{ "number": 33324, "repo": "bitcoin/bitcoin", "url": "https://github.com/bitcoin/bitcoin/pull/33324", "title": "blocks: add resumable reobfuscation for existing block files", "author": "l0rinc", "author_association": "MEMBER", "created_at": "2025-09-05T23:51:29Z", "updated_at": "2026-09-17T09:34:31Z", "age_days": 376, "draft": false, "labels": [ "Needs rebase" ], "milestone": null, "base": "master", "head_sha": "45ee237c5f9e154e597c3f36c1fe996cc8962572", "head_ref": "l0rinc/reobfuscate-blocks", "head_repo": "l0rinc/bitcoin", "head_history": [ { "t": "2025-09-06T00:29:25Z", "sha": "bb5037256a67a3afeb4b68fa3abf8000df5db769" }, { "t": "2025-09-07T04:59:28Z", "sha": "7df02ea3c79aa34587f87479f712a45ec32de24d" }, { "t": "2025-09-08T04:44:37Z", "sha": "2933b17d17399fe6092f8294a97b6292f93f1384" }, { "t": "2025-09-08T04:47:19Z", "sha": "d3962f65fae0dd5452b46e187569000a5088f432" }, { "t": "2025-09-10T03:17:32Z", "sha": "aa587f3740e39214d99da3fc44ce2878a0f4e32d" }, { "t": "2025-09-12T03:55:27Z", "sha": "ffb022179936f958678d1c38e60b4b41ff493dcd" }, { "t": "2025-09-20T06:58:21Z", "sha": "9b8bf69cef802d15abd745929ff73f531312a85f" }, { "t": "2025-09-20T20:09:22Z", "sha": "94ae42646b8358d5a8c2bcc209323554d5f5bb51" }, { "t": "2025-09-22T03:31:00Z", "sha": "af1d2ec33364980ed309591b041c1c079e39b07a" }, { "t": "2025-09-26T03:29:32Z", "sha": "3afb11fa4882f9be5c6a77cd1076dac3bfa772a0" }, { "t": "2025-10-03T00:59:37Z", "sha": "86c5afc2948511c0f4db3f7ba4f27f2d69b6ed78" }, { "t": "2025-12-10T22:21:30Z", "sha": "d1f2cfc817af6fcf8f284a57b744b6549d98fa58" }, { "t": "2026-01-25T17:53:32Z", "sha": "bcabb145e4d18059692a69bfa7bd7789542e3432" }, { "t": "2026-01-25T19:06:54Z", "sha": "e38dffd9dd30a3eaaa48aeaadd9613a82d026c57" }, { "t": "2026-01-25T19:07:44Z", "sha": "8492e024264ee388d84add1c86d024bd21ef0177" }, { "t": "2026-01-25T19:25:19Z", "sha": "6edb138730e877caff0c72b365887f8c2480385a" }, { "t": "2026-01-25T19:54:13Z", "sha": "dc7e43ef1b86057f2c1d897577d437a18dcc5d89" }, { "t": "2026-02-08T06:58:23Z", "sha": "f9bbef505cb91885ac748373bcfe378a2f1a6d6e" }, { "t": "2026-06-08T14:06:07Z", "sha": "6ab1eb624b238a55b82a2b6025b88abf424975b2" }, { "t": "2026-06-18T14:44:43Z", "sha": "afd59bd9159dad7ebaeee1ee2af1edba041761db" }, { "t": "2026-06-18T15:59:14Z", "sha": "181753328a8e27c7fac5300a44885843c3b6855f" }, { "t": "2026-07-12T18:57:10Z", "sha": "1f1d92a6696d5741352689bdb8273fc6ea83a0fa" }, { "t": "2026-07-22T22:26:07Z", "sha": "22790069f60fcb629de1577f6ea4e558883217ed" }, { "t": "2026-07-25T07:29:41Z", "sha": "d2bd095c19716a37c9aeefe99e6997882a0dec96" }, { "t": "2026-08-15T19:03:52Z", "sha": "9ff2767a99cd4fc4a7932913612615dd0061e1a8" }, { "t": "2026-08-15T19:45:17Z", "sha": "8c44cad5aa0cc2c9eabc8ffc475816a8118a3ddc" }, { "t": "2026-09-05T00:16:28Z", "sha": "53f86e980770db17c6e01bf38dc7115d0a349b92" }, { "t": "2026-09-05T01:02:34Z", "sha": "45ee237c5f9e154e597c3f36c1fe996cc8962572" } ], "additions": 451, "deletions": 43, "changed_files": 15, "commit_count": 9, "size_bucket": "L", "mergeable_state": "dirty", "bot": { "drahtbot": { "present": true, "reviews": { "concept_ack": [ { "login": "stickies-v", "url": "https://github.com/bitcoin/bitcoin/pull/33324#issuecomment-3268176314" } ], "stale_ack": [ { "login": "sedited", "url": "https://github.com/bitcoin/bitcoin/pull/33324#pullrequestreview-4527831296" } ] }, "conflicts": [ { "number": 36167, "title": "[RFC] Enable `-Wunused`", "author": "fanquake" }, { "number": 35948, "title": "init: correct first-run disk space estimate", "author": "l0rinc" }, { "number": 35139, "title": "test: Add thread-safe fast-failing test macros", "author": "maflcko" }, { "number": 34132, "title": "coins, dbwrapper: remove error catcher, make point-read failures fatal", "author": "l0rinc" }, { "number": 33637, "title": "refactor: optimize block index comparisons (1.4-6.8x faster)", "author": "l0rinc" }, { "number": 33593, "title": "guix: Use UCRT runtime for Windows release binaries", "author": "hebasto" }, { "number": 29700, "title": "kernel, refactor: return error status on all fatal errors", "author": "ryanofsky" } ] } }, "acks_parsed": { "sedited": { "kind": "ack", "hash": "181753328a8e27c7fac5300a44885843c3b6855f", "t": "2026-06-18T19:10:21Z", "stale": true }, "stickies-v": { "kind": "concept_ack", "hash": null, "t": "2025-09-08T22:06:31Z", "stale": false } }, "acks_tally": { "ack": 0, "stale_ack": 1, "concept_ack": 1, "approach_ack": 0, "nack": 0, "concept_nack": 0, "approach_nack": 0 }, "reviews": { "approved": 1, "changes_requested": 0, "distinct_reviewers": [ "ajtowns", "luke-jr", "sedited", "stickies-v" ] }, "signals": { "needs_rebase": true, "ci_failed": false, "mergeable_state": "dirty", "last_author_activity": "2026-09-05T01:05:10Z", "last_reviewer_activity": "2026-07-23T09:38:11Z", "last_reviewer": "sedited", "author_silent_days": 12, "waiting_on_author_days": 0, "days_since_update": 0 }, "refs": { "mentioned": [ 32451 ], "depends_on": [], "fixes": [], "linked_issues": [], "references": [ { "number": 32451, "type": "pull", "state": "closed", "merged": false, "merged_at": null, "title": "contrib: add xor-blocks tool to obfuscate blocks directory" } ], "conflicts": [ 36167, 35948, 35139, 34132, 33637, 33593, 29700 ] }, "stack": { "shares_commits_with": [], "based_on": [], "base_for": [] }, "review_paths": [ "src/init.cpp", "src/node/blockstorage.cpp" ], "body": "**Problem:** Nodes that synced before v28 may store block and undo files effectively unobfuscated with a zero XOR key. There is no built-in way to obfuscate those existing files, rotate their key, or deliberately replace it with a zero key without resyncing.\n\n**Fix:** Add `-reobfuscate-blocks[=]` to rewrite existing `blk*.dat` and `rev*.dat` files before chainstate loading. With no value or `=1`, the option generates a random 8-byte key. A 16-character hexadecimal value selects an exact key, including all zeros to remove obfuscation, while malformed values fail during startup. If the selected key is already active, block and undo files are left unchanged.\n\nFiles are rewritten in fixed-size chunks, staged with a `.reobfuscated` suffix, committed before the originals are removed, and renamed to their original names after all rewrites finish. The target key is staged as `xor.dat.reobfuscated`, which is also used to resume an interrupted migration. File modification times are preserved, and progress is reported in the log and through kernel notifications marked as resumable.\n\n**Benchmark:** The migration is single-threaded and processes one file at a time.\n\n| CPU | Storage | Block count | Size | Files | Time (min) | Blocks/min |\n|---|---|---:|---:|---:|---:|---:|\n| Apple M4 Max laptop | SSD | ~955k | ~854 GB | 11,232 | 10.18 | 93,804 |\n| Intel Core i9 | SSD | ~909k | ~725 GB | 10,238 | 23.1 | 39,351 |\n| Raspberry Pi 5 | SSD | ~914k | ~728 GB | 10,276 | 72.78 | 12,558 |\n| Intel Core i7 | HDD | ~909k | ~720 GB | 10,156 | 208.7 | 4,356 |\n| Raspberry Pi 4B | HDD | ~915k | ~730 GB | 10,304 | 1467 | 624 |\n\n**Similar work:**\n\n- [contrib: add xor-blocks tool to obfuscate blocks directory #32451](https://github.com/bitcoin/bitcoin/pull/32451)\n- [andrewtoth/blocks-xor](https://github.com/andrewtoth/blocks-xor)", "commits": [ { "sha": "79d331c2ff16adda19ffcca0fa61334275a0a306", "date": "2026-09-05T00:16:14Z", "message": "index: remove always-false obfuscation parameter\n\nAll `BaseIndex::DB` callers leave database obfuscation disabled, and the only explicit argument is `false`.\nRemove the parameter and rely on the `DBParams` default." }, { "sha": "5fc4699a00b295584ac71fe2302ed2d8ad466eb4", "date": "2026-09-05T00:16:15Z", "message": "refactor: add `Obfuscation` key helpers\n\nName the fixed-size key type and add helpers for parsing keys and deriving the XOR delta between keys.\nThis keeps the block reobfuscation commit focused on file migration." }, { "sha": "ec70a7a95eff8f41e226b64e8774de3af6666b6b", "date": "2026-09-05T00:16:15Z", "message": "node: extract XOR key file helpers\n\nCentralize `xor.dat` reading and creation so block reobfuscation can reuse the serialization and close-error handling." }, { "sha": "08e1f794d38624a3995f85fd0c82a452508fabd7", "date": "2026-09-05T00:16:15Z", "message": "node: extract block and undo file discovery\n\nExtract `blk` and `rev` discovery from `CleanupBlockRevFiles` so block reobfuscation can share a directory scan that handles gaps left by pruning." }, { "sha": "92526d48fa0fc5d9a1b0b701eaedc0b2afb63160", "date": "2026-09-05T00:16:15Z", "message": "node: log when block file XOR is inactive\n\nSuggest `-reobfuscate-blocks` when the key is zero and block-file XOR is enabled.\nSuppress the suggestion for `-blocksxor=0`, which explicitly leaves the files unobfuscated.\n\nCo-authored-by: sedited " }, { "sha": "022ef28eb3ce7fe9fec13071a1d2199c506bbc2c", "date": "2026-09-05T01:00:38Z", "message": "blocks: reobfuscate block and undo files\n\nRewrite existing block and undo files under a requested or random XOR key.\nStage each replacement and the target key before removing active files so interrupted migrations can resume safely.\nSkip file rewrites when the target key is already active.\n\nCo-authored-by: Anthony Towns " }, { "sha": "f7e9260b348db159fa25f73d96d695a819287d2d", "date": "2026-09-05T01:00:38Z", "message": "init: add block file reobfuscation option\n\nAdd `-reobfuscate-blocks` and run the requested or pending migration before chainstate loading.\nAccept a random or exact key, reject malformed values, and reject migration with `-blocksxor=0`.\n\nCo-authored-by: sedited " }, { "sha": "73d0588d395e0c42120c31366999b83b9190bfb9", "date": "2026-09-05T01:00:38Z", "message": "test: cover block file reobfuscation\n\nCover random and exact keys, same-key reruns, interrupted resume, invalid values, staged-key mismatches, and `-blocksxor=0`." }, { "sha": "45ee237c5f9e154e597c3f36c1fe996cc8962572", "date": "2026-09-05T01:00:38Z", "message": "gui: report block reobfuscation progress\n\nReport block reobfuscation through `kernel::Notifications` so GUI users can follow the long-running startup operation.\nSet `resume_possible` because a restart continues from `xor.dat.reobfuscated`.\n\nCo-authored-by: Ryan Ofsky \nCo-authored-by: sedited " } ], "timeline": [ { "t": "2025-09-06T00:29:25Z", "kind": "force_push", "who": "l0rinc", "commit": "bb5037256a67a3afeb4b68fa3abf8000df5db769" }, { "t": "2025-09-07T04:59:28Z", "kind": "force_push", "who": "l0rinc", "commit": "7df02ea3c79aa34587f87479f712a45ec32de24d" }, { "t": "2025-09-08T04:44:37Z", "kind": "force_push", "who": "l0rinc", "commit": "2933b17d17399fe6092f8294a97b6292f93f1384" }, { "t": "2025-09-08T04:47:19Z", "kind": "force_push", "who": "l0rinc", "commit": "d3962f65fae0dd5452b46e187569000a5088f432" }, { "t": "2025-09-08T08:21:29Z", "kind": "comment", "who": "sedited", "assoc": "MEMBER", "text": "Concept ACK" }, { "t": "2025-09-08T22:06:31Z", "kind": "comment", "who": "stickies-v", "assoc": "MEMBER", "text": "Concept ACK, this seems like useful functionality to expose.\n\n[quoted text omitted]\nI don't like using startup options for one-time operations (I feel the same about e.g. `-reindex`). Without having thought it through too much yet, maybe we can bundle this e.g. as part of `bitcoin-util` or a separate `bitcoin-xor-blocks` utility?\n\n[quoted text omitted]\nWith this PR, IIUC we'd have `-blocksxor`, `reobfuscate-blocks`, and the existence of the `xor.dat` file that all have some redundancy and thus potential for conflict (e,g. `blocksxor=0`, `reobfuscate-blocks=1`, and a non-zero `xor.dat` file). Reducing that complexity seems like it would be useful." }, { "t": "2025-09-10T03:17:32Z", "kind": "force_push", "who": "l0rinc", "commit": "aa587f3740e39214d99da3fc44ce2878a0f4e32d" }, { "t": "2025-09-11T05:10:14Z", "kind": "review_comment", "who": "ajtowns", "assoc": "MEMBER", "path": "src/init.cpp", "commit": "ffb022179936f958678d1c38e60b4b41ff493dcd", "in_reply_to": null, "text": "I was surprised when it hit 100% of the undo files then restarted on the block files and was much slower -- might be better to do the slow files first, or ideally to do block and undo files intermixed so you just have a single 0% to 100% run." }, { "t": "2025-09-11T05:22:07Z", "kind": "review_comment", "who": "ajtowns", "assoc": "MEMBER", "path": "src/init.cpp", "commit": "d1f2cfc817af6fcf8f284a57b744b6549d98fa58", "in_reply_to": null, "text": "Would be good to try to reset the timestamp of new_blocks to match that of old_blocks here.\n\n```c++\n // attempt to preserve timestamp\n fs::last_write_time(file + suffix, fs::last_write_time(file));\n```" }, { "t": "2025-09-11T05:23:45Z", "kind": "review_comment", "who": "ajtowns", "assoc": "MEMBER", "path": "src/init.cpp", "commit": "ffb022179936f958678d1c38e60b4b41ff493dcd", "in_reply_to": null, "text": "Rather than reading the entire blockfile into memory at once, consider chunking it:\n\n```c++\n size_t left = fs::file_size(file);\n while (left > 0) {\n size_t chunk = std::min(left, 2 * MAX_BLOCK_SERIALIZED_SIZE);\n buf.resize(chunk);\n old_blocks.read(buf);\n new_blocks.write_buffer(buf);\n left -= chunk;\n }\n```" }, { "t": "2025-09-11T05:43:17Z", "kind": "review_comment", "who": "ajtowns", "assoc": "MEMBER", "path": "src/init.cpp", "commit": "ffb022179936f958678d1c38e60b4b41ff493dcd", "in_reply_to": null, "text": "This doesn't seem like it would work correctly with pruning, when blk0000.dat has been deleted?" }, { "t": "2025-09-11T05:46:09Z", "kind": "review", "who": "ajtowns", "assoc": "MEMBER", "state": "COMMENTED", "commit": "aa587f3740e39214d99da3fc44ce2878a0f4e32d", "text": "Having it be a startup option like -reindex seems fine to me." }, { "t": "2025-09-11T05:49:50Z", "kind": "review_comment", "who": "l0rinc", "assoc": "MEMBER", "path": "src/init.cpp", "commit": "ffb022179936f958678d1c38e60b4b41ff493dcd", "in_reply_to": 2338622414, "text": "We could do that with the recently introduced buffered readers - but that's considerably slower.\nIs it a problem to read all of it in memory when we don't even have `dbcache` yet? The total memory usage is just 160 MB during migration, we should be fine until 1 GB at least, right?" }, { "t": "2025-09-11T06:20:54Z", "kind": "review_comment", "who": "ajtowns", "assoc": "MEMBER", "path": "src/init.cpp", "commit": "ffb022179936f958678d1c38e60b4b41ff493dcd", "in_reply_to": 2338622414, "text": "I don't think buffered readers is the right thing (that's for when you want to process small amounts of data while still reading it from the file in large chunks), and trying the above didn't seem particularly slow to me.\n\nI guess it could be simplified a bit to:\n\n```c++\nbuf.resize(2 * MAX_BLOCK_SERIALIZED_SIZE);\nwhile (true) {\n size_t size = old_blocks.detail_fread(buf);\n if (size == 0) break;\n new_blocks.write_buffer(std::span(buf, 0, size));\n}\n```" }, { "t": "2025-09-12T03:55:27Z", "kind": "force_push", "who": "l0rinc", "commit": "ffb022179936f958678d1c38e60b4b41ff493dcd" }, { "t": "2025-09-12T03:56:21Z", "kind": "review_comment", "who": "l0rinc", "assoc": "MEMBER", "path": "src/init.cpp", "commit": "ffb022179936f958678d1c38e60b4b41ff493dcd", "in_reply_to": 2338593448, "text": "I had that version before, but didn't like that the small and big files made the percentages look unevenly spaced. But I have reverted that version and shuffled the files, this should make the progress feel more uniform - thank you for the observation!" }, { "t": "2025-09-12T03:56:45Z", "kind": "review_comment", "who": "l0rinc", "assoc": "MEMBER", "path": "src/init.cpp", "commit": "d1f2cfc817af6fcf8f284a57b744b6549d98fa58", "in_reply_to": 2338618798, "text": "Done, thanks!" }, { "t": "2025-09-12T03:58:26Z", "kind": "review_comment", "who": "l0rinc", "assoc": "MEMBER", "path": "src/init.cpp", "commit": "ffb022179936f958678d1c38e60b4b41ff493dcd", "in_reply_to": 2338622414, "text": "What's problem would chunking solve in your opinion? I don't mind doing it, but the current version is slightly simpler and slightly faster, so I need at least *some* justification for giving up both :)" }, { "t": "2025-09-12T03:58:45Z", "kind": "review_comment", "who": "l0rinc", "assoc": "MEMBER", "path": "src/init.cpp", "commit": "ffb022179936f958678d1c38e60b4b41ff493dcd", "in_reply_to": 2338660825, "text": "Good call, changed it back to regex matching" }, { "t": "2025-09-12T12:36:26Z", "kind": "review_comment", "who": "ajtowns", "assoc": "MEMBER", "path": "src/init.cpp", "commit": "ffb022179936f958678d1c38e60b4b41ff493dcd", "in_reply_to": 2338622414, "text": "Loading a large file entirely into memory when it's not necessary is just bad practice. What if we changed to .blk files of 1GB each? What if we're running on a node that's memory constrained and configured dbcache down to 4MB?\n\nIf we're worried about speed, then doing it in parallel helps on my system since obfuscation ends up CPU bound when single-threaded -- with the current code, it takes 238s (4min); with 8 threads it's 65s; with 16 threads it's 47s. Using 16MB chunks (BLOCKFILE_CHUNK_SIZE), 8 threads is also ~128MB of memory, but a user on a severely memory constrained system could reduce the thread count if they wanted. Here's roughly what I'm thinking: https://github.com/ajtowns/bitcoin/commits/202509-reobfus/" }, { "t": "2025-09-12T13:48:56Z", "kind": "review_comment", "who": "l0rinc", "assoc": "MEMBER", "path": "src/init.cpp", "commit": "ffb022179936f958678d1c38e60b4b41ff493dcd", "in_reply_to": 2338622414, "text": "[quoted text omitted]\n\nThis is run on request, before anything else loads, it's not *that* large, only 160 MB memory is needed.\n\nFor reference, applying the mentioned `dbcache=4` (which isn't used here yet) still makes the node use > 1 GB memory:\n\nEdit: doing an actual massif memory measurement with `dbcache=4` and `-blocksonly` reveals that the actual memory usage is lower than that (but still higher than the 160MB needed for a single blockfile):\n```\nCommand: ./build/bin/bitcoind -datadir=/mnt/my_storage/BitcoinData -stopatheight=200000 -dbcache=4 -blocksonly -printtoconsole=1\nMassif arguments: --time-unit=ms --massif-out-file=/mnt/my_storage/logs/massif-e66f04d0131b8c2db13ddd649e9eb20910eb6d1d-200000-4.out\nms_print arguments: /mnt/my_storage/logs/massif-e66f04d0131b8c2db13ddd649e9eb20910eb6d1d-200000-4.out\n--------------------------------------------------------------------------------\n\n MB\n383.1^#\n |#\n |#\n |#\n |#\n |#\n |# : : ::: : : :::: @\n |# :@: :::@::::::::::::::@:::@::::::::::::::::::@:::::::@::::::@:::::::\n |#::::@::: :@::: ::::::::::@:::@::::::::::::::::::@:::::::@::::::@:::::::\n |#:: :@::: :@::: ::::::::::@:::@::::::::::::::::::@:::::::@::::::@:::::::\n |#:: :@::: :@::: ::::::::::@:::@::::::::::::::::::@:::::::@::::::@:::::::\n |#:: :@::: :@::: ::::::::::@:::@::::::::::::::::::@:::::::@::::::@:::::::\n |#:: :@::: :@::: ::::::::::@:::@::::::::::::::::::@:::::::@::::::@:::::::\n |#:: :@::: :@::: ::::::::::@:::@::::::::::::::::::@:::::::@::::::@:::::::\n |#:: :@::: :@::: ::::::::::@:::@::::::::::::::::::@:::::::@::::::@:::::::\n |#:: :@::: :@::: ::::::::::@:::@::::::::::::::::::@:::::::@::::::@:::::::\n |#:: :@::: :@::: ::::::::::@:::@::::::::::::::::::@:::::::@::::::@:::::::\n |#:: :@::: :@::: ::::::::::@:::@::::::::::::::::::@:::::::@::::::@:::::::\n |#:: :@::: :@::: ::::::::::@:::@::::::::::::::::::@:::::::@::::::@:::::::\n |#:: :@::: :@::: ::::::::::@:::@::::::::::::::::::@:::::::@::::::@:::::::\n 0 +----------------------------------------------------------------------->h\n 0 2.506\n```\n\n[quoted text omitted]\nMultithreading is indeed a very good argument for chunking, thanks a lot for the patch, I'll apply it soon and add you as coauthor!" }, { "t": "2025-09-15T07:32:02Z", "kind": "comment", "who": "ajtowns", "assoc": "MEMBER", "text": "tidy wants emplace_back over push_back" }, { "t": "2025-09-18T01:29:22Z", "kind": "comment", "who": "luke-jr", "assoc": "CONTRIBUTOR", "text": "I agree a separate utility for this seems better - this requires very little of the existing codebase, in theory.\n\nAlso suggest making the files with the same names, but in a new directory, and then atomically rename the directory when complete, rather than every single file." }, { "t": "2025-09-18T01:49:52Z", "kind": "comment", "who": "l0rinc", "assoc": "MEMBER", "text": "[quoted text omitted]\n\nCan you quote what you're agreeing with specifically, not sure who suggested that.\nBesides, @andrewtoth already has a tool for that, it was mentioned in the PR description.\n\n[quoted text omitted]\nI will think about it, could make sense, but in that case unrelated files should also be copied over (maybe duplicated to be safe) - and listing the directory content wouldn't make the progress obvious. What's wrong with the current approach?" }, { "t": "2025-09-20T06:58:21Z", "kind": "force_push", "who": "l0rinc", "commit": "9b8bf69cef802d15abd745929ff73f531312a85f" }, { "t": "2025-09-20T07:02:20Z", "kind": "review_comment", "who": "l0rinc", "assoc": "MEMBER", "path": "src/init.cpp", "commit": "ffb022179936f958678d1c38e60b4b41ff493dcd", "in_reply_to": 2338622414, "text": "Thanks again for the review, I have pushed a change to fix the CI and took a few suggestion from your branch (chunking, code simplifications), but kept the original file iteration with progress indicator for now.\nThe parallelization complicates the situation considerably, I will see if I can find a simpler way or if single-threaded execution is also acceptable.\nEdit: grouped the block and undo files for more uniform iteration instead of shuffling" }, { "t": "2025-09-20T20:09:22Z", "kind": "force_push", "who": "l0rinc", "commit": "94ae42646b8358d5a8c2bcc209323554d5f5bb51" }, { "t": "2025-09-22T03:31:00Z", "kind": "force_push", "who": "l0rinc", "commit": "af1d2ec33364980ed309591b041c1c079e39b07a" }, { "t": "2025-09-26T03:29:32Z", "kind": "force_push", "who": "l0rinc", "commit": "3afb11fa4882f9be5c6a77cd1076dac3bfa772a0" }, { "t": "2025-09-26T03:33:10Z", "kind": "review_comment", "who": "l0rinc", "assoc": "MEMBER", "path": "src/init.cpp", "commit": "ffb022179936f958678d1c38e60b4b41ff493dcd", "in_reply_to": 2338622414, "text": "I have pushed a new version (rebased, extended test), let me know what you think.\n\nI have implemented a very simple multithreaded version but I couldn't convince it to achieve *any* speedup whatsoever - I guess xor operations are a lot cheaper than disk reads/writes. The total CPU usage was at 20% even with 50 threads.\n\nI have pushed my threaded solution to https://github.com/l0rinc/bitcoin/pull/40/files#diff-b1e19192258d83199d8adaa5ac31f067af98f63554bfdd679bd8e8073815e69dR1361-R1379, but I kept the single-threaded version here." }, { "t": "2025-10-03T00:59:37Z", "kind": "force_push", "who": "l0rinc", "commit": "86c5afc2948511c0f4db3f7ba4f27f2d69b6ed78" }, { "t": "2025-10-03T01:01:44Z", "kind": "comment", "who": "l0rinc", "assoc": "MEMBER", "text": "Added kernel notifications (thanks @ryanofsky) and improved crash resistance at the very last step (final rename back to old names) - try it out with `./build/bin/bitcoin-qt -reobfuscate-blocks -stopatheight=1`." }, { "t": "2025-12-10T22:21:30Z", "kind": "force_push", "who": "l0rinc", "commit": "d1f2cfc817af6fcf8f284a57b744b6549d98fa58" }, { "t": "2026-01-25T17:53:32Z", "kind": "force_push", "who": "l0rinc", "commit": "bcabb145e4d18059692a69bfa7bd7789542e3432" }, { "t": "2026-01-25T19:06:54Z", "kind": "force_push", "who": "l0rinc", "commit": "e38dffd9dd30a3eaaa48aeaadd9613a82d026c57" }, { "t": "2026-01-25T19:07:44Z", "kind": "force_push", "who": "l0rinc", "commit": "8492e024264ee388d84add1c86d024bd21ef0177" }, { "t": "2026-01-25T19:25:19Z", "kind": "force_push", "who": "l0rinc", "commit": "6edb138730e877caff0c72b365887f8c2480385a" }, { "t": "2026-01-25T19:54:13Z", "kind": "force_push", "who": "l0rinc", "commit": "dc7e43ef1b86057f2c1d897577d437a18dcc5d89" }, { "t": "2026-02-08T06:58:23Z", "kind": "force_push", "who": "l0rinc", "commit": "f9bbef505cb91885ac748373bcfe378a2f1a6d6e" }, { "t": "2026-06-08T14:06:07Z", "kind": "force_push", "who": "l0rinc", "commit": "6ab1eb624b238a55b82a2b6025b88abf424975b2" }, { "t": "2026-06-11T15:27:30Z", "kind": "review_comment", "who": "sedited", "assoc": "MEMBER", "path": "src/node/blockstorage.cpp", "commit": "51a7b252e7d2faf483f2d596bca819a03b095052", "in_reply_to": null, "text": "Not sure if a warning is really warranted here. Feel like this pull request would have gotten more review over the past year if this were a pressing operation." }, { "t": "2026-06-11T21:15:43Z", "kind": "review_comment", "who": "sedited", "assoc": "MEMBER", "path": "src/init.cpp", "commit": "6ab1eb624b238a55b82a2b6025b88abf424975b2", "in_reply_to": null, "text": "This function is already very busy. This logic also doesn't really need much state. The notifications are available from the `node` object and the blocks dir could just be re-read from the args. How about moving it to right before the cache size calculations in `AppInitMain`. It could also be moved into its little function." }, { "t": "2026-06-11T21:25:12Z", "kind": "review_comment", "who": "sedited", "assoc": "MEMBER", "path": "src/init.cpp", "commit": "43eb6ec1b1421b7905b06d5441733edd51d64bd5", "in_reply_to": null, "text": "It feels a bit off that getting the length wrong is not fatal. Maybe we should put these in same statement and have separate branches for the \"0\" and empty case?" }, { "t": "2026-06-11T21:27:02Z", "kind": "review_comment", "who": "sedited", "assoc": "MEMBER", "path": "src/node/blockstorage.cpp", "commit": "1f1d92a6696d5741352689bdb8273fc6ea83a0fa", "in_reply_to": null, "text": "I don't quite understand what the purpose of this is. Where does the timestamp become relevant?" }, { "t": "2026-06-11T21:38:58Z", "kind": "review_comment", "who": "sedited", "assoc": "MEMBER", "path": "src/node/blockstorage.cpp", "commit": "6ab1eb624b238a55b82a2b6025b88abf424975b2", "in_reply_to": null, "text": "Could this be the first kernel notification where resume is possible?" }, { "t": "2026-06-11T21:50:36Z", "kind": "review", "who": "sedited", "assoc": "MEMBER", "state": "COMMENTED", "commit": "6ab1eb624b238a55b82a2b6025b88abf424975b2", "text": "The migration logic looks nice.\n\nI concur that this would be a good candidate for a separate c++ binary that also gets shipped in releases. However I'm not sure if this particular pull request is the practice case for ironing out how to launch such binaries from the GUI, how they would handle directory locks, and how they would communicate failure. The logic here is fairly straight forward. If somebody wants to take a stab at pioneering such a utility, that could easily be done later." }, { "t": "2026-06-18T13:32:43Z", "kind": "review_comment", "who": "l0rinc", "assoc": "MEMBER", "path": "src/node/blockstorage.cpp", "commit": "1f1d92a6696d5741352689bdb8273fc6ea83a0fa", "in_reply_to": 3399281100, "text": "Was explicitly requested to minimize unintended changes https://github.com/bitcoin/bitcoin/pull/33324#discussion_r2338618798" }, { "t": "2026-06-18T13:41:27Z", "kind": "review_comment", "who": "l0rinc", "assoc": "MEMBER", "path": "src/init.cpp", "commit": "43eb6ec1b1421b7905b06d5441733edd51d64bd5", "in_reply_to": 3399270951, "text": "Even I got confused trying to explain why I did it as such - thanks, split, covered with test, added you as coauthor." }, { "t": "2026-06-18T13:45:24Z", "kind": "review_comment", "who": "l0rinc", "assoc": "MEMBER", "path": "src/node/blockstorage.cpp", "commit": "6ab1eb624b238a55b82a2b6025b88abf424975b2", "in_reply_to": 3399344277, "text": "Good point, adjusted, added you as coauthor, thanks." }, { "t": "2026-06-18T13:45:39Z", "kind": "review_comment", "who": "l0rinc", "assoc": "MEMBER", "path": "src/init.cpp", "commit": "6ab1eb624b238a55b82a2b6025b88abf424975b2", "in_reply_to": 3399224377, "text": "Was hoping someone will suggest that :)" }, { "t": "2026-06-18T13:45:49Z", "kind": "review_comment", "who": "l0rinc", "assoc": "MEMBER", "path": "src/node/blockstorage.cpp", "commit": "51a7b252e7d2faf483f2d596bca819a03b095052", "in_reply_to": 3397152913, "text": "fair, done" }, { "t": "2026-06-18T14:44:43Z", "kind": "force_push", "who": "l0rinc", "commit": "afd59bd9159dad7ebaeee1ee2af1edba041761db" }, { "t": "2026-06-18T14:45:27Z", "kind": "review", "who": "l0rinc", "assoc": "MEMBER", "state": "COMMENTED", "commit": "afd59bd9159dad7ebaeee1ee2af1edba041761db", "text": "Rebased and folded in the latest review feedback. I also re-ran the reobfuscation check before pushing and updated the PR description with the refreshed numbers for M4.\n\nMain changes since the previous push:\n* Moved the startup reobfuscation handling out of `InitAndLoadChainstate` into `ReobfuscateBlocksIfRequested()`, called before cache sizing in `AppInitMain`.\n* Made `-reobfuscate-blocks` parsing stricter: only `0`, empty/`1`, or exact 16-character hex keys are accepted; wrong-length keys now fail early and are covered by the functional test.\n* Downgraded the unobfuscated-blocks startup notice from warning to info.\n* Marked GUI reobfuscation progress as resumable and reused a single translated progress title.\n* Addressed the tidy/test nits (`emplace_back`, named progress args, `assert_not_equal`).\n\n[quoted text omitted]\n@andrewtoth already has a separate tool for this, see https://github.com/bitcoin/bitcoin/pull/33324#issuecomment-3305103297\n\n@ajtowns, @stickies-v, @luke-jr, @ryanofsky care to take another look?" }, { "t": "2026-06-18T15:59:14Z", "kind": "force_push", "who": "l0rinc", "commit": "181753328a8e27c7fac5300a44885843c3b6855f" }, { "t": "2026-06-18T19:10:21Z", "kind": "review", "who": "sedited", "assoc": "MEMBER", "state": "APPROVED", "commit": "181753328a8e27c7fac5300a44885843c3b6855f", "text": "ACK 181753328a8e27c7fac5300a44885843c3b6855f\n\n[quoted text omitted]\nI saw and I looked at the pull request at the time, which is why I specifically mentioned a c++ tool. I think we should start using the `bitcoin` command more, and this would be a good candidate for it. As much as I like rust, I'm against adding released utilities written in yet another different language. With the code already here, it would probably be fairly straight forward to implement." }, { "t": "2026-07-12T18:57:10Z", "kind": "force_push", "who": "l0rinc", "commit": "1f1d92a6696d5741352689bdb8273fc6ea83a0fa" }, { "t": "2026-07-22T22:26:07Z", "kind": "force_push", "who": "l0rinc", "commit": "22790069f60fcb629de1577f6ea4e558883217ed" }, { "t": "2026-07-22T22:32:48Z", "kind": "comment", "who": "l0rinc", "assoc": "MEMBER", "text": "Rebased and pushed again to fix a minor crash-safety bug and a `-blocksxor=0` handling inconsistency - while simplifying the implementation a bit." }, { "t": "2026-07-23T09:38:11Z", "kind": "review_comment", "who": "sedited", "assoc": "MEMBER", "path": "src/node/blockstorage.cpp", "commit": "22790069f60fcb629de1577f6ea4e558883217ed", "in_reply_to": null, "text": "It does feel a bit weird to me to assert on what is not an invariant of the program, but rather an exceptional condition. Why not just throw or return here (and for the rename assertion). I think it would be preferable to just use the throwing fs::remove and fs::rename." }, { "t": "2026-07-25T07:29:41Z", "kind": "force_push", "who": "l0rinc", "commit": "d2bd095c19716a37c9aeefe99e6997882a0dec96" }, { "t": "2026-07-25T07:37:39Z", "kind": "review_comment", "who": "l0rinc", "assoc": "MEMBER", "path": "src/node/blockstorage.cpp", "commit": "22790069f60fcb629de1577f6ea4e558883217ed", "in_reply_to": 3637087440, "text": "Thanks, rebased and pushed: filesystem failures now propagate instead of aborting on assertions.\nI also did some cleanup, split the main feature commit into migration, startup wiring, and functional coverage, added `Obfuscation::Key` and made same-key reruns skip rewriting block and undo files.\nReran the benchmark on my M4 Max - finished 955k blocks in ~10 minutes." }, { "t": "2026-08-15T19:03:52Z", "kind": "force_push", "who": "l0rinc", "commit": "9ff2767a99cd4fc4a7932913612615dd0061e1a8" }, { "t": "2026-08-15T19:45:17Z", "kind": "force_push", "who": "l0rinc", "commit": "8c44cad5aa0cc2c9eabc8ffc475816a8118a3ddc" }, { "t": "2026-09-05T00:16:28Z", "kind": "force_push", "who": "l0rinc", "commit": "53f86e980770db17c6e01bf38dc7115d0a349b92" }, { "t": "2026-09-05T01:02:34Z", "kind": "force_push", "who": "l0rinc", "commit": "45ee237c5f9e154e597c3f36c1fe996cc8962572" }, { "t": "2026-09-05T01:05:10Z", "kind": "comment", "who": "l0rinc", "assoc": "MEMBER", "text": "Rebased separately and pushed some I/O error-handling hardening." } ], "labels_log": [ { "t": "2025-09-09T23:39:56Z", "action": "labeled", "label": "Needs rebase", "who": "DrahtBot" }, { "t": "2025-09-10T04:10:16Z", "action": "unlabeled", "label": "Needs rebase", "who": "DrahtBot" }, { "t": "2026-01-25T19:07:34Z", "action": "labeled", "label": "CI failed", "who": "DrahtBot" }, { "t": "2026-01-25T21:30:43Z", "action": "unlabeled", "label": "CI failed", "who": "DrahtBot" }, { "t": "2026-02-08T00:04:47Z", "action": "labeled", "label": "Needs rebase", "who": "DrahtBot" }, { "t": "2026-02-08T08:38:52Z", "action": "unlabeled", "label": "Needs rebase", "who": "DrahtBot" }, { "t": "2026-05-26T13:03:38Z", "action": "labeled", "label": "Needs rebase", "who": "DrahtBot" }, { "t": "2026-06-08T15:17:24Z", "action": "unlabeled", "label": "Needs rebase", "who": "DrahtBot" }, { "t": "2026-06-18T15:59:43Z", "action": "labeled", "label": "CI failed", "who": "DrahtBot" }, { "t": "2026-06-18T17:15:41Z", "action": "unlabeled", "label": "CI failed", "who": "DrahtBot" }, { "t": "2026-07-12T11:41:17Z", "action": "labeled", "label": "Needs rebase", "who": "DrahtBot" }, { "t": "2026-07-12T20:31:30Z", "action": "unlabeled", "label": "Needs rebase", "who": "DrahtBot" }, { "t": "2026-08-15T15:42:27Z", "action": "labeled", "label": "Needs rebase", "who": "DrahtBot" }, { "t": "2026-08-15T19:22:26Z", "action": "labeled", "label": "CI failed", "who": "DrahtBot" }, { "t": "2026-08-15T19:57:23Z", "action": "unlabeled", "label": "Needs rebase", "who": "DrahtBot" }, { "t": "2026-08-15T21:10:45Z", "action": "unlabeled", "label": "CI failed", "who": "DrahtBot" }, { "t": "2026-09-05T01:03:02Z", "action": "labeled", "label": "CI failed", "who": "DrahtBot" }, { "t": "2026-09-05T02:07:32Z", "action": "unlabeled", "label": "CI failed", "who": "DrahtBot" }, { "t": "2026-09-16T13:38:58Z", "action": "labeled", "label": "Needs rebase", "who": "DrahtBot" } ], "state_log": [ { "t": "2025-10-03T01:01:49Z", "kind": "ready_for_review", "who": "l0rinc" }, { "t": "2025-10-03T01:02:13Z", "kind": "renamed", "who": "l0rinc", "from": "RFC: blocks: add `-reobfuscate-blocks` arg to xor existing blk/rev on startup", "to": "blocks: add `-reobfuscate-blocks` arg to xor existing blk/rev on startup" }, { "t": "2025-12-11T12:54:49Z", "kind": "renamed", "who": "l0rinc", "from": "blocks: add `-reobfuscate-blocks` arg to xor existing blk/rev on startup", "to": "blocks: add `-reobfuscate-blocks` argument to enable (de)obfuscating existing blocks" }, { "t": "2026-07-22T19:31:48Z", "kind": "renamed", "who": "l0rinc", "from": "blocks: add `-reobfuscate-blocks` argument to enable (de)obfuscating existing blocks", "to": "blocks: add resumable reobfuscation for existing block files" } ], "text_chars": 16101, "text_tokens_estimate": 4025, "changed_paths": [ "src/index/base.cpp", "src/index/base.h", "src/index/txindex.cpp", "src/index/txospenderindex.cpp", "src/init.cpp", "src/node/blockstorage.cpp", "src/node/blockstorage.h", "src/streams.h", "src/test/fs_tests.cpp", "src/test/streams_tests.cpp", "src/util/fs_helpers.cpp", "src/util/fs_helpers.h", "src/util/obfuscation.h", "test/functional/feature_reobfuscation.py", "test/functional/test_runner.py" ], "files": [ { "path": "src/index/base.cpp", "add": 1, "del": 2 }, { "path": "src/index/base.h", "add": 1, "del": 1 }, { "path": "src/index/txindex.cpp", "add": 1, "del": 1 }, { "path": "src/index/txospenderindex.cpp", "add": 1, "del": 1 }, { "path": "src/init.cpp", "add": 34, "del": 0 }, { "path": "src/node/blockstorage.cpp", "add": 194, "del": 30 }, { "path": "src/node/blockstorage.h", "add": 12, "del": 0 }, { "path": "src/streams.h", "add": 1, "del": 1 }, { "path": "src/test/fs_tests.cpp", "add": 11, "del": 0 }, { "path": "src/test/streams_tests.cpp", "add": 27, "del": 1 }, { "path": "src/util/fs_helpers.cpp", "add": 7, "del": 5 }, { "path": "src/util/fs_helpers.h", "add": 3, "del": 1 }, { "path": "src/util/obfuscation.h", "add": 21, "del": 0 }, { "path": "test/functional/feature_reobfuscation.py", "add": 136, "del": 0 }, { "path": "test/functional/test_runner.py", "add": 1, "del": 0 } ], "test_lines": 176, "git": { "head": "45ee237c5f9e154e597c3f36c1fe996cc8962572", "head_matches_backup": true, "base": "4519933391dd23dbf1a4eceec6dd53d2e9e71cc3", "commits": [ { "sha": "79d331c2ff", "subject": "index: remove always-false obfuscation parameter", "files": 4, "add": 4, "del": 5 }, { "sha": "5fc4699a00", "subject": "refactor: add `Obfuscation` key helpers", "files": 3, "add": 49, "del": 2 }, { "sha": "ec70a7a95e", "subject": "node: extract XOR key file helpers", "files": 1, "add": 27, "del": 15 }, { "sha": "08e1f794d3", "subject": "node: extract block and undo file discovery", "files": 1, "add": 25, "del": 11 }, { "sha": "92526d48fa", "subject": "node: log when block file XOR is inactive", "files": 1, "add": 9, "del": 2 }, { "sha": "022ef28eb3", "subject": "blocks: reobfuscate block and undo files", "files": 6, "add": 160, "del": 10 }, { "sha": "f7e9260b34", "subject": "init: add block file reobfuscation option", "files": 1, "add": 34, "del": 0 }, { "sha": "73d0588d39", "subject": "test: cover block file reobfuscation", "files": 2, "add": 137, "del": 0 }, { "sha": "45ee237c5f", "subject": "gui: report block reobfuscation progress", "files": 3, "add": 9, "del": 1 } ], "patch_truncated": false }, "input_hash": "6161573b6aab69bd", "extracted_at": "2026-09-17T16:15:31+00:00" }