{
 "number": 36603,
 "repo": "bitcoin/bitcoin",
 "url": "https://github.com/bitcoin/bitcoin/pull/36603",
 "title": "Improve loadtxoutset performance",
 "author": "ajtowns",
 "author_association": "MEMBER",
 "created_at": "2026-10-22T10:25:07Z",
 "updated_at": "2026-10-30T14:52:01Z",
 "description_hash": "a97afd5be37ff047",
 "age_days": 10,
 "draft": false,
 "labels": [],
 "milestone": null,
 "base": "master",
 "head_sha": "fbda3acde3914cf9ce14a58070467c8989429881",
 "head_ref": "202610-loadtxoutset-speedy",
 "head_repo": "ajtowns/bitcoin",
 "head_history": [
  {
   "t": "2026-10-23T05:47:53Z",
   "sha": "420ca68611179905994fd6efc9ceb847b4d69679"
  },
  {
   "t": "2026-10-23T05:52:34Z",
   "sha": "51d7d9e66d1a0aac096ae0b4e60bf2f90334307b"
  },
  {
   "t": "2026-10-23T06:04:31Z",
   "sha": "32575f09eb25b881cc7d4e475a5494aa3ced1a66"
  },
  {
   "t": "2026-10-23T11:19:06Z",
   "sha": "e81eeb8ff565361d55ab2dafe0ac909b375e4344"
  },
  {
   "t": "2026-10-25T04:58:22Z",
   "sha": "4840412aa5f1dd18e65c65c836492e7a3855d16c"
  },
  {
   "t": "2026-10-25T05:07:00Z",
   "sha": "92c6b60734813ca580fe77f7b30e2005cc6df56c"
  },
  {
   "t": "2026-10-28T02:47:03Z",
   "sha": "fbda3acde3914cf9ce14a58070467c8989429881"
  }
 ],
 "additions": 1323,
 "deletions": 472,
 "changed_files": 27,
 "commit_count": 12,
 "size_bucket": "XL",
 "mergeable_state": "clean",
 "bot": {
  "drahtbot": {
   "present": true,
   "reviews": {},
   "conflicts": []
  }
 },
 "acks_parsed": {},
 "acks_tally": {
  "ack": 0,
  "stale_ack": 0,
  "concept_ack": 0,
  "approach_ack": 0,
  "nack": 0,
  "concept_nack": 0,
  "approach_nack": 0
 },
 "reviews": {
  "approved": 0,
  "changes_requested": 0,
  "distinct_reviewers": [
   "l0rinc"
  ]
 },
 "signals": {
  "needs_rebase": false,
  "ci_failed": false,
  "mergeable_state": "clean",
  "last_author_activity": "2026-10-28T02:47:03Z",
  "last_reviewer_activity": "2026-10-23T07:03:28Z",
  "last_reviewer": "l0rinc",
  "author_silent_days": 4,
  "waiting_on_author_days": 0,
  "days_since_update": 2
 },
 "refs": {
  "mentioned": [
   35054
  ],
  "depends_on": [],
  "fixes": [],
  "linked_issues": [],
  "references": [
   {
    "number": 35054,
    "type": "pull",
    "state": "open",
    "merged": false,
    "merged_at": null,
    "title": "p2p: UTXO set sharing"
   }
  ],
  "conflicts": []
 },
 "stack": {
  "shares_commits_with": [],
  "based_on": [],
  "base_for": []
 },
 "review_paths": [],
 "body": "Significantly improves loadtxoutset performance via a number of changes:\n\n * parallel threads for decoding, hashing, serializing and writing to leveldb\n * no fsync() gating writes\n * no redundant loading into CoinsView\n\nAlso adds a subcommand to bitcoin-util for getting a snapshot's hash_serialized_3, muhash, coin count and total amount values without importing to leveldb first.\n\nOn my systems, performance ends up bottlenecked by leveldb CPU usage.",
 "commits": [
  {
   "sha": "52583fba0d5a9a69a3b5a9d71a0a50f8f7a5aff8",
   "date": "2026-10-27T04:06:34Z",
   "message": "refactor: Move SnapshotMetadata out of node/\n\nThe snapshot file format is not node state: reading a snapshot needs no\nchainstate, so move it out. This prepares the way for adding snapshot\nparsing to bitcoin-util in a future commit.\n\nMostly move-only. The commit can be reviewed with the\n--color-moved=dimmed-zebra option for an easier review."
  },
  {
   "sha": "19fa00eb2d0152fe78e5c56b09620ca46814d0af",
   "date": "2026-10-27T04:06:34Z",
   "message": "refactor: Check the snapshot network in the caller\n\nSnapshotMetadata took the expected network in its constructor and compared\nit while parsing, throwing on mismatch. Split that check out as a separate\n`CheckNetwork` method, so that callers have the option to read snapshots\nfor arbitrary networks.\n\nAlso replace the per-object `const std::set` for checking supported\nversions with an inline function."
  },
  {
   "sha": "21bef3424851c04c63ddbd1a2b792a91e867f862",
   "date": "2026-10-28T02:45:42Z",
   "message": "util: Add DirectoryFilesCommit for syncing the contents of a directory"
  },
  {
   "sha": "f7de839fc951ba86cfec78bf67ab27e5e0477025",
   "date": "2026-10-28T02:45:44Z",
   "message": "dbwrapper: Add nosync option for quickly populating a db\n\nNormally, we want updates to the database to be fsynced to disk so that\nthe database is in a consistent state even after a crash. This adds an\noption to drop that guarantee, in favour of faster bulk writes, relying\non the caller to make other arrangements to ensure the resulting database\nis flushed to disk."
  },
  {
   "sha": "a73f679f3372945fa891c1b4ba6b037476c8b88c",
   "date": "2026-10-28T02:45:44Z",
   "message": "txdb: Add RawCoinDataDBBatch for directly populating a chainstate\n\nThis populates a chainstate leveldb (as a CDBWrapper) directly from a\nbatch (generated from Coin objects). This avoids the overhead of going\nthrough a CoinsView map, and allows splitting the batch creation and\nthe batch write/flush across different threads."
  },
  {
   "sha": "bbe85cc3dd86acdb21e57fb0c513297df1833b8d",
   "date": "2026-10-28T02:45:44Z",
   "message": "util: Add BlockingQueue\n\nMulti-producer, multi-consumer thread-safe queue with maximum capacity\nfor handing out jobs to threads."
  },
  {
   "sha": "8810862d1de6349dd7feab0e9a294887fe0fc8ee",
   "date": "2026-10-28T02:45:44Z",
   "message": "utxo_snapshot: More efficient snapshot imports\n\nAllows importing snapshots directly into leveldb (dbwrapper), without\npassing through Chainstate or CoinsView first. Supports multithreaded\nprocessing, with one thread reading, one thread hashing, one thread\nserializing for the database, and one thread populating the database.\n\nAdds an additional check that the coins in the snapshot are sorted in\ntxid order.\n\nNote that because v2 dump follow the database key order (which encodes\noutput indices as VARINTs), but the UTXO commitment requires outputs\nto be ordered numerically by output index, a sorting step is required\n(eg VARINT(16512) sorts less than VARINT(16511). We bound the total\nserialized transaction outputs by the consensus block weight limit to\nbound the amount of date being sorted in case of corrupt snapshot files.\n\nNothing calls it yet, so behaviour is unchanged.\n\nCo-Authored-By: Sjors Provoost <sjors@sprovoost.nl>"
  },
  {
   "sha": "49aa42ccaaf4e9107e76e7a938707b7adff0b9c7",
   "date": "2026-10-28T02:45:44Z",
   "message": "validation: Make ActivateSnapshot faster\n\nActivateSnapshot and, therefore, the loadtxoutset RPC now use the new\nutxo_snapshot.cpp pipeline.\n\nThe dump is no longer staged in a CCoinsViewCache and then hashed and\nflushed out of it, so `-dbcache` worth of cached coins during a mainnet\nload are gone along with the flush cycles. In addition, the hashing\nof the dump takes place in parallel with the import, rather than after\nit completes.\n\nThe old loader is left in place until the next commit removes it;\nnothing calls it any more."
  },
  {
   "sha": "ce01e3c30be9a0217a98283ee2ed208d21e2ac98",
   "date": "2026-10-28T02:45:44Z",
   "message": "validation/utxo_snapshot: Remove the old snapshot loader\n\nAfter the previous two commits, this code is no longer used, so remove it.\n\nThis also removes the circular dependency between node/utxo_snapshot\nand validation."
  },
  {
   "sha": "03fec5e4018af9c3f7cbbd30697cfc21a12f2dd7",
   "date": "2026-10-28T02:45:44Z",
   "message": "coins: Drop EmplaceCoinInternalDANGER which is no longer used outside of tests"
  },
  {
   "sha": "65caae102d01439a4b27021b221e9cc609e3a136",
   "date": "2026-10-28T02:45:44Z",
   "message": "utxo_snapshot: Add support for MuHash calculations"
  },
  {
   "sha": "fbda3acde3914cf9ce14a58070467c8989429881",
   "date": "2026-10-28T02:45:44Z",
   "message": "bitcoin-util: Add txoutsetsummary subcommand\n\nAdds a subcommand for quickly generating the summary of a serialized\nutxo set for verification purposes (including various hashes, coin count\nand value)."
  }
 ],
 "timeline": [
  {
   "t": "2026-10-22T10:30:33Z",
   "kind": "comment",
   "who": "ajtowns",
   "assoc": "MEMBER",
   "text": "The performance change I see on a Ryzen 9 3900X system with RAID HDD storage is ~11minutes with to ~1m35s for a load, consisting of ~1m15s of CPU time followed by ~20s of disk flushing; with SSD and a slightly faster CPU, total time is reduced from ~5m30s to ~50s (no significant flush delay after the CPU work is completed here). These numbers are with the block 965000 utxo set.",
   "id": "c:6338585430",
   "url": "https://github.com/bitcoin/bitcoin/pull/36603#issuecomment-6338585430",
   "hash": "01e8f1ca8f74a1e4",
   "edited": "2026-10-25T05:03:13Z"
  },
  {
   "t": "2026-10-23T05:47:53Z",
   "kind": "force_push",
   "who": "ajtowns",
   "commit": "420ca68611179905994fd6efc9ceb847b4d69679",
   "id": "push:420ca68611179905994fd6efc9ceb847b4d69679"
  },
  {
   "t": "2026-10-23T05:52:34Z",
   "kind": "force_push",
   "who": "ajtowns",
   "commit": "51d7d9e66d1a0aac096ae0b4e60bf2f90334307b",
   "id": "push:51d7d9e66d1a0aac096ae0b4e60bf2f90334307b"
  },
  {
   "t": "2026-10-23T06:04:31Z",
   "kind": "force_push",
   "who": "ajtowns",
   "commit": "32575f09eb25b881cc7d4e475a5494aa3ced1a66",
   "id": "push:32575f09eb25b881cc7d4e475a5494aa3ced1a66"
  },
  {
   "t": "2026-10-23T07:03:28Z",
   "kind": "comment",
   "who": "l0rinc",
   "assoc": "MEMBER",
   "text": "Can you please give us more context? Why do you consider an 11-minute load too slow? Is there a reason to think this is what's preventing people from using it - enough to justify yet more complexity in the most \"load-bearing\" parts of the code? Or is this just a demo to explore the possibilities?\n\nOver the past years we've discussed how relying on someone else's UTXO set makes us uncomfortable (see https://github.com/bitcoin/bitcoin/issues/32377 and https://github.com/bitcoin/bitcoin/pull/36598).\nThis change adds *even* more AssumeUTXO-specific complexity, which is exactly why we're considering removing it completely. I wouldn't object to adding it back later through IPC and the Kernel if there's still demand and doesn't add complexity - but there's no indication of that, especially since https://github.com/bitcoin/bitcoin/issues/36476 (possibly combined with SwiftSync) would already speed up actual UTXO set construction by an order of magnitude on resource-constrained nodes. Quickly loading someone else's UTXO set could still be a useful academic pursuit, but many of us are not comfortable polluting the code with multiple chainstates, which make consensus-critical paths harder to reason about.",
   "id": "c:6359437508",
   "url": "https://github.com/bitcoin/bitcoin/pull/36603#issuecomment-6359437508",
   "hash": "0acfe4dbc3b45cbb",
   "edited": null
  },
  {
   "t": "2026-10-23T11:19:06Z",
   "kind": "force_push",
   "who": "ajtowns",
   "commit": "e81eeb8ff565361d55ab2dafe0ac909b375e4344",
   "id": "push:e81eeb8ff565361d55ab2dafe0ac909b375e4344"
  },
  {
   "t": "2026-10-23T11:35:03Z",
   "kind": "comment",
   "who": "ajtowns",
   "assoc": "MEMBER",
   "text": "[quoted text omitted]\n\nIn [bips#2137](https://github.com/bitcoin/bips/pull/2137#issuecomment-6230472208) Sjors suggested splitting up the snapshot by coin to improve performance, which didn't seem sensible to me.\n\n[quoted text omitted]\nI don't consider it too slow; though it's possible it could be on slower systems than the ones I test on. But the reason it is that slow is because the database gets compacted multiple times, and it also seems to use excessive amounts of memory while loading, and there isn't really a good reason to do that. The difference between checking whether better performance is possible and actually making it available to users isn't that large.\n\n[quoted text omitted]\nAs [I've said on the list](https://gnusha.org/pi/bitcoindev/atCSAVprBE-tAnN3@erisian.com.au/T/#md0417be9181751fbb2c600c0337c2917c43018dd), I think the assumeutxo approach is a much better approach to resolving the \"fast startup of a new node\" challenge than trying to get a few extra percent out of IBD performance (which introduces its own large sets of complexities). If it's been decided that that's not one of this project's goals, that's fine I guess, but I haven't seen a project statement along those lines.\n\nMaking assumeutxo a reasonable default, rather than something for experts, does require significantly more complexity (ie, the various stuff for the BIP above, cf #35054). Likewise if we want to do FEC encoding so pruned nodes can help share utxo snapshots or support IBD.",
   "id": "c:6363949408",
   "url": "https://github.com/bitcoin/bitcoin/pull/36603#issuecomment-6363949408",
   "hash": "9cdf7912b1de0dd5",
   "edited": null
  },
  {
   "t": "2026-10-25T04:58:22Z",
   "kind": "force_push",
   "who": "ajtowns",
   "commit": "4840412aa5f1dd18e65c65c836492e7a3855d16c",
   "id": "push:4840412aa5f1dd18e65c65c836492e7a3855d16c"
  },
  {
   "t": "2026-10-25T05:07:00Z",
   "kind": "force_push",
   "who": "ajtowns",
   "commit": "92c6b60734813ca580fe77f7b30e2005cc6df56c",
   "id": "push:92c6b60734813ca580fe77f7b30e2005cc6df56c"
  },
  {
   "t": "2026-10-28T02:47:03Z",
   "kind": "force_push",
   "who": "ajtowns",
   "commit": "fbda3acde3914cf9ce14a58070467c8989429881",
   "id": "push:fbda3acde3914cf9ce14a58070467c8989429881"
  }
 ],
 "labels_log": [
  {
   "t": "2026-10-22T11:37:06Z",
   "action": "labeled",
   "label": "CI failed",
   "who": "DrahtBot"
  },
  {
   "t": "2026-10-23T12:20:52Z",
   "action": "unlabeled",
   "label": "CI failed",
   "who": "DrahtBot"
  },
  {
   "t": "2026-10-25T05:07:53Z",
   "action": "labeled",
   "label": "CI failed",
   "who": "DrahtBot"
  },
  {
   "t": "2026-10-25T06:12:24Z",
   "action": "unlabeled",
   "label": "CI failed",
   "who": "DrahtBot"
  },
  {
   "t": "2026-10-26T18:37:38Z",
   "action": "labeled",
   "label": "Needs rebase",
   "who": "DrahtBot"
  },
  {
   "t": "2026-10-28T03:33:43Z",
   "action": "unlabeled",
   "label": "Needs rebase",
   "who": "DrahtBot"
  }
 ],
 "state_log": [],
 "text_chars": 7224,
 "text_tokens_estimate": 1806,
 "changed_paths": [
  "src/bitcoin-util.cpp",
  "src/coins.cpp",
  "src/coins.h",
  "src/dbwrapper.cpp",
  "src/dbwrapper.h",
  "src/node/blockstorage.cpp",
  "src/node/utxo_snapshot.cpp",
  "src/node/utxo_snapshot.h",
  "src/rpc/blockchain.cpp",
  "src/test/coins_tests.cpp",
  "src/test/coinsviewoverlay_tests.cpp",
  "src/test/fuzz/coins_view.cpp",
  "src/test/fuzz/deserialize.cpp",
  "src/test/fuzz/utxo_snapshot.cpp",
  "src/test/util/chainstate.h",
  "src/test/util_tests.cpp",
  "src/test/validation_chainstatemanager_tests.cpp",
  "src/txdb.cpp",
  "src/txdb.h",
  "src/util/blockingqueue.h",
  "src/util/fs_helpers.cpp",
  "src/util/fs_helpers.h",
  "src/utxo_snapshot.h",
  "src/validation.cpp",
  "src/validation.h",
  "test/functional/feature_assumeutxo.py",
  "test/lint/lint-circular-dependencies.py"
 ],
 "files": [
  {
   "path": "src/bitcoin-util.cpp",
   "add": 137,
   "del": 0
  },
  {
   "path": "src/coins.cpp",
   "add": 0,
   "del": 10
  },
  {
   "path": "src/coins.h",
   "add": 0,
   "del": 9
  },
  {
   "path": "src/dbwrapper.cpp",
   "add": 51,
   "del": 0
  },
  {
   "path": "src/dbwrapper.h",
   "add": 6,
   "del": 0
  },
  {
   "path": "src/node/blockstorage.cpp",
   "add": 1,
   "del": 1
  },
  {
   "path": "src/node/utxo_snapshot.cpp",
   "add": 244,
   "del": 13
  },
  {
   "path": "src/node/utxo_snapshot.h",
   "add": 34,
   "del": 91
  },
  {
   "path": "src/rpc/blockchain.cpp",
   "add": 3,
   "del": 3
  },
  {
   "path": "src/test/coins_tests.cpp",
   "add": 3,
   "del": 24
  },
  {
   "path": "src/test/coinsviewoverlay_tests.cpp",
   "add": 6,
   "del": 3
  },
  {
   "path": "src/test/fuzz/coins_view.cpp",
   "add": 1,
   "del": 1
  },
  {
   "path": "src/test/fuzz/deserialize.cpp",
   "add": 2,
   "del": 3
  },
  {
   "path": "src/test/fuzz/utxo_snapshot.cpp",
   "add": 4,
   "del": 6
  },
  {
   "path": "src/test/util/chainstate.h",
   "add": 4,
   "del": 2
  },
  {
   "path": "src/test/util_tests.cpp",
   "add": 128,
   "del": 0
  },
  {
   "path": "src/test/validation_chainstatemanager_tests.cpp",
   "add": 61,
   "del": 1
  },
  {
   "path": "src/txdb.cpp",
   "add": 20,
   "del": 0
  },
  {
   "path": "src/txdb.h",
   "add": 31,
   "del": 0
  },
  {
   "path": "src/util/blockingqueue.h",
   "add": 135,
   "del": 0
  },
  {
   "path": "src/util/fs_helpers.cpp",
   "add": 16,
   "del": 0
  },
  {
   "path": "src/util/fs_helpers.h",
   "add": 6,
   "del": 0
  },
  {
   "path": "src/utxo_snapshot.h",
   "add": 305,
   "del": 0
  },
  {
   "path": "src/validation.cpp",
   "add": 120,
   "del": 286
  },
  {
   "path": "src/validation.h",
   "add": 1,
   "del": 15
  },
  {
   "path": "test/functional/feature_assumeutxo.py",
   "add": 4,
   "del": 3
  },
  {
   "path": "test/lint/lint-circular-dependencies.py",
   "add": 0,
   "del": 1
  }
 ],
 "test_lines": 257,
 "git": {
  "head": "fbda3acde3914cf9ce14a58070467c8989429881",
  "head_matches_backup": true,
  "base": "2b5b6fe231ff45a09d9e24411e4420d51ed45cab",
  "patch_id": "3ff3bf8b2b52b629",
  "commits": [
   {
    "sha": "52583fba0d",
    "subject": "refactor: Move SnapshotMetadata out of node/",
    "files": 9,
    "add": 110,
    "del": 101
   },
   {
    "sha": "19fa00eb2d",
    "subject": "refactor: Check the snapshot network in the caller",
    "files": 5,
    "add": 33,
    "del": 28
   },
   {
    "sha": "21bef34248",
    "subject": "util: Add DirectoryFilesCommit for syncing the contents of a directory",
    "files": 2,
    "add": 22,
    "del": 0
   },
   {
    "sha": "f7de839fc9",
    "subject": "dbwrapper: Add nosync option for quickly populating a db",
    "files": 2,
    "add": 57,
    "del": 0
   },
   {
    "sha": "a73f679f33",
    "subject": "txdb: Add RawCoinDataDBBatch for directly populating a chainstate",
    "files": 2,
    "add": 47,
    "del": 0
   },
   {
    "sha": "bbe85cc3dd",
    "subject": "util: Add BlockingQueue",
    "files": 2,
    "add": 263,
    "del": 0
   },
   {
    "sha": "8810862d1d",
    "subject": "utxo_snapshot: More efficient snapshot imports",
    "files": 4,
    "add": 542,
    "del": 0
   },
   {
    "sha": "49aa42ccaa",
    "subject": "validation: Make ActivateSnapshot faster",
    "files": 5,
    "add": 130,
    "del": 79
   },
   {
    "sha": "ce01e3c30b",
    "subject": "validation/utxo_snapshot: Remove the old snapshot loader",
    "files": 5,
    "add": 0,
    "del": 262
   },
   {
    "sha": "03fec5e401",
    "subject": "coins: Drop EmplaceCoinInternalDANGER which is no longer used outside of tests",
    "files": 5,
    "add": 10,
    "del": 47
   },
   {
    "sha": "65caae102d",
    "subject": "utxo_snapshot: Add support for MuHash calculations",
    "files": 1,
    "add": 17,
    "del": 0
   },
   {
    "sha": "fbda3acde3",
    "subject": "bitcoin-util: Add txoutsetsummary subcommand",
    "files": 1,
    "add": 137,
    "del": 0
   }
  ],
  "patch_truncated": true
 },
 "patch_id": "3ff3bf8b2b52b629",
 "input_hash": "605f3f60d2ee5b51",
 "extracted_at": "2026-11-02T01:09:49+00:00"
}