{
 "number": 31868,
 "repo": "bitcoin/bitcoin",
 "url": "https://github.com/bitcoin/bitcoin/pull/31868",
 "title": "[IBD] specialize block serialization",
 "author": "l0rinc",
 "author_association": "CONTRIBUTOR",
 "created_at": "2025-02-14T16:48:23Z",
 "updated_at": "2026-06-30T17:50:40Z",
 "age_days": 579,
 "draft": true,
 "labels": [
  "Needs rebase"
 ],
 "milestone": null,
 "base": "master",
 "head_sha": "41ef25fcbaf2c3d14f901ef3f56311a2d396a3d8",
 "head_ref": "lorinc/block-serialization-optimizations",
 "head_repo": "l0rinc/bitcoin",
 "head_history": [
  {
   "t": "2025-03-09T23:58:39Z",
   "sha": "c5bbf571ec2b4bdab3719667d4a95cb21bbcc0c2"
  },
  {
   "t": "2025-03-10T00:09:03Z",
   "sha": "19d20cee89099243b172718dc15eac354a4c6d59"
  },
  {
   "t": "2025-03-10T14:22:44Z",
   "sha": "f98fb619bc433e3a08cb5da9656bf7fc050e5c47"
  },
  {
   "t": "2025-03-10T14:54:42Z",
   "sha": "a9d311c54e6b4dd39d2ae4b8fe31b9261f7e1fbd"
  },
  {
   "t": "2025-03-20T11:45:10Z",
   "sha": "681aad4fdfb6dcae529904845e837f4978899b6f"
  },
  {
   "t": "2025-04-17T10:43:03Z",
   "sha": "787aea27a7e0cc82097fc7c3bb9b52aea4d8e9a2"
  },
  {
   "t": "2025-04-17T12:49:48Z",
   "sha": "5317366a83d4fb4fc0db85beeafef9b2ac0fa495"
  },
  {
   "t": "2025-04-19T18:55:33Z",
   "sha": "073e28b1e9e494389597e2ceb761ca1c842abe3c"
  },
  {
   "t": "2025-07-29T23:26:46Z",
   "sha": "a84f12e90a99f02d1808a4d49b5b037248c25ef3"
  },
  {
   "t": "2025-08-01T01:33:04Z",
   "sha": "f4cb5591a46ea59381dbb2e2ef8422affd936e21"
  },
  {
   "t": "2026-01-15T12:52:32Z",
   "sha": "006864181489fa18bf4a172a51ce4b8a8c3d2b1b"
  },
  {
   "t": "2026-01-15T12:54:44Z",
   "sha": "41ef25fcbaf2c3d14f901ef3f56311a2d396a3d8"
  }
 ],
 "additions": 234,
 "deletions": 98,
 "changed_files": 11,
 "commit_count": 5,
 "size_bucket": "M",
 "mergeable_state": "dirty",
 "bot": {
  "drahtbot": {
   "present": true,
   "reviews": {},
   "conflicts": [
    {
     "number": 34436,
     "title": "refactor: add overflow-safe `CeilDiv` helper and use it in unsigned callsites",
     "author": "l0rinc"
    },
    {
     "number": 34208,
     "title": "bench: add fluent API for untimed `setup` steps in nanobench",
     "author": "l0rinc"
    },
    {
     "number": 32729,
     "title": "test,refactor: extract script template helpers & widen sigop count coverage",
     "author": "l0rinc"
    },
    {
     "number": 31682,
     "title": "[IBD] specialize CheckBlock's input & coinbase checks",
     "author": "l0rinc"
    }
   ]
  }
 },
 "acks_parsed": {},
 "acks_tally": {
  "ack": 0,
  "stale_ack": 0,
  "concept_ack": 0,
  "approach_ack": 0,
  "nack": 0,
  "concept_nack": 0,
  "approach_nack": 0
 },
 "reviews": {
  "approved": 0,
  "changes_requested": 0,
  "distinct_reviewers": [
   "maflcko",
   "sedited",
   "theuni"
  ]
 },
 "signals": {
  "needs_rebase": true,
  "ci_failed": false,
  "mergeable_state": "dirty",
  "last_author_activity": "2026-06-30T17:50:40Z",
  "last_reviewer_activity": "2026-06-27T11:09:24Z",
  "last_reviewer": "maflcko",
  "author_silent_days": 78,
  "waiting_on_author_days": 0,
  "days_since_update": 78
 },
 "refs": {
  "mentioned": [
   31519
  ],
  "depends_on": [],
  "fixes": [],
  "linked_issues": [],
  "references": [
   {
    "number": 31519,
    "type": "pull",
    "state": "closed",
    "merged": true,
    "merged_at": "2025-03-20",
    "title": "refactor: Use std::span over Span"
   }
  ],
  "conflicts": [
   34436,
   34208,
   32729,
   31682
  ]
 },
 "stack": {
  "shares_commits_with": [],
  "based_on": [],
  "base_for": []
 },
 "review_paths": [
  "src/crypto/sha256.cpp",
  "src/serialize.h",
  "src/streams.h"
 ],
 "body": "This change is part of [[IBD] - Tracking PR for speeding up Initial Block Download](https://github.com/bitcoin/bitcoin/pull/32043)\n\n---\n\nThis PR is drafted until I remeasure everything after the recent merges and I need to find a way to simplify the 1 byte writes more nicely, I don't like all the specializations.\n\n---\n\n**Summary:** This PR contains a few different optimizations found by IBD profiling and via the newly added block serialization benchmarks. It also takes advantage of the recently merged [`std::span` changes](https://github.com/bitcoin/bitcoin/pull/31519) enabling propagating static extents.\n\nThe commits merge similar (de)serialization methods and separate them internally with `if constexpr`, similarly to how it has been [done here before](https://github.com/bitcoin/bitcoin/pull/28203). This enables further `SizeComputer` optimizations as well.\n\n**Context:** Since single byte writes are used very often (for every `(u)int8_t`, `std::byte`, or `bool`, and for every `VarInt`'s first byte, which is also needed for every `(pre)Vector`), it makes sense to avoid the generalized serialization infrastructure that isn't needed:\n* `AutoFile` write doesn't need to allocate a 4k buffer for a single byte now;\n* `VectorWriter` and `DataStream` avoid `memcpy`/`insert` calls;\n* `CSHA256::Write` can avoid `memcpy`.\n\n`DeserializeBlock` is dominated by hash calculations, so the optimizations barely affect it.\n\n**Measurements:**\n\nC compiler ............................ AppleClang 16.0.0.16000026\n\n[quoted text omitted]\n|            ns/block |             block/s |    err% |     total | benchmark\n|--------------------:|--------------------:|--------:|----------:|:----------\n|          195,610.62 |            5,112.20 |    0.3% |     11.00 | `SerializeBlock`\n|           12,061.83 |           82,906.19 |    0.1% |     11.01 | `SizeComputerBlock`\n[quoted text omitted]\n|            ns/block |             block/s |    err% |     total | benchmark\n|--------------------:|--------------------:|--------:|----------:|:----------\n|          174,569.19 |            5,728.39 |    0.6% |     10.89 | `SerializeBlock`\n|           10,241.16 |           97,645.21 |    0.0% |     11.00 | `SizeComputerBlock`\n\n[quoted text omitted]\n-----\n\nC++ compiler .......................... GNU 13.3.0\n\n[quoted text omitted]\n|            ns/block |             block/s |    err% |       ins/block |       cyc/block |    IPC |      bra/block |   miss% |     total | benchmark\n|--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:----------\n|          867,857.55 |            1,152.26 |    0.0% |    8,015,883.90 |    3,116,099.08 |  2.572 |   1,517,035.87 |    0.5% |     10.81 | `SerializeBlock`\n|           30,928.27 |           32,332.88 |    0.0% |      221,683.03 |      111,055.84 |  1.996 |      53,037.03 |    0.8% |     11.03 | `SizeComputerBlock`\n[quoted text omitted]\n|            ns/block |             block/s |    err% |       ins/block |       cyc/block |    IPC |      bra/block |   miss% |     total | benchmark\n|--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:----------\n|          615,000.56 |            1,626.01 |    0.0% |    8,015,883.64 |    2,208,340.88 |  3.630 |   1,517,035.62 |    0.5% |     10.56 | `SerializeBlock`\n|           25,676.76 |           38,945.72 |    0.0% |      159,390.03 |       92,202.10 |  1.729 |      42,131.03 |    0.9% |     11.00 | `SizeComputerBlock`\n\n[quoted text omitted]\n----\n\nWhile this wasn't the main motivation for the change, IBD on Ubuntu/GCC on SSD with i9 indicates a 2% speedup as well:\n\nDetails\n```bash\nCOMMITS=\"05314bde0b06b820225f10c6529b5afae128ff81 1cd94ec2511874ec68b92db34ad7ec7d9534fed1\"; \\\nSTOP_HEIGHT=880000; DBCACHE=10000; \\\nC_COMPILER=gcc; CXX_COMPILER=g++; \\\nhyperfine \\\n--export-json \"/mnt/my_storage/ibd-${COMMITS// /-}-${STOP_HEIGHT}-${DBCACHE}-${C_COMPILER}.json\" \\\n--runs 3 \\\n--parameter-list COMMIT ${COMMITS// /,} \\\n--prepare \"killall bitcoind || true; rm -rf /mnt/my_storage/BitcoinData/*; git checkout {COMMIT}; git clean -fxd; git reset --hard; cmake -B build -DCMAKE_BUILD_TYPE=Release -DENABLE_WALLET=OFF -DCMAKE_C_COMPILER=$C_COMPILER -DCMAKE_CXX_COMPILER=$CXX_COMPILER && cmake --build build -j$(nproc) --target bitcoind && ./build/bin/bitcoind -datadir=/mnt/my_storage/BitcoinData -stopatheight=1 -printtoconsole=0 || true\" \\\n--cleanup \"cp /mnt/my_storage/BitcoinData/debug.log /mnt/my_storage/logs/debug-{COMMIT}-$(date +%s).log || true\" \\\n\"COMPILER=$C_COMPILER COMMIT={COMMIT} ./build/bin/bitcoind -datadir=/mnt/my_storage/BitcoinData -stopatheight=$STOP_HEIGHT -dbcache=$DBCACHE -prune=550 -printtoconsole=0\"\nBenchmark 1: COMPILER=gcc COMMIT=05314bde0b06b820225f10c6529b5afae128ff81 ./build/bin/bitcoind -datadir=/mnt/my_storage/BitcoinData -stopatheight=880000 -dbcache=10000 -prune=550 -printtoconsole=0\n  Time (mean \u00b1 \u03c3):     33647.918 s \u00b1 508.655 s    [User: 71503.409 s, System: 4404.899 s]\n  Range (min \u2026 max):   33283.439 s \u2026 34229.026 s    3 runs\n\nBenchmark 2: COMPILER=gcc COMMIT=1cd94ec2511874ec68b92db34ad7ec7d9534fed1 ./build/bin/bitcoind -datadir=/mnt/my_storage/BitcoinData -stopatheight=880000 -dbcache=10000 -prune=550 -printtoconsole=0\n  Time (mean \u00b1 \u03c3):     33062.491 s \u00b1 183.335 s    [User: 71246.532 s, System: 4318.490 s]\n  Range (min \u2026 max):   32888.211 s \u2026 33253.706 s    3 runs\n\nSummary\n  COMPILER=gcc COMMIT=1cd94ec2511874ec68b92db34ad7ec7d9534fed1 ./build/bin/bitcoind -datadir=/mnt/my_storage/BitcoinData -stopatheight=880000 -dbcache=10000 -prune=550 -printtoconsole=0 ran\n    1.02 \u00b1 0.02 times faster than COMPILER=gcc COMMIT=05314bde0b06b820225f10c6529b5afae128ff81 ./build/bin/bitcoind -datadir=/mnt/my_storage/BitcoinData -stopatheight=880000 -dbcache=10000 -prune=550 -printtoconsole=0\n```",
 "commits": [
  {
   "sha": "7ba6866139bc4736b9d55e3a8d1b5000e7726cc9",
   "date": "2026-01-15T12:52:06Z",
   "message": "bench: measure block (size)serialization speed\n\nMeasure both full block serialization and size computation via `SizeComputer`.\n`SizeComputer` returns the exact final size of the serialized content without writing any bytes.\n\n[quoted text omitted]\n|            ns/block |             block/s |    err% |     total | benchmark\n|--------------------:|--------------------:|--------:|----------:|:----------\n|          195,610.62 |            5,112.20 |    0.3% |     11.00 | `SerializeBlock`\n|           12,061.83 |           82,906.19 |    0.1% |     11.01 | `SizeComputerBlock`\n\n[quoted text omitted]\n|            ns/block |             block/s |    err% |       ins/block |       cyc/block |    IPC |      bra/block |   miss% |     total | benchmark\n|--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:----------\n|          867,857.55 |            1,152.26 |    0.0% |    8,015,883.90 |    3,116,099.08 |  2.572 |   1,517,035.87 |    0.5% |     10.81 | `SerializeBlock`\n|           30,928.27 |           32,332.88 |    0.0% |      221,683.03 |      111,055.84 |  1.996 |      53,037.03 |    0.8% |     11.03 | `SizeComputerBlock`"
  },
  {
   "sha": "1aadf0da9d2609dd5ee8c6aec342c5bdb2492159",
   "date": "2026-01-15T12:52:09Z",
   "message": "refactor: reduce template bloat in primitive serialization\n\nMerged multiple template methods into single constexpr-delimited implementation to reduce template bloat (i.e. related functionality is grouped into a single method, but can be optimized because of C++20 constexpr conditions).\nThis unifies related methods that were only bound before by similar signatures - and enables `SizeComputer` optimizations later"
  },
  {
   "sha": "e2be418d704f97f713710557007a658658512755",
   "date": "2026-01-15T12:52:09Z",
   "message": "refactor: add explicit static extent to spans"
  },
  {
   "sha": "42a54c690131576f6f5f0c671ccaecffc364e08e",
   "date": "2026-01-15T12:52:09Z",
   "message": "optimization: merge `SizeComputer` specializations and add new overloads\n\nEndianness doesn\u2019t affect final size, so skip it in `SizeComputer`.\nFold existing overloads into one implementation, short\u2011circuiting logic when only the serialized size is needed.\n\n[quoted text omitted]\n|            ns/block |             block/s |    err% |     total | benchmark\n|--------------------:|--------------------:|--------:|----------:|:----------\n|          191,652.29 |            5,217.78 |    0.4% |     10.96 | `SerializeBlock`\n|           10,323.55 |           96,865.92 |    0.2% |     11.01 | `SizeComputerBlock`\n\n[quoted text omitted]\n|            ns/block |             block/s |    err% |       ins/block |       cyc/block |    IPC |      bra/block |   miss% |     total | benchmark\n|--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:----------\n|          614,847.32 |            1,626.42 |    0.0% |    8,015,883.64 |    2,207,628.07 |  3.631 |   1,517,035.62 |    0.5% |     10.56 | `SerializeBlock`\n|           26,020.31 |           38,431.52 |    0.0% |      159,390.03 |       93,438.33 |  1.706 |      42,131.03 |    0.9% |     11.00 | `SizeComputerBlock`"
  },
  {
   "sha": "41ef25fcbaf2c3d14f901ef3f56311a2d396a3d8",
   "date": "2026-01-15T12:53:21Z",
   "message": "optimization: add single byte writes\n\nSingle byte writes are used very often (used for every (u)int8_t or std::byte or bool and for every VarInt's first byte which is also needed for every (pre)Vector).\nIt makes sense to avoid the generalized serialization infrastructure that isn't needed:\n* AutoFile write doesn't need to allocate 4k buffer for a single byte now;\n* `VectorWriter` and `DataStream` avoids memcpy/insert calls.\n\n[quoted text omitted]\n|            ns/block |             block/s |    err% |     total | benchmark\n|--------------------:|--------------------:|--------:|----------:|:----------\n|          174,569.19 |            5,728.39 |    0.6% |     10.89 | `SerializeBlock`\n|           10,241.16 |           97,645.21 |    0.0% |     11.00 | `SizeComputerBlock`\n\n[quoted text omitted]\n|            ns/block |             block/s |    err% |       ins/block |       cyc/block |    IPC |      bra/block |   miss% |     total | benchmark\n|--------------------:|--------------------:|--------:|----------------:|----------------:|-------:|---------------:|--------:|----------:|:----------\n|          615,000.56 |            1,626.01 |    0.0% |    8,015,883.64 |    2,208,340.88 |  3.630 |   1,517,035.62 |    0.5% |     10.56 | `SerializeBlock`\n|           25,676.76 |           38,945.72 |    0.0% |      159,390.03 |       92,202.10 |  1.729 |      42,131.03 |    0.9% |     11.00 | `SizeComputerBlock`"
  }
 ],
 "timeline": [
  {
   "t": "2025-02-14T17:13:06Z",
   "kind": "review_comment",
   "who": "theuni",
   "assoc": "MEMBER",
   "path": "src/streams.h",
   "commit": "f4cb5591a46ea59381dbb2e2ef8422affd936e21",
   "in_reply_to": null,
   "text": "These are nice optims, but [as I mentoned here](https://github.com/bitcoin/bitcoin/pull/31519#discussion_r1929225185):\n\n[quoted text omitted]\nI think it makes sense to wait for the `std::span` replacement (#31519) to do this, that way we can specialize for any static extent instead which should compile down to nothing: https://compiler-explorer.com/z/97aY3bnK8"
  },
  {
   "t": "2025-02-14T20:33:23Z",
   "kind": "review_comment",
   "who": "l0rinc",
   "assoc": "CONTRIBUTOR",
   "path": "src/streams.h",
   "commit": "f4cb5591a46ea59381dbb2e2ef8422affd936e21",
   "in_reply_to": 1956480589,
   "text": "Absolutely, I already have other optimization ideas in mind after that's merged.\n\nThe reviewers can decide the preferred merge order, I don't mind rebasing or doing it in multiple PRs - there's a lot of work left with serialization anyway."
  },
  {
   "t": "2025-03-09T23:58:39Z",
   "kind": "force_push",
   "who": "l0rinc",
   "commit": "c5bbf571ec2b4bdab3719667d4a95cb21bbcc0c2"
  },
  {
   "t": "2025-03-09T23:59:46Z",
   "kind": "comment",
   "who": "l0rinc",
   "assoc": "CONTRIBUTOR",
   "text": "Drafting until https://github.com/bitcoin/bitcoin/pull/31519 is merged, as recommended in https://github.com/bitcoin/bitcoin/pull/31868#discussion_r1956480589"
  },
  {
   "t": "2025-03-10T00:01:15Z",
   "kind": "review_comment",
   "who": "l0rinc",
   "assoc": "CONTRIBUTOR",
   "path": "src/streams.h",
   "commit": "f4cb5591a46ea59381dbb2e2ef8422affd936e21",
   "in_reply_to": 1956480589,
   "text": "Rebased on top of https://github.com/bitcoin/bitcoin/pull/31519 and experimented with static extents - the speed is not the same as with bare `std::byte` parameters, but close enough and the code is more generalized.\nDrafting until the span PR is merged - suggestions for further investigations are welcome."
  },
  {
   "t": "2025-03-10T00:09:03Z",
   "kind": "force_push",
   "who": "l0rinc",
   "commit": "19d20cee89099243b172718dc15eac354a4c6d59"
  },
  {
   "t": "2025-03-10T14:22:44Z",
   "kind": "force_push",
   "who": "l0rinc",
   "commit": "f98fb619bc433e3a08cb5da9656bf7fc050e5c47"
  },
  {
   "t": "2025-03-10T14:54:42Z",
   "kind": "force_push",
   "who": "l0rinc",
   "commit": "a9d311c54e6b4dd39d2ae4b8fe31b9261f7e1fbd"
  },
  {
   "t": "2025-03-20T11:45:10Z",
   "kind": "force_push",
   "who": "l0rinc",
   "commit": "681aad4fdfb6dcae529904845e837f4978899b6f"
  },
  {
   "t": "2025-03-20T11:45:25Z",
   "kind": "review_comment",
   "who": "l0rinc",
   "assoc": "CONTRIBUTOR",
   "path": "src/streams.h",
   "commit": "f4cb5591a46ea59381dbb2e2ef8422affd936e21",
   "in_reply_to": 1956480589,
   "text": "Now that #31519 was merged, I've rebased an moved it out of draft."
  },
  {
   "t": "2025-04-17T08:59:18Z",
   "kind": "review_comment",
   "who": "sedited",
   "assoc": "CONTRIBUTOR",
   "path": "src/serialize.h",
   "commit": "41ef25fcbaf2c3d14f901ef3f56311a2d396a3d8",
   "in_reply_to": null,
   "text": "In commit c9a69f9088340df88017752e1016670141b6ad74:\n\nLooking at the concept, could this also be `unsigned char`?"
  },
  {
   "t": "2025-04-17T09:58:32Z",
   "kind": "review_comment",
   "who": "sedited",
   "assoc": "CONTRIBUTOR",
   "path": "src/serialize.h",
   "commit": "41ef25fcbaf2c3d14f901ef3f56311a2d396a3d8",
   "in_reply_to": null,
   "text": "Nit: Maybe add a description here too along the lines of:\nCheck if type contains a SizeComputer by seeing if the return type of T's GetStream() method is a SizeComputer."
  },
  {
   "t": "2025-04-17T10:43:03Z",
   "kind": "force_push",
   "who": "l0rinc",
   "commit": "787aea27a7e0cc82097fc7c3bb9b52aea4d8e9a2"
  },
  {
   "t": "2025-04-17T11:58:33Z",
   "kind": "review",
   "who": "sedited",
   "assoc": "CONTRIBUTOR",
   "state": "COMMENTED",
   "commit": "787aea27a7e0cc82097fc7c3bb9b52aea4d8e9a2",
   "text": "This looks good, but I am not really reproducing any of the performance changes on my machine yet. Maybe my laptop is just too unreliable though."
  },
  {
   "t": "2025-04-17T12:28:08Z",
   "kind": "comment",
   "who": "l0rinc",
   "assoc": "CONTRIBUTOR",
   "text": "[quoted text omitted]\n\nI just removed the single-byte specializations, they were quite ugly, will have to add it back somehow later - will draft it until I do that, thanks for checking!"
  },
  {
   "t": "2025-04-17T12:49:48Z",
   "kind": "force_push",
   "who": "l0rinc",
   "commit": "5317366a83d4fb4fc0db85beeafef9b2ac0fa495"
  },
  {
   "t": "2025-04-17T12:50:53Z",
   "kind": "comment",
   "who": "l0rinc",
   "assoc": "CONTRIBUTOR",
   "text": "@TheCharlatan, I've added back the single-byte optimizations (simplified slightly after rebase), your feedback is appreciated."
  },
  {
   "t": "2025-04-17T13:58:17Z",
   "kind": "review_comment",
   "who": "sedited",
   "assoc": "CONTRIBUTOR",
   "path": "src/crypto/sha256.cpp",
   "commit": "f4cb5591a46ea59381dbb2e2ef8422affd936e21",
   "in_reply_to": null,
   "text": "I'm not sure about all these crypto changes. Will this be noticable at all? Why only do it for the sha256 hasher? Maybe do these once/if the other single byte steam writer changes are accepted."
  },
  {
   "t": "2025-04-17T14:10:47Z",
   "kind": "review",
   "who": "sedited",
   "assoc": "CONTRIBUTOR",
   "state": "COMMENTED",
   "commit": "5317366a83d4fb4fc0db85beeafef9b2ac0fa495",
   "text": "This looks ok, albeit that I'm seeing more modest performance improvements on my end."
  },
  {
   "t": "2025-04-17T15:40:56Z",
   "kind": "comment",
   "who": "maflcko",
   "assoc": "MEMBER",
   "text": "[quoted text omitted]\n\nIf IBD speedup is not the main motivation, what else is the motivation, given that the title mentions `[IBD]`?\n\nAlso, what code part is hitting this during IBD, given that reads and writes are now buffered?"
  },
  {
   "t": "2025-04-17T15:46:39Z",
   "kind": "review_comment",
   "who": "l0rinc",
   "assoc": "CONTRIBUTOR",
   "path": "src/serialize.h",
   "commit": "41ef25fcbaf2c3d14f901ef3f56311a2d396a3d8",
   "in_reply_to": 2048540359,
   "text": "Yes:\n[quoted text omitted]"
  },
  {
   "t": "2025-04-17T15:47:52Z",
   "kind": "comment",
   "who": "l0rinc",
   "assoc": "CONTRIBUTOR",
   "text": "[quoted text omitted]\n\nDuring serialization we have very many 1 byte writes, this change avoids the heavy allocations.\nBut drafting until I have time to properly remeasure everything after the recent merge."
  },
  {
   "t": "2025-04-17T15:50:50Z",
   "kind": "comment",
   "who": "maflcko",
   "assoc": "MEMBER",
   "text": "I understand that there are many 1-byte writes, but they go into the BufferedReader/Writer, which is not modified here. The Buffered* then passes them to AutoFile, which is modified here, but never receives 1-byte writes anymore"
  },
  {
   "t": "2025-04-19T18:55:33Z",
   "kind": "force_push",
   "who": "l0rinc",
   "commit": "073e28b1e9e494389597e2ceb761ca1c842abe3c"
  },
  {
   "t": "2025-05-01T15:43:00Z",
   "kind": "comment",
   "who": "sedited",
   "assoc": "CONTRIBUTOR",
   "text": "[quoted text omitted]\n\nI think block serialization being fast could be useful for future kernel users. Reading and de-serializing blocks is one of the common operations I'd expect outside of just pure block validation."
  },
  {
   "t": "2025-05-04T13:15:13Z",
   "kind": "comment",
   "who": "maflcko",
   "assoc": "MEMBER",
   "text": "[quoted text omitted]\n\nOk, seems fine. It would be good to list the main motivation(s) in the OP. Currently, it only says IBD and that IBD *isn't* the main motivation. However, I don't understand how this could have any effect at all, from looking at the code, see https://github.com/bitcoin/bitcoin/pull/31868#issuecomment-2813398761."
  },
  {
   "t": "2025-05-04T13:49:40Z",
   "kind": "comment",
   "who": "l0rinc",
   "assoc": "CONTRIBUTOR",
   "text": "I will work on this more in the upcoming weeks, drafted it for now since I don't like the current single-byte specializations, have to find a better way. Suggestions are welcome."
  },
  {
   "t": "2025-05-04T14:46:48Z",
   "kind": "comment",
   "who": "maflcko",
   "assoc": "MEMBER",
   "text": "My guess is that the concept specialization here at most could have an effect on xor'd IO? This is because anything not xor'd won't need a buffer to (un)do the xor before reading or writing.\n\nThus, it just seems easier to use the existing BufferedReader/Writer in places that could benefit from it?\n\nHowever, for block serialization they are already used.\n\n(Edit: I misunderstood that this has benefits outside of BufferedReader. Please ignore this comment)\n\nAgain, no objection here, but it would be good to explain:\n\n* What is the goal of the changes here, and what real-world usage do they affect\n* How is it achieved and how do the benchmarks represent the real-world usage\n\nRight now:\n\n* It only states IBD as motivation and also that IBD isn't the motivation. Also the IBD speedup measurement seems outdated.\n* The added benchmark is about test-only code paths (DataStream) or p2p code paths (VectorWriter). It is unclear what the `AutoFile` changes are about. Also, it is unclear how this could possibly affect the kernel, given that the kernel doesn't call into test-only or p2p-only code paths and instead uses BufferedReader/Writer."
  },
  {
   "t": "2025-07-29T23:26:46Z",
   "kind": "force_push",
   "who": "l0rinc",
   "commit": "a84f12e90a99f02d1808a4d49b5b037248c25ef3"
  },
  {
   "t": "2025-07-29T23:34:29Z",
   "kind": "comment",
   "who": "l0rinc",
   "assoc": "CONTRIBUTOR",
   "text": "Rebased to solve conflict, the PR is still in draft mode until I remeasure its effect on IBD - and I need to find a nicer way to do the single-byte specializations, since they have a measurable effect but make the code super-ugly... Suggestions are welcome.\nIn the meantime I have split out the first refactor into https://github.com/bitcoin/bitcoin/pull/33093."
  },
  {
   "t": "2025-08-01T01:33:04Z",
   "kind": "force_push",
   "who": "l0rinc",
   "commit": "f4cb5591a46ea59381dbb2e2ef8422affd936e21"
  },
  {
   "t": "2025-09-01T10:35:57Z",
   "kind": "comment",
   "who": "maflcko",
   "assoc": "MEMBER",
   "text": "Are you still working on this?"
  },
  {
   "t": "2025-09-01T18:12:11Z",
   "kind": "comment",
   "who": "l0rinc",
   "assoc": "CONTRIBUTOR",
   "text": "Of course, paused it since I'm not exactly sure how do handle single-byte entries: they measurably speed up the scenarios, but the resulting code is quite ugly. Do you have a suggestion? Should I attempt that in a separate PR and ignore it here?\n\nEdit: draft because of the requests in https://github.com/bitcoin/bitcoin/pull/31868#issuecomment-2849258766 which I will need to remeasure"
  },
  {
   "t": "2025-09-02T07:26:25Z",
   "kind": "comment",
   "who": "maflcko",
   "assoc": "MEMBER",
   "text": "What was the speedup? I can only see your last comment, which says:\n\n[quoted text omitted]"
  },
  {
   "t": "2026-01-15T12:52:32Z",
   "kind": "force_push",
   "who": "l0rinc",
   "commit": "006864181489fa18bf4a172a51ce4b8a8c3d2b1b"
  },
  {
   "t": "2026-01-15T12:54:44Z",
   "kind": "force_push",
   "who": "l0rinc",
   "commit": "41ef25fcbaf2c3d14f901ef3f56311a2d396a3d8"
  },
  {
   "t": "2026-06-27T11:09:24Z",
   "kind": "comment",
   "who": "maflcko",
   "assoc": "MEMBER",
   "text": "Just checking in here. This still needs rebase, and I still wonder about the speedup, and the scope: https://github.com/bitcoin/bitcoin/pull/31868#issuecomment-2849258766\n\nIn any case, I read https://purplekarrot.net/blog/serialization.html and it introduces something like this behind a compilation boundary (to get faster compilation and the blog also claims an architectural cleanup):\n\n```cpp\nvoid serialize(CBlock const& block, writer_ref writer)\n{\n  auto encode = encode_block(witness::allow);\n  auto w = BufferedWriter<writer_ref>{writer};\n  encode(w, block);\n}\n```\n\nI wonder if this is an alternative to this pull request and if this will also result in possible runtime speedups as the 1-byte reads/writes don't hit a read/write syscall one-by-one in a hot loop, and instead go through a buffer.\n\nI can understand that compilation speed improvements or architectural cleanup is nice, but when there is no visible speedup or other benefit for external users, the motivation is weaker.\n\nJust wondering what the greater picture here is and if there is any visible speedup.\n\ncc @purpleKarrot"
  },
  {
   "t": "2026-06-30T17:50:40Z",
   "kind": "comment",
   "who": "l0rinc",
   "assoc": "CONTRIBUTOR",
   "text": "[quoted text omitted]\n\nThanks, valid concerns. I have a rebased and updated branch locally; once I finish the reviews and PRs I'm working on, I'll retest and adjust this one - see https://github.com/l0rinc/bitcoin/pull/201"
  }
 ],
 "labels_log": [
  {
   "t": "2025-02-16T05:49:10Z",
   "action": "labeled",
   "label": "CI failed",
   "who": "DrahtBot"
  },
  {
   "t": "2025-02-16T14:40:57Z",
   "action": "unlabeled",
   "label": "CI failed",
   "who": "DrahtBot"
  },
  {
   "t": "2025-03-10T00:09:10Z",
   "action": "labeled",
   "label": "CI failed",
   "who": "DrahtBot"
  },
  {
   "t": "2025-03-10T16:50:03Z",
   "action": "unlabeled",
   "label": "CI failed",
   "who": "DrahtBot"
  },
  {
   "t": "2025-03-20T10:10:53Z",
   "action": "labeled",
   "label": "Needs rebase",
   "who": "DrahtBot"
  },
  {
   "t": "2025-03-20T12:23:37Z",
   "action": "unlabeled",
   "label": "Needs rebase",
   "who": "DrahtBot"
  },
  {
   "t": "2025-05-16T08:17:11Z",
   "action": "labeled",
   "label": "CI failed",
   "who": "DrahtBot"
  },
  {
   "t": "2025-05-16T09:29:02Z",
   "action": "labeled",
   "label": "Needs rebase",
   "who": "DrahtBot"
  },
  {
   "t": "2025-07-30T01:10:59Z",
   "action": "unlabeled",
   "label": "Needs rebase",
   "who": "DrahtBot"
  },
  {
   "t": "2025-07-30T05:10:19Z",
   "action": "unlabeled",
   "label": "CI failed",
   "who": "DrahtBot"
  },
  {
   "t": "2025-11-19T10:20:32Z",
   "action": "labeled",
   "label": "Needs rebase",
   "who": "DrahtBot"
  },
  {
   "t": "2026-01-15T12:55:11Z",
   "action": "labeled",
   "label": "CI failed",
   "who": "DrahtBot"
  },
  {
   "t": "2026-01-15T14:55:56Z",
   "action": "unlabeled",
   "label": "Needs rebase",
   "who": "DrahtBot"
  },
  {
   "t": "2026-01-15T15:20:23Z",
   "action": "unlabeled",
   "label": "CI failed",
   "who": "DrahtBot"
  },
  {
   "t": "2026-03-11T11:31:34Z",
   "action": "labeled",
   "label": "Needs rebase",
   "who": "DrahtBot"
  }
 ],
 "state_log": [
  {
   "t": "2025-03-09T23:59:52Z",
   "kind": "convert_to_draft",
   "who": "l0rinc"
  },
  {
   "t": "2025-03-12T15:36:38Z",
   "kind": "renamed",
   "who": "l0rinc",
   "from": "optimization: speed up block serialization",
   "to": "[IBD] Specialize block serialization"
  },
  {
   "t": "2025-03-12T16:22:50Z",
   "kind": "renamed",
   "who": "l0rinc",
   "from": "[IBD] Specialize block serialization",
   "to": "[IBD] specialize block serialization"
  },
  {
   "t": "2025-03-20T11:45:36Z",
   "kind": "ready_for_review",
   "who": "l0rinc"
  },
  {
   "t": "2025-04-17T12:28:13Z",
   "kind": "convert_to_draft",
   "who": "l0rinc"
  },
  {
   "t": "2025-04-17T12:51:51Z",
   "kind": "ready_for_review",
   "who": "l0rinc"
  },
  {
   "t": "2025-04-17T15:43:09Z",
   "kind": "convert_to_draft",
   "who": "l0rinc"
  }
 ],
 "text_chars": 17212,
 "text_tokens_estimate": 4303,
 "changed_paths": [
  "src/bench/checkblock.cpp",
  "src/bench/rpc_blockchain.cpp",
  "src/crypto/sha256.cpp",
  "src/crypto/sha256.h",
  "src/hash.h",
  "src/serialize.h",
  "src/streams.cpp",
  "src/streams.h",
  "src/test/crypto_tests.cpp",
  "src/test/fuzz/autofile.cpp",
  "src/test/streams_tests.cpp"
 ],
 "files": [
  {
   "path": "src/bench/checkblock.cpp",
   "add": 31,
   "del": 6
  },
  {
   "path": "src/bench/rpc_blockchain.cpp",
   "add": 1,
   "del": 1
  },
  {
   "path": "src/crypto/sha256.cpp",
   "add": 15,
   "del": 0
  },
  {
   "path": "src/crypto/sha256.h",
   "add": 1,
   "del": 0
  },
  {
   "path": "src/hash.h",
   "add": 23,
   "del": 1
  },
  {
   "path": "src/serialize.h",
   "add": 126,
   "del": 84
  },
  {
   "path": "src/streams.cpp",
   "add": 13,
   "del": 0
  },
  {
   "path": "src/streams.h",
   "add": 18,
   "del": 0
  },
  {
   "path": "src/test/crypto_tests.cpp",
   "add": 1,
   "del": 1
  },
  {
   "path": "src/test/fuzz/autofile.cpp",
   "add": 2,
   "del": 2
  },
  {
   "path": "src/test/streams_tests.cpp",
   "add": 3,
   "del": 3
  }
 ],
 "test_lines": 51,
 "git": {
  "head": "41ef25fcbaf2c3d14f901ef3f56311a2d396a3d8",
  "head_matches_backup": true,
  "base": "baa554f7089d8ce8ecb11e78ae097c10bfa85d8d",
  "commits": [
   {
    "sha": "7ba6866139",
    "subject": "bench: measure block (size)serialization speed",
    "files": 2,
    "add": 30,
    "del": 4
   },
   {
    "sha": "1aadf0da9d",
    "subject": "refactor: reduce template bloat in primitive serialization",
    "files": 1,
    "add": 39,
    "del": 33
   },
   {
    "sha": "e2be418d70",
    "subject": "refactor: add explicit static extent to spans",
    "files": 1,
    "add": 14,
    "del": 15
   },
   {
    "sha": "42a54c6901",
    "subject": "optimization: merge `SizeComputer` specializations and add new overloads",
    "files": 1,
    "add": 73,
    "del": 42
   },
   {
    "sha": "41ef25fcba",
    "subject": "optimization: add single byte writes",
    "files": 11,
    "add": 84,
    "del": 10
   }
  ],
  "patch_truncated": false
 },
 "input_hash": "430b61a0989d503c",
 "extracted_at": "2026-09-17T16:15:31+00:00"
}