From b2b153aba67cbae96028939c57410d3c6ea4a2bd Mon Sep 17 00:00:00 2001 From: Eric Voskuil Date: Thu, 30 Jul 2026 23:44:25 -0400 Subject: [PATCH 1/5] Posix memory map control header page release. --- .../database/impl/memory/mmap_private.ipp | 3 + .../database/impl/memory/mmap_staging.ipp | 140 +++++++++++++++++- .../database/impl/memory/mmap_storage.ipp | 27 ++++ .../database/impl/primitives/arrayhead.ipp | 3 + .../database/impl/primitives/hashhead.ipp | 3 + .../database/impl/primitives/nohead.ipp | 2 + .../database/memory/interfaces/storage.hpp | 6 + include/bitcoin/database/memory/mmap.hpp | 17 +++ include/bitcoin/database/memory/mstage.hpp | 4 + src/memory/mstage.cpp | 45 ++++++ 10 files changed, 249 insertions(+), 1 deletion(-) diff --git a/include/bitcoin/database/impl/memory/mmap_private.ipp b/include/bitcoin/database/impl/memory/mmap_private.ipp index 70be61bd7..8f6ffb3b9 100644 --- a/include/bitcoin/database/impl/memory/mmap_private.ipp +++ b/include/bitcoin/database/impl/memory/mmap_private.ipp @@ -95,7 +95,10 @@ bool CLASS::unmap_all_(std::index_sequence) NOEXCEPT frontier_.store(zero); marks_.store(zero); dirty_.reset(); + intent_.reset(); + released_.reset(); words_ = zero; + engaged_.store(false); #endif return success; diff --git a/include/bitcoin/database/impl/memory/mmap_staging.ipp b/include/bitcoin/database/impl/memory/mmap_staging.ipp index ac4f830bf..3a2db742e 100644 --- a/include/bitcoin/database/impl/memory/mmap_staging.ipp +++ b/include/bitcoin/database/impl/memory/mmap_staging.ipp @@ -325,6 +325,8 @@ bool CLASS::stage_() NOEXCEPT const auto pages = ceilinged_divide(reserved, page_); words_ = ceilinged_divide(pages, page_bound); dirty_ = std::make_unique(words_); + intent_ = std::make_unique(words_); + released_ = std::make_unique(words_); } } @@ -448,6 +450,11 @@ bool CLASS::commit_(size_t size) NOEXCEPT dirty_ = std::move(grown); words_ = words; + + // The replacement reservation is fully anonymous (content was + // copied above), so released and intent page state resets. + intent_ = std::make_unique(words); + released_ = std::make_unique(words); } } @@ -837,9 +844,16 @@ void CLASS::settler_run_() NOEXCEPT // before reading, so racing writes remark and transfer on the next pass // (torn disk pages are unreachable, as live heads are only trusted // following a clean close). +// +// Memory scarcity additionally pages the head to its own file (the windows +// model): transfer, then release cold clean pages to read-only file mappings +// (reclaimable cache), restored to anonymous by prepare() before any write. +// Release waits one pass after engagement so all writers declare intent. TEMPLATE void CLASS::head_run_() NOEXCEPT { + const auto scarce = system_memory() / evict_factor; + // Ticks without a mark before idle draining. auto mark = marks_.load(); auto transferred = mark; @@ -858,7 +872,9 @@ void CLASS::head_run_() NOEXCEPT still = (top == mark) ? std::min(add1(still), idle_seconds) : zero; mark = top; - if ((still < idle_seconds) || (transferred == top)) + const auto scarcity = dirty_ && (system_free() < scarce); + if (!scarcity && + ((still < idle_seconds) || (transferred == top))) continue; { @@ -867,12 +883,19 @@ void CLASS::head_run_() NOEXCEPT if (!loaded_.load() || fault_.load()) continue; + const auto engaged = engaged_.load(); + if (scarcity && !engaged) + engaged_.store(true); + if (!transfer_(to_width(logical_.load())) || !sync_()) { set_first_code(error::fsync_failure); continue; } + + if (scarcity && engaged && !release_pages_()) + continue; } transferred = top; @@ -880,6 +903,121 @@ void CLASS::head_run_() NOEXCEPT } } +// Release cold clean head pages to read-only file mappings (reclaimable), +// full pages below logical only. Writer synchronization is a per-page bit +// protocol: prepare() declares intent then loads released; release stores +// released then loads intent (both sequentially consistent), so a page +// converts only when no write can land on it unrestored. A wrong release +// costs one restore. Conversion and restore serialize on restore_mutex_. +TEMPLATE +bool CLASS::release_pages_() NOEXCEPT +{ + using namespace system; + const auto bytes = to_width(logical_.load()); + const auto pages = bytes / page_; + const auto bound = std::min(words_, ceilinged_divide(pages, page_bound)); + + std::unique_lock restore_lock(restore_mutex_); + + size_t from{}; + size_t to{}; + const auto convert = [&]() NOEXCEPT + { + return (from >= to) || (mmap_settle( + std::next(memory_map_[zero], from), to - from, opened_[zero], + from) != fail); + }; + + for (size_t word{}; word < bound; ++word) + { + // Hot mask: pages written since the previous pass. Aging clears only + // the snapshot bits, so a concurrent declaration on a candidate page + // is retained for the live rechecks below. + const auto hot = intent_[word].load(); + intent_[word].fetch_and(system::bit_not(hot)); + const auto dirt = dirty_[word].load(relaxed); + const auto done = released_[word].load(relaxed); + + // Retain candidacy below the page bound (boundary word only). + auto bits = bit_not(bit_or(hot, bit_or(dirt, done))); + const auto first = word * page_bound; + if (pages < (first + page_bound)) + bits = bit_and(bits, mask_right(pages - first)); + + for (size_t bit{}; !is_zero(bits) && (bit < page_bound); ++bit) + { + if (!get_right(bits, bit)) + continue; + + bits = set_right(bits, bit, false); + const auto flag = bit_right(bit); + released_[word].fetch_or(flag); + + // A raced intent or mark invalidates only this page's release. + if (!is_zero(bit_and(intent_[word].load(), flag)) || + !is_zero(bit_and(dirty_[word].load(), flag))) + { + released_[word].fetch_and(bit_not(flag)); + continue; + } + + const auto start = (first + bit) * page_; + if (start == to) + { + to = start + page_; + continue; + } + + if (!convert()) + { + set_first_code(error::mmap_failure); + return false; + } + + from = start; + to = start + page_; + } + } + + if (!convert()) + { + set_first_code(error::mmap_failure); + return false; + } + + return true; +} + +// Restore released pages within [offset, offset+size) to writable anonymous +// memory (content preserved by atomic installation), before a declared write. +TEMPLATE +void CLASS::restore_(size_t offset, size_t size) NOEXCEPT +{ + using namespace system; + std::unique_lock restore_lock(restore_mutex_); + + auto page = offset / page_; + const auto end = (offset + sub1(size)) / page_; + while ((page <= end) && ((page / page_bound) < words_)) + { + const auto word = page / page_bound; + const auto flag = bit_right(page % page_bound); + if (!is_zero(bit_and(released_[word].load(), flag))) + { + if (mmap_restore(std::next(memory_map_[zero], page * page_), + page_) == fail) + { + set_first_code(error::mmap_failure); + return; + } + + released_[word].fetch_and(bit_not(flag)); + } + + ++page; + } +} + // Settle up to chunk completed rows: write under the shared remap lock // (completed extents are immutable, writers proceed), convert under a brief // exclusive. Durability remains a snapshot property (no sync here). diff --git a/include/bitcoin/database/impl/memory/mmap_storage.ipp b/include/bitcoin/database/impl/memory/mmap_storage.ipp index 6c7d9b3b6..ec3ef0fb3 100644 --- a/include/bitcoin/database/impl/memory/mmap_storage.ipp +++ b/include/bitcoin/database/impl/memory/mmap_storage.ipp @@ -187,6 +187,33 @@ code CLASS::reload() NOEXCEPT return error::reload_locked; } +TEMPLATE +void CLASS::prepare(size_t STAGING_ONLY(offset), + size_t STAGING_ONLY(size)) NOEXCEPT +{ +#if defined(MANAGE_STAGING) + if (is_zero(size) || !dirty_ || !engaged_.load(relaxed)) + return; + + // Declare intent before the write (sequentially consistent, pairing with + // the release protocol), then restore any released page in the range. + auto restore = false; + auto page = offset / page_; + const auto end = (offset + sub1(size)) / page_; + while ((page <= end) && ((page / page_bound) < words_)) + { + const auto word = page / page_bound; + const auto flag = system::bit_right(page % page_bound); + intent_[word].fetch_or(flag); + restore |= !is_zero(system::bit_and(released_[word].load(), flag)); + ++page; + } + + if (restore) + restore_(offset, size); +#endif +} + TEMPLATE void CLASS::mark(size_t STAGING_ONLY(offset), size_t STAGING_ONLY(size)) NOEXCEPT diff --git a/include/bitcoin/database/impl/primitives/arrayhead.ipp b/include/bitcoin/database/impl/primitives/arrayhead.ipp index a23bcf353..1f9b11f34 100644 --- a/include/bitcoin/database/impl/primitives/arrayhead.ipp +++ b/include/bitcoin/database/impl/primitives/arrayhead.ipp @@ -71,6 +71,7 @@ bool CLASS::clear() NOEXCEPT // count to zero, which is picked up in arraymap::reset(). Body file size // remains unchanged and subject to initialization size at each startup. So // there is no reduction until restart, which can include config change. + file_.prepare(zero, size()); std::fill_n(ptr.data(), size(), system::bit_all); file_.mark(zero, size()); return set_body_count(zero); @@ -120,6 +121,7 @@ bool CLASS::set_body_count(const Link& count) NOEXCEPT // Body count is written as the first value in link size, but since // offsetting is a multiple of cell size, a full cell is consumed for it. // In case of nomap or disabled there are no cells, so file is link size. + file_.prepare(zero, Link::size); to_array(ptr.data()) = count; file_.mark(zero, Link::size); return true; @@ -174,6 +176,7 @@ bool CLASS::push(const Link& link, const Link& index) NOEXCEPT if (!ptr) return false; + file_.prepare(position, bucket_size); if constexpr (aligned) { // Writes full padded word (0x00 fill). diff --git a/include/bitcoin/database/impl/primitives/hashhead.ipp b/include/bitcoin/database/impl/primitives/hashhead.ipp index 9b65628e7..1f2ca0c0d 100644 --- a/include/bitcoin/database/impl/primitives/hashhead.ipp +++ b/include/bitcoin/database/impl/primitives/hashhead.ipp @@ -72,6 +72,7 @@ bool CLASS::create() NOEXCEPT // std::memset/fill_n have identical performance (on win32). ////std::memset(ptr.data(), system::bit_all, allocation); + file_.prepare(start, allocation); std::fill_n(ptr.data(), allocation, system::bit_all); file_.mark(start, allocation); return set_body_count(zero); @@ -108,6 +109,7 @@ bool CLASS::set_body_count(const Link& count) NOEXCEPT // offsetting is a multiple of sell size, a full cell is consumed for it. // In case of disabled there are no cells, so file is link size. auto value = count.value; + file_.prepare(zero, Link::size); link_array(ptr.data()) = link_array(value); file_.mark(zero, Link::size); return true; @@ -202,6 +204,7 @@ inline bool CLASS::set_cell(bool& collision, bytes& next, const Link& current, if (is_null(raw)) return false; + file_.prepare(position, cell_size); const auto entropy = keys::thumb(key); if constexpr (aligned) { diff --git a/include/bitcoin/database/impl/primitives/nohead.ipp b/include/bitcoin/database/impl/primitives/nohead.ipp index 235b2953a..d823311a6 100644 --- a/include/bitcoin/database/impl/primitives/nohead.ipp +++ b/include/bitcoin/database/impl/primitives/nohead.ipp @@ -63,6 +63,7 @@ bool CLASS::clear() NOEXCEPT // count to zero, which is picked up in arraymap::reset(). Body file size // remains unchanged and subject to initialization size at each startup. So // there is no reduction until restart, which can include config change. + file_.prepare(zero, size()); std::fill_n(ptr.data(), size(), system::bit_all); file_.mark(zero, size()); return set_body_count(zero); @@ -112,6 +113,7 @@ bool CLASS::set_body_count(const Link& count) NOEXCEPT // Body count is written as the first value in link size, but since // offsetting is a multiple of cell size, a full cell is consumed for it. // In case of nomap or disabled there are no cells, so file is link size. + file_.prepare(zero, Link::size); to_array(ptr.data()) = count; file_.mark(zero, Link::size); return true; diff --git a/include/bitcoin/database/memory/interfaces/storage.hpp b/include/bitcoin/database/memory/interfaces/storage.hpp index f74d945aa..541fd81b4 100644 --- a/include/bitcoin/database/memory/interfaces/storage.hpp +++ b/include/bitcoin/database/memory/interfaces/storage.hpp @@ -58,6 +58,12 @@ class storage /// Clear disk full condition, fails if fault, must be loaded, idempotent. virtual code reload() NOEXCEPT = 0; + /// Declare content mutation of size bytes at offset (rewritable tables), + /// before the write. Restores released pages; no effect where unneeded. + virtual void prepare(size_t, size_t) NOEXCEPT + { + } + /// Report content mutation of size bytes at offset (rewritable tables), /// after the write. Advisory dirty tracking; no effect where unneeded. virtual void mark(size_t, size_t) NOEXCEPT diff --git a/include/bitcoin/database/memory/mmap.hpp b/include/bitcoin/database/memory/mmap.hpp index a9ae1dca0..23f1c3d56 100644 --- a/include/bitcoin/database/memory/mmap.hpp +++ b/include/bitcoin/database/memory/mmap.hpp @@ -118,6 +118,10 @@ class mmap /// Clear disk full condition, fails if fault, must be loaded, idempotent. code reload() NOEXCEPT override; + /// Declare content mutation, restoring released pages (unstaged + /// instances under the staging backend only; no effect otherwise). + void prepare(size_t offset, size_t size) NOEXCEPT override; + /// Report content mutation (advisory page-dirty tracking, unstaged /// instances under the staging backend only; no effect otherwise). void mark(size_t offset, size_t size) NOEXCEPT override; @@ -289,6 +293,11 @@ class mmap template bool sync_() NOEXCEPT; + // head page release (unstaged instances), synchronized with writers by + // the prepare/release bit protocol (see release_pages_). + bool release_pages_() NOEXCEPT; + void restore_(size_t offset, size_t size) NOEXCEPT; + // settle scheduler (instance-owned thread, load/unload lifecycle). void settler_start_() NOEXCEPT; void settler_stop_() NOEXCEPT; @@ -361,8 +370,16 @@ class mmap // These are protected by remap_mutex_. std::unique_ptr dirty_{}; + std::unique_ptr intent_{}; + std::unique_ptr released_{}; size_t words_{}; + // Set when the first head page releases (gates the prepare fast path). + std::atomic_bool engaged_{}; + + // Serializes page release against restore (prepare slow path). + mutable std::mutex restore_mutex_{}; + // These are protected by extent_mutex_. size_t page_{}; std::array ring_{}; diff --git a/include/bitcoin/database/memory/mstage.hpp b/include/bitcoin/database/memory/mstage.hpp index 36068f0b6..09678b58f 100644 --- a/include/bitcoin/database/memory/mstage.hpp +++ b/include/bitcoin/database/memory/mstage.hpp @@ -53,6 +53,10 @@ int mmap_unsettle(void* address, size_t size) NOEXCEPT; /// Release cached pages of a settled range (writes back any dirty first). int mmap_evict(void* address, size_t size) NOEXCEPT; +/// Atomically replace a released (read-only file-backed) range with writable +/// anonymous memory carrying its content (readers never observe zeros). +int mmap_restore(void* address, size_t size) NOEXCEPT; + /// Full-transfer positional file read/write (false on failure or early eof). bool pread_all(int fd, uint8_t* to, size_t size, size_t offset) NOEXCEPT; bool pwrite_all(int fd, const uint8_t* from, size_t size, diff --git a/src/memory/mstage.cpp b/src/memory/mstage.cpp index 0751fd473..8271585ae 100644 --- a/src/memory/mstage.cpp +++ b/src/memory/mstage.cpp @@ -18,6 +18,7 @@ */ #include #include +#include #include #include @@ -26,6 +27,10 @@ #include #include #include +#if defined(HAVE_APPLE) + #include + #include +#endif using namespace libbitcoin; using namespace libbitcoin::system; @@ -61,6 +66,46 @@ int mmap_evict(void* address, size_t size) NOEXCEPT return ::msync(address, size, MS_SYNC | MS_INVALIDATE); } +// Atomically install the source anonymous mapping over the target range, +// consuming the source (readers see old or new bytes, never zeros). +static int mmap_install(void* address, void* source, size_t size) NOEXCEPT +{ +#if defined(HAVE_APPLE) + auto target = reinterpret_cast(address); + vm_prot_t current{}; + vm_prot_t maximum{}; + if (::mach_vm_remap(::mach_task_self(), &target, size, 0, + VM_FLAGS_FIXED | VM_FLAGS_OVERWRITE, ::mach_task_self(), + reinterpret_cast(source), TRUE, ¤t, + &maximum, VM_INHERIT_DEFAULT) != KERN_SUCCESS) + return -1; + + return ::munmap(source, size); +#else + return ::mremap(source, size, size, MREMAP_MAYMOVE | MREMAP_FIXED, + address) == MAP_FAILED ? -1 : 0; +#endif +} + +int mmap_restore(void* address, size_t size) NOEXCEPT +{ + // Anonymous staging copy of the current (file) content. + const auto source = ::mmap(nullptr, size, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + + if (source == MAP_FAILED) + return -1; + + std::memcpy(source, address, size); + if (mmap_install(address, source, size) == -1) + { + ::munmap(source, size); + return -1; + } + + return 0; +} + bool pread_all(int fd, uint8_t* to, size_t size, size_t offset) NOEXCEPT { while (!is_zero(size)) From 81e77bde64d742e6a940f38183f2c1b7539f34c5 Mon Sep 17 00:00:00 2001 From: Eric Voskuil Date: Fri, 31 Jul 2026 10:44:10 -0400 Subject: [PATCH 2/5] Posix memory map control head release granularity. --- include/bitcoin/database/impl/memory/mmap_staging.ipp | 7 ++++--- include/bitcoin/database/memory/mmap.hpp | 5 +++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/include/bitcoin/database/impl/memory/mmap_staging.ipp b/include/bitcoin/database/impl/memory/mmap_staging.ipp index 3a2db742e..e5593cb1a 100644 --- a/include/bitcoin/database/impl/memory/mmap_staging.ipp +++ b/include/bitcoin/database/impl/memory/mmap_staging.ipp @@ -884,8 +884,9 @@ void CLASS::head_run_() NOEXCEPT continue; const auto engaged = engaged_.load(); - if (scarcity && !engaged) - engaged_.store(true); + if constexpr (head_release) + if (scarcity && !engaged) + engaged_.store(true); if (!transfer_(to_width(logical_.load())) || !sync_()) @@ -942,7 +943,7 @@ bool CLASS::release_pages_() NOEXCEPT auto bits = bit_not(bit_or(hot, bit_or(dirt, done))); const auto first = word * page_bound; if (pages < (first + page_bound)) - bits = bit_and(bits, mask_right(pages - first)); + bits = bit_and(bits, unmask_right(pages - first)); for (size_t bit{}; !is_zero(bits) && (bit < page_bound); ++bit) { diff --git a/include/bitcoin/database/memory/mmap.hpp b/include/bitcoin/database/memory/mmap.hpp index 23f1c3d56..d0cb94240 100644 --- a/include/bitcoin/database/memory/mmap.hpp +++ b/include/bitcoin/database/memory/mmap.hpp @@ -221,6 +221,11 @@ class mmap static constexpr size_t active_factor = 32; static constexpr size_t urgent_factor = 4; static constexpr size_t idle_seconds = 60; + + // Page-granular release fragments the address space beyond what host + // memory management tolerates (vma/vm_map_entry explosion); disabled + // pending run-granular conversion. + static constexpr bool head_release = false; static constexpr size_t headroom = 4; #if defined(STAGING_TELEMETRY) static constexpr size_t telemetry_seconds = 60; From f45ca9b947b3cab700bbf3b817356b030fcdb5d2 Mon Sep 17 00:00:00 2001 From: Eric Voskuil Date: Fri, 31 Jul 2026 16:55:54 -0400 Subject: [PATCH 3/5] Posix memory map fix recent race and regression. --- .../database/impl/memory/mmap_staging.ipp | 22 +++++++++++++++++-- include/bitcoin/database/memory/mmap.hpp | 5 +++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/include/bitcoin/database/impl/memory/mmap_staging.ipp b/include/bitcoin/database/impl/memory/mmap_staging.ipp index e5593cb1a..0ff85f0b2 100644 --- a/include/bitcoin/database/impl/memory/mmap_staging.ipp +++ b/include/bitcoin/database/impl/memory/mmap_staging.ipp @@ -647,6 +647,9 @@ bool CLASS::transfer_(size_t bytes) NOEXCEPT if (is_zero(bytes)) return true; + // One pass at a time: concurrent passes split the claimed dirty set. + std::unique_lock transfer_lock(transfer_mutex_); + // Untracked (multi-column unstaged) instances transfer in full. if (!dirty_) return pwrite_all(opened_[Column], memory_map_[Column], bytes, zero); @@ -689,14 +692,28 @@ bool CLASS::transfer_(size_t bytes) NOEXCEPT } if (!write()) + { + // Restore claimed marks (the failed range, the page that + // ended it, and this word's unwritten remainder) so failure + // is retryable, not lossy. + mark(from, to - from); + mark(start, end - start); + dirty_[word].fetch_or(bits, relaxed); return false; + } from = start; to = end; } } - return write(); + if (!write()) + { + mark(from, to - from); + return false; + } + + return true; } // Durability barrier for the column file. @@ -872,7 +889,8 @@ void CLASS::head_run_() NOEXCEPT still = (top == mark) ? std::min(add1(still), idle_seconds) : zero; mark = top; - const auto scarcity = dirty_ && (system_free() < scarce); + const auto scarcity = head_release && dirty_ && + (system_free() < scarce); if (!scarcity && ((still < idle_seconds) || (transferred == top))) continue; diff --git a/include/bitcoin/database/memory/mmap.hpp b/include/bitcoin/database/memory/mmap.hpp index d0cb94240..9d0c046ba 100644 --- a/include/bitcoin/database/memory/mmap.hpp +++ b/include/bitcoin/database/memory/mmap.hpp @@ -385,6 +385,11 @@ class mmap // Serializes page release against restore (prepare slow path). mutable std::mutex restore_mutex_{}; + // Serializes transfer passes (settler tick against flush), as concurrent + // passes split the claimed dirty set, allowing a flush to complete while + // claimed pages remain unwritten (a stale snapshot copy). + mutable std::mutex transfer_mutex_{}; + // These are protected by extent_mutex_. size_t page_{}; std::array ring_{}; From 762647dbf1a8e4cb524545fd14004bad98698695 Mon Sep 17 00:00:00 2001 From: Eric Voskuil Date: Fri, 31 Jul 2026 16:56:19 -0400 Subject: [PATCH 4/5] Include heads in store fault reports. --- .../database/impl/primitives/arrayhead.ipp | 12 ++++++ .../database/impl/primitives/arraymap.ipp | 5 ++- .../database/impl/primitives/hashhead.ipp | 12 ++++++ .../database/impl/primitives/hashmap.ipp | 5 ++- .../database/impl/primitives/hashmaps.ipp | 5 ++- .../database/impl/primitives/nohead.ipp | 12 ++++++ .../database/impl/primitives/nomap.ipp | 5 ++- .../database/impl/primitives/nomaps.ipp | 5 ++- .../database/impl/store/store_report.ipp | 42 +++++++++++++++++++ .../bitcoin/database/primitives/arrayhead.hpp | 6 +++ .../bitcoin/database/primitives/hashhead.hpp | 6 +++ .../bitcoin/database/primitives/nohead.hpp | 6 +++ 12 files changed, 111 insertions(+), 10 deletions(-) diff --git a/include/bitcoin/database/impl/primitives/arrayhead.ipp b/include/bitcoin/database/impl/primitives/arrayhead.ipp index 1f9b11f34..1748dd7dc 100644 --- a/include/bitcoin/database/impl/primitives/arrayhead.ipp +++ b/include/bitcoin/database/impl/primitives/arrayhead.ipp @@ -91,6 +91,18 @@ bool CLASS::create() NOEXCEPT return clear(); } +TEMPLATE +code CLASS::get_fault() const NOEXCEPT +{ + return file_.get_fault(); +} + +TEMPLATE +size_t CLASS::get_space() const NOEXCEPT +{ + return file_.get_space(); +} + TEMPLATE bool CLASS::verify() const NOEXCEPT { diff --git a/include/bitcoin/database/impl/primitives/arraymap.ipp b/include/bitcoin/database/impl/primitives/arraymap.ipp index 9c88e2ff6..7d0258688 100644 --- a/include/bitcoin/database/impl/primitives/arraymap.ipp +++ b/include/bitcoin/database/impl/primitives/arraymap.ipp @@ -133,13 +133,14 @@ bool CLASS::expand(const Link& count) NOEXCEPT TEMPLATE code CLASS::get_fault() const NOEXCEPT { - return body_.get_fault(); + const auto ec = head_.get_fault(); + return ec ? ec : body_.get_fault(); } TEMPLATE size_t CLASS::get_space() const NOEXCEPT { - return body_.get_space(); + return system::ceilinged_add(head_.get_space(), body_.get_space()); } TEMPLATE diff --git a/include/bitcoin/database/impl/primitives/hashhead.ipp b/include/bitcoin/database/impl/primitives/hashhead.ipp index 1f2ca0c0d..d546e399e 100644 --- a/include/bitcoin/database/impl/primitives/hashhead.ipp +++ b/include/bitcoin/database/impl/primitives/hashhead.ipp @@ -78,6 +78,18 @@ bool CLASS::create() NOEXCEPT return set_body_count(zero); } +TEMPLATE +code CLASS::get_fault() const NOEXCEPT +{ + return file_.get_fault(); +} + +TEMPLATE +size_t CLASS::get_space() const NOEXCEPT +{ + return file_.get_space(); +} + TEMPLATE bool CLASS::verify() const NOEXCEPT { diff --git a/include/bitcoin/database/impl/primitives/hashmap.ipp b/include/bitcoin/database/impl/primitives/hashmap.ipp index fad23f867..d5bc8e9b1 100644 --- a/include/bitcoin/database/impl/primitives/hashmap.ipp +++ b/include/bitcoin/database/impl/primitives/hashmap.ipp @@ -137,13 +137,14 @@ size_t CLASS::negative_search_count() const NOEXCEPT TEMPLATE code CLASS::get_fault() const NOEXCEPT { - return body_.get_fault(); + const auto ec = head_.get_fault(); + return ec ? ec : body_.get_fault(); } TEMPLATE size_t CLASS::get_space() const NOEXCEPT { - return body_.get_space(); + return system::ceilinged_add(head_.get_space(), body_.get_space()); } TEMPLATE diff --git a/include/bitcoin/database/impl/primitives/hashmaps.ipp b/include/bitcoin/database/impl/primitives/hashmaps.ipp index a2ae94263..a8f2440e7 100644 --- a/include/bitcoin/database/impl/primitives/hashmaps.ipp +++ b/include/bitcoin/database/impl/primitives/hashmaps.ipp @@ -129,13 +129,14 @@ size_t CLASS::negative_search_count() const NOEXCEPT TEMPLATE code CLASS::get_fault() const NOEXCEPT { - return body_.get_fault(); + const auto ec = head_.get_fault(); + return ec ? ec : body_.get_fault(); } TEMPLATE size_t CLASS::get_space() const NOEXCEPT { - return body_.get_space(); + return system::ceilinged_add(head_.get_space(), body_.get_space()); } TEMPLATE diff --git a/include/bitcoin/database/impl/primitives/nohead.ipp b/include/bitcoin/database/impl/primitives/nohead.ipp index d823311a6..28670bee1 100644 --- a/include/bitcoin/database/impl/primitives/nohead.ipp +++ b/include/bitcoin/database/impl/primitives/nohead.ipp @@ -83,6 +83,18 @@ bool CLASS::create() NOEXCEPT return clear(); } +TEMPLATE +code CLASS::get_fault() const NOEXCEPT +{ + return file_.get_fault(); +} + +TEMPLATE +size_t CLASS::get_space() const NOEXCEPT +{ + return file_.get_space(); +} + TEMPLATE bool CLASS::verify() const NOEXCEPT { diff --git a/include/bitcoin/database/impl/primitives/nomap.ipp b/include/bitcoin/database/impl/primitives/nomap.ipp index d2a53d16a..a0c5dd0d5 100644 --- a/include/bitcoin/database/impl/primitives/nomap.ipp +++ b/include/bitcoin/database/impl/primitives/nomap.ipp @@ -145,13 +145,14 @@ memory CLASS::get_memory() const NOEXCEPT TEMPLATE code CLASS::get_fault() const NOEXCEPT { - return body_.get_fault(); + const auto ec = head_.get_fault(); + return ec ? ec : body_.get_fault(); } TEMPLATE size_t CLASS::get_space() const NOEXCEPT { - return body_.get_space(); + return system::ceilinged_add(head_.get_space(), body_.get_space()); } TEMPLATE diff --git a/include/bitcoin/database/impl/primitives/nomaps.ipp b/include/bitcoin/database/impl/primitives/nomaps.ipp index ff89f5a93..7dca1d74a 100644 --- a/include/bitcoin/database/impl/primitives/nomaps.ipp +++ b/include/bitcoin/database/impl/primitives/nomaps.ipp @@ -110,13 +110,14 @@ bool CLASS::drop() NOEXCEPT TEMPLATE code CLASS::get_fault() const NOEXCEPT { - return body_.get_fault(); + const auto ec = head_.get_fault(); + return ec ? ec : body_.get_fault(); } TEMPLATE size_t CLASS::get_space() const NOEXCEPT { - return body_.get_space(); + return system::ceilinged_add(head_.get_space(), body_.get_space()); } TEMPLATE diff --git a/include/bitcoin/database/impl/store/store_report.ipp b/include/bitcoin/database/impl/store/store_report.ipp index 4e39efbfd..4319d8329 100644 --- a/include/bitcoin/database/impl/store/store_report.ipp +++ b/include/bitcoin/database/impl/store/store_report.ipp @@ -69,26 +69,47 @@ TEMPLATE code CLASS::get_fault() const NOEXCEPT { code ec{ error::success }; + if ((ec = header_head_.get_fault())) return ec; if ((ec = header_body_.get_fault())) return ec; + if ((ec = input_head_.get_fault())) return ec; if ((ec = input_body_.get_fault())) return ec; + if ((ec = output_head_.get_fault())) return ec; if ((ec = output_body_.get_fault())) return ec; + if ((ec = ins_head_.get_fault())) return ec; if ((ec = ins_body_.get_fault())) return ec; + if ((ec = outs_head_.get_fault())) return ec; if ((ec = outs_body_.get_fault())) return ec; + if ((ec = tx_head_.get_fault())) return ec; if ((ec = tx_body_.get_fault())) return ec; + if ((ec = txs_head_.get_fault())) return ec; if ((ec = txs_body_.get_fault())) return ec; + if ((ec = candidate_head_.get_fault())) return ec; if ((ec = candidate_body_.get_fault())) return ec; + if ((ec = confirmed_head_.get_fault())) return ec; if ((ec = confirmed_body_.get_fault())) return ec; + if ((ec = strong_tx_head_.get_fault())) return ec; if ((ec = strong_tx_body_.get_fault())) return ec; + if ((ec = ecdsa_head_.get_fault())) return ec; if ((ec = ecdsa_body_.get_fault())) return ec; + if ((ec = schnorr_head_.get_fault())) return ec; if ((ec = schnorr_body_.get_fault())) return ec; + if ((ec = silent_head_.get_fault())) return ec; if ((ec = silent_body_.get_fault())) return ec; + if ((ec = duplicate_head_.get_fault())) return ec; if ((ec = duplicate_body_.get_fault())) return ec; + if ((ec = prevalid_head_.get_fault())) return ec; if ((ec = prevalid_body_.get_fault())) return ec; + if ((ec = prevout_head_.get_fault())) return ec; if ((ec = prevout_body_.get_fault())) return ec; + if ((ec = validated_bk_head_.get_fault())) return ec; if ((ec = validated_bk_body_.get_fault())) return ec; + if ((ec = validated_tx_head_.get_fault())) return ec; if ((ec = validated_tx_body_.get_fault())) return ec; + if ((ec = address_head_.get_fault())) return ec; if ((ec = address_body_.get_fault())) return ec; + if ((ec = filter_bk_head_.get_fault())) return ec; if ((ec = filter_bk_body_.get_fault())) return ec; + if ((ec = filter_tx_head_.get_fault())) return ec; if ((ec = filter_tx_body_.get_fault())) return ec; return ec; } @@ -103,26 +124,47 @@ size_t CLASS::get_space() const NOEXCEPT total = system::ceilinged_add(total, storage.get_space()); }; + space(header_head_); space(header_body_); + space(input_head_); space(input_body_); + space(output_head_); space(output_body_); + space(ins_head_); space(ins_body_); + space(outs_head_); space(outs_body_); + space(tx_head_); space(tx_body_); + space(txs_head_); space(txs_body_); + space(candidate_head_); space(candidate_body_); + space(confirmed_head_); space(confirmed_body_); + space(strong_tx_head_); space(strong_tx_body_); + space(ecdsa_head_); space(ecdsa_body_); + space(schnorr_head_); space(schnorr_body_); + space(silent_head_); space(silent_body_); + space(duplicate_head_); space(duplicate_body_); + space(prevalid_head_); space(prevalid_body_); + space(prevout_head_); space(prevout_body_); + space(validated_bk_head_); space(validated_bk_body_); + space(validated_tx_head_); space(validated_tx_body_); + space(address_head_); space(address_body_); + space(filter_bk_head_); space(filter_bk_body_); + space(filter_tx_head_); space(filter_tx_body_); return total; diff --git a/include/bitcoin/database/primitives/arrayhead.hpp b/include/bitcoin/database/primitives/arrayhead.hpp index 1089ad6e4..b85142faa 100644 --- a/include/bitcoin/database/primitives/arrayhead.hpp +++ b/include/bitcoin/database/primitives/arrayhead.hpp @@ -52,6 +52,12 @@ class arrayhead /// Clear the existing index of all links. bool clear() NOEXCEPT; + /// The first fault code recorded by the head storage. + code get_fault() const NOEXCEPT; + + /// The space required by a failed head storage allocation. + size_t get_space() const NOEXCEPT; + /// False if head file size incorrect (not thread safe). bool verify() const NOEXCEPT; diff --git a/include/bitcoin/database/primitives/hashhead.hpp b/include/bitcoin/database/primitives/hashhead.hpp index 2bd8f3d0c..c16fa7bf7 100644 --- a/include/bitcoin/database/primitives/hashhead.hpp +++ b/include/bitcoin/database/primitives/hashhead.hpp @@ -50,6 +50,12 @@ class hashhead /// Create from empty head file (not thread safe). bool create() NOEXCEPT; + /// The first fault code recorded by the head storage. + code get_fault() const NOEXCEPT; + + /// The space required by a failed head storage allocation. + size_t get_space() const NOEXCEPT; + /// False if head file size incorrect (not thread safe). bool verify() const NOEXCEPT; diff --git a/include/bitcoin/database/primitives/nohead.hpp b/include/bitcoin/database/primitives/nohead.hpp index d7f5ee5ff..79a356758 100644 --- a/include/bitcoin/database/primitives/nohead.hpp +++ b/include/bitcoin/database/primitives/nohead.hpp @@ -51,6 +51,12 @@ class nohead /// Clear the existing index of all links. bool clear() NOEXCEPT; + /// The first fault code recorded by the head storage. + code get_fault() const NOEXCEPT; + + /// The space required by a failed head storage allocation. + size_t get_space() const NOEXCEPT; + /// False if head file size incorrect (not thread safe). bool verify() const NOEXCEPT; From 002affc6686c3bb15c9c51a316c60af232c6b5ca Mon Sep 17 00:00:00 2001 From: Eric Voskuil Date: Fri, 31 Jul 2026 22:29:00 -0400 Subject: [PATCH 5/5] Posix memory map change madvise config (linux). --- .../database/impl/memory/mmap_private.ipp | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/include/bitcoin/database/impl/memory/mmap_private.ipp b/include/bitcoin/database/impl/memory/mmap_private.ipp index 8f6ffb3b9..6013b9387 100644 --- a/include/bitcoin/database/impl/memory/mmap_private.ipp +++ b/include/bitcoin/database/impl/memory/mmap_private.ipp @@ -384,19 +384,28 @@ bool CLASS::finalize_(size_t const auto max = sub1(page); const auto target = to_width(size); const auto align = bit_and(ceilinged_add(target, max), bit_not(max)); - const auto advice = random_ ? MADV_RANDOM : MADV_SEQUENTIAL; - for (size_t offset{}; offset < align; offset += advise_chunk) + // Advice is elective (normal is the kernel default) and configured from + // the read pattern (see database::advice), as advising from the write + // pattern (structural) invites fault read amplification on random reads. + // Random access preloads (small heads, avoiding initial fault stalls). + if (access_ != advice::normal) { - const auto length = std::min(advise_chunk, align - offset); - const auto start = std::next(memory_map_[Column], offset); + const auto preload = (access_ == advice::random); + const auto behavior = preload ? MADV_RANDOM : MADV_SEQUENTIAL; - if (::madvise(start, length, advice) == fail || (random_ && - ::madvise(start, length, MADV_WILLNEED) == fail)) + for (size_t offset{}; offset < align; offset += advise_chunk) { - set_first_code(error::madvise_failure); - unmap_(size); - return false; + const auto length = std::min(advise_chunk, align - offset); + const auto start = std::next(memory_map_[Column], offset); + + if (::madvise(start, length, behavior) == fail || (preload && + ::madvise(start, length, MADV_WILLNEED) == fail)) + { + set_first_code(error::madvise_failure); + unmap_(size); + return false; + } } } #endif // !HAVE_MSC && !WITHOUT_MADVISE