From b2e8fdd78da155e0abc566a2a2baf1922f9ceed7 Mon Sep 17 00:00:00 2001 From: Eric Voskuil Date: Sat, 1 Aug 2026 03:28:25 -0400 Subject: [PATCH 1/2] Fix windows mmap advise config. --- src/file/utilities.cpp | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/src/file/utilities.cpp b/src/file/utilities.cpp index 73c57cd6b..05167207a 100644 --- a/src/file/utilities.cpp +++ b/src/file/utilities.cpp @@ -28,16 +28,10 @@ #include #include -#if defined(HAVE_MSC) - #define MSC_ONLY(name) name -#else - #define MSC_ONLY(name) -#endif - -#if defined(HAVE_LINUX) - #define LINUX_ONLY(name) name +#if defined(HAVE_MSC) || defined(HAVE_LINUX) + #define ADVISED_ONLY(name) name #else - #define LINUX_ONLY(name) + #define ADVISED_ONLY(name) #endif namespace libbitcoin { @@ -299,18 +293,26 @@ code space_ex(size_t& out, const path& filename) NOEXCEPT // File descriptor functions required for memory mapping. // ---------------------------------------------------------------------------- -int open(const path& filename, bool MSC_ONLY(random), - advice LINUX_ONLY(access)) NOEXCEPT +int open(const path& filename, bool, + advice ADVISED_ONLY(access)) NOEXCEPT { const auto path = system::extended_path(filename); int file_descriptor{}; #if defined(HAVE_MSC) + // Advice is elective (unhinted is the cache manager default) and + // configured from the read pattern (see database::advice). + // Order follows the advice enumeration. + static constexpr std::array hints + { + 0, _O_RANDOM, _O_SEQUENTIAL + }; + // _wsopen_s and wstring do not throw (but are unannotated). // sets file_descriptor = -1 and errno on error. - const auto access = (random ? _O_RANDOM : _O_SEQUENTIAL); + const auto hint = hints.at(to_value(access)); ::_wsopen_s(&file_descriptor, path.c_str(), - O_RDWR | _O_BINARY | access, _SH_DENYWR, _S_IREAD | _S_IWRITE); + O_RDWR | _O_BINARY | hint, _SH_DENYWR, _S_IREAD | _S_IWRITE); #elif defined(HAVE_LINUX) // open sets errno on failure. file_descriptor = ::open(path.c_str(), O_RDWR, S_IRUSR | S_IWUSR); @@ -327,7 +329,7 @@ int open(const path& filename, bool MSC_ONLY(random), // posix_fadvise returns error on failure. const auto result = ::posix_fadvise(file_descriptor, 0, 0, - advices.at(static_cast(access))); + advices.at(to_value(access))); if (!is_zero(result)) { close(file_descriptor); From dc66fd050c46c57475ff8079b09ae8a4579c00c8 Mon Sep 17 00:00:00 2001 From: Eric Voskuil Date: Sat, 1 Aug 2026 11:17:26 -0400 Subject: [PATCH 2/2] Posix memory map throttle staging chunk size. --- include/bitcoin/database/impl/memory/mmap.ipp | 16 ++++++++++++++-- .../database/impl/memory/mmap_staging.ipp | 6 +++++- include/bitcoin/database/memory/mmap.hpp | 2 ++ 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/include/bitcoin/database/impl/memory/mmap.ipp b/include/bitcoin/database/impl/memory/mmap.ipp index f9cd1ba81..a3b8746c4 100644 --- a/include/bitcoin/database/impl/memory/mmap.ipp +++ b/include/bitcoin/database/impl/memory/mmap.ipp @@ -24,6 +24,7 @@ #include #include #include +#include namespace libbitcoin { namespace database { @@ -112,6 +113,17 @@ size_t CLASS::to_capacity(size_t required) const NOEXCEPT // this design exists to prevent, moved from create to first touch). Growth is // chunked to bound slow path frequency, and clamped so that small tables do // not over-commit (the provisioned file requires no memory until committed). +// The commit chunk is scaled to memory (bounded by commit_chunk), as the +// committed but unused overhang is otherwise up to one full chunk for every +// instance, an outsized share of a small system under memory pressure. +TEMPLATE +size_t CLASS::to_chunk() NOEXCEPT +{ + static const auto chunk = std::min(commit_chunk, + system::possible_narrow_cast(system_memory() / chunk_scale)); + return chunk; +} + TEMPLATE size_t CLASS::to_growth(size_t required) const NOEXCEPT { @@ -120,7 +132,7 @@ size_t CLASS::to_growth(size_t required) const NOEXCEPT const auto expand = ceilinged_multiply(required, expansion_) / 100u; const auto expanded = ceilinged_add(required, expand); const auto chunked = std::max(expanded, - ceilinged_add(capacity_.load(), to_rows(commit_chunk))); + ceilinged_add(capacity_.load(), to_rows(to_chunk()))); return std::min(chunked, std::max(expanded, to_provision())); #else @@ -148,7 +160,7 @@ size_t CLASS::to_commitment() const NOEXCEPT { #if defined(MANAGE_STAGING) const auto logical = logical_.load(); - return std::min(to_provision(), std::max(logical, to_rows(commit_chunk))); + return std::min(to_provision(), std::max(logical, to_rows(to_chunk()))); #else // The classic mapping is file-backed, so commitment is provisioning. return to_provision(); diff --git a/include/bitcoin/database/impl/memory/mmap_staging.ipp b/include/bitcoin/database/impl/memory/mmap_staging.ipp index 0ff85f0b2..9a6d73571 100644 --- a/include/bitcoin/database/impl/memory/mmap_staging.ipp +++ b/include/bitcoin/database/impl/memory/mmap_staging.ipp @@ -732,7 +732,11 @@ bool CLASS::sync_() NOEXCEPT TEMPLATE void CLASS::settler_start_() NOEXCEPT { - limit_ = system_memory() / throttle_factor; + // Halve the staging debt allowance on small systems, as the debt is + // otherwise an outsized share of a small system under memory pressure. + const auto memory = system_memory(); + const auto small = memory < system::power2(36u); + limit_ = memory / (small ? (2u * throttle_factor) : throttle_factor); evicted_ = zero; settling_.store(true); settler_ = std::thread([this]() NOEXCEPT diff --git a/include/bitcoin/database/memory/mmap.hpp b/include/bitcoin/database/memory/mmap.hpp index 9d0c046ba..09e5d1901 100644 --- a/include/bitcoin/database/memory/mmap.hpp +++ b/include/bitcoin/database/memory/mmap.hpp @@ -201,6 +201,7 @@ class mmap return system::ceilinged_divide(bytes, row); } + static size_t to_chunk() NOEXCEPT; size_t to_capacity(size_t required) const NOEXCEPT; size_t to_growth(size_t required) const NOEXCEPT; size_t to_provision() const NOEXCEPT; @@ -214,6 +215,7 @@ class mmap static constexpr size_t settle_chunk = system::power2(28u); static constexpr size_t advise_chunk = system::power2(30u); static constexpr size_t commit_chunk = system::power2(28u); + static constexpr size_t chunk_scale = 256; static constexpr size_t evict_chunk = system::power2(30u); static constexpr size_t compress_factor = 32; static constexpr size_t evict_factor = 32;