Posix memory map work in progress.

include/bitcoin/database/impl/memory/mmap_private.ipp

@@ -95,7 +95,10 @@ bool CLASS::unmap_all_(std::index_sequence<Index...>) NOEXCEPT
     frontier_.store(zero);
     marks_.store(zero);
     dirty_.reset();
+    intent_.reset();
+    released_.reset();
     words_ = zero;
+    engaged_.store(false);
 #endif
 
     return success;
@@ -381,19 +384,28 @@ bool CLASS::finalize_(size_t
     const auto max = sub1(page);
     const auto target = to_width<Column>(size);
     const auto align = bit_and(ceilinged_add(target, max), bit_not(max));
-    const auto advice = random_ ? MADV_RANDOM : MADV_SEQUENTIAL;
 
-    for (size_t offset{}; offset < align; offset += advise_chunk)
+    // Advice is elective (normal is the kernel default) and configured from
+    // the read pattern (see database::advice), as advising from the write
+    // pattern (structural) invites fault read amplification on random reads.
+    // Random access preloads (small heads, avoiding initial fault stalls).
+    if (access_ != advice::normal)
     {
-        const auto length = std::min(advise_chunk, align - offset);
-        const auto start = std::next(memory_map_[Column], offset);
+        const auto preload = (access_ == advice::random);
+        const auto behavior = preload ? MADV_RANDOM : MADV_SEQUENTIAL;
 
-        if (::madvise(start, length, advice) == fail || (random_ &&
-            ::madvise(start, length, MADV_WILLNEED) == fail))
+        for (size_t offset{}; offset < align; offset += advise_chunk)
         {
-            set_first_code(error::madvise_failure);
-            unmap_<Column>(size);
-            return false;
+            const auto length = std::min(advise_chunk, align - offset);
+            const auto start = std::next(memory_map_[Column], offset);
+
+            if (::madvise(start, length, behavior) == fail || (preload &&
+                ::madvise(start, length, MADV_WILLNEED) == fail))
+            {
+                set_first_code(error::madvise_failure);
+                unmap_<Column>(size);
+                return false;
+            }
         }
     }
 #endif // !HAVE_MSC && !WITHOUT_MADVISE

include/bitcoin/database/impl/memory/mmap_staging.ipp

@@ -325,6 +325,8 @@ bool CLASS::stage_() NOEXCEPT
             const auto pages = ceilinged_divide(reserved, page_);
             words_ = ceilinged_divide(pages, page_bound);
             dirty_ = std::make_unique<dirty_bitmaps>(words_);
+            intent_ = std::make_unique<dirty_bitmaps>(words_);
+            released_ = std::make_unique<dirty_bitmaps>(words_);
         }
     }
 
@@ -448,6 +450,11 @@ bool CLASS::commit_(size_t size) NOEXCEPT
 
             dirty_ = std::move(grown);
             words_ = words;
+
+            // The replacement reservation is fully anonymous (content was
+            // copied above), so released and intent page state resets.
+            intent_ = std::make_unique<dirty_bitmaps>(words);
+            released_ = std::make_unique<dirty_bitmaps>(words);
         }
     }
 
@@ -640,6 +647,9 @@ bool CLASS::transfer_(size_t bytes) NOEXCEPT
     if (is_zero(bytes))
         return true;
 
+    // One pass at a time: concurrent passes split the claimed dirty set.
+    std::unique_lock transfer_lock(transfer_mutex_);
+
     // Untracked (multi-column unstaged) instances transfer in full.
     if (!dirty_)
         return pwrite_all(opened_[Column], memory_map_[Column], bytes, zero);
@@ -682,14 +692,28 @@ bool CLASS::transfer_(size_t bytes) NOEXCEPT
             }
 
             if (!write())
+            {
+                // Restore claimed marks (the failed range, the page that
+                // ended it, and this word's unwritten remainder) so failure
+                // is retryable, not lossy.
+                mark(from, to - from);
+                mark(start, end - start);
+                dirty_[word].fetch_or(bits, relaxed);
                 return false;
+            }
 
             from = start;
             to = end;
         }
     }
 
-    return write();
+    if (!write())
+    {
+        mark(from, to - from);
+        return false;
+    }
+
+    return true;
 }
 
 // Durability barrier for the column file.
@@ -837,9 +861,16 @@ void CLASS::settler_run_() NOEXCEPT
 // before reading, so racing writes remark and transfer on the next pass
 // (torn disk pages are unreachable, as live heads are only trusted
 // following a clean close).
+//
+// Memory scarcity additionally pages the head to its own file (the windows
+// model): transfer, then release cold clean pages to read-only file mappings
+// (reclaimable cache), restored to anonymous by prepare() before any write.
+// Release waits one pass after engagement so all writers declare intent.
 TEMPLATE
 void CLASS::head_run_() NOEXCEPT
 {
+    const auto scarce = system_memory() / evict_factor;
+
     // Ticks without a mark before idle draining.
     auto mark = marks_.load();
     auto transferred = mark;
@@ -858,7 +889,10 @@ void CLASS::head_run_() NOEXCEPT
         still = (top == mark) ? std::min(add1(still), idle_seconds) : zero;
         mark = top;
 
-        if ((still < idle_seconds) || (transferred == top))
+        const auto scarcity = head_release && dirty_ &&
+            (system_free() < scarce);
+        if (!scarcity &&
+            ((still < idle_seconds) || (transferred == top)))
             continue;
 
         {
@@ -867,19 +901,142 @@ void CLASS::head_run_() NOEXCEPT
             if (!loaded_.load() || fault_.load())
                 continue;
 
+            const auto engaged = engaged_.load();
+            if constexpr (head_release)
+                if (scarcity && !engaged)
+                    engaged_.store(true);
+
             if (!transfer_<zero>(to_width<zero>(logical_.load())) ||
                 !sync_<zero>())
             {
                 set_first_code(error::fsync_failure);
                 continue;
             }
+
+            if (scarcity && engaged && !release_pages_())
+                continue;
         }
 
         transferred = top;
         still = zero;
     }
 }
 
+// Release cold clean head pages to read-only file mappings (reclaimable),
+// full pages below logical only. Writer synchronization is a per-page bit
+// protocol: prepare() declares intent then loads released; release stores
+// released then loads intent (both sequentially consistent), so a page
+// converts only when no write can land on it unrestored. A wrong release
+// costs one restore. Conversion and restore serialize on restore_mutex_.
+TEMPLATE
+bool CLASS::release_pages_() NOEXCEPT
+{
+    using namespace system;
+    const auto bytes = to_width<zero>(logical_.load());
+    const auto pages = bytes / page_;
+    const auto bound = std::min(words_, ceilinged_divide(pages, page_bound));
+
+    std::unique_lock restore_lock(restore_mutex_);
+
+    size_t from{};
+    size_t to{};
+    const auto convert = [&]() NOEXCEPT
+    {
+        return (from >= to) || (mmap_settle(
+            std::next(memory_map_[zero], from), to - from, opened_[zero],
+            from) != fail);
+    };
+
+    for (size_t word{}; word < bound; ++word)
+    {
+        // Hot mask: pages written since the previous pass. Aging clears only
+        // the snapshot bits, so a concurrent declaration on a candidate page
+        // is retained for the live rechecks below.
+        const auto hot = intent_[word].load();
+        intent_[word].fetch_and(system::bit_not(hot));
+        const auto dirt = dirty_[word].load(relaxed);
+        const auto done = released_[word].load(relaxed);
+
+        // Retain candidacy below the page bound (boundary word only).
+        auto bits = bit_not(bit_or(hot, bit_or(dirt, done)));
+        const auto first = word * page_bound;
+        if (pages < (first + page_bound))
+            bits = bit_and(bits, unmask_right<uint64_t>(pages - first));
+
+        for (size_t bit{}; !is_zero(bits) && (bit < page_bound); ++bit)
+        {
+            if (!get_right(bits, bit))
+                continue;
+
+            bits = set_right(bits, bit, false);
+            const auto flag = bit_right<uint64_t>(bit);
+            released_[word].fetch_or(flag);
+
+            // A raced intent or mark invalidates only this page's release.
+            if (!is_zero(bit_and(intent_[word].load(), flag)) ||
+                !is_zero(bit_and(dirty_[word].load(), flag)))
+            {
+                released_[word].fetch_and(bit_not(flag));
+                continue;
+            }
+
+            const auto start = (first + bit) * page_;
+            if (start == to)
+            {
+                to = start + page_;
+                continue;
+            }
+
+            if (!convert())
+            {
+                set_first_code(error::mmap_failure);
+                return false;
+            }
+
+            from = start;
+            to = start + page_;
+        }
+    }
+
+    if (!convert())
+    {
+        set_first_code(error::mmap_failure);
+        return false;
+    }
+
+    return true;
+}
+
+// Restore released pages within [offset, offset+size) to writable anonymous
+// memory (content preserved by atomic installation), before a declared write.
+TEMPLATE
+void CLASS::restore_(size_t offset, size_t size) NOEXCEPT
+{
+    using namespace system;
+    std::unique_lock restore_lock(restore_mutex_);
+
+    auto page = offset / page_;
+    const auto end = (offset + sub1(size)) / page_;
+    while ((page <= end) && ((page / page_bound) < words_))
+    {
+        const auto word = page / page_bound;
+        const auto flag = bit_right<uint64_t>(page % page_bound);
+        if (!is_zero(bit_and(released_[word].load(), flag)))
+        {
+            if (mmap_restore(std::next(memory_map_[zero], page * page_),
+                page_) == fail)
+            {
+                set_first_code(error::mmap_failure);
+                return;
+            }
+
+            released_[word].fetch_and(bit_not(flag));
+        }
+
+        ++page;
+    }
+}
+
 // Settle up to chunk completed rows: write under the shared remap lock
 // (completed extents are immutable, writers proceed), convert under a brief
 // exclusive. Durability remains a snapshot property (no sync here).

include/bitcoin/database/impl/memory/mmap_storage.ipp

@@ -187,6 +187,33 @@ code CLASS::reload() NOEXCEPT
     return error::reload_locked;
 }
 
+TEMPLATE
+void CLASS::prepare(size_t STAGING_ONLY(offset),
+    size_t STAGING_ONLY(size)) NOEXCEPT
+{
+#if defined(MANAGE_STAGING)
+    if (is_zero(size) || !dirty_ || !engaged_.load(relaxed))
+        return;
+
+    // Declare intent before the write (sequentially consistent, pairing with
+    // the release protocol), then restore any released page in the range.
+    auto restore = false;
+    auto page = offset / page_;
+    const auto end = (offset + sub1(size)) / page_;
+    while ((page <= end) && ((page / page_bound) < words_))
+    {
+        const auto word = page / page_bound;
+        const auto flag = system::bit_right<uint64_t>(page % page_bound);
+        intent_[word].fetch_or(flag);
+        restore |= !is_zero(system::bit_and(released_[word].load(), flag));
+        ++page;
+    }
+
+    if (restore)
+        restore_(offset, size);
+#endif
+}
+
 TEMPLATE
 void CLASS::mark(size_t STAGING_ONLY(offset),
     size_t STAGING_ONLY(size)) NOEXCEPT

include/bitcoin/database/impl/primitives/arrayhead.ipp

@@ -71,6 +71,7 @@ bool CLASS::clear() NOEXCEPT
     // count to zero, which is picked up in arraymap::reset(). Body file size
     // remains unchanged and subject to initialization size at each startup. So
     // there is no reduction until restart, which can include config change.
+    file_.prepare(zero, size());
     std::fill_n(ptr.data(), size(), system::bit_all<uint8_t>);
     file_.mark(zero, size());
     return set_body_count(zero);
@@ -90,6 +91,18 @@ bool CLASS::create() NOEXCEPT
     return clear();
 }
 
+TEMPLATE
+code CLASS::get_fault() const NOEXCEPT
+{
+    return file_.get_fault();
+}
+
+TEMPLATE
+size_t CLASS::get_space() const NOEXCEPT
+{
+    return file_.get_space();
+}
+
 TEMPLATE
 bool CLASS::verify() const NOEXCEPT
 {
@@ -120,6 +133,7 @@ bool CLASS::set_body_count(const Link& count) NOEXCEPT
     // Body count is written as the first value in link size, but since
     // offsetting is a multiple of cell size, a full cell is consumed for it.
     // In case of nomap or disabled there are no cells, so file is link size.
+    file_.prepare(zero, Link::size);
     to_array<Link::size>(ptr.data()) = count;
     file_.mark(zero, Link::size);
     return true;
@@ -174,6 +188,7 @@ bool CLASS::push(const Link& link, const Link& index) NOEXCEPT
     if (!ptr)
         return false;
 
+    file_.prepare(position, bucket_size);
     if constexpr (aligned)
     {
         // Writes full padded word (0x00 fill).

include/bitcoin/database/impl/primitives/arraymap.ipp

@@ -133,13 +133,14 @@ bool CLASS::expand(const Link& count) NOEXCEPT
 TEMPLATE
 code CLASS::get_fault() const NOEXCEPT
 {
-    return body_.get_fault();
+    const auto ec = head_.get_fault();
+    return ec ? ec : body_.get_fault();
 }
 
 TEMPLATE
 size_t CLASS::get_space() const NOEXCEPT
 {
-    return body_.get_space();
+    return system::ceilinged_add(head_.get_space(), body_.get_space());
 }
 
 TEMPLATE