Apply the neighbour heuristic incrementally on full back-links (1.4–2.9× faster build)

Qvec.Core/QvecDatabase.cs

@@ -1650,15 +1650,43 @@ private bool ConnectNewNode(int newIndex, float[] newVector, int newLevel)
         {
             if (candidates.Length <= m) return candidates;
 
-            var ordered = candidates.Where(c => c.Id >= 0)
-                                    .OrderByDescending(c => c.Score)
-                                    .ToArray();
+            int live = 0;
+            for (int i = 0; i < candidates.Length; i++)
+                if (candidates[i].Id >= 0) live++;
+
+            var ordered = new (int Id, float Score)[live];
+            for (int i = 0, j = 0; i < candidates.Length; i++)
+                if (candidates[i].Id >= 0) ordered[j++] = candidates[i];
+
+            // Stable, like the OrderByDescending it replaces, so equal scores keep candidate
+            // order and the resulting graph is unchanged.
+            StableSortDescending(ordered);
 
             if (ordered.Length <= m) return ordered;
 
             return PruneNeighbors(ordered, m);
         }
 
+        /// <summary>
+        /// Stable descending sort by score. Equal scores keep their input order, exactly as the
+        /// LINQ OrderByDescending this replaces, so the resulting graph is unchanged.
+        /// </summary>
+        private static void StableSortDescending((int Id, float Score)[] items)
+        {
+            var order = new int[items.Length];
+            for (int i = 0; i < order.Length; i++) order[i] = i;
+
+            Array.Sort(order, (a, b) =>
+            {
+                int byScore = items[b].Score.CompareTo(items[a].Score);
+                return byScore != 0 ? byScore : a.CompareTo(b);
+            });
+
+            var sorted = new (int Id, float Score)[items.Length];
+            for (int i = 0; i < order.Length; i++) sorted[i] = items[order[i]];
+            sorted.CopyTo(items, 0);
+        }
+
         /// <summary>
         /// Neighbour selection heuristic from Malkov &amp; Yashunin, algorithm 4.
         /// Keeps a candidate only when it is closer to the owning node than to any candidate
@@ -1983,6 +2011,15 @@ private unsafe void WriteNeighborsAtLevel(int nodeIndex, int level, int[] neighb
         /// leaving whole groups of entries with outgoing edges only — unreachable from the
         /// entry point, and therefore invisible to search.
         /// </summary>
+        /// <remarks>
+        /// The heuristic is applied incrementally, the way Lucene's HNSW graph builder does it
+        /// (<c>findWorstNonDiverse</c>), instead of re-running the full O(M0²) selection over
+        /// all M0 + 1 candidates. Only the new node is unchecked: an existing neighbour can only
+        /// have become redundant because of the new node, and the new node has to be checked
+        /// against the neighbours closer than itself. That is O(M0) distance computations per
+        /// full back-link rather than O(M0²), and on 768-dimensional vectors, where every
+        /// distance is a 3 KiB memory read, the old path was 66 % of insert time.
+        /// </remarks>
         private void AddNeighborConnection(int existingNode, int level, int newNode)
         {
             int slots = NeighborsAtLevel(level);
@@ -2003,9 +2040,8 @@ private void AddNeighborConnection(int existingNode, int level, int newNode)
                     if (neighbors[i] == newNode) return;
                 }
 
-                // The list is full: re-run the selection heuristic over the existing neighbours
-                // plus the new node, all read in place from the mapped file. The new node's
-                // vector is already on disk at this point, so it needs no special casing.
+                // The list is full. Scores are recomputed because the graph section stores ids
+                // only; the new node's vector is already on disk, so it needs no special casing.
                 int candidateCount = slots + 1;
                 var candidates = new (int Id, float Score)[candidateCount];
 
@@ -2014,20 +2050,68 @@ private void AddNeighborConnection(int existingNode, int level, int newNode)
 
                 candidates[slots] = (newNode, StoredSimilarity(existingNode, newNode));
 
-                // Sorted by descending similarity for the pruning pass. Array.Sort runs the same
-                // introsort over the same comparison outcomes as the index-array sort it
-                // replaces, so tie ordering, and therefore the graph, is unchanged.
                 Array.Sort(candidates, DescendingScore.Instance);
 
-                var selected = PruneNeighbors(candidates, slots);
-                WriteNeighborsAtLevel(existingNode, level, selected);
+                int newPosition = 0;
+                while (candidates[newPosition].Id != newNode) newPosition++;
+                int evict = FindWorstNonDiverse(candidates, newPosition);
+
+                if (evict < 0)
+                {
+                    // Nothing is redundant because of the new node. The list may still hold
+                    // neighbours that were topped up past the diversity check when it was
+                    // built (keepPrunedConnections), and those are the ones to give up before
+                    // a diverse long-range link. Only the full heuristic can tell them apart.
+                    WriteNeighborsAtLevel(existingNode, level, PruneNeighbors(candidates, slots));
+                    return;
+                }
+
+                // Rejecting the new node leaves the stored list exactly as it was.
+                if (evict == newPosition) return;
+
+                for (int i = 0, j = 0; i < candidateCount; i++)
+                    if (i != evict) neighbors[j++] = candidates[i].Id;
+
+                WriteNeighborsAtLevel(existingNode, level, neighbors);
             }
             finally
             {
                 ArrayPool<int>.Shared.Return(neighbors);
             }
         }
 
+        /// <summary>
+        /// Walks the candidates from farthest to nearest and returns the index of the first one
+        /// that is closer to the new node than it is to the owner — the candidate the selection
+        /// heuristic would discard because of the insertion. For the new node itself every
+        /// nearer candidate is checked. Returns -1 when the new node makes nothing redundant.
+        /// </summary>
+        /// <param name="candidates">Sorted by descending similarity to the owner.</param>
+        /// <param name="newPosition">Index of the newly inserted node; the only candidate the existing neighbours were never checked against.</param>
+        private int FindWorstNonDiverse((int Id, float Score)[] candidates, int newPosition)
+        {
+            for (int i = candidates.Length - 1; i > 0; i--)
+            {
+                var candidate = candidates[i];
+
+                if (i == newPosition)
+                {
+                    for (int j = 0; j < i; j++)
+                    {
+                        if (StoredSimilarity(candidate.Id, candidates[j].Id) > candidate.Score)
+                            return i;
+                    }
+                }
+                else if (newPosition < i)
+                {
+                    if (StoredSimilarity(candidate.Id, candidates[newPosition].Id) > candidate.Score)
+                        return i;
+                }
+            }
+
+            return -1;
+        }
+
         private sealed class DescendingScore : IComparer<(int Id, float Score)>
         {
             public static readonly DescendingScore Instance = new();

README.md

@@ -34,14 +34,14 @@ Measured on **SIFT-1M** ([TexMex corpus](http://corpus-texmex.irisa.fr/)) agains
 
 | efSearch | recall@1 | recall@10 | QPS | mean latency |
 | ---: | ---: | ---: | ---: | ---: |
-| 10 | 89.7 % | 85.9 % | 8,110 | 0.123 ms |
-| 20 | 95.3 % | 93.7 % | 5,128 | 0.195 ms |
-| 40 | 98.2 % | 97.8 % | 3,231 | 0.310 ms |
-| 80 | 99.0 % | 99.4 % | 1,873 | 0.534 ms |
-| 160 | 99.2 % | 99.8 % | 1,023 | 0.978 ms |
-| 320 | 99.2 % | 99.9 % | 573 | 1.744 ms |
+| 10 | 88.0 % | 84.4 % | 6,734 | 0.148 ms |
+| 20 | 94.5 % | 92.5 % | 4,341 | 0.230 ms |
+| 40 | 97.6 % | 97.0 % | 2,624 | 0.381 ms |
+| 80 | 98.8 % | 99.0 % | 1,576 | 0.634 ms |
+| 160 | 99.3 % | 99.7 % | 863 | 1.158 ms |
+| 320 | 99.3 % | 99.9 % | 498 | 2.010 ms |
 
-Index build: 2,762 s (362 inserts/s), producing a 1,324 MiB file, with `indexSeed` pinned so the run can be reproduced. Build throughput is still the weakest number here: the insert path is now dominated by the O(M0²) distance arithmetic of the neighbour-selection heuristic and by memory latency once the file outgrows the CPU caches. Query performance is not affected by it.
+Index build: 1,631 s (613 inserts/s), producing a 1,324 MiB file, with `indexSeed` pinned so the run can be reproduced. The previous graph, built with the full O(M0²) neighbour heuristic on every back-link, took 2,279 s on the same day and machine and scored 85.9 / 93.7 / 97.8 / 99.4 / 99.8 / 99.9 % recall@10 on the same rows; the incremental heuristic ([design doc](docs/design-insert-prune.md)) trades up to 1.5 points of recall at the narrowest beam, and nothing from efSearch 160 up, for the faster build. Query throughput of the two graphs is identical within run-to-run noise (±5 %). Build is still single-threaded and memory-bound once the file outgrows the CPU caches.
 
 A single recall figure would be misleading, because any ANN index reaches 99% by widening the beam until it has effectively scanned everything. The honest unit is the whole curve, so pick the row that matches your latency budget.
 
@@ -319,7 +319,7 @@ Planned cloud work is tracked in design documents and the roadmap below.
 - **Full Native AOT support for the typed client** — Remove or replace reflection, expression compilation, and reflection-based JSON paths.
 - **ProjectReference analyzer flow for source generation** — Ensure the `[QvecIndexed]` generator is available when consuming `Qvec.Core.Client` through project references.
 - **Published benchmark methodology** — ✅ Done. `benchmarks/Qvec.Benchmarks` measures recall vs. QPS against the TexMex SIFT/GIST corpora and their published ground truth.
-- **Faster index construction** — Partly done: removing marshalling, pool and allocation overhead from the insert path took SIFT-1M from 242 to 362 inserts/s and doubled query throughput. What remains is algorithmic (the O(M0²) neighbour heuristic) and memory-bound; parallel construction and multi-accumulator SIMD kernels are the next candidates.
+- **Faster index construction** — Partly done. Removing marshalling, pool and allocation overhead took SIFT-1M from 242 to 362 inserts/s; replacing the O(M0²) re-run of the neighbour heuristic on every full back-link with an incremental O(M0) update ([design doc](docs/design-insert-prune.md)) took Cohere 100K (768 dims) from 59 to 170 inserts/s. Insert is still single-threaded; parallel construction is the next lever.
 - **Multi-vector support** — Store and search multiple embeddings, such as image + text, for one logical entry.
 
 ## License

benchmarks/README.md

@@ -74,6 +74,29 @@ VectorDBBench's index build is parallel, while Qvec's insert path is single-thre
 build time is not comparable at all — it is the honest measurement of where Qvec stands, not a
 like-for-like number.
 
+### Cohere 100K, measured
+
+`--dataset cohere100k --k 100`, `maxNeighbors = 32`, 12 logical cores, Windows 11. Build is
+single-threaded; queries at `--concurrency 12`.
+
+| mode | build | inserts/s | file | efSearch | recall@100 | QPS 1 thread | QPS 12 threads |
+| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
+| float | 589 s | 170 | 382 MiB | 100 | 97.6 % | 381 | 1,652 |
+| float | | | | 180 | 99.0 % | 263 | 1,097 |
+| int8 | 672 s | 149 | 164 MiB | 100 | 95.7 % | 549 | — |
+| int8 | | | | 180 | 96.6 % | 395 | — |
+| int8 | | | | 300 | 96.8 % | 308 | — |
+
+The float row is the incremental-prune build ([design doc](../docs/design-insert-prune.md)).
+Before it the same build took 1,696 s (59 inserts/s) and reached 97.9 % / 99.3 % at efSearch
+100 / 180 — the graph is not byte-identical, so recall is re-measured rather than assumed.
+Absolute QPS on this laptop varies by up to 40 % between sessions (the same index file gave
+585 and 347 QPS single-threaded on two different days), so compare QPS only within one table
+measured back to back; the design doc does that for old versus new graph. The int8 row was
+built with the old prune and shows the other open item plainly: without rescoring on the
+floats, int8 recall@100 plateaus at 96.8 % on Cohere, which is why Zvec's published figures
+use int8 *with* a refiner.
+
 ## Metric
 
 SIFT and GIST ground truth is **Euclidean**, Cohere is **Cosine**. The benchmark defaults to the

docs/design-insert-prune.md

@@ -0,0 +1,122 @@
+# Design: incremental neighbour pruning on insert
+
+## Problem
+
+Every insert into the HNSW graph creates up to M0 = 2·M back-links on layer 0: each of the
+new node's neighbours gets the new node appended to its own list. Once the graph has warmed up
+that list is almost always full, and the original code handled a full list by re-running the
+complete selection heuristic (Malkov & Yashunin, algorithm 4) over all M0 + 1 candidates:
+
+1. recompute the owner's similarity to every existing neighbour (M0 distances),
+2. sort,
+3. for each candidate in order, compare it against every already-selected candidate
+   (up to M0² / 2 distances).
+
+With M = 32 that is ~2,000 distance computations per full back-link and ~130,000 per insert.
+Each `StoredSimilarity` reads two vectors from the memory-mapped file, so at 768 dimensions
+(3 KiB per vector) an insert moved roughly 400 MB through the memory hierarchy. Profiling the
+Cohere 100K build with `dotnet-trace` put 66 % of insert time in `PruneNeighbors` and gave
+**59 inserts/s**, 1,696 s for 100K vectors.
+
+## Change
+
+`AddNeighborConnection` now applies the heuristic incrementally, the way Lucene's HNSW graph
+builder does in `findWorstNonDiverse`:
+
+1. Recompute the owner's similarity to the existing neighbours and the new node (M0 + 1
+   distances — unavoidable because the graph section stores ids only), and sort descending.
+2. Walk the candidates from farthest to nearest. A candidate is *non-diverse* if it is closer
+   to some nearer candidate than to the owner. Only the new node is *unchecked*, so:
+   - an existing neighbour needs to be compared against the new node only (1 distance), and
+     only if the new node is nearer than it;
+   - the new node itself is compared against every candidate nearer than it (≤ M0 distances).
+3. Evict the first non-diverse candidate found. If that is the new node, the stored list is
+   left untouched.
+
+Typical cost is O(M0) distances instead of O(M0²).
+
+### The fallback
+
+Lucene can stop there and evict the farthest candidate when nothing is non-diverse, because
+its lists only ever contain mutually diverse nodes. Qvec's lists do not: `SelectNeighborsHeuristic`
+tops an under-filled result up with the best discarded candidates (`keepPrunedConnections`),
+so a full list can hold neighbours that already fail the diversity test against each other.
+
+Evicting the farthest candidate in that situation drops a diverse long-range link in favour of
+a redundant short one. On a 200-vector Euclidean test set with widely varying magnitudes that
+was enough to leave seven nodes with in-degree zero — unreachable from the entry point — and
+fail `Search_WithEuclidean_ReturnsAnExactlyStoredVectorFirst`.
+
+So when the incremental walk finds nothing, the code falls back to the full heuristic
+(`PruneNeighbors`) over the M0 + 1 candidates, which is exactly the old behaviour and does
+tell the topped-up neighbours apart. Instrumented on clustered vectors the fallback fires in
+0.1 % (M = 32, 128 dims) to 0.7 % (M = 16, 64 dims) of full back-links, so it costs almost
+nothing and keeps the graph connected in the cases the incremental rule cannot judge.
+
+### Also
+
+`SelectNeighborsHeuristic` no longer uses LINQ (`Where` / `OrderByDescending` / `ToArray`); a
+stable index sort replaces it with identical ordering, so the forward selection produces the
+same graph as before.
+
+## What is not identical
+
+The graph is **not** byte-identical to the one the old code built. The old full re-run evicted
+the worst candidate that was non-diverse against the *selected set*; the incremental rule
+evicts the worst candidate that is non-diverse against the *new node*. Both are legitimate
+readings of the heuristic, but they can pick different victims, so recall had to be re-measured
+rather than proved unchanged.
+
+## Measurements
+
+Same machine (12 logical cores, Windows 11), same seeds, `maxNeighbors = 32`.
+
+| dataset | build before | build after | inserts/s before → after |
+| --- | ---: | ---: | ---: |
+| siftsmall (10K × 128) | ~7–10 s | 4.0 s | ~1,000–1,500 → 2,485 |
+| Cohere 100K (768, cosine) | 1,696 s | 589 s | 59 → 170 (2.9×) |
+| SIFT-1M (128, euclidean), same day | 2,279 s | 1,631 s | 439 → 613 (1.4×) |
+
+The gain is largest where a distance computation is most expensive: at 768 dimensions the
+O(M0²) pass was two thirds of the insert; at 128 dimensions, cache-resident, it was a smaller
+share and the search phase dominates.
+
+Recall, Cohere 100K, k = 100, both index files measured back to back with `--reuse-index`:
+
+| efSearch | recall@100 before | recall@100 after | QPS before | QPS after |
+| ---: | ---: | ---: | ---: | ---: |
+| 100 | 97.89 % | 97.60 % | 347 | 381 |
+| 180 | 99.29 % | 99.03 % | 247 | 263 |
+
+Recall, SIFT-1M, k = 10, both graphs rebuilt the same day and measured back to back twice:
+
+| efSearch | recall@10 before | recall@10 after | Δ | QPS before / after |
+| ---: | ---: | ---: | ---: | ---: |
+| 10 | 85.94 % | 84.44 % | −1.50 | 6,679–7,016 / 6,509–6,734 |
+| 20 | 93.72 % | 92.48 % | −1.24 | 3,951–4,101 / 4,252–4,341 |
+| 40 | 97.81 % | 97.03 % | −0.78 | 2,556–2,644 / 2,561–2,624 |
+| 80 | 99.35 % | 99.05 % | −0.30 | 1,488–1,573 / 1,456–1,576 |
+| 160 | 99.82 % | 99.73 % | −0.09 | 831–853 / 824–863 |
+| 320 | 99.91 % | 99.90 % | −0.01 | 470–475 / 484–498 |
+
+So the new graph is slightly less well connected at the narrowest beams — up to 1.5 points on
+SIFT-1M, 0.3 on Cohere — and indistinguishable from efSearch 160 up. Query throughput at a
+given efSearch is the same within run-to-run noise on both datasets. The recall/QPS frontier
+moves by roughly one efSearch step at the low end; a caller who needs the old recall at
+efSearch 10 gets it at efSearch 20 for ~35 % fewer QPS, and a caller at efSearch ≥ 80 sees no
+difference. siftsmall recall is unchanged (100 % recall@10 from efSearch 100).
+
+## Tests
+
+`NeighborDiversityTests` pins the required *outcome* of a full-list back-link on hand-checkable
+2-D and 3-D Euclidean constructions — new node rejected when redundant, the neighbour it makes
+redundant evicted, farthest evicted when everyone is diverse — plus a well-formedness sweep
+(no duplicates, no self-links, valid ids, no gaps) over clustered data under all three
+distance functions. The tests pass against both the old and the new implementation, which is
+the point: they describe the heuristic, not the shortcut.
+
+## Next
+
+Insert is still single-threaded. With the per-insert work now dominated by the search phase
+and M0 + 1 owner-score recomputations, parallel insert (per-node locks on neighbour lists, as
+in hnswlib) is the next lever, followed by measuring Cohere 1M.