Cohere 1M measurement and fix for k=100 concurrent-search scaling

Qvec.Core/QvecDatabase.cs

@@ -2737,11 +2737,16 @@ private static unsafe float DotProductUnsafe(float[] left, float* right, int dim
             for (; i < dim; i++) dot += left[i] * right[i];
             return dot;
         }
-        private string GetMetadata(int index)
+        // Reads through the raw mapping pointer rather than the accessor: every accessor call
+        // takes an interlocked ref on the shared SafeBuffer, and a top-100 result set makes a
+        // few hundred of them per query, which serialised concurrent searches on that one
+        // cache line (Cohere 1M, k = 100, 12 threads scaled 2× instead of 6×).
+        private unsafe string GetMetadata(int index)
         {
             long descriptorPos = (_metadataSectionOffset - HeaderSize) + (long)index * MetadataDescriptorSize;
-            long heapOffset = _dataAccessor.ReadInt64(descriptorPos);
-            int length = _dataAccessor.ReadInt32(descriptorPos + 8);
+            byte* descriptor = DataBasePointer + descriptorPos;
+            long heapOffset = System.Runtime.CompilerServices.Unsafe.ReadUnaligned<long>(descriptor);
+            int length = System.Runtime.CompilerServices.Unsafe.ReadUnaligned<int>(descriptor + 8);
 
             if (length <= 0) return string.Empty;
 
@@ -2752,16 +2757,7 @@ private string GetMetadata(int index)
                     $"heap capacity={_header.MetadataHeapCapacity}.");
             }
 
-            byte[] buffer = ArrayPool<byte>.Shared.Rent(length);
-            try
-            {
-                _dataAccessor.ReadArray((_metadataHeapOffset - HeaderSize) + heapOffset, buffer, 0, length);
-                return Encoding.UTF8.GetString(buffer, 0, length);
-            }
-            finally
-            {
-                ArrayPool<byte>.Shared.Return(buffer);
-            }
+            return Encoding.UTF8.GetString(DataBasePointer + (_metadataHeapOffset - HeaderSize) + heapOffset, length);
         }
 
         private void WriteGuidToDisk(int index, Guid guid)
@@ -2772,13 +2768,10 @@ private void WriteGuidToDisk(int index, Guid guid)
             _dataAccessor.WriteArray(offset, bytes, 0, GuidSize);
         }
 
-        private Guid ReadGuidFromDisk(int index)
+        private unsafe Guid ReadGuidFromDisk(int index)
         {
-            Span<byte> bytes = stackalloc byte[GuidSize];
             long offset = (_guidSectionOffset - HeaderSize) + (long)index * GuidSize;
-            for (int i = 0; i < GuidSize; i++)
-                bytes[i] = _dataAccessor.ReadByte(offset + i);
-            return new Guid(bytes);
+            return new Guid(new ReadOnlySpan<byte>(DataBasePointer + offset, GuidSize));
         }
 
         private void RebuildGuidIndex()

README.md

@@ -50,6 +50,17 @@ A single recall figure would be misleading, because any ANN index reaches 99% by
 
 Reproduce with `dotnet run -c Release --project benchmarks/Qvec.Benchmarks -- --dataset sift --download`. See [benchmarks/README.md](benchmarks/README.md).
 
+### Cohere 1M under concurrent load
+
+The configuration Zvec publishes for Cohere 1M (768 dims, cosine, recall@100, `M = 15`, `efSearch = 180`, 12 concurrent clients), on the same 12-core laptop:
+
+| mode | build (12 threads) | file | recall@100 | QPS 1 thread | QPS 12 threads |
+| --- | ---: | ---: | ---: | ---: | ---: |
+| float | 419 s | 3,376 MiB | 94.8 % | 565 | 3,764 |
+| int8 | 181 s | 1,194 MiB | 93.2 % | 839 | 6,841 |
+
+The full sweep, the single-threaded rows and what can and cannot be read into a comparison with Zvec's 16-vCPU figures are in [benchmarks/README.md](benchmarks/README.md#cohere-1m-measured).
+
 ### int8 quantization on siftsmall
 
 Same code, `--dataset siftsmall` (10,000 vectors, 128 dimensions, 100 queries), float versus `--quantization int8`, same seed:

benchmarks/Qvec.Benchmarks/Program.cs

@@ -63,6 +63,7 @@
     QueryCount = arguments.Int("queries") ?? int.MaxValue,
     EfSearchSweep = arguments.Ints("ef") ?? [10, 20, 40, 80, 160, 320, 640],
     Concurrency = arguments.Int("concurrency") ?? 1,
+    QueryPasses = arguments.Int("passes") ?? 1,
     BuildThreads = arguments.Int("threads") ?? 1,
     ReuseIndex = arguments.Flag("reuse-index"),
 };
@@ -149,6 +150,8 @@ Qvec recall/QPS benchmark.
               --ef <list>         comma-separated efSearch sweep, default 10,20,40,80,160,320,640
               --queries <int>     limit the number of queries
               --concurrency <n>   query threads, default 1; QPS is aggregate over all threads
+              --passes <n>        run the query set n times per efSearch row (default 1); use
+                                  5-10 on Cohere, whose 1,000 queries finish in under a second
               --threads <n>       index build threads (AddEntries), default 1; 0 = all cores.
                                   Builds with more than one thread are not byte-reproducible.
               --max-base <int>    index only a prefix of the base set (invalidates recall)

benchmarks/Qvec.Benchmarks/RecallBenchmark.cs

@@ -191,34 +191,29 @@ private static RecallQpsPoint MeasureOne(
         int queryCount = Math.Min(options.QueryCount, dataset.Queries.Count);
         int topK = options.TopK;
 
-        // Warm up so the first measured query is not paying for page faults on the mapped file
-        // and JIT of the search path. Without this the first efSearch in the sweep looks slow
-        // for reasons that have nothing to do with efSearch.
-        for (int q = 0; q < Math.Min(options.WarmupQueries, queryCount); q++)
-        {
-            db.Search(dataset.Queries.ToArray(q), topK, efSearch);
-        }
-
-        double recallAt1 = 0;
-        double recallAtK = 0;
-
         var queries = new float[queryCount][];
         for (int q = 0; q < queryCount; q++) queries[q] = dataset.Queries.ToArray(q);
 
-        var stopwatch = Stopwatch.StartNew();
         var results = new List<(Guid Id, float Score, string Metadata)>[queryCount];
-        if (options.Concurrency <= 1)
+
+        // Runs `count` queries (cycling through the set) on the configured number of threads.
+        // Each thread takes the next query off a shared counter, the way VectorDBBench's
+        // concurrent clients each run their own serial loop. Aggregate QPS is queries over
+        // wall-clock time, so a slow thread shows up honestly instead of being averaged away.
+        // Results are deterministic, so only the first pass is recorded for scoring.
+        void RunQueries(int count, bool record)
         {
-            for (int q = 0; q < queryCount; q++)
+            if (options.Concurrency <= 1)
             {
-                results[q] = db.Search(queries[q], topK, efSearch);
+                for (int i = 0; i < count; i++)
+                {
+                    int q = i % queryCount;
+                    var r = db.Search(queries[q], topK, efSearch);
+                    if (record && i < queryCount) results[q] = r;
+                }
+                return;
             }
-        }
-        else
-        {
-            // Each thread takes the next query off a shared counter, the way VectorDBBench's
-            // concurrent clients each run their own serial loop. Aggregate QPS is queries over
-            // wall-clock time, so a slow thread shows up honestly instead of being averaged away.
+
             int next = -1;
             var threads = new Thread[options.Concurrency];
             for (int t = 0; t < threads.Length; t++)
@@ -227,16 +222,42 @@ private static RecallQpsPoint MeasureOne(
                 {
                     while (true)
                     {
-                        int q = Interlocked.Increment(ref next);
-                        if (q >= queryCount) break;
-                        results[q] = db.Search(queries[q], topK, efSearch);
+                        int i = Interlocked.Increment(ref next);
+                        if (i >= count) break;
+                        int q = i % queryCount;
+                        var r = db.Search(queries[q], topK, efSearch);
+                        if (record && i < queryCount) results[q] = r;
                     }
                 });
                 threads[t].Start();
             }
 
             foreach (var thread in threads) thread.Join();
         }
+
+        // Warm up so the first measured query is not paying for page faults on the mapped file,
+        // thread start-up or tiered JIT of the search path. Without this the first efSearch in
+        // the sweep looks slow for reasons that have nothing to do with efSearch. The warm-up
+        // uses the same thread count as the measurement and runs at least one full pass over
+        // the query set and at least two seconds: a fresh process has to soft-fault every page
+        // of a multi-gigabyte mapping into its working set even when the file is cached.
+        var warmup = Stopwatch.StartNew();
+        RunQueries(Math.Max(options.WarmupQueries, queryCount), record: false);
+        while (warmup.Elapsed < options.MinimumWarmup)
+        {
+            RunQueries(queryCount, record: false);
+        }
+
+        double recallAt1 = 0;
+        double recallAtK = 0;
+
+        // With only 1,000 queries (Cohere) a row is over in half a second, which is mostly noise
+        // rather than throughput. Passes repeat the query set inside the timed region.
+        int passes = Math.Max(options.QueryPasses, 1);
+        int total = queryCount * passes;
+
+        var stopwatch = Stopwatch.StartNew();
+        RunQueries(total, record: true);
         stopwatch.Stop();
 
         // Scoring happens outside the timed region: mapping Guids back to base indices is our
@@ -267,13 +288,13 @@ private static RecallQpsPoint MeasureOne(
 
         // With N threads the wall time per query understates what one caller waits; multiply
         // back up so the column is an estimate of per-query latency under that load.
-        double meanLatencyMs = stopwatch.Elapsed.TotalMilliseconds * Math.Max(options.Concurrency, 1) / queryCount;
+        double meanLatencyMs = stopwatch.Elapsed.TotalMilliseconds * Math.Max(options.Concurrency, 1) / total;
 
         return new RecallQpsPoint(
             efSearch,
             recallAt1 / queryCount,
             recallAtK / queryCount,
-            queryCount / Math.Max(seconds, 1e-9),
+            total / Math.Max(seconds, 1e-9),
             meanLatencyMs);
     }
 }
@@ -290,12 +311,18 @@ public sealed class BenchmarkOptions
     public int TopK { get; init; } = 10;
     public int QueryCount { get; init; } = int.MaxValue;
     public int WarmupQueries { get; init; } = 100;
+
+    /// <summary>Each efSearch row warms up for at least this long before the timed region.</summary>
+    public TimeSpan MinimumWarmup { get; init; } = TimeSpan.FromSeconds(2);
     public int ProgressEvery { get; init; } = 100_000;
     public IReadOnlyList<int> EfSearchSweep { get; init; } = [10, 20, 40, 80, 160, 320, 640];
 
     /// <summary>Number of threads issuing queries; 1 is the single-threaded latency view.</summary>
     public int Concurrency { get; init; } = 1;
 
+    /// <summary>How many times the query set is run per efSearch row; QPS covers all passes.</summary>
+    public int QueryPasses { get; init; } = 1;
+
     /// <summary>
     /// Threads used to build the index through <see cref="QvecDatabase.AddEntries"/>; 1 gives
     /// the same graph as one-by-one <see cref="QvecDatabase.AddEntry"/>, 0 means all cores.

benchmarks/README.md

@@ -58,6 +58,15 @@ the mode so a float row and an int8 row cannot be confused for each other.
 it, so `--k`, `--ef` and `--concurrency` can be swept without paying for the build again. The
 build time is reported as zero in that case, not as the previous run's number.
 
+`--passes <n>` runs the query set `n` times per `efSearch` row inside the timed region. Cohere
+ships only 1,000 queries, which at several thousand QPS is over in a fraction of a second — too
+short to be a throughput number. Use 5–10 there. Recall is unaffected (the results are
+deterministic and only the first pass is scored). Each row is also preceded by a warm-up on the
+same thread count that runs at least one full pass and at least two seconds, because a fresh
+process must soft-fault every page of a multi-gigabyte mapping into its working set even when
+the OS has the file cached; without that the first row of a sweep looked 2–3× slower than the
+second for reasons unrelated to `efSearch`.
+
 ### Windows power throttling
 
 On startup the benchmark asks Windows to exempt it from power throttling (EcoQoS) and prints
@@ -114,7 +123,49 @@ at a cost of 0.1 pp recall@100. The float rows use the incremental prune
 1,696 s (59 inserts/s) and reached 97.9 % / 99.3 % at efSearch 100 / 180 — the graph is not
 byte-identical, so recall is re-measured rather than assumed. The int8 row shows the other open
 item plainly: without rescoring on the floats, int8 recall@100 plateaus at 96.8 % on Cohere,
-which is why Zvec's published figures use int8 *with* a refiner.
+which is why Zvec's published Cohere 10M figures use int8 *with* a refiner (their 1M run does not).
+
+### Cohere 1M, measured
+
+Zvec's published Cohere 1M run is `--quantize-type int8 --m 15 --ef-search 180` under 12–20
+concurrent clients on a 16-vCPU g9i.4xlarge ([their reproduction
+guide](https://zvec.org/en/docs/db/benchmarks/#cohere-1m)). The matching invocation here is
+
+```bash
+dotnet run -c Release --project benchmarks/Qvec.Benchmarks -- --dataset cohere1m --k 100 --m 15 --ef 100,180,320 --threads 0 --concurrency 12 --passes 10 [--quantization int8]
+```
+
+Snapdragon X Elite (12 cores, ARM64, laptop), Windows 11, throttling exemption in place, both
+indexes built the same evening with 12 threads, queries at `--concurrency 12`:
+
+| mode | build | inserts/s | file | efSearch | recall@1 | recall@100 | QPS 1 thread | QPS 12 threads | latency @ 12 |
+| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
+| float | 419 s | 2,385 | 3,376 MiB | 100 | 97.1 % | 90.3 % | — ‡ | 6,441 | 1.86 ms |
+| float | | | | 180 | 98.1 % | 94.8 % | 565 | 3,764 | 3.19 ms |
+| float | | | | 320 | 98.8 % | 97.4 % | 351 | 2,207 | 5.44 ms |
+| int8 | 181 s | 5,516 | 1,194 MiB | 100 | 93.1 % | 89.3 % | 1,395 | 13,540 | 0.89 ms |
+| int8 | | | | 180 | 94.4 % | 93.2 % | 839 | 6,841 | 1.75 ms |
+| int8 | | | | 320 | 95.4 % | 95.3 % | 502 | 4,714 | 2.55 ms |
+
+‡ The single-threaded float sweep was measured at `--passes 2`, where the first row had not
+finished faulting the 3.4 GB mapping in; it is omitted rather than reported wrong.
+
+Twelve query threads give 6.7× (float) and 8.2× (int8) over one. Zvec's chart for the same
+configuration reads as roughly 8–9 thousand QPS at recall@100 ≈ 0.93–0.94 on 16 vCPUs; the
+exact values are only published as an image, so treat that as approximate. Per core, the int8
+row at efSearch 180 (6,841 QPS / 12 cores) is in the same range, on different hardware, a
+different OS and a different day — which is as far as the comparison honestly goes. What the
+table does show without caveats is Qvec's own shape: int8 is 1.8× the float QPS at the same
+efSearch but loses 1.6 pp recall@100 at 180 and plateaus around 95 %, so a float refiner
+(`quantization-rescoring`) is the lever that would make the int8 row competitive on recall.
+
+Measuring this uncovered a scaling bug in the search path, fixed on the same branch. Result
+materialisation read the Guid and metadata of every hit through `MemoryMappedViewAccessor`,
+whose every call takes an interlocked reference on the shared `SafeBuffer`; at `k = 100` that is
+a few hundred atomic operations per query on one cache line, and twelve threads serialised on
+it. Cohere 1M float at efSearch 100 went from 1,883 to 6,441 QPS on 12 threads once the reads
+went through the raw mapping pointer like the distance computations already did. Single-threaded
+throughput was unaffected, which is why the SIFT numbers in the main README did not move.
 
 ## Metric