From f48ad75e51b4c447e51590b28135bb45f714154c Mon Sep 17 00:00:00 2001 From: "Claude - GLM-5.2 - Gustavo" Date: Thu, 20 Aug 2026 21:26:48 -0300 Subject: [PATCH] feat: ValorBrain memory provider + BEAM-100K result (71.4% avg score, 400 queries) Adds the ValorBrain memory provider (hybrid retrieval over REST, re-windowed ingest), the LLM providers needed to reproduce our reader/judge pair (gateway/glm/agy-direct, OpenAI-compatible plain-HTTP with reasoning-model fallback), and our complete BEAM-100K run: 400/400 queries, 71.4% avg score per criterion (304/400 = 76.0% binary correct), reader Gemini 3.6 Flash, judge GLM-5.2 (different family from the reader), avg retrieve 5.2s, avg context 12.6k tokens. Reproduced independently with a different reader (70.2%). Full results in outputs/beam/valorbrain/rag/100k.json. --- README.md | 23 + results-manifest.json | 1532 +++++++++++++------------ src/memory_bench/cli.py | 7 +- src/memory_bench/llm/__init__.py | 6 + src/memory_bench/llm/agy_direct.py | 75 ++ src/memory_bench/llm/gateway.py | 124 ++ src/memory_bench/llm/glm.py | 130 +++ src/memory_bench/memory/__init__.py | 2 + src/memory_bench/memory/valorbrain.py | 253 ++++ src/memory_bench/runner.py | 13 + 10 files changed, 1403 insertions(+), 762 deletions(-) create mode 100644 src/memory_bench/llm/agy_direct.py create mode 100644 src/memory_bench/llm/gateway.py create mode 100644 src/memory_bench/llm/glm.py create mode 100644 src/memory_bench/memory/valorbrain.py diff --git a/README.md b/README.md index bc44295..5dbf04e 100644 --- a/README.md +++ b/README.md @@ -67,3 +67,26 @@ Results are saved to `outputs/{dataset}/{memory}/{mode}/{domain}.json` and can b - Python ≥ 3.11 - `GEMINI_API_KEY` in `.env` or environment - For MemBench: set `MEMBENCH_DATA_PATH` to your local data directory + +--- + +## ValorBrain memory provider + +**ValorBrain** is a hybrid memory engine for AI agents: BM25 + dense vectors (pgvector) + reciprocal-rank fusion + entity-graph reranking + cross-encoder rerank, on PostgreSQL with row-level security, multi-tenant. The provider talks to a running engine over its REST API — retrieval goes through `/api/v1/memory/prepare`, the same production path agents use at runtime (full funnel: consolidation + timeline + snippet delivery), falling back to `/search` (hybrid) when prepare returns nothing. On ingest, ~100k-char AMB chunks are re-windowed into 8k-char windows with 800-char overlap, matching production conversational windows. + +### BEAM-100K result + +| | | +|---|---| +| **Accuracy (avg score per criterion)** | **71.4%** | +| Binary correct | 304/400 (76.0%) | +| Reader (answer) | Gemini 3.6 Flash (AGY gateway) | +| Judge | GLM-5.2 (Z.ai) — different family from the reader | +| Avg retrieve time | 5.2s | +| Avg context tokens | 12.6k | + +Run file: `outputs/beam/valorbrain/rag/100k.json` (400/400 queries, `--split 100k --memory valorbrain`). + +Reproduction: set `VALORBRAIN_URL`, `VALORBRAIN_TOKEN`, `VALORBRAIN_BENCHMARK_TENANT_ID` to a running engine + benchmark tenant, plus reader/judge keys, then `uv run amb run --dataset beam --split 100k --memory valorbrain --name valorbrain`. The judge is overridable via `OMB_JUDGE_LLM`/`OMB_JUDGE_MODEL` (see Known patches below) — we pin a judge from a different model family than the reader to avoid self-preference bias. + +The result was independently reproduced with a different reader/judge pair (stealth "Ox Alpha" reader via OpenCode Zen + GLM judge: 70.2% avg score, 76.0% binary on the same 400 queries), supporting that the score measures the memory system rather than one particular reader. diff --git a/results-manifest.json b/results-manifest.json index 23cfd95..ce1ebda 100644 --- a/results-manifest.json +++ b/results-manifest.json @@ -1,760 +1,776 @@ [ - { - "path": "outputs/beam/hindsight/rag/100k.json", - "dataset": "beam", - "run_name": "hindsight", - "memory": "hindsight", - "mode": "rag", - "split": "100k", - "total_queries": 400, - "correct": 367, - "accuracy": 0.8618214285714286, - "ingestion_time_ms": 0.0, - "ingested_docs": 0, - "avg_retrieve_time_ms": 2565.3, - "avg_context_tokens": 23662.0, - "category": null - }, - { - "path": "outputs/beam/hindsight/rag/1m.json", - "dataset": "beam", - "run_name": "hindsight", - "memory": "hindsight", - "mode": "rag", - "split": "1m", - "total_queries": 700, - "correct": 617, - "accuracy": 0.7912926675548582, - "ingestion_time_ms": 2714555.4, - "ingested_docs": 1228, - "avg_retrieve_time_ms": 2968.3, - "avg_context_tokens": 31664.8, - "category": null - }, - { - "path": "outputs/beam/hindsight/rag/500k.json", - "dataset": "beam", - "run_name": "hindsight", - "memory": "hindsight", - "mode": "rag", - "split": "500k", - "total_queries": 700, - "correct": 610, - "accuracy": 0.8009276816418098, - "ingestion_time_ms": 816268.7, - "ingested_docs": 364, - "avg_retrieve_time_ms": 1956.0, - "avg_context_tokens": 30055.3, - "category": null - }, - { - "path": "outputs/beam/hindsight/single-query/100k.json", - "dataset": "beam", - "run_name": "hindsight", - "memory": "hindsight", - "mode": "single-query", - "split": "100k", - "total_queries": 400, - "correct": 326, - "accuracy": 0.7336577408538804, - "ingestion_time_ms": 403580.5, - "ingested_docs": 6, - "avg_retrieve_time_ms": 6379.4, - "avg_context_tokens": 17654.6, - "category": null - }, - { - "path": "outputs/beam/hindsight/single-query/10m.json", - "dataset": "beam", - "run_name": "hindsight", - "memory": "hindsight", - "mode": "single-query", - "split": "10m", - "total_queries": 200, - "correct": 147, - "accuracy": 0.6407977206043332, - "ingestion_time_ms": 5504578.7, - "ingested_docs": 1, - "avg_retrieve_time_ms": 14744.9, - "avg_context_tokens": 27342.0, - "category": null - }, - { - "path": "outputs/beam/hindsight/single-query/1m.json", - "dataset": "beam", - "run_name": "hindsight", - "memory": "hindsight", - "mode": "single-query", - "split": "1m", - "total_queries": 700, - "correct": 585, - "accuracy": 0.7386039171368353, - "ingestion_time_ms": 347733.5, - "ingested_docs": 51, - "avg_retrieve_time_ms": 5517.7, - "avg_context_tokens": 23901.7, - "category": null - }, - { - "path": "outputs/beam/hindsight/single-query/500k.json", - "dataset": "beam", - "run_name": "hindsight", - "memory": "hindsight", - "mode": "single-query", - "split": "500k", - "total_queries": 700, - "correct": 541, - "accuracy": 0.7111989151837411, - "ingestion_time_ms": 21787574.7, - "ingested_docs": 962, - "avg_retrieve_time_ms": 5149.3, - "avg_context_tokens": 20529.5, - "category": null - }, - { - "path": "outputs/lifebench/hindsight/rag/en.json", - "dataset": "lifebench", - "run_name": "hindsight", - "memory": "hindsight", - "mode": "rag", - "split": "en", - "total_queries": 2003, - "correct": 1433, - "accuracy": 0.7154268597104343, - "ingestion_time_ms": 12088621.4, - "ingested_docs": 2877, - "avg_retrieve_time_ms": 705.2, - "avg_context_tokens": 34324.2, - "category": null - }, - { - "path": "outputs/lifebench/hybrid-search/rag/en.json", - "dataset": "lifebench", - "run_name": "hybrid-search", - "memory": "hybrid-search", - "mode": "rag", - "split": "en", - "total_queries": 2003, - "correct": 1221, - "accuracy": 0.6095856215676485, - "ingestion_time_ms": 13975390.8, - "ingested_docs": 3605, - "avg_retrieve_time_ms": 791.4, - "avg_context_tokens": 23544.1, - "category": null - }, - { - "path": "outputs/locomo/cognee/rag/locomo10.json", - "dataset": "locomo", - "run_name": "cognee", - "memory": "cognee", - "mode": "rag", - "split": "locomo10", - "total_queries": 152, - "correct": 122, - "accuracy": 0.8026315789473685, - "ingestion_time_ms": 289744.4, - "ingested_docs": 19, - "avg_retrieve_time_ms": 673.7, - "avg_context_tokens": 14723.5, - "category": null - }, - { - "path": "outputs/locomo/hybrid-search/rag/locomo10.json", - "dataset": "locomo", - "run_name": "hybrid-search", - "memory": "hybrid-search", - "mode": "rag", - "split": "locomo10", - "total_queries": 1540, - "correct": 1218, - "accuracy": 0.7909090909090909, - "ingestion_time_ms": 645739.8, - "ingested_docs": 272, - "avg_retrieve_time_ms": 219.3, - "avg_context_tokens": 22156.5, - "category": null - }, - { - "path": "outputs/locomo/locomo-hindsight/rag/locomo10.json", - "dataset": "locomo", - "run_name": "locomo-hindsight", - "memory": "hindsight", - "mode": "rag", - "split": "locomo10", - "total_queries": 1540, - "correct": 1417, - "accuracy": 0.9201298701298701, - "ingestion_time_ms": 330205.8, - "ingested_docs": 272, - "avg_retrieve_time_ms": 964.1, - "avg_context_tokens": 36235.4, - "category": null - }, - { - "path": "outputs/longmemeval/hindsight/rag/s.json", - "dataset": "longmemeval", - "run_name": "hindsight", - "memory": "hindsight", - "mode": "rag", - "split": "s", - "total_queries": 500, - "correct": 473, - "accuracy": 0.946, - "ingestion_time_ms": 30090034.3, - "ingested_docs": 11303, - "avg_retrieve_time_ms": 674.9, - "avg_context_tokens": 43624.5, - "category": null - }, - { - "path": "outputs/longmemeval/hybrid-search/rag/s.json", - "dataset": "longmemeval", - "run_name": "hybrid-search", - "memory": "hybrid-search", - "mode": "rag", - "split": "s", - "total_queries": 500, - "correct": 370, - "accuracy": 0.74, - "ingestion_time_ms": 57815933.6, - "ingested_docs": 23867, - "avg_retrieve_time_ms": 1599.5, - "avg_context_tokens": 23221.7, - "category": null - }, - { - "path": "outputs/personamem/cognee/rag/32k.json", - "dataset": "personamem", - "run_name": "cognee", - "memory": "cognee", - "mode": "rag", - "split": "32k", - "total_queries": 589, - "correct": 482, - "accuracy": 0.8183361629881154, - "ingestion_time_ms": 2335222.9, - "ingested_docs": 195, - "avg_retrieve_time_ms": 657.9, - "avg_context_tokens": 11848.2, - "category": null - }, - { - "path": "outputs/personamem/hindsight/rag/32k.json", - "dataset": "personamem", - "run_name": "hindsight", - "memory": "hindsight", - "mode": "rag", - "split": "32k", - "total_queries": 589, - "correct": 510, - "accuracy": 0.865874363327674, - "ingestion_time_ms": 1039008.5, - "ingested_docs": 195, - "avg_retrieve_time_ms": 674.9, - "avg_context_tokens": 15811.6, - "category": null - }, - { - "path": "outputs/personamem/hybrid-search/rag/32k.json", - "dataset": "personamem", - "run_name": "hybrid-search", - "memory": "hybrid-search", - "mode": "rag", - "split": "32k", - "total_queries": 589, - "correct": 497, - "accuracy": 0.8438030560271647, - "ingestion_time_ms": 1039209.5, - "ingested_docs": 195, - "avg_retrieve_time_ms": 362.2, - "avg_context_tokens": 24169.0, - "category": null - }, - { - "path": "outputs/sdebench/hindsight-claude-1/coding/boltons.json", - "dataset": "sdebench", - "run_name": "hindsight-claude-1", - "memory": "hindsight-coding", - "mode": "coding", - "split": "boltons", - "total_queries": 61, - "correct": 61, - "accuracy": 1.0, - "ingestion_time_ms": 0.0, - "ingested_docs": 0, - "avg_retrieve_time_ms": 80218.0, - "avg_context_tokens": 208.7, - "category": null, - "coding": true, - "agent": "claude-code", - "model": "claude-sonnet-5", - "tasks": 61, - "solved": 61, - "interventions": 24, - "cost_usd": 21.5, - "turns": 896, - "wall_s": 4893.0, - "tokens_in": 31433379, - "tokens_out": 277621 - }, - { - "path": "outputs/sdebench/hindsight-claude-2/coding/boltons.json", - "dataset": "sdebench", - "run_name": "hindsight-claude-2", - "memory": "hindsight-coding", - "mode": "coding", - "split": "boltons", - "total_queries": 61, - "correct": 61, - "accuracy": 1.0, - "ingestion_time_ms": 0.0, - "ingested_docs": 0, - "avg_retrieve_time_ms": 71614.8, - "avg_context_tokens": 216.1, - "category": null, - "coding": true, - "agent": "claude-code", - "model": "claude-sonnet-5", - "tasks": 61, - "solved": 61, - "interventions": 17, - "cost_usd": 19.53, - "turns": 832, - "wall_s": 4368.0, - "tokens_in": 29041651, - "tokens_out": 251341 - }, - { - "path": "outputs/sdebench/hindsight-claude-3/coding/boltons.json", - "dataset": "sdebench", - "run_name": "hindsight-claude-3", - "memory": "hindsight-coding", - "mode": "coding", - "split": "boltons", - "total_queries": 61, - "correct": 61, - "accuracy": 1.0, - "ingestion_time_ms": 0.0, - "ingested_docs": 0, - "avg_retrieve_time_ms": 73431.1, - "avg_context_tokens": 222.6, - "category": null, - "coding": true, - "agent": "claude-code", - "model": "claude-sonnet-5", - "tasks": 61, - "solved": 61, - "interventions": 25, - "cost_usd": 20.78, - "turns": 868, - "wall_s": 4479.0, - "tokens_in": 29943843, - "tokens_out": 255704 - }, - { - "path": "outputs/sdebench/hindsight-codex-1/coding/boltons.json", - "dataset": "sdebench", - "run_name": "hindsight-codex-1", - "memory": "hindsight-coding", - "mode": "coding", - "split": "boltons", - "total_queries": 61, - "correct": 61, - "accuracy": 1.0, - "ingestion_time_ms": 0.0, - "ingested_docs": 0, - "avg_retrieve_time_ms": 46752.5, - "avg_context_tokens": 243.0, - "category": null, - "coding": true, - "agent": "codex", - "model": "gpt-5.4-mini", - "tasks": 61, - "solved": 61, - "interventions": 25, - "cost_usd": 15.98, - "turns": 1392, - "wall_s": 2852.0, - "tokens_in": 32507346, - "tokens_out": 465613 - }, - { - "path": "outputs/sdebench/hindsight-codex-2/coding/boltons.json", - "dataset": "sdebench", - "run_name": "hindsight-codex-2", - "memory": "hindsight-coding", - "mode": "coding", - "split": "boltons", - "total_queries": 61, - "correct": 61, - "accuracy": 1.0, - "ingestion_time_ms": 0.0, - "ingested_docs": 0, - "avg_retrieve_time_ms": 47696.7, - "avg_context_tokens": 236.5, - "category": null, - "coding": true, - "agent": "codex", - "model": "gpt-5.4-mini", - "tasks": 61, - "solved": 61, - "interventions": 30, - "cost_usd": 16.78, - "turns": 1450, - "wall_s": 2910.0, - "tokens_in": 34187227, - "tokens_out": 484686 - }, - { - "path": "outputs/sdebench/hindsight-codex-3/coding/boltons.json", - "dataset": "sdebench", - "run_name": "hindsight-codex-3", - "memory": "hindsight-coding", - "mode": "coding", - "split": "boltons", - "total_queries": 61, - "correct": 61, - "accuracy": 1.0, - "ingestion_time_ms": 0.0, - "ingested_docs": 0, - "avg_retrieve_time_ms": 51391.8, - "avg_context_tokens": 226.2, - "category": null, - "coding": true, - "agent": "codex", - "model": "gpt-5.4-mini", - "tasks": 61, - "solved": 61, - "interventions": 31, - "cost_usd": 17.67, - "turns": 1461, - "wall_s": 3135.0, - "tokens_in": 35254187, - "tokens_out": 567986 - }, - { - "path": "outputs/sdebench/hindsight-opencode-1/coding/boltons.json", - "dataset": "sdebench", - "run_name": "hindsight-opencode-1", - "memory": "hindsight-coding", - "mode": "coding", - "split": "boltons", - "total_queries": 61, - "correct": 60, - "accuracy": 0.9836065573770492, - "ingestion_time_ms": 0.0, - "ingested_docs": 0, - "avg_retrieve_time_ms": 171332.8, - "avg_context_tokens": 249.1, - "category": null, - "coding": true, - "agent": "opencode", - "model": "google/gemini-3.5-flash", - "tasks": 61, - "solved": 60, - "interventions": 50, - "cost_usd": 31.77, - "turns": 2135, - "wall_s": 10451.0, - "tokens_in": 54775427, - "tokens_out": 840575 - }, - { - "path": "outputs/sdebench/hindsight-opencode-2/coding/boltons.json", - "dataset": "sdebench", - "run_name": "hindsight-opencode-2", - "memory": "hindsight-coding", - "mode": "coding", - "split": "boltons", - "total_queries": 61, - "correct": 61, - "accuracy": 1.0, - "ingestion_time_ms": 0.0, - "ingested_docs": 0, - "avg_retrieve_time_ms": 160829.5, - "avg_context_tokens": 220.8, - "category": null, - "coding": true, - "agent": "opencode", - "model": "google/gemini-3.5-flash", - "tasks": 61, - "solved": 61, - "interventions": 46, - "cost_usd": 32.3, - "turns": 2115, - "wall_s": 9811.0, - "tokens_in": 53326204, - "tokens_out": 797596 - }, - { - "path": "outputs/sdebench/hindsight-opencode-3/coding/boltons.json", - "dataset": "sdebench", - "run_name": "hindsight-opencode-3", - "memory": "hindsight-coding", - "mode": "coding", - "split": "boltons", - "total_queries": 61, - "correct": 61, - "accuracy": 1.0, - "ingestion_time_ms": 0.0, - "ingested_docs": 0, - "avg_retrieve_time_ms": 157729.5, - "avg_context_tokens": 237.3, - "category": null, - "coding": true, - "agent": "opencode", - "model": "google/gemini-3.5-flash", - "tasks": 61, - "solved": 61, - "interventions": 50, - "cost_usd": 37.14, - "turns": 2298, - "wall_s": 9622.0, - "tokens_in": 62036828, - "tokens_out": 849154 - }, - { - "path": "outputs/sdebench/vanilla-claude-1/coding/boltons.json", - "dataset": "sdebench", - "run_name": "vanilla-claude-1", - "memory": "vanilla", - "mode": "coding", - "split": "boltons", - "total_queries": 61, - "correct": 60, - "accuracy": 0.9836065573770492, - "ingestion_time_ms": 13.2, - "ingested_docs": 14706, - "avg_retrieve_time_ms": 87980.3, - "avg_context_tokens": 7.0, - "category": null, - "coding": true, - "agent": "claude-code", - "model": "claude-sonnet-5", - "tasks": 61, - "solved": 60, - "interventions": 56, - "cost_usd": 27.29, - "turns": 1015, - "wall_s": 5367.0, - "tokens_in": 36732480, - "tokens_out": 390756 - }, - { - "path": "outputs/sdebench/vanilla-claude-2/coding/boltons.json", - "dataset": "sdebench", - "run_name": "vanilla-claude-2", - "memory": "vanilla", - "mode": "coding", - "split": "boltons", - "total_queries": 61, - "correct": 60, - "accuracy": 0.9836065573770492, - "ingestion_time_ms": 13.9, - "ingested_docs": 14706, - "avg_retrieve_time_ms": 80416.4, - "avg_context_tokens": 7.0, - "category": null, - "coding": true, - "agent": "claude-code", - "model": "claude-sonnet-5", - "tasks": 61, - "solved": 60, - "interventions": 50, - "cost_usd": 26.33, - "turns": 971, - "wall_s": 4905.0, - "tokens_in": 34885807, - "tokens_out": 378763 - }, - { - "path": "outputs/sdebench/vanilla-claude-3/coding/boltons.json", - "dataset": "sdebench", - "run_name": "vanilla-claude-3", - "memory": "vanilla", - "mode": "coding", - "split": "boltons", - "total_queries": 61, - "correct": 61, - "accuracy": 1.0, - "ingestion_time_ms": 12.8, - "ingested_docs": 14706, - "avg_retrieve_time_ms": 85563.9, - "avg_context_tokens": 7.0, - "category": null, - "coding": true, - "agent": "claude-code", - "model": "claude-sonnet-5", - "tasks": 61, - "solved": 61, - "interventions": 49, - "cost_usd": 27.75, - "turns": 982, - "wall_s": 5219.0, - "tokens_in": 35801538, - "tokens_out": 403470 - }, - { - "path": "outputs/sdebench/vanilla-codex-1/coding/boltons.json", - "dataset": "sdebench", - "run_name": "vanilla-codex-1", - "memory": "vanilla", - "mode": "coding", - "split": "boltons", - "total_queries": 61, - "correct": 61, - "accuracy": 1.0, - "ingestion_time_ms": 16.8, - "ingested_docs": 14706, - "avg_retrieve_time_ms": 51154.1, - "avg_context_tokens": 7.0, - "category": null, - "coding": true, - "agent": "codex", - "model": "gpt-5.4-mini", - "tasks": 61, - "solved": 61, - "interventions": 80, - "cost_usd": 32.86, - "turns": 1781, - "wall_s": 3120.0, - "tokens_in": 64473703, - "tokens_out": 1113034 - }, - { - "path": "outputs/sdebench/vanilla-codex-2/coding/boltons.json", - "dataset": "sdebench", - "run_name": "vanilla-codex-2", - "memory": "vanilla", - "mode": "coding", - "split": "boltons", - "total_queries": 61, - "correct": 60, - "accuracy": 0.9836065573770492, - "ingestion_time_ms": 18.2, - "ingested_docs": 14706, - "avg_retrieve_time_ms": 53754.1, - "avg_context_tokens": 7.0, - "category": null, - "coding": true, - "agent": "codex", - "model": "gpt-5.4-mini", - "tasks": 61, - "solved": 60, - "interventions": 83, - "cost_usd": 37.65, - "turns": 1867, - "wall_s": 3279.0, - "tokens_in": 74249355, - "tokens_out": 1282729 - }, - { - "path": "outputs/sdebench/vanilla-codex-3/coding/boltons.json", - "dataset": "sdebench", - "run_name": "vanilla-codex-3", - "memory": "vanilla", - "mode": "coding", - "split": "boltons", - "total_queries": 61, - "correct": 61, - "accuracy": 1.0, - "ingestion_time_ms": 14.4, - "ingested_docs": 14706, - "avg_retrieve_time_ms": 55793.4, - "avg_context_tokens": 7.0, - "category": null, - "coding": true, - "agent": "codex", - "model": "gpt-5.4-mini", - "tasks": 61, - "solved": 61, - "interventions": 83, - "cost_usd": 35.12, - "turns": 1869, - "wall_s": 3403.0, - "tokens_in": 68638315, - "tokens_out": 1239311 - }, - { - "path": "outputs/sdebench/vanilla-opencode-1/coding/boltons.json", - "dataset": "sdebench", - "run_name": "vanilla-opencode-1", - "memory": "vanilla", - "mode": "coding", - "split": "boltons", - "total_queries": 61, - "correct": 61, - "accuracy": 1.0, - "ingestion_time_ms": 29.3, - "ingested_docs": 14706, - "avg_retrieve_time_ms": 185123.0, - "avg_context_tokens": 7.0, - "category": null, - "coding": true, - "agent": "opencode", - "model": "google/gemini-3.5-flash", - "tasks": 61, - "solved": 61, - "interventions": 74, - "cost_usd": 41.66, - "turns": 2369, - "wall_s": 11292.0, - "tokens_in": 65435546, - "tokens_out": 1121445 - }, - { - "path": "outputs/sdebench/vanilla-opencode-2/coding/boltons.json", - "dataset": "sdebench", - "run_name": "vanilla-opencode-2", - "memory": "vanilla", - "mode": "coding", - "split": "boltons", - "total_queries": 61, - "correct": 61, - "accuracy": 1.0, - "ingestion_time_ms": 14.8, - "ingested_docs": 14706, - "avg_retrieve_time_ms": 162785.2, - "avg_context_tokens": 7.0, - "category": null, - "coding": true, - "agent": "opencode", - "model": "google/gemini-3.5-flash", - "tasks": 61, - "solved": 61, - "interventions": 76, - "cost_usd": 39.52, - "turns": 2325, - "wall_s": 9930.0, - "tokens_in": 72344900, - "tokens_out": 1115870 - }, - { - "path": "outputs/sdebench/vanilla-opencode-3/coding/boltons.json", - "dataset": "sdebench", - "run_name": "vanilla-opencode-3", - "memory": "vanilla", - "mode": "coding", - "split": "boltons", - "total_queries": 61, - "correct": 61, - "accuracy": 1.0, - "ingestion_time_ms": 18.8, - "ingested_docs": 14706, - "avg_retrieve_time_ms": 174832.8, - "avg_context_tokens": 7.0, - "category": null, - "coding": true, - "agent": "opencode", - "model": "google/gemini-3.5-flash", - "tasks": 61, - "solved": 61, - "interventions": 69, - "cost_usd": 34.9, - "turns": 2427, - "wall_s": 10665.0, - "tokens_in": 67479034, - "tokens_out": 1049086 - } + { + "path": "outputs/beam/hindsight/rag/100k.json", + "dataset": "beam", + "run_name": "hindsight", + "memory": "hindsight", + "mode": "rag", + "split": "100k", + "total_queries": 400, + "correct": 367, + "accuracy": 0.8618214285714286, + "ingestion_time_ms": 0.0, + "ingested_docs": 0, + "avg_retrieve_time_ms": 2565.3, + "avg_context_tokens": 23662.0, + "category": null + }, + { + "path": "outputs/beam/hindsight/rag/1m.json", + "dataset": "beam", + "run_name": "hindsight", + "memory": "hindsight", + "mode": "rag", + "split": "1m", + "total_queries": 700, + "correct": 617, + "accuracy": 0.7912926675548582, + "ingestion_time_ms": 2714555.4, + "ingested_docs": 1228, + "avg_retrieve_time_ms": 2968.3, + "avg_context_tokens": 31664.8, + "category": null + }, + { + "path": "outputs/beam/hindsight/rag/500k.json", + "dataset": "beam", + "run_name": "hindsight", + "memory": "hindsight", + "mode": "rag", + "split": "500k", + "total_queries": 700, + "correct": 610, + "accuracy": 0.8009276816418098, + "ingestion_time_ms": 816268.7, + "ingested_docs": 364, + "avg_retrieve_time_ms": 1956.0, + "avg_context_tokens": 30055.3, + "category": null + }, + { + "path": "outputs/beam/hindsight/single-query/100k.json", + "dataset": "beam", + "run_name": "hindsight", + "memory": "hindsight", + "mode": "single-query", + "split": "100k", + "total_queries": 400, + "correct": 326, + "accuracy": 0.7336577408538804, + "ingestion_time_ms": 403580.5, + "ingested_docs": 6, + "avg_retrieve_time_ms": 6379.4, + "avg_context_tokens": 17654.6, + "category": null + }, + { + "path": "outputs/beam/hindsight/single-query/10m.json", + "dataset": "beam", + "run_name": "hindsight", + "memory": "hindsight", + "mode": "single-query", + "split": "10m", + "total_queries": 200, + "correct": 147, + "accuracy": 0.6407977206043332, + "ingestion_time_ms": 5504578.7, + "ingested_docs": 1, + "avg_retrieve_time_ms": 14744.9, + "avg_context_tokens": 27342.0, + "category": null + }, + { + "path": "outputs/beam/hindsight/single-query/1m.json", + "dataset": "beam", + "run_name": "hindsight", + "memory": "hindsight", + "mode": "single-query", + "split": "1m", + "total_queries": 700, + "correct": 585, + "accuracy": 0.7386039171368353, + "ingestion_time_ms": 347733.5, + "ingested_docs": 51, + "avg_retrieve_time_ms": 5517.7, + "avg_context_tokens": 23901.7, + "category": null + }, + { + "path": "outputs/beam/hindsight/single-query/500k.json", + "dataset": "beam", + "run_name": "hindsight", + "memory": "hindsight", + "mode": "single-query", + "split": "500k", + "total_queries": 700, + "correct": 541, + "accuracy": 0.7111989151837411, + "ingestion_time_ms": 21787574.7, + "ingested_docs": 962, + "avg_retrieve_time_ms": 5149.3, + "avg_context_tokens": 20529.5, + "category": null + }, + { + "path": "outputs/lifebench/hindsight/rag/en.json", + "dataset": "lifebench", + "run_name": "hindsight", + "memory": "hindsight", + "mode": "rag", + "split": "en", + "total_queries": 2003, + "correct": 1433, + "accuracy": 0.7154268597104343, + "ingestion_time_ms": 12088621.4, + "ingested_docs": 2877, + "avg_retrieve_time_ms": 705.2, + "avg_context_tokens": 34324.2, + "category": null + }, + { + "path": "outputs/lifebench/hybrid-search/rag/en.json", + "dataset": "lifebench", + "run_name": "hybrid-search", + "memory": "hybrid-search", + "mode": "rag", + "split": "en", + "total_queries": 2003, + "correct": 1221, + "accuracy": 0.6095856215676485, + "ingestion_time_ms": 13975390.8, + "ingested_docs": 3605, + "avg_retrieve_time_ms": 791.4, + "avg_context_tokens": 23544.1, + "category": null + }, + { + "path": "outputs/locomo/cognee/rag/locomo10.json", + "dataset": "locomo", + "run_name": "cognee", + "memory": "cognee", + "mode": "rag", + "split": "locomo10", + "total_queries": 152, + "correct": 122, + "accuracy": 0.8026315789473685, + "ingestion_time_ms": 289744.4, + "ingested_docs": 19, + "avg_retrieve_time_ms": 673.7, + "avg_context_tokens": 14723.5, + "category": null + }, + { + "path": "outputs/locomo/hybrid-search/rag/locomo10.json", + "dataset": "locomo", + "run_name": "hybrid-search", + "memory": "hybrid-search", + "mode": "rag", + "split": "locomo10", + "total_queries": 1540, + "correct": 1218, + "accuracy": 0.7909090909090909, + "ingestion_time_ms": 645739.8, + "ingested_docs": 272, + "avg_retrieve_time_ms": 219.3, + "avg_context_tokens": 22156.5, + "category": null + }, + { + "path": "outputs/locomo/locomo-hindsight/rag/locomo10.json", + "dataset": "locomo", + "run_name": "locomo-hindsight", + "memory": "hindsight", + "mode": "rag", + "split": "locomo10", + "total_queries": 1540, + "correct": 1417, + "accuracy": 0.9201298701298701, + "ingestion_time_ms": 330205.8, + "ingested_docs": 272, + "avg_retrieve_time_ms": 964.1, + "avg_context_tokens": 36235.4, + "category": null + }, + { + "path": "outputs/longmemeval/hindsight/rag/s.json", + "dataset": "longmemeval", + "run_name": "hindsight", + "memory": "hindsight", + "mode": "rag", + "split": "s", + "total_queries": 500, + "correct": 473, + "accuracy": 0.946, + "ingestion_time_ms": 30090034.3, + "ingested_docs": 11303, + "avg_retrieve_time_ms": 674.9, + "avg_context_tokens": 43624.5, + "category": null + }, + { + "path": "outputs/longmemeval/hybrid-search/rag/s.json", + "dataset": "longmemeval", + "run_name": "hybrid-search", + "memory": "hybrid-search", + "mode": "rag", + "split": "s", + "total_queries": 500, + "correct": 370, + "accuracy": 0.74, + "ingestion_time_ms": 57815933.6, + "ingested_docs": 23867, + "avg_retrieve_time_ms": 1599.5, + "avg_context_tokens": 23221.7, + "category": null + }, + { + "path": "outputs/personamem/cognee/rag/32k.json", + "dataset": "personamem", + "run_name": "cognee", + "memory": "cognee", + "mode": "rag", + "split": "32k", + "total_queries": 589, + "correct": 482, + "accuracy": 0.8183361629881154, + "ingestion_time_ms": 2335222.9, + "ingested_docs": 195, + "avg_retrieve_time_ms": 657.9, + "avg_context_tokens": 11848.2, + "category": null + }, + { + "path": "outputs/personamem/hindsight/rag/32k.json", + "dataset": "personamem", + "run_name": "hindsight", + "memory": "hindsight", + "mode": "rag", + "split": "32k", + "total_queries": 589, + "correct": 510, + "accuracy": 0.865874363327674, + "ingestion_time_ms": 1039008.5, + "ingested_docs": 195, + "avg_retrieve_time_ms": 674.9, + "avg_context_tokens": 15811.6, + "category": null + }, + { + "path": "outputs/personamem/hybrid-search/rag/32k.json", + "dataset": "personamem", + "run_name": "hybrid-search", + "memory": "hybrid-search", + "mode": "rag", + "split": "32k", + "total_queries": 589, + "correct": 497, + "accuracy": 0.8438030560271647, + "ingestion_time_ms": 1039209.5, + "ingested_docs": 195, + "avg_retrieve_time_ms": 362.2, + "avg_context_tokens": 24169.0, + "category": null + }, + { + "path": "outputs/sdebench/hindsight-claude-1/coding/boltons.json", + "dataset": "sdebench", + "run_name": "hindsight-claude-1", + "memory": "hindsight-coding", + "mode": "coding", + "split": "boltons", + "total_queries": 61, + "correct": 61, + "accuracy": 1.0, + "ingestion_time_ms": 0.0, + "ingested_docs": 0, + "avg_retrieve_time_ms": 80218.0, + "avg_context_tokens": 208.7, + "category": null, + "coding": true, + "agent": "claude-code", + "model": "claude-sonnet-5", + "tasks": 61, + "solved": 61, + "interventions": 24, + "cost_usd": 21.5, + "turns": 896, + "wall_s": 4893.0, + "tokens_in": 31433379, + "tokens_out": 277621 + }, + { + "path": "outputs/sdebench/hindsight-claude-2/coding/boltons.json", + "dataset": "sdebench", + "run_name": "hindsight-claude-2", + "memory": "hindsight-coding", + "mode": "coding", + "split": "boltons", + "total_queries": 61, + "correct": 61, + "accuracy": 1.0, + "ingestion_time_ms": 0.0, + "ingested_docs": 0, + "avg_retrieve_time_ms": 71614.8, + "avg_context_tokens": 216.1, + "category": null, + "coding": true, + "agent": "claude-code", + "model": "claude-sonnet-5", + "tasks": 61, + "solved": 61, + "interventions": 17, + "cost_usd": 19.53, + "turns": 832, + "wall_s": 4368.0, + "tokens_in": 29041651, + "tokens_out": 251341 + }, + { + "path": "outputs/sdebench/hindsight-claude-3/coding/boltons.json", + "dataset": "sdebench", + "run_name": "hindsight-claude-3", + "memory": "hindsight-coding", + "mode": "coding", + "split": "boltons", + "total_queries": 61, + "correct": 61, + "accuracy": 1.0, + "ingestion_time_ms": 0.0, + "ingested_docs": 0, + "avg_retrieve_time_ms": 73431.1, + "avg_context_tokens": 222.6, + "category": null, + "coding": true, + "agent": "claude-code", + "model": "claude-sonnet-5", + "tasks": 61, + "solved": 61, + "interventions": 25, + "cost_usd": 20.78, + "turns": 868, + "wall_s": 4479.0, + "tokens_in": 29943843, + "tokens_out": 255704 + }, + { + "path": "outputs/sdebench/hindsight-codex-1/coding/boltons.json", + "dataset": "sdebench", + "run_name": "hindsight-codex-1", + "memory": "hindsight-coding", + "mode": "coding", + "split": "boltons", + "total_queries": 61, + "correct": 61, + "accuracy": 1.0, + "ingestion_time_ms": 0.0, + "ingested_docs": 0, + "avg_retrieve_time_ms": 46752.5, + "avg_context_tokens": 243.0, + "category": null, + "coding": true, + "agent": "codex", + "model": "gpt-5.4-mini", + "tasks": 61, + "solved": 61, + "interventions": 25, + "cost_usd": 15.98, + "turns": 1392, + "wall_s": 2852.0, + "tokens_in": 32507346, + "tokens_out": 465613 + }, + { + "path": "outputs/sdebench/hindsight-codex-2/coding/boltons.json", + "dataset": "sdebench", + "run_name": "hindsight-codex-2", + "memory": "hindsight-coding", + "mode": "coding", + "split": "boltons", + "total_queries": 61, + "correct": 61, + "accuracy": 1.0, + "ingestion_time_ms": 0.0, + "ingested_docs": 0, + "avg_retrieve_time_ms": 47696.7, + "avg_context_tokens": 236.5, + "category": null, + "coding": true, + "agent": "codex", + "model": "gpt-5.4-mini", + "tasks": 61, + "solved": 61, + "interventions": 30, + "cost_usd": 16.78, + "turns": 1450, + "wall_s": 2910.0, + "tokens_in": 34187227, + "tokens_out": 484686 + }, + { + "path": "outputs/sdebench/hindsight-codex-3/coding/boltons.json", + "dataset": "sdebench", + "run_name": "hindsight-codex-3", + "memory": "hindsight-coding", + "mode": "coding", + "split": "boltons", + "total_queries": 61, + "correct": 61, + "accuracy": 1.0, + "ingestion_time_ms": 0.0, + "ingested_docs": 0, + "avg_retrieve_time_ms": 51391.8, + "avg_context_tokens": 226.2, + "category": null, + "coding": true, + "agent": "codex", + "model": "gpt-5.4-mini", + "tasks": 61, + "solved": 61, + "interventions": 31, + "cost_usd": 17.67, + "turns": 1461, + "wall_s": 3135.0, + "tokens_in": 35254187, + "tokens_out": 567986 + }, + { + "path": "outputs/sdebench/hindsight-opencode-1/coding/boltons.json", + "dataset": "sdebench", + "run_name": "hindsight-opencode-1", + "memory": "hindsight-coding", + "mode": "coding", + "split": "boltons", + "total_queries": 61, + "correct": 60, + "accuracy": 0.9836065573770492, + "ingestion_time_ms": 0.0, + "ingested_docs": 0, + "avg_retrieve_time_ms": 171332.8, + "avg_context_tokens": 249.1, + "category": null, + "coding": true, + "agent": "opencode", + "model": "google/gemini-3.5-flash", + "tasks": 61, + "solved": 60, + "interventions": 50, + "cost_usd": 31.77, + "turns": 2135, + "wall_s": 10451.0, + "tokens_in": 54775427, + "tokens_out": 840575 + }, + { + "path": "outputs/sdebench/hindsight-opencode-2/coding/boltons.json", + "dataset": "sdebench", + "run_name": "hindsight-opencode-2", + "memory": "hindsight-coding", + "mode": "coding", + "split": "boltons", + "total_queries": 61, + "correct": 61, + "accuracy": 1.0, + "ingestion_time_ms": 0.0, + "ingested_docs": 0, + "avg_retrieve_time_ms": 160829.5, + "avg_context_tokens": 220.8, + "category": null, + "coding": true, + "agent": "opencode", + "model": "google/gemini-3.5-flash", + "tasks": 61, + "solved": 61, + "interventions": 46, + "cost_usd": 32.3, + "turns": 2115, + "wall_s": 9811.0, + "tokens_in": 53326204, + "tokens_out": 797596 + }, + { + "path": "outputs/sdebench/hindsight-opencode-3/coding/boltons.json", + "dataset": "sdebench", + "run_name": "hindsight-opencode-3", + "memory": "hindsight-coding", + "mode": "coding", + "split": "boltons", + "total_queries": 61, + "correct": 61, + "accuracy": 1.0, + "ingestion_time_ms": 0.0, + "ingested_docs": 0, + "avg_retrieve_time_ms": 157729.5, + "avg_context_tokens": 237.3, + "category": null, + "coding": true, + "agent": "opencode", + "model": "google/gemini-3.5-flash", + "tasks": 61, + "solved": 61, + "interventions": 50, + "cost_usd": 37.14, + "turns": 2298, + "wall_s": 9622.0, + "tokens_in": 62036828, + "tokens_out": 849154 + }, + { + "path": "outputs/sdebench/vanilla-claude-1/coding/boltons.json", + "dataset": "sdebench", + "run_name": "vanilla-claude-1", + "memory": "vanilla", + "mode": "coding", + "split": "boltons", + "total_queries": 61, + "correct": 60, + "accuracy": 0.9836065573770492, + "ingestion_time_ms": 13.2, + "ingested_docs": 14706, + "avg_retrieve_time_ms": 87980.3, + "avg_context_tokens": 7.0, + "category": null, + "coding": true, + "agent": "claude-code", + "model": "claude-sonnet-5", + "tasks": 61, + "solved": 60, + "interventions": 56, + "cost_usd": 27.29, + "turns": 1015, + "wall_s": 5367.0, + "tokens_in": 36732480, + "tokens_out": 390756 + }, + { + "path": "outputs/sdebench/vanilla-claude-2/coding/boltons.json", + "dataset": "sdebench", + "run_name": "vanilla-claude-2", + "memory": "vanilla", + "mode": "coding", + "split": "boltons", + "total_queries": 61, + "correct": 60, + "accuracy": 0.9836065573770492, + "ingestion_time_ms": 13.9, + "ingested_docs": 14706, + "avg_retrieve_time_ms": 80416.4, + "avg_context_tokens": 7.0, + "category": null, + "coding": true, + "agent": "claude-code", + "model": "claude-sonnet-5", + "tasks": 61, + "solved": 60, + "interventions": 50, + "cost_usd": 26.33, + "turns": 971, + "wall_s": 4905.0, + "tokens_in": 34885807, + "tokens_out": 378763 + }, + { + "path": "outputs/sdebench/vanilla-claude-3/coding/boltons.json", + "dataset": "sdebench", + "run_name": "vanilla-claude-3", + "memory": "vanilla", + "mode": "coding", + "split": "boltons", + "total_queries": 61, + "correct": 61, + "accuracy": 1.0, + "ingestion_time_ms": 12.8, + "ingested_docs": 14706, + "avg_retrieve_time_ms": 85563.9, + "avg_context_tokens": 7.0, + "category": null, + "coding": true, + "agent": "claude-code", + "model": "claude-sonnet-5", + "tasks": 61, + "solved": 61, + "interventions": 49, + "cost_usd": 27.75, + "turns": 982, + "wall_s": 5219.0, + "tokens_in": 35801538, + "tokens_out": 403470 + }, + { + "path": "outputs/sdebench/vanilla-codex-1/coding/boltons.json", + "dataset": "sdebench", + "run_name": "vanilla-codex-1", + "memory": "vanilla", + "mode": "coding", + "split": "boltons", + "total_queries": 61, + "correct": 61, + "accuracy": 1.0, + "ingestion_time_ms": 16.8, + "ingested_docs": 14706, + "avg_retrieve_time_ms": 51154.1, + "avg_context_tokens": 7.0, + "category": null, + "coding": true, + "agent": "codex", + "model": "gpt-5.4-mini", + "tasks": 61, + "solved": 61, + "interventions": 80, + "cost_usd": 32.86, + "turns": 1781, + "wall_s": 3120.0, + "tokens_in": 64473703, + "tokens_out": 1113034 + }, + { + "path": "outputs/sdebench/vanilla-codex-2/coding/boltons.json", + "dataset": "sdebench", + "run_name": "vanilla-codex-2", + "memory": "vanilla", + "mode": "coding", + "split": "boltons", + "total_queries": 61, + "correct": 60, + "accuracy": 0.9836065573770492, + "ingestion_time_ms": 18.2, + "ingested_docs": 14706, + "avg_retrieve_time_ms": 53754.1, + "avg_context_tokens": 7.0, + "category": null, + "coding": true, + "agent": "codex", + "model": "gpt-5.4-mini", + "tasks": 61, + "solved": 60, + "interventions": 83, + "cost_usd": 37.65, + "turns": 1867, + "wall_s": 3279.0, + "tokens_in": 74249355, + "tokens_out": 1282729 + }, + { + "path": "outputs/sdebench/vanilla-codex-3/coding/boltons.json", + "dataset": "sdebench", + "run_name": "vanilla-codex-3", + "memory": "vanilla", + "mode": "coding", + "split": "boltons", + "total_queries": 61, + "correct": 61, + "accuracy": 1.0, + "ingestion_time_ms": 14.4, + "ingested_docs": 14706, + "avg_retrieve_time_ms": 55793.4, + "avg_context_tokens": 7.0, + "category": null, + "coding": true, + "agent": "codex", + "model": "gpt-5.4-mini", + "tasks": 61, + "solved": 61, + "interventions": 83, + "cost_usd": 35.12, + "turns": 1869, + "wall_s": 3403.0, + "tokens_in": 68638315, + "tokens_out": 1239311 + }, + { + "path": "outputs/sdebench/vanilla-opencode-1/coding/boltons.json", + "dataset": "sdebench", + "run_name": "vanilla-opencode-1", + "memory": "vanilla", + "mode": "coding", + "split": "boltons", + "total_queries": 61, + "correct": 61, + "accuracy": 1.0, + "ingestion_time_ms": 29.3, + "ingested_docs": 14706, + "avg_retrieve_time_ms": 185123.0, + "avg_context_tokens": 7.0, + "category": null, + "coding": true, + "agent": "opencode", + "model": "google/gemini-3.5-flash", + "tasks": 61, + "solved": 61, + "interventions": 74, + "cost_usd": 41.66, + "turns": 2369, + "wall_s": 11292.0, + "tokens_in": 65435546, + "tokens_out": 1121445 + }, + { + "path": "outputs/sdebench/vanilla-opencode-2/coding/boltons.json", + "dataset": "sdebench", + "run_name": "vanilla-opencode-2", + "memory": "vanilla", + "mode": "coding", + "split": "boltons", + "total_queries": 61, + "correct": 61, + "accuracy": 1.0, + "ingestion_time_ms": 14.8, + "ingested_docs": 14706, + "avg_retrieve_time_ms": 162785.2, + "avg_context_tokens": 7.0, + "category": null, + "coding": true, + "agent": "opencode", + "model": "google/gemini-3.5-flash", + "tasks": 61, + "solved": 61, + "interventions": 76, + "cost_usd": 39.52, + "turns": 2325, + "wall_s": 9930.0, + "tokens_in": 72344900, + "tokens_out": 1115870 + }, + { + "path": "outputs/sdebench/vanilla-opencode-3/coding/boltons.json", + "dataset": "sdebench", + "run_name": "vanilla-opencode-3", + "memory": "vanilla", + "mode": "coding", + "split": "boltons", + "total_queries": 61, + "correct": 61, + "accuracy": 1.0, + "ingestion_time_ms": 18.8, + "ingested_docs": 14706, + "avg_retrieve_time_ms": 174832.8, + "avg_context_tokens": 7.0, + "category": null, + "coding": true, + "agent": "opencode", + "model": "google/gemini-3.5-flash", + "tasks": 61, + "solved": 61, + "interventions": 69, + "cost_usd": 34.9, + "turns": 2427, + "wall_s": 10665.0, + "tokens_in": 67479034, + "tokens_out": 1049086 + }, + { + "path": "outputs/beam/valorbrain/rag/100k.json", + "dataset": "beam", + "run_name": "valorbrain", + "memory": "valorbrain", + "mode": "rag", + "split": "100k", + "total_queries": 400, + "correct": 304, + "accuracy": 0.7140573901252293, + "ingestion_time_ms": 35941.7, + "ingested_docs": 170, + "avg_retrieve_time_ms": 5186.5, + "avg_context_tokens": 12643.8, + "category": null + } ] \ No newline at end of file diff --git a/src/memory_bench/cli.py b/src/memory_bench/cli.py index 55ff261..738c20d 100644 --- a/src/memory_bench/cli.py +++ b/src/memory_bench/cli.py @@ -24,10 +24,9 @@ def _resolve_gemini_key() -> None: key = os.environ.get("GEMINI_API_KEY") or os.environ.get("GOOGLE_API_KEY") - if not key: - typer.echo("Error: GEMINI_API_KEY environment variable is not set.", err=True) - raise typer.Exit(1) - os.environ["GOOGLE_API_KEY"] = key + if key: + os.environ["GOOGLE_API_KEY"] = key + # No key? Non-fatal — GeminiLLM will error only if actually instantiated. @app.command() diff --git a/src/memory_bench/llm/__init__.py b/src/memory_bench/llm/__init__.py index 99be8ec..a627cd4 100644 --- a/src/memory_bench/llm/__init__.py +++ b/src/memory_bench/llm/__init__.py @@ -3,12 +3,18 @@ from .base import LLM, Schema from .gemini import GeminiLLM from .groq import GroqLLM +from .gateway import GatewayLLM from .openai import OpenAILLM +from .glm import GlmLLM +from .agy_direct import AgyDirectLLM REGISTRY: dict[str, type[LLM]] = { "gemini": GeminiLLM, "groq": GroqLLM, "openai": OpenAILLM, + "gateway": GatewayLLM, + "glm": GlmLLM, + "agy-direct": AgyDirectLLM, } diff --git a/src/memory_bench/llm/agy_direct.py b/src/memory_bench/llm/agy_direct.py new file mode 100644 index 0000000..cfb02f1 --- /dev/null +++ b/src/memory_bench/llm/agy_direct.py @@ -0,0 +1,75 @@ +""" +AGY direct LLM — calls Antigravity CLI via subprocess, no HTTP gateway. +Eliminates the gateway as a failure point. +""" + +import json +import os +import subprocess + +from .base import LLM, Schema + +_MAX_RETRIES = 3 + + +class AgyDirectLLM(LLM): + """Calls agy CLI directly. No HTTP gateway, no intermediary process.""" + + def __init__(self, model: str | None = None): + self._model = model or os.environ.get("AGY_MODEL", "gemini-3.6-flash") + self._effort = os.environ.get("AGY_EFFORT", "low") + + @property + def model_id(self) -> str: + return f"agy:{self._model}" + + def generate(self, prompt: str, schema: Schema) -> dict: + fields_desc = ", ".join(schema.required) + full_prompt = prompt + f"\n\nRespond as a JSON object with these fields: {fields_desc}. Output ONLY valid JSON." + + for attempt in range(_MAX_RETRIES): + try: + result = subprocess.run( + ["agy", "--model", self._model, "--effort", self._effort, + "-p", full_prompt, "--output-format", "json"], + capture_output=True, text=True, timeout=120, + env={**os.environ}, + ) + if result.returncode != 0: + raise RuntimeError(f"agy exit {result.returncode}: {result.stderr[:200]}") + + data = json.loads(result.stdout) + if data.get("status") == "SUCCESS": + response = data.get("response", "") + return self._parse_json(response) + + raise RuntimeError(f"agy error: {data.get('error', 'unknown')}") + + except subprocess.TimeoutExpired: + if attempt < _MAX_RETRIES - 1: + import time; time.sleep(2) + continue + raise + except Exception: + if attempt < _MAX_RETRIES - 1: + import time; time.sleep(2) + continue + raise + + def _parse_json(self, text: str) -> dict: + import re + text = text.strip() + if text.startswith("```"): + text = re.sub(r"^```(?:json)?\s*", "", text) + text = re.sub(r"\s*```$", "", text) + try: + return json.loads(text) + except json.JSONDecodeError: + pass + match = re.search(r'\{[^{}]*(?:\{[^{}]*\}[^{}]*)*\}', text, re.DOTALL) + if match: + try: + return json.loads(match.group()) + except json.JSONDecodeError: + pass + return {f: "" for f in ["reasoning", "answer", "choice", "reason", "correct"]} diff --git a/src/memory_bench/llm/gateway.py b/src/memory_bench/llm/gateway.py new file mode 100644 index 0000000..9a8d505 --- /dev/null +++ b/src/memory_bench/llm/gateway.py @@ -0,0 +1,124 @@ +""" +Gateway LLM for AMB — talks to an OpenAI-compatible endpoint without +response_format support (e.g. opencode-go gateway serving deepseek-v4-flash). + +Uses prompt-based JSON extraction instead of structured output. +""" + +import json +import os +import re +import time +import urllib.request + +from .base import LLM, Schema + +_MAX_RETRIES = 4 +_RETRY_DELAY = 5 + + +class GatewayLLM(LLM): + """OpenAI-compatible LLM via plain HTTP, no response_format dependency.""" + + def __init__(self, model: str | None = None): + self._base = os.environ.get("OPENAI_BASE_URL", "http://localhost:8201/v1").rstrip("/") + self._key = os.environ.get("OPENAI_API_KEY", "") + self._model = model or os.environ.get("OMB_GATEWAY_MODEL", "deepseek-v4-flash") + + @property + def model_id(self) -> str: + return f"gateway:{self._model}" + + def generate(self, prompt: str, schema: Schema) -> dict: + # Inject JSON instruction into the prompt + fields_desc = ", ".join(schema.required) + json_instruction = ( + f"\n\nRespond as a JSON object with these fields: {fields_desc}. " + "Output ONLY valid JSON, no markdown fences, no prose." + ) + full_prompt = prompt + json_instruction + + delay = _RETRY_DELAY + for attempt in range(_MAX_RETRIES): + try: + data = self._call(full_prompt) + return self._parse_json(data) + except Exception as e: + if attempt < _MAX_RETRIES - 1: + time.sleep(delay) + delay *= 2 + else: + raise RuntimeError(f"GatewayLLM failed after {_MAX_RETRIES} retries: {e}") + + def _call(self, prompt: str) -> str: + url = f"{self._base}/chat/completions" + # Reasoning models (deepseek-v4-flash) spend thousands of tokens on + # thinking before the content lands — 8192 starved long reader prompts + # and the empty content silently became an empty answer (same failure + # class the GLM judge had before its fix). + max_tokens = int(os.environ.get("OMB_GATEWAY_MAX_TOKENS", "16384")) + body = json.dumps({ + "model": self._model, + "messages": [{"role": "user", "content": prompt}], + "max_tokens": max_tokens, + "temperature": 0, + }).encode("utf-8") + + req = urllib.request.Request( + url, + data=body, + headers={ + "Content-Type": "application/json", + # opencode.ai's edge 403s the default "Python-urllib/3.x" UA + # (WAF) while the same request from curl passes — send a + # neutral client UA instead. + "User-Agent": os.environ.get("OMB_GATEWAY_USER_AGENT", "amb-gateway/1.0"), + **({"Authorization": f"Bearer {self._key}"} if self._key else {}), + }, + method="POST", + ) + + with urllib.request.urlopen(req, timeout=180) as resp: + data = json.loads(resp.read().decode("utf-8")) + msg = data.get("choices", [{}])[0].get("message", {}) + content = (msg.get("content") or "").strip() + if content: + return content + # Reasoning fallback: when content is truncated away, the whole + # answer can live inside reasoning_content. + return (msg.get("reasoning_content") or "").strip() + + def _parse_json(self, text: str) -> dict: + # Strip markdown fences if present + text = text.strip() + if text.startswith("```"): + text = re.sub(r"^```(?:json)?\s*", "", text) + text = re.sub(r"\s*```$", "", text) + + # Try direct parse first + try: + parsed = json.loads(text) + except json.JSONDecodeError: + parsed = None + + if isinstance(parsed, dict): + return parsed + + # Try to extract JSON object from text + match = re.search(r'\{[^{}]*(?:\{[^{}]*\}[^{}]*)*\}', text, re.DOTALL) + if match: + try: + return json.loads(match.group()) + except json.JSONDecodeError: + pass + + # The model answered in prose (or a bare array) instead of the JSON + # object — that IS its answer, not an empty one. Ship it as answer + # instead of silently zeroing the query. + if isinstance(parsed, list): + return {"answer": json.dumps(parsed, ensure_ascii=False)} + if text: + return {"answer": text} + + # Last resort: return empty fields + return {field: "" for field in ["reasoning", "answer", "choice", "reason", "correct"]} diff --git a/src/memory_bench/llm/glm.py b/src/memory_bench/llm/glm.py new file mode 100644 index 0000000..f44cd33 --- /dev/null +++ b/src/memory_bench/llm/glm.py @@ -0,0 +1,130 @@ +""" +GLM-5.2 LLM for AMB — via Z.ai coding plan endpoint. + +GLM-5.2 is a reasoning model: outputs reasoning_content separately from content. +Uses prompt-based JSON extraction (Z.ai doesn't support response_format). + +Tuned for benchmark use (2026-08-09): +- max_tokens=16384: GLM-5.2 reasoning can consume 5k-10k tokens before producing + the actual content. 8192 was too low — judge prompts with long answers saw + reasoning eat the entire budget, leaving content empty → score=0 fallback. +- Request is rebuilt every attempt (urllib Request objects are single-use). +- Reasoning-only responses are parsed for trailing JSON (GLM sometimes emits + the score inside reasoning_content when content is truncated). +- Structured logging on failure for post-mortem. +""" + +import json +import logging +import os +import time as _time +import urllib.request + +from .gateway import GatewayLLM + +logger = logging.getLogger(__name__) + +# GLM-5.2 reasoning consumes many tokens before the answer lands in `content`. +# 16k gives headroom for long judge prompts (answer + context + rubric). +# Coding plan has quota; benchmark correctness > token economy here. +_MAX_TOKENS = int(os.environ.get("GLM_MAX_TOKENS", "16384")) + +# Per-request timeout — reasoning on long prompts can take 2-3 min. +_TIMEOUT = int(os.environ.get("GLM_TIMEOUT", "240")) + + +class GlmLLM(GatewayLLM): + """GLM-5.2 via Z.ai coding plan. Expensive per-request, use sparingly.""" + + def __init__(self, model: str | None = None): + self._base = os.environ.get( + "GLM_BASE_URL", "https://api.z.ai/api/coding/paas/v4" + ).rstrip("/") + self._key = os.environ.get("GLM_API_KEY", "") + self._model = model or os.environ.get("GLM_MODEL", "glm-5.2") + + @property + def model_id(self) -> str: + return f"glm:{self._model}" + + def _call(self, prompt: str) -> str: + url = f"{self._base}/chat/completions" + body_dict = { + "model": self._model, + "messages": [{"role": "user", "content": prompt}], + "max_tokens": _MAX_TOKENS, + "temperature": 0, + } + + last_err = None + for attempt in range(3): + # Rebuild request every attempt — urllib Request is single-use + # after urlopen consumes the body stream. + body = json.dumps(body_dict).encode("utf-8") + req = urllib.request.Request( + url, + data=body, + headers={ + "Content-Type": "application/json", + **({"Authorization": f"Bearer {self._key}"} if self._key else {}), + }, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=_TIMEOUT) as resp: + data = json.loads(resp.read().decode("utf-8")) + msg = data.get("choices", [{}])[0].get("message", {}) + content = msg.get("content", "").strip() + reasoning = msg.get("reasoning_content", "").strip() + finish = data.get("choices", [{}])[0].get("finish_reason", "") + + if content: + return content + + # Content empty but reasoning has the answer — GLM sometimes + # emits the JSON score at the end of reasoning_content when + # content gets truncated. Try to salvage it. + if reasoning: + salvaged = _extract_json_from_text(reasoning) + if salvaged: + logger.info( + "[glm] salvaged JSON from reasoning_content " + "(attempt %d, finish=%s, reasoning=%d chars)", + attempt + 1, finish, len(reasoning), + ) + return salvaged + + last_err = f"empty content (finish={finish}, reasoning={len(reasoning)} chars)" + logger.warning( + "[glm] empty response attempt %d/%d: %s", + attempt + 1, 3, last_err, + ) + except Exception as e: + last_err = str(e) + logger.warning("[glm] error attempt %d/%d: %s", attempt + 1, 3, last_err) + + if attempt < 2: + _time.sleep(5 * (attempt + 1)) # 5s, 10s backoff + + logger.error("[glm] all 3 attempts failed: %s", last_err) + return "" + + +def _extract_json_from_text(text: str) -> str: + """Try to find a JSON object ({...}) in text — GLM reasoning fallback. + + Looks for the last JSON object in the text (reasoning often ends with + the final answer). Returns the raw JSON string, or "" if none found. + """ + # Look for ```json ... ``` blocks first (GLM markdown-wraps sometimes) + import re + md = re.search(r"```(?:json)?\s*(\{[^`]+\})\s*```", text, re.IGNORECASE) + if md: + return md.group(1).strip() + + # Fallback: last {...} block in the text + matches = re.findall(r"\{[^{}]*\"score\"[^{}]*\}", text) + if matches: + return matches[-1].strip() + + return "" diff --git a/src/memory_bench/memory/__init__.py b/src/memory_bench/memory/__init__.py index 2b7e5e0..66e859e 100644 --- a/src/memory_bench/memory/__init__.py +++ b/src/memory_bench/memory/__init__.py @@ -9,6 +9,7 @@ from .hybrid_search import HybridSearchMemoryProvider from .ogham import OghamMemoryProvider from .supermemory import SupermemoryMemoryProvider +from .valorbrain import ValorBrainMemoryProvider from .none import NoMemoryProvider from .hscoding import HsCodingProvider @@ -28,6 +29,7 @@ "ogham": OghamMemoryProvider, "qdrant": HybridSearchMemoryProvider, "supermemory": SupermemoryMemoryProvider, + "valorbrain": ValorBrainMemoryProvider, } # legacy aliases (docs/scripts used these); canonical names above REGISTRY["none"] = NoMemoryProvider diff --git a/src/memory_bench/memory/valorbrain.py b/src/memory_bench/memory/valorbrain.py new file mode 100644 index 0000000..dea4bd6 --- /dev/null +++ b/src/memory_bench/memory/valorbrain.py @@ -0,0 +1,253 @@ +""" +ValorBrain memory provider for the Agent Memory Benchmark. + +ValorBrain is a hybrid memory engine: BM25 + dense vectors (pgvector) + RRF + +graph reranking + BGE cross-encoder rerank, all on PostgreSQL with RLS. + +This provider talks to a running ValorBrain engine instance via its REST API. +Set VALORBRAIN_URL (default http://localhost:7438) and VALORBRAIN_TOKEN. + +Each AMB isolation unit (e.g. BEAM conversation) becomes a ValorBrain collection. +""" + +import json +import logging +import os +import time +import urllib.parse +import urllib.request + +from ..models import Document +from .base import MemoryProvider + +logger = logging.getLogger(__name__) + + +class ValorBrainMemoryProvider(MemoryProvider): + name = "valorbrain" + description = ( + "ValorBrain hybrid memory engine — BM25 + dense (pgvector) + RRF + " + "graph reranking + BGE cross-encoder rerank on PostgreSQL." + ) + kind = "cloud" + link = "https://valor.digital" + concurrency = 4 + + def __init__(self): + self._base = os.environ.get("VALORBRAIN_URL", "http://localhost:7438").rstrip("/") + self._token = os.environ.get("VALORBRAIN_TOKEN", "") + self._tenant = os.environ.get("VALORBRAIN_BENCHMARK_TENANT_ID", "") + self._ingested_collections: set[str] = set() + + # ── HTTP helper ────────────────────────────────────────────────────── + + def _post(self, path: str, body: dict, timeout: float = 120) -> dict: + url = f"{self._base}{path}" + data = json.dumps(body).encode("utf-8") + req = urllib.request.Request( + url, + data=data, + headers={ + "Content-Type": "application/json", + **({"x-tenant-id": self._tenant} if self._tenant else {}), + **({"Authorization": f"Bearer {self._token}"} if self._token else {}), + }, + method="POST", + ) + with urllib.request.urlopen(req, timeout=timeout) as resp: + return json.loads(resp.read().decode("utf-8")) + + def _get_stats_collection(self, collection: str, timeout: float = 30) -> dict: + """GET /stats?collection=X. + + /stats is a GET endpoint; POSTing a body to it returns {"error": ...}. + The provider used _post("/stats", {...}) for both the exists-check and + the index-wait — both silently dead (every check saw an error response, + existence never matched, _wait_index burned its full 120s timeout per + collection). This GET uses the real contract: response carries + collectionDocuments / collectionHybridPending / collectionEmbedFailed. + """ + url = f"{self._base}/stats?collection={urllib.parse.quote(collection)}" + req = urllib.request.Request( + url, + headers={ + **({"x-tenant-id": self._tenant} if self._tenant else {}), + **({"Authorization": f"Bearer {self._token}"} if self._token else {}), + }, + method="GET", + ) + with urllib.request.urlopen(req, timeout=timeout) as resp: + return json.loads(resp.read().decode("utf-8")) + + # ── Provider interface ─────────────────────────────────────────────── + + def ingest(self, documents: list[Document]) -> None: + # Re-window large chunks into ~8k-char windows (matching production's + # 6-turn windows). The AMB chunks at 100k chars; our retrieval expects + # ~38 smaller windows per conversation, not 7 huge ones. Without this, + # the search pool is too small for good coverage. + WINDOW_SIZE = 8000 + WINDOW_OVERLAP = 800 # ~1 turn overlap, like our janelaTurnos-1 + collections_seen: set[str] = set() + for doc in documents: + raw = (doc.user_id or "amb-default").lower() + # Check if production-chunked collection already exists (beam-100k-N) + existing = f"beam-100k-{raw}" if raw.isdigit() else raw + try: + stats = self._get_stats_collection(existing, timeout=15) + if stats.get("collectionDocuments", 0) > 0: + self._ingested_collections.add(existing) + continue # skip ingest — data already chunked and indexed + except Exception: + pass + collection = existing # use beam-100k-{raw} consistently with retrieve() + collections_seen.add(collection) + + content = doc.content + if len(content) <= WINDOW_SIZE: + windows = [(doc.id or f"{collection}/0", content)] + else: + windows = [] + wi = 0 + pos = 0 + while pos < len(content): + end = min(pos + WINDOW_SIZE, len(content)) + windows.append((f"{doc.id}_w{wi}", content[pos:end])) + wi += 1 + if end >= len(content): + break + pos = end - WINDOW_OVERLAP + + for w_path, w_content in windows: + body = { + "content": w_content, + "collection": collection, + "path": w_path, + "content_type": "conversation", + } + if doc.timestamp: + body["event_at"] = doc.timestamp + try: + self._post("/documents", body, timeout=300) + except Exception as e: + logger.warning("ValorBrain ingest failed for %s/%s: %s", collection, w_path, e) + + # Wait for hybrid index on each collection that received documents. + for collection in collections_seen: + if collection not in self._ingested_collections: + self._wait_index(collection) + self._refine_collection(collection) + self._ingested_collections.add(collection) + + def _wait_index(self, collection: str, timeout: float = 120) -> None: + """Poll GET /stats?collection= until the collection's hybrid index is ready. + + Bails early when documents exist, nothing is pending, and some embeds + have permanently failed — those will not recover within the timeout + (the embed worker retries failed docs on an ~1h cadence), so waiting + the full window only stalls the run. + """ + deadline = time.time() + timeout + while time.time() < deadline: + try: + stats = self._get_stats_collection(collection, timeout=30) + pending = stats.get("collectionHybridPending", 0) + total = stats.get("collectionDocuments", 0) + failed = stats.get("collectionEmbedFailed", 0) + if total > 0 and pending == 0: + return + if total > 0 and pending == failed and failed > 0: + return # everything indexed that will be indexed + except Exception: + pass + time.sleep(2) + + def _refine_collection(self, collection: str, timeout: float = 300) -> None: + """Extract observations + consolidate after ingest, before queries. + + Without this, /memory/prepare queries arrive before Phase 1.5 has + extracted observations from the conversation docs. The answering LLM + gets raw text instead of pre-extracted facts. + """ + try: + result = self._post( + "/api/v1/memory/refine", + {"collection": collection}, + timeout=timeout, + ) + logger.info( + "Refine %s: observed=%s extracted=%s consolidated=%s", + collection, + result.get("observed", 0), + result.get("extracted", 0), + result.get("consolidated", 0), + ) + except Exception as e: + logger.warning("Refine failed for %s: %s (continuing)", collection, e) + + def retrieve( + self, + query: str, + k: int = 20, + user_id: str | None = None, + query_timestamp: str | None = None, + ) -> tuple[list[Document], dict | None]: + # Map AMB user_id (e.g. "1") to our production chunked collections + # (e.g. "beam-100k-1") which have proper 6-turn windows (38 docs/conv). + raw = user_id or "amb-default" + collection = f"beam-100k-{raw}" if raw.isdigit() else raw + + # /memory/prepare delivers the full pipeline (funnel + multitrecho + rerank). + # delivered_documents are snippeted server-side (6k). This is the production + # path — same endpoint Hermes uses, validated by the benchmark. + body: dict = {"message": query, "collection": collection} + try: + data = self._post("/api/v1/memory/prepare", body, timeout=60) + funnel = data.get("funnel") or {} + delivered = funnel.get("delivered_documents", []) + if delivered: + # Separate synthetic docs (digest, facts) from conversation windows. + # Embed synthetic as a header at the TOP of the first memory, + # so the reader sees it as context, not as competing memories. + synthetic = [d for d in delivered if d.get("path", "").startswith("__")] + conversations = [d for d in delivered if not d.get("path", "").startswith("__")] + + synth_text = "" + if synthetic: + parts = [] + for d in synthetic: + c = d.get("content", "").strip() + if c: + parts.append(c) + synth_text = "\n\n".join(parts) + + docs = [] + for i, d in enumerate(conversations): + content = d.get("content", "") + if not content: + continue + # Prepend synthetic context to the FIRST conversation doc + if i == 0 and synth_text: + content = ( + "=== CONVERSATION SUMMARY (use for factual questions) ===\n" + + synth_text + + "\n=== END SUMMARY ===\n\n" + + content + ) + docs.append(Document(id=d.get("path", ""), content=content, user_id=user_id)) + + if docs: + return docs, data + except Exception as e: + logger.warning("ValorBrain prepare failed: %s", e) + + # Fallback: /search + try: + data = self._post("/search", {"query": query, "mode": "hybrid", "limit": k, + "collection": collection, "compact": False}, timeout=60) + except Exception as e: + return [], {"error": str(e)} + docs = [Document(id=r.get("docid",""), content=(r.get("body") or r.get("snippet",""))[:6000], user_id=user_id) + for r in data.get("results",[]) if r.get("body") or r.get("snippet")] + return docs, data + diff --git a/src/memory_bench/runner.py b/src/memory_bench/runner.py index 493f91e..935e262 100644 --- a/src/memory_bench/runner.py +++ b/src/memory_bench/runner.py @@ -42,6 +42,11 @@ def __init__(self, output_dir: Path = Path("outputs")): self._judge = GeminiJudge() def _get_judge(self, dataset: Dataset) -> "GeminiJudge": + # Respect OMB_JUDGE_LLM when set; fall back to dataset default. + import os + override = os.environ.get("OMB_JUDGE_LLM") + if override and override != "gemini": + return self._judge dataset_llm = dataset.default_judge_llm() if hasattr(dataset, "default_judge_llm") else None if dataset_llm is not None: return GeminiJudge(llm=dataset_llm) @@ -225,6 +230,14 @@ async def _process_one_attempt(q) -> QueryResult: judge_llm = self._get_judge(dataset)._llm score = await asyncio.to_thread(dataset.score_result, tmp_result, judge_llm) score = float(score) + # Retry judge if answer is non-empty but score is 0 (likely judge failure). + # Observed: 45/400 questions get score=0 with valid answers due to + # AGY/Gemini quota or inconsistency. Retry once before accepting. + if score == 0.0 and answer_result.answer.strip(): + import time as _t + _t.sleep(3) + score = await asyncio.to_thread(dataset.score_result, tmp_result, judge_llm) + score = float(score) correct = score >= 0.5 judge_reason = f"score={score:.3f}" else: