From baa32dcdd25de07eb90a705398d323268682c4df Mon Sep 17 00:00:00 2001
From: Matthew Wesney <Devaux.mail@gmail.com>
Date: Mon, 10 Aug 2026 11:10:38 -0400
Subject: [PATCH] Raise the default context window from 4096 to 8192

A user reported chats losing continuity, refined after testing to "larger
system prompts seem to be overloading the model." That diagnosis was right,
but not for the reason it looks like -- this has nothing to do with model
size or attention quality.

Traced the full path first: a real two-turn conversation driven through the
actual /api/v1/generations -> jobs -> GenerationService orchestration showed
turn 2 genuinely receiving turn 1's full persisted history. Persistence and
loading are not the problem.

The problem is SynthesisAgent.fit_history_to_context, working exactly as
designed against a budget that was too small. num_ctx is one Cortex setting
applied the same way regardless of which model is selected -- it is not
tied to the model's own native context. Measuring the real trimmer (not the
test double, which skips trimming and would have hidden this) against a
realistic 30-exchange conversation, at the old default of 4096:

    memory on, code-exec off:            10/30 exchanges survived
    memory on, code-exec eligible:        6/30 exchanges survived
    + a handful of stored memories:       4/30 exchanges survived

The built-in system_prompt.txt + memory_prompt.txt cost ~1332 tokens before a
single word of the conversation is counted; add the code-execution JIT prompt
(triggered by ordinary phrases like "run this") and it climbs to ~1973 -- out
of a 3072-token prompt budget (num_ctx minus the 1024-token output reservation).
Every one of the models actually installed on this machine natively supports
far more (qwen3:8b alone is 40960), so history was being discarded well
within what the model could have used, and nothing in the UI said so -- it
just looked like the model forgot.

8192 was verified against the worst realistic combination (memory + stored
facts + code-execution eligibility + custom system instructions) and keeps
the full 30-exchange conversation. Left the existing 16384 ceiling alone --
raising it trades directly against local RAM/VRAM for the KV cache, which is
a real product decision, not a bug fix.

Changed everywhere a copy of the old default lived: the Pydantic field, two
defensive num_ctx fallbacks in services/generation.py and services/llm.py
(hit when model_options lacks the key, which a real snapshot never does, but
now stays in step with the real default instead of quietly reintroducing the
old bug), and three frontend fallback constants shown before Settings loads.

Added a regression test that reads the shipped default from CortexSettings
rather than hardcoding it, so it stays meaningful if the default changes
again, and fails loudly with the exact surviving-exchange count if it ever
regresses.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
---
 backend/cortex_backend/core/settings.py       |  9 +++-
 backend/cortex_backend/services/generation.py |  6 ++-
 backend/cortex_backend/services/llm.py        |  5 +-
 contracts/openapi.json                        |  2 +-
 frontend/src/features/chat/ChatPage.tsx       |  2 +-
 .../src/features/chat/MessageComposer.tsx     |  2 +-
 .../src/features/settings/SettingsPanel.tsx   |  2 +-
 tests/test_chat_correctness.py                | 51 +++++++++++++++++++
 tests/test_settings_compatibility.py          |  2 +-
 9 files changed, 73 insertions(+), 8 deletions(-)

diff --git a/backend/cortex_backend/core/settings.py b/backend/cortex_backend/core/settings.py
index b06b940..0b323b7 100644
--- a/backend/cortex_backend/core/settings.py
+++ b/backend/cortex_backend/core/settings.py
@@ -54,7 +54,14 @@ class GenerationSettings(_SettingsModel):
     top_p: float = Field(default=0.9, ge=0.0, le=1.0)
     top_k: int = Field(default=40, ge=0, le=200)
     repeat_penalty: float = Field(default=1.1, ge=0.5, le=2.0)
-    num_ctx: int = Field(default=4096, ge=2048, le=16384)
+    # 4096 measured out at only 4-10 of 30 realistic exchanges surviving the
+    # context-budget trim once the built-in system/memory/code-execution
+    # prompts (up to ~2000 tokens) were accounted for -- history was being
+    # silently discarded well within what every locally installed model
+    # actually supports (the smallest here is 40960). 8192 keeps the full
+    # 30-exchange conversation even with memories and code-execution eligibility
+    # both active; see the discussion around PR raising this default.
+    num_ctx: int = Field(default=8192, ge=2048, le=16384)
     seed: int = Field(default=-1, ge=-1, le=2147483647)
     # No length cap: whatever doesn't fit in the configured context window is
     # already handled gracefully by the history/memory/attachment budget
diff --git a/backend/cortex_backend/services/generation.py b/backend/cortex_backend/services/generation.py
index 20bdf22..722d552 100644
--- a/backend/cortex_backend/services/generation.py
+++ b/backend/cortex_backend/services/generation.py
@@ -112,7 +112,11 @@ def generate(
         permanent_memories = (
             list(self._memory_loader()) if snapshot.memories_enabled else []
         )
-        num_ctx = int(snapshot.model_options.get("num_ctx", 4096))
+        # A real snapshot always carries num_ctx (GENERATION_OVERRIDE_FIELDS
+        # guarantees it); this fallback only matters for callers that build
+        # model_options by hand, so it stays in step with GenerationSettings'
+        # own default rather than reintroducing the old, too-small one.
+        num_ctx = int(snapshot.model_options.get("num_ctx", 8192))
         if snapshot.memories_enabled:
             self._publish(sink, snapshot, "thoughts", "Gathering thoughts...")
             engine = self._engine_factory(snapshot)
diff --git a/backend/cortex_backend/services/llm.py b/backend/cortex_backend/services/llm.py
index ffb9455..8c9901a 100644
--- a/backend/cortex_backend/services/llm.py
+++ b/backend/cortex_backend/services/llm.py
@@ -654,7 +654,10 @@ def generate(
             permanent_memories=permanent_memories,
             memories_enabled=memories_enabled,
             user_system_instructions=user_system_instructions,
-            num_ctx=int(api_options.get("num_ctx", 4096)),
+            # Kept in step with GenerationSettings.num_ctx's own default --
+            # a real call always carries num_ctx, so this only matters for
+            # options built by hand without one.
+            num_ctx=int(api_options.get("num_ctx", 8192)),
             code_execution_eligible=self.code_execution_eligible,
             bypass_system_prompt=self.bypass_system_prompt,
         )
diff --git a/contracts/openapi.json b/contracts/openapi.json
index c98f8c0..7ce8e97 100644
--- a/contracts/openapi.json
+++ b/contracts/openapi.json
@@ -1792,7 +1792,7 @@
             "type": "boolean"
           },
           "num_ctx": {
-            "default": 4096,
+            "default": 8192,
             "maximum": 16384.0,
             "minimum": 2048.0,
             "title": "Num Ctx",
diff --git a/frontend/src/features/chat/ChatPage.tsx b/frontend/src/features/chat/ChatPage.tsx
index e5f2324..17092cc 100644
--- a/frontend/src/features/chat/ChatPage.tsx
+++ b/frontend/src/features/chat/ChatPage.tsx
@@ -16,7 +16,7 @@ const DEFAULT_GENERATION_SETTINGS = {
   top_p: 0.9,
   top_k: 40,
   repeat_penalty: 1.1,
-  num_ctx: 4096,
+  num_ctx: 8192,
   seed: -1,
   system_instructions: "",
 };
diff --git a/frontend/src/features/chat/MessageComposer.tsx b/frontend/src/features/chat/MessageComposer.tsx
index 0a6a1a4..46a2382 100644
--- a/frontend/src/features/chat/MessageComposer.tsx
+++ b/frontend/src/features/chat/MessageComposer.tsx
@@ -88,7 +88,7 @@ const FALLBACK_GENERATION_DEFAULTS: GenerationSettings = {
   top_p: 0.9,
   top_k: 40,
   repeat_penalty: 1.1,
-  num_ctx: 4096,
+  num_ctx: 8192,
   seed: -1,
 };
 
diff --git a/frontend/src/features/settings/SettingsPanel.tsx b/frontend/src/features/settings/SettingsPanel.tsx
index ebe7300..083432e 100644
--- a/frontend/src/features/settings/SettingsPanel.tsx
+++ b/frontend/src/features/settings/SettingsPanel.tsx
@@ -204,7 +204,7 @@ export function SettingsPanel({
                 </div>
                 <div className="settings-field-row">
                   <label className="field-label" htmlFor="num-ctx">Context window
-                    <input id="num-ctx" type="number" min="2048" max="16384" step="1024" value={generation.num_ctx ?? 4096} onChange={(event) => update({ generation: { ...generation, num_ctx: Number(event.target.value) } })} />
+                    <input id="num-ctx" type="number" min="2048" max="16384" step="1024" value={generation.num_ctx ?? 8192} onChange={(event) => update({ generation: { ...generation, num_ctx: Number(event.target.value) } })} />
                   </label>
                   <label className="field-label" htmlFor="seed">Seed
                     <input id="seed" type="number" min="-1" max="2147483647" value={generation.seed ?? -1} onChange={(event) => update({ generation: { ...generation, seed: Number(event.target.value) } })} />
diff --git a/tests/test_chat_correctness.py b/tests/test_chat_correctness.py
index 6dfdb24..b8bc6bc 100644
--- a/tests/test_chat_correctness.py
+++ b/tests/test_chat_correctness.py
@@ -8,6 +8,7 @@
 from cortex_backend.repositories.chats import InMemoryChatRepository, LegacyDatabaseChatRepository
 from cortex_backend.repositories.legacy_storage import DatabaseManager
 from cortex_backend.core.generation import GenerationAttachment
+from cortex_backend.core.settings import CortexSettings
 from cortex_backend.services.llm import SynthesisAgent
 
 
@@ -117,6 +118,56 @@ def test_context_budget_keeps_recent_history_and_reserves_output(self):
         self.assertNotIn("old-0", history)
         self.assertEqual(SynthesisAgent.output_token_reservation(4096), 1024)
 
+    def test_default_context_window_survives_a_realistic_long_conversation(self):
+        """Regression guard for a bug where the shipped num_ctx default was
+        small enough that ordinary conversations lost most of their history
+        to the context-budget trim -- not because any model "forgot", but
+        because the built-in system/memory/code-execution prompts (up to
+        ~2000 tokens) ate most of an already-small budget before a single
+        word of the conversation was counted. At the old 4096 default, a
+        30-exchange conversation like this one kept as few as 4 of 30
+        exchanges. Reads the default from CortexSettings rather than
+        hardcoding it, so this stays meaningful if the default changes again.
+        """
+        turn = "Can you walk me through why the connection pool keeps timing out under load?"
+        reply = (
+            "The timeout usually means every connection is checked out and none are "
+            "returned before the next request needs one. Check whether connections "
+            "are closed in a finally block even on exceptions, and whether the pool "
+            "size actually matches your real concurrency."
+        )
+        messages = []
+        for index in range(30):
+            messages.append({"role": "user", "content": f"{turn} (turn {index})"})
+            messages.append({"role": "assistant", "content": f"{reply} (turn {index})"})
+
+        default_num_ctx = CortexSettings().generation.num_ctx
+        history = SynthesisAgent.fit_history_to_context(
+            messages,
+            query="Given all that, what should I change first?",
+            permanent_memories=[
+                "Prefers Python for backend work.",
+                "Works on a small internal tools team of four engineers.",
+                "Wants direct answers with caveats stated plainly.",
+                "Currently debugging a connection-pool timeout issue in production.",
+                "Uses PostgreSQL with SQLAlchemy's pooled engine.",
+            ],
+            memories_enabled=True,
+            user_system_instructions="Always include a code example when relevant, and be concise.",
+            num_ctx=default_num_ctx,
+            code_execution_eligible=True,
+        )
+
+        kept_exchanges = history.count("User: ")
+        self.assertGreaterEqual(
+            kept_exchanges,
+            25,
+            f"Only {kept_exchanges}/30 exchanges survived at the shipped default "
+            f"num_ctx={default_num_ctx} with memory and code-execution eligibility "
+            "both on -- the default is too small relative to the built-in prompt "
+            "overhead and conversations will appear to lose their memory.",
+        )
+
     def test_context_budget_trims_oversized_permanent_memory(self):
         memories = [f"memory-{index} " + ("detail " * 120) for index in range(20)]
 
diff --git a/tests/test_settings_compatibility.py b/tests/test_settings_compatibility.py
index f9c4948..7cad860 100644
--- a/tests/test_settings_compatibility.py
+++ b/tests/test_settings_compatibility.py
@@ -31,7 +31,7 @@ def test_defaults_match_the_web_runtime_contract(self):
         self.assertIsNone(settings.models.title)
         self.assertEqual(settings.models.translation, "translategemma:4b")
         self.assertEqual(settings.generation.temperature, 0.7)
-        self.assertEqual(settings.generation.num_ctx, 4096)
+        self.assertEqual(settings.generation.num_ctx, 8192)
         self.assertEqual(settings.generation.seed, -1)
         self.assertTrue(settings.execution.automatic_compute)
         self.assertTrue(settings.memory.enabled)