diff --git a/backend/cortex_backend/core/settings.py b/backend/cortex_backend/core/settings.py index b06b940..0b323b7 100644 --- a/backend/cortex_backend/core/settings.py +++ b/backend/cortex_backend/core/settings.py @@ -54,7 +54,14 @@ class GenerationSettings(_SettingsModel): top_p: float = Field(default=0.9, ge=0.0, le=1.0) top_k: int = Field(default=40, ge=0, le=200) repeat_penalty: float = Field(default=1.1, ge=0.5, le=2.0) - num_ctx: int = Field(default=4096, ge=2048, le=16384) + # 4096 measured out at only 4-10 of 30 realistic exchanges surviving the + # context-budget trim once the built-in system/memory/code-execution + # prompts (up to ~2000 tokens) were accounted for -- history was being + # silently discarded well within what every locally installed model + # actually supports (the smallest here is 40960). 8192 keeps the full + # 30-exchange conversation even with memories and code-execution eligibility + # both active; see the discussion around PR raising this default. + num_ctx: int = Field(default=8192, ge=2048, le=16384) seed: int = Field(default=-1, ge=-1, le=2147483647) # No length cap: whatever doesn't fit in the configured context window is # already handled gracefully by the history/memory/attachment budget diff --git a/backend/cortex_backend/services/generation.py b/backend/cortex_backend/services/generation.py index 20bdf22..722d552 100644 --- a/backend/cortex_backend/services/generation.py +++ b/backend/cortex_backend/services/generation.py @@ -112,7 +112,11 @@ def generate( permanent_memories = ( list(self._memory_loader()) if snapshot.memories_enabled else [] ) - num_ctx = int(snapshot.model_options.get("num_ctx", 4096)) + # A real snapshot always carries num_ctx (GENERATION_OVERRIDE_FIELDS + # guarantees it); this fallback only matters for callers that build + # model_options by hand, so it stays in step with GenerationSettings' + # own default rather than reintroducing the old, too-small one. + num_ctx = int(snapshot.model_options.get("num_ctx", 8192)) if snapshot.memories_enabled: self._publish(sink, snapshot, "thoughts", "Gathering thoughts...") engine = self._engine_factory(snapshot) diff --git a/backend/cortex_backend/services/llm.py b/backend/cortex_backend/services/llm.py index ffb9455..8c9901a 100644 --- a/backend/cortex_backend/services/llm.py +++ b/backend/cortex_backend/services/llm.py @@ -654,7 +654,10 @@ def generate( permanent_memories=permanent_memories, memories_enabled=memories_enabled, user_system_instructions=user_system_instructions, - num_ctx=int(api_options.get("num_ctx", 4096)), + # Kept in step with GenerationSettings.num_ctx's own default -- + # a real call always carries num_ctx, so this only matters for + # options built by hand without one. + num_ctx=int(api_options.get("num_ctx", 8192)), code_execution_eligible=self.code_execution_eligible, bypass_system_prompt=self.bypass_system_prompt, ) diff --git a/contracts/openapi.json b/contracts/openapi.json index c98f8c0..7ce8e97 100644 --- a/contracts/openapi.json +++ b/contracts/openapi.json @@ -1792,7 +1792,7 @@ "type": "boolean" }, "num_ctx": { - "default": 4096, + "default": 8192, "maximum": 16384.0, "minimum": 2048.0, "title": "Num Ctx", diff --git a/frontend/src/features/chat/ChatPage.tsx b/frontend/src/features/chat/ChatPage.tsx index e5f2324..17092cc 100644 --- a/frontend/src/features/chat/ChatPage.tsx +++ b/frontend/src/features/chat/ChatPage.tsx @@ -16,7 +16,7 @@ const DEFAULT_GENERATION_SETTINGS = { top_p: 0.9, top_k: 40, repeat_penalty: 1.1, - num_ctx: 4096, + num_ctx: 8192, seed: -1, system_instructions: "", }; diff --git a/frontend/src/features/chat/MessageComposer.tsx b/frontend/src/features/chat/MessageComposer.tsx index 0a6a1a4..46a2382 100644 --- a/frontend/src/features/chat/MessageComposer.tsx +++ b/frontend/src/features/chat/MessageComposer.tsx @@ -88,7 +88,7 @@ const FALLBACK_GENERATION_DEFAULTS: GenerationSettings = { top_p: 0.9, top_k: 40, repeat_penalty: 1.1, - num_ctx: 4096, + num_ctx: 8192, seed: -1, }; diff --git a/frontend/src/features/settings/SettingsPanel.tsx b/frontend/src/features/settings/SettingsPanel.tsx index ebe7300..083432e 100644 --- a/frontend/src/features/settings/SettingsPanel.tsx +++ b/frontend/src/features/settings/SettingsPanel.tsx @@ -204,7 +204,7 @@ export function SettingsPanel({