diff --git a/backend/cortex_backend/api/jobs.py b/backend/cortex_backend/api/jobs.py index bde4eb9..c8ab408 100644 --- a/backend/cortex_backend/api/jobs.py +++ b/backend/cortex_backend/api/jobs.py @@ -98,6 +98,7 @@ class _JobRecord: prepared: bool = False preparation_error: str | None = None cancel_event: Event = field(default_factory=Event) + commit_started: bool = False status: JobStatus = "queued" sequence: int = 0 error: str | None = None @@ -127,6 +128,39 @@ def publish_progress( payload = {"message": message, **dict(data or {})} self.publish_event("progress", phase=phase, data=payload) + def begin_commit( + self, + phase: str, + message: str, + *, + data: Mapping[str, Any] | None = None, + ) -> bool: + """Atomically cross the point after which cancellation is too late. + + The worker must call this immediately before its first durable result + mutation. Sharing the registry lock with :meth:`JobRegistry.cancel` + gives the two operations one unambiguous order: cancellation wins and + this returns ``False``, or commit wins and later cancellation is inert. + """ + payload = {"message": message, **dict(data or {})} + with self._registry._lock: + if ( + self._record.status != "running" + or self._record.cancel_event.is_set() + ): + return False + if self._record.commit_started: + return True + self._record.commit_started = True + self._registry._append_event( + self._record, + kind="progress", + status="running", + phase=phase, + data=payload, + ) + return True + def publish_event( self, kind: EventKind, @@ -392,7 +426,11 @@ def active_snapshot(self, *, kind: JobKind) -> JobSnapshot | None: def cancel(self, job_id: str, *, owner: str) -> JobSnapshot: record = self._owned_record(job_id, owner) with self._lock: - if record.status not in TERMINAL_STATUSES and record.status != "cancelling": + if ( + not record.commit_started + and record.status not in TERMINAL_STATUSES + and record.status != "cancelling" + ): record.cancel_event.set() self._append_event( record, @@ -443,6 +481,8 @@ async def shutdown(self) -> None: if record.status not in TERMINAL_STATUSES ] for record in records: + if record.commit_started: + continue if record.status != "cancelling": record.cancel_event.set() self._append_event( @@ -493,7 +533,9 @@ async def _run( with self._lock: if record.status in TERMINAL_STATUSES: return - if record.status == "cancelling" or record.cancel_event.is_set(): + if not record.commit_started and ( + record.status == "cancelling" or record.cancel_event.is_set() + ): self._finalize_cancellation(record) return data = dict( @@ -510,6 +552,7 @@ async def _run( with self._lock: if ( record.status not in TERMINAL_STATUSES + and not record.commit_started and (record.status == "cancelling" or record.cancel_event.is_set()) ): self._finalize_cancellation(record) @@ -518,7 +561,9 @@ async def _run( with self._lock: if record.status in TERMINAL_STATUSES: return - if record.status == "cancelling" or record.cancel_event.is_set(): + if not record.commit_started and ( + record.status == "cancelling" or record.cancel_event.is_set() + ): self._finalize_cancellation(record) return logging.error( diff --git a/backend/cortex_backend/api/routes.py b/backend/cortex_backend/api/routes.py index 99217bc..815df47 100644 --- a/backend/cortex_backend/api/routes.py +++ b/backend/cortex_backend/api/routes.py @@ -1869,23 +1869,10 @@ def runner(sink, cancel_event): history_messages=prepared_history, ) # The generation service checks cancellation around its model work, - # but the API owns the following persistence and optional title work. - # Keep those side effects behind explicit checkpoints as well. + # while the API owns streaming and persistence. Keep everything + # cancellable until begin_commit atomically seals the durable result. if cancel_event.is_set(): return {"cancelled": True} - code_execution_job_id = _queue_code_proposal( - request, - principal, - settings, - generation_snapshot.job_id, - result, - ) - if code_execution_job_id: - sink.publish_progress( - "code_approval", - "A local code task is waiting for your approval.", - data={"execution_job_id": code_execution_job_id}, - ) if result.thoughts: for delta in _chunks(result.thoughts): if cancel_event.is_set(): @@ -1904,14 +1891,7 @@ def runner(sink, cancel_event): data={"delta": delta}, ) - for memo in result.memory_command.additions: - if cancel_event.is_set(): - return {"cancelled": True} - deps.memories.add_memo(memo) - if cancel_event.is_set(): - return {"cancelled": True} - sink.publish_progress("persisting", "Saving the response.") - if cancel_event.is_set(): + if not sink.begin_commit("persisting", "Saving the response."): return {"cancelled": True} stats_payload = asdict(result.stats) if result.stats else None if target_message_id is None: @@ -1934,13 +1914,40 @@ def runner(sink, cancel_event): ) assistant_message_id = target_message_id - if cancel_event.is_set(): - return {"cancelled": True} + # The assistant turn is the canonical generation result. Code and + # memory actions are optional derivatives: queue them only after the + # answer exists, and never invalidate that answer if they fail. + code_execution_job_id = None + try: + code_execution_job_id = _queue_code_proposal( + request, + principal, + settings, + generation_snapshot.job_id, + result, + ) + except Exception as exc: + logging.warning( + "Cortex code proposal queueing failed (%s).", type(exc).__name__ + ) + if code_execution_job_id: + sink.publish_progress( + "code_approval", + "A local code task is waiting for your approval.", + data={"execution_job_id": code_execution_job_id}, + ) + + for memo in result.memory_command.additions: + try: + deps.memories.add_memo(memo) + except Exception as exc: + logging.warning( + "Cortex memory update failed (%s).", type(exc).__name__ + ) + updated_chat = deps.chats.get_chat(thread_id) or {"messages": []} title = str(updated_chat.get("title") or "New Chat") if target_message_id is None and title == "New Chat": - if cancel_event.is_set(): - return {"cancelled": True} raw_title = None title_generator = getattr( deps.generation, "generate_chat_title", None @@ -1955,8 +1962,6 @@ def runner(sink, cancel_event): "Cortex chat title generation failed (%s).", type(exc).__name__, ) - if cancel_event.is_set(): - return {"cancelled": True} generated_title = normalize_title(raw_title, fallback="") if ( not generated_title @@ -1964,8 +1969,6 @@ def runner(sink, cancel_event): ): generated_title = title_from_first_message(payload.user_input) if generated_title != title: - if cancel_event.is_set(): - return {"cancelled": True} try: deps.chats.rename_chat(thread_id, generated_title) title = generated_title @@ -1973,8 +1976,6 @@ def runner(sink, cancel_event): logging.warning( "Cortex title update failed (%s).", type(exc).__name__ ) - if cancel_event.is_set(): - return {"cancelled": True} updated_chat = deps.chats.get_chat(thread_id) or updated_chat return { "thread_id": thread_id, diff --git a/backend/cortex_backend/launcher/frontend.py b/backend/cortex_backend/launcher/frontend.py index e85bb67..ae3d768 100644 --- a/backend/cortex_backend/launcher/frontend.py +++ b/backend/cortex_backend/launcher/frontend.py @@ -200,14 +200,15 @@ def build_frontend( frontend_root = frontend_root.resolve() if not (frontend_root / "package.json").is_file(): raise FrontendBuildError("frontend/package.json is missing from the source checkout.") - lock = lock_digest(frontend_root) - node_major = _major_version("node") - npm_major = _major_version("npm") build_root = _stage_frontend_source(frontend_root) staging = build_root / f".cortex-dist-staging-{uuid.uuid4().hex}" dist = frontend_root / "dist" backup = frontend_root / f".cortex-dist-backup-{uuid.uuid4().hex}" try: + lock = lock_digest(build_root) + source = source_digest(build_root) + node_major = _major_version("node") + npm_major = _major_version("npm") _install_if_needed(build_root, lock) _run( [_tool_name("npm"), "run", "build", "--", "--outDir", str(staging)], @@ -217,7 +218,7 @@ def build_frontend( raise FrontendBuildError("Frontend build completed without index.html.") manifest = FrontendManifest( lock_digest=lock, - source_digest=source_digest(frontend_root), + source_digest=source, node_major=node_major, npm_major=npm_major, built_at=datetime.now(timezone.utc).isoformat(), diff --git a/frontend/src/app/App.test.tsx b/frontend/src/app/App.test.tsx index 5407acd..57468bb 100644 --- a/frontend/src/app/App.test.tsx +++ b/frontend/src/app/App.test.tsx @@ -3,6 +3,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import userEvent from "@testing-library/user-event"; import { App } from "./App"; import { CortexApi } from "../api/client"; +import { useChatStore } from "../stores/useChatStore"; import { ToastProvider } from "./ToastProvider"; describe("App", () => { @@ -42,6 +43,40 @@ describe("App", () => { expect(screen.queryByLabelText(/token/i)).not.toBeInTheDocument(); }); + it("returns to onboarding and clears a persisted generation when its stream session expires", async () => { + window.sessionStorage.setItem("cortex.session.token", "local-session"); + window.sessionStorage.setItem("cortex.active.generation", JSON.stringify({ + jobId: "job-expired", + threadId: "thread-expired", + lastEventId: 3, + })); + window.history.replaceState({}, "", "/chat/thread-expired"); + const json = (body: unknown, status = 200) => new Response(JSON.stringify(body), { + status, + headers: { "Content-Type": "application/json" }, + }); + const fetcher = vi.fn(async (input) => { + const url = String(input); + if (url.endsWith("/system")) return json({ status: "ok", preview: true, session_required: true, started_at: "2026-07-21T18:00:00Z" }); + if (url.endsWith("/chat-groups")) return json([]); + if (url.endsWith("/chats")) return json([{ id: "thread-expired", title: "Interrupted", timestamp: "2026-07-21T18:00:00Z" }]); + if (url.endsWith("/chats/thread-expired")) return json({ id: "thread-expired", title: "Interrupted", timestamp: "2026-07-21T18:00:00Z", revision: 1, messages: [] }); + if (url.endsWith("/settings")) return json({ settings: { models: { chat: "model-a", title: null }, appearance: { theme: "dark" } } }); + if (url.endsWith("/memories")) return json({ memos: [] }); + if (url.endsWith("/models")) return json({ required_models: [], optional_models: [], installed_models: ["model-a"], connection: { success: true, status: "connected", message: "Ready" } }); + if (url.endsWith("/generations/job-expired/events")) return json({ detail: "Local session expired." }, 401); + return json({ detail: "Unexpected test route." }, 404); + }); + + render(); + + expect(await screen.findByRole("heading", { name: "Start local workspace" })).toBeVisible(); + expect(fetcher.mock.calls.filter(([input]) => String(input).endsWith("/generations/job-expired/events"))).toHaveLength(1); + expect(window.sessionStorage.getItem("cortex.session.token")).toBeNull(); + expect(window.sessionStorage.getItem("cortex.active.generation")).toBeNull(); + expect(useChatStore.getState().generation).toMatchObject({ jobId: null, phase: "idle" }); + }); + it("opens the workspace when the model service is unavailable", async () => { window.sessionStorage.setItem("cortex.session.token", "local-session"); const json = (body: unknown, status = 200) => new Response(JSON.stringify(body), { diff --git a/frontend/src/app/App.tsx b/frontend/src/app/App.tsx index f5e2976..13af88c 100644 --- a/frontend/src/app/App.tsx +++ b/frontend/src/app/App.tsx @@ -482,7 +482,7 @@ function AuthenticatedWorkspace({ api, onSessionExpired }: { api: CortexApi; onS { if (route.kind === "chat") setSettingsReturnChatId(route.threadId); }} onRenameChat={renameChat} onDeleteChat={deleteChat} groups={groups} onCreateGroup={createGroup} onRenameGroup={renameGroup} onDeleteGroup={deleteGroup} onToggleGroup={toggleGroup} onMoveChat={moveChat}> {route.kind === "settings" ? - : } + : } Promise; onRescanModels: () => Promise; onChatChanged: (chat: ChatResponse) => void; onForked: (chat: ChatResponse) => void }) { +function ChatRoute({ threadId, api, runtimeReady, runtimeMessage, localModels, selectedModel, selectedModelSupportsVision, modelBusy, onSelectModel, onRescanModels, onChatChanged, onForked, onSessionExpired }: { threadId: string | null; api: CortexApi; runtimeReady: boolean; runtimeMessage: string | null; localModels: readonly string[]; selectedModel: string | null; selectedModelSupportsVision: boolean | null; modelBusy: boolean; onSelectModel: (model: string) => Promise; onRescanModels: () => Promise; onChatChanged: (chat: ChatResponse) => void; onForked: (chat: ChatResponse) => void; onSessionExpired: () => void }) { const navigate = useNavigate(); - return navigate(chatPath(id), { replace: true })} onChatChanged={onChatChanged} onForked={(chat) => { onForked(chat); navigate(chatPath(chat.id)); }} />; + return navigate(chatPath(id), { replace: true })} onChatChanged={onChatChanged} onForked={(chat) => { onForked(chat); navigate(chatPath(chat.id)); }} onSessionExpired={onSessionExpired} />; } function SettingsRoute({ activeChatId, ...props }: Omit & { activeChatId: string | null }) { diff --git a/frontend/src/features/chat/ChatPage.test.tsx b/frontend/src/features/chat/ChatPage.test.tsx index 5dd892a..a74e1cd 100644 --- a/frontend/src/features/chat/ChatPage.test.tsx +++ b/frontend/src/features/chat/ChatPage.test.tsx @@ -57,6 +57,7 @@ function renderChat(api: CortexApi, threadId = "thread-a", selectedModelSupports onThreadCreated={vi.fn()} onChatChanged={vi.fn()} onForked={vi.fn()} + onSessionExpired={vi.fn()} />, ); } @@ -163,6 +164,7 @@ describe("ChatPage composer integration", () => { onThreadCreated={vi.fn()} onChatChanged={vi.fn()} onForked={vi.fn()} + onSessionExpired={vi.fn()} />, ); @@ -198,6 +200,7 @@ describe("ChatPage composer integration", () => { onThreadCreated={vi.fn()} onChatChanged={vi.fn()} onForked={vi.fn()} + onSessionExpired={vi.fn()} />, ); await waitFor(() => expect(api.chat).toHaveBeenCalledWith("thread-b")); @@ -215,6 +218,7 @@ describe("ChatPage composer integration", () => { onThreadCreated={vi.fn()} onChatChanged={vi.fn()} onForked={vi.fn()} + onSessionExpired={vi.fn()} />, ); await waitFor(() => expect(api.chat).toHaveBeenCalledTimes(3)); @@ -250,6 +254,7 @@ describe("ChatPage composer integration", () => { onThreadCreated={setThreadId} onChatChanged={vi.fn()} onForked={vi.fn()} + onSessionExpired={vi.fn()} /> ); } @@ -296,6 +301,7 @@ describe("ChatPage composer integration", () => { onThreadCreated={setThreadId} onChatChanged={vi.fn()} onForked={vi.fn()} + onSessionExpired={vi.fn()} /> ); } @@ -364,6 +370,7 @@ describe("ChatPage composer integration", () => { onThreadCreated={setThreadId} onChatChanged={vi.fn()} onForked={vi.fn()} + onSessionExpired={vi.fn()} /> ); } @@ -448,6 +455,8 @@ describe("ChatPage composer integration", () => { await waitFor(() => expect(details).not.toHaveAttribute("open")); expect(screen.queryByText("Live")).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Stop generating" })).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Finishing response" })).toBeDisabled(); }); it("retains the exact draft if generation acceptance fails", async () => { @@ -501,6 +510,7 @@ describe("ChatPage composer integration", () => { onThreadCreated={vi.fn()} onChatChanged={vi.fn()} onForked={vi.fn()} + onSessionExpired={vi.fn()} />, ); const composerB = await screen.findByLabelText("Message Cortex"); @@ -521,6 +531,7 @@ describe("ChatPage composer integration", () => { onThreadCreated={vi.fn()} onChatChanged={vi.fn()} onForked={vi.fn()} + onSessionExpired={vi.fn()} />, ); await waitFor(() => { @@ -555,6 +566,7 @@ describe("ChatPage composer integration", () => { onThreadCreated={vi.fn()} onChatChanged={vi.fn()} onForked={vi.fn()} + onSessionExpired={vi.fn()} />, ); @@ -563,6 +575,46 @@ describe("ChatPage composer integration", () => { expect(screen.getByLabelText("Message Cortex")).toBeEnabled(); }); + it("does not stay stuck in stopping when cancellation loses to persistence", async () => { + const user = userEvent.setup(); + const cancelGeneration = vi.fn(async () => ({ + job_id: "job-committing", + kind: "generation" as const, + status: "running" as const, + sequence: 3, + })); + const api = chatApi({ + generate: vi.fn().mockResolvedValue({ + job_id: "job-committing", + kind: "generation", + status: "queued", + thread_id: "thread-a", + user_message_id: "message-1", + }), + cancelGeneration, + streamGeneration: vi.fn((_jobId, _onEvent, options: { signal?: AbortSignal } = {}) => + new Promise((_resolve, reject) => { + options.signal?.addEventListener( + "abort", + () => reject(new DOMException("Aborted", "AbortError")), + { once: true }, + ); + }), + ), + }); + renderChat(api); + + await user.type(await screen.findByLabelText("Message Cortex"), "Persist this answer"); + await user.click(screen.getByRole("button", { name: "Send message" })); + await user.click(await screen.findByRole("button", { name: "Stop generating" })); + + await waitFor(() => expect(cancelGeneration).toHaveBeenCalledWith("job-committing")); + expect(screen.queryByRole("button", { name: "Stopping response" })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Stop generating" })).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Finishing response" })).toBeDisabled(); + expect(screen.getByText("Finishing response...")).toBeVisible(); + }); + it("regenerates from the selected user turn instead of stale cross-chat or composer state", async () => { const user = userEvent.setup(); const originalAttachment: ChatAttachment = { @@ -632,6 +684,7 @@ describe("ChatPage composer integration", () => { onThreadCreated={vi.fn()} onChatChanged={vi.fn()} onForked={vi.fn()} + onSessionExpired={vi.fn()} />, ); expect(await screen.findByText("Answer from B")).toBeInTheDocument(); diff --git a/frontend/src/features/chat/ChatPage.tsx b/frontend/src/features/chat/ChatPage.tsx index 595c132..b07c3ef 100644 --- a/frontend/src/features/chat/ChatPage.tsx +++ b/frontend/src/features/chat/ChatPage.tsx @@ -35,6 +35,7 @@ type Props = { onThreadCreated: (threadId: string) => void; onChatChanged: (chat: ChatResponse) => void; onForked: (chat: ChatResponse) => void; + onSessionExpired: () => void; }; type ScopedError = { @@ -75,12 +76,13 @@ export function ChatPage({ onThreadCreated, onChatChanged, onForked, + onSessionExpired, }: Props) { const generation = useChatStore((state) => state.generation); const generationOptionsByThread = useChatStore((state) => state.generationOptionsByThread); const setThreadOptions = useChatStore((state) => state.setThreadOptions); const generationDefaults = useSettingsStore((state) => state.settings?.generation) ?? DEFAULT_GENERATION_SETTINGS; - const { start, consume, stop } = useGenerationStream(api); + const { start, consume, stop } = useGenerationStream(api, onSessionExpired); const [chat, setChat] = useState(null); const [resolvedThreadId, setResolvedThreadId] = useState(threadId); const [drafts, setDrafts] = useState>(() => ({ @@ -192,7 +194,7 @@ export function ChatPage({ : generation.phase === "stopping" ? "stopping" : generation.jobId - ? "generating" + ? generation.contentReady ? "finishing" : "generating" : starting ? "starting" : "ready"; @@ -396,7 +398,16 @@ export function ChatPage({ useChatStore.getState().markStopping(jobId); useChatStore.getState().setStatusText(jobId, "Stopping response..."); try { - await api.cancelGeneration(jobId); + const snapshot = await api.cancelGeneration(jobId); + if (snapshot.status !== "cancelling" && snapshot.status !== "cancelled") { + // Persistence has already crossed the backend's commit barrier, so a + // late stop is intentionally inert. Reflect that response instead of + // leaving the composer disabled in a false "Stopping" state while the + // durable answer finishes its optional bookkeeping. + useChatStore.getState().markContentReady(jobId); + useChatStore.getState().revertStopping(jobId); + useChatStore.getState().setStatusText(jobId, "Finishing response..."); + } } catch (requestError) { useChatStore.getState().revertStopping(jobId); setGenerationError({ diff --git a/frontend/src/features/chat/MessageComposer.test.tsx b/frontend/src/features/chat/MessageComposer.test.tsx index 7005be9..cdedca5 100644 --- a/frontend/src/features/chat/MessageComposer.test.tsx +++ b/frontend/src/features/chat/MessageComposer.test.tsx @@ -141,6 +141,19 @@ describe("MessageComposer", () => { stop.resolve(); }); + it("shows non-cancellable finishing work without offering another stop", async () => { + const user = userEvent.setup(); + const onStop = vi.fn(); + render(); + + expect(screen.getByText("Finishing response…")).toBeVisible(); + expect(screen.getByRole("button", { name: "Finishing response" })).toBeDisabled(); + expect(screen.queryByRole("button", { name: "Stop generating" })).not.toBeInTheDocument(); + await user.click(screen.getByLabelText("Message Cortex")); + await user.keyboard("{Escape}"); + expect(onStop).not.toHaveBeenCalled(); + }); + it("keeps drafting available while the local runtime is unavailable", async () => { const user = userEvent.setup(); render(); diff --git a/frontend/src/features/chat/MessageComposer.tsx b/frontend/src/features/chat/MessageComposer.tsx index 46a2382..6302a1e 100644 --- a/frontend/src/features/chat/MessageComposer.tsx +++ b/frontend/src/features/chat/MessageComposer.tsx @@ -50,7 +50,7 @@ function LlamaRuntimeBadge({ selectedModel }: { selectedModel: string }) { ); } -export type ComposerPhase = "ready" | "starting" | "generating" | "stopping" | "unavailable"; +export type ComposerPhase = "ready" | "starting" | "generating" | "stopping" | "finishing" | "unavailable"; export type MessageComposerProps = { value: string; @@ -134,7 +134,8 @@ export function MessageComposer({ && !attachmentsBusy && !imageInputBlocked; const isStopping = phase === "stopping"; - const isGenerating = phase === "generating" || isStopping; + const isFinishing = phase === "finishing"; + const isGenerating = phase === "generating" || isStopping || isFinishing; // Runtime availability gates sending, not workspace configuration. Users // must still be able to choose a discovered model while the current model // is missing or the local service is reconnecting. @@ -183,7 +184,7 @@ export function MessageComposer({ }; const stop = async () => { - if (!isGenerating || isStopping || stopPendingRef.current) return; + if (!isGenerating || isStopping || isFinishing || stopPendingRef.current) return; stopPendingRef.current = true; try { await onStop(); @@ -229,6 +230,8 @@ export function MessageComposer({ ? "Starting response…" : phase === "stopping" ? "Stopping response…" + : phase === "finishing" + ? "Finishing response…" : phase === "generating" ? generationElsewhere ? "Generating in another thread" : "Generating…" : phase === "unavailable" @@ -347,12 +350,12 @@ export function MessageComposer({ ) : (