From f1be6e9a5a76f6710200b699be836503cb353e4b Mon Sep 17 00:00:00 2001 From: Omri Katz <9701896+omridevk@users.noreply.github.com> Date: Sat, 15 Aug 2026 04:36:31 +0300 Subject: [PATCH 1/9] =?UTF-8?q?refactor(testkit):=20#354=20phase=200+1=20?= =?UTF-8?q?=E2=80=94=20staleness=20DI=20in=20core,=20real-core=20seams=20f?= =?UTF-8?q?or=20browser=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Core takes an injectable engine-staleness probe threaded through /health, rpc meta.engine and the mcp deps. bootCoreKit passes allowedOrigins, the staleness probe and harness transcript-history fixtures. createFakeHarness gains optional transcriptHistory rows, makeScriptedRun gains scriptTurn for multi-tool turns, and rpc-fault gains a releasable request gate plus preflight-aware, unroutable fault handlers. apps/conciv registers node-side vitest browser commands over those seams and proves the architecture with a held /rpc/chat/subscribe against a real booted core. Co-Authored-By: Claude Fable 5 --- .changeset/real-core-test-seams.md | 5 + .fallowrc.json | 4 +- apps/conciv/test/commands/core-control.ts | 150 ++++++++++++++++++ apps/conciv/test/commands/core-testkit.ts | 2 + .../test/core-request-gate.browser.test.ts | 77 +++++++++ apps/conciv/test/helpers/core-control.ts | 21 +++ apps/conciv/vitest.config.ts | 2 + packages/core/src/api/rpc/mount.ts | 2 + packages/core/src/api/rpc/router.ts | 3 +- packages/core/src/app.ts | 22 ++- .../test/api/engine-staleness-di.it.test.ts | 84 ++++++++++ packages/extension-testkit/src/core-kit.ts | 9 +- packages/extension-testkit/src/rpc-fault.ts | 77 +++++++-- .../test/core-kit-seams.it.test.ts | 66 ++++++++ .../src/create-fake-harness.ts | 77 +++++++-- packages/harness-testkit/src/scripted-run.ts | 88 ++++++---- packages/harness-testkit/src/testkit.ts | 1 + .../test/create-fake-harness.test.ts | 20 +++ .../harness-testkit/test/scripted-run.test.ts | 43 +++++ 19 files changed, 687 insertions(+), 66 deletions(-) create mode 100644 .changeset/real-core-test-seams.md create mode 100644 apps/conciv/test/commands/core-control.ts create mode 100644 apps/conciv/test/commands/core-testkit.ts create mode 100644 apps/conciv/test/core-request-gate.browser.test.ts create mode 100644 apps/conciv/test/helpers/core-control.ts create mode 100644 packages/core/test/api/engine-staleness-di.it.test.ts create mode 100644 packages/extension-testkit/test/core-kit-seams.it.test.ts diff --git a/.changeset/real-core-test-seams.md b/.changeset/real-core-test-seams.md new file mode 100644 index 000000000..69ff1e3bc --- /dev/null +++ b/.changeset/real-core-test-seams.md @@ -0,0 +1,5 @@ +--- +'@conciv/core': patch +--- + +makeApp accepts an injectable engine-staleness probe, shared by /health, the rpc meta.engine procedure and the mcp server instructions diff --git a/.fallowrc.json b/.fallowrc.json index d4382b29d..eea1659f8 100644 --- a/.fallowrc.json +++ b/.fallowrc.json @@ -149,6 +149,8 @@ "packages/extensions/tanstack/test/host/conciv/extensions/connect-probe.tsx", "packages/extensions/try-it/test/fixture/connect-pane-fixture.tsx", "e2e/harnesses/vite.*.config.ts", - "apps/conciv/test/helpers/fake-core-router.ts" + "apps/conciv/test/helpers/fake-core-router.ts", + "apps/conciv/test/commands/core-control.ts", + "apps/conciv/test/commands/core-testkit.ts" ] } diff --git a/apps/conciv/test/commands/core-control.ts b/apps/conciv/test/commands/core-control.ts new file mode 100644 index 000000000..43ebb308d --- /dev/null +++ b/apps/conciv/test/commands/core-control.ts @@ -0,0 +1,150 @@ +import {join} from 'node:path' +import type {BrowserCommand, BrowserCommandContext} from 'vitest/node' +import type {EngineStaleness} from '@conciv/contract' +import type {HarnessSessionMeta} from '@conciv/protocol/harness-types' +import type {ScriptedTurn} from '@conciv/harness-testkit' +import type {CoreKit} from './core-testkit.js' + +export type BootCoreInput = { + id: string + text?: string + history?: HarnessSessionMeta[] + allowedOrigins?: string[] +} + +export type BootCoreResult = {base: string; wsBase: string; bootMs: number} + +export type FaultSpec = {kind: 'fail'; path: string[]; status?: number} | {kind: 'gate'; path?: string[]} + +type Fault = {pending: () => number; release: () => Promise; dispose: () => Promise} + +type CoreTestkit = typeof import('./core-testkit.js') + +type FileState = { + kit: CoreKit | null + staleness: {value: EngineStaleness} + faults: Map + handles: {count: number} +} + +const FRESH: EngineStaleness = { + stale: false, + changed: [], + tracked: ['@conciv/core'], + bootedAt: 0, + fingerprint: 'testkit0000', +} + +const files = new Map() + +function testkitOf(ctx: BrowserCommandContext): Promise { + const nodeProject = ctx.project.vitest.getRootProject() + return nodeProject.import(join(ctx.project.config.root, 'test/commands/core-testkit.ts')) +} + +function stateOf(ctx: BrowserCommandContext): FileState { + const key = ctx.testPath ?? 'shared' + const existing = files.get(key) + if (existing) return existing + const created: FileState = {kit: null, staleness: {value: FRESH}, faults: new Map(), handles: {count: 0}} + files.set(key, created) + return created +} + +function kitOf(ctx: BrowserCommandContext): CoreKit { + const kit = stateOf(ctx).kit + if (!kit) throw new Error('no core is booted for this test file; call bootCore first') + return kit +} + +function faultOf(ctx: BrowserCommandContext, handle: string): Fault { + const fault = stateOf(ctx).faults.get(handle) + if (!fault) throw new Error(`no fault is installed under the handle "${handle}"`) + return fault +} + +const bootCore: BrowserCommand<[BootCoreInput]> = async (ctx, input): Promise => { + const state = stateOf(ctx) + if (state.kit) throw new Error('a core is already booted for this test file; call closeCore first') + const startedAt = Date.now() + const {bootCoreKit} = await testkitOf(ctx) + const kit = await bootCoreKit({ + id: input.id, + text: input.text, + history: input.history, + allowedOrigins: input.allowedOrigins, + staleness: () => state.staleness.value, + }) + state.kit = kit + return {base: kit.base, wsBase: kit.wsBase, bootMs: Date.now() - startedAt} +} + +const closeCore: BrowserCommand<[]> = async (ctx): Promise => { + const state = stateOf(ctx) + for (const fault of state.faults.values()) await fault.dispose() + state.faults.clear() + state.staleness.value = FRESH + const kit = state.kit + state.kit = null + if (!kit) return + kit.harness.script.release() + await kit.cleanup() +} + +const setStaleness: BrowserCommand<[EngineStaleness]> = (ctx, value): void => { + stateOf(ctx).staleness.value = value +} + +const holdTurn: BrowserCommand<[]> = (ctx): void => { + kitOf(ctx).harness.script.hold() +} + +const releaseTurn: BrowserCommand<[]> = (ctx): void => { + kitOf(ctx).harness.script.release() +} + +const scriptError: BrowserCommand<[string]> = (ctx, message): void => { + kitOf(ctx).harness.script.scriptError(message) +} + +const scriptTurn: BrowserCommand<[ScriptedTurn]> = (ctx, turn): string[] => kitOf(ctx).harness.script.scriptTurn(turn) + +const installFault: BrowserCommand<[FaultSpec]> = async (ctx, spec): Promise => { + const state = stateOf(ctx) + const {failRpcCalls, gateRpcCalls} = await testkitOf(ctx) + state.handles.count += 1 + const handle = `fault-${state.handles.count}` + if (spec.kind === 'gate') { + state.faults.set(handle, await gateRpcCalls(ctx.page, spec.path ? {path: spec.path} : {})) + return handle + } + const injector = await failRpcCalls(ctx.page, {path: spec.path, ...(spec.status ? {status: spec.status} : {})}) + state.faults.set(handle, { + pending: () => 0, + release: () => { + injector.repair() + return Promise.resolve() + }, + dispose: injector.dispose, + }) + return handle +} + +const releaseFault: BrowserCommand<[string]> = async (ctx, handle): Promise => { + await faultOf(ctx, handle).release() +} + +const faultPending: BrowserCommand<[string]> = (ctx, handle): number => faultOf(ctx, handle).pending() + +export const coreCommands = { + bootCore, + closeCore, + setStaleness, + holdTurn, + releaseTurn, + scriptError, + scriptTurn, + installFault, + releaseFault, + faultPending, +} diff --git a/apps/conciv/test/commands/core-testkit.ts b/apps/conciv/test/commands/core-testkit.ts new file mode 100644 index 000000000..e65b1a3cd --- /dev/null +++ b/apps/conciv/test/commands/core-testkit.ts @@ -0,0 +1,2 @@ +export {bootCoreKit, type CoreKit} from '@conciv/extension-testkit/core-kit' +export {failRpcCalls, gateRpcCalls} from '@conciv/extension-testkit/rpc-fault' diff --git a/apps/conciv/test/core-request-gate.browser.test.ts b/apps/conciv/test/core-request-gate.browser.test.ts new file mode 100644 index 000000000..e95f02239 --- /dev/null +++ b/apps/conciv/test/core-request-gate.browser.test.ts @@ -0,0 +1,77 @@ +import {afterAll, beforeAll, describe, expect, it, vi} from 'vitest' +import {EventType} from '@tanstack/ai' +import {makeRpcClient, type RpcClient} from '@conciv/contract' +import {coreControl} from './helpers/core-control.js' + +const core: {rpc: RpcClient | null} = {rpc: null} + +function statusOf(value: unknown): number | null { + if (typeof value !== 'object' || value === null || !('status' in value)) return null + return typeof value.status === 'number' ? value.status : null +} + +function chunkType(value: unknown): unknown { + if (typeof value !== 'object' || value === null || !('type' in value)) return null + return value.type +} + +function rpc(): RpcClient { + if (!core.rpc) throw new Error('the core kit did not boot') + return core.rpc +} + +beforeAll(async () => { + const booted = await coreControl.bootCore({id: 'request-gate', allowedOrigins: [window.location.origin]}) + core.rpc = makeRpcClient(booted.base) +}, 60_000) + +afterAll(async () => { + core.rpc = null + await coreControl.closeCore() +}, 30_000) + +describe('gated rpc requests against a real core', () => { + it('holds the subscribe request until release, then streams the real snapshot', async () => { + const {sessionId} = await rpc().sessions.create() + const gate = await coreControl.installFault({kind: 'gate', path: ['chat', 'subscribe']}) + const arrived = {value: false} + const subscription = rpc() + .chat.subscribe({sessionId}) + .then((iterator) => { + arrived.value = true + return iterator + }) + + await vi.waitUntil(async () => (await coreControl.faultPending(gate)) === 1, {timeout: 5_000, interval: 20}) + expect(arrived.value).toBe(false) + + await coreControl.releaseFault(gate) + const iterator = await subscription + const first = await iterator.next() + + expect(chunkType(first.value)).toBe(EventType.MESSAGES_SNAPSHOT) + expect(await coreControl.faultPending(gate)).toBe(0) + + await iterator.return?.() + }, 30_000) + + it('keeps a cross-origin injected failure an rpc rejection rather than a transport error', async () => { + const {sessionId} = await rpc().sessions.create() + const fault = await coreControl.installFault({kind: 'fail', path: ['sessions', 'rename'], status: 500}) + + const rejection = await rpc() + .sessions.rename({sessionId, title: 'renamed by the fault test'}) + .then( + () => null, + (reason: unknown) => reason, + ) + + expect(rejection).not.toBeInstanceOf(TypeError) + expect(String(rejection)).not.toMatch(/failed to fetch/i) + expect(statusOf(rejection)).toBe(500) + + await coreControl.releaseFault(fault) + const renamed = await rpc().sessions.rename({sessionId, title: 'renamed by the fault test'}) + expect(renamed.title).toBe('renamed by the fault test') + }, 30_000) +}) diff --git a/apps/conciv/test/helpers/core-control.ts b/apps/conciv/test/helpers/core-control.ts new file mode 100644 index 000000000..eb2274ea2 --- /dev/null +++ b/apps/conciv/test/helpers/core-control.ts @@ -0,0 +1,21 @@ +import {commands} from 'vitest/browser' +import type {EngineStaleness} from '@conciv/contract' +import type {ScriptedTurn} from '@conciv/harness-testkit' +import type {BootCoreInput, BootCoreResult, FaultSpec} from '../commands/core-control.js' + +declare module 'vitest/internal/browser' { + interface BrowserCommands { + bootCore: (input: BootCoreInput) => Promise + closeCore: () => Promise + setStaleness: (value: EngineStaleness) => Promise + holdTurn: () => Promise + releaseTurn: () => Promise + scriptError: (message: string) => Promise + scriptTurn: (turn: ScriptedTurn) => Promise + installFault: (spec: FaultSpec) => Promise + releaseFault: (handle: string) => Promise + faultPending: (handle: string) => Promise + } +} + +export const coreControl = commands diff --git a/apps/conciv/vitest.config.ts b/apps/conciv/vitest.config.ts index 3fca0ceab..3f038eb4e 100644 --- a/apps/conciv/vitest.config.ts +++ b/apps/conciv/vitest.config.ts @@ -3,6 +3,7 @@ import {defineConfig} from 'vitest/config' import {BaseSequencer, type TestSpecification} from 'vitest/node' import {serveRpcRouter} from '@conciv/harness-testkit/rpc-mounts' import {makeFakeCoreRouter} from './test/helpers/fake-core-router.js' +import {coreCommands} from './test/commands/core-control.js' import {playwright} from '@vitest/browser-playwright' import solidPlugin from 'vite-plugin-solid' import {browserOptimizeDeps, ciTest, ciTestSolidBrowser} from '@conciv/vitest-config' @@ -63,6 +64,7 @@ export default defineConfig({ headless: true, provider: playwright({}), instances: [{browser: 'chromium'}], + commands: coreCommands, }, }, }, diff --git a/packages/core/src/api/rpc/mount.ts b/packages/core/src/api/rpc/mount.ts index 5b88a320b..ecf56ef3d 100644 --- a/packages/core/src/api/rpc/mount.ts +++ b/packages/core/src/api/rpc/mount.ts @@ -2,6 +2,7 @@ import {implement} from '@orpc/server' import {contract} from '@conciv/contract' import type {RpcContext} from '@conciv/protocol/rpc-types' import type {ChatTool} from '@conciv/protocol/chat-types' +import type {EngineStaleness} from '@conciv/contract' import type {CompositeRpcRouter as CompositeRouterOf} from '@conciv/extension/rpc-mount' import type {ChatDeps} from '../../chat/runtime.js' import type {Compactor, Send} from '../../chat/run.js' @@ -18,6 +19,7 @@ export type RpcDeps = { openFromFrames: (frames: OpenSourceFrames) => Promise page: PageEnv registry: ToolRegistry + staleness: () => EngineStaleness askTimeoutMs?: number } diff --git a/packages/core/src/api/rpc/router.ts b/packages/core/src/api/rpc/router.ts index 8148c5ea3..f055afa9b 100644 --- a/packages/core/src/api/rpc/router.ts +++ b/packages/core/src/api/rpc/router.ts @@ -14,7 +14,6 @@ import {makeAskGate, requiresApproval} from '../../chat/gate.js' import {rowById} from '../../chat/session-rows.js' import {pageQueryStream} from '../../page-bus.js' import {symbolicateFrames} from '../../editor/symbolicate.js' -import {engineStaleness} from '../../lib/engine-stamp.js' import {chatRouter} from './chat.js' import {harnessMetaOf, sessionsRouter} from './sessions.js' import {os, type RpcDeps} from './mount.js' @@ -240,7 +239,7 @@ export function makeRpcRouter(deps: RpcDeps) { listCommands(chat, {sessionId: input.sessionId, origin: context.origin}), ), tools: os.meta.tools.handler(() => ({tools: deps.tools})), - engine: os.meta.engine.handler(() => engineStaleness()), + engine: os.meta.engine.handler(() => deps.staleness()), }, }) } diff --git a/packages/core/src/app.ts b/packages/core/src/app.ts index 9545b62b8..b30b8675a 100644 --- a/packages/core/src/app.ts +++ b/packages/core/src/app.ts @@ -1,7 +1,7 @@ import {existsSync} from 'node:fs' import {Hono} from 'hono' import {z} from 'zod' -import {EngineStalenessSchema} from '@conciv/contract' +import {EngineStalenessSchema, type EngineStaleness} from '@conciv/contract' import {upgradeWebSocket} from '@conciv/serve' import {HTTPException} from 'hono/http-exception' import type {HarnessAdapter} from '@conciv/protocol/harness-types' @@ -94,6 +94,8 @@ export type MakeAppOpts = { nativePageDir?: string nativeUrl?: () => string | undefined + + staleness?: () => EngineStaleness } export function slug(name: string): string { @@ -211,7 +213,11 @@ export const HealthSchema = z.object({ export type CoreVars = CorsVars & {chat: ChatDeps} & McpVars -function composeRoutes(vars: CoreVars, rpc: CompositeRpcRouter, onShutdown?: () => void) { +function composeRoutes( + vars: CoreVars, + rpc: CompositeRpcRouter, + deps: {staleness: () => EngineStaleness; onShutdown?: () => void}, +) { return new Hono<{Variables: CoreVars}>() .onError((error, c) => { if (error instanceof HTTPException) return c.json({message: error.message}, error.status) @@ -226,11 +232,11 @@ function composeRoutes(vars: CoreVars, rpc: CompositeRpcRouter, onShutdown?: () }) .use(corsMiddleware()) .get('/health', (c) => - c.json(HealthSchema.parse({ok: true, harness: vars.chat.harness.id, engine: engineStaleness()})), + c.json(HealthSchema.parse({ok: true, harness: vars.chat.harness.id, engine: deps.staleness()})), ) .post('/api/shutdown', (c) => { - if (!onShutdown) return c.json({message: 'shutdown not supported'}, 404) - setTimeout(onShutdown, 50) + if (!deps.onShutdown) return c.json({message: 'shutdown not supported'}, 404) + setTimeout(deps.onShutdown, 50) return c.json({ok: true}) }) .get( @@ -270,6 +276,7 @@ export async function makeApp(opts: MakeAppOpts): Promise { assertUniqueExtensionSlugs(extensions) const harness = opts.harness ?? requireHarness(opts.cfg.harness) + const staleness = opts.staleness ?? engineStaleness const db = openDb(opts.cfg.stateRoot) await recoverInterruptedRuns({db, harness, claudeHome: opts.claudeHome}) const asks = createAskRegistry() @@ -465,6 +472,7 @@ export async function makeApp(opts: MakeAppOpts): Promise { openFromFrames: (frames) => openSourceFromFrames(frames, opts.cwd, opts.openInEditor), page: pageEnv, registry, + staleness, ...(opts.askTimeoutMs === undefined ? {} : {askTimeoutMs: opts.askTimeoutMs}), }) @@ -493,11 +501,11 @@ export async function makeApp(opts: MakeAppOpts): Promise { sessionModel, sessionForNativeId: async (nativeId) => (await rowByNativeId(db, nativeId))?.id ?? null, noteToolCall: (sessionId, toolCallId, toolName) => asks.noteToolCall(sessionId, toolCallId, toolName), - staleness: engineStaleness, + staleness, }, }, compositeRpc, - opts.onShutdown, + {staleness, onShutdown: opts.onShutdown}, ) if (opts.nativePageDir) app.route(NATIVE_PAGE_PATH, makeNativePageApp(opts.nativePageDir)) diff --git a/packages/core/test/api/engine-staleness-di.it.test.ts b/packages/core/test/api/engine-staleness-di.it.test.ts new file mode 100644 index 000000000..c4992dc8c --- /dev/null +++ b/packages/core/test/api/engine-staleness-di.it.test.ts @@ -0,0 +1,84 @@ +import {mkdtempSync, rmSync} from 'node:fs' +import {tmpdir} from 'node:os' +import {join} from 'node:path' +import {afterEach, describe, expect, it} from 'vitest' +import type {EngineStaleness} from '@conciv/contract' +import {makeRpcClient, serveApp, type ServedApp} from '@conciv/harness-testkit' +import {HealthSchema, makeApp} from '../../src/app.js' +import {resolveConfig} from '../../src/config.js' + +const dirs: string[] = [] +const state = {served: undefined as ServedApp | undefined, dispose: undefined as (() => Promise) | undefined} + +const INJECTED: EngineStaleness = { + stale: true, + changed: ['@conciv/injected-probe'], + tracked: ['@conciv/injected-probe', '@conciv/core'], + bootedAt: 1_700_000_000_000, + fingerprint: 'injected1234', +} + +const INITIALIZE = JSON.stringify({ + jsonrpc: '2.0', + id: 1, + method: 'initialize', + params: {protocolVersion: '2025-06-18', capabilities: {}, clientInfo: {name: 'staleness-di-test', version: '0'}}, +}) + +afterEach(async () => { + await state.served?.close() + await state.dispose?.() + state.served = undefined + state.dispose = undefined + for (const dir of dirs.splice(0)) rmSync(dir, {recursive: true, force: true}) +}) + +async function serveWithProbe(staleness: () => EngineStaleness): Promise { + const root = mkdtempSync(join(tmpdir(), 'conciv-staleness-di-')) + dirs.push(root) + const {app, dispose} = await makeApp({cfg: resolveConfig({}, root), cwd: root, openInEditor: () => {}, staleness}) + state.dispose = dispose + const served = await serveApp(app.fetch) + state.served = served + return served +} + +describe('engine staleness dependency injection (IT, real http)', () => { + it('reports the injected probe consistently on /health, rpc meta.engine and the mcp instructions', async () => { + const calls = {count: 0} + const served = await serveWithProbe(() => { + calls.count += 1 + return INJECTED + }) + + const health = HealthSchema.parse(await (await fetch(`${served.base}/health`)).json()) + const engine = await makeRpcClient(served.base).meta.engine() + const mcp = await fetch(`${served.base}/api/mcp`, { + method: 'POST', + headers: {'content-type': 'application/json', accept: 'application/json, text/event-stream'}, + body: INITIALIZE, + }) + const instructions = await mcp.text() + + expect(health.engine).toEqual(INJECTED) + expect(engine).toEqual(INJECTED) + expect(instructions).toContain('@conciv/injected-probe') + expect(calls.count).toBeGreaterThanOrEqual(3) + }, 20_000) + + it('defaults to the real engine stamp when no probe is injected', async () => { + const root = mkdtempSync(join(tmpdir(), 'conciv-staleness-default-')) + dirs.push(root) + const {app, dispose} = await makeApp({cfg: resolveConfig({}, root), cwd: root, openInEditor: () => {}}) + state.dispose = dispose + const served = await serveApp(app.fetch) + state.served = served + + const health = HealthSchema.parse(await (await fetch(`${served.base}/health`)).json()) + const engine = await makeRpcClient(served.base).meta.engine() + + expect(health.engine.tracked).toContain('@conciv/core') + expect(engine.fingerprint).toBe(health.engine.fingerprint) + expect(engine.stale).toBe(false) + }, 20_000) +}) diff --git a/packages/extension-testkit/src/core-kit.ts b/packages/extension-testkit/src/core-kit.ts index 90119be7f..921e53100 100644 --- a/packages/extension-testkit/src/core-kit.ts +++ b/packages/extension-testkit/src/core-kit.ts @@ -2,7 +2,8 @@ import {createFakeHarness, createTestkit, type FakeHarness, type Kit} from '@con import {makeApp} from '@conciv/core/app' import type {AnyExtension} from '@conciv/extension' import type {ToolRegistry} from '@conciv/extension/registry' -import type {HarnessCommand, HarnessModel} from '@conciv/protocol/harness-types' +import type {HarnessCommand, HarnessModel, HarnessSessionMeta} from '@conciv/protocol/harness-types' +import type {EngineStaleness} from '@conciv/contract' export type CoreKit = Kit & {harness: FakeHarness; registry: ToolRegistry} @@ -12,6 +13,9 @@ export async function bootCoreKit(opts: { extensions?: AnyExtension[] models?: HarnessModel[] commands?: HarnessCommand[] + history?: HarnessSessionMeta[] + allowedOrigins?: string[] + staleness?: () => EngineStaleness nativePageDir?: string }): Promise { const harness = createFakeHarness({ @@ -19,6 +23,7 @@ export async function bootCoreKit(opts: { text: opts.text ?? 'Hello from conciv', models: opts.models, commands: opts.commands, + history: opts.history, }) const captured: {registry?: ToolRegistry} = {} const kit = await createTestkit(harness, async (env) => { @@ -37,6 +42,8 @@ export async function bootCoreKit(opts: { openInEditor: () => {}, harness: env.harness, extensions: opts.extensions, + allowedOrigins: opts.allowedOrigins, + staleness: opts.staleness, nativePageDir: opts.nativePageDir, }) captured.registry = registry diff --git a/packages/extension-testkit/src/rpc-fault.ts b/packages/extension-testkit/src/rpc-fault.ts index b2aede17e..da5a49d75 100644 --- a/packages/extension-testkit/src/rpc-fault.ts +++ b/packages/extension-testkit/src/rpc-fault.ts @@ -1,27 +1,40 @@ -import type {Page, WebSocketRoute} from 'playwright' +import type {Page, Route, WebSocketRoute} from 'playwright' import {toHttpPath} from '@orpc/client/standard' import {encodeResponseMessage, MessageType} from '@orpc/standard-server-peer' import {decodeRpcFrame} from './rpc-frames.js' const FAILURE_BODY = {json: {}, meta: []} -export type RpcFaultInjector = {repair: () => void} +const RPC_PREFIX = '/rpc' + +type UrlMatcher = (url: URL) => boolean +type RouteHandler = (route: Route) => Promise + +export type RpcFaultInjector = {repair: () => void; dispose: () => Promise} + +function pathMatcher(path: readonly string[] | undefined): UrlMatcher { + if (!path) return (url) => url.pathname === RPC_PREFIX || url.pathname.startsWith(`${RPC_PREFIX}/`) + const httpPath = toHttpPath(path) + return (url) => url.pathname.endsWith(httpPath) +} + +function isPreflight(route: Route): boolean { + return route.request().method() === 'OPTIONS' +} export async function failRpcCalls( page: Page, options: {path: readonly string[]; status?: number}, ): Promise { const status = options.status ?? 500 - const httpPath = toHttpPath(options.path) const broken = {value: true} + const matcher = pathMatcher(options.path) + const handler: RouteHandler = async (route) => { + if (isPreflight(route) || !broken.value) return route.continue() + await route.fulfill({status, contentType: 'application/json', body: JSON.stringify(FAILURE_BODY)}) + } - await page.route( - (url) => url.pathname.endsWith(httpPath), - async (route) => { - if (!broken.value) return route.continue() - await route.fulfill({status, contentType: 'application/json', body: JSON.stringify(FAILURE_BODY)}) - }, - ) + await page.route(matcher, handler) const holdSocket = (socket: WebSocketRoute): void => { const server = socket.connectToServer() @@ -51,6 +64,10 @@ export async function failRpcCalls( repair: () => { broken.value = false }, + dispose: async () => { + broken.value = false + await page.unroute(matcher, handler) + }, } } @@ -61,9 +78,9 @@ export async function holdRpcCalls(page: Page): Promise { const sockets = new Set() await page.route( - (url) => url.pathname.startsWith('/rpc/') || url.pathname === '/rpc', + (url) => url.pathname.startsWith('/rpc/') || url.pathname === RPC_PREFIX, async (route) => { - if (!held.value) return route.continue() + if (isPreflight(route) || !held.value) return route.continue() await route.abort('connectionrefused') }, ) @@ -94,3 +111,39 @@ export async function holdRpcCalls(page: Page): Promise { }, } } + +export type RpcGate = { + pending: () => number + release: () => Promise + dispose: () => Promise +} + +export async function gateRpcCalls(page: Page, options: {path?: readonly string[]} = {}): Promise { + const matcher = pathMatcher(options.path) + const open = {value: false} + const captured: Route[] = [] + const handler: RouteHandler = async (route) => { + if (isPreflight(route) || open.value) return route.continue() + captured.push(route) + } + + await page.route(matcher, handler) + + const settle = async (): Promise => { + const pending = captured.splice(0) + await Promise.all(pending.map((route) => route.continue().catch(() => {}))) + } + + return { + pending: () => captured.length, + release: async () => { + open.value = true + await settle() + }, + dispose: async () => { + open.value = true + await settle() + await page.unroute(matcher, handler) + }, + } +} diff --git a/packages/extension-testkit/test/core-kit-seams.it.test.ts b/packages/extension-testkit/test/core-kit-seams.it.test.ts new file mode 100644 index 000000000..9297a9bbe --- /dev/null +++ b/packages/extension-testkit/test/core-kit-seams.it.test.ts @@ -0,0 +1,66 @@ +import {afterEach, describe, expect, it} from 'vitest' +import type {EngineStaleness} from '@conciv/contract' +import {bootCoreKit, type CoreKit} from '../src/core-kit.js' + +const STALENESS: EngineStaleness = { + stale: true, + changed: ['@conciv/kit-probe'], + tracked: ['@conciv/kit-probe'], + bootedAt: 1_700_000_000_000, + fingerprint: 'kitprobe123', +} + +const ORIGIN = 'https://widget.example.test' + +const state = {kit: undefined as CoreKit | undefined} + +afterEach(async () => { + await state.kit?.cleanup() + state.kit = undefined +}) + +describe('bootCoreKit seams', () => { + it('adopts the harness transcript rows as external sessions with their message counts', async () => { + const kit = await bootCoreKit({ + id: 'seams-history', + history: [{id: 'native-1', derivedTitle: 'A native session', updatedAt: 1_700, messageCount: 7}], + }) + state.kit = kit + + const sessions = await kit.rpc.sessions.list({}) + const adopted = sessions.find((meta) => meta.native?.nativeId === 'native-1') + + expect(adopted?.origin).toBe('external') + expect(adopted?.messageCount).toBe(7) + expect(adopted?.title).toBe('A native session') + expect(adopted?.running).toBe(false) + }, 30_000) + + it('serves the injected staleness probe to the widget rpc surface', async () => { + const kit = await bootCoreKit({id: 'seams-staleness', staleness: () => STALENESS}) + state.kit = kit + + expect(await kit.rpc.meta.engine()).toEqual(STALENESS) + }, 30_000) + + it('lets a non-loopback origin through only when the kit declares it allowed', async () => { + const closed = await bootCoreKit({id: 'seams-origins-closed'}) + state.kit = closed + const rejected = await fetch(`${closed.base}/rpc/meta/engine`, { + method: 'OPTIONS', + headers: {origin: ORIGIN, 'access-control-request-method': 'POST'}, + }) + await closed.cleanup() + state.kit = undefined + + const open = await bootCoreKit({id: 'seams-origins-open', allowedOrigins: [ORIGIN]}) + state.kit = open + const accepted = await fetch(`${open.base}/rpc/meta/engine`, { + method: 'OPTIONS', + headers: {origin: ORIGIN, 'access-control-request-method': 'POST'}, + }) + + expect(rejected.status).toBe(403) + expect(accepted.headers.get('access-control-allow-origin')).toBe(ORIGIN) + }, 30_000) +}) diff --git a/packages/harness-testkit/src/create-fake-harness.ts b/packages/harness-testkit/src/create-fake-harness.ts index ac06fe1a3..d65529f78 100644 --- a/packages/harness-testkit/src/create-fake-harness.ts +++ b/packages/harness-testkit/src/create-fake-harness.ts @@ -4,7 +4,9 @@ import { type HarnessAdapter, type HarnessCommand, type HarnessConnectContext, + type HarnessHistory, type HarnessModel, + type HarnessSessionMeta, } from '@conciv/protocol/harness-types' import {makeTextAdapter} from '@conciv/harness' import {makeScriptedRun, type ScriptedRun} from './scripted-run.js' @@ -16,7 +18,6 @@ export type FakeHarness = HarnessAdapter & { const BASE_CAPABILITIES = { resume: false, permissionGate: 'none', - transcriptHistory: false, compaction: false, systemPrompt: 'none', mcp: 'none', @@ -24,36 +25,80 @@ const BASE_CAPABILITIES = { init: 'none', } as const +type SharedFields = { + id: string + binName: string + chatConfig: HarnessAdapter['chatConfig'] + models: HarnessModel[] | undefined + tty: {command(ctx: HarnessConnectContext): TtyCommand} | undefined +} + +function fixtureHistory(rows: HarnessSessionMeta[]): HarnessHistory { + return { + list: () => Promise.resolve(rows), + messages: () => Promise.resolve([]), + meta: (_cwd, sessionId) => Promise.resolve(rows.find((row) => row.id === sessionId) ?? null), + observe: () => ({ + revision: () => Promise.resolve({ok: false as const, reason: 'missing' as const, detail: 'fixture history'}), + read: () => Promise.resolve({ok: false as const, reason: 'missing' as const, detail: 'fixture history'}), + close: () => {}, + }), + } +} + +function buildAdapter( + shared: SharedFields, + history: HarnessHistory | undefined, + commands: HarnessCommand[] | undefined, +): HarnessAdapter { + const listCommands = commands ? () => Promise.resolve(commands) : undefined + if (history && listCommands) { + return defineHarness({ + ...shared, + capabilities: {...BASE_CAPABILITIES, transcriptHistory: true, slashCommands: 'live'}, + history, + commands: listCommands, + }) + } + if (history) { + return defineHarness({ + ...shared, + capabilities: {...BASE_CAPABILITIES, transcriptHistory: true, slashCommands: 'none'}, + history, + }) + } + if (listCommands) { + return defineHarness({ + ...shared, + capabilities: {...BASE_CAPABILITIES, transcriptHistory: false, slashCommands: 'live'}, + commands: listCommands, + }) + } + return defineHarness({ + ...shared, + capabilities: {...BASE_CAPABILITIES, transcriptHistory: false, slashCommands: 'none'}, + }) +} + export function createFakeHarness( opts: { id?: string text?: string models?: HarnessModel[] commands?: HarnessCommand[] + history?: HarnessSessionMeta[] tty?: {command(ctx: HarnessConnectContext): TtyCommand} } = {}, ): FakeHarness { const id = opts.id ?? 'fake-harness' const scripted = makeScriptedRun({text: opts.text}) - const commands = opts.commands - const shared = { + const shared: SharedFields = { id, binName: 'true', - chatConfig: (deps: Parameters[0]) => ({ - adapter: makeTextAdapter(id, () => scripted.chatStream(deps)), - }), + chatConfig: (deps) => ({adapter: makeTextAdapter(id, () => scripted.chatStream(deps))}), models: opts.models, tty: opts.tty, } - const adapter = commands - ? defineHarness({ - ...shared, - capabilities: {...BASE_CAPABILITIES, slashCommands: 'live'}, - commands: () => Promise.resolve(commands), - }) - : defineHarness({ - ...shared, - capabilities: {...BASE_CAPABILITIES, slashCommands: 'none'}, - }) + const adapter = buildAdapter(shared, opts.history ? fixtureHistory(opts.history) : undefined, opts.commands) return Object.assign(adapter, {script: scripted}) } diff --git a/packages/harness-testkit/src/scripted-run.ts b/packages/harness-testkit/src/scripted-run.ts index 6d8e863cf..d2760a31b 100644 --- a/packages/harness-testkit/src/scripted-run.ts +++ b/packages/harness-testkit/src/scripted-run.ts @@ -1,20 +1,61 @@ import {EventType, type StreamChunk} from '@tanstack/ai' import type {HarnessChatDeps} from '@conciv/protocol/harness-types' +export type ScriptedTurnToolCall = {name: string; input: unknown; result?: unknown} + +export type ScriptedTurn = {toolCalls: ScriptedTurnToolCall[]; text?: string} + export type ScriptedRun = { chatStream: (deps: HarnessChatDeps) => AsyncGenerator hold: () => void release: () => void scriptToolCall: (name: string, input: unknown, opts?: {blocking?: boolean}) => string + scriptTurn: (turn: ScriptedTurn) => string[] scriptCustomEvent: (name: string, value: unknown) => void scriptError: (message: string) => void } +type QueuedToolCall = {id: string; name: string; input: unknown; result: unknown} + +type QueuedTurn = {toolCalls: QueuedToolCall[]; text?: string} + +const THREAD = {threadId: 'scripted', runId: 'scripted'} as const + +function* requestChunks(call: {id: string; name: string; input: unknown}): Generator { + const toolCallId = call.id + yield {type: EventType.TOOL_CALL_START, toolCallId, toolCallName: call.name, toolName: call.name} + yield {type: EventType.TOOL_CALL_ARGS, toolCallId, delta: JSON.stringify(call.input)} + yield {type: EventType.TOOL_CALL_END, toolCallId} +} + +function resultChunk(toolCallId: string, result: unknown): StreamChunk { + return { + type: EventType.TOOL_CALL_RESULT, + messageId: `${toolCallId}-result`, + toolCallId, + content: JSON.stringify(result), + state: 'output-available', + } +} + +function* turnChunks(turn: QueuedTurn): Generator { + for (const call of turn.toolCalls) { + yield* requestChunks(call) + yield resultChunk(call.id, call.result) + } +} + +function* customEventChunks(events: {name: string; value: unknown}[]): Generator { + for (const event of events) yield {type: EventType.CUSTOM, name: event.name, value: event.value, ...THREAD} +} + export function makeScriptedRun(opts: {text?: string} = {}): ScriptedRun { + const defaultText = opts.text ?? 'ok' const gate = {held: false, waiting: new Set<() => void>()} const turns = {count: 0} const toolCalls = {count: 0} const queuedToolCalls: Array<{id: string; name: string; input: unknown; blocking: boolean}> = [] + const queuedTurns: QueuedTurn[] = [] const queuedCustomEvents: Array<{name: string; value: unknown}> = [] const queuedErrors: string[] = [] const hold = () => { @@ -32,6 +73,14 @@ export function makeScriptedRun(opts: {text?: string} = {}): ScriptedRun { queuedToolCalls.push({id: toolCallId, name, input, blocking: toolOpts.blocking ?? true}) return toolCallId } + const scriptTurn = (turn: ScriptedTurn) => { + const calls = turn.toolCalls.map((call) => { + toolCalls.count += 1 + return {id: `tc-${toolCalls.count}`, name: call.name, input: call.input, result: call.result ?? {ok: true}} + }) + queuedTurns.push({toolCalls: calls, text: turn.text}) + return calls.map((call) => call.id) + } const scriptCustomEvent = (name: string, value: unknown) => { queuedCustomEvents.push({name, value}) } @@ -41,40 +90,25 @@ export function makeScriptedRun(opts: {text?: string} = {}): ScriptedRun { const chatStream = async function* (deps: HarnessChatDeps): AsyncGenerator { turns.count += 1 const messageId = `scripted-${turns.count}` - yield {type: EventType.RUN_STARTED, threadId: 'scripted', runId: 'scripted'} - yield { - type: EventType.CUSTOM, - name: 'fake.session-id', - value: {sessionId: `fake-${deps.sessionId}`}, - threadId: 'scripted', - runId: 'scripted', - } - const toolCall = queuedToolCalls.shift() + yield {type: EventType.RUN_STARTED, ...THREAD} + yield {type: EventType.CUSTOM, name: 'fake.session-id', value: {sessionId: `fake-${deps.sessionId}`}, ...THREAD} + const scriptedTurn = queuedTurns.shift() + if (scriptedTurn) yield* turnChunks(scriptedTurn) + const toolCall = scriptedTurn ? undefined : queuedToolCalls.shift() if (toolCall) { - const toolCallId = toolCall.id - yield {type: EventType.TOOL_CALL_START, toolCallId, toolCallName: toolCall.name, toolName: toolCall.name} - yield {type: EventType.TOOL_CALL_ARGS, toolCallId, delta: JSON.stringify(toolCall.input)} - yield {type: EventType.TOOL_CALL_END, toolCallId} + yield* requestChunks(toolCall) if (toolCall.blocking) { - yield {type: EventType.RUN_FINISHED, threadId: 'scripted', runId: 'scripted', finishReason: 'tool_calls'} + yield {type: EventType.RUN_FINISHED, ...THREAD, finishReason: 'tool_calls'} return } - yield { - type: EventType.TOOL_CALL_RESULT, - messageId: `${toolCallId}-result`, - toolCallId, - content: JSON.stringify({ok: true}), - state: 'output-available', - } - } - for (const event of queuedCustomEvents.splice(0)) { - yield {type: EventType.CUSTOM, name: event.name, value: event.value, threadId: 'scripted', runId: 'scripted'} + yield resultChunk(toolCall.id, {ok: true}) } - yield {type: EventType.TEXT_MESSAGE_CONTENT, messageId, delta: opts.text ?? 'ok'} + yield* customEventChunks(queuedCustomEvents.splice(0)) + yield {type: EventType.TEXT_MESSAGE_CONTENT, messageId, delta: scriptedTurn?.text ?? defaultText} if (gate.held) await new Promise((resolve) => gate.waiting.add(resolve)) const failure = queuedErrors.shift() if (failure) throw new Error(failure) - yield {type: EventType.RUN_FINISHED, threadId: 'scripted', runId: 'scripted'} + yield {type: EventType.RUN_FINISHED, ...THREAD} } - return {chatStream, hold, release, scriptToolCall, scriptCustomEvent, scriptError} + return {chatStream, hold, release, scriptToolCall, scriptTurn, scriptCustomEvent, scriptError} } diff --git a/packages/harness-testkit/src/testkit.ts b/packages/harness-testkit/src/testkit.ts index d726d05b4..43481cc8a 100644 --- a/packages/harness-testkit/src/testkit.ts +++ b/packages/harness-testkit/src/testkit.ts @@ -11,6 +11,7 @@ export { } from './create-testkit.js' export {createTestHarness, type TestHarness} from './create-test-harness.js' export {createFakeHarness, type FakeHarness} from './create-fake-harness.js' +export type {ScriptedRun, ScriptedTurn, ScriptedTurnToolCall} from './scripted-run.js' export {harnessAvailable} from './harness-available.js' export { makeApprovingCallTool, diff --git a/packages/harness-testkit/test/create-fake-harness.test.ts b/packages/harness-testkit/test/create-fake-harness.test.ts index c8f7d16a0..2326b644c 100644 --- a/packages/harness-testkit/test/create-fake-harness.test.ts +++ b/packages/harness-testkit/test/create-fake-harness.test.ts @@ -15,6 +15,26 @@ const context: HarnessConnectContext = { hookUrl: null, } +describe('createFakeHarness transcript history', () => { + it('declares no transcript history by default', () => { + const harness = createFakeHarness() + expect(harness.capabilities.transcriptHistory).toBe(false) + expect(harness.history).toBeUndefined() + }) + + it('serves the injected rows as the harness transcript list', async () => { + const rows = [ + {id: 'external-1', derivedTitle: 'An external session', updatedAt: 1_700, messageCount: 7}, + {id: 'external-2', derivedTitle: 'Another one', updatedAt: 1_800, messageCount: 2}, + ] + const harness = createFakeHarness({history: rows}) + + expect(harness.capabilities.transcriptHistory).toBe(true) + expect(await harness.history?.list('/project')).toEqual(rows) + expect(await harness.history?.messages('/project', 'external-1')).toEqual([]) + }) +}) + describe('createFakeHarness tty', () => { it('has no tty by default', () => { expect(createFakeHarness().tty).toBeUndefined() diff --git a/packages/harness-testkit/test/scripted-run.test.ts b/packages/harness-testkit/test/scripted-run.test.ts index 455bc04ee..7cb29ff36 100644 --- a/packages/harness-testkit/test/scripted-run.test.ts +++ b/packages/harness-testkit/test/scripted-run.test.ts @@ -40,6 +40,49 @@ describe('makeScriptedRun', () => { expect(secondEmittedId).toBe(secondScriptedId) }) + it('keeps two independently queued tool calls in two separate turns', async () => { + const scripted = makeScriptedRun() + scripted.scriptToolCall('first_tool', {a: 1}) + scripted.scriptToolCall('second_tool', {b: 2}) + const drainTurn = async (): Promise => { + const chunks: StreamChunk[] = [] + for await (const chunk of scripted.chatStream(deps())) chunks.push(chunk) + return chunks + } + const first = await drainTurn() + const second = await drainTurn() + const startsIn = (chunks: StreamChunk[]): string[] => + chunks.flatMap((chunk) => (chunk.type === EventType.TOOL_CALL_START ? [chunk.toolCallName] : [])) + expect(startsIn(first)).toEqual(['first_tool']) + expect(startsIn(second)).toEqual(['second_tool']) + }) + + it('emits every tool call of a scripted turn, with its result, inside one turn', async () => { + const scripted = makeScriptedRun() + const ids = scripted.scriptTurn({ + toolCalls: [ + {name: 'first_tool', input: {a: 1}, result: {ok: 'one'}}, + {name: 'second_tool', input: {b: 2}, result: {ok: 'two'}}, + ], + text: 'both tools ran', + }) + const chunks: StreamChunk[] = [] + for await (const chunk of scripted.chatStream(deps())) chunks.push(chunk) + + const starts = chunks.flatMap((chunk) => (chunk.type === EventType.TOOL_CALL_START ? [chunk.toolCallId] : [])) + const results = chunks.flatMap((chunk) => + chunk.type === EventType.TOOL_CALL_RESULT ? [{id: chunk.toolCallId, content: chunk.content}] : [], + ) + const text = chunks.flatMap((chunk) => (chunk.type === EventType.TEXT_MESSAGE_CONTENT ? [chunk.delta] : [])) + expect(starts).toEqual(ids) + expect(results).toEqual([ + {id: ids[0], content: JSON.stringify({ok: 'one'})}, + {id: ids[1], content: JSON.stringify({ok: 'two'})}, + ]) + expect(text).toEqual(['both tools ran']) + expect(chunks.at(-1)?.type).toBe(EventType.RUN_FINISHED) + }) + it('holds the turn open until release()', async () => { const scripted = makeScriptedRun() scripted.hold() From 73cd8819131b620691f1f4c60e0dc04528970041 Mon Sep 17 00:00:00 2001 From: Omri Katz <9701896+omridevk@users.noreply.github.com> Date: Sat, 15 Aug 2026 05:26:30 +0300 Subject: [PATCH 2/9] =?UTF-8?q?test(conciv):=20#354=20phase=202=20?= =?UTF-8?q?=E2=80=94=20pane-harness=20group=20runs=20against=20the=20real?= =?UTF-8?q?=20core?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit chat-pane, session-selector, engine-staleness, launch-actions and page-session-card boot a real core per file and drive it through the real rpc surface: drafts seeded through drafts/set, held runs through the scripted harness, transcripts produced by real turns, faults injected at the playwright network layer. pane-harness takes the booted base and pins the browser rpc transport to fetch. Seams added along the way: an abort fault kind for transport failures, a per-boot rpc observer for wire-call counts, a resume-capable fake harness that reports the resumed session id instead of minting a new one, and createTerminalExtension({openTerminal}) so terminal launches are testable without spawning a real terminal window. Co-Authored-By: Claude Fable 5 --- .changeset/terminal-launch-opener-seam.md | 5 + apps/conciv/test/chat-pane.browser.test.tsx | 210 ++++++++++-------- apps/conciv/test/commands/core-control.ts | 72 +++++- apps/conciv/test/commands/core-testkit.ts | 4 +- .../test/engine-staleness.browser.test.tsx | 76 +++++-- apps/conciv/test/helpers/core-control.ts | 3 + apps/conciv/test/helpers/core-session.ts | 58 +++++ apps/conciv/test/helpers/pane-harness.tsx | 24 +- .../test/launch-actions.browser.test.tsx | 79 ++++--- .../test/page-session-card.browser.test.tsx | 59 +++-- .../test/session-selector.browser.test.tsx | 64 ++++-- packages/extension-testkit/src/core-kit.ts | 8 +- packages/extension-testkit/src/rpc-fault.ts | 21 ++ packages/extensions/terminal/src/server.ts | 45 ++-- .../src/create-fake-harness.ts | 25 ++- packages/harness-testkit/src/scripted-run.ts | 7 +- 16 files changed, 534 insertions(+), 226 deletions(-) create mode 100644 .changeset/terminal-launch-opener-seam.md create mode 100644 apps/conciv/test/helpers/core-session.ts diff --git a/.changeset/terminal-launch-opener-seam.md b/.changeset/terminal-launch-opener-seam.md new file mode 100644 index 000000000..2cf0ddb09 --- /dev/null +++ b/.changeset/terminal-launch-opener-seam.md @@ -0,0 +1,5 @@ +--- +'@conciv/extension-terminal': patch +--- + +the terminal extension is created through `createTerminalExtension({openTerminal})`, so the host that spawns the terminal window is injectable; the default export is unchanged diff --git a/apps/conciv/test/chat-pane.browser.test.tsx b/apps/conciv/test/chat-pane.browser.test.tsx index a4dc0700a..c8ac47c4b 100644 --- a/apps/conciv/test/chat-pane.browser.test.tsx +++ b/apps/conciv/test/chat-pane.browser.test.tsx @@ -1,111 +1,134 @@ import './helpers/utilities.css' -import {afterEach, expect, test} from 'vitest' +import {afterAll, afterEach, beforeAll, expect, test} from 'vitest' import {page, userEvent} from 'vitest/browser' +import type {RpcClient} from '@conciv/contract' import {ChatPane} from '../src/pane/chat-pane.js' -import {installFakeCore, sessionRow, type FakeCore, type FakeCoreConfig} from './helpers/fake-core.js' -import {mountPane, PANE_SESSION, type PaneMount} from './helpers/pane-harness.js' - -let core: FakeCore | null = null - -afterEach(() => { - core?.restore() - core = null +import {coreControl} from './helpers/core-control.js' +import {coreRpc, createSession, openTranscriptStream, runTurn, seedDraft, sendTurn} from './helpers/core-session.js' +import {mountPane, type PaneMount} from './helpers/pane-harness.js' + +const SEND_PATH = ['chat', 'send'] +const SUBSCRIBE_PATH = ['chat', 'subscribe'] + +const core = {base: ''} +const active: {pane: PaneMount | null; faults: string[]} = {pane: null, faults: []} + +beforeAll(async () => { + const booted = await coreControl.bootCore({id: 'chat-pane', allowedOrigins: [window.location.origin]}) + core.base = booted.base +}, 60_000) + +afterAll(async () => { + await coreControl.closeCore() +}, 30_000) + +afterEach(async () => { + for (const fault of active.faults.splice(0)) await coreControl.releaseFault(fault) + await coreControl.releaseTurn() + active.pane?.dispose() + active.pane = null }) -function mountChatPane(config: FakeCoreConfig = {}): PaneMount { - core = installFakeCore({sessions: [sessionRow({id: PANE_SESSION})], ...config}) - return mountPane(() => ) +async function installFault(spec: Parameters[0]): Promise { + const handle = await coreControl.installFault(spec) + active.faults.push(handle) + return handle } -const input = () => page.getByRole('textbox', {name: 'Message the conciv agent'}) -const removeGrab = () => page.getByRole('button', {name: 'Remove grabbed element'}) +async function newSession(): Promise<{rpc: RpcClient; sessionId: string}> { + const rpc = coreRpc(core.base) + return {rpc, sessionId: await createSession(rpc)} +} -function draftWithGrab(text: string): FakeCoreConfig['draft'] { - return { - sessionId: PANE_SESSION, - text: '', - selectionStart: 0, - selectionEnd: 0, - grabs: [text], - updatedAt: 1, - } +function mountChatPane(sessionId: string): PaneMount { + const mount = mountPane({base: core.base, sessionId}, () => ) + active.pane = mount + return mount } -async function sendWithStagedGrab(config: FakeCoreConfig): Promise { - mountChatPane({...config, draft: draftWithGrab('the grabbed hero section')}) +const input = () => page.getByRole('textbox', {name: 'Message the conciv agent'}) +const removeGrab = () => page.getByRole('button', {name: 'Remove grabbed element'}) +const notifications = () => page.getByRole('region', {name: /Notifications/}) +const stopButton = () => page.getByRole('button', {name: 'Stop generating'}) +const skeleton = () => page.getByRole('status', {name: 'Loading conversation'}) + +async function sendWithStagedGrab(): Promise { + const {rpc, sessionId} = await newSession() + await seedDraft(rpc, sessionId, {grabs: ['the grabbed hero section']}) + mountChatPane(sessionId) await expect.element(removeGrab()).toBeVisible() await input().fill('explain the section I grabbed') await userEvent.keyboard('{Enter}') } async function startStreamingRun(): Promise { - mountChatPane({holdRun: true}) + const {sessionId} = await newSession() + await coreControl.holdTurn() + mountChatPane(sessionId) await input().fill('first turn') await userEvent.keyboard('{Enter}') - await expect.element(page.getByRole('button', {name: 'Stop generating'})).toBeVisible() + await expect.element(stopButton()).toBeVisible() } test('restores the server-side draft text and staged grabs when the pane mounts', async () => { - mountChatPane({ - draft: { - sessionId: PANE_SESSION, - text: 'kept across the reload', - selectionStart: 22, - selectionEnd: 22, - grabs: ['a grabbed heading'], - updatedAt: 1, - }, - }) + const {rpc, sessionId} = await newSession() + await seedDraft(rpc, sessionId, {text: 'kept across the reload', grabs: ['a grabbed heading']}) + + mountChatPane(sessionId) await expect.element(input()).toHaveTextContent('kept across the reload') await expect.element(page.getByText('a grabbed heading')).toBeVisible() }) test('a rejected send keeps the draft in the composer and tells the user why', async () => { - mountChatPane({rejectSend: true}) + const {sessionId} = await newSession() + await installFault({kind: 'fail', path: SEND_PATH, status: 500}) + mountChatPane(sessionId) await expect.element(input()).toBeVisible() await input().fill('a message the server refuses') await userEvent.keyboard('{Enter}') - await expect - .element(page.getByRole('region', {name: /Notifications/})) - .toHaveTextContent(/Internal Server Error|could not be sent/) + await expect.element(notifications()).toHaveTextContent(/Internal Server Error|could not be sent/) await expect.element(input()).toHaveTextContent('a message the server refuses') }) test('sending drops the staged grab card at once, while the turn is still streaming', async () => { - await sendWithStagedGrab({holdRun: true}) + await coreControl.holdTurn() + await sendWithStagedGrab() - await expect.element(page.getByRole('button', {name: 'Stop generating'})).toBeVisible() + await expect.element(stopButton()).toBeVisible() await expect.element(removeGrab()).not.toBeInTheDocument() }) test('a send the server refuses puts the staged grab card back', async () => { - await sendWithStagedGrab({rejectSend: true}) + await installFault({kind: 'fail', path: SEND_PATH, status: 500}) + await sendWithStagedGrab() - await expect - .element(page.getByRole('region', {name: /Notifications/})) - .toHaveTextContent(/Internal Server Error|could not be sent/) + await expect.element(notifications()).toHaveTextContent(/Internal Server Error|could not be sent/) await expect.element(removeGrab()).toBeVisible() await expect.element(page.getByText('the grabbed hero section')).toBeVisible() }) test('a send that throws at the transport puts the staged grab card back', async () => { - await sendWithStagedGrab({throwSend: true}) + await installFault({kind: 'abort', path: SEND_PATH}) + await sendWithStagedGrab() - await expect.element(page.getByRole('region', {name: /Notifications/})).toHaveTextContent(/could not be sent|fetch/) + await expect.element(notifications()).toHaveTextContent(/could not be sent|fetch/) await expect.element(removeGrab()).toBeVisible() await expect.element(page.getByText('the grabbed hero section')).toBeVisible() }) test('a queued second send cannot cross-restore the grabs of the turn that failed', async () => { - const mount = mountChatPane({holdRun: true, draft: draftWithGrab('the grabbed hero section')}) + const {rpc, sessionId} = await newSession() + await seedDraft(rpc, sessionId, {grabs: ['the grabbed hero section']}) + await coreControl.holdTurn() + const mount = mountChatPane(sessionId) await expect.element(page.getByText('the grabbed hero section')).toBeVisible() await input().fill('turn A') await userEvent.keyboard('{Enter}') - await expect.element(page.getByRole('button', {name: 'Stop generating'})).toBeVisible() + await expect.element(stopButton()).toBeVisible() mount.pane.grabStore.stageAll([{text: 'the grabbed pricing table'}]) await expect.element(page.getByText('the grabbed pricing table')).toBeVisible() @@ -114,14 +137,16 @@ test('a queued second send cannot cross-restore the grabs of the turn that faile await expect.element(page.getByRole('button', {name: 'Remove from queue'})).toBeVisible() await expect.element(page.getByText('the grabbed pricing table')).not.toBeInTheDocument() - core?.push({type: 'RUN_ERROR', threadId: 'conciv_1', runId: 'conciv_run_1', message: 'turn A failed'}) + await coreControl.scriptError('turn A failed') + await coreControl.releaseTurn() await expect.element(page.getByText('the grabbed hero section')).toBeVisible() await expect.element(page.getByText('the grabbed pricing table')).not.toBeInTheDocument() }) test('sending announces thinking and then the reply through the live region', async () => { - mountChatPane() + const {sessionId} = await newSession() + mountChatPane(sessionId) await expect.element(input()).toBeVisible() await input().fill('rename the widget package') @@ -132,57 +157,67 @@ test('sending announces thinking and then the reply through the live region', as }) test('the refresh affordance re-subscribes and shows the transcript the server re-leads', async () => { - mountChatPane({ - snapshotFor: (subscribeIndex) => - subscribeIndex < 2 - ? [] - : [{id: 'a1', role: 'assistant', parts: [{type: 'text', content: 'the refreshed transcript'}]}], - }) + const {rpc, sessionId} = await newSession() + mountChatPane(sessionId) + await expect.element(page.getByText('How can I help you today?')).toBeVisible() - await expect.element(input()).toBeVisible() + const stream = await openTranscriptStream(rpc, sessionId) + const gate = await installFault({kind: 'gate', path: SUBSCRIBE_PATH}) await page.getByRole('button', {name: 'Refresh the conversation'}).click() + await coreControl.scriptTurn({toolCalls: [], text: 'the refreshed transcript'}) + await sendTurn(rpc, sessionId, 'lead the transcript from the server') + await stream.awaitTurnEnd() + stream.close() + await coreControl.releaseFault(gate) + await expect.element(page.getByText('the refreshed transcript')).toBeVisible() }) test('the refresh affordance is disabled while the run streams', async () => { - mountChatPane({holdRun: true}) + const {sessionId} = await newSession() + await coreControl.holdTurn() + mountChatPane(sessionId) await expect.element(input()).toBeVisible() await input().fill('start a run') await userEvent.keyboard('{Enter}') - await expect.element(page.getByRole('button', {name: 'Stop generating'})).toBeVisible() + await expect.element(stopButton()).toBeVisible() await expect.element(page.getByRole('button', {name: 'Refresh the conversation'})).toBeDisabled() }) test('the initial load shows a conversation skeleton until the snapshot arrives', async () => { - mountChatPane({holdSnapshot: true}) + const {sessionId} = await newSession() + const gate = await installFault({kind: 'gate', path: SUBSCRIBE_PATH}) + mountChatPane(sessionId) - await expect.element(page.getByRole('status', {name: 'Loading conversation'})).toBeVisible() + await expect.element(skeleton()).toBeVisible() - core?.releaseSnapshot() + await coreControl.releaseFault(gate) await expect.element(page.getByText('How can I help you today?')).toBeVisible() - await expect.element(page.getByRole('status', {name: 'Loading conversation'})).not.toBeInTheDocument() + await expect.element(skeleton()).not.toBeInTheDocument() }) test('the trailing control morphs from send to a single stop button while streaming, and back once the run stops', async () => { - mountChatPane({holdRun: true}) + const {sessionId} = await newSession() + await coreControl.holdTurn() + mountChatPane(sessionId) await expect.element(page.getByRole('button', {name: 'Send message'})).toBeVisible() - await expect.element(page.getByRole('button', {name: 'Stop generating'})).not.toBeInTheDocument() + await expect.element(stopButton()).not.toBeInTheDocument() await input().fill('start a run') await userEvent.keyboard('{Enter}') - await expect.element(page.getByRole('button', {name: 'Stop generating'})).toBeVisible() + await expect.element(stopButton()).toBeVisible() await expect.element(page.getByRole('button', {name: 'Send message'})).not.toBeInTheDocument() - await page.getByRole('button', {name: 'Stop generating'}).click() + await stopButton().click() await expect.element(page.getByRole('button', {name: 'Send message'})).toBeVisible() - await expect.element(page.getByRole('button', {name: 'Stop generating'})).not.toBeInTheDocument() + await expect.element(stopButton()).not.toBeInTheDocument() }) test('Enter while streaming queues the draft instead of sending or stopping', async () => { @@ -192,7 +227,7 @@ test('Enter while streaming queues the draft instead of sending or stopping', as await userEvent.keyboard('{Enter}') await expect.element(page.getByText('a queued follow-up')).toBeVisible() - await expect.element(page.getByRole('button', {name: 'Stop generating'})).toBeVisible() + await expect.element(stopButton()).toBeVisible() await expect.element(input()).toHaveTextContent('') }) @@ -208,7 +243,7 @@ test('clicking stop with a queue interrupts the run and flushes every queued mes await expect.element(page.getByText('bravo step')).toBeVisible() await expect.element(page.getByRole('button', {name: 'Remove from queue'}).first()).toBeVisible() - await page.getByRole('button', {name: 'Stop generating'}).click() + await stopButton().click() await expect.element(page.getByRole('button', {name: 'Remove from queue'})).not.toBeInTheDocument() await expect.element(page.getByText(/alpha step[\s\S]*bravo step/)).toBeVisible() @@ -217,7 +252,7 @@ test('clicking stop with a queue interrupts the run and flushes every queued mes test('clicking stop with an empty queue just stops the run', async () => { await startStreamingRun() - await page.getByRole('button', {name: 'Stop generating'}).click() + await stopButton().click() await expect.element(page.getByRole('button', {name: 'Send message'})).toBeVisible() }) @@ -234,7 +269,7 @@ test('Escape in the focused composer does what the stop button does and leaves t await expect.element(page.getByRole('button', {name: 'Remove from queue'})).not.toBeInTheDocument() await expect.element(page.getByText('queued while running')).toBeVisible() - await expect.element(page.getByRole('button', {name: 'Stop generating'})).toBeVisible() + await expect.element(stopButton()).toBeVisible() await expect.element(input()).toHaveTextContent('draft kept in place') }) @@ -244,26 +279,23 @@ test('Escape outside the composer does not stop the run', async () => { await userEvent.click(page.getByRole('log', {name: 'Announcements'})) await userEvent.keyboard('{Escape}') - await expect.element(page.getByRole('button', {name: 'Stop generating'})).toBeVisible() + await expect.element(stopButton()).toBeVisible() }) test('a new-session divider does not flash before the transcript snapshot hydrates', async () => { - mountChatPane({ - holdSnapshot: true, - markers: [{id: 'marker-1', sessionId: PANE_SESSION, afterTurn: 0, kind: 'new'}], - snapshotFor: () => [ - {id: 'u1', role: 'user', parts: [{type: 'text', content: 'restart with a clean slate'}]}, - {id: 'a1', role: 'assistant', parts: [{type: 'text', content: 'starting a fresh session'}]}, - ], - }) - - await expect.element(page.getByRole('status', {name: 'Loading conversation'})).toBeVisible() - await core?.idle() + const {rpc, sessionId} = await newSession() + await coreControl.scriptTurn({toolCalls: [], text: 'starting a fresh session'}) + await runTurn(rpc, sessionId, 'restart with a clean slate') + const gate = await installFault({kind: 'gate', path: SUBSCRIBE_PATH}) + const mount = mountChatPane(sessionId) + + await expect.element(skeleton()).toBeVisible() + await mount.queryClient.ensureQueryData(mount.data.utils.markers.list.queryOptions({input: {sessionId}})) await expect.element(page.getByRole('separator', {name: 'New session'})).not.toBeInTheDocument() - core?.releaseSnapshot() + await coreControl.releaseFault(gate) await expect.element(page.getByText('starting a fresh session')).toBeVisible() await expect.element(page.getByRole('separator', {name: 'New session'})).toBeVisible() - await expect.element(page.getByRole('status', {name: 'Loading conversation'})).not.toBeInTheDocument() + await expect.element(skeleton()).not.toBeInTheDocument() }) diff --git a/apps/conciv/test/commands/core-control.ts b/apps/conciv/test/commands/core-control.ts index 43ebb308d..5225da034 100644 --- a/apps/conciv/test/commands/core-control.ts +++ b/apps/conciv/test/commands/core-control.ts @@ -4,27 +4,39 @@ import type {EngineStaleness} from '@conciv/contract' import type {HarnessSessionMeta} from '@conciv/protocol/harness-types' import type {ScriptedTurn} from '@conciv/harness-testkit' import type {CoreKit} from './core-testkit.js' +import type {RpcObserver} from '@conciv/extension-testkit/rpc-observer' export type BootCoreInput = { id: string text?: string + resume?: boolean + displayName?: string + connect?: boolean + terminal?: boolean history?: HarnessSessionMeta[] allowedOrigins?: string[] } export type BootCoreResult = {base: string; wsBase: string; bootMs: number} -export type FaultSpec = {kind: 'fail'; path: string[]; status?: number} | {kind: 'gate'; path?: string[]} +export type FaultSpec = + | {kind: 'fail'; path: string[]; status?: number} + | {kind: 'abort'; path: string[]} + | {kind: 'gate'; path?: string[]} type Fault = {pending: () => number; release: () => Promise; dispose: () => Promise} type CoreTestkit = typeof import('./core-testkit.js') +type TerminalState = {launches: number; succeeds: boolean} + type FileState = { kit: CoreKit | null staleness: {value: EngineStaleness} faults: Map handles: {count: number} + terminal: TerminalState + observer: RpcObserver | null } const FRESH: EngineStaleness = { @@ -46,7 +58,14 @@ function stateOf(ctx: BrowserCommandContext): FileState { const key = ctx.testPath ?? 'shared' const existing = files.get(key) if (existing) return existing - const created: FileState = {kit: null, staleness: {value: FRESH}, faults: new Map(), handles: {count: 0}} + const created: FileState = { + kit: null, + staleness: {value: FRESH}, + faults: new Map(), + handles: {count: 0}, + terminal: {launches: 0, succeeds: true}, + observer: null, + } files.set(key, created) return created } @@ -57,6 +76,12 @@ function kitOf(ctx: BrowserCommandContext): CoreKit { return kit } +function observerOf(ctx: BrowserCommandContext): RpcObserver { + const observer = stateOf(ctx).observer + if (!observer) throw new Error('no core is booted for this test file; call bootCore first') + return observer +} + function faultOf(ctx: BrowserCommandContext, handle: string): Fault { const fault = stateOf(ctx).faults.get(handle) if (!fault) throw new Error(`no fault is installed under the handle "${handle}"`) @@ -67,15 +92,34 @@ const bootCore: BrowserCommand<[BootCoreInput]> = async (ctx, input): Promise => { + state.terminal.launches += 1 + return Promise.resolve(state.terminal.succeeds) + } + const {bootCoreKit, createTerminalExtension, observeRpc} = await testkitOf(ctx) const kit = await bootCoreKit({ id: input.id, text: input.text, + resume: input.resume, + displayName: input.displayName, history: input.history, allowedOrigins: input.allowedOrigins, staleness: () => state.staleness.value, + ...(input.terminal ? {extensions: [createTerminalExtension({openTerminal: openedTerminal})]} : {}), + ...(input.connect + ? { + connect: { + plan: (context) => ({ + argv: ['claude', '--resume', context.harnessSessionId ?? 'new'], + env: {}, + files: [], + }), + }, + } + : {}), }) state.kit = kit + state.observer = observeRpc(ctx.page) return {base: kit.base, wsBase: kit.wsBase, bootMs: Date.now() - startedAt} } @@ -84,6 +128,10 @@ const closeCore: BrowserCommand<[]> = async (ctx): Promise => { for (const fault of state.faults.values()) await fault.dispose() state.faults.clear() state.staleness.value = FRESH + state.terminal.launches = 0 + state.terminal.succeeds = true + state.observer?.dispose() + state.observer = null const kit = state.kit state.kit = null if (!kit) return @@ -109,16 +157,27 @@ const scriptError: BrowserCommand<[string]> = (ctx, message): void => { const scriptTurn: BrowserCommand<[ScriptedTurn]> = (ctx, turn): string[] => kitOf(ctx).harness.script.scriptTurn(turn) +const setTerminalLaunch: BrowserCommand<[boolean]> = (ctx, succeeds): void => { + stateOf(ctx).terminal.succeeds = succeeds +} + +const terminalLaunches: BrowserCommand<[]> = (ctx): number => stateOf(ctx).terminal.launches + +const rpcCallCount: BrowserCommand<[string[]]> = (ctx, path): number => observerOf(ctx).completedCount({path}) + const installFault: BrowserCommand<[FaultSpec]> = async (ctx, spec): Promise => { const state = stateOf(ctx) - const {failRpcCalls, gateRpcCalls} = await testkitOf(ctx) + const {abortRpcCalls, failRpcCalls, gateRpcCalls} = await testkitOf(ctx) state.handles.count += 1 const handle = `fault-${state.handles.count}` if (spec.kind === 'gate') { state.faults.set(handle, await gateRpcCalls(ctx.page, spec.path ? {path: spec.path} : {})) return handle } - const injector = await failRpcCalls(ctx.page, {path: spec.path, ...(spec.status ? {status: spec.status} : {})}) + const injector = + spec.kind === 'abort' + ? await abortRpcCalls(ctx.page, {path: spec.path}) + : await failRpcCalls(ctx.page, {path: spec.path, ...(spec.status ? {status: spec.status} : {})}) state.faults.set(handle, { pending: () => 0, release: () => { @@ -144,6 +203,9 @@ export const coreCommands = { releaseTurn, scriptError, scriptTurn, + setTerminalLaunch, + terminalLaunches, + rpcCallCount, installFault, releaseFault, faultPending, diff --git a/apps/conciv/test/commands/core-testkit.ts b/apps/conciv/test/commands/core-testkit.ts index e65b1a3cd..f445aa6a3 100644 --- a/apps/conciv/test/commands/core-testkit.ts +++ b/apps/conciv/test/commands/core-testkit.ts @@ -1,2 +1,4 @@ export {bootCoreKit, type CoreKit} from '@conciv/extension-testkit/core-kit' -export {failRpcCalls, gateRpcCalls} from '@conciv/extension-testkit/rpc-fault' +export {abortRpcCalls, failRpcCalls, gateRpcCalls} from '@conciv/extension-testkit/rpc-fault' +export {observeRpc, type RpcObserver} from '@conciv/extension-testkit/rpc-observer' +export {createTerminalExtension} from '@conciv/extension-terminal/server' diff --git a/apps/conciv/test/engine-staleness.browser.test.tsx b/apps/conciv/test/engine-staleness.browser.test.tsx index c348bb74d..f5890abae 100644 --- a/apps/conciv/test/engine-staleness.browser.test.tsx +++ b/apps/conciv/test/engine-staleness.browser.test.tsx @@ -1,26 +1,56 @@ import '@conciv/ui-kit-system/tokens.css' import './helpers/utilities.css' -import {afterEach, expect, test} from 'vitest' +import {afterAll, afterEach, beforeAll, expect, test} from 'vitest' import {page} from 'vitest/browser' +import type {EngineStaleness} from '@conciv/contract' import {EngineStaleNotice} from '../src/shell/engine-notice.js' -import {installFakeCore, sessionRow, type FakeCore} from './helpers/fake-core.js' -import {mountPane, PANE_SESSION} from './helpers/pane-harness.js' +import {coreControl} from './helpers/core-control.js' +import {coreRpc, createSession} from './helpers/core-session.js' +import {mountPane, type PaneMount} from './helpers/pane-harness.js' + +const core = {base: ''} +const mounted: {pane: PaneMount | null} = {pane: null} + +const TRACKED = ['@conciv/core', '@conciv/tools'] + +function staleness(stale: boolean, fingerprint: string): EngineStaleness { + return { + stale, + changed: stale ? ['@conciv/tools'] : [], + tracked: TRACKED, + bootedAt: 0, + fingerprint, + } +} + +beforeAll(async () => { + const booted = await coreControl.bootCore({id: 'engine-staleness', allowedOrigins: [window.location.origin]}) + core.base = booted.base +}, 60_000) -let core: FakeCore | null = null +afterAll(async () => { + await coreControl.closeCore() +}, 30_000) afterEach(() => { - core?.restore() - core = null + mounted.pane?.dispose() + mounted.pane = null }) -function mountNotice(config: Parameters[0] = {}): {refetch: () => Promise} { - core = installFakeCore({sessions: [sessionRow({id: PANE_SESSION})], ...config}) - const mounted = mountPane(() => ) - return {refetch: mounted.refetch} +async function mountNotice(engine: EngineStaleness): Promise { + await coreControl.setStaleness(engine) + const sessionId = await createSession(coreRpc(core.base)) + const pane = mountPane({base: core.base, sessionId}, () => ) + mounted.pane = pane + return pane +} + +async function settleEngine(pane: PaneMount): Promise { + await pane.queryClient.ensureQueryData(pane.data.utils.meta.engine.queryOptions()) } test('an engine running outdated code says so, and says it is the server code that moved', async () => { - mountNotice({engineStale: true}) + await mountNotice(staleness(true, 'stamp-boot')) await expect .element(page.getByRole('alert')) @@ -29,15 +59,15 @@ test('an engine running outdated code says so, and says it is the server code th }) test('an engine that matches the code on disk raises nothing at all', async () => { - mountNotice({engineStale: false}) + const pane = await mountNotice(staleness(false, 'stamp-boot')) - await core?.idle() + await settleEngine(pane) await expect.element(page.getByRole('alert')).not.toBeInTheDocument() }) test('the outdated-engine notice stands until it is dismissed by hand', async () => { - mountNotice({engineStale: true}) + await mountNotice(staleness(true, 'stamp-boot')) await expect.element(page.getByRole('alert')).toBeVisible() await page.getByRole('button', {name: 'Dismiss'}).click() @@ -46,33 +76,33 @@ test('the outdated-engine notice stands until it is dismissed by hand', async () }) test('restarting the engine takes the notice down without anyone dismissing it', async () => { - const mounted = mountNotice({engineStale: true}) + const pane = await mountNotice(staleness(true, 'stamp-boot')) await expect.element(page.getByRole('alert')).toBeVisible() - core?.setEngine({stale: false, fingerprint: 'stamp-restarted'}) - await mounted.refetch() + await coreControl.setStaleness(staleness(false, 'stamp-restarted')) + await pane.refetch() await expect.element(page.getByRole('alert')).not.toBeInTheDocument() }) test('a dismissed notice stays down while the engine is stale in the very same way', async () => { - const mounted = mountNotice({engineStale: true}) + const pane = await mountNotice(staleness(true, 'stamp-boot')) await page.getByRole('button', {name: 'Dismiss'}).click() await expect.element(page.getByRole('alert')).not.toBeInTheDocument() - await mounted.refetch() - await core?.idle() + await pane.refetch() + await settleEngine(pane) await expect.element(page.getByRole('alert')).not.toBeInTheDocument() }) test('a further rebuild speaks up again even after the earlier notice was dismissed', async () => { - const mounted = mountNotice({engineStale: true}) + const pane = await mountNotice(staleness(true, 'stamp-boot')) await page.getByRole('button', {name: 'Dismiss'}).click() await expect.element(page.getByRole('alert')).not.toBeInTheDocument() - core?.setEngine({stale: true, fingerprint: 'stamp-rebuilt-again'}) - await mounted.refetch() + await coreControl.setStaleness(staleness(true, 'stamp-rebuilt-again')) + await pane.refetch() await expect.element(page.getByRole('alert')).toBeVisible() }) diff --git a/apps/conciv/test/helpers/core-control.ts b/apps/conciv/test/helpers/core-control.ts index eb2274ea2..6792428a2 100644 --- a/apps/conciv/test/helpers/core-control.ts +++ b/apps/conciv/test/helpers/core-control.ts @@ -12,6 +12,9 @@ declare module 'vitest/internal/browser' { releaseTurn: () => Promise scriptError: (message: string) => Promise scriptTurn: (turn: ScriptedTurn) => Promise + setTerminalLaunch: (succeeds: boolean) => Promise + terminalLaunches: () => Promise + rpcCallCount: (path: string[]) => Promise installFault: (spec: FaultSpec) => Promise releaseFault: (handle: string) => Promise faultPending: (handle: string) => Promise diff --git a/apps/conciv/test/helpers/core-session.ts b/apps/conciv/test/helpers/core-session.ts new file mode 100644 index 000000000..4df9d492b --- /dev/null +++ b/apps/conciv/test/helpers/core-session.ts @@ -0,0 +1,58 @@ +import {EventType} from '@tanstack/ai' +import {makeRpcClient, type RpcClient} from '@conciv/contract' + +export type SeededDraft = {text?: string; grabs?: string[]} + +export type TranscriptStream = {awaitTurnEnd: () => Promise; close: () => void} + +export function coreRpc(base: string): RpcClient { + return makeRpcClient(base) +} + +export async function createSession(rpc: RpcClient): Promise { + const {sessionId} = await rpc.sessions.create() + return sessionId +} + +export async function seedDraft(rpc: RpcClient, sessionId: string, draft: SeededDraft): Promise { + const text = draft.text ?? '' + await rpc.drafts.set({ + sessionId, + text, + selectionStart: text.length, + selectionEnd: text.length, + grabs: draft.grabs ?? [], + }) +} + +function isFinal(chunk: unknown): boolean { + if (typeof chunk !== 'object' || chunk === null || !('type' in chunk)) return false + return chunk.type === EventType.RUN_FINISHED || chunk.type === EventType.RUN_ERROR +} + +export async function openTranscriptStream(rpc: RpcClient, sessionId: string): Promise { + const controller = new AbortController() + const iterator = await rpc.chat.subscribe({sessionId}, {signal: controller.signal}) + return { + awaitTurnEnd: async () => { + for await (const chunk of iterator) { + if (isFinal(chunk)) return + } + }, + close: () => controller.abort(), + } +} + +export async function sendTurn(rpc: RpcClient, sessionId: string, text: string): Promise { + await rpc.chat.send({sessionId, runId: crypto.randomUUID(), text}) +} + +export async function runTurn(rpc: RpcClient, sessionId: string, text: string): Promise { + const stream = await openTranscriptStream(rpc, sessionId) + try { + await sendTurn(rpc, sessionId, text) + await stream.awaitTurnEnd() + } finally { + stream.close() + } +} diff --git a/apps/conciv/test/helpers/pane-harness.tsx b/apps/conciv/test/helpers/pane-harness.tsx index 8fc2b3d89..bc8ffd8a1 100644 --- a/apps/conciv/test/helpers/pane-harness.tsx +++ b/apps/conciv/test/helpers/pane-harness.tsx @@ -1,7 +1,8 @@ import {createRoot, createSignal, type JSX} from 'solid-js' import {render} from '@solidjs/testing-library' import {QueryClient, QueryClientProvider} from '@tanstack/solid-query' -import {makeRpcClient} from '@conciv/contract' +import {browserRpcConnection, closeBrowserRpcConnection, makeRpcClient} from '@conciv/contract' +import '../../src/lib/api-base.js' import {HostApiProvider} from '@conciv/extension/host' import {AppContext, type AppContextValue} from '../../src/app/context.js' import {EngineReachabilityContext, makeEngineReachability} from '../../src/app/reachability.js' @@ -11,18 +12,19 @@ import { makePendingAttachmentQueue, type PaneContextValue, } from '../../src/app/pane-context.js' -import {makeAppData} from '../../src/data/app-data.js' +import {makeAppData, type AppData} from '../../src/data/app-data.js' import {parseConcivSettings} from '../../src/data/settings.js' import {makeLayerStack} from '../../src/shell/dialogs.js' import {NoticeContextProvider, NoticeSurface} from '../../src/shell/notice-context.js' -import {CORE_BASE} from './fake-core.js' -export const PANE_SESSION = 'conciv_1' +export type PaneMountOptions = {base: string; sessionId: string} export type PaneMount = { dispose: () => void announced: () => string[] pane: PaneContextValue + data: AppData + queryClient: QueryClient refetch: () => Promise } @@ -34,8 +36,10 @@ function AnnounceLog(props: {entries: () => string[]}): JSX.Element { ) } -export function mountPane(view: (pane: PaneContextValue) => JSX.Element): PaneMount { - const rpc = makeRpcClient(CORE_BASE) +export function mountPane(options: PaneMountOptions, view: (pane: PaneContextValue) => JSX.Element): PaneMount { + const rpc = makeRpcClient(options.base) + window.__CONCIV_API_BASE__ = options.base + browserRpcConnection(options.base, 'fetch') const queryClient = new QueryClient() const data = makeAppData(rpc, queryClient) const [announced, setAnnounced] = createSignal([]) @@ -55,10 +59,10 @@ export function mountPane(view: (pane: PaneContextValue) => JSX.Element): PaneMo connectBind: async () => '', connectMode: false, connectionGeneration: () => 0, - apiBase: () => CORE_BASE, + apiBase: () => options.base, } const pane: PaneContextValue = { - sessionId: () => PANE_SESSION, + sessionId: () => options.sessionId, running: () => false, viewLocked: () => false, setLockedFor: () => () => {}, @@ -96,9 +100,13 @@ export function mountPane(view: (pane: PaneContextValue) => JSX.Element): PaneMo dispose: () => { mounted.unmount() reachabilityRoot.dispose() + closeBrowserRpcConnection(options.base) + delete window.__CONCIV_API_BASE__ }, announced, pane, + data, + queryClient, refetch: () => queryClient.invalidateQueries({queryKey: data.utils.meta.engine.key()}), } } diff --git a/apps/conciv/test/launch-actions.browser.test.tsx b/apps/conciv/test/launch-actions.browser.test.tsx index 8b4c4dfda..80d852a7f 100644 --- a/apps/conciv/test/launch-actions.browser.test.tsx +++ b/apps/conciv/test/launch-actions.browser.test.tsx @@ -1,17 +1,17 @@ -import {afterEach, expect, test} from 'vitest' +import {afterAll, afterEach, beforeAll, expect, test} from 'vitest' import {page} from 'vitest/browser' import type {GrabApi} from '@conciv/grab' import {HostApiProvider} from '@conciv/extension/host' import {ComposerActions} from '../src/composer/actions.js' -import {installFakeCore, type FakeCore} from './helpers/fake-core.js' -import {mountPane, PANE_SESSION} from './helpers/pane-harness.js' +import {coreControl} from './helpers/core-control.js' +import {coreRpc, createSession} from './helpers/core-session.js' +import {mountPane, type PaneMount} from './helpers/pane-harness.js' -let core: FakeCore | null = null +const CONNECT_COMMAND_PATH = ['ext', 'terminal', 'connectCommand'] +const LAUNCH_PATH = ['ext', 'terminal', 'launch'] -afterEach(() => { - core?.restore() - core = null -}) +const core = {base: ''} +const mounted: {pane: PaneMount | null} = {pane: null} const grabApi: GrabApi = { pick: async () => null, @@ -23,13 +23,34 @@ const grabApi: GrabApi = { clear: () => {}, } -function mountActions(config: Parameters[0] = {}): FakeCore { - const fake = installFakeCore(config) - core = fake - mountPane(() => ( +beforeAll(async () => { + const booted = await coreControl.bootCore({ + id: 'launch-actions', + displayName: 'Claude', + connect: true, + terminal: true, + resume: true, + allowedOrigins: [window.location.origin], + }) + core.base = booted.base +}, 60_000) + +afterAll(async () => { + await coreControl.closeCore() +}, 30_000) + +afterEach(async () => { + mounted.pane?.dispose() + mounted.pane = null + await coreControl.setTerminalLaunch(true) +}) + +async function mountActions(): Promise { + const sessionId = await createSession(coreRpc(core.base)) + mounted.pane = mountPane({base: core.base, sessionId}, () => ( {}}> {}} onNewSession={() => {}} @@ -37,7 +58,6 @@ function mountActions(config: Parameters[0] = {}): FakeC /> )) - return fake } async function openMenu(): Promise { @@ -45,7 +65,7 @@ async function openMenu(): Promise { } test('the terminal menu offers launch and copy, and the connect candidates are gone', async () => { - mountActions() + await mountActions() await openMenu() @@ -55,44 +75,47 @@ test('the terminal menu offers launch and copy, and the connect candidates are g }) test('opening externally launches through the terminal extension', async () => { - const fake = mountActions({launchOk: true}) + await mountActions() await openMenu() await page.getByText('Open in Claude').click() await expect.element(page.getByText('Opened in Claude.')).toBeVisible() - expect(fake.calls.filter((call) => call.path === '/rpc/ext/terminal/launch')).toHaveLength(1) + expect(await coreControl.terminalLaunches()).toBe(1) + expect(await coreControl.rpcCallCount(LAUNCH_PATH)).toBe(1) }) test('when the terminal cannot open, the connect command is offered instead', async () => { - const fake = mountActions({launchOk: false}) + await coreControl.setTerminalLaunch(false) + await mountActions() + const before = await coreControl.rpcCallCount(CONNECT_COMMAND_PATH) await openMenu() await page.getByText('Open in Claude').click() - await expect - .element(page.getByText(/Command copied|Run in your terminal: claude --resume fake-session/)) - .toBeVisible() - expect(fake.calls.filter((call) => call.path === '/rpc/ext/terminal/connectCommand')).toHaveLength(1) + await expect.element(page.getByText(/Command copied|Run in your terminal: cd .*'claude' '--resume'/)).toBeVisible() + expect((await coreControl.rpcCallCount(CONNECT_COMMAND_PATH)) - before).toBe(1) }) test('copy command asks the terminal extension for the command', async () => { - const fake = mountActions() + await mountActions() + const before = await coreControl.rpcCallCount(CONNECT_COMMAND_PATH) await openMenu() await page.getByText('Copy command').click() - await expect - .element(page.getByText(/Command copied|Run in your terminal: claude --resume fake-session/)) - .toBeVisible() - expect(fake.calls.filter((call) => call.path === '/rpc/ext/terminal/connectCommand')).toHaveLength(1) + await expect.element(page.getByText(/Command copied|Run in your terminal: cd .*'claude' '--resume'/)).toBeVisible() + expect((await coreControl.rpcCallCount(CONNECT_COMMAND_PATH)) - before).toBe(1) }) test('a launch failure is surfaced instead of swallowed', async () => { - mountActions({launchRejects: true}) + const fault = await coreControl.installFault({kind: 'fail', path: LAUNCH_PATH, status: 500}) + await mountActions() await openMenu() await page.getByText('Open in Claude').click() await expect.element(page.getByText('Couldn’t open Claude.')).toBeVisible() + + await coreControl.releaseFault(fault) }) diff --git a/apps/conciv/test/page-session-card.browser.test.tsx b/apps/conciv/test/page-session-card.browser.test.tsx index 6bbec862d..701734099 100644 --- a/apps/conciv/test/page-session-card.browser.test.tsx +++ b/apps/conciv/test/page-session-card.browser.test.tsx @@ -1,46 +1,45 @@ import './helpers/utilities.css' -import {afterEach, expect, test} from 'vitest' +import {afterAll, afterEach, beforeAll, expect, test} from 'vitest' import {page} from 'vitest/browser' import {ChatPane} from '../src/pane/chat-pane.js' -import {installFakeCore, sessionRow, type FakeCore} from './helpers/fake-core.js' -import {mountPane, PANE_SESSION} from './helpers/pane-harness.js' +import {coreControl} from './helpers/core-control.js' +import {coreRpc, createSession, runTurn} from './helpers/core-session.js' +import {mountPane, type PaneMount} from './helpers/pane-harness.js' -let core: FakeCore | null = null +const core = {base: ''} +const mounted: {pane: PaneMount | null} = {pane: null} + +beforeAll(async () => { + const booted = await coreControl.bootCore({id: 'page-session-card', allowedOrigins: [window.location.origin]}) + core.base = booted.base +}, 60_000) + +afterAll(async () => { + await coreControl.closeCore() +}, 30_000) afterEach(() => { - core?.restore() - core = null + mounted.pane?.dispose() + mounted.pane = null }) -const HISTORY_TRANSCRIPT = [ - { - id: 'a1', - role: 'assistant', - parts: [ - { - type: 'tool-call', - id: 'f1', - name: 'page.fill', - arguments: '{"selector":"#name","value":"Ada"}', - state: 'complete', - }, - {type: 'tool-result', toolCallId: 'f1', content: '{"ok":true,"value":"Ada"}', state: 'complete'}, +test('a reloaded transcript of page acts renders one aggregated session card with the reply', async () => { + const rpc = coreRpc(core.base) + const sessionId = await createSession(rpc) + await coreControl.scriptTurn({ + toolCalls: [ + {name: 'page.fill', input: {selector: '#name', value: 'Ada'}, result: {ok: true, value: 'Ada'}}, { - type: 'tool-call', - id: 'f2', name: 'page.fill', - arguments: '{"selector":"#email","value":"ada@example.com"}', - state: 'complete', + input: {selector: '#email', value: 'ada@example.com'}, + result: {ok: true, value: 'ada@example.com'}, }, - {type: 'tool-result', toolCallId: 'f2', content: '{"ok":true,"value":"ada@example.com"}', state: 'complete'}, - {type: 'text', content: 'The profile form is filled in.'}, ], - }, -] + text: 'The profile form is filled in.', + }) + await runTurn(rpc, sessionId, 'fill in the profile form') -test('a reloaded transcript of page acts renders one aggregated session card with the reply', async () => { - core = installFakeCore({sessions: [sessionRow({id: PANE_SESSION})], snapshotFor: () => HISTORY_TRANSCRIPT}) - mountPane(() => ) + mounted.pane = mountPane({base: core.base, sessionId}, () => ) await expect.element(page.getByText('Edited the page'), {timeout: 5000}).toBeVisible() await expect.element(page.getByText('The profile form is filled in.')).toBeVisible() diff --git a/apps/conciv/test/session-selector.browser.test.tsx b/apps/conciv/test/session-selector.browser.test.tsx index 7f27410f6..675ea4d0b 100644 --- a/apps/conciv/test/session-selector.browser.test.tsx +++ b/apps/conciv/test/session-selector.browser.test.tsx @@ -1,30 +1,58 @@ -import {afterEach, expect, test} from 'vitest' +import {afterAll, afterEach, beforeAll, expect, test} from 'vitest' import {page} from 'vitest/browser' +import type {RpcClient} from '@conciv/contract' import {SessionSelector} from '../src/composer/session-selector.js' -import {installFakeCore, sessionRow, type FakeCore} from './helpers/fake-core.js' -import {mountPane} from './helpers/pane-harness.js' +import {coreControl} from './helpers/core-control.js' +import {coreRpc} from './helpers/core-session.js' +import {mountPane, type PaneMount} from './helpers/pane-harness.js' -let core: FakeCore | null = null +const EXTERNAL_ID = 'external-mid-run' -afterEach(() => { - core?.restore() - core = null -}) +const core = {base: ''} +const mounted: {pane: PaneMount | null} = {pane: null} -function mountSelector(): void { - core = installFakeCore({ - sessions: [ - sessionRow({id: 'conciv_1', title: 'the active session'}), - sessionRow({id: 'conciv_2', title: 'a session mid-run', running: true, origin: 'external'}), - ], +beforeAll(async () => { + const booted = await coreControl.bootCore({ + id: 'session-selector', + resume: true, + allowedOrigins: [window.location.origin], + history: [{id: EXTERNAL_ID, derivedTitle: 'a session mid-run', updatedAt: Date.now(), messageCount: 3}], }) - mountPane(() => ( - 'conciv_1'} onActivate={() => {}} onNewSession={() => {}} /> + core.base = booted.base +}, 60_000) + +afterAll(async () => { + await coreControl.closeCore() +}, 30_000) + +afterEach(async () => { + mounted.pane?.dispose() + mounted.pane = null + await coreControl.releaseTurn() +}) + +async function adoptExternalSession(rpc: RpcClient): Promise { + const listed = await rpc.sessions.list() + const external = listed.find((meta) => meta.native?.nativeId === EXTERNAL_ID) + if (!external?.native) throw new Error('the harness history fixture never reached the session list') + const {sessionId} = await rpc.sessions.open(external.native) + return sessionId +} + +async function mountSelector(): Promise { + const rpc = coreRpc(core.base) + const activeId = (await rpc.sessions.create()).sessionId + await rpc.sessions.rename({sessionId: activeId, title: 'the active session'}) + const externalId = await adoptExternalSession(rpc) + await coreControl.holdTurn() + await rpc.chat.send({sessionId: externalId, runId: crypto.randomUUID(), text: 'keep this session busy'}) + mounted.pane = mountPane({base: core.base, sessionId: activeId}, () => ( + activeId} onActivate={() => {}} onNewSession={() => {}} /> )) } test('a session with a live core run carries the running dot, an idle one does not', async () => { - mountSelector() + await mountSelector() await page.getByRole('button', {name: 'Session: the active session'}).click() @@ -33,7 +61,7 @@ test('a session with a live core run carries the running dot, an idle one does n }) test('the selector keeps freshness, message count and origin in the row description', async () => { - mountSelector() + await mountSelector() await page.getByRole('button', {name: 'Session: the active session'}).click() diff --git a/packages/extension-testkit/src/core-kit.ts b/packages/extension-testkit/src/core-kit.ts index 921e53100..619529d26 100644 --- a/packages/extension-testkit/src/core-kit.ts +++ b/packages/extension-testkit/src/core-kit.ts @@ -2,7 +2,7 @@ import {createFakeHarness, createTestkit, type FakeHarness, type Kit} from '@con import {makeApp} from '@conciv/core/app' import type {AnyExtension} from '@conciv/extension' import type {ToolRegistry} from '@conciv/extension/registry' -import type {HarnessCommand, HarnessModel, HarnessSessionMeta} from '@conciv/protocol/harness-types' +import type {HarnessCommand, HarnessConnect, HarnessModel, HarnessSessionMeta} from '@conciv/protocol/harness-types' import type {EngineStaleness} from '@conciv/contract' export type CoreKit = Kit & {harness: FakeHarness; registry: ToolRegistry} @@ -10,6 +10,9 @@ export type CoreKit = Kit & {harness: FakeHarness; registry: ToolRegistry} export async function bootCoreKit(opts: { id: string text?: string + resume?: boolean + displayName?: string + connect?: HarnessConnect extensions?: AnyExtension[] models?: HarnessModel[] commands?: HarnessCommand[] @@ -21,6 +24,9 @@ export async function bootCoreKit(opts: { const harness = createFakeHarness({ id: opts.id, text: opts.text ?? 'Hello from conciv', + ...(opts.resume ? {resume: true} : {}), + ...(opts.displayName ? {displayName: opts.displayName} : {}), + ...(opts.connect ? {connect: opts.connect} : {}), models: opts.models, commands: opts.commands, history: opts.history, diff --git a/packages/extension-testkit/src/rpc-fault.ts b/packages/extension-testkit/src/rpc-fault.ts index da5a49d75..738f7f26c 100644 --- a/packages/extension-testkit/src/rpc-fault.ts +++ b/packages/extension-testkit/src/rpc-fault.ts @@ -71,6 +71,27 @@ export async function failRpcCalls( } } +export async function abortRpcCalls(page: Page, options: {path: readonly string[]}): Promise { + const broken = {value: true} + const matcher = pathMatcher(options.path) + const handler: RouteHandler = async (route) => { + if (isPreflight(route) || !broken.value) return route.continue() + await route.abort('connectionrefused') + } + + await page.route(matcher, handler) + + return { + repair: () => { + broken.value = false + }, + dispose: async () => { + broken.value = false + await page.unroute(matcher, handler) + }, + } +} + export type RpcHold = {hold: () => void; release: () => void} export async function holdRpcCalls(page: Page): Promise { diff --git a/packages/extensions/terminal/src/server.ts b/packages/extensions/terminal/src/server.ts index dcd35e5f0..3d90d971e 100644 --- a/packages/extensions/terminal/src/server.ts +++ b/packages/extensions/terminal/src/server.ts @@ -5,7 +5,7 @@ import {os} from '@orpc/server' import {z} from 'zod' import {defineExtension, type ServerApi} from '@conciv/extension' import {SessionId} from '@conciv/protocol/chat-types' -import type {HarnessConnectContext, HarnessConnectPlan} from '@conciv/protocol/harness-types' +import type {HarnessConnectContext, HarnessConnectPlan, TerminalOpener} from '@conciv/protocol/harness-types' import {TtyClientControlSchema, type TtyClientControl} from '@conciv/protocol/terminal-types' import type {RpcContext} from '@conciv/protocol/rpc-types' import {createTtySessions, type TtySession, type TtySink} from './server/pty-sessions.js' @@ -23,6 +23,7 @@ const ESCAPE_KEY = String.fromCharCode(27) type TerminalRuntime = { server: ServerApi> tty: ReturnType + openTerminal?: TerminalOpener } type TerminalEnv = {Variables: {terminal: TerminalRuntime}} @@ -166,7 +167,13 @@ function makeTerminalRouter(runtime: TerminalRuntime) { const plan = await connectPlanFor(runtime, input.sessionId, input.model ?? null, apiBase(runtime, context)) if (!plan) throw errors.NO_CONNECT() const {server} = runtime - return {ok: await launchConnectPlan(plan, {cwd: server.cwd, stateDir: server.stateDir})} + return { + ok: await launchConnectPlan(plan, { + cwd: server.cwd, + stateDir: server.stateDir, + ...(runtime.openTerminal ? {openTerminal: runtime.openTerminal} : {}), + }), + } }), connectCommand: terminalOs .errors(noConnect) @@ -221,21 +228,25 @@ const app = new Hono().get( export type TerminalAppType = typeof app -export default defineExtension({name: TERMINAL_NAME}).server((server) => { - const tty = createTtySessions() - const runtime: TerminalRuntime = {server, tty} - return { - context: {}, - router: makeTerminalRouter(runtime), - app: new Hono() - .use(async (c, next) => { - c.set('terminal', runtime) - await next() - }) - .route('/', app), - dispose: () => tty.shutdown(), - } -}) +export function createTerminalExtension(opts: {openTerminal?: TerminalOpener} = {}) { + return defineExtension({name: TERMINAL_NAME}).server((server) => { + const tty = createTtySessions() + const runtime: TerminalRuntime = {server, tty, ...(opts.openTerminal ? {openTerminal: opts.openTerminal} : {})} + return { + context: {}, + router: makeTerminalRouter(runtime), + app: new Hono() + .use(async (c, next) => { + c.set('terminal', runtime) + await next() + }) + .route('/', app), + dispose: () => tty.shutdown(), + } + }) +} + +export default createTerminalExtension() function parseControl(text: string): TtyClientControl | null { if (!text.startsWith('{')) return null diff --git a/packages/harness-testkit/src/create-fake-harness.ts b/packages/harness-testkit/src/create-fake-harness.ts index d65529f78..15b707981 100644 --- a/packages/harness-testkit/src/create-fake-harness.ts +++ b/packages/harness-testkit/src/create-fake-harness.ts @@ -3,6 +3,7 @@ import { defineHarness, type HarnessAdapter, type HarnessCommand, + type HarnessConnect, type HarnessConnectContext, type HarnessHistory, type HarnessModel, @@ -28,6 +29,8 @@ const BASE_CAPABILITIES = { type SharedFields = { id: string binName: string + displayName?: string + connect?: HarnessConnect chatConfig: HarnessAdapter['chatConfig'] models: HarnessModel[] | undefined tty: {command(ctx: HarnessConnectContext): TtyCommand} | undefined @@ -48,6 +51,7 @@ function fixtureHistory(rows: HarnessSessionMeta[]): HarnessHistory { function buildAdapter( shared: SharedFields, + base: Omit & {resume: boolean}, history: HarnessHistory | undefined, commands: HarnessCommand[] | undefined, ): HarnessAdapter { @@ -55,7 +59,7 @@ function buildAdapter( if (history && listCommands) { return defineHarness({ ...shared, - capabilities: {...BASE_CAPABILITIES, transcriptHistory: true, slashCommands: 'live'}, + capabilities: {...base, transcriptHistory: true, slashCommands: 'live'}, history, commands: listCommands, }) @@ -63,20 +67,20 @@ function buildAdapter( if (history) { return defineHarness({ ...shared, - capabilities: {...BASE_CAPABILITIES, transcriptHistory: true, slashCommands: 'none'}, + capabilities: {...base, transcriptHistory: true, slashCommands: 'none'}, history, }) } if (listCommands) { return defineHarness({ ...shared, - capabilities: {...BASE_CAPABILITIES, transcriptHistory: false, slashCommands: 'live'}, + capabilities: {...base, transcriptHistory: false, slashCommands: 'live'}, commands: listCommands, }) } return defineHarness({ ...shared, - capabilities: {...BASE_CAPABILITIES, transcriptHistory: false, slashCommands: 'none'}, + capabilities: {...base, transcriptHistory: false, slashCommands: 'none'}, }) } @@ -84,6 +88,9 @@ export function createFakeHarness( opts: { id?: string text?: string + resume?: boolean + displayName?: string + connect?: HarnessConnect models?: HarnessModel[] commands?: HarnessCommand[] history?: HarnessSessionMeta[] @@ -95,10 +102,18 @@ export function createFakeHarness( const shared: SharedFields = { id, binName: 'true', + ...(opts.displayName ? {displayName: opts.displayName} : {}), + ...(opts.connect ? {connect: opts.connect} : {}), chatConfig: (deps) => ({adapter: makeTextAdapter(id, () => scripted.chatStream(deps))}), models: opts.models, tty: opts.tty, } - const adapter = buildAdapter(shared, opts.history ? fixtureHistory(opts.history) : undefined, opts.commands) + const capabilities = {...BASE_CAPABILITIES, resume: opts.resume ?? false} + const adapter = buildAdapter( + shared, + capabilities, + opts.history ? fixtureHistory(opts.history) : undefined, + opts.commands, + ) return Object.assign(adapter, {script: scripted}) } diff --git a/packages/harness-testkit/src/scripted-run.ts b/packages/harness-testkit/src/scripted-run.ts index d2760a31b..811d11ae8 100644 --- a/packages/harness-testkit/src/scripted-run.ts +++ b/packages/harness-testkit/src/scripted-run.ts @@ -45,6 +45,11 @@ function* turnChunks(turn: QueuedTurn): Generator { } } +function sessionIdChunk(deps: HarnessChatDeps): StreamChunk { + const sessionId = deps.resumeSessionId ?? `fake-${deps.sessionId}` + return {type: EventType.CUSTOM, name: 'fake.session-id', value: {sessionId}, ...THREAD} +} + function* customEventChunks(events: {name: string; value: unknown}[]): Generator { for (const event of events) yield {type: EventType.CUSTOM, name: event.name, value: event.value, ...THREAD} } @@ -91,7 +96,7 @@ export function makeScriptedRun(opts: {text?: string} = {}): ScriptedRun { turns.count += 1 const messageId = `scripted-${turns.count}` yield {type: EventType.RUN_STARTED, ...THREAD} - yield {type: EventType.CUSTOM, name: 'fake.session-id', value: {sessionId: `fake-${deps.sessionId}`}, ...THREAD} + yield sessionIdChunk(deps) const scriptedTurn = queuedTurns.shift() if (scriptedTurn) yield* turnChunks(scriptedTurn) const toolCall = scriptedTurn ? undefined : queuedToolCalls.shift() From 05965a3680057b9a4a3aca0cb7ba7b5f50a95554 Mon Sep 17 00:00:00 2001 From: Omri Katz <9701896+omridevk@users.noreply.github.com> Date: Sat, 15 Aug 2026 05:57:55 +0300 Subject: [PATCH 3/9] =?UTF-8?q?test(conciv):=20#354=20phase=203=20?= =?UTF-8?q?=E2=80=94=20shell=20and=20timing=20suites=20run=20against=20the?= =?UTF-8?q?=20real=20core?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drops installFakeCore from shell-harness and the six suites that used it: per-route delays become releasable playwright gates, network failure becomes a transport abort fault, and resolve/engine rejections become injected 500s. Adds rpcMark/awaitRpcCall so a wire fact can be awaited instead of polled, and lets an abort fault cover every rpc path. Co-Authored-By: Claude Fable 5 --- apps/conciv/test/chat-pane.browser.test.tsx | 14 +-- apps/conciv/test/commands/core-control.ts | 15 +++- apps/conciv/test/helpers/core-control.ts | 2 + apps/conciv/test/helpers/retry-recovery.ts | 4 +- apps/conciv/test/helpers/shell-harness.tsx | 35 ++++---- .../test/panel-connect.browser.test.tsx | 37 +++++--- .../panel-focus-stability.browser.test.tsx | 88 +++++++++---------- .../test/panel-min-height.browser.test.tsx | 72 +++++++-------- .../test/quick-add-pane.browser.test.tsx | 55 +++++++----- .../test/reachability-flows.browser.test.tsx | 83 +++++++++++------ .../test/route-boundary.browser.test.tsx | 83 ++++++++--------- packages/extension-testkit/src/rpc-fault.ts | 2 +- 12 files changed, 273 insertions(+), 217 deletions(-) diff --git a/apps/conciv/test/chat-pane.browser.test.tsx b/apps/conciv/test/chat-pane.browser.test.tsx index c8ac47c4b..bb7c14d62 100644 --- a/apps/conciv/test/chat-pane.browser.test.tsx +++ b/apps/conciv/test/chat-pane.browser.test.tsx @@ -107,7 +107,7 @@ test('a send the server refuses puts the staged grab card back', async () => { await expect.element(notifications()).toHaveTextContent(/Internal Server Error|could not be sent/) await expect.element(removeGrab()).toBeVisible() - await expect.element(page.getByText('the grabbed hero section')).toBeVisible() + await expect.element(page.getByText('the grabbed hero section', {exact: true})).toBeVisible() }) test('a send that throws at the transport puts the staged grab card back', async () => { @@ -116,7 +116,7 @@ test('a send that throws at the transport puts the staged grab card back', async await expect.element(notifications()).toHaveTextContent(/could not be sent|fetch/) await expect.element(removeGrab()).toBeVisible() - await expect.element(page.getByText('the grabbed hero section')).toBeVisible() + await expect.element(page.getByText('the grabbed hero section', {exact: true})).toBeVisible() }) test('a queued second send cannot cross-restore the grabs of the turn that failed', async () => { @@ -125,23 +125,23 @@ test('a queued second send cannot cross-restore the grabs of the turn that faile await coreControl.holdTurn() const mount = mountChatPane(sessionId) - await expect.element(page.getByText('the grabbed hero section')).toBeVisible() + await expect.element(page.getByText('the grabbed hero section', {exact: true})).toBeVisible() await input().fill('turn A') await userEvent.keyboard('{Enter}') await expect.element(stopButton()).toBeVisible() mount.pane.grabStore.stageAll([{text: 'the grabbed pricing table'}]) - await expect.element(page.getByText('the grabbed pricing table')).toBeVisible() + await expect.element(page.getByText('the grabbed pricing table', {exact: true})).toBeVisible() await input().fill('turn B') await userEvent.keyboard('{Enter}') await expect.element(page.getByRole('button', {name: 'Remove from queue'})).toBeVisible() - await expect.element(page.getByText('the grabbed pricing table')).not.toBeInTheDocument() + await expect.element(page.getByText('the grabbed pricing table', {exact: true})).not.toBeInTheDocument() await coreControl.scriptError('turn A failed') await coreControl.releaseTurn() - await expect.element(page.getByText('the grabbed hero section')).toBeVisible() - await expect.element(page.getByText('the grabbed pricing table')).not.toBeInTheDocument() + await expect.element(page.getByText('the grabbed hero section', {exact: true})).toBeVisible() + await expect.element(page.getByText('the grabbed pricing table', {exact: true})).not.toBeInTheDocument() }) test('sending announces thinking and then the reply through the live region', async () => { diff --git a/apps/conciv/test/commands/core-control.ts b/apps/conciv/test/commands/core-control.ts index 5225da034..94aad44fd 100644 --- a/apps/conciv/test/commands/core-control.ts +++ b/apps/conciv/test/commands/core-control.ts @@ -21,7 +21,7 @@ export type BootCoreResult = {base: string; wsBase: string; bootMs: number} export type FaultSpec = | {kind: 'fail'; path: string[]; status?: number} - | {kind: 'abort'; path: string[]} + | {kind: 'abort'; path?: string[]} | {kind: 'gate'; path?: string[]} type Fault = {pending: () => number; release: () => Promise; dispose: () => Promise} @@ -47,6 +47,8 @@ const FRESH: EngineStaleness = { fingerprint: 'testkit0000', } +const AWAIT_RPC_TIMEOUT_MS = 15_000 + const files = new Map() function testkitOf(ctx: BrowserCommandContext): Promise { @@ -165,6 +167,13 @@ const terminalLaunches: BrowserCommand<[]> = (ctx): number => stateOf(ctx).termi const rpcCallCount: BrowserCommand<[string[]]> = (ctx, path): number => observerOf(ctx).completedCount({path}) +const rpcMark: BrowserCommand<[]> = (ctx): number => observerOf(ctx).mark() + +const awaitRpcCall: BrowserCommand<[string[], number]> = async (ctx, path, since): Promise => { + const record = await observerOf(ctx).completed({path, since, timeout: AWAIT_RPC_TIMEOUT_MS}) + return record.status +} + const installFault: BrowserCommand<[FaultSpec]> = async (ctx, spec): Promise => { const state = stateOf(ctx) const {abortRpcCalls, failRpcCalls, gateRpcCalls} = await testkitOf(ctx) @@ -176,7 +185,7 @@ const installFault: BrowserCommand<[FaultSpec]> = async (ctx, spec): Promise 0, @@ -206,6 +215,8 @@ export const coreCommands = { setTerminalLaunch, terminalLaunches, rpcCallCount, + rpcMark, + awaitRpcCall, installFault, releaseFault, faultPending, diff --git a/apps/conciv/test/helpers/core-control.ts b/apps/conciv/test/helpers/core-control.ts index 6792428a2..d91c56a6a 100644 --- a/apps/conciv/test/helpers/core-control.ts +++ b/apps/conciv/test/helpers/core-control.ts @@ -15,6 +15,8 @@ declare module 'vitest/internal/browser' { setTerminalLaunch: (succeeds: boolean) => Promise terminalLaunches: () => Promise rpcCallCount: (path: string[]) => Promise + rpcMark: () => Promise + awaitRpcCall: (path: string[], since: number) => Promise installFault: (spec: FaultSpec) => Promise releaseFault: (handle: string) => Promise faultPending: (handle: string) => Promise diff --git a/apps/conciv/test/helpers/retry-recovery.ts b/apps/conciv/test/helpers/retry-recovery.ts index c9a0b9ec7..5500dd9b0 100644 --- a/apps/conciv/test/helpers/retry-recovery.ts +++ b/apps/conciv/test/helpers/retry-recovery.ts @@ -4,11 +4,11 @@ import {page} from 'vitest/browser' type Locator = ReturnType export async function expectRetryRecovers( - clearFailure: () => void, + clearFailure: () => Promise, editor: () => Locator, failureLocator: () => Locator, ): Promise { - clearFailure() + await clearFailure() await page.getByRole('button', {name: 'Retry'}).click() await expect.element(editor(), {timeout: 8000}).toBeVisible() await expect.element(failureLocator()).not.toBeInTheDocument() diff --git a/apps/conciv/test/helpers/shell-harness.tsx b/apps/conciv/test/helpers/shell-harness.tsx index b65a57347..9b158f283 100644 --- a/apps/conciv/test/helpers/shell-harness.tsx +++ b/apps/conciv/test/helpers/shell-harness.tsx @@ -1,41 +1,42 @@ +import {onlineManager} from '@tanstack/query-core' import {render} from '@solidjs/testing-library' import {RouterProvider, createMemoryHistory} from '@tanstack/solid-router' -import {makeBrowserRpcClient} from '@conciv/contract' +import {closeBrowserRpcConnection, makeBrowserRpcClient} from '@conciv/contract' import type {AnyExtension} from '@conciv/extension' +import '../../src/lib/api-base.js' import {parseConcivSettings} from '../../src/data/settings.js' import {createConcivRouter, disposeConcivRouter} from '../../src/router.js' -import {CORE_BASE, installFakeCore, sessionRow, type FakeCore, type FakeCoreConfig} from './fake-core.js' export type ShellHarness = { - mountShell: (entry: string, config?: FakeCoreConfig, extensions?: AnyExtension[]) => void - core: () => FakeCore | null + mountShell: (entry: string, extensions?: AnyExtension[]) => void dispose: () => void } -export function createShellHarness(sessionId: string): ShellHarness { - let core: FakeCore | null = null - let mountedRouter: ReturnType | null = null +export function createShellHarness(base: () => string): ShellHarness { + const mounted: {router: ReturnType | null} = {router: null} - const mountShell = (entry: string, config: FakeCoreConfig = {}, extensions: AnyExtension[] = []): void => { - core = installFakeCore({sessions: [sessionRow({id: sessionId})], ...config}) + const mountShell = (entry: string, extensions: AnyExtension[] = []): void => { + const apiBase = base() + window.__CONCIV_API_BASE__ = apiBase const router = createConcivRouter({ - rpc: makeBrowserRpcClient(CORE_BASE, {transport: 'fetch'}).rpc, + rpc: makeBrowserRpcClient(apiBase, {transport: 'fetch'}).rpc, history: createMemoryHistory({initialEntries: [entry]}), environment: {rootNode: document, document}, settings: parseConcivSettings(''), - apiBase: () => CORE_BASE, + apiBase: () => apiBase, extensions, }) - mountedRouter = router + mounted.router = router render(() => ) } const dispose = (): void => { - if (mountedRouter) disposeConcivRouter(mountedRouter) - mountedRouter = null - core?.restore() - core = null + if (mounted.router) disposeConcivRouter(mounted.router) + mounted.router = null + closeBrowserRpcConnection(base()) + onlineManager.setOnline(true) + delete window.__CONCIV_API_BASE__ } - return {mountShell, core: () => core, dispose} + return {mountShell, dispose} } diff --git a/apps/conciv/test/panel-connect.browser.test.tsx b/apps/conciv/test/panel-connect.browser.test.tsx index 6d576c52b..3b0b3776b 100644 --- a/apps/conciv/test/panel-connect.browser.test.tsx +++ b/apps/conciv/test/panel-connect.browser.test.tsx @@ -1,15 +1,27 @@ import './helpers/utilities.css' -import {afterEach, expect, test} from 'vitest' +import {afterAll, afterEach, beforeAll, expect, test} from 'vitest' import {page} from 'vitest/browser' import {defineExtension, getExtensionApi, type RegisterExtension} from '@conciv/extension' import {Show, type JSX} from 'solid-js' +import {coreControl} from './helpers/core-control.js' import {createShellHarness} from './helpers/shell-harness.js' import {expectRetryRecovers} from './helpers/retry-recovery.js' -const PANEL_SESSION = 'conciv_1' const FOUND_API_BASE = 'http://found.test' const CONNECT_PROBE_NAME = 'connect-probe' -const harness = createShellHarness(PANEL_SESSION) +const RESOLVE_PATH = ['sessions', 'resolve'] + +const core = {base: ''} +const harness = createShellHarness(() => core.base) + +beforeAll(async () => { + const booted = await coreControl.bootCore({id: 'panel-connect', allowedOrigins: [window.location.origin]}) + core.base = booted.base +}, 60_000) + +afterAll(async () => { + await coreControl.closeCore() +}, 30_000) afterEach(harness.dispose) @@ -35,8 +47,7 @@ declare module '@conciv/protocol/config-types' { interface ExtensionRegistry extends RegisterExtension {} } -const mountShell = (config: Parameters[1] = {}): void => - harness.mountShell('/panel/connect?open=true', config, [connectProbe]) +const mountShell = (): void => harness.mountShell('/panel/connect?open=true', [connectProbe]) const simulateFound = () => page.getByRole('button', {name: 'Simulate found'}) const bindFailure = () => page.getByText(/conciv couldn.t connect to that workspace/) @@ -44,18 +55,20 @@ const serverError = () => page.getByText('Internal Server Error') const editor = () => page.getByRole('textbox', {name: 'Message the conciv agent'}) test('a bind that fails with a transport failure shows the connect failure screen, and retrying against a healthy engine hands off to the panel', async () => { - mountShell({resolveTransportFails: true}) + const unreachable = await coreControl.installFault({kind: 'abort', path: RESOLVE_PATH}) + mountShell() await simulateFound().click() await expect.element(bindFailure(), {timeout: 8000}).toBeVisible() await expect.element(page.getByRole('button', {name: 'Retry'})).toBeVisible() - await expectRetryRecovers(() => harness.core()?.setResolveTransportFails(false), editor, bindFailure) -}) + await expectRetryRecovers(() => coreControl.releaseFault(unreachable), editor, bindFailure) +}, 30_000) test('a bind that fails with a server error shows the actual error, not the workspace-unreachable message', async () => { - mountShell({resolveRejects: true}) + const refused = await coreControl.installFault({kind: 'fail', path: RESOLVE_PATH, status: 500}) + mountShell() await expect.element(simulateFound(), {timeout: 8000}).toBeVisible() await simulateFound().click() @@ -63,8 +76,8 @@ test('a bind that fails with a server error shows the actual error, not the work await expect.element(serverError(), {timeout: 8000}).toBeVisible() await expect.element(bindFailure()).not.toBeInTheDocument() - await expectRetryRecovers(() => harness.core()?.setResolveRejects(false), editor, serverError) -}) + await expectRetryRecovers(() => coreControl.releaseFault(refused), editor, serverError) +}, 30_000) test('a bind that succeeds on the first try never shows the failure screen', async () => { mountShell() @@ -73,4 +86,4 @@ test('a bind that succeeds on the first try never shows the failure screen', asy await expect.element(editor(), {timeout: 8000}).toBeVisible() await expect.element(bindFailure()).not.toBeInTheDocument() -}) +}, 30_000) diff --git a/apps/conciv/test/panel-focus-stability.browser.test.tsx b/apps/conciv/test/panel-focus-stability.browser.test.tsx index d579d7c76..334ea2676 100644 --- a/apps/conciv/test/panel-focus-stability.browser.test.tsx +++ b/apps/conciv/test/panel-focus-stability.browser.test.tsx @@ -1,61 +1,57 @@ import './helpers/utilities.css' -import {afterEach, expect, test} from 'vitest' +import {afterAll, afterEach, beforeAll, expect, test, vi} from 'vitest' import {page} from 'vitest/browser' -import {render} from '@solidjs/testing-library' -import {RouterProvider, createMemoryHistory} from '@tanstack/solid-router' -import {makeRpcClient} from '@conciv/contract' -import {parseConcivSettings} from '../src/data/settings.js' -import {createConcivRouter} from '../src/router.js' -import {CORE_BASE, installFakeCore, sessionRow, type FakeCore} from './helpers/fake-core.js' - -const PANEL_SESSION = 'conciv_1' +import {coreControl} from './helpers/core-control.js' +import {coreRpc, createSession} from './helpers/core-session.js' +import {createShellHarness} from './helpers/shell-harness.js' + const SETTLED = {timeout: 1500} -let core: FakeCore | null = null - -afterEach(() => { - core?.restore() - core = null -}) - -function openPanel(config: Parameters[0] = {}): void { - core = installFakeCore({sessions: [sessionRow({id: PANEL_SESSION})], ...config}) - const router = createConcivRouter({ - rpc: makeRpcClient(CORE_BASE), - history: createMemoryHistory({initialEntries: [`/panel/${PANEL_SESSION}?open=true`]}), - environment: {rootNode: document, document}, - settings: parseConcivSettings(''), - }) - render(() => ) -} + +const core = {base: ''} +const harness = createShellHarness(() => core.base) + +beforeAll(async () => { + const booted = await coreControl.bootCore({id: 'panel-focus-stability', allowedOrigins: [window.location.origin]}) + core.base = booted.base +}, 60_000) + +afterAll(async () => { + await coreControl.closeCore() +}, 30_000) + +afterEach(harness.dispose) const editor = () => page.getByRole('textbox', {name: 'Message the conciv agent'}) -test('a composer that mounts after a slow draft load keeps the focus the panel gave it', async () => { - openPanel({delays: {'/rpc/drafts/get': 250}}) +async function openPanel(): Promise { + const sessionId = await createSession(coreRpc(core.base)) + harness.mountShell(`/panel/${sessionId}?open=true`) +} - await expect.element(editor(), SETTLED).toBeVisible() - const mountedNode = editor().element() - await core?.idle() - await expect.element(editor(), SETTLED).toHaveFocus() - expect(editor().element().isSameNode(mountedNode)).toBe(true) -}) +async function expectFocusSurvives(path: string[]): Promise { + const since = await coreControl.rpcMark() + const held = await coreControl.installFault({kind: 'gate', path}) + await openPanel() + await vi.waitUntil(async () => (await coreControl.faultPending(held)) > 0, {timeout: 5000, interval: 20}) -test('a composer that mounts while the harness metadata is still loading takes the focus the panel gave it', async () => { - openPanel({delays: {'/rpc/meta/models': 400}}) + await coreControl.releaseFault(held) await expect.element(editor(), SETTLED).toBeVisible() const mountedNode = editor().element() - await core?.idle() + await coreControl.awaitRpcCall(path, since) + await expect.element(editor(), SETTLED).toHaveFocus() expect(editor().element().isSameNode(mountedNode)).toBe(true) -}) +} -test('a composer that mounts while the transcript is still loading takes the focus the panel gave it', async () => { - openPanel({delays: {'/rpc/markers/list': 400}}) +test('a composer that mounts after a slow draft load keeps the focus the panel gave it', async () => { + await expectFocusSurvives(['drafts', 'get']) +}, 30_000) - await expect.element(editor(), SETTLED).toBeVisible() - const mountedNode = editor().element() - await core?.idle() - await expect.element(editor(), SETTLED).toHaveFocus() - expect(editor().element().isSameNode(mountedNode)).toBe(true) -}) +test('a composer that mounts while the harness metadata is still loading takes the focus the panel gave it', async () => { + await expectFocusSurvives(['meta', 'models']) +}, 30_000) + +test('a composer that mounts while the transcript is still loading takes the focus the panel gave it', async () => { + await expectFocusSurvives(['markers', 'list']) +}, 30_000) diff --git a/apps/conciv/test/panel-min-height.browser.test.tsx b/apps/conciv/test/panel-min-height.browser.test.tsx index d4b85fbdd..605975e01 100644 --- a/apps/conciv/test/panel-min-height.browser.test.tsx +++ b/apps/conciv/test/panel-min-height.browser.test.tsx @@ -1,48 +1,38 @@ import './helpers/utilities.css' -import {afterEach, expect, test} from 'vitest' +import {afterAll, afterEach, beforeAll, expect, test} from 'vitest' import {page} from 'vitest/browser' -import {render} from '@solidjs/testing-library' -import {RouterProvider, createMemoryHistory} from '@tanstack/solid-router' -import {makeRpcClient} from '@conciv/contract' -import {parseConcivSettings} from '../src/data/settings.js' -import {createConcivRouter} from '../src/router.js' -import {CORE_BASE, installFakeCore, sessionRow, type FakeCore} from './helpers/fake-core.js' - -const PANEL_SESSION = 'conciv_1' +import {coreControl} from './helpers/core-control.js' +import {coreRpc, createSession, seedDraft} from './helpers/core-session.js' +import {createShellHarness} from './helpers/shell-harness.js' + const STAGED = ['the grabbed hero section', 'the grabbed price row', 'the grabbed footer', 'the grabbed nav'] -let core: FakeCore | null = null - -afterEach(() => { - core?.restore() - core = null -}) - -function openPanel(): void { - core = installFakeCore({ - sessions: [sessionRow({id: PANEL_SESSION})], - draft: { - sessionId: PANEL_SESSION, - text: '', - selectionStart: 0, - selectionEnd: 0, - grabs: STAGED, - updatedAt: 1, - }, - }) - const router = createConcivRouter({ - rpc: makeRpcClient(CORE_BASE), - history: createMemoryHistory({initialEntries: [`/panel/${PANEL_SESSION}?open=true`]}), - environment: {rootNode: document, document}, - settings: parseConcivSettings(''), - }) - render(() => ) + +const core = {base: ''} +const harness = createShellHarness(() => core.base) + +beforeAll(async () => { + const booted = await coreControl.bootCore({id: 'panel-min-height', allowedOrigins: [window.location.origin]}) + core.base = booted.base +}, 60_000) + +afterAll(async () => { + await coreControl.closeCore() +}, 30_000) + +afterEach(harness.dispose) + +async function openPanel(): Promise { + const rpc = coreRpc(core.base) + const sessionId = await createSession(rpc) + await seedDraft(rpc, sessionId, {grabs: STAGED}) + harness.mountShell(`/panel/${sessionId}?open=true`) } const input = () => page.getByRole('textbox', {name: 'Message the conciv agent'}) test('the composer stays reachable when the viewport clamps the panel below its minimum height', async () => { await page.viewport(1000, 400) - openPanel() + await openPanel() await expect.element(page.getByText('the grabbed hero section')).toBeVisible() @@ -51,22 +41,22 @@ test('the composer stays reachable when the viewport clamps the panel below its await page.getByRole('button', {name: 'Send message'}).click() await expect.element(page.getByText('still typeable at the smallest panel')).toBeVisible() -}) +}, 30_000) test('the grabs strip resizes through the shared separator handle', async () => { await page.viewport(1000, 900) - openPanel() + await openPanel() const handle = page.getByRole('separator', {name: 'Resize grabs height'}) await expect.element(handle).toBeVisible() await expect.element(handle).toHaveAttribute('aria-valuenow', '288') await expect.element(page.getByText('the grabbed hero section')).toBeVisible() await expect.element(page.getByText('the grabbed nav')).toBeVisible() -}) +}, 30_000) test('the staged grabs come back into the flow once the panel has room again', async () => { await page.viewport(1000, 400) - openPanel() + await openPanel() await expect.element(input()).toBeVisible() @@ -80,4 +70,4 @@ test('the staged grabs come back into the flow once the panel has room again', a await input().fill('room to breathe') await page.getByRole('button', {name: 'Send message'}).click() await expect.element(page.getByText('room to breathe')).toBeVisible() -}) +}, 30_000) diff --git a/apps/conciv/test/quick-add-pane.browser.test.tsx b/apps/conciv/test/quick-add-pane.browser.test.tsx index edb3d6e1a..55d342252 100644 --- a/apps/conciv/test/quick-add-pane.browser.test.tsx +++ b/apps/conciv/test/quick-add-pane.browser.test.tsx @@ -1,48 +1,63 @@ import './helpers/utilities.css' -import {afterEach, expect, test} from 'vitest' +import {afterAll, afterEach, beforeAll, expect, test, vi} from 'vitest' import {page} from 'vitest/browser' +import {coreControl} from './helpers/core-control.js' +import {coreRpc, createSession} from './helpers/core-session.js' import {createShellHarness} from './helpers/shell-harness.js' -import {sessionRow} from './helpers/fake-core.js' import {expectRetryRecovers} from './helpers/retry-recovery.js' -const PANEL_SESSION = 'conciv_1' -const harness = createShellHarness(PANEL_SESSION) -const mountShell = (config: Parameters[1] = {}): void => harness.mountShell('/quick', config) +const RESOLVE_PATH = ['sessions', 'resolve'] + +const core = {base: ''} +const harness = createShellHarness(() => core.base) + +beforeAll(async () => { + const booted = await coreControl.bootCore({id: 'quick-add-pane', allowedOrigins: [window.location.origin]}) + core.base = booted.base +}, 60_000) + +afterAll(async () => { + await coreControl.closeCore() +}, 30_000) afterEach(harness.dispose) const startFailure = () => page.getByText(/conciv could not start a pane/) const editor = () => page.getByRole('textbox', {name: 'Message the conciv agent'}) +const closePane = () => page.getByRole('button', {name: 'Close pane'}) test('a quick terminal pane that fails to start shows a retry action', async () => { - mountShell({resolveRejects: true}) + const refused = await coreControl.installFault({kind: 'fail', path: RESOLVE_PATH, status: 500}) + harness.mountShell('/quick') await expect.element(startFailure(), {timeout: 8000}).toBeVisible() await expect.element(page.getByRole('button', {name: 'Retry'})).toBeVisible() -}) + + await coreControl.releaseFault(refused) +}, 30_000) test('retrying a failed quick terminal pane against a healthy engine starts it', async () => { - mountShell({resolveRejects: true}) + const refused = await coreControl.installFault({kind: 'fail', path: RESOLVE_PATH, status: 500}) + harness.mountShell('/quick') await expect.element(startFailure(), {timeout: 8000}).toBeVisible() - await expectRetryRecovers(() => harness.core()?.setResolveRejects(false), editor, startFailure) -}) + await expectRetryRecovers(() => coreControl.releaseFault(refused), editor, startFailure) +}, 30_000) test('rapid double-trigger creates exactly one pane', async () => { - harness.mountShell('/quick?panes=conciv_1&focus=0', { - sessions: [sessionRow({id: 'conciv_1'})], - delays: {'/rpc/sessions/resolve': 300}, - }) + const sessionId = await createSession(coreRpc(core.base)) + harness.mountShell(`/quick?panes=${sessionId}&focus=0`) await expect.element(editor(), {timeout: 8000}).toBeVisible() + const held = await coreControl.installFault({kind: 'gate', path: RESOLVE_PATH}) const splitButton = page.getByRole('button', {name: 'Split pane (Mod+D)'}) const firstClick = splitButton.click() const secondClick = splitButton.click() await Promise.all([firstClick, secondClick]) + await vi.waitUntil(async () => (await coreControl.faultPending(held)) === 1, {timeout: 5000, interval: 20}) + + await coreControl.releaseFault(held) - await harness.core()?.idle() - const addPaneCalls = harness - .core() - ?.calls.filter((call) => call.path === '/rpc/sessions/resolve' && Object.keys(call.body ?? {}).length === 0).length - expect(addPaneCalls).toBe(1) -}) + await expect.element(closePane().nth(1), {timeout: 8000}).toBeVisible() + await expect.element(closePane().nth(2)).not.toBeInTheDocument() +}, 30_000) diff --git a/apps/conciv/test/reachability-flows.browser.test.tsx b/apps/conciv/test/reachability-flows.browser.test.tsx index 343a5d802..e04e885db 100644 --- a/apps/conciv/test/reachability-flows.browser.test.tsx +++ b/apps/conciv/test/reachability-flows.browser.test.tsx @@ -1,12 +1,22 @@ import './helpers/utilities.css' -import {afterEach, expect, test} from 'vitest' +import {afterAll, afterEach, beforeAll, expect, test} from 'vitest' import {page, userEvent} from 'vitest/browser' +import {coreControl} from './helpers/core-control.js' +import {coreRpc, createSession} from './helpers/core-session.js' import {createShellHarness} from './helpers/shell-harness.js' import {expectRetryRecovers} from './helpers/retry-recovery.js' -const PANEL_SESSION = 'conciv_1' -const harness = createShellHarness(PANEL_SESSION) -const mountShell = harness.mountShell +const core = {base: ''} +const harness = createShellHarness(() => core.base) + +beforeAll(async () => { + const booted = await coreControl.bootCore({id: 'reachability-flows', allowedOrigins: [window.location.origin]}) + core.base = booted.base +}, 60_000) + +afterAll(async () => { + await coreControl.closeCore() +}, 30_000) afterEach(harness.dispose) @@ -16,13 +26,20 @@ const genericBoundary = () => page.getByText('Something went wrong!') const engineUnreachableNotice = () => page.getByText('conciv lost connection to the engine.') const serverError = () => page.getByText('Internal Server Error') +async function openSessionPanel(): Promise { + const sessionId = await createSession(coreRpc(core.base)) + harness.mountShell(`/panel/${sessionId}?open=true`) + await expect.element(editor(), {timeout: 8000}).toBeVisible() +} + test('a dead engine at boot shows our error screen, not the generic boundary, and Retry recovers it', async () => { - mountShell('/panel/latest?open=true', {networkFail: true}) + const outage = await coreControl.installFault({kind: 'abort'}) + harness.mountShell('/panel/latest?open=true') await expect.element(errorScreen(), {timeout: 8000}).toBeVisible() await expect.element(genericBoundary()).not.toBeInTheDocument() - harness.core()?.setNetworkFail(false) + await coreControl.releaseFault(outage) await page .getByRole('alert') .filter({hasText: /couldn.t reach the engine/}) @@ -30,40 +47,42 @@ test('a dead engine at boot shows our error screen, not the generic boundary, an .click() await expect.element(editor(), {timeout: 8000}).toBeVisible() -}) +}, 30_000) test('a server error while resolving /panel/latest shows the actual error, not the unreachable message', async () => { - mountShell('/panel/latest?open=true', {resolveRejects: true}) + const refused = await coreControl.installFault({kind: 'fail', path: ['sessions', 'resolve'], status: 500}) + harness.mountShell('/panel/latest?open=true') await expect.element(serverError(), {timeout: 8000}).toBeVisible() await expect.element(errorScreen()).not.toBeInTheDocument() await expect.element(genericBoundary()).not.toBeInTheDocument() - await expectRetryRecovers(() => harness.core()?.setResolveRejects(false), editor, serverError) -}) + await expectRetryRecovers(() => coreControl.releaseFault(refused), editor, serverError) +}, 30_000) test('a healthy engine resolves /panel/latest straight to the warm session', async () => { - mountShell('/panel/latest?open=true') + harness.mountShell('/panel/latest?open=true') await expect.element(editor(), {timeout: 8000}).toBeVisible() await expect.element(errorScreen()).not.toBeInTheDocument() -}) +}, 30_000) test('a sustained outage raises exactly one standing notice, and it clears once the engine returns', async () => { - mountShell(`/panel/${PANEL_SESSION}?open=true`) - await expect.element(editor(), {timeout: 8000}).toBeVisible() + await openSessionPanel() + + const outage = await coreControl.installFault({kind: 'abort'}) + await editor().fill('rename the widget package') - harness.core()?.setNetworkFail(true) await expect.element(engineUnreachableNotice(), {timeout: 8000}).toBeVisible() await expect.element(page.getByRole('button', {name: 'Retry'})).toBeVisible() - harness.core()?.setNetworkFail(false) + await coreControl.releaseFault(outage) await expect.element(engineUnreachableNotice(), {timeout: 8000}).not.toBeInTheDocument() -}) +}, 30_000) test('a 500 from an otherwise healthy engine never raises the unreachable notice', async () => { - mountShell(`/panel/${PANEL_SESSION}?open=true`, {rejectSend: true}) - await expect.element(editor(), {timeout: 8000}).toBeVisible() + const refused = await coreControl.installFault({kind: 'fail', path: ['chat', 'send'], status: 500}) + await openSessionPanel() await editor().fill('rename the widget package') await userEvent.keyboard('{Enter}') @@ -72,25 +91,31 @@ test('a 500 from an otherwise healthy engine never raises the unreachable notice .element(page.getByRole('region', {name: /Notifications/})) .toHaveTextContent(/Internal Server Error|could not be sent/) await expect.element(engineUnreachableNotice()).not.toBeInTheDocument() -}) + + await coreControl.releaseFault(refused) +}, 30_000) test('a failing engine-info probe on an otherwise healthy connection never raises the unreachable notice', async () => { - mountShell(`/panel/${PANEL_SESSION}?open=true`, {rejectEngineProbe: true}) + const refused = await coreControl.installFault({kind: 'fail', path: ['meta', 'engine'], status: 500}) + const since = await coreControl.rpcMark() + await openSessionPanel() - await expect.element(editor(), {timeout: 8000}).toBeVisible() - await harness.core()?.idle() + expect(await coreControl.awaitRpcCall(['meta', 'engine'], since)).toBe(500) await expect.element(engineUnreachableNotice()).not.toBeInTheDocument() -}) + + await coreControl.releaseFault(refused) +}, 30_000) test('the composer disables sending with a distinct message once the engine is unreachable', async () => { - mountShell(`/panel/${PANEL_SESSION}?open=true`) - await expect.element(editor(), {timeout: 8000}).toBeVisible() - await editor().fill('rename the widget package') + await openSessionPanel() - harness.core()?.setNetworkFail(true) + const outage = await coreControl.installFault({kind: 'abort'}) + await editor().fill('rename the widget package') await expect .element(page.getByRole('button', {name: 'conciv lost connection to the engine'}), {timeout: 8000}) .toBeVisible() await expect.element(page.getByRole('button', {name: 'conciv lost connection to the engine'})).toBeDisabled() -}) + + await coreControl.releaseFault(outage) +}, 30_000) diff --git a/apps/conciv/test/route-boundary.browser.test.tsx b/apps/conciv/test/route-boundary.browser.test.tsx index 0803c7e4b..5cc369b78 100644 --- a/apps/conciv/test/route-boundary.browser.test.tsx +++ b/apps/conciv/test/route-boundary.browser.test.tsx @@ -1,67 +1,70 @@ import './helpers/utilities.css' -import {afterEach, expect, test} from 'vitest' +import {afterAll, afterEach, beforeAll, expect, test} from 'vitest' import {page} from 'vitest/browser' -import {render} from '@solidjs/testing-library' -import {RouterProvider, createMemoryHistory} from '@tanstack/solid-router' -import {makeRpcClient} from '@conciv/contract' -import {parseConcivSettings} from '../src/data/settings.js' -import {createConcivRouter} from '../src/router.js' -import {CORE_BASE, installFakeCore, sessionRow, type FakeCore} from './helpers/fake-core.js' - -const PANEL_SESSION = 'conciv_1' -const HELD_ROUTE_MS = 1500 +import {coreControl} from './helpers/core-control.js' +import {coreRpc, createSession} from './helpers/core-session.js' +import {createShellHarness} from './helpers/shell-harness.js' + const WHILE_HELD = {timeout: 700} -let core: FakeCore | null = null - -afterEach(() => { - core?.restore() - core = null -}) - -const PANEL_ENTRY = `/panel/${PANEL_SESSION}?open=true` -const CLOSED_ENTRY = '/' - -function mountShell(entry: string, config: Parameters[0] = {}): void { - core = installFakeCore({sessions: [sessionRow({id: PANEL_SESSION})], ...config}) - const router = createConcivRouter({ - rpc: makeRpcClient(CORE_BASE), - history: createMemoryHistory({initialEntries: [entry]}), - environment: {rootNode: document, document}, - settings: parseConcivSettings(''), - }) - render(() => ) -} + +const core = {base: ''} +const harness = createShellHarness(() => core.base) + +beforeAll(async () => { + const booted = await coreControl.bootCore({id: 'route-boundary', allowedOrigins: [window.location.origin]}) + core.base = booted.base +}, 60_000) + +afterAll(async () => { + await coreControl.closeCore() +}, 30_000) + +afterEach(harness.dispose) const editor = () => page.getByRole('textbox', {name: 'Message the conciv agent'}) const launcher = () => page.getByRole('button', {name: 'Open conciv chat'}) const routePending = () => page.getByRole('progressbar', {name: 'Loading conciv'}) +async function openPanel(): Promise { + const sessionId = await createSession(coreRpc(core.base)) + harness.mountShell(`/panel/${sessionId}?open=true`) +} + test('the pane paints its composer while the element captures query is still in flight', async () => { - mountShell(PANEL_ENTRY, {delays: {'/rpc/captures/list': HELD_ROUTE_MS}}) + const held = await coreControl.installFault({kind: 'gate', path: ['captures', 'list']}) + await openPanel() await expect.element(editor(), WHILE_HELD).toBeVisible() await expect.element(routePending(), WHILE_HELD).not.toBeInTheDocument() -}) + + await coreControl.releaseFault(held) +}, 30_000) test('the shell keeps its launcher while the session list query is still in flight', async () => { - mountShell(CLOSED_ENTRY, {delays: {'/rpc/sessions/list': HELD_ROUTE_MS}}) + const held = await coreControl.installFault({kind: 'gate', path: ['sessions', 'list']}) + harness.mountShell('/') await expect.element(launcher(), WHILE_HELD).toBeVisible() await expect.element(routePending(), WHILE_HELD).not.toBeInTheDocument() -}) + + await coreControl.releaseFault(held) +}, 30_000) test('a pane whose queries all answer immediately never shows the route pending loader', async () => { - mountShell(PANEL_ENTRY) + await openPanel() await expect.element(editor(), WHILE_HELD).toBeVisible() - await core?.idle() await expect.element(routePending(), WHILE_HELD).not.toBeInTheDocument() -}) +}, 30_000) test('a slow beforeLoad reveals the route pending loader, then hands off to the pane', async () => { - mountShell('/panel/latest?open=true', {delays: {'/rpc/sessions/resolve': 900}}) + const held = await coreControl.installFault({kind: 'gate', path: ['sessions', 'resolve']}) + harness.mountShell('/panel/latest?open=true') await expect.element(routePending()).toBeVisible() - await expect.element(editor(), {timeout: 2000}).toBeVisible() + + await coreControl.releaseFault(held) + + await expect.element(editor(), {timeout: 8000}).toBeVisible() await expect.element(routePending()).not.toBeInTheDocument() -}) +}, 30_000) diff --git a/packages/extension-testkit/src/rpc-fault.ts b/packages/extension-testkit/src/rpc-fault.ts index 738f7f26c..2e613b64d 100644 --- a/packages/extension-testkit/src/rpc-fault.ts +++ b/packages/extension-testkit/src/rpc-fault.ts @@ -71,7 +71,7 @@ export async function failRpcCalls( } } -export async function abortRpcCalls(page: Page, options: {path: readonly string[]}): Promise { +export async function abortRpcCalls(page: Page, options: {path?: readonly string[]} = {}): Promise { const broken = {value: true} const matcher = pathMatcher(options.path) const handler: RouteHandler = async (route) => { From 44e57d4ac688ccc678c498e2a99f3c50b5d8fc80 Mon Sep 17 00:00:00 2001 From: Omri Katz <9701896+omridevk@users.noreply.github.com> Date: Sat, 15 Aug 2026 06:18:56 +0300 Subject: [PATCH 4/9] =?UTF-8?q?test(conciv):=20#354=20phases=204+5=20?= =?UTF-8?q?=E2=80=94=20close=20the=20fake-core=20chapter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit draft-storage boots a real core through bootCoreKit and verifies every write through the real drafts/get row: no globalThis.fetch monkeypatch, no hand-encoded {json, meta} envelope, no fake timers. The local proxy in front of the kit gains a path-scoped refusal (the failed-initial-read path) and per-path request counting (the debounce collapse), so faults land at the transport boundary and payload assertions come from the database. Deletes fake-core.ts, fake-core-router.ts, the fake-core-socket browser test (websocket coverage lives in transport-standalone.it.test.ts and the harness-testkit rpc suites) and the fakeCoreSocket vite plugin. Sweeps: failRpcCalls only installs its websocket route when a caller asks for it (playwright has no unrouteWebSocket, so an always-on handler accumulated across files on the persistent vitest page); the client boot helper drops a dead openTerminal option makeApp never accepted; apps/conciv drops the @orpc/server devDependency the deleted fake router was the last consumer of. Co-Authored-By: Claude Fable 5 --- .fallowrc.json | 1 - apps/conciv/package.json | 1 - apps/conciv/test/draft-storage.test.ts | 267 ++++++++++-------- .../test/fake-core-socket.browser.test.ts | 43 --- apps/conciv/test/helpers/fake-core-router.ts | 107 ------- apps/conciv/test/helpers/fake-core.ts | 243 ---------------- apps/conciv/test/helpers/proxy.ts | 22 +- apps/conciv/vitest.config.ts | 22 +- packages/client/test/helpers/boot.ts | 9 +- .../embed/tests/e2e/model-selector.it.test.ts | 2 +- packages/embed/tests/e2e/rpc-fault.it.test.ts | 2 +- packages/extension-testkit/src/rpc-fault.ts | 4 +- pnpm-lock.yaml | 3 - 13 files changed, 168 insertions(+), 558 deletions(-) delete mode 100644 apps/conciv/test/fake-core-socket.browser.test.ts delete mode 100644 apps/conciv/test/helpers/fake-core-router.ts delete mode 100644 apps/conciv/test/helpers/fake-core.ts diff --git a/.fallowrc.json b/.fallowrc.json index eea1659f8..64a3af315 100644 --- a/.fallowrc.json +++ b/.fallowrc.json @@ -149,7 +149,6 @@ "packages/extensions/tanstack/test/host/conciv/extensions/connect-probe.tsx", "packages/extensions/try-it/test/fixture/connect-pane-fixture.tsx", "e2e/harnesses/vite.*.config.ts", - "apps/conciv/test/helpers/fake-core-router.ts", "apps/conciv/test/commands/core-control.ts", "apps/conciv/test/commands/core-testkit.ts" ] diff --git a/apps/conciv/package.json b/apps/conciv/package.json index 178e4aeee..be66b8b4c 100644 --- a/apps/conciv/package.json +++ b/apps/conciv/package.json @@ -58,7 +58,6 @@ "@conciv/storage-history": "workspace:^", "@conciv/uno-preset": "workspace:*", "@conciv/vitest-config": "workspace:*", - "@orpc/server": "catalog:", "@solidjs/testing-library": "^0.8.10", "@tanstack/ai": "catalog:", "@tanstack/query-core": "^5.80.2", diff --git a/apps/conciv/test/draft-storage.test.ts b/apps/conciv/test/draft-storage.test.ts index 394ec8b1c..faa6e9faf 100644 --- a/apps/conciv/test/draft-storage.test.ts +++ b/apps/conciv/test/draft-storage.test.ts @@ -1,65 +1,56 @@ -import {afterEach, expect, test, vi} from 'vitest' -import {makeRpcClient, type DraftRow} from '@conciv/contract' -import {appendDraft, makeDraftStorage, type PaneDraftStorage} from '../src/pane/draft-storage.js' - -const BASE = 'http://conciv.test' -const SESSION = 'conciv_1' - -type Server = {row: DraftRow | null; writes: unknown[]; failReads: boolean} - -const realFetch = globalThis.fetch - -afterEach(() => { - globalThis.fetch = realFetch - vi.useRealTimers() +import {expect, test as baseTest} from 'vitest' +import {makeRpcClient, type RpcClient} from '@conciv/contract' +import {bootCoreKit, type CoreKit} from '@conciv/extension-testkit/core-kit' +import {appendDraft, makeDraftStorage} from '../src/pane/draft-storage.js' +import {createSession, seedDraft} from './helpers/core-session.js' +import {proxyTo, type ProxyCore} from './helpers/proxy.js' + +const DRAFTS_GET_PATH = '/rpc/drafts/get' +const DRAFTS_SET_PATH = '/rpc/drafts/set' +const SETTLED = {timeout: 4000, interval: 25} + +type StoredDraft = {sessionId: string; text: string; selectionStart: number; selectionEnd: number; grabs: string[]} + +const test = baseTest.extend<{kit: CoreKit; core: ProxyCore; sessionId: string}>({ + kit: [ + // oxlint-disable-next-line no-empty-pattern -- vitest's fixture parser requires the literal `{}` destructuring + async ({}, use) => { + const kit = await bootCoreKit({id: 'draft-storage'}) + await use(kit) + await kit.cleanup() + }, + {scope: 'file'}, + ], + core: async ({kit}, use) => { + const core = await proxyTo(kit.base) + await use(core) + await core.close() + }, + sessionId: async ({kit}, use) => { + await use(await createSession(kit.rpc)) + }, }) -function reply(value: unknown): Response { - return new Response(JSON.stringify({json: value, meta: []}), { - status: 200, - headers: {'content-type': 'application/json'}, - }) +function composerDraft(text: string, grabs: string[] = []): string { + return JSON.stringify({text, quote: null, grabs, attachments: []}) } -function installServer(server: Server): void { - globalThis.fetch = async (input: RequestInfo | URL, init?: RequestInit): Promise => { - const request = input instanceof Request ? input : new Request(input, init) - const path = new URL(request.url).pathname - if (path === '/rpc/drafts/get') { - if (server.failReads) return new Response('down', {status: 500}) - return reply(server.row) - } - if (path === '/rpc/drafts/set') { - const parsed: unknown = await request.json() - if (typeof parsed === 'object' && parsed !== null && 'json' in parsed) server.writes.push(parsed.json) - return reply({ok: true}) - } - throw new Error(`the fake core has no route for ${path}`) +async function storedDraft(rpc: RpcClient, sessionId: string): Promise { + const row = await rpc.drafts.get({sessionId}) + if (!row) return null + return { + sessionId: row.sessionId, + text: row.text, + selectionStart: row.selectionStart, + selectionEnd: row.selectionEnd, + grabs: row.grabs, } } -function draftRow(text: string, grabs: string[]): DraftRow { - return {sessionId: SESSION, text, selectionStart: text.length, selectionEnd: text.length, grabs, updatedAt: 1} -} +test('seeds the cache from the server draft row in the composer draft shape', async ({kit, core, sessionId}) => { + await seedDraft(kit.rpc, sessionId, {text: 'kept across the reload', grabs: ['a grabbed heading']}) -async function settleWrites(): Promise { - await vi.advanceTimersByTimeAsync(350) - await Promise.resolve() -} - -async function bootWritableStorage(): Promise<{server: Server; draftStorage: PaneDraftStorage}> { - const server: Server = {row: null, writes: [], failReads: false} - installServer(server) - const draftStorage = await makeDraftStorage(makeRpcClient(BASE), SESSION) - vi.useFakeTimers() - return {server, draftStorage} -} - -test('seeds the cache from the server draft row in the composer draft shape', async () => { - const server: Server = {row: draftRow('kept across the reload', ['a grabbed heading']), writes: [], failReads: false} - installServer(server) - - const draftStorage = await makeDraftStorage(makeRpcClient(BASE), SESSION) + const draftStorage = await makeDraftStorage(makeRpcClient(core.base), sessionId) expect(JSON.parse(draftStorage.storage.getItem('any') ?? '')).toEqual({ text: 'kept across the reload', @@ -69,104 +60,134 @@ test('seeds the cache from the server draft row in the composer draft shape', as }) }) -test('starts empty when the server has no draft', async () => { - const server: Server = {row: null, writes: [], failReads: false} - installServer(server) - - const draftStorage = await makeDraftStorage(makeRpcClient(BASE), SESSION) +test('starts empty when the server has no draft', async ({core, sessionId}) => { + const draftStorage = await makeDraftStorage(makeRpcClient(core.base), sessionId) expect(draftStorage.storage.getItem('any')).toBeNull() }) -test('writes the composer draft back to the server with the caret at the end', async () => { - const {server, draftStorage} = await bootWritableStorage() +test('writes the composer draft back to the server with the caret at the end', async ({kit, core, sessionId}) => { + const draftStorage = await makeDraftStorage(makeRpcClient(core.base), sessionId) - draftStorage.storage.setItem( - 'any', - JSON.stringify({text: 'a fresh draft', quote: null, grabs: ['a heading'], attachments: []}), - ) - await settleWrites() + draftStorage.storage.setItem('any', composerDraft('a fresh draft', ['a heading'])) - expect(server.writes).toEqual([ - {sessionId: SESSION, text: 'a fresh draft', selectionStart: 13, selectionEnd: 13, grabs: ['a heading']}, - ]) + await expect + .poll(() => storedDraft(kit.rpc, sessionId), SETTLED) + .toEqual({ + sessionId, + text: 'a fresh draft', + selectionStart: 13, + selectionEnd: 13, + grabs: ['a heading'], + }) }) -test('collapses a burst of writes into the last draft', async () => { - const {server, draftStorage} = await bootWritableStorage() - - draftStorage.storage.setItem('any', JSON.stringify({text: 'a', quote: null, grabs: [], attachments: []})) - draftStorage.storage.setItem('any', JSON.stringify({text: 'ab', quote: null, grabs: [], attachments: []})) - draftStorage.storage.setItem('any', JSON.stringify({text: 'abc', quote: null, grabs: [], attachments: []})) - await settleWrites() - - expect(server.writes).toEqual([{sessionId: SESSION, text: 'abc', selectionStart: 3, selectionEnd: 3, grabs: []}]) +test('collapses a burst of writes into the last draft', async ({kit, core, sessionId}) => { + const draftStorage = await makeDraftStorage(makeRpcClient(core.base), sessionId) + + draftStorage.storage.setItem('any', composerDraft('a')) + draftStorage.storage.setItem('any', composerDraft('ab')) + draftStorage.storage.setItem('any', composerDraft('abc')) + + await expect + .poll(() => storedDraft(kit.rpc, sessionId), SETTLED) + .toEqual({ + sessionId, + text: 'abc', + selectionStart: 3, + selectionEnd: 3, + grabs: [], + }) + expect(core.requestCount(DRAFTS_SET_PATH)).toBe(1) }) -test('keeps the latest value readable even when the payload cannot be persisted', async () => { - const {server, draftStorage} = await bootWritableStorage() +test('keeps the latest value readable even when the payload cannot be persisted', async ({kit, core, sessionId}) => { + const draftStorage = await makeDraftStorage(makeRpcClient(core.base), sessionId) draftStorage.storage.setItem('any', 'not json at all') - await settleWrites() - expect(draftStorage.storage.getItem('any')).toBe('not json at all') - expect(server.writes).toEqual([]) + + draftStorage.storage.setItem('any', composerDraft('a draft that parses')) + + await expect + .poll(() => storedDraft(kit.rpc, sessionId), SETTLED) + .toEqual({ + sessionId, + text: 'a draft that parses', + selectionStart: 19, + selectionEnd: 19, + grabs: [], + }) + expect(core.requestCount(DRAFTS_SET_PATH)).toBe(1) }) -test('persists the noted caret offsets with the draft text', async () => { - const {server, draftStorage} = await bootWritableStorage() +test('persists the noted caret offsets with the draft text', async ({kit, core, sessionId}) => { + const draftStorage = await makeDraftStorage(makeRpcClient(core.base), sessionId) - draftStorage.storage.setItem('any', JSON.stringify({text: 'say hello!', quote: null, grabs: [], attachments: []})) + draftStorage.storage.setItem('any', composerDraft('say hello!')) draftStorage.noteSelection({start: 4, end: 4}) - await settleWrites() - expect(server.writes).toEqual([ - {sessionId: SESSION, text: 'say hello!', selectionStart: 4, selectionEnd: 4, grabs: []}, - ]) + await expect + .poll(() => storedDraft(kit.rpc, sessionId), SETTLED) + .toEqual({ + sessionId, + text: 'say hello!', + selectionStart: 4, + selectionEnd: 4, + grabs: [], + }) }) -test('clamps noted offsets that fall beyond the persisted text', async () => { - const {server, draftStorage} = await bootWritableStorage() +test('clamps noted offsets that fall beyond the persisted text', async ({kit, core, sessionId}) => { + const draftStorage = await makeDraftStorage(makeRpcClient(core.base), sessionId) draftStorage.noteSelection({start: 40, end: 44}) - draftStorage.storage.setItem('any', JSON.stringify({text: 'short', quote: null, grabs: [], attachments: []})) - await settleWrites() - - expect(server.writes).toEqual([{sessionId: SESSION, text: 'short', selectionStart: 5, selectionEnd: 5, grabs: []}]) + draftStorage.storage.setItem('any', composerDraft('short')) + + await expect + .poll(() => storedDraft(kit.rpc, sessionId), SETTLED) + .toEqual({ + sessionId, + text: 'short', + selectionStart: 5, + selectionEnd: 5, + grabs: [], + }) }) -test('appends to the stored draft on a new line with the caret at the end', async () => { - const server: Server = {row: draftRow('a first line', ['a heading']), writes: [], failReads: false} - installServer(server) +test('appends to the stored draft on a new line with the caret at the end', async ({kit, core, sessionId}) => { + await seedDraft(kit.rpc, sessionId, {text: 'a first line', grabs: ['a heading']}) - await appendDraft(makeRpcClient(BASE), SESSION, 'a second line') + await appendDraft(makeRpcClient(core.base), sessionId, 'a second line') - expect(server.writes).toEqual([ - { - sessionId: SESSION, - text: 'a first line\na second line', - selectionStart: 26, - selectionEnd: 26, - grabs: ['a heading'], - }, - ]) + expect(await storedDraft(kit.rpc, sessionId)).toEqual({ + sessionId, + text: 'a first line\na second line', + selectionStart: 26, + selectionEnd: 26, + grabs: ['a heading'], + }) }) -test('survives a failed initial read and still accepts writes', async () => { - const server: Server = {row: null, writes: [], failReads: true} - installServer(server) +test('survives a failed initial read and still accepts writes', async ({kit, core, sessionId}) => { + await seedDraft(kit.rpc, sessionId, {text: 'unreachable at boot'}) + core.fail(DRAFTS_GET_PATH) + + const draftStorage = await makeDraftStorage(makeRpcClient(core.base), sessionId) + + expect(draftStorage.storage.getItem('any')).toBeNull() - const draftStorage = await makeDraftStorage(makeRpcClient(BASE), SESSION) - server.failReads = false - vi.useFakeTimers() - draftStorage.storage.setItem( - 'any', - JSON.stringify({text: 'after the outage', quote: null, grabs: [], attachments: []}), - ) - await settleWrites() + core.repair() + draftStorage.storage.setItem('any', composerDraft('after the outage')) expect(draftStorage.storage.getItem('any')).toContain('after the outage') - expect(server.writes).toEqual([ - {sessionId: SESSION, text: 'after the outage', selectionStart: 16, selectionEnd: 16, grabs: []}, - ]) + await expect + .poll(() => storedDraft(kit.rpc, sessionId), SETTLED) + .toEqual({ + sessionId, + text: 'after the outage', + selectionStart: 16, + selectionEnd: 16, + grabs: [], + }) }) diff --git a/apps/conciv/test/fake-core-socket.browser.test.ts b/apps/conciv/test/fake-core-socket.browser.test.ts deleted file mode 100644 index 9248dc814..000000000 --- a/apps/conciv/test/fake-core-socket.browser.test.ts +++ /dev/null @@ -1,43 +0,0 @@ -import {describe, expect, it} from 'vitest' -import {z} from 'zod' -import type {RpcClient} from '@conciv/contract' -import {rpcOverWebsocket} from '@conciv/harness-testkit/rpc-websocket-client' - -const AddressSchema = z.object({base: z.string(), wsUrl: z.string()}) - -async function openFakeCore(): Promise<{client: RpcClient; socket: WebSocket}> { - const payload: unknown = await fetch('/__fake-core').then((response) => response.json()) - const socket = new WebSocket(AddressSchema.parse(payload).wsUrl) - return {client: rpcOverWebsocket(socket), socket} -} - -describe('the apps/conciv fake core answers the production rpc client over a real websocket', () => { - it('serves the canned session list to a client that never touches http', async () => { - const {client, socket} = await openFakeCore() - const sessions = await client.sessions.list(undefined) - expect(sessions[0]?.id).toBe('conciv_1') - socket.close() - }) - - it('streams the transcript snapshot and the run chunks a send produces', async () => { - const {client, socket} = await openFakeCore() - const abort = new AbortController() - const stream = await client.chat.subscribe({sessionId: 'conciv_1'}, {signal: abort.signal}) - const seen: string[] = [] - async function collect(): Promise { - for await (const chunk of stream) { - const type = typeof chunk === 'object' && chunk !== null && 'type' in chunk ? String(chunk.type) : '' - seen.push(type) - if (type === 'RUN_FINISHED') return - } - } - const collected = collect() - await client.chat.send({sessionId: 'conciv_1', runId: 'conciv_run_1', text: 'hello'}) - await collected - abort.abort() - expect(seen[0]).toBe('MESSAGES_SNAPSHOT') - expect(seen).toContain('RUN_STARTED') - expect(seen.at(-1)).toBe('RUN_FINISHED') - socket.close() - }) -}) diff --git a/apps/conciv/test/helpers/fake-core-router.ts b/apps/conciv/test/helpers/fake-core-router.ts deleted file mode 100644 index a2b37988d..000000000 --- a/apps/conciv/test/helpers/fake-core-router.ts +++ /dev/null @@ -1,107 +0,0 @@ -import {implement} from '@orpc/server' -import {EventType, type StreamChunk} from '@tanstack/ai' -import {contract} from '@conciv/contract' -import type {RpcContext} from '@conciv/protocol/rpc-types' -import {sessionRow} from './fake-core.js' - -const os = implement(contract).$context() - -const RUN_ID = 'conciv_run_1' - -export type FakeCoreStream = { - push: (chunk: StreamChunk) => void - subscribeCount: () => number -} - -function makeStream(): { - stream: FakeCoreStream - subscribe: (signal: AbortSignal) => AsyncGenerator -} { - const listeners = new Set<(chunk: StreamChunk) => void>() - const opened = {count: 0} - return { - stream: { - push: (chunk) => { - for (const listener of listeners) listener(chunk) - }, - subscribeCount: () => opened.count, - }, - subscribe: async function* (signal: AbortSignal): AsyncGenerator { - opened.count += 1 - const queue: StreamChunk[] = [{type: EventType.MESSAGES_SNAPSHOT, messages: []}] - const waiter = {wake: () => {}} - const listener = (chunk: StreamChunk): void => { - queue.push(chunk) - waiter.wake() - } - listeners.add(listener) - try { - while (!signal.aborted) { - const next = queue.shift() - if (next !== undefined) { - yield next - continue - } - await new Promise((resolve) => { - waiter.wake = resolve - signal.addEventListener('abort', () => resolve(), {once: true}) - }) - } - } finally { - listeners.delete(listener) - } - }, - } -} - -export function makeFakeCoreRouter(): {router: ReturnType; stream: FakeCoreStream} { - const {stream, subscribe} = makeStream() - return {router: buildRouter(subscribe, stream), stream} -} - -function buildRouter(subscribe: (signal: AbortSignal) => AsyncGenerator, stream: FakeCoreStream) { - return { - sessions: { - list: os.sessions.list.handler(() => [sessionRow({id: 'conciv_1'})]), - create: os.sessions.create.handler(() => ({sessionId: 'conciv_2'})), - compact: os.sessions.compact.handler((): {ok: true} => ({ok: true})), - }, - drafts: { - get: os.drafts.get.handler(() => null), - set: os.drafts.set.handler((): {ok: true} => ({ok: true})), - }, - markers: { - list: os.markers.list.handler(() => []), - }, - captures: { - list: os.captures.list.handler(() => ({captures: [], cssBundles: {}})), - }, - meta: { - models: os.meta.models.handler(() => ({ - models: [{id: 'model-1', name: 'Fable'}], - defaultModel: 'model-1', - harness: {id: 'claude', name: 'Claude', canLaunch: true, imageInput: false}, - })), - commands: os.meta.commands.handler(() => ({commands: []})), - tools: os.meta.tools.handler(() => ({tools: []})), - engine: os.meta.engine.handler(() => ({ - stale: false, - changed: [], - tracked: ['@conciv/core'], - bootedAt: 0, - fingerprint: 'stamp-boot', - })), - }, - chat: { - subscribe: os.chat.subscribe.handler(({signal}) => subscribe(signal ?? new AbortController().signal)), - stop: os.chat.stop.handler((): {ok: true} => ({ok: true})), - send: os.chat.send.handler((): {ok: true; runId: string} => { - queueMicrotask(() => { - stream.push({type: EventType.RUN_STARTED, threadId: 'conciv_1', runId: RUN_ID}) - stream.push({type: EventType.RUN_FINISHED, threadId: 'conciv_1', runId: RUN_ID}) - }) - return {ok: true, runId: RUN_ID} - }), - }, - } -} diff --git a/apps/conciv/test/helpers/fake-core.ts b/apps/conciv/test/helpers/fake-core.ts deleted file mode 100644 index 3e15ba7f1..000000000 --- a/apps/conciv/test/helpers/fake-core.ts +++ /dev/null @@ -1,243 +0,0 @@ -import {onlineManager} from '@tanstack/query-core' -import { - browserRpcConnection, - closeBrowserRpcConnection, - type DraftRow, - type MarkerRow, - type SessionMeta, -} from '@conciv/contract' -import '../../src/lib/api-base.js' - -export const CORE_BASE = 'http://conciv.test' - -export type CoreCall = {path: string; body: unknown} - -export type FakeCore = { - calls: CoreCall[] - push: (chunk: unknown) => void - subscribeCount: () => number - releaseSnapshot: () => void - idle: () => Promise - restore: () => void - setEngine: (next: {stale: boolean; fingerprint?: string}) => void - setNetworkFail: (fail: boolean) => void - setResolveRejects: (fail: boolean) => void - setRejectEngineProbe: (fail: boolean) => void - setResolveTransportFails: (fail: boolean) => void -} - -const QUIET_MS = 60 - -export type FakeCoreConfig = { - draft?: DraftRow | null - delays?: Record - sessions?: SessionMeta[] - rejectSend?: boolean - throwSend?: boolean - snapshotFor?: (subscribeIndex: number) => unknown[] - holdSnapshot?: boolean - markers?: MarkerRow[] - holdRun?: boolean - launchOk?: boolean - launchRejects?: boolean - engineStale?: boolean - networkFail?: boolean - resolveRejects?: boolean - rejectEngineProbe?: boolean - resolveTransportFails?: boolean -} - -export function sessionRow(overrides: Partial & {id: string}): SessionMeta { - return { - title: 'rename the widget package', - updatedAt: Date.now(), - messageCount: 3, - running: false, - origin: 'conciv', - usage: null, - model: null, - hidden: false, - native: null, - ...overrides, - } -} - -function reply(value: unknown): Response { - return new Response(JSON.stringify({json: value, meta: []}), { - status: 200, - headers: {'content-type': 'application/json'}, - }) -} - -function frame(chunk: unknown): Uint8Array { - return new TextEncoder().encode(`event: message\ndata: ${JSON.stringify({json: chunk, meta: []})}\n\n`) -} - -async function bodyOf(request: Request): Promise { - try { - const parsed: unknown = await request.clone().json() - if (typeof parsed === 'object' && parsed !== null && 'json' in parsed) return parsed.json - return parsed - } catch { - return null - } -} - -const RUN_ID = 'conciv_run_1' - -function delayFor(schedule: number | number[] | undefined, callIndex: number): number { - if (schedule === undefined) return 0 - if (typeof schedule === 'number') return schedule - return schedule[Math.min(callIndex, schedule.length - 1)] ?? 0 -} - -export function installFakeCore(config: FakeCoreConfig = {}): FakeCore { - const realFetch = globalThis.fetch - const engine = {stale: config.engineStale ?? false, fingerprint: 'stamp-boot'} - const calls: CoreCall[] = [] - let subscribes = 0 - let snapshotReleased = false - let networkFail = config.networkFail ?? false - let resolveRejects = config.resolveRejects ?? false - let rejectEngineProbe = config.rejectEngineProbe ?? false - let resolveTransportFails = config.resolveTransportFails ?? false - if (typeof window !== 'undefined') window.__CONCIV_API_BASE__ = CORE_BASE - let inFlight = 0 - let quietTimer: ReturnType | undefined - const waitingForIdle: (() => void)[] = [] - const scheduleIdle = () => { - if (quietTimer !== undefined) clearTimeout(quietTimer) - if (inFlight > 0) return - quietTimer = setTimeout(() => { - for (const resolve of waitingForIdle.splice(0)) resolve() - }, QUIET_MS) - } - const core: FakeCore = { - calls, - push: () => {}, - subscribeCount: () => subscribes, - releaseSnapshot: () => { - snapshotReleased = true - }, - idle: () => - new Promise((resolve) => { - waitingForIdle.push(resolve) - scheduleIdle() - }), - restore: () => { - globalThis.fetch = realFetch - closeBrowserRpcConnection(CORE_BASE) - onlineManager.setOnline(true) - if (typeof window !== 'undefined') delete window.__CONCIV_API_BASE__ - }, - setEngine: (next) => { - engine.stale = next.stale - if (next.fingerprint !== undefined) engine.fingerprint = next.fingerprint - }, - setNetworkFail: (fail) => { - networkFail = fail - }, - setResolveRejects: (fail) => { - resolveRejects = fail - }, - setRejectEngineProbe: (fail) => { - rejectEngineProbe = fail - }, - setResolveTransportFails: (fail) => { - resolveTransportFails = fail - }, - } - - const liveStream = (signal: AbortSignal): Response => { - subscribes += 1 - const messages = config.snapshotFor?.(subscribes) ?? [] - const stream = new ReadableStream({ - start: (controller) => { - const sendSnapshot = () => controller.enqueue(frame({type: 'MESSAGES_SNAPSHOT', messages})) - const held = config.holdSnapshot === true && !snapshotReleased - if (held) core.releaseSnapshot = sendSnapshot - if (!held) sendSnapshot() - core.push = (chunk) => controller.enqueue(frame(chunk)) - signal.addEventListener('abort', () => { - core.push = () => {} - controller.error(new DOMException('aborted', 'AbortError')) - }) - }, - }) - return new Response(stream, {status: 200, headers: {'content-type': 'text/event-stream'}}) - } - - const routes: Record Response> = { - '/rpc/sessions/list': () => reply(config.sessions ?? [sessionRow({id: 'conciv_1'})]), - '/rpc/sessions/resolve': () => { - if (resolveTransportFails) throw new TypeError('Failed to fetch') - return resolveRejects - ? new Response('resolve refused', {status: 500}) - : reply({sessionId: config.sessions?.[0]?.id ?? 'conciv_1'}) - }, - '/rpc/sessions/create': () => reply({sessionId: 'conciv_2'}), - '/rpc/sessions/compact': () => reply({ok: true}), - '/rpc/drafts/get': () => reply(config.draft ?? null), - '/rpc/drafts/set': () => reply({ok: true}), - '/rpc/markers/list': () => reply(config.markers ?? []), - '/rpc/captures/list': () => reply({captures: [], cssBundles: {}}), - '/rpc/meta/models': () => - reply({ - models: [{id: 'model-1', name: 'Fable'}], - defaultModel: 'model-1', - harness: {id: 'claude', name: 'Claude', canLaunch: true, imageInput: false}, - }), - '/rpc/meta/commands': () => reply({commands: []}), - '/rpc/meta/engine': () => - rejectEngineProbe - ? new Response('engine probe refused', {status: 500}) - : reply({ - stale: engine.stale, - changed: engine.stale ? ['@conciv/tools'] : [], - tracked: ['@conciv/core', '@conciv/tools'], - bootedAt: 0, - fingerprint: engine.fingerprint, - }), - '/rpc/meta/tools': () => reply({tools: []}), - '/rpc/registry/catalog': () => reply([]), - '/rpc/chat/subscribe': (_body, signal) => liveStream(signal), - '/rpc/chat/stop': () => reply({ok: true}), - '/rpc/chat/send': () => { - if (config.throwSend) throw new TypeError('Failed to fetch') - if (config.rejectSend) return new Response('send refused', {status: 500}) - queueMicrotask(() => { - core.push({type: 'RUN_STARTED', threadId: 'conciv_1', runId: RUN_ID}) - if (config.holdRun) return - core.push({type: 'RUN_FINISHED', threadId: 'conciv_1', runId: RUN_ID, finishReason: 'stop'}) - }) - return reply({ok: true, runId: RUN_ID}) - }, - '/rpc/ext/terminal/launch': () => { - if (config.launchRejects) return new Response('no terminal', {status: 500}) - return reply({ok: config.launchOk ?? true}) - }, - '/rpc/ext/terminal/connectCommand': () => reply({command: 'claude --resume fake-session'}), - } - - globalThis.fetch = async (input: RequestInfo | URL, init?: RequestInit): Promise => { - const request = input instanceof Request ? input : new Request(input, init) - const url = new URL(request.url) - if (url.origin !== CORE_BASE) return realFetch(input, init) - if (networkFail) throw new TypeError('Failed to fetch') - const route = routes[url.pathname] - if (!route) throw new Error(`the fake core has no route for ${url.pathname}`) - inFlight += 1 - scheduleIdle() - const body = await bodyOf(request) - const priorCalls = calls.filter((call) => call.path === url.pathname).length - calls.push({path: url.pathname, body}) - const delay = delayFor(config.delays?.[url.pathname], priorCalls) - if (delay > 0) await new Promise((resolve) => setTimeout(resolve, delay)) - const response = route(body, request.signal) - inFlight -= 1 - scheduleIdle() - return response - } - browserRpcConnection(CORE_BASE, 'fetch') - return core -} diff --git a/apps/conciv/test/helpers/proxy.ts b/apps/conciv/test/helpers/proxy.ts index 88e5ac37c..9ec5497bf 100644 --- a/apps/conciv/test/helpers/proxy.ts +++ b/apps/conciv/test/helpers/proxy.ts @@ -5,8 +5,10 @@ import {listenLocal} from './listen-local.js' export type ProxyCore = { base: string port: number - requestCount: () => number + requestCount: (pathname?: string) => number wsConnectionCount: () => number + fail: (pathname: string) => void + repair: () => void close: () => Promise } @@ -21,11 +23,18 @@ function handshakeResponse(upstream: IncomingMessage): string { export async function proxyTo(targetBase: string, opts: {blockUpgrades?: boolean} = {}): Promise { const target = new URL(targetBase) - let count = 0 + const seen: string[] = [] + const refused = new Set() let upgrades = 0 const piped = new Set() const server: Server = createServer((req, res) => { - count += 1 + const pathname = new URL(req.url ?? '/', target).pathname + seen.push(pathname) + if (refused.has(pathname)) { + res.writeHead(500, {'content-type': 'text/plain'}) + res.end('the proxied core refused the call') + return + } const proxyReq = httpRequest( { hostname: target.hostname, @@ -78,8 +87,13 @@ export async function proxyTo(targetBase: string, opts: {blockUpgrades?: boolean return { base, port, - requestCount: () => count, + requestCount: (pathname) => + pathname === undefined ? seen.length : seen.filter((entry) => entry === pathname).length, wsConnectionCount: () => upgrades, + fail: (pathname) => { + refused.add(pathname) + }, + repair: () => refused.clear(), close: async () => { for (const socket of piped) socket.destroy() piped.clear() diff --git a/apps/conciv/vitest.config.ts b/apps/conciv/vitest.config.ts index 3f038eb4e..e3ba3b903 100644 --- a/apps/conciv/vitest.config.ts +++ b/apps/conciv/vitest.config.ts @@ -1,15 +1,10 @@ -import type {Plugin} from 'vite' import {defineConfig} from 'vitest/config' import {BaseSequencer, type TestSpecification} from 'vitest/node' -import {serveRpcRouter} from '@conciv/harness-testkit/rpc-mounts' -import {makeFakeCoreRouter} from './test/helpers/fake-core-router.js' import {coreCommands} from './test/commands/core-control.js' import {playwright} from '@vitest/browser-playwright' import solidPlugin from 'vite-plugin-solid' import {browserOptimizeDeps, ciTest, ciTestSolidBrowser} from '@conciv/vitest-config' -const FAKE_CORE_ADDRESS_PATH = '/__fake-core' - const SESSION_KILLER_SUITE = 'router-restore.browser.test.ts' class RunSessionKillerLastSequencer extends BaseSequencer { @@ -21,21 +16,6 @@ class RunSessionKillerLastSequencer extends BaseSequencer { } } -const fakeCoreSocket: Plugin = { - name: 'fake-core-socket', - async configureServer(server) { - const {router} = makeFakeCoreRouter() - const served = await serveRpcRouter({router}) - server.middlewares.use((req, res, next) => { - if (req.url !== FAKE_CORE_ADDRESS_PATH) return next() - res.setHeader('content-type', 'application/json') - res.end(JSON.stringify({base: served.base, wsUrl: served.wsUrl})) - }) - served.unref() - server.httpServer?.on('close', () => void served.close()) - }, -} - export default defineConfig({ test: { ...ciTest(), @@ -52,7 +32,7 @@ export default defineConfig({ }, }, { - plugins: [solidPlugin(), fakeCoreSocket], + plugins: [solidPlugin()], optimizeDeps: browserOptimizeDeps(), test: { ...ciTestSolidBrowser(), diff --git a/packages/client/test/helpers/boot.ts b/packages/client/test/helpers/boot.ts index cfb5149b7..2da4bda77 100644 --- a/packages/client/test/helpers/boot.ts +++ b/packages/client/test/helpers/boot.ts @@ -1,10 +1,4 @@ -import { - createFakeHarness, - createRecordingTerminalOpener, - createTestkit, - type FakeHarness, - type Kit, -} from '@conciv/harness-testkit' +import {createFakeHarness, createTestkit, type FakeHarness, type Kit} from '@conciv/harness-testkit' import type {AnyExtension} from '@conciv/extension' import {makeApp} from '@conciv/core/app' @@ -26,7 +20,6 @@ export async function bootClientKit(opts: {extensions?: AnyExtension[]} = {}): P }, cwd: env.cwd, openInEditor: () => {}, - openTerminal: createRecordingTerminalOpener().open, harness: env.harness, extensions: opts.extensions, }) diff --git a/packages/embed/tests/e2e/model-selector.it.test.ts b/packages/embed/tests/e2e/model-selector.it.test.ts index fcee71003..7cd1be3e2 100644 --- a/packages/embed/tests/e2e/model-selector.it.test.ts +++ b/packages/embed/tests/e2e/model-selector.it.test.ts @@ -15,7 +15,7 @@ test.describe('model selector error path', () => { test('offers a retry when meta.models fails and recovers on retry', async ({page}) => { test.setTimeout(180_000) - const models = await failRpcCalls(page, {path: ['meta', 'models']}) + const models = await failRpcCalls(page, {path: ['meta', 'models'], websocket: true}) await page.goto(suite.host().base, {waitUntil: 'domcontentloaded'}) await openPanel(page) diff --git a/packages/embed/tests/e2e/rpc-fault.it.test.ts b/packages/embed/tests/e2e/rpc-fault.it.test.ts index 060bf7784..e7b0faf17 100644 --- a/packages/embed/tests/e2e/rpc-fault.it.test.ts +++ b/packages/embed/tests/e2e/rpc-fault.it.test.ts @@ -6,7 +6,7 @@ const suite = setupWsProbeSuite() test.describe('rpc fault injection reaches calls that ride the websocket', () => { test('fails only the targeted procedure and lets it recover after repair', async ({page}) => { - const models = await failRpcCalls(page, {path: ['meta', 'models']}) + const models = await failRpcCalls(page, {path: ['meta', 'models'], websocket: true}) await page.goto(suite.host().base, {waitUntil: 'domcontentloaded'}) await page.evaluate((wsUrl) => window.__CONCIV_WS_PROBE__.connect(wsUrl), suite.socketUrl()) diff --git a/packages/extension-testkit/src/rpc-fault.ts b/packages/extension-testkit/src/rpc-fault.ts index 2e613b64d..79b8dd9e1 100644 --- a/packages/extension-testkit/src/rpc-fault.ts +++ b/packages/extension-testkit/src/rpc-fault.ts @@ -24,7 +24,7 @@ function isPreflight(route: Route): boolean { export async function failRpcCalls( page: Page, - options: {path: readonly string[]; status?: number}, + options: {path: readonly string[]; status?: number; websocket?: boolean}, ): Promise { const status = options.status ?? 500 const broken = {value: true} @@ -58,7 +58,7 @@ export async function failRpcCalls( server.onMessage((message) => socket.send(message)) } - await page.routeWebSocket((url) => url.pathname.endsWith('/rpc-ws'), holdSocket) + if (options.websocket) await page.routeWebSocket((url) => url.pathname.endsWith('/rpc-ws'), holdSocket) return { repair: () => { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 250043693..ea676707d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -242,9 +242,6 @@ importers: '@conciv/vitest-config': specifier: workspace:* version: link:../../packages/vitest-config - '@orpc/server': - specifier: 'catalog:' - version: 1.14.7(@opentelemetry/api@1.9.1)(crossws@0.4.10(srvx@0.11.22))(ws@8.21.0) '@solidjs/testing-library': specifier: ^0.8.10 version: 0.8.10(@solidjs/router@0.15.4(solid-js@1.9.14))(solid-js@1.9.14) From 271dd746a3eb1c30da66f6f414904d456c462a48 Mon Sep 17 00:00:00 2001 From: Omri Katz <9701896+omridevk@users.noreply.github.com> Date: Sat, 15 Aug 2026 06:24:16 +0300 Subject: [PATCH 5/9] test(harness-testkit): #354 drop the orphaned recording terminal opener createRecordingTerminalOpener lost its last consumer when the client boot helper stopped passing a dead openTerminal option to makeApp. Both testkits are private, so the export is not public API: fallow traced it to a single barrel re-export with zero references. The file carried nothing else. Co-Authored-By: Claude Fable 5 --- packages/harness-testkit/src/terminal-opener.ts | 14 -------------- packages/harness-testkit/src/testkit.ts | 1 - 2 files changed, 15 deletions(-) delete mode 100644 packages/harness-testkit/src/terminal-opener.ts diff --git a/packages/harness-testkit/src/terminal-opener.ts b/packages/harness-testkit/src/terminal-opener.ts deleted file mode 100644 index 35526c728..000000000 --- a/packages/harness-testkit/src/terminal-opener.ts +++ /dev/null @@ -1,14 +0,0 @@ -import type {TerminalOpener, TerminalOpenRequest} from '@conciv/protocol/harness-types' - -export type RecordingTerminalOpener = {open: TerminalOpener; opened: TerminalOpenRequest[]} - -export function createRecordingTerminalOpener(): RecordingTerminalOpener { - const opened: TerminalOpenRequest[] = [] - return { - opened, - open: (request) => { - opened.push({bin: request.bin, args: [...request.args]}) - return Promise.resolve(true) - }, - } -} diff --git a/packages/harness-testkit/src/testkit.ts b/packages/harness-testkit/src/testkit.ts index 43481cc8a..9edd5aad9 100644 --- a/packages/harness-testkit/src/testkit.ts +++ b/packages/harness-testkit/src/testkit.ts @@ -26,6 +26,5 @@ export { export {approvalIds} from './run-events.js' export {makeRpcClient, resolveSession, type RpcClient} from './session.js' export {harnessModes, type HarnessMode} from './harness-modes.js' -export {createRecordingTerminalOpener, type RecordingTerminalOpener} from './terminal-opener.js' export type {RunStream} from './run-stream.js' export type {RunEvents, SeenToolCall} from './run-events.js' From 554bb51e93641633466f4fce32847f3b424dc655 Mon Sep 17 00:00:00 2001 From: Omri Katz <9701896+omridevk@users.noreply.github.com> Date: Sat, 15 Aug 2026 06:47:22 +0300 Subject: [PATCH 6/9] test: review fixes for #354 PR Co-Authored-By: Claude Fable 5 --- apps/conciv/test/chat-pane.browser.test.tsx | 23 ++++++++----------- apps/conciv/test/helpers/tracked-faults.ts | 22 ++++++++++++++++++ .../test/launch-actions.browser.test.tsx | 4 +++- .../test/panel-connect.browser.test.tsx | 6 +++-- .../panel-focus-stability.browser.test.tsx | 4 +++- .../test/quick-add-pane.browser.test.tsx | 8 ++++--- .../test/reachability-flows.browser.test.tsx | 14 ++++++----- .../test/route-boundary.browser.test.tsx | 8 ++++--- packages/harness-testkit/src/scripted-run.ts | 21 +++++++---------- .../harness-testkit/test/scripted-run.test.ts | 17 ++++++++++++++ 10 files changed, 84 insertions(+), 43 deletions(-) create mode 100644 apps/conciv/test/helpers/tracked-faults.ts diff --git a/apps/conciv/test/chat-pane.browser.test.tsx b/apps/conciv/test/chat-pane.browser.test.tsx index bb7c14d62..bb9ee5b32 100644 --- a/apps/conciv/test/chat-pane.browser.test.tsx +++ b/apps/conciv/test/chat-pane.browser.test.tsx @@ -6,12 +6,14 @@ import {ChatPane} from '../src/pane/chat-pane.js' import {coreControl} from './helpers/core-control.js' import {coreRpc, createSession, openTranscriptStream, runTurn, seedDraft, sendTurn} from './helpers/core-session.js' import {mountPane, type PaneMount} from './helpers/pane-harness.js' +import {trackedFaults} from './helpers/tracked-faults.js' const SEND_PATH = ['chat', 'send'] const SUBSCRIBE_PATH = ['chat', 'subscribe'] const core = {base: ''} -const active: {pane: PaneMount | null; faults: string[]} = {pane: null, faults: []} +const active: {pane: PaneMount | null} = {pane: null} +const faults = trackedFaults() beforeAll(async () => { const booted = await coreControl.bootCore({id: 'chat-pane', allowedOrigins: [window.location.origin]}) @@ -23,18 +25,11 @@ afterAll(async () => { }, 30_000) afterEach(async () => { - for (const fault of active.faults.splice(0)) await coreControl.releaseFault(fault) await coreControl.releaseTurn() active.pane?.dispose() active.pane = null }) -async function installFault(spec: Parameters[0]): Promise { - const handle = await coreControl.installFault(spec) - active.faults.push(handle) - return handle -} - async function newSession(): Promise<{rpc: RpcClient; sessionId: string}> { const rpc = coreRpc(core.base) return {rpc, sessionId: await createSession(rpc)} @@ -82,7 +77,7 @@ test('restores the server-side draft text and staged grabs when the pane mounts' test('a rejected send keeps the draft in the composer and tells the user why', async () => { const {sessionId} = await newSession() - await installFault({kind: 'fail', path: SEND_PATH, status: 500}) + await faults.install({kind: 'fail', path: SEND_PATH, status: 500}) mountChatPane(sessionId) await expect.element(input()).toBeVisible() @@ -102,7 +97,7 @@ test('sending drops the staged grab card at once, while the turn is still stream }) test('a send the server refuses puts the staged grab card back', async () => { - await installFault({kind: 'fail', path: SEND_PATH, status: 500}) + await faults.install({kind: 'fail', path: SEND_PATH, status: 500}) await sendWithStagedGrab() await expect.element(notifications()).toHaveTextContent(/Internal Server Error|could not be sent/) @@ -111,7 +106,7 @@ test('a send the server refuses puts the staged grab card back', async () => { }) test('a send that throws at the transport puts the staged grab card back', async () => { - await installFault({kind: 'abort', path: SEND_PATH}) + await faults.install({kind: 'abort', path: SEND_PATH}) await sendWithStagedGrab() await expect.element(notifications()).toHaveTextContent(/could not be sent|fetch/) @@ -162,7 +157,7 @@ test('the refresh affordance re-subscribes and shows the transcript the server r await expect.element(page.getByText('How can I help you today?')).toBeVisible() const stream = await openTranscriptStream(rpc, sessionId) - const gate = await installFault({kind: 'gate', path: SUBSCRIBE_PATH}) + const gate = await faults.install({kind: 'gate', path: SUBSCRIBE_PATH}) await page.getByRole('button', {name: 'Refresh the conversation'}).click() await coreControl.scriptTurn({toolCalls: [], text: 'the refreshed transcript'}) @@ -189,7 +184,7 @@ test('the refresh affordance is disabled while the run streams', async () => { test('the initial load shows a conversation skeleton until the snapshot arrives', async () => { const {sessionId} = await newSession() - const gate = await installFault({kind: 'gate', path: SUBSCRIBE_PATH}) + const gate = await faults.install({kind: 'gate', path: SUBSCRIBE_PATH}) mountChatPane(sessionId) await expect.element(skeleton()).toBeVisible() @@ -286,7 +281,7 @@ test('a new-session divider does not flash before the transcript snapshot hydrat const {rpc, sessionId} = await newSession() await coreControl.scriptTurn({toolCalls: [], text: 'starting a fresh session'}) await runTurn(rpc, sessionId, 'restart with a clean slate') - const gate = await installFault({kind: 'gate', path: SUBSCRIBE_PATH}) + const gate = await faults.install({kind: 'gate', path: SUBSCRIBE_PATH}) const mount = mountChatPane(sessionId) await expect.element(skeleton()).toBeVisible() diff --git a/apps/conciv/test/helpers/tracked-faults.ts b/apps/conciv/test/helpers/tracked-faults.ts new file mode 100644 index 000000000..0dc872c0c --- /dev/null +++ b/apps/conciv/test/helpers/tracked-faults.ts @@ -0,0 +1,22 @@ +import {afterEach} from 'vitest' +import type {FaultSpec} from '../commands/core-control.js' +import {coreControl} from './core-control.js' + +export type TrackedFaults = { + install: (spec: FaultSpec) => Promise + releaseAll: () => Promise +} + +export function trackedFaults(): TrackedFaults { + const handles: string[] = [] + const install = async (spec: FaultSpec): Promise => { + const handle = await coreControl.installFault(spec) + handles.push(handle) + return handle + } + const releaseAll = async (): Promise => { + for (const handle of handles.splice(0)) await coreControl.releaseFault(handle) + } + afterEach(releaseAll) + return {install, releaseAll} +} diff --git a/apps/conciv/test/launch-actions.browser.test.tsx b/apps/conciv/test/launch-actions.browser.test.tsx index 80d852a7f..6e9141a9d 100644 --- a/apps/conciv/test/launch-actions.browser.test.tsx +++ b/apps/conciv/test/launch-actions.browser.test.tsx @@ -6,12 +6,14 @@ import {ComposerActions} from '../src/composer/actions.js' import {coreControl} from './helpers/core-control.js' import {coreRpc, createSession} from './helpers/core-session.js' import {mountPane, type PaneMount} from './helpers/pane-harness.js' +import {trackedFaults} from './helpers/tracked-faults.js' const CONNECT_COMMAND_PATH = ['ext', 'terminal', 'connectCommand'] const LAUNCH_PATH = ['ext', 'terminal', 'launch'] const core = {base: ''} const mounted: {pane: PaneMount | null} = {pane: null} +const faults = trackedFaults() const grabApi: GrabApi = { pick: async () => null, @@ -109,7 +111,7 @@ test('copy command asks the terminal extension for the command', async () => { }) test('a launch failure is surfaced instead of swallowed', async () => { - const fault = await coreControl.installFault({kind: 'fail', path: LAUNCH_PATH, status: 500}) + const fault = await faults.install({kind: 'fail', path: LAUNCH_PATH, status: 500}) await mountActions() await openMenu() diff --git a/apps/conciv/test/panel-connect.browser.test.tsx b/apps/conciv/test/panel-connect.browser.test.tsx index 3b0b3776b..99528d46e 100644 --- a/apps/conciv/test/panel-connect.browser.test.tsx +++ b/apps/conciv/test/panel-connect.browser.test.tsx @@ -5,6 +5,7 @@ import {defineExtension, getExtensionApi, type RegisterExtension} from '@conciv/ import {Show, type JSX} from 'solid-js' import {coreControl} from './helpers/core-control.js' import {createShellHarness} from './helpers/shell-harness.js' +import {trackedFaults} from './helpers/tracked-faults.js' import {expectRetryRecovers} from './helpers/retry-recovery.js' const FOUND_API_BASE = 'http://found.test' @@ -13,6 +14,7 @@ const RESOLVE_PATH = ['sessions', 'resolve'] const core = {base: ''} const harness = createShellHarness(() => core.base) +const faults = trackedFaults() beforeAll(async () => { const booted = await coreControl.bootCore({id: 'panel-connect', allowedOrigins: [window.location.origin]}) @@ -55,7 +57,7 @@ const serverError = () => page.getByText('Internal Server Error') const editor = () => page.getByRole('textbox', {name: 'Message the conciv agent'}) test('a bind that fails with a transport failure shows the connect failure screen, and retrying against a healthy engine hands off to the panel', async () => { - const unreachable = await coreControl.installFault({kind: 'abort', path: RESOLVE_PATH}) + const unreachable = await faults.install({kind: 'abort', path: RESOLVE_PATH}) mountShell() await simulateFound().click() @@ -67,7 +69,7 @@ test('a bind that fails with a transport failure shows the connect failure scree }, 30_000) test('a bind that fails with a server error shows the actual error, not the workspace-unreachable message', async () => { - const refused = await coreControl.installFault({kind: 'fail', path: RESOLVE_PATH, status: 500}) + const refused = await faults.install({kind: 'fail', path: RESOLVE_PATH, status: 500}) mountShell() await expect.element(simulateFound(), {timeout: 8000}).toBeVisible() diff --git a/apps/conciv/test/panel-focus-stability.browser.test.tsx b/apps/conciv/test/panel-focus-stability.browser.test.tsx index 334ea2676..b551f8685 100644 --- a/apps/conciv/test/panel-focus-stability.browser.test.tsx +++ b/apps/conciv/test/panel-focus-stability.browser.test.tsx @@ -4,11 +4,13 @@ import {page} from 'vitest/browser' import {coreControl} from './helpers/core-control.js' import {coreRpc, createSession} from './helpers/core-session.js' import {createShellHarness} from './helpers/shell-harness.js' +import {trackedFaults} from './helpers/tracked-faults.js' const SETTLED = {timeout: 1500} const core = {base: ''} const harness = createShellHarness(() => core.base) +const faults = trackedFaults() beforeAll(async () => { const booted = await coreControl.bootCore({id: 'panel-focus-stability', allowedOrigins: [window.location.origin]}) @@ -30,7 +32,7 @@ async function openPanel(): Promise { async function expectFocusSurvives(path: string[]): Promise { const since = await coreControl.rpcMark() - const held = await coreControl.installFault({kind: 'gate', path}) + const held = await faults.install({kind: 'gate', path}) await openPanel() await vi.waitUntil(async () => (await coreControl.faultPending(held)) > 0, {timeout: 5000, interval: 20}) diff --git a/apps/conciv/test/quick-add-pane.browser.test.tsx b/apps/conciv/test/quick-add-pane.browser.test.tsx index 55d342252..209798f29 100644 --- a/apps/conciv/test/quick-add-pane.browser.test.tsx +++ b/apps/conciv/test/quick-add-pane.browser.test.tsx @@ -4,12 +4,14 @@ import {page} from 'vitest/browser' import {coreControl} from './helpers/core-control.js' import {coreRpc, createSession} from './helpers/core-session.js' import {createShellHarness} from './helpers/shell-harness.js' +import {trackedFaults} from './helpers/tracked-faults.js' import {expectRetryRecovers} from './helpers/retry-recovery.js' const RESOLVE_PATH = ['sessions', 'resolve'] const core = {base: ''} const harness = createShellHarness(() => core.base) +const faults = trackedFaults() beforeAll(async () => { const booted = await coreControl.bootCore({id: 'quick-add-pane', allowedOrigins: [window.location.origin]}) @@ -27,7 +29,7 @@ const editor = () => page.getByRole('textbox', {name: 'Message the conciv agent' const closePane = () => page.getByRole('button', {name: 'Close pane'}) test('a quick terminal pane that fails to start shows a retry action', async () => { - const refused = await coreControl.installFault({kind: 'fail', path: RESOLVE_PATH, status: 500}) + const refused = await faults.install({kind: 'fail', path: RESOLVE_PATH, status: 500}) harness.mountShell('/quick') await expect.element(startFailure(), {timeout: 8000}).toBeVisible() @@ -37,7 +39,7 @@ test('a quick terminal pane that fails to start shows a retry action', async () }, 30_000) test('retrying a failed quick terminal pane against a healthy engine starts it', async () => { - const refused = await coreControl.installFault({kind: 'fail', path: RESOLVE_PATH, status: 500}) + const refused = await faults.install({kind: 'fail', path: RESOLVE_PATH, status: 500}) harness.mountShell('/quick') await expect.element(startFailure(), {timeout: 8000}).toBeVisible() @@ -49,7 +51,7 @@ test('rapid double-trigger creates exactly one pane', async () => { harness.mountShell(`/quick?panes=${sessionId}&focus=0`) await expect.element(editor(), {timeout: 8000}).toBeVisible() - const held = await coreControl.installFault({kind: 'gate', path: RESOLVE_PATH}) + const held = await faults.install({kind: 'gate', path: RESOLVE_PATH}) const splitButton = page.getByRole('button', {name: 'Split pane (Mod+D)'}) const firstClick = splitButton.click() const secondClick = splitButton.click() diff --git a/apps/conciv/test/reachability-flows.browser.test.tsx b/apps/conciv/test/reachability-flows.browser.test.tsx index e04e885db..734bdf4e5 100644 --- a/apps/conciv/test/reachability-flows.browser.test.tsx +++ b/apps/conciv/test/reachability-flows.browser.test.tsx @@ -4,10 +4,12 @@ import {page, userEvent} from 'vitest/browser' import {coreControl} from './helpers/core-control.js' import {coreRpc, createSession} from './helpers/core-session.js' import {createShellHarness} from './helpers/shell-harness.js' +import {trackedFaults} from './helpers/tracked-faults.js' import {expectRetryRecovers} from './helpers/retry-recovery.js' const core = {base: ''} const harness = createShellHarness(() => core.base) +const faults = trackedFaults() beforeAll(async () => { const booted = await coreControl.bootCore({id: 'reachability-flows', allowedOrigins: [window.location.origin]}) @@ -33,7 +35,7 @@ async function openSessionPanel(): Promise { } test('a dead engine at boot shows our error screen, not the generic boundary, and Retry recovers it', async () => { - const outage = await coreControl.installFault({kind: 'abort'}) + const outage = await faults.install({kind: 'abort'}) harness.mountShell('/panel/latest?open=true') await expect.element(errorScreen(), {timeout: 8000}).toBeVisible() @@ -50,7 +52,7 @@ test('a dead engine at boot shows our error screen, not the generic boundary, an }, 30_000) test('a server error while resolving /panel/latest shows the actual error, not the unreachable message', async () => { - const refused = await coreControl.installFault({kind: 'fail', path: ['sessions', 'resolve'], status: 500}) + const refused = await faults.install({kind: 'fail', path: ['sessions', 'resolve'], status: 500}) harness.mountShell('/panel/latest?open=true') await expect.element(serverError(), {timeout: 8000}).toBeVisible() @@ -70,7 +72,7 @@ test('a healthy engine resolves /panel/latest straight to the warm session', asy test('a sustained outage raises exactly one standing notice, and it clears once the engine returns', async () => { await openSessionPanel() - const outage = await coreControl.installFault({kind: 'abort'}) + const outage = await faults.install({kind: 'abort'}) await editor().fill('rename the widget package') await expect.element(engineUnreachableNotice(), {timeout: 8000}).toBeVisible() @@ -81,7 +83,7 @@ test('a sustained outage raises exactly one standing notice, and it clears once }, 30_000) test('a 500 from an otherwise healthy engine never raises the unreachable notice', async () => { - const refused = await coreControl.installFault({kind: 'fail', path: ['chat', 'send'], status: 500}) + const refused = await faults.install({kind: 'fail', path: ['chat', 'send'], status: 500}) await openSessionPanel() await editor().fill('rename the widget package') @@ -96,7 +98,7 @@ test('a 500 from an otherwise healthy engine never raises the unreachable notice }, 30_000) test('a failing engine-info probe on an otherwise healthy connection never raises the unreachable notice', async () => { - const refused = await coreControl.installFault({kind: 'fail', path: ['meta', 'engine'], status: 500}) + const refused = await faults.install({kind: 'fail', path: ['meta', 'engine'], status: 500}) const since = await coreControl.rpcMark() await openSessionPanel() @@ -109,7 +111,7 @@ test('a failing engine-info probe on an otherwise healthy connection never raise test('the composer disables sending with a distinct message once the engine is unreachable', async () => { await openSessionPanel() - const outage = await coreControl.installFault({kind: 'abort'}) + const outage = await faults.install({kind: 'abort'}) await editor().fill('rename the widget package') await expect diff --git a/apps/conciv/test/route-boundary.browser.test.tsx b/apps/conciv/test/route-boundary.browser.test.tsx index 5cc369b78..75ee54e21 100644 --- a/apps/conciv/test/route-boundary.browser.test.tsx +++ b/apps/conciv/test/route-boundary.browser.test.tsx @@ -4,11 +4,13 @@ import {page} from 'vitest/browser' import {coreControl} from './helpers/core-control.js' import {coreRpc, createSession} from './helpers/core-session.js' import {createShellHarness} from './helpers/shell-harness.js' +import {trackedFaults} from './helpers/tracked-faults.js' const WHILE_HELD = {timeout: 700} const core = {base: ''} const harness = createShellHarness(() => core.base) +const faults = trackedFaults() beforeAll(async () => { const booted = await coreControl.bootCore({id: 'route-boundary', allowedOrigins: [window.location.origin]}) @@ -31,7 +33,7 @@ async function openPanel(): Promise { } test('the pane paints its composer while the element captures query is still in flight', async () => { - const held = await coreControl.installFault({kind: 'gate', path: ['captures', 'list']}) + const held = await faults.install({kind: 'gate', path: ['captures', 'list']}) await openPanel() await expect.element(editor(), WHILE_HELD).toBeVisible() @@ -41,7 +43,7 @@ test('the pane paints its composer while the element captures query is still in }, 30_000) test('the shell keeps its launcher while the session list query is still in flight', async () => { - const held = await coreControl.installFault({kind: 'gate', path: ['sessions', 'list']}) + const held = await faults.install({kind: 'gate', path: ['sessions', 'list']}) harness.mountShell('/') await expect.element(launcher(), WHILE_HELD).toBeVisible() @@ -58,7 +60,7 @@ test('a pane whose queries all answer immediately never shows the route pending }, 30_000) test('a slow beforeLoad reveals the route pending loader, then hands off to the pane', async () => { - const held = await coreControl.installFault({kind: 'gate', path: ['sessions', 'resolve']}) + const held = await faults.install({kind: 'gate', path: ['sessions', 'resolve']}) harness.mountShell('/panel/latest?open=true') await expect.element(routePending()).toBeVisible() diff --git a/packages/harness-testkit/src/scripted-run.ts b/packages/harness-testkit/src/scripted-run.ts index 811d11ae8..e7899b23a 100644 --- a/packages/harness-testkit/src/scripted-run.ts +++ b/packages/harness-testkit/src/scripted-run.ts @@ -17,7 +17,7 @@ export type ScriptedRun = { type QueuedToolCall = {id: string; name: string; input: unknown; result: unknown} -type QueuedTurn = {toolCalls: QueuedToolCall[]; text?: string} +type QueuedTurn = {toolCalls: QueuedToolCall[]; text?: string; blocking: boolean} const THREAD = {threadId: 'scripted', runId: 'scripted'} as const @@ -41,7 +41,7 @@ function resultChunk(toolCallId: string, result: unknown): StreamChunk { function* turnChunks(turn: QueuedTurn): Generator { for (const call of turn.toolCalls) { yield* requestChunks(call) - yield resultChunk(call.id, call.result) + if (!turn.blocking) yield resultChunk(call.id, call.result) } } @@ -59,7 +59,6 @@ export function makeScriptedRun(opts: {text?: string} = {}): ScriptedRun { const gate = {held: false, waiting: new Set<() => void>()} const turns = {count: 0} const toolCalls = {count: 0} - const queuedToolCalls: Array<{id: string; name: string; input: unknown; blocking: boolean}> = [] const queuedTurns: QueuedTurn[] = [] const queuedCustomEvents: Array<{name: string; value: unknown}> = [] const queuedErrors: string[] = [] @@ -75,7 +74,8 @@ export function makeScriptedRun(opts: {text?: string} = {}): ScriptedRun { const scriptToolCall = (name: string, input: unknown, toolOpts: {blocking?: boolean} = {}) => { toolCalls.count += 1 const toolCallId = `tc-${toolCalls.count}` - queuedToolCalls.push({id: toolCallId, name, input, blocking: toolOpts.blocking ?? true}) + const blocking = toolOpts.blocking ?? true + queuedTurns.push({toolCalls: [{id: toolCallId, name, input, result: {ok: true}}], blocking}) return toolCallId } const scriptTurn = (turn: ScriptedTurn) => { @@ -83,7 +83,7 @@ export function makeScriptedRun(opts: {text?: string} = {}): ScriptedRun { toolCalls.count += 1 return {id: `tc-${toolCalls.count}`, name: call.name, input: call.input, result: call.result ?? {ok: true}} }) - queuedTurns.push({toolCalls: calls, text: turn.text}) + queuedTurns.push({toolCalls: calls, text: turn.text, blocking: false}) return calls.map((call) => call.id) } const scriptCustomEvent = (name: string, value: unknown) => { @@ -99,14 +99,9 @@ export function makeScriptedRun(opts: {text?: string} = {}): ScriptedRun { yield sessionIdChunk(deps) const scriptedTurn = queuedTurns.shift() if (scriptedTurn) yield* turnChunks(scriptedTurn) - const toolCall = scriptedTurn ? undefined : queuedToolCalls.shift() - if (toolCall) { - yield* requestChunks(toolCall) - if (toolCall.blocking) { - yield {type: EventType.RUN_FINISHED, ...THREAD, finishReason: 'tool_calls'} - return - } - yield resultChunk(toolCall.id, {ok: true}) + if (scriptedTurn?.blocking) { + yield {type: EventType.RUN_FINISHED, ...THREAD, finishReason: 'tool_calls'} + return } yield* customEventChunks(queuedCustomEvents.splice(0)) yield {type: EventType.TEXT_MESSAGE_CONTENT, messageId, delta: scriptedTurn?.text ?? defaultText} diff --git a/packages/harness-testkit/test/scripted-run.test.ts b/packages/harness-testkit/test/scripted-run.test.ts index 7cb29ff36..41ef74fa9 100644 --- a/packages/harness-testkit/test/scripted-run.test.ts +++ b/packages/harness-testkit/test/scripted-run.test.ts @@ -83,6 +83,23 @@ describe('makeScriptedRun', () => { expect(chunks.at(-1)?.type).toBe(EventType.RUN_FINISHED) }) + it('streams a queued tool call and a queued turn in the order they were scripted', async () => { + const scripted = makeScriptedRun() + scripted.scriptToolCall('first_tool', {a: 1}) + scripted.scriptTurn({toolCalls: [{name: 'second_tool', input: {b: 2}}], text: 'turn ran'}) + const drainTurn = async (): Promise => { + const chunks: StreamChunk[] = [] + for await (const chunk of scripted.chatStream(deps())) chunks.push(chunk) + return chunks + } + const first = await drainTurn() + const second = await drainTurn() + const startsIn = (chunks: StreamChunk[]): string[] => + chunks.flatMap((chunk) => (chunk.type === EventType.TOOL_CALL_START ? [chunk.toolCallName] : [])) + expect(startsIn(first)).toEqual(['first_tool']) + expect(startsIn(second)).toEqual(['second_tool']) + }) + it('holds the turn open until release()', async () => { const scripted = makeScriptedRun() scripted.hold() From bd129886c1acc45023d35be9ee3b61171cc5d62f Mon Sep 17 00:00:00 2001 From: Omri Katz <9701896+omridevk@users.noreply.github.com> Date: Sat, 15 Aug 2026 06:50:01 +0300 Subject: [PATCH 7/9] test: fold core-request-gate faults into the trackedFaults helper Co-Authored-By: Claude Fable 5 --- apps/conciv/test/core-request-gate.browser.test.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/conciv/test/core-request-gate.browser.test.ts b/apps/conciv/test/core-request-gate.browser.test.ts index e95f02239..190dfa95c 100644 --- a/apps/conciv/test/core-request-gate.browser.test.ts +++ b/apps/conciv/test/core-request-gate.browser.test.ts @@ -2,8 +2,10 @@ import {afterAll, beforeAll, describe, expect, it, vi} from 'vitest' import {EventType} from '@tanstack/ai' import {makeRpcClient, type RpcClient} from '@conciv/contract' import {coreControl} from './helpers/core-control.js' +import {trackedFaults} from './helpers/tracked-faults.js' const core: {rpc: RpcClient | null} = {rpc: null} +const faults = trackedFaults() function statusOf(value: unknown): number | null { if (typeof value !== 'object' || value === null || !('status' in value)) return null @@ -33,7 +35,7 @@ afterAll(async () => { describe('gated rpc requests against a real core', () => { it('holds the subscribe request until release, then streams the real snapshot', async () => { const {sessionId} = await rpc().sessions.create() - const gate = await coreControl.installFault({kind: 'gate', path: ['chat', 'subscribe']}) + const gate = await faults.install({kind: 'gate', path: ['chat', 'subscribe']}) const arrived = {value: false} const subscription = rpc() .chat.subscribe({sessionId}) @@ -57,7 +59,7 @@ describe('gated rpc requests against a real core', () => { it('keeps a cross-origin injected failure an rpc rejection rather than a transport error', async () => { const {sessionId} = await rpc().sessions.create() - const fault = await coreControl.installFault({kind: 'fail', path: ['sessions', 'rename'], status: 500}) + const fault = await faults.install({kind: 'fail', path: ['sessions', 'rename'], status: 500}) const rejection = await rpc() .sessions.rename({sessionId, title: 'renamed by the fault test'}) From 744c5ec57d29bad0d51261d43a011e8a2de6ab99 Mon Sep 17 00:00:00 2001 From: Omri Katz <9701896+omridevk@users.noreply.github.com> Date: Sat, 15 Aug 2026 11:43:03 +0300 Subject: [PATCH 8/9] test: #354 replace poll-based test waits with event-driven signals Copilot review findings on #508. The draft-storage suite polled `rpc.drafts.get` to wait out the 300ms debounced fire-and-forget write. The proxy the test already routes through knows exactly when a write completed, so it now exposes `mark()`/`awaitRequest(pathname)` resolved from its own response-finish event; every `expect.poll` becomes one armed wait plus one plain assertion. No product signal was added. `gateRpcCalls` gained `awaitCaptured(count)`, fulfilled at the capture site, threaded to the browser through a new `awaitFaultPending` command that keeps AWAIT_RPC_TIMEOUT_MS purely as a failure deadline. That retires the `vi.waitUntil(faultPending)` polls; `faultPending` stays for the direct-fact assertions in core-request-gate. Co-Authored-By: Claude Fable 5 --- apps/conciv/test/commands/core-control.ts | 31 ++++- .../test/core-request-gate.browser.test.ts | 4 +- apps/conciv/test/draft-storage.test.ts | 109 +++++++++--------- apps/conciv/test/helpers/core-control.ts | 1 + apps/conciv/test/helpers/proxy.ts | 50 ++++++++ .../panel-focus-stability.browser.test.tsx | 4 +- .../test/quick-add-pane.browser.test.tsx | 4 +- packages/extension-testkit/src/rpc-fault.ts | 15 +++ 8 files changed, 156 insertions(+), 62 deletions(-) diff --git a/apps/conciv/test/commands/core-control.ts b/apps/conciv/test/commands/core-control.ts index 94aad44fd..677a1b716 100644 --- a/apps/conciv/test/commands/core-control.ts +++ b/apps/conciv/test/commands/core-control.ts @@ -24,7 +24,12 @@ export type FaultSpec = | {kind: 'abort'; path?: string[]} | {kind: 'gate'; path?: string[]} -type Fault = {pending: () => number; release: () => Promise; dispose: () => Promise} +type Fault = { + pending: () => number + awaitCaptured: (count: number) => Promise + release: () => Promise + dispose: () => Promise +} type CoreTestkit = typeof import('./core-testkit.js') @@ -189,6 +194,9 @@ const installFault: BrowserCommand<[FaultSpec]> = async (ctx, spec): Promise 0, + awaitCaptured: async () => { + throw new Error(`the fault "${handle}" is a ${spec.kind} fault, which never captures pending requests`) + }, release: () => { injector.repair() return Promise.resolve() @@ -204,6 +212,26 @@ const releaseFault: BrowserCommand<[string]> = async (ctx, handle): Promise = (ctx, handle): number => faultOf(ctx, handle).pending() +const awaitFaultPending: BrowserCommand<[string, number]> = async (ctx, handle, count): Promise => { + const timer: {value: ReturnType | null} = {value: null} + const deadline = new Promise((_, reject) => { + timer.value = setTimeout( + () => + reject( + new Error( + `the fault "${handle}" captured ${faultOf(ctx, handle).pending()} of ${count} pending requests within ${AWAIT_RPC_TIMEOUT_MS}ms`, + ), + ), + AWAIT_RPC_TIMEOUT_MS, + ) + }) + try { + await Promise.race([faultOf(ctx, handle).awaitCaptured(count), deadline]) + } finally { + if (timer.value) clearTimeout(timer.value) + } +} + export const coreCommands = { bootCore, closeCore, @@ -220,4 +248,5 @@ export const coreCommands = { installFault, releaseFault, faultPending, + awaitFaultPending, } diff --git a/apps/conciv/test/core-request-gate.browser.test.ts b/apps/conciv/test/core-request-gate.browser.test.ts index 190dfa95c..6ab5f5fe2 100644 --- a/apps/conciv/test/core-request-gate.browser.test.ts +++ b/apps/conciv/test/core-request-gate.browser.test.ts @@ -1,4 +1,4 @@ -import {afterAll, beforeAll, describe, expect, it, vi} from 'vitest' +import {afterAll, beforeAll, describe, expect, it} from 'vitest' import {EventType} from '@tanstack/ai' import {makeRpcClient, type RpcClient} from '@conciv/contract' import {coreControl} from './helpers/core-control.js' @@ -44,7 +44,7 @@ describe('gated rpc requests against a real core', () => { return iterator }) - await vi.waitUntil(async () => (await coreControl.faultPending(gate)) === 1, {timeout: 5_000, interval: 20}) + await coreControl.awaitFaultPending(gate, 1) expect(arrived.value).toBe(false) await coreControl.releaseFault(gate) diff --git a/apps/conciv/test/draft-storage.test.ts b/apps/conciv/test/draft-storage.test.ts index faa6e9faf..64a3d3711 100644 --- a/apps/conciv/test/draft-storage.test.ts +++ b/apps/conciv/test/draft-storage.test.ts @@ -7,7 +7,6 @@ import {proxyTo, type ProxyCore} from './helpers/proxy.js' const DRAFTS_GET_PATH = '/rpc/drafts/get' const DRAFTS_SET_PATH = '/rpc/drafts/set' -const SETTLED = {timeout: 4000, interval: 25} type StoredDraft = {sessionId: string; text: string; selectionStart: number; selectionEnd: number; grabs: string[]} @@ -71,15 +70,15 @@ test('writes the composer draft back to the server with the caret at the end', a draftStorage.storage.setItem('any', composerDraft('a fresh draft', ['a heading'])) - await expect - .poll(() => storedDraft(kit.rpc, sessionId), SETTLED) - .toEqual({ - sessionId, - text: 'a fresh draft', - selectionStart: 13, - selectionEnd: 13, - grabs: ['a heading'], - }) + await core.awaitRequest(DRAFTS_SET_PATH) + + expect(await storedDraft(kit.rpc, sessionId)).toEqual({ + sessionId, + text: 'a fresh draft', + selectionStart: 13, + selectionEnd: 13, + grabs: ['a heading'], + }) }) test('collapses a burst of writes into the last draft', async ({kit, core, sessionId}) => { @@ -89,15 +88,15 @@ test('collapses a burst of writes into the last draft', async ({kit, core, sessi draftStorage.storage.setItem('any', composerDraft('ab')) draftStorage.storage.setItem('any', composerDraft('abc')) - await expect - .poll(() => storedDraft(kit.rpc, sessionId), SETTLED) - .toEqual({ - sessionId, - text: 'abc', - selectionStart: 3, - selectionEnd: 3, - grabs: [], - }) + await core.awaitRequest(DRAFTS_SET_PATH) + + expect(await storedDraft(kit.rpc, sessionId)).toEqual({ + sessionId, + text: 'abc', + selectionStart: 3, + selectionEnd: 3, + grabs: [], + }) expect(core.requestCount(DRAFTS_SET_PATH)).toBe(1) }) @@ -109,15 +108,15 @@ test('keeps the latest value readable even when the payload cannot be persisted' draftStorage.storage.setItem('any', composerDraft('a draft that parses')) - await expect - .poll(() => storedDraft(kit.rpc, sessionId), SETTLED) - .toEqual({ - sessionId, - text: 'a draft that parses', - selectionStart: 19, - selectionEnd: 19, - grabs: [], - }) + await core.awaitRequest(DRAFTS_SET_PATH) + + expect(await storedDraft(kit.rpc, sessionId)).toEqual({ + sessionId, + text: 'a draft that parses', + selectionStart: 19, + selectionEnd: 19, + grabs: [], + }) expect(core.requestCount(DRAFTS_SET_PATH)).toBe(1) }) @@ -127,15 +126,15 @@ test('persists the noted caret offsets with the draft text', async ({kit, core, draftStorage.storage.setItem('any', composerDraft('say hello!')) draftStorage.noteSelection({start: 4, end: 4}) - await expect - .poll(() => storedDraft(kit.rpc, sessionId), SETTLED) - .toEqual({ - sessionId, - text: 'say hello!', - selectionStart: 4, - selectionEnd: 4, - grabs: [], - }) + await core.awaitRequest(DRAFTS_SET_PATH) + + expect(await storedDraft(kit.rpc, sessionId)).toEqual({ + sessionId, + text: 'say hello!', + selectionStart: 4, + selectionEnd: 4, + grabs: [], + }) }) test('clamps noted offsets that fall beyond the persisted text', async ({kit, core, sessionId}) => { @@ -144,15 +143,15 @@ test('clamps noted offsets that fall beyond the persisted text', async ({kit, co draftStorage.noteSelection({start: 40, end: 44}) draftStorage.storage.setItem('any', composerDraft('short')) - await expect - .poll(() => storedDraft(kit.rpc, sessionId), SETTLED) - .toEqual({ - sessionId, - text: 'short', - selectionStart: 5, - selectionEnd: 5, - grabs: [], - }) + await core.awaitRequest(DRAFTS_SET_PATH) + + expect(await storedDraft(kit.rpc, sessionId)).toEqual({ + sessionId, + text: 'short', + selectionStart: 5, + selectionEnd: 5, + grabs: [], + }) }) test('appends to the stored draft on a new line with the caret at the end', async ({kit, core, sessionId}) => { @@ -181,13 +180,13 @@ test('survives a failed initial read and still accepts writes', async ({kit, cor draftStorage.storage.setItem('any', composerDraft('after the outage')) expect(draftStorage.storage.getItem('any')).toContain('after the outage') - await expect - .poll(() => storedDraft(kit.rpc, sessionId), SETTLED) - .toEqual({ - sessionId, - text: 'after the outage', - selectionStart: 16, - selectionEnd: 16, - grabs: [], - }) + await core.awaitRequest(DRAFTS_SET_PATH) + + expect(await storedDraft(kit.rpc, sessionId)).toEqual({ + sessionId, + text: 'after the outage', + selectionStart: 16, + selectionEnd: 16, + grabs: [], + }) }) diff --git a/apps/conciv/test/helpers/core-control.ts b/apps/conciv/test/helpers/core-control.ts index d91c56a6a..0f805a36f 100644 --- a/apps/conciv/test/helpers/core-control.ts +++ b/apps/conciv/test/helpers/core-control.ts @@ -20,6 +20,7 @@ declare module 'vitest/internal/browser' { installFault: (spec: FaultSpec) => Promise releaseFault: (handle: string) => Promise faultPending: (handle: string) => Promise + awaitFaultPending: (handle: string, count: number) => Promise } } diff --git a/apps/conciv/test/helpers/proxy.ts b/apps/conciv/test/helpers/proxy.ts index 9ec5497bf..20c8f884f 100644 --- a/apps/conciv/test/helpers/proxy.ts +++ b/apps/conciv/test/helpers/proxy.ts @@ -2,16 +2,26 @@ import {createServer, request as httpRequest, type IncomingMessage, type Server} import type {Duplex} from 'node:stream' import {listenLocal} from './listen-local.js' +const AWAIT_REQUEST_TIMEOUT_MS = 4000 + +export type AwaitRequestOptions = {since?: number; timeout?: number} + export type ProxyCore = { base: string port: number requestCount: (pathname?: string) => number wsConnectionCount: () => number + mark: () => number + awaitRequest: (pathname: string, options?: AwaitRequestOptions) => Promise fail: (pathname: string) => void repair: () => void close: () => Promise } +type Completion = {pathname: string; at: number} + +type Waiter = {pathname: string; since: number; deliver: () => void} + function handshakeResponse(upstream: IncomingMessage): string { const statusLine = `HTTP/1.1 ${upstream.statusCode ?? 101} ${upstream.statusMessage ?? 'Switching Protocols'}` const headers = upstream.rawHeaders.reduce((lines, value, index) => { @@ -27,9 +37,24 @@ export async function proxyTo(targetBase: string, opts: {blockUpgrades?: boolean const refused = new Set() let upgrades = 0 const piped = new Set() + const completions: Completion[] = [] + const waiters = new Set() + const sequence = {next: 0} + + const settle = (pathname: string): void => { + const at = (sequence.next += 1) + completions.push({pathname, at}) + for (const waiter of waiters) { + if (waiter.pathname !== pathname || at <= waiter.since) continue + waiters.delete(waiter) + waiter.deliver() + } + } + const server: Server = createServer((req, res) => { const pathname = new URL(req.url ?? '/', target).pathname seen.push(pathname) + res.on('finish', () => settle(pathname)) if (refused.has(pathname)) { res.writeHead(500, {'content-type': 'text/plain'}) res.end('the proxied core refused the call') @@ -90,6 +115,31 @@ export async function proxyTo(targetBase: string, opts: {blockUpgrades?: boolean requestCount: (pathname) => pathname === undefined ? seen.length : seen.filter((entry) => entry === pathname).length, wsConnectionCount: () => upgrades, + mark: () => sequence.next, + awaitRequest: (pathname, options = {}) => { + const since = options.since ?? sequence.next + if (completions.some((entry) => entry.pathname === pathname && entry.at > since)) return Promise.resolve() + const timeoutMs = options.timeout ?? AWAIT_REQUEST_TIMEOUT_MS + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + waiters.delete(waiter) + reject( + new Error( + `no proxied request to ${pathname} completed within ${timeoutMs}ms (completed paths: ${completions.map((entry) => entry.pathname).join(', ')})`, + ), + ) + }, timeoutMs) + const waiter: Waiter = { + pathname, + since, + deliver: () => { + clearTimeout(timer) + resolve() + }, + } + waiters.add(waiter) + }) + }, fail: (pathname) => { refused.add(pathname) }, diff --git a/apps/conciv/test/panel-focus-stability.browser.test.tsx b/apps/conciv/test/panel-focus-stability.browser.test.tsx index b551f8685..8aecd5e75 100644 --- a/apps/conciv/test/panel-focus-stability.browser.test.tsx +++ b/apps/conciv/test/panel-focus-stability.browser.test.tsx @@ -1,5 +1,5 @@ import './helpers/utilities.css' -import {afterAll, afterEach, beforeAll, expect, test, vi} from 'vitest' +import {afterAll, afterEach, beforeAll, expect, test} from 'vitest' import {page} from 'vitest/browser' import {coreControl} from './helpers/core-control.js' import {coreRpc, createSession} from './helpers/core-session.js' @@ -34,7 +34,7 @@ async function expectFocusSurvives(path: string[]): Promise { const since = await coreControl.rpcMark() const held = await faults.install({kind: 'gate', path}) await openPanel() - await vi.waitUntil(async () => (await coreControl.faultPending(held)) > 0, {timeout: 5000, interval: 20}) + await coreControl.awaitFaultPending(held, 1) await coreControl.releaseFault(held) diff --git a/apps/conciv/test/quick-add-pane.browser.test.tsx b/apps/conciv/test/quick-add-pane.browser.test.tsx index 209798f29..6d54c3cdc 100644 --- a/apps/conciv/test/quick-add-pane.browser.test.tsx +++ b/apps/conciv/test/quick-add-pane.browser.test.tsx @@ -1,5 +1,5 @@ import './helpers/utilities.css' -import {afterAll, afterEach, beforeAll, expect, test, vi} from 'vitest' +import {afterAll, afterEach, beforeAll, expect, test} from 'vitest' import {page} from 'vitest/browser' import {coreControl} from './helpers/core-control.js' import {coreRpc, createSession} from './helpers/core-session.js' @@ -56,7 +56,7 @@ test('rapid double-trigger creates exactly one pane', async () => { const firstClick = splitButton.click() const secondClick = splitButton.click() await Promise.all([firstClick, secondClick]) - await vi.waitUntil(async () => (await coreControl.faultPending(held)) === 1, {timeout: 5000, interval: 20}) + await coreControl.awaitFaultPending(held, 1) await coreControl.releaseFault(held) diff --git a/packages/extension-testkit/src/rpc-fault.ts b/packages/extension-testkit/src/rpc-fault.ts index 79b8dd9e1..f4801d283 100644 --- a/packages/extension-testkit/src/rpc-fault.ts +++ b/packages/extension-testkit/src/rpc-fault.ts @@ -135,17 +135,26 @@ export async function holdRpcCalls(page: Page): Promise { export type RpcGate = { pending: () => number + awaitCaptured: (count: number) => Promise release: () => Promise dispose: () => Promise } +type CaptureWaiter = {count: number; deliver: () => void} + export async function gateRpcCalls(page: Page, options: {path?: readonly string[]} = {}): Promise { const matcher = pathMatcher(options.path) const open = {value: false} const captured: Route[] = [] + const waiters = new Set() const handler: RouteHandler = async (route) => { if (isPreflight(route) || open.value) return route.continue() captured.push(route) + for (const waiter of waiters) { + if (captured.length < waiter.count) continue + waiters.delete(waiter) + waiter.deliver() + } } await page.route(matcher, handler) @@ -157,6 +166,12 @@ export async function gateRpcCalls(page: Page, options: {path?: readonly string[ return { pending: () => captured.length, + awaitCaptured: (count) => { + if (captured.length >= count) return Promise.resolve() + return new Promise((resolve) => { + waiters.add({count, deliver: resolve}) + }) + }, release: async () => { open.value = true await settle() From 45ddebd328d5a019afc98980d88a9778b5dc710f Mon Sep 17 00:00:00 2001 From: Omri Katz <9701896+omridevk@users.noreply.github.com> Date: Sat, 15 Aug 2026 11:56:21 +0300 Subject: [PATCH 9/9] test: #354 keep scripted null tool results and gate focus assertions (#508) scriptTurn defaulted result with ??, so an explicitly scripted null tool result serialized as {"ok":true}. Default only on undefined. panel-focus-stability released the request gate before the editor node and its focus were captured, so the gated request could complete before capture and the test no longer proved that an already-focused composer survives a pending dependency completing. The two "still loading" tests now capture and assert focus while the fault is still pending, then release. The draft-load test keeps its own shape: the composer lives behind a Show on the drafts.get resource, so it cannot mount until that request completes. Co-Authored-By: Claude Fable 5 --- .../panel-focus-stability.browser.test.tsx | 26 ++++++++++++++----- packages/harness-testkit/src/scripted-run.ts | 7 ++++- .../harness-testkit/test/scripted-run.test.ts | 11 ++++++++ 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/apps/conciv/test/panel-focus-stability.browser.test.tsx b/apps/conciv/test/panel-focus-stability.browser.test.tsx index 8aecd5e75..cd3f017f8 100644 --- a/apps/conciv/test/panel-focus-stability.browser.test.tsx +++ b/apps/conciv/test/panel-focus-stability.browser.test.tsx @@ -30,16 +30,18 @@ async function openPanel(): Promise { harness.mountShell(`/panel/${sessionId}?open=true`) } -async function expectFocusSurvives(path: string[]): Promise { +async function expectFocusSurvivesPendingRequest(path: string[]): Promise { const since = await coreControl.rpcMark() const held = await faults.install({kind: 'gate', path}) await openPanel() await coreControl.awaitFaultPending(held, 1) - await coreControl.releaseFault(held) - await expect.element(editor(), SETTLED).toBeVisible() const mountedNode = editor().element() + await expect.element(editor(), SETTLED).toHaveFocus() + expect(await coreControl.faultPending(held)).toBe(1) + + await coreControl.releaseFault(held) await coreControl.awaitRpcCall(path, since) await expect.element(editor(), SETTLED).toHaveFocus() @@ -47,13 +49,25 @@ async function expectFocusSurvives(path: string[]): Promise { } test('a composer that mounts after a slow draft load keeps the focus the panel gave it', async () => { - await expectFocusSurvives(['drafts', 'get']) + const path = ['drafts', 'get'] + const since = await coreControl.rpcMark() + const held = await faults.install({kind: 'gate', path}) + await openPanel() + await coreControl.awaitFaultPending(held, 1) + + await coreControl.releaseFault(held) + await coreControl.awaitRpcCall(path, since) + + await expect.element(editor(), SETTLED).toBeVisible() + const mountedNode = editor().element() + await expect.element(editor(), SETTLED).toHaveFocus() + expect(editor().element().isSameNode(mountedNode)).toBe(true) }, 30_000) test('a composer that mounts while the harness metadata is still loading takes the focus the panel gave it', async () => { - await expectFocusSurvives(['meta', 'models']) + await expectFocusSurvivesPendingRequest(['meta', 'models']) }, 30_000) test('a composer that mounts while the transcript is still loading takes the focus the panel gave it', async () => { - await expectFocusSurvives(['markers', 'list']) + await expectFocusSurvivesPendingRequest(['markers', 'list']) }, 30_000) diff --git a/packages/harness-testkit/src/scripted-run.ts b/packages/harness-testkit/src/scripted-run.ts index e7899b23a..de6ad6777 100644 --- a/packages/harness-testkit/src/scripted-run.ts +++ b/packages/harness-testkit/src/scripted-run.ts @@ -21,6 +21,11 @@ type QueuedTurn = {toolCalls: QueuedToolCall[]; text?: string; blocking: boolean const THREAD = {threadId: 'scripted', runId: 'scripted'} as const +function scriptedResult(call: ScriptedTurnToolCall): unknown { + if (call.result === undefined) return {ok: true} + return call.result +} + function* requestChunks(call: {id: string; name: string; input: unknown}): Generator { const toolCallId = call.id yield {type: EventType.TOOL_CALL_START, toolCallId, toolCallName: call.name, toolName: call.name} @@ -81,7 +86,7 @@ export function makeScriptedRun(opts: {text?: string} = {}): ScriptedRun { const scriptTurn = (turn: ScriptedTurn) => { const calls = turn.toolCalls.map((call) => { toolCalls.count += 1 - return {id: `tc-${toolCalls.count}`, name: call.name, input: call.input, result: call.result ?? {ok: true}} + return {id: `tc-${toolCalls.count}`, name: call.name, input: call.input, result: scriptedResult(call)} }) queuedTurns.push({toolCalls: calls, text: turn.text, blocking: false}) return calls.map((call) => call.id) diff --git a/packages/harness-testkit/test/scripted-run.test.ts b/packages/harness-testkit/test/scripted-run.test.ts index 41ef74fa9..0026d2c55 100644 --- a/packages/harness-testkit/test/scripted-run.test.ts +++ b/packages/harness-testkit/test/scripted-run.test.ts @@ -83,6 +83,17 @@ describe('makeScriptedRun', () => { expect(chunks.at(-1)?.type).toBe(EventType.RUN_FINISHED) }) + it('keeps a scripted null tool result as null instead of substituting the default', async () => { + const scripted = makeScriptedRun() + const ids = scripted.scriptTurn({toolCalls: [{name: 'nullish_tool', input: {a: 1}, result: null}]}) + const chunks: StreamChunk[] = [] + for await (const chunk of scripted.chatStream(deps())) chunks.push(chunk) + const results = chunks.flatMap((chunk) => + chunk.type === EventType.TOOL_CALL_RESULT ? [{id: chunk.toolCallId, content: chunk.content}] : [], + ) + expect(results).toEqual([{id: ids[0], content: 'null'}]) + }) + it('streams a queued tool call and a queued turn in the order they were scripted', async () => { const scripted = makeScriptedRun() scripted.scriptToolCall('first_tool', {a: 1})