From 11c0139b995d3cab2af326e867a73cf40b92f77f Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Wed, 1 Jul 2026 16:20:02 +0100 Subject: [PATCH 01/11] =?UTF-8?q?feat(run-ops):=20webapp=20routes=20?= =?UTF-8?q?=E2=80=94=20friendlyId=20reads,=20resolver=20auth=20re-check,?= =?UTF-8?q?=20co-location=20writes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.8 (1M context) --- apps/webapp/app/components/admin/debugRun.tsx | 37 +- apps/webapp/app/routes/@.runs.$runParam.ts | 31 +- .../route.tsx | 14 +- .../route.tsx | 2 +- .../route.tsx | 12 +- .../app/routes/api.v1.batches.$batchId.ts | 12 +- ....runs.$runFriendlyId.input-streams.wait.ts | 4 +- ...uns.$runFriendlyId.session-streams.wait.ts | 4 +- ...ens.$waitpointFriendlyId.callback.$hash.ts | 51 +-- ...ts.tokens.$waitpointFriendlyId.complete.ts | 27 +- .../app/routes/api.v1.waitpoints.tokens.ts | 12 +- .../app/routes/api.v2.batches.$batchId.ts | 12 +- apps/webapp/app/routes/api.v2.tasks.batch.ts | 15 +- apps/webapp/app/routes/api.v3.batches.ts | 15 +- ...ne.v1.runs.$runFriendlyId.wait.duration.ts | 4 + ...points.tokens.$waitpointFriendlyId.wait.ts | 18 +- ...g.projects.$projectParam.runs.$runParam.ts | 41 ++- .../projects.v3.$projectRef.runs.$runParam.ts | 25 +- .../routes/realtime.v1.batches.$batchId.ts | 11 +- .../app/routes/realtime.v1.runs.$runId.ts | 2 +- apps/webapp/app/routes/realtime.v1.runs.ts | 2 +- .../realtime.v1.streams.$runId.$streamId.ts | 18 +- ...am.runs.$runParam.idempotencyKey.reset.tsx | 41 ++- .../route.tsx | 37 +- .../route.tsx | 24 +- ...jectParam.env.$envParam.waitpoints.tags.ts | 12 +- .../resources.runs.$runParam.logs.download.ts | 59 ++-- .../app/routes/resources.runs.$runParam.ts | 121 +++---- .../resources.taskruns.$runParam.cancel.ts | 76 ++-- .../resources.taskruns.$runParam.debug.ts | 91 +++-- .../resources.taskruns.$runParam.replay.ts | 244 +++++++------ apps/webapp/app/routes/runs.$runParam.ts | 67 ++-- .../app/routes/sync.traces.runs.$traceId.ts | 15 +- ...nts.tokens.complete.crossSeamGuard.test.ts | 214 ++++++++++++ .../test/api.v1.waitpoints.tokens.test.ts | 258 ++++++++++++++ apps/webapp/test/crossSeamGuard.proof.test.ts | 215 ++++++++++++ .../waitpointCallback.controlPlane.test.ts | 324 ++++++++++++++++++ 37 files changed, 1618 insertions(+), 549 deletions(-) create mode 100644 apps/webapp/test/api.v1.waitpoints.tokens.complete.crossSeamGuard.test.ts create mode 100644 apps/webapp/test/api.v1.waitpoints.tokens.test.ts create mode 100644 apps/webapp/test/crossSeamGuard.proof.test.ts create mode 100644 apps/webapp/test/waitpointCallback.controlPlane.test.ts diff --git a/apps/webapp/app/components/admin/debugRun.tsx b/apps/webapp/app/components/admin/debugRun.tsx index 8fde0dd28d0..4c5d40d8bf8 100644 --- a/apps/webapp/app/components/admin/debugRun.tsx +++ b/apps/webapp/app/components/admin/debugRun.tsx @@ -77,6 +77,7 @@ function DebugRunData(props: UseDataFunctionReturn) { function DebugRunDataEngineV1({ run, + environment, queueConcurrencyLimit, queueCurrentConcurrency, envConcurrencyLimit, @@ -121,7 +122,7 @@ function DebugRunDataEngineV1({ Queue concurrency limit key @@ -228,9 +229,7 @@ function DebugRunDataEngineV1({ GET queue concurrency limit @@ -246,7 +245,7 @@ function DebugRunDataEngineV1({ Env current concurrency key @@ -256,7 +255,7 @@ function DebugRunDataEngineV1({ Get env current concurrency @@ -272,7 +271,7 @@ function DebugRunDataEngineV1({ Env reserve concurrency key @@ -282,9 +281,7 @@ function DebugRunDataEngineV1({ Get env reserve concurrency @@ -300,7 +297,7 @@ function DebugRunDataEngineV1({ Env concurrency limit key @@ -310,7 +307,7 @@ function DebugRunDataEngineV1({ GET env concurrency limit @@ -326,7 +323,7 @@ function DebugRunDataEngineV1({ Shared queue key @@ -337,7 +334,7 @@ function DebugRunDataEngineV1({ { }; const filters = BatchListFilters.parse(s); - const presenter = new BatchListPresenter(); + const presenter = new BatchListPresenter(undefined, undefined, { + runOpsNew: runOpsNewReplicaClient as unknown as PrismaClientOrTransaction, + runOpsLegacyReplica: runOpsLegacyReplica as unknown as PrismaClientOrTransaction, + controlPlaneReplica: $replica as unknown as PrismaClientOrTransaction, + splitEnabled: runOpsSplitReadEnabled, + }); const list = await presenter.call({ userId, projectId: project.id, diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.waitpoints.tokens.$waitpointParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.waitpoints.tokens.$waitpointParam/route.tsx index 48b842d34d1..cfc008e6792 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.waitpoints.tokens.$waitpointParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.waitpoints.tokens.$waitpointParam/route.tsx @@ -45,7 +45,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { } try { - const presenter = new WaitpointPresenter(); + const presenter = new WaitpointPresenter(undefined, undefined, {}); const result = await presenter.call({ friendlyId: waitpointParam, environmentId: environment.id, diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.waitpoints.tokens/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.waitpoints.tokens/route.tsx index 9d4d7d67433..011ad8a3680 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.waitpoints.tokens/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.waitpoints.tokens/route.tsx @@ -42,6 +42,12 @@ import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; import { WaitpointListPresenter } from "~/presenters/v3/WaitpointListPresenter.server"; import { requireUserId } from "~/services/session.server"; +import { + runOpsNewReplicaClient, + runOpsLegacyReplica, + runOpsSplitReadEnabled, + type PrismaClientOrTransaction, +} from "~/db.server"; import { docsPath, EnvironmentParamSchema, v3WaitpointTokenPath } from "~/utils/pathBuilder"; export const meta: MetaFunction = () => { @@ -88,7 +94,11 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { } try { - const presenter = new WaitpointListPresenter(); + const presenter = new WaitpointListPresenter(undefined, undefined, { + runOpsNew: runOpsNewReplicaClient as unknown as PrismaClientOrTransaction, + runOpsLegacyReplica: runOpsLegacyReplica as unknown as PrismaClientOrTransaction, + splitEnabled: runOpsSplitReadEnabled, + }); const result = await presenter.call({ environment, ...searchParams, diff --git a/apps/webapp/app/routes/api.v1.batches.$batchId.ts b/apps/webapp/app/routes/api.v1.batches.$batchId.ts index b485dc86054..01acd3d140f 100644 --- a/apps/webapp/app/routes/api.v1.batches.$batchId.ts +++ b/apps/webapp/app/routes/api.v1.batches.$batchId.ts @@ -1,7 +1,7 @@ import { json } from "@remix-run/server-runtime"; import { z } from "zod"; -import { $replica } from "~/db.server"; import { anyResource, createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; +import { runStore } from "~/v3/runStore.server"; const ParamsSchema = z.object({ batchId: z.string(), @@ -13,14 +13,8 @@ export const loader = createLoaderApiRoute( allowJWT: true, corsStrategy: "all", findResource: (params, auth) => { - return $replica.batchTaskRun.findFirst({ - where: { - friendlyId: params.batchId, - runtimeEnvironmentId: auth.environment.id, - }, - include: { - errors: true, - }, + return runStore.findBatchTaskRunByFriendlyId(params.batchId, auth.environment.id, { + include: { errors: true }, }); }, authorization: { diff --git a/apps/webapp/app/routes/api.v1.runs.$runFriendlyId.input-streams.wait.ts b/apps/webapp/app/routes/api.v1.runs.$runFriendlyId.input-streams.wait.ts index 091312a13b8..11d77b9f06f 100644 --- a/apps/webapp/app/routes/api.v1.runs.$runFriendlyId.input-streams.wait.ts +++ b/apps/webapp/app/routes/api.v1.runs.$runFriendlyId.input-streams.wait.ts @@ -78,8 +78,10 @@ const { action, loader } = createActionApiRoute( } } - // Step 1: Create the waitpoint + // Step 1: Create the waitpoint. Co-locate it with the owning run (run-ops split) so a ksuid + // run's input-stream waitpoint lands on the run's DB and its block edge resolves. const result = await engine.createManualWaitpoint({ + runId: run.id, environmentId: authentication.environment.id, projectId: authentication.environment.projectId, idempotencyKey: body.idempotencyKey, diff --git a/apps/webapp/app/routes/api.v1.runs.$runFriendlyId.session-streams.wait.ts b/apps/webapp/app/routes/api.v1.runs.$runFriendlyId.session-streams.wait.ts index cd88ef38281..a7052d2377e 100644 --- a/apps/webapp/app/routes/api.v1.runs.$runFriendlyId.session-streams.wait.ts +++ b/apps/webapp/app/routes/api.v1.runs.$runFriendlyId.session-streams.wait.ts @@ -99,8 +99,10 @@ const { action, loader } = createActionApiRoute( } } - // Step 1: Create the waitpoint. + // Step 1: Create the waitpoint. Co-locate it with the owning run (run-ops split) so a ksuid + // run's session-stream waitpoint lands on the run's DB and its block edge resolves. const result = await engine.createManualWaitpoint({ + runId: run.id, environmentId: authentication.environment.id, projectId: authentication.environment.projectId, idempotencyKey: body.idempotencyKey, diff --git a/apps/webapp/app/routes/api.v1.waitpoints.tokens.$waitpointFriendlyId.callback.$hash.ts b/apps/webapp/app/routes/api.v1.waitpoints.tokens.$waitpointFriendlyId.callback.$hash.ts index 5583b2e63c6..da7b9c2e1ab 100644 --- a/apps/webapp/app/routes/api.v1.waitpoints.tokens.$waitpointFriendlyId.callback.$hash.ts +++ b/apps/webapp/app/routes/api.v1.waitpoints.tokens.$waitpointFriendlyId.callback.$hash.ts @@ -2,11 +2,13 @@ import { type ActionFunctionArgs, json } from "@remix-run/server-runtime"; import { type CompleteWaitpointTokenResponseBody, stringifyIO } from "@trigger.dev/core/v3"; import { WaitpointId } from "@trigger.dev/core/v3/isomorphic"; import { z } from "zod"; -import { $replica } from "~/db.server"; +import type { PrismaReplicaClient } from "~/db.server"; import { env } from "~/env.server"; import { processWaitpointCompletionPacket } from "~/runEngine/concerns/waitpointCompletionPacket.server"; import { verifyHttpCallbackHash } from "~/services/httpCallback.server"; import { logger } from "~/services/logger.server"; +import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server"; +import { readThroughRun } from "~/v3/runOpsMigration/readThrough.server"; import { engine } from "~/v3/runEngine.server"; const paramsSchema = z.object({ @@ -32,36 +34,43 @@ export async function action({ request, params }: ActionFunctionArgs) { const waitpointId = WaitpointId.toId(waitpointFriendlyId); try { - const waitpoint = await $replica.waitpoint.findFirst({ - where: { - id: waitpointId, - }, - include: { - environment: { - include: { - project: true, - organization: true, - orgMember: true, - parentEnvironment: { - select: { - id: true, - apiKey: true, - }, - }, - }, + // Read through the split-aware run-ops read-through (passthrough in single-DB). The env is + // resolved below from the row; residency is classified off the waitpoint id, so env "" is fine. + const findWaitpoint = (client: PrismaReplicaClient) => + client.waitpoint.findFirst({ + where: { + id: waitpointId, }, - }, + select: { id: true, status: true, environmentId: true }, + }); + + const waitpointResult = await readThroughRun({ + runId: waitpointId, + environmentId: "", + readNew: (client) => findWaitpoint(client), + readLegacy: (replica) => findWaitpoint(replica), }); + const waitpoint = + waitpointResult.source === "new" || waitpointResult.source === "legacy-replica" + ? waitpointResult.value + : null; + if (!waitpoint) { return json({ error: "Waitpoint not found" }, { status: 404 }); } + const environment = await controlPlaneResolver.resolveAuthenticatedEnv(waitpoint.environmentId); + + if (!environment) { + return json({ error: "Waitpoint not found" }, { status: 404 }); + } + if ( !verifyHttpCallbackHash( waitpoint.id, hash, - waitpoint.environment.parentEnvironment?.apiKey ?? waitpoint.environment.apiKey + environment.parentEnvironment?.apiKey ?? environment.apiKey ) ) { return json({ error: "Invalid URL, hash doesn't match" }, { status: 401 }); @@ -79,7 +88,7 @@ export async function action({ request, params }: ActionFunctionArgs) { const stringifiedData = await stringifyIO(body); const finalData = await processWaitpointCompletionPacket( stringifiedData, - waitpoint.environment, + environment, `${WaitpointId.toFriendlyId(waitpointId)}/http-callback` ); diff --git a/apps/webapp/app/routes/api.v1.waitpoints.tokens.$waitpointFriendlyId.complete.ts b/apps/webapp/app/routes/api.v1.waitpoints.tokens.$waitpointFriendlyId.complete.ts index d268629c7d1..a4df0e83fd2 100644 --- a/apps/webapp/app/routes/api.v1.waitpoints.tokens.$waitpointFriendlyId.complete.ts +++ b/apps/webapp/app/routes/api.v1.waitpoints.tokens.$waitpointFriendlyId.complete.ts @@ -6,11 +6,12 @@ import { } from "@trigger.dev/core/v3"; import { WaitpointId } from "@trigger.dev/core/v3/isomorphic"; import { z } from "zod"; -import { $replica } from "~/db.server"; +import type { PrismaReplicaClient } from "~/db.server"; import { env } from "~/env.server"; import { logger } from "~/services/logger.server"; import { processWaitpointCompletionPacket } from "~/runEngine/concerns/waitpointCompletionPacket.server"; import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; +import { readThroughRun } from "~/v3/runOpsMigration/readThrough.server"; import { engine } from "~/v3/runEngine.server"; const { action, loader } = createActionApiRoute( @@ -33,13 +34,27 @@ const { action, loader } = createActionApiRoute( try { //check permissions - const waitpoint = await $replica.waitpoint.findFirst({ - where: { - id: waitpointId, - environmentId: authentication.environment.id, - }, + // Read through the split-aware run-ops read-through (passthrough in single-DB). + const findWaitpoint = (client: PrismaReplicaClient) => + client.waitpoint.findFirst({ + where: { + id: waitpointId, + environmentId: authentication.environment.id, + }, + }); + + const waitpointResult = await readThroughRun({ + runId: waitpointId, + environmentId: authentication.environment.id, + readNew: (client) => findWaitpoint(client), + readLegacy: (replica) => findWaitpoint(replica), }); + const waitpoint = + waitpointResult.source === "new" || waitpointResult.source === "legacy-replica" + ? waitpointResult.value + : null; + if (!waitpoint) { throw json({ error: "Waitpoint not found" }, { status: 404 }); } diff --git a/apps/webapp/app/routes/api.v1.waitpoints.tokens.ts b/apps/webapp/app/routes/api.v1.waitpoints.tokens.ts index b7ef988b728..3acb4b64cb0 100644 --- a/apps/webapp/app/routes/api.v1.waitpoints.tokens.ts +++ b/apps/webapp/app/routes/api.v1.waitpoints.tokens.ts @@ -9,6 +9,12 @@ import { ApiWaitpointListPresenter, ApiWaitpointListSearchParams, } from "~/presenters/v3/ApiWaitpointListPresenter.server"; +import { + runOpsNewReplicaClient, + runOpsLegacyReplica, + runOpsSplitReadEnabled, + type PrismaClientOrTransaction, +} from "~/db.server"; import { type AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; import { generateHttpCallbackUrl } from "~/services/httpCallback.server"; @@ -27,7 +33,11 @@ export const loader = createLoaderApiRoute( findResource: async () => 1, // This is a dummy function, we don't need to find a resource }, async ({ searchParams, authentication }) => { - const presenter = new ApiWaitpointListPresenter(); + const presenter = new ApiWaitpointListPresenter(undefined, undefined, { + runOpsNew: runOpsNewReplicaClient as unknown as PrismaClientOrTransaction, + runOpsLegacyReplica: runOpsLegacyReplica as unknown as PrismaClientOrTransaction, + splitEnabled: runOpsSplitReadEnabled, + }); const result = await presenter.call(authentication.environment, searchParams); return json(result); diff --git a/apps/webapp/app/routes/api.v2.batches.$batchId.ts b/apps/webapp/app/routes/api.v2.batches.$batchId.ts index deb3b788229..b6ef4136bd4 100644 --- a/apps/webapp/app/routes/api.v2.batches.$batchId.ts +++ b/apps/webapp/app/routes/api.v2.batches.$batchId.ts @@ -1,7 +1,7 @@ import { json } from "@remix-run/server-runtime"; import { z } from "zod"; -import { $replica } from "~/db.server"; import { anyResource, createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; +import { runStore } from "~/v3/runStore.server"; const ParamsSchema = z.object({ batchId: z.string(), @@ -13,14 +13,8 @@ export const loader = createLoaderApiRoute( allowJWT: true, corsStrategy: "all", findResource: (params, auth) => { - return $replica.batchTaskRun.findFirst({ - where: { - friendlyId: params.batchId, - runtimeEnvironmentId: auth.environment.id, - }, - include: { - errors: true, - }, + return runStore.findBatchTaskRunByFriendlyId(params.batchId, auth.environment.id, { + include: { errors: true }, }); }, authorization: { diff --git a/apps/webapp/app/routes/api.v2.tasks.batch.ts b/apps/webapp/app/routes/api.v2.tasks.batch.ts index 856e6f1b874..c766bb474f6 100644 --- a/apps/webapp/app/routes/api.v2.tasks.batch.ts +++ b/apps/webapp/app/routes/api.v2.tasks.batch.ts @@ -1,8 +1,8 @@ import { json } from "@remix-run/server-runtime"; import type { BatchTriggerTaskV3Response } from "@trigger.dev/core/v3"; import { BatchTriggerTaskV3RequestBody, generateJWT } from "@trigger.dev/core/v3"; -import { prisma } from "~/db.server"; import { env } from "~/env.server"; +import { runStore } from "~/v3/runStore.server"; import { RunEngineBatchTriggerService } from "~/runEngine/services/batchTrigger.server"; import type { AuthenticatedEnvironment } from "~/services/apiAuth.server"; import { getOneTimeUseToken } from "~/services/apiAuth.server"; @@ -90,16 +90,9 @@ const { action, loader } = createActionApiRoute( const cachedResponse = await handleRequestIdempotency(requestIdempotencyKey, { requestType: "batch-trigger", findCachedEntity: async (cachedRequestId) => { - return await prisma.batchTaskRun.findFirst({ - where: { - id: cachedRequestId, - runtimeEnvironmentId: authentication.environment.id, - }, - select: { - friendlyId: true, - runCount: true, - }, - }); + const batch = await runStore.findBatchTaskRunById(cachedRequestId); + if (!batch || batch.runtimeEnvironmentId !== authentication.environment.id) return null; + return batch; }, buildResponse: (cachedBatch) => ({ id: cachedBatch.friendlyId, diff --git a/apps/webapp/app/routes/api.v3.batches.ts b/apps/webapp/app/routes/api.v3.batches.ts index 033dda719b3..3179e2fa6aa 100644 --- a/apps/webapp/app/routes/api.v3.batches.ts +++ b/apps/webapp/app/routes/api.v3.batches.ts @@ -1,7 +1,6 @@ import { json } from "@remix-run/server-runtime"; import type { CreateBatchResponse } from "@trigger.dev/core/v3"; import { CreateBatchRequestBody, generateJWT } from "@trigger.dev/core/v3"; -import { prisma } from "~/db.server"; import { env } from "~/env.server"; import { BatchRateLimitExceededError } from "~/runEngine/concerns/batchLimits.server"; import { CreateBatchService } from "~/runEngine/services/createBatch.server"; @@ -19,6 +18,7 @@ import { import { sanitizeTriggerSource } from "~/utils/triggerSource"; import { ServiceValidationError } from "~/v3/services/baseService.server"; import { OutOfEntitlementError } from "~/v3/services/triggerTask.server"; +import { engine } from "~/v3/runEngine.server"; import { HeadersSchema } from "./api.v1.tasks.$taskId.trigger"; /** @@ -89,16 +89,9 @@ const { action, loader } = createActionApiRoute( >(body.idempotencyKey, { requestType: "create-batch", findCachedEntity: async (cachedRequestId) => { - return await prisma.batchTaskRun.findFirst({ - where: { - id: cachedRequestId, - runtimeEnvironmentId: authentication.environment.id, - }, - select: { - friendlyId: true, - runCount: true, - }, - }); + const batch = await engine.runStore.findBatchTaskRunById(cachedRequestId); + if (!batch || batch.runtimeEnvironmentId !== authentication.environment.id) return null; + return batch; }, buildResponse: (cachedBatch) => ({ id: cachedBatch.friendlyId, diff --git a/apps/webapp/app/routes/engine.v1.runs.$runFriendlyId.wait.duration.ts b/apps/webapp/app/routes/engine.v1.runs.$runFriendlyId.wait.duration.ts index b2f6c67949e..24f7f81ba0b 100644 --- a/apps/webapp/app/routes/engine.v1.runs.$runFriendlyId.wait.duration.ts +++ b/apps/webapp/app/routes/engine.v1.runs.$runFriendlyId.wait.duration.ts @@ -42,6 +42,10 @@ const { action } = createActionApiRoute( : undefined; const { waitpoint } = await engine.createDateTimeWaitpoint({ + // Co-locate the waitpoint with the run that blocks on it (run-ops split): a ksuid run lives + // on the dedicated DB, but the minted waitpoint id is always a cuid, so without the run id + // the waitpoint would route to the control-plane DB and the block edge would never resolve. + runId: run.id, projectId: authentication.environment.project.id, environmentId: authentication.environment.id, completedAfter: body.date, diff --git a/apps/webapp/app/routes/engine.v1.runs.$runFriendlyId.waitpoints.tokens.$waitpointFriendlyId.wait.ts b/apps/webapp/app/routes/engine.v1.runs.$runFriendlyId.waitpoints.tokens.$waitpointFriendlyId.wait.ts index ea17c339787..156171aff13 100644 --- a/apps/webapp/app/routes/engine.v1.runs.$runFriendlyId.waitpoints.tokens.$waitpointFriendlyId.wait.ts +++ b/apps/webapp/app/routes/engine.v1.runs.$runFriendlyId.waitpoints.tokens.$waitpointFriendlyId.wait.ts @@ -2,7 +2,8 @@ import { json } from "@remix-run/server-runtime"; import { type WaitForWaitpointTokenResponseBody } from "@trigger.dev/core/v3"; import { RunId, WaitpointId } from "@trigger.dev/core/v3/isomorphic"; import { z } from "zod"; -import { $replica } from "~/db.server"; +import type { PrismaReplicaClient } from "~/db.server"; +import { resolveWaitpointThroughReadThrough } from "~/runEngine/concerns/resolveWaitpointThroughReadThrough.server"; import { logger } from "~/services/logger.server"; import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; import { engine } from "~/v3/runEngine.server"; @@ -23,11 +24,16 @@ const { action } = createActionApiRoute( try { //check permissions - const waitpoint = await $replica.waitpoint.findFirst({ - where: { - id: waitpointId, - environmentId: authentication.environment.id, - }, + const waitpoint = await resolveWaitpointThroughReadThrough({ + waitpointId, + environmentId: authentication.environment.id, + read: (client: PrismaReplicaClient) => + client.waitpoint.findFirst({ + where: { + id: waitpointId, + environmentId: authentication.environment.id, + }, + }), }); if (!waitpoint) { diff --git a/apps/webapp/app/routes/orgs.$organizationSlug.projects.$projectParam.runs.$runParam.ts b/apps/webapp/app/routes/orgs.$organizationSlug.projects.$projectParam.runs.$runParam.ts index d5d4ab0f2f6..1687c45008f 100644 --- a/apps/webapp/app/routes/orgs.$organizationSlug.projects.$projectParam.runs.$runParam.ts +++ b/apps/webapp/app/routes/orgs.$organizationSlug.projects.$projectParam.runs.$runParam.ts @@ -1,10 +1,11 @@ import { redirect } from "@remix-run/router"; import { type LoaderFunctionArgs } from "@remix-run/server-runtime"; import { z } from "zod"; -import { prisma } from "~/db.server"; +import { $replica } from "~/db.server"; import { requireUserId } from "~/services/session.server"; import { ProjectParamSchema, v3RunPath } from "~/utils/pathBuilder"; import { runStore } from "~/v3/runStore.server"; +import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server"; const ParamSchema = ProjectParamSchema.extend({ runParam: z.string(), @@ -17,37 +18,45 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const run = await runStore.findRun( { friendlyId: runParam, - project: { - slug: projectParam, - organization: { - slug: organizationSlug, - members: { - some: { - userId, - }, - }, - }, - }, }, { select: { - runtimeEnvironment: true, + projectId: true, + runtimeEnvironmentId: true, }, - }, - prisma + } ); if (!run) { throw new Response("Not Found", { status: 404 }); } + const authorizedProject = await $replica.project.findFirst({ + where: { id: run.projectId, organization: { members: { some: { userId } } } }, + select: { id: true }, + }); + + if (!authorizedProject) { + throw new Response("Not Found", { status: 404 }); + } + + const environment = await controlPlaneResolver.resolveAuthenticatedEnv(run.runtimeEnvironmentId); + + if (!environment) { + throw new Response("Not Found", { status: 404 }); + } + + if (environment.project.slug !== projectParam || environment.organization.slug !== organizationSlug) { + throw new Response("Not Found", { status: 404 }); + } + return redirect( v3RunPath( { slug: organizationSlug, }, { slug: projectParam }, - { slug: run.runtimeEnvironment.slug }, + { slug: environment.slug }, { friendlyId: runParam } ) ); diff --git a/apps/webapp/app/routes/projects.v3.$projectRef.runs.$runParam.ts b/apps/webapp/app/routes/projects.v3.$projectRef.runs.$runParam.ts index 2a6cb34c913..300caa27ddf 100644 --- a/apps/webapp/app/routes/projects.v3.$projectRef.runs.$runParam.ts +++ b/apps/webapp/app/routes/projects.v3.$projectRef.runs.$runParam.ts @@ -4,6 +4,7 @@ import { prisma } from "~/db.server"; import { requireUserId } from "~/services/session.server"; import { v3RunSpanPath } from "~/utils/pathBuilder"; import { runStore } from "~/v3/runStore.server"; +import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server"; const ParamsSchema = z.object({ projectRef: z.string(), @@ -40,8 +41,10 @@ export async function loader({ params, request }: LoaderFunctionArgs) { friendlyId: validatedParams.runParam, }, { - include: { - runtimeEnvironment: true, + select: { + friendlyId: true, + spanId: true, + runtimeEnvironmentId: true, }, }, prisma @@ -51,16 +54,16 @@ export async function loader({ params, request }: LoaderFunctionArgs) { throw new Response("Not found", { status: 404 }); } + const environment = await controlPlaneResolver.resolveAuthenticatedEnv(run.runtimeEnvironmentId); + + if (!environment) { + throw new Response("Not found", { status: 404 }); + } + // Redirect to the project's runs page return redirect( - v3RunSpanPath( - { slug: project.organization.slug }, - { slug: project.slug }, - run.runtimeEnvironment, - run, - { - spanId: run.spanId, - } - ) + v3RunSpanPath({ slug: project.organization.slug }, { slug: project.slug }, environment, run, { + spanId: run.spanId, + }) ); } diff --git a/apps/webapp/app/routes/realtime.v1.batches.$batchId.ts b/apps/webapp/app/routes/realtime.v1.batches.$batchId.ts index 07906292d4d..29ad61d6646 100644 --- a/apps/webapp/app/routes/realtime.v1.batches.$batchId.ts +++ b/apps/webapp/app/routes/realtime.v1.batches.$batchId.ts @@ -1,8 +1,8 @@ import { z } from "zod"; -import { $replica } from "~/db.server"; import { getRequestAbortSignal } from "~/services/httpAsyncStorage.server"; import { resolveRealtimeStreamClient } from "~/services/realtime/resolveRealtimeStreamClient.server"; import { anyResource, createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; +import { runStore } from "~/v3/runStore.server"; const ParamsSchema = z.object({ batchId: z.string(), @@ -14,12 +14,7 @@ export const loader = createLoaderApiRoute( allowJWT: true, corsStrategy: "all", findResource: (params, auth) => { - return $replica.batchTaskRun.findFirst({ - where: { - friendlyId: params.batchId, - runtimeEnvironmentId: auth.environment.id, - }, - }); + return runStore.findBatchTaskRunByFriendlyId(params.batchId, auth.environment.id); }, authorization: { action: "read", @@ -29,7 +24,7 @@ export const loader = createLoaderApiRoute( }, }, async ({ authentication, request, resource: batchRun, apiVersion }) => { - // Pick the Electric proxy or the native backend per org (defaults to Electric); both implement streamBatch. + // Resolve the native realtime client; it implements streamBatch. const client = await resolveRealtimeStreamClient(authentication.environment); return client.streamBatch( diff --git a/apps/webapp/app/routes/realtime.v1.runs.$runId.ts b/apps/webapp/app/routes/realtime.v1.runs.$runId.ts index 2f4ad427d0a..51b47b22afd 100644 --- a/apps/webapp/app/routes/realtime.v1.runs.$runId.ts +++ b/apps/webapp/app/routes/realtime.v1.runs.$runId.ts @@ -48,7 +48,7 @@ export const loader = createLoaderApiRoute( }, }, async ({ authentication, request, resource: run, apiVersion }) => { - // Pick the Electric proxy or the native backend per org (defaults to Electric); both implement streamRun. + // Resolve the native realtime client; it implements streamRun. const client = await resolveRealtimeStreamClient(authentication.environment); return client.streamRun( diff --git a/apps/webapp/app/routes/realtime.v1.runs.ts b/apps/webapp/app/routes/realtime.v1.runs.ts index c0600b392bb..ca2998f2972 100644 --- a/apps/webapp/app/routes/realtime.v1.runs.ts +++ b/apps/webapp/app/routes/realtime.v1.runs.ts @@ -31,7 +31,7 @@ export const loader = createLoaderApiRoute( }, }, async ({ searchParams, authentication, request, apiVersion }) => { - // Pick the Electric proxy or the native backend per org (defaults to Electric); both implement streamRuns. + // Resolve the native realtime client; it implements streamRuns. const client = await resolveRealtimeStreamClient(authentication.environment); return client.streamRuns( diff --git a/apps/webapp/app/routes/realtime.v1.streams.$runId.$streamId.ts b/apps/webapp/app/routes/realtime.v1.streams.$runId.$streamId.ts index aaba0c33a35..420368c630a 100644 --- a/apps/webapp/app/routes/realtime.v1.streams.$runId.$streamId.ts +++ b/apps/webapp/app/routes/realtime.v1.streams.$runId.$streamId.ts @@ -5,6 +5,7 @@ import { getRequestAbortSignal } from "~/services/httpAsyncStorage.server"; import { getRealtimeStreamInstance } from "~/services/realtime/v1StreamsGlobal.server"; import { anyResource, createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; import { runStore } from "~/v3/runStore.server"; +import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server"; const ParamsSchema = z.object({ runId: z.string(), @@ -31,13 +32,7 @@ export async function action({ request, params }: ActionFunctionArgs) { id: true, friendlyId: true, streamBasinName: true, - runtimeEnvironment: { - include: { - project: true, - organization: true, - orgMember: true, - }, - }, + runtimeEnvironmentId: true, }, }, $replica @@ -47,6 +42,12 @@ export async function action({ request, params }: ActionFunctionArgs) { return new Response("Run not found", { status: 404 }); } + const environment = await controlPlaneResolver.resolveAuthenticatedEnv(run.runtimeEnvironmentId); + + if (!environment) { + return new Response("Run not found", { status: 404 }); + } + // Extract client ID from header, default to "default" if not provided const clientId = request.headers.get("X-Client-Id") || "default"; const streamVersion = request.headers.get("X-Stream-Version") || "v1"; @@ -67,8 +68,7 @@ export async function action({ request, params }: ActionFunctionArgs) { resumeFromChunkNumber = parsed; } - // The runtimeEnvironment from the run is already in the correct shape for AuthenticatedEnvironment - const realtimeStream = getRealtimeStreamInstance(run.runtimeEnvironment, streamVersion, { + const realtimeStream = getRealtimeStreamInstance(environment, streamVersion, { run, }); diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.idempotencyKey.reset.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.idempotencyKey.reset.tsx index f774db50b11..4d6662dba86 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.idempotencyKey.reset.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.idempotencyKey.reset.tsx @@ -1,5 +1,5 @@ -import { type ActionFunction } from "@remix-run/node"; -import { prisma } from "~/db.server"; +import { type ActionFunction, json } from "@remix-run/node"; +import { $replica, prisma } from "~/db.server"; import { jsonWithErrorMessage, jsonWithSuccessMessage } from "~/models/message.server"; import { logger } from "~/services/logger.server"; import { requireUserId } from "~/services/session.server"; @@ -15,41 +15,36 @@ export const action: ActionFunction = async ({ request, params }) => { const taskRun = await runStore.findRun( { friendlyId: runParam, - project: { - slug: projectParam, - organization: { - slug: organizationSlug, - members: { - some: { - userId, - }, - }, - }, - }, - runtimeEnvironment: { - slug: envParam, - }, }, { select: { id: true, idempotencyKey: true, taskIdentifier: true, + projectId: true, runtimeEnvironmentId: true, }, - }, - prisma + } ); if (!taskRun) { return jsonWithErrorMessage({}, request, "Run not found"); } + const authorizedProject = await $replica.project.findFirst({ + where: { id: taskRun.projectId, organization: { members: { some: { userId } } } }, + select: { id: true }, + }); + + if (!authorizedProject) { + return jsonWithErrorMessage({}, request, "Run not found"); + } + if (!taskRun.idempotencyKey) { return jsonWithErrorMessage({}, request, "This run does not have an idempotency key"); } - const environment = await prisma.runtimeEnvironment.findUnique({ + const environment = await prisma.runtimeEnvironment.findFirst({ where: { id: taskRun.runtimeEnvironmentId, }, @@ -66,6 +61,14 @@ export const action: ActionFunction = async ({ request, params }) => { return jsonWithErrorMessage({}, request, "Environment not found"); } + if ( + environment.slug !== envParam || + environment.project.slug !== projectParam || + environment.project.organization.slug !== organizationSlug + ) { + return jsonWithErrorMessage({}, request, "Run not found"); + } + const service = new ResetIdempotencyKeyService(); await service.call(taskRun.idempotencyKey, taskRun.taskIdentifier, { diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.streams.$streamKey/route.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.streams.$streamKey/route.tsx index ee5eda4c3a6..9e573e2f65a 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.streams.$streamKey/route.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.streams.$streamKey/route.tsx @@ -27,6 +27,7 @@ import { requireUserId } from "~/services/session.server"; import { cn } from "~/utils/cn"; import { v3RunStreamParamsSchema } from "~/utils/pathBuilder"; import { runStore } from "~/v3/runStore.server"; +import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server"; type ViewMode = "list" | "compact"; @@ -60,40 +61,34 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { } const run = await runStore.findRun( + { friendlyId: runParam, projectId: project.id }, { - friendlyId: runParam, - projectId: project.id, - }, - { - include: { - runtimeEnvironment: { - include: { - project: true, - organization: true, - orgMember: true, - }, - }, + select: { + id: true, + friendlyId: true, + realtimeStreamsVersion: true, + streamBasinName: true, + runtimeEnvironmentId: true, }, - }, - $replica + } ); if (!run) { throw new Response("Not Found", { status: 404 }); } - if (run.runtimeEnvironment.slug !== envParam) { + const environment = await controlPlaneResolver.resolveAuthenticatedEnv(run.runtimeEnvironmentId); + + if (!environment || environment.slug !== envParam) { throw new Response("Not Found", { status: 404 }); } // Get Last-Event-ID header for resuming from a specific position const lastEventId = request.headers.get("Last-Event-ID") || undefined; - const realtimeStream = getRealtimeStreamInstance( - run.runtimeEnvironment, - run.realtimeStreamsVersion, - { run } - ); + const realtimeStream = getRealtimeStreamInstance(environment, run.realtimeStreamsVersion, { + run: { streamBasinName: run.streamBasinName }, + }); return realtimeStream.streamResponse( request, @@ -211,7 +206,6 @@ export function RealtimeStreamViewer({ const handleScroll = () => { if (!scrollElement || !bottomElement) return; - // Clear any existing timeout if (scrollTimeout) { clearTimeout(scrollTimeout); } @@ -560,7 +554,6 @@ export function useRealtimeStream(resourcePath: string, startIndex?: number) { reader = stream.getReader(); - // Read from the stream while (true) { const { done, value } = await reader.read(); diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.waitpoints.$waitpointFriendlyId.complete/route.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.waitpoints.$waitpointFriendlyId.complete/route.tsx index fa065428471..9f3d90f1e25 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.waitpoints.$waitpointFriendlyId.complete/route.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.waitpoints.$waitpointFriendlyId.complete/route.tsx @@ -15,7 +15,8 @@ import { Paragraph } from "~/components/primitives/Paragraph"; import { SpinnerWhite } from "~/components/primitives/Spinner"; import { InfoIconTooltip } from "~/components/primitives/Tooltip"; import { LiveCountdown } from "~/components/runs/v3/LiveTimer"; -import { $replica } from "~/db.server"; +import { $replica, type PrismaReplicaClient } from "~/db.server"; +import { resolveWaitpointThroughReadThrough } from "~/runEngine/concerns/resolveWaitpointThroughReadThrough.server"; import { env } from "~/env.server"; import { useEnvironment } from "~/hooks/useEnvironment"; import { useOrganization } from "~/hooks/useOrganizations"; @@ -80,14 +81,19 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { const waitpointId = WaitpointId.toId(waitpointFriendlyId); - const waitpoint = await $replica.waitpoint.findFirst({ - select: { - projectId: true, - environmentId: true, - }, - where: { - id: waitpointId, - }, + const waitpoint = await resolveWaitpointThroughReadThrough({ + waitpointId, + environmentId: "", + read: (client: PrismaReplicaClient) => + client.waitpoint.findFirst({ + select: { + projectId: true, + environmentId: true, + }, + where: { + id: waitpointId, + }, + }), }); if (waitpoint?.projectId !== project.id) { diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.waitpoints.tags.ts b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.waitpoints.tags.ts index e8e5eea3643..d43b2f258b6 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.waitpoints.tags.ts +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.waitpoints.tags.ts @@ -4,6 +4,12 @@ import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; import { WaitpointTagListPresenter } from "~/presenters/v3/WaitpointTagListPresenter.server"; import { requireUserId } from "~/services/session.server"; +import { + runOpsNewReplicaClient, + runOpsLegacyReplica, + runOpsSplitReadEnabled, + type PrismaClientOrTransaction, +} from "~/db.server"; const Params = z.object({ organizationSlug: z.string(), @@ -28,7 +34,11 @@ export async function loader({ request, params }: LoaderFunctionArgs) { const search = new URL(request.url).searchParams; const name = search.get("name"); - const presenter = new WaitpointTagListPresenter(); + const presenter = new WaitpointTagListPresenter(undefined, undefined, { + runOpsNew: runOpsNewReplicaClient as unknown as PrismaClientOrTransaction, + runOpsLegacyReplica: runOpsLegacyReplica as unknown as PrismaClientOrTransaction, + splitEnabled: runOpsSplitReadEnabled, + }); const result = await presenter.call({ environmentId: environment.id, name: name ? decodeURIComponent(name) : undefined, diff --git a/apps/webapp/app/routes/resources.runs.$runParam.logs.download.ts b/apps/webapp/app/routes/resources.runs.$runParam.logs.download.ts index 99797e79b66..649f2ef268f 100644 --- a/apps/webapp/app/routes/resources.runs.$runParam.logs.download.ts +++ b/apps/webapp/app/routes/resources.runs.$runParam.logs.download.ts @@ -2,6 +2,7 @@ import type { LoaderFunctionArgs } from "@remix-run/server-runtime"; import { prisma } from "~/db.server"; import { env } from "~/env.server"; import { runStore } from "~/v3/runStore.server"; +import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server"; import { requireUser } from "~/services/session.server"; import { v3RunParamsSchema, v3RunPath } from "~/utils/pathBuilder"; import { createGzip } from "zlib"; @@ -27,19 +28,11 @@ export async function loader({ params, request }: LoaderFunctionArgs) { const showDebug = url.searchParams.get("showDebug") === "true" && user.admin; const filename = `${parsedParams.runParam}.${format.extension}`; - const run = await runStore.findRun( - { - friendlyId: parsedParams.runParam, - project: { - organization: { - members: { - some: { - userId: user.id, - }, - }, - }, - }, - }, + // Run-ops read keyed by friendlyId only (routes to the owning DB by residency). Org + // membership is a control-plane concern resolved separately below — joining it here is a + // cross-DB join that returns nothing once the run lives in run-ops. + let run = await runStore.findRun( + { friendlyId: parsedParams.runParam }, { select: { friendlyId: true, @@ -50,20 +43,28 @@ export async function loader({ params, request }: LoaderFunctionArgs) { completedAt: true, taskEventStore: true, taskIdentifier: true, - project: { select: { slug: true, organization: { select: { slug: true } } } }, - runtimeEnvironment: { select: { slug: true } }, }, - }, - prisma + } ); + // Authorize on the control-plane DB: the user must be a member of the run's org. A + // non-member is treated as not-found (matching the old scoped where) and falls through + // to the buffer fallback below. + if (run?.organizationId) { + const member = await prisma.orgMember.findFirst({ + where: { userId: user.id, organizationId: run.organizationId }, + select: { id: true }, + }); + if (!member) { + run = null; + } + } + if (!run || !run.organizationId) { - // Buffered run? It hasn't executed, so there's no trace to stream — but a - // 404 is wrong: the run does exist, the customer's "Download trace" button - // on the run-detail page generates this exact URL, and a 404 reads as "your - // run vanished" rather than "no trace yet". Verify the entry exists in the - // buffer (with the user as a member of the entry's org), and if so stream a - // single informational line instead of a 0-byte mystery. + // Buffered run? It hasn't executed, so there's no trace — but a 404 is wrong: + // the run does exist and reads as "your run vanished". If the buffer entry + // exists (and the user is a member of its org), stream one informational line + // instead of a 0-byte mystery. const buffer = getMollifierBuffer(); if (buffer) { const entry = await buffer.getEntry(parsedParams.runParam); @@ -83,6 +84,12 @@ export async function loader({ params, request }: LoaderFunctionArgs) { return new Response("Not found", { status: 404 }); } + const environment = await controlPlaneResolver.resolveAuthenticatedEnv(run.runtimeEnvironmentId); + + if (!environment) { + return new Response("Not found", { status: 404 }); + } + const eventRepository = await getEventRepositoryForStore(run.taskEventStore, run.organizationId); // Stream the trace straight from the store to the gzip response, one event at @@ -103,9 +110,9 @@ export async function loader({ params, request }: LoaderFunctionArgs) { traceId: run.traceId, taskIdentifier: run.taskIdentifier, runUrl: `${env.APP_ORIGIN}${v3RunPath( - run.project.organization, - run.project, - run.runtimeEnvironment, + environment.organization, + environment.project, + environment, { friendlyId: run.friendlyId } )}`, }; diff --git a/apps/webapp/app/routes/resources.runs.$runParam.ts b/apps/webapp/app/routes/resources.runs.$runParam.ts index ecd2e96a8f2..1384bcb4233 100644 --- a/apps/webapp/app/routes/resources.runs.$runParam.ts +++ b/apps/webapp/app/routes/resources.runs.$runParam.ts @@ -8,6 +8,7 @@ import { requireUserId } from "~/services/session.server"; import { v3RunParamsSchema } from "~/utils/pathBuilder"; import { machinePresetFromRun } from "~/v3/machinePresets.server"; import { runStore } from "~/v3/runStore.server"; +import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server"; import { FINAL_ATTEMPT_STATUSES, isFinalRunStatus } from "~/v3/taskStatus"; export type RunInspectorData = UseDataFunctionReturn; @@ -19,15 +20,6 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const run = await runStore.findRun( { friendlyId: parsedParams.runParam, - project: { - organization: { - members: { - some: { - userId, - }, - }, - }, - }, }, { select: { @@ -40,12 +32,10 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { isTest: true, runTags: true, machinePreset: true, - lockedToVersion: { - select: { - version: true, - sdkVersion: true, - }, - }, + runtimeEnvironmentId: true, + projectId: true, + lockedById: true, + lockedToVersionId: true, //status + duration status: true, startedAt: true, @@ -70,39 +60,11 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { baseCostInCents: true, costInCents: true, usageDurationMs: true, - //env - runtimeEnvironment: { - select: { id: true, slug: true, type: true }, - }, payload: true, payloadType: true, metadata: true, metadataType: true, maxAttempts: true, - project: { - include: { - organization: true, - }, - }, - lockedBy: { - select: { - filePath: true, - worker: { - select: { - deployment: { - select: { - friendlyId: true, - shortCode: true, - version: true, - runtime: true, - runtimeVersion: true, - git: true, - }, - }, - }, - }, - }, - }, parentTaskRun: { select: { friendlyId: true, @@ -114,18 +76,37 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { }, }, }, - }, - $replica + } ); if (!run) { throw new Response("Not found", { status: 404 }); } + const authorizedProject = await $replica.project.findFirst({ + where: { id: run.projectId, organization: { members: { some: { userId } } } }, + select: { id: true }, + }); + + if (!authorizedProject) { + throw new Response("Not found", { status: 404 }); + } + + const environment = await controlPlaneResolver.resolveAuthenticatedEnv(run.runtimeEnvironmentId); + + if (!environment) { + throw new Response("Run environment not found", { status: 404 }); + } + + const lockedWorker = await controlPlaneResolver.resolveRunLockedWorker({ + lockedById: run.lockedById, + lockedToVersionId: run.lockedToVersionId, + }); + const isFinished = isFinalRunStatus(run.status); const finishedAttempt = isFinished - ? await $replica.taskRunAttempt.findFirst({ + ? await runStore.findTaskRunAttempt({ select: { output: true, outputType: true, @@ -145,14 +126,14 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { finishedAttempt === null ? undefined : finishedAttempt.outputType === "application/store" - ? `/resources/packets/${run.runtimeEnvironment.id}/${finishedAttempt.output}` + ? `/resources/packets/${environment.id}/${finishedAttempt.output}` : typeof finishedAttempt.output !== "undefined" && finishedAttempt.output !== null ? await prettyPrintPacket(finishedAttempt.output, finishedAttempt.outputType ?? undefined) : undefined; const payload = run.payloadType === "application/store" - ? `/resources/packets/${run.runtimeEnvironment.id}/${run.payload}` + ? `/resources/packets/${environment.id}/${run.payload}` : typeof run.payload !== "undefined" && run.payload !== null ? await prettyPrintPacket(run.payload, run.payloadType ?? undefined) : undefined; @@ -173,7 +154,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { const context = { task: { id: run.taskIdentifier, - filePath: run.lockedBy?.filePath, + filePath: lockedWorker?.lockedBy?.filePath, exportName: "@deprecated", }, run: { @@ -187,7 +168,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { costInCents: run.costInCents, baseCostInCents: run.baseCostInCents, maxAttempts: run.maxAttempts ?? undefined, - version: run.lockedToVersion?.version, + version: lockedWorker?.lockedToVersion?.version, parentTaskRunId: run.parentTaskRun?.friendlyId ?? undefined, rootTaskRunId: run.rootTaskRun?.friendlyId ?? undefined, }, @@ -195,30 +176,30 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { name: run.queue, }, environment: { - id: run.runtimeEnvironment.id, - slug: run.runtimeEnvironment.slug, - type: run.runtimeEnvironment.type, + id: environment.id, + slug: environment.slug, + type: environment.type, }, organization: { - id: run.project.organization.id, - slug: run.project.organization.slug, - name: run.project.organization.title, + id: environment.organization.id, + slug: environment.organization.slug, + name: environment.organization.title, }, project: { - id: run.project.id, - ref: run.project.externalRef, - slug: run.project.slug, - name: run.project.name, + id: environment.project.id, + ref: environment.project.externalRef, + slug: environment.project.slug, + name: environment.project.name, }, machine: run.machinePreset ? machinePresetFromRun(run) : undefined, - deployment: run.lockedBy?.worker.deployment + deployment: lockedWorker?.lockedBy?.worker.deployment ? { - id: run.lockedBy.worker.deployment.friendlyId, - shortCode: run.lockedBy.worker.deployment.shortCode, - version: run.lockedBy.worker.deployment.version, - runtime: run.lockedBy.worker.deployment.runtime, - runtimeVersion: run.lockedBy.worker.deployment.runtimeVersion, - git: run.lockedBy.worker.deployment.git, + id: lockedWorker.lockedBy.worker.deployment.friendlyId, + shortCode: lockedWorker.lockedBy.worker.deployment.shortCode, + version: lockedWorker.lockedBy.worker.deployment.version, + runtime: lockedWorker.lockedBy.worker.deployment.runtime, + runtimeVersion: lockedWorker.lockedBy.worker.deployment.runtimeVersion, + git: lockedWorker.lockedBy.worker.deployment.git, } : undefined, }; @@ -235,10 +216,10 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { logsDeletedAt: run.logsDeletedAt, ttl: run.ttl, taskIdentifier: run.taskIdentifier, - version: run.lockedToVersion?.version, - sdkVersion: run.lockedToVersion?.sdkVersion, + version: lockedWorker?.lockedToVersion?.version, + sdkVersion: lockedWorker?.lockedToVersion?.sdkVersion, isTest: run.isTest, - environmentId: run.runtimeEnvironment.id, + environmentId: environment.id, schedule: await resolveSchedule(run.scheduleId ?? undefined), queue: { name: run.queue, diff --git a/apps/webapp/app/routes/resources.taskruns.$runParam.cancel.ts b/apps/webapp/app/routes/resources.taskruns.$runParam.cancel.ts index 672b9f99ba4..ce74becc7be 100644 --- a/apps/webapp/app/routes/resources.taskruns.$runParam.cancel.ts +++ b/apps/webapp/app/routes/resources.taskruns.$runParam.cancel.ts @@ -1,13 +1,14 @@ import { parseWithZod } from "@conform-to/zod"; import { json } from "@remix-run/node"; import { z } from "zod"; -import { $replica, prisma } from "~/db.server"; +import { prisma } from "~/db.server"; import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server"; import { logger } from "~/services/logger.server"; import { dashboardAction } from "~/services/routeBuilders/dashboardBuilder"; import { CancelTaskRunService } from "~/v3/services/cancelTaskRun.server"; import { getMollifierBuffer } from "~/v3/mollifier/mollifierBuffer.server"; import { runStore } from "~/v3/runStore.server"; +import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server"; export const cancelSchema = z.object({ redirectUrl: z.string(), @@ -21,13 +22,16 @@ const ParamSchema = z.object({ // user's role in it. The run may not be in Postgres yet (buffered during a // burst), so fall back to the buffer entry's org. async function resolveRunOrganizationId(runParam: string): Promise { + // Keyed by friendlyId only so the store routes to the owning run-ops DB. const run = await runStore.findRun( { friendlyId: runParam }, - { select: { project: { select: { organizationId: true } } } }, - $replica + { select: { runtimeEnvironmentId: true } } ); if (run) { - return run.project.organizationId; + const env = await controlPlaneResolver.resolveEnv(run.runtimeEnvironmentId); + if (env?.organizationId) { + return env.organizationId; + } } const buffer = getMollifierBuffer(); @@ -36,16 +40,20 @@ async function resolveRunOrganizationId(runParam: string): Promise `engine:runqueue:${key}`; @@ -149,6 +143,7 @@ export async function loader({ request, params }: LoaderFunctionArgs) { return typedjson({ engine: "V2", run, + environment, queueConcurrencyLimit, envConcurrencyLimit, queueCurrentConcurrency, diff --git a/apps/webapp/app/routes/resources.taskruns.$runParam.replay.ts b/apps/webapp/app/routes/resources.taskruns.$runParam.replay.ts index a231ef4dee8..77968ec149d 100644 --- a/apps/webapp/app/routes/resources.taskruns.$runParam.replay.ts +++ b/apps/webapp/app/routes/resources.taskruns.$runParam.replay.ts @@ -1,6 +1,7 @@ import { parseWithZod } from "@conform-to/zod"; import { json, type LoaderFunctionArgs } from "@remix-run/node"; import { type EnvironmentType, prettyPrintPacket } from "@trigger.dev/core/v3"; +import { type Prisma } from "@trigger.dev/database"; import { typedjson } from "remix-typedjson"; import { z } from "zod"; import { $replica, prisma } from "~/db.server"; @@ -25,6 +26,7 @@ import { queueTypeFromType } from "~/presenters/v3/QueueRetrievePresenter.server import { ReplayRunData } from "~/v3/replayTask"; import { RegionsPresenter } from "~/presenters/v3/RegionsPresenter.server"; import { runStore } from "~/v3/runStore.server"; +import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server"; const ParamSchema = z.object({ runParam: z.string(), @@ -42,8 +44,10 @@ export async function loader({ request, params }: LoaderFunctionArgs) { Object.fromEntries(new URL(request.url).searchParams) ); + // Run-ops read keyed by friendlyId only; project-scope + membership auth is + // resolved on the control plane below, keyed off the resolved run's projectId. let run = await runStore.findRun( - { friendlyId: runParam, project: { organization: { members: { some: { userId } } } } }, + { friendlyId: runParam }, { select: { payload: true, @@ -51,6 +55,7 @@ export async function loader({ request, params }: LoaderFunctionArgs) { seedMetadata: true, seedMetadataType: true, runtimeEnvironmentId: true, + projectId: true, concurrencyKey: true, maxAttempts: true, maxDurationInSeconds: true, @@ -62,46 +67,16 @@ export async function loader({ request, params }: LoaderFunctionArgs) { runTags: true, queue: true, taskIdentifier: true, - project: { - select: { - slug: true, - environments: { - select: { - id: true, - type: true, - slug: true, - branchName: true, - parentEnvironmentId: true, - orgMember: { - select: { - user: true, - }, - }, - }, - where: { - archivedAt: null, - OR: [ - { - type: { - in: ["PREVIEW", "STAGING", "PRODUCTION"], - }, - }, - { - type: "DEVELOPMENT", - orgMember: { - userId, - }, - }, - ], - }, - }, - }, - }, }, - }, - $replica + } ); + // project.environments is a project-rooted list read (not a single-env thread), + // so it stays a control-plane query keyed off the resolved run's projectId. + type ProjectWithEnvironments = NonNullable>>; + let projectEnvironments: ProjectWithEnvironments["environments"]; + let projectSlug: string; + let _synthetic: | (Awaited> & { __synth: true }) | undefined; @@ -125,40 +100,13 @@ export async function loader({ request, params }: LoaderFunctionArgs) { }); if (!buffered) throw new Response("Not Found", { status: 404 }); _synthetic = Object.assign(buffered, { __synth: true as const }); - // Scope the project lookup to the buffer entry's org as well as the - // env id. The prior `orgMember.findFirst` above confirms the user - // belongs to `entry.orgId`; pinning `organizationId` here means a - // malformed entry whose envId resolves to a different org can't leak - // that project's data through this loader. Mirrors the PG path's - // `project.organization.members.some.userId` scoping (lines 42-95) - // — the env filter and select shape are kept identical so the Replay - // dialog renders the same dropdown either way. - const orgProject = await $replica.project.findFirst({ - where: { - organizationId: entry.orgId, - environments: { some: { id: entry.envId } }, - }, - select: { - slug: true, - environments: { - select: { - id: true, - type: true, - slug: true, - branchName: true, - parentEnvironmentId: true, - orgMember: { select: { user: true } }, - }, - where: { - archivedAt: null, - OR: [ - { type: { in: ["PREVIEW", "STAGING", "PRODUCTION"] } }, - { type: "DEVELOPMENT", orgMember: { userId } }, - ], - }, - }, - }, - }); + // Pin the project lookup to the buffer entry's org (not just the env id) so a + // malformed entry whose envId resolves to a different org can't leak that + // project's data. Mirrors the PG path's org-membership scoping. + const orgProject = await loadProjectEnvironments( + { organizationId: entry.orgId, environments: { some: { id: entry.envId } } }, + userId + ); if (!orgProject) throw new Response("Not Found", { status: 404 }); run = { payload: buffered.payload, @@ -166,6 +114,7 @@ export async function loader({ request, params }: LoaderFunctionArgs) { seedMetadata: buffered.seedMetadata ?? null, seedMetadataType: buffered.seedMetadataType ?? null, runtimeEnvironmentId: entry.envId, + projectId: "", concurrencyKey: buffered.concurrencyKey ?? null, maxAttempts: buffered.maxAttempts ?? null, maxDurationInSeconds: buffered.maxDurationInSeconds ?? null, @@ -177,20 +126,30 @@ export async function loader({ request, params }: LoaderFunctionArgs) { runTags: buffered.runTags, queue: buffered.queue ?? "task/", taskIdentifier: buffered.taskIdentifier ?? "", - project: orgProject, } as unknown as typeof run; + projectEnvironments = orgProject.environments; + projectSlug = orgProject.slug; + } else { + // PG path: the run resolved from the run-ops store; fetch the env list from + // the control plane via projectId, re-applying the org-membership gate that + // used to live on the run-store join so a miss stays a 404. + const project = await loadProjectEnvironments( + { id: run.projectId, organization: { members: { some: { userId } } } }, + userId + ); + if (!project) { + throw new Response("Not Found", { status: 404 }); + } + projectEnvironments = project.environments; + projectSlug = project.slug; } if (!run) { throw new Response("Not Found", { status: 404 }); } - const runEnvironment = run.project.environments.find( - (env) => env.id === run.runtimeEnvironmentId - ); - const environmentOverride = run.project.environments.find( - (env) => env.id === environmentIdOverride - ); + const runEnvironment = projectEnvironments.find((env) => env.id === run.runtimeEnvironmentId); + const environmentOverride = projectEnvironments.find((env) => env.id === environmentIdOverride); const environment = environmentOverride ?? runEnvironment; if (!environment) { throw new Response("Environment not found", { status: 404 }); @@ -209,7 +168,7 @@ export async function loader({ request, params }: LoaderFunctionArgs) { prettyPrintPacket(run.payload, run.payloadType), new RegionsPresenter().call({ userId, - projectSlug: run.project.slug, + projectSlug, isAdmin: user.admin || user.isImpersonating, }), ]); @@ -249,7 +208,7 @@ export async function loader({ request, params }: LoaderFunctionArgs) { branchName: environment.branchName ?? undefined, }, environments: sortEnvironments( - run.project.environments + projectEnvironments .filter((env) => env.type !== "PREVIEW" || env.parentEnvironmentId !== null) .map((env) => ({ ...displayableEnvironment(env, userId), @@ -259,17 +218,62 @@ export async function loader({ request, params }: LoaderFunctionArgs) { }); } +// Project-rooted control-plane list read of the replay env dropdown. +function loadProjectEnvironments(where: Prisma.ProjectWhereInput, userId: string) { + return $replica.project.findFirst({ + where, + select: { + slug: true, + environments: { + select: { + id: true, + type: true, + slug: true, + branchName: true, + parentEnvironmentId: true, + orgMember: { + select: { + user: true, + }, + }, + }, + where: { + archivedAt: null, + OR: [ + { + type: { + in: ["PREVIEW", "STAGING", "PRODUCTION"], + }, + }, + { + type: "DEVELOPMENT", + orgMember: { + userId, + }, + }, + ], + }, + }, + }, + }); +} + // Resolve the run's organization so the RBAC auth scope can resolve the // user's role in it. The run may not be in Postgres yet (buffered during a // burst), so fall back to the buffer entry's org. async function resolveRunOrganizationId(runParam: string): Promise { + // Run-ops read keyed by friendlyId only; the store routes to the owning DB by + // residency off its own replica. Forwarding a control-plane client here would + // override that routing and miss any run that lives in the run-ops DB. const run = await runStore.findRun( { friendlyId: runParam }, - { select: { project: { select: { organizationId: true } } } }, - $replica + { select: { runtimeEnvironmentId: true } } ); if (run) { - return run.project.organizationId; + const env = await controlPlaneResolver.resolveEnv(run.runtimeEnvironmentId); + if (env?.organizationId) { + return env.organizationId; + } } const buffer = getMollifierBuffer(); @@ -278,16 +282,7 @@ async function resolveRunOrganizationId(runParam: string): Promise[0]["crossSeamGuard"]; + +afterEach(() => { + // Never let the ksuid mint mode leak into other webapp tests. + setKsuidMintEnabled(false); +}); + +function buildEngine(opts: { + prisma: any; + redisOptions: any; + crossSeamGuard?: CrossSeamGuard; +}) { + return new RunEngine({ + prisma: opts.prisma, + ...(opts.crossSeamGuard ? { crossSeamGuard: opts.crossSeamGuard } : {}), + worker: { + redis: opts.redisOptions, + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + queue: { + redis: opts.redisOptions, + }, + runLock: { + redis: opts.redisOptions, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0005, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); +} + +describe("waitpoint-token complete route — cross-seam guard", () => { + // (A) the completion path consults the guard FIRST with routeKind RESUME_TOKEN + // recording the waitpointId, then delegates and the waitpoint becomes COMPLETED. + containerTest( + "consults the guard first (RESUME_TOKEN), then completes (single-store)", + async ({ prisma, redisOptions }) => { + setKsuidMintEnabled(true); + + const seen: Array<{ waitpointId: string; routeKind: string }> = []; + const engine = buildEngine({ + prisma, + redisOptions, + crossSeamGuard: async ({ waitpointId, routeKind }) => { + seen.push({ waitpointId, routeKind }); + // Single-store / split-OFF returns the single ("legacy") store; the + // engine delegates regardless of decision.store. + return { store: "legacy", residency: "LEGACY", routeKind }; + }, + }); + + try { + const env = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const { waitpoint } = await engine.createManualWaitpoint({ + environmentId: env.id, + projectId: env.project.id, + }); + expect(waitpoint.status).toBe("PENDING"); + + await engine.completeWaitpoint({ + id: waitpoint.id, + output: { value: "{}", isError: false }, + }); + + // The guard was consulted first, with the right id + RESUME_TOKEN route kind. + expect(seen).toEqual([{ waitpointId: waitpoint.id, routeKind: "RESUME_TOKEN" }]); + + // The completion was then applied via delegation (single-store path). + const after = await prisma.waitpoint.findFirst({ where: { id: waitpoint.id } }); + expect(after?.status).toBe("COMPLETED"); + } finally { + await engine.quit(); + } + } + ); + + // (B) an injected guard that throws (unclassifiable) causes completeWaitpoint + // to reject and the waitpoint stays PENDING (loud, not silently applied). + containerTest( + "propagates a guard throw and leaves the waitpoint PENDING (loud)", + async ({ prisma, redisOptions }) => { + setKsuidMintEnabled(true); + + const engine = buildEngine({ + prisma, + redisOptions, + crossSeamGuard: async () => { + throw new Error("UnclassifiableRunId"); + }, + }); + + try { + const env = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const { waitpoint } = await engine.createManualWaitpoint({ + environmentId: env.id, + projectId: env.project.id, + }); + expect(waitpoint.status).toBe("PENDING"); + + await expect( + engine.completeWaitpoint({ id: waitpoint.id, output: { value: "{}", isError: false } }) + ).rejects.toThrow(); + + // The throw short-circuited before delegation — no silent local apply. + const after = await prisma.waitpoint.findFirst({ where: { id: waitpoint.id } }); + expect(after?.status).toBe("PENDING"); + } finally { + await engine.quit(); + } + } + ); +}); + +// (D) no-FK-abort: with the Waitpoint table split off control-plane, the env/project Cascade +// FKs are physically absent. Completing a waitpoint (status flip) must not trip a now-missing FK +// on EITHER the PG14 (legacy) or PG17 (new) store. Seed + complete on the SAME store (single-store +// write, no two-store router). The DB is never mocked: writes hit the real PG14/PG17 containers. +const WAITPOINT_CROSS_SEAM_FKS = [ + "Waitpoint_environmentId_fkey", + "Waitpoint_projectId_fkey", +] as const; + +async function dropWaitpointCrossSeamFks(prisma: PrismaClient) { + for (const c of WAITPOINT_CROSS_SEAM_FKS) { + await prisma.$executeRawUnsafe(`ALTER TABLE "Waitpoint" DROP CONSTRAINT IF EXISTS "${c}"`); + } +} + +let waitpointSeq = 0; + +// Seed a PENDING MANUAL waitpoint, then complete it (status flip) on the SAME store. +async function seedAndCompleteOnStore(store: PrismaClient) { + const s = waitpointSeq++; + const { id, friendlyId } = WaitpointId.generate(); + await store.waitpoint.create({ + data: { + id, + friendlyId, + type: "MANUAL", + status: "PENDING", + idempotencyKey: `idem_nofk_${s}`, + userProvidedIdempotencyKey: false, + // No matching env/project row on this store (they'd live on the other DB). + environmentId: `env_other_db_${s}`, + projectId: `proj_other_db_${s}`, + }, + }); + + await store.waitpoint.updateMany({ + where: { id, status: "PENDING" }, + data: { + status: "COMPLETED", + completedAt: new Date(), + output: JSON.stringify({ value: "{}" }), + outputType: "application/json", + outputIsError: false, + }, + }); + + return store.waitpoint.findFirst({ where: { id }, select: { id: true, status: true } }); +} + +describe("waitpoint-token complete route — no FK abort across the PG14<->17 boundary", () => { + heteroPostgresTest( + "completes a run-ops waitpoint on each version store without tripping the absent control-plane Cascade FK", + async ({ prisma14, prisma17 }) => { + setKsuidMintEnabled(true); + + const legacy = prisma14 as unknown as PrismaClient; + const next = prisma17 as unknown as PrismaClient; + + await dropWaitpointCrossSeamFks(legacy); + const completedOnLegacy = await seedAndCompleteOnStore(legacy); + expect(completedOnLegacy).not.toBeNull(); + expect(completedOnLegacy!.status).toBe("COMPLETED"); + + await dropWaitpointCrossSeamFks(next); + const completedOnNew = await seedAndCompleteOnStore(next); + expect(completedOnNew).not.toBeNull(); + expect(completedOnNew!.status).toBe("COMPLETED"); + } + ); +}); diff --git a/apps/webapp/test/api.v1.waitpoints.tokens.test.ts b/apps/webapp/test/api.v1.waitpoints.tokens.test.ts new file mode 100644 index 00000000000..73350230103 --- /dev/null +++ b/apps/webapp/test/api.v1.waitpoints.tokens.test.ts @@ -0,0 +1,258 @@ +import { describe, expect, afterEach, vi } from "vitest"; + +// These tests exercise the store-routed engine create/get seam and the +// residency-keyed id contract (the same id-shaping the create route's response +// uses). They do NOT drive the route's HTTP action — only the engine create/get +// seam behind it. + +import { RunEngine } from "@internal/run-engine"; +import { setupAuthenticatedEnvironment } from "@internal/run-engine/tests"; +import { containerTest, heteroPostgresTest } from "@internal/testcontainers"; +import { PostgresRunStore } from "@internal/run-store"; +import { + WaitpointId, + setKsuidMintEnabled, + ownerEngine, + KSUID_LENGTH, +} from "@trigger.dev/core/v3/isomorphic"; +import { Prisma } from "@trigger.dev/database"; +import { trace } from "@opentelemetry/api"; +import { nanoid } from "nanoid"; + +vi.setConfig({ testTimeout: 60_000 }); + +afterEach(() => { + // Never let the ksuid mint mode leak into other webapp tests. + setKsuidMintEnabled(false); +}); + +function buildEngine(opts: { + prisma: any; + redisOptions: any; + store?: ConstructorParameters[0]["store"]; +}) { + return new RunEngine({ + prisma: opts.prisma, + ...(opts.store ? { store: opts.store } : {}), + worker: { + redis: opts.redisOptions, + workers: 1, + tasksPerWorker: 10, + pollIntervalMs: 100, + }, + queue: { + redis: opts.redisOptions, + }, + runLock: { + redis: opts.redisOptions, + }, + machines: { + defaultMachine: "small-1x", + machines: { + "small-1x": { + name: "small-1x" as const, + cpu: 0.5, + memory: 0.5, + centsPerMs: 0.0001, + }, + }, + baseCostInCents: 0.0005, + }, + tracer: trace.getTracer("test", "0.0.0"), + }); +} + +describe("waitpoint-token create engine seam — residency-keyed id contract", () => { + // Test A: the create seam mints a KSUID WaitpointId on the run-ops engine. + containerTest( + "create mints a KSUID WaitpointId on the run-ops engine", + async ({ prisma, redisOptions }) => { + setKsuidMintEnabled(true); + + const engine = buildEngine({ prisma, redisOptions }); + + try { + const env = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const result = await engine.createManualWaitpoint({ + environmentId: env.id, + projectId: env.project.id, + timeout: new Date(Date.now() + 60_000), + }); + + // `WaitpointId.generate()` returns a bare (un-prefixed) internal id, so + // result.waitpoint.id is the raw 27-char ksuid body. + expect(result.waitpoint.id.length).toBe(KSUID_LENGTH); + expect(result.waitpoint.type).toBe("MANUAL"); + + // The waitpoint row exists on the run-ops store (single-DB: the container prisma). + const row = await prisma.waitpoint.findUnique({ where: { id: result.waitpoint.id } }); + expect(row).not.toBeNull(); + expect(row?.environmentId).toBe(env.id); + + // The exact response body id, computed as the route computes it. + const responseId = WaitpointId.toFriendlyId(result.waitpoint.id); + expect(responseId.startsWith("waitpoint_")).toBe(true); + expect(responseId).toBe("waitpoint_" + result.waitpoint.id); + expect(WaitpointId.fromFriendlyId(responseId)).toBe(result.waitpoint.id); + + // The id a client receives stays residency-classifiable to the owning + // store — the contract the completion route relies on to resolve the token. + expect(ownerEngine(WaitpointId.fromFriendlyId(responseId))).toBe("NEW"); + } finally { + await engine.quit(); + } + } + ); + + // Test B: the token id classifies to the owning (new) run-ops store and resolves back. + containerTest( + "token id classifies to the owning run-ops store and resolves back", + async ({ prisma, redisOptions }) => { + setKsuidMintEnabled(true); + + const engine = buildEngine({ prisma, redisOptions }); + + try { + const env = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + const result = await engine.createManualWaitpoint({ + environmentId: env.id, + projectId: env.project.id, + timeout: new Date(Date.now() + 60_000), + }); + + // A ksuid token id classifies to the NEW (run-ops) store. + expect(ownerEngine(result.waitpoint.id)).toBe("NEW"); + + // getWaitpoint resolves via this.runStore.findWaitpoint and returns the exact row. + const resolved = await engine.getWaitpoint({ + waitpointId: result.waitpoint.id, + environmentId: env.id, + projectId: env.project.id, + }); + expect(resolved).not.toBeNull(); + expect(resolved?.id).toBe(result.waitpoint.id); + } finally { + await engine.quit(); + } + } + ); + + // Test C: the control-plane WaitpointTag write stays control-plane — it cannot + // route through the run-ops store, which exposes no tag-write surface at all. + containerTest( + "control-plane WaitpointTag write stays control-plane, not on the run-ops store", + async ({ prisma, redisOptions }) => { + setKsuidMintEnabled(true); + + // A run-ops store that counts every waitpoint write that passes through it. + // Overrides mirror the base PostgresRunStore generics so a base-signature + // change can't silently detach the counter. + let waitpointWrites = 0; + class CountingPostgresRunStore extends PostgresRunStore { + async upsertWaitpoint( + args: Prisma.SelectSubset, + tx?: Parameters[1] + ): Promise> { + waitpointWrites++; + return super.upsertWaitpoint(args, tx); + } + async createWaitpoint( + args: Prisma.SelectSubset, + tx?: Parameters[1] + ): Promise> { + waitpointWrites++; + return super.createWaitpoint(args, tx); + } + } + + const countingStore = new CountingPostgresRunStore({ + prisma, + readOnlyPrisma: prisma, + }); + + const engine = buildEngine({ prisma, redisOptions, store: countingStore }); + + try { + const env = await setupAuthenticatedEnvironment(prisma, "PRODUCTION"); + + // Issue the control-plane tag write directly (the same upsert the + // createWaitpointTag model performs), against the container prisma — + // control-plane, never the run-ops store. + await prisma.waitpointTag.upsert({ + where: { environmentId_name: { environmentId: env.id, name: "t1" } }, + create: { name: "t1", environmentId: env.id, projectId: env.project.id }, + update: {}, + }); + + const result = await engine.createManualWaitpoint({ + environmentId: env.id, + projectId: env.project.id, + tags: ["t1"], + timeout: new Date(Date.now() + 60_000), + }); + expect(result.waitpoint.id.length).toBe(KSUID_LENGTH); + + // The tag landed on the control-plane client. + const tagRow = await prisma.waitpointTag.findFirst({ + where: { environmentId: env.id, name: "t1" }, + }); + expect(tagRow).not.toBeNull(); + + // The waitpoint went through the run-ops store (counting store fired). + expect(waitpointWrites).toBeGreaterThanOrEqual(1); + + // The run-ops store has no tag-write surface, so the partition rests on the + // two assertions above: the tag landed on control-plane and the waitpoint went through the store. + } finally { + await engine.quit(); + } + } + ); + + // Test D: a minted KSUID waitpoint resolves only to its owning store across the + // PG14↔PG17 version boundary. Store-only — no Redis/engine needed. + heteroPostgresTest( + "minted WaitpointId resolves only to its owning run-ops store across the version boundary", + async ({ prisma14, prisma17 }) => { + setKsuidMintEnabled(true); + + const store17 = new PostgresRunStore({ prisma: prisma17, readOnlyPrisma: prisma17 }); + const store14 = new PostgresRunStore({ prisma: prisma14, readOnlyPrisma: prisma14 }); + + const env = await setupAuthenticatedEnvironment(prisma17, "PRODUCTION"); + + const idempotencyKey = nanoid(24); + const generated = WaitpointId.generate(); + + const created = await store17.upsertWaitpoint({ + where: { + environmentId_idempotencyKey: { environmentId: env.id, idempotencyKey }, + }, + create: { + ...generated, + type: "MANUAL", + idempotencyKey, + userProvidedIdempotencyKey: false, + environmentId: env.id, + projectId: env.project.id, + }, + update: {}, + }); + + const id = created.id; + expect(id.length).toBe(KSUID_LENGTH); + expect(ownerEngine(id)).toBe("NEW"); + + // Byte-identical id resolves on the PG17 run-ops home. + const found17 = await store17.findWaitpoint({ where: { id } }, prisma17); + expect(found17).not.toBeNull(); + expect(found17?.id).toBe(id); + + // Residency invariant: the same id does NOT resolve on the PG14 legacy store. + const found14 = await store14.findWaitpoint({ where: { id } }, prisma14); + expect(found14).toBeNull(); + } + ); +}); diff --git a/apps/webapp/test/crossSeamGuard.proof.test.ts b/apps/webapp/test/crossSeamGuard.proof.test.ts new file mode 100644 index 00000000000..3b5a033779f --- /dev/null +++ b/apps/webapp/test/crossSeamGuard.proof.test.ts @@ -0,0 +1,215 @@ +import { heteroPostgresTest } from "@internal/testcontainers"; +import { PrismaClient } from "@trigger.dev/database"; +import { existsSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { + computeStoreForCompletion, + selectStoreForWaitpoint, +} from "~/v3/runOpsMigration/crossSeamGuard.server"; +import { + expectedCompleteWaitpointCallSites, + UNBLOCK_ROUTES, +} from "~/v3/runOpsMigration/unblockRouteCatalog"; +import { WaitpointId } from "@trigger.dev/core/v3/isomorphic"; + +const NEW_WP = WaitpointId.toFriendlyId("0".repeat(27)); // 27-char internal body → NEW +const LEGACY_WP = WaitpointId.toFriendlyId("c".repeat(25)); // 25-char internal body → LEGACY + +describe("cross-seam guard — exhaustive per-route store selection (Leg 1)", () => { + for (const route of UNBLOCK_ROUTES) { + it(`routes ${route.id} (${route.kind}) to new store for a NEW waitpoint`, () => { + const d = selectStoreForWaitpoint({ waitpointId: NEW_WP, routeKind: route.kind }); + expect(d.store).toBe("new"); + expect(d.residency).toBe("NEW"); + }); + + it(`routes ${route.id} (${route.kind}) to legacy store for a LEGACY waitpoint`, () => { + const d = selectStoreForWaitpoint({ waitpointId: LEGACY_WP, routeKind: route.kind }); + expect(d.store).toBe("legacy"); + expect(d.residency).toBe("LEGACY"); + }); + } +}); + +describe("cross-seam guard — single-DB no-op (Leg 2)", () => { + for (const route of UNBLOCK_ROUTES) { + it(`${route.id}: single-DB returns legacy without consulting the classifier`, () => { + const calls: string[] = []; + const d = computeStoreForCompletion( + { waitpointId: "anything-even-unclassifiable", routeKind: route.kind }, + { splitEnabled: false, classify: (id) => (calls.push(id), "NEW") } + ); + expect(d.store).toBe("legacy"); // the single store + expect(calls).toEqual([]); // classifier never consulted + }); + } +}); + +const ENGINE_FILES = [ + "internal-packages/run-engine/src/engine/index.ts", + "internal-packages/run-engine/src/engine/systems/waitpointSystem.ts", + "internal-packages/run-engine/src/engine/systems/ttlSystem.ts", + "internal-packages/run-engine/src/engine/systems/runAttemptSystem.ts", + "internal-packages/run-engine/src/engine/systems/batchSystem.ts", +]; + +function repoRoot(): string { + let dir = process.cwd(); + while (!existsSync(path.join(dir, "pnpm-workspace.yaml"))) { + const parent = path.dirname(dir); + if (parent === dir) throw new Error("repo root (pnpm-workspace.yaml) not found"); + dir = parent; + } + return dir; +} + +function tally(sites: string[]): Record { + const counts: Record = {}; + for (const site of sites) counts[site] = (counts[site] ?? 0) + 1; + return counts; +} + +describe("cross-seam guard — CI drift guard (Leg 3)", () => { + it("per-file completeWaitpoint( tally in source matches the catalog", () => { + const root = repoRoot(); + + // The regex matches tokens inside comments too — deliberate. Any textual + // addition forces catalog reconciliation, so a new call site cannot land + // without a matching entry. + const liveSites: string[] = []; + for (const file of ENGINE_FILES) { + const src = readFileSync(path.join(root, file), "utf8"); + const hits = (src.match(/completeWaitpoint\(/g) ?? []).length; + for (let i = 0; i < hits; i++) liveSites.push(file); + } + + const cataloguedSites = expectedCompleteWaitpointCallSites().map((s) => s.site); + + expect(tally(liveSites)).toEqual(tally(cataloguedSites)); + }); +}); + +// --------------------------------------------------------------------------- +// Leg 4 — PG14+PG17 hetero-fixture proof. The pure legs above prove the guard +// SELECTS the right store; this leg proves the selected store corresponds to the +// DB the Waitpoint row PHYSICALLY lives in, on a REAL heterogeneous PG14+PG17 +// fixture. NEVER mock. Seed Org->Project->Env (parents before children, or the +// required Waitpoint.projectId/environmentId FKs abort the insert) then the +// Waitpoint, on the matching DB ONLY: NEW residency on PG17, LEGACY on PG14. The +// cross-DB toBeNull checks then prove no ghost row leaked to the other version. +// Seed pattern copied from +// internal-packages/run-engine/src/engine/tests/crossVersionCompat.test.ts. +// --------------------------------------------------------------------------- + +const FIXED_TS = "2024-01-01 00:00:00+00"; + +async function seedOrgProjectEnv( + p: PrismaClient, + ids: { orgId: string; projectId: string; envId: string } +): Promise { + await p.$executeRawUnsafe( + `INSERT INTO "Organization" ("id","slug","title","createdAt","updatedAt") + VALUES ($1,$2,$3,$4::timestamptz,$4::timestamptz)`, + ids.orgId, + `${ids.orgId}-slug`, + `${ids.orgId}-title`, + FIXED_TS + ); + + await p.$executeRawUnsafe( + `INSERT INTO "Project" ("id","slug","name","externalRef","organizationId","createdAt","updatedAt") + VALUES ($1,$2,$3,$4,$5,$6::timestamptz,$6::timestamptz)`, + ids.projectId, + `${ids.projectId}-slug`, + `${ids.projectId}-name`, + `${ids.projectId}-ref`, + ids.orgId, + FIXED_TS + ); + + await p.$executeRawUnsafe( + `INSERT INTO "RuntimeEnvironment" + ("id","slug","apiKey","pkApiKey","shortcode","type","organizationId","projectId","createdAt","updatedAt") + VALUES ($1,$2,$3,$4,$5,'DEVELOPMENT',$6,$7,$8::timestamptz,$8::timestamptz)`, + ids.envId, + `${ids.envId}-slug`, + `${ids.envId}-apikey`, + `${ids.envId}-pkapikey`, + `${ids.envId}-short`, + ids.orgId, + ids.projectId, + FIXED_TS + ); +} + +async function seedWaitpoint( + p: PrismaClient, + ids: { waitpointId: string; projectId: string; envId: string; idempotencyKey: string } +): Promise { + await p.$executeRawUnsafe( + `INSERT INTO "Waitpoint" + ("id","friendlyId","type","status","idempotencyKey","userProvidedIdempotencyKey", + "projectId","environmentId","createdAt","updatedAt") + VALUES ($1,$2,'MANUAL','PENDING',$3,false,$4,$5,$6::timestamptz,$6::timestamptz)`, + ids.waitpointId, + `${ids.waitpointId}-friendly`, + ids.idempotencyKey, + ids.projectId, + ids.envId, + FIXED_TS + ); +} + +describe("cross-seam guard — PG14+PG17 hetero-fixture proof (Leg 4)", () => { + heteroPostgresTest( + "exhaustive routes resolve to the physically-correct store on PG14+PG17", + async ({ prisma14, prisma17 }) => { + const newWp = WaitpointId.toFriendlyId("0".repeat(27)); // NEW → PG17 + const legacyWp = WaitpointId.toFriendlyId("c".repeat(25)); // LEGACY → PG14 + + // Distinct parent chains per residency; each Waitpoint lives on its own DB + // ONLY so the cross-DB ghost assertions (toBeNull on the other version) hold. + await seedOrgProjectEnv(prisma17, { + orgId: "org_csm_new_0000000000000000000", + projectId: "proj_csm_new_00000000000000000", + envId: "env_csm_new_0000000000000000000", + }); + await seedWaitpoint(prisma17, { + waitpointId: newWp, + projectId: "proj_csm_new_00000000000000000", + envId: "env_csm_new_0000000000000000000", + idempotencyKey: "idem_csm_new", + }); + + await seedOrgProjectEnv(prisma14, { + orgId: "org_csm_legacy_0000000000000000", + projectId: "proj_csm_legacy_000000000000000", + envId: "env_csm_legacy_0000000000000000", + }); + await seedWaitpoint(prisma14, { + waitpointId: legacyWp, + projectId: "proj_csm_legacy_000000000000000", + envId: "env_csm_legacy_0000000000000000", + idempotencyKey: "idem_csm_legacy", + }); + + // Exhaustive over every unblock route: the selected store must match the + // DB the row physically lives in, with no cross-DB ghost in either direction. + for (const route of UNBLOCK_ROUTES) { + const dNew = selectStoreForWaitpoint({ waitpointId: newWp, routeKind: route.kind }); + expect(dNew.store).toBe("new"); + expect(await prisma17.waitpoint.findFirst({ where: { id: newWp } })).not.toBeNull(); + expect(await prisma14.waitpoint.findFirst({ where: { id: newWp } })).toBeNull(); + + const dLegacy = selectStoreForWaitpoint({ waitpointId: legacyWp, routeKind: route.kind }); + expect(dLegacy.store).toBe("legacy"); + expect(await prisma14.waitpoint.findFirst({ where: { id: legacyWp } })).not.toBeNull(); + expect(await prisma17.waitpoint.findFirst({ where: { id: legacyWp } })).toBeNull(); + } + }, + // First run boots two real Postgres containers (PG14 + PG17); the default + // 5s per-test timeout is far too short for the cold image pull + start. + 120_000 + ); +}); diff --git a/apps/webapp/test/waitpointCallback.controlPlane.test.ts b/apps/webapp/test/waitpointCallback.controlPlane.test.ts new file mode 100644 index 00000000000..80830a1b2a0 --- /dev/null +++ b/apps/webapp/test/waitpointCallback.controlPlane.test.ts @@ -0,0 +1,324 @@ +// Real PG14 (control-plane) + PG17 (run-ops) proof for the HTTP-callback waitpoint +// completion route after it was decomposed onto the ControlPlaneResolver. The waitpoint +// scalar row lives on PG17 (run-ops); the env (apiKey/project/org) + its branch parent live +// on PG14 (control-plane), with the cross-seam Waitpoint FKs dropped. The route reads the +// waitpoint scalars from run-ops and resolves the authenticated env (including the parent +// apiKey the hash check uses) from control-plane. The DB is never mocked; the .count() proof +// shows neither DB joins the other. +import { heteroPostgresTest } from "@internal/testcontainers"; +import { describe, expect, vi } from "vitest"; + +// The route under test reads the waitpoint scalars off the `$replica` singleton and resolves the +// authenticated env off the module-level `controlPlaneResolver` singleton, which reads the `prisma` +// singleton (split off -> controlPlanePrimary). We point each `~/db.server` proxy at the REAL +// container holding that data: `$replica` -> run-ops (PG17), `prisma` -> control-plane (PG14). The +// DB is NEVER mocked: the proxies forward to real testcontainer clients. (The route module also +// imports the run engine, but the branches exercised here all return before `engine.completeWaitpoint`.) +const replicaHolder = vi.hoisted(() => ({ client: undefined as any })); +const primaryHolder = vi.hoisted(() => ({ client: undefined as any })); + +vi.mock("~/db.server", async () => { + const { Prisma } = await import("@trigger.dev/database"); + const lazyProxy = (holder: { client: any }, label: string) => + new Proxy( + {}, + { + get(_t, prop) { + if (!holder.client) { + throw new Error(`${label} not set for this test`); + } + return holder.client[prop]; + }, + } + ); + return { + prisma: lazyProxy(primaryHolder, "primaryHolder.client"), + $replica: lazyProxy(replicaHolder, "replicaHolder.client"), + runOpsNewPrisma: lazyProxy(replicaHolder, "replicaHolder.client"), + runOpsNewReplica: lazyProxy(replicaHolder, "replicaHolder.client"), + runOpsLegacyReplica: lazyProxy(replicaHolder, "replicaHolder.client"), + sqlDatabaseSchema: Prisma.sql([`public`]), + }; +}); + +import type { PrismaClient } from "@trigger.dev/database"; +import { WaitpointId } from "@trigger.dev/core/v3/isomorphic"; +import { action } from "~/routes/api.v1.waitpoints.tokens.$waitpointFriendlyId.callback.$hash"; +import { generateHttpCallbackUrl } from "~/services/httpCallback.server"; +import { ControlPlaneCache } from "~/v3/runOpsMigration/controlPlaneCache.server"; +import { ControlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server"; + +vi.setConfig({ testTimeout: 60_000, hookTimeout: 60_000 }); + +function callbackRequest(body: unknown) { + const payload = JSON.stringify(body); + return new Request("http://localhost/callback", { + method: "POST", + headers: { "content-type": "application/json", "content-length": String(payload.length) }, + body: payload, + }); +} + +// Derives the same hash `verifyHttpCallbackHash` checks, via the production URL helper. +function hashFor(waitpointId: string, apiKey: string) { + const url = generateHttpCallbackUrl(waitpointId, apiKey); + return url.split("/").pop()!; +} + +const WAITPOINT_CROSS_SEAM_FKS = [ + "Waitpoint_environmentId_fkey", + "Waitpoint_projectId_fkey", +] as const; + +async function dropWaitpointCrossSeamFks(prisma: PrismaClient) { + for (const c of WAITPOINT_CROSS_SEAM_FKS) { + await prisma.$executeRawUnsafe(`ALTER TABLE "Waitpoint" DROP CONSTRAINT IF EXISTS "${c}"`); + } +} + +let n = 0; +async function seedControlPlane(prisma: PrismaClient, opts?: { withParent?: boolean }) { + const s = n++; + const organization = await prisma.organization.create({ + data: { title: `Org ${s}`, slug: `org-${s}` }, + }); + const project = await prisma.project.create({ + data: { + name: `P ${s}`, + slug: `p-${s}`, + externalRef: `proj_${s}`, + organizationId: organization.id, + }, + }); + const parent = opts?.withParent + ? await prisma.runtimeEnvironment.create({ + data: { + type: "PREVIEW", + slug: `preview-parent-${s}`, + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_parent_${s}`, + pkApiKey: `pk_parent_${s}`, + shortcode: `sc_parent_${s}`, + }, + }) + : null; + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: opts?.withParent ? "PREVIEW" : "PRODUCTION", + slug: `env-${s}`, + branchName: opts?.withParent ? "feat/x" : null, + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_${s}`, + pkApiKey: `pk_${s}`, + shortcode: `sc_${s}`, + parentEnvironmentId: parent?.id ?? null, + }, + }); + return { organization, project, environment, parent }; +} + +async function seedWaitpoint( + prisma: PrismaClient, + ctx: { environmentId: string; projectId: string } +) { + const s = n++; + return prisma.waitpoint.create({ + data: { + id: `waitpoint_${s}_pg17`, + friendlyId: `waitpoint_fr_${s}`, + type: "MANUAL", + status: "PENDING", + idempotencyKey: `idem_${s}`, + userProvidedIdempotencyKey: false, + environmentId: ctx.environmentId, + projectId: ctx.projectId, + }, + }); +} + +describe("waitpoint HTTP-callback cross-DB read-through", () => { + heteroPostgresTest( + "waitpoint resolves from run-ops; env apiKey resolves from control-plane", + async ({ prisma14, prisma17 }) => { + await dropWaitpointCrossSeamFks(prisma17 as unknown as PrismaClient); + const cp = await seedControlPlane(prisma14 as unknown as PrismaClient); + const waitpoint = await seedWaitpoint(prisma17 as unknown as PrismaClient, { + environmentId: cp.environment.id, + projectId: cp.project.id, + }); + + // Run-ops read: waitpoint scalars only, no environment relation. + const found = await (prisma17 as unknown as PrismaClient).waitpoint.findFirst({ + where: { id: waitpoint.id }, + select: { id: true, status: true, environmentId: true }, + }); + expect(found).not.toBeNull(); + expect(found!.environmentId).toBe(cp.environment.id); + + // Control-plane resolution of the authenticated env (passthrough mode). + const resolver = new ControlPlaneResolver({ + controlPlanePrimary: prisma14 as unknown as PrismaClient, + controlPlaneReplica: prisma14 as unknown as PrismaClient, + cache: new ControlPlaneCache(), + splitEnabled: () => false, + }); + const env = await resolver.resolveAuthenticatedEnv(found!.environmentId); + expect(env).not.toBeNull(); + // No parent env: the hash check falls back to the env's own apiKey. + expect(env!.parentEnvironment?.apiKey ?? env!.apiKey).toBe(cp.environment.apiKey); + expect(env!.organizationId).toBe(cp.organization.id); + + expect(await (prisma17 as unknown as PrismaClient).runtimeEnvironment.count()).toBe(0); + expect(await (prisma14 as unknown as PrismaClient).waitpoint.count()).toBe(0); + } + ); + + heteroPostgresTest( + "branch env: the hash check uses the parent apiKey resolved from control-plane", + async ({ prisma14, prisma17 }) => { + await dropWaitpointCrossSeamFks(prisma17 as unknown as PrismaClient); + const cp = await seedControlPlane(prisma14 as unknown as PrismaClient, { withParent: true }); + const waitpoint = await seedWaitpoint(prisma17 as unknown as PrismaClient, { + environmentId: cp.environment.id, + projectId: cp.project.id, + }); + + const found = await (prisma17 as unknown as PrismaClient).waitpoint.findFirst({ + where: { id: waitpoint.id }, + select: { id: true, status: true, environmentId: true }, + }); + + const resolver = new ControlPlaneResolver({ + controlPlanePrimary: prisma14 as unknown as PrismaClient, + controlPlaneReplica: prisma14 as unknown as PrismaClient, + cache: new ControlPlaneCache(), + splitEnabled: () => false, + }); + const env = await resolver.resolveAuthenticatedEnv(found!.environmentId); + expect(env!.parentEnvironment).not.toBeNull(); + // The route prefers the parent apiKey for the hash check on a branch env. + expect(env!.parentEnvironment?.apiKey ?? env!.apiKey).toBe(cp.parent!.apiKey); + + expect(await (prisma17 as unknown as PrismaClient).runtimeEnvironment.count()).toBe(0); + expect(await (prisma14 as unknown as PrismaClient).waitpoint.count()).toBe(0); + } + ); +}); + +// A waitpoint whose `id` matches `WaitpointId.toId(friendlyId)`, so the route's friendlyId->id +// conversion + the hash (computed over `id`) line up. +async function seedRoutableWaitpoint( + prisma: PrismaClient, + ctx: { environmentId: string; projectId: string }, + overrides?: { status?: "PENDING" | "COMPLETED"; output?: string } +) { + const s = n++; + const { id, friendlyId } = WaitpointId.generate(); + return prisma.waitpoint.create({ + data: { + id, + friendlyId, + type: "MANUAL", + status: overrides?.status ?? "PENDING", + idempotencyKey: `idem_${s}`, + userProvidedIdempotencyKey: false, + environmentId: ctx.environmentId, + projectId: ctx.projectId, + ...(overrides?.output + ? { output: overrides.output, outputType: "application/json", outputIsError: false } + : {}), + }, + }); +} + +// With split on (the test env sets RUN_OPS_SPLIT_ENABLED + both DB urls), the singleton resolver +// reads the env off the control-plane REPLICA, which is the `$replica` proxy the route also reads +// the waitpoint from. So for the route-level branches we co-locate the waitpoint + its env on the +// run-ops container the proxies point at; the genuine cross-DB residency (waitpoint PG17 / env PG14) +// is already proven by the resolver-isolation cases above. The DB is never mocked. +describe("waitpoint HTTP-callback route action (real containers)", () => { + heteroPostgresTest( + "env resolves null (no env row) -> 404 Waitpoint not found", + async ({ prisma17 }) => { + await dropWaitpointCrossSeamFks(prisma17 as unknown as PrismaClient); + // Seed the waitpoint but NOT its env, so the resolver returns null after the waitpoint is found. + const cp = await seedControlPlane(prisma17 as unknown as PrismaClient); + const waitpoint = await seedRoutableWaitpoint(prisma17 as unknown as PrismaClient, { + environmentId: `env_absent_${n++}`, + projectId: cp.project.id, + }); + + replicaHolder.client = prisma17; + primaryHolder.client = prisma17; + + const res = await action({ + request: callbackRequest({}), + params: { waitpointFriendlyId: waitpoint.friendlyId, hash: "deadbeef" }, + context: {} as never, + }); + + expect(res.status).toBe(404); + expect(await res.json()).toEqual({ error: "Waitpoint not found" }); + } + ); + + heteroPostgresTest("wrong hash -> 401 Invalid URL, hash doesn't match", async ({ prisma17 }) => { + await dropWaitpointCrossSeamFks(prisma17 as unknown as PrismaClient); + const cp = await seedControlPlane(prisma17 as unknown as PrismaClient); + const waitpoint = await seedRoutableWaitpoint(prisma17 as unknown as PrismaClient, { + environmentId: cp.environment.id, + projectId: cp.project.id, + }); + + replicaHolder.client = prisma17; + primaryHolder.client = prisma17; + + const res = await action({ + request: callbackRequest({}), + params: { waitpointFriendlyId: waitpoint.friendlyId, hash: "not-the-right-hash" }, + context: {} as never, + }); + + expect(res.status).toBe(401); + expect(await res.json()).toEqual({ error: "Invalid URL, hash doesn't match" }); + }); + + heteroPostgresTest( + "COMPLETED waitpoint short-circuits -> 200 success without mutating the row", + async ({ prisma17 }) => { + await dropWaitpointCrossSeamFks(prisma17 as unknown as PrismaClient); + const cp = await seedControlPlane(prisma17 as unknown as PrismaClient); + const existingOutput = JSON.stringify({ already: "done" }); + const waitpoint = await seedRoutableWaitpoint( + prisma17 as unknown as PrismaClient, + { environmentId: cp.environment.id, projectId: cp.project.id }, + { status: "COMPLETED", output: existingOutput } + ); + + replicaHolder.client = prisma17; + primaryHolder.client = prisma17; + + // No parent env -> the hash is computed over the env's own apiKey. + const hash = hashFor(waitpoint.id, cp.environment.apiKey); + + const res = await action({ + request: callbackRequest({ new: "data" }), + params: { waitpointFriendlyId: waitpoint.friendlyId, hash }, + context: {} as never, + }); + + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ success: true }); + + // The COMPLETED branch returns before `engine.completeWaitpoint`: the row is untouched. + const after = await (prisma17 as unknown as PrismaClient).waitpoint.findFirst({ + where: { id: waitpoint.id }, + select: { status: true, output: true }, + }); + expect(after!.status).toBe("COMPLETED"); + expect(after!.output).toBe(existingOutput); + } + ); +}); From 032cecfb02ff0eb2943d8533e20b24d7a8bfd461 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Wed, 1 Jul 2026 20:29:08 +0100 Subject: [PATCH 02/11] test(run-ops split): rewrite waitpoint-token tests to the always-cuid contract Waitpoint ids are always cuid; residency is decided by co-location routing, not id-shape. Remove the removed setKsuidMintEnabled global and flip the standalone-token assertions to cuid/LEGACY. Drive the NEW/cross-seam side from a ksuid run co-locating its (cuid) token on #new instead of a ksuid token, and keep the cross-seam guard consulted unconditionally on a plain cuid token. Co-Authored-By: Claude Opus 4.8 (1M context) --- ...nts.tokens.complete.crossSeamGuard.test.ts | 16 +- .../test/api.v1.waitpoints.tokens.test.ts | 260 +++++++++++++----- 2 files changed, 191 insertions(+), 85 deletions(-) diff --git a/apps/webapp/test/api.v1.waitpoints.tokens.complete.crossSeamGuard.test.ts b/apps/webapp/test/api.v1.waitpoints.tokens.complete.crossSeamGuard.test.ts index 48e6e0df65a..43c4dd1821b 100644 --- a/apps/webapp/test/api.v1.waitpoints.tokens.complete.crossSeamGuard.test.ts +++ b/apps/webapp/test/api.v1.waitpoints.tokens.complete.crossSeamGuard.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, afterEach, vi } from "vitest"; +import { describe, expect, vi } from "vitest"; // Regression for the waitpoint-token completion route path: the route calls // engine.completeWaitpoint, which must consult the cross-seam residency guard @@ -10,18 +10,13 @@ import { RunEngine } from "@internal/run-engine"; import { setupAuthenticatedEnvironment } from "@internal/run-engine/tests"; import { containerTest, heteroPostgresTest } from "@internal/testcontainers"; import type { PrismaClient } from "@trigger.dev/database"; -import { setKsuidMintEnabled, WaitpointId } from "@trigger.dev/core/v3/isomorphic"; +import { WaitpointId } from "@trigger.dev/core/v3/isomorphic"; import { trace } from "@opentelemetry/api"; vi.setConfig({ testTimeout: 60_000 }); type CrossSeamGuard = ConstructorParameters[0]["crossSeamGuard"]; -afterEach(() => { - // Never let the ksuid mint mode leak into other webapp tests. - setKsuidMintEnabled(false); -}); - function buildEngine(opts: { prisma: any; redisOptions: any; @@ -64,8 +59,6 @@ describe("waitpoint-token complete route — cross-seam guard", () => { containerTest( "consults the guard first (RESUME_TOKEN), then completes (single-store)", async ({ prisma, redisOptions }) => { - setKsuidMintEnabled(true); - const seen: Array<{ waitpointId: string; routeKind: string }> = []; const engine = buildEngine({ prisma, @@ -95,7 +88,6 @@ describe("waitpoint-token complete route — cross-seam guard", () => { // The guard was consulted first, with the right id + RESUME_TOKEN route kind. expect(seen).toEqual([{ waitpointId: waitpoint.id, routeKind: "RESUME_TOKEN" }]); - // The completion was then applied via delegation (single-store path). const after = await prisma.waitpoint.findFirst({ where: { id: waitpoint.id } }); expect(after?.status).toBe("COMPLETED"); } finally { @@ -109,8 +101,6 @@ describe("waitpoint-token complete route — cross-seam guard", () => { containerTest( "propagates a guard throw and leaves the waitpoint PENDING (loud)", async ({ prisma, redisOptions }) => { - setKsuidMintEnabled(true); - const engine = buildEngine({ prisma, redisOptions, @@ -195,8 +185,6 @@ describe("waitpoint-token complete route — no FK abort across the PG14<->17 bo heteroPostgresTest( "completes a run-ops waitpoint on each version store without tripping the absent control-plane Cascade FK", async ({ prisma14, prisma17 }) => { - setKsuidMintEnabled(true); - const legacy = prisma14 as unknown as PrismaClient; const next = prisma17 as unknown as PrismaClient; diff --git a/apps/webapp/test/api.v1.waitpoints.tokens.test.ts b/apps/webapp/test/api.v1.waitpoints.tokens.test.ts index 73350230103..6f9419ce7b5 100644 --- a/apps/webapp/test/api.v1.waitpoints.tokens.test.ts +++ b/apps/webapp/test/api.v1.waitpoints.tokens.test.ts @@ -1,31 +1,32 @@ -import { describe, expect, afterEach, vi } from "vitest"; +import { describe, expect, vi } from "vitest"; -// These tests exercise the store-routed engine create/get seam and the -// residency-keyed id contract (the same id-shaping the create route's response -// uses). They do NOT drive the route's HTTP action — only the engine create/get -// seam behind it. +// Store-routed engine create/get seam + the residency-keyed id contract behind +// the create route (not its HTTP action). A standalone MANUAL token is cuid → +// LEGACY; NEW residency is reached only by co-locating the token with a ksuid run. import { RunEngine } from "@internal/run-engine"; import { setupAuthenticatedEnvironment } from "@internal/run-engine/tests"; -import { containerTest, heteroPostgresTest } from "@internal/testcontainers"; -import { PostgresRunStore } from "@internal/run-store"; +import { + containerTest, + heteroRunOpsPostgresTest, + network, + redisContainer, + redisOptions, +} from "@internal/testcontainers"; +import { PostgresRunStore, RoutingRunStore, type CreateRunInput } from "@internal/run-store"; +import type { RunOpsPrismaClient } from "@internal/run-ops-database"; import { WaitpointId, - setKsuidMintEnabled, + RunId, + generateKsuidId, ownerEngine, - KSUID_LENGTH, + CUID_LENGTH, } from "@trigger.dev/core/v3/isomorphic"; -import { Prisma } from "@trigger.dev/database"; +import { Prisma, type PrismaClient } from "@trigger.dev/database"; import { trace } from "@opentelemetry/api"; -import { nanoid } from "nanoid"; vi.setConfig({ testTimeout: 60_000 }); -afterEach(() => { - // Never let the ksuid mint mode leak into other webapp tests. - setKsuidMintEnabled(false); -}); - function buildEngine(opts: { prisma: any; redisOptions: any; @@ -63,12 +64,10 @@ function buildEngine(opts: { } describe("waitpoint-token create engine seam — residency-keyed id contract", () => { - // Test A: the create seam mints a KSUID WaitpointId on the run-ops engine. + // Test A: a standalone token (no owning run) mints a cuid WaitpointId and stays LEGACY. containerTest( - "create mints a KSUID WaitpointId on the run-ops engine", + "create mints a cuid WaitpointId for a standalone token (LEGACY)", async ({ prisma, redisOptions }) => { - setKsuidMintEnabled(true); - const engine = buildEngine({ prisma, redisOptions }); try { @@ -80,12 +79,9 @@ describe("waitpoint-token create engine seam — residency-keyed id contract", ( timeout: new Date(Date.now() + 60_000), }); - // `WaitpointId.generate()` returns a bare (un-prefixed) internal id, so - // result.waitpoint.id is the raw 27-char ksuid body. - expect(result.waitpoint.id.length).toBe(KSUID_LENGTH); + expect(result.waitpoint.id.length).toBe(CUID_LENGTH); expect(result.waitpoint.type).toBe("MANUAL"); - // The waitpoint row exists on the run-ops store (single-DB: the container prisma). const row = await prisma.waitpoint.findUnique({ where: { id: result.waitpoint.id } }); expect(row).not.toBeNull(); expect(row?.environmentId).toBe(env.id); @@ -96,21 +92,17 @@ describe("waitpoint-token create engine seam — residency-keyed id contract", ( expect(responseId).toBe("waitpoint_" + result.waitpoint.id); expect(WaitpointId.fromFriendlyId(responseId)).toBe(result.waitpoint.id); - // The id a client receives stays residency-classifiable to the owning - // store — the contract the completion route relies on to resolve the token. - expect(ownerEngine(WaitpointId.fromFriendlyId(responseId))).toBe("NEW"); + expect(ownerEngine(WaitpointId.fromFriendlyId(responseId))).toBe("LEGACY"); } finally { await engine.quit(); } } ); - // Test B: the token id classifies to the owning (new) run-ops store and resolves back. + // Test B: the standalone token id classifies LEGACY and resolves back. containerTest( - "token id classifies to the owning run-ops store and resolves back", + "token id classifies to the legacy store and resolves back", async ({ prisma, redisOptions }) => { - setKsuidMintEnabled(true); - const engine = buildEngine({ prisma, redisOptions }); try { @@ -122,10 +114,8 @@ describe("waitpoint-token create engine seam — residency-keyed id contract", ( timeout: new Date(Date.now() + 60_000), }); - // A ksuid token id classifies to the NEW (run-ops) store. - expect(ownerEngine(result.waitpoint.id)).toBe("NEW"); + expect(ownerEngine(result.waitpoint.id)).toBe("LEGACY"); - // getWaitpoint resolves via this.runStore.findWaitpoint and returns the exact row. const resolved = await engine.getWaitpoint({ waitpointId: result.waitpoint.id, environmentId: env.id, @@ -144,8 +134,6 @@ describe("waitpoint-token create engine seam — residency-keyed id contract", ( containerTest( "control-plane WaitpointTag write stays control-plane, not on the run-ops store", async ({ prisma, redisOptions }) => { - setKsuidMintEnabled(true); - // A run-ops store that counts every waitpoint write that passes through it. // Overrides mirror the base PostgresRunStore generics so a base-signature // change can't silently detach the counter. @@ -192,7 +180,7 @@ describe("waitpoint-token create engine seam — residency-keyed id contract", ( tags: ["t1"], timeout: new Date(Date.now() + 60_000), }); - expect(result.waitpoint.id.length).toBe(KSUID_LENGTH); + expect(result.waitpoint.id.length).toBe(CUID_LENGTH); // The tag landed on the control-plane client. const tagRow = await prisma.waitpointTag.findFirst({ @@ -210,49 +198,179 @@ describe("waitpoint-token create engine seam — residency-keyed id contract", ( } } ); +}); - // Test D: a minted KSUID waitpoint resolves only to its owning store across the - // PG14↔PG17 version boundary. Store-only — no Redis/engine needed. - heteroPostgresTest( - "minted WaitpointId resolves only to its owning run-ops store across the version boundary", - async ({ prisma14, prisma17 }) => { - setKsuidMintEnabled(true); +const twoDbEngineTest = heteroRunOpsPostgresTest.extend<{ + redisContainer: any; + redisOptions: any; +}>({ + network, + redisContainer, + redisOptions, +}); - const store17 = new PostgresRunStore({ prisma: prisma17, readOnlyPrisma: prisma17 }); - const store14 = new PostgresRunStore({ prisma: prisma14, readOnlyPrisma: prisma14 }); +async function seedControlPlaneEnv(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "PRODUCTION", + slug: `prod-${suffix}`, + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_prod_${suffix}`, + pkApiKey: `pk_prod_${suffix}`, + shortcode: `short_${suffix}`, + maximumConcurrencyLimit: 10, + }, + }); + return { organization, project, environment }; +} - const env = await setupAuthenticatedEnvironment(prisma17, "PRODUCTION"); +function buildCreateRunInput(params: { + runId: string; + friendlyId: string; + organizationId: string; + projectId: string; + runtimeEnvironmentId: string; +}): CreateRunInput { + return { + data: { + id: params.runId, + engine: "V2", + status: "EXECUTING", + friendlyId: params.friendlyId, + runtimeEnvironmentId: params.runtimeEnvironmentId, + environmentType: "PRODUCTION", + organizationId: params.organizationId, + projectId: params.projectId, + taskIdentifier: "parent-task", + payload: "{}", + payloadType: "application/json", + context: {}, + traceContext: {}, + traceId: `trace_${params.runId}`, + spanId: `span_${params.runId}`, + runTags: [], + queue: "task/parent-task", + isTest: false, + taskEventStore: "taskEvent", + depth: 0, + createdAt: new Date("2024-01-01T00:00:00.000Z"), + }, + snapshot: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "Run was created", + runStatus: "PENDING", + environmentId: params.runtimeEnvironmentId, + environmentType: "PRODUCTION", + projectId: params.projectId, + organizationId: params.organizationId, + }, + }; +} - const idempotencyKey = nanoid(24); - const generated = WaitpointId.generate(); +async function seedExecutingKsuidRun( + prisma14: PrismaClient, + router: RoutingRunStore, + runId: string, + suffix: string +) { + const env = await seedControlPlaneEnv(prisma14, suffix); + + await router.createRun( + buildCreateRunInput({ + runId, + friendlyId: `run_${suffix}`, + organizationId: env.organization.id, + projectId: env.project.id, + runtimeEnvironmentId: env.environment.id, + }) + ); - const created = await store17.upsertWaitpoint({ - where: { - environmentId_idempotencyKey: { environmentId: env.id, idempotencyKey }, - }, - create: { - ...generated, - type: "MANUAL", - idempotencyKey, - userProvidedIdempotencyKey: false, - environmentId: env.id, + const created = await router.findLatestExecutionSnapshot(runId); + await router.createExecutionSnapshot( + { + run: { id: runId, status: "EXECUTING", attemptNumber: 1 }, + snapshot: { executionStatus: "EXECUTING", description: "run executing" }, + previousSnapshotId: created!.id, + environmentId: env.environment.id, + environmentType: "PRODUCTION", + projectId: env.project.id, + organizationId: env.organization.id, + }, + prisma14 + ); + + return env; +} + +function makeRouter(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) { + const newStore = new PostgresRunStore({ + prisma: prisma17 as never, + readOnlyPrisma: prisma17 as never, + schemaVariant: "dedicated", + }); + const legacyStore = new PostgresRunStore({ + prisma: prisma14, + readOnlyPrisma: prisma14, + schemaVariant: "legacy", + }); + return new RoutingRunStore({ new: newStore, legacy: legacyStore }); +} + +describe("waitpoint-token create engine seam — NEW residency via a ksuid run across the version boundary", () => { + // Test D: NEW residency comes from co-locating the token with a ksuid run; the token + // resolves only on its owning (#new) store across the PG14<->PG17 boundary, never #legacy. + twoDbEngineTest( + "a ksuid run's token co-locates on #new and resolves only there, not on #legacy", + async ({ prisma14, prisma17, redisOptions }) => { + const p14 = prisma14 as unknown as PrismaClient; + const router = makeRouter(p14, prisma17); + const engine = buildEngine({ prisma: prisma14, redisOptions, store: router }); + + try { + // A NEW-classified run id (explicit ksuid), mirroring the trigger-routing helper. + const runId = RunId.toFriendlyId(generateKsuidId()); + expect(ownerEngine(runId)).toBe("NEW"); + const env = await seedExecutingKsuidRun(p14, router, runId, "wpnew"); + + const { waitpoint } = await engine.createManualWaitpoint({ + runId, + environmentId: env.environment.id, projectId: env.project.id, - }, - update: {}, - }); + }); - const id = created.id; - expect(id.length).toBe(KSUID_LENGTH); - expect(ownerEngine(id)).toBe("NEW"); + // The token is always cuid (Option B); its NEW residency comes from the run. + expect(waitpoint.id.length).toBe(CUID_LENGTH); + expect(ownerEngine(waitpoint.id)).toBe("LEGACY"); - // Byte-identical id resolves on the PG17 run-ops home. - const found17 = await store17.findWaitpoint({ where: { id } }, prisma17); - expect(found17).not.toBeNull(); - expect(found17?.id).toBe(id); + // Co-location: the token lives on #new next to its run, not on #legacy. + const onNew = await prisma17.waitpoint.findUnique({ where: { id: waitpoint.id } }); + const onLegacy = await p14.waitpoint.findUnique({ where: { id: waitpoint.id } }); + expect(onNew).not.toBeNull(); + expect(onLegacy).toBeNull(); - // Residency invariant: the same id does NOT resolve on the PG14 legacy store. - const found14 = await store14.findWaitpoint({ where: { id } }, prisma14); - expect(found14).toBeNull(); + const resolved = await engine.getWaitpoint({ + waitpointId: waitpoint.id, + environmentId: env.environment.id, + projectId: env.project.id, + }); + expect(resolved?.id).toBe(waitpoint.id); + expect(await p14.waitpoint.findUnique({ where: { id: waitpoint.id } })).toBeNull(); + } finally { + await engine.quit(); + } } ); }); From 172b8c7d060bd48dec1a8661437ed8494a2f7b48 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Thu, 2 Jul 2026 14:06:34 +0100 Subject: [PATCH 03/11] fix(run-ops split): resolve public wait tokens across the split boundary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The public wait-token routes (complete, HTTP callback, retrieve) resolved the waitpoint with a bare read-through that defaulted its new-side client to the dedicated run-ops replica and gated on the async mint flag. A standalone wait token has a cuid id and, having no owning run, is written to the control-plane store, so under the split topology the run-ops replica does not hold it and the routes returned 404 "Waitpoint not found". Resolve these routes the same way the working waiter route does: fan out through the run-ops replica first, then the control-plane replica, so both a co-located (run-owned) waitpoint and a standalone control-plane token are found. Gate the fan-out on the URL-presence read gate rather than the mint flag, so read visibility spans both DBs whenever both are configured — including the window where both database URLs are set but the mint flag is off. The retrieve route hands the same fan-out clients and gate to ApiWaitpointPresenter. Adds a two-database testcontainer regression proving a control-plane-resident standalone token resolves via the legacy fallback under the read gate, and that the passthrough branch (gate off) misses it. Co-Authored-By: Claude Opus 4.8 (1M context) --- ...ens.$waitpointFriendlyId.callback.$hash.ts | 46 +++--- ...ts.tokens.$waitpointFriendlyId.complete.ts | 45 +++--- ....waitpoints.tokens.$waitpointFriendlyId.ts | 7 +- .../waitpointTokenResolve.server.test.ts | 148 ++++++++++++++++++ 4 files changed, 204 insertions(+), 42 deletions(-) create mode 100644 apps/webapp/app/v3/runOpsMigration/waitpointTokenResolve.server.test.ts diff --git a/apps/webapp/app/routes/api.v1.waitpoints.tokens.$waitpointFriendlyId.callback.$hash.ts b/apps/webapp/app/routes/api.v1.waitpoints.tokens.$waitpointFriendlyId.callback.$hash.ts index da7b9c2e1ab..310135adf59 100644 --- a/apps/webapp/app/routes/api.v1.waitpoints.tokens.$waitpointFriendlyId.callback.$hash.ts +++ b/apps/webapp/app/routes/api.v1.waitpoints.tokens.$waitpointFriendlyId.callback.$hash.ts @@ -2,13 +2,18 @@ import { type ActionFunctionArgs, json } from "@remix-run/server-runtime"; import { type CompleteWaitpointTokenResponseBody, stringifyIO } from "@trigger.dev/core/v3"; import { WaitpointId } from "@trigger.dev/core/v3/isomorphic"; import { z } from "zod"; -import type { PrismaReplicaClient } from "~/db.server"; +import { + $replica, + type PrismaReplicaClient, + runOpsNewReplica, + runOpsSplitReadEnabled, +} from "~/db.server"; import { env } from "~/env.server"; import { processWaitpointCompletionPacket } from "~/runEngine/concerns/waitpointCompletionPacket.server"; +import { resolveWaitpointThroughReadThrough } from "~/runEngine/concerns/resolveWaitpointThroughReadThrough.server"; import { verifyHttpCallbackHash } from "~/services/httpCallback.server"; import { logger } from "~/services/logger.server"; import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server"; -import { readThroughRun } from "~/v3/runOpsMigration/readThrough.server"; import { engine } from "~/v3/runEngine.server"; const paramsSchema = z.object({ @@ -34,28 +39,27 @@ export async function action({ request, params }: ActionFunctionArgs) { const waitpointId = WaitpointId.toId(waitpointFriendlyId); try { - // Read through the split-aware run-ops read-through (passthrough in single-DB). The env is - // resolved below from the row; residency is classified off the waitpoint id, so env "" is fine. - const findWaitpoint = (client: PrismaReplicaClient) => - client.waitpoint.findFirst({ - where: { - id: waitpointId, - }, - select: { id: true, status: true, environmentId: true }, - }); - - const waitpointResult = await readThroughRun({ - runId: waitpointId, + // Resolve wherever the waitpoint resides. The env is resolved below from the row; residency + // is classified off the waitpoint id, so env "" is fine. Fan-out reads the run-ops replica + // first, then the control-plane replica so both a co-located and a standalone token resolve, + // gated on the URL-presence read gate so the fan-out spans both DBs independent of the mint flag. + const waitpoint = await resolveWaitpointThroughReadThrough({ + waitpointId, environmentId: "", - readNew: (client) => findWaitpoint(client), - readLegacy: (replica) => findWaitpoint(replica), + read: (client: PrismaReplicaClient) => + client.waitpoint.findFirst({ + where: { + id: waitpointId, + }, + select: { id: true, status: true, environmentId: true }, + }), + deps: { + newClient: runOpsNewReplica, + legacyReplica: $replica, + splitEnabled: runOpsSplitReadEnabled, + }, }); - const waitpoint = - waitpointResult.source === "new" || waitpointResult.source === "legacy-replica" - ? waitpointResult.value - : null; - if (!waitpoint) { return json({ error: "Waitpoint not found" }, { status: 404 }); } diff --git a/apps/webapp/app/routes/api.v1.waitpoints.tokens.$waitpointFriendlyId.complete.ts b/apps/webapp/app/routes/api.v1.waitpoints.tokens.$waitpointFriendlyId.complete.ts index a4df0e83fd2..81358ef349f 100644 --- a/apps/webapp/app/routes/api.v1.waitpoints.tokens.$waitpointFriendlyId.complete.ts +++ b/apps/webapp/app/routes/api.v1.waitpoints.tokens.$waitpointFriendlyId.complete.ts @@ -6,12 +6,17 @@ import { } from "@trigger.dev/core/v3"; import { WaitpointId } from "@trigger.dev/core/v3/isomorphic"; import { z } from "zod"; -import type { PrismaReplicaClient } from "~/db.server"; +import { + $replica, + type PrismaReplicaClient, + runOpsNewReplica, + runOpsSplitReadEnabled, +} from "~/db.server"; import { env } from "~/env.server"; import { logger } from "~/services/logger.server"; import { processWaitpointCompletionPacket } from "~/runEngine/concerns/waitpointCompletionPacket.server"; +import { resolveWaitpointThroughReadThrough } from "~/runEngine/concerns/resolveWaitpointThroughReadThrough.server"; import { createActionApiRoute } from "~/services/routeBuilders/apiBuilder.server"; -import { readThroughRun } from "~/v3/runOpsMigration/readThrough.server"; import { engine } from "~/v3/runEngine.server"; const { action, loader } = createActionApiRoute( @@ -34,27 +39,27 @@ const { action, loader } = createActionApiRoute( try { //check permissions - // Read through the split-aware run-ops read-through (passthrough in single-DB). - const findWaitpoint = (client: PrismaReplicaClient) => - client.waitpoint.findFirst({ - where: { - id: waitpointId, - environmentId: authentication.environment.id, - }, - }); - - const waitpointResult = await readThroughRun({ - runId: waitpointId, + // Resolve wherever the waitpoint resides: a standalone token lives on the control-plane + // store, while a run-owned waitpoint co-locates with its run. Fan-out reads the run-ops + // replica first, then the control-plane replica so both residencies resolve, gated on the + // URL-presence read gate so the fan-out spans both DBs independent of the mint flag. + const waitpoint = await resolveWaitpointThroughReadThrough({ + waitpointId, environmentId: authentication.environment.id, - readNew: (client) => findWaitpoint(client), - readLegacy: (replica) => findWaitpoint(replica), + read: (client: PrismaReplicaClient) => + client.waitpoint.findFirst({ + where: { + id: waitpointId, + environmentId: authentication.environment.id, + }, + }), + deps: { + newClient: runOpsNewReplica, + legacyReplica: $replica, + splitEnabled: runOpsSplitReadEnabled, + }, }); - const waitpoint = - waitpointResult.source === "new" || waitpointResult.source === "legacy-replica" - ? waitpointResult.value - : null; - if (!waitpoint) { throw json({ error: "Waitpoint not found" }, { status: 404 }); } diff --git a/apps/webapp/app/routes/api.v1.waitpoints.tokens.$waitpointFriendlyId.ts b/apps/webapp/app/routes/api.v1.waitpoints.tokens.$waitpointFriendlyId.ts index be91b35b08e..5f16c6df671 100644 --- a/apps/webapp/app/routes/api.v1.waitpoints.tokens.$waitpointFriendlyId.ts +++ b/apps/webapp/app/routes/api.v1.waitpoints.tokens.$waitpointFriendlyId.ts @@ -2,6 +2,7 @@ import { json } from "@remix-run/server-runtime"; import { type WaitpointRetrieveTokenResponse } from "@trigger.dev/core/v3"; import { WaitpointId } from "@trigger.dev/core/v3/isomorphic"; import { z } from "zod"; +import { $replica, runOpsNewReplica, runOpsSplitReadEnabled } from "~/db.server"; import { ApiWaitpointPresenter } from "~/presenters/v3/ApiWaitpointPresenter.server"; import { createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; @@ -13,7 +14,11 @@ export const loader = createLoaderApiRoute( findResource: async () => 1, // This is a dummy function, we don't need to find a resource }, async ({ params, authentication }) => { - const presenter = new ApiWaitpointPresenter(); + const presenter = new ApiWaitpointPresenter(undefined, undefined, { + newClient: runOpsNewReplica, + legacyReplica: $replica, + splitEnabled: runOpsSplitReadEnabled, + }); const result: WaitpointRetrieveTokenResponse = await presenter.call( authentication.environment, WaitpointId.toId(params.waitpointFriendlyId) diff --git a/apps/webapp/app/v3/runOpsMigration/waitpointTokenResolve.server.test.ts b/apps/webapp/app/v3/runOpsMigration/waitpointTokenResolve.server.test.ts new file mode 100644 index 00000000000..30dabfb0d83 --- /dev/null +++ b/apps/webapp/app/v3/runOpsMigration/waitpointTokenResolve.server.test.ts @@ -0,0 +1,148 @@ +// A standalone wait token is minted with a cuid id and, having no owning run, is written to the +// control-plane store. Under the split topology the run-ops read replica is a distinct database +// that does not hold it, so the public token routes must fan out to the control-plane replica or +// the token is reported missing. These reads run as real queries against two containers. +import { heteroPostgresTest } from "@internal/testcontainers"; +import { WaitpointId } from "@trigger.dev/core/v3/isomorphic"; +import { describe, expect, vi } from "vitest"; +import type { PrismaClient } from "@trigger.dev/database"; +import type { PrismaReplicaClient } from "~/db.server"; +import { resolveWaitpointThroughReadThrough } from "~/runEngine/concerns/resolveWaitpointThroughReadThrough.server"; +import { readThroughRun } from "./readThrough.server"; + +vi.setConfig({ testTimeout: 60_000 }); + +async function seedControlPlaneEnv(prisma: PrismaClient, suffix: string) { + const organization = await prisma.organization.create({ + data: { title: `Org ${suffix}`, slug: `org-${suffix}` }, + }); + const project = await prisma.project.create({ + data: { + name: `Project ${suffix}`, + slug: `project-${suffix}`, + externalRef: `proj_${suffix}`, + organizationId: organization.id, + }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + type: "PRODUCTION", + slug: `prod-${suffix}`, + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_prod_${suffix}`, + pkApiKey: `pk_prod_${suffix}`, + shortcode: `short_${suffix}`, + maximumConcurrencyLimit: 10, + }, + }); + return { organization, project, environment }; +} + +async function seedStandaloneToken(prisma: PrismaClient, environmentId: string, projectId: string) { + const { id, friendlyId } = WaitpointId.generate(); + await prisma.waitpoint.create({ + data: { + id, + friendlyId, + type: "MANUAL", + status: "PENDING", + idempotencyKey: id, + userProvidedIdempotencyKey: false, + environmentId, + projectId, + }, + }); + return { id, friendlyId }; +} + +describe("public wait-token resolution across the split boundary", () => { + heteroPostgresTest( + "a control-plane-resident standalone token is found when the run-ops replica does not hold it", + async ({ prisma14, prisma17 }) => { + const { project, environment } = await seedControlPlaneEnv(prisma14, "token_cp"); + const { id: waitpointId } = await seedStandaloneToken(prisma14, environment.id, project.id); + + const waitpoint = await resolveWaitpointThroughReadThrough({ + waitpointId, + environmentId: environment.id, + read: (client: PrismaReplicaClient) => + client.waitpoint.findFirst({ + where: { id: waitpointId, environmentId: environment.id }, + }), + deps: { + newClient: prisma17 as unknown as PrismaReplicaClient, + legacyReplica: prisma14 as unknown as PrismaReplicaClient, + }, + }); + + expect(waitpoint).not.toBeNull(); + expect(waitpoint?.id).toBe(waitpointId); + } + ); + + heteroPostgresTest( + "pinning both reads at the run-ops replica misses the control-plane token", + async ({ prisma14, prisma17 }) => { + const { project, environment } = await seedControlPlaneEnv(prisma14, "token_miss"); + const { id: waitpointId } = await seedStandaloneToken(prisma14, environment.id, project.id); + + const waitpoint = await resolveWaitpointThroughReadThrough({ + waitpointId, + environmentId: environment.id, + read: (client: PrismaReplicaClient) => + client.waitpoint.findFirst({ + where: { id: waitpointId, environmentId: environment.id }, + }), + deps: { + newClient: prisma17 as unknown as PrismaReplicaClient, + legacyReplica: prisma17 as unknown as PrismaReplicaClient, + }, + }); + + expect(waitpoint).toBeNull(); + } + ); + + heteroPostgresTest( + "the read gate forces fan-out so a control-plane token resolves while the mint flag is off", + async ({ prisma14, prisma17 }) => { + const { project, environment } = await seedControlPlaneEnv(prisma14, "token_gate"); + const { id: waitpointId } = await seedStandaloneToken(prisma14, environment.id, project.id); + + const read = (client: PrismaReplicaClient) => + client.waitpoint.findFirst({ + where: { id: waitpointId, environmentId: environment.id }, + }); + + const gated = await resolveWaitpointThroughReadThrough({ + waitpointId, + environmentId: environment.id, + read, + deps: { + splitEnabled: true, + newClient: prisma17 as unknown as PrismaReplicaClient, + legacyReplica: prisma14 as unknown as PrismaReplicaClient, + }, + }); + + expect(gated).not.toBeNull(); + expect(gated?.id).toBe(waitpointId); + + const passthrough = await readThroughRun({ + runId: waitpointId, + environmentId: environment.id, + readNew: (c) => read(c), + readLegacy: (r) => read(r), + deps: { + splitEnabled: false, + newClient: prisma17 as unknown as PrismaReplicaClient, + legacyReplica: prisma14 as unknown as PrismaReplicaClient, + }, + }); + + expect(gated).not.toBeNull(); + expect(passthrough.source).toBe("not-found"); + } + ); +}); From cfa88eddc8299a47d452f8d8b5b38a357f638347 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Thu, 2 Jul 2026 14:37:00 +0100 Subject: [PATCH 04/11] chore(run-ops split): strip test-plan scaffolding from PR10 routes/tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Comment/label/title cleanup only — no product logic or test behavior changed. Removes leftover enumeration scaffolding across the PR10 files: Test A-D and Leg 1-4 case markers, (A)/(B)/(D) parenthesized markers, Step 1-3 prefixes, and --- comment framing, while preserving the behavioral prose and the describe/it titles. Co-Authored-By: Claude Opus 4.8 (1M context) --- ...pi.v1.runs.$runFriendlyId.input-streams.wait.ts | 6 +++--- ....v1.runs.$runFriendlyId.session-streams.wait.ts | 6 +++--- ...itpoints.tokens.complete.crossSeamGuard.test.ts | 6 +++--- apps/webapp/test/api.v1.waitpoints.tokens.test.ts | 8 ++++---- apps/webapp/test/crossSeamGuard.proof.test.ts | 14 ++++++-------- 5 files changed, 19 insertions(+), 21 deletions(-) diff --git a/apps/webapp/app/routes/api.v1.runs.$runFriendlyId.input-streams.wait.ts b/apps/webapp/app/routes/api.v1.runs.$runFriendlyId.input-streams.wait.ts index 11d77b9f06f..c2efce02347 100644 --- a/apps/webapp/app/routes/api.v1.runs.$runFriendlyId.input-streams.wait.ts +++ b/apps/webapp/app/routes/api.v1.runs.$runFriendlyId.input-streams.wait.ts @@ -78,7 +78,7 @@ const { action, loader } = createActionApiRoute( } } - // Step 1: Create the waitpoint. Co-locate it with the owning run (run-ops split) so a ksuid + // Create the waitpoint. Co-locate it with the owning run (run-ops split) so a ksuid // run's input-stream waitpoint lands on the run's DB and its block edge resolves. const result = await engine.createManualWaitpoint({ runId: run.id, @@ -90,7 +90,7 @@ const { action, loader } = createActionApiRoute( tags: bodyTags, }); - // Step 2: Cache the mapping in Redis for fast lookup from .send() + // Cache the mapping in Redis for fast lookup from .send() const ttlMs = timeout ? timeout.getTime() - Date.now() : undefined; await setInputStreamWaitpoint( run.friendlyId, @@ -99,7 +99,7 @@ const { action, loader } = createActionApiRoute( ttlMs && ttlMs > 0 ? ttlMs : undefined ); - // Step 3: Check if data was already sent to this input stream (race condition handling). + // Check if data was already sent to this input stream (race condition handling). // If .send() landed before .wait(), the data is in the S2 stream but no waitpoint // existed to complete. We check from the client's last known position. if (!result.isCached) { diff --git a/apps/webapp/app/routes/api.v1.runs.$runFriendlyId.session-streams.wait.ts b/apps/webapp/app/routes/api.v1.runs.$runFriendlyId.session-streams.wait.ts index a7052d2377e..0a34e6cddb4 100644 --- a/apps/webapp/app/routes/api.v1.runs.$runFriendlyId.session-streams.wait.ts +++ b/apps/webapp/app/routes/api.v1.runs.$runFriendlyId.session-streams.wait.ts @@ -99,7 +99,7 @@ const { action, loader } = createActionApiRoute( } } - // Step 1: Create the waitpoint. Co-locate it with the owning run (run-ops split) so a ksuid + // Create the waitpoint. Co-locate it with the owning run (run-ops split) so a ksuid // run's session-stream waitpoint lands on the run's DB and its block edge resolves. const result = await engine.createManualWaitpoint({ runId: run.id, @@ -111,7 +111,7 @@ const { action, loader } = createActionApiRoute( tags: bodyTags, }); - // Step 2: Register the waitpoint on the session channel so the next + // Register the waitpoint on the session channel so the next // append fires it. Keyed by (environmentId, addressingKey, io) — the // canonical string for the row, scoped to the environment because // externalIds are only unique per environment. The append handler @@ -126,7 +126,7 @@ const { action, loader } = createActionApiRoute( ttlMs && ttlMs > 0 ? ttlMs : undefined ); - // Step 3: Race-check. If a record landed on the channel before this + // Race-check. If a record landed on the channel before this // .wait() call, complete the waitpoint synchronously with that data // and remove the pending registration. if (!result.isCached) { diff --git a/apps/webapp/test/api.v1.waitpoints.tokens.complete.crossSeamGuard.test.ts b/apps/webapp/test/api.v1.waitpoints.tokens.complete.crossSeamGuard.test.ts index 43c4dd1821b..2b722e95a7d 100644 --- a/apps/webapp/test/api.v1.waitpoints.tokens.complete.crossSeamGuard.test.ts +++ b/apps/webapp/test/api.v1.waitpoints.tokens.complete.crossSeamGuard.test.ts @@ -54,7 +54,7 @@ function buildEngine(opts: { } describe("waitpoint-token complete route — cross-seam guard", () => { - // (A) the completion path consults the guard FIRST with routeKind RESUME_TOKEN + // The completion path consults the guard FIRST with routeKind RESUME_TOKEN // recording the waitpointId, then delegates and the waitpoint becomes COMPLETED. containerTest( "consults the guard first (RESUME_TOKEN), then completes (single-store)", @@ -96,7 +96,7 @@ describe("waitpoint-token complete route — cross-seam guard", () => { } ); - // (B) an injected guard that throws (unclassifiable) causes completeWaitpoint + // An injected guard that throws (unclassifiable) causes completeWaitpoint // to reject and the waitpoint stays PENDING (loud, not silently applied). containerTest( "propagates a guard throw and leaves the waitpoint PENDING (loud)", @@ -132,7 +132,7 @@ describe("waitpoint-token complete route — cross-seam guard", () => { ); }); -// (D) no-FK-abort: with the Waitpoint table split off control-plane, the env/project Cascade +// no-FK-abort: with the Waitpoint table split off control-plane, the env/project Cascade // FKs are physically absent. Completing a waitpoint (status flip) must not trip a now-missing FK // on EITHER the PG14 (legacy) or PG17 (new) store. Seed + complete on the SAME store (single-store // write, no two-store router). The DB is never mocked: writes hit the real PG14/PG17 containers. diff --git a/apps/webapp/test/api.v1.waitpoints.tokens.test.ts b/apps/webapp/test/api.v1.waitpoints.tokens.test.ts index 6f9419ce7b5..5c9ac229453 100644 --- a/apps/webapp/test/api.v1.waitpoints.tokens.test.ts +++ b/apps/webapp/test/api.v1.waitpoints.tokens.test.ts @@ -64,7 +64,7 @@ function buildEngine(opts: { } describe("waitpoint-token create engine seam — residency-keyed id contract", () => { - // Test A: a standalone token (no owning run) mints a cuid WaitpointId and stays LEGACY. + // A standalone token (no owning run) mints a cuid WaitpointId and stays LEGACY. containerTest( "create mints a cuid WaitpointId for a standalone token (LEGACY)", async ({ prisma, redisOptions }) => { @@ -99,7 +99,7 @@ describe("waitpoint-token create engine seam — residency-keyed id contract", ( } ); - // Test B: the standalone token id classifies LEGACY and resolves back. + // The standalone token id classifies LEGACY and resolves back. containerTest( "token id classifies to the legacy store and resolves back", async ({ prisma, redisOptions }) => { @@ -129,7 +129,7 @@ describe("waitpoint-token create engine seam — residency-keyed id contract", ( } ); - // Test C: the control-plane WaitpointTag write stays control-plane — it cannot + // The control-plane WaitpointTag write stays control-plane — it cannot // route through the run-ops store, which exposes no tag-write surface at all. containerTest( "control-plane WaitpointTag write stays control-plane, not on the run-ops store", @@ -330,7 +330,7 @@ function makeRouter(prisma14: PrismaClient, prisma17: RunOpsPrismaClient) { } describe("waitpoint-token create engine seam — NEW residency via a ksuid run across the version boundary", () => { - // Test D: NEW residency comes from co-locating the token with a ksuid run; the token + // NEW residency comes from co-locating the token with a ksuid run; the token // resolves only on its owning (#new) store across the PG14<->PG17 boundary, never #legacy. twoDbEngineTest( "a ksuid run's token co-locates on #new and resolves only there, not on #legacy", diff --git a/apps/webapp/test/crossSeamGuard.proof.test.ts b/apps/webapp/test/crossSeamGuard.proof.test.ts index 3b5a033779f..d5ab301f949 100644 --- a/apps/webapp/test/crossSeamGuard.proof.test.ts +++ b/apps/webapp/test/crossSeamGuard.proof.test.ts @@ -16,7 +16,7 @@ import { WaitpointId } from "@trigger.dev/core/v3/isomorphic"; const NEW_WP = WaitpointId.toFriendlyId("0".repeat(27)); // 27-char internal body → NEW const LEGACY_WP = WaitpointId.toFriendlyId("c".repeat(25)); // 25-char internal body → LEGACY -describe("cross-seam guard — exhaustive per-route store selection (Leg 1)", () => { +describe("cross-seam guard — exhaustive per-route store selection", () => { for (const route of UNBLOCK_ROUTES) { it(`routes ${route.id} (${route.kind}) to new store for a NEW waitpoint`, () => { const d = selectStoreForWaitpoint({ waitpointId: NEW_WP, routeKind: route.kind }); @@ -32,7 +32,7 @@ describe("cross-seam guard — exhaustive per-route store selection (Leg 1)", () } }); -describe("cross-seam guard — single-DB no-op (Leg 2)", () => { +describe("cross-seam guard — single-DB no-op", () => { for (const route of UNBLOCK_ROUTES) { it(`${route.id}: single-DB returns legacy without consulting the classifier`, () => { const calls: string[] = []; @@ -70,7 +70,7 @@ function tally(sites: string[]): Record { return counts; } -describe("cross-seam guard — CI drift guard (Leg 3)", () => { +describe("cross-seam guard — CI drift guard", () => { it("per-file completeWaitpoint( tally in source matches the catalog", () => { const root = repoRoot(); @@ -90,9 +90,8 @@ describe("cross-seam guard — CI drift guard (Leg 3)", () => { }); }); -// --------------------------------------------------------------------------- -// Leg 4 — PG14+PG17 hetero-fixture proof. The pure legs above prove the guard -// SELECTS the right store; this leg proves the selected store corresponds to the +// PG14+PG17 hetero-fixture proof. The pure-selection tests above prove the guard +// SELECTS the right store; this proves the selected store corresponds to the // DB the Waitpoint row PHYSICALLY lives in, on a REAL heterogeneous PG14+PG17 // fixture. NEVER mock. Seed Org->Project->Env (parents before children, or the // required Waitpoint.projectId/environmentId FKs abort the insert) then the @@ -100,7 +99,6 @@ describe("cross-seam guard — CI drift guard (Leg 3)", () => { // cross-DB toBeNull checks then prove no ghost row leaked to the other version. // Seed pattern copied from // internal-packages/run-engine/src/engine/tests/crossVersionCompat.test.ts. -// --------------------------------------------------------------------------- const FIXED_TS = "2024-01-01 00:00:00+00"; @@ -161,7 +159,7 @@ async function seedWaitpoint( ); } -describe("cross-seam guard — PG14+PG17 hetero-fixture proof (Leg 4)", () => { +describe("cross-seam guard — PG14+PG17 hetero-fixture proof", () => { heteroPostgresTest( "exhaustive routes resolve to the physically-correct store on PG14+PG17", async ({ prisma14, prisma17 }) => { From d1abfc4543a5d49f8e896acb825377788e463d70 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Thu, 2 Jul 2026 15:32:18 +0100 Subject: [PATCH 05/11] style(run-ops): apply oxfmt Co-Authored-By: Claude Opus 4.8 (1M context) --- ...rganizationSlug.projects.$projectParam.runs.$runParam.ts | 5 ++++- ...api.v1.waitpoints.tokens.complete.crossSeamGuard.test.ts | 6 +----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/apps/webapp/app/routes/orgs.$organizationSlug.projects.$projectParam.runs.$runParam.ts b/apps/webapp/app/routes/orgs.$organizationSlug.projects.$projectParam.runs.$runParam.ts index 1687c45008f..f2e138861e3 100644 --- a/apps/webapp/app/routes/orgs.$organizationSlug.projects.$projectParam.runs.$runParam.ts +++ b/apps/webapp/app/routes/orgs.$organizationSlug.projects.$projectParam.runs.$runParam.ts @@ -46,7 +46,10 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { throw new Response("Not Found", { status: 404 }); } - if (environment.project.slug !== projectParam || environment.organization.slug !== organizationSlug) { + if ( + environment.project.slug !== projectParam || + environment.organization.slug !== organizationSlug + ) { throw new Response("Not Found", { status: 404 }); } diff --git a/apps/webapp/test/api.v1.waitpoints.tokens.complete.crossSeamGuard.test.ts b/apps/webapp/test/api.v1.waitpoints.tokens.complete.crossSeamGuard.test.ts index 2b722e95a7d..aac8e62a1b8 100644 --- a/apps/webapp/test/api.v1.waitpoints.tokens.complete.crossSeamGuard.test.ts +++ b/apps/webapp/test/api.v1.waitpoints.tokens.complete.crossSeamGuard.test.ts @@ -17,11 +17,7 @@ vi.setConfig({ testTimeout: 60_000 }); type CrossSeamGuard = ConstructorParameters[0]["crossSeamGuard"]; -function buildEngine(opts: { - prisma: any; - redisOptions: any; - crossSeamGuard?: CrossSeamGuard; -}) { +function buildEngine(opts: { prisma: any; redisOptions: any; crossSeamGuard?: CrossSeamGuard }) { return new RunEngine({ prisma: opts.prisma, ...(opts.crossSeamGuard ? { crossSeamGuard: opts.crossSeamGuard } : {}), From 52eae91e1426320af37b72cc68cb1ea7f7eaf9a9 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Thu, 2 Jul 2026 20:08:49 +0100 Subject: [PATCH 06/11] fix(run-ops split): resolve org-membership authz on primary + full split deps for waitpoint detail Route the org-membership authorization gate through the primary client on the debug, run-inspector, and idempotency-key-reset routes so it matches the cancel and replay paths and never authorizes against lagging replica state. Restore the primary-DB org fallback in the replay action's RBAC scope resolver so the scope is never resolved without an org during replica lag. Pass the full split-read deps to WaitpointPresenter on the detail route so a waitpoint resident on the new run-ops DB resolves the same way it does on the list route. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../route.tsx | 12 +++++++++++- ...vParam.runs.$runParam.idempotencyKey.reset.tsx | 4 ++-- .../webapp/app/routes/resources.runs.$runParam.ts | 4 ++-- .../routes/resources.taskruns.$runParam.debug.ts | 4 ++-- .../routes/resources.taskruns.$runParam.replay.ts | 15 ++++++++++++++- 5 files changed, 31 insertions(+), 8 deletions(-) diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.waitpoints.tokens.$waitpointParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.waitpoints.tokens.$waitpointParam/route.tsx index cfc008e6792..a7405ad322d 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.waitpoints.tokens.$waitpointParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.waitpoints.tokens.$waitpointParam/route.tsx @@ -11,6 +11,12 @@ import { useProject } from "~/hooks/useProject"; import { findProjectBySlug } from "~/models/project.server"; import { findEnvironmentBySlug } from "~/models/runtimeEnvironment.server"; import { WaitpointPresenter } from "~/presenters/v3/WaitpointPresenter.server"; +import { + runOpsNewReplicaClient, + runOpsLegacyReplica, + runOpsSplitReadEnabled, + type PrismaClientOrTransaction, +} from "~/db.server"; import { requireUserId } from "~/services/session.server"; import { cn } from "~/utils/cn"; import { EnvironmentParamSchema, v3WaitpointTokensPath } from "~/utils/pathBuilder"; @@ -45,7 +51,11 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { } try { - const presenter = new WaitpointPresenter(undefined, undefined, {}); + const presenter = new WaitpointPresenter(undefined, undefined, { + newClient: runOpsNewReplicaClient as unknown as PrismaClientOrTransaction, + legacyReplica: runOpsLegacyReplica as unknown as PrismaClientOrTransaction, + splitEnabled: runOpsSplitReadEnabled, + }); const result = await presenter.call({ friendlyId: waitpointParam, environmentId: environment.id, diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.idempotencyKey.reset.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.idempotencyKey.reset.tsx index 4d6662dba86..06b615f4135 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.idempotencyKey.reset.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.idempotencyKey.reset.tsx @@ -1,5 +1,5 @@ import { type ActionFunction, json } from "@remix-run/node"; -import { $replica, prisma } from "~/db.server"; +import { prisma } from "~/db.server"; import { jsonWithErrorMessage, jsonWithSuccessMessage } from "~/models/message.server"; import { logger } from "~/services/logger.server"; import { requireUserId } from "~/services/session.server"; @@ -31,7 +31,7 @@ export const action: ActionFunction = async ({ request, params }) => { return jsonWithErrorMessage({}, request, "Run not found"); } - const authorizedProject = await $replica.project.findFirst({ + const authorizedProject = await prisma.project.findFirst({ where: { id: taskRun.projectId, organization: { members: { some: { userId } } } }, select: { id: true }, }); diff --git a/apps/webapp/app/routes/resources.runs.$runParam.ts b/apps/webapp/app/routes/resources.runs.$runParam.ts index 1384bcb4233..4b288d99c0e 100644 --- a/apps/webapp/app/routes/resources.runs.$runParam.ts +++ b/apps/webapp/app/routes/resources.runs.$runParam.ts @@ -3,7 +3,7 @@ import { prettyPrintPacket, TaskRunError } from "@trigger.dev/core/v3"; import type { UseDataFunctionReturn } from "remix-typedjson"; import { typedjson } from "remix-typedjson"; import { RUNNING_STATUSES } from "~/components/runs/v3/TaskRunStatus"; -import { $replica } from "~/db.server"; +import { $replica, prisma } from "~/db.server"; import { requireUserId } from "~/services/session.server"; import { v3RunParamsSchema } from "~/utils/pathBuilder"; import { machinePresetFromRun } from "~/v3/machinePresets.server"; @@ -83,7 +83,7 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { throw new Response("Not found", { status: 404 }); } - const authorizedProject = await $replica.project.findFirst({ + const authorizedProject = await prisma.project.findFirst({ where: { id: run.projectId, organization: { members: { some: { userId } } } }, select: { id: true }, }); diff --git a/apps/webapp/app/routes/resources.taskruns.$runParam.debug.ts b/apps/webapp/app/routes/resources.taskruns.$runParam.debug.ts index 6c8f985bfe1..8c6f4cdbb4f 100644 --- a/apps/webapp/app/routes/resources.taskruns.$runParam.debug.ts +++ b/apps/webapp/app/routes/resources.taskruns.$runParam.debug.ts @@ -1,7 +1,7 @@ import { type LoaderFunctionArgs } from "@remix-run/node"; import { typedjson } from "remix-typedjson"; import { z } from "zod"; -import { $replica } from "~/db.server"; +import { $replica, prisma } from "~/db.server"; import { requireUserId } from "~/services/session.server"; import { marqs } from "~/v3/marqs/index.server"; import { engine } from "~/v3/runEngine.server"; @@ -42,7 +42,7 @@ export async function loader({ request, params }: LoaderFunctionArgs) { // Authorize on the control-plane DB, keyed by the run's project — a non-member (or // unresolvable project) is indistinguishable from not-found (both 404), matching the // original scoped where. - const authorizedProject = await $replica.project.findFirst({ + const authorizedProject = await prisma.project.findFirst({ where: { id: run.projectId, organization: { members: { some: { userId } } } }, select: { id: true }, }); diff --git a/apps/webapp/app/routes/resources.taskruns.$runParam.replay.ts b/apps/webapp/app/routes/resources.taskruns.$runParam.replay.ts index 77968ec149d..8074a3d3b16 100644 --- a/apps/webapp/app/routes/resources.taskruns.$runParam.replay.ts +++ b/apps/webapp/app/routes/resources.taskruns.$runParam.replay.ts @@ -282,7 +282,20 @@ async function resolveRunOrganizationId(runParam: string): Promise Date: Thu, 2 Jul 2026 20:49:00 +0100 Subject: [PATCH 07/11] test(run-ops split): add runOpsSplitReadEnabled to the callback route test's db.server mock The callback route now resolves the waitpoint via resolveWaitpointThroughReadThrough, whose module reads runOpsSplitReadEnabled off ~/db.server at import. The test's vi.mock omitted that export, so the mocked module threw at import and the whole suite failed to collect. Provide it (split-on) to match the read path the route exercises; ksuid (NEW) waitpoint ids route to the runOpsNewReplica proxy pointing at the seeded container. Co-Authored-By: Claude Opus 4.8 (1M context) --- apps/webapp/test/waitpointCallback.controlPlane.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/webapp/test/waitpointCallback.controlPlane.test.ts b/apps/webapp/test/waitpointCallback.controlPlane.test.ts index 80830a1b2a0..54dc125be61 100644 --- a/apps/webapp/test/waitpointCallback.controlPlane.test.ts +++ b/apps/webapp/test/waitpointCallback.controlPlane.test.ts @@ -37,6 +37,9 @@ vi.mock("~/db.server", async () => { runOpsNewPrisma: lazyProxy(replicaHolder, "replicaHolder.client"), runOpsNewReplica: lazyProxy(replicaHolder, "replicaHolder.client"), runOpsLegacyReplica: lazyProxy(replicaHolder, "replicaHolder.client"), + // The route's read-through helper reads this off `~/db.server`; split-on routes ksuid (NEW) + // ids to the `runOpsNewReplica` proxy, which points at the seeded container. + runOpsSplitReadEnabled: true, sqlDatabaseSchema: Prisma.sql([`public`]), }; }); From f6bd082816394d935070b3c7fe5527c830afc414 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Fri, 3 Jul 2026 03:22:56 +0100 Subject: [PATCH 08/11] test(run-ops split): pin splitEnabled in the cross-boundary waitpoint-token test The "control-plane-resident standalone token is found when the run-ops replica does not hold it" case relied on the ambient RUN_OPS_SPLIT_ENABLED env being on. Locally .env sets it (plus distinct TASK_RUN_* URLs) so the resolver fans out new->legacy and finds the cuid-shaped token. In CI those vars are unset, so resolveWaitpointThroughReadThrough falls back to isSplitEnabled() -> false -> single-DB passthrough, which reads only the run-ops replica (prisma17), never the control-plane legacy replica (prisma14) that holds the token. The read returned null and the assertion "expected null not to be null" failed on CI shard 3. Inject deps.splitEnabled: true (mirroring the sibling read-gate test) so the fan-out is exercised deterministically regardless of ambient env. Test-only; preserves exactly what the case verifies (legacy fallback finds a control-plane token the new replica lacks). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../v3/runOpsMigration/waitpointTokenResolve.server.test.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/apps/webapp/app/v3/runOpsMigration/waitpointTokenResolve.server.test.ts b/apps/webapp/app/v3/runOpsMigration/waitpointTokenResolve.server.test.ts index 30dabfb0d83..e269d159a38 100644 --- a/apps/webapp/app/v3/runOpsMigration/waitpointTokenResolve.server.test.ts +++ b/apps/webapp/app/v3/runOpsMigration/waitpointTokenResolve.server.test.ts @@ -71,6 +71,11 @@ describe("public wait-token resolution across the split boundary", () => { where: { id: waitpointId, environmentId: environment.id }, }), deps: { + // Pin split-on explicitly: without it the fan-out gate falls back to the + // ambient RUN_OPS_SPLIT_ENABLED env, which is unset in CI (single-DB + // passthrough reads only the run-ops replica and never fans out to the + // control-plane legacy replica that holds this cuid-shaped token). + splitEnabled: true, newClient: prisma17 as unknown as PrismaReplicaClient, legacyReplica: prisma14 as unknown as PrismaReplicaClient, }, From daba69ae1a2e8c00d0f2c433b74aa233f2c4274d Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Fri, 3 Jul 2026 08:49:50 +0100 Subject: [PATCH 09/11] chore: add server-changes for pr09 Co-Authored-By: Claude Opus 4.8 (1M context) --- .server-changes/run-ops-split-route-read-routing.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .server-changes/run-ops-split-route-read-routing.md diff --git a/.server-changes/run-ops-split-route-read-routing.md b/.server-changes/run-ops-split-route-read-routing.md new file mode 100644 index 00000000000..62966e239d1 --- /dev/null +++ b/.server-changes/run-ops-split-route-read-routing.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: feature +--- + +Route dashboard and API run/waitpoint reads through the run store, and resolve public wait-token requests across both backing stores, so runs and tokens are found regardless of which store they reside on. From 9404124522a033f9af7fad2f63e185d630363007 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Fri, 3 Jul 2026 11:31:12 +0100 Subject: [PATCH 10/11] chore(run-ops): fix lint/format for main lint rules Co-Authored-By: Claude Opus 4.8 (1M context) --- ...tParam.env.$envParam.runs.$runParam.idempotencyKey.reset.tsx | 2 +- apps/webapp/app/routes/resources.taskruns.$runParam.debug.ts | 2 +- apps/webapp/test/api.v1.waitpoints.tokens.test.ts | 2 +- apps/webapp/test/crossSeamGuard.proof.test.ts | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.idempotencyKey.reset.tsx b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.idempotencyKey.reset.tsx index 06b615f4135..9567ce084e7 100644 --- a/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.idempotencyKey.reset.tsx +++ b/apps/webapp/app/routes/resources.orgs.$organizationSlug.projects.$projectParam.env.$envParam.runs.$runParam.idempotencyKey.reset.tsx @@ -1,4 +1,4 @@ -import { type ActionFunction, json } from "@remix-run/node"; +import { type ActionFunction } from "@remix-run/node"; import { prisma } from "~/db.server"; import { jsonWithErrorMessage, jsonWithSuccessMessage } from "~/models/message.server"; import { logger } from "~/services/logger.server"; diff --git a/apps/webapp/app/routes/resources.taskruns.$runParam.debug.ts b/apps/webapp/app/routes/resources.taskruns.$runParam.debug.ts index 8c6f4cdbb4f..99188cb711b 100644 --- a/apps/webapp/app/routes/resources.taskruns.$runParam.debug.ts +++ b/apps/webapp/app/routes/resources.taskruns.$runParam.debug.ts @@ -1,7 +1,7 @@ import { type LoaderFunctionArgs } from "@remix-run/node"; import { typedjson } from "remix-typedjson"; import { z } from "zod"; -import { $replica, prisma } from "~/db.server"; +import { prisma } from "~/db.server"; import { requireUserId } from "~/services/session.server"; import { marqs } from "~/v3/marqs/index.server"; import { engine } from "~/v3/runEngine.server"; diff --git a/apps/webapp/test/api.v1.waitpoints.tokens.test.ts b/apps/webapp/test/api.v1.waitpoints.tokens.test.ts index 5c9ac229453..0a05975c734 100644 --- a/apps/webapp/test/api.v1.waitpoints.tokens.test.ts +++ b/apps/webapp/test/api.v1.waitpoints.tokens.test.ts @@ -22,7 +22,7 @@ import { ownerEngine, CUID_LENGTH, } from "@trigger.dev/core/v3/isomorphic"; -import { Prisma, type PrismaClient } from "@trigger.dev/database"; +import type { Prisma, PrismaClient } from "@trigger.dev/database"; import { trace } from "@opentelemetry/api"; vi.setConfig({ testTimeout: 60_000 }); diff --git a/apps/webapp/test/crossSeamGuard.proof.test.ts b/apps/webapp/test/crossSeamGuard.proof.test.ts index d5ab301f949..ec59649dd41 100644 --- a/apps/webapp/test/crossSeamGuard.proof.test.ts +++ b/apps/webapp/test/crossSeamGuard.proof.test.ts @@ -1,5 +1,5 @@ import { heteroPostgresTest } from "@internal/testcontainers"; -import { PrismaClient } from "@trigger.dev/database"; +import type { PrismaClient } from "@trigger.dev/database"; import { existsSync, readFileSync } from "node:fs"; import path from "node:path"; import { describe, expect, it } from "vitest"; From 56ec38c4396a201a72d337b93cf7f145e98512e2 Mon Sep 17 00:00:00 2001 From: Daniel Sutton Date: Fri, 3 Jul 2026 17:28:50 +0100 Subject: [PATCH 11/11] fix(run-ops split): wire batch-results route with split read-through deps The api.v1.batches.$batchParam.results route constructed ApiBatchResultsPresenter with no read-through deps, collapsing call() to a passthrough read off the control-plane replica only. A NEW-resident (ksuid) batch on the dedicated run-ops DB was invisible to that read and the route returned 404. Wire the presenter with splitEnabled + newClient + legacyReplica, mirroring the batch-list and waitpoint token read routes. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../app/routes/api.v1.batches.$batchParam.results.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/apps/webapp/app/routes/api.v1.batches.$batchParam.results.ts b/apps/webapp/app/routes/api.v1.batches.$batchParam.results.ts index 818241317ee..cf46cebdcee 100644 --- a/apps/webapp/app/routes/api.v1.batches.$batchParam.results.ts +++ b/apps/webapp/app/routes/api.v1.batches.$batchParam.results.ts @@ -1,6 +1,7 @@ import type { LoaderFunctionArgs } from "@remix-run/server-runtime"; import { json } from "@remix-run/server-runtime"; import { z } from "zod"; +import { $replica, runOpsNewReplica, runOpsSplitReadEnabled } from "~/db.server"; import { ApiBatchResultsPresenter } from "~/presenters/v3/ApiBatchResultsPresenter.server"; import { authenticateApiRequest } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; @@ -28,7 +29,11 @@ export async function loader({ request, params }: LoaderFunctionArgs) { const { batchParam } = parsed.data; try { - const presenter = new ApiBatchResultsPresenter(); + const presenter = new ApiBatchResultsPresenter(undefined, undefined, { + newClient: runOpsNewReplica, + legacyReplica: $replica, + splitEnabled: runOpsSplitReadEnabled, + }); const result = await presenter.call(batchParam, authenticationResult.environment); if (!result) {