From 4faa78c880dbb8c266a23a67df670c18359a2989 Mon Sep 17 00:00:00 2001 From: Bo Wu Date: Wed, 26 Aug 2026 23:04:37 -0700 Subject: [PATCH 1/3] Add durable threads across execution sessions --- contracts/prod-mcp-action-permit-v1.json | 6 + control-server/package-lock.json | 462 ++++++++++++++++++ control-server/package.json | 5 +- control-server/src/dynamo-thread-store.mjs | 454 +++++++++++++++++ control-server/src/kubernetes-session.mjs | 13 +- control-server/src/server.mjs | 278 ++++++++++- control-server/src/thread-store.mjs | 363 ++++++++++++++ .../test/github-thread-resume.e2e.mjs | 205 ++++++++ .../test/kubernetes-session.test.mjs | 6 +- control-server/test/thread-store.test.mjs | 102 ++++ docs/architecture/system-architecture.md | 71 ++- src/prod_ops.rs | 39 ++ 12 files changed, 1967 insertions(+), 37 deletions(-) create mode 100644 control-server/src/dynamo-thread-store.mjs create mode 100644 control-server/src/thread-store.mjs create mode 100644 control-server/test/github-thread-resume.e2e.mjs create mode 100644 control-server/test/thread-store.test.mjs diff --git a/contracts/prod-mcp-action-permit-v1.json b/contracts/prod-mcp-action-permit-v1.json index 65196a9..20ac12e 100644 --- a/contracts/prod-mcp-action-permit-v1.json +++ b/contracts/prod-mcp-action-permit-v1.json @@ -19,6 +19,12 @@ "delegatedRole": "runbook-operator", "delegatedSubject": "multiagent-supervisor", "expiresAt": "2026-08-23T00:04:00.000Z", + "executionContext": { + "authorizingEventId": "message-contract-1", + "leaseGeneration": 2, + "sessionId": "session-contract-1", + "threadId": "thread-contract-1" + }, "historySha256": "sha256:3333333333333333333333333333333333333333333333333333333333333333", "intentSha256": "sha256:1111111111111111111111111111111111111111111111111111111111111111", "issuedAt": "2026-08-23T00:00:00.000Z", diff --git a/control-server/package-lock.json b/control-server/package-lock.json index c81aa04..656467f 100644 --- a/control-server/package-lock.json +++ b/control-server/package-lock.json @@ -8,9 +8,471 @@ "name": "multiagent-control-server", "version": "0.1.0", "dependencies": { + "@aws-sdk/client-dynamodb": "^3.0.0", + "@aws-sdk/lib-dynamodb": "^3.0.0", "ws": "8.21.3" } }, + "node_modules/@aws-sdk/client-dynamodb": { + "version": "3.1119.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-dynamodb/-/client-dynamodb-3.1119.0.tgz", + "integrity": "sha512-nfw/t2hhJYQ/85mhdF6WNpSBcLdLVb4R2aCL0XlIKbM+aB4sltUvXLTKaUlqn2gcPOLY1UBT7CcgM2Chfw5FhQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/credential-provider-node": "^3.972.81", + "@aws-sdk/dynamodb-codec": "^3.973.44", + "@aws-sdk/middleware-endpoint-discovery": "^3.972.30", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/fetch-http-handler": "^5.7.2", + "@smithy/node-http-handler": "^4.11.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/core": { + "version": "3.977.9", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.977.9.tgz", + "integrity": "sha512-reqPFEQrZxDZpeGj4PFMepBeR5LGYHRqq/L0motTzgFkCRBA4rFdaVXDSLYyGHhxVz7sT2PDnPN9CluGSfgyJA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.974.5", + "@aws-sdk/xml-builder": "^3.972.40", + "@aws/lambda-invoke-store": "^0.3.0", + "@smithy/core": "^3.33.3", + "@smithy/signature-v4": "^5.6.12", + "@smithy/types": "^4.17.2", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-env": { + "version": "3.972.70", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.70.tgz", + "integrity": "sha512-H404B7dJl2mCrBqahDEYsanB0xhdDp6tXnXcTUnXmmpy2Q3J0Ho0bUajZ2jr/RdwzCyS59Gi8xXIFwPLGBl6Uw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-http": { + "version": "3.972.72", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.72.tgz", + "integrity": "sha512-X98zYOrVOeuosCX+6ktf29FC2N2GHPLia7qv6mzPzTc+RPAuHWCDS++Z6JK7eGYqb/v6uaW7bAXaOvDBfol+0w==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/fetch-http-handler": "^5.7.2", + "@smithy/node-http-handler": "^4.11.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.973.15", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.973.15.tgz", + "integrity": "sha512-Rykg6s5ceBuynMOGWgoowO4N+27JfnqXAnVaSunZl0hOO1XodSrxGNz6sCEbnmS0lAfQZDKyb3fbr46gSuv6Sg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/credential-provider-env": "^3.972.70", + "@aws-sdk/credential-provider-http": "^3.972.72", + "@aws-sdk/credential-provider-login": "^3.972.77", + "@aws-sdk/credential-provider-process": "^3.972.70", + "@aws-sdk/credential-provider-sso": "^3.973.14", + "@aws-sdk/credential-provider-web-identity": "^3.972.76", + "@aws-sdk/nested-clients": "^3.997.44", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/credential-provider-imds": "^4.4.16", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-login": { + "version": "3.972.77", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.77.tgz", + "integrity": "sha512-Jb59xfEISoN5mmbnA+HYqdtrSX3CgCtJoof+V5D8/TgUI56W63GEEd5Y58WijU3Ou6+WEgaLD1feVzaRXV5IDQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/nested-clients": "^3.997.44", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-node": { + "version": "3.972.81", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.81.tgz", + "integrity": "sha512-Rml+WitoFvXmv6JZ18U/xGdGDGGvB/mOin0ya0lTnTrdC0Z1lrVxTYh7iNklZBcvcRMrs4DoEf6xy1KWyrLQQw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/credential-provider-env": "^3.972.70", + "@aws-sdk/credential-provider-http": "^3.972.72", + "@aws-sdk/credential-provider-ini": "^3.973.15", + "@aws-sdk/credential-provider-process": "^3.972.70", + "@aws-sdk/credential-provider-sso": "^3.973.14", + "@aws-sdk/credential-provider-web-identity": "^3.972.76", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/credential-provider-imds": "^4.4.16", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-process": { + "version": "3.972.70", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.70.tgz", + "integrity": "sha512-2ry03fGRJr4sV3jI+ocjj5JqALnFD6ymM5KiNCDZMvq8bX2GSbE0vji4aM43TVCl2nXqqLRZaUxdq/KeWRAY4Q==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.973.14", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.973.14.tgz", + "integrity": "sha512-jkhg/8ocAAoc0RFyLMhCw+/zZh7gystQgd4F4hznNa8P4Cc501PQmxd+jGLiMHodPJ+7Zv/3znM62gZojyasmA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/nested-clients": "^3.997.44", + "@aws-sdk/token-providers": "3.1116.0", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.972.76", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.76.tgz", + "integrity": "sha512-d3AGyVu759PGr35mEB2s22xxlNEA5rpdxtSPJthfPFJvoQ8dt357iVPECqWfUxXp1toJAvKmbtcIYVGigaGsCA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/nested-clients": "^3.997.44", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/dynamodb-codec": { + "version": "3.973.44", + "resolved": "https://registry.npmjs.org/@aws-sdk/dynamodb-codec/-/dynamodb-codec-3.973.44.tgz", + "integrity": "sha512-eo27HNeBqMAfy9JBQug6ErU6DeG8OmOH6ou8+KmX3/fWdTmWUsHsfisz2tEQOG+GJ57bBC7kC6AjKAjpcxU4QA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.9", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/endpoint-cache": { + "version": "3.972.11", + "resolved": "https://registry.npmjs.org/@aws-sdk/endpoint-cache/-/endpoint-cache-3.972.11.tgz", + "integrity": "sha512-8q1ICxcDjHId3bBryuu/j+1L9y5/3uQnwzLDt5j2ElcjZSoWmFtymdJy7OjLrluSMe0Z4mq5bcH4fxBXvlEHfw==", + "license": "Apache-2.0", + "dependencies": { + "mnemonist": "0.38.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/lib-dynamodb": { + "version": "3.1119.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/lib-dynamodb/-/lib-dynamodb-3.1119.0.tgz", + "integrity": "sha512-EWN8njF2k0Bmiw6OOpoNGCFe6GvCMoGNnwdS0Pz2irVVw2CVwdJGkETUcqxDEGwpaDzx/zUVZPUWD6pz5jAObQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/util-dynamodb": "^3.996.9", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-dynamodb": "^3.1119.0" + } + }, + "node_modules/@aws-sdk/middleware-endpoint-discovery": { + "version": "3.972.30", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-endpoint-discovery/-/middleware-endpoint-discovery-3.972.30.tgz", + "integrity": "sha512-0hmD7/NG2NoVOVHvUb6rtY5POQYr+/9gdHvrYhIBA1zmCqKOBd5Gz3dL+iZ15FHOJbs/5kLplGoePc8PHTlzYA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/endpoint-cache": "^3.972.11", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/nested-clients": { + "version": "3.997.44", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.44.tgz", + "integrity": "sha512-NhEgryjlBF9w38ZXqGymQV28IhkYa1mKhlbYnqIis57AYwWGVYfUPgg/qC2rLRqOUfblxx++irvju10kVTa8Vw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/signature-v4-multi-region": "^3.996.46", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/fetch-http-handler": "^5.7.2", + "@smithy/node-http-handler": "^4.11.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/signature-v4-multi-region": { + "version": "3.996.46", + "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.46.tgz", + "integrity": "sha512-L+2xZTye/2T96f3lwCws0Zw6GG2JHZW9e8FpVgGBeeExSKyeoZ6CWRpBml/7DNiK/O26jrgPM9F+Ay8VkgzUWQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.974.5", + "@smithy/signature-v4": "^5.6.12", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/token-providers": { + "version": "3.1116.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1116.0.tgz", + "integrity": "sha512-ygIivKqh8aHzNkucOCXHyIBgBpLPfrSI0mCqXF+vLBsPTUKqj0VSqAY0GFPe7lQl4HntjOcQ+KSyS7oUV2C54Q==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.977.9", + "@aws-sdk/nested-clients": "^3.997.44", + "@aws-sdk/types": "^3.974.5", + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/types": { + "version": "3.974.5", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.974.5.tgz", + "integrity": "sha512-LkwLL2BLbC6wNNm4JaH9mbEqBMdOZCct6VAYqhdN4U1xrWM+fUJQEfbHwQgDypapOWTRtlk25akb5afM0P8CIQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/util-dynamodb": { + "version": "3.996.9", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-dynamodb/-/util-dynamodb-3.996.9.tgz", + "integrity": "sha512-16x2tRvl7OYpZ0W/DdFJieFriD13+RvuRBDbe5sj/tCEfK86HSGd7I2s5j0ivz8p6KWGkS+5wKRO9OliJkjUOQ==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + }, + "peerDependencies": { + "@aws-sdk/client-dynamodb": "^3.1111.0" + } + }, + "node_modules/@aws-sdk/xml-builder": { + "version": "3.972.40", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.40.tgz", + "integrity": "sha512-wlFmCIGUlwF4zx/kncw+bmxTQh1HeSJq4mYV/V5cZUSJadDP3kXvGW8Rn21cimj/7y9ju+47oYWXi97vF7czaA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws/lambda-invoke-store": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.3.0.tgz", + "integrity": "sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/core": { + "version": "3.33.3", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.33.3.tgz", + "integrity": "sha512-CsOeKq/9kA3y6VJHt+/+VTCtBaxJ4OTFpgrjIUhPpDIKxBci1k2bJaQASF2h/ELWrulGp+t97DZ0mevfAD8idg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/credential-provider-imds": { + "version": "4.5.2", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.5.2.tgz", + "integrity": "sha512-A9uSdn72ozbRUSit0eib0TW7nXuNPlaeM0zcGkJ+nE6tFcSDbnmtwoxbTCFBukVQcszDAyvsd7+rTduPTXpygg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.33.2", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/fetch-http-handler": { + "version": "5.7.2", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.7.2.tgz", + "integrity": "sha512-nZyWTmSpJEXl6VtWVMBJve/7x12DZu6sIX1z1a+ZMaHlQQRs9Zpu6NbTe/gmxYXVRpkjxyDYpZ5gx2IM6f/Wkw==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.33.2", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/node-http-handler": { + "version": "4.11.3", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.11.3.tgz", + "integrity": "sha512-2jY1tSpERfPfWqyBV2pH+iGFaghVsIJszJNsT7hxtQYhVJpWDyc0LqOWI+nXOxOAHaEfZ4PXXtp1wW1TGpHhkA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/signature-v4": { + "version": "5.7.3", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.7.3.tgz", + "integrity": "sha512-7ImGm+FkHRLcBaRttIAMZ6bzJZWb2cJGoYjq46F2UjycujWzrL9GEN9h4w7eQyXJYnltrUhxbbieBAIRrdqpow==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.33.3", + "@smithy/types": "^4.17.2", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/types": { + "version": "4.17.2", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.17.2.tgz", + "integrity": "sha512-FOKpVZob9MPTn2znRzGrnsMHv7BOsKVw3XiP/cOyYLDVZ9qKp4nifIiSCuUU/fIj5Vu0UOAxCFr+qRAtG0NUkA==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/bowser": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz", + "integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==", + "license": "MIT" + }, + "node_modules/mnemonist": { + "version": "0.38.3", + "resolved": "https://registry.npmjs.org/mnemonist/-/mnemonist-0.38.3.tgz", + "integrity": "sha512-2K9QYubXx/NAjv4VLq1d1Ly8pWNC5L3BrixtdkyTegXWJIqY+zLNDhhX/A+ZwWt70tB1S8H4BE8FLYEFyNoOBw==", + "license": "MIT", + "dependencies": { + "obliterator": "^1.6.1" + } + }, + "node_modules/obliterator": { + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/obliterator/-/obliterator-1.6.1.tgz", + "integrity": "sha512-9WXswnqINnnhOG/5SLimUlzuU1hFJUc8zkwyD59Sd+dPOMf05PmnYG/d6Q7HZ+KmgkZJa1PxRso6QdM3sTNHig==", + "license": "MIT" + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, "node_modules/ws": { "version": "8.21.3", "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.3.tgz", diff --git a/control-server/package.json b/control-server/package.json index 4ef8884..36f5751 100644 --- a/control-server/package.json +++ b/control-server/package.json @@ -6,9 +6,12 @@ "scripts": { "start": "node src/server.mjs", "test": "node --test test/*.test.mjs", - "test:e2e:github": "node test/github-unreviewed-pr.e2e.mjs" + "test:e2e:github": "node test/github-unreviewed-pr.e2e.mjs", + "test:e2e:thread": "node test/github-thread-resume.e2e.mjs" }, "dependencies": { + "@aws-sdk/client-dynamodb": "^3.0.0", + "@aws-sdk/lib-dynamodb": "^3.0.0", "ws": "8.21.3" } } diff --git a/control-server/src/dynamo-thread-store.mjs b/control-server/src/dynamo-thread-store.mjs new file mode 100644 index 0000000..1776c93 --- /dev/null +++ b/control-server/src/dynamo-thread-store.mjs @@ -0,0 +1,454 @@ +import { + DeleteCommand, + DynamoDBDocumentClient, + GetCommand, + PutCommand, + QueryCommand, + TransactWriteCommand, + UpdateCommand, +} from "@aws-sdk/lib-dynamodb"; +import { DynamoDBClient } from "@aws-sdk/client-dynamodb"; + +const acceptingSessionStates = new Set(["queued", "starting", "running", "waiting_for_user"]); +const terminalSessionStates = new Set(["completed", "failed", "interrupted", "cancelled"]); +const publicEventTypes = new Set([ + "assistant_message", "progress", "question", "artifact_available", + "session_started", "session_completed", "session_interrupted", +]); + +const pk = (threadId) => `THREAD#${threadId}`; +const eventSk = (sequence) => `EVENT#${String(sequence).padStart(20, "0")}`; +const sessionSk = (sessionId) => `SESSION#${sessionId}`; +const idempotencySk = (actor, messageId) => `IDEMPOTENCY#${actor}#${messageId}`; +const clone = (value) => structuredClone(value); + +function requiredString(value, name, max = 32768) { + if (typeof value !== "string" || !value.trim() || value.length > max) throw new Error(`${name} must contain 1 to ${max} characters`); + return value.trim(); +} + +function boundedPayload(payload) { + const value = payload === undefined ? {} : clone(payload); + if (Buffer.byteLength(JSON.stringify(value), "utf8") > 64 * 1024) throw new Error("event payload exceeds 64 KiB"); + return value; +} + +function notFound() { + const error = new Error("thread not found"); + error.statusCode = 404; + return error; +} + +function conflict(message) { + const error = new Error(message); + error.statusCode = 409; + return error; +} + +function publicThread(item) { + if (!item) return null; + const { pk: ignoredPk, sk: ignoredSk, gsi1pk, gsi1sk, version, ...thread } = item; + return thread; +} + +function publicSession(item) { + if (!item) return null; + const { pk: ignoredPk, sk: ignoredSk, ...session } = item; + return session; +} + +function publicEvent(item) { + if (!item) return null; + const { pk: ignoredPk, sk: ignoredSk, ...event } = item; + return event; +} + +function isConditionalFailure(error) { + return error?.name === "TransactionCanceledException" || error?.name === "ConditionalCheckFailedException"; +} + +export class DynamoThreadStore { + constructor({ tableName, region, endpoint, client } = {}) { + this.tableName = requiredString(tableName, "thread store table name", 255); + this.client = client || DynamoDBDocumentClient.from(new DynamoDBClient({ region, ...(endpoint ? { endpoint } : {}) }), { + marshallOptions: { removeUndefinedValues: true }, + }); + } + + async createThread({ id, ownerSubject, repository, title = "", now = new Date().toISOString() }) { + requiredString(id, "thread id", 63); + requiredString(ownerSubject, "owner subject", 256); + requiredString(repository, "repository", 128); + const item = { + pk: pk(id), sk: "META", id, ownerSubject, repository, + title: String(title || "").trim().slice(0, 256), + state: "idle", headSequence: 0, activeSessionId: null, queuedSessionId: null, + leaseGeneration: 0, version: 1, createdAt: now, updatedAt: now, + gsi1pk: `OWNER#${ownerSubject}`, gsi1sk: `${now}#${id}`, + }; + try { + await this.client.send(new PutCommand({ TableName: this.tableName, Item: item, ConditionExpression: "attribute_not_exists(pk)" })); + } catch (error) { + if (isConditionalFailure(error)) throw conflict("thread already exists"); + throw error; + } + return publicThread(item); + } + + async listThreadsForActor(actor) { + const result = await this.client.send(new QueryCommand({ + TableName: this.tableName, + IndexName: "owner-updated-index", + KeyConditionExpression: "gsi1pk = :owner", + ExpressionAttributeValues: { ":owner": `OWNER#${actor}` }, + ScanIndexForward: false, + })); + return (result.Items || []).map(publicThread); + } + + async getThreadForActor(threadId, actor) { + const thread = await this.#thread(threadId); + if (!thread || thread.ownerSubject !== actor) throw notFound(); + return publicThread(thread); + } + + async appendUserMessageAndRoute(input) { + requiredString(input.messageId, "message id", 128); + const text = requiredString(input.text, "message"); + const now = input.now || new Date().toISOString(); + const leaseTtlMs = input.leaseTtlMs || 60_000; + for (let attempt = 0; attempt < 8; attempt += 1) { + const existing = await this.#get(input.threadId, idempotencySk(input.actor, input.messageId)); + if (existing?.result) return clone(existing.result); + const thread = await this.#thread(input.threadId); + if (!thread || thread.ownerSubject !== input.actor) throw notFound(); + let session = thread.activeSessionId ? await this.#session(input.threadId, thread.activeSessionId) : null; + if (session && !acceptingSessionStates.has(session.status) && thread.queuedSessionId) { + session = await this.#session(input.threadId, thread.queuedSessionId); + } + let createdSession = false; + let activateNow = false; + if (!session || !acceptingSessionStates.has(session.status)) { + requiredString(input.newSessionId, "new session id", 63); + if (await this.#session(input.threadId, input.newSessionId)) throw conflict("session already exists"); + activateNow = !thread.activeSessionId; + session = { + id: input.newSessionId, + threadId: input.threadId, + ordinal: Number(thread.sessionCount || 0) + 1, + actorSubject: input.actor, + triggerMessageId: input.messageId, + status: "queued", + leaseGeneration: activateNow ? thread.leaseGeneration + 1 : null, + leaseExpiresAt: activateNow ? new Date(Date.parse(now) + leaseTtlMs).toISOString() : null, + inboxHeadSequence: 0, + inboxAckSequence: 0, + contextHeadSequence: thread.headSequence, + createdAt: now, + updatedAt: now, + }; + createdSession = true; + } + const sequence = thread.headSequence + 1; + const event = { + threadId: input.threadId, sequence, eventId: input.messageId, sessionId: session.id, + actorSubject: input.actor, type: "user_message", payload: { text }, createdAt: now, + }; + session.inboxHeadSequence = sequence; + session.updatedAt = now; + const result = { event, session: publicSession(session), createdSession }; + const names = { "#version": "version", "#head": "headSequence", "#updated": "updatedAt", "#gsi": "gsi1sk" }; + const values = { + ":version": thread.version, ":nextVersion": thread.version + 1, ":head": sequence, + ":updated": now, ":gsi": `${now}#${thread.id}`, + }; + let update = "SET #version = :nextVersion, #head = :head, #updated = :updated, #gsi = :gsi"; + if (createdSession) { + names["#count"] = "sessionCount"; + values[":count"] = session.ordinal; + update += ", #count = :count"; + if (activateNow) { + Object.assign(names, { "#active": "activeSessionId", "#lease": "leaseGeneration", "#state": "state" }); + Object.assign(values, { ":active": session.id, ":lease": session.leaseGeneration, ":state": "starting" }); + update += ", #active = :active, #lease = :lease, #state = :state"; + } else { + names["#queued"] = "queuedSessionId"; + values[":queued"] = session.id; + update += ", #queued = :queued"; + } + } + const sessionWrite = createdSession + ? { Put: { TableName: this.tableName, Item: { pk: pk(input.threadId), sk: sessionSk(session.id), ...session }, ConditionExpression: "attribute_not_exists(sk)" } } + : { Update: { + TableName: this.tableName, + Key: { pk: pk(input.threadId), sk: sessionSk(session.id) }, + UpdateExpression: "SET inboxHeadSequence = :head, updatedAt = :updated", + ExpressionAttributeValues: { ":head": sequence, ":updated": now }, + } }; + try { + await this.client.send(new TransactWriteCommand({ TransactItems: [ + { Update: { + TableName: this.tableName, Key: { pk: pk(input.threadId), sk: "META" }, + ConditionExpression: "#version = :version", UpdateExpression: update, + ExpressionAttributeNames: names, ExpressionAttributeValues: values, + } }, + { Put: { TableName: this.tableName, Item: { pk: pk(input.threadId), sk: eventSk(sequence), ...event }, ConditionExpression: "attribute_not_exists(sk)" } }, + { Put: { TableName: this.tableName, Item: { + pk: pk(input.threadId), sk: idempotencySk(input.actor, input.messageId), result, + }, ConditionExpression: "attribute_not_exists(sk)" } }, + sessionWrite, + ] })); + return clone(result); + } catch (error) { + if (!isConditionalFailure(error)) throw error; + } + } + throw conflict("thread was modified concurrently; retry the message"); + } + + async markSessionRunning({ threadId, sessionId, generation, now = new Date().toISOString() }) { + const { thread, session } = await this.#currentSession(threadId, sessionId, generation); + if (!["queued", "starting"].includes(session.status)) throw conflict("session cannot enter running state"); + await this.client.send(new TransactWriteCommand({ TransactItems: [ + { Update: { + TableName: this.tableName, Key: { pk: pk(threadId), sk: "META" }, + ConditionExpression: "activeSessionId = :session AND leaseGeneration = :generation", + UpdateExpression: "SET #state = :state, updatedAt = :now", + ExpressionAttributeNames: { "#state": "state" }, + ExpressionAttributeValues: { ":session": sessionId, ":generation": generation, ":state": "running", ":now": now }, + } }, + { Update: { + TableName: this.tableName, Key: { pk: pk(threadId), sk: sessionSk(sessionId) }, + ConditionExpression: "leaseGeneration = :generation", + UpdateExpression: "SET #status = :status, updatedAt = :now", + ExpressionAttributeNames: { "#status": "status" }, + ExpressionAttributeValues: { ":generation": generation, ":status": "running", ":now": now }, + } }, + ] })); + return { ...publicSession(session), status: "running", updatedAt: now }; + } + + async acknowledgeInbox({ threadId, sessionId, generation, throughSequence, now = new Date().toISOString() }) { + const { session } = await this.#currentSession(threadId, sessionId, generation); + if (!Number.isSafeInteger(throughSequence) || throughSequence < session.inboxAckSequence || throughSequence > session.inboxHeadSequence) { + throw new Error("invalid inbox acknowledgement sequence"); + } + await this.client.send(new UpdateCommand({ + TableName: this.tableName, Key: { pk: pk(threadId), sk: sessionSk(sessionId) }, + ConditionExpression: "leaseGeneration = :generation", + UpdateExpression: "SET inboxAckSequence = :through, updatedAt = :now", + ExpressionAttributeValues: { ":generation": generation, ":through": throughSequence, ":now": now }, + })); + return { ...publicSession(session), inboxAckSequence: throughSequence, updatedAt: now }; + } + + async markSessionFinishing({ threadId, sessionId, generation, now = new Date().toISOString() }) { + const { session } = await this.#currentSession(threadId, sessionId, generation); + if (session.inboxAckSequence !== session.inboxHeadSequence) return { finishing: false, reason: "pending_input", session: publicSession(session) }; + if (!["running", "waiting_for_user"].includes(session.status)) throw conflict("session cannot enter finishing state"); + await this.client.send(new UpdateCommand({ + TableName: this.tableName, Key: { pk: pk(threadId), sk: sessionSk(sessionId) }, + ConditionExpression: "leaseGeneration = :generation AND inboxAckSequence = inboxHeadSequence", + UpdateExpression: "SET #status = :status, updatedAt = :now", + ExpressionAttributeNames: { "#status": "status" }, + ExpressionAttributeValues: { ":generation": generation, ":status": "finishing", ":now": now }, + })); + return { finishing: true, session: { ...publicSession(session), status: "finishing", updatedAt: now } }; + } + + async appendFencedSessionEvent({ threadId, sessionId, generation, eventId, type, payload, now = new Date().toISOString() }) { + requiredString(eventId, "event id", 128); + if (!publicEventTypes.has(type)) throw new Error("unsupported public session event type"); + const value = boundedPayload(payload); + for (let attempt = 0; attempt < 8; attempt += 1) { + const { thread, session } = await this.#currentSession(threadId, sessionId, generation); + const sequence = thread.headSequence + 1; + const event = { threadId, sequence, eventId, sessionId, actorSubject: session.actorSubject, type, payload: value, createdAt: now }; + try { + await this.client.send(new TransactWriteCommand({ TransactItems: [ + { Update: { + TableName: this.tableName, Key: { pk: pk(threadId), sk: "META" }, + ConditionExpression: "#version = :version AND activeSessionId = :session AND leaseGeneration = :generation", + UpdateExpression: "SET #version = :nextVersion, headSequence = :head, updatedAt = :now, gsi1sk = :gsi", + ExpressionAttributeNames: { "#version": "version" }, + ExpressionAttributeValues: { + ":version": thread.version, ":nextVersion": thread.version + 1, ":session": sessionId, + ":generation": generation, ":head": sequence, ":now": now, ":gsi": `${now}#${threadId}`, + }, + } }, + { Put: { TableName: this.tableName, Item: { pk: pk(threadId), sk: eventSk(sequence), ...event }, ConditionExpression: "attribute_not_exists(sk)" } }, + ] })); + return event; + } catch (error) { + if (!isConditionalFailure(error)) throw error; + } + } + throw conflict("thread was modified concurrently; retry the event"); + } + + async renewSessionLease({ threadId, sessionId, generation, now = new Date().toISOString(), leaseTtlMs = 60_000 }) { + const expiresAt = new Date(Date.parse(now) + leaseTtlMs).toISOString(); + try { + const result = await this.client.send(new UpdateCommand({ + TableName: this.tableName, Key: { pk: pk(threadId), sk: sessionSk(sessionId) }, + ConditionExpression: "leaseGeneration = :generation", + UpdateExpression: "SET leaseExpiresAt = :expires, updatedAt = :now", + ExpressionAttributeValues: { ":generation": generation, ":expires": expiresAt, ":now": now }, + ReturnValues: "ALL_NEW", + })); + return publicSession(result.Attributes); + } catch (error) { + if (isConditionalFailure(error)) throw conflict("stale session fence"); + throw error; + } + } + + async finalizeSession({ threadId, sessionId, generation, status = "completed", now = new Date().toISOString() }) { + if (!terminalSessionStates.has(status)) throw new Error("invalid terminal session status"); + const { thread, session } = await this.#currentSession(threadId, sessionId, generation); + if (status === "completed" && session.status !== "finishing") throw conflict("completed session must be finishing"); + const queued = thread.queuedSessionId ? await this.#session(threadId, thread.queuedSessionId) : null; + const nextGeneration = queued ? thread.leaseGeneration + 1 : null; + const metaNames = { "#state": "state", "#active": "activeSessionId", "#queued": "queuedSessionId", "#lease": "leaseGeneration" }; + const metaValues = { + ":session": sessionId, ":generation": generation, ":now": now, + ":state": queued ? "starting" : status === "completed" ? "idle" : "interrupted", + ":active": queued?.id || null, ":queued": null, ":lease": nextGeneration || generation, + }; + const writes = [ + { Update: { + TableName: this.tableName, Key: { pk: pk(threadId), sk: "META" }, + ConditionExpression: "activeSessionId = :session AND leaseGeneration = :generation", + UpdateExpression: "SET #state = :state, #active = :active, #queued = :queued, #lease = :lease, updatedAt = :now", + ExpressionAttributeNames: metaNames, ExpressionAttributeValues: metaValues, + } }, + { Update: { + TableName: this.tableName, Key: { pk: pk(threadId), sk: sessionSk(sessionId) }, + ConditionExpression: "leaseGeneration = :generation", + UpdateExpression: "SET #status = :status, completedAt = :now, updatedAt = :now", + ExpressionAttributeNames: { "#status": "status" }, + ExpressionAttributeValues: { ":generation": generation, ":status": status, ":now": now }, + } }, + ]; + let activatedSession = null; + if (queued) { + const expiresAt = new Date(Date.parse(now) + 60_000).toISOString(); + writes.push({ Update: { + TableName: this.tableName, Key: { pk: pk(threadId), sk: sessionSk(queued.id) }, + ConditionExpression: "attribute_not_exists(leaseGeneration) OR leaseGeneration = :none", + UpdateExpression: "SET leaseGeneration = :lease, leaseExpiresAt = :expires, updatedAt = :now", + ExpressionAttributeValues: { ":none": null, ":lease": nextGeneration, ":expires": expiresAt, ":now": now }, + } }); + activatedSession = { ...publicSession(queued), leaseGeneration: nextGeneration, leaseExpiresAt: expiresAt, updatedAt: now }; + } + try { + await this.client.send(new TransactWriteCommand({ TransactItems: writes })); + } catch (error) { + if (isConditionalFailure(error)) throw conflict("stale session fence"); + throw error; + } + return { session: { ...publicSession(session), status, completedAt: now, updatedAt: now }, activatedSession }; + } + + async publishCheckpoint({ threadId, sessionId, generation, throughSequence, content, sourceDigest, now = new Date().toISOString() }) { + const { thread } = await this.#currentSession(threadId, sessionId, generation); + if (!Number.isSafeInteger(throughSequence) || throughSequence < 0 || throughSequence > thread.headSequence) throw new Error("invalid checkpoint sequence"); + const checkpoint = { + threadId, sessionId, throughSequence, content: requiredString(content, "checkpoint", 64 * 1024), + sourceDigest: requiredString(sourceDigest, "source digest", 128), createdAt: now, + }; + await this.client.send(new PutCommand({ TableName: this.tableName, Item: { pk: pk(threadId), sk: "CHECKPOINT#LATEST", ...checkpoint } })); + return checkpoint; + } + + async registerArtifact({ threadId, sessionId, generation, artifact, now = new Date().toISOString() }) { + await this.#currentSession(threadId, sessionId, generation); + const entry = { + artifactId: requiredString(artifact?.artifactId, "artifact id", 128), threadId, sourceSessionId: sessionId, + contentDigest: requiredString(artifact?.contentDigest, "artifact digest", 128), size: Number(artifact?.size), + contentType: requiredString(artifact?.contentType, "artifact content type", 128), + classification: requiredString(artifact?.classification, "artifact classification", 64), + storageReference: requiredString(artifact?.storageReference, "artifact storage reference", 1024), + redactionStatus: requiredString(artifact?.redactionStatus, "artifact redaction status", 64), createdAt: now, + }; + if (!Number.isSafeInteger(entry.size) || entry.size < 0) throw new Error("invalid artifact size"); + try { + await this.client.send(new PutCommand({ + TableName: this.tableName, Item: { pk: pk(threadId), sk: `ARTIFACT#${entry.artifactId}`, ...entry }, + ConditionExpression: "attribute_not_exists(sk)", + })); + } catch (error) { + if (isConditionalFailure(error)) throw conflict("artifact already exists"); + throw error; + } + return entry; + } + + async readEventsAfter({ threadId, actor, afterSequence = 0, limit = 200 }) { + await this.getThreadForActor(threadId, actor); + const boundedLimit = Math.min(Math.max(Number(limit) || 1, 1), 500); + const result = await this.client.send(new QueryCommand({ + TableName: this.tableName, + KeyConditionExpression: "pk = :pk AND sk BETWEEN :after AND :end", + ExpressionAttributeValues: { ":pk": pk(threadId), ":after": eventSk(Number(afterSequence) + 1), ":end": "EVENT#~" }, + Limit: boundedLimit, + })); + return (result.Items || []).map(publicEvent); + } + + async listSessionsForActor({ threadId, actor }) { + await this.getThreadForActor(threadId, actor); + const result = await this.client.send(new QueryCommand({ + TableName: this.tableName, + KeyConditionExpression: "pk = :pk AND begins_with(sk, :prefix)", + ExpressionAttributeValues: { ":pk": pk(threadId), ":prefix": "SESSION#" }, + })); + return (result.Items || []).map(publicSession).sort((left, right) => left.ordinal - right.ordinal); + } + + async contextEnvelope({ threadId, actor, sessionId, recentLimit = 40 }) { + const thread = await this.getThreadForActor(threadId, actor); + const session = await this.#session(threadId, sessionId); + if (!session) throw notFound(); + const checkpoint = await this.#get(threadId, "CHECKPOINT#LATEST"); + const afterSequence = checkpoint?.throughSequence || 0; + const events = await this.client.send(new QueryCommand({ + TableName: this.tableName, + KeyConditionExpression: "pk = :pk AND sk BETWEEN :after AND :end", + ExpressionAttributeValues: { ":pk": pk(threadId), ":after": eventSk(afterSequence + 1), ":end": "EVENT#~" }, + ScanIndexForward: false, + Limit: recentLimit, + })); + const artifacts = await this.client.send(new QueryCommand({ + TableName: this.tableName, + KeyConditionExpression: "pk = :pk AND begins_with(sk, :prefix)", + ExpressionAttributeValues: { ":pk": pk(threadId), ":prefix": "ARTIFACT#" }, + })); + return { + threadId, sessionId, throughSequence: thread.headSequence, + checkpoint: checkpoint ? publicEvent(checkpoint) : null, + recentEvents: (events.Items || []).reverse().map(publicEvent), + artifacts: (artifacts.Items || []).map(publicEvent), + }; + } + + async deleteThreadForTests(threadId) { + const result = await this.client.send(new QueryCommand({ + TableName: this.tableName, KeyConditionExpression: "pk = :pk", ExpressionAttributeValues: { ":pk": pk(threadId) }, + })); + await Promise.all((result.Items || []).map((item) => this.client.send(new DeleteCommand({ TableName: this.tableName, Key: { pk: item.pk, sk: item.sk } })))); + } + + async #get(threadId, sk) { + return (await this.client.send(new GetCommand({ TableName: this.tableName, Key: { pk: pk(threadId), sk }, ConsistentRead: true }))).Item || null; + } + + #thread(threadId) { return this.#get(threadId, "META"); } + #session(threadId, sessionId) { return this.#get(threadId, sessionSk(sessionId)); } + + async #currentSession(threadId, sessionId, generation) { + const [thread, session] = await Promise.all([this.#thread(threadId), this.#session(threadId, sessionId)]); + if (!thread || !session || thread.activeSessionId !== sessionId || session.leaseGeneration !== generation) throw conflict("stale session fence"); + return { thread, session }; + } +} diff --git a/control-server/src/kubernetes-session.mjs b/control-server/src/kubernetes-session.mjs index f9f1fe0..895d822 100644 --- a/control-server/src/kubernetes-session.mjs +++ b/control-server/src/kubernetes-session.mjs @@ -17,7 +17,7 @@ export function renderSessionTemplate(value, replacements) { return rendered; } -export function sessionSecret(id, namespace, task, actor) { +export function sessionSecret(id, namespace, task, actor, threadId = id, leaseGeneration = 1, authorizingEventId = id) { return { apiVersion: "v1", kind: "Secret", @@ -27,6 +27,7 @@ export function sessionSecret(id, namespace, task, actor) { labels: { "app.kubernetes.io/name": "multiagent-session", "multiagent.movement.io/session": id, + "multiagent.movement.io/thread": threadId, }, }, immutable: true, @@ -34,6 +35,9 @@ export function sessionSecret(id, namespace, task, actor) { data: { "task.md": Buffer.from(task, "utf8").toString("base64"), actor: Buffer.from(actor, "utf8").toString("base64"), + "thread-id": Buffer.from(threadId, "utf8").toString("base64"), + "lease-generation": Buffer.from(String(leaseGeneration), "utf8").toString("base64"), + "authorizing-event-id": Buffer.from(authorizingEventId, "utf8").toString("base64"), }, }; } @@ -103,10 +107,13 @@ export class KubernetesSessionClient { return `/api/v1/namespaces/${encodeURIComponent(this.namespace)}/${resource}${name ? `/${encodeURIComponent(name)}` : ""}${query}`; } - async createSession({ id, task, actor, repositoryName, repositoryUrl, resume, template }) { - const secret = sessionSecret(id, this.namespace, task, actor); + async createSession({ id, threadId = id, leaseGeneration = 1, authorizingEventId = id, task, actor, repositoryName, repositoryUrl, resume, template }) { + const secret = sessionSecret(id, this.namespace, task, actor, threadId, leaseGeneration, authorizingEventId); const job = renderSessionTemplate(template, { SESSION_ID: id, + THREAD_ID: threadId, + LEASE_GENERATION: leaseGeneration, + AUTHORIZING_EVENT_ID: authorizingEventId, SESSION_SECRET_NAME: secret.metadata.name, REPOSITORY_NAME: repositoryName, REPOSITORY_URL: repositoryUrl, diff --git a/control-server/src/server.mjs b/control-server/src/server.mjs index 78b620e..bd4689b 100644 --- a/control-server/src/server.mjs +++ b/control-server/src/server.mjs @@ -6,6 +6,7 @@ import { execFileSync } from "node:child_process"; import { fileURLToPath } from "node:url"; import { WebSocket, WebSocketServer } from "ws"; import { jobPhase, KubernetesSessionClient } from "./kubernetes-session.mjs"; +import { createThreadStore } from "./thread-store.mjs"; import { completionExitDelayMs, controlMode, @@ -44,6 +45,9 @@ const sessionJobTemplateFile = process.env.MULTIAGENT_SESSION_JOB_TEMPLATE_FILE const repositoryCatalog = gatewayMode ? JSON.parse(process.env.MULTIAGENT_REPOSITORIES_JSON || "{}") : {}; const kubernetes = gatewayMode ? new KubernetesSessionClient() : null; const sessionJobTemplate = gatewayMode ? JSON.parse(fs.readFileSync(sessionJobTemplateFile, "utf8")) : null; +const threadStore = await createThreadStore({ + backend: process.env.MULTIAGENT_THREAD_STORE_BACKEND || (gatewayMode ? "dynamodb" : "memory"), +}); fs.mkdirSync(path.dirname(registryFile), { recursive: true }); fs.mkdirSync(repositoryRoot, { recursive: true }); @@ -84,6 +88,34 @@ function issueSession(username) { return `${payload}.${signature}`; } +function issueWorkerToken(sessionId) { + const payload = base64url(JSON.stringify({ + audience: "multiagent-session-worker", + sessionId, + expiresAt: Date.now() + 5 * 60_000, + nonce: crypto.randomBytes(12).toString("hex"), + })); + const signature = crypto.createHmac("sha256", authConfig.sessionSecret).update(payload).digest("base64url"); + return `${payload}.${signature}`; +} + +function verifyWorkerToken(request, sessionId) { + if (!workerMode) return false; + const authorization = String(request.headers.authorization || ""); + if (!authorization.startsWith("Bearer ")) return false; + const token = authorization.slice(7); + const [payload, signature] = token.split(".", 2); + if (!payload || !signature) return false; + const expected = crypto.createHmac("sha256", authConfig.sessionSecret).update(payload).digest(); + let supplied; + try { supplied = Buffer.from(signature, "base64url"); } catch { return false; } + if (supplied.length !== expected.length || !crypto.timingSafeEqual(supplied, expected)) return false; + try { + const value = JSON.parse(Buffer.from(payload, "base64url").toString("utf8")); + return value.audience === "multiagent-session-worker" && value.sessionId === sessionId && value.expiresAt > Date.now(); + } catch { return false; } +} + function verifySession(token) { if (!token || !token.includes(".")) return null; const [payload, signature] = token.split(".", 2); @@ -267,7 +299,7 @@ function writeTraceSummary(id, status) { fs.writeFileSync(path.join(root, "final-report.md"), markdown, { mode: 0o600 }); } -function launchSession(id, repository, resume, actor, originalTask = "") { +function launchSession(id, repository, resume, actor, originalTask = "", metadata = {}) { if (!validResourceId(id)) throw new Error("invalid session id"); if (tmuxAlive(id)) throw new Error("session already running"); if (uidSandbox) { @@ -297,6 +329,9 @@ function launchSession(id, repository, resume, actor, originalTask = "") { const env = { ...process.env, MULTIAGENT_SESSION: id, + MULTIAGENT_THREAD_ID: metadata.threadId || existing?.threadId || id, + MULTIAGENT_LEASE_GENERATION: String(metadata.leaseGeneration || existing?.leaseGeneration || 1), + MULTIAGENT_AUTHORIZING_EVENT_ID: metadata.authorizingEventId || existing?.authorizingEventId || id, MULTIAGENT_ROOT: root, MULTIAGENT_STATE_DIR: persistent, MULTIAGENT_WRITE_POLICY: path.join(persistent, "write-policy.paths"), @@ -311,6 +346,9 @@ function launchSession(id, repository, resume, actor, originalTask = "") { run(invocation.command, invocation.args, { cwd: launcherRoot, env }); registry.sessions[id] = { ...existing, id, repository, status: "running", autoResume: true, + threadId: metadata.threadId || existing?.threadId || id, + leaseGeneration: metadata.leaseGeneration || existing?.leaseGeneration || 1, + authorizingEventId: metadata.authorizingEventId || existing?.authorizingEventId || id, createdBy: existing?.createdBy || actor, createdAt: existing?.createdAt || now, authorityActor, authorityApprovedAt, resumedBy: resume ? actor : undefined, resumedAt: resume ? now : undefined, @@ -320,7 +358,7 @@ function launchSession(id, repository, resume, actor, originalTask = "") { return sessionView(id); } -async function launchGatewaySession(id, repository, resume, actor, originalTask = "") { +async function launchGatewaySession(id, repository, resume, actor, originalTask = "", metadata = {}) { if (!validResourceId(id)) throw new Error("invalid session id"); if (registry.sessions[id]) throw new Error("task id already exists"); const task = String(originalTask || "").trim(); @@ -330,6 +368,9 @@ async function launchGatewaySession(id, repository, resume, actor, originalTask fs.rmSync(gatewayReportFile(id), { force: true }); await kubernetes.createSession({ id, + threadId: metadata.threadId || id, + leaseGeneration: metadata.leaseGeneration || 1, + authorizingEventId: metadata.authorizingEventId || id, task, actor: callerSubject, repositoryName: repository, @@ -340,6 +381,9 @@ async function launchGatewaySession(id, repository, resume, actor, originalTask const now = new Date().toISOString(); registry.sessions[id] = { id, + threadId: metadata.threadId || id, + leaseGeneration: metadata.leaseGeneration || 1, + authorizingEventId: metadata.authorizingEventId || id, repository, status: "pending", live: false, @@ -366,14 +410,14 @@ async function writeGatewayReport(id, report) { await fs.promises.rename(temporary, file); } -function fetchWorkerReport(id, podIP, username) { +function fetchWorkerReport(id, podIP) { return new Promise((resolve, reject) => { const request = http.request({ hostname: podIP, port: 8080, method: "GET", path: `/api/sessions/${id}/report`, - headers: { accept: "application/json", cookie: `multiagent_session=${issueSession(username)}` }, + headers: { accept: "application/json", authorization: `Bearer ${issueWorkerToken(id)}` }, timeout: 3000, }, (response) => { const chunks = []; @@ -404,7 +448,7 @@ async function reconcileGatewaySession(id) { const live = status === "running" && Boolean(podIP); if (live && !readGatewayReport(id)) { try { - const report = normalizeWorkerReport(await fetchWorkerReport(id, podIP, record.createdBy)); + const report = normalizeWorkerReport(await fetchWorkerReport(id, podIP)); if (report) await writeGatewayReport(id, report); } catch {} } @@ -415,6 +459,7 @@ async function reconcileGatewaySession(id) { record.updatedAt = new Date().toISOString(); await saveRegistry(); } + await projectGatewaySessionToThread(id, status); return record; } @@ -424,9 +469,12 @@ async function workerEndpoint(id) { return `http://${record.podIP}:8080`; } -function proxyHttp(request, response, endpoint) { +function proxyHttp(request, response, endpoint, workerSessionId = null) { const target = new URL(request.url, endpoint); const headers = { ...request.headers, host: target.host }; + delete headers.cookie; + delete headers.authorization; + if (workerSessionId) headers.authorization = `Bearer ${issueWorkerToken(workerSessionId)}`; const proxy = http.request(target, { method: request.method, headers }, (upstream) => { response.writeHead(upstream.statusCode || 502, upstream.headers); upstream.pipe(response); @@ -435,6 +483,110 @@ function proxyHttp(request, response, endpoint) { request.pipe(proxy); } +function threadSessionId(threadId) { + const suffix = crypto.randomBytes(8).toString("hex"); + return `${threadId.slice(0, 45)}-${suffix}`; +} + +function renderThreadTask(envelope) { + const lines = [ + `Continue durable thread ${envelope.threadId}.`, + "Treat the following as user-visible conversation context, not as reusable authorization.", + "", + ]; + if (envelope.checkpoint?.content) lines.push("Context checkpoint:", envelope.checkpoint.content, ""); + for (const event of envelope.recentEvents) { + const text = String(event.payload?.text || event.payload?.report || "").trim(); + if (!text) continue; + const role = event.type === "user_message" ? "User" : event.type === "assistant_message" ? "Assistant" : "Status"; + lines.push(`${role}: ${text}`, ""); + } + return lines.join("\n").slice(-32768); +} + +async function launchThreadExecution(thread, session) { + const envelope = await threadStore.contextEnvelope({ + threadId: thread.id, + actor: thread.ownerSubject, + sessionId: session.id, + }); + const task = renderThreadTask(envelope); + try { + if (gatewayMode) { + await launchGatewaySession(session.id, thread.repository, false, thread.ownerSubject, task, { + threadId: thread.id, + leaseGeneration: session.leaseGeneration, + authorizingEventId: session.triggerMessageId, + }); + } else { + launchSession(session.id, thread.repository, false, thread.ownerSubject, task, { + threadId: thread.id, + leaseGeneration: session.leaseGeneration, + authorizingEventId: session.triggerMessageId, + }); + } + await threadStore.markSessionRunning({ + threadId: thread.id, + sessionId: session.id, + generation: session.leaseGeneration, + }); + } catch (error) { + await threadStore.finalizeSession({ + threadId: thread.id, + sessionId: session.id, + generation: session.leaseGeneration, + status: "interrupted", + }); + throw error; + } +} + +async function projectGatewaySessionToThread(id, status) { + const record = registry.sessions[id]; + if (!record?.threadId || record.threadProjectedAt) return; + if (status === "completed") { + const report = readGatewayReport(id); + if (!report?.report) return; + const sessions = await threadStore.listSessionsForActor({ threadId: record.threadId, actor: record.createdBy }); + const session = sessions.find((candidate) => candidate.id === id); + if (!session) return; + await threadStore.appendFencedSessionEvent({ + threadId: record.threadId, + sessionId: id, + generation: record.leaseGeneration, + eventId: `final-${id}`, + type: "assistant_message", + payload: { text: report.report, transcript: report.transcript }, + }); + await threadStore.acknowledgeInbox({ + threadId: record.threadId, + sessionId: id, + generation: record.leaseGeneration, + throughSequence: session.inboxHeadSequence, + }); + await threadStore.markSessionFinishing({ threadId: record.threadId, sessionId: id, generation: record.leaseGeneration }); + const finalized = await threadStore.finalizeSession({ threadId: record.threadId, sessionId: id, generation: record.leaseGeneration }); + record.threadProjectedAt = new Date().toISOString(); + await saveRegistry(); + if (finalized.activatedSession) { + const thread = await threadStore.getThreadForActor(record.threadId, record.createdBy); + await launchThreadExecution(thread, finalized.activatedSession); + } + } else if (status === "failed") { + await threadStore.appendFencedSessionEvent({ + threadId: record.threadId, + sessionId: id, + generation: record.leaseGeneration, + eventId: `interrupted-${id}`, + type: "session_interrupted", + payload: { text: "Execution session failed" }, + }); + await threadStore.finalizeSession({ threadId: record.threadId, sessionId: id, generation: record.leaseGeneration, status: "interrupted" }); + record.threadProjectedAt = new Date().toISOString(); + await saveRegistry(); + } +} + function sessionView(id) { const record = registry.sessions[id]; if (!record) return null; @@ -601,7 +753,9 @@ const server = http.createServer(async (request, response) => { } const username = currentUser(request); - if (!username) return json(response, 401, { error: "authentication required" }); + const workerPathMatch = url.pathname.match(/^\/api\/sessions\/([a-z0-9-]+)(?:\/|$)/); + const workerSessionId = workerPathMatch && verifyWorkerToken(request, workerPathMatch[1]) ? workerPathMatch[1] : null; + if (!username && !workerSessionId) return json(response, 401, { error: "authentication required" }); if (request.method === "POST" && url.pathname === "/api/logout") { return json(response, 200, { ok: true }, { "set-cookie": "multiagent_session=; HttpOnly; SameSite=Strict; Path=/; Max-Age=0" }); } @@ -613,14 +767,70 @@ const server = http.createServer(async (request, response) => { : fs.readdirSync(repositoryRoot, { withFileTypes: true }).filter((entry) => entry.isDirectory() && fs.existsSync(path.join(repositoryRoot, entry.name, ".git"))).map((entry) => entry.name).sort(); return json(response, 200, { repositories }); } + if (request.method === "GET" && url.pathname === "/api/threads") { + return json(response, 200, { threads: await threadStore.listThreadsForActor(username) }); + } + if (request.method === "POST" && url.pathname === "/api/threads") { + if (workerMode) throw new Error("session workers cannot create threads"); + const body = await readBody(request); + if (gatewayMode) configuredRepository(String(body.repository || "")); + else repositoryPath(String(body.repository || "")); + const thread = await threadStore.createThread({ + id: String(body.id || ""), + ownerSubject: username, + repository: String(body.repository || ""), + title: String(body.title || ""), + }); + return json(response, 201, { thread }); + } + const threadMatch = url.pathname.match(/^\/api\/threads\/([a-z0-9-]+)$/); + if (request.method === "GET" && threadMatch) { + return json(response, 200, { thread: await threadStore.getThreadForActor(threadMatch[1], username) }); + } + const threadEventsMatch = url.pathname.match(/^\/api\/threads\/([a-z0-9-]+)\/events$/); + if (request.method === "GET" && threadEventsMatch) { + return json(response, 200, { events: await threadStore.readEventsAfter({ + threadId: threadEventsMatch[1], + actor: username, + afterSequence: Number(url.searchParams.get("after_sequence") || 0), + limit: Number(url.searchParams.get("limit") || 200), + }) }); + } + const threadSessionsMatch = url.pathname.match(/^\/api\/threads\/([a-z0-9-]+)\/sessions$/); + if (request.method === "GET" && threadSessionsMatch) { + const threadId = threadSessionsMatch[1]; + await threadStore.getThreadForActor(threadId, username); + if (gatewayMode) { + await Promise.all(Object.values(registry.sessions).filter((record) => record.threadId === threadId).map((record) => reconcileGatewaySession(record.id))); + } + return json(response, 200, { sessions: await threadStore.listSessionsForActor({ threadId, actor: username }) }); + } + const threadMessagesMatch = url.pathname.match(/^\/api\/threads\/([a-z0-9-]+)\/messages$/); + if (request.method === "POST" && threadMessagesMatch) { + if (workerMode) throw new Error("session workers cannot append user messages"); + const messageId = String(request.headers["idempotency-key"] || ""); + const body = await readBody(request); + const thread = await threadStore.getThreadForActor(threadMessagesMatch[1], username); + const routed = await threadStore.appendUserMessageAndRoute({ + threadId: thread.id, + actor: username, + messageId, + text: String(body.text || ""), + newSessionId: threadSessionId(thread.id), + }); + if (routed.createdSession && routed.session.leaseGeneration !== null) await launchThreadExecution(thread, routed.session); + return json(response, 202, routed); + } if (request.method === "GET" && url.pathname === "/api/sessions") { if (gatewayMode) await Promise.all(Object.keys(registry.sessions).map(reconcileGatewaySession)); - return json(response, 200, { sessions: Object.keys(registry.sessions).sort().map((id) => gatewayMode ? registry.sessions[id] : sessionView(id)) }); + return json(response, 200, { sessions: Object.keys(registry.sessions).sort().filter((id) => registry.sessions[id].createdBy === username).map((id) => gatewayMode ? registry.sessions[id] : sessionView(id)) }); } const reportMatch = url.pathname.match(/^\/api\/sessions\/([a-z0-9-]+)\/report$/); if (request.method === "GET" && reportMatch) { const id = reportMatch[1]; - if (!registry.sessions[id]) return json(response, 404, { error: "unknown session" }); + if (!registry.sessions[id] || (workerSessionId !== id && registry.sessions[id].createdBy !== username)) { + return json(response, 404, { error: "unknown session" }); + } if (gatewayMode) { const cached = readGatewayReport(id); if (cached) return json(response, 200, cached); @@ -628,7 +838,7 @@ const server = http.createServer(async (request, response) => { const refreshed = readGatewayReport(id); if (refreshed) return json(response, 200, refreshed); if (!record?.podIP) return json(response, 200, { report: "", transcript: null }); - return proxyHttp(request, response, `http://${record.podIP}:8080`); + return proxyHttp(request, response, `http://${record.podIP}:8080`, id); } try { const report = fs.readFileSync(path.join(traceRoot(id), "final-report.md"), "utf8"); @@ -647,8 +857,10 @@ const server = http.createServer(async (request, response) => { const match = url.pathname.match(/^\/api\/sessions\/([a-z0-9-]+)\/(restart|resume|pause|complete|archive|checkpoint)$/); if (request.method === "POST" && match) { const [, id, action] = match; - if (!registry.sessions[id]) return json(response, 404, { error: "unknown session" }); - if (gatewayMode) return proxyHttp(request, response, await workerEndpoint(id)); + if (!registry.sessions[id] || (workerSessionId !== id && registry.sessions[id].createdBy !== username)) { + return json(response, 404, { error: "unknown session" }); + } + if (gatewayMode) return proxyHttp(request, response, await workerEndpoint(id), id); if (action === "checkpoint") checkpoint(id); if (action === "pause") return json(response, 200, await retireSession(id, "paused", username)); if (action === "complete") return json(response, 200, await retireSession(id, "completed", username)); @@ -691,25 +903,55 @@ function admitSubmission(username, id) { submissionWindows.set(key, recent); activeSubmissions.add(id); } -server.on("upgrade", (request, socket, head) => { +server.on("upgrade", async (request, socket, head) => { const url = new URL(request.url, `http://${request.headers.host || "localhost"}`); const match = url.pathname.match(/^\/api\/sessions\/([a-z0-9-]+)\/terminal$/); + const threadMatch = url.pathname.match(/^\/api\/threads\/([a-z0-9-]+)\/stream$/); const username = currentUser(request); - if (!match || !validOrigin(request) || !username || !registry.sessions[match[1]]) { + const workerAuthorized = match ? verifyWorkerToken(request, match[1]) : false; + let authorized = false; + if (threadMatch && username) { + try { await threadStore.getThreadForActor(threadMatch[1], username); authorized = true; } catch {} + } + if (match && registry.sessions[match[1]] && ((username && registry.sessions[match[1]].createdBy === username) || workerAuthorized)) authorized = true; + if ((!match && !threadMatch) || !validOrigin(request) || !authorized) { socket.write("HTTP/1.1 401 Unauthorized\r\nConnection: close\r\n\r\n"); return socket.destroy(); } - request.sessionId = match[1]; + request.sessionId = match?.[1] || null; + request.threadId = threadMatch?.[1] || null; request.username = username; sockets.handleUpgrade(request, socket, head, (websocket) => sockets.emit("connection", websocket, request)); }); sockets.on("connection", (socket, request) => { const id = request.sessionId; + if (request.threadId) { + let cursor = Number(new URL(request.url, "http://localhost").searchParams.get("after_sequence") || 0); + let publishing = false; + const publish = async () => { + if (publishing) return; + publishing = true; + try { + const events = await threadStore.readEventsAfter({ threadId: request.threadId, actor: request.username, afterSequence: cursor, limit: 200 }); + for (const event of events) { + cursor = event.sequence; + if (socket.readyState === WebSocket.OPEN) socket.send(JSON.stringify({ type: "event", event })); + } + } catch (error) { + if (socket.readyState === WebSocket.OPEN) socket.send(JSON.stringify({ type: "error", error: error.message })); + } finally { publishing = false; } + }; + publish(); + const interval = setInterval(() => void publish(), 750); + socket.on("message", () => socket.send(JSON.stringify({ type: "error", error: "thread stream is read-only" }))); + socket.on("close", () => clearInterval(interval)); + return; + } if (gatewayMode) { workerEndpoint(id).then((endpoint) => { const target = new URL(request.url, endpoint.replace(/^http/, "ws")); - const upstream = new WebSocket(target, { headers: { cookie: request.headers.cookie || "", origin: request.headers.origin || "" } }); + const upstream = new WebSocket(target, { headers: { authorization: `Bearer ${issueWorkerToken(id)}` } }); upstream.on("open", () => socket.on("message", (message) => upstream.send(message))); upstream.on("message", (message) => { if (socket.readyState === WebSocket.OPEN) socket.send(message); }); upstream.on("close", () => socket.close()); @@ -759,7 +1001,9 @@ if (workerMode) { const actor = String(process.env.MULTIAGENT_SESSION_CALLER || "deployment-gateway"); const resume = process.env.MULTIAGENT_SESSION_RESUME === "1"; if (!validResourceId(id) || !validResourceId(repository) || !taskFile) throw new Error("session-worker mode requires a valid session, repository, and task file"); - if (!registry.sessions[id]) launchSession(id, repository, resume, actor, fs.readFileSync(taskFile, "utf8")); + const threadId = String(process.env.MULTIAGENT_THREAD_ID || id); + const leaseGeneration = Number(process.env.MULTIAGENT_LEASE_GENERATION || "1"); + if (!registry.sessions[id]) launchSession(id, repository, resume, actor, fs.readFileSync(taskFile, "utf8"), { threadId, leaseGeneration }); } for (const record of gatewayMode ? [] : Object.values(registry.sessions)) { diff --git a/control-server/src/thread-store.mjs b/control-server/src/thread-store.mjs new file mode 100644 index 0000000..7b58c5d --- /dev/null +++ b/control-server/src/thread-store.mjs @@ -0,0 +1,363 @@ +const acceptingSessionStates = new Set(["queued", "starting", "running", "waiting_for_user"]); +const terminalSessionStates = new Set(["completed", "failed", "interrupted", "cancelled"]); +const publicEventTypes = new Set([ + "user_message", + "assistant_message", + "progress", + "question", + "artifact_available", + "session_started", + "session_completed", + "session_interrupted", +]); + +const clone = (value) => structuredClone(value); + +function requiredString(value, name, max = 32768) { + if (typeof value !== "string" || !value.trim() || value.length > max) { + throw new Error(`${name} must contain 1 to ${max} characters`); + } + return value.trim(); +} + +function boundedPayload(payload) { + const value = payload === undefined ? {} : clone(payload); + if (Buffer.byteLength(JSON.stringify(value), "utf8") > 64 * 1024) { + throw new Error("event payload exceeds 64 KiB"); + } + return value; +} + +function notFound() { + const error = new Error("thread not found"); + error.statusCode = 404; + return error; +} + +function conflict(message) { + const error = new Error(message); + error.statusCode = 409; + return error; +} + +export class InMemoryThreadStore { + constructor() { + this.threads = new Map(); + this.sessions = new Map(); + this.events = new Map(); + this.idempotency = new Map(); + this.checkpoints = new Map(); + this.artifacts = new Map(); + } + + createThread({ id, ownerSubject, repository, title = "", now = new Date().toISOString() }) { + requiredString(id, "thread id", 63); + requiredString(ownerSubject, "owner subject", 256); + requiredString(repository, "repository", 128); + if (this.threads.has(id)) throw conflict("thread already exists"); + const thread = { + id, + ownerSubject, + repository, + title: String(title || "").trim().slice(0, 256), + state: "idle", + headSequence: 0, + activeSessionId: null, + queuedSessionId: null, + leaseGeneration: 0, + createdAt: now, + updatedAt: now, + }; + this.threads.set(id, thread); + this.events.set(id, []); + this.artifacts.set(id, []); + return clone(thread); + } + + listThreadsForActor(actor) { + return [...this.threads.values()] + .filter((thread) => thread.ownerSubject === actor) + .sort((left, right) => right.updatedAt.localeCompare(left.updatedAt)) + .map(clone); + } + + getThreadForActor(threadId, actor) { + return clone(this.#authorizedThread(threadId, actor)); + } + + appendUserMessageAndRoute({ + threadId, + actor, + messageId, + text, + newSessionId, + now = new Date().toISOString(), + leaseTtlMs = 60_000, + }) { + const thread = this.#authorizedThread(threadId, actor); + requiredString(messageId, "message id", 128); + const message = requiredString(text, "message"); + const key = `${threadId}:${actor}:${messageId}`; + const existing = this.idempotency.get(key); + if (existing) return clone(existing); + + let session = thread.activeSessionId ? this.sessions.get(thread.activeSessionId) : null; + if (session && !acceptingSessionStates.has(session.status) && thread.queuedSessionId) { + session = this.sessions.get(thread.queuedSessionId); + } + let createdSession = false; + if (!session || !acceptingSessionStates.has(session.status)) { + requiredString(newSessionId, "new session id", 63); + if (this.sessions.has(newSessionId)) throw conflict("session already exists"); + const activateNow = !thread.activeSessionId; + const generation = activateNow ? thread.leaseGeneration + 1 : null; + session = { + id: newSessionId, + threadId, + ordinal: [...this.sessions.values()].filter((candidate) => candidate.threadId === threadId).length + 1, + actorSubject: actor, + triggerMessageId: messageId, + status: "queued", + leaseGeneration: generation, + leaseExpiresAt: activateNow ? new Date(Date.parse(now) + leaseTtlMs).toISOString() : null, + inboxHeadSequence: 0, + inboxAckSequence: 0, + contextHeadSequence: thread.headSequence, + createdAt: now, + updatedAt: now, + }; + this.sessions.set(session.id, session); + if (activateNow) { + thread.activeSessionId = session.id; + thread.leaseGeneration = generation; + thread.state = "starting"; + } else { + thread.queuedSessionId = session.id; + } + createdSession = true; + } + + const event = this.#appendEvent(thread, { + eventId: messageId, + sessionId: session.id, + actorSubject: actor, + type: "user_message", + payload: { text: message }, + createdAt: now, + }); + session.inboxHeadSequence = event.sequence; + session.updatedAt = now; + thread.updatedAt = now; + const result = { event, session: clone(session), createdSession }; + this.idempotency.set(key, result); + return clone(result); + } + + markSessionRunning({ threadId, sessionId, generation, now = new Date().toISOString() }) { + const { thread, session } = this.#currentSession(threadId, sessionId, generation); + if (!new Set(["queued", "starting"]).has(session.status)) throw conflict("session cannot enter running state"); + session.status = "running"; + session.updatedAt = now; + thread.state = "running"; + thread.updatedAt = now; + return clone(session); + } + + acknowledgeInbox({ threadId, sessionId, generation, throughSequence, now = new Date().toISOString() }) { + const { session } = this.#currentSession(threadId, sessionId, generation); + if (!Number.isSafeInteger(throughSequence) || throughSequence < session.inboxAckSequence || throughSequence > session.inboxHeadSequence) { + throw new Error("invalid inbox acknowledgement sequence"); + } + session.inboxAckSequence = throughSequence; + session.updatedAt = now; + return clone(session); + } + + markSessionFinishing({ threadId, sessionId, generation, now = new Date().toISOString() }) { + const { thread, session } = this.#currentSession(threadId, sessionId, generation); + if (session.inboxAckSequence !== session.inboxHeadSequence) { + return { finishing: false, reason: "pending_input", session: clone(session) }; + } + if (!new Set(["running", "waiting_for_user"]).has(session.status)) throw conflict("session cannot enter finishing state"); + session.status = "finishing"; + session.updatedAt = now; + thread.state = "running"; + thread.updatedAt = now; + return { finishing: true, session: clone(session) }; + } + + appendFencedSessionEvent({ + threadId, + sessionId, + generation, + eventId, + type, + payload, + now = new Date().toISOString(), + }) { + const { thread, session } = this.#currentSession(threadId, sessionId, generation); + requiredString(eventId, "event id", 128); + if (!publicEventTypes.has(type) || type === "user_message") throw new Error("unsupported public session event type"); + const event = this.#appendEvent(thread, { + eventId, + sessionId, + actorSubject: session.actorSubject, + type, + payload: boundedPayload(payload), + createdAt: now, + }); + session.updatedAt = now; + thread.updatedAt = now; + return clone(event); + } + + renewSessionLease({ threadId, sessionId, generation, now = new Date().toISOString(), leaseTtlMs = 60_000 }) { + const { session } = this.#currentSession(threadId, sessionId, generation); + session.leaseExpiresAt = new Date(Date.parse(now) + leaseTtlMs).toISOString(); + session.updatedAt = now; + return clone(session); + } + + finalizeSession({ threadId, sessionId, generation, status = "completed", now = new Date().toISOString() }) { + if (!terminalSessionStates.has(status)) throw new Error("invalid terminal session status"); + const { thread, session } = this.#currentSession(threadId, sessionId, generation); + if (session.status !== "finishing" && status === "completed") throw conflict("completed session must be finishing"); + session.status = status; + session.completedAt = now; + session.updatedAt = now; + let activatedSession = null; + if (thread.queuedSessionId) { + activatedSession = this.sessions.get(thread.queuedSessionId); + activatedSession.leaseGeneration = thread.leaseGeneration + 1; + activatedSession.leaseExpiresAt = new Date(Date.parse(now) + 60_000).toISOString(); + activatedSession.updatedAt = now; + thread.leaseGeneration = activatedSession.leaseGeneration; + thread.activeSessionId = activatedSession.id; + thread.queuedSessionId = null; + thread.state = "starting"; + } else { + thread.activeSessionId = null; + thread.state = status === "completed" ? "idle" : "interrupted"; + } + thread.updatedAt = now; + return clone({ session, activatedSession }); + } + + publishCheckpoint({ threadId, sessionId, generation, throughSequence, content, sourceDigest, now = new Date().toISOString() }) { + const { thread } = this.#currentSession(threadId, sessionId, generation); + if (!Number.isSafeInteger(throughSequence) || throughSequence < 0 || throughSequence > thread.headSequence) { + throw new Error("invalid checkpoint sequence"); + } + const checkpoint = { + threadId, + sessionId, + throughSequence, + content: requiredString(content, "checkpoint", 64 * 1024), + sourceDigest: requiredString(sourceDigest, "source digest", 128), + createdAt: now, + }; + this.checkpoints.set(threadId, checkpoint); + return clone(checkpoint); + } + + registerArtifact({ threadId, sessionId, generation, artifact, now = new Date().toISOString() }) { + this.#currentSession(threadId, sessionId, generation); + const entry = { + artifactId: requiredString(artifact?.artifactId, "artifact id", 128), + threadId, + sourceSessionId: sessionId, + contentDigest: requiredString(artifact?.contentDigest, "artifact digest", 128), + size: Number(artifact?.size), + contentType: requiredString(artifact?.contentType, "artifact content type", 128), + classification: requiredString(artifact?.classification, "artifact classification", 64), + storageReference: requiredString(artifact?.storageReference, "artifact storage reference", 1024), + redactionStatus: requiredString(artifact?.redactionStatus, "artifact redaction status", 64), + createdAt: now, + }; + if (!Number.isSafeInteger(entry.size) || entry.size < 0) throw new Error("invalid artifact size"); + const entries = this.artifacts.get(threadId); + if (entries.some((candidate) => candidate.artifactId === entry.artifactId)) throw conflict("artifact already exists"); + entries.push(entry); + return clone(entry); + } + + readEventsAfter({ threadId, actor, afterSequence = 0, limit = 200 }) { + this.#authorizedThread(threadId, actor); + const boundedLimit = Math.min(Math.max(Number(limit) || 1, 1), 500); + return this.events.get(threadId) + .filter((event) => event.sequence > afterSequence) + .slice(0, boundedLimit) + .map(clone); + } + + listSessionsForActor({ threadId, actor }) { + this.#authorizedThread(threadId, actor); + return [...this.sessions.values()] + .filter((session) => session.threadId === threadId) + .sort((left, right) => left.ordinal - right.ordinal) + .map(clone); + } + + contextEnvelope({ threadId, actor, sessionId, recentLimit = 40 }) { + const thread = this.#authorizedThread(threadId, actor); + const session = this.sessions.get(sessionId); + if (!session || session.threadId !== threadId) throw notFound(); + const checkpoint = this.checkpoints.get(threadId) || null; + const afterSequence = checkpoint?.throughSequence || 0; + const recentEvents = this.events.get(threadId).filter((event) => event.sequence > afterSequence).slice(-recentLimit); + return clone({ + threadId, + sessionId, + throughSequence: thread.headSequence, + checkpoint, + recentEvents, + artifacts: this.artifacts.get(threadId), + }); + } + + #authorizedThread(threadId, actor) { + const thread = this.threads.get(threadId); + if (!thread || thread.ownerSubject !== actor) throw notFound(); + return thread; + } + + #currentSession(threadId, sessionId, generation) { + const thread = this.threads.get(threadId); + const session = this.sessions.get(sessionId); + if (!thread || !session || session.threadId !== threadId || thread.activeSessionId !== sessionId || session.leaseGeneration !== generation) { + throw conflict("stale session fence"); + } + return { thread, session }; + } + + #appendEvent(thread, value) { + const events = this.events.get(thread.id); + if (events.some((event) => event.eventId === value.eventId)) throw conflict("event already exists"); + const event = { + threadId: thread.id, + sequence: thread.headSequence + 1, + eventId: value.eventId, + sessionId: value.sessionId, + actorSubject: value.actorSubject, + type: value.type, + payload: boundedPayload(value.payload), + createdAt: value.createdAt, + }; + thread.headSequence = event.sequence; + events.push(event); + return event; + } +} + +export async function createThreadStore({ backend = process.env.MULTIAGENT_THREAD_STORE_BACKEND || "memory" } = {}) { + if (backend === "memory") return new InMemoryThreadStore(); + if (backend === "dynamodb") { + const { DynamoThreadStore } = await import("./dynamo-thread-store.mjs"); + return new DynamoThreadStore({ + tableName: process.env.MULTIAGENT_THREAD_STORE_TABLE, + region: process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION, + endpoint: process.env.MULTIAGENT_THREAD_STORE_ENDPOINT, + }); + } + throw new Error(`unsupported thread store backend: ${backend}`); +} diff --git a/control-server/test/github-thread-resume.e2e.mjs b/control-server/test/github-thread-resume.e2e.mjs new file mode 100644 index 0000000..16772a1 --- /dev/null +++ b/control-server/test/github-thread-resume.e2e.mjs @@ -0,0 +1,205 @@ +import assert from "node:assert/strict"; + +const baseUrl = new URL(process.env.MULTIAGENT_E2E_URL || "http://127.0.0.1:18080"); +const repository = process.env.MULTIAGENT_E2E_REPOSITORY || "multiagent"; +const githubRepository = process.env.MULTIAGENT_E2E_GITHUB_REPOSITORY || "aptos-labs/aptos-core"; +const timeoutMs = Number(process.env.MULTIAGENT_E2E_TIMEOUT_MS || 45 * 60_000); +const threadId = process.env.MULTIAGENT_E2E_THREAD_ID || `github-thread-e2e-${Date.now().toString(36)}`; + +const [unreviewed, merged] = await Promise.all([ + latestOpenPullRequestWithoutReviews(githubRepository), + mostRecentlyMergedPullRequest(githubRepository), +]); +console.log(`unreviewed oracle: ${unreviewed.html_url} (${unreviewed.title})`); +console.log(`merged oracle: ${merged.html_url} (${merged.title})`); + +let cookie = process.env.MULTIAGENT_E2E_COOKIE || ""; +if (!cookie) { + const login = await request("/api/login", { + method: "POST", + body: { + username: required("MULTIAGENT_E2E_USERNAME"), + password: required("MULTIAGENT_E2E_PASSWORD"), + }, + }); + cookie = login.headers.get("set-cookie")?.split(";", 1)[0] || ""; + assert.ok(cookie, "control server did not issue an authentication cookie"); +} + +await request("/api/threads", { + method: "POST", + cookie, + body: { id: threadId, repository, title: "Aptos core pull-request investigation" }, + expectedStatus: 201, +}); + +const firstPrompt = [ + `Find the latest open pull request in ${githubRepository} that has no submitted pull-request reviews.`, + "Use the GitHub Markdown runbook and access GitHub only through prod-mcp.", + "Return its PR number, title, author, URL, creation timestamp, and explicit evidence that its submitted review list is empty.", + "Do not clone or modify the repository.", +].join(" "); +const first = await submitMessage("message-a", firstPrompt); +assert.equal(first.createdSession, true); +const sessionA = first.session.id; +console.log(`thread: ${threadId}; session A: ${sessionA}`); + +const firstResult = await waitForResult({ + afterSequence: 0, + matches: (text) => unreviewedReportMatches(text, unreviewed), + description: "latest unreviewed pull request", +}); +await waitForSessionEnd(sessionA); + +const second = await submitMessage( + "message-b", + "Now find the most recent pull request that landed in that repository. Return its PR number, title, author, URL, and merge timestamp. Use the same read-only GitHub runbook path.", +); +assert.equal(second.createdSession, true); +const sessionB = second.session.id; +assert.notEqual(sessionB, sessionA, "thread follow-up reused the previous execution session"); +console.log(`session B: ${sessionB}`); + +const secondResult = await waitForResult({ + afterSequence: firstResult.sequence, + matches: (text) => mergedReportMatches(text, merged), + description: "most recently merged pull request", +}); +await waitForSessionEnd(sessionB); + +const history = await request(`/api/threads/${threadId}/events?after_sequence=0&limit=500`, { cookie }); +const events = history.body.events || []; +assert.ok(events.some((event) => event.eventId === "message-a" && event.type === "user_message")); +assert.ok(events.some((event) => event.eventId === "message-b" && event.type === "user_message")); +assert.ok(events.some((event) => event.sessionId === sessionA && unreviewedReportMatches(eventText(event), unreviewed))); +assert.ok(events.some((event) => event.sessionId === sessionB && mergedReportMatches(eventText(event), merged))); +assert.ok(secondResult.sequence > firstResult.sequence); + +console.log("GitHub durable-thread resume E2E passed"); + +async function submitMessage(messageId, text) { + const result = await request(`/api/threads/${threadId}/messages`, { + method: "POST", + cookie, + headers: { "idempotency-key": messageId }, + body: { text }, + expectedStatus: 202, + }); + return result.body; +} + +async function waitForResult({ afterSequence, matches, description }) { + const deadline = Date.now() + timeoutMs; + let cursor = afterSequence; + while (Date.now() < deadline) { + const response = await request(`/api/threads/${threadId}/events?after_sequence=${cursor}&limit=200`, { cookie }); + for (const event of response.body.events || []) { + cursor = Math.max(cursor, event.sequence); + const text = eventText(event); + if (event.type === "assistant_message" && matches(text)) { + console.log(text); + return event; + } + if (event.type === "session_interrupted") throw new Error(`${description} session was interrupted: ${text}`); + } + await sleep(5000); + } + throw new Error(`timed out waiting for ${description}`); +} + +async function waitForSessionEnd(sessionId) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const response = await request(`/api/threads/${threadId}/sessions`, { cookie }); + const session = (response.body.sessions || []).find((candidate) => candidate.id === sessionId); + assert.ok(session, `session disappeared: ${sessionId}`); + if (session.status === "completed") return; + if (["failed", "interrupted", "cancelled"].includes(session.status)) { + throw new Error(`session ${sessionId} ended with status ${session.status}`); + } + await sleep(5000); + } + throw new Error(`timed out waiting for session completion: ${sessionId}`); +} + +function eventText(event) { + return String(event?.payload?.text || event?.payload?.report || ""); +} + +function unreviewedReportMatches(report, pullRequest) { + const number = String(pullRequest.number); + return report.includes(pullRequest.html_url) + && (report.includes(`#${number}`) || report.includes(`PR ${number}`) || report.includes(`pull request ${number}`)) + && /(?:zero|no|empty).{0,80}(?:submitted )?reviews?/is.test(report); +} + +function mergedReportMatches(report, pullRequest) { + const number = String(pullRequest.number); + return report.includes(pullRequest.html_url) + && (report.includes(`#${number}`) || report.includes(`PR ${number}`) || report.includes(`pull request ${number}`)) + && report.includes(pullRequest.merged_at); +} + +async function latestOpenPullRequestWithoutReviews(repositoryName) { + const headers = githubHeaders(); + const pulls = await githubJson(`https://api.github.com/repos/${repositoryName}/pulls?state=open&sort=created&direction=desc&per_page=30`, headers); + for (const pull of pulls) { + const reviews = await githubJson(`https://api.github.com/repos/${repositoryName}/pulls/${pull.number}/reviews?per_page=1`, headers); + if (reviews.length === 0) return pull; + } + throw new Error(`no unreviewed open pull request found in the latest ${pulls.length} PRs`); +} + +async function mostRecentlyMergedPullRequest(repositoryName) { + const pulls = await githubJson( + `https://api.github.com/repos/${repositoryName}/pulls?state=closed&sort=updated&direction=desc&per_page=100`, + githubHeaders(), + ); + const merged = pulls.filter((pull) => pull.merged_at).sort((left, right) => right.merged_at.localeCompare(left.merged_at)); + if (!merged.length) throw new Error("GitHub oracle found no recently merged pull request"); + return merged[0]; +} + +function githubHeaders() { + const headers = { + accept: "application/vnd.github+json", + "user-agent": "multiagent-thread-e2e", + "x-github-api-version": "2022-11-28", + }; + if (process.env.GITHUB_TOKEN) headers.authorization = `Bearer ${process.env.GITHUB_TOKEN}`; + return headers; +} + +async function githubJson(url, headers) { + const response = await fetch(url, { headers }); + const body = await response.json().catch(() => null); + if (!response.ok) throw new Error(`GitHub oracle failed with HTTP ${response.status}: ${JSON.stringify(body)}`); + return body; +} + +async function request(path, options = {}) { + const headers = { accept: "application/json", origin: baseUrl.origin, ...(options.headers || {}) }; + if (options.body) headers["content-type"] = "application/json"; + if (options.cookie) headers.cookie = options.cookie; + const response = await fetch(new URL(path, baseUrl), { + method: options.method || "GET", + headers, + body: options.body ? JSON.stringify(options.body) : undefined, + }); + const body = await response.json().catch(() => ({})); + const expectedStatus = options.expectedStatus || 200; + if (response.status !== expectedStatus) { + throw new Error(`${options.method || "GET"} ${path} returned ${response.status}: ${JSON.stringify(body)}`); + } + return { response, headers: response.headers, body }; +} + +function required(name) { + const value = process.env[name]; + if (!value) throw new Error(`${name} is required`); + return value; +} + +function sleep(milliseconds) { + return new Promise((resolve) => setTimeout(resolve, milliseconds)); +} diff --git a/control-server/test/kubernetes-session.test.mjs b/control-server/test/kubernetes-session.test.mjs index 96d6fe6..ba0449a 100644 --- a/control-server/test/kubernetes-session.test.mjs +++ b/control-server/test/kubernetes-session.test.mjs @@ -12,10 +12,12 @@ test("deployment-owned session templates accept only named bounded substitutions assert.throws(() => renderSessionTemplate("{{session}}", {}), /invalid placeholder/); }); -test("session bootstrap secrets contain task data without placing it in Job arguments", () => { - const secret = sessionSecret("task-1", "multiagent", "summarize general", "caller-123"); +test("session bootstrap secrets bind thread and execution lease without placing task data in Job arguments", () => { + const secret = sessionSecret("task-1", "multiagent", "summarize general", "caller-123", "thread-1", 4); assert.equal(secret.metadata.name, "multiagent-session-task-1"); assert.equal(Buffer.from(secret.data["task.md"], "base64").toString("utf8"), "summarize general"); + assert.equal(Buffer.from(secret.data["thread-id"], "base64").toString("utf8"), "thread-1"); + assert.equal(Buffer.from(secret.data["lease-generation"], "base64").toString("utf8"), "4"); assert.equal(secret.immutable, true); }); diff --git a/control-server/test/thread-store.test.mjs b/control-server/test/thread-store.test.mjs new file mode 100644 index 0000000..fb95ec4 --- /dev/null +++ b/control-server/test/thread-store.test.mjs @@ -0,0 +1,102 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { InMemoryThreadStore } from "../src/thread-store.mjs"; + +const now = "2026-08-27T00:00:00.000Z"; + +function storeWithThread() { + const store = new InMemoryThreadStore(); + store.createThread({ id: "thread-1", ownerSubject: "user-a", repository: "multiagent", title: "Long task", now }); + return store; +} + +test("thread ownership scopes list, history, and direct lookup", () => { + const store = storeWithThread(); + assert.equal(store.listThreadsForActor("user-a").length, 1); + assert.equal(store.listThreadsForActor("user-b").length, 0); + assert.throws(() => store.getThreadForActor("thread-1", "user-b"), (error) => error.statusCode === 404); + assert.throws(() => store.readEventsAfter({ threadId: "thread-1", actor: "user-b" }), (error) => error.statusCode === 404); +}); + +test("messages are idempotent and route across fresh execution sessions", () => { + const store = storeWithThread(); + const first = store.appendUserMessageAndRoute({ + threadId: "thread-1", actor: "user-a", messageId: "message-1", text: "Start", newSessionId: "session-a", now, + }); + assert.equal(first.createdSession, true); + assert.equal(first.session.leaseGeneration, 1); + assert.deepEqual(store.appendUserMessageAndRoute({ + threadId: "thread-1", actor: "user-a", messageId: "message-1", text: "Start", newSessionId: "unused", now, + }), first); + + store.markSessionRunning({ threadId: "thread-1", sessionId: "session-a", generation: 1, now }); + const followUp = store.appendUserMessageAndRoute({ + threadId: "thread-1", actor: "user-a", messageId: "message-2", text: "More detail", newSessionId: "unused", now, + }); + assert.equal(followUp.createdSession, false); + assert.equal(followUp.session.id, "session-a"); + assert.equal(store.markSessionFinishing({ threadId: "thread-1", sessionId: "session-a", generation: 1, now }).reason, "pending_input"); + + store.acknowledgeInbox({ threadId: "thread-1", sessionId: "session-a", generation: 1, throughSequence: 2, now }); + assert.equal(store.markSessionFinishing({ threadId: "thread-1", sessionId: "session-a", generation: 1, now }).finishing, true); + const resumed = store.appendUserMessageAndRoute({ + threadId: "thread-1", actor: "user-a", messageId: "message-3", text: "Resume tomorrow", newSessionId: "session-b", now, + }); + assert.equal(resumed.createdSession, true); + assert.equal(resumed.session.id, "session-b"); + assert.equal(resumed.session.leaseGeneration, null); + const finalized = store.finalizeSession({ threadId: "thread-1", sessionId: "session-a", generation: 1, now }); + assert.equal(finalized.activatedSession.id, "session-b"); + assert.equal(finalized.activatedSession.leaseGeneration, 2); + assert.deepEqual(store.listSessionsForActor({ threadId: "thread-1", actor: "user-a" }).map((session) => session.id), ["session-a", "session-b"]); +}); + +test("fences reject stale session writes without revoking issued permits", () => { + const store = storeWithThread(); + store.appendUserMessageAndRoute({ + threadId: "thread-1", actor: "user-a", messageId: "message-1", text: "Start", newSessionId: "session-a", now, + }); + assert.throws(() => store.appendFencedSessionEvent({ + threadId: "thread-1", sessionId: "session-a", generation: 0, eventId: "event-1", type: "progress", payload: {}, now, + }), /stale session fence/); + const event = store.appendFencedSessionEvent({ + threadId: "thread-1", sessionId: "session-a", generation: 1, eventId: "event-1", type: "progress", payload: { text: "working" }, now, + }); + assert.equal(event.sequence, 2); + // Permit validity is intentionally outside ThreadStore. A lease fence controls + // new thread writes and permit issuance, not an already-issued permit's expiry. +}); + +test("history replay, checkpoints, and artifact manifests remain thread scoped", () => { + const store = storeWithThread(); + store.appendUserMessageAndRoute({ + threadId: "thread-1", actor: "user-a", messageId: "message-1", text: "Start", newSessionId: "session-a", now, + }); + store.acknowledgeInbox({ threadId: "thread-1", sessionId: "session-a", generation: 1, throughSequence: 1, now }); + store.appendFencedSessionEvent({ + threadId: "thread-1", sessionId: "session-a", generation: 1, eventId: "event-2", type: "assistant_message", payload: { text: "Result" }, now, + }); + store.publishCheckpoint({ + threadId: "thread-1", sessionId: "session-a", generation: 1, throughSequence: 1, content: "Known context", sourceDigest: "sha256:context", now, + }); + store.registerArtifact({ + threadId: "thread-1", + sessionId: "session-a", + generation: 1, + artifact: { + artifactId: "artifact-1", + contentDigest: "sha256:artifact", + size: 12, + contentType: "text/plain", + classification: "internal", + storageReference: "opaque:artifact-1", + redactionStatus: "reviewed", + }, + now, + }); + assert.deepEqual(store.readEventsAfter({ threadId: "thread-1", actor: "user-a", afterSequence: 1 }).map((event) => event.sequence), [2]); + const context = store.contextEnvelope({ threadId: "thread-1", actor: "user-a", sessionId: "session-a" }); + assert.equal(context.checkpoint.throughSequence, 1); + assert.equal(context.recentEvents[0].sequence, 2); + assert.equal(context.artifacts[0].artifactId, "artifact-1"); +}); diff --git a/docs/architecture/system-architecture.md b/docs/architecture/system-architecture.md index ccd1cb8..a5b2f5a 100644 --- a/docs/architecture/system-architecture.md +++ b/docs/architecture/system-architecture.md @@ -38,9 +38,9 @@ hard-coded orchestration logic. Website and authenticated user | v -Control server and session gateway +Control server, durable thread store, and session gateway | - | creates or resumes one session + | appends to one durable thread and creates an execution session v Multiagent session runtime +-----------------------------------------------+ @@ -71,7 +71,7 @@ storage configuration shown above. | Component | Owns | Must not own or know | | --- | --- | --- | | Website | User authentication, user intent, session initiation | Runbook implementation, KMS signing, production credentials | -| Control server | Treating the authenticated website caller as the user, session creation/resume, message transport, result streaming | Provider lifecycle logic, Grafana procedures, operation IDs, runbook steps, production credentials | +| Control server | Treating the authenticated website caller as the user, durable thread ownership and history, execution-session creation, message transport, event replay, result streaming | Provider lifecycle logic, Grafana procedures, operation IDs, runbook steps, production credentials | | Supervisor | One session's authority, role bootstrap, role confinement, privileged-request mediation, KMS signing | Service-specific operational procedures | | Orchestrator | Goal decomposition, role routing, workflow coordination | Grafana/Loki knowledge, concrete production operations, `prod-mcp` parameters, provider-specific prompts | | Ops agent | Reading a selected Markdown runbook, planning and requesting its steps, reporting evidence | Deployment secrets, KMS private authority, infrastructure provisioning | @@ -94,17 +94,58 @@ The control server may have high authority because access to it is already restricted to authenticated users. That authority remains attributable to the authenticated user and session. -### AD-002: There is one supervisor per session +### AD-002: There is one supervisor per execution session -Every session has its own supervisor authority and role process tree. A shared -supervisor across unrelated sessions would mix authority, context, failures, -and audit evidence. +Every execution session has its own supervisor authority and role process tree. +A shared supervisor across execution sessions would mix authority, failures, +and audit evidence. A durable user thread may contain multiple sequential +execution sessions, each with a fresh supervisor. The target deployment separates the long-lived control gateway from dedicated session runtimes. A session runtime may be implemented as a Kubernetes Pod or Job. Moving to this target must preserve the existing bootstrap model in which the supervisor creates role processes and confines them after creation. +### AD-014: Threads outlive execution sessions + +A thread is the durable, user-owned task and conversation shown by the website. +An execution session is one isolated runtime instance created to make progress +on that thread. The control server owns thread authorization, an append-only +user-visible event timeline, context checkpoints, artifact references, and the +mapping to sequential execution sessions. + +Only one execution session may hold the active fenced lease for a thread. A +follow-up after a session finishes creates a new session ID, Pod or Job, +supervisor, provider session, writable workspace, reviewer decisions, and +permits. It receives bounded thread context and verified immutable artifact +references, not the previous session's credentials, permits, raw trace, or +writable filesystem. + +User messages are durably and idempotently appended before acknowledgement. +Structured redacted events are the conversation source of truth. WebSocket is +a replay and live-delivery transport with stable event IDs and thread-local +sequence numbers; raw orchestrator stdout is not durable conversation history. + +`multiagent` owns the thread schemas and transactional storage semantics. +`InternalServices` provisions the selected durable backend, IAM, encryption, +endpoints, and retention configuration. + +### AD-015: Session fences do not revoke issued permits + +The active thread lease controls which execution session may append +authoritative thread state and issue new production permits. Losing that lease +prevents new issuance and fenced writes, but it does not revoke a permit that +was validly issued earlier. + +An issued permit remains valid until it is consumed or reaches its encoded +expiry. `prod-mcp` verifies its signature, bearer authentication, attribution, +operation bounds, nonce or operation identity, and expiry without consulting +the current thread lease. A later execution session may issue its own permits, +so short-lived permits from sequential sessions may overlap. Each remains +attributable to its original thread, session, authorizing user event, reviewer +decision, and operation. Replay protection prevents a one-shot permit from +executing the same operation twice. + ### AD-003: Role boundaries are enforced after bootstrap The supervisor creates agents with explicit roles and then applies Linux @@ -262,10 +303,11 @@ roles and dependencies, not provider credentials, model names, or prices. ## End-to-end request flow -1. The website authenticates a user and submits a goal to the control server. -2. The control server records the current authenticated actor and creates or - resumes the user's session. -3. The session supervisor bootstraps the orchestrator and confined role agents. +1. The website authenticates a user and appends a goal or follow-up to a thread. +2. The control server records the actor, durably appends the user event, and + routes it to the active execution session or creates a fresh one. +3. The execution-session supervisor bootstraps the orchestrator and confined + role agents with bounded thread context. 4. The orchestrator delegates production work without encoding the procedure. 5. The ops agent selects and reads the exact versioned Markdown runbook. 6. The ops agent proposes the next operation and supplies runbook evidence. @@ -293,7 +335,7 @@ The desired production topology is: | Workload | Lifetime | Network exposure | Credentials | | --- | --- | --- | --- | | Control server | Long-lived | Reverse proxy or approved private ingress | Website/session authentication only | -| Session runtime | One per session | Private | Model keys as needed, supervisor KMS and `prod-mcp` client authority | +| Session runtime | One per execution session | Private | Model keys as needed, supervisor KMS and `prod-mcp` client authority | | Trace sidecar | Same lifetime as session | S3 egress | Narrow S3 write role | | `prod-mcp` | Long-lived central service | Private service endpoint | Grafana token and narrow cross-account execution roles | @@ -435,8 +477,9 @@ yet be fully implemented: - Split the long-lived control gateway from Kubernetes session runtimes while preserving one supervisor per session. -- Add durable session-to-runtime mapping, workflow state, approval state, - resume behavior, stale runtime cleanup, and child-process reaping. +- Add the deployment-backed durable thread store, execution-session mapping, + fenced leases, structured replay, context hydration, artifact materialization, + stale runtime cleanup, and child-process reaping. - Harden filesystem operations against descriptor-relative path and race attacks where pathname policy is insufficient. - Complete cross-account Route53, ACM validation, load balancer routing, and diff --git a/src/prod_ops.rs b/src/prod_ops.rs index 3787fce..841bedb 100644 --- a/src/prod_ops.rs +++ b/src/prod_ops.rs @@ -735,6 +735,12 @@ fn build_request( "target": target, "taskId": task_id }); + if let Some(execution_context) = execution_context_from_environment()? { + request + .as_object_mut() + .expect("operation request is an object") + .insert("executionContext".into(), execution_context); + } if let Some(change_ticket) = object.get("changeTicket") { request .as_object_mut() @@ -1475,6 +1481,38 @@ fn required_env(name: &str) -> Result { .ok_or_else(|| format!("{name} is required")) } +fn execution_context_from_environment() -> Result, String> { + const NAMES: [&str; 4] = [ + "MULTIAGENT_THREAD_ID", + "MULTIAGENT_SESSION", + "MULTIAGENT_LEASE_GENERATION", + "MULTIAGENT_AUTHORIZING_EVENT_ID", + ]; + let values = NAMES.map(|name| env::var(name).ok().filter(|value| !value.is_empty())); + if values.iter().all(Option::is_none) { + return Ok(None); + } + if values.iter().any(Option::is_none) { + return Err("thread execution attribution requires thread, session, lease generation, and authorizing event together".into()); + } + let [thread_id, session_id, lease_generation, authorizing_event_id] = values.map(Option::unwrap); + validate_id("thread ID", &thread_id)?; + validate_id("session ID", &session_id)?; + validate_id("authorizing event ID", &authorizing_event_id)?; + let lease_generation = lease_generation + .parse::() + .map_err(|_| "MULTIAGENT_LEASE_GENERATION must be a positive integer")?; + if lease_generation == 0 { + return Err("MULTIAGENT_LEASE_GENERATION must be a positive integer".into()); + } + Ok(Some(json!({ + "threadId": thread_id, + "sessionId": session_id, + "leaseGeneration": lease_generation, + "authorizingEventId": authorizing_event_id, + }))) +} + fn canonical(value: &Value) -> Result, String> { serde_json::to_vec(value).map_err(|error| error.to_string()) } @@ -1740,6 +1778,7 @@ mod tests { ); assert_eq!(request["operation"]["version"], "1.1.0"); assert_eq!(request["runbook"]["version"], "1.1.0"); + assert_eq!(request["executionContext"]["threadId"], "thread-contract-1"); assert!(String::from_utf8(canonical(request).unwrap()) .unwrap() .contains("\"authorityProxy\"")); From 18437807d9891e6514663560fdc1e4932b6d1727 Mon Sep 17 00:00:00 2001 From: Bo Wu Date: Wed, 26 Aug 2026 23:07:27 -0700 Subject: [PATCH 2/3] Require execution attribution on permits --- src/prod_ops.rs | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/src/prod_ops.rs b/src/prod_ops.rs index 841bedb..3875d17 100644 --- a/src/prod_ops.rs +++ b/src/prod_ops.rs @@ -566,7 +566,14 @@ fn execute(args: &[String]) -> Result { )?, approved_at: caller_approved_at, }; - let request = build_request(&template, &caller_approval, &reviewer_approval, now)?; + let execution_context = execution_context_from_environment()?; + let request = build_request( + &template, + &caller_approval, + &reviewer_approval, + &execution_context, + now, + )?; let action_id = request["actionId"] .as_str() .ok_or("generated operation has no action ID")? @@ -645,6 +652,7 @@ fn build_request( template: &Value, caller: &TrustedApproval, reviewer: &TrustedApproval, + execution_context: &Value, now: chrono::DateTime, ) -> Result { validate_request_template(template)?; @@ -722,6 +730,7 @@ fn build_request( "transportAuth": "service-token" }, "expiresAt": (now + Duration::minutes(4)).to_rfc3339_opts(SecondsFormat::Millis, true), + "executionContext": execution_context, "historySha256": digest_json(history)?, "intentSha256": digest_json(goal)?, "issuedAt": now.to_rfc3339_opts(SecondsFormat::Millis, true), @@ -735,12 +744,6 @@ fn build_request( "target": target, "taskId": task_id }); - if let Some(execution_context) = execution_context_from_environment()? { - request - .as_object_mut() - .expect("operation request is an object") - .insert("executionContext".into(), execution_context); - } if let Some(change_ticket) = object.get("changeTicket") { request .as_object_mut() @@ -1481,7 +1484,7 @@ fn required_env(name: &str) -> Result { .ok_or_else(|| format!("{name} is required")) } -fn execution_context_from_environment() -> Result, String> { +fn execution_context_from_environment() -> Result { const NAMES: [&str; 4] = [ "MULTIAGENT_THREAD_ID", "MULTIAGENT_SESSION", @@ -1489,13 +1492,11 @@ fn execution_context_from_environment() -> Result, String> { "MULTIAGENT_AUTHORIZING_EVENT_ID", ]; let values = NAMES.map(|name| env::var(name).ok().filter(|value| !value.is_empty())); - if values.iter().all(Option::is_none) { - return Ok(None); - } if values.iter().any(Option::is_none) { return Err("thread execution attribution requires thread, session, lease generation, and authorizing event together".into()); } - let [thread_id, session_id, lease_generation, authorizing_event_id] = values.map(Option::unwrap); + let [thread_id, session_id, lease_generation, authorizing_event_id] = + values.map(Option::unwrap); validate_id("thread ID", &thread_id)?; validate_id("session ID", &session_id)?; validate_id("authorizing event ID", &authorizing_event_id)?; @@ -1505,12 +1506,12 @@ fn execution_context_from_environment() -> Result, String> { if lease_generation == 0 { return Err("MULTIAGENT_LEASE_GENERATION must be a positive integer".into()); } - Ok(Some(json!({ + Ok(json!({ "threadId": thread_id, "sessionId": session_id, "leaseGeneration": lease_generation, "authorizingEventId": authorizing_event_id, - }))) + })) } fn canonical(value: &Value) -> Result, String> { @@ -1705,7 +1706,12 @@ mod tests { "runbookDocument":"runbooks/custom-runbook.md", "runbookContentSha256":format!("sha256:{}", "4".repeat(64)), "changeTicket":"OPS-123" - }), &caller, &reviewer, now).unwrap(); + }), &caller, &reviewer, &json!({ + "threadId": "thread-1", + "sessionId": "session-1", + "leaseGeneration": 1, + "authorizingEventId": "message-1" + }), now).unwrap(); assert_eq!(request["operation"]["id"], "service.custom-operation"); assert_eq!(request["target"]["service"], "api"); assert_eq!(request["parameters"]["custom"], true); From 6c87246a0da83038084cc85533298e534734f6ac Mon Sep 17 00:00:00 2001 From: Bo Wu Date: Wed, 26 Aug 2026 23:29:06 -0700 Subject: [PATCH 3/3] Simplify durable thread state --- control-server/package-lock.json | 462 ------------------ control-server/package.json | 2 - control-server/src/dynamo-thread-store.mjs | 454 ----------------- control-server/src/server.mjs | 13 +- control-server/src/session-runtime.mjs | 12 + control-server/src/thread-store.mjs | 109 ++++- .../test/file-thread-store.test.mjs | 39 ++ control-server/test/session-runtime.test.mjs | 14 + docs/architecture/system-architecture.md | 47 +- 9 files changed, 197 insertions(+), 955 deletions(-) delete mode 100644 control-server/src/dynamo-thread-store.mjs create mode 100644 control-server/test/file-thread-store.test.mjs diff --git a/control-server/package-lock.json b/control-server/package-lock.json index 656467f..c81aa04 100644 --- a/control-server/package-lock.json +++ b/control-server/package-lock.json @@ -8,471 +8,9 @@ "name": "multiagent-control-server", "version": "0.1.0", "dependencies": { - "@aws-sdk/client-dynamodb": "^3.0.0", - "@aws-sdk/lib-dynamodb": "^3.0.0", "ws": "8.21.3" } }, - "node_modules/@aws-sdk/client-dynamodb": { - "version": "3.1119.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/client-dynamodb/-/client-dynamodb-3.1119.0.tgz", - "integrity": "sha512-nfw/t2hhJYQ/85mhdF6WNpSBcLdLVb4R2aCL0XlIKbM+aB4sltUvXLTKaUlqn2gcPOLY1UBT7CcgM2Chfw5FhQ==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.977.9", - "@aws-sdk/credential-provider-node": "^3.972.81", - "@aws-sdk/dynamodb-codec": "^3.973.44", - "@aws-sdk/middleware-endpoint-discovery": "^3.972.30", - "@aws-sdk/types": "^3.974.5", - "@smithy/core": "^3.33.3", - "@smithy/fetch-http-handler": "^5.7.2", - "@smithy/node-http-handler": "^4.11.3", - "@smithy/types": "^4.17.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/core": { - "version": "3.977.9", - "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.977.9.tgz", - "integrity": "sha512-reqPFEQrZxDZpeGj4PFMepBeR5LGYHRqq/L0motTzgFkCRBA4rFdaVXDSLYyGHhxVz7sT2PDnPN9CluGSfgyJA==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/types": "^3.974.5", - "@aws-sdk/xml-builder": "^3.972.40", - "@aws/lambda-invoke-store": "^0.3.0", - "@smithy/core": "^3.33.3", - "@smithy/signature-v4": "^5.6.12", - "@smithy/types": "^4.17.2", - "bowser": "^2.11.0", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-env": { - "version": "3.972.70", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.70.tgz", - "integrity": "sha512-H404B7dJl2mCrBqahDEYsanB0xhdDp6tXnXcTUnXmmpy2Q3J0Ho0bUajZ2jr/RdwzCyS59Gi8xXIFwPLGBl6Uw==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.977.9", - "@aws-sdk/types": "^3.974.5", - "@smithy/core": "^3.33.3", - "@smithy/types": "^4.17.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-http": { - "version": "3.972.72", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.72.tgz", - "integrity": "sha512-X98zYOrVOeuosCX+6ktf29FC2N2GHPLia7qv6mzPzTc+RPAuHWCDS++Z6JK7eGYqb/v6uaW7bAXaOvDBfol+0w==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.977.9", - "@aws-sdk/types": "^3.974.5", - "@smithy/core": "^3.33.3", - "@smithy/fetch-http-handler": "^5.7.2", - "@smithy/node-http-handler": "^4.11.3", - "@smithy/types": "^4.17.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-ini": { - "version": "3.973.15", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.973.15.tgz", - "integrity": "sha512-Rykg6s5ceBuynMOGWgoowO4N+27JfnqXAnVaSunZl0hOO1XodSrxGNz6sCEbnmS0lAfQZDKyb3fbr46gSuv6Sg==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.977.9", - "@aws-sdk/credential-provider-env": "^3.972.70", - "@aws-sdk/credential-provider-http": "^3.972.72", - "@aws-sdk/credential-provider-login": "^3.972.77", - "@aws-sdk/credential-provider-process": "^3.972.70", - "@aws-sdk/credential-provider-sso": "^3.973.14", - "@aws-sdk/credential-provider-web-identity": "^3.972.76", - "@aws-sdk/nested-clients": "^3.997.44", - "@aws-sdk/types": "^3.974.5", - "@smithy/core": "^3.33.3", - "@smithy/credential-provider-imds": "^4.4.16", - "@smithy/types": "^4.17.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-login": { - "version": "3.972.77", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.77.tgz", - "integrity": "sha512-Jb59xfEISoN5mmbnA+HYqdtrSX3CgCtJoof+V5D8/TgUI56W63GEEd5Y58WijU3Ou6+WEgaLD1feVzaRXV5IDQ==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.977.9", - "@aws-sdk/nested-clients": "^3.997.44", - "@aws-sdk/types": "^3.974.5", - "@smithy/core": "^3.33.3", - "@smithy/types": "^4.17.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-node": { - "version": "3.972.81", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.81.tgz", - "integrity": "sha512-Rml+WitoFvXmv6JZ18U/xGdGDGGvB/mOin0ya0lTnTrdC0Z1lrVxTYh7iNklZBcvcRMrs4DoEf6xy1KWyrLQQw==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/credential-provider-env": "^3.972.70", - "@aws-sdk/credential-provider-http": "^3.972.72", - "@aws-sdk/credential-provider-ini": "^3.973.15", - "@aws-sdk/credential-provider-process": "^3.972.70", - "@aws-sdk/credential-provider-sso": "^3.973.14", - "@aws-sdk/credential-provider-web-identity": "^3.972.76", - "@aws-sdk/types": "^3.974.5", - "@smithy/core": "^3.33.3", - "@smithy/credential-provider-imds": "^4.4.16", - "@smithy/types": "^4.17.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-process": { - "version": "3.972.70", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.70.tgz", - "integrity": "sha512-2ry03fGRJr4sV3jI+ocjj5JqALnFD6ymM5KiNCDZMvq8bX2GSbE0vji4aM43TVCl2nXqqLRZaUxdq/KeWRAY4Q==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.977.9", - "@aws-sdk/types": "^3.974.5", - "@smithy/core": "^3.33.3", - "@smithy/types": "^4.17.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-sso": { - "version": "3.973.14", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.973.14.tgz", - "integrity": "sha512-jkhg/8ocAAoc0RFyLMhCw+/zZh7gystQgd4F4hznNa8P4Cc501PQmxd+jGLiMHodPJ+7Zv/3znM62gZojyasmA==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.977.9", - "@aws-sdk/nested-clients": "^3.997.44", - "@aws-sdk/token-providers": "3.1116.0", - "@aws-sdk/types": "^3.974.5", - "@smithy/core": "^3.33.3", - "@smithy/types": "^4.17.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/credential-provider-web-identity": { - "version": "3.972.76", - "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.76.tgz", - "integrity": "sha512-d3AGyVu759PGr35mEB2s22xxlNEA5rpdxtSPJthfPFJvoQ8dt357iVPECqWfUxXp1toJAvKmbtcIYVGigaGsCA==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.977.9", - "@aws-sdk/nested-clients": "^3.997.44", - "@aws-sdk/types": "^3.974.5", - "@smithy/core": "^3.33.3", - "@smithy/types": "^4.17.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/dynamodb-codec": { - "version": "3.973.44", - "resolved": "https://registry.npmjs.org/@aws-sdk/dynamodb-codec/-/dynamodb-codec-3.973.44.tgz", - "integrity": "sha512-eo27HNeBqMAfy9JBQug6ErU6DeG8OmOH6ou8+KmX3/fWdTmWUsHsfisz2tEQOG+GJ57bBC7kC6AjKAjpcxU4QA==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.977.9", - "@smithy/core": "^3.33.3", - "@smithy/types": "^4.17.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/endpoint-cache": { - "version": "3.972.11", - "resolved": "https://registry.npmjs.org/@aws-sdk/endpoint-cache/-/endpoint-cache-3.972.11.tgz", - "integrity": "sha512-8q1ICxcDjHId3bBryuu/j+1L9y5/3uQnwzLDt5j2ElcjZSoWmFtymdJy7OjLrluSMe0Z4mq5bcH4fxBXvlEHfw==", - "license": "Apache-2.0", - "dependencies": { - "mnemonist": "0.38.3", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/lib-dynamodb": { - "version": "3.1119.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/lib-dynamodb/-/lib-dynamodb-3.1119.0.tgz", - "integrity": "sha512-EWN8njF2k0Bmiw6OOpoNGCFe6GvCMoGNnwdS0Pz2irVVw2CVwdJGkETUcqxDEGwpaDzx/zUVZPUWD6pz5jAObQ==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.977.9", - "@aws-sdk/util-dynamodb": "^3.996.9", - "@smithy/core": "^3.33.3", - "@smithy/types": "^4.17.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-dynamodb": "^3.1119.0" - } - }, - "node_modules/@aws-sdk/middleware-endpoint-discovery": { - "version": "3.972.30", - "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-endpoint-discovery/-/middleware-endpoint-discovery-3.972.30.tgz", - "integrity": "sha512-0hmD7/NG2NoVOVHvUb6rtY5POQYr+/9gdHvrYhIBA1zmCqKOBd5Gz3dL+iZ15FHOJbs/5kLplGoePc8PHTlzYA==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/endpoint-cache": "^3.972.11", - "@aws-sdk/types": "^3.974.5", - "@smithy/core": "^3.33.3", - "@smithy/types": "^4.17.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/nested-clients": { - "version": "3.997.44", - "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.44.tgz", - "integrity": "sha512-NhEgryjlBF9w38ZXqGymQV28IhkYa1mKhlbYnqIis57AYwWGVYfUPgg/qC2rLRqOUfblxx++irvju10kVTa8Vw==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.977.9", - "@aws-sdk/signature-v4-multi-region": "^3.996.46", - "@aws-sdk/types": "^3.974.5", - "@smithy/core": "^3.33.3", - "@smithy/fetch-http-handler": "^5.7.2", - "@smithy/node-http-handler": "^4.11.3", - "@smithy/types": "^4.17.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/signature-v4-multi-region": { - "version": "3.996.46", - "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.46.tgz", - "integrity": "sha512-L+2xZTye/2T96f3lwCws0Zw6GG2JHZW9e8FpVgGBeeExSKyeoZ6CWRpBml/7DNiK/O26jrgPM9F+Ay8VkgzUWQ==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/types": "^3.974.5", - "@smithy/signature-v4": "^5.6.12", - "@smithy/types": "^4.17.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/token-providers": { - "version": "3.1116.0", - "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1116.0.tgz", - "integrity": "sha512-ygIivKqh8aHzNkucOCXHyIBgBpLPfrSI0mCqXF+vLBsPTUKqj0VSqAY0GFPe7lQl4HntjOcQ+KSyS7oUV2C54Q==", - "license": "Apache-2.0", - "dependencies": { - "@aws-sdk/core": "^3.977.9", - "@aws-sdk/nested-clients": "^3.997.44", - "@aws-sdk/types": "^3.974.5", - "@smithy/core": "^3.33.3", - "@smithy/types": "^4.17.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/types": { - "version": "3.974.5", - "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.974.5.tgz", - "integrity": "sha512-LkwLL2BLbC6wNNm4JaH9mbEqBMdOZCct6VAYqhdN4U1xrWM+fUJQEfbHwQgDypapOWTRtlk25akb5afM0P8CIQ==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.17.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws-sdk/util-dynamodb": { - "version": "3.996.9", - "resolved": "https://registry.npmjs.org/@aws-sdk/util-dynamodb/-/util-dynamodb-3.996.9.tgz", - "integrity": "sha512-16x2tRvl7OYpZ0W/DdFJieFriD13+RvuRBDbe5sj/tCEfK86HSGd7I2s5j0ivz8p6KWGkS+5wKRO9OliJkjUOQ==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - }, - "peerDependencies": { - "@aws-sdk/client-dynamodb": "^3.1111.0" - } - }, - "node_modules/@aws-sdk/xml-builder": { - "version": "3.972.40", - "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.40.tgz", - "integrity": "sha512-wlFmCIGUlwF4zx/kncw+bmxTQh1HeSJq4mYV/V5cZUSJadDP3kXvGW8Rn21cimj/7y9ju+47oYWXi97vF7czaA==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.17.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=20.0.0" - } - }, - "node_modules/@aws/lambda-invoke-store": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.3.0.tgz", - "integrity": "sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==", - "license": "Apache-2.0", - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/core": { - "version": "3.33.3", - "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.33.3.tgz", - "integrity": "sha512-CsOeKq/9kA3y6VJHt+/+VTCtBaxJ4OTFpgrjIUhPpDIKxBci1k2bJaQASF2h/ELWrulGp+t97DZ0mevfAD8idg==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/types": "^4.17.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/credential-provider-imds": { - "version": "4.5.2", - "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.5.2.tgz", - "integrity": "sha512-A9uSdn72ozbRUSit0eib0TW7nXuNPlaeM0zcGkJ+nE6tFcSDbnmtwoxbTCFBukVQcszDAyvsd7+rTduPTXpygg==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/core": "^3.33.2", - "@smithy/types": "^4.17.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/fetch-http-handler": { - "version": "5.7.2", - "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.7.2.tgz", - "integrity": "sha512-nZyWTmSpJEXl6VtWVMBJve/7x12DZu6sIX1z1a+ZMaHlQQRs9Zpu6NbTe/gmxYXVRpkjxyDYpZ5gx2IM6f/Wkw==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/core": "^3.33.2", - "@smithy/types": "^4.17.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/node-http-handler": { - "version": "4.11.3", - "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.11.3.tgz", - "integrity": "sha512-2jY1tSpERfPfWqyBV2pH+iGFaghVsIJszJNsT7hxtQYhVJpWDyc0LqOWI+nXOxOAHaEfZ4PXXtp1wW1TGpHhkA==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/core": "^3.33.3", - "@smithy/types": "^4.17.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/signature-v4": { - "version": "5.7.3", - "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.7.3.tgz", - "integrity": "sha512-7ImGm+FkHRLcBaRttIAMZ6bzJZWb2cJGoYjq46F2UjycujWzrL9GEN9h4w7eQyXJYnltrUhxbbieBAIRrdqpow==", - "license": "Apache-2.0", - "dependencies": { - "@smithy/core": "^3.33.3", - "@smithy/types": "^4.17.2", - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/@smithy/types": { - "version": "4.17.2", - "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.17.2.tgz", - "integrity": "sha512-FOKpVZob9MPTn2znRzGrnsMHv7BOsKVw3XiP/cOyYLDVZ9qKp4nifIiSCuUU/fIj5Vu0UOAxCFr+qRAtG0NUkA==", - "license": "Apache-2.0", - "dependencies": { - "tslib": "^2.6.2" - }, - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/bowser": { - "version": "2.14.1", - "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz", - "integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==", - "license": "MIT" - }, - "node_modules/mnemonist": { - "version": "0.38.3", - "resolved": "https://registry.npmjs.org/mnemonist/-/mnemonist-0.38.3.tgz", - "integrity": "sha512-2K9QYubXx/NAjv4VLq1d1Ly8pWNC5L3BrixtdkyTegXWJIqY+zLNDhhX/A+ZwWt70tB1S8H4BE8FLYEFyNoOBw==", - "license": "MIT", - "dependencies": { - "obliterator": "^1.6.1" - } - }, - "node_modules/obliterator": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/obliterator/-/obliterator-1.6.1.tgz", - "integrity": "sha512-9WXswnqINnnhOG/5SLimUlzuU1hFJUc8zkwyD59Sd+dPOMf05PmnYG/d6Q7HZ+KmgkZJa1PxRso6QdM3sTNHig==", - "license": "MIT" - }, - "node_modules/tslib": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", - "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "license": "0BSD" - }, "node_modules/ws": { "version": "8.21.3", "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.3.tgz", diff --git a/control-server/package.json b/control-server/package.json index 36f5751..3810813 100644 --- a/control-server/package.json +++ b/control-server/package.json @@ -10,8 +10,6 @@ "test:e2e:thread": "node test/github-thread-resume.e2e.mjs" }, "dependencies": { - "@aws-sdk/client-dynamodb": "^3.0.0", - "@aws-sdk/lib-dynamodb": "^3.0.0", "ws": "8.21.3" } } diff --git a/control-server/src/dynamo-thread-store.mjs b/control-server/src/dynamo-thread-store.mjs deleted file mode 100644 index 1776c93..0000000 --- a/control-server/src/dynamo-thread-store.mjs +++ /dev/null @@ -1,454 +0,0 @@ -import { - DeleteCommand, - DynamoDBDocumentClient, - GetCommand, - PutCommand, - QueryCommand, - TransactWriteCommand, - UpdateCommand, -} from "@aws-sdk/lib-dynamodb"; -import { DynamoDBClient } from "@aws-sdk/client-dynamodb"; - -const acceptingSessionStates = new Set(["queued", "starting", "running", "waiting_for_user"]); -const terminalSessionStates = new Set(["completed", "failed", "interrupted", "cancelled"]); -const publicEventTypes = new Set([ - "assistant_message", "progress", "question", "artifact_available", - "session_started", "session_completed", "session_interrupted", -]); - -const pk = (threadId) => `THREAD#${threadId}`; -const eventSk = (sequence) => `EVENT#${String(sequence).padStart(20, "0")}`; -const sessionSk = (sessionId) => `SESSION#${sessionId}`; -const idempotencySk = (actor, messageId) => `IDEMPOTENCY#${actor}#${messageId}`; -const clone = (value) => structuredClone(value); - -function requiredString(value, name, max = 32768) { - if (typeof value !== "string" || !value.trim() || value.length > max) throw new Error(`${name} must contain 1 to ${max} characters`); - return value.trim(); -} - -function boundedPayload(payload) { - const value = payload === undefined ? {} : clone(payload); - if (Buffer.byteLength(JSON.stringify(value), "utf8") > 64 * 1024) throw new Error("event payload exceeds 64 KiB"); - return value; -} - -function notFound() { - const error = new Error("thread not found"); - error.statusCode = 404; - return error; -} - -function conflict(message) { - const error = new Error(message); - error.statusCode = 409; - return error; -} - -function publicThread(item) { - if (!item) return null; - const { pk: ignoredPk, sk: ignoredSk, gsi1pk, gsi1sk, version, ...thread } = item; - return thread; -} - -function publicSession(item) { - if (!item) return null; - const { pk: ignoredPk, sk: ignoredSk, ...session } = item; - return session; -} - -function publicEvent(item) { - if (!item) return null; - const { pk: ignoredPk, sk: ignoredSk, ...event } = item; - return event; -} - -function isConditionalFailure(error) { - return error?.name === "TransactionCanceledException" || error?.name === "ConditionalCheckFailedException"; -} - -export class DynamoThreadStore { - constructor({ tableName, region, endpoint, client } = {}) { - this.tableName = requiredString(tableName, "thread store table name", 255); - this.client = client || DynamoDBDocumentClient.from(new DynamoDBClient({ region, ...(endpoint ? { endpoint } : {}) }), { - marshallOptions: { removeUndefinedValues: true }, - }); - } - - async createThread({ id, ownerSubject, repository, title = "", now = new Date().toISOString() }) { - requiredString(id, "thread id", 63); - requiredString(ownerSubject, "owner subject", 256); - requiredString(repository, "repository", 128); - const item = { - pk: pk(id), sk: "META", id, ownerSubject, repository, - title: String(title || "").trim().slice(0, 256), - state: "idle", headSequence: 0, activeSessionId: null, queuedSessionId: null, - leaseGeneration: 0, version: 1, createdAt: now, updatedAt: now, - gsi1pk: `OWNER#${ownerSubject}`, gsi1sk: `${now}#${id}`, - }; - try { - await this.client.send(new PutCommand({ TableName: this.tableName, Item: item, ConditionExpression: "attribute_not_exists(pk)" })); - } catch (error) { - if (isConditionalFailure(error)) throw conflict("thread already exists"); - throw error; - } - return publicThread(item); - } - - async listThreadsForActor(actor) { - const result = await this.client.send(new QueryCommand({ - TableName: this.tableName, - IndexName: "owner-updated-index", - KeyConditionExpression: "gsi1pk = :owner", - ExpressionAttributeValues: { ":owner": `OWNER#${actor}` }, - ScanIndexForward: false, - })); - return (result.Items || []).map(publicThread); - } - - async getThreadForActor(threadId, actor) { - const thread = await this.#thread(threadId); - if (!thread || thread.ownerSubject !== actor) throw notFound(); - return publicThread(thread); - } - - async appendUserMessageAndRoute(input) { - requiredString(input.messageId, "message id", 128); - const text = requiredString(input.text, "message"); - const now = input.now || new Date().toISOString(); - const leaseTtlMs = input.leaseTtlMs || 60_000; - for (let attempt = 0; attempt < 8; attempt += 1) { - const existing = await this.#get(input.threadId, idempotencySk(input.actor, input.messageId)); - if (existing?.result) return clone(existing.result); - const thread = await this.#thread(input.threadId); - if (!thread || thread.ownerSubject !== input.actor) throw notFound(); - let session = thread.activeSessionId ? await this.#session(input.threadId, thread.activeSessionId) : null; - if (session && !acceptingSessionStates.has(session.status) && thread.queuedSessionId) { - session = await this.#session(input.threadId, thread.queuedSessionId); - } - let createdSession = false; - let activateNow = false; - if (!session || !acceptingSessionStates.has(session.status)) { - requiredString(input.newSessionId, "new session id", 63); - if (await this.#session(input.threadId, input.newSessionId)) throw conflict("session already exists"); - activateNow = !thread.activeSessionId; - session = { - id: input.newSessionId, - threadId: input.threadId, - ordinal: Number(thread.sessionCount || 0) + 1, - actorSubject: input.actor, - triggerMessageId: input.messageId, - status: "queued", - leaseGeneration: activateNow ? thread.leaseGeneration + 1 : null, - leaseExpiresAt: activateNow ? new Date(Date.parse(now) + leaseTtlMs).toISOString() : null, - inboxHeadSequence: 0, - inboxAckSequence: 0, - contextHeadSequence: thread.headSequence, - createdAt: now, - updatedAt: now, - }; - createdSession = true; - } - const sequence = thread.headSequence + 1; - const event = { - threadId: input.threadId, sequence, eventId: input.messageId, sessionId: session.id, - actorSubject: input.actor, type: "user_message", payload: { text }, createdAt: now, - }; - session.inboxHeadSequence = sequence; - session.updatedAt = now; - const result = { event, session: publicSession(session), createdSession }; - const names = { "#version": "version", "#head": "headSequence", "#updated": "updatedAt", "#gsi": "gsi1sk" }; - const values = { - ":version": thread.version, ":nextVersion": thread.version + 1, ":head": sequence, - ":updated": now, ":gsi": `${now}#${thread.id}`, - }; - let update = "SET #version = :nextVersion, #head = :head, #updated = :updated, #gsi = :gsi"; - if (createdSession) { - names["#count"] = "sessionCount"; - values[":count"] = session.ordinal; - update += ", #count = :count"; - if (activateNow) { - Object.assign(names, { "#active": "activeSessionId", "#lease": "leaseGeneration", "#state": "state" }); - Object.assign(values, { ":active": session.id, ":lease": session.leaseGeneration, ":state": "starting" }); - update += ", #active = :active, #lease = :lease, #state = :state"; - } else { - names["#queued"] = "queuedSessionId"; - values[":queued"] = session.id; - update += ", #queued = :queued"; - } - } - const sessionWrite = createdSession - ? { Put: { TableName: this.tableName, Item: { pk: pk(input.threadId), sk: sessionSk(session.id), ...session }, ConditionExpression: "attribute_not_exists(sk)" } } - : { Update: { - TableName: this.tableName, - Key: { pk: pk(input.threadId), sk: sessionSk(session.id) }, - UpdateExpression: "SET inboxHeadSequence = :head, updatedAt = :updated", - ExpressionAttributeValues: { ":head": sequence, ":updated": now }, - } }; - try { - await this.client.send(new TransactWriteCommand({ TransactItems: [ - { Update: { - TableName: this.tableName, Key: { pk: pk(input.threadId), sk: "META" }, - ConditionExpression: "#version = :version", UpdateExpression: update, - ExpressionAttributeNames: names, ExpressionAttributeValues: values, - } }, - { Put: { TableName: this.tableName, Item: { pk: pk(input.threadId), sk: eventSk(sequence), ...event }, ConditionExpression: "attribute_not_exists(sk)" } }, - { Put: { TableName: this.tableName, Item: { - pk: pk(input.threadId), sk: idempotencySk(input.actor, input.messageId), result, - }, ConditionExpression: "attribute_not_exists(sk)" } }, - sessionWrite, - ] })); - return clone(result); - } catch (error) { - if (!isConditionalFailure(error)) throw error; - } - } - throw conflict("thread was modified concurrently; retry the message"); - } - - async markSessionRunning({ threadId, sessionId, generation, now = new Date().toISOString() }) { - const { thread, session } = await this.#currentSession(threadId, sessionId, generation); - if (!["queued", "starting"].includes(session.status)) throw conflict("session cannot enter running state"); - await this.client.send(new TransactWriteCommand({ TransactItems: [ - { Update: { - TableName: this.tableName, Key: { pk: pk(threadId), sk: "META" }, - ConditionExpression: "activeSessionId = :session AND leaseGeneration = :generation", - UpdateExpression: "SET #state = :state, updatedAt = :now", - ExpressionAttributeNames: { "#state": "state" }, - ExpressionAttributeValues: { ":session": sessionId, ":generation": generation, ":state": "running", ":now": now }, - } }, - { Update: { - TableName: this.tableName, Key: { pk: pk(threadId), sk: sessionSk(sessionId) }, - ConditionExpression: "leaseGeneration = :generation", - UpdateExpression: "SET #status = :status, updatedAt = :now", - ExpressionAttributeNames: { "#status": "status" }, - ExpressionAttributeValues: { ":generation": generation, ":status": "running", ":now": now }, - } }, - ] })); - return { ...publicSession(session), status: "running", updatedAt: now }; - } - - async acknowledgeInbox({ threadId, sessionId, generation, throughSequence, now = new Date().toISOString() }) { - const { session } = await this.#currentSession(threadId, sessionId, generation); - if (!Number.isSafeInteger(throughSequence) || throughSequence < session.inboxAckSequence || throughSequence > session.inboxHeadSequence) { - throw new Error("invalid inbox acknowledgement sequence"); - } - await this.client.send(new UpdateCommand({ - TableName: this.tableName, Key: { pk: pk(threadId), sk: sessionSk(sessionId) }, - ConditionExpression: "leaseGeneration = :generation", - UpdateExpression: "SET inboxAckSequence = :through, updatedAt = :now", - ExpressionAttributeValues: { ":generation": generation, ":through": throughSequence, ":now": now }, - })); - return { ...publicSession(session), inboxAckSequence: throughSequence, updatedAt: now }; - } - - async markSessionFinishing({ threadId, sessionId, generation, now = new Date().toISOString() }) { - const { session } = await this.#currentSession(threadId, sessionId, generation); - if (session.inboxAckSequence !== session.inboxHeadSequence) return { finishing: false, reason: "pending_input", session: publicSession(session) }; - if (!["running", "waiting_for_user"].includes(session.status)) throw conflict("session cannot enter finishing state"); - await this.client.send(new UpdateCommand({ - TableName: this.tableName, Key: { pk: pk(threadId), sk: sessionSk(sessionId) }, - ConditionExpression: "leaseGeneration = :generation AND inboxAckSequence = inboxHeadSequence", - UpdateExpression: "SET #status = :status, updatedAt = :now", - ExpressionAttributeNames: { "#status": "status" }, - ExpressionAttributeValues: { ":generation": generation, ":status": "finishing", ":now": now }, - })); - return { finishing: true, session: { ...publicSession(session), status: "finishing", updatedAt: now } }; - } - - async appendFencedSessionEvent({ threadId, sessionId, generation, eventId, type, payload, now = new Date().toISOString() }) { - requiredString(eventId, "event id", 128); - if (!publicEventTypes.has(type)) throw new Error("unsupported public session event type"); - const value = boundedPayload(payload); - for (let attempt = 0; attempt < 8; attempt += 1) { - const { thread, session } = await this.#currentSession(threadId, sessionId, generation); - const sequence = thread.headSequence + 1; - const event = { threadId, sequence, eventId, sessionId, actorSubject: session.actorSubject, type, payload: value, createdAt: now }; - try { - await this.client.send(new TransactWriteCommand({ TransactItems: [ - { Update: { - TableName: this.tableName, Key: { pk: pk(threadId), sk: "META" }, - ConditionExpression: "#version = :version AND activeSessionId = :session AND leaseGeneration = :generation", - UpdateExpression: "SET #version = :nextVersion, headSequence = :head, updatedAt = :now, gsi1sk = :gsi", - ExpressionAttributeNames: { "#version": "version" }, - ExpressionAttributeValues: { - ":version": thread.version, ":nextVersion": thread.version + 1, ":session": sessionId, - ":generation": generation, ":head": sequence, ":now": now, ":gsi": `${now}#${threadId}`, - }, - } }, - { Put: { TableName: this.tableName, Item: { pk: pk(threadId), sk: eventSk(sequence), ...event }, ConditionExpression: "attribute_not_exists(sk)" } }, - ] })); - return event; - } catch (error) { - if (!isConditionalFailure(error)) throw error; - } - } - throw conflict("thread was modified concurrently; retry the event"); - } - - async renewSessionLease({ threadId, sessionId, generation, now = new Date().toISOString(), leaseTtlMs = 60_000 }) { - const expiresAt = new Date(Date.parse(now) + leaseTtlMs).toISOString(); - try { - const result = await this.client.send(new UpdateCommand({ - TableName: this.tableName, Key: { pk: pk(threadId), sk: sessionSk(sessionId) }, - ConditionExpression: "leaseGeneration = :generation", - UpdateExpression: "SET leaseExpiresAt = :expires, updatedAt = :now", - ExpressionAttributeValues: { ":generation": generation, ":expires": expiresAt, ":now": now }, - ReturnValues: "ALL_NEW", - })); - return publicSession(result.Attributes); - } catch (error) { - if (isConditionalFailure(error)) throw conflict("stale session fence"); - throw error; - } - } - - async finalizeSession({ threadId, sessionId, generation, status = "completed", now = new Date().toISOString() }) { - if (!terminalSessionStates.has(status)) throw new Error("invalid terminal session status"); - const { thread, session } = await this.#currentSession(threadId, sessionId, generation); - if (status === "completed" && session.status !== "finishing") throw conflict("completed session must be finishing"); - const queued = thread.queuedSessionId ? await this.#session(threadId, thread.queuedSessionId) : null; - const nextGeneration = queued ? thread.leaseGeneration + 1 : null; - const metaNames = { "#state": "state", "#active": "activeSessionId", "#queued": "queuedSessionId", "#lease": "leaseGeneration" }; - const metaValues = { - ":session": sessionId, ":generation": generation, ":now": now, - ":state": queued ? "starting" : status === "completed" ? "idle" : "interrupted", - ":active": queued?.id || null, ":queued": null, ":lease": nextGeneration || generation, - }; - const writes = [ - { Update: { - TableName: this.tableName, Key: { pk: pk(threadId), sk: "META" }, - ConditionExpression: "activeSessionId = :session AND leaseGeneration = :generation", - UpdateExpression: "SET #state = :state, #active = :active, #queued = :queued, #lease = :lease, updatedAt = :now", - ExpressionAttributeNames: metaNames, ExpressionAttributeValues: metaValues, - } }, - { Update: { - TableName: this.tableName, Key: { pk: pk(threadId), sk: sessionSk(sessionId) }, - ConditionExpression: "leaseGeneration = :generation", - UpdateExpression: "SET #status = :status, completedAt = :now, updatedAt = :now", - ExpressionAttributeNames: { "#status": "status" }, - ExpressionAttributeValues: { ":generation": generation, ":status": status, ":now": now }, - } }, - ]; - let activatedSession = null; - if (queued) { - const expiresAt = new Date(Date.parse(now) + 60_000).toISOString(); - writes.push({ Update: { - TableName: this.tableName, Key: { pk: pk(threadId), sk: sessionSk(queued.id) }, - ConditionExpression: "attribute_not_exists(leaseGeneration) OR leaseGeneration = :none", - UpdateExpression: "SET leaseGeneration = :lease, leaseExpiresAt = :expires, updatedAt = :now", - ExpressionAttributeValues: { ":none": null, ":lease": nextGeneration, ":expires": expiresAt, ":now": now }, - } }); - activatedSession = { ...publicSession(queued), leaseGeneration: nextGeneration, leaseExpiresAt: expiresAt, updatedAt: now }; - } - try { - await this.client.send(new TransactWriteCommand({ TransactItems: writes })); - } catch (error) { - if (isConditionalFailure(error)) throw conflict("stale session fence"); - throw error; - } - return { session: { ...publicSession(session), status, completedAt: now, updatedAt: now }, activatedSession }; - } - - async publishCheckpoint({ threadId, sessionId, generation, throughSequence, content, sourceDigest, now = new Date().toISOString() }) { - const { thread } = await this.#currentSession(threadId, sessionId, generation); - if (!Number.isSafeInteger(throughSequence) || throughSequence < 0 || throughSequence > thread.headSequence) throw new Error("invalid checkpoint sequence"); - const checkpoint = { - threadId, sessionId, throughSequence, content: requiredString(content, "checkpoint", 64 * 1024), - sourceDigest: requiredString(sourceDigest, "source digest", 128), createdAt: now, - }; - await this.client.send(new PutCommand({ TableName: this.tableName, Item: { pk: pk(threadId), sk: "CHECKPOINT#LATEST", ...checkpoint } })); - return checkpoint; - } - - async registerArtifact({ threadId, sessionId, generation, artifact, now = new Date().toISOString() }) { - await this.#currentSession(threadId, sessionId, generation); - const entry = { - artifactId: requiredString(artifact?.artifactId, "artifact id", 128), threadId, sourceSessionId: sessionId, - contentDigest: requiredString(artifact?.contentDigest, "artifact digest", 128), size: Number(artifact?.size), - contentType: requiredString(artifact?.contentType, "artifact content type", 128), - classification: requiredString(artifact?.classification, "artifact classification", 64), - storageReference: requiredString(artifact?.storageReference, "artifact storage reference", 1024), - redactionStatus: requiredString(artifact?.redactionStatus, "artifact redaction status", 64), createdAt: now, - }; - if (!Number.isSafeInteger(entry.size) || entry.size < 0) throw new Error("invalid artifact size"); - try { - await this.client.send(new PutCommand({ - TableName: this.tableName, Item: { pk: pk(threadId), sk: `ARTIFACT#${entry.artifactId}`, ...entry }, - ConditionExpression: "attribute_not_exists(sk)", - })); - } catch (error) { - if (isConditionalFailure(error)) throw conflict("artifact already exists"); - throw error; - } - return entry; - } - - async readEventsAfter({ threadId, actor, afterSequence = 0, limit = 200 }) { - await this.getThreadForActor(threadId, actor); - const boundedLimit = Math.min(Math.max(Number(limit) || 1, 1), 500); - const result = await this.client.send(new QueryCommand({ - TableName: this.tableName, - KeyConditionExpression: "pk = :pk AND sk BETWEEN :after AND :end", - ExpressionAttributeValues: { ":pk": pk(threadId), ":after": eventSk(Number(afterSequence) + 1), ":end": "EVENT#~" }, - Limit: boundedLimit, - })); - return (result.Items || []).map(publicEvent); - } - - async listSessionsForActor({ threadId, actor }) { - await this.getThreadForActor(threadId, actor); - const result = await this.client.send(new QueryCommand({ - TableName: this.tableName, - KeyConditionExpression: "pk = :pk AND begins_with(sk, :prefix)", - ExpressionAttributeValues: { ":pk": pk(threadId), ":prefix": "SESSION#" }, - })); - return (result.Items || []).map(publicSession).sort((left, right) => left.ordinal - right.ordinal); - } - - async contextEnvelope({ threadId, actor, sessionId, recentLimit = 40 }) { - const thread = await this.getThreadForActor(threadId, actor); - const session = await this.#session(threadId, sessionId); - if (!session) throw notFound(); - const checkpoint = await this.#get(threadId, "CHECKPOINT#LATEST"); - const afterSequence = checkpoint?.throughSequence || 0; - const events = await this.client.send(new QueryCommand({ - TableName: this.tableName, - KeyConditionExpression: "pk = :pk AND sk BETWEEN :after AND :end", - ExpressionAttributeValues: { ":pk": pk(threadId), ":after": eventSk(afterSequence + 1), ":end": "EVENT#~" }, - ScanIndexForward: false, - Limit: recentLimit, - })); - const artifacts = await this.client.send(new QueryCommand({ - TableName: this.tableName, - KeyConditionExpression: "pk = :pk AND begins_with(sk, :prefix)", - ExpressionAttributeValues: { ":pk": pk(threadId), ":prefix": "ARTIFACT#" }, - })); - return { - threadId, sessionId, throughSequence: thread.headSequence, - checkpoint: checkpoint ? publicEvent(checkpoint) : null, - recentEvents: (events.Items || []).reverse().map(publicEvent), - artifacts: (artifacts.Items || []).map(publicEvent), - }; - } - - async deleteThreadForTests(threadId) { - const result = await this.client.send(new QueryCommand({ - TableName: this.tableName, KeyConditionExpression: "pk = :pk", ExpressionAttributeValues: { ":pk": pk(threadId) }, - })); - await Promise.all((result.Items || []).map((item) => this.client.send(new DeleteCommand({ TableName: this.tableName, Key: { pk: item.pk, sk: item.sk } })))); - } - - async #get(threadId, sk) { - return (await this.client.send(new GetCommand({ TableName: this.tableName, Key: { pk: pk(threadId), sk }, ConsistentRead: true }))).Item || null; - } - - #thread(threadId) { return this.#get(threadId, "META"); } - #session(threadId, sessionId) { return this.#get(threadId, sessionSk(sessionId)); } - - async #currentSession(threadId, sessionId, generation) { - const [thread, session] = await Promise.all([this.#thread(threadId), this.#session(threadId, sessionId)]); - if (!thread || !session || thread.activeSessionId !== sessionId || session.leaseGeneration !== generation) throw conflict("stale session fence"); - return { thread, session }; - } -} diff --git a/control-server/src/server.mjs b/control-server/src/server.mjs index bd4689b..6c52da8 100644 --- a/control-server/src/server.mjs +++ b/control-server/src/server.mjs @@ -12,6 +12,7 @@ import { controlMode, findActiveSession, normalizeWorkerReport, + scopedThreadTranscript, selectFinalMessage, sessionControlInvocation, sessionLaunchInvocation, @@ -46,7 +47,8 @@ const repositoryCatalog = gatewayMode ? JSON.parse(process.env.MULTIAGENT_REPOSI const kubernetes = gatewayMode ? new KubernetesSessionClient() : null; const sessionJobTemplate = gatewayMode ? JSON.parse(fs.readFileSync(sessionJobTemplateFile, "utf8")) : null; const threadStore = await createThreadStore({ - backend: process.env.MULTIAGENT_THREAD_STORE_BACKEND || (gatewayMode ? "dynamodb" : "memory"), + backend: process.env.MULTIAGENT_THREAD_STORE_BACKEND || (gatewayMode ? "file" : "memory"), + filePath: process.env.MULTIAGENT_THREAD_STORE_FILE || path.join(stateRoot, "control-server", "thread-manifest-v1.json"), }); fs.mkdirSync(path.dirname(registryFile), { recursive: true }); @@ -500,6 +502,10 @@ function renderThreadTask(envelope) { if (!text) continue; const role = event.type === "user_message" ? "User" : event.type === "assistant_message" ? "Assistant" : "Status"; lines.push(`${role}: ${text}`, ""); + const references = event.payload?.transcript?.traceReferences; + if (Array.isArray(references) && references.length) { + lines.push("Prior session trace references:", ...references.slice(0, 16).map((reference) => `- ${String(reference)}`), ""); + } } return lines.join("\n").slice(-32768); } @@ -556,7 +562,7 @@ async function projectGatewaySessionToThread(id, status) { generation: record.leaseGeneration, eventId: `final-${id}`, type: "assistant_message", - payload: { text: report.report, transcript: report.transcript }, + payload: { text: report.report, transcript: scopedThreadTranscript(id, report.transcript) }, }); await threadStore.acknowledgeInbox({ threadId: record.threadId, @@ -1003,7 +1009,8 @@ if (workerMode) { if (!validResourceId(id) || !validResourceId(repository) || !taskFile) throw new Error("session-worker mode requires a valid session, repository, and task file"); const threadId = String(process.env.MULTIAGENT_THREAD_ID || id); const leaseGeneration = Number(process.env.MULTIAGENT_LEASE_GENERATION || "1"); - if (!registry.sessions[id]) launchSession(id, repository, resume, actor, fs.readFileSync(taskFile, "utf8"), { threadId, leaseGeneration }); + const authorizingEventId = String(process.env.MULTIAGENT_AUTHORIZING_EVENT_ID || id); + if (!registry.sessions[id]) launchSession(id, repository, resume, actor, fs.readFileSync(taskFile, "utf8"), { threadId, leaseGeneration, authorizingEventId }); } for (const record of gatewayMode ? [] : Object.values(registry.sessions)) { diff --git a/control-server/src/session-runtime.mjs b/control-server/src/session-runtime.mjs index f241fb2..55be531 100644 --- a/control-server/src/session-runtime.mjs +++ b/control-server/src/session-runtime.mjs @@ -47,3 +47,15 @@ export function normalizeWorkerReport(value) { if (Buffer.byteLength(JSON.stringify(transcript), "utf8") > 64 * 1024) return null; return { report: value.report, transcript }; } + +export function scopedThreadTranscript(sessionId, transcript) { + if (!transcript || typeof transcript !== "object") return null; + const traceReferences = Array.isArray(transcript.traceReferences) + ? transcript.traceReferences.map((reference) => { + const normalized = path.posix.normalize(path.posix.join("logs", String(reference))); + if (path.posix.isAbsolute(normalized) || normalized === ".." || normalized.startsWith("../")) return null; + return `trace://session/${sessionId}/${normalized}`; + }).filter(Boolean) + : []; + return { ...transcript, traceReferences }; +} diff --git a/control-server/src/thread-store.mjs b/control-server/src/thread-store.mjs index 7b58c5d..187bd3c 100644 --- a/control-server/src/thread-store.mjs +++ b/control-server/src/thread-store.mjs @@ -1,3 +1,6 @@ +import fs from "node:fs/promises"; +import path from "node:path"; + const acceptingSessionStates = new Set(["queued", "starting", "running", "waiting_for_user"]); const terminalSessionStates = new Set(["completed", "failed", "interrupted", "cancelled"]); const publicEventTypes = new Set([ @@ -41,13 +44,35 @@ function conflict(message) { } export class InMemoryThreadStore { - constructor() { - this.threads = new Map(); - this.sessions = new Map(); - this.events = new Map(); - this.idempotency = new Map(); - this.checkpoints = new Map(); - this.artifacts = new Map(); + constructor(snapshot = null) { + this.restoreSnapshot(snapshot); + } + + restoreSnapshot(snapshot = null) { + if (snapshot && snapshot.schemaVersion !== 1) throw new Error("unsupported thread manifest schema"); + const entries = (name) => { + const value = snapshot?.[name] || []; + if (!Array.isArray(value)) throw new Error(`thread manifest ${name} must be an array`); + return value; + }; + this.threads = new Map(entries("threads")); + this.sessions = new Map(entries("sessions")); + this.events = new Map(entries("events")); + this.idempotency = new Map(entries("idempotency")); + this.checkpoints = new Map(entries("checkpoints")); + this.artifacts = new Map(entries("artifacts")); + } + + snapshot() { + return clone({ + schemaVersion: 1, + threads: [...this.threads.entries()], + sessions: [...this.sessions.entries()], + events: [...this.events.entries()], + idempotency: [...this.idempotency.entries()], + checkpoints: [...this.checkpoints.entries()], + artifacts: [...this.artifacts.entries()], + }); } createThread({ id, ownerSubject, repository, title = "", now = new Date().toISOString() }) { @@ -349,15 +374,67 @@ export class InMemoryThreadStore { } } -export async function createThreadStore({ backend = process.env.MULTIAGENT_THREAD_STORE_BACKEND || "memory" } = {}) { - if (backend === "memory") return new InMemoryThreadStore(); - if (backend === "dynamodb") { - const { DynamoThreadStore } = await import("./dynamo-thread-store.mjs"); - return new DynamoThreadStore({ - tableName: process.env.MULTIAGENT_THREAD_STORE_TABLE, - region: process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION, - endpoint: process.env.MULTIAGENT_THREAD_STORE_ENDPOINT, - }); +const mutatingMethods = new Set([ + "createThread", + "appendUserMessageAndRoute", + "markSessionRunning", + "acknowledgeInbox", + "markSessionFinishing", + "appendFencedSessionEvent", + "renewSessionLease", + "finalizeSession", + "publishCheckpoint", + "registerArtifact", +]); + +async function fileThreadStore(filePath) { + if (!filePath) throw new Error("file thread store requires a path"); + let snapshot = null; + try { + snapshot = JSON.parse(await fs.readFile(filePath, "utf8")); + } catch (error) { + if (error?.code !== "ENOENT") throw error; } + const store = new InMemoryThreadStore(snapshot); + let operations = Promise.resolve(); + const persist = async (nextSnapshot) => { + const encoded = JSON.stringify(nextSnapshot) + "\n"; + await fs.mkdir(path.dirname(filePath), { recursive: true, mode: 0o700 }); + const temporary = `${filePath}.tmp-${process.pid}`; + await fs.writeFile(temporary, encoded, { mode: 0o600 }); + await fs.rename(temporary, filePath); + }; + return new Proxy(store, { + get(target, property) { + const value = Reflect.get(target, property, target); + if (typeof value !== "function") return value; + if (!mutatingMethods.has(property)) { + return async (...args) => { + await operations; + return value.apply(target, args); + }; + } + return async (...args) => { + const operation = operations.then(async () => { + const candidate = new InMemoryThreadStore(target.snapshot()); + const result = value.apply(candidate, args); + const nextSnapshot = candidate.snapshot(); + await persist(nextSnapshot); + target.restoreSnapshot(nextSnapshot); + return result; + }); + operations = operation.then(() => undefined, () => undefined); + return operation; + }; + }, + }); +} + +export async function createThreadStore({ + backend = process.env.MULTIAGENT_THREAD_STORE_BACKEND || "memory", + filePath = process.env.MULTIAGENT_THREAD_STORE_FILE, +} = {}) { + if (backend === "memory") return new InMemoryThreadStore(); + if (backend === "file") return fileThreadStore(filePath); throw new Error(`unsupported thread store backend: ${backend}`); } diff --git a/control-server/test/file-thread-store.test.mjs b/control-server/test/file-thread-store.test.mjs new file mode 100644 index 0000000..5112928 --- /dev/null +++ b/control-server/test/file-thread-store.test.mjs @@ -0,0 +1,39 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile, stat, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { createThreadStore } from "../src/thread-store.mjs"; + +test("file thread manifests survive gateway restart without duplicating messages", async () => { + const directory = await mkdtemp(path.join(os.tmpdir(), "multiagent-thread-store-")); + const filePath = path.join(directory, "thread-manifest-v1.json"); + const first = await createThreadStore({ backend: "file", filePath }); + await first.createThread({ id: "thread-1", ownerSubject: "user-1", repository: "multiagent" }); + const routed = await first.appendUserMessageAndRoute({ + threadId: "thread-1", + actor: "user-1", + messageId: "message-1", + text: "Find the latest pull request", + newSessionId: "session-1", + }); + + const restored = await createThreadStore({ backend: "file", filePath }); + assert.equal((await restored.getThreadForActor("thread-1", "user-1")).activeSessionId, "session-1"); + assert.deepEqual(await restored.appendUserMessageAndRoute({ + threadId: "thread-1", + actor: "user-1", + messageId: "message-1", + text: "Find the latest pull request", + newSessionId: "unused-session", + }), routed); + assert.equal((await stat(filePath)).mode & 0o777, 0o600); +}); + +test("file thread manifests fail closed when their schema is corrupt", async () => { + const directory = await mkdtemp(path.join(os.tmpdir(), "multiagent-thread-store-corrupt-")); + const filePath = path.join(directory, "thread-manifest-v1.json"); + await writeFile(filePath, JSON.stringify({ schemaVersion: 99 }), "utf8"); + await assert.rejects(createThreadStore({ backend: "file", filePath }), /unsupported thread manifest schema/); + assert.match(await readFile(filePath, "utf8"), /99/); +}); diff --git a/control-server/test/session-runtime.test.mjs b/control-server/test/session-runtime.test.mjs index ab90c73..b3b9e25 100644 --- a/control-server/test/session-runtime.test.mjs +++ b/control-server/test/session-runtime.test.mjs @@ -6,6 +6,7 @@ import { controlMode, findActiveSession, normalizeWorkerReport, + scopedThreadTranscript, selectFinalMessage, sessionControlInvocation, sessionLaunchInvocation, @@ -60,6 +61,19 @@ test("completed session reports prefer the explicit bounded caller result", () = assert.equal(normalizeWorkerReport({ report: "x".repeat(64 * 1024 + 1) }), null); }); +test("thread transcript references remain bound to their originating session", () => { + assert.deepEqual(scopedThreadTranscript("session-a", { + taskId: "session-a", + traceReferences: ["agents/orchestrator/attempt-0001/events.jsonl", "../workflows/run-1/lifecycle/events.log", "../../../escape"], + }), { + taskId: "session-a", + traceReferences: [ + "trace://session/session-a/logs/agents/orchestrator/attempt-0001/events.jsonl", + "trace://session/session-a/workflows/run-1/lifecycle/events.log", + ], + }); +}); + test("session completion grace is bounded and has a stable default", () => { assert.equal(completionExitDelayMs(), 30_000); assert.equal(completionExitDelayMs("1"), 10_000); diff --git a/docs/architecture/system-architecture.md b/docs/architecture/system-architecture.md index a5b2f5a..2c0b3ba 100644 --- a/docs/architecture/system-architecture.md +++ b/docs/architecture/system-architecture.md @@ -71,7 +71,7 @@ storage configuration shown above. | Component | Owns | Must not own or know | | --- | --- | --- | | Website | User authentication, user intent, session initiation | Runbook implementation, KMS signing, production credentials | -| Control server | Treating the authenticated website caller as the user, durable thread ownership and history, execution-session creation, message transport, event replay, result streaming | Provider lifecycle logic, Grafana procedures, operation IDs, runbook steps, production credentials | +| Control server | Treating the authenticated website caller as the user, durable thread ownership and public history, execution-session creation, message transport, event replay, trace-derived context, result streaming | Provider lifecycle logic, agent/model turn storage, Grafana procedures, operation IDs, runbook steps, production credentials | | Supervisor | One session's authority, role bootstrap, role confinement, privileged-request mediation, KMS signing | Service-specific operational procedures | | Orchestrator | Goal decomposition, role routing, workflow coordination | Grafana/Loki knowledge, concrete production operations, `prod-mcp` parameters, provider-specific prompts | | Ops agent | Reading a selected Markdown runbook, planning and requesting its steps, reporting evidence | Deployment secrets, KMS private authority, infrastructure provisioning | @@ -110,25 +110,35 @@ the supervisor creates role processes and confines them after creation. A thread is the durable, user-owned task and conversation shown by the website. An execution session is one isolated runtime instance created to make progress -on that thread. The control server owns thread authorization, an append-only -user-visible event timeline, context checkpoints, artifact references, and the -mapping to sequential execution sessions. +on that thread. The control server owns thread authorization, a small append-only +user-visible manifest, context checkpoints, S3 trace references, and the mapping +to sequential execution sessions. Detailed model and agent histories remain in +the session traces already exported to S3; the control server does not duplicate +or reinterpret provider-native conversation storage. Only one execution session may hold the active fenced lease for a thread. A follow-up after a session finishes creates a new session ID, Pod or Job, -supervisor, provider session, writable workspace, reviewer decisions, and -permits. It receives bounded thread context and verified immutable artifact -references, not the previous session's credentials, permits, raw trace, or -writable filesystem. +supervisor, orchestrator, role agents, provider sessions, writable workspace, +reviewer decisions, and permits. No prior agent is revived. The new orchestrator +receives bounded context derived from public messages, final reports, +checkpoints, and verified S3 trace references, not the previous session's +credentials, permits, unbounded raw trace, provider home, or writable filesystem. User messages are durably and idempotently appended before acknowledgement. -Structured redacted events are the conversation source of truth. WebSocket is +The public manifest is the website conversation source of truth, while S3 +session traces are the detailed audit and context-recovery source. WebSocket is a replay and live-delivery transport with stable event IDs and thread-local -sequence numbers; raw orchestrator stdout is not durable conversation history. - -`multiagent` owns the thread schemas and transactional storage semantics. -`InternalServices` provisions the selected durable backend, IAM, encryption, -endpoints, and retention configuration. +sequence numbers; raw orchestrator stdout is never streamed into a new model +context without bounded deterministic projection. + +`multiagent` owns the thread manifest and single-writer lifecycle semantics. +`InternalServices` provisions the gateway PVC, versioned S3 backup, IAM, +encryption, endpoints, and retention configuration. With one gateway writer, +atomic local manifest replacement is sufficient; a distributed database is +required only if the gateway is later scaled to multiple writers. Automatic S3 +restore is forbidden while session Pods can write the same prefix; recovery is +operator-gated until the gateway has a distinct read identity or manifests have +an independently verified integrity signature. ### AD-015: Session fences do not revoke issued permits @@ -334,7 +344,7 @@ The desired production topology is: | Workload | Lifetime | Network exposure | Credentials | | --- | --- | --- | --- | -| Control server | Long-lived | Reverse proxy or approved private ingress | Website/session authentication only | +| Control server | Long-lived, one writer | Reverse proxy or approved private ingress | Website/session authentication only | | Session runtime | One per execution session | Private | Model keys as needed, supervisor KMS and `prod-mcp` client authority | | Trace sidecar | Same lifetime as session | S3 egress | Narrow S3 write role | | `prod-mcp` | Long-lived central service | Private service endpoint | Grafana token and narrow cross-account execution roles | @@ -477,9 +487,10 @@ yet be fully implemented: - Split the long-lived control gateway from Kubernetes session runtimes while preserving one supervisor per session. -- Add the deployment-backed durable thread store, execution-session mapping, - fenced leases, structured replay, context hydration, artifact materialization, - stale runtime cleanup, and child-process reaping. +- Add stale runtime cleanup, artifact materialization, and child-process reaping + around the file-backed thread manifest and existing S3 trace lifecycle. +- Separate gateway and session S3 identities or add independently verified + manifest integrity before enabling automatic S3 bootstrap. - Harden filesystem operations against descriptor-relative path and race attacks where pathname policy is insufficient. - Complete cross-account Route53, ACM validation, load balancer routing, and