From 29008ac8e189107eed805c8abba8f413395bd52d Mon Sep 17 00:00:00 2001 From: Julia Ortiz Date: Thu, 20 Aug 2026 13:11:31 -0300 Subject: [PATCH] fix: harden runtime correctness boundaries --- .changeset/clean-effect-contracts.md | 2 +- .changeset/reviewed-runtime-correctness.md | 23 ++ .gitignore | 2 + CLAUDE.md | 12 +- README.md | 35 +- biome.json | 3 +- docs/api-reference.md | 11 +- docs/storage-protocol-v1.md | 9 +- src/EngineRecord.ts | 88 +++-- src/PublicContracts.test.ts | 18 +- src/RetrySchedule.ts | 6 +- src/Schemas.test.ts | 56 +++ src/StorageProtocol.test.ts | 77 +++++ src/StorageProtocol.ts | 140 +++++++- src/TaskEngine.replies.test.ts | 37 ++ src/TaskEngine.test.ts | 156 ++++++++- src/TaskEngine.ts | 211 ++++++++++- src/TaskEvent.ts | 13 +- src/TaskEvents.test.ts | 43 +++ src/TaskQueue.test.ts | 384 ++++++++++++++++++++- src/TaskQueue.ts | 308 ++++++++++++++--- src/Worker.test.ts | 57 +++ src/Worker.ts | 155 ++++++++- src/lua/taskEngine.lua | 165 ++++++--- src/lua/taskEngine.ts | 2 +- 25 files changed, 1795 insertions(+), 218 deletions(-) create mode 100644 .changeset/reviewed-runtime-correctness.md diff --git a/.changeset/clean-effect-contracts.md b/.changeset/clean-effect-contracts.md index c965d88..be00e26 100644 --- a/.changeset/clean-effect-contracts.md +++ b/.changeset/clean-effect-contracts.md @@ -1,5 +1,5 @@ --- -"@effectmq/core": major +"@effectmq/core": patch --- Make public Effect contracts honest and restructure the package around focused diff --git a/.changeset/reviewed-runtime-correctness.md b/.changeset/reviewed-runtime-correctness.md new file mode 100644 index 0000000..061eeb7 --- /dev/null +++ b/.changeset/reviewed-runtime-correctness.md @@ -0,0 +1,23 @@ +--- +"@effectmq/core": patch +--- + +Fix queue correctness and boundary validation found during a repository-wide +review: + +- allow `Schema.Void` tasks to persist and recover successful completion; +- keep `wait` and `execute` subscribed across retryable failures; +- reject queue/task descriptors that do not match a persisted handle; +- validate numeric offer, lease, and retry-timestamp inputs before mutating + Redis; +- validate decoded storage values, preserve prototype-sensitive object keys, + and reject corrupt Redis numbers and cursors in typed error channels; +- schema-validate built-in failure events and keep their public type precise; +- keep stalled-attempt history out of handler retry schedules, settle attempts + when retry-policy evaluation fails, stop safely when retained history is too + short to replay, and retain terminal failures even when error history is + disabled; +- preserve retry-policy interruption for lease recovery and avoid full + wait-list scans on creation, acquisition, and non-waiting transitions; and +- reject unsafe worker concurrency, timing, and lease supervision options + before acquiring work or starting fibers. diff --git a/.gitignore b/.gitignore index 5578d63..1dc24fa 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ node_modules/ dist/ .next/ out/ +.vercel/ +.env.local *.tsbuildinfo .history/ llms/ diff --git a/CLAUDE.md b/CLAUDE.md index 8249c0b..2e520d2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -14,7 +14,8 @@ pnpm build # tsc -> dist/ (tests and src/testing are excl pnpm typecheck # strict production + complete test typecheck pnpm typecheck:src # production sources only pnpm typecheck:test # all tests and src/testing support -pnpm exec biome check src/ # lint (what CI runs) +pnpm lint # Biome lint across the repository +pnpm check # formatting, lint, types, architecture, Lua, docs, release pnpm lint:fix # biome check --write --unsafe pnpm test # vitest run (integration tests, needs Docker — see below) pnpm test:watch # vitest watch mode @@ -22,8 +23,6 @@ pnpm vitest run src/TaskQueue.test.ts # single test file pnpm vitest run -t "test name substring" # single test by name ``` -Note: the `pnpm lint` script runs `turbo run lint`, but turbo is not a dependency — it's stale. Use `pnpm exec biome check src/` (matches CI). - ### Tests need Docker Effectful tests use `@effect/vitest`. Redis integration suites install the suite-scoped `TestLayer` from `src/testing/redisLayer.ts`; it acquires and releases containers, clients, child processes, listeners, and temporary directories through Effect scopes. Never add a module-level `ManagedRuntime`, direct `Effect.runPromise` test runner, or top-level resource warming. Container acquisition has an explicit 60s hook timeout and normal tests have a 30s timeout (`vitest.config.ts`). The root `docker-compose.yml` Redis is for manual/local experimentation only — tests don't use it. @@ -41,7 +40,7 @@ Flat `src/` with a strict layering, top to bottom: - **`TaskQueue.ts`** — the high-level API users live in: `make` (bind a queue name to a task definition), `offer` (enqueue), `complete` (take → run handler → report outcome, applying the task's retry schedule on failure), `stream` (typed lifecycle events), `wait` / `execute` (await a task's terminal result). Decodes engine tasks/events against the task's schemas. - **`Scheduler.ts`** — cron-driven durable task materialization. Competing processes idempotently offer the same tick task; managed workers execute it with at-least-once delivery. - **`Task.ts`** — `Task.make`: the schema-bearing task definition (payload/success/error schemas, `idempotencyKey` — which *is* the task id, so same key = same task — `retry` as an Effect `Schedule`, `maxRetries` default 5, `null` = unbounded). -- **`TaskEngine.ts`** (the big one, ~1000 lines) — the low-level `Context.Service` implementing queue primitives as **atomic Lua scripts** (inline `/*lua*/` strings, built by `buildScripts`): create/take/writeSuccess/writeError, lock extend/remove, delayed + cron schedule state. Tasks move between Redis lists: `wait`, `scheduled`, `active`, `failed`, `success`. Every state change publishes to a per-queue Redis Stream (`::events`, via `XADD`); `stream` polls it with `XREAD` (default 1s). Consumers rarely call the engine directly — go through `TaskQueue`/`Scheduler`. +- **`TaskEngine.ts`** (the big one, ~1000 lines) — the low-level `Context.Service` implementing queue primitives through the atomic script generated from `src/lua/taskEngine.lua`: create/take/writeSuccess/writeError, lock extend/remove, delayed + cron schedule state. Tasks move between Redis indexes named `wait`, `scheduled`, `active`, `failed`, and `success`. Every state change publishes to a per-queue Redis Stream (`::events`, via `XADD`); `stream` uses a blocking `XREAD` with a default two-second poll interval. Consumers rarely call the engine directly — go through `TaskQueue`/`Scheduler`. - **`RedisPool.ts`** — the minimal service the engine depends on: just `send` + `eval`. Any Redis client can implement it. - **`NodeRedisPool.ts`** — the bundled `RedisPool` implementation using node-redis `createClientPool` (lazy connect, closed on layer scope end). Tests provide `RedisPool` via ioredis + testcontainers instead (`src/testing/redisLayer.ts`) — proof the service boundary works. - **`TaskRecord.ts`** — public typed task record schemas and storage codecs. @@ -50,7 +49,10 @@ Flat `src/` with a strict layering, top to bottom: Wiring: `TaskEngine.layer()` is the complete zero-requirement Node live graph and intentionally retains Redis operational services. `TaskEngine.layerNoDeps()` is the custom-client layer that requires `RedisPool`. -The library deliberately has **no built-in concurrency/rate limiting** — one `complete` processes one task, and callers compose concurrency from Effect primitives (fibers, semaphores, schedules). Don't add worker-pool machinery. +`Worker` provides built-in bounded local concurrency, lease supervision, +maintenance, and graceful draining. It does not provide distributed/global +concurrency or rate limiting; compose those policies explicitly at the +application boundary. ## Conventions diff --git a/README.md b/README.md index ce0749e..0670c00 100644 --- a/README.md +++ b/README.md @@ -85,8 +85,8 @@ A task is a *schema*, not a function. You declare what goes in (`payload`), what A tagged error makes failures pattern-matchable downstream, so reach for `Schema.TaggedError` rather than a bare struct. ```ts docs-check=email -import { Effect, Schedule, Schema, Semaphore, Stream } from "effect"; -import { Task, TaskQueue, type TaskHandler } from "@effectmq/core"; +import { Effect, Schedule, Schema, Stream } from "effect"; +import { Task, TaskQueue, type TaskHandler, Worker } from "@effectmq/core"; class EmailRejected extends Schema.TaggedError()( "EmailRejected", @@ -158,7 +158,7 @@ const repeatedWorkerProgram = Effect.gen(function* () { ## Streaming & events -The engine publishes a lifecycle event to a per-queue Redis Stream every time a task changes state. `TaskQueue.stream` hands you those events as an Effect `Stream`, decoded against your queue's schemas: `task.created` and `task.updated` carry fully-typed tasks, `task.failed` carries your typed error, `task.completed` carries your typed success value, and `task.moved` reports the list transition. +The engine publishes a lifecycle event to a per-queue Redis Stream every time a task changes state. `TaskQueue.stream` hands you those events as an Effect `Stream`, decoded against your queue's schemas: `task.created` and `task.updated` carry fully-typed tasks, `task.failed` carries your typed error or a built-in stalled/canceled error, `task.completed` carries your typed success value, and `task.moved` reports the list transition. ```ts docs-check=email const watch = TaskQueue.stream(emails).pipe( @@ -185,33 +185,26 @@ const offerAndWait = Effect.gen(function* () { }); ``` -`wait` reads durable state, subscribes from the handle's authoritative Redis cursor, and rechecks state after subscription, so completion before or during subscription is observed. Streams poll Redis (default every second); persist a retained cursor when building a resumable event consumer. +`wait` reads durable state, subscribes from the handle's authoritative Redis cursor, and rechecks state after subscription, so completion before or during subscription is observed. Streams use a blocking Redis read (default two-second block); persist a retained cursor when building a resumable event consumer. --- ## On concurrency -Differently than other queue libraries, `effectmq` doesn't have builtin concurrency, rate limiting, backpressure. It doesn't need to, it works perfectly with the Effect primitives you are used to. - -Effect gives you the fine control you need from your workers, so `complete` does exactly one task, and *you* decide how many run at once, with the same tools you use everywhere else: +`complete` processes exactly one task. For a long-running process, `Worker` +provides bounded local concurrency, lease supervision, maintenance, and graceful +draining: ```ts docs-check=email -// Concurrency example with semaphore -const worker = Effect.gen(function* () { - // At most 5 tasks in flight at any moment. - const semaphore = yield* Semaphore.make(5); - - yield* Semaphore.withPermit( - semaphore, - TaskQueue.complete(emails, handleSendEmail), - ).pipe( - Effect.forkScoped, // each worker is its own fiber - Effect.repeat(Schedule.forever), // ...that keeps pulling work - ); -}); +const worker = Worker.make(emails, handleSendEmail, { concurrency: 5 }); +const program = Worker.run(worker); ``` -Want a rate limit instead of a raw permit count? Compose one from a `Semaphore` and a `Schedule`. Want retries with jitter? `Schedule`. Want to fan out? Run more worker processes. None of it is our invention, all of it composes. The queue's job is to preserve eligible work and fence the current attempt; handlers remain at-least-once and must make external side effects idempotent. +The built-in worker does not impose distributed/global concurrency or rate +limits. Compose those policies from Effect primitives or external coordination, +and run more worker processes to fan out. The queue preserves eligible work and +fences the current attempt; handlers remain at-least-once and must make external +side effects idempotent. --- diff --git a/biome.json b/biome.json index de2c9ad..6f96445 100644 --- a/biome.json +++ b/biome.json @@ -10,7 +10,8 @@ "includes": [ "**", "!node_modules", - "!.next", + "!**/.next", + "!**/.vercel", "!dist", "!build", "!src/scratchpad", diff --git a/docs/api-reference.md b/docs/api-reference.md index 18e2c9f..8b12134 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -25,20 +25,23 @@ page describes the intended entry points and their contracts. - `execute(queue, payload, options?)` is offer followed by handle-based wait. - `completeOne(queue, handler, processing?)` acquires and supervises at most one attempt, returning whether work was processed. -- `complete(queue, handler, processing?)` processes one task and returns its id. +- `complete(queue, handler)` processes one task and returns its id. - `stream(queue, options?)` decodes versioned lifecycle events from a cursor. Important offer options include `taskId`, `delay`, `maxRetries`, completion policies, `onDuplicate`, and -`retainResultUntil: "current-task-settles"`. Processing options configure lease -duration, heartbeat interval, and bounded heartbeat transport retry. +`retainResultUntil: "current-task-settles"`. Numeric overrides are validated +before Redis is mutated. Processing options configure lease duration, +heartbeat interval, and bounded heartbeat transport retry. ## `Worker` - `make(queue, handler, options?)` describes a managed worker. - `run(worker)` runs scoped acquisition slots plus maintenance until interrupted. Options include concurrency, poll/maintenance intervals, drain - timeout, and processing supervision. + timeout, and processing supervision. Invalid concurrency, durations, or + processing settings fail with `WorkerConfigurationError` before any fibers + start. ## `Scheduler` diff --git a/docs/storage-protocol-v1.md b/docs/storage-protocol-v1.md index 7e9d5a2..988da41 100644 --- a/docs/storage-protocol-v1.md +++ b/docs/storage-protocol-v1.md @@ -43,13 +43,16 @@ reading their values, and fails when the old namespace has not been drained. After a task's Effect Schema has encoded it, v1 accepts: - `null`, strings, booleans, and `Uint8Array` -- finite numbers whose magnitude is at most `Number.MAX_SAFE_INTEGER` +- finite numbers whose magnitude is at most `Number.MAX_SAFE_INTEGER`, except + negative zero - arrays containing supported values - plain string-keyed objects containing supported values Empty arrays and objects, nested nulls, Unicode, binary bytes, and fractional -safe numbers round-trip losslessly. `undefined`, `bigint`, non-finite and unsafe -numbers, class instances, symbols, functions, and cyclic objects are rejected. +safe numbers round-trip losslessly. A top-level `undefined` success is accepted +for `Schema.Void`; `undefined` payloads, failures, and nested values remain +invalid. Negative zero, `bigint`, non-finite and unsafe numbers, class +instances, symbols, functions, and cyclic objects are rejected. Encoded user values are limited to 1 MiB by default. A task definition can override `storageLimits.maxValueBytes`; the same bound applies independently to payload, success, and typed-failure envelopes. diff --git a/src/EngineRecord.ts b/src/EngineRecord.ts index 4de6a32..c0277f9 100644 --- a/src/EngineRecord.ts +++ b/src/EngineRecord.ts @@ -38,16 +38,58 @@ export const TextFromBytes = Schema.Unknown.pipe( }), ); +const numberFromText = (integer: boolean) => + SchemaGetter.transformOrFail((value: string, options) => { + const pattern = integer + ? /^-?(?:0|[1-9]\d*)$/ + : /^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$/; + const parsed = Number(value); + return pattern.test(value) && + Number.isFinite(parsed) && + Math.abs(parsed) <= Number.MAX_SAFE_INTEGER + ? Effect.succeed(parsed) + : Effect.fail( + new SchemaIssue.InvalidValue( + { + message: integer + ? "Expected a safe decimal integer" + : "Expected a safe decimal number", + }, + value, + options, + ), + ); + }); + export const NumberFromBytes = TextFromBytes.pipe( - Schema.decodeTo(Schema.Number, { - decode: SchemaGetter.transform(Number), + Schema.decodeTo(Schema.Finite, { + decode: numberFromText(false), + encode: SchemaGetter.transform(String), + }), +); + +export const IntegerFromBytes = TextFromBytes.pipe( + Schema.decodeTo(Schema.Int, { + decode: numberFromText(true), encode: SchemaGetter.transform(String), }), ); export const BooleanFromBytes = TextFromBytes.pipe( Schema.decodeTo(Schema.Boolean, { - decode: SchemaGetter.transform((value) => value === "1"), + decode: SchemaGetter.transformOrFail((value, options) => + value === "1" + ? Effect.succeed(true) + : value === "0" + ? Effect.succeed(false) + : Effect.fail( + new SchemaIssue.InvalidValue( + { message: 'Expected Redis boolean "0" or "1"' }, + value, + options, + ), + ), + ), encode: SchemaGetter.transform((value) => (value ? "1" : "0")), }), ); @@ -57,28 +99,28 @@ const msgpackListFromBytes = (item: S) => export const EngineTaskSchema = Schema.Struct({ id: TextFromBytes, - protocolVersion: NumberFromBytes, + protocolVersion: IntegerFromBytes, schemaId: TextFromBytes, - generation: NumberFromBytes, + generation: IntegerFromBytes, name: TextFromBytes, delay: NumberFromBytes, - maxRetries: NumberFromBytes, - maxStalledCount: NumberFromBytes, - maxErrorEntries: NumberFromBytes, - maxRelationships: NumberFromBytes, - maxEventEntries: NumberFromBytes, - taskRecordRetentionMs: NumberFromBytes, - resultRetentionMs: NumberFromBytes, - terminalIndexRetentionMs: NumberFromBytes, - deadLetterRetentionMs: NumberFromBytes, - eventRetentionMs: NumberFromBytes, - attempt: NumberFromBytes, - handlerFailureCount: NumberFromBytes, - stalledAttemptCount: NumberFromBytes, + maxRetries: IntegerFromBytes, + maxStalledCount: IntegerFromBytes, + maxErrorEntries: IntegerFromBytes, + maxRelationships: IntegerFromBytes, + maxEventEntries: IntegerFromBytes, + taskRecordRetentionMs: IntegerFromBytes, + resultRetentionMs: IntegerFromBytes, + terminalIndexRetentionMs: IntegerFromBytes, + deadLetterRetentionMs: IntegerFromBytes, + eventRetentionMs: IntegerFromBytes, + attempt: IntegerFromBytes, + handlerFailureCount: IntegerFromBytes, + stalledAttemptCount: IntegerFromBytes, onSuccessPolicy: TextFromBytes.pipe(Schema.decodeTo(CompletionPolicySchema)), onFailurePolicy: TextFromBytes.pipe(Schema.decodeTo(CompletionPolicySchema)), - createdAt: NumberFromBytes.pipe(Schema.decodeTo(DateFromNumberSchema)), - updatedAt: NumberFromBytes.pipe(Schema.decodeTo(DateFromNumberSchema)), + createdAt: IntegerFromBytes.pipe(Schema.decodeTo(DateFromNumberSchema)), + updatedAt: IntegerFromBytes.pipe(Schema.decodeTo(DateFromNumberSchema)), payload: UnknownFromMsgpack, success: UnknownFromMsgpack.pipe(Schema.optional), errors: msgpackListFromBytes( @@ -100,11 +142,11 @@ export const EngineTaskSchema = Schema.Struct({ export type EngineTask = typeof EngineTaskSchema.Type; export const EngineTerminalResultSchema = Schema.Struct({ - protocolVersion: NumberFromBytes, + protocolVersion: IntegerFromBytes, schemaId: TextFromBytes, - generation: NumberFromBytes, + generation: IntegerFromBytes, outcome: TextFromBytes.pipe(Schema.decodeTo(TaskOutcomeSchema)), - settledAt: NumberFromBytes, + settledAt: IntegerFromBytes, success: UnknownFromMsgpack.pipe(Schema.optional), failure: UnknownFromMsgpack.pipe(Schema.optional), }); diff --git a/src/PublicContracts.test.ts b/src/PublicContracts.test.ts index 45534e9..f72efad 100644 --- a/src/PublicContracts.test.ts +++ b/src/PublicContracts.test.ts @@ -1,8 +1,9 @@ -import { expect, it } from "vitest"; import * as Context from "effect/Context"; import type * as Effect from "effect/Effect"; import type * as Layer from "effect/Layer"; import * as Schema from "effect/Schema"; +import type * as Stream from "effect/Stream"; +import { expect, it } from "vitest"; import type * as RedisPool from "./RedisPool.js"; import * as TaskEngine from "./TaskEngine.js"; import * as TaskQueue from "./TaskQueue.js"; @@ -71,7 +72,7 @@ const compilePublicContracts = () => { Equal, boolean> >; type CompleteOneError = Expect< - Equal, TaskQueue.CompleteError> + Equal, TaskQueue.CompleteOneError> >; type CompleteOneServices = Expect< Equal, Effect.Services> @@ -135,6 +136,18 @@ const compilePublicContracts = () => { >; type RejectAnyServices = ExpectFalse>>; + const events = TaskQueue.stream(queue); + type FailedEvent = Extract< + Stream.Success, + { readonly _tag: "task.failed" } + >; + type FailedEventError = Expect< + Equal< + FailedEvent["payload"]["error"], + boolean | TaskRecord.StalledErrorSchema | TaskRecord.CanceledErrorSchema + > + >; + const layerNoDeps = TaskEngine.layerNoDeps(); const liveLayer = TaskEngine.layer(); type LayerNoDepsRequirement = Expect< @@ -164,6 +177,7 @@ const compilePublicContracts = () => { | RejectUnknownError | RejectErasedServices | RejectAnyServices + | FailedEventError | LayerNoDepsRequirement | LiveLayerRequirement; }; diff --git a/src/RetrySchedule.ts b/src/RetrySchedule.ts index 09c9f64..a7e8856 100644 --- a/src/RetrySchedule.ts +++ b/src/RetrySchedule.ts @@ -49,7 +49,11 @@ export const nextRunAt = Effect.fnUntraced(function* ( (value) => Effect.succeed([value, -1] as const), ); if (delay === -1) return undefined; - time = error.timestamp.getTime() + Duration.toMillis(delay); + const delayMillis = Duration.toMillis(delay); + time = error.timestamp.getTime() + delayMillis; + if (!Number.isFinite(time) || Math.abs(time) > Number.MAX_SAFE_INTEGER) { + return undefined; + } } return time; }); diff --git a/src/Schemas.test.ts b/src/Schemas.test.ts index 9bb7fb7..a773c3f 100644 --- a/src/Schemas.test.ts +++ b/src/Schemas.test.ts @@ -1,5 +1,10 @@ import { expect, it } from "@effect/vitest"; import { Effect, Schema } from "effect"; +import { + BooleanFromBytes, + IntegerFromBytes, + NumberFromBytes, +} from "./EngineRecord.js"; import { UnknownFromMsgpack } from "./MessagePack.js"; it.effect("MessagePack round-trips supported values", () => @@ -29,3 +34,54 @@ it.effect("non-byte MessagePack input fails as a SchemaError", () => expect(error._tag).toBe("SchemaError"); }), ); + +it.effect("Redis numeric fields reject non-finite and nonnumeric values", () => + Effect.gen(function* () { + for (const value of [ + "NaN", + "Infinity", + "not-a-number", + String(Number.MAX_SAFE_INTEGER + 1), + "", + " ", + "0x10", + "1e2", + "+1", + "9007199254740992.5", + ]) { + const error = yield* Schema.decodeEffect(IntegerFromBytes)(value).pipe( + Effect.flip, + ); + expect(error._tag).toBe("SchemaError"); + } + }), +); + +it.effect("Redis finite numeric fields retain fractional values", () => + Effect.gen(function* () { + expect(yield* Schema.decodeEffect(NumberFromBytes)("0.5")).toBe(0.5); + expect(yield* Schema.decodeEffect(NumberFromBytes)("1e-7")).toBe(1e-7); + expect( + yield* Schema.decodeEffect(NumberFromBytes)( + yield* Schema.encodeEffect(NumberFromBytes)(1e-7), + ), + ).toBe(1e-7); + const error = yield* Schema.decodeEffect(NumberFromBytes)( + "9007199254740992.5", + ).pipe(Effect.flip); + expect(error._tag).toBe("SchemaError"); + }), +); + +it.effect("Redis boolean fields accept only zero and one", () => + Effect.gen(function* () { + expect(yield* Schema.decodeEffect(BooleanFromBytes)("0")).toBe(false); + expect(yield* Schema.decodeEffect(BooleanFromBytes)("1")).toBe(true); + for (const value of ["", "2", "false", "bogus"]) { + const error = yield* Schema.decodeEffect(BooleanFromBytes)(value).pipe( + Effect.flip, + ); + expect(error._tag).toBe("SchemaError"); + } + }), +); diff --git a/src/StorageProtocol.test.ts b/src/StorageProtocol.test.ts index d943601..c9c3fba 100644 --- a/src/StorageProtocol.test.ts +++ b/src/StorageProtocol.test.ts @@ -48,8 +48,14 @@ describe("StorageProtocol v1", () => { BigInt(1), Number.POSITIVE_INFINITY, Number.MAX_SAFE_INTEGER + 1, + -0, new Date(), cyclic, + { visible: true, [Symbol("hidden")]: false }, + Object.defineProperty({}, "hidden", { value: true }), + Object.defineProperty({}, "computed", { get: () => true }), + Object.assign([1], { extra: true }), + Object.assign([1], { [Symbol("hidden")]: true }), ]) { const error = yield* StorageProtocol.encodeValue( "schema", @@ -61,6 +67,77 @@ describe("StorageProtocol v1", () => { }), ); + it.effect("supports top-level undefined only for Schema.Void successes", () => + Effect.gen(function* () { + const encoded = yield* StorageProtocol.encodeValue( + "schema", + "success", + undefined, + ); + expect( + yield* StorageProtocol.decodeValue(encoded, "schema", "success"), + ).toBeUndefined(); + + const error = yield* StorageProtocol.encodeValue( + "schema", + "payload", + undefined, + ).pipe(Effect.flip); + expect(error._tag).toBe("UnsupportedStorageValue"); + }), + ); + + it.effect("preserves prototype-sensitive keys as ordinary data", () => + Effect.gen(function* () { + const value: Record = { + constructor: "constructor-value", + prototype: "prototype-value", + }; + Object.defineProperty(value, "__proto__", { + enumerable: true, + value: "proto-value", + }); + + const encoded = yield* StorageProtocol.encodeValue( + "schema", + "payload", + value, + ); + const decoded = yield* StorageProtocol.decodeValue( + encoded, + "schema", + "payload", + ); + expect(decoded).toEqual(value); + expect(Object.getPrototypeOf(decoded)).toBe(Object.prototype); + expect(Object.getOwnPropertyDescriptor(decoded, "__proto__")?.value).toBe( + "proto-value", + ); + }), + ); + + it.effect("rejects decoded values outside the lossless storage domain", () => + Effect.gen(function* () { + const external = new Packr({ useRecords: false }); + for (const unsupported of [ + Number.NaN, + Number.POSITIVE_INFINITY, + BigInt(Number.MAX_SAFE_INTEGER) + 1n, + new Date("2026-01-01T00:00:00.000Z"), + ]) { + const encoded = `effectmq:v1:${Buffer.from( + external.pack([1, "schema", "payload", unsupported]), + ).toString("base64")}`; + const error = yield* StorageProtocol.decodeValue( + encoded, + "schema", + "payload", + ).pipe(Effect.flip); + expect(error._tag).toBe("CorruptStorageValue"); + } + }), + ); + it.effect("size limits are checked on the encoded representation", () => Effect.gen(function* () { const error = yield* StorageProtocol.encodeValue( diff --git a/src/StorageProtocol.ts b/src/StorageProtocol.ts index 017bf30..7b8b381 100644 --- a/src/StorageProtocol.ts +++ b/src/StorageProtocol.ts @@ -176,14 +176,22 @@ export type StorageProtocolError = | UnsupportedProtocolVersion | SchemaIdentityMismatch; -const packr = new Packr({ useRecords: false, int64AsType: "number" }); +const packr = new Packr({ + useRecords: false, + mapsAsObjects: false, + int64AsType: "bigint", +}); const prefix = "effectmq:v1:"; const validate = ( value: unknown, path: string, seen: Set, + allowRootUndefined = false, ): UnsupportedStorageValue | undefined => { + if (value === undefined && path === "$" && allowRootUndefined) { + return undefined; + } if ( value === null || typeof value === "string" || @@ -193,7 +201,9 @@ const validate = ( return undefined; } if (typeof value === "number") { - return Number.isFinite(value) && Math.abs(value) <= Number.MAX_SAFE_INTEGER + return Number.isFinite(value) && + Math.abs(value) <= Number.MAX_SAFE_INTEGER && + !Object.is(value, -0) ? undefined : new UnsupportedStorageValue({ path, valueType: "unsafe number" }); } @@ -206,9 +216,33 @@ const validate = ( seen.add(value); if (Array.isArray(value)) { for (let index = 0; index < value.length; index++) { - const error = validate(value[index], `${path}[${index}]`, seen); + const descriptor = Object.getOwnPropertyDescriptor(value, String(index)); + if ( + descriptor === undefined || + !descriptor.enumerable || + !("value" in descriptor) + ) { + return new UnsupportedStorageValue({ + path: `${path}[${index}]`, + valueType: "sparse or non-data array entry", + }); + } + const error = validate(descriptor.value, `${path}[${index}]`, seen); if (error) return error; } + for (const key of Reflect.ownKeys(value)) { + if (key === "length") continue; + if ( + typeof key !== "string" || + !/^(?:0|[1-9]\d*)$/.test(key) || + Number(key) >= value.length + ) { + return new UnsupportedStorageValue({ + path, + valueType: "array with custom properties", + }); + } + } seen.delete(value); return undefined; } @@ -218,24 +252,73 @@ const validate = ( valueType: value.constructor?.name ?? "non-plain object", }); } - for (const [key, nested] of Object.entries(value)) { - const error = validate(nested, `${path}.${key}`, seen); + for (const key of Reflect.ownKeys(value)) { + if (typeof key !== "string") { + return new UnsupportedStorageValue({ + path, + valueType: "symbol-keyed object", + }); + } + const descriptor = Object.getOwnPropertyDescriptor(value, key); + if ( + descriptor === undefined || + !descriptor.enumerable || + !("value" in descriptor) + ) { + return new UnsupportedStorageValue({ + path: `${path}.${key}`, + valueType: "non-data property", + }); + } + const error = validate(descriptor.value, `${path}.${key}`, seen); if (error) return error; } seen.delete(value); return undefined; }; +const defineDataProperty = ( + target: Record, + key: string, + value: unknown, +) => { + Object.defineProperty(target, key, { + configurable: true, + enumerable: true, + value, + writable: true, + }); +}; + const normalizeDecoded = (value: unknown): unknown => { if (Buffer.isBuffer(value)) return new Uint8Array(value); + if (typeof value === "bigint") { + return value >= BigInt(Number.MIN_SAFE_INTEGER) && + value <= BigInt(Number.MAX_SAFE_INTEGER) + ? Number(value) + : value; + } if (Array.isArray(value)) return value.map(normalizeDecoded); - if (typeof value === "object" && value !== null) { - return Object.fromEntries( - Object.entries(value).map(([key, nested]) => [ - key, - normalizeDecoded(nested), - ]), - ); + if (value instanceof Map) { + const normalized: Record = {}; + for (const [key, nested] of value) { + if (typeof key !== "string") { + throw new TypeError("Storage object keys must be strings"); + } + defineDataProperty(normalized, key, normalizeDecoded(nested)); + } + return normalized; + } + if ( + typeof value === "object" && + value !== null && + Object.getPrototypeOf(value) === Object.prototype + ) { + const normalized: Record = {}; + for (const [key, nested] of Object.entries(value)) { + defineDataProperty(normalized, key, normalizeDecoded(nested)); + } + return normalized; } return value; }; @@ -243,10 +326,13 @@ const normalizeDecoded = (value: unknown): unknown => { /** * Encodes a lossless JavaScript value into an ASCII-safe MessagePack envelope. * - * Supported values are `null`, strings, booleans, finite safe numbers, + * Supported values are `null`, strings, booleans, finite safe numbers other + * than negative zero, * `Uint8Array`, arrays, and plain objects composed recursively from those - * values. Cycles, class instances, unsafe numbers, `undefined`, `bigint`, - * functions, and symbols fail with {@link UnsupportedStorageValue}. + * values. A top-level `undefined` success is supported for `Schema.Void`; + * nested `undefined` values and `undefined` payloads/failures remain invalid. + * Cycles, class instances, unsafe numbers, `bigint`, functions, and symbols + * fail with {@link UnsupportedStorageValue}. * * The size limit applies to the MessagePack bytes before base64 encoding. * @@ -284,7 +370,7 @@ export const encodeValue = ( > => Effect.gen(function* () { const unsupported = yield* Effect.try({ - try: () => validate(value, "$", new Set()), + try: () => validate(value, "$", new Set(), kind === "success"), catch: (cause) => new StorageEncodingError({ stage: "messagepack", @@ -392,5 +478,25 @@ export const decodeValue = ( message: `Expected ${expectedKind} envelope, received ${String(kind)}`, }); } - return normalizeDecoded(value); + const normalized = yield* Effect.try({ + try: () => normalizeDecoded(value), + catch: (cause) => + new CorruptStorageValue({ + message: "Stored value could not be normalized safely", + cause, + }), + }); + const unsupported = validate( + normalized, + "$", + new Set(), + expectedKind === "success", + ); + if (unsupported !== undefined) { + return yield* new CorruptStorageValue({ + message: `Stored value is outside the supported storage domain at ${unsupported.path}`, + cause: unsupported, + }); + } + return normalized; }); diff --git a/src/TaskEngine.replies.test.ts b/src/TaskEngine.replies.test.ts index 052da79..77bc2dc 100644 --- a/src/TaskEngine.replies.test.ts +++ b/src/TaskEngine.replies.test.ts @@ -111,3 +111,40 @@ it.effect("rejects odd stream field arrays", () => }); }), ); + +it.effect("rejects malformed caller cursors as typed failures", () => + Effect.gen(function* () { + const engine = yield* TaskEngine.makeWithRedis( + service({ scriptReply: ["0-0", "0-0", "0-0"] }), + ); + for (const cursor of [ + "not-a-stream-id", + "18446744073709551616-0", + "0-18446744073709551616", + "1".repeat(1_000), + ]) { + const error = yield* engine + .stream("queue", { cursor }) + .pipe(Stream.runHead, Effect.flip); + expect(error).toMatchObject({ _tag: "InvalidCursor", cursor }); + } + }), +); + +it.effect("rejects malformed persisted latest cursors as invalid replies", () => + Effect.gen(function* () { + const engine = yield* TaskEngine.makeWithRedis( + service({ scriptReply: ["0-0", "0-0", "$"] }), + ); + const error = yield* engine + .stream("queue") + .pipe(Stream.runHead, Effect.flip); + expect(error).toMatchObject({ + _tag: "TaskEngineError", + reason: { + _tag: "InvalidReply", + operation: "effectmq_eventCursors", + }, + }); + }), +); diff --git a/src/TaskEngine.test.ts b/src/TaskEngine.test.ts index 8697042..31397b7 100644 --- a/src/TaskEngine.test.ts +++ b/src/TaskEngine.test.ts @@ -1,6 +1,6 @@ -import { Effect, Metric } from "effect"; -import { Packr } from "msgpackr"; import { expect, layer } from "@effect/vitest"; +import { Duration, Effect, Metric } from "effect"; +import { Packr } from "msgpackr"; import { Observability, RedisPool, TaskEngine } from "./index.js"; import { getLists, TestLayer } from "./testing/redisLayer.js"; import { @@ -39,6 +39,64 @@ layer(TestLayer, { excludeTestServices: true, timeout: "60 seconds" })( }), ); + it.effect( + "rejects unsafe low-level numeric inputs before Redis mutation", + () => + Effect.gen(function* () { + const engine = yield* TaskEngine.TaskEngine; + const prefix = "invalid-low-level-numbers"; + const error = yield* engine + .createTask({ + id: "invalid", + name: "invalid", + payload: null, + delay: Number.NaN, + maxRetries: 0, + onSuccessPolicy: "keep", + onFailurePolicy: "keep", + prefix, + }) + .pipe(Effect.flip); + expect(error.reason).toMatchObject({ + _tag: "InvalidInput", + field: "delay", + }); + expect(yield* engine.getGeneration(prefix, "invalid")).toBe(0); + + const lockError = yield* engine + .takeTask(prefix, 1.5) + .pipe(Effect.flip); + expect(lockError.reason).toMatchObject({ + _tag: "InvalidInput", + field: "lockTimeout", + }); + + yield* engine.createTask({ + id: "retry-at", + name: "retry-at", + payload: null, + delay: 0, + maxRetries: 1, + onSuccessPolicy: "keep", + onFailurePolicy: "keep", + prefix, + }); + yield* takeTask(engine, prefix, 1_000); + const retryError = yield* writeError( + engine, + prefix, + "retry-at", + "failure", + Duration.infinity, + ).pipe(Effect.flip); + expect(retryError).toMatchObject({ + _tag: "TaskEngineError", + reason: { _tag: "InvalidInput", field: "retryAt" }, + }); + expect((yield* getLists(prefix)).active).toEqual(["retry-at"]); + }), + ); + it.effect("task inspection is capped, cursor-paginated, and ordered", () => Effect.gen(function* () { const engine = yield* TaskEngine.TaskEngine; @@ -421,6 +479,7 @@ layer(TestLayer, { excludeTestServices: true, timeout: "60 seconds" })( Effect.gen(function* () { const taskEngine = yield* TaskEngine.TaskEngine; const redis = yield* RedisPool.RedisPool; + const productionEngine = yield* TaskEngine.makeWithRedis(redis); const prefix = "same-state-active"; yield* taskEngine.createTask({ id: "active-1", @@ -432,11 +491,12 @@ layer(TestLayer, { excludeTestServices: true, timeout: "60 seconds" })( onFailurePolicy: "keep", prefix, }); - yield* takeTask(taskEngine, prefix, 30_000); + yield* takeTask(productionEngine, prefix, 30_000); // Simulate a partially corrupted pre-fix record. Any transition must // repair cross-state membership rather than preserving extra indexes. const keyPrefix = `~effectmq:v1:${prefix}`; + yield* redis.send("HDEL", `${keyPrefix}:task:active-1`, "currentList"); yield* redis.send("RPUSH", `${keyPrefix}:wait`, "active-1"); yield* redis.send( "ZADD", @@ -447,7 +507,7 @@ layer(TestLayer, { excludeTestServices: true, timeout: "60 seconds" })( yield* redis.send("ZADD", `${keyPrefix}:failed`, "1", "active-1"); yield* redis.send("ZADD", `${keyPrefix}:success`, "1", "active-1"); - yield* extendLock(taskEngine, prefix, "active-1", 30_000); + yield* extendLock(productionEngine, prefix, "active-1", 30_000); expect(yield* getLists(prefix)).toEqual({ active: ["active-1"], @@ -459,6 +519,43 @@ layer(TestLayer, { excludeTestServices: true, timeout: "60 seconds" })( }), ); + it.effect("stale rolling-deployment list markers fall back to repair", () => + Effect.gen(function* () { + const redis = yield* RedisPool.RedisPool; + const engine = yield* TaskEngine.makeWithRedis(redis); + const prefix = "stale-current-list"; + const keyPrefix = `~effectmq:v1:${prefix}`; + yield* engine.createTask({ + id: "stale", + name: "stale", + payload: null, + delay: 0, + maxRetries: 0, + onSuccessPolicy: "mark-as-success", + onFailurePolicy: "mark-as-failure", + prefix, + }); + yield* takeTask(engine, prefix, 30_000); + yield* writeSuccess(engine, prefix, "stale", "ok"); + yield* redis.send( + "HSET", + `${keyPrefix}:task:stale`, + "currentList", + "wait", + ); + + yield* engine.removeTask(prefix, "stale"); + + expect(yield* getLists(prefix)).toEqual({ + active: [], + failed: [], + scheduled: [], + success: [], + wait: [], + }); + }), + ); + it.effect("success happy path with delete on success policy", () => Effect.gen(function* () { const taskEngine = yield* TaskEngine.TaskEngine; @@ -689,6 +786,57 @@ layer(TestLayer, { excludeTestServices: true, timeout: "60 seconds" })( }), ); + it.effect("writeError treats zero as an immediate retry time", () => + Effect.gen(function* () { + const taskEngine = yield* TaskEngine.TaskEngine; + yield* TaskEngine.setMockTime(1000000000000); + const prefix = "retry-zero"; + yield* taskEngine.createTask({ + id: "r0", + name: "t", + payload: "p", + delay: 0, + maxRetries: 5, + onSuccessPolicy: "delete", + onFailurePolicy: "mark-as-failure", + prefix, + }); + + yield* takeTask(taskEngine, prefix, 30000); + yield* writeError(taskEngine, prefix, "r0", stalled(1), 0); + + const lists = yield* getLists(prefix); + expect(lists.wait).toEqual(["r0"]); + expect(lists.failed).toEqual([]); + }), + ); + + it.effect("retains a false terminal failure with zero error history", () => + Effect.gen(function* () { + const taskEngine = yield* TaskEngine.TaskEngine; + const prefix = "false-terminal-failure"; + yield* taskEngine.createTask({ + id: "false", + name: "t", + payload: "p", + delay: 0, + maxRetries: 0, + maxErrorEntries: 0, + onSuccessPolicy: "delete", + onFailurePolicy: "keep", + prefix, + }); + + yield* takeTask(taskEngine, prefix, 30000); + yield* writeError(taskEngine, prefix, "false", false); + + expect(yield* taskEngine.getResult(prefix, "false", 1)).toMatchObject({ + outcome: "failure", + failure: false, + }); + }), + ); + it.effect( "writeError without a retryAt applies the failure policy immediately", () => diff --git a/src/TaskEngine.ts b/src/TaskEngine.ts index c20f131..5fb0604 100644 --- a/src/TaskEngine.ts +++ b/src/TaskEngine.ts @@ -8,6 +8,8 @@ * * @module */ + +import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import * as Context from "effect/Context"; import * as Crypto from "effect/Crypto"; import * as Data from "effect/Data"; @@ -18,7 +20,6 @@ import * as Metric from "effect/Metric"; import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; -import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import { type EngineTask, type EngineTaskInsert, @@ -89,6 +90,12 @@ const validateConfig = ( * @since 0.1.0 */ export type TaskEngineErrorReason = + | { + readonly _tag: "InvalidInput"; + readonly operation: string; + readonly field: string; + readonly constraint: string; + } | { readonly _tag: "TransportFailure"; readonly operation: string } | { readonly _tag: "ScriptFailure"; readonly operation: string } | { @@ -209,6 +216,11 @@ export class CursorExpired extends Data.TaggedError("CursorExpired")<{ readonly earliest: string; }> {} +/** Indicates that a stream cursor is neither `$`, `0`, nor a Redis stream id. */ +export class InvalidCursor extends Data.TaggedError("InvalidCursor")<{ + readonly cursor: string; +}> {} + const diagnosticText = (cause: unknown, depth = 0): string => { if (depth >= 6) return String(cause); if (typeof cause !== "object" || cause === null) return String(cause); @@ -253,12 +265,51 @@ const classifyRedisFailure = ( const isLeaseLost = (error: TaskEngineError) => error.reason._tag === "LeaseLost"; -const compareStreamIds = (left: string, right: string): number => { - const [leftTime = "0", leftSequence = "0"] = left.split("-"); - const [rightTime = "0", rightSequence = "0"] = right.split("-"); - const timeDifference = BigInt(leftTime) - BigInt(rightTime); +const validatePositiveSafeInteger = ( + operation: string, + field: string, + value: number, +) => + Number.isSafeInteger(value) && value > 0 + ? Effect.void + : Effect.fail( + new TaskEngineError({ + reason: { + _tag: "InvalidInput", + operation, + field, + constraint: "a positive safe integer", + }, + cause: value, + }), + ); + +type ParsedStreamId = readonly [time: bigint, sequence: bigint]; + +const parseStreamId = (value: string): ParsedStreamId | undefined => { + const match = /^(\d+)(?:-(\d+))?$/.exec(value); + if ( + match === null || + match[1].length > 20 || + (match[2] !== undefined && match[2].length > 20) + ) { + return undefined; + } + const time = BigInt(match[1]); + const sequence = BigInt(match[2] ?? "0"); + const maxComponent = (1n << 64n) - 1n; + return time <= maxComponent && sequence <= maxComponent + ? [time, sequence] + : undefined; +}; + +const compareStreamIds = ( + [leftTime, leftSequence]: ParsedStreamId, + [rightTime, rightSequence]: ParsedStreamId, +): number => { + const timeDifference = leftTime - rightTime; if (timeDifference !== 0n) return timeDifference < 0n ? -1 : 1; - const sequenceDifference = BigInt(leftSequence) - BigInt(rightSequence); + const sequenceDifference = leftSequence - rightSequence; return sequenceDifference === 0n ? 0 : sequenceDifference < 0n ? -1 : 1; }; @@ -391,7 +442,7 @@ export class TaskEngine extends Context.Service< }, ) => Stream.Stream< Event, - TaskEngineError | CursorExpired | Schema.SchemaError + TaskEngineError | CursorExpired | InvalidCursor | Schema.SchemaError >; } >()("@effectmq/core/TaskEngine") {} @@ -668,6 +719,56 @@ export const makeWithRedis = ( ); const offerTask = Effect.fnUntraced(function* (task: EngineTaskInsert) { + const numericFields: ReadonlyArray< + readonly [keyof EngineTaskInsert, number, boolean] + > = [ + ["delay", task.delay, false], + ["maxRetries", task.maxRetries, true], + ["maxStalledCount", task.maxStalledCount ?? 1, true], + ["maxErrorEntries", task.maxErrorEntries ?? 100, true], + ["maxRelationships", task.maxRelationships ?? 1_000, true], + ["maxEventEntries", task.maxEventEntries ?? 10_000, true], + [ + "taskRecordRetentionMs", + task.taskRecordRetentionMs ?? 604_800_000, + true, + ], + ["resultRetentionMs", task.resultRetentionMs ?? 86_400_000, true], + [ + "terminalIndexRetentionMs", + task.terminalIndexRetentionMs ?? 604_800_000, + true, + ], + [ + "deadLetterRetentionMs", + task.deadLetterRetentionMs ?? 2_592_000_000, + true, + ], + ["eventRetentionMs", task.eventRetentionMs ?? 604_800_000, true], + ]; + for (const [field, value, integer] of numericFields) { + const minimum = + field === "maxRetries" ? -1 : field === "maxEventEntries" ? 1 : 0; + if ( + !Number.isFinite(value) || + Math.abs(value) > Number.MAX_SAFE_INTEGER || + value < minimum || + (integer && !Number.isSafeInteger(value)) + ) { + return yield* new TaskEngineError({ + reason: { + _tag: "InvalidInput", + operation: "effectmq_createTask", + field, + constraint: + field === "delay" + ? "a finite safe number greater than or equal to 0" + : `a safe integer greater than or equal to ${minimum}`, + }, + cause: value, + }); + } + } const retentionHolder = task.retentionHolder ? yield* pack({ ...task.retentionHolder, @@ -776,13 +877,41 @@ export const makeWithRedis = ( error: unknown, retryAt?: Duration.Input, ) { + const retryAtMillis = yield* Effect.try({ + try: () => (retryAt === undefined ? -1 : Duration.toMillis(retryAt)), + catch: (cause) => + new TaskEngineError({ + reason: { + _tag: "InvalidInput", + operation: "effectmq_writeError", + field: "retryAt", + constraint: "a valid finite safe timestamp", + }, + cause, + }), + }); + if ( + !Number.isFinite(retryAtMillis) || + Math.abs(retryAtMillis) > Number.MAX_SAFE_INTEGER || + retryAtMillis < -1 + ) { + return yield* new TaskEngineError({ + reason: { + _tag: "InvalidInput", + operation: "effectmq_writeError", + field: "retryAt", + constraint: "a finite safe timestamp or the terminal sentinel", + }, + cause: retryAtMillis, + }); + } yield* withLeaseFence( writeErrorFn( withPrefix(prefix), leaseToken, id, yield* pack(error), - String(retryAt ? Duration.toMillis(retryAt) : -1), + String(retryAtMillis), ), prefix, id, @@ -890,6 +1019,11 @@ export const makeWithRedis = ( prefix: string, lockTimeout: number, ) { + yield* validatePositiveSafeInteger( + "effectmq_takeTask", + "lockTimeout", + lockTimeout, + ); const crypto = yield* Crypto.Crypto; const leaseToken = yield* crypto.randomUUIDv4.pipe( Effect.map((uuid) => `lease/${uuid}`), @@ -929,12 +1063,25 @@ export const makeWithRedis = ( forceRemoveTask: (prefix, id) => forceRemoveTaskFn(withPrefix(prefix), id).pipe(Effect.asVoid), - extendLock: (prefix, id, leaseToken, lockTimeout) => - withLeaseFence( - extendLockFn(withPrefix(prefix), leaseToken, id, String(lockTimeout)), - prefix, - id, - ).pipe(Effect.asVoid), + extendLock: Effect.fnUntraced( + function* (prefix, id, leaseToken, lockTimeout) { + yield* validatePositiveSafeInteger( + "effectmq_extendLock", + "lockTimeout", + lockTimeout, + ); + return yield* withLeaseFence( + extendLockFn( + withPrefix(prefix), + leaseToken, + id, + String(lockTimeout), + ), + prefix, + id, + ).pipe(Effect.asVoid); + }, + ), removeLock: (prefix, id, leaseToken) => withLeaseFence( removeLockFn(withPrefix(prefix), leaseToken, id), @@ -1136,14 +1283,42 @@ export const makeWithRedis = ( ), latest: yield* decodeText("effectmq_eventCursors.latest", latest), }; + const firstId = parseStreamId(cursors.first); + const earliestId = parseStreamId(cursors.earliest); + const latestId = parseStreamId(cursors.latest); + if ( + firstId === undefined || + earliestId === undefined || + latestId === undefined + ) { + return yield* invalidReply( + "effectmq_eventCursors", + "Redis stream ids", + cursors, + ); + } const cursor = options.cursor ?? cursors.latest; + const requestedId = + options.cursor === undefined + ? latestId + : cursor === "$" + ? undefined + : parseStreamId(cursor); + if (cursor !== "$" && requestedId === undefined) { + return yield* new InvalidCursor({ cursor }); + } const streamWasTrimmed = cursors.first !== "0-0" && - compareStreamIds(cursors.earliest, cursors.first) > 0; + compareStreamIds(earliestId, firstId) > 0; + const requestedFromStart = + requestedId !== undefined && + requestedId[0] === 0n && + requestedId[1] === 0n; const requestedTrimmedEvent = - cursor === "0" || - (compareStreamIds(cursor, cursors.first) >= 0 && - compareStreamIds(cursor, cursors.earliest) < 0); + requestedFromStart || + (requestedId !== undefined && + compareStreamIds(requestedId, firstId) >= 0 && + compareStreamIds(requestedId, earliestId) < 0); if (streamWasTrimmed && requestedTrimmedEvent) { return yield* new CursorExpired({ requested: cursor, diff --git a/src/TaskEvent.ts b/src/TaskEvent.ts index 3a4f75a..e6d5a28 100644 --- a/src/TaskEvent.ts +++ b/src/TaskEvent.ts @@ -4,6 +4,7 @@ import { BooleanFromBytes, EngineTaskSchema, ExecutionStateSchema, + IntegerFromBytes, NumberFromBytes, TaskLists, TextFromBytes, @@ -14,8 +15,8 @@ import { CompletionPolicySchema } from "./TaskRecord.js"; const eventBase = { id: Schema.String, taskId: Schema.String, - generation: NumberFromBytes, - protocolVersion: NumberFromBytes, + generation: IntegerFromBytes, + protocolVersion: IntegerFromBytes, schemaId: TextFromBytes, }; @@ -44,7 +45,7 @@ export const EventSchema = Schema.Union([ failureKind: TextFromBytes.pipe( Schema.decodeTo(Schema.Literals(["handler", "stall"])), ), - attempt: NumberFromBytes, + attempt: IntegerFromBytes, terminal: BooleanFromBytes, }), }), @@ -68,9 +69,9 @@ export const EventSchema = Schema.Union([ Schema.decodeTo(ExecutionStateSchema), Schema.optional, ), - attempt: NumberFromBytes, - handlerFailureCount: NumberFromBytes, - stalledAttemptCount: NumberFromBytes, + attempt: IntegerFromBytes, + handlerFailureCount: IntegerFromBytes, + stalledAttemptCount: IntegerFromBytes, }), }), ]); diff --git a/src/TaskEvents.test.ts b/src/TaskEvents.test.ts index b99bdc6..0b5f693 100644 --- a/src/TaskEvents.test.ts +++ b/src/TaskEvents.test.ts @@ -358,6 +358,49 @@ layer(TestLayer, { excludeTestServices: true, timeout: "60 seconds" })( }), ); + it.effect("malformed built-in failures are schema-validated", () => + Effect.gen(function* () { + const queue = makeQueue("ev-corrupt-built-in"); + const engine = yield* TaskEngine.TaskEngine; + const redis = yield* RedisPool.RedisPool; + yield* TaskQueue.offer(queue, { userId: "corrupt", amount: 1 }); + const cursor = (yield* engine.eventCursors(queue.name)).latest; + const packr = new Packr({ useRecords: false }); + + yield* redis.send( + "XADD", + `~effectmq:v1:${queue.name}:events`, + "*", + "taskId", + "corrupt", + "generation", + "1", + "protocolVersion", + "1", + "schemaId", + queue.task.schemaId, + "_tag", + "task.failed", + "policy", + "keep", + "error", + packr.pack({ _tag: "~effectmq/Error/Stalled" }), + "failureKind", + "stall", + "attempt", + "1", + "terminal", + "1", + ); + + const error = yield* TaskQueue.stream(queue, { cursor }).pipe( + Stream.runHead, + Effect.flip, + ); + expect(error._tag).toBe("SchemaError"); + }), + ); + it.effect( "execute offers and resolves with the handler's success value", () => diff --git a/src/TaskQueue.test.ts b/src/TaskQueue.test.ts index bbd9bec..55710ac 100644 --- a/src/TaskQueue.test.ts +++ b/src/TaskQueue.test.ts @@ -1,5 +1,5 @@ import { describe, expect, layer } from "@effect/vitest"; -import { Deferred, Effect, Schedule, Schema } from "effect"; +import { Deferred, Duration, Effect, Fiber, Schedule, Schema } from "effect"; import * as PersistenceRedis from "effect/unstable/persistence/Redis"; import type { EngineTask } from "./EngineRecord.js"; import { RedisPool, Task, TaskEngine, TaskQueue } from "./index.js"; @@ -85,6 +85,42 @@ layer(TestLayer, { excludeTestServices: true, timeout: "60 seconds" })( }), ); + it.effect("offer validates numeric options before mutating Redis", () => + Effect.gen(function* () { + const engine = yield* TaskEngine.TaskEngine; + const queue = makeQueue("tq-invalid-offer-options"); + for (const [options, field] of [ + [{ delay: Number.NaN }, "delay"], + [{ delay: Number.POSITIVE_INFINITY }, "delay"], + [{ delay: -1 }, "delay"], + [{ maxRetries: 1.5 }, "maxRetries"], + [{ maxStalledCount: -1 }, "maxStalledCount"], + ] as const) { + const error = yield* TaskQueue.offer( + queue, + { userId: "invalid", amount: 1 }, + options, + ).pipe(Effect.flip); + expect(error).toMatchObject({ _tag: "TaskOptionsError", field }); + expect(yield* engine.getGeneration(queue.name, "invalid")).toBe(0); + } + }), + ); + + it.effect( + "fractional delays round-trip through Redis exponent form", + () => + Effect.gen(function* () { + const queue = makeQueue("tq-fractional-delay"); + const offered = yield* TaskQueue.offer( + queue, + { userId: "fractional", amount: 1 }, + { delay: 1e-7 }, + ); + expect(offered.task.delay).toBe(1e-7); + }), + ); + it.effect( "a task's configured byte limit is enforced while offering", () => @@ -109,6 +145,24 @@ layer(TestLayer, { excludeTestServices: true, timeout: "60 seconds" })( }), ); + it.effect("Schema.Void successes complete and remain waitable", () => + Effect.gen(function* () { + const definition = Task.make({ + name: "tq-void-success", + payload: { id: Schema.String }, + success: Schema.Void, + error: Schema.Never, + idempotencyKey: ({ id }) => id, + }); + const queue = TaskQueue.make(definition.name, definition); + const offered = yield* TaskQueue.offer(queue, { id: "void" }); + + yield* TaskQueue.complete(queue, () => Effect.void); + + expect(yield* TaskQueue.wait(queue, offered.handle)).toBeUndefined(); + }), + ); + it.effect("outcome byte limits fail before writing terminal state", () => Effect.gen(function* () { const successDefinition = Task.make({ @@ -597,6 +651,31 @@ layer(TestLayer, { excludeTestServices: true, timeout: "60 seconds" })( }), ); + it.effect( + "completeOne rejects invalid processing options before taking a task", + () => + Effect.gen(function* () { + const queue = makeQueue("tq-invalid-processing"); + yield* TaskQueue.offer(queue, { userId: "waiting", amount: 1 }); + + for (const lockTimeout of [0, 0.5]) { + const error = yield* TaskQueue.completeOne( + queue, + () => Effect.succeed("unused"), + { lockTimeout }, + ).pipe(Effect.flip); + + expect(error).toMatchObject({ + _tag: "ProcessingConfigurationError", + field: "lockTimeout", + }); + } + const lists = yield* getLists(queue.name); + expect(lists.wait).toEqual(["waiting"]); + expect(lists.active).toEqual([]); + }), + ); + it.effect( "a failing task with a retry schedule lands on the scheduled list", () => @@ -615,6 +694,309 @@ layer(TestLayer, { excludeTestServices: true, timeout: "60 seconds" })( }), ); + it.effect("an infinite retry delay settles terminally", () => + Effect.gen(function* () { + const definition = Task.make({ + name: "tq-infinite-retry-delay", + payload: { id: Schema.String }, + success: Schema.String, + error: Schema.Struct({ reason: Schema.String }), + idempotencyKey: ({ id }) => id, + retry: Schedule.spaced(Duration.infinity), + maxRetries: 1, + }); + const queue = TaskQueue.make(definition.name, definition); + yield* TaskQueue.offer( + queue, + { id: "infinite" }, + { onFailurePolicy: "mark-as-failure" }, + ); + + yield* TaskQueue.complete(queue, () => + Effect.fail({ reason: "terminal" }), + ); + + expect((yield* getLists(queue.name)).failed).toEqual(["infinite"]); + }), + ); + + it.effect( + "wait ignores retryable failures and observes final success", + () => + Effect.gen(function* () { + const definition = Task.make({ + name: "tq-wait-through-retry", + payload: { id: Schema.String }, + success: Schema.String, + error: Schema.Struct({ reason: Schema.String }), + idempotencyKey: ({ id }) => id, + retry: Schedule.spaced("5 millis"), + maxRetries: 1, + }); + const queue = TaskQueue.make(definition.name, definition); + const offered = yield* TaskQueue.offer(queue, { id: "retry" }); + const waiter = yield* TaskQueue.wait(queue, offered.handle).pipe( + Effect.forkChild, + ); + + yield* Effect.yieldNow; + yield* TaskQueue.complete(queue, () => + Effect.fail({ reason: "transient" }), + ); + yield* Effect.sleep("10 millis"); + yield* TaskQueue.complete(queue, () => Effect.succeed("recovered")); + + expect(yield* Fiber.join(waiter)).toBe("recovered"); + }), + ); + + it.effect( + "retry-policy failures settle the attempt and remain typed", + () => + Effect.gen(function* () { + const definition = Task.make({ + name: "tq-retry-policy-failure", + payload: { id: Schema.String }, + success: Schema.String, + error: Schema.Struct({ reason: Schema.String }), + idempotencyKey: ({ id }) => id, + retry: { + while: () => Effect.fail({ reason: "schedule-broken" }), + }, + }); + const queue = TaskQueue.make(definition.name, definition); + yield* TaskQueue.offer( + queue, + { id: "policy" }, + { onFailurePolicy: "mark-as-failure" }, + ); + + const error = yield* TaskQueue.complete(queue, () => + Effect.fail({ reason: "handler-failed" }), + ).pipe(Effect.flip); + + expect(error).toMatchObject({ + _tag: "RetryPolicyError", + queue: queue.name, + taskId: "policy", + cause: { reason: "schedule-broken" }, + }); + expect((yield* getLists(queue.name)).failed).toEqual(["policy"]); + }), + ); + + it.effect( + "retry-policy defects settle the attempt and remain typed", + () => + Effect.gen(function* () { + const definition = Task.make({ + name: "tq-retry-policy-defect", + payload: { id: Schema.String }, + success: Schema.String, + error: Schema.Struct({ reason: Schema.String }), + idempotencyKey: ({ id }) => id, + retry: { + while: () => { + throw new Error("schedule-defect"); + }, + }, + }); + const queue = TaskQueue.make(definition.name, definition); + yield* TaskQueue.offer( + queue, + { id: "policy" }, + { onFailurePolicy: "mark-as-failure" }, + ); + + const error = yield* TaskQueue.complete(queue, () => + Effect.fail({ reason: "handler-failed" }), + ).pipe(Effect.flip); + + expect(error).toMatchObject({ + _tag: "RetryPolicyError", + queue: queue.name, + taskId: "policy", + cause: new Error("schedule-defect"), + }); + expect((yield* getLists(queue.name)).failed).toEqual(["policy"]); + }), + ); + + it.effect( + "interrupting retry-policy evaluation leaves the attempt recoverable", + () => + Effect.gen(function* () { + const engine = yield* TaskEngine.TaskEngine; + const entered = yield* Deferred.make(); + const definition = Task.make({ + name: "tq-retry-policy-interrupt", + payload: { id: Schema.String }, + success: Schema.String, + error: Schema.Struct({ reason: Schema.String }), + idempotencyKey: ({ id }) => id, + retry: { + while: () => + Deferred.succeed(entered, undefined).pipe( + Effect.andThen(Effect.never), + ), + }, + }); + const queue = TaskQueue.make(definition.name, definition); + yield* TaskQueue.offer( + queue, + { id: "interrupted" }, + { onFailurePolicy: "mark-as-failure" }, + ); + + const fiber = yield* TaskQueue.complete(queue, () => + Effect.fail({ reason: "handler-failed" }), + ).pipe(Effect.forkChild); + yield* Deferred.await(entered); + yield* Fiber.interrupt(fiber); + + const task = yield* engine.getTask(queue.name, "interrupted"); + expect(task?.outcome).toBeUndefined(); + expect(task?.errors).toEqual([]); + expect((yield* getLists(queue.name)).active).toEqual([ + "interrupted", + ]); + }), + ); + + it.effect( + "truncated retry history settles instead of resetting the schedule", + () => + Effect.gen(function* () { + const definition = Task.make({ + name: "tq-truncated-retry-history", + payload: { id: Schema.String }, + success: Schema.String, + error: Schema.Struct({ reason: Schema.String }), + storageLimits: { maxErrorEntries: 1 }, + idempotencyKey: ({ id }) => id, + retry: Schedule.recurs(2), + maxRetries: 10, + }); + const queue = TaskQueue.make(definition.name, definition); + yield* TaskQueue.offer( + queue, + { id: "truncated" }, + { onFailurePolicy: "mark-as-failure" }, + ); + + yield* TaskQueue.complete(queue, () => + Effect.fail({ reason: "first" }), + ); + yield* TaskQueue.complete(queue, () => + Effect.fail({ reason: "second" }), + ); + const error = yield* TaskQueue.complete(queue, () => + Effect.fail({ reason: "third" }), + ).pipe(Effect.flip); + + expect(error).toMatchObject({ + _tag: "RetryPolicyError", + queue: queue.name, + taskId: "truncated", + }); + expect((yield* getLists(queue.name)).failed).toEqual(["truncated"]); + }), + ); + + it.effect( + "stalled history is not replayed into handler retry policy", + () => + Effect.gen(function* () { + const engine = yield* TaskEngine.TaskEngine; + const redis = yield* RedisPool.RedisPool; + const seen: Array<{ readonly reason: string }> = []; + const definition = Task.make({ + name: "tq-stall-retry-input", + payload: { id: Schema.String }, + success: Schema.String, + error: Schema.Struct({ reason: Schema.String }), + idempotencyKey: ({ id }) => id, + retry: { + while: (error) => { + seen.push(error); + return false; + }, + }, + }); + const queue = TaskQueue.make(definition.name, definition); + const now = 2_000_000_000_000; + yield* TaskEngine.setMockTime(now); + yield* TaskQueue.offer( + queue, + { id: "stalled" }, + { onFailurePolicy: "mark-as-failure" }, + ); + + const first = yield* engine.takeTask(queue.name, 100); + expect(first).not.toBeNull(); + yield* redis.send("DEL", `~effectmq:v1:${queue.name}:lock:stalled`); + yield* TaskEngine.stepMockTime(101); + yield* engine.maintain(queue.name); + + yield* TaskQueue.complete(queue, () => + Effect.fail({ reason: "handler" }), + ); + expect(seen).toEqual([{ reason: "handler" }]); + }), + ); + + it.effect( + "terminal failures remain waitable with zero error history", + () => + Effect.gen(function* () { + const definition = Task.make({ + name: "tq-zero-error-history", + payload: { id: Schema.String }, + success: Schema.String, + error: Schema.Struct({ reason: Schema.String }), + storageLimits: { maxErrorEntries: 0 }, + idempotencyKey: ({ id }) => id, + }); + const queue = TaskQueue.make(definition.name, definition); + const offered = yield* TaskQueue.offer( + queue, + { id: "failed" }, + { onFailurePolicy: "mark-as-failure" }, + ); + yield* TaskQueue.complete(queue, () => + Effect.fail({ reason: "terminal" }), + ); + + const error = yield* TaskQueue.wait(queue, offered.handle).pipe( + Effect.flip, + ); + expect(error).toMatchObject({ + _tag: "TaskFailed", + failure: { reason: "terminal" }, + }); + }), + ); + + it.effect("wait rejects a handle from another queue", () => + Effect.gen(function* () { + const queue = makeQueue("tq-handle-owner"); + const otherQueue = TaskQueue.make("tq-handle-other", queue.task); + const offered = yield* TaskQueue.offer(queue, { + userId: "owner", + amount: 1, + }); + + const error = yield* TaskQueue.wait(otherQueue, offered.handle).pipe( + Effect.flip, + ); + expect(error).toMatchObject({ + _tag: "TaskHandleMismatch", + expectedQueue: otherQueue.name, + actualQueue: queue.name, + }); + }), + ); + it.effect("maxRetries cap of 0 skips retries even with a schedule", () => Effect.gen(function* () { // cap at 0 → the first failure can't retry (0 errors is not < 0) diff --git a/src/TaskQueue.ts b/src/TaskQueue.ts index 2aeaf6a..5d55284 100644 --- a/src/TaskQueue.ts +++ b/src/TaskQueue.ts @@ -6,11 +6,14 @@ * * @module */ + +import * as Cause from "effect/Cause"; import * as Clock from "effect/Clock"; import type * as Crypto from "effect/Crypto"; import * as Data from "effect/Data"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; import * as Function from "effect/Function"; import * as Result from "effect/Result"; @@ -184,6 +187,13 @@ export interface TaskOptions { onDuplicate?: "return-existing" | "new-generation"; } +/** Indicates that per-offer timing or retry overrides are invalid. */ +export class TaskOptionsError extends Data.TaggedError("TaskOptionsError")<{ + readonly field: "delay" | "maxRetries" | "maxStalledCount"; + readonly constraint: string; + readonly actual: unknown; +}> {} + /** * Redis connection loss made it impossible to determine whether an offer was * committed. Retry with the same queue payload/idempotency identity; the @@ -213,6 +223,22 @@ export class RetentionContextRequired extends Data.TaggedError( readonly taskId: string; }> {} +/** A task retry schedule failed while deciding whether to run another attempt. */ +export class RetryPolicyError extends Data.TaggedError("RetryPolicyError")<{ + readonly queue: string; + readonly taskId: string; + readonly cause: unknown; +}> {} + +/** Indicates that lease supervision options cannot produce a safe attempt. */ +export class ProcessingConfigurationError extends Data.TaggedError( + "ProcessingConfigurationError", +)<{ + readonly field: keyof ProcessingOptions; + readonly constraint: string; + readonly actual: unknown; +}> {} + declare const TaskHandleSuccess: unique symbol; declare const TaskHandleError: unique symbol; @@ -285,10 +311,19 @@ export class CallerTimeout extends Data.TaggedError("CallerTimeout")<{ readonly timeout: Duration.Input; }> {} +/** The supplied queue descriptor does not match a persisted task handle. */ +export class TaskHandleMismatch extends Data.TaggedError("TaskHandleMismatch")<{ + readonly expectedQueue: string; + readonly actualQueue: string; + readonly expectedTaskName: string; + readonly actualTaskName: string; +}> {} + /** Recoverable failures produced while offering a task generation. */ export type OfferError = | IndeterminateWriteError | RetentionContextRequired + | TaskOptionsError | Task.TaskIdentityGenerationError | StorageProtocol.StorageProtocolError | TaskEngine.TaskEngineError @@ -311,10 +346,14 @@ export type OfferRequirements< /** Infrastructure and codec failures produced while completing an attempt. */ export type CompleteError = | StorageProtocol.StorageProtocolError + | RetryPolicyError | TaskEngine.LeaseLost | TaskEngine.TaskEngineError | Schema.SchemaError; +/** Completion failures plus invalid direct `completeOne` processing options. */ +export type CompleteOneError = CompleteError | ProcessingConfigurationError; + /** Services required by completion, including handler and retry environments. */ export type CompleteRequirements< Payload extends Schema.Top, @@ -339,7 +378,9 @@ export type WaitError = | TaskNotFound | ResultExpired | CallerTimeout + | TaskHandleMismatch | TaskEngine.CursorExpired + | TaskEngine.InvalidCursor | TaskEngine.TaskEngineError | StorageProtocol.StorageProtocolError | Schema.SchemaError; @@ -386,6 +427,14 @@ export type OfferOutcome< readonly task: Task.Task; readonly handle: TaskHandle; }; + +const hasBuiltInErrorTag = (value: unknown): boolean => + typeof value === "object" && + value !== null && + "_tag" in value && + Object.values(StorageProtocol.builtInErrorTags).some( + (tag) => tag === value._tag, + ); /** * Enqueues a typed payload and returns its exact generation handle. * @@ -444,6 +493,28 @@ export const offer = Effect.fnUntraced(function* < OfferRequirements > { yield* TaskInvariant.validate(queue.task); + const delay = options?.delay ?? 0; + if ( + !Number.isFinite(delay) || + Math.abs(delay) > Number.MAX_SAFE_INTEGER || + delay < 0 + ) { + return yield* new TaskOptionsError({ + field: "delay", + constraint: "a finite safe number greater than or equal to 0", + actual: options?.delay, + }); + } + for (const field of ["maxRetries", "maxStalledCount"] as const) { + const actual = options?.[field]; + if (actual !== undefined && (!Number.isSafeInteger(actual) || actual < 0)) { + return yield* new TaskOptionsError({ + field, + constraint: "a non-negative safe integer", + actual, + }); + } + } const encodePayload = Schema.encodeEffect(queue.task.payloadSchema); const id = options?.taskId ?? (yield* queue.task.idempotencyKey(payload)); const engine = yield* TaskEngine.TaskEngine; @@ -470,7 +541,7 @@ export const offer = Effect.fnUntraced(function* < queue.task.storageLimits, ), schemaId: queue.task.schemaId, - delay: options?.delay ?? 0, + delay, maxRetries: options?.maxRetries ?? -1, maxStalledCount: options?.maxStalledCount ?? 1, maxErrorEntries: queue.task.storageLimits.maxErrorEntries, @@ -507,6 +578,7 @@ export const offer = Effect.fnUntraced(function* < }), ); case "IndeterminateCommit": + case "InvalidReply": return Effect.fail( new IndeterminateWriteError({ cause, @@ -516,7 +588,7 @@ export const offer = Effect.fnUntraced(function* < ); case "TransportFailure": case "ScriptFailure": - case "InvalidReply": + case "InvalidInput": case "LeaseLost": return Effect.fail(cause); } @@ -626,25 +698,63 @@ const fail = Effect.fnUntraced(function* < : queue.task.maxRetries; const failedAt = new Date(yield* Clock.currentTimeMillis); - const retryAt = - queue.task.retrySchedule && attempt.task.handlerFailureCount < maxRetries - ? yield* nextRunAt( - queue.task.retrySchedule, - new Date(attempt.task.createdAt.getTime() + attempt.task.delay), - [...attempt.task.errors, { timestamp: failedAt, error: failure }], - ) - : undefined; + const encodedFailure = yield* StorageProtocol.encodeValue( + queue.task.schemaId, + "failure", + yield* encodeFailure(failure), + queue.task.storageLimits, + ); + const canRetry = + queue.task.retrySchedule !== undefined && + attempt.task.handlerFailureCount < maxRetries && + !hasBuiltInErrorTag(failure); + const handlerErrors = attempt.task.errors.filter( + ({ error }) => !hasBuiltInErrorTag(error), + ); + if (canRetry && handlerErrors.length < attempt.task.handlerFailureCount) { + yield* engine.writeError( + queue.name, + attempt.task.id, + attempt.leaseToken, + encodedFailure, + ); + return yield* new RetryPolicyError({ + queue: queue.name, + taskId: attempt.task.id, + cause: new Error( + "Retry schedule history was truncated before it could be replayed safely", + ), + }); + } + const retryDecision = canRetry + ? yield* nextRunAt( + queue.task.retrySchedule, + new Date(attempt.task.createdAt.getTime() + attempt.task.delay), + [...handlerErrors, { timestamp: failedAt, error: failure }], + ).pipe(Effect.exit) + : Exit.succeed(undefined); + if (Exit.isFailure(retryDecision)) { + if (Cause.hasInterruptsOnly(retryDecision.cause)) { + return yield* Effect.interrupt; + } + yield* engine.writeError( + queue.name, + attempt.task.id, + attempt.leaseToken, + encodedFailure, + ); + return yield* new RetryPolicyError({ + queue: queue.name, + taskId: attempt.task.id, + cause: Cause.squash(retryDecision.cause), + }); + } return yield* engine.writeError( queue.name, attempt.task.id, attempt.leaseToken, - yield* StorageProtocol.encodeValue( - queue.task.schemaId, - "failure", - yield* encodeFailure(failure), - queue.task.storageLimits, - ), - retryAt, + encodedFailure, + retryDecision.value, ); }); @@ -685,6 +795,94 @@ export interface ProcessingOptions { readonly heartbeatRetryCount?: number; } +interface ResolvedProcessingOptions { + readonly lockTimeout: number; + readonly lockRefresh: number; + readonly heartbeatRetryDelay: number; + readonly heartbeatRetryCount: number; +} + +const defaultProcessingOptions: ResolvedProcessingOptions = { + lockTimeout: 30_000, + lockRefresh: 10_000, + heartbeatRetryDelay: 250, + heartbeatRetryCount: 3, +}; + +const processingDuration = ( + field: keyof ProcessingOptions, + input: Duration.Input, + requireWholeMilliseconds = false, +) => + Effect.try({ + try: () => Duration.toMillis(input), + catch: () => + new ProcessingConfigurationError({ + field, + constraint: "a valid duration greater than zero", + actual: input, + }), + }).pipe( + Effect.filterOrFail( + (value) => + Number.isFinite(value) && + value > 0 && + (!requireWholeMilliseconds || Number.isSafeInteger(value)), + () => + new ProcessingConfigurationError({ + field, + constraint: requireWholeMilliseconds + ? "a positive safe-integer number of milliseconds" + : "a finite duration greater than zero", + actual: input, + }), + ), + ); + +const resolveProcessingOptions = Effect.fnUntraced(function* ( + options?: ProcessingOptions, +): Effect.fn.Return { + const lockTimeout = yield* processingDuration( + "lockTimeout", + options?.lockTimeout ?? Duration.seconds(30), + true, + ); + const lockRefresh = yield* processingDuration( + "lockRefresh", + options?.lockRefresh ?? Duration.seconds(10), + true, + ); + if (lockRefresh >= lockTimeout) { + return yield* new ProcessingConfigurationError({ + field: "lockRefresh", + constraint: "a duration shorter than lockTimeout", + actual: options?.lockRefresh ?? Duration.seconds(10), + }); + } + const heartbeatRetryDelay = yield* processingDuration( + "heartbeatRetryDelay", + options?.heartbeatRetryDelay ?? Duration.millis(250), + ); + const configuredRetryCount = options?.heartbeatRetryCount ?? 3; + if (!Number.isSafeInteger(configuredRetryCount) || configuredRetryCount < 0) { + return yield* new ProcessingConfigurationError({ + field: "heartbeatRetryCount", + constraint: "a non-negative safe integer", + actual: configuredRetryCount, + }); + } + const safetyWindow = lockTimeout - lockRefresh; + return { + lockTimeout, + lockRefresh, + heartbeatRetryDelay, + heartbeatRetryCount: Math.min( + configuredRetryCount, + Math.floor(safetyWindow / heartbeatRetryDelay), + ), + }; +}); + const processAttempt = Effect.fnUntraced(function* < Payload extends Schema.Top, Success extends Schema.Top, @@ -696,29 +894,16 @@ const processAttempt = Effect.fnUntraced(function* < self: TaskQueue, attempt: TaskAttempt, handler: TaskHandler, - options?: ProcessingOptions, + options: ResolvedProcessingOptions, ) { - const lockTimeout = Duration.toMillis( - options?.lockTimeout ?? Duration.seconds(30), - ); - const lockRefresh = Duration.toMillis( - options?.lockRefresh ?? Duration.seconds(10), - ); - const retryDelay = Math.max( - 1, - Duration.toMillis(options?.heartbeatRetryDelay ?? Duration.millis(250)), - ); - const safetyWindow = Math.max(0, lockTimeout - lockRefresh); - const heartbeatRetryCount = Math.min( - Math.max(0, options?.heartbeatRetryCount ?? 3), - Math.floor(safetyWindow / retryDelay), - ); + const { lockTimeout, lockRefresh, heartbeatRetryDelay, heartbeatRetryCount } = + options; const renew = extendLock(self, attempt, lockTimeout).pipe( Effect.retry({ while: (error) => error._tag === "TaskEngineError", times: heartbeatRetryCount, - schedule: Schedule.spaced(retryDelay), + schedule: Schedule.spaced(heartbeatRetryDelay), }), ); const heartBeat = renew.pipe( @@ -751,7 +936,8 @@ const processAttempt = Effect.fnUntraced(function* < * This operation polls until work is available, then returns the processed task * identifier. It is available in data-first and data-last forms. Handler * failures are recorded on the task and may schedule a retry; only decoding, - * storage, Redis, or ownership failures remain in the Effect failure channel. + * storage, Redis, ownership, or retry-policy evaluation failures remain in the + * Effect failure channel. * * **Gotchas** * @@ -832,8 +1018,11 @@ export const complete: { CompleteRequirements > { yield* TaskInvariant.validate(self.task); - const attempt = yield* takeUnsafe(self); - return yield* processAttempt(self, attempt, handler); + const options = defaultProcessingOptions; + const attempt = yield* takeUnsafe(self, { + lockTimeout: options.lockTimeout, + }); + return yield* processAttempt(self, attempt, handler, options); }), ); @@ -857,16 +1046,17 @@ export const completeOne = Effect.fnUntraced(function* < options?: ProcessingOptions, ): Effect.fn.Return< boolean, - CompleteError, + CompleteOneError, CompleteRequirements > { yield* TaskInvariant.validate(self.task); + const resolved = yield* resolveProcessingOptions(options); const attempt = yield* takeAvailable(self, { poll: false, - lockTimeout: options?.lockTimeout, + lockTimeout: resolved.lockTimeout, }); if (attempt === null) return false; - yield* processAttempt(self, attempt, handler, options); + yield* processAttempt(self, attempt, handler, resolved); return true; }); @@ -875,9 +1065,9 @@ export const completeOne = Effect.fnUntraced(function* < * * Wraps the engine's raw event stream and decodes each event's task-shaped * payload: `task.created`/`task.updated` yield typed {@link Task.Task}s, - * `task.failed` yields a typed error, and `task.completed` yields a typed - * success value. `cursor` resumes from a prior event id (defaults to now, so - * only future events are delivered). + * `task.failed` yields the task's typed error or a built-in stalled/canceled + * error, and `task.completed` yields a typed success value. `cursor` resumes + * from a prior event id (defaults to now, so only future events are delivered). * * **Gotchas** * @@ -948,16 +1138,11 @@ const streamEffect = Effect.fnUntraced(function* < }; } if (event._tag === "task.failed") { - const decodeError = Schema.decodeEffect(queue.task.errorSchema); + const decodeError = Schema.decodeEffect( + Schema.Union([TaskErrorSchema, queue.task.errorSchema]), + ); const rawFailure = event.payload.error; - const builtIn = - typeof rawFailure === "object" && - rawFailure !== null && - "_tag" in rawFailure && - Object.values(StorageProtocol.builtInErrorTags).includes( - rawFailure._tag as never, - ); - const failure = builtIn + const failure = hasBuiltInErrorTag(rawFailure) ? rawFailure : yield* StorageProtocol.decodeValue( rawFailure, @@ -968,7 +1153,7 @@ const streamEffect = Effect.fnUntraced(function* < ...event, payload: { ...event.payload, - error: builtIn ? failure : yield* decodeError(failure), + error: yield* decodeError(failure), }, }; } @@ -1035,6 +1220,14 @@ export const wait = Effect.fnUntraced(function* < WaitRequirements > { const operation = Effect.gen(function* () { + if (handle.queue !== queue.name || handle.taskName !== queue.task.name) { + return yield* new TaskHandleMismatch({ + expectedQueue: queue.name, + actualQueue: handle.queue, + expectedTaskName: queue.task.name, + actualTaskName: handle.taskName, + }); + } if (handle.protocolVersion !== StorageProtocol.protocolVersion) { return yield* new StorageProtocol.UnsupportedProtocolVersion({ encountered: handle.protocolVersion, @@ -1151,6 +1344,12 @@ export const wait = Effect.fnUntraced(function* < } const lastFailure = task.errors.at(-1)?.error; if (lastFailure === undefined) { + const result = yield* engine.getResult( + handle.queue, + handle.taskId, + handle.generation, + ); + if (result !== null) return yield* decodeTerminalResult(result); return yield* new StorageProtocol.CorruptStorageValue({ message: "Terminal failure has no error entry", }); @@ -1169,7 +1368,8 @@ export const wait = Effect.fnUntraced(function* < (event) => event.taskId === handle.taskId && event.generation === handle.generation && - (event._tag === "task.completed" || event._tag === "task.failed"), + (event._tag === "task.completed" || + (event._tag === "task.failed" && event.payload.terminal)), ), Stream.take(1), Stream.runCollect, diff --git a/src/Worker.test.ts b/src/Worker.test.ts index 67029bd..d875a0b 100644 --- a/src/Worker.test.ts +++ b/src/Worker.test.ts @@ -55,6 +55,63 @@ layer(TestLayer, { excludeTestServices: true, timeout: "60 seconds" })( }), ); + it.effect("rejects non-finite and out-of-range concurrency", () => + Effect.gen(function* () { + const queue = makeQueue("worker-invalid-concurrency"); + for (const concurrency of [ + Number.NaN, + Number.POSITIVE_INFINITY, + 0, + 1.5, + ]) { + const error = yield* Worker.run( + Worker.make(queue, () => Effect.succeed("unused"), { + concurrency, + }), + ).pipe(Effect.flip); + expect(error).toMatchObject({ + _tag: "WorkerConfigurationError", + field: "concurrency", + actual: concurrency, + }); + } + }), + ); + + it.effect("rejects unsafe timing and processing options", () => + Effect.gen(function* () { + const queue = makeQueue("worker-invalid-timing"); + const cases: ReadonlyArray = [ + [{ pollInterval: 0 }, "pollInterval"], + [{ maintenanceInterval: 0 }, "maintenanceInterval"], + [{ drainTimeout: -1 }, "drainTimeout"], + [{ processing: { lockTimeout: 0 } }, "processing.lockTimeout"], + [{ processing: { lockTimeout: 0.5 } }, "processing.lockTimeout"], + [ + { processing: { lockTimeout: 100, lockRefresh: 100 } }, + "processing.lockRefresh", + ], + [ + { processing: { heartbeatRetryDelay: 0 } }, + "processing.heartbeatRetryDelay", + ], + [ + { processing: { heartbeatRetryCount: 1.5 } }, + "processing.heartbeatRetryCount", + ], + ]; + for (const [options, field] of cases) { + const error = yield* Worker.run( + Worker.make(queue, () => Effect.succeed("unused"), options), + ).pipe(Effect.flip); + expect(error).toMatchObject({ + _tag: "WorkerConfigurationError", + field, + }); + } + }), + ); + it.effect("uses isolated worker and maintenance Redis roles", () => Effect.gen(function* () { const engine = yield* TaskEngine.TaskEngine; diff --git a/src/Worker.ts b/src/Worker.ts index 0f247ac..de15815 100644 --- a/src/Worker.ts +++ b/src/Worker.ts @@ -27,7 +27,7 @@ const TypeId = "~effectmq/Worker" as const; * @since 0.3.0 */ export interface WorkerOptions { - /** Number of independent acquire/process loops. Defaults to `1`. */ + /** Number of independent acquire/process loops (1-1000). Defaults to `1`. */ readonly concurrency?: number; /** Delay after an empty acquisition. Defaults to one second. */ readonly pollInterval?: Duration.Input; @@ -114,19 +114,152 @@ export const make = < class WorkerSlotStopped extends Data.TaggedError("WorkerSlotStopped") {} +/** Indicates that a worker option cannot produce a bounded runtime. */ +export class WorkerConfigurationError extends Data.TaggedError( + "WorkerConfigurationError", +)<{ + readonly field: + | keyof Omit + | `processing.${keyof TaskQueue.ProcessingOptions}`; + readonly constraint: string; + readonly actual: unknown; +}> {} + +interface ResolvedWorkerOptions { + readonly concurrency: number; + readonly pollInterval: number; + readonly maintenanceInterval: number; + readonly drainTimeout: number; + readonly processing: TaskQueue.ProcessingOptions; +} + +const invalidWorkerOption = ( + field: WorkerConfigurationError["field"], + constraint: string, + actual: unknown, +) => new WorkerConfigurationError({ field, constraint, actual }); + +const workerDuration = Effect.fnUntraced(function* ( + field: WorkerConfigurationError["field"], + input: Duration.Input, + allowZero: boolean, + requireWholeMilliseconds = false, +) { + const value = yield* Effect.try({ + try: () => Duration.toMillis(input), + catch: () => invalidWorkerOption(field, "a valid finite duration", input), + }); + if ( + !Number.isFinite(value) || + value < 0 || + (!allowZero && value === 0) || + (requireWholeMilliseconds && !Number.isSafeInteger(value)) + ) { + return yield* invalidWorkerOption( + field, + requireWholeMilliseconds + ? "a positive safe-integer number of milliseconds" + : allowZero + ? "a finite non-negative duration" + : "a finite duration greater than zero", + input, + ); + } + return value; +}); + +const resolveWorkerOptions = Effect.fnUntraced(function* ( + options: WorkerOptions, +): Effect.fn.Return { + const concurrency = options.concurrency ?? 1; + if ( + !Number.isSafeInteger(concurrency) || + concurrency < 1 || + concurrency > 1_000 + ) { + return yield* invalidWorkerOption( + "concurrency", + "a safe integer between 1 and 1000", + concurrency, + ); + } + const pollInterval = yield* workerDuration( + "pollInterval", + options.pollInterval ?? Duration.seconds(1), + false, + ); + const maintenanceInterval = yield* workerDuration( + "maintenanceInterval", + options.maintenanceInterval ?? Duration.seconds(1), + false, + ); + const drainTimeout = yield* workerDuration( + "drainTimeout", + options.drainTimeout ?? Duration.seconds(30), + true, + ); + const processing = options.processing ?? {}; + const lockTimeout = yield* workerDuration( + "processing.lockTimeout", + processing.lockTimeout ?? Duration.seconds(30), + false, + true, + ); + const lockRefresh = yield* workerDuration( + "processing.lockRefresh", + processing.lockRefresh ?? Duration.seconds(10), + false, + true, + ); + if (lockRefresh >= lockTimeout) { + return yield* invalidWorkerOption( + "processing.lockRefresh", + "a duration shorter than processing.lockTimeout", + processing.lockRefresh ?? Duration.seconds(10), + ); + } + const heartbeatRetryDelay = yield* workerDuration( + "processing.heartbeatRetryDelay", + processing.heartbeatRetryDelay ?? Duration.millis(250), + false, + ); + const heartbeatRetryCount = processing.heartbeatRetryCount ?? 3; + if (!Number.isSafeInteger(heartbeatRetryCount) || heartbeatRetryCount < 0) { + return yield* invalidWorkerOption( + "processing.heartbeatRetryCount", + "a non-negative safe integer", + heartbeatRetryCount, + ); + } + return { + concurrency, + pollInterval, + maintenanceInterval, + drainTimeout, + processing: { + lockTimeout, + lockRefresh, + heartbeatRetryDelay, + heartbeatRetryCount, + }, + }; +}); + /** * Runs a worker until interrupted. * * Independent acquisition fibers use the worker Redis role while a maintenance * fiber uses the maintenance role. Interruption stops new acquisitions, keeps * heartbeats alive while handlers drain, then interrupts any remainder after - * `drainTimeout`. + * `drainTimeout`. Invalid concurrency, timing, or processing configuration fails with + * {@link WorkerConfigurationError} before any worker fibers start. * * **Gotchas** * * Queue handlers have at-least-once delivery and must make externally visible * effects idempotent. Attempt and maintenance failures are logged and the loops - * continue, so this long-running Effect has `never` in its failure channel. + * continue. After configuration validation, operational failures are observed + * and the long-running worker does not fail. * * @category Operations * @since 0.3.0 @@ -142,7 +275,7 @@ export const run = Effect.fnUntraced(function* < worker: Worker, ): Effect.fn.Return< never, - never, + WorkerConfigurationError, | RedisConnectionRoles | Crypto.Crypto | QueueR @@ -152,6 +285,7 @@ export const run = Effect.fnUntraced(function* < | Error["EncodingServices"] > { yield* TaskInvariant.validate(worker.queue.task); + const options = yield* resolveWorkerOptions(worker.options); return yield* Effect.scoped( Effect.gen(function* () { const roles = yield* RedisConnectionRoles; @@ -163,15 +297,8 @@ export const run = Effect.fnUntraced(function* < ).pipe(Effect.orDie); const accepting = yield* Ref.make(true); const slots = yield* FiberSet.make(); - const pollInterval = worker.options.pollInterval ?? Duration.seconds(1); - const maintenanceInterval = - worker.options.maintenanceInterval ?? Duration.seconds(1); - const drainTimeout = worker.options.drainTimeout ?? Duration.seconds(30); - const concurrency = Math.max( - 1, - Math.floor(worker.options.concurrency ?? 1), - ); - + const { concurrency, drainTimeout, maintenanceInterval, pollInterval } = + options; // Registered after FiberSet.make, so this LIFO finalizer drains before // the set's own finalizer interrupts any remaining handlers. yield* Effect.addFinalizer(() => @@ -189,7 +316,7 @@ export const run = Effect.fnUntraced(function* < const processed = yield* TaskQueue.completeOne( worker.queue, worker.handler, - worker.options.processing, + options.processing, ).pipe( Effect.provideService(TaskEngine.TaskEngine, workerEngine), Effect.matchEffect({ diff --git a/src/lua/taskEngine.lua b/src/lua/taskEngine.lua index ba8cda0..7553918 100644 --- a/src/lua/taskEngine.lua +++ b/src/lua/taskEngine.lua @@ -14,6 +14,7 @@ local maintenanceRemaining = 100 -- set once per script invocation; helpers close over it local now = 0 +local debugMode = false local function redisNow() local time = redis.call("TIME") @@ -27,6 +28,11 @@ local function getNow(debug) return redisNow() end +local function isPositiveSafeInteger(value) + return value and value == value and value > 0 + and value <= 9007199254740991 and math.floor(value) == value +end + -- key helpers --------------------------------------------------------------- local function scheduleHash(prefix, name) return prefix .. ":schedule:" .. name end @@ -62,7 +68,9 @@ local function maintenanceCursorKey(prefix) return prefix .. ":maintenance:curso local function removeFromFailedList(prefix, id) return redis.call("ZREM", failedList(prefix), id) end local function removeFromSuccessList(prefix, id) return redis.call("ZREM", successList(prefix), id) end -local function removeFromWaitList(prefix, id) return redis.call("LREM", waitList(prefix), 0, id) end +local function removeFromWaitList(prefix, id, count) + return redis.call("LREM", waitList(prefix), count, id) +end local function removeFromDelayedList(prefix, id) return redis.call("ZREM", delayedList(prefix), id) end local function removeFromActiveLists(prefix, id) return redis.call("ZREM", activeList(prefix), id) end @@ -114,21 +122,48 @@ local function eventCursors(prefix) end local function removeFromCurrentLists(prefix, id) + local storedList = redis.call("HGET", taskHash(prefix, id), "currentList") local currentList = nil - if removeFromWaitList(prefix, id) > 0 then - currentList = currentList or "wait" - end - if removeFromDelayedList(prefix, id) > 0 then - currentList = currentList or "scheduled" - end - if removeFromActiveLists(prefix, id) > 0 then - currentList = currentList or "active" - end - if removeFromFailedList(prefix, id) > 0 then - currentList = currentList or "failed" - end - if removeFromSuccessList(prefix, id) > 0 then - currentList = currentList or "success" + local storedRemoval = 0 + if storedList == "wait" then + storedRemoval = removeFromWaitList(prefix, id, 1) + currentList = "wait" + elseif storedList == "scheduled" then + storedRemoval = removeFromDelayedList(prefix, id) + currentList = "scheduled" + elseif storedList == "active" then + storedRemoval = removeFromActiveLists(prefix, id) + currentList = "active" + elseif storedList == "failed" then + storedRemoval = removeFromFailedList(prefix, id) + currentList = "failed" + elseif storedList == "success" then + storedRemoval = removeFromSuccessList(prefix, id) + currentList = "success" + end + + -- Records created before currentList was introduced take one migration scan. + -- A stale marker from an older rolling-deployment writer also falls back. + -- Debug mode repairs deliberately injected cross-index corruption. + if storedList == false + or (storedList ~= "none" and storedRemoval == 0) + or debugMode then + if storedRemoval == 0 then currentList = nil end + if removeFromDelayedList(prefix, id) > 0 then + currentList = currentList or "scheduled" + end + if removeFromActiveLists(prefix, id) > 0 then + currentList = currentList or "active" + end + if removeFromFailedList(prefix, id) > 0 then + currentList = currentList or "failed" + end + if removeFromSuccessList(prefix, id) > 0 then + currentList = currentList or "success" + end + if removeFromWaitList(prefix, id, 0) > 0 then + currentList = currentList or "wait" + end end return currentList end @@ -167,8 +202,9 @@ end -- the add* helpers do not clear other lists; moveToList is the single entry -- point that removes from the current list, adds to the target, and emits task.moved -local function moveToList(prefix, id, list, readyAt) - local currentList = removeFromCurrentLists(prefix, id) +local function moveToList(prefix, id, list, readyAt, sourceKnownEmpty) + local currentList = nil + if not sourceKnownEmpty then currentList = removeFromCurrentLists(prefix, id) end if list == "wait" then addToWaitList(prefix, id) elseif list == "scheduled" then @@ -180,6 +216,11 @@ local function moveToList(prefix, id, list, readyAt) elseif list == "success" then addToSuccessList(prefix, id) end + if list then + redis.call("HSET", taskHash(prefix, id), "currentList", list) + else + redis.call("HSET", taskHash(prefix, id), "currentList", "none") + end -- Removing first repairs duplicate or cross-state membership. Re-add the -- target even when it is unchanged, then avoid publishing a fake move. @@ -215,9 +256,9 @@ local function moveToList(prefix, id, list, readyAt) publishEvent(prefix, id, "task.moved", fields) end -local function deleteTask(prefix, id) +local function deleteTask(prefix, id, sourceKnownEmpty) local generation = redis.call("HGET", taskHash(prefix, id), "generation") - moveToList(prefix, id, nil) + moveToList(prefix, id, nil, nil, sourceKnownEmpty) if generation then local member = cmsgpack.pack({ prefix, id, tonumber(generation) }) redis.call("ZREM", taskExpiryIndex(prefix), member) @@ -323,7 +364,7 @@ local function scheduleExpiry(index, member, retentionMs) redis.call("ZADD", index, now + tonumber(retentionMs), member) end -local function persistTerminalResult(prefix, id) +local function persistTerminalResult(prefix, id, terminalFailure) local generation = currentGeneration(prefix, id) local member = identityMember(prefix, id, generation) local outcome = getTaskField(prefix, id, "outcome") @@ -341,7 +382,10 @@ local function persistTerminalResult(prefix, id) redis.call("HSET", hash, "success", getTaskField(prefix, id, "success")) else local errors = getTaskErrors(prefix, id) - local failure = errors[#errors] and errors[#errors].error or nil + local failure = terminalFailure + if failure == nil then + failure = errors[#errors] and errors[#errors].error or nil + end if failure ~= nil then redis.call("HSET", hash, "failure", cmsgpack.pack(failure)) end redis.call("ZADD", deadLetterList(prefix), now, member) scheduleExpiry( @@ -443,7 +487,7 @@ local function applyCompletionPolicy(prefix, id) end if policy == "delete" and not hasRetentionHolds(prefix, id, generation) then - deleteTask(prefix, id) + deleteTask(prefix, id, true) end end @@ -481,9 +525,9 @@ local function releaseOwnedHolds(holderPrefix, holderId, holderGeneration) end end -local function settleTask(prefix, id) +local function settleTask(prefix, id, terminalFailure) local generation = currentGeneration(prefix, id) - persistTerminalResult(prefix, id) + persistTerminalResult(prefix, id, terminalFailure) scheduleTerminalRetention(prefix, id) applyCompletionPolicy(prefix, id) releaseOwnedHolds(prefix, id, generation) @@ -528,7 +572,7 @@ local function failTask(prefix, id, error, retryAt, failureKind) return end setTask(prefix, id, "outcome", "failure") - settleTask(prefix, id) + settleTask(prefix, id, error) end -- sync ----------------------------------------------------------------------- @@ -632,8 +676,11 @@ local function syncTerminalExpiry(prefix) if exists(taskHash(prefix, task.id)) == 1 and currentGeneration(prefix, task.id) == task.generation then - removeFromSuccessList(prefix, task.id) - removeFromFailedList(prefix, task.id) + local removed = removeFromSuccessList(prefix, task.id) + + removeFromFailedList(prefix, task.id) + if removed > 0 then + redis.call("HSET", taskHash(prefix, task.id), "currentList", "none") + end end redis.call("ZREM", index, member) end @@ -717,7 +764,6 @@ end register("effectmq_createTask", function(args) local prefix = args[2] - syncAll(prefix) local throwOnExists = args[3] == "1" local id = args[4] @@ -731,15 +777,39 @@ register("effectmq_createTask", function(args) local retentionHolder = args[11] ~= "" and cmsgpack.unpack(args[11]) or nil local creator = args[12] local onDuplicate = args[13] or "return-existing" - local maxStalledCount = tonumber(args[14]) or 1 - local maxErrorEntries = tonumber(args[16]) or 100 - local maxRelationships = tonumber(args[17]) or 1000 - local maxEventEntries = tonumber(args[18]) or 10000 - local taskRecordRetentionMs = tonumber(args[19]) or 604800000 - local resultRetentionMs = tonumber(args[20]) or 86400000 - local terminalIndexRetentionMs = tonumber(args[21]) or 604800000 - local deadLetterRetentionMs = tonumber(args[22]) or 2592000000 - local eventRetentionMs = tonumber(args[23]) or 604800000 + local maxStalledCount = tonumber(args[14]) + local maxErrorEntries = tonumber(args[16]) + local maxRelationships = tonumber(args[17]) + local maxEventEntries = tonumber(args[18]) + local taskRecordRetentionMs = tonumber(args[19]) + local resultRetentionMs = tonumber(args[20]) + local terminalIndexRetentionMs = tonumber(args[21]) + local deadLetterRetentionMs = tonumber(args[22]) + local eventRetentionMs = tonumber(args[23]) + local maxSafeInteger = 9007199254740991 + if not delay or delay ~= delay or delay < 0 or math.abs(delay) > maxSafeInteger then + return redis.error_reply("invalid delay") + end + local integerFields = { + { "maxRetries", maxRetries, -1 }, + { "maxStalledCount", maxStalledCount, 0 }, + { "maxErrorEntries", maxErrorEntries, 0 }, + { "maxRelationships", maxRelationships, 0 }, + { "maxEventEntries", maxEventEntries, 1 }, + { "taskRecordRetentionMs", taskRecordRetentionMs, 0 }, + { "resultRetentionMs", resultRetentionMs, 0 }, + { "terminalIndexRetentionMs", terminalIndexRetentionMs, 0 }, + { "deadLetterRetentionMs", deadLetterRetentionMs, 0 }, + { "eventRetentionMs", eventRetentionMs, 0 }, + } + for _, field in ipairs(integerFields) do + local value = field[2] + if not value or value ~= value or value < field[3] + or value > maxSafeInteger or math.floor(value) ~= value then + return redis.error_reply("invalid " .. field[1]) + end + end + syncAll(prefix) local existingTask = getTask(prefix, id) local replacedTask = nil @@ -829,9 +899,9 @@ register("effectmq_createTask", function(args) publishEvent(prefix, id, replacedTask and "task.updated" or "task.created", fields) if delay > 0 then - moveToList(prefix, id, "scheduled", now + delay) + moveToList(prefix, id, "scheduled", now + delay, true) else - moveToList(prefix, id, "wait") + moveToList(prefix, id, "wait", nil, true) end return { "created", latestEventCursor(prefix), getTask(prefix, id) } @@ -888,12 +958,16 @@ end) register("effectmq_writeError", function(args) local prefix = args[2] - syncAll(prefix) local leaseToken = args[3] local id = args[4] + local retryAt = tonumber(args[6]) + if not retryAt or retryAt ~= retryAt or retryAt < -1 + or math.abs(retryAt) > 9007199254740991 then + return redis.error_reply("invalid retryAt") + end local error = cmsgpack.unpack(args[5]) - local retryAt = tonumber(args[6]) or -1 local hash = taskHash(prefix, id) + syncAll(prefix) if exists(hash) == 0 then return redis.error_reply("Task not found") @@ -952,9 +1026,12 @@ end) register("effectmq_takeTask", function(args) local prefix = args[2] - syncAll(prefix) local leaseToken = args[3] local lockTimeout = tonumber(args[4]) + if not isPositiveSafeInteger(lockTimeout) then + return redis.error_reply("invalid lockTimeout") + end + syncAll(prefix) local taskId = popWaitList(prefix) @@ -980,6 +1057,9 @@ register("effectmq_extendLock", function(args) local leaseToken = args[3] local id = args[4] local lockTimeout = tonumber(args[5]) + if not isPositiveSafeInteger(lockTimeout) then + return redis.error_reply("invalid lockTimeout") + end local lock = getLockId(prefix, id) if not lock or lock ~= leaseToken then return redis.error_reply("LEASE_LOST") @@ -1096,6 +1176,7 @@ end -- MessagePack empty-array representation and must fail if read as another type. EMPTY_LIST = string.char(0x90) now = getNow(args[1]) +debugMode = args[1] == "1" maintenanceBatchSize = tonumber(ARGV[3]) or 100 maintenanceRemaining = maintenanceBatchSize return fn(args) diff --git a/src/lua/taskEngine.ts b/src/lua/taskEngine.ts index 4a2e9d8..254a74a 100644 --- a/src/lua/taskEngine.ts +++ b/src/lua/taskEngine.ts @@ -1,2 +1,2 @@ // generated from taskEngine.lua — do not edit -export default "-- The effectmq task engine as one content-addressed Redis script. The caller\n-- invokes it through SCRIPT LOAD/EVALSHA with an operation name, the debug\n-- flag (\"1\"/\"0\"), then the operation's own arguments.\n--\n-- Structured values (payload, errors, creator, success, event\n-- payloads) travel and rest as MessagePack: packed once at the Node\n-- boundary, unpacked once at the function entry point (cmsgpack here,\n-- effect's Msgpack codec on the Node side).\n\nlocal MOCKTIME_KEY = \"$$$effectmq/debug/mocktime\"\nlocal EMPTY_LIST = nil\nlocal maintenanceBatchSize = 100\nlocal maintenanceRemaining = 100\n\n-- set once per script invocation; helpers close over it\nlocal now = 0\n\nlocal function redisNow()\n local time = redis.call(\"TIME\")\n return (1000 * tonumber(time[1])) + math.floor(tonumber(time[2]) / 1000)\nend\n\nlocal function getNow(debug)\n if debug == \"1\" then\n return tonumber(redis.call(\"GET\", MOCKTIME_KEY) or redisNow())\n end\n return redisNow()\nend\n\n-- key helpers ---------------------------------------------------------------\n\nlocal function scheduleHash(prefix, name) return prefix .. \":schedule:\" .. name end\nlocal function taskHash(prefix, id) return prefix .. \":task:\" .. id end\nlocal function generationHash(prefix) return prefix .. \":generations\" end\nlocal function createdList(prefix) return prefix .. \":created\" end\nlocal function delayedList(prefix) return prefix .. \":scheduled\" end\nlocal function waitList(prefix) return prefix .. \":wait\" end\nlocal function successList(prefix) return prefix .. \":success\" end\nlocal function failedList(prefix) return prefix .. \":failed\" end\nlocal function activeList(prefix) return prefix .. \":active\" end\nlocal function eventStream(prefix) return prefix .. \":events\" end\nlocal function eventMetadata(prefix) return prefix .. \":events:metadata\" end\nlocal function lockHash(prefix, id) return prefix .. \":lock:\" .. id end\nlocal function resultHash(prefix, id, generation)\n return prefix .. \":result:\" .. id .. \":\" .. generation\nend\nlocal function taskExpiryIndex(prefix) return prefix .. \":expiry:tasks\" end\nlocal function resultExpiryIndex(prefix) return prefix .. \":expiry:results\" end\nlocal function terminalExpiryIndex(prefix) return prefix .. \":expiry:terminal-indexes\" end\nlocal function deadLetterList(prefix) return prefix .. \":dead-letter\" end\nlocal function deadLetterExpiryIndex(prefix) return prefix .. \":expiry:dead-letter\" end\nlocal function retentionHoldersKey(prefix, id, generation)\n return prefix .. \":task:\" .. id .. \":\" .. generation .. \":retained-by\"\nend\nlocal function retainedTasksKey(prefix, id, generation)\n return prefix .. \":task:\" .. id .. \":\" .. generation .. \":retains\"\nend\nlocal function retentionContinuationKey(prefix) return prefix .. \":retention-release-continuations\" end\nlocal function maintenanceCursorKey(prefix) return prefix .. \":maintenance:cursor\" end\n\n-- list membership -----------------------------------------------------------\n\nlocal function removeFromFailedList(prefix, id) return redis.call(\"ZREM\", failedList(prefix), id) end\nlocal function removeFromSuccessList(prefix, id) return redis.call(\"ZREM\", successList(prefix), id) end\nlocal function removeFromWaitList(prefix, id) return redis.call(\"LREM\", waitList(prefix), 0, id) end\nlocal function removeFromDelayedList(prefix, id) return redis.call(\"ZREM\", delayedList(prefix), id) end\nlocal function removeFromActiveLists(prefix, id) return redis.call(\"ZREM\", activeList(prefix), id) end\n\n-- fields is a flat {k1, v1, k2, v2, ...} list. Event data is published as\n-- individual stream fields (not one packed document) so that raw msgpack\n-- values (payload, success) are carried as binary-safe bulk strings — nesting\n-- them inside another msgpack document would corrupt them on decode.\nlocal function publishEvent(prefix, id, eventType, fields)\n local generation = redis.call(\"HGET\", taskHash(prefix, id), \"generation\") or \"0\"\n local protocolVersion = redis.call(\"HGET\", taskHash(prefix, id), \"protocolVersion\") or \"1\"\n local schemaId = redis.call(\"HGET\", taskHash(prefix, id), \"schemaId\") or \"unknown\"\n local maxEventEntries = redis.call(\"HGET\", taskHash(prefix, id), \"maxEventEntries\") or \"10000\"\n local eventRetentionMs = tonumber(\n redis.call(\"HGET\", taskHash(prefix, id), \"eventRetentionMs\") or \"604800000\"\n )\n local args = {\n \"XADD\", eventStream(prefix), \"MAXLEN\", \"~\", maxEventEntries, \"*\",\n \"taskId\", id,\n \"generation\", generation,\n \"protocolVersion\", protocolVersion,\n \"schemaId\", schemaId,\n \"_tag\", eventType,\n }\n for i = 1, #fields do\n args[#args + 1] = fields[i]\n end\n local eventId = redis.call(unpack(args))\n local cutoff = redisNow() - eventRetentionMs\n if cutoff < 0 then cutoff = 0 end\n redis.call(\"XTRIM\", eventStream(prefix), \"MINID\", \"~\", string.format(\"%.0f-0\", cutoff))\n redis.call(\"HSETNX\", eventMetadata(prefix), \"firstEventId\", eventId)\n return eventId\nend\n\nlocal function latestEventCursor(prefix)\n local entries = redis.call(\"XREVRANGE\", eventStream(prefix), \"+\", \"-\", \"COUNT\", 1)\n return entries[1] and entries[1][1] or \"0-0\"\nend\n\nlocal function eventCursors(prefix)\n local earliest = redis.call(\"XRANGE\", eventStream(prefix), \"-\", \"+\", \"COUNT\", 1)\n local latest = redis.call(\"XREVRANGE\", eventStream(prefix), \"+\", \"-\", \"COUNT\", 1)\n local firstEventId = redis.call(\"HGET\", eventMetadata(prefix), \"firstEventId\") or \"0-0\"\n return {\n firstEventId,\n earliest[1] and earliest[1][1] or \"0-0\",\n latest[1] and latest[1][1] or \"0-0\",\n }\nend\n\nlocal function removeFromCurrentLists(prefix, id)\n local currentList = nil\n if removeFromWaitList(prefix, id) > 0 then\n currentList = currentList or \"wait\"\n end\n if removeFromDelayedList(prefix, id) > 0 then\n currentList = currentList or \"scheduled\"\n end\n if removeFromActiveLists(prefix, id) > 0 then\n currentList = currentList or \"active\"\n end\n if removeFromFailedList(prefix, id) > 0 then\n currentList = currentList or \"failed\"\n end\n if removeFromSuccessList(prefix, id) > 0 then\n currentList = currentList or \"success\"\n end\n return currentList\nend\n\nlocal function addToActiveLists(prefix, id, expiresAt)\n return redis.call(\"ZADD\", activeList(prefix), expiresAt, id)\nend\nlocal function addToWaitList(prefix, id)\n return redis.call(\"RPUSH\", waitList(prefix), id)\nend\nlocal function addToDelayedList(prefix, id, readyAt)\n return redis.call(\"ZADD\", delayedList(prefix), readyAt, id)\nend\nlocal function addToSuccessList(prefix, id)\n return redis.call(\"ZADD\", successList(prefix), now, id)\nend\nlocal function addToFailedList(prefix, id)\n return redis.call(\"ZADD\", failedList(prefix), now, id)\nend\n\nlocal function executionState(prefix, id, list)\n if list == \"wait\" then return \"waiting\" end\n if list == \"active\" then return \"leased\" end\n if list == \"success\" then return \"succeeded\" end\n if list == \"failed\" then return \"failed\" end\n if list == \"scheduled\" then\n local handlerFailures = tonumber(redis.call(\"HGET\", taskHash(prefix, id), \"handlerFailureCount\") or \"0\")\n local stalledAttempts = tonumber(redis.call(\"HGET\", taskHash(prefix, id), \"stalledAttemptCount\") or \"0\")\n return (handlerFailures + stalledAttempts) > 0 and \"retry-scheduled\" or \"delayed\"\n end\n local outcome = redis.call(\"HGET\", taskHash(prefix, id), \"outcome\")\n if outcome == \"success\" then return \"succeeded\" end\n if outcome == \"failure\" then return \"failed\" end\n return nil\nend\n\n-- the add* helpers do not clear other lists; moveToList is the single entry\n-- point that removes from the current list, adds to the target, and emits task.moved\nlocal function moveToList(prefix, id, list, readyAt)\n local currentList = removeFromCurrentLists(prefix, id)\n if list == \"wait\" then\n addToWaitList(prefix, id)\n elseif list == \"scheduled\" then\n addToDelayedList(prefix, id, readyAt)\n elseif list == \"active\" then\n addToActiveLists(prefix, id, readyAt)\n elseif list == \"failed\" then\n addToFailedList(prefix, id)\n elseif list == \"success\" then\n addToSuccessList(prefix, id)\n end\n\n -- Removing first repairs duplicate or cross-state membership. Re-add the\n -- target even when it is unchanged, then avoid publishing a fake move.\n if currentList == list then\n return\n end\n\n local fields = {}\n local previousState = executionState(prefix, id, currentList)\n local newState = executionState(prefix, id, list)\n if currentList then\n fields[#fields + 1] = \"from\"\n fields[#fields + 1] = currentList\n end\n if list then\n fields[#fields + 1] = \"to\"\n fields[#fields + 1] = list\n end\n if previousState then\n fields[#fields + 1] = \"previousState\"\n fields[#fields + 1] = previousState\n end\n if newState then\n fields[#fields + 1] = \"newState\"\n fields[#fields + 1] = newState\n end\n fields[#fields + 1] = \"attempt\"\n fields[#fields + 1] = redis.call(\"HGET\", taskHash(prefix, id), \"attempt\") or \"0\"\n fields[#fields + 1] = \"handlerFailureCount\"\n fields[#fields + 1] = redis.call(\"HGET\", taskHash(prefix, id), \"handlerFailureCount\") or \"0\"\n fields[#fields + 1] = \"stalledAttemptCount\"\n fields[#fields + 1] = redis.call(\"HGET\", taskHash(prefix, id), \"stalledAttemptCount\") or \"0\"\n publishEvent(prefix, id, \"task.moved\", fields)\nend\n\nlocal function deleteTask(prefix, id)\n local generation = redis.call(\"HGET\", taskHash(prefix, id), \"generation\")\n moveToList(prefix, id, nil)\n if generation then\n local member = cmsgpack.pack({ prefix, id, tonumber(generation) })\n redis.call(\"ZREM\", taskExpiryIndex(prefix), member)\n redis.call(\"ZREM\", terminalExpiryIndex(prefix), member)\n end\n redis.call(\"ZREM\", createdList(prefix), id)\n return redis.call(\"DEL\", taskHash(prefix, id))\nend\n\nlocal function popWaitList(prefix) return redis.call(\"LINDEX\", waitList(prefix), 0) end\n\nlocal function getExpiredActiveList(prefix)\n if maintenanceRemaining <= 0 then return {} end\n return redis.call(\n \"ZRANGEBYSCORE\", activeList(prefix), 0, now,\n \"LIMIT\", 0, maintenanceRemaining\n )\nend\n\n-- task hash -----------------------------------------------------------------\n\n-- flat {\"id\", id, k1, v1, ...} entry list with raw hash values: structured\n-- fields stay msgpack bytes and are decoded on the Node side only — Lua never\n-- unpacks the payload, so it round-trips byte-exact (nested nulls included)\nlocal function getTask(prefix, id)\n local fields = redis.call(\"HGETALL\", taskHash(prefix, id))\n if #fields > 0 then\n return { \"id\", id, unpack(fields) }\n end\n return nil\nend\n\nlocal function getResult(prefix, id, generation)\n local fields = redis.call(\"HGETALL\", resultHash(prefix, id, generation))\n if #fields > 0 then return fields end\n return nil\nend\n\n-- append a task's entries to an event field list, prefixing each key\nlocal function appendTaskFields(fields, keyPrefix, entries)\n for i = 1, #entries, 2 do\n fields[#fields + 1] = keyPrefix .. entries[i]\n fields[#fields + 1] = entries[i + 1]\n end\nend\n\nlocal function getTaskField(prefix, id, field) return redis.call(\"HGET\", taskHash(prefix, id), field) end\nlocal function getTaskErrors(prefix, id)\n local raw = getTaskField(prefix, id, \"errors\")\n return raw and cmsgpack.unpack(raw) or {}\nend\nlocal function setTask(prefix, id, ...) return redis.call(\"HSET\", taskHash(prefix, id), \"updatedAt\", now, ...) end\nlocal function setTaskErrors(prefix, id, errors) return setTask(prefix, id, \"errors\", cmsgpack.pack(errors)) end\nlocal function appendTaskError(prefix, id, error, retryAt)\n local errorsList = getTaskErrors(prefix, id)\n local maxErrorEntries = tonumber(getTaskField(prefix, id, \"maxErrorEntries\") or \"100\")\n while #errorsList >= maxErrorEntries and #errorsList > 0 do\n table.remove(errorsList, 1)\n end\n if maxErrorEntries > 0 then\n errorsList[#errorsList + 1] = { error = error, timestamp = now, retryAt = retryAt }\n end\n setTaskErrors(prefix, id, errorsList)\n return errorsList\nend\n\n-- locks ----------------------------------------------------------------------\n\nlocal function exists(key) return redis.call(\"EXISTS\", key) end\nlocal function lockTask(prefix, id, leaseToken, lockTimeout)\n moveToList(prefix, id, \"active\", now + lockTimeout)\n redis.call(\"HINCRBY\", taskHash(prefix, id), \"attempt\", 1)\n return redis.call(\"SET\", lockHash(prefix, id), leaseToken, \"PX\", lockTimeout)\nend\nlocal function unlockTask(prefix, id) return redis.call(\"DEL\", lockHash(prefix, id)) end\nlocal function isLocked(prefix, id) return redis.call(\"EXISTS\", lockHash(prefix, id)) > 0 end\nlocal function getLockId(prefix, id) return redis.call(\"GET\", lockHash(prefix, id)) end\nlocal function isLockedBy(prefix, id, leaseToken) return getLockId(prefix, id) == leaseToken end\n\n-- explicit result retention --------------------------------------------------\n\n-- Relationships live in generation-keyed Redis sets. The member format is a\n-- canonical msgpack tuple so SADD is idempotent without relying on map order.\nlocal function identityMember(queue, id, generation)\n return cmsgpack.pack({ queue, id, tonumber(generation) })\nend\nlocal function decodeIdentity(member)\n local identity = cmsgpack.unpack(member)\n return {\n queue = identity[1],\n id = identity[2],\n generation = tonumber(identity[3]),\n }\nend\nlocal function currentGeneration(prefix, id)\n return tonumber(getTaskField(prefix, id, \"generation\"))\nend\nlocal function hasRetentionHolds(prefix, id, generation)\n return redis.call(\"SCARD\", retentionHoldersKey(prefix, id, generation)) > 0\nend\n\nlocal function scheduleExpiry(index, member, retentionMs)\n redis.call(\"ZADD\", index, now + tonumber(retentionMs), member)\nend\n\nlocal function persistTerminalResult(prefix, id)\n local generation = currentGeneration(prefix, id)\n local member = identityMember(prefix, id, generation)\n local outcome = getTaskField(prefix, id, \"outcome\")\n local hash = resultHash(prefix, id, generation)\n redis.call(\n \"HSET\",\n hash,\n \"protocolVersion\", getTaskField(prefix, id, \"protocolVersion\"),\n \"schemaId\", getTaskField(prefix, id, \"schemaId\"),\n \"generation\", generation,\n \"outcome\", outcome,\n \"settledAt\", now\n )\n if outcome == \"success\" then\n redis.call(\"HSET\", hash, \"success\", getTaskField(prefix, id, \"success\"))\n else\n local errors = getTaskErrors(prefix, id)\n local failure = errors[#errors] and errors[#errors].error or nil\n if failure ~= nil then redis.call(\"HSET\", hash, \"failure\", cmsgpack.pack(failure)) end\n redis.call(\"ZADD\", deadLetterList(prefix), now, member)\n scheduleExpiry(\n deadLetterExpiryIndex(prefix),\n member,\n getTaskField(prefix, id, \"deadLetterRetentionMs\")\n )\n end\n scheduleExpiry(\n resultExpiryIndex(prefix),\n member,\n getTaskField(prefix, id, \"resultRetentionMs\")\n )\nend\n\nlocal function scheduleTerminalRetention(prefix, id)\n local generation = currentGeneration(prefix, id)\n local member = identityMember(prefix, id, generation)\n scheduleExpiry(\n taskExpiryIndex(prefix),\n member,\n getTaskField(prefix, id, \"taskRecordRetentionMs\")\n )\n local outcome = getTaskField(prefix, id, \"outcome\")\n local policy = outcome == \"success\"\n and getTaskField(prefix, id, \"onSuccessPolicy\")\n or getTaskField(prefix, id, \"onFailurePolicy\")\n if policy == \"mark-as-success\" or policy == \"mark-as-failure\" then\n scheduleExpiry(\n terminalExpiryIndex(prefix),\n member,\n getTaskField(prefix, id, \"terminalIndexRetentionMs\")\n )\n end\nend\nlocal function validateLiveHolder(holder)\n if not holder or not holder.queue or not holder.id or not holder.generation then\n return \"invalid retention holder\"\n end\n if exists(taskHash(holder.queue, holder.id)) == 0 then\n return \"retention holder not found\"\n end\n if currentGeneration(holder.queue, holder.id) ~= tonumber(holder.generation) then\n return \"retention holder generation does not match\"\n end\n if getTaskField(holder.queue, holder.id, \"outcome\") then\n return \"retention holder is settled\"\n end\n return nil\nend\nlocal function acquireRetentionHold(holder, retainedPrefix, retainedId, retainedGeneration)\n local holderMember = identityMember(holder.queue, holder.id, holder.generation)\n local retainedMember = identityMember(retainedPrefix, retainedId, retainedGeneration)\n local holdersKey = retentionHoldersKey(retainedPrefix, retainedId, retainedGeneration)\n local retainedKey = retainedTasksKey(holder.queue, holder.id, holder.generation)\n\n -- Replay of the same relationship remains idempotent even at the cap.\n if redis.call(\"SISMEMBER\", holdersKey, holderMember) == 1 then return nil end\n\n local holderLimit = tonumber(getTaskField(holder.queue, holder.id, \"maxRelationships\") or \"1000\")\n local retainedLimit = tonumber(getTaskField(retainedPrefix, retainedId, \"maxRelationships\") or \"1000\")\n if redis.call(\"SCARD\", retainedKey) >= holderLimit then\n return \"STORAGE_RELATIONSHIP_LIMIT holder \" .. holderLimit\n end\n if redis.call(\"SCARD\", holdersKey) >= retainedLimit then\n return \"STORAGE_RELATIONSHIP_LIMIT retained \" .. retainedLimit\n end\n redis.call(\n \"SADD\",\n holdersKey,\n holderMember\n )\n redis.call(\n \"SADD\",\n retainedKey,\n retainedMember\n )\n return nil\nend\n\n-- Settlement is visible immediately through outcome and policy-selected\n-- terminal membership. A delete policy disposes the record only after the\n-- final explicit hold is gone.\nlocal function applyCompletionPolicy(prefix, id)\n if exists(taskHash(prefix, id)) == 0 then return end\n local generation = currentGeneration(prefix, id)\n local outcome = getTaskField(prefix, id, \"outcome\")\n if not outcome then return end\n local policy = outcome == \"success\"\n and getTaskField(prefix, id, \"onSuccessPolicy\")\n or getTaskField(prefix, id, \"onFailurePolicy\")\n\n if policy == \"mark-as-success\" then\n moveToList(prefix, id, \"success\")\n elseif policy == \"mark-as-failure\" then\n moveToList(prefix, id, \"failed\")\n else\n moveToList(prefix, id, nil)\n end\n\n if policy == \"delete\" and not hasRetentionHolds(prefix, id, generation) then\n deleteTask(prefix, id)\n end\nend\n\n-- Release at most one bounded batch. If work remains, the generation identity\n-- stays in a durable per-queue continuation index that later sync calls drain.\nlocal function releaseOwnedHolds(holderPrefix, holderId, holderGeneration)\n local key = retainedTasksKey(holderPrefix, holderId, holderGeneration)\n local holderMember = identityMember(holderPrefix, holderId, holderGeneration)\n local retainedMembers = maintenanceRemaining > 0\n and redis.call(\"SPOP\", key, maintenanceRemaining)\n or {}\n maintenanceRemaining = maintenanceRemaining - #retainedMembers\n for i, retainedMember in ipairs(retainedMembers) do\n local retained = decodeIdentity(retainedMember)\n redis.call(\n \"SREM\",\n retentionHoldersKey(retained.queue, retained.id, retained.generation),\n holderMember\n )\n if exists(taskHash(retained.queue, retained.id)) == 1\n and currentGeneration(retained.queue, retained.id) == retained.generation\n and getTaskField(retained.queue, retained.id, \"outcome\")\n and not hasRetentionHolds(retained.queue, retained.id, retained.generation)\n then\n applyCompletionPolicy(retained.queue, retained.id)\n end\n end\n\n local continuation = retentionContinuationKey(holderPrefix)\n if redis.call(\"SCARD\", key) > 0 then\n redis.call(\"ZADD\", continuation, now, holderMember)\n else\n redis.call(\"DEL\", key)\n redis.call(\"ZREM\", continuation, holderMember)\n end\nend\n\nlocal function settleTask(prefix, id)\n local generation = currentGeneration(prefix, id)\n persistTerminalResult(prefix, id)\n scheduleTerminalRetention(prefix, id)\n applyCompletionPolicy(prefix, id)\n releaseOwnedHolds(prefix, id, generation)\nend\n\nlocal function failTask(prefix, id, error, retryAt, failureKind)\n unlockTask(prefix, id)\n\n if failureKind == \"stall\" then\n redis.call(\"HINCRBY\", taskHash(prefix, id), \"stalledAttemptCount\", 1)\n else\n redis.call(\"HINCRBY\", taskHash(prefix, id), \"handlerFailureCount\", 1)\n end\n\n -- retryAt arrives as -1 (or nil) when no retry is scheduled; normalize\n -- to nil so it is omitted from the stored error and the event payload\n if (retryAt or -1) < 0 then retryAt = nil end\n appendTaskError(prefix, id, error, retryAt)\n local onFailurePolicy = getTaskField(prefix, id, \"onFailurePolicy\")\n\n local errorTag = type(error) == \"table\" and error._tag or nil\n\n local willRetry = errorTag ~= \"~effectmq/Error/Canceled\" and retryAt ~= nil\n local fields = {\n \"error\", cmsgpack.pack(error),\n \"policy\", onFailurePolicy,\n \"failureKind\", failureKind,\n \"attempt\", getTaskField(prefix, id, \"attempt\") or \"0\",\n \"terminal\", willRetry and \"0\" or \"1\",\n }\n if retryAt then\n fields[#fields + 1] = \"retryAt\"\n fields[#fields + 1] = retryAt\n end\n publishEvent(prefix, id, \"task.failed\", fields)\n if willRetry then\n if retryAt > now then\n moveToList(prefix, id, \"scheduled\", retryAt)\n else\n moveToList(prefix, id, \"wait\")\n end\n return\n end\n setTask(prefix, id, \"outcome\", \"failure\")\n settleTask(prefix, id)\nend\n\n-- sync -----------------------------------------------------------------------\n\nlocal function syncLocks(prefix)\n local activeIds = getExpiredActiveList(prefix)\n maintenanceRemaining = maintenanceRemaining - #activeIds\n for i, id in ipairs(activeIds) do\n if not isLocked(prefix, id) then\n local nextStalledCount = tonumber(getTaskField(prefix, id, \"stalledAttemptCount\") or \"0\") + 1\n local maxStalledCount = tonumber(getTaskField(prefix, id, \"maxStalledCount\") or \"1\")\n local retryAt = nextStalledCount > maxStalledCount and -1 or 0\n failTask(prefix, id, {\n _tag = \"~effectmq/Error/Stalled\",\n timestamp = now,\n }, retryAt, \"stall\")\n end\n end\nend\n\nlocal function syncDelayed(prefix)\n -- we lazily move tasks from the delayed list to the wait list so we need to sync\n -- before performing any other operations\n if maintenanceRemaining <= 0 then return end\n local items = redis.call(\n \"ZRANGEBYSCORE\", delayedList(prefix), 0, now,\n \"LIMIT\", 0, maintenanceRemaining\n )\n maintenanceRemaining = maintenanceRemaining - #items\n for i, item in ipairs(items) do\n moveToList(prefix, item, \"wait\")\n end\nend\n\nlocal function syncRetentionContinuations(prefix)\n if maintenanceRemaining <= 0 then return end\n local holders = redis.call(\n \"ZRANGE\",\n retentionContinuationKey(prefix),\n 0,\n 0\n )\n for i, holderMember in ipairs(holders) do\n local holder = decodeIdentity(holderMember)\n releaseOwnedHolds(holder.queue, holder.id, holder.generation)\n end\nend\n\nlocal function dueExpiryMembers(index)\n if maintenanceRemaining <= 0 then return {} end\n return redis.call(\n \"ZRANGEBYSCORE\", index, 0, now,\n \"LIMIT\", 0, maintenanceRemaining\n )\nend\n\nlocal function syncTaskExpiry(prefix)\n local index = taskExpiryIndex(prefix)\n local members = dueExpiryMembers(index)\n maintenanceRemaining = maintenanceRemaining - #members\n for i, member in ipairs(members) do\n local task = decodeIdentity(member)\n if exists(taskHash(prefix, task.id)) == 1\n and currentGeneration(prefix, task.id) == task.generation\n and getTaskField(prefix, task.id, \"outcome\")\n then\n if hasRetentionHolds(prefix, task.id, task.generation) then\n redis.call(\"ZADD\", index, now + 1000, member)\n else\n deleteTask(prefix, task.id)\n redis.call(\"DEL\", retentionHoldersKey(prefix, task.id, task.generation))\n redis.call(\"ZREM\", index, member)\n end\n else\n redis.call(\"ZREM\", index, member)\n end\n end\nend\n\nlocal function syncResultExpiry(prefix)\n local index = resultExpiryIndex(prefix)\n local members = dueExpiryMembers(index)\n maintenanceRemaining = maintenanceRemaining - #members\n for i, member in ipairs(members) do\n local result = decodeIdentity(member)\n if hasRetentionHolds(prefix, result.id, result.generation) then\n redis.call(\"ZADD\", index, now + 1000, member)\n else\n redis.call(\"DEL\", resultHash(prefix, result.id, result.generation))\n redis.call(\"ZREM\", index, member)\n end\n end\nend\n\nlocal function syncTerminalExpiry(prefix)\n local index = terminalExpiryIndex(prefix)\n local members = dueExpiryMembers(index)\n maintenanceRemaining = maintenanceRemaining - #members\n for i, member in ipairs(members) do\n local task = decodeIdentity(member)\n if exists(taskHash(prefix, task.id)) == 1\n and currentGeneration(prefix, task.id) == task.generation\n then\n removeFromSuccessList(prefix, task.id)\n removeFromFailedList(prefix, task.id)\n end\n redis.call(\"ZREM\", index, member)\n end\nend\n\nlocal function syncDeadLetterExpiry(prefix)\n local index = deadLetterExpiryIndex(prefix)\n local members = dueExpiryMembers(index)\n maintenanceRemaining = maintenanceRemaining - #members\n for i, member in ipairs(members) do\n redis.call(\"ZREM\", deadLetterList(prefix), member)\n redis.call(\"ZREM\", index, member)\n end\nend\n\nlocal function syncAll(prefix)\n local syncers = {\n syncDelayed,\n syncLocks,\n syncRetentionContinuations,\n syncTaskExpiry,\n syncResultExpiry,\n syncTerminalExpiry,\n syncDeadLetterExpiry,\n }\n local cursor = tonumber(redis.call(\"GET\", maintenanceCursorKey(prefix)) or \"0\")\n for offset = 0, #syncers - 1 do\n if maintenanceRemaining <= 0 then break end\n local index = ((cursor + offset) % #syncers) + 1\n syncers[index](prefix)\n end\n redis.call(\"SET\", maintenanceCursorKey(prefix), (cursor + 1) % #syncers)\nend\n\nlocal function firstScore(key)\n local entry = redis.call(\"ZRANGE\", key, 0, 0, \"WITHSCORES\")\n return entry[2] and tonumber(entry[2]) or nil\nend\n\nlocal function maintenanceSnapshot(prefix)\n local depth = redis.call(\"LLEN\", waitList(prefix))\n + redis.call(\"ZCARD\", delayedList(prefix))\n + redis.call(\"ZCARD\", activeList(prefix))\n local oldestCreated = firstScore(createdList(prefix))\n local oldestAge = oldestCreated and math.max(0, now - oldestCreated) or 0\n local dueBacklog = redis.call(\"ZCOUNT\", delayedList(prefix), 0, now)\n local expiredLeases = redis.call(\"ZCOUNT\", activeList(prefix), 0, now)\n local retentionBacklog = redis.call(\"ZCOUNT\", taskExpiryIndex(prefix), 0, now)\n + redis.call(\"ZCOUNT\", resultExpiryIndex(prefix), 0, now)\n + redis.call(\"ZCOUNT\", terminalExpiryIndex(prefix), 0, now)\n + redis.call(\"ZCOUNT\", deadLetterExpiryIndex(prefix), 0, now)\n local oldestDue = nil\n local dueIndexes = {\n delayedList(prefix), activeList(prefix), taskExpiryIndex(prefix),\n resultExpiryIndex(prefix), terminalExpiryIndex(prefix),\n deadLetterExpiryIndex(prefix), retentionContinuationKey(prefix),\n }\n for i, index in ipairs(dueIndexes) do\n local score = firstScore(index)\n if score and score <= now and (not oldestDue or score < oldestDue) then\n oldestDue = score\n end\n end\n return {\n depth,\n oldestAge,\n oldestDue and math.max(0, now - oldestDue) or 0,\n dueBacklog,\n expiredLeases,\n retentionBacklog,\n maintenanceBatchSize - maintenanceRemaining,\n }\nend\n\n-- entry points ---------------------------------------------------------------\n\nlocal operations = {}\nlocal function register(name, fn)\n operations[name] = fn\nend\n\nregister(\"effectmq_createTask\", function(args)\n local prefix = args[2]\n syncAll(prefix)\n local throwOnExists = args[3] == \"1\"\n\n local id = args[4]\n local name = args[5]\n local payload = args[6]\n local delay = tonumber(args[7])\n local maxRetries = tonumber(args[8])\n local onSuccessPolicy = args[9]\n local onFailurePolicy = args[10]\n -- Relationship identities arrive with fully-qualified queue prefixes.\n local retentionHolder = args[11] ~= \"\" and cmsgpack.unpack(args[11]) or nil\n local creator = args[12]\n local onDuplicate = args[13] or \"return-existing\"\n local maxStalledCount = tonumber(args[14]) or 1\n local maxErrorEntries = tonumber(args[16]) or 100\n local maxRelationships = tonumber(args[17]) or 1000\n local maxEventEntries = tonumber(args[18]) or 10000\n local taskRecordRetentionMs = tonumber(args[19]) or 604800000\n local resultRetentionMs = tonumber(args[20]) or 86400000\n local terminalIndexRetentionMs = tonumber(args[21]) or 604800000\n local deadLetterRetentionMs = tonumber(args[22]) or 2592000000\n local eventRetentionMs = tonumber(args[23]) or 604800000\n local existingTask = getTask(prefix, id)\n local replacedTask = nil\n\n if retentionHolder then\n local holderError = validateLiveHolder(retentionHolder)\n if holderError then return redis.error_reply(holderError) end\n end\n\n if existingTask ~= nil then\n if throwOnExists then\n return redis.error_reply(\"task already exists\")\n end\n if onDuplicate == \"return-existing\" then\n if retentionHolder then\n local retentionError = acquireRetentionHold(\n retentionHolder,\n prefix,\n id,\n currentGeneration(prefix, id)\n )\n if retentionError then return redis.error_reply(retentionError) end\n end\n return { \"existing\", latestEventCursor(prefix), existingTask }\n end\n if onDuplicate ~= \"new-generation\" then\n return redis.error_reply(\"invalid duplicate mode\")\n end\n if not getTaskField(prefix, id, \"outcome\") then\n return redis.error_reply(\"cannot create a new generation for a non-terminal task\")\n end\n if hasRetentionHolds(prefix, id, currentGeneration(prefix, id)) then\n return redis.error_reply(\"cannot replace a retained task generation\")\n end\n replacedTask = existingTask\n deleteTask(prefix, id)\n unlockTask(prefix, id)\n existingTask = nil\n end\n\n local generation = redis.call(\"HINCRBY\", generationHash(prefix), id, 1)\n setTask(\n prefix, id,\n \"generation\", generation,\n \"protocolVersion\", 1,\n \"schemaId\", args[15] or name,\n \"name\", name,\n \"createdAt\", now,\n \"payload\", payload,\n \"delay\", delay,\n \"maxRetries\", maxRetries,\n \"maxStalledCount\", maxStalledCount,\n \"maxErrorEntries\", maxErrorEntries,\n \"maxRelationships\", maxRelationships,\n \"maxEventEntries\", maxEventEntries,\n \"taskRecordRetentionMs\", taskRecordRetentionMs,\n \"resultRetentionMs\", resultRetentionMs,\n \"terminalIndexRetentionMs\", terminalIndexRetentionMs,\n \"deadLetterRetentionMs\", deadLetterRetentionMs,\n \"eventRetentionMs\", eventRetentionMs,\n \"attempt\", 0,\n \"handlerFailureCount\", 0,\n \"stalledAttemptCount\", 0,\n \"onSuccessPolicy\", onSuccessPolicy,\n \"onFailurePolicy\", onFailurePolicy,\n \"errors\", EMPTY_LIST\n )\n redis.call(\"ZADD\", createdList(prefix), now, id)\n if creator ~= \"\" then\n setTask(prefix, id, \"creator\", creator)\n end\n if retentionHolder then\n local retentionError = acquireRetentionHold(retentionHolder, prefix, id, generation)\n if retentionError then\n deleteTask(prefix, id)\n if redis.call(\"HINCRBY\", generationHash(prefix), id, -1) == 0 then\n redis.call(\"HDEL\", generationHash(prefix), id)\n end\n return redis.error_reply(retentionError)\n end\n end\n local newTask = getTask(prefix, id)\n local fields = {}\n if replacedTask then appendTaskFields(fields, \"existing:\", replacedTask) end\n appendTaskFields(fields, \"new:\", newTask)\n fields[#fields + 1] = \"state\"\n fields[#fields + 1] = delay > 0 and \"delayed\" or \"waiting\"\n publishEvent(prefix, id, replacedTask and \"task.updated\" or \"task.created\", fields)\n\n if delay > 0 then\n moveToList(prefix, id, \"scheduled\", now + delay)\n else\n moveToList(prefix, id, \"wait\")\n end\n\n return { \"created\", latestEventCursor(prefix), getTask(prefix, id) }\nend)\n\nregister(\"effectmq_getTask\", function(args)\n local prefix = args[2]\n local id = args[3]\n return getTask(prefix, id)\nend)\n\nregister(\"effectmq_getGeneration\", function(args)\n return tonumber(redis.call(\"HGET\", generationHash(args[2]), args[3]) or \"0\")\nend)\n\nregister(\"effectmq_getResult\", function(args)\n return getResult(args[2], args[3], tonumber(args[4]))\nend)\n\nregister(\"effectmq_maintain\", function(args)\n syncAll(args[2])\n return maintenanceSnapshot(args[2])\nend)\n\nregister(\"effectmq_eventCursors\", function(args)\n return eventCursors(args[2])\nend)\n\nregister(\"effectmq_writeSuccess\", function(args)\n local prefix = args[2]\n local leaseToken = args[3]\n local id = args[4]\n local result = args[5]\n local hash = taskHash(prefix, id)\n\n syncAll(prefix)\n\n if exists(hash) == 0 then\n return redis.error_reply(\"Task not found\")\n end\n if not isLockedBy(prefix, id, leaseToken) then\n return redis.error_reply(\"LEASE_LOST\")\n end\n unlockTask(prefix, id)\n setTask(prefix, id, \"success\", result, \"outcome\", \"success\")\n local successPolicy = getTaskField(prefix, id, \"onSuccessPolicy\")\n\n -- result stays raw msgpack bytes end-to-end; publish it as its own field\n publishEvent(prefix, id, \"task.completed\", { \"success\", result, \"policy\", successPolicy })\n\n settleTask(prefix, id)\n return\nend)\n\nregister(\"effectmq_writeError\", function(args)\n local prefix = args[2]\n syncAll(prefix)\n local leaseToken = args[3]\n local id = args[4]\n local error = cmsgpack.unpack(args[5])\n local retryAt = tonumber(args[6]) or -1\n local hash = taskHash(prefix, id)\n\n if exists(hash) == 0 then\n return redis.error_reply(\"Task not found\")\n end\n if not isLockedBy(prefix, id, leaseToken) then\n return redis.error_reply(\"LEASE_LOST\")\n end\n\n failTask(prefix, id, error, retryAt, \"handler\")\n return\nend)\n\nregister(\"effectmq_removeTask\", function(args)\n local prefix = args[2]\n syncAll(prefix)\n local id = args[3]\n\n local lock = getLockId(prefix, id)\n\n if lock then\n return redis.error_reply(\"cannot remove a leased task\")\n end\n if exists(taskHash(prefix, id)) == 0 then return end\n local generation = currentGeneration(prefix, id)\n if hasRetentionHolds(prefix, id, generation) then\n return redis.error_reply(\"task has active retention holds\")\n end\n releaseOwnedHolds(prefix, id, generation)\n deleteTask(prefix, id)\n return\nend)\n\nregister(\"effectmq_forceRemoveTask\", function(args)\n local prefix = args[2]\n syncAll(prefix)\n local id = args[3]\n if exists(taskHash(prefix, id)) == 0 then return end\n local generation = currentGeneration(prefix, id)\n\n -- Administrative removal may revoke the current attempt and inbound result\n -- holds. Holder-side set entries are harmless tombstones and are removed in\n -- bounded batches when those holders settle or are removed.\n unlockTask(prefix, id)\n releaseOwnedHolds(prefix, id, generation)\n redis.call(\"DEL\", retentionHoldersKey(prefix, id, generation))\n local member = identityMember(prefix, id, generation)\n redis.call(\"DEL\", resultHash(prefix, id, generation))\n redis.call(\"ZREM\", taskExpiryIndex(prefix), member)\n redis.call(\"ZREM\", resultExpiryIndex(prefix), member)\n redis.call(\"ZREM\", terminalExpiryIndex(prefix), member)\n redis.call(\"ZREM\", deadLetterList(prefix), member)\n redis.call(\"ZREM\", deadLetterExpiryIndex(prefix), member)\n deleteTask(prefix, id)\n return\nend)\n\nregister(\"effectmq_takeTask\", function(args)\n local prefix = args[2]\n syncAll(prefix)\n local leaseToken = args[3]\n local lockTimeout = tonumber(args[4])\n\n local taskId = popWaitList(prefix)\n\n -- LINDEX returns false (not nil) on an empty list\n if not taskId then\n return nil\n end\n -- sanity check: tasks on wait list should never be locked, but just in case\n if isLocked(prefix, taskId) then\n moveToList(prefix, taskId, \"active\", now + lockTimeout)\n return redis.error_reply(\"Task is locked by another worker\")\n end\n\n local lock = lockTask(prefix, taskId, leaseToken, lockTimeout)\n if lock == nil then\n return nil\n end\n return { leaseToken, getTask(prefix, taskId) }\nend)\n\nregister(\"effectmq_extendLock\", function(args)\n local prefix = args[2]\n local leaseToken = args[3]\n local id = args[4]\n local lockTimeout = tonumber(args[5])\n local lock = getLockId(prefix, id)\n if not lock or lock ~= leaseToken then\n return redis.error_reply(\"LEASE_LOST\")\n end\n redis.call(\"PEXPIRE\", lockHash(prefix, id), lockTimeout)\n -- Renewal is a same-state transition, but still runs through the canonical\n -- mover so a partially corrupt cross-index record is repaired atomically.\n moveToList(prefix, id, \"active\", now + lockTimeout)\n return\nend)\n\nregister(\"effectmq_removeLock\", function(args)\n local prefix = args[2]\n local leaseToken = args[3]\n local id = args[4]\n local lock = getLockId(prefix, id)\n if not lock or lock ~= leaseToken then\n return redis.error_reply(\"LEASE_LOST\")\n end\n unlockTask(prefix, id)\n -- A voluntary release is not a crash. Return the exact owned attempt to\n -- wait immediately without consuming the task's stalled-attempt budget.\n moveToList(prefix, id, \"wait\")\n return\nend)\n\nregister(\"effectmq_setSchedule\", function(args)\n local prefix = args[2]\n local name = args[3]\n local next = tonumber(args[4])\n local hash = scheduleHash(prefix, name)\n redis.call(\"HSETNX\", hash, \"next\", next)\n\n return tonumber(redis.call(\"HGET\", hash, \"next\"))\nend)\n\nregister(\"effectmq_consumeSchedule\", function(args)\n local prefix = args[2]\n local name = args[3]\n local currentToConsume = tonumber(args[4])\n local nextToSet = tonumber(args[5])\n local hash = scheduleHash(prefix, name)\n\n -- consumed is reported as 1/0 rather than a boolean: Redis converts a\n -- Lua false to a null reply, which truncates the returned array\n local currentSchedule = tonumber(redis.call(\"HGET\", hash, \"next\"))\n -- if schedule is not set, we return nil\n if not currentSchedule then\n return { 0 }\n end\n\n -- if the expected current schedule is not equal to the current schedule,\n -- we assume the \"next\" schedule has been calculated relative to the wrong time\n -- so we discard it and send the acual current schedule so the worker can use it to try again\n if currentSchedule ~= currentToConsume then\n return { 0, currentSchedule }\n end\n\n -- if the current schedule match, but the vent is still in the future, we also discard it\n if now < currentSchedule then\n return { 0, currentSchedule }\n end\n\n -- if the event is in the past, we can consume it and schedule the next event\n if currentSchedule < nextToSet then\n redis.call(\"HSET\", hash, \"next\", nextToSet)\n return { 1, nextToSet }\n end\n\n return { 0, currentSchedule }\nend)\n\nregister(\"effectmq_listTasks\", function(args)\n local prefix = args[2]\n syncAll(prefix)\n local list = args[3]\n local offset = tonumber(args[4]) or 0\n local limit = tonumber(args[5]) or 100\n local items\n if list == \"scheduled\" then\n items = redis.call(\"ZRANGE\", delayedList(prefix), offset, offset + limit)\n elseif list == \"wait\" then\n items = redis.call(\"LRANGE\", waitList(prefix), offset, offset + limit)\n elseif list == \"active\" then\n items = redis.call(\"ZRANGE\", activeList(prefix), offset, offset + limit)\n elseif list == \"failed\" then\n items = redis.call(\"ZRANGE\", failedList(prefix), offset, offset + limit)\n elseif list == \"success\" then\n items = redis.call(\"ZRANGE\", successList(prefix), offset, offset + limit)\n else\n return redis.error_reply(\"Invalid list\")\n end\n local hasMore = #items > limit\n if hasMore then table.remove(items, #items) end\n local result = { hasMore and tostring(offset + limit) or \"\" }\n for i, item in ipairs(items) do result[#result + 1] = item end\n return result\nend)\n\nlocal operation = ARGV[1]\nlocal fn = operations[operation]\nif not fn then\n return redis.error_reply(\"Unknown effectmq operation: \" .. tostring(operation))\nend\n\n-- Preserve the operation-local layout while the universal maintenance batch\n-- travels beside the debug flag: args[1] is debug, args[2...] are operation\n-- arguments, and ARGV[3] never leaks into an operation.\nlocal args = { ARGV[2] }\nfor i = 4, #ARGV do\n args[#args + 1] = ARGV[i]\nend\n-- cmsgpack.pack({}) encodes an ambiguous empty map. 0x90 is the canonical\n-- MessagePack empty-array representation and must fail if read as another type.\nEMPTY_LIST = string.char(0x90)\nnow = getNow(args[1])\nmaintenanceBatchSize = tonumber(ARGV[3]) or 100\nmaintenanceRemaining = maintenanceBatchSize\nreturn fn(args)\n"; +export default "-- The effectmq task engine as one content-addressed Redis script. The caller\n-- invokes it through SCRIPT LOAD/EVALSHA with an operation name, the debug\n-- flag (\"1\"/\"0\"), then the operation's own arguments.\n--\n-- Structured values (payload, errors, creator, success, event\n-- payloads) travel and rest as MessagePack: packed once at the Node\n-- boundary, unpacked once at the function entry point (cmsgpack here,\n-- effect's Msgpack codec on the Node side).\n\nlocal MOCKTIME_KEY = \"$$$effectmq/debug/mocktime\"\nlocal EMPTY_LIST = nil\nlocal maintenanceBatchSize = 100\nlocal maintenanceRemaining = 100\n\n-- set once per script invocation; helpers close over it\nlocal now = 0\nlocal debugMode = false\n\nlocal function redisNow()\n local time = redis.call(\"TIME\")\n return (1000 * tonumber(time[1])) + math.floor(tonumber(time[2]) / 1000)\nend\n\nlocal function getNow(debug)\n if debug == \"1\" then\n return tonumber(redis.call(\"GET\", MOCKTIME_KEY) or redisNow())\n end\n return redisNow()\nend\n\nlocal function isPositiveSafeInteger(value)\n return value and value == value and value > 0\n and value <= 9007199254740991 and math.floor(value) == value\nend\n\n-- key helpers ---------------------------------------------------------------\n\nlocal function scheduleHash(prefix, name) return prefix .. \":schedule:\" .. name end\nlocal function taskHash(prefix, id) return prefix .. \":task:\" .. id end\nlocal function generationHash(prefix) return prefix .. \":generations\" end\nlocal function createdList(prefix) return prefix .. \":created\" end\nlocal function delayedList(prefix) return prefix .. \":scheduled\" end\nlocal function waitList(prefix) return prefix .. \":wait\" end\nlocal function successList(prefix) return prefix .. \":success\" end\nlocal function failedList(prefix) return prefix .. \":failed\" end\nlocal function activeList(prefix) return prefix .. \":active\" end\nlocal function eventStream(prefix) return prefix .. \":events\" end\nlocal function eventMetadata(prefix) return prefix .. \":events:metadata\" end\nlocal function lockHash(prefix, id) return prefix .. \":lock:\" .. id end\nlocal function resultHash(prefix, id, generation)\n return prefix .. \":result:\" .. id .. \":\" .. generation\nend\nlocal function taskExpiryIndex(prefix) return prefix .. \":expiry:tasks\" end\nlocal function resultExpiryIndex(prefix) return prefix .. \":expiry:results\" end\nlocal function terminalExpiryIndex(prefix) return prefix .. \":expiry:terminal-indexes\" end\nlocal function deadLetterList(prefix) return prefix .. \":dead-letter\" end\nlocal function deadLetterExpiryIndex(prefix) return prefix .. \":expiry:dead-letter\" end\nlocal function retentionHoldersKey(prefix, id, generation)\n return prefix .. \":task:\" .. id .. \":\" .. generation .. \":retained-by\"\nend\nlocal function retainedTasksKey(prefix, id, generation)\n return prefix .. \":task:\" .. id .. \":\" .. generation .. \":retains\"\nend\nlocal function retentionContinuationKey(prefix) return prefix .. \":retention-release-continuations\" end\nlocal function maintenanceCursorKey(prefix) return prefix .. \":maintenance:cursor\" end\n\n-- list membership -----------------------------------------------------------\n\nlocal function removeFromFailedList(prefix, id) return redis.call(\"ZREM\", failedList(prefix), id) end\nlocal function removeFromSuccessList(prefix, id) return redis.call(\"ZREM\", successList(prefix), id) end\nlocal function removeFromWaitList(prefix, id, count)\n return redis.call(\"LREM\", waitList(prefix), count, id)\nend\nlocal function removeFromDelayedList(prefix, id) return redis.call(\"ZREM\", delayedList(prefix), id) end\nlocal function removeFromActiveLists(prefix, id) return redis.call(\"ZREM\", activeList(prefix), id) end\n\n-- fields is a flat {k1, v1, k2, v2, ...} list. Event data is published as\n-- individual stream fields (not one packed document) so that raw msgpack\n-- values (payload, success) are carried as binary-safe bulk strings — nesting\n-- them inside another msgpack document would corrupt them on decode.\nlocal function publishEvent(prefix, id, eventType, fields)\n local generation = redis.call(\"HGET\", taskHash(prefix, id), \"generation\") or \"0\"\n local protocolVersion = redis.call(\"HGET\", taskHash(prefix, id), \"protocolVersion\") or \"1\"\n local schemaId = redis.call(\"HGET\", taskHash(prefix, id), \"schemaId\") or \"unknown\"\n local maxEventEntries = redis.call(\"HGET\", taskHash(prefix, id), \"maxEventEntries\") or \"10000\"\n local eventRetentionMs = tonumber(\n redis.call(\"HGET\", taskHash(prefix, id), \"eventRetentionMs\") or \"604800000\"\n )\n local args = {\n \"XADD\", eventStream(prefix), \"MAXLEN\", \"~\", maxEventEntries, \"*\",\n \"taskId\", id,\n \"generation\", generation,\n \"protocolVersion\", protocolVersion,\n \"schemaId\", schemaId,\n \"_tag\", eventType,\n }\n for i = 1, #fields do\n args[#args + 1] = fields[i]\n end\n local eventId = redis.call(unpack(args))\n local cutoff = redisNow() - eventRetentionMs\n if cutoff < 0 then cutoff = 0 end\n redis.call(\"XTRIM\", eventStream(prefix), \"MINID\", \"~\", string.format(\"%.0f-0\", cutoff))\n redis.call(\"HSETNX\", eventMetadata(prefix), \"firstEventId\", eventId)\n return eventId\nend\n\nlocal function latestEventCursor(prefix)\n local entries = redis.call(\"XREVRANGE\", eventStream(prefix), \"+\", \"-\", \"COUNT\", 1)\n return entries[1] and entries[1][1] or \"0-0\"\nend\n\nlocal function eventCursors(prefix)\n local earliest = redis.call(\"XRANGE\", eventStream(prefix), \"-\", \"+\", \"COUNT\", 1)\n local latest = redis.call(\"XREVRANGE\", eventStream(prefix), \"+\", \"-\", \"COUNT\", 1)\n local firstEventId = redis.call(\"HGET\", eventMetadata(prefix), \"firstEventId\") or \"0-0\"\n return {\n firstEventId,\n earliest[1] and earliest[1][1] or \"0-0\",\n latest[1] and latest[1][1] or \"0-0\",\n }\nend\n\nlocal function removeFromCurrentLists(prefix, id)\n local storedList = redis.call(\"HGET\", taskHash(prefix, id), \"currentList\")\n local currentList = nil\n local storedRemoval = 0\n if storedList == \"wait\" then\n storedRemoval = removeFromWaitList(prefix, id, 1)\n currentList = \"wait\"\n elseif storedList == \"scheduled\" then\n storedRemoval = removeFromDelayedList(prefix, id)\n currentList = \"scheduled\"\n elseif storedList == \"active\" then\n storedRemoval = removeFromActiveLists(prefix, id)\n currentList = \"active\"\n elseif storedList == \"failed\" then\n storedRemoval = removeFromFailedList(prefix, id)\n currentList = \"failed\"\n elseif storedList == \"success\" then\n storedRemoval = removeFromSuccessList(prefix, id)\n currentList = \"success\"\n end\n\n -- Records created before currentList was introduced take one migration scan.\n -- A stale marker from an older rolling-deployment writer also falls back.\n -- Debug mode repairs deliberately injected cross-index corruption.\n if storedList == false\n or (storedList ~= \"none\" and storedRemoval == 0)\n or debugMode then\n if storedRemoval == 0 then currentList = nil end\n if removeFromDelayedList(prefix, id) > 0 then\n currentList = currentList or \"scheduled\"\n end\n if removeFromActiveLists(prefix, id) > 0 then\n currentList = currentList or \"active\"\n end\n if removeFromFailedList(prefix, id) > 0 then\n currentList = currentList or \"failed\"\n end\n if removeFromSuccessList(prefix, id) > 0 then\n currentList = currentList or \"success\"\n end\n if removeFromWaitList(prefix, id, 0) > 0 then\n currentList = currentList or \"wait\"\n end\n end\n return currentList\nend\n\nlocal function addToActiveLists(prefix, id, expiresAt)\n return redis.call(\"ZADD\", activeList(prefix), expiresAt, id)\nend\nlocal function addToWaitList(prefix, id)\n return redis.call(\"RPUSH\", waitList(prefix), id)\nend\nlocal function addToDelayedList(prefix, id, readyAt)\n return redis.call(\"ZADD\", delayedList(prefix), readyAt, id)\nend\nlocal function addToSuccessList(prefix, id)\n return redis.call(\"ZADD\", successList(prefix), now, id)\nend\nlocal function addToFailedList(prefix, id)\n return redis.call(\"ZADD\", failedList(prefix), now, id)\nend\n\nlocal function executionState(prefix, id, list)\n if list == \"wait\" then return \"waiting\" end\n if list == \"active\" then return \"leased\" end\n if list == \"success\" then return \"succeeded\" end\n if list == \"failed\" then return \"failed\" end\n if list == \"scheduled\" then\n local handlerFailures = tonumber(redis.call(\"HGET\", taskHash(prefix, id), \"handlerFailureCount\") or \"0\")\n local stalledAttempts = tonumber(redis.call(\"HGET\", taskHash(prefix, id), \"stalledAttemptCount\") or \"0\")\n return (handlerFailures + stalledAttempts) > 0 and \"retry-scheduled\" or \"delayed\"\n end\n local outcome = redis.call(\"HGET\", taskHash(prefix, id), \"outcome\")\n if outcome == \"success\" then return \"succeeded\" end\n if outcome == \"failure\" then return \"failed\" end\n return nil\nend\n\n-- the add* helpers do not clear other lists; moveToList is the single entry\n-- point that removes from the current list, adds to the target, and emits task.moved\nlocal function moveToList(prefix, id, list, readyAt, sourceKnownEmpty)\n local currentList = nil\n if not sourceKnownEmpty then currentList = removeFromCurrentLists(prefix, id) end\n if list == \"wait\" then\n addToWaitList(prefix, id)\n elseif list == \"scheduled\" then\n addToDelayedList(prefix, id, readyAt)\n elseif list == \"active\" then\n addToActiveLists(prefix, id, readyAt)\n elseif list == \"failed\" then\n addToFailedList(prefix, id)\n elseif list == \"success\" then\n addToSuccessList(prefix, id)\n end\n if list then\n redis.call(\"HSET\", taskHash(prefix, id), \"currentList\", list)\n else\n redis.call(\"HSET\", taskHash(prefix, id), \"currentList\", \"none\")\n end\n\n -- Removing first repairs duplicate or cross-state membership. Re-add the\n -- target even when it is unchanged, then avoid publishing a fake move.\n if currentList == list then\n return\n end\n\n local fields = {}\n local previousState = executionState(prefix, id, currentList)\n local newState = executionState(prefix, id, list)\n if currentList then\n fields[#fields + 1] = \"from\"\n fields[#fields + 1] = currentList\n end\n if list then\n fields[#fields + 1] = \"to\"\n fields[#fields + 1] = list\n end\n if previousState then\n fields[#fields + 1] = \"previousState\"\n fields[#fields + 1] = previousState\n end\n if newState then\n fields[#fields + 1] = \"newState\"\n fields[#fields + 1] = newState\n end\n fields[#fields + 1] = \"attempt\"\n fields[#fields + 1] = redis.call(\"HGET\", taskHash(prefix, id), \"attempt\") or \"0\"\n fields[#fields + 1] = \"handlerFailureCount\"\n fields[#fields + 1] = redis.call(\"HGET\", taskHash(prefix, id), \"handlerFailureCount\") or \"0\"\n fields[#fields + 1] = \"stalledAttemptCount\"\n fields[#fields + 1] = redis.call(\"HGET\", taskHash(prefix, id), \"stalledAttemptCount\") or \"0\"\n publishEvent(prefix, id, \"task.moved\", fields)\nend\n\nlocal function deleteTask(prefix, id, sourceKnownEmpty)\n local generation = redis.call(\"HGET\", taskHash(prefix, id), \"generation\")\n moveToList(prefix, id, nil, nil, sourceKnownEmpty)\n if generation then\n local member = cmsgpack.pack({ prefix, id, tonumber(generation) })\n redis.call(\"ZREM\", taskExpiryIndex(prefix), member)\n redis.call(\"ZREM\", terminalExpiryIndex(prefix), member)\n end\n redis.call(\"ZREM\", createdList(prefix), id)\n return redis.call(\"DEL\", taskHash(prefix, id))\nend\n\nlocal function popWaitList(prefix) return redis.call(\"LINDEX\", waitList(prefix), 0) end\n\nlocal function getExpiredActiveList(prefix)\n if maintenanceRemaining <= 0 then return {} end\n return redis.call(\n \"ZRANGEBYSCORE\", activeList(prefix), 0, now,\n \"LIMIT\", 0, maintenanceRemaining\n )\nend\n\n-- task hash -----------------------------------------------------------------\n\n-- flat {\"id\", id, k1, v1, ...} entry list with raw hash values: structured\n-- fields stay msgpack bytes and are decoded on the Node side only — Lua never\n-- unpacks the payload, so it round-trips byte-exact (nested nulls included)\nlocal function getTask(prefix, id)\n local fields = redis.call(\"HGETALL\", taskHash(prefix, id))\n if #fields > 0 then\n return { \"id\", id, unpack(fields) }\n end\n return nil\nend\n\nlocal function getResult(prefix, id, generation)\n local fields = redis.call(\"HGETALL\", resultHash(prefix, id, generation))\n if #fields > 0 then return fields end\n return nil\nend\n\n-- append a task's entries to an event field list, prefixing each key\nlocal function appendTaskFields(fields, keyPrefix, entries)\n for i = 1, #entries, 2 do\n fields[#fields + 1] = keyPrefix .. entries[i]\n fields[#fields + 1] = entries[i + 1]\n end\nend\n\nlocal function getTaskField(prefix, id, field) return redis.call(\"HGET\", taskHash(prefix, id), field) end\nlocal function getTaskErrors(prefix, id)\n local raw = getTaskField(prefix, id, \"errors\")\n return raw and cmsgpack.unpack(raw) or {}\nend\nlocal function setTask(prefix, id, ...) return redis.call(\"HSET\", taskHash(prefix, id), \"updatedAt\", now, ...) end\nlocal function setTaskErrors(prefix, id, errors) return setTask(prefix, id, \"errors\", cmsgpack.pack(errors)) end\nlocal function appendTaskError(prefix, id, error, retryAt)\n local errorsList = getTaskErrors(prefix, id)\n local maxErrorEntries = tonumber(getTaskField(prefix, id, \"maxErrorEntries\") or \"100\")\n while #errorsList >= maxErrorEntries and #errorsList > 0 do\n table.remove(errorsList, 1)\n end\n if maxErrorEntries > 0 then\n errorsList[#errorsList + 1] = { error = error, timestamp = now, retryAt = retryAt }\n end\n setTaskErrors(prefix, id, errorsList)\n return errorsList\nend\n\n-- locks ----------------------------------------------------------------------\n\nlocal function exists(key) return redis.call(\"EXISTS\", key) end\nlocal function lockTask(prefix, id, leaseToken, lockTimeout)\n moveToList(prefix, id, \"active\", now + lockTimeout)\n redis.call(\"HINCRBY\", taskHash(prefix, id), \"attempt\", 1)\n return redis.call(\"SET\", lockHash(prefix, id), leaseToken, \"PX\", lockTimeout)\nend\nlocal function unlockTask(prefix, id) return redis.call(\"DEL\", lockHash(prefix, id)) end\nlocal function isLocked(prefix, id) return redis.call(\"EXISTS\", lockHash(prefix, id)) > 0 end\nlocal function getLockId(prefix, id) return redis.call(\"GET\", lockHash(prefix, id)) end\nlocal function isLockedBy(prefix, id, leaseToken) return getLockId(prefix, id) == leaseToken end\n\n-- explicit result retention --------------------------------------------------\n\n-- Relationships live in generation-keyed Redis sets. The member format is a\n-- canonical msgpack tuple so SADD is idempotent without relying on map order.\nlocal function identityMember(queue, id, generation)\n return cmsgpack.pack({ queue, id, tonumber(generation) })\nend\nlocal function decodeIdentity(member)\n local identity = cmsgpack.unpack(member)\n return {\n queue = identity[1],\n id = identity[2],\n generation = tonumber(identity[3]),\n }\nend\nlocal function currentGeneration(prefix, id)\n return tonumber(getTaskField(prefix, id, \"generation\"))\nend\nlocal function hasRetentionHolds(prefix, id, generation)\n return redis.call(\"SCARD\", retentionHoldersKey(prefix, id, generation)) > 0\nend\n\nlocal function scheduleExpiry(index, member, retentionMs)\n redis.call(\"ZADD\", index, now + tonumber(retentionMs), member)\nend\n\nlocal function persistTerminalResult(prefix, id, terminalFailure)\n local generation = currentGeneration(prefix, id)\n local member = identityMember(prefix, id, generation)\n local outcome = getTaskField(prefix, id, \"outcome\")\n local hash = resultHash(prefix, id, generation)\n redis.call(\n \"HSET\",\n hash,\n \"protocolVersion\", getTaskField(prefix, id, \"protocolVersion\"),\n \"schemaId\", getTaskField(prefix, id, \"schemaId\"),\n \"generation\", generation,\n \"outcome\", outcome,\n \"settledAt\", now\n )\n if outcome == \"success\" then\n redis.call(\"HSET\", hash, \"success\", getTaskField(prefix, id, \"success\"))\n else\n local errors = getTaskErrors(prefix, id)\n local failure = terminalFailure\n if failure == nil then\n failure = errors[#errors] and errors[#errors].error or nil\n end\n if failure ~= nil then redis.call(\"HSET\", hash, \"failure\", cmsgpack.pack(failure)) end\n redis.call(\"ZADD\", deadLetterList(prefix), now, member)\n scheduleExpiry(\n deadLetterExpiryIndex(prefix),\n member,\n getTaskField(prefix, id, \"deadLetterRetentionMs\")\n )\n end\n scheduleExpiry(\n resultExpiryIndex(prefix),\n member,\n getTaskField(prefix, id, \"resultRetentionMs\")\n )\nend\n\nlocal function scheduleTerminalRetention(prefix, id)\n local generation = currentGeneration(prefix, id)\n local member = identityMember(prefix, id, generation)\n scheduleExpiry(\n taskExpiryIndex(prefix),\n member,\n getTaskField(prefix, id, \"taskRecordRetentionMs\")\n )\n local outcome = getTaskField(prefix, id, \"outcome\")\n local policy = outcome == \"success\"\n and getTaskField(prefix, id, \"onSuccessPolicy\")\n or getTaskField(prefix, id, \"onFailurePolicy\")\n if policy == \"mark-as-success\" or policy == \"mark-as-failure\" then\n scheduleExpiry(\n terminalExpiryIndex(prefix),\n member,\n getTaskField(prefix, id, \"terminalIndexRetentionMs\")\n )\n end\nend\nlocal function validateLiveHolder(holder)\n if not holder or not holder.queue or not holder.id or not holder.generation then\n return \"invalid retention holder\"\n end\n if exists(taskHash(holder.queue, holder.id)) == 0 then\n return \"retention holder not found\"\n end\n if currentGeneration(holder.queue, holder.id) ~= tonumber(holder.generation) then\n return \"retention holder generation does not match\"\n end\n if getTaskField(holder.queue, holder.id, \"outcome\") then\n return \"retention holder is settled\"\n end\n return nil\nend\nlocal function acquireRetentionHold(holder, retainedPrefix, retainedId, retainedGeneration)\n local holderMember = identityMember(holder.queue, holder.id, holder.generation)\n local retainedMember = identityMember(retainedPrefix, retainedId, retainedGeneration)\n local holdersKey = retentionHoldersKey(retainedPrefix, retainedId, retainedGeneration)\n local retainedKey = retainedTasksKey(holder.queue, holder.id, holder.generation)\n\n -- Replay of the same relationship remains idempotent even at the cap.\n if redis.call(\"SISMEMBER\", holdersKey, holderMember) == 1 then return nil end\n\n local holderLimit = tonumber(getTaskField(holder.queue, holder.id, \"maxRelationships\") or \"1000\")\n local retainedLimit = tonumber(getTaskField(retainedPrefix, retainedId, \"maxRelationships\") or \"1000\")\n if redis.call(\"SCARD\", retainedKey) >= holderLimit then\n return \"STORAGE_RELATIONSHIP_LIMIT holder \" .. holderLimit\n end\n if redis.call(\"SCARD\", holdersKey) >= retainedLimit then\n return \"STORAGE_RELATIONSHIP_LIMIT retained \" .. retainedLimit\n end\n redis.call(\n \"SADD\",\n holdersKey,\n holderMember\n )\n redis.call(\n \"SADD\",\n retainedKey,\n retainedMember\n )\n return nil\nend\n\n-- Settlement is visible immediately through outcome and policy-selected\n-- terminal membership. A delete policy disposes the record only after the\n-- final explicit hold is gone.\nlocal function applyCompletionPolicy(prefix, id)\n if exists(taskHash(prefix, id)) == 0 then return end\n local generation = currentGeneration(prefix, id)\n local outcome = getTaskField(prefix, id, \"outcome\")\n if not outcome then return end\n local policy = outcome == \"success\"\n and getTaskField(prefix, id, \"onSuccessPolicy\")\n or getTaskField(prefix, id, \"onFailurePolicy\")\n\n if policy == \"mark-as-success\" then\n moveToList(prefix, id, \"success\")\n elseif policy == \"mark-as-failure\" then\n moveToList(prefix, id, \"failed\")\n else\n moveToList(prefix, id, nil)\n end\n\n if policy == \"delete\" and not hasRetentionHolds(prefix, id, generation) then\n deleteTask(prefix, id, true)\n end\nend\n\n-- Release at most one bounded batch. If work remains, the generation identity\n-- stays in a durable per-queue continuation index that later sync calls drain.\nlocal function releaseOwnedHolds(holderPrefix, holderId, holderGeneration)\n local key = retainedTasksKey(holderPrefix, holderId, holderGeneration)\n local holderMember = identityMember(holderPrefix, holderId, holderGeneration)\n local retainedMembers = maintenanceRemaining > 0\n and redis.call(\"SPOP\", key, maintenanceRemaining)\n or {}\n maintenanceRemaining = maintenanceRemaining - #retainedMembers\n for i, retainedMember in ipairs(retainedMembers) do\n local retained = decodeIdentity(retainedMember)\n redis.call(\n \"SREM\",\n retentionHoldersKey(retained.queue, retained.id, retained.generation),\n holderMember\n )\n if exists(taskHash(retained.queue, retained.id)) == 1\n and currentGeneration(retained.queue, retained.id) == retained.generation\n and getTaskField(retained.queue, retained.id, \"outcome\")\n and not hasRetentionHolds(retained.queue, retained.id, retained.generation)\n then\n applyCompletionPolicy(retained.queue, retained.id)\n end\n end\n\n local continuation = retentionContinuationKey(holderPrefix)\n if redis.call(\"SCARD\", key) > 0 then\n redis.call(\"ZADD\", continuation, now, holderMember)\n else\n redis.call(\"DEL\", key)\n redis.call(\"ZREM\", continuation, holderMember)\n end\nend\n\nlocal function settleTask(prefix, id, terminalFailure)\n local generation = currentGeneration(prefix, id)\n persistTerminalResult(prefix, id, terminalFailure)\n scheduleTerminalRetention(prefix, id)\n applyCompletionPolicy(prefix, id)\n releaseOwnedHolds(prefix, id, generation)\nend\n\nlocal function failTask(prefix, id, error, retryAt, failureKind)\n unlockTask(prefix, id)\n\n if failureKind == \"stall\" then\n redis.call(\"HINCRBY\", taskHash(prefix, id), \"stalledAttemptCount\", 1)\n else\n redis.call(\"HINCRBY\", taskHash(prefix, id), \"handlerFailureCount\", 1)\n end\n\n -- retryAt arrives as -1 (or nil) when no retry is scheduled; normalize\n -- to nil so it is omitted from the stored error and the event payload\n if (retryAt or -1) < 0 then retryAt = nil end\n appendTaskError(prefix, id, error, retryAt)\n local onFailurePolicy = getTaskField(prefix, id, \"onFailurePolicy\")\n\n local errorTag = type(error) == \"table\" and error._tag or nil\n\n local willRetry = errorTag ~= \"~effectmq/Error/Canceled\" and retryAt ~= nil\n local fields = {\n \"error\", cmsgpack.pack(error),\n \"policy\", onFailurePolicy,\n \"failureKind\", failureKind,\n \"attempt\", getTaskField(prefix, id, \"attempt\") or \"0\",\n \"terminal\", willRetry and \"0\" or \"1\",\n }\n if retryAt then\n fields[#fields + 1] = \"retryAt\"\n fields[#fields + 1] = retryAt\n end\n publishEvent(prefix, id, \"task.failed\", fields)\n if willRetry then\n if retryAt > now then\n moveToList(prefix, id, \"scheduled\", retryAt)\n else\n moveToList(prefix, id, \"wait\")\n end\n return\n end\n setTask(prefix, id, \"outcome\", \"failure\")\n settleTask(prefix, id, error)\nend\n\n-- sync -----------------------------------------------------------------------\n\nlocal function syncLocks(prefix)\n local activeIds = getExpiredActiveList(prefix)\n maintenanceRemaining = maintenanceRemaining - #activeIds\n for i, id in ipairs(activeIds) do\n if not isLocked(prefix, id) then\n local nextStalledCount = tonumber(getTaskField(prefix, id, \"stalledAttemptCount\") or \"0\") + 1\n local maxStalledCount = tonumber(getTaskField(prefix, id, \"maxStalledCount\") or \"1\")\n local retryAt = nextStalledCount > maxStalledCount and -1 or 0\n failTask(prefix, id, {\n _tag = \"~effectmq/Error/Stalled\",\n timestamp = now,\n }, retryAt, \"stall\")\n end\n end\nend\n\nlocal function syncDelayed(prefix)\n -- we lazily move tasks from the delayed list to the wait list so we need to sync\n -- before performing any other operations\n if maintenanceRemaining <= 0 then return end\n local items = redis.call(\n \"ZRANGEBYSCORE\", delayedList(prefix), 0, now,\n \"LIMIT\", 0, maintenanceRemaining\n )\n maintenanceRemaining = maintenanceRemaining - #items\n for i, item in ipairs(items) do\n moveToList(prefix, item, \"wait\")\n end\nend\n\nlocal function syncRetentionContinuations(prefix)\n if maintenanceRemaining <= 0 then return end\n local holders = redis.call(\n \"ZRANGE\",\n retentionContinuationKey(prefix),\n 0,\n 0\n )\n for i, holderMember in ipairs(holders) do\n local holder = decodeIdentity(holderMember)\n releaseOwnedHolds(holder.queue, holder.id, holder.generation)\n end\nend\n\nlocal function dueExpiryMembers(index)\n if maintenanceRemaining <= 0 then return {} end\n return redis.call(\n \"ZRANGEBYSCORE\", index, 0, now,\n \"LIMIT\", 0, maintenanceRemaining\n )\nend\n\nlocal function syncTaskExpiry(prefix)\n local index = taskExpiryIndex(prefix)\n local members = dueExpiryMembers(index)\n maintenanceRemaining = maintenanceRemaining - #members\n for i, member in ipairs(members) do\n local task = decodeIdentity(member)\n if exists(taskHash(prefix, task.id)) == 1\n and currentGeneration(prefix, task.id) == task.generation\n and getTaskField(prefix, task.id, \"outcome\")\n then\n if hasRetentionHolds(prefix, task.id, task.generation) then\n redis.call(\"ZADD\", index, now + 1000, member)\n else\n deleteTask(prefix, task.id)\n redis.call(\"DEL\", retentionHoldersKey(prefix, task.id, task.generation))\n redis.call(\"ZREM\", index, member)\n end\n else\n redis.call(\"ZREM\", index, member)\n end\n end\nend\n\nlocal function syncResultExpiry(prefix)\n local index = resultExpiryIndex(prefix)\n local members = dueExpiryMembers(index)\n maintenanceRemaining = maintenanceRemaining - #members\n for i, member in ipairs(members) do\n local result = decodeIdentity(member)\n if hasRetentionHolds(prefix, result.id, result.generation) then\n redis.call(\"ZADD\", index, now + 1000, member)\n else\n redis.call(\"DEL\", resultHash(prefix, result.id, result.generation))\n redis.call(\"ZREM\", index, member)\n end\n end\nend\n\nlocal function syncTerminalExpiry(prefix)\n local index = terminalExpiryIndex(prefix)\n local members = dueExpiryMembers(index)\n maintenanceRemaining = maintenanceRemaining - #members\n for i, member in ipairs(members) do\n local task = decodeIdentity(member)\n if exists(taskHash(prefix, task.id)) == 1\n and currentGeneration(prefix, task.id) == task.generation\n then\n local removed = removeFromSuccessList(prefix, task.id)\n + removeFromFailedList(prefix, task.id)\n if removed > 0 then\n redis.call(\"HSET\", taskHash(prefix, task.id), \"currentList\", \"none\")\n end\n end\n redis.call(\"ZREM\", index, member)\n end\nend\n\nlocal function syncDeadLetterExpiry(prefix)\n local index = deadLetterExpiryIndex(prefix)\n local members = dueExpiryMembers(index)\n maintenanceRemaining = maintenanceRemaining - #members\n for i, member in ipairs(members) do\n redis.call(\"ZREM\", deadLetterList(prefix), member)\n redis.call(\"ZREM\", index, member)\n end\nend\n\nlocal function syncAll(prefix)\n local syncers = {\n syncDelayed,\n syncLocks,\n syncRetentionContinuations,\n syncTaskExpiry,\n syncResultExpiry,\n syncTerminalExpiry,\n syncDeadLetterExpiry,\n }\n local cursor = tonumber(redis.call(\"GET\", maintenanceCursorKey(prefix)) or \"0\")\n for offset = 0, #syncers - 1 do\n if maintenanceRemaining <= 0 then break end\n local index = ((cursor + offset) % #syncers) + 1\n syncers[index](prefix)\n end\n redis.call(\"SET\", maintenanceCursorKey(prefix), (cursor + 1) % #syncers)\nend\n\nlocal function firstScore(key)\n local entry = redis.call(\"ZRANGE\", key, 0, 0, \"WITHSCORES\")\n return entry[2] and tonumber(entry[2]) or nil\nend\n\nlocal function maintenanceSnapshot(prefix)\n local depth = redis.call(\"LLEN\", waitList(prefix))\n + redis.call(\"ZCARD\", delayedList(prefix))\n + redis.call(\"ZCARD\", activeList(prefix))\n local oldestCreated = firstScore(createdList(prefix))\n local oldestAge = oldestCreated and math.max(0, now - oldestCreated) or 0\n local dueBacklog = redis.call(\"ZCOUNT\", delayedList(prefix), 0, now)\n local expiredLeases = redis.call(\"ZCOUNT\", activeList(prefix), 0, now)\n local retentionBacklog = redis.call(\"ZCOUNT\", taskExpiryIndex(prefix), 0, now)\n + redis.call(\"ZCOUNT\", resultExpiryIndex(prefix), 0, now)\n + redis.call(\"ZCOUNT\", terminalExpiryIndex(prefix), 0, now)\n + redis.call(\"ZCOUNT\", deadLetterExpiryIndex(prefix), 0, now)\n local oldestDue = nil\n local dueIndexes = {\n delayedList(prefix), activeList(prefix), taskExpiryIndex(prefix),\n resultExpiryIndex(prefix), terminalExpiryIndex(prefix),\n deadLetterExpiryIndex(prefix), retentionContinuationKey(prefix),\n }\n for i, index in ipairs(dueIndexes) do\n local score = firstScore(index)\n if score and score <= now and (not oldestDue or score < oldestDue) then\n oldestDue = score\n end\n end\n return {\n depth,\n oldestAge,\n oldestDue and math.max(0, now - oldestDue) or 0,\n dueBacklog,\n expiredLeases,\n retentionBacklog,\n maintenanceBatchSize - maintenanceRemaining,\n }\nend\n\n-- entry points ---------------------------------------------------------------\n\nlocal operations = {}\nlocal function register(name, fn)\n operations[name] = fn\nend\n\nregister(\"effectmq_createTask\", function(args)\n local prefix = args[2]\n local throwOnExists = args[3] == \"1\"\n\n local id = args[4]\n local name = args[5]\n local payload = args[6]\n local delay = tonumber(args[7])\n local maxRetries = tonumber(args[8])\n local onSuccessPolicy = args[9]\n local onFailurePolicy = args[10]\n -- Relationship identities arrive with fully-qualified queue prefixes.\n local retentionHolder = args[11] ~= \"\" and cmsgpack.unpack(args[11]) or nil\n local creator = args[12]\n local onDuplicate = args[13] or \"return-existing\"\n local maxStalledCount = tonumber(args[14])\n local maxErrorEntries = tonumber(args[16])\n local maxRelationships = tonumber(args[17])\n local maxEventEntries = tonumber(args[18])\n local taskRecordRetentionMs = tonumber(args[19])\n local resultRetentionMs = tonumber(args[20])\n local terminalIndexRetentionMs = tonumber(args[21])\n local deadLetterRetentionMs = tonumber(args[22])\n local eventRetentionMs = tonumber(args[23])\n local maxSafeInteger = 9007199254740991\n if not delay or delay ~= delay or delay < 0 or math.abs(delay) > maxSafeInteger then\n return redis.error_reply(\"invalid delay\")\n end\n local integerFields = {\n { \"maxRetries\", maxRetries, -1 },\n { \"maxStalledCount\", maxStalledCount, 0 },\n { \"maxErrorEntries\", maxErrorEntries, 0 },\n { \"maxRelationships\", maxRelationships, 0 },\n { \"maxEventEntries\", maxEventEntries, 1 },\n { \"taskRecordRetentionMs\", taskRecordRetentionMs, 0 },\n { \"resultRetentionMs\", resultRetentionMs, 0 },\n { \"terminalIndexRetentionMs\", terminalIndexRetentionMs, 0 },\n { \"deadLetterRetentionMs\", deadLetterRetentionMs, 0 },\n { \"eventRetentionMs\", eventRetentionMs, 0 },\n }\n for _, field in ipairs(integerFields) do\n local value = field[2]\n if not value or value ~= value or value < field[3]\n or value > maxSafeInteger or math.floor(value) ~= value then\n return redis.error_reply(\"invalid \" .. field[1])\n end\n end\n syncAll(prefix)\n local existingTask = getTask(prefix, id)\n local replacedTask = nil\n\n if retentionHolder then\n local holderError = validateLiveHolder(retentionHolder)\n if holderError then return redis.error_reply(holderError) end\n end\n\n if existingTask ~= nil then\n if throwOnExists then\n return redis.error_reply(\"task already exists\")\n end\n if onDuplicate == \"return-existing\" then\n if retentionHolder then\n local retentionError = acquireRetentionHold(\n retentionHolder,\n prefix,\n id,\n currentGeneration(prefix, id)\n )\n if retentionError then return redis.error_reply(retentionError) end\n end\n return { \"existing\", latestEventCursor(prefix), existingTask }\n end\n if onDuplicate ~= \"new-generation\" then\n return redis.error_reply(\"invalid duplicate mode\")\n end\n if not getTaskField(prefix, id, \"outcome\") then\n return redis.error_reply(\"cannot create a new generation for a non-terminal task\")\n end\n if hasRetentionHolds(prefix, id, currentGeneration(prefix, id)) then\n return redis.error_reply(\"cannot replace a retained task generation\")\n end\n replacedTask = existingTask\n deleteTask(prefix, id)\n unlockTask(prefix, id)\n existingTask = nil\n end\n\n local generation = redis.call(\"HINCRBY\", generationHash(prefix), id, 1)\n setTask(\n prefix, id,\n \"generation\", generation,\n \"protocolVersion\", 1,\n \"schemaId\", args[15] or name,\n \"name\", name,\n \"createdAt\", now,\n \"payload\", payload,\n \"delay\", delay,\n \"maxRetries\", maxRetries,\n \"maxStalledCount\", maxStalledCount,\n \"maxErrorEntries\", maxErrorEntries,\n \"maxRelationships\", maxRelationships,\n \"maxEventEntries\", maxEventEntries,\n \"taskRecordRetentionMs\", taskRecordRetentionMs,\n \"resultRetentionMs\", resultRetentionMs,\n \"terminalIndexRetentionMs\", terminalIndexRetentionMs,\n \"deadLetterRetentionMs\", deadLetterRetentionMs,\n \"eventRetentionMs\", eventRetentionMs,\n \"attempt\", 0,\n \"handlerFailureCount\", 0,\n \"stalledAttemptCount\", 0,\n \"onSuccessPolicy\", onSuccessPolicy,\n \"onFailurePolicy\", onFailurePolicy,\n \"errors\", EMPTY_LIST\n )\n redis.call(\"ZADD\", createdList(prefix), now, id)\n if creator ~= \"\" then\n setTask(prefix, id, \"creator\", creator)\n end\n if retentionHolder then\n local retentionError = acquireRetentionHold(retentionHolder, prefix, id, generation)\n if retentionError then\n deleteTask(prefix, id)\n if redis.call(\"HINCRBY\", generationHash(prefix), id, -1) == 0 then\n redis.call(\"HDEL\", generationHash(prefix), id)\n end\n return redis.error_reply(retentionError)\n end\n end\n local newTask = getTask(prefix, id)\n local fields = {}\n if replacedTask then appendTaskFields(fields, \"existing:\", replacedTask) end\n appendTaskFields(fields, \"new:\", newTask)\n fields[#fields + 1] = \"state\"\n fields[#fields + 1] = delay > 0 and \"delayed\" or \"waiting\"\n publishEvent(prefix, id, replacedTask and \"task.updated\" or \"task.created\", fields)\n\n if delay > 0 then\n moveToList(prefix, id, \"scheduled\", now + delay, true)\n else\n moveToList(prefix, id, \"wait\", nil, true)\n end\n\n return { \"created\", latestEventCursor(prefix), getTask(prefix, id) }\nend)\n\nregister(\"effectmq_getTask\", function(args)\n local prefix = args[2]\n local id = args[3]\n return getTask(prefix, id)\nend)\n\nregister(\"effectmq_getGeneration\", function(args)\n return tonumber(redis.call(\"HGET\", generationHash(args[2]), args[3]) or \"0\")\nend)\n\nregister(\"effectmq_getResult\", function(args)\n return getResult(args[2], args[3], tonumber(args[4]))\nend)\n\nregister(\"effectmq_maintain\", function(args)\n syncAll(args[2])\n return maintenanceSnapshot(args[2])\nend)\n\nregister(\"effectmq_eventCursors\", function(args)\n return eventCursors(args[2])\nend)\n\nregister(\"effectmq_writeSuccess\", function(args)\n local prefix = args[2]\n local leaseToken = args[3]\n local id = args[4]\n local result = args[5]\n local hash = taskHash(prefix, id)\n\n syncAll(prefix)\n\n if exists(hash) == 0 then\n return redis.error_reply(\"Task not found\")\n end\n if not isLockedBy(prefix, id, leaseToken) then\n return redis.error_reply(\"LEASE_LOST\")\n end\n unlockTask(prefix, id)\n setTask(prefix, id, \"success\", result, \"outcome\", \"success\")\n local successPolicy = getTaskField(prefix, id, \"onSuccessPolicy\")\n\n -- result stays raw msgpack bytes end-to-end; publish it as its own field\n publishEvent(prefix, id, \"task.completed\", { \"success\", result, \"policy\", successPolicy })\n\n settleTask(prefix, id)\n return\nend)\n\nregister(\"effectmq_writeError\", function(args)\n local prefix = args[2]\n local leaseToken = args[3]\n local id = args[4]\n local retryAt = tonumber(args[6])\n if not retryAt or retryAt ~= retryAt or retryAt < -1\n or math.abs(retryAt) > 9007199254740991 then\n return redis.error_reply(\"invalid retryAt\")\n end\n local error = cmsgpack.unpack(args[5])\n local hash = taskHash(prefix, id)\n syncAll(prefix)\n\n if exists(hash) == 0 then\n return redis.error_reply(\"Task not found\")\n end\n if not isLockedBy(prefix, id, leaseToken) then\n return redis.error_reply(\"LEASE_LOST\")\n end\n\n failTask(prefix, id, error, retryAt, \"handler\")\n return\nend)\n\nregister(\"effectmq_removeTask\", function(args)\n local prefix = args[2]\n syncAll(prefix)\n local id = args[3]\n\n local lock = getLockId(prefix, id)\n\n if lock then\n return redis.error_reply(\"cannot remove a leased task\")\n end\n if exists(taskHash(prefix, id)) == 0 then return end\n local generation = currentGeneration(prefix, id)\n if hasRetentionHolds(prefix, id, generation) then\n return redis.error_reply(\"task has active retention holds\")\n end\n releaseOwnedHolds(prefix, id, generation)\n deleteTask(prefix, id)\n return\nend)\n\nregister(\"effectmq_forceRemoveTask\", function(args)\n local prefix = args[2]\n syncAll(prefix)\n local id = args[3]\n if exists(taskHash(prefix, id)) == 0 then return end\n local generation = currentGeneration(prefix, id)\n\n -- Administrative removal may revoke the current attempt and inbound result\n -- holds. Holder-side set entries are harmless tombstones and are removed in\n -- bounded batches when those holders settle or are removed.\n unlockTask(prefix, id)\n releaseOwnedHolds(prefix, id, generation)\n redis.call(\"DEL\", retentionHoldersKey(prefix, id, generation))\n local member = identityMember(prefix, id, generation)\n redis.call(\"DEL\", resultHash(prefix, id, generation))\n redis.call(\"ZREM\", taskExpiryIndex(prefix), member)\n redis.call(\"ZREM\", resultExpiryIndex(prefix), member)\n redis.call(\"ZREM\", terminalExpiryIndex(prefix), member)\n redis.call(\"ZREM\", deadLetterList(prefix), member)\n redis.call(\"ZREM\", deadLetterExpiryIndex(prefix), member)\n deleteTask(prefix, id)\n return\nend)\n\nregister(\"effectmq_takeTask\", function(args)\n local prefix = args[2]\n local leaseToken = args[3]\n local lockTimeout = tonumber(args[4])\n if not isPositiveSafeInteger(lockTimeout) then\n return redis.error_reply(\"invalid lockTimeout\")\n end\n syncAll(prefix)\n\n local taskId = popWaitList(prefix)\n\n -- LINDEX returns false (not nil) on an empty list\n if not taskId then\n return nil\n end\n -- sanity check: tasks on wait list should never be locked, but just in case\n if isLocked(prefix, taskId) then\n moveToList(prefix, taskId, \"active\", now + lockTimeout)\n return redis.error_reply(\"Task is locked by another worker\")\n end\n\n local lock = lockTask(prefix, taskId, leaseToken, lockTimeout)\n if lock == nil then\n return nil\n end\n return { leaseToken, getTask(prefix, taskId) }\nend)\n\nregister(\"effectmq_extendLock\", function(args)\n local prefix = args[2]\n local leaseToken = args[3]\n local id = args[4]\n local lockTimeout = tonumber(args[5])\n if not isPositiveSafeInteger(lockTimeout) then\n return redis.error_reply(\"invalid lockTimeout\")\n end\n local lock = getLockId(prefix, id)\n if not lock or lock ~= leaseToken then\n return redis.error_reply(\"LEASE_LOST\")\n end\n redis.call(\"PEXPIRE\", lockHash(prefix, id), lockTimeout)\n -- Renewal is a same-state transition, but still runs through the canonical\n -- mover so a partially corrupt cross-index record is repaired atomically.\n moveToList(prefix, id, \"active\", now + lockTimeout)\n return\nend)\n\nregister(\"effectmq_removeLock\", function(args)\n local prefix = args[2]\n local leaseToken = args[3]\n local id = args[4]\n local lock = getLockId(prefix, id)\n if not lock or lock ~= leaseToken then\n return redis.error_reply(\"LEASE_LOST\")\n end\n unlockTask(prefix, id)\n -- A voluntary release is not a crash. Return the exact owned attempt to\n -- wait immediately without consuming the task's stalled-attempt budget.\n moveToList(prefix, id, \"wait\")\n return\nend)\n\nregister(\"effectmq_setSchedule\", function(args)\n local prefix = args[2]\n local name = args[3]\n local next = tonumber(args[4])\n local hash = scheduleHash(prefix, name)\n redis.call(\"HSETNX\", hash, \"next\", next)\n\n return tonumber(redis.call(\"HGET\", hash, \"next\"))\nend)\n\nregister(\"effectmq_consumeSchedule\", function(args)\n local prefix = args[2]\n local name = args[3]\n local currentToConsume = tonumber(args[4])\n local nextToSet = tonumber(args[5])\n local hash = scheduleHash(prefix, name)\n\n -- consumed is reported as 1/0 rather than a boolean: Redis converts a\n -- Lua false to a null reply, which truncates the returned array\n local currentSchedule = tonumber(redis.call(\"HGET\", hash, \"next\"))\n -- if schedule is not set, we return nil\n if not currentSchedule then\n return { 0 }\n end\n\n -- if the expected current schedule is not equal to the current schedule,\n -- we assume the \"next\" schedule has been calculated relative to the wrong time\n -- so we discard it and send the acual current schedule so the worker can use it to try again\n if currentSchedule ~= currentToConsume then\n return { 0, currentSchedule }\n end\n\n -- if the current schedule match, but the vent is still in the future, we also discard it\n if now < currentSchedule then\n return { 0, currentSchedule }\n end\n\n -- if the event is in the past, we can consume it and schedule the next event\n if currentSchedule < nextToSet then\n redis.call(\"HSET\", hash, \"next\", nextToSet)\n return { 1, nextToSet }\n end\n\n return { 0, currentSchedule }\nend)\n\nregister(\"effectmq_listTasks\", function(args)\n local prefix = args[2]\n syncAll(prefix)\n local list = args[3]\n local offset = tonumber(args[4]) or 0\n local limit = tonumber(args[5]) or 100\n local items\n if list == \"scheduled\" then\n items = redis.call(\"ZRANGE\", delayedList(prefix), offset, offset + limit)\n elseif list == \"wait\" then\n items = redis.call(\"LRANGE\", waitList(prefix), offset, offset + limit)\n elseif list == \"active\" then\n items = redis.call(\"ZRANGE\", activeList(prefix), offset, offset + limit)\n elseif list == \"failed\" then\n items = redis.call(\"ZRANGE\", failedList(prefix), offset, offset + limit)\n elseif list == \"success\" then\n items = redis.call(\"ZRANGE\", successList(prefix), offset, offset + limit)\n else\n return redis.error_reply(\"Invalid list\")\n end\n local hasMore = #items > limit\n if hasMore then table.remove(items, #items) end\n local result = { hasMore and tostring(offset + limit) or \"\" }\n for i, item in ipairs(items) do result[#result + 1] = item end\n return result\nend)\n\nlocal operation = ARGV[1]\nlocal fn = operations[operation]\nif not fn then\n return redis.error_reply(\"Unknown effectmq operation: \" .. tostring(operation))\nend\n\n-- Preserve the operation-local layout while the universal maintenance batch\n-- travels beside the debug flag: args[1] is debug, args[2...] are operation\n-- arguments, and ARGV[3] never leaks into an operation.\nlocal args = { ARGV[2] }\nfor i = 4, #ARGV do\n args[#args + 1] = ARGV[i]\nend\n-- cmsgpack.pack({}) encodes an ambiguous empty map. 0x90 is the canonical\n-- MessagePack empty-array representation and must fail if read as another type.\nEMPTY_LIST = string.char(0x90)\nnow = getNow(args[1])\ndebugMode = args[1] == \"1\"\nmaintenanceBatchSize = tonumber(ARGV[3]) or 100\nmaintenanceRemaining = maintenanceBatchSize\nreturn fn(args)\n";