diff --git a/packages/bb-app/src/launcher.ts b/packages/bb-app/src/launcher.ts index 491cb54f91..138fab06e2 100644 --- a/packages/bb-app/src/launcher.ts +++ b/packages/bb-app/src/launcher.ts @@ -79,7 +79,11 @@ const HEALTH_CHECK_INTERVAL_MS = 100; const HEALTH_CHECK_REQUEST_TIMEOUT_MS = 1_000; const MANAGED_PROCESS_TERMINATION_TIMEOUT_MS = 5_000; const MANAGED_PROCESS_KILL_TIMEOUT_MS = 1_000; -const MANAGED_PROCESS_RESTART_RETRY_DELAY_MS = 1_000; +const MANAGED_PROCESS_RESTART_INITIAL_DELAY_MS = 1_000; +const MANAGED_PROCESS_RESTART_MAX_DELAY_MS = 60_000; +const MANAGED_PROCESS_RESTART_MAX_CONSECUTIVE_FAILURES = 8; +export const MANAGED_PROCESS_RESTART_EXHAUSTED_EXIT_CODE = 75; +const MANAGED_PROCESS_STABLE_RUNTIME_MS = 60_000; const START_COMMAND = "start"; const STOP_COMMAND = "stop"; const STOP_TIMEOUT_MS = 15_000; @@ -221,6 +225,8 @@ interface ResolveWorktreeRuntimePolicyArgs { interface RunBbAppOptions { beforeServerStart?: () => Promise | void; + delayMilliseconds?: DelayMillisecondsFn; + runtimeState?: BbAppRuntimeState; worktreePolicy: WorktreeRuntimePolicy | null; } @@ -331,7 +337,7 @@ type StartManagedProcess = () => Promise; export type DelayMillisecondsFn = ( args: DelayMillisecondsArgs, ) => Promise; -export type FullStackSupervisionResult = "shutdown" | "stopped"; +export type FullStackSupervisionResult = "failed" | "shutdown" | "stopped"; type ResolveWaitForProcessExitWithTimeout = ( result: WaitForProcessExitWithTimeoutResult, ) => void; @@ -406,17 +412,27 @@ interface StartFullStackDaemonProcessArgs { interface RestartManagedProcessArgs { context: BbAppStartContext; delayMilliseconds: DelayMillisecondsFn; + failureCounts: ManagedProcessFailureCounts; isShutdownRequested: () => boolean; processName: ManagedProcessName; + shutdownSignal?: AbortSignal; start: StartManagedProcess; } +interface ManagedProcessFailureCounts { + daemon: number; + server: number; +} + interface SuperviseFullStackProcessesArgs { context: BbAppStartContext; delayMilliseconds: DelayMillisecondsFn; + failureCounts?: ManagedProcessFailureCounts; isHealthyServerAnswering?: (url: string) => Promise; isShutdownRequested: () => boolean; + now?: () => number; processes: ManagedFullStackProcesses; + shutdownSignal?: AbortSignal; startDaemon: StartManagedProcess; startServer: StartManagedProcess; } @@ -438,6 +454,7 @@ interface LogManagedProcessStartupFailureContextArgs { export interface DelayMillisecondsArgs { ms: number; + signal?: AbortSignal; } interface WaitForServerHealthArgs { @@ -2556,12 +2573,50 @@ function toExitCode(result: ProcessExitResult): number { return result.signal === null ? 1 : 128; } -function delayMilliseconds(args: DelayMillisecondsArgs): Promise { +function isStartupStorageError(error: unknown): error is NodeJS.ErrnoException { + if (!(error instanceof Error) || !("code" in error)) { + return false; + } + return ["EACCES", "EIO", "ENOSPC", "EPERM", "EROFS"].includes( + String(error.code), + ); +} + +function reportTerminalStartupStorageFailure(args: { + error: NodeJS.ErrnoException; + message: string; +}): void { + process.stderr.write(`bb-app: ${args.message}: ${args.error.message}\n`); + process.exitCode = MANAGED_PROCESS_RESTART_EXHAUSTED_EXIT_CODE; +} + +export function delayMilliseconds(args: DelayMillisecondsArgs): Promise { + if (args.signal?.aborted) { + return Promise.resolve(); + } return new Promise((resolvePromise) => { - setTimeout(resolvePromise, args.ms); + let timeout: ReturnType | undefined; + const finish = (): void => { + if (timeout !== undefined) { + clearTimeout(timeout); + } + args.signal?.removeEventListener("abort", finish); + resolvePromise(); + }; + timeout = setTimeout(finish, args.ms); + args.signal?.addEventListener("abort", finish, { once: true }); }); } +export function calculateManagedProcessRestartDelay( + consecutiveFailure: number, +): number { + return Math.min( + MANAGED_PROCESS_RESTART_INITIAL_DELAY_MS * 2 ** (consecutiveFailure - 1), + MANAGED_PROCESS_RESTART_MAX_DELAY_MS, + ); +} + async function terminateProcessIfRunning( args: TerminateProcessIfRunningArgs, ): Promise { @@ -3122,7 +3177,27 @@ async function startFullStackDaemonProcess( async function restartManagedProcess( args: RestartManagedProcessArgs, ): Promise { - while (!args.isShutdownRequested()) { + while ( + args.failureCounts[args.processName] < + MANAGED_PROCESS_RESTART_MAX_CONSECUTIVE_FAILURES && + !args.isShutdownRequested() + ) { + const delayMs = calculateManagedProcessRestartDelay( + args.failureCounts[args.processName], + ); + log( + yellow("!"), + `${formatManagedProcessName(args.processName)} restart attempt ${args.failureCounts[args.processName]}/${MANAGED_PROCESS_RESTART_MAX_CONSECUTIVE_FAILURES - 1} in ${delayMs}ms`, + ); + await args.delayMilliseconds({ + ms: delayMs, + ...(args.shutdownSignal === undefined + ? {} + : { signal: args.shutdownSignal }), + }); + if (args.isShutdownRequested()) { + return null; + } beginStep(`Restarting ${formatManagedProcessLabel(args.processName)}`); try { const processRun = await args.start(); @@ -3143,12 +3218,16 @@ async function restartManagedProcess( context: args.context, processName: args.processName, }); - await args.delayMilliseconds({ - ms: MANAGED_PROCESS_RESTART_RETRY_DELAY_MS, - }); + args.failureCounts[args.processName] += 1; } } + if (!args.isShutdownRequested()) { + log( + red("✗"), + `${formatManagedProcessName(args.processName)} failed ${MANAGED_PROCESS_RESTART_MAX_CONSECUTIVE_FAILURES} consecutive startup attempts; stopping bb`, + ); + } return null; } @@ -3170,6 +3249,23 @@ export async function terminateManagedFullStackProcesses( export async function superviseFullStackProcesses( args: SuperviseFullStackProcessesArgs, ): Promise { + const failureCounts = args.failureCounts ?? { daemon: 0, server: 0 }; + const now = args.now ?? Date.now; + const startedAt = new WeakMap(); + const exitedAt = new WeakMap(); + const trackManagedProcessRun = (processRun: ManagedProcessRun): void => { + startedAt.set(processRun, now()); + void processRun.exit.then(() => { + exitedAt.set(processRun, now()); + }); + }; + if (args.processes.serverRun !== null) { + trackManagedProcessRun(args.processes.serverRun); + } + if (args.processes.daemonRun !== null) { + trackManagedProcessRun(args.processes.daemonRun); + } + while (!args.isShutdownRequested()) { const serverRun = args.processes.serverRun; const daemonRun = args.processes.daemonRun; @@ -3212,45 +3308,69 @@ export async function superviseFullStackProcesses( )} - restarting ${formatManagedProcessLabel(exitedProcess.processName)}`, ); + const managedRun = + exitedProcess.processName === "server" ? serverRun : daemonRun; + const runStartedAt = startedAt.get(managedRun); + const runExitedAt = exitedAt.get(managedRun) ?? now(); + if ( + runStartedAt !== undefined && + runExitedAt - runStartedAt >= MANAGED_PROCESS_STABLE_RUNTIME_MS + ) { + failureCounts[exitedProcess.processName] = 0; + } + failureCounts[exitedProcess.processName] += 1; + if (exitedProcess.processName === "server") { - await args.delayMilliseconds({ - ms: MANAGED_PROCESS_RESTART_RETRY_DELAY_MS, - }); - if (args.isShutdownRequested()) { - return "shutdown"; - } const restartedServer = await restartManagedProcess({ context: args.context, delayMilliseconds: args.delayMilliseconds, + failureCounts, isShutdownRequested: args.isShutdownRequested, processName: "server", + ...(args.shutdownSignal === undefined + ? {} + : { shutdownSignal: args.shutdownSignal }), start: args.startServer, }); if (restartedServer === null) { - return "shutdown"; + if (args.isShutdownRequested()) { + return "shutdown"; + } + await terminateManagedFullStackProcesses({ + processes: args.processes, + signal: "SIGTERM", + }); + return "failed"; } + trackManagedProcessRun(restartedServer); continue; } if (args.processes.daemonRun === daemonRun) { args.processes.daemonRun = null; } - await args.delayMilliseconds({ - ms: MANAGED_PROCESS_RESTART_RETRY_DELAY_MS, - }); - if (args.isShutdownRequested()) { - return "shutdown"; - } const restartedDaemon = await restartManagedProcess({ context: args.context, delayMilliseconds: args.delayMilliseconds, + failureCounts, isShutdownRequested: args.isShutdownRequested, processName: "daemon", + ...(args.shutdownSignal === undefined + ? {} + : { shutdownSignal: args.shutdownSignal }), start: args.startDaemon, }); if (restartedDaemon === null) { - return "shutdown"; + if (args.isShutdownRequested()) { + return "shutdown"; + } + await terminateManagedFullStackProcesses({ + processes: args.processes, + signal: "SIGTERM", + }); + return "failed"; } + trackManagedProcessRun(restartedDaemon); } return "shutdown"; } @@ -3263,6 +3383,10 @@ export async function completeFullStackSupervision( } if (args.supervisionResult === "shutdown") { process.exitCode = 0; + return; + } + if (args.supervisionResult === "failed") { + process.exitCode = MANAGED_PROCESS_RESTART_EXHAUSTED_EXIT_CODE; } } @@ -3347,21 +3471,35 @@ export async function runBbApp( return; } - const runtime = await resolveBbAppRuntimeState({ - entrypointUrl: import.meta.url, - env: process.env, - homeDir: homedir(), - options: parsedArgs.options, - serverUrlMode: - command.kind === "config" || - command.kind === "env" || - command.kind === "host-daemon" - ? "managed" - : "local", - ...(options.worktreePolicy === null - ? {} - : { worktreePolicy: options.worktreePolicy }), - }); + let runtime: BbAppRuntimeState; + try { + runtime = + options.runtimeState ?? + (await resolveBbAppRuntimeState({ + entrypointUrl: import.meta.url, + env: process.env, + homeDir: homedir(), + options: parsedArgs.options, + serverUrlMode: + command.kind === "config" || + command.kind === "env" || + command.kind === "host-daemon" + ? "managed" + : "local", + ...(options.worktreePolicy === null + ? {} + : { worktreePolicy: options.worktreePolicy }), + })); + } catch (error) { + if (command.kind !== "start" || !isStartupStorageError(error)) { + throw error; + } + reportTerminalStartupStorageFailure({ + error, + message: "could not read startup configuration", + }); + return; + } if (command.kind === "start") { const configuredServerBindHost = runtime.serverEnv.BB_SERVER_BIND_HOST; @@ -3434,6 +3572,8 @@ export async function runBbApp( assertBbAppArtifacts(runtime.context); const context = runtime.context; + const managedProcessDelayMilliseconds = + options.delayMilliseconds ?? delayMilliseconds; const serverListenerUrl = resolveServerListenerUrl({ bindHost: runtime.serverEnv.BB_SERVER_BIND_HOST, port: context.serverPort, @@ -3454,16 +3594,29 @@ export async function runBbApp( warnExistingDaemonLock(runtime.context.daemonLockDir); } - const runtimeRecordOwned = await claimBbAppRuntimeFile({ - dataDir: context.dataDir, - entryPath: resolveLauncherEntryPath(), - pid: process.pid, - serverUrl: context.serverUrl, - startedAt: new Date().toISOString(), - surface: - parseAppSurface(runtime.env[APP_SURFACE_ENV_NAME]) ?? DEFAULT_APP_SURFACE, - version: context.appVersion, - }); + let runtimeRecordOwned: boolean; + try { + runtimeRecordOwned = await claimBbAppRuntimeFile({ + dataDir: context.dataDir, + entryPath: resolveLauncherEntryPath(), + pid: process.pid, + serverUrl: context.serverUrl, + startedAt: new Date().toISOString(), + surface: + parseAppSurface(runtime.env[APP_SURFACE_ENV_NAME]) ?? + DEFAULT_APP_SURFACE, + version: context.appVersion, + }); + } catch (error) { + if (!isStartupStorageError(error)) { + throw error; + } + reportTerminalStartupStorageFailure({ + error, + message: `could not create runtime file in ${context.dataDir}`, + }); + return; + } if (!runtimeRecordOwned) { warnExistingRuntimeRecord(context.dataDir); } @@ -3472,6 +3625,8 @@ export async function runBbApp( daemonRun: null, serverRun: null, }; + const failureCounts: ManagedProcessFailureCounts = { daemon: 0, server: 0 }; + const shutdownController = new AbortController(); let shuttingDown = false; let shutdownPromise: Promise | null = null; @@ -3481,6 +3636,7 @@ export async function runBbApp( return shutdownPromise; } shuttingDown = true; + shutdownController.abort(); shutdownPromise = (async () => { process.stdout.write("\n"); log(dim("●"), "Shutting down"); @@ -3516,10 +3672,26 @@ export async function runBbApp( context, processName: "server", }); - outputBuffer.flush(); - process.exitCode = 1; - await shutdown("SIGTERM"); - return; + failureCounts.server += 1; + const restartedServer = await restartManagedProcess({ + context, + delayMilliseconds: managedProcessDelayMilliseconds, + failureCounts, + isShutdownRequested, + processName: "server", + shutdownSignal: shutdownController.signal, + start: startServer, + }); + if (restartedServer === null) { + const stoppedBySignal = isShutdownRequested(); + outputBuffer.flush(); + await shutdown("SIGTERM"); + await completeFullStackSupervision({ + shutdownPromise, + supervisionResult: stoppedBySignal ? "shutdown" : "failed", + }); + return; + } } endStep(green("✓"), `Server listening on ${cyan(serverListenerUrl)}`); @@ -3546,10 +3718,26 @@ export async function runBbApp( context, processName: "daemon", }); - outputBuffer.flush(); - process.exitCode = 1; - await shutdown("SIGTERM"); - return; + failureCounts.daemon += 1; + const restartedDaemon = await restartManagedProcess({ + context, + delayMilliseconds: managedProcessDelayMilliseconds, + failureCounts, + isShutdownRequested, + processName: "daemon", + shutdownSignal: shutdownController.signal, + start: startDaemon, + }); + if (restartedDaemon === null) { + const stoppedBySignal = isShutdownRequested(); + outputBuffer.flush(); + await shutdown("SIGTERM"); + await completeFullStackSupervision({ + shutdownPromise, + supervisionResult: stoppedBySignal ? "shutdown" : "failed", + }); + return; + } } endStep(green("✓"), "Host daemon running"); @@ -3569,23 +3757,42 @@ export async function runBbApp( outputBuffer.flush(); const supervisionResult = await superviseFullStackProcesses({ context, - delayMilliseconds, + delayMilliseconds: managedProcessDelayMilliseconds, + failureCounts, isShutdownRequested, processes, + shutdownSignal: shutdownController.signal, startDaemon, startServer, }); await completeFullStackSupervision({ shutdownPromise, supervisionResult }); } catch (error) { await shutdown("SIGTERM"); + if (isStartupStorageError(error)) { + reportTerminalStartupStorageFailure({ + error, + message: "startup storage operation failed", + }); + return; + } throw error; } finally { removeSignalForwarding(); if (runtimeRecordOwned) { - await clearOwnBbAppRuntimeFile({ - dataDir: context.dataDir, - pid: process.pid, - }); + try { + await clearOwnBbAppRuntimeFile({ + dataDir: context.dataDir, + pid: process.pid, + }); + } catch (error) { + if (!isStartupStorageError(error)) { + throw error; + } + reportTerminalStartupStorageFailure({ + error, + message: `could not remove runtime file in ${context.dataDir}`, + }); + } } } } diff --git a/packages/bb-app/test/index.test.ts b/packages/bb-app/test/index.test.ts index 57dded4509..bf86028b54 100644 --- a/packages/bb-app/test/index.test.ts +++ b/packages/bb-app/test/index.test.ts @@ -1,5 +1,7 @@ import { execFileSync, spawn } from "node:child_process"; import { + chmodSync, + existsSync, mkdirSync, mkdtempSync, readdirSync, @@ -23,7 +25,10 @@ import { resolvePortFromEnv } from "@bb/config/runtime"; import { assertBbAppArtifacts, assertBbHostArtifacts, + calculateManagedProcessRestartDelay, completeFullStackSupervision, + delayMilliseconds, + MANAGED_PROCESS_RESTART_EXHAUSTED_EXIT_CODE, createDaemonEnv, createHostEnrollKeyRequestBody, createServerEnv, @@ -2090,7 +2095,7 @@ describe("bb-app launcher", () => { } }); - it("throttles repeated healthy child exits before restarting", async () => { + it("backs off repeated short-lived child lifecycles before restarting", async () => { const restartThrottle = new ControlledDelay(); const supervisor = createFakeSupervisor(); const firstServerRun = supervisor.serverRuns[0]; @@ -2125,7 +2130,7 @@ describe("bb-app launcher", () => { delay: restartThrottle, index: 1, }); - expect(secondDelay.ms).toBe(1_000); + expect(secondDelay.ms).toBe(2_000); expect(supervisor.serverRuns).toHaveLength(2); expect(supervisor.processes.serverRun).toBeNull(); secondDelay.resolve(); @@ -2142,6 +2147,403 @@ describe("bb-app launcher", () => { ); }); + it("aborts an active restart backoff during shutdown", async () => { + const shutdownController = new AbortController(); + const supervisor = createFakeSupervisor(); + const initialServerRun = supervisor.serverRuns[0]; + let delaySignal: AbortSignal | undefined; + const supervision = superviseFullStackProcesses({ + context: createTestStartContext(), + delayMilliseconds: ({ signal }) => + new Promise((resolvePromise) => { + delaySignal = signal; + signal?.addEventListener("abort", () => resolvePromise(), { + once: true, + }); + }), + isHealthyServerAnswering: async () => false, + isShutdownRequested: supervisor.shutdownRequested, + processes: supervisor.processes, + shutdownSignal: shutdownController.signal, + startDaemon: supervisor.daemonStart, + startServer: supervisor.serverStart, + }); + + initialServerRun.exitWith({ code: 1, signal: null }); + for ( + let attempt = 0; + attempt < 50 && delaySignal === undefined; + attempt += 1 + ) { + await delay({ ms: 1 }); + } + expect(delaySignal).toBe(shutdownController.signal); + + supervisor.setShutdownRequested(true); + const shutdownPromise = terminateManagedFullStackProcesses({ + processes: supervisor.processes, + signal: "SIGTERM", + }); + shutdownController.abort(); + + await expect(Promise.race([supervision, delay({ ms: 100 })])).resolves.toBe( + "shutdown", + ); + await shutdownPromise; + expect(supervisor.serverRuns).toHaveLength(1); + }); + + it("cancels the default restart delay when shutdown is requested", async () => { + const shutdownController = new AbortController(); + const restartDelay = delayMilliseconds({ + ms: 60_000, + signal: shutdownController.signal, + }); + + shutdownController.abort(); + + await expect( + Promise.race([restartDelay, delay({ ms: 100 })]), + ).resolves.toBeUndefined(); + }); + + it("resets restart backoff after a sustained child lifecycle", async () => { + const restartThrottle = new ControlledDelay(); + const supervisor = createFakeSupervisor(); + const initialServerRun = supervisor.serverRuns[0]; + let now = 0; + const supervision = superviseFullStackProcesses({ + context: createTestStartContext(), + delayMilliseconds: (args) => restartThrottle.delayMilliseconds(args), + isHealthyServerAnswering: async () => false, + isShutdownRequested: supervisor.shutdownRequested, + now: () => now, + processes: supervisor.processes, + startDaemon: supervisor.daemonStart, + startServer: supervisor.serverStart, + }); + + initialServerRun.exitWith({ code: 1, signal: null }); + const firstDelay = await waitForDelayCall({ + delay: restartThrottle, + index: 0, + }); + expect(firstDelay.ms).toBe(1_000); + firstDelay.resolve(); + + const restartedServer = await waitForProcessReplacement({ + currentRun: () => supervisor.processes.serverRun, + previousRun: initialServerRun, + }); + now = 60_000; + expect(restartedServer).toBe(supervisor.serverRuns[1]); + supervisor.serverRuns[1]!.exitWith({ + code: 1, + signal: null, + }); + const secondDelay = await waitForDelayCall({ + delay: restartThrottle, + index: 1, + }); + expect(secondDelay.ms).toBe(1_000); + + const stopped = stopFakeSupervisor(supervisor, supervision); + secondDelay.resolve(); + await expect(stopped).resolves.toBe("shutdown"); + }); + + it("uses a child exit time rather than another child's retry time for backoff reset", async () => { + const restartThrottle = new ControlledDelay(); + const supervisor = createFakeSupervisor(); + const initialServerRun = supervisor.serverRuns[0]; + const initialDaemonRun = supervisor.daemonRuns[0]; + let now = 0; + const supervision = superviseFullStackProcesses({ + context: createTestStartContext(), + delayMilliseconds: (args) => restartThrottle.delayMilliseconds(args), + failureCounts: { daemon: 4, server: 0 }, + isHealthyServerAnswering: async () => false, + isShutdownRequested: supervisor.shutdownRequested, + now: () => now, + processes: supervisor.processes, + startDaemon: supervisor.daemonStart, + startServer: supervisor.serverStart, + }); + + initialServerRun.exitWith({ code: 1, signal: null }); + const serverDelay = await waitForDelayCall({ + delay: restartThrottle, + index: 0, + }); + now = 1; + initialDaemonRun.exitWith({ code: 1, signal: null }); + await Promise.resolve(); + now = 60_001; + serverDelay.resolve(); + + const daemonDelay = await waitForDelayCall({ + delay: restartThrottle, + index: 1, + }); + expect(daemonDelay.ms).toBe(16_000); + + const stopped = stopFakeSupervisor(supervisor, supervision); + daemonDelay.resolve(); + await expect(stopped).resolves.toBe("shutdown"); + }); + + it("backs off then stops after bounded consecutive startup failures", async () => { + const restartThrottle = new ControlledDelay(); + const supervisor = createFakeSupervisor(); + const initialServerRun = supervisor.serverRuns[0]; + const startupFailure = new Error("data directory is unwritable"); + const supervision = superviseFullStackProcesses({ + context: createTestStartContext(), + delayMilliseconds: (args) => restartThrottle.delayMilliseconds(args), + isHealthyServerAnswering: async () => false, + isShutdownRequested: supervisor.shutdownRequested, + processes: supervisor.processes, + startDaemon: supervisor.daemonStart, + startServer: async () => Promise.reject(startupFailure), + }); + + initialServerRun.exitWith({ code: 1, signal: null }); + for (const [index, expectedDelay] of [ + 1_000, 2_000, 4_000, 8_000, 16_000, 32_000, 60_000, + ].entries()) { + const delayCall = await waitForDelayCall({ + delay: restartThrottle, + index, + }); + expect(delayCall.ms).toBe(expectedDelay); + delayCall.resolve(); + } + + const supervisionResult = await supervision; + expect(supervisionResult).toBe("failed"); + expect(supervisor.serverRuns).toHaveLength(1); + expect(supervisor.daemonRuns[0]?.terminationSignals).toEqual(["SIGTERM"]); + const previousExitCode = process.exitCode; + try { + process.exitCode = 0; + await completeFullStackSupervision({ + shutdownPromise: null, + supervisionResult, + }); + expect(process.exitCode).toBe( + MANAGED_PROCESS_RESTART_EXHAUSTED_EXIT_CODE, + ); + } finally { + process.exitCode = previousExitCode; + } + }); + + it("calculates capped exponential managed-process restart delays", () => { + expect(calculateManagedProcessRestartDelay(1)).toBe(1_000); + expect(calculateManagedProcessRestartDelay(2)).toBe(2_000); + expect(calculateManagedProcessRestartDelay(7)).toBe(60_000); + }); + + it("reports unreadable startup configuration with the terminal code", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "bb-app-unreadable-data-")); + chmodSync(dataDir, 0o400); + const previousExitCode = process.exitCode; + const stderr = vi + .spyOn(process.stderr, "write") + .mockImplementation(() => true); + + try { + process.exitCode = 0; + await runBbApp(["--data-dir", dataDir], { worktreePolicy: null }); + + expect(process.exitCode).toBe( + MANAGED_PROCESS_RESTART_EXHAUSTED_EXIT_CODE, + ); + expect(stderr).toHaveBeenCalledWith( + expect.stringMatching( + /bb-app: could not read startup configuration.*(?:EACCES|permission denied)/u, + ), + ); + } finally { + stderr.mockRestore(); + chmodSync(dataDir, 0o700); + process.exitCode = previousExitCode; + } + }); + + it("reports an unwritable runtime file and exits with the terminal code", async () => { + const root = mkdtempSync(join(tmpdir(), "bb-app-runtime-file-")); + const dataDir = join(root, "data"); + const appDistDir = join(root, "app"); + const daemonBundleDir = join(root, "host-daemon"); + const chunksDir = join(daemonBundleDir, "bb-chunks"); + mkdirSync(dataDir, { recursive: true }); + mkdirSync(appDistDir, { recursive: true }); + mkdirSync(chunksDir, { recursive: true }); + writeFileSync(join(appDistDir, "index.html"), ""); + writeFileSync(join(daemonBundleDir, "daemon-bundle.mjs"), ""); + writeFileSync(join(daemonBundleDir, "bb"), ""); + writeFileSync(join(chunksDir, "chunk.js"), ""); + writeFileSync(join(daemonBundleDir, "bb-provider-bridge-worker.mjs"), ""); + writeFileSync(join(daemonBundleDir, "bb-parcel-watcher-child.mjs"), ""); + writeFileSync(join(daemonBundleDir, "bb-plugin-host-worker.mjs"), ""); + writeFileSync(join(root, "server.mjs"), ""); + chmodSync(dataDir, 0o500); + const previousExitCode = process.exitCode; + const stderr = vi + .spyOn(process.stderr, "write") + .mockImplementation(() => true); + + try { + process.exitCode = 0; + await runBbApp([], { + runtimeState: { + config: {}, + context: { + appDistDir, + appVersion: "0.0.0-test", + configFile: join(dataDir, "config.json"), + daemonBundleDir, + daemonEntry: join(daemonBundleDir, "daemon-bundle.mjs"), + daemonLockDir: join(dataDir, "daemon.lock.lock"), + daemonLockFile: join(dataDir, "daemon.lock"), + daemonPort: 49387, + dataDir, + dbPath: join(dataDir, "bb.db"), + envFile: join(dataDir, "env.json"), + logDir: join(dataDir, "logs"), + packageRoot: root, + serverEntry: join(root, "server.mjs"), + serverPort: 49386, + serverUrl: "http://127.0.0.1:49386", + }, + env: {}, + serverEnv: {}, + }, + worktreePolicy: null, + }); + + expect(process.exitCode).toBe( + MANAGED_PROCESS_RESTART_EXHAUSTED_EXIT_CODE, + ); + expect(stderr).toHaveBeenCalledWith( + expect.stringMatching( + /bb-app: could not create runtime file.*(?:EACCES|permission denied)/u, + ), + ); + expect(existsSync(join(dataDir, "bb-app-runtime.json"))).toBe(false); + } finally { + stderr.mockRestore(); + chmodSync(dataDir, 0o700); + process.exitCode = previousExitCode; + } + }); + + it("bounds unwritable data startup failures through the real bb-app supervisor path", async () => { + const root = mkdtempSync(join(tmpdir(), "bb-app-cold-start-")); + const dataDir = join(root, "data"); + const logsDir = join(dataDir, "logs"); + const appDistDir = join(root, "app"); + const daemonBundleDir = join(root, "host-daemon"); + const chunksDir = join(daemonBundleDir, "bb-chunks"); + const attemptsPath = join(root, "attempts"); + const serverEntry = join(root, "server.mjs"); + const restartDelays: number[] = []; + mkdirSync(logsDir, { recursive: true }); + mkdirSync(appDistDir, { recursive: true }); + mkdirSync(chunksDir, { recursive: true }); + writeFileSync(join(appDistDir, "index.html"), ""); + writeFileSync(join(daemonBundleDir, "daemon-bundle.mjs"), ""); + writeFileSync(join(daemonBundleDir, "bb"), ""); + writeFileSync(join(chunksDir, "chunk.js"), ""); + writeFileSync(join(daemonBundleDir, "bb-provider-bridge-worker.mjs"), ""); + writeFileSync(join(daemonBundleDir, "bb-parcel-watcher-child.mjs"), ""); + writeFileSync(join(daemonBundleDir, "bb-plugin-host-worker.mjs"), ""); + writeFileSync( + serverEntry, + `import { appendFileSync, chmodSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +appendFileSync(process.env.BB_TEST_ATTEMPTS_PATH, "attempt\\n"); +chmodSync(process.env.BB_DATA_DIR, 0o500); +try { + writeFileSync(join(process.env.BB_DATA_DIR, "logs", "probe"), "x"); +} catch (error) { + process.stderr.write(\`SqliteError: \${error.message}\\n\`); +} +process.exitCode = 1; +`, + ); + chmodSync(logsDir, 0o500); + const previousExitCode = process.exitCode; + const stderr = vi + .spyOn(process.stderr, "write") + .mockImplementation(() => true); + + try { + process.exitCode = 0; + await runBbApp([], { + delayMilliseconds: async ({ ms }) => { + restartDelays.push(ms); + }, + runtimeState: { + config: {}, + context: { + appDistDir, + appVersion: "0.0.0-test", + configFile: join(dataDir, "config.json"), + daemonBundleDir, + daemonEntry: join(daemonBundleDir, "daemon-bundle.mjs"), + daemonLockDir: join(dataDir, "daemon.lock.lock"), + daemonLockFile: join(dataDir, "daemon.lock"), + daemonPort: 49387, + dataDir, + dbPath: join(dataDir, "bb.db"), + envFile: join(dataDir, "env.json"), + logDir: logsDir, + packageRoot: root, + serverEntry, + serverPort: 49386, + serverUrl: "http://127.0.0.1:49386", + }, + env: { + BB_DATA_DIR: dataDir, + BB_TEST_ATTEMPTS_PATH: attemptsPath, + }, + serverEnv: { + BB_DATA_DIR: dataDir, + BB_TEST_ATTEMPTS_PATH: attemptsPath, + }, + }, + worktreePolicy: null, + }); + + expect( + readFileSync(attemptsPath, "utf8").trim().split("\n"), + ).toHaveLength(8); + expect(restartDelays).toEqual([ + 1_000, 2_000, 4_000, 8_000, 16_000, 32_000, 60_000, + ]); + expect(process.exitCode).toBe( + MANAGED_PROCESS_RESTART_EXHAUSTED_EXIT_CODE, + ); + expect( + readdirSync(logsDir).filter((entry) => + entry.startsWith("process-server-startupFailure-"), + ), + ).toHaveLength(0); + expect(stderr).toHaveBeenCalledWith( + expect.stringMatching( + /bb-app: could not remove runtime file.*(?:EACCES|permission denied)/u, + ), + ); + } finally { + stderr.mockRestore(); + chmodSync(dataDir, 0o700); + chmodSync(logsDir, 0o700); + process.exitCode = previousExitCode; + } + }); + it("limits npm package metadata to documented runtimes", () => { const metadata = readPackageMetadata(); diff --git a/packages/process-utils/src/index.ts b/packages/process-utils/src/index.ts index e9073e948c..1d8e98ed0a 100644 --- a/packages/process-utils/src/index.ts +++ b/packages/process-utils/src/index.ts @@ -1,7 +1,13 @@ export * from "./plugin-process-paths.js"; import type { ChildProcess, StdioOptions } from "node:child_process"; import { randomUUID } from "node:crypto"; -import { mkdirSync, writeFileSync } from "node:fs"; +import { + mkdirSync, + readdirSync, + renameSync, + unlinkSync, + writeFileSync, +} from "node:fs"; import { lstat, readdir, readlink, realpath } from "node:fs/promises"; import { basename, @@ -100,6 +106,7 @@ interface WriteSafeProcessDiagnosticReportArgs extends SafeProcessDiagnosticsOpt const MAX_DIAGNOSTIC_ERROR_CAUSE_DEPTH = 8; const MAX_DIAGNOSTIC_AGGREGATE_ERRORS = 8; +const MAX_SAFE_PROCESS_DIAGNOSTIC_REPORTS_PER_PROCESS_AND_KIND = 5; interface SafeProcessDiagnosticError { name: string; @@ -572,12 +579,10 @@ export function writeSafeProcessDiagnosticReport( (args.createReportId ?? randomUUID)(), ); const processName = sanitizeDiagnosticFilenamePart(args.processName); - const reportPath = join( - args.logsDir, - `process-${processName}-${args.kind}-${formatDiagnosticTimestamp( - occurredAt, - )}-${reportId}.json`, - ); + const reportFileName = `process-${processName}-${args.kind}-${formatDiagnosticTimestamp( + occurredAt, + )}-${reportId}.json`; + const reportPath = join(args.logsDir, reportFileName); const report: SafeProcessDiagnosticReport = { diagnosticVersion: 1, kind: args.kind, @@ -593,9 +598,33 @@ export function writeSafeProcessDiagnosticReport( error: serializeDiagnosticError(args.error), }; - writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`, { - encoding: "utf8", - }); + const reportPrefix = `process-${processName}-${args.kind}-`; + const existingReports = readdirSync(args.logsDir) + .filter( + (entry) => entry.startsWith(reportPrefix) && entry.endsWith(".json"), + ) + .sort(); + for ( + let index = 0; + index <= + existingReports.length - + MAX_SAFE_PROCESS_DIAGNOSTIC_REPORTS_PER_PROCESS_AND_KIND; + index += 1 + ) { + unlinkSync(join(args.logsDir, existingReports[index]!)); + } + const temporaryReportPath = join(args.logsDir, `.${reportFileName}.tmp`); + try { + writeFileSync(temporaryReportPath, `${JSON.stringify(report, null, 2)}\n`, { + encoding: "utf8", + }); + renameSync(temporaryReportPath, reportPath); + } catch (error) { + try { + unlinkSync(temporaryReportPath); + } catch {} + throw error; + } return reportPath; } diff --git a/packages/process-utils/test/index.test.ts b/packages/process-utils/test/index.test.ts index a4f0980cb4..befc2be1b2 100644 --- a/packages/process-utils/test/index.test.ts +++ b/packages/process-utils/test/index.test.ts @@ -1,5 +1,12 @@ import { once } from "node:events"; -import { existsSync, mkdtempSync, readFileSync } from "node:fs"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + statSync, +} from "node:fs"; import { join } from "node:path"; import { tmpdir } from "node:os"; import { describe, expect, it } from "vitest"; @@ -108,6 +115,59 @@ describe("process utils", () => { } }); + it("caps diagnostic reports at five per process and kind", () => { + const logsDir = join( + mkdtempSync(join(tmpdir(), "bb-process-utils-report-")), + "logs", + ); + for (let index = 0; index < 6; index += 1) { + writeSafeProcessDiagnosticReport({ + kind: "startupFailure", + logsDir, + processName: "test-process", + error: new Error(`startup failed ${index}`), + now: () => new Date(`2026-06-01T12:00:0${index}.000Z`), + createReportId: () => `report-id-${index}`, + }); + } + + const reports = readdirSync(logsDir).filter((entry) => + entry.startsWith("process-test-process-startupFailure-"), + ); + expect(reports).toHaveLength(5); + expect(reports).not.toContain( + "process-test-process-startupFailure-2026-06-01T12-00-00-000Z-report-id-0.json", + ); + for (const report of reports) { + expect(statSync(join(logsDir, report)).size).toBeGreaterThan(0); + } + }); + + it("cleans up a complete temporary report when its final rename fails", () => { + const logsDir = join( + mkdtempSync(join(tmpdir(), "bb-process-utils-report-")), + "logs", + ); + const reportFileName = + "process-test-process-startupFailure-2026-06-01T12-00-00-000Z-rename-failure.json"; + mkdirSync(join(logsDir, reportFileName), { recursive: true }); + + expect(() => + writeSafeProcessDiagnosticReport({ + kind: "startupFailure", + logsDir, + processName: "test-process", + error: new Error("data directory is unwritable"), + now: () => new Date("2026-06-01T12:00:00.000Z"), + createReportId: () => "rename-failure", + }), + ).toThrow(); + expect(existsSync(join(logsDir, `.${reportFileName}.tmp`))).toBe(false); + expect( + readdirSync(logsDir).filter((entry) => entry.endsWith(".tmp")), + ).toEqual([]); + }); + it("writes nested error causes", () => { const logsDir = join( mkdtempSync(join(tmpdir(), "bb-process-utils-report-")),