From aa2b128c041ba1859bdc72b4d3215abc2bdf8ae0 Mon Sep 17 00:00:00 2001 From: James Date: Sat, 25 Jul 2026 07:49:28 +0000 Subject: [PATCH] fix(producer): type video extraction failures --- packages/engine/src/index.ts | 6 + .../src/services/videoFrameExtractor.test.ts | 157 ++++++ .../src/services/videoFrameExtractor.ts | 526 ++++++++++++++++-- .../engine/src/utils/urlDownloader.test.ts | 13 +- packages/engine/src/utils/urlDownloader.ts | 5 +- .../producer/src/server.errorCode.test.ts | 30 + packages/producer/src/server.ts | 21 + .../producer/src/services/distributed/plan.ts | 1 + .../src/services/render/observability.ts | 2 + .../render/stages/extractVideosStage.test.ts | 153 ++++- .../render/stages/extractVideosStage.ts | 150 ++++- .../src/services/renderOrchestrator.ts | 4 + 12 files changed, 1009 insertions(+), 59 deletions(-) create mode 100644 packages/producer/src/server.errorCode.test.ts diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts index 5996ee7a29..26ad8e12ab 100644 --- a/packages/engine/src/index.ts +++ b/packages/engine/src/index.ts @@ -184,12 +184,18 @@ export { createFrameLookupTable, FrameLookupTable, analyzeClipMediaFit, + classifyVideoExtractionError, + isVideoSourceExtractionError, + runVideoExtractionWithRetry, + VideoSourceExtractionError, type VideoElement, type ImageElement, type ExtractedFrames, type ExtractionOptions, type ExtractionResult, type ExtractionPhaseBreakdown, + type VideoExtractionFailure, + type VideoExtractionFailureKind, type VideoFrameFormat, VIDEO_FRAME_FORMATS, isVideoFrameFormat, diff --git a/packages/engine/src/services/videoFrameExtractor.test.ts b/packages/engine/src/services/videoFrameExtractor.test.ts index 89ad8b29dc..d494565db9 100644 --- a/packages/engine/src/services/videoFrameExtractor.test.ts +++ b/packages/engine/src/services/videoFrameExtractor.test.ts @@ -1,3 +1,4 @@ +// fallow-ignore-file code-duplication import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { existsSync, @@ -25,6 +26,9 @@ import { decoderForCodec, getFrameAtTime, analyzeClipMediaFit, + classifyVideoExtractionError, + runVideoExtractionWithRetry, + VideoSourceExtractionError, type VideoElement, type ExtractedFrames, type ExtractionResult, @@ -41,6 +45,120 @@ import { COMPLETE_SENTINEL, GC_MARKER, SCHEMA_PREFIX } from "./extractionCache.j // synthesized VFR fixture. const HAS_FFMPEG = spawnSync("ffmpeg", ["-version"]).status === 0; +describe("video extraction failure taxonomy and bounded retry", () => { + it("classifies missing and transient HTTP sources without exposing retry ambiguity", () => { + expect(classifyVideoExtractionError(new Error("HTTP 404: Not Found"))).toMatchObject({ + kind: "download_not_found", + retryable: false, + }); + expect(classifyVideoExtractionError(new Error("HTTP 503: Service Unavailable"))).toMatchObject({ + kind: "download_transient", + retryable: true, + }); + }); + + it("retries one transient failure, cleaning partial output before the retry", async () => { + const retryDir = mkdtempSync(join(tmpdir(), "hf-extract-retry-")); + const partialPath = join(retryDir, "frame-00001.jpg"); + let attempts = 0; + try { + const outcome = await runVideoExtractionWithRetry( + async () => { + attempts += 1; + if (attempts === 1) { + writeFileSync(partialPath, "partial"); + throw new VideoSourceExtractionError( + "ffmpeg_timeout", + true, + "Video frame extraction timed out", + ); + } + expect(existsSync(partialPath)).toBe(false); + return "frames"; + }, + { + maxTransientRetries: 1, + onRetry: () => { + rmSync(retryDir, { recursive: true, force: true }); + mkdirSync(retryDir, { recursive: true }); + }, + }, + ); + + expect(outcome).toEqual({ result: "frames", retries: 1 }); + expect(attempts).toBe(2); + } finally { + rmSync(retryDir, { recursive: true, force: true }); + } + }); + + it("does not retry deterministic or caller-aborted failures", async () => { + let deterministicAttempts = 0; + await expect( + runVideoExtractionWithRetry(async () => { + deterministicAttempts += 1; + throw new VideoSourceExtractionError( + "zero_output", + false, + "Video source produced no decodable frames", + ); + }), + ).rejects.toMatchObject({ kind: "zero_output", retryable: false }); + expect(deterministicAttempts).toBe(1); + + const controller = new AbortController(); + controller.abort(); + let abortedAttempts = 0; + await expect( + runVideoExtractionWithRetry( + async () => { + abortedAttempts += 1; + throw new VideoSourceExtractionError( + "download_transient", + true, + "Video source download failed transiently", + ); + }, + { signal: controller.signal }, + ), + ).rejects.toMatchObject({ kind: "cancelled", retryable: false }); + expect(abortedAttempts).toBe(0); + }); + + it("does not retry transient extraction failures unless the caller opts in", async () => { + let attempts = 0; + await expect( + runVideoExtractionWithRetry(async () => { + attempts += 1; + throw new VideoSourceExtractionError( + "ffmpeg_timeout", + true, + "Video frame extraction timed out", + ); + }), + ).rejects.toMatchObject({ kind: "ffmpeg_timeout", retryable: true }); + expect(attempts).toBe(1); + }); + + it("fails closed to zero retries for a non-finite runtime retry budget", async () => { + let attempts = 0; + await expect( + runVideoExtractionWithRetry( + async () => { + attempts += 1; + throw new VideoSourceExtractionError( + "ffmpeg_timeout", + true, + "Video frame extraction timed out", + ); + }, + { maxTransientRetries: Number.NaN }, + ), + ).rejects.toMatchObject({ kind: "ffmpeg_timeout", retryable: true }); + expect(attempts).toBe(1); + }); +}); + // Codec-based alpha defaulting replaces tag-based detection (the // alpha_mode/ALPHA_MODE case bug — see ffprobe.test.ts for the regression // pin on that). The extractor uses these helpers for two decisions: @@ -966,6 +1084,45 @@ describe.skipIf(!HAS_FFMPEG)("extractAllVideoFrames on a VFR source", () => { return src; } + it("rejects a media start beyond source duration before invoking FFmpeg", async () => { + const src = await synthCfrClip("zero-output-src.mp4", 1); + const outputDir = join(FIXTURE_DIR, "out-zero-output"); + await expect( + extractVideoFramesRange(src, "past-eof", 2, 1, { fps: 30, outputDir }), + ).rejects.toMatchObject({ + kind: "media_start_out_of_range", + retryable: false, + }); + }, 60_000); + + it("preserves legacy metadata rejection unless typed aggregation is explicitly enabled", async () => { + const src = join(FIXTURE_DIR, "invalid-probe.mp4"); + writeFileSync(src, "not a media container"); + const video = cfrClipElement("invalid-probe", src, 1); + + await expect( + extractAllVideoFrames([video], FIXTURE_DIR, { + fps: 30, + outputDir: join(FIXTURE_DIR, "out-invalid-probe-legacy"), + }), + ).rejects.toThrow(); + + const collected = await extractAllVideoFrames([video], FIXTURE_DIR, { + fps: 30, + outputDir: join(FIXTURE_DIR, "out-invalid-probe-typed"), + collectProbeFailures: true, + }); + expect(collected.success).toBe(false); + expect(collected.extracted).toEqual([]); + expect(collected.errors).toEqual([ + expect.objectContaining({ + videoId: "invalid-probe", + kind: "invalid_media", + retryable: false, + }), + ]); + }, 60_000); + async function synthHdrTaggedClip(name: string, durationSeconds: number): Promise { const src = join(FIXTURE_DIR, name); const synth = await runFfmpeg([ diff --git a/packages/engine/src/services/videoFrameExtractor.ts b/packages/engine/src/services/videoFrameExtractor.ts index 953b3433f8..b27e137976 100644 --- a/packages/engine/src/services/videoFrameExtractor.ts +++ b/packages/engine/src/services/videoFrameExtractor.ts @@ -17,7 +17,7 @@ import { isHdrColorSpace as isHdrColorSpaceUtil, type HdrTransfer, } from "../utils/hdr.js"; -import { downloadToTemp, isHttpUrl } from "../utils/urlDownloader.js"; +import { downloadToTemp, isHttpUrl, UrlDownloadError } from "../utils/urlDownloader.js"; import { runFfmpeg } from "../utils/runFfmpeg.js"; import { DEFAULT_CONFIG, type EngineConfig } from "../config.js"; import { unwrapTemplate } from "../utils/htmlTemplate.js"; @@ -83,6 +83,17 @@ export interface ExtractionOptions { quality?: number; format?: VideoFrameFormat; sdrToHdrTransfer?: HdrTransfer; + /** + * Bounded per-source FFmpeg retries. Default 0 preserves stable behavior; + * the producer may canary at most one retry after observing typed failures. + */ + maxTransientRetries?: number; + /** + * Collect metadata-probe failures into `ExtractionResult.errors` instead + * of preserving the legacy Promise rejection. Default false; only the + * candidate enforce lane may opt into typed aggregation. + */ + collectProbeFailures?: boolean; } const EXTRACT_CACHE_MIN_AGE_MS = 60 * 60 * 1000; @@ -136,12 +147,267 @@ export interface ExtractionPhaseBreakdown { extractMs: number; cacheHits: number; cacheMisses: number; + /** Number of per-source transient failures retried inside this extraction. */ + transientRetries?: number; +} + +export type VideoExtractionFailureKind = + | "cancelled" + | "source_missing" + | "source_rejected" + | "download_not_found" + | "download_transient" + | "invalid_media" + | "media_start_out_of_range" + | "ffmpeg_unavailable" + | "ffmpeg_timeout" + | "ffmpeg_transient" + | "ffmpeg_failed" + | "zero_output" + | "internal"; + +export interface VideoExtractionFailure { + videoId: string; + /** Always populated by this engine version; optional for source compatibility with older consumers. */ + kind?: VideoExtractionFailureKind; + /** Always populated by this engine version; absent legacy values fail closed. */ + retryable?: boolean; + /** + * Operator diagnostic retained inside the engine result. Producer-facing + * errors must summarize `kind`/counts and must not forward this field: it + * can contain a local path or a signed source URL. + */ + error: string; +} + +export class VideoSourceExtractionError extends Error { + readonly hyperframesVideoSourceExtractionError = true as const; + + constructor( + readonly kind: VideoExtractionFailureKind, + readonly retryable: boolean, + message: string, + readonly diagnostic: string = message, + ) { + super(message); + this.name = "VideoSourceExtractionError"; + } +} + +export function isVideoSourceExtractionError(error: unknown): error is VideoSourceExtractionError { + return ( + typeof error === "object" && + error !== null && + (error as { hyperframesVideoSourceExtractionError?: unknown }) + .hyperframesVideoSourceExtractionError === true + ); +} + +function boundedTransientRetryBudget(value: number | undefined): 0 | 1 { + return Number.isFinite(value) && (value ?? 0) >= 1 ? 1 : 0; +} + +function errorText(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +/** + * Convert legacy/raw downloader and filesystem errors into the bounded + * extraction taxonomy. New extraction code should throw + * `VideoSourceExtractionError` directly; this classifier keeps older utility + * boundaries safe while they migrate. + */ +export function classifyVideoExtractionError(error: unknown): VideoSourceExtractionError { + if (isVideoSourceExtractionError(error)) return error; + const diagnostic = errorText(error); + const lowered = diagnostic.toLowerCase(); + + if (error instanceof UrlDownloadError) { + if (error.kind === "cancelled") { + return new VideoSourceExtractionError( + "cancelled", + false, + "Video extraction cancelled", + diagnostic, + ); + } + if (error.kind === "http_not_found") { + return new VideoSourceExtractionError( + "download_not_found", + false, + "Video source was not found", + diagnostic, + ); + } + if (error.kind === "http_rejected") { + return new VideoSourceExtractionError( + "source_rejected", + false, + "Video source download was rejected", + diagnostic, + ); + } + if (error.retryable) { + return new VideoSourceExtractionError( + "download_transient", + true, + "Video source download failed transiently", + diagnostic, + ); + } + return new VideoSourceExtractionError( + "internal", + false, + "Video source download failed internally", + diagnostic, + ); + } + if (lowered.includes("cancelled") || lowered.includes("aborted")) { + return new VideoSourceExtractionError( + "cancelled", + false, + "Video extraction cancelled", + diagnostic, + ); + } + if (lowered.includes("video file not found")) { + return new VideoSourceExtractionError( + "source_missing", + false, + "Video source is missing", + diagnostic, + ); + } + if ( + lowered.includes("only https urls are permitted") || + lowered.includes("private/reserved address") || + lowered.includes("invalid url") + ) { + return new VideoSourceExtractionError( + "source_rejected", + false, + "Video source URL is not permitted", + diagnostic, + ); + } + const httpStatus = diagnostic.match(/\bHTTP\s+(\d{3})\b/i)?.[1]; + if (httpStatus) { + const status = Number(httpStatus); + if (status === 404 || status === 410) { + return new VideoSourceExtractionError( + "download_not_found", + false, + "Video source was not found", + diagnostic, + ); + } + if (status === 408 || status === 429 || status >= 500) { + return new VideoSourceExtractionError( + "download_transient", + true, + "Video source download failed transiently", + diagnostic, + ); + } + return new VideoSourceExtractionError( + "source_rejected", + false, + "Video source download was rejected", + diagnostic, + ); + } + if ( + lowered.includes("[urldownloader] download timeout") || + lowered.includes("[urldownloader] download failed") || + lowered.includes("fetch failed") || + lowered.includes("network") + ) { + return new VideoSourceExtractionError( + "download_transient", + true, + "Video source download failed transiently", + diagnostic, + ); + } + if (lowered.includes("ffprobe not found")) { + return new VideoSourceExtractionError( + "ffmpeg_unavailable", + false, + "FFprobe is unavailable", + diagnostic, + ); + } + if (lowered.includes("ffprobe deadline")) { + return new VideoSourceExtractionError( + "ffmpeg_timeout", + true, + "Video inspection timed out", + diagnostic, + ); + } + if ( + lowered.includes("ffprobe") || + lowered.includes("failed to parse ffprobe output") || + lowered.includes("no video stream found") + ) { + return new VideoSourceExtractionError( + "invalid_media", + false, + "Video source could not be inspected", + diagnostic, + ); + } + return new VideoSourceExtractionError( + "internal", + false, + "Video extraction failed internally", + diagnostic, + ); +} + +export async function runVideoExtractionWithRetry( + operation: () => Promise, + options: { + signal?: AbortSignal; + onRetry?: () => Promise | void; + maxTransientRetries?: number; + } = {}, +): Promise<{ result: T; retries: number }> { + const maxTransientRetries = boundedTransientRetryBudget(options.maxTransientRetries); + let retries = 0; + for (;;) { + if (options.signal?.aborted) { + throw new VideoSourceExtractionError("cancelled", false, "Video extraction cancelled"); + } + try { + return { result: await operation(), retries }; + } catch (error) { + const classified = classifyVideoExtractionError(error); + if (options.signal?.aborted) { + throw new VideoSourceExtractionError( + "cancelled", + false, + "Video extraction cancelled", + classified.diagnostic, + ); + } + if ( + classified.kind === "cancelled" || + !classified.retryable || + retries >= maxTransientRetries + ) { + throw classified; + } + retries += 1; + await options.onRetry?.(); + } + } } export interface ExtractionResult { success: boolean; extracted: ExtractedFrames[]; - errors: Array<{ videoId: string; error: string }>; + errors: VideoExtractionFailure[]; totalFramesExtracted: number; durationMs: number; phaseBreakdown: ExtractionPhaseBreakdown; @@ -270,7 +536,28 @@ export async function extractVideoFramesRange( const videoOutputDir = outputDirOverride ?? join(outputDir, videoId); if (!existsSync(videoOutputDir)) mkdirSync(videoOutputDir, { recursive: true }); - const metadata = await extractMediaMetadata(videoPath); + let metadata: VideoMetadata; + try { + metadata = await extractMediaMetadata(videoPath); + } catch (error) { + throw classifyVideoExtractionError(error); + } + if (!(metadata.durationSeconds > 0)) { + throw new VideoSourceExtractionError( + "invalid_media", + false, + "Video source has no positive duration", + `Video source duration is ${metadata.durationSeconds}s`, + ); + } + if (startTime >= metadata.durationSeconds) { + throw new VideoSourceExtractionError( + "media_start_out_of_range", + false, + "Video media start is outside the source duration", + `Video media start ${startTime}s is outside source duration ${metadata.durationSeconds}s`, + ); + } const format = resolveFrameFormat(metadata, options.format); const framePattern = `${FRAME_FILENAME_PREFIX}%05d.${format}`; const outputPattern = join(videoOutputDir, framePattern); @@ -328,13 +615,24 @@ export async function extractVideoFramesRange( const processResult = await runFfmpeg(args, { signal, timeout: ffmpegProcessTimeout }); if (processResult.terminationReason === "abort") { - throw new Error("Video frame extraction cancelled"); + throw new VideoSourceExtractionError("cancelled", false, "Video extraction cancelled"); } if (processResult.terminationReason === "spawn_error") { if ((processResult.error as NodeJS.ErrnoException | undefined)?.code === "ENOENT") { - throw new Error("[FFmpeg] ffmpeg not found"); + throw new VideoSourceExtractionError( + "ffmpeg_unavailable", + false, + "FFmpeg is unavailable", + "[FFmpeg] ffmpeg not found", + ); } - throw processResult.error ?? new Error(processResult.stderr); + const diagnostic = processResult.error?.message || processResult.stderr; + throw new VideoSourceExtractionError( + "ffmpeg_transient", + true, + "FFmpeg could not be started", + diagnostic, + ); } if (!processResult.success) { // With the SDR-to-HDR remap folded into this pass, a filter failure @@ -344,12 +642,30 @@ export async function extractVideoFramesRange( const hdrPrefix = options.sdrToHdrTransfer ? `SDR→HDR conversion failed (colorspace filter in extract pass, target ${options.sdrToHdrTransfer}): ` : ""; - const timeoutSuffix = - processResult.terminationReason === "deadline" - ? ` (timed out after ${ffmpegProcessTimeout} ms)` - : ""; - throw new Error( - `${hdrPrefix}FFmpeg exited with code ${processResult.exitCode}${timeoutSuffix}: ${processResult.stderr.slice(-500)}`, + const timedOut = processResult.terminationReason === "deadline"; + const timeoutSuffix = timedOut ? ` (timed out after ${ffmpegProcessTimeout} ms)` : ""; + const diagnostic = + `${hdrPrefix}FFmpeg exited with code ${processResult.exitCode}${timeoutSuffix}: ` + + processResult.stderr.slice(-500); + if (timedOut) { + throw new VideoSourceExtractionError( + "ffmpeg_timeout", + true, + "Video frame extraction timed out", + diagnostic, + ); + } + const transientIo = + /resource temporarily unavailable|device or resource busy|input\/output error/i.test( + processResult.stderr, + ); + throw new VideoSourceExtractionError( + transientIo ? "ffmpeg_transient" : "ffmpeg_failed", + transientIo, + transientIo + ? "Video frame extraction hit a transient I/O failure" + : "Video source could not be decoded", + diagnostic, ); } @@ -360,6 +676,14 @@ export async function extractVideoFramesRange( files.forEach((file, index) => { framePaths.set(index, join(videoOutputDir, file)); }); + if (framePaths.size === 0 && duration > 0) { + throw new VideoSourceExtractionError( + "zero_output", + false, + "Video source produced no decodable frames", + `FFmpeg exited successfully but produced no frames (start=${startTime}, duration=${duration})`, + ); + } return { videoId, @@ -694,7 +1018,7 @@ export async function extractAllVideoFrames( ): Promise { const startTime = Date.now(); const extracted: ExtractedFrames[] = []; - const errors: Array<{ videoId: string; error: string }> = []; + const errors: VideoExtractionFailure[] = []; let totalFramesExtracted = 0; const breakdown: ExtractionPhaseBreakdown = { resolveMs: 0, @@ -711,6 +1035,10 @@ export async function extractAllVideoFrames( extractMs: 0, cacheHits: 0, cacheMisses: 0, + transientRetries: 0, + }; + const recordTransientRetries = (count: number): void => { + breakdown.transientRetries = (breakdown.transientRetries ?? 0) + count; }; // Phase 1: Resolve paths and download remote videos @@ -730,7 +1058,9 @@ export async function extractAllVideoFrames( if (isHttpUrl(videoPath)) { const downloadDir = join(options.outputDir, "_downloads"); mkdirSync(downloadDir, { recursive: true }); - videoPath = await downloadToTemp(videoPath, downloadDir, undefined, signal); + videoPath = await downloadToTemp(videoPath, downloadDir, undefined, signal, () => + recordTransientRetries(1), + ); } if (!existsSync(videoPath)) { @@ -747,12 +1077,23 @@ export async function extractAllVideoFrames( `(e.g. src="assets/foo.mp4") over "../assets/foo.mp4".\n`, ); } - errors.push({ videoId: video.id, error: `Video file not found: ${videoPath}` }); + errors.push({ + videoId: video.id, + kind: "source_missing", + retryable: false, + error: `Video file not found: ${videoPath}`, + }); continue; } resolvedVideos.push({ video, videoPath }); } catch (err) { - errors.push({ videoId: video.id, error: err instanceof Error ? err.message : String(err) }); + const classified = classifyVideoExtractionError(err); + errors.push({ + videoId: video.id, + kind: classified.kind, + retryable: classified.retryable, + error: classified.diagnostic, + }); } } @@ -782,9 +1123,46 @@ export async function extractAllVideoFrames( // Phase 2: Probe color spaces and normalize if mixed HDR/SDR const phase2ProbeStart = Date.now(); - const videoMetadata = await Promise.all( - resolvedVideos.map(({ videoPath }) => extractMediaMetadata(videoPath)), + const metadataResults = await Promise.all( + resolvedVideos.map(async ({ video, videoPath }, index) => { + try { + // Keep the default/off path byte-for-byte compatible with the legacy + // Promise.all rejection. Classification is introduced only when a + // bounded retry or explicit typed aggregation is enabled. + const attempted = + !options.collectProbeFailures && + boundedTransientRetryBudget(options.maxTransientRetries) === 0 + ? { result: await extractMediaMetadata(videoPath), retries: 0 } + : await runVideoExtractionWithRetry(() => extractMediaMetadata(videoPath), { + signal, + maxTransientRetries: options.maxTransientRetries, + onRetry: () => recordTransientRetries(1), + }); + return { + video, + videoPath, + metadata: attempted.result, + cacheKeyInput: cacheKeyInputs[index] ?? null, + }; + } catch (error) { + if (!options.collectProbeFailures) throw error; + errors.push(extractionError(video.id, error)); + return null; + } + }), ); + const probedVideos = metadataResults.filter((entry) => entry !== null); + resolvedVideos.splice( + 0, + resolvedVideos.length, + ...probedVideos.map(({ video, videoPath }) => ({ video, videoPath })), + ); + cacheKeyInputs.splice( + 0, + cacheKeyInputs.length, + ...probedVideos.map(({ cacheKeyInput }) => cacheKeyInput), + ); + const videoMetadata = probedVideos.map(({ metadata }) => metadata); const videoColorSpaces = videoMetadata.map((m) => m.colorSpace); // Canonical per-index record of the SDR-to-HDR transform decision. BOTH the // cache key (transform discriminator) and the extraction options read from @@ -829,6 +1207,8 @@ export async function extractAllVideoFrames( if (entry.video.mediaStart >= metadata.durationSeconds) { errors.push({ videoId: entry.video.id, + kind: "media_start_out_of_range", + retryable: false, error: `SDR→HDR conversion skipped: mediaStart (${entry.video.mediaStart}s) ≥ source duration (${metadata.durationSeconds}s)`, }); hdrSkippedIndices.add(i); @@ -865,12 +1245,10 @@ export async function extractAllVideoFrames( const vfrPreflightStart = Date.now(); for (let i = 0; i < resolvedVideos.length; i++) { if (signal?.aborted) break; - const entry = resolvedVideos[i]; - if (!entry) continue; const vfrProbeStart = Date.now(); - const metadata = await extractMediaMetadata(entry.videoPath); + const metadata = videoMetadata[i]; breakdown.vfrProbeMs += Date.now() - vfrProbeStart; - if (metadata.isVFR) breakdown.vfrPreflightCount += 1; + if (metadata?.isVFR) breakdown.vfrPreflightCount += 1; } breakdown.vfrPreflightMs = Date.now() - vfrPreflightStart; @@ -888,17 +1266,19 @@ export async function extractAllVideoFrames( } } - function extractionError(videoId: string, err: unknown): { videoId: string; error: string } { - return { videoId, error: err instanceof Error ? err.message : String(err) }; + function extractionError(videoId: string, err: unknown): VideoExtractionFailure { + const classified = classifyVideoExtractionError(err); + return { + videoId, + kind: classified.kind, + retryable: classified.retryable, + error: classified.diagnostic, + }; } - type PreparedExtractionResult = - | { work: PreparedExtraction } - | { error: { videoId: string; error: string } }; + type PreparedExtractionResult = { work: PreparedExtraction } | { error: VideoExtractionFailure }; - type ExtractionOutcome = - | { result: ExtractedFrames } - | { error: { videoId: string; error: string } }; + type ExtractionOutcome = { result: ExtractedFrames } | { error: VideoExtractionFailure }; function scopedExtractionOptions(work: PreparedExtraction): ExtractionOptions { return { ...options, format: work.format, sdrToHdrTransfer: work.sdrToHdrTransfer }; @@ -951,32 +1331,62 @@ export async function extractAllVideoFrames( }; } - async function extractDirectMiss(miss: UniqueExtractionMiss): Promise { + async function extractDirectMiss( + miss: UniqueExtractionMiss, + maxTransientRetries = options.maxTransientRetries ?? 0, + ): Promise { const { work, cacheTarget } = miss; if (!cacheTarget) { - return extractVideoFramesRange( - work.videoPath, - work.video.id, - work.video.mediaStart, - work.videoDuration, - scopedExtractionOptions(work), - signal, - config, + const outputDir = join(options.outputDir, work.video.id); + const attempted = await runVideoExtractionWithRetry( + () => + extractVideoFramesRange( + work.videoPath, + work.video.id, + work.video.mediaStart, + work.videoDuration, + scopedExtractionOptions(work), + signal, + config, + ), + { + signal, + maxTransientRetries, + onRetry: () => { + recordTransientRetries(1); + rmSync(outputDir, { recursive: true, force: true }); + }, + }, ); + return attempted.result; } const partialDir = partialCacheEntryDir(cacheTarget.entry); + rmSync(partialDir, { recursive: true, force: true }); mkdirSync(partialDir, { recursive: true }); - const result = await extractVideoFramesRange( - work.videoPath, - work.video.id, - work.video.mediaStart, - work.videoDuration, - scopedExtractionOptions(work), - signal, - config, - partialDir, + const attempted = await runVideoExtractionWithRetry( + () => + extractVideoFramesRange( + work.videoPath, + work.video.id, + work.video.mediaStart, + work.videoDuration, + scopedExtractionOptions(work), + signal, + config, + partialDir, + ), + { + signal, + maxTransientRetries, + onRetry: () => { + recordTransientRetries(1); + rmSync(partialDir, { recursive: true, force: true }); + mkdirSync(partialDir, { recursive: true }); + }, + }, ); + const result = attempted.result; const published = publishCacheEntry(cacheTarget.entry, partialDir); if (!published.published) { breakdown.cachePublishFailures += 1; @@ -985,9 +1395,12 @@ export async function extractAllVideoFrames( return rehydratePublishedCache(work, cacheTarget); } - async function executeDirectMiss(miss: UniqueExtractionMiss): Promise { + async function executeDirectMiss( + miss: UniqueExtractionMiss, + maxTransientRetries = options.maxTransientRetries ?? 0, + ): Promise { try { - return { result: await extractDirectMiss(miss) }; + return { result: await extractDirectMiss(miss, maxTransientRetries) }; } catch (err) { return { error: extractionError(miss.work.video.id, err) }; } @@ -1036,6 +1449,10 @@ export async function extractAllVideoFrames( try { rmSync(tempDir, { recursive: true, force: true }); + // A long union can hit the fixed FFmpeg deadline even when each shorter + // member range succeeds. Do not retry the optimization itself; preserve + // the established grouped→direct fallback and apply bounded retries only + // to the individual source ranges below. const superset = await extractVideoFramesRange( first.videoPath, group.groupId, @@ -1164,7 +1581,14 @@ export async function extractAllVideoFrames( const message = isFollower ? `[shared extraction, leader ${outcome.error.videoId}] ${outcome.error.error}` : outcome.error.error; - return { error: { videoId: prepared.work.video.id, error: message } }; + return { + error: { + videoId: prepared.work.video.id, + kind: outcome.error.kind, + retryable: outcome.error.retryable, + error: message, + }, + }; } return { result: { ...outcome.result, videoId: prepared.work.video.id } }; }); diff --git a/packages/engine/src/utils/urlDownloader.test.ts b/packages/engine/src/utils/urlDownloader.test.ts index b544ee8ca7..f36453cc10 100644 --- a/packages/engine/src/utils/urlDownloader.test.ts +++ b/packages/engine/src/utils/urlDownloader.test.ts @@ -141,10 +141,21 @@ describe("downloadToTemp atomic publication and bounded retry", () => { .mockResolvedValueOnce(new Response("complete")); vi.stubGlobal("fetch", fetchMock); const dir = makeTempDir(); + const onTransientRetry = vi.fn(); - const path = await downloadToTemp("https://cdn.example/retry-503.mp4", dir, 1_000); + const path = await downloadToTemp( + "https://cdn.example/retry-503.mp4", + dir, + 1_000, + undefined, + onTransientRetry, + ); expect(fetchMock).toHaveBeenCalledTimes(2); + expect(onTransientRetry).toHaveBeenCalledOnce(); + expect(onTransientRetry).toHaveBeenCalledWith( + expect.objectContaining({ kind: "http_transient", retryable: true }), + ); expect(readFileSync(path, "utf8")).toBe("complete"); expect(temporaryDownloadEntries(dir)).toEqual([]); }); diff --git a/packages/engine/src/utils/urlDownloader.ts b/packages/engine/src/utils/urlDownloader.ts index 894e698767..8383b26a2b 100644 --- a/packages/engine/src/utils/urlDownloader.ts +++ b/packages/engine/src/utils/urlDownloader.ts @@ -326,6 +326,7 @@ async function downloadWithRetry( localPath: string, timeoutMs: number, signal?: AbortSignal, + onTransientRetry?: (error: UrlDownloadError) => void, ): Promise { const maxTransientRetries = 1; for (let attempt = 0; ; attempt += 1) { @@ -334,6 +335,7 @@ async function downloadWithRetry( } catch (error) { const classified = classifyDownloadFailure(error); if (!classified.retryable || attempt >= maxTransientRetries) throw classified; + onTransientRetry?.(classified); } } } @@ -343,6 +345,7 @@ export async function downloadToTemp( destDir: string, timeoutMs: number = 300000, signal?: AbortSignal, + onTransientRetry?: (error: UrlDownloadError) => void, ): Promise { // Reject non-HTTPS URLs and private/reserved address ranges before // touching the cache or filesystem — customer-supplied compositions must @@ -368,7 +371,7 @@ export async function downloadToTemp( if (hasCompleteFile(localPath)) return localPath; - const downloadPromise = downloadWithRetry(url, localPath, timeoutMs, signal); + const downloadPromise = downloadWithRetry(url, localPath, timeoutMs, signal, onTransientRetry); const trackedDownload = downloadPromise.finally(() => { inFlightDownloads.delete(inFlightKey); }); diff --git a/packages/producer/src/server.errorCode.test.ts b/packages/producer/src/server.errorCode.test.ts new file mode 100644 index 0000000000..b372a20481 --- /dev/null +++ b/packages/producer/src/server.errorCode.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from "vitest"; +import { extractSafeRenderErrorCode } from "./server.js"; +import { VideoExtractionStageError } from "./services/render/stages/extractVideosStage.js"; + +describe("extractSafeRenderErrorCode", () => { + it("preserves allowlisted typed extraction codes", () => { + const deterministic = new VideoExtractionStageError("VIDEO_SOURCE_UNRENDERABLE", false, [ + { kind: "invalid_media", count: 1 }, + ]); + const exhausted = new VideoExtractionStageError("VIDEO_EXTRACTION_FAILED", true, [ + { kind: "ffmpeg_timeout", count: 1 }, + ]); + + expect(extractSafeRenderErrorCode(deterministic)).toBe("VIDEO_SOURCE_UNRENDERABLE"); + expect(extractSafeRenderErrorCode(exhausted)).toBe("VIDEO_EXTRACTION_FAILED"); + }); + + it("accepts the same bounded structural code across wrapped module boundaries", () => { + expect(extractSafeRenderErrorCode({ code: "VIDEO_SOURCE_UNRENDERABLE" })).toBe( + "VIDEO_SOURCE_UNRENDERABLE", + ); + }); + + it("does not forward arbitrary codes or parse message text", () => { + expect(extractSafeRenderErrorCode({ code: "INTERNAL_ERROR" })).toBeUndefined(); + expect( + extractSafeRenderErrorCode(new Error("failed [VIDEO_SOURCE_UNRENDERABLE; secret=/tmp/x]")), + ).toBeUndefined(); + }); +}); diff --git a/packages/producer/src/server.ts b/packages/producer/src/server.ts index 45af6b222f..da85b89e06 100644 --- a/packages/producer/src/server.ts +++ b/packages/producer/src/server.ts @@ -118,6 +118,23 @@ interface PreparedRenderInput { } const DEFAULT_SERVER_FPS = { num: 30, den: 1 } as const; +const SAFE_RENDER_ERROR_CODES = new Set([ + "VIDEO_SOURCE_UNRENDERABLE", + "VIDEO_EXTRACTION_FAILED", +] as const); + +/** + * Preserve only bounded producer error codes across JSON/SSE. Never derive a + * code from the message: it may contain local paths or signed source URLs. + */ +export function extractSafeRenderErrorCode(error: unknown): string | undefined { + if (typeof error !== "object" || error === null || !("code" in error)) return undefined; + const code = (error as { code?: unknown }).code; + return typeof code === "string" && + SAFE_RENDER_ERROR_CODES.has(code as "VIDEO_SOURCE_UNRENDERABLE" | "VIDEO_EXTRACTION_FAILED") + ? code + : undefined; +} function parseServerFps(value: unknown): RenderInput["fps"] { if (typeof value !== "number" && typeof value !== "string") return DEFAULT_SERVER_FPS; @@ -524,6 +541,7 @@ async function writeRenderStreamFailure(input: { return; } const errorMsg = error instanceof Error ? error.message : String(error); + const errorCode = extractSafeRenderErrorCode(error); const elapsedMs = Date.now() - startedAtMs; log.error("render-stream failed", { requestId, @@ -536,6 +554,7 @@ async function writeRenderStreamFailure(input: { type: "error", requestId, error: errorMsg, + errorCode, stage: job.currentStage, elapsedMs, errorDetails: job.errorDetails ?? null, @@ -684,6 +703,7 @@ export function createRenderHandlers(options: HandlerOptions = {}): RenderHandle } catch (error) { const durationMs = Date.now() - t0; const errorMsg = error instanceof Error ? error.message : String(error); + const errorCode = extractSafeRenderErrorCode(error); log.error("render failed", { requestId, durationMs, @@ -695,6 +715,7 @@ export function createRenderHandlers(options: HandlerOptions = {}): RenderHandle success: false, requestId, error: errorMsg, + errorCode, stage: job.currentStage, durationMs, errorDetails: job.errorDetails ?? null, diff --git a/packages/producer/src/services/distributed/plan.ts b/packages/producer/src/services/distributed/plan.ts index 8f11d7bd55..fe8580572d 100644 --- a/packages/producer/src/services/distributed/plan.ts +++ b/packages/producer/src/services/distributed/plan.ts @@ -928,6 +928,7 @@ export async function plan( assertNotAborted, materializeSymlinks: true, }); + if (extractResult.failureToEnforce) throw extractResult.failureToEnforce; // Skip `extractResult.frameLookup.cleanup()`: it would rm-rf each // video's outputDir, but in `plan()` those directories ARE the source // material the renames below move into `planDir/video-frames/`. diff --git a/packages/producer/src/services/render/observability.ts b/packages/producer/src/services/render/observability.ts index 1ee58dcfc7..23bc938143 100644 --- a/packages/producer/src/services/render/observability.ts +++ b/packages/producer/src/services/render/observability.ts @@ -109,6 +109,8 @@ export interface RenderExtractionObservability { vfrPreflightCount?: number; cacheHits?: number; cacheMisses?: number; + /** Per-source transient download/metadata/FFmpeg retries performed during extraction. */ + transientRetries?: number; /** * Per-clip captured-vs-expected-frame gauges. Emitted by the parity gate * at extract finalization (see `videoFrameCoverage.ts`). Undefined when diff --git a/packages/producer/src/services/render/stages/extractVideosStage.test.ts b/packages/producer/src/services/render/stages/extractVideosStage.test.ts index 640d90f526..2933c400ec 100644 --- a/packages/producer/src/services/render/stages/extractVideosStage.test.ts +++ b/packages/producer/src/services/render/stages/extractVideosStage.test.ts @@ -1,6 +1,17 @@ import { describe, expect, it } from "vitest"; -import { appendAutoDetectedVideoAudio, shouldCopyExtractedFrames } from "./extractVideosStage.js"; -import type { ExtractedFrames, VideoElement } from "@hyperframes/engine"; +import type { + ExtractedFrames, + ExtractionResult, + VideoElement, + VideoExtractionFailure, +} from "@hyperframes/engine"; +import { + appendAutoDetectedVideoAudio, + assertVideoExtractionSucceeded, + resolveVideoExtractionPolicy, + shouldCopyExtractedFrames, + VideoExtractionStageError, +} from "./extractVideosStage.js"; function makeVideo(overrides: Partial = {}): VideoElement { return { @@ -35,6 +46,33 @@ function makeExtracted(videoId: string, fileHasAudio: boolean): ExtractedFrames } as ExtractedFrames; } +function extractionResult(errors: VideoExtractionFailure[]): ExtractionResult { + return { + success: errors.length === 0, + errors, + extracted: [], + totalFramesExtracted: 0, + durationMs: 1, + phaseBreakdown: { + resolveMs: 0, + cachePublishFailures: 0, + cacheGcEvictions: 0, + cacheGcBytesFreed: 0, + cacheAgedPartialsCleared: 0, + hdrProbeMs: 0, + hdrPreflightMs: 0, + hdrPreflightCount: 0, + vfrProbeMs: 0, + vfrPreflightMs: 0, + vfrPreflightCount: 0, + extractMs: 0, + cacheHits: 0, + cacheMisses: 0, + transientRetries: 0, + }, + }; +} + describe("appendAutoDetectedVideoAudio", () => { it("adds audio for an audible video whose file has an audio track", () => { const composition = { videos: [makeVideo()], audios: [] as never[] }; @@ -92,3 +130,114 @@ describe("shouldCopyExtractedFrames", () => { expect(shouldCopyExtractedFrames("linux")).toBe(false); }); }); + +describe("resolveVideoExtractionPolicy", () => { + it("preserves stable behavior by default", () => { + expect(resolveVideoExtractionPolicy({})).toEqual({ + failureMode: "off", + maxTransientRetries: 0, + }); + }); + + it("allows only the bounded candidate rollout values", () => { + expect( + resolveVideoExtractionPolicy({ + HF_VIDEO_EXTRACTION_FAILURE_MODE: "observe", + HF_VIDEO_EXTRACTION_MAX_RETRIES: "1", + }), + ).toEqual({ failureMode: "observe", maxTransientRetries: 1 }); + expect( + resolveVideoExtractionPolicy({ + HF_VIDEO_EXTRACTION_FAILURE_MODE: "unexpected", + HF_VIDEO_EXTRACTION_MAX_RETRIES: "1", + }), + ).toEqual({ failureMode: "off", maxTransientRetries: 0 }); + }); +}); + +describe("assertVideoExtractionSucceeded", () => { + it("accepts a complete extraction", () => { + expect(() => assertVideoExtractionSucceeded(extractionResult([]))).not.toThrow(); + }); + + it("fails deterministic media errors without forwarding paths or signed URLs", () => { + const result = extractionResult([ + { + videoId: "narrator", + kind: "zero_output", + retryable: false, + error: + "FFmpeg failed for /tmp/render/secret.mp4 from https://cdn.example/x?Signature=secret", + }, + { + videoId: "missing", + kind: "source_missing", + retryable: false, + error: "Video file not found: /tmp/private/input.mp4", + }, + ]); + + let caught: unknown; + try { + assertVideoExtractionSucceeded(result); + } catch (error) { + caught = error; + } + + expect(caught).toBeInstanceOf(VideoExtractionStageError); + expect(caught).toMatchObject({ + code: "VIDEO_SOURCE_UNRENDERABLE", + retryable: false, + failures: [ + { kind: "source_missing", count: 1 }, + { kind: "zero_output", count: 1 }, + ], + }); + expect((caught as Error).message).not.toContain("/tmp/"); + expect((caught as Error).message).not.toContain("Signature"); + }); + + it("keeps exhausted transient failures retryable and collapses duplicate kinds", () => { + const result = extractionResult([ + { + videoId: "a", + kind: "download_transient", + retryable: true, + error: "HTTP 503", + }, + { + videoId: "b", + kind: "download_transient", + retryable: true, + error: "HTTP 503", + }, + ]); + + expect(() => assertVideoExtractionSucceeded(result)).toThrow( + expect.objectContaining({ + code: "VIDEO_EXTRACTION_FAILED", + retryable: true, + failures: [{ kind: "download_transient", count: 2 }], + }), + ); + }); + + it("fails closed for legacy failures without a kind or retryability", () => { + expect(() => + assertVideoExtractionSucceeded( + extractionResult([ + { + videoId: "legacy", + error: "legacy extraction error", + }, + ]), + ), + ).toThrow( + expect.objectContaining({ + code: "VIDEO_SOURCE_UNRENDERABLE", + retryable: false, + failures: [{ kind: "internal", count: 1 }], + }), + ); + }); +}); diff --git a/packages/producer/src/services/render/stages/extractVideosStage.ts b/packages/producer/src/services/render/stages/extractVideosStage.ts index 7f133b789d..df1956db6e 100644 --- a/packages/producer/src/services/render/stages/extractVideosStage.ts +++ b/packages/producer/src/services/render/stages/extractVideosStage.ts @@ -34,15 +34,19 @@ import { type CaptureVideoMetadataHint, type EngineConfig, type ExtractedFrames, + type ExtractionResult, type FrameLookupTable, type HdrTransfer, + type VideoExtractionFailureKind, type VideoColorSpace, + classifyVideoExtractionError, createFrameLookupTable, detectTransfer, extractAllVideoFrames, extractMediaMetadata, isHdrColorSpace, resolveProjectRelativeSrc, + runVideoExtractionWithRetry, } from "@hyperframes/engine"; import { fpsToNumber } from "@hyperframes/core"; import { @@ -94,6 +98,11 @@ export interface ExtractVideosStageResult { imageColorSpaces: (VideoColorSpace | null)[]; /** Wall-clock ms for the video extraction phase. */ videoExtractMs: number; + /** + * Candidate-only typed failure gate. Callers throw this only after their + * extraction telemetry checkpoint has been emitted. + */ + failureToEnforce: VideoExtractionStageError | null; } /** @@ -108,6 +117,98 @@ export function shouldCopyExtractedFrames(platform: NodeJS.Platform): boolean { return platform === "win32"; } +export type VideoExtractionStageErrorCode = "VIDEO_SOURCE_UNRENDERABLE" | "VIDEO_EXTRACTION_FAILED"; + +export interface VideoExtractionStageFailureSummary { + kind: VideoExtractionFailureKind; + count: number; +} + +export type VideoExtractionFailureMode = "off" | "observe" | "enforce"; + +export interface VideoExtractionPolicy { + failureMode: VideoExtractionFailureMode; + maxTransientRetries: 0 | 1; +} + +/** + * Candidate-lane rollout controls. Stable behavior remains unchanged unless + * explicitly enabled in the producer environment. + */ +export function resolveVideoExtractionPolicy( + env: Readonly> = process.env, +): VideoExtractionPolicy { + const rawMode = env.HF_VIDEO_EXTRACTION_FAILURE_MODE?.trim().toLowerCase(); + const failureMode: VideoExtractionFailureMode = + rawMode === "observe" || rawMode === "enforce" ? rawMode : "off"; + const maxTransientRetries = + failureMode !== "off" && env.HF_VIDEO_EXTRACTION_MAX_RETRIES?.trim() === "1" ? 1 : 0; + return { failureMode, maxTransientRetries }; +} + +/** + * Producer-safe terminal error for per-source extraction failures. + * + * `ExtractionResult.errors[].error` intentionally retains local diagnostics + * and can contain signed URLs or filesystem paths. This error carries only a + * bounded taxonomy/count summary so the HTTP/Temporal boundary can transport + * the cause without leaking those values. + */ +export class VideoExtractionStageError extends Error { + constructor( + readonly code: VideoExtractionStageErrorCode, + readonly retryable: boolean, + readonly failures: readonly VideoExtractionStageFailureSummary[], + ) { + const total = failures.reduce((sum, failure) => sum + failure.count, 0); + const breakdown = failures.map((failure) => `${failure.kind}=${failure.count}`).join(","); + super(`Video extraction failed for ${total} source(s) [${code}; ${breakdown}]`); + this.name = "VideoExtractionStageError"; + } +} + +export function assertVideoExtractionSucceeded(result: ExtractionResult): void { + const error = buildVideoExtractionStageError(result); + if (error) throw error; +} + +function buildVideoExtractionStageError( + result: ExtractionResult, +): VideoExtractionStageError | null { + if (result.success && result.errors.length === 0) return null; + const counts = new Map(); + for (const failure of result.errors) { + const kind = failure.kind ?? "internal"; + counts.set(kind, (counts.get(kind) ?? 0) + 1); + } + const failures = Array.from(counts, ([kind, count]) => ({ kind, count })).sort((a, b) => + a.kind.localeCompare(b.kind), + ); + const retryable = + result.errors.length > 0 && result.errors.every((failure) => failure.retryable === true); + return new VideoExtractionStageError( + retryable ? "VIDEO_EXTRACTION_FAILED" : "VIDEO_SOURCE_UNRENDERABLE", + retryable, + failures, + ); +} + +function applyVideoExtractionFailurePolicy( + result: ExtractionResult, + policy: VideoExtractionPolicy, + log?: ProducerLogger, +): VideoExtractionStageError | null { + const error = buildVideoExtractionStageError(result); + if (!error || policy.failureMode === "off") return null; + log?.warn("Video extraction produced typed source failures", { + mode: policy.failureMode, + code: error.code, + retryable: error.retryable, + failures: error.failures, + }); + return policy.failureMode === "enforce" ? error : null; +} + export async function runExtractVideosStage( input: ExtractVideosStageInput, ): Promise { @@ -124,9 +225,11 @@ export async function runExtractVideosStage( } = input; const stage2Start = Date.now(); + const extractionPolicy = resolveVideoExtractionPolicy(); let frameLookup: FrameLookupTable | null = null; let extractionResult: Awaited> | null = null; + let failureToEnforce: VideoExtractionStageError | null = null; let videoReadinessSkipIds: string[] = []; let videoMetadataHints: CaptureVideoMetadataHint[] = []; @@ -136,6 +239,7 @@ export async function runExtractVideosStage( // avoid ffprobe overhead when the user has explicitly opted out. const nativeHdrVideoIds = new Set(); const videoTransfers = new Map(); + let hdrProbeTransientRetries = 0; if (job.config.hdrMode !== "force-sdr" && composition.videos.length > 0) { log?.info("Probing video color spaces...", { videoCount: composition.videos.length }); await Promise.all( @@ -149,10 +253,42 @@ export async function runExtractVideosStage( ? v.src : resolveProjectRelativeSrc(v.src, projectDir, compiledDir); if (!existsSync(videoPath)) return; - const meta = await extractMediaMetadata(videoPath); - if (isHdrColorSpace(meta.colorSpace)) { - nativeHdrVideoIds.add(v.id); - videoTransfers.set(v.id, detectTransfer(meta.colorSpace)); + try { + // Retries are separately opt-in from the failure gate. With the + // default zero budget this remains the exact legacy single probe. + const attempted = + extractionPolicy.maxTransientRetries === 0 + ? { result: await extractMediaMetadata(videoPath), retries: 0 } + : await runVideoExtractionWithRetry(() => extractMediaMetadata(videoPath), { + signal: abortSignal, + maxTransientRetries: extractionPolicy.maxTransientRetries, + onRetry: () => { + hdrProbeTransientRetries += 1; + }, + }); + const meta = attempted.result; + if (isHdrColorSpace(meta.colorSpace)) { + nativeHdrVideoIds.add(v.id); + videoTransfers.set(v.id, detectTransfer(meta.colorSpace)); + } + } catch (error) { + if (extractionPolicy.failureMode !== "off") { + const classified = classifyVideoExtractionError(error); + log?.warn("Video HDR metadata probe failed", { + mode: extractionPolicy.failureMode, + kind: classified.kind, + retryable: classified.retryable, + transientRetries: hdrProbeTransientRetries, + }); + if (extractionPolicy.failureMode === "enforce") { + throw new VideoExtractionStageError( + classified.retryable ? "VIDEO_EXTRACTION_FAILED" : "VIDEO_SOURCE_UNRENDERABLE", + classified.retryable, + [{ kind: classified.kind, count: 1 }], + ); + } + } + throw error; } }), ); @@ -207,12 +343,17 @@ export async function runExtractVideosStage( fps: fpsToNumber(job.config.fps), outputDir: join(compiledDir, "__hyperframes_video_frames"), format: job.config.videoFrameFormat ?? "auto", + maxTransientRetries: extractionPolicy.maxTransientRetries, + collectProbeFailures: extractionPolicy.failureMode === "enforce", }, abortSignal, { extractCacheDir: cfg.extractCacheDir, extractCacheMaxBytes: cfg.extractCacheMaxBytes }, compiledDir, ); + extractionResult.phaseBreakdown.transientRetries = + (extractionResult.phaseBreakdown.transientRetries ?? 0) + hdrProbeTransientRetries; assertNotAborted(); + failureToEnforce = applyVideoExtractionFailurePolicy(extractionResult, extractionPolicy, log); materializeExtractedFramesForCompiledDir(extractionResult.extracted, compiledDir, { materializeSymlinks, @@ -243,6 +384,7 @@ export async function runExtractVideosStage( hdrImageSrcPaths, imageColorSpaces, videoExtractMs, + failureToEnforce, }; } diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index c9a0c81292..8859dbc5d8 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -230,6 +230,7 @@ function summarizeExtractionObservability( vfrPreflightCount: phaseBreakdown?.vfrPreflightCount, cacheHits: phaseBreakdown?.cacheHits, cacheMisses: phaseBreakdown?.cacheMisses, + transientRetries: phaseBreakdown?.transientRetries, ...coverageGauges, authoredTimedClipCount, }; @@ -2103,6 +2104,7 @@ async function executeRenderPipeline(input: { imageTransfers, hdrImageSrcPaths, imageColorSpaces, + failureToEnforce, } = extractResult; perfStages.videoExtractMs = extractResult.videoExtractMs; @@ -2138,9 +2140,11 @@ async function executeRenderPipeline(input: { vfrPreflightMs: extractionObservability.vfrPreflightMs ?? null, cacheHits: extractionObservability.cacheHits ?? null, cacheMisses: extractionObservability.cacheMisses ?? null, + transientRetries: extractionObservability.transientRetries ?? null, minVideoFrameCoverageRatio: extractionObservability.minVideoFrameCoverageRatio ?? null, authoredTimedClipCount: extractionObservability.authoredTimedClipCount ?? null, }); + if (failureToEnforce) throw failureToEnforce; // Gate AFTER the checkpoint so a coverage-failed render still emits // the observability row (partial telemetry is still worth having). // `assertVideoFrameCoverage` no-ops on an empty report list AND on a