Skip to content

Write HTML. Render video. Built for agents.

fix(producer): renders leave no captured frames behind when interrupted (#4694)

Miguel Ángelf7ef18dca80c
diff --git a/packages/cli/src/utils/renderCancellation.test.ts b/packages/cli/src/utils/renderCancellation.test.ts
index f732274fcd..b1281fa272 100644
--- a/packages/cli/src/utils/renderCancellation.test.ts
+++ b/packages/cli/src/utils/renderCancellation.test.ts
@@ -2,6 +2,9 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { createRenderCancellationScope, stoppedByCancelSignal } from "./renderCancellation.js";
+const { setHostHandlesSigint } = vi.hoisted(() => ({ setHostHandlesSigint: vi.fn() }));
+vi.mock("@hyperframes/engine", () => ({ setHostHandlesSigint }));
+
afterEach(() => {
vi.useRealTimers();
});
@@ -29,6 +32,17 @@ describe("createRenderCancellationScope ancestor monitoring", () => {
cancellation.dispose();
});
+ it("owns Ctrl+C for the engine's browsers while the scope is open", () => {
+ const cancellation = createRenderCancellationScope({
+ signalTarget: { on: vi.fn(), off: vi.fn() },
+ pid: 100,
+ parentPid: () => 1,
+ });
+ expect(setHostHandlesSigint).toHaveBeenLastCalledWith(true);
+ cancellation.dispose();
+ expect(setHostHandlesSigint).toHaveBeenLastCalledWith(false);
+ });
+
it("keeps termination handlers installed until cancellation cleanup finishes", () => {
const handlers = new Map<string, () => void>();
const signalTarget = {
diff --git a/packages/cli/src/utils/renderCancellation.ts b/packages/cli/src/utils/renderCancellation.ts
index ef60cd3d45..9977aca4bf 100644
--- a/packages/cli/src/utils/renderCancellation.ts
+++ b/packages/cli/src/utils/renderCancellation.ts
@@ -1,3 +1,4 @@
+import { setHostHandlesSigint } from "@hyperframes/engine";
import {
processAncestorSnapshot,
processIdentity,
@@ -115,6 +116,7 @@ export function createRenderCancellationScope(
const detached = options.detached ?? process.env.HYPERFRAMES_RENDER_DETACHED === "1";
const signalHandlers = installSignalHandlers(detached, signalTarget, abort);
+ setHostHandlesSigint(true);
const lookupIdentity = options.identity ?? processIdentity;
const isAlive = options.isAlive ?? processIsAlive;
const platform = options.platform ?? process.platform;
@@ -155,6 +157,7 @@ export function createRenderCancellationScope(
disposed = true;
if (watchdogTimer !== undefined) clearInterval(watchdogTimer);
for (const [signal, handler] of signalHandlers) signalTarget.off(signal, handler);
+ setHostHandlesSigint(false);
},
};
}
diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts
index 431aff25be..4b98af1600 100644
--- a/packages/engine/src/index.ts
+++ b/packages/engine/src/index.ts
@@ -77,6 +77,7 @@ export {
export { chromeMajorCeiling } from "./services/chromeHostCeiling.js";
export {
acquireBrowser,
+ setHostHandlesSigint,
releaseBrowser,
drainBrowserPool,
closeBrowserPool,
diff --git a/packages/engine/src/services/browserManager.ts b/packages/engine/src/services/browserManager.ts
index 861728014d..dafe45173e 100644
--- a/packages/engine/src/services/browserManager.ts
+++ b/packages/engine/src/services/browserManager.ts
@@ -31,6 +31,15 @@ export type {
let _puppeteer: PuppeteerNode | undefined;
+let hostHandlesSigint = false;
+
+/** Set while the host cancels renders on Ctrl+C; Puppeteer's own handler would exit before any cleanup ran. */
+export function setHostHandlesSigint(owned: boolean): void {
+ hostHandlesSigint = owned;
+}
+
+export const sigintLaunchOptions = () => ({ handleSIGINT: !hostHandlesSigint });
+
interface WebGlProbeInfo {
hasWebGL: boolean;
vendor: string;
@@ -74,6 +83,7 @@ async function probeHardwareWebGlInfo(
let probeBrowser: Browser | undefined;
try {
probeBrowser = await ppt.launch({
+ ...sigintLaunchOptions(),
headless: true,
args: options.args,
defaultViewport: { width: 64, height: 64 },
@@ -699,6 +709,7 @@ async function launchBrowser(
let browser: Browser | undefined;
try {
browser = await ppt.launch({
+ ...sigintLaunchOptions(),
headless: true,
args: [...fingerprint.args],
defaultViewport: null,
@@ -730,6 +741,7 @@ async function launchBrowser(
);
captureMode = "screenshot";
browser = await ppt.launch({
+ ...sigintLaunchOptions(),
headless: true,
args: stripBeginFrameFlags([...fingerprint.args]),
defaultViewport: null,
diff --git a/packages/engine/src/services/frameCapture-warmupTicks.test.ts b/packages/engine/src/services/frameCapture-warmupTicks.test.ts
index f38839ca60..d5abd9cbc6 100644
--- a/packages/engine/src/services/frameCapture-warmupTicks.test.ts
+++ b/packages/engine/src/services/frameCapture-warmupTicks.test.ts
@@ -14,6 +14,7 @@ import {
LOCKED_WARMUP_TICKS,
deriveBeginFrameTimelineTicks,
deriveBeginFrameTimeTicks,
+ cancelWarmupOnError,
driveWarmupTicks,
prepareBeginFrameTimeline,
warmupFrameTimeTicks,
@@ -54,6 +55,45 @@ async function runWithSimulatedPageLoad(
return state;
}
+describe("driveWarmupTicks — cancelled", () => {
+ it.each([false, true])(
+ "stops once cancelled even while every tick fails (locked: %s)",
+ async (locked) => {
+ const state = makeState();
+ let calls = 0;
+ await driveWarmupTicks(
+ {
+ intervalMs: 33,
+ lockWarmupTicks: locked,
+ // Yields to the event loop so a loop that never stops fails on the test timeout instead of hanging.
+ sleep: () => new Promise((settle) => setImmediate(settle)),
+ tick: async () => {
+ if (++calls === 3) state.cancelled = true;
+ throw new Error("Target closed");
+ },
+ },
+ state,
+ );
+ expect(calls).toBe(3);
+ },
+ );
+});
+
+describe("cancelWarmupOnError", () => {
+ it("stops the warm-up loop when setup fails and leaves it alone when setup succeeds", async () => {
+ const failed = makeState();
+ const setupError = new Error("render_cancelled");
+ await expect(cancelWarmupOnError(failed, () => Promise.reject(setupError))).rejects.toBe(
+ setupError,
+ );
+ expect(failed.cancelled).toBe(true);
+
+ const succeeded = makeState();
+ await expect(cancelWarmupOnError(succeeded, async () => "ready")).resolves.toBe("ready");
+ expect(succeeded.cancelled).toBeUndefined();
+ });
+});
+
describe("driveWarmupTicks — unlocked", () => {
it("stops when state.running flips false", async () => {
const state = await runWithSimulatedPageLoad(false, 10);
diff --git a/packages/engine/src/services/frameCapture.ts b/packages/engine/src/services/frameCapture.ts
index 9ac974d65a..dac4eaf3be 100644
--- a/packages/engine/src/services/frameCapture.ts
+++ b/packages/engine/src/services/frameCapture.ts
@@ -679,6 +679,8 @@ export const LOCKED_WARMUP_TICKS = 60;
export interface WarmupTickState {
running: boolean;
ticks: number;
+ /** Set when session init fails: stops the loop in either mode, since a closed page fails every tick. */
+ cancelled?: boolean;
}
export interface WarmupTickOptions {
@@ -794,12 +796,25 @@ export function prepareBeginFrameTimeline(
};
}
+/** Runs BeginFrame session setup; a failure stops the warm-up loop, which would otherwise tick forever. */
+export async function cancelWarmupOnError<T>(
+ state: WarmupTickState,
+ setup: () => Promise<T>,
+): Promise<T> {
+ try {
+ return await setup();
+ } catch (error) {
+ state.cancelled = true;
+ throw error;
+ }
+}
+
export async function driveWarmupTicks(
options: WarmupTickOptions,
state: WarmupTickState,
): Promise<void> {
const sleep = options.sleep ?? realSleep;
- while (true) {
+ while (!state.cancelled) {
if (options.lockWarmupTicks) {
// Locked mode exits on the iteration count, ignoring `state.running` —
// the caller flips `running=false` after page-readiness but we keep
@@ -2462,95 +2477,93 @@ export async function initializeSession(session: CaptureSession): Promise<void>
warmupLoopPromise.catch(() => {});
logInitPhase("warmup loop started");
- await gotoEntryPage();
- logInitPhase("page.goto complete");
+ await cancelWarmupOnError(warmupState, async () => {
+ await gotoEntryPage();
+ logInitPhase("page.goto complete");
- // Flush the GSAP proxy queue synchronously. In BeginFrame mode the rAF-based
- // batch drain runs on the warmup loop's 33ms ticks — for tween-heavy
- // compositions this is the dominant init cost. Flushing synchronously
- // eliminates the wait entirely.
- await page.evaluate(`window.__hfFlushSync?.()`);
- logInitPhase("GSAP proxy flush complete");
+ // Flush the GSAP proxy queue synchronously. In BeginFrame mode the rAF-based
+ // batch drain runs on the warmup loop's 33ms ticks — for tween-heavy
+ // compositions this is the dominant init cost. Flushing synchronously
+ // eliminates the wait entirely.
+ await page.evaluate(`window.__hfFlushSync?.()`);
+ logInitPhase("GSAP proxy flush complete");
- // Poll for window.__hf readiness using manual evaluate loop (waitForFunction
- // uses rAF polling internally, which won't fire in beginFrame mode).
- const pageReadyTimeout = session.config?.playerReadyTimeout ?? DEFAULT_CONFIG.playerReadyTimeout;
- try {
+ // Poll for window.__hf readiness using manual evaluate loop (waitForFunction
+ // uses rAF polling internally, which won't fire in beginFrame mode).
+ const pageReadyTimeout =
+ session.config?.playerReadyTimeout ?? DEFAULT_CONFIG.playerReadyTimeout;
await pollHfReady(page, pageReadyTimeout);
logInitPhase("pollHfReady complete");
- } catch (err) {
- warmupState.running = false;
- throw err;
- }
- session.subTimelineWaitOutcome = await pollSubCompositionTimelines(
- page,
- pageReadyTimeout,
- undefined,
- () => session.scriptLoadFailures,
- undefined,
- (ids) => {
- session.pendingTimelineIds = [...ids];
- },
- );
- logInitPhase(`pollSubCompositionTimelines complete (${session.subTimelineWaitOutcome})`);
- recordSubTimelineWarning(session, pageReadyTimeout);
-
- await applyVideoMetadataHints(page, session.options.videoMetadataHints);
- logInitPhase("applyVideoMetadataHints complete");
-
- // Run independent readiness checks in parallel — videos, images, fonts,
- // and Tailwind don't depend on each other's completion.
- const bfSkipVideoIds = session.options.skipReadinessVideoIds ?? [];
- const [bfVideosReady] = await Promise.all([
- pollVideosReady(page, bfSkipVideoIds, pageReadyTimeout),
- pollImagesReady(page, pageReadyTimeout).then(async (ready) => {
- if (!ready) {
- const failedImages = await page.evaluate(() => {
- return Array.from(document.querySelectorAll("img"))
- .filter((img) => {
- const ie = img as HTMLImageElement;
- const src = ie.getAttribute("src") || "";
- if (!src || src.startsWith("data:")) return false;
- return !(ie.complete && ie.naturalWidth > 0);
- })
- .map((img) => (img as HTMLImageElement).src || img.getAttribute("src") || "(no src)")
- .join(", ");
- });
- console.warn(
- `[FrameCapture] Some image elements did not load within ${pageReadyTimeout}ms: ${failedImages}. ` +
- `Continuing render — affected images may appear blank/missing in early frames.`,
- );
- }
- await decodeAllImages(page);
- return ready;
- }),
- page.evaluate(`document.fonts?.ready`),
- waitForOptionalTailwindReady(page, pageReadyTimeout),
- ]);
- logInitPhase("media + fonts + tailwind ready");
-
- if (!bfVideosReady) {
- const failedVideos = await page.evaluate((skipIdList: readonly string[]) => {
- const skip = new Set(skipIdList);
- return Array.from(document.querySelectorAll("video"))
- .filter((v) => !skip.has(v.id))
- .filter((v) => (v as HTMLVideoElement).readyState < 2 && !(v as HTMLVideoElement).error)
- .map((v) => (v as HTMLVideoElement).src || v.getAttribute("src") || "(no src)")
- .join(", ");
- }, bfSkipVideoIds);
- console.warn(
- `[FrameCapture] Some video elements did not decode within ${pageReadyTimeout}ms: ${failedVideos}. ` +
- `Continuing render — affected videos will appear as blank/black frames.`,
+ session.subTimelineWaitOutcome = await pollSubCompositionTimelines(
+ page,
+ pageReadyTimeout,
+ undefined,
+ () => session.scriptLoadFailures,
+ undefined,
+ (ids) => {
+ session.pendingTimelineIds = [...ids];
+ },
);
- }
- recordCaptureWarnings(
- session,
- await collectMediaReadinessWarnings(page, bfSkipVideoIds, pageReadyTimeout),
- );
- await recordLiveMapWarning(session, page);
+ logInitPhase(`pollSubCompositionTimelines complete (${session.subTimelineWaitOutcome})`);
+ recordSubTimelineWarning(session, pageReadyTimeout);
- await recordSessionInitTelemetry(session, initStart);
+ await applyVideoMetadataHints(page, session.options.videoMetadataHints);
+ logInitPhase("applyVideoMetadataHints complete");
+
+ // Run independent readiness checks in parallel — videos, images, fonts,
+ // and Tailwind don't depend on each other's completion.
+ const bfSkipVideoIds = session.options.skipReadinessVideoIds ?? [];
+ const [bfVideosReady] = await Promise.all([
+ pollVideosReady(page, bfSkipVideoIds, pageReadyTimeout),
+ pollImagesReady(page, pageReadyTimeout).then(async (ready) => {
+ if (!ready) {
+ const failedImages = await page.evaluate(() => {
+ return Array.from(document.querySelectorAll("img"))
+ .filter((img) => {
+ const ie = img as HTMLImageElement;
+ const src = ie.getAttribute("src") || "";
+ if (!src || src.startsWith("data:")) return false;
+ return !(ie.complete && ie.naturalWidth > 0);
+ })
+ .map((img) => (img as HTMLImageElement).src || img.getAttribute("src") || "(no src)")
+ .join(", ");
+ });
+ console.warn(
+ `[FrameCapture] Some image elements did not load within ${pageReadyTimeout}ms: ${failedImages}. ` +
+ `Continuing render — affected images may appear blank/missing in early frames.`,
+ );
+ }
+ await decodeAllImages(page);
+ return ready;
+ }),
+ page.evaluate(`document.fonts?.ready`),
+ waitForOptionalTailwindReady(page, pageReadyTimeout),
+ ]);
+ logInitPhase("media + fonts + tailwind ready");
+
+ if (!bfVideosReady) {
+ const failedVideos = await page.evaluate((skipIdList: readonly string[]) => {
+ const skip = new Set(skipIdList);
+ return Array.from(document.querySelectorAll("video"))
+ .filter((v) => !skip.has(v.id))
+ .filter((v) => (v as HTMLVideoElement).readyState < 2 && !(v as HTMLVideoElement).error)
+ .map((v) => (v as HTMLVideoElement).src || v.getAttribute("src") || "(no src)")
+ .join(", ");
+ }, bfSkipVideoIds);
+ console.warn(
+ `[FrameCapture] Some video elements did not decode within ${pageReadyTimeout}ms: ${failedVideos}. ` +
+ `Continuing render — affected videos will appear as blank/black frames.`,
+ );
+ }
+ recordCaptureWarnings(
+ session,
+ await collectMediaReadinessWarnings(page, bfSkipVideoIds, pageReadyTimeout),
+ );
+ await recordLiveMapWarning(session, page);
+
+ await recordSessionInitTelemetry(session, initStart);
+ });
// Stop warmup, then drain the loop in BOTH modes before any further
// BeginFrame on this session (drawElement init, the render-frame commit tick
diff --git a/packages/engine/src/services/hdrCapture.ts b/packages/engine/src/services/hdrCapture.ts
index 93ac78f23e..6a1867ae8b 100644
--- a/packages/engine/src/services/hdrCapture.ts
+++ b/packages/engine/src/services/hdrCapture.ts
@@ -23,6 +23,7 @@ import type { Page, Browser, PuppeteerNode } from "puppeteer-core";
import { existsSync, readdirSync } from "fs";
import { join } from "path";
import { homedir } from "os";
+import { sigintLaunchOptions } from "./browserManager.js";
// ── PQ (SMPTE 2084) OETF ─────────────────────────────────────────────────────
@@ -282,6 +283,7 @@ export async function launchHdrBrowser(
}
const browser = await ppt.launch({
+ ...sigintLaunchOptions(),
headless: false,
executablePath: chromePath,
args: buildHdrChromeArgs(width, height),
diff --git a/packages/engine/src/services/parallelCoordinator-peerAbort.test.ts b/packages/engine/src/services/parallelCoordinator-peerAbort.test.ts
index c389a80f53..88aa7c3b50 100644
--- a/packages/engine/src/services/parallelCoordinator-peerAbort.test.ts
+++ b/packages/engine/src/services/parallelCoordinator-peerAbort.test.ts
@@ -62,6 +62,44 @@ describe("executeParallelCapture peer abort", () => {
}
});
+ it("closes a browser whose launch finishes after the cancel before the render rejects", async () => {
+ const root = mkdtempSync(join(tmpdir(), "hf-late-launch-"));
+ const lateSession = {} as CaptureSession;
+ let finishLaunch: (session: CaptureSession) => void = () => {};
+ const closeCaptureSession = vi.fn().mockResolvedValue(undefined);
+ vi.doMock("./frameCapture.js", () => ({
+ createCaptureSession: vi.fn(
+ () => new Promise<CaptureSession>((settle) => (finishLaunch = settle)),
+ ),
+ initializeSession: vi.fn(async () => {}),
+ captureFrame: vi.fn(),
+ captureFrameToBuffer: vi.fn(),
+ captureFrameToBufferPipelined: vi.fn(),
+ closeCaptureSession,
+ getCapturePerfSummary: vi.fn(() => ({ frames: 0 })),
+ }));
+
+ try {
+ const { executeParallelCapture } = await import("./parallelCoordinator.js");
+ const controller = new AbortController();
+ const result = executeParallelCapture(
+ "http://127.0.0.1",
+ root,
+ [{ workerId: 0, startFrame: 0, endFrame: 3, outputDir: join(root, "worker-0") }],
+ { width: 320, height: 180, fps: { num: 30, den: 1 } },
+ () => null,
+ controller.signal,
+ );
+ controller.abort();
+ setTimeout(() => finishLaunch(lateSession), 20);
+
+ await expect(result).rejects.toMatchObject({ name: "CaptureFailure" });
+ expect(closeCaptureSession).toHaveBeenCalledWith(lateSession);
+ } finally {
+ rmSync(root, { recursive: true, force: true });
+ }
+ });
+
// heygen-com/hyperframes#3441: a worker wedged INSIDE a native capture call
// (WSL2 hangs the very first drawElement/BeginFrame call at frame 0 with no
// error) must actually be unstuck once the caller's `signal` aborts — e.g.
diff --git a/packages/engine/src/services/parallelCoordinator.ts b/packages/engine/src/services/parallelCoordinator.ts
index f01deb7e18..457f22b631 100644
--- a/packages/engine/src/services/parallelCoordinator.ts
+++ b/packages/engine/src/services/parallelCoordinator.ts
@@ -544,6 +544,8 @@ export function withParallelWorkerDeadline<T>(
});
}
+const LATE_LAUNCH_CLOSE_WAIT_MS = 10_000;
+
function resolveParallelWorkerTimeoutMs(enabled: boolean): number {
if (!enabled) return 0;
const raw = process.env.HF_DE_PARALLEL_PHASE_TIMEOUT_MS;
@@ -881,15 +883,22 @@ async function executeWorkerTask(
};
try {
- session = await runPhase("browser_launch", () =>
- createCaptureSession(
- serverUrl,
- task.outputDir,
- captureOptions,
- createBeforeCaptureHook(),
- workerConfig,
- ),
+ const launch = createCaptureSession(
+ serverUrl,
+ task.outputDir,
+ captureOptions,
+ createBeforeCaptureHook(),
+ workerConfig,
);
+ session = await runPhase("browser_launch", () => launch).catch(async (error: unknown) => {
+ // Close a launch that lost to cancel or its deadline before settling: Puppeteer's exit hook skips browsers.
+ const closing = launch.then(closeCaptureSession).catch(() => {});
+ await Promise.race([
+ closing,
+ new Promise((settle) => setTimeout(settle, LATE_LAUNCH_CLOSE_WAIT_MS).unref()),
+ ]);
+ throw error;
+ });
const activeSession = session;
browserExecutable = activeSession.browser?.process?.()?.spawnfile || browserExecutable;
logParDebug(() => `[par:w${task.workerId}] session created`);
diff --git a/packages/producer/scripts/test-classification.mjs b/packages/producer/scripts/test-classification.mjs
index afc9af2b51..84bb47a807 100644
--- a/packages/producer/scripts/test-classification.mjs
+++ b/packages/producer/scripts/test-classification.mjs
@@ -29,6 +29,7 @@ const INTEGRATION_TEST_FILES = new Set([
"src/services/htmlCompiler.mediaType.test.ts",
"src/services/htmlCompiler.naturalDuration.test.ts",
"src/services/render/hlsRender.integration.test.ts",
+ "src/services/render/renderCancel.integration.test.ts",
"src/services/render/stages/compileStage.mediaType.test.ts",
"src/services/render/stages/gifEncodeArgs.test.ts",
"src/utils/audioRegression.test.ts",
diff --git a/packages/producer/src/services/render/artifactTransaction.ts b/packages/producer/src/services/render/artifactTransaction.ts
index 172bfdfb9a..296a63a817 100644
--- a/packages/producer/src/services/render/artifactTransaction.ts
+++ b/packages/producer/src/services/render/artifactTransaction.ts
@@ -2,7 +2,6 @@ import {
closeSync,
existsSync,
fstatSync,
- mkdtempSync,
openSync,
readSync,
readdirSync,
@@ -13,6 +12,7 @@ import { basename, dirname, extname, join, resolve } from "node:path";
import { HLS_MASTER_PLAYLIST, HLS_VIDEO_PLAYLIST } from "@hyperframes/engine";
import { extractMediaMetadata } from "../../utils/ffprobe.js";
import type { RenderOutputFormat } from "./renderFormat.js";
+import { TRANSACTION_BACKUP, createOwnedRenderDir } from "./renderDirOwner.js";
export type ArtifactKind = "file" | "directory";
@@ -145,7 +145,7 @@ function createSiblingTransactionDirectory(destination: string): string {
// mkdtemp reserves the directory atomically and creates it with private
// permissions. Keeping it beside the destination preserves same-filesystem
// rename semantics without exposing predictable files in a shared temp dir.
- return mkdtempSync(join(parent, `.${stem}.hf-transaction-`));
+ return createOwnedRenderDir(join(parent, `.${stem}.hf-transaction-`));
}
function assertReadableNonEmptyFile(path: string): void {
@@ -271,7 +271,7 @@ export class ArtifactTransaction {
this.destinationPath = resolve(destinationPath);
this.transactionDirectory = createSiblingTransactionDirectory(this.destinationPath);
this.stagingPath = join(this.transactionDirectory, basename(this.destinationPath));
- this.backupPath = join(this.transactionDirectory, "backup");
+ this.backupPath = join(this.transactionDirectory, TRANSACTION_BACKUP);
this.durationProbe = durationProbe;
}
diff --git a/packages/producer/src/services/render/renderCancel.fixture.ts b/packages/producer/src/services/render/renderCancel.fixture.ts
new file mode 100644
index 0000000000..66f9218930
--- /dev/null
+++ b/packages/producer/src/services/render/renderCancel.fixture.ts
@@ -0,0 +1,25 @@
+// fallow-ignore-file unused-file
+import { resolveConfig, setHostHandlesSigint } from "@hyperframes/engine";
+import { createRenderJob, executeRenderJob } from "../renderOrchestrator.js";
+
+const [projectDir, outputPath] = process.argv.slice(2);
+if (!projectDir || !outputPath) throw new Error("Missing fixture arguments");
+
+// Same contract as the CLI: the host owns Ctrl+C, aborts the render, and the render cleans up after itself.
+const controller = new AbortController();
+process.once("SIGINT", () => controller.abort(new Error("render_cancelled_by_sigint")));
+setHostHandlesSigint(true);
+const silent = { info() {}, warn() {}, error() {}, debug() {} };
+const job = createRenderJob({
+ fps: 30,
+ quality: "draft",
+ hdrMode: "force-sdr",
+ workers: 2,
+ logger: silent,
+ producerConfig: resolveConfig({ browserGpuMode: "software" }),
+});
+try {
+ await executeRenderJob(job, projectDir, outputPath, undefined, controller.signal);
+} catch {
+ process.exitCode = 1;
+}
diff --git a/packages/producer/src/services/render/renderCancel.integration.test.ts b/packages/producer/src/services/render/renderCancel.integration.test.ts
new file mode 100644
index 0000000000..c2920521db
--- /dev/null
+++ b/packages/producer/src/services/render/renderCancel.integration.test.ts
@@ -0,0 +1,135 @@
+/**
+ * A render interrupted mid-capture must leave no frames behind. Needs Chrome + ffmpeg (integration lane). Multi-worker
+ * capture with parallel streaming off is the path that writes frames to disk.
+ */
+import { type ChildProcess, spawn, spawnSync } from "node:child_process";
+import { once } from "node:events";
+import {
+ existsSync,
+ mkdirSync,
+ mkdtempSync,
+ readFileSync,
+ readdirSync,
+ rmSync,
+ writeFileSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
+import { resolveConfig } from "@hyperframes/engine";
+import { createRenderJob, executeRenderJob } from "../renderOrchestrator.js";
+
+const FIXTURE = join(import.meta.dirname, "renderCancel.fixture.ts");
+const composition = (seconds: number) => `<!DOCTYPE html>
+<html><head><style>body { margin: 0; background: #111; }</style></head>
+<body>
+ <div id="root" data-composition-id="cancel" data-start="0" data-duration="${seconds}" data-width="320"
+ data-height="180" data-fps="30" data-no-timeline>
+ <section class="clip" style="position:absolute;inset:0;background:#036" data-start="0"
+ data-duration="${seconds}" data-track-index="1"></section>
+ </div>
+</body></html>
+`;
+
+const renderTempDirs = (dir: string): string[] =>
+ readdirSync(dir).filter((name) => name.startsWith("work-") || name.includes(".hf-transaction-"));
+const hasFrames = (dir: string): boolean =>
+ readdirSync(dir, { recursive: true, encoding: "utf-8" }).some((path) =>
+ /frame_\d+\.(jpg|png)$/.test(path),
+ );
+
+describe("render cancelled mid-capture", () => {
+ let root: string;
+ let outputDir: string;
+
+ beforeAll(() => {
+ root = mkdtempSync(join(tmpdir(), "hf-render-cancel-"));
+ outputDir = join(root, "renders");
+ mkdirSync(join(root, "long"));
+ writeFileSync(join(root, "long", "index.html"), composition(60));
+ mkdirSync(join(root, "short"));
+ writeFileSync(join(root, "short", "index.html"), composition(1));
+ });
+ afterAll(() => rmSync(root, { recursive: true, force: true }));
+
+ const start = (name: string): ChildProcess =>
+ spawn(
+ process.execPath,
+ ["--import", "tsx", FIXTURE, join(root, "long"), join(outputDir, name)],
+ {
+ env: { ...process.env, HF_CAPTURE_PARALLEL_STREAM: "false" },
+ stdio: "ignore",
+ },
+ );
+ const framesLand = () =>
+ vi.waitFor(() => expect(existsSync(outputDir) && hasFrames(outputDir)).toBe(true), {
+ timeout: 180_000,
+ interval: 100,
+ });
+
+ /** Stops a render still running after a failed assertion, letting it close its own browsers first. */
+ async function stop(child: ChildProcess): Promise<void> {
+ if (child.exitCode !== null || child.signalCode !== null) return;
+ child.kill("SIGINT");
+ const exited = once(child, "exit");
+ const timer = setTimeout(() => child.kill("SIGKILL"), 30_000);
+ await exited;
+ clearTimeout(timer);
+ }
+
+ it("removes its work dir and staging dir on Ctrl+C", async () => {
+ const child = start("cancelled.mp4");
+ try {
+ await framesLand();
+ const workDir = renderTempDirs(outputDir).find((name) => name.startsWith("work-"))!;
+ expect(JSON.parse(readFileSync(join(outputDir, workDir, "owner.json"), "utf-8")).pid).toBe(
+ child.pid,
+ );
+
+ const exited = once(child, "exit");
+ child.kill("SIGINT");
+ await exited;
+ expect(renderTempDirs(outputDir)).toEqual([]);
+ } finally {
+ await stop(child);
+ }
+ }, 240_000);
+
+ it("the next render into the folder reclaims what a killed render left", async () => {
+ const child = start("killed.mp4");
+ try {
+ await framesLand();
+ } finally {
+ if (child.exitCode === null && child.signalCode === null) {
+ // Chrome runs in its own process group, so a killed render orphans it; reap it by pid.
+ const browsers = spawnSync("pgrep", ["-P", String(child.pid)], {
+ encoding: "utf-8",
+ }).stdout;
+ const exited = once(child, "exit");
+ child.kill("SIGKILL");
+ await exited;
+ for (const pid of browsers.split("\n").filter(Boolean)) {
+ try {
+ process.kill(Number(pid), "SIGKILL");
+ } catch {
+ // Already gone.
+ }
+ }
+ }
+ }
+ expect(renderTempDirs(outputDir)).not.toEqual([]);
+
+ const job = createRenderJob({
+ fps: 30,
+ quality: "draft",
+ hdrMode: "force-sdr",
+ workers: 1,
+ logger: { info() {}, warn() {}, error() {}, debug() {} },
+ producerConfig: resolveConfig({ browserGpuMode: "software" }),
+ });
+ await executeRenderJob(job, join(root, "short"), join(outputDir, "next.mp4"));
+
+ expect(renderTempDirs(outputDir)).toEqual([]);
+ expect(existsSync(join(outputDir, "next.mp4"))).toBe(true);
+ }, 300_000);
+});
diff --git a/packages/producer/src/services/render/renderDirOwner.test.ts b/packages/producer/src/services/render/renderDirOwner.test.ts
new file mode 100644
index 0000000000..977f9efe60
--- /dev/null
+++ b/packages/producer/src/services/render/renderDirOwner.test.ts
@@ -0,0 +1,60 @@
+import { spawnSync } from "node:child_process";
+import { mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { basename, join } from "node:path";
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import { TRANSACTION_BACKUP, createOwnedRenderDir, createRenderWorkDir } from "./renderDirOwner.js";
+
+const JOB = "0f8c2b1e-5d3a-4c7b-9e21-6a4f0d8b3c19";
+
+describe("createRenderWorkDir", () => {
+ let parent: string;
+ let self: { pid: number; host: string; boot?: string; pidns?: string };
+ let dead: typeof self;
+ const plant = (name: string, owner?: object): string => {
+ mkdirSync(join(parent, name));
+ writeFileSync(join(parent, name, "frame_000001.jpg"), "x");
+ if (owner) writeFileSync(join(parent, name, "owner.json"), JSON.stringify(owner));
+ return name;
+ };
+
+ beforeEach(() => {
+ parent = mkdtempSync(join(tmpdir(), "hf-render-dir-owner-"));
+ // This process's own stamp, so planted owners differ from it only where a case says so.
+ const own = createOwnedRenderDir(join(parent, "self-"));
+ self = JSON.parse(readFileSync(join(own, "owner.json"), "utf-8"));
+ rmSync(own, { recursive: true });
+ dead = { ...self, pid: spawnSync(process.execPath, ["-e", ""]).pid! };
+ });
+ afterEach(() => rmSync(parent, { recursive: true, force: true }));
+
+ it("reclaims only render temp dirs whose owner on this host has exited", () => {
+ plant(`work-${JOB}-aB3dE9`, dead);
+ plant(".out.hf-transaction-Zx81Qa", dead);
+ plant("hf-render-Qw12Er", dead);
+ const kept = [
+ basename(createOwnedRenderDir(join(parent, `work-${JOB}-`))),
+ plant(`work-${JOB}-Cc0000`, { ...dead, host: `${self.host}-container` }),
+ plant(`work-${JOB}-Nn0000`, { ...dead, pidns: "pid:[1]" }),
+ plant(`work-${JOB}-Bb0000`, { ...dead, boot: "another-boot-or-machine" }),
+ plant(`work-${JOB}-Dd0000`, { ...self, pid: 1 }),
+ plant(`work-${JOB}-Ee0000`),
+ plant(`work-${JOB}-Ff0000`, { ...dead, pid: String(dead.pid) }),
+ plant("work-in-progress", dead),
+ plant("keep", dead),
+ ];
+
+ const created = basename(createRenderWorkDir(join(parent, `work-${JOB}-`), parent));
+
+ expect(readdirSync(parent).sort()).toEqual([...kept, created].sort());
+ });
+
+ it("keeps a dead render's staging dir that still holds the previous output", () => {
+ const staging = plant(".out.hf-transaction-Gg0000", dead);
+ writeFileSync(join(parent, staging, TRANSACTION_BACKUP), "previous render");
+
+ createRenderWorkDir(join(parent, `work-${JOB}-`), parent);
+
+ expect(readdirSync(parent)).toContain(staging);
+ });
+});
diff --git a/packages/producer/src/services/render/renderDirOwner.ts b/packages/producer/src/services/render/renderDirOwner.ts
new file mode 100644
index 0000000000..c9651fcabb
--- /dev/null
+++ b/packages/producer/src/services/render/renderDirOwner.ts
@@ -0,0 +1,111 @@
+import {
+ existsSync,
+ mkdtempSync,
+ readFileSync,
+ readdirSync,
+ readlinkSync,
+ rmSync,
+ writeFileSync,
+} from "node:fs";
+import { hostname, uptime } from "node:os";
+import { dirname, join } from "node:path";
+
+const OWNER_FILE = "owner.json";
+/** Where a staging dir parks the previous output while swapping in the new one. */
+export const TRANSACTION_BACKUP = "backup";
+// The names mkdtemp gives a render's temp dirs: work-<job uuid>-, hf-render- (Windows) and .<output>.hf-transaction-.
+const RENDER_TEMP_DIR =
+ /^(work-[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}|hf-render|\..+\.hf-transaction)-[A-Za-z0-9]{6}$/;
+
+/**
+ * Where a pid means this process: host and boot, plus the Linux pid namespace. Containers and machines on a shared
+ * folder can share a hostname, and every normal Linux host has the same root pid namespace id.
+ */
+function pidScope(): { host: string; boot?: string; pidns?: string } {
+ const attempt = (read: () => string) => {
+ try {
+ return read();
+ } catch {
+ return undefined;
+ }
+ };
+ return {
+ host: hostname(),
+ // Without /proc (macOS, Windows) the boot time, to the minute, stands in for the boot id.
+ boot:
+ attempt(() => readFileSync("/proc/sys/kernel/random/boot_id", "utf-8").trim()) ??
+ String(Math.round(Date.now() / 60_000 - uptime() / 60)),
+ pidns: attempt(() => readlinkSync("/proc/self/ns/pid")),
+ };
+}
+
+/** A render's private temp dir, stamped with the process that owns it so a later sweep can tell it was abandoned. */
+export function createOwnedRenderDir(prefix: string): string {
+ const dir = mkdtempSync(prefix);
+ const owner = { pid: process.pid, ...pidScope() };
+ writeFileSync(join(dir, OWNER_FILE), JSON.stringify(owner));
+ return dir;
+}
+
+function alive(pid: number): boolean {
+ try {
+ process.kill(pid, 0);
+ return true;
+ } catch (error) {
+ return (error as NodeJS.ErrnoException).code === "EPERM";
+ }
+}
+
+/**
+ * True only for an owner in this process's pid scope that has exited; no owner file, another scope or an unreadable
+ * owner all mean "not provably abandoned".
+ */
+function ownerIsGone(dir: string): boolean {
+ let owner: { pid?: unknown; host?: unknown; boot?: unknown; pidns?: unknown };
+ try {
+ owner = JSON.parse(readFileSync(join(dir, OWNER_FILE), "utf-8"));
+ } catch {
+ return false;
+ }
+ const scope = pidScope();
+ const { pid } = owner;
+ return (
+ owner.host === scope.host &&
+ owner.boot === scope.boot &&
+ owner.pidns === scope.pidns &&
+ typeof pid === "number" &&
+ Number.isInteger(pid) &&
+ pid > 0 &&
+ !alive(pid)
+ );
+}
+
+/** A staging dir that still holds {@link TRANSACTION_BACKUP} (its render died mid-swap) is left alone. */
+function removeAbandonedRenderDirs(parent: string): void {
+ let names: string[];
+ try {
+ names = readdirSync(parent);
+ } catch {
+ return;
+ }
+ for (const name of names) {
+ const dir = join(parent, name);
+ if (
+ !RENDER_TEMP_DIR.test(name) ||
+ !ownerIsGone(dir) ||
+ existsSync(join(dir, TRANSACTION_BACKUP))
+ )
+ continue;
+ try {
+ rmSync(dir, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 });
+ } catch {
+ // Reclaiming a dead render's dir is best-effort; it must never fail the render that found it.
+ }
+ }
+}
+
+/** Creates a render's work dir after reclaiming dirs of renders that were killed outright (no cleanup ran). */
+export function createRenderWorkDir(prefix: string, outputDir: string): string {
+ for (const dir of new Set([outputDir, dirname(prefix)])) removeAbandonedRenderDirs(dir);
+ return createOwnedRenderDir(prefix);
+}
diff --git a/packages/producer/src/services/render/stages/probeStage.test.ts b/packages/producer/src/services/render/stages/probeStage.test.ts
index b6ee9e5ed3..2528d82635 100644
--- a/packages/producer/src/services/render/stages/probeStage.test.ts
+++ b/packages/producer/src/services/render/stages/probeStage.test.ts
@@ -555,6 +555,24 @@ describe("runProbeStage — forceScreenshot threading", () => {
expect(fileServerCloseCallCount).toBe(1);
});
+ it.each([1, 2])(
+ "closes probe-owned resources when cancellation lands at the browser's check %i",
+ async (check) => {
+ resetRetryMocks();
+ const { runProbeStage } = await import("./probeStage.js");
+ const input = makeProbeInput({});
+ let checks = 0;
+ input.assertNotAborted = () => {
+ if (++checks === check) throw new Error("render cancelled");
+ };
+
+ await expect(runProbeStage(input)).rejects.toThrow("render cancelled");
+
+ expect(fileServerCloseCallCount).toBe(1);
+ expect(closeCaptureSessionCallCount).toBe(check === 1 ? 0 : 1);
+ },
+ );
+
it("launches a probe when a static-duration composition inserts video at runtime", async () => {
capturedCfgs.length = 0;
const { runProbeStage } = await import("./probeStage.js");
diff --git a/packages/producer/src/services/render/stages/probeStage.ts b/packages/producer/src/services/render/stages/probeStage.ts
index 2488ed4693..9f6f354576 100644
--- a/packages/producer/src/services/render/stages/probeStage.ts
+++ b/packages/producer/src/services/render/stages/probeStage.ts
@@ -307,417 +307,408 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
hasInsertedMedia,
});
- if (needsBrowser) {
- const reasons = [];
- if (composition.duration <= 0) reasons.push("root duration unknown");
- if (compiled.unresolvedCompositions.length > 0)
- reasons.push(`${compiled.unresolvedCompositions.length} unresolved composition(s)`);
- if (hasAutoStart) reasons.push("auto-start video(s)");
- if (hasScriptedAudio) reasons.push("scripted audio volume");
- if (hasInsertedMedia) reasons.push("runtime-created or source-mutated media");
- if (hasVariableMedia) reasons.push("variable-bound media source(s)");
-
- log.info("Launching browser for composition probe...", {
- reasons,
- });
+ try {
+ if (needsBrowser) {
+ const reasons = [];
+ if (composition.duration <= 0) reasons.push("root duration unknown");
+ if (compiled.unresolvedCompositions.length > 0)
+ reasons.push(`${compiled.unresolvedCompositions.length} unresolved composition(s)`);
+ if (hasAutoStart) reasons.push("auto-start video(s)");
+ if (hasScriptedAudio) reasons.push("scripted audio volume");
+ if (hasInsertedMedia) reasons.push("runtime-created or source-mutated media");
+ if (hasVariableMedia) reasons.push("variable-bound media source(s)");
+
+ log.info("Launching browser for composition probe...", {
+ reasons,
+ });
- fileServer = await createFileServer({
- projectDir,
- compiledDir: join(workDir, "compiled"),
- port: 0,
- preHeadScripts: [VIRTUAL_TIME_SHIM],
- fps: job.config.fps,
- });
- assertNotAborted();
+ fileServer = await createFileServer({
+ projectDir,
+ compiledDir: join(workDir, "compiled"),
+ port: 0,
+ preHeadScripts: [VIRTUAL_TIME_SHIM],
+ fps: job.config.fps,
+ });
+ assertNotAborted();
- // Motion blur averages the sub-frame samples pixel by pixel, so the frames have to be
- // captured losslessly even when the output container wants no alpha. This is the one
- // place a render picks its capture format; the encoder reads the result rather than
- // re-deriving it, so the two cannot disagree about what is on disk.
- const motionBlur = job.config.motionBlur;
- const captureFormat = resolveCaptureImageFormat({ needsAlpha, motionBlur });
- const captureOpts: CaptureOptions = {
- width,
- height,
- fps: job.config.fps,
- format: captureFormat,
- quality: captureFormat === "png" ? undefined : 80,
- variables: job.config.variables,
- deviceScaleFactor,
- motionBlur,
- requiresWebGpu: compositionRequiresWebGpu(compiled.html),
- };
-
- const PROBE_MAX_ATTEMPTS = 2;
- for (let attempt = 1; attempt <= PROBE_MAX_ATTEMPTS; attempt++) {
- const attemptStart = Date.now();
- try {
- log.info("Creating capture session...", { attempt, maxAttempts: PROBE_MAX_ATTEMPTS });
- probeSession = await createCaptureSession(
- fileServer.url,
- join(workDir, "probe"),
- captureOpts,
- null,
- probeCfg,
- );
- log.info("Waiting for composition to initialize...", { attempt });
- const heartbeat = setInterval(() => {
- const elapsed = ((Date.now() - attemptStart) / 1000).toFixed(1);
- log.info(`Still waiting for browser initialization... (${elapsed}s elapsed)`);
- }, 30_000);
+ // Motion blur averages the sub-frame samples pixel by pixel, so the frames have to be
+ // captured losslessly even when the output container wants no alpha. This is the one
+ // place a render picks its capture format; the encoder reads the result rather than
+ // re-deriving it, so the two cannot disagree about what is on disk.
+ const motionBlur = job.config.motionBlur;
+ const captureFormat = resolveCaptureImageFormat({ needsAlpha, motionBlur });
+ const captureOpts: CaptureOptions = {
+ width,
+ height,
+ fps: job.config.fps,
+ format: captureFormat,
+ quality: captureFormat === "png" ? undefined : 80,
+ variables: job.config.variables,
+ deviceScaleFactor,
+ motionBlur,
+ requiresWebGpu: compositionRequiresWebGpu(compiled.html),
+ };
+
+ const PROBE_MAX_ATTEMPTS = 2;
+ for (let attempt = 1; attempt <= PROBE_MAX_ATTEMPTS; attempt++) {
+ const attemptStart = Date.now();
try {
- await initializeSession(probeSession);
- } finally {
- clearInterval(heartbeat);
- }
- } catch (err) {
- const isTransient = isTransientBrowserError(err);
- const errMsg = err instanceof Error ? err.message : String(err);
- log.warn("Browser probe attempt failed", {
- attempt,
- maxAttempts: PROBE_MAX_ATTEMPTS,
- isTransient,
- error: errMsg,
- elapsedMs: Date.now() - attemptStart,
- });
-
- if (probeSession) {
+ log.info("Creating capture session...", { attempt, maxAttempts: PROBE_MAX_ATTEMPTS });
+ probeSession = await createCaptureSession(
+ fileServer.url,
+ join(workDir, "probe"),
+ captureOpts,
+ null,
+ probeCfg,
+ );
+ log.info("Waiting for composition to initialize...", { attempt });
+ const heartbeat = setInterval(() => {
+ const elapsed = ((Date.now() - attemptStart) / 1000).toFixed(1);
+ log.info(`Still waiting for browser initialization... (${elapsed}s elapsed)`);
+ }, 30_000);
try {
- await closeCaptureSession(probeSession);
- } catch (closeErr) {
- log.warn("Failed to close crashed probe session", {
- error: closeErr instanceof Error ? closeErr.message : String(closeErr),
- });
+ await initializeSession(probeSession);
+ } finally {
+ clearInterval(heartbeat);
}
- probeSession = null;
- }
-
- if (isTransient && attempt < PROBE_MAX_ATTEMPTS) {
- log.info("Retrying with a fresh browser session...", {
- attempt: attempt + 1,
+ } catch (err) {
+ const isTransient = isTransientBrowserError(err);
+ const errMsg = err instanceof Error ? err.message : String(err);
+ log.warn("Browser probe attempt failed", {
+ attempt,
maxAttempts: PROBE_MAX_ATTEMPTS,
+ isTransient,
+ error: errMsg,
+ elapsedMs: Date.now() - attemptStart,
});
- assertNotAborted();
- continue;
+
+ if (probeSession) {
+ try {
+ await closeCaptureSession(probeSession);
+ } catch (closeErr) {
+ log.warn("Failed to close crashed probe session", {
+ error: closeErr instanceof Error ? closeErr.message : String(closeErr),
+ });
+ }
+ probeSession = null;
+ }
+
+ if (isTransient && attempt < PROBE_MAX_ATTEMPTS) {
+ log.info("Retrying with a fresh browser session...", {
+ attempt: attempt + 1,
+ maxAttempts: PROBE_MAX_ATTEMPTS,
+ });
+ assertNotAborted();
+ continue;
+ }
+ throw err;
}
- throw err;
+ log.info("Composition ready", {
+ attempt,
+ initMs: Date.now() - attemptStart,
+ });
+ break;
}
- log.info("Composition ready", {
- attempt,
- initMs: Date.now() - attemptStart,
- });
- break;
- }
- assertNotAborted();
- // After the retry loop, probeSession is guaranteed non-null (the loop
- // either breaks with a valid session or throws on the last attempt).
- if (!probeSession) {
- throw new Error("Browser probe completed without a capture session");
- }
- lastBrowserConsole = probeSession.browserConsoleBuffer;
-
- // BeginFrame liveness probe. On SwiftShader, heavy-layer compositions
- // (multi-group nested opacity caption animations — style-N prod comps)
- // stall the FIRST BeginFrame indefinitely (tested to 30 min). The
- // auto-worker calibration catches this via its capped protocol timeout,
- // but renders with explicit `--workers N` skip calibration and would
- // hang for the full protocol timeout. One bounded BeginFrame here gives
- // ground truth for every render that probes a browser: on stall,
- // relaunch the probe session in screenshot mode and tell the sequencer
- // (via `beginFrameStalled`) to route the whole render through
- // screenshot capture — the path the baseline already uses for these
- // comps. Healthy comps pay one extra composited frame (<1s on GPU, a
- // few seconds on SwiftShader).
- if (probeSession.launchCaptureMode === "beginframe") {
- const probeTimeoutMs =
- Number(process.env.PRODUCER_BEGINFRAME_PROBE_TIMEOUT_MS) > 0
- ? Number(process.env.PRODUCER_BEGINFRAME_PROBE_TIMEOUT_MS)
- : 30_000;
- const livenessStart = Date.now();
- // Tick inside the post-warmup cushion: warmup < probe < first capture
- // keeps the session's BeginFrame frameTimeTicks monotonic.
- const probeTick = deriveBeginFrameProbeTimeTicks(
- probeSession.beginFrameTimeTicks,
- probeSession.beginFrameIntervalMs,
- );
- const alive = await probeBeginFrameLiveness(
- probeSession.page,
- probeTimeoutMs,
- probeTick,
- probeSession.beginFrameIntervalMs,
- );
assertNotAborted();
- if (alive) {
- log.info("BeginFrame liveness probe passed", {
- probeMs: Date.now() - livenessStart,
- });
- } else {
- beginFrameStalled = true;
- log.warn(
- "[Render] BeginFrame liveness probe timed out — this composition stalls " +
- "BeginFrame on this host (SwiftShader heavy-layer pattern). Relaunching " +
- "the probe browser in screenshot capture mode; the render will use " +
- "screenshot capture throughout.",
- { probeTimeoutMs },
+ // After the retry loop, probeSession is guaranteed non-null (the loop
+ // either breaks with a valid session or throws on the last attempt).
+ if (!probeSession) {
+ throw new Error("Browser probe completed without a capture session");
+ }
+ lastBrowserConsole = probeSession.browserConsoleBuffer;
+
+ // One bounded BeginFrame: on SwiftShader some heavy-layer comps stall the first one indefinitely, and
+ // `--workers N` renders skip the calibration that catches it. On a stall, relaunch in screenshot mode and set
+ // `beginFrameStalled` so the whole render captures by screenshot; healthy comps pay one extra frame.
+ if (probeSession.launchCaptureMode === "beginframe") {
+ const probeTimeoutMs =
+ Number(process.env.PRODUCER_BEGINFRAME_PROBE_TIMEOUT_MS) > 0
+ ? Number(process.env.PRODUCER_BEGINFRAME_PROBE_TIMEOUT_MS)
+ : 30_000;
+ const livenessStart = Date.now();
+ // Tick inside the post-warmup cushion: warmup < probe < first capture
+ // keeps the session's BeginFrame frameTimeTicks monotonic.
+ const probeTick = deriveBeginFrameProbeTimeTicks(
+ probeSession.beginFrameTimeTicks,
+ probeSession.beginFrameIntervalMs,
);
- lastBrowserConsole = probeSession.browserConsoleBuffer;
- await closeCaptureSession(probeSession).catch(() => {});
- probeSession = await createCaptureSession(
- fileServer.url,
- join(workDir, "probe-screenshot"),
- captureOpts,
- null,
- { ...probeCfg, forceScreenshot: true },
+ const alive = await probeBeginFrameLiveness(
+ probeSession.page,
+ probeTimeoutMs,
+ probeTick,
+ probeSession.beginFrameIntervalMs,
);
- await initializeSession(probeSession);
assertNotAborted();
- lastBrowserConsole = probeSession.browserConsoleBuffer;
+ if (alive) {
+ log.info("BeginFrame liveness probe passed", {
+ probeMs: Date.now() - livenessStart,
+ });
+ } else {
+ beginFrameStalled = true;
+ log.warn(
+ "[Render] BeginFrame liveness probe timed out — this composition stalls " +
+ "BeginFrame on this host (SwiftShader heavy-layer pattern). Relaunching " +
+ "the probe browser in screenshot capture mode; the render will use " +
+ "screenshot capture throughout.",
+ { probeTimeoutMs },
+ );
+ lastBrowserConsole = probeSession.browserConsoleBuffer;
+ await closeCaptureSession(probeSession).catch(() => {});
+ probeSession = await createCaptureSession(
+ fileServer.url,
+ join(workDir, "probe-screenshot"),
+ captureOpts,
+ null,
+ { ...probeCfg, forceScreenshot: true },
+ );
+ await initializeSession(probeSession);
+ assertNotAborted();
+ lastBrowserConsole = probeSession.browserConsoleBuffer;
+ }
}
- }
- // Bind the session only after the BeginFrame fallback, which may close the
- // original browser and replace it with a screenshot-mode session. Every
- // downstream probe must use the live replacement rather than the closed
- // session captured before the fallback.
- const session = probeSession;
+ // Bind the session only after the BeginFrame fallback, which may close the
+ // original browser and replace it with a screenshot-mode session. Every
+ // downstream probe must use the live replacement rather than the closed
+ // session captured before the fallback.
+ const session = probeSession;
- // Discover root composition duration
- if (composition.duration <= 0) {
- log.info("Discovering composition duration...");
- const discoveredDuration = await getCompositionDuration(session);
- assertNotAborted();
- log.info("Probed composition duration from browser", {
- discoveredDuration,
- staticDuration: compiled.staticDuration,
- });
- composition.duration = discoveredDuration;
- } else {
- log.info("Using static duration from data-duration attribute", {
- duration: composition.duration,
- });
- }
+ // Discover root composition duration
+ if (composition.duration <= 0) {
+ log.info("Discovering composition duration...");
+ const discoveredDuration = await getCompositionDuration(session);
+ assertNotAborted();
+ log.info("Probed composition duration from browser", {
+ discoveredDuration,
+ staticDuration: compiled.staticDuration,
+ });
+ composition.duration = discoveredDuration;
+ } else {
+ log.info("Using static duration from data-duration attribute", {
+ duration: composition.duration,
+ });
+ }
- // Resolve unresolved composition durations via window.__timelines
- if (compiled.unresolvedCompositions.length > 0) {
- const resolutions = await resolveCompositionDurations(
- session.page,
- compiled.unresolvedCompositions,
- );
- assertNotAborted();
- if (resolutions.length > 0) {
- compiled = await recompileWithResolutions(
- compiled,
- resolutions,
- projectDir,
- join(workDir, "downloads"),
+ // Resolve unresolved composition durations via window.__timelines
+ if (compiled.unresolvedCompositions.length > 0) {
+ const resolutions = await resolveCompositionDurations(
+ session.page,
+ compiled.unresolvedCompositions,
);
assertNotAborted();
- // Update composition metadata with re-parsed media
- composition.videos = compiled.videos;
- composition.audios = compiled.audios;
- composition.images = compiled.images;
- writeCompiledArtifacts(compiled, workDir, Boolean(job.config.debug));
+ if (resolutions.length > 0) {
+ compiled = await recompileWithResolutions(
+ compiled,
+ resolutions,
+ projectDir,
+ join(workDir, "downloads"),
+ );
+ assertNotAborted();
+ // Update composition metadata with re-parsed media
+ composition.videos = compiled.videos;
+ composition.audios = compiled.audios;
+ composition.images = compiled.images;
+ writeCompiledArtifacts(compiled, workDir, Boolean(job.config.debug));
+ }
}
- }
- // Discover media elements from browser DOM (catches dynamically-set src)
- log.info("Discovering media assets from browser DOM...");
- const browserMedia = await discoverMediaFromBrowser(session.page);
- assertNotAborted();
- if (browserMedia.length > 0) {
- const existingVideoIds = new Set(composition.videos.map((v) => v.id));
- const existingAudioIds = new Set(composition.audios.map((a) => a.id));
- const existingImageIds = new Set(composition.images.map((image) => image.id));
+ // Discover media elements from browser DOM (catches dynamically-set src)
+ log.info("Discovering media assets from browser DOM...");
+ const browserMedia = await discoverMediaFromBrowser(session.page);
+ assertNotAborted();
+ if (browserMedia.length > 0) {
+ const existingVideoIds = new Set(composition.videos.map((v) => v.id));
+ const existingAudioIds = new Set(composition.audios.map((a) => a.id));
+ const existingImageIds = new Set(composition.images.map((image) => image.id));
- pruneMutedBrowserMedia(composition, browserMedia, existingAudioIds);
+ pruneMutedBrowserMedia(composition, browserMedia, existingAudioIds);
- for (const el of browserMedia) {
- if (!el.src || el.src === "about:blank") continue;
- if (el.muted && el.tagName === "audio") continue;
+ for (const el of browserMedia) {
+ if (!el.src || el.src === "about:blank") continue;
+ if (el.muted && el.tagName === "audio") continue;
- // Convert absolute localhost URLs back to relative paths
- let src = el.src;
- if (fileServer && src.startsWith(fileServer.url)) {
- src = src.slice(fileServer.url.length).replace(/^\//, "");
- }
+ // Convert absolute localhost URLs back to relative paths
+ let src = el.src;
+ if (fileServer && src.startsWith(fileServer.url)) {
+ src = src.slice(fileServer.url.length).replace(/^\//, "");
+ }
- if (el.tagName === "video") {
- // fallow-ignore-next-line code-duplication
- if (existingVideoIds.has(el.id)) {
- // Reconcile to browser/runtime media metadata (runtime src can differ from static HTML).
- const existing = composition.videos.find((v) => v.id === el.id);
- if (existing) {
- const sourceChanged = existing.src !== src;
- if (sourceChanged) {
- existing.src = src;
- }
- const projectedEnd = projectBrowserEndToCompositionTimeline(
- existing.start,
- el.start,
- resolveBrowserMediaEnd(el.start, el.end, el.duration),
- );
- existing.end = reconcileBrowserMediaEnd(
- existing.end,
- projectedEnd,
- sourceChanged,
- el.durationInferred,
- );
- if (
- el.mediaStart > 0 &&
- (existing.mediaStart <= 0 ||
- Math.abs(existing.mediaStart - el.mediaStart) > BROWSER_MEDIA_EPSILON)
- ) {
- existing.mediaStart = el.mediaStart;
- }
- if (el.hasAudio && !el.muted && !existing.hasAudio) {
- existing.hasAudio = true;
- }
- if (el.loop && !existing.loop) {
- existing.loop = true;
+ if (el.tagName === "video") {
+ // fallow-ignore-next-line code-duplication
+ if (existingVideoIds.has(el.id)) {
+ // Reconcile to browser/runtime media metadata (runtime src can differ from static HTML).
+ const existing = composition.videos.find((v) => v.id === el.id);
+ if (existing) {
+ const sourceChanged = existing.src !== src;
+ if (sourceChanged) {
+ existing.src = src;
+ }
+ const projectedEnd = projectBrowserEndToCompositionTimeline(
+ existing.start,
+ el.start,
+ resolveBrowserMediaEnd(el.start, el.end, el.duration),
+ );
+ existing.end = reconcileBrowserMediaEnd(
+ existing.end,
+ projectedEnd,
+ sourceChanged,
+ el.durationInferred,
+ );
+ if (
+ el.mediaStart > 0 &&
+ (existing.mediaStart <= 0 ||
+ Math.abs(existing.mediaStart - el.mediaStart) > BROWSER_MEDIA_EPSILON)
+ ) {
+ existing.mediaStart = el.mediaStart;
+ }
+ if (el.hasAudio && !el.muted && !existing.hasAudio) {
+ existing.hasAudio = true;
+ }
+ if (el.loop && !existing.loop) {
+ existing.loop = true;
+ }
}
+ } else {
+ // New video discovered from browser
+ composition.videos.push({
+ id: el.id,
+ src,
+ start: el.start,
+ end: resolveBrowserMediaEnd(el.start, el.end, el.duration),
+ mediaStart: el.mediaStart,
+ loop: el.loop,
+ hasAudio: el.hasAudio && !el.muted,
+ });
+ existingVideoIds.add(el.id);
}
- } else {
- // New video discovered from browser
- composition.videos.push({
- id: el.id,
- src,
- start: el.start,
- end: resolveBrowserMediaEnd(el.start, el.end, el.duration),
- mediaStart: el.mediaStart,
- loop: el.loop,
- hasAudio: el.hasAudio && !el.muted,
- });
- existingVideoIds.add(el.id);
- }
- } else if (el.tagName === "audio") {
- // fallow-ignore-next-line code-duplication
- if (existingAudioIds.has(el.id)) {
- const existing = composition.audios.find((a) => a.id === el.id);
- if (existing) {
- const sourceChanged = existing.src !== src;
- if (sourceChanged) {
- existing.src = src;
- }
- const projectedEnd = projectBrowserEndToCompositionTimeline(
- existing.start,
- el.start,
- resolveBrowserMediaEnd(el.start, el.end, el.duration),
- );
- existing.end = reconcileBrowserMediaEnd(
- existing.end,
- projectedEnd,
- sourceChanged,
- el.durationInferred,
- );
- if (
- el.mediaStart > 0 &&
- (existing.mediaStart <= 0 ||
- Math.abs(existing.mediaStart - el.mediaStart) > BROWSER_MEDIA_EPSILON)
- ) {
- existing.mediaStart = el.mediaStart;
- }
- if (
- el.volume > 0 &&
- Math.abs((existing.volume ?? 1) - el.volume) > BROWSER_MEDIA_EPSILON
- ) {
- existing.volume = el.volume;
+ } else if (el.tagName === "audio") {
+ // fallow-ignore-next-line code-duplication
+ if (existingAudioIds.has(el.id)) {
+ const existing = composition.audios.find((a) => a.id === el.id);
+ if (existing) {
+ const sourceChanged = existing.src !== src;
+ if (sourceChanged) {
+ existing.src = src;
+ }
+ const projectedEnd = projectBrowserEndToCompositionTimeline(
+ existing.start,
+ el.start,
+ resolveBrowserMediaEnd(el.start, el.end, el.duration),
+ );
+ existing.end = reconcileBrowserMediaEnd(
+ existing.end,
+ projectedEnd,
+ sourceChanged,
+ el.durationInferred,
+ );
+ if (
+ el.mediaStart > 0 &&
+ (existing.mediaStart <= 0 ||
+ Math.abs(existing.mediaStart - el.mediaStart) > BROWSER_MEDIA_EPSILON)
+ ) {
+ existing.mediaStart = el.mediaStart;
+ }
+ if (
+ el.volume > 0 &&
+ Math.abs((existing.volume ?? 1) - el.volume) > BROWSER_MEDIA_EPSILON
+ ) {
+ existing.volume = el.volume;
+ }
}
+ } else {
+ composition.audios.push({
+ id: el.id,
+ src,
+ start: el.start,
+ end: resolveBrowserMediaEnd(el.start, el.end, el.duration),
+ mediaStart: el.mediaStart,
+ layer: 0,
+ volume: el.volume,
+ type: "audio",
+ });
+ existingAudioIds.add(el.id);
}
- } else {
- composition.audios.push({
- id: el.id,
- src,
- start: el.start,
- end: resolveBrowserMediaEnd(el.start, el.end, el.duration),
- mediaStart: el.mediaStart,
- layer: 0,
- volume: el.volume,
- type: "audio",
- });
- existingAudioIds.add(el.id);
- }
- } else if (el.tagName === "image") {
- if (existingImageIds.has(el.id)) {
- const existing = composition.images.find((image) => image.id === el.id);
- if (existing) {
- existing.src = src;
- const runtimeEnd = resolveBrowserMediaEnd(el.start, el.end, el.duration);
- const projectedEnd = projectBrowserEndToCompositionTimeline(
- existing.start,
- el.start,
- runtimeEnd,
- );
- if (
- projectedEnd > existing.start &&
- Math.abs(existing.end - projectedEnd) > BROWSER_MEDIA_EPSILON
- ) {
- existing.end = projectedEnd;
+ } else if (el.tagName === "image") {
+ if (existingImageIds.has(el.id)) {
+ const existing = composition.images.find((image) => image.id === el.id);
+ if (existing) {
+ existing.src = src;
+ const runtimeEnd = resolveBrowserMediaEnd(el.start, el.end, el.duration);
+ const projectedEnd = projectBrowserEndToCompositionTimeline(
+ existing.start,
+ el.start,
+ runtimeEnd,
+ );
+ if (
+ projectedEnd > existing.start &&
+ Math.abs(existing.end - projectedEnd) > BROWSER_MEDIA_EPSILON
+ ) {
+ existing.end = projectedEnd;
+ }
}
+ } else {
+ composition.images.push({
+ id: el.id,
+ src,
+ start: el.start,
+ end: resolveBrowserMediaEnd(el.start, el.end, el.duration),
+ });
+ existingImageIds.add(el.id);
}
- } else {
- composition.images.push({
- id: el.id,
- src,
- start: el.start,
- end: resolveBrowserMediaEnd(el.start, el.end, el.duration),
- });
- existingImageIds.add(el.id);
}
}
}
- }
- if (composition.audios.length > 0) {
- log.info("Discovering audio volume automation...", {
- audioCount: composition.audios.length,
- });
- const automation = await discoverAudioVolumeAutomationFromTimeline(
- session.page,
- composition.audios.map((audio) => audio.id),
- composition.duration,
- fpsToNumber(job.config.fps),
- );
- assertNotAborted();
- if (automation.length > 0) {
- const byId = new Map(automation.map((entry) => [entry.id, entry.keyframes]));
- for (const audio of composition.audios) {
- const keyframes = byId.get(audio.id);
- if (!keyframes || keyframes.length === 0) continue;
- audio.volumeKeyframes = keyframes;
- log.info(`[Probe] Runtime audio volume automation: ${audio.id}`, {
- keyframeCount: keyframes.length,
- });
+ if (composition.audios.length > 0) {
+ log.info("Discovering audio volume automation...", {
+ audioCount: composition.audios.length,
+ });
+ const automation = await discoverAudioVolumeAutomationFromTimeline(
+ session.page,
+ composition.audios.map((audio) => audio.id),
+ composition.duration,
+ fpsToNumber(job.config.fps),
+ );
+ assertNotAborted();
+ if (automation.length > 0) {
+ const byId = new Map(automation.map((entry) => [entry.id, entry.keyframes]));
+ for (const audio of composition.audios) {
+ const keyframes = byId.get(audio.id);
+ if (!keyframes || keyframes.length === 0) continue;
+ audio.volumeKeyframes = keyframes;
+ log.info(`[Probe] Runtime audio volume automation: ${audio.id}`, {
+ keyframeCount: keyframes.length,
+ });
+ }
}
}
- }
- // Runtime video discovery: for videos with auto-injected timing (data-hf-auto-start),
- // seek the GSAP timeline to find actual scene visibility windows and override start/end.
- if (composition.videos.length > 0) {
- log.info("Discovering video visibility windows...", {
- videoCount: composition.videos.length,
- });
- const visibilityWindows = await discoverVideoVisibilityFromTimeline(
- session.page,
- composition.duration,
- );
- assertNotAborted();
+ // Runtime video discovery: for videos with auto-injected timing (data-hf-auto-start),
+ // seek the GSAP timeline to find actual scene visibility windows and override start/end.
+ if (composition.videos.length > 0) {
+ log.info("Discovering video visibility windows...", {
+ videoCount: composition.videos.length,
+ });
+ const visibilityWindows = await discoverVideoVisibilityFromTimeline(
+ session.page,
+ composition.duration,
+ );
+ assertNotAborted();
- for (const win of visibilityWindows) {
- const video = composition.videos.find((v) => v.id === win.videoId);
- if (!video) continue;
- if (win.visibleStart >= 0 && win.visibleEnd > win.visibleStart) {
- video.start = win.visibleStart;
- video.end = win.visibleEnd;
- log.info(
- `[Probe] Runtime video discovery: ${video.id} visible ${win.visibleStart.toFixed(2)}s–${win.visibleEnd.toFixed(2)}s`,
- );
+ for (const win of visibilityWindows) {
+ const video = composition.videos.find((v) => v.id === win.videoId);
+ if (!video) continue;
+ if (win.visibleStart >= 0 && win.visibleEnd > win.visibleStart) {
+ video.start = win.visibleStart;
+ video.end = win.visibleEnd;
+ log.info(
+ `[Probe] Runtime video discovery: ${video.id} visible ${win.visibleStart.toFixed(2)}s–${win.visibleEnd.toFixed(2)}s`,
+ );
+ }
}
}
}
- }
- try {
await preflightCompositionAssetMediaTypes({
projectDir,
compiledDir: join(workDir, "compiled"),
@@ -730,8 +721,8 @@ export async function runProbeStage(input: ProbeStageInput): Promise<ProbeStageR
assertNotAborted();
} catch (error) {
// The orchestrator only takes ownership after this stage returns. Until
- // then, any post-browser validation failure must release both resources
- // here or a deterministic user error strands Chrome and its file server.
+ // then any failure, a cancel included, must release both resources here
+ // or it strands Chrome and its file server.
if (probeSession) {
try {
await closeCaptureSession(probeSession);
diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts
index f9c4694375..54167745cf 100644
--- a/packages/producer/src/services/renderOrchestrator.ts
+++ b/packages/producer/src/services/renderOrchestrator.ts
@@ -34,7 +34,6 @@
import {
existsSync,
mkdirSync,
- mkdtempSync,
readFileSync,
readdirSync,
rmSync,
@@ -139,6 +138,7 @@ import {
buildArtifactExpectation,
commitArtifactTransaction,
} from "./render/artifactTransaction.js";
+import { createRenderWorkDir } from "./render/renderDirOwner.js";
import {
capturePathForPlanKind,
createCapturePlan,
@@ -2855,7 +2855,7 @@ export async function executeRenderJob(
if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });
const workDir = job.config.debug
? join(debugDir, job.id)
- : mkdtempSync(resolveRenderWorkDirPrefix(outputPath, job.id));
+ : createRenderWorkDir(resolveRenderWorkDirPrefix(outputPath, job.id), outputDir);
const pipelineStart = Date.now();
const baseLog = job.config.logger ?? defaultLogger;
const logPath = job.config.debug ? join(workDir, "render.log") : null;
diff --git a/scripts/test-reachability.json b/scripts/test-reachability.json
index dae5d42c29..3f904a6f66 100644
--- a/scripts/test-reachability.json
+++ b/scripts/test-reachability.json
@@ -61,7 +61,7 @@
],
"sources": {
"packages/producer/scripts/run-test-lane.mjs": "bff02680f19eebcce0df72cdeedbbc05edfdcede3f3f60658fc0cb18ed8c59f6",
- "packages/producer/scripts/test-classification.mjs": "e68069c96f9c77a8fb415eab2f31a0e730234b352a456a0ccc95b811053132fd",
+ "packages/producer/scripts/test-classification.mjs": "fa9452d63a9f3b76ab6be7ff001acddfbf3eb117159d932dcd1510eb88b4b5c6",
".github/workflows/ci.yml#job#producer-source-tests": "3709c6ac6c9f5c7c66f57fc6c1c106aec4ffe906f919eb7cdf2a6ce1d4e0974c",
"packages/producer/package.json#script#test:unit": "ae6beda8319d166774be958ea5593cd27dbf7f6c9bce1d787a3fe28dfc3477b1",
"packages/producer/package.json#script#test:integration": "008589f37d026877595a13501cbb66abd5cbe50e21fdcac6904f45466c3f475d",