Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
/**
* Parallel Coordinator Service
*
* Coordinates parallel frame capture across multiple Puppeteer sessions.
* Auto-detects optimal worker count based on CPU/memory.
*/
import { cpus, freemem } from "os";
import { existsSync, mkdirSync, readdirSync } from "fs";
import { copyFile, readFile, rename } from "fs/promises";
import { join } from "path";
import { getHeapStatistics } from "v8";
import {
createCaptureSession,
initializeSession,
closeCaptureSession,
captureFrame,
captureFrameToBufferPipelined,
captureFrameToBuffer,
getCapturePerfSummary,
DrawElementVerificationError,
type CaptureSession,
type CaptureOptions,
type CapturePerfSummary,
type BeforeCaptureHook,
} from "./frameCapture.js";
import { psnrDb, resolveDeVerifyMinDb } from "../utils/psnr.js";
import { DEFAULT_CONFIG, type EngineConfig } from "../config.js";
import { assertSwiftShader } from "../utils/assertSwiftShader.js";
import { readWebGlVendorInfoFromCanvas } from "../utils/readWebGlVendorInfoFromCanvas.js";
import { resolveHeadlessShellPath } from "./browserManager.js";
import { getSystemTotalMb } from "./systemMemory.js";
import {
CaptureFailure,
classifyCaptureFailure,
isFatalCaptureFailure,
type CaptureWorkerDiagnostic,
} from "./captureFailure.js";
export interface WorkerTask {
workerId: number;
startFrame: number;
endFrame: number;
outputDir: string;
/**
* Offset subtracted from the absolute frame index when naming the captured
* file (`frame_<i - outputFrameOffset>.{ext}`). Default 0. Distributed
* chunks set this to the chunk's absolute startFrame so file names land
* 0-indexed within the chunk's range — the encoder reads frames
* sequentially without an `-start_number` override. The per-frame TIME
* calculation still uses the absolute frame index.
*/
outputFrameOffset?: number;
/**
* Frame stride for interleaved distribution (HF_DE_PARALLEL_STREAM spike):
* the worker captures startFrame, startFrame+stride, … < endFrame. Default 1
* (contiguous range). Interleaving keeps the ordered streaming writer's
* reorder window at O(workerCount) frames instead of O(totalFrames/N).
*/
frameStride?: number;
}
export interface WorkerResult {
workerId: number;
framesCaptured: number;
startFrame: number;
endFrame: number;
/**
* Mirrors the originating `WorkerTask.frameStride` (default 1). Required by
* `expectedFramesForTask` — without it, every interleaved (stride > 1)
* worker's expected count is computed as the full contiguous range instead
* of range/stride, so a fully-successful worker looks like it under-captured
* and gets misclassified as a silent death (see `synthesizeSilentWorkerExitError`).
*/
frameStride?: number;
durationMs: number;
perf?: CapturePerfSummary;
error?: string;
diagnostics?: string[];
failure?: CaptureFailure;
}
export interface ParallelProgress {
totalFrames: number;
capturedFrames: number;
activeWorkers: number;
workerProgress: Map<number, number>;
/** Latest lifecycle transition; absent on ordinary completed-frame updates. */
latestWorkerPhase?: ParallelWorkerPhaseDiagnostic;
/** Disk capture only: the frame file this update finished writing, readable from now on. */
landedFrame?: LandedFrame;
}
/** A captured frame file: `index` is its file index (`frame_<index>`), `path` where it was written. */
export interface LandedFrame {
index: number;
path: string;
}
export type ParallelWorkerPhase =
| "browser_launch"
| "browser_probe"
| "session_init"
| "frame_capture"
| "frame_encode";
export interface ParallelWorkerPhaseDiagnostic {
workerId: number;
phase: ParallelWorkerPhase;
frameIndex?: number;
browserExecutable: string;
browserVersion: string;
canvasDrawElement: boolean | "unknown";
gpuBackend: string;
}
export interface WorkerSizingConfig extends Partial<
Pick<
EngineConfig,
"concurrency" | "coresPerWorker" | "minParallelFrames" | "largeRenderThreshold"
>
> {
/**
* Relative per-frame capture cost for auto worker sizing. Values above 1
* represent compositions that put more CPU pressure on each Chrome worker
* than a plain DOM screenshot. Explicit --workers requests ignore this hint.
*/
captureCostMultiplier?: number;
}
type WorkerBrowserPoolDecision = {
parallel?: boolean;
};
// System-memory budget per parallel worker. Each worker is a full Chrome
// process (SwiftShader compositor + raster threads) plus its share of parent-
// process frame buffering — the old 256MB figure was ~6× under a measured
// Chrome-under-capture RSS and let memory-constrained hosts overcommit (wild
// 16GB black-slab report; 0.7.66 heap-OOM report on 24GB with 6 auto workers).
const MEMORY_PER_WORKER_MB = 1536;
// Parent-process V8 heap the coordinator itself needs regardless of worker
// count (compile artifacts, puppeteer sessions, encoder state).
const HEAP_RESERVED_MB = 1024;
// Parent-process V8 heap consumed per worker (protocol buffers + in-flight
// frame buffers). Derived from the field OOM: 6 workers exhausted a ~4GB
// default heap ⇒ >~500MB/worker + base. Caps auto sizing; an explicit
// `--workers N` is still the operator's call.
const HEAP_PER_WORKER_MB = 640;
const MIN_WORKERS = 1;
const MAX_WORKER_DIAGNOSTIC_LINES = 8;
// Hard ceiling on explicit `--workers N` requests. Above this, the cost of
// CDP-protocol dispatch through Node's main event loop and OS scheduling
// noise overwhelms any further parallelism. Bumped from 10 → 24 in hf#732
// follow-up so high-core hosts (32-96+ cores) can actually surface the
// hardware to renders that are CPU-bound on DOM capture.
const ABSOLUTE_MAX_WORKERS = 24;
// `auto` concurrency picks this many workers as the upper bound. Bumped
// from a hardcoded 6 → CPU-scaled value (floor(cpuCount/8), floor at 6,
// ceiling at 16) in hf#732 follow-up. Rationale: the prior fixed cap of 6
// left ~90 cores idle on the validation host and forced users to pass
// `--workers N` to opt in. Now `auto` matches what a thoughtful operator
// would pick by hand. The /8 divisor leaves headroom for each Chrome
// worker's SwiftShader compositor + the shader-blend thread pool, both of
// which are themselves CPU-heavy.
function defaultSafeMaxWorkers(): number {
return Math.max(6, Math.min(16, Math.floor(cpus().length / 8)));
}
const MIN_FRAMES_PER_WORKER = 30;
// Every parallel worker gets its own browser: pages in one browser share six
// connections to the file server, which a media-heavy page holds (#5548), and
// sharing also crashes BeginFrame and serializes screenshots.
export function shouldDisableBrowserPoolForParallelWorker({
parallel,
}: WorkerBrowserPoolDecision): boolean {
return Boolean(parallel);
}
export function selectWorkerDiagnostics(
lines: readonly string[],
maxLines: number = MAX_WORKER_DIAGNOSTIC_LINES,
): string[] {
return lines
.filter((line) =>
/\[(FrameCapture:ERROR|Browser:ERROR|Browser:PAGEERROR|Browser:REQUESTFAILED|Browser:HTTP\d{3})\]/.test(
line,
),
)
.slice(-maxLines);
}
function compactDiagnosticLine(line: string): string {
return line.replace(/\s+/g, " ").trim();
}
/**
* Expected frame count for a worker task, honoring its stride. Contiguous
* tasks (stride 1) expect `endFrame - startFrame`; interleaved tasks
* (stride > 1) expect `ceil((endFrame - startFrame) / stride)`, matching
* the loop shape in `captureFrameRange`.
*/
export function expectedFramesForTask(task: {
startFrame: number;
endFrame: number;
frameStride?: number;
}): number {
const stride = task.frameStride ?? 1;
return Math.max(0, Math.ceil((task.endFrame - task.startFrame) / stride));
}
/**
* Synthetic terminal-error message for a worker whose exit didn't produce
* an explicit error string but under-captured its expected frame range.
* Field signal ts=1784042064: a 1292s Windows render hard-exited during
* capture with no final error string, leaving the operator with no
* actionable trace. This message surfaces the shortfall + reruns hint
* so downstream telemetry (and operators grepping logs) can classify the
* failure instead of it disappearing silently.
*/
export function synthesizeSilentWorkerExitError(
result: Pick<WorkerResult, "workerId" | "framesCaptured" | "startFrame" | "endFrame">,
expectedFrames: number,
): string {
return (
`worker ${result.workerId} exited without terminal error string ` +
`(framesCaptured=${result.framesCaptured}, expected=${expectedFrames}, ` +
`range=[${result.startFrame}, ${result.endFrame})). ` +
`Field signal ts=1784042064: this class of failure has been reported; ` +
`consider re-run with --workers=1 to isolate.`
);
}
/**
* A worker may return without an error string yet with `framesCaptured`
* below its task's expected count — the silent-exit shape field signal
* ts=1784042064 called out. Synthesize a terminal error string in-place so
* the caller's failure filter treats it as a failure (and so the caller's
* failure message actually names what went wrong). Requires each result to
* carry `frameStride` (see `WorkerResult`) — without it, an interleaved
* worker's true `framesCaptured` (range/stride) is compared against the full
* contiguous range and every successful interleaved worker false-positives.
*/
export function flagSilentWorkerExits(results: WorkerResult[]): void {
for (const r of results) {
if (!r.error && r.framesCaptured < expectedFramesForTask(r)) {
r.error = synthesizeSilentWorkerExitError(r, expectedFramesForTask(r));
}
}
}
export function formatWorkerFailure(result: WorkerResult): string {
const errorText =
result.error && result.error.length > 0
? result.error
: synthesizeSilentWorkerExitError(result, expectedFramesForTask(result));
const base = `Worker ${result.workerId}: ${errorText}`;
if (!result.diagnostics || result.diagnostics.length === 0) return base;
const diagnostics = result.diagnostics.map(compactDiagnosticLine).join(" | ");
return `${base}; diagnostics: ${diagnostics}`;
}
/** Which constraint produced the final auto-sized worker count. */
export type WorkerSizingBound =
| "explicit"
| "too_few_frames"
| "cpu"
| "memory"
| "frames"
| "max_workers"
| "min_parallel_floor"
| "contention"
| "heap";
/**
* Full provenance of a worker-sizing decision. Threaded into render
* observability/telemetry so fleet data can answer "why N workers?" and
* "would the heap budget have prevented this OOM?" without a repro.
*/
export interface WorkerSizing {
workers: number;
boundBy: WorkerSizingBound;
cpuBasedWorkers: number;
memoryBasedWorkers: number;
frameBasedWorkers: number;
effectiveMaxWorkers: number;
/**
* How many workers the parent process's V8 heap can feed (see
* HEAP_PER_WORKER_MB); auto sizing never exceeds it.
*/
heapBasedWorkers: number;
/** V8 `heap_size_limit` for the parent process, MB. */
heapLimitMb: number;
totalMemoryMb: number;
cpuCount: number;
captureCostMultiplier: number;
/** true when the chosen count exceeds the heap budget (only an explicit request can). */
exceedsHeapAdvisory: boolean;
}
/**
* Compute the auto worker count together with the full sizing provenance.
* `calculateOptimalWorkers` is the thin legacy wrapper returning `.workers`.
*/
// The branch count is the decision provenance itself — each if records WHICH
// constraint bound, which is the entire point of the function.
// fallow-ignore-next-line complexity
export function computeWorkerSizing(
totalFrames: number,
requested?: number,
config?: WorkerSizingConfig,
): WorkerSizing {
const cpuCount = cpus().length;
// Use total memory instead of free memory — macOS reports misleadingly low
// freemem() because it aggressively caches files in "inactive" memory that
// is immediately reclaimable.
const totalMemoryMb = getSystemTotalMb();
const heapLimitMb = Math.round(getHeapStatistics().heap_size_limit / (1024 * 1024));
const heapBasedWorkers = Math.max(
1,
Math.floor((heapLimitMb - HEAP_RESERVED_MB) / HEAP_PER_WORKER_MB),
);
const cpuBasedWorkers = Math.max(1, cpuCount - 2);
const memoryBasedWorkers = Math.max(1, Math.floor((totalMemoryMb * 0.5) / MEMORY_PER_WORKER_MB));
const frameBasedWorkers = Math.floor(totalFrames / MIN_FRAMES_PER_WORKER);
const captureCostMultiplier = Math.max(1, config?.captureCostMultiplier ?? 1);
const base = {
cpuBasedWorkers,
memoryBasedWorkers,
frameBasedWorkers,
heapBasedWorkers,
heapLimitMb,
totalMemoryMb,
cpuCount,
captureCostMultiplier,
};
const finish = (workers: number, boundBy: WorkerSizingBound, effectiveMaxWorkers: number) => ({
workers,
boundBy,
effectiveMaxWorkers,
...base,
exceedsHeapAdvisory: workers > heapBasedWorkers,
});
if (requested !== undefined) {
return finish(
Math.max(MIN_WORKERS, Math.min(ABSOLUTE_MAX_WORKERS, requested)),
"explicit",
ABSOLUTE_MAX_WORKERS,
);
}
// Resolve effective values: config overrides → DEFAULT_CONFIG fallback.
const effectiveMaxWorkers = (() => {
const concurrency = config?.concurrency ?? DEFAULT_CONFIG.concurrency;
if (concurrency !== "auto") {
return Math.max(MIN_WORKERS, Math.min(ABSOLUTE_MAX_WORKERS, Math.floor(concurrency)));
}
return defaultSafeMaxWorkers();
})();
const effectiveCoresPerWorker = config?.coresPerWorker ?? DEFAULT_CONFIG.coresPerWorker;
const effectiveMinParallelFrames = config?.minParallelFrames ?? DEFAULT_CONFIG.minParallelFrames;
const effectiveLargeRenderThreshold =
config?.largeRenderThreshold ?? DEFAULT_CONFIG.largeRenderThreshold;
if (totalFrames < MIN_FRAMES_PER_WORKER * 2) {
return finish(1, "too_few_frames", effectiveMaxWorkers);
}
const optimal = Math.min(cpuBasedWorkers, memoryBasedWorkers, frameBasedWorkers);
const optimalBound: WorkerSizingBound =
optimal === cpuBasedWorkers ? "cpu" : optimal === memoryBasedWorkers ? "memory" : "frames";
const minWorkersForJob = totalFrames >= effectiveMinParallelFrames ? 2 : MIN_WORKERS;
let finalWorkers = Math.max(minWorkersForJob, Math.min(effectiveMaxWorkers, optimal));
let boundBy: WorkerSizingBound =
finalWorkers === optimal
? optimalBound
: finalWorkers === effectiveMaxWorkers && effectiveMaxWorkers < optimal
? "max_workers"
: "min_parallel_floor";
// Adaptive scaling: cap workers for large or expensive renders to prevent
// CPU contention. Each Chrome process (with SwiftShader) is CPU-heavy; too
// many concurrent captures can starve the compositor and surface as CDP
// protocol timeouts. Scale proportionally to CPU count and composition cost:
// 8 cores → 2 workers, 16 cores → 5 workers, 32 cores → 10 workers.
const weightedFrames = totalFrames * captureCostMultiplier;
const contentionThreshold = Math.max(
effectiveMinParallelFrames,
Math.floor(effectiveLargeRenderThreshold / 3),
);
if (totalFrames >= effectiveLargeRenderThreshold || weightedFrames >= contentionThreshold) {
const weightedCoresPerWorker = effectiveCoresPerWorker * captureCostMultiplier;
const cpuScaledMax = Math.max(MIN_WORKERS, Math.floor(cpuCount / weightedCoresPerWorker));
if (finalWorkers > cpuScaledMax) {
finalWorkers = cpuScaledMax;
boundBy = "contention";
}
}
if (finalWorkers > heapBasedWorkers) {
finalWorkers = heapBasedWorkers;
boundBy = "heap";
}
return finish(finalWorkers, boundBy, effectiveMaxWorkers);
}
export function calculateOptimalWorkers(
totalFrames: number,
requested?: number,
config?: WorkerSizingConfig,
): number {
return computeWorkerSizing(totalFrames, requested, config).workers;
}
export function distributeFrames(
totalFrames: number,
workerCount: number,
workDir: string,
rangeStart: number = 0,
): WorkerTask[] {
const tasks: WorkerTask[] = [];
const framesPerWorker = Math.ceil(totalFrames / workerCount);
for (let i = 0; i < workerCount; i++) {
const startFrame = rangeStart + i * framesPerWorker;
const endFrame = Math.min(rangeStart + (i + 1) * framesPerWorker, rangeStart + totalFrames);
if (startFrame >= rangeStart + totalFrames) break;
tasks.push({
workerId: i,
startFrame,
endFrame,
outputDir: join(workDir, `worker-${i}`),
outputFrameOffset: rangeStart,
});
}
return tasks;
}
/**
* Interleaved (round-robin) distribution: worker i captures frames
* i, i+N, i+2N, …. Seek-based capture makes stride access free (every frame
* is an absolute seek), and the streaming reorder window shrinks from
* totalFrames/N to N — contiguous chunks serialize workers behind the
* ordered writer (worker 1's first frame waits for ALL of worker 0's).
* HF_DE_PARALLEL_STREAM spike; disk-path capture keeps contiguous chunks.
*/
export function distributeFramesInterleaved(
totalFrames: number,
workerCount: number,
workDir: string,
rangeStart: number = 0,
): WorkerTask[] {
const tasks: WorkerTask[] = [];
for (let i = 0; i < workerCount && i < totalFrames; i++) {
tasks.push({
workerId: i,
startFrame: rangeStart + i,
endFrame: rangeStart + totalFrames,
frameStride: workerCount,
outputDir: join(workDir, `worker-${i}`),
outputFrameOffset: rangeStart,
});
}
return tasks;
}
/**
* Decide whether a parallel worker should run the per-worker SwiftShader
* assertion. Gated to worker 0 only: workers within a chunk share the same
* Chrome binary, flags, and OS/driver state, so one verification per chunk
* is sufficient. See `heygen-com/hyperframes#955`.
*/
export function shouldVerifyWorkerGpu(workerId: number, config?: Partial<EngineConfig>): boolean {
return config?.browserGpuMode === "software" && workerId === 0;
}
/**
* Race a single in-flight capture call against `signal` actually firing.
*
* `captureFrame`/`captureFrameToBuffer`/`captureFrameToBufferPipelined` take
* no abort signal of their own — a native browser call wedged inside one of
* them (WSL2 hangs the very first drawElement/BeginFrame capture at frame 0
* with no error, heygen-com/hyperframes#3441) cannot be cancelled, only
* raced. Without this, the `signal?.aborted` checks at the top of the
* `captureFrameRange` loop are a no-op the moment a worker is already
* awaiting a hung call: nothing revisits that check until the await settles,
* which on a genuine hang is never. The DE parallel-router's stall watchdog
* (`captureStreamingStage.ts`) does fire `stallController.abort()` after
* `HF_DE_STALL_MS`, but until this call actually observes the signal, that
* abort has no effect on an already-wedged worker — `executeParallelCapture`'s
* `Promise.all` waits forever, so the render hangs indefinitely and the CLI's
* circuit breaker (which only runs after `executeRenderJob` settles) never
* gets a chance to trip.
*/
export function withParallelWorkerDeadline<T>(
promise: Promise<T>,
diagnostic: ParallelWorkerPhaseDiagnostic,
timeoutMs: number,
signal?: AbortSignal,
): Promise<T> {
if (signal?.aborted) return Promise.reject(new Error("Parallel worker cancelled"));
return new Promise<T>((resolve, reject) => {
let timer: ReturnType<typeof setTimeout> | undefined;
const cleanup = () => {
if (timer) clearTimeout(timer);
signal?.removeEventListener("abort", onAbort);
};
const onAbort = () => {
cleanup();
reject(new Error("Parallel worker cancelled"));
};
signal?.addEventListener("abort", onAbort, { once: true });
if (timeoutMs > 0) {
timer = setTimeout(() => {
cleanup();
const error = new Error(
`Parallel worker operation timeout exceeded after ${timeoutMs}ms: ` +
`worker=${diagnostic.workerId} phase=${diagnostic.phase} ` +
`frame=${diagnostic.frameIndex ?? "n/a"} ` +
`browser=${diagnostic.browserExecutable} version=${diagnostic.browserVersion} ` +
`CanvasDrawElement=${diagnostic.canvasDrawElement} gpu=${diagnostic.gpuBackend}`,
);
error.name = "ParallelWorkerPhaseTimeoutError";
reject(error);
}, timeoutMs);
timer.unref?.();
}
promise.then(
(value) => {
cleanup();
resolve(value);
},
(error) => {
cleanup();
reject(error);
},
);
});
}
const LATE_LAUNCH_CLOSE_WAIT_MS = 10_000;
function resolveParallelWorkerTimeoutMs(enabled: boolean): number {
if (!enabled) return 0;
const raw = process.env.HF_DE_PARALLEL_PHASE_TIMEOUT_MS;
if (raw === "0") return 0;
const parsed = raw ? Number(raw) : Number.NaN;
return Number.isFinite(parsed) && parsed > 0 ? parsed : 30_000;
}
// fallow-ignore-next-line complexity
async function captureFrameRange(
session: CaptureSession,
task: WorkerTask,
captureOptions: CaptureOptions,
signal: AbortSignal | undefined,
onFrameCaptured:
| ((workerId: number, frameIndex: number, landedFrame?: LandedFrame) => void)
| undefined,
onFrameBuffer:
| ((frameIndex: number, buffer: Buffer, session: CaptureSession) => Promise<void>)
| undefined,
phaseTimeoutMs: number,
diagnosticFor: (phase: ParallelWorkerPhase, frameIndex?: number) => ParallelWorkerPhaseDiagnostic,
onWorkerPhase: ((diagnostic: ParallelWorkerPhaseDiagnostic) => void) | undefined,
): Promise<number> {
let framesCaptured = 0;
const outputOffset = task.outputFrameOffset ?? 0;
const stride = task.frameStride ?? 1;
const runCaptureOperation = <T>(
phase: ParallelWorkerPhase,
frameIndex: number,
operation: () => Promise<T>,
): Promise<T> => {
const diagnostic = diagnosticFor(phase, frameIndex);
if (framesCaptured === 0) onWorkerPhase?.(diagnostic);
return withParallelWorkerDeadline(operation(), diagnostic, phaseTimeoutMs, signal);
};
const awaitEncode = (frameIndex: number, encodeResult: Promise<Buffer>): Promise<Buffer> => {
const diagnostic = diagnosticFor("frame_encode", frameIndex);
if (framesCaptured === 0) onWorkerPhase?.(diagnostic);
return withParallelWorkerDeadline(encodeResult, diagnostic, phaseTimeoutMs, signal);
};
// Depth-2 pipelined drawElement produce (HF_DE_PARALLEL_STREAM spike): frame
// k's in-page worker encode overlaps frame k+stride's produce phase — the
// same shape as the sequential worker-encode loop. Only engaged when the
// session's encode worker initialized (drawElement mode) and frames stream
// back via onFrameBuffer; the ordered writer's waitForFrame provides the
// cross-worker backpressure (each worker runs at most `stride` frames ahead).
// NOTE: this branch fires for any stride, but production only ever reaches
// it via HF_DE_PARALLEL_STREAM, which always uses interleaved distribution
// (stride = workerCount). The stride=1 (contiguous) path through here is
// validation-only — exercised by tests wiring onFrameBuffer with a
// contiguous multi-worker task, not a shape real renders take. Don't
// "simplify" the flag checks around this without accounting for that.
if (onFrameBuffer && session.workerEncodeEnabled) {
const dbg = process.env.HF_DE_PAR_DEBUG === "1";
const dbgT0 = Date.now();
const dbgWin = 40 * stride;
let prev: { idx: number; encodeResult: Promise<Buffer> } | null = null;
for (let i = task.startFrame; i < task.endFrame; i += stride) {
if (signal?.aborted) throw new Error("Parallel worker cancelled");
const time = (i * captureOptions.fps.den) / captureOptions.fps.num;
if (dbg && i < task.startFrame + dbgWin) {
console.log(`[par:w${task.workerId}] +${Date.now() - dbgT0}ms produce ${i} start`);
}
const { encodeResult } = await runCaptureOperation("frame_capture", i, () =>
captureFrameToBufferPipelined(session, i - outputOffset, time),
);
// Marks the promise "handled" for Node's unhandled-rejection detector
// without affecting the real `await prev.encodeResult` below — if a
// later iteration throws (abort, downstream writeFrame failure) before
// this frame's encode is drained, it's abandoned rather than awaited,
// and would otherwise surface as an unhandled rejection during teardown.
encodeResult.catch(() => {});
if (dbg && i < task.startFrame + dbgWin) {
console.log(`[par:w${task.workerId}] +${Date.now() - dbgT0}ms produce ${i} kicked`);
}
if (prev) {
if (dbg && prev.idx < task.startFrame + dbgWin) {
console.log(
`[par:w${task.workerId}] +${Date.now() - dbgT0}ms drain ${prev.idx} await-encode`,
);
}
const buf = await awaitEncode(prev.idx, prev.encodeResult);
if (dbg && prev.idx < task.startFrame + dbgWin) {
console.log(
`[par:w${task.workerId}] +${Date.now() - dbgT0}ms drain ${prev.idx} encoded ${buf.length}B`,
);
}
await onFrameBuffer(prev.idx, buf, session);
if (dbg && prev.idx < task.startFrame + dbgWin) {
console.log(`[par:w${task.workerId}] +${Date.now() - dbgT0}ms drain ${prev.idx} written`);
}
framesCaptured++;
if (onFrameCaptured) onFrameCaptured(task.workerId, prev.idx);
}
prev = { idx: i, encodeResult };
}
if (prev) {
await onFrameBuffer(prev.idx, await awaitEncode(prev.idx, prev.encodeResult), session);
framesCaptured++;
if (onFrameCaptured) onFrameCaptured(task.workerId, prev.idx);
}
return framesCaptured;
}
for (let i = task.startFrame; i < task.endFrame; i += stride) {
if (signal?.aborted) throw new Error("Parallel worker cancelled");
const time = (i * captureOptions.fps.den) / captureOptions.fps.num;
const fileFrameIdx = i - outputOffset;
let landedFrame: LandedFrame | undefined;
if (onFrameBuffer) {
const { buffer } = await runCaptureOperation("frame_capture", i, () =>
captureFrameToBuffer(session, fileFrameIdx, time),
);
await onFrameBuffer(i, buffer, session);
} else {
const { path } = await runCaptureOperation("frame_capture", i, () =>
captureFrame(session, fileFrameIdx, time),
);
landedFrame = { index: fileFrameIdx, path };
}
framesCaptured++;
if (onFrameCaptured) onFrameCaptured(task.workerId, i, landedFrame);
}
return framesCaptured;
}
/**
* The armed self-verify sample indices this task actually captured: inside
* `[startFrame, endFrame)` and on the task's stride lattice. Mirrors the
* capture loop in `captureFrameRange` (`i += stride` from `startFrame`).
*/
export function selectVerifySampleIndicesForTask(
sampleIndices: Iterable<number>,
task: Pick<WorkerTask, "startFrame" | "endFrame" | "frameStride">,
): number[] {
const stride = task.frameStride ?? 1;
const selected: number[] = [];
for (const idx of sampleIndices) {
if (idx < task.startFrame || idx >= task.endFrame) continue;
if ((idx - task.startFrame) % stride !== 0) continue;
selected.push(idx);
}
return selected.sort((a, b) => a - b);
}
/**
* Disk-path drawElement self-verification (PRINFRA-352). Parallel DISK
* workers arm the same pre-injection ground-truth samples as the streaming
* path (`resolveParallelDeVerifySamples` even raises the density for
* multi-worker capture) — but only the streaming drain ever CHECKED them,
* so an explicit `--experimental-fast-capture --workers N` render shipped
* unverified drawElement frames. On a 16GB host, two concurrent
* hardware-GPU Chrome instances hit the documented compositor-tile-eviction
* damage class (frames displaced into vertical strips for one worker's
* whole range — reads as "corruption from the exact worker boundary").
*
* After a worker's range completes, re-read its captured files for the
* sampled indices and PSNR-compare against the session's ground truth.
* A breach throws `DrawElementVerificationError`, which the orchestrator's
* existing pinned-fallback retry converts into a screenshot re-render —
* the same recovery the streaming drain gets.
*/
/** HF_DE_PAR_DEBUG=1 gated per-worker trace line (message built lazily). */
function logParDebug(message: () => string): void {
if (process.env.HF_DE_PAR_DEBUG === "1") console.log(message());
}
/**
* Throw the verification error for a sample below the PSNR floor; log the
* pass otherwise. Split from the sampling loop for the complexity gate.
*/
function assertDiskSampleAboveFloor(
db: number,
verifyMinDb: number,
idx: number,
workerId: number,
): void {
if (db < verifyMinDb) {
// Message keeps the contiguous "drawElement self-verify" phrase —
// captureFailure's VERIFICATION_ERROR_PATTERNS classifies on it.
throw new DrawElementVerificationError(
`drawElement self-verify failed at frame ${idx} (disk path, worker ${workerId}): ` +
`${db.toFixed(1)}dB < ${verifyMinDb}dB vs pre-injection screenshot`,
{ kind: "psnr", frameIndex: idx, failedDb: db, verifyThresholdDb: verifyMinDb },
);
}
console.log(
`[Parallel] drawElement disk self-verify passed (worker ${workerId}, frame ${idx}, ` +
`${db === Infinity ? "inf" : db.toFixed(1)}dB)`,
);
}
/**
* Distinguishes infrastructure-class ffmpeg failures (spawn ENOENT, missing
* `psnr` filter) from per-sample noise (readFile races, transient tmpdir
* EPERM). Only the infrastructure class should terminate the render — the
* ffmpeg preflight in `initDrawElementOrTransparentBackground` catches these
* at bootstrap, so surfacing them here means the preflight was bypassed or
* the host ffmpeg changed mid-render.
*
* Exported for testing; the discriminator is a pure error-shape read.
*/
export function isFfmpegInfrastructureFailure(err: unknown): boolean {
if (!err || typeof err !== "object") return false;
const record = err as { code?: unknown; message?: unknown; stderr?: unknown };
if (record.code === "ENOENT") return true;
const message = typeof record.message === "string" ? record.message : "";
const stderr = typeof record.stderr === "string" ? record.stderr : "";
const text = `${message}\n${stderr}`;
// Spawn-side failures ("spawn ffmpeg ENOENT") and filter-side failures
// ("No such filter: 'psnr'", ffmpeg <=5 emits "Unknown filter 'psnr'").
return /\bENOENT\b|No such filter|Unknown filter/i.test(text);
}
/**
* Compare one captured frame file against its ground truth. Returns the
* PSNR, or null on per-sample noise (readFile races, transient EPERM,
* unparseable ffmpeg output on a single sample) — a skipped sample is not
* damage evidence and must not fail the capture. Re-throws when the error
* shape indicates the ffmpeg install itself is broken (missing binary or
* missing `psnr` filter): the drawElement self-verify safety net cannot
* possibly run in that state, and continuing would silently ship every
* remaining frame unverified.
*/
async function psnrForDiskSample(
framePath: string,
truth: Buffer,
workerId: number,
idx: number,
): Promise<number | null> {
try {
return await psnrDb(await readFile(framePath), truth);
} catch (err) {
if (isFfmpegInfrastructureFailure(err)) {
const detail = err instanceof Error ? err.message : String(err);
throw new Error(
`[Parallel] drawElement disk self-verify aborted (worker ${workerId}, frame ${idx}): ` +
`ffmpeg or the \`psnr\` filter is unavailable (${detail}). The preflight in ` +
"initDrawElementOrTransparentBackground normally catches this at bootstrap; if you " +
"hit this after a successful preflight, ffmpeg was replaced mid-render or " +
"HYPERFRAMES_FFMPEG_PATH now points at a different binary.",
{ cause: err },
);
}
console.warn(
`[Parallel] drawElement disk self-verify sample skipped (worker ${workerId}, ` +
`frame ${idx}): ${err instanceof Error ? err.message : String(err)}`,
);
return null;
}
}
// Branches are the gate conditions themselves (mode/armed/streaming guards +
// per-sample skip/breach) — already decomposed into psnrForDiskSample +
// assertDiskSampleAboveFloor; further splitting obscures the check.
// fallow-ignore-next-line complexity
export async function verifyDiskDrawElementSamples(
session: CaptureSession,
task: WorkerTask,
streaming: boolean,
): Promise<void> {
// Streaming capture verifies every sampled frame in the drain guard already.
if (streaming || session.captureMode !== "drawelement") return;
const truths = session.deVerifyFrames;
if (!truths || truths.size === 0) return;
const verifyMinDb = resolveDeVerifyMinDb();
const ext = session.options.format === "png" ? "png" : "jpg";
const offset = task.outputFrameOffset ?? 0;
for (const idx of selectVerifySampleIndicesForTask(truths.keys(), task)) {
const truth = truths.get(idx);
if (!truth) continue;
const framePath = join(task.outputDir, `frame_${String(idx - offset).padStart(6, "0")}.${ext}`);
const db = await psnrForDiskSample(framePath, truth, task.workerId, idx);
if (db === null) continue;
assertDiskSampleAboveFloor(db, verifyMinDb, idx, task.workerId);
}
}
// Inherited worker-lifecycle shape (session create → verify GPU → init →
// capture → self-verify → perf, with a classifying catch + closing finally);
// flagged only because the disk self-verify call shifted its line range into
// the changed-code audit. Not restructured by this PR.
// fallow-ignore-next-line complexity
async function executeWorkerTask(
task: WorkerTask,
serverUrl: string,
captureOptions: CaptureOptions,
createBeforeCaptureHook: () => BeforeCaptureHook | null,
signal?: AbortSignal,
onFrameCaptured?: (workerId: number, frameIndex: number, landedFrame?: LandedFrame) => void,
onFrameBuffer?: (frameIndex: number, buffer: Buffer, session: CaptureSession) => Promise<void>,
config?: Partial<EngineConfig>,
parallel?: boolean,
onFailure?: (failure: CaptureFailure) => void,
onWorkerPhase?: (diagnostic: ParallelWorkerPhaseDiagnostic) => void,
): Promise<WorkerResult> {
const startTime = Date.now();
let framesCaptured = 0;
if (!existsSync(task.outputDir)) mkdirSync(task.outputDir, { recursive: true });
let session: CaptureSession | null = null;
let perf: CapturePerfSummary | undefined;
const needsSeparateBrowsers = shouldDisableBrowserPoolForParallelWorker({ parallel });
const workerConfig: Partial<EngineConfig> | undefined = needsSeparateBrowsers
? { ...config, enableBrowserPool: false }
: config;
const phaseTimeoutMs = resolveParallelWorkerTimeoutMs(
Boolean(parallel && onFrameBuffer && workerConfig?.useDrawElement),
);
let browserVersion = "unknown";
let canvasDrawElement: boolean | "unknown" = "unknown";
let gpuBackend = "unknown";
let browserExecutable = resolveHeadlessShellPath(workerConfig) ?? "system/default";
const diagnosticFor = (
phase: ParallelWorkerPhase,
frameIndex?: number,
): ParallelWorkerPhaseDiagnostic => ({
workerId: task.workerId,
phase,
frameIndex,
browserExecutable,
browserVersion,
canvasDrawElement,
gpuBackend,
});
const runPhase = <T>(phase: ParallelWorkerPhase, operation: () => Promise<T>): Promise<T> => {
const diagnostic = diagnosticFor(phase);
onWorkerPhase?.(diagnostic);
return withParallelWorkerDeadline(operation(), diagnostic, phaseTimeoutMs, signal);
};
try {
const launch = createCaptureSession(
serverUrl,
task.outputDir,
captureOptions,
createBeforeCaptureHook(),
workerConfig,
);
session = await runPhase("browser_launch", () => launch).catch(async (error: unknown) => {
// Close a launch that lost to cancel or its deadline before settling: Puppeteer's exit hook skips browsers.
const closing = launch.then(closeCaptureSession).catch(() => {});
await Promise.race([
closing,
new Promise((settle) => setTimeout(settle, LATE_LAUNCH_CLOSE_WAIT_MS).unref()),
]);
throw error;
});
const activeSession = session;
browserExecutable = activeSession.browser?.process?.()?.spawnfile || browserExecutable;
logParDebug(() => `[par:w${task.workerId}] session created`);
browserVersion = await runPhase(
"browser_probe",
async () => (await activeSession.browser?.version?.().catch(() => "unknown")) ?? "unknown",
);
await runPhase("session_init", async () => {
// Worker-0-only SwiftShader assertion — see `shouldVerifyWorkerGpu` and #955.
if (shouldVerifyWorkerGpu(task.workerId, workerConfig)) {
await assertSwiftShader(activeSession.page, readWebGlVendorInfoFromCanvas);
}
await initializeSession(activeSession);
});
canvasDrawElement = activeSession.captureMode === "drawelement";
gpuBackend = activeSession.gpuRenderer ?? "unknown";
logParDebug(
() =>
`[par:w${task.workerId}] init done (mode=${session?.captureMode} workerEncode=${session?.workerEncodeEnabled === true})`,
);
framesCaptured = await captureFrameRange(
activeSession,
task,
captureOptions,
signal,
onFrameCaptured,
onFrameBuffer,
phaseTimeoutMs,
diagnosticFor,
onWorkerPhase,
);
await verifyDiskDrawElementSamples(activeSession, task, Boolean(onFrameBuffer));
perf = getCapturePerfSummary(activeSession);
return {
workerId: task.workerId,
framesCaptured,
startFrame: task.startFrame,
endFrame: task.endFrame,
frameStride: task.frameStride,
durationMs: Date.now() - startTime,
perf,
};
} catch (error) {
const diagnostics = session ? selectWorkerDiagnostics(session.browserConsoleBuffer) : [];
const workerDiagnostic: CaptureWorkerDiagnostic = {
workerId: task.workerId,
framesCaptured,
startFrame: task.startFrame,
endFrame: task.endFrame,
lines: diagnostics,
};
const failure = classifyCaptureFailure(error, {
signal,
workerDiagnostics: [workerDiagnostic],
});
onFailure?.(failure);
return {
workerId: task.workerId,
framesCaptured,
startFrame: task.startFrame,
endFrame: task.endFrame,
frameStride: task.frameStride,
durationMs: Date.now() - startTime,
perf,
error: failure.message,
diagnostics: diagnostics.length > 0 ? diagnostics : undefined,
failure,
};
} finally {
if (session) await closeCaptureSession(session).catch(() => {});
}
}
/**
* drawElement self-verify sample count for multi-worker capture. Each worker
* arms the same shared sample grid but drains only ~1/N of it, and N
* concurrent hardware-GPU browsers are exactly where compositor-tile damage
* shows up (wild 0.7.52 black-slab report) — so density rises with worker
* count: 4 base + 2 per extra worker, clamped to the verify path's max of 8.
* A caller-set value passes through untouched, and explicit HF_DE_VERIFY
* still overrides inside the session.
*/
export function resolveParallelDeVerifySamples(
callerValue: number | undefined,
workerCount: number,
): number | undefined {
if (callerValue !== undefined) return callerValue;
if (workerCount <= 1) return undefined;
return Math.min(8, 4 + 2 * (workerCount - 1));
}
/**
* Whether one worker's failure ends the whole pool. On the disk path a
* transient death (Target closed, Page crashed) is not fatal: the
* orchestrator's adaptive retry re-captures that worker's missing frames. On
* the streaming path (`onFrameBuffer` present) there is no per-worker retry
* and the dead worker's frames are gone, so its peers would park in the
* ordered writer waiting for a frame that never comes until the producer's
* no-progress watchdog relabelled the death as a stall a minute later.
* Every non-cancelled failure is therefore pool-fatal there; `cancelled`
* means the pool was already aborted and there is nothing left to propagate.
*/
export function isPoolFatalWorkerFailure(failure: CaptureFailure, streaming: boolean): boolean {
if (streaming) return failure.kind !== "cancelled";
return isFatalCaptureFailure(failure);
}
export interface ParallelCaptureHooks {
/**
* The first pool-fatal worker failure, delivered BEFORE peers are aborted
* so a streaming caller can release anything parked on the dead worker's
* frame (the ordered writer) with the original error, not a stall.
*/
onWorkerFailure?: (failure: CaptureFailure) => void;
}
/**
* The pool's single failure gate. Both halves of the contract the streaming
* stage relies on live here so they can be pinned without a browser: the
* first pool-fatal failure reaches `hooks.onWorkerFailure` with the ORIGINAL
* `CaptureFailure` before the peers are aborted, and a hook that throws cannot
* skip that abort (the pool still rejects with its own classified failure,
* not the hook's error). Later failures are ignored: the first one owns the
* abort reason. `onFailure` runs synchronously at the tail of a worker's
* catch, before that worker's promise settles, so the hook always precedes
* the pool's rejection. Exported for tests.
*/
export function createPoolFailureHandler(args: {
streaming: boolean;
peerController: AbortController;
hooks?: ParallelCaptureHooks;
}): {
onFailure: (failure: CaptureFailure) => void;
firstFatalFailure: () => CaptureFailure | undefined;
} {
let firstFatalFailure: CaptureFailure | undefined;
return {
firstFatalFailure: () => firstFatalFailure,
onFailure: (failure) => {
if (firstFatalFailure || !isPoolFatalWorkerFailure(failure, args.streaming)) return;
firstFatalFailure = failure;
try {
args.hooks?.onWorkerFailure?.(failure);
} catch {
// A caller-supplied hook must not be able to disable the pool abort;
// the worker's classified failure is what the pool reports.
} finally {
args.peerController.abort(failure);
}
},
};
}
export async function executeParallelCapture(
serverUrl: string,
workDir: string,
tasks: WorkerTask[],
captureOptions: CaptureOptions,
createBeforeCaptureHook: () => BeforeCaptureHook | null,
signal?: AbortSignal,
onProgress?: (progress: ParallelProgress) => void,
onFrameBuffer?: (frameIndex: number, buffer: Buffer, session: CaptureSession) => Promise<void>,
config?: Partial<EngineConfig>,
hooks?: ParallelCaptureHooks,
): Promise<WorkerResult[]> {
// `endFrame - startFrame` is the correct per-task frame count for contiguous
// tasks (stride 1), but for interleaved tasks (stride = workerCount) each
// task spans nearly the full range while only actually capturing 1/stride
// of it — dividing by stride here matches the loop in `captureFrameRange`
// (`i += stride`) so progress doesn't plateau at ~1/workerCount.
const totalFrames = tasks.reduce(
(sum, t) => sum + Math.ceil((t.endFrame - t.startFrame) / (t.frameStride ?? 1)),
0,
);
const workerProgress = new Map<number, number>();
const workerPhases = new Map<number, ParallelWorkerPhaseDiagnostic>();
for (const task of tasks) workerProgress.set(task.workerId, 0);
const onFrameCaptured = (workerId: number, _frameIndex: number, landedFrame?: LandedFrame) => {
const current = workerProgress.get(workerId) || 0;
workerProgress.set(workerId, current + 1);
if (onProgress) {
const capturedFrames = Array.from(workerProgress.values()).reduce((a, b) => a + b, 0);
onProgress({
totalFrames,
capturedFrames,
activeWorkers: tasks.length,
workerProgress: new Map(workerProgress),
...(landedFrame && { landedFrame }),
});
}
};
const onWorkerPhase = (diagnostic: ParallelWorkerPhaseDiagnostic) => {
workerPhases.set(diagnostic.workerId, diagnostic);
if (!onProgress) return;
const capturedFrames = Array.from(workerProgress.values()).reduce((a, b) => a + b, 0);
onProgress({
totalFrames,
capturedFrames,
activeWorkers: tasks.length,
workerProgress: new Map(workerProgress),
latestWorkerPhase: diagnostic,
});
};
const parallel = tasks.length > 1;
const deVerifySamples = resolveParallelDeVerifySamples(
captureOptions.deVerifySamples,
tasks.length,
);
const workerCaptureOptions: CaptureOptions =
deVerifySamples === captureOptions.deVerifySamples
? captureOptions
: { ...captureOptions, deVerifySamples };
const peerController = new AbortController();
const workerSignal = signal
? AbortSignal.any([signal, peerController.signal])
: peerController.signal;
const failureHandler = createPoolFailureHandler({
streaming: Boolean(onFrameBuffer),
peerController,
hooks,
});
const onFailure = failureHandler.onFailure;
const results = await Promise.all(
tasks.map((task) =>
executeWorkerTask(
task,
serverUrl,
workerCaptureOptions,
createBeforeCaptureHook,
workerSignal,
onFrameCaptured,
onFrameBuffer,
config,
parallel,
onFailure,
onWorkerPhase,
),
),
);
flagSilentWorkerExits(results);
const errors = results.filter((r) => r.failure || r.error);
if (errors.length > 0) {
const errorMessages = errors.map(formatWorkerFailure).join("; ");
const representative =
failureHandler.firstFatalFailure() ?? errors.find((result) => result.failure)?.failure;
const workerDiagnostics = errors.flatMap((result) => result.failure?.workerDiagnostics ?? []);
throw new CaptureFailure({
kind: representative?.kind ?? "io",
message: `[Parallel] Capture failed: ${errorMessages}`,
cause: representative,
workerDiagnostics,
});
}
return results;
}
export async function mergeWorkerFrames(
workDir: string,
tasks: WorkerTask[],
outputDir: string,
): Promise<number> {
if (!existsSync(outputDir)) mkdirSync(outputDir, { recursive: true });
let totalFrames = 0;
const sortedTasks = [...tasks].sort((a, b) => a.startFrame - b.startFrame);
for (const task of sortedTasks) {
if (!existsSync(task.outputDir)) {
continue;
}
const files = readdirSync(task.outputDir)
.filter((f) => f.startsWith("frame_") && (f.endsWith(".jpg") || f.endsWith(".png")))
.sort();
const copyTasks = files.map(async (file) => {
const sourcePath = join(task.outputDir, file);
const targetPath = join(outputDir, file);
try {
await rename(sourcePath, targetPath);
} catch {
await copyFile(sourcePath, targetPath);
}
});
await Promise.all(copyTasks);
totalFrames += files.length;
}
return totalFrames;
}
export function getSystemResources(): {
cpuCores: number;
totalMemoryMB: number;
freeMemoryMB: number;
recommendedWorkers: number;
} {
return {
cpuCores: cpus().length,
totalMemoryMB: getSystemTotalMb(),
freeMemoryMB: Math.round(freemem() / (1024 * 1024)),
recommendedWorkers: calculateOptimalWorkers(1000),
};
}