Skip to content

Code-first video & image, rendered natively on WebGPU.

Log render phases compactly so agents can follow long exports

Claude4f5339eee88f
diff --git a/.changeset/render-progress-phases.md b/.changeset/render-progress-phases.md
new file mode 100644
index 00000000..8aa6c40c
--- /dev/null
+++ b/.changeset/render-progress-phases.md
@@ -0,0 +1,9 @@
+---
+"framefields": minor
+---
+
+Clearer render progress, sized for agents' context:
+- `renderVideo` logs each phase (prepare, frames, audio mix, audio encode, mux, GIF conversion) and a summary with per-phase timings. Frames log at every 10% with fps and ETA instead of every 10 frames, so an export prints about 15 lines whatever its length.
+- A phase that runs long logs `Still …` lines (after 10 s, then backing off to every 5 min) naming the phase and its progress, so the audio mix or the mux after the last frame no longer looks like a hang. A failed render names the phase it stopped in.
+- `onProgress` reports the new phases `prepare`, `audio-encode`, `mux`, `convert` and `done` alongside `render` and `audio`, with `elapsedMs`, and `fps`, `etaSec`, `phaseProgress` and `detail` where they apply. The audio mix reports each clip it mixes (`mixAudioTracks` takes an `onProgress` option), which also keeps the render worker's stall detector from firing on long mixes.
+- Removes the unused `video-worker` entry from `@framefields/renderer`.
diff --git a/apps/renderer-service/src/engine.gpu.test.ts b/apps/renderer-service/src/engine.gpu.test.ts
index fd4dc128..f7c08197 100644
--- a/apps/renderer-service/src/engine.gpu.test.ts
+++ b/apps/renderer-service/src/engine.gpu.test.ts
@@ -72,15 +72,24 @@ gpu("engine hooks", () => {
backgroundColor: "#204060",
});
- it("reports progress for every frame, then the audio mix", async () => {
+ it("reports progress for every frame, then each phase after them", async () => {
const seen: string[] = [];
const result = await film().renderVideo({
onProgress: (p) => seen.push(`${p.phase}:${p.frame}/${p.totalFrames}`),
});
await result.cleanup();
- expect(seen.filter((s) => s.startsWith("render:"))).toHaveLength(48);
- expect(seen.at(-2)).toBe("render:48/48");
- expect(seen.at(-1)).toBe("audio:48/48");
+ expect(seen.slice(0, 3)).toEqual([
+ "prepare:0/48",
+ "render:0/48",
+ "render:1/48",
+ ]);
+ // The phase start, then one event per frame.
+ expect(seen.filter((s) => s.startsWith("render:"))).toHaveLength(49);
+ const last = seen.indexOf("render:48/48");
+ expect(seen[last + 1]).toBe("audio:48/48");
+ expect(seen).toContain("audio-encode:48/48");
+ expect(seen).toContain("mux:48/48");
+ expect(seen.at(-1)).toBe("done:48/48");
}, 120_000);
it("aborts between frames with the signal's reason", async () => {
diff --git a/apps/renderer-service/src/render/render-step.ts b/apps/renderer-service/src/render/render-step.ts
index 00d5900c..4769f42a 100644
--- a/apps/renderer-service/src/render/render-step.ts
+++ b/apps/renderer-service/src/render/render-step.ts
@@ -8,6 +8,7 @@ import type {
StepContext,
} from "../jobs/pipeline.js";
import type { JobInput, RenderSettings } from "../jobs/schema.js";
+import type { JobProgress } from "../jobs/store.js";
import { rendererLogger } from "../logger.js";
import { gpuStatus } from "../worker/gpu.js";
import { workerHealth } from "../worker/health.js";
@@ -189,6 +190,7 @@ async function renderOutput(
const { ext, contentType } = containerFor(codec);
const started = Date.now();
let lastEmit = 0;
+ let lastPhase: string | undefined;
let expectedFrames: number | undefined;
const result = await output.target.renderVideo({
outputPath: `${base}.${ext}`,
@@ -198,30 +200,24 @@ async function renderOutput(
qa: settings.qa,
renderer,
signal: controller.signal,
- onProgress: (p: {
- phase: string;
- frame: number;
- totalFrames: number;
- }) => {
+ onProgress: (p: EngineProgress) => {
stall.touch();
expectedFrames = p.totalFrames;
+ // The job reports its own "done" after the upload.
+ if (p.phase === "done") return;
const now = Date.now();
+ const lastFrame = p.phase === "render" && p.frame >= p.totalFrames;
if (
- now - lastEmit < PROGRESS_INTERVAL_MS &&
- p.frame < p.totalFrames
+ p.phase === lastPhase &&
+ !lastFrame &&
+ now - lastEmit < PROGRESS_INTERVAL_MS
)
return;
lastEmit = now;
- const fps = p.frame / Math.max(0.001, (now - started) / 1000);
+ lastPhase = p.phase;
ctx.progress({
- phase: p.phase === "audio" ? "encoding" : "rendering",
+ ...toJobProgress(p, (now - started) / 1000),
output: count > 1 ? index : undefined,
- frame: p.frame,
- total: p.totalFrames,
- fps: Math.round(fps * 10) / 10,
- etaSec: Math.round(
- (p.totalFrames - p.frame) / Math.max(fps, 0.001),
- ),
});
},
});
@@ -320,6 +316,61 @@ async function renderOutput(
}
}
+/**
+ * What `renderVideo({ onProgress })` reports. Engines before the phase
+ * breakdown send only "render" and "audio", without the optional fields.
+ */
+interface EngineProgress {
+ phase: string;
+ frame: number;
+ totalFrames: number;
+ fps?: number;
+ etaSec?: number;
+ detail?: string;
+}
+
+/** What each engine phase after the frames is doing, for the job message. */
+const AFTER_FRAMES: Record<string, string> = {
+ audio: "mixing audio",
+ "audio-encode": "encoding audio",
+ mux: "muxing",
+ convert: "converting to GIF",
+};
+
+/** Maps an engine progress event to the job's progress. */
+export function toJobProgress(
+ p: EngineProgress,
+ elapsedSec: number,
+): JobProgress {
+ if (p.phase === "prepare") {
+ return {
+ phase: "rendering",
+ frame: 0,
+ total: p.totalFrames,
+ message: "preparing",
+ };
+ }
+ if (p.phase === "render") {
+ const fps = p.fps ?? p.frame / Math.max(0.001, elapsedSec);
+ return {
+ phase: "rendering",
+ frame: p.frame,
+ total: p.totalFrames,
+ fps: Math.round(fps * 10) / 10,
+ etaSec:
+ p.etaSec ??
+ Math.round((p.totalFrames - p.frame) / Math.max(fps, 0.001)),
+ };
+ }
+ const label = AFTER_FRAMES[p.phase] ?? p.phase;
+ return {
+ phase: "encoding",
+ frame: p.frame,
+ total: p.totalFrames,
+ message: p.detail ? `${label} (${p.detail})` : label,
+ };
+}
+
export function containerFor(codec: string): {
ext: string;
contentType: string;
diff --git a/apps/renderer-service/src/render/render.test.ts b/apps/renderer-service/src/render/render.test.ts
index eef5539c..8ae4c9f5 100644
--- a/apps/renderer-service/src/render/render.test.ts
+++ b/apps/renderer-service/src/render/render.test.ts
@@ -8,7 +8,7 @@ import { CrashJournal } from "../jobs/journal.js";
import { testConfig } from "../test-helpers.js";
import { recycleReason } from "../worker/recycle.js";
import { buildPlan, isRenderable } from "./load.js";
-import { classifyRenderError } from "./render-step.js";
+import { classifyRenderError, toJobProgress } from "./render-step.js";
import { ExitTrap, openSandbox } from "./sandbox.js";
let ws: string;
@@ -404,3 +404,49 @@ describe("crash journal", () => {
expect(journal.recover()).toBeUndefined();
});
});
+
+describe("render progress", () => {
+ it("maps frames to rendering, with the engine's rate when it sends one", () => {
+ expect(
+ toJobProgress(
+ { phase: "render", frame: 60, totalFrames: 120, fps: 30, etaSec: 2 },
+ 4,
+ ),
+ ).toEqual({
+ phase: "rendering",
+ frame: 60,
+ total: 120,
+ fps: 30,
+ etaSec: 2,
+ });
+ // Older engines send no rate: it is worked out from the elapsed time.
+ expect(
+ toJobProgress({ phase: "render", frame: 60, totalFrames: 120 }, 4),
+ ).toMatchObject({ fps: 15, etaSec: 4 });
+ });
+
+ it("names the phase after the frames in the message", () => {
+ expect(
+ toJobProgress(
+ {
+ phase: "audio",
+ frame: 120,
+ totalFrames: 120,
+ detail: "3/7 clips: music.mp3",
+ },
+ 9,
+ ),
+ ).toEqual({
+ phase: "encoding",
+ frame: 120,
+ total: 120,
+ message: "mixing audio (3/7 clips: music.mp3)",
+ });
+ expect(
+ toJobProgress({ phase: "mux", frame: 120, totalFrames: 120 }, 9),
+ ).toMatchObject({ phase: "encoding", message: "muxing" });
+ expect(
+ toJobProgress({ phase: "prepare", frame: 0, totalFrames: 120 }, 0),
+ ).toMatchObject({ phase: "rendering", frame: 0, message: "preparing" });
+ });
+});
diff --git a/packages/compositions/src/utils/audio-extractor.ts b/packages/compositions/src/utils/audio-extractor.ts
index fe14e0ad..2efc4065 100644
--- a/packages/compositions/src/utils/audio-extractor.ts
+++ b/packages/compositions/src/utils/audio-extractor.ts
@@ -548,6 +548,35 @@ class MixBus {
}
}
+/** Progress reported by `mixAudioTracks` as each audio or video clip is mixed. */
+export interface AudioMixProgress {
+ /** Clips mixed so far. */
+ done: number;
+ /** Audio and video clips in the tree; muted or empty ones may be skipped. */
+ total: number;
+ /** File name of the clip just mixed (no directory or query string). */
+ source?: string;
+}
+
+export interface MixAudioOptions {
+ onProgress?: (progress: AudioMixProgress) => void;
+}
+
+/** A clip's file name for progress, without paths or signed-URL queries. */
+function sourceLabel(url: string): string {
+ if (url.startsWith("data:")) return "inline audio";
+ const name = url.split(/[?#]/)[0].split(/[\\/]/).pop();
+ return name ? decodeURIComponentSafe(name) : "audio";
+}
+
+function decodeURIComponentSafe(s: string): string {
+ try {
+ return decodeURIComponent(s);
+ } catch {
+ return s;
+ }
+}
+
/**
* Recursively parses the VirtualMediaData tree and mixes all active audio tracks
* into a single unified stereo PCM stream (Float32Array channels).
@@ -558,6 +587,7 @@ export async function mixAudioTracks(
targetSampleRate = 48000,
device?: GPUDevice,
renderId?: string,
+ options: MixAudioOptions = {},
): Promise<{ channels: Float32Array[]; sampleRate: number }> {
mediaLogger.debug(
`[AudioExtractor] mixAudioTracks started. FPS: ${fps}, targetSampleRate: ${targetSampleRate}`,
@@ -844,6 +874,14 @@ export async function mixAudioTracks(
}
await yieldToMain();
}
+ if (mediaType === "Audio" || mediaType === "Video") {
+ clipsDone++;
+ options.onProgress?.({
+ done: clipsDone,
+ total: clipsTotal,
+ ...(params.sourceUrl && { source: sourceLabel(params.sourceUrl) }),
+ });
+ }
}
// Traverse children recursively
@@ -938,6 +976,26 @@ export async function mixAudioTracks(
}
const normalized = normalizeTimeline(virtualMedia, fps);
+
+ // Clips to mix, for progress: one per segment of each audio or video source.
+ let clipsTotal = 0;
+ let clipsDone = 0;
+ if (options.onProgress) {
+ const count = (node: VirtualMediaData | undefined) => {
+ if (!node) return;
+ const op = node.operation;
+ if (op?.op === "source") {
+ const mediaType = getMediaType(node);
+ if (mediaType === "Audio" || mediaType === "Video") {
+ clipsTotal += Math.max(1, op.timeline?.segments?.length ?? 0);
+ }
+ }
+ for (const child of node.children ?? []) count(child);
+ };
+ count(normalized);
+ options.onProgress({ done: 0, total: clipsTotal });
+ }
+
const rootChannels = await traverse(normalized, 0, 0, 1, 1, null);
// Output mixed buffer is stereo; a root mix of the same length is used as is
diff --git a/packages/framefields/src/index.ts b/packages/framefields/src/index.ts
index 70aed9a2..060ee7c0 100644
--- a/packages/framefields/src/index.ts
+++ b/packages/framefields/src/index.ts
@@ -1094,7 +1094,11 @@ export interface RenderVideoOptions {
* `signal.reason` and no file is left behind.
*/
signal?: AbortSignal;
- /** Called as frames finish (`phase: "render"`), then once for the audio mix. */
+ /**
+ * Called as the render moves through its phases: "prepare", each frame
+ * ("render"), "audio" (per clip mixed), "audio-encode", "mux", "convert"
+ * (GIF only) and "done". The engine logs the same phases on its own.
+ */
onProgress?: (progress: RenderProgress) => void;
}
diff --git a/packages/framefields/src/render-progress.test.ts b/packages/framefields/src/render-progress.test.ts
new file mode 100644
index 00000000..f3ef2576
--- /dev/null
+++ b/packages/framefields/src/render-progress.test.ts
@@ -0,0 +1,94 @@
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { describe, expect, it } from "vitest";
+import {
+ Composition,
+ encodeStereoWav,
+ Layer,
+ type RenderProgress,
+} from "./index.js";
+
+async function tone(dir: string): Promise<string> {
+ const sampleRate = 48_000;
+ const samples = new Float32Array(sampleRate);
+ for (let i = 0; i < samples.length; i++) {
+ samples[i] = 0.3 * Math.sin((2 * Math.PI * 440 * i) / sampleRate);
+ }
+ const file = path.join(dir, "tone.wav");
+ await fs.writeFile(file, encodeStereoWav([samples, samples], { sampleRate }));
+ return file;
+}
+
+function film(audio: string): Composition {
+ const comp = new Composition({
+ width: 64,
+ height: 64,
+ fps: 30,
+ durationFrames: 30,
+ backgroundColor: "#202020",
+ });
+ comp.add(Layer.audio(audio, { id: "tone", durationFrames: 30 }));
+ return comp;
+}
+
+/** Phases in the order they were first reported. */
+function phases(events: RenderProgress[]): string[] {
+ return events
+ .map((e) => e.phase)
+ .filter((phase, i, all) => all.indexOf(phase) === i);
+}
+
+describe("renderVideo progress", () => {
+ it("reports each phase in order, with the audio clips mixed", async () => {
+ const dir = await fs.mkdtemp(path.join(os.tmpdir(), "ff-progress-"));
+ const events: RenderProgress[] = [];
+ const result = await film(await tone(dir)).renderVideo({
+ outputPath: path.join(dir, "out.mp4"),
+ onProgress: (p) => events.push({ ...p }),
+ });
+ await result.cleanup();
+
+ expect(phases(events)).toEqual([
+ "prepare",
+ "render",
+ "audio",
+ "audio-encode",
+ "mux",
+ "done",
+ ]);
+ const frames = events.filter((e) => e.phase === "render");
+ expect(frames.at(-1)).toMatchObject({ frame: 30, totalFrames: 30 });
+ for (let i = 1; i < frames.length; i++) {
+ expect(frames[i].frame).toBeGreaterThanOrEqual(frames[i - 1].frame);
+ }
+ expect(events.filter((e) => e.phase === "audio").at(-1)).toMatchObject({
+ detail: "1/1 clips: tone.wav",
+ phaseProgress: 1,
+ });
+ expect(
+ events.filter((e) => e.phase === "audio-encode").at(-1)?.phaseProgress,
+ ).toBe(1);
+ const elapsed = events.map((e) => e.elapsedMs);
+ expect(elapsed).toEqual([...elapsed].sort((a, b) => a - b));
+ }, 60_000);
+
+ it("skips the frame phase for audio-only output", async () => {
+ const dir = await fs.mkdtemp(path.join(os.tmpdir(), "ff-progress-"));
+ const events: RenderProgress[] = [];
+ const result = await film(await tone(dir)).renderVideo({
+ outputPath: path.join(dir, "out.m4a"),
+ codec: "aac",
+ onProgress: (p) => events.push({ ...p }),
+ });
+ await result.cleanup();
+
+ expect(phases(events)).toEqual([
+ "prepare",
+ "audio",
+ "audio-encode",
+ "mux",
+ "done",
+ ]);
+ }, 60_000);
+});
diff --git a/packages/renderer/src/headless-webgpu-renderer.ts b/packages/renderer/src/headless-webgpu-renderer.ts
index d4a71b82..b6bcfb9d 100644
--- a/packages/renderer/src/headless-webgpu-renderer.ts
+++ b/packages/renderer/src/headless-webgpu-renderer.ts
@@ -50,6 +50,7 @@ import { preloadFonts } from "./asset-preloader.js";
import { bootstrapMediabunny } from "./bootstrap-mediabunny.js";
import { discoverAndRegisterNodeRenderers } from "./dynamic-node-discovery.js";
import { rendererLogger } from "./logger.js";
+import { type RenderProgress, RenderReporter } from "./render-reporter.js";
// Vision reports its inference backend once per process (see
// VISION_BACKEND_LISTENER in @framefields/vision).
@@ -264,35 +265,6 @@ if (typeof skiaProto.getAttribute === "undefined") {
skiaProto.getAttribute = () => null;
}
-// ─── Types ──────────────────────────────────────────────────────────────────────
-
-export interface WorkerInput {
- virtualMedia: VirtualMediaData;
- width: number;
- height: number;
- fps: number;
- startFrame: number;
- endFrame: number;
- renderId: string;
- outputFormat: "png" | "pixels";
- workerIndex: number;
-}
-
-export type WorkerMessage =
- | { type: "frame"; renderId: string; frameIndex: number; pixels: Uint8Array }
- | { type: "png"; renderId: string; data: Uint8Array }
- | { type: "done"; renderId: string }
- | {
- type: "lut";
- renderId: string;
- raw: {
- points: Array<[number, number, number]>;
- size: number;
- type: "1D" | "3D";
- };
- }
- | { type: "error"; renderId?: string; error: string };
-
// ─── Concurrency Limiter ───────────────────────────────────────────────────────
const CPU_COUNT = os.cpus().length;
@@ -376,6 +348,14 @@ const returnStillId = (id: string): void => {
// ─── Helpers ───────────────────────────────────────────────────────────────────
+async function fileSize(file: string): Promise<number | undefined> {
+ try {
+ return (await fs.stat(file)).size;
+ } catch {
+ return undefined;
+ }
+}
+
function toEvenDimension(n: number): number {
const v = Math.round(n);
return v % 2 === 0 ? v : v + 1;
@@ -1071,7 +1051,11 @@ export class HeadlessWebGPURenderer {
* `signal.reason` and the partial output is deleted.
*/
signal?: AbortSignal;
- /** Called after each frame is queued for encoding, then once for audio. */
+ /**
+ * Called as the render moves through its phases: "prepare", each
+ * frame ("render"), "audio" (mixing, per clip), "audio-encode",
+ * "mux", "convert" (GIF only) and "done".
+ */
onProgress?: (progress: RenderProgress) => void;
},
): Promise<{
@@ -1149,8 +1133,28 @@ export class HeadlessWebGPURenderer {
"[HeadlessWebGPURenderer] Acquiring slot for video render (inline)",
);
+ if (renderSemaphore.stats.active >= renderSemaphore.stats.max) {
+ rendererLogger.info(
+ { renderId, ...renderSemaphore.stats },
+ `[HeadlessWebGPURenderer] Waiting for a render slot (${renderSemaphore.stats.active} running, ${renderSemaphore.stats.queued} queued)`,
+ );
+ }
await renderSemaphore.acquire();
+ const reporter = new RenderReporter({
+ renderId,
+ totalFrames,
+ onProgress: options?.onProgress,
+ });
+ const seconds = `${(durationMs / 1000).toFixed(1)}s`;
+ reporter.stage(
+ "prepare",
+ isAudioOnly
+ ? `Preparing ${seconds} of audio (${audioCodec})`
+ : `Preparing ${totalFrames} frames (${seconds}) at ${width}x${height}, ${fps} fps, ${codecOption}`,
+ { width, height, fps, durationMs, codec: codecOption },
+ );
+
let lastCpuUsage = process.cpuUsage();
let lastCpuTime = process.hrtime.bigint();
const getCpuUsage = () => {
@@ -1232,7 +1236,6 @@ export class HeadlessWebGPURenderer {
let outputStarted = false;
let outputFinalized = false;
- const device = await ensureDevice();
// Resources for inline rendering (cleaned up in finally block)
let surface: NodeSurfaceProvider | undefined;
@@ -1240,6 +1243,8 @@ export class HeadlessWebGPURenderer {
let pipeline: ZeroCopyWebCodecsPipeline | undefined;
try {
+ // Inside the try, so a device failure still releases the slot.
+ const device = await ensureDevice();
await output.start();
outputStarted = true;
await preloadFonts(virtualMedia, device);
@@ -1302,15 +1307,10 @@ export class HeadlessWebGPURenderer {
};
await prePass(0);
+ reporter.stage("render", `Rendering ${totalFrames} frames`);
// Render and encode frames using pipelined double-buffered DMA staging ring
for (let i = 0; i < totalFrames; i++) {
options?.signal?.throwIfAborted();
- if (i % 10 === 0) {
- rendererLogger.info(
- { renderId, frame: i, totalFrames },
- `[HeadlessWebGPURenderer] Rendering frame ${i}/${totalFrames}`,
- );
- }
if (ctx && surface && renderer && pipeline) {
// Assets first used on a later frame (a LUT, a glyph) get the
@@ -1363,11 +1363,7 @@ export class HeadlessWebGPURenderer {
);
await pipeline.enqueueFrame(i, targetTexture, encoder);
- options?.onProgress?.({
- phase: "render",
- frame: i + 1,
- totalFrames,
- });
+ if (i + 1 < totalFrames) reporter.frameDone(i + 1);
} else {
throw new Error("Invalid render context state");
}
@@ -1375,6 +1371,8 @@ export class HeadlessWebGPURenderer {
if (pipeline) {
await pipeline.drain();
+ // 100% once the last frames left the staging ring.
+ reporter.frameDone(totalFrames);
const pipelineStats = pipeline.getStats();
if (inspector && qaOptions) {
videoQa = inspector.stats(
@@ -1395,17 +1393,22 @@ export class HeadlessWebGPURenderer {
// ── Audio ──────────────────────────────────────────────────────────────
options?.signal?.throwIfAborted();
- options?.onProgress?.({
- phase: "audio",
- frame: totalFrames,
- totalFrames,
- });
+ reporter.stage("audio", "Mixing audio");
const { channels, sampleRate } = await mixAudioTracks(
virtualMedia,
fps,
48_000,
device,
renderId,
+ {
+ onProgress: ({ done, total, source }) =>
+ reporter.update(
+ total > 0
+ ? `${done}/${total} clips${source ? `: ${source}` : ""}`
+ : "no audio clips",
+ total > 0 ? done / total : undefined,
+ ),
+ },
);
const numChannels = channels.length;
@@ -1416,6 +1419,10 @@ export class HeadlessWebGPURenderer {
const totalSamples = channels[0].length;
const chunkSize = Math.round(0.1 * sampleRate);
+ reporter.stage(
+ "audio-encode",
+ `Encoding ${(totalSamples / sampleRate).toFixed(1)}s of audio (${audioCodec})`,
+ );
for (let offset = 0; offset < totalSamples; offset += chunkSize) {
const currentChunk = Math.min(chunkSize, totalSamples - offset);
@@ -1442,9 +1449,16 @@ export class HeadlessWebGPURenderer {
} finally {
sample.close();
}
+ const written = offset + currentChunk;
+ reporter.update(
+ `${(written / sampleRate).toFixed(1)}s/${(totalSamples / sampleRate).toFixed(1)}s`,
+ written / totalSamples,
+ );
}
options?.signal?.throwIfAborted();
+ // Flushes the encoders and writes the container index.
+ reporter.stage("mux", `Muxing ${extension}`);
await output.finalize();
outputFinalized = true;
@@ -1467,15 +1481,16 @@ export class HeadlessWebGPURenderer {
};
if (codecOption === "gif") {
- rendererLogger.debug(
- "[HeadlessWebGPURenderer] Converting rendered MP4 to high-quality GIF...",
- );
+ reporter.stage("convert", "Converting to GIF (ffmpeg)");
await convertMp4ToGif(tempFilePath, tempGifPath);
+ reporter.finish({ bytes: await fileSize(tempGifPath) });
return { filePath: tempGifPath, cleanup, ...(qa && { qa }) };
}
+ reporter.finish({ bytes: await fileSize(tempFilePath) });
return { filePath: tempFilePath, cleanup, ...(qa && { qa }) };
} catch (error) {
+ reporter.fail(error, options?.signal?.aborted === true);
if (outputStarted && !outputFinalized) {
try {
await output.finalize();
@@ -1489,6 +1504,7 @@ export class HeadlessWebGPURenderer {
} catch {}
throw error;
} finally {
+ reporter.dispose();
if (statsInterval) {
clearInterval(statsInterval);
}
@@ -1529,14 +1545,5 @@ export class HeadlessWebGPURenderer {
}
}
-/** Progress reported by `renderVideo({ onProgress })`. */
-export interface RenderProgress {
- /** "render" while frames are drawn and encoded, "audio" while mixing. */
- phase: "render" | "audio";
- /** Frames finished so far (1-based). */
- frame: number;
- totalFrames: number;
-}
-
export const HeadlessMediaRenderer = HeadlessWebGPURenderer;
export type HeadlessMediaRenderer = HeadlessWebGPURenderer;
diff --git a/packages/renderer/src/index.ts b/packages/renderer/src/index.ts
index a8418280..44c33604 100644
--- a/packages/renderer/src/index.ts
+++ b/packages/renderer/src/index.ts
@@ -5,5 +5,6 @@ export * from "./headless-webgpu-renderer.js";
export * from "./job-reset.js";
export * from "./logger.js";
export * from "./node-manifest.js";
+export type { RenderPhase, RenderProgress } from "./render-reporter.js";
export * from "./video-qa.js";
export * from "./zero-copy-webcodecs-pipeline.js";
diff --git a/packages/renderer/src/render-reporter.ts b/packages/renderer/src/render-reporter.ts
new file mode 100644
index 00000000..cacce7b7
--- /dev/null
+++ b/packages/renderer/src/render-reporter.ts
@@ -0,0 +1,323 @@
+import { rendererLogger } from "./logger.js";
+
+/**
+ * Stages of `renderVideo`, in order. Audio-only renders skip "render";
+ * "convert" runs only for GIF output.
+ */
+export type RenderPhase =
+ | "prepare"
+ | "render"
+ | "audio"
+ | "audio-encode"
+ | "mux"
+ | "convert"
+ | "done";
+
+/** Progress reported by `renderVideo({ onProgress })`. */
+export interface RenderProgress {
+ /**
+ * "prepare" loads fonts and warms the pipeline, "render" draws and encodes
+ * frames, "audio" mixes the soundtrack, "audio-encode" encodes it, "mux"
+ * writes the container, "convert" makes the GIF, "done" is the last event.
+ */
+ phase: RenderPhase;
+ /** Frames finished so far (1-based); `totalFrames` once frames are done. */
+ frame: number;
+ totalFrames: number;
+ /** Time since the render started. */
+ elapsedMs: number;
+ /** Share of the current phase done (0–1), when it can be measured. */
+ phaseProgress?: number;
+ /** Frames per second so far ("render" only). */
+ fps?: number;
+ /** Estimated seconds until the frames are done ("render" only). */
+ etaSec?: number;
+ /** What the phase is working on, e.g. "3/7 sources: music.mp3". */
+ detail?: string;
+}
+
+interface ReporterLog {
+ info(obj: object, msg: string): void;
+ debug(obj: object, msg: string): void;
+ warn(obj: object, msg: string): void;
+ error(obj: object, msg: string): void;
+}
+
+interface RenderReporterOptions {
+ renderId: string;
+ totalFrames: number;
+ onProgress?: (progress: RenderProgress) => void;
+ /** Percent step between frame log lines (default 10). */
+ milestonePercent?: number;
+ log?: ReporterLog;
+ now?: () => number;
+}
+
+const LOG_PREFIX = "[HeadlessWebGPURenderer]";
+/** A milestone reached sooner than this after the last line is not logged. */
+const MIN_LOG_GAP_MS = 1000;
+/**
+ * Quiet time before the first "still …" line of a phase. Each one doubles it
+ * up to the cap, so a long phase adds a few lines rather than one a minute.
+ */
+const HEARTBEAT_FIRST_MS = 10_000;
+const HEARTBEAT_MAX_MS = 5 * 60_000;
+
+const ACTIVITY: Record<RenderPhase, string> = {
+ prepare: "preparing",
+ render: "rendering",
+ audio: "mixing audio",
+ "audio-encode": "encoding audio",
+ mux: "muxing",
+ convert: "converting to GIF",
+ done: "finishing",
+};
+
+/**
+ * Reports a render's progress to the log and to `onProgress`.
+ *
+ * The log stays short whatever the length of the film, so it fits an agent's
+ * context: one line per phase, frame lines only at percent milestones, and a
+ * "still …" line when a phase goes quiet, backing off from 10 s to 5 min.
+ * `onProgress` gets every update; the "still …" lines are log-only, so a stall
+ * detector fed by `onProgress` still sees a hang.
+ */
+export class RenderReporter {
+ private readonly renderId: string;
+ private readonly totalFrames: number;
+ private readonly onProgress?: (progress: RenderProgress) => void;
+ private readonly milestonePercent: number;
+ private readonly log: ReporterLog;
+ private readonly now: () => number;
+ private readonly startedAt: number;
+ private readonly phaseMs = new Map<RenderPhase, number>();
+
+ private phase: RenderPhase = "prepare";
+ private phaseStartedAt: number;
+ private frame = 0;
+ private phaseProgress?: number;
+ private detail?: string;
+ private lastMilestone = 0;
+ /** Last phase or milestone line; "still …" lines don't count. */
+ private lastLogAt: number;
+ private heartbeatMs = HEARTBEAT_FIRST_MS;
+ private heartbeat?: ReturnType<typeof setTimeout>;
+
+ constructor(options: RenderReporterOptions) {
+ this.renderId = options.renderId;
+ this.totalFrames = options.totalFrames;
+ this.onProgress = options.onProgress;
+ this.milestonePercent = options.milestonePercent ?? 10;
+ this.log = options.log ?? rendererLogger;
+ this.now = options.now ?? (() => performance.now());
+ this.startedAt = this.now();
+ this.phaseStartedAt = this.startedAt;
+ this.lastLogAt = this.startedAt;
+ }
+
+ /** Starts a phase: logs `message` and emits a progress event. */
+ stage(phase: RenderPhase, message: string, fields: object = {}): void {
+ this.closePhase();
+ this.phase = phase;
+ this.phaseProgress = undefined;
+ this.detail = undefined;
+ this.heartbeatMs = HEARTBEAT_FIRST_MS;
+ if (phase !== "render" && phase !== "prepare")
+ this.frame = this.totalFrames;
+ this.info(message, fields);
+ this.emit();
+ }
+
+ /** Records a finished frame; logs at each milestone. */
+ frameDone(frame: number): void {
+ this.frame = frame;
+ const percent = (frame / this.totalFrames) * 100;
+ const milestone =
+ frame >= this.totalFrames
+ ? 100
+ : Math.floor(percent / this.milestonePercent) * this.milestonePercent;
+ const { fps, etaSec } = this.rate();
+ if (milestone > this.lastMilestone) {
+ this.lastMilestone = milestone;
+ if (milestone === 100 || this.now() - this.lastLogAt >= MIN_LOG_GAP_MS) {
+ const tail =
+ milestone === 100
+ ? `in ${formatDuration(this.now() - this.phaseStartedAt)}`
+ : `eta ${formatDuration(etaSec * 1000)}`;
+ this.info(
+ `Rendered ${milestone}% (${frame}/${this.totalFrames} frames) · ${fps.toFixed(1)} fps · ${tail}`,
+ { fps: round1(fps), etaSec },
+ );
+ }
+ }
+ this.emit({ fps: round1(fps), etaSec });
+ }
+
+ /** Updates what the current phase is doing; shown in "still …" lines. */
+ update(detail: string | undefined, phaseProgress?: number): void {
+ this.detail = detail;
+ this.phaseProgress = phaseProgress;
+ this.emit();
+ }
+
+ /** Logs the summary and emits the "done" event. */
+ finish(fields: { bytes?: number } = {}): void {
+ this.closePhase();
+ this.phase = "done";
+ this.frame = this.totalFrames;
+ this.stopHeartbeat();
+ const total = this.now() - this.startedAt;
+ const parts = [...this.phaseMs].map(
+ ([phase, ms]) => `${phase} ${formatDuration(ms)}`,
+ );
+ const size =
+ fields.bytes !== undefined ? ` (${formatBytes(fields.bytes)})` : "";
+ this.log.info(
+ {
+ renderId: this.renderId,
+ phase: "done",
+ elapsedMs: Math.round(total),
+ phaseMs: Object.fromEntries(
+ [...this.phaseMs].map(([phase, ms]) => [phase, Math.round(ms)]),
+ ),
+ ...fields,
+ },
+ `${LOG_PREFIX} Done in ${formatDuration(total)}${size}: ${parts.join(" · ")}`,
+ );
+ this.emit();
+ }
+
+ /** Logs which phase the render stopped in. */
+ fail(error: unknown, aborted: boolean): void {
+ this.stopHeartbeat();
+ const elapsed = formatDuration(this.now() - this.startedAt);
+ const fields = {
+ renderId: this.renderId,
+ phase: this.phase,
+ frame: this.frame,
+ totalFrames: this.totalFrames,
+ elapsedMs: Math.round(this.now() - this.startedAt),
+ };
+ if (aborted) {
+ this.log.warn(
+ fields,
+ `${LOG_PREFIX} Aborted while ${ACTIVITY[this.phase]} after ${elapsed}`,
+ );
+ } else {
+ this.log.error(
+ { ...fields, err: error },
+ `${LOG_PREFIX} Failed while ${ACTIVITY[this.phase]} after ${elapsed}`,
+ );
+ }
+ }
+
+ /** Stops the "still …" timer. Safe to call more than once. */
+ dispose(): void {
+ this.stopHeartbeat();
+ }
+
+ private closePhase(): void {
+ const now = this.now();
+ this.phaseMs.set(
+ this.phase,
+ (this.phaseMs.get(this.phase) ?? 0) + (now - this.phaseStartedAt),
+ );
+ this.phaseStartedAt = now;
+ }
+
+ private rate(): { fps: number; etaSec: number } {
+ const seconds = Math.max(0.001, (this.now() - this.phaseStartedAt) / 1000);
+ const fps = this.frame / seconds;
+ const etaSec = Math.round(
+ (this.totalFrames - this.frame) / Math.max(fps, 0.001),
+ );
+ return { fps, etaSec };
+ }
+
+ private info(message: string, fields: object = {}): void {
+ this.log.info(
+ {
+ renderId: this.renderId,
+ phase: this.phase,
+ frame: this.frame,
+ totalFrames: this.totalFrames,
+ ...fields,
+ },
+ `${LOG_PREFIX} ${message}`,
+ );
+ this.lastLogAt = this.now();
+ this.armHeartbeat();
+ }
+
+ private emit(extra: Partial<RenderProgress> = {}): void {
+ this.onProgress?.({
+ phase: this.phase,
+ frame: this.frame,
+ totalFrames: this.totalFrames,
+ elapsedMs: Math.round(this.now() - this.startedAt),
+ ...(this.phaseProgress !== undefined && {
+ phaseProgress: this.phaseProgress,
+ }),
+ ...(this.detail !== undefined && { detail: this.detail }),
+ ...extra,
+ });
+ }
+
+ private armHeartbeat(): void {
+ this.stopHeartbeat();
+ this.heartbeat = setTimeout(() => this.beat(), this.heartbeatMs);
+ // Never keeps the process alive on its own.
+ this.heartbeat.unref?.();
+ }
+
+ private beat(): void {
+ const inPhase = formatDuration(this.now() - this.phaseStartedAt);
+ let what = "";
+ if (this.phase === "render") {
+ what = ` (frame ${this.frame}/${this.totalFrames})`;
+ } else if (this.detail || this.phaseProgress !== undefined) {
+ const pct =
+ this.phaseProgress !== undefined
+ ? `${Math.floor(this.phaseProgress * 100)}%`
+ : "";
+ what = ` (${[pct, this.detail].filter(Boolean).join(", ")})`;
+ }
+ this.log.info(
+ {
+ renderId: this.renderId,
+ phase: this.phase,
+ frame: this.frame,
+ totalFrames: this.totalFrames,
+ },
+ `${LOG_PREFIX} Still ${ACTIVITY[this.phase]}${what}, ${inPhase} in this phase`,
+ );
+ this.heartbeatMs = Math.min(this.heartbeatMs * 2, HEARTBEAT_MAX_MS);
+ this.armHeartbeat();
+ }
+
+ private stopHeartbeat(): void {
+ if (this.heartbeat) clearTimeout(this.heartbeat);
+ this.heartbeat = undefined;
+ }
+}
+
+/** "4.2s", "2m 05s", "1h 02m". */
+export function formatDuration(ms: number): string {
+ const s = Math.max(0, ms) / 1000;
+ if (s < 60) return `${s.toFixed(1)}s`;
+ const total = Math.round(s);
+ const h = Math.floor(total / 3600);
+ const m = Math.floor((total % 3600) / 60);
+ const sec = total % 60;
+ if (h > 0) return `${h}h ${String(m).padStart(2, "0")}m`;
+ return `${m}m ${String(sec).padStart(2, "0")}s`;
+}
+
+function formatBytes(bytes: number): string {
+ if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
+ return `${(bytes / 1024 / 1024).toFixed(1)} MB`;
+}
+
+function round1(n: number): number {
+ return Math.round(n * 10) / 10;
+}
diff --git a/packages/renderer/src/tests/render-reporter.test.ts b/packages/renderer/src/tests/render-reporter.test.ts
new file mode 100644
index 00000000..f4df720e
--- /dev/null
+++ b/packages/renderer/src/tests/render-reporter.test.ts
@@ -0,0 +1,147 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import {
+ formatDuration,
+ type RenderProgress,
+ RenderReporter,
+} from "../render-reporter.js";
+
+function setup(totalFrames: number) {
+ const lines: string[] = [];
+ const events: RenderProgress[] = [];
+ const push = (_obj: object, msg: string) => {
+ lines.push(msg.replace("[HeadlessWebGPURenderer] ", ""));
+ };
+ const reporter = new RenderReporter({
+ renderId: "vid-test",
+ totalFrames,
+ onProgress: (p) => events.push(p),
+ log: { info: push, debug: () => {}, warn: push, error: push },
+ now: () => Date.now(),
+ });
+ return { reporter, lines, events };
+}
+
+describe("RenderReporter", () => {
+ beforeEach(() => {
+ vi.useFakeTimers();
+ });
+ afterEach(() => {
+ vi.useRealTimers();
+ });
+
+ it("logs a bounded number of frame lines whatever the length", () => {
+ for (const totalFrames of [300, 30_000]) {
+ const { reporter, lines, events } = setup(totalFrames);
+ reporter.stage("render", "Rendering");
+ for (let f = 1; f <= totalFrames; f++) {
+ vi.advanceTimersByTime(100);
+ reporter.frameDone(f);
+ }
+ reporter.dispose();
+ const frameLines = lines.filter((l) => l.startsWith("Rendered "));
+ expect(frameLines).toHaveLength(10);
+ expect(frameLines[0]).toMatch(
+ new RegExp(
+ `^Rendered 10% \\(${totalFrames / 10}/${totalFrames} frames\\) · 10\\.0 fps · eta `,
+ ),
+ );
+ expect(frameLines.at(-1)).toMatch(/^Rendered 100% .* in /);
+ // onProgress still sees every frame.
+ expect(events.filter((e) => e.phase === "render")).toHaveLength(
+ totalFrames + 1,
+ );
+ }
+ });
+
+ it("skips milestones reached within a second, but always logs 100%", () => {
+ const { reporter, lines } = setup(30);
+ reporter.stage("render", "Rendering");
+ for (let f = 1; f <= 30; f++) {
+ vi.advanceTimersByTime(10);
+ reporter.frameDone(f);
+ }
+ reporter.dispose();
+ expect(lines).toEqual([
+ "Rendering",
+ expect.stringMatching(/^Rendered 100% \(30\/30 frames\)/),
+ ]);
+ });
+
+ it("logs 'still …' lines with backoff, without emitting progress", () => {
+ const { reporter, lines, events } = setup(100);
+ reporter.stage("mux", "Muxing mp4");
+ const before = events.length;
+ vi.advanceTimersByTime(20 * 60_000);
+ reporter.dispose();
+ const still = lines.filter((l) => l.startsWith("Still "));
+ // 10 s, 30 s, 70 s, 150 s, 310 s, then every 5 min: 610 s, 910 s.
+ expect(still).toHaveLength(7);
+ expect(still[0]).toBe("Still muxing, 10.0s in this phase");
+ expect(still[1]).toBe("Still muxing, 30.0s in this phase");
+ expect(events.length).toBe(before);
+ });
+
+ it("restarts the backoff on a new phase and shows the phase's detail", () => {
+ const { reporter, lines } = setup(100);
+ reporter.stage("mux", "Muxing mp4");
+ vi.advanceTimersByTime(40_000);
+ reporter.stage("audio", "Mixing audio");
+ reporter.update("3/7 clips: music.mp3", 3 / 7);
+ vi.advanceTimersByTime(10_000);
+ reporter.dispose();
+ expect(lines.at(-1)).toBe(
+ "Still mixing audio (42%, 3/7 clips: music.mp3), 10.0s in this phase",
+ );
+ });
+
+ it("summarizes phase durations and reports done", () => {
+ const { reporter, lines, events } = setup(60);
+ reporter.stage("prepare", "Preparing");
+ vi.advanceTimersByTime(1_000);
+ reporter.stage("render", "Rendering");
+ vi.advanceTimersByTime(4_000);
+ reporter.frameDone(60);
+ reporter.stage("audio", "Mixing audio");
+ vi.advanceTimersByTime(500);
+ reporter.stage("mux", "Muxing mp4");
+ vi.advanceTimersByTime(250);
+ reporter.finish({ bytes: 3 * 1024 * 1024 });
+ vi.advanceTimersByTime(60_000);
+ expect(lines.at(-1)).toBe(
+ "Done in 5.8s (3.0 MB): prepare 1.0s · render 4.0s · audio 0.5s · mux 0.3s",
+ );
+ expect(events.at(-1)).toMatchObject({
+ phase: "done",
+ frame: 60,
+ totalFrames: 60,
+ elapsedMs: 5750,
+ });
+ expect(events.map((e) => e.phase)).toEqual([
+ "prepare",
+ "render",
+ "render",
+ "audio",
+ "mux",
+ "done",
+ ]);
+ });
+
+ it("names the phase a render failed or was aborted in", () => {
+ const { reporter, lines } = setup(10);
+ reporter.stage("audio", "Mixing audio");
+ reporter.fail(new Error("boom"), false);
+ reporter.fail(new Error("stop"), true);
+ expect(lines.slice(-2)).toEqual([
+ "Failed while mixing audio after 0.0s",
+ "Aborted while mixing audio after 0.0s",
+ ]);
+ });
+});
+
+describe("formatDuration", () => {
+ it("formats seconds, minutes and hours", () => {
+ expect(formatDuration(4_240)).toBe("4.2s");
+ expect(formatDuration(125_000)).toBe("2m 05s");
+ expect(formatDuration(3_720_000)).toBe("1h 02m");
+ });
+});
diff --git a/packages/renderer/src/video-worker.ts b/packages/renderer/src/video-worker.ts
deleted file mode 100644
index 823511a0..00000000
--- a/packages/renderer/src/video-worker.ts
+++ /dev/null
@@ -1,436 +0,0 @@
-import { parentPort } from "node:worker_threads";
-import {
- compositionStateStore,
- drawCompositionTree,
-} from "@framefields/compositions";
-import type { VirtualMediaData } from "@framefields/core";
-import {
- clearAllVideoCache,
- createHeadlessDOMElement,
- ensureDevice,
- initHeadlessWebGPU,
- installIntersectionObserverStub,
- lutStore,
- mediaDecoderCache,
- NodeSurfaceProvider,
- patchCanvasPrototypeForDOM,
- type RenderContextValue,
- Renderer2D,
- resetDeviceInstance,
- shaderStore,
- textureCache,
-} from "@framefields/webgpu-renderers";
-import sharp from "sharp";
-import { Canvas as SkiaCanvas, Image as SkiaImage } from "skia-canvas";
-import { preloadFonts } from "./asset-preloader.js";
-import { DmaStagingRing } from "./dma-staging-ring.js";
-import { discoverAndRegisterNodeRenderers } from "./dynamic-node-discovery.js";
-import { rendererLogger } from "./logger.js";
-
-const globalObj = globalThis as unknown as Record<string, unknown>;
-
-if (typeof globalThis !== "undefined") {
- globalObj.__IS_HEADLESS_RENDERER__ = true;
- globalObj.__GATEWAI_DELAYS__ ??= new Set<number>();
-}
-
-import { bootstrapMediabunny } from "./bootstrap-mediabunny.js";
-
-bootstrapMediabunny();
-
-if (typeof globalThis.requestAnimationFrame === "undefined") {
- globalObj.requestAnimationFrame = (cb: () => void) => setTimeout(cb, 0);
- globalObj.cancelAnimationFrame = (id: number) => clearTimeout(id);
-}
-
-const dummyDomElement = createHeadlessDOMElement();
-
-if (typeof globalThis.OffscreenCanvas === "undefined") {
- globalObj.OffscreenCanvas = SkiaCanvas;
-}
-
-if (typeof globalThis.HTMLCanvasElement === "undefined") {
- globalObj.HTMLCanvasElement = SkiaCanvas;
-}
-
-if (typeof globalThis.Image === "undefined") {
- globalObj.Image = SkiaImage;
-}
-
-if (typeof globalThis.window === "undefined") {
- globalObj.window = globalThis;
-}
-
-installIntersectionObserverStub();
-
-if (typeof globalThis.document === "undefined") {
- globalObj.document = {
- createElement: (tag: string) => {
- if (tag === "canvas") return new SkiaCanvas(1, 1);
- return { ...dummyDomElement };
- },
- getElementsByTagName: () => [],
- documentElement: dummyDomElement,
- body: dummyDomElement,
- };
-} else {
- const docObj = globalThis.document as unknown as Record<string, unknown>;
- if (typeof docObj.getElementsByTagName === "undefined") {
- docObj.getElementsByTagName = () => [];
- }
- docObj.documentElement ??= dummyDomElement;
- docObj.body ??= dummyDomElement;
-}
-
-const skiaProto = SkiaCanvas.prototype as unknown as Record<string, unknown>;
-patchCanvasPrototypeForDOM(skiaProto, dummyDomElement);
-if (typeof skiaProto.getAttribute === "undefined") {
- skiaProto.getAttribute = () => null;
-}
-
-export interface WorkerInput {
- virtualMedia: VirtualMediaData;
- width: number;
- height: number;
- fps: number;
- startFrame: number;
- endFrame: number;
- renderId: string;
- outputFormat: "png" | "pixels";
- workerIndex: number;
-}
-
-export type WorkerMessage =
- | { type: "frame"; renderId: string; frameIndex: number; pixels: Uint8Array }
- | { type: "png"; renderId: string; data: Uint8Array }
- | { type: "done"; renderId: string }
- | {
- type: "lut";
- renderId: string;
- raw: {
- points: Array<[number, number, number]>;
- size: number;
- type: "1D" | "3D";
- };
- }
- | { type: "error"; renderId?: string; error: string };
-
-let isInitialized = false;
-let initPromise: Promise<void> | null = null;
-
-function initializeWorker(): Promise<void> {
- if (isInitialized) return Promise.resolve();
- if (initPromise) return initPromise;
-
- initPromise = (async () => {
- try {
- rendererLogger.debug(
- "[video-worker] Initializing headless WebGPU in worker thread…",
- );
- await initHeadlessWebGPU();
- await ensureDevice();
- rendererLogger.debug(
- "[video-worker] Scanning node renderers in worker thread…",
- );
- await discoverAndRegisterNodeRenderers();
- isInitialized = true;
- initPromise = null;
- rendererLogger.debug(
- "[video-worker] Worker thread initialization complete.",
- );
- } catch (error) {
- initPromise = null;
- throw error;
- }
- })();
-
- return initPromise;
-}
-
-if (parentPort) {
- if (process.env.GATEWAI_UNIT_TEST === "true") {
- globalThis.fetch = async () => {
- throw new Error("Network requests are disabled in unit tests");
- };
- }
-
- parentPort.on("message", (task: any) => {
- if (!task || task.type !== "start") return;
-
- void (async () => {
- try {
- const {
- virtualMedia,
- width,
- height,
- fps,
- startFrame,
- endFrame,
- renderId,
- outputFormat,
- workerIndex,
- } = task.input;
-
- rendererLogger.debug(
- { renderId, workerIndex },
- `[video-worker] Worker ${workerIndex} initializing renderer`,
- );
- await initializeWorker();
- const device = await ensureDevice();
- await preloadFonts(virtualMedia, device);
- const surface = new NodeSurfaceProvider(device, width, height);
- const stagingRing = new DmaStagingRing(device, {
- width,
- height,
- capacity: 2,
- });
- const renderer = new Renderer2D(device, surface.colorFormat);
- const ctx: RenderContextValue = { device, renderer, surface };
- rendererLogger.debug(
- { renderId, workerIndex },
- `[video-worker] Worker ${workerIndex} setup complete, starting render loop`,
- );
-
- let lutSub: (() => void) | undefined;
- let onTaskMessage: ((msg: any) => void) | undefined;
- const ackPromises = new Map<number, () => void>();
- let lastAckedFrame = startFrame - 1;
-
- try {
- onTaskMessage = (msg: any) => {
- if (msg && msg.type === "ack") {
- const acked = msg.frameIndex as number;
- if (acked > lastAckedFrame) {
- lastAckedFrame = acked;
- }
- const resolve = ackPromises.get(acked);
- if (resolve) {
- resolve();
- ackPromises.delete(acked);
- }
- }
- };
- parentPort!.on("message", onTaskMessage);
-
- lutSub = lutStore.onChange((key) => {
- const raw = lutStore.getRawData(key);
- if (raw) {
- parentPort!.postMessage({
- type: "lut",
- renderId,
- raw,
- } satisfies WorkerMessage);
- }
- });
-
- // Pre-pass: run drawCompositionTree once on the startFrame to trigger and await all LUT extractions/loads
- {
- rendererLogger.debug(
- { renderId, workerIndex },
- `[video-worker] Pre-pass rendering startFrame ${startFrame} to trigger/await all LUTs`,
- );
- const preEncoder = device.createCommandEncoder();
- const dummyTex = renderer.getTemporaryTexture(width, height);
- const dummyView = dummyTex.createView();
-
- const preClear = renderer.beginFrame(
- preEncoder,
- dummyView,
- { r: 0, g: 0, b: 0, a: 0 },
- width,
- height,
- "clear",
- );
- preClear.end();
-
- await drawCompositionTree(
- ctx,
- preEncoder,
- dummyView,
- dummyTex,
- width,
- height,
- virtualMedia,
- {
- frame: startFrame,
- compositionFrame: startFrame,
- fps,
- isHeadless: true,
- renderId,
- virtualMedia,
- containerWidth: width,
- containerHeight: height,
- isVideoMode: virtualMedia.operation?.dataType === "Video",
- },
- );
-
- device.queue.submit([preEncoder.finish()]);
- await lutStore.awaitAllPending(device);
- }
-
- // Compute backpressure limit based on frame size to prevent
- // native heap corruption from excessive pending pixel buffers.
- // Target: ~120 MB max pending per worker.
- const frameSizeBytes = width * height * 4;
- const maxPendingBytes = 120 * 1024 * 1024;
- const maxUnacked = Math.max(
- 2,
- Math.min(5, Math.floor(maxPendingBytes / frameSizeBytes)),
- );
-
- const inFlightWorkerFrames: number[] = [];
- const consumeWorkerFrame = async (frameIdx: number) => {
- const pixelsArr = await stagingRing.readAndReleaseFrame(frameIdx);
- if (outputFormat === "png") {
- const pngBuffer = await sharp(Buffer.from(pixelsArr), {
- raw: { width, height, channels: 4 },
- })
- .png()
- .toBuffer();
- parentPort!.postMessage({
- type: "png",
- renderId,
- data: new Uint8Array(pngBuffer),
- } satisfies WorkerMessage);
- } else {
- parentPort!.postMessage(
- {
- type: "frame",
- renderId,
- frameIndex: frameIdx,
- pixels: pixelsArr,
- } satisfies WorkerMessage,
- [pixelsArr.buffer as ArrayBuffer],
- );
- }
- };
-
- for (let i = startFrame; i <= endFrame; i++) {
- // Implement backpressure: do not render too far ahead of encoding
- while (i - lastAckedFrame > maxUnacked) {
- await new Promise<void>((resolve) => {
- ackPromises.set(i - maxUnacked, resolve);
- });
- }
-
- if (inFlightWorkerFrames.length >= stagingRing.capacity) {
- const prevFrame = inFlightWorkerFrames.shift();
- if (prevFrame !== undefined) {
- await consumeWorkerFrame(prevFrame);
- }
- }
-
- if (i % 10 === 0) {
- rendererLogger.info(
- { renderId, workerIndex, frame: i },
- `[video-worker] Rendering frame ${i} (Worker ${workerIndex})`,
- );
- }
- compositionStateStore.setState(renderId, i, fps, true);
- mediaDecoderCache.tick();
-
- const encoder = device.createCommandEncoder();
- const targetView = surface.getCurrentTextureView();
- const targetTexture = surface.getCurrentTexture();
-
- const clearPass = renderer.beginFrame(
- encoder,
- targetView,
- { r: 0, g: 0, b: 0, a: 0 },
- width,
- height,
- "clear",
- );
- clearPass.end();
-
- await drawCompositionTree(
- ctx,
- encoder,
- targetView,
- targetTexture,
- width,
- height,
- virtualMedia,
- {
- frame: i,
- compositionFrame: i,
- fps,
- isHeadless: true,
- renderId,
- virtualMedia,
- containerWidth: width,
- containerHeight: height,
- isVideoMode: virtualMedia.operation?.dataType === "Video",
- },
- );
-
- await stagingRing.prepareSlot(i);
- stagingRing.stageTexture(encoder, targetTexture, i);
- device.queue.submit([encoder.finish()]);
- stagingRing.scheduleMap(i);
- inFlightWorkerFrames.push(i);
- }
-
- while (inFlightWorkerFrames.length > 0) {
- const remainingFrame = inFlightWorkerFrames.shift();
- if (remainingFrame !== undefined) {
- await consumeWorkerFrame(remainingFrame);
- }
- }
-
- if (outputFormat !== "png") {
- parentPort!.postMessage({
- type: "done",
- renderId,
- } satisfies WorkerMessage);
- }
- } finally {
- if (onTaskMessage) {
- try {
- parentPort!.off("message", onTaskMessage);
- } catch {}
- }
- if (lutSub) {
- try {
- lutSub();
- } catch {}
- }
- try {
- mediaDecoderCache.destroy();
- clearAllVideoCache();
- } catch {}
- try {
- shaderStore.clear(renderId);
- } catch {}
- try {
- device.destroy();
- } catch (e) {}
- try {
- resetDeviceInstance();
- } catch (e) {}
- try {
- renderer.destroy();
- } catch (e) {}
- try {
- surface.destroy();
- } catch (e) {}
- try {
- stagingRing.destroy();
- } catch (e) {}
- try {
- textureCache.destroy();
- } catch (e) {}
- }
- } catch (error) {
- parentPort!.postMessage({
- type: "error",
- renderId: task?.input?.renderId,
- error:
- error instanceof Error
- ? (error.stack ?? error.message)
- : String(error),
- } satisfies WorkerMessage);
- }
- })();
- });
-}
diff --git a/packages/renderer/tsdown.config.ts b/packages/renderer/tsdown.config.ts
index 23157ae1..639f19a2 100644
--- a/packages/renderer/tsdown.config.ts
+++ b/packages/renderer/tsdown.config.ts
@@ -1,12 +1,7 @@
import { defineConfig } from "tsdown";
export default defineConfig({
- entry: [
- "src/index.ts",
- "src/test-runner.ts",
- "src/offscreen-canvas.ts",
- "src/video-worker.ts",
- ],
+ entry: ["src/index.ts", "src/test-runner.ts", "src/offscreen-canvas.ts"],
format: ["esm"],
dts: true,
clean: true,
diff --git a/plugins/framefields/skills/framefields-render/SKILL.md b/plugins/framefields/skills/framefields-render/SKILL.md
index 82dc08b2..04e84993 100644
--- a/plugins/framefields/skills/framefields-render/SKILL.md
+++ b/plugins/framefields/skills/framefields-render/SKILL.md
@@ -233,6 +233,29 @@ tsx src/render.ts preview
tsx src/render.ts
```
+### Reading render progress
+
+`renderVideo` logs its own progress to stdout, one JSON line per event (the text is in `msg`). The log stays short however long the film is, about 15 lines for an export:
+
+```text
+Preparing 1800 frames (60.0s) at 1920x1080, 30 fps, h264
+Rendering 1800 frames
+Rendered 10% (180/1800 frames) · 41.2 fps · eta 39.3s
+…
+Rendered 100% (1800/1800 frames) · 41.8 fps · in 43.1s
+Mixing audio
+Encoding 60.0s of audio (aac)
+Muxing mp4
+Done in 51.4s (14.2 MB): prepare 1.2s · render 43.1s · audio 3.2s · audio-encode 1.1s · mux 0.8s
+```
+
+- **Work continues after the last frame.** "Mixing audio" decodes every audio and video clip in the film, then the soundtrack is encoded and "Muxing" writes the file. With many or long clips this takes a while. It is not a hang; wait for `Done in …`.
+- **A quiet phase prints `Still …` lines**, e.g. `Still mixing audio (42%, 3/7 clips: music.mp3), 1m 10s in this phase`: after 10 s, 30 s, 70 s, then backing off to one every 5 min. Each says which phase is running and for how long. A render is stuck only when no line arrives for well over the last gap. Killing it at any point leaves no video.
+- **A failure names its phase:** `Failed while mixing audio after 12.3s`, with the error.
+- **Run long exports in the background** with the longest timeout your harness allows, and read the output when it finishes rather than polling it.
+- `LOG_LEVEL=warn` silences these lines; `LOG_LEVEL=debug` adds per-stage detail.
+- For your own progress output, `onProgress` gets every event as `{ phase, frame, totalFrames, elapsedMs, phaseProgress?, fps?, etaSec?, detail? }`. `phase` runs `prepare`, `render` (audio-only output skips it), `audio`, `audio-encode`, `mux`, `convert` (GIF only), `done`.
+
---
## Built-in Video QA (`renderVideo({ qa })`)
@@ -302,7 +325,8 @@ ffmpeg -i output/final-video.mp4 -af "volumedetect" -f null -
| Audio cuts out early | Missing `durationFrames` on audio layer | Explicitly assign `durationFrames: totalFrames` on `Layer.audio()`. |
| Vision cutout / mask is empty on a single `renderFrame` | Vision reads the previous rendered frame; frame 0 has none | Render two or more consecutive frames with the same `HeadlessMediaRenderer` and keep the last. `renderVideo` handles this automatically. |
| Ghosted double of the subject in a vision cutout | Frames rendered out of order (frame grid, or jumping from frame 3 to 48), so the cutout uses pixels from another frame | Verify vision layers with sequential frames or the exported MP4, not `renderFrameGrid`. |
-| Script hangs after rendering | Background worker thread pool held open | Always include `process.exit(0)` at the end of render scripts. |
+| Output stops after `Rendered 100%` | Audio is being mixed, encoded or muxed | Not a hang: wait for `Still …` lines and `Done in …` (see **Reading render progress**). |
+| Script hangs after `Done in …` | Background worker thread pool held open | Always include `process.exit(0)` at the end of render scripts. |
| Render very slow on a Linux server | Frames drawn on a CPU adapter (lavapipe) | `await describeGpu()` shows `software: true`; fix the GPU's Vulkan driver (`vulkaninfo --summary`). |
| Encoder fails to open (`h264_rkmpp`, NVENC) | framefields < 2.2 uses the hardware encoder untested | Use framefields 2.2+, which falls back to libx264 etc., or set `FRAMEFIELDS_HW_CODECS=off`. |
| Render worker job fails | Project not render-ready (lockfile, export, LFS) | Read the job's `code` and `hint`; checklist at https://framefields.dev/docs/guides/render-service/ |
diff --git a/plugins/framefields/skills/framefields/SKILL.md b/plugins/framefields/skills/framefields/SKILL.md
index dfbf7b5e..c7a1654e 100644
--- a/plugins/framefields/skills/framefields/SKILL.md
+++ b/plugins/framefields/skills/framefields/SKILL.md
@@ -255,7 +255,7 @@ Never go straight to a full video render. On a machine without a GPU every rende
2. `npm run frames -- 0 30 60` and look at the PNGs in `output/`. A black frame usually means a missing font, a layer outside the canvas, or opacity stuck at 0.
3. `npm run grid` and check the motion reads smoothly across the contact sheet.
4. `npm run preview` when a person is watching: they play the film with sound before you export (see below).
-5. `npm run render` once frames and motion look right.
+5. `npm run render` once frames and motion look right. Run it in the background with a long timeout. It logs each phase (`Rendered 40% …`, `Mixing audio`, `Muxing mp4`, `Done in …`) and a `Still …` line when one runs long, so a pause after the last frame is the audio mix or mux, not a hang.
Show the user the frame PNGs or the grid before the full render. `framefields-render` covers pixel-probe tests, render QA (`renderVideo({ qa })`), and diagnosing blank frames. If you suspect an issue or defect in the engine itself or ai gateway, see [section 9](#9-report-engine-issues).