Code-first video & image, rendered natively on WebGPU.
Log render phases compactly so agents can follow long exports
Claude4f5339eee88f
diff --git a/.changeset/render-progress-phases.md b/.changeset/render-progress-phases.mdnew file mode 100644index 00000000..8aa6c40c--- /dev/null+++ b/.changeset/render-progress-phases.md@@ -0,0 +1,9 @@+---+"framefields": minor+---++Clearer render progress, sized for agents' context:+- `renderVideo` logs each phase (prepare, frames, audio mix, audio encode, mux, GIF conversion) and a summary with per-phase timings. Frames log at every 10% with fps and ETA instead of every 10 frames, so an export prints about 15 lines whatever its length.+- A phase that runs long logs `Still …` lines (after 10 s, then backing off to every 5 min) naming the phase and its progress, so the audio mix or the mux after the last frame no longer looks like a hang. A failed render names the phase it stopped in.+- `onProgress` reports the new phases `prepare`, `audio-encode`, `mux`, `convert` and `done` alongside `render` and `audio`, with `elapsedMs`, and `fps`, `etaSec`, `phaseProgress` and `detail` where they apply. The audio mix reports each clip it mixes (`mixAudioTracks` takes an `onProgress` option), which also keeps the render worker's stall detector from firing on long mixes.+- Removes the unused `video-worker` entry from `@framefields/renderer`.diff --git a/apps/renderer-service/src/engine.gpu.test.ts b/apps/renderer-service/src/engine.gpu.test.tsindex fd4dc128..f7c08197 100644--- a/apps/renderer-service/src/engine.gpu.test.ts+++ b/apps/renderer-service/src/engine.gpu.test.ts@@ -72,15 +72,24 @@ gpu("engine hooks", () => {backgroundColor: "#204060",});- it("reports progress for every frame, then the audio mix", async () => {+ it("reports progress for every frame, then each phase after them", async () => {const seen: string[] = [];const result = await film().renderVideo({onProgress: (p) => seen.push(`${p.phase}:${p.frame}/${p.totalFrames}`),});await result.cleanup();- expect(seen.filter((s) => s.startsWith("render:"))).toHaveLength(48);- expect(seen.at(-2)).toBe("render:48/48");- expect(seen.at(-1)).toBe("audio:48/48");+ expect(seen.slice(0, 3)).toEqual([+ "prepare:0/48",+ "render:0/48",+ "render:1/48",+ ]);+ // The phase start, then one event per frame.+ expect(seen.filter((s) => s.startsWith("render:"))).toHaveLength(49);+ const last = seen.indexOf("render:48/48");+ expect(seen[last + 1]).toBe("audio:48/48");+ expect(seen).toContain("audio-encode:48/48");+ expect(seen).toContain("mux:48/48");+ expect(seen.at(-1)).toBe("done:48/48");}, 120_000);it("aborts between frames with the signal's reason", async () => {diff --git a/apps/renderer-service/src/render/render-step.ts b/apps/renderer-service/src/render/render-step.tsindex 00d5900c..4769f42a 100644--- a/apps/renderer-service/src/render/render-step.ts+++ b/apps/renderer-service/src/render/render-step.ts@@ -8,6 +8,7 @@ import type {StepContext,} from "../jobs/pipeline.js";import type { JobInput, RenderSettings } from "../jobs/schema.js";+import type { JobProgress } from "../jobs/store.js";import { rendererLogger } from "../logger.js";import { gpuStatus } from "../worker/gpu.js";import { workerHealth } from "../worker/health.js";@@ -189,6 +190,7 @@ async function renderOutput(const { ext, contentType } = containerFor(codec);const started = Date.now();let lastEmit = 0;+ let lastPhase: string | undefined;let expectedFrames: number | undefined;const result = await output.target.renderVideo({outputPath: `${base}.${ext}`,@@ -198,30 +200,24 @@ async function renderOutput(qa: settings.qa,renderer,signal: controller.signal,- onProgress: (p: {- phase: string;- frame: number;- totalFrames: number;- }) => {+ onProgress: (p: EngineProgress) => {stall.touch();expectedFrames = p.totalFrames;+ // The job reports its own "done" after the upload.+ if (p.phase === "done") return;const now = Date.now();+ const lastFrame = p.phase === "render" && p.frame >= p.totalFrames;if (- now - lastEmit < PROGRESS_INTERVAL_MS &&- p.frame < p.totalFrames+ p.phase === lastPhase &&+ !lastFrame &&+ now - lastEmit < PROGRESS_INTERVAL_MS)return;lastEmit = now;- const fps = p.frame / Math.max(0.001, (now - started) / 1000);+ lastPhase = p.phase;ctx.progress({- phase: p.phase === "audio" ? "encoding" : "rendering",+ ...toJobProgress(p, (now - started) / 1000),output: count > 1 ? index : undefined,- frame: p.frame,- total: p.totalFrames,- fps: Math.round(fps * 10) / 10,- etaSec: Math.round(- (p.totalFrames - p.frame) / Math.max(fps, 0.001),- ),});},});@@ -320,6 +316,61 @@ async function renderOutput(}}+/**+ * What `renderVideo({ onProgress })` reports. Engines before the phase+ * breakdown send only "render" and "audio", without the optional fields.+ */+interface EngineProgress {+ phase: string;+ frame: number;+ totalFrames: number;+ fps?: number;+ etaSec?: number;+ detail?: string;+}++/** What each engine phase after the frames is doing, for the job message. */+const AFTER_FRAMES: Record<string, string> = {+ audio: "mixing audio",+ "audio-encode": "encoding audio",+ mux: "muxing",+ convert: "converting to GIF",+};++/** Maps an engine progress event to the job's progress. */+export function toJobProgress(+ p: EngineProgress,+ elapsedSec: number,+): JobProgress {+ if (p.phase === "prepare") {+ return {+ phase: "rendering",+ frame: 0,+ total: p.totalFrames,+ message: "preparing",+ };+ }+ if (p.phase === "render") {+ const fps = p.fps ?? p.frame / Math.max(0.001, elapsedSec);+ return {+ phase: "rendering",+ frame: p.frame,+ total: p.totalFrames,+ fps: Math.round(fps * 10) / 10,+ etaSec:+ p.etaSec ??+ Math.round((p.totalFrames - p.frame) / Math.max(fps, 0.001)),+ };+ }+ const label = AFTER_FRAMES[p.phase] ?? p.phase;+ return {+ phase: "encoding",+ frame: p.frame,+ total: p.totalFrames,+ message: p.detail ? `${label} (${p.detail})` : label,+ };+}+export function containerFor(codec: string): {ext: string;contentType: string;diff --git a/apps/renderer-service/src/render/render.test.ts b/apps/renderer-service/src/render/render.test.tsindex eef5539c..8ae4c9f5 100644--- a/apps/renderer-service/src/render/render.test.ts+++ b/apps/renderer-service/src/render/render.test.ts@@ -8,7 +8,7 @@ import { CrashJournal } from "../jobs/journal.js";import { testConfig } from "../test-helpers.js";import { recycleReason } from "../worker/recycle.js";import { buildPlan, isRenderable } from "./load.js";-import { classifyRenderError } from "./render-step.js";+import { classifyRenderError, toJobProgress } from "./render-step.js";import { ExitTrap, openSandbox } from "./sandbox.js";let ws: string;@@ -404,3 +404,49 @@ describe("crash journal", () => {expect(journal.recover()).toBeUndefined();});});++describe("render progress", () => {+ it("maps frames to rendering, with the engine's rate when it sends one", () => {+ expect(+ toJobProgress(+ { phase: "render", frame: 60, totalFrames: 120, fps: 30, etaSec: 2 },+ 4,+ ),+ ).toEqual({+ phase: "rendering",+ frame: 60,+ total: 120,+ fps: 30,+ etaSec: 2,+ });+ // Older engines send no rate: it is worked out from the elapsed time.+ expect(+ toJobProgress({ phase: "render", frame: 60, totalFrames: 120 }, 4),+ ).toMatchObject({ fps: 15, etaSec: 4 });+ });++ it("names the phase after the frames in the message", () => {+ expect(+ toJobProgress(+ {+ phase: "audio",+ frame: 120,+ totalFrames: 120,+ detail: "3/7 clips: music.mp3",+ },+ 9,+ ),+ ).toEqual({+ phase: "encoding",+ frame: 120,+ total: 120,+ message: "mixing audio (3/7 clips: music.mp3)",+ });+ expect(+ toJobProgress({ phase: "mux", frame: 120, totalFrames: 120 }, 9),+ ).toMatchObject({ phase: "encoding", message: "muxing" });+ expect(+ toJobProgress({ phase: "prepare", frame: 0, totalFrames: 120 }, 0),+ ).toMatchObject({ phase: "rendering", frame: 0, message: "preparing" });+ });+});diff --git a/packages/compositions/src/utils/audio-extractor.ts b/packages/compositions/src/utils/audio-extractor.tsindex fe14e0ad..2efc4065 100644--- a/packages/compositions/src/utils/audio-extractor.ts+++ b/packages/compositions/src/utils/audio-extractor.ts@@ -548,6 +548,35 @@ class MixBus {}}+/** Progress reported by `mixAudioTracks` as each audio or video clip is mixed. */+export interface AudioMixProgress {+ /** Clips mixed so far. */+ done: number;+ /** Audio and video clips in the tree; muted or empty ones may be skipped. */+ total: number;+ /** File name of the clip just mixed (no directory or query string). */+ source?: string;+}++export interface MixAudioOptions {+ onProgress?: (progress: AudioMixProgress) => void;+}++/** A clip's file name for progress, without paths or signed-URL queries. */+function sourceLabel(url: string): string {+ if (url.startsWith("data:")) return "inline audio";+ const name = url.split(/[?#]/)[0].split(/[\\/]/).pop();+ return name ? decodeURIComponentSafe(name) : "audio";+}++function decodeURIComponentSafe(s: string): string {+ try {+ return decodeURIComponent(s);+ } catch {+ return s;+ }+}+/*** Recursively parses the VirtualMediaData tree and mixes all active audio tracks* into a single unified stereo PCM stream (Float32Array channels).@@ -558,6 +587,7 @@ export async function mixAudioTracks(targetSampleRate = 48000,device?: GPUDevice,renderId?: string,+ options: MixAudioOptions = {},): Promise<{ channels: Float32Array[]; sampleRate: number }> {mediaLogger.debug(`[AudioExtractor] mixAudioTracks started. FPS: ${fps}, targetSampleRate: ${targetSampleRate}`,@@ -844,6 +874,14 @@ export async function mixAudioTracks(}await yieldToMain();}+ if (mediaType === "Audio" || mediaType === "Video") {+ clipsDone++;+ options.onProgress?.({+ done: clipsDone,+ total: clipsTotal,+ ...(params.sourceUrl && { source: sourceLabel(params.sourceUrl) }),+ });+ }}// Traverse children recursively@@ -938,6 +976,26 @@ export async function mixAudioTracks(}const normalized = normalizeTimeline(virtualMedia, fps);++ // Clips to mix, for progress: one per segment of each audio or video source.+ let clipsTotal = 0;+ let clipsDone = 0;+ if (options.onProgress) {+ const count = (node: VirtualMediaData | undefined) => {+ if (!node) return;+ const op = node.operation;+ if (op?.op === "source") {+ const mediaType = getMediaType(node);+ if (mediaType === "Audio" || mediaType === "Video") {+ clipsTotal += Math.max(1, op.timeline?.segments?.length ?? 0);+ }+ }+ for (const child of node.children ?? []) count(child);+ };+ count(normalized);+ options.onProgress({ done: 0, total: clipsTotal });+ }+const rootChannels = await traverse(normalized, 0, 0, 1, 1, null);// Output mixed buffer is stereo; a root mix of the same length is used as isdiff --git a/packages/framefields/src/index.ts b/packages/framefields/src/index.tsindex 70aed9a2..060ee7c0 100644--- a/packages/framefields/src/index.ts+++ b/packages/framefields/src/index.ts@@ -1094,7 +1094,11 @@ export interface RenderVideoOptions {* `signal.reason` and no file is left behind.*/signal?: AbortSignal;- /** Called as frames finish (`phase: "render"`), then once for the audio mix. */+ /**+ * Called as the render moves through its phases: "prepare", each frame+ * ("render"), "audio" (per clip mixed), "audio-encode", "mux", "convert"+ * (GIF only) and "done". The engine logs the same phases on its own.+ */onProgress?: (progress: RenderProgress) => void;}diff --git a/packages/framefields/src/render-progress.test.ts b/packages/framefields/src/render-progress.test.tsnew file mode 100644index 00000000..f3ef2576--- /dev/null+++ b/packages/framefields/src/render-progress.test.ts@@ -0,0 +1,94 @@+import fs from "node:fs/promises";+import os from "node:os";+import path from "node:path";+import { describe, expect, it } from "vitest";+import {+ Composition,+ encodeStereoWav,+ Layer,+ type RenderProgress,+} from "./index.js";++async function tone(dir: string): Promise<string> {+ const sampleRate = 48_000;+ const samples = new Float32Array(sampleRate);+ for (let i = 0; i < samples.length; i++) {+ samples[i] = 0.3 * Math.sin((2 * Math.PI * 440 * i) / sampleRate);+ }+ const file = path.join(dir, "tone.wav");+ await fs.writeFile(file, encodeStereoWav([samples, samples], { sampleRate }));+ return file;+}++function film(audio: string): Composition {+ const comp = new Composition({+ width: 64,+ height: 64,+ fps: 30,+ durationFrames: 30,+ backgroundColor: "#202020",+ });+ comp.add(Layer.audio(audio, { id: "tone", durationFrames: 30 }));+ return comp;+}++/** Phases in the order they were first reported. */+function phases(events: RenderProgress[]): string[] {+ return events+ .map((e) => e.phase)+ .filter((phase, i, all) => all.indexOf(phase) === i);+}++describe("renderVideo progress", () => {+ it("reports each phase in order, with the audio clips mixed", async () => {+ const dir = await fs.mkdtemp(path.join(os.tmpdir(), "ff-progress-"));+ const events: RenderProgress[] = [];+ const result = await film(await tone(dir)).renderVideo({+ outputPath: path.join(dir, "out.mp4"),+ onProgress: (p) => events.push({ ...p }),+ });+ await result.cleanup();++ expect(phases(events)).toEqual([+ "prepare",+ "render",+ "audio",+ "audio-encode",+ "mux",+ "done",+ ]);+ const frames = events.filter((e) => e.phase === "render");+ expect(frames.at(-1)).toMatchObject({ frame: 30, totalFrames: 30 });+ for (let i = 1; i < frames.length; i++) {+ expect(frames[i].frame).toBeGreaterThanOrEqual(frames[i - 1].frame);+ }+ expect(events.filter((e) => e.phase === "audio").at(-1)).toMatchObject({+ detail: "1/1 clips: tone.wav",+ phaseProgress: 1,+ });+ expect(+ events.filter((e) => e.phase === "audio-encode").at(-1)?.phaseProgress,+ ).toBe(1);+ const elapsed = events.map((e) => e.elapsedMs);+ expect(elapsed).toEqual([...elapsed].sort((a, b) => a - b));+ }, 60_000);++ it("skips the frame phase for audio-only output", async () => {+ const dir = await fs.mkdtemp(path.join(os.tmpdir(), "ff-progress-"));+ const events: RenderProgress[] = [];+ const result = await film(await tone(dir)).renderVideo({+ outputPath: path.join(dir, "out.m4a"),+ codec: "aac",+ onProgress: (p) => events.push({ ...p }),+ });+ await result.cleanup();++ expect(phases(events)).toEqual([+ "prepare",+ "audio",+ "audio-encode",+ "mux",+ "done",+ ]);+ }, 60_000);+});diff --git a/packages/renderer/src/headless-webgpu-renderer.ts b/packages/renderer/src/headless-webgpu-renderer.tsindex d4a71b82..b6bcfb9d 100644--- a/packages/renderer/src/headless-webgpu-renderer.ts+++ b/packages/renderer/src/headless-webgpu-renderer.ts@@ -50,6 +50,7 @@ import { preloadFonts } from "./asset-preloader.js";import { bootstrapMediabunny } from "./bootstrap-mediabunny.js";import { discoverAndRegisterNodeRenderers } from "./dynamic-node-discovery.js";import { rendererLogger } from "./logger.js";+import { type RenderProgress, RenderReporter } from "./render-reporter.js";// Vision reports its inference backend once per process (see// VISION_BACKEND_LISTENER in @framefields/vision).@@ -264,35 +265,6 @@ if (typeof skiaProto.getAttribute === "undefined") {skiaProto.getAttribute = () => null;}-// ─── Types ──────────────────────────────────────────────────────────────────────--export interface WorkerInput {- virtualMedia: VirtualMediaData;- width: number;- height: number;- fps: number;- startFrame: number;- endFrame: number;- renderId: string;- outputFormat: "png" | "pixels";- workerIndex: number;-}--export type WorkerMessage =- | { type: "frame"; renderId: string; frameIndex: number; pixels: Uint8Array }- | { type: "png"; renderId: string; data: Uint8Array }- | { type: "done"; renderId: string }- | {- type: "lut";- renderId: string;- raw: {- points: Array<[number, number, number]>;- size: number;- type: "1D" | "3D";- };- }- | { type: "error"; renderId?: string; error: string };-// ─── Concurrency Limiter ───────────────────────────────────────────────────────const CPU_COUNT = os.cpus().length;@@ -376,6 +348,14 @@ const returnStillId = (id: string): void => {// ─── Helpers ───────────────────────────────────────────────────────────────────+async function fileSize(file: string): Promise<number | undefined> {+ try {+ return (await fs.stat(file)).size;+ } catch {+ return undefined;+ }+}+function toEvenDimension(n: number): number {const v = Math.round(n);return v % 2 === 0 ? v : v + 1;@@ -1071,7 +1051,11 @@ export class HeadlessWebGPURenderer {* `signal.reason` and the partial output is deleted.*/signal?: AbortSignal;- /** Called after each frame is queued for encoding, then once for audio. */+ /**+ * Called as the render moves through its phases: "prepare", each+ * frame ("render"), "audio" (mixing, per clip), "audio-encode",+ * "mux", "convert" (GIF only) and "done".+ */onProgress?: (progress: RenderProgress) => void;},): Promise<{@@ -1149,8 +1133,28 @@ export class HeadlessWebGPURenderer {"[HeadlessWebGPURenderer] Acquiring slot for video render (inline)",);+ if (renderSemaphore.stats.active >= renderSemaphore.stats.max) {+ rendererLogger.info(+ { renderId, ...renderSemaphore.stats },+ `[HeadlessWebGPURenderer] Waiting for a render slot (${renderSemaphore.stats.active} running, ${renderSemaphore.stats.queued} queued)`,+ );+ }await renderSemaphore.acquire();+ const reporter = new RenderReporter({+ renderId,+ totalFrames,+ onProgress: options?.onProgress,+ });+ const seconds = `${(durationMs / 1000).toFixed(1)}s`;+ reporter.stage(+ "prepare",+ isAudioOnly+ ? `Preparing ${seconds} of audio (${audioCodec})`+ : `Preparing ${totalFrames} frames (${seconds}) at ${width}x${height}, ${fps} fps, ${codecOption}`,+ { width, height, fps, durationMs, codec: codecOption },+ );+let lastCpuUsage = process.cpuUsage();let lastCpuTime = process.hrtime.bigint();const getCpuUsage = () => {@@ -1232,7 +1236,6 @@ export class HeadlessWebGPURenderer {let outputStarted = false;let outputFinalized = false;- const device = await ensureDevice();// Resources for inline rendering (cleaned up in finally block)let surface: NodeSurfaceProvider | undefined;@@ -1240,6 +1243,8 @@ export class HeadlessWebGPURenderer {let pipeline: ZeroCopyWebCodecsPipeline | undefined;try {+ // Inside the try, so a device failure still releases the slot.+ const device = await ensureDevice();await output.start();outputStarted = true;await preloadFonts(virtualMedia, device);@@ -1302,15 +1307,10 @@ export class HeadlessWebGPURenderer {};await prePass(0);+ reporter.stage("render", `Rendering ${totalFrames} frames`);// Render and encode frames using pipelined double-buffered DMA staging ringfor (let i = 0; i < totalFrames; i++) {options?.signal?.throwIfAborted();- if (i % 10 === 0) {- rendererLogger.info(- { renderId, frame: i, totalFrames },- `[HeadlessWebGPURenderer] Rendering frame ${i}/${totalFrames}`,- );- }if (ctx && surface && renderer && pipeline) {// Assets first used on a later frame (a LUT, a glyph) get the@@ -1363,11 +1363,7 @@ export class HeadlessWebGPURenderer {);await pipeline.enqueueFrame(i, targetTexture, encoder);- options?.onProgress?.({- phase: "render",- frame: i + 1,- totalFrames,- });+ if (i + 1 < totalFrames) reporter.frameDone(i + 1);} else {throw new Error("Invalid render context state");}@@ -1375,6 +1371,8 @@ export class HeadlessWebGPURenderer {if (pipeline) {await pipeline.drain();+ // 100% once the last frames left the staging ring.+ reporter.frameDone(totalFrames);const pipelineStats = pipeline.getStats();if (inspector && qaOptions) {videoQa = inspector.stats(@@ -1395,17 +1393,22 @@ export class HeadlessWebGPURenderer {// ── Audio ──────────────────────────────────────────────────────────────options?.signal?.throwIfAborted();- options?.onProgress?.({- phase: "audio",- frame: totalFrames,- totalFrames,- });+ reporter.stage("audio", "Mixing audio");const { channels, sampleRate } = await mixAudioTracks(virtualMedia,fps,48_000,device,renderId,+ {+ onProgress: ({ done, total, source }) =>+ reporter.update(+ total > 0+ ? `${done}/${total} clips${source ? `: ${source}` : ""}`+ : "no audio clips",+ total > 0 ? done / total : undefined,+ ),+ },);const numChannels = channels.length;@@ -1416,6 +1419,10 @@ export class HeadlessWebGPURenderer {const totalSamples = channels[0].length;const chunkSize = Math.round(0.1 * sampleRate);+ reporter.stage(+ "audio-encode",+ `Encoding ${(totalSamples / sampleRate).toFixed(1)}s of audio (${audioCodec})`,+ );for (let offset = 0; offset < totalSamples; offset += chunkSize) {const currentChunk = Math.min(chunkSize, totalSamples - offset);@@ -1442,9 +1449,16 @@ export class HeadlessWebGPURenderer {} finally {sample.close();}+ const written = offset + currentChunk;+ reporter.update(+ `${(written / sampleRate).toFixed(1)}s/${(totalSamples / sampleRate).toFixed(1)}s`,+ written / totalSamples,+ );}options?.signal?.throwIfAborted();+ // Flushes the encoders and writes the container index.+ reporter.stage("mux", `Muxing ${extension}`);await output.finalize();outputFinalized = true;@@ -1467,15 +1481,16 @@ export class HeadlessWebGPURenderer {};if (codecOption === "gif") {- rendererLogger.debug(- "[HeadlessWebGPURenderer] Converting rendered MP4 to high-quality GIF...",- );+ reporter.stage("convert", "Converting to GIF (ffmpeg)");await convertMp4ToGif(tempFilePath, tempGifPath);+ reporter.finish({ bytes: await fileSize(tempGifPath) });return { filePath: tempGifPath, cleanup, ...(qa && { qa }) };}+ reporter.finish({ bytes: await fileSize(tempFilePath) });return { filePath: tempFilePath, cleanup, ...(qa && { qa }) };} catch (error) {+ reporter.fail(error, options?.signal?.aborted === true);if (outputStarted && !outputFinalized) {try {await output.finalize();@@ -1489,6 +1504,7 @@ export class HeadlessWebGPURenderer {} catch {}throw error;} finally {+ reporter.dispose();if (statsInterval) {clearInterval(statsInterval);}@@ -1529,14 +1545,5 @@ export class HeadlessWebGPURenderer {}}-/** Progress reported by `renderVideo({ onProgress })`. */-export interface RenderProgress {- /** "render" while frames are drawn and encoded, "audio" while mixing. */- phase: "render" | "audio";- /** Frames finished so far (1-based). */- frame: number;- totalFrames: number;-}-export const HeadlessMediaRenderer = HeadlessWebGPURenderer;export type HeadlessMediaRenderer = HeadlessWebGPURenderer;diff --git a/packages/renderer/src/index.ts b/packages/renderer/src/index.tsindex a8418280..44c33604 100644--- a/packages/renderer/src/index.ts+++ b/packages/renderer/src/index.ts@@ -5,5 +5,6 @@ export * from "./headless-webgpu-renderer.js";export * from "./job-reset.js";export * from "./logger.js";export * from "./node-manifest.js";+export type { RenderPhase, RenderProgress } from "./render-reporter.js";export * from "./video-qa.js";export * from "./zero-copy-webcodecs-pipeline.js";diff --git a/packages/renderer/src/render-reporter.ts b/packages/renderer/src/render-reporter.tsnew file mode 100644index 00000000..cacce7b7--- /dev/null+++ b/packages/renderer/src/render-reporter.ts@@ -0,0 +1,323 @@+import { rendererLogger } from "./logger.js";++/**+ * Stages of `renderVideo`, in order. Audio-only renders skip "render";+ * "convert" runs only for GIF output.+ */+export type RenderPhase =+ | "prepare"+ | "render"+ | "audio"+ | "audio-encode"+ | "mux"+ | "convert"+ | "done";++/** Progress reported by `renderVideo({ onProgress })`. */+export interface RenderProgress {+ /**+ * "prepare" loads fonts and warms the pipeline, "render" draws and encodes+ * frames, "audio" mixes the soundtrack, "audio-encode" encodes it, "mux"+ * writes the container, "convert" makes the GIF, "done" is the last event.+ */+ phase: RenderPhase;+ /** Frames finished so far (1-based); `totalFrames` once frames are done. */+ frame: number;+ totalFrames: number;+ /** Time since the render started. */+ elapsedMs: number;+ /** Share of the current phase done (0–1), when it can be measured. */+ phaseProgress?: number;+ /** Frames per second so far ("render" only). */+ fps?: number;+ /** Estimated seconds until the frames are done ("render" only). */+ etaSec?: number;+ /** What the phase is working on, e.g. "3/7 sources: music.mp3". */+ detail?: string;+}++interface ReporterLog {+ info(obj: object, msg: string): void;+ debug(obj: object, msg: string): void;+ warn(obj: object, msg: string): void;+ error(obj: object, msg: string): void;+}++interface RenderReporterOptions {+ renderId: string;+ totalFrames: number;+ onProgress?: (progress: RenderProgress) => void;+ /** Percent step between frame log lines (default 10). */+ milestonePercent?: number;+ log?: ReporterLog;+ now?: () => number;+}++const LOG_PREFIX = "[HeadlessWebGPURenderer]";+/** A milestone reached sooner than this after the last line is not logged. */+const MIN_LOG_GAP_MS = 1000;+/**+ * Quiet time before the first "still …" line of a phase. Each one doubles it+ * up to the cap, so a long phase adds a few lines rather than one a minute.+ */+const HEARTBEAT_FIRST_MS = 10_000;+const HEARTBEAT_MAX_MS = 5 * 60_000;++const ACTIVITY: Record<RenderPhase, string> = {+ prepare: "preparing",+ render: "rendering",+ audio: "mixing audio",+ "audio-encode": "encoding audio",+ mux: "muxing",+ convert: "converting to GIF",+ done: "finishing",+};++/**+ * Reports a render's progress to the log and to `onProgress`.+ *+ * The log stays short whatever the length of the film, so it fits an agent's+ * context: one line per phase, frame lines only at percent milestones, and a+ * "still …" line when a phase goes quiet, backing off from 10 s to 5 min.+ * `onProgress` gets every update; the "still …" lines are log-only, so a stall+ * detector fed by `onProgress` still sees a hang.+ */+export class RenderReporter {+ private readonly renderId: string;+ private readonly totalFrames: number;+ private readonly onProgress?: (progress: RenderProgress) => void;+ private readonly milestonePercent: number;+ private readonly log: ReporterLog;+ private readonly now: () => number;+ private readonly startedAt: number;+ private readonly phaseMs = new Map<RenderPhase, number>();++ private phase: RenderPhase = "prepare";+ private phaseStartedAt: number;+ private frame = 0;+ private phaseProgress?: number;+ private detail?: string;+ private lastMilestone = 0;+ /** Last phase or milestone line; "still …" lines don't count. */+ private lastLogAt: number;+ private heartbeatMs = HEARTBEAT_FIRST_MS;+ private heartbeat?: ReturnType<typeof setTimeout>;++ constructor(options: RenderReporterOptions) {+ this.renderId = options.renderId;+ this.totalFrames = options.totalFrames;+ this.onProgress = options.onProgress;+ this.milestonePercent = options.milestonePercent ?? 10;+ this.log = options.log ?? rendererLogger;+ this.now = options.now ?? (() => performance.now());+ this.startedAt = this.now();+ this.phaseStartedAt = this.startedAt;+ this.lastLogAt = this.startedAt;+ }++ /** Starts a phase: logs `message` and emits a progress event. */+ stage(phase: RenderPhase, message: string, fields: object = {}): void {+ this.closePhase();+ this.phase = phase;+ this.phaseProgress = undefined;+ this.detail = undefined;+ this.heartbeatMs = HEARTBEAT_FIRST_MS;+ if (phase !== "render" && phase !== "prepare")+ this.frame = this.totalFrames;+ this.info(message, fields);+ this.emit();+ }++ /** Records a finished frame; logs at each milestone. */+ frameDone(frame: number): void {+ this.frame = frame;+ const percent = (frame / this.totalFrames) * 100;+ const milestone =+ frame >= this.totalFrames+ ? 100+ : Math.floor(percent / this.milestonePercent) * this.milestonePercent;+ const { fps, etaSec } = this.rate();+ if (milestone > this.lastMilestone) {+ this.lastMilestone = milestone;+ if (milestone === 100 || this.now() - this.lastLogAt >= MIN_LOG_GAP_MS) {+ const tail =+ milestone === 100+ ? `in ${formatDuration(this.now() - this.phaseStartedAt)}`+ : `eta ${formatDuration(etaSec * 1000)}`;+ this.info(+ `Rendered ${milestone}% (${frame}/${this.totalFrames} frames) · ${fps.toFixed(1)} fps · ${tail}`,+ { fps: round1(fps), etaSec },+ );+ }+ }+ this.emit({ fps: round1(fps), etaSec });+ }++ /** Updates what the current phase is doing; shown in "still …" lines. */+ update(detail: string | undefined, phaseProgress?: number): void {+ this.detail = detail;+ this.phaseProgress = phaseProgress;+ this.emit();+ }++ /** Logs the summary and emits the "done" event. */+ finish(fields: { bytes?: number } = {}): void {+ this.closePhase();+ this.phase = "done";+ this.frame = this.totalFrames;+ this.stopHeartbeat();+ const total = this.now() - this.startedAt;+ const parts = [...this.phaseMs].map(+ ([phase, ms]) => `${phase} ${formatDuration(ms)}`,+ );+ const size =+ fields.bytes !== undefined ? ` (${formatBytes(fields.bytes)})` : "";+ this.log.info(+ {+ renderId: this.renderId,+ phase: "done",+ elapsedMs: Math.round(total),+ phaseMs: Object.fromEntries(+ [...this.phaseMs].map(([phase, ms]) => [phase, Math.round(ms)]),+ ),+ ...fields,+ },+ `${LOG_PREFIX} Done in ${formatDuration(total)}${size}: ${parts.join(" · ")}`,+ );+ this.emit();+ }++ /** Logs which phase the render stopped in. */+ fail(error: unknown, aborted: boolean): void {+ this.stopHeartbeat();+ const elapsed = formatDuration(this.now() - this.startedAt);+ const fields = {+ renderId: this.renderId,+ phase: this.phase,+ frame: this.frame,+ totalFrames: this.totalFrames,+ elapsedMs: Math.round(this.now() - this.startedAt),+ };+ if (aborted) {+ this.log.warn(+ fields,+ `${LOG_PREFIX} Aborted while ${ACTIVITY[this.phase]} after ${elapsed}`,+ );+ } else {+ this.log.error(+ { ...fields, err: error },+ `${LOG_PREFIX} Failed while ${ACTIVITY[this.phase]} after ${elapsed}`,+ );+ }+ }++ /** Stops the "still …" timer. Safe to call more than once. */+ dispose(): void {+ this.stopHeartbeat();+ }++ private closePhase(): void {+ const now = this.now();+ this.phaseMs.set(+ this.phase,+ (this.phaseMs.get(this.phase) ?? 0) + (now - this.phaseStartedAt),+ );+ this.phaseStartedAt = now;+ }++ private rate(): { fps: number; etaSec: number } {+ const seconds = Math.max(0.001, (this.now() - this.phaseStartedAt) / 1000);+ const fps = this.frame / seconds;+ const etaSec = Math.round(+ (this.totalFrames - this.frame) / Math.max(fps, 0.001),+ );+ return { fps, etaSec };+ }++ private info(message: string, fields: object = {}): void {+ this.log.info(+ {+ renderId: this.renderId,+ phase: this.phase,+ frame: this.frame,+ totalFrames: this.totalFrames,+ ...fields,+ },+ `${LOG_PREFIX} ${message}`,+ );+ this.lastLogAt = this.now();+ this.armHeartbeat();+ }++ private emit(extra: Partial<RenderProgress> = {}): void {+ this.onProgress?.({+ phase: this.phase,+ frame: this.frame,+ totalFrames: this.totalFrames,+ elapsedMs: Math.round(this.now() - this.startedAt),+ ...(this.phaseProgress !== undefined && {+ phaseProgress: this.phaseProgress,+ }),+ ...(this.detail !== undefined && { detail: this.detail }),+ ...extra,+ });+ }++ private armHeartbeat(): void {+ this.stopHeartbeat();+ this.heartbeat = setTimeout(() => this.beat(), this.heartbeatMs);+ // Never keeps the process alive on its own.+ this.heartbeat.unref?.();+ }++ private beat(): void {+ const inPhase = formatDuration(this.now() - this.phaseStartedAt);+ let what = "";+ if (this.phase === "render") {+ what = ` (frame ${this.frame}/${this.totalFrames})`;+ } else if (this.detail || this.phaseProgress !== undefined) {+ const pct =+ this.phaseProgress !== undefined+ ? `${Math.floor(this.phaseProgress * 100)}%`+ : "";+ what = ` (${[pct, this.detail].filter(Boolean).join(", ")})`;+ }+ this.log.info(+ {+ renderId: this.renderId,+ phase: this.phase,+ frame: this.frame,+ totalFrames: this.totalFrames,+ },+ `${LOG_PREFIX} Still ${ACTIVITY[this.phase]}${what}, ${inPhase} in this phase`,+ );+ this.heartbeatMs = Math.min(this.heartbeatMs * 2, HEARTBEAT_MAX_MS);+ this.armHeartbeat();+ }++ private stopHeartbeat(): void {+ if (this.heartbeat) clearTimeout(this.heartbeat);+ this.heartbeat = undefined;+ }+}++/** "4.2s", "2m 05s", "1h 02m". */+export function formatDuration(ms: number): string {+ const s = Math.max(0, ms) / 1000;+ if (s < 60) return `${s.toFixed(1)}s`;+ const total = Math.round(s);+ const h = Math.floor(total / 3600);+ const m = Math.floor((total % 3600) / 60);+ const sec = total % 60;+ if (h > 0) return `${h}h ${String(m).padStart(2, "0")}m`;+ return `${m}m ${String(sec).padStart(2, "0")}s`;+}++function formatBytes(bytes: number): string {+ if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;+ return `${(bytes / 1024 / 1024).toFixed(1)} MB`;+}++function round1(n: number): number {+ return Math.round(n * 10) / 10;+}diff --git a/packages/renderer/src/tests/render-reporter.test.ts b/packages/renderer/src/tests/render-reporter.test.tsnew file mode 100644index 00000000..f4df720e--- /dev/null+++ b/packages/renderer/src/tests/render-reporter.test.ts@@ -0,0 +1,147 @@+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import {+ formatDuration,+ type RenderProgress,+ RenderReporter,+} from "../render-reporter.js";++function setup(totalFrames: number) {+ const lines: string[] = [];+ const events: RenderProgress[] = [];+ const push = (_obj: object, msg: string) => {+ lines.push(msg.replace("[HeadlessWebGPURenderer] ", ""));+ };+ const reporter = new RenderReporter({+ renderId: "vid-test",+ totalFrames,+ onProgress: (p) => events.push(p),+ log: { info: push, debug: () => {}, warn: push, error: push },+ now: () => Date.now(),+ });+ return { reporter, lines, events };+}++describe("RenderReporter", () => {+ beforeEach(() => {+ vi.useFakeTimers();+ });+ afterEach(() => {+ vi.useRealTimers();+ });++ it("logs a bounded number of frame lines whatever the length", () => {+ for (const totalFrames of [300, 30_000]) {+ const { reporter, lines, events } = setup(totalFrames);+ reporter.stage("render", "Rendering");+ for (let f = 1; f <= totalFrames; f++) {+ vi.advanceTimersByTime(100);+ reporter.frameDone(f);+ }+ reporter.dispose();+ const frameLines = lines.filter((l) => l.startsWith("Rendered "));+ expect(frameLines).toHaveLength(10);+ expect(frameLines[0]).toMatch(+ new RegExp(+ `^Rendered 10% \\(${totalFrames / 10}/${totalFrames} frames\\) · 10\\.0 fps · eta `,+ ),+ );+ expect(frameLines.at(-1)).toMatch(/^Rendered 100% .* in /);+ // onProgress still sees every frame.+ expect(events.filter((e) => e.phase === "render")).toHaveLength(+ totalFrames + 1,+ );+ }+ });++ it("skips milestones reached within a second, but always logs 100%", () => {+ const { reporter, lines } = setup(30);+ reporter.stage("render", "Rendering");+ for (let f = 1; f <= 30; f++) {+ vi.advanceTimersByTime(10);+ reporter.frameDone(f);+ }+ reporter.dispose();+ expect(lines).toEqual([+ "Rendering",+ expect.stringMatching(/^Rendered 100% \(30\/30 frames\)/),+ ]);+ });++ it("logs 'still …' lines with backoff, without emitting progress", () => {+ const { reporter, lines, events } = setup(100);+ reporter.stage("mux", "Muxing mp4");+ const before = events.length;+ vi.advanceTimersByTime(20 * 60_000);+ reporter.dispose();+ const still = lines.filter((l) => l.startsWith("Still "));+ // 10 s, 30 s, 70 s, 150 s, 310 s, then every 5 min: 610 s, 910 s.+ expect(still).toHaveLength(7);+ expect(still[0]).toBe("Still muxing, 10.0s in this phase");+ expect(still[1]).toBe("Still muxing, 30.0s in this phase");+ expect(events.length).toBe(before);+ });++ it("restarts the backoff on a new phase and shows the phase's detail", () => {+ const { reporter, lines } = setup(100);+ reporter.stage("mux", "Muxing mp4");+ vi.advanceTimersByTime(40_000);+ reporter.stage("audio", "Mixing audio");+ reporter.update("3/7 clips: music.mp3", 3 / 7);+ vi.advanceTimersByTime(10_000);+ reporter.dispose();+ expect(lines.at(-1)).toBe(+ "Still mixing audio (42%, 3/7 clips: music.mp3), 10.0s in this phase",+ );+ });++ it("summarizes phase durations and reports done", () => {+ const { reporter, lines, events } = setup(60);+ reporter.stage("prepare", "Preparing");+ vi.advanceTimersByTime(1_000);+ reporter.stage("render", "Rendering");+ vi.advanceTimersByTime(4_000);+ reporter.frameDone(60);+ reporter.stage("audio", "Mixing audio");+ vi.advanceTimersByTime(500);+ reporter.stage("mux", "Muxing mp4");+ vi.advanceTimersByTime(250);+ reporter.finish({ bytes: 3 * 1024 * 1024 });+ vi.advanceTimersByTime(60_000);+ expect(lines.at(-1)).toBe(+ "Done in 5.8s (3.0 MB): prepare 1.0s · render 4.0s · audio 0.5s · mux 0.3s",+ );+ expect(events.at(-1)).toMatchObject({+ phase: "done",+ frame: 60,+ totalFrames: 60,+ elapsedMs: 5750,+ });+ expect(events.map((e) => e.phase)).toEqual([+ "prepare",+ "render",+ "render",+ "audio",+ "mux",+ "done",+ ]);+ });++ it("names the phase a render failed or was aborted in", () => {+ const { reporter, lines } = setup(10);+ reporter.stage("audio", "Mixing audio");+ reporter.fail(new Error("boom"), false);+ reporter.fail(new Error("stop"), true);+ expect(lines.slice(-2)).toEqual([+ "Failed while mixing audio after 0.0s",+ "Aborted while mixing audio after 0.0s",+ ]);+ });+});++describe("formatDuration", () => {+ it("formats seconds, minutes and hours", () => {+ expect(formatDuration(4_240)).toBe("4.2s");+ expect(formatDuration(125_000)).toBe("2m 05s");+ expect(formatDuration(3_720_000)).toBe("1h 02m");+ });+});diff --git a/packages/renderer/src/video-worker.ts b/packages/renderer/src/video-worker.tsdeleted file mode 100644index 823511a0..00000000--- a/packages/renderer/src/video-worker.ts+++ /dev/null@@ -1,436 +0,0 @@-import { parentPort } from "node:worker_threads";-import {- compositionStateStore,- drawCompositionTree,-} from "@framefields/compositions";-import type { VirtualMediaData } from "@framefields/core";-import {- clearAllVideoCache,- createHeadlessDOMElement,- ensureDevice,- initHeadlessWebGPU,- installIntersectionObserverStub,- lutStore,- mediaDecoderCache,- NodeSurfaceProvider,- patchCanvasPrototypeForDOM,- type RenderContextValue,- Renderer2D,- resetDeviceInstance,- shaderStore,- textureCache,-} from "@framefields/webgpu-renderers";-import sharp from "sharp";-import { Canvas as SkiaCanvas, Image as SkiaImage } from "skia-canvas";-import { preloadFonts } from "./asset-preloader.js";-import { DmaStagingRing } from "./dma-staging-ring.js";-import { discoverAndRegisterNodeRenderers } from "./dynamic-node-discovery.js";-import { rendererLogger } from "./logger.js";--const globalObj = globalThis as unknown as Record<string, unknown>;--if (typeof globalThis !== "undefined") {- globalObj.__IS_HEADLESS_RENDERER__ = true;- globalObj.__GATEWAI_DELAYS__ ??= new Set<number>();-}--import { bootstrapMediabunny } from "./bootstrap-mediabunny.js";--bootstrapMediabunny();--if (typeof globalThis.requestAnimationFrame === "undefined") {- globalObj.requestAnimationFrame = (cb: () => void) => setTimeout(cb, 0);- globalObj.cancelAnimationFrame = (id: number) => clearTimeout(id);-}--const dummyDomElement = createHeadlessDOMElement();--if (typeof globalThis.OffscreenCanvas === "undefined") {- globalObj.OffscreenCanvas = SkiaCanvas;-}--if (typeof globalThis.HTMLCanvasElement === "undefined") {- globalObj.HTMLCanvasElement = SkiaCanvas;-}--if (typeof globalThis.Image === "undefined") {- globalObj.Image = SkiaImage;-}--if (typeof globalThis.window === "undefined") {- globalObj.window = globalThis;-}--installIntersectionObserverStub();--if (typeof globalThis.document === "undefined") {- globalObj.document = {- createElement: (tag: string) => {- if (tag === "canvas") return new SkiaCanvas(1, 1);- return { ...dummyDomElement };- },- getElementsByTagName: () => [],- documentElement: dummyDomElement,- body: dummyDomElement,- };-} else {- const docObj = globalThis.document as unknown as Record<string, unknown>;- if (typeof docObj.getElementsByTagName === "undefined") {- docObj.getElementsByTagName = () => [];- }- docObj.documentElement ??= dummyDomElement;- docObj.body ??= dummyDomElement;-}--const skiaProto = SkiaCanvas.prototype as unknown as Record<string, unknown>;-patchCanvasPrototypeForDOM(skiaProto, dummyDomElement);-if (typeof skiaProto.getAttribute === "undefined") {- skiaProto.getAttribute = () => null;-}--export interface WorkerInput {- virtualMedia: VirtualMediaData;- width: number;- height: number;- fps: number;- startFrame: number;- endFrame: number;- renderId: string;- outputFormat: "png" | "pixels";- workerIndex: number;-}--export type WorkerMessage =- | { type: "frame"; renderId: string; frameIndex: number; pixels: Uint8Array }- | { type: "png"; renderId: string; data: Uint8Array }- | { type: "done"; renderId: string }- | {- type: "lut";- renderId: string;- raw: {- points: Array<[number, number, number]>;- size: number;- type: "1D" | "3D";- };- }- | { type: "error"; renderId?: string; error: string };--let isInitialized = false;-let initPromise: Promise<void> | null = null;--function initializeWorker(): Promise<void> {- if (isInitialized) return Promise.resolve();- if (initPromise) return initPromise;-- initPromise = (async () => {- try {- rendererLogger.debug(- "[video-worker] Initializing headless WebGPU in worker thread…",- );- await initHeadlessWebGPU();- await ensureDevice();- rendererLogger.debug(- "[video-worker] Scanning node renderers in worker thread…",- );- await discoverAndRegisterNodeRenderers();- isInitialized = true;- initPromise = null;- rendererLogger.debug(- "[video-worker] Worker thread initialization complete.",- );- } catch (error) {- initPromise = null;- throw error;- }- })();-- return initPromise;-}--if (parentPort) {- if (process.env.GATEWAI_UNIT_TEST === "true") {- globalThis.fetch = async () => {- throw new Error("Network requests are disabled in unit tests");- };- }-- parentPort.on("message", (task: any) => {- if (!task || task.type !== "start") return;-- void (async () => {- try {- const {- virtualMedia,- width,- height,- fps,- startFrame,- endFrame,- renderId,- outputFormat,- workerIndex,- } = task.input;-- rendererLogger.debug(- { renderId, workerIndex },- `[video-worker] Worker ${workerIndex} initializing renderer`,- );- await initializeWorker();- const device = await ensureDevice();- await preloadFonts(virtualMedia, device);- const surface = new NodeSurfaceProvider(device, width, height);- const stagingRing = new DmaStagingRing(device, {- width,- height,- capacity: 2,- });- const renderer = new Renderer2D(device, surface.colorFormat);- const ctx: RenderContextValue = { device, renderer, surface };- rendererLogger.debug(- { renderId, workerIndex },- `[video-worker] Worker ${workerIndex} setup complete, starting render loop`,- );-- let lutSub: (() => void) | undefined;- let onTaskMessage: ((msg: any) => void) | undefined;- const ackPromises = new Map<number, () => void>();- let lastAckedFrame = startFrame - 1;-- try {- onTaskMessage = (msg: any) => {- if (msg && msg.type === "ack") {- const acked = msg.frameIndex as number;- if (acked > lastAckedFrame) {- lastAckedFrame = acked;- }- const resolve = ackPromises.get(acked);- if (resolve) {- resolve();- ackPromises.delete(acked);- }- }- };- parentPort!.on("message", onTaskMessage);-- lutSub = lutStore.onChange((key) => {- const raw = lutStore.getRawData(key);- if (raw) {- parentPort!.postMessage({- type: "lut",- renderId,- raw,- } satisfies WorkerMessage);- }- });-- // Pre-pass: run drawCompositionTree once on the startFrame to trigger and await all LUT extractions/loads- {- rendererLogger.debug(- { renderId, workerIndex },- `[video-worker] Pre-pass rendering startFrame ${startFrame} to trigger/await all LUTs`,- );- const preEncoder = device.createCommandEncoder();- const dummyTex = renderer.getTemporaryTexture(width, height);- const dummyView = dummyTex.createView();-- const preClear = renderer.beginFrame(- preEncoder,- dummyView,- { r: 0, g: 0, b: 0, a: 0 },- width,- height,- "clear",- );- preClear.end();-- await drawCompositionTree(- ctx,- preEncoder,- dummyView,- dummyTex,- width,- height,- virtualMedia,- {- frame: startFrame,- compositionFrame: startFrame,- fps,- isHeadless: true,- renderId,- virtualMedia,- containerWidth: width,- containerHeight: height,- isVideoMode: virtualMedia.operation?.dataType === "Video",- },- );-- device.queue.submit([preEncoder.finish()]);- await lutStore.awaitAllPending(device);- }-- // Compute backpressure limit based on frame size to prevent- // native heap corruption from excessive pending pixel buffers.- // Target: ~120 MB max pending per worker.- const frameSizeBytes = width * height * 4;- const maxPendingBytes = 120 * 1024 * 1024;- const maxUnacked = Math.max(- 2,- Math.min(5, Math.floor(maxPendingBytes / frameSizeBytes)),- );-- const inFlightWorkerFrames: number[] = [];- const consumeWorkerFrame = async (frameIdx: number) => {- const pixelsArr = await stagingRing.readAndReleaseFrame(frameIdx);- if (outputFormat === "png") {- const pngBuffer = await sharp(Buffer.from(pixelsArr), {- raw: { width, height, channels: 4 },- })- .png()- .toBuffer();- parentPort!.postMessage({- type: "png",- renderId,- data: new Uint8Array(pngBuffer),- } satisfies WorkerMessage);- } else {- parentPort!.postMessage(- {- type: "frame",- renderId,- frameIndex: frameIdx,- pixels: pixelsArr,- } satisfies WorkerMessage,- [pixelsArr.buffer as ArrayBuffer],- );- }- };-- for (let i = startFrame; i <= endFrame; i++) {- // Implement backpressure: do not render too far ahead of encoding- while (i - lastAckedFrame > maxUnacked) {- await new Promise<void>((resolve) => {- ackPromises.set(i - maxUnacked, resolve);- });- }-- if (inFlightWorkerFrames.length >= stagingRing.capacity) {- const prevFrame = inFlightWorkerFrames.shift();- if (prevFrame !== undefined) {- await consumeWorkerFrame(prevFrame);- }- }-- if (i % 10 === 0) {- rendererLogger.info(- { renderId, workerIndex, frame: i },- `[video-worker] Rendering frame ${i} (Worker ${workerIndex})`,- );- }- compositionStateStore.setState(renderId, i, fps, true);- mediaDecoderCache.tick();-- const encoder = device.createCommandEncoder();- const targetView = surface.getCurrentTextureView();- const targetTexture = surface.getCurrentTexture();-- const clearPass = renderer.beginFrame(- encoder,- targetView,- { r: 0, g: 0, b: 0, a: 0 },- width,- height,- "clear",- );- clearPass.end();-- await drawCompositionTree(- ctx,- encoder,- targetView,- targetTexture,- width,- height,- virtualMedia,- {- frame: i,- compositionFrame: i,- fps,- isHeadless: true,- renderId,- virtualMedia,- containerWidth: width,- containerHeight: height,- isVideoMode: virtualMedia.operation?.dataType === "Video",- },- );-- await stagingRing.prepareSlot(i);- stagingRing.stageTexture(encoder, targetTexture, i);- device.queue.submit([encoder.finish()]);- stagingRing.scheduleMap(i);- inFlightWorkerFrames.push(i);- }-- while (inFlightWorkerFrames.length > 0) {- const remainingFrame = inFlightWorkerFrames.shift();- if (remainingFrame !== undefined) {- await consumeWorkerFrame(remainingFrame);- }- }-- if (outputFormat !== "png") {- parentPort!.postMessage({- type: "done",- renderId,- } satisfies WorkerMessage);- }- } finally {- if (onTaskMessage) {- try {- parentPort!.off("message", onTaskMessage);- } catch {}- }- if (lutSub) {- try {- lutSub();- } catch {}- }- try {- mediaDecoderCache.destroy();- clearAllVideoCache();- } catch {}- try {- shaderStore.clear(renderId);- } catch {}- try {- device.destroy();- } catch (e) {}- try {- resetDeviceInstance();- } catch (e) {}- try {- renderer.destroy();- } catch (e) {}- try {- surface.destroy();- } catch (e) {}- try {- stagingRing.destroy();- } catch (e) {}- try {- textureCache.destroy();- } catch (e) {}- }- } catch (error) {- parentPort!.postMessage({- type: "error",- renderId: task?.input?.renderId,- error:- error instanceof Error- ? (error.stack ?? error.message)- : String(error),- } satisfies WorkerMessage);- }- })();- });-}diff --git a/packages/renderer/tsdown.config.ts b/packages/renderer/tsdown.config.tsindex 23157ae1..639f19a2 100644--- a/packages/renderer/tsdown.config.ts+++ b/packages/renderer/tsdown.config.ts@@ -1,12 +1,7 @@import { defineConfig } from "tsdown";export default defineConfig({- entry: [- "src/index.ts",- "src/test-runner.ts",- "src/offscreen-canvas.ts",- "src/video-worker.ts",- ],+ entry: ["src/index.ts", "src/test-runner.ts", "src/offscreen-canvas.ts"],format: ["esm"],dts: true,clean: true,diff --git a/plugins/framefields/skills/framefields-render/SKILL.md b/plugins/framefields/skills/framefields-render/SKILL.mdindex 82dc08b2..04e84993 100644--- a/plugins/framefields/skills/framefields-render/SKILL.md+++ b/plugins/framefields/skills/framefields-render/SKILL.md@@ -233,6 +233,29 @@ tsx src/render.ts previewtsx src/render.ts```+### Reading render progress++`renderVideo` logs its own progress to stdout, one JSON line per event (the text is in `msg`). The log stays short however long the film is, about 15 lines for an export:++```text+Preparing 1800 frames (60.0s) at 1920x1080, 30 fps, h264+Rendering 1800 frames+Rendered 10% (180/1800 frames) · 41.2 fps · eta 39.3s+…+Rendered 100% (1800/1800 frames) · 41.8 fps · in 43.1s+Mixing audio+Encoding 60.0s of audio (aac)+Muxing mp4+Done in 51.4s (14.2 MB): prepare 1.2s · render 43.1s · audio 3.2s · audio-encode 1.1s · mux 0.8s+```++- **Work continues after the last frame.** "Mixing audio" decodes every audio and video clip in the film, then the soundtrack is encoded and "Muxing" writes the file. With many or long clips this takes a while. It is not a hang; wait for `Done in …`.+- **A quiet phase prints `Still …` lines**, e.g. `Still mixing audio (42%, 3/7 clips: music.mp3), 1m 10s in this phase`: after 10 s, 30 s, 70 s, then backing off to one every 5 min. Each says which phase is running and for how long. A render is stuck only when no line arrives for well over the last gap. Killing it at any point leaves no video.+- **A failure names its phase:** `Failed while mixing audio after 12.3s`, with the error.+- **Run long exports in the background** with the longest timeout your harness allows, and read the output when it finishes rather than polling it.+- `LOG_LEVEL=warn` silences these lines; `LOG_LEVEL=debug` adds per-stage detail.+- For your own progress output, `onProgress` gets every event as `{ phase, frame, totalFrames, elapsedMs, phaseProgress?, fps?, etaSec?, detail? }`. `phase` runs `prepare`, `render` (audio-only output skips it), `audio`, `audio-encode`, `mux`, `convert` (GIF only), `done`.+---## Built-in Video QA (`renderVideo({ qa })`)@@ -302,7 +325,8 @@ ffmpeg -i output/final-video.mp4 -af "volumedetect" -f null -| Audio cuts out early | Missing `durationFrames` on audio layer | Explicitly assign `durationFrames: totalFrames` on `Layer.audio()`. || Vision cutout / mask is empty on a single `renderFrame` | Vision reads the previous rendered frame; frame 0 has none | Render two or more consecutive frames with the same `HeadlessMediaRenderer` and keep the last. `renderVideo` handles this automatically. || Ghosted double of the subject in a vision cutout | Frames rendered out of order (frame grid, or jumping from frame 3 to 48), so the cutout uses pixels from another frame | Verify vision layers with sequential frames or the exported MP4, not `renderFrameGrid`. |-| Script hangs after rendering | Background worker thread pool held open | Always include `process.exit(0)` at the end of render scripts. |+| Output stops after `Rendered 100%` | Audio is being mixed, encoded or muxed | Not a hang: wait for `Still …` lines and `Done in …` (see **Reading render progress**). |+| Script hangs after `Done in …` | Background worker thread pool held open | Always include `process.exit(0)` at the end of render scripts. || Render very slow on a Linux server | Frames drawn on a CPU adapter (lavapipe) | `await describeGpu()` shows `software: true`; fix the GPU's Vulkan driver (`vulkaninfo --summary`). || Encoder fails to open (`h264_rkmpp`, NVENC) | framefields < 2.2 uses the hardware encoder untested | Use framefields 2.2+, which falls back to libx264 etc., or set `FRAMEFIELDS_HW_CODECS=off`. || Render worker job fails | Project not render-ready (lockfile, export, LFS) | Read the job's `code` and `hint`; checklist at https://framefields.dev/docs/guides/render-service/ |diff --git a/plugins/framefields/skills/framefields/SKILL.md b/plugins/framefields/skills/framefields/SKILL.mdindex dfbf7b5e..c7a1654e 100644--- a/plugins/framefields/skills/framefields/SKILL.md+++ b/plugins/framefields/skills/framefields/SKILL.md@@ -255,7 +255,7 @@ Never go straight to a full video render. On a machine without a GPU every rende2. `npm run frames -- 0 30 60` and look at the PNGs in `output/`. A black frame usually means a missing font, a layer outside the canvas, or opacity stuck at 0.3. `npm run grid` and check the motion reads smoothly across the contact sheet.4. `npm run preview` when a person is watching: they play the film with sound before you export (see below).-5. `npm run render` once frames and motion look right.+5. `npm run render` once frames and motion look right. Run it in the background with a long timeout. It logs each phase (`Rendered 40% …`, `Mixing audio`, `Muxing mp4`, `Done in …`) and a `Still …` line when one runs long, so a pause after the last frame is the audio mix or mux, not a hang.Show the user the frame PNGs or the grid before the full render. `framefields-render` covers pixel-probe tests, render QA (`renderVideo({ qa })`), and diagnosing blank frames. If you suspect an issue or defect in the engine itself or ai gateway, see [section 9](#9-report-engine-issues).