Code-first video & image, rendered natively on WebGPU.
import { exec, spawn } from "node:child_process";
import { randomUUID } from "node:crypto";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import {
type AudioMix,
compositionStateStore,
drawCompositionTree,
mixAudioTracks,
preflightMediaTime,
} from "@framefields/compositions";
import {
frameCount,
getMediaType,
resolveFrameRequest,
takeRenderDiagnostics,
updateClockSignals,
type VirtualMediaData,
} from "@framefields/core";
import {
clearAllVideoCache,
ensureDevice,
getRenderer2D,
initHeadlessWebGPU,
inputStore,
lutStore,
mediaDecoderCache,
NodeSurfaceProvider,
type RenderContextValue,
type Renderer2D,
SlugFontCache,
shaderStore,
textureCache,
WebGPUAudioProcessor,
} from "@framefields/webgpu-renderers";
import {
AudioSample,
AudioSampleSource,
FilePathTarget,
Mp3OutputFormat,
Mp4OutputFormat,
Output,
QUALITY_HIGH,
QUALITY_LOW,
QUALITY_VERY_HIGH,
QUALITY_VERY_LOW,
VideoSampleSource,
WebMOutputFormat,
} from "mediabunny";
import sharp from "sharp";
import { Canvas as SkiaCanvas, Image as SkiaImage } from "skia-canvas";
import { preloadFonts } from "./asset-preloader.js";
import { bootstrapMediabunny } from "./bootstrap-mediabunny.js";
import { discoverAndRegisterNodeRenderers } from "./dynamic-node-discovery.js";
import { rendererLogger } from "./logger.js";
import { type RenderProgress, RenderReporter } from "./render-reporter.js";
// Vision reports its inference backend once per process (see
// VISION_BACKEND_LISTENER in @framefields/vision).
(globalThis as Record<symbol, unknown>)[
Symbol.for("framefields.vision.backend")
] = (info: { backend: string; sharedDevice: boolean; reason?: string }) => {
rendererLogger.info(
info,
info.backend === "webgpu"
? "[vision] Inference on the GPU (WebGPU)"
: `[vision] Inference on the ${info.backend.toUpperCase()}`,
);
};
import type { FrameCostSample } from "./chunk-plan.js";
import {
type AudioQaStats,
analyzeAudio,
buildQaReport,
FrameInspector,
type VideoQaOptions,
type VideoQaReport,
type VideoQaStats,
} from "./video-qa.js";
import { ZeroCopyWebCodecsPipeline } from "./zero-copy-webcodecs-pipeline.js";
async function convertMp4ToGif(
inputMp4Path: string,
outputGifPath: string,
): Promise<void> {
return new Promise<void>((resolve, reject) => {
const ffmpeg = spawn("ffmpeg", [
"-i",
inputMp4Path,
"-vf",
"split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse",
"-y",
outputGifPath,
]);
let errorOutput = "";
ffmpeg.stderr.on("data", (data) => {
errorOutput += data.toString();
});
ffmpeg.on("close", async (code) => {
if (code === 0) {
resolve();
} else {
rendererLogger.error(
{ code, err: errorOutput },
"[convertMp4ToGif] ffmpeg conversion failed",
);
reject(new Error(`ffmpeg exited with code ${code}: ${errorOutput}`));
}
});
ffmpeg.on("error", (err) => {
rendererLogger.error({ err }, "[convertMp4ToGif] ffmpeg spawn error");
reject(err);
});
});
}
// ─── Bootstrap ─────────────────────────────────────────────────────────────────
bootstrapMediabunny();
const globalObj = globalThis as unknown as Record<string, unknown>;
if (typeof globalThis !== "undefined") {
globalObj.__IS_HEADLESS_RENDERER__ = true;
globalObj.__GATEWAI_DELAYS__ ??= new Set<number>();
}
if (typeof globalThis.requestAnimationFrame === "undefined") {
globalObj.requestAnimationFrame = (cb: () => void) => setTimeout(cb, 0);
globalObj.cancelAnimationFrame = (id: number) => clearTimeout(id);
}
const dummyDomElement = {
clientWidth: 1920,
clientHeight: 1080,
getBoundingClientRect: () => ({
x: 0,
y: 0,
left: 0,
top: 0,
right: 1920,
bottom: 1080,
width: 1920,
height: 1080,
}),
style: {},
appendChild: () => {},
removeChild: () => {},
addEventListener: () => {},
removeEventListener: () => {},
getAttribute: () => null,
};
if (typeof globalThis.OffscreenCanvas === "undefined") {
(globalObj.OffscreenCanvas as unknown) = SkiaCanvas;
}
if (typeof globalThis.HTMLCanvasElement === "undefined") {
(globalObj.HTMLCanvasElement as unknown) = SkiaCanvas;
}
if (typeof globalThis.Image === "undefined") {
(globalObj.Image as unknown) = SkiaImage;
}
if (typeof globalThis.window === "undefined") {
(globalObj.window as unknown) = globalThis;
}
if (!globalThis.location) {
(globalObj.location as unknown) = new URL("http://localhost");
}
if (typeof globalThis.IntersectionObserver === "undefined") {
(globalObj.IntersectionObserver as unknown) = class IntersectionObserver {
observe() {}
unobserve() {}
disconnect() {}
};
}
if (typeof globalThis.document === "undefined") {
(globalObj.document as unknown) = {
createElement: (tag: string) => {
if (tag === "canvas") return new SkiaCanvas(1, 1);
return { ...dummyDomElement };
},
getElementsByTagName: () => [],
documentElement: dummyDomElement,
body: dummyDomElement,
};
} else {
const docObj = globalThis.document as unknown as Record<string, unknown>;
if (typeof docObj.getElementsByTagName === "undefined") {
docObj.getElementsByTagName = () => [];
}
docObj.documentElement ??= dummyDomElement;
docObj.body ??= dummyDomElement;
}
const skiaProto = SkiaCanvas.prototype as unknown as Record<string, unknown>;
if (typeof skiaProto.getBoundingClientRect === "undefined") {
skiaProto.getBoundingClientRect = function (this: {
width?: number;
height?: number;
}) {
const width = this.width || 512;
const height = this.height || 512;
return {
x: 0,
y: 0,
left: 0,
top: 0,
right: width,
bottom: height,
width,
height,
};
};
}
if (!("style" in skiaProto)) {
Object.defineProperty(skiaProto, "style", {
get(this: { _style?: Record<string, unknown> }) {
this._style ??= {};
return this._style;
},
configurable: true,
});
}
if (!("clientWidth" in skiaProto)) {
Object.defineProperty(skiaProto, "clientWidth", {
get(this: { width?: number }) {
return this.width || 512;
},
configurable: true,
});
}
if (!("clientHeight" in skiaProto)) {
Object.defineProperty(skiaProto, "clientHeight", {
get(this: { height?: number }) {
return this.height || 512;
},
configurable: true,
});
}
if (!("parentElement" in skiaProto)) {
Object.defineProperty(skiaProto, "parentElement", {
get() {
return dummyDomElement;
},
configurable: true,
});
}
if (typeof skiaProto.addEventListener === "undefined") {
skiaProto.addEventListener = () => {};
}
if (typeof skiaProto.removeEventListener === "undefined") {
skiaProto.removeEventListener = () => {};
}
if (typeof skiaProto.getAttribute === "undefined") {
skiaProto.getAttribute = () => null;
}
// ─── Concurrency Limiter ───────────────────────────────────────────────────────
const CPU_COUNT = os.cpus().length;
const TOTAL_WORKER_BUDGET = Math.max(1, CPU_COUNT - 3);
// Balance concurrency (renders processed in parallel) and internal parallelism (workers per render).
// For rendering stability on headless instances, we limit MAX_CONCURRENT_RENDERS to 1-3.
const MAX_CONCURRENT_RENDERS = Math.max(
1,
Math.min(3, Math.floor(TOTAL_WORKER_BUDGET / 3)),
);
const QUEUE_TIMEOUT_MS = 2 * 60 * 60_000; // 2 hours
class RenderSemaphore {
private _running = 0;
private readonly _max: number;
private readonly _queue: Array<{
resolve: () => void;
reject: (err: Error) => void;
timer: ReturnType<typeof setTimeout>;
}> = [];
constructor(max: number) {
this._max = max;
}
acquire(timeoutMs = QUEUE_TIMEOUT_MS): Promise<void> {
if (this._running < this._max) {
this._running++;
return Promise.resolve();
}
return new Promise<void>((resolve, reject) => {
const timer = setTimeout(() => {
const idx = this._queue.findIndex((e) => e.resolve === resolve);
if (idx !== -1) this._queue.splice(idx, 1);
reject(
new Error(
`Render queue timeout after ${timeoutMs}ms ` +
`(${this._queue.length} queued, ${this._running}/${this._max} active)`,
),
);
}, timeoutMs);
this._queue.push({ resolve, reject, timer });
});
}
release(): void {
if (this._queue.length > 0) {
const next = this._queue.shift()!;
clearTimeout(next.timer);
next.resolve();
} else {
this._running--;
}
}
get stats() {
return {
active: this._running,
queued: this._queue.length,
max: this._max,
};
}
}
export const renderSemaphore = new RenderSemaphore(MAX_CONCURRENT_RENDERS);
/**
* Ids for still renders without a caller id. Media caches (video decoders,
* last-good frames) are keyed by render id, so a fresh id per still would
* open a new decoder for every frame of a frame-by-frame export. Stills
* borrow an idle id instead: consecutive stills reuse it (and its decoders);
* concurrent stills get distinct ones.
*/
const idleStillIds: string[] = [];
const takeStillId = (): string => idleStillIds.pop() ?? `img-${randomUUID()}`;
const returnStillId = (id: string): void => {
idleStillIds.push(id);
};
// ─── Helpers ───────────────────────────────────────────────────────────────────
async function fileSize(file: string): Promise<number | undefined> {
try {
return (await fs.stat(file)).size;
} catch {
return undefined;
}
}
/** Composition frames `start` (inclusive) to `end` (exclusive). */
export interface FrameRange {
start: number;
end: number;
}
/** The frames a render covers, checked against the composition's length. */
function resolveFrameRange(
frames: FrameRange | undefined,
total: number,
): FrameRange & { total: number } {
if (!frames) return { start: 0, end: total, total };
const { start, end } = frames;
if (
!Number.isInteger(start) ||
!Number.isInteger(end) ||
start < 0 ||
end <= start ||
end > total
)
throw new Error(
`Invalid frames { start: ${start}, end: ${end} }: need integers with 0 <= start < end <= ${total} (the composition's frame count).`,
);
return { start, end, total };
}
/**
* Mixes the composition's whole audio timeline and encodes it into `source`
* in 100 ms samples. Returns its QA stats when `qaOptions` is set.
*/
async function encodeAudioTrack(job: {
virtualMedia: VirtualMediaData;
fps: number;
device: GPUDevice;
renderId: string;
source: AudioSampleSource;
codec: string;
reporter: RenderReporter;
qaOptions?: VideoQaOptions;
}): Promise<AudioQaStats | undefined> {
const { reporter } = job;
reporter.stage("audio", "Mixing audio");
const { channels, sampleRate } = await mixAudioTracks(
job.virtualMedia,
job.fps,
48_000,
job.device,
job.renderId,
{
onProgress: ({ done, total, source }) =>
reporter.update(
total > 0
? `${done}/${total} clips${source ? `: ${source}` : ""}`
: "no audio clips",
total > 0 ? done / total : undefined,
),
},
);
const numChannels = channels.length;
if (numChannels === 0) throw new Error("mixAudioTracks returned no channels");
const totalSamples = channels[0].length;
const chunkSize = Math.round(0.1 * sampleRate);
reporter.stage(
"audio-encode",
`Encoding ${(totalSamples / sampleRate).toFixed(1)}s of audio (${job.codec})`,
);
for (let offset = 0; offset < totalSamples; offset += chunkSize) {
const currentChunk = Math.min(chunkSize, totalSamples - offset);
// A fresh buffer per chunk: the encoder may read a sample's data after
// add() resolves, so a buffer reused across chunks gets overwritten by
// the next chunk before it is encoded (audible as 100 ms garbling).
const interleaved = new Float32Array(currentChunk * numChannels);
for (let i = 0; i < currentChunk; i++) {
for (let ch = 0; ch < numChannels; ch++) {
interleaved[i * numChannels + ch] = channels[ch]?.[offset + i] ?? 0;
}
}
const sample = new AudioSample({
data: interleaved,
format: "f32",
numberOfChannels: numChannels,
sampleRate,
timestamp: offset / sampleRate,
});
try {
await job.source.add(sample);
} finally {
sample.close();
}
const written = offset + currentChunk;
reporter.update(
`${(written / sampleRate).toFixed(1)}s/${(totalSamples / sampleRate).toFixed(1)}s`,
written / totalSamples,
);
}
return job.qaOptions
? analyzeAudio(channels, sampleRate, job.qaOptions)
: undefined;
}
function toEvenDimension(n: number): number {
const v = Math.round(n);
return v % 2 === 0 ? v : v + 1;
}
function resolveQuality(quality?: string) {
switch (quality) {
case "very_low":
return QUALITY_VERY_LOW;
case "low":
return QUALITY_LOW;
case "high":
return QUALITY_HIGH;
case "very_high":
return QUALITY_VERY_HIGH;
default:
return QUALITY_VERY_HIGH;
}
}
// ─── VRAM telemetry ────────────────────────────────────────────────────────────
//
// nvidia-smi is probed asynchronously and cached. The previous implementation
// ran `execSync` from inside the 5s stats interval, which blocked the event
// loop — and therefore the frame loop and DMA map callbacks — for tens of ms on
// exactly the NVIDIA hosts that render the most. `getVramUsage()` now returns
// the latest sample; `refreshVramUsage()` refreshes it off the event loop.
let cachedVramUsage = "N/A";
let vramProbeInFlight = false;
function refreshVramUsage(): void {
if (vramProbeInFlight) return;
vramProbeInFlight = true;
exec(
"nvidia-smi --query-gpu=memory.used,memory.total --format=csv,noheader,nounits",
{ encoding: "utf-8", timeout: 2000 },
(err, stdout) => {
vramProbeInFlight = false;
if (err || !stdout) return;
const [used, total] = stdout
.trim()
.split(",")
.map((s) => s.trim());
if (used && total) {
cachedVramUsage = `${used}MB / ${total}MB`;
}
},
);
}
function getVramUsage(): string {
return cachedVramUsage;
}
// ─── DMA staging ring depth ────────────────────────────────────────────────────
//
// The ring overlaps GPU rendering of frame i+1 with the DMA readback and
// hardware encode of frame i. A deeper ring hides more encoder variance at the
// cost of VRAM (~8.3MB/slot at 1080p RGBA). Default stays at 2 so behavior is
// unchanged unless a caller or the environment opts in.
const DEFAULT_RING_CAPACITY = 2;
const MIN_RING_CAPACITY = 2;
const MAX_RING_CAPACITY = 8;
function resolveRingCapacity(explicit?: number): number {
const fromEnv = Number(process.env.FRAMEFIELDS_RING_CAPACITY);
const requested =
explicit ?? (Number.isFinite(fromEnv) && fromEnv > 0 ? fromEnv : NaN);
if (!Number.isFinite(requested) || requested <= 0) {
return DEFAULT_RING_CAPACITY;
}
return Math.max(
MIN_RING_CAPACITY,
Math.min(MAX_RING_CAPACITY, Math.floor(requested)),
);
}
// ─── Reusable render surface pool ──────────────────────────────────────────────
//
// `renderImage` is called once per still frame (frame-by-frame exports,
// scrubbing, the renderer-service image path). Allocating and destroying the
// native render target + readback staging buffer on every call is pure overhead:
// the surface is cleared at the start of each frame, so it can be reused.
// Surfaces are only checked out while the render semaphore is held, so two
// concurrent renders never share one.
class NodeSurfacePool {
private readonly idle: Array<{
device: GPUDevice;
surface: NodeSurfaceProvider;
}> = [];
private readonly maxIdle = 2;
acquire(
device: GPUDevice,
width: number,
height: number,
): NodeSurfaceProvider {
const index = this.idle.findIndex(
(entry) =>
entry.device === device &&
entry.surface.width === width &&
entry.surface.height === height,
);
if (index >= 0) {
const [entry] = this.idle.splice(index, 1);
if (entry) return entry.surface;
}
// Drop any surfaces belonging to a previous device (it was lost and
// `ensureDevice` created a new one); they are no longer usable.
for (let i = this.idle.length - 1; i >= 0; i--) {
const entry = this.idle[i];
if (entry && entry.device !== device) {
this.idle.splice(i, 1);
try {
entry.surface.destroy();
} catch {}
}
}
return new NodeSurfaceProvider(device, width, height);
}
release(device: GPUDevice, surface: NodeSurfaceProvider): void {
while (this.idle.length >= this.maxIdle) {
const evicted = this.idle.shift();
try {
evicted?.surface.destroy();
} catch {}
}
this.idle.push({ device, surface });
}
destroy(): void {
for (const entry of this.idle.splice(0)) {
try {
entry.surface.destroy();
} catch {}
}
}
}
const nodeSurfacePool = new NodeSurfacePool();
function formatLutAsCube(lut: {
points: Array<[number, number, number]>;
size: number;
type: "1D" | "3D";
}): string {
const lines = [
"# Created by Gatewai",
lut.type === "3D" ? `LUT_3D_SIZE ${lut.size}` : `LUT_1D_SIZE ${lut.size}`,
"",
];
for (const p of lut.points) {
lines.push(`${p[0].toFixed(6)} ${p[1].toFixed(6)} ${p[2].toFixed(6)}`);
}
return lines.join("\n");
}
// ─── HeadlessWebGPURenderer ───────────────────────────────────────────────────
export class HeadlessWebGPURenderer {
private static isInitialized = false;
private static initPromise: Promise<void> | null = null;
private static initFailureCount = 0;
private static readonly MAX_INIT_FAILURES = 3;
private static nextRetryAt = 0;
public static initialize(): Promise<void> {
if (HeadlessWebGPURenderer.isInitialized) return Promise.resolve();
if (
HeadlessWebGPURenderer.initFailureCount >=
HeadlessWebGPURenderer.MAX_INIT_FAILURES
) {
return Promise.reject(
new Error(
"GPU initialization permanently failed — circuit breaker open",
),
);
}
if (Date.now() < HeadlessWebGPURenderer.nextRetryAt) {
return Promise.reject(new Error("GPU initialization in backoff period"));
}
if (HeadlessWebGPURenderer.initPromise) {
return HeadlessWebGPURenderer.initPromise;
}
HeadlessWebGPURenderer.initPromise = (async () => {
try {
rendererLogger.debug(
"[HeadlessWebGPURenderer] Initializing headless WebGPU…",
);
await initHeadlessWebGPU();
rendererLogger.debug(
"[HeadlessWebGPURenderer] Scanning node renderers…",
);
await discoverAndRegisterNodeRenderers();
HeadlessWebGPURenderer.isInitialized = true;
HeadlessWebGPURenderer.initPromise = null;
HeadlessWebGPURenderer.initFailureCount = 0;
HeadlessWebGPURenderer.nextRetryAt = 0;
rendererLogger.debug(
"[HeadlessWebGPURenderer] Initialization complete.",
);
} catch (error) {
HeadlessWebGPURenderer.initFailureCount++;
HeadlessWebGPURenderer.nextRetryAt =
Date.now() + 5000 * 2 ** HeadlessWebGPURenderer.initFailureCount;
HeadlessWebGPURenderer.initPromise = null; // allow retry
rendererLogger.error(
{
err: error instanceof Error ? error.message : String(error),
failureCount: HeadlessWebGPURenderer.initFailureCount,
nextRetryAt: new Date(
HeadlessWebGPURenderer.nextRetryAt,
).toISOString(),
},
"[HeadlessWebGPURenderer] Initialization failed",
);
throw error;
}
})();
return HeadlessWebGPURenderer.initPromise;
}
public async renderImage(
virtualMedia: VirtualMediaData,
frame = 0,
fps = 24,
options: {
renderId?: string;
/** "rgba" / "raw" skips image encoding and returns width*height*4 raw pixels. */
format?: "png" | "rgba" | "raw" | "jpeg" | "jpg" | "webp";
} = {},
): Promise<Buffer> {
await HeadlessWebGPURenderer.initialize();
await preflightMediaTime(virtualMedia, fps);
const width = toEvenDimension(virtualMedia.metadata.width ?? 0);
const height = toEvenDimension(virtualMedia.metadata.height ?? 0);
if (!width || !height)
throw new Error(`Invalid dimensions: ${width}x${height}`);
// A caller rendering consecutive frames (e.g. a scrubbing preview) passes a
// stable id so compiled timelines are reused across frames, as renderVideo
// does. Never render two frames with the same id concurrently.
const pooledId = options.renderId ? undefined : takeStillId();
const renderId = options.renderId ?? (pooledId as string);
rendererLogger.debug(
{ renderId, frame, ...renderSemaphore.stats },
"[HeadlessWebGPURenderer] Acquiring slot for image render",
);
await renderSemaphore.acquire();
let surface: NodeSurfaceProvider | undefined;
let renderer: Renderer2D | undefined;
const device = await ensureDevice();
try {
await preloadFonts(virtualMedia, device);
surface = nodeSurfacePool.acquire(device, width, height);
renderer = getRenderer2D(device, surface.colorFormat);
const ctx: RenderContextValue = { device, renderer, surface };
updateClockSignals(
frame,
fps,
virtualMedia.metadata.durationMs ?? undefined,
);
const needsPrePass =
frame === 0 ||
lutStore.hasPending(device) ||
SlugFontCache.hasPending();
if (needsPrePass) {
const preEncoder = device.createCommandEncoder();
const dummyTex = renderer.getTemporaryTexture(width, height);
const dummyView = dummyTex.createView();
const preClear = renderer.beginFrame(
preEncoder,
dummyView,
{ r: 0, g: 0, b: 0, a: 0 },
width,
height,
"clear",
);
preClear.end();
await drawCompositionTree(
ctx,
preEncoder,
dummyView,
dummyTex,
width,
height,
virtualMedia,
{
frame,
compositionFrame: frame,
fps,
isHeadless: true,
renderId,
virtualMedia,
containerWidth: width,
containerHeight: height,
isVideoMode: virtualMedia.operation?.dataType === "Video",
},
);
device.queue.submit([preEncoder.finish()]);
await lutStore.awaitAllPending(device);
}
compositionStateStore.setState(renderId, frame, fps, true);
const encoder = device.createCommandEncoder();
const targetView = surface.getCurrentTextureView();
const targetTexture = surface.getCurrentTexture();
const clearPass = renderer.beginFrame(
encoder,
targetView,
{ r: 0, g: 0, b: 0, a: 0 },
width,
height,
"clear",
);
clearPass.end();
await drawCompositionTree(
ctx,
encoder,
targetView,
targetTexture,
width,
height,
virtualMedia,
{
frame,
compositionFrame: frame,
fps,
isHeadless: true,
renderId,
virtualMedia,
containerWidth: width,
containerHeight: height,
isVideoMode: virtualMedia.operation?.dataType === "Video",
},
);
device.queue.submit([encoder.finish()]);
const pixels = await surface.readPixels();
const pixelsArr = new Uint8Array(
pixels.buffer,
pixels.byteOffset,
pixels.byteLength,
);
if (options.format === "rgba" || options.format === "raw")
return Buffer.from(pixelsArr);
const img = sharp(Buffer.from(pixelsArr), {
raw: { width, height, channels: 4 },
});
if (options.format === "jpeg" || options.format === "jpg") {
return img.jpeg().toBuffer();
}
if (options.format === "webp") {
return img.webp().toBuffer();
}
return img.png().toBuffer();
} finally {
try {
clearAllVideoCache();
} catch {}
try {
shaderStore.clear(renderId);
} catch {}
// Keep this render's decoders (and the singleton WebGPU device) alive:
// renderImage is called repeatedly for multi-frame exports, and the next
// still borrows the same id, so it reuses them and seeks. Decoders no
// still has used for a while are released by the per-frame tick.
try {
mediaDecoderCache.tick();
} catch {}
if (pooledId) returnStillId(pooledId);
// The render surface (target texture + readback buffer) returns to the
// pool instead of being destroyed, so repeated stills skip the native
// allocation. Cached media textures are still released to bound VRAM.
if (surface) nodeSurfacePool.release(device, surface);
try {
textureCache.destroy();
} catch {}
renderSemaphore.release();
rendererLogger.debug(
{ renderId, ...renderSemaphore.stats },
"[HeadlessWebGPURenderer] Image render slot released",
);
}
}
/**
* Renders a single frame to a PNG buffer at a specified timestamp in milliseconds or frame index.
*/
public async renderFrame(
source: VirtualMediaData | { toVirtualMedia(): VirtualMediaData },
options: {
atMs?: number;
frame?: number;
fps?: number;
/** Reuse compiled state across calls; see `renderImage`. */
renderId?: string;
/** "rgba" / "raw" returns raw width*height*4 pixels instead of an encoded image. */
format?: "png" | "rgba" | "raw" | "jpeg" | "jpg" | "webp";
} = {},
): Promise<Buffer> {
const vm =
typeof source === "object" &&
source !== null &&
"toVirtualMedia" in source
? (source as { toVirtualMedia(): VirtualMediaData }).toVirtualMedia()
: (source as VirtualMediaData);
const fps = options.fps ?? (vm.metadata?.fps || 30);
const { frame } = resolveFrameRequest(options, fps);
return this.renderImage(vm, frame, fps, {
renderId: options.renderId,
format: options.format,
});
}
/**
* Measures what frames cost to render across the timeline, for planning a
* split render (`planChunks`). At `samples` evenly spaced points (default
* 12) it renders three consecutive frames and times the last two, so
* effects that carry state (trails, particles) are caught up and their
* replay is not counted. Encoding is not included.
*/
public async probeRender(
source: VirtualMediaData | { toVirtualMedia(): VirtualMediaData },
options: {
samples?: number;
signal?: AbortSignal;
/** Called after each sample point with how many are done. */
onSample?: (done: number, total: number) => void;
} = {},
): Promise<{
totalFrames: number;
fps: number;
durationMs: number;
samples: FrameCostSample[];
}> {
const vm =
"toVirtualMedia" in source
? (source as { toVirtualMedia(): VirtualMediaData }).toVirtualMedia()
: source;
const fps = vm.metadata.fps;
if (!fps) throw new Error("probeRender: the composition has no fps.");
let durationMs = vm.metadata.durationMs;
if (!durationMs || durationMs <= 0) {
durationMs = await this.resolveMediaDuration(vm, fps);
vm.metadata.durationMs = durationMs;
}
const totalFrames = frameCount(durationMs, fps);
const count = Math.max(
1,
Math.min(Math.floor(options.samples ?? 12), totalFrames),
);
const last = Math.max(0, totalFrames - 3);
const renderId = `probe-${randomUUID()}`;
const render = (frame: number) =>
this.renderImage(vm, Math.min(frame, totalFrames - 1), fps, {
renderId,
format: "rgba",
});
const samples: FrameCostSample[] = [];
try {
for (let i = 0; i < count; i++) {
options.signal?.throwIfAborted();
const at = count === 1 ? 0 : Math.round((i * last) / (count - 1));
await render(at);
const started = performance.now();
await render(at + 1);
await render(at + 2);
samples.push({
frame: Math.min(at + 1, totalFrames - 1),
ms: (performance.now() - started) / 2,
});
options.onSample?.(i + 1, count);
}
} finally {
mediaDecoderCache.clearNode(renderId);
}
return { totalFrames, fps, durationMs, samples };
}
/**
* Mixes the composition's full audio timeline (clips, effects, synthesized
* tracks) to stereo PCM, exactly as `renderVideo` muxes it.
*/
/**
* Mixes the timeline's audio. `window` mixes only that stretch (the same
* samples as that part of a full mix); `master: false` skips the master
* stage, for callers that master with the whole film's `peak` later.
*/
public async renderAudio(
virtualMedia: VirtualMediaData,
options: {
fps?: number;
sampleRate?: number;
window?: { startSec: number; endSec: number };
master?: boolean;
} = {},
): Promise<AudioMix> {
await HeadlessWebGPURenderer.initialize();
const fps = options.fps ?? (virtualMedia.metadata?.fps || 30);
await preflightMediaTime(virtualMedia, fps);
await renderSemaphore.acquire();
try {
const device = await ensureDevice();
const renderId = `aud-${randomUUID()}`;
try {
return await mixAudioTracks(
virtualMedia,
fps,
options.sampleRate ?? 48_000,
device,
renderId,
{ window: options.window, master: options.master },
);
} finally {
// The mix's effect state (delay lines, reverb tails) on the GPU.
WebGPUAudioProcessor.clearCache(renderId, device);
}
} finally {
renderSemaphore.release();
}
}
public async renderLut(
virtualMedia: VirtualMediaData,
frame = 0,
fps = 24,
): Promise<Buffer> {
await HeadlessWebGPURenderer.initialize();
const width = 32;
const height = 32;
const renderId = `lut-${randomUUID()}`;
rendererLogger.debug(
{ renderId, frame, ...renderSemaphore.stats },
"[HeadlessWebGPURenderer] Acquiring slot for LUT render",
);
await renderSemaphore.acquire();
let surface: NodeSurfaceProvider | undefined;
let renderer: Renderer2D | undefined;
const device = await ensureDevice();
let lutData: any = null;
const lutSub = lutStore.onChange((key) => {
const raw = lutStore.getRawData(key);
if (raw) {
lutData = raw;
}
});
try {
await preloadFonts(virtualMedia, device);
surface = new NodeSurfaceProvider(device, width, height);
renderer = getRenderer2D(device, surface.colorFormat);
const ctx: RenderContextValue = { device, renderer, surface };
const needsPrePass =
frame === 0 ||
lutStore.hasPending(device) ||
SlugFontCache.hasPending();
if (needsPrePass) {
const preEncoder = device.createCommandEncoder();
const dummyTex = renderer.getTemporaryTexture(width, height);
const dummyView = dummyTex.createView();
const preClear = renderer.beginFrame(
preEncoder,
dummyView,
{ r: 0, g: 0, b: 0, a: 0 },
width,
height,
"clear",
);
preClear.end();
await drawCompositionTree(
ctx,
preEncoder,
dummyView,
dummyTex,
width,
height,
virtualMedia,
{
frame,
compositionFrame: frame,
fps,
isHeadless: true,
renderId,
virtualMedia,
containerWidth: width,
containerHeight: height,
isVideoMode: virtualMedia.operation?.dataType === "Video",
},
);
device.queue.submit([preEncoder.finish()]);
await lutStore.awaitAllPending(device);
}
compositionStateStore.setState(renderId, frame, fps, true);
const encoder = device.createCommandEncoder();
const targetView = surface.getCurrentTextureView();
const targetTexture = surface.getCurrentTexture();
const clearPass = renderer.beginFrame(
encoder,
targetView,
{ r: 0, g: 0, b: 0, a: 0 },
width,
height,
"clear",
);
clearPass.end();
await drawCompositionTree(
ctx,
encoder,
targetView,
targetTexture,
width,
height,
virtualMedia,
{
frame,
compositionFrame: frame,
fps,
isHeadless: true,
renderId,
virtualMedia,
containerWidth: width,
containerHeight: height,
isVideoMode: virtualMedia.operation?.dataType === "Video",
},
);
device.queue.submit([encoder.finish()]);
await lutStore.awaitAllPending(device);
if (!lutData) {
lutData = lutStore.getAnyRawData();
}
if (!lutData) {
throw new Error("No LUT data generated during rendering");
}
const cubeContent = formatLutAsCube(lutData);
return Buffer.from(cubeContent, "utf-8");
} finally {
try {
lutSub();
} catch {}
try {
// Only this render's decoders: another render may be running.
mediaDecoderCache.clearNode(renderId);
clearAllVideoCache();
} catch {}
try {
shaderStore.clear(renderId);
} catch {}
try {
surface?.destroy();
} catch {}
try {
textureCache.destroy();
} catch {}
renderSemaphore.release();
rendererLogger.debug(
{ renderId, ...renderSemaphore.stats },
"[HeadlessWebGPURenderer] LUT render slot released",
);
}
}
private async resolveMediaDuration(
virtualMedia: VirtualMediaData,
fps: number,
): Promise<number> {
let maxDurationMs = 0;
const queue: VirtualMediaData[] = [virtualMedia];
const sources = new Set<string>();
while (queue.length > 0) {
const current = queue.shift()!;
if (
current.metadata?.durationMs &&
current.metadata.durationMs > maxDurationMs
) {
maxDurationMs = current.metadata.durationMs;
}
const op = current.operation as any;
if (op) {
if (typeof op.source === "string") sources.add(op.source);
if (typeof op.url === "string") sources.add(op.url);
if (typeof op.inputHandleId === "string") sources.add(op.inputHandleId);
if (Array.isArray(op.layout)) {
for (const node of op.layout) {
if (node.inputHandleId) sources.add(node.inputHandleId);
if (node.src) sources.add(node.src);
if (typeof node.durationFrames === "number") {
const start =
typeof node.startFrame === "number" ? node.startFrame : 0;
const durMs = ((start + node.durationFrames) / fps) * 1000;
if (durMs > maxDurationMs) maxDurationMs = durMs;
}
}
}
}
if (current.children) {
queue.push(...current.children);
}
}
for (const src of sources) {
try {
const input = await inputStore.acquire(src);
const sec = await input.computeDuration();
if (Number.isFinite(sec) && sec > 0) {
const ms = Math.round(sec * 1000);
if (ms > maxDurationMs) maxDurationMs = ms;
}
} catch (_) {}
}
return maxDurationMs > 0 ? maxDurationMs : 1000;
}
public async renderVideo(
virtualMedia: VirtualMediaData,
options?: {
codec?: string;
audioCodec?: string;
quality?: string;
concurrency?: number;
/**
* Depth of the DMA staging ring (default 2, clamped 2–8). A deeper
* ring overlaps more encoder latency at the cost of VRAM
* (~8.3MB/slot at 1080p RGBA). Also settable via
* `FRAMEFIELDS_RING_CAPACITY`.
*/
ringCapacity?: number;
/**
* Measure the output while it renders (black and frozen frames,
* dropped frames, loudness, clipping, silence, document warnings)
* and return the findings as `qa`. `true` uses default thresholds.
*/
qa?: boolean | VideoQaOptions;
/**
* Aborts the render between frames; the promise rejects with
* `signal.reason` and the partial output is deleted.
*/
signal?: AbortSignal;
/**
* Called as the render moves through its phases: "prepare", each
* frame ("render"), "audio" (mixing, per clip), "audio-encode",
* "mux", "convert" (GIF only) and "done".
*/
onProgress?: (progress: RenderProgress) => void;
/**
* Renders only composition frames `start` (inclusive) to `end`
* (exclusive), as one chunk of a render split across workers. The
* chunk's timestamps start at 0 and its first frame is a keyframe,
* so `stitchChunks` joins chunks without re-encoding. Effects that
* carry state between frames rebuild it before `start`.
*/
frames?: FrameRange;
/**
* `false` writes no audio track, as for chunks: the timeline's audio
* is exported once with `codec: "aac"` and added by `stitchChunks`.
*/
audio?: boolean;
},
): Promise<{
filePath: string;
cleanup: () => Promise<void>;
qa?: VideoQaReport;
}> {
await HeadlessWebGPURenderer.initialize();
const codecOption = options?.codec ?? "h264";
const isMp3 = codecOption === "mp3";
const isWebM =
codecOption === "vp8" || codecOption === "vp9" || codecOption === "opus";
const mediaType = getMediaType(virtualMedia);
const isLut =
mediaType === "LUT" ||
(virtualMedia as any).operation?.dataType === "LUT";
const isAudioOnly =
isMp3 ||
codecOption === "aac" ||
codecOption === "opus" ||
mediaType === "Audio";
const fps =
virtualMedia.metadata.fps ?? (isAudioOnly || isLut ? 24 : undefined);
if (!fps) {
const dataType = mediaType ?? (virtualMedia as any).dataType ?? "Unknown";
const op =
(virtualMedia as any).operation?.op ??
(virtualMedia as any).op ??
"Unknown";
throw new Error(
`FPS is missing from virtualMedia metadata (dataType: "${dataType}", operation: "${op}")`,
);
}
await preflightMediaTime(virtualMedia, fps);
const width = toEvenDimension(
virtualMedia.metadata.width ?? (isAudioOnly ? 1280 : 0),
);
const height = toEvenDimension(
virtualMedia.metadata.height ?? (isAudioOnly ? 720 : 0),
);
if (!width || !height)
throw new Error(`Invalid dimensions: ${width}x${height}`);
let durationMs = virtualMedia.metadata.durationMs;
if (!durationMs || durationMs <= 0) {
durationMs = await this.resolveMediaDuration(virtualMedia, fps);
virtualMedia.metadata.durationMs = durationMs;
}
const range = resolveFrameRange(
options?.frames,
frameCount(durationMs, fps),
);
if (isAudioOnly && (options?.frames || options?.audio === false))
throw new Error(
`An audio-only render (codec "${codecOption}") takes neither \`frames\` nor \`audio: false\`.`,
);
const totalFrames = range.end - range.start;
const renderId = `vid-${randomUUID()}`;
const qaOptions: VideoQaOptions | undefined =
options?.qa === true ? {} : options?.qa ? options.qa : undefined;
const inspector = qaOptions ? new FrameInspector(qaOptions) : undefined;
let videoQa: VideoQaStats | undefined;
let audioQa: AudioQaStats | undefined;
const videoCodec: "avc" | "vp9" | "vp8" =
codecOption === "vp9" ? "vp9" : codecOption === "vp8" ? "vp8" : "avc";
const audioCodec: "aac" | "opus" | "mp3" =
(options?.audioCodec as any) ?? (isMp3 ? "mp3" : isWebM ? "opus" : "aac");
const quality = resolveQuality(options?.quality);
rendererLogger.debug(
{
renderId,
totalFrames,
fps,
width,
height,
codec: codecOption,
...renderSemaphore.stats,
},
"[HeadlessWebGPURenderer] Acquiring slot for video render (inline)",
);
if (renderSemaphore.stats.active >= renderSemaphore.stats.max) {
rendererLogger.info(
{ renderId, ...renderSemaphore.stats },
`[HeadlessWebGPURenderer] Waiting for a render slot (${renderSemaphore.stats.active} running, ${renderSemaphore.stats.queued} queued)`,
);
}
await renderSemaphore.acquire();
const reporter = new RenderReporter({
renderId,
totalFrames,
onProgress: options?.onProgress,
});
const seconds = `${(durationMs / 1000).toFixed(1)}s`;
reporter.stage(
"prepare",
isAudioOnly
? `Preparing ${seconds} of audio (${audioCodec})`
: `Preparing ${totalFrames} frames (${seconds}) at ${width}x${height}, ${fps} fps, ${codecOption}`,
{ width, height, fps, durationMs, codec: codecOption },
);
let lastCpuUsage = process.cpuUsage();
let lastCpuTime = process.hrtime.bigint();
const getCpuUsage = () => {
const curCpuUsage = process.cpuUsage();
const curCpuTime = process.hrtime.bigint();
const userDiff = curCpuUsage.user - lastCpuUsage.user;
const sysDiff = curCpuUsage.system - lastCpuUsage.system;
const timeDiff = Number(curCpuTime - lastCpuTime) / 1000;
lastCpuUsage = curCpuUsage;
lastCpuTime = curCpuTime;
if (timeDiff === 0) return "0.0%";
const cpus = os.cpus().length || 1;
const percent = (((userDiff + sysDiff) / timeDiff) * 100) / cpus;
return `${percent.toFixed(1)}%`;
};
const getRamUsage = () => {
const mem = process.memoryUsage();
const freeMem = os.freemem();
const totalMem = os.totalmem();
const usedMem = totalMem - freeMem;
const heapUsedMb = Math.round(mem.heapUsed / 1024 / 1024);
const heapTotalMb = Math.round(mem.heapTotal / 1024 / 1024);
const rssMb = Math.round(mem.rss / 1024 / 1024);
const sysUsedMb = Math.round(usedMem / 1024 / 1024);
const sysTotalMb = Math.round(totalMem / 1024 / 1024);
return `Process: ${rssMb}MB RSS (${heapUsedMb}MB/${heapTotalMb}MB Heap), System: ${sysUsedMb}MB/${sysTotalMb}MB`;
};
let statsInterval: NodeJS.Timeout | undefined;
if (process.env.NODE_ENV === "production") {
statsInterval = setInterval(() => {
// Kick off an async sample; the log line reports the previous one
// rather than blocking the event loop to gather a fresh value.
refreshVramUsage();
rendererLogger.info(
{
renderId,
cpu: getCpuUsage(),
ram: getRamUsage(),
vram: getVramUsage(),
renderQueue: renderSemaphore.stats,
},
"[HeadlessWebGPURenderer] Performance Stats",
);
}, 5000);
}
const tempDir = os.tmpdir();
const extension = isMp3 ? "mp3" : isWebM ? "webm" : "mp4";
const tempFilePath = path.join(tempDir, `render-${renderId}.${extension}`);
const tempGifPath = path.join(tempDir, `render-${renderId}.gif`);
const target = new FilePathTarget(tempFilePath);
const output = new Output({
format: isMp3
? new Mp3OutputFormat()
: isWebM
? new WebMOutputFormat()
: new Mp4OutputFormat(),
target,
});
let videoSource: VideoSampleSource | undefined;
if (!isAudioOnly) {
videoSource = new VideoSampleSource({
codec: videoCodec,
bitrate: quality,
latencyMode: "realtime",
// The codec server options pick hardware or software (video-encoders.ts).
hardwareAcceleration: "prefer-hardware",
} as any);
output.addVideoTrack(videoSource);
}
let audioSource: AudioSampleSource | undefined;
if (options?.audio !== false) {
audioSource = new AudioSampleSource({
codec: audioCodec,
bitrate: quality,
});
output.addAudioTrack(audioSource);
}
let outputStarted = false;
let outputFinalized = false;
// Resources for inline rendering (cleaned up in finally block)
// The device this render's audio effects keep their state on.
let gpu: GPUDevice | undefined;
let surface: NodeSurfaceProvider | undefined;
let renderer: Renderer2D | undefined;
let pipeline: ZeroCopyWebCodecsPipeline | undefined;
try {
// Inside the try, so a device failure still releases the slot.
const device = await ensureDevice();
gpu = device;
await output.start();
outputStarted = true;
await preloadFonts(virtualMedia, device);
if (!isAudioOnly && videoSource) {
surface = new NodeSurfaceProvider(device, width, height);
renderer = getRenderer2D(device, surface.colorFormat);
pipeline = new ZeroCopyWebCodecsPipeline(device, {
width,
height,
fps,
videoSource,
ringCapacity: resolveRingCapacity(options?.ringCapacity),
onFrame: inspector
? (rgba) => inspector.inspect(rgba, width, height)
: undefined,
});
const ctx: RenderContextValue | null =
surface && renderer ? { device, renderer, surface } : null;
// Pre-pass: draws a frame whose output is dropped so the LUT
// extractions and glyph uploads it starts finish before the frame
// is drawn for real.
const prePass = async (frame: number) => {
if (!ctx || !renderer) return;
const preEncoder = device.createCommandEncoder();
const dummyTex = renderer.getTemporaryTexture(width, height);
const dummyView = dummyTex.createView();
const preClear = renderer.beginFrame(
preEncoder,
dummyView,
{ r: 0, g: 0, b: 0, a: 0 },
width,
height,
"clear",
);
preClear.end();
await drawCompositionTree(
ctx,
preEncoder,
dummyView,
dummyTex,
width,
height,
virtualMedia,
{
frame,
compositionFrame: frame,
fps,
isHeadless: true,
renderId,
virtualMedia,
containerWidth: width,
containerHeight: height,
isVideoMode: virtualMedia.operation?.dataType === "Video",
},
);
device.queue.submit([preEncoder.finish()]);
await lutStore.awaitAllPending(device);
};
updateClockSignals(range.start, fps, durationMs);
await prePass(range.start);
reporter.stage(
"render",
range.start === 0 && totalFrames === range.total
? `Rendering ${totalFrames} frames`
: `Rendering frames ${range.start}–${range.end - 1} of ${range.total}`,
);
// Render and encode frames using pipelined double-buffered DMA staging ring
for (let i = range.start; i < range.end; i++) {
options?.signal?.throwIfAborted();
if (ctx && surface && renderer && pipeline) {
// Assets first used on a later frame (a LUT, a glyph) get the
// same pre-pass as the first frame, as renderImage and the
// preview do.
if (
i > range.start &&
(lutStore.hasPending(device) || SlugFontCache.hasPending())
) {
updateClockSignals(i, fps, durationMs);
await prePass(i);
}
compositionStateStore.setState(renderId, i, fps, true);
updateClockSignals(i, fps, durationMs);
// Release decoders of clips that ended a while ago.
mediaDecoderCache.tick();
const encoder = device.createCommandEncoder();
const targetView = surface.getCurrentTextureView();
const targetTexture = surface.getCurrentTexture();
const clearPass = renderer.beginFrame(
encoder,
targetView,
{ r: 0, g: 0, b: 0, a: 0 },
width,
height,
"clear",
);
clearPass.end();
await drawCompositionTree(
ctx,
encoder,
targetView,
targetTexture,
width,
height,
virtualMedia,
{
frame: i,
compositionFrame: i,
fps,
isHeadless: true,
renderId,
virtualMedia,
containerWidth: width,
containerHeight: height,
isVideoMode: virtualMedia.operation?.dataType === "Video",
},
);
// Chunk-local index: the pipeline timestamps frames from it.
const done = i - range.start + 1;
await pipeline.enqueueFrame(done - 1, targetTexture, encoder);
if (done < totalFrames) reporter.frameDone(done);
} else {
throw new Error("Invalid render context state");
}
}
if (pipeline) {
await pipeline.drain();
// 100% once the last frames left the staging ring.
reporter.frameDone(totalFrames);
const pipelineStats = pipeline.getStats();
if (inspector && qaOptions) {
videoQa = inspector.stats(
totalFrames,
pipelineStats.totalFramesEncoded,
fps,
width,
height,
qaOptions,
);
}
rendererLogger.debug(
{ renderId, totalFrames, ...pipelineStats },
`[HeadlessWebGPURenderer] All frames rendered and encoded (avg cycle: ${pipelineStats.averageCycleTimeMs.toFixed(2)}ms, throughput: ${pipelineStats.throughputFps.toFixed(1)} fps)`,
);
}
}
// ── Audio ──────────────────────────────────────────────────────────────
if (audioSource) {
options?.signal?.throwIfAborted();
audioQa = await encodeAudioTrack({
virtualMedia,
fps,
device,
renderId,
source: audioSource,
codec: audioCodec,
reporter,
qaOptions,
});
}
options?.signal?.throwIfAborted();
// Flushes the encoders and writes the container index.
reporter.stage("mux", `Muxing ${extension}`);
await output.finalize();
outputFinalized = true;
const qa = qaOptions
? buildQaReport({
video: videoQa,
audio: audioQa,
diagnostics: takeRenderDiagnostics(renderId),
options: qaOptions,
})
: undefined;
const cleanup = async () => {
try {
await fs.unlink(tempFilePath).catch(() => {});
if (codecOption === "gif") {
await fs.unlink(tempGifPath).catch(() => {});
}
} catch {}
};
if (codecOption === "gif") {
reporter.stage("convert", "Converting to GIF (ffmpeg)");
await convertMp4ToGif(tempFilePath, tempGifPath);
reporter.finish({ bytes: await fileSize(tempGifPath) });
return { filePath: tempGifPath, cleanup, ...(qa && { qa }) };
}
reporter.finish({ bytes: await fileSize(tempFilePath) });
return { filePath: tempFilePath, cleanup, ...(qa && { qa }) };
} catch (error) {
reporter.fail(error, options?.signal?.aborted === true);
if (outputStarted && !outputFinalized) {
try {
await output.finalize();
} catch {}
}
try {
await fs.unlink(tempFilePath).catch(() => {});
if (codecOption === "gif") {
await fs.unlink(tempGifPath).catch(() => {});
}
} catch {}
throw error;
} finally {
reporter.dispose();
if (statsInterval) {
clearInterval(statsInterval);
}
try {
renderer?.destroy();
} catch {}
try {
pipeline?.destroy();
} catch {}
try {
surface?.destroy();
} catch {}
try {
// Only this render's decoders: another render may be running.
mediaDecoderCache.clearNode(renderId);
} catch {}
try {
clearAllVideoCache();
} catch {}
try {
textureCache.destroy();
} catch {}
try {
WebGPUAudioProcessor.clearCache(renderId, gpu);
} catch {}
try {
shaderStore.clear(renderId);
} catch {}
// Drop anything reported for this render that QA did not collect.
takeRenderDiagnostics(renderId);
renderSemaphore.release();
rendererLogger.debug(
{ renderId, ...renderSemaphore.stats },
"[HeadlessWebGPURenderer] Video render slot released",
);
}
}
}
export const HeadlessMediaRenderer = HeadlessWebGPURenderer;
export type HeadlessMediaRenderer = HeadlessWebGPURenderer;