Code-first video & image, rendered natively on WebGPU.
import fs from "node:fs/promises";
import path from "node:path";
import { RenderError } from "../errors.js";
import type {
CheckoutResult,
Pipeline,
RenderedFile,
StepContext,
} from "../jobs/pipeline.js";
import type { JobInput, RenderSettings } from "../jobs/schema.js";
import type { JobProgress } from "../jobs/store.js";
import { isUrl } from "../jobs/validate.js";
import { rendererLogger } from "../logger.js";
import { downloadS3, downloadUrl } from "../output/sinks.js";
import { gpuStatus } from "../worker/gpu.js";
import { workerHealth } from "../worker/health.js";
import {
type BoundEngine,
bindEngine,
type EngineModule,
type EngineRenderer,
markDepsLoaded,
} from "./engine.js";
import { clearWorkspaceRequireCache } from "./engine-alias.js";
import { buildPlan, importEntry, type PlanOutput } from "./load.js";
import { openSandbox } from "./sandbox.js";
const PROGRESS_INTERVAL_MS = 500;
/**
* Pipeline step: imports the repo's entry into this process and renders every
* output of its plan on the worker's warm GPU device (PLAN.md §8).
*/
export const render: Pipeline["render"] = async (checkout, input, ctx) => {
// The project's own framefields, bound once per worker (§7).
const engine = await bindEngine(checkout.workdir, checkout.root);
markDepsLoaded(checkout.root);
if (engine.workerHooks) {
// Path-keyed caches from the previous job may hold stale bytes (§8.2).
engine.mod.resetRenderCaches?.();
engine.mod.FontManager?.clear();
} else {
// No cache reset, progress or abort in this version: render it, then
// start fresh rather than risk stale caches in the next job.
workerHealth.requestRecycle(
`framefields ${engine.version} lacks worker hooks`,
);
ctx.warn(
"ENGINE_NO_WORKER_HOOKS",
`framefields ${engine.version} predates worker support (progress, cancel, cache reset); the worker restarts after this job.`,
);
}
const outDir = path.join(ctx.jobDir, "out");
await fs.mkdir(outDir, { recursive: true });
// The render may be aborted by the job (cancel/timeout) or by the stall
// detector; both go through this controller.
const controller = new AbortController();
const forward = () => controller.abort(ctx.signal.reason);
if (ctx.signal.aborted) forward();
ctx.signal.addEventListener("abort", forward, { once: true });
const sandbox = openSandbox({
cwd: checkout.workdir,
argv1: path.join(checkout.workdir, input.target.entry),
env: input.env,
});
const warnings: string[] = [];
let dispose: (() => Promise<void>) | undefined;
try {
const work = (async () => {
// A stitch joins files already rendered: no composition to load.
if (input.render.kind === "stitch")
return [
await stitchOutput(input.render, outDir, ctx, controller, engine),
];
const files: RenderedFile[] = [];
const plan = await withTimeout(
loadPlan(checkout, input, ctx.jobId, (d) => {
dispose = d;
}),
ctx.config.limits.timeouts.load,
"load",
);
for (const output of plan) {
const { Composition, Media } = engine.mod;
const known =
(Composition && output.target instanceof Composition) ||
(Media && output.target instanceof Media);
if (!known) {
warnings.push(output.name);
}
}
if (warnings.length) {
ctx.warn(
"ENGINE_FOREIGN_INSTANCE",
`Outputs ${warnings.join(", ")} were built by a different copy of framefields than the project's; rendering them with the project's engine.`,
);
}
for (const [index, output] of plan.entries()) {
controller.signal.throwIfAborted();
files.push(
...(await renderOutput(
output,
index,
plan.length,
input.render,
outDir,
ctx,
controller,
engine,
)),
);
}
return files;
})();
const files = await raceWithGrace(
work,
sandbox.failed,
controller.signal,
ctx.config.limits.killGraceSec,
);
return {
files,
engine: {
framefields: engine.version,
source: "project",
workerHooks: engine.workerHooks,
gpu: gpuStatus().adapter,
},
};
} catch (err) {
throw classifyRenderError(
controller.signal.aborted ? controller.signal.reason : err,
checkout.root,
);
} finally {
ctx.signal.removeEventListener("abort", forward);
sandbox.restore();
clearWorkspaceRequireCache();
await dispose?.().catch(() => {});
if (engine.workerHooks) {
// Free this job's decoders, textures and fonts now rather than when
// the next job starts: an idle worker shouldn't hold them.
try {
engine.mod.resetRenderCaches?.();
engine.mod.FontManager?.clear();
} catch (err) {
rendererLogger.warn({ err: String(err) }, "Render cache reset failed");
}
}
}
};
async function loadPlan(
checkout: CheckoutResult,
input: JobInput,
jobId: string,
onLoaded: (dispose: () => Promise<void>) => void,
): Promise<PlanOutput[]> {
const { mod, dispose } = await importEntry(
checkout.workdir,
input.target.entry,
jobId,
);
onLoaded(dispose);
return buildPlan(
mod,
input.target.export,
input.target.args,
input.target.select,
);
}
/** Renders one plan output into one or more files. */
async function renderOutput(
output: PlanOutput,
index: number,
count: number,
defaults: RenderSettings,
outDir: string,
ctx: StepContext,
controller: AbortController,
engine: BoundEngine,
): Promise<RenderedFile[]> {
const settings = { ...defaults, ...output.render } as RenderSettings;
const renderer = new engine.mod.HeadlessMediaRenderer();
// Without progress events there is nothing to measure stalls by.
const stall = stallWatch(
engine.workerHooks ? ctx.config.limits.stallTimeoutSec : 0,
controller,
);
const base = path.join(outDir, `${index}-${output.name}`);
try {
switch (settings.kind) {
case "video":
case "audio": {
const codec =
settings.codec ?? (settings.kind === "audio" ? "aac" : "h264");
const { ext, contentType } = containerFor(codec);
const started = Date.now();
let lastEmit = 0;
let lastPhase: string | undefined;
let expectedFrames: number | undefined;
const chunk =
settings.kind === "video" ? chunkOptions(settings, engine) : {};
const result = await output.target.renderVideo({
outputPath: `${base}.${ext}`,
...chunk,
codec,
audioCodec: settings.audioCodec,
quality: settings.quality,
qa: settings.qa,
renderer,
signal: controller.signal,
onProgress: (p: EngineProgress) => {
stall.touch();
expectedFrames = p.totalFrames;
// The job reports its own "done" after the upload.
if (p.phase === "done") return;
const now = Date.now();
const lastFrame = p.phase === "render" && p.frame >= p.totalFrames;
if (
p.phase === lastPhase &&
!lastFrame &&
now - lastEmit < PROGRESS_INTERVAL_MS
)
return;
lastEmit = now;
lastPhase = p.phase;
ctx.progress({
...toJobProgress(p, (now - started) / 1000),
output: count > 1 ? index : undefined,
});
},
});
return [
{
name: output.name,
path: result.filePath,
ext,
contentType,
expectedFrames,
...(result.qa !== undefined && { qa: result.qa }),
},
];
}
case "probe":
return [
await probeOutput(
output,
settings,
base,
ctx,
controller,
engine,
renderer,
stall.touch,
count > 1 ? index : undefined,
),
];
case "stitch":
throw new RenderError(
"INTERNAL",
"Stitch jobs are handled before the plan loads.",
);
case "frames": {
const files: RenderedFile[] = [];
const frames = settings.frames ?? [0];
for (const [i, frame] of frames.entries()) {
controller.signal.throwIfAborted();
const png = await output.target.renderFrame({ frame, renderer });
stall.touch();
const file = `${base}-f${String(frame).padStart(5, "0")}.png`;
await fs.writeFile(file, png);
files.push({
name: `${output.name}-f${frame}`,
path: file,
ext: "png",
contentType: "image/png",
});
ctx.progress({
phase: "rendering",
output: count > 1 ? index : undefined,
frame: i + 1,
total: frames.length,
});
}
return files;
}
case "frameGrid": {
if (!output.target.renderFrameGrid) {
throw new RenderError(
"EXPORT_INVALID",
`Output ${output.name} can't render a frame grid.`,
{ phase: "render" },
);
}
const grid = settings.grid ?? {};
const png = await output.target.renderFrameGrid({
startFrame: grid.from,
endFrame: grid.to,
stepFrames: grid.step,
...(grid.columns && { columns: grid.columns }),
renderer,
});
const file = `${base}-grid.png`;
await fs.writeFile(file, png);
return [
{
name: output.name,
path: file,
ext: "png",
contentType: "image/png",
},
];
}
case "lut": {
const vm =
output.target.toVirtualMedia?.() ?? output.target.virtualMedia;
if (!vm) {
throw new RenderError(
"EXPORT_INVALID",
`Output ${output.name} has no virtual media to bake a LUT from.`,
{
phase: "render",
},
);
}
const cube = await renderer.renderLut(
vm as never,
settings.frames?.[0] ?? 0,
);
const file = `${base}.cube`;
await fs.writeFile(file, cube);
return [
{
name: output.name,
path: file,
ext: "cube",
contentType: "text/plain",
},
];
}
}
} finally {
stall.stop();
}
}
/** A function the project's framefields must export for this job, or ENGINE_UNSUPPORTED. */
function requireEngine<K extends "planChunks" | "stitchChunks">(
engine: BoundEngine,
name: K,
what: string,
): NonNullable<EngineModule[K]> {
const fn = engine.mod[name];
if (typeof fn !== "function") {
throw new RenderError(
"ENGINE_UNSUPPORTED",
`framefields ${engine.version} can't run ${what}: it has no \`${name}\`.`,
{
phase: "render",
hint: "Update framefields in the project to a version with split renders.",
},
);
}
return fn as NonNullable<EngineModule[K]>;
}
/**
* `renderVideo` options for one chunk of a split render. An engine without
* them would ignore `frames` and render the whole film, so it is refused.
*/
function chunkOptions(
settings: RenderSettings,
engine: BoundEngine,
): { frames?: { start: number; end: number }; audio?: false } {
if (!settings.range && settings.audio !== false) return {};
requireEngine(engine, "stitchChunks", "chunked renders");
return {
...(settings.range && {
frames: { start: settings.range.from, end: settings.range.to },
}),
...(settings.audio === false && { audio: false }),
};
}
/**
* kind=probe: times frames across the timeline and plans the split render
* with the project's own `planChunks`, written as one JSON file.
*/
async function probeOutput(
output: PlanOutput,
settings: RenderSettings,
base: string,
ctx: StepContext,
controller: AbortController,
engine: BoundEngine,
renderer: Pick<EngineRenderer, "probeRender">,
touch: () => void,
outputIndex: number | undefined,
): Promise<RenderedFile> {
const planChunks = requireEngine(engine, "planChunks", "probe jobs");
const vm = output.target.toVirtualMedia?.() ?? output.target.virtualMedia;
if (!renderer.probeRender || !vm) {
throw new RenderError(
"EXPORT_INVALID",
`Output ${output.name} can't be probed.`,
{ phase: "render" },
);
}
const options = settings.probe ?? {};
const measured = await renderer.probeRender(vm as never, {
samples: options.samples,
signal: controller.signal,
onSample: (done, total) => {
touch();
ctx.progress({
phase: "rendering",
output: outputIndex,
frame: done,
total,
message: "probing frame costs",
});
},
});
const plan = planChunks({
totalFrames: measured.totalFrames,
samples: measured.samples,
workers: options.workers,
maxWorkers: options.maxWorkers,
overheadSec: options.overheadSec,
efficiency: options.efficiency,
minChunkFrames: options.minChunkFrames,
});
const file = `${base}-probe.json`;
await fs.writeFile(file, JSON.stringify({ ...measured, plan }));
return {
name: output.name,
path: file,
ext: "json",
contentType: "application/json",
};
}
/**
* kind=stitch: downloads the chunks (and audio) from the output bucket and
* joins them with the project's `stitchChunks`, without re-encoding.
*/
async function stitchOutput(
settings: RenderSettings,
outDir: string,
ctx: StepContext,
controller: AbortController,
engine: BoundEngine,
): Promise<RenderedFile> {
const stitch = settings.stitch;
if (!stitch)
throw new RenderError("JOB_INVALID", "render.stitch is missing.");
const bucket = stitch.bucket ?? ctx.config.storage.bucket;
const stitchChunks = requireEngine(engine, "stitchChunks", "stitch jobs");
const inDir = path.join(ctx.jobDir, "in");
await fs.mkdir(inDir, { recursive: true });
const inputs = [...stitch.chunks, ...(stitch.audio ? [stitch.audio] : [])];
let downloaded = 0;
const local = await mapLimit(inputs, 4, async (input, i) => {
const file = path.join(inDir, `${i}-${path.basename(inputPath(input))}`);
if (isUrl(input)) await downloadUrl(input, file, controller.signal);
else if (bucket)
await downloadS3(ctx.config, bucket, input, file, controller.signal);
else
throw new RenderError(
"JOB_INVALID",
"Stitch inputs given as keys need STORAGE_BUCKET or stitch.bucket.",
);
ctx.progress({
phase: "rendering",
frame: ++downloaded,
total: inputs.length,
message: "downloading chunks",
});
return file;
});
const ext = path.extname(inputPath(stitch.chunks[0])).slice(1) || "mp4";
const { contentType } = containerFor(ext === "webm" ? "vp9" : "h264");
const outputPath = path.join(outDir, `0-main.${ext}`);
ctx.progress({
phase: "encoding",
message: `stitching ${stitch.chunks.length} chunks`,
});
const result = await stitchChunks({
chunks: local.slice(0, stitch.chunks.length),
audio: stitch.audio ? local.at(-1) : undefined,
outputPath,
signal: controller.signal,
});
return {
name: "main",
path: outputPath,
ext,
contentType,
expectedFrames: result.frames,
};
}
/** A stitch input's path: the key, or a URL's path without its signature. */
function inputPath(input: string): string {
return isUrl(input) ? new URL(input).pathname : input;
}
/** `Promise.all` over `items`, at most `limit` at a time, results in order. */
async function mapLimit<T, R>(
items: readonly T[],
limit: number,
fn: (item: T, index: number) => Promise<R>,
): Promise<R[]> {
const results = new Array<R>(items.length);
let next = 0;
const run = async () => {
while (next < items.length) {
const i = next++;
results[i] = await fn(items[i], i);
}
};
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, run));
return results;
}
/**
* What `renderVideo({ onProgress })` reports. Engines before the phase
* breakdown send only "render" and "audio", without the optional fields.
*/
interface EngineProgress {
phase: string;
frame: number;
totalFrames: number;
fps?: number;
etaSec?: number;
detail?: string;
}
/** What each engine phase after the frames is doing, for the job message. */
const AFTER_FRAMES: Record<string, string> = {
audio: "mixing audio",
"audio-encode": "encoding audio",
mux: "muxing",
convert: "converting to GIF",
};
/** Maps an engine progress event to the job's progress. */
export function toJobProgress(
p: EngineProgress,
elapsedSec: number,
): JobProgress {
if (p.phase === "prepare") {
return {
phase: "rendering",
frame: 0,
total: p.totalFrames,
message: "preparing",
};
}
if (p.phase === "render") {
const fps = p.fps ?? p.frame / Math.max(0.001, elapsedSec);
return {
phase: "rendering",
frame: p.frame,
total: p.totalFrames,
fps: Math.round(fps * 10) / 10,
etaSec:
p.etaSec ??
Math.round((p.totalFrames - p.frame) / Math.max(fps, 0.001)),
};
}
const label = AFTER_FRAMES[p.phase] ?? p.phase;
return {
phase: "encoding",
frame: p.frame,
total: p.totalFrames,
message: p.detail ? `${label} (${p.detail})` : label,
};
}
export function containerFor(codec: string): {
ext: string;
contentType: string;
} {
switch (codec) {
case "vp8":
case "vp9":
return { ext: "webm", contentType: "video/webm" };
case "gif":
return { ext: "gif", contentType: "image/gif" };
case "mp3":
return { ext: "mp3", contentType: "audio/mpeg" };
case "aac":
return { ext: "m4a", contentType: "audio/mp4" };
case "opus":
return { ext: "webm", contentType: "audio/webm" };
default:
return { ext: "mp4", contentType: "video/mp4" };
}
}
/** Aborts with RENDER_STALLED when no progress arrives for `seconds`. */
function stallWatch(seconds: number, controller: AbortController) {
let last = Date.now();
const timer =
seconds > 0
? setInterval(() => {
if (
Date.now() - last > seconds * 1000 &&
!controller.signal.aborted
) {
controller.abort(
new RenderError(
"RENDER_STALLED",
`No render progress for ${seconds}s.`,
{ phase: "render" },
),
);
}
}, 1000)
: undefined;
timer?.unref();
return {
touch: () => {
last = Date.now();
},
stop: () => clearInterval(timer),
};
}
function withTimeout<T>(
promise: Promise<T>,
seconds: number,
what: string,
): Promise<T> {
if (seconds <= 0) return promise;
let timer: NodeJS.Timeout | undefined;
return Promise.race([
promise,
new Promise<never>((_, reject) => {
timer = setTimeout(
() =>
reject(
new RenderError(
"COMPOSITION_BUILD_FAILED",
`Loading the entry took longer than ${seconds}s (${what}).`,
{
phase: "load",
hint: "Is the export waiting on something that never resolves? Raise FF_TIMEOUT_LOAD_SEC if it's just slow.",
},
),
),
seconds * 1000,
);
}),
]).finally(() => clearTimeout(timer));
}
/**
* Waits for the render, a stray error from repo code, or an abort. Code that
* ignores the abort gets `graceSec`; after that the job is reported anyway and
* the worker recycles, since it can't stop that code in-process (§8.3).
*/
async function raceWithGrace<T>(
work: Promise<T>,
failed: Promise<never>,
signal: AbortSignal,
graceSec: number,
): Promise<T> {
let timer: NodeJS.Timeout | undefined;
const abandoned = new Promise<never>((_, reject) => {
const arm = () => {
timer = setTimeout(() => {
workerHealth.requestRecycle("render ignored abort");
reject(signal.reason);
}, graceSec * 1000);
};
if (signal.aborted) arm();
else signal.addEventListener("abort", arm, { once: true });
});
work.catch(() => {}); // may settle after we've given up on it
try {
return await Promise.race([work, failed, abandoned]);
} finally {
clearTimeout(timer);
}
}
export function classifyRenderError(
err: unknown,
repoRoot: string,
): RenderError {
const scrub = (s: string) => s.replaceAll(repoRoot, "<repo>");
if (err instanceof RenderError) {
if (err.code === "GPU_DEVICE_LOST" || err.code === "RENDER_OOM")
workerHealth.requestRecycle(err.code);
return err;
}
const message = err instanceof Error ? err.message : String(err);
const stack =
err instanceof Error ? projectFrames(err.stack ?? "", repoRoot) : undefined;
const details = stack ? { stack } : undefined;
if (
/device (was )?lost|GPUDevice.*lost|DeviceLost|VK_ERROR_DEVICE_LOST/i.test(
message,
)
) {
workerHealth.requestRecycle("GPU device lost");
return new RenderError("GPU_DEVICE_LOST", scrub(message), {
phase: "render",
details,
cause: err,
});
}
if (
/out of memory|OutOfMemory|Failed to allocate|allocation failed|ERR_OUT_OF_MEMORY/i.test(
message,
)
) {
workerHealth.requestRecycle("out of memory");
return new RenderError("RENDER_OOM", scrub(message), {
phase: "render",
details,
cause: err,
});
}
if (
/EACCES|EPERM|permission denied/i.test(message) &&
/node_modules/.test(message)
) {
return new RenderError("RENDER_FAILED", scrub(message), {
phase: "render",
hint: "node_modules files are shared read-only between projects (FF_SHARED_MODULES); write to a temp dir instead, or set FF_SHARED_MODULES=false.",
details,
cause: err,
});
}
if (/Cannot open local media file|ENOENT|no such file/i.test(message)) {
return new RenderError("ASSET_NOT_FOUND", scrub(message), {
phase: "render",
details,
hint: "If the file is tracked with LFS, make sure source.lfs.include covers it.",
cause: err,
});
}
if (
/UnsupportedInputFormat|Unsupported.*format|Invalid data found/i.test(
message,
)
) {
return new RenderError("ASSET_NOT_FOUND", scrub(message), {
phase: "render",
details,
hint: "The file isn't decodable media. An undownloaded LFS pointer looks like this too.",
cause: err,
});
}
return new RenderError("RENDER_FAILED", scrub(message), {
phase: "render",
details,
cause: err,
});
}
/** Keeps stack frames from the project, with paths made repo-relative. */
function projectFrames(stack: string, repoRoot: string): string | undefined {
const frames = stack
.split("\n")
.filter((line) => line.includes(repoRoot) && !line.includes("node_modules"))
.slice(0, 15)
.map((line) => line.replaceAll(repoRoot, "<repo>").trim());
return frames.length ? frames.join("\n") : undefined;
}