Skip to content

Code-first video & image, rendered natively on WebGPU.

21.1 KB
Raw Blame
import fs from "node:fs/promises";
import path from "node:path";
import { RenderError } from "../errors.js";
import type {
	CheckoutResult,
	Pipeline,
	RenderedFile,
	StepContext,
} from "../jobs/pipeline.js";
import type { JobInput, RenderSettings } from "../jobs/schema.js";
import type { JobProgress } from "../jobs/store.js";
import { isUrl } from "../jobs/validate.js";
import { rendererLogger } from "../logger.js";
import { downloadS3, downloadUrl } from "../output/sinks.js";
import { gpuStatus } from "../worker/gpu.js";
import { workerHealth } from "../worker/health.js";
import {
	type BoundEngine,
	bindEngine,
	type EngineModule,
	type EngineRenderer,
	markDepsLoaded,
} from "./engine.js";
import { clearWorkspaceRequireCache } from "./engine-alias.js";
import { buildPlan, importEntry, type PlanOutput } from "./load.js";
import { openSandbox } from "./sandbox.js";

const PROGRESS_INTERVAL_MS = 500;

/**
 * Pipeline step: imports the repo's entry into this process and renders every
 * output of its plan on the worker's warm GPU device (PLAN.md §8).
 */
export const render: Pipeline["render"] = async (checkout, input, ctx) => {
	// The project's own framefields, bound once per worker (§7).
	const engine = await bindEngine(checkout.workdir, checkout.root);
	markDepsLoaded(checkout.root);
	if (engine.workerHooks) {
		// Path-keyed caches from the previous job may hold stale bytes (§8.2).
		engine.mod.resetRenderCaches?.();
		engine.mod.FontManager?.clear();
	} else {
		// No cache reset, progress or abort in this version: render it, then
		// start fresh rather than risk stale caches in the next job.
		workerHealth.requestRecycle(
			`framefields ${engine.version} lacks worker hooks`,
		);
		ctx.warn(
			"ENGINE_NO_WORKER_HOOKS",
			`framefields ${engine.version} predates worker support (progress, cancel, cache reset); the worker restarts after this job.`,
		);
	}

	const outDir = path.join(ctx.jobDir, "out");
	await fs.mkdir(outDir, { recursive: true });

	// The render may be aborted by the job (cancel/timeout) or by the stall
	// detector; both go through this controller.
	const controller = new AbortController();
	const forward = () => controller.abort(ctx.signal.reason);
	if (ctx.signal.aborted) forward();
	ctx.signal.addEventListener("abort", forward, { once: true });

	const sandbox = openSandbox({
		cwd: checkout.workdir,
		argv1: path.join(checkout.workdir, input.target.entry),
		env: input.env,
	});
	const warnings: string[] = [];
	let dispose: (() => Promise<void>) | undefined;
	try {
		const work = (async () => {
			// A stitch joins files already rendered: no composition to load.
			if (input.render.kind === "stitch")
				return [
					await stitchOutput(input.render, outDir, ctx, controller, engine),
				];
			const files: RenderedFile[] = [];
			const plan = await withTimeout(
				loadPlan(checkout, input, ctx.jobId, (d) => {
					dispose = d;
				}),
				ctx.config.limits.timeouts.load,
				"load",
			);
			for (const output of plan) {
				const { Composition, Media } = engine.mod;
				const known =
					(Composition && output.target instanceof Composition) ||
					(Media && output.target instanceof Media);
				if (!known) {
					warnings.push(output.name);
				}
			}
			if (warnings.length) {
				ctx.warn(
					"ENGINE_FOREIGN_INSTANCE",
					`Outputs ${warnings.join(", ")} were built by a different copy of framefields than the project's; rendering them with the project's engine.`,
				);
			}
			for (const [index, output] of plan.entries()) {
				controller.signal.throwIfAborted();
				files.push(
					...(await renderOutput(
						output,
						index,
						plan.length,
						input.render,
						outDir,
						ctx,
						controller,
						engine,
					)),
				);
			}
			return files;
		})();

		const files = await raceWithGrace(
			work,
			sandbox.failed,
			controller.signal,
			ctx.config.limits.killGraceSec,
		);
		return {
			files,
			engine: {
				framefields: engine.version,
				source: "project",
				workerHooks: engine.workerHooks,
				gpu: gpuStatus().adapter,
			},
		};
	} catch (err) {
		throw classifyRenderError(
			controller.signal.aborted ? controller.signal.reason : err,
			checkout.root,
		);
	} finally {
		ctx.signal.removeEventListener("abort", forward);
		sandbox.restore();
		clearWorkspaceRequireCache();
		await dispose?.().catch(() => {});
		if (engine.workerHooks) {
			// Free this job's decoders, textures and fonts now rather than when
			// the next job starts: an idle worker shouldn't hold them.
			try {
				engine.mod.resetRenderCaches?.();
				engine.mod.FontManager?.clear();
			} catch (err) {
				rendererLogger.warn({ err: String(err) }, "Render cache reset failed");
			}
		}
	}
};

async function loadPlan(
	checkout: CheckoutResult,
	input: JobInput,
	jobId: string,
	onLoaded: (dispose: () => Promise<void>) => void,
): Promise<PlanOutput[]> {
	const { mod, dispose } = await importEntry(
		checkout.workdir,
		input.target.entry,
		jobId,
	);
	onLoaded(dispose);
	return buildPlan(
		mod,
		input.target.export,
		input.target.args,
		input.target.select,
	);
}

/** Renders one plan output into one or more files. */
async function renderOutput(
	output: PlanOutput,
	index: number,
	count: number,
	defaults: RenderSettings,
	outDir: string,
	ctx: StepContext,
	controller: AbortController,
	engine: BoundEngine,
): Promise<RenderedFile[]> {
	const settings = { ...defaults, ...output.render } as RenderSettings;
	const renderer = new engine.mod.HeadlessMediaRenderer();
	// Without progress events there is nothing to measure stalls by.
	const stall = stallWatch(
		engine.workerHooks ? ctx.config.limits.stallTimeoutSec : 0,
		controller,
	);
	const base = path.join(outDir, `${index}-${output.name}`);

	try {
		switch (settings.kind) {
			case "video":
			case "audio": {
				const codec =
					settings.codec ?? (settings.kind === "audio" ? "aac" : "h264");
				const { ext, contentType } = containerFor(codec);
				const started = Date.now();
				let lastEmit = 0;
				let lastPhase: string | undefined;
				let expectedFrames: number | undefined;
				const chunk =
					settings.kind === "video" ? chunkOptions(settings, engine) : {};
				const result = await output.target.renderVideo({
					outputPath: `${base}.${ext}`,
					...chunk,
					codec,
					audioCodec: settings.audioCodec,
					quality: settings.quality,
					qa: settings.qa,
					renderer,
					signal: controller.signal,
					onProgress: (p: EngineProgress) => {
						stall.touch();
						expectedFrames = p.totalFrames;
						// The job reports its own "done" after the upload.
						if (p.phase === "done") return;
						const now = Date.now();
						const lastFrame = p.phase === "render" && p.frame >= p.totalFrames;
						if (
							p.phase === lastPhase &&
							!lastFrame &&
							now - lastEmit < PROGRESS_INTERVAL_MS
						)
							return;
						lastEmit = now;
						lastPhase = p.phase;
						ctx.progress({
							...toJobProgress(p, (now - started) / 1000),
							output: count > 1 ? index : undefined,
						});
					},
				});
				return [
					{
						name: output.name,
						path: result.filePath,
						ext,
						contentType,
						expectedFrames,
						...(result.qa !== undefined && { qa: result.qa }),
					},
				];
			}
			case "probe":
				return [
					await probeOutput(
						output,
						settings,
						base,
						ctx,
						controller,
						engine,
						renderer,
						stall.touch,
						count > 1 ? index : undefined,
					),
				];
			case "stitch":
				throw new RenderError(
					"INTERNAL",
					"Stitch jobs are handled before the plan loads.",
				);
			case "frames": {
				const files: RenderedFile[] = [];
				const frames = settings.frames ?? [0];
				for (const [i, frame] of frames.entries()) {
					controller.signal.throwIfAborted();
					const png = await output.target.renderFrame({ frame, renderer });
					stall.touch();
					const file = `${base}-f${String(frame).padStart(5, "0")}.png`;
					await fs.writeFile(file, png);
					files.push({
						name: `${output.name}-f${frame}`,
						path: file,
						ext: "png",
						contentType: "image/png",
					});
					ctx.progress({
						phase: "rendering",
						output: count > 1 ? index : undefined,
						frame: i + 1,
						total: frames.length,
					});
				}
				return files;
			}
			case "frameGrid": {
				if (!output.target.renderFrameGrid) {
					throw new RenderError(
						"EXPORT_INVALID",
						`Output ${output.name} can't render a frame grid.`,
						{ phase: "render" },
					);
				}
				const grid = settings.grid ?? {};
				const png = await output.target.renderFrameGrid({
					startFrame: grid.from,
					endFrame: grid.to,
					stepFrames: grid.step,
					...(grid.columns && { columns: grid.columns }),
					renderer,
				});
				const file = `${base}-grid.png`;
				await fs.writeFile(file, png);
				return [
					{
						name: output.name,
						path: file,
						ext: "png",
						contentType: "image/png",
					},
				];
			}
			case "lut": {
				const vm =
					output.target.toVirtualMedia?.() ?? output.target.virtualMedia;
				if (!vm) {
					throw new RenderError(
						"EXPORT_INVALID",
						`Output ${output.name} has no virtual media to bake a LUT from.`,
						{
							phase: "render",
						},
					);
				}
				const cube = await renderer.renderLut(
					vm as never,
					settings.frames?.[0] ?? 0,
				);
				const file = `${base}.cube`;
				await fs.writeFile(file, cube);
				return [
					{
						name: output.name,
						path: file,
						ext: "cube",
						contentType: "text/plain",
					},
				];
			}
		}
	} finally {
		stall.stop();
	}
}

/** A function the project's framefields must export for this job, or ENGINE_UNSUPPORTED. */
function requireEngine<K extends "planChunks" | "stitchChunks">(
	engine: BoundEngine,
	name: K,
	what: string,
): NonNullable<EngineModule[K]> {
	const fn = engine.mod[name];
	if (typeof fn !== "function") {
		throw new RenderError(
			"ENGINE_UNSUPPORTED",
			`framefields ${engine.version} can't run ${what}: it has no \`${name}\`.`,
			{
				phase: "render",
				hint: "Update framefields in the project to a version with split renders.",
			},
		);
	}
	return fn as NonNullable<EngineModule[K]>;
}

/**
 * `renderVideo` options for one chunk of a split render. An engine without
 * them would ignore `frames` and render the whole film, so it is refused.
 */
function chunkOptions(
	settings: RenderSettings,
	engine: BoundEngine,
): { frames?: { start: number; end: number }; audio?: false } {
	if (!settings.range && settings.audio !== false) return {};
	requireEngine(engine, "stitchChunks", "chunked renders");
	return {
		...(settings.range && {
			frames: { start: settings.range.from, end: settings.range.to },
		}),
		...(settings.audio === false && { audio: false }),
	};
}

/**
 * kind=probe: times frames across the timeline and plans the split render
 * with the project's own `planChunks`, written as one JSON file.
 */
async function probeOutput(
	output: PlanOutput,
	settings: RenderSettings,
	base: string,
	ctx: StepContext,
	controller: AbortController,
	engine: BoundEngine,
	renderer: Pick<EngineRenderer, "probeRender">,
	touch: () => void,
	outputIndex: number | undefined,
): Promise<RenderedFile> {
	const planChunks = requireEngine(engine, "planChunks", "probe jobs");
	const vm = output.target.toVirtualMedia?.() ?? output.target.virtualMedia;
	if (!renderer.probeRender || !vm) {
		throw new RenderError(
			"EXPORT_INVALID",
			`Output ${output.name} can't be probed.`,
			{ phase: "render" },
		);
	}
	const options = settings.probe ?? {};
	const measured = await renderer.probeRender(vm as never, {
		samples: options.samples,
		signal: controller.signal,
		onSample: (done, total) => {
			touch();
			ctx.progress({
				phase: "rendering",
				output: outputIndex,
				frame: done,
				total,
				message: "probing frame costs",
			});
		},
	});
	const plan = planChunks({
		totalFrames: measured.totalFrames,
		samples: measured.samples,
		workers: options.workers,
		maxWorkers: options.maxWorkers,
		overheadSec: options.overheadSec,
		efficiency: options.efficiency,
		minChunkFrames: options.minChunkFrames,
	});
	const file = `${base}-probe.json`;
	await fs.writeFile(file, JSON.stringify({ ...measured, plan }));
	return {
		name: output.name,
		path: file,
		ext: "json",
		contentType: "application/json",
	};
}

/**
 * kind=stitch: downloads the chunks (and audio) from the output bucket and
 * joins them with the project's `stitchChunks`, without re-encoding.
 */
async function stitchOutput(
	settings: RenderSettings,
	outDir: string,
	ctx: StepContext,
	controller: AbortController,
	engine: BoundEngine,
): Promise<RenderedFile> {
	const stitch = settings.stitch;
	if (!stitch)
		throw new RenderError("JOB_INVALID", "render.stitch is missing.");
	const bucket = stitch.bucket ?? ctx.config.storage.bucket;
	const stitchChunks = requireEngine(engine, "stitchChunks", "stitch jobs");
	const inDir = path.join(ctx.jobDir, "in");
	await fs.mkdir(inDir, { recursive: true });
	const inputs = [...stitch.chunks, ...(stitch.audio ? [stitch.audio] : [])];
	let downloaded = 0;
	const local = await mapLimit(inputs, 4, async (input, i) => {
		const file = path.join(inDir, `${i}-${path.basename(inputPath(input))}`);
		if (isUrl(input)) await downloadUrl(input, file, controller.signal);
		else if (bucket)
			await downloadS3(ctx.config, bucket, input, file, controller.signal);
		else
			throw new RenderError(
				"JOB_INVALID",
				"Stitch inputs given as keys need STORAGE_BUCKET or stitch.bucket.",
			);
		ctx.progress({
			phase: "rendering",
			frame: ++downloaded,
			total: inputs.length,
			message: "downloading chunks",
		});
		return file;
	});
	const ext = path.extname(inputPath(stitch.chunks[0])).slice(1) || "mp4";
	const { contentType } = containerFor(ext === "webm" ? "vp9" : "h264");
	const outputPath = path.join(outDir, `0-main.${ext}`);
	ctx.progress({
		phase: "encoding",
		message: `stitching ${stitch.chunks.length} chunks`,
	});
	const result = await stitchChunks({
		chunks: local.slice(0, stitch.chunks.length),
		audio: stitch.audio ? local.at(-1) : undefined,
		outputPath,
		signal: controller.signal,
	});
	return {
		name: "main",
		path: outputPath,
		ext,
		contentType,
		expectedFrames: result.frames,
	};
}

/** A stitch input's path: the key, or a URL's path without its signature. */
function inputPath(input: string): string {
	return isUrl(input) ? new URL(input).pathname : input;
}

/** `Promise.all` over `items`, at most `limit` at a time, results in order. */
async function mapLimit<T, R>(
	items: readonly T[],
	limit: number,
	fn: (item: T, index: number) => Promise<R>,
): Promise<R[]> {
	const results = new Array<R>(items.length);
	let next = 0;
	const run = async () => {
		while (next < items.length) {
			const i = next++;
			results[i] = await fn(items[i], i);
		}
	};
	await Promise.all(Array.from({ length: Math.min(limit, items.length) }, run));
	return results;
}

/**
 * What `renderVideo({ onProgress })` reports. Engines before the phase
 * breakdown send only "render" and "audio", without the optional fields.
 */
interface EngineProgress {
	phase: string;
	frame: number;
	totalFrames: number;
	fps?: number;
	etaSec?: number;
	detail?: string;
}

/** What each engine phase after the frames is doing, for the job message. */
const AFTER_FRAMES: Record<string, string> = {
	audio: "mixing audio",
	"audio-encode": "encoding audio",
	mux: "muxing",
	convert: "converting to GIF",
};

/** Maps an engine progress event to the job's progress. */
export function toJobProgress(
	p: EngineProgress,
	elapsedSec: number,
): JobProgress {
	if (p.phase === "prepare") {
		return {
			phase: "rendering",
			frame: 0,
			total: p.totalFrames,
			message: "preparing",
		};
	}
	if (p.phase === "render") {
		const fps = p.fps ?? p.frame / Math.max(0.001, elapsedSec);
		return {
			phase: "rendering",
			frame: p.frame,
			total: p.totalFrames,
			fps: Math.round(fps * 10) / 10,
			etaSec:
				p.etaSec ??
				Math.round((p.totalFrames - p.frame) / Math.max(fps, 0.001)),
		};
	}
	const label = AFTER_FRAMES[p.phase] ?? p.phase;
	return {
		phase: "encoding",
		frame: p.frame,
		total: p.totalFrames,
		message: p.detail ? `${label} (${p.detail})` : label,
	};
}

export function containerFor(codec: string): {
	ext: string;
	contentType: string;
} {
	switch (codec) {
		case "vp8":
		case "vp9":
			return { ext: "webm", contentType: "video/webm" };
		case "gif":
			return { ext: "gif", contentType: "image/gif" };
		case "mp3":
			return { ext: "mp3", contentType: "audio/mpeg" };
		case "aac":
			return { ext: "m4a", contentType: "audio/mp4" };
		case "opus":
			return { ext: "webm", contentType: "audio/webm" };
		default:
			return { ext: "mp4", contentType: "video/mp4" };
	}
}

/** Aborts with RENDER_STALLED when no progress arrives for `seconds`. */
function stallWatch(seconds: number, controller: AbortController) {
	let last = Date.now();
	const timer =
		seconds > 0
			? setInterval(() => {
					if (
						Date.now() - last > seconds * 1000 &&
						!controller.signal.aborted
					) {
						controller.abort(
							new RenderError(
								"RENDER_STALLED",
								`No render progress for ${seconds}s.`,
								{ phase: "render" },
							),
						);
					}
				}, 1000)
			: undefined;
	timer?.unref();
	return {
		touch: () => {
			last = Date.now();
		},
		stop: () => clearInterval(timer),
	};
}

function withTimeout<T>(
	promise: Promise<T>,
	seconds: number,
	what: string,
): Promise<T> {
	if (seconds <= 0) return promise;
	let timer: NodeJS.Timeout | undefined;
	return Promise.race([
		promise,
		new Promise<never>((_, reject) => {
			timer = setTimeout(
				() =>
					reject(
						new RenderError(
							"COMPOSITION_BUILD_FAILED",
							`Loading the entry took longer than ${seconds}s (${what}).`,
							{
								phase: "load",
								hint: "Is the export waiting on something that never resolves? Raise FF_TIMEOUT_LOAD_SEC if it's just slow.",
							},
						),
					),
				seconds * 1000,
			);
		}),
	]).finally(() => clearTimeout(timer));
}

/**
 * Waits for the render, a stray error from repo code, or an abort. Code that
 * ignores the abort gets `graceSec`; after that the job is reported anyway and
 * the worker recycles, since it can't stop that code in-process (§8.3).
 */
async function raceWithGrace<T>(
	work: Promise<T>,
	failed: Promise<never>,
	signal: AbortSignal,
	graceSec: number,
): Promise<T> {
	let timer: NodeJS.Timeout | undefined;
	const abandoned = new Promise<never>((_, reject) => {
		const arm = () => {
			timer = setTimeout(() => {
				workerHealth.requestRecycle("render ignored abort");
				reject(signal.reason);
			}, graceSec * 1000);
		};
		if (signal.aborted) arm();
		else signal.addEventListener("abort", arm, { once: true });
	});
	work.catch(() => {}); // may settle after we've given up on it
	try {
		return await Promise.race([work, failed, abandoned]);
	} finally {
		clearTimeout(timer);
	}
}

export function classifyRenderError(
	err: unknown,
	repoRoot: string,
): RenderError {
	const scrub = (s: string) => s.replaceAll(repoRoot, "<repo>");
	if (err instanceof RenderError) {
		if (err.code === "GPU_DEVICE_LOST" || err.code === "RENDER_OOM")
			workerHealth.requestRecycle(err.code);
		return err;
	}
	const message = err instanceof Error ? err.message : String(err);
	const stack =
		err instanceof Error ? projectFrames(err.stack ?? "", repoRoot) : undefined;
	const details = stack ? { stack } : undefined;

	if (
		/device (was )?lost|GPUDevice.*lost|DeviceLost|VK_ERROR_DEVICE_LOST/i.test(
			message,
		)
	) {
		workerHealth.requestRecycle("GPU device lost");
		return new RenderError("GPU_DEVICE_LOST", scrub(message), {
			phase: "render",
			details,
			cause: err,
		});
	}
	if (
		/out of memory|OutOfMemory|Failed to allocate|allocation failed|ERR_OUT_OF_MEMORY/i.test(
			message,
		)
	) {
		workerHealth.requestRecycle("out of memory");
		return new RenderError("RENDER_OOM", scrub(message), {
			phase: "render",
			details,
			cause: err,
		});
	}
	if (
		/EACCES|EPERM|permission denied/i.test(message) &&
		/node_modules/.test(message)
	) {
		return new RenderError("RENDER_FAILED", scrub(message), {
			phase: "render",
			hint: "node_modules files are shared read-only between projects (FF_SHARED_MODULES); write to a temp dir instead, or set FF_SHARED_MODULES=false.",
			details,
			cause: err,
		});
	}
	if (/Cannot open local media file|ENOENT|no such file/i.test(message)) {
		return new RenderError("ASSET_NOT_FOUND", scrub(message), {
			phase: "render",
			details,
			hint: "If the file is tracked with LFS, make sure source.lfs.include covers it.",
			cause: err,
		});
	}
	if (
		/UnsupportedInputFormat|Unsupported.*format|Invalid data found/i.test(
			message,
		)
	) {
		return new RenderError("ASSET_NOT_FOUND", scrub(message), {
			phase: "render",
			details,
			hint: "The file isn't decodable media. An undownloaded LFS pointer looks like this too.",
			cause: err,
		});
	}
	return new RenderError("RENDER_FAILED", scrub(message), {
		phase: "render",
		details,
		cause: err,
	});
}

/** Keeps stack frames from the project, with paths made repo-relative. */
function projectFrames(stack: string, repoRoot: string): string | undefined {
	const frames = stack
		.split("\n")
		.filter((line) => line.includes(repoRoot) && !line.includes("node_modules"))
		.slice(0, 15)
		.map((line) => line.replaceAll(repoRoot, "<repo>").trim());
	return frames.length ? frames.join("\n") : undefined;
}