Skip to content

Code-first video & image, rendered natively on WebGPU.

fix(renderer): pick working video encoders; software WebGPU fallback

okanaslcf657279fa76
diff --git a/.changeset/renderer-worker-gpu.md b/.changeset/renderer-worker-gpu.md
new file mode 100644
index 00000000..3299b193
--- /dev/null
+++ b/.changeset/renderer-worker-gpu.md
@@ -0,0 +1,11 @@
+---
+"framefields": minor
+"@framefields/renderer": minor
+"@framefields/webgpu-renderers": minor
+---
+
+Headless rendering for long-lived workers and any Linux machine:
+
+- Video encoding uses a hardware encoder (NVENC, VideoToolbox) only after testing that it works, and software encoders (libx264, libx265, libvpx, SVT-AV1) otherwise. This fixes H.264/HEVC renders on linux-arm64, where `h264_rkmpp` was picked without Rockchip hardware, and on NVIDIA GPUs without NVENC (A100, H100). Set `FRAMEFIELDS_HW_CODECS=off` to always encode in software.
+- WebGPU falls back to a software adapter (e.g. lavapipe) when no GPU adapter is available. The new `describeGpu()` reports the adapter in use and whether it is a CPU one.
+- `renderVideo` accepts `signal` (AbortSignal) and `onProgress`; `resetRenderCaches()` clears per-render caches between jobs in one process.
diff --git a/packages/framefields/src/index.ts b/packages/framefields/src/index.ts
index 5baf8cbf..86afb25a 100644
--- a/packages/framefields/src/index.ts
+++ b/packages/framefields/src/index.ts
@@ -2914,10 +2914,12 @@ export {
export {
type AudioQaStats,
computeGridLayout,
+ describeGpu,
type FrameGridOptions,
type FrameGridRenderable,
type FrameSamplePoint,
formatQaReport,
+ type GpuAdapterDescription,
HeadlessMediaRenderer,
HeadlessWebGPURenderer,
type QaIssue,
diff --git a/packages/framefields/src/renderer/index.ts b/packages/framefields/src/renderer/index.ts
index 3b4b3b07..c333bb1d 100644
--- a/packages/framefields/src/renderer/index.ts
+++ b/packages/framefields/src/renderer/index.ts
@@ -13,7 +13,11 @@ export {
type VideoQaStats,
} from "@framefields/renderer";
-export { initHeadlessWebGPU } from "@framefields/webgpu-renderers";
+export {
+ describeGpu,
+ type GpuAdapterDescription,
+ initHeadlessWebGPU,
+} from "@framefields/webgpu-renderers";
export {
computeGridLayout,
diff --git a/packages/renderer/package.json b/packages/renderer/package.json
index 88775902..881af79c 100644
--- a/packages/renderer/package.json
+++ b/packages/renderer/package.json
@@ -102,6 +102,7 @@
"@mediabunny/server": "^1.51.0",
"@napi-rs/webcodecs": "^1.3.1",
"mediabunny": "^1.51.0",
+ "node-av": "^6.0.0",
"pino": "^10.3.1",
"sharp": "^0.35.4",
"skia-canvas": "^3.0.8",
diff --git a/packages/renderer/src/bootstrap-mediabunny.ts b/packages/renderer/src/bootstrap-mediabunny.ts
index 764d604c..3bc783e6 100644
--- a/packages/renderer/src/bootstrap-mediabunny.ts
+++ b/packages/renderer/src/bootstrap-mediabunny.ts
@@ -1,6 +1,7 @@
import fsSync from "node:fs";
import { registerMediabunnyServer } from "@mediabunny/server";
import * as webcodecs from "@napi-rs/webcodecs";
+import { videoCodecServerOptions } from "./video-encoders.js";
export const isAMD = (): boolean => {
if (process.platform !== "linux") return false;
@@ -29,14 +30,8 @@ export const isAMD = (): boolean => {
};
export const bootstrapMediabunny = (): void => {
- const getHardwareContextOption = () => {
- if (isAMD()) {
- return { hardwareContext: null };
- }
- return {};
- };
-
- registerMediabunnyServer(getHardwareContextOption());
+ // Hardware codecs only where they work; software encoders otherwise.
+ registerMediabunnyServer(videoCodecServerOptions(isAMD()) as never);
for (const key of Object.keys(webcodecs)) {
if (typeof (globalThis as any)[key] === "undefined") {
diff --git a/packages/renderer/src/headless-webgpu-renderer.ts b/packages/renderer/src/headless-webgpu-renderer.ts
index d8ea1785..54aeabb1 100644
--- a/packages/renderer/src/headless-webgpu-renderer.ts
+++ b/packages/renderer/src/headless-webgpu-renderer.ts
@@ -47,7 +47,7 @@ import {
import sharp from "sharp";
import { Canvas as SkiaCanvas, Image as SkiaImage } from "skia-canvas";
import { preloadFonts } from "./asset-preloader.js";
-import { bootstrapMediabunny, isAMD } from "./bootstrap-mediabunny.js";
+import { bootstrapMediabunny } from "./bootstrap-mediabunny.js";
import { discoverAndRegisterNodeRenderers } from "./dynamic-node-discovery.js";
import { rendererLogger } from "./logger.js";
import {
@@ -1188,7 +1188,8 @@ export class HeadlessWebGPURenderer {
codec: videoCodec,
bitrate: quality,
latencyMode: "realtime",
- hardwareAcceleration: isAMD() ? "prefer-software" : "prefer-hardware",
+ // The codec server options pick hardware or software (video-encoders.ts).
+ hardwareAcceleration: "prefer-hardware",
} as any);
output.addVideoTrack(videoSource);
}
diff --git a/packages/renderer/src/tests/video-encoders.test.ts b/packages/renderer/src/tests/video-encoders.test.ts
new file mode 100644
index 00000000..a3630844
--- /dev/null
+++ b/packages/renderer/src/tests/video-encoders.test.ts
@@ -0,0 +1,44 @@
+import { createRequire } from "node:module";
+import { pathToFileURL } from "node:url";
+import type * as NodeAvModule from "node-av";
+import { afterEach, describe, expect, it } from "vitest";
+import { videoCodecServerOptions } from "../video-encoders.js";
+
+// node-av exactly as @mediabunny/server resolves it.
+const require = createRequire(import.meta.url);
+const av = (await import(
+ pathToFileURL(
+ createRequire(require.resolve("@mediabunny/server")).resolve("node-av"),
+ ).href
+)) as typeof NodeAvModule;
+
+describe("videoCodecServerOptions", () => {
+ afterEach(() => {
+ delete process.env.FRAMEFIELDS_HW_CODECS;
+ });
+
+ it("resolves to a working hardware context or null, never throws", async () => {
+ const { hardwareContext } = videoCodecServerOptions(false);
+ const hw = await hardwareContext(av.AV_CODEC_ID_H264);
+ expect(hw === null || hw instanceof av.HardwareContext).toBe(true);
+ });
+
+ it("makes software encoders the default for each codec", async () => {
+ await videoCodecServerOptions(true).hardwareContext(av.AV_CODEC_ID_H264);
+ // node-av's linux-arm64 build lists h264_rkmpp/hevc_rkmpp first.
+ expect(av.Codec.findEncoder(av.AV_CODEC_ID_H264)?.name).toBe("libx264");
+ expect(av.Codec.findEncoder(av.AV_CODEC_ID_HEVC)?.name).toBe("libx265");
+ expect(av.Codec.findEncoder(av.AV_CODEC_ID_VP9)?.name).toBe("libvpx-vp9");
+ });
+
+ it("keeps every codec in software when forced", async () => {
+ const { hardwareContext } = videoCodecServerOptions(true);
+ await expect(hardwareContext(av.AV_CODEC_ID_H264)).resolves.toBeNull();
+ });
+
+ it("turns hardware codecs off with FRAMEFIELDS_HW_CODECS=off", async () => {
+ process.env.FRAMEFIELDS_HW_CODECS = "off";
+ const { hardwareContext } = videoCodecServerOptions(false);
+ await expect(hardwareContext(av.AV_CODEC_ID_H264)).resolves.toBeNull();
+ });
+});
diff --git a/packages/renderer/src/video-encoders.ts b/packages/renderer/src/video-encoders.ts
new file mode 100644
index 00000000..7e2aef0e
--- /dev/null
+++ b/packages/renderer/src/video-encoders.ts
@@ -0,0 +1,188 @@
+import { createRequire } from "node:module";
+import { pathToFileURL } from "node:url";
+import type * as NodeAvModule from "node-av";
+import { rendererLogger } from "./logger.js";
+
+type NodeAv = typeof NodeAvModule;
+type HardwareContext = NodeAvModule.HardwareContext;
+type Codec = NodeAvModule.Codec;
+
+let nodeAv: Promise<NodeAv> | undefined;
+
+/**
+ * node-av as @mediabunny/server loads it, with software encoders as the
+ * default. The patch and the HardwareContext instances returned to the server
+ * must come from that exact module instance, so the file is resolved from the
+ * server package and imported natively (a `require()` can be routed through
+ * a loader such as tsx and produce a second copy).
+ */
+function loadNodeAv(): Promise<NodeAv> {
+ nodeAv ??= (async () => {
+ const require = createRequire(import.meta.url);
+ const server = require.resolve("@mediabunny/server");
+ const entry = createRequire(server).resolve("node-av");
+ const av = (await import(pathToFileURL(entry).href)) as NodeAv;
+ preferSoftwareEncoders(av);
+ return av;
+ })();
+ return nodeAv;
+}
+
+/**
+ * Software encoders to use per codec, best first. @mediabunny/server takes
+ * `Codec.findEncoder(id)` as its software encoder, which returns whichever
+ * encoder FFmpeg registered first. In node-av's linux-arm64 build that is a
+ * hardware encoder (h264_rkmpp, hevc_rkmpp), which fails without Rockchip
+ * hardware.
+ */
+const SOFTWARE_ENCODERS: Record<string, string[]> = {
+ AV_CODEC_ID_H264: ["libx264", "libopenh264"],
+ AV_CODEC_ID_HEVC: ["libx265"],
+ AV_CODEC_ID_VP8: ["libvpx"],
+ AV_CODEC_ID_VP9: ["libvpx-vp9"],
+ AV_CODEC_ID_AV1: ["libsvtav1", "libaom-av1", "libdav1d"],
+};
+
+let patched = false;
+
+/** Makes `Codec.findEncoder` return a software encoder when one exists. */
+function preferSoftwareEncoders(av: NodeAv): void {
+ if (patched) return;
+ patched = true;
+ const byId = new Map<number, string[]>();
+ for (const [constant, names] of Object.entries(SOFTWARE_ENCODERS)) {
+ const id = (av as unknown as Record<string, unknown>)[constant];
+ if (typeof id === "number") byId.set(id, names);
+ }
+ const findEncoder = av.Codec.findEncoder.bind(av.Codec);
+ av.Codec.findEncoder = ((id: Parameters<typeof findEncoder>[0]) => {
+ for (const name of byId.get(id as number) ?? []) {
+ const codec = av.Codec.findEncoderByName(name as never);
+ if (codec) return codec;
+ }
+ return findEncoder(id);
+ }) as typeof av.Codec.findEncoder;
+}
+
+/**
+ * Opens the encoder with the settings @mediabunny/server uses and encodes one
+ * frame. A hardware encoder can be compiled in and still be unusable: no such
+ * hardware (rkmpp on a non-Rockchip board), or a GPU without an encoder block
+ * (NVIDIA A100/H100 have no NVENC).
+ */
+async function encoderWorks(av: NodeAv, codec: Codec): Promise<boolean> {
+ const ctx = new av.CodecContext();
+ const frame = new av.Frame();
+ const packet = new av.Packet();
+ try {
+ ctx.allocContext3(codec);
+ const formats = codec.pixelFormats;
+ const pixelFormat =
+ !formats || formats.includes(av.AV_PIX_FMT_YUV420P)
+ ? av.AV_PIX_FMT_YUV420P
+ : formats[0];
+ ctx.width = 256;
+ ctx.height = 256;
+ ctx.pixelFormat = pixelFormat;
+ ctx.timeBase = new av.Rational(1, 1e6);
+ ctx.framerate = new av.Rational(30, 1);
+ ctx.gopSize = 60;
+ ctx.bitRate = 1_000_000n;
+ if ((await ctx.open2()) < 0) return false;
+
+ frame.alloc();
+ frame.format = pixelFormat;
+ frame.width = 256;
+ frame.height = 256;
+ if (frame.getBuffer() < 0) return false;
+ frame.pts = 0n;
+ packet.alloc();
+ if ((await ctx.sendFrame(frame)) < 0) return false;
+ if ((await ctx.sendFrame(null)) < 0) return false;
+ for (;;) {
+ const ret = await ctx.receivePacket(packet);
+ if (ret >= 0) return true;
+ if (ret !== av.AVERROR_EAGAIN) return false;
+ }
+ } catch {
+ return false;
+ } finally {
+ packet.free();
+ frame.free();
+ ctx.freeContext();
+ }
+}
+
+/** True when the env turns hardware video encoding/decoding off. */
+function hardwareDisabled(): boolean {
+ const value = process.env.FRAMEFIELDS_HW_CODECS?.toLowerCase();
+ return value === "0" || value === "off" || value === "false";
+}
+
+/**
+ * Returns the `hardwareContext` option for registerMediabunnyServer: the
+ * platform's hardware context for codecs whose hardware encoder actually
+ * works (NVENC, VideoToolbox, ...), and null otherwise, so that codec is
+ * encoded in software. Each codec is tested once per process.
+ *
+ * The server calls this before it picks any encoder (unless the encoder is
+ * created with "prefer-software"), so the software default is patched in by
+ * then.
+ */
+function hardwareContextResolver(
+ forceSoftware: boolean,
+): (codecId: number) => Promise<HardwareContext | null> {
+ let hw: HardwareContext | null | undefined;
+ const verdicts = new Map<number, Promise<HardwareContext | null>>();
+
+ const resolve = async (codecId: number): Promise<HardwareContext | null> => {
+ const av = await loadNodeAv();
+ if (forceSoftware || hardwareDisabled()) return null;
+ if (hw === undefined) hw = av.HardwareContext.auto();
+ if (!hw) return null;
+ const encoder = hw.getEncoderCodec(codecId as never);
+ // Decode-only hardware: the encoder falls back to software on its own.
+ if (!encoder) return hw;
+ if (await encoderWorks(av, encoder)) {
+ rendererLogger.info(
+ { encoder: encoder.name, device: hw.deviceTypeName },
+ "[video-encoders] Using hardware video encoder",
+ );
+ return hw;
+ }
+ rendererLogger.warn(
+ { encoder: encoder.name, device: hw.deviceTypeName },
+ "[video-encoders] Hardware video encoder unusable; encoding in software",
+ );
+ return null;
+ };
+
+ return (codecId) => {
+ let verdict = verdicts.get(codecId);
+ if (!verdict) {
+ // A failed hardware probe means software; a failed load surfaces.
+ verdict = resolve(codecId).catch(async (err) => {
+ await loadNodeAv();
+ rendererLogger.warn(
+ { err: err instanceof Error ? err.message : String(err) },
+ "[video-encoders] Hardware probe failed; encoding in software",
+ );
+ return null;
+ });
+ verdicts.set(codecId, verdict);
+ }
+ return verdict;
+ };
+}
+
+/**
+ * Options for registerMediabunnyServer: hardware video codecs when they work,
+ * software (libx264, libx265, libvpx, SVT-AV1) otherwise. `forceSoftware`
+ * keeps every codec in software (used on AMD). Create encoders with
+ * "prefer-hardware" so this decides; "prefer-software" skips it.
+ */
+export function videoCodecServerOptions(forceSoftware: boolean): {
+ hardwareContext: (codecId: number) => Promise<HardwareContext | null>;
+} {
+ return { hardwareContext: hardwareContextResolver(forceSoftware) };
+}
diff --git a/packages/webgpu-renderers/src/device.ts b/packages/webgpu-renderers/src/device.ts
index 657b3885..4c496ce9 100644
--- a/packages/webgpu-renderers/src/device.ts
+++ b/packages/webgpu-renderers/src/device.ts
@@ -2,6 +2,45 @@ let deviceInstance: GPUDevice | null = null;
let loadPromise: Promise<GPUDevice> | null = null;
+/** The adapter behind the rendering device. */
+export interface GpuAdapterDescription {
+ vendor: string;
+ architecture: string;
+ device: string;
+ description: string;
+ /** A CPU implementation (lavapipe/llvmpipe, SwiftShader, WARP). */
+ software: boolean;
+}
+
+let adapterInfo: GpuAdapterDescription | null = null;
+
+const SOFTWARE_ADAPTER =
+ /llvmpipe|lavapipe|softpipe|swiftshader|\bwarp\b|software/i;
+
+function readAdapterInfo(adapter: any): GpuAdapterDescription {
+ const info = adapter.info ?? {};
+ const out = {
+ vendor: String(info.vendor ?? ""),
+ architecture: String(info.architecture ?? ""),
+ device: String(info.device ?? ""),
+ description: String(info.description ?? ""),
+ };
+ const software =
+ info.isFallbackAdapter === true ||
+ adapter.isFallbackAdapter === true ||
+ SOFTWARE_ADAPTER.test(
+ `${out.vendor} ${out.architecture} ${out.device} ${out.description}`,
+ );
+ return { ...out, software };
+}
+
+/** Initialises the rendering device if needed and describes its adapter. */
+export async function describeGpu(): Promise<GpuAdapterDescription> {
+ await ensureDevice();
+ if (!adapterInfo) throw new Error("WebGPU adapter unknown");
+ return adapterInfo;
+}
+
export async function acquireDevice(
opts?: GPURequestAdapterOptions,
): Promise<GPUDevice> {
@@ -28,11 +67,18 @@ export async function acquireDevice(
});
}
}
- const adapter = await (globalThis as any).navigator.gpu.requestAdapter({
- powerPreference: "high-performance",
- ...opts,
- });
+ const gpu = (globalThis as any).navigator.gpu;
+ // No GPU: fall back to a software adapter (e.g. lavapipe, SwiftShader).
+ const adapter =
+ (await gpu.requestAdapter({
+ powerPreference: "high-performance",
+ ...opts,
+ })) ??
+ (opts?.forceFallbackAdapter
+ ? null
+ : await gpu.requestAdapter({ ...opts, forceFallbackAdapter: true }));
if (!adapter) throw new Error("WebGPU not available");
+ adapterInfo = readAdapterInfo(adapter);
const requiredFeatures: string[] = [];
if (adapter.features.has("float32-filterable")) {
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index adcb5b1b..2ec4f4b3 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -2774,6 +2774,9 @@ importers:
mediabunny:
specifier: ^1.51.0
version: 1.51.0
+ node-av:
+ specifier: 6.0.0
+ version: 6.0.0
pino:
specifier: ^10.3.1
version: 10.3.1