Code-first video & image, rendered natively on WebGPU.
fix(renderer): pick working video encoders; software WebGPU fallback
okanaslcf657279fa76
diff --git a/.changeset/renderer-worker-gpu.md b/.changeset/renderer-worker-gpu.mdnew file mode 100644index 00000000..3299b193--- /dev/null+++ b/.changeset/renderer-worker-gpu.md@@ -0,0 +1,11 @@+---+"framefields": minor+"@framefields/renderer": minor+"@framefields/webgpu-renderers": minor+---++Headless rendering for long-lived workers and any Linux machine:++- Video encoding uses a hardware encoder (NVENC, VideoToolbox) only after testing that it works, and software encoders (libx264, libx265, libvpx, SVT-AV1) otherwise. This fixes H.264/HEVC renders on linux-arm64, where `h264_rkmpp` was picked without Rockchip hardware, and on NVIDIA GPUs without NVENC (A100, H100). Set `FRAMEFIELDS_HW_CODECS=off` to always encode in software.+- WebGPU falls back to a software adapter (e.g. lavapipe) when no GPU adapter is available. The new `describeGpu()` reports the adapter in use and whether it is a CPU one.+- `renderVideo` accepts `signal` (AbortSignal) and `onProgress`; `resetRenderCaches()` clears per-render caches between jobs in one process.diff --git a/packages/framefields/src/index.ts b/packages/framefields/src/index.tsindex 5baf8cbf..86afb25a 100644--- a/packages/framefields/src/index.ts+++ b/packages/framefields/src/index.ts@@ -2914,10 +2914,12 @@ export {export {type AudioQaStats,computeGridLayout,+ describeGpu,type FrameGridOptions,type FrameGridRenderable,type FrameSamplePoint,formatQaReport,+ type GpuAdapterDescription,HeadlessMediaRenderer,HeadlessWebGPURenderer,type QaIssue,diff --git a/packages/framefields/src/renderer/index.ts b/packages/framefields/src/renderer/index.tsindex 3b4b3b07..c333bb1d 100644--- a/packages/framefields/src/renderer/index.ts+++ b/packages/framefields/src/renderer/index.ts@@ -13,7 +13,11 @@ export {type VideoQaStats,} from "@framefields/renderer";-export { initHeadlessWebGPU } from "@framefields/webgpu-renderers";+export {+ describeGpu,+ type GpuAdapterDescription,+ initHeadlessWebGPU,+} from "@framefields/webgpu-renderers";export {computeGridLayout,diff --git a/packages/renderer/package.json b/packages/renderer/package.jsonindex 88775902..881af79c 100644--- a/packages/renderer/package.json+++ b/packages/renderer/package.json@@ -102,6 +102,7 @@"@mediabunny/server": "^1.51.0","@napi-rs/webcodecs": "^1.3.1","mediabunny": "^1.51.0",+ "node-av": "^6.0.0","pino": "^10.3.1","sharp": "^0.35.4","skia-canvas": "^3.0.8",diff --git a/packages/renderer/src/bootstrap-mediabunny.ts b/packages/renderer/src/bootstrap-mediabunny.tsindex 764d604c..3bc783e6 100644--- a/packages/renderer/src/bootstrap-mediabunny.ts+++ b/packages/renderer/src/bootstrap-mediabunny.ts@@ -1,6 +1,7 @@import fsSync from "node:fs";import { registerMediabunnyServer } from "@mediabunny/server";import * as webcodecs from "@napi-rs/webcodecs";+import { videoCodecServerOptions } from "./video-encoders.js";export const isAMD = (): boolean => {if (process.platform !== "linux") return false;@@ -29,14 +30,8 @@ export const isAMD = (): boolean => {};export const bootstrapMediabunny = (): void => {- const getHardwareContextOption = () => {- if (isAMD()) {- return { hardwareContext: null };- }- return {};- };-- registerMediabunnyServer(getHardwareContextOption());+ // Hardware codecs only where they work; software encoders otherwise.+ registerMediabunnyServer(videoCodecServerOptions(isAMD()) as never);for (const key of Object.keys(webcodecs)) {if (typeof (globalThis as any)[key] === "undefined") {diff --git a/packages/renderer/src/headless-webgpu-renderer.ts b/packages/renderer/src/headless-webgpu-renderer.tsindex d8ea1785..54aeabb1 100644--- a/packages/renderer/src/headless-webgpu-renderer.ts+++ b/packages/renderer/src/headless-webgpu-renderer.ts@@ -47,7 +47,7 @@ import {import sharp from "sharp";import { Canvas as SkiaCanvas, Image as SkiaImage } from "skia-canvas";import { preloadFonts } from "./asset-preloader.js";-import { bootstrapMediabunny, isAMD } from "./bootstrap-mediabunny.js";+import { bootstrapMediabunny } from "./bootstrap-mediabunny.js";import { discoverAndRegisterNodeRenderers } from "./dynamic-node-discovery.js";import { rendererLogger } from "./logger.js";import {@@ -1188,7 +1188,8 @@ export class HeadlessWebGPURenderer {codec: videoCodec,bitrate: quality,latencyMode: "realtime",- hardwareAcceleration: isAMD() ? "prefer-software" : "prefer-hardware",+ // The codec server options pick hardware or software (video-encoders.ts).+ hardwareAcceleration: "prefer-hardware",} as any);output.addVideoTrack(videoSource);}diff --git a/packages/renderer/src/tests/video-encoders.test.ts b/packages/renderer/src/tests/video-encoders.test.tsnew file mode 100644index 00000000..a3630844--- /dev/null+++ b/packages/renderer/src/tests/video-encoders.test.ts@@ -0,0 +1,44 @@+import { createRequire } from "node:module";+import { pathToFileURL } from "node:url";+import type * as NodeAvModule from "node-av";+import { afterEach, describe, expect, it } from "vitest";+import { videoCodecServerOptions } from "../video-encoders.js";++// node-av exactly as @mediabunny/server resolves it.+const require = createRequire(import.meta.url);+const av = (await import(+ pathToFileURL(+ createRequire(require.resolve("@mediabunny/server")).resolve("node-av"),+ ).href+)) as typeof NodeAvModule;++describe("videoCodecServerOptions", () => {+ afterEach(() => {+ delete process.env.FRAMEFIELDS_HW_CODECS;+ });++ it("resolves to a working hardware context or null, never throws", async () => {+ const { hardwareContext } = videoCodecServerOptions(false);+ const hw = await hardwareContext(av.AV_CODEC_ID_H264);+ expect(hw === null || hw instanceof av.HardwareContext).toBe(true);+ });++ it("makes software encoders the default for each codec", async () => {+ await videoCodecServerOptions(true).hardwareContext(av.AV_CODEC_ID_H264);+ // node-av's linux-arm64 build lists h264_rkmpp/hevc_rkmpp first.+ expect(av.Codec.findEncoder(av.AV_CODEC_ID_H264)?.name).toBe("libx264");+ expect(av.Codec.findEncoder(av.AV_CODEC_ID_HEVC)?.name).toBe("libx265");+ expect(av.Codec.findEncoder(av.AV_CODEC_ID_VP9)?.name).toBe("libvpx-vp9");+ });++ it("keeps every codec in software when forced", async () => {+ const { hardwareContext } = videoCodecServerOptions(true);+ await expect(hardwareContext(av.AV_CODEC_ID_H264)).resolves.toBeNull();+ });++ it("turns hardware codecs off with FRAMEFIELDS_HW_CODECS=off", async () => {+ process.env.FRAMEFIELDS_HW_CODECS = "off";+ const { hardwareContext } = videoCodecServerOptions(false);+ await expect(hardwareContext(av.AV_CODEC_ID_H264)).resolves.toBeNull();+ });+});diff --git a/packages/renderer/src/video-encoders.ts b/packages/renderer/src/video-encoders.tsnew file mode 100644index 00000000..7e2aef0e--- /dev/null+++ b/packages/renderer/src/video-encoders.ts@@ -0,0 +1,188 @@+import { createRequire } from "node:module";+import { pathToFileURL } from "node:url";+import type * as NodeAvModule from "node-av";+import { rendererLogger } from "./logger.js";++type NodeAv = typeof NodeAvModule;+type HardwareContext = NodeAvModule.HardwareContext;+type Codec = NodeAvModule.Codec;++let nodeAv: Promise<NodeAv> | undefined;++/**+ * node-av as @mediabunny/server loads it, with software encoders as the+ * default. The patch and the HardwareContext instances returned to the server+ * must come from that exact module instance, so the file is resolved from the+ * server package and imported natively (a `require()` can be routed through+ * a loader such as tsx and produce a second copy).+ */+function loadNodeAv(): Promise<NodeAv> {+ nodeAv ??= (async () => {+ const require = createRequire(import.meta.url);+ const server = require.resolve("@mediabunny/server");+ const entry = createRequire(server).resolve("node-av");+ const av = (await import(pathToFileURL(entry).href)) as NodeAv;+ preferSoftwareEncoders(av);+ return av;+ })();+ return nodeAv;+}++/**+ * Software encoders to use per codec, best first. @mediabunny/server takes+ * `Codec.findEncoder(id)` as its software encoder, which returns whichever+ * encoder FFmpeg registered first. In node-av's linux-arm64 build that is a+ * hardware encoder (h264_rkmpp, hevc_rkmpp), which fails without Rockchip+ * hardware.+ */+const SOFTWARE_ENCODERS: Record<string, string[]> = {+ AV_CODEC_ID_H264: ["libx264", "libopenh264"],+ AV_CODEC_ID_HEVC: ["libx265"],+ AV_CODEC_ID_VP8: ["libvpx"],+ AV_CODEC_ID_VP9: ["libvpx-vp9"],+ AV_CODEC_ID_AV1: ["libsvtav1", "libaom-av1", "libdav1d"],+};++let patched = false;++/** Makes `Codec.findEncoder` return a software encoder when one exists. */+function preferSoftwareEncoders(av: NodeAv): void {+ if (patched) return;+ patched = true;+ const byId = new Map<number, string[]>();+ for (const [constant, names] of Object.entries(SOFTWARE_ENCODERS)) {+ const id = (av as unknown as Record<string, unknown>)[constant];+ if (typeof id === "number") byId.set(id, names);+ }+ const findEncoder = av.Codec.findEncoder.bind(av.Codec);+ av.Codec.findEncoder = ((id: Parameters<typeof findEncoder>[0]) => {+ for (const name of byId.get(id as number) ?? []) {+ const codec = av.Codec.findEncoderByName(name as never);+ if (codec) return codec;+ }+ return findEncoder(id);+ }) as typeof av.Codec.findEncoder;+}++/**+ * Opens the encoder with the settings @mediabunny/server uses and encodes one+ * frame. A hardware encoder can be compiled in and still be unusable: no such+ * hardware (rkmpp on a non-Rockchip board), or a GPU without an encoder block+ * (NVIDIA A100/H100 have no NVENC).+ */+async function encoderWorks(av: NodeAv, codec: Codec): Promise<boolean> {+ const ctx = new av.CodecContext();+ const frame = new av.Frame();+ const packet = new av.Packet();+ try {+ ctx.allocContext3(codec);+ const formats = codec.pixelFormats;+ const pixelFormat =+ !formats || formats.includes(av.AV_PIX_FMT_YUV420P)+ ? av.AV_PIX_FMT_YUV420P+ : formats[0];+ ctx.width = 256;+ ctx.height = 256;+ ctx.pixelFormat = pixelFormat;+ ctx.timeBase = new av.Rational(1, 1e6);+ ctx.framerate = new av.Rational(30, 1);+ ctx.gopSize = 60;+ ctx.bitRate = 1_000_000n;+ if ((await ctx.open2()) < 0) return false;++ frame.alloc();+ frame.format = pixelFormat;+ frame.width = 256;+ frame.height = 256;+ if (frame.getBuffer() < 0) return false;+ frame.pts = 0n;+ packet.alloc();+ if ((await ctx.sendFrame(frame)) < 0) return false;+ if ((await ctx.sendFrame(null)) < 0) return false;+ for (;;) {+ const ret = await ctx.receivePacket(packet);+ if (ret >= 0) return true;+ if (ret !== av.AVERROR_EAGAIN) return false;+ }+ } catch {+ return false;+ } finally {+ packet.free();+ frame.free();+ ctx.freeContext();+ }+}++/** True when the env turns hardware video encoding/decoding off. */+function hardwareDisabled(): boolean {+ const value = process.env.FRAMEFIELDS_HW_CODECS?.toLowerCase();+ return value === "0" || value === "off" || value === "false";+}++/**+ * Returns the `hardwareContext` option for registerMediabunnyServer: the+ * platform's hardware context for codecs whose hardware encoder actually+ * works (NVENC, VideoToolbox, ...), and null otherwise, so that codec is+ * encoded in software. Each codec is tested once per process.+ *+ * The server calls this before it picks any encoder (unless the encoder is+ * created with "prefer-software"), so the software default is patched in by+ * then.+ */+function hardwareContextResolver(+ forceSoftware: boolean,+): (codecId: number) => Promise<HardwareContext | null> {+ let hw: HardwareContext | null | undefined;+ const verdicts = new Map<number, Promise<HardwareContext | null>>();++ const resolve = async (codecId: number): Promise<HardwareContext | null> => {+ const av = await loadNodeAv();+ if (forceSoftware || hardwareDisabled()) return null;+ if (hw === undefined) hw = av.HardwareContext.auto();+ if (!hw) return null;+ const encoder = hw.getEncoderCodec(codecId as never);+ // Decode-only hardware: the encoder falls back to software on its own.+ if (!encoder) return hw;+ if (await encoderWorks(av, encoder)) {+ rendererLogger.info(+ { encoder: encoder.name, device: hw.deviceTypeName },+ "[video-encoders] Using hardware video encoder",+ );+ return hw;+ }+ rendererLogger.warn(+ { encoder: encoder.name, device: hw.deviceTypeName },+ "[video-encoders] Hardware video encoder unusable; encoding in software",+ );+ return null;+ };++ return (codecId) => {+ let verdict = verdicts.get(codecId);+ if (!verdict) {+ // A failed hardware probe means software; a failed load surfaces.+ verdict = resolve(codecId).catch(async (err) => {+ await loadNodeAv();+ rendererLogger.warn(+ { err: err instanceof Error ? err.message : String(err) },+ "[video-encoders] Hardware probe failed; encoding in software",+ );+ return null;+ });+ verdicts.set(codecId, verdict);+ }+ return verdict;+ };+}++/**+ * Options for registerMediabunnyServer: hardware video codecs when they work,+ * software (libx264, libx265, libvpx, SVT-AV1) otherwise. `forceSoftware`+ * keeps every codec in software (used on AMD). Create encoders with+ * "prefer-hardware" so this decides; "prefer-software" skips it.+ */+export function videoCodecServerOptions(forceSoftware: boolean): {+ hardwareContext: (codecId: number) => Promise<HardwareContext | null>;+} {+ return { hardwareContext: hardwareContextResolver(forceSoftware) };+}diff --git a/packages/webgpu-renderers/src/device.ts b/packages/webgpu-renderers/src/device.tsindex 657b3885..4c496ce9 100644--- a/packages/webgpu-renderers/src/device.ts+++ b/packages/webgpu-renderers/src/device.ts@@ -2,6 +2,45 @@ let deviceInstance: GPUDevice | null = null;let loadPromise: Promise<GPUDevice> | null = null;+/** The adapter behind the rendering device. */+export interface GpuAdapterDescription {+ vendor: string;+ architecture: string;+ device: string;+ description: string;+ /** A CPU implementation (lavapipe/llvmpipe, SwiftShader, WARP). */+ software: boolean;+}++let adapterInfo: GpuAdapterDescription | null = null;++const SOFTWARE_ADAPTER =+ /llvmpipe|lavapipe|softpipe|swiftshader|\bwarp\b|software/i;++function readAdapterInfo(adapter: any): GpuAdapterDescription {+ const info = adapter.info ?? {};+ const out = {+ vendor: String(info.vendor ?? ""),+ architecture: String(info.architecture ?? ""),+ device: String(info.device ?? ""),+ description: String(info.description ?? ""),+ };+ const software =+ info.isFallbackAdapter === true ||+ adapter.isFallbackAdapter === true ||+ SOFTWARE_ADAPTER.test(+ `${out.vendor} ${out.architecture} ${out.device} ${out.description}`,+ );+ return { ...out, software };+}++/** Initialises the rendering device if needed and describes its adapter. */+export async function describeGpu(): Promise<GpuAdapterDescription> {+ await ensureDevice();+ if (!adapterInfo) throw new Error("WebGPU adapter unknown");+ return adapterInfo;+}+export async function acquireDevice(opts?: GPURequestAdapterOptions,): Promise<GPUDevice> {@@ -28,11 +67,18 @@ export async function acquireDevice(});}}- const adapter = await (globalThis as any).navigator.gpu.requestAdapter({- powerPreference: "high-performance",- ...opts,- });+ const gpu = (globalThis as any).navigator.gpu;+ // No GPU: fall back to a software adapter (e.g. lavapipe, SwiftShader).+ const adapter =+ (await gpu.requestAdapter({+ powerPreference: "high-performance",+ ...opts,+ })) ??+ (opts?.forceFallbackAdapter+ ? null+ : await gpu.requestAdapter({ ...opts, forceFallbackAdapter: true }));if (!adapter) throw new Error("WebGPU not available");+ adapterInfo = readAdapterInfo(adapter);const requiredFeatures: string[] = [];if (adapter.features.has("float32-filterable")) {diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex adcb5b1b..2ec4f4b3 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -2774,6 +2774,9 @@ importers:mediabunny:specifier: ^1.51.0version: 1.51.0+ node-av:+ specifier: 6.0.0+ version: 6.0.0pino:specifier: ^10.3.1version: 10.3.1