Skip to content

Code-first video & image, rendered natively on WebGPU.

perf(render): non-blocking VRAM telemetry, configurable DMA ring, surface/decode-buffer reuse

okanasl51b0daf51867
diff --git a/.changeset/render-perf-headless.md b/.changeset/render-perf-headless.md
new file mode 100644
index 00000000..56697ef3
--- /dev/null
+++ b/.changeset/render-perf-headless.md
@@ -0,0 +1,12 @@
+---
+"framefields": patch
+"@framefields/renderer": patch
+"@framefields/webgpu-renderers": patch
+---
+
+Rendering performance for the headless / server-GPU path:
+
+- VRAM telemetry no longer blocks the event loop. The renderer's 5-second stats probe ran `nvidia-smi` with `execSync`; it is now sampled asynchronously and cached, so it can no longer stall the frame loop and DMA callbacks on NVIDIA hosts.
+- `renderVideo` accepts `ringCapacity` (also `FRAMEFIELDS_RING_CAPACITY`, clamped 2–8) to tune the DMA staging ring depth. The default stays at 2.
+- `renderImage` reuses render surfaces across stills instead of allocating and destroying the native render target and readback staging buffer on every call.
+- Headless video decode reuses a single RGBA staging buffer per decoder instead of allocating a full-frame buffer for every decoded frame.
diff --git a/packages/framefields/src/index.ts b/packages/framefields/src/index.ts
index 6c09fc21..8890c714 100644
--- a/packages/framefields/src/index.ts
+++ b/packages/framefields/src/index.ts
@@ -866,6 +866,13 @@ export interface RenderVideoOptions {
audioCodec?: string;
quality?: string;
concurrency?: number;
+ /**
+ * Depth of the DMA staging ring used to overlap GPU rendering with hardware
+ * encoding (default 2, clamped 2–8). A deeper ring hides more encoder
+ * latency at the cost of VRAM (~8.3MB/slot at 1080p RGBA). Also settable
+ * via `FRAMEFIELDS_RING_CAPACITY`.
+ */
+ ringCapacity?: number;
outputPath?: string;
renderer?: HeadlessMediaRenderer;
/**
diff --git a/packages/renderer/src/headless-webgpu-renderer.ts b/packages/renderer/src/headless-webgpu-renderer.ts
index f1b835d1..dd8d14ef 100644
--- a/packages/renderer/src/headless-webgpu-renderer.ts
+++ b/packages/renderer/src/headless-webgpu-renderer.ts
@@ -1,4 +1,4 @@
-import { execSync, spawn } from "node:child_process";
+import { exec, spawn } from "node:child_process";
import { randomUUID } from "node:crypto";
import fs from "node:fs/promises";
import os from "node:os";
@@ -64,9 +64,9 @@ import {
mediaDecoderCache,
NodeSurfaceProvider,
type RenderContextValue,
- Renderer2D,
- shaderStore,
+ type Renderer2D,
SlugFontCache,
+ shaderStore,
textureCache,
WebGPUAudioProcessor,
} from "@framefields/webgpu-renderers";
@@ -371,6 +371,131 @@ function resolveQuality(quality?: string) {
}
}
+// ─── VRAM telemetry ────────────────────────────────────────────────────────────
+//
+// nvidia-smi is probed asynchronously and cached. The previous implementation
+// ran `execSync` from inside the 5s stats interval, which blocked the event
+// loop — and therefore the frame loop and DMA map callbacks — for tens of ms on
+// exactly the NVIDIA hosts that render the most. `getVramUsage()` now returns
+// the latest sample; `refreshVramUsage()` refreshes it off the event loop.
+
+let cachedVramUsage = "N/A";
+let vramProbeInFlight = false;
+
+function refreshVramUsage(): void {
+ if (vramProbeInFlight) return;
+ vramProbeInFlight = true;
+ exec(
+ "nvidia-smi --query-gpu=memory.used,memory.total --format=csv,noheader,nounits",
+ { encoding: "utf-8", timeout: 2000 },
+ (err, stdout) => {
+ vramProbeInFlight = false;
+ if (err || !stdout) return;
+ const [used, total] = stdout
+ .trim()
+ .split(",")
+ .map((s) => s.trim());
+ if (used && total) {
+ cachedVramUsage = `${used}MB / ${total}MB`;
+ }
+ },
+ );
+}
+
+function getVramUsage(): string {
+ return cachedVramUsage;
+}
+
+// ─── DMA staging ring depth ────────────────────────────────────────────────────
+//
+// The ring overlaps GPU rendering of frame i+1 with the DMA readback and
+// hardware encode of frame i. A deeper ring hides more encoder variance at the
+// cost of VRAM (~8.3MB/slot at 1080p RGBA). Default stays at 2 so behavior is
+// unchanged unless a caller or the environment opts in.
+
+const DEFAULT_RING_CAPACITY = 2;
+const MIN_RING_CAPACITY = 2;
+const MAX_RING_CAPACITY = 8;
+
+function resolveRingCapacity(explicit?: number): number {
+ const fromEnv = Number(process.env.FRAMEFIELDS_RING_CAPACITY);
+ const requested =
+ explicit ?? (Number.isFinite(fromEnv) && fromEnv > 0 ? fromEnv : NaN);
+ if (!Number.isFinite(requested) || requested <= 0) {
+ return DEFAULT_RING_CAPACITY;
+ }
+ return Math.max(
+ MIN_RING_CAPACITY,
+ Math.min(MAX_RING_CAPACITY, Math.floor(requested)),
+ );
+}
+
+// ─── Reusable render surface pool ──────────────────────────────────────────────
+//
+// `renderImage` is called once per still frame (frame-by-frame exports,
+// scrubbing, the renderer-service image path). Allocating and destroying the
+// native render target + readback staging buffer on every call is pure overhead:
+// the surface is cleared at the start of each frame, so it can be reused.
+// Surfaces are only checked out while the render semaphore is held, so two
+// concurrent renders never share one.
+
+class NodeSurfacePool {
+ private readonly idle: Array<{
+ device: GPUDevice;
+ surface: NodeSurfaceProvider;
+ }> = [];
+ private readonly maxIdle = 2;
+
+ acquire(
+ device: GPUDevice,
+ width: number,
+ height: number,
+ ): NodeSurfaceProvider {
+ const index = this.idle.findIndex(
+ (entry) =>
+ entry.device === device &&
+ entry.surface.width === width &&
+ entry.surface.height === height,
+ );
+ if (index >= 0) {
+ const [entry] = this.idle.splice(index, 1);
+ if (entry) return entry.surface;
+ }
+ // Drop any surfaces belonging to a previous device (it was lost and
+ // `ensureDevice` created a new one); they are no longer usable.
+ for (let i = this.idle.length - 1; i >= 0; i--) {
+ const entry = this.idle[i];
+ if (entry && entry.device !== device) {
+ this.idle.splice(i, 1);
+ try {
+ entry.surface.destroy();
+ } catch {}
+ }
+ }
+ return new NodeSurfaceProvider(device, width, height);
+ }
+
+ release(device: GPUDevice, surface: NodeSurfaceProvider): void {
+ if (this.idle.length >= this.maxIdle) {
+ try {
+ surface.destroy();
+ } catch {}
+ return;
+ }
+ this.idle.push({ device, surface });
+ }
+
+ destroy(): void {
+ for (const entry of this.idle.splice(0)) {
+ try {
+ entry.surface.destroy();
+ } catch {}
+ }
+ }
+}
+
+const nodeSurfacePool = new NodeSurfacePool();
+
function formatLutAsCube(lut: {
points: Array<[number, number, number]>;
size: number;
@@ -491,7 +616,7 @@ export class HeadlessWebGPURenderer {
try {
await preloadFonts(virtualMedia, device);
- surface = new NodeSurfaceProvider(device, width, height);
+ surface = nodeSurfacePool.acquire(device, width, height);
renderer = getRenderer2D(device, surface.colorFormat);
const ctx: RenderContextValue = { device, renderer, surface };
updateClockSignals(
@@ -610,10 +735,10 @@ export class HeadlessWebGPURenderer {
// "UnsupportedInputFormatError". renderImage is called repeatedly for
// multi-frame exports, so keep the decoder (and the singleton WebGPU
// device) alive to reuse + seek across frames — mirroring renderVideo.
- // Per-frame textures / surfaces are still released to bound VRAM.
- try {
- surface?.destroy();
- } catch {}
+ // The render surface (target texture + readback buffer) returns to the
+ // pool instead of being destroyed, so repeated stills skip the native
+ // allocation. Cached media textures are still released to bound VRAM.
+ if (surface) nodeSurfacePool.release(device, surface);
try {
textureCache.destroy();
} catch {}
@@ -895,6 +1020,13 @@ export class HeadlessWebGPURenderer {
audioCodec?: string;
quality?: string;
concurrency?: number;
+ /**
+ * Depth of the DMA staging ring (default 2, clamped 2–8). A deeper
+ * ring overlaps more encoder latency at the cost of VRAM
+ * (~8.3MB/slot at 1080p RGBA). Also settable via
+ * `FRAMEFIELDS_RING_CAPACITY`.
+ */
+ ringCapacity?: number;
/**
* Measure the output while it renders (black and frozen frames,
* dropped frames, loudness, clipping, silence, document warnings)
@@ -1008,29 +1140,12 @@ export class HeadlessWebGPURenderer {
return `Process: ${rssMb}MB RSS (${heapUsedMb}MB/${heapTotalMb}MB Heap), System: ${sysUsedMb}MB/${sysTotalMb}MB`;
};
- const getVramUsage = () => {
- try {
- const out = execSync(
- "nvidia-smi --query-gpu=memory.used,memory.total --format=csv,noheader,nounits",
- {
- encoding: "utf-8",
- stdio: ["ignore", "pipe", "ignore"],
- },
- );
- const [used, total] = out
- .trim()
- .split(",")
- .map((s) => s.trim());
- if (used && total) {
- return `${used}MB / ${total}MB`;
- }
- } catch {}
- return "N/A";
- };
-
let statsInterval: NodeJS.Timeout | undefined;
if (process.env.NODE_ENV === "production") {
statsInterval = setInterval(() => {
+ // Kick off an async sample; the log line reports the previous one
+ // rather than blocking the event loop to gather a fresh value.
+ refreshVramUsage();
rendererLogger.info(
{
renderId,
@@ -1095,7 +1210,7 @@ export class HeadlessWebGPURenderer {
height,
fps,
videoSource,
- ringCapacity: 2,
+ ringCapacity: resolveRingCapacity(options?.ringCapacity),
onFrame: inspector
? (rgba) => inspector.inspect(rgba, width, height)
: undefined,
diff --git a/packages/webgpu-renderers/src/nodes/media-decoder.ts b/packages/webgpu-renderers/src/nodes/media-decoder.ts
index 2a92afd4..696a1e61 100644
--- a/packages/webgpu-renderers/src/nodes/media-decoder.ts
+++ b/packages/webgpu-renderers/src/nodes/media-decoder.ts
@@ -86,6 +86,14 @@ export class VideoDecoder extends BaseMediaDecoder<
private nextSample: VideoSample | null = null;
private headlessQueue: VideoSample[] = [];
private isDecoding = false;
+ /**
+ * Reused RGBA staging array for headless decode. Only one decode is in
+ * flight per decoder (`isDecoding` serializes them) and callers consume the
+ * pixels synchronously (uploadFrameToTexture → writeTexture copies before
+ * the next decode can run), so the array is never read after it is
+ * overwritten. Avoids a full-frame allocation on every decoded frame.
+ */
+ private decodeBuffer: Uint8Array | null = null;
private nextRequest: {
timestampSec: number;
resolve: (value: VideoFrameData | null) => void;
@@ -148,6 +156,7 @@ export class VideoDecoder extends BaseMediaDecoder<
} catch (_) {}
}
this.headlessQueue = [];
+ this.decodeBuffer = null;
this.iteratorStartTimestamp = -1;
}
@@ -301,7 +310,12 @@ export class VideoDecoder extends BaseMediaDecoder<
const width = currentSample.displayWidth;
const height = currentSample.displayHeight;
- const buffer = new Uint8Array(width * height * 4);
+ const byteLength = width * height * 4;
+ let buffer = this.decodeBuffer;
+ if (!buffer || buffer.length !== byteLength) {
+ buffer = new Uint8Array(byteLength);
+ this.decodeBuffer = buffer;
+ }
await currentSample.copyTo(buffer, { format: "RGBA" });
if (this.currentFetchId !== fetchId || this.destroyed || !this.input)
return null;