Skip to content

Code-first video & image, rendered natively on WebGPU.

fix(webgpu-renderers): resolve native memory leaks in headless lottie and media decoders

okanasl1c5919f46f58
diff --git a/.changeset/fix-headless-lottie-leak.md b/.changeset/fix-headless-lottie-leak.md
new file mode 100644
index 00000000..6a95f0bf
--- /dev/null
+++ b/.changeset/fix-headless-lottie-leak.md
@@ -0,0 +1,7 @@
+---
+"framefields": patch
+---
+
+Fix native C++ memory leaks in headless video and still renders:
+- Decouple headless Lottie rendering from `skia-canvas`, reading straight-alpha pixels directly from the underlying ThorVG WASM linear buffer (`_dotLottieCore.get_pixel_buffer()`) into an instance-reused upload buffer before writing to the WebGPU texture, eliminating megabytes of native heap leakage per frame.
+- Add frame-based eviction (`mediaDecoderCache.tick()`) for video and audio decoders of clips that ended over 120 frames ago, and pool still render IDs to prevent decoder re-instantiation across consecutive stills.
diff --git a/packages/renderer/src/headless-webgpu-renderer.ts b/packages/renderer/src/headless-webgpu-renderer.ts
index dc304891..d4a71b82 100644
--- a/packages/renderer/src/headless-webgpu-renderer.ts
+++ b/packages/renderer/src/headless-webgpu-renderer.ts
@@ -361,6 +361,19 @@ class RenderSemaphore {
export const renderSemaphore = new RenderSemaphore(MAX_CONCURRENT_RENDERS);
+/**
+ * Ids for still renders without a caller id. Media caches (video decoders,
+ * last-good frames) are keyed by render id, so a fresh id per still would
+ * open a new decoder for every frame of a frame-by-frame export. Stills
+ * borrow an idle id instead: consecutive stills reuse it (and its decoders);
+ * concurrent stills get distinct ones.
+ */
+const idleStillIds: string[] = [];
+const takeStillId = (): string => idleStillIds.pop() ?? `img-${randomUUID()}`;
+const returnStillId = (id: string): void => {
+ idleStillIds.push(id);
+};
+
// ─── Helpers ───────────────────────────────────────────────────────────────────
function toEvenDimension(n: number): number {
@@ -614,7 +627,8 @@ export class HeadlessWebGPURenderer {
// A caller rendering consecutive frames (e.g. a scrubbing preview) passes a
// stable id so compiled timelines are reused across frames, as renderVideo
// does. Never render two frames with the same id concurrently.
- const renderId = options.renderId ?? `img-${randomUUID()}`;
+ const pooledId = options.renderId ? undefined : takeStillId();
+ const renderId = options.renderId ?? (pooledId as string);
rendererLogger.debug(
{ renderId, frame, ...renderSemaphore.stats },
@@ -743,12 +757,14 @@ export class HeadlessWebGPURenderer {
try {
shaderStore.clear(renderId);
} catch {}
- // NOTE: do NOT mediaDecoderCache.destroy() or device.destroy()/reset here.
- // The mediabunny Input (native demuxer over file://) can only be created
- // ONCE per URL in a process — a 2nd Input fails format detection with
- // "UnsupportedInputFormatError". renderImage is called repeatedly for
- // multi-frame exports, so keep the decoder (and the singleton WebGPU
- // device) alive to reuse + seek across frames — mirroring renderVideo.
+ // Keep this render's decoders (and the singleton WebGPU device) alive:
+ // renderImage is called repeatedly for multi-frame exports, and the next
+ // still borrows the same id, so it reuses them and seeks. Decoders no
+ // still has used for a while are released by the per-frame tick.
+ try {
+ mediaDecoderCache.tick();
+ } catch {}
+ if (pooledId) returnStillId(pooledId);
// The render surface (target texture + readback buffer) returns to the
// pool instead of being destroyed, so repeated stills skip the native
// allocation. Cached media textures are still released to bound VRAM.
@@ -955,7 +971,8 @@ export class HeadlessWebGPURenderer {
lutSub();
} catch {}
try {
- mediaDecoderCache.destroy();
+ // Only this render's decoders: another render may be running.
+ mediaDecoderCache.clearNode(renderId);
clearAllVideoCache();
} catch {}
try {
@@ -1307,6 +1324,8 @@ export class HeadlessWebGPURenderer {
}
compositionStateStore.setState(renderId, i, fps, true);
updateClockSignals(i, fps, durationMs);
+ // Release decoders of clips that ended a while ago.
+ mediaDecoderCache.tick();
const encoder = device.createCommandEncoder();
const targetView = surface.getCurrentTextureView();
@@ -1483,7 +1502,8 @@ export class HeadlessWebGPURenderer {
surface?.destroy();
} catch {}
try {
- mediaDecoderCache.destroy();
+ // Only this render's decoders: another render may be running.
+ mediaDecoderCache.clearNode(renderId);
} catch {}
try {
clearAllVideoCache();
diff --git a/packages/renderer/src/video-worker.ts b/packages/renderer/src/video-worker.ts
index c93930f6..823511a0 100644
--- a/packages/renderer/src/video-worker.ts
+++ b/packages/renderer/src/video-worker.ts
@@ -327,6 +327,7 @@ if (parentPort) {
);
}
compositionStateStore.setState(renderId, i, fps, true);
+ mediaDecoderCache.tick();
const encoder = device.createCommandEncoder();
const targetView = surface.getCurrentTextureView();
diff --git a/packages/webgpu-renderers/src/nodes/lottie.test.ts b/packages/webgpu-renderers/src/nodes/lottie.test.ts
index c8354182..b4ee64ad 100644
--- a/packages/webgpu-renderers/src/nodes/lottie.test.ts
+++ b/packages/webgpu-renderers/src/nodes/lottie.test.ts
@@ -198,5 +198,30 @@ describe("Lottie Node", () => {
}),
);
});
+
+ it("should upload directly via writeTexture when _dotLottieCore returns pixel buffer", async () => {
+ const rawBuffer = new Uint8Array(200 * 200 * 4);
+ rawBuffer.fill(200);
+ (mockDotLottieInstance as Record<string, unknown>)._dotLottieCore = {
+ get_pixel_buffer: vi.fn().mockReturnValue(rawBuffer),
+ };
+
+ const props = {
+ src: "wasm-test.json",
+ frame: 0,
+ fps: 30,
+ dstRect: { x: 0, y: 0, width: 100, height: 100 },
+ };
+
+ await drawLottieNode(mockCtx, mockPass, props);
+
+ expect(mockDevice.queue.writeTexture).toHaveBeenCalledWith(
+ expect.objectContaining({ texture: expect.any(Object) }),
+ expect.any(Uint8Array),
+ expect.objectContaining({ bytesPerRow: 200 * 4 }),
+ [200, 200],
+ );
+ delete (mockDotLottieInstance as Record<string, unknown>)._dotLottieCore;
+ });
});
});
diff --git a/packages/webgpu-renderers/src/nodes/lottie.ts b/packages/webgpu-renderers/src/nodes/lottie.ts
index 51b61e4a..52a8d332 100644
--- a/packages/webgpu-renderers/src/nodes/lottie.ts
+++ b/packages/webgpu-renderers/src/nodes/lottie.ts
@@ -24,11 +24,17 @@ function decodeBase64(base64: string): string {
return atob(base64);
}
-function extractFontFamiliesFromLottie(lottieJson: any): string[] {
+function extractFontFamiliesFromLottie(
+ lottieJson: Record<string, unknown> | null | undefined,
+): string[] {
const fonts = new Set<string>();
// 1. Extract from fonts.list (both fName and fFamily)
- const fontsList = lottieJson?.fonts?.list;
+ const fontsList = (
+ lottieJson?.fonts as
+ | { list?: Array<{ fName?: string; fFamily?: string }> }
+ | undefined
+ )?.list;
if (Array.isArray(fontsList)) {
for (const entry of fontsList) {
if (entry?.fName && typeof entry.fName === "string") {
@@ -45,22 +51,28 @@ function extractFontFamiliesFromLottie(lottieJson: any): string[] {
if (!Array.isArray(layers)) return [...fonts];
for (const layer of layers) {
- if (layer?.ty !== 5) continue; // ty: 5 is Text Layer
+ const l = layer as {
+ ty?: number;
+ t?: { d?: { k?: unknown }; k?: unknown };
+ };
+ if (l?.ty !== 5) continue; // ty: 5 is Text Layer
- const k = layer?.t?.d?.k ?? layer?.t?.k;
+ const k = l?.t?.d?.k ?? l?.t?.k;
if (!k) continue;
const kArr = Array.isArray(k) ? k : [k];
for (const kf of kArr) {
// Handle keyframe wrappers (s, e) AND direct document objects (kf)
- const possibleDocs = [kf?.s, kf?.e, kf].filter(Boolean);
+ const kfObj = kf as { s?: unknown; e?: unknown } | undefined;
+ const possibleDocs = [kfObj?.s, kfObj?.e, kf].filter(Boolean);
for (const doc of possibleDocs) {
// Handle both single document objects and arrays of documents
const docArr = Array.isArray(doc) ? doc : [doc];
for (const d of docArr) {
- if (d?.f && typeof d.f === "string") {
- fonts.add(d.f);
+ const docItem = d as { f?: string } | undefined;
+ if (docItem?.f && typeof docItem.f === "string") {
+ fonts.add(docItem.f);
}
}
}
@@ -110,7 +122,7 @@ async function registerLottieFont(family: string): Promise<void> {
if (isBrowser) {
try {
await localFontManager.loadFont(family, fontUrl);
- } catch (err) {
+ } catch (_err) {
// Fallback to original family name
fontUrl = GetFontAssetUrl(family);
try {
@@ -193,12 +205,59 @@ async function registerLottieFont(family: string): Promise<void> {
}
}
-class LottieInstance {
+class HeadlessLottieCanvas {
+ public width: number;
+ public height: number;
+ public style: Record<string, unknown> = {};
+
+ constructor(width: number, height: number) {
+ this.width = width;
+ this.height = height;
+ }
+
+ addEventListener(): void {}
+ removeEventListener(): void {}
+ getContext(contextId: string): unknown {
+ if (contextId === "2d") {
+ return {
+ fillRect(): void {},
+ clearRect(): void {},
+ putImageData(): void {},
+ createImageData(width: number, height: number) {
+ return {
+ width,
+ height,
+ data: new Uint8ClampedArray(width * height * 4),
+ };
+ },
+ getImageData(_sx: number, _sy: number, sw: number, sh: number) {
+ return {
+ width: sw,
+ height: sh,
+ data: new Uint8ClampedArray(sw * sh * 4),
+ };
+ },
+ };
+ }
+ return null;
+ }
+ getBoundingClientRect(): {
+ left: number;
+ top: number;
+ width: number;
+ height: number;
+ } {
+ return { left: 0, top: 0, width: this.width, height: this.height };
+ }
+}
+
+export class LottieInstance {
private dotLottie: DotLottie;
private canvas: OffscreenCanvas;
private loadPromise: Promise<void>;
private _width: number;
private _height: number;
+ private uploadBuffer: Uint8Array | null = null;
constructor(
src: string,
@@ -208,7 +267,16 @@ class LottieInstance {
) {
this._width = width;
this._height = height;
- this.canvas = new OffscreenCanvas(width, height);
+ const isBrowser =
+ typeof window !== "undefined" &&
+ typeof document !== "undefined" &&
+ typeof document.createElement === "function" &&
+ typeof document.getElementById === "function";
+
+ this.canvas = isBrowser
+ ? new OffscreenCanvas(width, height)
+ : (new HeadlessLottieCanvas(width, height) as unknown as OffscreenCanvas);
+
this.dotLottie = new DotLottie({
canvas: this.canvas,
...(data ? { data } : { src }),
@@ -264,6 +332,45 @@ class LottieInstance {
await this.loadPromise;
}
+ renderPixels(
+ frame: number,
+ fit?: "contain" | "cover" | "fill",
+ ): Uint8Array | null {
+ const loopedFrame = Math.floor(frame) % Math.max(1, this.totalFrames);
+ this.dotLottie.setLayout({
+ align: [0.5, 0.5],
+ fit: (fit || "contain") as Fit,
+ });
+ this.dotLottie.resize();
+ this.dotLottie.setFrame(loopedFrame);
+ try {
+ (this.dotLottie as { render?: () => void }).render?.();
+ } catch (_) {}
+
+ const core = (
+ this.dotLottie as unknown as {
+ _dotLottieCore?: { get_pixel_buffer: () => Uint8Array };
+ }
+ )._dotLottieCore;
+
+ if (core && typeof core.get_pixel_buffer === "function") {
+ const buf = core.get_pixel_buffer();
+ const expectedLength = this._width * this._height * 4;
+ if (buf && buf.byteLength === expectedLength) {
+ if (
+ !this.uploadBuffer ||
+ this.uploadBuffer.byteLength !== expectedLength
+ ) {
+ this.uploadBuffer = new Uint8Array(expectedLength);
+ }
+ this.uploadBuffer.set(buf);
+ return premultiplyPixels(this.uploadBuffer);
+ }
+ }
+
+ return null;
+ }
+
renderFrame(
frame: number,
fit?: "contain" | "cover" | "fill",
@@ -285,6 +392,7 @@ class LottieInstance {
try {
this.dotLottie.destroy();
} catch (_) {}
+ this.uploadBuffer = null;
}
}
@@ -343,13 +451,8 @@ async function ensureWasmConfigured(): Promise<void> {
if (!isBrowser) {
if (typeof globalThis.OffscreenCanvas === "undefined") {
- try {
- const { Canvas } = await import(
- /* webpackIgnore: true */ "skia-canvas"
- );
- (globalThis as unknown as Record<string, unknown>).OffscreenCanvas =
- Canvas;
- } catch {}
+ (globalThis as unknown as Record<string, unknown>).OffscreenCanvas =
+ HeadlessLottieCanvas;
}
polyfillHeadlessDOM();
// Add Response instanceof patch to handle cross-realm ESM loader issues in worker threads
@@ -432,7 +535,7 @@ class LottieLoader {
const resolvedSrc = src;
let lottieData: string | ArrayBuffer | undefined;
try {
- let lottieJson: any = null;
+ let lottieJson: Record<string, unknown> | null = null;
const isNode =
typeof process !== "undefined" && Boolean(process.versions?.node);
const isRemote =
@@ -465,7 +568,9 @@ class LottieLoader {
filePath = pathModule.resolve(process.cwd(), filePath);
}
try {
- const fs = await import(/* webpackIgnore: true */ "fs/promises");
+ const fs = await import(
+ /* webpackIgnore: true */ "node:fs/promises"
+ );
if (filePath.endsWith(".json")) {
const jsonContent = await fs.readFile(filePath, "utf-8");
lottieJson = JSON.parse(jsonContent);
@@ -556,8 +661,6 @@ export async function drawLottieNode(
let tex = textureCache.acquire(cacheKey);
if (!tex) {
- const canvas = instance.renderFrame(loopedFrame, fitMode);
-
tex = ctx.device.createTexture({
size: [instance.width, instance.height],
format: "rgba8unorm",
@@ -567,72 +670,42 @@ export async function drawLottieNode(
GPUTextureUsage.RENDER_ATTACHMENT,
});
- try {
- ctx.device.queue.copyExternalImageToTexture(
- { source: canvas as unknown as GPUImageCopyExternalImageSource },
- { texture: tex, premultipliedAlpha: true },
+ const pixelBuffer = instance.renderPixels(loopedFrame, fitMode);
+ if (pixelBuffer) {
+ ctx.device.queue.writeTexture(
+ { texture: tex },
+ pixelBuffer,
+ { bytesPerRow: instance.width * 4 },
[instance.width, instance.height],
);
- } catch {
- const c2d = canvas.getContext("2d");
- if (c2d) {
- const imgData = (
- c2d as unknown as OffscreenCanvasRenderingContext2D
- ).getImageData(0, 0, instance.width, instance.height);
- // getImageData returns straight alpha.
- const pixelBuffer = premultiplyPixels(
- new Uint8Array(
- imgData.data.buffer,
- imgData.data.byteOffset,
- imgData.data.byteLength,
- ),
- );
- ctx.device.queue.writeTexture(
- { texture: tex },
- pixelBuffer,
- { bytesPerRow: instance.width * 4 },
+ } else {
+ const canvas = instance.renderFrame(loopedFrame, fitMode);
+ try {
+ ctx.device.queue.copyExternalImageToTexture(
+ { source: canvas as unknown as GPUImageCopyExternalImageSource },
+ { texture: tex, premultipliedAlpha: true },
[instance.width, instance.height],
);
- } else {
- const gl =
- canvas.getContext("webgl") ||
- canvas.getContext(
- "experimental-webgl" as unknown as OffscreenRenderingContextId,
+ } catch {
+ const c2d = canvas.getContext("2d");
+ if (c2d) {
+ const imgData = (
+ c2d as unknown as OffscreenCanvasRenderingContext2D
+ ).getImageData(0, 0, instance.width, instance.height);
+ // getImageData returns straight alpha.
+ const fallbackBuffer = premultiplyPixels(
+ new Uint8Array(
+ imgData.data.buffer,
+ imgData.data.byteOffset,
+ imgData.data.byteLength,
+ ),
);
- if (gl) {
- const pixels = new Uint8Array(instance.width * instance.height * 4);
- (gl as any).readPixels(
- 0,
- 0,
- instance.width,
- instance.height,
- (gl as any).RGBA,
- (gl as any).UNSIGNED_BYTE,
- pixels,
- );
-
- // WebGL pixels are flipped vertically, so flip them back for WebGPU/2D compatibility
- const flipped = new Uint8Array(pixels.length);
- const bytesPerRow = instance.width * 4;
- for (let y = 0; y < instance.height; y++) {
- const srcOffset = y * bytesPerRow;
- const dstOffset = (instance.height - 1 - y) * bytesPerRow;
- flipped.set(
- pixels.subarray(srcOffset, srcOffset + bytesPerRow),
- dstOffset,
- );
- }
-
ctx.device.queue.writeTexture(
{ texture: tex },
- flipped,
- { bytesPerRow: bytesPerRow },
+ fallbackBuffer,
+ { bytesPerRow: instance.width * 4 },
[instance.width, instance.height],
);
- } else {
- console.error(
- "[LottieRenderer] Could not acquire 2D or WebGL context to read pixels.",
- );
}
}
}
diff --git a/packages/webgpu-renderers/src/nodes/media-decoder.test.ts b/packages/webgpu-renderers/src/nodes/media-decoder.test.ts
index 22430999..93c97368 100644
--- a/packages/webgpu-renderers/src/nodes/media-decoder.test.ts
+++ b/packages/webgpu-renderers/src/nodes/media-decoder.test.ts
@@ -178,3 +178,36 @@ describe("Media Decoder", () => {
});
});
});
+
+describe("mediaDecoderCache frame-based release", () => {
+ beforeEach(() => {
+ mediaDecoderCache.destroy();
+ });
+
+ it("releases a decoder no layer has drawn for the idle window", () => {
+ mediaDecoderCache.getVideo("file:///a.mp4", true, "render-1-l0");
+ mediaDecoderCache.getVideo("file:///b.mp4", true, "render-1-l1");
+ expect(mediaDecoderCache.size).toBe(2);
+ // l1 keeps being drawn; l0's clip has ended.
+ for (let i = 0; i < 130; i++) {
+ mediaDecoderCache.getVideo("file:///b.mp4", true, "render-1-l1");
+ mediaDecoderCache.tick();
+ }
+ expect(mediaDecoderCache.size).toBe(1);
+ });
+
+ it("keeps decoders that are used every frame", () => {
+ for (let i = 0; i < 500; i++) {
+ mediaDecoderCache.getVideo("file:///a.mp4", true, "still-pool-0");
+ mediaDecoderCache.tick();
+ }
+ expect(mediaDecoderCache.size).toBe(1);
+ });
+
+ it("clearNode drops only the given render's decoders", () => {
+ mediaDecoderCache.getVideo("file:///a.mp4", true, "vid-1-l0");
+ mediaDecoderCache.getVideo("file:///a.mp4", true, "vid-2-l0");
+ mediaDecoderCache.clearNode("vid-1");
+ expect(mediaDecoderCache.size).toBe(1);
+ });
+});
diff --git a/packages/webgpu-renderers/src/nodes/media-decoder.ts b/packages/webgpu-renderers/src/nodes/media-decoder.ts
index 619bacdd..65a37e1a 100644
--- a/packages/webgpu-renderers/src/nodes/media-decoder.ts
+++ b/packages/webgpu-renderers/src/nodes/media-decoder.ts
@@ -490,9 +490,45 @@ class MediaDecoderCache {
private videoDecoders = new Map<string, VideoDecoder>();
private audioDecoders = new Map<string, AudioDecoder>();
private lastAccessed = new Map<string, number>();
+ /** Rendered frames so far (see `tick`), and the frame each decoder was last used on. */
+ private ticks = 0;
+ private lastTick = new Map<string, number>();
private touchKey(key: string) {
this.lastAccessed.set(key, Date.now());
+ this.lastTick.set(key, this.ticks);
+ }
+
+ private dropKey(key: string): void {
+ const videoDecoder = this.videoDecoders.get(key);
+ if (videoDecoder) {
+ try {
+ videoDecoder.destroy();
+ } catch (_) {}
+ this.videoDecoders.delete(key);
+ }
+ const audioDecoder = this.audioDecoders.get(key);
+ if (audioDecoder) {
+ try {
+ audioDecoder.destroy();
+ } catch (_) {}
+ this.audioDecoders.delete(key);
+ }
+ this.lastAccessed.delete(key);
+ this.lastTick.delete(key);
+ }
+
+ /**
+ * Called once per rendered frame by headless renders. A decoder holds a
+ * full-frame RGBA buffer and native decoder state, so one that no layer has
+ * drawn for `idleFrames` frames (its clip ended, or a still render used a
+ * one-off id) is released instead of waiting for the wall-clock prune.
+ */
+ tick(idleFrames = 120): void {
+ this.ticks++;
+ for (const [key, last] of this.lastTick) {
+ if (this.ticks - last > idleFrames) this.dropKey(key);
+ }
}
destroy(): void {
@@ -510,24 +546,18 @@ class MediaDecoderCache {
}
this.audioDecoders.clear();
this.lastAccessed.clear();
+ this.lastTick.clear();
+ }
+
+ /** Decoders currently open (for tests and diagnostics). */
+ get size(): number {
+ return this.videoDecoders.size + this.audioDecoders.size;
}
private prune(maxAgeMs = 300000) {
const now = Date.now();
for (const [key, lastTime] of this.lastAccessed.entries()) {
- if (now - lastTime > maxAgeMs) {
- const videoDecoder = this.videoDecoders.get(key);
- if (videoDecoder) {
- videoDecoder.destroy();
- this.videoDecoders.delete(key);
- }
- const audioDecoder = this.audioDecoders.get(key);
- if (audioDecoder) {
- audioDecoder.destroy();
- this.audioDecoders.delete(key);
- }
- this.lastAccessed.delete(key);
- }
+ if (now - lastTime > maxAgeMs) this.dropKey(key);
}
}
@@ -556,19 +586,9 @@ class MediaDecoderCache {
}
clearNode(nodeId: string) {
- for (const [key, decoder] of this.videoDecoders) {
- if (key.includes(nodeId)) {
- decoder.destroy();
- this.videoDecoders.delete(key);
- this.lastAccessed.delete(key);
- }
- }
- for (const [key, decoder] of this.audioDecoders) {
- if (key.includes(nodeId)) {
- decoder.destroy();
- this.audioDecoders.delete(key);
- this.lastAccessed.delete(key);
- }
+ const keys = [...this.videoDecoders.keys(), ...this.audioDecoders.keys()];
+ for (const key of keys) {
+ if (key.includes(nodeId)) this.dropKey(key);
}
}
}