Code-first video & image, rendered natively on WebGPU.
fix(webgpu-renderers): resolve native memory leaks in headless lottie and media decoders
okanasl1c5919f46f58
diff --git a/.changeset/fix-headless-lottie-leak.md b/.changeset/fix-headless-lottie-leak.mdnew file mode 100644index 00000000..6a95f0bf--- /dev/null+++ b/.changeset/fix-headless-lottie-leak.md@@ -0,0 +1,7 @@+---+"framefields": patch+---++Fix native C++ memory leaks in headless video and still renders:+- Decouple headless Lottie rendering from `skia-canvas`, reading straight-alpha pixels directly from the underlying ThorVG WASM linear buffer (`_dotLottieCore.get_pixel_buffer()`) into an instance-reused upload buffer before writing to the WebGPU texture, eliminating megabytes of native heap leakage per frame.+- Add frame-based eviction (`mediaDecoderCache.tick()`) for video and audio decoders of clips that ended over 120 frames ago, and pool still render IDs to prevent decoder re-instantiation across consecutive stills.diff --git a/packages/renderer/src/headless-webgpu-renderer.ts b/packages/renderer/src/headless-webgpu-renderer.tsindex dc304891..d4a71b82 100644--- a/packages/renderer/src/headless-webgpu-renderer.ts+++ b/packages/renderer/src/headless-webgpu-renderer.ts@@ -361,6 +361,19 @@ class RenderSemaphore {export const renderSemaphore = new RenderSemaphore(MAX_CONCURRENT_RENDERS);+/**+ * Ids for still renders without a caller id. Media caches (video decoders,+ * last-good frames) are keyed by render id, so a fresh id per still would+ * open a new decoder for every frame of a frame-by-frame export. Stills+ * borrow an idle id instead: consecutive stills reuse it (and its decoders);+ * concurrent stills get distinct ones.+ */+const idleStillIds: string[] = [];+const takeStillId = (): string => idleStillIds.pop() ?? `img-${randomUUID()}`;+const returnStillId = (id: string): void => {+ idleStillIds.push(id);+};+// ─── Helpers ───────────────────────────────────────────────────────────────────function toEvenDimension(n: number): number {@@ -614,7 +627,8 @@ export class HeadlessWebGPURenderer {// A caller rendering consecutive frames (e.g. a scrubbing preview) passes a// stable id so compiled timelines are reused across frames, as renderVideo// does. Never render two frames with the same id concurrently.- const renderId = options.renderId ?? `img-${randomUUID()}`;+ const pooledId = options.renderId ? undefined : takeStillId();+ const renderId = options.renderId ?? (pooledId as string);rendererLogger.debug({ renderId, frame, ...renderSemaphore.stats },@@ -743,12 +757,14 @@ export class HeadlessWebGPURenderer {try {shaderStore.clear(renderId);} catch {}- // NOTE: do NOT mediaDecoderCache.destroy() or device.destroy()/reset here.- // The mediabunny Input (native demuxer over file://) can only be created- // ONCE per URL in a process — a 2nd Input fails format detection with- // "UnsupportedInputFormatError". renderImage is called repeatedly for- // multi-frame exports, so keep the decoder (and the singleton WebGPU- // device) alive to reuse + seek across frames — mirroring renderVideo.+ // Keep this render's decoders (and the singleton WebGPU device) alive:+ // renderImage is called repeatedly for multi-frame exports, and the next+ // still borrows the same id, so it reuses them and seeks. Decoders no+ // still has used for a while are released by the per-frame tick.+ try {+ mediaDecoderCache.tick();+ } catch {}+ if (pooledId) returnStillId(pooledId);// The render surface (target texture + readback buffer) returns to the// pool instead of being destroyed, so repeated stills skip the native// allocation. Cached media textures are still released to bound VRAM.@@ -955,7 +971,8 @@ export class HeadlessWebGPURenderer {lutSub();} catch {}try {- mediaDecoderCache.destroy();+ // Only this render's decoders: another render may be running.+ mediaDecoderCache.clearNode(renderId);clearAllVideoCache();} catch {}try {@@ -1307,6 +1324,8 @@ export class HeadlessWebGPURenderer {}compositionStateStore.setState(renderId, i, fps, true);updateClockSignals(i, fps, durationMs);+ // Release decoders of clips that ended a while ago.+ mediaDecoderCache.tick();const encoder = device.createCommandEncoder();const targetView = surface.getCurrentTextureView();@@ -1483,7 +1502,8 @@ export class HeadlessWebGPURenderer {surface?.destroy();} catch {}try {- mediaDecoderCache.destroy();+ // Only this render's decoders: another render may be running.+ mediaDecoderCache.clearNode(renderId);} catch {}try {clearAllVideoCache();diff --git a/packages/renderer/src/video-worker.ts b/packages/renderer/src/video-worker.tsindex c93930f6..823511a0 100644--- a/packages/renderer/src/video-worker.ts+++ b/packages/renderer/src/video-worker.ts@@ -327,6 +327,7 @@ if (parentPort) {);}compositionStateStore.setState(renderId, i, fps, true);+ mediaDecoderCache.tick();const encoder = device.createCommandEncoder();const targetView = surface.getCurrentTextureView();diff --git a/packages/webgpu-renderers/src/nodes/lottie.test.ts b/packages/webgpu-renderers/src/nodes/lottie.test.tsindex c8354182..b4ee64ad 100644--- a/packages/webgpu-renderers/src/nodes/lottie.test.ts+++ b/packages/webgpu-renderers/src/nodes/lottie.test.ts@@ -198,5 +198,30 @@ describe("Lottie Node", () => {}),);});++ it("should upload directly via writeTexture when _dotLottieCore returns pixel buffer", async () => {+ const rawBuffer = new Uint8Array(200 * 200 * 4);+ rawBuffer.fill(200);+ (mockDotLottieInstance as Record<string, unknown>)._dotLottieCore = {+ get_pixel_buffer: vi.fn().mockReturnValue(rawBuffer),+ };++ const props = {+ src: "wasm-test.json",+ frame: 0,+ fps: 30,+ dstRect: { x: 0, y: 0, width: 100, height: 100 },+ };++ await drawLottieNode(mockCtx, mockPass, props);++ expect(mockDevice.queue.writeTexture).toHaveBeenCalledWith(+ expect.objectContaining({ texture: expect.any(Object) }),+ expect.any(Uint8Array),+ expect.objectContaining({ bytesPerRow: 200 * 4 }),+ [200, 200],+ );+ delete (mockDotLottieInstance as Record<string, unknown>)._dotLottieCore;+ });});});diff --git a/packages/webgpu-renderers/src/nodes/lottie.ts b/packages/webgpu-renderers/src/nodes/lottie.tsindex 51b61e4a..52a8d332 100644--- a/packages/webgpu-renderers/src/nodes/lottie.ts+++ b/packages/webgpu-renderers/src/nodes/lottie.ts@@ -24,11 +24,17 @@ function decodeBase64(base64: string): string {return atob(base64);}-function extractFontFamiliesFromLottie(lottieJson: any): string[] {+function extractFontFamiliesFromLottie(+ lottieJson: Record<string, unknown> | null | undefined,+): string[] {const fonts = new Set<string>();// 1. Extract from fonts.list (both fName and fFamily)- const fontsList = lottieJson?.fonts?.list;+ const fontsList = (+ lottieJson?.fonts as+ | { list?: Array<{ fName?: string; fFamily?: string }> }+ | undefined+ )?.list;if (Array.isArray(fontsList)) {for (const entry of fontsList) {if (entry?.fName && typeof entry.fName === "string") {@@ -45,22 +51,28 @@ function extractFontFamiliesFromLottie(lottieJson: any): string[] {if (!Array.isArray(layers)) return [...fonts];for (const layer of layers) {- if (layer?.ty !== 5) continue; // ty: 5 is Text Layer+ const l = layer as {+ ty?: number;+ t?: { d?: { k?: unknown }; k?: unknown };+ };+ if (l?.ty !== 5) continue; // ty: 5 is Text Layer- const k = layer?.t?.d?.k ?? layer?.t?.k;+ const k = l?.t?.d?.k ?? l?.t?.k;if (!k) continue;const kArr = Array.isArray(k) ? k : [k];for (const kf of kArr) {// Handle keyframe wrappers (s, e) AND direct document objects (kf)- const possibleDocs = [kf?.s, kf?.e, kf].filter(Boolean);+ const kfObj = kf as { s?: unknown; e?: unknown } | undefined;+ const possibleDocs = [kfObj?.s, kfObj?.e, kf].filter(Boolean);for (const doc of possibleDocs) {// Handle both single document objects and arrays of documentsconst docArr = Array.isArray(doc) ? doc : [doc];for (const d of docArr) {- if (d?.f && typeof d.f === "string") {- fonts.add(d.f);+ const docItem = d as { f?: string } | undefined;+ if (docItem?.f && typeof docItem.f === "string") {+ fonts.add(docItem.f);}}}@@ -110,7 +122,7 @@ async function registerLottieFont(family: string): Promise<void> {if (isBrowser) {try {await localFontManager.loadFont(family, fontUrl);- } catch (err) {+ } catch (_err) {// Fallback to original family namefontUrl = GetFontAssetUrl(family);try {@@ -193,12 +205,59 @@ async function registerLottieFont(family: string): Promise<void> {}}-class LottieInstance {+class HeadlessLottieCanvas {+ public width: number;+ public height: number;+ public style: Record<string, unknown> = {};++ constructor(width: number, height: number) {+ this.width = width;+ this.height = height;+ }++ addEventListener(): void {}+ removeEventListener(): void {}+ getContext(contextId: string): unknown {+ if (contextId === "2d") {+ return {+ fillRect(): void {},+ clearRect(): void {},+ putImageData(): void {},+ createImageData(width: number, height: number) {+ return {+ width,+ height,+ data: new Uint8ClampedArray(width * height * 4),+ };+ },+ getImageData(_sx: number, _sy: number, sw: number, sh: number) {+ return {+ width: sw,+ height: sh,+ data: new Uint8ClampedArray(sw * sh * 4),+ };+ },+ };+ }+ return null;+ }+ getBoundingClientRect(): {+ left: number;+ top: number;+ width: number;+ height: number;+ } {+ return { left: 0, top: 0, width: this.width, height: this.height };+ }+}++export class LottieInstance {private dotLottie: DotLottie;private canvas: OffscreenCanvas;private loadPromise: Promise<void>;private _width: number;private _height: number;+ private uploadBuffer: Uint8Array | null = null;constructor(src: string,@@ -208,7 +267,16 @@ class LottieInstance {) {this._width = width;this._height = height;- this.canvas = new OffscreenCanvas(width, height);+ const isBrowser =+ typeof window !== "undefined" &&+ typeof document !== "undefined" &&+ typeof document.createElement === "function" &&+ typeof document.getElementById === "function";++ this.canvas = isBrowser+ ? new OffscreenCanvas(width, height)+ : (new HeadlessLottieCanvas(width, height) as unknown as OffscreenCanvas);+this.dotLottie = new DotLottie({canvas: this.canvas,...(data ? { data } : { src }),@@ -264,6 +332,45 @@ class LottieInstance {await this.loadPromise;}+ renderPixels(+ frame: number,+ fit?: "contain" | "cover" | "fill",+ ): Uint8Array | null {+ const loopedFrame = Math.floor(frame) % Math.max(1, this.totalFrames);+ this.dotLottie.setLayout({+ align: [0.5, 0.5],+ fit: (fit || "contain") as Fit,+ });+ this.dotLottie.resize();+ this.dotLottie.setFrame(loopedFrame);+ try {+ (this.dotLottie as { render?: () => void }).render?.();+ } catch (_) {}++ const core = (+ this.dotLottie as unknown as {+ _dotLottieCore?: { get_pixel_buffer: () => Uint8Array };+ }+ )._dotLottieCore;++ if (core && typeof core.get_pixel_buffer === "function") {+ const buf = core.get_pixel_buffer();+ const expectedLength = this._width * this._height * 4;+ if (buf && buf.byteLength === expectedLength) {+ if (+ !this.uploadBuffer ||+ this.uploadBuffer.byteLength !== expectedLength+ ) {+ this.uploadBuffer = new Uint8Array(expectedLength);+ }+ this.uploadBuffer.set(buf);+ return premultiplyPixels(this.uploadBuffer);+ }+ }++ return null;+ }+renderFrame(frame: number,fit?: "contain" | "cover" | "fill",@@ -285,6 +392,7 @@ class LottieInstance {try {this.dotLottie.destroy();} catch (_) {}+ this.uploadBuffer = null;}}@@ -343,13 +451,8 @@ async function ensureWasmConfigured(): Promise<void> {if (!isBrowser) {if (typeof globalThis.OffscreenCanvas === "undefined") {- try {- const { Canvas } = await import(- /* webpackIgnore: true */ "skia-canvas"- );- (globalThis as unknown as Record<string, unknown>).OffscreenCanvas =- Canvas;- } catch {}+ (globalThis as unknown as Record<string, unknown>).OffscreenCanvas =+ HeadlessLottieCanvas;}polyfillHeadlessDOM();// Add Response instanceof patch to handle cross-realm ESM loader issues in worker threads@@ -432,7 +535,7 @@ class LottieLoader {const resolvedSrc = src;let lottieData: string | ArrayBuffer | undefined;try {- let lottieJson: any = null;+ let lottieJson: Record<string, unknown> | null = null;const isNode =typeof process !== "undefined" && Boolean(process.versions?.node);const isRemote =@@ -465,7 +568,9 @@ class LottieLoader {filePath = pathModule.resolve(process.cwd(), filePath);}try {- const fs = await import(/* webpackIgnore: true */ "fs/promises");+ const fs = await import(+ /* webpackIgnore: true */ "node:fs/promises"+ );if (filePath.endsWith(".json")) {const jsonContent = await fs.readFile(filePath, "utf-8");lottieJson = JSON.parse(jsonContent);@@ -556,8 +661,6 @@ export async function drawLottieNode(let tex = textureCache.acquire(cacheKey);if (!tex) {- const canvas = instance.renderFrame(loopedFrame, fitMode);-tex = ctx.device.createTexture({size: [instance.width, instance.height],format: "rgba8unorm",@@ -567,72 +670,42 @@ export async function drawLottieNode(GPUTextureUsage.RENDER_ATTACHMENT,});- try {- ctx.device.queue.copyExternalImageToTexture(- { source: canvas as unknown as GPUImageCopyExternalImageSource },- { texture: tex, premultipliedAlpha: true },+ const pixelBuffer = instance.renderPixels(loopedFrame, fitMode);+ if (pixelBuffer) {+ ctx.device.queue.writeTexture(+ { texture: tex },+ pixelBuffer,+ { bytesPerRow: instance.width * 4 },[instance.width, instance.height],);- } catch {- const c2d = canvas.getContext("2d");- if (c2d) {- const imgData = (- c2d as unknown as OffscreenCanvasRenderingContext2D- ).getImageData(0, 0, instance.width, instance.height);- // getImageData returns straight alpha.- const pixelBuffer = premultiplyPixels(- new Uint8Array(- imgData.data.buffer,- imgData.data.byteOffset,- imgData.data.byteLength,- ),- );- ctx.device.queue.writeTexture(- { texture: tex },- pixelBuffer,- { bytesPerRow: instance.width * 4 },+ } else {+ const canvas = instance.renderFrame(loopedFrame, fitMode);+ try {+ ctx.device.queue.copyExternalImageToTexture(+ { source: canvas as unknown as GPUImageCopyExternalImageSource },+ { texture: tex, premultipliedAlpha: true },[instance.width, instance.height],);- } else {- const gl =- canvas.getContext("webgl") ||- canvas.getContext(- "experimental-webgl" as unknown as OffscreenRenderingContextId,+ } catch {+ const c2d = canvas.getContext("2d");+ if (c2d) {+ const imgData = (+ c2d as unknown as OffscreenCanvasRenderingContext2D+ ).getImageData(0, 0, instance.width, instance.height);+ // getImageData returns straight alpha.+ const fallbackBuffer = premultiplyPixels(+ new Uint8Array(+ imgData.data.buffer,+ imgData.data.byteOffset,+ imgData.data.byteLength,+ ),);- if (gl) {- const pixels = new Uint8Array(instance.width * instance.height * 4);- (gl as any).readPixels(- 0,- 0,- instance.width,- instance.height,- (gl as any).RGBA,- (gl as any).UNSIGNED_BYTE,- pixels,- );-- // WebGL pixels are flipped vertically, so flip them back for WebGPU/2D compatibility- const flipped = new Uint8Array(pixels.length);- const bytesPerRow = instance.width * 4;- for (let y = 0; y < instance.height; y++) {- const srcOffset = y * bytesPerRow;- const dstOffset = (instance.height - 1 - y) * bytesPerRow;- flipped.set(- pixels.subarray(srcOffset, srcOffset + bytesPerRow),- dstOffset,- );- }-ctx.device.queue.writeTexture({ texture: tex },- flipped,- { bytesPerRow: bytesPerRow },+ fallbackBuffer,+ { bytesPerRow: instance.width * 4 },[instance.width, instance.height],);- } else {- console.error(- "[LottieRenderer] Could not acquire 2D or WebGL context to read pixels.",- );}}}diff --git a/packages/webgpu-renderers/src/nodes/media-decoder.test.ts b/packages/webgpu-renderers/src/nodes/media-decoder.test.tsindex 22430999..93c97368 100644--- a/packages/webgpu-renderers/src/nodes/media-decoder.test.ts+++ b/packages/webgpu-renderers/src/nodes/media-decoder.test.ts@@ -178,3 +178,36 @@ describe("Media Decoder", () => {});});});++describe("mediaDecoderCache frame-based release", () => {+ beforeEach(() => {+ mediaDecoderCache.destroy();+ });++ it("releases a decoder no layer has drawn for the idle window", () => {+ mediaDecoderCache.getVideo("file:///a.mp4", true, "render-1-l0");+ mediaDecoderCache.getVideo("file:///b.mp4", true, "render-1-l1");+ expect(mediaDecoderCache.size).toBe(2);+ // l1 keeps being drawn; l0's clip has ended.+ for (let i = 0; i < 130; i++) {+ mediaDecoderCache.getVideo("file:///b.mp4", true, "render-1-l1");+ mediaDecoderCache.tick();+ }+ expect(mediaDecoderCache.size).toBe(1);+ });++ it("keeps decoders that are used every frame", () => {+ for (let i = 0; i < 500; i++) {+ mediaDecoderCache.getVideo("file:///a.mp4", true, "still-pool-0");+ mediaDecoderCache.tick();+ }+ expect(mediaDecoderCache.size).toBe(1);+ });++ it("clearNode drops only the given render's decoders", () => {+ mediaDecoderCache.getVideo("file:///a.mp4", true, "vid-1-l0");+ mediaDecoderCache.getVideo("file:///a.mp4", true, "vid-2-l0");+ mediaDecoderCache.clearNode("vid-1");+ expect(mediaDecoderCache.size).toBe(1);+ });+});diff --git a/packages/webgpu-renderers/src/nodes/media-decoder.ts b/packages/webgpu-renderers/src/nodes/media-decoder.tsindex 619bacdd..65a37e1a 100644--- a/packages/webgpu-renderers/src/nodes/media-decoder.ts+++ b/packages/webgpu-renderers/src/nodes/media-decoder.ts@@ -490,9 +490,45 @@ class MediaDecoderCache {private videoDecoders = new Map<string, VideoDecoder>();private audioDecoders = new Map<string, AudioDecoder>();private lastAccessed = new Map<string, number>();+ /** Rendered frames so far (see `tick`), and the frame each decoder was last used on. */+ private ticks = 0;+ private lastTick = new Map<string, number>();private touchKey(key: string) {this.lastAccessed.set(key, Date.now());+ this.lastTick.set(key, this.ticks);+ }++ private dropKey(key: string): void {+ const videoDecoder = this.videoDecoders.get(key);+ if (videoDecoder) {+ try {+ videoDecoder.destroy();+ } catch (_) {}+ this.videoDecoders.delete(key);+ }+ const audioDecoder = this.audioDecoders.get(key);+ if (audioDecoder) {+ try {+ audioDecoder.destroy();+ } catch (_) {}+ this.audioDecoders.delete(key);+ }+ this.lastAccessed.delete(key);+ this.lastTick.delete(key);+ }++ /**+ * Called once per rendered frame by headless renders. A decoder holds a+ * full-frame RGBA buffer and native decoder state, so one that no layer has+ * drawn for `idleFrames` frames (its clip ended, or a still render used a+ * one-off id) is released instead of waiting for the wall-clock prune.+ */+ tick(idleFrames = 120): void {+ this.ticks++;+ for (const [key, last] of this.lastTick) {+ if (this.ticks - last > idleFrames) this.dropKey(key);+ }}destroy(): void {@@ -510,24 +546,18 @@ class MediaDecoderCache {}this.audioDecoders.clear();this.lastAccessed.clear();+ this.lastTick.clear();+ }++ /** Decoders currently open (for tests and diagnostics). */+ get size(): number {+ return this.videoDecoders.size + this.audioDecoders.size;}private prune(maxAgeMs = 300000) {const now = Date.now();for (const [key, lastTime] of this.lastAccessed.entries()) {- if (now - lastTime > maxAgeMs) {- const videoDecoder = this.videoDecoders.get(key);- if (videoDecoder) {- videoDecoder.destroy();- this.videoDecoders.delete(key);- }- const audioDecoder = this.audioDecoders.get(key);- if (audioDecoder) {- audioDecoder.destroy();- this.audioDecoders.delete(key);- }- this.lastAccessed.delete(key);- }+ if (now - lastTime > maxAgeMs) this.dropKey(key);}}@@ -556,19 +586,9 @@ class MediaDecoderCache {}clearNode(nodeId: string) {- for (const [key, decoder] of this.videoDecoders) {- if (key.includes(nodeId)) {- decoder.destroy();- this.videoDecoders.delete(key);- this.lastAccessed.delete(key);- }- }- for (const [key, decoder] of this.audioDecoders) {- if (key.includes(nodeId)) {- decoder.destroy();- this.audioDecoders.delete(key);- this.lastAccessed.delete(key);- }+ const keys = [...this.videoDecoders.keys(), ...this.audioDecoders.keys()];+ for (const key of keys) {+ if (key.includes(nodeId)) this.dropKey(key);}}}