Code-first video & image, rendered natively on WebGPU.
fix(audio): start every mix with fresh effect state
okanasl1064591c2d19
diff --git a/.changeset/audio-effect-state.md b/.changeset/audio-effect-state.mdnew file mode 100644index 00000000..c91b1410--- /dev/null+++ b/.changeset/audio-effect-state.md@@ -0,0 +1,5 @@+---+"framefields": patch+---++Audio effects start every mix from silence. Effect state on the GPU (delay lines, reverb tails, envelopes) was keyed by render id and carried into the next mix that reused one, so mixing again under the same id (the preview, block by block) heard the previous mix's echoes: a ping-pong delay was off by up to 0.43. `mixAudioTracks` now resets it first (`WebGPUAudioProcessor.resetState`, which keeps compiled pipelines), `renderAudio` frees each mix's state, and the video export's cleanup, which passed no device and so freed nothing, now releases it.diff --git a/packages/compositions/src/utils/audio-extractor.ts b/packages/compositions/src/utils/audio-extractor.tsindex 6780485b..44414103 100644--- a/packages/compositions/src/utils/audio-extractor.ts+++ b/packages/compositions/src/utils/audio-extractor.ts@@ -12,6 +12,7 @@ import {inputStore,resolveInputUrl,shaderStore,+ WebGPUAudioProcessor,} from "@framefields/webgpu-renderers";import { AudioSampleSink } from "mediabunny";import { computeRenderParams } from "./apply-operations.js";@@ -681,6 +682,11 @@ export async function mixAudioTracks();await waitForDelayRender();+ // Effects keep state on the GPU (delay lines, reverb tails) that carries+ // across the chunks of one mix. A mix starts from silence, also when a+ // caller mixes again under the same render id (the preview, block by block).+ if (device) WebGPUAudioProcessor.resetState(renderId ?? "global", device);+// Register all signals from the composition treeshaderStore.registerSignalsFromComposition(renderId, virtualMedia);diff --git a/packages/framefields/src/audio-mix-window.test.ts b/packages/framefields/src/audio-mix-window.test.tsindex 675ac64f..66f25f0d 100644--- a/packages/framefields/src/audio-mix-window.test.ts+++ b/packages/framefields/src/audio-mix-window.test.ts@@ -6,8 +6,10 @@import fs from "node:fs/promises";import os from "node:os";import path from "node:path";+import { mixAudioTracks } from "@framefields/compositions";+import { ensureDevice } from "@framefields/webgpu-renderers";import { beforeAll, describe, expect, it } from "vitest";-import { AudioFade, AudioReverb } from "./effects/index.js";+import { AudioDelay, AudioFade, AudioReverb } from "./effects/index.js";import { Composition, encodeStereoWav, Layer } from "./index.js";import { HeadlessMediaRenderer } from "./renderer/index.js";@@ -132,4 +134,32 @@ describe("windowed audio mix", () => {1e-3,);}, 60_000);++ it("starts every mix with fresh effect state, whatever its render id", async () => {+ // A ping-pong delay keeps its echoes on the GPU between calls. The+ // preview mixes block after block under one render id: each mix must+ // still start from silence, as an export does.+ const comp = new Composition({+ width: 64,+ height: 64,+ fps: FPS,+ durationMs: 2000,+ });+ comp.addAudio(+ Layer.audio(source, { durationFrames: 60 }).apply(+ new AudioDelay({ delayTime: 0.25, feedback: 0.6, pingPong: true }),+ ),+ );+ const vm = comp.toVirtualMedia();+ await mix(comp); // registers the audio processors+ const device = await ensureDevice();+ const once = () =>+ mixAudioTracks(vm, FPS, SR, device, "preview-audio", { master: false });+ const first = await once();+ const second = await once();+ expect(maxDiff(first.channels, second.channels, 0)).toBe(0);+ // And the same as an export, which gets a fresh render id.+ const exported = await mix(comp);+ expect(maxDiff(exported.channels, first.channels, 0)).toBe(0);+ }, 60_000);});diff --git a/packages/renderer/src/headless-webgpu-renderer.ts b/packages/renderer/src/headless-webgpu-renderer.tsindex d54f2323..ad7e738e 100644--- a/packages/renderer/src/headless-webgpu-renderer.ts+++ b/packages/renderer/src/headless-webgpu-renderer.ts@@ -991,14 +991,20 @@ export class HeadlessWebGPURenderer {await renderSemaphore.acquire();try {const device = await ensureDevice();- return await mixAudioTracks(- virtualMedia,- fps,- options.sampleRate ?? 48_000,- device,- `aud-${randomUUID()}`,- { window: options.window, master: options.master },- );+ const renderId = `aud-${randomUUID()}`;+ try {+ return await mixAudioTracks(+ virtualMedia,+ fps,+ options.sampleRate ?? 48_000,+ device,+ renderId,+ { window: options.window, master: options.master },+ );+ } finally {+ // The mix's effect state (delay lines, reverb tails) on the GPU.+ WebGPUAudioProcessor.clearCache(renderId, device);+ }} finally {renderSemaphore.release();}@@ -1450,6 +1456,8 @@ export class HeadlessWebGPURenderer {let outputFinalized = false;// Resources for inline rendering (cleaned up in finally block)+ // The device this render's audio effects keep their state on.+ let gpu: GPUDevice | undefined;let surface: NodeSurfaceProvider | undefined;let renderer: Renderer2D | undefined;let pipeline: ZeroCopyWebCodecsPipeline | undefined;@@ -1457,6 +1465,7 @@ export class HeadlessWebGPURenderer {try {// Inside the try, so a device failure still releases the slot.const device = await ensureDevice();+ gpu = device;await output.start();outputStarted = true;await preloadFonts(virtualMedia, device);@@ -1700,7 +1709,7 @@ export class HeadlessWebGPURenderer {} catch {}try {- WebGPUAudioProcessor.clearCache(renderId);+ WebGPUAudioProcessor.clearCache(renderId, gpu);} catch {}try {shaderStore.clear(renderId);diff --git a/packages/webgpu-renderers/src/audio/webgpu-audio-processor.ts b/packages/webgpu-renderers/src/audio/webgpu-audio-processor.tsindex 24279329..b40cbb3d 100644--- a/packages/webgpu-renderers/src/audio/webgpu-audio-processor.ts+++ b/packages/webgpu-renderers/src/audio/webgpu-audio-processor.ts@@ -96,6 +96,25 @@ export class WebGPUAudioProcessor {return deviceCache?.get(cacheKey);}+ /**+ * Zeroes the state every effect keeps for `renderId` (delay lines,+ * envelopes, reverb tails), as on its first use. Pipelines and buffers+ * stay, so a mix that starts again doesn't compile them again.+ */+ public static resetState(renderId: string, device: GPUDevice): void {+ const cache = WebGPUAudioProcessor.devicePipelineCache.get(device);+ if (!cache) return;+ const prefix = `${renderId}_`;+ for (const [key, cached] of cache.entries()) {+ if (!key.startsWith(prefix)) continue;+ device.queue.writeBuffer(+ cached.stateBuffer,+ 0,+ new Float32Array(cached.stateBuffer.size / 4),+ );+ }+ }+public static clearCache(renderId: string, device?: GPUDevice): void {if (!device) return;const cache = WebGPUAudioProcessor.devicePipelineCache.get(device);