Skip to content

Code-first video & image, rendered natively on WebGPU.

fix(3d): layers sharing a mesh keep their own audio deformation

okanasl5f2ff2efc9bd
diff --git a/.changeset/audio-deform-shared-mesh.md b/.changeset/audio-deform-shared-mesh.md
new file mode 100644
index 00000000..698e6846
--- /dev/null
+++ b/.changeset/audio-deform-shared-mesh.md
@@ -0,0 +1,5 @@
+---
+"framefields": patch
+---
+
+Layers that share one mesh with `deformWithAudio` keep their own deformation: each dispatch writes its own output, where every layer used to draw whichever was deformed last. Audio deformation also streaks under camera motion blur.
diff --git a/packages/framefields/src/audio-deform-motion-render.test.ts b/packages/framefields/src/audio-deform-motion-render.test.ts
new file mode 100644
index 00000000..ea8067d1
--- /dev/null
+++ b/packages/framefields/src/audio-deform-motion-render.test.ts
@@ -0,0 +1,136 @@
+/**
+ * Audio-deformed meshes through the real renderer: layers sharing one mesh
+ * keep their own deformation (each dispatch has its own output; before,
+ * every layer drew the last one's), and motion blur streaks the deformation
+ * itself, the same after a seek as in order.
+ */
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+import { Composition, Layer, Layer3D, type LayoutNode } from "./index.js";
+import { computeMse, decodePngPixels } from "./test-utils/pixels.js";
+
+const SIZE = 400;
+const C = SIZE / 2;
+const FPS = 30;
+/** One second of silence: ripple and twist still move by time alone. */
+const silence = new Float32Array(48000);
+
+let dir = "";
+let ribbon = "";
+
+/** A flat ribbon facing the camera, `w` × `h`, cut into `rows` horizontal strips. */
+function ribbonObj(w: number, h: number, rows: number): string {
+ let out = "";
+ for (let j = 0; j <= rows; j++) {
+ const y = (j / rows - 0.5) * h;
+ for (const x of [-w / 2, 0, w / 2])
+ out += `v ${x} ${y} 0\nvt ${x / w + 0.5} ${j / rows}\nvn 0 0 -1\n`;
+ }
+ for (let j = 0; j < rows; j++)
+ for (let i = 0; i < 2; i++) {
+ const a = j * 3 + i + 1;
+ const b = a + 1;
+ const c = a + 4;
+ const d = a + 3;
+ out += `f ${a}/${a}/${a} ${b}/${b}/${b} ${c}/${c}/${c}\n`;
+ out += `f ${a}/${a}/${a} ${c}/${c}/${c} ${d}/${d}/${d}\n`;
+ }
+ return out;
+}
+
+beforeAll(async () => {
+ dir = await fs.mkdtemp(path.join(os.tmpdir(), "framefields-audio-deform-"));
+ ribbon = path.join(dir, "ribbon.obj");
+ await fs.writeFile(ribbon, ribbonObj(120, 240, 24));
+});
+afterAll(async () => {
+ await fs.rm(dir, { recursive: true, force: true });
+});
+
+function scene(layers: LayoutNode[], motionBlur: boolean): Composition {
+ const comp = new Composition({
+ width: SIZE,
+ height: SIZE,
+ fps: FPS,
+ durationFrames: 30,
+ backgroundColor: "#000000",
+ });
+ comp.add([
+ Layer.camera({
+ x: C,
+ y: C,
+ z: -C / Math.tan((25 * Math.PI) / 180),
+ targetX: C,
+ targetY: C,
+ targetZ: 0,
+ motionBlur,
+ }),
+ ...layers,
+ ]);
+ return comp;
+}
+
+const deformed = (
+ id: string,
+ x: number,
+ mode: "twist" | "ripple",
+ amplitudeMultiplier: number,
+) =>
+ Layer3D.obj(ribbon, {
+ id,
+ x,
+ y: C,
+ z: 0,
+ color: "#ffffff",
+ material: "unlit",
+ }).deformWithAudio({
+ audioData: [silence],
+ mode,
+ amplitudeMultiplier,
+ }) as LayoutNode;
+
+/** Lit pixels in row `y` between x0 and x1. */
+function width(pixels: Uint8Array, y: number, x0: number, x1: number) {
+ let n = 0;
+ for (let x = x0; x < x1; x++)
+ if ((pixels[(y * SIZE + x) * 4] ?? 0) > 128) n++;
+ return n;
+}
+
+describe("audio-deformed meshes", () => {
+ it("keeps each layer's own deformation of a shared mesh", async () => {
+ // Same mesh: the left one untwisted, the right one twisted hard.
+ const comp = scene(
+ [
+ deformed("flat", C - 90, "twist", 0),
+ deformed("twisted", C + 90, "twist", 150),
+ ],
+ false,
+ );
+ const { pixels } = await decodePngPixels(
+ await comp.renderFrame({ frame: 0 }),
+ );
+ // Near the top the twisted ribbon has turned edge-on; the flat one has not.
+ const top = C - 100;
+ const flat = width(pixels, top, 0, C);
+ const twisted = width(pixels, top, C, SIZE);
+ expect(flat).toBeGreaterThan(100);
+ expect(twisted).toBeLessThan(flat * 0.8);
+ });
+
+ it("streaks a deformation by its own motion, the same after a seek", async () => {
+ const ripple = () => [deformed("ripple", C, "ripple", 6)];
+ const sharp = await decodePngPixels(
+ await scene(ripple(), false).renderFrame({ frame: 15 }),
+ );
+ const blurredPng = await scene(ripple(), true).renderFrame({ frame: 15 });
+ const blurred = await decodePngPixels(blurredPng);
+ // The mesh holds still; only its vertices move, and now they blur.
+ expect(computeMse(sharp.pixels, blurred.pixels)).toBeGreaterThan(0.5);
+ const inOrder = scene(ripple(), true);
+ for (const f of [12, 13, 14]) await inOrder.renderFrame({ frame: f });
+ expect(await inOrder.renderFrame({ frame: 15 })).toEqual(blurredPng);
+ });
+});
diff --git a/packages/webgpu-renderers/src/renderer3d/audio-mesh-deform-pipeline.ts b/packages/webgpu-renderers/src/renderer3d/audio-mesh-deform-pipeline.ts
index 48d02ff7..43632192 100644
--- a/packages/webgpu-renderers/src/renderer3d/audio-mesh-deform-pipeline.ts
+++ b/packages/webgpu-renderers/src/renderer3d/audio-mesh-deform-pipeline.ts
@@ -30,7 +30,12 @@ export class AudioMeshDeformPipeline {
public pipeline: GPUComputePipeline;
public bindGroupLayout: GPUBindGroupLayout;
private uniformPool: BufferPool;
- private deformedBuffers = new WeakMap<GPUBuffer, GPUBuffer>();
+ /**
+ * One output per dispatch this frame: layers sharing a mesh deform it
+ * differently, and each draw (and motion blur's copy) reads its own.
+ */
+ private outputs: GPUBuffer[] = [];
+ private outputIndex = 0;
// 8 floats = 32 bytes for DeformParams uniform
private uniformData = new ArrayBuffer(32);
@@ -90,24 +95,25 @@ export class AudioMeshDeformPipeline {
});
}
- public getOrCreateDeformedBuffer(
- srcBuffer: GPUBuffer,
- byteLength: number,
- ): GPUBuffer {
- let dst = this.deformedBuffers.get(srcBuffer);
+ /** The next output of this frame, at least `byteLength` bytes. */
+ private nextOutput(byteLength: number): GPUBuffer {
+ let dst = this.outputs[this.outputIndex];
if (!dst || dst.size < byteLength) {
- dst?.destroy();
+ if (dst) {
+ const old = dst;
+ this.device.queue.onSubmittedWorkDone().then(() => old.destroy());
+ }
dst = this.device.createBuffer({
label: "DeformedMeshVertexBuffer",
size: Math.max(64, Math.ceil(byteLength / 4) * 4),
usage:
GPUBufferUsage.VERTEX |
GPUBufferUsage.STORAGE |
- GPUBufferUsage.COPY_DST |
GPUBufferUsage.COPY_SRC,
});
- this.deformedBuffers.set(srcBuffer, dst);
+ this.outputs[this.outputIndex] = dst;
}
+ this.outputIndex++;
return dst;
}
@@ -124,10 +130,7 @@ export class AudioMeshDeformPipeline {
sampleRate = 48000,
} = options;
const byteLength = vertexCount * 64; // 16 floats * 4 bytes
- const dstVertexBuffer = this.getOrCreateDeformedBuffer(
- srcVertexBuffer,
- byteLength,
- );
+ const dstVertexBuffer = this.nextOutput(byteLength);
const modeInt = DEFORM_MODES[config.mode] ?? 0;
@@ -193,9 +196,11 @@ export class AudioMeshDeformPipeline {
public resetPools(): void {
this.uniformPool.reset();
+ this.outputIndex = 0;
}
public destroy(): void {
this.uniformPool.destroy();
+ for (const output of this.outputs) output.destroy();
}
}