Code-first video & image, rendered natively on WebGPU.
fix(3d): layers sharing a mesh keep their own audio deformation
okanasl5f2ff2efc9bd
diff --git a/.changeset/audio-deform-shared-mesh.md b/.changeset/audio-deform-shared-mesh.mdnew file mode 100644index 00000000..698e6846--- /dev/null+++ b/.changeset/audio-deform-shared-mesh.md@@ -0,0 +1,5 @@+---+"framefields": patch+---++Layers that share one mesh with `deformWithAudio` keep their own deformation: each dispatch writes its own output, where every layer used to draw whichever was deformed last. Audio deformation also streaks under camera motion blur.diff --git a/packages/framefields/src/audio-deform-motion-render.test.ts b/packages/framefields/src/audio-deform-motion-render.test.tsnew file mode 100644index 00000000..ea8067d1--- /dev/null+++ b/packages/framefields/src/audio-deform-motion-render.test.ts@@ -0,0 +1,136 @@+/**+ * Audio-deformed meshes through the real renderer: layers sharing one mesh+ * keep their own deformation (each dispatch has its own output; before,+ * every layer drew the last one's), and motion blur streaks the deformation+ * itself, the same after a seek as in order.+ */+import fs from "node:fs/promises";+import os from "node:os";+import path from "node:path";+import { afterAll, beforeAll, describe, expect, it } from "vitest";+import { Composition, Layer, Layer3D, type LayoutNode } from "./index.js";+import { computeMse, decodePngPixels } from "./test-utils/pixels.js";++const SIZE = 400;+const C = SIZE / 2;+const FPS = 30;+/** One second of silence: ripple and twist still move by time alone. */+const silence = new Float32Array(48000);++let dir = "";+let ribbon = "";++/** A flat ribbon facing the camera, `w` × `h`, cut into `rows` horizontal strips. */+function ribbonObj(w: number, h: number, rows: number): string {+ let out = "";+ for (let j = 0; j <= rows; j++) {+ const y = (j / rows - 0.5) * h;+ for (const x of [-w / 2, 0, w / 2])+ out += `v ${x} ${y} 0\nvt ${x / w + 0.5} ${j / rows}\nvn 0 0 -1\n`;+ }+ for (let j = 0; j < rows; j++)+ for (let i = 0; i < 2; i++) {+ const a = j * 3 + i + 1;+ const b = a + 1;+ const c = a + 4;+ const d = a + 3;+ out += `f ${a}/${a}/${a} ${b}/${b}/${b} ${c}/${c}/${c}\n`;+ out += `f ${a}/${a}/${a} ${c}/${c}/${c} ${d}/${d}/${d}\n`;+ }+ return out;+}++beforeAll(async () => {+ dir = await fs.mkdtemp(path.join(os.tmpdir(), "framefields-audio-deform-"));+ ribbon = path.join(dir, "ribbon.obj");+ await fs.writeFile(ribbon, ribbonObj(120, 240, 24));+});+afterAll(async () => {+ await fs.rm(dir, { recursive: true, force: true });+});++function scene(layers: LayoutNode[], motionBlur: boolean): Composition {+ const comp = new Composition({+ width: SIZE,+ height: SIZE,+ fps: FPS,+ durationFrames: 30,+ backgroundColor: "#000000",+ });+ comp.add([+ Layer.camera({+ x: C,+ y: C,+ z: -C / Math.tan((25 * Math.PI) / 180),+ targetX: C,+ targetY: C,+ targetZ: 0,+ motionBlur,+ }),+ ...layers,+ ]);+ return comp;+}++const deformed = (+ id: string,+ x: number,+ mode: "twist" | "ripple",+ amplitudeMultiplier: number,+) =>+ Layer3D.obj(ribbon, {+ id,+ x,+ y: C,+ z: 0,+ color: "#ffffff",+ material: "unlit",+ }).deformWithAudio({+ audioData: [silence],+ mode,+ amplitudeMultiplier,+ }) as LayoutNode;++/** Lit pixels in row `y` between x0 and x1. */+function width(pixels: Uint8Array, y: number, x0: number, x1: number) {+ let n = 0;+ for (let x = x0; x < x1; x++)+ if ((pixels[(y * SIZE + x) * 4] ?? 0) > 128) n++;+ return n;+}++describe("audio-deformed meshes", () => {+ it("keeps each layer's own deformation of a shared mesh", async () => {+ // Same mesh: the left one untwisted, the right one twisted hard.+ const comp = scene(+ [+ deformed("flat", C - 90, "twist", 0),+ deformed("twisted", C + 90, "twist", 150),+ ],+ false,+ );+ const { pixels } = await decodePngPixels(+ await comp.renderFrame({ frame: 0 }),+ );+ // Near the top the twisted ribbon has turned edge-on; the flat one has not.+ const top = C - 100;+ const flat = width(pixels, top, 0, C);+ const twisted = width(pixels, top, C, SIZE);+ expect(flat).toBeGreaterThan(100);+ expect(twisted).toBeLessThan(flat * 0.8);+ });++ it("streaks a deformation by its own motion, the same after a seek", async () => {+ const ripple = () => [deformed("ripple", C, "ripple", 6)];+ const sharp = await decodePngPixels(+ await scene(ripple(), false).renderFrame({ frame: 15 }),+ );+ const blurredPng = await scene(ripple(), true).renderFrame({ frame: 15 });+ const blurred = await decodePngPixels(blurredPng);+ // The mesh holds still; only its vertices move, and now they blur.+ expect(computeMse(sharp.pixels, blurred.pixels)).toBeGreaterThan(0.5);+ const inOrder = scene(ripple(), true);+ for (const f of [12, 13, 14]) await inOrder.renderFrame({ frame: f });+ expect(await inOrder.renderFrame({ frame: 15 })).toEqual(blurredPng);+ });+});diff --git a/packages/webgpu-renderers/src/renderer3d/audio-mesh-deform-pipeline.ts b/packages/webgpu-renderers/src/renderer3d/audio-mesh-deform-pipeline.tsindex 48d02ff7..43632192 100644--- a/packages/webgpu-renderers/src/renderer3d/audio-mesh-deform-pipeline.ts+++ b/packages/webgpu-renderers/src/renderer3d/audio-mesh-deform-pipeline.ts@@ -30,7 +30,12 @@ export class AudioMeshDeformPipeline {public pipeline: GPUComputePipeline;public bindGroupLayout: GPUBindGroupLayout;private uniformPool: BufferPool;- private deformedBuffers = new WeakMap<GPUBuffer, GPUBuffer>();+ /**+ * One output per dispatch this frame: layers sharing a mesh deform it+ * differently, and each draw (and motion blur's copy) reads its own.+ */+ private outputs: GPUBuffer[] = [];+ private outputIndex = 0;// 8 floats = 32 bytes for DeformParams uniformprivate uniformData = new ArrayBuffer(32);@@ -90,24 +95,25 @@ export class AudioMeshDeformPipeline {});}- public getOrCreateDeformedBuffer(- srcBuffer: GPUBuffer,- byteLength: number,- ): GPUBuffer {- let dst = this.deformedBuffers.get(srcBuffer);+ /** The next output of this frame, at least `byteLength` bytes. */+ private nextOutput(byteLength: number): GPUBuffer {+ let dst = this.outputs[this.outputIndex];if (!dst || dst.size < byteLength) {- dst?.destroy();+ if (dst) {+ const old = dst;+ this.device.queue.onSubmittedWorkDone().then(() => old.destroy());+ }dst = this.device.createBuffer({label: "DeformedMeshVertexBuffer",size: Math.max(64, Math.ceil(byteLength / 4) * 4),usage:GPUBufferUsage.VERTEX |GPUBufferUsage.STORAGE |- GPUBufferUsage.COPY_DST |GPUBufferUsage.COPY_SRC,});- this.deformedBuffers.set(srcBuffer, dst);+ this.outputs[this.outputIndex] = dst;}+ this.outputIndex++;return dst;}@@ -124,10 +130,7 @@ export class AudioMeshDeformPipeline {sampleRate = 48000,} = options;const byteLength = vertexCount * 64; // 16 floats * 4 bytes- const dstVertexBuffer = this.getOrCreateDeformedBuffer(- srcVertexBuffer,- byteLength,- );+ const dstVertexBuffer = this.nextOutput(byteLength);const modeInt = DEFORM_MODES[config.mode] ?? 0;@@ -193,9 +196,11 @@ export class AudioMeshDeformPipeline {public resetPools(): void {this.uniformPool.reset();+ this.outputIndex = 0;}public destroy(): void {this.uniformPool.destroy();+ for (const output of this.outputs) output.destroy();}}