Code-first video & image, rendered natively on WebGPU.
signals for stereo
okanasl5644bf2347d2
diff --git a/examples/24_audio_capabilities/README.md b/examples/24_audio_capabilities/README.mdindex c9838628..e67bc849 100644--- a/examples/24_audio_capabilities/README.md+++ b/examples/24_audio_capabilities/README.md@@ -1,9 +1,14 @@# Framefields Audio Capabilities Example (`24_audio_capabilities`)-Demonstrates code-first WebGPU audio DSP and soundtrack composition in Framefields:+Demonstrates code-first WebGPU audio DSP, multi-track mixing, and procedural sound design in Framefields:+1. **Step 1: Baseline / Initial Audio** — Unprocessed score soundtrack loaded with explicit timing (`durationFrames`, `volume`).-2. **Step 2: Faded Audio** — WebGPU compute `AudioFade` DSP shader performing smooth envelope gain modulation (2.0s linear fade-in and 2.0s scurve fade-out).+2. **Step 2: Faded Audio** — WebGPU compute `AudioFade` DSP shader performing smooth envelope gain modulation (2.0s linear fade-in and 2.0s s-curve fade-out).3. **Step 3: Faded + Reverb Audio** — Multi-stage WebGPU audio DSP pipeline chaining `AudioFade` directly into `AudioReverb` (room size 0.85, wet mix 0.6, stereo damping 0.2).+4. **Step 4: Ping-Pong Delay** — WebGPU compute `AudioDelay` creating spatial bouncing stereo reflections (0.28s delay time, 45% feedback, 55% wet mix).+5. **Step 5: Parametric EQ Filter** — WebGPU compute `AudioParametricEq` biquad low-pass filter (750 Hz cutoff, Q factor 1.2).+6. **Step 6: Dynamics Compressor** — WebGPU compute `AudioCompressor` punchy dynamics control (-22 dB threshold, 6:1 ratio, 8 dB soft knee, +4 dB makeup gain).+7. **Step 7: Stereo Panning** — WebGPU compute `StereoPanning` with dynamic `Signal` modulation smoothly sweeping the soundstage from full left to full right across the clip (pan: -1.0 → +1.0).Each stage is exported both as uncompressed **16-bit PCM WAV** audio and as high-quality **MP4 video** with synchronized audio.@@ -12,7 +17,7 @@ Each stage is exported both as uncompressed **16-bit PCM WAV** audio and as high## Quick Start```bash-# Render all 3 audio steps (.wav) and encoded videos (.mp4)+# Render all 7 audio steps (.wav) and encoded videos (.mp4)pnpm --filter @framefields/example-24-audio-capabilities render# Render only WAV audio files@@ -33,6 +38,10 @@ When running `pnpm render`, files are written to `examples/24_audio_capabilities- `output/step1_initial_audio.wav` & `output/step1_initial.mp4`- `output/step2_faded_audio.wav` & `output/step2_faded.mp4`- `output/step3_faded_reverb_audio.wav` & `output/step3_faded_reverb.mp4`+- `output/step4_delay_audio.wav` & `output/step4_delay.mp4`+- `output/step5_parametric_eq_audio.wav` & `output/step5_parametric_eq.mp4`+- `output/step6_compressor_audio.wav` & `output/step6_compressor.mp4`+- `output/step7_stereo_pan_audio.wav` & `output/step7_stereo_pan.mp4`---@@ -41,13 +50,14 @@ When running `pnpm render`, files are written to `examples/24_audio_capabilities```tsimport { Composition, Layer, Effect } from "framefields";-const comp = new Composition({ width: 1280, height: 720, fps: 30, duration: 6 });+const comp = new Composition({ width: 1280, height: 720, fps: 30, durationMs: 6000 });-// Add audio soundtrack with chained DSP effects+// Add audio soundtrack with chained WebGPU DSP effectscomp.addAudio(Layer.audio("assets/score.mp3", { durationFrames: 180, volume: 1.0 }).apply(Effect.audioFade({ fadeInDuration: 2.0, fadeOutDuration: 2.0 })).apply(Effect.audioReverb({ roomSize: 0.85, damping: 0.2, wet: 0.6, dry: 0.7 }))+ .apply(Effect.audioDelay({ delayTime: 0.28, feedback: 0.45, pingPong: true })));// Render video with audiodiff --git a/examples/24_audio_capabilities/src/audio-pipeline.ts b/examples/24_audio_capabilities/src/audio-pipeline.tsindex 15bd4921..cb4da34c 100644--- a/examples/24_audio_capabilities/src/audio-pipeline.ts+++ b/examples/24_audio_capabilities/src/audio-pipeline.ts@@ -1,5 +1,5 @@import path from "node:path";-import { Composition, Effect, Layer } from "framefields";+import { Composition, Effect, Layer, Signal } from "framefields";export const FPS = 30;export const DURATION_SEC = 6;@@ -169,3 +169,162 @@ export function buildStep3FadedReverbAudio(): Composition {return comp;}++/**+ * Step 4: Ping-Pong Delay.+ * Applies WebGPU AudioDelay with spatial bouncing echoes.+ */+export function buildStep4DelayAudio(): Composition {+ const comp = new Composition({+ width: W,+ height: H,+ fps: FPS,+ durationMs: DURATION_MS,+ backgroundColor: "#0d0f12",+ });++ comp.add(+ createCard(+ "Step 4: Ping-Pong Delay",+ "WebGPU AudioDelay: 0.28s stereo delay time, 45% feedback, 55% wet",+ "SPATIAL ECHO DSP",+ ),+ );++ comp.addAudio(+ Layer.audio(SCORE_PATH, {+ id: "audio-step4-delay",+ durationFrames: DURATION_FRAMES,+ volume: 1.0,+ }).apply(+ Effect.audioDelay({+ delayTime: 0.28,+ feedback: 0.45,+ wet: 0.55,+ dry: 0.85,+ pingPong: true,+ }),+ ),+ );++ return comp;+}++/**+ * Step 5: Parametric EQ Filter.+ * Applies WebGPU AudioParametricEq low-pass biquad filter.+ */+export function buildStep5ParametricEqAudio(): Composition {+ const comp = new Composition({+ width: W,+ height: H,+ fps: FPS,+ durationMs: DURATION_MS,+ backgroundColor: "#0d0f12",+ });++ comp.add(+ createCard(+ "Step 5: Parametric Low-Pass EQ",+ "WebGPU AudioParametricEq: 750 Hz cutoff biquad low-pass filter (Q: 1.2)",+ "BIQUAD FILTER DSP",+ ),+ );++ comp.addAudio(+ Layer.audio(SCORE_PATH, {+ id: "audio-step5-eq",+ durationFrames: DURATION_FRAMES,+ volume: 1.0,+ }).apply(+ Effect.audioParametricEq({+ type: "lowPass",+ frequency: 750,+ q: 1.2,+ gain: 0,+ }),+ ),+ );++ return comp;+}++/**+ * Step 6: Dynamics Compressor.+ * Applies WebGPU AudioCompressor for punchy dynamics with soft-knee and makeup gain.+ */+export function buildStep6CompressorAudio(): Composition {+ const comp = new Composition({+ width: W,+ height: H,+ fps: FPS,+ durationMs: DURATION_MS,+ backgroundColor: "#0d0f12",+ });++ comp.add(+ createCard(+ "Step 6: Dynamics Compressor",+ "WebGPU AudioCompressor: -22 dB threshold, 6:1 ratio, +4 dB makeup gain",+ "DYNAMICS DSP",+ ),+ );++ comp.addAudio(+ Layer.audio(SCORE_PATH, {+ id: "audio-step6-compressor",+ durationFrames: DURATION_FRAMES,+ volume: 1.0,+ }).apply(+ Effect.audioCompressor({+ threshold: -22,+ ratio: 6,+ attack: 0.005,+ release: 0.15,+ knee: 8,+ makeupGain: 4,+ }),+ ),+ );++ return comp;+}++/**+ * Step 7: Stereo Panning.+ * Dynamic signal modulation sweeping soundstage from left to right (-1.0 to +1.0).+ */+export function buildStep7StereoPanningAudio(): Composition {+ const comp = new Composition({+ width: W,+ height: H,+ fps: FPS,+ durationMs: DURATION_MS,+ backgroundColor: "#0d0f12",+ });++ // Smooth linear progress sweep from hard left (-1.0) to hard right (+1.0)+ const panSignal = Signal.programmatic((ctx) => -1.0 + 2.0 * ctx.progress);++ comp.add(+ createCard(+ "Step 7: Stereo Panning",+ "WebGPU StereoPanning: dynamic signal sweep from left to right (pan: -1.0 → +1.0)",+ "STEREO PANNING",+ ),+ );++ comp.addAudio(+ Layer.audio(SCORE_PATH, {+ id: "audio-step7-pan",+ durationFrames: DURATION_FRAMES,+ volume: 1.0,+ }).apply(+ Effect.stereoPanning({+ pan: panSignal,+ }),+ ),+ );++ return comp;+}diff --git a/examples/24_audio_capabilities/src/audio.test.ts b/examples/24_audio_capabilities/src/audio.test.tsindex 6d9011d2..8715b9a4 100644--- a/examples/24_audio_capabilities/src/audio.test.ts+++ b/examples/24_audio_capabilities/src/audio.test.ts@@ -1,36 +1,39 @@-import { HeadlessMediaRenderer } from "framefields";+import { encodeStereoWav, HeadlessMediaRenderer } from "framefields";import { describe, expect, it } from "vitest";import {buildStep1InitialAudio,buildStep2FadedAudio,buildStep3FadedReverbAudio,+ buildStep4DelayAudio,+ buildStep5ParametricEqAudio,+ buildStep6CompressorAudio,+ buildStep7StereoPanningAudio,DURATION_FRAMES,DURATION_MS,FPS,SAMPLE_RATE,} from "./audio-pipeline.js";-import { pcmToWav } from "./wav.js";describe("Example 24: Audio Capabilities", () => {- it("compositions compile and export valid specs", () => {- const comp1 = buildStep1InitialAudio();- const comp2 = buildStep2FadedAudio();- const comp3 = buildStep3FadedReverbAudio();+ it("compositions compile and export valid specs across all 7 steps", () => {+ const steps = [+ buildStep1InitialAudio(),+ buildStep2FadedAudio(),+ buildStep3FadedReverbAudio(),+ buildStep4DelayAudio(),+ buildStep5ParametricEqAudio(),+ buildStep6CompressorAudio(),+ buildStep7StereoPanningAudio(),+ ];- expect(comp1.durationMs).toBe(DURATION_MS);- expect(comp2.durationMs).toBe(DURATION_MS);- expect(comp3.durationMs).toBe(DURATION_MS);-- const spec1 = comp1.toSpec();- const spec2 = comp2.toSpec();- const spec3 = comp3.toSpec();-- expect(spec1.layout.some((n) => n.kind === "media")).toBe(true);- expect(spec2.layout.some((n) => n.kind === "media")).toBe(true);- expect(spec3.layout.some((n) => n.kind === "media")).toBe(true);+ for (const comp of steps) {+ expect(comp.durationMs).toBe(DURATION_MS);+ const spec = comp.toSpec();+ expect(spec.layout.some((n) => n.kind === "media")).toBe(true);+ }});- it("renders audio PCM channels across all 3 steps", async () => {+ it("renders audio PCM channels and validates DSP effects", async () => {const renderer = new HeadlessMediaRenderer();const pcm1 = await renderer.renderAudio(@@ -45,23 +48,44 @@ describe("Example 24: Audio Capabilities", () => {buildStep3FadedReverbAudio().toVirtualMedia({ durationMs: DURATION_MS }),{ fps: FPS, sampleRate: SAMPLE_RATE },);+ const pcm4 = await renderer.renderAudio(+ buildStep4DelayAudio().toVirtualMedia({ durationMs: DURATION_MS }),+ { fps: FPS, sampleRate: SAMPLE_RATE },+ );+ const pcm5 = await renderer.renderAudio(+ buildStep5ParametricEqAudio().toVirtualMedia({ durationMs: DURATION_MS }),+ { fps: FPS, sampleRate: SAMPLE_RATE },+ );+ const pcm6 = await renderer.renderAudio(+ buildStep6CompressorAudio().toVirtualMedia({ durationMs: DURATION_MS }),+ { fps: FPS, sampleRate: SAMPLE_RATE },+ );+ const pcm7 = await renderer.renderAudio(+ buildStep7StereoPanningAudio().toVirtualMedia({+ durationMs: DURATION_MS,+ }),+ { fps: FPS, sampleRate: SAMPLE_RATE },+ );const expectedSamples = Math.ceil((DURATION_FRAMES / FPS) * SAMPLE_RATE);- expect(pcm1.channels.length).toBe(2);- expect(pcm2.channels.length).toBe(2);- expect(pcm3.channels.length).toBe(2);- expect(pcm1.channels[0].length).toBe(expectedSamples);- expect(pcm2.channels[0].length).toBe(expectedSamples);- expect(pcm3.channels[0].length).toBe(expectedSamples);+ const allPcms = [pcm1, pcm2, pcm3, pcm4, pcm5, pcm6, pcm7];++ for (const p of allPcms) {+ expect(p.channels.length).toBe(2);+ expect(p.channels[0].length).toBe(expectedSamples);+ expect(p.channels[1].length).toBe(expectedSamples);+ }- // Encode WAV and verify RIFF structure- const wav = pcmToWav(pcm1.channels, SAMPLE_RATE);+ // 1. Validate RIFF WAV output+ const wav = encodeStereoWav([pcm1.channels[0], pcm1.channels[1]], {+ sampleRate: SAMPLE_RATE,+ });expect(wav.subarray(0, 4).toString("ascii")).toBe("RIFF");expect(wav.subarray(8, 12).toString("ascii")).toBe("WAVE");expect(wav.subarray(12, 16).toString("ascii")).toBe("fmt ");expect(wav.subarray(36, 40).toString("ascii")).toBe("data");- // Calculate RMS in early fade-in window (0.0s to 1.5s)+ // 2. Validate Step 2 AudioFade attenuation in early window (0.0s to 1.5s)const earlyCount = Math.round(1.5 * SAMPLE_RATE);let sum1 = 0;let sum2 = 0;@@ -69,14 +93,12 @@ describe("Example 24: Audio Capabilities", () => {sum1 += (pcm1.channels[0][i] ?? 0) ** 2;sum2 += (pcm2.channels[0][i] ?? 0) ** 2;}- const rms1 = Math.sqrt(sum1 / earlyCount);- const rms2 = Math.sqrt(sum2 / earlyCount);-- // Fade-in should attenuate the early window- expect(rms2).toBeLessThan(rms1);+ expect(Math.sqrt(sum2 / earlyCount)).toBeLessThan(+ Math.sqrt(sum1 / earlyCount),+ );- // Step 3 (Reverb) should differ from Step 2 in the sustain/tail region- let diffCount = 0;+ // 3. Validate Step 3 Reverb tail difference vs Step 2+ let reverbDiff = 0;const midStart = Math.round(2.0 * SAMPLE_RATE);const midEnd = Math.round(4.0 * SAMPLE_RATE);for (let i = midStart; i < midEnd; i++) {@@ -84,9 +106,54 @@ describe("Example 24: Audio Capabilities", () => {Math.abs((pcm3.channels[0][i] ?? 0) - (pcm2.channels[0][i] ?? 0)) >0.001) {- diffCount++;+ reverbDiff++;}}- expect(diffCount).toBeGreaterThan(0);- });+ expect(reverbDiff).toBeGreaterThan(0);++ // 4. Validate Step 4 Ping-Pong Delay echo differences+ let delayDiff = 0;+ for (let i = midStart; i < midEnd; i++) {+ if (+ Math.abs((pcm4.channels[0][i] ?? 0) - (pcm1.channels[0][i] ?? 0)) >+ 0.001+ ) {+ delayDiff++;+ }+ }+ expect(delayDiff).toBeGreaterThan(0);++ // 5. Validate Step 5 Low-Pass EQ filters high frequencies+ let eqDiff = 0;+ for (let i = midStart; i < midEnd; i++) {+ if (+ Math.abs((pcm5.channels[0][i] ?? 0) - (pcm1.channels[0][i] ?? 0)) >+ 0.001+ ) {+ eqDiff++;+ }+ }+ expect(eqDiff).toBeGreaterThan(0);++ // 6. Validate Step 7 Stereo Panning dynamic sweep: early window panned left, late window panned right+ const earlyStart = Math.round(0.5 * SAMPLE_RATE);+ const earlyEnd = Math.round(1.5 * SAMPLE_RATE);+ let leftSumEarly = 0;+ let rightSumEarly = 0;+ for (let i = earlyStart; i < earlyEnd; i++) {+ leftSumEarly += (pcm7.channels[0][i] ?? 0) ** 2;+ rightSumEarly += (pcm7.channels[1][i] ?? 0) ** 2;+ }+ expect(leftSumEarly).toBeGreaterThan(rightSumEarly);++ const lateStart = Math.round(4.5 * SAMPLE_RATE);+ const lateEnd = Math.round(5.5 * SAMPLE_RATE);+ let leftSumLate = 0;+ let rightSumLate = 0;+ for (let i = lateStart; i < lateEnd; i++) {+ leftSumLate += (pcm7.channels[0][i] ?? 0) ** 2;+ rightSumLate += (pcm7.channels[1][i] ?? 0) ** 2;+ }+ expect(rightSumLate).toBeGreaterThan(leftSumLate);+ }, 30_000);});diff --git a/examples/24_audio_capabilities/src/render.ts b/examples/24_audio_capabilities/src/render.tsindex 794e0c8c..62c638ad 100644--- a/examples/24_audio_capabilities/src/render.ts+++ b/examples/24_audio_capabilities/src/render.ts@@ -4,27 +4,34 @@* 1. Step 1: Baseline / Initial Audio (score.mp3 without effects)* 2. Step 2: Faded Audio (with WebGPU AudioFade effect)* 3. Step 3: Faded + Reverb Audio (AudioFade + AudioReverb DSP chain)+ * 4. Step 4: Ping-Pong Delay (AudioDelay spatial stereo echo)+ * 5. Step 5: Parametric EQ Filter (AudioParametricEq biquad low-pass filter)+ * 6. Step 6: Dynamics Compressor (AudioCompressor dynamics limiter & makeup gain)+ * 7. Step 7: Stereo Panning (StereoPanning spatial soundstage balance)** Exports both uncompressed 16-bit WAV PCM files and encoded MP4 videos.** Usage:- * pnpm render # Renders all 3 audio steps (.wav) and videos (.mp4)+ * pnpm render # Renders all audio steps (.wav) and videos (.mp4)* pnpm render wav # Renders only WAV audio files*/import fs from "node:fs/promises";import path from "node:path";-import { HeadlessMediaRenderer } from "framefields";+import { encodeStereoWav, HeadlessMediaRenderer } from "framefields";import {buildStep1InitialAudio,buildStep2FadedAudio,buildStep3FadedReverbAudio,+ buildStep4DelayAudio,+ buildStep5ParametricEqAudio,+ buildStep6CompressorAudio,+ buildStep7StereoPanningAudio,DURATION_FRAMES,DURATION_MS,DURATION_SEC,FPS,SAMPLE_RATE,} from "./audio-pipeline.js";-import { pcmToWav } from "./wav.js";const OUTPUT_DIR = path.resolve(import.meta.dirname, "../output");const mode = process.argv[2] ?? "all";@@ -67,6 +74,34 @@ async function main() {description:"Multi-stage chain: AudioFade → AudioReverb (room 0.85, wet 0.6)",},+ {+ name: "Step 4: Ping-Pong Delay",+ slug: "step4_delay",+ builder: buildStep4DelayAudio,+ description:+ "WebGPU AudioDelay: spatial ping-pong stereo echo (0.28s, 45% feedback)",+ },+ {+ name: "Step 5: Parametric EQ Filter",+ slug: "step5_parametric_eq",+ builder: buildStep5ParametricEqAudio,+ description:+ "WebGPU AudioParametricEq: 750 Hz biquad low-pass filter (Q: 1.2)",+ },+ {+ name: "Step 6: Dynamics Compressor",+ slug: "step6_compressor",+ builder: buildStep6CompressorAudio,+ description:+ "WebGPU AudioCompressor: -22 dB threshold, 6:1 ratio, +4 dB makeup gain",+ },+ {+ name: "Step 7: Stereo Panning",+ slug: "step7_stereo_pan",+ builder: buildStep7StereoPanningAudio,+ description:+ "WebGPU StereoPanning: dynamic signal sweep from left to right (pan: -1.0 → +1.0)",+ },];console.log("=========================================================");@@ -91,20 +126,23 @@ async function main() {const pcmDuration = ((Date.now() - pcmStart) / 1000).toFixed(2);// 2. Export 16-bit WAV- const wavBuffer = pcmToWav(pcm.channels, SAMPLE_RATE);+ const left = pcm.channels[0] ?? new Float32Array(0);+ const right = pcm.channels[1] ?? left;+ const wavBuffer = encodeStereoWav([left, right], {+ sampleRate: SAMPLE_RATE,+ });const wavPath = path.join(OUTPUT_DIR, `${step.slug}_audio.wav`);await fs.writeFile(wavPath, wavBuffer);// Compute RMS metrics across key timeline windows- const ch0 = pcm.channels[0] ?? new Float32Array(0);- const fadeInRms = calculateRms(ch0, 0, Math.round(1.5 * SAMPLE_RATE));+ const fadeInRms = calculateRms(left, 0, Math.round(1.5 * SAMPLE_RATE));const sustainRms = calculateRms(- ch0,+ left,Math.round(2.5 * SAMPLE_RATE),Math.round(3.5 * SAMPLE_RATE),);const fadeOutRms = calculateRms(- ch0,+ left,Math.round(4.5 * SAMPLE_RATE),Math.round(6.0 * SAMPLE_RATE),);diff --git a/examples/24_audio_capabilities/src/wav.ts b/examples/24_audio_capabilities/src/wav.tsdeleted file mode 100644index 011c5db1..00000000--- a/examples/24_audio_capabilities/src/wav.ts+++ /dev/null@@ -1,49 +0,0 @@-/**- * Encodes stereo/mono Float32Array PCM channel buffers to a 16-bit PCM RIFF WAV buffer.- */-export function pcmToWav(- channels: readonly Float32Array[],- sampleRate = 48000,-): Buffer {- const numChannels = channels.length;- if (numChannels === 0) {- throw new Error("Cannot encode WAV: channels array is empty");- }- const numSamples = channels[0]?.length ?? 0;- const byteRate = sampleRate * numChannels * 2;- const blockAlign = numChannels * 2;- const dataSize = numSamples * numChannels * 2;- const buffer = Buffer.alloc(44 + dataSize);-- // RIFF chunk header- buffer.write("RIFF", 0);- buffer.writeUInt32LE(36 + dataSize, 4);- buffer.write("WAVE", 8);-- // "fmt " subchunk- buffer.write("fmt ", 12);- buffer.writeUInt32LE(16, 16); // Subchunk1Size for PCM- buffer.writeUInt16LE(1, 20); // AudioFormat: 1 (PCM)- buffer.writeUInt16LE(numChannels, 22);- buffer.writeUInt32LE(sampleRate, 24);- buffer.writeUInt32LE(byteRate, 28);- buffer.writeUInt16LE(blockAlign, 32);- buffer.writeUInt16LE(16, 34); // BitsPerSample: 16-- // "data" subchunk- buffer.write("data", 36);- buffer.writeUInt32LE(dataSize, 40);-- let offset = 44;- for (let i = 0; i < numSamples; i++) {- for (let c = 0; c < numChannels; c++) {- const channelData = channels[c];- const sample = Math.max(-1, Math.min(1, channelData?.[i] ?? 0));- const intSample = sample < 0 ? sample * 0x8000 : sample * 0x7fff;- buffer.writeInt16LE(Math.round(intSample), offset);- offset += 2;- }- }-- return buffer;-}diff --git a/nodes/node-stereo-panning/src/renderers/audio-processor.test.ts b/nodes/node-stereo-panning/src/renderers/audio-processor.test.tsindex 57ff7ad6..626b8348 100644--- a/nodes/node-stereo-panning/src/renderers/audio-processor.test.ts+++ b/nodes/node-stereo-panning/src/renderers/audio-processor.test.ts@@ -112,4 +112,38 @@ describe("Stereo Panning Audio Processor", () => {expect(channels[1][0]).toBeCloseTo(0.5, 5);});});++ it("should dynamically modulate panning from left to right when pan is driven by a signal", async () => {+ await runWithDevice(async (device) => {+ const numSamples = 200;+ const channels = createChannels(numSamples, 0.5, true);++ // Signal sweeping pan from -1.0 (full left) to +1.0 (full right)+ const panSignal = {+ value: -1.0,+ peek: () => -1.0,+ get: (ctx?: { progress?: number }) => {+ const p = ctx?.progress ?? 0;+ return -1.0 + 2.0 * p;+ },+ };++ const virtualMedia = createVirtualMedia({ pan: panSignal });++ await stereoPanningAudioProcessor(channels, SAMPLE_RATE, virtualMedia, {+ device,+ frame: 0,+ fps: 24,+ renderId: "test-panning-signal-sweep",+ });++ // At start (sample 0): pan ~ -1.0 -> Left gain = 1.0 (0.5), Right gain = 0.0 (0.0)+ expect(channels[0][0]).toBeCloseTo(0.5, 4);+ expect(channels[1][0]).toBeCloseTo(0.0, 4);++ // At end (sample numSamples - 1): pan ~ +1.0 -> Left gain = 0.0 (0.0), Right gain = 1.0 (0.5)+ expect(channels[0][numSamples - 1]).toBeCloseTo(0.0, 4);+ expect(channels[1][numSamples - 1]).toBeCloseTo(0.5, 4);+ });+ });});diff --git a/nodes/node-stereo-panning/src/renderers/audio-processor.ts b/nodes/node-stereo-panning/src/renderers/audio-processor.tsindex 5e3bf2de..09737a86 100644--- a/nodes/node-stereo-panning/src/renderers/audio-processor.ts+++ b/nodes/node-stereo-panning/src/renderers/audio-processor.ts@@ -1,22 +1,29 @@-import { type AudioProcessor, resolveNumber } from "@framefields/node-sdk";+import {+ type AudioProcessor,+ isSignal,+ resolveNumber,+} from "@framefields/node-sdk";import { WebGPUAudioProcessor } from "@framefields/webgpu-renderers";+const PARAM_ORDER = ["pan"];+const PANNING_SHADER_TEMPLATE = () => `struct Uniforms {sampleRate : f32,pan : f32,+ hasPanSig : f32,baseTime : f32,frame : f32,numSamples : f32,numChannels : f32,pad1 : f32,- pad2 : f32,};@group(0) @binding(0) var<uniform> u : Uniforms;@group(0) @binding(1) var<storage, read> inputChannels : array<f32>;@group(0) @binding(2) var<storage, read_write> outputChannels : array<f32>;@group(0) @binding(3) var<storage, read_write> state : array<f32>;+@group(0) @binding(4) var<storage, read> panSignal : array<f32>;@compute @workgroup_size(1)fn main(@builtin(global_invocation_id) gid : vec3<u32>) {@@ -24,18 +31,22 @@ fn main(@builtin(global_invocation_id) gid : vec3<u32>) {let numSamples = u32(u.numSamples);let numChannels = u32(u.numChannels);-- let clampedPan = max(-1.0f, min(1.0f, u.pan));- var leftGain = 1.0f;- var rightGain = 1.0f;-- if (clampedPan < 0.0f) {- rightGain = 1.0f + clampedPan;- } else if (clampedPan > 0.0f) {- leftGain = 1.0f - clampedPan;- }for (var index = 0u; index < numSamples; index = index + 1u) {+ var currentPan = u.pan;+ if (u.hasPanSig > 0.5f) {+ currentPan = panSignal[index];+ }+ let clampedPan = max(-1.0f, min(1.0f, currentPan));+ var leftGain = 1.0f;+ var rightGain = 1.0f;++ if (clampedPan < 0.0f) {+ rightGain = 1.0f + clampedPan;+ } else if (clampedPan > 0.0f) {+ leftGain = 1.0f - clampedPan;+ }+if (numChannels > 1u) {outputChannels[index] = inputChannels[index] * leftGain;outputChannels[numSamples + index] = inputChannels[numSamples + index] * rightGain;@@ -59,12 +70,35 @@ export const stereoPanningAudioProcessor: AudioProcessor = async (throw new Error("GPUDevice is required for WebGPU Stereo Panning.");}- const pan = resolveNumber(- (node.operation as Record<string, unknown> | undefined)?.pan,- 0,- );+ const op = (node.operation as Record<string, unknown> | undefined) || {};+ const inputs =+ (op.inputs as Record<+ string,+ { connectionValid?: boolean; outputItem?: { type?: string } }+ >) || {};++ const pan = resolveNumber(op.pan, 0);const clampedPan = Math.max(-1, Math.min(1, pan));+ const isHandleConnected = (handleIdKey: string): boolean => {+ const handleId = op[handleIdKey];+ if (typeof handleId !== "string" || !handleId) return false;+ const input = inputs[handleId];+ return Boolean(+ input?.connectionValid &&+ input.outputItem &&+ (input.outputItem.type === "Signal" ||+ input.outputItem.type === "Number"),+ );+ };++ const hasPanSig =+ isSignal(op.pan) ||+ isSignal((op.signals as Record<string, unknown> | undefined)?.pan) ||+ isHandleConnected("panHandleId")+ ? 1.0+ : 0.0;+if (channels.length === 0) return;const numSamples = channels[0].length;if (numSamples === 0) return;@@ -77,9 +111,7 @@ export const stereoPanningAudioProcessor: AudioProcessor = async (const numChannels = channels.length;- const nodeId =- ((node.operation as Record<string, unknown> | undefined)?.id as string) ||- "stereo-panning";+ const nodeId = (op.id as string) || "stereo-panning";const frame = ctx.frame ?? 0;const fps = ctx.fps ?? 24;@@ -95,12 +127,12 @@ export const stereoPanningAudioProcessor: AudioProcessor = async (() => [sampleRate,clampedPan,+ hasPanSig,0.0, // baseTimeframe,numSamples,numChannels,- 0,- 0, // padding+ 0.0, // padding],8, // Aligned to 16 bytes (8 * 4 = 32 bytes)1, // stateFloatCount@@ -108,5 +140,8 @@ export const stereoPanningAudioProcessor: AudioProcessor = async (true,ctx?.elapsedMs,ctx?.durationMs,+ undefined,+ undefined,+ PARAM_ORDER,);};diff --git a/nodes/node-stereo-panning/src/shared/config.ts b/nodes/node-stereo-panning/src/shared/config.tsindex 3e0c89c8..7fc0d8f4 100644--- a/nodes/node-stereo-panning/src/shared/config.ts+++ b/nodes/node-stereo-panning/src/shared/config.ts@@ -1,16 +1,21 @@-import { AudioResultSchema, VideoResultSchema } from "@framefields/node-sdk";+import {+ AudioResultSchema,+ configBuilder,+ VideoResultSchema,+} from "@framefields/node-sdk";import { z } from "zod";-export const StereoPanningNodeConfigSchema = z.object({- pan: z- .number()- .min(-1)- .max(1)- .default(0)- .describe(- "Stereo panning value: -1 (full left) to 1 (full right), 0 is center",- ),-});+export const stereoPanningConfig = configBuilder()+ .field("pan", z.number().min(-1).max(1).default(0), {+ bindable: true,+ dataTypes: ["Number", "Signal"],+ label: "Stereo Panning",+ description:+ "Stereo panning value: -1 (full left) to 1 (full right), 0 is center. Can be modulated by a static number or dynamic signal.",+ })+ .build();++export const StereoPanningNodeConfigSchema = stereoPanningConfig.schema;export type StereoPanningNodeConfig = z.infer<typeof StereoPanningNodeConfigSchemadiff --git a/packages/framefields/src/effects/generated/catalog.json b/packages/framefields/src/effects/generated/catalog.jsonindex 7505134b..da90b3f6 100644--- a/packages/framefields/src/effects/generated/catalog.json+++ b/packages/framefields/src/effects/generated/catalog.json@@ -2053,12 +2053,12 @@"props": [{"name": "pan",- "type": "number",- "signal": false,+ "type": "EffectProp<number>",+ "signal": true,"default": 0,"min": -1,"max": 1,- "description": "Stereo panning value: -1 (full left) to 1 (full right), 0 is center"+ "description": "Stereo panning value: -1 (full left) to 1 (full right), 0 is center. Can be modulated by a static number or dynamic signal."}]},diff --git a/packages/framefields/src/effects/generated/classes.ts b/packages/framefields/src/effects/generated/classes.tsindex a55717db..9e08c2b4 100644--- a/packages/framefields/src/effects/generated/classes.ts+++ b/packages/framefields/src/effects/generated/classes.ts@@ -2561,11 +2561,11 @@ export class ResizerScaler extends Effect<ResizerScalerProps> {export interface StereoPanningProps {/**- * Stereo panning value: -1 (full left) to 1 (full right), 0 is center- * Range -1 to 1.+ * Stereo panning value: -1 (full left) to 1 (full right), 0 is center. Can be modulated by a static number or dynamic signal.+ * Range -1 to 1. Accepts a signal.* @default 0*/- pan?: number;+ pan?: EffectProp<number>;}const stereoPanningDefaults: StereoPanningProps = {diff --git a/packages/node-sdk/renderer/index.ts b/packages/node-sdk/renderer/index.tsindex 57194659..52f81b95 100644--- a/packages/node-sdk/renderer/index.ts+++ b/packages/node-sdk/renderer/index.ts@@ -74,6 +74,7 @@ export type AudioProcessor = () => void | Promise<void>;export {+ isSignal,resolveBoolean,resolveNumber,resolveParam,diff --git a/packages/webgpu-renderers/src/audio/webgpu-audio-processor.ts b/packages/webgpu-renderers/src/audio/webgpu-audio-processor.tsindex be52ebe8..594386f2 100644--- a/packages/webgpu-renderers/src/audio/webgpu-audio-processor.ts+++ b/packages/webgpu-renderers/src/audio/webgpu-audio-processor.ts@@ -1,4 +1,4 @@-import type { VirtualMediaData } from "@framefields/core";+import { isSignal, type VirtualMediaData } from "@framefields/core";import { shaderStore } from "../shader-store.js";import { signalRegistry } from "../signals/signal-registry.js";@@ -273,6 +273,53 @@ export class WebGPUAudioProcessor {const hasBinding = (idx: number) =>new RegExp(`@binding\\s*\\(\\s*${idx}\\s*\\)`).test(fullShaderCode);+ const createdBuffers: GPUBuffer[] = [];++ if (isStatic && paramOrder && paramOrder.length > 0) {+ for (const p of paramOrder) {+ if (signalBufferMap.has(p)) continue;+ const directVal =+ op?.[p] ?? (op?.signals as Record<string, unknown> | undefined)?.[p];+ if (isSignal(directVal)) {+ const signalData = new Float32Array(numSamples);+ const durationSec =+ durationSeconds > 0 ? durationSeconds : numSamples / _sampleRate;+ for (let s = 0; s < numSamples; s++) {+ const t = elapsedSeconds + s / _sampleRate;+ const f = t * fps;+ const progress = durationSec > 0 ? t / durationSec : 0;+ signalData[s] =+ Number(+ directVal.get({+ frame: f,+ fps,+ time: t,+ duration: durationSec * fps,+ durationMs: durationSec * 1000,+ progress,+ deltaTime: 1 / _sampleRate,+ }),+ ) || 0;+ }+ const buf = device.createBuffer({+ size: Math.max(16, Math.ceil(signalData.byteLength / 16) * 16),+ usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,+ mappedAtCreation: true,+ label: `audio_direct_signal_${nodeId}_${p}`,+ });+ new Float32Array(buf.getMappedRange()).set(signalData);+ buf.unmap();+ signalBufferMap.set(p, buf);+ createdBuffers.push(buf);+ activeBindings.push({+ parameterName: p,+ signalNodeId: `direct_${p}`,+ signalFnName: "direct_sig",+ });+ }+ }+ }+if (isStatic) {if (paramOrder && paramOrder.length > 0) {for (let i = 0; i < paramOrder.length; i++) {@@ -354,190 +401,196 @@ export class WebGPUAudioProcessor {// 3. Process in chunks to prevent GPU driver timeout (TDR) and browser freezingconst CHUNK_SIZE = 65536;- for (let offset = 0; offset < numSamples; offset += CHUNK_SIZE) {- const chunkSamples = Math.min(CHUNK_SIZE, numSamples - offset);-- // Determine if we can reuse the cached storage and uniform buffers- const canReuse =- cached.inputBuffer &&- cached.outputBuffer &&- cached.uniformBuffer &&- cached.allocatedSamples &&- cached.allocatedSamples >= chunkSamples &&- cached.allocatedChannels &&- cached.allocatedChannels >= numChannels &&- cached.allocatedUniformsFloatCount &&- cached.allocatedUniformsFloatCount >= uniformsFloatCount;-- let inputBuffer: GPUBuffer;- let outputBuffer: GPUBuffer;- let uniformBuffer: GPUBuffer;-- if (canReuse) {- inputBuffer = cached.inputBuffer!;- outputBuffer = cached.outputBuffer!;- uniformBuffer = cached.uniformBuffer!;- } else {- const allocSamples = Math.max(chunkSamples, CHUNK_SIZE);- const allocChannels = numChannels;- const allocUniforms = uniformsFloatCount;-- const freshInput = device.createBuffer({- size: allocChannels * allocSamples * 4,- usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,- label: `audio_input_temp_${nodeId}_chunk_${offset}`,- });-- const freshOutput = device.createBuffer({- size: allocChannels * allocSamples * 4,- usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC,- label: `audio_output_temp_${nodeId}_chunk_${offset}`,- });+ try {+ for (let offset = 0; offset < numSamples; offset += CHUNK_SIZE) {+ const chunkSamples = Math.min(CHUNK_SIZE, numSamples - offset);++ // Determine if we can reuse the cached storage and uniform buffers+ const canReuse =+ cached.inputBuffer &&+ cached.outputBuffer &&+ cached.uniformBuffer &&+ cached.allocatedSamples &&+ cached.allocatedSamples >= chunkSamples &&+ cached.allocatedChannels &&+ cached.allocatedChannels >= numChannels &&+ cached.allocatedUniformsFloatCount &&+ cached.allocatedUniformsFloatCount >= uniformsFloatCount;++ let inputBuffer: GPUBuffer;+ let outputBuffer: GPUBuffer;+ let uniformBuffer: GPUBuffer;++ if (canReuse) {+ inputBuffer = cached.inputBuffer!;+ outputBuffer = cached.outputBuffer!;+ uniformBuffer = cached.uniformBuffer!;+ } else {+ const allocSamples = Math.max(chunkSamples, CHUNK_SIZE);+ const allocChannels = numChannels;+ const allocUniforms = uniformsFloatCount;++ const freshInput = device.createBuffer({+ size: allocChannels * allocSamples * 4,+ usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,+ label: `audio_input_temp_${nodeId}_chunk_${offset}`,+ });++ const freshOutput = device.createBuffer({+ size: allocChannels * allocSamples * 4,+ usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC,+ label: `audio_output_temp_${nodeId}_chunk_${offset}`,+ });++ const freshUniform = device.createBuffer({+ size: allocUniforms * 4,+ usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST,+ label: `audio_uniforms_temp_${nodeId}_chunk_${offset}`,+ });++ inputBuffer = freshInput;+ outputBuffer = freshOutput;+ uniformBuffer = freshUniform;++ cached.inputBuffer?.destroy();+ cached.outputBuffer?.destroy();+ cached.uniformBuffer?.destroy();++ cached.inputBuffer = freshInput;+ cached.outputBuffer = freshOutput;+ cached.uniformBuffer = freshUniform;+ cached.allocatedSamples = allocSamples;+ cached.allocatedChannels = allocChannels;+ cached.allocatedUniformsFloatCount = allocUniforms;+ }- const freshUniform = device.createBuffer({- size: allocUniforms * 4,- usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST,- label: `audio_uniforms_temp_${nodeId}_chunk_${offset}`,+ // stagingBuffer is ALWAYS allocated fresh for this chunk to prevent reuse/mapping bugs+ const stagingBuffer = device.createBuffer({+ size: numChannels * chunkSamples * 4,+ usage: GPUBufferUsage.MAP_READ | GPUBufferUsage.COPY_DST,+ label: `audio_staging_temp_${nodeId}_chunk_${offset}`,});- inputBuffer = freshInput;- outputBuffer = freshOutput;- uniformBuffer = freshUniform;-- cached.inputBuffer?.destroy();- cached.outputBuffer?.destroy();- cached.uniformBuffer?.destroy();-- cached.inputBuffer = freshInput;- cached.outputBuffer = freshOutput;- cached.uniformBuffer = freshUniform;- cached.allocatedSamples = allocSamples;- cached.allocatedChannels = allocChannels;- cached.allocatedUniformsFloatCount = allocUniforms;- }-- // stagingBuffer is ALWAYS allocated fresh for this chunk to prevent reuse/mapping bugs- const stagingBuffer = device.createBuffer({- size: numChannels * chunkSamples * 4,- usage: GPUBufferUsage.MAP_READ | GPUBufferUsage.COPY_DST,- label: `audio_staging_temp_${nodeId}_chunk_${offset}`,- });-- const entries: GPUBindGroupEntry[] = [- { binding: 0, resource: { buffer: uniformBuffer } },- { binding: 1, resource: { buffer: inputBuffer } },- { binding: 2, resource: { buffer: outputBuffer } },- { binding: 3, resource: { buffer: cached.stateBuffer } },- ];-- if (isStatic) {- for (let i = 0; i < signalBuffers.length; i++) {- const bindingIndex = 4 + i;- if (hasBinding(bindingIndex)) {- const sigBuf = signalBuffers[i];- if (sigBuf.size >= (offset + chunkSamples) * 4) {- entries.push({- binding: bindingIndex,- resource: {- buffer: sigBuf,- offset: offset * 4,- size: chunkSamples * 4,- },- });- } else {- entries.push({- binding: bindingIndex,- resource: {- buffer: sigBuf,- offset: 0,- size: sigBuf.size,- },- });+ const entries: GPUBindGroupEntry[] = [+ { binding: 0, resource: { buffer: uniformBuffer } },+ { binding: 1, resource: { buffer: inputBuffer } },+ { binding: 2, resource: { buffer: outputBuffer } },+ { binding: 3, resource: { buffer: cached.stateBuffer } },+ ];++ if (isStatic) {+ for (let i = 0; i < signalBuffers.length; i++) {+ const bindingIndex = 4 + i;+ if (hasBinding(bindingIndex)) {+ const sigBuf = signalBuffers[i];+ if (sigBuf.size >= (offset + chunkSamples) * 4) {+ entries.push({+ binding: bindingIndex,+ resource: {+ buffer: sigBuf,+ offset: offset * 4,+ size: chunkSamples * 4,+ },+ });+ } else {+ entries.push({+ binding: bindingIndex,+ resource: {+ buffer: sigBuf,+ offset: 0,+ size: sigBuf.size,+ },+ });+ }}}}- }- const bindGroup = device.createBindGroup({- layout: cached.pipeline.getBindGroupLayout(0),- entries,- });+ const bindGroup = device.createBindGroup({+ layout: cached.pipeline.getBindGroupLayout(0),+ entries,+ });- // Upload Input Channels- const flatInput = new Float32Array(numChannels * chunkSamples);- for (let c = 0; c < numChannels; c++) {- flatInput.set(- channels[c].subarray(offset, offset + chunkSamples),- c * chunkSamples,+ // Upload Input Channels+ const flatInput = new Float32Array(numChannels * chunkSamples);+ for (let c = 0; c < numChannels; c++) {+ flatInput.set(+ channels[c].subarray(offset, offset + chunkSamples),+ c * chunkSamples,+ );+ }+ device.queue.writeBuffer(+ inputBuffer,+ 0,+ flatInput as unknown as GPUAllowSharedBufferSource,);- }- device.queue.writeBuffer(- inputBuffer,- 0,- flatInput as unknown as GPUAllowSharedBufferSource,- );- // Compute uniforms for this chunk- const chunkBaseTime = frame / fps + offset / _sampleRate;- const uniforms = getUniforms(chunkBaseTime);+ // Compute uniforms for this chunk+ const chunkBaseTime = frame / fps + offset / _sampleRate;+ const uniforms = getUniforms(chunkBaseTime);- // Replace numSamples with chunkSamples inside the uniforms array- let numSamplesIndex = -1;- for (let i = uniforms.length - 2; i >= 0; i--) {- if (uniforms[i] === numSamples && uniforms[i + 1] === numChannels) {- numSamplesIndex = i;- break;+ // Replace numSamples with chunkSamples inside the uniforms array+ let numSamplesIndex = -1;+ for (let i = uniforms.length - 2; i >= 0; i--) {+ if (uniforms[i] === numSamples && uniforms[i + 1] === numChannels) {+ numSamplesIndex = i;+ break;+ }+ }+ if (numSamplesIndex === -1) {+ numSamplesIndex = uniforms.indexOf(numSamples);+ }+ if (numSamplesIndex !== -1) {+ uniforms[numSamplesIndex] = chunkSamples;}- }- if (numSamplesIndex === -1) {- numSamplesIndex = uniforms.indexOf(numSamples);- }- if (numSamplesIndex !== -1) {- uniforms[numSamplesIndex] = chunkSamples;- }- device.queue.writeBuffer(- uniformBuffer,- 0,- new Float32Array(uniforms) as unknown as GPUAllowSharedBufferSource,- );+ device.queue.writeBuffer(+ uniformBuffer,+ 0,+ new Float32Array(uniforms) as unknown as GPUAllowSharedBufferSource,+ );- // Execute Compute Pass and Copy to Staging- const commandEncoder = device.createCommandEncoder({- label: `audio_encoder_${nodeId}_chunk_${offset}`,- });- const passEncoder = commandEncoder.beginComputePass({- label: `audio_pass_${nodeId}_chunk_${offset}`,- });- passEncoder.setPipeline(cached.pipeline);- passEncoder.setBindGroup(0, bindGroup);- passEncoder.dispatchWorkgroups(1);- passEncoder.end();+ // Execute Compute Pass and Copy to Staging+ const commandEncoder = device.createCommandEncoder({+ label: `audio_encoder_${nodeId}_chunk_${offset}`,+ });+ const passEncoder = commandEncoder.beginComputePass({+ label: `audio_pass_${nodeId}_chunk_${offset}`,+ });+ passEncoder.setPipeline(cached.pipeline);+ passEncoder.setBindGroup(0, bindGroup);+ passEncoder.dispatchWorkgroups(1);+ passEncoder.end();- commandEncoder.copyBufferToBuffer(- outputBuffer,- 0,- stagingBuffer,- 0,- numChannels * chunkSamples * 4,- );+ commandEncoder.copyBufferToBuffer(+ outputBuffer,+ 0,+ stagingBuffer,+ 0,+ numChannels * chunkSamples * 4,+ );- device.queue.submit([commandEncoder.finish()]);+ device.queue.submit([commandEncoder.finish()]);- // Map & Read back- await stagingBuffer.mapAsync(GPUMapMode.READ);- const mappedRange = new Float32Array(stagingBuffer.getMappedRange());+ // Map & Read back+ await stagingBuffer.mapAsync(GPUMapMode.READ);+ const mappedRange = new Float32Array(stagingBuffer.getMappedRange());- for (let c = 0; c < numChannels; c++) {- channels[c].set(- mappedRange.subarray(c * chunkSamples, (c + 1) * chunkSamples),- offset,- );- }+ for (let c = 0; c < numChannels; c++) {+ channels[c].set(+ mappedRange.subarray(c * chunkSamples, (c + 1) * chunkSamples),+ offset,+ );+ }- stagingBuffer.unmap();+ stagingBuffer.unmap();- await yieldToMain();+ await yieldToMain();+ }+ } finally {+ for (const b of createdBuffers) {+ b.destroy();+ }}}}diff --git a/plugins/framefields/skills/framefields-effects/references/effects-catalog.md b/plugins/framefields/skills/framefields-effects/references/effects-catalog.mdindex b162465a..b47816c4 100644--- a/plugins/framefields/skills/framefields-effects/references/effects-catalog.md+++ b/plugins/framefields/skills/framefields-effects/references/effects-catalog.md@@ -599,7 +599,7 @@ new StereoPanning({ pan: 0 });| Prop | Type | Default | Notes || --- | --- | --- | --- |-| `pan` | `number` | `0` | Range -1 to 1. Stereo panning value: -1 (full left) to 1 (full right), 0 is center |+| `pan` | `EffectProp<number>` | `0` | Range -1 to 1. Stereo panning value: -1 (full left) to 1 (full right), 0 is center. Can be modulated by a static number or dynamic signal. |### Vignette