Code-first video & image, rendered natively on WebGPU.
import {
type AudioProcessor,
resolveNumber,
resolveString,
} from "@framefields/node-sdk";
import {
type AudioSignalComputeConfig,
AudioSignalComputePipeline,
shaderStore,
signalRegistry,
} from "@framefields/webgpu-renderers";
export const audioSignalExtractorAudioProcessor: AudioProcessor = async (
channels,
sampleRate,
virtualMedia,
ctx,
) => {
if (!ctx?.device) return;
if (!channels || channels.length === 0 || channels[0].length === 0) return;
const op = (virtualMedia.operation as Record<string, unknown>) || {};
const nodeId =
(op.nodeId as string) || (op.id as string) || "audio-signal-extractor";
const fps = ctx.fps ?? 24;
const mode = resolveString(op.extractionMode, "");
const curveStr = resolveString(op.curve, "");
const config: AudioSignalComputeConfig = {
extractionMode: (mode ||
undefined) as AudioSignalComputeConfig["extractionMode"],
attackMs: resolveNumber(op.attackMs, 10),
releaseMs: resolveNumber(op.releaseMs, 120),
sensitivity: resolveNumber(op.sensitivity, 1.0),
noiseFloorDb: resolveNumber(op.noiseFloorDb, -45),
dynamicRangeDb: resolveNumber(op.dynamicRangeDb, 36),
smoothing: resolveNumber(op.smoothing, 0.15),
curve: (curveStr || undefined) as AudioSignalComputeConfig["curve"],
beatThreshold: resolveNumber(op.beatThreshold, 0.55),
beatDecayMs: resolveNumber(op.beatDecayMs, 80),
};
// 1. WebGPU feature extraction on the audio channels produced by the pipeline
const signals = await AudioSignalComputePipeline.extractFeatures(
ctx.device,
channels,
sampleRate,
fps,
config,
512,
);
// 2. Register signals at signal store (signalRegistry & shaderStore)
signalRegistry.registerBuffer(
ctx.device,
nodeId,
signals.primaryBuffer,
ctx.renderId,
);
signalRegistry.registerBuffer(
ctx.device,
`${nodeId}_beat`,
signals.beatBuffer,
ctx.renderId,
);
signalRegistry.registerBuffer(
ctx.device,
`${nodeId}_bass`,
signals.bassBuffer,
ctx.renderId,
);
signalRegistry.registerBuffer(
ctx.device,
`${nodeId}_energy`,
signals.energyBuffer,
ctx.renderId,
);
signalRegistry.registerTexture(
ctx.device,
nodeId,
signals.texture,
signals.textureView,
ctx.renderId,
);
signalRegistry.registerTexture(
ctx.device,
`${nodeId}_beat`,
signals.texture,
signals.textureView,
ctx.renderId,
);
signalRegistry.registerTexture(
ctx.device,
`${nodeId}_bass`,
signals.texture,
signals.textureView,
ctx.renderId,
);
signalRegistry.registerTexture(
ctx.device,
`${nodeId}_energy`,
signals.texture,
signals.textureView,
ctx.renderId,
);
const safeNodeId = nodeId.replace(/[^a-zA-Z0-9]/g, "_");
const fnName = `signal_${safeNodeId}`;
const wgsl = `
fn ${fnName}(t: f32, x: f32, y: f32, z: f32, i: u32, n: u32, frame: u32, color: vec4f, t_elapsed: f32, duration: f32) -> f32 {
return 0.0f;
}
`;
shaderStore.register(
nodeId,
{
wgsl,
name: fnName,
outputType: "f32",
fnParams: [],
},
ctx.renderId,
);
};