Skip to content

Code-first video & image, rendered natively on WebGPU.

5.5 KB
Raw Blame
import { decodeAudioSource } from "@framefields/compositions";
import {
	type FrameSignal,
	frameArraySignal,
	type GPUTextureBinding,
} from "@framefields/core";
import {
	type AudioPitchTrackConfig,
	type AudioSignalComputeConfig,
	AudioSignalComputePipeline,
	ensureDevice,
	trackAudioPitch,
} from "@framefields/webgpu-renderers";

export interface AudioSignalOptions extends AudioSignalComputeConfig {
	fps?: number;
	targetWidth?: number;
	previewMode?: "waveform" | "envelope" | "beat_markers" | "spectrum";
}

export interface ExtractedAudioSignalsBundle {
	primary: FrameSignal<number>;
	beat: FrameSignal<number>;
	bass: FrameSignal<number>;
	energy: FrameSignal<number>;
	durationSec: number;
	numFrames: number;
	sampleRate: number;
	textureView: unknown;
	texture: unknown;
	primaryBuffer: unknown;
	beatBuffer: unknown;
	bassBuffer: unknown;
	energyBuffer: unknown;
	seekFrame(frame: number): void;
}

export interface AudioPitchOptions extends AudioPitchTrackConfig {
	fps?: number;
}

/** Fundamental frequency per frame, e.g. an engine note (RPM) or a melody line. */
export interface AudioPitchSignals {
	/** Pitch in Hz; interpolated across unvoiced gaps so it never drops to 0 mid-track. */
	hz: FrameSignal<number>;
	/** 0..1 periodicity of the measured frame, 0 where unvoiced. */
	confidence: FrameSignal<number>;
	/** 1 where a pitch was measured, 0 where `hz` is interpolated. */
	voiced: FrameSignal<number>;
	samples: { hz: Float32Array; confidence: Float32Array; voiced: Float32Array };
	fps: number;
	numFrames: number;
	durationSec: number;
}

async function decodeOrThrow(source: string, purpose: string) {
	const decoded = await decodeAudioSource(source);
	if (!decoded || decoded.channels.length === 0) {
		throw new Error(`Failed to decode audio source for ${purpose}: ${source}`);
	}
	return decoded;
}

export const AudioSignal = {
	async extractFromChannels(
		channels: Float32Array[],
		sampleRate: number,
		options: AudioSignalOptions = {},
	): Promise<ExtractedAudioSignalsBundle> {
		const device = await ensureDevice();
		const fps = options.fps ?? 24;
		const targetWidth = options.targetWidth ?? 512;

		const extracted = await AudioSignalComputePipeline.extractFeatures(
			device,
			channels,
			sampleRate,
			fps,
			options,
			targetWidth,
		);

		const samples = extracted.channelSamples ?? {
			primary: new Float32Array(extracted.numFrames),
			beat: new Float32Array(extracted.numFrames),
			bass: new Float32Array(extracted.numFrames),
			energy: new Float32Array(extracted.numFrames),
		};

		const primaryBinding: GPUTextureBinding = {
			nodeId: "audio_primary",
			channel: "primary",
			textureView: extracted.textureView,
			texture: extracted.texture,
			buffer: extracted.primaryBuffer,
			stats: extracted.stats.primary,
		};
		const beatBinding: GPUTextureBinding = {
			nodeId: "audio_beat",
			channel: "beat",
			textureView: extracted.textureView,
			texture: extracted.texture,
			buffer: extracted.beatBuffer,
			stats: extracted.stats.beat,
		};
		const bassBinding: GPUTextureBinding = {
			nodeId: "audio_bass",
			channel: "bass",
			textureView: extracted.textureView,
			texture: extracted.texture,
			buffer: extracted.bassBuffer,
			stats: extracted.stats.bass,
		};
		const energyBinding: GPUTextureBinding = {
			nodeId: "audio_energy",
			channel: "energy",
			textureView: extracted.textureView,
			texture: extracted.texture,
			buffer: extracted.energyBuffer,
			stats: extracted.stats.energy,
		};

		const primary = frameArraySignal(samples.primary, fps, primaryBinding);
		const beat = frameArraySignal(samples.beat, fps, beatBinding);
		const bass = frameArraySignal(samples.bass, fps, bassBinding);
		const energy = frameArraySignal(samples.energy, fps, energyBinding);

		return {
			primary,
			beat,
			bass,
			energy,
			durationSec: extracted.durationSec,
			numFrames: extracted.numFrames,
			sampleRate,
			textureView: extracted.textureView,
			texture: extracted.texture,
			primaryBuffer: extracted.primaryBuffer,
			beatBuffer: extracted.beatBuffer,
			bassBuffer: extracted.bassBuffer,
			energyBuffer: extracted.energyBuffer,
			seekFrame(frame: number) {
				primary.seekFrame?.(frame);
				beat.seekFrame?.(frame);
				bass.seekFrame?.(frame);
				energy.seekFrame?.(frame);
			},
		};
	},

	async extract(
		source: string,
		options: AudioSignalOptions = {},
	): Promise<ExtractedAudioSignalsBundle> {
		const decoded = await decodeOrThrow(source, "feature extraction");
		return AudioSignal.extractFromChannels(
			decoded.channels,
			decoded.sampleRate,
			options,
		);
	},

	/** Tracks the fundamental frequency of the downmixed channels on the GPU. */
	async pitchFromChannels(
		channels: Float32Array[],
		sampleRate: number,
		options: AudioPitchOptions = {},
	): Promise<AudioPitchSignals> {
		const device = await ensureDevice();
		const fps = options.fps ?? 24;
		const track = await trackAudioPitch(
			device,
			channels,
			sampleRate,
			fps,
			options,
		);
		return {
			hz: frameArraySignal(track.hz, fps),
			confidence: frameArraySignal(track.confidence, fps),
			voiced: frameArraySignal(track.voiced, fps),
			samples: {
				hz: track.hz,
				confidence: track.confidence,
				voiced: track.voiced,
			},
			fps,
			numFrames: track.numFrames,
			durationSec: track.durationSec,
		};
	},

	/** Decodes `source` (audio or video file) and tracks its pitch. */
	async pitch(
		source: string,
		options: AudioPitchOptions = {},
	): Promise<AudioPitchSignals> {
		const decoded = await decodeOrThrow(source, "pitch tracking");
		return AudioSignal.pitchFromChannels(
			decoded.channels,
			decoded.sampleRate,
			options,
		);
	},
};