Skip to content

Code-first video & image, rendered natively on WebGPU.

2.0 KB
Raw Blame
/**
 * Per-frame envelopes for the picture, from the separated stems: the drum
 * stem's kick band (below ~120 Hz) and snare band (~1.5–4 kHz) as 0–1 onset
 * envelopes, and the vocal stem's loudness.
 */
import { FPS } from "../grid.js";
import { SR, type Stereo, Svf } from "./dsp.js";

export interface Envelopes {
	fps: number;
	kick: number[];
	snare: number[];
	vocal: number[];
}

const HOP = SR / FPS;

function bandEnergy(
	mix: Stereo,
	band: (x: number, f: Svf) => number,
): Float32Array {
	const frames = Math.floor(mix[0].length / HOP);
	const out = new Float32Array(frames);
	const filter = new Svf();
	for (let f = 0; f < frames; f++) {
		let e = 0;
		for (let i = f * HOP; i < (f + 1) * HOP; i++) {
			const y = band((mix[0][i] + mix[1][i]) * 0.5, filter);
			e += y * y;
		}
		out[f] = Math.log10(e / HOP + 1e-9);
	}
	return out;
}

/** Positive log-energy flux, normalised to its 98th percentile and clamped to 0–1. */
function onset(energy: Float32Array): Float32Array {
	const flux = new Float32Array(energy.length);
	for (let i = 1; i < energy.length; i++)
		flux[i] = Math.max(0, energy[i] - energy[i - 1]);
	const ref =
		[...flux].sort((a, b) => a - b)[Math.floor(flux.length * 0.98)] || 1;
	return flux.map((v) => Math.min(1, v / ref));
}

/** Holds each onset and lets it decay, so a hit reads for a few frames. */
function decay(onsets: Float32Array, frames: number): number[] {
	const out: number[] = [];
	let level = 0;
	for (const v of onsets) {
		level = Math.max(v, level * Math.exp(-1 / frames));
		out.push(Number(level.toFixed(4)));
	}
	return out;
}

export function envelopes(drums: Stereo, vocals: Stereo): Envelopes {
	const kick = onset(bandEnergy(drums, (x, f) => f.run(x, 120, 0.3).lp));
	const snare = onset(bandEnergy(drums, (x, f) => f.run(x, 2400, 0.55).bp));
	const loud = bandEnergy(vocals, (x) => x);
	const lo = Math.min(...loud);
	const hi = Math.max(...loud);
	return {
		fps: FPS,
		kick: decay(kick, 5),
		snare: decay(snare, 4),
		vocal: [...loud].map((v) => Number(((v - lo) / (hi - lo)).toFixed(4))),
	};
}