Skip to content

Write HTML. Render video. Built for agents.

feat(core): signal rhythm is rate-independent, refuses grids in the tracker, follows chords (#5644)

Xuanru Li695110e46ba4
diff --git a/packages/core/src/audioResample.test.ts b/packages/core/src/audioResample.test.ts
new file mode 100644
index 0000000000..6d6daca630
--- /dev/null
+++ b/packages/core/src/audioResample.test.ts
@@ -0,0 +1,35 @@
+import { describe, expect, it } from "vitest";
+import { resample } from "./audioResample";
+
+const tone = (hz: number, rate: number, seconds: number) =>
+ Float32Array.from({ length: seconds * rate }, (_, i) => Math.sin((2 * Math.PI * hz * i) / rate));
+
+const rms = (x: Float32Array) => Math.sqrt(x.reduce((a, v) => a + v * v, 0) / x.length);
+
+/** Upward zero crossings per second, over the middle of the signal. */
+function frequency(x: Float32Array, rate: number): number {
+ const from = Math.floor(x.length / 4);
+ const to = Math.floor((3 * x.length) / 4);
+ let crossings = 0;
+ for (let i = from + 1; i < to; i++) if (x[i - 1]! < 0 && x[i]! >= 0) crossings++;
+ return (crossings * rate) / (to - from);
+}
+
+describe("resample", () => {
+ it("keeps a tone's pitch and level from 48 kHz to 22.05 kHz", () => {
+ const out = resample(tone(1000, 48_000, 2), 48_000, 22_050);
+ expect(out.length).toBe(44_100);
+ expect(frequency(out, 22_050)).toBeCloseTo(1000, -1);
+ expect(rms(out.subarray(1000, -1000))).toBeCloseTo(Math.SQRT1_2, 2);
+ });
+
+ it("removes what the lower rate cannot hold instead of folding it down", () => {
+ const out = resample(tone(15_000, 48_000, 1), 48_000, 22_050);
+ expect(rms(out.subarray(1000, -1000))).toBeLessThan(0.01);
+ });
+
+ it("returns the samples untouched at the same rate", () => {
+ const x = tone(440, 22_050, 1);
+ expect(resample(x, 22_050, 22_050)).toBe(x);
+ });
+});
diff --git a/packages/core/src/audioResample.ts b/packages/core/src/audioResample.ts
new file mode 100644
index 0000000000..60add57aec
--- /dev/null
+++ b/packages/core/src/audioResample.ts
@@ -0,0 +1,38 @@
+/** Band-limited resampling by windowed-sinc interpolation, so analysis reads the same at any decode rate. */
+
+const ZERO_CROSSINGS = 8;
+const STEPS_PER_CROSSING = 256;
+/** Cutoff as a share of the lower Nyquist, leaving the window's transition band below it. */
+const CUTOFF = 0.95;
+
+const KERNEL = Float64Array.from({ length: ZERO_CROSSINGS * STEPS_PER_CROSSING + 2 }, (_, i) => {
+ const x = i / STEPS_PER_CROSSING;
+ if (x >= ZERO_CROSSINGS) return 0;
+ const sinc = x === 0 ? 1 : Math.sin(Math.PI * x) / (Math.PI * x);
+ return sinc * (0.5 + 0.5 * Math.cos((Math.PI * x) / ZERO_CROSSINGS));
+});
+
+function kernelAt(x: number): number {
+ const at = Math.abs(x) * STEPS_PER_CROSSING;
+ const i = Math.floor(at);
+ if (i + 1 >= KERNEL.length) return 0;
+ const frac = at - i;
+ return KERNEL[i]! * (1 - frac) + KERNEL[i + 1]! * frac;
+}
+
+export function resample(samples: Float32Array, from: number, to: number): Float32Array {
+ if (from === to) return samples;
+ const ratio = to / from;
+ const scale = Math.min(1, ratio) * CUTOFF;
+ const reach = ZERO_CROSSINGS / scale;
+ const out = new Float32Array(Math.floor(samples.length * ratio));
+ for (let n = 0; n < out.length; n++) {
+ const t = n / ratio;
+ const last = Math.min(samples.length - 1, Math.floor(t + reach));
+ let sum = 0;
+ for (let k = Math.max(0, Math.ceil(t - reach)); k <= last; k++)
+ sum += samples[k]! * kernelAt((t - k) * scale);
+ out[n] = sum * scale;
+ }
+ return out;
+}
diff --git a/packages/core/src/audioRhythm.test.ts b/packages/core/src/audioRhythm.test.ts
index 2667d5d40c..760a85c1d5 100644
--- a/packages/core/src/audioRhythm.test.ts
+++ b/packages/core/src/audioRhythm.test.ts
@@ -15,14 +15,14 @@ function random(seed = 3) {
}
/** A drum pattern on a known grid: a kick with a low thump on each bar's first beat, a click on the others. */
-function drums(beatTimes: number[], meter: number, seconds: number): AudioSignal {
- const samples = new Float32Array(Math.round(seconds * RATE));
+function drums(beatTimes: number[], meter: number, seconds: number, rate = RATE): AudioSignal {
+ const samples = new Float32Array(Math.round(seconds * rate));
const noise = random();
beatTimes.forEach((t, k) => {
const downbeat = k % meter === 0;
- const start = Math.round(t * RATE);
- for (let i = 0; i < 0.12 * RATE && start + i < samples.length; i++) {
- const since = i / RATE;
+ const start = Math.round(t * rate);
+ for (let i = 0; i < 0.12 * rate && start + i < samples.length; i++) {
+ const since = i / rate;
const click = (downbeat ? 0.5 : 0.35) * noise() * Math.exp(-since / 0.02);
const thump = downbeat
? 0.8 * Math.sin(2 * Math.PI * 55 * since) * Math.exp(-since / 0.06)
@@ -30,6 +30,31 @@ function drums(beatTimes: number[], meter: number, seconds: number): AudioSignal
samples[start + i]! += click + thump;
}
});
+ return { sampleRate: rate, channels: [samples] };
+}
+
+/** The same hit on every beat, over chords that change every four beats from beat `firstChange`. */
+function chordsOverEvenHits(
+ beatTimes: number[],
+ firstChange: number,
+ seconds: number,
+): AudioSignal {
+ const samples = new Float32Array(Math.round(seconds * RATE));
+ const noise = random(7);
+ const roots = [220, 175, 262, 196];
+ beatTimes.forEach((t) => {
+ const start = Math.round(t * RATE);
+ for (let i = 0; i < 0.08 * RATE && start + i < samples.length; i++)
+ samples[start + i]! += 0.4 * noise() * Math.exp(-i / RATE / 0.02);
+ });
+ const period = beatTimes[1]! - beatTimes[0]!;
+ for (let i = 0; i < samples.length; i++) {
+ const t = i / RATE;
+ const bar = Math.floor((t - beatTimes[firstChange]!) / (4 * period));
+ const root = roots[((bar % 4) + 4) % 4]!;
+ for (const ratio of [1, 1.26, 1.5])
+ samples[i]! += 0.08 * Math.sin(2 * Math.PI * root * ratio * t);
+ }
return { sampleRate: RATE, channels: [samples] };
}
@@ -66,6 +91,23 @@ describe("rhythm", () => {
expect(curve[curve.length - 1]!.bpm).toBeCloseTo(133, -1);
});
+ it("gives the same beats at any sample rate and for mono or the same sound on two channels", () => {
+ const truth = grid(0.25, 0.5, 19.5);
+ const reference = rhythm(drums(truth, 4, 20)).beats.times;
+ const wide = drums(truth, 4, 20, 44_100).channels[0]!;
+ const stereo = rhythm({ sampleRate: 44_100, channels: [wide, wide.slice()] }).beats.times;
+ expect(stereo).toHaveLength(reference.length);
+ stereo.forEach((t, k) => expect(Math.abs(t - reference[k]!)).toBeLessThanOrEqual(0.011));
+ });
+
+ it("puts the bar where the chords change when every beat hits the same", () => {
+ const truth = grid(0.25, 0.5, 23.5);
+ const { downbeats } = rhythm(chordsOverEvenHits(truth, 1, 24));
+ expect(downbeats.meter).toBe(4);
+ const changes = truth.filter((_, k) => k % 4 === 1);
+ expect(recall(downbeats.times, changes)).toBeGreaterThan(0.9);
+ });
+
it("finds no steady beat in silence or in noise, and none in a clip too short to tell", () => {
const silent = rhythm({ sampleRate: RATE, channels: [new Float32Array(10 * RATE)] });
expect(silent.beats).toMatchObject({ times: [], reason: "no steady beat" });
@@ -75,6 +117,7 @@ describe("rhythm", () => {
channels: [Float32Array.from({ length: 10 * RATE }, () => 0.3 * noise())],
});
expect(hiss.beats.times).toEqual([]);
+ expect(hiss.downbeats.times).toEqual([]);
const short = rhythm(drums(grid(0.25, 0.5, 3), 4, 3));
expect(short.beats).toMatchObject({ times: [], reason: "too short for a steady beat" });
});
diff --git a/packages/core/src/audioRhythm.ts b/packages/core/src/audioRhythm.ts
index 74fa42d36e..a0608630f3 100644
--- a/packages/core/src/audioRhythm.ts
+++ b/packages/core/src/audioRhythm.ts
@@ -4,6 +4,7 @@
*/
import type { AudioSignal } from "./audioAnalysis";
+import { resample } from "./audioResample";
import { fft } from "./fft";
export interface Beats {
@@ -18,7 +19,7 @@ export interface Downbeats {
times: number[];
/** Beats per bar, 3 or 4; null when there are too few beats to tell. */
meter: number | null;
- /** "low" for a guess from where the low end hits hardest. */
+ /** "low": a guess from where the harmony changes and the low end hits. */
confidence?: "low";
}
@@ -39,28 +40,39 @@ export interface Rhythm {
downbeats: Downbeats;
}
-/** Log energy per frame in log-spaced bands, the frames `fps` per second. */
+/** Log energy per frame in log-spaced bands, `fps` frames per second, and a slower pitch-class profile. */
export interface BandFrames {
fps: number;
bands: Float64Array[];
+ chroma: Float64Array[];
+ chromaFps: number;
}
+/** Every signal is analyzed at this rate and as the mean of its channels' power, so decode rate and layout cannot change a result. */
+const ANALYSIS_RATE = 22050;
+const FRAME_SIZE = 1024;
+const CHROMA_SIZE = 4096;
+const CHROMA_FPS = 10;
const FRAMES_PER_SECOND = 100;
const LOW_BAND_HZ = 150;
const SLOWEST_BPM = 40;
const FASTEST_BPM = 240;
const TIGHTNESS = 100;
+/** The onset curve peaks this much ahead of where listeners place the beat, so beats move this much later. */
+const ONSET_LAG_SECONDS = 0.015;
const round = (t: number) => Math.round(t * 1000) / 1000;
-function bandEdges(sampleRate: number, size: number): number[] {
- const binHz = sampleRate / size;
+function bandEdges(): number[] {
+ const binHz = ANALYSIS_RATE / FRAME_SIZE;
const edges: number[] = [];
- for (let hz = 30; hz < Math.min(9000, sampleRate / 2); hz *= 2 ** (1 / 5))
- edges.push(Math.max(1, Math.round(hz / binHz)));
+ for (let hz = 30; hz < 9000; hz *= 2 ** (1 / 5)) edges.push(Math.max(1, Math.round(hz / binHz)));
return edges;
}
+const hann = (size: number) =>
+ Float64Array.from({ length: size }, (_, i) => 0.5 - 0.5 * Math.cos((2 * Math.PI * i) / size));
+
/** Add each band's power in the Hann-windowed frame centred on `center` into `into`. */
function addFramePower(
samples: Float32Array,
@@ -86,24 +98,61 @@ function addFramePower(
}
}
-/** Per-band log power, summed over channels so a polarity-flipped channel cannot cancel. */
-export function bandFrames(signal: AudioSignal): BandFrames {
- const { sampleRate, channels } = signal;
- const hop = Math.max(1, Math.round(sampleRate / FRAMES_PER_SECOND));
- const size = 2 ** Math.round(Math.log2(sampleRate * 0.046));
- const window = Float64Array.from(
- { length: size },
- (_, i) => 0.5 - 0.5 * Math.cos((2 * Math.PI * i) / size),
- );
- const edges = bandEdges(sampleRate, size);
- const length = channels[0]?.length ?? 0;
+/** Per-band log power of every frame, averaged over channels so a polarity-flipped channel cannot cancel. */
+function bandPower(channels: Float32Array[]): Float64Array[] {
+ const window = hann(FRAME_SIZE);
+ const edges = bandEdges();
+ const hop = ANALYSIS_RATE / FRAMES_PER_SECOND;
const bands: Float64Array[] = [];
- for (let center = 0; center < length; center += hop) {
- const power = new Float64Array(Math.max(0, edges.length - 1));
- for (const channel of channels) addFramePower(channel, center, window, edges, power);
- bands.push(power.map((p) => Math.log1p(1e9 * p)));
+ for (let f = 0; Math.round(f * hop) < (channels[0]?.length ?? 0); f++) {
+ const power = new Float64Array(edges.length - 1);
+ for (const channel of channels)
+ addFramePower(channel, Math.round(f * hop), window, edges, power);
+ bands.push(power.map((p) => Math.log1p((1e9 * p) / channels.length)));
+ }
+ return bands;
+}
+
+/** Index of the pitch class, A = 9, of each FFT bin from 55 Hz to 2 kHz; -1 outside that range. */
+const PITCH_CLASS = Array.from({ length: CHROMA_SIZE / 2 }, (_, bin) => {
+ const hz = (bin * ANALYSIS_RATE) / CHROMA_SIZE;
+ if (hz < 55 || hz > 2000) return -1;
+ return (((Math.round(12 * Math.log2(hz / 440)) + 9) % 12) + 12) % 12;
+});
+
+function chromaFrames(channels: Float32Array[]): Float64Array[] {
+ const window = hann(CHROMA_SIZE);
+ const hop = ANALYSIS_RATE / CHROMA_FPS;
+ const re = new Float64Array(CHROMA_SIZE);
+ const im = new Float64Array(CHROMA_SIZE);
+ const frames: Float64Array[] = [];
+ for (let f = 0; Math.round(f * hop) < (channels[0]?.length ?? 0); f++) {
+ const chroma = new Float64Array(12);
+ const first = Math.round(f * hop) - CHROMA_SIZE / 2;
+ for (const channel of channels) {
+ for (let i = 0; i < CHROMA_SIZE; i++) {
+ const at = first + i;
+ re[i] = at >= 0 && at < channel.length ? channel[at]! * window[i]! : 0;
+ im[i] = 0;
+ }
+ fft(re, im);
+ PITCH_CLASS.forEach((pc, bin) => {
+ if (pc >= 0) chroma[pc]! += Math.hypot(re[bin]!, im[bin]!);
+ });
+ }
+ frames.push(chroma);
}
- return { fps: sampleRate / hop, bands };
+ return frames;
+}
+
+function bandFrames(signal: AudioSignal): BandFrames {
+ const channels = signal.channels.map((c) => resample(c, signal.sampleRate, ANALYSIS_RATE));
+ return {
+ fps: FRAMES_PER_SECOND,
+ bands: bandPower(channels),
+ chroma: chromaFrames(channels),
+ chromaFps: CHROMA_FPS,
+ };
}
/** Subtract the mean of the surrounding ±`radius` frames and keep what rises above it. */
@@ -131,10 +180,9 @@ interface OnsetCurves {
low: Float64Array;
}
-function onsetCurves({ fps, bands }: BandFrames, sampleRate: number): OnsetCurves {
- const size = 2 ** Math.round(Math.log2(sampleRate * 0.046));
- const lowBands = bandEdges(sampleRate, size).filter(
- (bin) => (bin * sampleRate) / size < LOW_BAND_HZ,
+function onsetCurves({ fps, bands }: BandFrames): OnsetCurves {
+ const lowBands = bandEdges().filter(
+ (bin) => (bin * ANALYSIS_RATE) / FRAME_SIZE < LOW_BAND_HZ,
).length;
const broad = new Float64Array(bands.length);
const low = new Float64Array(bands.length);
@@ -170,7 +218,7 @@ function autocorrelation(curve: Float64Array, maxLag: number): Float64Array {
}
const tempoPrior = (lag: number, fps: number) =>
- Math.exp(-0.5 * Math.log2(60 / (lag / fps) / 120) ** 2);
+ Math.exp(-0.5 * (Math.log2(60 / (lag / fps) / 120) / 0.8) ** 2);
interface LagRange {
shortest: number;
@@ -256,13 +304,13 @@ function trimmed(beats: number[], local: Float64Array): number[] {
const LOCAL_WINDOW_SECONDS = 8;
-/** The period through the piece: eight-second windows a second apart, held near the global period, median-smoothed. */
+/** The period through the piece: eight-second windows a second apart, held within 10% of the global period, median-smoothed. */
function localPeriods(curve: Float64Array, fps: number, period: number): (frame: number) => number {
const window = Math.round(LOCAL_WINDOW_SECONDS * fps);
const hop = Math.round(fps);
const range = {
- shortest: Math.max(fullRange(fps).shortest, Math.floor(period * 0.7)),
- longest: Math.min(fullRange(fps).longest, Math.ceil(period * 1.4)),
+ shortest: Math.max(fullRange(fps).shortest, Math.floor(period * 0.9)),
+ longest: Math.min(fullRange(fps).longest, Math.ceil(period * 1.1)),
};
const centres: number[] = [];
const periods: number[] = [];
@@ -303,25 +351,81 @@ function onsetLift(frames: number[], local: Float64Array): number {
return overall > 0 ? onBeats / overall - 1 : 0;
}
-/** The meter and bar phase whose beats carry the most low-band onset energy against the others. */
-function barPhase(beats: number[], curves: OnsetCurves): { meter: number; phase: number } {
- const strength = (f: number) =>
- Math.max(
- ...[-1, 0, 1, 2].map((d) => (curves.low[f + d] ?? 0) + 0.3 * (curves.broad[f + d] ?? 0)),
- );
- const levels = beats.map(strength);
+const mean = (values: readonly number[]) =>
+ values.reduce((a, b) => a + b, 0) / (values.length || 1);
+
+function zScored(values: number[]): number[] {
+ const centre = mean(values);
+ const spread = Math.sqrt(mean(values.map((v) => (v - centre) ** 2))) || 1;
+ return values.map((v) => (v - centre) / spread);
+}
+
+const CHROMA_REACH_SECONDS = CHROMA_SIZE / 2 / ANALYSIS_RATE;
+
+/** The unit-length mean chroma from one time to another, in seconds, leaving out frames that already hear the next onset. */
+function chromaBetween({ chroma, chromaFps }: BandFrames, from: number, to: number): number[] {
+ const middle = Math.round(((from + to) / 2) * chromaFps);
+ let first = Math.ceil(from * chromaFps);
+ let end = Math.floor((to - CHROMA_REACH_SECONDS) * chromaFps) + 1;
+ if (end <= first) [first, end] = [middle, middle + 1];
+ const sum = new Array<number>(12).fill(0);
+ for (let f = first; f < end; f++) for (let c = 0; c < 12; c++) sum[c]! += chroma[f]?.[c] ?? 0;
+ const length = Math.hypot(...sum) || 1;
+ return sum.map((v) => v / length);
+}
+
+const cosine = (a: number[], b: number[]) => a.reduce((sum, v, i) => sum + v * b[i]!, 0);
+
+/** How much the harmony changes at each beat: the chroma `span` beats before it against the `span` after. */
+function harmonicChange(beats: number[], fps: number, frames: BandFrames, span: number): number[] {
+ return beats.map((f, k) => {
+ if (k < span || k + span >= beats.length) return 0;
+ const before = chromaBetween(frames, beats[k - span]! / fps, f / fps);
+ return 1 - cosine(before, chromaBetween(frames, f / fps, beats[k + span]! / fps));
+ });
+}
+
+/** Each beat's evidence of starting a bar: a low-end hit, a chord change, and a chord change over half a bar. */
+function barStartEvidence(beats: number[], curves: OnsetCurves, frames: BandFrames): number[] {
+ const lowHit = (f: number) => Math.max(...[-1, 0, 1, 2].map((d) => curves.low[f + d] ?? 0));
+ const cues = [
+ zScored(beats.map(lowHit)),
+ zScored(harmonicChange(beats, curves.fps, frames, 1)),
+ zScored(harmonicChange(beats, curves.fps, frames, 2)),
+ ];
+ return beats.map((_, k) => cues.reduce((sum, cue) => sum + cue[k]!, 0));
+}
+
+/** Most music is in four, so three must win by a clear margin. */
+const TRIPLE_METER_HANDICAP = 0.7;
+
+function barPhase(beats: number[], curves: OnsetCurves, frames: BandFrames) {
+ const evidence = barStartEvidence(beats, curves, frames);
let best = { meter: 4, phase: 0, contrast: -Infinity };
for (const meter of [4, 3])
for (let phase = 0; phase < meter; phase++) {
- const on = levels.filter((_, k) => (k - phase) % meter === 0);
- const off = levels.filter((_, k) => (k - phase) % meter !== 0);
- const mean = (v: number[]) => v.reduce((a, b) => a + b, 0) / (v.length || 1);
- const contrast = (mean(on) - mean(off)) * (meter === 3 ? 0.85 : 1);
+ const on = evidence.filter((_, k) => (k - phase) % meter === 0);
+ const off = evidence.filter((_, k) => (k - phase) % meter !== 0);
+ const contrast = (mean(on) - mean(off)) * (meter === 3 ? TRIPLE_METER_HANDICAP : 1);
if (contrast > best.contrast) best = { meter, phase, contrast };
}
return best;
}
+function downbeatsOf(
+ beatList: number[],
+ times: number[],
+ curves: OnsetCurves,
+ frames: BandFrames,
+): Downbeats {
+ const bar = beatList.length >= 6 ? barPhase(beatList, curves, frames) : null;
+ return {
+ times: bar ? times.filter((_, k) => (k - bar.phase) % bar.meter === 0) : [],
+ meter: bar?.meter ?? null,
+ confidence: "low",
+ };
+}
+
export function medianInterval(times: readonly number[]): number | null {
const gaps = times
.slice(1)
@@ -339,15 +443,15 @@ const noRhythm = (confidence: number, reason = "no steady beat"): Rhythm => ({
const SHORTEST_SECONDS = 6;
/** Below this the beats are more likely the tracker's own grid than the music's. */
-export const STEADY_BEAT_CONFIDENCE = 0.05;
+export const STEADY_BEAT_CONFIDENCE = 0.15;
/** Clarity of the period, scaled down when the beats are under three times as strong as the onsets around them. */
const beatConfidence = (clarity: number, lift: number) =>
round(Math.max(0, Math.min(1, clarity)) * Math.max(0, Math.min(1, lift / 3)));
-/** Beats and downbeats of `frames`, the band energies of a signal at `sampleRate`. */
-export function rhythmOf(frames: BandFrames, sampleRate: number): Rhythm {
- const curves = onsetCurves(frames, sampleRate);
+/** Beats and downbeats of `frames`, a signal's band energies and chroma. */
+function rhythmOf(frames: BandFrames): Rhythm {
+ const curves = onsetCurves(frames);
const { period, clarity } = beatPeriod(curves.broad, curves.fps);
if (curves.broad.length < SHORTEST_SECONDS * curves.fps)
return noRhythm(0, "too short for a steady beat");
@@ -355,16 +459,8 @@ export function rhythmOf(frames: BandFrames, sampleRate: number): Rhythm {
const { frames: beatList, local } = beatFrames(curves.broad, curves.fps, period);
const confidence = beatConfidence(clarity, onsetLift(beatList, local));
if (beatList.length < 4 || confidence < STEADY_BEAT_CONFIDENCE) return noRhythm(confidence);
- const times = beatList.map((f) => round(f / curves.fps));
- const bar = beatList.length >= 6 ? barPhase(beatList, curves) : null;
- return {
- beats: { times, confidence },
- downbeats: {
- times: bar ? times.filter((_, k) => (k - bar.phase) % bar.meter === 0) : [],
- meter: bar?.meter ?? null,
- confidence: "low",
- },
- };
+ const times = beatList.map((f) => round(f / curves.fps + ONSET_LAG_SECONDS));
+ return { beats: { times, confidence }, downbeats: downbeatsOf(beatList, times, curves, frames) };
}
const TEMPO_STEP_SECONDS = 4;
@@ -405,5 +501,5 @@ export function bandFramesOf(signal: AudioSignal): BandFrames {
export function rhythm(signal: AudioSignal): Rhythm {
const frames = bandFramesOf(signal);
const entry = measured.get(signal)!;
- return (entry.rhythm ??= rhythmOf(frames, signal.sampleRate));
+ return (entry.rhythm ??= rhythmOf(frames));
}
diff --git a/packages/core/src/audioSections.ts b/packages/core/src/audioSections.ts
index 334adb8d6e..efd4cd0907 100644
--- a/packages/core/src/audioSections.ts
+++ b/packages/core/src/audioSections.ts
@@ -1,5 +1,5 @@
/**
- * Sections from signal processing: band energies averaged into half-second blocks, a self-similarity matrix, and
+ * Sections from signal processing: band energies and chroma averaged into half-second blocks, a self-similarity matrix, and
* Foote's checkerboard novelty, whose peaks are where the music changes. Boundaries snap to the nearest beat.
*/
@@ -47,14 +47,25 @@ function standardizeBands(blocks: number[][]): void {
}
}
-/** Band energies averaged over blocks, each band standardized over time and each block scaled to unit length. */
-function blockFeatures({ fps, bands }: BandFrames): number[][] {
+/** Block means of the band energies and of the log chroma, each part unit length within a block. */
+function blockParts({ fps, bands, chroma, chromaFps }: BandFrames): number[][] {
const per = Math.max(1, Math.round(BLOCK_SECONDS * fps));
+ const chromaPer = Math.max(1, Math.round(BLOCK_SECONDS * chromaFps));
const blocks: number[][] = [];
- for (let start = 0; start + per <= bands.length; start += per)
- blocks.push(meanOf(bands.slice(start, start + per)));
+ for (let start = 0, b = 0; start + per <= bands.length; start += per, b++) {
+ const pitch = meanOf(chroma.slice(b * chromaPer, (b + 1) * chromaPer)).map(Math.log1p);
+ blocks.push([...meanOf(bands.slice(start, start + per)), ...unit(pitch)]);
+ }
+ return blocks;
+}
+
+function blockFeatures(frames: BandFrames): number[][] {
+ const blocks = blockParts(frames);
standardizeBands(blocks);
- return blocks.map(unit);
+ const split = frames.bands[0]?.length ?? 0;
+ return blocks.map((block) =>
+ [...unit(block.slice(0, split)), ...unit(block.slice(split))].map((v) => v * Math.SQRT1_2),
+ );
}
function unit(vector: number[]): number[] {
diff --git a/packages/core/src/beats/beatDetection.test.ts b/packages/core/src/beats/beatDetection.test.ts
index d8e4244db2..841180c175 100644
--- a/packages/core/src/beats/beatDetection.test.ts
+++ b/packages/core/src/beats/beatDetection.test.ts
@@ -1,6 +1,7 @@
import { describe, expect, it, vi } from "vitest";
-import type { AudioSignal } from "../audioAnalysis";
+import { signalProvider, type AudioSignal } from "../audioAnalysis";
+import { STEADY_BEAT_CONFIDENCE, rhythm } from "../audioRhythm";
import { analyzeMusicFromBuffer, analyzeMusicFromSignal } from "./beatDetection";
const RATE = 22_050;
@@ -105,6 +106,17 @@ describe("analyzeMusicFromSignal", () => {
}
});
+ it("refuses an ambient pad in the tracker itself, so every caller of the signal tracker agrees", async () => {
+ const ambient = mono(pad(30));
+ const { beats, downbeats } = rhythm(ambient);
+ expect(beats).toMatchObject({ times: [], reason: "no steady beat" });
+ expect(beats.confidence).toBeLessThan(STEADY_BEAT_CONFIDENCE);
+ expect(downbeats.times).toEqual([]);
+ await expect(signalProvider.analyze("beats", ambient, {})).resolves.toMatchObject({
+ times: [],
+ });
+ });
+
it("waits on the caller's pause before each long stage", async () => {
const samples = new Float32Array(8 * RATE);
clicks(samples, grid(0.25, 0.5, 7.5), 0.6);
diff --git a/packages/core/src/beats/beatDetection.ts b/packages/core/src/beats/beatDetection.ts
index f133bd8e14..63a271fc03 100644
--- a/packages/core/src/beats/beatDetection.ts
+++ b/packages/core/src/beats/beatDetection.ts
@@ -1,5 +1,5 @@
import { energyMix, type AudioSignal } from "../audioAnalysis";
-import { bandFrames, rhythmOf, tempoOf } from "../audioRhythm";
+import { bandFramesOf, rhythm, tempoOf } from "../audioRhythm";
export interface MusicBeatAnalysis {
beatTimes: number[];
@@ -40,8 +40,6 @@ export function strengthAtTime(
return Math.min(1, computeRmsAt(analysis.channelData, analysis.sampleRate, time) / analysis.peak);
}
-/** Below this tracker confidence the grid is more likely the tracker's than the music's, so none is reported. */
-const MIN_CONFIDENCE = 0.2;
/** At or above this the tempo is reported as "high" confidence. */
const HIGH_CONFIDENCE = 0.35;
@@ -62,13 +60,13 @@ export async function analyzeMusicFromSignal(
{ pause }: MusicAnalysisOptions = {},
): Promise<MusicBeatAnalysis> {
await pause?.();
- const frames = bandFrames(signal);
+ bandFramesOf(signal);
await pause?.();
- const { beats } = rhythmOf(frames, signal.sampleRate);
+ const { beats } = rhythm(signal);
await pause?.();
const channelData = energyMix(signal);
const confidence = beats.confidence ?? 0;
- const times = confidence >= MIN_CONFIDENCE ? beats.times : [];
+ const times = beats.times;
const { strengths, peak } = beatStrengths(times, channelData, signal.sampleRate);
const bpm = tempoOf(times).value;
return {