Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
feat(core): signal rhythm is rate-independent, refuses grids in the tracker, follows chords (#5644)
Xuanru Li695110e46ba4
diff --git a/packages/core/src/audioResample.test.ts b/packages/core/src/audioResample.test.tsnew file mode 100644index 0000000000..6d6daca630--- /dev/null+++ b/packages/core/src/audioResample.test.ts@@ -0,0 +1,35 @@+import { describe, expect, it } from "vitest";+import { resample } from "./audioResample";++const tone = (hz: number, rate: number, seconds: number) =>+ Float32Array.from({ length: seconds * rate }, (_, i) => Math.sin((2 * Math.PI * hz * i) / rate));++const rms = (x: Float32Array) => Math.sqrt(x.reduce((a, v) => a + v * v, 0) / x.length);++/** Upward zero crossings per second, over the middle of the signal. */+function frequency(x: Float32Array, rate: number): number {+ const from = Math.floor(x.length / 4);+ const to = Math.floor((3 * x.length) / 4);+ let crossings = 0;+ for (let i = from + 1; i < to; i++) if (x[i - 1]! < 0 && x[i]! >= 0) crossings++;+ return (crossings * rate) / (to - from);+}++describe("resample", () => {+ it("keeps a tone's pitch and level from 48 kHz to 22.05 kHz", () => {+ const out = resample(tone(1000, 48_000, 2), 48_000, 22_050);+ expect(out.length).toBe(44_100);+ expect(frequency(out, 22_050)).toBeCloseTo(1000, -1);+ expect(rms(out.subarray(1000, -1000))).toBeCloseTo(Math.SQRT1_2, 2);+ });++ it("removes what the lower rate cannot hold instead of folding it down", () => {+ const out = resample(tone(15_000, 48_000, 1), 48_000, 22_050);+ expect(rms(out.subarray(1000, -1000))).toBeLessThan(0.01);+ });++ it("returns the samples untouched at the same rate", () => {+ const x = tone(440, 22_050, 1);+ expect(resample(x, 22_050, 22_050)).toBe(x);+ });+});diff --git a/packages/core/src/audioResample.ts b/packages/core/src/audioResample.tsnew file mode 100644index 0000000000..60add57aec--- /dev/null+++ b/packages/core/src/audioResample.ts@@ -0,0 +1,38 @@+/** Band-limited resampling by windowed-sinc interpolation, so analysis reads the same at any decode rate. */++const ZERO_CROSSINGS = 8;+const STEPS_PER_CROSSING = 256;+/** Cutoff as a share of the lower Nyquist, leaving the window's transition band below it. */+const CUTOFF = 0.95;++const KERNEL = Float64Array.from({ length: ZERO_CROSSINGS * STEPS_PER_CROSSING + 2 }, (_, i) => {+ const x = i / STEPS_PER_CROSSING;+ if (x >= ZERO_CROSSINGS) return 0;+ const sinc = x === 0 ? 1 : Math.sin(Math.PI * x) / (Math.PI * x);+ return sinc * (0.5 + 0.5 * Math.cos((Math.PI * x) / ZERO_CROSSINGS));+});++function kernelAt(x: number): number {+ const at = Math.abs(x) * STEPS_PER_CROSSING;+ const i = Math.floor(at);+ if (i + 1 >= KERNEL.length) return 0;+ const frac = at - i;+ return KERNEL[i]! * (1 - frac) + KERNEL[i + 1]! * frac;+}++export function resample(samples: Float32Array, from: number, to: number): Float32Array {+ if (from === to) return samples;+ const ratio = to / from;+ const scale = Math.min(1, ratio) * CUTOFF;+ const reach = ZERO_CROSSINGS / scale;+ const out = new Float32Array(Math.floor(samples.length * ratio));+ for (let n = 0; n < out.length; n++) {+ const t = n / ratio;+ const last = Math.min(samples.length - 1, Math.floor(t + reach));+ let sum = 0;+ for (let k = Math.max(0, Math.ceil(t - reach)); k <= last; k++)+ sum += samples[k]! * kernelAt((t - k) * scale);+ out[n] = sum * scale;+ }+ return out;+}diff --git a/packages/core/src/audioRhythm.test.ts b/packages/core/src/audioRhythm.test.tsindex 2667d5d40c..760a85c1d5 100644--- a/packages/core/src/audioRhythm.test.ts+++ b/packages/core/src/audioRhythm.test.ts@@ -15,14 +15,14 @@ function random(seed = 3) {}/** A drum pattern on a known grid: a kick with a low thump on each bar's first beat, a click on the others. */-function drums(beatTimes: number[], meter: number, seconds: number): AudioSignal {- const samples = new Float32Array(Math.round(seconds * RATE));+function drums(beatTimes: number[], meter: number, seconds: number, rate = RATE): AudioSignal {+ const samples = new Float32Array(Math.round(seconds * rate));const noise = random();beatTimes.forEach((t, k) => {const downbeat = k % meter === 0;- const start = Math.round(t * RATE);- for (let i = 0; i < 0.12 * RATE && start + i < samples.length; i++) {- const since = i / RATE;+ const start = Math.round(t * rate);+ for (let i = 0; i < 0.12 * rate && start + i < samples.length; i++) {+ const since = i / rate;const click = (downbeat ? 0.5 : 0.35) * noise() * Math.exp(-since / 0.02);const thump = downbeat? 0.8 * Math.sin(2 * Math.PI * 55 * since) * Math.exp(-since / 0.06)@@ -30,6 +30,31 @@ function drums(beatTimes: number[], meter: number, seconds: number): AudioSignalsamples[start + i]! += click + thump;}});+ return { sampleRate: rate, channels: [samples] };+}++/** The same hit on every beat, over chords that change every four beats from beat `firstChange`. */+function chordsOverEvenHits(+ beatTimes: number[],+ firstChange: number,+ seconds: number,+): AudioSignal {+ const samples = new Float32Array(Math.round(seconds * RATE));+ const noise = random(7);+ const roots = [220, 175, 262, 196];+ beatTimes.forEach((t) => {+ const start = Math.round(t * RATE);+ for (let i = 0; i < 0.08 * RATE && start + i < samples.length; i++)+ samples[start + i]! += 0.4 * noise() * Math.exp(-i / RATE / 0.02);+ });+ const period = beatTimes[1]! - beatTimes[0]!;+ for (let i = 0; i < samples.length; i++) {+ const t = i / RATE;+ const bar = Math.floor((t - beatTimes[firstChange]!) / (4 * period));+ const root = roots[((bar % 4) + 4) % 4]!;+ for (const ratio of [1, 1.26, 1.5])+ samples[i]! += 0.08 * Math.sin(2 * Math.PI * root * ratio * t);+ }return { sampleRate: RATE, channels: [samples] };}@@ -66,6 +91,23 @@ describe("rhythm", () => {expect(curve[curve.length - 1]!.bpm).toBeCloseTo(133, -1);});+ it("gives the same beats at any sample rate and for mono or the same sound on two channels", () => {+ const truth = grid(0.25, 0.5, 19.5);+ const reference = rhythm(drums(truth, 4, 20)).beats.times;+ const wide = drums(truth, 4, 20, 44_100).channels[0]!;+ const stereo = rhythm({ sampleRate: 44_100, channels: [wide, wide.slice()] }).beats.times;+ expect(stereo).toHaveLength(reference.length);+ stereo.forEach((t, k) => expect(Math.abs(t - reference[k]!)).toBeLessThanOrEqual(0.011));+ });++ it("puts the bar where the chords change when every beat hits the same", () => {+ const truth = grid(0.25, 0.5, 23.5);+ const { downbeats } = rhythm(chordsOverEvenHits(truth, 1, 24));+ expect(downbeats.meter).toBe(4);+ const changes = truth.filter((_, k) => k % 4 === 1);+ expect(recall(downbeats.times, changes)).toBeGreaterThan(0.9);+ });+it("finds no steady beat in silence or in noise, and none in a clip too short to tell", () => {const silent = rhythm({ sampleRate: RATE, channels: [new Float32Array(10 * RATE)] });expect(silent.beats).toMatchObject({ times: [], reason: "no steady beat" });@@ -75,6 +117,7 @@ describe("rhythm", () => {channels: [Float32Array.from({ length: 10 * RATE }, () => 0.3 * noise())],});expect(hiss.beats.times).toEqual([]);+ expect(hiss.downbeats.times).toEqual([]);const short = rhythm(drums(grid(0.25, 0.5, 3), 4, 3));expect(short.beats).toMatchObject({ times: [], reason: "too short for a steady beat" });});diff --git a/packages/core/src/audioRhythm.ts b/packages/core/src/audioRhythm.tsindex 74fa42d36e..a0608630f3 100644--- a/packages/core/src/audioRhythm.ts+++ b/packages/core/src/audioRhythm.ts@@ -4,6 +4,7 @@*/import type { AudioSignal } from "./audioAnalysis";+import { resample } from "./audioResample";import { fft } from "./fft";export interface Beats {@@ -18,7 +19,7 @@ export interface Downbeats {times: number[];/** Beats per bar, 3 or 4; null when there are too few beats to tell. */meter: number | null;- /** "low" for a guess from where the low end hits hardest. */+ /** "low": a guess from where the harmony changes and the low end hits. */confidence?: "low";}@@ -39,28 +40,39 @@ export interface Rhythm {downbeats: Downbeats;}-/** Log energy per frame in log-spaced bands, the frames `fps` per second. */+/** Log energy per frame in log-spaced bands, `fps` frames per second, and a slower pitch-class profile. */export interface BandFrames {fps: number;bands: Float64Array[];+ chroma: Float64Array[];+ chromaFps: number;}+/** Every signal is analyzed at this rate and as the mean of its channels' power, so decode rate and layout cannot change a result. */+const ANALYSIS_RATE = 22050;+const FRAME_SIZE = 1024;+const CHROMA_SIZE = 4096;+const CHROMA_FPS = 10;const FRAMES_PER_SECOND = 100;const LOW_BAND_HZ = 150;const SLOWEST_BPM = 40;const FASTEST_BPM = 240;const TIGHTNESS = 100;+/** The onset curve peaks this much ahead of where listeners place the beat, so beats move this much later. */+const ONSET_LAG_SECONDS = 0.015;const round = (t: number) => Math.round(t * 1000) / 1000;-function bandEdges(sampleRate: number, size: number): number[] {- const binHz = sampleRate / size;+function bandEdges(): number[] {+ const binHz = ANALYSIS_RATE / FRAME_SIZE;const edges: number[] = [];- for (let hz = 30; hz < Math.min(9000, sampleRate / 2); hz *= 2 ** (1 / 5))- edges.push(Math.max(1, Math.round(hz / binHz)));+ for (let hz = 30; hz < 9000; hz *= 2 ** (1 / 5)) edges.push(Math.max(1, Math.round(hz / binHz)));return edges;}+const hann = (size: number) =>+ Float64Array.from({ length: size }, (_, i) => 0.5 - 0.5 * Math.cos((2 * Math.PI * i) / size));+/** Add each band's power in the Hann-windowed frame centred on `center` into `into`. */function addFramePower(samples: Float32Array,@@ -86,24 +98,61 @@ function addFramePower(}}-/** Per-band log power, summed over channels so a polarity-flipped channel cannot cancel. */-export function bandFrames(signal: AudioSignal): BandFrames {- const { sampleRate, channels } = signal;- const hop = Math.max(1, Math.round(sampleRate / FRAMES_PER_SECOND));- const size = 2 ** Math.round(Math.log2(sampleRate * 0.046));- const window = Float64Array.from(- { length: size },- (_, i) => 0.5 - 0.5 * Math.cos((2 * Math.PI * i) / size),- );- const edges = bandEdges(sampleRate, size);- const length = channels[0]?.length ?? 0;+/** Per-band log power of every frame, averaged over channels so a polarity-flipped channel cannot cancel. */+function bandPower(channels: Float32Array[]): Float64Array[] {+ const window = hann(FRAME_SIZE);+ const edges = bandEdges();+ const hop = ANALYSIS_RATE / FRAMES_PER_SECOND;const bands: Float64Array[] = [];- for (let center = 0; center < length; center += hop) {- const power = new Float64Array(Math.max(0, edges.length - 1));- for (const channel of channels) addFramePower(channel, center, window, edges, power);- bands.push(power.map((p) => Math.log1p(1e9 * p)));+ for (let f = 0; Math.round(f * hop) < (channels[0]?.length ?? 0); f++) {+ const power = new Float64Array(edges.length - 1);+ for (const channel of channels)+ addFramePower(channel, Math.round(f * hop), window, edges, power);+ bands.push(power.map((p) => Math.log1p((1e9 * p) / channels.length)));+ }+ return bands;+}++/** Index of the pitch class, A = 9, of each FFT bin from 55 Hz to 2 kHz; -1 outside that range. */+const PITCH_CLASS = Array.from({ length: CHROMA_SIZE / 2 }, (_, bin) => {+ const hz = (bin * ANALYSIS_RATE) / CHROMA_SIZE;+ if (hz < 55 || hz > 2000) return -1;+ return (((Math.round(12 * Math.log2(hz / 440)) + 9) % 12) + 12) % 12;+});++function chromaFrames(channels: Float32Array[]): Float64Array[] {+ const window = hann(CHROMA_SIZE);+ const hop = ANALYSIS_RATE / CHROMA_FPS;+ const re = new Float64Array(CHROMA_SIZE);+ const im = new Float64Array(CHROMA_SIZE);+ const frames: Float64Array[] = [];+ for (let f = 0; Math.round(f * hop) < (channels[0]?.length ?? 0); f++) {+ const chroma = new Float64Array(12);+ const first = Math.round(f * hop) - CHROMA_SIZE / 2;+ for (const channel of channels) {+ for (let i = 0; i < CHROMA_SIZE; i++) {+ const at = first + i;+ re[i] = at >= 0 && at < channel.length ? channel[at]! * window[i]! : 0;+ im[i] = 0;+ }+ fft(re, im);+ PITCH_CLASS.forEach((pc, bin) => {+ if (pc >= 0) chroma[pc]! += Math.hypot(re[bin]!, im[bin]!);+ });+ }+ frames.push(chroma);}- return { fps: sampleRate / hop, bands };+ return frames;+}++function bandFrames(signal: AudioSignal): BandFrames {+ const channels = signal.channels.map((c) => resample(c, signal.sampleRate, ANALYSIS_RATE));+ return {+ fps: FRAMES_PER_SECOND,+ bands: bandPower(channels),+ chroma: chromaFrames(channels),+ chromaFps: CHROMA_FPS,+ };}/** Subtract the mean of the surrounding ±`radius` frames and keep what rises above it. */@@ -131,10 +180,9 @@ interface OnsetCurves {low: Float64Array;}-function onsetCurves({ fps, bands }: BandFrames, sampleRate: number): OnsetCurves {- const size = 2 ** Math.round(Math.log2(sampleRate * 0.046));- const lowBands = bandEdges(sampleRate, size).filter(- (bin) => (bin * sampleRate) / size < LOW_BAND_HZ,+function onsetCurves({ fps, bands }: BandFrames): OnsetCurves {+ const lowBands = bandEdges().filter(+ (bin) => (bin * ANALYSIS_RATE) / FRAME_SIZE < LOW_BAND_HZ,).length;const broad = new Float64Array(bands.length);const low = new Float64Array(bands.length);@@ -170,7 +218,7 @@ function autocorrelation(curve: Float64Array, maxLag: number): Float64Array {}const tempoPrior = (lag: number, fps: number) =>- Math.exp(-0.5 * Math.log2(60 / (lag / fps) / 120) ** 2);+ Math.exp(-0.5 * (Math.log2(60 / (lag / fps) / 120) / 0.8) ** 2);interface LagRange {shortest: number;@@ -256,13 +304,13 @@ function trimmed(beats: number[], local: Float64Array): number[] {const LOCAL_WINDOW_SECONDS = 8;-/** The period through the piece: eight-second windows a second apart, held near the global period, median-smoothed. */+/** The period through the piece: eight-second windows a second apart, held within 10% of the global period, median-smoothed. */function localPeriods(curve: Float64Array, fps: number, period: number): (frame: number) => number {const window = Math.round(LOCAL_WINDOW_SECONDS * fps);const hop = Math.round(fps);const range = {- shortest: Math.max(fullRange(fps).shortest, Math.floor(period * 0.7)),- longest: Math.min(fullRange(fps).longest, Math.ceil(period * 1.4)),+ shortest: Math.max(fullRange(fps).shortest, Math.floor(period * 0.9)),+ longest: Math.min(fullRange(fps).longest, Math.ceil(period * 1.1)),};const centres: number[] = [];const periods: number[] = [];@@ -303,25 +351,81 @@ function onsetLift(frames: number[], local: Float64Array): number {return overall > 0 ? onBeats / overall - 1 : 0;}-/** The meter and bar phase whose beats carry the most low-band onset energy against the others. */-function barPhase(beats: number[], curves: OnsetCurves): { meter: number; phase: number } {- const strength = (f: number) =>- Math.max(- ...[-1, 0, 1, 2].map((d) => (curves.low[f + d] ?? 0) + 0.3 * (curves.broad[f + d] ?? 0)),- );- const levels = beats.map(strength);+const mean = (values: readonly number[]) =>+ values.reduce((a, b) => a + b, 0) / (values.length || 1);++function zScored(values: number[]): number[] {+ const centre = mean(values);+ const spread = Math.sqrt(mean(values.map((v) => (v - centre) ** 2))) || 1;+ return values.map((v) => (v - centre) / spread);+}++const CHROMA_REACH_SECONDS = CHROMA_SIZE / 2 / ANALYSIS_RATE;++/** The unit-length mean chroma from one time to another, in seconds, leaving out frames that already hear the next onset. */+function chromaBetween({ chroma, chromaFps }: BandFrames, from: number, to: number): number[] {+ const middle = Math.round(((from + to) / 2) * chromaFps);+ let first = Math.ceil(from * chromaFps);+ let end = Math.floor((to - CHROMA_REACH_SECONDS) * chromaFps) + 1;+ if (end <= first) [first, end] = [middle, middle + 1];+ const sum = new Array<number>(12).fill(0);+ for (let f = first; f < end; f++) for (let c = 0; c < 12; c++) sum[c]! += chroma[f]?.[c] ?? 0;+ const length = Math.hypot(...sum) || 1;+ return sum.map((v) => v / length);+}++const cosine = (a: number[], b: number[]) => a.reduce((sum, v, i) => sum + v * b[i]!, 0);++/** How much the harmony changes at each beat: the chroma `span` beats before it against the `span` after. */+function harmonicChange(beats: number[], fps: number, frames: BandFrames, span: number): number[] {+ return beats.map((f, k) => {+ if (k < span || k + span >= beats.length) return 0;+ const before = chromaBetween(frames, beats[k - span]! / fps, f / fps);+ return 1 - cosine(before, chromaBetween(frames, f / fps, beats[k + span]! / fps));+ });+}++/** Each beat's evidence of starting a bar: a low-end hit, a chord change, and a chord change over half a bar. */+function barStartEvidence(beats: number[], curves: OnsetCurves, frames: BandFrames): number[] {+ const lowHit = (f: number) => Math.max(...[-1, 0, 1, 2].map((d) => curves.low[f + d] ?? 0));+ const cues = [+ zScored(beats.map(lowHit)),+ zScored(harmonicChange(beats, curves.fps, frames, 1)),+ zScored(harmonicChange(beats, curves.fps, frames, 2)),+ ];+ return beats.map((_, k) => cues.reduce((sum, cue) => sum + cue[k]!, 0));+}++/** Most music is in four, so three must win by a clear margin. */+const TRIPLE_METER_HANDICAP = 0.7;++function barPhase(beats: number[], curves: OnsetCurves, frames: BandFrames) {+ const evidence = barStartEvidence(beats, curves, frames);let best = { meter: 4, phase: 0, contrast: -Infinity };for (const meter of [4, 3])for (let phase = 0; phase < meter; phase++) {- const on = levels.filter((_, k) => (k - phase) % meter === 0);- const off = levels.filter((_, k) => (k - phase) % meter !== 0);- const mean = (v: number[]) => v.reduce((a, b) => a + b, 0) / (v.length || 1);- const contrast = (mean(on) - mean(off)) * (meter === 3 ? 0.85 : 1);+ const on = evidence.filter((_, k) => (k - phase) % meter === 0);+ const off = evidence.filter((_, k) => (k - phase) % meter !== 0);+ const contrast = (mean(on) - mean(off)) * (meter === 3 ? TRIPLE_METER_HANDICAP : 1);if (contrast > best.contrast) best = { meter, phase, contrast };}return best;}+function downbeatsOf(+ beatList: number[],+ times: number[],+ curves: OnsetCurves,+ frames: BandFrames,+): Downbeats {+ const bar = beatList.length >= 6 ? barPhase(beatList, curves, frames) : null;+ return {+ times: bar ? times.filter((_, k) => (k - bar.phase) % bar.meter === 0) : [],+ meter: bar?.meter ?? null,+ confidence: "low",+ };+}+export function medianInterval(times: readonly number[]): number | null {const gaps = times.slice(1)@@ -339,15 +443,15 @@ const noRhythm = (confidence: number, reason = "no steady beat"): Rhythm => ({const SHORTEST_SECONDS = 6;/** Below this the beats are more likely the tracker's own grid than the music's. */-export const STEADY_BEAT_CONFIDENCE = 0.05;+export const STEADY_BEAT_CONFIDENCE = 0.15;/** Clarity of the period, scaled down when the beats are under three times as strong as the onsets around them. */const beatConfidence = (clarity: number, lift: number) =>round(Math.max(0, Math.min(1, clarity)) * Math.max(0, Math.min(1, lift / 3)));-/** Beats and downbeats of `frames`, the band energies of a signal at `sampleRate`. */-export function rhythmOf(frames: BandFrames, sampleRate: number): Rhythm {- const curves = onsetCurves(frames, sampleRate);+/** Beats and downbeats of `frames`, a signal's band energies and chroma. */+function rhythmOf(frames: BandFrames): Rhythm {+ const curves = onsetCurves(frames);const { period, clarity } = beatPeriod(curves.broad, curves.fps);if (curves.broad.length < SHORTEST_SECONDS * curves.fps)return noRhythm(0, "too short for a steady beat");@@ -355,16 +459,8 @@ export function rhythmOf(frames: BandFrames, sampleRate: number): Rhythm {const { frames: beatList, local } = beatFrames(curves.broad, curves.fps, period);const confidence = beatConfidence(clarity, onsetLift(beatList, local));if (beatList.length < 4 || confidence < STEADY_BEAT_CONFIDENCE) return noRhythm(confidence);- const times = beatList.map((f) => round(f / curves.fps));- const bar = beatList.length >= 6 ? barPhase(beatList, curves) : null;- return {- beats: { times, confidence },- downbeats: {- times: bar ? times.filter((_, k) => (k - bar.phase) % bar.meter === 0) : [],- meter: bar?.meter ?? null,- confidence: "low",- },- };+ const times = beatList.map((f) => round(f / curves.fps + ONSET_LAG_SECONDS));+ return { beats: { times, confidence }, downbeats: downbeatsOf(beatList, times, curves, frames) };}const TEMPO_STEP_SECONDS = 4;@@ -405,5 +501,5 @@ export function bandFramesOf(signal: AudioSignal): BandFrames {export function rhythm(signal: AudioSignal): Rhythm {const frames = bandFramesOf(signal);const entry = measured.get(signal)!;- return (entry.rhythm ??= rhythmOf(frames, signal.sampleRate));+ return (entry.rhythm ??= rhythmOf(frames));}diff --git a/packages/core/src/audioSections.ts b/packages/core/src/audioSections.tsindex 334adb8d6e..efd4cd0907 100644--- a/packages/core/src/audioSections.ts+++ b/packages/core/src/audioSections.ts@@ -1,5 +1,5 @@/**- * Sections from signal processing: band energies averaged into half-second blocks, a self-similarity matrix, and+ * Sections from signal processing: band energies and chroma averaged into half-second blocks, a self-similarity matrix, and* Foote's checkerboard novelty, whose peaks are where the music changes. Boundaries snap to the nearest beat.*/@@ -47,14 +47,25 @@ function standardizeBands(blocks: number[][]): void {}}-/** Band energies averaged over blocks, each band standardized over time and each block scaled to unit length. */-function blockFeatures({ fps, bands }: BandFrames): number[][] {+/** Block means of the band energies and of the log chroma, each part unit length within a block. */+function blockParts({ fps, bands, chroma, chromaFps }: BandFrames): number[][] {const per = Math.max(1, Math.round(BLOCK_SECONDS * fps));+ const chromaPer = Math.max(1, Math.round(BLOCK_SECONDS * chromaFps));const blocks: number[][] = [];- for (let start = 0; start + per <= bands.length; start += per)- blocks.push(meanOf(bands.slice(start, start + per)));+ for (let start = 0, b = 0; start + per <= bands.length; start += per, b++) {+ const pitch = meanOf(chroma.slice(b * chromaPer, (b + 1) * chromaPer)).map(Math.log1p);+ blocks.push([...meanOf(bands.slice(start, start + per)), ...unit(pitch)]);+ }+ return blocks;+}++function blockFeatures(frames: BandFrames): number[][] {+ const blocks = blockParts(frames);standardizeBands(blocks);- return blocks.map(unit);+ const split = frames.bands[0]?.length ?? 0;+ return blocks.map((block) =>+ [...unit(block.slice(0, split)), ...unit(block.slice(split))].map((v) => v * Math.SQRT1_2),+ );}function unit(vector: number[]): number[] {diff --git a/packages/core/src/beats/beatDetection.test.ts b/packages/core/src/beats/beatDetection.test.tsindex d8e4244db2..841180c175 100644--- a/packages/core/src/beats/beatDetection.test.ts+++ b/packages/core/src/beats/beatDetection.test.ts@@ -1,6 +1,7 @@import { describe, expect, it, vi } from "vitest";-import type { AudioSignal } from "../audioAnalysis";+import { signalProvider, type AudioSignal } from "../audioAnalysis";+import { STEADY_BEAT_CONFIDENCE, rhythm } from "../audioRhythm";import { analyzeMusicFromBuffer, analyzeMusicFromSignal } from "./beatDetection";const RATE = 22_050;@@ -105,6 +106,17 @@ describe("analyzeMusicFromSignal", () => {}});+ it("refuses an ambient pad in the tracker itself, so every caller of the signal tracker agrees", async () => {+ const ambient = mono(pad(30));+ const { beats, downbeats } = rhythm(ambient);+ expect(beats).toMatchObject({ times: [], reason: "no steady beat" });+ expect(beats.confidence).toBeLessThan(STEADY_BEAT_CONFIDENCE);+ expect(downbeats.times).toEqual([]);+ await expect(signalProvider.analyze("beats", ambient, {})).resolves.toMatchObject({+ times: [],+ });+ });+it("waits on the caller's pause before each long stage", async () => {const samples = new Float32Array(8 * RATE);clicks(samples, grid(0.25, 0.5, 7.5), 0.6);diff --git a/packages/core/src/beats/beatDetection.ts b/packages/core/src/beats/beatDetection.tsindex f133bd8e14..63a271fc03 100644--- a/packages/core/src/beats/beatDetection.ts+++ b/packages/core/src/beats/beatDetection.ts@@ -1,5 +1,5 @@import { energyMix, type AudioSignal } from "../audioAnalysis";-import { bandFrames, rhythmOf, tempoOf } from "../audioRhythm";+import { bandFramesOf, rhythm, tempoOf } from "../audioRhythm";export interface MusicBeatAnalysis {beatTimes: number[];@@ -40,8 +40,6 @@ export function strengthAtTime(return Math.min(1, computeRmsAt(analysis.channelData, analysis.sampleRate, time) / analysis.peak);}-/** Below this tracker confidence the grid is more likely the tracker's than the music's, so none is reported. */-const MIN_CONFIDENCE = 0.2;/** At or above this the tempo is reported as "high" confidence. */const HIGH_CONFIDENCE = 0.35;@@ -62,13 +60,13 @@ export async function analyzeMusicFromSignal({ pause }: MusicAnalysisOptions = {},): Promise<MusicBeatAnalysis> {await pause?.();- const frames = bandFrames(signal);+ bandFramesOf(signal);await pause?.();- const { beats } = rhythmOf(frames, signal.sampleRate);+ const { beats } = rhythm(signal);await pause?.();const channelData = energyMix(signal);const confidence = beats.confidence ?? 0;- const times = confidence >= MIN_CONFIDENCE ? beats.times : [];+ const times = beats.times;const { strengths, peak } = beatStrengths(times, channelData, signal.sampleRate);const bpm = tempoOf(times).value;return {