Skip to content

Write HTML. Render video. Built for agents.

feat(cli): audio answers beats, downbeats, bpm, sections and describe through a provider ladder (#5632)

Xuanru Li8e340d220ea4
diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx
index 3b65380cc7..77290194e7 100644
--- a/docs/packages/cli.mdx
+++ b/docs/packages/cli.mdx
@@ -540,9 +540,57 @@ For a file not in the film yet, `--place-at <s>` answers as if it started at tha
film second; `--skip-file-seconds <s>` cuts that much from the start of the file
first, and the result warns when the cut part held the file's loudest moment.
-Every measurement comes from a provider ladder. Today the only provider is
-local signal processing; `--quality fast|local|best` limits which providers may
-run, `--provider onsets=signal` pins one, and `--full` shows each ladder step.
+`--only` also asks for the music's structure, which is not measured unless asked:
+
+- **beats**: beat times, or none with a `reason` (`no steady beat`, or too short
+ to tell) when the music has no beat to follow.
+- **downbeats**: the first beat of each bar, with the `meter` (3 or 4). From
+ signal processing these are a guess (`confidence: "low"`).
+- **bpm**: the tempo read off the beats (median interval), with a `curve` every
+ 4 s for music that speeds up or slows down. It always comes from the provider
+ that answered `beats`.
+- **sections**: spans where the sound stays alike, each with a letter; a part
+ that returns gets its earlier letter.
+- **describe**: labels for what the audio holds, each with a score.
+
+Word timing is not here: use [`transcribe`](#transcribe).
+
+Every measurement comes from a provider ladder, tried in this order until one
+answers:
+
+| Capability | Ladder |
+| --- | --- |
+| moments, onsets, loudness, silence, envelope, kind | `signal` |
+| beats, downbeats (and so bpm) | `api`, `apple-mu`, `beat-this`, `signal` |
+| sections | `api`, `apple-mu`, `signal` |
+| describe | `api`, `apple-sound` |
+
+- `api` is an audio analysis service, used when `HYPERFRAMES_AUDIO_API_URL` and
+ `HYPERFRAMES_AUDIO_API_KEY` are both set. The audio is uploaded. Every beat,
+ downbeat or section boundary it answers moves onto the nearest beat signal
+ processing measured, when one is within a quarter beat; the rest are listed in
+ `snap.unsnapped`, and `snap.medianShiftMs` says how far the moved ones went.
+- `apple-mu` is Apple's Music Understanding, which needs macOS 27. It is not
+ available yet.
+- `beat-this` is [Beat This!](https://github.com/CPJKU/beat_this), used when
+ `HYPERFRAMES_BEAT_THIS_PYTHON` names a Python with `beat-this` installed and
+ its `small0` weights are on disk (`HYPERFRAMES_BEAT_THIS_CHECKPOINT`, or
+ `beat_this-small0.ckpt` in the Torch hub cache). The CLI never downloads them;
+ to set it up once:
+
+ ```bash
+ uv init --bare ~/.beat-this
+ uv add --project ~/.beat-this "beat-this @ git+https://github.com/CPJKU/beat_this" torch torchaudio soundfile
+ ~/.beat-this/.venv/bin/python -c "from beat_this.inference import load_checkpoint; load_checkpoint('small0')"
+ export HYPERFRAMES_BEAT_THIS_PYTHON=~/.beat-this/.venv/bin/python
+ ```
+- `apple-sound` is macOS's built-in sound classifier, through a small Swift
+ helper compiled once with the Xcode command line tools.
+
+A provider that cannot run, or fails, is passed over with a warning that says
+why. `--quality fast|local|best` limits which providers may run (`local` keeps
+the audio on this machine, `fast` allows only signal processing),
+`--provider beats=signal` pins one, and `--full` shows each ladder step.
Tune the analysis with `--onset-method`, `--threshold`, `--band lo,hi`,
`--hop-ms`, `--onset-ratio`, `--tail-db` and `--silence-db`. Decoding needs
FFmpeg.
diff --git a/packages/cli/src/audio/analysis/analyze.test.ts b/packages/cli/src/audio/analysis/analyze.test.ts
index 37847f690a..68e54b7873 100644
--- a/packages/cli/src/audio/analysis/analyze.test.ts
+++ b/packages/cli/src/audio/analysis/analyze.test.ts
@@ -38,10 +38,48 @@ describe("analyzeSignal", () => {
expect(analysis.values.moments).toEqual({ onset: 1.1, steepest: 1.1, peak: 1.1, tailEnd: 1.2 });
});
- it("warns, and answers nothing, for a capability no provider offers yet", async () => {
- const analysis = await analyzeSignal(tone, { capabilities: ["beats"] });
- expect(analysis.values.beats).toBeNull();
- expect(analysis.warnings).toEqual(["beats: no provider answered (none offers it)"]);
+ it("warns, and answers nothing, for a capability no provider offers", async () => {
+ const analysis = await analyzeSignal(tone, { capabilities: ["describe"] }, [signalProvider]);
+ expect(analysis.values.describe).toBeNull();
+ expect(analysis.warnings).toEqual(["describe: no provider answered (none offers it)"]);
+ });
+
+ it("reads tempo off the beats of whichever provider placed them, in file seconds", async () => {
+ const grid: AudioAnalysisProvider<AudioSignal> = {
+ id: "grid",
+ local: true,
+ capabilities: ["beats"],
+ analyze: async () => ({ times: [0, 0.5, 1, 1.5, 2, 2.5, 3, 3.5, 4] }),
+ };
+ const analysis = await analyzeSignal(tone, { capabilities: ["bpm", "beats"], fileOffset: 10 }, [
+ grid,
+ signalProvider,
+ ]);
+ expect(analysis.provenance).toEqual({ bpm: "grid", beats: "grid" });
+ expect(analysis.values.bpm).toEqual({
+ value: 120,
+ curve: [
+ { t: 10, bpm: 120 },
+ { t: 14, bpm: 120 },
+ ],
+ });
+ expect((analysis.values.beats as { times: number[] }).times[0]).toBe(10);
+ });
+
+ it("keeps a service off the ladder at quality fast", async () => {
+ const service: AudioAnalysisProvider<AudioSignal> = {
+ id: "service",
+ local: false,
+ capabilities: ["beats"],
+ analyze: async () => ({ times: [1] }),
+ };
+ const analysis = await analyzeSignal(
+ tone,
+ { capabilities: ["beats"], quality: "fast", options: {} },
+ [service, signalProvider],
+ );
+ expect(analysis.provenance.beats).toBe("signal");
+ expect(analysis.warnings).toEqual([]);
});
});
diff --git a/packages/cli/src/audio/analysis/analyze.ts b/packages/cli/src/audio/analysis/analyze.ts
index 2f4b2bf5f6..bc9850bd6c 100644
--- a/packages/cli/src/audio/analysis/analyze.ts
+++ b/packages/cli/src/audio/analysis/analyze.ts
@@ -2,12 +2,19 @@ import {
runLadder,
signalDuration,
signalProvider,
+ tempoOf,
type AnalysisQuality,
type AudioAnalysisProvider,
type AudioCapability,
type AudioSignal,
+ type Beats,
+ type LadderResult,
type LadderStep,
} from "@hyperframes/core/audio-analysis";
+import { apiProvider } from "./providers/api.js";
+import { appleMuProvider, appleSoundProvider } from "./providers/apple.js";
+import { beatThisProvider } from "./providers/beatThis.js";
+import { shiftedTimes } from "./fileTime.js";
/** What `hyperframes audio` asks for when `--only` is not given. */
export const DEFAULT_CAPABILITIES: readonly AudioCapability[] = [
@@ -19,9 +26,12 @@ export const DEFAULT_CAPABILITIES: readonly AudioCapability[] = [
"envelope",
];
-/** Every provider the CLI knows, in default ladder order. Model and service providers join this list. */
+/**
+ * Every provider the CLI knows, in default ladder order: each capability tries the ones that offer it in this order.
+ * beats, downbeats: api, apple-mu, beat-this, signal. sections: api, apple-mu, signal. describe: api, apple-sound.
+ */
function audioAnalysisProviders(): AudioAnalysisProvider<AudioSignal>[] {
- return [signalProvider];
+ return [apiProvider, appleMuProvider, beatThisProvider, appleSoundProvider, signalProvider];
}
export interface AnalyzeOptions {
@@ -54,36 +64,6 @@ export function envelopeHopMs(seconds: number, asked?: number): number {
const round = (t: number) => Math.round(t * 1000) / 1000;
-/** Move a capability's times from signal seconds to file seconds. */
-function toFileTime(capability: AudioCapability, value: unknown, offset: number): unknown {
- if (value === null || typeof value !== "object") return value;
- const at = (t: number) => round(t + offset);
- switch (capability) {
- case "moments": {
- const m = value as Record<string, number>;
- return Object.fromEntries(Object.entries(m).map(([k, t]) => [k, at(t)]));
- }
- case "onsets": {
- const o = value as { events: { t: number; strength: number }[] };
- return { ...o, events: o.events.map((e) => ({ ...e, t: at(e.t) })) };
- }
- case "silence":
- return (value as { start: number; end: number }[]).map((s) => ({
- start: at(s.start),
- end: at(s.end),
- }));
- case "loudness": {
- const l = value as { clipping: { count: number; firstAt: number | null } };
- const firstAt = l.clipping.firstAt === null ? null : at(l.clipping.firstAt);
- return { ...l, clipping: { ...l.clipping, firstAt } };
- }
- case "envelope":
- return { startsAt: round(offset), ...(value as object) };
- default:
- return value;
- }
-}
-
function within(value: unknown, from: number, to: number): unknown {
if (!value || typeof value !== "object" || !("events" in value)) return value;
const onsets = value as { events: { t: number }[] };
@@ -123,6 +103,9 @@ export async function analyzeSignal(
request: AnalyzeOptions,
providers: readonly AudioAnalysisProvider<AudioSignal>[] = audioAnalysisProviders(),
): Promise<Analysis> {
+ const { beats, bpm } = request.provider ?? {};
+ if (beats && bpm && beats !== bpm) throw new Error("beats and bpm must use the same provider");
+ const beatProvider = beats ?? bpm;
const framing = frame(signal, request.context);
const analysis: Analysis = {
analyzed: round(signalDuration(framing.windowed)),
@@ -131,20 +114,37 @@ export async function analyzeSignal(
ladder: {},
warnings: [],
};
+ const ladders = new Map<AudioCapability, Promise<LadderResult>>();
+ const ladderFor = (capability: AudioCapability) => {
+ let result = ladders.get(capability);
+ if (!result) {
+ const withContext = capability === "onsets" && framing.cut;
+ result = runLadder(capability, withContext ? signal : framing.windowed, providers, {
+ quality: request.quality,
+ provider: capability === "beats" ? beatProvider : request.provider?.[capability],
+ options: optionsFor(capability, request, analysis.analyzed),
+ });
+ ladders.set(capability, result);
+ }
+ return result;
+ };
for (const capability of request.capabilities) {
- // Onsets see the context, so a window's edge is not mistaken for a hit, then keep only the window's own.
const withContext = capability === "onsets" && framing.cut;
- const result = await runLadder(capability, withContext ? signal : framing.windowed, providers, {
- quality: request.quality,
- provider: request.provider?.[capability],
- options: optionsFor(capability, request, analysis.analyzed),
- });
+ const result =
+ capability === "bpm" ? tempoFromBeats(await ladderFor("beats")) : await ladderFor(capability);
const value = withContext ? within(result.value, framing.before, framing.end) : result.value;
const offset = (request.fileOffset ?? 0) - (withContext ? framing.before : 0);
- analysis.values[capability] = toFileTime(capability, value, offset);
+ analysis.values[capability] = shiftedTimes(capability, value, offset);
analysis.provenance[capability] = result.provenance;
analysis.ladder[capability] = result.steps;
- analysis.warnings.push(...result.warnings);
+ const said = new Set(analysis.warnings);
+ analysis.warnings.push(...result.warnings.filter((w) => !said.has(w)));
}
return analysis;
}
+
+/** Tempo is read off the beats of whichever provider placed them, so the two never disagree. */
+function tempoFromBeats(beats: LadderResult): LadderResult {
+ const times = (beats.value as Beats | null)?.times;
+ return { ...beats, value: times ? tempoOf(times) : null };
+}
diff --git a/packages/cli/src/audio/analysis/fileTime.ts b/packages/cli/src/audio/analysis/fileTime.ts
new file mode 100644
index 0000000000..5f683ed691
--- /dev/null
+++ b/packages/cli/src/audio/analysis/fileTime.ts
@@ -0,0 +1,57 @@
+import type {
+ AudioCapability,
+ Onsets,
+ SnapReport,
+ Span,
+ Tempo,
+} from "@hyperframes/core/audio-analysis";
+
+const round = (t: number) => Math.round(t * 1000) / 1000;
+
+type At = (t: number) => number;
+
+/** A snap report's kept times are times too. */
+function withSnap<V extends { snap?: SnapReport }>(value: V, at: At): V {
+ return value.snap
+ ? { ...value, snap: { ...value.snap, unsnapped: value.snap.unsnapped.map(at) } }
+ : value;
+}
+
+const SHIFTS: Partial<Record<AudioCapability, (value: never, at: At, offset: number) => unknown>> =
+ {
+ moments: (m: Record<string, number>, at) =>
+ Object.fromEntries(Object.entries(m).map(([k, t]) => [k, at(t)])),
+ onsets: (o: Onsets, at) => ({ ...o, events: o.events.map((e) => ({ ...e, t: at(e.t) })) }),
+ silence: (spans: Span[], at) => spans.map((s) => ({ start: at(s.start), end: at(s.end) })),
+ loudness: (l: { clipping: { count: number; firstAt: number | null } }, at) => ({
+ ...l,
+ clipping: {
+ ...l.clipping,
+ firstAt: l.clipping.firstAt === null ? null : at(l.clipping.firstAt),
+ },
+ }),
+ envelope: (e: object, _at, offset) => ({ startsAt: round(offset), ...e }),
+ beats: (b: { times: number[]; snap?: SnapReport }, at) =>
+ withSnap({ ...b, times: b.times.map(at) }, at),
+ downbeats: (b: { times: number[]; snap?: SnapReport }, at) =>
+ withSnap({ ...b, times: b.times.map(at) }, at),
+ bpm: (tempo: Tempo, at) => ({
+ ...tempo,
+ curve: tempo.curve.map((p) => ({ ...p, t: at(p.t) })),
+ }),
+ sections: (s: { spans: Span[]; snap?: SnapReport }, at) =>
+ withSnap(
+ {
+ ...s,
+ spans: s.spans.map((span) => ({ ...span, start: at(span.start), end: at(span.end) })),
+ },
+ at,
+ ),
+ };
+
+/** Move a capability's times from signal seconds to file seconds. */
+export function shiftedTimes(capability: AudioCapability, value: unknown, offset: number): unknown {
+ if (value === null || typeof value !== "object") return value;
+ const shift = SHIFTS[capability];
+ return shift ? shift(value as never, (t) => round(t + offset), offset) : value;
+}
diff --git a/packages/cli/src/audio/analysis/format.ts b/packages/cli/src/audio/analysis/format.ts
index ffc37a47ee..32dd0d9c0d 100644
--- a/packages/cli/src/audio/analysis/format.ts
+++ b/packages/cli/src/audio/analysis/format.ts
@@ -1,10 +1,15 @@
import type {
+ Beats,
+ Downbeats,
Envelope,
KindGuess,
Loudness,
Moments,
Onsets,
+ Sections,
+ SnapReport,
Span,
+ Tempo,
} from "@hyperframes/core/audio-analysis";
import type { AudioItem, AudioReport } from "./run.js";
import { filmTime } from "./targets.js";
@@ -60,6 +65,50 @@ function silenceLine(silence: Span[], at: At): string | null {
: null;
}
+function snapNote(snap: SnapReport | undefined): string {
+ if (!snap) return "";
+ const total = snap.snapped + snap.unsnapped.length;
+ const median = snap.medianShiftMs === null ? "" : `, median ${snap.medianShiftMs} ms`;
+ return `; ${snap.snapped}/${total} snapped to ${snap.reference}${median}`;
+}
+
+const listed = (times: number[], at: At) => {
+ const shown = times.slice(0, ONSETS_SHOWN).map(at).join(" ");
+ return times.length > ONSETS_SHOWN ? `${shown} +${times.length - ONSETS_SHOWN}` : shown;
+};
+
+function beatsLine(beats: Beats & { snap?: SnapReport }, at: At): string {
+ if (!beats.times.length) return ` beats none${beats.reason ? ` (${beats.reason})` : ""}`;
+ return ` beats ${beats.times.length}: ${listed(beats.times, at)}${snapNote(beats.snap)}`;
+}
+
+function downbeatsLine(downbeats: Downbeats & { snap?: SnapReport }, at: At): string | null {
+ if (!downbeats.times.length) return null;
+ const meter = downbeats.meter ? ` in ${downbeats.meter}` : "";
+ const low = downbeats.confidence === "low" ? ", a guess" : "";
+ return ` downbeats${meter}${low}: ${listed(downbeats.times, at)}${snapNote(downbeats.snap)}`;
+}
+
+function tempoLine(tempo: Tempo): string | null {
+ if (tempo.value === null) return null;
+ const bpms = tempo.curve.map((p) => p.bpm);
+ const spread =
+ bpms.length && Math.max(...bpms) - Math.min(...bpms) >= 2
+ ? ` (${Math.min(...bpms)}–${Math.max(...bpms)})`
+ : "";
+ return ` tempo ${tempo.value} BPM${spread}`;
+}
+
+function sectionsLine(sections: Sections, at: At): string {
+ const spans = sections.spans.map((s) => `${s.label} ${at(s.start)}–${at(s.end)}`).join(" ");
+ return ` sections ${spans}${snapNote(sections.snap)}`;
+}
+
+function describeLine(description: { text?: string; tags: { label: string; score: number }[] }) {
+ const tags = description.tags.map((t) => `${t.label} ${t.score.toFixed(2)}`).join(", ");
+ return ` sounds like ${[description.text, tags].filter(Boolean).join(" — ")}`;
+}
+
/** One line per measurement the item holds, in reading order. */
const LINES: [string, (value: never, at: At) => string | null][] = [
["moments", momentsLine],
@@ -67,6 +116,11 @@ const LINES: [string, (value: never, at: At) => string | null][] = [
["loudness", loudnessLine],
["silence", silenceLine],
["envelope", (envelope: Envelope) => ` ${sparkline(envelope)}`],
+ ["describe", describeLine],
+ ["bpm", tempoLine],
+ ["beats", beatsLine],
+ ["downbeats", downbeatsLine],
+ ["sections", sectionsLine],
];
function headline(item: AudioItem, where: string): string {
diff --git a/packages/cli/src/audio/analysis/providers/api.test.ts b/packages/cli/src/audio/analysis/providers/api.test.ts
new file mode 100644
index 0000000000..8488f0bb76
--- /dev/null
+++ b/packages/cli/src/audio/analysis/providers/api.test.ts
@@ -0,0 +1,244 @@
+import { createServer, type IncomingMessage, type Server } from "node:http";
+import type { AddressInfo } from "node:net";
+import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
+import {
+ rhythm,
+ signalProvider,
+ type AudioSignal,
+ type AudioCapability,
+} from "@hyperframes/core/audio-analysis";
+import { analyzeSignal } from "../analyze.js";
+import { apiProvider, type AudioApiRequest, type AudioApiResponse } from "./api.js";
+
+const RATE = 22_050;
+
+/** Clicks every half second from 0.25 s: a grid the signal tracker finds. */
+function clicks(seconds: number): AudioSignal {
+ const samples = new Float32Array(seconds * RATE);
+ let state = 7;
+ for (let t = 0.25; t < seconds; t += 0.5)
+ for (let i = 0; i < 0.05 * RATE; i++) {
+ state = (state * 1_103_515_245 + 12_345) % 2_147_483_648;
+ samples[Math.round(t * RATE) + i]! +=
+ 0.5 * ((state / 2_147_483_648) * 2 - 1) * Math.exp(-i / (0.01 * RATE));
+ }
+ return { sampleRate: RATE, channels: [samples] };
+}
+
+interface Received {
+ method: string;
+ authorization: string | undefined;
+ request: AudioApiRequest;
+ audio: { type: string; bytes: Uint8Array };
+}
+
+let server: Server;
+let received: Received[] = [];
+let reply: (request: AudioApiRequest) => { status: number; body: unknown } = () => ({
+ status: 500,
+ body: {},
+});
+
+async function readForm(req: IncomingMessage) {
+ const chunks: Buffer[] = [];
+ for await (const chunk of req) chunks.push(chunk as Buffer);
+ const form = await new Request("http://stand-in", {
+ method: "POST",
+ headers: { "content-type": req.headers["content-type"]! },
+ body: Buffer.concat(chunks),
+ }).formData();
+ const audio = form.get("audio") as Blob;
+ return {
+ request: JSON.parse(await (form.get("request") as Blob).text()) as AudioApiRequest,
+ audio: { type: audio.type, bytes: new Uint8Array(await audio.arrayBuffer()) },
+ };
+}
+
+beforeAll(async () => {
+ server = createServer(async (req, res) => {
+ const form = await readForm(req);
+ received.push({ method: req.method!, authorization: req.headers.authorization, ...form });
+ const { status, body } = reply(form.request);
+ res.writeHead(status, { "content-type": "application/json" }).end(JSON.stringify(body));
+ });
+ await new Promise<void>((done) => server.listen(0, "127.0.0.1", done));
+});
+
+afterAll(() => new Promise<void>((done) => server.close(() => done())));
+
+afterEach(() => {
+ received = [];
+ delete process.env.HYPERFRAMES_AUDIO_API_URL;
+ delete process.env.HYPERFRAMES_AUDIO_API_KEY;
+});
+
+function configure() {
+ process.env.HYPERFRAMES_AUDIO_API_URL = `http://127.0.0.1:${(server.address() as AddressInfo).port}/v1/audio`;
+ process.env.HYPERFRAMES_AUDIO_API_KEY = "test-key";
+}
+
+describe("apiProvider", () => {
+ it("is unavailable until both the URL and the key are set", () => {
+ expect(apiProvider.available!()).toEqual({
+ ok: false,
+ reason: "HYPERFRAMES_AUDIO_API_URL is not set",
+ });
+ process.env.HYPERFRAMES_AUDIO_API_URL = "http://127.0.0.1:1/";
+ expect(apiProvider.available!()).toEqual({
+ ok: false,
+ reason: "HYPERFRAMES_AUDIO_API_KEY is not set",
+ });
+ process.env.HYPERFRAMES_AUDIO_API_KEY = "k";
+ expect(apiProvider.available!()).toEqual({ ok: true });
+ });
+
+ it("posts the audio and the capabilities, and snaps each answered beat onto the signal's own", async () => {
+ configure();
+ const signal = clicks(12);
+ const measured = rhythm(signal).beats.times;
+ // The service is 30 ms late on every beat, and invents one between two real beats.
+ const answered = [...measured.map((t) => t + 0.03), 3.5].sort((a, b) => a - b);
+ reply = () => ({
+ status: 200,
+ body: {
+ version: 1,
+ source: "stand-in",
+ beats: { times: answered },
+ downbeats: { times: measured.filter((_, k) => k % 4 === 0).map((t) => t - 0.02), meter: 4 },
+ } satisfies AudioApiResponse,
+ });
+ const analysis = await analyzeSignal(signal, { capabilities: ["beats", "downbeats", "bpm"] }, [
+ apiProvider,
+ signalProvider,
+ ]);
+
+ expect(received).toHaveLength(1);
+ const [call] = received;
+ expect(call).toMatchObject({ method: "POST", authorization: "Bearer test-key" });
+ expect(call!.request).toEqual({
+ version: 1,
+ capabilities: ["beats", "downbeats"],
+ audio: { sampleRate: RATE, channels: 1, duration: 12 },
+ });
+ expect(call!.audio.type).toBe("audio/wav");
+ expect(Buffer.from(call!.audio.bytes.subarray(0, 4)).toString()).toBe("RIFF");
+ expect(call!.audio.bytes.length).toBe(44 + 12 * RATE * 2);
+
+ expect(analysis.provenance).toEqual({ beats: "api", downbeats: "api", bpm: "api" });
+ const beats = analysis.values.beats as { times: number[]; snap: object; source: string };
+ expect(beats.times).toEqual([...measured, 3.5].sort((a, b) => a - b));
+ expect(beats.source).toBe("stand-in");
+ expect(beats.snap).toEqual({
+ reference: "signal beats",
+ toleranceMs: 125,
+ snapped: measured.length,
+ unsnapped: [3.5],
+ medianShiftMs: 30,
+ });
+ expect((analysis.values.downbeats as { times: number[] }).times).toEqual(
+ measured.filter((_, k) => k % 4 === 0),
+ );
+ expect((analysis.values.bpm as { value: number }).value).toBeCloseTo(120, 0);
+ });
+
+ it.each<{ capabilities: AudioCapability[] }>([
+ { capabilities: ["bpm"] },
+ { capabilities: ["beats", "bpm"] },
+ { capabilities: ["bpm", "beats"] },
+ ])("keeps a BPM signal pin local for $capabilities", async ({ capabilities }) => {
+ configure();
+ reply = () => ({ status: 200, body: { version: 1, beats: { times: [0.5, 1, 1.5] } } });
+ const analysis = await analyzeSignal(clicks(8), {
+ capabilities,
+ provider: { bpm: "signal" },
+ });
+ expect(received).toHaveLength(0);
+ expect(analysis.provenance.bpm).toBe("signal");
+ expect(analysis.values.bpm).toMatchObject({ value: expect.any(Number) });
+ if (capabilities.includes("beats")) expect(analysis.provenance.beats).toBe("signal");
+ });
+
+ it.each([
+ { beats: "api", bpm: "signal" },
+ { beats: "signal", bpm: "api" },
+ ])(
+ "rejects conflicting beats/BPM pins before any capability uploads audio: %j",
+ async (provider) => {
+ configure();
+ reply = () => ({
+ status: 200,
+ body: { version: 1, describe: { tags: [] }, beats: { times: [0.5, 1] } },
+ });
+ await expect(
+ analyzeSignal(clicks(8), {
+ capabilities: ["describe", "beats", "bpm"],
+ provider,
+ }),
+ ).rejects.toThrow("beats and bpm must use the same provider");
+ expect(received).toHaveLength(0);
+ },
+ );
+
+ it("asks for sections and describe on their own, and moves inner section boundaries onto beats", async () => {
+ configure();
+ const signal = clicks(12);
+ const beat = rhythm(signal).beats.times.find((t) => Math.abs(t - 6.25) < 0.05)!;
+ reply = () => ({
+ status: 200,
+ body: {
+ version: 1,
+ sections: {
+ spans: [
+ { start: 0, end: 6.3, label: "intro" },
+ { start: 6.3, end: 12 },
+ ],
+ },
+ describe: { text: "clicks", tags: [{ label: "click", score: 0.9 }] },
+ },
+ });
+ const analysis = await analyzeSignal(signal, { capabilities: ["sections", "describe"] }, [
+ apiProvider,
+ signalProvider,
+ ]);
+ expect(received.map((r) => r.request.capabilities)).toEqual([["sections"], ["describe"]]);
+ expect(analysis.values.sections).toMatchObject({
+ spans: [
+ { start: 0, end: beat, label: "intro" },
+ { start: beat, end: 12 },
+ ],
+ snap: { snapped: 2, unsnapped: [] },
+ });
+ expect(analysis.values.describe).toEqual({
+ text: "clicks",
+ tags: [{ label: "click", score: 0.9 }],
+ });
+ });
+
+ it("falls through to signal processing with a warning when the service fails or answers in another shape", async () => {
+ configure();
+ reply = () => ({ status: 503, body: { error: { message: "busy" } } });
+ const failed = await analyzeSignal(clicks(8), { capabilities: ["beats"] }, [
+ apiProvider,
+ signalProvider,
+ ]);
+ expect(failed.provenance.beats).toBe("signal");
+ expect(failed.warnings).toEqual([
+ "beats: used signal; api failed (the service answered 503: busy)",
+ ]);
+
+ reply = () => ({ status: 200, body: { version: 1, beats: { times: ["soon"] } } });
+ const garbled = await analyzeSignal(clicks(8), { capabilities: ["beats"] }, [
+ apiProvider,
+ signalProvider,
+ ]);
+ expect(garbled.provenance.beats).toBe("signal");
+ expect(garbled.warnings[0]).toContain("api failed (the service answered beats in a shape");
+ });
+
+ it("never sends the key anywhere but the authorization header", async () => {
+ configure();
+ reply = () => ({ status: 401, body: { error: { message: "bad key" } } });
+ const analysis = await analyzeSignal(clicks(8), { capabilities: ["describe"] }, [apiProvider]);
+ expect(JSON.stringify(analysis)).not.toContain("test-key");
+ });
+});
diff --git a/packages/cli/src/audio/analysis/providers/api.ts b/packages/cli/src/audio/analysis/providers/api.ts
new file mode 100644
index 0000000000..5659f19206
--- /dev/null
+++ b/packages/cli/src/audio/analysis/providers/api.ts
@@ -0,0 +1,197 @@
+import {
+ medianInterval,
+ rhythm,
+ snapper,
+ type AudioAnalysisProvider,
+ type AudioCapability,
+ type AudioSignal,
+ type Snapper,
+} from "@hyperframes/core/audio-analysis";
+import { AVAILABLE, encodeWav, perSignal, rounded, unavailable } from "./shared.js";
+
+/**
+ * The audio analysis service contract.
+ *
+ * Request: `POST <HYPERFRAMES_AUDIO_API_URL>`, exactly as configured, with
+ * `Authorization: Bearer <HYPERFRAMES_AUDIO_API_KEY>` and a `multipart/form-data` body of two parts:
+ * - `request`: `application/json`, an {@link AudioApiRequest};
+ * - `audio`: `audio/wav`, 16-bit PCM of the analyzed span, channels as decoded.
+ *
+ * Response: `200` with `application/json`, an {@link AudioApiResponse}. Every time is in seconds from the start of the
+ * uploaded audio. A capability the service cannot answer is left out of the response. Any other status is a failure;
+ * its body may be `{ "error": { "message": string } }`.
+ */
+export interface AudioApiRequest {
+ version: 1;
+ capabilities: AudioApiCapability[];
+ audio: { sampleRate: number; channels: number; duration: number };
+}
+
+export type AudioApiCapability = "beats" | "downbeats" | "sections" | "describe";
+
+export interface AudioApiResponse {
+ version: 1;
+ /** What answered, as the service wants it reported. */
+ source?: string;
+ beats?: { times: number[] };
+ downbeats?: { times: number[]; meter?: number | null };
+ sections?: { spans: { start: number; end: number; label?: string }[] };
+ describe?: { text?: string; tags: { label: string; score: number }[] };
+}
+
+const URL_ENV = "HYPERFRAMES_AUDIO_API_URL";
+const KEY_ENV = "HYPERFRAMES_AUDIO_API_KEY";
+const TIMEOUT_ENV = "HYPERFRAMES_AUDIO_API_TIMEOUT_MS";
+const DEFAULT_TIMEOUT_MS = 120_000;
+
+/** Beats and downbeats come from one request; the others each from their own. */
+const askedTogether = (capability: AudioApiCapability): AudioApiCapability[] =>
+ capability === "beats" || capability === "downbeats" ? ["beats", "downbeats"] : [capability];
+
+function config() {
+ const url = process.env[URL_ENV]?.trim();
+ const key = process.env[KEY_ENV]?.trim();
+ const timeout = Number(process.env[TIMEOUT_ENV]);
+ return { url, key, timeoutMs: timeout > 0 ? timeout : DEFAULT_TIMEOUT_MS };
+}
+
+const isNumberList = (value: unknown): value is number[] =>
+ Array.isArray(value) && value.every((t) => typeof t === "number" && Number.isFinite(t));
+
+function checked(capability: AudioApiCapability, body: AudioApiResponse): void {
+ const value = body[capability] as Record<string, unknown> | undefined;
+ if (!value || typeof value !== "object")
+ throw new Error(`the service did not answer ${capability}`);
+ const valid = {
+ beats: () => isNumberList(value.times),
+ downbeats: () => isNumberList(value.times),
+ sections: () =>
+ Array.isArray(value.spans) &&
+ value.spans.every((s: { start?: unknown; end?: unknown }) => isNumberList([s.start, s.end])),
+ describe: () =>
+ Array.isArray(value.tags) &&
+ value.tags.every(
+ (t: { label?: unknown; score?: unknown }) =>
+ typeof t.label === "string" && typeof t.score === "number",
+ ),
+ }[capability];
+ if (!valid())
+ throw new Error(`the service answered ${capability} in a shape this CLI cannot read`);
+}
+
+async function failure(response: Response): Promise<Error> {
+ const text = await response.text().catch(() => "");
+ let detail = text.slice(0, 200);
+ try {
+ detail = (JSON.parse(text) as { error?: { message?: string } }).error?.message ?? detail;
+ } catch {
+ // not JSON: keep the text
+ }
+ return new Error(`the service answered ${response.status}${detail ? `: ${detail}` : ""}`);
+}
+
+async function ask(signal: AudioSignal, capabilities: AudioApiCapability[]) {
+ const { url, key, timeoutMs } = config();
+ if (!url || !key) throw new Error(`${URL_ENV} and ${KEY_ENV} must both be set`);
+ const request: AudioApiRequest = {
+ version: 1,
+ capabilities,
+ audio: {
+ sampleRate: signal.sampleRate,
+ channels: signal.channels.length,
+ duration: (signal.channels[0]?.length ?? 0) / signal.sampleRate,
+ },
+ };
+ const form = new FormData();
+ form.append("request", new Blob([JSON.stringify(request)], { type: "application/json" }));
+ form.append(
+ "audio",
+ new Blob([new Uint8Array(encodeWav(signal))], { type: "audio/wav" }),
+ "audio.wav",
+ );
+ const response = await fetch(url, {
+ method: "POST",
+ headers: { authorization: `Bearer ${key}` },
+ body: form,
+ signal: AbortSignal.timeout(timeoutMs),
+ });
+ if (!response.ok) throw await failure(response);
+ return (await response.json()) as AudioApiResponse;
+}
+
+const requests = new Map<string, (signal: AudioSignal) => Promise<AudioApiResponse>>();
+
+function answerFor(signal: AudioSignal, capability: AudioApiCapability) {
+ const together = askedTogether(capability);
+ const key = together.join(",");
+ let request = requests.get(key);
+ if (!request) requests.set(key, (request = perSignal((s) => ask(s, together))));
+ return request(signal);
+}
+
+/** The signal's own beats, which an answer's times move onto when one is within a quarter of the answer's beat. */
+function beatSnapper(signal: AudioSignal, beatSeconds: number | null): Snapper {
+ const measured = rhythm(signal).beats.times;
+ const period = beatSeconds ?? medianInterval(measured) ?? 0;
+ return snapper(measured, period / 4, "signal beats");
+}
+
+function snappedBeats(signal: AudioSignal, body: AudioApiResponse) {
+ const times = body.beats!.times;
+ const onBeat = beatSnapper(signal, medianInterval(times));
+ return { times: rounded(times.map(onBeat.snap)), snap: onBeat.report() };
+}
+
+function snappedDownbeats(signal: AudioSignal, body: AudioApiResponse) {
+ const { times, meter = null } = body.downbeats!;
+ const beat = body.beats ? medianInterval(body.beats.times) : null;
+ const bar = medianInterval(times);
+ const onBeat = beatSnapper(signal, beat ?? (bar && meter ? bar / meter : null));
+ return { times: rounded(times.map(onBeat.snap)), meter, snap: onBeat.report() };
+}
+
+function snappedSections(signal: AudioSignal, body: AudioApiResponse) {
+ const onBeat = beatSnapper(signal, null);
+ const end = (signal.channels[0]?.length ?? 0) / signal.sampleRate;
+ const move = (t: number) => (t <= 0 || t >= end ? t : onBeat.snap(t));
+ const spans = body.sections!.spans.map((s) => ({
+ ...s,
+ start: rounded([move(s.start)])[0]!,
+ end: rounded([move(s.end)])[0]!,
+ }));
+ return { spans, snap: onBeat.report() };
+}
+
+/** The answer for one capability, its times moved onto what the signal measured. */
+function measured(capability: AudioApiCapability, signal: AudioSignal, body: AudioApiResponse) {
+ const value =
+ capability === "beats"
+ ? snappedBeats(signal, body)
+ : capability === "downbeats"
+ ? snappedDownbeats(signal, body)
+ : capability === "sections"
+ ? snappedSections(signal, body)
+ : body.describe!;
+ return body.source ? { ...value, source: body.source } : value;
+}
+
+const API_CAPABILITIES: readonly AudioCapability[] = ["beats", "downbeats", "sections", "describe"];
+
+/** HeyGen's audio analysis service, or any service that speaks {@link AudioApiRequest}, configured by environment. */
+export const apiProvider: AudioAnalysisProvider<AudioSignal> = {
+ id: "api",
+ local: false,
+ capabilities: API_CAPABILITIES,
+ available() {
+ const { url, key } = config();
+ if (!url) return unavailable(`${URL_ENV} is not set`);
+ if (!key) return unavailable(`${KEY_ENV} is not set`);
+ return AVAILABLE;
+ },
+ async analyze(capability, signal) {
+ const asked = capability as AudioApiCapability;
+ const body = await answerFor(signal, asked);
+ checked(asked, body);
+ return measured(asked, signal, body);
+ },
+};
diff --git a/packages/cli/src/audio/analysis/providers/apple.ts b/packages/cli/src/audio/analysis/providers/apple.ts
new file mode 100644
index 0000000000..296697a72b
--- /dev/null
+++ b/packages/cli/src/audio/analysis/providers/apple.ts
@@ -0,0 +1,171 @@
+import { createHash } from "node:crypto";
+import { existsSync } from "node:fs";
+import { mkdir, rename, writeFile } from "node:fs/promises";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import type { AudioAnalysisProvider, AudioSignal } from "@hyperframes/core/audio-analysis";
+import { AVAILABLE, runFile, unavailable, withWavFile } from "./shared.js";
+
+const SOUND_ANALYSIS = "/System/Library/Frameworks/SoundAnalysis.framework";
+const MUSIC_UNDERSTANDING_MACOS = 27;
+
+/** Apple's built-in sound classifier over the whole file: each label's mean confidence across its windows. */
+const SOUND_TAGGER = `
+import Foundation
+import SoundAnalysis
+
+final class Collector: NSObject, SNResultsObserving {
+ var sums: [String: Double] = [:]
+ var windows = 0
+ var failure: Error?
+ func request(_ request: SNRequest, didProduce result: SNResult) {
+ guard let result = result as? SNClassificationResult else { return }
+ windows += 1
+ for c in result.classifications { sums[c.identifier, default: 0] += c.confidence }
+ }
+ func request(_ request: SNRequest, didFailWithError error: Error) { failure = error }
+}
+
+do {
+ let analyzer = try SNAudioFileAnalyzer(url: URL(fileURLWithPath: CommandLine.arguments[1]))
+ let collector = Collector()
+ try analyzer.add(try SNClassifySoundRequest(classifierIdentifier: .version1), withObserver: collector)
+ analyzer.analyze()
+ if let failure = collector.failure { throw failure }
+ let count = Double(max(1, collector.windows))
+ var tags: [[String: Any]] = []
+ for (label, sum) in collector.sums.sorted(by: { $0.value > $1.value }).prefix(10) {
+ let score: Double = sum / count
+ if score >= 0.01 { tags.append(["label": label, "score": (score * 1000).rounded() / 1000]) }
+ }
+ let json = try JSONSerialization.data(withJSONObject: ["tags": tags, "windows": collector.windows])
+ FileHandle.standardOutput.write(json)
+} catch {
+ FileHandle.standardError.write(Data("\\(error)".utf8))
+ exit(1)
+}
+`;
+
+/** The classifier listens in windows of about three seconds and says nothing about a shorter file. */
+const CLASSIFIER_WINDOW_SECONDS = 3;
+
+function atLeastOneWindow(signal: AudioSignal): AudioSignal {
+ const length = Math.ceil(CLASSIFIER_WINDOW_SECONDS * signal.sampleRate);
+ if ((signal.channels[0]?.length ?? 0) >= length) return signal;
+ return {
+ ...signal,
+ channels: signal.channels.map((channel) => {
+ const padded = new Float32Array(length);
+ padded.set(channel);
+ return padded;
+ }),
+ };
+}
+
+const helperDir = () => join(homedir(), ".cache", "hyperframes", "audio-helpers");
+
+function helperPath(name: string, source: string): string {
+ const hash = createHash("sha256").update(source).digest("hex").slice(0, 12);
+ return join(helperDir(), `${name}-${hash}`);
+}
+
+async function hasSwiftCompiler(): Promise<boolean> {
+ try {
+ await runFile("xcrun", ["--find", "swiftc"], { timeout: 30_000 });
+ return true;
+ } catch {
+ return false;
+ }
+}
+
+const firstLine = (error: unknown) =>
+ ((error as { stderr?: string }).stderr || String(error)).trim().split("\n")[0];
+
+const builds = new Map<string, Promise<string>>();
+
+/** The helper compiled from `source`, built once per source into the cache; Swift is never fetched. */
+function compiled(name: string, source: string): Promise<string> {
+ const binary = helperPath(name, source);
+ if (existsSync(binary)) return Promise.resolve(binary);
+ let build = builds.get(binary);
+ if (!build) {
+ build = (async () => {
+ await mkdir(helperDir(), { recursive: true });
+ const file = `${binary}.swift`;
+ await writeFile(file, source);
+ // Through xcrun, so the compiler finds the SDK it belongs to.
+ const args = [
+ "--sdk",
+ "macosx",
+ "swiftc",
+ "-O",
+ "-swift-version",
+ "5",
+ file,
+ "-o",
+ `${binary}.tmp`,
+ ];
+ await runFile("xcrun", args, { timeout: 5 * 60_000 }).catch((error: unknown) => {
+ throw new Error(`could not build the ${name} helper: ${firstLine(error)}`);
+ });
+ await rename(`${binary}.tmp`, binary);
+ return binary;
+ })();
+ builds.set(binary, build);
+ }
+ return build;
+}
+
+/** The macOS major version, or null off macOS. */
+async function macosMajor(): Promise<number | null> {
+ if (process.platform !== "darwin") return null;
+ const { stdout } = await runFile("sw_vers", ["-productVersion"], { timeout: 10_000 });
+ return Number.parseInt(stdout, 10) || null;
+}
+
+/** Labels for what the audio holds, from Apple's on-device sound classifier (SoundAnalysis, macOS 12+). */
+export const appleSoundProvider: AudioAnalysisProvider<AudioSignal> = {
+ id: "apple-sound",
+ local: true,
+ capabilities: ["describe"],
+ async available() {
+ if (process.platform !== "darwin") return unavailable("macOS only");
+ if (!existsSync(SOUND_ANALYSIS))
+ return unavailable("SoundAnalysis.framework is not on this Mac");
+ if (existsSync(helperPath("apple-sound", SOUND_TAGGER))) return AVAILABLE;
+ if (!(await hasSwiftCompiler())) return unavailable("no Swift compiler to build the helper");
+ return AVAILABLE;
+ },
+ async analyze(_capability, signal) {
+ const helper = await compiled("apple-sound", SOUND_TAGGER);
+ const { stdout } = await withWavFile(atLeastOneWindow(signal), (wav) =>
+ runFile(helper, [wav], { timeout: 5 * 60_000, maxBuffer: 8 * 1024 * 1024 }),
+ );
+ const { tags } = JSON.parse(stdout) as { tags: { label: string; score: number }[] };
+ return {
+ tags: tags.map((t) => ({ label: t.label, score: Math.round(t.score * 1000) / 1000 })),
+ };
+ },
+};
+
+/**
+ * Apple's on-device Music Understanding (WWDC26, macOS 27). No SDK this CLI has been built against offers it, so the
+ * helper is not written yet: the provider reports why it cannot run instead of answering.
+ */
+export const appleMuProvider: AudioAnalysisProvider<AudioSignal> = {
+ id: "apple-mu",
+ local: true,
+ capabilities: ["beats", "downbeats", "sections"],
+ async available() {
+ const major = await macosMajor();
+ if (major === null) return unavailable("macOS only");
+ if (major < MUSIC_UNDERSTANDING_MACOS)
+ return unavailable(
+ `Music Understanding needs macOS ${MUSIC_UNDERSTANDING_MACOS}; this Mac runs ${major}`,
+ );
+ return unavailable("no Music Understanding helper yet: it needs the macOS 27 SDK to build");
+ },
+ analyze() {
+ return Promise.reject(new Error("Music Understanding is not available"));
+ },
+};
diff --git a/packages/cli/src/audio/analysis/providers/beatThis.ts b/packages/cli/src/audio/analysis/providers/beatThis.ts
new file mode 100644
index 0000000000..dff283e2ad
--- /dev/null
+++ b/packages/cli/src/audio/analysis/providers/beatThis.ts
@@ -0,0 +1,85 @@
+import { existsSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+import type { AudioAnalysisProvider, AudioSignal } from "@hyperframes/core/audio-analysis";
+import {
+ AVAILABLE,
+ downbeatsWithMeter,
+ perSignal,
+ rounded,
+ runFile,
+ unavailable,
+ withWavFile,
+} from "./shared.js";
+
+const PYTHON_ENV = "HYPERFRAMES_BEAT_THIS_PYTHON";
+const CHECKPOINT_ENV = "HYPERFRAMES_BEAT_THIS_CHECKPOINT";
+
+/** Where Beat This! keeps the small0 weights after its own first download, unless a path is configured. */
+function checkpointPath(): string {
+ const configured = process.env[CHECKPOINT_ENV]?.trim();
+ if (configured) return configured;
+ const torchHome = process.env.TORCH_HOME ?? join(homedir(), ".cache", "torch");
+ return join(torchHome, "hub", "checkpoints", "beat_this-small0.ckpt");
+}
+
+const FIND_PACKAGE =
+ "import importlib.util,sys; sys.exit(importlib.util.find_spec('beat_this') is None)";
+
+// The checkpoint is a local file path, so Beat This! loads it from disk and never downloads.
+const TRACK = `
+import json, sys
+from beat_this.inference import File2Beats
+beats, downbeats = File2Beats(checkpoint_path=sys.argv[2], device="cpu", dbn=False)(sys.argv[1])
+print(json.dumps({"beats": [float(t) for t in beats], "downbeats": [float(t) for t in downbeats]}))
+`;
+
+const installed = new Map<string, Promise<boolean>>();
+
+function hasPackage(python: string): Promise<boolean> {
+ let found = installed.get(python);
+ if (!found) {
+ found = runFile(python, ["-c", FIND_PACKAGE], { timeout: 30_000, windowsHide: true }).then(
+ () => true,
+ () => false,
+ );
+ installed.set(python, found);
+ }
+ return found;
+}
+
+const track = perSignal((signal: AudioSignal) =>
+ withWavFile(signal, async (wav) => {
+ const python = process.env[PYTHON_ENV]!.trim();
+ const { stdout } = await runFile(python, ["-c", TRACK, wav, checkpointPath()], {
+ maxBuffer: 64 * 1024 * 1024,
+ timeout: 10 * 60_000,
+ windowsHide: true,
+ });
+ const answer = JSON.parse(stdout.trim().split("\n").pop()!) as {
+ beats: number[];
+ downbeats: number[];
+ };
+ return { beats: rounded(answer.beats), downbeats: rounded(answer.downbeats) };
+ }),
+);
+
+/** CPJKU's Beat This! (MIT), opt-in: a Python with `beat-this` installed and its small0 weights on disk. */
+export const beatThisProvider: AudioAnalysisProvider<AudioSignal> = {
+ id: "beat-this",
+ local: true,
+ capabilities: ["beats", "downbeats"],
+ async available() {
+ const python = process.env[PYTHON_ENV]?.trim();
+ if (!python) return unavailable(`${PYTHON_ENV} is not set`);
+ if (!existsSync(python)) return unavailable(`${PYTHON_ENV} names no file`);
+ if (!existsSync(checkpointPath()))
+ return unavailable(`no Beat This! weights at ${checkpointPath()}`);
+ if (!(await hasPackage(python))) return unavailable(`beat_this is not installed for ${python}`);
+ return AVAILABLE;
+ },
+ async analyze(capability, signal) {
+ const { beats, downbeats } = await track(signal);
+ return capability === "beats" ? { times: beats } : downbeatsWithMeter(beats, downbeats);
+ },
+};
diff --git a/packages/cli/src/audio/analysis/providers/localProviders.test.ts b/packages/cli/src/audio/analysis/providers/localProviders.test.ts
new file mode 100644
index 0000000000..11f8bb84e0
--- /dev/null
+++ b/packages/cli/src/audio/analysis/providers/localProviders.test.ts
@@ -0,0 +1,137 @@
+import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, afterEach, describe, expect, it } from "vitest";
+import { signalProvider, type AudioSignal } from "@hyperframes/core/audio-analysis";
+import { analyzeSignal } from "../analyze.js";
+import { appleMuProvider, appleSoundProvider } from "./apple.js";
+import { beatThisProvider } from "./beatThis.js";
+
+const dir = mkdtempSync(join(tmpdir(), "hf-providers-"));
+afterAll(() => rmSync(dir, { recursive: true, force: true }));
+
+const tone: AudioSignal = {
+ sampleRate: 8000,
+ channels: [Float32Array.from({ length: 8000 * 2 }, (_, i) => 0.3 * Math.sin(i / 3))],
+};
+
+/** Stands in for a Python with beat_this: answers the package probe, and tracks only a real WAV with real weights. */
+function fakePython(hasPackage: boolean): string {
+ const path = join(dir, hasPackage ? "python" : "python-bare");
+ writeFileSync(
+ path,
+ `#!/bin/sh
+case "$2" in *find_spec*) exit ${hasPackage ? 0 : 1};; esac
+[ -f "$3" ] && [ -f "$4" ] || exit 2
+head -c 4 "$3" | grep -q RIFF || exit 3
+echo "loading weights"
+echo '{"beats": [0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5], "downbeats": [0.5, 2.5, 4.5]}'
+`,
+ );
+ chmodSync(path, 0o755);
+ return path;
+}
+
+const weights = join(dir, "beat_this-small0.ckpt");
+writeFileSync(weights, "weights");
+
+afterEach(() => {
+ delete process.env.HYPERFRAMES_BEAT_THIS_PYTHON;
+ delete process.env.HYPERFRAMES_BEAT_THIS_CHECKPOINT;
+});
+
+describe.skipIf(process.platform === "win32")("beatThisProvider", () => {
+ it("is unavailable until a Python with beat_this and the weights are both on this machine", async () => {
+ expect(await beatThisProvider.available!()).toEqual({
+ ok: false,
+ reason: "HYPERFRAMES_BEAT_THIS_PYTHON is not set",
+ });
+ process.env.HYPERFRAMES_BEAT_THIS_PYTHON = join(dir, "missing");
+ expect(await beatThisProvider.available!()).toMatchObject({
+ reason: "HYPERFRAMES_BEAT_THIS_PYTHON names no file",
+ });
+ process.env.HYPERFRAMES_BEAT_THIS_PYTHON = fakePython(true);
+ process.env.HYPERFRAMES_BEAT_THIS_CHECKPOINT = join(dir, "absent.ckpt");
+ expect(await beatThisProvider.available!()).toMatchObject({
+ reason: expect.stringContaining("no Beat This! weights"),
+ });
+ process.env.HYPERFRAMES_BEAT_THIS_CHECKPOINT = weights;
+ expect(await beatThisProvider.available!()).toEqual({ ok: true });
+ process.env.HYPERFRAMES_BEAT_THIS_PYTHON = fakePython(false);
+ expect(await beatThisProvider.available!()).toMatchObject({
+ reason: expect.stringContaining("beat_this is not installed"),
+ });
+ });
+
+ it("answers beats and downbeats from one run, with the meter from beats per bar", async () => {
+ process.env.HYPERFRAMES_BEAT_THIS_PYTHON = fakePython(true);
+ process.env.HYPERFRAMES_BEAT_THIS_CHECKPOINT = weights;
+ const analysis = await analyzeSignal(tone, { capabilities: ["beats", "downbeats", "bpm"] }, [
+ beatThisProvider,
+ signalProvider,
+ ]);
+ expect(analysis.provenance).toEqual({
+ beats: "beat-this",
+ downbeats: "beat-this",
+ bpm: "beat-this",
+ });
+ expect(analysis.values.downbeats).toEqual({ times: [0.5, 2.5, 4.5], meter: 4 });
+ expect(analysis.values.bpm).toMatchObject({ value: 120 });
+ });
+
+ it("falls through to signal processing when its run fails", async () => {
+ const crashing = join(dir, "python-crashing");
+ writeFileSync(
+ crashing,
+ `#!/bin/sh\ncase "$2" in *find_spec*) exit 0;; esac\necho "out of memory" >&2\nexit 1\n`,
+ );
+ chmodSync(crashing, 0o755);
+ process.env.HYPERFRAMES_BEAT_THIS_PYTHON = crashing;
+ process.env.HYPERFRAMES_BEAT_THIS_CHECKPOINT = weights;
+ const silent = { ...tone, channels: [new Float32Array(8000 * 7)] };
+ const analysis = await analyzeSignal(silent, { capabilities: ["beats"] }, [
+ beatThisProvider,
+ signalProvider,
+ ]);
+ expect(analysis.provenance.beats).toBe("signal");
+ expect(analysis.warnings[0]).toMatch(
+ /^beats: used signal; beat-this failed \(.*out of memory/s,
+ );
+ });
+});
+
+describe("appleMuProvider", () => {
+ it("says why it cannot run instead of answering", async () => {
+ const availability = await appleMuProvider.available!();
+ expect(availability.ok).toBe(false);
+ if (!availability.ok)
+ expect(availability.reason).toMatch(
+ process.platform === "darwin" ? /Music Understanding/ : /^macOS only$/,
+ );
+ });
+});
+
+describe("appleSoundProvider", () => {
+ it("is unavailable off macOS", async () => {
+ const platform = Object.getOwnPropertyDescriptor(process, "platform")!;
+ Object.defineProperty(process, "platform", { value: "linux" });
+ try {
+ expect(await appleSoundProvider.available!()).toEqual({ ok: false, reason: "macOS only" });
+ } finally {
+ Object.defineProperty(process, "platform", platform);
+ }
+ });
+
+ it.skipIf(process.platform !== "darwin")(
+ "labels what a short sound holds, using the built-in classifier",
+ async () => {
+ if (!(await appleSoundProvider.available!()).ok) return;
+ const value = (await appleSoundProvider.analyze("describe", tone, {})) as {
+ tags: { label: string; score: number }[];
+ };
+ expect(value.tags.length).toBeGreaterThan(0);
+ for (const tag of value.tags) expect(tag.score).toBeGreaterThan(0);
+ },
+ 300_000,
+ );
+});
diff --git a/packages/cli/src/audio/analysis/providers/shared.ts b/packages/cli/src/audio/analysis/providers/shared.ts
new file mode 100644
index 0000000000..c6e88153d9
--- /dev/null
+++ b/packages/cli/src/audio/analysis/providers/shared.ts
@@ -0,0 +1,76 @@
+import { execFile } from "node:child_process";
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { promisify } from "node:util";
+import type { AudioSignal, Availability, Downbeats } from "@hyperframes/core/audio-analysis";
+import { medianInterval } from "@hyperframes/core/audio-analysis";
+
+export const runFile = promisify(execFile);
+
+export const unavailable = (reason: string): Availability => ({ ok: false, reason });
+export const AVAILABLE: Availability = { ok: true };
+
+/** 16-bit PCM WAV of the signal, channels interleaved. */
+export function encodeWav(signal: AudioSignal): Buffer {
+ const channels = signal.channels.length;
+ const frames = signal.channels[0]?.length ?? 0;
+ const data = frames * channels * 2;
+ const wav = Buffer.alloc(44 + data);
+ wav.write("RIFF", 0);
+ wav.writeUInt32LE(36 + data, 4);
+ wav.write("WAVEfmt ", 8);
+ wav.writeUInt32LE(16, 16);
+ wav.writeUInt16LE(1, 20);
+ wav.writeUInt16LE(channels, 22);
+ wav.writeUInt32LE(signal.sampleRate, 24);
+ wav.writeUInt32LE(signal.sampleRate * channels * 2, 28);
+ wav.writeUInt16LE(channels * 2, 32);
+ wav.writeUInt16LE(16, 34);
+ wav.write("data", 36);
+ wav.writeUInt32LE(data, 40);
+ let offset = 44;
+ for (let i = 0; i < frames; i++)
+ for (const channel of signal.channels) {
+ const sample = Math.max(-1, Math.min(1, channel[i]!));
+ wav.writeInt16LE(Math.round(sample * 32767), offset);
+ offset += 2;
+ }
+ return wav;
+}
+
+/** Runs `use` with the signal written to a WAV file that is removed afterwards. */
+export async function withWavFile<T>(
+ signal: AudioSignal,
+ use: (path: string) => Promise<T>,
+): Promise<T> {
+ const dir = await mkdtemp(join(tmpdir(), "hf-audio-"));
+ try {
+ const path = join(dir, "audio.wav");
+ await writeFile(path, encodeWav(signal));
+ return await use(path);
+ } finally {
+ await rm(dir, { recursive: true, force: true });
+ }
+}
+
+/** One answer per signal, shared by every capability a provider computes together. */
+export function perSignal<T>(compute: (signal: AudioSignal) => Promise<T>) {
+ const answers = new WeakMap<AudioSignal, Promise<T>>();
+ return (signal: AudioSignal): Promise<T> => {
+ let answer = answers.get(signal);
+ if (!answer) answers.set(signal, (answer = compute(signal)));
+ return answer;
+ };
+}
+
+/** Downbeats from a model's beat and downbeat times; the meter is the median count of beats per bar. */
+export function downbeatsWithMeter(beats: number[], downbeats: number[]): Downbeats {
+ const bar = medianInterval(downbeats);
+ const beat = medianInterval(beats);
+ const meter = bar && beat ? Math.round(bar / beat) : null;
+ return { times: downbeats, meter: meter && meter > 0 ? meter : null };
+}
+
+const round = (t: number) => Math.round(t * 1000) / 1000;
+export const rounded = (times: readonly number[]) => times.map(round);
diff --git a/packages/cli/src/commands/audio.ts b/packages/cli/src/commands/audio.ts
index ac1a92517a..85cc7900a5 100644
--- a/packages/cli/src/commands/audio.ts
+++ b/packages/cli/src/commands/audio.ts
@@ -25,6 +25,10 @@ export const examples: Example[] = [
],
["Every sound playing at 12 s in the film", "hyperframes audio --at 12"],
["One clip of the film, as JSON", "hyperframes audio --element '#riser' --json"],
+ [
+ "The beat grid, tempo and sections of a music bed",
+ "hyperframes audio music.mp3 --only beats,downbeats,bpm,sections",
+ ],
[
"Only onsets, from 60–200 Hz, finer",
"hyperframes audio drums.wav --only onsets --band 60,200 --hop-ms 2",
@@ -34,8 +38,9 @@ export const examples: Example[] = [
const CAPABILITIES = new Set<string>([
...DEFAULT_CAPABILITIES,
"beats",
+ "downbeats",
+ "bpm",
"sections",
- "words",
"describe",
]);
const QUALITIES = new Set<AnalysisQuality>(["best", "local", "fast"]);
@@ -61,6 +66,8 @@ function capabilities(raw: string | undefined): AudioCapability[] {
.split(",")
.map((c) => c.trim())
.filter(Boolean);
+ if (asked.includes("words"))
+ throw new CliUsageError("words come from hyperframes transcribe, not hyperframes audio");
const unknown = asked.filter((c) => !CAPABILITIES.has(c));
if (unknown.length)
throw new CliUsageError(
@@ -177,7 +184,7 @@ export default defineCommand({
},
only: {
type: "string",
- description: `Capabilities to measure (default ${DEFAULT_CAPABILITIES.join(",")})`,
+ description: `Capabilities to measure (default ${DEFAULT_CAPABILITIES.join(",")}; also beats,downbeats,bpm,sections,describe)`,
},
quality: {
type: "string",
diff --git a/packages/core/src/audioAnalysis.test.ts b/packages/core/src/audioAnalysis.test.ts
index 3dfcdfe8b7..c366e91725 100644
--- a/packages/core/src/audioAnalysis.test.ts
+++ b/packages/core/src/audioAnalysis.test.ts
@@ -240,7 +240,28 @@ describe("runLadder", () => {
it("falls through a failing provider and says so", async () => {
const result = await runLadder("moments", tone, [broken, signalProvider]);
expect(result.provenance).toBe("signal");
- expect(result.warnings).toEqual(["moments: broken failed (model missing); used signal"]);
+ expect(result.warnings).toEqual(["moments: used signal; broken failed (model missing)"]);
+ });
+
+ it("falls through a provider that cannot run here, or whose check throws, and says why", async () => {
+ const keyless = { ...service, available: () => ({ ok: false as const, reason: "no key" }) };
+ const throwing = {
+ ...broken,
+ id: "throwing",
+ available: () => {
+ throw new Error("probe crashed");
+ },
+ };
+ const result = await runLadder("moments", tone, [keyless, throwing, signalProvider]);
+ expect(result.provenance).toBe("signal");
+ expect(result.steps.map((s) => [s.provider, s.outcome, s.reason])).toEqual([
+ ["service", "unavailable", "no key"],
+ ["throwing", "unavailable", "probe crashed"],
+ ["signal", "answered", undefined],
+ ]);
+ expect(result.warnings).toEqual([
+ "moments: used signal; service unavailable (no key); throwing unavailable (probe crashed)",
+ ]);
});
it("follows a ladder given in place of the default, and reports a capability no provider offers", async () => {
@@ -248,8 +269,8 @@ describe("runLadder", () => {
ladder: ["signal", "service"],
});
expect(result.provenance).toBe("signal");
- const beats = await runLadder("beats", tone, [signalProvider]);
- expect(beats).toMatchObject({ value: null, provenance: null });
- expect(beats.warnings).toEqual(["beats: no provider answered (none offers it)"]);
+ const described = await runLadder("describe", tone, [signalProvider]);
+ expect(described).toMatchObject({ value: null, provenance: null });
+ expect(described.warnings).toEqual(["describe: no provider answered (none offers it)"]);
});
});
diff --git a/packages/core/src/audioAnalysis.ts b/packages/core/src/audioAnalysis.ts
index 99e51f3c0f..7098a363cf 100644
--- a/packages/core/src/audioAnalysis.ts
+++ b/packages/core/src/audioAnalysis.ts
@@ -3,8 +3,10 @@
* Pure functions of PCM samples, so they run the same in Node, Bun and the browser. Times are seconds into the signal.
*/
-import type { AudioAnalysisProvider } from "./audioAnalysisLadder";
+import type { AudioAnalysisProvider, AudioCapability } from "./audioAnalysisLadder";
import { fft } from "./fft";
+import { rhythm } from "./audioRhythm";
+import { sections, type SectionOptions } from "./audioSections";
/** Decoded audio: one Float32Array per channel, all the same length. */
export interface AudioSignal {
@@ -595,29 +597,43 @@ export function classify(signal: AudioSignal): KindGuess {
}
export * from "./audioAnalysisLadder";
+export * from "./audioRhythm";
+export * from "./audioSections";
+export * from "./audioSnap";
+
+const SIGNAL_MEASURES: Partial<
+ Record<AudioCapability, (input: AudioSignal, options: Record<string, unknown>) => unknown>
+> = {
+ beats: (input) => rhythm(input).beats,
+ downbeats: (input) => rhythm(input).downbeats,
+ sections: (input, options) => sections(input, options as SectionOptions),
+ envelope: (input, options) => envelope(input, options as EnvelopeOptions),
+ onsets: (input, options) => onsets(input, options as OnsetOptions),
+ moments: (input, options) => moments(input, options as MomentOptions),
+ loudness: (input) => loudness(input),
+ silence: (input, options) => silence(input, options as SilenceOptions),
+ kind: (input) => classify(input),
+};
/** Signal processing as a ladder provider: always available, the measured ground every other provider snaps to. */
export const signalProvider: AudioAnalysisProvider<AudioSignal> = {
id: "signal",
local: true,
signal: true,
- capabilities: ["envelope", "onsets", "moments", "loudness", "silence", "kind"],
+ capabilities: [
+ "envelope",
+ "onsets",
+ "moments",
+ "loudness",
+ "silence",
+ "kind",
+ "beats",
+ "downbeats",
+ "sections",
+ ],
async analyze(capability, input, options) {
- switch (capability) {
- case "envelope":
- return envelope(input, options as EnvelopeOptions);
- case "onsets":
- return onsets(input, options as OnsetOptions);
- case "moments":
- return moments(input, options as MomentOptions);
- case "loudness":
- return loudness(input);
- case "silence":
- return silence(input, options as SilenceOptions);
- case "kind":
- return classify(input);
- default:
- throw new Error(`signal processing does not measure ${capability}`);
- }
+ const measure = SIGNAL_MEASURES[capability];
+ if (!measure) throw new Error(`signal processing does not measure ${capability}`);
+ return measure(input, options);
},
};
diff --git a/packages/core/src/audioAnalysisLadder.ts b/packages/core/src/audioAnalysisLadder.ts
index 728e7bbadb..c9e35cb36d 100644
--- a/packages/core/src/audioAnalysisLadder.ts
+++ b/packages/core/src/audioAnalysisLadder.ts
@@ -12,13 +12,17 @@ export type AudioCapability =
| "silence"
| "kind"
| "beats"
+ | "downbeats"
+ | "bpm"
| "sections"
- | "words"
| "describe";
/** `best` may use any provider; `local` none that sends audio off the machine; `fast` only signal processing. */
export type AnalysisQuality = "best" | "local" | "fast";
+/** Whether a provider can run here now, and if not, what is missing. */
+export type Availability = { ok: true } | { ok: false; reason: string };
+
export interface AudioAnalysisProvider<Input = unknown> {
id: string;
/** Runs on this machine; a provider that is not local sends the audio to a service. */
@@ -27,7 +31,7 @@ export interface AudioAnalysisProvider<Input = unknown> {
signal?: boolean;
capabilities: readonly AudioCapability[];
/** Whether it can run here now: a key, a platform, a model on disk. Missing means always. */
- available?: () => boolean | Promise<boolean>;
+ available?: () => Availability | Promise<Availability>;
analyze: (
capability: AudioCapability,
input: Input,
@@ -78,6 +82,18 @@ export function exclusion<Input>(
return null;
}
+const message = (error: unknown) => (error instanceof Error ? error.message : String(error));
+
+async function availabilityOf<Input>(
+ provider: AudioAnalysisProvider<Input>,
+): Promise<Availability> {
+ try {
+ return provider.available ? await provider.available() : { ok: true };
+ } catch (error) {
+ return { ok: false, reason: message(error) };
+ }
+}
+
/** Why a provider cannot take this step, or null when it can. */
async function blocked<Input>(
id: string,
@@ -89,11 +105,22 @@ async function blocked<Input>(
return { provider: id, outcome: "unavailable", reason: `no provider ${id} for ${capability}` };
const excluded = exclusion(candidate, quality);
if (excluded) return { provider: id, outcome: "excluded", reason: excluded };
- if (candidate.available && !(await candidate.available()))
- return { provider: id, outcome: "unavailable" };
+ const availability = await availabilityOf(candidate);
+ if (!availability.ok)
+ return { provider: id, outcome: "unavailable", reason: availability.reason };
return null;
}
+/** One line naming the providers above `answered` that could not answer, and why; null when none was passed over. */
+function passedOver(capability: AudioCapability, steps: LadderStep[], answered: string | null) {
+ const skipped = steps.filter((s) => s.outcome === "unavailable" || s.outcome === "failed");
+ if (!skipped.length) return null;
+ const why = skipped
+ .map((s) => `${s.provider} ${s.outcome}${s.reason ? ` (${s.reason})` : ""}`)
+ .join("; ");
+ return answered ? `${capability}: used ${answered}; ${why}` : `${capability}: ${why}`;
+}
+
export async function runLadder<Input, Value = unknown>(
capability: AudioCapability,
input: Input,
@@ -112,21 +139,19 @@ export async function runLadder<Input, Value = unknown>(
}
try {
const value = (await candidate!.analyze(capability, input, options)) as Value;
+ const warning = passedOver(capability, steps, id);
steps.push({ provider: id, outcome: "answered" });
- const warnings = steps
- .filter((step) => step.outcome === "failed")
- .map((step) => `${capability}: ${step.provider} failed (${step.reason}); used ${id}`);
- return { value, provenance: id, steps, warnings };
+ return { value, provenance: id, steps, warnings: warning ? [warning] : [] };
} catch (error) {
- const reason = error instanceof Error ? error.message : String(error);
- steps.push({ provider: id, outcome: "failed", reason });
+ steps.push({ provider: id, outcome: "failed", reason: message(error) });
}
}
const none = order.length ? "" : " (none offers it)";
+ const why = passedOver(capability, steps, null)?.slice(capability.length + 2);
return {
value: null,
provenance: null,
steps,
- warnings: [`${capability}: no provider answered${none}`],
+ warnings: [`${capability}: no provider answered${none}${why ? `; ${why}` : ""}`],
};
}
diff --git a/packages/core/src/audioRhythm.test.ts b/packages/core/src/audioRhythm.test.ts
new file mode 100644
index 0000000000..e176dbd7ad
--- /dev/null
+++ b/packages/core/src/audioRhythm.test.ts
@@ -0,0 +1,156 @@
+import { describe, expect, it } from "vitest";
+import type { AudioSignal } from "./audioAnalysis";
+import { rhythm, tempoOf } from "./audioRhythm";
+import { sections } from "./audioSections";
+import { snapper } from "./audioSnap";
+
+const RATE = 22_050;
+
+function random(seed = 3) {
+ let state = seed;
+ return () => {
+ state = (state * 1_103_515_245 + 12_345) % 2_147_483_648;
+ return (state / 2_147_483_648) * 2 - 1;
+ };
+}
+
+/** A drum pattern on a known grid: a kick with a low thump on each bar's first beat, a click on the others. */
+function drums(beatTimes: number[], meter: number, seconds: number): AudioSignal {
+ const samples = new Float32Array(Math.round(seconds * RATE));
+ const noise = random();
+ beatTimes.forEach((t, k) => {
+ const downbeat = k % meter === 0;
+ const start = Math.round(t * RATE);
+ for (let i = 0; i < 0.12 * RATE && start + i < samples.length; i++) {
+ const since = i / RATE;
+ const click = (downbeat ? 0.5 : 0.35) * noise() * Math.exp(-since / 0.02);
+ const thump = downbeat
+ ? 0.8 * Math.sin(2 * Math.PI * 55 * since) * Math.exp(-since / 0.06)
+ : 0;
+ samples[start + i]! += click + thump;
+ }
+ });
+ return { sampleRate: RATE, channels: [samples] };
+}
+
+const grid = (from: number, period: number, until: number) =>
+ Array.from({ length: Math.floor((until - from) / period) + 1 }, (_, k) => from + k * period);
+
+/** Share of `truth` with a detected time within 70 ms, the usual beat-tracking tolerance. */
+const recall = (found: number[], truth: number[]) =>
+ truth.filter((t) => found.some((f) => Math.abs(f - t) <= 0.07)).length / truth.length;
+
+describe("rhythm", () => {
+ it("places beats on a steady grid, with its tempo and its bars", () => {
+ const truth = grid(0.25, 0.5, 19.5);
+ const { beats, downbeats } = rhythm(drums(truth, 4, 20));
+ expect(recall(beats.times, truth)).toBeGreaterThan(0.95);
+ expect(beats.times.length).toBeLessThanOrEqual(truth.length + 1);
+ expect(beats.confidence).toBeGreaterThan(0.3);
+ expect(tempoOf(beats.times).value).toBeCloseTo(120, 0);
+ expect(downbeats).toMatchObject({ meter: 4, confidence: "low" });
+ expect(
+ recall(
+ downbeats.times,
+ truth.filter((_, k) => k % 4 === 0),
+ ),
+ ).toBeGreaterThan(0.9);
+ });
+
+ it("follows a tempo that changes partway through", () => {
+ const truth = [...grid(0.25, 0.6, 15), ...grid(15.25, 0.45, 30)];
+ const { beats } = rhythm(drums(truth, 4, 30));
+ expect(recall(beats.times, truth)).toBeGreaterThan(0.85);
+ const curve = tempoOf(beats.times).curve;
+ expect(curve[0]!.bpm).toBeCloseTo(100, -1);
+ expect(curve[curve.length - 1]!.bpm).toBeCloseTo(133, -1);
+ });
+
+ it("finds no steady beat in silence or in noise, and none in a clip too short to tell", () => {
+ const silent = rhythm({ sampleRate: RATE, channels: [new Float32Array(10 * RATE)] });
+ expect(silent.beats).toMatchObject({ times: [], reason: "no steady beat" });
+ const noise = random(9);
+ const hiss = rhythm({
+ sampleRate: RATE,
+ channels: [Float32Array.from({ length: 10 * RATE }, () => 0.3 * noise())],
+ });
+ expect(hiss.beats.times).toEqual([]);
+ const short = rhythm(drums(grid(0.25, 0.5, 3), 4, 3));
+ expect(short.beats).toMatchObject({ times: [], reason: "too short for a steady beat" });
+ });
+});
+
+describe("tempoOf", () => {
+ it("has no tempo without two beats", () => {
+ expect(tempoOf([1])).toEqual({ value: null, curve: [] });
+ });
+});
+
+describe("sections", () => {
+ /** Noise filtered differently per part, each a steady texture, with a beat throughout. */
+ function parts(spec: { seconds: number; bright: boolean }[]): AudioSignal {
+ const total = spec.reduce((a, p) => a + p.seconds, 0);
+ const drumsTrack = drums(grid(0.25, 0.5, total - 0.5), 4, total).channels[0]!;
+ const noise = random(5);
+ let low = 0;
+ let at = 0;
+ for (const part of spec) {
+ for (let i = Math.round(at * RATE); i < Math.round((at + part.seconds) * RATE); i++) {
+ const white = noise();
+ low = 0.98 * low + 0.02 * white;
+ drumsTrack[i]! += part.bright ? 0.15 * (white - low) : 2 * low;
+ }
+ at += part.seconds;
+ }
+ return { sampleRate: RATE, channels: [drumsTrack] };
+ }
+
+ it("splits where the sound changes, on a beat, and gives a returning part its earlier letter", () => {
+ const found = sections(
+ parts([
+ { seconds: 20, bright: false },
+ { seconds: 20, bright: true },
+ { seconds: 20, bright: false },
+ ]),
+ );
+ expect(found.spans.map((s) => s.label)).toEqual(["A", "B", "A"]);
+ expect(found.spans[1]!.start).toBeCloseTo(20, 0);
+ expect(found.spans[2]!.start).toBeCloseTo(40, 0);
+ expect(found.spans[2]!.end).toBe(60);
+ expect(found.snap).toMatchObject({ reference: "signal beats", snapped: 2, unsnapped: [] });
+ });
+
+ it.each([1, 4, 8])(
+ "keeps 30 seconds of silence as one section with a %s-second minimum",
+ (minSeconds) => {
+ const signal = { sampleRate: RATE, channels: [new Float32Array(30 * RATE)] };
+ expect(sections(signal, { minSeconds }).spans).toEqual([{ start: 0, end: 30, label: "A" }]);
+ },
+ );
+
+ it("keeps a short sound as one section", () => {
+ expect(sections(parts([{ seconds: 5, bright: true }])).spans).toEqual([
+ { start: 0, end: 5, label: "A" },
+ ]);
+ });
+});
+
+describe("snapper", () => {
+ it("moves each time to the nearest measured one within tolerance and keeps the rest, counting both", () => {
+ const onBeat = snapper([1, 2, 3, 4], 0.1, "signal beats");
+ expect([0.95, 2.04, 2.5, 3.98].map(onBeat.snap)).toEqual([1, 2, 2.5, 4]);
+ expect(onBeat.report()).toEqual({
+ reference: "signal beats",
+ toleranceMs: 100,
+ snapped: 3,
+ unsnapped: [2.5],
+ medianShiftMs: 40,
+ });
+ });
+
+ it("keeps every time when nothing was measured", () => {
+ const none = snapper([], 0.1, "signal beats");
+ expect(none.snap(1.23)).toBe(1.23);
+ expect(none.report()).toMatchObject({ snapped: 0, unsnapped: [1.23], medianShiftMs: null });
+ });
+});
diff --git a/packages/core/src/audioRhythm.ts b/packages/core/src/audioRhythm.ts
new file mode 100644
index 0000000000..9d1bd4f783
--- /dev/null
+++ b/packages/core/src/audioRhythm.ts
@@ -0,0 +1,399 @@
+/**
+ * Beats, downbeats and tempo from signal processing alone: a log-band spectral-flux onset curve, an autocorrelation
+ * tempo with a log-normal prior, and dynamic-programming beat placement (Ellis 2007). Times are seconds into the signal.
+ */
+
+import type { AudioSignal } from "./audioAnalysis";
+import { fft } from "./fft";
+
+export interface Beats {
+ /** Empty when there is no steady beat, with `reason` saying so. */
+ times: number[];
+ /** 0–1: how strongly the onsets repeat at the beat period, scaled by how much stronger they are on the beats. */
+ confidence?: number;
+ reason?: string;
+}
+
+export interface Downbeats {
+ times: number[];
+ /** Beats per bar, 3 or 4; null when there are too few beats to tell. */
+ meter: number | null;
+ /** "low" for a guess from where the low end hits hardest. */
+ confidence?: "low";
+}
+
+export interface TempoPoint {
+ t: number;
+ bpm: number;
+}
+
+export interface Tempo {
+ /** Beats per minute from the median beat interval; null without at least two beats. */
+ value: number | null;
+ /** The tempo over the beats around each point, every few seconds, for music that speeds up or slows down. */
+ curve: TempoPoint[];
+}
+
+export interface Rhythm {
+ beats: Beats;
+ downbeats: Downbeats;
+}
+
+/** Log energy per frame in log-spaced bands, the frames `fps` per second. */
+export interface BandFrames {
+ fps: number;
+ bands: Float64Array[];
+}
+
+const FRAMES_PER_SECOND = 100;
+const LOW_BAND_HZ = 150;
+const SLOWEST_BPM = 40;
+const FASTEST_BPM = 240;
+const TIGHTNESS = 100;
+
+const round = (t: number) => Math.round(t * 1000) / 1000;
+
+function bandEdges(sampleRate: number, size: number): number[] {
+ const binHz = sampleRate / size;
+ const edges: number[] = [];
+ for (let hz = 30; hz < Math.min(9000, sampleRate / 2); hz *= 2 ** (1 / 5))
+ edges.push(Math.max(1, Math.round(hz / binHz)));
+ return edges;
+}
+
+/** Add each band's power in the Hann-windowed frame centred on `center` into `into`. */
+function addFramePower(
+ samples: Float32Array,
+ center: number,
+ window: Float64Array,
+ edges: number[],
+ into: Float64Array,
+): void {
+ const size = window.length;
+ const re = new Float64Array(size);
+ const im = new Float64Array(size);
+ const first = center - size / 2;
+ for (let i = 0; i < size; i++) {
+ const at = first + i;
+ re[i] = at >= 0 && at < samples.length ? samples[at]! * window[i]! : 0;
+ }
+ fft(re, im);
+ for (let b = 0; b + 1 < edges.length; b++) {
+ const end = Math.max(edges[b]! + 1, edges[b + 1]!);
+ let power = 0;
+ for (let k = edges[b]!; k < end; k++) power += re[k]! * re[k]! + im[k]! * im[k]!;
+ into[b]! += power / (size * size * (end - edges[b]!));
+ }
+}
+
+/** Per-band log power, summed over channels so a polarity-flipped channel cannot cancel. */
+export function bandFrames(signal: AudioSignal): BandFrames {
+ const { sampleRate, channels } = signal;
+ const hop = Math.max(1, Math.round(sampleRate / FRAMES_PER_SECOND));
+ const size = 2 ** Math.round(Math.log2(sampleRate * 0.046));
+ const window = Float64Array.from(
+ { length: size },
+ (_, i) => 0.5 - 0.5 * Math.cos((2 * Math.PI * i) / size),
+ );
+ const edges = bandEdges(sampleRate, size);
+ const length = channels[0]?.length ?? 0;
+ const bands: Float64Array[] = [];
+ for (let center = 0; center < length; center += hop) {
+ const power = new Float64Array(Math.max(0, edges.length - 1));
+ for (const channel of channels) addFramePower(channel, center, window, edges, power);
+ bands.push(power.map((p) => Math.log1p(1e9 * p)));
+ }
+ return { fps: sampleRate / hop, bands };
+}
+
+/** Subtract the mean of the surrounding ±`radius` frames and keep what rises above it. */
+function aboveLocalMean(curve: Float64Array, radius: number): Float64Array {
+ const sums = new Float64Array(curve.length + 1);
+ for (let i = 0; i < curve.length; i++) sums[i + 1] = sums[i]! + curve[i]!;
+ return curve.map((value, i) => {
+ const from = Math.max(0, i - radius);
+ const to = Math.min(curve.length, i + radius + 1);
+ return Math.max(0, value - (sums[to]! - sums[from]!) / (to - from));
+ });
+}
+
+function standardized(curve: Float64Array): Float64Array {
+ const mean = curve.reduce((a, b) => a + b, 0) / (curve.length || 1);
+ const spread = Math.sqrt(curve.reduce((a, v) => a + (v - mean) ** 2, 0) / (curve.length || 1));
+ return spread > 0 ? curve.map((v) => v / spread) : curve.map(() => 0);
+}
+
+interface OnsetCurves {
+ fps: number;
+ /** Rectified rise summed over every band. */
+ broad: Float64Array;
+ /** Rectified rise of the bands under 150 Hz, where kick drums and bass mark the bar. */
+ low: Float64Array;
+}
+
+function onsetCurves({ fps, bands }: BandFrames, sampleRate: number): OnsetCurves {
+ const size = 2 ** Math.round(Math.log2(sampleRate * 0.046));
+ const lowBands = bandEdges(sampleRate, size).filter(
+ (bin) => (bin * sampleRate) / size < LOW_BAND_HZ,
+ ).length;
+ const broad = new Float64Array(bands.length);
+ const low = new Float64Array(bands.length);
+ for (let f = 1; f < bands.length; f++) {
+ const frame = bands[f]!;
+ const previous = bands[f - 1]!;
+ for (let b = 0; b < frame.length; b++) {
+ const rise = Math.max(0, frame[b]! - previous[b]!);
+ broad[f]! += rise;
+ if (b < lowBands) low[f]! += rise;
+ }
+ }
+ const radius = Math.round(0.4 * fps);
+ return {
+ fps,
+ broad: standardized(aboveLocalMean(broad, radius)),
+ low: standardized(aboveLocalMean(low, radius)),
+ };
+}
+
+/** Autocorrelation of `curve` at lags 0…`maxLag`, normalized by lag 0. */
+function autocorrelation(curve: Float64Array, maxLag: number): Float64Array {
+ const mean = curve.reduce((a, b) => a + b, 0) / (curve.length || 1);
+ const centred = curve.map((v) => v - mean);
+ const ac = new Float64Array(maxLag + 1);
+ for (let lag = 0; lag <= maxLag && lag < centred.length; lag++) {
+ let sum = 0;
+ for (let i = lag; i < centred.length; i++) sum += centred[i]! * centred[i - lag]!;
+ ac[lag] = sum;
+ }
+ const zero = ac[0] || 1;
+ return ac.map((v) => v / zero);
+}
+
+const tempoPrior = (lag: number, fps: number) =>
+ Math.exp(-0.5 * Math.log2(60 / (lag / fps) / 120) ** 2);
+
+interface LagRange {
+ shortest: number;
+ longest: number;
+}
+
+const fullRange = (fps: number): LagRange => ({
+ shortest: Math.max(1, Math.floor((60 / FASTEST_BPM) * fps)),
+ longest: Math.ceil((60 / SLOWEST_BPM) * fps),
+});
+
+/** The beat period in frames: prior-weighted autocorrelation plus half its value at twice the lag. */
+function beatPeriod(
+ curve: Float64Array,
+ fps: number,
+ { shortest, longest }: LagRange = fullRange(fps),
+): { period: number; clarity: number } {
+ const ac = autocorrelation(curve, 2 * longest + 1);
+ const at = (lag: number) => ac[Math.min(ac.length - 1, Math.max(0, lag))]!;
+ const score = (lag: number) =>
+ tempoPrior(lag, fps) *
+ (at(lag) + 0.5 * at(2 * lag) + 0.25 * (at(2 * lag - 1) + at(2 * lag + 1)));
+ let best = shortest;
+ for (let lag = shortest; lag <= longest; lag++) if (score(lag) > score(best)) best = lag;
+ const [a, b, c] = [score(best - 1), score(best), score(best + 1)];
+ const curvature = a - 2 * b + c;
+ const shift = curvature ? Math.max(-0.5, Math.min(0.5, (0.5 * (a - c)) / curvature)) : 0;
+ return { period: best + shift, clarity: Math.max(0, Math.min(1, at(best))) };
+}
+
+/** The onset curve smoothed by a Gaussian a thirty-second of a beat wide. */
+function smoothed(curve: Float64Array, period: number): Float64Array {
+ const half = Math.max(1, Math.round(period / 8));
+ const kernel = Array.from({ length: 2 * half + 1 }, (_, i) =>
+ Math.exp(-0.5 * (((i - half) * 32) / period) ** 2),
+ );
+ return curve.map((_, i) => {
+ let sum = 0;
+ for (let j = 0; j < kernel.length; j++) sum += kernel[j]! * (curve[i + j - half] ?? 0);
+ return sum;
+ });
+}
+
+/** Ellis's recursion: each frame's best score from a predecessor about one period back, penalized by log deviation. */
+function cumulativeScore(local: Float64Array, periodAt: (frame: number) => number) {
+ const score = new Float64Array(local.length);
+ const back = new Int32Array(local.length).fill(-1);
+ for (let i = 0; i < local.length; i++) {
+ const period = periodAt(i);
+ let best = -Infinity;
+ let from = -1;
+ for (let j = Math.max(0, Math.round(i - 2 * period)); j <= i - Math.round(period / 2); j++) {
+ const candidate = score[j]! - TIGHTNESS * Math.log((i - j) / period) ** 2;
+ if (candidate > best) [best, from] = [candidate, j];
+ }
+ score[i] = local[i]! + Math.max(0, best);
+ back[i] = best > 0 ? from : -1;
+ }
+ return { score, back };
+}
+
+/** The last local maximum of the cumulative score that reaches half the median of all of them. */
+function lastBeat(score: Float64Array): number {
+ const maxima: number[] = [];
+ for (let i = 1; i + 1 < score.length; i++)
+ if (score[i]! > score[i - 1]! && score[i]! >= score[i + 1]!) maxima.push(i);
+ if (!maxima.length) return score.length - 1;
+ const median = maxima.map((i) => score[i]!).sort((a, b) => a - b)[maxima.length >> 1]!;
+ for (let k = maxima.length - 1; k >= 0; k--)
+ if (score[maxima[k]!]! >= 0.5 * median) return maxima[k]!;
+ return maxima[maxima.length - 1]!;
+}
+
+/** Drop beats at either end whose onset strength is under half the RMS of the beats' strengths. */
+function trimmed(beats: number[], local: Float64Array): number[] {
+ const rms = Math.sqrt(beats.reduce((a, b) => a + local[b]! ** 2, 0) / (beats.length || 1));
+ let first = 0;
+ let end = beats.length;
+ while (first < end && local[beats[first]!]! < 0.5 * rms) first++;
+ while (end > first && local[beats[end - 1]!]! < 0.5 * rms) end--;
+ return beats.slice(first, end);
+}
+
+const LOCAL_WINDOW_SECONDS = 8;
+
+/** The period through the piece: eight-second windows a second apart, held near the global period, median-smoothed. */
+function localPeriods(curve: Float64Array, fps: number, period: number): (frame: number) => number {
+ const window = Math.round(LOCAL_WINDOW_SECONDS * fps);
+ const hop = Math.round(fps);
+ const range = {
+ shortest: Math.max(fullRange(fps).shortest, Math.floor(period * 0.7)),
+ longest: Math.min(fullRange(fps).longest, Math.ceil(period * 1.4)),
+ };
+ const centres: number[] = [];
+ const periods: number[] = [];
+ for (let start = 0; start === 0 || start + window <= curve.length; start += hop) {
+ const end = Math.min(curve.length, start + window);
+ periods.push(beatPeriod(curve.subarray(start, end), fps, range).period);
+ centres.push((start + end) / 2);
+ }
+ const median = periods.map((_, i) => {
+ const near = periods.slice(Math.max(0, i - 2), i + 3).sort((a, b) => a - b);
+ return near[near.length >> 1]!;
+ });
+ return (frame) => interpolate(centres, median, frame);
+}
+
+function interpolate(xs: number[], ys: number[], x: number): number {
+ if (x <= xs[0]!) return ys[0]!;
+ for (let i = 1; i < xs.length; i++)
+ if (x <= xs[i]!) {
+ const t = (x - xs[i - 1]!) / (xs[i]! - xs[i - 1]!);
+ return ys[i - 1]! * (1 - t) + ys[i]! * t;
+ }
+ return ys[ys.length - 1]!;
+}
+
+function beatFrames(curve: Float64Array, fps: number, period: number) {
+ const local = smoothed(curve, period);
+ const { score, back } = cumulativeScore(local, localPeriods(curve, fps, period));
+ const beats: number[] = [];
+ for (let i = lastBeat(score); i >= 0; i = back[i]!) beats.push(i);
+ return { frames: trimmed(beats.reverse(), local), local };
+}
+
+/** How much stronger the smoothed onsets are on the beats than on average, as a share: 0 is no stronger. */
+function onsetLift(frames: number[], local: Float64Array): number {
+ const overall = local.reduce((a, b) => a + b, 0) / (local.length || 1);
+ const onBeats = frames.reduce((a, f) => a + local[f]!, 0) / (frames.length || 1);
+ return overall > 0 ? onBeats / overall - 1 : 0;
+}
+
+/** The meter and bar phase whose beats carry the most low-band onset energy against the others. */
+function barPhase(beats: number[], curves: OnsetCurves): { meter: number; phase: number } {
+ const strength = (f: number) =>
+ Math.max(
+ ...[-1, 0, 1, 2].map((d) => (curves.low[f + d] ?? 0) + 0.3 * (curves.broad[f + d] ?? 0)),
+ );
+ const levels = beats.map(strength);
+ let best = { meter: 4, phase: 0, contrast: -Infinity };
+ for (const meter of [4, 3])
+ for (let phase = 0; phase < meter; phase++) {
+ const on = levels.filter((_, k) => (k - phase) % meter === 0);
+ const off = levels.filter((_, k) => (k - phase) % meter !== 0);
+ const mean = (v: number[]) => v.reduce((a, b) => a + b, 0) / (v.length || 1);
+ const contrast = (mean(on) - mean(off)) * (meter === 3 ? 0.85 : 1);
+ if (contrast > best.contrast) best = { meter, phase, contrast };
+ }
+ return best;
+}
+
+export function medianInterval(times: readonly number[]): number | null {
+ const gaps = times
+ .slice(1)
+ .map((t, i) => t - times[i]!)
+ .sort((a, b) => a - b);
+ return gaps.length ? gaps[gaps.length >> 1]! : null;
+}
+
+const noRhythm = (confidence: number, reason = "no steady beat"): Rhythm => ({
+ beats: { times: [], confidence, reason },
+ downbeats: { times: [], meter: null, confidence: "low" },
+});
+
+/** Two bars of 4/4 at 80 BPM: anything shorter has too few beats to call steady. */
+const SHORTEST_SECONDS = 6;
+
+/** Below this the beats are more likely the tracker's own grid than the music's. */
+export const STEADY_BEAT_CONFIDENCE = 0.05;
+
+/** Clarity of the period, scaled down when the beats are under three times as strong as the onsets around them. */
+const beatConfidence = (clarity: number, lift: number) =>
+ round(Math.max(0, Math.min(1, clarity)) * Math.max(0, Math.min(1, lift / 3)));
+
+/** Beats and downbeats of `frames`, the band energies of a signal at `sampleRate`. */
+export function rhythmOf(frames: BandFrames, sampleRate: number): Rhythm {
+ const curves = onsetCurves(frames, sampleRate);
+ const { period, clarity } = beatPeriod(curves.broad, curves.fps);
+ if (curves.broad.length < SHORTEST_SECONDS * curves.fps)
+ return noRhythm(0, "too short for a steady beat");
+ if (!curves.broad.some((v) => v > 0) || curves.broad.length < 2 * period) return noRhythm(0);
+ const { frames: beatList, local } = beatFrames(curves.broad, curves.fps, period);
+ const confidence = beatConfidence(clarity, onsetLift(beatList, local));
+ if (beatList.length < 4 || confidence < STEADY_BEAT_CONFIDENCE) return noRhythm(confidence);
+ const times = beatList.map((f) => round(f / curves.fps));
+ const bar = beatList.length >= 6 ? barPhase(beatList, curves) : null;
+ return {
+ beats: { times, confidence },
+ downbeats: {
+ times: bar ? times.filter((_, k) => (k - bar.phase) % bar.meter === 0) : [],
+ meter: bar?.meter ?? null,
+ confidence: "low",
+ },
+ };
+}
+
+const TEMPO_STEP_SECONDS = 4;
+
+/** Tempo from beat times, by whichever provider placed them: the median interval, and a curve over ±4 s windows. */
+export function tempoOf(times: readonly number[]): Tempo {
+ const interval = medianInterval(times);
+ if (!interval) return { value: null, curve: [] };
+ const bpm = (gap: number) => Math.round(600 / gap) / 10;
+ const curve: TempoPoint[] = [];
+ const last = times[times.length - 1]!;
+ for (let t = times[0]!; t <= last; t += TEMPO_STEP_SECONDS) {
+ const gap = medianInterval(times.filter((b) => Math.abs(b - t) <= TEMPO_STEP_SECONDS));
+ if (gap) curve.push({ t: round(t), bpm: bpm(gap) });
+ }
+ return { value: bpm(interval), curve };
+}
+
+const measured = new WeakMap<AudioSignal, { frames: BandFrames; rhythm?: Rhythm }>();
+
+/** Band energies of `signal`, computed once per signal for every capability that reads them. */
+export function bandFramesOf(signal: AudioSignal): BandFrames {
+ let entry = measured.get(signal);
+ if (!entry) measured.set(signal, (entry = { frames: bandFrames(signal) }));
+ return entry.frames;
+}
+
+export function rhythm(signal: AudioSignal): Rhythm {
+ const frames = bandFramesOf(signal);
+ const entry = measured.get(signal)!;
+ return (entry.rhythm ??= rhythmOf(frames, signal.sampleRate));
+}
diff --git a/packages/core/src/audioSections.ts b/packages/core/src/audioSections.ts
new file mode 100644
index 0000000000..334adb8d6e
--- /dev/null
+++ b/packages/core/src/audioSections.ts
@@ -0,0 +1,139 @@
+/**
+ * Sections from signal processing: band energies averaged into half-second blocks, a self-similarity matrix, and
+ * Foote's checkerboard novelty, whose peaks are where the music changes. Boundaries snap to the nearest beat.
+ */
+
+import type { AudioSignal } from "./audioAnalysis";
+import { bandFramesOf, medianInterval, rhythm, type BandFrames } from "./audioRhythm";
+import { snapper, type SnapReport } from "./audioSnap";
+
+export interface Section {
+ start: number;
+ end: number;
+ /** Sections that sound alike share a letter. */
+ label: string;
+}
+
+export interface Sections {
+ spans: Section[];
+ snap?: SnapReport;
+}
+
+export interface SectionOptions {
+ /** Shortest section, in seconds. */
+ minSeconds?: number;
+}
+
+const BLOCK_SECONDS = 0.5;
+/** Checkerboard half-width: eight seconds of music either side of a candidate boundary. */
+const KERNEL_BLOCKS = 16;
+const SAME_SECTION = 0.9;
+
+const round = (t: number) => Math.round(t * 1000) / 1000;
+
+function meanOf(frames: ArrayLike<number>[]): number[] {
+ const sum = new Array<number>(frames[0]?.length ?? 0).fill(0);
+ for (const frame of frames) for (let b = 0; b < sum.length; b++) sum[b]! += frame[b]!;
+ return sum.map((v) => v / (frames.length || 1));
+}
+
+/** Each band moved to zero mean and unit spread over time, in place. */
+function standardizeBands(blocks: number[][]): void {
+ for (let b = 0; b < (blocks[0]?.length ?? 0); b++) {
+ const column = blocks.map((block) => block[b]!);
+ const mean = column.reduce((a, v) => a + v, 0) / column.length;
+ const spread = Math.sqrt(column.reduce((a, v) => a + (v - mean) ** 2, 0) / column.length);
+ for (const block of blocks) block[b] = (block[b]! - mean) / (spread || 1);
+ }
+}
+
+/** Band energies averaged over blocks, each band standardized over time and each block scaled to unit length. */
+function blockFeatures({ fps, bands }: BandFrames): number[][] {
+ const per = Math.max(1, Math.round(BLOCK_SECONDS * fps));
+ const blocks: number[][] = [];
+ for (let start = 0; start + per <= bands.length; start += per)
+ blocks.push(meanOf(bands.slice(start, start + per)));
+ standardizeBands(blocks);
+ return blocks.map(unit);
+}
+
+function unit(vector: number[]): number[] {
+ const length = Math.hypot(...vector) || 1;
+ return vector.map((v) => v / length);
+}
+
+const dot = (a: number[], b: number[]) => a.reduce((sum, v, i) => sum + v * b[i]!, 0);
+
+/** The checkerboard kernel, offsets -KERNEL_BLOCKS…KERNEL_BLOCKS-1 each way: + within a side, − across, tapered. */
+const CHECKERBOARD = (() => {
+ const offsets = Array.from({ length: 2 * KERNEL_BLOCKS }, (_, k) => k - KERNEL_BLOCKS);
+ const taper = (k: number) =>
+ Math.exp(-0.5 * (((k < 0 ? -k - 1 : k) + 0.5) / (KERNEL_BLOCKS / 2)) ** 2);
+ return offsets.map((a) => offsets.map((b) => (a < 0 === b < 0 ? 1 : -1) * taper(a) * taper(b)));
+})();
+
+/** Foote novelty at block `i`: similarity within each side of it minus similarity across it. */
+function noveltyAt(blocks: number[][], i: number): number {
+ let sum = 0;
+ CHECKERBOARD.forEach((row, a) => {
+ const x = blocks[i + a - KERNEL_BLOCKS];
+ if (!x) return;
+ row.forEach((weight, b) => {
+ const y = blocks[i + b - KERNEL_BLOCKS];
+ if (y) sum += weight * dot(x, y);
+ });
+ });
+ return sum;
+}
+
+const novelty = (blocks: number[][]) => blocks.map((_, i) => noveltyAt(blocks, i));
+
+/** Blocks where novelty peaks over its neighbourhood and clears the mean by a standard deviation. */
+function boundaryBlocks(curve: number[], minBlocks: number): number[] {
+ const mean = curve.reduce((a, b) => a + b, 0) / (curve.length || 1);
+ const spread = Math.sqrt(curve.reduce((a, v) => a + (v - mean) ** 2, 0) / (curve.length || 1));
+ const peaks: number[] = [];
+ for (let i = minBlocks; i <= curve.length - minBlocks; i++) {
+ const neighbourhood = curve.slice(Math.max(0, i - minBlocks), i + minBlocks + 1);
+ if (curve[i]! < Math.max(...neighbourhood) || curve[i]! <= mean + spread) continue;
+ if (!peaks.length || i - peaks[peaks.length - 1]! >= minBlocks) peaks.push(i);
+ }
+ return peaks;
+}
+
+/** A letter per span: the letter of the first earlier span whose mean features are this alike, else a new one. */
+function labels(blocks: number[][], edges: number[]): string[] {
+ const means: number[][] = [];
+ const letters: string[] = [];
+ for (let s = 0; s + 1 < edges.length; s++) {
+ const part = blocks.slice(edges[s]!, edges[s + 1]!);
+ const mean = unit(part[0]!.map((_, b) => part.reduce((a, block) => a + block[b]!, 0)));
+ const match = means.findIndex((other) => dot(other, mean) >= SAME_SECTION);
+ letters.push(
+ match >= 0 ? letters[match]! : String.fromCharCode(65 + (new Set(letters).size % 26)),
+ );
+ means.push(mean);
+ }
+ return letters;
+}
+
+/** Half the median beat interval: a boundary moves to a beat, never past one. */
+const beatTolerance = (beats: number[]) => (medianInterval(beats) ?? 0) / 2;
+
+export function sections(signal: AudioSignal, options: SectionOptions = {}): Sections {
+ const duration = round((signal.channels[0]?.length ?? 0) / signal.sampleRate);
+ const blocks = blockFeatures(bandFramesOf(signal));
+ const minBlocks = Math.max(1, Math.round((options.minSeconds ?? 4) / BLOCK_SECONDS));
+ if (blocks.length < 2 * minBlocks) return { spans: [{ start: 0, end: duration, label: "A" }] };
+ const edges = [0, ...boundaryBlocks(novelty(blocks), minBlocks), blocks.length];
+ const letters = labels(blocks, edges);
+ const beats = rhythm(signal).beats.times;
+ const onBeat = snapper(beats, beatTolerance(beats), "signal beats");
+ const times = edges.map((e, i) =>
+ i === 0 ? 0 : i === edges.length - 1 ? duration : round(onBeat.snap(e * BLOCK_SECONDS)),
+ );
+ return {
+ spans: letters.map((label, s) => ({ start: times[s]!, end: times[s + 1]!, label })),
+ snap: onBeat.report(),
+ };
+}
diff --git a/packages/core/src/audioSnap.ts b/packages/core/src/audioSnap.ts
new file mode 100644
index 0000000000..55cc6ff5cd
--- /dev/null
+++ b/packages/core/src/audioSnap.ts
@@ -0,0 +1,62 @@
+/** Moving reported times onto measured ones, one at a time, and saying how far they moved. */
+
+export interface SnapReport {
+ /** What the times were moved onto. */
+ reference: string;
+ toleranceMs: number;
+ /** Times moved onto a measured one. */
+ snapped: number;
+ /** Times with nothing measured within the tolerance, kept as reported. */
+ unsnapped: number[];
+ /** Median distance between a reported time and the measured one it moved to, in ms: low means they agreed. */
+ medianShiftMs: number | null;
+}
+
+/** Index of the value in ascending `sorted` nearest `t`. */
+function nearestIndex(sorted: readonly number[], t: number): number {
+ let low = 0;
+ let high = sorted.length - 1;
+ while (low < high) {
+ const mid = (low + high) >> 1;
+ if (sorted[mid]! < t) low = mid + 1;
+ else high = mid;
+ }
+ return low > 0 && t - sorted[low - 1]! <= sorted[low]! - t ? low - 1 : low;
+}
+
+export interface Snapper {
+ snap: (t: number) => number;
+ report: () => SnapReport;
+}
+
+/** Snaps each time to the nearest of `measured` (ascending) when one is within `toleranceSeconds`, else keeps it. */
+export function snapper(
+ measured: readonly number[],
+ toleranceSeconds: number,
+ reference: string,
+): Snapper {
+ const shifts: number[] = [];
+ const unsnapped: number[] = [];
+ return {
+ snap(t) {
+ const nearest = measured.length ? measured[nearestIndex(measured, t)]! : null;
+ if (nearest === null || Math.abs(nearest - t) > toleranceSeconds) {
+ unsnapped.push(t);
+ return t;
+ }
+ shifts.push(Math.abs(nearest - t));
+ return nearest;
+ },
+ report() {
+ const sorted = [...shifts].sort((a, b) => a - b);
+ const median = sorted.length ? sorted[sorted.length >> 1]! : null;
+ return {
+ reference,
+ toleranceMs: Math.round(toleranceSeconds * 1000),
+ snapped: shifts.length,
+ unsnapped: [...unsnapped],
+ medianShiftMs: median === null ? null : Math.round(median * 1000),
+ };
+ },
+ };
+}