Write HTML. Render video. Built for agents.
JavaScript aiai-agentsanimationclaude-codeffmpegframeworkgsaphtmlhtml-to-videomcpmotion-graphicspuppeteerrenderingtypescriptvideo-editor
feat(cli): audio answers beats, downbeats, bpm, sections and describe through a provider ladder (#5632)
Xuanru Li8e340d220ea4
diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdxindex 3b65380cc7..77290194e7 100644--- a/docs/packages/cli.mdx+++ b/docs/packages/cli.mdx@@ -540,9 +540,57 @@ For a file not in the film yet, `--place-at <s>` answers as if it started at thafilm second; `--skip-file-seconds <s>` cuts that much from the start of the filefirst, and the result warns when the cut part held the file's loudest moment.-Every measurement comes from a provider ladder. Today the only provider is-local signal processing; `--quality fast|local|best` limits which providers may-run, `--provider onsets=signal` pins one, and `--full` shows each ladder step.+`--only` also asks for the music's structure, which is not measured unless asked:++- **beats**: beat times, or none with a `reason` (`no steady beat`, or too short+ to tell) when the music has no beat to follow.+- **downbeats**: the first beat of each bar, with the `meter` (3 or 4). From+ signal processing these are a guess (`confidence: "low"`).+- **bpm**: the tempo read off the beats (median interval), with a `curve` every+ 4 s for music that speeds up or slows down. It always comes from the provider+ that answered `beats`.+- **sections**: spans where the sound stays alike, each with a letter; a part+ that returns gets its earlier letter.+- **describe**: labels for what the audio holds, each with a score.++Word timing is not here: use [`transcribe`](#transcribe).++Every measurement comes from a provider ladder, tried in this order until one+answers:++| Capability | Ladder |+| --- | --- |+| moments, onsets, loudness, silence, envelope, kind | `signal` |+| beats, downbeats (and so bpm) | `api`, `apple-mu`, `beat-this`, `signal` |+| sections | `api`, `apple-mu`, `signal` |+| describe | `api`, `apple-sound` |++- `api` is an audio analysis service, used when `HYPERFRAMES_AUDIO_API_URL` and+ `HYPERFRAMES_AUDIO_API_KEY` are both set. The audio is uploaded. Every beat,+ downbeat or section boundary it answers moves onto the nearest beat signal+ processing measured, when one is within a quarter beat; the rest are listed in+ `snap.unsnapped`, and `snap.medianShiftMs` says how far the moved ones went.+- `apple-mu` is Apple's Music Understanding, which needs macOS 27. It is not+ available yet.+- `beat-this` is [Beat This!](https://github.com/CPJKU/beat_this), used when+ `HYPERFRAMES_BEAT_THIS_PYTHON` names a Python with `beat-this` installed and+ its `small0` weights are on disk (`HYPERFRAMES_BEAT_THIS_CHECKPOINT`, or+ `beat_this-small0.ckpt` in the Torch hub cache). The CLI never downloads them;+ to set it up once:++ ```bash+ uv init --bare ~/.beat-this+ uv add --project ~/.beat-this "beat-this @ git+https://github.com/CPJKU/beat_this" torch torchaudio soundfile+ ~/.beat-this/.venv/bin/python -c "from beat_this.inference import load_checkpoint; load_checkpoint('small0')"+ export HYPERFRAMES_BEAT_THIS_PYTHON=~/.beat-this/.venv/bin/python+ ```+- `apple-sound` is macOS's built-in sound classifier, through a small Swift+ helper compiled once with the Xcode command line tools.++A provider that cannot run, or fails, is passed over with a warning that says+why. `--quality fast|local|best` limits which providers may run (`local` keeps+the audio on this machine, `fast` allows only signal processing),+`--provider beats=signal` pins one, and `--full` shows each ladder step.Tune the analysis with `--onset-method`, `--threshold`, `--band lo,hi`,`--hop-ms`, `--onset-ratio`, `--tail-db` and `--silence-db`. Decoding needsFFmpeg.diff --git a/packages/cli/src/audio/analysis/analyze.test.ts b/packages/cli/src/audio/analysis/analyze.test.tsindex 37847f690a..68e54b7873 100644--- a/packages/cli/src/audio/analysis/analyze.test.ts+++ b/packages/cli/src/audio/analysis/analyze.test.ts@@ -38,10 +38,48 @@ describe("analyzeSignal", () => {expect(analysis.values.moments).toEqual({ onset: 1.1, steepest: 1.1, peak: 1.1, tailEnd: 1.2 });});- it("warns, and answers nothing, for a capability no provider offers yet", async () => {- const analysis = await analyzeSignal(tone, { capabilities: ["beats"] });- expect(analysis.values.beats).toBeNull();- expect(analysis.warnings).toEqual(["beats: no provider answered (none offers it)"]);+ it("warns, and answers nothing, for a capability no provider offers", async () => {+ const analysis = await analyzeSignal(tone, { capabilities: ["describe"] }, [signalProvider]);+ expect(analysis.values.describe).toBeNull();+ expect(analysis.warnings).toEqual(["describe: no provider answered (none offers it)"]);+ });++ it("reads tempo off the beats of whichever provider placed them, in file seconds", async () => {+ const grid: AudioAnalysisProvider<AudioSignal> = {+ id: "grid",+ local: true,+ capabilities: ["beats"],+ analyze: async () => ({ times: [0, 0.5, 1, 1.5, 2, 2.5, 3, 3.5, 4] }),+ };+ const analysis = await analyzeSignal(tone, { capabilities: ["bpm", "beats"], fileOffset: 10 }, [+ grid,+ signalProvider,+ ]);+ expect(analysis.provenance).toEqual({ bpm: "grid", beats: "grid" });+ expect(analysis.values.bpm).toEqual({+ value: 120,+ curve: [+ { t: 10, bpm: 120 },+ { t: 14, bpm: 120 },+ ],+ });+ expect((analysis.values.beats as { times: number[] }).times[0]).toBe(10);+ });++ it("keeps a service off the ladder at quality fast", async () => {+ const service: AudioAnalysisProvider<AudioSignal> = {+ id: "service",+ local: false,+ capabilities: ["beats"],+ analyze: async () => ({ times: [1] }),+ };+ const analysis = await analyzeSignal(+ tone,+ { capabilities: ["beats"], quality: "fast", options: {} },+ [service, signalProvider],+ );+ expect(analysis.provenance.beats).toBe("signal");+ expect(analysis.warnings).toEqual([]);});});diff --git a/packages/cli/src/audio/analysis/analyze.ts b/packages/cli/src/audio/analysis/analyze.tsindex 2f4b2bf5f6..bc9850bd6c 100644--- a/packages/cli/src/audio/analysis/analyze.ts+++ b/packages/cli/src/audio/analysis/analyze.ts@@ -2,12 +2,19 @@ import {runLadder,signalDuration,signalProvider,+ tempoOf,type AnalysisQuality,type AudioAnalysisProvider,type AudioCapability,type AudioSignal,+ type Beats,+ type LadderResult,type LadderStep,} from "@hyperframes/core/audio-analysis";+import { apiProvider } from "./providers/api.js";+import { appleMuProvider, appleSoundProvider } from "./providers/apple.js";+import { beatThisProvider } from "./providers/beatThis.js";+import { shiftedTimes } from "./fileTime.js";/** What `hyperframes audio` asks for when `--only` is not given. */export const DEFAULT_CAPABILITIES: readonly AudioCapability[] = [@@ -19,9 +26,12 @@ export const DEFAULT_CAPABILITIES: readonly AudioCapability[] = ["envelope",];-/** Every provider the CLI knows, in default ladder order. Model and service providers join this list. */+/**+ * Every provider the CLI knows, in default ladder order: each capability tries the ones that offer it in this order.+ * beats, downbeats: api, apple-mu, beat-this, signal. sections: api, apple-mu, signal. describe: api, apple-sound.+ */function audioAnalysisProviders(): AudioAnalysisProvider<AudioSignal>[] {- return [signalProvider];+ return [apiProvider, appleMuProvider, beatThisProvider, appleSoundProvider, signalProvider];}export interface AnalyzeOptions {@@ -54,36 +64,6 @@ export function envelopeHopMs(seconds: number, asked?: number): number {const round = (t: number) => Math.round(t * 1000) / 1000;-/** Move a capability's times from signal seconds to file seconds. */-function toFileTime(capability: AudioCapability, value: unknown, offset: number): unknown {- if (value === null || typeof value !== "object") return value;- const at = (t: number) => round(t + offset);- switch (capability) {- case "moments": {- const m = value as Record<string, number>;- return Object.fromEntries(Object.entries(m).map(([k, t]) => [k, at(t)]));- }- case "onsets": {- const o = value as { events: { t: number; strength: number }[] };- return { ...o, events: o.events.map((e) => ({ ...e, t: at(e.t) })) };- }- case "silence":- return (value as { start: number; end: number }[]).map((s) => ({- start: at(s.start),- end: at(s.end),- }));- case "loudness": {- const l = value as { clipping: { count: number; firstAt: number | null } };- const firstAt = l.clipping.firstAt === null ? null : at(l.clipping.firstAt);- return { ...l, clipping: { ...l.clipping, firstAt } };- }- case "envelope":- return { startsAt: round(offset), ...(value as object) };- default:- return value;- }-}-function within(value: unknown, from: number, to: number): unknown {if (!value || typeof value !== "object" || !("events" in value)) return value;const onsets = value as { events: { t: number }[] };@@ -123,6 +103,9 @@ export async function analyzeSignal(request: AnalyzeOptions,providers: readonly AudioAnalysisProvider<AudioSignal>[] = audioAnalysisProviders(),): Promise<Analysis> {+ const { beats, bpm } = request.provider ?? {};+ if (beats && bpm && beats !== bpm) throw new Error("beats and bpm must use the same provider");+ const beatProvider = beats ?? bpm;const framing = frame(signal, request.context);const analysis: Analysis = {analyzed: round(signalDuration(framing.windowed)),@@ -131,20 +114,37 @@ export async function analyzeSignal(ladder: {},warnings: [],};+ const ladders = new Map<AudioCapability, Promise<LadderResult>>();+ const ladderFor = (capability: AudioCapability) => {+ let result = ladders.get(capability);+ if (!result) {+ const withContext = capability === "onsets" && framing.cut;+ result = runLadder(capability, withContext ? signal : framing.windowed, providers, {+ quality: request.quality,+ provider: capability === "beats" ? beatProvider : request.provider?.[capability],+ options: optionsFor(capability, request, analysis.analyzed),+ });+ ladders.set(capability, result);+ }+ return result;+ };for (const capability of request.capabilities) {- // Onsets see the context, so a window's edge is not mistaken for a hit, then keep only the window's own.const withContext = capability === "onsets" && framing.cut;- const result = await runLadder(capability, withContext ? signal : framing.windowed, providers, {- quality: request.quality,- provider: request.provider?.[capability],- options: optionsFor(capability, request, analysis.analyzed),- });+ const result =+ capability === "bpm" ? tempoFromBeats(await ladderFor("beats")) : await ladderFor(capability);const value = withContext ? within(result.value, framing.before, framing.end) : result.value;const offset = (request.fileOffset ?? 0) - (withContext ? framing.before : 0);- analysis.values[capability] = toFileTime(capability, value, offset);+ analysis.values[capability] = shiftedTimes(capability, value, offset);analysis.provenance[capability] = result.provenance;analysis.ladder[capability] = result.steps;- analysis.warnings.push(...result.warnings);+ const said = new Set(analysis.warnings);+ analysis.warnings.push(...result.warnings.filter((w) => !said.has(w)));}return analysis;}++/** Tempo is read off the beats of whichever provider placed them, so the two never disagree. */+function tempoFromBeats(beats: LadderResult): LadderResult {+ const times = (beats.value as Beats | null)?.times;+ return { ...beats, value: times ? tempoOf(times) : null };+}diff --git a/packages/cli/src/audio/analysis/fileTime.ts b/packages/cli/src/audio/analysis/fileTime.tsnew file mode 100644index 0000000000..5f683ed691--- /dev/null+++ b/packages/cli/src/audio/analysis/fileTime.ts@@ -0,0 +1,57 @@+import type {+ AudioCapability,+ Onsets,+ SnapReport,+ Span,+ Tempo,+} from "@hyperframes/core/audio-analysis";++const round = (t: number) => Math.round(t * 1000) / 1000;++type At = (t: number) => number;++/** A snap report's kept times are times too. */+function withSnap<V extends { snap?: SnapReport }>(value: V, at: At): V {+ return value.snap+ ? { ...value, snap: { ...value.snap, unsnapped: value.snap.unsnapped.map(at) } }+ : value;+}++const SHIFTS: Partial<Record<AudioCapability, (value: never, at: At, offset: number) => unknown>> =+ {+ moments: (m: Record<string, number>, at) =>+ Object.fromEntries(Object.entries(m).map(([k, t]) => [k, at(t)])),+ onsets: (o: Onsets, at) => ({ ...o, events: o.events.map((e) => ({ ...e, t: at(e.t) })) }),+ silence: (spans: Span[], at) => spans.map((s) => ({ start: at(s.start), end: at(s.end) })),+ loudness: (l: { clipping: { count: number; firstAt: number | null } }, at) => ({+ ...l,+ clipping: {+ ...l.clipping,+ firstAt: l.clipping.firstAt === null ? null : at(l.clipping.firstAt),+ },+ }),+ envelope: (e: object, _at, offset) => ({ startsAt: round(offset), ...e }),+ beats: (b: { times: number[]; snap?: SnapReport }, at) =>+ withSnap({ ...b, times: b.times.map(at) }, at),+ downbeats: (b: { times: number[]; snap?: SnapReport }, at) =>+ withSnap({ ...b, times: b.times.map(at) }, at),+ bpm: (tempo: Tempo, at) => ({+ ...tempo,+ curve: tempo.curve.map((p) => ({ ...p, t: at(p.t) })),+ }),+ sections: (s: { spans: Span[]; snap?: SnapReport }, at) =>+ withSnap(+ {+ ...s,+ spans: s.spans.map((span) => ({ ...span, start: at(span.start), end: at(span.end) })),+ },+ at,+ ),+ };++/** Move a capability's times from signal seconds to file seconds. */+export function shiftedTimes(capability: AudioCapability, value: unknown, offset: number): unknown {+ if (value === null || typeof value !== "object") return value;+ const shift = SHIFTS[capability];+ return shift ? shift(value as never, (t) => round(t + offset), offset) : value;+}diff --git a/packages/cli/src/audio/analysis/format.ts b/packages/cli/src/audio/analysis/format.tsindex ffc37a47ee..32dd0d9c0d 100644--- a/packages/cli/src/audio/analysis/format.ts+++ b/packages/cli/src/audio/analysis/format.ts@@ -1,10 +1,15 @@import type {+ Beats,+ Downbeats,Envelope,KindGuess,Loudness,Moments,Onsets,+ Sections,+ SnapReport,Span,+ Tempo,} from "@hyperframes/core/audio-analysis";import type { AudioItem, AudioReport } from "./run.js";import { filmTime } from "./targets.js";@@ -60,6 +65,50 @@ function silenceLine(silence: Span[], at: At): string | null {: null;}+function snapNote(snap: SnapReport | undefined): string {+ if (!snap) return "";+ const total = snap.snapped + snap.unsnapped.length;+ const median = snap.medianShiftMs === null ? "" : `, median ${snap.medianShiftMs} ms`;+ return `; ${snap.snapped}/${total} snapped to ${snap.reference}${median}`;+}++const listed = (times: number[], at: At) => {+ const shown = times.slice(0, ONSETS_SHOWN).map(at).join(" ");+ return times.length > ONSETS_SHOWN ? `${shown} +${times.length - ONSETS_SHOWN}` : shown;+};++function beatsLine(beats: Beats & { snap?: SnapReport }, at: At): string {+ if (!beats.times.length) return ` beats none${beats.reason ? ` (${beats.reason})` : ""}`;+ return ` beats ${beats.times.length}: ${listed(beats.times, at)}${snapNote(beats.snap)}`;+}++function downbeatsLine(downbeats: Downbeats & { snap?: SnapReport }, at: At): string | null {+ if (!downbeats.times.length) return null;+ const meter = downbeats.meter ? ` in ${downbeats.meter}` : "";+ const low = downbeats.confidence === "low" ? ", a guess" : "";+ return ` downbeats${meter}${low}: ${listed(downbeats.times, at)}${snapNote(downbeats.snap)}`;+}++function tempoLine(tempo: Tempo): string | null {+ if (tempo.value === null) return null;+ const bpms = tempo.curve.map((p) => p.bpm);+ const spread =+ bpms.length && Math.max(...bpms) - Math.min(...bpms) >= 2+ ? ` (${Math.min(...bpms)}–${Math.max(...bpms)})`+ : "";+ return ` tempo ${tempo.value} BPM${spread}`;+}++function sectionsLine(sections: Sections, at: At): string {+ const spans = sections.spans.map((s) => `${s.label} ${at(s.start)}–${at(s.end)}`).join(" ");+ return ` sections ${spans}${snapNote(sections.snap)}`;+}++function describeLine(description: { text?: string; tags: { label: string; score: number }[] }) {+ const tags = description.tags.map((t) => `${t.label} ${t.score.toFixed(2)}`).join(", ");+ return ` sounds like ${[description.text, tags].filter(Boolean).join(" — ")}`;+}+/** One line per measurement the item holds, in reading order. */const LINES: [string, (value: never, at: At) => string | null][] = [["moments", momentsLine],@@ -67,6 +116,11 @@ const LINES: [string, (value: never, at: At) => string | null][] = [["loudness", loudnessLine],["silence", silenceLine],["envelope", (envelope: Envelope) => ` ${sparkline(envelope)}`],+ ["describe", describeLine],+ ["bpm", tempoLine],+ ["beats", beatsLine],+ ["downbeats", downbeatsLine],+ ["sections", sectionsLine],];function headline(item: AudioItem, where: string): string {diff --git a/packages/cli/src/audio/analysis/providers/api.test.ts b/packages/cli/src/audio/analysis/providers/api.test.tsnew file mode 100644index 0000000000..8488f0bb76--- /dev/null+++ b/packages/cli/src/audio/analysis/providers/api.test.ts@@ -0,0 +1,244 @@+import { createServer, type IncomingMessage, type Server } from "node:http";+import type { AddressInfo } from "node:net";+import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";+import {+ rhythm,+ signalProvider,+ type AudioSignal,+ type AudioCapability,+} from "@hyperframes/core/audio-analysis";+import { analyzeSignal } from "../analyze.js";+import { apiProvider, type AudioApiRequest, type AudioApiResponse } from "./api.js";++const RATE = 22_050;++/** Clicks every half second from 0.25 s: a grid the signal tracker finds. */+function clicks(seconds: number): AudioSignal {+ const samples = new Float32Array(seconds * RATE);+ let state = 7;+ for (let t = 0.25; t < seconds; t += 0.5)+ for (let i = 0; i < 0.05 * RATE; i++) {+ state = (state * 1_103_515_245 + 12_345) % 2_147_483_648;+ samples[Math.round(t * RATE) + i]! +=+ 0.5 * ((state / 2_147_483_648) * 2 - 1) * Math.exp(-i / (0.01 * RATE));+ }+ return { sampleRate: RATE, channels: [samples] };+}++interface Received {+ method: string;+ authorization: string | undefined;+ request: AudioApiRequest;+ audio: { type: string; bytes: Uint8Array };+}++let server: Server;+let received: Received[] = [];+let reply: (request: AudioApiRequest) => { status: number; body: unknown } = () => ({+ status: 500,+ body: {},+});++async function readForm(req: IncomingMessage) {+ const chunks: Buffer[] = [];+ for await (const chunk of req) chunks.push(chunk as Buffer);+ const form = await new Request("http://stand-in", {+ method: "POST",+ headers: { "content-type": req.headers["content-type"]! },+ body: Buffer.concat(chunks),+ }).formData();+ const audio = form.get("audio") as Blob;+ return {+ request: JSON.parse(await (form.get("request") as Blob).text()) as AudioApiRequest,+ audio: { type: audio.type, bytes: new Uint8Array(await audio.arrayBuffer()) },+ };+}++beforeAll(async () => {+ server = createServer(async (req, res) => {+ const form = await readForm(req);+ received.push({ method: req.method!, authorization: req.headers.authorization, ...form });+ const { status, body } = reply(form.request);+ res.writeHead(status, { "content-type": "application/json" }).end(JSON.stringify(body));+ });+ await new Promise<void>((done) => server.listen(0, "127.0.0.1", done));+});++afterAll(() => new Promise<void>((done) => server.close(() => done())));++afterEach(() => {+ received = [];+ delete process.env.HYPERFRAMES_AUDIO_API_URL;+ delete process.env.HYPERFRAMES_AUDIO_API_KEY;+});++function configure() {+ process.env.HYPERFRAMES_AUDIO_API_URL = `http://127.0.0.1:${(server.address() as AddressInfo).port}/v1/audio`;+ process.env.HYPERFRAMES_AUDIO_API_KEY = "test-key";+}++describe("apiProvider", () => {+ it("is unavailable until both the URL and the key are set", () => {+ expect(apiProvider.available!()).toEqual({+ ok: false,+ reason: "HYPERFRAMES_AUDIO_API_URL is not set",+ });+ process.env.HYPERFRAMES_AUDIO_API_URL = "http://127.0.0.1:1/";+ expect(apiProvider.available!()).toEqual({+ ok: false,+ reason: "HYPERFRAMES_AUDIO_API_KEY is not set",+ });+ process.env.HYPERFRAMES_AUDIO_API_KEY = "k";+ expect(apiProvider.available!()).toEqual({ ok: true });+ });++ it("posts the audio and the capabilities, and snaps each answered beat onto the signal's own", async () => {+ configure();+ const signal = clicks(12);+ const measured = rhythm(signal).beats.times;+ // The service is 30 ms late on every beat, and invents one between two real beats.+ const answered = [...measured.map((t) => t + 0.03), 3.5].sort((a, b) => a - b);+ reply = () => ({+ status: 200,+ body: {+ version: 1,+ source: "stand-in",+ beats: { times: answered },+ downbeats: { times: measured.filter((_, k) => k % 4 === 0).map((t) => t - 0.02), meter: 4 },+ } satisfies AudioApiResponse,+ });+ const analysis = await analyzeSignal(signal, { capabilities: ["beats", "downbeats", "bpm"] }, [+ apiProvider,+ signalProvider,+ ]);++ expect(received).toHaveLength(1);+ const [call] = received;+ expect(call).toMatchObject({ method: "POST", authorization: "Bearer test-key" });+ expect(call!.request).toEqual({+ version: 1,+ capabilities: ["beats", "downbeats"],+ audio: { sampleRate: RATE, channels: 1, duration: 12 },+ });+ expect(call!.audio.type).toBe("audio/wav");+ expect(Buffer.from(call!.audio.bytes.subarray(0, 4)).toString()).toBe("RIFF");+ expect(call!.audio.bytes.length).toBe(44 + 12 * RATE * 2);++ expect(analysis.provenance).toEqual({ beats: "api", downbeats: "api", bpm: "api" });+ const beats = analysis.values.beats as { times: number[]; snap: object; source: string };+ expect(beats.times).toEqual([...measured, 3.5].sort((a, b) => a - b));+ expect(beats.source).toBe("stand-in");+ expect(beats.snap).toEqual({+ reference: "signal beats",+ toleranceMs: 125,+ snapped: measured.length,+ unsnapped: [3.5],+ medianShiftMs: 30,+ });+ expect((analysis.values.downbeats as { times: number[] }).times).toEqual(+ measured.filter((_, k) => k % 4 === 0),+ );+ expect((analysis.values.bpm as { value: number }).value).toBeCloseTo(120, 0);+ });++ it.each<{ capabilities: AudioCapability[] }>([+ { capabilities: ["bpm"] },+ { capabilities: ["beats", "bpm"] },+ { capabilities: ["bpm", "beats"] },+ ])("keeps a BPM signal pin local for $capabilities", async ({ capabilities }) => {+ configure();+ reply = () => ({ status: 200, body: { version: 1, beats: { times: [0.5, 1, 1.5] } } });+ const analysis = await analyzeSignal(clicks(8), {+ capabilities,+ provider: { bpm: "signal" },+ });+ expect(received).toHaveLength(0);+ expect(analysis.provenance.bpm).toBe("signal");+ expect(analysis.values.bpm).toMatchObject({ value: expect.any(Number) });+ if (capabilities.includes("beats")) expect(analysis.provenance.beats).toBe("signal");+ });++ it.each([+ { beats: "api", bpm: "signal" },+ { beats: "signal", bpm: "api" },+ ])(+ "rejects conflicting beats/BPM pins before any capability uploads audio: %j",+ async (provider) => {+ configure();+ reply = () => ({+ status: 200,+ body: { version: 1, describe: { tags: [] }, beats: { times: [0.5, 1] } },+ });+ await expect(+ analyzeSignal(clicks(8), {+ capabilities: ["describe", "beats", "bpm"],+ provider,+ }),+ ).rejects.toThrow("beats and bpm must use the same provider");+ expect(received).toHaveLength(0);+ },+ );++ it("asks for sections and describe on their own, and moves inner section boundaries onto beats", async () => {+ configure();+ const signal = clicks(12);+ const beat = rhythm(signal).beats.times.find((t) => Math.abs(t - 6.25) < 0.05)!;+ reply = () => ({+ status: 200,+ body: {+ version: 1,+ sections: {+ spans: [+ { start: 0, end: 6.3, label: "intro" },+ { start: 6.3, end: 12 },+ ],+ },+ describe: { text: "clicks", tags: [{ label: "click", score: 0.9 }] },+ },+ });+ const analysis = await analyzeSignal(signal, { capabilities: ["sections", "describe"] }, [+ apiProvider,+ signalProvider,+ ]);+ expect(received.map((r) => r.request.capabilities)).toEqual([["sections"], ["describe"]]);+ expect(analysis.values.sections).toMatchObject({+ spans: [+ { start: 0, end: beat, label: "intro" },+ { start: beat, end: 12 },+ ],+ snap: { snapped: 2, unsnapped: [] },+ });+ expect(analysis.values.describe).toEqual({+ text: "clicks",+ tags: [{ label: "click", score: 0.9 }],+ });+ });++ it("falls through to signal processing with a warning when the service fails or answers in another shape", async () => {+ configure();+ reply = () => ({ status: 503, body: { error: { message: "busy" } } });+ const failed = await analyzeSignal(clicks(8), { capabilities: ["beats"] }, [+ apiProvider,+ signalProvider,+ ]);+ expect(failed.provenance.beats).toBe("signal");+ expect(failed.warnings).toEqual([+ "beats: used signal; api failed (the service answered 503: busy)",+ ]);++ reply = () => ({ status: 200, body: { version: 1, beats: { times: ["soon"] } } });+ const garbled = await analyzeSignal(clicks(8), { capabilities: ["beats"] }, [+ apiProvider,+ signalProvider,+ ]);+ expect(garbled.provenance.beats).toBe("signal");+ expect(garbled.warnings[0]).toContain("api failed (the service answered beats in a shape");+ });++ it("never sends the key anywhere but the authorization header", async () => {+ configure();+ reply = () => ({ status: 401, body: { error: { message: "bad key" } } });+ const analysis = await analyzeSignal(clicks(8), { capabilities: ["describe"] }, [apiProvider]);+ expect(JSON.stringify(analysis)).not.toContain("test-key");+ });+});diff --git a/packages/cli/src/audio/analysis/providers/api.ts b/packages/cli/src/audio/analysis/providers/api.tsnew file mode 100644index 0000000000..5659f19206--- /dev/null+++ b/packages/cli/src/audio/analysis/providers/api.ts@@ -0,0 +1,197 @@+import {+ medianInterval,+ rhythm,+ snapper,+ type AudioAnalysisProvider,+ type AudioCapability,+ type AudioSignal,+ type Snapper,+} from "@hyperframes/core/audio-analysis";+import { AVAILABLE, encodeWav, perSignal, rounded, unavailable } from "./shared.js";++/**+ * The audio analysis service contract.+ *+ * Request: `POST <HYPERFRAMES_AUDIO_API_URL>`, exactly as configured, with+ * `Authorization: Bearer <HYPERFRAMES_AUDIO_API_KEY>` and a `multipart/form-data` body of two parts:+ * - `request`: `application/json`, an {@link AudioApiRequest};+ * - `audio`: `audio/wav`, 16-bit PCM of the analyzed span, channels as decoded.+ *+ * Response: `200` with `application/json`, an {@link AudioApiResponse}. Every time is in seconds from the start of the+ * uploaded audio. A capability the service cannot answer is left out of the response. Any other status is a failure;+ * its body may be `{ "error": { "message": string } }`.+ */+export interface AudioApiRequest {+ version: 1;+ capabilities: AudioApiCapability[];+ audio: { sampleRate: number; channels: number; duration: number };+}++export type AudioApiCapability = "beats" | "downbeats" | "sections" | "describe";++export interface AudioApiResponse {+ version: 1;+ /** What answered, as the service wants it reported. */+ source?: string;+ beats?: { times: number[] };+ downbeats?: { times: number[]; meter?: number | null };+ sections?: { spans: { start: number; end: number; label?: string }[] };+ describe?: { text?: string; tags: { label: string; score: number }[] };+}++const URL_ENV = "HYPERFRAMES_AUDIO_API_URL";+const KEY_ENV = "HYPERFRAMES_AUDIO_API_KEY";+const TIMEOUT_ENV = "HYPERFRAMES_AUDIO_API_TIMEOUT_MS";+const DEFAULT_TIMEOUT_MS = 120_000;++/** Beats and downbeats come from one request; the others each from their own. */+const askedTogether = (capability: AudioApiCapability): AudioApiCapability[] =>+ capability === "beats" || capability === "downbeats" ? ["beats", "downbeats"] : [capability];++function config() {+ const url = process.env[URL_ENV]?.trim();+ const key = process.env[KEY_ENV]?.trim();+ const timeout = Number(process.env[TIMEOUT_ENV]);+ return { url, key, timeoutMs: timeout > 0 ? timeout : DEFAULT_TIMEOUT_MS };+}++const isNumberList = (value: unknown): value is number[] =>+ Array.isArray(value) && value.every((t) => typeof t === "number" && Number.isFinite(t));++function checked(capability: AudioApiCapability, body: AudioApiResponse): void {+ const value = body[capability] as Record<string, unknown> | undefined;+ if (!value || typeof value !== "object")+ throw new Error(`the service did not answer ${capability}`);+ const valid = {+ beats: () => isNumberList(value.times),+ downbeats: () => isNumberList(value.times),+ sections: () =>+ Array.isArray(value.spans) &&+ value.spans.every((s: { start?: unknown; end?: unknown }) => isNumberList([s.start, s.end])),+ describe: () =>+ Array.isArray(value.tags) &&+ value.tags.every(+ (t: { label?: unknown; score?: unknown }) =>+ typeof t.label === "string" && typeof t.score === "number",+ ),+ }[capability];+ if (!valid())+ throw new Error(`the service answered ${capability} in a shape this CLI cannot read`);+}++async function failure(response: Response): Promise<Error> {+ const text = await response.text().catch(() => "");+ let detail = text.slice(0, 200);+ try {+ detail = (JSON.parse(text) as { error?: { message?: string } }).error?.message ?? detail;+ } catch {+ // not JSON: keep the text+ }+ return new Error(`the service answered ${response.status}${detail ? `: ${detail}` : ""}`);+}++async function ask(signal: AudioSignal, capabilities: AudioApiCapability[]) {+ const { url, key, timeoutMs } = config();+ if (!url || !key) throw new Error(`${URL_ENV} and ${KEY_ENV} must both be set`);+ const request: AudioApiRequest = {+ version: 1,+ capabilities,+ audio: {+ sampleRate: signal.sampleRate,+ channels: signal.channels.length,+ duration: (signal.channels[0]?.length ?? 0) / signal.sampleRate,+ },+ };+ const form = new FormData();+ form.append("request", new Blob([JSON.stringify(request)], { type: "application/json" }));+ form.append(+ "audio",+ new Blob([new Uint8Array(encodeWav(signal))], { type: "audio/wav" }),+ "audio.wav",+ );+ const response = await fetch(url, {+ method: "POST",+ headers: { authorization: `Bearer ${key}` },+ body: form,+ signal: AbortSignal.timeout(timeoutMs),+ });+ if (!response.ok) throw await failure(response);+ return (await response.json()) as AudioApiResponse;+}++const requests = new Map<string, (signal: AudioSignal) => Promise<AudioApiResponse>>();++function answerFor(signal: AudioSignal, capability: AudioApiCapability) {+ const together = askedTogether(capability);+ const key = together.join(",");+ let request = requests.get(key);+ if (!request) requests.set(key, (request = perSignal((s) => ask(s, together))));+ return request(signal);+}++/** The signal's own beats, which an answer's times move onto when one is within a quarter of the answer's beat. */+function beatSnapper(signal: AudioSignal, beatSeconds: number | null): Snapper {+ const measured = rhythm(signal).beats.times;+ const period = beatSeconds ?? medianInterval(measured) ?? 0;+ return snapper(measured, period / 4, "signal beats");+}++function snappedBeats(signal: AudioSignal, body: AudioApiResponse) {+ const times = body.beats!.times;+ const onBeat = beatSnapper(signal, medianInterval(times));+ return { times: rounded(times.map(onBeat.snap)), snap: onBeat.report() };+}++function snappedDownbeats(signal: AudioSignal, body: AudioApiResponse) {+ const { times, meter = null } = body.downbeats!;+ const beat = body.beats ? medianInterval(body.beats.times) : null;+ const bar = medianInterval(times);+ const onBeat = beatSnapper(signal, beat ?? (bar && meter ? bar / meter : null));+ return { times: rounded(times.map(onBeat.snap)), meter, snap: onBeat.report() };+}++function snappedSections(signal: AudioSignal, body: AudioApiResponse) {+ const onBeat = beatSnapper(signal, null);+ const end = (signal.channels[0]?.length ?? 0) / signal.sampleRate;+ const move = (t: number) => (t <= 0 || t >= end ? t : onBeat.snap(t));+ const spans = body.sections!.spans.map((s) => ({+ ...s,+ start: rounded([move(s.start)])[0]!,+ end: rounded([move(s.end)])[0]!,+ }));+ return { spans, snap: onBeat.report() };+}++/** The answer for one capability, its times moved onto what the signal measured. */+function measured(capability: AudioApiCapability, signal: AudioSignal, body: AudioApiResponse) {+ const value =+ capability === "beats"+ ? snappedBeats(signal, body)+ : capability === "downbeats"+ ? snappedDownbeats(signal, body)+ : capability === "sections"+ ? snappedSections(signal, body)+ : body.describe!;+ return body.source ? { ...value, source: body.source } : value;+}++const API_CAPABILITIES: readonly AudioCapability[] = ["beats", "downbeats", "sections", "describe"];++/** HeyGen's audio analysis service, or any service that speaks {@link AudioApiRequest}, configured by environment. */+export const apiProvider: AudioAnalysisProvider<AudioSignal> = {+ id: "api",+ local: false,+ capabilities: API_CAPABILITIES,+ available() {+ const { url, key } = config();+ if (!url) return unavailable(`${URL_ENV} is not set`);+ if (!key) return unavailable(`${KEY_ENV} is not set`);+ return AVAILABLE;+ },+ async analyze(capability, signal) {+ const asked = capability as AudioApiCapability;+ const body = await answerFor(signal, asked);+ checked(asked, body);+ return measured(asked, signal, body);+ },+};diff --git a/packages/cli/src/audio/analysis/providers/apple.ts b/packages/cli/src/audio/analysis/providers/apple.tsnew file mode 100644index 0000000000..296697a72b--- /dev/null+++ b/packages/cli/src/audio/analysis/providers/apple.ts@@ -0,0 +1,171 @@+import { createHash } from "node:crypto";+import { existsSync } from "node:fs";+import { mkdir, rename, writeFile } from "node:fs/promises";+import { homedir } from "node:os";+import { join } from "node:path";+import type { AudioAnalysisProvider, AudioSignal } from "@hyperframes/core/audio-analysis";+import { AVAILABLE, runFile, unavailable, withWavFile } from "./shared.js";++const SOUND_ANALYSIS = "/System/Library/Frameworks/SoundAnalysis.framework";+const MUSIC_UNDERSTANDING_MACOS = 27;++/** Apple's built-in sound classifier over the whole file: each label's mean confidence across its windows. */+const SOUND_TAGGER = `+import Foundation+import SoundAnalysis++final class Collector: NSObject, SNResultsObserving {+ var sums: [String: Double] = [:]+ var windows = 0+ var failure: Error?+ func request(_ request: SNRequest, didProduce result: SNResult) {+ guard let result = result as? SNClassificationResult else { return }+ windows += 1+ for c in result.classifications { sums[c.identifier, default: 0] += c.confidence }+ }+ func request(_ request: SNRequest, didFailWithError error: Error) { failure = error }+}++do {+ let analyzer = try SNAudioFileAnalyzer(url: URL(fileURLWithPath: CommandLine.arguments[1]))+ let collector = Collector()+ try analyzer.add(try SNClassifySoundRequest(classifierIdentifier: .version1), withObserver: collector)+ analyzer.analyze()+ if let failure = collector.failure { throw failure }+ let count = Double(max(1, collector.windows))+ var tags: [[String: Any]] = []+ for (label, sum) in collector.sums.sorted(by: { $0.value > $1.value }).prefix(10) {+ let score: Double = sum / count+ if score >= 0.01 { tags.append(["label": label, "score": (score * 1000).rounded() / 1000]) }+ }+ let json = try JSONSerialization.data(withJSONObject: ["tags": tags, "windows": collector.windows])+ FileHandle.standardOutput.write(json)+} catch {+ FileHandle.standardError.write(Data("\\(error)".utf8))+ exit(1)+}+`;++/** The classifier listens in windows of about three seconds and says nothing about a shorter file. */+const CLASSIFIER_WINDOW_SECONDS = 3;++function atLeastOneWindow(signal: AudioSignal): AudioSignal {+ const length = Math.ceil(CLASSIFIER_WINDOW_SECONDS * signal.sampleRate);+ if ((signal.channels[0]?.length ?? 0) >= length) return signal;+ return {+ ...signal,+ channels: signal.channels.map((channel) => {+ const padded = new Float32Array(length);+ padded.set(channel);+ return padded;+ }),+ };+}++const helperDir = () => join(homedir(), ".cache", "hyperframes", "audio-helpers");++function helperPath(name: string, source: string): string {+ const hash = createHash("sha256").update(source).digest("hex").slice(0, 12);+ return join(helperDir(), `${name}-${hash}`);+}++async function hasSwiftCompiler(): Promise<boolean> {+ try {+ await runFile("xcrun", ["--find", "swiftc"], { timeout: 30_000 });+ return true;+ } catch {+ return false;+ }+}++const firstLine = (error: unknown) =>+ ((error as { stderr?: string }).stderr || String(error)).trim().split("\n")[0];++const builds = new Map<string, Promise<string>>();++/** The helper compiled from `source`, built once per source into the cache; Swift is never fetched. */+function compiled(name: string, source: string): Promise<string> {+ const binary = helperPath(name, source);+ if (existsSync(binary)) return Promise.resolve(binary);+ let build = builds.get(binary);+ if (!build) {+ build = (async () => {+ await mkdir(helperDir(), { recursive: true });+ const file = `${binary}.swift`;+ await writeFile(file, source);+ // Through xcrun, so the compiler finds the SDK it belongs to.+ const args = [+ "--sdk",+ "macosx",+ "swiftc",+ "-O",+ "-swift-version",+ "5",+ file,+ "-o",+ `${binary}.tmp`,+ ];+ await runFile("xcrun", args, { timeout: 5 * 60_000 }).catch((error: unknown) => {+ throw new Error(`could not build the ${name} helper: ${firstLine(error)}`);+ });+ await rename(`${binary}.tmp`, binary);+ return binary;+ })();+ builds.set(binary, build);+ }+ return build;+}++/** The macOS major version, or null off macOS. */+async function macosMajor(): Promise<number | null> {+ if (process.platform !== "darwin") return null;+ const { stdout } = await runFile("sw_vers", ["-productVersion"], { timeout: 10_000 });+ return Number.parseInt(stdout, 10) || null;+}++/** Labels for what the audio holds, from Apple's on-device sound classifier (SoundAnalysis, macOS 12+). */+export const appleSoundProvider: AudioAnalysisProvider<AudioSignal> = {+ id: "apple-sound",+ local: true,+ capabilities: ["describe"],+ async available() {+ if (process.platform !== "darwin") return unavailable("macOS only");+ if (!existsSync(SOUND_ANALYSIS))+ return unavailable("SoundAnalysis.framework is not on this Mac");+ if (existsSync(helperPath("apple-sound", SOUND_TAGGER))) return AVAILABLE;+ if (!(await hasSwiftCompiler())) return unavailable("no Swift compiler to build the helper");+ return AVAILABLE;+ },+ async analyze(_capability, signal) {+ const helper = await compiled("apple-sound", SOUND_TAGGER);+ const { stdout } = await withWavFile(atLeastOneWindow(signal), (wav) =>+ runFile(helper, [wav], { timeout: 5 * 60_000, maxBuffer: 8 * 1024 * 1024 }),+ );+ const { tags } = JSON.parse(stdout) as { tags: { label: string; score: number }[] };+ return {+ tags: tags.map((t) => ({ label: t.label, score: Math.round(t.score * 1000) / 1000 })),+ };+ },+};++/**+ * Apple's on-device Music Understanding (WWDC26, macOS 27). No SDK this CLI has been built against offers it, so the+ * helper is not written yet: the provider reports why it cannot run instead of answering.+ */+export const appleMuProvider: AudioAnalysisProvider<AudioSignal> = {+ id: "apple-mu",+ local: true,+ capabilities: ["beats", "downbeats", "sections"],+ async available() {+ const major = await macosMajor();+ if (major === null) return unavailable("macOS only");+ if (major < MUSIC_UNDERSTANDING_MACOS)+ return unavailable(+ `Music Understanding needs macOS ${MUSIC_UNDERSTANDING_MACOS}; this Mac runs ${major}`,+ );+ return unavailable("no Music Understanding helper yet: it needs the macOS 27 SDK to build");+ },+ analyze() {+ return Promise.reject(new Error("Music Understanding is not available"));+ },+};diff --git a/packages/cli/src/audio/analysis/providers/beatThis.ts b/packages/cli/src/audio/analysis/providers/beatThis.tsnew file mode 100644index 0000000000..dff283e2ad--- /dev/null+++ b/packages/cli/src/audio/analysis/providers/beatThis.ts@@ -0,0 +1,85 @@+import { existsSync } from "node:fs";+import { homedir } from "node:os";+import { join } from "node:path";+import type { AudioAnalysisProvider, AudioSignal } from "@hyperframes/core/audio-analysis";+import {+ AVAILABLE,+ downbeatsWithMeter,+ perSignal,+ rounded,+ runFile,+ unavailable,+ withWavFile,+} from "./shared.js";++const PYTHON_ENV = "HYPERFRAMES_BEAT_THIS_PYTHON";+const CHECKPOINT_ENV = "HYPERFRAMES_BEAT_THIS_CHECKPOINT";++/** Where Beat This! keeps the small0 weights after its own first download, unless a path is configured. */+function checkpointPath(): string {+ const configured = process.env[CHECKPOINT_ENV]?.trim();+ if (configured) return configured;+ const torchHome = process.env.TORCH_HOME ?? join(homedir(), ".cache", "torch");+ return join(torchHome, "hub", "checkpoints", "beat_this-small0.ckpt");+}++const FIND_PACKAGE =+ "import importlib.util,sys; sys.exit(importlib.util.find_spec('beat_this') is None)";++// The checkpoint is a local file path, so Beat This! loads it from disk and never downloads.+const TRACK = `+import json, sys+from beat_this.inference import File2Beats+beats, downbeats = File2Beats(checkpoint_path=sys.argv[2], device="cpu", dbn=False)(sys.argv[1])+print(json.dumps({"beats": [float(t) for t in beats], "downbeats": [float(t) for t in downbeats]}))+`;++const installed = new Map<string, Promise<boolean>>();++function hasPackage(python: string): Promise<boolean> {+ let found = installed.get(python);+ if (!found) {+ found = runFile(python, ["-c", FIND_PACKAGE], { timeout: 30_000, windowsHide: true }).then(+ () => true,+ () => false,+ );+ installed.set(python, found);+ }+ return found;+}++const track = perSignal((signal: AudioSignal) =>+ withWavFile(signal, async (wav) => {+ const python = process.env[PYTHON_ENV]!.trim();+ const { stdout } = await runFile(python, ["-c", TRACK, wav, checkpointPath()], {+ maxBuffer: 64 * 1024 * 1024,+ timeout: 10 * 60_000,+ windowsHide: true,+ });+ const answer = JSON.parse(stdout.trim().split("\n").pop()!) as {+ beats: number[];+ downbeats: number[];+ };+ return { beats: rounded(answer.beats), downbeats: rounded(answer.downbeats) };+ }),+);++/** CPJKU's Beat This! (MIT), opt-in: a Python with `beat-this` installed and its small0 weights on disk. */+export const beatThisProvider: AudioAnalysisProvider<AudioSignal> = {+ id: "beat-this",+ local: true,+ capabilities: ["beats", "downbeats"],+ async available() {+ const python = process.env[PYTHON_ENV]?.trim();+ if (!python) return unavailable(`${PYTHON_ENV} is not set`);+ if (!existsSync(python)) return unavailable(`${PYTHON_ENV} names no file`);+ if (!existsSync(checkpointPath()))+ return unavailable(`no Beat This! weights at ${checkpointPath()}`);+ if (!(await hasPackage(python))) return unavailable(`beat_this is not installed for ${python}`);+ return AVAILABLE;+ },+ async analyze(capability, signal) {+ const { beats, downbeats } = await track(signal);+ return capability === "beats" ? { times: beats } : downbeatsWithMeter(beats, downbeats);+ },+};diff --git a/packages/cli/src/audio/analysis/providers/localProviders.test.ts b/packages/cli/src/audio/analysis/providers/localProviders.test.tsnew file mode 100644index 0000000000..11f8bb84e0--- /dev/null+++ b/packages/cli/src/audio/analysis/providers/localProviders.test.ts@@ -0,0 +1,137 @@+import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { afterAll, afterEach, describe, expect, it } from "vitest";+import { signalProvider, type AudioSignal } from "@hyperframes/core/audio-analysis";+import { analyzeSignal } from "../analyze.js";+import { appleMuProvider, appleSoundProvider } from "./apple.js";+import { beatThisProvider } from "./beatThis.js";++const dir = mkdtempSync(join(tmpdir(), "hf-providers-"));+afterAll(() => rmSync(dir, { recursive: true, force: true }));++const tone: AudioSignal = {+ sampleRate: 8000,+ channels: [Float32Array.from({ length: 8000 * 2 }, (_, i) => 0.3 * Math.sin(i / 3))],+};++/** Stands in for a Python with beat_this: answers the package probe, and tracks only a real WAV with real weights. */+function fakePython(hasPackage: boolean): string {+ const path = join(dir, hasPackage ? "python" : "python-bare");+ writeFileSync(+ path,+ `#!/bin/sh+case "$2" in *find_spec*) exit ${hasPackage ? 0 : 1};; esac+[ -f "$3" ] && [ -f "$4" ] || exit 2+head -c 4 "$3" | grep -q RIFF || exit 3+echo "loading weights"+echo '{"beats": [0.5, 1.0, 1.5, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5], "downbeats": [0.5, 2.5, 4.5]}'+`,+ );+ chmodSync(path, 0o755);+ return path;+}++const weights = join(dir, "beat_this-small0.ckpt");+writeFileSync(weights, "weights");++afterEach(() => {+ delete process.env.HYPERFRAMES_BEAT_THIS_PYTHON;+ delete process.env.HYPERFRAMES_BEAT_THIS_CHECKPOINT;+});++describe.skipIf(process.platform === "win32")("beatThisProvider", () => {+ it("is unavailable until a Python with beat_this and the weights are both on this machine", async () => {+ expect(await beatThisProvider.available!()).toEqual({+ ok: false,+ reason: "HYPERFRAMES_BEAT_THIS_PYTHON is not set",+ });+ process.env.HYPERFRAMES_BEAT_THIS_PYTHON = join(dir, "missing");+ expect(await beatThisProvider.available!()).toMatchObject({+ reason: "HYPERFRAMES_BEAT_THIS_PYTHON names no file",+ });+ process.env.HYPERFRAMES_BEAT_THIS_PYTHON = fakePython(true);+ process.env.HYPERFRAMES_BEAT_THIS_CHECKPOINT = join(dir, "absent.ckpt");+ expect(await beatThisProvider.available!()).toMatchObject({+ reason: expect.stringContaining("no Beat This! weights"),+ });+ process.env.HYPERFRAMES_BEAT_THIS_CHECKPOINT = weights;+ expect(await beatThisProvider.available!()).toEqual({ ok: true });+ process.env.HYPERFRAMES_BEAT_THIS_PYTHON = fakePython(false);+ expect(await beatThisProvider.available!()).toMatchObject({+ reason: expect.stringContaining("beat_this is not installed"),+ });+ });++ it("answers beats and downbeats from one run, with the meter from beats per bar", async () => {+ process.env.HYPERFRAMES_BEAT_THIS_PYTHON = fakePython(true);+ process.env.HYPERFRAMES_BEAT_THIS_CHECKPOINT = weights;+ const analysis = await analyzeSignal(tone, { capabilities: ["beats", "downbeats", "bpm"] }, [+ beatThisProvider,+ signalProvider,+ ]);+ expect(analysis.provenance).toEqual({+ beats: "beat-this",+ downbeats: "beat-this",+ bpm: "beat-this",+ });+ expect(analysis.values.downbeats).toEqual({ times: [0.5, 2.5, 4.5], meter: 4 });+ expect(analysis.values.bpm).toMatchObject({ value: 120 });+ });++ it("falls through to signal processing when its run fails", async () => {+ const crashing = join(dir, "python-crashing");+ writeFileSync(+ crashing,+ `#!/bin/sh\ncase "$2" in *find_spec*) exit 0;; esac\necho "out of memory" >&2\nexit 1\n`,+ );+ chmodSync(crashing, 0o755);+ process.env.HYPERFRAMES_BEAT_THIS_PYTHON = crashing;+ process.env.HYPERFRAMES_BEAT_THIS_CHECKPOINT = weights;+ const silent = { ...tone, channels: [new Float32Array(8000 * 7)] };+ const analysis = await analyzeSignal(silent, { capabilities: ["beats"] }, [+ beatThisProvider,+ signalProvider,+ ]);+ expect(analysis.provenance.beats).toBe("signal");+ expect(analysis.warnings[0]).toMatch(+ /^beats: used signal; beat-this failed \(.*out of memory/s,+ );+ });+});++describe("appleMuProvider", () => {+ it("says why it cannot run instead of answering", async () => {+ const availability = await appleMuProvider.available!();+ expect(availability.ok).toBe(false);+ if (!availability.ok)+ expect(availability.reason).toMatch(+ process.platform === "darwin" ? /Music Understanding/ : /^macOS only$/,+ );+ });+});++describe("appleSoundProvider", () => {+ it("is unavailable off macOS", async () => {+ const platform = Object.getOwnPropertyDescriptor(process, "platform")!;+ Object.defineProperty(process, "platform", { value: "linux" });+ try {+ expect(await appleSoundProvider.available!()).toEqual({ ok: false, reason: "macOS only" });+ } finally {+ Object.defineProperty(process, "platform", platform);+ }+ });++ it.skipIf(process.platform !== "darwin")(+ "labels what a short sound holds, using the built-in classifier",+ async () => {+ if (!(await appleSoundProvider.available!()).ok) return;+ const value = (await appleSoundProvider.analyze("describe", tone, {})) as {+ tags: { label: string; score: number }[];+ };+ expect(value.tags.length).toBeGreaterThan(0);+ for (const tag of value.tags) expect(tag.score).toBeGreaterThan(0);+ },+ 300_000,+ );+});diff --git a/packages/cli/src/audio/analysis/providers/shared.ts b/packages/cli/src/audio/analysis/providers/shared.tsnew file mode 100644index 0000000000..c6e88153d9--- /dev/null+++ b/packages/cli/src/audio/analysis/providers/shared.ts@@ -0,0 +1,76 @@+import { execFile } from "node:child_process";+import { mkdtemp, rm, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import { join } from "node:path";+import { promisify } from "node:util";+import type { AudioSignal, Availability, Downbeats } from "@hyperframes/core/audio-analysis";+import { medianInterval } from "@hyperframes/core/audio-analysis";++export const runFile = promisify(execFile);++export const unavailable = (reason: string): Availability => ({ ok: false, reason });+export const AVAILABLE: Availability = { ok: true };++/** 16-bit PCM WAV of the signal, channels interleaved. */+export function encodeWav(signal: AudioSignal): Buffer {+ const channels = signal.channels.length;+ const frames = signal.channels[0]?.length ?? 0;+ const data = frames * channels * 2;+ const wav = Buffer.alloc(44 + data);+ wav.write("RIFF", 0);+ wav.writeUInt32LE(36 + data, 4);+ wav.write("WAVEfmt ", 8);+ wav.writeUInt32LE(16, 16);+ wav.writeUInt16LE(1, 20);+ wav.writeUInt16LE(channels, 22);+ wav.writeUInt32LE(signal.sampleRate, 24);+ wav.writeUInt32LE(signal.sampleRate * channels * 2, 28);+ wav.writeUInt16LE(channels * 2, 32);+ wav.writeUInt16LE(16, 34);+ wav.write("data", 36);+ wav.writeUInt32LE(data, 40);+ let offset = 44;+ for (let i = 0; i < frames; i++)+ for (const channel of signal.channels) {+ const sample = Math.max(-1, Math.min(1, channel[i]!));+ wav.writeInt16LE(Math.round(sample * 32767), offset);+ offset += 2;+ }+ return wav;+}++/** Runs `use` with the signal written to a WAV file that is removed afterwards. */+export async function withWavFile<T>(+ signal: AudioSignal,+ use: (path: string) => Promise<T>,+): Promise<T> {+ const dir = await mkdtemp(join(tmpdir(), "hf-audio-"));+ try {+ const path = join(dir, "audio.wav");+ await writeFile(path, encodeWav(signal));+ return await use(path);+ } finally {+ await rm(dir, { recursive: true, force: true });+ }+}++/** One answer per signal, shared by every capability a provider computes together. */+export function perSignal<T>(compute: (signal: AudioSignal) => Promise<T>) {+ const answers = new WeakMap<AudioSignal, Promise<T>>();+ return (signal: AudioSignal): Promise<T> => {+ let answer = answers.get(signal);+ if (!answer) answers.set(signal, (answer = compute(signal)));+ return answer;+ };+}++/** Downbeats from a model's beat and downbeat times; the meter is the median count of beats per bar. */+export function downbeatsWithMeter(beats: number[], downbeats: number[]): Downbeats {+ const bar = medianInterval(downbeats);+ const beat = medianInterval(beats);+ const meter = bar && beat ? Math.round(bar / beat) : null;+ return { times: downbeats, meter: meter && meter > 0 ? meter : null };+}++const round = (t: number) => Math.round(t * 1000) / 1000;+export const rounded = (times: readonly number[]) => times.map(round);diff --git a/packages/cli/src/commands/audio.ts b/packages/cli/src/commands/audio.tsindex ac1a92517a..85cc7900a5 100644--- a/packages/cli/src/commands/audio.ts+++ b/packages/cli/src/commands/audio.ts@@ -25,6 +25,10 @@ export const examples: Example[] = [],["Every sound playing at 12 s in the film", "hyperframes audio --at 12"],["One clip of the film, as JSON", "hyperframes audio --element '#riser' --json"],+ [+ "The beat grid, tempo and sections of a music bed",+ "hyperframes audio music.mp3 --only beats,downbeats,bpm,sections",+ ],["Only onsets, from 60–200 Hz, finer","hyperframes audio drums.wav --only onsets --band 60,200 --hop-ms 2",@@ -34,8 +38,9 @@ export const examples: Example[] = [const CAPABILITIES = new Set<string>([...DEFAULT_CAPABILITIES,"beats",+ "downbeats",+ "bpm","sections",- "words","describe",]);const QUALITIES = new Set<AnalysisQuality>(["best", "local", "fast"]);@@ -61,6 +66,8 @@ function capabilities(raw: string | undefined): AudioCapability[] {.split(",").map((c) => c.trim()).filter(Boolean);+ if (asked.includes("words"))+ throw new CliUsageError("words come from hyperframes transcribe, not hyperframes audio");const unknown = asked.filter((c) => !CAPABILITIES.has(c));if (unknown.length)throw new CliUsageError(@@ -177,7 +184,7 @@ export default defineCommand({},only: {type: "string",- description: `Capabilities to measure (default ${DEFAULT_CAPABILITIES.join(",")})`,+ description: `Capabilities to measure (default ${DEFAULT_CAPABILITIES.join(",")}; also beats,downbeats,bpm,sections,describe)`,},quality: {type: "string",diff --git a/packages/core/src/audioAnalysis.test.ts b/packages/core/src/audioAnalysis.test.tsindex 3dfcdfe8b7..c366e91725 100644--- a/packages/core/src/audioAnalysis.test.ts+++ b/packages/core/src/audioAnalysis.test.ts@@ -240,7 +240,28 @@ describe("runLadder", () => {it("falls through a failing provider and says so", async () => {const result = await runLadder("moments", tone, [broken, signalProvider]);expect(result.provenance).toBe("signal");- expect(result.warnings).toEqual(["moments: broken failed (model missing); used signal"]);+ expect(result.warnings).toEqual(["moments: used signal; broken failed (model missing)"]);+ });++ it("falls through a provider that cannot run here, or whose check throws, and says why", async () => {+ const keyless = { ...service, available: () => ({ ok: false as const, reason: "no key" }) };+ const throwing = {+ ...broken,+ id: "throwing",+ available: () => {+ throw new Error("probe crashed");+ },+ };+ const result = await runLadder("moments", tone, [keyless, throwing, signalProvider]);+ expect(result.provenance).toBe("signal");+ expect(result.steps.map((s) => [s.provider, s.outcome, s.reason])).toEqual([+ ["service", "unavailable", "no key"],+ ["throwing", "unavailable", "probe crashed"],+ ["signal", "answered", undefined],+ ]);+ expect(result.warnings).toEqual([+ "moments: used signal; service unavailable (no key); throwing unavailable (probe crashed)",+ ]);});it("follows a ladder given in place of the default, and reports a capability no provider offers", async () => {@@ -248,8 +269,8 @@ describe("runLadder", () => {ladder: ["signal", "service"],});expect(result.provenance).toBe("signal");- const beats = await runLadder("beats", tone, [signalProvider]);- expect(beats).toMatchObject({ value: null, provenance: null });- expect(beats.warnings).toEqual(["beats: no provider answered (none offers it)"]);+ const described = await runLadder("describe", tone, [signalProvider]);+ expect(described).toMatchObject({ value: null, provenance: null });+ expect(described.warnings).toEqual(["describe: no provider answered (none offers it)"]);});});diff --git a/packages/core/src/audioAnalysis.ts b/packages/core/src/audioAnalysis.tsindex 99e51f3c0f..7098a363cf 100644--- a/packages/core/src/audioAnalysis.ts+++ b/packages/core/src/audioAnalysis.ts@@ -3,8 +3,10 @@* Pure functions of PCM samples, so they run the same in Node, Bun and the browser. Times are seconds into the signal.*/-import type { AudioAnalysisProvider } from "./audioAnalysisLadder";+import type { AudioAnalysisProvider, AudioCapability } from "./audioAnalysisLadder";import { fft } from "./fft";+import { rhythm } from "./audioRhythm";+import { sections, type SectionOptions } from "./audioSections";/** Decoded audio: one Float32Array per channel, all the same length. */export interface AudioSignal {@@ -595,29 +597,43 @@ export function classify(signal: AudioSignal): KindGuess {}export * from "./audioAnalysisLadder";+export * from "./audioRhythm";+export * from "./audioSections";+export * from "./audioSnap";++const SIGNAL_MEASURES: Partial<+ Record<AudioCapability, (input: AudioSignal, options: Record<string, unknown>) => unknown>+> = {+ beats: (input) => rhythm(input).beats,+ downbeats: (input) => rhythm(input).downbeats,+ sections: (input, options) => sections(input, options as SectionOptions),+ envelope: (input, options) => envelope(input, options as EnvelopeOptions),+ onsets: (input, options) => onsets(input, options as OnsetOptions),+ moments: (input, options) => moments(input, options as MomentOptions),+ loudness: (input) => loudness(input),+ silence: (input, options) => silence(input, options as SilenceOptions),+ kind: (input) => classify(input),+};/** Signal processing as a ladder provider: always available, the measured ground every other provider snaps to. */export const signalProvider: AudioAnalysisProvider<AudioSignal> = {id: "signal",local: true,signal: true,- capabilities: ["envelope", "onsets", "moments", "loudness", "silence", "kind"],+ capabilities: [+ "envelope",+ "onsets",+ "moments",+ "loudness",+ "silence",+ "kind",+ "beats",+ "downbeats",+ "sections",+ ],async analyze(capability, input, options) {- switch (capability) {- case "envelope":- return envelope(input, options as EnvelopeOptions);- case "onsets":- return onsets(input, options as OnsetOptions);- case "moments":- return moments(input, options as MomentOptions);- case "loudness":- return loudness(input);- case "silence":- return silence(input, options as SilenceOptions);- case "kind":- return classify(input);- default:- throw new Error(`signal processing does not measure ${capability}`);- }+ const measure = SIGNAL_MEASURES[capability];+ if (!measure) throw new Error(`signal processing does not measure ${capability}`);+ return measure(input, options);},};diff --git a/packages/core/src/audioAnalysisLadder.ts b/packages/core/src/audioAnalysisLadder.tsindex 728e7bbadb..c9e35cb36d 100644--- a/packages/core/src/audioAnalysisLadder.ts+++ b/packages/core/src/audioAnalysisLadder.ts@@ -12,13 +12,17 @@ export type AudioCapability =| "silence"| "kind"| "beats"+ | "downbeats"+ | "bpm"| "sections"- | "words"| "describe";/** `best` may use any provider; `local` none that sends audio off the machine; `fast` only signal processing. */export type AnalysisQuality = "best" | "local" | "fast";+/** Whether a provider can run here now, and if not, what is missing. */+export type Availability = { ok: true } | { ok: false; reason: string };+export interface AudioAnalysisProvider<Input = unknown> {id: string;/** Runs on this machine; a provider that is not local sends the audio to a service. */@@ -27,7 +31,7 @@ export interface AudioAnalysisProvider<Input = unknown> {signal?: boolean;capabilities: readonly AudioCapability[];/** Whether it can run here now: a key, a platform, a model on disk. Missing means always. */- available?: () => boolean | Promise<boolean>;+ available?: () => Availability | Promise<Availability>;analyze: (capability: AudioCapability,input: Input,@@ -78,6 +82,18 @@ export function exclusion<Input>(return null;}+const message = (error: unknown) => (error instanceof Error ? error.message : String(error));++async function availabilityOf<Input>(+ provider: AudioAnalysisProvider<Input>,+): Promise<Availability> {+ try {+ return provider.available ? await provider.available() : { ok: true };+ } catch (error) {+ return { ok: false, reason: message(error) };+ }+}+/** Why a provider cannot take this step, or null when it can. */async function blocked<Input>(id: string,@@ -89,11 +105,22 @@ async function blocked<Input>(return { provider: id, outcome: "unavailable", reason: `no provider ${id} for ${capability}` };const excluded = exclusion(candidate, quality);if (excluded) return { provider: id, outcome: "excluded", reason: excluded };- if (candidate.available && !(await candidate.available()))- return { provider: id, outcome: "unavailable" };+ const availability = await availabilityOf(candidate);+ if (!availability.ok)+ return { provider: id, outcome: "unavailable", reason: availability.reason };return null;}+/** One line naming the providers above `answered` that could not answer, and why; null when none was passed over. */+function passedOver(capability: AudioCapability, steps: LadderStep[], answered: string | null) {+ const skipped = steps.filter((s) => s.outcome === "unavailable" || s.outcome === "failed");+ if (!skipped.length) return null;+ const why = skipped+ .map((s) => `${s.provider} ${s.outcome}${s.reason ? ` (${s.reason})` : ""}`)+ .join("; ");+ return answered ? `${capability}: used ${answered}; ${why}` : `${capability}: ${why}`;+}+export async function runLadder<Input, Value = unknown>(capability: AudioCapability,input: Input,@@ -112,21 +139,19 @@ export async function runLadder<Input, Value = unknown>(}try {const value = (await candidate!.analyze(capability, input, options)) as Value;+ const warning = passedOver(capability, steps, id);steps.push({ provider: id, outcome: "answered" });- const warnings = steps- .filter((step) => step.outcome === "failed")- .map((step) => `${capability}: ${step.provider} failed (${step.reason}); used ${id}`);- return { value, provenance: id, steps, warnings };+ return { value, provenance: id, steps, warnings: warning ? [warning] : [] };} catch (error) {- const reason = error instanceof Error ? error.message : String(error);- steps.push({ provider: id, outcome: "failed", reason });+ steps.push({ provider: id, outcome: "failed", reason: message(error) });}}const none = order.length ? "" : " (none offers it)";+ const why = passedOver(capability, steps, null)?.slice(capability.length + 2);return {value: null,provenance: null,steps,- warnings: [`${capability}: no provider answered${none}`],+ warnings: [`${capability}: no provider answered${none}${why ? `; ${why}` : ""}`],};}diff --git a/packages/core/src/audioRhythm.test.ts b/packages/core/src/audioRhythm.test.tsnew file mode 100644index 0000000000..e176dbd7ad--- /dev/null+++ b/packages/core/src/audioRhythm.test.ts@@ -0,0 +1,156 @@+import { describe, expect, it } from "vitest";+import type { AudioSignal } from "./audioAnalysis";+import { rhythm, tempoOf } from "./audioRhythm";+import { sections } from "./audioSections";+import { snapper } from "./audioSnap";++const RATE = 22_050;++function random(seed = 3) {+ let state = seed;+ return () => {+ state = (state * 1_103_515_245 + 12_345) % 2_147_483_648;+ return (state / 2_147_483_648) * 2 - 1;+ };+}++/** A drum pattern on a known grid: a kick with a low thump on each bar's first beat, a click on the others. */+function drums(beatTimes: number[], meter: number, seconds: number): AudioSignal {+ const samples = new Float32Array(Math.round(seconds * RATE));+ const noise = random();+ beatTimes.forEach((t, k) => {+ const downbeat = k % meter === 0;+ const start = Math.round(t * RATE);+ for (let i = 0; i < 0.12 * RATE && start + i < samples.length; i++) {+ const since = i / RATE;+ const click = (downbeat ? 0.5 : 0.35) * noise() * Math.exp(-since / 0.02);+ const thump = downbeat+ ? 0.8 * Math.sin(2 * Math.PI * 55 * since) * Math.exp(-since / 0.06)+ : 0;+ samples[start + i]! += click + thump;+ }+ });+ return { sampleRate: RATE, channels: [samples] };+}++const grid = (from: number, period: number, until: number) =>+ Array.from({ length: Math.floor((until - from) / period) + 1 }, (_, k) => from + k * period);++/** Share of `truth` with a detected time within 70 ms, the usual beat-tracking tolerance. */+const recall = (found: number[], truth: number[]) =>+ truth.filter((t) => found.some((f) => Math.abs(f - t) <= 0.07)).length / truth.length;++describe("rhythm", () => {+ it("places beats on a steady grid, with its tempo and its bars", () => {+ const truth = grid(0.25, 0.5, 19.5);+ const { beats, downbeats } = rhythm(drums(truth, 4, 20));+ expect(recall(beats.times, truth)).toBeGreaterThan(0.95);+ expect(beats.times.length).toBeLessThanOrEqual(truth.length + 1);+ expect(beats.confidence).toBeGreaterThan(0.3);+ expect(tempoOf(beats.times).value).toBeCloseTo(120, 0);+ expect(downbeats).toMatchObject({ meter: 4, confidence: "low" });+ expect(+ recall(+ downbeats.times,+ truth.filter((_, k) => k % 4 === 0),+ ),+ ).toBeGreaterThan(0.9);+ });++ it("follows a tempo that changes partway through", () => {+ const truth = [...grid(0.25, 0.6, 15), ...grid(15.25, 0.45, 30)];+ const { beats } = rhythm(drums(truth, 4, 30));+ expect(recall(beats.times, truth)).toBeGreaterThan(0.85);+ const curve = tempoOf(beats.times).curve;+ expect(curve[0]!.bpm).toBeCloseTo(100, -1);+ expect(curve[curve.length - 1]!.bpm).toBeCloseTo(133, -1);+ });++ it("finds no steady beat in silence or in noise, and none in a clip too short to tell", () => {+ const silent = rhythm({ sampleRate: RATE, channels: [new Float32Array(10 * RATE)] });+ expect(silent.beats).toMatchObject({ times: [], reason: "no steady beat" });+ const noise = random(9);+ const hiss = rhythm({+ sampleRate: RATE,+ channels: [Float32Array.from({ length: 10 * RATE }, () => 0.3 * noise())],+ });+ expect(hiss.beats.times).toEqual([]);+ const short = rhythm(drums(grid(0.25, 0.5, 3), 4, 3));+ expect(short.beats).toMatchObject({ times: [], reason: "too short for a steady beat" });+ });+});++describe("tempoOf", () => {+ it("has no tempo without two beats", () => {+ expect(tempoOf([1])).toEqual({ value: null, curve: [] });+ });+});++describe("sections", () => {+ /** Noise filtered differently per part, each a steady texture, with a beat throughout. */+ function parts(spec: { seconds: number; bright: boolean }[]): AudioSignal {+ const total = spec.reduce((a, p) => a + p.seconds, 0);+ const drumsTrack = drums(grid(0.25, 0.5, total - 0.5), 4, total).channels[0]!;+ const noise = random(5);+ let low = 0;+ let at = 0;+ for (const part of spec) {+ for (let i = Math.round(at * RATE); i < Math.round((at + part.seconds) * RATE); i++) {+ const white = noise();+ low = 0.98 * low + 0.02 * white;+ drumsTrack[i]! += part.bright ? 0.15 * (white - low) : 2 * low;+ }+ at += part.seconds;+ }+ return { sampleRate: RATE, channels: [drumsTrack] };+ }++ it("splits where the sound changes, on a beat, and gives a returning part its earlier letter", () => {+ const found = sections(+ parts([+ { seconds: 20, bright: false },+ { seconds: 20, bright: true },+ { seconds: 20, bright: false },+ ]),+ );+ expect(found.spans.map((s) => s.label)).toEqual(["A", "B", "A"]);+ expect(found.spans[1]!.start).toBeCloseTo(20, 0);+ expect(found.spans[2]!.start).toBeCloseTo(40, 0);+ expect(found.spans[2]!.end).toBe(60);+ expect(found.snap).toMatchObject({ reference: "signal beats", snapped: 2, unsnapped: [] });+ });++ it.each([1, 4, 8])(+ "keeps 30 seconds of silence as one section with a %s-second minimum",+ (minSeconds) => {+ const signal = { sampleRate: RATE, channels: [new Float32Array(30 * RATE)] };+ expect(sections(signal, { minSeconds }).spans).toEqual([{ start: 0, end: 30, label: "A" }]);+ },+ );++ it("keeps a short sound as one section", () => {+ expect(sections(parts([{ seconds: 5, bright: true }])).spans).toEqual([+ { start: 0, end: 5, label: "A" },+ ]);+ });+});++describe("snapper", () => {+ it("moves each time to the nearest measured one within tolerance and keeps the rest, counting both", () => {+ const onBeat = snapper([1, 2, 3, 4], 0.1, "signal beats");+ expect([0.95, 2.04, 2.5, 3.98].map(onBeat.snap)).toEqual([1, 2, 2.5, 4]);+ expect(onBeat.report()).toEqual({+ reference: "signal beats",+ toleranceMs: 100,+ snapped: 3,+ unsnapped: [2.5],+ medianShiftMs: 40,+ });+ });++ it("keeps every time when nothing was measured", () => {+ const none = snapper([], 0.1, "signal beats");+ expect(none.snap(1.23)).toBe(1.23);+ expect(none.report()).toMatchObject({ snapped: 0, unsnapped: [1.23], medianShiftMs: null });+ });+});diff --git a/packages/core/src/audioRhythm.ts b/packages/core/src/audioRhythm.tsnew file mode 100644index 0000000000..9d1bd4f783--- /dev/null+++ b/packages/core/src/audioRhythm.ts@@ -0,0 +1,399 @@+/**+ * Beats, downbeats and tempo from signal processing alone: a log-band spectral-flux onset curve, an autocorrelation+ * tempo with a log-normal prior, and dynamic-programming beat placement (Ellis 2007). Times are seconds into the signal.+ */++import type { AudioSignal } from "./audioAnalysis";+import { fft } from "./fft";++export interface Beats {+ /** Empty when there is no steady beat, with `reason` saying so. */+ times: number[];+ /** 0–1: how strongly the onsets repeat at the beat period, scaled by how much stronger they are on the beats. */+ confidence?: number;+ reason?: string;+}++export interface Downbeats {+ times: number[];+ /** Beats per bar, 3 or 4; null when there are too few beats to tell. */+ meter: number | null;+ /** "low" for a guess from where the low end hits hardest. */+ confidence?: "low";+}++export interface TempoPoint {+ t: number;+ bpm: number;+}++export interface Tempo {+ /** Beats per minute from the median beat interval; null without at least two beats. */+ value: number | null;+ /** The tempo over the beats around each point, every few seconds, for music that speeds up or slows down. */+ curve: TempoPoint[];+}++export interface Rhythm {+ beats: Beats;+ downbeats: Downbeats;+}++/** Log energy per frame in log-spaced bands, the frames `fps` per second. */+export interface BandFrames {+ fps: number;+ bands: Float64Array[];+}++const FRAMES_PER_SECOND = 100;+const LOW_BAND_HZ = 150;+const SLOWEST_BPM = 40;+const FASTEST_BPM = 240;+const TIGHTNESS = 100;++const round = (t: number) => Math.round(t * 1000) / 1000;++function bandEdges(sampleRate: number, size: number): number[] {+ const binHz = sampleRate / size;+ const edges: number[] = [];+ for (let hz = 30; hz < Math.min(9000, sampleRate / 2); hz *= 2 ** (1 / 5))+ edges.push(Math.max(1, Math.round(hz / binHz)));+ return edges;+}++/** Add each band's power in the Hann-windowed frame centred on `center` into `into`. */+function addFramePower(+ samples: Float32Array,+ center: number,+ window: Float64Array,+ edges: number[],+ into: Float64Array,+): void {+ const size = window.length;+ const re = new Float64Array(size);+ const im = new Float64Array(size);+ const first = center - size / 2;+ for (let i = 0; i < size; i++) {+ const at = first + i;+ re[i] = at >= 0 && at < samples.length ? samples[at]! * window[i]! : 0;+ }+ fft(re, im);+ for (let b = 0; b + 1 < edges.length; b++) {+ const end = Math.max(edges[b]! + 1, edges[b + 1]!);+ let power = 0;+ for (let k = edges[b]!; k < end; k++) power += re[k]! * re[k]! + im[k]! * im[k]!;+ into[b]! += power / (size * size * (end - edges[b]!));+ }+}++/** Per-band log power, summed over channels so a polarity-flipped channel cannot cancel. */+export function bandFrames(signal: AudioSignal): BandFrames {+ const { sampleRate, channels } = signal;+ const hop = Math.max(1, Math.round(sampleRate / FRAMES_PER_SECOND));+ const size = 2 ** Math.round(Math.log2(sampleRate * 0.046));+ const window = Float64Array.from(+ { length: size },+ (_, i) => 0.5 - 0.5 * Math.cos((2 * Math.PI * i) / size),+ );+ const edges = bandEdges(sampleRate, size);+ const length = channels[0]?.length ?? 0;+ const bands: Float64Array[] = [];+ for (let center = 0; center < length; center += hop) {+ const power = new Float64Array(Math.max(0, edges.length - 1));+ for (const channel of channels) addFramePower(channel, center, window, edges, power);+ bands.push(power.map((p) => Math.log1p(1e9 * p)));+ }+ return { fps: sampleRate / hop, bands };+}++/** Subtract the mean of the surrounding ±`radius` frames and keep what rises above it. */+function aboveLocalMean(curve: Float64Array, radius: number): Float64Array {+ const sums = new Float64Array(curve.length + 1);+ for (let i = 0; i < curve.length; i++) sums[i + 1] = sums[i]! + curve[i]!;+ return curve.map((value, i) => {+ const from = Math.max(0, i - radius);+ const to = Math.min(curve.length, i + radius + 1);+ return Math.max(0, value - (sums[to]! - sums[from]!) / (to - from));+ });+}++function standardized(curve: Float64Array): Float64Array {+ const mean = curve.reduce((a, b) => a + b, 0) / (curve.length || 1);+ const spread = Math.sqrt(curve.reduce((a, v) => a + (v - mean) ** 2, 0) / (curve.length || 1));+ return spread > 0 ? curve.map((v) => v / spread) : curve.map(() => 0);+}++interface OnsetCurves {+ fps: number;+ /** Rectified rise summed over every band. */+ broad: Float64Array;+ /** Rectified rise of the bands under 150 Hz, where kick drums and bass mark the bar. */+ low: Float64Array;+}++function onsetCurves({ fps, bands }: BandFrames, sampleRate: number): OnsetCurves {+ const size = 2 ** Math.round(Math.log2(sampleRate * 0.046));+ const lowBands = bandEdges(sampleRate, size).filter(+ (bin) => (bin * sampleRate) / size < LOW_BAND_HZ,+ ).length;+ const broad = new Float64Array(bands.length);+ const low = new Float64Array(bands.length);+ for (let f = 1; f < bands.length; f++) {+ const frame = bands[f]!;+ const previous = bands[f - 1]!;+ for (let b = 0; b < frame.length; b++) {+ const rise = Math.max(0, frame[b]! - previous[b]!);+ broad[f]! += rise;+ if (b < lowBands) low[f]! += rise;+ }+ }+ const radius = Math.round(0.4 * fps);+ return {+ fps,+ broad: standardized(aboveLocalMean(broad, radius)),+ low: standardized(aboveLocalMean(low, radius)),+ };+}++/** Autocorrelation of `curve` at lags 0…`maxLag`, normalized by lag 0. */+function autocorrelation(curve: Float64Array, maxLag: number): Float64Array {+ const mean = curve.reduce((a, b) => a + b, 0) / (curve.length || 1);+ const centred = curve.map((v) => v - mean);+ const ac = new Float64Array(maxLag + 1);+ for (let lag = 0; lag <= maxLag && lag < centred.length; lag++) {+ let sum = 0;+ for (let i = lag; i < centred.length; i++) sum += centred[i]! * centred[i - lag]!;+ ac[lag] = sum;+ }+ const zero = ac[0] || 1;+ return ac.map((v) => v / zero);+}++const tempoPrior = (lag: number, fps: number) =>+ Math.exp(-0.5 * Math.log2(60 / (lag / fps) / 120) ** 2);++interface LagRange {+ shortest: number;+ longest: number;+}++const fullRange = (fps: number): LagRange => ({+ shortest: Math.max(1, Math.floor((60 / FASTEST_BPM) * fps)),+ longest: Math.ceil((60 / SLOWEST_BPM) * fps),+});++/** The beat period in frames: prior-weighted autocorrelation plus half its value at twice the lag. */+function beatPeriod(+ curve: Float64Array,+ fps: number,+ { shortest, longest }: LagRange = fullRange(fps),+): { period: number; clarity: number } {+ const ac = autocorrelation(curve, 2 * longest + 1);+ const at = (lag: number) => ac[Math.min(ac.length - 1, Math.max(0, lag))]!;+ const score = (lag: number) =>+ tempoPrior(lag, fps) *+ (at(lag) + 0.5 * at(2 * lag) + 0.25 * (at(2 * lag - 1) + at(2 * lag + 1)));+ let best = shortest;+ for (let lag = shortest; lag <= longest; lag++) if (score(lag) > score(best)) best = lag;+ const [a, b, c] = [score(best - 1), score(best), score(best + 1)];+ const curvature = a - 2 * b + c;+ const shift = curvature ? Math.max(-0.5, Math.min(0.5, (0.5 * (a - c)) / curvature)) : 0;+ return { period: best + shift, clarity: Math.max(0, Math.min(1, at(best))) };+}++/** The onset curve smoothed by a Gaussian a thirty-second of a beat wide. */+function smoothed(curve: Float64Array, period: number): Float64Array {+ const half = Math.max(1, Math.round(period / 8));+ const kernel = Array.from({ length: 2 * half + 1 }, (_, i) =>+ Math.exp(-0.5 * (((i - half) * 32) / period) ** 2),+ );+ return curve.map((_, i) => {+ let sum = 0;+ for (let j = 0; j < kernel.length; j++) sum += kernel[j]! * (curve[i + j - half] ?? 0);+ return sum;+ });+}++/** Ellis's recursion: each frame's best score from a predecessor about one period back, penalized by log deviation. */+function cumulativeScore(local: Float64Array, periodAt: (frame: number) => number) {+ const score = new Float64Array(local.length);+ const back = new Int32Array(local.length).fill(-1);+ for (let i = 0; i < local.length; i++) {+ const period = periodAt(i);+ let best = -Infinity;+ let from = -1;+ for (let j = Math.max(0, Math.round(i - 2 * period)); j <= i - Math.round(period / 2); j++) {+ const candidate = score[j]! - TIGHTNESS * Math.log((i - j) / period) ** 2;+ if (candidate > best) [best, from] = [candidate, j];+ }+ score[i] = local[i]! + Math.max(0, best);+ back[i] = best > 0 ? from : -1;+ }+ return { score, back };+}++/** The last local maximum of the cumulative score that reaches half the median of all of them. */+function lastBeat(score: Float64Array): number {+ const maxima: number[] = [];+ for (let i = 1; i + 1 < score.length; i++)+ if (score[i]! > score[i - 1]! && score[i]! >= score[i + 1]!) maxima.push(i);+ if (!maxima.length) return score.length - 1;+ const median = maxima.map((i) => score[i]!).sort((a, b) => a - b)[maxima.length >> 1]!;+ for (let k = maxima.length - 1; k >= 0; k--)+ if (score[maxima[k]!]! >= 0.5 * median) return maxima[k]!;+ return maxima[maxima.length - 1]!;+}++/** Drop beats at either end whose onset strength is under half the RMS of the beats' strengths. */+function trimmed(beats: number[], local: Float64Array): number[] {+ const rms = Math.sqrt(beats.reduce((a, b) => a + local[b]! ** 2, 0) / (beats.length || 1));+ let first = 0;+ let end = beats.length;+ while (first < end && local[beats[first]!]! < 0.5 * rms) first++;+ while (end > first && local[beats[end - 1]!]! < 0.5 * rms) end--;+ return beats.slice(first, end);+}++const LOCAL_WINDOW_SECONDS = 8;++/** The period through the piece: eight-second windows a second apart, held near the global period, median-smoothed. */+function localPeriods(curve: Float64Array, fps: number, period: number): (frame: number) => number {+ const window = Math.round(LOCAL_WINDOW_SECONDS * fps);+ const hop = Math.round(fps);+ const range = {+ shortest: Math.max(fullRange(fps).shortest, Math.floor(period * 0.7)),+ longest: Math.min(fullRange(fps).longest, Math.ceil(period * 1.4)),+ };+ const centres: number[] = [];+ const periods: number[] = [];+ for (let start = 0; start === 0 || start + window <= curve.length; start += hop) {+ const end = Math.min(curve.length, start + window);+ periods.push(beatPeriod(curve.subarray(start, end), fps, range).period);+ centres.push((start + end) / 2);+ }+ const median = periods.map((_, i) => {+ const near = periods.slice(Math.max(0, i - 2), i + 3).sort((a, b) => a - b);+ return near[near.length >> 1]!;+ });+ return (frame) => interpolate(centres, median, frame);+}++function interpolate(xs: number[], ys: number[], x: number): number {+ if (x <= xs[0]!) return ys[0]!;+ for (let i = 1; i < xs.length; i++)+ if (x <= xs[i]!) {+ const t = (x - xs[i - 1]!) / (xs[i]! - xs[i - 1]!);+ return ys[i - 1]! * (1 - t) + ys[i]! * t;+ }+ return ys[ys.length - 1]!;+}++function beatFrames(curve: Float64Array, fps: number, period: number) {+ const local = smoothed(curve, period);+ const { score, back } = cumulativeScore(local, localPeriods(curve, fps, period));+ const beats: number[] = [];+ for (let i = lastBeat(score); i >= 0; i = back[i]!) beats.push(i);+ return { frames: trimmed(beats.reverse(), local), local };+}++/** How much stronger the smoothed onsets are on the beats than on average, as a share: 0 is no stronger. */+function onsetLift(frames: number[], local: Float64Array): number {+ const overall = local.reduce((a, b) => a + b, 0) / (local.length || 1);+ const onBeats = frames.reduce((a, f) => a + local[f]!, 0) / (frames.length || 1);+ return overall > 0 ? onBeats / overall - 1 : 0;+}++/** The meter and bar phase whose beats carry the most low-band onset energy against the others. */+function barPhase(beats: number[], curves: OnsetCurves): { meter: number; phase: number } {+ const strength = (f: number) =>+ Math.max(+ ...[-1, 0, 1, 2].map((d) => (curves.low[f + d] ?? 0) + 0.3 * (curves.broad[f + d] ?? 0)),+ );+ const levels = beats.map(strength);+ let best = { meter: 4, phase: 0, contrast: -Infinity };+ for (const meter of [4, 3])+ for (let phase = 0; phase < meter; phase++) {+ const on = levels.filter((_, k) => (k - phase) % meter === 0);+ const off = levels.filter((_, k) => (k - phase) % meter !== 0);+ const mean = (v: number[]) => v.reduce((a, b) => a + b, 0) / (v.length || 1);+ const contrast = (mean(on) - mean(off)) * (meter === 3 ? 0.85 : 1);+ if (contrast > best.contrast) best = { meter, phase, contrast };+ }+ return best;+}++export function medianInterval(times: readonly number[]): number | null {+ const gaps = times+ .slice(1)+ .map((t, i) => t - times[i]!)+ .sort((a, b) => a - b);+ return gaps.length ? gaps[gaps.length >> 1]! : null;+}++const noRhythm = (confidence: number, reason = "no steady beat"): Rhythm => ({+ beats: { times: [], confidence, reason },+ downbeats: { times: [], meter: null, confidence: "low" },+});++/** Two bars of 4/4 at 80 BPM: anything shorter has too few beats to call steady. */+const SHORTEST_SECONDS = 6;++/** Below this the beats are more likely the tracker's own grid than the music's. */+export const STEADY_BEAT_CONFIDENCE = 0.05;++/** Clarity of the period, scaled down when the beats are under three times as strong as the onsets around them. */+const beatConfidence = (clarity: number, lift: number) =>+ round(Math.max(0, Math.min(1, clarity)) * Math.max(0, Math.min(1, lift / 3)));++/** Beats and downbeats of `frames`, the band energies of a signal at `sampleRate`. */+export function rhythmOf(frames: BandFrames, sampleRate: number): Rhythm {+ const curves = onsetCurves(frames, sampleRate);+ const { period, clarity } = beatPeriod(curves.broad, curves.fps);+ if (curves.broad.length < SHORTEST_SECONDS * curves.fps)+ return noRhythm(0, "too short for a steady beat");+ if (!curves.broad.some((v) => v > 0) || curves.broad.length < 2 * period) return noRhythm(0);+ const { frames: beatList, local } = beatFrames(curves.broad, curves.fps, period);+ const confidence = beatConfidence(clarity, onsetLift(beatList, local));+ if (beatList.length < 4 || confidence < STEADY_BEAT_CONFIDENCE) return noRhythm(confidence);+ const times = beatList.map((f) => round(f / curves.fps));+ const bar = beatList.length >= 6 ? barPhase(beatList, curves) : null;+ return {+ beats: { times, confidence },+ downbeats: {+ times: bar ? times.filter((_, k) => (k - bar.phase) % bar.meter === 0) : [],+ meter: bar?.meter ?? null,+ confidence: "low",+ },+ };+}++const TEMPO_STEP_SECONDS = 4;++/** Tempo from beat times, by whichever provider placed them: the median interval, and a curve over ±4 s windows. */+export function tempoOf(times: readonly number[]): Tempo {+ const interval = medianInterval(times);+ if (!interval) return { value: null, curve: [] };+ const bpm = (gap: number) => Math.round(600 / gap) / 10;+ const curve: TempoPoint[] = [];+ const last = times[times.length - 1]!;+ for (let t = times[0]!; t <= last; t += TEMPO_STEP_SECONDS) {+ const gap = medianInterval(times.filter((b) => Math.abs(b - t) <= TEMPO_STEP_SECONDS));+ if (gap) curve.push({ t: round(t), bpm: bpm(gap) });+ }+ return { value: bpm(interval), curve };+}++const measured = new WeakMap<AudioSignal, { frames: BandFrames; rhythm?: Rhythm }>();++/** Band energies of `signal`, computed once per signal for every capability that reads them. */+export function bandFramesOf(signal: AudioSignal): BandFrames {+ let entry = measured.get(signal);+ if (!entry) measured.set(signal, (entry = { frames: bandFrames(signal) }));+ return entry.frames;+}++export function rhythm(signal: AudioSignal): Rhythm {+ const frames = bandFramesOf(signal);+ const entry = measured.get(signal)!;+ return (entry.rhythm ??= rhythmOf(frames, signal.sampleRate));+}diff --git a/packages/core/src/audioSections.ts b/packages/core/src/audioSections.tsnew file mode 100644index 0000000000..334adb8d6e--- /dev/null+++ b/packages/core/src/audioSections.ts@@ -0,0 +1,139 @@+/**+ * Sections from signal processing: band energies averaged into half-second blocks, a self-similarity matrix, and+ * Foote's checkerboard novelty, whose peaks are where the music changes. Boundaries snap to the nearest beat.+ */++import type { AudioSignal } from "./audioAnalysis";+import { bandFramesOf, medianInterval, rhythm, type BandFrames } from "./audioRhythm";+import { snapper, type SnapReport } from "./audioSnap";++export interface Section {+ start: number;+ end: number;+ /** Sections that sound alike share a letter. */+ label: string;+}++export interface Sections {+ spans: Section[];+ snap?: SnapReport;+}++export interface SectionOptions {+ /** Shortest section, in seconds. */+ minSeconds?: number;+}++const BLOCK_SECONDS = 0.5;+/** Checkerboard half-width: eight seconds of music either side of a candidate boundary. */+const KERNEL_BLOCKS = 16;+const SAME_SECTION = 0.9;++const round = (t: number) => Math.round(t * 1000) / 1000;++function meanOf(frames: ArrayLike<number>[]): number[] {+ const sum = new Array<number>(frames[0]?.length ?? 0).fill(0);+ for (const frame of frames) for (let b = 0; b < sum.length; b++) sum[b]! += frame[b]!;+ return sum.map((v) => v / (frames.length || 1));+}++/** Each band moved to zero mean and unit spread over time, in place. */+function standardizeBands(blocks: number[][]): void {+ for (let b = 0; b < (blocks[0]?.length ?? 0); b++) {+ const column = blocks.map((block) => block[b]!);+ const mean = column.reduce((a, v) => a + v, 0) / column.length;+ const spread = Math.sqrt(column.reduce((a, v) => a + (v - mean) ** 2, 0) / column.length);+ for (const block of blocks) block[b] = (block[b]! - mean) / (spread || 1);+ }+}++/** Band energies averaged over blocks, each band standardized over time and each block scaled to unit length. */+function blockFeatures({ fps, bands }: BandFrames): number[][] {+ const per = Math.max(1, Math.round(BLOCK_SECONDS * fps));+ const blocks: number[][] = [];+ for (let start = 0; start + per <= bands.length; start += per)+ blocks.push(meanOf(bands.slice(start, start + per)));+ standardizeBands(blocks);+ return blocks.map(unit);+}++function unit(vector: number[]): number[] {+ const length = Math.hypot(...vector) || 1;+ return vector.map((v) => v / length);+}++const dot = (a: number[], b: number[]) => a.reduce((sum, v, i) => sum + v * b[i]!, 0);++/** The checkerboard kernel, offsets -KERNEL_BLOCKS…KERNEL_BLOCKS-1 each way: + within a side, − across, tapered. */+const CHECKERBOARD = (() => {+ const offsets = Array.from({ length: 2 * KERNEL_BLOCKS }, (_, k) => k - KERNEL_BLOCKS);+ const taper = (k: number) =>+ Math.exp(-0.5 * (((k < 0 ? -k - 1 : k) + 0.5) / (KERNEL_BLOCKS / 2)) ** 2);+ return offsets.map((a) => offsets.map((b) => (a < 0 === b < 0 ? 1 : -1) * taper(a) * taper(b)));+})();++/** Foote novelty at block `i`: similarity within each side of it minus similarity across it. */+function noveltyAt(blocks: number[][], i: number): number {+ let sum = 0;+ CHECKERBOARD.forEach((row, a) => {+ const x = blocks[i + a - KERNEL_BLOCKS];+ if (!x) return;+ row.forEach((weight, b) => {+ const y = blocks[i + b - KERNEL_BLOCKS];+ if (y) sum += weight * dot(x, y);+ });+ });+ return sum;+}++const novelty = (blocks: number[][]) => blocks.map((_, i) => noveltyAt(blocks, i));++/** Blocks where novelty peaks over its neighbourhood and clears the mean by a standard deviation. */+function boundaryBlocks(curve: number[], minBlocks: number): number[] {+ const mean = curve.reduce((a, b) => a + b, 0) / (curve.length || 1);+ const spread = Math.sqrt(curve.reduce((a, v) => a + (v - mean) ** 2, 0) / (curve.length || 1));+ const peaks: number[] = [];+ for (let i = minBlocks; i <= curve.length - minBlocks; i++) {+ const neighbourhood = curve.slice(Math.max(0, i - minBlocks), i + minBlocks + 1);+ if (curve[i]! < Math.max(...neighbourhood) || curve[i]! <= mean + spread) continue;+ if (!peaks.length || i - peaks[peaks.length - 1]! >= minBlocks) peaks.push(i);+ }+ return peaks;+}++/** A letter per span: the letter of the first earlier span whose mean features are this alike, else a new one. */+function labels(blocks: number[][], edges: number[]): string[] {+ const means: number[][] = [];+ const letters: string[] = [];+ for (let s = 0; s + 1 < edges.length; s++) {+ const part = blocks.slice(edges[s]!, edges[s + 1]!);+ const mean = unit(part[0]!.map((_, b) => part.reduce((a, block) => a + block[b]!, 0)));+ const match = means.findIndex((other) => dot(other, mean) >= SAME_SECTION);+ letters.push(+ match >= 0 ? letters[match]! : String.fromCharCode(65 + (new Set(letters).size % 26)),+ );+ means.push(mean);+ }+ return letters;+}++/** Half the median beat interval: a boundary moves to a beat, never past one. */+const beatTolerance = (beats: number[]) => (medianInterval(beats) ?? 0) / 2;++export function sections(signal: AudioSignal, options: SectionOptions = {}): Sections {+ const duration = round((signal.channels[0]?.length ?? 0) / signal.sampleRate);+ const blocks = blockFeatures(bandFramesOf(signal));+ const minBlocks = Math.max(1, Math.round((options.minSeconds ?? 4) / BLOCK_SECONDS));+ if (blocks.length < 2 * minBlocks) return { spans: [{ start: 0, end: duration, label: "A" }] };+ const edges = [0, ...boundaryBlocks(novelty(blocks), minBlocks), blocks.length];+ const letters = labels(blocks, edges);+ const beats = rhythm(signal).beats.times;+ const onBeat = snapper(beats, beatTolerance(beats), "signal beats");+ const times = edges.map((e, i) =>+ i === 0 ? 0 : i === edges.length - 1 ? duration : round(onBeat.snap(e * BLOCK_SECONDS)),+ );+ return {+ spans: letters.map((label, s) => ({ start: times[s]!, end: times[s + 1]!, label })),+ snap: onBeat.report(),+ };+}diff --git a/packages/core/src/audioSnap.ts b/packages/core/src/audioSnap.tsnew file mode 100644index 0000000000..55cc6ff5cd--- /dev/null+++ b/packages/core/src/audioSnap.ts@@ -0,0 +1,62 @@+/** Moving reported times onto measured ones, one at a time, and saying how far they moved. */++export interface SnapReport {+ /** What the times were moved onto. */+ reference: string;+ toleranceMs: number;+ /** Times moved onto a measured one. */+ snapped: number;+ /** Times with nothing measured within the tolerance, kept as reported. */+ unsnapped: number[];+ /** Median distance between a reported time and the measured one it moved to, in ms: low means they agreed. */+ medianShiftMs: number | null;+}++/** Index of the value in ascending `sorted` nearest `t`. */+function nearestIndex(sorted: readonly number[], t: number): number {+ let low = 0;+ let high = sorted.length - 1;+ while (low < high) {+ const mid = (low + high) >> 1;+ if (sorted[mid]! < t) low = mid + 1;+ else high = mid;+ }+ return low > 0 && t - sorted[low - 1]! <= sorted[low]! - t ? low - 1 : low;+}++export interface Snapper {+ snap: (t: number) => number;+ report: () => SnapReport;+}++/** Snaps each time to the nearest of `measured` (ascending) when one is within `toleranceSeconds`, else keeps it. */+export function snapper(+ measured: readonly number[],+ toleranceSeconds: number,+ reference: string,+): Snapper {+ const shifts: number[] = [];+ const unsnapped: number[] = [];+ return {+ snap(t) {+ const nearest = measured.length ? measured[nearestIndex(measured, t)]! : null;+ if (nearest === null || Math.abs(nearest - t) > toleranceSeconds) {+ unsnapped.push(t);+ return t;+ }+ shifts.push(Math.abs(nearest - t));+ return nearest;+ },+ report() {+ const sorted = [...shifts].sort((a, b) => a - b);+ const median = sorted.length ? sorted[sorted.length >> 1]! : null;+ return {+ reference,+ toleranceMs: Math.round(toleranceSeconds * 1000),+ snapped: shifts.length,+ unsnapped: [...unsnapped],+ medianShiftMs: median === null ? null : Math.round(median * 1000),+ };+ },+ };+}