Skip to content

Write HTML. Render video. Built for agents.

feat(cli): hyperframes audio measures where sounds land, in film time (#5602)

Xuanru Libb49858b8a4d
diff --git a/docs/packages/cli.mdx b/docs/packages/cli.mdx
index c186b36622..3b65380cc7 100644
--- a/docs/packages/cli.mdx
+++ b/docs/packages/cli.mdx
@@ -20,17 +20,17 @@ npm install -g hyperframes
## Find your command
-| You want to | Use |
-| ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| Start a project | [`init`](#init), [`add`](#add), [`catalog`](#catalog) |
-| Bring in source material | [`capture`](#capture), [`transcribe`](#transcribe), [`tts`](#tts), [`remove-background`](#remove-background), [`media-treatment`](#media-treatment), [`beats`](#beats), [`normalize-audio`](#normalize-audio) |
-| Look at it, or share it | [`preview`](#preview), [`present`](#present-and-play), [`play`](#present-and-play), [`publish`](#publish), [`open`](#open-and-catch-up), [`catch-up`](#open-and-catch-up) |
-| Find problems | [`lint`](#lint), [`check`](#check), [`snapshot`](#snapshot), [`keyframes`](#keyframes), [`compare`](#compare-and-grade-compare), [`grade-compare`](#compare-and-grade-compare) |
-| Edit from the terminal | [`timeline`](#timeline), [`history`](#history) |
-| Make a file | [`render`](#render), [`benchmark`](#benchmark) |
-| Render somewhere else | [`cloud`](#hyperframes-cloud), [`lambda`](#hyperframes-lambda), [`cloudrun`](#hyperframes-cloudrun) |
-| Fix your setup | [`doctor`](#doctor), [`browser`](#browser), [`upgrade`](#upgrade), [`docs`](#docs), [`info`](#info-and-compositions), [`compositions`](#info-and-compositions) |
-| Connect other tools | [`auth`](#hyperframes-auth), [`skills`](#skills), [`figma`](#figma-and-events), [`telemetry`](#telemetry), [`feedback`](#feedback) |
+| You want to | Use |
+| ------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Start a project | [`init`](#init), [`add`](#add), [`catalog`](#catalog) |
+| Bring in source material | [`capture`](#capture), [`transcribe`](#transcribe), [`tts`](#tts), [`remove-background`](#remove-background), [`media-treatment`](#media-treatment), [`beats`](#beats), [`audio`](#audio), [`normalize-audio`](#normalize-audio) |
+| Look at it, or share it | [`preview`](#preview), [`present`](#present-and-play), [`play`](#present-and-play), [`publish`](#publish), [`open`](#open-and-catch-up), [`catch-up`](#open-and-catch-up) |
+| Find problems | [`lint`](#lint), [`check`](#check), [`snapshot`](#snapshot), [`keyframes`](#keyframes), [`compare`](#compare-and-grade-compare), [`grade-compare`](#compare-and-grade-compare) |
+| Edit from the terminal | [`timeline`](#timeline), [`history`](#history) |
+| Make a file | [`render`](#render), [`benchmark`](#benchmark) |
+| Render somewhere else | [`cloud`](#hyperframes-cloud), [`lambda`](#hyperframes-lambda), [`cloudrun`](#hyperframes-cloudrun) |
+| Fix your setup | [`doctor`](#doctor), [`browser`](#browser), [`upgrade`](#upgrade), [`docs`](#docs), [`info`](#info-and-compositions), [`compositions`](#info-and-compositions) |
+| Connect other tools | [`auth`](#hyperframes-auth), [`skills`](#skills), [`figma`](#figma-and-events), [`telemetry`](#telemetry), [`feedback`](#feedback) |
`validate`, `inspect`, and `layout` still work but are
[deprecated](#deprecated-validate-inspect-and-layout). Use `check` instead.
@@ -504,6 +504,49 @@ Needs a local Chrome, the same one `render` uses. Run
within a frame or two — a different headless-Chrome audio sample rate can shift
a beat slightly.
+### `audio`
+
+Measure how sound moves in time, so a cut or an animation can land on it.
+
+```bash
+npx hyperframes audio sfx/whoosh.mp3 # one file, in file seconds
+npx hyperframes audio sfx/hit.wav --place-at 4.2 # not in the film yet: where it lands if it starts at 4.2 s
+npx hyperframes audio --element '#riser' # one clip of the film, in film seconds
+npx hyperframes audio --at 12 # every sound playing at 12 s
+npx hyperframes audio --range 10,14 --only onsets # every hit between 10 and 14 s
+```
+
+Each sound reports:
+
+- **moments**: where it starts (RMS first reaches 20% of its peak), where it
+ rises fastest, its loudest point, and where its tail fades 40 dB under the peak.
+ A short impact accents on its peak; a slow whoosh accents on its steepest rise.
+- **onsets**: every hit, with a strength relative to the strongest.
+- **loudness**: integrated and short-term-max LUFS (ITU-R BS.1770, matching
+ FFmpeg's `ebur128` to 0.1 LU), sample and true peak, clipping.
+- **silence**: quiet spans under `--silence-db` (−50 dBFS).
+- **kind**: a first guess of `sfx`, `voice` or `music`, with the features behind it.
+- **envelope**: the level over time, about 2,000 values per sound.
+
+A film query (`--element`, `--at`, `--range`) reads the project in
+the current directory or `--dir`, finds the `<audio>` and `<video>` clips it
+covers — nested compositions included — and analyzes only the part of each file
+that plays. `--json` returns times in file seconds with each clip's placement
+(`start`, `mediaStart`, `playbackRate`, and `dataStart`, the element's own
+`data-start`, local to its composition), so film time is
+`start + (t − mediaStart) / playbackRate`; the text output prints film time.
+
+For a file not in the film yet, `--place-at <s>` answers as if it started at that
+film second; `--skip-file-seconds <s>` cuts that much from the start of the file
+first, and the result warns when the cut part held the file's loudest moment.
+
+Every measurement comes from a provider ladder. Today the only provider is
+local signal processing; `--quality fast|local|best` limits which providers may
+run, `--provider onsets=signal` pins one, and `--full` shows each ladder step.
+Tune the analysis with `--onset-method`, `--threshold`, `--band lo,hi`,
+`--hop-ms`, `--onset-ratio`, `--tail-db` and `--silence-db`. Decoding needs
+FFmpeg.
+
## Look at it
### `normalize-audio`
diff --git a/packages/cli/README.md b/packages/cli/README.md
index ba8f0be874..925cb9e6b2 100644
--- a/packages/cli/README.md
+++ b/packages/cli/README.md
@@ -47,6 +47,16 @@ previews with `--status`, `--stop`, `--list`, and `--kill-all`. Add `--json` to
managed lifecycle commands for machine-readable output. `--foreground --json`
prints the ready-session envelope once, then remains attached until stopped.
+### `audio`
+
+Measure where sounds land: onsets, the moment each sound peaks, loudness and
+silence. Give files, or query the film's own clips in film time:
+
+```bash
+npx hyperframes audio sfx/whoosh.mp3
+npx hyperframes audio --at 12 --json
+```
+
### `normalize-audio`
Measure two local authored audio clips with integrated LUFS and match the target
diff --git a/packages/cli/src/audio/analysis/analyze.test.ts b/packages/cli/src/audio/analysis/analyze.test.ts
new file mode 100644
index 0000000000..37847f690a
--- /dev/null
+++ b/packages/cli/src/audio/analysis/analyze.test.ts
@@ -0,0 +1,54 @@
+import { describe, expect, it } from "vitest";
+import {
+ signalProvider,
+ type AudioAnalysisProvider,
+ type AudioSignal,
+} from "@hyperframes/core/audio-analysis";
+import { analyzeSignal, envelopeHopMs } from "./analyze.js";
+
+const RATE = 8000;
+const tone: AudioSignal = {
+ sampleRate: RATE,
+ channels: [Float32Array.from({ length: RATE }, (_, i) => (i > RATE / 2 ? 0.5 * Math.sin(i) : 0))],
+};
+
+describe("analyzeSignal", () => {
+ it("reports times in file seconds when the signal starts partway into the file", async () => {
+ const analysis = await analyzeSignal(tone, {
+ capabilities: ["moments", "envelope"],
+ fileOffset: 2,
+ });
+ expect((analysis.values.moments as { onset: number }).onset).toBeCloseTo(2.5, 2);
+ expect(analysis.values.envelope).toMatchObject({ startsAt: 2 });
+ });
+
+ it("follows a provider pinned to a capability and says which one answered", async () => {
+ const fixed: AudioAnalysisProvider<AudioSignal> = {
+ id: "fixed",
+ local: true,
+ capabilities: ["moments"],
+ analyze: async () => ({ onset: 0.1, steepest: 0.1, peak: 0.1, tailEnd: 0.2 }),
+ };
+ const analysis = await analyzeSignal(
+ tone,
+ { capabilities: ["moments", "loudness"], provider: { moments: "fixed" }, fileOffset: 1 },
+ [signalProvider, fixed],
+ );
+ expect(analysis.provenance).toEqual({ moments: "fixed", loudness: "signal" });
+ expect(analysis.values.moments).toEqual({ onset: 1.1, steepest: 1.1, peak: 1.1, tailEnd: 1.2 });
+ });
+
+ it("warns, and answers nothing, for a capability no provider offers yet", async () => {
+ const analysis = await analyzeSignal(tone, { capabilities: ["beats"] });
+ expect(analysis.values.beats).toBeNull();
+ expect(analysis.warnings).toEqual(["beats: no provider answered (none offers it)"]);
+ });
+});
+
+describe("envelopeHopMs", () => {
+ it("keeps short sounds at 5 ms and coarsens long ones to about 2,000 values", () => {
+ expect(envelopeHopMs(1)).toBe(5);
+ expect(envelopeHopMs(180)).toBe(90);
+ expect(envelopeHopMs(180, 20)).toBe(20);
+ });
+});
diff --git a/packages/cli/src/audio/analysis/analyze.ts b/packages/cli/src/audio/analysis/analyze.ts
new file mode 100644
index 0000000000..2f4b2bf5f6
--- /dev/null
+++ b/packages/cli/src/audio/analysis/analyze.ts
@@ -0,0 +1,150 @@
+import {
+ runLadder,
+ signalDuration,
+ signalProvider,
+ type AnalysisQuality,
+ type AudioAnalysisProvider,
+ type AudioCapability,
+ type AudioSignal,
+ type LadderStep,
+} from "@hyperframes/core/audio-analysis";
+
+/** What `hyperframes audio` asks for when `--only` is not given. */
+export const DEFAULT_CAPABILITIES: readonly AudioCapability[] = [
+ "kind",
+ "moments",
+ "onsets",
+ "loudness",
+ "silence",
+ "envelope",
+];
+
+/** Every provider the CLI knows, in default ladder order. Model and service providers join this list. */
+function audioAnalysisProviders(): AudioAnalysisProvider<AudioSignal>[] {
+ return [signalProvider];
+}
+
+export interface AnalyzeOptions {
+ capabilities: readonly AudioCapability[];
+ quality?: AnalysisQuality;
+ /** Pin a capability to one provider, skipping the ladder. */
+ provider?: Partial<Record<AudioCapability, string>>;
+ /** Per-capability options handed to the provider. */
+ options?: Partial<Record<AudioCapability, Record<string, unknown>>>;
+ /** Seconds into the file where the signal begins; every reported time is shifted by it. */
+ fileOffset?: number;
+ /** Seconds of signal around the analyzed window, there so onsets at its edges are told from the cut. */
+ context?: { before: number; after: number };
+}
+
+export interface Analysis {
+ /** Seconds of signal analyzed. */
+ analyzed: number;
+ values: Partial<Record<AudioCapability, unknown>>;
+ provenance: Partial<Record<AudioCapability, string | null>>;
+ ladder: Partial<Record<AudioCapability, LadderStep[]>>;
+ warnings: string[];
+}
+
+/** Long signals get a coarser envelope, so one item stays near 2,000 values. */
+export function envelopeHopMs(seconds: number, asked?: number): number {
+ if (asked !== undefined) return asked;
+ return Math.max(5, Math.ceil((seconds * 1000) / 2000));
+}
+
+const round = (t: number) => Math.round(t * 1000) / 1000;
+
+/** Move a capability's times from signal seconds to file seconds. */
+function toFileTime(capability: AudioCapability, value: unknown, offset: number): unknown {
+ if (value === null || typeof value !== "object") return value;
+ const at = (t: number) => round(t + offset);
+ switch (capability) {
+ case "moments": {
+ const m = value as Record<string, number>;
+ return Object.fromEntries(Object.entries(m).map(([k, t]) => [k, at(t)]));
+ }
+ case "onsets": {
+ const o = value as { events: { t: number; strength: number }[] };
+ return { ...o, events: o.events.map((e) => ({ ...e, t: at(e.t) })) };
+ }
+ case "silence":
+ return (value as { start: number; end: number }[]).map((s) => ({
+ start: at(s.start),
+ end: at(s.end),
+ }));
+ case "loudness": {
+ const l = value as { clipping: { count: number; firstAt: number | null } };
+ const firstAt = l.clipping.firstAt === null ? null : at(l.clipping.firstAt);
+ return { ...l, clipping: { ...l.clipping, firstAt } };
+ }
+ case "envelope":
+ return { startsAt: round(offset), ...(value as object) };
+ default:
+ return value;
+ }
+}
+
+function within(value: unknown, from: number, to: number): unknown {
+ if (!value || typeof value !== "object" || !("events" in value)) return value;
+ const onsets = value as { events: { t: number }[] };
+ return { ...onsets, events: onsets.events.filter((e) => e.t >= from && e.t < to) };
+}
+
+interface Framing {
+ /** The signal without its context, which every capability but onsets measures. */
+ windowed: AudioSignal;
+ /** Seconds of context before the window, and where the window ends, in signal seconds. */
+ before: number;
+ end: number;
+ cut: boolean;
+}
+
+function frame(signal: AudioSignal, context: AnalyzeOptions["context"]): Framing {
+ const before = context?.before ?? 0;
+ const skip = Math.round(before * signal.sampleRate);
+ const length = signal.channels[0]?.length ?? 0;
+ const end = Math.max(skip, length - Math.round((context?.after ?? 0) * signal.sampleRate));
+ const cut = skip > 0 || end < length;
+ const windowed = cut
+ ? { ...signal, channels: signal.channels.map((c) => c.subarray(skip, end)) }
+ : signal;
+ return { windowed, before, end: end / signal.sampleRate, cut };
+}
+
+function optionsFor(capability: AudioCapability, request: AnalyzeOptions, seconds: number) {
+ const options = { ...request.options?.[capability] };
+ if (capability === "envelope")
+ options.hopMs = envelopeHopMs(seconds, options.hopMs as number | undefined);
+ return options;
+}
+
+export async function analyzeSignal(
+ signal: AudioSignal,
+ request: AnalyzeOptions,
+ providers: readonly AudioAnalysisProvider<AudioSignal>[] = audioAnalysisProviders(),
+): Promise<Analysis> {
+ const framing = frame(signal, request.context);
+ const analysis: Analysis = {
+ analyzed: round(signalDuration(framing.windowed)),
+ values: {},
+ provenance: {},
+ ladder: {},
+ warnings: [],
+ };
+ for (const capability of request.capabilities) {
+ // Onsets see the context, so a window's edge is not mistaken for a hit, then keep only the window's own.
+ const withContext = capability === "onsets" && framing.cut;
+ const result = await runLadder(capability, withContext ? signal : framing.windowed, providers, {
+ quality: request.quality,
+ provider: request.provider?.[capability],
+ options: optionsFor(capability, request, analysis.analyzed),
+ });
+ const value = withContext ? within(result.value, framing.before, framing.end) : result.value;
+ const offset = (request.fileOffset ?? 0) - (withContext ? framing.before : 0);
+ analysis.values[capability] = toFileTime(capability, value, offset);
+ analysis.provenance[capability] = result.provenance;
+ analysis.ladder[capability] = result.steps;
+ analysis.warnings.push(...result.warnings);
+ }
+ return analysis;
+}
diff --git a/packages/cli/src/audio/analysis/decode.ts b/packages/cli/src/audio/analysis/decode.ts
new file mode 100644
index 0000000000..ef86377d24
--- /dev/null
+++ b/packages/cli/src/audio/analysis/decode.ts
@@ -0,0 +1,103 @@
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { findFfBinary } from "@hyperframes/parsers/ff-binaries";
+import type { AudioSignal } from "@hyperframes/core/audio-analysis";
+
+const run = promisify(execFile);
+
+export interface AudioStreamInfo {
+ sampleRate: number;
+ channels: number;
+ duration: number;
+}
+
+function requiredBinary(name: "ffmpeg" | "ffprobe"): string {
+ const path = findFfBinary(name, { configuredMustExist: true });
+ if (!path) throw new Error(`${name} is required to analyze audio; run: npx hyperframes doctor`);
+ return path;
+}
+
+/** The first audio stream's format, or `null` when the file has no audio. */
+export async function probeAudio(file: string): Promise<AudioStreamInfo | null> {
+ const { stdout } = await run(
+ requiredBinary("ffprobe"),
+ [
+ "-v",
+ "error",
+ "-select_streams",
+ "a:0",
+ "-show_streams",
+ "-show_format",
+ "-of",
+ "json",
+ "--",
+ file,
+ ],
+ { maxBuffer: 8 * 1024 * 1024, windowsHide: true },
+ );
+ const parsed = JSON.parse(stdout) as {
+ streams?: { sample_rate?: string; channels?: number; duration?: string }[];
+ format?: { duration?: string };
+ };
+ const stream = parsed.streams?.[0];
+ if (!stream) return null;
+ return {
+ sampleRate: Number(stream.sample_rate),
+ channels: stream.channels ?? 1,
+ duration: Number(stream.duration ?? parsed.format?.duration ?? 0),
+ };
+}
+
+export interface DecodeWindow {
+ /** Seconds into the file. */
+ from?: number;
+ /** Seconds of file to read; the rest of the file when absent. */
+ seconds?: number;
+}
+
+export interface Decoded {
+ signal: AudioSignal;
+ stream: AudioStreamInfo;
+}
+
+/** Decode the first audio stream at its own sample rate, each channel kept apart; `null` when the file has none. */
+export async function decodeAudio(
+ file: string,
+ window: DecodeWindow = {},
+ probed?: AudioStreamInfo,
+): Promise<Decoded | null> {
+ const info = probed ?? (await probeAudio(file));
+ if (!info) return null;
+ const seek = window.from ? ["-ss", String(window.from)] : [];
+ const length = window.seconds === undefined ? [] : ["-t", String(window.seconds)];
+ const { stdout } = await run(
+ requiredBinary("ffmpeg"),
+ [
+ "-v",
+ "error",
+ ...seek,
+ "-i",
+ file,
+ ...length,
+ "-map",
+ "0:a:0",
+ "-f",
+ "f32le",
+ "-acodec",
+ "pcm_f32le",
+ "-",
+ ],
+ { encoding: "buffer", maxBuffer: 2 * 1024 * 1024 * 1024, windowsHide: true },
+ );
+ const bytes = stdout.byteLength - (stdout.byteLength % 4);
+ const interleaved = new Float32Array(
+ stdout.buffer.slice(stdout.byteOffset, stdout.byteOffset + bytes),
+ );
+ const frames = Math.floor(interleaved.length / info.channels);
+ const channels = Array.from({ length: info.channels }, (_, c) => {
+ const samples = new Float32Array(frames);
+ for (let i = 0; i < frames; i++) samples[i] = interleaved[i * info.channels + c]!;
+ return samples;
+ });
+ return { signal: { sampleRate: info.sampleRate, channels }, stream: info };
+}
diff --git a/packages/cli/src/audio/analysis/format.test.ts b/packages/cli/src/audio/analysis/format.test.ts
new file mode 100644
index 0000000000..94e13ba3ec
--- /dev/null
+++ b/packages/cli/src/audio/analysis/format.test.ts
@@ -0,0 +1,52 @@
+import { describe, expect, it } from "vitest";
+import { formatReport, sparkline } from "./format.js";
+import type { AudioItem } from "./run.js";
+
+const item = (fields: Partial<AudioItem>): AudioItem => ({
+ file: "sfx/hit.wav",
+ ref: "#hit",
+ element: "audio",
+ placement: { start: 10, end: 11, dataStart: 10, mediaStart: 0.5, playbackRate: 1, volume: 1 },
+ timebase: "file",
+ window: { from: 0.5, to: 1.5 },
+ duration: 1.5,
+ sampleRate: 48000,
+ channels: 2,
+ provenance: {},
+ warnings: [],
+ ...fields,
+});
+
+describe("sparkline", () => {
+ it("draws the quiet floor low and the loudest slice high, one character per slice", () => {
+ const line = sparkline({ hopMs: 10, scale: "db", values: [-80, -80, -40, -6, -3, -20] }, 6);
+ expect(line[0]).toBe("▁");
+ expect(line.at(-2)).toBe("█");
+ expect([...line]).toHaveLength(6);
+ });
+});
+
+describe("formatReport", () => {
+ it("prints file seconds as film seconds through the clip's placement", () => {
+ const text = formatReport({
+ version: 1,
+ items: [item({ moments: { onset: 0.6, steepest: 0.61, peak: 0.62, tailEnd: 1.2 } })],
+ });
+ expect(text).toContain("onset 10.100 steepest 10.110 peak 10.120 tail end 10.700");
+ });
+
+ it("caps the hit list and says how many more there are", () => {
+ const events = Array.from({ length: 15 }, (_, i) => ({ t: 0.5 + i * 0.05, strength: 1 }));
+ const text = formatReport({
+ version: 1,
+ items: [item({ onsets: { method: "spectral-flux", events } })],
+ });
+ expect(text).toMatch(/hits (\S+ ){12}\+3$/m);
+ });
+
+ it("shows an item's error in place of its measurements", () => {
+ expect(formatReport({ version: 1, items: [item({ error: "file not found" })] })).toBe(
+ "sfx/hit.wav #hit: file not found",
+ );
+ });
+});
diff --git a/packages/cli/src/audio/analysis/format.ts b/packages/cli/src/audio/analysis/format.ts
new file mode 100644
index 0000000000..ffc37a47ee
--- /dev/null
+++ b/packages/cli/src/audio/analysis/format.ts
@@ -0,0 +1,97 @@
+import type {
+ Envelope,
+ KindGuess,
+ Loudness,
+ Moments,
+ Onsets,
+ Span,
+} from "@hyperframes/core/audio-analysis";
+import type { AudioItem, AudioReport } from "./run.js";
+import { filmTime } from "./targets.js";
+
+const BARS = "▁▂▃▄▅▆▇█";
+const SPARK_WIDTH = 48;
+const ONSETS_SHOWN = 12;
+
+const s = (t: number) => t.toFixed(3);
+
+/** One character per slice of the envelope, from the floor of what it holds to its loudest. */
+export function sparkline(envelope: Envelope, width = SPARK_WIDTH): string {
+ const { values } = envelope;
+ if (values.length === 0) return "";
+ const per = Math.max(1, Math.ceil(values.length / width));
+ const slices: number[] = [];
+ for (let i = 0; i < values.length; i += per) slices.push(Math.max(...values.slice(i, i + per)));
+ const top = Math.max(...slices);
+ const floor = Math.max(Math.min(...slices), top - 60);
+ if (top - floor < 1e-9) return BARS[0]!.repeat(slices.length);
+ return slices
+ .map(
+ (v) => BARS[Math.round(((Math.max(v, floor) - floor) / (top - floor)) * (BARS.length - 1))],
+ )
+ .join("");
+}
+
+type At = (t: number) => string;
+
+function momentsLine(moments: Moments, at: At): string {
+ return ` onset ${at(moments.onset)} steepest ${at(moments.steepest)} peak ${at(moments.peak)} tail end ${at(moments.tailEnd)}`;
+}
+
+function hitsLine(onsets: Onsets, at: At): string | null {
+ if (!onsets.events.length) return null;
+ const shown = onsets.events
+ .slice(0, ONSETS_SHOWN)
+ .map((e) => `${at(e.t)}(${e.strength.toFixed(2)})`);
+ const more = onsets.events.length - shown.length;
+ return ` hits ${shown.join(" ")}${more > 0 ? ` +${more}` : ""}`;
+}
+
+function loudnessLine(loudness: Loudness, at: At): string {
+ const lufs = loudness.lufsIntegrated === null ? "below gate" : `${loudness.lufsIntegrated} LUFS`;
+ const { count, firstAt } = loudness.clipping;
+ const clip = count && firstAt !== null ? `, clips ${count}× from ${at(firstAt)}` : "";
+ return ` loudness ${lufs}, true peak ${loudness.truePeakDb} dBTP${clip}`;
+}
+
+function silenceLine(silence: Span[], at: At): string | null {
+ return silence.length
+ ? ` silent ${silence.map((g) => `${at(g.start)}–${at(g.end)}`).join(" ")}`
+ : null;
+}
+
+/** One line per measurement the item holds, in reading order. */
+const LINES: [string, (value: never, at: At) => string | null][] = [
+ ["moments", momentsLine],
+ ["onsets", hitsLine],
+ ["loudness", loudnessLine],
+ ["silence", silenceLine],
+ ["envelope", (envelope: Envelope) => ` ${sparkline(envelope)}`],
+];
+
+function headline(item: AudioItem, where: string): string {
+ const kind = item.kind as KindGuess | null | undefined;
+ const span = item.placement
+ ? `film ${s(item.placement.start)}–${s(item.placement.end)}, file from ${s(item.placement.mediaStart)}`
+ : `${s(item.duration ?? 0)} s`;
+ return `${where} ${span}${kind ? ` ${kind.value} (${kind.confidence})` : ""}`;
+}
+
+function formatItem(item: AudioItem): string[] {
+ const where = item.ref ? `${item.file} ${item.ref}` : item.file;
+ if (item.error) return [`${where}: ${item.error}`];
+ const at: At = (t) => s(item.placement ? filmTime(item.placement, t) : t);
+ const lines = [headline(item, where)];
+ for (const [capability, line] of LINES) {
+ const value = item[capability];
+ const text = value ? line(value as never, at) : null;
+ if (text) lines.push(text);
+ }
+ for (const warning of item.warnings) lines.push(` ! ${warning}`);
+ return lines;
+}
+
+export function formatReport(report: AudioReport): string {
+ if (report.items.length === 0) return "No audio matched.";
+ return report.items.map((item) => formatItem(item).join("\n")).join("\n\n");
+}
diff --git a/packages/cli/src/audio/analysis/run.test.ts b/packages/cli/src/audio/analysis/run.test.ts
new file mode 100644
index 0000000000..aaa72fc316
--- /dev/null
+++ b/packages/cli/src/audio/analysis/run.test.ts
@@ -0,0 +1,147 @@
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, describe, expect, it } from "vitest";
+import { findFfBinary } from "@hyperframes/parsers/ff-binaries";
+import type { Moments, Onsets } from "@hyperframes/core/audio-analysis";
+import { formatReport } from "./format.js";
+import { analyzeTargets, previewTarget } from "./run.js";
+
+const RATE = 48_000;
+const dir = mkdtempSync(join(tmpdir(), "hf-audio-run-"));
+afterAll(() => rmSync(dir, { recursive: true, force: true }));
+
+/** A stereo 32-bit float WAV of `seconds`, each sample from `fill`. */
+function wav(name: string, seconds: number, fill: (t: number) => number): string {
+ seed = 3;
+ const frames = Math.round(seconds * RATE);
+ const data = Buffer.alloc(frames * 8);
+ for (let i = 0; i < frames; i++) {
+ const v = fill(i / RATE);
+ data.writeFloatLE(v, i * 8);
+ data.writeFloatLE(v, i * 8 + 4);
+ }
+ const header = Buffer.alloc(44);
+ header.write("RIFF", 0);
+ header.writeUInt32LE(36 + data.length, 4);
+ header.write("WAVEfmt ", 8);
+ header.writeUInt32LE(16, 16);
+ header.writeUInt16LE(3, 20);
+ header.writeUInt16LE(2, 22);
+ header.writeUInt32LE(RATE, 24);
+ header.writeUInt32LE(RATE * 8, 28);
+ header.writeUInt16LE(8, 32);
+ header.writeUInt16LE(32, 34);
+ header.write("data", 36);
+ header.writeUInt32LE(data.length, 40);
+ const path = join(dir, name);
+ writeFileSync(path, Buffer.concat([header, data]));
+ return path;
+}
+
+let seed = 3; // every file starts the same noise, so no test depends on which ran before it
+const noise = () =>
+ ((seed = (seed * 1_103_515_245 + 12_345) % 2_147_483_648) / 2_147_483_648) * 2 - 1;
+const hit = (at: number) => (t: number) =>
+ t < at ? 0 : 0.8 * Math.exp(-(t - at) / 0.04) * noise();
+
+const ffmpeg = findFfBinary("ffmpeg") && findFfBinary("ffprobe");
+
+describe.skipIf(!ffmpeg)("analyzeTargets", () => {
+ it("reports a file's moments and hits in file seconds, with its format", async () => {
+ const path = wav("two-hits.wav", 2, (t) => hit(0.5)(t) + 0.5 * hit(1.3)(t));
+ const report = await analyzeTargets([await previewTarget(path, "two-hits.wav", null)], {
+ capabilities: ["moments", "onsets"],
+ });
+ const [item] = report.items;
+ expect(item).toMatchObject({ sampleRate: RATE, channels: 2, element: null, placement: null });
+ expect((item!.moments as Moments).onset).toBeCloseTo(0.5, 2);
+ const hits = (item!.onsets as Onsets).events.map((e) => e.t);
+ expect(hits.map((t) => Math.round(t * 100) / 100)).toEqual([0.5, 1.3]);
+ });
+
+ it("places a previewed file in the film and prints film seconds", async () => {
+ const path = wav("one-hit.wav", 1, hit(0.25));
+ const target = await previewTarget(path, "one-hit.wav", 4, 0.1);
+ expect(target.placement).toEqual({
+ start: 4,
+ end: 4.9,
+ dataStart: 4,
+ mediaStart: 0.1,
+ playbackRate: 1,
+ volume: 1,
+ });
+ const report = await analyzeTargets([target], { capabilities: ["moments"] });
+ expect((report.items[0]!.moments as Moments).onset).toBeCloseTo(0.25, 2);
+ expect(formatReport(report)).toMatch(/onset 4\.1[45]\d/);
+ });
+
+ it("warns when the seconds skipped from a placed file held its loudest moment, and refuses to skip all of it", async () => {
+ const path = wav("late-cut.wav", 1, hit(0.1));
+ const report = await analyzeTargets([await previewTarget(path, "late-cut.wav", 2, 0.3)], {
+ capabilities: ["moments"],
+ });
+ expect(report.items[0]!.warnings).toEqual([
+ "the loudest part of late-cut.wav is in the first 0.3 s, which is skipped",
+ ]);
+ const kept = await analyzeTargets([await previewTarget(path, "late-cut.wav", 2, 0.05)], {
+ capabilities: ["moments"],
+ });
+ expect(kept.items[0]!.warnings).toEqual([]);
+ await expect(previewTarget(path, "late-cut.wav", 2, 1)).rejects.toThrow(/leaves nothing/);
+ });
+
+ it("warns when the loudest hit is only inside the margin before the kept window", async () => {
+ const tone = (at: number, gain: number) => (t: number) =>
+ t < at || t >= at + 0.03 ? 0 : gain * Math.sin(2 * Math.PI * 1000 * t);
+ const path = wav("margin.wav", 2.4, (t) => tone(0.502, 0.8)(t) + tone(1.202, 0.25)(t));
+ const warned = async (skip: number) =>
+ (
+ await analyzeTargets([await previewTarget(path, "margin.wav", 4, skip)], {
+ capabilities: ["moments"],
+ })
+ ).items[0]!.warnings;
+ expect(await warned(0.55)).toEqual([
+ "the loudest part of margin.wav is in the first 0.55 s, which is skipped",
+ ]);
+ expect(await warned(0.65)).toEqual([
+ "the loudest part of margin.wav is in the first 0.65 s, which is skipped",
+ ]);
+ });
+
+ it("finds no hit where a window cuts into a sound that is already playing", async () => {
+ const chord = (t: number) =>
+ Array.from({ length: 40 }, (_, k) => Math.sin(2 * Math.PI * 110 * (k + 1) * t + k)).reduce(
+ (a, b) => a + b,
+ ) / 40;
+ const path = wav("bed.wav", 3, (t) => 0.9 * chord(t) + 0.125 * hit(2)(t));
+ const report = await analyzeTargets(
+ [
+ {
+ path,
+ file: "bed.wav",
+ ref: "#bed",
+ kind: "audio",
+ placement: null,
+ window: { from: 1, seconds: 1.5 },
+ },
+ ],
+ { capabilities: ["onsets"] },
+ );
+ const hits = (report.items[0]!.onsets as Onsets).events.map((e) => e.t);
+ expect(hits.some((t) => Math.abs(t - 2) < 0.01)).toBe(true);
+ expect(hits.filter((t) => Math.abs(t - 2) >= 0.01)).toEqual([]);
+ });
+
+ it("reports a file it cannot read on that item and goes on to the next", async () => {
+ const broken = join(dir, "notes.txt");
+ writeFileSync(broken, "not audio");
+ const good = wav("good.wav", 0.5, hit(0.1));
+ const report = await analyzeTargets(
+ [await previewTarget(broken, "notes.txt", null), await previewTarget(good, "good.wav", null)],
+ { capabilities: ["moments"] },
+ );
+ expect(report.items[0]!.error).toMatch(/^could not read: /);
+ expect(report.items[1]!.moments).not.toBeNull();
+ });
+});
diff --git a/packages/cli/src/audio/analysis/run.ts b/packages/cli/src/audio/analysis/run.ts
new file mode 100644
index 0000000000..e1c1272e6a
--- /dev/null
+++ b/packages/cli/src/audio/analysis/run.ts
@@ -0,0 +1,177 @@
+import { existsSync } from "node:fs";
+import {
+ envelope,
+ type AudioCapability,
+ type AudioSignal,
+ type LadderStep,
+} from "@hyperframes/core/audio-analysis";
+import { analyzeSignal, type AnalyzeOptions } from "./analyze.js";
+import { decodeAudio, probeAudio, type AudioStreamInfo } from "./decode.js";
+import type { AudioTarget, Placement } from "./targets.js";
+
+export interface AudioItem {
+ file: string;
+ ref: string | null;
+ /** The clip's element in the film; `null` for a file given by path. */
+ element: "audio" | "video" | null;
+ placement: Placement | null;
+ /** Times below are file seconds; film time = placement.start + (t − placement.mediaStart) / placement.playbackRate. */
+ timebase: "file";
+ window: { from: number; to: number } | null;
+ duration: number | null;
+ sampleRate: number | null;
+ channels: number | null;
+ provenance: Partial<Record<AudioCapability, string | null>>;
+ ladder?: Partial<Record<AudioCapability, LadderStep[]>>;
+ warnings: string[];
+ error?: string;
+ [capability: string]: unknown;
+}
+
+export interface AudioReport {
+ version: 1;
+ items: AudioItem[];
+}
+
+const round = (t: number) => Math.round(t * 1000) / 1000;
+
+/** Onsets at a window's edges need the audio just past them to tell a hit from the cut. */
+const EDGE_CONTEXT_SECONDS = 0.1;
+
+/** A file the film does not play yet, placed where the caller would put it. */
+export async function previewTarget(
+ path: string,
+ file: string,
+ at: number | null,
+ mediaStart = 0,
+): Promise<AudioTarget> {
+ if (at === null) return { path, file, ref: null, kind: "file", placement: null };
+ const stream = existsSync(path) ? await probeAudio(path) : null;
+ if (stream && mediaStart >= stream.duration)
+ throw new Error(
+ `skipping ${mediaStart} s leaves nothing of ${file}, which is ${round(stream.duration)} s long`,
+ );
+ const length = stream ? stream.duration - mediaStart : 0;
+ return {
+ path,
+ file,
+ ref: null,
+ kind: "file",
+ placement: {
+ start: at,
+ end: round(at + length),
+ dataStart: at,
+ mediaStart,
+ playbackRate: 1,
+ volume: 1,
+ },
+ window: { from: mediaStart, seconds: length },
+ };
+}
+
+const emptyItem = (target: AudioTarget): AudioItem => ({
+ file: target.file,
+ ref: target.ref,
+ element: target.kind === "file" ? null : target.kind,
+ placement: target.placement,
+ timebase: "file",
+ window: null,
+ duration: null,
+ sampleRate: null,
+ channels: null,
+ provenance: {},
+ warnings: [],
+});
+
+function unreadable(target: AudioTarget): string | null {
+ if (/^[a-z][a-z\d+.-]*:\/\//i.test(target.file)) return "remote sources are not analyzed";
+ if (!existsSync(target.path)) return "file not found";
+ return null;
+}
+
+/** The part of the file to decode: the window, with up to `EDGE_CONTEXT_SECONDS` either side the file has. */
+function decodeSpan(target: AudioTarget, stream: AudioStreamInfo) {
+ const from = target.window?.from ?? 0;
+ const seconds = target.window?.seconds ?? stream.duration - from;
+ const before = Math.min(from, EDGE_CONTEXT_SECONDS);
+ const after = Math.max(0, Math.min(EDGE_CONTEXT_SECONDS, stream.duration - from - seconds));
+ return {
+ from,
+ read: { from: from - before, seconds: seconds + before + after },
+ context: { before, after },
+ };
+}
+
+const loudest = (signal: AudioSignal) =>
+ Math.max(0, ...envelope(signal, { hopMs: 10, scale: "linear" }).values);
+
+/** The decoded span without the margin added so an onset at the window edge is not the cut. */
+function retained(signal: AudioSignal, before: number, after: number): AudioSignal {
+ const skip = Math.round(before * signal.sampleRate);
+ const length = signal.channels[0]?.length ?? 0;
+ const end = Math.max(skip, length - Math.round(after * signal.sampleRate));
+ if (!skip && end >= length) return signal;
+ return { ...signal, channels: signal.channels.map((channel) => channel.subarray(skip, end)) };
+}
+
+/** A file placed after cutting its head says so when the cut part held its loudest moment. */
+async function skippedPeak(target: AudioTarget, kept: AudioSignal, stream: AudioStreamInfo) {
+ const skipped = target.kind === "file" ? (target.window?.from ?? 0) : 0;
+ if (skipped <= 0) return [];
+ const head = await decodeAudio(target.path, { from: 0, seconds: skipped }, stream);
+ if (!head || loudest(head.signal) <= loudest(kept)) return [];
+ return [
+ `the loudest part of ${target.file} is in the first ${round(skipped)} s, which is skipped`,
+ ];
+}
+
+async function analyzeTarget(
+ target: AudioTarget,
+ request: AnalyzeOptions,
+ full: boolean,
+): Promise<AudioItem> {
+ const item = emptyItem(target);
+ const problem = unreadable(target);
+ if (problem) return { ...item, error: problem };
+ const stream = await probeAudio(target.path);
+ if (!stream) return { ...item, error: "no audio stream" };
+ const { from, read, context } = decodeSpan(target, stream);
+ const decoded = await decodeAudio(target.path, read, stream);
+ if (!decoded) return { ...item, error: "no audio stream" };
+ const analysis = await analyzeSignal(decoded.signal, { ...request, fileOffset: from, context });
+ return {
+ ...item,
+ window: { from: round(from), to: round(from + analysis.analyzed) },
+ duration: round(stream.duration),
+ sampleRate: stream.sampleRate,
+ channels: stream.channels,
+ ...analysis.values,
+ provenance: analysis.provenance,
+ ...(full ? { ladder: analysis.ladder } : {}),
+ warnings: [
+ ...analysis.warnings,
+ ...(await skippedPeak(
+ target,
+ retained(decoded.signal, context.before, context.after),
+ stream,
+ )),
+ ],
+ };
+}
+
+export async function analyzeTargets(
+ targets: readonly AudioTarget[],
+ request: AnalyzeOptions,
+ full = false,
+): Promise<AudioReport> {
+ const items: AudioItem[] = [];
+ for (const target of targets) {
+ try {
+ items.push(await analyzeTarget(target, request, full));
+ } catch (error) {
+ const reason = error instanceof Error ? error.message.split("\n")[0]! : String(error);
+ items.push({ ...emptyItem(target), error: `could not read: ${reason}` });
+ }
+ }
+ return { version: 1, items };
+}
diff --git a/packages/cli/src/audio/analysis/targets.test.ts b/packages/cli/src/audio/analysis/targets.test.ts
new file mode 100644
index 0000000000..cefc9b442f
--- /dev/null
+++ b/packages/cli/src/audio/analysis/targets.test.ts
@@ -0,0 +1,146 @@
+import { resolve } from "node:path";
+import { describe, expect, it } from "vitest";
+import type { ProjectTimeline, TimelineRow } from "../../timeline/describeProject.js";
+import { fileTime, filmTargets, filmTime, type Placement } from "./targets.js";
+
+function row(
+ fields: Partial<TimelineRow> & Pick<TimelineRow, "ref" | "kind" | "absStart" | "absEnd">,
+): TimelineRow {
+ return {
+ warnings: [],
+ id: fields.ref.replace(/^#/, ""),
+ label: null,
+ trackKind: fields.kind === "video" ? "video" : "audio",
+ start: fields.absStart,
+ duration: fields.absEnd - fields.absStart,
+ end: fields.absEnd,
+ file: "index.html",
+ index: 0,
+ nested: false,
+ host: null,
+ hostRow: null,
+ children: [],
+ trackIndex: 0,
+ src: "audio/a.wav",
+ sourceFile: null,
+ volume: null,
+ lanes: [],
+ playbackRate: null,
+ audioGroup: null,
+ role: null,
+ durationAuthored: true,
+ durationSource: "authored",
+ pendingReason: null,
+ laneError: null,
+ syncOffsetFrames: null,
+ mediaStart: 0,
+ ...fields,
+ };
+}
+
+const timeline: ProjectTimeline = {
+ duration: 20,
+ tracks: [
+ {
+ kind: "video",
+ rows: [
+ row({ ref: "#clip", kind: "video", absStart: 0, absEnd: 6, src: "clip.mp4" }),
+ row({ ref: "#silent", kind: "video", absStart: 0, absEnd: 6, src: "b.mp4", volume: 0 }),
+ ],
+ },
+ {
+ kind: "audio",
+ rows: [
+ row({ ref: "#score", kind: "audio", absStart: 0, absEnd: 20, src: "audio/score.wav" }),
+ row({
+ ref: "#hit",
+ kind: "audio",
+ absStart: 4.2,
+ absEnd: 5,
+ src: "sfx/hit.wav",
+ mediaStart: 0.1,
+ }),
+ row({
+ ref: "#fast",
+ kind: "audio",
+ absStart: 8,
+ absEnd: 10,
+ src: "../shared/riser.wav",
+ start: 2,
+ file: "compositions/scene.html",
+ playbackRate: 2,
+ mediaStart: 1,
+ }),
+ ],
+ },
+ ],
+};
+
+describe("film and file time", () => {
+ const placed: Placement = {
+ start: 8,
+ end: 10,
+ dataStart: 8,
+ mediaStart: 1,
+ playbackRate: 2,
+ volume: 1,
+ };
+
+ it("maps a file second to the film through the clip's start, in-point and rate, and back", () => {
+ expect(filmTime(placed, 3)).toBe(9);
+ expect(fileTime(placed, 9)).toBe(3);
+ });
+});
+
+describe("filmTargets", () => {
+ const ids = (query: Parameters<typeof filmTargets>[2]) =>
+ filmTargets("/p", timeline, query).map((t) => t.ref);
+
+ it("lists every clip that sounds, leaving out muted ones", () => {
+ expect(ids({ by: "all" })).toEqual(["#clip", "#score", "#hit", "#fast"]);
+ });
+
+ it("picks the clips playing at a second or in a range", () => {
+ expect(ids({ by: "at", t: 4.5 })).toEqual(["#clip", "#score", "#hit"]);
+ expect(ids({ by: "at", t: 5 })).toEqual(["#clip", "#score"]);
+ expect(ids({ by: "range", start: 5.5, end: 8.5 })).toEqual(["#clip", "#score", "#fast"]);
+ });
+
+ it("reads only the part of the file a range plays, through the in-point and rate", () => {
+ const [fast] = filmTargets("/p", timeline, { by: "range", start: 9, end: 12 }).filter(
+ (t) => t.ref === "#fast",
+ );
+ expect(fast!.window).toEqual({ from: 3, seconds: 2 });
+ });
+
+ it("resolves a source against the file that declares the clip, and keeps its local data-start", () => {
+ const [fast] = filmTargets("/p", timeline, { by: "element", ref: "#fast" });
+ expect(fast!.path).toBe(resolve("/p/shared/riser.wav"));
+ expect(fast!.placement).toEqual({
+ start: 8,
+ end: 10,
+ dataStart: 2,
+ mediaStart: 1,
+ playbackRate: 2,
+ volume: 1,
+ });
+ });
+
+ it("says when an element is missing or is not sound", () => {
+ expect(() => filmTargets("/p", timeline, { by: "element", ref: "#nope" })).toThrow(
+ /was not found/,
+ );
+ const withTitle: ProjectTimeline = {
+ ...timeline,
+ tracks: [
+ {
+ kind: "graphics",
+ rows: [row({ ref: "#title", kind: "div", absStart: 0, absEnd: 2, src: null })],
+ },
+ ],
+ };
+ expect(() => filmTargets("/p", withTitle, { by: "element", ref: "#title" })).toThrow(
+ /not an audio or video clip/,
+ );
+ });
+});
diff --git a/packages/cli/src/audio/analysis/targets.ts b/packages/cli/src/audio/analysis/targets.ts
new file mode 100644
index 0000000000..16884bc39e
--- /dev/null
+++ b/packages/cli/src/audio/analysis/targets.ts
@@ -0,0 +1,114 @@
+import { dirname, join, resolve } from "node:path";
+import type { ProjectTimeline, TimelineRow } from "../../timeline/describeProject.js";
+
+/** Where a file sits in the film: film time = start + (file time − mediaStart) / playbackRate. */
+export interface Placement {
+ start: number;
+ end: number;
+ /** The element's own `data-start`: local to its composition, so it differs from `start` inside a sub-composition. */
+ dataStart: number;
+ mediaStart: number;
+ playbackRate: number;
+ volume: number;
+}
+
+export interface AudioTarget {
+ /** Absolute path of the media file. */
+ path: string;
+ /** The path as the project or the caller wrote it. */
+ file: string;
+ ref: string | null;
+ kind: "audio" | "video" | "file";
+ placement: Placement | null;
+ /** The part of the file to analyze, in file seconds; the whole file when absent. */
+ window?: { from: number; seconds: number };
+}
+
+export type FilmQuery =
+ | { by: "all" }
+ | { by: "element"; ref: string }
+ | { by: "at"; t: number }
+ | { by: "range"; start: number; end: number };
+
+const round = (t: number) => Math.round(t * 1000) / 1000;
+
+function placementOf(row: TimelineRow): Placement {
+ return {
+ start: row.absStart,
+ end: row.absEnd,
+ dataStart: row.start,
+ mediaStart: row.mediaStart ?? 0,
+ playbackRate: row.playbackRate ?? 1,
+ volume: row.volume ?? 1,
+ };
+}
+
+export function filmTime(placement: Placement, fileTime: number): number {
+ return round(placement.start + (fileTime - placement.mediaStart) / placement.playbackRate);
+}
+
+export function fileTime(placement: Placement, filmTime: number): number {
+ return round(placement.mediaStart + (filmTime - placement.start) * placement.playbackRate);
+}
+
+const soundingRows = (timeline: ProjectTimeline) =>
+ timeline.tracks
+ .filter((track) => track.kind === "audio" || track.kind === "video")
+ .flatMap((track) => track.rows)
+ .filter((row) => row.src && row.volume !== 0 && /^(audio|video)$/.test(row.kind))
+ .sort((a, b) => a.absStart - b.absStart);
+
+function matches(row: TimelineRow, query: FilmQuery): boolean {
+ switch (query.by) {
+ case "all":
+ case "element":
+ return true;
+ case "at":
+ return row.absStart <= query.t && query.t < row.absEnd;
+ case "range":
+ return row.absStart < query.end && query.start < row.absEnd;
+ }
+}
+
+function targetOf(projectDir: string, row: TimelineRow, query: FilmQuery): AudioTarget {
+ const placement = placementOf(row);
+ const from = query.by === "range" ? Math.max(placement.start, query.start) : placement.start;
+ const to = query.by === "range" ? Math.min(placement.end, query.end) : placement.end;
+ const declaredIn = dirname(join(projectDir, row.file));
+ return {
+ path: resolve(declaredIn, row.src!),
+ file: row.src!,
+ ref: row.ref || null,
+ kind: row.kind as "audio" | "video",
+ placement,
+ window: {
+ from: fileTime(placement, from),
+ seconds: round((to - from) * placement.playbackRate),
+ },
+ };
+}
+
+/** The film's sounding clips a query covers, each with the part of its file that plays. */
+export function filmTargets(
+ projectDir: string,
+ timeline: ProjectTimeline,
+ query: FilmQuery,
+): AudioTarget[] {
+ if (query.by === "element") {
+ const id = query.ref.startsWith("#") ? query.ref.slice(1) : null;
+ const found = timeline.tracks
+ .flatMap((track) => track.rows)
+ .filter((row) => row.ref === query.ref || (id !== null && row.id === id));
+ if (found.length !== 1)
+ throw new Error(
+ `${query.ref} ${found.length ? `matches ${found.length} clips` : "was not found"}; use a ref from hyperframes timeline`,
+ );
+ const row = found[0]!;
+ if (!row.src || !/^(audio|video)$/.test(row.kind))
+ throw new Error(`${query.ref} is not an audio or video clip`);
+ return [targetOf(projectDir, row, query)];
+ }
+ return soundingRows(timeline)
+ .filter((row) => matches(row, query))
+ .map((row) => targetOf(projectDir, row, query));
+}
diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts
index 1ff2d2195d..d934e0c8ca 100644
--- a/packages/cli/src/cli.ts
+++ b/packages/cli/src/cli.ts
@@ -135,6 +135,7 @@ const commandLoaders = {
lint: () => import("./commands/lint.js").then((m) => m.default),
check: () => import("./commands/check.js").then((m) => m.default),
beats: () => import("./commands/beats.js").then((m) => m.default),
+ audio: () => import("./commands/audio.js").then((m) => m.default),
"normalize-audio": () => import("./commands/normalize-audio.js").then((m) => m.default),
inspect: () => import("./commands/inspect.js").then((m) => m.default),
keyframes: () => import("./commands/keyframes.js").then((m) => m.default),
diff --git a/packages/cli/src/commands/audio.ts b/packages/cli/src/commands/audio.ts
new file mode 100644
index 0000000000..ac1a92517a
--- /dev/null
+++ b/packages/cli/src/commands/audio.ts
@@ -0,0 +1,224 @@
+import { resolve } from "node:path";
+import { defineCommand } from "citty";
+import type {
+ AnalysisQuality,
+ AudioCapability,
+ OnsetMethod,
+} from "@hyperframes/core/audio-analysis";
+import type { Example } from "./_examples.js";
+import { DEFAULT_CAPABILITIES, type AnalyzeOptions } from "../audio/analysis/analyze.js";
+import { formatReport } from "../audio/analysis/format.js";
+import { analyzeTargets, previewTarget } from "../audio/analysis/run.js";
+import { filmTargets, type AudioTarget, type FilmQuery } from "../audio/analysis/targets.js";
+import { describeProject } from "../timeline/describeProject.js";
+import { CliUsageError } from "../utils/commandResult.js";
+import { ensureDOMParser } from "../utils/dom.js";
+import { resolveProject } from "../utils/project.js";
+import { assertKnownFlags } from "../utils/reject-unknown-flags.js";
+import { withMeta } from "../utils/updateCheck.js";
+
+export const examples: Example[] = [
+ ["Where a sound effect lands, its hits and loudness", "hyperframes audio sfx/whoosh.mp3"],
+ [
+ "Where a file not in the film yet would land if it started at 4.2 s",
+ "hyperframes audio sfx/hit.wav --place-at 4.2",
+ ],
+ ["Every sound playing at 12 s in the film", "hyperframes audio --at 12"],
+ ["One clip of the film, as JSON", "hyperframes audio --element '#riser' --json"],
+ [
+ "Only onsets, from 60–200 Hz, finer",
+ "hyperframes audio drums.wav --only onsets --band 60,200 --hop-ms 2",
+ ],
+];
+
+const CAPABILITIES = new Set<string>([
+ ...DEFAULT_CAPABILITIES,
+ "beats",
+ "sections",
+ "words",
+ "describe",
+]);
+const QUALITIES = new Set<AnalysisQuality>(["best", "local", "fast"]);
+
+function number(name: string, raw: string | undefined): number | undefined {
+ if (raw === undefined) return undefined;
+ const value = Number(raw);
+ if (!Number.isFinite(value)) throw new CliUsageError(`--${name} must be a number, got ${raw}`);
+ return value;
+}
+
+function pair(name: string, raw: string | undefined): [number, number] | undefined {
+ if (raw === undefined) return undefined;
+ const parts = raw.split(",").map(Number);
+ if (parts.length !== 2 || parts.some((n) => !Number.isFinite(n)) || parts[0]! >= parts[1]!)
+ throw new CliUsageError(`--${name} takes two increasing numbers, like 2.5,4`);
+ return parts as [number, number];
+}
+
+function capabilities(raw: string | undefined): AudioCapability[] {
+ if (!raw) return [...DEFAULT_CAPABILITIES];
+ const asked = raw
+ .split(",")
+ .map((c) => c.trim())
+ .filter(Boolean);
+ const unknown = asked.filter((c) => !CAPABILITIES.has(c));
+ if (unknown.length)
+ throw new CliUsageError(
+ `--only does not know ${unknown.join(", ")}; use ${[...CAPABILITIES].join(", ")}`,
+ );
+ return asked as AudioCapability[];
+}
+
+function pins(raw: string | undefined): AnalyzeOptions["provider"] {
+ if (!raw) return undefined;
+ return Object.fromEntries(
+ raw.split(",").map((entry) => {
+ const [capability, id] = entry.split("=").map((s) => s.trim());
+ if (!capability || !id || !CAPABILITIES.has(capability))
+ throw new CliUsageError(`--provider takes capability=provider pairs, like onsets=signal`);
+ return [capability, id];
+ }),
+ );
+}
+
+function filmQuery(args: AudioFlags): FilmQuery | null {
+ const given = (["element", "at", "range"] as const).filter((k) => args[k] !== undefined);
+ if (given.length > 1) throw new CliUsageError(`use one of --element, --at, --range`);
+ if (args.element) return { by: "element", ref: args.element };
+ if (args.at !== undefined) return { by: "at", t: number("at", args.at)! };
+ const range = pair("range", args.range);
+ return range ? { by: "range", start: range[0], end: range[1] } : null;
+}
+
+interface AudioFlags {
+ dir?: string;
+ element?: string;
+ at?: string;
+ range?: string;
+ "place-at"?: string;
+ "skip-file-seconds"?: string;
+ only?: string;
+ quality?: string;
+ provider?: string;
+ "onset-method"?: string;
+ threshold?: string;
+ band?: string;
+ "hop-ms"?: string;
+ "onset-ratio"?: string;
+ "tail-db"?: string;
+ "silence-db"?: string;
+}
+
+function analyzeRequest(flags: AudioFlags): AnalyzeOptions {
+ const quality = (flags.quality ?? "best") as AnalysisQuality;
+ if (!QUALITIES.has(quality)) throw new CliUsageError("--quality is best, local or fast");
+ const method = flags["onset-method"] as OnsetMethod | undefined;
+ if (method && method !== "spectral-flux" && method !== "energy")
+ throw new CliUsageError("--onset-method is spectral-flux or energy");
+ const band = pair("band", flags.band);
+ return {
+ capabilities: capabilities(flags.only),
+ quality,
+ provider: pins(flags.provider),
+ options: {
+ onsets: { method, band, threshold: number("threshold", flags.threshold) },
+ envelope: { band, hopMs: number("hop-ms", flags["hop-ms"]) },
+ moments: {
+ onsetRatio: number("onset-ratio", flags["onset-ratio"]),
+ tailDb: number("tail-db", flags["tail-db"]),
+ },
+ silence: { thresholdDb: number("silence-db", flags["silence-db"]) },
+ },
+ };
+}
+
+async function fileTargets(files: string[], flags: AudioFlags): Promise<AudioTarget[]> {
+ const start = number("place-at", flags["place-at"]);
+ if (start !== undefined && files.length !== 1)
+ throw new CliUsageError("--place-at places exactly one file");
+ const skip = number("skip-file-seconds", flags["skip-file-seconds"]);
+ if (skip !== undefined && start === undefined)
+ throw new CliUsageError("--skip-file-seconds only works with --place-at");
+ const mediaStart = skip ?? 0;
+ const cwd = process.cwd();
+ return Promise.all(
+ files.map((file) => previewTarget(resolve(cwd, file), file, start ?? null, mediaStart)),
+ );
+}
+
+async function clipTargets(flags: AudioFlags, query: FilmQuery | null): Promise<AudioTarget[]> {
+ const project = resolveProject(flags.dir);
+ ensureDOMParser();
+ const timeline = await describeProject(project.indexPath);
+ return filmTargets(project.dir, timeline, query ?? { by: "all" });
+}
+
+export default defineCommand({
+ meta: {
+ name: "audio",
+ description: "Measure how sound moves in time: where it lands, its hits, loudness and silence",
+ },
+ args: {
+ files: { type: "positional", description: "Audio or video files to analyze", required: false },
+ dir: { type: "string", description: "Project directory, for film queries (default: current)" },
+ element: {
+ type: "string",
+ description: "One audio or video clip of the film: #id, hf:id or audio/N",
+ },
+ at: { type: "string", description: "Every sound playing at this film second" },
+ range: { type: "string", description: "Every sound playing in start,end film seconds" },
+ "place-at": {
+ type: "string",
+ description: "For one file not in the film yet: the film second it would start at",
+ },
+ "skip-file-seconds": {
+ type: "string",
+ description: "With --place-at: seconds cut from the start of the file",
+ },
+ only: {
+ type: "string",
+ description: `Capabilities to measure (default ${DEFAULT_CAPABILITIES.join(",")})`,
+ },
+ quality: {
+ type: "string",
+ description: "best (any provider), local (this machine), fast (signal only)",
+ },
+ provider: { type: "string", description: "Pin capabilities to providers: onsets=signal,..." },
+ "onset-method": { type: "string", description: "spectral-flux (default) or energy" },
+ threshold: { type: "string", description: "Onset strength floor, 0–1 (default 0.1)" },
+ band: { type: "string", description: "Measure onsets and envelope in lo,hi Hz only" },
+ "hop-ms": {
+ type: "string",
+ description: "Envelope hop in ms (default: about 2,000 values per item)",
+ },
+ "onset-ratio": {
+ type: "string",
+ description: "Moments: onset is where RMS first reaches this share of the peak (0.2)",
+ },
+ "tail-db": {
+ type: "string",
+ description: "Moments: the tail ends this many dB under the peak (-40)",
+ },
+ "silence-db": { type: "string", description: "Silence below this level in dBFS (-50)" },
+ full: {
+ type: "boolean",
+ description: "Include each capability's provider ladder",
+ default: false,
+ },
+ json: { type: "boolean", description: "Output as JSON", default: false },
+ },
+ setup({ rawArgs, cmd }) {
+ assertKnownFlags({ args: cmd.args }, rawArgs);
+ },
+ async run({ args }) {
+ const flags = args as unknown as AudioFlags;
+ const files = args._.map(String);
+ const query = filmQuery(flags);
+ if (files.length && query) throw new CliUsageError("give files or a film query, not both");
+ const targets = files.length
+ ? await fileTargets(files, flags)
+ : await clipTargets(flags, query);
+ const report = await analyzeTargets(targets, analyzeRequest(flags), Boolean(args.full));
+ console.log(args.json ? JSON.stringify(withMeta(report)) : formatReport(report));
+ },
+});
diff --git a/packages/cli/src/help.ts b/packages/cli/src/help.ts
index b9e04f181e..82a7cc7fb1 100644
--- a/packages/cli/src/help.ts
+++ b/packages/cli/src/help.ts
@@ -41,6 +41,7 @@ const GROUPS: Group[] = [
"Runtime-validate a composition in headless Chrome (JS errors, missing assets, contrast)",
],
["beats", "Detect beats in the music track and write beats/<audio>.json"],
+ ["audio", "Measure where sounds land: onsets, hits, loudness, silence, in film time"],
["inspect", "Inspect rendered visual layout across the timeline"],
["keyframes", "Inspect keyframes and render onion-shot diagnostics"],
["snapshot", "Capture key frames as PNG screenshots for visual verification"],
diff --git a/packages/cli/src/timeline/describeProject.inPoint.test.ts b/packages/cli/src/timeline/describeProject.inPoint.test.ts
index 2fde62feee..1caf194d04 100644
--- a/packages/cli/src/timeline/describeProject.inPoint.test.ts
+++ b/packages/cli/src/timeline/describeProject.inPoint.test.ts
@@ -17,16 +17,20 @@ const SCENE = `<template id="scene-template"><div data-composition-id="scene" da
beforeAll(ensureDOMParser);
+/** The video rows of a project made of `index` and `compositions/scene.html`. */
+async function videoRows(index: string, prefix: string) {
+ const dir = mkdtempSync(join(tmpdir(), prefix));
+ onTestFinished(() => rmSync(dir, { recursive: true, force: true }));
+ mkdirSync(join(dir, "compositions"));
+ writeFileSync(join(dir, "index.html"), index);
+ writeFileSync(join(dir, "compositions", "scene.html"), SCENE);
+ const timeline = await describeProject(join(dir, "index.html"));
+ return timeline.tracks.find((t) => t.kind === "video")!.rows;
+}
+
describe("a sub-composition's in-point", () => {
it("shifts each later split half's videos by its in-point and cuts them to that half", async () => {
- const dir = mkdtempSync(join(tmpdir(), "hf-timeline-inpoint-"));
- onTestFinished(() => rmSync(dir, { recursive: true, force: true }));
- mkdirSync(join(dir, "compositions"));
- writeFileSync(join(dir, "index.html"), SPLIT_INDEX);
- writeFileSync(join(dir, "compositions", "scene.html"), SCENE);
-
- const timeline = await describeProject(join(dir, "index.html"));
- const videos = timeline.tracks.find((t) => t.kind === "video")!.rows;
+ const videos = await videoRows(SPLIT_INDEX, "hf-timeline-inpoint-");
// The first half's in-point is 0, so it shows its children's authored extent, as before.
expect(videos.map((r) => [r.host, r.id, r.absStart, r.absEnd])).toEqual([
@@ -39,18 +43,24 @@ describe("a sub-composition's in-point", () => {
]);
});
+ it("moves each cut clip's media in-point by the part the cut skips", async () => {
+ const videos = await videoRows(SPLIT_INDEX, "hf-timeline-inpoint-media-");
+
+ expect(videos.map((r) => [r.host, r.id, r.mediaStart])).toEqual([
+ ["scene", "v1", 1],
+ ["scene", "v2", 5],
+ ["scene-split", "v2", 6],
+ ["scene", "v3", 2],
+ ["scene-split", "v3", 2],
+ ["scene-split-split", "v3", 3],
+ ]);
+ });
+
it("cuts nothing for a host whose in-point is 0, as Studio stamps on every inserted scene", async () => {
- const dir = mkdtempSync(join(tmpdir(), "hf-timeline-inpoint0-"));
- onTestFinished(() => rmSync(dir, { recursive: true, force: true }));
- mkdirSync(join(dir, "compositions"));
- writeFileSync(
- join(dir, "index.html"),
+ const videos = await videoRows(
`<div data-composition-id="main" data-duration="12"><div id="scene" data-composition-src="compositions/scene.html" data-start="1" data-duration="3" data-playback-start="0"></div></div>`,
+ "hf-timeline-inpoint0-",
);
- writeFileSync(join(dir, "compositions", "scene.html"), SCENE);
-
- const timeline = await describeProject(join(dir, "index.html"));
- const videos = timeline.tracks.find((t) => t.kind === "video")!.rows;
expect(videos.map((r) => [r.id, r.absStart, r.absEnd])).toEqual([
["v1", 1, 5],
diff --git a/packages/cli/src/timeline/describeProject.test.ts b/packages/cli/src/timeline/describeProject.test.ts
index 16d1d24c9f..586fb1b6c9 100644
--- a/packages/cli/src/timeline/describeProject.test.ts
+++ b/packages/cli/src/timeline/describeProject.test.ts
@@ -276,6 +276,7 @@ describe("describeProject", () => {
);
// (0.72 - 0.2) / 2, hand-computed
expect(row!.duration).toBeCloseTo(0.26, 1);
+ expect(row!.mediaStart).toBe(0.2);
});
it("takes a media leaf's duration from the probe when none is authored", async () => {
diff --git a/packages/cli/src/timeline/describeProject.ts b/packages/cli/src/timeline/describeProject.ts
index af41810915..efffc30bb0 100644
--- a/packages/cli/src/timeline/describeProject.ts
+++ b/packages/cli/src/timeline/describeProject.ts
@@ -71,6 +71,7 @@ export interface TimelineRow extends ClipFact {
children: RowPointer[];
/** Signed frames from its `data-sync-origin` partner (+ = late); `null` unpaired or rates differ. */
syncOffsetFrames: number | null;
+ mediaStart: number | null;
}
export interface RowPointer {
@@ -330,6 +331,9 @@ function describeRowFields(scope: DocScope, node: DomNode, timing: RowTiming): C
elementId: el.id || null,
hfId: el.getAttribute("data-hf-id"),
syncOffsetFrames: syncOffsetOf(el, scope.fps),
+ mediaStart: /^(video|audio)$/.test(kind)
+ ? readMediaOffsetSeconds((name) => el.getAttribute(name))
+ : null,
};
}
@@ -378,7 +382,9 @@ async function readSubComposition(
function cutRow(row: ClipDraft, slots: readonly HostSlot[]): ClipDraft[] {
const kept = cutToHostSlots({ start: row.absStart, end: row.absEnd }, slots);
if (kept.end === kept.start && row.absEnd > row.absStart) return [];
- return [{ ...row, absStart: roundMs(kept.start), absEnd: roundMs(kept.end) }];
+ const skipped = (kept.start - row.absStart) * (row.playbackRate ?? 1);
+ const mediaStart = row.mediaStart === null ? null : roundMs(row.mediaStart + skipped);
+ return [{ ...row, absStart: roundMs(kept.start), absEnd: roundMs(kept.end), mediaStart }];
}
/** The file's real path when it is a regular file inside the project (symlinks resolved), else null. */
diff --git a/packages/cli/src/timeline/timeExpr.test.ts b/packages/cli/src/timeline/timeExpr.test.ts
index 31b7730742..dadc9eed52 100644
--- a/packages/cli/src/timeline/timeExpr.test.ts
+++ b/packages/cli/src/timeline/timeExpr.test.ts
@@ -33,6 +33,7 @@ const row: TimelineRow = {
pendingReason: null,
laneError: null,
syncOffsetFrames: null,
+ mediaStart: 0,
};
describe("parseTimeExpression", () => {
diff --git a/packages/core/package-subpaths.json b/packages/core/package-subpaths.json
index 84b0e4c070..149b071752 100644
--- a/packages/core/package-subpaths.json
+++ b/packages/core/package-subpaths.json
@@ -254,6 +254,12 @@
"types": "./dist/audibleVideo.d.ts",
"environments": ["browser", "bun", "node"]
},
+ "./audio-analysis": {
+ "source": "./src/audioAnalysis.ts",
+ "runtime": "./dist/audioAnalysis.js",
+ "types": "./dist/audioAnalysis.d.ts",
+ "environments": ["browser", "bun", "node"]
+ },
"./audio-gain": {
"source": "./src/audioGain.ts",
"runtime": "./dist/audioGain.js",
diff --git a/packages/core/package.json b/packages/core/package.json
index 007d1156cd..671f643445 100644
--- a/packages/core/package.json
+++ b/packages/core/package.json
@@ -266,6 +266,12 @@
"import": "./src/audibleVideo.ts",
"types": "./src/audibleVideo.ts"
},
+ "./audio-analysis": {
+ "bun": "./src/audioAnalysis.ts",
+ "node": "./dist/audioAnalysis.js",
+ "import": "./src/audioAnalysis.ts",
+ "types": "./src/audioAnalysis.ts"
+ },
"./audio-gain": {
"bun": "./src/audioGain.ts",
"node": "./dist/audioGain.js",
@@ -702,6 +708,10 @@
"import": "./dist/audibleVideo.js",
"types": "./dist/audibleVideo.d.ts"
},
+ "./audio-analysis": {
+ "import": "./dist/audioAnalysis.js",
+ "types": "./dist/audioAnalysis.d.ts"
+ },
"./audio-gain": {
"import": "./dist/audioGain.js",
"types": "./dist/audioGain.d.ts"
diff --git a/packages/core/src/audioAnalysis.test.ts b/packages/core/src/audioAnalysis.test.ts
new file mode 100644
index 0000000000..3dfcdfe8b7
--- /dev/null
+++ b/packages/core/src/audioAnalysis.test.ts
@@ -0,0 +1,255 @@
+import { describe, expect, it } from "vitest";
+import {
+ classify,
+ envelope,
+ loudness,
+ moments,
+ onsets,
+ runLadder,
+ signalProvider,
+ silence,
+ type AudioAnalysisProvider,
+ type AudioSignal,
+} from "./audioAnalysis";
+
+const RATE = 48_000;
+
+/** A seeded noise source, so every run measures the same signal. */
+function noise(seed = 1) {
+ let state = seed;
+ return () => {
+ state = (state * 1_103_515_245 + 12_345) % 2_147_483_648;
+ return (state / 2_147_483_648) * 2 - 1;
+ };
+}
+
+function signal(
+ seconds: number,
+ fill: (t: number, random: () => number) => number,
+ channels = 1,
+): AudioSignal {
+ const random = noise();
+ const samples = Float32Array.from({ length: Math.round(seconds * RATE) }, (_, i) =>
+ fill(i / RATE, random),
+ );
+ return { sampleRate: RATE, channels: Array.from({ length: channels }, () => samples) };
+}
+
+/** Silence, then a noise burst that peaks `rise` seconds after `at` and decays exponentially. */
+const impact =
+ (at: number, rise: number, decay = 0.08) =>
+ (t: number, random: () => number) => {
+ if (t < at) return 0;
+ const since = t - at;
+ const gain = since < rise ? since / rise : Math.exp(-(since - rise) / decay);
+ return 0.8 * gain * random();
+ };
+
+describe("moments", () => {
+ it("finds where an impact starts, rises fastest, peaks and fades", () => {
+ const hit = moments(signal(1, impact(0.3, 0.02)))!;
+ expect(hit.onset).toBeGreaterThanOrEqual(0.3);
+ expect(hit.onset).toBeLessThan(0.31);
+ expect(hit.peak).toBeCloseTo(0.32, 2);
+ expect(hit.steepest).toBeGreaterThanOrEqual(hit.onset);
+ expect(hit.steepest).toBeLessThanOrEqual(hit.peak);
+ expect(hit.tailEnd).toBeGreaterThan(0.6);
+ });
+
+ it("puts a slow swell's peak at its end, well after its steepest rise", () => {
+ const swell = moments(
+ signal(1.5, (t, random) => (t < 1.2 ? 0.8 * (t / 1.2) ** 3 * random() : 0)),
+ )!;
+ expect(swell.peak).toBeGreaterThan(1.1);
+ expect(swell.onset).toBeGreaterThan(0.6);
+ expect(swell.onset).toBeLessThan(swell.peak);
+ });
+
+ it("has no moments in silence", () => {
+ expect(moments(signal(0.5, () => 0))).toBeNull();
+ });
+});
+
+describe("opposite polarity", () => {
+ const reversed = signal(3, impact(1, 0.01, 0.05));
+ reversed.channels.push(Float32Array.from(reversed.channels[0]!, (sample) => -sample));
+
+ it("still hears a channel that is the reverse of the other", () => {
+ expect(moments(reversed)?.peak).toBeCloseTo(1.01, 2);
+ for (const method of ["spectral-flux", "energy"] as const)
+ expect(
+ onsets(reversed, { method }).events.some((event) => Math.abs(event.t - 1) < 0.02),
+ ).toBe(true);
+ expect(Math.max(...envelope(reversed, { hopMs: 10 }).values)).toBeGreaterThan(-40);
+ expect(silence(reversed).some((span) => span.start <= 1.01 && span.end > 1.05)).toBe(false);
+ });
+});
+
+describe("onsets", () => {
+ it("finds each hit of a three-hit effect, the loudest strongest, by spectral flux and by energy", () => {
+ const hits = [0.2, 0.55, 0.9];
+ const gains = [1, 0.4, 0.7];
+ const fill = (t: number, random: () => number) =>
+ hits.reduce((sum, at, i) => sum + gains[i]! * impact(at, 0.003, 0.05)(t, random), 0);
+ for (const method of ["spectral-flux", "energy"] as const) {
+ const found = onsets(signal(1.2, fill), { method });
+ expect(found.events.map((e) => e.t).every((t, i) => Math.abs(t - hits[i]!) < 0.015)).toBe(
+ true,
+ );
+ expect(found.events).toHaveLength(3);
+ expect(found.events[0]!.strength).toBe(1);
+ }
+ });
+
+ it("finds a hit 16 dB under the loudest one", () => {
+ const fill = (t: number, random: () => number) =>
+ impact(0.2, 0.003, 0.05)(t, random) + 0.16 * impact(0.6, 0.003, 0.05)(t, random);
+ const found = onsets(signal(1, fill)).events.map((e) => e.t);
+ expect(found.some((t) => Math.abs(t - 0.6) < 0.015)).toBe(true);
+ });
+
+ it("finds nothing in silence", () => {
+ expect(onsets(signal(0.5, () => 0)).events).toEqual([]);
+ });
+});
+
+describe("envelope", () => {
+ it("reads a steady tone at its RMS level, in dB or linear", () => {
+ const tone = signal(0.5, (t) => 0.5 * Math.sin(2 * Math.PI * 440 * t));
+ const db = envelope(tone, { hopMs: 10 });
+ expect(db.values.length).toBe(50);
+ expect(db.values[25]).toBeCloseTo(20 * Math.log10(0.5 / Math.SQRT2), 0);
+ expect(envelope(tone, { scale: "linear" }).values[50]).toBeCloseTo(0.5 / Math.SQRT2, 2);
+ });
+
+ it("measures only the band asked for", () => {
+ const tone = signal(0.5, (t) => 0.5 * Math.sin(2 * Math.PI * 440 * t));
+ const inBand = envelope(tone, { band: [300, 600], hopMs: 10 }).values[25]!;
+ const outOfBand = envelope(tone, { band: [4000, 8000], hopMs: 10 }).values[25]!;
+ expect(inBand).toBeCloseTo(20 * Math.log10(0.5 / Math.SQRT2), 0);
+ expect(outOfBand).toBeLessThan(inBand - 40);
+ });
+});
+
+describe("loudness", () => {
+ it("reads a full-scale 997 Hz sine in one channel at -3.01 LUFS, as BS.1770 specifies", () => {
+ const sine = signal(3, (t) => Math.sin(2 * Math.PI * 997 * t));
+ const measured = loudness(sine);
+ expect(measured.lufsIntegrated).toBeCloseTo(-3.0, 1);
+ expect(measured.samplePeakDb).toBeCloseTo(0, 1);
+ expect(measured.clipping.count).toBeGreaterThan(0);
+ });
+
+ it("sums channels: the same sine in two channels is 3 dB louder", () => {
+ const one = loudness(signal(3, (t) => 0.5 * Math.sin(2 * Math.PI * 997 * t)));
+ const two = loudness(signal(3, (t) => 0.5 * Math.sin(2 * Math.PI * 997 * t), 2));
+ expect(two.lufsIntegrated! - one.lufsIntegrated!).toBeCloseTo(3.01, 1);
+ });
+
+ it("sees an inter-sample peak above the sample peak", () => {
+ const quarter = signal(1, (t) => Math.sin(2 * Math.PI * (RATE / 4) * t + Math.PI / 4));
+ const measured = loudness(quarter);
+ expect(measured.truePeakDb).toBeGreaterThan(measured.samplePeakDb + 2);
+ });
+
+ it("has no integrated loudness for silence", () => {
+ expect(loudness(signal(1, () => 0)).lufsIntegrated).toBeNull();
+ });
+});
+
+describe("silence", () => {
+ it("returns the quiet spans long enough to count", () => {
+ const gaps = silence(
+ signal(2, (t, random) => (t < 0.5 || (t > 1.2 && t < 1.25) ? 0 : 0.3 * random())),
+ );
+ expect(gaps).toHaveLength(1);
+ expect(gaps[0]!.start).toBe(0);
+ expect(gaps[0]!.end).toBeCloseTo(0.5, 1);
+ });
+});
+
+describe("classify", () => {
+ const gaps = noise(7);
+ const syllables: number[] = [];
+ for (let at = 0.1; at < 10; at += 0.12 + 0.25 * Math.abs(gaps())) syllables.push(at);
+ /** Noise in irregular syllable-length bursts over a quiet room tone. */
+ const speechLike = (t: number, random: () => number) => {
+ const into = syllables.map((at) => t - at).find((d) => d >= 0 && d < 0.15);
+ const voiced = into === undefined ? 0 : 0.5 * Math.sin((Math.PI * into) / 0.15);
+ return (voiced + 0.02) * random();
+ };
+
+ it("calls a short sound an effect", () => {
+ expect(classify(signal(1, impact(0.1, 0.01))).value).toBe("sfx");
+ });
+
+ it("calls a long sound that only fades away an effect, not a bed", () => {
+ expect(classify(signal(10, impact(0.1, 0.01, 1.2))).value).toBe("sfx");
+ });
+
+ it("calls a steady beat over a held pad music", () => {
+ const beat = (t: number, random: () => number) =>
+ 0.15 * Math.sin(2 * Math.PI * 220 * t) + impact(0, 0.003, 0.04)(t % 0.5, random);
+ expect(classify(signal(10, beat)).value).toBe("music");
+ });
+
+ it("calls irregular syllable-length bursts voice, in a long take and in a short line", () => {
+ expect(classify(signal(10, speechLike)).value).toBe("voice");
+ expect(classify(signal(3, speechLike)).value).toBe("voice");
+ });
+
+ it("reports what the guess was made from", () => {
+ expect(Object.keys(classify(signal(3, speechLike)).features)).toEqual([
+ "duration",
+ "regularity",
+ "syllables",
+ "sustain",
+ ]);
+ });
+});
+
+describe("runLadder", () => {
+ const tone = signal(0.2, (t) => 0.5 * Math.sin(2 * Math.PI * 440 * t));
+ const service: AudioAnalysisProvider<AudioSignal> = {
+ id: "service",
+ local: false,
+ capabilities: ["moments"],
+ analyze: async () => ({ onset: 0, steepest: 0, peak: 0, tailEnd: 0 }),
+ };
+ const broken: AudioAnalysisProvider<AudioSignal> = {
+ id: "broken",
+ local: true,
+ capabilities: ["moments"],
+ analyze: async () => {
+ throw new Error("model missing");
+ },
+ };
+
+ it("uses the first provider allowed at the quality asked for", async () => {
+ const providers = [service, signalProvider];
+ expect((await runLadder("moments", tone, providers)).provenance).toBe("service");
+ const local = await runLadder("moments", tone, providers, { quality: "local" });
+ expect(local.provenance).toBe("signal");
+ expect(local.steps[0]).toEqual({
+ provider: "service",
+ outcome: "excluded",
+ reason: "quality local keeps audio on this machine",
+ });
+ });
+
+ it("falls through a failing provider and says so", async () => {
+ const result = await runLadder("moments", tone, [broken, signalProvider]);
+ expect(result.provenance).toBe("signal");
+ expect(result.warnings).toEqual(["moments: broken failed (model missing); used signal"]);
+ });
+
+ it("follows a ladder given in place of the default, and reports a capability no provider offers", async () => {
+ const result = await runLadder("moments", tone, [service, signalProvider], {
+ ladder: ["signal", "service"],
+ });
+ expect(result.provenance).toBe("signal");
+ const beats = await runLadder("beats", tone, [signalProvider]);
+ expect(beats).toMatchObject({ value: null, provenance: null });
+ expect(beats.warnings).toEqual(["beats: no provider answered (none offers it)"]);
+ });
+});
diff --git a/packages/core/src/audioAnalysis.ts b/packages/core/src/audioAnalysis.ts
new file mode 100644
index 0000000000..99e51f3c0f
--- /dev/null
+++ b/packages/core/src/audioAnalysis.ts
@@ -0,0 +1,623 @@
+/**
+ * Measurements of decoded audio over time: an energy envelope, onsets, the moments a sound lands, loudness and silence.
+ * Pure functions of PCM samples, so they run the same in Node, Bun and the browser. Times are seconds into the signal.
+ */
+
+import type { AudioAnalysisProvider } from "./audioAnalysisLadder";
+import { fft } from "./fft";
+
+/** Decoded audio: one Float32Array per channel, all the same length. */
+export interface AudioSignal {
+ sampleRate: number;
+ channels: Float32Array[];
+}
+
+export type EnvelopeScale = "db" | "linear";
+
+export interface EnvelopeOptions {
+ hopMs?: number;
+ /** RMS window; twice the hop when left out. */
+ windowMs?: number;
+ scale?: EnvelopeScale;
+ /** Measure only this frequency band, in Hz. */
+ band?: [number, number];
+}
+
+export interface Envelope {
+ hopMs: number;
+ scale: EnvelopeScale;
+ /** One value per hop, the first centred on time 0. */
+ values: number[];
+}
+
+export type OnsetMethod = "spectral-flux" | "energy";
+
+export interface OnsetOptions {
+ method?: OnsetMethod;
+ band?: [number, number];
+ /** Smallest normalized strength, 0–1, that counts as an onset. */
+ threshold?: number;
+ /** Onsets closer than this keep only the stronger. */
+ minGapMs?: number;
+ hopMs?: number;
+}
+
+export interface Onset {
+ t: number;
+ /** Relative to the strongest onset in the signal, 0–1. */
+ strength: number;
+}
+
+export interface Onsets {
+ method: OnsetMethod;
+ events: Onset[];
+}
+
+export interface MomentOptions {
+ /** The onset is where RMS first reaches this fraction of its peak. */
+ onsetRatio?: number;
+ /** The tail ends where RMS last stays above the peak plus this many dB. */
+ tailDb?: number;
+ hopMs?: number;
+}
+
+/** Where a sound lands. Null for a silent signal. */
+export interface Moments {
+ /** RMS first reaches `onsetRatio` of its peak. */
+ onset: number;
+ /** The largest rise in RMS between neighbouring hops. */
+ steepest: number;
+ /** The loudest hop. */
+ peak: number;
+ /** The last hop within `tailDb` of the peak. */
+ tailEnd: number;
+}
+
+export interface Loudness {
+ /** ITU-R BS.1770 gated integrated loudness; null when every block is below the absolute gate. */
+ lufsIntegrated: number | null;
+ /** The loudest 3 s window. */
+ lufsShortTermMax: number | null;
+ samplePeakDb: number;
+ /** A 4× oversampled estimate of the true peak, in dBTP. */
+ truePeakDb: number;
+ clipping: { count: number; firstAt: number | null };
+}
+
+export interface SilenceOptions {
+ thresholdDb?: number;
+ minMs?: number;
+ hopMs?: number;
+}
+
+export interface Span {
+ start: number;
+ end: number;
+}
+
+export type AudioKind = "sfx" | "music" | "voice" | "mixed";
+
+export interface KindGuess {
+ value: AudioKind;
+ /** 0–1. Signal heuristics only: treat below 0.6 as a guess. */
+ confidence: number;
+ /** What the guess was made from. */
+ features: KindFeatures;
+}
+
+export interface KindFeatures {
+ duration: number;
+ /** Strongest autocorrelation of the onset curve at beat-like lags (0.3–1.5 s), 0–1. */
+ regularity: number;
+ /** Share of envelope modulation at syllable rate (3–8 Hz), 0–1. */
+ syllables: number;
+ /** Share of 50 ms frames within 20 dB of the loud end (95th percentile), 0–1: high for beds and speech, low for hits. */
+ sustain: number;
+}
+
+const DB_FLOOR = -120;
+
+export const signalDuration = (signal: AudioSignal): number =>
+ (signal.channels[0]?.length ?? 0) / signal.sampleRate;
+
+/** Per-sample energy. Channels are squared first, so a reversed channel stays audible. */
+function energyMix(signal: AudioSignal): Float32Array {
+ const [first, ...rest] = signal.channels;
+ if (!first) return new Float32Array(0);
+ if (!rest.length) return first;
+ const mixed = new Float32Array(first.length);
+ for (const channel of signal.channels)
+ for (let i = 0; i < mixed.length; i++) mixed[i]! += channel[i]! * channel[i]!;
+ const scale = 1 / signal.channels.length;
+ for (let i = 0; i < mixed.length; i++) mixed[i] = Math.sqrt(mixed[i]! * scale);
+ return mixed;
+}
+
+const toDb = (amplitude: number) =>
+ amplitude > 0 ? Math.max(DB_FLOOR, 20 * Math.log10(amplitude)) : DB_FLOOR;
+const hopSamples = (sampleRate: number, hopMs: number) =>
+ Math.max(1, Math.round((sampleRate * hopMs) / 1000));
+const round = (t: number) => Math.round(t * 1e4) / 1e4;
+
+function rmsFrames(
+ samples: Float32Array,
+ sampleRate: number,
+ hopMs: number,
+ windowMs: number,
+): number[] {
+ const hop = hopSamples(sampleRate, hopMs);
+ const half = Math.max(1, Math.round((sampleRate * windowMs) / 2000));
+ const frames: number[] = [];
+ for (let center = 0; center < samples.length; center += hop) {
+ const from = Math.max(0, center - half);
+ const to = Math.min(samples.length, center + half);
+ let sum = 0;
+ for (let i = from; i < to; i++) sum += samples[i]! * samples[i]!;
+ frames.push(to > from ? Math.sqrt(sum / (to - from)) : 0);
+ }
+ return frames;
+}
+
+const fftSizeFor = (sampleRate: number) => 2 ** Math.round(Math.log2(sampleRate * 0.023));
+
+/** Magnitude spectra of Hann-windowed frames centred on each hop, limited to `band`. */
+/** The FFT bins inside `band`, as [first, end). */
+function bandBins(sampleRate: number, size: number, band?: [number, number]): [number, number] {
+ const binHz = sampleRate / size;
+ const low = Math.max(1, Math.floor((band?.[0] ?? 0) / binHz));
+ return [low, Math.max(low, Math.min(size / 2, Math.ceil((band?.[1] ?? sampleRate / 2) / binHz)))];
+}
+
+/** Load the Hann-windowed frame centred on `center` into `re`, zero outside the signal. */
+function loadFrame(
+ samples: Float32Array,
+ center: number,
+ window: Float64Array,
+ re: Float64Array,
+ im: Float64Array,
+): void {
+ const first = center - window.length / 2;
+ for (let i = 0; i < window.length; i++) {
+ const at = first + i;
+ re[i] = at >= 0 && at < samples.length ? samples[at]! * window[i]! : 0;
+ im[i] = 0;
+ }
+}
+
+function spectra(
+ samples: Float32Array,
+ sampleRate: number,
+ hopMs: number,
+ band?: [number, number],
+): Float64Array[] {
+ const size = fftSizeFor(sampleRate);
+ const hop = hopSamples(sampleRate, hopMs);
+ const window = Float64Array.from(
+ { length: size },
+ (_, i) => 0.5 - 0.5 * Math.cos((2 * Math.PI * i) / size),
+ );
+ const [low, high] = bandBins(sampleRate, size, band);
+ const frames: Float64Array[] = [];
+ const re = new Float64Array(size);
+ const im = new Float64Array(size);
+ for (let center = 0; center < samples.length; center += hop) {
+ loadFrame(samples, center, window, re, im);
+ fft(re, im);
+ const magnitudes = new Float64Array(high - low);
+ for (let k = low; k < high; k++) magnitudes[k - low] = Math.hypot(re[k]!, im[k]!);
+ frames.push(magnitudes);
+ }
+ return frames;
+}
+
+function averageMagnitudes(frames: Float64Array[]): Float64Array {
+ const mixed = new Float64Array(frames[0]?.length ?? 0);
+ for (const frame of frames)
+ for (let bin = 0; bin < mixed.length; bin++) mixed[bin]! += frame[bin]!;
+ const scale = frames.length || 1;
+ for (let bin = 0; bin < mixed.length; bin++) mixed[bin]! /= scale;
+ return mixed;
+}
+
+/** Mean magnitude spectrum across channels, so opposite polarity cannot cancel a bin. */
+function meanSpectra(signal: AudioSignal, hopMs: number, band?: [number, number]): Float64Array[] {
+ const stacked = signal.channels.map((channel) =>
+ spectra(channel, signal.sampleRate, hopMs, band),
+ );
+ const count = stacked[0]?.length ?? 0;
+ const mixed: Float64Array[] = [];
+ for (let frame = 0; frame < count; frame++)
+ mixed.push(averageMagnitudes(stacked.map((channel) => channel[frame]!)));
+ return mixed;
+}
+
+export function envelope(signal: AudioSignal, options: EnvelopeOptions = {}): Envelope {
+ const { hopMs = 5, scale = "db", band } = options;
+ const windowMs = options.windowMs ?? hopMs * 2;
+ const size = fftSizeFor(signal.sampleRate);
+ // Parseval over the band's bins; a Hann window keeps 0.375 of a frame's power.
+ const amplitudes = band
+ ? meanSpectra(signal, hopMs, band).map((frame) => {
+ let sum = 0;
+ for (const magnitude of frame) sum += magnitude * magnitude;
+ return Math.sqrt((2 * sum) / (size * size * 0.375));
+ })
+ : rmsFrames(energyMix(signal), signal.sampleRate, hopMs, windowMs);
+ return { hopMs, scale, values: amplitudes.map((a) => (scale === "db" ? toDb(a) : a)) };
+}
+
+/** Positive change between neighbouring frames: spectral flux of log magnitudes, or the rise in RMS. */
+function noveltyCurve(
+ signal: AudioSignal,
+ method: OnsetMethod,
+ hopMs: number,
+ band?: [number, number],
+): number[] {
+ const samples = energyMix(signal);
+ if (method === "energy") {
+ const rms = rmsFrames(samples, signal.sampleRate, hopMs, hopMs * 2);
+ return rms.map((value, i) => Math.max(0, value - (rms[i - 1] ?? 0)));
+ }
+ const frames = meanSpectra(signal, hopMs, band).map((frame) =>
+ frame.map((m) => Math.log1p(100 * m)),
+ );
+ // A frame cut off by the end of the signal splatters across the spectrum; that is the edge, not a hit.
+ const lastWhole = Math.floor(
+ (samples.length - fftSizeFor(signal.sampleRate) / 2) / hopSamples(signal.sampleRate, hopMs),
+ );
+ return frames.map((frame, i) => {
+ const previous = frames[i - 1];
+ if (!previous || i > lastWhole) return 0;
+ let flux = 0;
+ for (let k = 0; k < frame.length; k++) flux += Math.max(0, frame[k]! - previous[k]!);
+ return flux;
+ });
+}
+
+/** Peaks of the novelty curve that stand out from their neighbourhood. */
+/** A local maximum within `gap` frames that clears the mean of its ±`context` neighbourhood by 5% of the top. */
+function isOnsetPeak(
+ curve: number[],
+ i: number,
+ gap: number,
+ context: number,
+ top: number,
+): boolean {
+ const value = curve[i]!;
+ let sum = 0;
+ let count = 0;
+ for (let j = Math.max(0, i - context); j <= Math.min(curve.length - 1, i + context); j++) {
+ if (Math.abs(j - i) <= gap && curve[j]! > value) return false;
+ sum += curve[j]!;
+ count++;
+ }
+ return value >= sum / count + 0.05 * top;
+}
+
+export function onsets(signal: AudioSignal, options: OnsetOptions = {}): Onsets {
+ const { method = "spectral-flux", band, threshold = 0.1, minGapMs = 30, hopMs = 5 } = options;
+ const curve = noveltyCurve(signal, method, hopMs, band);
+ const top = Math.max(0, ...curve);
+ if (top <= 0) return { method, events: [] };
+ const gap = Math.max(1, Math.round(minGapMs / hopMs));
+ const context = Math.max(gap, Math.round(100 / hopMs));
+ const hopSeconds = hopSamples(signal.sampleRate, hopMs) / signal.sampleRate;
+ const events: Onset[] = [];
+ for (let i = 0; i < curve.length; i++) {
+ const strength = curve[i]! / top;
+ if (strength < threshold || !isOnsetPeak(curve, i, gap, context, top)) continue;
+ const onset = { t: round(i * hopSeconds), strength: round(strength) };
+ const last = events[events.length - 1];
+ if (!last || onset.t - last.t >= minGapMs / 1000) events.push(onset);
+ else if (onset.strength > last.strength) events[events.length - 1] = onset;
+ }
+ return { method, events };
+}
+
+export function moments(signal: AudioSignal, options: MomentOptions = {}): Moments | null {
+ const { onsetRatio = 0.2, tailDb = -40, hopMs = 1 } = options;
+ const rms = rmsFrames(energyMix(signal), signal.sampleRate, hopMs, Math.max(4, hopMs * 4));
+ const hopSeconds = hopSamples(signal.sampleRate, hopMs) / signal.sampleRate;
+ let peakIndex = 0;
+ for (let i = 1; i < rms.length; i++) if (rms[i]! > rms[peakIndex]!) peakIndex = i;
+ const peak = rms[peakIndex] ?? 0;
+ if (peak <= 0) return null;
+ let onsetIndex = rms.findIndex((value) => value >= peak * onsetRatio);
+ if (onsetIndex < 0) onsetIndex = peakIndex;
+ let steepestIndex = onsetIndex;
+ let steepestRise = -Infinity;
+ for (let i = 1; i <= peakIndex; i++) {
+ const rise = rms[i]! - rms[i - 1]!;
+ if (rise > steepestRise) [steepestRise, steepestIndex] = [rise, i];
+ }
+ const floor = peak * 10 ** (tailDb / 20);
+ let tailIndex = peakIndex;
+ for (let i = rms.length - 1; i > peakIndex; i--)
+ if (rms[i]! >= floor) {
+ tailIndex = i;
+ break;
+ }
+ return {
+ onset: round(onsetIndex * hopSeconds),
+ steepest: round(steepestIndex * hopSeconds),
+ peak: round(peakIndex * hopSeconds),
+ tailEnd: round(tailIndex * hopSeconds),
+ };
+}
+
+type Biquad = [number, number, number, number, number];
+
+/** BS.1770 K-weighting at any sample rate: a high-shelf pre-filter, then the RLB high-pass. */
+function kWeighting(sampleRate: number): [Biquad, Biquad] {
+ let k = Math.tan((Math.PI * 1681.974450955533) / sampleRate);
+ const q = 0.7071752369554196;
+ const vh = 10 ** (3.999843853973347 / 20);
+ const vb = vh ** 0.4996667741545416;
+ let a0 = 1 + k / q + k * k;
+ const shelf: Biquad = [
+ (vh + (vb * k) / q + k * k) / a0,
+ (2 * (k * k - vh)) / a0,
+ (vh - (vb * k) / q + k * k) / a0,
+ (2 * (k * k - 1)) / a0,
+ (1 - k / q + k * k) / a0,
+ ];
+ k = Math.tan((Math.PI * 38.13547087602444) / sampleRate);
+ const q2 = 0.5003270373238773;
+ a0 = 1 + k / q2 + k * k;
+ const highPass: Biquad = [1, -2, 1, (2 * (k * k - 1)) / a0, (1 - k / q2 + k * k) / a0];
+ return [shelf, highPass];
+}
+
+function filtered(
+ samples: Float32Array | Float64Array,
+ [b0, b1, b2, a1, a2]: Biquad,
+): Float64Array {
+ const out = new Float64Array(samples.length);
+ let x1 = 0,
+ x2 = 0,
+ y1 = 0,
+ y2 = 0;
+ for (let i = 0; i < samples.length; i++) {
+ const x = samples[i]!;
+ const y = b0 * x + b1 * x1 + b2 * x2 - a1 * y1 - a2 * y2;
+ x2 = x1;
+ x1 = x;
+ y2 = y1;
+ y1 = y;
+ out[i] = y;
+ }
+ return out;
+}
+
+const blockLoudness = (meanSquare: number) =>
+ meanSquare > 0 ? -0.691 + 10 * Math.log10(meanSquare) : -Infinity;
+
+/** Running sum of squares, so any block's power is one subtraction. */
+function squareSums(channel: Float64Array): Float64Array {
+ const sums = new Float64Array(channel.length + 1);
+ for (let i = 0; i < channel.length; i++) sums[i + 1] = sums[i]! + channel[i]! * channel[i]!;
+ return sums;
+}
+
+/** Summed channel power of each block of `blockSeconds`, stepped by `stepSeconds`. */
+function blockPowers(
+ sums: Float64Array[],
+ sampleRate: number,
+ blockSeconds: number,
+ stepSeconds: number,
+): number[] {
+ const length = (sums[0]?.length ?? 1) - 1;
+ if (!length) return [];
+ const block = Math.min(length, Math.round(blockSeconds * sampleRate));
+ const step = Math.round(stepSeconds * sampleRate);
+ const powers: number[] = [];
+ for (let start = 0; start + block <= length; start += step)
+ powers.push(
+ sums.reduce((power, sum) => power + (sum[start + block]! - sum[start]!) / block, 0),
+ );
+ return powers;
+}
+
+const TRUE_PEAK_OVERSAMPLE = 4;
+const TRUE_PEAK_TAPS = 12;
+
+const TRUE_PEAK_KERNELS = Array.from({ length: TRUE_PEAK_OVERSAMPLE - 1 }, (_, phase) => {
+ const offset = (phase + 1) / TRUE_PEAK_OVERSAMPLE;
+ return Float64Array.from({ length: 2 * TRUE_PEAK_TAPS }, (_, tap) => {
+ const x = tap - TRUE_PEAK_TAPS + 1 - offset;
+ const sinc = x === 0 ? 1 : Math.sin(Math.PI * x) / (Math.PI * x);
+ return sinc * (0.5 + 0.5 * Math.cos((Math.PI * x) / TRUE_PEAK_TAPS));
+ });
+});
+
+/**
+ * The 4× oversampled peak, interpolated only between samples at least half the sample peak: overshooting both
+ * neighbours by 6 dB takes energy near Nyquist that real program material does not carry.
+ */
+function truePeak(channel: Float32Array): number {
+ let samplePeak = 0;
+ for (let i = 0; i < channel.length; i++) samplePeak = Math.max(samplePeak, Math.abs(channel[i]!));
+ const floor = samplePeak / 2;
+ const taps = 2 * TRUE_PEAK_TAPS;
+ let peak = samplePeak;
+ for (let n = 0; n + taps <= channel.length; n++) {
+ if (
+ Math.abs(channel[n + TRUE_PEAK_TAPS - 1]!) < floor &&
+ Math.abs(channel[n + TRUE_PEAK_TAPS]!) < floor
+ )
+ continue;
+ for (const kernel of TRUE_PEAK_KERNELS) {
+ let y = 0;
+ for (let tap = 0; tap < taps; tap++) y += channel[n + tap]! * kernel[tap]!;
+ if (y > peak) peak = y;
+ else if (-y > peak) peak = -y;
+ }
+ }
+ return peak;
+}
+
+export function loudness(signal: AudioSignal): Loudness {
+ const [shelf, highPass] = kWeighting(signal.sampleRate);
+ const sums = signal.channels.map((channel) =>
+ squareSums(filtered(filtered(channel, shelf), highPass)),
+ );
+ const momentary = blockPowers(sums, signal.sampleRate, 0.4, 0.1);
+ const audible = momentary.filter((power) => blockLoudness(power) > -70);
+ let lufsIntegrated: number | null = null;
+ if (audible.length) {
+ const relative = blockLoudness(audible.reduce((a, b) => a + b, 0) / audible.length) - 10;
+ const gated = audible.filter((power) => blockLoudness(power) > relative);
+ if (gated.length)
+ lufsIntegrated =
+ Math.round(blockLoudness(gated.reduce((a, b) => a + b, 0) / gated.length) * 10) / 10;
+ }
+ const shortTerm = blockPowers(sums, signal.sampleRate, 3, 0.1)
+ .map(blockLoudness)
+ .filter(Number.isFinite);
+ let samplePeak = 0;
+ let count = 0;
+ let firstAt: number | null = null;
+ for (const channel of signal.channels)
+ for (let i = 0; i < channel.length; i++) {
+ const magnitude = Math.abs(channel[i]!);
+ samplePeak = Math.max(samplePeak, magnitude);
+ if (magnitude >= 0.999) {
+ count++;
+ const t = i / signal.sampleRate;
+ if (firstAt === null || t < firstAt) firstAt = round(t);
+ }
+ }
+ return {
+ lufsIntegrated,
+ lufsShortTermMax: shortTerm.length ? Math.round(Math.max(...shortTerm) * 10) / 10 : null,
+ samplePeakDb: Math.round(toDb(samplePeak) * 10) / 10,
+ truePeakDb: Math.round(toDb(Math.max(0, ...signal.channels.map(truePeak))) * 10) / 10,
+ clipping: { count, firstAt },
+ };
+}
+
+export function silence(signal: AudioSignal, options: SilenceOptions = {}): Span[] {
+ const { thresholdDb = -50, minMs = 200, hopMs = 5 } = options;
+ const db = rmsFrames(energyMix(signal), signal.sampleRate, hopMs, hopMs * 2).map(toDb);
+ const hopSeconds = hopSamples(signal.sampleRate, hopMs) / signal.sampleRate;
+ const spans: Span[] = [];
+ let from: number | null = null;
+ for (let i = 0; i <= db.length; i++) {
+ const quiet = i < db.length && db[i]! < thresholdDb;
+ if (quiet && from === null) from = i;
+ if (!quiet && from !== null) {
+ if ((i - from) * hopSeconds * 1000 >= minMs)
+ spans.push({
+ start: round(from * hopSeconds),
+ end: round(Math.min(i * hopSeconds, signalDuration(signal))),
+ });
+ from = null;
+ }
+ }
+ return spans;
+}
+
+/** How regular the onsets are: the strongest normalized autocorrelation of the novelty curve at beat-like lags. */
+function periodicity(curve: number[], hopMs: number): number {
+ const mean = curve.reduce((a, b) => a + b, 0) / (curve.length || 1);
+ const centred = curve.map((v) => v - mean);
+ const energy = centred.reduce((a, v) => a + v * v, 0);
+ if (!energy) return 0;
+ let best = 0;
+ for (
+ let lag = Math.round(300 / hopMs);
+ lag <= Math.round(1500 / hopMs) && lag < centred.length;
+ lag++
+ ) {
+ let sum = 0;
+ for (let i = lag; i < centred.length; i++) sum += centred[i]! * centred[i - lag]!;
+ best = Math.max(best, sum / energy);
+ }
+ return best;
+}
+
+/** How much of the envelope's modulation sits at syllable rate (3–8 Hz). */
+function syllableModulation(signal: AudioSignal): number {
+ const hopMs = 10;
+ const rms = rmsFrames(energyMix(signal), signal.sampleRate, hopMs, 25);
+ const size = 2 ** Math.floor(Math.log2(Math.min(rms.length, 1024)));
+ if (size < 64) return 0;
+ const re = Float64Array.from(rms.slice(0, size));
+ const mean = re.reduce((a, b) => a + b, 0) / size;
+ for (let i = 0; i < size; i++) re[i] = re[i]! - mean;
+ const im = new Float64Array(size);
+ fft(re, im);
+ const rate = 1000 / hopMs;
+ let inBand = 0;
+ let total = 0;
+ for (let k = 1; k < size / 2; k++) {
+ const power = re[k]! * re[k]! + im[k]! * im[k]!;
+ const hz = (k * rate) / size;
+ total += power;
+ if (hz >= 3 && hz <= 8) inBand += power;
+ }
+ return total ? inBand / total : 0;
+}
+
+function sustainShare(signal: AudioSignal): number {
+ const levels = rmsFrames(energyMix(signal), signal.sampleRate, 50, 100).map(toDb);
+ if (!levels.length) return 0;
+ const loud = [...levels].sort((a, b) => a - b)[Math.floor((levels.length - 1) * 0.95)]!;
+ return levels.filter((level) => level > loud - 20).length / levels.length;
+}
+
+export function kindFeatures(signal: AudioSignal): KindFeatures {
+ return {
+ duration: round(signalDuration(signal)),
+ regularity: round(periodicity(noveltyCurve(signal, "spectral-flux", 10), 10)),
+ syllables: round(syllableModulation(signal)),
+ sustain: round(sustainShare(signal)),
+ };
+}
+
+/**
+ * A first guess at what the audio is. Length decides most of it: beds run long, effects short. Long audio is voice
+ * when its envelope moves at syllable rate without a beat; short audio is voice when it also holds its level.
+ */
+export function classify(signal: AudioSignal): KindGuess {
+ const features = kindFeatures(signal);
+ const { duration, regularity, syllables, sustain } = features;
+ const speechLike = syllables >= 0.35 && regularity < 0.35;
+ if (duration >= 8) {
+ if (sustain < 0.4) return { value: "sfx", confidence: 0.6, features };
+ if (speechLike)
+ return { value: "voice", confidence: round(Math.min(0.9, 0.4 + syllables)), features };
+ return { value: "music", confidence: round(Math.min(0.9, 0.5 + regularity)), features };
+ }
+ if (duration >= 1.5 && syllables >= 0.17 && regularity < 0.3 && sustain >= 0.7)
+ return { value: "voice", confidence: speechLike ? 0.8 : 0.6, features };
+ return { value: "sfx", confidence: duration <= 2 ? 0.85 : 0.65, features };
+}
+
+export * from "./audioAnalysisLadder";
+
+/** Signal processing as a ladder provider: always available, the measured ground every other provider snaps to. */
+export const signalProvider: AudioAnalysisProvider<AudioSignal> = {
+ id: "signal",
+ local: true,
+ signal: true,
+ capabilities: ["envelope", "onsets", "moments", "loudness", "silence", "kind"],
+ async analyze(capability, input, options) {
+ switch (capability) {
+ case "envelope":
+ return envelope(input, options as EnvelopeOptions);
+ case "onsets":
+ return onsets(input, options as OnsetOptions);
+ case "moments":
+ return moments(input, options as MomentOptions);
+ case "loudness":
+ return loudness(input);
+ case "silence":
+ return silence(input, options as SilenceOptions);
+ case "kind":
+ return classify(input);
+ default:
+ throw new Error(`signal processing does not measure ${capability}`);
+ }
+ },
+};
diff --git a/packages/core/src/audioAnalysisLadder.ts b/packages/core/src/audioAnalysisLadder.ts
new file mode 100644
index 0000000000..728e7bbadb
--- /dev/null
+++ b/packages/core/src/audioAnalysisLadder.ts
@@ -0,0 +1,132 @@
+/**
+ * The provider ladder for audio analysis. Each capability lists providers best first; a call takes the first one
+ * that is available within the requested quality and answers, falling through on failure. The result says which
+ * provider answered and why the ones above it did not.
+ */
+
+export type AudioCapability =
+ | "envelope"
+ | "onsets"
+ | "moments"
+ | "loudness"
+ | "silence"
+ | "kind"
+ | "beats"
+ | "sections"
+ | "words"
+ | "describe";
+
+/** `best` may use any provider; `local` none that sends audio off the machine; `fast` only signal processing. */
+export type AnalysisQuality = "best" | "local" | "fast";
+
+export interface AudioAnalysisProvider<Input = unknown> {
+ id: string;
+ /** Runs on this machine; a provider that is not local sends the audio to a service. */
+ local: boolean;
+ /** Pure signal processing: no model, no download. */
+ signal?: boolean;
+ capabilities: readonly AudioCapability[];
+ /** Whether it can run here now: a key, a platform, a model on disk. Missing means always. */
+ available?: () => boolean | Promise<boolean>;
+ analyze: (
+ capability: AudioCapability,
+ input: Input,
+ options: Record<string, unknown>,
+ ) => Promise<unknown>;
+}
+
+export type LadderOutcome = "answered" | "unavailable" | "excluded" | "failed";
+
+export interface LadderStep {
+ provider: string;
+ outcome: LadderOutcome;
+ reason?: string;
+}
+
+export interface LadderResult<Value = unknown> {
+ value: Value | null;
+ /** The provider that answered; null when none could. */
+ provenance: string | null;
+ steps: LadderStep[];
+ warnings: string[];
+}
+
+export interface LadderOptions {
+ quality?: AnalysisQuality;
+ /** This capability's ladder, best first, in place of the default. */
+ ladder?: readonly string[];
+ /** Use only this provider. */
+ provider?: string;
+ options?: Record<string, unknown>;
+}
+
+/** The default order per capability: every registered provider that offers it, in registration order. */
+export function defaultLadder<Input>(
+ capability: AudioCapability,
+ providers: readonly AudioAnalysisProvider<Input>[],
+): string[] {
+ return providers.filter((p) => p.capabilities.includes(capability)).map((p) => p.id);
+}
+
+/** Why a provider may not be used at this quality, or null when it may. */
+export function exclusion<Input>(
+ provider: AudioAnalysisProvider<Input>,
+ quality: AnalysisQuality,
+): string | null {
+ if (quality === "fast" && !provider.signal) return "quality fast allows signal processing only";
+ if (quality === "local" && !provider.local) return "quality local keeps audio on this machine";
+ return null;
+}
+
+/** Why a provider cannot take this step, or null when it can. */
+async function blocked<Input>(
+ id: string,
+ candidate: AudioAnalysisProvider<Input> | undefined,
+ capability: AudioCapability,
+ quality: AnalysisQuality,
+): Promise<LadderStep | null> {
+ if (!candidate?.capabilities.includes(capability))
+ return { provider: id, outcome: "unavailable", reason: `no provider ${id} for ${capability}` };
+ const excluded = exclusion(candidate, quality);
+ if (excluded) return { provider: id, outcome: "excluded", reason: excluded };
+ if (candidate.available && !(await candidate.available()))
+ return { provider: id, outcome: "unavailable" };
+ return null;
+}
+
+export async function runLadder<Input, Value = unknown>(
+ capability: AudioCapability,
+ input: Input,
+ providers: readonly AudioAnalysisProvider<Input>[],
+ { quality = "best", ladder, provider, options = {} }: LadderOptions = {},
+): Promise<LadderResult<Value>> {
+ const byId = new Map(providers.map((p) => [p.id, p]));
+ const order = provider ? [provider] : (ladder ?? defaultLadder(capability, providers));
+ const steps: LadderStep[] = [];
+ for (const id of order) {
+ const candidate = byId.get(id);
+ const skip = await blocked(id, candidate, capability, quality);
+ if (skip) {
+ steps.push(skip);
+ continue;
+ }
+ try {
+ const value = (await candidate!.analyze(capability, input, options)) as Value;
+ steps.push({ provider: id, outcome: "answered" });
+ const warnings = steps
+ .filter((step) => step.outcome === "failed")
+ .map((step) => `${capability}: ${step.provider} failed (${step.reason}); used ${id}`);
+ return { value, provenance: id, steps, warnings };
+ } catch (error) {
+ const reason = error instanceof Error ? error.message : String(error);
+ steps.push({ provider: id, outcome: "failed", reason });
+ }
+ }
+ const none = order.length ? "" : " (none offers it)";
+ return {
+ value: null,
+ provenance: null,
+ steps,
+ warnings: [`${capability}: no provider answered${none}`],
+ };
+}
diff --git a/packages/core/src/audioCarve.ts b/packages/core/src/audioCarve.ts
index 0b0fce582a..bd09a14400 100644
--- a/packages/core/src/audioCarve.ts
+++ b/packages/core/src/audioCarve.ts
@@ -17,6 +17,7 @@ import {
type HfAudioFxChain,
type HfAudioFxNode,
} from "./audioFx.js";
+import { fft } from "./fft";
export const HF_AUDIO_CARVE_ATTR = "data-fx-carve";
@@ -399,42 +400,6 @@ function powerSpectrum(
return { freqs, power };
}
-/** In-place iterative radix-2 FFT. */
-function fft(re: Float64Array, im: Float64Array): void {
- const n = re.length;
- for (let i = 1, j = 0; i < n; i++) {
- let bit = n >> 1;
- for (; j & bit; bit >>= 1) j ^= bit;
- j ^= bit;
- if (i < j) {
- [re[i], re[j]] = [re[j]!, re[i]!];
- [im[i], im[j]] = [im[j]!, im[i]!];
- }
- }
- for (let len = 2; len <= n; len <<= 1) {
- const ang = (-2 * Math.PI) / len;
- const wr = Math.cos(ang);
- const wi = Math.sin(ang);
- for (let i = 0; i < n; i += len) {
- let cr = 1;
- let ci = 0;
- for (let k = 0; k < len / 2; k++) {
- const ur = re[i + k]!;
- const ui = im[i + k]!;
- const vr = re[i + k + len / 2]! * cr - im[i + k + len / 2]! * ci;
- const vi = re[i + k + len / 2]! * ci + im[i + k + len / 2]! * cr;
- re[i + k] = ur + vr;
- im[i + k] = ui + vi;
- re[i + k + len / 2] = ur - vr;
- im[i + k + len / 2] = ui - vi;
- const ncr = cr * wr - ci * wi;
- ci = cr * wi + ci * wr;
- cr = ncr;
- }
- }
- }
-}
-
function bandPower(freqs: number[], power: number[], center: number): number {
const lo = center / Math.pow(2, 1 / 6);
const hi = center * Math.pow(2, 1 / 6);
diff --git a/packages/core/src/fft.ts b/packages/core/src/fft.ts
new file mode 100644
index 0000000000..3ca509a359
--- /dev/null
+++ b/packages/core/src/fft.ts
@@ -0,0 +1,35 @@
+/** In-place iterative radix-2 FFT. */
+export function fft(re: Float64Array, im: Float64Array): void {
+ const n = re.length;
+ for (let i = 1, j = 0; i < n; i++) {
+ let bit = n >> 1;
+ for (; j & bit; bit >>= 1) j ^= bit;
+ j ^= bit;
+ if (i < j) {
+ [re[i], re[j]] = [re[j]!, re[i]!];
+ [im[i], im[j]] = [im[j]!, im[i]!];
+ }
+ }
+ for (let len = 2; len <= n; len <<= 1) {
+ const ang = (-2 * Math.PI) / len;
+ const wr = Math.cos(ang);
+ const wi = Math.sin(ang);
+ for (let i = 0; i < n; i += len) {
+ let cr = 1;
+ let ci = 0;
+ for (let k = 0; k < len / 2; k++) {
+ const ur = re[i + k]!;
+ const ui = im[i + k]!;
+ const vr = re[i + k + len / 2]! * cr - im[i + k + len / 2]! * ci;
+ const vi = re[i + k + len / 2]! * ci + im[i + k + len / 2]! * cr;
+ re[i + k] = ur + vr;
+ im[i + k] = ui + vi;
+ re[i + k + len / 2] = ur - vr;
+ im[i + k + len / 2] = ui - vi;
+ const ncr = cr * wr - ci * wi;
+ ci = cr * wi + ci * wr;
+ cr = ncr;
+ }
+ }
+ }
+}