Skip to content

Write HTML. Render video. Built for agents.

6.5 KB
Raw Blame
import {
  runLadder,
  signalDuration,
  signalProvider,
  tempoOf,
  type AnalysisQuality,
  type AudioAnalysisProvider,
  type AudioCapability,
  type AudioSignal,
  type Beats,
  type LadderResult,
  type LadderStep,
} from "@hyperframes/core/audio-analysis";
import { apiProvider } from "./providers/api.js";
import { appleMuProvider, appleSoundProvider } from "./providers/apple.js";
import { beatThisProvider } from "./providers/beatThis.js";
import { shiftedTimes } from "./fileTime.js";

/** What `hyperframes audio` asks for when `--only` is not given. */
export const DEFAULT_CAPABILITIES: readonly AudioCapability[] = [
  "kind",
  "moments",
  "onsets",
  "loudness",
  "silence",
  "envelope",
];

/**
 * Every provider the CLI knows, in default ladder order: each capability tries the ones that offer it in this order.
 * beats, downbeats: api, apple-mu, beat-this, signal. sections: api, apple-mu, signal. describe: api, apple-sound.
 */
function audioAnalysisProviders(): AudioAnalysisProvider<AudioSignal>[] {
  return [apiProvider, appleMuProvider, beatThisProvider, appleSoundProvider, signalProvider];
}

export interface AnalyzeOptions {
  capabilities: readonly AudioCapability[];
  /** Defaults to local: audio leaves the machine only when best is asked for. */
  quality?: AnalysisQuality;
  /** Pin a capability to one provider, skipping the ladder. */
  provider?: Partial<Record<AudioCapability, string>>;
  /** Per-capability options handed to the provider. */
  options?: Partial<Record<AudioCapability, Record<string, unknown>>>;
  /** Seconds into the file where the signal begins; every reported time is shifted by it. */
  fileOffset?: number;
  /** Seconds of signal around the analyzed window, there so onsets at its edges are told from the cut. */
  context?: { before: number; after: number };
}

export interface Analysis {
  /** Seconds of signal analyzed. */
  analyzed: number;
  values: Partial<Record<AudioCapability, unknown>>;
  provenance: Partial<Record<AudioCapability, string | null>>;
  ladder: Partial<Record<AudioCapability, LadderStep[]>>;
  warnings: string[];
}

/** Long signals get a coarser envelope, so one item stays near 2,000 values. */
export function envelopeHopMs(seconds: number, asked?: number): number {
  if (asked !== undefined) return asked;
  return Math.max(5, Math.ceil((seconds * 1000) / 2000));
}

const round = (t: number) => Math.round(t * 1000) / 1000;

function within(value: unknown, from: number, to: number): unknown {
  if (!value || typeof value !== "object" || !("events" in value)) return value;
  const onsets = value as { events: { t: number }[] };
  return { ...onsets, events: onsets.events.filter((e) => e.t >= from && e.t < to) };
}

interface Framing {
  /** The signal without its context, which every capability but onsets measures. */
  windowed: AudioSignal;
  /** Seconds of context before the window, and where the window ends, in signal seconds. */
  before: number;
  end: number;
  cut: boolean;
}

function frame(signal: AudioSignal, context: AnalyzeOptions["context"]): Framing {
  const before = context?.before ?? 0;
  const skip = Math.round(before * signal.sampleRate);
  const length = signal.channels[0]?.length ?? 0;
  const end = Math.max(skip, length - Math.round((context?.after ?? 0) * signal.sampleRate));
  const cut = skip > 0 || end < length;
  const windowed = cut
    ? { ...signal, channels: signal.channels.map((c) => c.subarray(skip, end)) }
    : signal;
  return { windowed, before, end: end / signal.sampleRate, cut };
}

function optionsFor(capability: AudioCapability, request: AnalyzeOptions, seconds: number) {
  const options = { ...request.options?.[capability] };
  if (capability === "envelope")
    options.hopMs = envelopeHopMs(seconds, options.hopMs as number | undefined);
  return options;
}

export async function analyzeSignal(
  signal: AudioSignal,
  request: AnalyzeOptions,
  providers: readonly AudioAnalysisProvider<AudioSignal>[] = audioAnalysisProviders(),
): Promise<Analysis> {
  const quality = request.quality ?? "local";
  const { beats, bpm } = request.provider ?? {};
  if (beats && bpm && beats !== bpm) throw new Error("beats and bpm must use the same provider");
  const beatProvider = beats ?? bpm;
  const framing = frame(signal, request.context);
  const analysis: Analysis = {
    analyzed: round(signalDuration(framing.windowed)),
    values: {},
    provenance: {},
    ladder: {},
    warnings: [],
  };
  const ladders = new Map<AudioCapability, Promise<LadderResult>>();
  const ladderFor = (capability: AudioCapability) => {
    let result = ladders.get(capability);
    if (!result) {
      const withContext = capability === "onsets" && framing.cut;
      result = runLadder(capability, withContext ? signal : framing.windowed, providers, {
        quality,
        provider: capability === "beats" ? beatProvider : request.provider?.[capability],
        options: optionsFor(capability, request, analysis.analyzed),
      });
      ladders.set(capability, result);
    }
    return result;
  };
  for (const capability of request.capabilities) {
    const withContext = capability === "onsets" && framing.cut;
    const result =
      capability === "bpm" ? tempoFromBeats(await ladderFor("beats")) : await ladderFor(capability);
    const value = withContext ? within(result.value, framing.before, framing.end) : result.value;
    const offset = (request.fileOffset ?? 0) - (withContext ? framing.before : 0);
    analysis.values[capability] = shiftedTimes(capability, value, offset);
    analysis.provenance[capability] = result.provenance;
    analysis.ladder[capability] = result.steps;
    const said = new Set(analysis.warnings);
    analysis.warnings.push(...result.warnings.filter((w) => !said.has(w)));
  }
  analysis.warnings.push(...serviceHint(quality, analysis, request.capabilities));
  return analysis;
}

function serviceHint(
  quality: AnalysisQuality,
  analysis: Analysis,
  capabilities: readonly AudioCapability[],
): string[] {
  const leftToService = (capability: AudioCapability) =>
    analysis.values[capability] == null &&
    (analysis.ladder[capability] ?? []).some((s) => s.outcome === "excluded");
  return quality === "local" && capabilities.some(leftToService)
    ? ["a service could have answered what is missing: --quality best uploads the audio to HeyGen"]
    : [];
}

/** Tempo is read off the beats of whichever provider placed them, so the two never disagree. */
function tempoFromBeats(beats: LadderResult): LadderResult {
  const times = (beats.value as Beats | null)?.times;
  return { ...beats, value: times ? tempoOf(times) : null };
}