Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

22.0 KB
Raw Blame
// Reference-audio analysis (#2106) — pure helpers behind the Rounds
// "analyze a reference's attached audio" view. Three concerns, all
// side-effect-free and unit-tested (no Web Audio here — callers hand us the
// decoded Float32 samples):
//
//   1. extractPitchTrack — offline f0 extraction over decoded PCM: frame the
//      samples, run `detectFrequency` (the shared McLeod/NSDF core the live
//      tuner uses) per frame, and emit the same `[{ tMs, hz, clarity }]` track
//      shape `singToScore` consumes. Runs much faster than realtime because
//      it never waits on an analyser loop.
//   2. proposeSegmentScore — the solo-segment pipeline: extract a track from a
//      labeled time range, transcribe it through `transcribePitchTrack`
//      (segment → quantize to the round's tempo grid → key-aware spelling),
//      and wrap the body in a score header so it renders/plays standalone.
//   3. diffScoreBars — the review-side comparison: per-bar pitch-class
//      sequences of a proposed part vs the stored `scorePart`, so the UI can
//      highlight exactly which bars disagree before the user applies anything.
//
// Pure local DSP — zero LLM calls (the AI Provider Usage Policy applies to the
// feature this backs; nothing here touches a provider).

import { detectFrequency } from './pitchDetect.js';
import { transcribePitchTrack, DEFAULT_CLARITY_THRESHOLD } from './singToScore.js';
import { parseScore } from './scoreNotation.js';

// === Offline pitch extraction ==========================================

// Analysis window per frame. 2048 samples spans ≥2 periods of the lowest
// searched pitch at both common rates (16 kHz mic captures, 44.1/48 kHz file
// uploads), matching the live tuner's analyser fftSize.
export const DEFAULT_FRAME_SIZE = 2048;
// Hop between frames, in ms — ~25 ms (~40 tracks/sec) resolves note onsets
// well below singToScore's 70 ms minimum-note floor without paying for the
// O(frame·lag) NSDF on every sample.
export const DEFAULT_HOP_MS = 25;

/**
 * Extract a pitch track from decoded mono PCM — the offline counterpart of the
 * live `createPitchTracker`, producing the exact frame shape `singToScore`
 * consumes. Unclear/silent frames are kept (hz: null) so rests survive into
 * segmentation.
 *
 * @param {Float32Array|number[]} samples — mono PCM (one AudioBuffer channel).
 * @param {number} sampleRate — samples per second of `samples`.
 * @param {object} [opts]
 * @param {number} [opts.startMs=0] — analyze from this offset.
 * @param {number} [opts.endMs] — analyze up to this offset (default: the end).
 * @param {number} [opts.frameSize] — analysis window in samples.
 * @param {number} [opts.hopMs] — hop between frames in ms.
 * @param {number} [opts.minHz=70] / [opts.maxHz=1200] — vocal search band
 *   (the live tracker's defaults).
 * @returns {Array<{tMs:number, hz:number|null, clarity:number}>} — `tMs` is
 *   relative to `startMs`, so a segment's track starts at 0.
 */
export const extractPitchTrack = (samples, sampleRate, opts = {}) => {
  const {
    startMs = 0,
    endMs = null,
    frameSize = DEFAULT_FRAME_SIZE,
    hopMs = DEFAULT_HOP_MS,
    minHz = 70,
    maxHz = 1200,
  } = opts;
  const total = samples?.length || 0;
  if (!total || !Number.isFinite(sampleRate) || sampleRate <= 0) return [];

  const startSample = Math.max(0, Math.floor((startMs / 1000) * sampleRate));
  const endSample = endMs != null
    ? Math.min(total, Math.ceil((endMs / 1000) * sampleRate))
    : total;
  const hop = Math.max(1, Math.round((hopMs / 1000) * sampleRate));

  const track = [];
  for (let pos = startSample; pos + frameSize <= endSample; pos += hop) {
    const frame = samples.subarray
      ? samples.subarray(pos, pos + frameSize)
      : samples.slice(pos, pos + frameSize);
    const res = detectFrequency(frame, { sampleRate, minHz, maxHz });
    track.push({
      tMs: Math.round(((pos - startSample) / sampleRate) * 1000),
      hz: res ? res.hz : null,
      clarity: res ? res.clarity : 0,
    });
  }
  return track;
};

// === Score-text assembly ================================================

/**
 * Wrap a transcribed measure body in a lead-sheet header (key/tempo/time) so
 * the proposed part renders and plays standalone through ScoreSheet /
 * parseScore, matching the header shape the stored scoreParts carry. Returns
 * '' when there's no body (nothing was sung in the segment).
 */
export const buildScoreText = ({ key, tempo, beatsPerBar = 4, beatValue = 4, body } = {}) => {
  if (!body) return '';
  const lines = [];
  if (key) lines.push(`key: ${key}`);
  if (Number.isFinite(tempo) && tempo > 0) lines.push(`tempo: ${tempo}`);
  if (beatsPerBar !== 4 || beatValue !== 4) lines.push(`time: ${beatsPerBar}/${beatValue}`);
  lines.push('', body);
  return lines.join('\n').trimStart();
};

// Reference audio (a phone speaker re-recorded through a mic, or a screen
// recording's compressed track) is noisier than a direct vocal take, so the
// solo-segment pipeline defaults to a slightly laxer clarity gate than the
// live sing-to-score threshold — strict enough to reject hiss, loose enough
// that a clean sung note through a speaker still registers.
export const REFERENCE_CLARITY_THRESHOLD = 0.75;

/**
 * Solo-segment extraction (#2106 Stage 3): pitch-track a labeled time range of
 * decoded reference audio and transcribe it into a proposed lead-sheet score
 * on the round's tempo/key grid. Pure DSP — no LLM.
 *
 * @param {Float32Array} samples — mono PCM of the WHOLE reference audio.
 * @param {number} sampleRate
 * @param {object} opts
 * @param {number} opts.startMs / {number} opts.endMs — the segment bounds.
 * @param {number} [opts.bpm] — the round's tempo (quantization grid).
 * @param {string} [opts.key] — the round's key (enharmonic spelling + header).
 * @param {number} [opts.beatsPerBar=4] / [opts.beatValue=4]
 * @param {number} [opts.clarityThreshold] — override the reference gate.
 * @returns {{ track: Array, body: string, text: string }} — the raw pitch
 *   track (for debugging/inspection), the bare measure body, and the
 *   header-wrapped score text ready for ScoreSheet / scoreParts.
 */
export const proposeSegmentScore = (samples, sampleRate, opts = {}) => {
  const {
    startMs = 0, endMs = null, bpm, key,
    beatsPerBar = 4, beatValue = 4,
    clarityThreshold = REFERENCE_CLARITY_THRESHOLD,
  } = opts;
  const track = extractPitchTrack(samples, sampleRate, { startMs, endMs });
  const body = transcribePitchTrack(track, {
    bpm, key, beatsPerBar, beatValue,
    segmentOpts: { clarityThreshold: clarityThreshold ?? DEFAULT_CLARITY_THRESHOLD },
  });
  return { track, body, text: buildScoreText({ key, tempo: bpm, beatsPerBar, beatValue, body }) };
};

// === Stacked-mix extraction (spectral diff, #2121) ======================
//
// Layered TikTok builds often never expose a new voice ALONE — it enters over
// a mix that already carries the earlier layers. The solo pipeline above can't
// touch those. This section extracts the NEW voice from the stacked segment by
// spectral subtraction: average the magnitude spectrum of a "backing" window
// taken just BEFORE the voice enters (the layers already present), subtract
// that static profile from each frame of the "after" window, and run
// frequency-domain f0 estimation on the residual — which is dominated by the
// newly-added harmonic series. The resulting `[{ tMs, hz, clarity }]` track
// feeds the exact same `transcribePitchTrack` half `proposeSegmentScore` uses.
//
// Still pure local DSP — a hand-rolled radix-2 FFT (PortOS keeps its audio
// stack library-free, same rationale as `pitchDetect.js`'s NSDF core) and a
// weighted harmonic-sum pitch estimator. Zero LLM calls.

// Spectral analysis window. A power of two is required by the radix-2 FFT; 2048
// spans ≥2 periods of the lowest searched pitch at both common rates and gives
// ~7–21 Hz bins (16–48 kHz) — coarse alone, but the harmonic-sum estimator
// interpolates between bins over a fine candidate grid, so effective f0
// resolution is far finer than the bin spacing.
export const DEFAULT_FFT_SIZE = 2048;

// Spectral subtraction can only estimate the backing profile; residual hiss and
// reverb tails leave the extracted voice noisier than a clean solo take, so the
// stacked pipeline defaults to a laxer clarity gate than the solo threshold.
export const STACKED_CLARITY_THRESHOLD = 0.55;

// In-place iterative radix-2 Cooley–Tukey FFT over parallel real/imag arrays
// (length must be a power of two). Decimation-in-time with an initial
// bit-reversal permutation — the textbook form, kept tiny and dependency-free.
const fftInPlace = (re, im) => {
  const n = re.length;
  // Bit-reversal permutation.
  for (let i = 1, j = 0; i < n; i++) {
    let bit = n >> 1;
    for (; j & bit; bit >>= 1) j ^= bit;
    j ^= bit;
    if (i < j) {
      const tr = re[i]; re[i] = re[j]; re[j] = tr;
      const ti = im[i]; im[i] = im[j]; im[j] = ti;
    }
  }
  // Butterflies, doubling the transform length each stage.
  for (let len = 2; len <= n; len <<= 1) {
    const ang = (-2 * Math.PI) / len;
    const wRe = Math.cos(ang);
    const wIm = Math.sin(ang);
    for (let start = 0; start < n; start += len) {
      let curRe = 1;
      let curIm = 0;
      for (let k = 0; k < len >> 1; k++) {
        const i0 = start + k;
        const i1 = i0 + (len >> 1);
        const tRe = re[i1] * curRe - im[i1] * curIm;
        const tIm = re[i1] * curIm + im[i1] * curRe;
        re[i1] = re[i0] - tRe;
        im[i1] = im[i0] - tIm;
        re[i0] += tRe;
        im[i0] += tIm;
        const nextRe = curRe * wRe - curIm * wIm;
        curIm = curRe * wIm + curIm * wRe;
        curRe = nextRe;
      }
    }
  }
};

/**
 * Hann-windowed magnitude spectrum of a real frame. Returns the first
 * `fftSize/2` magnitude bins (bin b ↔ b·sampleRate/fftSize Hz). The window
 * tapers frame edges so a note that doesn't align to the frame boundary
 * doesn't smear energy across the whole spectrum. Returns null if the frame is
 * too short for the requested (power-of-two) size.
 *
 * @param {Float32Array|number[]} frame — real PCM samples (length ≥ fftSize).
 * @param {object} [opts]
 * @param {number} [opts.fftSize] — power-of-two transform length.
 * @returns {Float32Array|null} — `fftSize/2` magnitudes, or null.
 */
export const magnitudeSpectrum = (frame, { fftSize = DEFAULT_FFT_SIZE } = {}) => {
  const size = frame?.length || 0;
  if (size < fftSize || fftSize < 2 || (fftSize & (fftSize - 1)) !== 0) return null;
  const re = new Float64Array(fftSize);
  const im = new Float64Array(fftSize);
  for (let i = 0; i < fftSize; i++) {
    // Hann window: 0.5·(1 − cos(2πi/(N−1))).
    const w = 0.5 * (1 - Math.cos((2 * Math.PI * i) / (fftSize - 1)));
    re[i] = frame[i] * w;
  }
  fftInPlace(re, im);
  const half = fftSize >> 1;
  const mag = new Float32Array(half);
  for (let b = 0; b < half; b++) mag[b] = Math.hypot(re[b], im[b]);
  return mag;
};

/**
 * Average Hann-windowed magnitude spectrum across the frames of a time window —
 * the static "backing" profile for spectral subtraction. Slides `fftSize`-wide
 * frames by `hopMs` over [startMs, endMs) and means the per-frame magnitudes.
 *
 * @returns {Float32Array|null} — mean magnitude bins, or null when the window
 *   holds no full frame (too short to analyze — the caller degrades to no
 *   subtraction rather than crashing).
 */
export const averageMagnitudeSpectrum = (samples, sampleRate, opts = {}) => {
  const {
    startMs = 0, endMs = null,
    fftSize = DEFAULT_FFT_SIZE, hopMs = DEFAULT_HOP_MS,
  } = opts;
  const total = samples?.length || 0;
  if (!total || !Number.isFinite(sampleRate) || sampleRate <= 0) return null;
  const startSample = Math.max(0, Math.floor((startMs / 1000) * sampleRate));
  const endSample = endMs != null
    ? Math.min(total, Math.ceil((endMs / 1000) * sampleRate))
    : total;
  const hop = Math.max(1, Math.round((hopMs / 1000) * sampleRate));

  const half = fftSize >> 1;
  const sum = new Float64Array(half);
  let frames = 0;
  for (let pos = startSample; pos + fftSize <= endSample; pos += hop) {
    const frame = samples.subarray
      ? samples.subarray(pos, pos + fftSize)
      : samples.slice(pos, pos + fftSize);
    const mag = magnitudeSpectrum(frame, { fftSize });
    if (!mag) continue;
    for (let b = 0; b < half; b++) sum[b] += mag[b];
    frames += 1;
  }
  if (!frames) return null;
  const avg = new Float32Array(half);
  for (let b = 0; b < half; b++) avg[b] = sum[b] / frames;
  return avg;
};

// Linear-interpolated magnitude at an arbitrary frequency (bins are discrete;
// a candidate f0 and its harmonics rarely land on a bin center). Out-of-range
// frequencies read as 0.
const interpMagAt = (mag, hz, sampleRate, fftSize) => {
  const bin = (hz * fftSize) / sampleRate;
  if (bin < 0 || bin >= mag.length - 1) return 0;
  const lo = Math.floor(bin);
  const frac = bin - lo;
  return mag[lo] * (1 - frac) + mag[lo + 1] * frac;
};

// Median of a Float32Array/number list (copy-sorted; small arrays here).
const medianOf = (values) => {
  if (!values.length) return 0;
  const sorted = Array.prototype.slice.call(values).sort((a, b) => a - b);
  const mid = sorted.length >> 1;
  return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2;
};

/**
 * Estimate the fundamental frequency of a magnitude spectrum by weighted
 * harmonic sum. For each candidate f0 on a ~5-cent grid across [minHz, maxHz]
 * we sum interpolated magnitudes at its harmonics k·f0 (k = 1…H, below
 * Nyquist) weighted by 1/k. The 1/k weighting biases toward the TRUE
 * fundamental over its octave errors: f0/2 misses the strong first harmonic
 * (its k=1 term lands on empty spectrum) and 2·f0 drops the fundamental term
 * entirely, so both usually score below the real f0 — the frequency-domain
 * analogue of the NSDF core's first-strong-peak trick. It is a bias, not a
 * guarantee: when spectral subtraction has gutted the fundamental bin (backing
 * energy overlapped it), 2·f0 can still win and the note reads an octave high —
 * which is why the result is a reviewable/editable PROPOSAL, not an auto-apply.
 * `clarity` is the residual prominence of the winning
 * candidate over the grid's median score, in [0, 1]: a clean harmonic series
 * towers over a mostly-empty grid (→ 1); broadband noise scores flat (→ 0).
 *
 * @returns {{ hz:number, clarity:number }|null} — null for an empty/flat
 *   spectrum with no candidate carrying energy.
 */
export const estimateSpectralF0 = (mag, sampleRate, fftSize, opts = {}) => {
  const { minHz = 70, maxHz = 1200, maxHarmonics = 8 } = opts;
  if (!mag?.length || !Number.isFinite(sampleRate) || sampleRate <= 0) return null;
  const nyquist = sampleRate / 2;
  const hiHz = Math.min(maxHz, nyquist);
  if (minHz <= 0 || hiHz <= minHz) return null;

  // ~5-cent candidate spacing (2^(5/1200)) — finer than a semitone by 20×,
  // far below the note-onset resolution the transcription grid needs.
  const step = Math.pow(2, 5 / 1200);
  const scores = [];
  let best = null;
  for (let f = minHz; f <= hiHz; f *= step) {
    let s = 0;
    for (let k = 1; k <= maxHarmonics; k++) {
      const hk = k * f;
      if (hk >= nyquist) break;
      s += interpMagAt(mag, hk, sampleRate, fftSize) / k;
    }
    scores.push(s);
    if (!best || s > best.score) best = { hz: f, score: s };
  }
  if (!best || best.score <= 0) return null;
  const floor = medianOf(scores);
  const clarity = Math.max(0, Math.min(1, 1 - floor / best.score));
  return { hz: best.hz, clarity };
};

/**
 * Extract a pitch track for a NEW voice that enters over a stacked mix (#2121).
 * Builds a backing magnitude profile from [bgStartMs, bgEndMs) — a slice just
 * before the voice enters, carrying only the earlier layers — then, per frame
 * of the [startMs, endMs) "after" window, subtracts that profile
 * (over-subtraction factor `overSubtract`, rectified at 0) and runs
 * `estimateSpectralF0` on the residual. Emits the same segment-relative
 * `[{ tMs, hz, clarity }]` shape as `extractPitchTrack`, so it drops straight
 * into `transcribePitchTrack`. With no usable backing window it degrades to
 * plain spectral f0 on the mix (no subtraction) rather than failing.
 *
 * @param {Float32Array} samples — mono PCM of the WHOLE reference audio.
 * @param {number} sampleRate
 * @param {object} opts
 * @param {number} opts.startMs / {number} opts.endMs — the stacked segment.
 * @param {number} [opts.bgStartMs] / {number} [opts.bgEndMs] — backing window.
 * @param {number} [opts.overSubtract=1] — spectral-subtraction strength.
 * @param {number} [opts.fftSize] / {number} [opts.hopMs] / {number} [opts.minHz] / {number} [opts.maxHz]
 * @returns {Array<{tMs:number, hz:number|null, clarity:number}>}
 */
export const extractSpectralDiffTrack = (samples, sampleRate, opts = {}) => {
  const {
    startMs = 0, endMs = null,
    bgStartMs = null, bgEndMs = null,
    fftSize = DEFAULT_FFT_SIZE, hopMs = DEFAULT_HOP_MS,
    minHz = 70, maxHz = 1200, overSubtract = 1,
  } = opts;
  const total = samples?.length || 0;
  if (!total || !Number.isFinite(sampleRate) || sampleRate <= 0) return [];

  // Backing profile from the pre-entrance window (null → no subtraction).
  const backing = (bgStartMs != null && bgEndMs != null && bgEndMs > bgStartMs)
    ? averageMagnitudeSpectrum(samples, sampleRate, { startMs: bgStartMs, endMs: bgEndMs, fftSize, hopMs })
    : null;

  const startSample = Math.max(0, Math.floor((startMs / 1000) * sampleRate));
  const endSample = endMs != null
    ? Math.min(total, Math.ceil((endMs / 1000) * sampleRate))
    : total;
  const hop = Math.max(1, Math.round((hopMs / 1000) * sampleRate));
  const half = fftSize >> 1;

  const track = [];
  for (let pos = startSample; pos + fftSize <= endSample; pos += hop) {
    const frame = samples.subarray
      ? samples.subarray(pos, pos + fftSize)
      : samples.slice(pos, pos + fftSize);
    const mag = magnitudeSpectrum(frame, { fftSize });
    const tMs = Math.round(((pos - startSample) / sampleRate) * 1000);
    if (!mag) { track.push({ tMs, hz: null, clarity: 0 }); continue; }
    // Spectral subtraction: rectify (max 0) so removed bins can't go negative.
    if (backing) {
      for (let b = 0; b < half; b++) {
        const r = mag[b] - overSubtract * backing[b];
        mag[b] = r > 0 ? r : 0;
      }
    }
    const res = estimateSpectralF0(mag, sampleRate, fftSize, { minHz, maxHz });
    track.push({ tMs, hz: res ? res.hz : null, clarity: res ? res.clarity : 0 });
  }
  return track;
};

/**
 * Stacked-mix counterpart of `proposeSegmentScore` (#2121): pitch-track the new
 * voice out of a stacked segment via spectral diff, then transcribe it onto the
 * round's tempo/key grid. Pure DSP — no LLM.
 *
 * @param {Float32Array} samples — mono PCM of the WHOLE reference audio.
 * @param {number} sampleRate
 * @param {object} opts — the `extractSpectralDiffTrack` bounds plus
 *   `bpm`/`key`/`beatsPerBar`/`beatValue` (quantization + spelling) and an
 *   optional `clarityThreshold` override.
 * @returns {{ track: Array, body: string, text: string }}
 */
export const proposeStackedSegmentScore = (samples, sampleRate, opts = {}) => {
  const {
    startMs = 0, endMs = null, bgStartMs = null, bgEndMs = null,
    bpm, key, beatsPerBar = 4, beatValue = 4,
    overSubtract = 1,
    clarityThreshold = STACKED_CLARITY_THRESHOLD,
  } = opts;
  const track = extractSpectralDiffTrack(samples, sampleRate, {
    startMs, endMs, bgStartMs, bgEndMs, overSubtract,
  });
  const body = transcribePitchTrack(track, {
    bpm, key, beatsPerBar, beatValue,
    segmentOpts: { clarityThreshold: clarityThreshold ?? DEFAULT_CLARITY_THRESHOLD },
  });
  return { track, body, text: buildScoreText({ key, tempo: bpm, beatsPerBar, beatValue, body }) };
};

// === Proposed-vs-existing bar diff ======================================

// Note letter → pitch class (semitones above C), and accidental → shift.
// Mirrors pitchDetect's tables (kept local — they aren't exported there).
const LETTER_PC = { C: 0, D: 2, E: 4, F: 5, G: 7, A: 9, B: 11 };
const ACCIDENTAL_SHIFT = { '': 0, n: 0, '#': 1, '##': 2, b: -1, bb: -2 };

// Display names for pitch classes (sharp spelling — labels only; the score
// text itself is already spelled from the key signature). Re-export of the
// canonical table in pianoKeyboard.js so note and chord labels can't drift
// apart (#2477 promoted the shared constant).
export { PITCH_CLASS_NAMES } from './pianoKeyboard.js';

// Pitch class (0–11) of a parsed note's pitch, or null when unmappable.
const notePitchClass = (pitch) => {
  const base = LETTER_PC[pitch?.letter];
  const shift = ACCIDENTAL_SHIFT[pitch?.accidental || ''];
  if (base == null || shift == null) return null;
  return ((base + shift) % 12 + 12) % 12;
};

/**
 * Per-bar pitch-class sequences of a score — one array of pitch classes
 * (rests skipped, octaves collapsed) per measure. The octave collapse is
 * deliberate: a reference singer and the stored part may sit an octave apart
 * while still being "the same part".
 */
export const barPitchClasses = (scoreText) =>
  parseScore(scoreText).measures.map((m) =>
    m.notes.filter((n) => !n.rest).map((n) => notePitchClass(n.pitch)).filter((pc) => pc != null));

/**
 * Compare a proposed part against an existing scorePart, bar by bar (#2106
 * Stage 4). A bar matches when both scores have that measure AND the ordered
 * pitch-class sequences agree (rhythm differences within a bar don't flag —
 * the review staffs make those visible; the highlight is for wrong NOTES).
 *
 * @returns {Array<{bar:number, proposed:number[]|null, existing:number[]|null, match:boolean}>}
 *   — one row per bar of the longer score; a side missing that bar is null
 *   (and the bar can't match).
 */
export const diffScoreBars = (proposedText, existingText) => {
  const a = barPitchClasses(proposedText);
  const b = barPitchClasses(existingText);
  const bars = Math.max(a.length, b.length);
  const out = [];
  for (let i = 0; i < bars; i++) {
    const proposed = i < a.length ? a[i] : null;
    const existing = i < b.length ? b[i] : null;
    const match = proposed != null && existing != null
      && proposed.length === existing.length
      && proposed.every((pc, j) => pc === existing[j]);
    out.push({ bar: i + 1, proposed, existing, match });
  }
  return out;
};