Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

10.0 KB
Raw Blame
/**
 * Loudness fingerprints of a stretch of a song (#9266).
 *
 * A lip-sync take is valid for the audio it was cut from, not for the whole
 * file: a re-mastered song that only changes bars outside a take's window must
 * not invalidate the take. The fingerprint is a 100 Hz RMS envelope in integer
 * dBFS (−90..0), laid on the SONG's own frame grid (frame k covers
 * [k/100, (k+1)/100) s), so one decode of a master serves every take and two
 * fingerprints of the same song time line up frame for frame.
 *
 * Stored shape: `{ version, rateHz, startFrame, db }` where `db` is base64 of an
 * Int8Array — at most 3,000 bytes for a 30 s window.
 *
 * What it answers is "did the timing of this window's sound change?" — which
 * is what mouth motion follows (syllable onsets, holds, breaths). A change that
 * keeps the 10 ms loudness contour intact to r ≥ 0.95 and ≤ 1.5 dB (pitch
 * correction, an EQ tweak, a re-encode) is deliberately NOT a change. It is not
 * a content hash: a different vocal performed with an identical 10 ms
 * dynamic contour would pass, which no real re-master produces.
 */

import { spawn } from './childProcess.js';
import { findFfmpeg } from './ffmpeg.js';
import { safeChildProcessOptions } from './processEnv.js';

export const AUDIO_FINGERPRINT_VERSION = 1;
export const FINGERPRINT_RATE_HZ = 100;
const DECODE_RATE_HZ = 8000;
const SAMPLES_PER_FRAME = DECODE_RATE_HZ / FINGERPRINT_RATE_HZ;
const FLOOR_DB = -90;
// Below this level the envelope is codec noise and dither, which a re-encode
// moves by tens of dB without anything audible changing — compare above it.
const COMPARE_FLOOR_DB = -60;
// A window whose envelope barely moves (sustained tone, silence) has no shape
// to correlate; the mean level difference alone judges it.
const MIN_SHAPE_STD_DB = 1;
// A dense, compressed chorus has little loudness shape (σ of a few dB), so a
// plain re-encode's ±1 dB frame jitter alone pulls r to 0.976–0.978 while the
// mean moves only 0.2–0.3 dB (measured on a real re-mastered song whose
// windows were otherwise identical). 0.95 keeps those; a real edit is caught
// by the mean-delta and local-change rules, which a re-encode never trips.
export const FINGERPRINT_MIN_CORRELATION = 0.95;
export const FINGERPRINT_MAX_MEAN_DELTA_DB = 1.5;
// A window-wide mean dilutes a short local change (a silenced second of a
// 30 s steady window moves the mean ~1.3 dB), so a quarter second of frames
// that each moved by 10 dB or more is a change on its own.
const LOCAL_CHANGE_DB = 10;
const LOCAL_CHANGE_FRAMES = 25;

/**
 * Decode `path` to 8 kHz mono and return its whole 100 Hz RMS envelope as an
 * Int8Array of dBFS. The PCM is folded into frames as it streams, so memory is
 * one byte per 10 ms whatever the song length. Throws when ffmpeg is missing
 * or the decode fails.
 */
export async function computeRmsEnvelope(path) {
  const ffmpeg = await findFfmpeg();
  if (!ffmpeg) throw new Error('ffmpeg is required to fingerprint audio');
  return new Promise((resolve, reject) => {
    const proc = spawn(ffmpeg, [
      '-v', 'error', '-i', path, '-vn', '-ac', '1', '-ar', String(DECODE_RATE_HZ), '-f', 's16le', '-acodec', 'pcm_s16le', 'pipe:1',
    ], safeChildProcessOptions({ stdio: ['ignore', 'pipe', 'pipe'] }));
    const frames = [];
    let sumSq = 0;
    let count = 0;
    let carry = null;
    let stderr = '';
    const pushFrame = () => {
      const rms = Math.sqrt(sumSq / count) / 32768;
      frames.push(rms > 0 ? Math.max(FLOOR_DB, Math.min(0, Math.round(20 * Math.log10(rms)))) : FLOOR_DB);
      sumSq = 0;
      count = 0;
    };
    proc.stdout.on('data', (chunk) => {
      const buf = carry ? Buffer.concat([carry, chunk]) : chunk;
      const whole = buf.length - (buf.length % 2);
      carry = whole < buf.length ? buf.subarray(whole) : null;
      for (let at = 0; at < whole; at += 2) {
        const s = buf.readInt16LE(at);
        sumSq += s * s;
        if (++count === SAMPLES_PER_FRAME) pushFrame();
      }
    });
    proc.stderr.on('data', (chunk) => { stderr = (stderr + chunk.toString()).slice(-500); });
    proc.on('error', reject);
    proc.on('close', (code) => {
      if (code !== 0) { reject(new Error(`ffmpeg could not decode the audio: ${stderr.trim() || `exit ${code}`}`)); return; }
      resolve(Int8Array.from(frames));
    });
  });
}

// Whole frames inside [startSec, endSec).
const windowFrames = ({ startSec, endSec }) => {
  const start = Math.ceil(startSec * FINGERPRINT_RATE_HZ - 1e-6);
  const end = Math.floor(endSec * FINGERPRINT_RATE_HZ + 1e-6);
  return { start, end };
};

/** The stored fingerprint of `window` cut from a whole-song envelope, or null when the song is shorter than the window. */
export function windowFingerprint(envelope, window) {
  const { start, end } = windowFrames(window);
  if (!(end > start) || end > envelope.length) return null;
  const db = envelope.slice(start, end);
  return {
    version: AUDIO_FINGERPRINT_VERSION,
    rateHz: FINGERPRINT_RATE_HZ,
    startFrame: start,
    db: Buffer.from(db.buffer, db.byteOffset, db.byteLength).toString('base64'),
  };
}

const decodeFingerprint = (fingerprint) => {
  if (fingerprint?.version !== AUDIO_FINGERPRINT_VERSION || fingerprint.rateHz !== FINGERPRINT_RATE_HZ) return null;
  if (!Number.isInteger(fingerprint.startFrame) || fingerprint.startFrame < 0 || typeof fingerprint.db !== 'string') return null;
  const bytes = Buffer.from(fingerprint.db, 'base64');
  return bytes.length > 0 ? new Int8Array(bytes.buffer, bytes.byteOffset, bytes.length) : null;
};

/**
 * Does `envelope` (a whole song) still carry the audio a stored fingerprint
 * recorded? Returns `{ same, correlation, meanDeltaDb }`; `same` is false when
 * the fingerprint is unreadable or the song no longer reaches the window.
 * Pearson correlation below 0.95, a mean |ΔdB| above 1.5 dB, or a quarter
 * second moved by 10 dB or more is a change.
 */
export function compareWindowFingerprint(fingerprint, envelope) {
  const stored = decodeFingerprint(fingerprint);
  if (!stored || fingerprint.startFrame + stored.length > envelope.length) return { same: false, correlation: null, meanDeltaDb: null };
  const n = stored.length;
  const a = new Float64Array(n);
  const b = new Float64Array(n);
  let sumA = 0;
  let sumB = 0;
  let sumDelta = 0;
  let run = 0;
  let localChange = false;
  for (let i = 0; i < n; i++) {
    a[i] = Math.max(COMPARE_FLOOR_DB, stored[i]);
    b[i] = Math.max(COMPARE_FLOOR_DB, envelope[fingerprint.startFrame + i]);
    sumA += a[i];
    sumB += b[i];
    const delta = Math.abs(a[i] - b[i]);
    sumDelta += delta;
    run = delta >= LOCAL_CHANGE_DB ? run + 1 : 0;
    if (run >= LOCAL_CHANGE_FRAMES) localChange = true;
  }
  const meanA = sumA / n;
  const meanB = sumB / n;
  let cov = 0;
  let varA = 0;
  let varB = 0;
  for (let i = 0; i < n; i++) {
    cov += (a[i] - meanA) * (b[i] - meanB);
    varA += (a[i] - meanA) ** 2;
    varB += (b[i] - meanB) ** 2;
  }
  const shaped = Math.sqrt(varA / n) >= MIN_SHAPE_STD_DB && Math.sqrt(varB / n) >= MIN_SHAPE_STD_DB;
  const correlation = shaped ? cov / Math.sqrt(varA * varB) : null;
  const meanDeltaDb = sumDelta / n;
  const same = !localChange && meanDeltaDb <= FINGERPRINT_MAX_MEAN_DELTA_DB && (correlation == null || correlation >= FINGERPRINT_MIN_CORRELATION);
  return { same, correlation, meanDeltaDb };
}

// A lip-sync provider's lead padding is a fraction of a second; a clip that
// starts early is rarer and smaller. Searching only this span keeps one
// syllable from matching a repeat of itself a bar away.
const OFFSET_MIN_LAG_SEC = -0.5;
const OFFSET_MAX_LAG_SEC = 1.5;
// Below this the two contours are not the same performance shifted in time.
export const OFFSET_MIN_CORRELATION = 0.6;

/**
 * Where `reference`'s sound sits inside `candidate`: the lag (seconds) at which
 * `candidate[t + lag]` best matches `reference[t]`, by Pearson correlation of
 * the two 100 Hz envelopes above the codec-noise floor, refined to a fraction
 * of a frame. Positive = the candidate's sound starts later (leading padding).
 * Returns `{ offsetSec, correlation }`, or null when nothing matches well
 * enough (a flat or silent window, a different performance).
 */
export function envelopeOffsetSec(reference, candidate, { minLagSec = OFFSET_MIN_LAG_SEC, maxLagSec = OFFSET_MAX_LAG_SEC } = {}) {
  const ref = Float64Array.from(reference, (v) => Math.max(COMPARE_FLOOR_DB, v));
  const cand = Float64Array.from(candidate, (v) => Math.max(COMPARE_FLOOR_DB, v));
  const minOverlap = Math.max(FINGERPRINT_RATE_HZ, Math.ceil(ref.length * 0.6));
  const scores = new Map();
  const score = (lag) => {
    const from = Math.max(0, -lag);
    const to = Math.min(ref.length, cand.length - lag);
    const n = to - from;
    if (n < minOverlap) return null;
    let sumA = 0;
    let sumB = 0;
    for (let i = from; i < to; i++) { sumA += ref[i]; sumB += cand[i + lag]; }
    const meanA = sumA / n;
    const meanB = sumB / n;
    let cov = 0;
    let varA = 0;
    let varB = 0;
    for (let i = from; i < to; i++) {
      const a = ref[i] - meanA;
      const b = cand[i + lag] - meanB;
      cov += a * b;
      varA += a * a;
      varB += b * b;
    }
    if (Math.sqrt(varA / n) < MIN_SHAPE_STD_DB || Math.sqrt(varB / n) < MIN_SHAPE_STD_DB) return null;
    return cov / Math.sqrt(varA * varB);
  };
  let best = null;
  for (let lag = Math.round(minLagSec * FINGERPRINT_RATE_HZ); lag <= Math.round(maxLagSec * FINGERPRINT_RATE_HZ); lag++) {
    const r = score(lag);
    scores.set(lag, r);
    if (r != null && (best == null || r > best.r)) best = { lag, r };
  }
  if (!best || best.r < OFFSET_MIN_CORRELATION) return null;
  // Parabolic peak through the neighbors: the envelope is 10 ms frames, a
  // video frame is ~42 ms, so the refinement only keeps rounding honest.
  const left = scores.get(best.lag - 1);
  const right = scores.get(best.lag + 1);
  const curve = left != null && right != null ? left - 2 * best.r + right : 0;
  const shift = curve < 0 ? Math.max(-0.5, Math.min(0.5, (left - right) / (2 * curve))) : 0;
  return { offsetSec: Math.round((best.lag + shift) / FINGERPRINT_RATE_HZ * 1000) / 1000, correlation: Math.round(best.r * 1000) / 1000 };
}