Skip to content

Open source CapCut alternative.

TypeScript
17.7 KB
Raw Blame
#!/usr/bin/env bun
/**
 * Speech-cut eval: does the cut land where a listener wants it?
 *
 * A recut is judged by ear, and the ear is strict about one thing above all —
 * a word must finish. Half a syllable missing at a cut is heard immediately,
 * however good the rest of the edit is, and "make it fast paced" is the ask
 * that produces it: cuts get planned tight against cue timings, cue timings
 * run short of the audio, and the tail of every sentence goes with the cut.
 *
 * So the truth is built rather than measured. A take of known words renders to
 * audio, a model's cut is planned on top of it from a transcript with the
 * errors real transcripts have, and the production placement path runs over
 * both. Every number below is measured against where the words actually are:
 *
 *   clipped — speech the cut removed. Anything above zero is the bug.
 *   air     — the quiet left between the cut and the word it keeps.
 *
 * Each case reports the model's own edges and the placed ones, because the
 * placement is the thing on trial. Budgets are enforced by default; the run
 * takes a couple of seconds with no model, no network, and no credits.
 *
 *   npm run eval:cut-edges [--only <case>] [--out path] [--no-enforce] [--verbose]
 */

import { mkdir, writeFile } from "node:fs/promises";
import path from "node:path";
import { scanSpeech, type PcmChunk } from "../src/cut/lib/audioScan";
import { AIR, MIN_AIR, placeEdge, settleJoint, type Pace } from "../src/cut/lib/cutRefine";

const SITE = path.resolve(import.meta.dir, "..");
const REPORT = path.resolve(SITE, "..", "evals", "cut-edges.latest-report.json");

const argv = process.argv.slice(2);
const arg = (name: string) => {
  const i = argv.indexOf(name);
  return i >= 0 ? argv[i + 1] : undefined;
};
const ONLY = arg("--only");
const OUT = arg("--out") ?? REPORT;
const ENFORCE = !argv.includes("--no-enforce");
const VERBOSE = argv.includes("--verbose");

const RATE = 16000;

// ── Deterministic everything ────────────────────────────────────────────────
// A budget that trips one time in five is worse than no budget.
const rng = (seed: number) => () => {
  seed = (seed * 1103515245 + 12345) % 2147483648;
  return seed / 2147483648 - 0.5;
};

// ── The take ────────────────────────────────────────────────────────────────

interface Word {
  sentence: number;
  start: number;
  end: number;
}

interface Take {
  words: Word[];
  sentences: { start: number; end: number }[];
  duration: number;
}

/** A take of `sentences` sentences: words of syllable length, a breath between
 * them, a pause between sentences, and `lead` of room before anyone speaks. */
function speak(opts: {
  sentences: number;
  wordsPerSentence: number;
  wordGap: number;
  sentenceGap: number;
  lead: number;
  seed: number;
}): Take {
  const rnd = rng(opts.seed);
  const words: Word[] = [];
  const sentences: Take["sentences"] = [];
  let t = opts.lead;
  for (let si = 0; si < opts.sentences; si++) {
    const start = t;
    for (let wi = 0; wi < opts.wordsPerSentence; wi++) {
      const dur = 0.24 + Math.abs(rnd()) * 0.4;
      words.push({ sentence: si, start: t, end: t + dur });
      t += dur + (wi + 1 < opts.wordsPerSentence ? opts.wordGap : 0);
    }
    sentences.push({ start, end: t });
    t += opts.sentenceGap;
  }
  return { words, sentences, duration: t + 1 };
}

interface Room {
  floorDb: number;
  bedDb?: number;
  /** Peak amplitude of the voice — a recording nobody normalized is quiet. */
  level: number;
  seed: number;
}

/** The take as audio: a burst per word, shaped like a syllable rather than a
 * flat block, over whatever room the case asks for. */
function render(take: Take, room: Room): Float32Array {
  const rnd = rng(room.seed);
  const out = new Float32Array(Math.ceil(take.duration * RATE));
  const floor = Math.pow(10, room.floorDb / 20);
  for (let i = 0; i < out.length; i++) out[i] = rnd() * floor * 2;
  if (room.bedDb !== undefined) {
    const bed = Math.pow(10, room.bedDb / 20);
    for (let i = 0; i < out.length; i++) out[i] += Math.sin((2 * Math.PI * 110 * i) / RATE) * bed;
  }
  for (const w of take.words) {
    const from = Math.round(w.start * RATE);
    const to = Math.min(out.length, Math.round(w.end * RATE));
    const len = Math.max(1, to - from);
    for (let i = from; i < to; i++) {
      const p = (i - from) / len;
      // Attack, hold, decay — a word does not start and stop square, and the
      // decay is the part a tight cut eats.
      const env = Math.min(1, p / 0.1) * Math.min(1, (1 - p) / 0.18);
      const wobble = 0.7 + 0.3 * Math.sin(p * Math.PI * 4);
      out[i] += rnd() * room.level * env * wobble;
    }
  }
  return out;
}

/** The rendered take, handed over in uneven spans the way a decoder does. */
async function* pcm(audio: Float32Array, from: number, to: number): AsyncGenerator<PcmChunk> {
  const first = Math.max(0, Math.floor(from * RATE));
  const last = Math.min(audio.length, Math.ceil(to * RATE));
  for (let start = first; start < last; start += 4321) {
    const end = Math.min(last, start + 4321);
    yield { channels: [audio.slice(start, end)], timestamp: start / RATE, sampleRate: RATE };
  }
}

// ── The model's cut ─────────────────────────────────────────────────────────

/** How the assistant placed its own edges before anything was refined.
 *  tight — right on the cue boundary, the pace-chasing habit this eval exists
 *          for; the cue runs short of the audio, so the edge is often already
 *          inside the word.
 *  loose — a beat of spare air on each side, what the skill actually asks for.
 *  wide  — a whole second of it, from a model working off a coarse look. */
type Style = "tight" | "loose" | "wide";

const SPARE: Record<Style, number> = { tight: 0, loose: 0.35, wide: 1 };

/** An edge the model asked for, and the word it is meant to keep. */
interface Cut {
  kind: "in" | "out";
  /** Where the model put it. */
  t: number;
  /** The true boundary of the word this side keeps. */
  word: number;
}

/**
 * Cuts at every sentence joint: the end of one sentence and the start of the
 * next, as the model would place them from a transcript. Cue times carry the
 * error real transcripts carry — a word's cue ends before its sound does, and
 * the boundary jitters either way.
 */
function planCuts(take: Take, style: Style, seed: number): Cut[] {
  const rnd = rng(seed);
  const cuts: Cut[] = [];
  for (const [i, s] of take.sentences.entries()) {
    // The recognizer clips the tail of the last word and jitters the boundary.
    const jitter = () => rnd() * 0.2;
    if (i + 1 < take.sentences.length)
      cuts.push({ kind: "out", t: s.end - 0.08 + jitter() + SPARE[style], word: s.end });
    if (i > 0) cuts.push({ kind: "in", t: s.start + 0.04 + jitter() - SPARE[style], word: s.start });
  }
  return cuts;
}

// ── Scoring ─────────────────────────────────────────────────────────────────

const round3 = (n: number) => Math.round(n * 1000) / 1000;
const quantile = (xs: number[], q: number) =>
  xs.length === 0 ? 0 : round3([...xs].sort((a, b) => a - b)[Math.min(xs.length - 1, Math.floor(q * xs.length))]);

interface Score {
  edges: number;
  /** Speech the cut removed, seconds. Zero is the only good answer. */
  worstClipped: number;
  clippedEdges: number;
  /** Quiet left between the cut and the word, seconds. */
  medianAir: number;
  tightestAir: number;
  widestAir: number;
}

/** Air at one edge: how far the cut sits from the word it keeps. Negative when
 * the cut is inside the word — the measurement of a clipped word. */
const airAt = (cut: Cut, t: number) => (cut.kind === "out" ? t - cut.word : cut.word - t);

function score(cuts: Cut[], placed: number[]): Score {
  const air = cuts.map((c, i) => airAt(c, placed[i]));
  const clipped = air.map((a) => Math.max(0, -a));
  return {
    edges: cuts.length,
    worstClipped: round3(Math.max(0, ...clipped)),
    clippedEdges: clipped.filter((c) => c > 0.005).length,
    medianAir: quantile(air, 0.5),
    tightestAir: round3(Math.min(...air)),
    widestAir: round3(Math.max(...air)),
  };
}

// ── Cases ───────────────────────────────────────────────────────────────────

interface Budget {
  /** Speech may not be cut off. Kept as a number so the report shows the slack. */
  clipped: number;
  /** The quiet at the tightest edge, and at the widest. */
  minAir: number;
  maxAir: number;
}

/** The span the tool reads around one edge — REACH plus room to measure the
 * recording's levels, the same window refine_speech_cuts opens. */
const WINDOW = 3;

function refine(audio: Float32Array, duration: number, cuts: Cut[], pace: Pace): Promise<number[]> {
  return Promise.all(
    cuts.map(async (cut) => {
      const from = Math.max(0, cut.t - WINDOW);
      const to = Math.min(duration, cut.t + WINDOW);
      const scan = await scanSpeech(pcm(audio, from, to), { from, to });
      return placeEdge(scan.segments, cut.t, cut.kind, pace, scan).t;
    })
  );
}

const talk = speak({ sentences: 7, wordsPerSentence: 5, wordGap: 0.08, sentenceGap: 0.62, lead: 0.9, seed: 1 });
const rapid = speak({ sentences: 8, wordsPerSentence: 4, wordGap: 0.05, sentenceGap: 0.24, lead: 0.4, seed: 2 });
// Long beats between sentences, the take a coarse plan leaves dead air in.
const airy = speak({ sentences: 6, wordsPerSentence: 5, wordGap: 0.08, sentenceGap: 2.4, lead: 1.4, seed: 6 });

const quietRoom: Room = { floorDb: -60, level: 0.6, seed: 3 };
const loudRoom: Room = { floorDb: -38, level: 0.6, seed: 5 };
const unnormalized: Room = { floorDb: -78, level: 0.05, seed: 7 };
const musicBed: Room = { floorDb: -58, bedDb: -34, level: 0.6, seed: 9 };

interface Spec {
  name: string;
  what: string;
  take: Take;
  room: Room;
  style: Style;
  pace: Pace;
  budget: Budget;
}

const SPECS: Spec[] = [
  {
    name: "fast/tight-cuts",
    what: "The reported bug: a fast-paced recut planned right on the cue boundaries, in a quiet room. Cue times run short of the audio, so half these edges start out inside a word.",
    take: talk,
    room: quietRoom,
    style: "tight",
    pace: "fast",
    budget: { clipped: 0, minAir: MIN_AIR.in, maxAir: 0.34 },
  },
  {
    name: "fast/loud-room",
    what: "The same tight plan over a room 22dB louder — a floor close enough to the voice that a fixed threshold reads the pauses as speech.",
    take: talk,
    room: loudRoom,
    style: "tight",
    pace: "fast",
    budget: { clipped: 0, minAir: MIN_AIR.in, maxAir: 0.34 },
  },
  {
    name: "fast/unnormalized",
    what: "A recording nobody levelled: the voice sits at -37dBFS, under any fixed silence threshold worth using on a normal file.",
    take: talk,
    room: unnormalized,
    style: "tight",
    pace: "fast",
    budget: { clipped: 0, minAir: MIN_AIR.in, maxAir: 0.34 },
  },
  {
    name: "fast/music-bed",
    what: "A steady bed under the whole take: nothing is ever truly quiet, and the gaps between sentences are the only readable edges.",
    take: talk,
    room: musicBed,
    style: "tight",
    pace: "fast",
    budget: { clipped: 0, minAir: MIN_AIR.in, maxAir: 0.4 },
  },
  {
    name: "fast/rapid-cadence",
    what: "Sentences a quarter second apart, cut fast: the pause cannot hold the full air, so both sides take what there is and no word is touched.",
    take: rapid,
    room: quietRoom,
    style: "tight",
    pace: "fast",
    budget: { clipped: 0, minAir: 0.05, maxAir: 0.34 },
  },
  {
    name: "fast/loose-plan",
    what: "The skill's own instruction followed — cut loose, refine after. The air comes back to the pace from the other direction.",
    take: talk,
    room: quietRoom,
    style: "loose",
    pace: "fast",
    budget: { clipped: 0, minAir: MIN_AIR.in, maxAir: 0.34 },
  },
  {
    name: "fast/wide-plan",
    what: "Sentences held two and a half seconds apart and a second of spare air either side, from a model working off a coarse look. Dead air is the failure here, not clipping.",
    take: airy,
    room: quietRoom,
    style: "wide",
    pace: "fast",
    budget: { clipped: 0, minAir: MIN_AIR.in, maxAir: 0.34 },
  },
  {
    name: "natural/tight-cuts",
    what: "The default pace over the same tight plan — every edge a quarter second clear of the word.",
    take: talk,
    room: quietRoom,
    style: "tight",
    pace: "natural",
    budget: { clipped: 0, minAir: 0.1, maxAir: 0.45 },
  },
  {
    name: "relaxed/tight-cuts",
    what: "The slowest pace: the piece breathes, and the same edges sit further out without running into the next sentence.",
    take: talk,
    room: quietRoom,
    style: "tight",
    pace: "relaxed",
    budget: { clipped: 0, minAir: 0.12, maxAir: 0.7 },
  },
];

// ── Run ─────────────────────────────────────────────────────────────────────

interface Result {
  name: string;
  what: string;
  pace: Pace;
  before: Score;
  after: Score;
  budget: Budget;
  breaches: string[];
}

const specs = SPECS.filter((c) => !ONLY || c.name.includes(ONLY));
if (specs.length === 0) {
  console.error(`No case matches ${ONLY}. Cases: ${SPECS.map((c) => c.name).join(", ")}`);
  process.exit(2);
}

const report: Result[] = [];
for (const spec of specs) {
  const audio = render(spec.take, spec.room);
  const cuts = planCuts(spec.take, spec.style, spec.room.seed + 1);
  const placed = await refine(audio, spec.take.duration, cuts, spec.pace);
  const before = score(cuts, cuts.map((c) => c.t));
  const after = score(cuts, placed);
  const breaches: string[] = [];
  if (after.worstClipped > spec.budget.clipped)
    breaches.push(`clipped ${after.worstClipped}s of speech at ${after.clippedEdges} edge(s)`);
  if (after.tightestAir < spec.budget.minAir)
    breaches.push(`tightest edge ${after.tightestAir}s < ${spec.budget.minAir}s of air`);
  if (after.widestAir > spec.budget.maxAir)
    breaches.push(`widest edge ${after.widestAir}s > ${spec.budget.maxAir}s of air`);
  // The pass has to earn its place: placing must not leave an edge worse off
  // than the model had it.
  if (after.worstClipped > before.worstClipped)
    breaches.push(`placing made it worse: ${before.worstClipped}s → ${after.worstClipped}s clipped`);
  report.push({ name: spec.name, what: spec.what, pace: spec.pace, before, after, budget: spec.budget, breaches });
}

// One joint, both edges of a removed sliver inside a pause too short to hold
// the air either side wants. Nothing may replay across it and neither word may
// be touched — the case that used to revert both edges to the model's numbers.
const jointCase = (() => {
  const take = speak({ sentences: 2, wordsPerSentence: 4, wordGap: 0.06, sentenceGap: 0.18, lead: 0.6, seed: 4 });
  const audio = render(take, quietRoom);
  const gapStart = take.sentences[0].end;
  const gapEnd = take.sentences[1].start;
  const mid = (gapStart + gapEnd) / 2;
  const run = async (pace: Pace) => {
    const scan = await scanSpeech(pcm(audio, 0, take.duration), { from: 0, to: take.duration });
    const out = placeEdge(scan.segments, mid - 0.03, "out", pace, scan);
    const into = placeEdge(scan.segments, mid + 0.03, "in", pace, scan);
    const joint = out.t > into.t ? settleJoint(out, into) : null;
    const outT = joint ?? out.t;
    const inT = joint ?? into.t;
    return {
      pace,
      crossed: round3(Math.max(0, out.t - into.t)),
      settled: joint !== null,
      replay: round3(Math.max(0, outT - inT)),
      outAir: round3(outT - gapStart),
      inAir: round3(gapEnd - inT),
    };
  };
  return { take, run };
})();

const joints = await Promise.all((["fast", "natural"] as Pace[]).map(jointCase.run));
const jointBreaches = joints.flatMap((j) => {
  const bad: string[] = [];
  if (j.replay > 0) bad.push(`${j.pace}: ${j.replay}s of source plays twice across the joint`);
  if (j.outAir < 0) bad.push(`${j.pace}: the joint cuts ${-j.outAir}s off the word before it`);
  if (j.inAir < 0) bad.push(`${j.pace}: the joint cuts ${-j.inAir}s off the word after it`);
  return bad;
});

for (const r of report) {
  const mark = r.breaches.length === 0 ? "ok  " : "FAIL";
  console.log(
    `${mark} ${r.name.padEnd(22)} clipped ${r.before.worstClipped.toFixed(3)}s → ${r.after.worstClipped.toFixed(3)}s   ` +
      `air ${r.after.tightestAir.toFixed(3)}–${r.after.widestAir.toFixed(3)}s (median ${r.after.medianAir.toFixed(3)}s, pace wants ${AIR[r.pace].in}–${AIR[r.pace].out}s)`
  );
  if (VERBOSE) console.log(`     ${r.what}`);
  for (const b of r.breaches) console.log(`     ✗ ${b}`);
}
console.log(
  `${jointBreaches.length === 0 ? "ok  " : "FAIL"} ${"joint/short-pause".padEnd(22)} ` +
    joints.map((j) => `${j.pace}: ${j.settled ? "settled" : "held"} ${j.outAir}s / ${j.inAir}s`).join("   ")
);
for (const b of jointBreaches) console.log(`     ✗ ${b}`);

const failed = report.filter((r) => r.breaches.length > 0);
await mkdir(path.dirname(OUT), { recursive: true });
await writeFile(
  OUT,
  `${JSON.stringify(
    { cases: report, joint: joints, jointBreaches, failed: failed.map((f) => f.name) },
    null,
    2
  )}\n`
);
const total = report.length + 1;
const passed = total - failed.length - (jointBreaches.length > 0 ? 1 : 0);
console.log(`\n${passed}/${total} within budget — report at ${OUT}`);

if ((failed.length > 0 || jointBreaches.length > 0) && ENFORCE) process.exit(1);