Open source CapCut alternative.
TypeScript
#!/usr/bin/env bun
/**
* Caption timing eval: when the words appear, do they appear on the voice?
*
* A caption's timing has one truth — the moment the speaker actually says the
* word — and no eyeball on a preview settles whether a line is 200ms late.
* So the truth is built rather than measured: a script of lines with exact
* onsets renders to audio, a transcriber's answer is simulated on top of it
* with the errors real transcribers make, and the production timing path runs
* over both. Every number below is |reported − true| in seconds.
*
* Two paths, the two ways words get times:
* captions — a transcript grouped into cues (server/transcribe.ts), then
* re-timed against the mix (lib/cueSync.ts → lib/cueAlign.ts).
* lyrics — the user's own text aligned to a degraded transcript
* (lib/lyricSync.ts), then re-timed the same way. This is the
* path where half the words are never recognized, so most line
* times start out interpolated between the ones that were.
*
* Each case reports before and after the alignment pass, because the pass is
* the thing on trial: if `after` is not better than `before`, reading the
* waveform bought nothing. Budgets are enforced by default — this is a
* regression guard, and it runs in a couple of seconds with no model, no
* network, and no credits.
*
* npm run eval:cut-sync [--only <case>] [--out path] [--no-enforce] [--verbose]
*/
import { mkdir, writeFile } from "node:fs/promises";
import path from "node:path";
import { syncToSpeech } from "../src/cut/lib/cueSync";
import { syncLines, type TimedWord } from "../src/cut/lib/lyricSync";
import { groupWords } from "../src/cut/server/transcribe";
const SITE = path.resolve(import.meta.dir, "..");
const REPORT = path.resolve(SITE, "..", "evals", "cut-sync.latest-report.json");
const argv = process.argv.slice(2);
const arg = (name: string) => {
const i = argv.indexOf(name);
return i >= 0 ? argv[i + 1] : undefined;
};
const ONLY = arg("--only");
const OUT = arg("--out") ?? REPORT;
const ENFORCE = !argv.includes("--no-enforce");
const VERBOSE = argv.includes("--verbose");
const RATE = 16000;
// ── Deterministic noise ─────────────────────────────────────────────────────
// Nothing here may vary run to run: a budget that trips one time in five is
// worse than no budget.
const rng = (seed: number) => () => {
seed = (seed * 1103515245 + 12345) % 2147483648;
return seed / 2147483648 - 0.5;
};
// ── The truth ───────────────────────────────────────────────────────────────
interface TrueWord {
line: number;
w: string;
start: number;
end: number;
}
interface Script {
words: TrueWord[];
lines: { text: string; start: number; end: number }[];
duration: number;
}
const WORDS =
"the light came down over the harbour and nobody moved a hand until morning broke again on every quiet street we knew".split(
" "
);
/**
* A script of `lines` lines, each a few words long, laid out with real
* cadence: syllable-length words, a breath between them, a longer beat
* between lines. `lead` is the silence before anyone speaks.
*/
function script(opts: {
lines: number;
wordsPerLine: number;
wordGap: number;
lineGap: number;
lead: number;
seed: number;
}): Script {
const rnd = rng(opts.seed);
const words: TrueWord[] = [];
const lines: Script["lines"] = [];
let t = opts.lead;
for (let li = 0; li < opts.lines; li++) {
const start = t;
const text: string[] = [];
for (let wi = 0; wi < opts.wordsPerLine; wi++) {
const dur = 0.22 + Math.abs(rnd()) * 0.34;
const w = WORDS[(li * opts.wordsPerLine + wi) % WORDS.length];
words.push({ line: li, w, start: t, end: t + dur });
text.push(w);
t += dur + (wi + 1 < opts.wordsPerLine ? opts.wordGap : 0);
}
lines.push({ text: text.join(" "), start, end: t });
t += opts.lineGap;
}
return { words, lines, duration: t + 1 };
}
/**
* The script as audio: a burst per word, shaped like a syllable rather than a
* flat block, over whatever floor the condition asks for. `bed` lays a steady
* music-like tone under the whole thing — the case where quiet between lines
* is the only edge the envelope can read.
*/
function render(s: Script, opts: { floorDb: number; bedDb?: number; seed: number }): Float32Array {
const rnd = rng(opts.seed);
const out = new Float32Array(Math.ceil(s.duration * RATE));
const floor = Math.pow(10, opts.floorDb / 20);
for (let i = 0; i < out.length; i++) out[i] = rnd() * floor;
if (opts.bedDb !== undefined) {
const bed = Math.pow(10, opts.bedDb / 20);
for (let i = 0; i < out.length; i++) out[i] += Math.sin((2 * Math.PI * 110 * i) / RATE) * bed;
}
for (const w of s.words) {
const from = Math.round(w.start * RATE);
const to = Math.min(out.length, Math.round(w.end * RATE));
const len = Math.max(1, to - from);
for (let i = from; i < to; i++) {
const p = (i - from) / len;
// Attack, hold, decay — a word does not start and stop square.
const env = Math.min(1, p / 0.12) * Math.min(1, (1 - p) / 0.2);
// Two syllables' worth of amplitude wobble inside the word.
const wobble = 0.7 + 0.3 * Math.sin(p * Math.PI * 4);
out[i] += rnd() * 0.55 * env * wobble;
}
}
return out;
}
// ── The transcriber's answer ────────────────────────────────────────────────
type Flaw = "lag" | "leadin" | "drift" | "clean";
/**
* What a recognizer hands back for this audio. Every flaw here is one real
* transcribers make: a constant lag, the silence before a sentence handed to
* its first word, and a clock that drifts as a long chunk runs on.
*/
function heard(s: Script, flaw: Flaw): TimedWord[] {
const out: TimedWord[] = [];
for (const [i, w] of s.words.entries()) {
let t0 = w.start;
let t1 = w.end;
if (flaw === "lag") {
t0 += 0.25;
t1 += 0.25;
}
if (flaw === "drift") {
const k = (i / s.words.length) * 0.6;
t0 += k;
t1 += k;
}
if (flaw === "leadin" && s.words[i - 1] && s.words[i - 1].line !== w.line) {
// The pause before the line is billed to the line's first word.
t0 = s.words[i - 1].end;
}
if (flaw === "leadin" && i === 0) t0 = 0;
out.push({ t0, t1, w: w.w });
}
return out;
}
/** A recognizer on singing: words missed outright, words misheard. */
function degrade(words: TimedWord[], opts: { drop: number; wrong: number; seed: number }): TimedWord[] {
const rnd = rng(opts.seed);
const out: TimedWord[] = [];
for (const w of words) {
const r = Math.abs(rnd()) * 2;
if (r < opts.drop) continue;
out.push(r < opts.drop + opts.wrong ? { ...w, w: `${w.w}zz` } : w);
}
return out;
}
// ── Scoring ─────────────────────────────────────────────────────────────────
const abs = (n: number) => Math.abs(n);
const round3 = (n: number) => Math.round(n * 1000) / 1000;
const quantile = (xs: number[], q: number) =>
xs.length === 0 ? 0 : round3([...xs].sort((a, b) => a - b)[Math.min(xs.length - 1, Math.floor(q * xs.length))]);
interface Score {
lines: number;
medianStart: number;
p90Start: number;
worstStart: number;
within100ms: number;
within250ms: number;
}
function score(got: { start: number }[], truth: { start: number }[]): Score {
const errs = truth.map((t, i) => (got[i] ? abs(got[i].start - t.start) : Infinity));
const finite = errs.filter((e) => Number.isFinite(e));
return {
lines: truth.length,
medianStart: quantile(finite, 0.5),
p90Start: quantile(finite, 0.9),
worstStart: round3(Math.max(0, ...finite)),
within100ms: round3(errs.filter((e) => e <= 0.1).length / errs.length),
within250ms: round3(errs.filter((e) => e <= 0.25).length / errs.length),
};
}
// ── Cases ───────────────────────────────────────────────────────────────────
interface Case {
name: string;
what: string;
run: () => { before: Score; after: Score; evidence: boolean };
/** Budgets on the aligned result. A case that breaks one fails the eval. */
budget: { medianStart: number; p90Start: number; within250ms: number };
}
const clean = { floorDb: -60, seed: 3 };
const noisy = { floorDb: -38, seed: 5 };
const bed = { floorDb: -58, bedDb: -30, seed: 9 };
/** The caption path: a transcript grouped into cues, then aligned. */
function captionCase(
name: string,
what: string,
s: Script,
audio: Float32Array,
flaw: Flaw,
budget: Case["budget"]
): Case {
return {
name,
what,
budget,
run: () => {
// The production grouping decides where the cue boundaries fall — a long
// line becomes two cues — so the truth for each cue is read off the words
// it actually carries, not off the line it came from.
const cues = groupWords(heard(s, flaw).map((w) => ({ t0: w.t0, t1: w.t1, w: w.w })));
let cursor = 0;
const truth = cues.map((c) => {
const count = c.words?.length ?? 1;
const span = s.words.slice(cursor, cursor + count);
cursor += count;
return { start: span[0].start, end: span[span.length - 1].end };
});
const report = syncToSpeech(cues, audio, RATE, { reach: 0.6 });
return {
before: score(cues, truth),
after: score(report.items, truth),
evidence: report.evidence,
};
},
};
}
/** The lyric path: the user's own words against a transcript that heard half
* of them, then aligned. */
function lyricCase(
name: string,
what: string,
s: Script,
audio: Float32Array,
opts: { drop: number; wrong: number },
budget: Case["budget"]
): Case {
return {
name,
what,
budget,
run: () => {
const transcript = degrade(heard(s, "leadin"), { ...opts, seed: 11 });
const timed = syncLines(
s.lines.map((l) => l.text),
transcript,
{ from: 0, to: s.duration }
);
const report = syncToSpeech(timed, audio, RATE, { reach: 0.6 });
return {
before: score(timed, s.lines),
after: score(report.items, s.lines),
evidence: report.evidence,
};
},
};
}
const talk = script({ lines: 8, wordsPerLine: 5, wordGap: 0.07, lineGap: 0.55, lead: 0.9, seed: 1 });
const fast = script({ lines: 10, wordsPerLine: 4, wordGap: 0.05, lineGap: 0.28, lead: 0.4, seed: 2 });
const song = script({ lines: 8, wordsPerLine: 5, wordGap: 0.09, lineGap: 0.7, lead: 1.4, seed: 4 });
const CASES: Case[] = [
captionCase(
"captions/lead-in",
"The silence before each sentence billed to its first word — the on-device transcriber's habit.",
talk,
render(talk, clean),
"leadin",
{ medianStart: 0.1, p90Start: 0.15, within250ms: 1 }
),
captionCase(
"captions/lag",
"Every cue a quarter second late; the shape is right, the placement is not.",
talk,
render(talk, clean),
"lag",
{ medianStart: 0.1, p90Start: 0.15, within250ms: 1 }
),
captionCase(
"captions/drift",
"A clock that slips as the chunk runs on, ending 0.6s out — the hosted transcriber's, unchunked. " +
"The tail budget allows one cue: once the slip exceeds the line gap, a caption's true onset sits " +
"mid-speech in the previous line's run, which no waveform can testify to. Chunking bounds real " +
"drift far under that (cloudTranscribe.ts).",
talk,
render(talk, clean),
"drift",
{ medianStart: 0.12, p90Start: 0.9, within250ms: 0.8 }
),
captionCase(
"captions/noisy-room",
"The same lag over a room 22dB louder — a noise floor close enough to the voice to blur its edges.",
talk,
render(talk, noisy),
"lag",
{ medianStart: 0.12, p90Start: 0.2, within250ms: 1 }
),
captionCase(
"captions/music-bed",
"A steady bed under the whole cut: the gaps between lines are the only readable quiet.",
talk,
render(talk, bed),
"lag",
{ medianStart: 0.15, p90Start: 0.3, within250ms: 0.9 }
),
captionCase(
"captions/fast-cadence",
"Lines a quarter second apart, where a snap that overshoots lands on the wrong line.",
fast,
render(fast, clean),
"leadin",
{ medianStart: 0.12, p90Start: 0.2, within250ms: 1 }
),
lyricCase(
"lyrics/half-heard",
"The user's lyrics against a transcript that missed a third of the words and misheard a quarter.",
song,
render(song, clean),
{ drop: 0.35, wrong: 0.25 },
{ medianStart: 0.12, p90Start: 0.25, within250ms: 0.9 }
),
lyricCase(
"lyrics/mostly-unheard",
"Singing the recognizer barely follows: two words in three unheard, so most line times are interpolated.",
song,
render(song, clean),
{ drop: 0.68, wrong: 0.2 },
{ medianStart: 0.15, p90Start: 0.35, within250ms: 0.85 }
),
];
// ── Run ─────────────────────────────────────────────────────────────────────
const cases = CASES.filter((c) => !ONLY || c.name.includes(ONLY));
if (cases.length === 0) {
console.error(`No case matches ${ONLY}. Cases: ${CASES.map((c) => c.name).join(", ")}`);
process.exit(2);
}
const results = cases.map((c) => {
const { before, after, evidence } = c.run();
const breaches: string[] = [];
if (after.medianStart > c.budget.medianStart)
breaches.push(`median ${after.medianStart}s > ${c.budget.medianStart}s`);
if (after.p90Start > c.budget.p90Start) breaches.push(`p90 ${after.p90Start}s > ${c.budget.p90Start}s`);
if (after.within250ms < c.budget.within250ms)
breaches.push(`within 250ms ${after.within250ms} < ${c.budget.within250ms}`);
// The pass has to earn its place: aligning must not leave lines worse off
// than the transcript had them.
if (after.medianStart > before.medianStart + 0.02)
breaches.push(`alignment made it worse: ${before.medianStart}s → ${after.medianStart}s`);
return { name: c.name, what: c.what, evidence, before, after, budget: c.budget, breaches };
});
const failed = results.filter((r) => r.breaches.length > 0);
for (const r of results) {
const mark = r.breaches.length === 0 ? "ok " : "FAIL";
console.log(
`${mark} ${r.name.padEnd(24)} median ${r.before.medianStart.toFixed(3)}s → ${r.after.medianStart.toFixed(3)}s ` +
`p90 ${r.after.p90Start.toFixed(3)}s within 250ms ${(r.after.within250ms * 100).toFixed(0)}%` +
(r.evidence ? "" : " (the mix offered no edges)")
);
if (VERBOSE) console.log(` ${r.what}`);
for (const b of r.breaches) console.log(` ✗ ${b}`);
}
await mkdir(path.dirname(OUT), { recursive: true });
await writeFile(
OUT,
`${JSON.stringify({ cases: results, failed: failed.map((f) => f.name) }, null, 2)}\n`
);
console.log(`\n${results.length - failed.length}/${results.length} within budget — report at ${OUT}`);
if (failed.length > 0 && ENFORCE) process.exit(1);