Skip to content

Open source CapCut alternative.

TypeScript
18.7 KB
Raw Blame
/**
 * Watch-selection benchmark: how well does each frame-selection method turn a
 * video into ≤36 contact-sheet cells?
 *
 * Methods, both fed the same candidate pass (the production sampler's steady
 * floor at max(1s, duration/150), software decode for determinism):
 *   A  fixed interval — the old sampler: steady clamp(range/32, 2, 30)s
 *   C  dense+dedup    — the shipped selector (lib/watch/dedupSelector)
 *
 * Fixtures build deterministically into dist/watch-bench/ (gitignored):
 *   montage    — 20 stock clips re-encoded and concatenated; ground truth =
 *                the known cut times (does each segment get a frame?)
 *   slideshow  — synthesized text cards on a static background; ground truth =
 *                the card-change times (the settled-channel case: thin strokes
 *                that measure ~0% on a coarse grid)
 *   talking heads — stock character clips as-is (dedup should collapse these)
 *   jfk-rice   — public-domain speech footage via yt-dlp when available
 *
 * Outputs: dist/watch-bench/stats.csv, per-video keep/drop report.html for
 * method C, candidate JPEGs for eyeballing. Run: npx bun scripts/bench-watch.ts
 */
import { spawn } from "node:child_process";
import { mkdir, readdir, rm, writeFile } from "node:fs/promises";
import { existsSync } from "node:fs";
import path from "node:path";
import { createDedupSelector } from "../src/cut/lib/watch/dedupSelector";
import { SIGNATURE_SIZE, type FrameDecision } from "../src/cut/lib/watch/types";

const SITE = path.resolve(import.meta.dir, "..");
const OUT = path.resolve(SITE, "..", "dist", "watch-bench");
const STOCK = path.join(SITE, "public", "cut-stock-video");
const RAW_BYTES = SIGNATURE_SIZE * SIGNATURE_SIZE * 3;
const MAX_KEPT = 36;

const clamp = (v: number, lo: number, hi: number) => Math.max(lo, Math.min(hi, v));

interface RunOut {
  code: number | null;
  stdout: Buffer;
  stderr: string;
}

function run(cmd: string, args: string[], onStdout?: (d: Buffer) => void): Promise<RunOut> {
  return new Promise((resolve, reject) => {
    const p = spawn(cmd, args);
    const chunks: Buffer[] = [];
    let stderr = "";
    p.stdout.on("data", (d: Buffer) => (onStdout ? onStdout(d) : chunks.push(d)));
    p.stderr.on("data", (d) => {
      if (stderr.length < 4_000_000) stderr += d.toString();
    });
    p.on("error", reject);
    p.on("close", (code) => resolve({ code, stdout: Buffer.concat(chunks), stderr }));
  });
}

async function ffmpeg(args: string[], onStdout?: (d: Buffer) => void): Promise<RunOut> {
  const r = await run("ffmpeg", ["-hide_banner", "-nostats", "-y", ...args], onStdout);
  if (r.code !== 0 && !onStdout)
    throw new Error(`ffmpeg failed: ${r.stderr.split("\n").slice(-4).join("\n")}`);
  return r;
}

// ---------------------------------------------------------------- fixtures

/** Deterministic PRNG so the slideshow renders identically every run. */
function lcg(seed: number) {
  let s = seed >>> 0;
  return () => ((s = (s * 1664525 + 1013904223) >>> 0), s / 2 ** 32);
}

const CARD_W = 640;
const CARD_H = 360;
const CARD_SECONDS = 6;
const CARD_COUNT = 8;

/** One text card as a PPM (P6) buffer: a block digit header plus rows of
 * 2px "text" strokes whose lengths change per card — thin ink on a static
 * background, the change a coarse grid cannot see. */
function renderCard(card: number): Buffer {
  const px = new Uint8Array(CARD_W * CARD_H * 3);
  for (let i = 0; i < px.length; i += 3) {
    px[i] = 16;
    px[i + 1] = 16;
    px[i + 2] = 20;
  }
  const fill = (x: number, y: number, w: number, h: number) => {
    for (let yy = y; yy < y + h && yy < CARD_H; yy++)
      for (let xx = x; xx < x + w && xx < CARD_W; xx++) {
        const i = (yy * CARD_W + xx) * 3;
        px[i] = 235;
        px[i + 1] = 235;
        px[i + 2] = 240;
      }
  };
  // Seven-segment "chapter" digit, so sheets are eyeball-checkable. It only
  // advances every second card — the in-between transitions change nothing
  // but the 2px stroke rows, the change only the settled channel can see.
  const SEGS: Record<string, [number, number, number, number]> = {
    top: [0, 0, 60, 8],
    mid: [0, 46, 60, 8],
    bot: [0, 92, 60, 8],
    tl: [0, 0, 8, 50],
    tr: [52, 0, 8, 50],
    bl: [0, 50, 8, 50],
    br: [52, 50, 8, 50],
  };
  const DIGIT: Record<number, string[]> = {
    0: ["top", "bot", "tl", "tr", "bl", "br"],
    1: ["tr", "br"],
    2: ["top", "mid", "bot", "tr", "bl"],
    3: ["top", "mid", "bot", "tr", "br"],
    4: ["mid", "tl", "tr", "br"],
    5: ["top", "mid", "bot", "tl", "br"],
    6: ["top", "mid", "bot", "tl", "bl", "br"],
    7: ["top", "tr", "br"],
    8: ["top", "mid", "bot", "tl", "tr", "bl", "br"],
    9: ["top", "mid", "bot", "tl", "tr", "br"],
  };
  for (const seg of DIGIT[(Math.floor(card / 2) + 1) % 10]) {
    const [x, y, w, h] = SEGS[seg];
    fill(40 + x, 30 + y, w, h);
  }
  // "Body text": a seeded paragraph of thin word-run strokes. Line pitch,
  // word lengths, and gaps are the card's identity — different text lays its
  // ink at different spots, the way real caption swaps do.
  const rand = lcg(1000 + card * 77);
  const lineH = 14 + Math.floor(rand() * 9);
  const lines = 7 + Math.floor(rand() * 4);
  for (let row = 0; row < lines; row++) {
    const y = 160 + row * lineH + Math.floor(rand() * 4);
    let x = 40 + Math.floor(rand() * 30);
    while (x < 560) {
      const w = 20 + Math.floor(rand() * 60);
      if (rand() > 0.25) fill(x, y, Math.min(w, 580 - x), 3);
      x += w + 8 + Math.floor(rand() * 18);
    }
  }
  return Buffer.concat([Buffer.from(`P6\n${CARD_W} ${CARD_H}\n255\n`), Buffer.from(px)]);
}

const WRITE_SECONDS = 40;
const WRITE_STEP = 2; // a new word-run lands every 2s
// One page for the whole clip: a cleared page would leave the old full page
// in the selector's kept window, sitting exactly where the new ink lands —
// the window's changed-vs-everything rule then reads the new ink as already
// seen. Page swaps are a hard-cut case; the montage covers those.
const WRITE_PAGE = WRITE_SECONDS;

/** One second of the "handwriting" fixture: dark word-runs accumulating on a
 * bright static page, one more every WRITE_STEP seconds — a slow thin-stroke
 * morph, the settled channel's hardest case. */
function renderWritingFrame(sec: number): Buffer {
  const px = new Uint8Array(CARD_W * CARD_H * 3);
  for (let i = 0; i < px.length; i += 3) {
    px[i] = 235;
    px[i + 1] = 233;
    px[i + 2] = 225;
  }
  const ink = (x: number, y: number, w: number, h: number) => {
    for (let yy = y; yy < y + h && yy < CARD_H; yy++)
      for (let xx = x; xx < x + w && xx < CARD_W; xx++) {
        const i = (yy * CARD_W + xx) * 3;
        px[i] = 30;
        px[i + 1] = 28;
        px[i + 2] = 40;
      }
  };
  const page = Math.floor(sec / WRITE_PAGE);
  const steps = Math.floor((sec % WRITE_PAGE) / WRITE_STEP) + 1;
  const rand = lcg(4000 + page * 131);
  let x = 60;
  let y = 60;
  for (let s = 0; s < steps; s++) {
    const w = 40 + Math.floor(rand() * 90);
    if (x + w > 580) {
      x = 60;
      y += 24 + Math.floor(rand() * 6);
    }
    ink(x, y, w, 3);
    x += w + 14 + Math.floor(rand() * 12);
  }
  return Buffer.concat([Buffer.from(`P6\n${CARD_W} ${CARD_H}\n255\n`), Buffer.from(px)]);
}

interface Fixture {
  slug: string;
  file: string;
  /** Ground-truth state boundaries (ascending): a good selection has ≥1 kept
   * frame inside every [boundary[i], boundary[i+1]) span. */
  states?: number[];
}

async function buildFixtures(): Promise<Fixture[]> {
  const fx = path.join(OUT, "fixtures");
  await mkdir(fx, { recursive: true });
  const fixtures: Fixture[] = [];

  // montage: 20 stock clips, 7s each, one size, concatenated. Real encoded
  // part durations drift from the asked-for 7s, so the ground-truth cut
  // times are measured, then remembered beside the file.
  const montage = path.join(fx, "montage.mp4");
  const montageStates = path.join(fx, "montage.states.json");
  const stock = (await readdir(STOCK)).filter((f) => f.endsWith(".mp4")).sort().slice(0, 20);
  if (!existsSync(montage) || !existsSync(montageStates)) {
    const parts: string[] = [];
    const cuts: number[] = [];
    let at = 0;
    for (const [i, f] of stock.entries()) {
      const part = path.join(fx, `part-${String(i).padStart(2, "0")}.ts`);
      await ffmpeg([
        "-ss", "0", "-t", "7", "-i", path.join(STOCK, f),
        "-vf", "scale=640:360:force_original_aspect_ratio=increase,crop=640:360,fps=30",
        "-an", "-c:v", "libx264", "-preset", "veryfast", "-crf", "23", part,
      ]);
      parts.push(part);
      cuts.push(at);
      const probe = await run("ffprobe", [
        "-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", part,
      ]);
      at += parseFloat(probe.stdout.toString()) || 7;
    }
    const list = path.join(fx, "concat.txt");
    await writeFile(list, parts.map((p) => `file '${p}'`).join("\n"));
    await ffmpeg(["-f", "concat", "-safe", "0", "-i", list, "-c", "copy", montage]);
    await writeFile(montageStates, JSON.stringify(cuts));
    await Promise.all(parts.map((p) => rm(p, { force: true })));
    await rm(list, { force: true });
  }
  fixtures.push({
    slug: "montage",
    file: montage,
    states: JSON.parse(await Bun.file(montageStates).text()) as number[],
  });

  // slideshow: synthesized text cards, CARD_SECONDS each.
  const slideshow = path.join(fx, "slideshow.mp4");
  if (!existsSync(slideshow)) {
    for (let c = 0; c < CARD_COUNT; c++)
      await writeFile(path.join(fx, `card-${String(c).padStart(3, "0")}.ppm`), renderCard(c));
    await ffmpeg([
      "-framerate", `1/${CARD_SECONDS}`, "-i", path.join(fx, "card-%03d.ppm"),
      "-r", "30", "-pix_fmt", "yuv420p", "-c:v", "libx264", "-preset", "veryfast", "-crf", "20",
      slideshow,
    ]);
    for (let c = 0; c < CARD_COUNT; c++)
      await rm(path.join(fx, `card-${String(c).padStart(3, "0")}.ppm`), { force: true });
  }
  fixtures.push({
    slug: "slideshow",
    file: slideshow,
    states: Array.from({ length: CARD_COUNT }, (_, i) => i * CARD_SECONDS),
  });

  // handwriting: ink accumulating on a static page.
  const writing = path.join(fx, "writing.mp4");
  if (!existsSync(writing)) {
    for (let s = 0; s < WRITE_SECONDS; s++)
      await writeFile(path.join(fx, `write-${String(s).padStart(3, "0")}.ppm`), renderWritingFrame(s));
    await ffmpeg([
      "-framerate", "1", "-i", path.join(fx, "write-%03d.ppm"),
      "-r", "30", "-pix_fmt", "yuv420p", "-c:v", "libx264", "-preset", "veryfast", "-crf", "20",
      writing,
    ]);
    for (let s = 0; s < WRITE_SECONDS; s++)
      await rm(path.join(fx, `write-${String(s).padStart(3, "0")}.ppm`), { force: true });
  }
  fixtures.push({
    slug: "writing",
    file: writing,
    states: Array.from({ length: WRITE_SECONDS / WRITE_STEP }, (_, i) => i * WRITE_STEP),
  });

  // talking heads: character clips as-is (no ground truth — the measure is
  // how few frames a static speaker costs).
  for (const slug of ["character-cafe-analyst", "character-startup-founder"]) {
    const file = path.join(STOCK, `${slug}.mp4`);
    if (existsSync(file)) fixtures.push({ slug, file });
  }

  // jfk-rice: public-domain speech footage (1962), 60–240s section.
  const jfk = path.join(fx, "jfk-rice.mp4");
  if (!existsSync(jfk)) {
    const probe = await run("yt-dlp", ["--version"]).catch(() => null);
    if (probe && probe.code === 0) {
      console.log("fetching jfk-rice.mp4 (yt-dlp)...");
      const dl = await run("yt-dlp", [
        "--download-sections", "*60-240",
        "-f", "bv*[height<=480]+ba/b[height<=480]",
        "--merge-output-format", "mp4",
        "-o", jfk,
        "https://www.youtube.com/watch?v=WZyRbnpGyzQ",
      ]);
      if (dl.code !== 0) console.log("  jfk-rice download failed — skipping that fixture");
    } else {
      console.log("yt-dlp not found — skipping the jfk-rice fixture");
    }
  }
  if (existsSync(jfk)) fixtures.push({ slug: "jfk-rice", file: jfk });

  return fixtures;
}

// ---------------------------------------------------- candidate extraction

interface Candidate {
  t: number;
  jpg: string; // path on disk
  raw: Uint8Array;
}

/** The production candidate pass (software decode): a steady floor at
 * max(1s, duration/150), split into 480px JPEGs and raw selector frames. */
async function extractCandidates(file: string, dir: string): Promise<{ cands: Candidate[]; ms: number }> {
  await rm(dir, { recursive: true, force: true });
  await mkdir(dir, { recursive: true });
  const probe = await run("ffprobe", [
    "-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", file,
  ]);
  const dur = parseFloat(probe.stdout.toString()) || 0;
  const interval = Math.max(1, dur / 150);
  const graph = [
    // showinfo sits after select, so its log lines are exactly the emitted
    // candidates in order — the only place the source timestamps survive.
    `[0:v]select='isnan(prev_selected_t)+gte(t-prev_selected_t,${interval})',` +
      "showinfo,split=2[jl][rl]",
    "[jl]scale=480:-2[jpg]",
    `[rl]scale=${SIGNATURE_SIZE}:${SIGNATURE_SIZE}:flags=area,format=rgb24[raw]`,
  ].join(";");
  const raws: Uint8Array[] = [];
  let pendingBuf: Buffer = Buffer.alloc(0);
  const t0 = performance.now();
  const r = await ffmpeg(
    [
      "-loglevel", "info",
      "-i", file,
      "-filter_complex", graph,
      "-map", "[jpg]", "-fps_mode", "vfr", "-frames:v", "700", "-q:v", "5",
      path.join(dir, "cand-%04d.jpg"),
      "-map", "[raw]", "-fps_mode", "vfr", "-frames:v", "700", "-f", "rawvideo", "pipe:1",
    ],
    (d) => {
      pendingBuf = pendingBuf.length === 0 ? d : Buffer.concat([pendingBuf, d]);
      while (pendingBuf.length >= RAW_BYTES) {
        raws.push(new Uint8Array(pendingBuf.subarray(0, RAW_BYTES)));
        pendingBuf = Buffer.from(pendingBuf.subarray(RAW_BYTES));
      }
    }
  );
  const ms = performance.now() - t0;
  if (r.code !== 0) throw new Error(`candidate pass failed: ${r.stderr.split("\n").slice(-4).join("\n")}`);
  const meta: { t: number }[] = [];
  for (const line of r.stderr.split("\n")) {
    const t = /Parsed_showinfo.*pts_time:\s*(-?[\d.]+)/.exec(line);
    if (t) meta.push({ t: Math.max(0, parseFloat(t[1])) });
  }
  const files = (await readdir(dir)).filter((f) => f.endsWith(".jpg")).sort();
  const n = Math.min(files.length, meta.length, raws.length);
  const cands: Candidate[] = [];
  for (let i = 0; i < n; i++) cands.push({ t: meta[i].t, jpg: files[i], raw: raws[i] });
  return { cands, ms };
}

// ----------------------------------------------------------------- methods

/** A: the old cloud sampler — one frame per steady interval, capped at 36. */
function methodA(cands: Candidate[], from: number, to: number): number[] {
  const interval = clamp((to - from) / 32, 2, 30);
  const kept: number[] = [];
  for (let t = from; t < to && kept.length < MAX_KEPT; t += interval) {
    let best = -1;
    for (let i = 0; i < cands.length; i++) {
      if (best === -1 || Math.abs(cands[i].t - t) < Math.abs(cands[best].t - t)) best = i;
    }
    if (best >= 0 && kept[kept.length - 1] !== best) kept.push(best);
  }
  return kept;
}

/** C: the shipped selector. */
function methodC(cands: Candidate[]): { kept: number[]; decisions: FrameDecision[]; ms: number } {
  const t0 = performance.now();
  const selector = createDedupSelector({ maxFrames: MAX_KEPT });
  for (const c of cands)
    selector.push({ width: SIGNATURE_SIZE, height: SIGNATURE_SIZE, channels: 3, data: c.raw });
  const sel = selector.finish();
  return { kept: sel.kept, decisions: sel.decisions, ms: performance.now() - t0 };
}

// ------------------------------------------------------------------ scoring

/** Fraction of ground-truth states holding at least one kept frame. */
function coverage(cands: Candidate[], kept: number[], states: number[], end: number): number {
  const times = kept.map((i) => cands[i].t);
  let hit = 0;
  for (let s = 0; s < states.length; s++) {
    const lo = states[s];
    const hi = s + 1 < states.length ? states[s + 1] : end;
    if (times.some((t) => t >= lo && t < hi)) hit++;
  }
  return hit / states.length;
}

function reportHtml(cands: Candidate[], decisions: FrameDecision[], slug: string): string {
  const rows = decisions
    .map((d) => {
      const c = cands[d.index];
      if (!c) return "";
      const cls = d.verdict === "drop" ? "drop" : d.verdict === "thinned" ? "thin" : "keep";
      const extra =
        d.settledScore !== undefined
          ? ` · settled ${d.settledScore.toFixed(1)}%/${d.settledGate!.toFixed(1)}%`
          : "";
      return (
        `<figure class="${cls}"><img src="cands/${c.jpg}" loading="lazy">` +
        `<figcaption>#${d.index} @${c.t.toFixed(1)}s · ${d.verdict} · dist ${d.globalDist.toFixed(1)}%${extra}</figcaption></figure>`
      );
    })
    .join("");
  return `<!doctype html><meta charset="utf-8"><title>${slug} — watch selection</title>
<style>body{font:14px system-ui;margin:20px;background:#111;color:#ddd}
.grid{display:flex;flex-wrap:wrap;gap:10px}figure{margin:0;width:200px}
img{width:100%;border-radius:4px}figcaption{font-size:11px;color:#999;padding:2px 0}
.drop img{opacity:.35;outline:2px solid #a33}.thin img{opacity:.35;outline:2px solid #a80}
.keep img{outline:2px solid #3a6}</style>
<h2>${slug}</h2><p>green kept · red duplicate · orange removed by the 36-frame cap</p>
<div class="grid">${rows}</div>`;
}

// --------------------------------------------------------------------- main

const fixtures = await buildFixtures();
const csv: string[] = ["video,method,candidates,kept,sheets,coverage,selectMs,extractMs"];
for (const f of fixtures) {
  const dir = path.join(OUT, f.slug);
  const { cands, ms: extractMs } = await extractCandidates(f.file, path.join(dir, "cands"));
  if (cands.length === 0) {
    console.log(`${f.slug}: no candidates — skipped`);
    continue;
  }
  const from = 0;
  const to = cands[cands.length - 1].t + 1;
  const a = methodA(cands, from, to);
  const c = methodC(cands);
  await writeFile(path.join(dir, "report.html"), reportHtml(cands, c.decisions, f.slug));

  console.log(`\n== ${f.slug} (${cands.length} candidates, ${to.toFixed(0)}s, extract ${(extractMs / 1000).toFixed(1)}s)`);
  for (const [name, kept, ms] of [
    ["A fixed", a, 0],
    ["C dedup", c.kept, c.ms],
  ] as const) {
    const cov = f.states ? coverage(cands, [...kept], f.states, to) : null;
    const sheets = Math.ceil(kept.length / 9);
    console.log(
      `  ${name}: kept ${String(kept.length).padStart(3)}  sheets ${sheets}` +
        (cov !== null ? `  state-coverage ${(cov * 100).toFixed(0)}%` : "") +
        (ms ? `  select ${ms.toFixed(0)}ms` : "")
    );
    csv.push(
      `${f.slug},${name.split(" ")[0]},${cands.length},${kept.length},${sheets},` +
        `${cov !== null ? cov.toFixed(3) : ""},${ms ? ms.toFixed(0) : ""},${extractMs.toFixed(0)}`
    );
  }
}
await writeFile(path.join(OUT, "stats.csv"), csv.join("\n") + "\n");
console.log(`\nstats: ${path.join(OUT, "stats.csv")}`);
console.log(`reports: ${OUT}/<slug>/report.html`);