Skip to content

Open source CapCut alternative.

TypeScript
5.0 KB
Raw Blame
/**
 * Model-comparison eval: run the same brief through several model suites and
 * compare them side by side. Swapping the video (or any) model is swapping one
 * id in a suite selection; the harness runs each suite through the real
 * orchestrator and reports coverage, shots, fallbacks, lip-sync, and wall-clock.
 *
 * Today it runs against the fake registry, so it proves the comparison
 * mechanics without spending credits. To compare real models, register real
 * adapters in the registry and pass a real (recording) editor + real reference
 * ids — the recording editor already captures every produced clip, which is
 * what a side-by-side gallery renders.
 *
 *   node_modules/.bin/bun run scripts/genvideo-eval.ts "a video of me and my son, cinematic"
 */

import { assertCoverage } from "../src/cut/lib/genvideo/coverage";
import { FakeEditor } from "../src/cut/lib/genvideo/editor";
import { fakeRegistry } from "../src/cut/lib/genvideo/registry";
import { VideoOrchestrator } from "../src/cut/lib/genvideo/orchestrator";
import type { RoleName } from "../src/cut/lib/genvideo/capabilities";
import type { VideoProject } from "../src/cut/lib/genvideo/types";

const FPS = 30;

/** The suites to compare. Each differs from the others by one model id. Add a
 * real entry the moment a real adapter is registered. */
const SUITES: { name: string; selection: Partial<Record<RoleName, string>> }[] = [
  { name: "fast-video", selection: { video: "fake-fast" } },
  { name: "pro-video", selection: { video: "fake-pro" } },
];

interface SuiteMetrics {
  suite: string;
  videoModel: string;
  coverageOk: boolean;
  shots: number;
  durationSec: number;
  placed: number;
  fallbacks: number;
  lipSyncShots: number;
  producedClips: string[];
  wallMs: number;
}

async function runSuite(name: string, selection: Partial<Record<RoleName, string>>, brief: string): Promise<SuiteMetrics> {
  const registry = fakeRegistry();
  const suite = registry.buildSuite(selection, name);
  const project: VideoProject = {
    id: `eval:${name}`,
    brief,
    references: [{ mediaId: "ref:user", kind: "image", purpose: "character", name: "the user" }],
    audioMode: "generated",
    fps: FPS,
    durationFrames: 0,
    transcript: [],
    style: "",
    suiteLabel: suite.label,
    characters: [],
    locations: [],
    shots: [],
    phase: "brief",
    breakdownApproved: true, // evals don't pause for a human
    createdAt: 0,
    updatedAt: 0,
    targetSeconds: 30,
  };
  const editor = new FakeEditor({ fps: FPS, durationFrames: 0, aspect: "9:16" }, []);
  const orch = new VideoOrchestrator(project, { editor, suite, emit: () => {}, persist: () => {}, sleep: async () => {} });

  const t0 = performance.now();
  await orch.run();
  const wallMs = performance.now() - t0;

  let coverageOk = true;
  try {
    assertCoverage(project.shots, project.durationFrames);
  } catch {
    coverageOk = false;
  }
  return {
    suite: name,
    videoModel: selection.video ?? "(default)",
    coverageOk,
    shots: project.shots.length,
    durationSec: Math.round((project.durationFrames / FPS) * 10) / 10,
    placed: project.shots.filter((s) => s.status === "placed").length,
    fallbacks: project.shots.filter((s) => s.status === "failed").length,
    lipSyncShots: project.shots.filter((s) => s.lipSynced).length,
    producedClips: project.shots.map((s) => s.clip ?? "(none)"),
    wallMs: Math.round(wallMs * 10) / 10,
  };
}

function table(rows: SuiteMetrics[]): string {
  const head = ["suite", "video model", "coverage", "shots", "dur(s)", "placed", "fallback", "lip-sync", "wall(ms)"];
  const body = rows.map((r) => [
    r.suite,
    r.videoModel,
    r.coverageOk ? "ok" : "BROKEN",
    String(r.shots),
    String(r.durationSec),
    String(r.placed),
    String(r.fallbacks),
    String(r.lipSyncShots),
    String(r.wallMs),
  ]);
  const widths = head.map((h, i) => Math.max(h.length, ...body.map((row) => row[i].length)));
  const fmt = (cols: string[]) => cols.map((c, i) => c.padEnd(widths[i])).join("  ");
  return [fmt(head), fmt(widths.map((w) => "-".repeat(w))), ...body.map(fmt)].join("\n");
}

async function main(): Promise<void> {
  const brief = process.argv[2] ?? "a video of me and my son at the beach, cinematic";
  console.log(`Model-comparison eval\nbrief: "${brief}"\n`);
  const rows: SuiteMetrics[] = [];
  for (const s of SUITES) rows.push(await runSuite(s.name, s.selection, brief));

  console.log(table(rows));

  const broken = rows.filter((r) => !r.coverageOk);
  if (broken.length) {
    console.log(`\n✗ ${broken.length} suite(s) failed the coverage invariant: ${broken.map((r) => r.suite).join(", ")}`);
    process.exit(1);
  }
  // With fakes every suite is instant and correct; the ranking mechanic is what
  // matters — fewest fallbacks, then fastest. Real runs make this meaningful.
  const best = [...rows].sort((a, b) => a.fallbacks - b.fallbacks || a.wallMs - b.wallMs)[0];
  console.log(`\n✓ all suites covered the audio. Leading suite by fallbacks then speed: ${best.suite}`);
}

main().catch((e) => {
  console.error(e);
  process.exit(1);
});