Skip to content

Open source CapCut alternative.

TypeScript
9.3 KB
Raw Blame
#!/usr/bin/env bun
/**
 * The Cut chat eval: behavior and speed in one run.
 *
 * Each case replays a real composer turn (system prompt, tool catalog,
 * <attached_assets>, <editor_state>, inline audio) against the live chat
 * model through the hosted Responses route and asserts on what the model
 * does — questions answer in chat without project-mutating tool calls, edit
 * requests reach the right tools — while timing the whole path: the gate,
 * time-to-first-token, every round, every tool serve.
 *
 * Run with the site dev server up:
 *   bun run scripts/eval-cut-chat.ts
 *     [--base http://localhost:3000]
 *     [--only <case>[,<case>…]] [--bucket chat|single-tool|multi-tool]
 *     [--runs N]                      default 1; use 3+ for latency stats
 *     [--model <registryKey|rawId>]   pin BOTH chat roles to one model (unrouted)
 *     [--simple-model <id>] [--complex-model <id>]  override one routed role
 *     [--matrix]                      run every CANDIDATES row, compare
 *     [--no-judge]                    skip the voice/taste judge call
 *     [--enforce-budgets]             exit 1 when a case breaches its budget
 *     [--out <path>]                  report path; a full run defaults to
 *                                     <repo>/evals/cut-chat.latest-report.json,
 *                                     a narrowed run writes only with --out
 *
 * Auth is the dev bypass header (scripts only — never the app), so runs are
 * dev-server-only and spend no credits. The spoken fixture is synthesized
 * locally with macOS `say`, so the transcript assertion is deterministic.
 * See scripts/lib/cut-eval/README.md for the improvement loop.
 */

import { mkdirSync, writeFileSync } from "node:fs";
import { dirname, join, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import {
  geminiModelRoles,
  geminiModels,
  resolveGeminiModel,
} from "../src/lib/inference/gemini-models";
import { cases, type Bucket, type EvalCase } from "./lib/cut-eval/cases";
import { makeFixtureAudio } from "./lib/cut-eval/fixtures";
import { defaultRunConfig, runCase, type RunConfig } from "./lib/cut-eval/harness";
import {
  buildBucketSummaries,
  buildCaseReport,
  toRunReport,
  type CaseReport,
  type ConfigReport,
  type LatencyReport,
  type RunReport,
} from "./lib/cut-eval/report";

// Model configs the --matrix mode compares. Add a row when trialing a new
// model id; keep "baseline" matching geminiModelRoles so the comparison always
// includes what production runs (the routed pair).
const CANDIDATES: { label: string; simple: string; complex: string }[] = [
  {
    label: "baseline",
    simple: geminiModelRoles.chatSimple,
    complex: geminiModelRoles.chat,
  },
  {
    label: "all-flash",
    simple: geminiModels.flash,
    complex: geminiModels.flash,
  },
  {
    label: "all-flashLite",
    simple: geminiModels.flashLite,
    complex: geminiModels.flashLite,
  },
];

const args = process.argv.slice(2);
const argValue = (flag: string) => {
  const i = args.indexOf(flag);
  return i >= 0 ? args[i + 1] : undefined;
};
const BASE = argValue("--base") ?? "http://localhost:3000";
const ONLY = argValue("--only");
/** Print each tool call's arguments and the reply on a failing run. */
const TRACE = args.includes("--trace");
const BUCKET = argValue("--bucket") as Bucket | undefined;
const RUNS = Number(argValue("--runs") ?? 1);
const MATRIX = args.includes("--matrix");
const NO_JUDGE = args.includes("--no-judge");
const ENFORCE_BUDGETS = args.includes("--enforce-budgets");
const OUT_ARG = argValue("--out");
const DEFAULT_OUT = join(
  dirname(fileURLToPath(import.meta.url)),
  "../../evals/cut-chat.latest-report.json"
);
// A narrowed run would overwrite the committed baseline with a partial
// report, so it only writes when --out names a destination.
const OUT = OUT_ARG ? resolve(OUT_ARG) : ONLY || BUCKET ? null : resolve(DEFAULT_OUT);

/** A registry key (flash, flashLite, …) or a raw model id. */
// A registry key becomes its id; a role name or a raw id goes out as given —
// the responses route resolves a role, the way it does for the page.
const resolveModel = (v: string): string =>
  (geminiModels as Record<string, string>)[v] ?? v;
// Reports name ids, whatever the config sent.
const idOf = resolveGeminiModel;

const ms = (v: number | null | undefined) => (typeof v === "number" ? `${v}ms` : "—");
const pct = (v: number) => `${Math.round(v * 100)}%`;

function printTable(header: string[], rows: string[][]) {
  const all = [header, ...rows];
  const widths = header.map((_, i) => Math.max(...all.map((r) => r[i].length)));
  for (const [n, row] of all.entries()) {
    console.log(row.map((cell, i) => cell.padEnd(widths[i])).join("  "));
    if (n === 0) console.log(widths.map((w) => "-".repeat(w)).join("  "));
  }
}

async function runConfig(
  label: string,
  cfg: RunConfig,
  selected: EvalCase[]
): Promise<ConfigReport> {
  const simpleId = idOf(cfg.simpleModel);
  const complexId = idOf(cfg.complexModel);
  const chatDesc = simpleId === complexId ? simpleId : `${simpleId} | complex→${complexId}`;
  console.log(`\n== ${label}  chat=${chatDesc}  judge=typesafe`);
  const caseReports: CaseReport[] = [];
  for (const c of selected) {
    const runs: RunReport[] = [];
    for (let run = 1; run <= RUNS; run++) {
      try {
        runs.push(toRunReport(await runCase(c, cfg)));
      } catch (err) {
        runs.push(toRunReport(null, err instanceof Error ? err.message : String(err)));
      }
    }
    const report = buildCaseReport(c, runs);
    caseReports.push(report);
    const l = report.latency;
    console.log(
      `  ${report.passRate === 1 ? "ok  " : report.passRate > 0 ? "part" : "FAIL"} ${c.name}` +
        `  pass ${pct(report.passRate)}  total p50 ${ms(l?.totalMs.p50)}  ttft p50 ${ms(l?.ttftMs?.p50)}` +
        `  rounds ${l ? l.rounds.mean : "—"}` +
        (runs.find((r) => r.instant) ? `  instant ${runs.find((r) => r.instant)?.instant}` : "") +
        (report.budgetBreaches.length > 0 ? `  BUDGET: ${report.budgetBreaches.join("; ")}` : "")
    );
    for (const r of runs.filter((r) => !r.pass)) {
      for (const n of r.error ? [r.error] : r.notes) console.log(`       - ${n}`);
      if (r.trace.length > 0) {
        console.log(`       tools: ${r.trace.map((t) => t.name).join(" → ")}`);
        if (TRACE)
          for (const t of r.trace) console.log(`         ${t.name} ${JSON.stringify(t.args).slice(0, 200)}`);
      }
    }
    for (const r of runs.filter((r) => r.judgeNote)) console.log(`       judge: ${r.judgeNote}`);
  }
  return {
    label,
    chatModel: chatDesc,
    buckets: buildBucketSummaries(caseReports),
    cases: caseReports,
  };
}

async function main() {
  const audio = makeFixtureAudio();
  const only = ONLY?.split(",").map((s) => s.trim());
  const selected = cases(audio).filter(
    (c) => (!only || only.includes(c.name)) && (!BUCKET || c.bucket === BUCKET)
  );
  if (selected.length === 0)
    throw new Error(ONLY ? `No case named "${ONLY}".` : `No cases in bucket "${BUCKET}".`);

  // --model pins BOTH chat roles to one model (an unrouted config);
  // --simple-model / --complex-model override a role each. The default is
  // production's routed pair.
  const both = argValue("--model");
  const defaults = defaultRunConfig(BASE);
  const judge = !NO_JUDGE;
  const configs = MATRIX
    ? CANDIDATES.map((cand) => ({
        label: cand.label,
        cfg: {
          base: BASE,
          simpleModel: cand.simple,
          complexModel: cand.complex,
          judge,
        },
      }))
    : [
        {
          label: "config",
          cfg: {
            base: BASE,
            simpleModel: resolveModel(
              argValue("--simple-model") ?? both ?? defaults.simpleModel
            ),
            complexModel: resolveModel(
              argValue("--complex-model") ?? both ?? defaults.complexModel
            ),
            judge,
          },
        },
      ];

  const report: LatencyReport = {
    schema: "cut-chat-eval/v1",
    generatedAt: new Date().toISOString(),
    base: BASE,
    runsPerCase: RUNS,
    configs: [],
  };
  for (const { label, cfg } of configs) {
    report.configs.push(await runConfig(label, cfg, selected));
  }

  console.log("");
  printTable(
    ["config", "bucket", "cases", "pass", "gate p50", "ttft p50", "total p50", "rounds"],
    report.configs.flatMap((c) =>
      (Object.entries(c.buckets) as [Bucket, NonNullable<ConfigReport["buckets"][Bucket]>][]).map(
        ([bucket, s]) => [
          c.label,
          bucket,
          String(s.cases),
          pct(s.passRate),
          ms(s.gateMs?.p50),
          ms(s.ttftMs?.p50),
          ms(s.totalMs?.p50),
          s.meanRounds === null ? "—" : String(s.meanRounds),
        ]
      )
    )
  );

  if (OUT) {
    mkdirSync(dirname(OUT), { recursive: true });
    writeFileSync(OUT, `${JSON.stringify(report, null, 2)}\n`);
    console.log(`\nReport: ${OUT}`);
  }

  const dead = report.configs.flatMap((c) =>
    c.cases.filter((x) => x.passRate === 0).map((x) => `${c.label}/${x.name}`)
  );
  const breaches = report.configs.flatMap((c) =>
    c.cases.filter((x) => x.budgetBreaches.length > 0).map((x) => `${c.label}/${x.name}`)
  );
  if (dead.length > 0) {
    console.log(`\nNo passing runs: ${dead.join(", ")}`);
    process.exit(1);
  }
  if (ENFORCE_BUDGETS && breaches.length > 0) {
    console.log(`\nBudget breaches: ${breaches.join(", ")}`);
    process.exit(1);
  }
}

void main();