Skip to content

Open source CapCut alternative.

TypeScript
8.7 KB
Raw Blame
#!/usr/bin/env bun
/**
 * The Cut routing eval: what the turn judge decides for a labelled set of
 * asks — the skill it attaches, the tool areas it declares, the gate side —
 * and how the CUT_JUDGE thresholds trade suggestion
 * accuracy against declared tool count.
 *
 * Every case is judged once (the answers are cached for the run), then the
 * route is composed under a grid of thresholds in code, so a sweep costs no
 * extra calls. The report names the best triple; the registry defaults are
 * set from it.
 *
 * Run with the site dev server up:
 *   bun run scripts/eval-cut-routing.ts
 *     [--base http://localhost:3000]
 *     [--only <substring>]            cases whose request contains it
 *     [--no-sweep]                    defaults only
 *     [--filler]                      also judge the filler-word fixture
 *
 * Auth is the dev bypass header (scripts only — never the app), so runs are
 * dev-server-only and spend no credits.
 */

import { CUT_JUDGE } from "../src/cut/lib/chatRuntime";
import { noul } from "../src/lib/inference/judge";
import { AI_SKILL_INDEX, areaTools, CORE_TOOLS, TOOL_AREA_NAMES } from "../src/cut/server/ai/catalog";
import { judgeTurnState, pickSkill, routeTurn, TURN_JUDGE_QUESTIONS, type TurnJudgeAnswers } from "../src/cut/lib/turnJudge";
import { EDITOR_STATE, FILLER_CUES } from "./lib/cut-eval/fixtures";
import { routingCases, type RoutingCase } from "./lib/cut-eval/routingCases";

const args = process.argv.slice(2);
const argValue = (flag: string) => {
  const i = args.indexOf(flag);
  return i >= 0 ? args[i + 1] : undefined;
};
const BASE = argValue("--base") ?? "http://localhost:3000";
const ONLY = argValue("--only");
const SWEEP = !args.includes("--no-sweep");
const FILLER = args.includes("--filler");

async function judge(state: unknown, questions: Record<string, unknown>) {
  const res = await fetch(`${BASE}/api/inference/judge`, {
    method: "POST",
    headers: {
      "Content-Type": "application/json",
      "x-donkey-client-id": "donkey-cut-eval",
      "x-donkey-dev-auth-bypass": "1",
    },
    body: JSON.stringify({ state, questions }),
  });
  if (!res.ok) throw new Error(`judge ${res.status}: ${await res.text()}`);
  return (await res.json()) as { answers: Record<string, { type: string; noul?: number; choice?: string; confidence?: number }>; usage: { input_tokens: number } };
}

function messagesFor(c: RoutingCase) {
  const recent = (c.recent ?? []).map((t, i) => ({ id: `r${i}`, role: t.role, parts: [{ type: "text", text: t.text }] }));
  return [...recent, { id: "ask", role: "user", parts: [{ type: "text", text: c.request }] }] as never;
}

interface Judged {
  c: RoutingCase;
  answers: TurnJudgeAnswers;
  ms: number;
  tokens: number;
}

interface Summary {
  skillWrong: number;
  skillNeedless: number;
  skillMissed: number;
  areaMisses: number;
  intentWrong: number;
  meanTools: number;
}

function summarize(judged: Judged[], settings: typeof CUT_JUDGE): Summary {
  let skillWrong = 0;
  let skillNeedless = 0;
  let skillMissed = 0;
  let areaMisses = 0;
  let intentWrong = 0;
  let tools = 0;
  for (const j of judged) {
    const r = routeTurn(j.answers, settings);
    const skill = pickSkill(j.answers, settings);
    const ok = skillMatches(j.c, skill);
    if (!ok && skill && j.c.skill) skillWrong++;
    if (!ok && skill && !j.c.skill) skillNeedless++;
    if (!ok && !skill) skillMissed++;
    if (j.c.areas.some((a) => !r.areas.includes(a))) areaMisses++;
    if (j.c.intent && r.intent !== j.c.intent) intentWrong++;
    tools += r.intent === "chat" ? 0 : CORE_TOOLS.length + areaTools(r.areas).length;
  }
  return {
    skillWrong,
    skillNeedless,
    skillMissed,
    areaMisses,
    intentWrong,
    meanTools: Math.round(tools / Math.max(1, judged.length)),
  };
}

const skillMatches = (c: RoutingCase, skill: string | null) =>
  Array.isArray(c.skill) ? c.skill.includes(skill) : skill === c.skill;

const pct = (n: number, d: number) => `${Math.round((100 * n) / Math.max(1, d))}%`;

async function main() {
  const selected = routingCases.filter((c) => !ONLY || c.request.includes(ONLY));
  if (selected.length === 0) throw new Error(`No case matching "${ONLY}".`);
  const defaults = CUT_JUDGE;
  console.log(`== routing  cases=${selected.length}  skills=${AI_SKILL_INDEX.length}  areas=${TOOL_AREA_NAMES.length}`);

  const judged: Judged[] = [];
  for (const c of selected) {
    const t0 = performance.now();
    const state = judgeTurnState(messagesFor(c), EDITOR_STATE);
    const { answers, usage } = await judge(state, TURN_JUDGE_QUESTIONS);
    const j: Judged = { c, answers: answers as unknown as TurnJudgeAnswers, ms: performance.now() - t0, tokens: usage.input_tokens };
    judged.push(j);
    const r = routeTurn(j.answers, defaults);
    const skill = pickSkill(j.answers, defaults);
    const skillOk = skillMatches(c, skill);
    const areasOk = c.areas.every((a) => r.areas.includes(a));
    const intentOk = !c.intent || r.intent === c.intent;
    const mark = skillOk && areasOk && intentOk ? "ok  " : "MISS";
    console.log(
      `  ${mark} ${c.request.slice(0, 60).padEnd(60)}  ${r.intent.padEnd(7)}  skill=${String(skill).padEnd(22)}  areas=${r.areas.length}  ${Math.round(j.ms)}ms`,
    );
    if (!skillOk) {
      const a = j.answers;
      const gate = a.needs_reference.noul.toFixed(2);
      const fit = skill ? (a[`fits::${skill}`] as { noul?: number })?.noul?.toFixed(2) : "-";
      console.log(`       skill: expected ${String(c.skill)}, got ${skill}  needs_reference=${gate}  choice=${a.skill.choice}@${a.skill.confidence.toFixed(2)} fit=${fit}`);
    }
    if (!areasOk) console.log(`       areas: missing ${c.areas.filter((a) => !r.areas.includes(a)).join(", ")} (got ${r.areas.join(", ") || "none"})`);
    if (!intentOk) console.log(`       intent: expected ${c.intent}, got ${r.intent}`);
  }

  const ms = judged.map((j) => j.ms).sort((a, b) => a - b);
  const p50 = ms[Math.floor(ms.length / 2)];
  const meanTokens = Math.round(judged.reduce((s, j) => s + j.tokens, 0) / judged.length);
  const d = summarize(judged, defaults);
  console.log(`\n== defaults  skillGate=${defaults.skillGate} skillFits=${defaults.skillFits} toolArea=${defaults.toolArea}`);
  console.log(
    `  skill wrong ${pct(d.skillWrong, judged.length)}  needless ${pct(d.skillNeedless, judged.length)}  missed ${pct(d.skillMissed, judged.length)}` +
      `  area misses ${pct(d.areaMisses, judged.length)}  intent wrong ${pct(d.intentWrong, judged.length)}  mean tools ${d.meanTools}` +
      `  judge p50 ${Math.round(p50)}ms  mean input tokens ${meanTokens}`,
  );

  if (SWEEP) {
    const grid = { skillGate: [0.2, 0.3, 0.4, 0.5], skillFits: [0.2, 0.3, 0.4, 0.5], toolArea: [0.15, 0.2, 0.25, 0.3, 0.4, 0.5] };
    const rows: { s: typeof defaults; r: Summary }[] = [];
    for (const skillGate of grid.skillGate)
      for (const skillFits of grid.skillFits)
        for (const toolArea of grid.toolArea) {
          const s = { ...defaults, skillGate, skillFits, toolArea };
          rows.push({ s, r: summarize(judged, s) });
        }
    // Misses cost more than tokens: order by area misses, then wrong or
    // needless skills, then missed skills, then the declared count.
    rows.sort(
      (a, b) =>
        a.r.areaMisses - b.r.areaMisses ||
        a.r.skillWrong + a.r.skillNeedless - (b.r.skillWrong + b.r.skillNeedless) ||
        a.r.skillMissed - b.r.skillMissed ||
        a.r.meanTools - b.r.meanTools,
    );
    console.log(`\n== sweep (best first)`);
    for (const { s, r } of rows.slice(0, 8))
      console.log(
        `  gate=${s.skillGate} fits=${s.skillFits} area=${s.toolArea}  area misses ${r.areaMisses}  skill wrong ${r.skillWrong} needless ${r.skillNeedless} missed ${r.skillMissed}  mean tools ${r.meanTools}`,
      );
  }

  if (FILLER) {
    console.log(`\n== filler words`);
    const words = FILLER_CUES.flatMap((c) => c.words.map((w, i) => ({ key: `word::${c.id}::${i}`, cueId: c.id, i, w: w.w })));
    const questions = Object.fromEntries(
      words.map((w) => [
        w.key,
        noul(
          `In \`cues\`, is word ${w.i} of cue "${w.cueId}" a disfluency or false start a careful editor would cut out of the speech — an um, uh, er, hmm; a stranded "like", "you know", "I mean", "sort of"; a stutter or an immediately repeated word — judged in the sentence it sits in?`,
        ),
      ]),
    );
    const state = { cues: FILLER_CUES.map((c) => ({ id: c.id, text: c.text, words: c.words.map((w, i) => ({ i, w: w.w })) })) };
    const { answers } = await judge(state, questions);
    for (const w of words) {
      const p = answers[w.key]?.noul ?? 0;
      console.log(`  ${p >= defaults.fillerCut ? "CUT " : "keep"} ${w.w.padEnd(12)} ${p.toFixed(2)}`);
    }
  }
}

main().catch((err) => {
  console.error(err instanceof Error ? err.message : err);
  process.exit(1);
});