Skip to content

Open source CapCut alternative.

TypeScript

feat(site): routing eval

David Le294865f8f799
diff --git a/site/package.json b/site/package.json
index 568e407a..c9e99b9a 100644
--- a/site/package.json
+++ b/site/package.json
@@ -23,6 +23,7 @@
"bench:watch": "bun run scripts/bench-watch.ts",
"eval:cut-chat": "bun run scripts/eval-cut-chat.ts",
"eval:cut-queue": "bun run scripts/eval-cut-queue.ts",
+ "eval:cut-routing": "bun run scripts/eval-cut-routing.ts",
"eval:chatgpt-workflow": "bun run scripts/eval-chatgpt-workflow.ts",
"eval:cut-headless": "bun run scripts/eval-cut-headless.ts",
"email:dev": "email dev --dir src/emails --port 3001",
diff --git a/site/scripts/eval-cut-routing.ts b/site/scripts/eval-cut-routing.ts
new file mode 100644
index 00000000..da19f290
--- /dev/null
+++ b/site/scripts/eval-cut-routing.ts
@@ -0,0 +1,200 @@
+#!/usr/bin/env bun
+/**
+ * The Cut routing eval: what the turn judge decides for a labelled set of
+ * asks — the skill it attaches, the tool areas it declares, the gate side —
+ * and how the thresholds in the `cutJudge` setting trade suggestion
+ * accuracy against declared tool count.
+ *
+ * Every case is judged once (the answers are cached for the run), then the
+ * route is composed under a grid of thresholds in code, so a sweep costs no
+ * extra calls. The report names the best triple; the registry defaults are
+ * set from it.
+ *
+ * Run with the site dev server up:
+ * bun run scripts/eval-cut-routing.ts
+ * [--base http://localhost:3000]
+ * [--only <substring>] cases whose request contains it
+ * [--no-sweep] defaults only
+ * [--filler] also judge the filler-word fixture
+ *
+ * Auth is the dev bypass header (scripts only — never the app), so runs are
+ * dev-server-only and spend no credits.
+ */
+
+import { SETTINGS } from "../src/lib/config/registry";
+import { noul } from "../src/lib/inference/judge";
+import { AI_SKILL_INDEX, areaTools, CORE_TOOLS, TOOL_AREA_NAMES } from "../src/cut/server/ai/catalog";
+import { judgeTurnState, pickSkill, routeTurn, TURN_JUDGE_QUESTIONS, type TurnJudgeAnswers } from "../src/cut/lib/turnJudge";
+import { EDITOR_STATE, FILLER_CUES } from "./lib/cut-eval/fixtures";
+import { routingCases, type RoutingCase } from "./lib/cut-eval/routingCases";
+
+const args = process.argv.slice(2);
+const argValue = (flag: string) => {
+ const i = args.indexOf(flag);
+ return i >= 0 ? args[i + 1] : undefined;
+};
+const BASE = argValue("--base") ?? "http://localhost:3000";
+const ONLY = argValue("--only");
+const SWEEP = !args.includes("--no-sweep");
+const FILLER = args.includes("--filler");
+
+async function judge(state: unknown, questions: Record<string, unknown>) {
+ const res = await fetch(`${BASE}/api/inference/judge`, {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ "x-donkey-client-id": "donkey-cut-eval",
+ "x-donkey-dev-auth-bypass": "1",
+ },
+ body: JSON.stringify({ state, questions }),
+ });
+ if (!res.ok) throw new Error(`judge ${res.status}: ${await res.text()}`);
+ return (await res.json()) as { answers: Record<string, { type: string; noul?: number; choice?: string; confidence?: number }>; usage: { input_tokens: number } };
+}
+
+function messagesFor(c: RoutingCase) {
+ const recent = (c.recent ?? []).map((t, i) => ({ id: `r${i}`, role: t.role, parts: [{ type: "text", text: t.text }] }));
+ return [...recent, { id: "ask", role: "user", parts: [{ type: "text", text: c.request }] }] as never;
+}
+
+interface Judged {
+ c: RoutingCase;
+ answers: TurnJudgeAnswers;
+ ms: number;
+ tokens: number;
+}
+
+interface Summary {
+ skillWrong: number;
+ skillNeedless: number;
+ skillMissed: number;
+ areaMisses: number;
+ intentWrong: number;
+ meanTools: number;
+}
+
+function summarize(judged: Judged[], settings: typeof SETTINGS.cutJudge.default): Summary {
+ let skillWrong = 0;
+ let skillNeedless = 0;
+ let skillMissed = 0;
+ let areaMisses = 0;
+ let intentWrong = 0;
+ let tools = 0;
+ for (const j of judged) {
+ const r = routeTurn(j.answers, settings);
+ const skill = pickSkill(j.answers, settings);
+ const ok = skillMatches(j.c, skill);
+ if (!ok && skill && j.c.skill) skillWrong++;
+ if (!ok && skill && !j.c.skill) skillNeedless++;
+ if (!ok && !skill) skillMissed++;
+ if (j.c.areas.some((a) => !r.areas.includes(a))) areaMisses++;
+ if (j.c.intent && r.intent !== j.c.intent) intentWrong++;
+ tools += r.intent === "chat" ? 0 : CORE_TOOLS.length + areaTools(r.areas).length;
+ }
+ return {
+ skillWrong,
+ skillNeedless,
+ skillMissed,
+ areaMisses,
+ intentWrong,
+ meanTools: Math.round(tools / Math.max(1, judged.length)),
+ };
+}
+
+const skillMatches = (c: RoutingCase, skill: string | null) =>
+ Array.isArray(c.skill) ? c.skill.includes(skill) : skill === c.skill;
+
+const pct = (n: number, d: number) => `${Math.round((100 * n) / Math.max(1, d))}%`;
+
+async function main() {
+ const selected = routingCases.filter((c) => !ONLY || c.request.includes(ONLY));
+ if (selected.length === 0) throw new Error(`No case matching "${ONLY}".`);
+ const defaults = SETTINGS.cutJudge.default;
+ console.log(`== routing cases=${selected.length} skills=${AI_SKILL_INDEX.length} areas=${TOOL_AREA_NAMES.length}`);
+
+ const judged: Judged[] = [];
+ for (const c of selected) {
+ const t0 = performance.now();
+ const state = judgeTurnState(messagesFor(c), EDITOR_STATE);
+ const { answers, usage } = await judge(state, TURN_JUDGE_QUESTIONS);
+ const j: Judged = { c, answers: answers as unknown as TurnJudgeAnswers, ms: performance.now() - t0, tokens: usage.input_tokens };
+ judged.push(j);
+ const r = routeTurn(j.answers, defaults);
+ const skill = pickSkill(j.answers, defaults);
+ const skillOk = skillMatches(c, skill);
+ const areasOk = c.areas.every((a) => r.areas.includes(a));
+ const intentOk = !c.intent || r.intent === c.intent;
+ const mark = skillOk && areasOk && intentOk ? "ok " : "MISS";
+ console.log(
+ ` ${mark} ${c.request.slice(0, 60).padEnd(60)} ${r.intent.padEnd(7)} skill=${String(skill).padEnd(22)} areas=${r.areas.length} ${Math.round(j.ms)}ms`,
+ );
+ if (!skillOk) {
+ const a = j.answers;
+ const gate = a.needs_reference.noul.toFixed(2);
+ const fit = skill ? (a[`fits::${skill}`] as { noul?: number })?.noul?.toFixed(2) : "-";
+ console.log(` skill: expected ${String(c.skill)}, got ${skill} needs_reference=${gate} choice=${a.skill.choice}@${a.skill.confidence.toFixed(2)} fit=${fit}`);
+ }
+ if (!areasOk) console.log(` areas: missing ${c.areas.filter((a) => !r.areas.includes(a)).join(", ")} (got ${r.areas.join(", ") || "none"})`);
+ if (!intentOk) console.log(` intent: expected ${c.intent}, got ${r.intent}`);
+ }
+
+ const ms = judged.map((j) => j.ms).sort((a, b) => a - b);
+ const p50 = ms[Math.floor(ms.length / 2)];
+ const meanTokens = Math.round(judged.reduce((s, j) => s + j.tokens, 0) / judged.length);
+ const d = summarize(judged, defaults);
+ console.log(`\n== defaults skillGate=${defaults.skillGate} skillFits=${defaults.skillFits} toolArea=${defaults.toolArea}`);
+ console.log(
+ ` skill wrong ${pct(d.skillWrong, judged.length)} needless ${pct(d.skillNeedless, judged.length)} missed ${pct(d.skillMissed, judged.length)}` +
+ ` area misses ${pct(d.areaMisses, judged.length)} intent wrong ${pct(d.intentWrong, judged.length)} mean tools ${d.meanTools}` +
+ ` judge p50 ${Math.round(p50)}ms mean input tokens ${meanTokens}`,
+ );
+
+ if (SWEEP) {
+ const grid = { skillGate: [0.2, 0.3, 0.4, 0.5], skillFits: [0.2, 0.3, 0.4, 0.5], toolArea: [0.15, 0.2, 0.25, 0.3, 0.4, 0.5] };
+ const rows: { s: typeof defaults; r: Summary }[] = [];
+ for (const skillGate of grid.skillGate)
+ for (const skillFits of grid.skillFits)
+ for (const toolArea of grid.toolArea) {
+ const s = { ...defaults, skillGate, skillFits, toolArea };
+ rows.push({ s, r: summarize(judged, s) });
+ }
+ // Misses cost more than tokens: order by area misses, then wrong or
+ // needless skills, then missed skills, then the declared count.
+ rows.sort(
+ (a, b) =>
+ a.r.areaMisses - b.r.areaMisses ||
+ a.r.skillWrong + a.r.skillNeedless - (b.r.skillWrong + b.r.skillNeedless) ||
+ a.r.skillMissed - b.r.skillMissed ||
+ a.r.meanTools - b.r.meanTools,
+ );
+ console.log(`\n== sweep (best first)`);
+ for (const { s, r } of rows.slice(0, 8))
+ console.log(
+ ` gate=${s.skillGate} fits=${s.skillFits} area=${s.toolArea} area misses ${r.areaMisses} skill wrong ${r.skillWrong} needless ${r.skillNeedless} missed ${r.skillMissed} mean tools ${r.meanTools}`,
+ );
+ }
+
+ if (FILLER) {
+ console.log(`\n== filler words`);
+ const words = FILLER_CUES.flatMap((c) => c.words.map((w, i) => ({ key: `word::${c.id}::${i}`, cueId: c.id, i, w: w.w })));
+ const questions = Object.fromEntries(
+ words.map((w) => [
+ w.key,
+ noul(
+ `In \`cues\`, is word ${w.i} of cue "${w.cueId}" a disfluency or false start a careful editor would cut out of the speech — an um, uh, er, hmm; a stranded "like", "you know", "I mean", "sort of"; a stutter or an immediately repeated word — judged in the sentence it sits in?`,
+ ),
+ ]),
+ );
+ const state = { cues: FILLER_CUES.map((c) => ({ id: c.id, text: c.text, words: c.words.map((w, i) => ({ i, w: w.w })) })) };
+ const { answers } = await judge(state, questions);
+ for (const w of words) {
+ const p = answers[w.key]?.noul ?? 0;
+ console.log(` ${p >= defaults.fillerCut ? "CUT " : "keep"} ${w.w.padEnd(12)} ${p.toFixed(2)}`);
+ }
+ }
+}
+
+main().catch((err) => {
+ console.error(err instanceof Error ? err.message : err);
+ process.exit(1);
+});
diff --git a/site/scripts/lib/cut-eval/routingCases.ts b/site/scripts/lib/cut-eval/routingCases.ts
new file mode 100644
index 00000000..8a13a00b
--- /dev/null
+++ b/site/scripts/lib/cut-eval/routingCases.ts
@@ -0,0 +1,113 @@
+// The routing eval's labelled turns: what the judge should decide for each.
+// `skill` is the document a careful editor opens first (null when none, a
+// list when more than one reads right), and
+// `areas` are the tool areas the ask cannot be finished without — the eval
+// checks that every gold area is declared, so a routed turn never dead-ends.
+// `intent` pins the gate side where it matters.
+
+export interface RoutingCase {
+ request: string;
+ recent?: { role: "user" | "assistant"; text: string }[];
+ skill: string | (string | null)[] | null;
+ areas: string[];
+ intent?: "chat" | "simple" | "complex";
+}
+
+export const routingCases: RoutingCase[] = [
+ // ── uncovered turns: no skill, no work ────────────────────────────────
+ { request: "hi", skill: null, areas: [], intent: "chat" },
+ { request: "thanks, that's perfect", skill: null, areas: [], intent: "chat" },
+ { request: "ok bye", skill: null, areas: [], intent: "chat" },
+ { request: "how long is my cut right now?", skill: null, areas: [], intent: "simple" },
+ { request: "what's on the timeline?", skill: null, areas: [], intent: "simple" },
+ { request: "which clips have captions?", skill: null, areas: [], intent: "simple" },
+ { request: "write me a caption for tiktok about this beach day", skill: null, areas: [], intent: "simple" },
+ { request: "give me three title ideas for this video", skill: null, areas: [] },
+ // ── single plainly named tools: no skill ──────────────────────────────
+ { request: "mute the second clip", skill: null, areas: ["timeline"], intent: "simple" },
+ { request: "delete the last title", skill: null, areas: ["timeline"], intent: "simple" },
+ { request: "rename the project to Beach Day", skill: null, areas: ["top_bar"], intent: "simple" },
+ { request: "switch the aspect to 16:9", skill: null, areas: ["top_bar"], intent: "simple" },
+ { request: "undo that", skill: null, areas: ["editor"], intent: "simple" },
+ { request: "hide track 1", skill: null, areas: ["timeline"], intent: "simple" },
+ { request: "seek to 12 seconds", skill: null, areas: ["preview"], intent: "simple" },
+ { request: "show me the frame at 4 seconds", skill: null, areas: ["preview"], intent: "simple" },
+ { request: "open the export dialog", skill: null, areas: ["top_bar"], intent: "simple" },
+ { request: "select the first clip", skill: null, areas: ["timeline"], intent: "simple" },
+ {
+ request: "yes do it",
+ recent: [
+ { role: "user", text: "could you clean up my captions? lots of filler words" },
+ { role: "assistant", text: "Happy to — I'd tidy the five cues on track 0, dropping the ums and uhs. Want me to go ahead?" },
+ ],
+ skill: [null, "audio-and-subtitles"],
+ areas: ["subtitles"],
+ intent: "simple",
+ },
+ // ── editor-overview ───────────────────────────────────────────────────
+ { request: "where do generated images end up, the media tab?", skill: ["editor-overview", "media-and-library"], areas: [] },
+ { request: "how do the tracks stack, which one is in front?", skill: ["editor-overview", "timeline-editing"], areas: [] },
+ // ── timeline-editing ──────────────────────────────────────────────────
+ { request: "move the third clip to the start and close the gap it leaves", skill: "timeline-editing", areas: ["timeline"] },
+ { request: "group the intro clips so they move together", skill: "timeline-editing", areas: ["timeline"] },
+ { request: "put the b-roll on a track above the interview from 5 to 9 seconds", skill: "timeline-editing", areas: ["timeline"] },
+ // ── color-grading ─────────────────────────────────────────────────────
+ { request: "make the second clip warmer", skill: "color-grading", areas: ["color"], intent: "simple" },
+ { request: "the footage looks flat and milky, fix the blacks", skill: "color-grading", areas: ["color"] },
+ { request: "match the color of my clip to this reference", skill: "color-grading", areas: ["color"] },
+ // ── background-removal ────────────────────────────────────────────────
+ { request: "remove the background from the presenter clip", skill: "background-removal", areas: ["removal"] },
+ { request: "cut me out of this and put the city behind me", skill: "background-removal", areas: ["removal"] },
+ { request: "the cutout has rough hair edges, refine it", skill: "background-removal", areas: ["removal"] },
+ // ── watching-and-cutting ──────────────────────────────────────────────
+ { request: "cut out the dead air", skill: "watching-and-cutting", areas: ["timeline"], intent: "complex" },
+ { request: "cut the filler words out of my video", skill: "watching-and-cutting", areas: ["timeline"], intent: "complex" },
+ { request: "clip the best moment from the interview", skill: "watching-and-cutting", areas: ["timeline"] },
+ { request: "split wherever the scene changes", skill: "watching-and-cutting", areas: ["timeline"], intent: "complex" },
+ // ── transitions-and-fades ─────────────────────────────────────────────
+ { request: "add a crossfade between the first two clips", skill: "transitions-and-fades", areas: ["transitions"], intent: "simple" },
+ { request: "fade the last clip out to black", skill: "transitions-and-fades", areas: ["transitions"] },
+ { request: "make the title slide in", skill: "transitions-and-fades", areas: ["animation"] },
+ // ── graphics ──────────────────────────────────────────────────────────
+ { request: "add a title that says Summer 2026 in the top third with a dark plate", skill: "graphics", areas: ["timeline"] },
+ { request: "put a rounded rectangle behind the title", skill: "graphics", areas: ["elements"] },
+ { request: "build a 2x2 grid of my four clips like this reference", skill: "graphics", areas: ["timeline", "inspector"], intent: "complex" },
+ { request: "add a sticker of a smiling sun in the corner", skill: "graphics", areas: ["elements"] },
+ // ── text-videos ───────────────────────────────────────────────────────
+ { request: "build a lyric video from this song", skill: "text-videos", areas: ["subtitles", "timeline"], intent: "complex" },
+ { request: "make a quote card video with the three quotes I pasted", skill: "text-videos", areas: ["timeline"], intent: "complex" },
+ { request: "kinetic type of what he says, make it punchy", skill: "text-videos", areas: ["timeline", "subtitles"], intent: "complex" },
+ // ── text-creativity ───────────────────────────────────────────────────
+ { request: "the text looks boring and samey, make it creative", skill: "text-creativity", areas: ["inspector", "animation"] },
+ { request: "give the words more life, every line is dead center", skill: "text-creativity", areas: ["inspector"] },
+ // ── audio-and-subtitles ───────────────────────────────────────────────
+ { request: "make the music as loud as the voice", skill: "audio-and-subtitles", areas: ["inspector"] },
+ { request: "add a voiceover reading my script in a warm older man's voice", skill: "audio-and-subtitles", areas: ["audio"] },
+ { request: "translate the captions to Korean on a second track", skill: "audio-and-subtitles", areas: ["subtitles"] },
+ { request: "the captions are off from the speech, retime them", skill: "audio-and-subtitles", areas: ["subtitles"] },
+ // ── ai-generation ─────────────────────────────────────────────────────
+ { request: "generate a 5 second clip of a rainy tokyo street at night", skill: "ai-generation", areas: ["video_gen"] },
+ { request: "find me some stock footage of a busy city", skill: "ai-generation", areas: ["stock"], intent: "simple" },
+ { request: "make an image of a golden retriever on a surfboard", skill: "ai-generation", areas: ["image_gen"] },
+ { request: "add a whoosh at the cut", skill: ["ai-generation", "audio-and-subtitles"], areas: ["stock"] },
+ { request: "import this tiktok link into my project", skill: "ai-generation", areas: [] },
+ { request: "the beach clip feels off — make me a better version, more of a golden-hour sunset", skill: "ai-generation", areas: ["video_gen"] },
+ // ── scene-productions ─────────────────────────────────────────────────
+ { request: "turn my narration into a smooth 2D cartoon", skill: "scene-productions", areas: ["scene"] },
+ { request: "make a narrated three-shot story about a kite that gets away", skill: "scene-productions", areas: ["scene"], intent: "complex" },
+ { request: "regenerate shot 2 with the boy holding the kite", skill: "scene-productions", areas: ["scene"] },
+ // ── editing-taste ─────────────────────────────────────────────────────
+ { request: "tighten this up and make it flow better", skill: ["editing-taste", "watching-and-cutting"], areas: ["timeline"], intent: "complex" },
+ { request: "should I put a transition on every cut?", skill: ["editing-taste", "transitions-and-fades"], areas: [] },
+ // ── media-and-library ─────────────────────────────────────────────────
+ { request: "save this cutout to my library", skill: "media-and-library", areas: ["side_panel"] },
+ { request: "move the generated voiceover into media", skill: "media-and-library", areas: ["side_panel"] },
+ { request: "save this project as a template", skill: "media-and-library", areas: ["library"] },
+ // ── replicating-a-project ─────────────────────────────────────────────
+ { request: "replicate this project with my footage: https://donkeycut.com/app/p/abc123", skill: "replicating-a-project", areas: [], intent: "complex" },
+ { request: "bring over the title style from https://donkeycut.com/s/xyz", skill: "replicating-a-project", areas: [] },
+ // ── publish-and-export ────────────────────────────────────────────────
+ { request: "export a 4K version for youtube", skill: "publish-and-export", areas: ["top_bar"] },
+ { request: "set the tiktok caption and tags for this", skill: "publish-and-export", areas: ["side_panel"] },
+ { request: "export just the selected part as prores", skill: "publish-and-export", areas: ["top_bar"] },
+];