Open source CapCut alternative.
TypeScript
feat(site): routing eval
David Le294865f8f799
diff --git a/site/package.json b/site/package.jsonindex 568e407a..c9e99b9a 100644--- a/site/package.json+++ b/site/package.json@@ -23,6 +23,7 @@"bench:watch": "bun run scripts/bench-watch.ts","eval:cut-chat": "bun run scripts/eval-cut-chat.ts","eval:cut-queue": "bun run scripts/eval-cut-queue.ts",+ "eval:cut-routing": "bun run scripts/eval-cut-routing.ts","eval:chatgpt-workflow": "bun run scripts/eval-chatgpt-workflow.ts","eval:cut-headless": "bun run scripts/eval-cut-headless.ts","email:dev": "email dev --dir src/emails --port 3001",diff --git a/site/scripts/eval-cut-routing.ts b/site/scripts/eval-cut-routing.tsnew file mode 100644index 00000000..da19f290--- /dev/null+++ b/site/scripts/eval-cut-routing.ts@@ -0,0 +1,200 @@+#!/usr/bin/env bun+/**+ * The Cut routing eval: what the turn judge decides for a labelled set of+ * asks — the skill it attaches, the tool areas it declares, the gate side —+ * and how the thresholds in the `cutJudge` setting trade suggestion+ * accuracy against declared tool count.+ *+ * Every case is judged once (the answers are cached for the run), then the+ * route is composed under a grid of thresholds in code, so a sweep costs no+ * extra calls. The report names the best triple; the registry defaults are+ * set from it.+ *+ * Run with the site dev server up:+ * bun run scripts/eval-cut-routing.ts+ * [--base http://localhost:3000]+ * [--only <substring>] cases whose request contains it+ * [--no-sweep] defaults only+ * [--filler] also judge the filler-word fixture+ *+ * Auth is the dev bypass header (scripts only — never the app), so runs are+ * dev-server-only and spend no credits.+ */++import { SETTINGS } from "../src/lib/config/registry";+import { noul } from "../src/lib/inference/judge";+import { AI_SKILL_INDEX, areaTools, CORE_TOOLS, TOOL_AREA_NAMES } from "../src/cut/server/ai/catalog";+import { judgeTurnState, pickSkill, routeTurn, TURN_JUDGE_QUESTIONS, type TurnJudgeAnswers } from "../src/cut/lib/turnJudge";+import { EDITOR_STATE, FILLER_CUES } from "./lib/cut-eval/fixtures";+import { routingCases, type RoutingCase } from "./lib/cut-eval/routingCases";++const args = process.argv.slice(2);+const argValue = (flag: string) => {+ const i = args.indexOf(flag);+ return i >= 0 ? args[i + 1] : undefined;+};+const BASE = argValue("--base") ?? "http://localhost:3000";+const ONLY = argValue("--only");+const SWEEP = !args.includes("--no-sweep");+const FILLER = args.includes("--filler");++async function judge(state: unknown, questions: Record<string, unknown>) {+ const res = await fetch(`${BASE}/api/inference/judge`, {+ method: "POST",+ headers: {+ "Content-Type": "application/json",+ "x-donkey-client-id": "donkey-cut-eval",+ "x-donkey-dev-auth-bypass": "1",+ },+ body: JSON.stringify({ state, questions }),+ });+ if (!res.ok) throw new Error(`judge ${res.status}: ${await res.text()}`);+ return (await res.json()) as { answers: Record<string, { type: string; noul?: number; choice?: string; confidence?: number }>; usage: { input_tokens: number } };+}++function messagesFor(c: RoutingCase) {+ const recent = (c.recent ?? []).map((t, i) => ({ id: `r${i}`, role: t.role, parts: [{ type: "text", text: t.text }] }));+ return [...recent, { id: "ask", role: "user", parts: [{ type: "text", text: c.request }] }] as never;+}++interface Judged {+ c: RoutingCase;+ answers: TurnJudgeAnswers;+ ms: number;+ tokens: number;+}++interface Summary {+ skillWrong: number;+ skillNeedless: number;+ skillMissed: number;+ areaMisses: number;+ intentWrong: number;+ meanTools: number;+}++function summarize(judged: Judged[], settings: typeof SETTINGS.cutJudge.default): Summary {+ let skillWrong = 0;+ let skillNeedless = 0;+ let skillMissed = 0;+ let areaMisses = 0;+ let intentWrong = 0;+ let tools = 0;+ for (const j of judged) {+ const r = routeTurn(j.answers, settings);+ const skill = pickSkill(j.answers, settings);+ const ok = skillMatches(j.c, skill);+ if (!ok && skill && j.c.skill) skillWrong++;+ if (!ok && skill && !j.c.skill) skillNeedless++;+ if (!ok && !skill) skillMissed++;+ if (j.c.areas.some((a) => !r.areas.includes(a))) areaMisses++;+ if (j.c.intent && r.intent !== j.c.intent) intentWrong++;+ tools += r.intent === "chat" ? 0 : CORE_TOOLS.length + areaTools(r.areas).length;+ }+ return {+ skillWrong,+ skillNeedless,+ skillMissed,+ areaMisses,+ intentWrong,+ meanTools: Math.round(tools / Math.max(1, judged.length)),+ };+}++const skillMatches = (c: RoutingCase, skill: string | null) =>+ Array.isArray(c.skill) ? c.skill.includes(skill) : skill === c.skill;++const pct = (n: number, d: number) => `${Math.round((100 * n) / Math.max(1, d))}%`;++async function main() {+ const selected = routingCases.filter((c) => !ONLY || c.request.includes(ONLY));+ if (selected.length === 0) throw new Error(`No case matching "${ONLY}".`);+ const defaults = SETTINGS.cutJudge.default;+ console.log(`== routing cases=${selected.length} skills=${AI_SKILL_INDEX.length} areas=${TOOL_AREA_NAMES.length}`);++ const judged: Judged[] = [];+ for (const c of selected) {+ const t0 = performance.now();+ const state = judgeTurnState(messagesFor(c), EDITOR_STATE);+ const { answers, usage } = await judge(state, TURN_JUDGE_QUESTIONS);+ const j: Judged = { c, answers: answers as unknown as TurnJudgeAnswers, ms: performance.now() - t0, tokens: usage.input_tokens };+ judged.push(j);+ const r = routeTurn(j.answers, defaults);+ const skill = pickSkill(j.answers, defaults);+ const skillOk = skillMatches(c, skill);+ const areasOk = c.areas.every((a) => r.areas.includes(a));+ const intentOk = !c.intent || r.intent === c.intent;+ const mark = skillOk && areasOk && intentOk ? "ok " : "MISS";+ console.log(+ ` ${mark} ${c.request.slice(0, 60).padEnd(60)} ${r.intent.padEnd(7)} skill=${String(skill).padEnd(22)} areas=${r.areas.length} ${Math.round(j.ms)}ms`,+ );+ if (!skillOk) {+ const a = j.answers;+ const gate = a.needs_reference.noul.toFixed(2);+ const fit = skill ? (a[`fits::${skill}`] as { noul?: number })?.noul?.toFixed(2) : "-";+ console.log(` skill: expected ${String(c.skill)}, got ${skill} needs_reference=${gate} choice=${a.skill.choice}@${a.skill.confidence.toFixed(2)} fit=${fit}`);+ }+ if (!areasOk) console.log(` areas: missing ${c.areas.filter((a) => !r.areas.includes(a)).join(", ")} (got ${r.areas.join(", ") || "none"})`);+ if (!intentOk) console.log(` intent: expected ${c.intent}, got ${r.intent}`);+ }++ const ms = judged.map((j) => j.ms).sort((a, b) => a - b);+ const p50 = ms[Math.floor(ms.length / 2)];+ const meanTokens = Math.round(judged.reduce((s, j) => s + j.tokens, 0) / judged.length);+ const d = summarize(judged, defaults);+ console.log(`\n== defaults skillGate=${defaults.skillGate} skillFits=${defaults.skillFits} toolArea=${defaults.toolArea}`);+ console.log(+ ` skill wrong ${pct(d.skillWrong, judged.length)} needless ${pct(d.skillNeedless, judged.length)} missed ${pct(d.skillMissed, judged.length)}` ++ ` area misses ${pct(d.areaMisses, judged.length)} intent wrong ${pct(d.intentWrong, judged.length)} mean tools ${d.meanTools}` ++ ` judge p50 ${Math.round(p50)}ms mean input tokens ${meanTokens}`,+ );++ if (SWEEP) {+ const grid = { skillGate: [0.2, 0.3, 0.4, 0.5], skillFits: [0.2, 0.3, 0.4, 0.5], toolArea: [0.15, 0.2, 0.25, 0.3, 0.4, 0.5] };+ const rows: { s: typeof defaults; r: Summary }[] = [];+ for (const skillGate of grid.skillGate)+ for (const skillFits of grid.skillFits)+ for (const toolArea of grid.toolArea) {+ const s = { ...defaults, skillGate, skillFits, toolArea };+ rows.push({ s, r: summarize(judged, s) });+ }+ // Misses cost more than tokens: order by area misses, then wrong or+ // needless skills, then missed skills, then the declared count.+ rows.sort(+ (a, b) =>+ a.r.areaMisses - b.r.areaMisses ||+ a.r.skillWrong + a.r.skillNeedless - (b.r.skillWrong + b.r.skillNeedless) ||+ a.r.skillMissed - b.r.skillMissed ||+ a.r.meanTools - b.r.meanTools,+ );+ console.log(`\n== sweep (best first)`);+ for (const { s, r } of rows.slice(0, 8))+ console.log(+ ` gate=${s.skillGate} fits=${s.skillFits} area=${s.toolArea} area misses ${r.areaMisses} skill wrong ${r.skillWrong} needless ${r.skillNeedless} missed ${r.skillMissed} mean tools ${r.meanTools}`,+ );+ }++ if (FILLER) {+ console.log(`\n== filler words`);+ const words = FILLER_CUES.flatMap((c) => c.words.map((w, i) => ({ key: `word::${c.id}::${i}`, cueId: c.id, i, w: w.w })));+ const questions = Object.fromEntries(+ words.map((w) => [+ w.key,+ noul(+ `In \`cues\`, is word ${w.i} of cue "${w.cueId}" a disfluency or false start a careful editor would cut out of the speech — an um, uh, er, hmm; a stranded "like", "you know", "I mean", "sort of"; a stutter or an immediately repeated word — judged in the sentence it sits in?`,+ ),+ ]),+ );+ const state = { cues: FILLER_CUES.map((c) => ({ id: c.id, text: c.text, words: c.words.map((w, i) => ({ i, w: w.w })) })) };+ const { answers } = await judge(state, questions);+ for (const w of words) {+ const p = answers[w.key]?.noul ?? 0;+ console.log(` ${p >= defaults.fillerCut ? "CUT " : "keep"} ${w.w.padEnd(12)} ${p.toFixed(2)}`);+ }+ }+}++main().catch((err) => {+ console.error(err instanceof Error ? err.message : err);+ process.exit(1);+});diff --git a/site/scripts/lib/cut-eval/routingCases.ts b/site/scripts/lib/cut-eval/routingCases.tsnew file mode 100644index 00000000..8a13a00b--- /dev/null+++ b/site/scripts/lib/cut-eval/routingCases.ts@@ -0,0 +1,113 @@+// The routing eval's labelled turns: what the judge should decide for each.+// `skill` is the document a careful editor opens first (null when none, a+// list when more than one reads right), and+// `areas` are the tool areas the ask cannot be finished without — the eval+// checks that every gold area is declared, so a routed turn never dead-ends.+// `intent` pins the gate side where it matters.++export interface RoutingCase {+ request: string;+ recent?: { role: "user" | "assistant"; text: string }[];+ skill: string | (string | null)[] | null;+ areas: string[];+ intent?: "chat" | "simple" | "complex";+}++export const routingCases: RoutingCase[] = [+ // ── uncovered turns: no skill, no work ────────────────────────────────+ { request: "hi", skill: null, areas: [], intent: "chat" },+ { request: "thanks, that's perfect", skill: null, areas: [], intent: "chat" },+ { request: "ok bye", skill: null, areas: [], intent: "chat" },+ { request: "how long is my cut right now?", skill: null, areas: [], intent: "simple" },+ { request: "what's on the timeline?", skill: null, areas: [], intent: "simple" },+ { request: "which clips have captions?", skill: null, areas: [], intent: "simple" },+ { request: "write me a caption for tiktok about this beach day", skill: null, areas: [], intent: "simple" },+ { request: "give me three title ideas for this video", skill: null, areas: [] },+ // ── single plainly named tools: no skill ──────────────────────────────+ { request: "mute the second clip", skill: null, areas: ["timeline"], intent: "simple" },+ { request: "delete the last title", skill: null, areas: ["timeline"], intent: "simple" },+ { request: "rename the project to Beach Day", skill: null, areas: ["top_bar"], intent: "simple" },+ { request: "switch the aspect to 16:9", skill: null, areas: ["top_bar"], intent: "simple" },+ { request: "undo that", skill: null, areas: ["editor"], intent: "simple" },+ { request: "hide track 1", skill: null, areas: ["timeline"], intent: "simple" },+ { request: "seek to 12 seconds", skill: null, areas: ["preview"], intent: "simple" },+ { request: "show me the frame at 4 seconds", skill: null, areas: ["preview"], intent: "simple" },+ { request: "open the export dialog", skill: null, areas: ["top_bar"], intent: "simple" },+ { request: "select the first clip", skill: null, areas: ["timeline"], intent: "simple" },+ {+ request: "yes do it",+ recent: [+ { role: "user", text: "could you clean up my captions? lots of filler words" },+ { role: "assistant", text: "Happy to — I'd tidy the five cues on track 0, dropping the ums and uhs. Want me to go ahead?" },+ ],+ skill: [null, "audio-and-subtitles"],+ areas: ["subtitles"],+ intent: "simple",+ },+ // ── editor-overview ───────────────────────────────────────────────────+ { request: "where do generated images end up, the media tab?", skill: ["editor-overview", "media-and-library"], areas: [] },+ { request: "how do the tracks stack, which one is in front?", skill: ["editor-overview", "timeline-editing"], areas: [] },+ // ── timeline-editing ──────────────────────────────────────────────────+ { request: "move the third clip to the start and close the gap it leaves", skill: "timeline-editing", areas: ["timeline"] },+ { request: "group the intro clips so they move together", skill: "timeline-editing", areas: ["timeline"] },+ { request: "put the b-roll on a track above the interview from 5 to 9 seconds", skill: "timeline-editing", areas: ["timeline"] },+ // ── color-grading ─────────────────────────────────────────────────────+ { request: "make the second clip warmer", skill: "color-grading", areas: ["color"], intent: "simple" },+ { request: "the footage looks flat and milky, fix the blacks", skill: "color-grading", areas: ["color"] },+ { request: "match the color of my clip to this reference", skill: "color-grading", areas: ["color"] },+ // ── background-removal ────────────────────────────────────────────────+ { request: "remove the background from the presenter clip", skill: "background-removal", areas: ["removal"] },+ { request: "cut me out of this and put the city behind me", skill: "background-removal", areas: ["removal"] },+ { request: "the cutout has rough hair edges, refine it", skill: "background-removal", areas: ["removal"] },+ // ── watching-and-cutting ──────────────────────────────────────────────+ { request: "cut out the dead air", skill: "watching-and-cutting", areas: ["timeline"], intent: "complex" },+ { request: "cut the filler words out of my video", skill: "watching-and-cutting", areas: ["timeline"], intent: "complex" },+ { request: "clip the best moment from the interview", skill: "watching-and-cutting", areas: ["timeline"] },+ { request: "split wherever the scene changes", skill: "watching-and-cutting", areas: ["timeline"], intent: "complex" },+ // ── transitions-and-fades ─────────────────────────────────────────────+ { request: "add a crossfade between the first two clips", skill: "transitions-and-fades", areas: ["transitions"], intent: "simple" },+ { request: "fade the last clip out to black", skill: "transitions-and-fades", areas: ["transitions"] },+ { request: "make the title slide in", skill: "transitions-and-fades", areas: ["animation"] },+ // ── graphics ──────────────────────────────────────────────────────────+ { request: "add a title that says Summer 2026 in the top third with a dark plate", skill: "graphics", areas: ["timeline"] },+ { request: "put a rounded rectangle behind the title", skill: "graphics", areas: ["elements"] },+ { request: "build a 2x2 grid of my four clips like this reference", skill: "graphics", areas: ["timeline", "inspector"], intent: "complex" },+ { request: "add a sticker of a smiling sun in the corner", skill: "graphics", areas: ["elements"] },+ // ── text-videos ───────────────────────────────────────────────────────+ { request: "build a lyric video from this song", skill: "text-videos", areas: ["subtitles", "timeline"], intent: "complex" },+ { request: "make a quote card video with the three quotes I pasted", skill: "text-videos", areas: ["timeline"], intent: "complex" },+ { request: "kinetic type of what he says, make it punchy", skill: "text-videos", areas: ["timeline", "subtitles"], intent: "complex" },+ // ── text-creativity ───────────────────────────────────────────────────+ { request: "the text looks boring and samey, make it creative", skill: "text-creativity", areas: ["inspector", "animation"] },+ { request: "give the words more life, every line is dead center", skill: "text-creativity", areas: ["inspector"] },+ // ── audio-and-subtitles ───────────────────────────────────────────────+ { request: "make the music as loud as the voice", skill: "audio-and-subtitles", areas: ["inspector"] },+ { request: "add a voiceover reading my script in a warm older man's voice", skill: "audio-and-subtitles", areas: ["audio"] },+ { request: "translate the captions to Korean on a second track", skill: "audio-and-subtitles", areas: ["subtitles"] },+ { request: "the captions are off from the speech, retime them", skill: "audio-and-subtitles", areas: ["subtitles"] },+ // ── ai-generation ─────────────────────────────────────────────────────+ { request: "generate a 5 second clip of a rainy tokyo street at night", skill: "ai-generation", areas: ["video_gen"] },+ { request: "find me some stock footage of a busy city", skill: "ai-generation", areas: ["stock"], intent: "simple" },+ { request: "make an image of a golden retriever on a surfboard", skill: "ai-generation", areas: ["image_gen"] },+ { request: "add a whoosh at the cut", skill: ["ai-generation", "audio-and-subtitles"], areas: ["stock"] },+ { request: "import this tiktok link into my project", skill: "ai-generation", areas: [] },+ { request: "the beach clip feels off — make me a better version, more of a golden-hour sunset", skill: "ai-generation", areas: ["video_gen"] },+ // ── scene-productions ─────────────────────────────────────────────────+ { request: "turn my narration into a smooth 2D cartoon", skill: "scene-productions", areas: ["scene"] },+ { request: "make a narrated three-shot story about a kite that gets away", skill: "scene-productions", areas: ["scene"], intent: "complex" },+ { request: "regenerate shot 2 with the boy holding the kite", skill: "scene-productions", areas: ["scene"] },+ // ── editing-taste ─────────────────────────────────────────────────────+ { request: "tighten this up and make it flow better", skill: ["editing-taste", "watching-and-cutting"], areas: ["timeline"], intent: "complex" },+ { request: "should I put a transition on every cut?", skill: ["editing-taste", "transitions-and-fades"], areas: [] },+ // ── media-and-library ─────────────────────────────────────────────────+ { request: "save this cutout to my library", skill: "media-and-library", areas: ["side_panel"] },+ { request: "move the generated voiceover into media", skill: "media-and-library", areas: ["side_panel"] },+ { request: "save this project as a template", skill: "media-and-library", areas: ["library"] },+ // ── replicating-a-project ─────────────────────────────────────────────+ { request: "replicate this project with my footage: https://donkeycut.com/app/p/abc123", skill: "replicating-a-project", areas: [], intent: "complex" },+ { request: "bring over the title style from https://donkeycut.com/s/xyz", skill: "replicating-a-project", areas: [] },+ // ── publish-and-export ────────────────────────────────────────────────+ { request: "export a 4K version for youtube", skill: "publish-and-export", areas: ["top_bar"] },+ { request: "set the tiktok caption and tags for this", skill: "publish-and-export", areas: ["side_panel"] },+ { request: "export just the selected part as prores", skill: "publish-and-export", areas: ["top_bar"] },+];