Skip to content

Open source CapCut alternative.

TypeScript

test(site): queue triage eval

David Le6dd7e879320b
diff --git a/site/package.json b/site/package.json
index f7884f83..8a8ac3da 100644
--- a/site/package.json
+++ b/site/package.json
@@ -22,6 +22,7 @@
"worker:build": "bun build --target=node --minify --external=pg-native --external=skia-canvas --external=mediabunny --external=@mediabunny/server --external=node-web-audio-api --loader .css:text src/cut/worker/main.ts --outfile dist/cut-worker/main.js",
"bench:watch": "bun run scripts/bench-watch.ts",
"eval:cut-chat": "bun run scripts/eval-cut-chat.ts",
+ "eval:cut-queue": "bun run scripts/eval-cut-queue.ts",
"eval:chatgpt-workflow": "bun run scripts/eval-chatgpt-workflow.ts",
"eval:cut-headless": "bun run scripts/eval-cut-headless.ts",
"email:dev": "email dev --dir src/emails --port 3001",
diff --git a/site/scripts/eval-cut-queue.ts b/site/scripts/eval-cut-queue.ts
new file mode 100644
index 00000000..e837ebd4
--- /dev/null
+++ b/site/scripts/eval-cut-queue.ts
@@ -0,0 +1,266 @@
+#!/usr/bin/env bun
+/**
+ * The Cut queue triage eval: where a message sent mid-turn goes.
+ *
+ * Each case is a running turn plus the messages the user sent while it ran,
+ * placed by the production triage call — fold into the running turn, spawn
+ * a parallel thread, or wait in the tray — and checked against the place
+ * each message must get. A wrong fold does the work in the wrong turn, a
+ * wrong spawn runs two agents over the same clips, and a wrong queue makes
+ * the user wait; every case pins one of those lines.
+ *
+ * The live fold case then runs a real turn through the production loop and
+ * folds a message into it mid-turn, the way the page does: the fold has to
+ * land in the turn's context and the turn has to act on it.
+ *
+ * Run with the site dev server up:
+ * bun run scripts/eval-cut-queue.ts
+ * [--base http://localhost:3000]
+ * [--only <case>[,<case>…]]
+ * [--runs N] default 1; 3+ shows a flaky verdict
+ * [--gate-model <registryKey|rawId>]
+ * [--no-live] skip the live fold turn
+ *
+ * Auth is the dev bypass header (scripts only — never the app), so runs are
+ * dev-server-only and spend no credits.
+ */
+
+import type { UIMessage } from "ai";
+import type { Message } from "@earendil-works/pi-ai";
+import { geminiModelRoleNames, geminiModels, resolveGeminiModel } from "../src/lib/inference/gemini-models";
+import {
+ dropPiSession,
+ foldIntoCutChat,
+ readPiSession,
+ streamCutChat,
+ triageQueuedMessages,
+ type CutAgentDeps,
+} from "../src/cut/lib/pi/cutAgent";
+import type { QueueVerdict } from "../src/cut/lib/queueTriage";
+import { AUDIO_STATE, EDITOR_STATE, SAFE_TOOLS, SCENE_PLANNED, serveSafeTool, userTurn } from "./lib/cut-eval/fixtures";
+import { queueCases, type QueueCase } from "./lib/cut-eval/queueCases";
+
+const args = process.argv.slice(2);
+const argValue = (flag: string) => {
+ const i = args.indexOf(flag);
+ return i >= 0 ? args[i + 1] : undefined;
+};
+const BASE = argValue("--base") ?? "http://localhost:3000";
+const ONLY = argValue("--only")?.split(",").map((s) => s.trim());
+const RUNS = Number(argValue("--runs") ?? 1);
+const LIVE = !args.includes("--no-live");
+const GATE = (() => {
+ const v = argValue("--gate-model");
+ if (!v) return geminiModelRoleNames.fastDecision;
+ return (geminiModels as Record<string, string>)[v] ?? v;
+})();
+
+/** The eval's deps: the dev server as the inference proxy. The triage uses
+ * the post function and the gate model alone. */
+const deps: CutAgentDeps = {
+ post: (payload, signal) =>
+ fetch(`${BASE}/api/inference/responses`, {
+ method: "POST",
+ headers: {
+ "Content-Type": "application/json",
+ "x-donkey-client-id": "donkey-cut-eval",
+ "x-donkey-dev-auth-bypass": "1",
+ },
+ body: JSON.stringify(payload),
+ signal,
+ }),
+ execTool: async () => ({}),
+ models: { simple: GATE, complex: GATE, gate: GATE },
+ buildContext: () => ({}),
+ resolveRefs: async () => [],
+};
+
+interface RunResult {
+ ms: number;
+ wrong: { text: string; expected: QueueVerdict; got: QueueVerdict }[];
+}
+
+async function runCase(c: QueueCase): Promise<RunResult> {
+ const t0 = performance.now();
+ const rows = c.rows.map((r, i) => ({ id: `r${i}`, text: r.text }));
+ const verdicts = await triageQueuedMessages({
+ anchor: c.anchor,
+ waiting: (c.waiting ?? []).map((text, i) => ({ id: `w${i}`, text })),
+ rows,
+ deps,
+ });
+ const wrong: RunResult["wrong"] = [];
+ c.rows.forEach((r, i) => {
+ const got = verdicts.get(`r${i}`) ?? "queue";
+ if (got !== r.expect) wrong.push({ text: r.text, expected: r.expect, got });
+ });
+ return { ms: performance.now() - t0, wrong };
+}
+
+/** A real turn with a fold landing mid-way: "add a title" goes out, and the
+ * moment its first tool call is served, "make it blue" folds in — the way the
+ * page steers a triaged row into the live turn. The fold must resolve as
+ * taken, sit in the thread's context as a user message, and show in the
+ * turn's work: a second overlay edit or a reply that covers the color. */
+async function runLiveFold(): Promise<string[]> {
+ const notes: string[] = [];
+ const threadId = `eval-queue-fold-${Date.now()}`;
+ const trace: { name: string; args: Record<string, unknown> }[] = [];
+ let folded: Promise<boolean> | null = null;
+ const live: CutAgentDeps = {
+ ...deps,
+ models: {
+ simple: geminiModelRoleNames.chatSimple,
+ complex: geminiModelRoleNames.chat,
+ gate: geminiModelRoleNames.fastDecision,
+ },
+ buildContext: () => EDITOR_STATE,
+ execTool: async (name, args) => {
+ trace.push({ name, args });
+ if (name === "add_title") {
+ folded ??= foldIntoCutChat({ threadId, text: "make it blue", attachments: [], deps: live });
+ return { ok: true, id: "t-new", start: 0, len: 3 };
+ }
+ if (name === "update_overlay" || name === "update_title") return { ok: true };
+ if (SAFE_TOOLS.has(name)) return serveSafeTool(name, EDITOR_STATE);
+ return { error: "eval: this tool is disabled for this turn — do not retry it" };
+ },
+ };
+ const messages = [userTurn("Add a title that says Welcome at the start")] as unknown as UIMessage[];
+ let reply = "";
+ const stream = streamCutChat({ threadId, model: geminiModelRoleNames.chat, messages, deps: live });
+ const reader = stream.getReader();
+ for (;;) {
+ const { done, value } = await reader.read();
+ if (done) break;
+ const chunk = value as unknown as { type: string; delta?: string; errorText?: string };
+ if (chunk.type === "text-delta" && chunk.delta) reply += chunk.delta;
+ if (chunk.type === "error" && chunk.errorText) notes.push(`turn error: ${chunk.errorText}`);
+ }
+ if (!folded) notes.push("add_title never ran, so nothing folded");
+ else if (!(await folded)) notes.push("the fold was handed back: the turn ended before taking it");
+ const session = readPiSession(threadId) ?? [];
+ const inContext = session.some(
+ (m) =>
+ (m as Message).role === "user" &&
+ (m as { fold?: true }).fold === true &&
+ JSON.stringify((m as Message).content).includes("make it blue")
+ );
+ if (!inContext) notes.push("the fold is missing from the thread's stored context");
+ const styled = trace.some(
+ (t, i) => i > 0 && (t.name === "update_overlay" || t.name === "update_title" || t.name === "add_title") && /blue|#[0-9a-f]{3,8}/i.test(JSON.stringify(t.args))
+ );
+ if (!styled && !/blue/i.test(reply)) notes.push("the turn never acted on the fold (no blue edit, no mention)");
+ dropPiSession(threadId);
+ console.log(
+ ` ${notes.length === 0 ? "ok " : "FAIL"} live-fold-lands-in-running-turn tools: ${trace.map((t) => t.name).join(" → ")}`
+ );
+ for (const n of notes) console.log(` - ${n}`);
+ if (notes.length > 0) console.log(` reply: ${reply.trim().slice(0, 200)}`);
+ return notes;
+}
+
+/** A fold the turn cannot do yet by construction: a scene plan goes out and
+ * stops at the storyboard for the user's approval, and "once it's rendered,
+ * put soft music under it" folds in on the plan call. The shots cannot
+ * render this turn, so the turn must hand the music ask to the queue in the
+ * user's words and make no music now. */
+async function runLiveQueueBack(): Promise<string[]> {
+ const notes: string[] = [];
+ const threadId = `eval-queue-back-${Date.now()}`;
+ const trace: { name: string; args: Record<string, unknown> }[] = [];
+ let folded: Promise<boolean> | null = null;
+ const live: CutAgentDeps = {
+ ...deps,
+ models: {
+ simple: geminiModelRoleNames.chatSimple,
+ complex: geminiModelRoleNames.chat,
+ gate: geminiModelRoleNames.fastDecision,
+ },
+ buildContext: () => AUDIO_STATE,
+ execTool: async (name, args) => {
+ trace.push({ name, args });
+ if (name === "generate_scene") {
+ folded ??= foldIntoCutChat({
+ threadId,
+ text: "once it's rendered, put some soft music under it",
+ attachments: [],
+ deps: live,
+ });
+ return SCENE_PLANNED;
+ }
+ if (name === "queue_message") return { queued: true, note: "It runs as its own turn after this one finishes." };
+ if (SAFE_TOOLS.has(name)) return serveSafeTool(name, AUDIO_STATE);
+ return { error: "eval: this tool is disabled for this turn — do not retry it" };
+ },
+ };
+ const messages = [
+ userTurn("turn my narration audio into a smooth 2D cartoon", { state: AUDIO_STATE }),
+ ] as unknown as UIMessage[];
+ let reply = "";
+ const stream = streamCutChat({ threadId, model: geminiModelRoleNames.chat, messages, deps: live });
+ const reader = stream.getReader();
+ for (;;) {
+ const { done, value } = await reader.read();
+ if (done) break;
+ const chunk = value as unknown as { type: string; delta?: string; errorText?: string };
+ if (chunk.type === "text-delta" && chunk.delta) reply += chunk.delta;
+ if (chunk.type === "error" && chunk.errorText) notes.push(`turn error: ${chunk.errorText}`);
+ }
+ if (!folded) notes.push("add_title never ran, so nothing folded");
+ else if (!(await folded)) notes.push("the fold was handed back: the turn ended before taking it");
+ if (!trace.some((t) => t.name === "generate_scene")) notes.push("generate_scene never ran, so nothing folded");
+ const queued = trace.find((t) => t.name === "queue_message");
+ if (!queued) notes.push("the turn never called queue_message for the music");
+ else if (!/music/i.test(String(queued.args.text ?? ""))) notes.push(`queue_message text lost the ask: ${JSON.stringify(queued.args)}`);
+ if (trace.some((t) => /music|approve_scene|add_clip|stock/.test(t.name))) notes.push("the turn acted on the music, or approved the plan, in the same turn");
+ dropPiSession(threadId);
+ console.log(
+ ` ${notes.length === 0 ? "ok " : "FAIL"} live-fold-queues-what-needs-the-result tools: ${trace.map((t) => t.name).join(" → ")}`
+ );
+ for (const n of notes) console.log(` - ${n}`);
+ if (notes.length > 0) console.log(` reply: ${reply.trim().slice(0, 200)}`);
+ return notes;
+}
+
+async function main() {
+ const selected = queueCases.filter((c) => !ONLY || ONLY.includes(c.name));
+ if (selected.length === 0 && !LIVE) throw new Error(`No case named "${ONLY?.join(", ")}".`);
+ console.log(`== queue triage gate=${resolveGeminiModel(GATE)} runs=${RUNS}`);
+ let failed = 0;
+ const times: number[] = [];
+ for (const c of selected) {
+ const runs: RunResult[] = [];
+ for (let i = 0; i < RUNS; i++) runs.push(await runCase(c));
+ const passes = runs.filter((r) => r.wrong.length === 0).length;
+ const p50 = [...runs.map((r) => r.ms)].sort((a, b) => a - b)[Math.floor(runs.length / 2)];
+ times.push(p50);
+ const mark = passes === runs.length ? "ok " : passes > 0 ? "part" : "FAIL";
+ if (passes === 0) failed++;
+ console.log(` ${mark} ${c.name} pass ${passes}/${runs.length} p50 ${Math.round(p50)}ms`);
+ for (const r of runs) {
+ for (const w of r.wrong)
+ console.log(` - "${w.text}" → ${w.got}, expected ${w.expected}`);
+ }
+ }
+ const sorted = [...times].sort((a, b) => a - b);
+ if (sorted.length > 0)
+ console.log(
+ `\n${selected.length - failed}/${selected.length} cases pass; triage p50 ${Math.round(sorted[Math.floor(sorted.length / 2)])}ms, max ${Math.round(sorted.at(-1) ?? 0)}ms`
+ );
+ if (LIVE) {
+ console.log("\n== live fold");
+ if (!ONLY || ONLY.includes("live-fold-lands-in-running-turn")) {
+ if ((await runLiveFold()).length > 0) failed++;
+ }
+ if (!ONLY || ONLY.includes("live-fold-queues-what-needs-the-result")) {
+ if ((await runLiveQueueBack()).length > 0) failed++;
+ }
+ }
+ if (failed > 0) process.exit(1);
+}
+
+main().catch((err) => {
+ console.error(err instanceof Error ? err.message : String(err));
+ process.exit(1);
+});
diff --git a/site/scripts/lib/cut-eval/README.md b/site/scripts/lib/cut-eval/README.md
index bae543b6..fa3041ae 100644
--- a/site/scripts/lib/cut-eval/README.md
+++ b/site/scripts/lib/cut-eval/README.md
@@ -41,6 +41,19 @@ Per-case `budget` values in `cases.ts` are checked against passing-run p50s.
They report by default; `--enforce-budgets` turns a breach into exit 1. The run
always exits 1 when a case has zero passing runs.
+## The queue triage eval
+
+`npm run eval:cut-queue -- [--runs N] [--only <case>] [--gate-model <id>]`
+checks where a message sent mid-turn goes. Each case in `queueCases.ts` is a
+running turn (its ask and the tools it has run so far, plus what other chats
+of the project are doing) and the messages the user sent while it ran, each
+with the place it must get: `fold` into the running turn, `spawn` a parallel
+thread, or `queue` in the tray. The cases run the production triage call
+against the dev server on the gate model and print one line per case; a wrong
+verdict prints the message, what it got and what it should have got. It exits
+1 when a case never passes. Add a case whenever a real message landed in the
+wrong place.
+
## The report
A full run writes `evals/cut-chat.latest-report.json` (schema
diff --git a/site/scripts/lib/cut-eval/queueCases.ts b/site/scripts/lib/cut-eval/queueCases.ts
new file mode 100644
index 00000000..1401c61b
--- /dev/null
+++ b/site/scripts/lib/cut-eval/queueCases.ts
@@ -0,0 +1,121 @@
+/**
+ * The queue triage eval cases. Each case is a running turn — its ask and the
+ * tools it has run so far — plus the messages the user sent while it ran,
+ * each with the place it must be given: fold (the running turn takes it),
+ * spawn (a parallel thread runs it now), queue (it waits for the result).
+ */
+
+import type { QueueVerdict, TriageAnchor } from "../../../src/cut/lib/queueTriage";
+
+export interface QueueCase {
+ name: string;
+ anchor: TriageAnchor;
+ /** Rows already in the tray when the triaged ones arrive. */
+ waiting?: string[];
+ rows: { text: string; expect: QueueVerdict }[];
+}
+
+const running = (ask: string, progress: string[] = [], elsewhere: string[] = []): TriageAnchor => ({
+ ask,
+ progress,
+ elsewhere,
+});
+
+export const queueCases: QueueCase[] = [
+ // Refinements of the running ask fold.
+ {
+ name: "fold-refine-same-job",
+ anchor: running("Cut the silences out of clip 2", ["detect_silence", "split_clip (running)"]),
+ rows: [{ text: "also tighten the gaps to 0.2s", expect: "fold" }],
+ },
+ {
+ name: "fold-style-the-title-being-added",
+ anchor: running("Add a title that says Welcome at the start", ["add_text"]),
+ rows: [{ text: "make it blue and a bit bigger", expect: "fold" }],
+ },
+ {
+ name: "fold-correction",
+ anchor: running("Remove the filler words", ["detect_filler", "split_clip", "delete_clip (running)"]),
+ rows: [{ text: "actually leave the first um in, it sounds natural", expect: "fold" }],
+ },
+ {
+ name: "fold-question-on-question",
+ anchor: running("What's in this video?", ["watch_video (running)"]),
+ rows: [{ text: "and how long is it?", expect: "fold" }],
+ },
+ {
+ name: "fold-extra-piece-same-job",
+ anchor: running("Add captions to the whole video", ["captions_generate (running)"]),
+ rows: [{ text: "and put them near the bottom, not the middle", expect: "fold" }],
+ },
+ {
+ name: "fold-timeline-edit-during-restructure",
+ anchor: running("Cut the pauses and tighten the whole video", ["detect_silence", "split_clip", "delete_clip (running)"]),
+ rows: [{ text: "add a title card at the start too", expect: "fold" }],
+ },
+
+ // Independent, non-colliding work spawns.
+ {
+ name: "spawn-question-during-edit",
+ anchor: running("Remove the filler words", ["detect_filler", "split_clip (running)"]),
+ rows: [{ text: "what resolution is the source footage?", expect: "spawn" }],
+ },
+ {
+ name: "spawn-audio-level-during-captions",
+ anchor: running("Add captions to the whole video", ["captions_generate (running)"]),
+ rows: [{ text: "lower the background music to 20%", expect: "spawn" }],
+ },
+ {
+ name: "spawn-grade-other-clip-during-title",
+ anchor: running("Add a title that says Welcome at the start", ["add_text"]),
+ rows: [{ text: "warm up the color on the interview clip", expect: "spawn" }],
+ },
+ {
+ name: "spawn-footage-question-during-generation",
+ anchor: running("Generate a 5 second intro of a sunrise over the city", ["generate_video (running)"]),
+ rows: [{ text: "does the second clip have any speech in it?", expect: "spawn" }],
+ },
+
+ // Work that needs the finished result waits.
+ {
+ name: "queue-export-after",
+ anchor: running("Cut the silences out of clip 2", ["detect_silence", "split_clip (running)"]),
+ rows: [{ text: "then export it at 1080p", expect: "queue" }],
+ },
+ {
+ name: "queue-share-when-done",
+ anchor: running("Remove the filler words", ["detect_filler (running)"]),
+ rows: [{ text: "when you're done, share the link with me", expect: "queue" }],
+ },
+ {
+ name: "queue-thumbnail-of-final-cut",
+ anchor: running("Cut the pauses and tighten the whole video", ["detect_silence", "split_clip (running)"]),
+ rows: [{ text: "after that, grab the best frame for a thumbnail", expect: "queue" }],
+ },
+
+ // Look-ahead: several rows placed at once against the turn that just started.
+ {
+ name: "lookahead-mixed",
+ anchor: running("Cut the silences out of clip 2", []),
+ rows: [
+ { text: "and remove the filler words while you're at it", expect: "fold" },
+ { text: "what's the aspect ratio of this project?", expect: "spawn" },
+ { text: "then export it", expect: "queue" },
+ ],
+ },
+ {
+ name: "lookahead-with-waiting-context",
+ anchor: running("Add a title that says Welcome at the start", ["add_text (running)"]),
+ waiting: ["then export it"],
+ rows: [{ text: "center it", expect: "fold" }],
+ },
+
+ // Work running in another chat is a collision too.
+ {
+ name: "queue-collides-with-other-chat",
+ anchor: running("Add captions to the whole video", ["captions_generate (running)"], [
+ "Cut the silences and tighten the whole video",
+ ]),
+ rows: [{ text: "add a sticker at 12 seconds", expect: "queue" }],
+ },
+];