Open source CapCut alternative.
TypeScript
test(site): queue triage eval
David Le6dd7e879320b
diff --git a/site/package.json b/site/package.jsonindex f7884f83..8a8ac3da 100644--- a/site/package.json+++ b/site/package.json@@ -22,6 +22,7 @@"worker:build": "bun build --target=node --minify --external=pg-native --external=skia-canvas --external=mediabunny --external=@mediabunny/server --external=node-web-audio-api --loader .css:text src/cut/worker/main.ts --outfile dist/cut-worker/main.js","bench:watch": "bun run scripts/bench-watch.ts","eval:cut-chat": "bun run scripts/eval-cut-chat.ts",+ "eval:cut-queue": "bun run scripts/eval-cut-queue.ts","eval:chatgpt-workflow": "bun run scripts/eval-chatgpt-workflow.ts","eval:cut-headless": "bun run scripts/eval-cut-headless.ts","email:dev": "email dev --dir src/emails --port 3001",diff --git a/site/scripts/eval-cut-queue.ts b/site/scripts/eval-cut-queue.tsnew file mode 100644index 00000000..e837ebd4--- /dev/null+++ b/site/scripts/eval-cut-queue.ts@@ -0,0 +1,266 @@+#!/usr/bin/env bun+/**+ * The Cut queue triage eval: where a message sent mid-turn goes.+ *+ * Each case is a running turn plus the messages the user sent while it ran,+ * placed by the production triage call — fold into the running turn, spawn+ * a parallel thread, or wait in the tray — and checked against the place+ * each message must get. A wrong fold does the work in the wrong turn, a+ * wrong spawn runs two agents over the same clips, and a wrong queue makes+ * the user wait; every case pins one of those lines.+ *+ * The live fold case then runs a real turn through the production loop and+ * folds a message into it mid-turn, the way the page does: the fold has to+ * land in the turn's context and the turn has to act on it.+ *+ * Run with the site dev server up:+ * bun run scripts/eval-cut-queue.ts+ * [--base http://localhost:3000]+ * [--only <case>[,<case>…]]+ * [--runs N] default 1; 3+ shows a flaky verdict+ * [--gate-model <registryKey|rawId>]+ * [--no-live] skip the live fold turn+ *+ * Auth is the dev bypass header (scripts only — never the app), so runs are+ * dev-server-only and spend no credits.+ */++import type { UIMessage } from "ai";+import type { Message } from "@earendil-works/pi-ai";+import { geminiModelRoleNames, geminiModels, resolveGeminiModel } from "../src/lib/inference/gemini-models";+import {+ dropPiSession,+ foldIntoCutChat,+ readPiSession,+ streamCutChat,+ triageQueuedMessages,+ type CutAgentDeps,+} from "../src/cut/lib/pi/cutAgent";+import type { QueueVerdict } from "../src/cut/lib/queueTriage";+import { AUDIO_STATE, EDITOR_STATE, SAFE_TOOLS, SCENE_PLANNED, serveSafeTool, userTurn } from "./lib/cut-eval/fixtures";+import { queueCases, type QueueCase } from "./lib/cut-eval/queueCases";++const args = process.argv.slice(2);+const argValue = (flag: string) => {+ const i = args.indexOf(flag);+ return i >= 0 ? args[i + 1] : undefined;+};+const BASE = argValue("--base") ?? "http://localhost:3000";+const ONLY = argValue("--only")?.split(",").map((s) => s.trim());+const RUNS = Number(argValue("--runs") ?? 1);+const LIVE = !args.includes("--no-live");+const GATE = (() => {+ const v = argValue("--gate-model");+ if (!v) return geminiModelRoleNames.fastDecision;+ return (geminiModels as Record<string, string>)[v] ?? v;+})();++/** The eval's deps: the dev server as the inference proxy. The triage uses+ * the post function and the gate model alone. */+const deps: CutAgentDeps = {+ post: (payload, signal) =>+ fetch(`${BASE}/api/inference/responses`, {+ method: "POST",+ headers: {+ "Content-Type": "application/json",+ "x-donkey-client-id": "donkey-cut-eval",+ "x-donkey-dev-auth-bypass": "1",+ },+ body: JSON.stringify(payload),+ signal,+ }),+ execTool: async () => ({}),+ models: { simple: GATE, complex: GATE, gate: GATE },+ buildContext: () => ({}),+ resolveRefs: async () => [],+};++interface RunResult {+ ms: number;+ wrong: { text: string; expected: QueueVerdict; got: QueueVerdict }[];+}++async function runCase(c: QueueCase): Promise<RunResult> {+ const t0 = performance.now();+ const rows = c.rows.map((r, i) => ({ id: `r${i}`, text: r.text }));+ const verdicts = await triageQueuedMessages({+ anchor: c.anchor,+ waiting: (c.waiting ?? []).map((text, i) => ({ id: `w${i}`, text })),+ rows,+ deps,+ });+ const wrong: RunResult["wrong"] = [];+ c.rows.forEach((r, i) => {+ const got = verdicts.get(`r${i}`) ?? "queue";+ if (got !== r.expect) wrong.push({ text: r.text, expected: r.expect, got });+ });+ return { ms: performance.now() - t0, wrong };+}++/** A real turn with a fold landing mid-way: "add a title" goes out, and the+ * moment its first tool call is served, "make it blue" folds in — the way the+ * page steers a triaged row into the live turn. The fold must resolve as+ * taken, sit in the thread's context as a user message, and show in the+ * turn's work: a second overlay edit or a reply that covers the color. */+async function runLiveFold(): Promise<string[]> {+ const notes: string[] = [];+ const threadId = `eval-queue-fold-${Date.now()}`;+ const trace: { name: string; args: Record<string, unknown> }[] = [];+ let folded: Promise<boolean> | null = null;+ const live: CutAgentDeps = {+ ...deps,+ models: {+ simple: geminiModelRoleNames.chatSimple,+ complex: geminiModelRoleNames.chat,+ gate: geminiModelRoleNames.fastDecision,+ },+ buildContext: () => EDITOR_STATE,+ execTool: async (name, args) => {+ trace.push({ name, args });+ if (name === "add_title") {+ folded ??= foldIntoCutChat({ threadId, text: "make it blue", attachments: [], deps: live });+ return { ok: true, id: "t-new", start: 0, len: 3 };+ }+ if (name === "update_overlay" || name === "update_title") return { ok: true };+ if (SAFE_TOOLS.has(name)) return serveSafeTool(name, EDITOR_STATE);+ return { error: "eval: this tool is disabled for this turn — do not retry it" };+ },+ };+ const messages = [userTurn("Add a title that says Welcome at the start")] as unknown as UIMessage[];+ let reply = "";+ const stream = streamCutChat({ threadId, model: geminiModelRoleNames.chat, messages, deps: live });+ const reader = stream.getReader();+ for (;;) {+ const { done, value } = await reader.read();+ if (done) break;+ const chunk = value as unknown as { type: string; delta?: string; errorText?: string };+ if (chunk.type === "text-delta" && chunk.delta) reply += chunk.delta;+ if (chunk.type === "error" && chunk.errorText) notes.push(`turn error: ${chunk.errorText}`);+ }+ if (!folded) notes.push("add_title never ran, so nothing folded");+ else if (!(await folded)) notes.push("the fold was handed back: the turn ended before taking it");+ const session = readPiSession(threadId) ?? [];+ const inContext = session.some(+ (m) =>+ (m as Message).role === "user" &&+ (m as { fold?: true }).fold === true &&+ JSON.stringify((m as Message).content).includes("make it blue")+ );+ if (!inContext) notes.push("the fold is missing from the thread's stored context");+ const styled = trace.some(+ (t, i) => i > 0 && (t.name === "update_overlay" || t.name === "update_title" || t.name === "add_title") && /blue|#[0-9a-f]{3,8}/i.test(JSON.stringify(t.args))+ );+ if (!styled && !/blue/i.test(reply)) notes.push("the turn never acted on the fold (no blue edit, no mention)");+ dropPiSession(threadId);+ console.log(+ ` ${notes.length === 0 ? "ok " : "FAIL"} live-fold-lands-in-running-turn tools: ${trace.map((t) => t.name).join(" → ")}`+ );+ for (const n of notes) console.log(` - ${n}`);+ if (notes.length > 0) console.log(` reply: ${reply.trim().slice(0, 200)}`);+ return notes;+}++/** A fold the turn cannot do yet by construction: a scene plan goes out and+ * stops at the storyboard for the user's approval, and "once it's rendered,+ * put soft music under it" folds in on the plan call. The shots cannot+ * render this turn, so the turn must hand the music ask to the queue in the+ * user's words and make no music now. */+async function runLiveQueueBack(): Promise<string[]> {+ const notes: string[] = [];+ const threadId = `eval-queue-back-${Date.now()}`;+ const trace: { name: string; args: Record<string, unknown> }[] = [];+ let folded: Promise<boolean> | null = null;+ const live: CutAgentDeps = {+ ...deps,+ models: {+ simple: geminiModelRoleNames.chatSimple,+ complex: geminiModelRoleNames.chat,+ gate: geminiModelRoleNames.fastDecision,+ },+ buildContext: () => AUDIO_STATE,+ execTool: async (name, args) => {+ trace.push({ name, args });+ if (name === "generate_scene") {+ folded ??= foldIntoCutChat({+ threadId,+ text: "once it's rendered, put some soft music under it",+ attachments: [],+ deps: live,+ });+ return SCENE_PLANNED;+ }+ if (name === "queue_message") return { queued: true, note: "It runs as its own turn after this one finishes." };+ if (SAFE_TOOLS.has(name)) return serveSafeTool(name, AUDIO_STATE);+ return { error: "eval: this tool is disabled for this turn — do not retry it" };+ },+ };+ const messages = [+ userTurn("turn my narration audio into a smooth 2D cartoon", { state: AUDIO_STATE }),+ ] as unknown as UIMessage[];+ let reply = "";+ const stream = streamCutChat({ threadId, model: geminiModelRoleNames.chat, messages, deps: live });+ const reader = stream.getReader();+ for (;;) {+ const { done, value } = await reader.read();+ if (done) break;+ const chunk = value as unknown as { type: string; delta?: string; errorText?: string };+ if (chunk.type === "text-delta" && chunk.delta) reply += chunk.delta;+ if (chunk.type === "error" && chunk.errorText) notes.push(`turn error: ${chunk.errorText}`);+ }+ if (!folded) notes.push("add_title never ran, so nothing folded");+ else if (!(await folded)) notes.push("the fold was handed back: the turn ended before taking it");+ if (!trace.some((t) => t.name === "generate_scene")) notes.push("generate_scene never ran, so nothing folded");+ const queued = trace.find((t) => t.name === "queue_message");+ if (!queued) notes.push("the turn never called queue_message for the music");+ else if (!/music/i.test(String(queued.args.text ?? ""))) notes.push(`queue_message text lost the ask: ${JSON.stringify(queued.args)}`);+ if (trace.some((t) => /music|approve_scene|add_clip|stock/.test(t.name))) notes.push("the turn acted on the music, or approved the plan, in the same turn");+ dropPiSession(threadId);+ console.log(+ ` ${notes.length === 0 ? "ok " : "FAIL"} live-fold-queues-what-needs-the-result tools: ${trace.map((t) => t.name).join(" → ")}`+ );+ for (const n of notes) console.log(` - ${n}`);+ if (notes.length > 0) console.log(` reply: ${reply.trim().slice(0, 200)}`);+ return notes;+}++async function main() {+ const selected = queueCases.filter((c) => !ONLY || ONLY.includes(c.name));+ if (selected.length === 0 && !LIVE) throw new Error(`No case named "${ONLY?.join(", ")}".`);+ console.log(`== queue triage gate=${resolveGeminiModel(GATE)} runs=${RUNS}`);+ let failed = 0;+ const times: number[] = [];+ for (const c of selected) {+ const runs: RunResult[] = [];+ for (let i = 0; i < RUNS; i++) runs.push(await runCase(c));+ const passes = runs.filter((r) => r.wrong.length === 0).length;+ const p50 = [...runs.map((r) => r.ms)].sort((a, b) => a - b)[Math.floor(runs.length / 2)];+ times.push(p50);+ const mark = passes === runs.length ? "ok " : passes > 0 ? "part" : "FAIL";+ if (passes === 0) failed++;+ console.log(` ${mark} ${c.name} pass ${passes}/${runs.length} p50 ${Math.round(p50)}ms`);+ for (const r of runs) {+ for (const w of r.wrong)+ console.log(` - "${w.text}" → ${w.got}, expected ${w.expected}`);+ }+ }+ const sorted = [...times].sort((a, b) => a - b);+ if (sorted.length > 0)+ console.log(+ `\n${selected.length - failed}/${selected.length} cases pass; triage p50 ${Math.round(sorted[Math.floor(sorted.length / 2)])}ms, max ${Math.round(sorted.at(-1) ?? 0)}ms`+ );+ if (LIVE) {+ console.log("\n== live fold");+ if (!ONLY || ONLY.includes("live-fold-lands-in-running-turn")) {+ if ((await runLiveFold()).length > 0) failed++;+ }+ if (!ONLY || ONLY.includes("live-fold-queues-what-needs-the-result")) {+ if ((await runLiveQueueBack()).length > 0) failed++;+ }+ }+ if (failed > 0) process.exit(1);+}++main().catch((err) => {+ console.error(err instanceof Error ? err.message : String(err));+ process.exit(1);+});diff --git a/site/scripts/lib/cut-eval/README.md b/site/scripts/lib/cut-eval/README.mdindex bae543b6..fa3041ae 100644--- a/site/scripts/lib/cut-eval/README.md+++ b/site/scripts/lib/cut-eval/README.md@@ -41,6 +41,19 @@ Per-case `budget` values in `cases.ts` are checked against passing-run p50s.They report by default; `--enforce-budgets` turns a breach into exit 1. The runalways exits 1 when a case has zero passing runs.+## The queue triage eval++`npm run eval:cut-queue -- [--runs N] [--only <case>] [--gate-model <id>]`+checks where a message sent mid-turn goes. Each case in `queueCases.ts` is a+running turn (its ask and the tools it has run so far, plus what other chats+of the project are doing) and the messages the user sent while it ran, each+with the place it must get: `fold` into the running turn, `spawn` a parallel+thread, or `queue` in the tray. The cases run the production triage call+against the dev server on the gate model and print one line per case; a wrong+verdict prints the message, what it got and what it should have got. It exits+1 when a case never passes. Add a case whenever a real message landed in the+wrong place.+## The reportA full run writes `evals/cut-chat.latest-report.json` (schemadiff --git a/site/scripts/lib/cut-eval/queueCases.ts b/site/scripts/lib/cut-eval/queueCases.tsnew file mode 100644index 00000000..1401c61b--- /dev/null+++ b/site/scripts/lib/cut-eval/queueCases.ts@@ -0,0 +1,121 @@+/**+ * The queue triage eval cases. Each case is a running turn — its ask and the+ * tools it has run so far — plus the messages the user sent while it ran,+ * each with the place it must be given: fold (the running turn takes it),+ * spawn (a parallel thread runs it now), queue (it waits for the result).+ */++import type { QueueVerdict, TriageAnchor } from "../../../src/cut/lib/queueTriage";++export interface QueueCase {+ name: string;+ anchor: TriageAnchor;+ /** Rows already in the tray when the triaged ones arrive. */+ waiting?: string[];+ rows: { text: string; expect: QueueVerdict }[];+}++const running = (ask: string, progress: string[] = [], elsewhere: string[] = []): TriageAnchor => ({+ ask,+ progress,+ elsewhere,+});++export const queueCases: QueueCase[] = [+ // Refinements of the running ask fold.+ {+ name: "fold-refine-same-job",+ anchor: running("Cut the silences out of clip 2", ["detect_silence", "split_clip (running)"]),+ rows: [{ text: "also tighten the gaps to 0.2s", expect: "fold" }],+ },+ {+ name: "fold-style-the-title-being-added",+ anchor: running("Add a title that says Welcome at the start", ["add_text"]),+ rows: [{ text: "make it blue and a bit bigger", expect: "fold" }],+ },+ {+ name: "fold-correction",+ anchor: running("Remove the filler words", ["detect_filler", "split_clip", "delete_clip (running)"]),+ rows: [{ text: "actually leave the first um in, it sounds natural", expect: "fold" }],+ },+ {+ name: "fold-question-on-question",+ anchor: running("What's in this video?", ["watch_video (running)"]),+ rows: [{ text: "and how long is it?", expect: "fold" }],+ },+ {+ name: "fold-extra-piece-same-job",+ anchor: running("Add captions to the whole video", ["captions_generate (running)"]),+ rows: [{ text: "and put them near the bottom, not the middle", expect: "fold" }],+ },+ {+ name: "fold-timeline-edit-during-restructure",+ anchor: running("Cut the pauses and tighten the whole video", ["detect_silence", "split_clip", "delete_clip (running)"]),+ rows: [{ text: "add a title card at the start too", expect: "fold" }],+ },++ // Independent, non-colliding work spawns.+ {+ name: "spawn-question-during-edit",+ anchor: running("Remove the filler words", ["detect_filler", "split_clip (running)"]),+ rows: [{ text: "what resolution is the source footage?", expect: "spawn" }],+ },+ {+ name: "spawn-audio-level-during-captions",+ anchor: running("Add captions to the whole video", ["captions_generate (running)"]),+ rows: [{ text: "lower the background music to 20%", expect: "spawn" }],+ },+ {+ name: "spawn-grade-other-clip-during-title",+ anchor: running("Add a title that says Welcome at the start", ["add_text"]),+ rows: [{ text: "warm up the color on the interview clip", expect: "spawn" }],+ },+ {+ name: "spawn-footage-question-during-generation",+ anchor: running("Generate a 5 second intro of a sunrise over the city", ["generate_video (running)"]),+ rows: [{ text: "does the second clip have any speech in it?", expect: "spawn" }],+ },++ // Work that needs the finished result waits.+ {+ name: "queue-export-after",+ anchor: running("Cut the silences out of clip 2", ["detect_silence", "split_clip (running)"]),+ rows: [{ text: "then export it at 1080p", expect: "queue" }],+ },+ {+ name: "queue-share-when-done",+ anchor: running("Remove the filler words", ["detect_filler (running)"]),+ rows: [{ text: "when you're done, share the link with me", expect: "queue" }],+ },+ {+ name: "queue-thumbnail-of-final-cut",+ anchor: running("Cut the pauses and tighten the whole video", ["detect_silence", "split_clip (running)"]),+ rows: [{ text: "after that, grab the best frame for a thumbnail", expect: "queue" }],+ },++ // Look-ahead: several rows placed at once against the turn that just started.+ {+ name: "lookahead-mixed",+ anchor: running("Cut the silences out of clip 2", []),+ rows: [+ { text: "and remove the filler words while you're at it", expect: "fold" },+ { text: "what's the aspect ratio of this project?", expect: "spawn" },+ { text: "then export it", expect: "queue" },+ ],+ },+ {+ name: "lookahead-with-waiting-context",+ anchor: running("Add a title that says Welcome at the start", ["add_text (running)"]),+ waiting: ["then export it"],+ rows: [{ text: "center it", expect: "fold" }],+ },++ // Work running in another chat is a collision too.+ {+ name: "queue-collides-with-other-chat",+ anchor: running("Add captions to the whole video", ["captions_generate (running)"], [+ "Cut the silences and tighten the whole video",+ ]),+ rows: [{ text: "add a sticker at 12 seconds", expect: "queue" }],+ },+];