Open source CapCut alternative.
TypeScript
#!/usr/bin/env bun
/**
* The Cut queue triage eval: where a message sent mid-turn goes.
*
* Each case is a running turn plus the messages the user sent while it ran,
* placed by the production triage call — fold into the running turn, spawn
* a parallel thread, or wait in the tray — and checked against the place
* each message must get. A wrong fold does the work in the wrong turn, a
* wrong spawn runs two agents over the same clips, and a wrong queue makes
* the user wait; every case pins one of those lines.
*
* The live fold case then runs a real turn through the production loop and
* folds a message into it mid-turn, the way the page does: the fold has to
* land in the turn's context and the turn has to act on it.
*
* Run with the site dev server up:
* bun run scripts/eval-cut-queue.ts
* [--base http://localhost:3000]
* [--only <case>[,<case>…]]
* [--runs N] default 1; 3+ shows a flaky verdict
* [--no-live] skip the live fold turn
*
* Auth is the dev bypass header (scripts only — never the app), so runs are
* dev-server-only and spend no credits.
*/
import type { UIMessage } from "ai";
import type { Message } from "@earendil-works/pi-ai";
import { geminiModelRoleNames } from "../src/lib/inference/gemini-models";
import {
dropPiSession,
foldIntoCutChat,
readPiSession,
streamCutChat,
triageQueuedMessages,
type CutAgentDeps,
} from "../src/cut/lib/pi/cutAgent";
import type { QueueVerdict } from "../src/cut/lib/queueTriage";
import { AUDIO_STATE, EDITOR_STATE, SAFE_TOOLS, SCENE_PLANNED, serveSafeTool, userTurn } from "./lib/cut-eval/fixtures";
import { queueCases, type QueueCase } from "./lib/cut-eval/queueCases";
import { DEFAULT_LOCALE } from "../src/i18n/locales";
const args = process.argv.slice(2);
const argValue = (flag: string) => {
const i = args.indexOf(flag);
return i >= 0 ? args[i + 1] : undefined;
};
const BASE = argValue("--base") ?? "http://localhost:3000";
const ONLY = argValue("--only")?.split(",").map((s) => s.trim());
const RUNS = Number(argValue("--runs") ?? 1);
const LIVE = !args.includes("--no-live");
/** The eval's deps: the dev server as the inference proxy. The triage uses
* the judge post alone. */
const devPost =
(path: string) =>
(payload: Record<string, unknown>, signal?: AbortSignal) =>
fetch(`${BASE}${path}`, {
method: "POST",
headers: {
"Content-Type": "application/json",
"x-donkey-client-id": "donkey-cut-eval",
"x-donkey-dev-auth-bypass": "1",
},
body: JSON.stringify(payload),
signal,
});
const deps: CutAgentDeps = {
post: devPost("/api/inference/responses"),
judge: devPost("/api/inference/judge"),
execTool: async () => ({}),
models: { simple: geminiModelRoleNames.chatSimple, complex: geminiModelRoleNames.chat },
buildContext: () => ({}),
resolveRefs: async () => [],
};
interface RunResult {
ms: number;
wrong: { text: string; expected: QueueVerdict; got: QueueVerdict }[];
}
async function runCase(c: QueueCase): Promise<RunResult> {
const t0 = performance.now();
const rows = c.rows.map((r, i) => ({ id: `r${i}`, text: r.text }));
const verdicts = await triageQueuedMessages({
anchor: c.anchor,
waiting: (c.waiting ?? []).map((text, i) => ({ id: `w${i}`, text })),
rows,
deps,
});
const wrong: RunResult["wrong"] = [];
c.rows.forEach((r, i) => {
const got = verdicts.get(`r${i}`) ?? "queue";
if (got !== r.expect) wrong.push({ text: r.text, expected: r.expect, got });
});
return { ms: performance.now() - t0, wrong };
}
/** A real turn with a fold landing mid-way: "add a title" goes out, and the
* moment its first tool call is served, "make it blue" folds in — the way the
* page steers a triaged row into the live turn. The fold must resolve as
* taken, sit in the thread's context as a user message, and show in the
* turn's work: a second overlay edit or a reply that covers the color. */
async function runLiveFold(): Promise<string[]> {
const notes: string[] = [];
const threadId = `eval-queue-fold-${Date.now()}`;
const trace: { name: string; args: Record<string, unknown> }[] = [];
let folded: Promise<boolean> | null = null;
const live: CutAgentDeps = {
...deps,
models: {
simple: geminiModelRoleNames.chatSimple,
complex: geminiModelRoleNames.chat,
},
buildContext: () => EDITOR_STATE,
execTool: async (name, args) => {
trace.push({ name, args });
if (name === "add_title") {
folded ??= foldIntoCutChat({ threadId, text: "make it blue", attachments: [], deps: live });
return { ok: true, id: "t-new", start: 0, len: 3 };
}
if (name === "update_overlay" || name === "update_title") return { ok: true };
if (SAFE_TOOLS.has(name)) return serveSafeTool(name, EDITOR_STATE);
return { error: "eval: this tool is disabled for this turn — do not retry it" };
},
};
const messages = [userTurn("Add a title that says Welcome at the start")] as unknown as UIMessage[];
let reply = "";
const stream = streamCutChat({ threadId, model: geminiModelRoleNames.chat, messages, locale: DEFAULT_LOCALE, deps: live });
const reader = stream.getReader();
for (;;) {
const { done, value } = await reader.read();
if (done) break;
const chunk = value as unknown as { type: string; delta?: string; errorText?: string };
if (chunk.type === "text-delta" && chunk.delta) reply += chunk.delta;
if (chunk.type === "error" && chunk.errorText) notes.push(`turn error: ${chunk.errorText}`);
}
if (!folded) notes.push("add_title never ran, so nothing folded");
else if (!(await folded)) notes.push("the fold was handed back: the turn ended before taking it");
const session = readPiSession(threadId) ?? [];
const inContext = session.some(
(m) =>
(m as Message).role === "user" &&
(m as { fold?: true }).fold === true &&
JSON.stringify((m as Message).content).includes("make it blue")
);
if (!inContext) notes.push("the fold is missing from the thread's stored context");
const styled = trace.some(
(t, i) => i > 0 && (t.name === "update_overlay" || t.name === "update_title" || t.name === "add_title") && /blue|#[0-9a-f]{3,8}/i.test(JSON.stringify(t.args))
);
if (!styled && !/blue/i.test(reply)) notes.push("the turn never acted on the fold (no blue edit, no mention)");
dropPiSession(threadId);
console.log(
` ${notes.length === 0 ? "ok " : "FAIL"} live-fold-lands-in-running-turn tools: ${trace.map((t) => t.name).join(" → ")}`
);
for (const n of notes) console.log(` - ${n}`);
if (notes.length > 0) console.log(` reply: ${reply.trim().slice(0, 200)}`);
return notes;
}
/** A fold the turn cannot do yet by construction: a scene plan goes out and
* stops at the storyboard for the user's approval, and "once it's rendered,
* put soft music under it" folds in on the plan call. The shots cannot
* render this turn, so the turn must hand the music ask to the queue in the
* user's words and make no music now. */
async function runLiveQueueBack(): Promise<string[]> {
const notes: string[] = [];
const threadId = `eval-queue-back-${Date.now()}`;
const trace: { name: string; args: Record<string, unknown> }[] = [];
let folded: Promise<boolean> | null = null;
const live: CutAgentDeps = {
...deps,
models: {
simple: geminiModelRoleNames.chatSimple,
complex: geminiModelRoleNames.chat,
},
buildContext: () => AUDIO_STATE,
execTool: async (name, args) => {
trace.push({ name, args });
if (name === "generate_scene") {
folded ??= foldIntoCutChat({
threadId,
text: "once it's rendered, put some soft music under it",
attachments: [],
deps: live,
});
return SCENE_PLANNED;
}
if (name === "queue_message") return { queued: true, note: "It runs as its own turn after this one finishes." };
if (SAFE_TOOLS.has(name)) return serveSafeTool(name, AUDIO_STATE);
return { error: "eval: this tool is disabled for this turn — do not retry it" };
},
};
const messages = [
userTurn("turn my narration audio into a smooth 2D cartoon", { state: AUDIO_STATE }),
] as unknown as UIMessage[];
let reply = "";
const stream = streamCutChat({ threadId, model: geminiModelRoleNames.chat, messages, locale: DEFAULT_LOCALE, deps: live });
const reader = stream.getReader();
for (;;) {
const { done, value } = await reader.read();
if (done) break;
const chunk = value as unknown as { type: string; delta?: string; errorText?: string };
if (chunk.type === "text-delta" && chunk.delta) reply += chunk.delta;
if (chunk.type === "error" && chunk.errorText) notes.push(`turn error: ${chunk.errorText}`);
}
if (!folded) notes.push("add_title never ran, so nothing folded");
else if (!(await folded)) notes.push("the fold was handed back: the turn ended before taking it");
if (!trace.some((t) => t.name === "generate_scene")) notes.push("generate_scene never ran, so nothing folded");
const queued = trace.find((t) => t.name === "queue_message");
if (!queued) notes.push("the turn never called queue_message for the music");
else if (!/music/i.test(String(queued.args.text ?? ""))) notes.push(`queue_message text lost the ask: ${JSON.stringify(queued.args)}`);
if (trace.some((t) => /music|approve_scene|add_clip|stock/.test(t.name))) notes.push("the turn acted on the music, or approved the plan, in the same turn");
dropPiSession(threadId);
console.log(
` ${notes.length === 0 ? "ok " : "FAIL"} live-fold-queues-what-needs-the-result tools: ${trace.map((t) => t.name).join(" → ")}`
);
for (const n of notes) console.log(` - ${n}`);
if (notes.length > 0) console.log(` reply: ${reply.trim().slice(0, 200)}`);
return notes;
}
async function main() {
const selected = queueCases.filter((c) => !ONLY || ONLY.includes(c.name));
if (selected.length === 0 && !LIVE) throw new Error(`No case named "${ONLY?.join(", ")}".`);
console.log(`== queue triage judge=typesafe runs=${RUNS}`);
let failed = 0;
const times: number[] = [];
for (const c of selected) {
const runs: RunResult[] = [];
for (let i = 0; i < RUNS; i++) runs.push(await runCase(c));
const passes = runs.filter((r) => r.wrong.length === 0).length;
const p50 = [...runs.map((r) => r.ms)].sort((a, b) => a - b)[Math.floor(runs.length / 2)];
times.push(p50);
const mark = passes === runs.length ? "ok " : passes > 0 ? "part" : "FAIL";
if (passes === 0) failed++;
console.log(` ${mark} ${c.name} pass ${passes}/${runs.length} p50 ${Math.round(p50)}ms`);
for (const r of runs) {
for (const w of r.wrong)
console.log(` - "${w.text}" → ${w.got}, expected ${w.expected}`);
}
}
const sorted = [...times].sort((a, b) => a - b);
if (sorted.length > 0)
console.log(
`\n${selected.length - failed}/${selected.length} cases pass; triage p50 ${Math.round(sorted[Math.floor(sorted.length / 2)])}ms, max ${Math.round(sorted.at(-1) ?? 0)}ms`
);
if (LIVE) {
console.log("\n== live fold");
if (!ONLY || ONLY.includes("live-fold-lands-in-running-turn")) {
if ((await runLiveFold()).length > 0) failed++;
}
if (!ONLY || ONLY.includes("live-fold-queues-what-needs-the-result")) {
if ((await runLiveQueueBack()).length > 0) failed++;
}
}
if (failed > 0) process.exit(1);
}
main().catch((err) => {
console.error(err instanceof Error ? err.message : String(err));
process.exit(1);
});