Open source CapCut alternative.
TypeScript
refactor(site): eval runs the production router
David Lec35bf26b8c6d
diff --git a/site/scripts/eval-cut-chat.ts b/site/scripts/eval-cut-chat.tsindex 7e628b1a..b1fbb46b 100644--- a/site/scripts/eval-cut-chat.ts+++ b/site/scripts/eval-cut-chat.ts@@ -14,7 +14,8 @@* [--base http://localhost:3000]* [--only <case>] [--bucket chat|single-tool|multi-tool]* [--runs N] default 1; use 3+ for latency stats- * [--model <registryKey|rawId>] chat model (flash, flashLite, or a raw id)+ * [--model <registryKey|rawId>] pin BOTH chat roles to one model (unrouted)+ * [--simple-model <id>] [--complex-model <id>] override one routed role* [--gate-model <registryKey|rawId>]* [--matrix] run every CANDIDATES row, compare* [--enforce-budgets] exit 1 when a case breaches its budget@@ -47,11 +48,26 @@ import {// Model configs the --matrix mode compares. Add a row when trialing a new// model id; keep "baseline" matching geminiModelRoles so the comparison always-// includes what production runs.-const CANDIDATES: { label: string; chat: string; gate: string }[] = [- { label: "baseline", chat: geminiModelRoles.chat, gate: geminiModelRoles.fastDecision },- { label: "chat=flashLite", chat: geminiModels.flashLite, gate: geminiModelRoles.fastDecision },- { label: "gate=flash", chat: geminiModelRoles.chat, gate: geminiModels.flash },+// includes what production runs (the routed pair).+const CANDIDATES: { label: string; simple: string; complex: string; gate: string }[] = [+ {+ label: "baseline",+ simple: geminiModelRoles.chatSimple,+ complex: geminiModelRoles.chat,+ gate: geminiModelRoles.fastDecision,+ },+ {+ label: "all-flash",+ simple: geminiModels.flash,+ complex: geminiModels.flash,+ gate: geminiModelRoles.fastDecision,+ },+ {+ label: "all-flashLite",+ simple: geminiModels.flashLite,+ complex: geminiModels.flashLite,+ gate: geminiModelRoles.fastDecision,+ },];const args = process.argv.slice(2);@@ -95,7 +111,11 @@ async function runConfig(cfg: RunConfig,selected: EvalCase[]): Promise<ConfigReport> {- console.log(`\n== ${label} chat=${cfg.chatModel} gate=${cfg.gateModel}`);+ const chatDesc =+ cfg.simpleModel === cfg.complexModel+ ? cfg.simpleModel+ : `${cfg.simpleModel} | complex→${cfg.complexModel}`;+ console.log(`\n== ${label} chat=${chatDesc} gate=${cfg.gateModel}`);const caseReports: CaseReport[] = [];for (const c of selected) {const runs: RunReport[] = [];@@ -123,7 +143,7 @@ async function runConfig(}return {label,- chatModel: cfg.chatModel,+ chatModel: chatDesc,gateModel: cfg.gateModel,buckets: buildBucketSummaries(caseReports),cases: caseReports,@@ -138,17 +158,26 @@ async function main() {if (selected.length === 0)throw new Error(ONLY ? `No case named "${ONLY}".` : `No cases in bucket "${BUCKET}".`);+ // --model pins BOTH chat roles to one model (an unrouted config);+ // --simple-model / --complex-model override a role each. The default is+ // production's routed pair.+ const both = argValue("--model");const configs = MATRIX? CANDIDATES.map((cand) => ({label: cand.label,- cfg: { base: BASE, chatModel: cand.chat, gateModel: cand.gate },+ cfg: { base: BASE, simpleModel: cand.simple, complexModel: cand.complex, gateModel: cand.gate },})): [{label: "config",cfg: {base: BASE,- chatModel: resolveModel(argValue("--model") ?? geminiModelRoles.chat),+ simpleModel: resolveModel(+ argValue("--simple-model") ?? both ?? geminiModelRoles.chatSimple+ ),+ complexModel: resolveModel(+ argValue("--complex-model") ?? both ?? geminiModelRoles.chat+ ),gateModel: resolveModel(argValue("--gate-model") ?? geminiModelRoles.fastDecision),},},diff --git a/site/scripts/lib/cut-eval/README.md b/site/scripts/lib/cut-eval/README.mdindex a4848b2c..0172dfae 100644--- a/site/scripts/lib/cut-eval/README.md+++ b/site/scripts/lib/cut-eval/README.md@@ -4,8 +4,8 @@ One command measures the chat assistant's behavior and speed together:```npm run eval:cut-chat -- [--runs N] [--only <case>] [--bucket chat|single-tool|multi-tool]- [--model flash|flashLite|<raw id>] [--gate-model ...]- [--matrix] [--enforce-budgets] [--out <path>]+ [--model <id>] [--simple-model <id>] [--complex-model <id>]+ [--gate-model <id>] [--matrix] [--enforce-budgets] [--out <path>]```It replays real composer turns against the live chat model through the dev@@ -27,9 +27,14 @@ fast path), `single-tool` edits (one decisive call), and `multi-tool` editswith input assembly, which is negligible, so this is the same critical paththe user perceives.-`--matrix` runs every row of `CANDIDATES` in `eval-cut-chat.ts` and prints a-comparison table. Model flags and candidate rows resolve through the registry-in `src/lib/inference/gemini-models.ts`; raw model ids pass through.+The default config is production's: the gate's three-way verdict (chat /+simple / complex) withholds tools on chat turns and routes simple turns to the+light chat model. `--model` pins both roles to one model for an unrouted+comparison; each run's report records the model every turn actually ran on+(`roundModel`). `--matrix` runs every row of `CANDIDATES` in+`eval-cut-chat.ts` and prints a comparison table. Model flags and candidate+rows resolve through the registry in `src/lib/inference/gemini-models.ts`; raw+model ids pass through.Per-case `budget` values in `cases.ts` are checked against passing-run p50s.They report by default; `--enforce-budgets` turns a breach into exit 1. The rundiff --git a/site/scripts/lib/cut-eval/cases.ts b/site/scripts/lib/cut-eval/cases.tsindex 3ddbdef8..b2cb6baa 100644--- a/site/scripts/lib/cut-eval/cases.ts+++ b/site/scripts/lib/cut-eval/cases.ts@@ -9,7 +9,6 @@* multi-tool — composed edits over several calls*/-import type { TurnIntent } from "../../../src/cut/lib/turnIntent";import {AUDIO_STATE,CLIP_REFS,@@ -48,9 +47,10 @@ export interface EvalCase {/** Latency guard: the turn fails if the trace exceeds this many calls. A* simple ask must not detour through skill reads or state polls. */maxToolCalls?: number;- /** The tool gate's required verdict for the turn ("chat" turns run with no- * tool declarations, so tool calls become impossible). */- gate?: TurnIntent;+ /** The gate side the turn must classify to: "chat" turns run with no tool+ * declarations (tool calls become impossible), "work" covers both routing+ * verdicts (simple and complex). */+ gate?: "chat" | "work";/** Editor snapshot served to get_state for this case (default EDITOR_STATE). */state?: unknown;/** Per-run tool interceptor (fresh per run); a non-undefined return serves@@ -460,7 +460,12 @@ export function cases(audio: { dataBase64: string; mimeType: string }): EvalCasestatus: "done",track: 0,locale: "en-US",- cues: 3,+ count: 3,+ cues: [+ { id: "ncue1", start: 0, end: 2.1, text: "So today we're at the beach with the dogs" },+ { id: "ncue2", start: 2.1, end: 4.9, text: "they absolutely love the water" },+ { id: "ncue3", start: 4.9, end: 7.5, text: "so let's watch them play" },+ ],note: "Re-transcribed the tightened cut.",},subtitles_remove_track: { removed: 0 },diff --git a/site/scripts/lib/cut-eval/fixtures.ts b/site/scripts/lib/cut-eval/fixtures.tsindex f609caf7..07e9ee6e 100644--- a/site/scripts/lib/cut-eval/fixtures.ts+++ b/site/scripts/lib/cut-eval/fixtures.ts@@ -436,6 +436,12 @@ export function makeTimelineSim(base: typeof FILLER_STATE) {speed: 1,})),});+ // Mirrors production aiTools: every timeline mutation returns the updated+ // rows so the model keeps cutting from ids in the result.+ const rows = () => ({+ track0: clips.map((c) => ({ id: c.id, start: c.start, len: r2(c.out - c.in) })),+ soundtrack: audio.map((a) => ({ id: a.id, start: a.start, len: r2(a.out - a.in) })),+ });return (name: string, args: Record<string, unknown>): unknown => {if (name === "get_state") return snapshot();if (name === "refine_speech_cuts") {@@ -443,7 +449,7 @@ export function makeTimelineSim(base: typeof FILLER_STATE) {const missing = ids.filter((id) => !clips.some((c) => c.id === id));if (ids.length === 0 || missing.length > 0)return { error: `No such clip(s): ${missing.join(", ") || "(none given)"}.` };- return { refined: ids, note: "Edges re-trimmed into the nearest pauses." };+ return { refined: ids, note: "Edges re-trimmed into the nearest pauses.", ...rows() };}if (name === "split_at") {const t = Number(args.t);@@ -462,7 +468,7 @@ export function makeTimelineSim(base: typeof FILLER_STATE) {remember();clips = cut(clips);audio = cut(audio);- return { split: true, videoClips: clips.length };+ return { split: true, ...rows() };}if (name === "delete_item") {const id = String(args.id);@@ -471,13 +477,13 @@ export function makeTimelineSim(base: typeof FILLER_STATE) {remember();clips = clips.filter((c) => c.id !== id);ripple();- return { deleted: { kind: "clip", id } };+ return { deleted: { kind: "clip", id }, ...rows() };}if (args.kind === "audio") {if (!audio.some((a) => a.id === id)) return { error: `No audio with id ${id}.` };remember();audio = audio.filter((a) => a.id !== id);- return { deleted: { kind: "audio", id } };+ return { deleted: { kind: "audio", id }, ...rows() };}return undefined;}@@ -490,7 +496,7 @@ export function makeTimelineSim(base: typeof FILLER_STATE) {remember();clips = clips.map((x) => (x === c ? { ...x, in: nextIn, out: nextOut } : x));ripple();- return { in: nextIn, out: nextOut, len: r2(nextOut - nextIn) };+ return { in: nextIn, out: nextOut, len: r2(nextOut - nextIn), ...rows() };}if (name === "place_clip") {const c = clips.find((x) => x.id === String(args.clipId));@@ -501,7 +507,7 @@ export function makeTimelineSim(base: typeof FILLER_STATE) {.map((x) => (x === c ? { ...x, start: at } : x)).sort((a, b) => a.start - b.start);ripple();- return { start: at };+ return { start: at, ...rows() };}if (name === "update_audio") {const a = audio.find((x) => x.id === String(args.id));@@ -525,7 +531,7 @@ export function makeTimelineSim(base: typeof FILLER_STATE) {clips = prev.clips;audio = prev.audio;}- return { ok: true, videoClips: clips.length };+ return { ok: true, ...rows() };}if (name === "delete_cue") {const cue = cues.find((c) => c.id === String(args.id));diff --git a/site/scripts/lib/cut-eval/harness.ts b/site/scripts/lib/cut-eval/harness.tsindex 06b1048e..b8c109ad 100644--- a/site/scripts/lib/cut-eval/harness.ts+++ b/site/scripts/lib/cut-eval/harness.ts@@ -34,15 +34,19 @@ const TRANSIENT_RETRIES = 2;export interface RunConfig {/** Dev server base URL. */base: string;- /** Model the chat rounds run on. */- chatModel: string;- /** Model the tool gate runs on. */+ /** Model a simple-verdict turn's rounds run on. */+ simpleModel: string;+ /** Model a complex-verdict (or fail-open) turn's rounds run on. */+ complexModel: string;+ /** Model the gate/router call runs on. */gateModel: string;}+/** Production's config: the three-way gate routes between the two chat roles. */export const defaultRunConfig = (base: string): RunConfig => ({base,- chatModel: geminiModelRoles.chat,+ simpleModel: geminiModelRoles.chatSimple,+ complexModel: geminiModelRoles.chat,gateModel: geminiModelRoles.fastDecision,});@@ -90,6 +94,8 @@ export interface CaseResult {violations: string[];notes: string[];intent: TurnIntent;+ /** The model the rounds ran on — differs per turn under a router config. */+ roundModel: string;timings: RunTimings;}@@ -145,19 +151,21 @@ const itemText = (item: Item): string => {.trim();};-/** The production tool gate, replicated from geminiChat.ts: a message with- * attachments is work by construction; otherwise the fast-decision model- * judges the newest message. Fails open to "work". */+/** The production gate and router, replicated from geminiChat.ts: a message+ * with attachments is complex by construction; otherwise the gate model's+ * three-way verdict withholds tools (chat) or picks the loop's model (simple /+ * complex). Fails open to complex. */export async function classifyIntent(input: Item[],cfg: RunConfig,tally: RetryTally-): Promise<{ intent: TurnIntent; gateMs: number; gateSkipped: boolean }> {+): Promise<{ intent: TurnIntent; roundModel: string; gateMs: number; gateSkipped: boolean }> {const lastUser = [...input].reverse().find((i) => i.role === "user");const raw =((lastUser?.content as { text?: string }[]) ?? []).find((p) => typeof p.text === "string")?.text ?? "";- if (raw.includes("<attached_assets>")) return { intent: "work", gateMs: 0, gateSkipped: true };+ if (raw.includes("<attached_assets>"))+ return { intent: "complex", roundModel: cfg.complexModel, gateMs: 0, gateSkipped: true };const turns = input.map((i) => ({role: (i.role === "user" ? "user" : "assistant") as "user" | "assistant",text: itemText(i),@@ -172,8 +180,10 @@ export async function classifyIntent(});if (res.ok) {const body = (await res.json()) as { output_text?: string };+ const intent = parseTurnIntent(body.output_text);return {- intent: parseTurnIntent(body.output_text),+ intent,+ roundModel: intent === "complex" ? cfg.complexModel : cfg.simpleModel,gateMs: performance.now() - t0,gateSkipped: false,};@@ -181,7 +191,7 @@ export async function classifyIntent(tally.retries++;tally.retryMs += performance.now() - t0;if (!RETRYABLE_STATUS.has(res.status) || attempt >= TRANSIENT_RETRIES)- return { intent: "work", gateMs: 0, gateSkipped: false };+ return { intent: "complex", roundModel: cfg.complexModel, gateMs: 0, gateSkipped: false };await sleepTallied(800 * (attempt + 1), tally);}}@@ -291,7 +301,7 @@ export async function runCase(c: EvalCase, cfg: RunConfig): Promise<CaseResult>// with input assembly, which is negligible — serial gate + round 1 is the// same critical path the user perceives.)const gate = await classifyIntent(input, cfg, tally);- const tools = gate.intent === "work" ? toolDeclarations : undefined;+ const tools = gate.intent === "chat" ? undefined : toolDeclarations;const sim = c.simulate?.();const trace: TraceEntry[] = [];const rounds: RoundTiming[] = [];@@ -308,7 +318,7 @@ export async function runCase(c: EvalCase, cfg: RunConfig): Promise<CaseResult>cfg,{donkeyProvider: "gemini",- model: cfg.chatModel,+ model: gate.roundModel,instructions: systemPrompt(),input,...(tools ? { tools } : {}),@@ -392,7 +402,10 @@ export async function runCase(c: EvalCase, cfg: RunConfig): Promise<CaseResult>input.push({ role: "user", content: responseParts });}- if (c.gate && gate.intent !== c.gate) notes.push(`gate said ${gate.intent}, expected ${c.gate}`);+ // Cases assert the gate side of the verdict; simple-vs-complex is routing,+ // scored by latency, so either satisfies a "work" expectation.+ const gateSide = gate.intent === "chat" ? "chat" : "work";+ if (c.gate && gateSide !== c.gate) notes.push(`gate said ${gate.intent}, expected ${c.gate}`);if (!c.reply.test(reply)) notes.push(`reply did not match ${c.reply}`);for (const t of c.requiredTools ?? []) {if (!trace.some((e) => e.name === t)) notes.push(`required tool ${t} was never called`);@@ -421,6 +434,7 @@ export async function runCase(c: EvalCase, cfg: RunConfig): Promise<CaseResult>violations,notes,intent: gate.intent,+ roundModel: gate.roundModel,timings,};}diff --git a/site/scripts/lib/cut-eval/report.ts b/site/scripts/lib/cut-eval/report.tsindex c933e9b8..6e4d9349 100644--- a/site/scripts/lib/cut-eval/report.ts+++ b/site/scripts/lib/cut-eval/report.ts@@ -19,6 +19,8 @@ export interface Agg {export interface RunReport {pass: boolean;intent: string;+ /** The model the rounds ran on — differs per turn under a router config. */+ roundModel?: string;notes: string[];trace: TraceEntry[];timings: RunTimings | null;@@ -94,6 +96,7 @@ export function toRunReport(result: CaseResult | null, error?: string): RunReporreturn {pass: result.pass,intent: result.intent,+ roundModel: result.roundModel,notes: result.notes,trace: result.trace,timings: result.timings,