Skip to content

Open source CapCut alternative.

TypeScript

refactor(site): eval runs the production router

David Lec35bf26b8c6d
diff --git a/site/scripts/eval-cut-chat.ts b/site/scripts/eval-cut-chat.ts
index 7e628b1a..b1fbb46b 100644
--- a/site/scripts/eval-cut-chat.ts
+++ b/site/scripts/eval-cut-chat.ts
@@ -14,7 +14,8 @@
* [--base http://localhost:3000]
* [--only <case>] [--bucket chat|single-tool|multi-tool]
* [--runs N] default 1; use 3+ for latency stats
- * [--model <registryKey|rawId>] chat model (flash, flashLite, or a raw id)
+ * [--model <registryKey|rawId>] pin BOTH chat roles to one model (unrouted)
+ * [--simple-model <id>] [--complex-model <id>] override one routed role
* [--gate-model <registryKey|rawId>]
* [--matrix] run every CANDIDATES row, compare
* [--enforce-budgets] exit 1 when a case breaches its budget
@@ -47,11 +48,26 @@ import {
// Model configs the --matrix mode compares. Add a row when trialing a new
// model id; keep "baseline" matching geminiModelRoles so the comparison always
-// includes what production runs.
-const CANDIDATES: { label: string; chat: string; gate: string }[] = [
- { label: "baseline", chat: geminiModelRoles.chat, gate: geminiModelRoles.fastDecision },
- { label: "chat=flashLite", chat: geminiModels.flashLite, gate: geminiModelRoles.fastDecision },
- { label: "gate=flash", chat: geminiModelRoles.chat, gate: geminiModels.flash },
+// includes what production runs (the routed pair).
+const CANDIDATES: { label: string; simple: string; complex: string; gate: string }[] = [
+ {
+ label: "baseline",
+ simple: geminiModelRoles.chatSimple,
+ complex: geminiModelRoles.chat,
+ gate: geminiModelRoles.fastDecision,
+ },
+ {
+ label: "all-flash",
+ simple: geminiModels.flash,
+ complex: geminiModels.flash,
+ gate: geminiModelRoles.fastDecision,
+ },
+ {
+ label: "all-flashLite",
+ simple: geminiModels.flashLite,
+ complex: geminiModels.flashLite,
+ gate: geminiModelRoles.fastDecision,
+ },
];
const args = process.argv.slice(2);
@@ -95,7 +111,11 @@ async function runConfig(
cfg: RunConfig,
selected: EvalCase[]
): Promise<ConfigReport> {
- console.log(`\n== ${label} chat=${cfg.chatModel} gate=${cfg.gateModel}`);
+ const chatDesc =
+ cfg.simpleModel === cfg.complexModel
+ ? cfg.simpleModel
+ : `${cfg.simpleModel} | complex→${cfg.complexModel}`;
+ console.log(`\n== ${label} chat=${chatDesc} gate=${cfg.gateModel}`);
const caseReports: CaseReport[] = [];
for (const c of selected) {
const runs: RunReport[] = [];
@@ -123,7 +143,7 @@ async function runConfig(
}
return {
label,
- chatModel: cfg.chatModel,
+ chatModel: chatDesc,
gateModel: cfg.gateModel,
buckets: buildBucketSummaries(caseReports),
cases: caseReports,
@@ -138,17 +158,26 @@ async function main() {
if (selected.length === 0)
throw new Error(ONLY ? `No case named "${ONLY}".` : `No cases in bucket "${BUCKET}".`);
+ // --model pins BOTH chat roles to one model (an unrouted config);
+ // --simple-model / --complex-model override a role each. The default is
+ // production's routed pair.
+ const both = argValue("--model");
const configs = MATRIX
? CANDIDATES.map((cand) => ({
label: cand.label,
- cfg: { base: BASE, chatModel: cand.chat, gateModel: cand.gate },
+ cfg: { base: BASE, simpleModel: cand.simple, complexModel: cand.complex, gateModel: cand.gate },
}))
: [
{
label: "config",
cfg: {
base: BASE,
- chatModel: resolveModel(argValue("--model") ?? geminiModelRoles.chat),
+ simpleModel: resolveModel(
+ argValue("--simple-model") ?? both ?? geminiModelRoles.chatSimple
+ ),
+ complexModel: resolveModel(
+ argValue("--complex-model") ?? both ?? geminiModelRoles.chat
+ ),
gateModel: resolveModel(argValue("--gate-model") ?? geminiModelRoles.fastDecision),
},
},
diff --git a/site/scripts/lib/cut-eval/README.md b/site/scripts/lib/cut-eval/README.md
index a4848b2c..0172dfae 100644
--- a/site/scripts/lib/cut-eval/README.md
+++ b/site/scripts/lib/cut-eval/README.md
@@ -4,8 +4,8 @@ One command measures the chat assistant's behavior and speed together:
```
npm run eval:cut-chat -- [--runs N] [--only <case>] [--bucket chat|single-tool|multi-tool]
- [--model flash|flashLite|<raw id>] [--gate-model ...]
- [--matrix] [--enforce-budgets] [--out <path>]
+ [--model <id>] [--simple-model <id>] [--complex-model <id>]
+ [--gate-model <id>] [--matrix] [--enforce-budgets] [--out <path>]
```
It replays real composer turns against the live chat model through the dev
@@ -27,9 +27,14 @@ fast path), `single-tool` edits (one decisive call), and `multi-tool` edits
with input assembly, which is negligible, so this is the same critical path
the user perceives.
-`--matrix` runs every row of `CANDIDATES` in `eval-cut-chat.ts` and prints a
-comparison table. Model flags and candidate rows resolve through the registry
-in `src/lib/inference/gemini-models.ts`; raw model ids pass through.
+The default config is production's: the gate's three-way verdict (chat /
+simple / complex) withholds tools on chat turns and routes simple turns to the
+light chat model. `--model` pins both roles to one model for an unrouted
+comparison; each run's report records the model every turn actually ran on
+(`roundModel`). `--matrix` runs every row of `CANDIDATES` in
+`eval-cut-chat.ts` and prints a comparison table. Model flags and candidate
+rows resolve through the registry in `src/lib/inference/gemini-models.ts`; raw
+model ids pass through.
Per-case `budget` values in `cases.ts` are checked against passing-run p50s.
They report by default; `--enforce-budgets` turns a breach into exit 1. The run
diff --git a/site/scripts/lib/cut-eval/cases.ts b/site/scripts/lib/cut-eval/cases.ts
index 3ddbdef8..b2cb6baa 100644
--- a/site/scripts/lib/cut-eval/cases.ts
+++ b/site/scripts/lib/cut-eval/cases.ts
@@ -9,7 +9,6 @@
* multi-tool — composed edits over several calls
*/
-import type { TurnIntent } from "../../../src/cut/lib/turnIntent";
import {
AUDIO_STATE,
CLIP_REFS,
@@ -48,9 +47,10 @@ export interface EvalCase {
/** Latency guard: the turn fails if the trace exceeds this many calls. A
* simple ask must not detour through skill reads or state polls. */
maxToolCalls?: number;
- /** The tool gate's required verdict for the turn ("chat" turns run with no
- * tool declarations, so tool calls become impossible). */
- gate?: TurnIntent;
+ /** The gate side the turn must classify to: "chat" turns run with no tool
+ * declarations (tool calls become impossible), "work" covers both routing
+ * verdicts (simple and complex). */
+ gate?: "chat" | "work";
/** Editor snapshot served to get_state for this case (default EDITOR_STATE). */
state?: unknown;
/** Per-run tool interceptor (fresh per run); a non-undefined return serves
@@ -460,7 +460,12 @@ export function cases(audio: { dataBase64: string; mimeType: string }): EvalCase
status: "done",
track: 0,
locale: "en-US",
- cues: 3,
+ count: 3,
+ cues: [
+ { id: "ncue1", start: 0, end: 2.1, text: "So today we're at the beach with the dogs" },
+ { id: "ncue2", start: 2.1, end: 4.9, text: "they absolutely love the water" },
+ { id: "ncue3", start: 4.9, end: 7.5, text: "so let's watch them play" },
+ ],
note: "Re-transcribed the tightened cut.",
},
subtitles_remove_track: { removed: 0 },
diff --git a/site/scripts/lib/cut-eval/fixtures.ts b/site/scripts/lib/cut-eval/fixtures.ts
index f609caf7..07e9ee6e 100644
--- a/site/scripts/lib/cut-eval/fixtures.ts
+++ b/site/scripts/lib/cut-eval/fixtures.ts
@@ -436,6 +436,12 @@ export function makeTimelineSim(base: typeof FILLER_STATE) {
speed: 1,
})),
});
+ // Mirrors production aiTools: every timeline mutation returns the updated
+ // rows so the model keeps cutting from ids in the result.
+ const rows = () => ({
+ track0: clips.map((c) => ({ id: c.id, start: c.start, len: r2(c.out - c.in) })),
+ soundtrack: audio.map((a) => ({ id: a.id, start: a.start, len: r2(a.out - a.in) })),
+ });
return (name: string, args: Record<string, unknown>): unknown => {
if (name === "get_state") return snapshot();
if (name === "refine_speech_cuts") {
@@ -443,7 +449,7 @@ export function makeTimelineSim(base: typeof FILLER_STATE) {
const missing = ids.filter((id) => !clips.some((c) => c.id === id));
if (ids.length === 0 || missing.length > 0)
return { error: `No such clip(s): ${missing.join(", ") || "(none given)"}.` };
- return { refined: ids, note: "Edges re-trimmed into the nearest pauses." };
+ return { refined: ids, note: "Edges re-trimmed into the nearest pauses.", ...rows() };
}
if (name === "split_at") {
const t = Number(args.t);
@@ -462,7 +468,7 @@ export function makeTimelineSim(base: typeof FILLER_STATE) {
remember();
clips = cut(clips);
audio = cut(audio);
- return { split: true, videoClips: clips.length };
+ return { split: true, ...rows() };
}
if (name === "delete_item") {
const id = String(args.id);
@@ -471,13 +477,13 @@ export function makeTimelineSim(base: typeof FILLER_STATE) {
remember();
clips = clips.filter((c) => c.id !== id);
ripple();
- return { deleted: { kind: "clip", id } };
+ return { deleted: { kind: "clip", id }, ...rows() };
}
if (args.kind === "audio") {
if (!audio.some((a) => a.id === id)) return { error: `No audio with id ${id}.` };
remember();
audio = audio.filter((a) => a.id !== id);
- return { deleted: { kind: "audio", id } };
+ return { deleted: { kind: "audio", id }, ...rows() };
}
return undefined;
}
@@ -490,7 +496,7 @@ export function makeTimelineSim(base: typeof FILLER_STATE) {
remember();
clips = clips.map((x) => (x === c ? { ...x, in: nextIn, out: nextOut } : x));
ripple();
- return { in: nextIn, out: nextOut, len: r2(nextOut - nextIn) };
+ return { in: nextIn, out: nextOut, len: r2(nextOut - nextIn), ...rows() };
}
if (name === "place_clip") {
const c = clips.find((x) => x.id === String(args.clipId));
@@ -501,7 +507,7 @@ export function makeTimelineSim(base: typeof FILLER_STATE) {
.map((x) => (x === c ? { ...x, start: at } : x))
.sort((a, b) => a.start - b.start);
ripple();
- return { start: at };
+ return { start: at, ...rows() };
}
if (name === "update_audio") {
const a = audio.find((x) => x.id === String(args.id));
@@ -525,7 +531,7 @@ export function makeTimelineSim(base: typeof FILLER_STATE) {
clips = prev.clips;
audio = prev.audio;
}
- return { ok: true, videoClips: clips.length };
+ return { ok: true, ...rows() };
}
if (name === "delete_cue") {
const cue = cues.find((c) => c.id === String(args.id));
diff --git a/site/scripts/lib/cut-eval/harness.ts b/site/scripts/lib/cut-eval/harness.ts
index 06b1048e..b8c109ad 100644
--- a/site/scripts/lib/cut-eval/harness.ts
+++ b/site/scripts/lib/cut-eval/harness.ts
@@ -34,15 +34,19 @@ const TRANSIENT_RETRIES = 2;
export interface RunConfig {
/** Dev server base URL. */
base: string;
- /** Model the chat rounds run on. */
- chatModel: string;
- /** Model the tool gate runs on. */
+ /** Model a simple-verdict turn's rounds run on. */
+ simpleModel: string;
+ /** Model a complex-verdict (or fail-open) turn's rounds run on. */
+ complexModel: string;
+ /** Model the gate/router call runs on. */
gateModel: string;
}
+/** Production's config: the three-way gate routes between the two chat roles. */
export const defaultRunConfig = (base: string): RunConfig => ({
base,
- chatModel: geminiModelRoles.chat,
+ simpleModel: geminiModelRoles.chatSimple,
+ complexModel: geminiModelRoles.chat,
gateModel: geminiModelRoles.fastDecision,
});
@@ -90,6 +94,8 @@ export interface CaseResult {
violations: string[];
notes: string[];
intent: TurnIntent;
+ /** The model the rounds ran on — differs per turn under a router config. */
+ roundModel: string;
timings: RunTimings;
}
@@ -145,19 +151,21 @@ const itemText = (item: Item): string => {
.trim();
};
-/** The production tool gate, replicated from geminiChat.ts: a message with
- * attachments is work by construction; otherwise the fast-decision model
- * judges the newest message. Fails open to "work". */
+/** The production gate and router, replicated from geminiChat.ts: a message
+ * with attachments is complex by construction; otherwise the gate model's
+ * three-way verdict withholds tools (chat) or picks the loop's model (simple /
+ * complex). Fails open to complex. */
export async function classifyIntent(
input: Item[],
cfg: RunConfig,
tally: RetryTally
-): Promise<{ intent: TurnIntent; gateMs: number; gateSkipped: boolean }> {
+): Promise<{ intent: TurnIntent; roundModel: string; gateMs: number; gateSkipped: boolean }> {
const lastUser = [...input].reverse().find((i) => i.role === "user");
const raw =
((lastUser?.content as { text?: string }[]) ?? []).find((p) => typeof p.text === "string")
?.text ?? "";
- if (raw.includes("<attached_assets>")) return { intent: "work", gateMs: 0, gateSkipped: true };
+ if (raw.includes("<attached_assets>"))
+ return { intent: "complex", roundModel: cfg.complexModel, gateMs: 0, gateSkipped: true };
const turns = input.map((i) => ({
role: (i.role === "user" ? "user" : "assistant") as "user" | "assistant",
text: itemText(i),
@@ -172,8 +180,10 @@ export async function classifyIntent(
});
if (res.ok) {
const body = (await res.json()) as { output_text?: string };
+ const intent = parseTurnIntent(body.output_text);
return {
- intent: parseTurnIntent(body.output_text),
+ intent,
+ roundModel: intent === "complex" ? cfg.complexModel : cfg.simpleModel,
gateMs: performance.now() - t0,
gateSkipped: false,
};
@@ -181,7 +191,7 @@ export async function classifyIntent(
tally.retries++;
tally.retryMs += performance.now() - t0;
if (!RETRYABLE_STATUS.has(res.status) || attempt >= TRANSIENT_RETRIES)
- return { intent: "work", gateMs: 0, gateSkipped: false };
+ return { intent: "complex", roundModel: cfg.complexModel, gateMs: 0, gateSkipped: false };
await sleepTallied(800 * (attempt + 1), tally);
}
}
@@ -291,7 +301,7 @@ export async function runCase(c: EvalCase, cfg: RunConfig): Promise<CaseResult>
// with input assembly, which is negligible — serial gate + round 1 is the
// same critical path the user perceives.)
const gate = await classifyIntent(input, cfg, tally);
- const tools = gate.intent === "work" ? toolDeclarations : undefined;
+ const tools = gate.intent === "chat" ? undefined : toolDeclarations;
const sim = c.simulate?.();
const trace: TraceEntry[] = [];
const rounds: RoundTiming[] = [];
@@ -308,7 +318,7 @@ export async function runCase(c: EvalCase, cfg: RunConfig): Promise<CaseResult>
cfg,
{
donkeyProvider: "gemini",
- model: cfg.chatModel,
+ model: gate.roundModel,
instructions: systemPrompt(),
input,
...(tools ? { tools } : {}),
@@ -392,7 +402,10 @@ export async function runCase(c: EvalCase, cfg: RunConfig): Promise<CaseResult>
input.push({ role: "user", content: responseParts });
}
- if (c.gate && gate.intent !== c.gate) notes.push(`gate said ${gate.intent}, expected ${c.gate}`);
+ // Cases assert the gate side of the verdict; simple-vs-complex is routing,
+ // scored by latency, so either satisfies a "work" expectation.
+ const gateSide = gate.intent === "chat" ? "chat" : "work";
+ if (c.gate && gateSide !== c.gate) notes.push(`gate said ${gate.intent}, expected ${c.gate}`);
if (!c.reply.test(reply)) notes.push(`reply did not match ${c.reply}`);
for (const t of c.requiredTools ?? []) {
if (!trace.some((e) => e.name === t)) notes.push(`required tool ${t} was never called`);
@@ -421,6 +434,7 @@ export async function runCase(c: EvalCase, cfg: RunConfig): Promise<CaseResult>
violations,
notes,
intent: gate.intent,
+ roundModel: gate.roundModel,
timings,
};
}
diff --git a/site/scripts/lib/cut-eval/report.ts b/site/scripts/lib/cut-eval/report.ts
index c933e9b8..6e4d9349 100644
--- a/site/scripts/lib/cut-eval/report.ts
+++ b/site/scripts/lib/cut-eval/report.ts
@@ -19,6 +19,8 @@ export interface Agg {
export interface RunReport {
pass: boolean;
intent: string;
+ /** The model the rounds ran on — differs per turn under a router config. */
+ roundModel?: string;
notes: string[];
trace: TraceEntry[];
timings: RunTimings | null;
@@ -94,6 +96,7 @@ export function toRunReport(result: CaseResult | null, error?: string): RunRepor
return {
pass: result.pass,
intent: result.intent,
+ roundModel: result.roundModel,
notes: result.notes,
trace: result.trace,
timings: result.timings,