Open source CapCut alternative.
TypeScript
feat(site): latency evals for the Cut chat assistant
David Le687aeddd1317
diff --git a/site/package.json b/site/package.jsonindex f2dbfdfe..f0314714 100644--- a/site/package.json+++ b/site/package.json@@ -21,6 +21,7 @@"engine:build": "bash scripts/build-cut-engine.sh","worker:build": "bun build --target=node --minify --external=pg-native src/cut/worker/main.ts --outfile dist/cut-worker/main.js","eval:cut-chat": "bun run scripts/eval-cut-chat.ts",+ "eval:cut-latency": "bun run scripts/eval-cut-latency.ts","email:dev": "email dev --dir src/emails --port 3001","test": "bun test src packages/effects-kit/src","test:genvideo": "bun run scripts/genvideo-selftest.ts",diff --git a/site/scripts/eval-cut-chat.ts b/site/scripts/eval-cut-chat.tsindex c08b3943..676b2ca5 100644--- a/site/scripts/eval-cut-chat.ts+++ b/site/scripts/eval-cut-chat.ts@@ -14,22 +14,14 @@* Auth is the dev bypass header (scripts only — never the app), so runs are* dev-server-only and spend no credits. The spoken fixture is synthesized* locally with macOS `say`, so the transcript assertion is deterministic.+ *+ * The cases, fixtures, and runner live in scripts/lib/cut-eval/ and are shared+ * with the latency eval (eval-cut-latency.ts).*/-import { spawnSync } from "node:child_process";-import { mkdtempSync, readFileSync } from "node:fs";-import { tmpdir } from "node:os";-import { join } from "node:path";-import { geminiModelRoles } from "../src/lib/inference/gemini-models";-import { AI_SKILL_INDEX, AI_SKILLS, AI_TOOLS, attachedAssetsBlock, systemPrompt, toolsRanBlock } from "../src/cut/server/ai/catalog";-import {- parseTurnIntent,- TURN_INTENT_PROMPT,- turnIntentInput,- type TurnIntent,-} from "../src/cut/lib/turnIntent";--type Item = Record<string, unknown>;+import { cases } from "./lib/cut-eval/cases";+import { makeFixtureAudio } from "./lib/cut-eval/fixtures";+import { defaultRunConfig, runCase } from "./lib/cut-eval/harness";const args = process.argv.slice(2);const argValue = (flag: string) => {@@ -39,1247 +31,23 @@ const argValue = (flag: string) => {const BASE = argValue("--base") ?? "http://localhost:3000";const ONLY = argValue("--only");const RUNS = Number(argValue("--runs") ?? 1);-// Matches the production chat loop's tool-round cap (geminiChat.ts).-const MAX_ROUNDS = 24;--// Tools that read or steer the view without changing the cut or spending-// credits. Anything outside this set counts as a mutation for the evals.-const SAFE_TOOLS = new Set([- "get_state",- "list_skills",- "read_skill",- "capture_frame",- "watch_video",- "detect_silence",- "listen_audio",- "library_list",- "stock_search",- "list_voices",- "seek",- "select",- "set_playing",- "set_view",-]);--// ----------------------------------------------------------------------------// Fixtures--const SPOKEN_LINE = "Hi, this is Mason.";--/** Synthesize the fixture line locally (macOS say + afconvert), base64 wav. */-function makeFixtureAudio(): { dataBase64: string; mimeType: string } {- const dir = mkdtempSync(join(tmpdir(), "cut-eval-"));- const aiff = join(dir, "line.aiff");- const wav = join(dir, "line.wav");- const say = spawnSync("say", ["-o", aiff, SPOKEN_LINE]);- if (say.status !== 0) throw new Error(`say failed: ${say.stderr}`);- const conv = spawnSync("afconvert", ["-f", "WAVE", "-d", "LEI16@22050", "-c", "1", aiff, wav]);- if (conv.status !== 0) throw new Error(`afconvert failed: ${conv.stderr}`);- return { dataBase64: readFileSync(wav).toString("base64"), mimeType: "audio/wav" };-}--const VOICE_ASSET = {- id: "a-vo1",- name: "AI voice — Hi, this is",- type: "audio",- duration: 1.6,- origin: "voiceover",-};--/** Photos dropped on the chat composer: imported as plain user media (no- * origin), referenced by the message's attachment metadata. */-const PHOTO_ASSETS = [- { id: "a-i1", name: "dog-park.jpg", type: "image" },- { id: "a-i2", name: "dog-beach.jpg", type: "image" },-];--const PHOTO_REFS = PHOTO_ASSETS.map((a, i) => ({- scope: "project",- id: a.id,- name: a.name,- kind: "image",- url: `http://127.0.0.1:41417/media/${a.id}.jpg`,- handle: `i${i + 1}`,-}));--/** Hand-built snapshot mirroring buildAiContext's shape (aiContext.ts): one- * 12.5s video clip on track 0, the voiceover on the soundtrack, no captions.- * Keep the keys in sync with buildAiContext when its shape changes. */-const EDITOR_STATE = {- project: {- id: "p-eval",- name: "Eval project",- duration: 12.5,- aspect: "9:16",- frame: "1080x1920",- },- playhead: 0,- skimmer: null,- playing: false,- selection: null,- media: [- { id: "a-v1", name: "beach.mp4", type: "video", duration: 20 },- VOICE_ASSET,- ...PHOTO_ASSETS,- ],- mediaTruncated: false,- videoTrack: [- {- index: 0,- id: "c1",- asset: "beach.mp4",- start: 0,- len: 12.5,- in: 0,- out: 12.5,- sourceDuration: 20,- muted: false,- framing: "fit",- speed: 1,- },- ],- overlayVideo: [],- soundtrack: [- {- id: "au1",- asset: VOICE_ASSET.name,- start: 0,- len: 1.6,- in: 0,- out: 1.6,- volume: 1,- fadeIn: 0,- fadeOut: 0,- duck: 0.4,- },- ],- titles: [],- subtitles: {- count: 0,- showOnVideo: true,- showOnTimeline: true,- activeTrack: 0,- tracks: [{ track: 0, locale: "en-US", cues: 0 }],- status: "idle",- cues: [],- cuesTruncated: false,- },- publish: { caption: "", tags: "", soundTitle: "", handle: "" },- view: { pxPerSec: 60, timelineH: 260, exportDialogOpen: false },-};--/** Two abutting track-0 clips for the effects cases (transition, animation,- * and look control). Keys mirror buildAiContext, like EDITOR_STATE. */-const TWO_CLIP_STATE = {- ...EDITOR_STATE,- videoTrack: [- {- index: 0,- id: "c1",- asset: "beach.mp4",- start: 0,- len: 6,- in: 0,- out: 6,- sourceDuration: 20,- muted: false,- framing: "fit",- speed: 1,- },- {- index: 1,- id: "c2",- asset: "beach.mp4",- start: 6,- len: 6.5,- in: 6,- out: 12.5,- sourceDuration: 20,- muted: false,- framing: "fit",- speed: 1,- },- ],-};--/** The same pair already joined by a crossfade, for the edge-override case. */-const CROSSFADED_STATE = {- ...TWO_CLIP_STATE,- videoTrack: [- { ...TWO_CLIP_STATE.videoTrack[0], transitionToNext: { style: "crossfade", seconds: 0.5 } },- { ...TWO_CLIP_STATE.videoTrack[1], start: 5.5 },- ],-};--/** A user-imported narration file for the scene-production cases. */-const NARRATION_ASSET = { id: "a-au1", name: "narration.mp3", type: "audio", duration: 24 };--const VOICE_REF = {- scope: "project",- id: VOICE_ASSET.id,- name: VOICE_ASSET.name,- kind: "audio",- url: `http://127.0.0.1:41417/media/${VOICE_ASSET.id}.wav`,- duration: VOICE_ASSET.duration,- handle: "a1",-};--/** The base snapshot with a spoken transcript on track 0 — filler words at- * known cue timings, so "cut the filler" has real ranges to act on. */-const FILLER_CUES = [- { id: "cue1", start: 0, end: 1.8, text: "Um, so today we're" },- { id: "cue2", start: 1.8, end: 4.2, text: "at the beach with the dogs" },- { id: "cue3", start: 4.2, end: 5.6, text: "and, uh, you know," },- { id: "cue4", start: 5.6, end: 9, text: "they absolutely love the water" },- { id: "cue5", start: 9, end: 12.5, text: "so let's watch them play" },-];--const FILLER_STATE = {- ...EDITOR_STATE,- subtitles: {- ...EDITOR_STATE.subtitles,- count: FILLER_CUES.length,- tracks: [{ track: 0, locale: "en-US", cues: FILLER_CUES.length }],- cues: FILLER_CUES,- },-};--/** The base snapshot plus the narration import — the scene cases' spine. */-const AUDIO_STATE = {- ...EDITOR_STATE,- media: [...EDITOR_STATE.media, NARRATION_ASSET],-};--/** A tweet photo import_url landed in an earlier turn: chat-owned (aiTools- * tags chat imports origin "chat"), named by the post's text, not yet placed.- * A past turn replays as its text plus a ledger of the tool names it ran, so- * the model must find this asset through `media` in the snapshot. */-const TWEET_ASSET = {- id: "a-tw1",- name: "first snow of the year ❄️",- type: "image",- origin: "chat",-};--const TWEET_STATE = {- ...EDITOR_STATE,- media: [...EDITOR_STATE.media, TWEET_ASSET],-};--/** The project after a styling turn finished: the reference import landed in- * media, the three city clips are already joined by the crosszooms that turn- * set, and every clip is still unmuted. Everything the previous turn did is- * visible here, so redoing any of it is plainly redundant. */-const cityClip = (n: number, name: string, start: number, styled: boolean) => ({- index: n - 1,- id: `c${n}`,- asset: name,- start,- len: 4.7,- in: 0,- out: 4.7,- sourceDuration: 4.7,- muted: false,- framing: "fit",- speed: 1,- ...(styled ? { transitionToNext: { style: "crosszoom", seconds: 1 } } : {}),-});-const STYLED_STATE = {- ...EDITOR_STATE,- media: [- ...EDITOR_STATE.media,- TWEET_ASSET,- { id: "a-c1", name: "seoul.mp4", type: "video", duration: 4.7, origin: "generated" },- { id: "a-c2", name: "san-francisco.mp4", type: "video", duration: 4.7, origin: "generated" },- { id: "a-c3", name: "toronto.mp4", type: "video", duration: 4.7, origin: "generated" },- ],- videoTrack: [- cityClip(1, "seoul.mp4", 0, true),- cityClip(2, "san-francisco.mp4", 3.7, true),- cityClip(3, "toronto.mp4", 7.4, false),- ],- soundtrack: [],-};--/** A finished scene run's timeline: three generated takes, each clip carrying- * its plan shot number (sceneShot), the narration as the spine. */-const sceneClip = (n: number, start: number, len: number) => ({- index: n - 1,- id: `sc-${n}`,- asset: `shot ${n} take.mp4`,- start,- len,- in: 0,- out: len,- sourceDuration: 10,- muted: true,- framing: "fit",- speed: 1,- sceneShot: n,-});-const SCENE_DONE_STATE = {- ...EDITOR_STATE,- media: [- ...EDITOR_STATE.media,- NARRATION_ASSET,- { id: "t-1", name: "shot 1 take.mp4", type: "video", duration: 10, origin: "generated" },- { id: "t-2", name: "shot 2 take.mp4", type: "video", duration: 10, origin: "generated" },- { id: "t-3", name: "shot 3 take.mp4", type: "video", duration: 10, origin: "generated" },- ],- videoTrack: [sceneClip(1, 0, 3), sceneClip(2, 3, 3), sceneClip(3, 6, 3)],- soundtrack: [{ id: "au-n", asset: NARRATION_ASSET.name, start: 0, len: 9, in: 0, out: 9, volume: 1 }],-};--/** The clip mention refs the composer attaches for "@c1"/"@c2". */-const CLIP_REFS = [- { scope: "clip", id: "sc-1", name: "clip 1", kind: "video", url: "file:sc-1", duration: 3, handle: "c1" },- { scope: "clip", id: "sc-2", name: "clip 2", kind: "video", url: "file:sc-2", duration: 3, handle: "c2" },-];--/** What generate_scene returns after planning (mirrors aiTools' note). */-const SCENE_PLANNED = {- planned: true,- shots: 5,- note: "Planned 5 shots over the audio. A plan card below lists the shots for the user, so keep your reply to one short line — don't re-describe the shots or the timing. Just ask them to confirm; when they do, call approve_scene (each shot spends credits, so don't approve on your own).",-};--/** Scene-case interceptor: asserts on generate_scene's arguments and fails- * the case if the model self-approves the plan. */-function makeSceneSim(check: (args: Record<string, unknown>) => void) {- return () => (name: string, args: Record<string, unknown>): unknown => {- if (name === "generate_scene") {- check(args);- return SCENE_PLANNED;- }- if (name === "approve_scene") throw new Error("approve_scene before the user confirmed");- return undefined;- };-}--/** A composer turn as geminiChat's inputFromMessages builds it (keep the- * envelope text in sync with geminiChat.ts). */-function userTurn(- text: string,- opts?: {- attachAudio?: { dataBase64: string; mimeType: string };- attachRefs?: unknown[];- state?: unknown;- }-): Item {- let full = text;- const extra: Item[] = [];- const refs = [...(opts?.attachAudio ? [VOICE_REF] : []), ...(opts?.attachRefs ?? [])];- full += attachedAssetsBlock(refs);- if (opts?.attachAudio) {- extra.push({ text: `Attached audio "${VOICE_REF.name}":` });- extra.push({ type: "input_audio", ...opts.attachAudio });- }- full += `\n\n<editor_state>\n${JSON.stringify(opts?.state ?? EDITOR_STATE)}\n</editor_state>`;- return { role: "user", content: [{ text: full }, ...extra] };-}--/** Earlier turns in a multi-message case: plain text, no envelope — production- * attaches the editor snapshot to the newest user message alone. */-const plainUserTurn = (text: string): Item => ({ role: "user", content: [{ text }] });-const assistantTurn = (text: string): Item => ({ role: "assistant", content: [{ text }] });--/** An earlier assistant turn that ran tools, carrying the same <tools_ran>- * ledger production replays it with (catalog's toolsRanBlock). */-const assistantToolTurn = (text: string, tools: string[]): Item => ({- role: "assistant",- content: [{ text: `${text}${toolsRanBlock(tools)}` }],-});--/** A tiny track-0 simulator for composed cut flows: splits, trims, deletes,- * and undo evolve real state, so get_state shows the model its own edits and- * later calls can use the new clip ids. A frozen snapshot can't support a- * multi-step cut — the model chases clips that "aren't there" and stalls. */-function makeTimelineSim(base: typeof FILLER_STATE) {- interface SimClip {- id: string;- start: number;- in: number;- out: number;- }- const r2 = (x: number) => Math.round(x * 100) / 100;- let clips: SimClip[] = base.videoTrack.map((c) => ({- id: c.id,- start: c.start,- in: c.in,- out: c.out,- }));- let cues = base.subtitles.cues.map((c) => ({ ...c }));- let audio: SimClip[] = base.soundtrack.map((a) => ({- id: a.id,- start: a.start,- in: a.in,- out: a.out,- }));- const history: { clips: SimClip[]; audio: SimClip[] }[] = [];- const remember = () => history.push({ clips, audio });- let n = 0;- const ripple = () => {- let t = 0;- clips = clips.map((c) => {- const next = { ...c, start: r2(t) };- t += c.out - c.in;- return next;- });- };- const snapshot = () => ({- ...base,- project: { ...base.project, duration: r2(clips.reduce((s, c) => s + c.out - c.in, 0)) },- subtitles: {- ...base.subtitles,- count: cues.length,- tracks: [{ track: 0, locale: "en-US", cues: cues.length }],- cues,- },- soundtrack: audio.map((a) => ({- id: a.id,- asset: VOICE_ASSET.name,- start: a.start,- len: r2(a.out - a.in),- in: a.in,- out: a.out,- volume: 1,- fadeIn: 0,- fadeOut: 0,- duck: 0.4,- })),- videoTrack: clips.map((c, i) => ({- index: i,- id: c.id,- asset: "beach.mp4",- start: c.start,- len: r2(c.out - c.in),- in: c.in,- out: c.out,- sourceDuration: 20,- muted: false,- framing: "fit",- speed: 1,- })),- });- return (name: string, args: Record<string, unknown>): unknown => {- if (name === "get_state") return snapshot();- if (name === "split_at") {- const t = Number(args.t);- const splittable = (x: SimClip) => t > x.start + 0.05 && t < x.start + (x.out - x.in) - 0.05;- const cut = (list: SimClip[]) =>- list.flatMap((x) => {- if (!splittable(x)) return [x];- const at = r2(x.in + (t - x.start));- return [- { id: `${x.id}-${++n}`, start: x.start, in: x.in, out: at },- { id: `${x.id}-${++n}`, start: r2(t), in: at, out: x.out },- ];- });- if (!Number.isFinite(t) || (!clips.some(splittable) && !audio.some(splittable)))- return { error: "Nothing to split at that time." };- remember();- clips = cut(clips);- audio = cut(audio);- return { split: true, videoClips: clips.length };- }- if (name === "delete_item") {- const id = String(args.id);- if (args.kind === "clip") {- if (!clips.some((c) => c.id === id)) return { error: `No clip with id ${id}.` };- remember();- clips = clips.filter((c) => c.id !== id);- ripple();- return { deleted: { kind: "clip", id } };- }- if (args.kind === "audio") {- if (!audio.some((a) => a.id === id)) return { error: `No audio with id ${id}.` };- remember();- audio = audio.filter((a) => a.id !== id);- return { deleted: { kind: "audio", id } };- }- return undefined;- }- if (name === "trim_clip") {- const c = clips.find((x) => x.id === String(args.clipId));- if (!c) return { error: `No video clip with id ${String(args.clipId)}.` };- const nextIn = typeof args.in === "number" ? args.in : c.in;- const nextOut = typeof args.out === "number" ? args.out : c.out;- if (nextOut - nextIn < 0.1) return { error: "Clip must stay at least 0.1s long." };- remember();- clips = clips.map((x) => (x === c ? { ...x, in: nextIn, out: nextOut } : x));- ripple();- return { in: nextIn, out: nextOut, len: r2(nextOut - nextIn) };- }- if (name === "place_clip") {- const c = clips.find((x) => x.id === String(args.clipId));- if (!c) return { error: `No video clip with id ${String(args.clipId)}.` };- remember();- const at = typeof args.start === "number" ? Math.max(0, args.start) : c.start;- clips = clips- .map((x) => (x === c ? { ...x, start: at } : x))- .sort((a, b) => a.start - b.start);- ripple();- return { start: at };- }- if (name === "undo") {- const prev = history.pop();- if (prev) {- clips = prev.clips;- audio = prev.audio;- }- return { ok: true, videoClips: clips.length };- }- if (name === "delete_cue") {- const cue = cues.find((c) => c.id === String(args.id));- if (!cue) return { error: `No subtitle cue with id ${String(args.id)}.` };- cues = cues.filter((c) => c !== cue);- return { deleted: cue.id };- }- if (name === "update_cue") {- const cue = cues.find((c) => c.id === String(args.id));- if (!cue) return { error: `No subtitle cue with id ${String(args.id)}.` };- if (typeof args.text === "string") cue.text = args.text;- if (typeof args.start === "number") cue.start = args.start;- if (typeof args.end === "number") cue.end = args.end;- return { id: cue.id, text: cue.text, start: cue.start, end: cue.end };- }- if (name === "merge_cue") {- const i = cues.findIndex((c) => c.id === String(args.id));- if (i < 0) return { error: `No subtitle cue with id ${String(args.id)}.` };- if (i === 0) return { error: "That is its track's first cue — nothing before it to merge into." };- cues[i - 1] = { ...cues[i - 1], end: cues[i].end, text: `${cues[i - 1].text} ${cues[i].text}` };- cues.splice(i, 1);- return { mergedInto: "previous cue" };- }- return undefined;- };-}--// ----------------------------------------------------------------------------// Cases--interface EvalCase {- name: string;- input: () => Item[];- /** The final reply must match. */- reply: RegExp;- /** Tools that MUST appear across the turn (each stubbed to succeed). */- requiredTools?: string[];- /** At least one of these must appear — for flows with several valid cuts. */- anyTools?: string[];- /** Latency guard: the turn fails if the trace exceeds this many calls. A- * simple ask must not detour through skill reads or state polls. */- maxToolCalls?: number;- /** The tool gate's required verdict for the turn ("chat" turns run with no- * tool declarations, so tool calls become impossible). */- gate?: TurnIntent;- /** Editor snapshot served to get_state for this case (default EDITOR_STATE). */- state?: unknown;- /** Per-run tool interceptor (fresh per run); a non-undefined return serves- * the call before stubs and safe tools. */- simulate?: () => (name: string, args: Record<string, unknown>) => unknown;- /** Stub results for expected tool calls. */- stubs?: Record<string, unknown>;-}--function cases(audio: { dataBase64: string; mimeType: string }): EvalCase[] {- return [- {- // The regression from the screenshot: asking for an attached audio's- // text must be answered from the inline audio — in chat, no captions- // written onto the project.- name: "transcribe-attached-audio",- input: () => [userTurn("get the text from this", { attachAudio: audio })],- reply: /mason/i,- },- {- // The tool gate (the "hi" regression, which once fired an unsolicited- // subtitles_generate): a bare greeting asks for nothing, so the turn- // must classify "chat" and run with every tool declaration withheld.- name: "greeting-is-gated",- input: () => [userTurn("hi")],- reply: /help|what would you/i,- gate: "chat",- maxToolCalls: 0,- },- {- // Same gate mid-conversation: thanks after landed work requests nothing.- name: "thanks-is-gated",- input: () => [- plainUserTurn("trim the first clip down to 5 seconds"),- assistantTurn("Trimmed! The first clip now runs 5 seconds."),- userTurn("nice, thank you!"),- ],- reply: /\S/,- gate: "chat",- maxToolCalls: 0,- },- {- // Context rides into the gate: a terse follow-up is a request — its- // referent lives in the earlier turns — so it classifies "work" and the- // caption cleanup actually lands.- name: "follow-up-do-it-works",- input: () => [- plainUserTurn("could you clean up my captions? lots of filler words"),- assistantTurn(- "Happy to — I'd tidy the five cues on track 0, dropping the ums and uhs. Want me to go ahead?"- ),- userTurn("yes do it", { state: FILLER_STATE }),- ],- reply: /cue|caption|filler|clean|tidi|done|\bum\b|\buh\b|remove|swept/i,- gate: "work",- anyTools: ["update_cue", "delete_cue", "merge_cue"],- state: FILLER_STATE,- simulate: () => makeTimelineSim(FILLER_STATE),- },- {- // Plain questions answer from the snapshot; at most safe reads.- name: "question-answers-in-chat",- input: () => [userTurn("how long is my cut right now?")],- reply: /12[.,]?5?/,- },- {- // The "drag photos in, ask for a movie" flow: an explicit ask to put- // attached media in the cut must reach add_clip for each photo. Styling- // follow-ups a movie ask can reasonably trigger are stubbed too.- name: "photos-request-places-clips",- input: () => [- userTurn("stitch these two photos into a short movie", { attachRefs: PHOTO_REFS }),- ],- reply: /photo|movie|timeline|cut/i,- requiredTools: ["add_clip"],- stubs: {- add_clip: { id: "c-new", kind: "image", index: 1, start: 12.5, len: 8 },- set_transition: { ok: true },- set_project_fade: { ok: true },- add_title: { ok: true },- },- },- {- // Attached media with a question stays a chat deliverable: ideas in- // text, the photos stay off the timeline (add_clip would be a violation).- name: "photos-question-stays-in-chat",- input: () => [- userTurn("what could you make with these two photos?", { attachRefs: PHOTO_REFS }),- ],- // Any engaged ideas answer counts — the real assertion is that no- // mutating tool ran on a question turn.- reply: /photo|movie|montage|slideshow|cut|idea|story|transition|split|dog|pup|beach|park/i,- },- {- // Destructive and organizing tools stay sheathed without an explicit- // command: venting about clutter deletes and files nothing.- name: "clutter-complaint-deletes-nothing",- input: () => [userTurn("ugh, my project is getting cluttered with files")],- reply: /media|asset|file|clean|organi|tidy|clutter|remove|delete/i,- },- {- // Fast path: a plain text-only video ask is ONE generate_video call —- // no scene pipeline, no image staging, no skill-doc detour first.- name: "video-ask-single-generate",- input: () => [userTurn("generate a video about dogs")],- reply: /video|render|dog|minute/i,- requiredTools: ["generate_video"],- maxToolCalls: 2,- stubs: {- generate_video: {- kind: "video",- started: true,- jobId: "job-dogs",- addToTimeline: false,- note:- "Rendering — it previews in this chat when it lands, in a minute or two. It stays in the chat until the user places it.",- },- },- },- {- // The screenshot regression: iterating on an existing clip ("make me a- // better version / closer to the original / a different take") ADDS a- // fresh generation and leaves the clip in place. Without an explicit- // "delete / replace / remove this", deleting the user's existing work- // to "make room" is wrong — the trap word "instead"/"version" is not a- // delete instruction.- name: "iterate-keeps-existing-clip",- input: () => [- userTurn("the beach clip feels off — make me a better version, more of a golden-hour sunset"),- ],- reply: /version|render|sunset|golden|minute|new|preview|kept|left|place/i,- requiredTools: ["generate_video"],- simulate: () => (name) => {- if (name === "delete_item")- throw new Error("delete_item — iterating is not a delete/replace; the existing clip must stay");- if (name === "delete_asset")- throw new Error("delete_asset — iterating must not remove the existing clip's media");- return undefined;- },- stubs: {- generate_video: {- kind: "video",- started: true,- jobId: "job-sunset",- addToTimeline: false,- note:- "Rendering a new take — it previews in this chat when it lands, in a minute or two. Your current clip stays put.",- },- },- },- {- // The screenshot regression: after a turn that imported a reference and- // styled the cut, "mute all clips" mutes and stops. A replayed- // conversation carries no tool traffic, so without the <tools_ran>- // ledger the model reads its own last reply as an unfinished plan and- // runs the whole thing again — re-importing, re-watching, re-styling —- // then narrates that as the answer. The snapshot names every clip and- // its muted flag, so the mute needs no reads either.- name: "mute-all-does-not-redo-prior-turn",- input: () => [- plainUserTurn(- "use the illustration style at https://x.com/example/status/1 and blend the three city clips together"- ),- assistantToolTurn(- "Imported the reference illustration and blended Seoul, San Francisco, and Toronto with 1-second crosszooms.",- ["import_url", "watch_video", "watch_video", "watch_video", "set_transition", "set_transition"]- ),- userTurn("mute all clips", { state: STYLED_STATE }),- ],- reply: /mut/i,- requiredTools: ["set_clip_muted"],- maxToolCalls: 3,- state: STYLED_STATE,- simulate: () => (name) => {- if (name === "import_url" || name === "watch_video" || name === "set_transition")- throw new Error(`${name} — the previous turn already ran it; "mute all clips" only mutes`);- return undefined;- },- stubs: { set_clip_muted: { muted: true } },- },- {- // The tweet-import flow: after import_url landed a photo in an earlier- // turn, "the exact same thing but…" must anchor generate_image on that- // asset via reference_asset_ids — the compose pass and image model see- // the real photo only through the reference, so a text-only render of- // the model's guess loses the source image entirely.- name: "imported-photo-edit-references-photo",- input: () => [- plainUserTurn("https://x.com/oggii_0/status/2079613105571828043"),- assistantTurn(- `Imported the photo from that post — it's on a card in this chat. The post says: "${TWEET_ASSET.name}".`- ),- userTurn("generate the exact same thing but put her in a santa hat", {- state: TWEET_STATE,- }),- ],- reply: /santa|hat|image|render|preview/i,- requiredTools: ["generate_image"],- state: TWEET_STATE,- simulate: () => (name, args) => {- if (name === "generate_image") {- const refs = Array.isArray(args.reference_asset_ids)- ? args.reference_asset_ids.map(String)- : [];- if (!refs.includes(TWEET_ASSET.id))- throw new Error(- `generate_image must reference ${TWEET_ASSET.id}, got ${JSON.stringify(args.reference_asset_ids)}`- );- return {- assetId: "a-gen1",- kind: "image",- note: "Rendered — the image previews on a card in this chat.",- };- }- if (name === "watch_video")- return { note: "Eval: the photo shows a young woman smiling outdoors in falling snow." };- if (name === "generate_video" || name === "generate_scene")- throw new Error(`${name} called — an edited copy of a photo is a generate_image ask`);- return undefined;- },- },- {- // Fast path: audio generation goes straight to voiceover_generate- // (list_voices is a fine extra hop, more than that is a detour).- name: "audio-ask-single-generate",- input: () => [userTurn("generate a voiceover that says welcome to the dog show")],- reply: /voice|audio|narrat|welcome/i,- requiredTools: ["voiceover_generate"],- maxToolCalls: 3,- stubs: {- voiceover_generate: {- assetId: "a-tts2",- name: "AI voice — welcome to the dog show",- start: 0,- duration: 2.3,- voice: "aoede",- },- },- },- {- // Background music goes straight to generate_music (not voiceover, which- // is speech; not generate_scene, which is a whole narrated production).- name: "music-ask-single-generate",- input: () => [userTurn("add some chill background music to this video")],- reply: /music|track|bed|chill|background/i,- requiredTools: ["generate_music"],- maxToolCalls: 3,- stubs: {- generate_music: {- assetId: "a-music1",- name: "chill background music",- duration: 30,- addedToTimeline: false,- },- },- },- {- // Fast path: a direct edit is one trim_clip on the clip in the snapshot.- name: "trim-ask-single-tool",- input: () => [userTurn("trim the first clip down to 5 seconds")],- reply: /trim|5|second/i,- requiredTools: ["trim_clip"],- maxToolCalls: 2,- stubs: { trim_clip: { in: 0, out: 5, len: 5 } },- },- {- // Effects: a styled transition ask is one set_transition on the LEADING- // clip with the catalog style id (a skill read on the way is fine).- name: "transition-ask-styled",- input: () => [- userTurn("add a push down transition between the two clips", { state: TWO_CLIP_STATE }),- ],- reply: /push|transition/i,- requiredTools: ["set_transition"],- maxToolCalls: 3,- state: TWO_CLIP_STATE,- simulate: () => (name, args) => {- if (name !== "set_transition") return undefined;- if (args.clipId !== "c1")- throw new Error(`set_transition on ${args.clipId} — the transition belongs to the leading clip c1`);- if (args.style !== "pushdown")- throw new Error(`style ${args.style} — expected pushdown`);- return { id: "c1", transition: Number(args.seconds) || 0.5, style: "pushdown" };- },- },- {- // Directional comprehension: slide names are the motion direction, so- // "slide in from the left" moves rightward — slideright on c2's "in".- name: "slide-in-direction-correct",- input: () => [- userTurn("make the second clip slide in from the left", { state: TWO_CLIP_STATE }),- ],- reply: /slide/i,- requiredTools: ["set_animation"],- maxToolCalls: 3,- state: TWO_CLIP_STATE,- simulate: () => (name, args) => {- if (name !== "set_animation") return undefined;- if (args.clipId !== "c2" || args.which !== "in")- throw new Error(`set_animation ${args.clipId}/${args.which} — expected c2 "in"`);- if (args.style !== "slideright")- throw new Error(`style ${args.style} — "from the left" moves rightward (slideright)`);- return { id: "c2", which: "in", style: "slideright", seconds: Number(args.seconds) || 0.5 };- },- },- {- // Looks route to set_look with a catalog id — not a hand-rolled- // set_color_grade approximation.- name: "look-ask-single-tool",- input: () => [- userTurn("give the second clip a retro VHS look", { state: TWO_CLIP_STATE }),- ],- reply: /vhs|retro|look/i,- requiredTools: ["set_look"],- maxToolCalls: 3,- state: TWO_CLIP_STATE,- simulate: () => (name, args) => {- if (name !== "set_look") return undefined;- if (args.clipId !== "c2") throw new Error(`set_look on ${args.clipId} — expected c2`);- if (args.style !== "vhs") throw new Error(`style ${args.style} — expected vhs`);- return { id: "c2", look: "vhs", amount: 1 };- },- },- {- // Last pick wins per edge: swapping a crossfading joint for an entrance- // animation is a set_animation — the override clears the transition- // itself (an explicit set_transition 0 first is fine, not required).- // "Pop" exists only as an animation, so the routing is unambiguous- // (a wipe ask could legitimately become a wipe *transition* instead).- name: "animation-overrides-transition",- input: () => [- userTurn("get rid of the crossfade — make the second clip pop in instead", {- state: CROSSFADED_STATE,- }),- ],- reply: /pop/i,- requiredTools: ["set_animation"],- maxToolCalls: 4,- state: CROSSFADED_STATE,- simulate: () => (name, args) => {- if (name === "set_transition") return { id: "c1", transition: 0, style: "crossfade" };- if (name !== "set_animation") return undefined;- if (args.clipId !== "c2" || args.which !== "in")- throw new Error(`set_animation ${args.clipId}/${args.which} — expected c2 "in"`);- if (args.style !== "pop") throw new Error(`style ${args.style} — expected pop`);- return { id: "c2", which: "in", style: "pop", seconds: 0.5 };- },- },- {- // Driving the editor on the user's behalf: with a transcript in the- // snapshot, "cut the filler words" must land real timeline cuts (split- // then delete, or a trim) — not just talk about them, and not merely- // rewrite the captions.- name: "filler-words-cut-with-editor",- input: () => [userTurn("cut the filler words out of my video", { state: FILLER_STATE })],- reply: /filler|um|uh|cut|remove|trim/i,- anyTools: ["split_at", "delete_item", "trim_clip"],- state: FILLER_STATE,- simulate: () => makeTimelineSim(FILLER_STATE),- },- {- // Control: a real edit request must still act with tools — guards- // against over-suppressing tool use.- name: "edit-request-still-uses-tools",- input: () => [userTurn("add captions to my video")],- reply: /caption|subtitle/i,- requiredTools: ["subtitles_generate"],- stubs: {- subtitles_generate: {- status: "done",- track: 0,- locale: "en-US",- cues: 12,- note: "Transcribed the cut onto subtitle track 0.",- },- },- },- {- // A cartoon ask over project audio is one generate_scene plan: the- // audio rides as the spine, the plan waits for the user's go-ahead,- // and a 9:16 project takes 9:16 shots.- name: "cartoon-from-audio-plans-scene",- input: () => [- userTurn("turn my narration audio into a smooth 2D cartoon", { state: AUDIO_STATE }),- ],- reply: /shot|plan|confirm|approv|cartoon|scene/i,- requiredTools: ["generate_scene"],- state: AUDIO_STATE,- simulate: makeSceneSim((args) => {- if (String(args.from_audio_asset_id ?? "") !== NARRATION_ASSET.id)- throw new Error(- `generate_scene must animate ${NARRATION_ASSET.id}, got ${JSON.stringify(args.from_audio_asset_id)}`- );- if (args.aspect !== undefined && args.aspect !== "9:16")- throw new Error(- `generate_scene aspect must match the 9:16 project, got ${JSON.stringify(args.aspect)}`- );- }),- },- {- // Foreign source audio: the speech language rides audio_language so the- // on-device recognizer matches, and the audio still spines the cut.- name: "cartoon-korean-audio-passes-language",- input: () => [- userTurn("my narration audio is in Korean — turn it into a smooth 2D cartoon", {- state: AUDIO_STATE,- }),- ],- reply: /shot|plan|confirm|approv|cartoon|scene|korean/i,- requiredTools: ["generate_scene"],- state: AUDIO_STATE,- simulate: makeSceneSim((args) => {- if (String(args.from_audio_asset_id ?? "") !== NARRATION_ASSET.id)- throw new Error("generate_scene must animate the narration audio");- if (!/^ko(-|$)/i.test(String(args.audio_language ?? "")))- throw new Error(- `audio_language must name Korean, got ${JSON.stringify(args.audio_language)}`- );- }),- },- {- // A style image attached to the ask anchors the scene's look: its asset- // id must ride reference_asset_ids into the plan.- name: "cartoon-style-reference-anchors-look",- input: () => [- userTurn("make a cartoon episode from my narration audio in the style of this picture", {- state: AUDIO_STATE,- attachRefs: [PHOTO_REFS[0]],- }),- ],- reply: /shot|plan|confirm|approv|cartoon|scene|style/i,- requiredTools: ["generate_scene"],- state: AUDIO_STATE,- simulate: makeSceneSim((args) => {- const refs = Array.isArray(args.reference_asset_ids)- ? args.reference_asset_ids.map(String)- : [];- if (!refs.includes(PHOTO_ASSETS[0].id))- throw new Error(- `reference_asset_ids must carry ${PHOTO_ASSETS[0].id}, got ${JSON.stringify(args.reference_asset_ids)}`- );- }),- },- {- // A "how would you" cartoon question is a chat deliverable: approach in- // words, the scene pipeline untouched (generate_scene would violate).- name: "cartoon-question-stays-in-chat",- input: () => [- userTurn("how would you turn my narration audio into a 2D cartoon?", {- state: AUDIO_STATE,- }),- ],- reply: /cartoon|scene|shot|animate|style|narration/i,- state: AUDIO_STATE,- },- {- // Timeline mentions: "@c1 and @c2 are too similar" must map through the- // clip attachments to videoTrack's sceneShot numbers and revise those- // exact shots — regenerate_shot with the complaint riding as the note,- // never a fresh generate_video/generate_scene.- name: "clip-mentions-redo-their-shots",- input: () => [- userTurn(- "clip 1 and clip 2 are too similar and they don't match the audio. fix them",- { attachRefs: CLIP_REFS, state: SCENE_DONE_STATE }- ),- ],- reply: /shot|redo|regenerat/i,- requiredTools: ["regenerate_shot"],- state: SCENE_DONE_STATE,- simulate: () => {- const redone = new Set<number>();- return (name, args) => {- if (name === "regenerate_shot") {- const n = Number(args.n);- if (n !== 1 && n !== 2) throw new Error(`regenerate_shot ${n} — the mentions were shots 1 and 2`);- if (!String(args.note ?? "").trim()) throw new Error("regenerate_shot without the user's note");- redone.add(n);- return { ok: true, message: `Redoing shot ${n}…` };- }- if (name === "generate_video" || name === "generate_scene" || name === "restyle_scene")- throw new Error(`${name} called — a clip complaint revises its shot, not a new render`);- return undefined;- };- },- },- ];-}--// ----------------------------------------------------------------------------// Runner--const toolDeclarations = AI_TOOLS.map((t) => ({- type: "function",- name: t.name,- description: t.description,- parameters: t.inputSchema,-}));--/** A case item's composer text with the envelope stripped back off — the- * gate's classifier sees the raw turn, before the snapshot rides on. */-const itemText = (item: Item): string => {- const first = (item.content as { text?: string }[] | undefined)?.find(- (p) => typeof p.text === "string"- );- return (first?.text ?? "")- .split("\n\n<attached_assets>")[0]- .split("\n\n<editor_state>")[0]- .trim();-};--/** The production tool gate, replicated from geminiChat.ts: a message with- * attachments is work by construction; otherwise the fast-decision model- * judges the newest message. Fails open to "work". */-async function classifyIntent(input: Item[]): Promise<TurnIntent> {- const lastUser = [...input].reverse().find((i) => i.role === "user");- const raw =- ((lastUser?.content as { text?: string }[]) ?? []).find((p) => typeof p.text === "string")- ?.text ?? "";- if (raw.includes("<attached_assets>")) return "work";- const turns = input.map((i) => ({- role: (i.role === "user" ? "user" : "assistant") as "user" | "assistant",- text: itemText(i),- }));- for (let attempt = 0; ; attempt++) {- const res = await fetch(`${BASE}/api/inference/responses`, {- method: "POST",- headers: {- "Content-Type": "application/json",- "x-donkey-client-id": "donkey-cut-eval",- "x-donkey-dev-auth-bypass": "1",- },- body: JSON.stringify({- donkeyProvider: "gemini",- model: geminiModelRoles.fastDecision,- instructions: TURN_INTENT_PROMPT,- input: turnIntentInput(turns),- }),- });- if (res.ok) {- const body = (await res.json()) as { output_text?: string };- return parseTurnIntent(body.output_text);- }- if (![429, 500, 502, 503, 504].includes(res.status) || attempt >= 2) return "work";- await new Promise((r) => setTimeout(r, 800 * (attempt + 1)));- }-}--function serveSafeTool(name: string, state: unknown): unknown {- if (name === "get_state") {- // The fixture snapshot is frozen — it can't reflect this turn's stubbed- // edits. Say so, or the model sees its adds "missing" and re-adds in a- // loop (the live store never has this problem).- return {- ...(state as Record<string, unknown>),- note: "Snapshot may lag this turn's edits — trust each tool's own result.",- };- }- if (name === "list_skills") return { skills: AI_SKILL_INDEX };- if (name === "library_list") return { folders: [], assets: [], templates: [] };- if (name === "detect_silence") return { silences: [] };- return { ok: true };-}--interface CaseResult {- pass: boolean;- reply: string;- trace: string[];- violations: string[];- notes: string[];- intent: TurnIntent;-}--async function runCase(c: EvalCase): Promise<CaseResult> {- const input = c.input();- // The gate runs first, exactly as in production: a "chat" verdict strips- // the tool declarations from the whole turn.- const intent = await classifyIntent(input);- const tools = intent === "work" ? toolDeclarations : undefined;- const sim = c.simulate?.();- const trace: string[] = [];- const violations: string[] = [];- const notes: string[] = [];- let reply = "";- let emptyRounds = 0;-- for (let round = 0; round < MAX_ROUNDS; round++) {- // A transient upstream failure (rate limit, Gemini 5xx) gets a couple of- // retries so it doesn't read as a behavioral failure of the case.- let res: Response;- for (let attempt = 0; ; attempt++) {- res = await fetch(`${BASE}/api/inference/responses`, {- method: "POST",- headers: {- "Content-Type": "application/json",- "x-donkey-client-id": "donkey-cut-eval",- "x-donkey-dev-auth-bypass": "1",- },- body: JSON.stringify({- donkeyProvider: "gemini",- model: geminiModelRoles.chat,- instructions: systemPrompt(),- input,- ...(tools ? { tools } : {}),- }),- });- if (res.ok || ![429, 500, 502, 503, 504].includes(res.status) || attempt >= 2) break;- await new Promise((r) => setTimeout(r, 800 * (attempt + 1)));- }- if (!res.ok) throw new Error(`responses ${res.status}: ${(await res.text()).slice(0, 300)}`);- const body = (await res.json()) as {- output_text?: string;- output?: { type?: string; id?: string; name?: string; arguments?: unknown; thoughtSignature?: string }[];- };-- const text = (body.output_text ?? "").trim();- if (text) reply = reply ? `${reply}\n${text}` : text;-- const calls = (body.output ?? []).filter((o) => o.type === "function_call");- if (calls.length === 0) {- // A no-call round with nothing said all turn is the degenerate empty- // STOP; the production loop (geminiChat.ts) re-asks the same input a- // couple of times, so the eval does too rather than scoring it silent.- if (!reply && emptyRounds < 2) {- emptyRounds++;- continue;- }- break;- }- emptyRounds = 0;-- const assistantParts: Item[] = text ? [{ text }] : [];- const responseParts: Item[] = [];- for (const call of calls) {- const name = String(call.name ?? "unknown");- trace.push(name);- const id = call.id ? String(call.id) : `call-${trace.length}`;- const part: Item = {- functionCall: {- id,- name,- args: call.arguments && typeof call.arguments === "object" ? call.arguments : {},- },- };- if (call.thoughtSignature) part.thoughtSignature = call.thoughtSignature;- assistantParts.push(part);-- let response: unknown;- const argsObj =- call.arguments && typeof call.arguments === "object"- ? (call.arguments as Record<string, unknown>)- : {};- const simulated = sim?.(name, argsObj);- if (name === "read_skill") {- const doc = AI_SKILLS[String(argsObj.name ?? "")];- response = doc ? { doc } : { error: "No such skill." };- } else if (simulated !== undefined) {- response = simulated;- } else if (c.stubs && name in c.stubs) {- response = c.stubs[name];- } else if (SAFE_TOOLS.has(name)) {- response = serveSafeTool(name, c.state ?? EDITOR_STATE);- } else {- violations.push(name);- response = { error: "eval: this tool is disabled for this turn" };- }- responseParts.push({- type: "function_response",- id,- name,- response:- response && typeof response === "object" && !Array.isArray(response)- ? response- : { result: response ?? null },- });- }- input.push({ role: "assistant", content: assistantParts });- input.push({ role: "user", content: responseParts });- }-- if (c.gate && intent !== c.gate) notes.push(`gate said ${intent}, expected ${c.gate}`);- if (!c.reply.test(reply)) notes.push(`reply did not match ${c.reply}`);- for (const t of c.requiredTools ?? []) {- if (!trace.includes(t)) notes.push(`required tool ${t} was never called`);- }- if (c.anyTools && !c.anyTools.some((t) => trace.includes(t)))- notes.push(`none of [${c.anyTools.join(", ")}] were called`);- if (c.maxToolCalls !== undefined && trace.length > c.maxToolCalls)- notes.push(`slow: ${trace.length} tool calls (cap ${c.maxToolCalls})`);- if (violations.length > 0) notes.push(`mutating tools called: ${violations.join(", ")}`);- return { pass: notes.length === 0, reply, trace, violations, notes, intent };-}async function main() {const audio = makeFixtureAudio();const all = cases(audio).filter((c) => !ONLY || c.name === ONLY);if (all.length === 0) throw new Error(`No case named "${ONLY}".`);+ const cfg = defaultRunConfig(BASE);let failed = 0;for (const c of all) {for (let run = 1; run <= RUNS; run++) {const label = RUNS > 1 ? `${c.name} [${run}/${RUNS}]` : c.name;try {- const r = await runCase(c);+ const r = await runCase(c, cfg);+ const names = r.trace.map((t) => t.name);const tools =- r.trace.length > 0- ? ` tools: ${r.trace.join(" → ")}`+ names.length > 0+ ? ` tools: ${names.join(" → ")}`: r.intent === "chat"? " tools: none (gated)": " tools: none";diff --git a/site/scripts/eval-cut-latency.ts b/site/scripts/eval-cut-latency.tsnew file mode 100644index 00000000..8faa9a6d--- /dev/null+++ b/site/scripts/eval-cut-latency.ts@@ -0,0 +1,195 @@+#!/usr/bin/env bun+/**+ * Latency evals for the Cut assistant chat: how fast the model reaches the+ * right tools, and how fast plain conversation turns complete.+ *+ * Runs the shared cases (scripts/lib/cut-eval/) with full timing — gate,+ * time-to-first-token, per-round wall, tool serve — and writes a+ * machine-readable report (schema cut-chat-latency/v1) beside a human table.+ * Latency aggregates count passing runs only; pass rate rides alongside.+ *+ * Run with the site dev server up:+ * bun run scripts/eval-cut-latency.ts+ * [--base http://localhost:3000]+ * [--only <case>] [--bucket chat|single-tool|multi-tool]+ * [--runs N] default 3+ * [--model <registryKey|rawId>] chat model (flash, flashLite, or a raw id)+ * [--gate-model <registryKey|rawId>]+ * [--matrix] run every CANDIDATES row, compare+ * [--enforce-budgets] exit 1 when a case breaches its budget+ * [--out <path>] default <repo>/evals/cut-chat-latency.latest-report.json+ *+ * See scripts/lib/cut-eval/README.md for the improvement loop.+ */++import { mkdirSync, writeFileSync } from "node:fs";+import { dirname, join, resolve } from "node:path";+import { fileURLToPath } from "node:url";+import { geminiModelRoles, geminiModels } from "../src/lib/inference/gemini-models";+import { cases, type Bucket, type EvalCase } from "./lib/cut-eval/cases";+import { makeFixtureAudio } from "./lib/cut-eval/fixtures";+import { runCase, type RunConfig } from "./lib/cut-eval/harness";+import {+ buildBucketSummaries,+ buildCaseReport,+ toRunReport,+ type CaseReport,+ type ConfigReport,+ type LatencyReport,+ type RunReport,+} from "./lib/cut-eval/report";++// Model configs the --matrix mode compares. Add a row when trialing a new+// model id; keep "baseline" matching geminiModelRoles so the comparison always+// includes what production runs.+const CANDIDATES: { label: string; chat: string; gate: string }[] = [+ { label: "baseline", chat: geminiModelRoles.chat, gate: geminiModelRoles.fastDecision },+ { label: "chat=flashLite", chat: geminiModels.flashLite, gate: geminiModelRoles.fastDecision },+ { label: "gate=flash", chat: geminiModelRoles.chat, gate: geminiModels.flash },+];++const args = process.argv.slice(2);+const argValue = (flag: string) => {+ const i = args.indexOf(flag);+ return i >= 0 ? args[i + 1] : undefined;+};+const BASE = argValue("--base") ?? "http://localhost:3000";+const ONLY = argValue("--only");+const BUCKET = argValue("--bucket") as Bucket | undefined;+const RUNS = Number(argValue("--runs") ?? 3);+const MATRIX = args.includes("--matrix");+const ENFORCE_BUDGETS = args.includes("--enforce-budgets");+const OUT = resolve(+ argValue("--out") ??+ join(dirname(fileURLToPath(import.meta.url)), "../../evals/cut-chat-latency.latest-report.json")+);++/** A registry key (flash, flashLite, …) or a raw model id. */+const resolveModel = (v: string): string =>+ (geminiModels as Record<string, string>)[v] ?? v;++const ms = (v: number | null | undefined) => (typeof v === "number" ? `${v}ms` : "—");+const pct = (v: number) => `${Math.round(v * 100)}%`;++function printTable(header: string[], rows: string[][]) {+ const all = [header, ...rows];+ const widths = header.map((_, i) => Math.max(...all.map((r) => r[i].length)));+ for (const [n, row] of all.entries()) {+ console.log(row.map((cell, i) => cell.padEnd(widths[i])).join(" "));+ if (n === 0) console.log(widths.map((w) => "-".repeat(w)).join(" "));+ }+}++async function runConfig(+ label: string,+ cfg: RunConfig,+ selected: EvalCase[]+): Promise<ConfigReport> {+ console.log(`\n== ${label} chat=${cfg.chatModel} gate=${cfg.gateModel}`);+ const caseReports: CaseReport[] = [];+ for (const c of selected) {+ const runs: RunReport[] = [];+ for (let run = 1; run <= RUNS; run++) {+ try {+ runs.push(toRunReport(await runCase(c, cfg)));+ } catch (err) {+ runs.push(toRunReport(null, err instanceof Error ? err.message : String(err)));+ }+ }+ const report = buildCaseReport(c, runs);+ caseReports.push(report);+ const l = report.latency;+ console.log(+ ` ${report.passRate === 1 ? "ok " : report.passRate > 0 ? "part" : "FAIL"} ${c.name}` ++ ` pass ${pct(report.passRate)} total p50 ${ms(l?.totalMs.p50)} ttft p50 ${ms(l?.ttftMs?.p50)}` ++ ` rounds ${l ? l.rounds.mean : "—"}` ++ (report.budgetBreaches.length > 0 ? ` BUDGET: ${report.budgetBreaches.join("; ")}` : "")+ );+ for (const r of runs.filter((r) => !r.pass)) {+ for (const n of r.error ? [r.error] : r.notes) console.log(` - ${n}`);+ }+ }+ return {+ label,+ chatModel: cfg.chatModel,+ gateModel: cfg.gateModel,+ buckets: buildBucketSummaries(caseReports),+ cases: caseReports,+ };+}++async function main() {+ const audio = makeFixtureAudio();+ const selected = cases(audio).filter(+ (c) => (!ONLY || c.name === ONLY) && (!BUCKET || c.bucket === BUCKET)+ );+ if (selected.length === 0)+ throw new Error(ONLY ? `No case named "${ONLY}".` : `No cases in bucket "${BUCKET}".`);++ const configs = MATRIX+ ? CANDIDATES.map((cand) => ({+ label: cand.label,+ cfg: { base: BASE, chatModel: cand.chat, gateModel: cand.gate },+ }))+ : [+ {+ label: "config",+ cfg: {+ base: BASE,+ chatModel: resolveModel(argValue("--model") ?? geminiModelRoles.chat),+ gateModel: resolveModel(argValue("--gate-model") ?? geminiModelRoles.fastDecision),+ },+ },+ ];++ const report: LatencyReport = {+ schema: "cut-chat-latency/v1",+ generatedAt: new Date().toISOString(),+ base: BASE,+ runsPerCase: RUNS,+ configs: [],+ };+ for (const { label, cfg } of configs) {+ report.configs.push(await runConfig(label, cfg, selected));+ }++ console.log("");+ printTable(+ ["config", "bucket", "cases", "pass", "gate p50", "ttft p50", "total p50", "rounds"],+ report.configs.flatMap((c) =>+ (Object.entries(c.buckets) as [Bucket, NonNullable<ConfigReport["buckets"][Bucket]>][]).map(+ ([bucket, s]) => [+ c.label,+ bucket,+ String(s.cases),+ pct(s.passRate),+ ms(s.gateMs?.p50),+ ms(s.ttftMs?.p50),+ ms(s.totalMs?.p50),+ s.meanRounds === null ? "—" : String(s.meanRounds),+ ]+ )+ )+ );++ mkdirSync(dirname(OUT), { recursive: true });+ writeFileSync(OUT, `${JSON.stringify(report, null, 2)}\n`);+ console.log(`\nReport: ${OUT}`);++ const dead = report.configs.flatMap((c) =>+ c.cases.filter((x) => x.passRate === 0).map((x) => `${c.label}/${x.name}`)+ );+ const breaches = report.configs.flatMap((c) =>+ c.cases.filter((x) => x.budgetBreaches.length > 0).map((x) => `${c.label}/${x.name}`)+ );+ if (dead.length > 0) {+ console.log(`\nNo passing runs: ${dead.join(", ")}`);+ process.exit(1);+ }+ if (ENFORCE_BUDGETS && breaches.length > 0) {+ console.log(`\nBudget breaches: ${breaches.join(", ")}`);+ process.exit(1);+ }+}++void main();diff --git a/site/scripts/lib/cut-eval/README.md b/site/scripts/lib/cut-eval/README.mdnew file mode 100644index 00000000..c1b64ebb--- /dev/null+++ b/site/scripts/lib/cut-eval/README.md@@ -0,0 +1,73 @@+# Cut chat evals++Two entrypoints share the fixtures, cases, and runner in this folder:++- `npm run eval:cut-chat` — behavior: does the model do the right thing?+- `npm run eval:cut-latency` — speed: how fast does it get there?++Both replay real composer turns against the live chat model through the dev+server's hosted Responses route. Start `next dev` on :3000 first. Auth is the+dev bypass header, so runs spend no credits. The spoken fixture is synthesized+with macOS `say`, so runs need a Mac.++## Latency eval++```+npm run eval:cut-latency -- [--runs 3] [--only <case>] [--bucket chat|single-tool|multi-tool]+ [--model flash|flashLite|<raw id>] [--gate-model ...]+ [--matrix] [--enforce-budgets] [--out <path>]+```++Every case carries a bucket: `chat` turns (greetings, questions — the gated+fast path), `single-tool` edits (one decisive call), and `multi-tool` edits+(composed cuts). Each run records the gate's classify time, time-to-first-token+(rounds stream, as production does), per-round wall time, tool-serve time, and+round count. Latency aggregates (p50/p95/mean) count passing runs only — a+wrong answer's speed means nothing — and pass rate is reported alongside so a+fast-but-wrong config stays visible.++The eval runs the gate before round 1. Production overlaps it with input+assembly, which is negligible, so this is the same critical path the user+perceives.++`--matrix` runs every row of `CANDIDATES` in `eval-cut-latency.ts` and prints a+comparison table. Model flags and candidate rows resolve through the registry+in `src/lib/inference/gemini-models.ts`; raw model ids pass through.++Per-case `budget` values in `cases.ts` are checked against passing-run p50s.+They report by default; `--enforce-budgets` turns a breach into exit 1. The run+always exits 1 when a case has zero passing runs.++## The report++Each run writes `evals/cut-chat-latency.latest-report.json` (schema+`cut-chat-latency/v1`, types in `report.ts`): per config → per bucket and per+case → pass rate, latency aggregates, budget breaches, and every run's full+timings and tool trace. The committed copy is the baseline; `git diff` on it+shows what a change did to speed and correctness.++## Improving latency (the loop)++1. Make one change: bump a role in `src/lib/inference/gemini-models.ts`, edit+ the system prompt or a tool description in the catalog, or add a+ `CANDIDATES` row for a new model id.+2. Run `npm run eval:cut-latency -- --matrix --runs 3` (narrow with `--bucket`+ or `--only` while iterating).+3. Compare against the baseline: `git diff evals/cut-chat-latency.latest-report.json`.+4. Keep the change only if pass rate holds and the p50s improve.+5. Commit the new report with the change, so the baseline tracks what ships.++What to look at per bucket: `chat` turns live and die on gate p50 + TTFT;+`single-tool` on total p50 and whether rounds stays near 2; `multi-tool` on+round count — every extra round is a full round-trip carrying the whole+conversation and all ~75 tool declarations.++Adding a case: append to `cases.ts` with a `bucket`, stub any mutating tool it+should call, and use `simulate` to assert on arguments. Run it alone with+`--only <name>` and `--runs 5` to check it's stable before relying on it.++## Behavior eval++`npm run eval:cut-chat -- [--only <case>] [--runs N]` prints PASS/FAIL per case+and exits 1 on any failure. Same cases, no timing, no report file — use it as+the quick regression check while the latency eval is the measurement tool.diff --git a/site/scripts/lib/cut-eval/cases.ts b/site/scripts/lib/cut-eval/cases.tsnew file mode 100644index 00000000..493d6526--- /dev/null+++ b/site/scripts/lib/cut-eval/cases.ts@@ -0,0 +1,574 @@+/**+ * The Cut chat eval cases. Each case replays a real composer turn and asserts+ * on what the model does: questions get answered in chat without+ * project-mutating tool calls, edit requests still reach for tools.+ *+ * Every case carries a latency bucket:+ * chat — the fast path: gated greetings and question turns+ * single-tool — one decisive call reaches the right tool+ * multi-tool — composed edits over several calls+ */++import type { TurnIntent } from "../../../src/cut/lib/turnIntent";+import {+ AUDIO_STATE,+ CLIP_REFS,+ CROSSFADED_STATE,+ FILLER_STATE,+ makeSceneSim,+ makeTimelineSim,+ NARRATION_ASSET,+ PHOTO_ASSETS,+ PHOTO_REFS,+ SCENE_DONE_STATE,+ STYLED_STATE,+ TWEET_ASSET,+ TWEET_STATE,+ TWO_CLIP_STATE,+ type Item,+ assistantToolTurn,+ assistantTurn,+ plainUserTurn,+ userTurn,+} from "./fixtures";++export type Bucket = "chat" | "single-tool" | "multi-tool";++export interface EvalCase {+ name: string;+ /** Latency bucket the case aggregates under. */+ bucket: Bucket;+ input: () => Item[];+ /** The final reply must match. */+ reply: RegExp;+ /** Tools that MUST appear across the turn (each stubbed to succeed). */+ requiredTools?: string[];+ /** At least one of these must appear — for flows with several valid cuts. */+ anyTools?: string[];+ /** Latency guard: the turn fails if the trace exceeds this many calls. A+ * simple ask must not detour through skill reads or state polls. */+ maxToolCalls?: number;+ /** The tool gate's required verdict for the turn ("chat" turns run with no+ * tool declarations, so tool calls become impossible). */+ gate?: TurnIntent;+ /** Editor snapshot served to get_state for this case (default EDITOR_STATE). */+ state?: unknown;+ /** Per-run tool interceptor (fresh per run); a non-undefined return serves+ * the call before stubs and safe tools. */+ simulate?: () => (name: string, args: Record<string, unknown>) => unknown;+ /** Stub results for expected tool calls. */+ stubs?: Record<string, unknown>;+ /** Latency budgets checked against passing-run p50s by the latency eval;+ * report-only unless --enforce-budgets. Set from baseline numbers. */+ budget?: { totalMs?: number; ttftMs?: number; gateMs?: number };+}++export function cases(audio: { dataBase64: string; mimeType: string }): EvalCase[] {+ return [+ {+ // The regression from the screenshot: asking for an attached audio's+ // text must be answered from the inline audio — in chat, no captions+ // written onto the project.+ name: "transcribe-attached-audio",+ bucket: "chat",+ input: () => [userTurn("get the text from this", { attachAudio: audio })],+ reply: /mason/i,+ },+ {+ // The tool gate (the "hi" regression, which once fired an unsolicited+ // subtitles_generate): a bare greeting asks for nothing, so the turn+ // must classify "chat" and run with every tool declaration withheld.+ name: "greeting-is-gated",+ bucket: "chat",+ input: () => [userTurn("hi")],+ reply: /help|what would you/i,+ gate: "chat",+ maxToolCalls: 0,+ budget: { gateMs: 2000, ttftMs: 4500, totalMs: 5000 },+ },+ {+ // Same gate mid-conversation: thanks after landed work requests nothing.+ name: "thanks-is-gated",+ bucket: "chat",+ input: () => [+ plainUserTurn("trim the first clip down to 5 seconds"),+ assistantTurn("Trimmed! The first clip now runs 5 seconds."),+ userTurn("nice, thank you!"),+ ],+ reply: /\S/,+ gate: "chat",+ maxToolCalls: 0,+ },+ {+ // Context rides into the gate: a terse follow-up is a request — its+ // referent lives in the earlier turns — so it classifies "work" and the+ // caption cleanup actually lands.+ name: "follow-up-do-it-works",+ bucket: "multi-tool",+ input: () => [+ plainUserTurn("could you clean up my captions? lots of filler words"),+ assistantTurn(+ "Happy to — I'd tidy the five cues on track 0, dropping the ums and uhs. Want me to go ahead?"+ ),+ userTurn("yes do it", { state: FILLER_STATE }),+ ],+ reply: /cue|caption|filler|clean|tidi|done|\bum\b|\buh\b|remove|swept/i,+ gate: "work",+ anyTools: ["update_cue", "delete_cue", "merge_cue"],+ state: FILLER_STATE,+ simulate: () => makeTimelineSim(FILLER_STATE),+ },+ {+ // Plain questions answer from the snapshot; at most safe reads.+ name: "question-answers-in-chat",+ bucket: "chat",+ input: () => [userTurn("how long is my cut right now?")],+ reply: /12[.,]?5?/,+ budget: { totalMs: 6500 },+ },+ {+ // The "drag photos in, ask for a movie" flow: an explicit ask to put+ // attached media in the cut must reach add_clip for each photo. Styling+ // follow-ups a movie ask can reasonably trigger are stubbed too.+ name: "photos-request-places-clips",+ bucket: "multi-tool",+ input: () => [+ userTurn("stitch these two photos into a short movie", { attachRefs: PHOTO_REFS }),+ ],+ reply: /photo|movie|timeline|cut/i,+ requiredTools: ["add_clip"],+ stubs: {+ add_clip: { id: "c-new", kind: "image", index: 1, start: 12.5, len: 8 },+ set_transition: { ok: true },+ set_project_fade: { ok: true },+ add_title: { ok: true },+ },+ },+ {+ // Attached media with a question stays a chat deliverable: ideas in+ // text, the photos stay off the timeline (add_clip would be a violation).+ name: "photos-question-stays-in-chat",+ bucket: "chat",+ input: () => [+ userTurn("what could you make with these two photos?", { attachRefs: PHOTO_REFS }),+ ],+ // Any engaged ideas answer counts — the real assertion is that no+ // mutating tool ran on a question turn.+ reply: /photo|movie|montage|slideshow|cut|idea|story|transition|split|dog|pup|beach|park/i,+ },+ {+ // Destructive and organizing tools stay sheathed without an explicit+ // command: venting about clutter deletes and files nothing.+ name: "clutter-complaint-deletes-nothing",+ bucket: "chat",+ input: () => [userTurn("ugh, my project is getting cluttered with files")],+ reply: /media|asset|file|clean|organi|tidy|clutter|remove|delete/i,+ },+ {+ // Fast path: a plain text-only video ask is ONE generate_video call —+ // no scene pipeline, no image staging, no skill-doc detour first.+ name: "video-ask-single-generate",+ bucket: "single-tool",+ input: () => [userTurn("generate a video about dogs")],+ reply: /video|render|dog|minute/i,+ requiredTools: ["generate_video"],+ maxToolCalls: 2,+ budget: { totalMs: 14000 },+ stubs: {+ generate_video: {+ kind: "video",+ started: true,+ jobId: "job-dogs",+ addToTimeline: false,+ note:+ "Rendering — it previews in this chat when it lands, in a minute or two. It stays in the chat until the user places it.",+ },+ },+ },+ {+ // The screenshot regression: iterating on an existing clip ("make me a+ // better version / closer to the original / a different take") ADDS a+ // fresh generation and leaves the clip in place. Without an explicit+ // "delete / replace / remove this", deleting the user's existing work+ // to "make room" is wrong — the trap word "instead"/"version" is not a+ // delete instruction.+ name: "iterate-keeps-existing-clip",+ bucket: "single-tool",+ input: () => [+ userTurn("the beach clip feels off — make me a better version, more of a golden-hour sunset"),+ ],+ reply: /version|render|sunset|golden|minute|new|preview|kept|left|place/i,+ requiredTools: ["generate_video"],+ simulate: () => (name) => {+ if (name === "delete_item")+ throw new Error("delete_item — iterating is not a delete/replace; the existing clip must stay");+ if (name === "delete_asset")+ throw new Error("delete_asset — iterating must not remove the existing clip's media");+ return undefined;+ },+ stubs: {+ generate_video: {+ kind: "video",+ started: true,+ jobId: "job-sunset",+ addToTimeline: false,+ note:+ "Rendering a new take — it previews in this chat when it lands, in a minute or two. Your current clip stays put.",+ },+ },+ },+ {+ // The screenshot regression: after a turn that imported a reference and+ // styled the cut, "mute all clips" mutes and stops. A replayed+ // conversation carries no tool traffic, so without the <tools_ran>+ // ledger the model reads its own last reply as an unfinished plan and+ // runs the whole thing again — re-importing, re-watching, re-styling —+ // then narrates that as the answer. The snapshot names every clip and+ // its muted flag, so the mute needs no reads either.+ name: "mute-all-does-not-redo-prior-turn",+ bucket: "single-tool",+ input: () => [+ plainUserTurn(+ "use the illustration style at https://x.com/example/status/1 and blend the three city clips together"+ ),+ assistantToolTurn(+ "Imported the reference illustration and blended Seoul, San Francisco, and Toronto with 1-second crosszooms.",+ ["import_url", "watch_video", "watch_video", "watch_video", "set_transition", "set_transition"]+ ),+ userTurn("mute all clips", { state: STYLED_STATE }),+ ],+ reply: /mut/i,+ requiredTools: ["set_clip_muted"],+ maxToolCalls: 3,+ state: STYLED_STATE,+ simulate: () => (name) => {+ if (name === "import_url" || name === "watch_video" || name === "set_transition")+ throw new Error(`${name} — the previous turn already ran it; "mute all clips" only mutes`);+ return undefined;+ },+ stubs: { set_clip_muted: { muted: true } },+ },+ {+ // The tweet-import flow: after import_url landed a photo in an earlier+ // turn, "the exact same thing but…" must anchor generate_image on that+ // asset via reference_asset_ids — the compose pass and image model see+ // the real photo only through the reference, so a text-only render of+ // the model's guess loses the source image entirely.+ name: "imported-photo-edit-references-photo",+ bucket: "single-tool",+ input: () => [+ plainUserTurn("https://x.com/oggii_0/status/2079613105571828043"),+ assistantTurn(+ `Imported the photo from that post — it's on a card in this chat. The post says: "${TWEET_ASSET.name}".`+ ),+ userTurn("generate the exact same thing but put her in a santa hat", {+ state: TWEET_STATE,+ }),+ ],+ reply: /santa|hat|image|render|preview/i,+ requiredTools: ["generate_image"],+ state: TWEET_STATE,+ simulate: () => (name, args) => {+ if (name === "generate_image") {+ const refs = Array.isArray(args.reference_asset_ids)+ ? args.reference_asset_ids.map(String)+ : [];+ if (!refs.includes(TWEET_ASSET.id))+ throw new Error(+ `generate_image must reference ${TWEET_ASSET.id}, got ${JSON.stringify(args.reference_asset_ids)}`+ );+ return {+ assetId: "a-gen1",+ kind: "image",+ note: "Rendered — the image previews on a card in this chat.",+ };+ }+ if (name === "watch_video")+ return { note: "Eval: the photo shows a young woman smiling outdoors in falling snow." };+ if (name === "generate_video" || name === "generate_scene")+ throw new Error(`${name} called — an edited copy of a photo is a generate_image ask`);+ return undefined;+ },+ },+ {+ // Fast path: audio generation goes straight to voiceover_generate+ // (list_voices is a fine extra hop, more than that is a detour).+ name: "audio-ask-single-generate",+ bucket: "single-tool",+ input: () => [userTurn("generate a voiceover that says welcome to the dog show")],+ reply: /voice|audio|narrat|welcome/i,+ requiredTools: ["voiceover_generate"],+ maxToolCalls: 3,+ stubs: {+ voiceover_generate: {+ assetId: "a-tts2",+ name: "AI voice — welcome to the dog show",+ start: 0,+ duration: 2.3,+ voice: "aoede",+ },+ },+ },+ {+ // Background music goes straight to generate_music (not voiceover, which+ // is speech; not generate_scene, which is a whole narrated production).+ name: "music-ask-single-generate",+ bucket: "single-tool",+ input: () => [userTurn("add some chill background music to this video")],+ reply: /music|track|bed|chill|background/i,+ requiredTools: ["generate_music"],+ maxToolCalls: 3,+ stubs: {+ generate_music: {+ assetId: "a-music1",+ name: "chill background music",+ duration: 30,+ addedToTimeline: false,+ },+ },+ },+ {+ // Fast path: a direct edit is one trim_clip on the clip in the snapshot.+ name: "trim-ask-single-tool",+ bucket: "single-tool",+ input: () => [userTurn("trim the first clip down to 5 seconds")],+ reply: /trim|5|second/i,+ requiredTools: ["trim_clip"],+ maxToolCalls: 2,+ stubs: { trim_clip: { in: 0, out: 5, len: 5 } },+ budget: { totalMs: 10000 },+ },+ {+ // Effects: a styled transition ask is one set_transition on the LEADING+ // clip with the catalog style id (a skill read on the way is fine).+ name: "transition-ask-styled",+ bucket: "single-tool",+ input: () => [+ userTurn("add a push down transition between the two clips", { state: TWO_CLIP_STATE }),+ ],+ reply: /push|transition/i,+ requiredTools: ["set_transition"],+ maxToolCalls: 3,+ state: TWO_CLIP_STATE,+ simulate: () => (name, args) => {+ if (name !== "set_transition") return undefined;+ if (args.clipId !== "c1")+ throw new Error(`set_transition on ${args.clipId} — the transition belongs to the leading clip c1`);+ if (args.style !== "pushdown")+ throw new Error(`style ${args.style} — expected pushdown`);+ return { id: "c1", transition: Number(args.seconds) || 0.5, style: "pushdown" };+ },+ },+ {+ // Directional comprehension: slide names are the motion direction, so+ // "slide in from the left" moves rightward — slideright on c2's "in".+ name: "slide-in-direction-correct",+ bucket: "single-tool",+ input: () => [+ userTurn("make the second clip slide in from the left", { state: TWO_CLIP_STATE }),+ ],+ reply: /slide/i,+ requiredTools: ["set_animation"],+ maxToolCalls: 3,+ state: TWO_CLIP_STATE,+ simulate: () => (name, args) => {+ if (name !== "set_animation") return undefined;+ if (args.clipId !== "c2" || args.which !== "in")+ throw new Error(`set_animation ${args.clipId}/${args.which} — expected c2 "in"`);+ if (args.style !== "slideright")+ throw new Error(`style ${args.style} — "from the left" moves rightward (slideright)`);+ return { id: "c2", which: "in", style: "slideright", seconds: Number(args.seconds) || 0.5 };+ },+ },+ {+ // Looks route to set_look with a catalog id — not a hand-rolled+ // set_color_grade approximation.+ name: "look-ask-single-tool",+ bucket: "single-tool",+ input: () => [+ userTurn("give the second clip a retro VHS look", { state: TWO_CLIP_STATE }),+ ],+ reply: /vhs|retro|look/i,+ requiredTools: ["set_look"],+ maxToolCalls: 3,+ state: TWO_CLIP_STATE,+ simulate: () => (name, args) => {+ if (name !== "set_look") return undefined;+ if (args.clipId !== "c2") throw new Error(`set_look on ${args.clipId} — expected c2`);+ if (args.style !== "vhs") throw new Error(`style ${args.style} — expected vhs`);+ return { id: "c2", look: "vhs", amount: 1 };+ },+ },+ {+ // Last pick wins per edge: swapping a crossfading joint for an entrance+ // animation is a set_animation — the override clears the transition+ // itself (an explicit set_transition 0 first is fine, not required).+ // "Pop" exists only as an animation, so the routing is unambiguous+ // (a wipe ask could legitimately become a wipe *transition* instead).+ name: "animation-overrides-transition",+ bucket: "single-tool",+ input: () => [+ userTurn("get rid of the crossfade — make the second clip pop in instead", {+ state: CROSSFADED_STATE,+ }),+ ],+ reply: /pop/i,+ requiredTools: ["set_animation"],+ maxToolCalls: 4,+ state: CROSSFADED_STATE,+ simulate: () => (name, args) => {+ if (name === "set_transition") return { id: "c1", transition: 0, style: "crossfade" };+ if (name !== "set_animation") return undefined;+ if (args.clipId !== "c2" || args.which !== "in")+ throw new Error(`set_animation ${args.clipId}/${args.which} — expected c2 "in"`);+ if (args.style !== "pop") throw new Error(`style ${args.style} — expected pop`);+ return { id: "c2", which: "in", style: "pop", seconds: 0.5 };+ },+ },+ {+ // Driving the editor on the user's behalf: with a transcript in the+ // snapshot, "cut the filler words" must land real timeline cuts (split+ // then delete, or a trim) — not just talk about them, and not merely+ // rewrite the captions.+ name: "filler-words-cut-with-editor",+ bucket: "multi-tool",+ input: () => [userTurn("cut the filler words out of my video", { state: FILLER_STATE })],+ reply: /filler|um|uh|cut|remove|trim/i,+ anyTools: ["split_at", "delete_item", "trim_clip"],+ state: FILLER_STATE,+ simulate: () => makeTimelineSim(FILLER_STATE),+ },+ {+ // Control: a real edit request must still act with tools — guards+ // against over-suppressing tool use.+ name: "edit-request-still-uses-tools",+ bucket: "single-tool",+ input: () => [userTurn("add captions to my video")],+ reply: /caption|subtitle/i,+ requiredTools: ["subtitles_generate"],+ stubs: {+ subtitles_generate: {+ status: "done",+ track: 0,+ locale: "en-US",+ cues: 12,+ note: "Transcribed the cut onto subtitle track 0.",+ },+ },+ },+ {+ // A cartoon ask over project audio is one generate_scene plan: the+ // audio rides as the spine, the plan waits for the user's go-ahead,+ // and a 9:16 project takes 9:16 shots.+ name: "cartoon-from-audio-plans-scene",+ bucket: "single-tool",+ input: () => [+ userTurn("turn my narration audio into a smooth 2D cartoon", { state: AUDIO_STATE }),+ ],+ reply: /shot|plan|confirm|approv|cartoon|scene/i,+ requiredTools: ["generate_scene"],+ state: AUDIO_STATE,+ simulate: makeSceneSim((args) => {+ if (String(args.from_audio_asset_id ?? "") !== NARRATION_ASSET.id)+ throw new Error(+ `generate_scene must animate ${NARRATION_ASSET.id}, got ${JSON.stringify(args.from_audio_asset_id)}`+ );+ if (args.aspect !== undefined && args.aspect !== "9:16")+ throw new Error(+ `generate_scene aspect must match the 9:16 project, got ${JSON.stringify(args.aspect)}`+ );+ }),+ },+ {+ // Foreign source audio: the speech language rides audio_language so the+ // on-device recognizer matches, and the audio still spines the cut.+ name: "cartoon-korean-audio-passes-language",+ bucket: "single-tool",+ input: () => [+ userTurn("my narration audio is in Korean — turn it into a smooth 2D cartoon", {+ state: AUDIO_STATE,+ }),+ ],+ reply: /shot|plan|confirm|approv|cartoon|scene|korean/i,+ requiredTools: ["generate_scene"],+ state: AUDIO_STATE,+ simulate: makeSceneSim((args) => {+ if (String(args.from_audio_asset_id ?? "") !== NARRATION_ASSET.id)+ throw new Error("generate_scene must animate the narration audio");+ if (!/^ko(-|$)/i.test(String(args.audio_language ?? "")))+ throw new Error(+ `audio_language must name Korean, got ${JSON.stringify(args.audio_language)}`+ );+ }),+ },+ {+ // A style image attached to the ask anchors the scene's look: its asset+ // id must ride reference_asset_ids into the plan.+ name: "cartoon-style-reference-anchors-look",+ bucket: "single-tool",+ input: () => [+ userTurn("make a cartoon episode from my narration audio in the style of this picture", {+ state: AUDIO_STATE,+ attachRefs: [PHOTO_REFS[0]],+ }),+ ],+ reply: /shot|plan|confirm|approv|cartoon|scene|style/i,+ requiredTools: ["generate_scene"],+ state: AUDIO_STATE,+ simulate: makeSceneSim((args) => {+ const refs = Array.isArray(args.reference_asset_ids)+ ? args.reference_asset_ids.map(String)+ : [];+ if (!refs.includes(PHOTO_ASSETS[0].id))+ throw new Error(+ `reference_asset_ids must carry ${PHOTO_ASSETS[0].id}, got ${JSON.stringify(args.reference_asset_ids)}`+ );+ }),+ },+ {+ // A "how would you" cartoon question is a chat deliverable: approach in+ // words, the scene pipeline untouched (generate_scene would violate).+ name: "cartoon-question-stays-in-chat",+ bucket: "chat",+ input: () => [+ userTurn("how would you turn my narration audio into a 2D cartoon?", {+ state: AUDIO_STATE,+ }),+ ],+ reply: /cartoon|scene|shot|animate|style|narration/i,+ state: AUDIO_STATE,+ },+ {+ // Timeline mentions: "@c1 and @c2 are too similar" must map through the+ // clip attachments to videoTrack's sceneShot numbers and revise those+ // exact shots — regenerate_shot with the complaint riding as the note,+ // never a fresh generate_video/generate_scene.+ name: "clip-mentions-redo-their-shots",+ bucket: "multi-tool",+ input: () => [+ userTurn(+ "clip 1 and clip 2 are too similar and they don't match the audio. fix them",+ { attachRefs: CLIP_REFS, state: SCENE_DONE_STATE }+ ),+ ],+ reply: /shot|redo|regenerat/i,+ requiredTools: ["regenerate_shot"],+ state: SCENE_DONE_STATE,+ simulate: () => {+ const redone = new Set<number>();+ return (name, args) => {+ if (name === "regenerate_shot") {+ const n = Number(args.n);+ if (n !== 1 && n !== 2) throw new Error(`regenerate_shot ${n} — the mentions were shots 1 and 2`);+ if (!String(args.note ?? "").trim()) throw new Error("regenerate_shot without the user's note");+ redone.add(n);+ return { ok: true, message: `Redoing shot ${n}…` };+ }+ if (name === "generate_video" || name === "generate_scene" || name === "restyle_scene")+ throw new Error(`${name} called — a clip complaint revises its shot, not a new render`);+ return undefined;+ };+ },+ },+ ];+}diff --git a/site/scripts/lib/cut-eval/fixtures.ts b/site/scripts/lib/cut-eval/fixtures.tsnew file mode 100644index 00000000..779fb302--- /dev/null+++ b/site/scripts/lib/cut-eval/fixtures.ts@@ -0,0 +1,548 @@+/**+ * Shared fixtures for the Cut chat evals: editor snapshots, composer-turn+ * builders, tool simulators, and the safe-tool server. Everything here is+ * deterministic so the correctness and latency entrypoints measure the model,+ * not the harness.+ */++import { spawnSync } from "node:child_process";+import { mkdtempSync, readFileSync } from "node:fs";+import { tmpdir } from "node:os";+import { join } from "node:path";+import {+ AI_SKILL_INDEX,+ attachedAssetsBlock,+ toolsRanBlock,+} from "../../../src/cut/server/ai/catalog";++export type Item = Record<string, unknown>;++// Tools that read or steer the view without changing the cut or spending+// credits. Anything outside this set counts as a mutation for the evals.+export const SAFE_TOOLS = new Set([+ "get_state",+ "list_skills",+ "read_skill",+ "capture_frame",+ "watch_video",+ "detect_silence",+ "listen_audio",+ "library_list",+ "stock_search",+ "list_voices",+ "seek",+ "select",+ "set_playing",+ "set_view",+]);++// ---------------------------------------------------------------------------+// Fixtures++export const SPOKEN_LINE = "Hi, this is Mason.";++/** Synthesize the fixture line locally (macOS say + afconvert), base64 wav. */+export function makeFixtureAudio(): { dataBase64: string; mimeType: string } {+ const dir = mkdtempSync(join(tmpdir(), "cut-eval-"));+ const aiff = join(dir, "line.aiff");+ const wav = join(dir, "line.wav");+ const say = spawnSync("say", ["-o", aiff, SPOKEN_LINE]);+ if (say.status !== 0) throw new Error(`say failed: ${say.stderr}`);+ const conv = spawnSync("afconvert", ["-f", "WAVE", "-d", "LEI16@22050", "-c", "1", aiff, wav]);+ if (conv.status !== 0) throw new Error(`afconvert failed: ${conv.stderr}`);+ return { dataBase64: readFileSync(wav).toString("base64"), mimeType: "audio/wav" };+}++export const VOICE_ASSET = {+ id: "a-vo1",+ name: "AI voice — Hi, this is",+ type: "audio",+ duration: 1.6,+ origin: "voiceover",+};++/** Photos dropped on the chat composer: imported as plain user media (no+ * origin), referenced by the message's attachment metadata. */+export const PHOTO_ASSETS = [+ { id: "a-i1", name: "dog-park.jpg", type: "image" },+ { id: "a-i2", name: "dog-beach.jpg", type: "image" },+];++export const PHOTO_REFS = PHOTO_ASSETS.map((a, i) => ({+ scope: "project",+ id: a.id,+ name: a.name,+ kind: "image",+ url: `http://127.0.0.1:41417/media/${a.id}.jpg`,+ handle: `i${i + 1}`,+}));++/** Hand-built snapshot mirroring buildAiContext's shape (aiContext.ts): one+ * 12.5s video clip on track 0, the voiceover on the soundtrack, no captions.+ * Keep the keys in sync with buildAiContext when its shape changes. */+export const EDITOR_STATE = {+ project: {+ id: "p-eval",+ name: "Eval project",+ duration: 12.5,+ aspect: "9:16",+ frame: "1080x1920",+ },+ playhead: 0,+ skimmer: null,+ playing: false,+ selection: null,+ media: [+ { id: "a-v1", name: "beach.mp4", type: "video", duration: 20 },+ VOICE_ASSET,+ ...PHOTO_ASSETS,+ ],+ mediaTruncated: false,+ videoTrack: [+ {+ index: 0,+ id: "c1",+ asset: "beach.mp4",+ start: 0,+ len: 12.5,+ in: 0,+ out: 12.5,+ sourceDuration: 20,+ muted: false,+ framing: "fit",+ speed: 1,+ },+ ],+ overlayVideo: [],+ soundtrack: [+ {+ id: "au1",+ asset: VOICE_ASSET.name,+ start: 0,+ len: 1.6,+ in: 0,+ out: 1.6,+ volume: 1,+ fadeIn: 0,+ fadeOut: 0,+ duck: 0.4,+ },+ ],+ titles: [],+ subtitles: {+ count: 0,+ showOnVideo: true,+ showOnTimeline: true,+ activeTrack: 0,+ tracks: [{ track: 0, locale: "en-US", cues: 0 }],+ status: "idle",+ cues: [],+ cuesTruncated: false,+ },+ publish: { caption: "", tags: "", soundTitle: "", handle: "" },+ view: { pxPerSec: 60, timelineH: 260, exportDialogOpen: false },+};++/** Two abutting track-0 clips for the effects cases (transition, animation,+ * and look control). Keys mirror buildAiContext, like EDITOR_STATE. */+export const TWO_CLIP_STATE = {+ ...EDITOR_STATE,+ videoTrack: [+ {+ index: 0,+ id: "c1",+ asset: "beach.mp4",+ start: 0,+ len: 6,+ in: 0,+ out: 6,+ sourceDuration: 20,+ muted: false,+ framing: "fit",+ speed: 1,+ },+ {+ index: 1,+ id: "c2",+ asset: "beach.mp4",+ start: 6,+ len: 6.5,+ in: 6,+ out: 12.5,+ sourceDuration: 20,+ muted: false,+ framing: "fit",+ speed: 1,+ },+ ],+};++/** The same pair already joined by a crossfade, for the edge-override case. */+export const CROSSFADED_STATE = {+ ...TWO_CLIP_STATE,+ videoTrack: [+ { ...TWO_CLIP_STATE.videoTrack[0], transitionToNext: { style: "crossfade", seconds: 0.5 } },+ { ...TWO_CLIP_STATE.videoTrack[1], start: 5.5 },+ ],+};++/** A user-imported narration file for the scene-production cases. */+export const NARRATION_ASSET = { id: "a-au1", name: "narration.mp3", type: "audio", duration: 24 };++export const VOICE_REF = {+ scope: "project",+ id: VOICE_ASSET.id,+ name: VOICE_ASSET.name,+ kind: "audio",+ url: `http://127.0.0.1:41417/media/${VOICE_ASSET.id}.wav`,+ duration: VOICE_ASSET.duration,+ handle: "a1",+};++/** The base snapshot with a spoken transcript on track 0 — filler words at+ * known cue timings, so "cut the filler" has real ranges to act on. */+export const FILLER_CUES = [+ { id: "cue1", start: 0, end: 1.8, text: "Um, so today we're" },+ { id: "cue2", start: 1.8, end: 4.2, text: "at the beach with the dogs" },+ { id: "cue3", start: 4.2, end: 5.6, text: "and, uh, you know," },+ { id: "cue4", start: 5.6, end: 9, text: "they absolutely love the water" },+ { id: "cue5", start: 9, end: 12.5, text: "so let's watch them play" },+];++export const FILLER_STATE = {+ ...EDITOR_STATE,+ subtitles: {+ ...EDITOR_STATE.subtitles,+ count: FILLER_CUES.length,+ tracks: [{ track: 0, locale: "en-US", cues: FILLER_CUES.length }],+ cues: FILLER_CUES,+ },+};++/** The base snapshot plus the narration import — the scene cases' spine. */+export const AUDIO_STATE = {+ ...EDITOR_STATE,+ media: [...EDITOR_STATE.media, NARRATION_ASSET],+};++/** A tweet photo import_url landed in an earlier turn: chat-owned (aiTools+ * tags chat imports origin "chat"), named by the post's text, not yet placed.+ * A past turn replays as its text plus a ledger of the tool names it ran, so+ * the model must find this asset through `media` in the snapshot. */+export const TWEET_ASSET = {+ id: "a-tw1",+ name: "first snow of the year ❄️",+ type: "image",+ origin: "chat",+};++export const TWEET_STATE = {+ ...EDITOR_STATE,+ media: [...EDITOR_STATE.media, TWEET_ASSET],+};++/** The project after a styling turn finished: the reference import landed in+ * media, the three city clips are already joined by the crosszooms that turn+ * set, and every clip is still unmuted. Everything the previous turn did is+ * visible here, so redoing any of it is plainly redundant. */+const cityClip = (n: number, name: string, start: number, styled: boolean) => ({+ index: n - 1,+ id: `c${n}`,+ asset: name,+ start,+ len: 4.7,+ in: 0,+ out: 4.7,+ sourceDuration: 4.7,+ muted: false,+ framing: "fit",+ speed: 1,+ ...(styled ? { transitionToNext: { style: "crosszoom", seconds: 1 } } : {}),+});+export const STYLED_STATE = {+ ...EDITOR_STATE,+ media: [+ ...EDITOR_STATE.media,+ TWEET_ASSET,+ { id: "a-c1", name: "seoul.mp4", type: "video", duration: 4.7, origin: "generated" },+ { id: "a-c2", name: "san-francisco.mp4", type: "video", duration: 4.7, origin: "generated" },+ { id: "a-c3", name: "toronto.mp4", type: "video", duration: 4.7, origin: "generated" },+ ],+ videoTrack: [+ cityClip(1, "seoul.mp4", 0, true),+ cityClip(2, "san-francisco.mp4", 3.7, true),+ cityClip(3, "toronto.mp4", 7.4, false),+ ],+ soundtrack: [],+};++/** A finished scene run's timeline: three generated takes, each clip carrying+ * its plan shot number (sceneShot), the narration as the spine. */+const sceneClip = (n: number, start: number, len: number) => ({+ index: n - 1,+ id: `sc-${n}`,+ asset: `shot ${n} take.mp4`,+ start,+ len,+ in: 0,+ out: len,+ sourceDuration: 10,+ muted: true,+ framing: "fit",+ speed: 1,+ sceneShot: n,+});+export const SCENE_DONE_STATE = {+ ...EDITOR_STATE,+ media: [+ ...EDITOR_STATE.media,+ NARRATION_ASSET,+ { id: "t-1", name: "shot 1 take.mp4", type: "video", duration: 10, origin: "generated" },+ { id: "t-2", name: "shot 2 take.mp4", type: "video", duration: 10, origin: "generated" },+ { id: "t-3", name: "shot 3 take.mp4", type: "video", duration: 10, origin: "generated" },+ ],+ videoTrack: [sceneClip(1, 0, 3), sceneClip(2, 3, 3), sceneClip(3, 6, 3)],+ soundtrack: [{ id: "au-n", asset: NARRATION_ASSET.name, start: 0, len: 9, in: 0, out: 9, volume: 1 }],+};++/** The clip mention refs the composer attaches for "@c1"/"@c2". */+export const CLIP_REFS = [+ { scope: "clip", id: "sc-1", name: "clip 1", kind: "video", url: "file:sc-1", duration: 3, handle: "c1" },+ { scope: "clip", id: "sc-2", name: "clip 2", kind: "video", url: "file:sc-2", duration: 3, handle: "c2" },+];++/** What generate_scene returns after planning (mirrors aiTools' note). */+export const SCENE_PLANNED = {+ planned: true,+ shots: 5,+ note: "Planned 5 shots over the audio. A plan card below lists the shots for the user, so keep your reply to one short line — don't re-describe the shots or the timing. Just ask them to confirm; when they do, call approve_scene (each shot spends credits, so don't approve on your own).",+};++/** Scene-case interceptor: asserts on generate_scene's arguments and fails+ * the case if the model self-approves the plan. */+export function makeSceneSim(check: (args: Record<string, unknown>) => void) {+ return () => (name: string, args: Record<string, unknown>): unknown => {+ if (name === "generate_scene") {+ check(args);+ return SCENE_PLANNED;+ }+ if (name === "approve_scene") throw new Error("approve_scene before the user confirmed");+ return undefined;+ };+}++/** A composer turn as geminiChat's inputFromMessages builds it (keep the+ * envelope text in sync with geminiChat.ts). */+export function userTurn(+ text: string,+ opts?: {+ attachAudio?: { dataBase64: string; mimeType: string };+ attachRefs?: unknown[];+ state?: unknown;+ }+): Item {+ let full = text;+ const extra: Item[] = [];+ const refs = [...(opts?.attachAudio ? [VOICE_REF] : []), ...(opts?.attachRefs ?? [])];+ full += attachedAssetsBlock(refs);+ if (opts?.attachAudio) {+ extra.push({ text: `Attached audio "${VOICE_REF.name}":` });+ extra.push({ type: "input_audio", ...opts.attachAudio });+ }+ full += `\n\n<editor_state>\n${JSON.stringify(opts?.state ?? EDITOR_STATE)}\n</editor_state>`;+ return { role: "user", content: [{ text: full }, ...extra] };+}++/** Earlier turns in a multi-message case: plain text, no envelope — production+ * attaches the editor snapshot to the newest user message alone. */+export const plainUserTurn = (text: string): Item => ({ role: "user", content: [{ text }] });+export const assistantTurn = (text: string): Item => ({ role: "assistant", content: [{ text }] });++/** An earlier assistant turn that ran tools, carrying the same <tools_ran>+ * ledger production replays it with (catalog's toolsRanBlock). */+export const assistantToolTurn = (text: string, tools: string[]): Item => ({+ role: "assistant",+ content: [{ text: `${text}${toolsRanBlock(tools)}` }],+});++/** A tiny track-0 simulator for composed cut flows: splits, trims, deletes,+ * and undo evolve real state, so get_state shows the model its own edits and+ * later calls can use the new clip ids. A frozen snapshot can't support a+ * multi-step cut — the model chases clips that "aren't there" and stalls. */+export function makeTimelineSim(base: typeof FILLER_STATE) {+ interface SimClip {+ id: string;+ start: number;+ in: number;+ out: number;+ }+ const r2 = (x: number) => Math.round(x * 100) / 100;+ let clips: SimClip[] = base.videoTrack.map((c) => ({+ id: c.id,+ start: c.start,+ in: c.in,+ out: c.out,+ }));+ let cues = base.subtitles.cues.map((c) => ({ ...c }));+ let audio: SimClip[] = base.soundtrack.map((a) => ({+ id: a.id,+ start: a.start,+ in: a.in,+ out: a.out,+ }));+ const history: { clips: SimClip[]; audio: SimClip[] }[] = [];+ const remember = () => history.push({ clips, audio });+ let n = 0;+ const ripple = () => {+ let t = 0;+ clips = clips.map((c) => {+ const next = { ...c, start: r2(t) };+ t += c.out - c.in;+ return next;+ });+ };+ const snapshot = () => ({+ ...base,+ project: { ...base.project, duration: r2(clips.reduce((s, c) => s + c.out - c.in, 0)) },+ subtitles: {+ ...base.subtitles,+ count: cues.length,+ tracks: [{ track: 0, locale: "en-US", cues: cues.length }],+ cues,+ },+ soundtrack: audio.map((a) => ({+ id: a.id,+ asset: VOICE_ASSET.name,+ start: a.start,+ len: r2(a.out - a.in),+ in: a.in,+ out: a.out,+ volume: 1,+ fadeIn: 0,+ fadeOut: 0,+ duck: 0.4,+ })),+ videoTrack: clips.map((c, i) => ({+ index: i,+ id: c.id,+ asset: "beach.mp4",+ start: c.start,+ len: r2(c.out - c.in),+ in: c.in,+ out: c.out,+ sourceDuration: 20,+ muted: false,+ framing: "fit",+ speed: 1,+ })),+ });+ return (name: string, args: Record<string, unknown>): unknown => {+ if (name === "get_state") return snapshot();+ if (name === "split_at") {+ const t = Number(args.t);+ const splittable = (x: SimClip) => t > x.start + 0.05 && t < x.start + (x.out - x.in) - 0.05;+ const cut = (list: SimClip[]) =>+ list.flatMap((x) => {+ if (!splittable(x)) return [x];+ const at = r2(x.in + (t - x.start));+ return [+ { id: `${x.id}-${++n}`, start: x.start, in: x.in, out: at },+ { id: `${x.id}-${++n}`, start: r2(t), in: at, out: x.out },+ ];+ });+ if (!Number.isFinite(t) || (!clips.some(splittable) && !audio.some(splittable)))+ return { error: "Nothing to split at that time." };+ remember();+ clips = cut(clips);+ audio = cut(audio);+ return { split: true, videoClips: clips.length };+ }+ if (name === "delete_item") {+ const id = String(args.id);+ if (args.kind === "clip") {+ if (!clips.some((c) => c.id === id)) return { error: `No clip with id ${id}.` };+ remember();+ clips = clips.filter((c) => c.id !== id);+ ripple();+ return { deleted: { kind: "clip", id } };+ }+ if (args.kind === "audio") {+ if (!audio.some((a) => a.id === id)) return { error: `No audio with id ${id}.` };+ remember();+ audio = audio.filter((a) => a.id !== id);+ return { deleted: { kind: "audio", id } };+ }+ return undefined;+ }+ if (name === "trim_clip") {+ const c = clips.find((x) => x.id === String(args.clipId));+ if (!c) return { error: `No video clip with id ${String(args.clipId)}.` };+ const nextIn = typeof args.in === "number" ? args.in : c.in;+ const nextOut = typeof args.out === "number" ? args.out : c.out;+ if (nextOut - nextIn < 0.1) return { error: "Clip must stay at least 0.1s long." };+ remember();+ clips = clips.map((x) => (x === c ? { ...x, in: nextIn, out: nextOut } : x));+ ripple();+ return { in: nextIn, out: nextOut, len: r2(nextOut - nextIn) };+ }+ if (name === "place_clip") {+ const c = clips.find((x) => x.id === String(args.clipId));+ if (!c) return { error: `No video clip with id ${String(args.clipId)}.` };+ remember();+ const at = typeof args.start === "number" ? Math.max(0, args.start) : c.start;+ clips = clips+ .map((x) => (x === c ? { ...x, start: at } : x))+ .sort((a, b) => a.start - b.start);+ ripple();+ return { start: at };+ }+ if (name === "undo") {+ const prev = history.pop();+ if (prev) {+ clips = prev.clips;+ audio = prev.audio;+ }+ return { ok: true, videoClips: clips.length };+ }+ if (name === "delete_cue") {+ const cue = cues.find((c) => c.id === String(args.id));+ if (!cue) return { error: `No subtitle cue with id ${String(args.id)}.` };+ cues = cues.filter((c) => c !== cue);+ return { deleted: cue.id };+ }+ if (name === "update_cue") {+ const cue = cues.find((c) => c.id === String(args.id));+ if (!cue) return { error: `No subtitle cue with id ${String(args.id)}.` };+ if (typeof args.text === "string") cue.text = args.text;+ if (typeof args.start === "number") cue.start = args.start;+ if (typeof args.end === "number") cue.end = args.end;+ return { id: cue.id, text: cue.text, start: cue.start, end: cue.end };+ }+ if (name === "merge_cue") {+ const i = cues.findIndex((c) => c.id === String(args.id));+ if (i < 0) return { error: `No subtitle cue with id ${String(args.id)}.` };+ if (i === 0) return { error: "That is its track's first cue — nothing before it to merge into." };+ cues[i - 1] = { ...cues[i - 1], end: cues[i].end, text: `${cues[i - 1].text} ${cues[i].text}` };+ cues.splice(i, 1);+ return { mergedInto: "previous cue" };+ }+ return undefined;+ };+}++/** Serve a read-only tool from the fixture snapshot. */+export function serveSafeTool(name: string, state: unknown): unknown {+ if (name === "get_state") {+ // The fixture snapshot is frozen — it can't reflect this turn's stubbed+ // edits. Say so, or the model sees its adds "missing" and re-adds in a+ // loop (the live store never has this problem).+ return {+ ...(state as Record<string, unknown>),+ note: "Snapshot may lag this turn's edits — trust each tool's own result.",+ };+ }+ if (name === "list_skills") return { skills: AI_SKILL_INDEX };+ if (name === "library_list") return { folders: [], assets: [], templates: [] };+ if (name === "detect_silence") return { silences: [] };+ return { ok: true };+}diff --git a/site/scripts/lib/cut-eval/harness.ts b/site/scripts/lib/cut-eval/harness.tsnew file mode 100644index 00000000..06b1048e--- /dev/null+++ b/site/scripts/lib/cut-eval/harness.ts@@ -0,0 +1,426 @@+/**+ * The eval runner: one case = one chat turn replayed against the live model+ * through the hosted Responses route, with the production loop's shape — the+ * tool gate first, then streamed tool rounds until the model settles.+ *+ * Every run is timed. Rounds stream (stream:true, as production does), so the+ * harness can stamp time-to-first-token; the gate, each round, and each tool+ * serve get their own wall-clock. Retry attempts and backoff sleeps accumulate+ * separately in retryMs so transient upstream hiccups never pollute the+ * latency numbers.+ */++import {+ AI_SKILLS,+ AI_TOOLS,+ systemPrompt,+} from "../../../src/cut/server/ai/catalog";+import {+ parseTurnIntent,+ TURN_INTENT_PROMPT,+ turnIntentInput,+ type TurnIntent,+} from "../../../src/cut/lib/turnIntent";+import { geminiModelRoles } from "../../../src/lib/inference/gemini-models";+import { EDITOR_STATE, SAFE_TOOLS, serveSafeTool, type Item } from "./fixtures";+import type { EvalCase } from "./cases";++// Matches the production chat loop's tool-round cap (geminiChat.ts).+export const MAX_ROUNDS = 24;++const RETRYABLE_STATUS = new Set([429, 500, 502, 503, 504]);+const TRANSIENT_RETRIES = 2;++export interface RunConfig {+ /** Dev server base URL. */+ base: string;+ /** Model the chat rounds run on. */+ chatModel: string;+ /** Model the tool gate runs on. */+ gateModel: string;+}++export const defaultRunConfig = (base: string): RunConfig => ({+ base,+ chatModel: geminiModelRoles.chat,+ gateModel: geminiModelRoles.fastDecision,+});++export interface RoundTiming {+ /** Wall time of the round's successful request, request start → body settled. */+ roundMs: number;+ /** Round start → first visible (non-whitespace) text delta, if any text streamed. */+ firstDeltaMs: number | null;+ calls: { name: string; execMs: number }[];+}++export interface RunTimings {+ /** Wall time of the gate's successful classify call. */+ gateMs: number;+ /** True when attachments classified the turn "work" with no model call. */+ gateSkipped: boolean;+ /** Turn start → first visible text delta anywhere in the turn. */+ ttftMs: number | null;+ /** Σ successful round wall times. */+ modelMs: number;+ /** Σ tool-serve wall times (stubs and sims today, so ≈0 — kept separate so+ * real tool execution never pollutes model timing). */+ toolMs: number;+ /** gateMs + modelMs + toolMs — the turn's critical path, retries excluded. */+ totalMs: number;+ /** Failed attempts plus backoff sleeps, across the gate and every round. */+ retryMs: number;+ retries: number;+ rounds: RoundTiming[];+}++export interface TraceEntry {+ name: string;+ /** Zero-based round the call arrived in. */+ round: number;+ /** Turn start → this call being served. */+ atMs: number;+ execMs: number;+}++export interface CaseResult {+ pass: boolean;+ reply: string;+ trace: TraceEntry[];+ violations: string[];+ notes: string[];+ intent: TurnIntent;+ timings: RunTimings;+}++interface RetryTally {+ retryMs: number;+ retries: number;+}++interface ResponseBody {+ output_text?: string;+ output?: {+ type?: string;+ id?: string;+ name?: string;+ arguments?: unknown;+ thoughtSignature?: string;+ }[];+}++const toolDeclarations = AI_TOOLS.map((t) => ({+ type: "function",+ name: t.name,+ description: t.description,+ parameters: t.inputSchema,+}));++const post = (cfg: RunConfig, body: Record<string, unknown>) =>+ fetch(`${cfg.base}/api/inference/responses`, {+ method: "POST",+ headers: {+ "Content-Type": "application/json",+ "x-donkey-client-id": "donkey-cut-eval",+ "x-donkey-dev-auth-bypass": "1",+ },+ body: JSON.stringify(body),+ });++const sleepTallied = async (ms: number, tally: RetryTally) => {+ const t0 = performance.now();+ await new Promise((r) => setTimeout(r, ms));+ tally.retryMs += performance.now() - t0;+};++/** A case item's composer text with the envelope stripped back off — the+ * gate's classifier sees the raw turn, before the snapshot rides on. */+const itemText = (item: Item): string => {+ const first = (item.content as { text?: string }[] | undefined)?.find(+ (p) => typeof p.text === "string"+ );+ return (first?.text ?? "")+ .split("\n\n<attached_assets>")[0]+ .split("\n\n<editor_state>")[0]+ .trim();+};++/** The production tool gate, replicated from geminiChat.ts: a message with+ * attachments is work by construction; otherwise the fast-decision model+ * judges the newest message. Fails open to "work". */+export async function classifyIntent(+ input: Item[],+ cfg: RunConfig,+ tally: RetryTally+): Promise<{ intent: TurnIntent; gateMs: number; gateSkipped: boolean }> {+ const lastUser = [...input].reverse().find((i) => i.role === "user");+ const raw =+ ((lastUser?.content as { text?: string }[]) ?? []).find((p) => typeof p.text === "string")+ ?.text ?? "";+ if (raw.includes("<attached_assets>")) return { intent: "work", gateMs: 0, gateSkipped: true };+ const turns = input.map((i) => ({+ role: (i.role === "user" ? "user" : "assistant") as "user" | "assistant",+ text: itemText(i),+ }));+ for (let attempt = 0; ; attempt++) {+ const t0 = performance.now();+ const res = await post(cfg, {+ donkeyProvider: "gemini",+ model: cfg.gateModel,+ instructions: TURN_INTENT_PROMPT,+ input: turnIntentInput(turns),+ });+ if (res.ok) {+ const body = (await res.json()) as { output_text?: string };+ return {+ intent: parseTurnIntent(body.output_text),+ gateMs: performance.now() - t0,+ gateSkipped: false,+ };+ }+ tally.retries++;+ tally.retryMs += performance.now() - t0;+ if (!RETRYABLE_STATUS.has(res.status) || attempt >= TRANSIENT_RETRIES)+ return { intent: "work", gateMs: 0, gateSkipped: false };+ await sleepTallied(800 * (attempt + 1), tally);+ }+}++/** The route's SSE frames (mirrors geminiChat.ts): every text delta goes out+ * through onDelta, and the terminal `response.completed` frame — the same body+ * a plain round returns — is the return value. */+async function readStreamedRound(+ res: Response,+ onDelta: (delta: string) => void+): Promise<ResponseBody> {+ const reader = res.body?.getReader();+ if (!reader) throw new Error("Gemini returned no response stream.");+ const decoder = new TextDecoder();+ let buffer = "";+ let completed: ResponseBody | null = null;+ for (;;) {+ const { done, value } = await reader.read();+ if (done) break;+ buffer += decoder.decode(value, { stream: true });+ for (;;) {+ const split = buffer.indexOf("\n\n");+ if (split === -1) break;+ const frame = buffer.slice(0, split);+ buffer = buffer.slice(split + 2);+ let isError = false;+ let data = "";+ for (const line of frame.split("\n")) {+ if (line.startsWith("event: ")) isError = line.slice(7).trim() === "error";+ else if (line.startsWith("data: ")) data += line.slice(6);+ }+ if (!data || data === "[DONE]") continue;+ let event: { type?: string; delta?: string; response?: ResponseBody; message?: string };+ try {+ event = JSON.parse(data) as typeof event;+ } catch {+ continue;+ }+ if (isError) throw new Error(event.message || "Gemini stream failed.");+ if (event.type === "response.output_text.delta" && typeof event.delta === "string") {+ onDelta(event.delta);+ } else if (event.type === "response.completed" && event.response) {+ completed = event.response;+ }+ }+ }+ if (!completed) throw new Error("Gemini stream ended early.");+ return completed;+}++/** One streamed round with transient retries. A broken stream retries only+ * while no delta has arrived (production's rule); failed attempts and backoff+ * land in the tally, so roundMs is the successful request alone. */+async function postRoundTimed(+ cfg: RunConfig,+ payload: Record<string, unknown>,+ tally: RetryTally,+ onFirstDelta: () => void+): Promise<{ body: ResponseBody; roundMs: number; firstDeltaMs: number | null }> {+ for (let attempt = 0; ; attempt++) {+ const t0 = performance.now();+ let res: Response;+ try {+ res = await post(cfg, { ...payload, stream: true });+ } catch (err) {+ tally.retries++;+ tally.retryMs += performance.now() - t0;+ if (attempt >= TRANSIENT_RETRIES) throw err;+ await sleepTallied(800 * (attempt + 1), tally);+ continue;+ }+ if (res.ok) {+ let firstDeltaMs: number | null = null;+ let delivered = false;+ try {+ const body = await readStreamedRound(res, (delta) => {+ delivered = true;+ if (firstDeltaMs === null && delta.trim()) {+ firstDeltaMs = performance.now() - t0;+ onFirstDelta();+ }+ });+ return { body, roundMs: performance.now() - t0, firstDeltaMs };+ } catch (err) {+ tally.retries++;+ tally.retryMs += performance.now() - t0;+ if (delivered || attempt >= TRANSIENT_RETRIES) throw err;+ await sleepTallied(800 * (attempt + 1), tally);+ continue;+ }+ }+ const failMs = performance.now() - t0;+ if (!RETRYABLE_STATUS.has(res.status) || attempt >= TRANSIENT_RETRIES)+ throw new Error(`responses ${res.status}: ${(await res.text()).slice(0, 300)}`);+ tally.retries++;+ tally.retryMs += failMs;+ await sleepTallied(800 * (attempt + 1), tally);+ }+}++export async function runCase(c: EvalCase, cfg: RunConfig): Promise<CaseResult> {+ const turnStart = performance.now();+ const tally: RetryTally = { retryMs: 0, retries: 0 };+ const input = c.input();+ // The gate runs first, exactly as in production: a "chat" verdict strips+ // the tool declarations from the whole turn. (Production overlaps the gate+ // with input assembly, which is negligible — serial gate + round 1 is the+ // same critical path the user perceives.)+ const gate = await classifyIntent(input, cfg, tally);+ const tools = gate.intent === "work" ? toolDeclarations : undefined;+ const sim = c.simulate?.();+ const trace: TraceEntry[] = [];+ const rounds: RoundTiming[] = [];+ const violations: string[] = [];+ const notes: string[] = [];+ let reply = "";+ let emptyRounds = 0;+ let ttftMs: number | null = null;+ let modelMs = 0;+ let toolMs = 0;++ for (let round = 0; round < MAX_ROUNDS; round++) {+ const { body, roundMs, firstDeltaMs } = await postRoundTimed(+ cfg,+ {+ donkeyProvider: "gemini",+ model: cfg.chatModel,+ instructions: systemPrompt(),+ input,+ ...(tools ? { tools } : {}),+ },+ tally,+ () => {+ if (ttftMs === null) ttftMs = performance.now() - turnStart;+ }+ );+ modelMs += roundMs;+ const roundTiming: RoundTiming = { roundMs, firstDeltaMs, calls: [] };+ rounds.push(roundTiming);++ const text = (body.output_text ?? "").trim();+ if (text) reply = reply ? `${reply}\n${text}` : text;++ const calls = (body.output ?? []).filter((o) => o.type === "function_call");+ if (calls.length === 0) {+ // A no-call round with nothing said all turn is the degenerate empty+ // STOP; the production loop (geminiChat.ts) re-asks the same input a+ // couple of times, so the eval does too rather than scoring it silent.+ if (!reply && emptyRounds < 2) {+ emptyRounds++;+ continue;+ }+ break;+ }+ emptyRounds = 0;++ const assistantParts: Item[] = text ? [{ text }] : [];+ const responseParts: Item[] = [];+ for (const call of calls) {+ const name = String(call.name ?? "unknown");+ const id = call.id ? String(call.id) : `call-${trace.length + 1}`;+ const part: Item = {+ functionCall: {+ id,+ name,+ args: call.arguments && typeof call.arguments === "object" ? call.arguments : {},+ },+ };+ if (call.thoughtSignature) part.thoughtSignature = call.thoughtSignature;+ assistantParts.push(part);++ let response: unknown;+ const argsObj =+ call.arguments && typeof call.arguments === "object"+ ? (call.arguments as Record<string, unknown>)+ : {};+ const atMs = performance.now() - turnStart;+ const e0 = performance.now();+ const simulated = sim?.(name, argsObj);+ if (name === "read_skill") {+ const doc = AI_SKILLS[String(argsObj.name ?? "")];+ response = doc ? { doc } : { error: "No such skill." };+ } else if (simulated !== undefined) {+ response = simulated;+ } else if (c.stubs && name in c.stubs) {+ response = c.stubs[name];+ } else if (SAFE_TOOLS.has(name)) {+ response = serveSafeTool(name, c.state ?? EDITOR_STATE);+ } else {+ violations.push(name);+ response = { error: "eval: this tool is disabled for this turn" };+ }+ const execMs = performance.now() - e0;+ toolMs += execMs;+ trace.push({ name, round, atMs, execMs });+ roundTiming.calls.push({ name, execMs });+ responseParts.push({+ type: "function_response",+ id,+ name,+ response:+ response && typeof response === "object" && !Array.isArray(response)+ ? response+ : { result: response ?? null },+ });+ }+ input.push({ role: "assistant", content: assistantParts });+ input.push({ role: "user", content: responseParts });+ }++ if (c.gate && gate.intent !== c.gate) notes.push(`gate said ${gate.intent}, expected ${c.gate}`);+ if (!c.reply.test(reply)) notes.push(`reply did not match ${c.reply}`);+ for (const t of c.requiredTools ?? []) {+ if (!trace.some((e) => e.name === t)) notes.push(`required tool ${t} was never called`);+ }+ if (c.anyTools && !c.anyTools.some((t) => trace.some((e) => e.name === t)))+ notes.push(`none of [${c.anyTools.join(", ")}] were called`);+ if (c.maxToolCalls !== undefined && trace.length > c.maxToolCalls)+ notes.push(`slow: ${trace.length} tool calls (cap ${c.maxToolCalls})`);+ if (violations.length > 0) notes.push(`mutating tools called: ${violations.join(", ")}`);++ const timings: RunTimings = {+ gateMs: gate.gateMs,+ gateSkipped: gate.gateSkipped,+ ttftMs,+ modelMs,+ toolMs,+ totalMs: gate.gateMs + modelMs + toolMs,+ retryMs: tally.retryMs,+ retries: tally.retries,+ rounds,+ };+ return {+ pass: notes.length === 0,+ reply,+ trace,+ violations,+ notes,+ intent: gate.intent,+ timings,+ };+}diff --git a/site/scripts/lib/cut-eval/report.ts b/site/scripts/lib/cut-eval/report.tsnew file mode 100644index 00000000..475006a8--- /dev/null+++ b/site/scripts/lib/cut-eval/report.ts@@ -0,0 +1,167 @@+/**+ * The latency eval's report: schema cut-chat-latency/v1. One report holds one+ * or more model configs; each config holds per-case runs with full timings and+ * per-bucket aggregates. Latency aggregates count passing runs only — a wrong+ * answer's speed means nothing — with pass rate reported alongside so a+ * fast-but-wrong config stays visible.+ */++import type { Bucket, EvalCase } from "./cases";+import type { CaseResult, RunTimings, TraceEntry } from "./harness";++export interface Agg {+ p50: number;+ p95: number;+ mean: number;+ n: number;+}++export interface RunReport {+ pass: boolean;+ intent: string;+ notes: string[];+ trace: TraceEntry[];+ timings: RunTimings | null;+ /** Set when the run threw (dev server down, non-retryable upstream error). */+ error?: string;+}++export interface CaseLatency {+ gateMs: Agg;+ ttftMs: Agg | null;+ totalMs: Agg;+ modelMs: Agg;+ toolMs: Agg;+ rounds: { mean: number; max: number };+}++export interface CaseReport {+ name: string;+ bucket: Bucket;+ passRate: number;+ budgetBreaches: string[];+ /** Passing runs only; null when no run passed. */+ latency: CaseLatency | null;+ runs: RunReport[];+}++export interface BucketSummary {+ cases: number;+ passRate: number;+ gateMs: Agg | null;+ ttftMs: Agg | null;+ totalMs: Agg | null;+ meanRounds: number | null;+}++export interface ConfigReport {+ label: string;+ chatModel: string;+ gateModel: string;+ buckets: Partial<Record<Bucket, BucketSummary>>;+ cases: CaseReport[];+}++export interface LatencyReport {+ schema: "cut-chat-latency/v1";+ generatedAt: string;+ base: string;+ runsPerCase: number;+ configs: ConfigReport[];+}++const r0 = (x: number) => Math.round(x);++const percentile = (sorted: number[], p: number): number => {+ if (sorted.length === 0) return 0;+ const idx = Math.min(sorted.length - 1, Math.ceil((p / 100) * sorted.length) - 1);+ return sorted[Math.max(0, idx)];+};++export function agg(values: number[]): Agg | null {+ if (values.length === 0) return null;+ const sorted = [...values].sort((a, b) => a - b);+ return {+ p50: r0(percentile(sorted, 50)),+ p95: r0(percentile(sorted, 95)),+ mean: r0(values.reduce((s, v) => s + v, 0) / values.length),+ n: values.length,+ };+}++export function toRunReport(result: CaseResult | null, error?: string): RunReport {+ if (!result) return { pass: false, intent: "work", notes: [], trace: [], timings: null, error };+ return {+ pass: result.pass,+ intent: result.intent,+ notes: result.notes,+ trace: result.trace,+ timings: result.timings,+ };+}++export function buildCaseReport(c: EvalCase, runs: RunReport[]): CaseReport {+ const passing = runs.filter((r) => r.pass && r.timings).map((r) => r.timings as RunTimings);+ const passRate = runs.length > 0 ? passing.length / runs.length : 0;+ let latency: CaseLatency | null = null;+ if (passing.length > 0) {+ const ttfts = passing.map((t) => t.ttftMs).filter((v): v is number => v !== null);+ latency = {+ gateMs: agg(passing.map((t) => t.gateMs)) as Agg,+ ttftMs: agg(ttfts),+ totalMs: agg(passing.map((t) => t.totalMs)) as Agg,+ modelMs: agg(passing.map((t) => t.modelMs)) as Agg,+ toolMs: agg(passing.map((t) => t.toolMs)) as Agg,+ rounds: {+ mean: Math.round((passing.reduce((s, t) => s + t.rounds.length, 0) / passing.length) * 10) / 10,+ max: Math.max(...passing.map((t) => t.rounds.length)),+ },+ };+ }+ const budgetBreaches: string[] = [];+ if (c.budget && latency) {+ const checks: [string, number | undefined, Agg | null][] = [+ ["totalMs", c.budget.totalMs, latency.totalMs],+ ["ttftMs", c.budget.ttftMs, latency.ttftMs],+ ["gateMs", c.budget.gateMs, latency.gateMs],+ ];+ for (const [key, budget, a] of checks) {+ if (budget !== undefined && a && a.p50 > budget)+ budgetBreaches.push(`${key} p50 ${a.p50} > budget ${budget}`);+ }+ }+ return { name: c.name, bucket: c.bucket, passRate, budgetBreaches, latency, runs };+}++export function buildBucketSummaries(+ caseReports: CaseReport[]+): Partial<Record<Bucket, BucketSummary>> {+ const buckets: Partial<Record<Bucket, BucketSummary>> = {};+ for (const bucket of new Set(caseReports.map((c) => c.bucket))) {+ const cs = caseReports.filter((c) => c.bucket === bucket);+ const withLatency = cs.filter((c) => c.latency !== null);+ const flat = <K extends "gateMs" | "totalMs" | "modelMs">(key: K) =>+ agg(withLatency.map((c) => (c.latency as CaseLatency)[key].p50));+ buckets[bucket] = {+ cases: cs.length,+ passRate:+ Math.round((cs.reduce((s, c) => s + c.passRate, 0) / cs.length) * 100) / 100,+ gateMs: flat("gateMs"),+ ttftMs: agg(+ withLatency+ .map((c) => c.latency?.ttftMs?.p50)+ .filter((v): v is number => typeof v === "number")+ ),+ totalMs: flat("totalMs"),+ meanRounds:+ withLatency.length > 0+ ? Math.round(+ (withLatency.reduce((s, c) => s + (c.latency as CaseLatency).rounds.mean, 0) /+ withLatency.length) *+ 10+ ) / 10+ : null,+ };+ }+ return buckets;+}