Skip to content

Open source CapCut alternative.

TypeScript

refactor(site): one entrypoint for the Cut chat eval

David Le3cecbd4ba5ab
diff --git a/evals/cut-chat-latency.latest-report.json b/evals/cut-chat.latest-report.json
similarity index 99%
rename from evals/cut-chat-latency.latest-report.json
rename to evals/cut-chat.latest-report.json
index e70e56e1..2a7aa21c 100644
--- a/evals/cut-chat-latency.latest-report.json
+++ b/evals/cut-chat.latest-report.json
@@ -1,5 +1,5 @@
{
- "schema": "cut-chat-latency/v1",
+ "schema": "cut-chat-eval/v1",
"generatedAt": "2026-08-04T12:45:08.327Z",
"base": "http://localhost:3000",
"runsPerCase": 3,
diff --git a/site/package.json b/site/package.json
index f0314714..f2dbfdfe 100644
--- a/site/package.json
+++ b/site/package.json
@@ -21,7 +21,6 @@
"engine:build": "bash scripts/build-cut-engine.sh",
"worker:build": "bun build --target=node --minify --external=pg-native src/cut/worker/main.ts --outfile dist/cut-worker/main.js",
"eval:cut-chat": "bun run scripts/eval-cut-chat.ts",
- "eval:cut-latency": "bun run scripts/eval-cut-latency.ts",
"email:dev": "email dev --dir src/emails --port 3001",
"test": "bun test src packages/effects-kit/src",
"test:genvideo": "bun run scripts/genvideo-selftest.ts",
diff --git a/site/scripts/eval-cut-chat.ts b/site/scripts/eval-cut-chat.ts
index 676b2ca5..7e628b1a 100644
--- a/site/scripts/eval-cut-chat.ts
+++ b/site/scripts/eval-cut-chat.ts
@@ -1,27 +1,58 @@
#!/usr/bin/env bun
/**
- * Behavior evals for the Cut assistant chat.
+ * The Cut chat eval: behavior and speed in one run.
*
* Each case replays a real composer turn (system prompt, tool catalog,
* <attached_assets>, <editor_state>, inline audio) against the live chat
* model through the hosted Responses route and asserts on what the model
- * does: questions get answered in chat without project-mutating tool calls,
- * edit requests still reach for tools.
+ * does — questions answer in chat without project-mutating tool calls, edit
+ * requests reach the right tools — while timing the whole path: the gate,
+ * time-to-first-token, every round, every tool serve.
*
* Run with the site dev server up:
- * bun run scripts/eval-cut-chat.ts [--base http://localhost:3000] [--only <case>] [--runs N]
+ * bun run scripts/eval-cut-chat.ts
+ * [--base http://localhost:3000]
+ * [--only <case>] [--bucket chat|single-tool|multi-tool]
+ * [--runs N] default 1; use 3+ for latency stats
+ * [--model <registryKey|rawId>] chat model (flash, flashLite, or a raw id)
+ * [--gate-model <registryKey|rawId>]
+ * [--matrix] run every CANDIDATES row, compare
+ * [--enforce-budgets] exit 1 when a case breaches its budget
+ * [--out <path>] report path; a full run defaults to
+ * <repo>/evals/cut-chat.latest-report.json,
+ * a narrowed run writes only with --out
*
* Auth is the dev bypass header (scripts only — never the app), so runs are
* dev-server-only and spend no credits. The spoken fixture is synthesized
* locally with macOS `say`, so the transcript assertion is deterministic.
- *
- * The cases, fixtures, and runner live in scripts/lib/cut-eval/ and are shared
- * with the latency eval (eval-cut-latency.ts).
+ * See scripts/lib/cut-eval/README.md for the improvement loop.
*/
-import { cases } from "./lib/cut-eval/cases";
+import { mkdirSync, writeFileSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { geminiModelRoles, geminiModels } from "../src/lib/inference/gemini-models";
+import { cases, type Bucket, type EvalCase } from "./lib/cut-eval/cases";
import { makeFixtureAudio } from "./lib/cut-eval/fixtures";
-import { defaultRunConfig, runCase } from "./lib/cut-eval/harness";
+import { runCase, type RunConfig } from "./lib/cut-eval/harness";
+import {
+ buildBucketSummaries,
+ buildCaseReport,
+ toRunReport,
+ type CaseReport,
+ type ConfigReport,
+ type LatencyReport,
+ type RunReport,
+} from "./lib/cut-eval/report";
+
+// Model configs the --matrix mode compares. Add a row when trialing a new
+// model id; keep "baseline" matching geminiModelRoles so the comparison always
+// includes what production runs.
+const CANDIDATES: { label: string; chat: string; gate: string }[] = [
+ { label: "baseline", chat: geminiModelRoles.chat, gate: geminiModelRoles.fastDecision },
+ { label: "chat=flashLite", chat: geminiModels.flashLite, gate: geminiModelRoles.fastDecision },
+ { label: "gate=flash", chat: geminiModelRoles.chat, gate: geminiModels.flash },
+];
const args = process.argv.slice(2);
const argValue = (flag: string) => {
@@ -30,46 +61,149 @@ const argValue = (flag: string) => {
};
const BASE = argValue("--base") ?? "http://localhost:3000";
const ONLY = argValue("--only");
+const BUCKET = argValue("--bucket") as Bucket | undefined;
const RUNS = Number(argValue("--runs") ?? 1);
+const MATRIX = args.includes("--matrix");
+const ENFORCE_BUDGETS = args.includes("--enforce-budgets");
+const OUT_ARG = argValue("--out");
+const DEFAULT_OUT = join(
+ dirname(fileURLToPath(import.meta.url)),
+ "../../evals/cut-chat.latest-report.json"
+);
+// A narrowed run would overwrite the committed baseline with a partial
+// report, so it only writes when --out names a destination.
+const OUT = OUT_ARG ? resolve(OUT_ARG) : ONLY || BUCKET ? null : resolve(DEFAULT_OUT);
-async function main() {
- const audio = makeFixtureAudio();
- const all = cases(audio).filter((c) => !ONLY || c.name === ONLY);
- if (all.length === 0) throw new Error(`No case named "${ONLY}".`);
- const cfg = defaultRunConfig(BASE);
+/** A registry key (flash, flashLite, …) or a raw model id. */
+const resolveModel = (v: string): string =>
+ (geminiModels as Record<string, string>)[v] ?? v;
+
+const ms = (v: number | null | undefined) => (typeof v === "number" ? `${v}ms` : "—");
+const pct = (v: number) => `${Math.round(v * 100)}%`;
- let failed = 0;
- for (const c of all) {
+function printTable(header: string[], rows: string[][]) {
+ const all = [header, ...rows];
+ const widths = header.map((_, i) => Math.max(...all.map((r) => r[i].length)));
+ for (const [n, row] of all.entries()) {
+ console.log(row.map((cell, i) => cell.padEnd(widths[i])).join(" "));
+ if (n === 0) console.log(widths.map((w) => "-".repeat(w)).join(" "));
+ }
+}
+
+async function runConfig(
+ label: string,
+ cfg: RunConfig,
+ selected: EvalCase[]
+): Promise<ConfigReport> {
+ console.log(`\n== ${label} chat=${cfg.chatModel} gate=${cfg.gateModel}`);
+ const caseReports: CaseReport[] = [];
+ for (const c of selected) {
+ const runs: RunReport[] = [];
for (let run = 1; run <= RUNS; run++) {
- const label = RUNS > 1 ? `${c.name} [${run}/${RUNS}]` : c.name;
try {
- const r = await runCase(c, cfg);
- const names = r.trace.map((t) => t.name);
- const tools =
- names.length > 0
- ? ` tools: ${names.join(" → ")}`
- : r.intent === "chat"
- ? " tools: none (gated)"
- : " tools: none";
- if (r.pass) {
- console.log(`PASS ${label}${tools}`);
- } else {
- failed++;
- console.log(`FAIL ${label}${tools}`);
- for (const n of r.notes) console.log(` - ${n}`);
- console.log(` reply: ${r.reply.slice(0, 200) || "(empty)"}`);
- }
+ runs.push(toRunReport(await runCase(c, cfg)));
} catch (err) {
- failed++;
- console.log(`FAIL ${label} — ${err instanceof Error ? err.message : err}`);
+ runs.push(toRunReport(null, err instanceof Error ? err.message : String(err)));
}
}
+ const report = buildCaseReport(c, runs);
+ caseReports.push(report);
+ const l = report.latency;
+ console.log(
+ ` ${report.passRate === 1 ? "ok " : report.passRate > 0 ? "part" : "FAIL"} ${c.name}` +
+ ` pass ${pct(report.passRate)} total p50 ${ms(l?.totalMs.p50)} ttft p50 ${ms(l?.ttftMs?.p50)}` +
+ ` rounds ${l ? l.rounds.mean : "—"}` +
+ (report.budgetBreaches.length > 0 ? ` BUDGET: ${report.budgetBreaches.join("; ")}` : "")
+ );
+ for (const r of runs.filter((r) => !r.pass)) {
+ for (const n of r.error ? [r.error] : r.notes) console.log(` - ${n}`);
+ if (r.trace.length > 0)
+ console.log(` tools: ${r.trace.map((t) => t.name).join(" → ")}`);
+ }
+ }
+ return {
+ label,
+ chatModel: cfg.chatModel,
+ gateModel: cfg.gateModel,
+ buckets: buildBucketSummaries(caseReports),
+ cases: caseReports,
+ };
+}
+
+async function main() {
+ const audio = makeFixtureAudio();
+ const selected = cases(audio).filter(
+ (c) => (!ONLY || c.name === ONLY) && (!BUCKET || c.bucket === BUCKET)
+ );
+ if (selected.length === 0)
+ throw new Error(ONLY ? `No case named "${ONLY}".` : `No cases in bucket "${BUCKET}".`);
+
+ const configs = MATRIX
+ ? CANDIDATES.map((cand) => ({
+ label: cand.label,
+ cfg: { base: BASE, chatModel: cand.chat, gateModel: cand.gate },
+ }))
+ : [
+ {
+ label: "config",
+ cfg: {
+ base: BASE,
+ chatModel: resolveModel(argValue("--model") ?? geminiModelRoles.chat),
+ gateModel: resolveModel(argValue("--gate-model") ?? geminiModelRoles.fastDecision),
+ },
+ },
+ ];
+
+ const report: LatencyReport = {
+ schema: "cut-chat-eval/v1",
+ generatedAt: new Date().toISOString(),
+ base: BASE,
+ runsPerCase: RUNS,
+ configs: [],
+ };
+ for (const { label, cfg } of configs) {
+ report.configs.push(await runConfig(label, cfg, selected));
+ }
+
+ console.log("");
+ printTable(
+ ["config", "bucket", "cases", "pass", "gate p50", "ttft p50", "total p50", "rounds"],
+ report.configs.flatMap((c) =>
+ (Object.entries(c.buckets) as [Bucket, NonNullable<ConfigReport["buckets"][Bucket]>][]).map(
+ ([bucket, s]) => [
+ c.label,
+ bucket,
+ String(s.cases),
+ pct(s.passRate),
+ ms(s.gateMs?.p50),
+ ms(s.ttftMs?.p50),
+ ms(s.totalMs?.p50),
+ s.meanRounds === null ? "—" : String(s.meanRounds),
+ ]
+ )
+ )
+ );
+
+ if (OUT) {
+ mkdirSync(dirname(OUT), { recursive: true });
+ writeFileSync(OUT, `${JSON.stringify(report, null, 2)}\n`);
+ console.log(`\nReport: ${OUT}`);
+ }
+
+ const dead = report.configs.flatMap((c) =>
+ c.cases.filter((x) => x.passRate === 0).map((x) => `${c.label}/${x.name}`)
+ );
+ const breaches = report.configs.flatMap((c) =>
+ c.cases.filter((x) => x.budgetBreaches.length > 0).map((x) => `${c.label}/${x.name}`)
+ );
+ if (dead.length > 0) {
+ console.log(`\nNo passing runs: ${dead.join(", ")}`);
+ process.exit(1);
}
- if (failed > 0) {
- console.log(`\n${failed} failure(s).`);
+ if (ENFORCE_BUDGETS && breaches.length > 0) {
+ console.log(`\nBudget breaches: ${breaches.join(", ")}`);
process.exit(1);
}
- console.log("\nAll cases passed.");
}
void main();
diff --git a/site/scripts/eval-cut-latency.ts b/site/scripts/eval-cut-latency.ts
deleted file mode 100644
index 8faa9a6d..00000000
--- a/site/scripts/eval-cut-latency.ts
+++ /dev/null
@@ -1,195 +0,0 @@
-#!/usr/bin/env bun
-/**
- * Latency evals for the Cut assistant chat: how fast the model reaches the
- * right tools, and how fast plain conversation turns complete.
- *
- * Runs the shared cases (scripts/lib/cut-eval/) with full timing — gate,
- * time-to-first-token, per-round wall, tool serve — and writes a
- * machine-readable report (schema cut-chat-latency/v1) beside a human table.
- * Latency aggregates count passing runs only; pass rate rides alongside.
- *
- * Run with the site dev server up:
- * bun run scripts/eval-cut-latency.ts
- * [--base http://localhost:3000]
- * [--only <case>] [--bucket chat|single-tool|multi-tool]
- * [--runs N] default 3
- * [--model <registryKey|rawId>] chat model (flash, flashLite, or a raw id)
- * [--gate-model <registryKey|rawId>]
- * [--matrix] run every CANDIDATES row, compare
- * [--enforce-budgets] exit 1 when a case breaches its budget
- * [--out <path>] default <repo>/evals/cut-chat-latency.latest-report.json
- *
- * See scripts/lib/cut-eval/README.md for the improvement loop.
- */
-
-import { mkdirSync, writeFileSync } from "node:fs";
-import { dirname, join, resolve } from "node:path";
-import { fileURLToPath } from "node:url";
-import { geminiModelRoles, geminiModels } from "../src/lib/inference/gemini-models";
-import { cases, type Bucket, type EvalCase } from "./lib/cut-eval/cases";
-import { makeFixtureAudio } from "./lib/cut-eval/fixtures";
-import { runCase, type RunConfig } from "./lib/cut-eval/harness";
-import {
- buildBucketSummaries,
- buildCaseReport,
- toRunReport,
- type CaseReport,
- type ConfigReport,
- type LatencyReport,
- type RunReport,
-} from "./lib/cut-eval/report";
-
-// Model configs the --matrix mode compares. Add a row when trialing a new
-// model id; keep "baseline" matching geminiModelRoles so the comparison always
-// includes what production runs.
-const CANDIDATES: { label: string; chat: string; gate: string }[] = [
- { label: "baseline", chat: geminiModelRoles.chat, gate: geminiModelRoles.fastDecision },
- { label: "chat=flashLite", chat: geminiModels.flashLite, gate: geminiModelRoles.fastDecision },
- { label: "gate=flash", chat: geminiModelRoles.chat, gate: geminiModels.flash },
-];
-
-const args = process.argv.slice(2);
-const argValue = (flag: string) => {
- const i = args.indexOf(flag);
- return i >= 0 ? args[i + 1] : undefined;
-};
-const BASE = argValue("--base") ?? "http://localhost:3000";
-const ONLY = argValue("--only");
-const BUCKET = argValue("--bucket") as Bucket | undefined;
-const RUNS = Number(argValue("--runs") ?? 3);
-const MATRIX = args.includes("--matrix");
-const ENFORCE_BUDGETS = args.includes("--enforce-budgets");
-const OUT = resolve(
- argValue("--out") ??
- join(dirname(fileURLToPath(import.meta.url)), "../../evals/cut-chat-latency.latest-report.json")
-);
-
-/** A registry key (flash, flashLite, …) or a raw model id. */
-const resolveModel = (v: string): string =>
- (geminiModels as Record<string, string>)[v] ?? v;
-
-const ms = (v: number | null | undefined) => (typeof v === "number" ? `${v}ms` : "—");
-const pct = (v: number) => `${Math.round(v * 100)}%`;
-
-function printTable(header: string[], rows: string[][]) {
- const all = [header, ...rows];
- const widths = header.map((_, i) => Math.max(...all.map((r) => r[i].length)));
- for (const [n, row] of all.entries()) {
- console.log(row.map((cell, i) => cell.padEnd(widths[i])).join(" "));
- if (n === 0) console.log(widths.map((w) => "-".repeat(w)).join(" "));
- }
-}
-
-async function runConfig(
- label: string,
- cfg: RunConfig,
- selected: EvalCase[]
-): Promise<ConfigReport> {
- console.log(`\n== ${label} chat=${cfg.chatModel} gate=${cfg.gateModel}`);
- const caseReports: CaseReport[] = [];
- for (const c of selected) {
- const runs: RunReport[] = [];
- for (let run = 1; run <= RUNS; run++) {
- try {
- runs.push(toRunReport(await runCase(c, cfg)));
- } catch (err) {
- runs.push(toRunReport(null, err instanceof Error ? err.message : String(err)));
- }
- }
- const report = buildCaseReport(c, runs);
- caseReports.push(report);
- const l = report.latency;
- console.log(
- ` ${report.passRate === 1 ? "ok " : report.passRate > 0 ? "part" : "FAIL"} ${c.name}` +
- ` pass ${pct(report.passRate)} total p50 ${ms(l?.totalMs.p50)} ttft p50 ${ms(l?.ttftMs?.p50)}` +
- ` rounds ${l ? l.rounds.mean : "—"}` +
- (report.budgetBreaches.length > 0 ? ` BUDGET: ${report.budgetBreaches.join("; ")}` : "")
- );
- for (const r of runs.filter((r) => !r.pass)) {
- for (const n of r.error ? [r.error] : r.notes) console.log(` - ${n}`);
- }
- }
- return {
- label,
- chatModel: cfg.chatModel,
- gateModel: cfg.gateModel,
- buckets: buildBucketSummaries(caseReports),
- cases: caseReports,
- };
-}
-
-async function main() {
- const audio = makeFixtureAudio();
- const selected = cases(audio).filter(
- (c) => (!ONLY || c.name === ONLY) && (!BUCKET || c.bucket === BUCKET)
- );
- if (selected.length === 0)
- throw new Error(ONLY ? `No case named "${ONLY}".` : `No cases in bucket "${BUCKET}".`);
-
- const configs = MATRIX
- ? CANDIDATES.map((cand) => ({
- label: cand.label,
- cfg: { base: BASE, chatModel: cand.chat, gateModel: cand.gate },
- }))
- : [
- {
- label: "config",
- cfg: {
- base: BASE,
- chatModel: resolveModel(argValue("--model") ?? geminiModelRoles.chat),
- gateModel: resolveModel(argValue("--gate-model") ?? geminiModelRoles.fastDecision),
- },
- },
- ];
-
- const report: LatencyReport = {
- schema: "cut-chat-latency/v1",
- generatedAt: new Date().toISOString(),
- base: BASE,
- runsPerCase: RUNS,
- configs: [],
- };
- for (const { label, cfg } of configs) {
- report.configs.push(await runConfig(label, cfg, selected));
- }
-
- console.log("");
- printTable(
- ["config", "bucket", "cases", "pass", "gate p50", "ttft p50", "total p50", "rounds"],
- report.configs.flatMap((c) =>
- (Object.entries(c.buckets) as [Bucket, NonNullable<ConfigReport["buckets"][Bucket]>][]).map(
- ([bucket, s]) => [
- c.label,
- bucket,
- String(s.cases),
- pct(s.passRate),
- ms(s.gateMs?.p50),
- ms(s.ttftMs?.p50),
- ms(s.totalMs?.p50),
- s.meanRounds === null ? "—" : String(s.meanRounds),
- ]
- )
- )
- );
-
- mkdirSync(dirname(OUT), { recursive: true });
- writeFileSync(OUT, `${JSON.stringify(report, null, 2)}\n`);
- console.log(`\nReport: ${OUT}`);
-
- const dead = report.configs.flatMap((c) =>
- c.cases.filter((x) => x.passRate === 0).map((x) => `${c.label}/${x.name}`)
- );
- const breaches = report.configs.flatMap((c) =>
- c.cases.filter((x) => x.budgetBreaches.length > 0).map((x) => `${c.label}/${x.name}`)
- );
- if (dead.length > 0) {
- console.log(`\nNo passing runs: ${dead.join(", ")}`);
- process.exit(1);
- }
- if (ENFORCE_BUDGETS && breaches.length > 0) {
- console.log(`\nBudget breaches: ${breaches.join(", ")}`);
- process.exit(1);
- }
-}
-
-void main();
diff --git a/site/scripts/lib/cut-eval/README.md b/site/scripts/lib/cut-eval/README.md
index c1b64ebb..a4848b2c 100644
--- a/site/scripts/lib/cut-eval/README.md
+++ b/site/scripts/lib/cut-eval/README.md
@@ -1,36 +1,33 @@
-# Cut chat evals
+# Cut chat eval
-Two entrypoints share the fixtures, cases, and runner in this folder:
+One command measures the chat assistant's behavior and speed together:
-- `npm run eval:cut-chat` — behavior: does the model do the right thing?
-- `npm run eval:cut-latency` — speed: how fast does it get there?
+```
+npm run eval:cut-chat -- [--runs N] [--only <case>] [--bucket chat|single-tool|multi-tool]
+ [--model flash|flashLite|<raw id>] [--gate-model ...]
+ [--matrix] [--enforce-budgets] [--out <path>]
+```
-Both replay real composer turns against the live chat model through the dev
+It replays real composer turns against the live chat model through the dev
server's hosted Responses route. Start `next dev` on :3000 first. Auth is the
dev bypass header, so runs spend no credits. The spoken fixture is synthesized
with macOS `say`, so runs need a Mac.
-## Latency eval
-
-```
-npm run eval:cut-latency -- [--runs 3] [--only <case>] [--bucket chat|single-tool|multi-tool]
- [--model flash|flashLite|<raw id>] [--gate-model ...]
- [--matrix] [--enforce-budgets] [--out <path>]
-```
+Every case asserts behavior (the reply, the tools called, the gate verdict,
+forbidden tools) and records timing: the gate's classify time, time-to-first-
+token (rounds stream, as production does), per-round wall time, tool-serve
+time, and round count. The default single run per case is the quick regression
+check; `--runs 3` gives real p50/p95s. Latency aggregates count passing runs
+only — a wrong answer's speed means nothing — and pass rate is reported
+alongside so a fast-but-wrong config stays visible.
Every case carries a bucket: `chat` turns (greetings, questions — the gated
fast path), `single-tool` edits (one decisive call), and `multi-tool` edits
-(composed cuts). Each run records the gate's classify time, time-to-first-token
-(rounds stream, as production does), per-round wall time, tool-serve time, and
-round count. Latency aggregates (p50/p95/mean) count passing runs only — a
-wrong answer's speed means nothing — and pass rate is reported alongside so a
-fast-but-wrong config stays visible.
-
-The eval runs the gate before round 1. Production overlaps it with input
-assembly, which is negligible, so this is the same critical path the user
-perceives.
+(composed cuts). The eval runs the gate before round 1; production overlaps it
+with input assembly, which is negligible, so this is the same critical path
+the user perceives.
-`--matrix` runs every row of `CANDIDATES` in `eval-cut-latency.ts` and prints a
+`--matrix` runs every row of `CANDIDATES` in `eval-cut-chat.ts` and prints a
comparison table. Model flags and candidate rows resolve through the registry
in `src/lib/inference/gemini-models.ts`; raw model ids pass through.
@@ -40,20 +37,22 @@ always exits 1 when a case has zero passing runs.
## The report
-Each run writes `evals/cut-chat-latency.latest-report.json` (schema
-`cut-chat-latency/v1`, types in `report.ts`): per config → per bucket and per
+A full run writes `evals/cut-chat.latest-report.json` (schema
+`cut-chat-eval/v1`, types in `report.ts`): per config → per bucket and per
case → pass rate, latency aggregates, budget breaches, and every run's full
-timings and tool trace. The committed copy is the baseline; `git diff` on it
-shows what a change did to speed and correctness.
+timings and tool trace. A run narrowed by `--only` or `--bucket` writes only
+when `--out` names a destination, so iterating never clobbers the baseline.
+The committed copy is the baseline; `git diff` on it shows what a change did
+to speed and correctness.
## Improving latency (the loop)
1. Make one change: bump a role in `src/lib/inference/gemini-models.ts`, edit
the system prompt or a tool description in the catalog, or add a
`CANDIDATES` row for a new model id.
-2. Run `npm run eval:cut-latency -- --matrix --runs 3` (narrow with `--bucket`
+2. Run `npm run eval:cut-chat -- --matrix --runs 3` (narrow with `--bucket`
or `--only` while iterating).
-3. Compare against the baseline: `git diff evals/cut-chat-latency.latest-report.json`.
+3. Compare against the baseline: `git diff evals/cut-chat.latest-report.json`.
4. Keep the change only if pass rate holds and the p50s improve.
5. Commit the new report with the change, so the baseline tracks what ships.
@@ -65,9 +64,3 @@ conversation and all ~75 tool declarations.
Adding a case: append to `cases.ts` with a `bucket`, stub any mutating tool it
should call, and use `simulate` to assert on arguments. Run it alone with
`--only <name>` and `--runs 5` to check it's stable before relying on it.
-
-## Behavior eval
-
-`npm run eval:cut-chat -- [--only <case>] [--runs N]` prints PASS/FAIL per case
-and exits 1 on any failure. Same cases, no timing, no report file — use it as
-the quick regression check while the latency eval is the measurement tool.
diff --git a/site/scripts/lib/cut-eval/report.ts b/site/scripts/lib/cut-eval/report.ts
index 475006a8..c933e9b8 100644
--- a/site/scripts/lib/cut-eval/report.ts
+++ b/site/scripts/lib/cut-eval/report.ts
@@ -1,6 +1,6 @@
/**
- * The latency eval's report: schema cut-chat-latency/v1. One report holds one
- * or more model configs; each config holds per-case runs with full timings and
+ * The eval's report: schema cut-chat-eval/v1. One report holds one or more
+ * model configs; each config holds per-case runs with full timings and
* per-bucket aggregates. Latency aggregates count passing runs only — a wrong
* answer's speed means nothing — with pass rate reported alongside so a
* fast-but-wrong config stays visible.
@@ -63,7 +63,7 @@ export interface ConfigReport {
}
export interface LatencyReport {
- schema: "cut-chat-latency/v1";
+ schema: "cut-chat-eval/v1";
generatedAt: string;
base: string;
runsPerCase: number;