Open source CapCut alternative.
TypeScript
test(site): the chat eval sends model roles like the page does
David Lec67c4b42f3a9
diff --git a/site/scripts/eval-cut-chat.ts b/site/scripts/eval-cut-chat.tsindex 94f55d3e..8f68fb40 100644--- a/site/scripts/eval-cut-chat.ts+++ b/site/scripts/eval-cut-chat.ts@@ -33,7 +33,12 @@import { mkdirSync, writeFileSync } from "node:fs";import { dirname, join, resolve } from "node:path";import { fileURLToPath } from "node:url";-import { geminiModelRoles, geminiModels } from "../src/lib/inference/gemini-models";+import {+ geminiModelRoleNames,+ geminiModelRoles,+ geminiModels,+ resolveGeminiModel,+} from "../src/lib/inference/gemini-models";import { cases, type Bucket, type EvalCase } from "./lib/cut-eval/cases";import { makeFixtureAudio } from "./lib/cut-eval/fixtures";import { runCase, type RunConfig } from "./lib/cut-eval/harness";@@ -93,8 +98,12 @@ const DEFAULT_OUT = join(const OUT = OUT_ARG ? resolve(OUT_ARG) : ONLY || BUCKET ? null : resolve(DEFAULT_OUT);/** A registry key (flash, flashLite, …) or a raw model id. */+// A registry key becomes its id; a role name or a raw id goes out as given —+// the responses route resolves a role, the way it does for the page.const resolveModel = (v: string): string =>(geminiModels as Record<string, string>)[v] ?? v;+// Reports name ids, whatever the config sent.+const idOf = resolveGeminiModel;const ms = (v: number | null | undefined) => (typeof v === "number" ? `${v}ms` : "—");const pct = (v: number) => `${Math.round(v * 100)}%`;@@ -113,11 +122,10 @@ async function runConfig(cfg: RunConfig,selected: EvalCase[]): Promise<ConfigReport> {- const chatDesc =- cfg.simpleModel === cfg.complexModel- ? cfg.simpleModel- : `${cfg.simpleModel} | complex→${cfg.complexModel}`;- console.log(`\n== ${label} chat=${chatDesc} gate=${cfg.gateModel}`);+ const simpleId = idOf(cfg.simpleModel);+ const complexId = idOf(cfg.complexModel);+ const chatDesc = simpleId === complexId ? simpleId : `${simpleId} | complex→${complexId}`;+ console.log(`\n== ${label} chat=${chatDesc} gate=${idOf(cfg.gateModel)}`);const caseReports: CaseReport[] = [];for (const c of selected) {const runs: RunReport[] = [];@@ -147,7 +155,7 @@ async function runConfig(return {label,chatModel: chatDesc,- gateModel: cfg.gateModel,+ gateModel: idOf(cfg.gateModel),buckets: buildBucketSummaries(caseReports),cases: caseReports,};@@ -184,12 +192,14 @@ async function main() {cfg: {base: BASE,simpleModel: resolveModel(- argValue("--simple-model") ?? both ?? geminiModelRoles.chatSimple+ argValue("--simple-model") ?? both ?? geminiModelRoleNames.chatSimple),complexModel: resolveModel(- argValue("--complex-model") ?? both ?? geminiModelRoles.chat+ argValue("--complex-model") ?? both ?? geminiModelRoleNames.chat+ ),+ gateModel: resolveModel(+ argValue("--gate-model") ?? geminiModelRoleNames.fastDecision),- gateModel: resolveModel(argValue("--gate-model") ?? geminiModelRoles.fastDecision),judgeModel,},},diff --git a/site/scripts/lib/cut-eval/harness.ts b/site/scripts/lib/cut-eval/harness.tsindex 6ea1595e..4437ca98 100644--- a/site/scripts/lib/cut-eval/harness.ts+++ b/site/scripts/lib/cut-eval/harness.ts@@ -12,7 +12,11 @@import type { UIMessage } from "ai";import { AI_SKILLS } from "../../../src/cut/server/ai/catalog";-import { geminiModelRoles } from "../../../src/lib/inference/gemini-models";+import {+ geminiModelRoleNames,+ geminiModelRoles,+ resolveGeminiModel,+} from "../../../src/lib/inference/gemini-models";import { streamCutChat, dropPiSession, type CutAgentDeps } from "../../../src/cut/lib/pi/cutAgent";import type { TurnIntent } from "../../../src/cut/lib/turnIntent";import { EDITOR_STATE, SAFE_TOOLS, serveSafeTool } from "./fixtures";@@ -32,12 +36,13 @@ export interface RunConfig {judgeModel: string | null;}-/** Production's config: the three-way gate routes between the two chat roles. */+/** Production's config: the three-way gate routes between the two chat roles,+ * named the way the page names them so the route resolves the ids. */export const defaultRunConfig = (base: string): RunConfig => ({base,- simpleModel: geminiModelRoles.chatSimple,- complexModel: geminiModelRoles.chat,- gateModel: geminiModelRoles.fastDecision,+ simpleModel: geminiModelRoleNames.chatSimple,+ complexModel: geminiModelRoleNames.chat,+ gateModel: geminiModelRoleNames.fastDecision,judgeModel: geminiModelRoles.fastDecision,});@@ -279,7 +284,7 @@ export async function runCase(c: EvalCase, cfg: RunConfig): Promise<CaseResult>notes,judgeNote,intent,- roundModel: intent === "simple" ? cfg.simpleModel : cfg.complexModel,+ roundModel: resolveGeminiModel(intent === "simple" ? cfg.simpleModel : cfg.complexModel),timings,};}