Claude Code. Any Model. The most powerful AI coding agent now speaks every language.
chore(routing): record the proxy decision per row in the route gate
Jack Rudenko56fe2fbcff85
diff --git a/scripts/route-table-snapshot.ts b/scripts/route-table-snapshot.tsindex 1e5315bd..327462f4 100644--- a/scripts/route-table-snapshot.ts+++ b/scripts/route-table-snapshot.ts@@ -28,6 +28,23 @@* mismatch is an error there, not a warning, because a diff across two generations* cannot tell the code change from the catalog change.*+ * Two flags make `--strict` usable for a change meant to move a KNOWN set of things:+ *+ * diff --strict --expect expected.json before.json after.json+ * diff --strict --new-column decision before.json after.json+ *+ * `--expect` names each allowed difference in advance, as a JSON list of+ * `{ "model", "column", "old", "new" }` (a missing `old` or `new` means the column is+ * absent on that side). A difference the list names exactly is allowed; any other+ * difference fails, and so does a listed difference the two snapshots do not show.+ * `--new-column <name>` (repeatable) skips a column the before snapshot does not carry+ * at all, which is how a capture that gained a column is compared with one that+ * predates it. Naming a column the before snapshot does carry is refused.+ *+ * Each row also records `decision`: what the proxy does with the id before `route()`+ * could see it (`proxyRouteDecision`). The row's chain is `route()`'s answer whatever+ * the decision says; the proxy asks `route()` only when the decision is `bare`.+ ** Environment-dependent by design. It calls the real `route()`, so it sees this* machine's credentials and `defaultProvider`. Both snapshots must come from the* same machine with the same configuration, which is why each file records them.@@ -38,11 +55,20 @@ import { homedir } from "node:os";import { join } from "node:path";import { loadConfig } from "../packages/cli/src/profile-config.js";import { readAllModelsCache } from "../packages/cli/src/providers/all-models-cache.js";+import {+ type ProxyRouteDecision,+ proxyRouteDecision,+} from "../packages/cli/src/providers/native-route.js";import { route } from "../packages/cli/src/providers/routing-rules.js";/** One model's resolved chain, flattened to the strings a request would carry. */interface RouteRow {modelId: string;+ /**+ * What the proxy does with the id before routing; only `bare` reaches `route()`.+ * Absent from snapshots captured before this column existed.+ */+ decision?: ProxyRouteDecision["type"];kind: "ok" | "no-route";/** `provider@model` per hop, primary first. Empty for `no-route`. */chain: string[];@@ -81,20 +107,23 @@ async function capture(outPath: string, limit?: number): Promise<void> {const batch = ids.slice(start, start + CONCURRENCY);const settled = await Promise.all(batch.map(async (modelId): Promise<RouteRow> => {+ const decision = proxyRouteDecision(modelId).type;try {const plan = await route(modelId);if (plan.kind === "ok") {return {modelId,+ decision,kind: "ok",chain: [plan.primary, ...plan.fallbacks].map((hop) => hop.modelSpec),};}- return { modelId, kind: "no-route", chain: [], reason: plan.reason };+ return { modelId, decision, kind: "no-route", chain: [], reason: plan.reason };} catch (error) {// A throw is itself a finding: record it rather than losing the row.return {modelId,+ decision,kind: "no-route",chain: [],reason: `threw: ${error instanceof Error ? error.message : String(error)}`,@@ -122,6 +151,14 @@ async function capture(outPath: string, limit?: number): Promise<void> {console.log(`Wrote ${outPath}: ${rows.length} models, ${routed} routed, ${rows.length - routed} with no route, generation ${snapshot.catalogGenerationId}.`);+ const decisions = new Map<string, number>();+ for (const row of rows) {+ const decision = row.decision ?? "(none)";+ decisions.set(decision, (decisions.get(decision) ?? 0) + 1);+ }+ console.log(+ `Proxy decision per row: ${[...decisions].map(([decision, count]) => `${decision} ${count}`).join(", ")}.`+ );}function loadSnapshot(path: string): Snapshot {@@ -307,9 +344,183 @@ function section(title: string, lines: string[], verdict: "must-be-empty" | "revif (lines.length > 40) console.log(` … ${lines.length - 40} more`);}-function diff(beforePath: string, afterPath: string, strict: boolean): void {+/** An invocation or input the gate cannot run with. Printed as an error; exit 2. */+class UsageError extends Error {}++/**+ * One difference the design names in advance: `column` of `model` goes from `old` to+ * `new`. A missing `old` or `new` means the column is absent on that side.+ */+interface ExpectedDifference {+ model: string;+ column: string;+ old?: unknown;+ new?: unknown;+}++const EXPECTATION_KEYS = new Set(["model", "column", "old", "new"]);++function readJson(path: string, flag: string): unknown {+ try {+ return JSON.parse(readFileSync(path, "utf-8"));+ } catch (error) {+ throw new UsageError(`${flag} ${path}: ${error instanceof Error ? error.message : error}`);+ }+}++function toExpectation(item: unknown, where: string): ExpectedDifference {+ if (typeof item !== "object" || item === null || Array.isArray(item)) {+ throw new UsageError(`${where}: not an object`);+ }+ const fields = item as Record<string, unknown>;+ const unknownKeys = Object.keys(fields).filter((key) => !EXPECTATION_KEYS.has(key));+ if (unknownKeys.length > 0)+ throw new UsageError(`${where}: unknown key ${unknownKeys.join(", ")}`);+ if (typeof fields.model !== "string" || typeof fields.column !== "string") {+ throw new UsageError(`${where}: needs a string "model" and a string "column"`);+ }+ if (fields.column === "modelId") throw new UsageError(`${where}: modelId names the row`);+ if (sameValue(fields.old, fields.new)) {+ throw new UsageError(`${where}: "old" equals "new", so it names no difference`);+ }+ return { model: fields.model, column: fields.column, old: fields.old, new: fields.new };+}++/** The `--expect` file, refused whole if any entry is malformed or listed twice. */+function loadExpectations(path: string): ExpectedDifference[] {+ const parsed = readJson(path, "--expect");+ if (!Array.isArray(parsed)) throw new UsageError(`--expect ${path}: not a JSON list`);+ const seen = new Set<string>();+ return parsed.map((item: unknown, index) => {+ const where = `--expect ${path}, entry ${index}`;+ const expectation = toExpectation(item, where);+ const key = JSON.stringify([expectation.model, expectation.column]);+ if (seen.has(key)) {+ throw new UsageError(`${where}: ${expectation.model} ${expectation.column} is listed twice`);+ }+ seen.add(key);+ return expectation;+ });+}++function columnOf(row: RouteRow, column: string): unknown {+ return (row as unknown as Record<string, unknown>)[column];+}++/** A copy of `row` with `column` set to `value`, or without it when `value` is undefined. */+function withColumn(row: RouteRow, column: string, value: unknown): RouteRow {+ const copy: Record<string, unknown> = { ...row };+ if (value === undefined) delete copy[column];+ else copy[column] = value;+ return copy as unknown as RouteRow;+}++/**+ * Take each new column out of `after`, once `before` is shown to carry it on no row.+ * A column `before` does carry is not new, and skipping it would hide a real difference.+ */+function dropNewColumns(+ before: Snapshot,+ after: Snapshot,+ columns: string[]+): { after: Snapshot; notes: string[] } {+ const notes: string[] = [];+ let rows = after.rows;+ for (const column of columns) {+ const carriedBefore = before.rows.filter((row) => Object.hasOwn(row, column)).length;+ if (carriedBefore > 0) {+ throw new UsageError(+ `--new-column ${column}: the before snapshot carries it on ${carriedBefore} of ${before.rows.length} rows, so it is not new. Drop the flag to compare it.`+ );+ }+ const carriedAfter = rows.filter((row) => Object.hasOwn(row, column)).length;+ notes.push(+ `New column "${column}": on no before row and ${carriedAfter} of ${rows.length} after rows; not compared.`+ );+ rows = rows.map((row) => withColumn(row, column, undefined));+ }+ return { after: { ...after, rows }, notes };+}++/**+ * Match each expected difference against the two snapshots. A match is taken out of+ * `after` (its column set back to the before value), so the comparison never sees it.+ * Anything else is unmet: the model is missing, or a side holds another value.+ */+function applyExpectations(+ before: Snapshot,+ after: Snapshot,+ expected: ExpectedDifference[]+): { after: Snapshot; matched: string[]; unmet: string[] } {+ const beforeRows = new Map(before.rows.map((row) => [row.modelId, row]));+ const afterRows = new Map(after.rows.map((row) => [row.modelId, row]));+ const matched: string[] = [];+ const unmet: string[] = [];+ for (const expectation of expected) {+ const { model, column } = expectation;+ const named = `${model}: ${column}: ${showValue(expectation.old)} => ${showValue(expectation.new)}`;+ const beforeRow = beforeRows.get(model);+ const afterRow = afterRows.get(model);+ if (!beforeRow || !afterRow) {+ unmet.push(`${named} (no such model in the ${beforeRow ? "after" : "before"} snapshot)`);+ continue;+ }+ const was = columnOf(beforeRow, column);+ const now = columnOf(afterRow, column);+ if (sameValue(was, expectation.old) && sameValue(now, expectation.new)) {+ matched.push(named);+ afterRows.set(model, withColumn(afterRow, column, was));+ } else {+ unmet.push(`${named} (found ${showValue(was)} => ${showValue(now)})`);+ }+ }+ const rows = after.rows.map((row) => afterRows.get(row.modelId) ?? row);+ return { after: { ...after, rows }, matched, unmet };+}++interface StrictOptions {+ /** A JSON list of expected differences. */+ expectPath?: string;+ /** Columns the before snapshot does not carry; not compared. */+ newColumns: string[];+}++interface StrictAdjustment {+ after: Snapshot;+ notes: string[];+ matched: string[];+ unmet: string[];+}++function adjustForStrict(+ before: Snapshot,+ after: Snapshot,+ options: StrictOptions+): StrictAdjustment {+ const expected = options.expectPath ? loadExpectations(options.expectPath) : [];+ const both = expected.find((expectation) => options.newColumns.includes(expectation.column));+ if (both) {+ throw new UsageError(+ `${both.column} is both a --new-column (not compared) and an --expect column (compared)`+ );+ }+ const dropped = dropNewColumns(before, after, options.newColumns);+ const applied = applyExpectations(before, dropped.after, expected);+ return { notes: dropped.notes, ...applied };+}++function diff(+ beforePath: string,+ afterPath: string,+ strict: boolean,+ options: StrictOptions+): void {const before = loadSnapshot(beforePath);- const after = loadSnapshot(afterPath);+ const loaded = loadSnapshot(afterPath);+ const adjusted: StrictAdjustment = strict+ ? adjustForStrict(before, loaded, options)+ : { after: loaded, notes: [], matched: [], unmet: [] };+ const after = adjusted.after;reportSetupMismatch(before, after, strict);const table = compareTables(before, after);@@ -325,6 +536,7 @@ function diff(beforePath: string, afterPath: string, strict: boolean): void {console.log(`Route table: ${before.rows.length} models before, ${after.rows.length} after, generation ${after.catalogGenerationId}.`);+ for (const note of adjusted.notes) console.log(note);if (strict) {section("Snapshot header changed", header, "must-be-empty");section("Model listed twice in one snapshot", duplicates, "must-be-empty");@@ -347,30 +559,105 @@ function diff(beforePath: string, afterPath: string, strict: boolean): void {section("No-route reason or hint changed", table.noRouteTextChanged, "must-be-empty");section("Other row fields changed", table.otherFieldChanged, "must-be-empty");+ if (options.expectPath) {+ section("Expected differences NOT found", adjusted.unmet, "must-be-empty");+ section("Expected differences found, allowed", adjusted.matched, "review");+ }+ strictVerdict(header.length + duplicates.length, table, adjusted);+}++/** Print the strict verdict and exit: 0 only when nothing but expected differences remain. */+function strictVerdict(headerAndDuplicates: number, table: TableDiff, adjusted: StrictAdjustment) {const differences =- header.length +- duplicates.length ++ headerAndDuplicates ++ adjusted.unmet.length +Object.values(table).reduce((sum, lines) => sum + lines.length, 0);+ const clean =+ adjusted.matched.length === 0+ ? "no difference"+ : `no difference beyond the ${adjusted.matched.length} expected`;console.log(- `\nVerdict (strict): ${differences === 0 ? "no difference" : `${differences} difference${differences === 1 ? "" : "s"}, each to be named by the design or fixed`}.`+ `\nVerdict (strict): ${differences === 0 ? clean : `${differences} difference${differences === 1 ? "" : "s"}, each to be named by the design or fixed`}.`);process.exit(differences === 0 ? 0 : 1);}-const args = process.argv.slice(2);-const flags = args.filter((arg) => arg.startsWith("--"));-const [command, a, b] = args.filter((arg) => !arg.startsWith("--"));+/** Flags that take a value, as `--name value` or `--name=value`. */+const VALUE_FLAGS = new Set(["--expect", "--new-column"]);++interface CommandLine {+ positional: string[];+ /** Every other `--` token, verbatim (`--strict`, `--limit=N`, or a typo). */+ flags: string[];+ values: Map<string, string[]>;+ problems: string[];+}++/** Record the value flag at `argv[index]`; returns the index of the last token it used. */+function readValueFlag(argv: string[], index: number, line: CommandLine): number {+ const arg = argv[index];+ const name = arg.split("=")[0];+ const inline = arg.includes("=") ? arg.slice(name.length + 1) : undefined;+ const used = inline === undefined ? index + 1 : index;+ const value = inline ?? argv[used];+ if (!value || value.startsWith("--")) line.problems.push(`${name} needs a value`);+ else line.values.set(name, [...(line.values.get(name) ?? []), value]);+ return used;+}++function parseCommandLine(argv: string[]): CommandLine {+ const line: CommandLine = { positional: [], flags: [], values: new Map(), problems: [] };+ for (let index = 0; index < argv.length; index++) {+ const arg = argv[index];+ if (!arg.startsWith("--")) line.positional.push(arg);+ else if (!VALUE_FLAGS.has(arg.split("=")[0])) line.flags.push(arg);+ else index = readValueFlag(argv, index, line);+ }+ return line;+}++/** Why a `diff` command line cannot run; empty when it can. */+function diffProblems(line: CommandLine): string[] {+ const problems = [...line.problems];+ // An unknown flag is refused, not ignored: a mistyped --strict must not run the plain gate and pass.+ const unknown = line.flags.filter((flag) => flag !== "--strict");+ if (unknown.length > 0) problems.push(`unknown flag ${unknown.join(", ")}`);+ if (line.values.size > 0 && !line.flags.includes("--strict")) {+ problems.push("--expect and --new-column apply only with --strict");+ }+ if ((line.values.get("--expect") ?? []).length > 1) problems.push("--expect may be given once");+ return problems;+}++function runDiff(line: CommandLine, beforePath: string, afterPath: string): void {+ try {+ diff(beforePath, afterPath, line.flags.includes("--strict"), {+ expectPath: line.values.get("--expect")?.[0],+ newColumns: line.values.get("--new-column") ?? [],+ });+ } catch (error) {+ if (!(error instanceof UsageError)) throw error;+ console.error(`ERROR: ${error.message}`);+ process.exit(2);+ }+}++const commandLine = parseCommandLine(process.argv.slice(2));+const [command, a, b] = commandLine.positional;+const problems = command === "diff" ? diffProblems(commandLine) : [];if (command === "capture" && a) {- const limitArg = flags.find((arg) => arg.startsWith("--limit="));+ const limitArg = commandLine.flags.find((arg) => arg.startsWith("--limit="));await capture(a, limitArg ? Number(limitArg.split("=")[1]) : undefined);-} else if (command === "diff" && a && b && flags.every((flag) => flag === "--strict")) {- // An unknown flag is refused, not ignored: a mistyped --strict must not run the plain gate and pass.- diff(a, b, flags.includes("--strict"));+} else if (command === "diff" && a && b && problems.length === 0) {+ runDiff(commandLine, a, b);} else {+ for (const problem of problems) console.error(`ERROR: ${problem}`);console.error("usage:\n" +" bun run scripts/route-table-snapshot.ts capture <out.json> [--limit=N]\n" +- " bun run scripts/route-table-snapshot.ts diff [--strict] <before.json> <after.json>"+ " bun run scripts/route-table-snapshot.ts diff [--strict] <before.json> <after.json>\n" ++ " bun run scripts/route-table-snapshot.ts diff --strict [--expect <expected.json>]\n" ++ " [--new-column <name>]... <before.json> <after.json>");process.exit(2);}