Skip to content

Claude Code. Any Model. The most powerful AI coding agent now speaks every language.

chore(routing): record the proxy decision per row in the route gate

Jack Rudenko56fe2fbcff85
diff --git a/scripts/route-table-snapshot.ts b/scripts/route-table-snapshot.ts
index 1e5315bd..327462f4 100644
--- a/scripts/route-table-snapshot.ts
+++ b/scripts/route-table-snapshot.ts
@@ -28,6 +28,23 @@
* mismatch is an error there, not a warning, because a diff across two generations
* cannot tell the code change from the catalog change.
*
+ * Two flags make `--strict` usable for a change meant to move a KNOWN set of things:
+ *
+ * diff --strict --expect expected.json before.json after.json
+ * diff --strict --new-column decision before.json after.json
+ *
+ * `--expect` names each allowed difference in advance, as a JSON list of
+ * `{ "model", "column", "old", "new" }` (a missing `old` or `new` means the column is
+ * absent on that side). A difference the list names exactly is allowed; any other
+ * difference fails, and so does a listed difference the two snapshots do not show.
+ * `--new-column <name>` (repeatable) skips a column the before snapshot does not carry
+ * at all, which is how a capture that gained a column is compared with one that
+ * predates it. Naming a column the before snapshot does carry is refused.
+ *
+ * Each row also records `decision`: what the proxy does with the id before `route()`
+ * could see it (`proxyRouteDecision`). The row's chain is `route()`'s answer whatever
+ * the decision says; the proxy asks `route()` only when the decision is `bare`.
+ *
* Environment-dependent by design. It calls the real `route()`, so it sees this
* machine's credentials and `defaultProvider`. Both snapshots must come from the
* same machine with the same configuration, which is why each file records them.
@@ -38,11 +55,20 @@ import { homedir } from "node:os";
import { join } from "node:path";
import { loadConfig } from "../packages/cli/src/profile-config.js";
import { readAllModelsCache } from "../packages/cli/src/providers/all-models-cache.js";
+import {
+ type ProxyRouteDecision,
+ proxyRouteDecision,
+} from "../packages/cli/src/providers/native-route.js";
import { route } from "../packages/cli/src/providers/routing-rules.js";
/** One model's resolved chain, flattened to the strings a request would carry. */
interface RouteRow {
modelId: string;
+ /**
+ * What the proxy does with the id before routing; only `bare` reaches `route()`.
+ * Absent from snapshots captured before this column existed.
+ */
+ decision?: ProxyRouteDecision["type"];
kind: "ok" | "no-route";
/** `provider@model` per hop, primary first. Empty for `no-route`. */
chain: string[];
@@ -81,20 +107,23 @@ async function capture(outPath: string, limit?: number): Promise<void> {
const batch = ids.slice(start, start + CONCURRENCY);
const settled = await Promise.all(
batch.map(async (modelId): Promise<RouteRow> => {
+ const decision = proxyRouteDecision(modelId).type;
try {
const plan = await route(modelId);
if (plan.kind === "ok") {
return {
modelId,
+ decision,
kind: "ok",
chain: [plan.primary, ...plan.fallbacks].map((hop) => hop.modelSpec),
};
}
- return { modelId, kind: "no-route", chain: [], reason: plan.reason };
+ return { modelId, decision, kind: "no-route", chain: [], reason: plan.reason };
} catch (error) {
// A throw is itself a finding: record it rather than losing the row.
return {
modelId,
+ decision,
kind: "no-route",
chain: [],
reason: `threw: ${error instanceof Error ? error.message : String(error)}`,
@@ -122,6 +151,14 @@ async function capture(outPath: string, limit?: number): Promise<void> {
console.log(
`Wrote ${outPath}: ${rows.length} models, ${routed} routed, ${rows.length - routed} with no route, generation ${snapshot.catalogGenerationId}.`
);
+ const decisions = new Map<string, number>();
+ for (const row of rows) {
+ const decision = row.decision ?? "(none)";
+ decisions.set(decision, (decisions.get(decision) ?? 0) + 1);
+ }
+ console.log(
+ `Proxy decision per row: ${[...decisions].map(([decision, count]) => `${decision} ${count}`).join(", ")}.`
+ );
}
function loadSnapshot(path: string): Snapshot {
@@ -307,9 +344,183 @@ function section(title: string, lines: string[], verdict: "must-be-empty" | "rev
if (lines.length > 40) console.log(` … ${lines.length - 40} more`);
}
-function diff(beforePath: string, afterPath: string, strict: boolean): void {
+/** An invocation or input the gate cannot run with. Printed as an error; exit 2. */
+class UsageError extends Error {}
+
+/**
+ * One difference the design names in advance: `column` of `model` goes from `old` to
+ * `new`. A missing `old` or `new` means the column is absent on that side.
+ */
+interface ExpectedDifference {
+ model: string;
+ column: string;
+ old?: unknown;
+ new?: unknown;
+}
+
+const EXPECTATION_KEYS = new Set(["model", "column", "old", "new"]);
+
+function readJson(path: string, flag: string): unknown {
+ try {
+ return JSON.parse(readFileSync(path, "utf-8"));
+ } catch (error) {
+ throw new UsageError(`${flag} ${path}: ${error instanceof Error ? error.message : error}`);
+ }
+}
+
+function toExpectation(item: unknown, where: string): ExpectedDifference {
+ if (typeof item !== "object" || item === null || Array.isArray(item)) {
+ throw new UsageError(`${where}: not an object`);
+ }
+ const fields = item as Record<string, unknown>;
+ const unknownKeys = Object.keys(fields).filter((key) => !EXPECTATION_KEYS.has(key));
+ if (unknownKeys.length > 0)
+ throw new UsageError(`${where}: unknown key ${unknownKeys.join(", ")}`);
+ if (typeof fields.model !== "string" || typeof fields.column !== "string") {
+ throw new UsageError(`${where}: needs a string "model" and a string "column"`);
+ }
+ if (fields.column === "modelId") throw new UsageError(`${where}: modelId names the row`);
+ if (sameValue(fields.old, fields.new)) {
+ throw new UsageError(`${where}: "old" equals "new", so it names no difference`);
+ }
+ return { model: fields.model, column: fields.column, old: fields.old, new: fields.new };
+}
+
+/** The `--expect` file, refused whole if any entry is malformed or listed twice. */
+function loadExpectations(path: string): ExpectedDifference[] {
+ const parsed = readJson(path, "--expect");
+ if (!Array.isArray(parsed)) throw new UsageError(`--expect ${path}: not a JSON list`);
+ const seen = new Set<string>();
+ return parsed.map((item: unknown, index) => {
+ const where = `--expect ${path}, entry ${index}`;
+ const expectation = toExpectation(item, where);
+ const key = JSON.stringify([expectation.model, expectation.column]);
+ if (seen.has(key)) {
+ throw new UsageError(`${where}: ${expectation.model} ${expectation.column} is listed twice`);
+ }
+ seen.add(key);
+ return expectation;
+ });
+}
+
+function columnOf(row: RouteRow, column: string): unknown {
+ return (row as unknown as Record<string, unknown>)[column];
+}
+
+/** A copy of `row` with `column` set to `value`, or without it when `value` is undefined. */
+function withColumn(row: RouteRow, column: string, value: unknown): RouteRow {
+ const copy: Record<string, unknown> = { ...row };
+ if (value === undefined) delete copy[column];
+ else copy[column] = value;
+ return copy as unknown as RouteRow;
+}
+
+/**
+ * Take each new column out of `after`, once `before` is shown to carry it on no row.
+ * A column `before` does carry is not new, and skipping it would hide a real difference.
+ */
+function dropNewColumns(
+ before: Snapshot,
+ after: Snapshot,
+ columns: string[]
+): { after: Snapshot; notes: string[] } {
+ const notes: string[] = [];
+ let rows = after.rows;
+ for (const column of columns) {
+ const carriedBefore = before.rows.filter((row) => Object.hasOwn(row, column)).length;
+ if (carriedBefore > 0) {
+ throw new UsageError(
+ `--new-column ${column}: the before snapshot carries it on ${carriedBefore} of ${before.rows.length} rows, so it is not new. Drop the flag to compare it.`
+ );
+ }
+ const carriedAfter = rows.filter((row) => Object.hasOwn(row, column)).length;
+ notes.push(
+ `New column "${column}": on no before row and ${carriedAfter} of ${rows.length} after rows; not compared.`
+ );
+ rows = rows.map((row) => withColumn(row, column, undefined));
+ }
+ return { after: { ...after, rows }, notes };
+}
+
+/**
+ * Match each expected difference against the two snapshots. A match is taken out of
+ * `after` (its column set back to the before value), so the comparison never sees it.
+ * Anything else is unmet: the model is missing, or a side holds another value.
+ */
+function applyExpectations(
+ before: Snapshot,
+ after: Snapshot,
+ expected: ExpectedDifference[]
+): { after: Snapshot; matched: string[]; unmet: string[] } {
+ const beforeRows = new Map(before.rows.map((row) => [row.modelId, row]));
+ const afterRows = new Map(after.rows.map((row) => [row.modelId, row]));
+ const matched: string[] = [];
+ const unmet: string[] = [];
+ for (const expectation of expected) {
+ const { model, column } = expectation;
+ const named = `${model}: ${column}: ${showValue(expectation.old)} => ${showValue(expectation.new)}`;
+ const beforeRow = beforeRows.get(model);
+ const afterRow = afterRows.get(model);
+ if (!beforeRow || !afterRow) {
+ unmet.push(`${named} (no such model in the ${beforeRow ? "after" : "before"} snapshot)`);
+ continue;
+ }
+ const was = columnOf(beforeRow, column);
+ const now = columnOf(afterRow, column);
+ if (sameValue(was, expectation.old) && sameValue(now, expectation.new)) {
+ matched.push(named);
+ afterRows.set(model, withColumn(afterRow, column, was));
+ } else {
+ unmet.push(`${named} (found ${showValue(was)} => ${showValue(now)})`);
+ }
+ }
+ const rows = after.rows.map((row) => afterRows.get(row.modelId) ?? row);
+ return { after: { ...after, rows }, matched, unmet };
+}
+
+interface StrictOptions {
+ /** A JSON list of expected differences. */
+ expectPath?: string;
+ /** Columns the before snapshot does not carry; not compared. */
+ newColumns: string[];
+}
+
+interface StrictAdjustment {
+ after: Snapshot;
+ notes: string[];
+ matched: string[];
+ unmet: string[];
+}
+
+function adjustForStrict(
+ before: Snapshot,
+ after: Snapshot,
+ options: StrictOptions
+): StrictAdjustment {
+ const expected = options.expectPath ? loadExpectations(options.expectPath) : [];
+ const both = expected.find((expectation) => options.newColumns.includes(expectation.column));
+ if (both) {
+ throw new UsageError(
+ `${both.column} is both a --new-column (not compared) and an --expect column (compared)`
+ );
+ }
+ const dropped = dropNewColumns(before, after, options.newColumns);
+ const applied = applyExpectations(before, dropped.after, expected);
+ return { notes: dropped.notes, ...applied };
+}
+
+function diff(
+ beforePath: string,
+ afterPath: string,
+ strict: boolean,
+ options: StrictOptions
+): void {
const before = loadSnapshot(beforePath);
- const after = loadSnapshot(afterPath);
+ const loaded = loadSnapshot(afterPath);
+ const adjusted: StrictAdjustment = strict
+ ? adjustForStrict(before, loaded, options)
+ : { after: loaded, notes: [], matched: [], unmet: [] };
+ const after = adjusted.after;
reportSetupMismatch(before, after, strict);
const table = compareTables(before, after);
@@ -325,6 +536,7 @@ function diff(beforePath: string, afterPath: string, strict: boolean): void {
console.log(
`Route table: ${before.rows.length} models before, ${after.rows.length} after, generation ${after.catalogGenerationId}.`
);
+ for (const note of adjusted.notes) console.log(note);
if (strict) {
section("Snapshot header changed", header, "must-be-empty");
section("Model listed twice in one snapshot", duplicates, "must-be-empty");
@@ -347,30 +559,105 @@ function diff(beforePath: string, afterPath: string, strict: boolean): void {
section("No-route reason or hint changed", table.noRouteTextChanged, "must-be-empty");
section("Other row fields changed", table.otherFieldChanged, "must-be-empty");
+ if (options.expectPath) {
+ section("Expected differences NOT found", adjusted.unmet, "must-be-empty");
+ section("Expected differences found, allowed", adjusted.matched, "review");
+ }
+ strictVerdict(header.length + duplicates.length, table, adjusted);
+}
+
+/** Print the strict verdict and exit: 0 only when nothing but expected differences remain. */
+function strictVerdict(headerAndDuplicates: number, table: TableDiff, adjusted: StrictAdjustment) {
const differences =
- header.length +
- duplicates.length +
+ headerAndDuplicates +
+ adjusted.unmet.length +
Object.values(table).reduce((sum, lines) => sum + lines.length, 0);
+ const clean =
+ adjusted.matched.length === 0
+ ? "no difference"
+ : `no difference beyond the ${adjusted.matched.length} expected`;
console.log(
- `\nVerdict (strict): ${differences === 0 ? "no difference" : `${differences} difference${differences === 1 ? "" : "s"}, each to be named by the design or fixed`}.`
+ `\nVerdict (strict): ${differences === 0 ? clean : `${differences} difference${differences === 1 ? "" : "s"}, each to be named by the design or fixed`}.`
);
process.exit(differences === 0 ? 0 : 1);
}
-const args = process.argv.slice(2);
-const flags = args.filter((arg) => arg.startsWith("--"));
-const [command, a, b] = args.filter((arg) => !arg.startsWith("--"));
+/** Flags that take a value, as `--name value` or `--name=value`. */
+const VALUE_FLAGS = new Set(["--expect", "--new-column"]);
+
+interface CommandLine {
+ positional: string[];
+ /** Every other `--` token, verbatim (`--strict`, `--limit=N`, or a typo). */
+ flags: string[];
+ values: Map<string, string[]>;
+ problems: string[];
+}
+
+/** Record the value flag at `argv[index]`; returns the index of the last token it used. */
+function readValueFlag(argv: string[], index: number, line: CommandLine): number {
+ const arg = argv[index];
+ const name = arg.split("=")[0];
+ const inline = arg.includes("=") ? arg.slice(name.length + 1) : undefined;
+ const used = inline === undefined ? index + 1 : index;
+ const value = inline ?? argv[used];
+ if (!value || value.startsWith("--")) line.problems.push(`${name} needs a value`);
+ else line.values.set(name, [...(line.values.get(name) ?? []), value]);
+ return used;
+}
+
+function parseCommandLine(argv: string[]): CommandLine {
+ const line: CommandLine = { positional: [], flags: [], values: new Map(), problems: [] };
+ for (let index = 0; index < argv.length; index++) {
+ const arg = argv[index];
+ if (!arg.startsWith("--")) line.positional.push(arg);
+ else if (!VALUE_FLAGS.has(arg.split("=")[0])) line.flags.push(arg);
+ else index = readValueFlag(argv, index, line);
+ }
+ return line;
+}
+
+/** Why a `diff` command line cannot run; empty when it can. */
+function diffProblems(line: CommandLine): string[] {
+ const problems = [...line.problems];
+ // An unknown flag is refused, not ignored: a mistyped --strict must not run the plain gate and pass.
+ const unknown = line.flags.filter((flag) => flag !== "--strict");
+ if (unknown.length > 0) problems.push(`unknown flag ${unknown.join(", ")}`);
+ if (line.values.size > 0 && !line.flags.includes("--strict")) {
+ problems.push("--expect and --new-column apply only with --strict");
+ }
+ if ((line.values.get("--expect") ?? []).length > 1) problems.push("--expect may be given once");
+ return problems;
+}
+
+function runDiff(line: CommandLine, beforePath: string, afterPath: string): void {
+ try {
+ diff(beforePath, afterPath, line.flags.includes("--strict"), {
+ expectPath: line.values.get("--expect")?.[0],
+ newColumns: line.values.get("--new-column") ?? [],
+ });
+ } catch (error) {
+ if (!(error instanceof UsageError)) throw error;
+ console.error(`ERROR: ${error.message}`);
+ process.exit(2);
+ }
+}
+
+const commandLine = parseCommandLine(process.argv.slice(2));
+const [command, a, b] = commandLine.positional;
+const problems = command === "diff" ? diffProblems(commandLine) : [];
if (command === "capture" && a) {
- const limitArg = flags.find((arg) => arg.startsWith("--limit="));
+ const limitArg = commandLine.flags.find((arg) => arg.startsWith("--limit="));
await capture(a, limitArg ? Number(limitArg.split("=")[1]) : undefined);
-} else if (command === "diff" && a && b && flags.every((flag) => flag === "--strict")) {
- // An unknown flag is refused, not ignored: a mistyped --strict must not run the plain gate and pass.
- diff(a, b, flags.includes("--strict"));
+} else if (command === "diff" && a && b && problems.length === 0) {
+ runDiff(commandLine, a, b);
} else {
+ for (const problem of problems) console.error(`ERROR: ${problem}`);
console.error(
"usage:\n" +
" bun run scripts/route-table-snapshot.ts capture <out.json> [--limit=N]\n" +
- " bun run scripts/route-table-snapshot.ts diff [--strict] <before.json> <after.json>"
+ " bun run scripts/route-table-snapshot.ts diff [--strict] <before.json> <after.json>\n" +
+ " bun run scripts/route-table-snapshot.ts diff --strict [--expect <expected.json>]\n" +
+ " [--new-column <name>]... <before.json> <after.json>"
);
process.exit(2);
}