Skip to content

Claude Code. Any Model. The most powerful AI coding agent now speaks every language.

test(mcp): pin preflight probing the routing chain hop by hop

Jack Rudenkoe8734975a864
diff --git a/packages/cli/src/handlers/shared/hop-decision.test.ts b/packages/cli/src/handlers/shared/hop-decision.test.ts
new file mode 100644
index 00000000..67906f42
--- /dev/null
+++ b/packages/cli/src/handlers/shared/hop-decision.test.ts
@@ -0,0 +1,256 @@
+import { expect, test } from "bun:test";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+// HOME must be set before imports: Bun caches homedir() at process startup.
+if (process.env.CLAUDISH_W2_TEST_CHILD !== import.meta.path) {
+ test("W2: isolated HOME child", () => {
+ const home = mkdtempSync(join(tmpdir(), "claudish-w2-"));
+ try {
+ const child = Bun.spawnSync([process.execPath, "test", import.meta.path], {
+ cwd: process.cwd(),
+ env: {
+ ...process.env,
+ HOME: home,
+ CLAUDISH_W2_TEST_CHILD: import.meta.path,
+ CLAUDISH_DISABLE_KEYCHAIN: "1",
+ CLAUDISH_DISABLE_OP: "1",
+ CLAUDISH_DISABLE_CATALOG_WARM: "1",
+ },
+ timeout: 18_000,
+ });
+ const output =
+ new TextDecoder().decode(child.stdout) + new TextDecoder().decode(child.stderr);
+ console.info(output);
+ expect(child.exitCode, output).toBe(0);
+ } finally {
+ rmSync(home, { recursive: true, force: true });
+ }
+ }, 20_000);
+} else {
+ const check = (name: string, run: () => unknown) =>
+ test(name, async () => {
+ await run();
+ }, 10_000);
+
+ const { afterEach } = await import("bun:test");
+ const { readFileSync } = await import("node:fs");
+ const { Hono } = await import("hono");
+ const { decideHopFailure } = await import("./hop-decision.js");
+ const { FallbackHandler } = await import("../fallback-handler.js");
+ const { probeLink } = await import("../../providers/probe-live.js");
+ const { preflightModel } = await import("../../providers/preflight-chain.js");
+ const { setConfigFileOverride } = await import("../../profile-config.js");
+ setConfigFileOverride(join(process.env.HOME!, "config.json"));
+ const originalFetch = globalThis.fetch;
+ const blockedFetch = Object.assign(async () => {
+ throw new Error("W2 forbids upstream network");
+ }, originalFetch);
+ globalThis.fetch = blockedFetch;
+ afterEach(() => {
+ globalThis.fetch = blockedFetch;
+ });
+
+ type Case = {
+ name: string;
+ status: number;
+ headers?: Record<string, string>;
+ provider: string;
+ action: "advance" | "stop";
+ reason: import("./hop-decision.js").HopFailureReason;
+ thrown?: boolean;
+ };
+ const cases: Case[] = [
+ {
+ name: "connection fault",
+ status: 400,
+ headers: { "x-claudish-connection-error": "1" },
+ provider: "kimi-coding",
+ action: "stop",
+ reason: "non-retryable",
+ },
+ {
+ name: "stream failure",
+ status: 400,
+ headers: { "x-claudish-stream-failure": "1" },
+ provider: "kimi-coding",
+ action: "stop",
+ reason: "stream-failure",
+ },
+ {
+ name: "recovery hold",
+ status: 503,
+ headers: { "x-claudish-recovery": "1" },
+ provider: "kimi-coding",
+ action: "stop",
+ reason: "recovery-hold",
+ },
+ {
+ name: "metered credential rejection",
+ status: 400,
+ headers: { "x-claudish-auth-verdict": "terminal" },
+ provider: "openai",
+ action: "stop",
+ reason: "credential-hold",
+ },
+ {
+ name: "subscription credential rejection",
+ status: 400,
+ headers: { "x-claudish-auth-verdict": "terminal" },
+ provider: "kimi-coding",
+ action: "advance",
+ reason: "subscription-credential-rejected",
+ },
+ { name: "404", status: 404, provider: "openai", action: "advance", reason: "retryable" },
+ { name: "plain 500", status: 500, provider: "openai", action: "stop", reason: "non-retryable" },
+ {
+ name: "thrown hop / marked 500",
+ status: 500,
+ headers: { "x-claudish-unhandled-error": "1" },
+ provider: "openai",
+ action: "advance",
+ reason: "thrown",
+ thrown: true,
+ },
+ ];
+
+ for (const item of cases) {
+ check(`G2 / 2.1-2.4: runtime, probe and preflight agree on ${item.name}`, async () => {
+ const input: import("./hop-decision.js").HopFailureInput = item.thrown
+ ? { kind: "thrown" }
+ : {
+ kind: "response",
+ status: item.status,
+ headers: new Headers(item.headers),
+ body: "",
+ provider: item.provider,
+ name: item.provider,
+ };
+ expect(decideHopFailure(input)).toEqual({ action: item.action, reason: item.reason });
+
+ const response = () => new Response(null, { status: item.status, headers: item.headers });
+ let secondCalls = 0;
+ const runtime = new FallbackHandler([
+ {
+ name: item.provider,
+ provider: item.provider,
+ handler: {
+ handle: async () => {
+ if (item.thrown) throw new Error("W2 thrown hop");
+ return response();
+ },
+ shutdown: async () => {},
+ },
+ },
+ {
+ name: "W2 second",
+ provider: "openrouter",
+ handler: {
+ handle: async () => {
+ secondCalls++;
+ return new Response(null, { status: 204 });
+ },
+ shutdown: async () => {},
+ },
+ },
+ ]);
+ const app = new Hono();
+ app.post("/", (context) => runtime.handle(context, {}));
+ try {
+ const observed = await app.request("http://w2.test/", { method: "POST" });
+ expect(secondCalls).toBe(item.action === "advance" ? 1 : 0);
+ expect(observed.status).toBe(item.action === "advance" ? 204 : item.status);
+ } finally {
+ await runtime.shutdown();
+ }
+
+ globalThis.fetch = Object.assign(async () => response(), originalFetch);
+ const probe = await probeLink(
+ "http://w2.test",
+ { provider: item.provider, modelSpec: `${item.provider}@w2`, hasCredentials: true },
+ 100
+ );
+ expect(probe.chainAction).toBe(item.action);
+ expect(probe.chainReason).toBe(item.reason);
+ let calls = 0;
+ const result = await preflightModel("w2-parity", {
+ explain: async () => ({
+ requestedModel: "w2-parity",
+ routedModel: "w2-parity",
+ source: "catalog",
+ outcome: { kind: "ok" },
+ warnings: [],
+ candidates: [item.provider, "openrouter"].map((provider, index) => ({
+ provider,
+ displayName: provider,
+ modelSpec: `${provider}@wire-${index}`,
+ wireId: `wire-${index}`,
+ position: "candidate" as const,
+ outcome: "kept" as const,
+ })),
+ }),
+ probe: async () => (++calls === 1 ? probe : { state: "live", latencyMs: 0 }),
+ });
+ expect(calls).toBe(item.action === "advance" ? 2 : 1);
+ expect(result.kind).toBe(item.action === "advance" ? "served" : "failed");
+ });
+ }
+
+ check("G2 / 2.1: connection marker takes priority over quota wording and auth verdict", () => {
+ expect(
+ decideHopFailure({
+ kind: "response",
+ status: 400,
+ headers: new Headers({
+ "x-claudish-connection-error": "1",
+ "x-claudish-auth-verdict": "terminal",
+ }),
+ body: "quota",
+ provider: "kimi-coding",
+ name: "Kimi Coding",
+ }).action
+ ).toBe("stop");
+ });
+
+ check("G2 / 2.2: captured content followed by SSE error stops after-200", async () => {
+ const stream = readFileSync(
+ join(
+ import.meta.dir,
+ "../../test-fixtures/sse-responses/regression-zai-glm5-instream-error.sse"
+ )
+ );
+ globalThis.fetch = Object.assign(
+ async () => new Response(stream, { headers: { "content-type": "text/event-stream" } }),
+ originalFetch
+ );
+ const result = await probeLink(
+ "http://w2.test",
+ { provider: "openai", modelSpec: "openai@w2", hasCredentials: true },
+ 100
+ );
+ expect(result.state).not.toBe("live");
+ expect(result.chainAction).toBe("stop");
+ expect(result.chainReason).toBe("after-200");
+ });
+
+ check("G2 / 2.2: captured live stream has no chainAction", async () => {
+ const stream = readFileSync(
+ join(
+ import.meta.dir,
+ "../../test-fixtures/sse-responses/minimax-m25-turn1-thinking-text-tool.sse"
+ )
+ );
+ globalThis.fetch = Object.assign(
+ async () => new Response(stream, { headers: { "content-type": "text/event-stream" } }),
+ originalFetch
+ );
+ const result = await probeLink(
+ "http://w2.test",
+ { provider: "openai", modelSpec: "openai@w2", hasCredentials: true },
+ 100
+ );
+ expect(result.state).toBe("live");
+ expect(result.chainAction).toBeUndefined();
+ });
+}
diff --git a/packages/cli/src/providers/preflight-chain.test.ts b/packages/cli/src/providers/preflight-chain.test.ts
new file mode 100644
index 00000000..4dd8c1d5
--- /dev/null
+++ b/packages/cli/src/providers/preflight-chain.test.ts
@@ -0,0 +1,284 @@
+import { expect, test } from "bun:test";
+import { mkdtempSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+
+// HOME must be set before imports: Bun caches homedir() at process startup.
+if (process.env.CLAUDISH_W2_TEST_CHILD !== import.meta.path) {
+ test("W2: isolated HOME child", () => {
+ const home = mkdtempSync(join(tmpdir(), "claudish-w2-"));
+ try {
+ const child = Bun.spawnSync([process.execPath, "test", import.meta.path], {
+ cwd: process.cwd(),
+ env: {
+ ...process.env,
+ HOME: home,
+ CLAUDISH_W2_TEST_CHILD: import.meta.path,
+ CLAUDISH_DISABLE_KEYCHAIN: "1",
+ CLAUDISH_DISABLE_OP: "1",
+ CLAUDISH_DISABLE_CATALOG_WARM: "1",
+ },
+ timeout: 18_000,
+ });
+ const output =
+ new TextDecoder().decode(child.stdout) + new TextDecoder().decode(child.stderr);
+ console.info(output);
+ expect(child.exitCode, output).toBe(0);
+ } finally {
+ rmSync(home, { recursive: true, force: true });
+ }
+ }, 20_000);
+} else {
+ const check = (name: string, run: () => unknown) =>
+ test(name, async () => {
+ await run();
+ }, 10_000);
+
+ const { preflightModel, renderPreflightReport, hopBilling } = await import(
+ "./preflight-chain.js"
+ );
+ const { describeProbeState } = await import("./probe-live.js");
+ const { setConfigFileOverride } = await import("../profile-config.js");
+ setConfigFileOverride(join(process.env.HOME!, "config.json"));
+ const blockedFetch = globalThis.fetch;
+ globalThis.fetch = Object.assign(async () => {
+ throw new Error("W2 forbids upstream network");
+ }, blockedFetch);
+
+ const model = "w2-chain-model";
+ const explanation = (): import("./routing-rules.js").RouteExplanation => ({
+ requestedModel: model,
+ routedModel: model,
+ source: "catalog",
+ catalog: "found",
+ candidates: [
+ {
+ provider: "kimi-coding",
+ displayName: "Kimi Coding",
+ modelSpec: "kc@wire-one",
+ wireId: "wire-one",
+ position: "candidate",
+ tier: "subscription",
+ outcome: "kept",
+ },
+ {
+ provider: "openai",
+ displayName: "OpenAI",
+ modelSpec: "openai@wire-two",
+ wireId: "wire-two",
+ position: "candidate",
+ tier: "native",
+ outcome: "kept",
+ },
+ {
+ provider: "openrouter",
+ displayName: "OpenRouter",
+ modelSpec: "or@wire-three",
+ wireId: "wire-three",
+ position: "fallback",
+ tier: "fallback",
+ outcome: "kept",
+ },
+ ],
+ outcome: { kind: "ok" },
+ warnings: [],
+ });
+ const rejected = (): import("./probe-live.js").ProbeResult => ({
+ state: "auth-failed",
+ latencyMs: 1,
+ httpStatus: 400,
+ errorMessage: "W2 rejected hop",
+ chainAction: "advance",
+ chainReason: "subscription-credential-rejected",
+ });
+ const live = (): import("./probe-live.js").ProbeResult => ({ state: "live", latencyMs: 1 });
+
+ check(
+ "G1 / 2.3: hop 2 serves with its provider, billing and model spec; hop 3 is not probed",
+ async () => {
+ const calls: string[] = [];
+ const result = await preflightModel(model, {
+ explain: async () => explanation(),
+ probe: async (target) => {
+ calls.push(target.provider);
+ return target.provider === "kimi-coding" ? rejected() : live();
+ },
+ });
+ expect(calls).toEqual(["kimi-coding", "openai"]);
+ expect(result.kind).toBe("served");
+ if (result.kind !== "served") throw new Error("Expected serving second hop");
+ expect(result.served).toMatchObject({
+ provider: "openai",
+ billing: "metered",
+ wireId: "wire-two",
+ index: 2,
+ });
+ expect(result.served.probeSpec).toContain("@wire-two");
+ expect(result.failed).toHaveLength(1);
+ expect(result.failed[0]).toMatchObject({
+ index: 1,
+ provider: "kimi-coding",
+ billing: "subscription",
+ action: "advance",
+ reason: "subscription-credential-rejected",
+ });
+ expect(result.untried).toBe(1);
+ expect(result.total).toBe(3);
+ const report = renderPreflightReport([result], { probed: true });
+ expect(report).toContain("| Model | Provider | Billing | Status | Model spec |");
+ expect(report).toContain("| OpenAI | metered |");
+ expect(report).toContain("✅ live · hop 2/3");
+ expect(report).toContain("(+1 untried)");
+ expect(report).toContain("subscription: 0 · metered: 1");
+ expect(report).toContain("PER TOKEN");
+ expect(report).not.toContain("Drop or replace before running");
+ const failedLines = report
+ .split("\n")
+ .filter((line) => line.startsWith("- ") && line.includes("hop 1/3"));
+ expect(failedLines).toHaveLength(1);
+ expect(failedLines[0]).toContain("subscription");
+ expect(failedLines[0]).toContain(describeProbeState(rejected()));
+ expect(failedLines[0]).toContain("moved to hop 2");
+ expect(report).not.toContain("wire-three");
+ }
+ );
+
+ check("G1 / 2.7: only exhausted and no-route models appear in Drop or replace", async () => {
+ const served = await preflightModel(model, {
+ explain: async () => explanation(),
+ probe: async () => live(),
+ });
+ const exhausted = await preflightModel("w2-exhausted", {
+ explain: async () => explanation(),
+ probe: async () => rejected(),
+ });
+ expect(exhausted.kind).toBe("failed");
+ if (exhausted.kind !== "failed") throw new Error("Expected exhausted chain");
+ expect(exhausted.failed).toHaveLength(3);
+ expect(exhausted.heldAt).toBeUndefined();
+ expect(exhausted.untried).toBe(0);
+ const noRoute = await preflightModel("w2-no-route", {
+ probe: async () => {
+ throw new Error("No-route must not probe");
+ },
+ explain: async () => ({
+ ...explanation(),
+ candidates: [],
+ outcome: {
+ kind: "no-route",
+ cause: "catalog-empty",
+ reason: "W2 no mapped route",
+ hint: "W2 add a route",
+ },
+ }),
+ });
+ expect(noRoute).toMatchObject({
+ kind: "no-route",
+ reason: "W2 no mapped route",
+ hint: "W2 add a route",
+ });
+ const report = renderPreflightReport([served, exhausted, noRoute], { probed: true });
+ const drop = report
+ .split("\n")
+ .find((line) => line.includes("Drop or replace before running"))!;
+ expect(drop).toContain("`w2-exhausted`");
+ expect(drop).toContain("`w2-no-route`");
+ expect(drop).not.toContain("`w2-chain-model`");
+ expect(report).toContain("❌ every hop failed (3)");
+ expect(report).toContain("W2 no mapped route");
+ expect(report).toContain("W2 add a route");
+ });
+
+ check("G2 / 2.4: a stop verdict holds hop 1 and leaves later hops untried", async () => {
+ let calls = 0;
+ const result = await preflightModel(model, {
+ explain: async () => explanation(),
+ probe: async () => {
+ calls++;
+ return { ...rejected(), chainAction: "stop", chainReason: "credential-hold" };
+ },
+ });
+ expect(calls).toBe(1);
+ expect(result.kind).toBe("failed");
+ if (result.kind !== "failed") throw new Error("Expected held chain");
+ expect(result.heldAt).toMatchObject({ index: 1, action: "stop", reason: "credential-hold" });
+ expect(result.untried).toBe(2);
+ const report = renderPreflightReport([result], { probed: true });
+ expect(report).toContain("chain stops at hop 1/3");
+ expect(report).toContain("chain stops here");
+ });
+
+ check(
+ "G2 / 2.9: absent chainAction stops with no-decision instead of trying a live hop 2",
+ async () => {
+ let calls = 0;
+ const result = await preflightModel(model, {
+ explain: async () => explanation(),
+ probe: async () => (++calls === 1 ? { state: "key-missing", latencyMs: 0 } : live()),
+ });
+ expect(calls).toBe(1);
+ expect(result.kind).toBe("failed");
+ if (result.kind !== "failed") throw new Error("Expected no-decision hold");
+ expect(result.heldAt).toMatchObject({ index: 1, action: "stop", reason: "no-decision" });
+ expect(renderPreflightReport([result], { probed: true })).toContain(
+ "chain stops here (no hop decision)"
+ );
+ }
+ );
+
+ check("G2 / 2.10: a throwing probe is contained as probe-error and never advances", async () => {
+ let calls = 0;
+ const result = await preflightModel(model, {
+ explain: async () => explanation(),
+ probe: async () => {
+ calls++;
+ throw new Error("boom");
+ },
+ });
+ expect(calls).toBe(1);
+ expect(result.kind).toBe("failed");
+ if (result.kind !== "failed") throw new Error("Expected contained probe error");
+ expect(result.heldAt).toMatchObject({ index: 1, action: "stop", reason: "probe-error" });
+ expect(renderPreflightReport([result], { probed: true })).toContain(
+ "❌ probe error · chain stops here"
+ );
+ });
+
+ check(
+ "G3 / 2.5: native Claude is neither success nor failure and bypasses explain and probe",
+ async () => {
+ const result = await preflightModel("opus", {
+ explain: async () => {
+ throw new Error("Native auth must not be routed");
+ },
+ probe: async () => {
+ throw new Error("Native auth must not be probed");
+ },
+ });
+ expect(result.kind).toBe("native");
+ const report = renderPreflightReport([result], { probed: true });
+ expect(report).toContain("native (not probed): 1");
+ expect(report).toContain("not probed");
+ expect(report).toContain("own auth");
+ expect(report).not.toContain("Drop or replace before running");
+ expect(report).not.toContain("✅");
+ expect(report).not.toContain("❌");
+ }
+ );
+
+ check("G1 / 2.6: plan-only preserves the chain without sending a probe", async () => {
+ const result = await preflightModel(model, { explain: async () => explanation(), probe: null });
+ expect(result.kind).toBe("planned");
+ if (result.kind !== "planned") throw new Error("Expected unprobed plan");
+ expect(result.hops).toHaveLength(3);
+ expect(result.hops.every((hop) => hop.result === undefined)).toBe(true);
+ const report = renderPreflightReport([result], { probed: false });
+ expect(report).toContain("| Kimi Coding | subscription |");
+ expect(report).toContain("not probed");
+ expect(report).toContain("subscription: 1 · metered: 0");
+ expect(report).not.toContain("PER TOKEN");
+ expect(hopBilling("kimi-coding")).toBe("subscription");
+ expect(hopBilling("openai")).toBe("metered");
+ expect(hopBilling("ollama")).toBe("local");
+ });
+}