Claude Code. Any Model. The most powerful AI coding agent now speaks every language.
fix(tui): stop Test All failing every provider when the probe catalog is unreadable
Jack Rudenkoe5d3cc543ee5
diff --git a/packages/cli/src/providers/probe-catalog.test.ts b/packages/cli/src/providers/probe-catalog.test.tsindex d9633f45..956e0f04 100644--- a/packages/cli/src/providers/probe-catalog.test.ts+++ b/packages/cli/src/providers/probe-catalog.test.ts@@ -14,6 +14,7 @@ import { join } from "node:path";import {type ProbeModelsResponse,+ describeProbeCatalogFailure,fetchProbeModels,isCacheFresh,readProbeModelsCache,@@ -39,6 +40,12 @@ const SAMPLE: ProbeModelsResponse = {},};+const REAL_426_BODY =+ '{"contractVersion":3,"error":{"code":"catalog_client_upgrade_required","message":"This endpoint requires catalog contract version 3","minimumContractVersion":3}}';++const REAL_V3_PROBE_MODELS_BODY =+ '{"contractVersion":3,"generationId":"g-20260918072314542-9c5a9567","generatedAt":"2026-09-18T07:23:14.542Z","data":{"routes":{"deepseek/direct-api":{"modelId":"deepseek-v4.1-flash","externalModelId":"deepseek-flash","route":{"routeId":"deepseek","routeProfileId":"direct-api"},"confidence":"api_official"}}}}';+describe("readProbeModelsCache / writeProbeModelsCache", () => {let tmp: ReturnType<typeof makeTmpPath>;beforeEach(() => {@@ -139,9 +146,33 @@ describe("fetchProbeModels", () => {if (outcome.kind === "ok") expect(outcome.data).toEqual(SAMPLE);});- test("returns http with status on non-2xx", async () => {+ test("returns incompatible with the server contract version on 426", async () => {+ globalThis.fetch = mock(+ async () => new Response(REAL_426_BODY, { status: 426 })+ ) as unknown as typeof fetch;+ const outcome = await fetchProbeModels("http://stub.local", 1000);+ expect(outcome).toEqual({ kind: "incompatible", serverContractVersion: 3 });+ });++ test("returns incompatible with an unknown server contract version on an empty 426", async () => {globalThis.fetch = mock(- async () => new Response("", { status: 503 })+ async () => new Response(null, { status: 426 })+ ) as unknown as typeof fetch;+ const outcome = await fetchProbeModels("http://stub.local", 1000);+ expect(outcome).toEqual({ kind: "incompatible", serverContractVersion: null });+ });++ test("returns incompatible before validating the shape of a newer 200 response", async () => {+ globalThis.fetch = mock(+ async () => new Response(REAL_V3_PROBE_MODELS_BODY, { status: 200 })+ ) as unknown as typeof fetch;+ const outcome = await fetchProbeModels("http://stub.local", 1000);+ expect(outcome).toEqual({ kind: "incompatible", serverContractVersion: 3 });+ });++ test("returns http with status on a non-contract non-2xx response with no body", async () => {+ globalThis.fetch = mock(+ async () => new Response(null, { status: 503 })) as unknown as typeof fetch;const outcome = await fetchProbeModels("http://stub.local", 1000);expect(outcome).toEqual({ kind: "http", status: 503 });@@ -175,6 +206,36 @@ describe("fetchProbeModels", () => {});});+describe("describeProbeCatalogFailure", () => {+ test("describes a known incompatible version without blaming connectivity", () => {+ const message = describeProbeCatalogFailure({+ kind: "incompatible",+ serverContractVersion: 3,+ });+ expect(message).toContain("contract v3");+ expect(message).not.toContain("could not reach");+ });++ test("describes an unknown incompatible version without blaming connectivity", () => {+ const message = describeProbeCatalogFailure({+ kind: "incompatible",+ serverContractVersion: null,+ });+ expect(message).not.toContain("could not reach");+ });++ test("describes timeout and network failures as connectivity failures", () => {+ expect(describeProbeCatalogFailure({ kind: "timeout" })).toContain("could not reach");+ expect(describeProbeCatalogFailure({ kind: "network", reason: "ECONNREFUSED" })).toContain(+ "could not reach"+ );+ });++ test("includes the HTTP status in an HTTP failure", () => {+ expect(describeProbeCatalogFailure({ kind: "http", status: 503 })).toContain("503");+ });+});+describe("getProbeModel", () => {// getProbeModel reads from ~/.claudish/probe-models.json directly. The// shape contract — string-valued, non-empty, indexed by claudish providerdiff --git a/packages/cli/src/providers/probe-catalog.ts b/packages/cli/src/providers/probe-catalog.tsindex dc6e9361..359f19b4 100644--- a/packages/cli/src/providers/probe-catalog.ts+++ b/packages/cli/src/providers/probe-catalog.ts@@ -12,14 +12,22 @@* `models-index/TASK_probe_models_endpoint.md`.** Lazy fetch: the first caller to need a probe model triggers a network- * fetch. Subsequent reads hit the disk cache. On fetch failure the cache- * stays empty and callers see `null` — the TUI surfaces this as- * "could not reach model catalog" rather than running with stale data.+ * fetch. Subsequent reads hit the disk cache. A failed fetch never overwrites+ * the cache, and `getProbeModel` returns the last cached pick whatever its age.+ *+ * Using an old pick is safe, which is why a failed fetch is not fatal to the+ * TUI. The pick is only the first model to TRY: the probe is a live request to+ * the provider, so an outdated pick cannot produce a false pass. At worst it+ * fails model-not-found and the TUI moves on to endpoint discovery. Failing+ * every provider on a failed fetch — the previous behaviour — turned the v3+ * cutover (this endpoint answers 426 to a v2 build) into 18 red rows, none of+ * which had contacted its provider.*/import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";import { homedir } from "node:os";import { dirname, join } from "node:path";+import { isIncompatibleContractVersion, parseContractEnvelope } from "./catalog-compatibility.js";const PROBE_MODELS_URL = "https://us-central1-claudish-6da10.cloudfunctions.net/probeModels";// 1h TTL (matches the server's own instance cache). A 24h TTL meant a@@ -48,10 +56,43 @@ export type FetchOutcome =| { kind: "timeout" }| { kind: "network"; reason: string }| { kind: "http"; status: number }- | { kind: "invalid"; reason: string };+ | { kind: "invalid"; reason: string }+ /**+ * The endpoint answered in a catalog contract this build cannot read: a 426,+ * or a body whose `contractVersion` is newer than this build supports. Kept+ * apart from `http` because it is not a connectivity problem, and a caller+ * that reports it as one sends the user to debug a working network.+ */+ | { kind: "incompatible"; serverContractVersion: number | null };let _inFlight: Promise<FetchOutcome> | null = null;+/**+ * One line naming why the probe catalog supplied no fresh pick, for the TUI.+ *+ * Only a connectivity failure says "could not reach". A contract mismatch says+ * what it is: the network is fine, and a message that blames it sends the user+ * to debug the wrong thing.+ */+export function describeProbeCatalogFailure(+ outcome: Exclude<FetchOutcome, { kind: "ok" }>+): string {+ switch (outcome.kind) {+ case "incompatible":+ return outcome.serverContractVersion === null+ ? "model catalog uses a newer contract than this build reads"+ : `model catalog uses contract v${outcome.serverContractVersion}, newer than this build reads`;+ case "http":+ return `model catalog returned HTTP ${outcome.status}`;+ case "timeout":+ return "could not reach model catalog (timeout)";+ case "network":+ return `could not reach model catalog (${outcome.reason})`;+ case "invalid":+ return `model catalog response unreadable (${outcome.reason})`;+ }+}+export function readProbeModelsCache(path: string = PROBE_MODELS_CACHE_PATH): ProbeModelsResponse | null {@@ -102,7 +143,22 @@ export async function fetchProbeModels(};}- if (!response.ok) return { kind: "http", status: response.status };+ if (!response.ok) {+ // /probeModels negotiates the catalog contract like queryModels does, so at+ // the v3 cutover it answers this build 426. Read the body before calling it+ // a plain HTTP failure. The body is optional: a bare 426 is still a verdict.+ let errorBody: unknown = null;+ try {+ errorBody = await response.json();+ } catch {+ // No JSON body. The status alone decides below.+ }+ const envelope = parseContractEnvelope(errorBody);+ if (response.status === 426 || isIncompatibleContractVersion(envelope.contractVersion)) {+ return { kind: "incompatible", serverContractVersion: envelope.contractVersion };+ }+ return { kind: "http", status: response.status };+ }let body: unknown;try {@@ -113,6 +169,13 @@ export async function fetchProbeModels(reason: e instanceof Error ? e.message : "json parse error",};}+ // A 200 in a newer contract is the same verdict. It must be caught here,+ // before `isValidResponse`, which would otherwise report it as "missing+ // providers map" and hide the real cause.+ const bodyEnvelope = parseContractEnvelope(body);+ if (isIncompatibleContractVersion(bodyEnvelope.contractVersion)) {+ return { kind: "incompatible", serverContractVersion: bodyEnvelope.contractVersion };+ }if (!isValidResponse(body)) {return { kind: "invalid", reason: "missing providers map" };}diff --git a/packages/cli/src/tui/App.tsx b/packages/cli/src/tui/App.tsxindex cb801890..2eb9c33b 100644--- a/packages/cli/src/tui/App.tsx+++ b/packages/cli/src/tui/App.tsx@@ -88,6 +88,7 @@ import {} from "../providers/onepassword.js";import type { SdkAuth } from "../providers/onepassword.js";import {+ describeProbeCatalogFailure,discoverProbeModelFromEndpoint,ensureProbeModelsCached,forceRefreshProbeModels,@@ -1305,17 +1306,15 @@ export function App({ requestLogin }: AppProps = {}) {}}- const outcome = await ensureProbeModelsCached();- if (outcome.kind !== "ok") {- setTestResults((prev) => ({- ...prev,- [provName]: {- status: "failed",- error: `could not reach model catalog (${outcome.kind})`,- },- }));- return;- }+ // A failed probe-catalog fetch is NOT fatal. The catalog only supplies the+ // FIRST model to try: the probe below is a live request, so the last cached+ // pick (whatever its age) cannot produce a false pass, and endpoint+ // discovery covers a provider with no pick at all. This once returned a+ // failure here instead, and when /probeModels moved to catalog contract v3+ // (a 426 to this build) every provider went red without being contacted.+ const catalogOutcome = await ensureProbeModelsCached();+ const catalogProblem =+ catalogOutcome.kind === "ok" ? undefined : describeProbeCatalogFailure(catalogOutcome);const startMs = Date.now();try {@@ -1399,9 +1398,20 @@ export function App({ requestLogin }: AppProps = {}) {...prev,[provName]: {status: prov.isLocal ? "unavailable" : "failed",- error: lastDiscoveryReason- ? `no probe model: ${lastDiscoveryReason}`- : "no probe model available",+ // Name the catalog problem too when there was one: with no pick+ // AND no discovery, both sources failed, and the discovery reason+ // alone would hide the one the user can act on.+ error: [+ lastDiscoveryReason+ ? `no probe model: ${lastDiscoveryReason}`+ : "no probe model available",+ catalogProblem,+ ]+ .filter((part): part is string => Boolean(part))+ // Both parts can be full sentences ("The operation timed out."),+ // so drop a trailing period rather than print ".;" between them.+ .map((part) => part.replace(/\.$/, ""))+ .join(" · "),ms,},}));