Skip to content

Claude Code. Any Model. The most powerful AI coding agent now speaks every language.

fix(tui): stop Test All failing every provider when the probe catalog is unreadable

Jack Rudenkoe5d3cc543ee5
diff --git a/packages/cli/src/providers/probe-catalog.test.ts b/packages/cli/src/providers/probe-catalog.test.ts
index d9633f45..956e0f04 100644
--- a/packages/cli/src/providers/probe-catalog.test.ts
+++ b/packages/cli/src/providers/probe-catalog.test.ts
@@ -14,6 +14,7 @@ import { join } from "node:path";
import {
type ProbeModelsResponse,
+ describeProbeCatalogFailure,
fetchProbeModels,
isCacheFresh,
readProbeModelsCache,
@@ -39,6 +40,12 @@ const SAMPLE: ProbeModelsResponse = {
},
};
+const REAL_426_BODY =
+ '{"contractVersion":3,"error":{"code":"catalog_client_upgrade_required","message":"This endpoint requires catalog contract version 3","minimumContractVersion":3}}';
+
+const REAL_V3_PROBE_MODELS_BODY =
+ '{"contractVersion":3,"generationId":"g-20260918072314542-9c5a9567","generatedAt":"2026-09-18T07:23:14.542Z","data":{"routes":{"deepseek/direct-api":{"modelId":"deepseek-v4.1-flash","externalModelId":"deepseek-flash","route":{"routeId":"deepseek","routeProfileId":"direct-api"},"confidence":"api_official"}}}}';
+
describe("readProbeModelsCache / writeProbeModelsCache", () => {
let tmp: ReturnType<typeof makeTmpPath>;
beforeEach(() => {
@@ -139,9 +146,33 @@ describe("fetchProbeModels", () => {
if (outcome.kind === "ok") expect(outcome.data).toEqual(SAMPLE);
});
- test("returns http with status on non-2xx", async () => {
+ test("returns incompatible with the server contract version on 426", async () => {
+ globalThis.fetch = mock(
+ async () => new Response(REAL_426_BODY, { status: 426 })
+ ) as unknown as typeof fetch;
+ const outcome = await fetchProbeModels("http://stub.local", 1000);
+ expect(outcome).toEqual({ kind: "incompatible", serverContractVersion: 3 });
+ });
+
+ test("returns incompatible with an unknown server contract version on an empty 426", async () => {
globalThis.fetch = mock(
- async () => new Response("", { status: 503 })
+ async () => new Response(null, { status: 426 })
+ ) as unknown as typeof fetch;
+ const outcome = await fetchProbeModels("http://stub.local", 1000);
+ expect(outcome).toEqual({ kind: "incompatible", serverContractVersion: null });
+ });
+
+ test("returns incompatible before validating the shape of a newer 200 response", async () => {
+ globalThis.fetch = mock(
+ async () => new Response(REAL_V3_PROBE_MODELS_BODY, { status: 200 })
+ ) as unknown as typeof fetch;
+ const outcome = await fetchProbeModels("http://stub.local", 1000);
+ expect(outcome).toEqual({ kind: "incompatible", serverContractVersion: 3 });
+ });
+
+ test("returns http with status on a non-contract non-2xx response with no body", async () => {
+ globalThis.fetch = mock(
+ async () => new Response(null, { status: 503 })
) as unknown as typeof fetch;
const outcome = await fetchProbeModels("http://stub.local", 1000);
expect(outcome).toEqual({ kind: "http", status: 503 });
@@ -175,6 +206,36 @@ describe("fetchProbeModels", () => {
});
});
+describe("describeProbeCatalogFailure", () => {
+ test("describes a known incompatible version without blaming connectivity", () => {
+ const message = describeProbeCatalogFailure({
+ kind: "incompatible",
+ serverContractVersion: 3,
+ });
+ expect(message).toContain("contract v3");
+ expect(message).not.toContain("could not reach");
+ });
+
+ test("describes an unknown incompatible version without blaming connectivity", () => {
+ const message = describeProbeCatalogFailure({
+ kind: "incompatible",
+ serverContractVersion: null,
+ });
+ expect(message).not.toContain("could not reach");
+ });
+
+ test("describes timeout and network failures as connectivity failures", () => {
+ expect(describeProbeCatalogFailure({ kind: "timeout" })).toContain("could not reach");
+ expect(describeProbeCatalogFailure({ kind: "network", reason: "ECONNREFUSED" })).toContain(
+ "could not reach"
+ );
+ });
+
+ test("includes the HTTP status in an HTTP failure", () => {
+ expect(describeProbeCatalogFailure({ kind: "http", status: 503 })).toContain("503");
+ });
+});
+
describe("getProbeModel", () => {
// getProbeModel reads from ~/.claudish/probe-models.json directly. The
// shape contract — string-valued, non-empty, indexed by claudish provider
diff --git a/packages/cli/src/providers/probe-catalog.ts b/packages/cli/src/providers/probe-catalog.ts
index dc6e9361..359f19b4 100644
--- a/packages/cli/src/providers/probe-catalog.ts
+++ b/packages/cli/src/providers/probe-catalog.ts
@@ -12,14 +12,22 @@
* `models-index/TASK_probe_models_endpoint.md`.
*
* Lazy fetch: the first caller to need a probe model triggers a network
- * fetch. Subsequent reads hit the disk cache. On fetch failure the cache
- * stays empty and callers see `null` — the TUI surfaces this as
- * "could not reach model catalog" rather than running with stale data.
+ * fetch. Subsequent reads hit the disk cache. A failed fetch never overwrites
+ * the cache, and `getProbeModel` returns the last cached pick whatever its age.
+ *
+ * Using an old pick is safe, which is why a failed fetch is not fatal to the
+ * TUI. The pick is only the first model to TRY: the probe is a live request to
+ * the provider, so an outdated pick cannot produce a false pass. At worst it
+ * fails model-not-found and the TUI moves on to endpoint discovery. Failing
+ * every provider on a failed fetch — the previous behaviour — turned the v3
+ * cutover (this endpoint answers 426 to a v2 build) into 18 red rows, none of
+ * which had contacted its provider.
*/
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { homedir } from "node:os";
import { dirname, join } from "node:path";
+import { isIncompatibleContractVersion, parseContractEnvelope } from "./catalog-compatibility.js";
const PROBE_MODELS_URL = "https://us-central1-claudish-6da10.cloudfunctions.net/probeModels";
// 1h TTL (matches the server's own instance cache). A 24h TTL meant a
@@ -48,10 +56,43 @@ export type FetchOutcome =
| { kind: "timeout" }
| { kind: "network"; reason: string }
| { kind: "http"; status: number }
- | { kind: "invalid"; reason: string };
+ | { kind: "invalid"; reason: string }
+ /**
+ * The endpoint answered in a catalog contract this build cannot read: a 426,
+ * or a body whose `contractVersion` is newer than this build supports. Kept
+ * apart from `http` because it is not a connectivity problem, and a caller
+ * that reports it as one sends the user to debug a working network.
+ */
+ | { kind: "incompatible"; serverContractVersion: number | null };
let _inFlight: Promise<FetchOutcome> | null = null;
+/**
+ * One line naming why the probe catalog supplied no fresh pick, for the TUI.
+ *
+ * Only a connectivity failure says "could not reach". A contract mismatch says
+ * what it is: the network is fine, and a message that blames it sends the user
+ * to debug the wrong thing.
+ */
+export function describeProbeCatalogFailure(
+ outcome: Exclude<FetchOutcome, { kind: "ok" }>
+): string {
+ switch (outcome.kind) {
+ case "incompatible":
+ return outcome.serverContractVersion === null
+ ? "model catalog uses a newer contract than this build reads"
+ : `model catalog uses contract v${outcome.serverContractVersion}, newer than this build reads`;
+ case "http":
+ return `model catalog returned HTTP ${outcome.status}`;
+ case "timeout":
+ return "could not reach model catalog (timeout)";
+ case "network":
+ return `could not reach model catalog (${outcome.reason})`;
+ case "invalid":
+ return `model catalog response unreadable (${outcome.reason})`;
+ }
+}
+
export function readProbeModelsCache(
path: string = PROBE_MODELS_CACHE_PATH
): ProbeModelsResponse | null {
@@ -102,7 +143,22 @@ export async function fetchProbeModels(
};
}
- if (!response.ok) return { kind: "http", status: response.status };
+ if (!response.ok) {
+ // /probeModels negotiates the catalog contract like queryModels does, so at
+ // the v3 cutover it answers this build 426. Read the body before calling it
+ // a plain HTTP failure. The body is optional: a bare 426 is still a verdict.
+ let errorBody: unknown = null;
+ try {
+ errorBody = await response.json();
+ } catch {
+ // No JSON body. The status alone decides below.
+ }
+ const envelope = parseContractEnvelope(errorBody);
+ if (response.status === 426 || isIncompatibleContractVersion(envelope.contractVersion)) {
+ return { kind: "incompatible", serverContractVersion: envelope.contractVersion };
+ }
+ return { kind: "http", status: response.status };
+ }
let body: unknown;
try {
@@ -113,6 +169,13 @@ export async function fetchProbeModels(
reason: e instanceof Error ? e.message : "json parse error",
};
}
+ // A 200 in a newer contract is the same verdict. It must be caught here,
+ // before `isValidResponse`, which would otherwise report it as "missing
+ // providers map" and hide the real cause.
+ const bodyEnvelope = parseContractEnvelope(body);
+ if (isIncompatibleContractVersion(bodyEnvelope.contractVersion)) {
+ return { kind: "incompatible", serverContractVersion: bodyEnvelope.contractVersion };
+ }
if (!isValidResponse(body)) {
return { kind: "invalid", reason: "missing providers map" };
}
diff --git a/packages/cli/src/tui/App.tsx b/packages/cli/src/tui/App.tsx
index cb801890..2eb9c33b 100644
--- a/packages/cli/src/tui/App.tsx
+++ b/packages/cli/src/tui/App.tsx
@@ -88,6 +88,7 @@ import {
} from "../providers/onepassword.js";
import type { SdkAuth } from "../providers/onepassword.js";
import {
+ describeProbeCatalogFailure,
discoverProbeModelFromEndpoint,
ensureProbeModelsCached,
forceRefreshProbeModels,
@@ -1305,17 +1306,15 @@ export function App({ requestLogin }: AppProps = {}) {
}
}
- const outcome = await ensureProbeModelsCached();
- if (outcome.kind !== "ok") {
- setTestResults((prev) => ({
- ...prev,
- [provName]: {
- status: "failed",
- error: `could not reach model catalog (${outcome.kind})`,
- },
- }));
- return;
- }
+ // A failed probe-catalog fetch is NOT fatal. The catalog only supplies the
+ // FIRST model to try: the probe below is a live request, so the last cached
+ // pick (whatever its age) cannot produce a false pass, and endpoint
+ // discovery covers a provider with no pick at all. This once returned a
+ // failure here instead, and when /probeModels moved to catalog contract v3
+ // (a 426 to this build) every provider went red without being contacted.
+ const catalogOutcome = await ensureProbeModelsCached();
+ const catalogProblem =
+ catalogOutcome.kind === "ok" ? undefined : describeProbeCatalogFailure(catalogOutcome);
const startMs = Date.now();
try {
@@ -1399,9 +1398,20 @@ export function App({ requestLogin }: AppProps = {}) {
...prev,
[provName]: {
status: prov.isLocal ? "unavailable" : "failed",
- error: lastDiscoveryReason
- ? `no probe model: ${lastDiscoveryReason}`
- : "no probe model available",
+ // Name the catalog problem too when there was one: with no pick
+ // AND no discovery, both sources failed, and the discovery reason
+ // alone would hide the one the user can act on.
+ error: [
+ lastDiscoveryReason
+ ? `no probe model: ${lastDiscoveryReason}`
+ : "no probe model available",
+ catalogProblem,
+ ]
+ .filter((part): part is string => Boolean(part))
+ // Both parts can be full sentences ("The operation timed out."),
+ // so drop a trailing period rather than print ".;" between them.
+ .map((part) => part.replace(/\.$/, ""))
+ .join(" · "),
ms,
},
}));