Skip to content

Claude Code. Any Model. The most powerful AI coding agent now speaks every language.

refactor(adapters): extract FallbackHandler's hop decision

Jack Rudenkoa41955b455bb
diff --git a/packages/cli/src/handlers/fallback-handler.ts b/packages/cli/src/handlers/fallback-handler.ts
index 046c832c..b34860c7 100644
--- a/packages/cli/src/handlers/fallback-handler.ts
+++ b/packages/cli/src/handlers/fallback-handler.ts
@@ -14,18 +14,21 @@ import { log, logStderr } from "../logger.js";
import { redactForClient } from "../redact.js";
import { ComposedHandler, getRecoveryHint } from "./composed-handler.js";
import { extractUpstreamStatus } from "./shared/anthropic-error.js";
-import { authVerdict, isCredentialRejection } from "./shared/auth-failure.js";
+import { authVerdict } from "./shared/auth-failure.js";
import { abortedResponse, callAborted } from "./shared/call-context.js";
+import { decideHopFailure } from "./shared/hop-decision.js";
import { hasQuotaExhaustionWording } from "./shared/quota-exhaustion.js";
-import {
- hasStreamFailureMarker,
- isClaudishConnectionVerdict,
- isRecoveryHoldResponse,
- isStreamFailureResponse,
-} from "./shared/recovery-marker.js";
import { isSubscriptionProvider } from "./shared/remote-provider-types.js";
import type { ModelHandler } from "./types.js";
+// Moved to `hop-decision.ts`, which the probe shares; re-exported for the
+// existing importers and tests.
+export {
+ isProvider,
+ isRetryableError,
+ isSubscriptionCredentialRejection,
+} from "./shared/hop-decision.js";
+
export interface FallbackCandidate {
/** Human-readable provider name for logging */
name: string;
@@ -102,120 +105,101 @@ export class FallbackHandler implements ModelHandler {
return response;
}
- // ── THE RECOVERY MARKER, CHECKED BEFORE THE BODY IS EVEN READ ─────────
- //
- // A recovery-exhaustion 503 is OUR status, not an upstream one. It says
- // "claudish held this request for its whole deadline, could not reach
- // the host, and a surface is painting that fact" — so it must leave
- // this handler UNTOUCHED, for two separate reasons:
- //
- // 1. It is not a reason to advance the chain. During a network outage
- // the next candidate is unreachable for the same reason, so the
- // chain would burn every candidate — and per the standing CLAUDE.md
- // invariant, advancing off a SUBSCRIPTION_PROVIDERS candidate onto
- // a metered one quotes the user a real per-token price for a fault
- // that had nothing to do with any provider.
- // 2. It must not be FOLDED into a combined error either. Below,
- // `formatCombinedError` emits `exhaustedChainStatus(errors)`, which
- // returns 503 only if EVERY accumulated error is transient — so one
- // earlier auth/404 failure would turn our 503 into a terminal 400,
- // Claude Code would never re-POST, and tier 2 would never begin.
- //
- // Returning it verbatim answers both at once. This is a HEADER and not a
- // status, so the "read the upstream status via `extractUpstreamStatus`,
- // never the wire status" doctrine is untouched. It cannot be forged by
- // an upstream — see `recovery-marker.ts` for the two source facts that
- // make that structural rather than hopeful.
- //
- // NOT a wording rule, and that is the whole point: `isRetryableError`'s
- // first statement is a status-agnostic phrase match whose list contains
- // the bare substring "quota", so a 503 whose message merely MENTIONED a
- // quota would have advanced the chain and spent the user's money.
- if (isRecoveryHoldResponse(response)) {
- logStderr(
- `[Fallback] ${name} is unreachable and claudish is still retrying it — ` +
- "holding the chain here rather than switching providers mid-outage."
- );
- return response;
- }
-
- // ── THE STREAM-FAILURE MARKER, SAME PLACE AND SAME REASONS ────────────
- //
- // A non-streamed turn whose stream ended on an SSE `error` (the provider
- // answered 200, then the stream failed) is answered by the JSON collector
- // with an error response. In streaming mode the same failure never comes
- // back through here as an error, so neither may this one: it is not the
- // provider's verdict on the request, and its body can quote an upstream
- // message containing quota wording. Returned verbatim, before the body is
- // read, so it is neither advanced past nor folded with earlier errors.
- if (isStreamFailureResponse(response)) {
- return response;
- }
+ // Clone before reading body so we can still return the original if needed.
+ // Read BEFORE the decision, also for a recovery hold, a stream failure and
+ // a credential verdict: all three are claudish's own small JSON bodies,
+ // and each is still returned verbatim below.
+ const errorBody = await response.clone().text();
- // A CREDENTIAL VERDICT leaves this handler untouched too, for the
- // recovery hold's second reason: folded into `formatCombinedError`, a
- // transient 503 behind an earlier candidate's 404 became a terminal 400
- // (`exhaustedChainStatus` is 503 only when EVERY error is transient), so
- // Claude Code stopped instead of retrying the same hop, and the verdict
- // header was dropped. A terminal verdict keeps its exact message and
- // `upstream_status` the same way (`auth-failure.ts`).
- //
- // One exception, for a TERMINAL verdict on a subscription (an expired,
- // wrong or revoked credential): the chain moves on to its next hop,
- // metered included (project owner, 2026-10-21). An expired key must not
- // block the chain, and a metered key the user configured is consent to
- // be billed by it. The move is announced, once per provider and status
- // (`announceCredentialRejection`). The recovery hold and the
- // stream-failure marker are returned above, before this point. The 400
- // arm's connection marker (`x-claudish-connection-error`) is not, so it
- // is excluded here from BOTH the header case and the body-wording case.
- // With a credential verdict header it is returned verbatim just below;
- // without one it reaches `isRetryableError`, which refuses it first, and
- // it is folded or returned as any other stop. A network fault
- // therefore never walks onto metered billing, even when its body quotes
- // an upstream 401 in credential wording.
- const connectionFault = isClaudishConnectionVerdict(response.headers);
- const verdict = authVerdict(response.headers);
- const subscriptionRejected =
- !connectionFault &&
- verdict === "terminal" &&
- !!provider &&
- isSubscriptionProvider(provider);
- if (verdict && !subscriptionRejected) {
- if (errors.length > 0) {
+ // ONE decision, shared with the probe (`hop-decision.ts`, which carries
+ // the reasons for its order). Each reason maps onto exactly the branch
+ // this handler has always taken for it.
+ const decision = decideHopFailure({
+ kind: "response",
+ status: response.status,
+ headers: response.headers,
+ body: errorBody,
+ provider,
+ name,
+ });
+
+ switch (decision.reason) {
+ // ── THE RECOVERY MARKER ─────────────────────────────────────────────
+ //
+ // A recovery-exhaustion 503 is OUR status, not an upstream one. It says
+ // "claudish held this request for its whole deadline, could not reach
+ // the host, and a surface is painting that fact" — so it must leave
+ // this handler UNTOUCHED, for two separate reasons:
+ //
+ // 1. It is not a reason to advance the chain. During a network outage
+ // the next candidate is unreachable for the same reason, so the
+ // chain would burn every candidate — and per the standing CLAUDE.md
+ // invariant, advancing off a SUBSCRIPTION_PROVIDERS candidate onto
+ // a metered one quotes the user a real per-token price for a fault
+ // that had nothing to do with any provider.
+ // 2. It must not be FOLDED into a combined error either. Below,
+ // `formatCombinedError` emits `exhaustedChainStatus(errors)`, which
+ // returns 503 only if EVERY accumulated error is transient — so one
+ // earlier auth/404 failure would turn our 503 into a terminal 400,
+ // Claude Code would never re-POST, and tier 2 would never begin.
+ //
+ // Returning it verbatim answers both at once.
+ case "recovery-hold":
logStderr(
- `[Fallback] ${name}: credential ${verdict === "terminal" ? "rejected" : "temporarily unavailable"} — stopping the chain here.`
+ `[Fallback] ${name} is unreachable and claudish is still retrying it — ` +
+ "holding the chain here rather than switching providers mid-outage."
);
- }
- return response;
- }
-
- // Clone before reading body so we can still return the original if needed
- const errorBody = await response.clone().text();
-
- if (
- subscriptionRejected ||
- (!connectionFault &&
- isSubscriptionCredentialRejection(response.status, errorBody, provider))
- ) {
- errors.push({ provider: name, status: response.status, message: errorBody });
- if (!firstRejection) firstRejection = { name, response, body: errorBody };
- if (!isLast) {
- const next = this.candidates[(idx + 1) % this.candidates.length];
- announceCredentialRejection({ name, provider }, response.status, errorBody, next);
- }
- continue;
- }
-
- // Non-retryable error (rate limit, server error, bad format) — stop trying
- if (!isRetryableError(response.status, errorBody, name, response.headers, provider)) {
- if (errors.length > 0) {
- // We had previous fallback attempts; show combined error
+ return response;
+
+ // ── THE STREAM-FAILURE MARKER, SAME REASONS ─────────────────────────
+ //
+ // A non-streamed turn whose stream ended on an SSE `error` (the provider
+ // answered 200, then the stream failed) is answered by the JSON collector
+ // with an error response. In streaming mode the same failure never comes
+ // back through here as an error, so neither may this one: it is not the
+ // provider's verdict on the request. Neither advanced past nor folded.
+ case "stream-failure":
+ return response;
+
+ // A CREDENTIAL VERDICT leaves this handler untouched too, for the
+ // recovery hold's second reason: folded into `formatCombinedError`, a
+ // transient 503 behind an earlier candidate's 404 became a terminal 400,
+ // so Claude Code stopped instead of retrying the same hop, and the
+ // verdict header was dropped. A terminal verdict keeps its exact message
+ // and `upstream_status` the same way (`auth-failure.ts`).
+ case "credential-hold":
+ if (errors.length > 0) {
+ const verdict = authVerdict(response.headers);
+ logStderr(
+ `[Fallback] ${name}: credential ${verdict === "terminal" ? "rejected" : "temporarily unavailable"} — stopping the chain here.`
+ );
+ }
+ return response;
+
+ // A TERMINAL verdict on a subscription (an expired, wrong or revoked
+ // credential) moves to the next hop, metered included (project owner,
+ // 2026-10-21): an expired key must not block the chain, and a metered
+ // key the user configured is consent to be billed by it. The move is
+ // announced, once per provider and status.
+ case "subscription-credential-rejected": {
errors.push({ provider: name, status: response.status, message: errorBody });
- return this.formatCombinedError(c, errors, payload.model, firstRejection);
+ if (!firstRejection) firstRejection = { name, response, body: errorBody };
+ if (!isLast) {
+ const next = this.candidates[(idx + 1) % this.candidates.length];
+ announceCredentialRejection({ name, provider }, response.status, errorBody, next);
+ }
+ continue;
}
- // First and only attempt — return original response as-is
- return response;
+
+ // Non-retryable error (rate limit, server error, bad format) — stop trying
+ case "non-retryable":
+ if (errors.length > 0) {
+ // We had previous fallback attempts; show combined error
+ errors.push({ provider: name, status: response.status, message: errorBody });
+ return this.formatCombinedError(c, errors, payload.model, firstRejection);
+ }
+ // First and only attempt — return original response as-is
+ return response;
}
// Retryable (auth/not-found) — log and try next provider
@@ -241,9 +225,17 @@ export class FallbackHandler implements ModelHandler {
}
} catch (err: any) {
if (callAborted(c)) return abortedResponse();
- errors.push({ provider: name, status: 0, message: redactForClient(String(err?.message)) });
- if (!isLast) {
- logStderr(`[Fallback] ${name} error: ${err.message}, trying next provider...`);
+ // A thrown hop advances (`decideHopFailure({kind: "thrown"})`), as it
+ // always has; the probe judges a thrown hop by the same rule.
+ if (decideHopFailure({ kind: "thrown" }).action === "advance") {
+ errors.push({
+ provider: name,
+ status: 0,
+ message: redactForClient(String(err?.message)),
+ });
+ if (!isLast) {
+ logStderr(`[Fallback] ${name} error: ${err.message}, trying next provider...`);
+ }
}
}
}
@@ -311,37 +303,6 @@ export class FallbackHandler implements ModelHandler {
}
}
-/**
- * A subscription candidate's dead or rejected login, answered WITHOUT the
- * verdict header. The header covers every 401 a ComposedHandler places; this
- * covers a candidate that answers one without it, so the rule holds for every
- * subscription provider and not only for the code paths that mint the header.
- * Asked of the uid — a display name is never a subscription.
- */
-export function isSubscriptionCredentialRejection(
- status: number,
- errorBody: string,
- providerId?: string
-): boolean {
- if (!providerId || !isSubscriptionProvider(providerId)) return false;
- const recovered = status === 400 ? extractUpstreamStatus(errorBody) : undefined;
- // A credential rejection only: a 401 that says the MODEL is not carried
- // (OpenCode Zen Go's "Model <id> is not supported") is a per-model miss and
- // advances, as it always did (`isCredentialRejection`).
- if ((status === 401 || recovered === 401) && isCredentialRejection(401, errorBody)) {
- return true;
- }
- // A RAW 403 credential rejection counts too (measured: `403
- // {"error":{"type":"invalid_api_key"}}` from a subscription hop advanced
- // onto the metered `meta@`). Raw only: the upstream body still carries the
- // provider's own `error.type`, so `isCredentialRejection` can tell Zen Go's
- // `403 RegionError` (a model opt-in, which advances) from a dead login. A
- // RECOVERED 403 cannot be told apart that way — claudish's remapped 400
- // body says `invalid_request_error` — so it keeps advancing; a 403 the
- // ComposedHandler placed already carries the terminal verdict header.
- return status === 403 && isCredentialRejection(403, errorBody);
-}
-
/** `<provider uid>|<upstream status>` of every rejection already announced on stderr. */
const announcedRejections = new Set<string>();
@@ -396,259 +357,6 @@ export function announceCredentialRejection(
logStderr(line);
}
-/**
- * Determine if an HTTP error is retryable (should try next provider).
- * Auth errors, billing errors, rate limits, model-not-found errors and an
- * unavailable endpoint (502/503/504) warrant trying a different provider. A
- * plain 500 does NOT: it may carry account state, and it is the one 5xx a
- * request's own payload can provoke on every provider alike.
- */
-export function isRetryableError(
- status: number,
- errorBody: string,
- provider?: string,
- headers?: Headers,
- providerId?: string
-): boolean {
- // ── CLAUDISH'S OWN "CANNOT REACH THE HOST" IS NEVER A REASON TO ADVANCE ────
- //
- // FIRST, above the quota wording check, and the ORDER is the whole point.
- // The next statement is `hasQuotaExhaustionWording(errorBody)`, which is
- // deliberately status-agnostic and whose phrase list contains the bare
- // substring "quota" — so one of our own connection verdicts that merely
- // mentioned one would be read as a spent subscription and walk the user onto
- // metered billing in the middle of a network outage. The error text quotes
- // the endpoint HOST and URL, so "merely mentioned" is not hypothetical: a
- // host named `quota-…` reproduced it, and only that word did.
- //
- // BOTH arms, via one predicate. The 503 handoff carries `x-claudish-recovery`
- // and is already returned verbatim by `handle()` above, so for that arm this
- // branch is a second independent mechanism. The 400 arm — every headless run,
- // where no banner can exist — carries `x-claudish-connection-error`, and for
- // IT this branch is the only mechanism there is. Asking the combined
- // predicate rather than either half is what stops chain-safety from being a
- // property of which arm happened to answer.
- if (isClaudishConnectionVerdict(headers)) return false;
- // A stream that failed after a 200 is not a provider's verdict either; see
- // `STREAM_FAILURE_HEADER` in recovery-marker.ts. Above the wording match for
- // the same reason as the line above.
- if (hasStreamFailureMarker(headers)) return false;
-
- // ── A CREDENTIAL VERDICT: TERMINAL ON A SUBSCRIPTION ADVANCES, ELSE HOLDS ──
- //
- // `ComposedHandler` marks its answer to a credential failure it could place
- // (`auth-failure.ts`): `terminal` — a credential that cannot succeed on
- // retry, answered 400 so the message lands inline — or `transient` — a
- // 5xx/429 on the credential path, which the SAME hop answers on Claude Code's
- // own retry, so the next candidate is the wrong remedy for it. A terminal
- // verdict on a SUBSCRIPTION hop (an expired, wrong or revoked credential)
- // advances to the next hop, metered included (project owner, 2026-10-21:
- // an expired key must not block the chain). A terminal verdict on any other
- // hop still holds. Above the quota wording and the `upstream_status`
- // recovery below, because a terminal 400 carries `upstream_status: 401`,
- // which that recovery reads as retryable whatever the hop.
- const verdict = authVerdict(headers);
- if (verdict) return verdict === "terminal" && !!providerId && isSubscriptionProvider(providerId);
-
- // A spent subscription allowance is retryable AT THE CHAIN LEVEL: this
- // provider cannot serve, but the next one can.
- //
- // Checked FIRST, and on wording rather than status, because the transport has
- // already decided this is terminal for itself and surfaced it as 400 (see the
- // "terminal errors become 400" doctrine). 400 is otherwise non-retryable, so a
- // status-based check here would stop the chain dead — which is exactly the
- // regression that showed up as a bare `minimax-m2.5` hard-failing while Zen
- // Go's 5-hour window was spent and metered MiniMax stood ready.
- //
- // The billing change this causes is announced rather than prevented — see the
- // warning at the advance site below.
- //
- // A first attempt made them terminal, to stop a user being moved from their
- // subscription onto per-token billing without being told. That was the wrong
- // lever, for two measured reasons:
- //
- // 1. It was unnecessary for the case it was meant to protect. An explicit
- // `kc@k3` / `zgo@model` spec resolves to exactly ONE candidate, so there
- // is nothing to advance to and the "Out of quota" error surfaces whatever
- // this function returns. Terminal only ever affected chains claudish
- // assembled on the user's behalf.
- // 2. In those chains it cost availability outright. With `opencode-zen-go`
- // now sitting ahead of the metered APIs, a bare `minimax-m2.5` hard-failed
- // whenever the Go plan's rolling 5-hour window was spent — a request that
- // had worked a moment earlier via the metered provider. Verified live:
- // Zen Go answers `429 GoUsageLimitError "5-hour usage limit reached.
- // Resets in 3hr 17min"`, and treating that as terminal stopped the chain
- // dead instead of falling through to a provider that was ready to serve.
- //
- // So the chain advances, and the billing change is made LOUD instead of being
- // prevented at the cost of the request.
- if (hasQuotaExhaustionWording(errorBody)) return true;
-
- // A SUBSCRIPTION's rejected credential, answered without the verdict header,
- // advances like the terminal verdict above (`handle()` announces it).
- if (isSubscriptionCredentialRejection(status, errorBody, providerId)) return true;
-
- // ── Recover the ORIGINAL status from a remapped terminal error (#148) ──────
- //
- // `composed-handler` remaps terminal upstream failures (401/403/terminal-429)
- // to 400 BEFORE this handler ever sees them, because a 400 is what makes
- // Claude Code render the reason inline instead of looping on "API error ·
- // Retrying". Each candidate in the chain IS a ComposedHandler, so by the time
- // the status reaches here it is 400, falls into the model-not-found branch
- // below, matches none of its phrases, and the chain STOPS at the first
- // provider.
- //
- // Which inverts the intent exactly. "Terminal" means THIS provider will not
- // recover on retry, and that is precisely when the next one should be tried.
- // The errors that most warrant a fallback were the only ones that could no
- // longer trigger one.
- //
- // The quota half of this was fixed first, by the wording check above — that
- // covers a spent plan, which is the common case. What is left is a genuine
- // credential failure: a revoked or rotated key hard-fails a model that three
- // other credentialed providers in the same chain would have served.
- //
- // Deliberately ADDITIVE. It only ever turns a `false` into a `true`, using
- // the status the upstream actually returned, and falls through to the
- // unchanged status logic when there is no `upstream_status` to recover. A
- // remapped 400 that carries something non-retryable (say an upstream 400) is
- // left to the existing branches, so nothing that used to surface now hides.
- const upstream = status === 400 ? extractUpstreamStatus(errorBody) : undefined;
- if (upstream === 401 || upstream === 403 || upstream === 402 || upstream === 429) {
- return true;
- }
-
- // Auth errors — different provider might have valid credentials
- if (status === 401 || status === 403) return true;
-
- // Payment required — billing/credit issue specific to this provider
- if (status === 402) return true;
-
- // Not found — model doesn't exist on this provider
- if (status === 404) return true;
-
- // Rate limited — per-provider limit, a different provider may have capacity
- if (status === 429) return true;
-
- // Unavailable — a gateway or overload answer is about THIS endpoint, not about
- // the model anywhere else. A bare `grok-4.7` stopped at its first candidate on
- // Zen Go's `503 Upstream request failed: Endpoint is unavailable.` while the
- // five candidates behind it answered (`--probe grok-4.7`, 2026-09-24), and
- // Claude Code's retry loop re-sent the same chain to the same dead hop.
- //
- // The old exclusion argued a 503 "cannot silently switch the user off the model
- // they pinned". A pinned `provider@model` builds no FallbackHandler at all
- // (proxy-server: one candidate → its handler directly), so inside a chain that
- // protected nothing. claudish's OWN "cannot reach the host" 503 still holds the
- // chain: the connection-verdict check at the top of this function answers first,
- // and `handle()` returns a recovery-hold response before it gets here.
- if (status === 502 || status === 503 || status === 504) return true;
-
- const lower = errorBody.toLowerCase();
-
- // Unprocessable (422) — some providers (OpenRouter) use this for model unavailability
- if (status === 422) {
- if (
- lower.includes("not available") ||
- lower.includes("model not found") ||
- lower.includes("not supported")
- ) {
- return true;
- }
- }
-
- // Bad request — only retryable if it's a model-not-found variant
- if (status === 400) {
- if (
- lower.includes("model not found") ||
- lower.includes("not registered") ||
- lower.includes("does not exist") ||
- lower.includes("unknown model") ||
- lower.includes("unsupported model") ||
- lower.includes("no healthy deployment") ||
- // Gemini Code Assist config-terminal error (the F1-F7 path returns 400 to
- // surface it inline for an EXPLICIT ag@ selection). But this handler
- // only runs for BARE-NAME auto-routing, where Gemini is just the first
- // candidate — a missing project / revoked-client verdict must advance the
- // chain to the next provider (e.g. OpenRouter), not abort it. When Gemini
- // is the LAST candidate the caller returns this same 400 anyway, so the
- // inline-surface behavior is preserved for the single-provider case.
- lower.includes("requires a google cloud project") ||
- lower.includes("unsupported_client")
- ) {
- return true;
- }
-
- // Antigravity rejects some SERVED variants at generation time with a
- // generic "Request contains an invalid argument" — `gemini-3.1-pro-high`
- // is in both the account's dynamic models catalog and the cloud models
- // catalog (marked the family default), yet 400s on every call. Without
- // this, a bare `gemini-3.1-pro-*` aborts the whole chain instead of
- // advancing to a provider that works, which is strictly worse than the 404
- // it used to produce.
- //
- // Deliberately scoped to this provider: "invalid argument" is generic
- // enough that a blanket rule would mask real payload bugs elsewhere by
- // silently advancing instead of surfacing them. Still bare-name only — an
- // explicit `ag@` selection has no next candidate, so the caller returns
- // this same 400 and the message reaches the user inline.
- if (provider?.toLowerCase().includes("antigravity") && lower.includes("invalid argument")) {
- return true;
- }
-
- // An AGGREGATOR reporting that its UPSTREAM rejected the request is terminal
- // for this candidate and says nothing about the next one — the two claims are
- // different, and conflating them costs the chain a provider that works.
- //
- // Zen Go fronts other vendors' models, so a rejection here is the upstream's
- // verdict on the request SHAPE, not on the model's availability anywhere:
- //
- // {"error":{"type":"server_error","message":"Error from provider (Console
- // Go): Upstream request failed: [bad_request_error] invalid params,
- // invalid tool type: (2013)"}}
- //
- // That wording matches none of the model-not-found tests above, so a bare
- // `minimax-m2.5` died at the first candidate while the metered MiniMax API and
- // OpenRouter sat behind it, credentialed and ready. Same defect class as the
- // spent-subscription case at the top of this function, one axis over: that one
- // separated "exhausted for this provider" from "exhausted for the chain", this
- // one separates "can't serve this request" from "nobody can serve it".
- //
- // Keyed on the aggregator's own framing (`Upstream request failed` / `Error
- // from provider (`) rather than on the upstream's message, which is
- // vendor-specific and unbounded. Provider-scoped for the same reason the
- // Antigravity rule above is: a blanket "advance on any 400" would silently
- // paper over real payload bugs in providers that speak for themselves.
- // NOTE ON THE PROVIDER STRING: what arrives here is the candidate's DISPLAY
- // name ("OpenCode Zen Go"), not the canonical id ("opencode-zen-go") —
- // proxy-server builds candidates as `{ name: candidate.displayName }`. So the
- // match is made on a punctuation-stripped form, which accepts both. The
- // Antigravity rule above only works because its display name happens to be a
- // single word; do not copy its literal shape for a hyphenated provider.
- if (
- isProvider(provider, "opencodezen") &&
- (lower.includes("upstream request failed") || lower.includes("error from provider ("))
- ) {
- return true;
- }
- }
-
- // Server errors (500) — only retryable if it's a billing/credit issue
- // (some providers misuse 500 for account-level problems)
- if (status === 500) {
- if (
- lower.includes("insufficient balance") ||
- lower.includes("insufficient credit") ||
- lower.includes("quota exceeded") ||
- lower.includes("billing")
- ) {
- return true;
- }
- }
-
- return false;
-}
-
/**
* The status an EXHAUSTED chain reports — terminal 400, or transient 503.
*
@@ -731,23 +439,6 @@ export function exhaustedChainStatus(
return errors.every(isTransient) ? 503 : 400;
}
-/**
- * Match a candidate against a provider family, tolerating the two spellings the
- * same provider reaches this file under: the canonical id (`opencode-zen-go`) and
- * the display name (`OpenCode Zen Go`). Both normalize to `opencodezengo`, so a
- * `needle` of `opencodezen` matches either, and both Zen tiers.
- *
- * Exported for tests — the display-name-vs-id distinction is exactly the kind of
- * thing that silently disables a rule.
- */
-export function isProvider(provider: string | undefined, needle: string): boolean {
- if (!provider) return false;
- return provider
- .toLowerCase()
- .replace(/[^a-z0-9]/g, "")
- .includes(needle);
-}
-
/**
* The upstream's own words from a claudish error body (`provider_message`,
* which carries no hint, provider name or status), else its message.
diff --git a/packages/cli/src/handlers/shared/hop-decision.ts b/packages/cli/src/handlers/shared/hop-decision.ts
new file mode 100644
index 00000000..0e51abb0
--- /dev/null
+++ b/packages/cli/src/handlers/shared/hop-decision.ts
@@ -0,0 +1,394 @@
+/**
+ * What a routing chain does after one hop failed: move to the next hop, or stop here.
+ *
+ * ONE function, {@link decideHopFailure}, shared by the two places that must agree:
+ * `FallbackHandler.handle` (the runtime walking a routing chain) and the probe
+ * (`probe-live.ts`), which preflight uses to say which hop of a chain would serve. A
+ * second copy of the rule is how preflight once reported "drop or replace" for a model
+ * whose later hop served (model-failures-2026-10-21.md, D6).
+ *
+ * The order is the runtime's, and it is the whole point. claudish's own recovery hold
+ * and stream-failure marker stop the chain before anything else; a credential verdict
+ * holds unless it is a terminal one on a subscription hop; a subscription's rejected
+ * credential advances; then `isRetryableError`, whose FIRST statement refuses claudish's
+ * connection verdict, so a network fault never walks a subscription user onto metered
+ * billing (CLAUDE.md, the five auth sites).
+ *
+ * Kept apart from `fallback-handler.ts` so the probe can import it without loading
+ * `composed-handler.ts` and the adapter stack behind it.
+ */
+import { extractUpstreamStatus } from "./anthropic-error.js";
+import { authVerdict, isCredentialRejection } from "./auth-failure.js";
+import { hasQuotaExhaustionWording } from "./quota-exhaustion.js";
+import {
+ hasRecoveryMarker,
+ hasStreamFailureMarker,
+ isClaudishConnectionVerdict,
+} from "./recovery-marker.js";
+import { isSubscriptionProvider } from "./remote-provider-types.js";
+
+/** Why a chain advanced or stopped at a failed hop. */
+export type HopFailureReason =
+ /** claudish's own recovery hold (`x-claudish-recovery`): stop, response verbatim. */
+ | "recovery-hold"
+ /** A stream that failed after a 200 (`x-claudish-stream-failure`): stop, verbatim. */
+ | "stream-failure"
+ /** A credential verdict that is not a subscription's terminal rejection: stop, verbatim. */
+ | "credential-hold"
+ /** A subscription's terminal verdict, or its credential rejection in the body: advance. */
+ | "subscription-credential-rejected"
+ /** `isRetryableError` true: advance. */
+ | "retryable"
+ /** `isRetryableError` false: stop. */
+ | "non-retryable"
+ /** The hop's handler threw: advance. */
+ | "thrown";
+
+export interface HopFailureDecision {
+ action: "advance" | "stop";
+ reason: HopFailureReason;
+}
+
+export type HopFailureInput =
+ | {
+ kind: "response";
+ status: number;
+ headers: Headers;
+ body: string;
+ /** The claudish provider uid (`kimi-coding`); a billing question is asked of it. */
+ provider?: string;
+ /** The hop's display name, which some wording rules match. */
+ name: string;
+ }
+ | { kind: "thrown" };
+
+/**
+ * The decision `FallbackHandler.handle` makes for a hop that did not answer `ok`, as a
+ * pure function. A thrown hop advances, exactly as the handler's `catch` always has.
+ */
+export function decideHopFailure(input: HopFailureInput): HopFailureDecision {
+ if (input.kind === "thrown") return { action: "advance", reason: "thrown" };
+ const { status, headers, body, provider, name } = input;
+
+ if (hasRecoveryMarker(headers)) return { action: "stop", reason: "recovery-hold" };
+ if (hasStreamFailureMarker(headers)) return { action: "stop", reason: "stream-failure" };
+
+ const connectionFault = isClaudishConnectionVerdict(headers);
+ const verdict = authVerdict(headers);
+ const subscriptionRejected =
+ !connectionFault && verdict === "terminal" && !!provider && isSubscriptionProvider(provider);
+ if (verdict && !subscriptionRejected) return { action: "stop", reason: "credential-hold" };
+
+ if (
+ subscriptionRejected ||
+ (!connectionFault && isSubscriptionCredentialRejection(status, body, provider))
+ ) {
+ return { action: "advance", reason: "subscription-credential-rejected" };
+ }
+
+ if (!isRetryableError(status, body, name, headers, provider)) {
+ return { action: "stop", reason: "non-retryable" };
+ }
+ return { action: "advance", reason: "retryable" };
+}
+
+/**
+ * A subscription candidate's dead or rejected login, answered WITHOUT the
+ * verdict header. The header covers every 401 a ComposedHandler places; this
+ * covers a candidate that answers one without it, so the rule holds for every
+ * subscription provider and not only for the code paths that mint the header.
+ * Asked of the uid — a display name is never a subscription.
+ */
+export function isSubscriptionCredentialRejection(
+ status: number,
+ errorBody: string,
+ providerId?: string
+): boolean {
+ if (!providerId || !isSubscriptionProvider(providerId)) return false;
+ const recovered = status === 400 ? extractUpstreamStatus(errorBody) : undefined;
+ // A credential rejection only: a 401 that says the MODEL is not carried
+ // (OpenCode Zen Go's "Model <id> is not supported") is a per-model miss and
+ // advances, as it always did (`isCredentialRejection`).
+ if ((status === 401 || recovered === 401) && isCredentialRejection(401, errorBody)) {
+ return true;
+ }
+ // A RAW 403 credential rejection counts too (measured: `403
+ // {"error":{"type":"invalid_api_key"}}` from a subscription hop advanced
+ // onto the metered `meta@`). Raw only: the upstream body still carries the
+ // provider's own `error.type`, so `isCredentialRejection` can tell Zen Go's
+ // `403 RegionError` (a model opt-in, which advances) from a dead login. A
+ // RECOVERED 403 cannot be told apart that way — claudish's remapped 400
+ // body says `invalid_request_error` — so it keeps advancing; a 403 the
+ // ComposedHandler placed already carries the terminal verdict header.
+ return status === 403 && isCredentialRejection(403, errorBody);
+}
+
+/**
+ * Determine if an HTTP error is retryable (should try next provider).
+ * Auth errors, billing errors, rate limits, model-not-found errors and an
+ * unavailable endpoint (502/503/504) warrant trying a different provider. A
+ * plain 500 does NOT: it may carry account state, and it is the one 5xx a
+ * request's own payload can provoke on every provider alike.
+ */
+export function isRetryableError(
+ status: number,
+ errorBody: string,
+ provider?: string,
+ headers?: Headers,
+ providerId?: string
+): boolean {
+ // ── CLAUDISH'S OWN "CANNOT REACH THE HOST" IS NEVER A REASON TO ADVANCE ────
+ //
+ // FIRST, above the quota wording check, and the ORDER is the whole point.
+ // The next statement is `hasQuotaExhaustionWording(errorBody)`, which is
+ // deliberately status-agnostic and whose phrase list contains the bare
+ // substring "quota" — so one of our own connection verdicts that merely
+ // mentioned one would be read as a spent subscription and walk the user onto
+ // metered billing in the middle of a network outage. The error text quotes
+ // the endpoint HOST and URL, so "merely mentioned" is not hypothetical: a
+ // host named `quota-…` reproduced it, and only that word did.
+ //
+ // BOTH arms, via one predicate. The 503 handoff carries `x-claudish-recovery`
+ // and is already returned verbatim by `handle()` above, so for that arm this
+ // branch is a second independent mechanism. The 400 arm — every headless run,
+ // where no banner can exist — carries `x-claudish-connection-error`, and for
+ // IT this branch is the only mechanism there is. Asking the combined
+ // predicate rather than either half is what stops chain-safety from being a
+ // property of which arm happened to answer.
+ if (isClaudishConnectionVerdict(headers)) return false;
+ // A stream that failed after a 200 is not a provider's verdict either; see
+ // `STREAM_FAILURE_HEADER` in recovery-marker.ts. Above the wording match for
+ // the same reason as the line above.
+ if (hasStreamFailureMarker(headers)) return false;
+
+ // ── A CREDENTIAL VERDICT: TERMINAL ON A SUBSCRIPTION ADVANCES, ELSE HOLDS ──
+ //
+ // `ComposedHandler` marks its answer to a credential failure it could place
+ // (`auth-failure.ts`): `terminal` — a credential that cannot succeed on
+ // retry, answered 400 so the message lands inline — or `transient` — a
+ // 5xx/429 on the credential path, which the SAME hop answers on Claude Code's
+ // own retry, so the next candidate is the wrong remedy for it. A terminal
+ // verdict on a SUBSCRIPTION hop (an expired, wrong or revoked credential)
+ // advances to the next hop, metered included (project owner, 2026-10-21:
+ // an expired key must not block the chain). A terminal verdict on any other
+ // hop still holds. Above the quota wording and the `upstream_status`
+ // recovery below, because a terminal 400 carries `upstream_status: 401`,
+ // which that recovery reads as retryable whatever the hop.
+ const verdict = authVerdict(headers);
+ if (verdict) return verdict === "terminal" && !!providerId && isSubscriptionProvider(providerId);
+
+ // A spent subscription allowance is retryable AT THE CHAIN LEVEL: this
+ // provider cannot serve, but the next one can.
+ //
+ // Checked FIRST, and on wording rather than status, because the transport has
+ // already decided this is terminal for itself and surfaced it as 400 (see the
+ // "terminal errors become 400" doctrine). 400 is otherwise non-retryable, so a
+ // status-based check here would stop the chain dead — which is exactly the
+ // regression that showed up as a bare `minimax-m2.5` hard-failing while Zen
+ // Go's 5-hour window was spent and metered MiniMax stood ready.
+ //
+ // The billing change this causes is announced rather than prevented — see the
+ // warning at the advance site below.
+ //
+ // A first attempt made them terminal, to stop a user being moved from their
+ // subscription onto per-token billing without being told. That was the wrong
+ // lever, for two measured reasons:
+ //
+ // 1. It was unnecessary for the case it was meant to protect. An explicit
+ // `kc@k3` / `zgo@model` spec resolves to exactly ONE candidate, so there
+ // is nothing to advance to and the "Out of quota" error surfaces whatever
+ // this function returns. Terminal only ever affected chains claudish
+ // assembled on the user's behalf.
+ // 2. In those chains it cost availability outright. With `opencode-zen-go`
+ // now sitting ahead of the metered APIs, a bare `minimax-m2.5` hard-failed
+ // whenever the Go plan's rolling 5-hour window was spent — a request that
+ // had worked a moment earlier via the metered provider. Verified live:
+ // Zen Go answers `429 GoUsageLimitError "5-hour usage limit reached.
+ // Resets in 3hr 17min"`, and treating that as terminal stopped the chain
+ // dead instead of falling through to a provider that was ready to serve.
+ //
+ // So the chain advances, and the billing change is made LOUD instead of being
+ // prevented at the cost of the request.
+ if (hasQuotaExhaustionWording(errorBody)) return true;
+
+ // A SUBSCRIPTION's rejected credential, answered without the verdict header,
+ // advances like the terminal verdict above (`handle()` announces it).
+ if (isSubscriptionCredentialRejection(status, errorBody, providerId)) return true;
+
+ // ── Recover the ORIGINAL status from a remapped terminal error (#148) ──────
+ //
+ // `composed-handler` remaps terminal upstream failures (401/403/terminal-429)
+ // to 400 BEFORE this handler ever sees them, because a 400 is what makes
+ // Claude Code render the reason inline instead of looping on "API error ·
+ // Retrying". Each candidate in the chain IS a ComposedHandler, so by the time
+ // the status reaches here it is 400, falls into the model-not-found branch
+ // below, matches none of its phrases, and the chain STOPS at the first
+ // provider.
+ //
+ // Which inverts the intent exactly. "Terminal" means THIS provider will not
+ // recover on retry, and that is precisely when the next one should be tried.
+ // The errors that most warrant a fallback were the only ones that could no
+ // longer trigger one.
+ //
+ // The quota half of this was fixed first, by the wording check above — that
+ // covers a spent plan, which is the common case. What is left is a genuine
+ // credential failure: a revoked or rotated key hard-fails a model that three
+ // other credentialed providers in the same chain would have served.
+ //
+ // Deliberately ADDITIVE. It only ever turns a `false` into a `true`, using
+ // the status the upstream actually returned, and falls through to the
+ // unchanged status logic when there is no `upstream_status` to recover. A
+ // remapped 400 that carries something non-retryable (say an upstream 400) is
+ // left to the existing branches, so nothing that used to surface now hides.
+ const upstream = status === 400 ? extractUpstreamStatus(errorBody) : undefined;
+ if (upstream === 401 || upstream === 403 || upstream === 402 || upstream === 429) {
+ return true;
+ }
+
+ // Auth errors — different provider might have valid credentials
+ if (status === 401 || status === 403) return true;
+
+ // Payment required — billing/credit issue specific to this provider
+ if (status === 402) return true;
+
+ // Not found — model doesn't exist on this provider
+ if (status === 404) return true;
+
+ // Rate limited — per-provider limit, a different provider may have capacity
+ if (status === 429) return true;
+
+ // Unavailable — a gateway or overload answer is about THIS endpoint, not about
+ // the model anywhere else. A bare `grok-4.7` stopped at its first candidate on
+ // Zen Go's `503 Upstream request failed: Endpoint is unavailable.` while the
+ // five candidates behind it answered (`--probe grok-4.7`, 2026-09-24), and
+ // Claude Code's retry loop re-sent the same chain to the same dead hop.
+ //
+ // The old exclusion argued a 503 "cannot silently switch the user off the model
+ // they pinned". A pinned `provider@model` builds no FallbackHandler at all
+ // (proxy-server: one candidate → its handler directly), so inside a chain that
+ // protected nothing. claudish's OWN "cannot reach the host" 503 still holds the
+ // chain: the connection-verdict check at the top of this function answers first,
+ // and `handle()` returns a recovery-hold response before it gets here.
+ if (status === 502 || status === 503 || status === 504) return true;
+
+ const lower = errorBody.toLowerCase();
+
+ // Unprocessable (422) — some providers (OpenRouter) use this for model unavailability
+ if (status === 422) {
+ if (
+ lower.includes("not available") ||
+ lower.includes("model not found") ||
+ lower.includes("not supported")
+ ) {
+ return true;
+ }
+ }
+
+ // Bad request — only retryable if it's a model-not-found variant
+ if (status === 400) {
+ if (
+ lower.includes("model not found") ||
+ lower.includes("not registered") ||
+ lower.includes("does not exist") ||
+ lower.includes("unknown model") ||
+ lower.includes("unsupported model") ||
+ lower.includes("no healthy deployment") ||
+ // Gemini Code Assist config-terminal error (the F1-F7 path returns 400 to
+ // surface it inline for an EXPLICIT ag@ selection). But this handler
+ // only runs for BARE-NAME auto-routing, where Gemini is just the first
+ // candidate — a missing project / revoked-client verdict must advance the
+ // chain to the next provider (e.g. OpenRouter), not abort it. When Gemini
+ // is the LAST candidate the caller returns this same 400 anyway, so the
+ // inline-surface behavior is preserved for the single-provider case.
+ lower.includes("requires a google cloud project") ||
+ lower.includes("unsupported_client")
+ ) {
+ return true;
+ }
+
+ // Antigravity rejects some SERVED variants at generation time with a
+ // generic "Request contains an invalid argument" — `gemini-3.1-pro-high`
+ // is in both the account's dynamic models catalog and the cloud models
+ // catalog (marked the family default), yet 400s on every call. Without
+ // this, a bare `gemini-3.1-pro-*` aborts the whole chain instead of
+ // advancing to a provider that works, which is strictly worse than the 404
+ // it used to produce.
+ //
+ // Deliberately scoped to this provider: "invalid argument" is generic
+ // enough that a blanket rule would mask real payload bugs elsewhere by
+ // silently advancing instead of surfacing them. Still bare-name only — an
+ // explicit `ag@` selection has no next candidate, so the caller returns
+ // this same 400 and the message reaches the user inline.
+ if (provider?.toLowerCase().includes("antigravity") && lower.includes("invalid argument")) {
+ return true;
+ }
+
+ // An AGGREGATOR reporting that its UPSTREAM rejected the request is terminal
+ // for this candidate and says nothing about the next one — the two claims are
+ // different, and conflating them costs the chain a provider that works.
+ //
+ // Zen Go fronts other vendors' models, so a rejection here is the upstream's
+ // verdict on the request SHAPE, not on the model's availability anywhere:
+ //
+ // {"error":{"type":"server_error","message":"Error from provider (Console
+ // Go): Upstream request failed: [bad_request_error] invalid params,
+ // invalid tool type: (2013)"}}
+ //
+ // That wording matches none of the model-not-found tests above, so a bare
+ // `minimax-m2.5` died at the first candidate while the metered MiniMax API and
+ // OpenRouter sat behind it, credentialed and ready. Same defect class as the
+ // spent-subscription case at the top of this function, one axis over: that one
+ // separated "exhausted for this provider" from "exhausted for the chain", this
+ // one separates "can't serve this request" from "nobody can serve it".
+ //
+ // Keyed on the aggregator's own framing (`Upstream request failed` / `Error
+ // from provider (`) rather than on the upstream's message, which is
+ // vendor-specific and unbounded. Provider-scoped for the same reason the
+ // Antigravity rule above is: a blanket "advance on any 400" would silently
+ // paper over real payload bugs in providers that speak for themselves.
+ // NOTE ON THE PROVIDER STRING: what arrives here is the candidate's DISPLAY
+ // name ("OpenCode Zen Go"), not the canonical id ("opencode-zen-go") —
+ // proxy-server builds candidates as `{ name: candidate.displayName }`. So the
+ // match is made on a punctuation-stripped form, which accepts both. The
+ // Antigravity rule above only works because its display name happens to be a
+ // single word; do not copy its literal shape for a hyphenated provider.
+ if (
+ isProvider(provider, "opencodezen") &&
+ (lower.includes("upstream request failed") || lower.includes("error from provider ("))
+ ) {
+ return true;
+ }
+ }
+
+ // Server errors (500) — only retryable if it's a billing/credit issue
+ // (some providers misuse 500 for account-level problems)
+ if (status === 500) {
+ if (
+ lower.includes("insufficient balance") ||
+ lower.includes("insufficient credit") ||
+ lower.includes("quota exceeded") ||
+ lower.includes("billing")
+ ) {
+ return true;
+ }
+ }
+
+ return false;
+}
+
+/**
+ * Match a candidate against a provider family, tolerating the two spellings the
+ * same provider reaches this file under: the canonical id (`opencode-zen-go`) and
+ * the display name (`OpenCode Zen Go`). Both normalize to `opencodezengo`, so a
+ * `needle` of `opencodezen` matches either, and both Zen tiers.
+ *
+ * Exported for tests — the display-name-vs-id distinction is exactly the kind of
+ * thing that silently disables a rule.
+ */
+export function isProvider(provider: string | undefined, needle: string): boolean {
+ if (!provider) return false;
+ return provider
+ .toLowerCase()
+ .replace(/[^a-z0-9]/g, "")
+ .includes(needle);
+}