Skip to content

Use Claude Code, Codex, VSCode, Pi, and OpenCode (and 6 other harnesses) for free (1.3B+ free tokens) from your terminal, app, IDE, or phone, and now from the browser with native browser sessions (multi-harness + multi-model) like OpenClaw (voice supported + ToS friendly)

Python

Fix pre-output model fallback transitions (#1553)

Ali Khokhara945a1f1b8b4
diff --git a/.env.example b/.env.example
index add20b06..f7efe77b 100644
--- a/.env.example
+++ b/.env.example
@@ -202,7 +202,7 @@ OLLAMA_BASE_URL="http://localhost:11434"
# MODEL_SONNET=<provider/model-id>
# MODEL_HAIKU=<provider/model-id>
MODEL="nvidia_nim/nvidia/nemotron-3-super-120b-a12b"
-# Optional ordered cross-client fallbacks after retryable provider failures exhaust.
+# Optional ordered cross-client fallbacks when a provider/model fails before output.
# A request may reach and consume usage from more than one provider.
# MODEL_FALLBACKS="groq/llama-3.3-70b-versatile,open_router/openrouter/free"
diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md
index 96c331e4..3455fd1d 100644
--- a/ARCHITECTURE.md
+++ b/ARCHITECTURE.md
@@ -619,16 +619,18 @@ publish the private upstream model as the response model.
Fallback execution is a bounded first-frame state machine in
[application/execution.py](src/free_claude_code/application/execution.py). The
executor opens the primary first and resolves each later provider lazily. It
-advances only when the current candidate raises a retryable `ExecutionFailure`
-before emitting any non-empty protocol chunk and another configured target
-exists. It closes the abandoned iterator before resolving the next provider and
-deep-copies the original routed request for every candidate, changing only the
-upstream model. Authentication, permission, billing, invalid request, context
-overflow, deterministic preflight, unexpected exceptions, empty completion,
-timeouts, cancellation, and disconnect do not advance. Once any protocol frame
-is emitted, the candidate is committed and existing terminal-error behavior is
-authoritative; this prevents duplicate lifecycles and output. Final exhaustion
-re-raises the last exact failure.
+advances when the current provider has finalized any `ExecutionFailure` before
+emitting a non-empty protocol chunk and another configured target exists.
+`retryable` remains provider-local recovery metadata; it does not control the
+application's forward transition to a different target. The executor closes the
+abandoned iterator before resolving the next provider and deep-copies the
+original routed request for every candidate, changing only the upstream model.
+Deterministic preflight or validation failures, unexpected exceptions, the
+application-owned progress deadline, cancellation, disconnect, and empty
+completion remain terminal. Once any protocol frame is emitted, the candidate
+is committed and existing terminal-error behavior is authoritative; this
+prevents duplicate lifecycles and output. Final exhaustion re-raises the last
+exact failure.
Every candidate keeps the original request ID, public response model, input
token count, reasoning policy, messages, system prompt, tools, and generation
diff --git a/pyproject.toml b/pyproject.toml
index cbe1e272..8566519c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
[project]
name = "free-claude-code"
-version = "5.14.5"
+version = "5.14.6"
description = "Local proxy connecting coding agents to OpenAI-compatible AI providers"
readme = "README.md"
requires-python = ">=3.14.0"
diff --git a/src/free_claude_code/application/execution.py b/src/free_claude_code/application/execution.py
index ca4b1fca..249f2848 100644
--- a/src/free_claude_code/application/execution.py
+++ b/src/free_claude_code/application/execution.py
@@ -100,7 +100,7 @@ class ProviderExecutor:
"candidate_count": candidate_count,
"failure_kind": failure.kind.value,
"status_code": failure.status_code,
- "provider_retryable": True,
+ "provider_retryable": failure.retryable,
}
if self._generation_id is not None:
fields["generation_id"] = self._generation_id
@@ -233,25 +233,37 @@ class ProviderExecutor:
provider_stream: AsyncIterator[str] | None = None
candidate_committed = False
- advance_failure: ExecutionFailure | None = None
+ candidate_failure: ExecutionFailure | None = None
try:
- provider_stream = provider.stream_response(
- candidate_request,
- input_tokens=input_tokens,
- request_id=request_id,
- response_model=gateway_model,
- reasoning=routed.reasoning,
- )
- while True:
+ try:
+ provider_stream = provider.stream_response(
+ candidate_request,
+ input_tokens=input_tokens,
+ request_id=request_id,
+ response_model=gateway_model,
+ reasoning=routed.reasoning,
+ )
+ except ExecutionFailure as failure:
+ candidate_failure = failure
+
+ if provider_stream is None and candidate_failure is None:
+ raise TypeError(
+ "provider stream_response must return an async iterator"
+ )
+ while provider_stream is not None:
if loop.time() >= progress_deadline:
raise self._progress_timeout_failure(
request_id=request_id,
provider_id=target.provider_id,
)
progress_timeout = asyncio.timeout_at(progress_deadline)
+ read_failure: ExecutionFailure | None = None
try:
async with progress_timeout:
- chunk = await anext(provider_stream)
+ try:
+ chunk = await anext(provider_stream)
+ except ExecutionFailure as failure:
+ read_failure = failure
except StopAsyncIteration:
break
except TimeoutError as exc:
@@ -261,6 +273,14 @@ class ProviderExecutor:
request_id=request_id,
provider_id=target.provider_id,
) from exc
+ if progress_timeout.expired():
+ raise self._progress_timeout_failure(
+ request_id=request_id,
+ provider_id=target.provider_id,
+ )
+ if read_failure is not None:
+ candidate_failure = read_failure
+ break
if not chunk:
await asyncio.sleep(0)
continue
@@ -276,32 +296,40 @@ class ProviderExecutor:
)
yield chunk
progress_deadline = loop.time() + self._progress_timeout_seconds
- except ExecutionFailure as failure:
- if (
- candidate_committed
- or not failure.retryable
- or index + 1 >= len(candidates)
- ):
- raise
- advance_failure = failure
finally:
if provider_stream is not None:
- await close_stream_input(
- provider_stream,
- owner="provider_executor",
- source="api",
- preserved_error=sys.exception(),
+ active_error = sys.exception()
+ preserved_error = active_error or candidate_failure
+ cleanup_timeout = asyncio.timeout_at(
+ progress_deadline if active_error is None else None
)
+ try:
+ async with cleanup_timeout:
+ await close_stream_input(
+ provider_stream,
+ owner="provider_executor",
+ source="api",
+ preserved_error=preserved_error,
+ )
+ except TimeoutError as exc:
+ if not cleanup_timeout.expired():
+ raise
+ raise self._progress_timeout_failure(
+ request_id=request_id,
+ provider_id=target.provider_id,
+ ) from exc
- if advance_failure is None:
+ if candidate_failure is None:
return
+ if candidate_committed or index + 1 >= len(candidates):
+ raise candidate_failure
next_target = candidates[index + 1]
self._trace_fallback_started(
request_id=request_id,
wire_api=wire_api,
failed=target,
selected=next_target,
- failure=advance_failure,
+ failure=candidate_failure,
candidate_index=index + 2,
candidate_count=len(candidates),
)
diff --git a/src/free_claude_code/config/admin/manifest.py b/src/free_claude_code/config/admin/manifest.py
index c87c0a85..37477cee 100644
--- a/src/free_claude_code/config/admin/manifest.py
+++ b/src/free_claude_code/config/admin/manifest.py
@@ -132,8 +132,8 @@ _NON_PROVIDER_FIELDS: tuple[ConfigFieldSpec, ...] = (
"model_list",
settings_attr="model_fallbacks",
description=(
- "Tried in order after the selected model exhausts retryable provider "
- "failures. Applies to every client. One request may reach multiple "
+ "Tried in order when the selected provider/model fails before output "
+ "starts. Applies to every client. One request may reach multiple "
"providers and consume usage at each."
),
),
diff --git a/tests/api/test_admin.py b/tests/api/test_admin.py
index 778a37a3..996e7bac 100644
--- a/tests/api/test_admin.py
+++ b/tests/api/test_admin.py
@@ -467,6 +467,7 @@ def test_admin_config_masks_secrets_and_exposes_manifest(monkeypatch, tmp_path):
assert fallback_field["value"] is None
assert fallback_field["nullable"] is True
assert fallback_field["restart_required"] is False
+ assert "fails before output" in fallback_field["description"]
assert "every client" in fallback_field["description"]
assert "multiple providers" in fallback_field["description"]
reasoning_policy = next(
diff --git a/tests/api/test_model_fallback.py b/tests/api/test_model_fallback.py
index 9001fe7e..f94738ca 100644
--- a/tests/api/test_model_fallback.py
+++ b/tests/api/test_model_fallback.py
@@ -61,6 +61,36 @@ def test_responses_fallback_emits_one_stable_response_lifecycle() -> None:
assert "fallback worked" in response.text
+ ("path", "payload"),
+ (
+ ("/v1/messages", messages_payload(stream=True)),
+ ("/v1/responses", responses_payload()),
+ ),
+)
+def test_nonretryable_provider_failure_uses_fallback_before_output(
+ path: str,
+ payload: dict[str, object],
+) -> None:
+ primary = ControlledFallbackProvider(
+ failure=ExecutionFailure(
+ kind=FailureKind.PERMISSION,
+ status_code=403,
+ message="primary quota exhausted",
+ retryable=False,
+ )
+ )
+ fallback = ControlledFallbackProvider(text="fallback worked")
+
+ with fallback_client(primary, fallback) as client:
+ response = client.post(path, json=payload)
+
+ assert response.status_code == 200
+ assert "fallback worked" in response.text
+ assert "primary quota exhausted" not in response.text
+ assert primary.close_calls == fallback.close_calls == 1
+
+
@pytest.mark.parametrize(
("path", "payload"),
(
diff --git a/tests/application/test_execution.py b/tests/application/test_execution.py
index a9581a11..4ab1ff20 100644
--- a/tests/application/test_execution.py
+++ b/tests/application/test_execution.py
@@ -172,6 +172,73 @@ class FailingStreamConstructionProvider(FakeProvider):
raise RuntimeError("stream construction failed")
+class ExecutionFailureStreamConstructionProvider(FakeProvider):
+ def __init__(self, failure: ExecutionFailure) -> None:
+ super().__init__()
+ self._failure = failure
+
+ def stream_response(
+ self,
+ request: MessagesRequest,
+ input_tokens: int = 0,
+ *,
+ request_id: str | None = None,
+ response_model: str | None = None,
+ reasoning: ReasoningPolicy,
+ ) -> AsyncIterator[str]:
+ del request, input_tokens, request_id, response_model, reasoning
+ raise self._failure
+
+
+class CloseControlledProvider(FakeProvider):
+ def __init__(
+ self,
+ failure: ExecutionFailure,
+ *,
+ close_delay_seconds: float = 0.0,
+ close_error: Exception | None = None,
+ ) -> None:
+ super().__init__()
+ self._failure = failure
+ self._close_delay_seconds = close_delay_seconds
+ self._close_error = close_error
+ self._read = False
+
+ def stream_response(
+ self,
+ request: MessagesRequest,
+ input_tokens: int = 0,
+ *,
+ request_id: str | None = None,
+ response_model: str | None = None,
+ reasoning: ReasoningPolicy,
+ ) -> AsyncIterator[str]:
+ self._record_stream_call(
+ request,
+ input_tokens=input_tokens,
+ request_id=request_id,
+ response_model=response_model,
+ reasoning=reasoning,
+ )
+ return self
+
+ def __aiter__(self) -> AsyncIterator[str]:
+ return self
+
+ async def __anext__(self) -> str:
+ if self._read:
+ raise StopAsyncIteration
+ self._read = True
+ raise self._failure
+
+ async def aclose(self) -> None:
+ self.stream_close_calls += 1
+ if self._close_delay_seconds:
+ await asyncio.sleep(self._close_delay_seconds)
+ if self._close_error is not None:
+ raise self._close_error
+
+
def _target(provider_id: str, provider_model: str) -> ProviderModelTarget:
return ProviderModelTarget(
provider_id=provider_id,
@@ -474,12 +541,46 @@ async def test_unexpected_primary_failure_does_not_select_fallback() -> None:
@pytest.mark.asyncio
-async def test_nonretryable_failure_never_resolves_fallback() -> None:
- primary_failure = _execution_failure("rejected", retryable=False)
- primary = ControlledProvider([primary_failure])
+async def test_invalid_none_stream_remains_terminal() -> None:
+ primary = FakeProvider()
fallback = FakeProvider()
resolved_ids: list[str] = []
+ def resolve(provider_id: str) -> FakeProvider:
+ resolved_ids.append(provider_id)
+ return {"provider": primary, "fallback": fallback}[provider_id]
+
+ executor = ProviderExecutor(resolve, progress_timeout_seconds=60.0)
+ with patch.object(primary, "stream_response", return_value=None):
+ stream = executor.stream(
+ _routed_request(_target("fallback", "model")),
+ wire_api="messages",
+ raw_log_label="FULL_PAYLOAD",
+ raw_log_payload={},
+ request_id="req_invalid_stream",
+ )
+ with pytest.raises(TypeError):
+ await anext(stream)
+
+ assert resolved_ids == ["provider"]
+ assert fallback.stream_calls == []
+
+
[email protected]("failure_kind", tuple(FailureKind))
+async def test_nonretryable_provider_failure_selects_fallback_before_first_frame(
+ failure_kind: FailureKind,
+) -> None:
+ primary_failure = ExecutionFailure(
+ kind=failure_kind,
+ status_code=500,
+ message="provider rejected candidate",
+ retryable=False,
+ )
+ primary = ControlledProvider([primary_failure])
+ fallback = ControlledProvider(["fallback-frame"])
+ resolved_ids: list[str] = []
+
def resolve(provider_id: str) -> FakeProvider:
resolved_ids.append(provider_id)
return {"provider": primary, "fallback": fallback}[provider_id]
@@ -493,11 +594,45 @@ async def test_nonretryable_failure_never_resolves_fallback() -> None:
request_id="req_rejected",
)
- with pytest.raises(ExecutionFailure) as exc_info:
- await anext(stream)
+ with patch("free_claude_code.application.execution.trace_event") as trace_mock:
+ chunks = [chunk async for chunk in stream]
- assert exc_info.value is primary_failure
- assert resolved_ids == ["provider"]
+ assert chunks == ["fallback-frame"]
+ assert resolved_ids == ["provider", "fallback"]
+ fallback_started = next(
+ call.kwargs
+ for call in trace_mock.call_args_list
+ if call.kwargs.get("event") == "free_claude_code.model_fallback.started"
+ )
+ assert fallback_started["failure_kind"] == failure_kind.value
+ assert fallback_started["provider_retryable"] is False
+
+
+async def test_nonretryable_stream_construction_failure_selects_fallback() -> None:
+ primary_failure = ExecutionFailure(
+ kind=FailureKind.PERMISSION,
+ status_code=403,
+ message="provider denied candidate",
+ retryable=False,
+ )
+ primary = ExecutionFailureStreamConstructionProvider(primary_failure)
+ fallback = ControlledProvider(["fallback-frame"])
+ providers = {"provider": primary, "fallback": fallback}
+ executor = ProviderExecutor(
+ providers.__getitem__,
+ progress_timeout_seconds=60.0,
+ )
+ stream = executor.stream(
+ _routed_request(_target("fallback", "model")),
+ wire_api="messages",
+ raw_log_label="FULL_PAYLOAD",
+ raw_log_payload={},
+ request_id="req_construction_failure",
+ )
+
+ assert [chunk async for chunk in stream] == ["fallback-frame"]
+ assert fallback.stream_calls[0]["request_id"] == "req_construction_failure"
@pytest.mark.asyncio
@@ -731,6 +866,104 @@ async def test_progress_timeout_before_first_chunk_is_canonical_and_correlated()
]
+async def test_application_progress_timeout_never_resolves_fallback() -> None:
+ primary = ControlledProvider([WaitStep()])
+ fallback = FakeProvider()
+ resolved_ids: list[str] = []
+
+ def resolve(provider_id: str) -> FakeProvider:
+ resolved_ids.append(provider_id)
+ return {"provider": primary, "fallback": fallback}[provider_id]
+
+ executor = ProviderExecutor(resolve, progress_timeout_seconds=0.02)
+ stream = executor.stream(
+ _routed_request(_target("fallback", "model")),
+ wire_api="messages",
+ raw_log_label="FULL_PAYLOAD",
+ raw_log_payload={},
+ request_id="req_terminal_progress_timeout",
+ )
+
+ with pytest.raises(ExecutionFailure) as exc_info:
+ await anext(stream)
+
+ assert exc_info.value.kind is FailureKind.TIMEOUT
+ assert exc_info.value.status_code == 504
+ assert exc_info.value.retryable is False
+ assert resolved_ids == ["provider"]
+ assert primary.stream_close_calls == 1
+ assert fallback.stream_calls == []
+
+
+async def test_provider_cleanup_cannot_delay_fallback_past_progress_deadline() -> None:
+ primary = CloseControlledProvider(
+ _execution_failure("primary overloaded"),
+ close_delay_seconds=0.05,
+ )
+ fallback = FakeProvider()
+ resolved_ids: list[str] = []
+
+ def resolve(provider_id: str) -> FakeProvider:
+ resolved_ids.append(provider_id)
+ return {"provider": primary, "fallback": fallback}[provider_id]
+
+ executor = ProviderExecutor(resolve, progress_timeout_seconds=0.02)
+ stream = executor.stream(
+ _routed_request(_target("fallback", "model")),
+ wire_api="messages",
+ raw_log_label="FULL_PAYLOAD",
+ raw_log_payload={},
+ request_id="req_cleanup_timeout",
+ )
+
+ with (
+ patch("free_claude_code.application.execution.trace_event") as trace_mock,
+ pytest.raises(ExecutionFailure) as exc_info,
+ ):
+ await anext(stream)
+
+ assert exc_info.value.kind is FailureKind.TIMEOUT
+ assert exc_info.value.status_code == 504
+ assert resolved_ids == ["provider"]
+ assert primary.stream_close_calls == 1
+ assert fallback.stream_calls == []
+ assert all(
+ call.kwargs.get("event") != "free_claude_code.model_fallback.started"
+ for call in trace_mock.call_args_list
+ )
+
+
+async def test_cleanup_failure_trace_names_preserved_provider_failure() -> None:
+ failure = _execution_failure("primary overloaded")
+ provider = CloseControlledProvider(
+ failure,
+ close_error=RuntimeError("close failed"),
+ )
+ stream = _executor_stream(
+ provider,
+ timeout_seconds=1.0,
+ request_id="req_close_failure_trace",
+ )
+
+ with (
+ patch("free_claude_code.core.trace.trace_event") as trace_mock,
+ pytest.raises(ExecutionFailure) as exc_info,
+ ):
+ await anext(stream)
+
+ assert exc_info.value is failure
+ close_trace = next(
+ call.kwargs
+ for call in trace_mock.call_args_list
+ if call.kwargs.get("event") == "stream.input.close_failed"
+ )
+ assert close_trace["close_exc_type"] == "RuntimeError"
+ assert close_trace["preserved_exc_type"] == "ExecutionFailure"
+
+
@pytest.mark.asyncio
async def test_progress_timeout_renews_after_output_without_crossing_yield() -> None:
second_wait = WaitStep()
diff --git a/tests/providers/test_failure_policy.py b/tests/providers/test_failure_policy.py
index 60497b7b..ec93b6c7 100644
--- a/tests/providers/test_failure_policy.py
+++ b/tests/providers/test_failure_policy.py
@@ -255,6 +255,42 @@ _CASES = (
500,
False,
),
+ _ClassificationCase(
+ "statusless_openai_tokenrouter_bad_request",
+ lambda: _statusless_openai_error(
+ "stream embedded error",
+ {
+ "object": "error",
+ "message": (
+ "Invalid request: Disaggregated request received without "
+ "bootstrap room id"
+ ),
+ "type": "BAD_REQUEST",
+ "param": None,
+ "code": 400,
+ },
+ ),
+ FailureKind.UPSTREAM,
+ 500,
+ False,
+ ),
+ _ClassificationCase(
+ "openai_insufficient_user_quota",
+ lambda: _openai_status_error(
+ openai.PermissionDeniedError,
+ status_code=403,
+ message="insufficient user quota",
+ body={
+ "error": {
+ "message": "insufficient user quota",
+ "code": "insufficient_user_quota",
+ }
+ },
+ ),
+ FailureKind.PERMISSION,
+ 403,
+ False,
+ ),
_ClassificationCase(
"http_401_maps_authentication",
lambda: _http_status_error(401, "Unauthorized"),
diff --git a/uv.lock b/uv.lock
index efa2b2be..d768b367 100644
--- a/uv.lock
+++ b/uv.lock
@@ -607,7 +607,7 @@ wheels = [
[[package]]
name = "free-claude-code"
-version = "5.14.5"
+version = "5.14.6"
source = { editable = "." }
dependencies = [
{ name = "aiohttp" },