Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
fix(model-hub): name the failure an exhausted recovery waited out, not the request (#2480)
Call White0d19fb1e8b83
diff --git a/core/agent_core/ai/errors.py b/core/agent_core/ai/errors.pyindex 5164d74f3..e34417d36 100644--- a/core/agent_core/ai/errors.py+++ b/core/agent_core/ai/errors.py@@ -142,6 +142,17 @@ def classify_error(effective_code = codeif not effective_code or _is_generic_code(effective_code):effective_code = extracted_code or effective_code+ if effective_code == _MODEL_HUB_RECOVERY_EXHAUSTED:+ # Model Hub already retried within its recovery window and gave up: never retry again, and name the failure+ # it waited out rather than the request (the generic 4xx rule below would say the request was rejected).+ return ProviderError(+ kind=_RECOVERY_CAUSE_KINDS.get(_recovery_cause(body) or "", "server"),+ message=message,+ retryable=False,+ retry_after_s=retry_after,+ status=status,+ partial=partial,+ )kind = _classify_kind(status=status,message=f"{body}\n{message}",@@ -209,6 +220,27 @@ def is_overflow_message(message: str, *, status: int | None = None) -> bool:return any(pattern.search(message) for pattern in _OVERFLOW_RE)+# Model Hub's ending after its recovery window (``core/handlers/model_hub/retry.py``) and the source settlement reasons+# its envelope's ``cause`` carries (``core/handlers/model_hub/classification.py``), as kinds.+_MODEL_HUB_RECOVERY_EXHAUSTED = "model_hub_recovery_exhausted"+_RECOVERY_CAUSE_KINDS = {+ "server_error": "server",+ "network": "network",+ "rate_limited": "rate_limit",+ "quota_exhausted": "rate_limit",+}+++def _recovery_cause(body: str) -> str | None:+ try:+ value = json.loads(body)+ except (TypeError, ValueError):+ return None+ error = value.get("error") if isinstance(value, Mapping) else None+ cause = error.get("cause") if isinstance(error, Mapping) else None+ return cause if isinstance(cause, str) else None++def _classify_kind(*,status: int | None,diff --git a/core/handlers/model_hub/service.py b/core/handlers/model_hub/service.pyindex e2ecd4f0e..4d52afde1 100644--- a/core/handlers/model_hub/service.py+++ b/core/handlers/model_hub/service.py@@ -8855,12 +8855,12 @@ class ModelHubService:facts = turn_outcome.supply_factsif facts is None:raise AssertionError("no-candidate outcome must carry supply facts")+ expired = recovery_request is not None and recovery_request.expiredraise ModelHubError(- RECOVERY_EXHAUSTED_CODE- if recovery_request is not None and recovery_request.expired- else "mapping_target_unavailable",+ RECOVERY_EXHAUSTED_CODE if expired else "mapping_target_unavailable",status=409,supply_state=facts.supply_state,+ data=_recovery_cause(recovery_request.reason) if expired else None,blockers=exact_hop_blockers(projection_resolution),turn_outcome=turn_outcome,)@@ -9215,6 +9215,11 @@ class ModelHubService:RECOVERY_EXHAUSTED_CODE if window_closed else "mapping_target_unavailable",status=503,supply_state=final_facts.supply_state,+ data=(+ _recovery_cause(failed_reason or (recovery_request.reason if recovery_request is not None else None))+ if window_closed+ else None+ ),blockers=exact_hop_blockers(final_resolution),turn_outcome=turn_outcome,)@@ -9254,7 +9259,7 @@ class ModelHubService:raisepending.start()if pending.expired:- raise self._recovery_exhausted(exc) from exc+ raise self._recovery_exhausted(exc, pending.reason) from excconfig = self.store.load()resolution = self._invocation_resolution(config, kwargs["backend"], kwargs["model_id"], supply_channel,@@ -9278,34 +9283,42 @@ class ModelHubService:]selected = min(waits, key=lambda item: item[0]) if waits else Nonedelay = selected[0] if selected is not None else pending.remaining- if delay >= pending.remaining:- # Another admitted owner can still wake this request early.- if not any(hop.recovery == "in_flight" for hop in waitable):- raise self._recovery_exhausted(exc) from exc- delay = pending.remaininghop = selected[1] if selected is not None else waitable[0]pending.source_id = hop.source_idannotation = annotations.get(hop.source_id)pending.reason = annotation.reason if annotation else pending.reason+ if delay >= pending.remaining:+ # Another admitted owner can still wake this request early.+ if not any(hop.recovery == "in_flight" for hop in waitable):+ raise self._recovery_exhausted(exc, pending.reason) from exc+ delay = pending.remainingchanged = self.recovery.changedpending.publish("waiting", next_eligible_at=hop.retry_at)await self.recovery.wait(delay, changed)if pending.expired:# Preserve this request's admission-expiry cause even# if an owner succeeded or configuration changed on wake.- raise self._recovery_exhausted(exc) from exc+ raise self._recovery_exhausted(exc, pending.reason) from excfinally:pending.clear()@staticmethod- def _recovery_exhausted(error: ModelHubError) -> ModelHubError:+ def _recovery_exhausted(error: ModelHubError, reason: str | None) -> ModelHubError:+ # The walk's own failure is the most recent one; otherwise the reason the request was waiting out.return ModelHubError(RECOVERY_EXHAUSTED_CODE, status=error.status,supply_state=error.supply_state, blockers=error.blockers,+ data=error.data if error.code == RECOVERY_EXHAUSTED_CODE and error.data else _recovery_cause(reason),turn_outcome=error.turn_outcome,)+def _recovery_cause(reason: str | None) -> dict[str, str] | None:+ """The failure an exhausted recovery waited out, for the caller to classify (``classification.py`` reasons)."""++ return {"cause": reason} if reason else None++def create_default_service(*,adapter: Optional[EngineAdapter] = None,diff --git a/core/handlers/model_hub/turn_gateway.py b/core/handlers/model_hub/turn_gateway.pyindex e85349866..a387a579d 100644--- a/core/handlers/model_hub/turn_gateway.py+++ b/core/handlers/model_hub/turn_gateway.py@@ -1055,15 +1055,17 @@ class ModelHubTurnGateway:self._commit_and_render_turn_outcome(execution, terminalizer, turn_outcome)# Native compatibility is keyed by the caller backend. Keep its# transport result separate from actual upstream provenance.+ ending = {+ "type": RECOVERY_EXHAUSTED_CODE,+ "code": RECOVERY_EXHAUSTED_CODE,+ "message": RECOVERY_EXHAUSTED_MESSAGE,+ }+ cause = exc.data.get("cause")+ if isinstance(cause, str):+ # The failure recovery waited out, so a caller can name it instead of blaming the request.+ ending["cause"] = causeresponse = web.json_response(- {- "type": "error",- "error": {- "type": RECOVERY_EXHAUSTED_CODE,- "code": RECOVERY_EXHAUSTED_CODE,- "message": RECOVERY_EXHAUSTED_MESSAGE,- },- },+ {"type": "error", "error": ending},status=400 if backend == "codex" else 424,headers={"Cache-Control": "no-store", "X-Content-Type-Options": "nosniff"},)diff --git a/docs/plans/model-hub-contracts/api.md b/docs/plans/model-hub-contracts/api.mdindex db5ecdf96..6e26e33e3 100644--- a/docs/plans/model-hub-contracts/api.md+++ b/docs/plans/model-hub-contracts/api.md@@ -1277,14 +1277,24 @@ Only exhaustion of automatic temporary recovery uses this exact terminal body:"error": {"type": "model_hub_recovery_exhausted","code": "model_hub_recovery_exhausted",- "message": "Automatic recovery has ended. Try again or choose another model."+ "message": "Automatic recovery has ended. Try again or choose another model.",+ "cause": "server_error"}}```-The HTTP status is 400 for backend `codex`, 424 for backend `claude` or-`opencode`, independently of protocol. No Retry-After, raw upstream text or-upstream error detail is sent on this branch. Real provenance remains unchanged;+`cause` names the failure the request was waiting out when recovery ended, in+the structured recovery family the `recovery_snapshot` `reason` uses:+`server_error`, `rate_limited`, `quota_exhausted` or `network`. The walk's own+most recent failure wins over the waited hop's reason. It is omitted when no+reason is known, and it is never raw upstream text. Native clients ignore it;+Avibe's own agent classifies the ending by it, so the user is told what failed+rather than that the request was rejected, and never retries an ending the hub+already retried.++The HTTP status is 400 for backend `codex`, 424 for backend `claude`,+`opencode` or `vibey`, independently of protocol. No Retry-After, raw upstream+text or upstream error detail is sent on this branch. Real provenance remains unchanged;the wire status is not reclassified as a Source/request failure. In particular,Codex's internal InvalidRequest interpretation is an intentional transportcompatibility measure, not public `invalid_request_error`.diff --git a/tests/agent_core/ai/test_provider_adapters.py b/tests/agent_core/ai/test_provider_adapters.pyindex 8b654aee6..b2516a461 100644--- a/tests/agent_core/ai/test_provider_adapters.py+++ b/tests/agent_core/ai/test_provider_adapters.py@@ -4574,6 +4574,44 @@ async def test_model_hubs_stream_interrupted_frame_is_the_interrupted_kind(adaptassert (error.kind, error.retryable) == ("interrupted", True)+# Model Hub's ending once its recovery window ran out (``turn_gateway.py``): an HTTP 424 whose envelope names the+# failure the hub waited out. It already retried that failure, so the loop must not retry again, and the copy must+# name the failure, not blame the request as the generic 4xx rule would.+ ("adapter_type", "protocol"),+ [(OpenAIResponsesAdapter, "openai_responses"), (OpenAIChatAdapter, "openai_chat"), (AnthropicAdapter, "anthropic")],+)+ ("cause", "kind"),+ [+ ("server_error", "server"),+ ("network", "network"),+ ("rate_limited", "rate_limit"),+ ("quota_exhausted", "rate_limit"),+ (None, "server"),+ ],+)+async def test_model_hubs_recovery_exhausted_ending_names_its_cause(adapter_type, protocol, cause, kind) -> None:+ ending = {+ "type": "model_hub_recovery_exhausted",+ "code": "model_hub_recovery_exhausted",+ "message": "Automatic recovery has ended. Try again or choose another model.",+ }+ if cause is not None:+ ending["cause"] = cause++ def handler(_: httpx.Request) -> httpx.Response:+ return httpx.Response(424, json={"type": "error", "error": ending})++ async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:+ events = await _events(adapter_type(client), _request(protocol))++ error = events[-1]+ assert isinstance(error, ProviderError)+ assert (error.kind, error.retryable) == (kind, False)++# CLIProxyAPI nests the class under `error`; OpenAI's reference carries it at the root._SERVER_ERROR_FRAMES = {"cpa-nested": 'data: {"type":"error","error":{"code":"request_timeout","message":"stream error: stream 'diff --git a/tests/scenarios/model_hub/test_model_hub_turn_contract_scenarios.py b/tests/scenarios/model_hub/test_model_hub_turn_contract_scenarios.pyindex 281927bf7..ddbf21881 100644--- a/tests/scenarios/model_hub/test_model_hub_turn_contract_scenarios.py+++ b/tests/scenarios/model_hub/test_model_hub_turn_contract_scenarios.py@@ -390,6 +390,7 @@ def test_mh_turn_takeover_is_silent_and_provenance_names_the_serving_hop(tmp_pat"type": "model_hub_recovery_exhausted","code": "model_hub_recovery_exhausted","message": "Automatic recovery has ended. Try again or choose another model.",+ "cause": "quota_exhausted",},}assert exhausted_result["provenance"]["outcome"] == "exhausted"diff --git a/tests/test_model_hub_l3.py b/tests/test_model_hub_l3.pyindex 0a5570d05..b5f006115 100644--- a/tests/test_model_hub_l3.py+++ b/tests/test_model_hub_l3.py@@ -3154,6 +3154,7 @@ def test_gateway_cooldown_uses_one_cancellable_recovery_window("type": "model_hub_recovery_exhausted","code": "model_hub_recovery_exhausted","message": "Automatic recovery has ended. Try again or choose another model.",+ "cause": "server_error",},}assert "Retry-After" not in response.headersdiff --git a/tests/test_model_hub_retry_lifecycle.py b/tests/test_model_hub_retry_lifecycle.pyindex 21d87b5b5..71374d078 100644--- a/tests/test_model_hub_retry_lifecycle.py+++ b/tests/test_model_hub_retry_lifecycle.py@@ -377,18 +377,19 @@ def test_failure_persistence_error_keeps_temporary_admission_and_original_terminservice.store = FreshStore(service.store.load())before = copy.deepcopy(service.store.persisted)service.store.write_error = True+ reason = code or ("server_error" if status == 503 else "rate_limited")async with gateway_client(service, backend, models[backend]) as (gateway, client, url, headers):async with client.post(url, headers=headers, json={"model": "shared-model"}) as response:assert response.status == (400 if backend == "codex" else 424)- assert (await response.json())["error"]["code"] == RECOVERY_EXHAUSTED_CODE+ ending = (await response.json())["error"]+ # The ending names the failure recovery waited out, whichever way the window closed.+ assert (ending["code"], ending["cause"]) == (RECOVERY_EXHAUSTED_CODE, reason)projection = gateway.correlation.terminal_projection("turn_lifecycle", backend=backend)if backend == "opencode":assert projection is None # Shared-server HTTP does not prove an exact Turn.else:assert projection is not None- assert [blocker.reason for blocker in projection.supply_facts.blockers] == [- code or ("server_error" if status == 503 else "rate_limited"),- ]+ assert [blocker.reason for blocker in projection.supply_facts.blockers] == [reason]gateway.correlation.settle("turn_lifecycle", settled_by=SETTLED_BY_TERMINAL_RESULT)record = service.provenance.get("turn_lifecycle")assert [attempt["http_status"] for attempt in record["failed_attempts"]] == [status] * attempts@@ -512,7 +513,7 @@ def test_expiry_remains_a_closed_terminal_when_live_supply_changes(tmp_path, bac"type": "error","error": {"type": RECOVERY_EXHAUSTED_CODE, "code": RECOVERY_EXHAUSTED_CODE,- "message": RECOVERY_EXHAUSTED_MESSAGE,+ "message": RECOVERY_EXHAUSTED_MESSAGE, "cause": "network",},}assert service.adapter.invocations == []diff --git a/tests/test_model_hub_retry_policy.py b/tests/test_model_hub_retry_policy.pyindex b51c9f2c3..bba1450a5 100644--- a/tests/test_model_hub_retry_policy.py+++ b/tests/test_model_hub_retry_policy.py@@ -480,6 +480,8 @@ def test_real_gateway_has_one_half_open_owner_and_interruptible_waiters(tmp_path"type": RECOVERY_EXHAUSTED_CODE,"code": RECOVERY_EXHAUSTED_CODE,"message": RECOVERY_EXHAUSTED_MESSAGE,+ # The failure this request waited out names the ending, not the request.+ "cause": "network",},}assert len(calls) == 1 and not owner.done()