Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

fix(model-hub): name the failure an exhausted recovery waited out, not the request (#2480)

Call White0d19fb1e8b83
diff --git a/core/agent_core/ai/errors.py b/core/agent_core/ai/errors.py
index 5164d74f3..e34417d36 100644
--- a/core/agent_core/ai/errors.py
+++ b/core/agent_core/ai/errors.py
@@ -142,6 +142,17 @@ def classify_error(
effective_code = code
if not effective_code or _is_generic_code(effective_code):
effective_code = extracted_code or effective_code
+ if effective_code == _MODEL_HUB_RECOVERY_EXHAUSTED:
+ # Model Hub already retried within its recovery window and gave up: never retry again, and name the failure
+ # it waited out rather than the request (the generic 4xx rule below would say the request was rejected).
+ return ProviderError(
+ kind=_RECOVERY_CAUSE_KINDS.get(_recovery_cause(body) or "", "server"),
+ message=message,
+ retryable=False,
+ retry_after_s=retry_after,
+ status=status,
+ partial=partial,
+ )
kind = _classify_kind(
status=status,
message=f"{body}\n{message}",
@@ -209,6 +220,27 @@ def is_overflow_message(message: str, *, status: int | None = None) -> bool:
return any(pattern.search(message) for pattern in _OVERFLOW_RE)
+# Model Hub's ending after its recovery window (``core/handlers/model_hub/retry.py``) and the source settlement reasons
+# its envelope's ``cause`` carries (``core/handlers/model_hub/classification.py``), as kinds.
+_MODEL_HUB_RECOVERY_EXHAUSTED = "model_hub_recovery_exhausted"
+_RECOVERY_CAUSE_KINDS = {
+ "server_error": "server",
+ "network": "network",
+ "rate_limited": "rate_limit",
+ "quota_exhausted": "rate_limit",
+}
+
+
+def _recovery_cause(body: str) -> str | None:
+ try:
+ value = json.loads(body)
+ except (TypeError, ValueError):
+ return None
+ error = value.get("error") if isinstance(value, Mapping) else None
+ cause = error.get("cause") if isinstance(error, Mapping) else None
+ return cause if isinstance(cause, str) else None
+
+
def _classify_kind(
*,
status: int | None,
diff --git a/core/handlers/model_hub/service.py b/core/handlers/model_hub/service.py
index e2ecd4f0e..4d52afde1 100644
--- a/core/handlers/model_hub/service.py
+++ b/core/handlers/model_hub/service.py
@@ -8855,12 +8855,12 @@ class ModelHubService:
facts = turn_outcome.supply_facts
if facts is None:
raise AssertionError("no-candidate outcome must carry supply facts")
+ expired = recovery_request is not None and recovery_request.expired
raise ModelHubError(
- RECOVERY_EXHAUSTED_CODE
- if recovery_request is not None and recovery_request.expired
- else "mapping_target_unavailable",
+ RECOVERY_EXHAUSTED_CODE if expired else "mapping_target_unavailable",
status=409,
supply_state=facts.supply_state,
+ data=_recovery_cause(recovery_request.reason) if expired else None,
blockers=exact_hop_blockers(projection_resolution),
turn_outcome=turn_outcome,
)
@@ -9215,6 +9215,11 @@ class ModelHubService:
RECOVERY_EXHAUSTED_CODE if window_closed else "mapping_target_unavailable",
status=503,
supply_state=final_facts.supply_state,
+ data=(
+ _recovery_cause(failed_reason or (recovery_request.reason if recovery_request is not None else None))
+ if window_closed
+ else None
+ ),
blockers=exact_hop_blockers(final_resolution),
turn_outcome=turn_outcome,
)
@@ -9254,7 +9259,7 @@ class ModelHubService:
raise
pending.start()
if pending.expired:
- raise self._recovery_exhausted(exc) from exc
+ raise self._recovery_exhausted(exc, pending.reason) from exc
config = self.store.load()
resolution = self._invocation_resolution(
config, kwargs["backend"], kwargs["model_id"], supply_channel,
@@ -9278,34 +9283,42 @@ class ModelHubService:
]
selected = min(waits, key=lambda item: item[0]) if waits else None
delay = selected[0] if selected is not None else pending.remaining
- if delay >= pending.remaining:
- # Another admitted owner can still wake this request early.
- if not any(hop.recovery == "in_flight" for hop in waitable):
- raise self._recovery_exhausted(exc) from exc
- delay = pending.remaining
hop = selected[1] if selected is not None else waitable[0]
pending.source_id = hop.source_id
annotation = annotations.get(hop.source_id)
pending.reason = annotation.reason if annotation else pending.reason
+ if delay >= pending.remaining:
+ # Another admitted owner can still wake this request early.
+ if not any(hop.recovery == "in_flight" for hop in waitable):
+ raise self._recovery_exhausted(exc, pending.reason) from exc
+ delay = pending.remaining
changed = self.recovery.changed
pending.publish("waiting", next_eligible_at=hop.retry_at)
await self.recovery.wait(delay, changed)
if pending.expired:
# Preserve this request's admission-expiry cause even
# if an owner succeeded or configuration changed on wake.
- raise self._recovery_exhausted(exc) from exc
+ raise self._recovery_exhausted(exc, pending.reason) from exc
finally:
pending.clear()
@staticmethod
- def _recovery_exhausted(error: ModelHubError) -> ModelHubError:
+ def _recovery_exhausted(error: ModelHubError, reason: str | None) -> ModelHubError:
+ # The walk's own failure is the most recent one; otherwise the reason the request was waiting out.
return ModelHubError(
RECOVERY_EXHAUSTED_CODE, status=error.status,
supply_state=error.supply_state, blockers=error.blockers,
+ data=error.data if error.code == RECOVERY_EXHAUSTED_CODE and error.data else _recovery_cause(reason),
turn_outcome=error.turn_outcome,
)
+def _recovery_cause(reason: str | None) -> dict[str, str] | None:
+ """The failure an exhausted recovery waited out, for the caller to classify (``classification.py`` reasons)."""
+
+ return {"cause": reason} if reason else None
+
+
def create_default_service(
*,
adapter: Optional[EngineAdapter] = None,
diff --git a/core/handlers/model_hub/turn_gateway.py b/core/handlers/model_hub/turn_gateway.py
index e85349866..a387a579d 100644
--- a/core/handlers/model_hub/turn_gateway.py
+++ b/core/handlers/model_hub/turn_gateway.py
@@ -1055,15 +1055,17 @@ class ModelHubTurnGateway:
self._commit_and_render_turn_outcome(execution, terminalizer, turn_outcome)
# Native compatibility is keyed by the caller backend. Keep its
# transport result separate from actual upstream provenance.
+ ending = {
+ "type": RECOVERY_EXHAUSTED_CODE,
+ "code": RECOVERY_EXHAUSTED_CODE,
+ "message": RECOVERY_EXHAUSTED_MESSAGE,
+ }
+ cause = exc.data.get("cause")
+ if isinstance(cause, str):
+ # The failure recovery waited out, so a caller can name it instead of blaming the request.
+ ending["cause"] = cause
response = web.json_response(
- {
- "type": "error",
- "error": {
- "type": RECOVERY_EXHAUSTED_CODE,
- "code": RECOVERY_EXHAUSTED_CODE,
- "message": RECOVERY_EXHAUSTED_MESSAGE,
- },
- },
+ {"type": "error", "error": ending},
status=400 if backend == "codex" else 424,
headers={"Cache-Control": "no-store", "X-Content-Type-Options": "nosniff"},
)
diff --git a/docs/plans/model-hub-contracts/api.md b/docs/plans/model-hub-contracts/api.md
index db5ecdf96..6e26e33e3 100644
--- a/docs/plans/model-hub-contracts/api.md
+++ b/docs/plans/model-hub-contracts/api.md
@@ -1277,14 +1277,24 @@ Only exhaustion of automatic temporary recovery uses this exact terminal body:
"error": {
"type": "model_hub_recovery_exhausted",
"code": "model_hub_recovery_exhausted",
- "message": "Automatic recovery has ended. Try again or choose another model."
+ "message": "Automatic recovery has ended. Try again or choose another model.",
+ "cause": "server_error"
}
}
```
-The HTTP status is 400 for backend `codex`, 424 for backend `claude` or
-`opencode`, independently of protocol. No Retry-After, raw upstream text or
-upstream error detail is sent on this branch. Real provenance remains unchanged;
+`cause` names the failure the request was waiting out when recovery ended, in
+the structured recovery family the `recovery_snapshot` `reason` uses:
+`server_error`, `rate_limited`, `quota_exhausted` or `network`. The walk's own
+most recent failure wins over the waited hop's reason. It is omitted when no
+reason is known, and it is never raw upstream text. Native clients ignore it;
+Avibe's own agent classifies the ending by it, so the user is told what failed
+rather than that the request was rejected, and never retries an ending the hub
+already retried.
+
+The HTTP status is 400 for backend `codex`, 424 for backend `claude`,
+`opencode` or `vibey`, independently of protocol. No Retry-After, raw upstream
+text or upstream error detail is sent on this branch. Real provenance remains unchanged;
the wire status is not reclassified as a Source/request failure. In particular,
Codex's internal InvalidRequest interpretation is an intentional transport
compatibility measure, not public `invalid_request_error`.
diff --git a/tests/agent_core/ai/test_provider_adapters.py b/tests/agent_core/ai/test_provider_adapters.py
index 8b654aee6..b2516a461 100644
--- a/tests/agent_core/ai/test_provider_adapters.py
+++ b/tests/agent_core/ai/test_provider_adapters.py
@@ -4574,6 +4574,44 @@ async def test_model_hubs_stream_interrupted_frame_is_the_interrupted_kind(adapt
assert (error.kind, error.retryable) == ("interrupted", True)
+# Model Hub's ending once its recovery window ran out (``turn_gateway.py``): an HTTP 424 whose envelope names the
+# failure the hub waited out. It already retried that failure, so the loop must not retry again, and the copy must
+# name the failure, not blame the request as the generic 4xx rule would.
+ ("adapter_type", "protocol"),
+ [(OpenAIResponsesAdapter, "openai_responses"), (OpenAIChatAdapter, "openai_chat"), (AnthropicAdapter, "anthropic")],
+)
+ ("cause", "kind"),
+ [
+ ("server_error", "server"),
+ ("network", "network"),
+ ("rate_limited", "rate_limit"),
+ ("quota_exhausted", "rate_limit"),
+ (None, "server"),
+ ],
+)
+async def test_model_hubs_recovery_exhausted_ending_names_its_cause(adapter_type, protocol, cause, kind) -> None:
+ ending = {
+ "type": "model_hub_recovery_exhausted",
+ "code": "model_hub_recovery_exhausted",
+ "message": "Automatic recovery has ended. Try again or choose another model.",
+ }
+ if cause is not None:
+ ending["cause"] = cause
+
+ def handler(_: httpx.Request) -> httpx.Response:
+ return httpx.Response(424, json={"type": "error", "error": ending})
+
+ async with httpx.AsyncClient(transport=httpx.MockTransport(handler)) as client:
+ events = await _events(adapter_type(client), _request(protocol))
+
+ error = events[-1]
+ assert isinstance(error, ProviderError)
+ assert (error.kind, error.retryable) == (kind, False)
+
+
# CLIProxyAPI nests the class under `error`; OpenAI's reference carries it at the root.
_SERVER_ERROR_FRAMES = {
"cpa-nested": 'data: {"type":"error","error":{"code":"request_timeout","message":"stream error: stream '
diff --git a/tests/scenarios/model_hub/test_model_hub_turn_contract_scenarios.py b/tests/scenarios/model_hub/test_model_hub_turn_contract_scenarios.py
index 281927bf7..ddbf21881 100644
--- a/tests/scenarios/model_hub/test_model_hub_turn_contract_scenarios.py
+++ b/tests/scenarios/model_hub/test_model_hub_turn_contract_scenarios.py
@@ -390,6 +390,7 @@ def test_mh_turn_takeover_is_silent_and_provenance_names_the_serving_hop(tmp_pat
"type": "model_hub_recovery_exhausted",
"code": "model_hub_recovery_exhausted",
"message": "Automatic recovery has ended. Try again or choose another model.",
+ "cause": "quota_exhausted",
},
}
assert exhausted_result["provenance"]["outcome"] == "exhausted"
diff --git a/tests/test_model_hub_l3.py b/tests/test_model_hub_l3.py
index 0a5570d05..b5f006115 100644
--- a/tests/test_model_hub_l3.py
+++ b/tests/test_model_hub_l3.py
@@ -3154,6 +3154,7 @@ def test_gateway_cooldown_uses_one_cancellable_recovery_window(
"type": "model_hub_recovery_exhausted",
"code": "model_hub_recovery_exhausted",
"message": "Automatic recovery has ended. Try again or choose another model.",
+ "cause": "server_error",
},
}
assert "Retry-After" not in response.headers
diff --git a/tests/test_model_hub_retry_lifecycle.py b/tests/test_model_hub_retry_lifecycle.py
index 21d87b5b5..71374d078 100644
--- a/tests/test_model_hub_retry_lifecycle.py
+++ b/tests/test_model_hub_retry_lifecycle.py
@@ -377,18 +377,19 @@ def test_failure_persistence_error_keeps_temporary_admission_and_original_termin
service.store = FreshStore(service.store.load())
before = copy.deepcopy(service.store.persisted)
service.store.write_error = True
+ reason = code or ("server_error" if status == 503 else "rate_limited")
async with gateway_client(service, backend, models[backend]) as (gateway, client, url, headers):
async with client.post(url, headers=headers, json={"model": "shared-model"}) as response:
assert response.status == (400 if backend == "codex" else 424)
- assert (await response.json())["error"]["code"] == RECOVERY_EXHAUSTED_CODE
+ ending = (await response.json())["error"]
+ # The ending names the failure recovery waited out, whichever way the window closed.
+ assert (ending["code"], ending["cause"]) == (RECOVERY_EXHAUSTED_CODE, reason)
projection = gateway.correlation.terminal_projection("turn_lifecycle", backend=backend)
if backend == "opencode":
assert projection is None # Shared-server HTTP does not prove an exact Turn.
else:
assert projection is not None
- assert [blocker.reason for blocker in projection.supply_facts.blockers] == [
- code or ("server_error" if status == 503 else "rate_limited"),
- ]
+ assert [blocker.reason for blocker in projection.supply_facts.blockers] == [reason]
gateway.correlation.settle("turn_lifecycle", settled_by=SETTLED_BY_TERMINAL_RESULT)
record = service.provenance.get("turn_lifecycle")
assert [attempt["http_status"] for attempt in record["failed_attempts"]] == [status] * attempts
@@ -512,7 +513,7 @@ def test_expiry_remains_a_closed_terminal_when_live_supply_changes(tmp_path, bac
"type": "error",
"error": {
"type": RECOVERY_EXHAUSTED_CODE, "code": RECOVERY_EXHAUSTED_CODE,
- "message": RECOVERY_EXHAUSTED_MESSAGE,
+ "message": RECOVERY_EXHAUSTED_MESSAGE, "cause": "network",
},
}
assert service.adapter.invocations == []
diff --git a/tests/test_model_hub_retry_policy.py b/tests/test_model_hub_retry_policy.py
index b51c9f2c3..bba1450a5 100644
--- a/tests/test_model_hub_retry_policy.py
+++ b/tests/test_model_hub_retry_policy.py
@@ -480,6 +480,8 @@ def test_real_gateway_has_one_half_open_owner_and_interruptible_waiters(tmp_path
"type": RECOVERY_EXHAUSTED_CODE,
"code": RECOVERY_EXHAUSTED_CODE,
"message": RECOVERY_EXHAUSTED_MESSAGE,
+ # The failure this request waited out names the ending, not the request.
+ "cause": "network",
},
}
assert len(calls) == 1 and not owner.done()