Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
fix(vibey): continue after transient hub recovery exhaustion (#2720)
Call Whitef8829a23bdc9
diff --git a/core/agent_core/ai/errors.py b/core/agent_core/ai/errors.pyindex 1fbed40f1..dbacfc82f 100644--- a/core/agent_core/ai/errors.py+++ b/core/agent_core/ai/errors.py@@ -148,12 +148,16 @@ def classify_error(if not effective_code or _is_generic_code(effective_code):effective_code = extracted_code or effective_codeif effective_code == _MODEL_HUB_RECOVERY_EXHAUSTED:- # Model Hub already retried within its recovery window and gave up: never retry again, and name the failure- # it waited out rather than the request (the generic 4xx rule below would say the request was rejected).+ # Model Hub already retried the same request within its recovery window. Do not replay that request again,+ # but permit Vibey's bounded transcript-only continuation for a pre-output server/network failure: it resumes+ # from committed context without rerunning committed tools or resending the original input.+ recovery_cause = _recovery_cause(body)+ kind = _RECOVERY_CAUSE_KINDS.get(recovery_cause or "", "server")return ProviderError(- kind=_RECOVERY_CAUSE_KINDS.get(_recovery_cause(body) or "", "server"),+ kind=kind,message=message,retryable=False,+ continuable=partial is None and recovery_cause in {"server_error", "network"},retry_after_s=retry_after,status=status,partial=partial,@@ -235,6 +239,10 @@ _RECOVERY_CAUSE_KINDS = {"rate_limited": "rate_limit","quota_exhausted": "rate_limit",}+# Model Hub has already spent its own recovery window. A fresh transcript-only+# continuation is safe only for an explicit pre-output server/network cause:+# it cannot replay committed tools or resend the original input. Keep rate limits,+# quota exhaustion, missing causes, and future causes terminal.def _recovery_cause(body: str) -> str | None:diff --git a/docs/plans/agent-core-contracts/provider-event.schema.json b/docs/plans/agent-core-contracts/provider-event.schema.jsonindex 468b7cda0..fe33e5463 100644--- a/docs/plans/agent-core-contracts/provider-event.schema.json+++ b/docs/plans/agent-core-contracts/provider-event.schema.json@@ -42,7 +42,7 @@"type": { "const": "error" },"kind": { "enum": ["overflow", "rate_limit", "overloaded", "network", "stalled", "server", "interrupted", "auth", "invalid_request", "aborted", "unknown"] },"retryable": { "type": "boolean" },- "continuable": { "type": "boolean", "default": false, "description": "Explicit permission to continue from committed context, separate from same-request replay. Classified transient failures allow it; recovery exhaustion, refusals, permanent and local failures do not." },+ "continuable": { "type": "boolean", "default": false, "description": "Explicit permission to continue from committed context, separate from same-request replay. Classified transient failures allow it; a pre-output recovery exhaustion with an explicit server_error or network cause may allow the bounded transcript-only continuation; missing/future causes, refusals, permanent and local failures do not." },"retry_after_s": { "type": "number", "minimum": 0 },"after_thinking": { "type": "boolean", "description": "The response streamed thinking and nothing else before this transient or unknown error; the loop retries it at most once per request (provider.md)." },"status": { "type": "integer", "description": "HTTP status, when there was one." },diff --git a/docs/plans/agent-core-contracts/provider.md b/docs/plans/agent-core-contracts/provider.mdindex 931823f77..b969a7e85 100644--- a/docs/plans/agent-core-contracts/provider.md+++ b/docs/plans/agent-core-contracts/provider.md@@ -119,10 +119,13 @@ Error classification is part of the contract because the loop branches on it:`retryable` permits replay of the same request. `continuable` (default false) separately permits a new runfrom the committed transcript after the failed run closes. Classified transient errors grant continuation even-when streamed output forbids replay. Model Hub's `model_hub_recovery_exhausted` ending, explicit pin/opaque-refusals, permanent/protocol errors, unclassified failures, and local preparation errors grant no continuation.-Neither the terminal sanitizer nor a downstream consumer may infer permission from `kind` alone. In particular,-a recovery-exhausted error naming `server` stays non-retryable and non-continuable after thinking, too.+when streamed output forbids replay. A `model_hub_recovery_exhausted` ending remains non-retryable: for an+explicit pre-output `server` or `network` cause, it may grant Vibey one of its existing bounded transcript-only+continuation runs, which cannot replay the original input or committed tools. Rate-limit/quota exhaustion, missing or+future causes, explicit pin/opaque refusals, permanent/protocol errors, unclassified failures, and local preparation+errors grant no continuation. Neither the terminal sanitizer nor a downstream consumer may infer permission from+`kind` alone. A recovery-exhausted error after thinking remains non-continuable, because the partial response is not+safe to replay or continue implicitly.The loop carries both permissions and `retry_after_s` into the terminal `AgentError` unchanged. These fields donot grant permission to replay committed tools or manufacture user input; Vibey uses `Agent.resume` under itsdiff --git a/docs/plans/model-hub-retry-experience.md b/docs/plans/model-hub-retry-experience.mdindex 79e63def9..8e2a33559 100644--- a/docs/plans/model-hub-retry-experience.md+++ b/docs/plans/model-hub-retry-experience.md@@ -22,9 +22,11 @@ The intended experience is:Other requests may use backups or wait; they do not all hit the relay.4. If that request produces valid model output, let work continue. If itfails again, increase the delay. Timer expiry alone proves nothing.-5. If automatic recovery ends, say that this request has ended and offer- the existing safe Retry action where supported. Later supplier recovery- must not silently revive a terminal Turn.+5. If automatic recovery ends, the gateway request ends truthfully. For a pre-output `server` or `network` failure,+ Vibey may use its existing bounded transcript-only continuation once the failed Turn closes; this does not replay the+ original input or committed tools. Recovery exhaustion after thinking, rate-limit/quota exhaustion, or any ambiguous+ acceptance remains terminal and uses the existing explicit Retry action where supported. Later supplier recovery must+ not silently revive a terminal Turn.The incident's observed `server_error` classification does not itself provea socket failure: a relay can return 503 because its own upstream is down.diff --git a/tests/agent_core/ai/test_provider_adapters.py b/tests/agent_core/ai/test_provider_adapters.pyindex b180bebee..ddbfd8e6b 100644--- a/tests/agent_core/ai/test_provider_adapters.py+++ b/tests/agent_core/ai/test_provider_adapters.py@@ -4686,24 +4686,26 @@ async def test_model_hubs_stream_interrupted_frame_is_the_interrupted_kind(adapt# Model Hub's ending once its recovery window ran out (``turn_gateway.py``): an HTTP 424 whose envelope names the-# failure the hub waited out. It already retried that failure, so the loop must not retry again, and the copy must-# name the failure, not blame the request as the generic 4xx rule would.+# failure the hub waited out. The loop must not replay that same request; a pre-output server/network cause may still+# permit Vibey's safe transcript-only continuation, while the copy must name the failure rather than blame the request.@pytest.mark.asyncio@pytest.mark.parametrize(("adapter_type", "protocol"),[(OpenAIResponsesAdapter, "openai_responses"), (OpenAIChatAdapter, "openai_chat"), (AnthropicAdapter, "anthropic")],)@pytest.mark.parametrize(- ("cause", "kind"),+ ("cause", "kind", "continuable"),[- ("server_error", "server"),- ("network", "network"),- ("rate_limited", "rate_limit"),- ("quota_exhausted", "rate_limit"),- (None, "server"),+ ("server_error", "server", True),+ ("network", "network", True),+ ("rate_limited", "rate_limit", False),+ ("quota_exhausted", "rate_limit", False),+ (None, "server", False),],)-async def test_model_hubs_recovery_exhausted_ending_names_its_cause(adapter_type, protocol, cause, kind) -> None:+async def test_model_hubs_recovery_exhausted_ending_names_its_cause(+ adapter_type, protocol, cause, kind, continuable+) -> None:ending = {"type": "model_hub_recovery_exhausted","code": "model_hub_recovery_exhausted",@@ -4720,7 +4722,7 @@ async def test_model_hubs_recovery_exhausted_ending_names_its_cause(adapter_typeerror = events[-1]assert isinstance(error, ProviderError)- assert (error.kind, error.retryable, error.continuable) == (kind, False, False)+ assert (error.kind, error.retryable, error.continuable) == (kind, False, continuable)# CLIProxyAPI nests the class under `error`; OpenAI's reference carries it at the root.@@ -4774,20 +4776,20 @@ async def test_only_a_drop_after_thinking_alone_is_marked_after_thinking(@pytest.mark.parametrize("output", ["none", "thinking", "text"])@pytest.mark.parametrize(- "failure,continuable",+ "failure",[- ("classified transient", True),- ("explicit transient", True),- ("eof", True),- ("stalled", True),- ("recovery exhausted", False),- ("pin refusal", False),- ("malformed", False),- ("unknown wire", False),- ("local exception", False),+ "classified transient",+ "explicit transient",+ "eof",+ "stalled",+ "recovery exhausted",+ "pin refusal",+ "malformed",+ "unknown wire",+ "local exception",],)-def test_stream_error_producers_preserve_continuation_permission(output, failure, continuable):+def test_stream_error_producers_preserve_continuation_permission(output, failure):"""C-2: terminal sanitization must not resurrect exhausted/refused/local failures after thinking.Prior tests covered kinds and replay only, so the outer adapter could undo their decision.@@ -4820,8 +4822,13 @@ def test_stream_error_producers_preserve_continuation_permission(output, failureelse:error = assembler.exception(RuntimeError("local failure"))terminal = assembler.terminal(error)- assert terminal.continuable is continuable- expected_retry = (continuable and output != "text") or (failure == "unknown wire" and output == "thinking")+ expected_continuable = failure in {"classified transient", "explicit transient", "eof", "stalled"}+ if failure == "recovery exhausted":+ expected_continuable = output == "none"+ assert terminal.continuable is expected_continuable+ expected_retry = (+ expected_continuable and failure != "recovery exhausted" and output != "text"+ ) or (failure == "unknown wire" and output == "thinking")assert terminal.retryable is expected_retryassert terminal.after_thinking is (expected_retry and output == "thinking")diff --git a/tests/test_vibey_agent.py b/tests/test_vibey_agent.pyindex a8ac795e9..4b6dca1da 100644--- a/tests/test_vibey_agent.py+++ b/tests/test_vibey_agent.py@@ -1427,7 +1427,7 @@ async def test_a_transient_failure_shows_one_notice_after_five_continuations(("streamed eof", True),("streamed rate limit", True),("rate limit past budget", False),- ("recovery exhausted", False),+ ("recovery exhausted", True),("recovery exhausted after thinking", False),("unknown", False),("malformed", False),@@ -1436,7 +1436,8 @@ async def test_a_transient_failure_shows_one_notice_after_five_continuations(async def test_provider_continuation_decision_reaches_vibey(engine, session, tmp_path, published, continuation_policy, failure, continues):- """Classification -> loop -> adapter: neither an exhausted Hub nor a permanent stream fault may be retried.+ """Classification -> loop -> adapter: exhausted Hub recovery may use one safe transcript continuation only when+ no output was produced; it must never replay a request after thinking or replay committed tools.Kind-only scripted tests missed real producers and erased Retry-After before the consumer saw it."""