Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

fix(vibey): continue after transient hub recovery exhaustion (#2720)

Call Whitef8829a23bdc9
diff --git a/core/agent_core/ai/errors.py b/core/agent_core/ai/errors.py
index 1fbed40f1..dbacfc82f 100644
--- a/core/agent_core/ai/errors.py
+++ b/core/agent_core/ai/errors.py
@@ -148,12 +148,16 @@ def classify_error(
if not effective_code or _is_generic_code(effective_code):
effective_code = extracted_code or effective_code
if effective_code == _MODEL_HUB_RECOVERY_EXHAUSTED:
- # Model Hub already retried within its recovery window and gave up: never retry again, and name the failure
- # it waited out rather than the request (the generic 4xx rule below would say the request was rejected).
+ # Model Hub already retried the same request within its recovery window. Do not replay that request again,
+ # but permit Vibey's bounded transcript-only continuation for a pre-output server/network failure: it resumes
+ # from committed context without rerunning committed tools or resending the original input.
+ recovery_cause = _recovery_cause(body)
+ kind = _RECOVERY_CAUSE_KINDS.get(recovery_cause or "", "server")
return ProviderError(
- kind=_RECOVERY_CAUSE_KINDS.get(_recovery_cause(body) or "", "server"),
+ kind=kind,
message=message,
retryable=False,
+ continuable=partial is None and recovery_cause in {"server_error", "network"},
retry_after_s=retry_after,
status=status,
partial=partial,
@@ -235,6 +239,10 @@ _RECOVERY_CAUSE_KINDS = {
"rate_limited": "rate_limit",
"quota_exhausted": "rate_limit",
}
+# Model Hub has already spent its own recovery window. A fresh transcript-only
+# continuation is safe only for an explicit pre-output server/network cause:
+# it cannot replay committed tools or resend the original input. Keep rate limits,
+# quota exhaustion, missing causes, and future causes terminal.
def _recovery_cause(body: str) -> str | None:
diff --git a/docs/plans/agent-core-contracts/provider-event.schema.json b/docs/plans/agent-core-contracts/provider-event.schema.json
index 468b7cda0..fe33e5463 100644
--- a/docs/plans/agent-core-contracts/provider-event.schema.json
+++ b/docs/plans/agent-core-contracts/provider-event.schema.json
@@ -42,7 +42,7 @@
"type": { "const": "error" },
"kind": { "enum": ["overflow", "rate_limit", "overloaded", "network", "stalled", "server", "interrupted", "auth", "invalid_request", "aborted", "unknown"] },
"retryable": { "type": "boolean" },
- "continuable": { "type": "boolean", "default": false, "description": "Explicit permission to continue from committed context, separate from same-request replay. Classified transient failures allow it; recovery exhaustion, refusals, permanent and local failures do not." },
+ "continuable": { "type": "boolean", "default": false, "description": "Explicit permission to continue from committed context, separate from same-request replay. Classified transient failures allow it; a pre-output recovery exhaustion with an explicit server_error or network cause may allow the bounded transcript-only continuation; missing/future causes, refusals, permanent and local failures do not." },
"retry_after_s": { "type": "number", "minimum": 0 },
"after_thinking": { "type": "boolean", "description": "The response streamed thinking and nothing else before this transient or unknown error; the loop retries it at most once per request (provider.md)." },
"status": { "type": "integer", "description": "HTTP status, when there was one." },
diff --git a/docs/plans/agent-core-contracts/provider.md b/docs/plans/agent-core-contracts/provider.md
index 931823f77..b969a7e85 100644
--- a/docs/plans/agent-core-contracts/provider.md
+++ b/docs/plans/agent-core-contracts/provider.md
@@ -119,10 +119,13 @@ Error classification is part of the contract because the loop branches on it:
`retryable` permits replay of the same request. `continuable` (default false) separately permits a new run
from the committed transcript after the failed run closes. Classified transient errors grant continuation even
-when streamed output forbids replay. Model Hub's `model_hub_recovery_exhausted` ending, explicit pin/opaque
-refusals, permanent/protocol errors, unclassified failures, and local preparation errors grant no continuation.
-Neither the terminal sanitizer nor a downstream consumer may infer permission from `kind` alone. In particular,
-a recovery-exhausted error naming `server` stays non-retryable and non-continuable after thinking, too.
+when streamed output forbids replay. A `model_hub_recovery_exhausted` ending remains non-retryable: for an
+explicit pre-output `server` or `network` cause, it may grant Vibey one of its existing bounded transcript-only
+continuation runs, which cannot replay the original input or committed tools. Rate-limit/quota exhaustion, missing or
+future causes, explicit pin/opaque refusals, permanent/protocol errors, unclassified failures, and local preparation
+errors grant no continuation. Neither the terminal sanitizer nor a downstream consumer may infer permission from
+`kind` alone. A recovery-exhausted error after thinking remains non-continuable, because the partial response is not
+safe to replay or continue implicitly.
The loop carries both permissions and `retry_after_s` into the terminal `AgentError` unchanged. These fields do
not grant permission to replay committed tools or manufacture user input; Vibey uses `Agent.resume` under its
diff --git a/docs/plans/model-hub-retry-experience.md b/docs/plans/model-hub-retry-experience.md
index 79e63def9..8e2a33559 100644
--- a/docs/plans/model-hub-retry-experience.md
+++ b/docs/plans/model-hub-retry-experience.md
@@ -22,9 +22,11 @@ The intended experience is:
Other requests may use backups or wait; they do not all hit the relay.
4. If that request produces valid model output, let work continue. If it
fails again, increase the delay. Timer expiry alone proves nothing.
-5. If automatic recovery ends, say that this request has ended and offer
- the existing safe Retry action where supported. Later supplier recovery
- must not silently revive a terminal Turn.
+5. If automatic recovery ends, the gateway request ends truthfully. For a pre-output `server` or `network` failure,
+ Vibey may use its existing bounded transcript-only continuation once the failed Turn closes; this does not replay the
+ original input or committed tools. Recovery exhaustion after thinking, rate-limit/quota exhaustion, or any ambiguous
+ acceptance remains terminal and uses the existing explicit Retry action where supported. Later supplier recovery must
+ not silently revive a terminal Turn.
The incident's observed `server_error` classification does not itself prove
a socket failure: a relay can return 503 because its own upstream is down.
diff --git a/tests/agent_core/ai/test_provider_adapters.py b/tests/agent_core/ai/test_provider_adapters.py
index b180bebee..ddbfd8e6b 100644
--- a/tests/agent_core/ai/test_provider_adapters.py
+++ b/tests/agent_core/ai/test_provider_adapters.py
@@ -4686,24 +4686,26 @@ async def test_model_hubs_stream_interrupted_frame_is_the_interrupted_kind(adapt
# Model Hub's ending once its recovery window ran out (``turn_gateway.py``): an HTTP 424 whose envelope names the
-# failure the hub waited out. It already retried that failure, so the loop must not retry again, and the copy must
-# name the failure, not blame the request as the generic 4xx rule would.
+# failure the hub waited out. The loop must not replay that same request; a pre-output server/network cause may still
+# permit Vibey's safe transcript-only continuation, while the copy must name the failure rather than blame the request.
@pytest.mark.asyncio
@pytest.mark.parametrize(
("adapter_type", "protocol"),
[(OpenAIResponsesAdapter, "openai_responses"), (OpenAIChatAdapter, "openai_chat"), (AnthropicAdapter, "anthropic")],
)
@pytest.mark.parametrize(
- ("cause", "kind"),
+ ("cause", "kind", "continuable"),
[
- ("server_error", "server"),
- ("network", "network"),
- ("rate_limited", "rate_limit"),
- ("quota_exhausted", "rate_limit"),
- (None, "server"),
+ ("server_error", "server", True),
+ ("network", "network", True),
+ ("rate_limited", "rate_limit", False),
+ ("quota_exhausted", "rate_limit", False),
+ (None, "server", False),
],
)
-async def test_model_hubs_recovery_exhausted_ending_names_its_cause(adapter_type, protocol, cause, kind) -> None:
+async def test_model_hubs_recovery_exhausted_ending_names_its_cause(
+ adapter_type, protocol, cause, kind, continuable
+) -> None:
ending = {
"type": "model_hub_recovery_exhausted",
"code": "model_hub_recovery_exhausted",
@@ -4720,7 +4722,7 @@ async def test_model_hubs_recovery_exhausted_ending_names_its_cause(adapter_type
error = events[-1]
assert isinstance(error, ProviderError)
- assert (error.kind, error.retryable, error.continuable) == (kind, False, False)
+ assert (error.kind, error.retryable, error.continuable) == (kind, False, continuable)
# CLIProxyAPI nests the class under `error`; OpenAI's reference carries it at the root.
@@ -4774,20 +4776,20 @@ async def test_only_a_drop_after_thinking_alone_is_marked_after_thinking(
@pytest.mark.parametrize("output", ["none", "thinking", "text"])
@pytest.mark.parametrize(
- "failure,continuable",
+ "failure",
[
- ("classified transient", True),
- ("explicit transient", True),
- ("eof", True),
- ("stalled", True),
- ("recovery exhausted", False),
- ("pin refusal", False),
- ("malformed", False),
- ("unknown wire", False),
- ("local exception", False),
+ "classified transient",
+ "explicit transient",
+ "eof",
+ "stalled",
+ "recovery exhausted",
+ "pin refusal",
+ "malformed",
+ "unknown wire",
+ "local exception",
],
)
-def test_stream_error_producers_preserve_continuation_permission(output, failure, continuable):
+def test_stream_error_producers_preserve_continuation_permission(output, failure):
"""C-2: terminal sanitization must not resurrect exhausted/refused/local failures after thinking.
Prior tests covered kinds and replay only, so the outer adapter could undo their decision.
@@ -4820,8 +4822,13 @@ def test_stream_error_producers_preserve_continuation_permission(output, failure
else:
error = assembler.exception(RuntimeError("local failure"))
terminal = assembler.terminal(error)
- assert terminal.continuable is continuable
- expected_retry = (continuable and output != "text") or (failure == "unknown wire" and output == "thinking")
+ expected_continuable = failure in {"classified transient", "explicit transient", "eof", "stalled"}
+ if failure == "recovery exhausted":
+ expected_continuable = output == "none"
+ assert terminal.continuable is expected_continuable
+ expected_retry = (
+ expected_continuable and failure != "recovery exhausted" and output != "text"
+ ) or (failure == "unknown wire" and output == "thinking")
assert terminal.retryable is expected_retry
assert terminal.after_thinking is (expected_retry and output == "thinking")
diff --git a/tests/test_vibey_agent.py b/tests/test_vibey_agent.py
index a8ac795e9..4b6dca1da 100644
--- a/tests/test_vibey_agent.py
+++ b/tests/test_vibey_agent.py
@@ -1427,7 +1427,7 @@ async def test_a_transient_failure_shows_one_notice_after_five_continuations(
("streamed eof", True),
("streamed rate limit", True),
("rate limit past budget", False),
- ("recovery exhausted", False),
+ ("recovery exhausted", True),
("recovery exhausted after thinking", False),
("unknown", False),
("malformed", False),
@@ -1436,7 +1436,8 @@ async def test_a_transient_failure_shows_one_notice_after_five_continuations(
async def test_provider_continuation_decision_reaches_vibey(
engine, session, tmp_path, published, continuation_policy, failure, continues
):
- """Classification -> loop -> adapter: neither an exhausted Hub nor a permanent stream fault may be retried.
+ """Classification -> loop -> adapter: exhausted Hub recovery may use one safe transcript continuation only when
+ no output was produced; it must never replay a request after thinking or replay committed tools.
Kind-only scripted tests missed real producers and erased Retry-After before the consumer saw it.
"""