Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

fix(model-hub): recover quietly before reporting terminal failures (#1966)

Call White4eaac3e650bd
diff --git a/core/backend_failure.py b/core/backend_failure.py
index b9102e8ec..138e11588 100644
--- a/core/backend_failure.py
+++ b/core/backend_failure.py
@@ -129,6 +129,39 @@ def _failure_texts(backend_name: str, diagnostic: Any, display_text: Any) -> tup
return error, visible
+def _model_hub_failure_text(
+ controller: Any, context: Any, request: Any, backend: str
+) -> str | None:
+ """Render exact live Hub evidence, never parse a native exception wrapper.
+
+ The registry owns ambiguity, pending-request, and Stop guards. Without that
+ evidence the original backend error remains authoritative; an earlier source
+ outage must not disguise an unrelated native/tool failure.
+ """
+
+ from core.handlers.model_hub.provenance import (
+ TurnOutcomeProjectionInput,
+ render_turn_outcome_copy,
+ )
+
+ gateway = getattr(controller, "model_hub_turn_gateway", None)
+ registry = getattr(gateway, "correlation", None)
+ project = getattr(registry, "terminal_projection", None)
+ if not callable(project):
+ return None
+ for source in (getattr(request, "context", None), context):
+ payload = getattr(source, "platform_specific", None) or {}
+ turn_id = payload.get("turn_token")
+ if not isinstance(turn_id, str) or not turn_id.strip():
+ continue
+ projection = project(turn_id.strip(), backend=backend)
+ if not isinstance(projection, TurnOutcomeProjectionInput):
+ continue
+ language = str(getattr(getattr(controller, "config", None), "language", "en") or "en")
+ return render_turn_outcome_copy(projection, language)
+ return None
+
+
def backend_failure_notification_output(
context: Any,
backend: str,
@@ -328,6 +361,9 @@ async def emit_backend_failure(
backend_name = str(backend or "backend").strip() or "backend"
error, visible = _failure_texts(backend_name, diagnostic, display_text)
+ hub_visible = _model_hub_failure_text(controller, context, request, backend_name)
+ if hub_visible is not None:
+ visible = hub_visible
terminal = _terminal_output(request, output)
harness_run_id = _harness_run_identity(context, request)
owns_failure_contract = bool(
diff --git a/core/controller.py b/core/controller.py
index c16f52676..52f729e99 100644
--- a/core/controller.py
+++ b/core/controller.py
@@ -709,6 +709,11 @@ class Controller:
self.model_hub_service,
language_provider=lambda: self.config.language,
)
+ from core.model_hub_progress import publish_recovery_changed
+
+ self.model_hub_turn_gateway.correlation.on_recovery_changed = (
+ lambda turn_id: publish_recovery_changed(self, turn_id)
+ )
self.model_hub_runtime = ModelHubRuntimeRouter(
service=self.model_hub_service,
turn_gateway=self.model_hub_turn_gateway,
diff --git a/core/handlers/model_hub/adapter.py b/core/handlers/model_hub/adapter.py
index 960a32666..28a16009e 100644
--- a/core/handlers/model_hub/adapter.py
+++ b/core/handlers/model_hub/adapter.py
@@ -12,6 +12,7 @@ from __future__ import annotations
import asyncio
from dataclasses import dataclass
+from datetime import datetime
from enum import Enum
from typing import Any, AsyncIterator, Callable, Final, Literal, Mapping, Protocol, Sequence
@@ -124,6 +125,14 @@ class RawCallOutcome:
# would otherwise take its token report with it, and a vendor that reported
# tokens billed for them whether or not the call ended well.
usage: ProtocolUsageReport | None = None
+ # Only failed HTTP responses supply retry advice: ASCII, at most 128
+ # characters, unparsed. L2 owns validation and the resulting retry policy.
+ retry_after: str | None = None
+ # Aware UTC receipt time of those headers, before reading the error body.
+ response_received_at: datetime | None = None
+ # Recognized model output or protocol success, independently of permissive
+ # forwarding. An unrecognized HTTP 200 alone cannot prove Source recovery.
+ recovery_verified: bool = False
class ObservationOutcome(str, Enum):
diff --git a/core/handlers/model_hub/provenance.py b/core/handlers/model_hub/provenance.py
index 6b898dee8..6f44aabc8 100644
--- a/core/handlers/model_hub/provenance.py
+++ b/core/handlers/model_hub/provenance.py
@@ -102,7 +102,9 @@ def exact_hop_blockers(
inspection.source is not None
and EVENT_REASON_AUTHORITY.get(str(reason)) != "structural"
):
- if inspection.source.state.status == "cooldown":
+ if inspection.backoff or inspection.recovery == "in_flight":
+ reason = "network" if inspection.backoff else "cooldown"
+ elif inspection.cooldown:
reason = "cooldown"
elif inspection.source.state.detail_key is not None:
reason = SOURCE_DETAIL_EVENT_REASONS.get(
@@ -405,9 +407,9 @@ def turn_supply_facts(
"waiting" if resolution.supply_status == "waiting" else "interrupted"
)
cooling = tuple(
- source
- for source in resolution.matching_sources
- if source.state.status == "cooldown" and source.state.retry_at
+ hop
+ for hop in resolution.inspected_hops
+ if hop.temporary_blocker and not hop.runnable
)
blockers: list[TurnSupplyBlocker] = []
for inspection in resolution.inspected_hops:
@@ -416,9 +418,10 @@ def turn_supply_facts(
source = inspection.source
reason = inspection.reason
if reason not in EVENT_REASON_AUTHORITY and source is not None:
- reason = SOURCE_DETAIL_EVENT_REASONS.get(
- source.state.detail_key or "",
- reason,
+ reason = (
+ inspection.cooldown_reason
+ if inspection.cooldown_reason in EVENT_REASON_AUTHORITY
+ else SOURCE_DETAIL_EVENT_REASONS.get(source.state.detail_key or "", reason)
)
if reason not in EVENT_REASON_AUTHORITY:
continue
@@ -443,9 +446,9 @@ def turn_supply_facts(
backend=resolution.backend,
model=model,
supply_state=supply_state,
- source=", ".join(source.display_name for source in cooling),
+ source=", ".join(hop.source.display_name for hop in cooling if hop.source is not None),
retry_at=min(
- (source.state.retry_at or "" for source in cooling),
+ (hop.retry_at for hop in cooling if hop.retry_at),
key=parse_model_hub_timestamp,
default="",
),
@@ -533,6 +536,7 @@ class TurnTrace:
terminal_outcome: TurnOutcomeProjectionInput | None = None
admission_closed: bool = False
outcome_frozen: bool = False
+ recovery_requests: dict[str, dict] = field(default_factory=dict)
@property
def pending_attempt(self) -> Optional[AttemptIdentity]:
@@ -616,6 +620,7 @@ class GatewayTurnTerminalizer:
self._stream_started = False
self._attempt_started = False
self._downstream_canceled = False
+ self._recovery_closed = False
self.on_attribution_released: Callable[[], None] | None = None
def __enter__(self) -> "GatewayTurnTerminalizer":
@@ -718,6 +723,15 @@ class GatewayTurnTerminalizer:
def mark_stream_started(self) -> None:
self._stream_started = True
+ def update_recovery(self, snapshot: dict | None) -> None:
+ if self._recovery_closed:
+ return
+ if snapshot is None:
+ self._recovery_closed = True
+ self._registry.update_recovery(
+ self.turn_id, backend=self._backend, request_id=self._request_id, snapshot=snapshot,
+ )
+
def record_turn_outcome(
self,
turn_outcome: TurnOutcomeProjectionInput | None,
@@ -844,13 +858,82 @@ class BoundedProvenanceStore:
class TurnCorrelationRegistry:
"""Correlate process credentials to the existing Workbench turn token."""
- def __init__(self, store: BoundedProvenanceStore):
+ def __init__(
+ self, store: BoundedProvenanceStore,
+ *, on_recovery_changed: Callable[[str], None] | None = None,
+ ):
self.store = store
self._lock = threading.RLock()
self._scopes: dict[ScopeKey, ProcessScope] = {}
self._credentials: dict[str, GatewayCredential] = {}
self._turn_scopes: dict[str, set[ScopeKey]] = {}
self._traces: dict[str, TurnTrace] = {}
+ self.on_recovery_changed = on_recovery_changed
+
+ def _readable_trace(
+ self, turn_id: str, backend: str | None = None, *, allow_closed: bool = False,
+ ) -> TurnTrace | None:
+ """Called under the registry lock; attribution is never inferred."""
+
+ trace = self._traces.get(turn_id)
+ if (
+ trace is None or trace.ambiguous or trace.outcome_frozen
+ or (trace.admission_closed and not allow_closed)
+ or (backend is not None and trace.agent != backend)
+ ):
+ return None
+ keys = self._turn_scopes.get(turn_id, set())
+ if not keys or any(
+ key[0] != trace.agent or (scope := self._scopes.get(key)) is None
+ or scope.untracked_use or turn_id in scope.ambiguous_turns
+ or turn_id not in scope.active_turns
+ for key in keys
+ ):
+ return None
+ return trace
+
+ def terminal_projection(self, turn_id: str, *, backend: str) -> TurnOutcomeProjectionInput | None:
+ with self._lock:
+ trace = self._readable_trace(turn_id, backend, allow_closed=True)
+ if (
+ trace is None or trace.pending_attempts or trace.terminal_outcome is None
+ or trace.terminal_outcome.outcome in {"served", "canceled"}
+ ):
+ return None
+ return trace.terminal_outcome
+
+ def recovery_snapshot(self, turn_id: str) -> list[dict]:
+ with self._lock:
+ trace = self._readable_trace(turn_id)
+ return [dict(item) for item in trace.recovery_requests.values()] if trace else []
+
+ def update_recovery(
+ self, turn_id: str | None, *, backend: str, request_id: str, snapshot: dict | None,
+ ) -> None:
+ if turn_id is None:
+ return
+ changed = False
+ with self._lock:
+ trace = self._readable_trace(turn_id, backend)
+ if trace is None:
+ return
+ previous = trace.recovery_requests.get(request_id)
+ current = {**snapshot, "request_id": request_id} if snapshot is not None else None
+ if previous != current:
+ changed = True
+ if current is None:
+ trace.recovery_requests.pop(request_id, None)
+ else:
+ trace.recovery_requests[request_id] = current
+ if changed and self.on_recovery_changed is not None:
+ self._notify_recovery_changed(turn_id)
+
+ def _notify_recovery_changed(self, turn_id: str) -> None:
+ if self.on_recovery_changed is not None:
+ try:
+ self.on_recovery_changed(turn_id)
+ except Exception:
+ logger.exception("Could not publish Model Hub recovery activity")
@staticmethod
def _scope_key(backend: str, process_scope: str) -> ScopeKey:
@@ -1535,6 +1618,10 @@ class TurnCorrelationRegistry:
trace = self._traces.get(normalized)
if trace is None or trace.outcome_frozen:
return
+ if self.terminal_projection(normalized, backend=trace.agent) is not None:
+ # The native failure is reporting the gateway's completed
+ # failure, not rejecting content that the gateway served.
+ return
trace.outcome_frozen = True
identity = trace.pending_attempt
payload = (
@@ -1586,7 +1673,10 @@ class TurnCorrelationRegistry:
return
if decision.action == "fallback" and decision.reason is not None:
trace.failed_attempts.append(
- {**identity.payload(), "reason": decision.reason}
+ {
+ **identity.payload(), "reason": decision.reason,
+ **({"http_status": outcome.http_status} if type(outcome.http_status) is int and 100 <= outcome.http_status <= 599 else {}),
+ }
)
return
if decision.action == "surface":
@@ -1616,15 +1706,20 @@ class TurnCorrelationRegistry:
if trace is None or trace.admission_closed:
return
trace.admission_closed = True
+ had_recovery = bool(trace.recovery_requests)
+ trace.recovery_requests.clear()
if settled_by == SETTLED_BY_STOPPED:
# Preserve the exact facts present at Stop. Teardown may expose
# a later producer success, but cannot change who ended the turn.
trace.outcome_frozen = True
+ if had_recovery:
+ self._notify_recovery_changed(turn_id)
def settle(self, turn_id: str, *, settled_by: Optional[str], ts: Optional[str] = None) -> None:
normalized_turn_id = str(turn_id or "").strip()
if not normalized_turn_id:
return
+ self.close_turn_admission(normalized_turn_id, settled_by=settled_by)
with self._lock:
trace = self._traces.pop(normalized_turn_id, None)
scope_keys = self._turn_scopes.pop(normalized_turn_id, set())
diff --git a/core/handlers/model_hub/resolver.py b/core/handlers/model_hub/resolver.py
index 8258c5d27..7a38c53b0 100644
--- a/core/handlers/model_hub/resolver.py
+++ b/core/handlers/model_hub/resolver.py
@@ -5,7 +5,7 @@ from __future__ import annotations
import re
from dataclasses import dataclass, replace
from datetime import datetime, timezone
-from typing import Literal
+from typing import Literal, Mapping
from config.v2_config import (
MODEL_HUB_BACKENDS,
@@ -30,6 +30,16 @@ class EffectiveModelRoute:
hops: tuple[ModelHubRouteHopConfig, ...]
+@dataclass(frozen=True)
+class SourceRecoveryAnnotation:
+ reason: str
+ retry_at: str | None = None
+ in_flight: bool = False
+ # Recovery retires only this exact old disk observation, never a later
+ # cooldown in a fresh config or a draft with the same Source identity.
+ retired_cooldown: tuple[str | None, str | None] | None = None
+
+
@dataclass(frozen=True)
class ExactHopInspection:
"""Canonical identity and live eligibility for one effective Route hop."""
@@ -46,6 +56,19 @@ class ExactHopInspection:
structural_blocker: bool
reason: str | None
retry_at: str | None
+ backoff: bool = False
+ recovery: Literal["eligible", "in_flight"] | None = None
+ cooldown: bool = False
+ cooldown_reason: str | None = None
+
+ @property
+ def temporary_blocker(self) -> bool:
+ return (
+ self.source is not None and self.supply_eligible
+ and not self.structural_blocker
+ and self.source.state.status not in {"needs_action", "error"}
+ and (self.backoff or self.cooldown or self.recovery == "in_flight")
+ )
@property
def identity(self) -> tuple[BackendName, str, str | None, str | None]:
@@ -283,6 +306,7 @@ def inspect_exact_hop(
now: datetime | None = None,
unavailable_source_ids: frozenset[str] = frozenset(),
supply_channel: Literal["hub"] | None = None,
+ live_recovery: Mapping[str, SourceRecoveryAnnotation] | None = None,
) -> ExactHopInspection:
"""Inspect one exact hop without matching, substituting, or reordering it."""
@@ -345,6 +369,38 @@ def inspect_exact_hop(
structural_blocker = True
elif source.state.status in {"needs_action", "error"}:
reason = source.state.detail_key
+ annotation = (live_recovery or {}).get(source.id) if source.supply_channel == "hub" else None
+ backoff = False
+ recovery = None
+ cooldown = source.state.status == "cooldown"
+ cooldown_reason = None
+ retry_at = source.state.retry_at if source.state.status == "cooldown" else None
+ if annotation is not None and supply_eligible and source.state.status not in {"needs_action", "error"}:
+ if not structural_blocker:
+ retired = (
+ cooldown and annotation.retired_cooldown is not None
+ and annotation.retired_cooldown == (source.state.retry_at, source.state.detail_key)
+ )
+ if retired:
+ cooldown = False
+ if annotation.reason != "recovery" and (annotation.reason != "network" or not cooldown):
+ cooldown_reason = annotation.reason
+ if annotation.reason == "recovery":
+ if retired:
+ runnable = True
+ retry_at = None
+ elif annotation.retry_at is not None:
+ runnable = False
+ retry_at = annotation.retry_at
+ backoff = annotation.reason == "network" and not cooldown
+ cooldown = not backoff
+ if backoff:
+ reason = "models.source.backoff.connection_failed"
+ else:
+ recovery = "in_flight" if annotation.in_flight else "eligible"
+ runnable = not annotation.in_flight
+ cooldown = False
+ retry_at = None
return ExactHopInspection(
backend=backend,
menu_model=menu_model,
@@ -357,11 +413,11 @@ def inspect_exact_hop(
runnable=runnable,
structural_blocker=structural_blocker,
reason=reason,
- retry_at=(
- source.state.retry_at
- if source.state.status == "cooldown"
- else None
- ),
+ retry_at=retry_at,
+ backoff=backoff,
+ recovery=recovery,
+ cooldown=cooldown,
+ cooldown_reason=cooldown_reason,
)
@@ -376,11 +432,7 @@ def _supply_status(
if any(inspection.runnable for inspection in inspections):
return "ok" if all(inspection.runnable for inspection in inspections) else "degraded"
if all(
- inspection.source is not None
- and inspection.source.id not in unavailable_source_ids
- and inspection.supply_eligible
- and inspection.source.state.status == "cooldown"
- and not source_retry_ready(inspection.source, now)
+ inspection.temporary_blocker
for inspection in inspections
):
return "waiting"
@@ -395,6 +447,7 @@ def resolve_model_hub_turn(
now: datetime | None = None,
unavailable_source_ids: frozenset[str] = frozenset(),
supply_channel: Literal["hub"] | None = None,
+ live_recovery: Mapping[str, SourceRecoveryAnnotation] | None = None,
) -> ModelHubTurnResolution:
"""Annotate the effective route without changing its membership or tier."""
@@ -431,6 +484,7 @@ def resolve_model_hub_turn(
now=now,
unavailable_source_ids=unavailable_source_ids,
supply_channel=supply_channel,
+ live_recovery=live_recovery,
)
for hop in route.hops
)
@@ -443,6 +497,7 @@ def resolve_model_hub_turn(
now=now,
unavailable_source_ids=unavailable_source_ids,
supply_channel=supply_channel,
+ live_recovery=live_recovery,
).reason
if not route.hops
else None
@@ -526,7 +581,12 @@ def resolve_model_hub_turn(
route_reason=route_reason,
recoverable_source_ids=recoverable,
supply_status=_supply_status(
- inspected_hops,
+ tuple(
+ hop for hop in inspected_hops
+ if supply_channel is None or hop.source is None
+ or hop.source.supply_channel == supply_channel
+ or hop.structural_blocker or hop.source.state.status in {"needs_action", "error"}
+ ),
now=now,
unavailable_source_ids=unavailable_source_ids,
),
diff --git a/core/handlers/model_hub/retry.py b/core/handlers/model_hub/retry.py
new file mode 100644
index 000000000..8cf965a3c
--- /dev/null
+++ b/core/handlers/model_hub/retry.py
@@ -0,0 +1,310 @@
+"""Service-owned, identity-scoped recovery admission for pending inference."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import random
+import time
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from email.utils import parsedate_to_datetime
+from typing import Awaitable, Callable
+
+from config.v2_config import ModelHubConfig, ModelHubSourceConfig
+
+from .resolver import SourceRecoveryAnnotation, parse_model_hub_timestamp
+
+
+logger = logging.getLogger(__name__)
+RECOVERY_WINDOW_SECONDS = 120.0
+RECOVERY_EXHAUSTED_CODE = "model_hub_recovery_exhausted"
+RECOVERY_EXHAUSTED_MESSAGE = "Automatic recovery has ended. Try again or choose another model."
+RETRY_DELAYS = {
+ "network": (1, 2, 4, 8, 16, 30),
+ "server_error": (30, 60, 120),
+ "rate_limited": (60, 120, 240, 300),
+ "quota_exhausted": (300,),
+}
+
+
+def source_identity(source: ModelHubSourceConfig) -> tuple:
+ """Configuration identity, independent of mutable health and inventory."""
+
+ return (
+ source.kind, source.vendor, source.protocol, source.supply_channel,
+ source.base_url, source.credential_ref, source.verification_pending,
+ )
+
+
+def retry_after_deadline(value: str | None, received_at: datetime) -> datetime | None:
+ """Validate the bounded advice without reflecting its contents in diagnostics."""
+
+ if value is None:
+ return None
+ try:
+ if not isinstance(value, str) or len(value) > 128 or not value.isascii():
+ raise ValueError
+ value = value.strip()
+ if value.isdecimal():
+ deadline = received_at + timedelta(seconds=int(value))
+ else:
+ deadline = parsedate_to_datetime(value)
+ if deadline.tzinfo is None:
+ raise ValueError
+ if deadline < received_at:
+ raise ValueError
+ return deadline
+ except (ValueError, TypeError, OverflowError):
+ logger.debug("Ignored invalid or past Model Hub Retry-After advice")
+ return None
+
+
+@dataclass
+class RecoveryRequest:
+ policy: RecoveryPolicy
+ started: float | None = None
+ started_at: datetime | None = None
+ window_end: datetime | None = None
+ attempt_count: int = 0
+ source_id: str | None = None
+ reason: str | None = None
+ observer: Callable[[dict | None], None] | None = None
+
+ def start(self) -> None:
+ if self.started is None:
+ self.started = self.policy.monotonic()
+ self.started_at = self.policy.now()
+ self.window_end = self.started_at + timedelta(seconds=self.policy.window_seconds)
+
+ @property
+ def remaining(self) -> float:
+ if self.started is None:
+ return self.policy.window_seconds
+ return max(0.0, self.policy.window_seconds - (self.policy.monotonic() - self.started))
+
+ @property
+ def expired(self) -> bool:
+ return self.started is not None and self.remaining <= 0
+
+ def publish(self, phase: str, *, next_eligible_at: str | None = None) -> None:
+ if self.observer is not None and self.started is not None:
+ self.observer({
+ "phase": phase,
+ "started_at": self.started_at.isoformat() if self.started_at else None,
+ "attempt_count": self.attempt_count,
+ "source_id": self.source_id,
+ "reason": self.reason,
+ "next_eligible_at": next_eligible_at,
+ "window_end": self.window_end.isoformat() if self.window_end else None,
+ })
+
+ def clear(self) -> None:
+ if self.observer is not None:
+ self.observer(None)
+
+
+@dataclass
+class _SourceRecovery:
+ identity: tuple
+ reason: str
+ failures: int
+ retry_at: datetime
+ eligible_at: float
+ owner: int | None = None
+ failed_generation: int | None = None
+
+
+class RecoveryPolicy:
+ """Live deadlines and half-open ownership; the service supplies synchronization."""
+
+ def __init__(
+ self,
+ *,
+ now: Callable[[], datetime],
+ monotonic: Callable[[], float] = time.monotonic,
+ jitter: Callable[[float, float], float] = random.uniform,
+ sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
+ window_seconds: float = RECOVERY_WINDOW_SECONDS,
+ ):
+ self.now = now
+ self.monotonic = monotonic
+ self.jitter = jitter
+ self.sleep = sleep
+ self.window_seconds = window_seconds
+ self._sources: dict[str, _SourceRecovery] = {}
+ # Exact persisted observations superseded by verified inference. Keep
+ # them only while a failed save leaves that same cooldown on disk.
+ self._retired_cooldowns: dict[str, tuple] = {}
+ self.changed = asyncio.Event()
+
+ def notify(self) -> None:
+ # Replacing the event avoids lost wakeups and lets all current waiters wake.
+ previous, self.changed = self.changed, asyncio.Event()
+ previous.set()
+
+ def reconcile(self, config: ModelHubConfig) -> None:
+ sources = {source.id: source for source in config.sources if source.supply_channel == "hub"}
+ removed = [
+ source_id for source_id, state in self._sources.items()
+ if source_id not in sources or source_identity(sources[source_id]) != state.identity
+ ]
+ for source_id in removed:
+ del self._sources[source_id]
+ retired = [
+ source_id for source_id, observation in self._retired_cooldowns.items()
+ if source_id not in sources or self._cooldown_observation(sources[source_id]) != observation
+ ]
+ for source_id in retired:
+ del self._retired_cooldowns[source_id]
+ if removed or retired:
+ self.notify()
+
+ @staticmethod
+ def _cooldown_observation(source: ModelHubSourceConfig) -> tuple | None:
+ if source.state.status != "cooldown":
+ return None
+ return source_identity(source), source.state.retry_at, source.state.detail_key
+
+ def _state(self, source: ModelHubSourceConfig) -> _SourceRecovery | None:
+ # Native calls have no HTTP attempt/success owner. Their shipped fixed
+ # cooldown and native retry lifecycle remain outside this coordinator.
+ if source.supply_channel != "hub":
+ return None
+ state = self._sources.get(source.id)
+ if state is not None and state.identity != source_identity(source):
+ del self._sources[source.id]
+ self.notify()
+ state = None
+ retired = self._retired_cooldowns.get(source.id)
+ if retired is not None:
+ if retired == self._cooldown_observation(source):
+ return state
+ del self._retired_cooldowns[source.id]
+ # Restore eligibility from shipped cooldowns, never a recovered verdict.
+ if state is None and source.state.status == "cooldown" and source.state.retry_at:
+ retry_at = parse_model_hub_timestamp(source.state.retry_at)
+ state = _SourceRecovery(
+ source_identity(source),
+ (source.state.detail_key or "").rsplit(".", 1)[-1],
+ 0,
+ retry_at,
+ self.monotonic() + max(0.0, (retry_at - self.now()).total_seconds()),
+ )
+ self._sources[source.id] = state
+ elif (
+ state is not None and source.state.status == "cooldown"
+ and source.state.retry_at
+ ):
+ persisted = parse_model_hub_timestamp(source.state.retry_at)
+ if persisted > state.retry_at:
+ state.retry_at = persisted
+ state.eligible_at = max(
+ state.eligible_at,
+ self.monotonic() + max(0.0, (persisted - self.now()).total_seconds()),
+ )
+ return state
+
+ def annotations(self, config: ModelHubConfig) -> dict[str, SourceRecoveryAnnotation]:
+ self.reconcile(config)
+ result = {}
+ for source in config.sources:
+ state = self._state(source)
+ retired = self._retired_cooldowns.get(source.id)
+ if state is None:
+ if retired is not None:
+ result[source.id] = SourceRecoveryAnnotation(
+ reason="recovery", retired_cooldown=retired[1:],
+ )
+ continue
+ remaining = max(0.0, state.eligible_at - self.monotonic())
+ result[source.id] = SourceRecoveryAnnotation(
+ reason=state.reason,
+ # Project remaining monotonic time onto the current wall clock.
+ retry_at=(self.now() + timedelta(seconds=remaining)).isoformat() if remaining else None,
+ in_flight=state.owner is not None,
+ retired_cooldown=retired[1:] if retired is not None else None,
+ )
+ return result
+
+ def failed(
+ self,
+ source: ModelHubSourceConfig,
+ reason: str,
+ *,
+ generation: int,
+ retry_after: str | None = None,
+ response_received_at: datetime | None = None,
+ ) -> tuple[datetime, bool]:
+ state = self._state(source)
+ if state is not None and state.failed_generation is not None and generation <= state.failed_generation:
+ return state.retry_at, False
+ failures = state.failures + 1 if state is not None else 1
+ delays = RETRY_DELAYS[reason]
+ base, cap = delays[min(failures - 1, len(delays) - 1)], delays[-1]
+ local = self.jitter(base, min(cap, base * 1.2))
+ now, mono = self.now(), self.monotonic()
+ received = response_received_at
+ if received is None or received.tzinfo is None:
+ received = now
+ upstream = retry_after_deadline(retry_after, received)
+ deadline = max(now + timedelta(seconds=local), upstream or now)
+ eligible_at = mono + (deadline - now).total_seconds()
+ if state is not None:
+ eligible_at = max(eligible_at, state.eligible_at)
+ deadline = max(deadline, state.retry_at, now + timedelta(seconds=eligible_at - mono))
+ self._sources[source.id] = _SourceRecovery(
+ source_identity(source), reason, failures, deadline, eligible_at,
+ failed_generation=generation,
+ )
+ self.notify()
+ return deadline, True
+
+ def claim(self, source: ModelHubSourceConfig, generation: int) -> bool:
+ state = self._state(source)
+ if state is None:
+ return True
+ if state.owner is not None or state.eligible_at > self.monotonic():
+ return False
+ state.owner = generation
+ self.notify()
+ return True
+
+ def release(self, source_id: str, generation: int | None) -> None:
+ state = self._sources.get(source_id)
+ if state is not None and generation is not None and state.owner == generation:
+ state.owner = None
+ self.notify()
+
+ def succeeded(self, source: ModelHubSourceConfig) -> bool:
+ state = self._state(source)
+ if state is None:
+ return False
+ del self._sources[source.id]
+ observation = self._cooldown_observation(source)
+ if observation is not None:
+ self._retired_cooldowns[source.id] = observation
+ self.notify()
+ return True
+
+ def verification_retired(self, source_id: str, previous: tuple, current: tuple) -> None:
+ """Retiring a verification marker does not replace the credential."""
+
+ state = self._sources.get(source_id)
+ if state is not None and state.identity == previous:
+ state.identity = current
+ retired = self._retired_cooldowns.get(source_id)
+ if retired is not None and retired[0] == previous:
+ self._retired_cooldowns[source_id] = (current, *retired[1:])
+
+ async def wait(self, delay: float, changed: asyncio.Event) -> None:
+ """Wait without owning the configuration lock or a transport."""
+
+ timer = asyncio.create_task(self.sleep(delay))
+ notification = asyncio.create_task(changed.wait())
+ try:
+ await asyncio.wait((timer, notification), return_when=asyncio.FIRST_COMPLETED)
+ finally:
+ timer.cancel()
+ notification.cancel()
+ await asyncio.gather(timer, notification, return_exceptions=True)
diff --git a/core/handlers/model_hub/service.py b/core/handlers/model_hub/service.py
index 70d1436fa..76cbe7726 100644
--- a/core/handlers/model_hub/service.py
+++ b/core/handlers/model_hub/service.py
@@ -119,6 +119,7 @@ from .stream_wire import ProtocolSSEState
from .resolver import (
BackendName,
ModelHubTurnResolution,
+ SourceRecoveryAnnotation,
allowed_origins,
effective_model_route,
inspect_exact_hop,
@@ -132,6 +133,7 @@ from .resolver import (
source_runnable,
)
from .revocations import CredentialRevocationJournal
+from .retry import RECOVERY_EXHAUSTED_CODE, RecoveryPolicy, RecoveryRequest, RETRY_DELAYS, source_identity
from .usage import USAGE_DEFAULT_WINDOW_DAYS, BoundedUsageLedger, SourceIdentity, UsageWriter
CONTRACT_VERSION = 10
@@ -409,6 +411,10 @@ class _InvocationPlanChanged(Exception):
"""No transport was admitted; recompute the remaining effective route."""
+class _RecoveryWindowClosed(Exception):
+ """The request may finish its admitted inference, but cannot start another."""
+
+
@dataclass(frozen=True)
class ResolvedInvocation:
backend: BackendName
@@ -779,6 +785,7 @@ class ModelHubService:
Callable[[BackendName], Awaitable[None]]
] = None,
now: Callable[[], datetime] = _utc_now,
+ recovery: RecoveryPolicy | None = None,
):
self.store = store
self.adapter = adapter
@@ -810,6 +817,7 @@ class ModelHubService:
self.cli_presence_refresh = cli_presence_refresh
self.backend_catalog_changed = backend_catalog_changed
self.now = now
+ self.recovery = recovery or RecoveryPolicy(now=lambda: self.now())
self.native_source_ready: Callable[[BackendName, ModelHubSourceConfig], bool] = (
lambda _backend, _source: True
)
@@ -820,6 +828,7 @@ class ModelHubService:
self._source_create_nonces: set[str] = set()
self._next_settlement_generation = PRE_ATTEMPT_SETTLEMENT_GENERATION
self._latest_source_attempt_generation: dict[str, int] = {}
+ self._source_attempt_identities: dict[str, tuple] = {}
self._engine_synced = False
self._engine_preparation_failed = False
self._runtime_install_reconcile_lock = asyncio.Lock()
@@ -978,6 +987,8 @@ class ModelHubService:
def _save_config(self, config: ModelHubConfig) -> ModelHubConfig:
canonical = ModelHubConfig.from_payload(config.to_payload())
self.store.save(canonical)
+ self.recovery.reconcile(canonical)
+ self.recovery.notify()
return canonical
def _save_projection_neutral(
@@ -1012,8 +1023,30 @@ class ModelHubService:
self._latest_source_attempt_generation[source_id] = (
self._next_settlement_generation
)
+ source = next((item for item in self.store.load().sources if item.id == source_id), None)
+ if source is not None:
+ self._source_attempt_identities[source_id] = source_identity(source)
return self._next_settlement_generation
+ def _settlement_current(self, source: ModelHubSourceConfig, generation: int | None) -> bool:
+ return (
+ generation is not None
+ and generation >= self._latest_source_attempt_generation.get(source.id, generation)
+ and self._source_attempt_identities.get(source.id, source_identity(source)) == source_identity(source)
+ )
+
+ def recovery_annotations(self, config: ModelHubConfig) -> dict[str, SourceRecoveryAnnotation]:
+ """Read live health once; preview/guard drafts cannot mutate its owner."""
+
+ current = self.store.load()
+ annotations = self.recovery.annotations(current)
+ identities = {source.id: source_identity(source) for source in current.sources}
+ return {
+ source.id: annotations[source.id]
+ for source in config.sources
+ if source.id in annotations and identities.get(source.id) == source_identity(source)
+ }
+
def _ensure_config_writable(self) -> None:
ensure_writable = getattr(self.store, "ensure_writable", None)
if callable(ensure_writable):
@@ -3249,6 +3282,7 @@ class ModelHubService:
model_id,
now=self.now(),
unavailable_source_ids=unavailable_source_ids,
+ live_recovery=self.recovery_annotations(config),
)
if resolution.candidates:
continue
@@ -3662,7 +3696,9 @@ class ModelHubService:
def get_agent_sources(self, backend: str) -> dict:
config = self.store.load()
- return self._agent_payload(config, self._agent(config, backend))
+ return self._agent_payload(
+ config, self._agent(config, backend), live_recovery=self.recovery.annotations(config),
+ )
async def set_agent_chain(self, backend: str, model_id: object, payload: object) -> dict:
if (
@@ -3808,7 +3844,10 @@ class ModelHubService:
config = self.store.load()
return self._catalog_models_payload(config.agents[backend])
- def _agent_payload(self, config: ModelHubConfig, agent: ModelHubAgentSupplyConfig) -> dict:
+ def _agent_payload(
+ self, config: ModelHubConfig, agent: ModelHubAgentSupplyConfig,
+ *, live_recovery: Mapping[str, SourceRecoveryAnnotation] | None = None,
+ ) -> dict:
backend = cast(BackendName, agent.backend)
builtin_models = (
[model.id for model in agent.models]
@@ -3818,12 +3857,15 @@ class ModelHubService:
requested_model = self._requested_model(agent)
unavailable_source_ids = self._unavailable_native_sources(config, backend)
now = self.now()
+ if live_recovery is None:
+ live_recovery = self.recovery_annotations(config)
resolution = resolve_model_hub_turn(
config,
backend,
requested_model,
now=now,
unavailable_source_ids=unavailable_source_ids,
+ live_recovery=live_recovery,
)
menu_model_ids = [model.id for model in agent.models]
model_supply = [
@@ -3836,6 +3878,7 @@ class ModelHubService:
for model_id in menu_model_ids
for model_resolution in (resolve_model_hub_turn(
config, backend, model_id, now=now, unavailable_source_ids=unavailable_source_ids,
+ live_recovery=live_recovery,
),)
]
selected_model_id = (
@@ -3889,6 +3932,7 @@ class ModelHubService:
requested,
now=now,
unavailable_source_ids=unavailable_source_ids,
+ live_recovery=live_recovery,
)
named_agents.append(
{
@@ -4110,7 +4154,11 @@ class ModelHubService:
def list_agents(self) -> list[dict]:
config = self.store.load()
- return [self._agent_payload(config, config.agents[backend]) for backend in ("claude", "codex", "opencode")]
+ live_recovery = self.recovery.annotations(config)
+ return [
+ self._agent_payload(config, config.agents[backend], live_recovery=live_recovery)
+ for backend in ("claude", "codex", "opencode")
+ ]
def refresh_cli_presence(
self,
@@ -4971,6 +5019,7 @@ class ModelHubService:
*,
now: Optional[datetime] = None,
unavailable_source_ids: Optional[frozenset[str]] = None,
+ live_recovery: Mapping[str, SourceRecoveryAnnotation] | None = None,
) -> dict:
agent = self._agent(config, backend)
if agent.mode == "direct":
@@ -4990,6 +5039,7 @@ class ModelHubService:
model_id,
now=observed_at,
unavailable_source_ids=unavailable,
+ live_recovery=live_recovery if live_recovery is not None else self.recovery_annotations(config),
)
chain: list[dict] = []
for inspection in resolution.inspected_hops:
@@ -5009,8 +5059,10 @@ class ModelHubService:
continue
status = source.state.status
health = (
- "healthy"
- if status in {"active", "standby"}
+ "backoff" if inspection.backoff
+ else "cooldown" if inspection.cooldown
+ else "healthy"
+ if status in {"active", "standby", "cooldown"}
else status
)
chain.append(
@@ -5022,6 +5074,7 @@ class ModelHubService:
"runnable": inspection.runnable,
"reason": inspection.reason,
"retry_at": inspection.retry_at,
+ **({"recovery": inspection.recovery} if inspection.recovery is not None else {}),
}
)
current = next(
@@ -5040,13 +5093,18 @@ class ModelHubService:
"route_origin": resolution.route_origin,
"chain": chain,
"current": current,
- "supply_state": self._chain_supply_state(chain),
+ "supply_state": (
+ "ok" if resolution.candidate_hops
+ else "waiting" if resolution.supply_status == "waiting"
+ else "interrupted"
+ ),
}
def agent_chain(self, backend: str, model_id: object) -> dict:
if backend not in MODEL_HUB_BACKENDS or not isinstance(model_id, str) or not model_id:
raise ModelHubError("mapping_target_unavailable", status=409)
- return self._agent_chain(self.store.load(), backend, model_id)
+ config = self.store.load()
+ return self._agent_chain(config, backend, model_id, live_recovery=self.recovery.annotations(config))
def agent_chains(self, backend: str) -> list[dict]:
if backend not in MODEL_HUB_BACKENDS:
@@ -5061,6 +5119,7 @@ class ModelHubService:
config,
cast(BackendName, backend),
)
+ live_recovery = self.recovery.annotations(config)
return [
self._agent_chain(
config,
@@ -5068,6 +5127,7 @@ class ModelHubService:
model_id,
now=observed_at,
unavailable_source_ids=unavailable_source_ids,
+ live_recovery=live_recovery,
)
for model_id in self._agent_model_ids(agent, requested_model)
]
@@ -5243,7 +5303,9 @@ class ModelHubService:
decision = classify_outcome(outcome)
if decision.action == "refresh":
decision = ResolutionDecision("fallback", reason="credential_revoked")
- error = None if succeeded else self._probe_failure(outcome, decision)[0]
+ error = None if succeeded else self._probe_failure(
+ outcome, decision, recovery_managed=False,
+ )[0]
return {
"source_id": source.id,
"model_id": model_id,
@@ -5257,13 +5319,21 @@ class ModelHubService:
def _probe_failure(
outcome: RawCallOutcome,
decision: ResolutionDecision,
+ *,
+ recovery_managed: bool,
) -> tuple[str, Optional[EventReason]]:
if decision.action == "surface":
return "models.source.error.unclassified", None
if outcome.kind == RawOutcomeKind.NETWORK_ERROR:
- return "models.source.cooldown.network", "network"
+ return (
+ "models.source.backoff.connection_failed" if recovery_managed else "models.source.cooldown.network",
+ "network",
+ )
if outcome.kind == RawOutcomeKind.TIMEOUT:
- return "models.source.cooldown.timeout", "network"
+ return (
+ "models.source.backoff.connection_failed" if recovery_managed else "models.source.cooldown.timeout",
+ "network",
+ )
if decision.reason in {
"credential_expired",
"credential_revoked",
@@ -5402,26 +5472,12 @@ class ModelHubService:
started_at = time.monotonic()
settlement_generation = None
- def admitted() -> None:
+ def admitted(generation: int) -> None:
nonlocal settlement_generation
- settlement_generation = self._reserve_settlement_generation(source.id)
-
- handle = await self._invoke_admitted(
- source=source,
- model_id=resolved_model,
- requested_model_id=chain_payload["model_id"],
- request=self._probe_request(source, resolved_model, backend),
- stream=False,
- backend=cast(BackendName, backend),
- excluded_source_ids=set(),
- on_admitted=admitted,
- )
- if handle.stream is not None:
- async for _chunk in handle.stream:
- pass
- outcome = await self._engine_call(handle.outcome())
- decision = await self._classify_source_outcome(source, outcome)
- if decision.action == "refresh":
+ settlement_generation = generation
+
+ handle: InvokeHandle | None = None
+ try:
handle = await self._invoke_admitted(
source=source,
model_id=resolved_model,
@@ -5430,48 +5486,75 @@ class ModelHubService:
stream=False,
backend=cast(BackendName, backend),
excluded_source_ids=set(),
- exact_retry=True,
on_admitted=admitted,
)
if handle.stream is not None:
async for _chunk in handle.stream:
pass
outcome = await self._engine_call(handle.outcome())
- decision = classify_outcome(outcome, refresh_attempted=True)
- elapsed_ms = max(0, round((time.monotonic() - started_at) * 1000))
- reachable = decision.action == "return"
- if reachable:
- await self._verify_successful_source(
- source.id, source.credential_ref, verification_pending, outcome,
- )
- error_key: Optional[str] = None
- latency_ms: Optional[int] = elapsed_ms
- if not reachable:
- error_key, event_reason = self._probe_failure(outcome, decision)
- if error_key in {
- "models.source.cooldown.network",
- "models.source.cooldown.timeout",
- }:
- latency_ms = None
- if event_reason is not None:
- await self._settle_fallback_source(
- source,
- decision,
+ decision = await self._classify_source_outcome(source, outcome)
+ if decision.action == "refresh":
+ await handle.close_stream()
+ self.recovery.release(source.id, settlement_generation)
+ handle = await self._invoke_admitted(
+ source=source,
+ model_id=resolved_model,
+ requested_model_id=chain_payload["model_id"],
+ request=self._probe_request(source, resolved_model, backend),
+ stream=False,
backend=cast(BackendName, backend),
- model_id=chain_payload["model_id"],
- detail_key=error_key,
- settlement_generation=settlement_generation,
+ excluded_source_ids=set(),
+ exact_retry=True,
+ on_admitted=admitted,
)
- return {
- "contract_version": PROBE_RESULT_CONTRACT_VERSION,
- "backend": backend,
- "channel": "hub",
- "reachable": reachable,
- "source_id": source.id,
- "model_id": resolved_model,
- "latency_ms": latency_ms,
- "error": error_key,
- }
+ if handle.stream is not None:
+ async for _chunk in handle.stream:
+ pass
+ outcome = await self._engine_call(handle.outcome())
+ decision = classify_outcome(outcome, refresh_attempted=True)
+ elapsed_ms = max(0, round((time.monotonic() - started_at) * 1000))
+ reachable = decision.action == "return"
+ if reachable:
+ if self._verified_recovery_outcome(outcome):
+ await self._record_recovery_success(
+ source.id, settlement_generation,
+ backend=cast(BackendName, backend), model_id=chain_payload["model_id"],
+ )
+ await self._verify_successful_source(
+ source.id, source.credential_ref, verification_pending, outcome,
+ )
+ error_key: Optional[str] = None
+ latency_ms: Optional[int] = elapsed_ms
+ if not reachable:
+ error_key, event_reason = self._probe_failure(outcome, decision, recovery_managed=True)
+ if error_key == "models.source.backoff.connection_failed":
+ latency_ms = None
+ if event_reason is not None:
+ await self._settle_fallback_source(
+ source,
+ decision,
+ backend=cast(BackendName, backend),
+ model_id=chain_payload["model_id"],
+ detail_key=error_key,
+ settlement_generation=settlement_generation,
+ outcome=outcome,
+ )
+ return {
+ "contract_version": PROBE_RESULT_CONTRACT_VERSION,
+ "backend": backend,
+ "channel": "hub",
+ "reachable": reachable,
+ "source_id": source.id,
+ "model_id": resolved_model,
+ "latency_ms": latency_ms,
+ "error": error_key,
+ }
+ finally:
+ try:
+ if handle is not None:
+ await await_owned_task(asyncio.create_task(handle.close_stream()))
+ finally:
+ self.recovery.release(source.id, settlement_generation)
@staticmethod
def note_turn_mode(
@@ -6124,38 +6207,43 @@ class ModelHubService:
self,
resolution: ModelHubTurnResolution,
) -> None:
- if not resolution.recoverable_source_ids:
- return
+ # Compatibility for existing callers: elapsed timers only admit a try.
+ self.recovery.annotations(self.store.load())
+
+ async def _record_recovery_success(
+ self, source_id: str, generation: int | None, *, backend: BackendName, model_id: str,
+ ) -> None:
async with self._mutation_lock:
config = self.store.load()
- previous = self._clone_config(config)
- config_changed = False
- recovered_sources: list[ModelHubSourceConfig] = []
- for source_id in resolution.recoverable_source_ids:
- source = next(
- (item for item in config.sources if item.id == source_id),
- None,
+ source = next((item for item in config.sources if item.id == source_id), None)
+ if (
+ source is None or not self._settlement_current(source, generation)
+ or source.supply_channel != "hub"
+ or source.state.status in {"needs_action", "error"}
+ ):
+ return
+ recovered = self.recovery.succeeded(source)
+ if recovered and source.state.status == "cooldown":
+ previous = self._clone_config(config)
+ source.state = ModelHubSourceStateConfig(status="standby")
+ try:
+ self._save_runtime_config(previous, config)
+ except OSError:
+ # Inference is authoritative; the policy remembers exactly
+ # which stale persisted cooldown must not be imported again.
+ logger.warning("Could not persist Model Hub recovered state")
+ if recovered:
+ self._record_event(
+ agent=cast(EventAgent, backend), kind="recover", model_id=model_id,
+ reason="recovery", to_source=source.id, to_label=source.display_name, now=self.now(),
)
- if source is None or source.state.status != "cooldown":
- continue
- recovered_source = source_after_cooldown_recovery(source, self.now())
- if recovered_source is source:
- continue
- source.state = recovered_source.state
- config_changed = True
- recovered_sources.append(source)
- if config_changed:
- if self._save_runtime_config(previous, config):
- for source in recovered_sources:
- self._record_event(
- agent=cast(EventAgent, resolution.backend),
- kind="recover",
- model_id=resolution.requested_model,
- reason="recovery",
- to_source=source.id,
- to_label=source.display_name,
- now=self.now(),
- )
+
+ @staticmethod
+ def _verified_recovery_outcome(outcome: RawCallOutcome) -> bool:
+ # Buffered compatibility admission also sets stream_started on unknown
+ # HTTP 200 bodies. Only the protocol evidence bit proves success here;
+ # actual streaming output is read separately from the handle observer.
+ return outcome.kind is RawOutcomeKind.SUCCESS and outcome.recovery_verified
def _write_cooldown_locked(
self,
@@ -6167,8 +6255,9 @@ class ModelHubService:
model_id: str,
detail_key: Optional[str] = None,
emit_event: bool = True,
+ retry_at: datetime | None = None,
) -> bool:
- retry_at = self.now() + timedelta(seconds=decision.cooldown_seconds)
+ retry_at = retry_at or self.now() + timedelta(seconds=decision.cooldown_seconds)
if (
source.state.status == "cooldown"
and source.state.retry_at is not None
@@ -6182,7 +6271,13 @@ class ModelHubService:
retry_at=retry_at.isoformat(),
detail_key=detail_key or f"models.source.cooldown.{decision.reason}",
)
- persisted = self._save_runtime_config(previous, config)
+ try:
+ persisted = self._save_runtime_config(previous, config)
+ except OSError:
+ # HTTP recovery already owns the live deadline. Failed telemetry
+ # cannot replace the upstream result or erase temporary eligibility.
+ logger.warning("Could not persist Model Hub cooldown state")
+ persisted = False
if persisted and not already_cooling and emit_event:
self._record_event(
agent=agent,
@@ -6205,6 +6300,7 @@ class ModelHubService:
emit_event: bool = True,
detail_key: Optional[str] = None,
settlement_generation: Optional[int] = None,
+ outcome: RawCallOutcome | None = None,
) -> tuple[EventReason, bool]:
"""Persist one fallback-class Source result before the turn settles."""
@@ -6212,7 +6308,7 @@ class ModelHubService:
raise AssertionError("fallback-class outcome must retain its Source reason")
event_reason = cast(EventReason, decision.reason)
settlement_rule = source_settlement_rule(event_reason)
- if not settlement_rule.may_write_health:
+ if decision.reason == "network" and outcome is not None and outcome.stream_started:
return event_reason, False
# Generations are minted at attempt start and nowhere else. A settlement
# that carries none cannot prove it is not superseded, so it does not
@@ -6223,20 +6319,36 @@ class ModelHubService:
return event_reason, False
generation = settlement_generation
async with self._mutation_lock:
+ self.recovery.release(source.id, generation)
config = self.store.load()
try:
current = self._source(config, source.id)
except ModelHubError:
return event_reason, False
- latest_generation = self._latest_source_attempt_generation.get(
- current.id,
- generation,
- )
- if generation < latest_generation:
+ if not self._settlement_current(current, generation):
return event_reason, False
if not source_settlement_allowed(current.state.status, event_reason):
return event_reason, False
if settlement_rule.status == "cooldown":
+ retry_at = None
+ changed = True
+ if current.supply_channel == "hub":
+ retry_at, changed = self.recovery.failed(
+ current, event_reason,
+ generation=generation,
+ retry_after=outcome.retry_after if outcome is not None else None,
+ response_received_at=outcome.response_received_at if outcome is not None else None,
+ )
+ if not changed:
+ return event_reason, False
+ if not settlement_rule.may_write_health:
+ if emit_event:
+ self._record_event(
+ agent=cast(EventAgent, backend), kind="cooldown", model_id=model_id,
+ reason=event_reason, from_source=current.id,
+ from_label=current.display_name, now=self.now(),
+ )
+ return event_reason, False
persisted = self._write_cooldown_locked(
config,
current,
@@ -6245,6 +6357,7 @@ class ModelHubService:
model_id=model_id,
detail_key=detail_key,
emit_event=emit_event,
+ retry_at=retry_at,
)
else:
blocker_detail_key = detail_key or {
@@ -6263,6 +6376,7 @@ class ModelHubService:
reason=event_reason,
emit_event=emit_event,
)
+ self.recovery.release(source.id, generation)
return event_reason, persisted
def _inspect_terminal_chain(
@@ -6283,6 +6397,7 @@ class ModelHubService:
config,
backend,
),
+ live_recovery=self.recovery_annotations(config),
)
return config, resolution
@@ -6388,7 +6503,10 @@ class ModelHubService:
"""Retire pending verification only for the current credential's call."""
if outcome.kind is not RawOutcomeKind.SUCCESS or verification_pending is None:
return
+ retired_identity: tuple[tuple, tuple] | None = None
+
def clear_current(config: ModelHubConfig) -> bool:
+ nonlocal retired_identity
current = next((item for item in config.sources if item.id == source_id), None)
if (
current is None
@@ -6396,12 +6514,19 @@ class ModelHubService:
or current.verification_pending != verification_pending
):
return False
+ previous_identity = source_identity(current)
current.verification_pending = None
+ retired_identity = (previous_identity, source_identity(current))
return True
try:
async with self._mutation_lock:
self.store.mutate(clear_current)
+ if retired_identity is not None:
+ previous_identity, current_identity = retired_identity
+ self.recovery.verification_retired(source_id, previous_identity, current_identity)
+ if self._source_attempt_identities.get(source_id) == previous_identity:
+ self._source_attempt_identities[source_id] = current_identity
except ValueError as exc:
if "recovery warnings" not in str(exc):
raise
@@ -6421,6 +6546,27 @@ class ModelHubService:
) -> HandleSettlement:
"""Settle every consumed hub handle before its terminal facts are exposed."""
+ try:
+ return await self._settle_handle_outcome(
+ resolved, outcome, termination_origin=termination_origin, record_attempt=record_attempt,
+ )
+ finally:
+ if resolved is not None:
+ self.recovery.release(resolved.source_id, resolved.settlement_generation)
+
+ async def _settle_handle_outcome(
+ self,
+ resolved: ResolvedInvocation | None,
+ outcome: RawCallOutcome | None,
+ *,
+ termination_origin: HandleTerminationOrigin,
+ record_attempt: Callable[[RawCallOutcome, ResolutionDecision], None],
+ ) -> HandleSettlement:
+ if resolved is not None and resolved.handle is not None:
+ await self._observe_handle_recovery(
+ resolved.source_id, resolved.settlement_generation, resolved.handle,
+ backend=resolved.backend, model_id=resolved.requested_model_id,
+ )
if termination_origin == "downstream_cancel" and outcome is None:
return HandleSettlement(
outcome=outcome,
@@ -6443,6 +6589,11 @@ class ModelHubService:
)
record_attempt(outcome, decision)
if decision.action == "return":
+ if self._verified_recovery_outcome(outcome):
+ await self._record_recovery_success(
+ resolved.source_id, resolved.settlement_generation,
+ backend=resolved.backend, model_id=resolved.requested_model_id,
+ )
await self._verify_successful_source(
resolved.source_id, resolved.credential_ref, resolved.verification_pending, outcome,
)
@@ -6475,6 +6626,7 @@ class ModelHubService:
backend=resolved.backend,
model_id=resolved.requested_model_id,
settlement_generation=resolved.settlement_generation,
+ outcome=outcome,
)
return HandleSettlement(
outcome=outcome,
@@ -6533,6 +6685,7 @@ class ModelHubService:
now=self.now(),
unavailable_source_ids=self._unavailable_native_sources(config, backend),
supply_channel=supply_channel,
+ live_recovery=self.recovery_annotations(config),
)
if resolution.channel == "direct":
raise ModelHubError("mapping_target_unavailable", status=409)
@@ -6550,11 +6703,14 @@ class ModelHubService:
excluded_source_ids: set[str],
supply_channel: Literal["hub"] | None = None,
exact_retry: bool = False,
- on_admitted: Callable[[], None] | None = None,
+ on_admitted: Callable[[int], None] | None = None,
+ recovery_request: RecoveryRequest | None = None,
) -> InvokeHandle:
while True:
await self._mutation_lock.acquire()
held = True
+ generation: int | None = None
+ admission_confirmed = False
def release_exclusion() -> None:
nonlocal held
@@ -6563,9 +6719,11 @@ class ModelHubService:
self._mutation_lock.release()
def admitted() -> None:
+ nonlocal admission_confirmed
+ admission_confirmed = True
release_exclusion()
if on_admitted is not None:
- on_admitted()
+ on_admitted(generation)
try:
config = self.store.load()
@@ -6588,12 +6746,25 @@ class ModelHubService:
):
raise _InvocationPlanChanged
if self._engine_synced:
+ if recovery_request is not None and recovery_request.expired:
+ raise _RecoveryWindowClosed
+ generation = self._reserve_settlement_generation(source.id)
+ if not self.recovery.claim(source, generation):
+ raise _InvocationPlanChanged
# Lock order matches config sync. The adapter hands exclusion
# back only after owning the transport that sync must drain.
return await self._engine_call(self.adapter.invoke(
source.id, model_id, request, stream, backend, on_admitted=admitted,
))
+ except BaseException:
+ self.recovery.release(source.id, generation)
+ raise
finally:
+ if not admission_confirmed:
+ # A completed local engine failure may return without a
+ # transport or on_admitted. It owns neither an attempt nor
+ # a half-open slot after this call, even on normal return.
+ self.recovery.release(source.id, generation)
release_exclusion()
# A canceled configuration transaction left reconciliation pending.
# Use its existing owner outside the lock, then revalidate the plan.
@@ -6611,9 +6782,17 @@ class ModelHubService:
excluded_source_ids: set[str],
supply_channel: Literal["hub"] | None = None,
exact_retry: bool = False,
- on_admitted: Callable[[], None] | None = None,
+ on_admitted: Callable[[int], None] | None = None,
+ recovery_request: RecoveryRequest | None = None,
) -> tuple[InvokeHandle, Optional[RawCallOutcome], asyncio.CancelledError | None]:
acquired_handle: InvokeHandle | None = None
+ generation: int | None = None
+
+ def admitted(value: int) -> None:
+ nonlocal generation
+ generation = value
+ if on_admitted is not None:
+ on_admitted(value)
async def meter_observed(
observed: ProtocolSSEState | None,
@@ -6648,7 +6827,8 @@ class ModelHubService:
excluded_source_ids=excluded_source_ids,
supply_channel=supply_channel,
exact_retry=exact_retry,
- on_admitted=on_admitted,
+ on_admitted=admitted,
+ recovery_request=recovery_request,
)
except InvokeCancelledError as cancelled:
await meter_observed(cancelled.observed, None)
@@ -6675,6 +6855,7 @@ class ModelHubService:
try:
handle, outcome = await await_owned_task(attempt_task)
except BaseException:
+ self.recovery.release(source.id, generation)
raise caught
if cancelled is not None and handle.stream is not None:
@@ -6689,6 +6870,7 @@ class ModelHubService:
try:
outcome = await await_owned_task(cleanup_task)
except BaseException:
+ self.recovery.release(source.id, generation)
raise cancelled
return handle, outcome, cancelled
@@ -6729,7 +6911,13 @@ class ModelHubService:
backend=backend,
model_id=requested_model_id,
settlement_generation=settlement_generation,
+ outcome=outcome,
+ )
+ elif decision.action == "return" and self._verified_recovery_outcome(outcome):
+ await self._record_recovery_success(
+ source.id, settlement_generation, backend=backend, model_id=requested_model_id,
)
+ self.recovery.release(source.id, settlement_generation)
async def _meter_call(
self,
@@ -6839,6 +7027,7 @@ class ModelHubService:
stream: bool = False,
supply_channel: Literal["hub"] | None = None,
attempt_observer: Optional[AttemptObserver] = None,
+ recovery_request: RecoveryRequest | None = None,
) -> ResolvedInvocation:
if backend not in {"claude", "codex", "opencode"}:
raise ModelHubError("mapping_target_unavailable")
@@ -6856,13 +7045,6 @@ class ModelHubService:
resolution = self._invocation_resolution(
config, cast(BackendName, backend), model_id, supply_channel,
)
- if resolution.recoverable_source_ids:
- await self._recover_resolution_sources(resolution)
- async with self._mutation_lock:
- config = self.store.load()
- resolution = self._invocation_resolution(
- config, cast(BackendName, backend), model_id, supply_channel,
- )
event_agent = cast(EventAgent, backend)
candidate_hops = list(resolution.candidate_hops)
if not candidate_hops:
@@ -6880,7 +7062,9 @@ class ModelHubService:
if facts is None:
raise AssertionError("no-candidate outcome must carry supply facts")
raise ModelHubError(
- "mapping_target_unavailable",
+ RECOVERY_EXHAUSTED_CODE
+ if recovery_request is not None and recovery_request.expired
+ else "mapping_target_unavailable",
status=409,
supply_state=facts.supply_state,
blockers=exact_hop_blockers(projection_resolution),
@@ -6889,9 +7073,14 @@ class ModelHubService:
failed_source: Optional[ModelHubSourceConfig] = None
failed_reason: Optional[EventReason] = None
+ window_closed = False
+ non_retryable_failure = False
globally_blocked_source_ids: set[str] = set()
while True:
async with self._mutation_lock:
+ if recovery_request is not None and recovery_request.expired:
+ window_closed = not non_retryable_failure
+ break
config = self.store.load()
resolution = self._invocation_resolution(
config, cast(BackendName, backend), model_id, supply_channel,
@@ -6929,9 +7118,13 @@ class ModelHubService:
engine_prepared = True
settlement_generation = None
- def admitted() -> None:
+ def admitted(generation: int) -> None:
nonlocal settlement_generation
- settlement_generation = self._reserve_settlement_generation(source.id)
+ settlement_generation = generation
+ if recovery_request is not None:
+ recovery_request.attempt_count += 1
+ recovery_request.source_id = source.id
+ recovery_request.publish("attempting")
if attempt_observer is not None:
attempt_observer(
source.id,
@@ -6955,11 +7148,16 @@ class ModelHubService:
excluded_source_ids=globally_blocked_source_ids,
supply_channel=supply_channel,
on_admitted=admitted,
+ recovery_request=recovery_request,
)
except _InvocationPlanChanged:
continue
+ except _RecoveryWindowClosed:
+ window_closed = not non_retryable_failure
+ break
if outcome is None:
if cancelled is not None:
+ self.recovery.release(source.id, settlement_generation)
raise cancelled
self._emit_switch(
agent=event_agent,
@@ -6994,6 +7192,7 @@ class ModelHubService:
)
raise cancelled
if decision.action == "refresh":
+ self.recovery.release(source.id, settlement_generation)
# The engine refreshes its credential internally; L2 retries the
# exact same source once and never falls through on a second 401.
try:
@@ -7008,13 +7207,18 @@ class ModelHubService:
supply_channel=supply_channel,
exact_retry=True,
on_admitted=admitted,
+ recovery_request=recovery_request,
)
except _InvocationPlanChanged:
if attempt_observer is not None:
attempt_observer(source.id, target_model, "hub", False, outcome, decision, (), ())
continue
+ except _RecoveryWindowClosed:
+ window_closed = not non_retryable_failure
+ break
if outcome is None:
if cancelled is not None:
+ self.recovery.release(source.id, settlement_generation)
raise cancelled
self._emit_switch(
agent=event_agent,
@@ -7060,9 +7264,14 @@ class ModelHubService:
(),
)
if decision.action == "return":
+ if self._verified_recovery_outcome(outcome):
+ await self._record_recovery_success(
+ source.id, settlement_generation, backend=cast(BackendName, backend), model_id=model_id,
+ )
await self._verify_successful_source(
source.id, source.credential_ref, verification_pending, outcome,
)
+ self.recovery.release(source.id, settlement_generation)
self._emit_switch(
agent=event_agent,
model_id=model_id,
@@ -7083,6 +7292,7 @@ class ModelHubService:
verification_pending=verification_pending,
)
if decision.action == "surface":
+ self.recovery.release(source.id, settlement_generation)
source_transition_persisted: bool | None = None
if outcome.stream_started and decision.reason is not None:
source_transition_persisted = False
@@ -7092,6 +7302,7 @@ class ModelHubService:
backend=cast(BackendName, backend),
model_id=model_id,
settlement_generation=settlement_generation,
+ outcome=outcome,
)
raise ModelHubError(
decision.error_code or outcome.error_code or "engine_down",
@@ -7111,12 +7322,17 @@ class ModelHubService:
),
)
if decision.action == "fallback":
+ non_retryable_failure |= decision.reason not in RETRY_DELAYS
+ if recovery_request is not None and decision.reason in RETRY_DELAYS:
+ recovery_request.start()
+ recovery_request.reason = decision.reason
event_reason, _persisted = await self._settle_fallback_source(
source,
decision,
backend=cast(BackendName, backend),
model_id=model_id,
settlement_generation=settlement_generation,
+ outcome=outcome,
)
globally_blocked_source_ids.add(source.id)
failed_source = source
@@ -7139,13 +7355,94 @@ class ModelHubService:
if final_facts is None:
raise AssertionError("exhausted outcome must carry supply facts")
raise ModelHubError(
- "mapping_target_unavailable",
+ RECOVERY_EXHAUSTED_CODE if window_closed else "mapping_target_unavailable",
status=503,
supply_state=final_facts.supply_state,
blockers=exact_hop_blockers(final_resolution),
turn_outcome=turn_outcome,
)
+ async def _observe_handle_recovery(
+ self, source_id: str, generation: int | None, handle: InvokeHandle,
+ *, backend: BackendName, model_id: str,
+ ) -> None:
+ observed = handle.observed
+ proved = observed is not None and observed.model_output_started
+ if proved:
+ await self._record_recovery_success(
+ source_id, generation, backend=backend, model_id=model_id,
+ )
+
+ async def resolve_with_recovery(
+ self, *, recovery_observer: Callable[[dict | None], None] | None = None, **kwargs,
+ ) -> ResolvedInvocation:
+ """One pending model request, including all fallback passes and waits."""
+
+ if self.recovery.window_seconds == 0:
+ # Explicit opt-out for single-walk probes and characterization tests.
+ return await self.resolve(**kwargs)
+ pending = RecoveryRequest(self.recovery, observer=recovery_observer)
+ try:
+ while True:
+ try:
+ return await self.resolve(**kwargs, recovery_request=pending)
+ except ModelHubError as exc:
+ projection = exc.turn_outcome
+ if (
+ projection is None or projection.outcome not in {"no_candidate", "exhausted"}
+ or projection.supply_facts is None
+ or projection.supply_facts.supply_state != "waiting"
+ ):
+ raise
+ pending.start()
+ if pending.expired:
+ raise self._recovery_exhausted(exc) from exc
+ config = self.store.load()
+ resolution = self._invocation_resolution(
+ config, kwargs["backend"], kwargs["model_id"], kwargs.get("supply_channel"),
+ )
+ if resolution.candidate_hops:
+ continue
+ if resolution.supply_status != "waiting":
+ raise
+ annotations = self.recovery.annotations(config)
+ waits = [
+ (
+ max(0.0, (parse_model_hub_timestamp(hop.retry_at) - self.now()).total_seconds()),
+ hop,
+ )
+ for hop in resolution.inspected_hops
+ if hop.temporary_blocker and hop.recovery != "in_flight" and hop.retry_at
+ ]
+ selected = min(waits, key=lambda item: item[0]) if waits else None
+ delay = selected[0] if selected is not None else pending.remaining
+ if delay >= pending.remaining:
+ # Another admitted owner can still wake this request early.
+ if not any(hop.recovery == "in_flight" for hop in resolution.inspected_hops):
+ raise self._recovery_exhausted(exc) from exc
+ delay = pending.remaining
+ hop = selected[1] if selected is not None else resolution.inspected_hops[0]
+ pending.source_id = hop.source_id
+ annotation = annotations.get(hop.source_id)
+ pending.reason = annotation.reason if annotation else pending.reason
+ changed = self.recovery.changed
+ pending.publish("waiting", next_eligible_at=hop.retry_at)
+ await self.recovery.wait(delay, changed)
+ if pending.expired:
+ # Preserve this request's admission-expiry cause even
+ # if an owner succeeded or configuration changed on wake.
+ raise self._recovery_exhausted(exc) from exc
+ finally:
+ pending.clear()
+
+ @staticmethod
+ def _recovery_exhausted(error: ModelHubError) -> ModelHubError:
+ return ModelHubError(
+ RECOVERY_EXHAUSTED_CODE, status=error.status,
+ supply_state=error.supply_state, blockers=error.blockers,
+ turn_outcome=error.turn_outcome,
+ )
+
def create_default_service(
*,
diff --git a/core/handlers/model_hub/stream_wire.py b/core/handlers/model_hub/stream_wire.py
index d803b89fe..b25e25774 100644
--- a/core/handlers/model_hub/stream_wire.py
+++ b/core/handlers/model_hub/stream_wire.py
@@ -435,6 +435,7 @@ class ProtocolObservation:
usage: ProtocolUsageReport | None = None
error_type_candidates: tuple[str, ...] = ()
error_code_candidates: tuple[str, ...] = ()
+ recovery_verified: bool = False
@dataclass(frozen=True)
@@ -483,6 +484,10 @@ class ProtocolStreamTaxonomy:
terminal_event_name: str | None
render_terminal_event: Callable[[str, str, int], dict[str, object]]
usage: ProtocolUsageTaxonomy
+ # Positive buffered recovery evidence, not a response-admission schema.
+ # Project only these finite selectors and the result container's shape.
+ buffered_success_selectors: tuple[tuple[JSONPath, tuple[str, ...]], ...] = ()
+ buffered_output_array_path: JSONPath | None = None
PROTOCOL_STREAM_TAXONOMY: Final[Mapping[str, ProtocolStreamTaxonomy]] = {
@@ -521,6 +526,8 @@ PROTOCOL_STREAM_TAXONOMY: Final[Mapping[str, ProtocolStreamTaxonomy]] = {
success_literal=None,
sequence_number_path=None,
buffered_error_envelope_paths=(("error",),),
+ buffered_success_selectors=((("type",), ("message",)),),
+ buffered_output_array_path=("content",),
terminal_event_name="error",
render_terminal_event=_anthropic_terminal_event,
usage=ProtocolUsageTaxonomy(
@@ -627,6 +634,11 @@ PROTOCOL_STREAM_TAXONOMY: Final[Mapping[str, ProtocolStreamTaxonomy]] = {
success_literal=None,
sequence_number_path=("sequence_number",),
buffered_error_envelope_paths=(("error",),),
+ buffered_success_selectors=(
+ (("object",), ("response",)),
+ (("status",), ("completed", "incomplete")),
+ ),
+ buffered_output_array_path=("output",),
terminal_event_name="error",
render_terminal_event=_responses_terminal_event,
usage=ProtocolUsageTaxonomy(
@@ -695,6 +707,8 @@ PROTOCOL_STREAM_TAXONOMY: Final[Mapping[str, ProtocolStreamTaxonomy]] = {
success_literal=(None, b"[DONE]"),
sequence_number_path=None,
buffered_error_envelope_paths=(("error",),),
+ buffered_success_selectors=((("object",), ("chat.completion",)),),
+ buffered_output_array_path=("choices",),
terminal_event_name=None,
render_terminal_event=_chat_terminal_event,
usage=ProtocolUsageTaxonomy(
@@ -838,6 +852,9 @@ def _usage_from_scalar_paths(
def _protocol_projection_paths(protocol: str) -> frozenset[JSONPath]:
taxonomy = PROTOCOL_STREAM_TAXONOMY[protocol]
paths: set[JSONPath] = {()}
+ paths.update(path for path, _values in taxonomy.buffered_success_selectors)
+ if taxonomy.buffered_output_array_path is not None:
+ paths.add(taxonomy.buffered_output_array_path)
for envelope in (*taxonomy.terminal_envelopes, *taxonomy.model_output_envelopes):
paths.add(envelope.selector_path)
if isinstance(envelope, ProtocolTerminalEnvelope):
@@ -859,6 +876,20 @@ def _protocol_projection_paths(protocol: str) -> frozenset[JSONPath]:
return frozenset(paths)
+def _buffered_recovery_verified(
+ taxonomy: ProtocolStreamTaxonomy,
+ scalars: Mapping[JSONPath, object],
+ arrays: AbstractSet[JSONPath],
+) -> bool:
+ """Recognize a completed native result without validating every body field."""
+
+ return (
+ bool(taxonomy.buffered_success_selectors)
+ and taxonomy.buffered_output_array_path in arrays
+ and all(scalars.get(path) in values for path, values in taxonomy.buffered_success_selectors)
+ )
+
+
def _protocol_error_paths(taxonomy: ProtocolStreamTaxonomy) -> frozenset[JSONPath]:
return frozenset(
(
@@ -927,7 +958,7 @@ class ProtocolFactProjector:
) -> ProtocolObservation:
literal = None if self._literal_too_long else bytes(self._literal)
if streamed and self.taxonomy.success_literal == (event_name, literal):
- return ProtocolObservation(outcome="served")
+ return ProtocolObservation(outcome="served", recovery_verified=True)
if not self._parser.finish() or () not in self._maps:
return ProtocolObservation(outcome="served" if not streamed else None)
@@ -956,6 +987,10 @@ class ProtocolFactProjector:
usage=usage,
error_type_candidates=(type_candidates if matched_paths else ()),
error_code_candidates=(code_candidates if matched_paths else ()),
+ recovery_verified=(
+ not matched_paths
+ and _buffered_recovery_verified(self.taxonomy, self._scalars, self._arrays)
+ ),
)
for envelope in self.taxonomy.terminal_envelopes:
@@ -1002,11 +1037,13 @@ class ProtocolFactProjector:
if envelope.terminal_outcome == "failed_terminal"
else ()
),
+ recovery_verified=model_output_started or envelope.terminal_outcome == "served",
)
return ProtocolObservation(
model_output_started=model_output_started,
sequence_number=sequence_number,
usage=usage,
+ recovery_verified=model_output_started,
)
def _visit(
@@ -1158,7 +1195,7 @@ def observe_protocol_response(
if data is None:
return ProtocolObservation()
if streamed and taxonomy.success_literal == (event_name, data):
- return ProtocolObservation(outcome="served")
+ return ProtocolObservation(outcome="served", recovery_verified=True)
payload = _try_parse_payload(data)
if not isinstance(payload, dict):
if not streamed:
@@ -1166,6 +1203,15 @@ def observe_protocol_response(
return ProtocolObservation()
usage = extract_protocol_usage(protocol, payload)
+ # This legacy byte helper retains its forwarding policy, but its recovery
+ # evidence shares the incremental reader's handling of malformed JSON.
+ projector = ProtocolFactProjector(protocol)
+ projector.feed(data)
+ recovery_verified = projector.finish(
+ streamed=streamed,
+ event_name=event_name,
+ previous_sequence_number=previous_sequence_number,
+ ).recovery_verified
if not streamed:
if any(
@@ -1179,7 +1225,11 @@ def observe_protocol_response(
error_envelope_paths=taxonomy.buffered_error_envelope_paths,
usage=usage,
)
- return ProtocolObservation(outcome="served", usage=usage)
+ return ProtocolObservation(
+ outcome="served",
+ usage=usage,
+ recovery_verified=recovery_verified,
+ )
sequence_number: int | None = None
if taxonomy.sequence_number_path is not None:
@@ -1216,11 +1266,13 @@ def observe_protocol_response(
),
sequence_number=sequence_number,
usage=usage,
+ recovery_verified=recovery_verified,
)
return ProtocolObservation(
model_output_started=model_output_started,
sequence_number=sequence_number,
usage=usage,
+ recovery_verified=recovery_verified,
)
@@ -1326,6 +1378,7 @@ class ProtocolSSEState:
usage=self.usage,
error_type_candidates=self.error_type_candidates,
error_code_candidates=self.error_code_candidates,
+ recovery_verified=self.reached_model,
)
return None
diff --git a/core/handlers/model_hub/turn_gateway.py b/core/handlers/model_hub/turn_gateway.py
index 1e7b99271..e654c15be 100644
--- a/core/handlers/model_hub/turn_gateway.py
+++ b/core/handlers/model_hub/turn_gateway.py
@@ -42,6 +42,7 @@ from .provenance import (
render_turn_outcome_copy,
)
from .request import ModelHubRequest
+from .retry import RECOVERY_EXHAUSTED_CODE, RECOVERY_EXHAUSTED_MESSAGE
from .resolver import parse_model_hub_timestamp
from .stream_wire import (
ProtocolSSEState,
@@ -847,33 +848,19 @@ class ModelHubTurnGateway:
translation = translate_opencode_tool_names(payload)
payload = translation.request
execution.response_tool_aliases = translation.response_aliases
- for retry in range(2):
- try:
- resolved = await self.service.resolve(
- backend=backend,
- model_id=resolution_model,
- request=ModelHubRequest(
- payload,
- protocol=_REQUEST_PROTOCOLS[endpoint],
- headers=protocol_headers,
- ),
- stream=stream,
- supply_channel="hub",
- attempt_observer=observe_attempt,
- )
- break
- except ModelHubError as exc:
- delay = self._cooldown_retry_delay(exc.turn_outcome)
- if (
- retry
- or delay is None
- or exc.turn_outcome is None
- or exc.turn_outcome.outcome != "no_candidate"
- ):
- raise
- # Native callers may ignore Retry-After. With no runnable
- # hop, wait for one known recovery before admission.
- await asyncio.sleep(delay)
+ resolved = await self.service.resolve_with_recovery(
+ backend=backend,
+ model_id=resolution_model,
+ request=ModelHubRequest(
+ payload,
+ protocol=_REQUEST_PROTOCOLS[endpoint],
+ headers=protocol_headers,
+ ),
+ stream=stream,
+ supply_channel="hub",
+ attempt_observer=observe_attempt,
+ recovery_observer=terminalizer.update_recovery,
+ )
except ModelHubError as exc:
turn_outcome = exc.turn_outcome
if turn_outcome is None and exc.code == "engine_down":
@@ -882,6 +869,22 @@ class ModelHubTurnGateway:
terminalizer.engine_down()
elif turn_outcome is not None and turn_outcome.outcome == "no_candidate" and exc.supply_state is not None:
terminalizer.mark_no_candidate(exc.supply_state, exc.blockers)
+ if exc.code == RECOVERY_EXHAUSTED_CODE:
+ self._commit_and_render_turn_outcome(execution, terminalizer, turn_outcome)
+ # Native compatibility is keyed by the caller backend. Keep its
+ # transport result separate from actual upstream provenance.
+ return web.json_response(
+ {
+ "type": "error",
+ "error": {
+ "type": RECOVERY_EXHAUSTED_CODE,
+ "code": RECOVERY_EXHAUSTED_CODE,
+ "message": RECOVERY_EXHAUSTED_MESSAGE,
+ },
+ },
+ status=400 if backend == "codex" else 424,
+ headers={"Cache-Control": "no-store", "X-Content-Type-Options": "nosniff"},
+ )
return self._terminal_error_response(
execution,
terminalizer,
@@ -894,6 +897,12 @@ class ModelHubTurnGateway:
if resolved.handle is not None and resolved.handle.stream is not None:
execution.handle = resolved.handle
resources.push_async_callback(resolved.handle.close_stream)
+ # The execution boundary must own cleanup before an awaited health
+ # write: cancellation here still settles and meters this handle.
+ await self.service._observe_handle_recovery(
+ resolved.source_id, resolved.settlement_generation, resolved.handle,
+ backend=resolved.backend, model_id=resolved.requested_model_id,
+ )
return await self._resolved_response(
request,
resolved,
diff --git a/core/message_dispatcher.py b/core/message_dispatcher.py
index b692a628a..306e3cc3a 100644
--- a/core/message_dispatcher.py
+++ b/core/message_dispatcher.py
@@ -937,12 +937,19 @@ class ConsolidatedMessageDispatcher:
if done:
footer = self._status_footer_text(context, elapsed_s=elapsed_s, done=True, reason=reason)
else:
+ from core.model_hub_progress import recovery_status_text
+
+ recovery_label = recovery_status_text(
+ self.controller,
+ (context.platform_specific or {}).get("turn_token"),
+ str(getattr(getattr(self.controller, "config", None), "language", "en") or "en"),
+ )
last = self._status_last_activity_at.get(consolidated_key, started)
tick = self._status_render_tick.get(consolidated_key, 0)
self._status_render_tick[consolidated_key] = tick + 1
hourglass = "⏳" if tick % 2 == 0 else "⌛"
backend_dead = self._backend_dead(context)
- body = self._decorate_body_with_action_time(
+ body = recovery_label or self._decorate_body_with_action_time(
context, body, now - last, backend_dead=backend_dead
)
footer = self._status_footer_text(
diff --git a/core/model_hub_progress.py b/core/model_hub_progress.py
new file mode 100644
index 000000000..289ddc607
--- /dev/null
+++ b/core/model_hub_progress.py
@@ -0,0 +1,75 @@
+"""Read-only presentation of live Model Hub recovery; never owns a retry."""
+
+from __future__ import annotations
+
+import math
+from datetime import datetime, timezone
+from typing import Any
+
+from core.handlers.model_hub.resolver import parse_model_hub_timestamp
+from vibe.i18n import t
+
+
+def recovery_snapshot(controller: Any, turn_id: str | None) -> list[dict]:
+ if not isinstance(turn_id, str) or not turn_id.strip():
+ return []
+ gateway = getattr(controller, "model_hub_turn_gateway", None)
+ read = getattr(getattr(gateway, "correlation", None), "recovery_snapshot", None)
+ if not callable(read):
+ return []
+ return read(turn_id.strip())
+
+
+def publish_recovery_changed(controller: Any, turn_id: str) -> None:
+ """Invalidate the existing Session read projection, without chat delivery."""
+
+ from core.inbox_events import bus
+
+ manager = getattr(controller, "session_turns", None)
+ for session_id, entry in tuple(getattr(manager, "in_flight", {}).items()):
+ if entry.task.done():
+ continue
+ payload = getattr(entry.context, "platform_specific", None) or {}
+ if payload.get("turn_token") == turn_id:
+ bus.publish(
+ "session.activity",
+ {"session_id": session_id, "event": "model_recovery"},
+ )
+
+
+def recovery_status_text(
+ controller: Any,
+ turn_id: str | None,
+ language: str,
+ *,
+ now: datetime | None = None,
+) -> str | None:
+ """Debounce a status label for an existing IM bubble; emit no messages."""
+
+ observed_at = now or datetime.now(timezone.utc)
+ for snapshot in recovery_snapshot(controller, turn_id):
+ started_at = snapshot.get("started_at")
+ if not isinstance(started_at, str):
+ continue
+ try:
+ age = (observed_at - parse_model_hub_timestamp(started_at)).total_seconds()
+ except (ValueError, OverflowError):
+ continue
+ if age < 5:
+ continue
+ phase = snapshot.get("phase")
+ if phase == "attempting":
+ return t("modelHub.recovery.attempting", language)
+ if phase != "waiting":
+ continue
+ retry_at = snapshot.get("next_eligible_at")
+ if isinstance(retry_at, str):
+ try:
+ remaining = (parse_model_hub_timestamp(retry_at) - observed_at).total_seconds()
+ except (ValueError, OverflowError):
+ remaining = 0
+ if remaining > 0:
+ return t("modelHub.recovery.waiting", language, seconds=math.ceil(remaining))
+ # A read-side clock can establish eligibility, never actual recovery.
+ return t("modelHub.recovery.pending", language)
+ return None
diff --git a/core/session_turns.py b/core/session_turns.py
index 50d915ad6..8b1210541 100644
--- a/core/session_turns.py
+++ b/core/session_turns.py
@@ -7672,11 +7672,13 @@ class SessionTurnManager:
entry = self.in_flight.get(session_id)
active = entry is not None and not entry.task.done()
native_turn_started = False
+ active_turn_id: str | None = None
backend = ""
backend_alive: Optional[bool] = None
owner: dict[str, Any] | None = None
if active and entry is not None:
payload = getattr(entry.context, "platform_specific", None) or {}
+ active_turn_id = str(payload.get("turn_token") or "").strip() or None
target = payload.get("agent_session_target")
if isinstance(target, dict):
backend = str(target.get("agent_backend") or "").strip()
@@ -7718,6 +7720,7 @@ class SessionTurnManager:
durable_turn = delivery_store.active_turn(conn, session_id)
if durable_turn is not None:
active = True
+ active_turn_id = str(durable_turn["id"])
native_turn_started = durable_turn["state"] == "active"
backend = str(durable_turn.get("backend") or "").strip()
initial = delivery_store.delivery_for_turn(
@@ -7825,6 +7828,12 @@ class SessionTurnManager:
result["backend"] = backend
if owner is not None:
result["owner"] = owner
+ if active and active_turn_id:
+ from core.model_hub_progress import recovery_snapshot
+
+ recovery = recovery_snapshot(self.controller, active_turn_id)
+ if recovery:
+ result["model_recovery"] = recovery
return result
async def release_for_service_shutdown(self) -> int:
diff --git a/docs/plans/model-hub-contracts/adapter-interface.py b/docs/plans/model-hub-contracts/adapter-interface.py
index 960a32666..28a16009e 100644
--- a/docs/plans/model-hub-contracts/adapter-interface.py
+++ b/docs/plans/model-hub-contracts/adapter-interface.py
@@ -12,6 +12,7 @@ from __future__ import annotations
import asyncio
from dataclasses import dataclass
+from datetime import datetime
from enum import Enum
from typing import Any, AsyncIterator, Callable, Final, Literal, Mapping, Protocol, Sequence
@@ -124,6 +125,14 @@ class RawCallOutcome:
# would otherwise take its token report with it, and a vendor that reported
# tokens billed for them whether or not the call ended well.
usage: ProtocolUsageReport | None = None
+ # Only failed HTTP responses supply retry advice: ASCII, at most 128
+ # characters, unparsed. L2 owns validation and the resulting retry policy.
+ retry_after: str | None = None
+ # Aware UTC receipt time of those headers, before reading the error body.
+ response_received_at: datetime | None = None
+ # Recognized model output or protocol success, independently of permissive
+ # forwarding. An unrecognized HTTP 200 alone cannot prove Source recovery.
+ recovery_verified: bool = False
class ObservationOutcome(str, Enum):
diff --git a/docs/plans/model-hub-contracts/agent-chain.schema.json b/docs/plans/model-hub-contracts/agent-chain.schema.json
index a08403144..cb5542736 100644
--- a/docs/plans/model-hub-contracts/agent-chain.schema.json
+++ b/docs/plans/model-hub-contracts/agent-chain.schema.json
@@ -71,6 +71,10 @@
"description": "Source-global health mapped to this hop, plus live backoff. backoff is an in-memory connection throttle before the first user-visible model-output byte and is never persisted on Source or configuration. It is projected only over an otherwise healthy hop whose exact Source/model pair is eligible and not explicitly retired; API-key inventory absence is not a blocker; subscriptions retain existing model membership admission; every durable Source/capability blocker suppresses it."
},
"runnable": { "type": "boolean" },
+ "recovery": {
+ "enum": ["eligible", "in_flight"],
+ "description": "Optional live recovery annotation. Eligibility expiry is not verified recovery; in_flight means another real request owns half-open admission. Neither value is persisted in Source state."
+ },
"reason": {
"anyOf": [
{ "const": "native_cli_unavailable" },
@@ -96,6 +100,24 @@
}
},
"allOf": [
+ {
+ "if": { "required": ["recovery"] },
+ "then": {
+ "properties": {
+ "health": { "const": "healthy" },
+ "reason": { "type": "null" },
+ "retry_at": { "type": "null" }
+ }
+ }
+ },
+ {
+ "if": { "properties": { "recovery": { "const": "eligible" } }, "required": ["recovery"] },
+ "then": { "properties": { "runnable": { "const": true } } }
+ },
+ {
+ "if": { "properties": { "recovery": { "const": "in_flight" } }, "required": ["recovery"] },
+ "then": { "properties": { "runnable": { "const": false } } }
+ },
{
"if": { "properties": { "health": { "enum": ["needs_action", "error"] } }, "required": ["health"] },
"then": { "properties": { "runnable": { "const": false }, "retry_at": { "type": "null" } } }
@@ -135,7 +157,7 @@
},
"supply_state": {
"enum": ["ok", "waiting", "interrupted"],
- "description": "ok has a runnable hop; waiting is a non-empty chain whose only blockers are Source cooldown or live connection backoff and which has no process blocker; interrupted covers an empty chain or an action/process blocker. A durable blocker cannot be hidden by a simultaneous live backoff."
+ "description": "ok has a runnable hop; waiting is a non-empty chain whose only blockers are Source cooldown, live connection backoff or a half-open owner and which has no process blocker; interrupted covers an empty chain or an action/process blocker. A durable blocker cannot be hidden by simultaneous live recovery."
}
},
"allOf": [
@@ -169,10 +191,13 @@
"minItems": 1,
"items": {
"properties": {
- "health": { "enum": ["cooldown", "backoff"] },
"runnable": { "const": false },
"reason": { "enum": [null, "models.source.backoff.connection_failed"] }
},
+ "anyOf": [
+ { "properties": { "health": { "enum": ["cooldown", "backoff"] } }, "required": ["health"] },
+ { "properties": { "health": { "const": "healthy" }, "recovery": { "const": "in_flight" } }, "required": ["health", "recovery"] }
+ ],
"required": ["health", "runnable", "reason"]
}
}
@@ -186,10 +211,13 @@
"minItems": 1,
"items": {
"properties": {
- "health": { "enum": ["cooldown", "backoff"] },
"runnable": { "const": false },
"reason": { "enum": [null, "models.source.backoff.connection_failed"] }
},
+ "anyOf": [
+ { "properties": { "health": { "enum": ["cooldown", "backoff"] } }, "required": ["health"] },
+ { "properties": { "health": { "const": "healthy" }, "recovery": { "const": "in_flight" } }, "required": ["health", "recovery"] }
+ ],
"required": ["health", "runnable", "reason"]
}
}
diff --git a/docs/plans/model-hub-contracts/api.md b/docs/plans/model-hub-contracts/api.md
index 9d91f3074..b481b72c0 100644
--- a/docs/plans/model-hub-contracts/api.md
+++ b/docs/plans/model-hub-contracts/api.md
@@ -889,7 +889,17 @@ turn without changing the effective plan or origin.
Each item carries `channel`, Source-global health or the distinct live `backoff`
overlay, process-aware `runnable`, and nullable `reason`. The complete axiom is:
-`runnable = source-health-permits AND no-live-backoff AND process-available`.
+`runnable = source-health-permits AND no-live-backoff AND no-half-open-owner AND process-available`.
+
+An affected hop optionally includes `recovery: "eligible" | "in_flight"`.
+Expiry only makes it eligible; it does not write Source state or emit a
+recovery event. `in_flight` means a single real request owns half-open
+admission and forces `runnable: false`. With no stronger blocker its read
+health is `healthy`, `reason` and `retry_at` are null, and it rolls up as
+temporary `waiting`. `needs_action`, `error`, Source/model absence and native
+process unavailability retain precedence and cannot become temporary through
+this annotation. The field is absent outside eligible/in-flight recovery,
+including future cooldown and network deadlines.
Process availability is definitionally true for `channel: "hub"` in v2; there is no
configuration knob for it. For `native_cli`, `reason: "native_cli_unavailable"` is an
@@ -1035,10 +1045,15 @@ The backoff deadline expiring makes the hop runnable without persistence. Before
chain, AgentSupply, or probe response is validated and serialized, the read assembler
captures one assembly time and normalizes every expired live overlay to that Source's
underlying non-backoff health and runnability; it never emits a stale `backoff` or expired
-live `retry_at`. The first later user-visible model-output byte produced by that same
-affected Source, Source endpoint/credential replacement, or process reconstruction
+live `retry_at`. The first later recognized model output or `SUCCESS` with
+`recovery_verified: true` produced by that same affected Source, Source
+endpoint/credential replacement, or process reconstruction
clears its deadline and consecutive-failure streak. Output from a different fallback
-Source does not clear the affected Source's streak. The delay is capped at 30 seconds.
+Source does not clear the affected Source's streak. Unknown buffered HTTP 200
+admission, including its compatibility `stream_started: true`, is not recovery
+evidence. Actual streaming output is the separate handle observer fact;
+recognized empty protocol completion is valid success. The local network
+delay is capped at 30 seconds; valid longer upstream retry advice is retained.
For a native hop whose process is simultaneously unavailable, the live deadline remains
visible but `native_cli_unavailable` takes the single reason slot and the chain remains
`interrupted`; restoring the process reveals any still-live connection backoff.
@@ -1058,6 +1073,120 @@ closed live key and validates only as `channel: hub`, `reachable: false`, and
`latency_ms: null`; native probes and Hub probes with measured latency cannot carry it.
It never reports the removed persistent network/timeout cooldown keys.
+The independently selected saved-Source model test (`probe_source`) is not an
+Agent probe or recovery request: it calls once without changing Source health
+or scheduling retry. Its legacy `models.source.cooldown.network` and
+`models.source.cooldown.timeout` error identities remain display-only results,
+not persisted cooldowns. It must not use the live backoff copy that promises
+a forthcoming retry. This distinction is explicit at the shared probe-result
+formatter's call boundary; it does not introduce a second policy.
+
+## Live request recovery and terminal projection
+
+The service is the only recovery-policy owner. `resolve_with_recovery` is the
+gateway entry point; `resolve` remains a single walk for manual probes and
+explicit characterization tests. §4.3 of `model-hub.md` owns the delay families,
+jitter, Retry-After validation, identity fencing and the shared 120-second
+admission window. `service._invoke_admitted` reserves the existing attempt
+generation while holding mutation exclusion, after exact-route revalidation
+and before half-open claim/transport admission. Its admission callback forwards
+that generation; settlement never mints one. A completed pre-admission local
+failure releases its provisional claim without invoking that callback or
+counting an attempt. `native_cli` is outside this HTTP coordinator and retains
+its fixed native cooldown and launch-eligibility semantics.
+
+`SourceRecoveryAnnotation.retired_cooldown` is private, optional evidence naming
+the exact old `(retry_at, detail_key)` observation retired by verified success
+for the live Source identity. It is not a public field or a permanent healthy
+override. The policy drops it when a fresh authority read observes different
+cooldown data, retirement on disk, removal or identity replacement. Draft reads
+cannot consume it for a different cooldown or mutate the live authority.
+`ExactHopInspection.cooldown` and `cooldown_reason` carry effective transient
+facts from that same canonical inspector when persistence fails. Chain health
+and waiting eligibility use the effective cooldown; `exact_hop_blockers`
+retains its existing `cooldown` identity, while `turn_supply_facts` uses only a
+known effective EventReason. Neither invents an upstream status or bypasses
+stronger blockers. No new wire field, enum, classifier or retry owner is added.
+
+Write failure during temporary cooldown or verified-recovery bookkeeping cannot
+replace inference results. An already expired admission remains explicitly
+`model_hub_recovery_exhausted` across owner success and config/engine-preparation
+races; elapsed time cannot relabel an actual permanent, request, engine or
+post-output terminal result.
+
+`TurnCorrelationRegistry.recovery_snapshot(turn_id: str) -> list[dict]` is a
+read-only, in-memory hook for existing `turn_state` consumers. Each item is one
+pending model request, not one Session or Source:
+
+| Field | Type | Meaning |
+| --- | --- | --- |
+| `request_id` | nonempty string | existing gateway request correlation identity |
+| `phase` | `"waiting" \| "attempting"` | delayed admission or an admitted recovery attempt |
+| `attempt_count` | integer, at least 0 | admitted attempts in this request, including the first attempt |
+| `started_at` | aware UTC ISO-8601 string | beginning of the request's automatic recovery window |
+| `window_end` | aware UTC ISO-8601 string | fixed window end; never reset at fallback |
+| `source_id` | string or null | current affected/attempted Source |
+| `reason` | string or null | structured recovery family, never raw upstream text |
+| `next_eligible_at` | aware UTC ISO-8601 string or null | scheduled admission; null while owned/in flight or unscheduled |
+
+No snapshot is published before recovery is needed. `started_at` remains
+stable across waits and reloads, so a consumer can debounce its existing
+working indicator for five seconds without deriving policy constants.
+`GatewayTurnTerminalizer.update_recovery(snapshot)` binds service observations
+to the existing request id. It clears on completion/cancellation and rejects
+late updates after request finalization. There are no persisted snapshots,
+new endpoints, event families, notices, outbox entries or per-second updates.
+
+The registry accepts optional `on_recovery_changed: Callable[[str], None]`.
+It calls this with the Turn id after releasing its lock, only for material
+snapshot changes or clears. The orchestrator binds it to the existing
+SessionTurnManager live owner and `session.activity`/`turn_state` flow.
+Snapshot reads return defensive copies and exclude ambiguous, poisoned,
+untracked, closed, frozen, missing or mismatched-owner traces. Shared OpenCode
+processes without exact Turn correlation therefore expose no snapshot;
+this hook does not infer identity from prompts or native thread headers.
+
+`terminal_projection(turn_id: str, *, backend: str) ->
+TurnOutcomeProjectionInput | None` is the corresponding authoritative terminal
+read. Under the same lock it requires exact backend/scope ownership,
+unambiguous attribution, no pending attempts, and an existing non-served,
+non-canceled terminal outcome. Stop/frozen and poisoned/untracked traces
+return null. Normally closed but still live exact terminal traces remain
+readable until settlement; historical store records never override unrelated
+native errors. A backend callback reporting an already committed Hub failure
+preserves it: `record_turn_outcome -> fail_hub_attempt -> terminal_projection`
+does not freeze it as rejection of served content. Real served-to-protocol
+rejection and Stop retain their original guards.
+
+### Recovery-exhausted native wire compatibility
+
+Only exhaustion of automatic temporary recovery uses this exact terminal body:
+
+```json
+{
+ "type": "error",
+ "error": {
+ "type": "model_hub_recovery_exhausted",
+ "code": "model_hub_recovery_exhausted",
+ "message": "Automatic recovery has ended. Try again or choose another model."
+ }
+}
+```
+
+The HTTP status is 400 for backend `codex`, 424 for backend `claude` or
+`opencode`, independently of protocol. No Retry-After, raw upstream text or
+upstream error detail is sent on this branch. Real provenance remains unchanged;
+the wire status is not reclassified as a Source/request failure. In particular,
+Codex's internal InvalidRequest interpretation is an intentional transport
+compatibility measure, not public `invalid_request_error`.
+
+Auth, invalid-request, engine and post-output failures are not remapped.
+Hub launch injection disables only Codex provider request retries and
+unbounded connection retries, plus Claude request retries in environment and
+settings. Stream retries, watchdog and ordinary continuation are unchanged.
+The binary audit and lost-terminal transport residual are recorded in
+`model-hub-inference-deadlines.md` and `model-hub-retry-experience.md`.
+
## Latest recorded turn
The Recorded Error Detail Closure in `model-hub-routing-modes.md` (`0de3d2f47`,
@@ -1083,6 +1212,12 @@ Unknown upstream strings, raw bodies, messages, headers and credentials are not
retained. These optional observations do not change classification, fallback or Source
health. Historical records may omit both fields and keep their existing read behavior.
+`failed_attempts[].http_status` is independently optional: a strict integer
+100 through 599 when observed, omitted when absent (never boolean or null).
+It records the original upstream status, for example 503 even when native
+compatibility uses 400. Older retained records without it remain readable.
+This additive diagnostic changes neither the failure reason nor classifier.
+
The dialog independently reads this projection on demand and labels it "Latest recorded
turn" / "最近已记录回合". Its error panel and details action use the same structured record,
recorded timestamp and exact historical Source/model identifiers. Deleted Sources keep
diff --git a/docs/plans/model-hub-contracts/turn-provenance.schema.json b/docs/plans/model-hub-contracts/turn-provenance.schema.json
index b2af1db3e..48c325279 100644
--- a/docs/plans/model-hub-contracts/turn-provenance.schema.json
+++ b/docs/plans/model-hub-contracts/turn-provenance.schema.json
@@ -88,6 +88,12 @@
"channel": { "enum": ["native_cli", "hub"] },
"stripped_reasoning_efforts": { "$ref": "#/definitions/strippedReasoningEfforts" },
"declared_reasoning_efforts": { "$ref": "#/definitions/declaredReasoningEfforts" },
+ "http_status": {
+ "type": "integer",
+ "minimum": 100,
+ "maximum": 599,
+ "description": "Optional actual upstream HTTP status, independent of the native-compatible recovery-exhausted gateway response. Absent for unobserved/native transport failures and historical records."
+ },
"reason": {
"enum": [
"quota_exhausted",
diff --git a/docs/plans/model-hub-e2e-test-plan.md b/docs/plans/model-hub-e2e-test-plan.md
index d9a2765ee..b5bb82d5e 100644
--- a/docs/plans/model-hub-e2e-test-plan.md
+++ b/docs/plans/model-hub-e2e-test-plan.md
@@ -68,13 +68,13 @@ Out of scope (documented, not dropped silently):
|---|---|---|---|
| D1 | add api_key source first, subscription-class source second | **current**: appended at tail (key burns first). Baseline assert-current; expected value flips with decision D-1 | baseline |
| D2 | turn with hop0 healthy | served; provenance records hop; usage metered | assert |
-| D3 | hop0 → 429 before first output | same-turn failover to hop1; `switch` event; hop0 cooldown 60s; chat copy unchanged (silent) | assert |
+| D3 | hop0 → 429 before first output | same-turn failover to hop1; `switch` event; hop0 initial local cooldown 60–72s; chat copy unchanged (silent) | assert |
| D4 | hop0 → quota (`insufficient_quota`) | fallback, cooldown 300s | assert |
| D5 | hop0 → 401 (api_key, non-refreshable) | `credential_revoked` needs_action; UI shows repair action; no mid-turn refresh attempt | assert |
-| D6 | hop0 → 5xx | fallback 30s cooldown; retry-after header **ignored** (document; decide with D-4 whether to keep) | baseline |
+| D6 | hop0 → 5xx | source backoff with positive bounded jitter; later valid Retry-After survives the managed engine and takes precedence | assert |
| D7 | stream interrupt after first output | terminal frame injected per protocol; **no replay/failover**; source still settles; next turn resolves hop1 (takeover pill visible) | assert |
-| D8 | cooldown `retry_at` elapses | next resolve recovers source, chain returns to hop0, `recover` event | assert |
-| D9 | all hops failing | 503 `mapping_target_unavailable` + waiting copy with retry_at | assert |
+| D8 | cooldown `retry_at` elapses | hop0 becomes eligible; only recognized output or valid empty completion clears its state and emits `recover` | unit/loopback covered; local Incus outstanding |
+| D9 | all temporary hops keep failing | one closed recovery-exhausted result (Codex 400, Claude/OpenCode 424); localized final copy and original upstream provenance | gateway/scenario covered; local Incus outstanding |
| D10 | chat selects effort `high`; fallback hop declares tiers `[]` | each exact adapter attempt retains the requested effort; turn provenance identifies the serving hop without inventing stripping. This fixture covers the Avibe gateway/adapter boundary, not the proxy engine translator | assert |
| D11 | `POST /{backend}/v1/messages/count_tokens` | currently 404 `not_found_error` (D-2 decision: fix or document impact on Claude Code auto-compact) | baseline |
| D12 | env/catalog injection for claude/codex/opencode | `ANTHROPIC_BASE_URL/TOKEN` only for claude; codex catalog neutralized 4 keys; opencode overlay model projection shape | assert |
@@ -161,7 +161,10 @@ Unit/contract suites already cover: config validation, guard structure, classifi
- **D-1 subscription-first placement.** Restore "subscription before API key" at placement time (surgical change in `_apply_source_placement`, reuse dead `recommended_source_order`), incl. one-time re-sort of existing configs? Current shipped contract says append-at-tail. Affects D1.
- **D-2 `count_tokens` 404.** Serve it (proxy to upstream tokenizers or estimate) or document Claude Code context-estimation degradation in hub mode? Affects D11.
- **D-3 missing `modelHub.*` browser i18n keys + absent Python-bundle codes.** Fix now (makes B11 pass) or baseline as expected-fail for this round?
-- **D-4 Retry-After honoring.** Keep flat cooldowns (simple, predictable) or honor upstream reset headers? Affects D6.
+- **D-4 Retry-After honoring (approved 2026-09-09).** Use the later of
+ reason-specific backoff and valid upstream advice measured at header receipt.
+ D6 asserts the real managed-engine forwarding boundary. See
+ [bounded quiet recovery](model-hub-retry-experience.md).
- **D-5 tier provenance ladder (approved 2026-09-03).** Apply the first available
`upstream > catalog > user > null` rung; managed tiers are read-only, a refresh
override is observable, and silent stripping is attributed to the exact attempt.
diff --git a/docs/plans/model-hub-implementation.md b/docs/plans/model-hub-implementation.md
index 97027c491..e73ef957d 100644
--- a/docs/plans/model-hub-implementation.md
+++ b/docs/plans/model-hub-implementation.md
@@ -229,6 +229,7 @@ that reconciliation may change content but is not expected to change file owners
| **I4 Sources / Gateway UI** *(tentative; one lane, two increments)* | codex | `ui/src/components/settings/models/**`; `ui/src/i18n/*.json`; `vibe/i18n/*.json`; UI tests. Both i18n trees are I4-exclusive. The current increment consumes the already-specified install/unsupported-host surfaces for AC-38/AC-43; each model row may open frame 02, render the drawing, and consume the chain read projection, but no chain-write caller is wired and Save remains disabled. Exact-chain editing and its UI tests belong to the second increment only after K5's third-round G-32 specification merges. That second increment sends visible noninterrupting hop removal as one ordinary PUT and echoes both refusal-plan arrays unchanged only when protected supply would be interrupted, after I7 freezes both fixtures; the current increment continues to consume master's pre-I7 refusal shape and must not add inactive confirmation fields. It consumes AC-36/37/39–42/44–50 only after K5 round 2 changes their G rows from gaps to contract references and registers the live-backoff presentation; AC-51's read-before-retry client rule activates in the second increment after I7 freezes the deletion/recreate fixture. AC-53's materialization-error `interrupted_pairs` consumer activates only in that second increment after K5 round 2 registers E6 and I7 freezes positive/negative API payload fixtures. AC-54's `accept_unavailable_inventory` producer also belongs to the second increment after K5 round 2 names state ⑤ and I7 freezes both server-result cells; clean creation and pull-origin state ⑤′ never emit it. This is an ordering split inside I4, not a new lane identity. **Revised through the 2026-08-12 04:40 PM ruling.** | current increment: I1 envelopes + K4 contracts; second increment: K5 rounds 1–3 merged + frozen I7/I2/I3 fixtures |
| **I7 contract-completion implementation (backend)** | codex | After K6 merges, exclusive ownership transfers from I1 for `config/v2_config.py`; `core/handlers/model_hub/{service,resolver,classification,rpc,request,errors,events}.py`; `modules/agents/model_hub.py`; `vibe/{ui_server,model_hub_client}.py`; `tests/test_model_hub_config.py`; `tests/test_model_hub_api.py`; and `tests/test_model_hub_resolution.py`. For the Model Hub failure-callback seam only, I7 also exclusively owns `modules/agents/base.py`, `modules/agents/claude_agent.py`, `modules/agents/codex/{agent,event_handler}.py`, `modules/agents/opencode/{agent,poll_loop}.py`, exact cross-backend phase fixtures in `tests/{test_claude_agent_sessions,test_codex_agent,test_opencode_server}.py`, and mechanical signature consumers in `tests/{test_claude_cli_path,test_multi_platform_runtime}.py`. I7 owns AC-35–AC-47 and AC-50–AC-54's backend producers/tests plus AC-48's API/service half. `service.py` is the sole shared guard-plan owner and implements every guard-totality row and the error-to-required-array relation; it also enforces AC-52's both-channel re-auth acknowledgement before any OAuth adapter call and AC-53's materialization-error response condition, and AC-54's unavailable-inventory consent gate against the repeated server observation. `modules/agents/model_hub.py` owns the native classification/state decision: backend callbacks supply raw failure shape and phase only, and no native path writes a persistent network cooldown. The transferred resolver/classifier/event boundary owns `retired: false` filtering and all four network-failure cells; its post-output executor/provenance fixture consumes K6's G-34 row without inventing an outcome id, copy key, or payload shape. I7 delegates OAuth tuple claim/coalescing and terminal-flow expiry to I3's registry; its service/API half branches explicit cancellation by nonce presence, retaining the I3 flow for nonce-bearing reconciliation and forgetting only no-nonce flows. Its first AC-50 mechanical action is the sole owner-authorized I2-file exception: delete the strict xfail marker from `tests/test_model_hub_l3.py::test_probe_transport_failures_await_ac50_backoff_contract` in the same commit that makes its forward contract assertions XPASS; it otherwise consumes I2 managed-Gateway facts and I3 OAuth fixtures without editing their files. It never edits contracts, UI/i18n, scenario trees, or any other I2–I5 file. **Revised through the 2026-08-12 04:40 PM ruling.** | K4 + #1312 + K6 merged; every listed transfer and native callback seam activates only after all three merge |
| **I5 scenario validation** *(tentative)* | codex | `tests/scenarios/model_hub/**` except I3's native-import scenario; `tests/scenario_harness/**` | I1–I4 + I7 |
+| **I8 bounded recovery presentation integration** *(orchestrator-approved 2026-09-09)* | codex | `core/model_hub_progress.py` consumes canonical resolver timestamps and the read-only request recovery snapshot for the existing live working/activity surface. It does not classify, retry, write Source health, or own another recovery timer. Controller, native-terminal and UI integration remain orchestrator-owned; this registration gives the new resolver consumer one explicit owner without transferring core policy ownership. | Approved recovery contracts and the core policy/provenance hooks |
| **I6 default-on availability** *(owner-approved 2026-09-06)* | codex | Replace the shared capability resolver's absent-variable default with enabled; retain its explicit deployment/emergency opt-out. Verify controller aggregate construction, public API/capability projection and internal RPC with no environment opt-in. Saved backend modes, routing, model selection and engine-install consent remain unchanged. This supersedes the historical default-off and full gate-deletion directions below; it is not a constant-true shell. | Owner requested a PR to make Model Hub available by default. Full merged LOCAL live/visual acceptance remains outstanding and is not waived or claimed by this release-default change. |
**Executor ruling (owner, 2026-08-09; I7/K5/K6 registry completed 2026-08-12).** Every new K5, K6, and I1–I7 lane uses
diff --git a/docs/plans/model-hub-inference-deadlines.md b/docs/plans/model-hub-inference-deadlines.md
index 1c0c81abd..2f0178070 100644
--- a/docs/plans/model-hub-inference-deadlines.md
+++ b/docs/plans/model-hub-inference-deadlines.md
@@ -24,10 +24,15 @@ not block unrelated requests.
response has arrived, so model execution cannot consume it.
- Audit backend idle reclamation, gateway cancellation, and native caller
timeouts before closing the change. Record evidence and scope decisions here.
-- Treat a known source cooldown as delayed admission, with one cancellable wait
- and a fresh resolution at its existing recovery time. Only retry a request
- that had no runnable candidate; never replay visible output or restart a
- failed attempt inside the same gateway request.
+- Treat temporary Source failures and cooldowns as delayed admission under one
+ service-owned 120-second recovery window per pending model request. Re-read
+ configuration after cancellable waits and share the window across fallback
+ passes. Retry pre-output failures only; never replay model output.
+- Treat cooldown expiry as eligibility, not success. One pending real request
+ owns half-open admission; only recognized output or verified protocol success
+ records recovery. Window expiry never cancels admitted inference.
+- Pass all-temporary Hub supply through native preflight immediately, so native
+ acceptance and the first model request do not own independent wait windows.
- Recognize Responses reasoning-text and compatible Chat reasoning-content
deltas as model output using the existing protocol taxonomy.
- Keep production configuration and the running service unchanged during
@@ -49,7 +54,7 @@ classification, usage, response replay, source isolation, and shutdown tests.
| Gateway teardown and cancellation drain | Runs after completion/cancellation | Retain cleanup bounds |
| Codex and Claude active-session reclamation | Age backstop overrides durable active ownership | Require the existing ownership veto at both reclamation checks; retain repair of orphaned local flags |
| Codex SSE idle | Native default is 300,000 ms, after HTTP headers | Retain the native policy; zero expires immediately, not disabled. No arbitrary huge override and no early gateway headers |
-| Source cooldown | A waiting route returns 409; Codex does not honor HTTP Retry-After | Delay no-candidate admission until the known retry time once; re-resolve current configuration; use 503 and Retry-After for remaining time-recoverable errors |
+| Source cooldown | A waiting route returns 409; Codex does not honor HTTP Retry-After | One shared recovery window covers pre-output retries and delayed admission; terminal compatibility maps only recovery exhaustion to Codex 400 or Claude/OpenCode 424, with no Retry-After |
| Prelude output classification | Responses reasoning text and Chat reasoning content were treated as metadata | Extend the existing taxonomy and acceptance fixtures |
| Prelude storage | Memory spills at 256 KiB; replay has deliberately no total byte ceiling | Preserve lossless replay and cancellation cleanup, not a new response-size rejection policy |
| Codex local control RPC | 120-second acknowledgement deadline; turns run via notifications | Retain, not an inference deadline; resetting initialization on one slow RPC is a separate transport issue |
@@ -106,6 +111,35 @@ Verified against the installed Codex 0.153.2 and its exact official source tag:
The native SSE cap and a provider's own limits remain outside the removed
Avibe inference deadlines; this PR does not claim end-to-end infinite patience.
+### Bounded recovery compatibility amendment — September 9, 2026
+
+The approved binary audit in `model-hub-retry-experience.md` supersedes the
+earlier once-only gateway cooldown wait. Codex 0.153.2 issued six requests for
+424/422 despite provider retries being zero and unbounded connection retries
+being disabled; 400 produced one. Claude 2.1.263 and OpenCode 1.18.18 produced
+one for 424. All three accepted headers delayed by 121 seconds. These are
+orchestrator-owned binary audit results, not a claim that core unit tests ran
+the native binaries.
+
+The single exhausted domain type/code is `model_hub_recovery_exhausted` with
+neutral wire text, mapped by backend (not upstream protocol). Codex's internal
+400/InvalidRequest transport interpretation is intentional compatibility, not
+the public `invalid_request_error` classification. Original HTTP status and
+Source failure reasons remain in provenance. Only exhausted automatic recovery
+uses this branch; auth, request incompatibility, engine errors and post-output
+failure retain their established behavior.
+
+Hub-only launch injection sets Codex provider `request_max_retries=0` and
+`features.unbounded_connection_retries=false`, plus Claude environment and
+settings `CLAUDE_CODE_MAX_RETRIES=0`. Codex `stream_max_retries`, Claude watchdog,
+and normal continuation remain unchanged. The OpenCode native poll-loop
+consumer and all native terminal/rendering integration are orchestrator-owned.
+
+The received-terminal audit does not prove exactly-once behavior if the
+terminal response is lost. Existing request identity does not identify every
+native invocation across arrivals. No prompt-based dedupe, Turn blackout or
+Source blackout is introduced to hide that transport residual.
+
## Validation evidence
- Real loopback HTTP tests cover all three protocols, five delayed inference
diff --git a/docs/plans/model-hub-retry-experience.md b/docs/plans/model-hub-retry-experience.md
new file mode 100644
index 000000000..2f65c79a0
--- /dev/null
+++ b/docs/plans/model-hub-retry-experience.md
@@ -0,0 +1,510 @@
+# Model Hub retry, recovery, and failure experience
+
+Status: Approved for implementation on September 9, 2026. Runtime changes are
+not deployed by approval. The defaults below are the implementation target,
+not claims about deployed behavior.
+
+## Start with the reported case
+
+`gpt-6-astra` had one configured supplier, `relay-gpt`, and that relay was
+unreachable during the incident. Avibe eventually reported a Codex-wrapped
+local 503 and promised automatic recovery at an absolute UTC timestamp.
+The promise described a cooldown expiring, not a successful connection and
+not a guarantee that the failed conversation would resume.
+
+The intended experience is:
+
+1. Try another runnable supplier immediately, if the configured chain has one.
+2. If the only options are temporarily unavailable, keep the pending request
+ alive within a bounded recovery window. Show “waiting to retry,” not a
+ terminal failure or a promise that the provider will recover.
+3. When a supplier becomes eligible, let one waiting real request test it.
+ Other requests may use backups or wait; they do not all hit the relay.
+4. If that request produces valid model output, let work continue. If it
+ fails again, increase the delay. Timer expiry alone proves nothing.
+5. If automatic recovery ends, say that this request has ended and offer
+ the existing safe Retry action where supported. Later supplier recovery
+ must not silently revive a terminal Turn.
+
+The incident's observed `server_error` classification does not itself prove
+a socket failure: a relay can return 503 because its own upstream is down.
+User-facing specificity must follow the recorded wire evidence.
+
+## Incident evidence and baseline contracts
+
+The incident was investigated on Avibe `3.0.15rc7`. The initial design was checked
+against source baseline `49813f73471720e2e00d38fdbff1d82e8f4d58a7`.
+An in-flight engine-intent change is not assumed to be deployed or required.
+
+| Boundary | Pre-change baseline / recorded incident evidence |
+| --- | --- |
+| Source selection | A fallback-class failure excludes that Source for the current resolution and tries the next eligible configured hop. No candidate yields `no_candidate`; a failed walk yields `exhausted`. |
+| Cooldown | Server/overload failures use 30 seconds, rate limits 60 seconds, and recoverable quota failures 300 seconds. Delays are fixed, without jitter or consecutive-failure growth. Actionable balance/auth failures retain their separate rules. |
+| Unclassified connection failure | Classified for fallback, without a durable Source health write. The complete live network-backoff behavior described in the contract is not implemented in these inspected paths. |
+| Cooldown expiry | The Source can return to active/standby and emit `recover` without an inference succeeding. |
+| Gateway admission | Waits once for a known future deadline only after `no_candidate`, then resolves again. It does not wait-and-retry `exhausted` inside the same gateway request. Remaining temporary unavailability returns 503 with `Retry-After`. |
+| New Turn admission | Runtime routing can reject unavailable supply before reaching the gateway's waiting path. |
+| Engine and native caller | CPA request retry is disabled and Hub owns cooling. In the inspected Codex version, native retries still apply and do not honor `Retry-After`; retrying notifications can be hidden until the final failure. |
+| Failure recovery button | A separate, user-initiated Delivery action exists. It is not permission for automatic Turn replay. |
+
+The incident's 29 failed attempts were an accumulated result, not one setting
+named “29 retries.” Diagnosis should expose counts by layer and request.
+
+Existing authorities to reuse:
+
+- [Model Hub](model-hub.md) and its [API contract](model-hub-contracts/api.md):
+ classification, routing, blocker precedence, live connection backoff,
+ settlement freshness, event semantics, and the closed terminal-copy matrix.
+- [Inference deadlines](model-hub-inference-deadlines.md): connected inference
+ ends on completion, concrete failure, or owner cancellation, not elapsed time.
+- [Failed-Turn Retry](backend-failure-retry.md): accepted versus `not_written`
+ retry, durable ownership, and stale/duplicate-action guards.
+- [Delivery foundation](session-delivery-foundation.md): retained input,
+ admission, and native acceptance ownership.
+
+The approved implementation revises three baseline contracts:
+
+- Permit bounded retries of retryable, pre-output failures within one gateway
+ request; the baseline inference-deadlines contract permitted only delayed
+ admission after `no_candidate`.
+- Replace timer-based recovery claims with eligibility plus observed recovery;
+ add single-request recovery admission and increasing shaped-error cooldowns.
+- Add jitter and valid empty-completion recovery evidence to the existing
+ network-backoff design. Its documented 30-second maximum remains intact.
+
+Canonical contracts, production interfaces, localization keys, and mirror
+fixtures must change together in the implementation PR. Approval alone does
+not change a running installation.
+
+## Ownership and safety boundaries
+
+| Owner | Responsibility |
+| --- | --- |
+| Existing Turn / Delivery manager | Own input, native acceptance, queue order, cancellation, final settlement, and explicit user Retry. A waiting subphase is not a new Turn or Delivery. |
+| Existing Model Hub service and resolver | Own classification, configured fallback, per-Source eligibility, backoff, and a shared in-memory recovery admission coordinator. |
+| Gateway and runtime router | Gateway consumes the service policy. Router launches an all-temporary Hub chain without pre-native waiting; only the arriving gateway request owns a recovery window. Neither owns a second retry counter. |
+| CPA | Execute the selected attempt. Keep inference request retries disabled; preserve the existing bounded credential-refresh exception. |
+| Native backend | Execute the agent and any supported continuation. For `channel=hub`, redundant whole-HTTP-request retry must be reconciled with Hub ownership. For `native_cli`, Hub cannot control unobserved provider requests. |
+| Web / IM | Render controller-owned progress and terminal facts. Never retry model requests from a view timer. |
+
+No new persistent task state machine, retry table, standalone probe worker,
+or notification scheduler is needed. The coordinator belongs to the existing
+Model Hub service lifecycle; existing controller dispatch/IPC carries progress.
+
+The retry unit is one pending model inference request, not an Agent Turn.
+Retries stop once the first user-visible model-output byte is committed,
+including recognized reasoning and tool-call output. Headers, SSE comments,
+and protocol metadata are not output. Preserve lossless prelude handling.
+
+Pre-output inference retry does not guarantee exactly-once upstream billing:
+a dropped connection may hide completed inference. Keep every observed usage
+report exactly once locally. Do not extend this policy to replaying upstream
+side-effecting operations without a verified idempotency contract.
+
+Successful fallback uses only the current configured chain and its existing
+mapping semantics. It does not rewrite route order, select an unrelated
+model, restart a backend, or change credentials implicitly.
+
+## Recovery policy
+
+### One bounded automatic recovery window
+
+Approved default: **120 seconds per pending model request**, starting
+at its first retryable failure or first blocked admission. The same window
+covers fallback passes, cooldown waits, and recovery admission; it never resets
+because another Source was tried. For an all-temporary Hub chain, preflight
+constructs the gateway launch immediately without waiting, starting a recovery
+clock, or reserving a half-open slot. The first gateway request then owns its
+window. Dispatch acknowledgement and native acceptance retain their existing
+owners and deadlines.
+
+This is an admission limit, not an inference timeout:
+
+- Start another attempt only while the window remains open. An attempt already
+ connected may finish after it expires, however slowly the model thinks.
+- Do not cancel a live attempt on the window timer. If it subsequently fails
+ and the window has expired, do not admit another automatic attempt.
+- A terminal classification ends recovery immediately. A runnable backup can
+ be used immediately; there is no mandatory sleep between different Sources.
+- When nothing is runnable, wait interruptibly for eligibility, relevant
+ configuration change, cancellation, or window expiry. Re-resolve current
+ configuration on wake; never cache a candidate list across the wait.
+- If the next permitted attempt is beyond the remaining window, end automatic
+ recovery now and show the next eligible time as advisory. Do not spend two
+ minutes waiting when the recorded quota reset is five minutes away.
+- Only an all-temporary blocked chain is an automatic waiting case. Preserve
+ `interrupted` and its remedies if any blocker needs user action; do not hide
+ it behind a cooling Source.
+
+Two minutes allows two recovery opportunities after typical 30-second and
+60-second transient delays while bounding unattended outage waits. It is a
+starting policy, not a measured optimum. Do not add a user-facing
+timeout control in the first release.
+
+Native transport compatibility is a release gate, not an assumption. Validate
+the full window through each supported backend before enabling this loop.
+Retain legitimate native stream continuation and native idle policies; do not
+send fake output/early success headers, set enormous timeouts, or disable all
+stream retries to keep a waiting connection alive. If a backend cannot support
+the contract, resolve that ownership boundary before rollout and document the
+limitation; do not claim uniform automatic recovery.
+
+For Hub-managed traffic, disable independently controllable HTTP retries and
+use the exact-version-tested closed terminal described below. The selected
+native callers terminate after receiving it; OpenCode's Avibe-side automatic
+`continue` also recognizes it. Existing request/Turn correlation does not provide
+cross-retry invocation identity. If the terminal response is lost, a subsequent
+HTTP request cannot reliably be distinguished from a legitimate later model
+call; no exactly-once delivery guarantee is claimed. Do not compensate with
+prompt deduplication or a whole-Turn blacklist.
+Direct `native_cli` traffic retains native retry ownership; only its observable
+preflight availability and terminal rendering participate in this proposal.
+
+### Backoff and cooldown
+
+Count consecutive eligible failed attempts for the same Source identity.
+Do not count waiters, native error notifications, elapsed timers, or failures
+whose settlement generation is stale.
+
+| Classified failure | Approved local delay, before upstream advice | Storage / action |
+| --- | --- | --- |
+| Unclassified pre-output connection failure | 1, 2, 4, 8, 16, 30 seconds; cap 30 | Reuse contracted live Source-scoped backoff; no Source/config write. |
+| Retryable server/overload | 30, 60, 120 seconds; cap 120 | Keep existing shaped-error Source cooldown; streak is live state. |
+| Recoverable rate limit | 60, 120, 240, 300 seconds; cap 300 | Keep existing cooldown classification. |
+| Recoverable quota exhaustion | 300 seconds | Keep existing cooldown; normally beyond the automatic window. |
+| Refreshable credential rejection | Existing same-Source retry once | No independent retry loop; a second rejection follows the existing terminal rule. |
+| Bad key, revoked access, balance requiring action, request/schema/context/model errors, local engine failure | No new automatic retry | Keep existing classification/remedy; `engine_down` never blames or cools an upstream Source. |
+
+Apply bounded positive jitter to the local base: a sampled delay in
+`[base, min(cap, base * 1.2)]`. This preserves minimum delays and hard caps;
+at the cap, source-scoped admission still prevents a waiting-client stampede.
+Do not sample a separate delay per waiter. Shared state owns one deadline.
+
+Carry upstream retry advice through the existing `RawCallOutcome` boundary.
+The adapter preserves a bounded, allowlisted `Retry-After` value; the Hub
+parses nonnegative delay-seconds or a valid HTTP date against one captured
+response time. The effective deadline is the later of local backoff and valid
+upstream advice. A Source deadline must not move earlier due to another failure.
+Invalid, overflowing, or past values cannot cause a tight loop; fall back to
+the local policy and record only a sanitized diagnostic.
+
+Do not clamp a valid long upstream delay down to the local cap and retry early.
+If it is outside this request's window, terminate this request truthfully while
+keeping the advisory deadline. Compare aware timestamps using the existing
+shared parser; use monotonic elapsed time for in-process scheduling.
+
+### Recovery admission, not timer-based health
+
+Cooldown expiry means **eligible to try**, not **recovered**. Represent
+eligibility and a recovery request using the existing runtime annotation
+boundary, not new persistent Source statuses.
+
+For each affected Source, one existing pending request may hold recovery
+admission. It sends its own inference; there is no synthetic probe traffic.
+Other waiters use eligible fallback Sources or await that admission's result.
+Recheck admission under the service's existing synchronization before invoking
+the engine; never hold the configuration lock while sleeping or doing I/O.
+
+Valid recovery evidence is the same Source and identity generation producing
+recognized model output, or completing a valid successful buffered/empty
+response. A TCP connection, HTTP 200 headers, heartbeat, or elapsed timer is
+insufficient. Output proves service for this attempt, not permanent health.
+
+On that evidence, clear the affected streak/deadline, release recovery
+admission, and record a recovery observation. Ordinary traffic may then resume.
+A later shaped stream failure follows its existing terminal/health rule; it
+cannot make the already-consumed request retryable. Success on a backup never
+clears the failed primary's streak. No traffic means no recovery claim.
+
+Reuse the existing attempt-start settlement generation and Source identity
+fences; do not add a second freshness authority. At most one failed recovery
+admission advances the streak. Older concurrent outcomes may retain their
+history/usage but cannot overwrite newer Source facts.
+
+If the recovery request is canceled, release its admission after connection
+cleanup without asserting success or inventing a failure. Wake another live
+waiter when eligible. A slow connected recovery request is not killed to free
+the slot: other waiters can use backups or exhaust their own recovery windows.
+Release must check ownership so late cleanup cannot unlock a newer request.
+
+Use Source scope for connection failures and the existing generic server
+classification. Explicit request/model incompatibility must remain local to
+that request and must not poison other models. Do not infer model scope from
+a free-text 503 or add a persistent model-health table. A future transient
+model-scoped policy requires explicit classifier evidence and corresponding
+exact-hop contract changes; it is not silently bundled into this revision.
+
+## Turn lifecycle and user actions
+
+- **Before native acceptance:** retain the same original Delivery, prompt,
+ attachments, and queue ownership. Waiting does not mean “native accepted.”
+ Do not resubmit input or reopen a backend merely to check supply.
+- **After acceptance:** waiting is a progress subphase of the existing live
+ Turn, not a reversal to pre-admission state. A gateway request retry must
+ not rerun earlier tools or restart the Agent.
+- **Stop / disconnect:** propagate authoritative cancellation through waits,
+ recovery admission, the active transport, and bounded cleanup. A browser
+ disconnect alone follows the existing conversation ownership policy; a
+ canceled gateway caller cannot leave detached inference running.
+- **New input:** retain existing queue/steer/interrupt rules. A later message
+ does not silently replace the request being retried or reset its budget.
+- **Model selection:** use existing selection semantics; changing the default
+ is not consent to rewrite an accepted request. Users may Stop and submit a
+ new request with another model. No special “switch and replay” action.
+- **Source edit/removal:** wake affected waiters and re-resolve. Endpoint or
+ credential replacement invalidates live backoff and recovery admission for
+ the old identity. Preserve existing revocation/transport lifecycle behavior;
+ old results cannot mutate the new identity or claim recovery for it.
+- **Controller restart:** rebuild live throttle state through the existing
+ lifecycle and reconcile durable Turns/Deliveries normally. Do not persist
+ timers as new jobs or auto-resubmit an ambiguously accepted native start.
+- **Explicit Retry after failure:** reuse the
+ [existing action](backend-failure-retry.md). Accepted failed Turns use the
+ existing `continue` admission; proven `not_written` reuses retained input.
+ Unknown acceptance never licenses replay. Keep double-click/stale-Turn
+ guards, drafts, and attachments unchanged.
+
+## Presentation and evidence
+
+Two independent facts must be visible: **is this request still live?** and
+**when may this Source be tried again?** A terminal Turn can coexist with a
+Source that is temporarily unavailable; `supply_state=waiting` alone cannot
+justify telling that user “we are still retrying.”
+
+| Situation | Example Chinese copy | English equivalent |
+| --- | --- | --- |
+| Live wait, connection failure proved | `relay-gpt 暂时无法连接,正在等待重试。预计 30 秒后再次尝试。` | `Cannot connect to relay-gpt. Waiting to retry in about 30 seconds.` |
+| Live wait, only upstream 503 known | `relay-gpt 暂时无法处理请求,正在等待重试。` | `relay-gpt is temporarily unable to handle the request. Waiting to retry.` |
+| Deadline elapsed, recovery attempt active | `正在尝试重新连接 relay-gpt…` | `Trying relay-gpt again…` |
+| Automatic recovery ended, no usable backup | `本次请求未完成:relay-gpt 持续不可用,且没有可用备用源。自动重试已结束,你可以重试或切换模型。` | `This request could not complete: relay-gpt remains unavailable and no backup is usable. Automatic retry has ended. You can retry or choose another model.` |
+| Provider asks for a longer wait | `本次请求未完成,自动重试已结束。relay-gpt 预计 5 分钟后可再次尝试。` | `This request could not complete; automatic retry has ended. relay-gpt may be tried again in about 5 minutes.` |
+| Output interrupted | `本次输出已中断,自动重试已停止。你可以重试,让 Agent 继续处理。` | `Output was interrupted; automatic retry has stopped. You can retry to let the agent continue.` |
+| Eligibility on Models page, no recovery evidence yet | `冷却已结束,可再次尝试;尚未验证恢复。` | `Cooldown ended; eligible to retry. Recovery has not been verified.` |
+
+These are semantic examples, not a second string authority. Preserve existing
+actionable auth/configuration remedies and the no-fallback model distinctions.
+Terminal rendering must come from the canonical closed matrix, with its enum,
+key, backend projection, and fixture consumers updated together.
+
+Default copy omits the backend wrapper, local gateway URL, and raw UTC
+timestamp. Use relative time, with local absolute time and timezone in details
+when useful. A frontend countdown reaching zero changes to “retry eligible”
+or “checking status,” never “recovered” without a server fact.
+
+For short interruptions, use the existing working indicator. Reveal a waiting
+detail after five seconds of continuing recovery; this is display debounce,
+not a new retry timer. Web updates one live working/activity surface, including
+before the first assistant/tool activity row. The existing Activity model has
+no waiting row type today: extend its live status projection deliberately,
+rather than faking assistant text or recording a failed terminal notification.
+
+IM uses the current Turn's progress delivery path to edit an existing supported
+status bubble. When that feature is disabled or unavailable, keep the wait
+silent rather than adding a waiting notice.
+Do not send each backoff step or a per-second countdown as new messages.
+Terminal failure is emitted once through existing delivery. Successful
+fallback adds no standalone chat notification; normal output resumes and
+Models/Usage retain the supplier history. IM Retry buttons are not assumed
+to exist; preserve existing platform actions.
+
+### One producer of structured facts
+
+Extend the existing request correlation and provenance instead of parsing
+native exception strings in each UI. The minimum facts and their consumers are:
+
+| Fact | Producer / consumer |
+| --- | --- |
+| Exact Turn/Delivery, gateway request, and attempt identities | Existing ownership/correlation; controller progress and terminal settlement reject stale updates. No new cross-session correlation key. |
+| Live recovery phase, attempt count, next eligible time, window end | Hub coordinator; runtime router/gateway publish through existing controller dispatch/IPC. Views render a non-authoritative snapshot. |
+| Source/model, failure layer, reason, upstream status, gateway status | Existing adapter/classifier/provenance; terminal renderer and expandable diagnostics distinguish relay failure from local engine failure. |
+| Retry advice and header-receipt time | The adapter carries only bounded `Retry-After` and an aware receipt instant to the service policy. No new upstream header/body/request-ID diagnostic surface is introduced. |
+| Observed recovery versus eligibility expiry | Service settlement; Source read projection and event feed. Historical timer-based `recover` entries must not become proof of successful inference. |
+
+Wire names belong in the existing canonical interfaces during implementation;
+this table defines responsibilities, not another schema. The source resolution
+event feed remains a pull/debug record, not an outbox. Do not add recipient or
+audience fields to it.
+
+Only the controller's existing Turn owner may finalize the conversation. Clear
+obsolete no-candidate/exhausted projections when a new attempt is admitted;
+late failure or progress cannot overwrite success, Stop, or a newer Turn.
+Reload must reconcile the live snapshot with durable terminal state. Historical
+records without new facts remain readable and make no inferred retry promise.
+
+## Delivery sequence
+
+1. **Truthful state and copy.** Separate eligibility from observed recovery;
+ distinguish live waiting from terminal failure; preserve diagnostic layer
+ evidence and reuse existing failure actions. Correct current text even when
+ no automatic retry remains. Amend canonical contracts and mirrors with the
+ affected implementation. Do not advertise the new loop before it exists.
+2. **One recovery policy.** Complete contracted live network backoff, add
+ Retry-After propagation, the bounded admission window, shared recovery
+ admission, and runtime-router/gateway parity. Reconcile native retry
+ ownership and wire live Web/IM progress in the same complete behavior slice.
+3. **Validate and tune.** Run the acceptance matrix below, including native
+ callers and restart/cancellation. Use retry counts, recovery delay, terminal
+ outcomes, and Stop latency from existing observability to evaluate the
+ defaults. Add no background probing or automatic production tuning.
+
+These slices share resolver, settlement, provenance, and gateway contracts.
+Keep one owner for those changes; do not parallelize incompatible core edits.
+Rebase against merged engine/runtime work without treating an open PR as a
+runtime fact. UI implementation must reuse the design system and update the
+authoritative design when its visible structure changes.
+
+## Acceptance matrix
+
+The labels below are local to this proposal. Allocate canonical scenario IDs
+in `tests/scenarios/model_hub/catalog.yaml` during implementation.
+
+| Case | Required evidence |
+| --- | --- |
+| Sole relay fails, then succeeds | Pre-output recovery stays within one pending request, delays grow, real output resumes, and no terminal failure is emitted. |
+| Sole relay keeps failing | Admission window is not reset by fallback/native retries; no attempts start after expiry; exactly one final failure says automatic retry ended. |
+| Delay exceeds remaining window | No pointless full-window sleep or early retry; preserve the provider's later advisory time. |
+| Startup versus active Turn | Same temporary-supply policy; original startup Delivery and attachments survive; waiting never fabricates native acceptance. |
+| Concurrent waiting Turns/backends | One recovery admission per affected Source; backups proceed; waiters do not each increment the streak or allocate a separate probe. |
+| Recovery owner canceled or slow | No lock/transport leaks; canceled owner releases only its own admission; another waiter can proceed; slow connected inference is not killed. |
+| Delayed headers / first output / buffered completion | Success beyond the former transport timeout and beyond the admission window; no health penalty, retry, or duplicate usage caused by elapsed inference time. |
+| Stream boundary | Heartbeats/headers alone do not prove recovery; reasoning and tool output do. No Hub replay after output, including engine/transport failure and cancellation races. Valid empty completion also releases recovery admission. |
+| Advice and clocks | Missing/malformed/past/large Retry-After, differing UTC offsets, clock shifts, and deterministic jitter; no stale serialized deadline or busy loop. |
+| Stronger blockers and fault scope | Bad credentials, quota requiring action, local engine failure, mixed blockers, and explicit model errors retain correct remedies; waiting cannot hide them or corrupt unrelated Source/model state. |
+| Source replacement / stale settlement | Old success/failure/cleanup cannot mutate the replacement or release its recovery admission; backup success cannot clear primary failure history. |
+| Restart / unknown native acceptance | No revived terminal Turn, duplicate original prompt, or fabricated acceptance; live snapshot and durable owner reconcile through existing recovery. |
+| Retry / new input / model change | Existing double-click, stale-Turn, queued-input, `not_written`, and accepted-continue tests stay green; no draft loss or automatic tool replay. |
+| Web and IM | One live waiting presentation, no retry-message storm, readable local timing, redacted details, reload consistency, and one truthful terminal. Successful fallback remains quiet. |
+| Native retry ownership | Exact-version Codex, Claude, and OpenCode callers cannot multiply a Hub recovery episode; legitimate native continuation remains intact. Direct-native scope is explicitly distinguished. |
+
+Use clock-controlled resolver/service tests, real loopback HTTP for Responses,
+Messages, and Chat Completions (streaming and buffered), existing usage and
+provenance tests, and UI projection tests. Verify native integration and at
+least one IM delivery path in isolated developer-local Incus with a controllable
+mock upstream. Do not use real relay credentials, paid inference, the running
+workstation service, or remote tenant environments.
+
+Validation is reported by layer in the implementation PR: clock-controlled
+tests, real loopback adapter/gateway tests, fixed-version native caller audit,
+and UI tests/build are distinct evidence. Integrated controller/native/IM
+acceptance in local Incus and browser visual checks must be reported separately;
+neither unit success nor the native-only audit implies that they passed.
+Deployment and service restart require separate authorization.
+
+## Implementation ownership and boundary contract
+
+The implementation orchestrator owns integration and all contract revisions.
+The first shared contract commit establishes these additive boundaries before
+delegation:
+
+- `RawCallOutcome.retry_after: str | None = None`: the engine client supplies
+ only the bounded ASCII `Retry-After` HTTP header on failed responses (maximum
+ 128 characters). No arbitrary headers, credentials, or body excerpts are
+ added. The Model Hub retry policy consumes and validates this advice.
+ Existing constructors remain valid; the adapter-interface mirror changes
+ with the production dataclass.
+- `RawCallOutcome.response_received_at: datetime | None = None`: the client
+ captures an aware UTC instant on receipt of failed HTTP response headers.
+ The Hub resolves delay-seconds and HTTP dates against that instant, not the
+ later error-body read or settlement time. Missing evidence falls back to
+ the Hub's captured classification time.
+- `RawCallOutcome.recovery_verified: bool = False`: recovery-only evidence,
+ supplied by recognized model output or a recognized successful protocol
+ terminal/body. It does not tighten the existing permissive response-admission
+ policy. A generic HTTP 200, unrecognized JSON, or malformed body may still be
+ passed through under that policy but cannot produce a recovery claim.
+ `ProtocolObservation` carries the same additive boolean from the existing
+ protocol projector to the adapter. The adapter lane owns this evidence
+ projection and its mirror/tests; core only consumes it.
+- The core lane owns the single in-memory retry coordinator, resolver live
+ annotations, Source settlement, gateway retry loop, runtime-router admission,
+ their canonical contracts, and tests. Recovery waits are cancelable and a
+ request retry never becomes a Delivery replay.
+- The adapter lane owns the three additive outcome members, HTTP propagation,
+ positive protocol recovery-evidence projection, their mirror, and focused
+ wire tests. It does not classify, sleep, retry, or change engine retry
+ configuration.
+- Native retry settings, Turn correlation, progress, and terminal presentation
+ are integrated only after the native-boundary audit. No lane may deduplicate
+ distinct model requests using prompt text or implement a second retry owner.
+
+Live presentation consumes the core lane's public registry methods:
+
+- `terminal_projection(turn_id: str, *, backend: str)` returns an exact,
+ unambiguous, non-frozen terminal failure projection with no pending peer
+ request, or `None`. Shared failure presentation uses it instead of parsing
+ native error wrappers. History replay never queries live state.
+- `recovery_snapshot(turn_id: str) -> list[dict]` returns only live unambiguous
+ request progress: `request_id: str`, `phase: waiting | attempting`,
+ `attempt_count: int`, `started_at: str`, `window_end: str`,
+ `source_id: str | None`, `reason: str | None`,
+ `next_eligible_at: str | None`. Timestamps are aware UTC ISO strings.
+- The registry's optional `on_recovery_changed: Callable[[str], None]`
+ callback reports material changes after releasing its lock. The controller
+ maps the exact live Turn to its Session and publishes the existing
+ `session.activity` invalidation with `event=model_recovery`. The existing
+ `turn-state` response adds optional `model_recovery` with that snapshot.
+- `failed_attempts[].http_status` optionally retains the actual upstream
+ response status as an integer from 100 through 599; omit it when no response
+ status was observed. It is independent of the gateway's native-compatibility
+ 400/424 status and adds no upstream header/body exposure.
+- Web reuses the existing working/Activity label after a five-second debounce,
+ with no new notification or view-owned retry. IM concise status rendering
+ reads the same snapshot on its existing heartbeat and edits its existing
+ bubble; platforms or preferences without progress retain existing behavior
+ instead of receiving a new unsolicited waiting notice.
+
+No subtask may modify another lane's files or open/merge a PR independently.
+All work is integrated into one implementation PR and validated at its exact
+head. Any missing cross-boundary behavior is reported to the orchestrator, not
+silently omitted from the accepted contract.
+
+### Native terminal compatibility decision
+
+The [native audit record](model-hub-retry-native-compatibility.md) separates
+observed binary behavior, isolation, significant launch settings, and remaining
+acceptance layers.
+
+The domain result for an exhausted automatic recovery episode is
+`model_hub_recovery_exhausted`, in both `error.type` and `error.code`. Its wire
+message is exactly:
+
+> Automatic recovery has ended. Try again or choose another model.
+
+The JSON envelope has top-level `type: error`. It includes no upstream body,
+provider diagnostic, absolute retry time, or `Retry-After`. Provenance retains
+the actual failure and supply facts; localized user presentation reads those
+facts rather than interpreting the transport status as a user mistake.
+
+Native compatibility testing on 2026-09-09 found that a uniform 424 or 422 is
+not terminal for Codex 0.153.2: its outer loop made six requests even with
+provider HTTP retries disabled. The same closed error at 400 made one request.
+Claude 2.1.263 and OpenCode 1.18.18 made one request at 424. Therefore the
+gateway maps this domain result to HTTP 400 for Codex and HTTP 424 for Claude
+and OpenCode, by backend identity, not by the selected wire protocol. This is
+an intentional Codex transport-compatibility mapping: Codex internally calls
+400 `InvalidRequest`, but the Hub neither emits `invalid_request_error` nor
+claims the user's input is invalid. Do not remap unrelated authentication,
+request-validation, post-output, or local-engine failures to this result.
+
+Hub launches set Codex provider `request_max_retries=0` and
+`features.unbounded_connection_retries=false`; `stream_max_retries` remains
+unchanged. Claude Hub launches fix `CLAUDE_CODE_MAX_RETRIES=0` in both process
+environment and launch settings. Keep its independent normal stream
+continuation and fallback behavior; the native audit also verified that 424
+is terminal with its retry watchdog enabled. Direct-native launches and
+user-owned configuration are unchanged.
+
+OpenCode's Avibe-side automatic `continue` is bypassed only for a current
+assistant error already admitted by the existing baseline/liveness checks,
+with a bound Hub launch, `APIError`, integer `statusCode=424`, and exact
+`error.type` and `error.code` in parsed `responseBody`. Never parse the
+truncated diagnostic text or disable the global retry setting. Explicit
+Retry and ordinary native continuation keep their existing ownership.
+
+The isolated native audit also held response headers for 121 seconds before
+the selected terminal response; all three main calls remained single requests.
+This is native caller evidence, not integrated controller or Incus acceptance.
+A lost terminal response still cannot be distinguished from a later legitimate
+model call using today's routing/Turn identifiers. No exactly-once guarantee,
+prompt hash, Turn blacklist, or Source blacklist is introduced to hide that
+transport-delivery limit.
diff --git a/docs/plans/model-hub-retry-native-compatibility.md b/docs/plans/model-hub-retry-native-compatibility.md
new file mode 100644
index 000000000..55e7d02ee
--- /dev/null
+++ b/docs/plans/model-hub-retry-native-compatibility.md
@@ -0,0 +1,139 @@
+# Model Hub native terminal compatibility
+
+Verification date: September 9, 2026.
+
+This records the isolated native audit supporting the transport mapping in
+[bounded quiet recovery](model-hub-retry-experience.md#native-terminal-compatibility-decision).
+It is not evidence of an integrated controller, managed-engine, browser, or
+Incus acceptance pass.
+
+## Isolation and procedure
+
+The audit ran installed Codex 0.153.2, Claude Code 2.1.263, and OpenCode 1.18.18
+against an ephemeral Node HTTP mock bound to `127.0.0.1`. A macOS Seatbelt
+sandbox denied network access except the mock's exact port, writes outside the
+temporary audit root except `/dev/null`, reads from the real agent/Avibe homes
+and Keychains, and securityd access. An outside-root write was observed to fail
+while a scratch-root write succeeded.
+
+Each native child received a fresh environment whitelist and separate temporary
+home, Codex home, Claude config directory, XDG config/cache/data/state directories,
+and temporary directory. Authentication values were explicitly non-secret mock
+fixtures. No production credentials, paid inference, running service operations,
+or user configuration changes were involved.
+
+The mock recorded timestamps, method, path, model, streaming mode, and initial
+system input. It returned a synthetic model catalog for model discovery and the
+following JSON for inference, with `Content-Type: application/json` and
+`Cache-Control: no-store`:
+
+```json
+{
+ "type": "error",
+ "error": {
+ "type": "model_hub_recovery_exhausted",
+ "code": "model_hub_recovery_exhausted",
+ "message": "Automatic recovery has ended. Try again or choose another model."
+ }
+}
+```
+
+The final cases used neither `Retry-After` nor `x-should-retry`. Delayed cases
+withheld all headers for 121,000 ms before returning that terminal response.
+OpenCode title generation received an immediate response and was counted
+separately from the main call using the distinct title-generator system input.
+
+Only Responses and Anthropic Messages were exercised by the native audit.
+Chat Completions is covered by adapter/gateway loopback tests, not by this native
+binary evidence.
+
+## Native settings exercised
+
+Codex used `exec --json --ephemeral --skip-git-repo-check`, a synthetic local
+model catalog, and an isolated custom provider with `wire_api="responses"`,
+`requires_openai_auth=false`, `supports_websockets=false`, and its base URL and
+fixture token directed solely at the mock. The significant retry overrides
+were:
+
+```toml
+model_providers.audit.request_max_retries = 0
+features.unbounded_connection_retries = false
+```
+
+`stream_max_retries` remained unchanged.
+
+Claude used `-p --output-format stream-json --verbose`, the fixture model,
+`--setting-sources '' --no-session-persistence --tools ''`,
+`--strict-mcp-config --mcp-config '{"mcpServers":{}}'`, and a mock
+`ANTHROPIC_BASE_URL`/fixture auth token. Auto-update, telemetry, error reporting,
+and nonessential traffic were disabled. `CLAUDE_CODE_MAX_RETRIES`, API timeouts,
+and non-streaming fallback were not overridden for this audit. The main cases
+set the retry watchdog to zero; a separate 424 case enabled it. Therefore these
+results prove the terminal response itself is sufficient for that Claude
+version, independently of the implementation's additional Hub-only retry limit.
+
+OpenCode used `run --format json`, one enabled synthetic provider/model,
+`permission={"*":"deny"}`, and disabled auto-update and model fetching. Provider
+options contained only the mock URL and a non-secret token fixture. The
+Responses cases used `@ai-sdk/openai`; the Anthropic cases used
+`@ai-sdk/anthropic`. No global native retry behavior was changed.
+
+## Observed results
+
+These counts are main inference requests, not model discovery or title calls.
+
+| Native binary / protocol | HTTP 424 | HTTP 422 | HTTP 400 |
+| --- | ---: | ---: | ---: |
+| Codex 0.153.2 / Responses | 6 | 6 | 1 |
+| Claude Code 2.1.263 / Messages | 1 | 1 | 1 |
+| OpenCode 1.18.18 / Responses | 1 | 1 | 1 |
+
+Codex 424/422 reported five reconnects before failing. Codex 400 immediately
+reported `turn.failed`.
+
+| Additional case | Main requests | Title requests | Process elapsed |
+| --- | ---: | ---: | ---: |
+| Codex 400 after 121-second header delay | 1 | 0 | 121,089 ms |
+| Claude 424 after 121-second header delay | 1 | 0 | 121,652 ms |
+| OpenCode Responses 424 after 121-second header delay | 1 | 1 | 123,522 ms |
+| OpenCode Anthropic 424, immediate | 1 | 1 | 2,259 ms |
+| OpenCode Anthropic 400, immediate | 1 | 1 | 1,683 ms |
+| Claude 424 with retry watchdog enabled | 1 | 0 | 585 ms |
+
+All listed cases exited with code 1, without a harness timeout or terminating
+signal. Immediate-matrix elapsed times were not retained and are not inferred.
+
+OpenCode preserved `APIError`, integer `data.statusCode=424`,
+`data.isRetryable=false`, and a JSON-string `data.responseBody` containing the
+closed envelope. Claude preserved the status and neutral message but classified
+the custom type as `error: unknown` and `terminal_reason: api_error`.
+
+## Explanation and evidence limits
+
+Version-pinned source inspection identified the relevant native ownership:
+
+- Codex `rust-v0.153.2`: `codex-rs/codex-api/src/api_bridge.rs` maps HTTP 400
+ to `InvalidRequest`, while 424/422 become `UnexpectedStatus`.
+ `codex-rs/protocol/src/error.rs` defines retryability.
+ `codex-rs/core/src/session/turn.rs` and `responses_retry.rs` apply the outer
+ stream retry budget independently of provider HTTP retries.
+- Codex `codex-rs/model-provider-info/src/lib.rs` separates request and stream
+ retry settings; the audited default stream budget was five.
+- OpenCode `v1.18.18`: `packages/opencode/src/session/retry.ts` checks status,
+ native retryability, and message/body patterns. The closed neutral envelope
+ did not match its retry patterns. `session/message-v2.ts` preserves the
+ structured native error used by Avibe's narrowly scoped `continue` bypass.
+- Claude's rolling environment-variable documentation is not a public,
+ version-pinned classifier. Its classification evidence here is the actual
+ installed 2.1.263 binary.
+
+The audit's temporary harness and model catalog were deleted during cleanup.
+This document preserves the procedure, significant options, and observed
+results, not an immediately runnable retained harness. Reproduction requires
+recreating the isolated catalog and OS sandbox.
+
+The integrated implementation still needs its own exact-head tests and the
+declared Incus/managed-engine/IM/browser acceptance. Native compatibility must
+be rechecked when supported native versions change. A lost terminal response
+does not create a cross-retry request identity, so this evidence is not an
+exactly-once delivery guarantee.
diff --git a/docs/plans/model-hub.md b/docs/plans/model-hub.md
index 385f2e048..44d77b172 100644
--- a/docs/plans/model-hub.md
+++ b/docs/plans/model-hub.md
@@ -587,7 +587,9 @@ else EXHAUSTED(classify_blockers(C))
```
`inspect_exact_hop` checks only whether that configured hop can run now. `healthy` and
-an elapsed cooldown are runnable. An unelapsed cooldown, `needs_action`, `error`, a
+an elapsed cooldown are eligible, unless a real request already owns half-open
+admission. Eligibility is not proof of recovery. An unelapsed cooldown, live
+connection backoff, `needs_action`, `error`, a
missing/deleted Source, a configured model no longer callable by that Source, or an
unavailable native CLI process is not runnable; the hop stays at its configured
position. The canonical reasons are respectively the existing classified Source reason,
@@ -716,14 +718,19 @@ overlay and keeps that durable/self-healing blocker's established health, reason
retry facts. The one process-layer exception is simultaneous `native_cli` unavailability:
its actionable `native_cli_unavailable` reason takes the single reason slot while the
backoff health and deadline remain visible and the chain is `interrupted`.
-Deadline expiry makes the hop runnable again without a write. The first subsequent
-user-visible model-output byte produced by that same affected Source clears both
-deadline and streak automatically; successful fallback output from another Source does
-not. Source endpoint/credential replacement and process reconstruction also clear them
+Deadline expiry makes the hop eligible again without a write or `recover` event. The
+first subsequent recognized model output, or `SUCCESS` with `recovery_verified: true`,
+produced by that same affected Source clears both deadline and streak automatically.
+Permissive buffered HTTP 200 admission and its compatibility `stream_started` flag
+are not recovery evidence; actual streaming output is read from
+`handle.observed.model_output_started`. Recognized empty protocol completion is valid
+success. Successful fallback output from another Source does not clear the affected
+Source. Source endpoint/credential replacement and process reconstruction also clear them
because the state is in-memory and identity-specific. Before an API read is serialized,
the assembler captures one read time and normalizes an expired overlay to the Source's
underlying non-backoff health and runnability; a stale backoff deadline never crosses the
-API boundary. The maximum is 30 seconds. This family never uses
+API boundary. The local network schedule caps at 30 seconds; valid longer upstream
+advice is not shortened to that cap. This family never uses
`models.source.cooldown.*`, never writes `Source.state`, and never creates a permanent
health verdict.
@@ -755,11 +762,89 @@ The final mirror registry checks the closed (classification, credential capabili
row and fails any extra retry or unlisted remedy.
Because health is source-global, a cooldown created through one backend affects every
-route using that Source. Because every turn runs the algorithm again, an elapsed
-cooldown naturally restores the configured leading hop without mutating order.
+route using that Source. Because every request runs the algorithm again, an elapsed
+cooldown naturally makes the configured leading hop eligible without mutating order.
The Model Gateway and Usage pages remain the pull surfaces for takeover state,
connector color, recent switches, and usage; provenance remains a debug affordance.
+**Bounded automatic recovery (owner amendment, September 9, 2026).** One
+`ModelHubService.recovery` owns live Source deadlines and half-open admission.
+The gateway calls `resolve_with_recovery`; the existing `resolve` remains one
+walk for explicit probes and single-walk characterization. There is no background
+probe scheduler or prompt/Turn/Source blackout deduplication.
+
+Each pending model HTTP request has one 120-second automatic admission window,
+starting at its first retryable failure or temporarily blocked admission. Fallback
+passes and Source changes never reset it. The service re-reads the effective route
+after waits and immediately tries a runnable fallback. It waits only for
+all-temporary supply; empty routes and mixed action/process/capability blockers
+retain their existing terminal rules. If the next eligibility time is outside
+the remaining window, it ends immediately; an in-flight owner may instead wake
+waiters before their window closes. Window expiry prevents another admission,
+never cancels an already connected slow inference, and never permits replay
+after output. Explicit Stop and downstream disconnection retain cancellation.
+An expired admission remains the explicit exhausted-recovery domain result even
+if another owner recovers or a route changes before this waiter resumes or
+finishes engine preparation. An actual admitted request's permanent, request,
+engine, or post-output failure retains its own terminal classification.
+
+Consecutive failed HTTP attempts on the same Hub Source identity use:
+
+- network: 1, 2, 4, 8, 16, 30 seconds;
+- server error: 30, 60, 120 seconds;
+- rate limit: 60, 120, 240, 300 seconds;
+- quota exhaustion: 300 seconds.
+
+Each family stays at its final value. Positive jitter adds at most 20%, capped
+at that family's local maximum, and never makes a local delay shorter. A valid
+`Retry-After` integer or HTTP date uses the response-header receipt instant,
+not body completion; admission uses the later of upstream advice and local
+backoff. Malformed, past, oversized, and non-ASCII advice is ignored with a
+redacted diagnostic. Scheduling and window duration are monotonic; aware UTC
+timestamps are read projections. Network state is never persisted. Existing
+shaped cooldown persistence remains compatible with earlier releases.
+Direct `native_cli` failures do not enter this HTTP coordinator: no Hub handle
+can prove their later success. They retain fixed native cooldowns (server 30,
+rate 60, quota 300 seconds), existing deadline-based launch eligibility and
+native display semantics, without live streaks, half-open ownership or
+timer-generated `recover` events. Unclassified native connection failures remain
+non-persistent and do not create a Hub backoff.
+
+An eligible affected Source admits one real request as half-open owner across
+waiting Sessions. Ownership and stale settlement fencing reuse the existing
+attempt-start generation. Cancellation releases that owner only after transport
+cleanup; old generations and replaced endpoint/credential identities cannot clear
+or extend the new identity's state. A valid recovery clears the streak and emits
+one `recover`; mere timer expiry emits none. Live reads expose optional
+`recovery: eligible | in_flight` on AgentChain hops. `in_flight` is not runnable;
+with no stronger blocker it is temporary `waiting`, with nullable `retry_at`.
+The same annotation feeds service, runtime launch, API chain and AgentSupply reads.
+If the adapter returns a completed local failure without transport admission,
+release the provisional half-open claim on that normal return as well as on
+exception or cancellation. Do not synthesize `on_admitted`, attempt observations
+or retry counts for a request the engine never owned.
+
+Temporary cooldown persistence and recovery bookkeeping are observational.
+A failed write cannot destroy valid output or replace the actual upstream
+failure. The same in-memory policy supplies effective shaped cooldown health,
+deadline and reason even when disk still says `standby`. After verified success,
+it retires only the exact old persisted cooldown observation for the same Source
+identity while that observation remains on disk. Fresh reads must not import it
+again, emit duplicate recovery, or retain its old failure streak. A different
+persisted cooldown, observed on-disk retirement, identity replacement or removal
+invalidates that retirement; action, configuration and inventory blockers still
+win. This is not a permanent healthy override, another freshness generation or
+a persistent recovery record. Canonical hop inspection supplies these private
+effective facts to chain serialization and both terminal blocker projections;
+public health/recovery enums and the public reason whitelist are unchanged.
+
+For an all-temporary chain with Hub supply, runtime preflight prepares the Hub
+launch immediately. The first model request owns the window; no pre-native wait
+spends a second window or delays native acceptance. Connected inference remains
+independent of this admission budget. The exhausted native wire response and
+read-only request-correlation hooks are specified in `model-hub-contracts/api.md`;
+neither changes historical provenance or authorizes an extra notification surface.
+
**Settlement freshness is owned by the attempt that started (round-7 audit, fixed
round-8).** Because health is source-global and every turn re-runs the algorithm, a
failure only describes the Source *as the failing attempt used it*. Each attempt
@@ -943,7 +1028,9 @@ and every stored route position. (Top-up is the third tap
and needs no *replacement* route of ours — no credential of ours changes; it is a
link out to the vendor.)
-**A normal turn never probes a blocked Source in hope that it recovered.** The explicit
+**A normal turn never probes an action-blocked Source in hope that it recovered.**
+Temporary eligibility admits the pending real request under the bounded policy
+above; it does not launch an independent probe. The explicit
Source-details recovery path is `POST /api/models/sources/<id>/refresh`, the same saved
mutation defined in §4.1. It may test `needs_action` or `error` after a
user acts; a successful current observation clears the blocker without recreating or
@@ -959,7 +1046,7 @@ splits — on whether the user owes an action:
| --- | --- | --- | --- |
| `ok` | 正常 | — | serving from the intended head of the chain |
| `degraded` | 降级 | — | serving via a fallback, and/or some sources in the chain are down |
-| `waiting` | 暂时全部在冷却 | **yes** | nothing runnable right now, but every blocker is a persisted cooldown or live connection backoff — recovers unattended at the earliest `retry_at` |
+| `waiting` | 暂时全部在冷却 | **yes** | nothing runnable right now, but every blocker is an effective cooldown, live connection backoff, or an in-flight half-open owner — eligibility may return unattended; recovery still needs actual success |
| `interrupted` | 无可用来源 | **no** | nothing runnable and the effective chain is empty or at least one hop has a non-self-healing blocker: `needs_action`, `error`, `source_missing`, `model_unsupported`, or `native_cli_unavailable` |
These four values are the **only backend-level supply-health wording**. The Gateway
@@ -984,12 +1071,14 @@ detail/remedy copy for restoring the sanctioned local CLI; it is never presented
upstream Source cooldown.
`waiting` exists to keep the surfacing rule below consistent. An agent whose
-sources are *all* in persisted cooldown or live connection backoff has nothing runnable,
+sources are *all* in effective cooldown, live connection backoff, or half-open
+ownership has nothing runnable,
but nothing is owed either —
-it heals itself in minutes. Collapsing that into `interrupted` would tell the user to
-go fix a problem that resolves before they finish reading the sentence, which is
-exactly what the self-healing tier is supposed to prevent. The Turn-outcome copy matrix
-renders its recovery time rather than a fault; `current` is null in both states, so
+automatic recovery can proceed without user action. Collapsing that into `interrupted` would tell the user to
+repair a Source that may only need time. This does not promise that it will
+recover or that a terminal request remains live. The Turn-outcome copy matrix
+states when automatic recovery has ended; live progress and Source eligibility
+are separate read projections. `current` is null in both states, so
neither ever renders a stale 使用中.
**Two grains, one taxonomy.** `supply_status` above is the **agent** rollup, and it
@@ -1008,7 +1097,7 @@ rollup stays what its name says. One taxonomy, two grains, and only one definiti
The predicate itself is stated **once, here**, and every contract that carries either
grain points back at this table rather than restating it: `interrupted` when the chain
is empty **or at least one blocker needs the user**, `waiting` only when every blocker
-is a persisted cooldown or live connection backoff. The asymmetry is deliberate and
+is an effective cooldown, live connection backoff, or half-open ownership. The asymmetry is deliberate and
load-bearing — `interrupted` is the
OR-branch, `waiting` the AND-branch, so a chain holding one cooling source and one
revoked key is `interrupted`. Reading it as "every member needs the user" leaves that
diff --git a/modules/agents/model_hub.py b/modules/agents/model_hub.py
index e28bfb690..a822228d8 100644
--- a/modules/agents/model_hub.py
+++ b/modules/agents/model_hub.py
@@ -8,7 +8,7 @@ import json
import os
import re
import shutil
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field, replace
from pathlib import Path
from typing import Any, Callable, Literal, Mapping, Optional, cast
@@ -343,6 +343,7 @@ def build_claude_hub_env(
result["ANTHROPIC_BASE_URL"] = launch.gateway_base_url
result["ANTHROPIC_AUTH_TOKEN"] = launch.gateway_token
result["CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY"] = "1"
+ result["CLAUDE_CODE_MAX_RETRIES"] = "0"
else:
# A native_cli hop keeps the user's official CLI authentication.
result = dict(base_env)
@@ -388,6 +389,10 @@ def build_codex_hub_launch(
"-c",
f"model_providers.{provider}.requires_openai_auth=false",
"-c",
+ f"model_providers.{provider}.request_max_retries=0",
+ "-c",
+ "features.unbounded_connection_retries=false",
+ "-c",
f"model_catalog_json={format_toml_basic_string(str(model_catalog_path))}",
]
return overrides + list(base_args), env
@@ -541,19 +546,28 @@ class ModelHubRuntimeRouter:
now=self.service.now(),
unavailable_source_ids=self._unavailable_native_source_ids(config, backend),
supply_channel=supply_channel,
+ live_recovery=self.service.recovery_annotations(config),
)
- if not resolution.recoverable_source_ids:
- return config, resolution
- await self.service._recover_resolution_sources(resolution)
- config = self.service.store.load()
- return config, resolve_model_hub_turn(
- config,
- backend,
- requested_model,
- now=self.service.now(),
- unavailable_source_ids=self._unavailable_native_source_ids(config, backend),
- supply_channel=supply_channel,
- )
+ if (
+ self.turn_gateway is not None and resolution.source is None
+ and resolution.supply_status == "waiting"
+ and resolution.inspected_hops
+ and any(
+ hop.source is not None and hop.source.supply_channel == "hub"
+ for hop in resolution.inspected_hops
+ )
+ ):
+ # Preflight prepares the existing native delivery. Its first model
+ # request owns recovery admission; no second startup wait/window.
+ first = next(
+ hop for hop in resolution.inspected_hops
+ if hop.source is not None and hop.source.supply_channel == "hub"
+ )
+ resolution = replace(
+ resolution, channel="hub", source=first.source,
+ target_model=first.model_id or requested_model,
+ )
+ return config, resolution
@staticmethod
def _route_key(launch: ModelHubLaunch) -> tuple[BackendName, str]:
@@ -1129,8 +1143,11 @@ class ModelHubRuntimeRouter:
provider["models"][identifier] = projected_model
projected_identifiers.append(identifier)
model_provider_ids.append((identifier, overlay_provider_id))
- if resolution.candidate_hops:
- candidate = resolution.candidate_hops[0]
+ if resolution.source is not None:
+ candidate = (
+ resolution.candidate_hops[0] if resolution.candidate_hops
+ else next(hop for hop in resolution.inspected_hops if hop.source_id == resolution.source.id)
+ )
if candidate.source is None or candidate.model_id is None:
continue
available_identifiers.append(identifier)
diff --git a/modules/agents/opencode/poll_loop.py b/modules/agents/opencode/poll_loop.py
index af6c6444f..77801221f 100644
--- a/modules/agents/opencode/poll_loop.py
+++ b/modules/agents/opencode/poll_loop.py
@@ -3,6 +3,7 @@
from __future__ import annotations
import asyncio
+import json
import logging
import math
import time
@@ -13,11 +14,12 @@ from config.v2_config import (
DEFAULT_OPENCODE_ERROR_RETRY_LIMIT,
)
from core.backend_failure import emit_backend_failure
+from core.handlers.model_hub.retry import RECOVERY_EXHAUSTED_CODE
from core.message_context import build_context_session_key
from core.message_output import terminal_output_for, terminal_turn_output
from core.processing_indicator import STOPPED_REACTION_EMOJI
from modules.agents.base import AgentRequest
-from modules.agents.model_hub import bind_persisted_launch
+from modules.agents.model_hub import bind_persisted_launch, launch_for_context
from modules.im import MessageContext
from vibe.i18n import t as i18n_t
@@ -49,6 +51,35 @@ def _opencode_error_text(error: object) -> str:
return f"{error_name} - {error_message[:500]}".strip(" -")
+def _is_model_hub_recovery_exhausted(context: MessageContext, error: object) -> bool:
+ """Recognize the Hub's closed terminal, never a retry-looking error string."""
+
+ launch = launch_for_context(context)
+ if launch is None or launch.channel != "hub" or launch.backend != "opencode":
+ return False
+ if not isinstance(error, dict) or error.get("name") != "APIError":
+ return False
+ data = error.get("data")
+ if not isinstance(data, dict) or type(data.get("statusCode")) is not int or data["statusCode"] != 424:
+ return False
+ body = data.get("responseBody")
+ # This is recognition of a small closed envelope, not a response-body limit.
+ if not isinstance(body, str) or len(body) > 4096:
+ return False
+ try:
+ payload = json.loads(body)
+ except (ValueError, RecursionError):
+ return False
+ if not isinstance(payload, dict) or payload.get("type") != "error":
+ return False
+ detail = payload.get("error")
+ return (
+ isinstance(detail, dict)
+ and detail.get("type") == RECOVERY_EXHAUSTED_CODE
+ and detail.get("code") == RECOVERY_EXHAUSTED_CODE
+ )
+
+
def _message_info(message: Dict[str, Any]) -> Dict[str, Any]:
info = message.get("info")
return info if isinstance(info, dict) else {}
@@ -677,6 +708,7 @@ class OpenCodePollLoop:
if msg_error and last_id != last_error_message_id:
last_error_message_id = last_id
diagnostic = _opencode_error_text(msg_error)
+ hub_recovery_ended = _is_model_hub_recovery_exhausted(request.context, msg_error)
logger.warning(
"OpenCode message error detected for %s: %s (retry %d/%d)",
@@ -686,7 +718,10 @@ class OpenCodePollLoop:
error_retry_limit,
)
- if error_retry_count < error_retry_limit:
+ if (
+ error_retry_count < error_retry_limit
+ and not hub_recovery_ended
+ ):
error_retry_count += 1
logger.info(
"Auto-retrying OpenCode session %s with 'continue' (attempt %d/%d)",
@@ -723,7 +758,11 @@ class OpenCodePollLoop:
)
await self._record_model_hub_failure(request.context, diagnostic)
- message = self._t("error.opencodeBackendError", error=diagnostic)
+ message = (
+ self._t("modelHub.recovery.ended")
+ if hub_recovery_ended
+ else self._t("error.opencodeBackendError", error=diagnostic)
+ )
await emit_backend_failure(
self._agent.controller,
request.context,
diff --git a/tests/e2e/test_model_hub_routing.py b/tests/e2e/test_model_hub_routing.py
index e14c33261..1599e6d83 100644
--- a/tests/e2e/test_model_hub_routing.py
+++ b/tests/e2e/test_model_hub_routing.py
@@ -5,7 +5,7 @@ from __future__ import annotations
import asyncio
import json
import time
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
from pathlib import Path
import aiohttp
@@ -872,12 +872,15 @@ def _assert_probe_cooldown_and_next_request_selection(
expected_detail: str,
expected_reason: str,
expected_delay: int,
+ expected_max_delay: float | None = None,
) -> None:
first, second = _prepare_probe_chain(
app, first_upstream, second_upstream
)
first_upstream.configure(auth=auth_behavior)
+ before_probe = datetime.now(timezone.utc)
failed = _probe(app)
+ after_probe = datetime.now(timezone.utc)
failed_body = failed.json()
assert failed.status == 200, failed_body
assert failed_body["probe"]["reachable"] is False
@@ -886,8 +889,9 @@ def _assert_probe_cooldown_and_next_request_selection(
assert first_state["status"] == "cooldown"
assert first_state["detail_key"] == expected_detail
retry_at = datetime.fromisoformat(first_state["retry_at"])
- delay = (retry_at - datetime.now(timezone.utc)).total_seconds()
- assert expected_delay - 8 <= delay <= expected_delay + 2
+ upper = expected_max_delay if expected_max_delay is not None else expected_delay
+ assert before_probe + timedelta(seconds=expected_delay) <= retry_at
+ assert retry_at <= after_probe + timedelta(seconds=upper)
next_probe = _probe(app)
next_body = next_probe.json()
@@ -923,6 +927,7 @@ def test_d3_rate_limit_moves_the_next_api_probe_to_hop_one(
expected_detail="models.source.cooldown.rate_limited",
expected_reason="rate_limited",
expected_delay=60,
+ expected_max_delay=72,
)
@@ -983,14 +988,14 @@ def test_d5_nonrefreshable_401_marks_the_key_for_repair(
assert next_probe.json()["probe"]["source_id"] == second["id"]
-def test_d6_server_error_ignores_retry_after_and_uses_flat_cooldown(
+def test_d6_server_error_preserves_later_retry_after_through_real_engine(
model_hub_app_factory,
mock_llm_upstream,
) -> None:
- """D6 API subset: a 5xx uses 30s despite Retry-After: 600."""
+ """D6 API subset: a 5xx preserves Retry-After: 600 through the real engine."""
- # D-4 is open. The current contract deliberately records the flat 30s
- # server-error cooldown rather than honoring the upstream Retry-After.
+ # D-4 is approved: later upstream advice wins over local backoff/jitter.
+ # This asserts the actual managed-engine header boundary, not just EngineClient.
with MockLLMUpstream() as second_upstream:
with _engine_app(model_hub_app_factory) as app:
_assert_probe_cooldown_and_next_request_selection(
@@ -1000,7 +1005,7 @@ def test_d6_server_error_ignores_retry_after_and_uses_flat_cooldown(
auth_behavior="5xx",
expected_detail="models.source.cooldown.server_error",
expected_reason="server_error",
- expected_delay=30,
+ expected_delay=600,
)
diff --git a/tests/scenarios/INDEX.yaml b/tests/scenarios/INDEX.yaml
index 85004ed4f..a75e6f213 100644
--- a/tests/scenarios/INDEX.yaml
+++ b/tests/scenarios/INDEX.yaml
@@ -151,6 +151,14 @@ capabilities:
- tests/test_model_hub_injection.py
- tests/test_model_hub_runtime.py
- tests/test_model_hub_inference_wait.py
+ - tests/test_model_hub_retry_advice.py
+ - tests/test_model_hub_retry_policy.py
+ - tests/test_model_hub_retry_lifecycle.py
+ - tests/test_model_hub_recovery_evidence.py
+ - tests/test_model_hub_progress.py
+ - tests/test_model_hub_failure_presentation.py
+ - tests/test_opencode_model_hub_retry.py
+ - ui/src/lib/modelHubRecovery.test.ts
- tests/test_model_hub_request_body_budget.py
- tests/test_model_hub_l3.py
- tests/test_model_hub_usage.py
diff --git a/tests/scenarios/model_hub/catalog.yaml b/tests/scenarios/model_hub/catalog.yaml
index 29615b6b1..b73c07f9c 100644
--- a/tests/scenarios/model_hub/catalog.yaml
+++ b/tests/scenarios/model_hub/catalog.yaml
@@ -9,6 +9,14 @@ capability:
- tests/test_model_hub_injection.py
- tests/test_model_hub_runtime.py
- tests/test_model_hub_inference_wait.py
+ - tests/test_model_hub_retry_advice.py
+ - tests/test_model_hub_retry_policy.py
+ - tests/test_model_hub_retry_lifecycle.py
+ - tests/test_model_hub_recovery_evidence.py
+ - tests/test_model_hub_progress.py
+ - tests/test_model_hub_failure_presentation.py
+ - tests/test_opencode_model_hub_retry.py
+ - ui/src/lib/modelHubRecovery.test.ts
- tests/test_model_hub_request_body_budget.py
- tests/test_model_hub_l3.py
- tests/test_model_hub_usage.py
@@ -77,6 +85,90 @@ status_legend:
expected_fail: false
priority: false
scenarios:
+ - id: MH-RETRY-WINDOW-001
+ name: Every fallback pass shares one admission window without replaying output
+ status: covered
+ kind: recovery
+ layer: unit
+ test: tests/test_model_hub_retry_policy.py::test_one_window_covers_all_fallback_passes_without_replaying_after_output
+ - id: MH-RETRY-RECOVERY-001
+ name: Real protocol output or empty completion proves recovery while unknown success does not
+ status: covered
+ kind: recovery
+ layer: unit
+ test: tests/test_model_hub_retry_policy.py::test_real_gateway_retries_preoutput_then_preserves_one_response
+ - id: MH-RETRY-CONCURRENCY-001
+ name: Concurrent requests share one half-open slot through cancellation and slow inference
+ status: covered
+ kind: recovery
+ layer: unit
+ test: tests/test_model_hub_retry_policy.py::test_real_gateway_has_one_half_open_owner_and_interruptible_waiters
+ - id: MH-RETRY-NATIVE-001
+ name: OpenCode consumes the Hub terminal without automatically sending continue
+ status: covered
+ kind: recovery
+ layer: unit
+ test: tests/test_opencode_model_hub_retry.py::test_hub_terminal_skips_automatic_continue_and_emits_one_failure
+ - id: MH-RETRY-OWNERSHIP-001
+ name: A production adapter returning without admission releases its provisional half-open claim
+ status: covered
+ kind: recovery
+ layer: unit
+ test: tests/test_model_hub_retry_lifecycle.py::test_unavailable_production_client_releases_unadmitted_half_open_claim
+ - id: MH-RETRY-NATIVE-002
+ name: Native CLI settlements retain fixed cooldowns without entering HTTP recovery streaks
+ status: covered
+ kind: recovery
+ layer: unit
+ test: tests/test_model_hub_retry_lifecycle.py::test_native_failure_never_enters_http_recovery_streak
+ - id: MH-RETRY-PERSISTENCE-001
+ name: Verified recovery survives failed persistence and fresh reads of the retired cooldown
+ status: covered
+ kind: recovery
+ layer: unit
+ test: tests/test_model_hub_retry_lifecycle.py::test_verified_response_survives_failed_recovery_persistence_and_fresh_reads
+ - id: MH-RETRY-EXPIRY-001
+ name: An expired waiter retains its closed terminal when live supply changes
+ status: covered
+ kind: recovery
+ layer: unit
+ test: tests/test_model_hub_retry_lifecycle.py::test_expiry_remains_a_closed_terminal_when_live_supply_changes
+ - id: MH-RETRY-EXPIRY-002
+ name: Window expiry does not relabel an observed hard failure as temporary recovery exhaustion
+ status: covered
+ kind: recovery
+ layer: unit
+ test: tests/test_model_hub_retry_lifecycle.py::test_expiry_after_mixed_real_failures_keeps_the_action_blocker_terminal
+ - id: MH-RETRY-ADVICE-001
+ name: Failed HTTP advice retains the receipt clock across delayed or broken bodies
+ status: covered
+ kind: contract
+ layer: unit
+ test: tests/test_model_hub_retry_advice.py::test_retry_advice_survives_error_body_delays_and_failures
+ - id: MH-RETRY-EVIDENCE-001
+ name: Unknown buffered success passes through without proving recovery
+ status: covered
+ kind: contract
+ layer: unit
+ test: tests/test_model_hub_recovery_evidence.py::test_unknown_buffered_success_is_forwarded_without_recovery_evidence
+ - id: MH-RETRY-PROGRESS-001
+ name: Recovery progress invalidates only its exact live Session
+ status: covered
+ kind: projection
+ layer: unit
+ test: tests/test_model_hub_progress.py::test_http_recovery_reaches_live_turn_state_and_clears_without_chat
+ - id: MH-RETRY-COPY-001
+ name: Terminal copy says recovery ended instead of promising automatic restoration
+ status: covered
+ kind: copy
+ layer: unit
+ test: tests/test_model_hub_failure_presentation.py::test_gateway_terminal_survives_native_recorder_before_shared_notice
+ - id: MH-RETRY-WEB-001
+ name: The quiet Web label cannot infer recovery from either deadline
+ status: covered
+ kind: projection
+ layer: unit
+ test: ui/src/lib/modelHubRecovery.test.ts::MH-RETRY-WEB-001
- id: MH-CATALOG-001
name: Every scenario, observation, and canonical test resolves to discoverable executable evidence
status: covered
@@ -861,11 +953,11 @@ scenarios:
layer: e2e-auto
test: tests/e2e/test_model_hub_routing.py::test_d5_nonrefreshable_401_marks_the_key_for_repair
- id: D6
- name: A server error uses the current flat cooldown
- status: baseline
+ name: A server error honors later upstream advice through the real engine
+ status: assert
kind: failover
layer: e2e-auto
- test: tests/e2e/test_model_hub_routing.py::test_d6_server_error_ignores_retry_after_and_uses_flat_cooldown
+ test: tests/e2e/test_model_hub_routing.py::test_d6_server_error_preserves_later_retry_after_through_real_engine
- id: D7
name: A partial stream terminates without replaying another source
status: skip
@@ -875,21 +967,17 @@ scenarios:
owning_layer: future live-turn suite (turn gateway)
evidence: docs/plans/model-hub-e2e-test-plan.md#d-per-turn-routing-fallback-capability-behavior-c41c50-c74c81
- id: D8
- name: Cooldown expiry recovers hop zero and emits recovery
- status: skip
+ name: Cooldown expiry permits a real attempt and verified output proves recovery
+ status: covered
kind: recovery
- layer: contract
- reason: deterministic cooldown expiry requires a controllable clock across resolution and recovery
- owning_layer: unit/contract suite (controllable-time resolution)
- evidence: docs/plans/model-hub-e2e-test-plan.md#d-per-turn-routing-fallback-capability-behavior-c41c50-c74c81
+ layer: unit
+ test: tests/test_model_hub_retry_policy.py::test_real_gateway_retries_preoutput_then_preserves_one_response
- id: D9
- name: Exhausting every hop returns mapping_target_unavailable
- status: skip
+ name: Exhausting temporary recovery returns one closed terminal with original upstream provenance
+ status: covered
kind: failover
- layer: turn-gateway
- reason: same-turn route exhaustion requires the private credential issued to a live backend turn
- owning_layer: future live-turn suite (turn gateway)
- evidence: docs/plans/model-hub-e2e-test-plan.md#d-per-turn-routing-fallback-capability-behavior-c41c50-c74c81
+ layer: scenario
+ test: tests/scenarios/model_hub/test_model_hub_turn_contract_scenarios.py::test_mh_turn_takeover_is_silent_and_provenance_names_the_serving_hop
- id: D10
name: Reasoning intent survives exact-provider fallback without strip telemetry
status: assert
@@ -1011,7 +1099,6 @@ next_priority:
- B8
- B9
- D1
- - D6
- F1
- G2
- G3
diff --git a/tests/scenarios/model_hub/test_model_hub_live_resolution_scenarios.py b/tests/scenarios/model_hub/test_model_hub_live_resolution_scenarios.py
index ef9092645..04e9bc5a3 100644
--- a/tests/scenarios/model_hub/test_model_hub_live_resolution_scenarios.py
+++ b/tests/scenarios/model_hub/test_model_hub_live_resolution_scenarios.py
@@ -7,7 +7,7 @@ from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from pathlib import Path
from types import SimpleNamespace
-from typing import AsyncIterator
+from typing import AsyncIterator, Callable
import aiohttp
import pytest
@@ -78,6 +78,25 @@ class AdapterResult:
code: str | None = None
body: bytes | None = None
stream_started: bool = False
+ recovery_verified: bool = False
+
+
+class ScenarioClock:
+ """Advance eligibility and its displayed wall time together."""
+
+ def __init__(self) -> None:
+ self.wall = datetime(2026, 7, 25, tzinfo=timezone.utc)
+ self.elapsed = 0.0
+
+ def now(self) -> datetime:
+ return self.wall
+
+ def monotonic(self) -> float:
+ return self.elapsed
+
+ def advance(self, duration: timedelta) -> None:
+ self.wall += duration
+ self.elapsed += duration.total_seconds()
class InvokeHandle:
@@ -160,6 +179,7 @@ class AdapterBoundaryFake:
stream_started=result.stream_started,
model_id=model_id,
source_id=source_id,
+ recovery_verified=result.recovery_verified,
)
return InvokeHandle(outcome, result.body)
@@ -232,8 +252,9 @@ def _service(
adapter: AdapterBoundaryFake,
*,
now,
+ monotonic: Callable[[], float] | None = None,
) -> ModelHubService:
- return ModelHubService(
+ service = ModelHubService(
store=store,
adapter=adapter,
events=BoundedEventLog(tmp_path / "events.json"),
@@ -241,6 +262,9 @@ def _service(
revocations=CredentialRevocationJournal(tmp_path / "revocations.json"),
now=now,
)
+ if monotonic is not None:
+ service.recovery.monotonic = monotonic
+ return service
async def _post_turn(
@@ -256,7 +280,7 @@ async def _post_turn(
"messages": [],
"stream": stream,
}
- async with aiohttp.ClientSession(trust_env=False) as client:
+ async with aiohttp.ClientSession(trust_env=False, timeout=aiohttp.ClientTimeout(total=5)) as client:
async with client.post(
f"{launch.gateway_base_url}/v1/{endpoint}",
headers=headers,
@@ -362,6 +386,7 @@ def test_codex_backend_model_id_survives_until_gateway_resolution(
(AdapterResult(RawOutcomeKind.NETWORK_ERROR), "network"),
],
)
[email protected]("recovery_verified", [False, True], ids=["unverified", "verified"])
def test_mh_res_live_001_pre_stream_failure_falls_back_within_turn(
tmp_path: Path,
backend: str,
@@ -369,20 +394,32 @@ def test_mh_res_live_001_pre_stream_failure_falls_back_within_turn(
endpoint: str,
failed: AdapterResult,
reason: str,
+ recovery_verified: bool,
) -> None:
"""MH-RES-LIVE-001: the real turn gateway completes on candidate two."""
async def exercise() -> None:
- clock = [datetime(2026, 7, 25, tzinfo=timezone.utc)]
+ clock = ScenarioClock()
+ candidate_body = b'{"recovered":true}' if recovery_verified else b'{"unknown":true}'
adapter = AdapterBoundaryFake(
[
failed,
- AdapterResult(RawOutcomeKind.SUCCESS, status=200, body=b'{"ok":true}'),
- AdapterResult(RawOutcomeKind.SUCCESS, status=200, body=b'{"recovered":true}'),
+ AdapterResult(
+ RawOutcomeKind.SUCCESS, status=200, body=b'{"ok":true}', recovery_verified=True,
+ ),
+ AdapterResult(
+ RawOutcomeKind.SUCCESS,
+ status=200,
+ body=candidate_body,
+ # Buffered forwarding also sets this legacy flag; it alone
+ # cannot certify that an unrecognized response recovered.
+ stream_started=True,
+ recovery_verified=recovery_verified,
+ ),
]
)
store = MemoryStore(_config(_source("src_primary1"), _source("src_backup01")))
- service = _service(tmp_path, store, adapter, now=lambda: clock[0])
+ service = _service(tmp_path, store, adapter, now=clock.now, monotonic=clock.monotonic)
gateway = ModelHubTurnGateway(service)
router = ModelHubRuntimeRouter(service=service, turn_gateway=gateway)
try:
@@ -391,7 +428,9 @@ def test_mh_res_live_001_pre_stream_failure_falls_back_within_turn(
assert overlay is not None
launch = await router.resolve_opencode_overlay_launch(overlay, requested_model)
else:
- launch = await router.resolve(backend, requested_model)
+ launch = await router.resolve(
+ backend, requested_model, process_scope="scenario-fallback", turn_id="turn_fallback",
+ )
status, body = await _post_turn(launch, endpoint=endpoint)
assert status == 200
assert body == b'{"ok":true}'
@@ -399,13 +438,46 @@ def test_mh_res_live_001_pre_stream_failure_falls_back_within_turn(
events = service.list_events(limit=5)
if failed.kind is RawOutcomeKind.NETWORK_ERROR:
assert store.load().sources[0].state.status == "standby"
- assert [event["kind"] for event in events] == ["switch"]
+ assert store.load().sources[0].state.retry_at is None
+ assert store.load().sources[0].state.detail_key is None
else:
assert store.load().sources[0].state.status == "cooldown"
- assert [event["kind"] for event in events[:2]] == ["switch", "cooldown"]
- assert events[1]["reason"] == reason
-
- clock[0] += timedelta(minutes=6)
+ initial_retry_at = store.load().sources[0].state.retry_at
+ initial_detail_key = store.load().sources[0].state.detail_key
+ assert [event["kind"] for event in events] == ["switch", "cooldown"]
+ assert events[1]["reason"] == reason
+ assert events[1]["from_source"] == "src_primary1"
+ # Successful output from the backup cannot recover the primary.
+ assert not any(event["kind"] == "recover" for event in events)
+ if backend != "opencode":
+ router.settle_turn(
+ "turn_fallback", settled_by=SETTLED_BY_TERMINAL_RESULT, ts=clock.now().isoformat(),
+ )
+ provenance = service.get_turn_provenance("turn_fallback")
+ assert provenance is not None
+ assert provenance["turn_id"] == "turn_fallback"
+ assert provenance["agent"] == backend
+ assert provenance["requested_model_id"] == requested_model
+ assert provenance["outcome"] == "served"
+ assert provenance["served"]["source_id"] == "src_backup01"
+ assert provenance["served"]["configured_model_id"] == _source_model_id(backend)
+ assert provenance["served"]["channel"] == "hub"
+ assert len(provenance["failed_attempts"]) == 1
+ attempt = provenance["failed_attempts"][0]
+ assert attempt["source_id"] == "src_primary1"
+ assert attempt["configured_model_id"] == _source_model_id(backend)
+ assert attempt["channel"] == "hub"
+ assert attempt["reason"] == reason
+ if failed.status is None:
+ assert "http_status" not in attempt
+ else:
+ assert type(attempt["http_status"]) is int
+ assert attempt["http_status"] == failed.status
+
+ clock.advance(timedelta(minutes=6))
+ chain = service.agent_chain(backend, requested_model)["chain"]
+ assert chain[0]["source_id"] == "src_primary1"
+ assert chain[0]["recovery"] == "eligible"
if backend == "opencode":
recovered_overlay = await router.prepare_opencode_overlay()
assert recovered_overlay is not None
@@ -415,13 +487,32 @@ def test_mh_res_live_001_pre_stream_failure_falls_back_within_turn(
)
else:
recovered_launch = await router.resolve(backend, requested_model)
+ assert not any(event["kind"] == "recover" for event in service.list_events(limit=10))
recovered_status, recovered_body = await _post_turn(
recovered_launch,
endpoint=endpoint,
)
assert recovered_status == 200
- assert recovered_body == b'{"recovered":true}'
+ assert recovered_body == candidate_body
assert adapter.invocations[-1][0] == "src_primary1"
+ recovery_events = [
+ event for event in service.list_events(limit=10) if event["kind"] == "recover"
+ ]
+ assert len(recovery_events) == int(recovery_verified)
+ primary = store.load().sources[0]
+ if recovery_verified:
+ assert recovery_events[0]["reason"] == "recovery"
+ assert recovery_events[0]["to_source"] == primary.id == "src_primary1"
+ assert primary.state.status == "standby"
+ assert primary.state.retry_at is None
+ assert primary.state.detail_key is None
+ else:
+ assert primary.state.status == (
+ "standby" if failed.kind is RawOutcomeKind.NETWORK_ERROR else "cooldown"
+ )
+ assert primary.state.retry_at == initial_retry_at
+ assert primary.state.detail_key == initial_detail_key
+ assert service.agent_chain(backend, requested_model)["chain"][0]["recovery"] == "eligible"
finally:
await gateway.close()
@@ -571,7 +662,7 @@ def test_mh_res_live_004_hub_subscription_falls_back_to_api_key_within_turn(
"""MH-RES-LIVE-004: a hub subscription quota failure serves the next hop in the same turn and marks the switch as entered_metered."""
async def exercise() -> None:
- clock = [datetime(2026, 7, 25, tzinfo=timezone.utc)]
+ clock = ScenarioClock()
subscription = _source("src_primary1", kind="subscription")
api_key = _source("src_backup01")
assert subscription.kind == "subscription"
@@ -586,12 +677,23 @@ def test_mh_res_live_004_hub_subscription_falls_back_to_api_key_within_turn(
status=429,
code="quota_exceeded",
),
- AdapterResult(RawOutcomeKind.SUCCESS, status=200, body=b'{"ok":true}'),
- AdapterResult(RawOutcomeKind.SUCCESS, status=200, body=b'{"recovered":true}'),
+ AdapterResult(
+ RawOutcomeKind.SUCCESS, status=200, body=b'{"ok":true}', recovery_verified=True,
+ ),
+ AdapterResult(
+ RawOutcomeKind.SUCCESS, status=200, body=b'{"unknown":true}', stream_started=True,
+ ),
+ AdapterResult(
+ RawOutcomeKind.SUCCESS,
+ status=200,
+ body=b'{"recovered":true}',
+ stream_started=True,
+ recovery_verified=True,
+ ),
]
)
store = MemoryStore(_config(subscription, api_key))
- service = _service(tmp_path, store, adapter, now=lambda: clock[0])
+ service = _service(tmp_path, store, adapter, now=clock.now, monotonic=clock.monotonic)
gateway = ModelHubTurnGateway(service)
router = ModelHubRuntimeRouter(service=service, turn_gateway=gateway)
try:
@@ -608,7 +710,7 @@ def test_mh_res_live_004_hub_subscription_falls_back_to_api_key_within_turn(
assert cooled.id == "src_primary1"
assert cooled.state.status == "cooldown"
assert cooled.state.detail_key == "models.source.cooldown.quota_exhausted"
- assert cooled.state.retry_at == (clock[0] + timedelta(seconds=300)).isoformat()
+ assert cooled.state.retry_at == (clock.now() + timedelta(seconds=300)).isoformat()
events = service.list_events(limit=5)
assert [event["kind"] for event in events[:2]] == ["switch", "cooldown"]
assert events[0]["from_source"] == "src_primary1"
@@ -617,7 +719,17 @@ def test_mh_res_live_004_hub_subscription_falls_back_to_api_key_within_turn(
assert events[0]["billing_note"] == "entered_metered"
assert events[1]["reason"] == "quota_exhausted"
- clock[0] += timedelta(minutes=6)
+ clock.advance(timedelta(minutes=6))
+ candidate = await router.resolve("claude", _requested_model("claude"))
+ assert candidate.source_id == "src_primary1"
+ assert store.load().sources[0].state.status == "cooldown"
+ assert not any(event["kind"] == "recover" for event in service.list_events(limit=10))
+ unverified_status, unverified_body = await _post_turn(candidate)
+ assert unverified_status == 200
+ assert unverified_body == b'{"unknown":true}'
+ assert adapter.invocations[-1][0] == "src_primary1"
+ assert store.load().sources[0].state.status == "cooldown"
+ assert not any(event["kind"] == "recover" for event in service.list_events(limit=10))
recovered_status, recovered_body = await _post_turn(
await router.resolve("claude", _requested_model("claude"))
)
@@ -679,16 +791,29 @@ def test_mh_effort_001_unknown_capability_preserves_intent_and_turn_completes(
asyncio.run(exercise())
+ ("upstream_status", "upstream_code", "wire_status", "wire_code", "terminal_reason"),
+ [
+ (429, "rate_limited", 429, "stream_interrupted", "stream_interrupted"),
+ (503, "request_too_large", 400, "upstream_request_invalid", "invalid_parameter"),
+ ],
+)
def test_turn_gateway_nonstream_buffer_surfaces_terminal_failure(
tmp_path: Path,
+ upstream_status: int,
+ upstream_code: str,
+ wire_status: int,
+ wire_code: str,
+ terminal_reason: str,
) -> None:
async def exercise() -> None:
+ now = datetime(2026, 7, 25, tzinfo=timezone.utc)
adapter = AdapterBoundaryFake(
[
AdapterResult(
RawOutcomeKind.HTTP_ERROR,
- status=429,
- code="rate_limited",
+ status=upstream_status,
+ code=upstream_code,
body=b'{"partial":',
stream_started=True,
),
@@ -699,16 +824,36 @@ def test_turn_gateway_nonstream_buffer_surfaces_terminal_failure(
tmp_path,
store,
adapter,
- now=lambda: datetime(2026, 7, 25, tzinfo=timezone.utc),
+ now=lambda: now,
)
gateway = ModelHubTurnGateway(service)
router = ModelHubRuntimeRouter(service=service, turn_gateway=gateway)
try:
status, body = await _post_turn(
- await router.resolve("claude", _requested_model("claude")),
+ await router.resolve(
+ "claude", _requested_model("claude"),
+ process_scope="scenario-buffered-failure", turn_id="turn_buffered_failure",
+ ),
+ )
+ assert status == wire_status
+ assert json.loads(body)["error"]["code"] == wire_code
+ assert [call[0] for call in adapter.invocations] == ["src_primary1"]
+ router.settle_turn(
+ "turn_buffered_failure", settled_by=SETTLED_BY_TERMINAL_RESULT, ts=now.isoformat(),
)
- assert status == 429
- assert json.loads(body)["error"]["code"] == "stream_interrupted"
+ provenance = service.get_turn_provenance("turn_buffered_failure")
+ assert provenance is not None
+ assert provenance["outcome"] == "failed_terminal"
+ assert provenance["served"] is None
+ terminal = provenance["terminal_error"]
+ assert terminal["source_id"] == "src_primary1"
+ assert terminal["configured_model_id"] == _source_model_id("claude")
+ assert terminal["channel"] == "hub"
+ assert terminal["reason"] == terminal_reason
+ assert terminal["stream_started"] is True
+ # Raw upstream status is evidence, not the gateway's projection.
+ assert type(terminal["http_status"]) is int
+ assert terminal["http_status"] == upstream_status
finally:
await gateway.close()
diff --git a/tests/scenarios/model_hub/test_model_hub_turn_contract_scenarios.py b/tests/scenarios/model_hub/test_model_hub_turn_contract_scenarios.py
index 1f8a55bfc..62b9a3e3d 100644
--- a/tests/scenarios/model_hub/test_model_hub_turn_contract_scenarios.py
+++ b/tests/scenarios/model_hub/test_model_hub_turn_contract_scenarios.py
@@ -258,7 +258,7 @@ async def _post_stream(launch) -> tuple[int, bytes, BaseException | None]:
def test_mh_turn_takeover_is_silent_and_provenance_names_the_serving_hop(tmp_path: Path) -> None:
- """MH-TAKEOVER-001: fallback serves in order, or settles exhausted after the same route ends."""
+ """MH-TAKEOVER-001 / D9: ordered fallback or closed exhaustion preserves upstream evidence."""
menu_model = fixed_model("claude")
first = source("src_takeover01", [menu_model])
@@ -381,16 +381,24 @@ def test_mh_turn_takeover_is_silent_and_provenance_names_the_serving_hop(tmp_pat
assert result["provenance"]["failed_attempts"][0]["source_id"] == first.id
assert result["provenance"]["served"]["source_id"] == second.id
Draft7Validator(_provenance_schema()).validate(result["provenance"])
- assert exhausted_result["status"] == 503
- assert (
- exhausted_result["body"]["error"]["code"]
- == "mapping_target_unavailable"
- )
+ assert exhausted_result["status"] == 424
+ assert exhausted_result["body"] == {
+ "type": "error",
+ "error": {
+ "type": "model_hub_recovery_exhausted",
+ "code": "model_hub_recovery_exhausted",
+ "message": "Automatic recovery has ended. Try again or choose another model.",
+ },
+ }
assert exhausted_result["provenance"]["outcome"] == "exhausted"
assert [attempt["source_id"] for attempt in exhausted_result["provenance"]["failed_attempts"]] == [
exhausted_first.id,
exhausted_last.id,
]
+ for attempt in exhausted_result["provenance"]["failed_attempts"]:
+ assert attempt["reason"] == "quota_exhausted"
+ assert type(attempt["http_status"]) is int
+ assert attempt["http_status"] == 429
assert exhausted_result["provenance"]["served"] is None
Draft7Validator(_provenance_schema()).validate(exhausted_result["provenance"])
diff --git a/tests/test_controller_model_hub_gate.py b/tests/test_controller_model_hub_gate.py
index a45183677..89bc988ad 100644
--- a/tests/test_controller_model_hub_gate.py
+++ b/tests/test_controller_model_hub_gate.py
@@ -4,12 +4,13 @@ import asyncio
import json
import threading
from types import SimpleNamespace
-from unittest.mock import AsyncMock
+from unittest.mock import AsyncMock, Mock
import pytest
from config.v2_config import V2Config
from core.controller import Controller
+from core.handlers.model_hub.provenance import BoundedProvenanceStore, TurnCorrelationRegistry
from modules.agents.model_hub import resolve_model_hub_launch, resolve_opencode_overlay_launch
@@ -44,7 +45,7 @@ def test_controller_leaves_model_hub_aggregate_absent_when_explicitly_disabled(m
@pytest.mark.parametrize("env_value", [None, "1"])
-def test_controller_builds_one_model_hub_aggregate_by_default_or_explicit_enable(monkeypatch, env_value):
+def test_controller_builds_one_model_hub_aggregate_by_default_or_explicit_enable(monkeypatch, tmp_path, env_value):
import core.handlers.model_hub as model_hub
import core.handlers.model_hub.turn_gateway as turn_gateway
import modules.agents.model_hub as agent_model_hub
@@ -61,6 +62,9 @@ def test_controller_builds_one_model_hub_aggregate_by_default_or_explicit_enable
calls.append(("gateway", value, language_provider))
self.service = value
self.language_provider = language_provider
+ self.correlation = TurnCorrelationRegistry(
+ BoundedProvenanceStore(tmp_path / "provenance.json"),
+ )
class Router:
def __init__(self, *, service, turn_gateway):
@@ -166,6 +170,46 @@ def test_controller_builds_one_model_hub_aggregate_by_default_or_explicit_enable
("router", service, controller.model_hub_turn_gateway),
]
+ # Exercise the initializer's real callback binding, not a second test-only
+ # binding: one registry publishes material changes for its live Session.
+ registry = controller.model_hub_turn_gateway.correlation
+ assert callable(registry.on_recovery_changed)
+ token = registry.credentials("codex", "fixture-controller", "turn-live")
+ registry.begin_gateway_request(
+ backend="codex", token=token, requested_model_id="agent-model",
+ )
+ controller.session_turns = SimpleNamespace(in_flight={
+ "ses-live": SimpleNamespace(
+ task=SimpleNamespace(done=lambda: False),
+ context=SimpleNamespace(platform_specific={"turn_token": "turn-live"}),
+ ),
+ "ses-peer": SimpleNamespace(
+ task=SimpleNamespace(done=lambda: False),
+ context=SimpleNamespace(platform_specific={"turn_token": "turn-peer"}),
+ ),
+ })
+ publish = Mock()
+ monkeypatch.setattr("core.inbox_events.bus.publish", publish)
+ progress = {
+ "phase": "waiting", "attempt_count": 1, "source_id": "src_recovery01",
+ "reason": "network", "started_at": "2026-09-09T00:00:00+00:00",
+ "next_eligible_at": "2026-09-09T00:00:01+00:00",
+ "window_end": "2026-09-09T00:02:00+00:00",
+ }
+ registry.update_recovery("turn-live", backend="codex", request_id="one", snapshot=progress)
+ registry.update_recovery("turn-live", backend="codex", request_id="one", snapshot=dict(progress))
+ publish.assert_called_once_with(
+ "session.activity", {"session_id": "ses-live", "event": "model_recovery"},
+ )
+ assert registry.recovery_snapshot("turn-live") == [{**progress, "request_id": "one"}]
+ publish.reset_mock()
+ registry.update_recovery("turn-live", backend="codex", request_id="one", snapshot=None)
+ publish.assert_called_once_with(
+ "session.activity", {"session_id": "ses-live", "event": "model_recovery"},
+ )
+ assert registry.recovery_snapshot("turn-live") == []
+ assert not registry.store.path.exists()
+
refresh_callbacks[0]()
async def drain_refresh() -> None:
diff --git a/tests/test_message_dispatcher_status_bubble.py b/tests/test_message_dispatcher_status_bubble.py
index 71d1e1468..a9107b7ca 100644
--- a/tests/test_message_dispatcher_status_bubble.py
+++ b/tests/test_message_dispatcher_status_bubble.py
@@ -125,6 +125,40 @@ def _dispatcher(controller, *, now=1000.0, disable_heartbeat=True):
class StatusBubbleProcessTests(unittest.IsolatedAsyncioTestCase):
+ async def test_recovery_reuses_existing_bubble_without_notification_or_step(self):
+ from datetime import datetime, timedelta, timezone
+ from types import SimpleNamespace
+
+ controller = _StubController(platform="slack")
+ rows = []
+ controller.model_hub_turn_gateway = SimpleNamespace(
+ correlation=SimpleNamespace(recovery_snapshot=lambda _turn_id: rows),
+ )
+ dispatcher = _dispatcher(controller)
+ context = _ctx()
+ context.platform_specific = {"turn_token": "turn-exact"}
+ await dispatcher.begin_status_bubble(context)
+ key = dispatcher._get_consolidated_message_key(context)
+ message_id = dispatcher._consolidated_message_ids[key]
+ now = datetime.now(timezone.utc)
+ rows.append({
+ "phase": "waiting",
+ "started_at": (now - timedelta(seconds=10)).isoformat(),
+ "next_eligible_at": (now + timedelta(seconds=20)).isoformat(),
+ })
+ await dispatcher._status_heartbeat_render_once(context, controller.im_client, key, message_id)
+ self.assertEqual(len(controller.im_client.sent), 1)
+ self.assertIn("Waiting to retry", controller.im_client.edits[-1][1])
+ self.assertEqual(dispatcher._status_step_count.get(key, 0), 0)
+ body, _footer = dispatcher._compose_status_message(
+ context, key, done=True, result_body="Final answer",
+ )
+ self.assertEqual(body, "Final answer")
+ rows.clear()
+ await dispatcher._status_heartbeat_render_once(context, controller.im_client, key, message_id)
+ self.assertNotIn("retry", controller.im_client.edits[-1][1])
+ self.assertEqual(len(controller.im_client.sent), 1)
+
async def test_missing_progress_getter_defaults_off(self):
controller = object()
d = ConsolidatedMessageDispatcher(controller)
diff --git a/tests/test_model_hub_failure_presentation.py b/tests/test_model_hub_failure_presentation.py
new file mode 100644
index 000000000..043444db1
--- /dev/null
+++ b/tests/test_model_hub_failure_presentation.py
@@ -0,0 +1,195 @@
+"""User-visible failure copy consumes exact Hub facts, not native error text."""
+
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, Mock
+
+import aiohttp
+import pytest
+
+from config.v2_config import ModelHubBackendModelConfig, ModelHubSourceStateConfig
+from core.backend_failure import emit_backend_failure, emit_replayed_backend_failure
+from core.handlers.model_hub.adapter import RawOutcomeKind
+from core.handlers.model_hub.provenance import (
+ render_turn_outcome_copy,
+)
+from core.handlers.model_hub.turn_gateway import ModelHubTurnGateway
+from modules.agents.model_hub import ModelHubRuntimeRouter, bind_launch
+from modules.im import MessageContext
+from tests.test_model_hub_l3 import _canonicalize_fixed_test_routes, _outcome
+from tests.test_model_hub_resolution import _config, _service, _source
+from tests.test_model_hub_retry_policy import clock_service
+from vibe.i18n import t
+
+
+def unavailable(tmp_path, backend="codex"):
+ source = _source("src_copy0001", ("模型/alpha",))
+ source.display_name = "Relay 服务"
+ source.state = ModelHubSourceStateConfig(
+ status="cooldown",
+ detail_key="models.source.cooldown.server_error",
+ retry_at="2026-09-09T04:30:00+00:00",
+ )
+ config = _config([source], model="模型/alpha")
+ config.agents[backend].models.append(ModelHubBackendModelConfig(id="模型/alpha", origin="manual"))
+ service, _, _ = _service(tmp_path, config)
+ config, resolution = service._inspect_terminal_chain(backend=backend, model_id="模型/alpha")
+ return service._produce_exhausted_terminal_outcome(
+ config=config, resolution=resolution,
+ )
+
+
[email protected]("backend", ["codex", "claude", "opencode"])
[email protected]("platform", ["avibe", "slack"])
[email protected]("language", ["en", "zh"])
+async def test_shared_failure_uses_exact_hub_copy_and_preserves_terminal_evidence(
+ tmp_path, backend, platform, language,
+):
+ projection = unavailable(tmp_path, backend)
+ project = Mock(return_value=projection)
+ controller = SimpleNamespace(
+ config=SimpleNamespace(language=language),
+ model_hub_turn_gateway=SimpleNamespace(
+ correlation=SimpleNamespace(terminal_projection=project),
+ ),
+ emit_agent_message=AsyncMock(return_value="msg-failed"),
+ )
+ context = MessageContext(
+ user_id="user", channel_id="channel", platform=platform,
+ platform_specific={"turn_token": "turn-exact"},
+ )
+ diagnostic = (
+ "Codex turn failed: unexpected status 503 Service Unavailable; "
+ "url: http://127.0.0.1:49800/codex/v1/responses"
+ )
+
+ await emit_backend_failure(controller, context, backend, diagnostic)
+
+ project.assert_called_once_with("turn-exact", backend=backend)
+ notify, terminal = controller.emit_agent_message.await_args_list
+ assert notify.args[1] == "notify"
+ assert notify.args[2] == render_turn_outcome_copy(projection, language)
+ assert "127.0.0.1" not in notify.args[2]
+ assert "2026-09-09" not in notify.args[2]
+ assert "Relay 服务" in notify.args[2]
+ assert notify.kwargs["output"].metadata["event"] == "backend_failure"
+ assert terminal.args[1:] == ("result", "")
+ assert terminal.kwargs["terminal_error"] == diagnostic
+ assert terminal.kwargs["level"] == "silent"
+
+
[email protected]("identity", [None, "", 123, "different-turn"])
+async def test_no_exact_projection_keeps_unrelated_native_failure(identity):
+ project = Mock(return_value=None)
+ controller = SimpleNamespace(
+ model_hub_turn_gateway=SimpleNamespace(
+ correlation=SimpleNamespace(terminal_projection=project),
+ ),
+ emit_agent_message=AsyncMock(),
+ )
+ context = MessageContext(
+ user_id="user", channel_id="channel", platform="avibe",
+ platform_specific={"turn_token": identity},
+ )
+ await emit_backend_failure(
+ controller, context, "codex", "native failed",
+ display_text="Native process exited unexpectedly",
+ )
+ assert controller.emit_agent_message.await_args_list[0].args[2] == (
+ "Native process exited unexpectedly"
+ )
+ if identity == "different-turn":
+ project.assert_called_once_with(identity, backend="codex")
+ else:
+ project.assert_not_called()
+
+
+async def test_historical_notice_does_not_consume_a_live_turn_projection(tmp_path):
+ project = Mock(return_value=unavailable(tmp_path))
+ controller = SimpleNamespace(
+ model_hub_turn_gateway=SimpleNamespace(
+ correlation=SimpleNamespace(terminal_projection=project),
+ ),
+ emit_agent_message=AsyncMock(),
+ )
+ context = MessageContext(
+ user_id="user", channel_id="channel", platform="avibe",
+ platform_specific={"turn_token": "new-live-turn"},
+ )
+ await emit_replayed_backend_failure(
+ controller, context, "codex", "old failure",
+ display_text="Original failure notice", failure_id="old-failure",
+ )
+ project.assert_not_called()
+ assert controller.emit_agent_message.await_count == 1
+ assert controller.emit_agent_message.await_args.args[2] == "Original failure notice"
+
+
[email protected]("backend,endpoint,status", [
+ ("codex", "responses", 400), ("claude", "messages", 424),
+])
[email protected]("language", ["en", "zh"])
+async def test_gateway_terminal_survives_native_recorder_before_shared_notice(
+ tmp_path, backend, endpoint, status, language,
+):
+ """MH-RETRY-COPY-001: actual HTTP terminal -> native recorder -> one clean notice."""
+ service, clock = clock_service(tmp_path, outcomes=[
+ _outcome(RawOutcomeKind.NETWORK_ERROR, source_id="src_recovery01")
+ for _ in range(20)
+ ])
+ service.store.load().sources[0].display_name = "Relay 服务"
+ models = _canonicalize_fixed_test_routes(service)
+ gateway = ModelHubTurnGateway(service)
+ router = ModelHubRuntimeRouter(service=service, turn_gateway=gateway)
+ context = MessageContext(
+ user_id="user", channel_id="channel", platform="avibe",
+ platform_specific={"turn_token": "turn-copy"},
+ )
+ controller = SimpleNamespace(
+ config=SimpleNamespace(language=language),
+ model_hub_turn_gateway=gateway,
+ emit_agent_message=AsyncMock(return_value="msg-failed"),
+ )
+ try:
+ launch = await router.resolve(
+ backend, models[backend], process_scope="fixture-copy", turn_id="turn-copy",
+ )
+ bind_launch(context, launch)
+ async with aiohttp.ClientSession(trust_env=False, timeout=aiohttp.ClientTimeout(total=5)) as client:
+ async with client.post(
+ f"{launch.gateway_base_url}/v1/{endpoint}",
+ headers={"Authorization": f"Bearer {launch.gateway_token}"},
+ json={"model": launch.runtime_model, "stream": False},
+ ) as response:
+ assert response.status == status
+ assert (await response.json())["error"]["code"] == "model_hub_recovery_exhausted"
+ assert "Retry-After" not in response.headers
+ assert clock.elapsed == 91
+ assert len(service.adapter.invocations) == 8
+ controller.emit_agent_message.assert_not_awaited()
+ projection = gateway.correlation.terminal_projection("turn-copy", backend=backend)
+ assert projection is not None
+ diagnostic = f"Native error {status}, URL http://127.0.0.1/model-request"
+ assert await router.record_native_failure(context, diagnostic) is False
+ assert gateway.correlation.terminal_projection("turn-copy", backend=backend) == projection
+ await emit_backend_failure(controller, context, backend, diagnostic)
+ notice, terminal = controller.emit_agent_message.await_args_list
+ assert notice.args[2] == render_turn_outcome_copy(projection, language)
+ assert "Relay 服务" in notice.args[2]
+ assert "127.0.0.1" not in notice.args[2]
+ assert ("已结束" if language == "zh" else "has ended") in notice.args[2]
+ assert terminal.kwargs["terminal_error"] == diagnostic
+ assert gateway.correlation.recovery_snapshot("turn-copy") == []
+ finally:
+ await gateway.close()
+
+
[email protected]("language", ["en", "zh"])
+def test_terminal_copy_never_promises_automatic_recovery(language):
+ """MH-RETRY-COPY-001: final failures never promise that a timer will recover a Source."""
+ params = {"model": "test", "source": "relay", "retry_at": "2026-09-09T04:00:00Z"}
+ for key in ("modelHub.launch.waiting", "modelHub.launch.waiting_without_retry"):
+ text = t(key, language, **params)
+ assert ("已结束" if language == "zh" else "has ended") in text
+ assert "2026-09-09" not in text
+ assert "自动恢复。" not in text
+ assert "has recovered" not in text
diff --git a/tests/test_model_hub_injection.py b/tests/test_model_hub_injection.py
index 54dd0284e..357f010de 100644
--- a/tests/test_model_hub_injection.py
+++ b/tests/test_model_hub_injection.py
@@ -130,6 +130,7 @@ def test_hub_launch_masks_inherited_claude_auth_and_injects_gateway():
"CLAUDE_CODE_OAUTH_TOKEN": "oauth-token",
"CLAUDE_CODE_MAX_CONTEXT_TOKENS": "999999",
"CLAUDE_CODE_MAX_OUTPUT_TOKENS": "999999",
+ "CLAUDE_CODE_MAX_RETRIES": "9",
"PATH": "/bin",
},
launch,
@@ -137,6 +138,7 @@ def test_hub_launch_masks_inherited_claude_auth_and_injects_gateway():
assert env["ANTHROPIC_BASE_URL"] == launch.gateway_base_url
assert env["ANTHROPIC_AUTH_TOKEN"] == launch.gateway_token
assert env["CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY"] == "1"
+ assert env["CLAUDE_CODE_MAX_RETRIES"] == "0"
assert env["CLAUDE_CODE_MAX_CONTEXT_TOKENS"] == "999999"
assert env["CLAUDE_CODE_MAX_OUTPUT_TOKENS"] == "999999"
assert env["CLAUDE_CODE_OAUTH_TOKEN"] == ""
@@ -169,6 +171,8 @@ def test_claude_hub_settings_own_connection_after_native_and_sdk_env_merges():
assert effective_env["PATH"] == "/bin"
assert settings["autoMemoryEnabled"] is False
assert settings["apiKeyHelper"] == ""
+ settings = json.loads(claude_settings_for_launch('{"env":{"CLAUDE_CODE_MAX_RETRIES":"9"}}', launch))
+ assert settings["env"]["CLAUDE_CODE_MAX_RETRIES"] == "0"
@pytest.mark.parametrize("channel", [None, "direct", "native_cli"])
@@ -307,6 +311,9 @@ def test_codex_hub_launch_uses_responses_wire_api_and_ephemeral_token(tmp_path):
assert 'wire_api="responses"' in rendered
assert "model_provider=\"avibe_model_hub\"" in rendered
assert f'model_catalog_json="{catalog_path}"' in rendered
+ assert "model_providers.avibe_model_hub.request_max_retries=0" in args
+ assert "features.unbounded_connection_retries=false" in args
+ assert not any("stream_max_retries" in arg for arg in args)
assert env == {"AVIBE_MODEL_HUB_TOKEN": "local-test-token"}
diff --git a/tests/test_model_hub_l3.py b/tests/test_model_hub_l3.py
index 1be5a8daf..b577b2a60 100644
--- a/tests/test_model_hub_l3.py
+++ b/tests/test_model_hub_l3.py
@@ -497,9 +497,15 @@ def _terminal_resolution_facts(
source_id=(source.id if source is not None else None),
model_id=model,
reason=blocker_reason,
+ temporary_blocker=(source is not None and source.state.status == "cooldown"),
+ retry_at=source.state.retry_at if source is not None else None,
+ backoff=False,
+ recovery=None,
+ cooldown=(source is not None and source.state.status == "cooldown"),
+ cooldown_reason=None,
),
)
- if blocker_reason is not None
+ if blocker_reason is not None or (source is not None and source.state.status == "cooldown")
else ()
)
candidates = (
@@ -1275,9 +1281,10 @@ def _service(
sources: list[ModelHubSourceConfig],
outcomes: list[RawCallOutcome] | None = None,
live_handles: list[LiveInvokeHandle | BlockingLiveInvokeHandle] | None = None,
+ recovery_window_seconds: float = 0,
) -> ModelHubService:
store = MemoryStore(_config(sources))
- return ModelHubService(
+ service = ModelHubService(
store=store,
adapter=ProbeAdapter(outcomes or [], live_handles),
events=BoundedEventLog(tmp_path / "events.json"),
@@ -1287,6 +1294,12 @@ def _service(
now=lambda: NOW,
requested_model_override=store.requested_model,
)
+ # This fixture characterizes a single resolver walk. Recovery behavior opts
+ # into its own clocked policy; ordinary production requests default to 120s.
+ service.recovery.window_seconds = recovery_window_seconds
+ service.recovery.monotonic = lambda: (service.now() - NOW).total_seconds()
+ service.recovery.jitter = lambda low, high: low
+ return service
def _canonicalize_fixed_test_routes(
@@ -2961,7 +2974,7 @@ def test_gateway_preserves_exhausted_provenance_after_all_hops_fallback(
("opencode", "chat/completions", "openai_chat"),
],
)
-def test_gateway_cooldown_is_one_cancellable_retry(
+def test_gateway_cooldown_uses_one_cancellable_recovery_window(
tmp_path: Path,
already_cooling: bool,
ending: str,
@@ -2969,7 +2982,7 @@ def test_gateway_cooldown_is_one_cancellable_retry(
endpoint: str,
protocol: str,
) -> None:
- """MH-RUNTIME-009: known recovery, not caller retry speed, owns cooldown waiting."""
+ """MH-RUNTIME-009: one pending request owns all bounded recovery admission."""
async def exercise() -> None:
source = _source(
@@ -2985,7 +2998,8 @@ def test_gateway_cooldown_is_one_cancellable_retry(
service = _service(
tmp_path,
sources=[source],
- outcomes=([failure] if not already_cooling else []) + [final],
+ outcomes=([failure] if not already_cooling else []) + ([final] * 4 if ending == "fail" else [final]),
+ recovery_window_seconds=120,
)
fixed = _canonicalize_fixed_test_routes(service)
model = fixed[backend] if backend in fixed else "shared-model"
@@ -3031,19 +3045,8 @@ def test_gateway_cooldown_is_one_cancellable_retry(
source_id=source.id,
)
try:
- with patch("core.handlers.model_hub.turn_gateway.asyncio.sleep", side_effect=wait_for_recovery):
+ with patch.object(service.recovery, "sleep", side_effect=wait_for_recovery):
async with aiohttp.ClientSession(trust_env=False) as client:
- if not already_cooling:
- first = await client.post(
- f"{base_url}/v1/{endpoint}",
- json=payload,
- headers={"Authorization": f"Bearer {token}"},
- )
- await first.read()
- assert first.status == 503
- assert first.headers["Retry-After"] == "30"
- if ending == "admitted_cancel":
- service.adapter.invoke = invoke_until_stopped
request = asyncio.create_task(client.post(
f"{base_url}/v1/{endpoint}",
json=payload,
@@ -3059,6 +3062,7 @@ def test_gateway_cooldown_is_one_cancellable_retry(
await request
await asyncio.wait_for(wait_ended.wait(), timeout=2)
elif ending == "admitted_cancel":
+ service.adapter.invoke = invoke_until_stopped
release.set()
await asyncio.wait_for(admitted.wait(), timeout=2)
gateway.correlation.close_turn_admission(
@@ -3082,25 +3086,26 @@ def test_gateway_cooldown_is_one_cancellable_retry(
response = await asyncio.wait_for(request, timeout=2)
await response.read()
assert response.status == {
- "recover": 200, "fail": 503, "config_change": 409, "extend_recover": 503,
+ "recover": 200, "fail": 400 if backend == "codex" else 424,
+ "config_change": 409, "extend_recover": 200,
}[ending]
- if ending in {"fail", "extend_recover"}:
- assert response.headers["Retry-After"] == "30"
- else:
- assert "Retry-After" not in response.headers
- if ending == "extend_recover":
- clock["now"] += timedelta(seconds=30)
- recovered = await client.post(
- f"{base_url}/v1/{endpoint}",
- json=payload,
- headers={"Authorization": f"Bearer {token}"},
- )
- await recovered.read()
- assert recovered.status == 200
- assert delays == [30.0]
+ if ending == "fail":
+ assert await response.json() == {
+ "type": "error",
+ "error": {
+ "type": "model_hub_recovery_exhausted",
+ "code": "model_hub_recovery_exhausted",
+ "message": "Automatic recovery has ended. Try again or choose another model.",
+ },
+ }
+ assert "Retry-After" not in response.headers
+ assert delays == (
+ [30.0, 30.0 if already_cooling else 60.0] if ending == "fail"
+ else [30.0, 30.0] if ending == "extend_recover" else [30.0]
+ )
expected = (0 if already_cooling else 1) + (
ending in {"recover", "fail", "extend_recover", "admitted_cancel"}
- )
+ ) + (ending == "fail")
assert len(service.adapter.invocations) == expected
finally:
if not request.done():
@@ -3171,10 +3176,11 @@ def test_cooldown_selection_uses_instants_through_probe_and_gateway(tmp_path, re
with pytest.raises(ModelHubError) as probe:
await service.probe_agent("codex", model)
assert probe.value.code == "probe_no_candidate"
- assert probe.value.data["supply"]["retry_at"] == early
+ expected_retry_at = (NOW + timedelta(seconds=delay)).isoformat()
+ assert probe.value.data["supply"]["retry_at"] == expected_retry_at
with pytest.raises(ModelHubError) as failed:
await service.resolve(backend="codex", model_id=model, request={}, supply_channel="hub")
- assert failed.value.turn_outcome.supply_facts.retry_at == early
+ assert failed.value.turn_outcome.supply_facts.retry_at == expected_retry_at
gateway = ModelHubTurnGateway(service, now=lambda: clock["now"])
assert gateway._cooldown_retry_delay(failed.value.turn_outcome) == delay
@@ -3189,7 +3195,8 @@ def test_cooldown_selection_uses_instants_through_probe_and_gateway(tmp_path, re
requested_model_id=model, resolved_model_id="shared-model", source_id=first.id,
)
try:
- with patch("core.handlers.model_hub.turn_gateway.asyncio.sleep", side_effect=wait_for_recovery):
+ service.recovery.window_seconds = 120
+ with patch.object(service.recovery, "sleep", side_effect=wait_for_recovery):
async with aiohttp.ClientSession(trust_env=False) as client:
response = await client.post(
f"{base_url}/v1/responses",
@@ -9083,10 +9090,6 @@ def test_chain_projection_and_probe_latency_partition(tmp_path: Path) -> None:
_assert_valid("probe-result.schema.json", not_found)
- strict=True,
- reason="AC-50: cooldown.network → backoff.connection_failed 契约先行,实现随 I7 落地",
-)
def test_probe_transport_failures_await_ac50_backoff_contract(tmp_path: Path) -> None:
for name, kind in (
("network", RawOutcomeKind.NETWORK_ERROR),
@@ -9566,7 +9569,7 @@ def test_shared_source_cooldown_emits_only_on_state_transition(
)
return first_persisted, repeated_persisted
- assert asyncio.run(cool_twice()) == (True, False)
+ assert asyncio.run(cool_twice()) == (True, True)
events = service.list_events(limit=20)
assert len(events) == 1
diff --git a/tests/test_model_hub_progress.py b/tests/test_model_hub_progress.py
new file mode 100644
index 000000000..b9e50252a
--- /dev/null
+++ b/tests/test_model_hub_progress.py
@@ -0,0 +1,220 @@
+"""Live recovery changes status only; it never emits a chat notification."""
+
+import asyncio
+from contextlib import nullcontext
+from datetime import datetime, timedelta, timezone
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, Mock, patch
+
+import aiohttp
+import pytest
+
+from core.handlers.model_hub.adapter import RawOutcomeKind
+from core.handlers.model_hub.turn_gateway import ModelHubTurnGateway
+from core.model_hub_progress import (
+ publish_recovery_changed,
+ recovery_snapshot,
+ recovery_status_text,
+)
+from core.session_turns import SessionTurnManager, Turn
+from tests.test_model_hub_l3 import _outcome
+from tests.test_model_hub_retry_policy import clock_service
+
+
+NOW = datetime(2026, 9, 9, 4, 30, tzinfo=timezone.utc)
+
+
+def snapshot(**overrides):
+ return {
+ "request_id": "request-exact",
+ "phase": "waiting",
+ "attempt_count": 1,
+ "started_at": (NOW - timedelta(seconds=5)).isoformat(),
+ "window_end": (NOW + timedelta(seconds=115)).isoformat(),
+ "source_id": "src_unprinted",
+ "reason": "server_error",
+ "next_eligible_at": (NOW + timedelta(seconds=25)).isoformat(),
+ **overrides,
+ }
+
+
+def controller_for(rows):
+ return SimpleNamespace(
+ model_hub_turn_gateway=SimpleNamespace(
+ correlation=SimpleNamespace(recovery_snapshot=Mock(return_value=rows)),
+ ),
+ )
+
+
[email protected]("language", ["en", "zh"])
+def test_progress_uses_relative_time_after_debounce(language):
+ controller = controller_for([snapshot()])
+ text = recovery_status_text(controller, "turn-exact", language, now=NOW)
+ assert "25" in text
+ assert "src_unprinted" not in text and "2026" not in text
+ assert recovery_status_text(
+ controller, "turn-exact", language, now=NOW - timedelta(milliseconds=1),
+ ) is None
+ # Reload reads the original start rather than starting a new debounce.
+ assert recovery_status_text(
+ controller, "turn-exact", language, now=NOW + timedelta(seconds=10),
+ ) is not None
+
+
[email protected]("deadline", [None, "invalid", (NOW - timedelta(seconds=1)).isoformat()])
+def test_deadline_expiry_is_not_recovery(deadline):
+ controller = controller_for([snapshot(next_eligible_at=deadline)])
+ text = recovery_status_text(controller, "turn-exact", "en", now=NOW)
+ assert text == "Waiting for the next model attempt…"
+ assert "recovered" not in text
+
+
+def test_attempting_and_cleared_progress():
+ controller = controller_for([snapshot(phase="attempting", next_eligible_at=None)])
+ assert recovery_status_text(controller, "turn-exact", "en", now=NOW) == (
+ "Retrying the model request…"
+ )
+ controller.model_hub_turn_gateway.correlation.recovery_snapshot.return_value = []
+ assert recovery_status_text(controller, "turn-exact", "en", now=NOW) is None
+
+
+ snapshot(started_at="invalid"),
+ snapshot(started_at=None),
+ snapshot(started_at=(NOW + timedelta(seconds=30)).isoformat()),
+ snapshot(phase="unknown"),
+])
+def test_unusable_progress_keeps_normal_working_indicator(row):
+ assert recovery_status_text(controller_for([row]), "turn-exact", "en", now=NOW) is None
+
+
+def test_no_turn_or_gateway_has_no_snapshot():
+ controller = controller_for([snapshot()])
+ assert recovery_snapshot(controller, None) == []
+ controller.model_hub_turn_gateway.correlation.recovery_snapshot.assert_not_called()
+ assert recovery_snapshot(SimpleNamespace(), "turn-exact") == []
+
+
+def test_only_the_exact_live_session_gets_read_invalidation():
+ """MH-RETRY-PROGRESS-001: material recovery updates invalidate only their live Session."""
+ def entry(turn_id, done=False):
+ return SimpleNamespace(
+ context=SimpleNamespace(platform_specific={"turn_token": turn_id}),
+ task=SimpleNamespace(done=lambda: done),
+ )
+
+ controller = SimpleNamespace(session_turns=SimpleNamespace(in_flight={
+ "ses-live": entry("turn-exact"),
+ "ses-peer": entry("turn-peer"),
+ "ses-old": entry("turn-exact", done=True),
+ }))
+ with patch("core.inbox_events.bus.publish") as publish:
+ publish_recovery_changed(controller, "turn-exact")
+ publish.assert_called_once_with(
+ "session.activity", {"session_id": "ses-live", "event": "model_recovery"},
+ )
+
+
+def test_turn_state_reads_recovery_only_for_its_current_live_owner():
+ rows = [snapshot()]
+ controller = controller_for(rows)
+ context = SimpleNamespace(platform_specific={
+ "turn_token": "turn-exact",
+ "agent_session_target": {"agent_backend": "codex"},
+ })
+ manager = SessionTurnManager(controller=controller)
+ manager._engine = SimpleNamespace(begin=lambda: nullcontext(object()))
+ manager.in_flight["ses-live"] = Turn(
+ task=SimpleNamespace(done=lambda: False), context=context,
+ )
+ with patch("core.session_turns.delivery_store.list_queued", return_value=[]):
+ state = manager.turn_state("ses-live")
+ assert state["in_flight"] is True
+ assert state["model_recovery"] == rows
+ reader = controller.model_hub_turn_gateway.correlation.recovery_snapshot
+ reader.assert_called_once_with("turn-exact")
+
+ manager.in_flight.clear()
+ reader.reset_mock()
+ with patch("core.session_turns.delivery_store.list_queued", return_value=[]):
+ state = manager.turn_state("ses-live")
+ assert "model_recovery" not in state
+ reader.assert_not_called()
+
+
[email protected]("ending", ["success", "stop"])
+async def test_http_recovery_reaches_live_turn_state_and_clears_without_chat(tmp_path, ending):
+ """MH-RETRY-PROGRESS-001: gateway producer -> registry callback -> Session read."""
+ service, clock = clock_service(tmp_path, outcomes=[
+ _outcome(RawOutcomeKind.HTTP_ERROR, source_id="src_recovery01", status=503),
+ _outcome(RawOutcomeKind.SUCCESS, source_id="src_recovery01"),
+ ])
+ waiting, release = asyncio.Event(), asyncio.Event()
+
+ async def controlled_wait(delay):
+ clock.advance(5)
+ waiting.set()
+ await release.wait()
+ clock.advance(delay - 5)
+
+ service.recovery.sleep = controlled_wait
+ gateway = ModelHubTurnGateway(service)
+ controller = SimpleNamespace(model_hub_turn_gateway=gateway, emit_agent_message=AsyncMock())
+ manager = SessionTurnManager(controller=controller)
+ controller.session_turns = manager
+ manager._engine = SimpleNamespace(begin=lambda: nullcontext(object()))
+ context = SimpleNamespace(platform_specific={
+ "turn_token": "turn-progress", "agent_session_target": {"agent_backend": "codex"},
+ })
+ gateway.correlation.on_recovery_changed = lambda turn: publish_recovery_changed(controller, turn)
+ base, token = await gateway.endpoint(
+ "codex", process_scope="fixture-progress", turn_id="turn-progress",
+ requested_model_id="shared-model", resolved_model_id="shared-model", source_id="src_recovery01",
+ )
+ request_task = None
+ try:
+ with (
+ patch("core.inbox_events.bus.publish") as publish,
+ patch("core.session_turns.delivery_store.list_queued", return_value=[]),
+ ):
+ async with aiohttp.ClientSession(trust_env=False, timeout=aiohttp.ClientTimeout(total=5)) as client:
+ async def post():
+ async with client.post(
+ f"{base}/v1/responses", headers={"Authorization": f"Bearer {token}"},
+ json={"model": "shared-model", "stream": False},
+ ) as response:
+ await response.read()
+ return response.status
+
+ request_task = asyncio.create_task(post())
+ manager.in_flight["ses-live"] = Turn(task=request_task, context=context)
+ await asyncio.wait_for(waiting.wait(), 2)
+ row, = manager.turn_state("ses-live")["model_recovery"]
+ assert row["phase"] == "waiting" and row["attempt_count"] == 1
+ assert row["request_id"] and row["source_id"] == "src_recovery01"
+ assert recovery_status_text(controller, "turn-progress", "en", now=clock.now()) == (
+ "Waiting to retry the model request in about 25s"
+ )
+ assert not request_task.done() and len(service.adapter.invocations) == 1
+ if ending == "success":
+ release.set()
+ assert await asyncio.wait_for(request_task, 2) == 200
+ else:
+ drain = gateway.finalize_turn("turn-progress", settled_by="stopped", finish=lambda: None)
+ assert drain is not None
+ await asyncio.wait_for(drain, 2)
+ assert len(service.adapter.invocations) == 1
+ await asyncio.gather(request_task, return_exceptions=True)
+ assert gateway.correlation.recovery_snapshot("turn-progress") == []
+ assert len(publish.call_args_list) >= 2
+ for call in publish.call_args_list:
+ assert call.args == (
+ "session.activity", {"session_id": "ses-live", "event": "model_recovery"},
+ )
+ controller.emit_agent_message.assert_not_awaited()
+ finally:
+ if request_task is not None:
+ if not request_task.done():
+ request_task.cancel()
+ await asyncio.gather(request_task, return_exceptions=True)
+ await gateway.close()
diff --git a/tests/test_model_hub_provenance.py b/tests/test_model_hub_provenance.py
index 799063ec5..36918577e 100644
--- a/tests/test_model_hub_provenance.py
+++ b/tests/test_model_hub_provenance.py
@@ -2,22 +2,197 @@ from __future__ import annotations
import asyncio
import json
+from datetime import timedelta
from pathlib import Path
+from types import SimpleNamespace
import pytest
from core.handlers.model_hub.adapter import RawCallOutcome, RawOutcomeKind
-from core.handlers.model_hub.classification import UPSTREAM_MACHINE_ERROR_CODES, classify_outcome
-from core.handlers.model_hub.provenance import BoundedProvenanceStore, TurnCorrelationRegistry
+from core.handlers.model_hub.classification import UPSTREAM_MACHINE_ERROR_CODES, ResolutionDecision, classify_outcome
+from core.handlers.model_hub.provenance import (
+ BoundedProvenanceStore, TurnCorrelationRegistry, produce_turn_outcome, render_turn_outcome_copy,
+)
from core.handlers.model_hub.rpc import dispatch_model_hub_rpc
from core.handlers.model_hub.service import ModelHubError
-from core.run_settlement import SETTLED_BY_TERMINAL_RESULT
-from tests.test_model_hub_resolution import _service
+from core.run_settlement import SETTLED_BY_TERMINAL_RESULT, SETTLED_BY_STOPPED
+from modules.agents.model_hub import ModelHubLaunch, ModelHubRuntimeRouter, bind_launch
+from tests.test_model_hub_resolution import _config, _service, _source
from tests.test_model_hub_routing_modes import MODEL, _loaded_catalog_config, _sparse_config
from tests.ui_server_test_helpers import csrf_headers
from vibe import model_hub_client, ui_server
+def _live_registry(tmp_path, callback=None):
+ registry = TurnCorrelationRegistry(
+ BoundedProvenanceStore(tmp_path / "records.json"), on_recovery_changed=callback,
+ )
+ token = registry.credentials("codex", "fixture", "turn-live")
+ registry.begin_gateway_request(backend="codex", token=token, requested_model_id="shared-model")
+ return registry
+
+
+ "valid", "wrong_backend", "missing", "pending", "ambiguous", "poisoned",
+ "closed", "stopped", "served", "canceled", "settled",
+])
+def test_live_terminal_projection_requires_exact_unfrozen_owner(tmp_path, guard):
+ registry = _live_registry(tmp_path)
+ projection = produce_turn_outcome("turn.engine_down")
+ registry.record_turn_outcome("turn-live", projection)
+ if guard == "pending":
+ registry.begin_attempt(
+ "turn-live", source_id="src_primary01", resolved_model_id="shared-model",
+ channel="hub", via_mapping=False,
+ )
+ elif guard == "ambiguous":
+ registry._traces["turn-live"].ambiguous = True
+ elif guard == "poisoned":
+ registry._scopes[("codex", "fixture")].untracked_use = True
+ elif guard in {"closed", "stopped"}:
+ registry.close_turn_admission(
+ "turn-live", settled_by=SETTLED_BY_STOPPED if guard == "stopped" else SETTLED_BY_TERMINAL_RESULT,
+ )
+ elif guard in {"served", "canceled"}:
+ registry.record_turn_outcome("turn-live", produce_turn_outcome(f"turn.{guard}"))
+ elif guard == "settled":
+ registry.settle("turn-live", settled_by=SETTLED_BY_TERMINAL_RESULT)
+ assert registry.terminal_projection(
+ "missing" if guard == "missing" else "turn-live",
+ backend="claude" if guard == "wrong_backend" else "codex",
+ ) == (projection if guard in {"valid", "closed"} else None)
+
+
+def test_recovery_callbacks_are_material_post_lock_and_snapshots_are_copies(tmp_path):
+ calls = []
+ registry = _live_registry(tmp_path)
+
+ def changed(turn_id):
+ assert not registry._lock._is_owned()
+ calls.append(registry.recovery_snapshot(turn_id))
+
+ registry.on_recovery_changed = changed
+ snapshot = {
+ "phase": "waiting", "attempt_count": 1, "source_id": "src_primary01",
+ "reason": "network", "started_at": "2026-09-09T00:00:00+00:00",
+ "next_eligible_at": None, "window_end": "2026-09-09T00:02:00+00:00",
+ }
+ registry.update_recovery("turn-live", backend="codex", request_id="one", snapshot=snapshot)
+ registry.update_recovery("turn-live", backend="codex", request_id="one", snapshot=dict(snapshot))
+ registry.update_recovery("turn-live", backend="claude", request_id="wrong", snapshot=snapshot)
+ assert len(calls) == 1
+ external = registry.recovery_snapshot("turn-live")
+ external[0]["phase"] = "corrupted"
+ assert registry.recovery_snapshot("turn-live")[0]["phase"] == "waiting"
+ registry.update_recovery("turn-live", backend="codex", request_id="two", snapshot=snapshot)
+ registry.update_recovery("turn-live", backend="codex", request_id="one", snapshot=None)
+ assert [item["request_id"] for item in registry.recovery_snapshot("turn-live")] == ["two"]
+ registry.update_recovery("turn-live", backend="codex", request_id="two", snapshot=None)
+ registry.update_recovery("turn-live", backend="codex", request_id="two", snapshot=None)
+ assert len(calls) == 4 and calls[-1] == []
+ assert not registry.store.path.exists()
+
+
+def test_native_hub_failure_keeps_the_gateway_terminal_projection_readable(tmp_path):
+ registry = _live_registry(tmp_path)
+ projection = produce_turn_outcome("turn.engine_down")
+ registry.record_turn_outcome("turn-live", projection)
+ registry.fail_hub_attempt("turn-live")
+ assert registry.terminal_projection("turn-live", backend="codex") == projection
+ assert not registry._traces["turn-live"].outcome_frozen
+ registry.close_turn_admission("turn-live", settled_by=SETTLED_BY_STOPPED)
+ registry.fail_hub_attempt("turn-live")
+ assert registry.terminal_projection("turn-live", backend="codex") is None
+
+
[email protected]("terminal", ["exhausted", "no_candidate"])
+def test_native_failure_callback_consumes_exact_hub_terminal_copy(tmp_path, terminal):
+ async def run():
+ source = _source("src_callback01", (MODEL,), status="cooldown")
+ service, store, _ = _service(tmp_path, _config([source], model=MODEL))
+ source.state.retry_at = (service.now() + timedelta(seconds=300)).isoformat()
+ registry = TurnCorrelationRegistry(service.provenance)
+ token = registry.credentials("claude", "fixture", "turn-callback")
+ registry.begin_gateway_request(backend="claude", token=token, requested_model_id=MODEL)
+ config, resolution = service._inspect_terminal_chain(backend="claude", model_id=MODEL)
+ projection = produce_turn_outcome(f"turn.{terminal}" if terminal == "exhausted" else "turn.no_candidate.blocked",
+ config=config, resolution=resolution)
+ registry.record_turn_outcome("turn-callback", projection)
+ expected_text = render_turn_outcome_copy(projection, "en")
+ before = store.config.to_payload()
+ router = ModelHubRuntimeRouter(
+ service=service, turn_gateway=SimpleNamespace(correlation=registry),
+ overlay_path=tmp_path / "overlay.json",
+ )
+ context = SimpleNamespace(platform_specific={"turn_token": "turn-callback"})
+ bind_launch(context, ModelHubLaunch(
+ backend="claude", channel="hub", requested_model=MODEL,
+ target_model=MODEL, runtime_model=MODEL, source_id=source.id,
+ ))
+ assert await router.record_native_failure(context, "HTTP 424 opaque native exception") is False
+ authoritative = registry.terminal_projection("turn-callback", backend="claude")
+ assert authoritative == projection
+ assert render_turn_outcome_copy(authoritative, "en") == expected_text
+ assert expected_text and "opaque" not in expected_text and "424" not in expected_text
+ assert store.config.to_payload() == before
+ assert registry.terminal_projection("turn-callback", backend="codex") is None
+ registry.close_turn_admission("turn-callback", settled_by=SETTLED_BY_STOPPED)
+ assert registry.terminal_projection("turn-callback", backend="claude") is None
+ asyncio.run(run())
+
+
[email protected]("status", [100, 503, 599, None, True, 99, 600])
+def test_failed_attempt_http_status_is_additive_strict_and_history_safe(tmp_path, status):
+ import jsonschema
+
+ registry = _live_registry(tmp_path)
+ registry.begin_attempt(
+ "turn-live", source_id="src_primary01", resolved_model_id=MODEL, channel="hub", via_mapping=False,
+ )
+ registry.finish_attempt(
+ "turn-live",
+ outcome=RawCallOutcome(
+ kind=RawOutcomeKind.HTTP_ERROR, http_status=status,
+ error_code=None, redacted_message="not retained", stream_started=False,
+ model_id=MODEL, source_id="src_primary01",
+ ),
+ decision=ResolutionDecision("fallback", reason="server_error", cooldown_seconds=30),
+ )
+ registry.settle("turn-live", settled_by=SETTLED_BY_TERMINAL_RESULT)
+ record = registry.store.get("turn-live")
+ attempt = record["failed_attempts"][0]
+ expected = {"source_id": "src_primary01", "configured_model_id": MODEL, "channel": "hub", "reason": "server_error"}
+ if type(status) is int and 100 <= status <= 599:
+ expected["http_status"] = status
+ assert attempt == expected
+ schema = json.loads(
+ (Path(__file__).parents[1] / "docs/plans/model-hub-contracts/turn-provenance.schema.json").read_text()
+ )
+ validator = jsonschema.Draft7Validator(schema)
+ assert not list(validator.iter_errors(record))
+ for invalid in [True, None, 99, 600, "503"]:
+ attempt["http_status"] = invalid
+ assert list(validator.iter_errors(record))
+
+
[email protected]("guard", ["ambiguous", "poisoned", "closed", "stopped", "settled"])
+def test_live_recovery_snapshots_never_revive_invalid_or_terminal_owners(tmp_path, guard):
+ registry = _live_registry(tmp_path)
+ registry.update_recovery("turn-live", backend="codex", request_id="one", snapshot={"phase": "waiting"})
+ if guard == "ambiguous":
+ registry._traces["turn-live"].ambiguous = True
+ elif guard == "poisoned":
+ registry._scopes[("codex", "fixture")].untracked_use = True
+ elif guard == "settled":
+ registry.settle("turn-live", settled_by=SETTLED_BY_TERMINAL_RESULT)
+ else:
+ registry.close_turn_admission(
+ "turn-live", settled_by=SETTLED_BY_STOPPED if guard == "stopped" else SETTLED_BY_TERMINAL_RESULT,
+ )
+ registry.update_recovery("turn-live", backend="codex", request_id="one", snapshot={"phase": "attempting"})
+ assert registry.recovery_snapshot("turn-live") == []
+
+
def _record(turn_id, *, backend="claude", model=MODEL, outcome="failed_terminal"):
identity = {"source_id": "src_deleted01", "configured_model_id": "unknown-model", "channel": "hub"}
return {
diff --git a/tests/test_model_hub_recovery_evidence.py b/tests/test_model_hub_recovery_evidence.py
new file mode 100644
index 000000000..258be5689
--- /dev/null
+++ b/tests/test_model_hub_recovery_evidence.py
@@ -0,0 +1,340 @@
+"""Protocol recovery evidence is stricter than permissive HTTP forwarding."""
+
+from __future__ import annotations
+
+import asyncio
+import io
+import json
+from dataclasses import replace
+
+from aiohttp import web
+import pytest
+
+from core.handlers.model_hub.adapter import RawCallOutcome, RawOutcomeKind
+from core.handlers.model_hub.stream_wire import (
+ ProtocolFactProjector,
+ ProtocolObservation,
+ ProtocolSSEState,
+ observe_buffered_protocol_response,
+ observe_protocol_response,
+)
+from tests.test_model_hub_retry_advice import WIRE, _engine, _invoke, _source
+from vibe.model_hub_runtime import client as client_module
+
+
+BUFFERED = {
+ "anthropic": (
+ {"type": "message", "role": "assistant"},
+ "content",
+ [{"type": "text", "text": "已恢复"}],
+ ),
+ "openai_responses": (
+ {"object": "response", "status": "completed"},
+ "output",
+ [{"type": "message", "content": [{"type": "output_text", "text": "已恢复"}]}],
+ ),
+ "openai_chat": (
+ {"object": "chat.completion"},
+ "choices",
+ [{"index": 0, "message": {"role": "assistant", "content": "已恢复"}, "finish_reason": "stop"}],
+ ),
+}
+METADATA = {
+ "anthropic": b'event: message_start\ndata: {"type":"message_start","message":{"role":"assistant"}}\n\n',
+ "openai_responses": b'event: response.created\ndata: {"type":"response.created"}\n\n',
+ "openai_chat": b'data: {"choices":[{"delta":{"role":"assistant"}}]}\n\n',
+}
+TERMINAL = {
+ "anthropic": b'event: message_stop\ndata: {"type":"message_stop"}\n\n',
+ "openai_responses": b'event: response.completed\ndata: {"type":"response.completed"}\n\n',
+ "openai_chat": b"data: [DONE]\n\n",
+}
+REASONING = {
+ "anthropic": (
+ b'event: content_block_delta\ndata: {"type":"content_block_delta",'
+ b'"delta":{"type":"thinking_delta","thinking":"reasoning"}}\n\n'
+ ),
+ "openai_responses": (
+ b'event: response.reasoning_text.delta\ndata: {"type":"response.reasoning_text.delta","delta":"reasoning"}\n\n'
+ ),
+ "openai_chat": b'data: {"choices":[{"delta":{"reasoning_content":"reasoning"}}]}\n\n',
+}
+TOOL_OUTPUT = {
+ "anthropic": (
+ b'event: content_block_start\ndata: {"type":"content_block_start",'
+ b'"content_block":{"type":"tool_use","name":"example","input":{}}}\n\n'
+ ),
+ "openai_responses": (
+ b'event: response.function_call_arguments.delta\n'
+ b'data: {"type":"response.function_call_arguments.delta","delta":"{}"}\n\n'
+ ),
+ "openai_chat": b'data: {"choices":[{"delta":{"tool_calls":[{"id":"tool-call"}]}}]}\n\n',
+}
+
+
+def _body(protocol: str, *, empty: bool = False) -> bytes:
+ selectors, array_path, output = BUFFERED[protocol]
+ return json.dumps({**selectors, array_path: [] if empty else output}, ensure_ascii=False).encode()
+
+
+def test_recovery_evidence_is_additive_and_participates_in_equality() -> None:
+ outcome = RawCallOutcome(RawOutcomeKind.SUCCESS, 200, None, None, False, "model-a", "source-a")
+ observation = ProtocolObservation(outcome="served")
+ assert outcome.recovery_verified is False
+ assert observation.recovery_verified is False
+ assert replace(outcome, recovery_verified=True) != outcome
+ assert replace(observation, recovery_verified=True) != observation
+
+
[email protected]("protocol", BUFFERED)
[email protected]("malformed", [False, True], ids=["valid", "nonstandard-json"])
+def test_stream_evidence_agrees_between_legacy_and_incremental_observation(protocol: str, malformed: bool) -> None:
+ frame = REASONING[protocol]
+ event = next(
+ (line.removeprefix(b"event: ").decode() for line in frame.splitlines() if line.startswith(b"event: ")),
+ None,
+ )
+ payload = next(line.removeprefix(b"data: ") for line in frame.splitlines() if line.startswith(b"data: "))
+ if malformed:
+ payload = payload[:-1] + b',"unknown":NaN}'
+ legacy = observe_protocol_response(protocol, streamed=True, data=payload, event_name=event)
+ projector = ProtocolFactProjector(protocol)
+ projector.feed(payload)
+ projected = projector.finish(streamed=True, event_name=event)
+ assert legacy.recovery_verified is (not malformed)
+ assert projected.recovery_verified is legacy.recovery_verified
+
+
[email protected]("protocol", BUFFERED)
[email protected]("empty", [False, True], ids=["nonempty", "empty"])
+async def test_buffered_native_completion_proves_recovery_without_changing_bytes(
+ protocol: str, empty: bool,
+) -> None:
+ payload = _body(protocol, empty=empty)
+
+ async def respond(_request: web.Request) -> web.Response:
+ return web.Response(body=payload, content_type="application/json")
+
+ async with _engine(respond) as (client, requests):
+ outcome, forwarded = await _invoke(client, protocol, False)
+
+ assert outcome.kind is RawOutcomeKind.SUCCESS
+ assert outcome.recovery_verified is True
+ assert outcome.retry_after is None
+ assert outcome.response_received_at is None
+ assert forwarded == payload
+ assert len(requests) == 1
+ assert observe_buffered_protocol_response(protocol, io.BytesIO(payload)).recovery_verified is True
+ assert observe_protocol_response(protocol, streamed=False, data=payload).recovery_verified is True
+
+
[email protected]("protocol", BUFFERED)
+ "payload",
+ [
+ pytest.param(b"invalid", id="malformed"),
+ pytest.param(b'{"id":', id="truncated-json"),
+ pytest.param(b"[]", id="array"),
+ pytest.param(b"null", id="null"),
+ pytest.param(b"42", id="number"),
+ pytest.param(b'"ok"', id="string"),
+ pytest.param(b"{}", id="empty-object"),
+ pytest.param(b'{"id":"response"}', id="unknown-object"),
+ pytest.param(b'{"status":"ok","healthy":true}', id="health-object"),
+ ],
+)
+async def test_unknown_buffered_success_is_forwarded_without_recovery_evidence(protocol: str, payload: bytes) -> None:
+ """MH-RETRY-EVIDENCE-001: permissive forwarding cannot prove Source recovery."""
+ async def respond(_request: web.Request) -> web.Response:
+ return web.Response(body=payload, content_type="application/json")
+
+ async with _engine(respond) as (client, requests):
+ outcome, forwarded = await _invoke(client, protocol, False)
+
+ assert outcome.kind is RawOutcomeKind.SUCCESS
+ assert outcome.http_status == 200
+ assert outcome.recovery_verified is False
+ assert forwarded == payload
+ assert len(requests) == 1
+ assert observe_buffered_protocol_response(protocol, io.BytesIO(payload)).recovery_verified is False
+ assert observe_protocol_response(protocol, streamed=False, data=payload).recovery_verified is False
+
+
[email protected]("protocol", BUFFERED)
+ "mutation",
+ ["missing-array", "wrong-array-type", "missing-discriminator", "nested-result", "truncated", "nan", "duplicate-array"],
+)
+async def test_buffered_evidence_requires_the_recognized_shape_and_complete_json(protocol: str, mutation: str) -> None:
+ selectors, array_path, _output = BUFFERED[protocol]
+ document = json.loads(_body(protocol, empty=True))
+ if mutation == "missing-array":
+ document.pop(array_path)
+ elif mutation == "wrong-array-type":
+ document[array_path] = {}
+ elif mutation == "missing-discriminator":
+ document.pop(next(iter(selectors)))
+ elif mutation == "nested-result":
+ document = {"result": document}
+ payload = json.dumps(document).encode()
+ if mutation == "truncated":
+ payload = payload[:-1]
+ elif mutation == "nan":
+ payload = payload[:-1] + b',"unknown":NaN}'
+ elif mutation == "duplicate-array":
+ payload = payload[:-1] + b',"' + array_path.encode() + b'":null}'
+
+ async def respond(_request: web.Request) -> web.Response:
+ return web.Response(body=payload, content_type="application/json")
+
+ async with _engine(respond) as (client, _requests):
+ outcome, forwarded = await _invoke(client, protocol, False)
+
+ assert outcome.kind is RawOutcomeKind.SUCCESS
+ assert outcome.recovery_verified is False
+ assert forwarded == payload
+ assert observe_buffered_protocol_response(protocol, io.BytesIO(payload)).recovery_verified is False
+ assert observe_protocol_response(protocol, streamed=False, data=payload).recovery_verified is False
+
+
[email protected]("status", ["completed", "incomplete", "queued", "in_progress", "failed", "cancelled"])
+async def test_buffered_responses_need_a_completed_status_for_recovery(status: str) -> None:
+ payload = json.dumps({"object": "response", "status": status, "output": [], "error": None}).encode()
+
+ async def respond(_request: web.Request) -> web.Response:
+ return web.Response(body=payload, content_type="application/json")
+
+ async with _engine(respond) as (client, _requests):
+ outcome, forwarded = await _invoke(client, "openai_responses", False)
+
+ assert outcome.kind is RawOutcomeKind.SUCCESS
+ assert outcome.recovery_verified is (status in {"completed", "incomplete"})
+ assert forwarded == payload
+
+
[email protected]("protocol", BUFFERED)
[email protected]("failure", ["http", "envelope"])
+async def test_failed_response_never_proves_recovery_from_a_success_shaped_body(protocol: str, failure: str) -> None:
+ document = json.loads(_body(protocol))
+ if failure == "envelope":
+ document["error"] = {"type": "server_error"}
+
+ async def respond(_request: web.Request) -> web.Response:
+ return web.json_response(document, status=503 if failure == "http" else 200)
+
+ async with _engine(respond) as (client, _requests):
+ outcome, forwarded = await _invoke(client, protocol, False)
+
+ assert outcome.kind is RawOutcomeKind.HTTP_ERROR
+ assert outcome.recovery_verified is False
+ assert forwarded == b""
+
+
[email protected]("protocol", BUFFERED)
+ "body_mode",
+ ["output", "reasoning", "tool", "empty-completion", "metadata", "heartbeat", "no-event", "unknown", "error"],
+)
+async def test_streaming_recovery_uses_only_recognized_output_or_success(protocol: str, body_mode: str) -> None:
+ payload = {
+ "output": WIRE[protocol][2],
+ "reasoning": REASONING[protocol] + TERMINAL[protocol],
+ "tool": TOOL_OUTPUT[protocol] + TERMINAL[protocol],
+ "empty-completion": METADATA[protocol] + TERMINAL[protocol],
+ "metadata": METADATA[protocol],
+ "heartbeat": b": keepalive\n\nevent: ping\n\n",
+ "no-event": b"",
+ "unknown": b'data: {"unknown":"shape"}\n\n',
+ "error": WIRE[protocol][3],
+ }[body_mode]
+ verified = body_mode in {"output", "reasoning", "tool", "empty-completion"}
+
+ async def respond(_request: web.Request) -> web.Response:
+ return web.Response(body=payload, content_type="text/event-stream")
+
+ async with _engine(respond) as (client, requests):
+ outcome, forwarded = await _invoke(client, protocol, True)
+
+ assert outcome.recovery_verified is verified
+ if verified:
+ assert outcome.kind is RawOutcomeKind.SUCCESS
+ assert outcome.stream_started is (body_mode != "empty-completion")
+ assert forwarded == payload
+ else:
+ assert outcome.kind is (RawOutcomeKind.HTTP_ERROR if body_mode == "error" else RawOutcomeKind.NETWORK_ERROR)
+ assert forwarded == b""
+ assert len(requests) == 1
+
+ state = ProtocolSSEState(protocol)
+ state.observe(payload)
+ terminal = state.terminal_observation()
+ if terminal is not None:
+ assert terminal.recovery_verified is verified
+
+
[email protected]("protocol", BUFFERED)
[email protected]("ending", ["success", "error", "disconnect"])
+async def test_model_output_proves_recovery_before_stream_terminal(protocol: str, ending: str) -> None:
+ release_terminal = asyncio.Event()
+
+ async def respond(request: web.Request) -> web.StreamResponse:
+ response = web.StreamResponse(headers={"Content-Type": "text/event-stream"})
+ await response.prepare(request)
+ await response.write(REASONING[protocol])
+ await release_terminal.wait()
+ if ending == "disconnect":
+ assert request.transport is not None
+ request.transport.close()
+ else:
+ await response.write(TERMINAL[protocol] if ending == "success" else WIRE[protocol][3])
+ await response.write_eof()
+ return response
+
+ async with _engine(respond) as (client, requests):
+ handle = await asyncio.wait_for(client.invoke(_source(protocol), "model-a", {}, stream=True), timeout=2)
+ try:
+ assert handle.observed is not None
+ assert handle.observed.model_output_started is True
+ assert handle.observed.terminal_outcome is None
+ assert handle.outcome_available is False
+ release_terminal.set()
+ forwarded = b"".join([chunk async for chunk in handle.stream])
+ outcome = await handle.outcome()
+ finally:
+ release_terminal.set()
+ await handle.close_stream()
+
+ assert outcome.kind is {
+ "success": RawOutcomeKind.SUCCESS,
+ "error": RawOutcomeKind.HTTP_ERROR,
+ "disconnect": RawOutcomeKind.NETWORK_ERROR,
+ }[ending]
+ assert outcome.recovery_verified is True
+ assert outcome.stream_started is True
+ assert forwarded.startswith(REASONING[protocol])
+ assert len(requests) == 1
+
+
[email protected]("protocol", BUFFERED)
+async def test_large_buffered_result_preserves_bytes_and_bounded_evidence_projection(
+ monkeypatch: pytest.MonkeyPatch, protocol: str,
+) -> None:
+ # Force the existing spool path; neither the new evidence nor a late native
+ # discriminator may impose a response size limit or keep the body in memory.
+ monkeypatch.setattr(client_module, "_PRELUDE_MEMORY_BYTES", 64)
+ payload = b'{"unknown":"' + b"x" * (1024 * 1024) + b'",' + _body(protocol)[1:]
+ projector = ProtocolFactProjector(protocol)
+ for offset in range(0, len(payload), 4096):
+ projector.feed(payload[offset:offset + 4096])
+ assert projector.retained_bytes < 32 * 1024
+ assert projector.finish(streamed=False).recovery_verified is True
+
+ async def respond(_request: web.Request) -> web.Response:
+ return web.Response(body=payload, content_type="application/json")
+
+ async with _engine(respond) as (client, _requests):
+ outcome, forwarded = await _invoke(client, protocol, False)
+
+ assert outcome.kind is RawOutcomeKind.SUCCESS
+ assert outcome.recovery_verified is True
+ assert forwarded == payload
diff --git a/tests/test_model_hub_resolution.py b/tests/test_model_hub_resolution.py
index 486542ab1..b7f66447b 100644
--- a/tests/test_model_hub_resolution.py
+++ b/tests/test_model_hub_resolution.py
@@ -157,6 +157,9 @@ class FakeInvokeHandle:
async def outcome(self) -> RawCallOutcome:
return self._outcome
+ async def close_stream(self) -> None:
+ return None
+
class FakeAdapter:
def __init__(self, discovered: tuple[str, ...] = ("claude-opus-4-6",)):
diff --git a/tests/test_model_hub_retry_advice.py b/tests/test_model_hub_retry_advice.py
new file mode 100644
index 000000000..0b047260d
--- /dev/null
+++ b/tests/test_model_hub_retry_advice.py
@@ -0,0 +1,319 @@
+"""Retry advice crosses the adapter boundary without adding retry ownership."""
+
+from __future__ import annotations
+
+import asyncio
+import socket
+from contextlib import asynccontextmanager
+from datetime import datetime, timedelta, timezone
+
+from aiohttp import web
+import pytest
+
+from core.handlers.model_hub.adapter import RawCallOutcome, RawOutcomeKind
+from core.handlers.model_hub.stream_wire import ProtocolUsageReport
+from vibe.model_hub_runtime import client as client_module
+from vibe.model_hub_runtime.client import EngineClient, EngineConnection
+from vibe.model_hub_runtime.state import SourceRecord
+
+
+WIRE = {
+ "anthropic": (
+ "/v1/messages",
+ b'{"type":"message","content":[{"type":"text","text":"ok"}]}',
+ b'event: content_block_delta\ndata: {"type":"content_block_delta",'
+ b'"delta":{"type":"text_delta","text":"ok"}}\n\n'
+ b'event: message_stop\ndata: {"type":"message_stop"}\n\n',
+ b'event: error\ndata: {"type":"error","error":{"type":"api_error"}}\n\n',
+ ),
+ "openai_responses": (
+ "/v1/responses",
+ b'{"output":[{"content":[{"type":"output_text","text":"ok"}]}]}',
+ b'event: response.output_text.delta\ndata: {"type":"response.output_text.delta","delta":"ok"}\n\n'
+ b'event: response.completed\ndata: {"type":"response.completed"}\n\n',
+ b'event: response.failed\ndata: {"type":"response.failed","response":'
+ b'{"error":{"type":"server_error"}}}\n\n',
+ ),
+ "openai_chat": (
+ "/v1/chat/completions",
+ b'{"choices":[{"message":{"role":"assistant","content":"ok"}}]}',
+ b'data: {"choices":[{"delta":{"content":"ok"}}]}\n\ndata: [DONE]\n\n',
+ b'data: {"error":{"type":"server_error"}}\n\n',
+ ),
+}
+ERROR_BODY = (
+ b'{"error":{"type":"server_error","message":"private body detail","retry_after":"999"}}'
+)
+
+
+def _source(protocol: str) -> SourceRecord:
+ return SourceRecord(
+ source_id="src_retryadvice",
+ vendor="custom",
+ protocol=protocol,
+ base_url="https://unused.example.test",
+ credential_ref="cred_retryadvice",
+ allowed_origins=(),
+ model_ids=("model-a",),
+ prefix="retry-advice",
+ )
+
+
+@asynccontextmanager
+async def _engine(respond, *, timeout: float = 2):
+ requests = []
+
+ async def handle(request: web.Request) -> web.StreamResponse:
+ requests.append((request.path, await request.json()))
+ return await respond(request)
+
+ app = web.Application()
+ app.router.add_post("/{path:.*}", handle)
+ runner = web.AppRunner(app, handler_cancellation=True)
+ await runner.setup()
+ try:
+ await web.TCPSite(runner, "127.0.0.1", 0).start()
+ port = runner.addresses[0][1]
+ client = EngineClient(
+ EngineConnection(f"http://127.0.0.1:{port}", "fixture-management", "fixture-gateway"),
+ timeout=timeout,
+ )
+ yield client, requests
+ finally:
+ await runner.cleanup()
+
+
+async def _invoke(client: EngineClient, protocol: str, stream: bool):
+ cleanups = []
+ handle = await client.invoke(
+ _source(protocol),
+ "model-a",
+ {},
+ stream=stream,
+ on_transport_done=lambda: cleanups.append(True),
+ )
+ try:
+ body = b"" if handle.stream is None else b"".join([chunk async for chunk in handle.stream])
+ outcome = await handle.outcome()
+ finally:
+ await handle.close_stream()
+ assert cleanups
+ return outcome, body
+
+
[email protected]("optional_count", [0, 3])
+def test_existing_outcome_constructors_keep_retry_advice_optional(optional_count: int) -> None:
+ usage = ProtocolUsageReport.of(input_tokens=12, cached_input_tokens=0, output_tokens=3)
+ optional = ("server_error", ("server_error",), usage)
+ outcome = RawCallOutcome(
+ RawOutcomeKind.HTTP_ERROR, 503, None, None, False, "model-a", "src_retryadvice",
+ *optional[:optional_count],
+ )
+ assert outcome.retry_after is None
+ assert outcome.response_received_at is None
+ assert outcome.usage == (usage if optional_count else None)
+
+
[email protected]("protocol", WIRE)
[email protected]("stream", [False, True], ids=["buffered", "streaming"])
+ ("advice", "expected"),
+ [
+ pytest.param("60", "60", id="delay-seconds"),
+ pytest.param("0", "0", id="zero"),
+ pytest.param("Wed, 09 Sep 2026 04:20:00 GMT", "Wed, 09 Sep 2026 04:20:00 GMT", id="http-date"),
+ pytest.param("Sun, 06 Nov 1994 08:49:37 GMT", "Sun, 06 Nov 1994 08:49:37 GMT", id="past-date"),
+ pytest.param("-1", "-1", id="negative-left-to-policy"),
+ pytest.param("1.5", "1.5", id="fraction-left-to-policy"),
+ pytest.param("not-a-date", "not-a-date", id="malformed-left-to-policy"),
+ pytest.param("9" * 128, "9" * 128, id="overflow-left-to-policy"),
+ pytest.param("9" * 129, None, id="oversized-dropped-not-truncated"),
+ pytest.param("60" + "x" * 127, None, id="oversized-valid-prefix"),
+ pytest.param("12", None, id="nonascii-digits"),
+ pytest.param("60秒", None, id="nonascii-suffix"),
+ pytest.param("", "", id="empty-left-to-policy"),
+ pytest.param(None, None, id="missing"),
+ ],
+)
+async def test_failed_http_response_carries_only_bounded_ascii_advice(
+ protocol: str, stream: bool, advice: str | None, expected: str | None,
+) -> None:
+ async def respond(_request: web.Request) -> web.Response:
+ headers = {
+ "Authorization": "private response credential",
+ "X-Retry-After": "888",
+ "X-Private-Detail": "private header detail",
+ }
+ if advice is not None:
+ headers["rEtRy-AfTeR"] = advice
+ return web.Response(status=503, body=ERROR_BODY, headers=headers, content_type="application/json")
+
+ async with _engine(respond) as (client, requests):
+ before = datetime.now(timezone.utc)
+ outcome, body = await _invoke(client, protocol, stream)
+ after = datetime.now(timezone.utc)
+
+ assert outcome.kind is RawOutcomeKind.HTTP_ERROR
+ assert outcome.http_status == 503
+ assert outcome.stream_started is False
+ assert outcome.retry_after == expected
+ assert outcome.response_received_at is not None
+ assert outcome.response_received_at.tzinfo is timezone.utc
+ assert before <= outcome.response_received_at <= after
+ assert outcome.error_type == "server_error"
+ assert outcome.error_candidates == ("server_error",)
+ assert "private" not in repr(outcome)
+ assert body == b""
+ assert requests == [(WIRE[protocol][0], {"model": "retry-advice/model-a", "stream": stream})]
+
+
[email protected]("protocol", WIRE)
[email protected]("stream", [False, True], ids=["buffered", "streaming"])
[email protected]("body_mode", ["slow", "timeout", "disconnect", "malformed"])
+async def test_retry_advice_survives_error_body_delays_and_failures(
+ monkeypatch: pytest.MonkeyPatch, protocol: str, stream: bool, body_mode: str,
+) -> None:
+ """MH-RETRY-ADVICE-001: advice keeps the header receipt clock across body failures."""
+ receipt = datetime(2026, 9, 9, 4, 15, tzinfo=timezone.utc)
+ current_time = receipt
+ clock_samples = []
+ sampled = asyncio.Event()
+ release_body = asyncio.Event()
+
+ class Clock:
+ @staticmethod
+ def now(tz):
+ assert tz is timezone.utc
+ clock_samples.append(current_time)
+ sampled.set()
+ return current_time
+
+ monkeypatch.setattr(client_module, "datetime", Clock)
+
+ async def respond(request: web.Request) -> web.StreamResponse:
+ response = web.StreamResponse(
+ status=429,
+ headers={
+ "Retry-After": "60",
+ "Content-Type": "application/json",
+ "Content-Length": str(len(ERROR_BODY)),
+ },
+ )
+ await response.prepare(request)
+ await release_body.wait()
+ if body_mode == "timeout":
+ return response
+ if body_mode == "disconnect":
+ await response.write(ERROR_BODY[:5])
+ assert request.transport is not None
+ request.transport.close()
+ else:
+ await response.write(b"x" * len(ERROR_BODY) if body_mode == "malformed" else ERROR_BODY)
+ await response.write_eof()
+ return response
+
+ async with _engine(respond, timeout=0.25 if body_mode == "timeout" else 2) as (client, requests):
+ pending = asyncio.create_task(_invoke(client, protocol, stream))
+ try:
+ # The server cannot finish its body until the receipt clock is sampled.
+ await asyncio.wait_for(sampled.wait(), timeout=2)
+ assert not pending.done()
+ current_time += timedelta(minutes=5)
+ if body_mode != "timeout":
+ release_body.set()
+ outcome, body = await asyncio.wait_for(pending, timeout=3)
+ finally:
+ release_body.set()
+ if not pending.done():
+ pending.cancel()
+ await asyncio.gather(pending, return_exceptions=True)
+
+ assert clock_samples == [receipt]
+ assert outcome.response_received_at == receipt
+ assert outcome.retry_after == "60"
+ assert outcome.kind is RawOutcomeKind.HTTP_ERROR
+ assert outcome.http_status == 429
+ assert outcome.stream_started is False
+ assert outcome.error_type == ("server_error" if body_mode == "slow" else None)
+ assert body == b""
+ assert len(requests) == 1
+
+
[email protected]("protocol", WIRE)
[email protected]("stream", [False, True], ids=["buffered", "streaming"])
+ ("body_mode", "expected_kind"),
+ [
+ ("success", RawOutcomeKind.SUCCESS),
+ ("shaped_error", RawOutcomeKind.HTTP_ERROR),
+ ("empty", RawOutcomeKind.NETWORK_ERROR),
+ ],
+)
+async def test_http_200_never_supplies_retry_advice(
+ protocol: str, stream: bool, body_mode: str, expected_kind: RawOutcomeKind,
+) -> None:
+ _, buffered, streamed, stream_error = WIRE[protocol]
+ payload = {
+ "success": streamed if stream else buffered,
+ "shaped_error": stream_error if stream else ERROR_BODY,
+ "empty": b"",
+ }[body_mode]
+
+ async def respond(_request: web.Request) -> web.Response:
+ return web.Response(
+ body=payload,
+ headers={"Retry-After": "60"},
+ content_type="text/event-stream" if stream else "application/json",
+ )
+
+ async with _engine(respond) as (client, requests):
+ outcome, body = await _invoke(client, protocol, stream)
+
+ assert outcome.kind is expected_kind
+ assert outcome.http_status == 200
+ assert outcome.retry_after is None
+ assert outcome.response_received_at is None
+ assert body == (payload if body_mode == "success" else b"")
+ assert len(requests) == 1
+
+
+async def test_connection_failure_has_no_response_receipt_or_retry_advice() -> None:
+ # Reserve a local port without listening: no real service can own this target.
+ with socket.socket() as reserved:
+ reserved.bind(("127.0.0.1", 0))
+ port = reserved.getsockname()[1]
+ client = EngineClient(
+ EngineConnection(f"http://127.0.0.1:{port}", "fixture-management", "fixture-gateway"),
+ )
+ outcome, body = await _invoke(client, "openai_responses", False)
+
+ assert outcome.kind is RawOutcomeKind.NETWORK_ERROR
+ assert outcome.http_status is None
+ assert outcome.retry_after is None
+ assert outcome.response_received_at is None
+ assert body == b""
+
+
[email protected]("protocol", WIRE)
+async def test_local_registration_failure_keeps_its_identity_with_http_receipt(protocol: str) -> None:
+ async def respond(_request: web.Request) -> web.Response:
+ return web.json_response(
+ {
+ "type": "error",
+ "error": {"type": "api_error", "message": "unknown provider for model retry-advice/model-a"},
+ },
+ status=502,
+ headers={"Retry-After": "60"},
+ )
+
+ async with _engine(respond) as (client, requests):
+ outcome, body = await _invoke(client, protocol, False)
+
+ assert outcome.kind is RawOutcomeKind.NETWORK_ERROR
+ assert outcome.error_code == "engine_down"
+ assert outcome.http_status == 502
+ assert outcome.retry_after == "60"
+ assert outcome.response_received_at is not None
+ assert outcome.response_received_at.tzinfo is timezone.utc
+ assert body == b""
+ assert len(requests) == 1
diff --git a/tests/test_model_hub_retry_lifecycle.py b/tests/test_model_hub_retry_lifecycle.py
new file mode 100644
index 000000000..9aa8af3ae
--- /dev/null
+++ b/tests/test_model_hub_retry_lifecycle.py
@@ -0,0 +1,639 @@
+"""Recovery ownership and persistence boundaries from the first exact-head review."""
+
+from __future__ import annotations
+
+import asyncio
+import copy
+from contextlib import asynccontextmanager
+from dataclasses import replace
+from datetime import timedelta
+from types import SimpleNamespace
+
+import aiohttp
+from aiohttp import web
+import pytest
+
+from config.v2_config import ModelHubConfig, ModelHubRouteConfig, ModelHubRouteHopConfig, ModelHubSourceStateConfig
+from core.handlers.model_hub.adapter import RawOutcomeKind
+from core.handlers.model_hub.retry import RECOVERY_EXHAUSTED_CODE, RECOVERY_EXHAUSTED_MESSAGE
+from core.handlers.model_hub.service import ModelHubError
+from core.handlers.model_hub.turn_gateway import ModelHubTurnGateway
+from core.run_settlement import SETTLED_BY_TERMINAL_RESULT
+from modules.agents.model_hub import ModelHubRuntimeRouter, bind_launch
+from tests.test_model_hub_l3 import _assert_valid, _canonicalize_fixed_test_routes, _outcome, _source
+from tests.test_model_hub_retry_policy import WIRE, clock_service, fail
+from vibe.model_hub_runtime.adapter import CLIProxyEngineAdapter
+from vibe.model_hub_runtime.client import EngineClient, EngineConnection
+from vibe.model_hub_runtime.state import EngineStateStore
+from vibe.model_hub_runtime.supervisor import EngineUnavailableError
+
+
+BACKENDS = {
+ "codex": ("responses", "openai_responses"),
+ "claude": ("messages", "anthropic"),
+ "opencode": ("chat/completions", "openai_chat"),
+}
+
+
+class FreshStore:
+ """Every read reconstructs persisted data; failed writes cannot mutate it."""
+
+ def __init__(self, config):
+ self.persisted = copy.deepcopy(config.to_payload())
+ self.write_error = False
+
+ def load(self):
+ return ModelHubConfig.from_payload(copy.deepcopy(self.persisted))
+
+ def save(self, config):
+ if self.write_error:
+ if isinstance(self.write_error, BaseException):
+ raise self.write_error
+ raise OSError("fixture: read-only configuration")
+ self.persisted = copy.deepcopy(config.to_payload())
+
+ def mutate(self, operation):
+ config = self.load()
+ if operation(config):
+ self.save(config)
+
+
+def configured_service(tmp_path, **kwargs):
+ service, clock = clock_service(tmp_path, **kwargs)
+ models = _canonicalize_fixed_test_routes(service)
+ models["opencode"] = "shared-model"
+ return service, clock, models
+
+
+@asynccontextmanager
+async def gateway_client(service, backend, model):
+ gateway = ModelHubTurnGateway(service)
+ source = service.store.load().sources[0]
+ base, token = await gateway.endpoint(
+ backend, process_scope="lifecycle", turn_id="turn_lifecycle",
+ requested_model_id=model, resolved_model_id="shared-model", source_id=source.id,
+ )
+ try:
+ async with aiohttp.ClientSession(trust_env=False) as client:
+ yield gateway, client, f"{base}/v1/{BACKENDS[backend][0]}", {"Authorization": f"Bearer {token}"}
+ finally:
+ await gateway.close()
+
+
+@asynccontextmanager
+async def loopback_engine(tmp_path, service, respond):
+ """Use the production client/admission/stream observer against local HTTP."""
+ config = service.store.load()
+ source = config.sources[0]
+ state = EngineStateStore(tmp_path / "engine")
+ source.credential_ref = state.store_api_key("fixture", vendor=source.vendor, protocol=source.protocol)
+ service.store.save(config)
+ state.sync_sources(service._bindings(config))
+ app = web.Application()
+ app.router.add_post("/{path:.*}", respond)
+ runner = web.AppRunner(app, handler_cancellation=True)
+ await runner.setup()
+ await web.TCPSite(runner, "127.0.0.1", 0).start()
+ engine_client = EngineClient(
+ EngineConnection(f"http://127.0.0.1:{runner.addresses[0][1]}", "fixture-management", "fixture-gateway"),
+ )
+ adapter = CLIProxyEngineAdapter(supervisor=SimpleNamespace(client=lambda: engine_client), state_store=state)
+ adapter.ensure_installed = service.adapter.ensure_installed
+ service.adapter = adapter
+ service._engine_synced = True
+ try:
+ yield adapter, engine_client
+ finally:
+ await runner.cleanup()
+
+
[email protected]("stream", [False, True])
+def test_unavailable_production_client_releases_unadmitted_half_open_claim(tmp_path, stream):
+ """MH-RETRY-OWNERSHIP-001: completed engine-down never fires on_admitted."""
+ async def run():
+ service, clock, models = configured_service(tmp_path)
+ source = service.store.load().sources[0]
+ calls = []
+
+ async def respond(request):
+ calls.append(await request.json())
+ return web.Response(body=WIRE["openai_responses"][3], content_type="application/json")
+
+ def unavailable():
+ raise EngineUnavailableError("fixture: no running engine client")
+
+ async with loopback_engine(tmp_path, service, respond) as (adapter, engine_client):
+ await fail(service, source)
+ clock.advance(1)
+ adapter.supervisor.client = unavailable
+ observed = []
+ snapshots = []
+ with pytest.raises(ModelHubError, match="engine_down"):
+ await service.resolve_with_recovery(
+ backend="codex", model_id=models["codex"], request={}, stream=stream,
+ attempt_observer=lambda *args: observed.append(args),
+ recovery_observer=snapshots.append,
+ )
+ assert observed == [] # Not an admitted request, and not another failed Source attempt.
+ assert not any(snapshot and snapshot["attempt_count"] for snapshot in snapshots)
+ assert adapter._active_transports == 0 and not service._mutation_lock.locked()
+ entry = service.recovery._sources[source.id]
+ assert entry.owner is None and entry.failures == 1
+ assert service.agent_chain("codex", models["codex"])["chain"][0]["recovery"] == "eligible"
+ # The next real request can acquire the same half-open admission.
+ adapter.supervisor.client = lambda: engine_client
+ async with gateway_client(service, "codex", models["codex"]) as (_gateway, client, url, headers):
+ async with client.post(url, headers=headers, json={"model": "shared-model"}) as response:
+ assert response.status == 200
+ assert await response.read() == WIRE["openai_responses"][3]
+ assert len(calls) == 1 and adapter._active_transports == 0
+ assert source.id not in service.recovery._sources
+ asyncio.run(run())
+
+
[email protected]("backend", ["codex", "claude"])
[email protected]("diagnostic,delay", [
+ ("503 server error", 30), ("429 rate limit", 60), ("quota exhausted", 300), ("connection failed", 0),
+])
+def test_native_failure_never_enters_http_recovery_streak(tmp_path, backend, diagnostic, delay):
+ """MH-RETRY-NATIVE-002: native success has no Hub handle to clear a streak."""
+ async def run():
+ source = _source(
+ "src_native001", "Native", channel="native_cli", status="active",
+ vendor="anthropic" if backend == "claude" else "openai",
+ )
+ service, clock, models = configured_service(tmp_path, sources=[source])
+ source.models[0].id = models[backend]
+ service.store.config.agents[backend].routes[models[backend]] = ModelHubRouteConfig(
+ hops=(ModelHubRouteHopConfig(source.id, models[backend]),),
+ )
+ router = ModelHubRuntimeRouter(
+ service=service, native_cli_ready=lambda _backend: True, overlay_path=tmp_path / "overlay.json",
+ )
+ for _ in range(3):
+ launch = await router.resolve(backend, models[backend])
+ assert launch.channel == "native_cli"
+ context = SimpleNamespace(platform_specific={})
+ bind_launch(context, launch)
+ assert await router.record_native_failure(context, diagnostic) is bool(delay)
+ current = service.store.load().sources[0]
+ if delay:
+ assert current.state.retry_at == (clock.now() + timedelta(seconds=delay)).isoformat()
+ assert service.recovery.annotations(service.store.load()) == {}
+ assert service.recovery._sources == {}
+ clock.advance(delay)
+ assert service.agent_chain(backend, models[backend])["chain"][0]["runnable"]
+ assert service.adapter.invocations == []
+ assert not any(event["kind"] == "recover" for event in service.events.list())
+ asyncio.run(run())
+
+
[email protected]("backend", BACKENDS)
[email protected]("stream", [False, True])
[email protected]("write_failure", ["io", "recovery_warning"])
+def test_verified_response_survives_failed_recovery_persistence_and_fresh_reads(tmp_path, backend, stream, write_failure):
+ """MH-RETRY-PERSISTENCE-001: stored cooldown cannot resurrect after recovery."""
+ async def run():
+ service, clock, models = configured_service(tmp_path)
+ metadata, output, terminal, buffered = WIRE[BACKENDS[backend][1]]
+ body = metadata + output + terminal if stream else buffered
+
+ async def respond(_request):
+ return web.Response(body=body, content_type="text/event-stream" if stream else "application/json")
+
+ async with loopback_engine(tmp_path, service, respond) as (adapter, _engine_client):
+ service.store = FreshStore(service.store.load())
+ source = service.store.load().sources[0]
+ await fail(service, source, "server_error")
+ clock.advance(30)
+ before = copy.deepcopy(service.store.persisted)
+ service.store.write_error = True if write_failure == "io" else ValueError("fixture: recovery warnings")
+ async with gateway_client(service, backend, models[backend]) as (_gateway, client, url, headers):
+ async with client.post(url, headers=headers, json={"model": "shared-model", "stream": stream}) as response:
+ assert response.status == 200
+ assert await response.read() == body
+ assert adapter._active_transports == 0
+ assert service.store.persisted == before # A fresh disk view still has the old cooldown.
+ for _ in range(3):
+ chain = service.agent_chain(backend, models[backend])
+ assert chain["chain"][0]["health"] == "healthy"
+ assert chain["chain"][0]["runnable"] is True
+ assert "recovery" not in chain["chain"][0]
+ assert sum(event["kind"] == "recover" for event in service.events.list()) == 1
+ # A later real failure starts at the first delay, even if the stored old
+ # cooldown could not be retired. No further caller loses its response.
+ await fail(service, service.store.load().sources[0], "server_error")
+ assert service.recovery._sources[source.id].failures == 1
+ assert service.agent_chain(backend, models[backend])["supply_state"] == "waiting"
+ asyncio.run(run())
+
+
[email protected]("change", [
+ "none", "new_deadline", "new_reason", "needs_action", "error", "credential", "verification",
+ "endpoint", "deleted", "retired_model", "disk_cleared",
+])
+def test_recovered_disk_observation_is_exact_and_never_masks_new_authority(tmp_path, change):
+ async def run():
+ service, clock, models = configured_service(tmp_path)
+ service.store = FreshStore(service.store.load())
+ source = service.store.load().sources[0]
+ await fail(service, source, "server_error")
+ clock.advance(30)
+ generation = service._reserve_settlement_generation(source.id)
+ service.store.write_error = True
+ await service._record_recovery_success(source.id, generation, backend="codex", model_id=models["codex"])
+ assert service.agent_chain("codex", models["codex"])["chain"][0]["health"] == "healthy"
+ config = service.store.load()
+ current = config.sources[0]
+ if change == "new_deadline":
+ current.state.retry_at = (clock.now() + timedelta(seconds=60)).isoformat()
+ elif change == "new_reason":
+ current.state.detail_key = "models.source.cooldown.rate_limited"
+ elif change in {"needs_action", "error"}:
+ current.state = ModelHubSourceStateConfig(
+ status=change,
+ detail_key="models.source.error.unclassified" if change == "error"
+ else "models.source.needs_action.credential_revoked",
+ )
+ elif change == "credential":
+ current.credential_ref = "cred_replaced"
+ elif change == "verification":
+ current.verification_pending = "vp_" + "1" * 32
+ elif change == "endpoint":
+ current.base_url = "https://replacement.invalid"
+ elif change == "deleted":
+ config.sources = []
+ for agent in config.agents.values():
+ agent.sources.order = []
+ agent.routes = {}
+ elif change == "retired_model":
+ current.models[0].retired = True
+ elif change == "disk_cleared":
+ current.state = ModelHubSourceStateConfig(status="standby")
+ service.store.persisted = copy.deepcopy(config.to_payload())
+ chain = service.agent_chain("codex", models["codex"])
+ _assert_valid("agent-chain.schema.json", chain)
+ if change == "deleted":
+ assert chain["chain"] == [] and chain["supply_state"] == "interrupted"
+ assert service.recovery._sources == service.recovery._retired_cooldowns == {}
+ return
+ hop = chain["chain"][0]
+ if change in {"none", "disk_cleared"}:
+ assert hop["health"] == "healthy" and hop["runnable"]
+ assert "recovery" not in hop
+ elif change == "new_deadline":
+ assert hop["health"] == "cooldown" and not hop["runnable"]
+ assert chain["supply_state"] == "waiting"
+ elif change in {"new_reason", "credential", "verification", "endpoint"}:
+ assert hop["recovery"] == "eligible" # Not proven healthy for this new observation.
+ else:
+ assert not hop["runnable"] and chain["supply_state"] == "interrupted"
+ # Re-delivery of the old success cannot retire a replacement identity.
+ if change in {"credential", "verification", "endpoint"}:
+ await service._record_recovery_success(source.id, generation, backend="codex", model_id=models["codex"])
+ assert service.agent_chain("codex", models["codex"])["chain"][0]["recovery"] == "eligible"
+ assert sum(event["kind"] == "recover" for event in service.events.list()) == 1
+ if change == "disk_cleared":
+ # Even the identical old payload is new once its retirement was
+ # observed on disk; the tombstone is not a permanent Source flag.
+ config.sources[0].state = replace(source.state, status="cooldown", retry_at=clock.now().isoformat(),
+ detail_key="models.source.cooldown.server_error")
+ service.store.persisted = config.to_payload()
+ assert service.agent_chain("codex", models["codex"])["chain"][0]["recovery"] == "eligible"
+ asyncio.run(run())
+
+
+def test_recovered_annotation_cannot_retire_a_different_preview_cooldown(tmp_path):
+ async def run():
+ service, clock, models = configured_service(tmp_path)
+ service.store = FreshStore(service.store.load())
+ source = service.store.load().sources[0]
+ await fail(service, source, "server_error")
+ clock.advance(30)
+ service.store.write_error = True
+ await service._record_recovery_success(
+ source.id, service._reserve_settlement_generation(source.id),
+ backend="codex", model_id=models["codex"],
+ )
+ draft = service.store.load()
+ draft.sources[0].state.retry_at = (clock.now() + timedelta(seconds=300)).isoformat()
+ assert service._agent_chain(draft, "codex", models["codex"])["supply_state"] == "waiting"
+ assert service.agent_chain("codex", models["codex"])["chain"][0]["health"] == "healthy"
+ asyncio.run(run())
+
+
[email protected]("backend", BACKENDS)
[email protected]("status,code,attempts", [(503, None, 3), (429, None, 2), (429, "quota_exhausted", 1)])
+def test_failure_persistence_error_keeps_temporary_admission_and_original_terminal(tmp_path, backend, status, code, attempts):
+ async def run():
+ service, _clock, models = configured_service(tmp_path, outcomes=[
+ _outcome(RawOutcomeKind.HTTP_ERROR, source_id="src_recovery01", status=status, code=code)
+ for _ in range(attempts)
+ ])
+ service.store = FreshStore(service.store.load())
+ before = copy.deepcopy(service.store.persisted)
+ service.store.write_error = True
+ async with gateway_client(service, backend, models[backend]) as (gateway, client, url, headers):
+ async with client.post(url, headers=headers, json={"model": "shared-model"}) as response:
+ assert response.status == (400 if backend == "codex" else 424)
+ assert (await response.json())["error"]["code"] == RECOVERY_EXHAUSTED_CODE
+ projection = gateway.correlation.terminal_projection("turn_lifecycle", backend=backend)
+ if backend == "opencode":
+ assert projection is None # Shared-server HTTP does not prove an exact Turn.
+ else:
+ assert projection is not None
+ assert [blocker.reason for blocker in projection.supply_facts.blockers] == [
+ code or ("server_error" if status == 503 else "rate_limited"),
+ ]
+ gateway.correlation.settle("turn_lifecycle", settled_by=SETTLED_BY_TERMINAL_RESULT)
+ record = service.provenance.get("turn_lifecycle")
+ assert [attempt["http_status"] for attempt in record["failed_attempts"]] == [status] * attempts
+ assert len(service.adapter.invocations) == attempts
+ assert service.store.persisted == before
+ chain = service.agent_chain(backend, models[backend])
+ assert chain["supply_state"] == "waiting"
+ assert chain["chain"][0]["health"] == "cooldown"
+ _assert_valid("agent-chain.schema.json", chain)
+ with pytest.raises(ModelHubError) as terminal:
+ await service.resolve(backend=backend, model_id=models[backend], request={})
+ assert [blocker.reason for blocker in terminal.value.blockers] == ["cooldown"]
+ asyncio.run(run())
+
+
[email protected]("persisted", [False, True])
[email protected]("reason,delay", [
+ ("server_error", 30), ("rate_limited", 60), ("quota_exhausted", 300), ("network", 1),
+])
[email protected]("transition", ["none", "needs_action", "error", "retired_model", "recovered"])
+def test_terminal_supply_projection_uses_effective_transient_facts_and_stronger_blockers(
+ tmp_path, persisted, reason, delay, transition,
+):
+ async def run():
+ service, clock, models = configured_service(tmp_path)
+ service.store = FreshStore(service.store.load())
+ service.store.write_error = not persisted
+ source = service.store.load().sources[0]
+ await fail(service, source, reason)
+ config = service.store.load()
+ if transition in {"needs_action", "error"}:
+ config.sources[0].state = ModelHubSourceStateConfig(
+ status=transition,
+ detail_key="models.source.error.unclassified" if transition == "error"
+ else "models.source.needs_action.credential_revoked",
+ )
+ elif transition == "retired_model":
+ config.sources[0].models[0].retired = True
+ service.store.persisted = config.to_payload()
+ if transition == "recovered":
+ clock.advance(delay)
+ service.store.write_error = True
+ await service._record_recovery_success(
+ source.id, service._reserve_settlement_generation(source.id),
+ backend="codex", model_id=models["codex"],
+ )
+ chain = service.agent_chain("codex", models["codex"])
+ assert chain["supply_state"] == "ok" and chain["chain"][0]["health"] == "healthy"
+ _config, resolution = service._inspect_terminal_chain(backend="codex", model_id=models["codex"])
+ assert resolution.candidate_hops[0].cooldown_reason is None
+ return
+ with pytest.raises(ModelHubError) as terminal:
+ await service.resolve(backend="codex", model_id=models["codex"], request={})
+ expected = {
+ "none": reason, "needs_action": "credential_revoked",
+ "error": "unclassified_error", "retired_model": "model_unsupported",
+ }[transition]
+ facts = terminal.value.turn_outcome.supply_facts
+ assert [blocker.reason for blocker in facts.blockers] == [expected]
+ assert facts.supply_state == ("waiting" if transition == "none" else "interrupted")
+ assert bool(facts.retry_at) is (transition == "none")
+ assert [blocker.reason for blocker in terminal.value.blockers] == [
+ ("network" if reason == "network" else "cooldown") if transition == "none" else expected,
+ ]
+ asyncio.run(run())
+
+
+def test_persisted_cooldown_without_failure_evidence_does_not_invent_server_error(tmp_path):
+ async def run():
+ service, clock, models = configured_service(tmp_path)
+ source = service.store.load().sources[0]
+ source.state = ModelHubSourceStateConfig(
+ status="cooldown", retry_at=(clock.now() + timedelta(seconds=30)).isoformat(),
+ )
+ with pytest.raises(ModelHubError) as terminal:
+ await service.resolve(backend="codex", model_id=models["codex"], request={})
+ assert [blocker.reason for blocker in terminal.value.blockers] == ["cooldown"]
+ assert terminal.value.turn_outcome.supply_facts.blockers == ()
+ asyncio.run(run())
+
+
[email protected]("backend", BACKENDS)
[email protected]("wake", ["owner_success", "runnable_edit", "blocked_edit"])
+def test_expiry_remains_a_closed_terminal_when_live_supply_changes(tmp_path, backend, wake):
+ """MH-RETRY-EXPIRY-001: an exhausted waiter is not a fresh route failure."""
+ async def run():
+ service, clock, models = configured_service(tmp_path)
+ source = service.store.load().sources[0]
+ await fail(service, source)
+ clock.advance(1)
+ generation = service._reserve_settlement_generation(source.id)
+ assert service.recovery.claim(source, generation)
+ entered, release = asyncio.Event(), asyncio.Event()
+
+ async def held_wait(_seconds):
+ entered.set()
+ await release.wait()
+
+ service.recovery.sleep = held_wait
+ async with gateway_client(service, backend, models[backend]) as (_gateway, client, url, headers):
+ task = asyncio.create_task(client.post(url, headers=headers, json={"model": "shared-model"}))
+ try:
+ await asyncio.wait_for(entered.wait(), 2)
+ clock.advance(121)
+ if wake == "owner_success":
+ await service._record_recovery_success(
+ source.id, generation, backend=backend, model_id=models[backend],
+ )
+ elif wake == "runnable_edit":
+ source.credential_ref = "cred_replaced"
+ service.recovery.reconcile(service.store.load())
+ else:
+ source.state = ModelHubSourceStateConfig(
+ status="needs_action", detail_key="models.source.needs_action.credential_revoked",
+ )
+ release.set()
+ response = await asyncio.wait_for(task, 2)
+ assert response.status == (400 if backend == "codex" else 424)
+ assert "Retry-After" not in response.headers
+ assert await response.json() == {
+ "type": "error",
+ "error": {
+ "type": RECOVERY_EXHAUSTED_CODE, "code": RECOVERY_EXHAUSTED_CODE,
+ "message": RECOVERY_EXHAUSTED_MESSAGE,
+ },
+ }
+ assert service.adapter.invocations == []
+ finally:
+ release.set()
+ task.cancel()
+ await asyncio.gather(task, return_exceptions=True)
+ asyncio.run(run())
+
+
[email protected]("backend", BACKENDS)
[email protected]("change", ["none", "owner_success", "blocked_edit"])
+def test_window_expiry_during_engine_preparation_preserves_admission_cause(tmp_path, backend, change):
+ async def run():
+ service, clock, models = configured_service(tmp_path)
+ source = service.store.load().sources[0]
+ await fail(service, source)
+ original_prepare = service._prepare_engine_for_demand
+
+ async def expire_during_preparation(**kwargs):
+ clock.advance(121)
+ if change == "owner_success":
+ await service._record_recovery_success(
+ source.id, service._latest_source_attempt_generation[source.id],
+ backend=backend, model_id=models[backend],
+ )
+ elif change == "blocked_edit":
+ source.state = ModelHubSourceStateConfig(
+ status="needs_action", detail_key="models.source.needs_action.credential_revoked",
+ )
+ await original_prepare(**kwargs)
+
+ service._prepare_engine_for_demand = expire_during_preparation
+ async with gateway_client(service, backend, models[backend]) as (gateway, client, url, headers):
+ async with client.post(url, headers=headers, json={"model": "shared-model"}) as response:
+ assert response.status == (400 if backend == "codex" else 424)
+ assert (await response.json())["error"]["code"] == RECOVERY_EXHAUSTED_CODE
+ assert "Retry-After" not in response.headers
+ projection = gateway.correlation.terminal_projection("turn_lifecycle", backend=backend)
+ if backend == "opencode":
+ assert projection is None
+ else:
+ # The wire retains admission expiry while the canonical
+ # projection still reports current supply, not a stale outage.
+ assert projection.supply_facts.supply_state == (
+ "interrupted" if change == "blocked_edit" else "waiting"
+ )
+ assert [blocker.reason for blocker in projection.supply_facts.blockers] == (
+ ["credential_revoked"] if change == "blocked_edit" else []
+ )
+ assert clock.delays == [1] and service.adapter.invocations == []
+ assert not service._mutation_lock.locked()
+ asyncio.run(run())
+
+
[email protected]("backend", BACKENDS)
[email protected]("ending", ["temporary", "invalid", "auth", "engine", "post_output"])
+def test_admitted_terminal_classification_is_not_replaced_by_elapsed_window(tmp_path, backend, ending):
+ async def run():
+ service, clock, models = configured_service(tmp_path)
+ metadata, output, _terminal, _buffered = WIRE[BACKENDS[backend][1]]
+ calls = []
+
+ async def respond(request):
+ calls.append(await request.json())
+ if len(calls) == 1:
+ return web.json_response({"error": {"type": "server_error"}}, status=503)
+ clock.advance(121) # A connected inference may outlive admission.
+ if ending == "post_output":
+ error = b'data: {"type":"error","error":{"type":"server_error","message":"fixture"}}\n\n'
+ return web.Response(body=metadata + output + error, content_type="text/event-stream")
+ status, code = {
+ "temporary": (503, "server_error"),
+ "invalid": (400, "invalid_request_error"),
+ "auth": (403, "invalid_api_key"),
+ "engine": (502, "engine_down"),
+ }[ending]
+ return web.json_response({"error": {"type": code}}, status=status)
+
+ async with loopback_engine(tmp_path, service, respond) as (adapter, engine_client):
+ if ending == "engine":
+ client_reads = []
+
+ def client_or_late_engine_failure():
+ client_reads.append(True)
+ if len(client_reads) == 1:
+ return engine_client
+ clock.advance(121)
+ raise EngineUnavailableError("fixture: engine disappeared before admission")
+
+ adapter.supervisor.client = client_or_late_engine_failure
+ async with gateway_client(service, backend, models[backend]) as (_gateway, client, url, headers):
+ async with client.post(
+ url, headers=headers, json={"model": "shared-model", "stream": ending == "post_output"},
+ ) as response:
+ body = await response.read()
+ if ending == "temporary":
+ assert response.status == (400 if backend == "codex" else 424)
+ assert RECOVERY_EXHAUSTED_CODE.encode() in body
+ else:
+ assert RECOVERY_EXHAUSTED_CODE.encode() not in body
+ if ending == "post_output":
+ assert response.status == 200 and output in body
+ elif ending == "invalid":
+ assert response.status == 400
+ assert adapter._active_transports == 0
+ assert len(calls) == (1 if ending == "engine" else 2) and clock.delays == [30]
+ asyncio.run(run())
+
+
[email protected]("backend", BACKENDS)
+def test_expiry_after_mixed_real_failures_keeps_the_action_blocker_terminal(tmp_path, backend):
+ """MH-RETRY-EXPIRY-002: elapsed time cannot erase an admitted hard failure."""
+ async def run():
+ first, second = _source("src_primary01", "Primary"), _source("src_backup001", "Backup")
+ service, clock, models = configured_service(tmp_path, sources=[first, second], outcomes=[
+ _outcome(RawOutcomeKind.NETWORK_ERROR, source_id=first.id),
+ _outcome(RawOutcomeKind.NETWORK_ERROR, source_id=second.id),
+ _outcome(RawOutcomeKind.HTTP_ERROR, source_id=first.id, status=403),
+ _outcome(RawOutcomeKind.HTTP_ERROR, source_id=second.id, status=503),
+ ])
+ original_invoke = service.adapter.invoke
+ admissions = []
+
+ async def late_last_attempt(*args, **kwargs):
+ on_admitted = kwargs["on_admitted"]
+
+ def admitted():
+ admissions.append((args[0], args[1]))
+ on_admitted()
+
+ kwargs["on_admitted"] = admitted
+ handle = await original_invoke(*args, **kwargs)
+ if len(service.adapter.invocations) == 4:
+ clock.advance(121)
+ return handle
+
+ service.adapter.invoke = late_last_attempt
+ async with gateway_client(service, backend, models[backend]) as (gateway, client, url, headers):
+ async with client.post(url, headers=headers, json={"model": "shared-model"}) as response:
+ assert response.status == 503
+ payload = await response.json()
+ assert payload["error"]["code"] == payload["error"]["type"] == "mapping_target_unavailable"
+ projection = gateway.correlation.terminal_projection("turn_lifecycle", backend=backend)
+ if backend == "opencode":
+ assert projection is None
+ assert service.provenance.get("turn_lifecycle") is None
+ else:
+ assert projection is not None and projection.outcome == "exhausted"
+ assert projection.supply_facts.supply_state == "interrupted"
+ assert [(blocker.source, blocker.reason) for blocker in projection.supply_facts.blockers] == [
+ ("Primary", "credential_revoked"), ("Backup", "server_error"),
+ ]
+ gateway.correlation.settle("turn_lifecycle", settled_by=SETTLED_BY_TERMINAL_RESULT)
+ record = service.provenance.get("turn_lifecycle")
+ assert record["outcome"] == "exhausted"
+ assert record["failed_attempts"] == [
+ {"source_id": source_id, "configured_model_id": "shared-model", "channel": "hub",
+ "reason": reason, **({"http_status": status} if status else {})}
+ for source_id, reason, status in [
+ (first.id, "network", None), (second.id, "network", None),
+ (first.id, "credential_revoked", 403), (second.id, "server_error", 503),
+ ]
+ ]
+ assert len(service.adapter.invocations) == 4 and clock.delays == [1]
+ assert admissions == [(source_id, "shared-model") for source_id in [first.id, second.id] * 2]
+ assert service.agent_chain(backend, models[backend])["supply_state"] == "interrupted"
+ asyncio.run(run())
diff --git a/tests/test_model_hub_retry_policy.py b/tests/test_model_hub_retry_policy.py
new file mode 100644
index 000000000..77a9ca029
--- /dev/null
+++ b/tests/test_model_hub_retry_policy.py
@@ -0,0 +1,614 @@
+"""Bounded recovery at the shared resolver and actual gateway HTTP boundary."""
+
+from __future__ import annotations
+
+import asyncio
+import copy
+from datetime import datetime, timedelta
+from email.utils import format_datetime
+from types import SimpleNamespace
+
+import aiohttp
+from aiohttp import web
+import pytest
+
+from config.v2_config import ModelHubBackendModelConfig, ModelHubSourceStateConfig
+from core.handlers.model_hub.adapter import RawOutcomeKind
+from core.handlers.model_hub.classification import ResolutionDecision
+from core.handlers.model_hub.retry import (
+ RECOVERY_EXHAUSTED_CODE, RECOVERY_EXHAUSTED_MESSAGE, RecoveryPolicy, retry_after_deadline,
+)
+from core.handlers.model_hub.service import ModelHubError
+from core.handlers.model_hub.turn_gateway import ModelHubTurnGateway
+from modules.agents.model_hub import ModelHubRuntimeRouter
+from tests.test_model_hub_inference_wait import WIRE as INFERENCE_WIRE
+from tests.test_model_hub_l3 import (
+ NOW, _assert_valid, _canonicalize_fixed_test_routes, _outcome, _service, _source,
+)
+from vibe.model_hub_runtime.adapter import CLIProxyEngineAdapter
+from vibe.model_hub_runtime.client import EngineClient, EngineConnection
+from vibe.model_hub_runtime.state import EngineStateStore
+
+
+# The inference-deadline fixtures deliberately exercise permissive responses.
+# Recovery additionally needs recognized terminal envelopes.
+WIRE = {
+ **INFERENCE_WIRE,
+ "openai_responses": (*INFERENCE_WIRE["openai_responses"][:3],
+ b'{"object":"response","status":"completed","output":[{"content":[{"type":"output_text","text":"ok"}]}]}'),
+ "openai_chat": (*INFERENCE_WIRE["openai_chat"][:3],
+ b'{"object":"chat.completion","choices":[{"message":{"role":"assistant","content":"ok"}}]}'),
+}
+
+
+class Clock:
+ def __init__(self):
+ self.elapsed = 0.0
+ self.offset = 0.0
+ self.delays = []
+
+ def now(self):
+ return NOW + timedelta(seconds=self.elapsed + self.offset)
+
+ def advance(self, seconds):
+ self.elapsed += seconds
+
+ async def sleep(self, seconds):
+ self.delays.append(seconds)
+ self.advance(seconds)
+ await asyncio.sleep(0)
+
+ def policy(self):
+ return RecoveryPolicy(
+ now=self.now, monotonic=lambda: self.elapsed,
+ jitter=lambda low, high: low, sleep=self.sleep,
+ )
+
+
+def clock_service(tmp_path, *, sources=None, outcomes=None):
+ clock = Clock()
+ service = _service(
+ tmp_path, sources=sources or [_source("src_recovery01", "Recovering")],
+ outcomes=outcomes,
+ )
+ service.now = clock.now
+ service.recovery = clock.policy()
+ # Keep the exact common fixture route as a real catalog model so health
+ # persistence validates the same shape used by the inference tests.
+ for backend in ("claude", "codex"):
+ service.store.config.agents[backend].models.append(
+ ModelHubBackendModelConfig(id="shared-model", origin="manual"),
+ )
+ return service, clock
+
+
+async def fail(service, source, reason="network", *, generation=None, outcome=None):
+ generation = generation if generation is not None else service._reserve_settlement_generation(source.id)
+ return await service._settle_fallback_source(
+ source, ResolutionDecision("fallback", reason=reason),
+ backend="codex", model_id="shared-model", settlement_generation=generation, outcome=outcome,
+ )
+
+
[email protected]("reason,delays", [
+ ("network", [1, 2, 4, 8, 16, 30, 30]),
+ ("server_error", [30, 60, 120, 120]),
+ ("rate_limited", [60, 120, 240, 300, 300]),
+ ("quota_exhausted", [300, 300]),
+])
+def test_shared_streak_counts_attempts_and_caps_positive_jitter(tmp_path, reason, delays):
+ async def run():
+ service, clock = clock_service(tmp_path)
+ source = service.store.load().sources[0]
+ before = copy.deepcopy(service.store.load().to_payload())
+ for expected in delays:
+ await fail(service, source, reason)
+ annotation = service.recovery.annotations(service.store.load())[source.id]
+ assert (datetime.fromisoformat(annotation.retry_at) - clock.now()).total_seconds() == expected
+ clock.advance(expected)
+ if reason == "network":
+ assert service.store.load().to_payload() == before
+ service.recovery.jitter = lambda low, high: high
+ service.recovery._sources.clear()
+ if reason != "network":
+ source = service.store.load().sources[0]
+ source.state = ModelHubSourceStateConfig(status="standby")
+ await fail(service, source, reason)
+ annotation = service.recovery.annotations(service.store.load())[source.id]
+ assert (datetime.fromisoformat(annotation.retry_at) - clock.now()).total_seconds() == min(delays[0] * 1.2, delays[-1])
+ asyncio.run(run())
+
+
[email protected]("value", [None, "", "-1", "+1", "1.5", "NaN", "x" * 129, "4", "9" * 128, "Wed, 01 Jan 2020 00:00:00 GMT"])
+def test_invalid_advice_cannot_replace_local_policy(value):
+ assert retry_after_deadline(value, NOW) is None
+
+
+def test_retry_after_uses_response_receipt_and_keeps_long_upstream_advice(tmp_path):
+ async def run():
+ service, clock = clock_service(tmp_path)
+ source = service.store.load().sources[0]
+ for advice in ("300", format_datetime(NOW + timedelta(seconds=300), usegmt=True)):
+ service.recovery._sources.clear()
+ clock.elapsed = 20
+ outcome = SimpleNamespace(retry_after=advice, response_received_at=NOW, stream_started=False)
+ await fail(service, source, "server_error", outcome=outcome)
+ assert service.recovery.annotations(service.store.load())[source.id].retry_at == (NOW + timedelta(seconds=300)).isoformat()
+ with pytest.raises(ModelHubError):
+ await service.resolve_with_recovery(
+ backend="codex", model_id="shared-model", request={}, supply_channel="hub",
+ )
+ assert clock.delays == []
+ assert service.adapter.invocations == []
+ asyncio.run(run())
+
+
+def test_offsets_and_wall_clock_shifts_do_not_change_monotonic_admission(tmp_path):
+ async def run():
+ service, clock = clock_service(tmp_path)
+ source = service.store.load().sources[0]
+ await fail(service, source)
+ clock.offset = -3600
+ hop = service.agent_chain("codex", "shared-model")["chain"][0]
+ assert (datetime.fromisoformat(hop["retry_at"]) - clock.now()).total_seconds() == 1
+ clock.advance(1)
+ assert service.agent_chain("codex", "shared-model")["chain"][0]["recovery"] == "eligible"
+ assert service.store.load().sources[0].state.status == "standby"
+ assert not any(event["kind"] == "recover" for event in service.events.list())
+ asyncio.run(run())
+
+
[email protected]("blocker", ["none", "cooldown", "needs_action", "error", "retired", "missing"])
+def test_live_network_overlay_preserves_stronger_blockers(tmp_path, blocker):
+ async def run():
+ service, clock = clock_service(tmp_path)
+ source = service.store.load().sources[0]
+ await fail(service, source)
+ if blocker in {"cooldown", "needs_action", "error"}:
+ source.state = ModelHubSourceStateConfig(
+ status=blocker,
+ retry_at=(clock.now() + timedelta(seconds=60)).isoformat() if blocker == "cooldown" else None,
+ detail_key={
+ "cooldown": "models.source.cooldown.rate_limited",
+ "needs_action": "models.source.needs_action.credential_revoked",
+ "error": "models.source.error.unclassified",
+ }[blocker],
+ )
+ if blocker == "retired":
+ source.models[0].retired = True
+ if blocker == "missing":
+ service.store.config.sources = []
+ chain = service.agent_chain("codex", "shared-model")
+ _assert_valid("agent-chain.schema.json", chain)
+ hop = chain["chain"][0]
+ assert not hop["runnable"]
+ assert hop["health"] == {
+ "none": "backoff", "cooldown": "cooldown", "retired": "healthy",
+ "missing": "error", "needs_action": "needs_action", "error": "error",
+ }[blocker]
+ assert chain["supply_state"] == ("waiting" if blocker in {"none", "cooldown"} else "interrupted")
+ agents = {agent["backend"]: agent for agent in service.list_agents()}
+ assert agents["codex"]["supply_status"] == chain["supply_state"]
+ asyncio.run(run())
+
+
+def test_stale_duplicate_settlement_and_replacement_cannot_advance_or_unlock(tmp_path):
+ async def run():
+ service, clock = clock_service(tmp_path)
+ source = service.store.load().sources[0]
+ first = service._reserve_settlement_generation(source.id)
+ await fail(service, source, generation=first)
+ await fail(service, source, generation=first)
+ assert service.recovery._sources[source.id].failures == 1
+ clock.advance(1)
+ second = service._reserve_settlement_generation(source.id)
+ assert service.recovery.claim(source, second)
+ service.recovery.release(source.id, first)
+ assert service.recovery._sources[source.id].owner == second
+ await fail(service, source, generation=first)
+ assert service.recovery._sources[source.id].owner == second
+ old = copy.deepcopy(source)
+ source.credential_ref = "cred_replaced"
+ assert service.recovery.annotations(service.store.load()) == {}
+ await fail(service, old, generation=second)
+ assert service.recovery.annotations(service.store.load()) == {}
+ assert source.state.status == "standby"
+ asyncio.run(run())
+
+
+def test_fallback_success_does_not_clear_primary_and_eligibility_is_not_recovery(tmp_path):
+ async def run():
+ first, second = _source("src_primary01", "Primary"), _source("src_backup001", "Backup")
+ service, clock = clock_service(tmp_path, sources=[first, second], outcomes=[
+ _outcome(RawOutcomeKind.NETWORK_ERROR, source_id=first.id),
+ _outcome(RawOutcomeKind.SUCCESS, source_id=second.id, stream_started=True),
+ ])
+ result = await service.resolve_with_recovery(backend="codex", model_id="shared-model", request={})
+ assert result.source_id == second.id
+ assert clock.delays == []
+ assert first.id in service.recovery._sources
+ assert not any(event["kind"] == "recover" for event in service.events.list())
+ asyncio.run(run())
+
+
+def test_one_window_covers_all_fallback_passes_without_replaying_after_output(tmp_path):
+ """MH-RETRY-WINDOW-001: admission shares one clock across failed attempts."""
+ async def run():
+ service, clock = clock_service(tmp_path, outcomes=[
+ _outcome(RawOutcomeKind.NETWORK_ERROR, source_id="src_recovery01")
+ for _ in range(20)
+ ])
+ snapshots = []
+ with pytest.raises(ModelHubError):
+ await service.resolve_with_recovery(
+ backend="codex", model_id="shared-model", request={}, recovery_observer=snapshots.append,
+ )
+ assert clock.delays == [1, 2, 4, 8, 16, 30, 30]
+ assert len(service.adapter.invocations) == 8
+ assert clock.elapsed == 91
+ assert snapshots[-1] is None
+ assert {snapshot["window_end"] for snapshot in snapshots if snapshot} == {(NOW + timedelta(seconds=120)).isoformat()}
+ assert {snapshot["started_at"] for snapshot in snapshots if snapshot} == {NOW.isoformat()}
+ asyncio.run(run())
+
+
[email protected]("backend", ["codex", "claude", "opencode"])
+def test_startup_passes_temporary_hub_supply_without_spending_another_window(tmp_path, backend):
+ async def run():
+ service, clock = clock_service(tmp_path)
+ models = _canonicalize_fixed_test_routes(service)
+ source = service.store.load().sources[0]
+ await fail(service, source)
+ gateway = ModelHubTurnGateway(service)
+ router = ModelHubRuntimeRouter(service=service, turn_gateway=gateway, overlay_path=tmp_path / "overlay.json")
+ try:
+ model = models.get(backend, "shared-model")
+ launch = await router.resolve(backend, model, process_scope="/repo", turn_id="turn_startup")
+ assert launch.channel == "hub" and launch.source_id == source.id
+ if backend == "opencode":
+ overlay = await router.prepare_opencode_overlay()
+ assert "shared-model" in overlay.available_identifiers
+ assert clock.delays == []
+ assert service.adapter.invocations == []
+ assert gateway.correlation.recovery_snapshot("turn_startup") == []
+ finally:
+ await gateway.close()
+ asyncio.run(run())
+
+
[email protected]("protocol", WIRE)
[email protected]("streaming,verified,empty", [
+ (False, True, False), (True, True, False), (False, False, False),
+ (False, True, True), (True, True, True),
+])
+def test_real_gateway_retries_preoutput_then_preserves_one_response(tmp_path, protocol, streaming, verified, empty):
+ """MH-RETRY-RECOVERY-001 / D8: real loopback evidence, never timer-based recovery."""
+
+ async def run():
+ calls = []
+ metadata, output, terminal, buffered = WIRE[protocol]
+ if not verified:
+ buffered = b'{"unknown":"permissive compatibility response"}'
+ if empty:
+ output = b""
+ buffered = {
+ "openai_responses": b'{"object":"response","status":"completed","output":[]}',
+ "openai_chat": b'{"object":"chat.completion","choices":[]}',
+ "anthropic": b'{"type":"message","content":[]}',
+ }[protocol]
+
+ async def respond(request):
+ calls.append(await request.json())
+ if len(calls) <= 2:
+ return web.json_response({"error": {"type": "server_error"}}, status=503)
+ # An admitted recovery attempt may think beyond the 120s window.
+ clock.advance(121)
+ return web.Response(
+ body=metadata + output + terminal if streaming else buffered,
+ content_type="text/event-stream" if streaming else "application/json",
+ )
+
+ app = web.Application()
+ app.router.add_post("/{path:.*}", respond)
+ runner = web.AppRunner(app, handler_cancellation=True)
+ await runner.setup()
+ await web.TCPSite(runner, "127.0.0.1", 0).start()
+ backend, endpoint = {
+ "openai_responses": ("codex", "responses"),
+ "openai_chat": ("opencode", "chat/completions"),
+ "anthropic": ("claude", "messages"),
+ }[protocol]
+ source = _source("src_loopback01", "Loopback", protocol=protocol, vendor="anthropic" if backend == "claude" else "openai")
+ state = EngineStateStore(tmp_path / "engine")
+ source.credential_ref = state.store_api_key("fixture-key", vendor=source.vendor, protocol=protocol)
+ service, clock = clock_service(tmp_path, sources=[source])
+ models = _canonicalize_fixed_test_routes(service)
+ state.sync_sources(service._bindings(service.store.load()))
+ client = EngineClient(
+ EngineConnection(f"http://127.0.0.1:{runner.addresses[0][1]}", "fixture-management", "fixture-gateway"),
+ timeout=0.1,
+ )
+ adapter = CLIProxyEngineAdapter(supervisor=SimpleNamespace(client=lambda: client), state_store=state)
+ adapter.ensure_installed = service.adapter.ensure_installed
+ service.adapter = adapter
+ service._engine_synced = True
+ gateway = ModelHubTurnGateway(service)
+ base_url, token = await gateway.endpoint(
+ backend, process_scope="/repo", turn_id="turn_recovered",
+ requested_model_id=models.get(backend, "shared-model"),
+ resolved_model_id="shared-model", source_id=source.id,
+ )
+ try:
+ async with aiohttp.ClientSession(trust_env=False) as downstream:
+ async with downstream.post(
+ f"{base_url}/v1/{endpoint}",
+ headers={"Authorization": f"Bearer {token}"},
+ json={"model": "shared-model", "stream": streaming},
+ ) as response:
+ assert response.status == 200
+ assert await response.read() == (metadata + output + terminal if streaming else buffered)
+ assert len(calls) == 3
+ assert calls[0] == calls[1] == calls[2]
+ assert clock.delays == [30, 60]
+ assert adapter._active_transports == 0
+ assert gateway.correlation.recovery_snapshot("turn_recovered") == []
+ assert (source.id not in service.recovery._sources) is verified
+ assert sum(event["kind"] == "recover" for event in service.events.list()) == int(verified)
+ finally:
+ await gateway.close()
+ await runner.cleanup()
+ asyncio.run(run())
+
+
[email protected]("protocol", WIRE)
[email protected]("ending", ["cancel_owner", "slow_owner"])
+def test_real_gateway_has_one_half_open_owner_and_interruptible_waiters(tmp_path, protocol, ending):
+ """MH-RETRY-CONCURRENCY-001: real HTTP owners and waiters release only their slot."""
+ async def run():
+ connected, release, owner_closed, waiting = (asyncio.Event() for _ in range(4))
+ calls = []
+ buffered = WIRE[protocol][3]
+
+ async def respond(request):
+ calls.append(await request.json())
+ if len(calls) == 1:
+ connected.set()
+ try:
+ await release.wait()
+ finally:
+ owner_closed.set()
+ return web.Response(body=buffered, content_type="application/json")
+
+ app = web.Application()
+ app.router.add_post("/{path:.*}", respond)
+ runner = web.AppRunner(app, handler_cancellation=True)
+ await runner.setup()
+ await web.TCPSite(runner, "127.0.0.1", 0).start()
+ backend, endpoint = {
+ "openai_responses": ("codex", "responses"),
+ "openai_chat": ("opencode", "chat/completions"),
+ "anthropic": ("claude", "messages"),
+ }[protocol]
+ source = _source("src_concurrent1", "Concurrent", protocol=protocol)
+ state = EngineStateStore(tmp_path / "engine")
+ source.credential_ref = state.store_api_key("fixture-key", vendor=source.vendor, protocol=protocol)
+ service, clock = clock_service(tmp_path, sources=[source])
+ models = _canonicalize_fixed_test_routes(service)
+ await fail(service, source)
+ clock.advance(1)
+ state.sync_sources(service._bindings(service.store.load()))
+ client = EngineClient(
+ EngineConnection(f"http://127.0.0.1:{runner.addresses[0][1]}", "fixture-management", "fixture-gateway"),
+ timeout=0.1,
+ )
+ adapter = CLIProxyEngineAdapter(supervisor=SimpleNamespace(client=lambda: client), state_store=state)
+ adapter.ensure_installed = service.adapter.ensure_installed
+ service.adapter = adapter
+ service._engine_synced = True
+
+ async def wait_until_changed(delay):
+ waiting.set()
+ await asyncio.Future()
+
+ service.recovery.sleep = wait_until_changed
+ gateway = ModelHubTurnGateway(service)
+ model = models.get(backend, "shared-model")
+ base_url, first_token = await gateway.endpoint(
+ backend, process_scope="owner", turn_id="turn_owner",
+ requested_model_id=model, resolved_model_id="shared-model", source_id=source.id,
+ )
+ _, second_token = await gateway.endpoint(
+ backend, process_scope="waiter", turn_id="turn_waiter",
+ requested_model_id=model, resolved_model_id="shared-model", source_id=source.id,
+ )
+ tasks = []
+ try:
+ async with aiohttp.ClientSession(trust_env=False) as downstream:
+ def post(token):
+ return asyncio.create_task(downstream.post(
+ f"{base_url}/v1/{endpoint}",
+ headers={"Authorization": f"Bearer {token}"},
+ json={"model": "shared-model", "stream": False},
+ ))
+
+ owner = post(first_token)
+ tasks.append(owner)
+ await asyncio.wait_for(connected.wait(), 2)
+ generation = service.recovery._sources[source.id].owner
+ assert generation is not None
+ waiter = post(second_token)
+ tasks.append(waiter)
+ await asyncio.wait_for(waiting.wait(), 2)
+ assert len(calls) == 1 and not waiter.done()
+ chain = service.agent_chain(backend, model)
+ assert chain["chain"][0]["recovery"] == "in_flight"
+ assert chain["chain"][0]["runnable"] is False
+ assert chain["supply_state"] == "waiting"
+ _assert_valid("agent-chain.schema.json", chain)
+ snapshot = gateway.correlation.recovery_snapshot("turn_waiter")
+ if backend == "opencode":
+ # Shared-server requests have no exact Turn owner yet.
+ assert snapshot == []
+ else:
+ assert len(snapshot) == 1 and snapshot[0]["phase"] == "waiting"
+ if ending == "cancel_owner":
+ owner.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await owner
+ await asyncio.wait_for(owner_closed.wait(), 2)
+ response = await asyncio.wait_for(waiter, 2)
+ assert response.status == 200 and await response.read() == buffered
+ assert len(calls) == 2
+ else:
+ clock.advance(121)
+ service.recovery.notify()
+ response = await asyncio.wait_for(waiter, 2)
+ assert response.status == (400 if backend == "codex" else 424)
+ assert "Retry-After" not in response.headers
+ assert await response.json() == {
+ "type": "error",
+ "error": {
+ "type": RECOVERY_EXHAUSTED_CODE,
+ "code": RECOVERY_EXHAUSTED_CODE,
+ "message": RECOVERY_EXHAUSTED_MESSAGE,
+ },
+ }
+ assert len(calls) == 1 and not owner.done()
+ assert service.recovery._sources[source.id].owner == generation
+ release.set()
+ response = await asyncio.wait_for(owner, 2)
+ assert response.status == 200 and await response.read() == buffered
+ await gateway._drain_turn_requests("turn_owner")
+ await gateway._drain_turn_requests("turn_waiter")
+ assert adapter._active_transports == 0
+ assert source.id not in service.recovery._sources
+ assert gateway.correlation.recovery_snapshot("turn_waiter") == []
+ assert sum(event["kind"] == "recover" for event in service.events.list()) == 1
+ finally:
+ release.set()
+ for task in tasks:
+ task.cancel()
+ await asyncio.gather(*tasks, return_exceptions=True)
+ await gateway.close()
+ await runner.cleanup()
+ asyncio.run(run())
+
+
+def test_permissive_success_releases_admission_without_verified_recovery(tmp_path):
+ async def run():
+ service, clock = clock_service(tmp_path, outcomes=[
+ _outcome(RawOutcomeKind.SUCCESS, source_id="src_recovery01", stream_started=True),
+ ])
+ source = service.store.load().sources[0]
+ source.verification_pending = "vp_fixture"
+ await fail(service, source)
+ clock.advance(1)
+ result = await service.resolve_with_recovery(backend="codex", model_id="shared-model", request={})
+ assert result.outcome.kind is RawOutcomeKind.SUCCESS
+ entry = service.recovery._sources[source.id]
+ assert entry.owner is None and entry.failures == 1
+ assert service.agent_chain("codex", "shared-model")["chain"][0]["recovery"] == "eligible"
+ assert not any(event["kind"] == "recover" for event in service.events.list())
+ asyncio.run(run())
+
+
+def test_draft_preview_does_not_clear_live_health_and_mixed_action_blocks_waiting(tmp_path):
+ async def run():
+ first, second = _source("src_primary01", "Primary"), _source("src_backup001", "Backup")
+ service, clock = clock_service(tmp_path, sources=[first, second])
+ await fail(service, first)
+ draft = copy.deepcopy(service.store.load())
+ draft.sources[0].credential_ref = "cred_draft"
+ assert first.id not in service.recovery_annotations(draft)
+ assert first.id in service.recovery_annotations(service.store.load())
+ second.state = ModelHubSourceStateConfig(
+ status="needs_action", detail_key="models.source.needs_action.credential_revoked",
+ )
+ with pytest.raises(ModelHubError) as failed:
+ await service.resolve_with_recovery(backend="codex", model_id="shared-model", request={})
+ assert failed.value.supply_state == "interrupted"
+ assert clock.delays == [] and service.adapter.invocations == []
+ asyncio.run(run())
+
+
[email protected]("read", ["agent_chain", "agent_chains", "get_agent_sources", "list_agents"])
+def test_public_live_reads_capture_config_and_health_once(tmp_path, read):
+ async def run():
+ service, _clock = clock_service(tmp_path)
+ await fail(service, service.store.load().sources[0])
+ counts = {"config": 0, "health": 0}
+ load, annotations = service.store.load, service.recovery.annotations
+
+ def capture_config():
+ counts["config"] += 1
+ return load()
+
+ def capture_health(config):
+ counts["health"] += 1
+ return annotations(config)
+
+ service.store.load = capture_config
+ service.recovery.annotations = capture_health
+ args = {
+ "agent_chain": ("codex", "shared-model"),
+ "agent_chains": ("codex",),
+ "get_agent_sources": ("codex",),
+ "list_agents": (),
+ }[read]
+ result = getattr(service, read)(*args)
+ assert result
+ assert counts == {"config": 1, "health": 1}
+ asyncio.run(run())
+
+
+def test_half_open_projection_cannot_hide_action_or_native_blockers(tmp_path):
+ async def run():
+ service, clock = clock_service(tmp_path)
+ source = service.store.load().sources[0]
+ await fail(service, source)
+ clock.advance(1)
+ generation = service._reserve_settlement_generation(source.id)
+ assert service.recovery.claim(source, generation)
+ chain = service.agent_chain("codex", "shared-model")
+ assert chain["chain"][0]["recovery"] == "in_flight"
+ _assert_valid("agent-chain.schema.json", chain)
+ for mutation in [
+ {"runnable": True},
+ {"health": "needs_action", "reason": "models.source.needs_action.credential_revoked"},
+ {"reason": "source_missing"},
+ {"channel": "native_cli", "reason": "native_cli_unavailable"},
+ {"recovery": "eligible"},
+ ]:
+ invalid = copy.deepcopy(chain)
+ invalid["chain"][0].update(mutation)
+ with pytest.raises(AssertionError):
+ _assert_valid("agent-chain.schema.json", invalid)
+ source.state = ModelHubSourceStateConfig(
+ status="needs_action", detail_key="models.source.needs_action.credential_revoked",
+ )
+ blocked = service.agent_chain("codex", "shared-model")
+ assert "recovery" not in blocked["chain"][0]
+ assert blocked["supply_state"] == "interrupted"
+ _assert_valid("agent-chain.schema.json", blocked)
+ for phase in ("eligible", "in_flight"):
+ blocked["chain"][0]["recovery"] = phase
+ with pytest.raises(AssertionError):
+ _assert_valid("agent-chain.schema.json", blocked)
+ asyncio.run(run())
+
+
[email protected]("kind,suffix", [
+ (RawOutcomeKind.NETWORK_ERROR, "network"), (RawOutcomeKind.TIMEOUT, "timeout"),
+])
+def test_selected_source_probe_does_not_claim_or_schedule_recovery(tmp_path, kind, suffix):
+ async def run():
+ service, clock = clock_service(tmp_path, outcomes=[_outcome(kind, source_id="src_recovery01")])
+ before = copy.deepcopy(service.store.load().to_payload())
+ result = await service.probe_source("src_recovery01", {"model": "shared-model"})
+ assert result["reachable"] is False
+ assert result["error"] == f"models.source.cooldown.{suffix}"
+ assert len(service.adapter.invocations) == 1
+ assert service.recovery._sources == {} and clock.delays == []
+ assert service.store.load().to_payload() == before
+ assert service.events.list() == []
+ asyncio.run(run())
diff --git a/tests/test_model_hub_structure_guards.py b/tests/test_model_hub_structure_guards.py
index 4e1ff38dc..ed94cf9c4 100644
--- a/tests/test_model_hub_structure_guards.py
+++ b/tests/test_model_hub_structure_guards.py
@@ -465,7 +465,7 @@ def test_g3_settlement_returns_are_consumed() -> None:
def test_settlement_generations_are_reserved_only_at_attempt_start() -> None:
allowed_owners = {
- SERVICE: {"admitted"},
+ SERVICE: {"_invoke_admitted"},
ROUTER: {"resolve"},
}
for path, expected in allowed_owners.items():
@@ -483,16 +483,24 @@ def test_settlement_generations_are_reserved_only_at_attempt_start() -> None:
}
assert owners == expected
if path == SERVICE:
- admission_owners = set()
- for node in ast.walk(tree):
- if not isinstance(node, ast.Call) or _call_name(node) != "_reserve_settlement_generation":
- continue
- parent = parents[node]
- while not isinstance(parent, ast.FunctionDef):
- parent = parents[parent]
- assert parent.name == "admitted"
- admission_owners.add(_owner_name(parent, parents))
- assert admission_owners == {"_probe_agent_once", "resolve"}
+ admission = _functions(tree)["_invoke_admitted"]
+ source = ast.get_source_segment(path.read_text(encoding="utf-8"), admission)
+ assert source is not None
+ # The generation is reserved under the same mutation exclusion as
+ # candidate revalidation and the half-open claim, before invocation.
+ acquire_at = source.index("await self._mutation_lock.acquire()")
+ revalidate_at = source.index("self._invocation_resolution(", acquire_at)
+ reserve_at = source.index("self._reserve_settlement_generation(", revalidate_at)
+ claim_at = source.index("self.recovery.claim(", reserve_at)
+ invoke_at = source.index("self.adapter.invoke(", claim_at)
+ assert acquire_at < revalidate_at < reserve_at < claim_at < invoke_at
+ admitted = _functions(admission)["admitted"]
+ assert any(
+ isinstance(node, ast.Call)
+ and _call_name(node) == "on_admitted"
+ and [ast.unparse(arg) for arg in node.args] == ["generation"]
+ for node in ast.walk(admitted)
+ )
settlement_owner = _functions(_tree(SERVICE))["_settle_fallback_source"]
assert not any(
@@ -1371,7 +1379,7 @@ def test_malformed_stream_data_is_transparent_to_a_later_terminal(data: bytes) -
state.observe(
b'event: response.completed\ndata: {"type":"response.completed","sequence_number":1}\n\n'
)
- assert state.terminal_observation() == ProtocolObservation(outcome="served")
+ assert state.terminal_observation() == ProtocolObservation(outcome="served", recovery_verified=True)
@pytest.mark.parametrize("next_sequence", (1, 0))
@@ -1384,7 +1392,7 @@ def test_responses_sequence_order_is_ignored_for_settlement(next_sequence: int)
+ b"}\n\n"
)
state.observe(b'event: response.completed\ndata: {"type":"response.completed"}\n\n')
- assert state.terminal_observation() == ProtocolObservation(outcome="served")
+ assert state.terminal_observation() == ProtocolObservation(outcome="served", recovery_verified=True)
def test_responses_sequence_numbers_accept_a_strictly_increasing_stream() -> None:
@@ -1393,14 +1401,14 @@ def test_responses_sequence_numbers_accept_a_strictly_increasing_stream() -> Non
state.observe(
b'event: response.completed\ndata: {"type":"response.completed","sequence_number":1}\n\n'
)
- assert state.terminal_observation() == ProtocolObservation(outcome="served")
+ assert state.terminal_observation() == ProtocolObservation(outcome="served", recovery_verified=True)
def test_responses_missing_sequence_is_ignored_for_settlement() -> None:
state = ProtocolSSEState("openai_responses")
state.observe(b'event: response.created\ndata: {"type":"response.created"}\n\n')
state.observe(b'event: response.completed\ndata: {"type":"response.completed"}\n\n')
- assert state.terminal_observation() == ProtocolObservation(outcome="served")
+ assert state.terminal_observation() == ProtocolObservation(outcome="served", recovery_verified=True)
@pytest.mark.parametrize(
@@ -1417,7 +1425,7 @@ def test_spec_ignorable_stream_frames_do_not_poison_terminal_proof(prefix: bytes
state.observe(
b'event: response.completed\ndata: {"type":"response.completed","sequence_number":0}\n\n'
)
- assert state.terminal_observation() == ProtocolObservation(outcome="served")
+ assert state.terminal_observation() == ProtocolObservation(outcome="served", recovery_verified=True)
@pytest.mark.parametrize("protocol", tuple(BUFFERED_ERROR_TRUST_ROOT_FIXTURES))
@@ -1519,7 +1527,7 @@ def test_chat_projection_facts_do_not_grow_with_choice_count() -> None:
)
assert projector.finish(streamed=True).model_output_started is True
- assert projector._nonempty == {(), ("choices", "*", "delta", "content")}
+ assert projector._nonempty == {(), ("choices",), ("choices", "*", "delta", "content")}
assert projector._scoped_nonempty == set()
@@ -1702,7 +1710,7 @@ def test_observer_abandons_large_non_string_metadata_without_affecting_next_fram
)
assert state.model_output_started is False
- assert state.terminal_observation() == ProtocolObservation(outcome="served")
+ assert state.terminal_observation() == ProtocolObservation(outcome="served", recovery_verified=True)
def test_large_terminal_projects_its_discriminator_after_unrelated_structure() -> None:
@@ -1763,7 +1771,7 @@ def test_initial_utf8_bom_is_normalized_for_terminal_observation(split_at: int)
payload = b'\xef\xbb\xbfevent: response.completed\ndata: {"type":"response.completed"}\n\n'
state.observe(payload[:split_at])
state.observe(payload[split_at:])
- assert state.terminal_observation() == ProtocolObservation(outcome="served")
+ assert state.terminal_observation() == ProtocolObservation(outcome="served", recovery_verified=True)
def test_complete_frame_after_terminal_cannot_change_the_fact() -> None:
@@ -1775,7 +1783,7 @@ def test_complete_frame_after_terminal_cannot_change_the_fact() -> None:
state = ProtocolSSEState("openai_responses")
state.observe(b'event: response.completed\ndata: {"type":"response.completed"}\n\n')
state.observe(frame)
- assert state.terminal_observation() == ProtocolObservation(outcome="served")
+ assert state.terminal_observation() == ProtocolObservation(outcome="served", recovery_verified=True)
@pytest.mark.parametrize(
diff --git a/tests/test_opencode_model_hub_retry.py b/tests/test_opencode_model_hub_retry.py
new file mode 100644
index 000000000..4721619a4
--- /dev/null
+++ b/tests/test_opencode_model_hub_retry.py
@@ -0,0 +1,199 @@
+"""OpenCode consumes Hub finality without automatically replaying a Turn."""
+
+from __future__ import annotations
+
+import asyncio
+import copy
+import json
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+
+from modules.agents.base import AgentRequest
+from modules.agents.model_hub import ModelHubLaunch, bind_launch
+from modules.agents.opencode.poll_loop import (
+ OpenCodePollLoop,
+ _is_model_hub_recovery_exhausted,
+)
+from modules.im import MessageContext
+from vibe.i18n import t
+
+
+def _context(channel="hub", backend="opencode"):
+ context = MessageContext(user_id="user", channel_id="channel", platform="slack")
+ if channel is not None:
+ bind_launch(context, ModelHubLaunch(
+ backend=backend,
+ channel=channel,
+ requested_model="model",
+ target_model="model",
+ runtime_model="model",
+ source_id="source",
+ ))
+ return context
+
+
+def _error():
+ return {
+ "name": "APIError",
+ "data": {
+ "statusCode": 424,
+ "isRetryable": False,
+ "message": "Automatic recovery has ended. Try again or choose another model.",
+ "responseBody": json.dumps({
+ "type": "error",
+ "error": {
+ "type": "model_hub_recovery_exhausted",
+ "code": "model_hub_recovery_exhausted",
+ "message": "Automatic recovery has ended. Try again or choose another model.",
+ },
+ }),
+ },
+ }
+
+
+def test_exact_native_error_recognizes_hub_finality():
+ assert _is_model_hub_recovery_exhausted(_context(), _error())
+
+
[email protected]("channel", [None, "direct", "native_cli"])
+def test_same_error_cannot_change_non_hub_retry_policy(channel):
+ assert not _is_model_hub_recovery_exhausted(_context(channel), _error())
+
+
[email protected]("backend", ["codex", "claude"])
+def test_other_backend_launch_does_not_prove_opencode_finality(backend):
+ assert not _is_model_hub_recovery_exhausted(_context(backend=backend), _error())
+
+
[email protected]("replacement", [None, [], "model_hub_recovery_exhausted", {}, {"name": "UnknownError"}])
+def test_non_structured_native_error_retains_normal_policy(replacement):
+ assert not _is_model_hub_recovery_exhausted(_context(), replacement)
+
+
[email protected]("status", [None, "424", 424.0, True, 400, 429, 503])
+def test_only_exact_terminal_http_status_qualifies(status):
+ error = _error()
+ error["data"]["statusCode"] = status
+ assert not _is_model_hub_recovery_exhausted(_context(), error)
+
+
+ None, {}, "invalid", "[]", "null", "{}", "[" * 1100 + "]" * 1100,
+ " " * 4097,
+ '{"error":{"type":"model_hub_recovery_exhausted","code":"model_hub_recovery_exhausted"}}',
+ '{"type":"error","error":{"code":"model_hub_recovery_exhausted"}}',
+ '{"type":"error","error":{"type":"model_hub_recovery_exhausted"}}',
+ '{"type":"error","error":{"type":"model_hub_recovery_exhausted","code":"server_error"}}',
+])
+def test_diagnostic_words_without_exact_wire_evidence_do_not_suppress_retry(body):
+ error = _error()
+ error["data"]["responseBody"] = body
+ assert not _is_model_hub_recovery_exhausted(_context(), error)
+
+
+async def _poll(monkeypatch, *, channel="hub", error=None, baseline=False, busy=False, language="en"):
+ monkeypatch.setattr("modules.agents.opencode.poll_loop._POLL_INTERVAL_SECONDS", 0)
+ controller = SimpleNamespace(
+ config=SimpleNamespace(language=language),
+ agent_auth_service=SimpleNamespace(maybe_emit_auth_recovery_message=AsyncMock(return_value=False)),
+ emit_agent_message=AsyncMock(),
+ )
+ agent = SimpleNamespace(
+ controller=controller,
+ opencode_config=SimpleNamespace(error_retry_limit=2, active_turn_timeout_seconds=0),
+ record_model_hub_native_failure=AsyncMock(),
+ _extract_response_text=lambda message: message["parts"][0]["text"] if message["parts"] else "",
+ )
+ failed = {
+ "info": {
+ "id": "failed",
+ "role": "assistant",
+ "time": {"completed": 1},
+ "error": copy.deepcopy(error if error is not None else _error()),
+ },
+ "parts": [],
+ }
+ success = {
+ "info": {"id": "later", "role": "assistant", "time": {"completed": 2}, "finish": "stop"},
+ "parts": [{"type": "text", "text": "later legitimate output"}],
+ }
+ first_snapshot = [failed]
+ if busy:
+ first_snapshot.append({
+ "info": {"id": "pending-inject", "role": "user", "time": {}},
+ "parts": [{"type": "text", "text": "new legitimate input"}],
+ })
+ server = SimpleNamespace(
+ list_messages=AsyncMock(side_effect=[first_snapshot, [failed, success]]),
+ get_session_status=AsyncMock(return_value={"type": "busy"} if busy else None),
+ prompt_async=AsyncMock(),
+ )
+ request = AgentRequest(
+ context=_context(channel),
+ message="original prompt",
+ user_message="original prompt",
+ working_path="/tmp/model-hub-fixture",
+ base_session_id="base",
+ composite_session_id="base:fixture",
+ session_key="slack::channel",
+ )
+ result = await asyncio.wait_for(
+ OpenCodePollLoop(agent).run_prompt_poll(
+ request,
+ server,
+ "native-session",
+ agent_to_use=None,
+ model_dict={"providerID": "provider", "modelID": "model"},
+ reasoning_effort=None,
+ baseline_message_ids={"failed"} if baseline else set(),
+ ),
+ timeout=2,
+ )
+ return result, server, agent, controller
+
+
[email protected]("language", ["en", "zh"])
+async def test_hub_terminal_skips_automatic_continue_and_emits_one_failure(monkeypatch, language):
+ """MH-RETRY-NATIVE-001: one Hub terminal settles without an automatic Turn replay."""
+ result, server, agent, controller = await _poll(monkeypatch, language=language)
+ assert result == (None, False)
+ server.prompt_async.assert_not_awaited()
+ agent.record_model_hub_native_failure.assert_awaited_once()
+ calls = controller.emit_agent_message.await_args_list
+ assert [call.args[1] for call in calls] == ["notify", "result"]
+ assert sum(call.args[1] == "notify" for call in calls) == 1
+ assert sum(bool(call.kwargs["output"].completes_turn) for call in calls) == 1
+ # The shared OpenCode route need not have exact Turn/source attribution to
+ # display the closed request result already proved by the native error.
+ assert calls[0].args[2] == t("modelHub.recovery.ended", language)
+ assert "APIError" not in calls[0].args[2]
+ assert "APIError" in calls[-1].kwargs["terminal_error"]
+
+
[email protected]("channel", [None, "direct", "native_cli"])
+async def test_ordinary_native_failure_still_uses_existing_continue(monkeypatch, channel):
+ result, server, agent, controller = await _poll(monkeypatch, channel=channel)
+ assert result == ("later legitimate output", True)
+ server.prompt_async.assert_awaited_once()
+ assert server.prompt_async.await_args.kwargs["text"] == "continue"
+ agent.record_model_hub_native_failure.assert_not_awaited()
+ controller.emit_agent_message.assert_not_awaited()
+
+
+async def test_unrelated_hub_error_keeps_existing_retry(monkeypatch):
+ error = _error()
+ error["data"]["statusCode"] = 503
+ result, server, agent, _controller = await _poll(monkeypatch, error=error)
+ assert result == ("later legitimate output", True)
+ server.prompt_async.assert_awaited_once()
+ agent.record_model_hub_native_failure.assert_not_awaited()
+
+
[email protected]("guard", ["baseline", "busy"])
+async def test_old_or_still_live_assistant_is_not_terminalized(monkeypatch, guard):
+ result, server, agent, _controller = await _poll(monkeypatch, **{guard: True})
+ assert result == ("later legitimate output", True)
+ server.prompt_async.assert_not_awaited()
+ agent.record_model_hub_native_failure.assert_not_awaited()
diff --git a/ui/src/components/workbench/AgentActivityGroup.tsx b/ui/src/components/workbench/AgentActivityGroup.tsx
index 9f11f5415..b5a6ac3e0 100644
--- a/ui/src/components/workbench/AgentActivityGroup.tsx
+++ b/ui/src/components/workbench/AgentActivityGroup.tsx
@@ -397,6 +397,7 @@ export const ActivityCard: React.FC<{
showToolCalls: boolean;
onToggleTools: () => void;
onDisableActivity?: () => void;
+ statusLabel?: string | null;
}> = ({
rows,
startedAtMs,
@@ -405,6 +406,7 @@ export const ActivityCard: React.FC<{
showToolCalls,
onToggleTools,
onDisableActivity,
+ statusLabel,
}) => {
const { t } = useTranslation();
const [nowMs, setNowMs] = useState(() => Date.now());
@@ -465,7 +467,7 @@ export const ActivityCard: React.FC<{
className="flex min-w-0 flex-1 items-center gap-2 text-left"
>
<Loader2 className="size-3.5 shrink-0 animate-spin text-mint-ink" aria-hidden="true" />
- <span className="text-[12px] font-medium text-mint-ink">{t('chat.agentActivity.running')}</span>
+ <span className="text-[12px] font-medium text-mint-ink">{statusLabel || t('chat.agentActivity.running')}</span>
{rows.length > 0 && <span className="text-[12px] text-muted">· {stepLabel(t, rows.length)}</span>}
<span className="ml-auto shrink-0 font-mono text-[11px] text-muted">{clock}</span>
</button>
diff --git a/ui/src/components/workbench/ChatPage.hydration.test.tsx b/ui/src/components/workbench/ChatPage.hydration.test.tsx
index 4ec232c9f..500ab7632 100644
--- a/ui/src/components/workbench/ChatPage.hydration.test.tsx
+++ b/ui/src/components/workbench/ChatPage.hydration.test.tsx
@@ -4,9 +4,12 @@ import { act, cleanup, fireEvent, render, screen, waitFor } from '@testing-libra
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { MemoryRouter, Route, Routes, useNavigate } from 'react-router-dom';
import type { TurnActivityGroupWire } from '../../lib/agentActivity';
+import type { ModelRecoveryState, SessionRuntimeState } from '../../context/ApiContext';
+import type { ComposerProps } from './Composer';
const mocks = vi.hoisted(() => ({
api: {
+ cancelSession: vi.fn(),
connectWorkbenchEvents: vi.fn(),
getCachedSessionDraft: vi.fn(),
getSession: vi.fn(),
@@ -22,6 +25,7 @@ const mocks = vi.hoisted(() => ({
onSessionArchived: vi.fn(),
},
events: null as null | {
+ onSessionActivity: (data: { session_id: string; event: string }) => void;
onConnected: () => void;
onAuthorizationChanged: (data: {
resource_kinds?: string[];
@@ -31,6 +35,7 @@ const mocks = vi.hoisted(() => ({
onTurnStart: (data: { session_id: string }) => void;
onTurnEnd: (data: { session_id: string }) => void;
},
+ composer: null as ComposerProps | null,
authorizationCapabilities: {
can_chat: true,
can_manage_instance: false,
@@ -109,7 +114,10 @@ vi.mock('./useShowPageAnnotation', () => ({
}));
vi.mock('./Composer', () => ({
- Composer: () => null,
+ Composer: (props: ComposerProps) => {
+ mocks.composer = props;
+ return null;
+ },
}));
import { ChatPage } from './ChatPage';
@@ -130,7 +138,8 @@ const deferred = <T,>(): Deferred<T> => {
return { promise, resolve, reject };
};
-const idleTurnState = {
+const idleTurnState: SessionRuntimeState = {
+ in_flight: false,
foreground: 'idle',
native_turn_started: false,
pending_input_count: 0,
@@ -145,7 +154,7 @@ const bootstrapPayload = (sessionId: string) => ({
agents: [],
default_agent_name: null,
config: { ui: {} },
- messages: [],
+ messages: [] as ReturnType<typeof projectedMessage>[],
next_before_id: null,
turn_state: idleTurnState,
queued: [],
@@ -204,6 +213,7 @@ describe('ChatPage transcript hydration', () => {
sessionRow = deferred();
bootstrap = deferred();
mocks.events = null;
+ mocks.composer = null;
vi.clearAllMocks();
Object.assign(mocks.authorizationCapabilities, {
can_chat: true,
@@ -217,6 +227,7 @@ describe('ChatPage transcript hydration', () => {
mocks.events = events;
return () => {};
});
+ mocks.api.cancelSession.mockResolvedValue({ ok: true });
mocks.api.getCachedSessionDraft.mockReturnValue(null);
mocks.api.getSession.mockReturnValue(sessionRow.promise);
mocks.api.getSessionActivity.mockResolvedValue({ groups: [] });
@@ -234,6 +245,7 @@ describe('ChatPage transcript hydration', () => {
afterEach(() => {
cleanup();
vi.unstubAllGlobals();
+ vi.useRealTimers();
});
it('keeps the loading view when SSE Session-row recovery beats transcript bootstrap', async () => {
@@ -1107,4 +1119,201 @@ describe('ChatPage transcript hydration', () => {
expect(screen.queryByRole('button', { name: 'chat.agentActivity.disable' })).toBeNull();
expect(mocks.api.mutateConfig).not.toHaveBeenCalled();
});
+
+ describe('quiet model recovery in the existing progress slots', () => {
+ const now = Date.parse('2026-09-09T04:00:10Z');
+ const recovery = (overrides: Partial<ModelRecoveryState> = {}): ModelRecoveryState => ({
+ request_id: 'pending-request',
+ phase: 'waiting',
+ attempt_count: 1,
+ started_at: new Date(now - 6000).toISOString(),
+ window_end: new Date(now + 114_000).toISOString(),
+ next_eligible_at: new Date(now + 24_000).toISOString(),
+ source_id: 'private-source',
+ reason: 'https://private-host.invalid/diagnostic',
+ ...overrides,
+ });
+ const running = (model_recovery: ModelRecoveryState[] = [recovery()]): SessionRuntimeState => ({
+ ...idleTurnState, in_flight: true, foreground: 'running', model_recovery,
+ });
+ const invalidate = () => mocks.events?.onSessionActivity({ session_id: 'session-new', event: 'model_recovery' });
+ const mount = async (state = running(), activity = false) => {
+ mocks.api.getSession.mockResolvedValue({ id: 'session-new' });
+ mocks.api.getSessionBootstrap.mockResolvedValue({
+ ...bootstrapPayload('session-new'),
+ config: { ui: { show_agent_activity: activity } },
+ draft: { text: 'unsent draft 草稿' },
+ turn_state: state,
+ });
+ mocks.api.getTurnState.mockResolvedValue(state);
+ const view = render(
+ <MemoryRouter initialEntries={['/chat/session-new']}>
+ <SessionSwitcher />
+ <SessionSwitcher sessionId="session-new" label="return chat" />
+ <Routes><Route path="/chat/:sessionId" element={<ChatPage />} /></Routes>
+ </MemoryRouter>,
+ );
+ await act(async () => Promise.resolve());
+ expect(screen.queryByText('common.loading')).toBeNull();
+ return view;
+ };
+ beforeEach(() => {
+ vi.useFakeTimers();
+ vi.setSystemTime(now);
+ });
+
+ it.each([false, true])('reuses the label with Activity enabled=%s, preserving controls and rows', async (activity) => {
+ const { container } = await mount(running([recovery({ started_at: new Date(now).toISOString() })]), activity);
+ if (activity) {
+ act(() => mocks.events?.onMessageNew({
+ ...projectedMessage('step', 'existing tool narration'), author: 'agent', type: 'assistant', source: 'agent',
+ }));
+ }
+ const generic = activity ? 'chat.agentActivity.running' : 'chat.thinking';
+ const labelSlot = screen.getByText(generic);
+ const structure = container.querySelectorAll('*').length;
+ expect(container.querySelectorAll('[data-message-id]')).toHaveLength(0);
+ act(() => vi.advanceTimersByTime(4999));
+ expect(screen.queryByText('chat.modelRecovery.waitingSeconds')).toBeNull();
+ act(() => vi.advanceTimersByTime(1));
+ expect(screen.getByText('chat.modelRecovery.waitingSeconds')).toBe(labelSlot);
+ expect(container.querySelectorAll('*')).toHaveLength(structure);
+ expect(container.querySelectorAll('[data-message-id]')).toHaveLength(0);
+ expect(container.textContent).not.toMatch(/private-source|https:|2026-09|UTC/);
+ expect(mocks.composer?.busy).toBe(true);
+ expect(mocks.composer?.initialDraft).toBe('unsent draft 草稿');
+ if (activity) expect(screen.getByText('existing tool narration')).toBeTruthy();
+ else expect(container.querySelectorAll('.vr-typing-dot')).toHaveLength(3);
+
+ await act(async () => mocks.composer?.onStop?.());
+ expect(mocks.api.cancelSession).toHaveBeenCalledExactlyOnceWith('session-new');
+ expect(mocks.composer?.busy).toBe(true);
+ expect(screen.getByText('chat.modelRecovery.waitingSeconds')).toBeTruthy();
+ });
+
+ it('restores a mature snapshot immediately, follows invalidations, and clears without ending work', async () => {
+ await mount();
+ expect(screen.getAllByText('chat.modelRecovery.waitingSeconds')).toHaveLength(1);
+ mocks.api.getTurnState.mockResolvedValue(running([recovery({ phase: 'attempting' })]));
+ await act(async () => invalidate());
+ expect(screen.getByText('chat.modelRecovery.attempting')).toBeTruthy();
+ mocks.api.getTurnState.mockResolvedValue(running([]));
+ await act(async () => invalidate());
+ expect(screen.queryByText('chat.modelRecovery.attempting')).toBeNull();
+ expect(screen.getByText('chat.thinking')).toBeTruthy();
+ expect(mocks.composer?.busy).toBe(true);
+ expect(mocks.composer?.initialDraft).toBe('unsent draft 草稿');
+ });
+
+ it('keeps the mature label when the first activity row replaces the thinking bubble', async () => {
+ const { container } = await mount(running(), true);
+ expect(container.querySelectorAll('.vr-typing-dot')).toHaveLength(3);
+ expect(screen.getByText('chat.modelRecovery.waitingSeconds')).toBeTruthy();
+ act(() => mocks.events?.onMessageNew({
+ ...projectedMessage('step', 'first activity'), author: 'agent', type: 'assistant', source: 'agent',
+ }));
+ expect(container.querySelectorAll('.vr-typing-dot')).toHaveLength(0);
+ expect(screen.getByText('first activity')).toBeTruthy();
+ expect(screen.getAllByText('chat.modelRecovery.waitingSeconds')).toHaveLength(1);
+ });
+
+ it('does not hide pending peer recovery when a terminal message arrives', async () => {
+ await mount();
+ act(() => mocks.events?.onMessageNew({
+ ...projectedMessage('peer-result', 'one request finished'),
+ author: 'agent', type: 'result', source: 'agent',
+ }));
+ expect(screen.getByText('one request finished')).toBeTruthy();
+ expect(screen.getByText('chat.modelRecovery.waitingSeconds')).toBeTruthy();
+ expect(mocks.composer?.busy).toBe(true);
+ });
+
+ it('ignores other sessions and rejects an older wait after a newer clear response', async () => {
+ await mount();
+ const stale = deferred<SessionRuntimeState>();
+ mocks.api.getTurnState.mockReturnValueOnce(stale.promise).mockResolvedValue(running([]));
+ await act(async () => mocks.events?.onSessionActivity({ session_id: 'elsewhere', event: 'model_recovery' }));
+ expect(mocks.api.getTurnState).not.toHaveBeenCalled();
+ act(invalidate);
+ await act(async () => invalidate());
+ expect(screen.getByText('chat.thinking')).toBeTruthy();
+ await act(async () => stale.resolve(running()));
+ expect(screen.queryByText('chat.modelRecovery.waitingSeconds')).toBeNull();
+ expect(mocks.composer?.busy).toBe(true);
+ });
+
+ it.each(['start', 'end'] as const)('clears on turn %s and rejects an in-flight old snapshot', async (event) => {
+ await mount();
+ const stale = deferred<SessionRuntimeState>();
+ mocks.api.getTurnState.mockReturnValueOnce(stale.promise).mockResolvedValue(idleTurnState);
+ act(invalidate);
+ await act(async () => {
+ if (event === 'start') mocks.events?.onTurnStart({ session_id: 'session-new' });
+ else mocks.events?.onTurnEnd({ session_id: 'session-new' });
+ });
+ expect(screen.queryByText('chat.modelRecovery.waitingSeconds')).toBeNull();
+ await act(async () => stale.resolve(running()));
+ expect(screen.queryByText('chat.modelRecovery.waitingSeconds')).toBeNull();
+ expect(mocks.composer?.busy).toBe(event === 'start');
+ });
+
+ it.each(['start', 'end'] as const)('does not let a delayed bootstrap revive recovery after turn %s', async (event) => {
+ const stale = deferred<ReturnType<typeof bootstrapPayload>>();
+ mocks.api.getSession.mockResolvedValue({ id: 'session-new' });
+ mocks.api.getSessionBootstrap.mockReturnValue(stale.promise);
+ render(
+ <MemoryRouter initialEntries={['/chat/session-new']}>
+ <Routes><Route path="/chat/:sessionId" element={<ChatPage />} /></Routes>
+ </MemoryRouter>,
+ );
+ await act(async () => Promise.resolve());
+ await act(async () => {
+ if (event === 'start') mocks.events?.onTurnStart({ session_id: 'session-new' });
+ else mocks.events?.onTurnEnd({ session_id: 'session-new' });
+ });
+ await act(async () => stale.resolve({ ...bootstrapPayload('session-new'), turn_state: running() }));
+ expect(screen.queryByText('common.loading')).toBeNull();
+ expect(screen.queryByText('chat.modelRecovery.waitingSeconds')).toBeNull();
+ expect(mocks.composer?.busy).toBe(event === 'start');
+ });
+
+ it('rejects a stale response through A → B → A navigation', async () => {
+ await mount();
+ const stale = deferred<SessionRuntimeState>();
+ mocks.api.getTurnState.mockReturnValueOnce(stale.promise);
+ act(invalidate);
+ mocks.api.getSessionBootstrap.mockImplementation((id: string) => Promise.resolve(bootstrapPayload(id)));
+ await act(async () => screen.getByText('switch chat').click());
+ expect(screen.queryByText('chat.modelRecovery.waitingSeconds')).toBeNull();
+ await act(async () => screen.getByText('return chat').click());
+ await act(async () => stale.resolve(running()));
+ expect(screen.queryByText('chat.modelRecovery.waitingSeconds')).toBeNull();
+ expect(mocks.composer?.busy).toBe(false);
+ });
+
+ it('clears a dropped end from an idle snapshot while preserving unknown-state recovery', async () => {
+ await mount();
+ mocks.api.getTurnState.mockResolvedValue({ ...idleTurnState, foreground: 'unknown', in_flight: null });
+ await act(async () => invalidate());
+ expect(screen.getByText('chat.modelRecovery.waitingSeconds')).toBeTruthy();
+ act(() => vi.advanceTimersByTime(5000));
+ mocks.api.getTurnState.mockResolvedValue({ ...idleTurnState, model_recovery: [recovery()] });
+ await act(async () => invalidate());
+ expect(screen.queryByText('chat.modelRecovery.waitingSeconds')).toBeNull();
+ expect(mocks.composer?.busy).toBe(false);
+ });
+
+ it('does not let an older bootstrap revive a dropped end recovered by the idle poll', async () => {
+ await mount();
+ const stale = deferred<ReturnType<typeof bootstrapPayload>>();
+ mocks.api.getSessionBootstrap.mockReturnValue(stale.promise);
+ await act(async () => mocks.events?.onAuthorizationChanged({ resource_kinds: [] }));
+ act(() => vi.advanceTimersByTime(5000));
+ mocks.api.getTurnState.mockResolvedValue(idleTurnState);
+ await act(async () => invalidate());
+ await act(async () => stale.resolve({ ...bootstrapPayload('session-new'), turn_state: running() }));
+ expect(screen.queryByText('chat.modelRecovery.waitingSeconds')).toBeNull();
+ expect(mocks.composer?.busy).toBe(false);
+ });
+ });
});
diff --git a/ui/src/components/workbench/ChatPage.tsx b/ui/src/components/workbench/ChatPage.tsx
index 7d031cf75..f2853fa25 100644
--- a/ui/src/components/workbench/ChatPage.tsx
+++ b/ui/src/components/workbench/ChatPage.tsx
@@ -151,6 +151,7 @@ import {
import { errorMessage } from '@/lib/errorMessage';
import { pendingInitialMessageHandoff } from '@/lib/chatInitialMessage';
import { sessionAgentDisplayName } from './sessionAgentName';
+import { useModelHubRecovery } from '../../lib/modelHubRecovery';
// While a turn is in flight, reconcile the working/Stop state against the
// controller on this cadence (the backend ``GET /turn-state`` is authoritative).
@@ -688,13 +689,15 @@ export const ChatPage: React.FC = () => {
setLiveStartedAt(next.startedAt);
}, []);
// Lifecycle guards for ``syncTurnState``'s clear-on-idle (Codex P2):
- // - ``turnEpochRef`` bumps every time a turn STARTS (local send / send-now /
- // observed ``turn.start``). syncTurnState captures it before its request and
- // refuses to clear if it changed meanwhile — so an idle snapshot can't stomp
- // a turn that started WHILE the request was in flight.
+ // - ``turnEpochRef`` fences lifecycle changes and confirmed live state.
+ // Runtime reads capture it before requesting: a stale snapshot cannot clear
+ // a newer Turn or revive a settled Turn's recovery.
// - ``workingSetAtRef`` records when we last set working true, so syncTurnState
// can ignore an idle reading that lands inside the post-send registration gap.
const turnEpochRef = useRef(0);
+ // Material recovery invalidations may overlap: only the latest read can
+ // install a snapshot, including an empty one that clears an earlier wait.
+ const turnStateRequestRef = useRef(0);
const workingSetAtRef = useRef(0);
// A single pending "re-check after the post-send grace expires" timer + a ref
// to the latest syncTurnState, so an idle reading that arrives INSIDE the grace
@@ -707,6 +710,9 @@ export const ChatPage: React.FC = () => {
// every "a turn is starting now" path so clear-on-idle stays race-safe. Also sets
// ``workingRef`` synchronously so a settle refresh in the same tick reads it.
const markWorking = useCallback(() => {
+ if (!workingRef.current) {
+ setRuntimeState((current) => ({ ...current, model_recovery: [] }));
+ }
// Authoritative running can arrive after an early idle read on navigation.
// Resume before hydrating, so the next live row cannot discard that history.
if (liveStateRef.current.settled) dispatchLive({ type: 'turn_start' });
@@ -1363,16 +1369,19 @@ export const ChatPage: React.FC = () => {
const syncTurnState = useCallback(async (options?: { quiet?: boolean }) => {
if (!sessionId) return;
const epochAtRequest = turnEpochRef.current;
+ const request = ++turnStateRequestRef.current;
try {
const res = await api.getTurnState(sessionId, { handleError: !options?.quiet });
- if (sessionId !== sessionIdRef.current) return;
+ if (sessionId !== sessionIdRef.current
+ || turnEpochRef.current !== epochAtRequest
+ || request !== turnStateRequestRef.current) return;
if (res.foreground === 'unknown') return;
- setRuntimeState(res);
if (res.foreground === 'running') {
// markWorking (not setWorking): bump the epoch + timestamp so an OLDER
// overlapping sync whose idle response lands AFTER this one can't clear
// the Stop we just confirmed live — its captured epoch is now stale (P2).
markWorking();
+ setRuntimeState(res);
scheduleActivityRefresh();
return;
}
@@ -1382,10 +1391,12 @@ export const ChatPage: React.FC = () => {
// (2) we're past the post-send registration grace — a turn we just sent may
// not be in the controller's in-flight map yet, making this idle a
// false negative.
- if (turnEpochRef.current !== epochAtRequest) return;
const sinceSet = Date.now() - workingSetAtRef.current;
+ // Idle cannot carry live recovery, even inside the optimistic send grace.
+ setRuntimeState({ ...res, model_recovery: [] });
if (sinceSet > WORKING_SETTLE_GRACE_MS) {
const recoveredDroppedTurnEnd = workingRef.current;
+ if (recoveredDroppedTurnEnd) turnEpochRef.current += 1;
activityForegroundRef.current = 'idle';
workingRef.current = false;
setWorking(false);
@@ -1442,6 +1453,7 @@ export const ChatPage: React.FC = () => {
// route/config state, and current turn state. Fetch them as one bootstrap
// payload so remote links don't pay a tunnel round-trip per widget.
const bootstrapIsCurrent = await sessionRowRefreshGateRef.current.begin();
+ const epochAtRequest = turnEpochRef.current;
const bootstrap = await api.getSessionBootstrap(sessionId);
// Drop a response if the user switched chats or a newer bootstrap for
// this route began while it was in flight.
@@ -1499,18 +1511,26 @@ export const ChatPage: React.FC = () => {
}
setHydratedTranscriptSessionId(sessionId);
setFailedBootstrapSessionId(null);
- activityForegroundRef.current = bootstrap.turn_state.foreground;
setQueue(bootstrap.queued ?? []);
setInitialDraft(bootstrap.draft?.text ?? '');
- setRuntimeState(bootstrap.turn_state);
// Restore Stop for a turn that is still running (e.g. opened in another tab
// or reloaded mid-turn). markWorking on the live branch so a racing
// syncTurnState idle response can't clear it; an idle load is authoritative
// for the fresh page, so clear directly (Codex P2).
- if (bootstrap.turn_state.foreground === 'running') markWorking();
- else if (bootstrap.turn_state.foreground === 'idle') {
- workingRef.current = false;
- setWorking(false);
+ // Hydrate the transcript even if live lifecycle events outran this read,
+ // but never reinstall an older Turn's working state or recovery snapshot.
+ if (epochAtRequest === turnEpochRef.current) {
+ activityForegroundRef.current = bootstrap.turn_state.foreground;
+ if (bootstrap.turn_state.foreground === 'running') markWorking();
+ else if (bootstrap.turn_state.foreground === 'idle') {
+ workingRef.current = false;
+ setWorking(false);
+ }
+ setRuntimeState({
+ ...bootstrap.turn_state,
+ model_recovery: bootstrap.turn_state.foreground === 'running'
+ ? bootstrap.turn_state.model_recovery : [],
+ });
}
// Reconcile only after restoring the live generation above, so a recovered
// running turn's history request is tagged with the generation it belongs to.
@@ -1539,6 +1559,7 @@ export const ChatPage: React.FC = () => {
// and the merge in ``refresh`` only ever unions same-session rows.
useEffect(() => {
bootstrapRequestGenerationRef.current += 1;
+ turnEpochRef.current += 1;
// The gate is session-scoped. A PATCH for the previous chat may still be
// pending after navigation, but it must never hold the new chat's bootstrap
// or recovery reads hostage.
@@ -1676,7 +1697,7 @@ export const ChatPage: React.FC = () => {
// markWorking (not setWorking): bump the epoch so a syncTurnState idle
// reading already in flight can't clear this freshly-started turn.
if (data.session_id === sessionIdRef.current) {
- setRuntimeState((current) => ({ ...current, in_flight: true, foreground: 'running' }));
+ setRuntimeState((current) => ({ ...current, in_flight: true, foreground: 'running', model_recovery: [] }));
// Agent Activity: a new turn begins → bump the generation (fresh empty
// buffer). Any stale rows from the previous turn become invisible by
// construction and can never merge with the new turn's rows.
@@ -1689,8 +1710,9 @@ export const ChatPage: React.FC = () => {
// or user cancel) — the authoritative end of the working state. There is
// no turn-duration timeout, so this only fires on a REAL terminal signal.
if (data.session_id === sessionIdRef.current) {
+ turnEpochRef.current += 1;
activityForegroundRef.current = 'idle';
- setRuntimeState((current) => ({ ...current, in_flight: false, foreground: 'idle' }));
+ setRuntimeState((current) => ({ ...current, in_flight: false, foreground: 'idle', model_recovery: [] }));
workingRef.current = false;
setWorking(false);
// Agent Activity: the turn settled (result / error / interrupt) → mark the
@@ -1714,6 +1736,10 @@ export const ChatPage: React.FC = () => {
if (data.session_id === sessionIdRef.current) void refreshQueue();
},
onSessionActivity: (data) => {
+ if (data.session_id === sessionIdRef.current && data.event === 'model_recovery') {
+ void syncTurnState({ quiet: true });
+ return;
+ }
if (data.session_id === sessionIdRef.current && data.event === 'archived') {
// The session you're viewing was archived (here or in another tab) —
// archive is terminal, so cancel any prepared external launch before
@@ -2843,6 +2869,7 @@ export const ChatPage: React.FC = () => {
session={session}
agentDisplayName={agentDisplayName}
working={working}
+ modelRecovery={runtimeState.model_recovery}
hasOlder={!!olderCursor}
loadingOlder={loadingOlder}
onLoadOlder={loadOlderMessages}
@@ -3681,6 +3708,7 @@ interface TranscriptProps {
session: WorkbenchSession;
agentDisplayName: string | null;
working: boolean;
+ modelRecovery?: SessionRuntimeState['model_recovery'];
hasOlder: boolean;
loadingOlder: boolean;
onLoadOlder: () => void | Promise<boolean>;
@@ -3744,6 +3772,7 @@ export const Transcript: React.FC<TranscriptProps> = ({
session,
agentDisplayName,
working,
+ modelRecovery,
hasOlder,
loadingOlder,
onLoadOlder,
@@ -3765,6 +3794,7 @@ export const Transcript: React.FC<TranscriptProps> = ({
footer,
}) => {
const { t } = useTranslation();
+ const recovery = useModelHubRecovery(modelRecovery, working);
const navigate = useNavigate();
const { openApp } = useWindowManager();
const fileViewer = useFileViewer();
@@ -3901,7 +3931,7 @@ export const Transcript: React.FC<TranscriptProps> = ({
// a dropped turn.end that the idle poll recovered — invisible by construction.
const liveActive = !!activity?.enabled && activity.liveRows.length > 0;
const showActivityCard = shouldShowRunningCard(!!activity?.enabled, working, activity?.liveRows.length ?? 0);
- const showThinking = working && !lastIsAgentTerminal && !liveActive;
+ const showThinking = working && (!lastIsAgentTerminal || recovery.active) && !liveActive;
// Render one settled-turn chip (before/after its anchor message, or at the top).
// Plain render helper (not a component) so it stays referentially simple.
const renderActivityChip = (group: ActivityGroup) =>
@@ -4299,12 +4329,14 @@ export const Transcript: React.FC<TranscriptProps> = ({
showToolCalls={activity.showToolCalls}
onToggleTools={activity.onToggleTools}
onDisableActivity={activity.onDisable}
+ statusLabel={recovery.label}
/>
) : showThinking ? (
<ThinkingBubble
session={session}
agentDisplayName={agentDisplayName}
onShowActivity={!activity?.enabled ? activity?.onEnable : undefined}
+ statusLabel={recovery.label}
/>
) : null}
{footer}
@@ -4362,7 +4394,8 @@ export const ThinkingBubble: React.FC<{
session: WorkbenchSession;
agentDisplayName: string | null;
onShowActivity?: () => void;
-}> = ({ session, agentDisplayName, onShowActivity }) => {
+ statusLabel?: string | null;
+}> = ({ session, agentDisplayName, onShowActivity, statusLabel }) => {
const { t } = useTranslation();
const dots = (
<div className="flex items-center gap-1 py-0.5">
@@ -4377,7 +4410,7 @@ export const ThinkingBubble: React.FC<{
<div className="flex items-center gap-2 px-0.5">
<RoleAvatar tone="mint"><Bot /></RoleAvatar>
<span className="text-[11px] font-medium text-muted">
- {agentDisplayName || session.agent_name || t('chat.thinking')}
+ {statusLabel || agentDisplayName || session.agent_name || t('chat.thinking')}
</span>
</div>
{onShowActivity ? (
diff --git a/ui/src/context/ApiContext.tsx b/ui/src/context/ApiContext.tsx
index d60c09bbb..a27d8fcfa 100644
--- a/ui/src/context/ApiContext.tsx
+++ b/ui/src/context/ApiContext.tsx
@@ -1385,6 +1385,17 @@ export type SessionActivityState = {
schedule_type?: 'at' | 'cron' | null;
};
+export type ModelRecoveryState = {
+ request_id: string;
+ phase: 'waiting' | 'attempting';
+ attempt_count: number;
+ started_at: string;
+ window_end: string;
+ source_id: string | null;
+ reason: string | null;
+ next_eligible_at: string | null;
+};
+
export type SessionRuntimeState = {
in_flight: boolean | null;
foreground: 'idle' | 'running' | 'unknown';
@@ -1395,6 +1406,7 @@ export type SessionRuntimeState = {
connection: 'connected' | 'reconnecting' | 'disconnected' | 'unknown';
backend?: string;
recovered_agent_status?: boolean;
+ model_recovery?: ModelRecoveryState[];
};
export type WorkbenchSessionBootstrap = {
diff --git a/ui/src/i18n/en.json b/ui/src/i18n/en.json
index 203f6f3da..88d2999d3 100644
--- a/ui/src/i18n/en.json
+++ b/ui/src/i18n/en.json
@@ -4138,6 +4138,11 @@
"scrollToBottom": "Scroll to latest",
"streaming": "Streaming",
"thinking": "Thinking…",
+ "modelRecovery": {
+ "waitingSeconds": "Waiting to retry in about {{seconds}}s",
+ "waiting": "Waiting to retry…",
+ "attempting": "Trying again…"
+ },
"notifyLabel": "Notify",
"retryRequested": "Retry requested",
"retryFailed": "Could not request retry. You can try again safely.",
diff --git a/ui/src/i18n/zh.json b/ui/src/i18n/zh.json
index 97e017bfd..a1f84c0d5 100644
--- a/ui/src/i18n/zh.json
+++ b/ui/src/i18n/zh.json
@@ -4138,6 +4138,11 @@
"scrollToBottom": "回到最新",
"streaming": "实时输出中",
"thinking": "思考中…",
+ "modelRecovery": {
+ "waitingSeconds": "等待重试,预计 {{seconds}} 秒",
+ "waiting": "等待重试…",
+ "attempting": "正在重试…"
+ },
"notifyLabel": "通知",
"retryRequested": "已请求重试",
"retryFailed": "未能提交重试,可以再次点击。",
diff --git a/ui/src/lib/modelHubRecovery.test.ts b/ui/src/lib/modelHubRecovery.test.ts
new file mode 100644
index 000000000..cf87cd48c
--- /dev/null
+++ b/ui/src/lib/modelHubRecovery.test.ts
@@ -0,0 +1,174 @@
+/** @vitest-environment jsdom */
+
+import { act, cleanup, renderHook } from '@testing-library/react';
+import { createInstance } from 'i18next';
+import { createElement, type ReactNode } from 'react';
+import { I18nextProvider, initReactI18next } from 'react-i18next';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import type { ModelRecoveryState } from '../context/ApiContext';
+import en from '../i18n/en.json';
+import zh from '../i18n/zh.json';
+import { useModelHubRecovery } from './modelHubRecovery';
+
+const i18n = createInstance();
+void i18n.use(initReactI18next).init({
+ lng: 'en',
+ resources: { en: { translation: en }, zh: { translation: zh } },
+ interpolation: { escapeValue: false },
+});
+const wrapper = ({ children }: { children: ReactNode }) => createElement(I18nextProvider, { i18n }, children);
+const start = Date.parse('2026-09-09T04:00:00Z');
+const progress = (overrides: Partial<ModelRecoveryState> = {}): ModelRecoveryState => ({
+ request_id: 'request-1',
+ phase: 'waiting',
+ attempt_count: 1,
+ started_at: new Date(start).toISOString(),
+ window_end: new Date(start + 120_000).toISOString(),
+ source_id: 'private-source',
+ reason: 'https://private-host.invalid/diagnostic',
+ next_eligible_at: new Date(start + 30_000).toISOString(),
+ ...overrides,
+});
+const mount = (snapshot: unknown = [progress()], working = true) => renderHook(
+ ({ snapshot, working }) => useModelHubRecovery(snapshot, working),
+ { initialProps: { snapshot, working }, wrapper },
+);
+
+beforeEach(async () => {
+ vi.useFakeTimers();
+ vi.setSystemTime(start);
+ await i18n.changeLanguage('en');
+});
+afterEach(() => {
+ cleanup();
+ vi.useRealTimers();
+});
+
+describe('quiet Model Hub recovery label', () => {
+ it('keeps the generic indicator for five seconds and clears a short recovery silently', () => {
+ const { result, rerender } = mount();
+ expect(result.current).toEqual({ active: true, label: null });
+ act(() => vi.advanceTimersByTime(4999));
+ expect(result.current.label).toBeNull();
+ rerender({ snapshot: [], working: true });
+ act(() => vi.advanceTimersByTime(1000));
+ expect(result.current).toEqual({ active: false, label: null });
+ });
+
+ it('reveals at server start + 5s when a reload lands between display ticks', () => {
+ vi.setSystemTime(start + 3250);
+ const { result } = mount();
+ act(() => vi.advanceTimersByTime(1749));
+ expect(result.current.label).toBeNull();
+ act(() => vi.advanceTimersByTime(1));
+ expect(result.current.label).toBe('Waiting to retry in about 25s');
+ });
+
+ it('restores a mature wait immediately on reload and keeps its debounce through updates', () => {
+ vi.setSystemTime(start + 6000);
+ const { result, rerender } = mount();
+ expect(result.current.label).toBe('Waiting to retry in about 24s');
+ rerender({
+ snapshot: [progress({ attempt_count: 2, next_eligible_at: new Date(start + 40_000).toISOString() })],
+ working: true,
+ });
+ expect(result.current.label).toBe('Waiting to retry in about 34s');
+ });
+
+ it('MH-RETRY-WEB-001: never infers an attempt, recovery, or end from either deadline expiring', () => {
+ vi.setSystemTime(start + 29_000);
+ const { result, rerender } = mount();
+ expect(result.current.label).toBe('Waiting to retry in about 1s');
+ act(() => vi.advanceTimersByTime(1000));
+ expect(result.current.label).toBe('Waiting to retry…');
+ act(() => vi.advanceTimersByTime(100_000));
+ expect(result.current).toEqual({ active: true, label: 'Waiting to retry…' });
+ rerender({ snapshot: [progress({ phase: 'attempting' })], working: true });
+ expect(result.current.label).toBe('Trying again…');
+ act(() => vi.advanceTimersByTime(120_000));
+ expect(result.current.label).toBe('Trying again…');
+ });
+
+ it('clears immediately on server clear or idle and debounces a new request from its own start', () => {
+ vi.setSystemTime(start + 6000);
+ const { result, rerender } = mount();
+ rerender({ snapshot: [], working: true });
+ expect(result.current.label).toBeNull();
+ rerender({ snapshot: [progress()], working: false });
+ expect(result.current).toEqual({ active: false, label: null });
+ rerender({ snapshot: [progress({ request_id: 'new', started_at: new Date(Date.now()).toISOString() })], working: true });
+ expect(result.current).toEqual({ active: true, label: null });
+ act(() => vi.advanceTimersByTime(5000));
+ expect(result.current.label).toBe('Waiting to retry in about 19s');
+ });
+
+ it('keeps one stable oldest-request label while peer requests are added, reordered, and removed', () => {
+ vi.setSystemTime(start + 10_000);
+ const oldest = progress();
+ const peer = progress({ request_id: 'peer', phase: 'attempting', started_at: new Date(start + 1000).toISOString() });
+ const { result, rerender } = mount([peer, oldest]);
+ expect(result.current.label).toBe('Waiting to retry in about 20s');
+ rerender({ snapshot: [oldest, peer], working: true });
+ expect(result.current.label).toBe('Waiting to retry in about 20s');
+ rerender({ snapshot: [peer], working: true });
+ expect(result.current.label).toBe('Trying again…');
+ });
+
+ it.each([
+ undefined, null, {}, [null], ['waiting'],
+ [{ request_id: 'incomplete' }],
+ [progress({ phase: 'recovered' as 'waiting' })],
+ [progress({ request_id: '' })],
+ [progress({ attempt_count: -1 })],
+ [progress({ attempt_count: 1.5 })],
+ [progress({ attempt_count: '2' as unknown as number })],
+ [progress({ started_at: 'bad' })],
+ [progress({ started_at: '2026-02-30T00:00:00Z' })],
+ [progress({ started_at: '2026-09-09T04:00:00' })],
+ [progress({ window_end: 'bad' })],
+ [progress({ window_end: new Date(start - 1000).toISOString() })],
+ ])('leaves malformed recovery to the existing generic indicator: %j', (snapshot) => {
+ vi.setSystemTime(start + 6000);
+ const { result, rerender } = mount();
+ rerender({ snapshot, working: true });
+ expect(result.current).toEqual({ active: false, label: null });
+ });
+
+ it.each([null, 'invalid', '2026-09-09T04:00:00', 100, {}])(
+ 'uses generic waiting for unavailable/malformed advisory time: %j',
+ (next_eligible_at) => {
+ vi.setSystemTime(start + 6000);
+ const { result } = mount([{ ...progress(), next_eligible_at }]);
+ expect(result.current.label).toBe('Waiting to retry…');
+ },
+ );
+
+ it('accepts UTC offsets and fractional precision from the controller, ignoring malformed peers', () => {
+ vi.setSystemTime(start + 6000);
+ const { result } = mount([null, progress({
+ started_at: '2026-09-09T04:00:00.000000+00:00',
+ window_end: '2026-09-09T04:02:00.000000+00:00',
+ next_eligible_at: '2026-09-09T04:00:30.000000+00:00',
+ })]);
+ expect(result.current.label).toBe('Waiting to retry in about 24s');
+ expect(result.current.label).not.toMatch(/private|https|2026|UTC/);
+ });
+
+ it('renders localized waiting and attempting without diagnostic fields', async () => {
+ await i18n.changeLanguage('zh');
+ vi.setSystemTime(start + 6000);
+ const { result, rerender } = mount();
+ expect(result.current.label).toBe('等待重试,预计 24 秒');
+ rerender({ snapshot: [progress({ next_eligible_at: null })], working: true });
+ expect(result.current.label).toBe('等待重试…');
+ rerender({ snapshot: [progress({ phase: 'attempting' })], working: true });
+ expect(result.current.label).toBe('正在重试…');
+ });
+
+ it('catches up after a suspended tab without restarting the debounce', () => {
+ const { result } = mount();
+ vi.setSystemTime(start + 8000);
+ act(() => document.dispatchEvent(new Event('visibilitychange')));
+ expect(result.current.label).toBe('Waiting to retry in about 22s');
+ });
+});
diff --git a/ui/src/lib/modelHubRecovery.ts b/ui/src/lib/modelHubRecovery.ts
new file mode 100644
index 000000000..a0e56e8fc
--- /dev/null
+++ b/ui/src/lib/modelHubRecovery.ts
@@ -0,0 +1,84 @@
+import { useEffect, useMemo, useState } from 'react';
+import { useTranslation } from 'react-i18next';
+import { startVisibleTicker } from './useVisibleNow';
+
+const REVEAL_AFTER_MS = 5000;
+
+type RecoveryProgress = {
+ requestId: string;
+ phase: 'waiting' | 'attempting';
+ startedAt: number;
+ nextEligibleAt: number | null;
+};
+
+function utcTimestamp(value: unknown): number | null {
+ if (typeof value !== 'string'
+ || !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|\+00:00)$/.test(value)) return null;
+ const parsed = Date.parse(value);
+ // Date.parse normalizes impossible calendar dates such as February 30.
+ return Number.isFinite(parsed) && new Date(parsed).toISOString().slice(0, 19) === value.slice(0, 19)
+ ? parsed : null;
+}
+
+function selectRecovery(snapshot: unknown): RecoveryProgress | null {
+ if (!Array.isArray(snapshot)) return null;
+ let selected: RecoveryProgress | null = null;
+ for (const entry of snapshot) {
+ if (!entry || typeof entry !== 'object'
+ || typeof entry.request_id !== 'string' || !entry.request_id.trim()
+ || (entry.phase !== 'waiting' && entry.phase !== 'attempting')
+ || !Number.isSafeInteger(entry.attempt_count) || entry.attempt_count < 0) continue;
+ const startedAt = utcTimestamp(entry.started_at);
+ const windowEnd = utcTimestamp(entry.window_end);
+ if (startedAt === null || windowEnd === null || windowEnd < startedAt) continue;
+ // One label follows the oldest pending request, independent of array order.
+ // Source/reason are diagnostic fields and never participate in display text.
+ if (!selected || startedAt < selected.startedAt
+ || (startedAt === selected.startedAt && entry.request_id < selected.requestId)) {
+ selected = {
+ requestId: entry.request_id,
+ phase: entry.phase,
+ startedAt,
+ nextEligibleAt: utcTimestamp(entry.next_eligible_at),
+ };
+ }
+ }
+ return selected;
+}
+
+/** A display clock only: server snapshots own attempts, clear, and settlement. */
+export function useModelHubRecovery(snapshot: unknown, working: boolean): {
+ active: boolean;
+ label: string | null;
+} {
+ const { t } = useTranslation();
+ const recovery = useMemo(() => working ? selectRecovery(snapshot) : null, [snapshot, working]);
+ const [now, setNow] = useState(() => Date.now());
+ useEffect(() => {
+ if (!recovery) return;
+ const tick = () => setNow(Date.now());
+ const stop = startVisibleTicker(tick, 1000);
+ // Reveal at the server start + 5s even when hydration lands between ticks.
+ const remaining = recovery.startedAt + REVEAL_AFTER_MS - Date.now();
+ const reveal = remaining > 0 && remaining <= REVEAL_AFTER_MS
+ ? window.setTimeout(tick, remaining) : null;
+ return () => {
+ stop();
+ if (reveal !== null) window.clearTimeout(reveal);
+ };
+ }, [recovery]);
+
+ if (!recovery || now - recovery.startedAt < REVEAL_AFTER_MS) {
+ return { active: recovery !== null, label: null };
+ }
+ if (recovery.phase === 'attempting') {
+ return { active: true, label: t('chat.modelRecovery.attempting') };
+ }
+ const seconds = recovery.nextEligibleAt === null ? 0 : Math.ceil((recovery.nextEligibleAt - now) / 1000);
+ return {
+ active: true,
+ label: seconds > 0
+ ? t('chat.modelRecovery.waitingSeconds', { seconds })
+ : t('chat.modelRecovery.waiting'),
+ };
+}
diff --git a/vibe/i18n/en.json b/vibe/i18n/en.json
index 9bf85f32c..5a5035659 100644
--- a/vibe/i18n/en.json
+++ b/vibe/i18n/en.json
@@ -1017,10 +1017,16 @@
"native_cli_unavailable": "CLI login is not available in this process"
}
},
+ "recovery": {
+ "waiting": "Waiting to retry the model request in about {seconds}s",
+ "attempting": "Retrying the model request…",
+ "pending": "Waiting for the next model attempt…",
+ "ended": "This model request could not complete. Automatic recovery has ended. You can retry later or choose another model."
+ },
"launch": {
"retry": "The next turn has already switched routes. Retry directly.",
- "waiting": "Model {model} is temporarily unavailable. Waiting for {source} to recover at {retry_at}.",
- "waiting_without_retry": "The route for model {model} has recovered. Retry directly.",
+ "waiting": "This request could not complete: {source} is temporarily unavailable for model {model}, and no backup is usable. Automatic recovery has ended. You can retry later or choose another model.",
+ "waiting_without_retry": "This request could not complete. Automatic recovery has ended. You can retry model {model} or choose another model.",
"interrupted": "Model {model} is unavailable: {blockers}. Open Models to resolve it.",
"route_unconfigured": "Model {model} has no configured route. Open Models to configure one.",
"request_incompatible": "This request is incompatible; switching Sources will not help."
diff --git a/vibe/i18n/zh.json b/vibe/i18n/zh.json
index 85235e978..3412a69cb 100644
--- a/vibe/i18n/zh.json
+++ b/vibe/i18n/zh.json
@@ -1017,10 +1017,16 @@
"native_cli_unavailable": "当前进程无法使用 CLI 登录"
}
},
+ "recovery": {
+ "waiting": "正在等待重试模型请求,预计 {seconds} 秒后再次尝试",
+ "attempting": "正在重试模型请求…",
+ "pending": "正在等待下一次模型请求…",
+ "ended": "本次模型请求未完成,自动恢复已结束。你可以稍后重试或切换模型。"
+ },
"launch": {
"retry": "已自动切换到备用供应商,直接重试即可",
- "waiting": "模型 {model} 暂时不可用,等待 {source} 于 {retry_at} 自动恢复。",
- "waiting_without_retry": "模型 {model} 的路由已恢复,直接重试即可。",
+ "waiting": "本次请求未完成:{source} 暂时无法提供模型 {model},且没有可用备用源。自动恢复已结束,你可以稍后重试或切换模型。",
+ "waiting_without_retry": "本次请求未完成,自动恢复已结束。你可以重试模型 {model} 或切换模型。",
"interrupted": "模型 {model} 当前不可用:{blockers}。请前往「模型」页处理。",
"route_unconfigured": "模型 {model} 尚未配置路由。请前往「模型」页配置。",
"request_incompatible": "此请求不兼容;切换模型供应商也无法解决。"
diff --git a/vibe/model_hub_runtime/client.py b/vibe/model_hub_runtime/client.py
index 016261d6b..24ab7d4aa 100644
--- a/vibe/model_hub_runtime/client.py
+++ b/vibe/model_hub_runtime/client.py
@@ -12,6 +12,7 @@ import urllib.parse
import urllib.request
from collections.abc import Awaitable, Callable
from dataclasses import dataclass, field, replace
+from datetime import datetime, timezone
from decimal import Decimal
from typing import Any, AsyncIterator, BinaryIO, Literal, Mapping, TypeVar, cast
@@ -390,6 +391,8 @@ class EngineClient:
# completion, transport failure, or owner cancellation ends that wait.
session = aiohttp.ClientSession(timeout=timeout, trust_env=False)
response: aiohttp.ClientResponse | None = None
+ retry_after: str | None = None
+ response_received_at: datetime | None = None
first_received = False
model_output_started = False
ownership_transferred = False
@@ -412,6 +415,14 @@ class EngineClient:
if outcome.usage is None and wire_state is not None and wire_state.usage is not None:
outcome = replace(outcome, usage=wire_state.usage)
+ if wire_state is not None and wire_state.model_output_started:
+ outcome = replace(outcome, recovery_verified=True)
+ if response_received_at is not None:
+ outcome = replace(
+ outcome,
+ retry_after=retry_after,
+ response_received_at=response_received_at,
+ )
return completed_handle(outcome)
try:
@@ -422,6 +433,10 @@ class EngineClient:
allow_redirects=False,
)
if response.status >= 300:
+ response_received_at = datetime.now(timezone.utc)
+ retry_after = response.headers.get("Retry-After")
+ if retry_after is not None and (len(retry_after) > 128 or not retry_after.isascii()):
+ retry_after = None
error_body = _StreamPrelude()
response_deadline = time.monotonic() + self.timeout
try:
@@ -463,6 +478,7 @@ class EngineClient:
outcome="failed_terminal",
error_payload=payload,
message=f"upstream returned HTTP {response.status}",
+ recovery_verified=False,
),
source=source,
model_id=model_id,
@@ -1408,6 +1424,8 @@ async def _response_stream(
if on_transport_done is not None:
on_transport_done()
if outcome is not None and not outcome_future.done():
+ if wire_state.model_output_started:
+ outcome = replace(outcome, recovery_verified=True)
outcome_future.set_result(outcome)
@@ -1453,6 +1471,7 @@ def _reduce_protocol_observation(
http_status=http_status,
stream_started=stream_started,
usage=observation.usage,
+ recovery_verified=observation.recovery_verified,
)
if observation.outcome == "failed_terminal":
projected_types = tuple(
@@ -1485,6 +1504,7 @@ def _reduce_protocol_observation(
message=observation.message or "upstream returned a protocol error event",
stream_started=stream_started,
usage=observation.usage,
+ recovery_verified=observation.recovery_verified,
)
return _outcome(
kind=RawOutcomeKind.PROTOCOL_ERROR,
@@ -1494,6 +1514,7 @@ def _reduce_protocol_observation(
message=observation.message or "upstream emitted invalid protocol data",
stream_started=stream_started,
usage=observation.usage,
+ recovery_verified=observation.recovery_verified,
)
@@ -1558,6 +1579,7 @@ def _outcome(
message: str | None = None,
stream_started: bool = False,
usage: ProtocolUsageReport | None = None,
+ recovery_verified: bool = False,
) -> RawCallOutcome:
return RawCallOutcome(
kind=kind,
@@ -1570,6 +1592,7 @@ def _outcome(
error_type=error_type,
error_candidates=error_candidates,
usage=usage,
+ recovery_verified=recovery_verified,
)