Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
"""Bounded recovery at the shared resolver and actual gateway HTTP boundary."""
from __future__ import annotations
import asyncio
import copy
from dataclasses import replace
from datetime import datetime, timedelta
from email.utils import format_datetime
from types import SimpleNamespace
import aiohttp
from aiohttp import web
import pytest
from config.v2_config import ModelHubBackendModelConfig, ModelHubSourceStateConfig
from core.handlers.model_hub.adapter import RawOutcomeKind
from core.handlers.model_hub.classification import ResolutionDecision
from core.handlers.model_hub.retry import (
RECOVERY_EXHAUSTED_CODE, RECOVERY_EXHAUSTED_MESSAGE, RecoveryPolicy, retry_after_deadline,
)
from core.handlers.model_hub.service import ModelHubError
from core.handlers.model_hub.turn_gateway import ModelHubTurnGateway
from modules.agents.model_hub import ModelHubRuntimeRouter
from tests.test_model_hub_inference_wait import WIRE as INFERENCE_WIRE
from tests.test_model_hub_l3 import (
NOW, _assert_valid, _canonicalize_fixed_test_routes, _outcome, _service, _source,
)
from vibe.model_hub_runtime.adapter import CLIProxyEngineAdapter
from vibe.model_hub_runtime.client import EngineClient, EngineConnection
from vibe.model_hub_runtime.state import EngineStateStore
# The inference-deadline fixtures deliberately exercise permissive responses.
# Recovery additionally needs recognized terminal envelopes.
WIRE = {
**INFERENCE_WIRE,
"openai_responses": (*INFERENCE_WIRE["openai_responses"][:3],
b'{"object":"response","status":"completed","output":[{"content":[{"type":"output_text","text":"ok"}]}]}'),
"openai_chat": (*INFERENCE_WIRE["openai_chat"][:3],
b'{"object":"chat.completion","choices":[{"message":{"role":"assistant","content":"ok"}}]}'),
}
class Clock:
def __init__(self):
self.elapsed = 0.0
self.offset = 0.0
self.delays = []
def now(self):
return NOW + timedelta(seconds=self.elapsed + self.offset)
def advance(self, seconds):
self.elapsed += seconds
async def sleep(self, seconds):
self.delays.append(seconds)
self.advance(seconds)
await asyncio.sleep(0)
def policy(self):
return RecoveryPolicy(
now=self.now, monotonic=lambda: self.elapsed,
jitter=lambda low, high: low, sleep=self.sleep,
)
def clock_service(tmp_path, *, sources=None, outcomes=None):
clock = Clock()
service = _service(
tmp_path, sources=sources or [_source("src_recovery01", "Recovering")],
outcomes=outcomes,
)
service.now = clock.now
service.recovery = clock.policy()
# Keep the exact common fixture route as a real catalog model so health
# persistence validates the same shape used by the inference tests.
for backend in ("claude", "codex"):
service.store.config.agents[backend].models.append(
ModelHubBackendModelConfig(id="shared-model", origin="manual"),
)
return service, clock
async def fail(service, source, reason="network", *, generation=None, outcome=None):
generation = generation if generation is not None else service._reserve_settlement_generation(source.id)
return await service._settle_fallback_source(
source, ResolutionDecision("fallback", reason=reason),
backend="codex", model_id="shared-model", settlement_generation=generation, outcome=outcome,
)
@pytest.mark.parametrize("reason,delays", [
("network", [1, 2, 4, 8, 16, 30, 30]),
("server_error", [30, 60, 120, 120]),
("rate_limited", [60, 120, 240, 300, 300]),
("quota_exhausted", [300, 300]),
])
def test_shared_streak_counts_attempts_and_caps_positive_jitter(tmp_path, reason, delays):
async def run():
service, clock = clock_service(tmp_path)
source = service.store.load().sources[0]
before = copy.deepcopy(service.store.load().to_payload())
for expected in delays:
await fail(service, source, reason)
annotation = service.recovery.annotations(service.store.load())[source.id]
assert (datetime.fromisoformat(annotation.retry_at) - clock.now()).total_seconds() == expected
clock.advance(expected)
if reason == "network":
assert service.store.load().to_payload() == before
service.recovery.jitter = lambda low, high: high
service.recovery._sources.clear()
if reason != "network":
source = service.store.load().sources[0]
source.state = ModelHubSourceStateConfig(status="standby")
await fail(service, source, reason)
annotation = service.recovery.annotations(service.store.load())[source.id]
assert (datetime.fromisoformat(annotation.retry_at) - clock.now()).total_seconds() == min(delays[0] * 1.2, delays[-1])
asyncio.run(run())
@pytest.mark.parametrize("value", [None, "", "-1", "+1", "1.5", "NaN", "x" * 129, "4", "9" * 128, "Wed, 01 Jan 2020 00:00:00 GMT"])
def test_invalid_advice_cannot_replace_local_policy(value):
assert retry_after_deadline(value, NOW) is None
def test_retry_after_uses_response_receipt_and_keeps_long_upstream_advice(tmp_path):
async def run():
service, clock = clock_service(tmp_path)
source = service.store.load().sources[0]
for advice in ("300", format_datetime(NOW + timedelta(seconds=300), usegmt=True)):
service.recovery._sources.clear()
clock.elapsed = 20
outcome = SimpleNamespace(retry_after=advice, response_received_at=NOW, stream_started=False)
await fail(service, source, "server_error", outcome=outcome)
assert service.recovery.annotations(service.store.load())[source.id].retry_at == (NOW + timedelta(seconds=300)).isoformat()
with pytest.raises(ModelHubError):
await service.resolve_with_recovery(
backend="codex", model_id="shared-model", request={}, supply_channel="hub",
)
assert clock.delays == []
assert service.adapter.invocations == []
asyncio.run(run())
def test_retry_after_ms_reaches_recovery_as_fractional_seconds(tmp_path):
async def run():
service, clock = clock_service(tmp_path)
source = service.store.load().sources[0]
outcome = SimpleNamespace(
retry_after=None,
retry_after_ms=1500,
response_received_at=NOW,
stream_started=False,
)
await fail(service, source, "network", outcome=outcome)
retry_at = service.recovery.annotations(service.store.load())[source.id].retry_at
assert retry_at == (NOW + timedelta(milliseconds=1500)).isoformat()
asyncio.run(run())
async def run():
service, clock = clock_service(tmp_path)
source = service.store.load().sources[0]
await fail(service, source)
clock.offset = -3600
hop = service.agent_chain("codex", "shared-model")["chain"][0]
assert (datetime.fromisoformat(hop["retry_at"]) - clock.now()).total_seconds() == 1
clock.advance(1)
assert service.agent_chain("codex", "shared-model")["chain"][0]["recovery"] == "eligible"
assert service.store.load().sources[0].state.status == "standby"
assert not any(event["kind"] == "recover" for event in service.events.list())
asyncio.run(run())
@pytest.mark.parametrize("blocker", ["none", "cooldown", "needs_action", "error", "retired", "missing"])
def test_live_network_overlay_preserves_stronger_blockers(tmp_path, blocker):
async def run():
service, clock = clock_service(tmp_path)
source = service.store.load().sources[0]
await fail(service, source)
if blocker in {"cooldown", "needs_action", "error"}:
source.state = ModelHubSourceStateConfig(
status=blocker,
retry_at=(clock.now() + timedelta(seconds=60)).isoformat() if blocker == "cooldown" else None,
detail_key={
"cooldown": "models.source.cooldown.rate_limited",
"needs_action": "models.source.needs_action.credential_revoked",
"error": "models.source.error.unclassified",
}[blocker],
)
if blocker == "retired":
source.models[0].retired = True
if blocker == "missing":
service.store.config.sources = []
chain = service.agent_chain("codex", "shared-model")
_assert_valid("agent-chain.schema.json", chain)
hop = chain["chain"][0]
assert not hop["runnable"]
assert hop["health"] == {
"none": "backoff", "cooldown": "cooldown", "retired": "healthy",
"missing": "error", "needs_action": "needs_action", "error": "error",
}[blocker]
assert chain["supply_state"] == ("waiting" if blocker in {"none", "cooldown"} else "interrupted")
agents = {agent["backend"]: agent for agent in service.list_agents()}
assert agents["codex"]["supply_status"] == chain["supply_state"]
asyncio.run(run())
def test_stale_duplicate_settlement_and_replacement_cannot_advance_or_unlock(tmp_path):
async def run():
service, clock = clock_service(tmp_path)
source = service.store.load().sources[0]
first = service._reserve_settlement_generation(source.id)
await fail(service, source, generation=first)
await fail(service, source, generation=first)
assert service.recovery._sources[source.id].failures == 1
clock.advance(1)
second = service._reserve_settlement_generation(source.id)
assert service.recovery.claim(source, second)
service.recovery.release(source.id, first)
assert service.recovery._sources[source.id].owner == second
await fail(service, source, generation=first)
assert service.recovery._sources[source.id].owner == second
old = copy.deepcopy(source)
source.credential_ref = "cred_replaced"
assert service.recovery.annotations(service.store.load()) == {}
await fail(service, old, generation=second)
assert service.recovery.annotations(service.store.load()) == {}
assert source.state.status == "standby"
asyncio.run(run())
def test_fallback_success_does_not_clear_primary_and_eligibility_is_not_recovery(tmp_path):
async def run():
first, second = _source("src_primary01", "Primary"), _source("src_backup001", "Backup")
service, clock = clock_service(tmp_path, sources=[first, second], outcomes=[
_outcome(RawOutcomeKind.NETWORK_ERROR, source_id=first.id),
_outcome(RawOutcomeKind.SUCCESS, source_id=second.id, stream_started=True),
])
result = await service.resolve_with_recovery(backend="codex", model_id="shared-model", request={})
assert result.source_id == second.id
assert clock.delays == []
assert first.id in service.recovery._sources
assert not any(event["kind"] == "recover" for event in service.events.list())
asyncio.run(run())
@pytest.mark.parametrize(("advice", "waited"), [("120", []), ("119", [118.95])], ids=["at-the-window-end", "inside"])
def test_a_retry_the_window_cannot_admit_ends_the_request_at_once(tmp_path, advice, waited):
"""MH-RETRY-WINDOW-001: the window opens at the refusal, after the response its Retry-After counts from, so advice
equal to the window lands just inside it. Waiting for it would end the window as the retry fell due, with no
retry; the request ends at once instead, keeping the cooldown. Advice a second shorter is waited for and tried."""
async def run():
refused = replace(
_outcome(RawOutcomeKind.HTTP_ERROR, status=429, source_id="src_recovery01"),
retry_after=advice, response_received_at=NOW - timedelta(seconds=0.05),
)
service, clock = clock_service(tmp_path, outcomes=[
refused, _outcome(RawOutcomeKind.SUCCESS, source_id="src_recovery01", stream_started=True),
])
async def late_sleep(seconds):
# A real wake lands after its timer: the loop, then the lock, before the attempt is admitted.
await clock.sleep(seconds)
clock.advance(0.1)
service.recovery.sleep = late_sleep
if waited:
result = await service.resolve_with_recovery(backend="codex", model_id="shared-model", request={})
assert result.source_id == "src_recovery01"
else:
with pytest.raises(ModelHubError) as raised:
await service.resolve_with_recovery(backend="codex", model_id="shared-model", request={})
assert raised.value.code == RECOVERY_EXHAUSTED_CODE
assert clock.delays == pytest.approx(waited)
assert len(service.adapter.invocations) == 1 + len(waited)
asyncio.run(run())
def test_one_window_covers_all_fallback_passes_without_replaying_after_output(tmp_path):
"""MH-RETRY-WINDOW-001: admission shares one clock across failed attempts."""
async def run():
service, clock = clock_service(tmp_path, outcomes=[
_outcome(RawOutcomeKind.NETWORK_ERROR, source_id="src_recovery01")
for _ in range(20)
])
snapshots = []
with pytest.raises(ModelHubError):
await service.resolve_with_recovery(
backend="codex", model_id="shared-model", request={}, recovery_observer=snapshots.append,
)
assert clock.delays == [1, 2, 4, 8, 16, 30, 30]
assert len(service.adapter.invocations) == 8
assert clock.elapsed == 91
assert snapshots[-1] is None
assert {snapshot["window_end"] for snapshot in snapshots if snapshot} == {(NOW + timedelta(seconds=120)).isoformat()}
assert {snapshot["started_at"] for snapshot in snapshots if snapshot} == {NOW.isoformat()}
asyncio.run(run())
@pytest.mark.parametrize("action_sibling", [None, "before", "after"])
@pytest.mark.parametrize("backend", ["codex", "claude", "opencode"])
def test_startup_passes_temporary_hub_supply_without_spending_another_window(tmp_path, backend, action_sibling):
async def run():
cooling = _source("src_recovery01", "Recovering")
sources = [cooling]
if action_sibling is not None:
action = _source("src_backup001", "Backup")
action.state = ModelHubSourceStateConfig(
status="needs_action", detail_key="models.source.needs_action.credential_revoked",
)
sources = [action, cooling] if action_sibling == "before" else [cooling, action]
service, clock = clock_service(tmp_path, sources=sources)
models = _canonicalize_fixed_test_routes(service)
source = next(item for item in service.store.load().sources if item.id == cooling.id)
await fail(service, source)
gateway = ModelHubTurnGateway(service)
router = ModelHubRuntimeRouter(service=service, turn_gateway=gateway, overlay_path=tmp_path / "overlay.json")
try:
model = models.get(backend, "shared-model")
launch = await router.resolve(backend, model, process_scope="/repo", turn_id="turn_startup")
assert launch.channel == "hub" and launch.source_id == source.id
if backend == "opencode":
overlay = await router.prepare_opencode_overlay()
assert "shared-model" in overlay.available_identifiers
assert clock.delays == []
assert service.adapter.invocations == []
assert gateway.correlation.recovery_snapshot("turn_startup") == []
finally:
await gateway.close()
asyncio.run(run())
@pytest.mark.parametrize("protocol", WIRE)
@pytest.mark.parametrize("streaming,verified,empty", [
(False, True, False), (True, True, False), (False, False, False),
(False, True, True), (True, True, True),
])
def test_real_gateway_retries_preoutput_then_preserves_one_response(tmp_path, protocol, streaming, verified, empty):
"""MH-RETRY-RECOVERY-001 / D8: real loopback evidence, never timer-based recovery."""
async def run():
calls = []
metadata, output, terminal, buffered = WIRE[protocol]
if not verified:
buffered = b'{"unknown":"permissive compatibility response"}'
if empty:
output = b""
buffered = {
"openai_responses": b'{"object":"response","status":"completed","output":[]}',
"openai_chat": b'{"object":"chat.completion","choices":[]}',
"anthropic": b'{"type":"message","content":[]}',
}[protocol]
async def respond(request):
calls.append(await request.json())
if len(calls) <= 2:
return web.json_response({"error": {"type": "server_error"}}, status=503)
# An admitted recovery attempt may think beyond the 120s window.
clock.advance(121)
return web.Response(
body=metadata + output + terminal if streaming else buffered,
content_type="text/event-stream" if streaming else "application/json",
)
app = web.Application()
app.router.add_post("/{path:.*}", respond)
runner = web.AppRunner(app, handler_cancellation=True)
await runner.setup()
await web.TCPSite(runner, "127.0.0.1", 0).start()
backend, endpoint = {
"openai_responses": ("codex", "responses"),
"openai_chat": ("opencode", "chat/completions"),
"anthropic": ("claude", "messages"),
}[protocol]
source = _source("src_loopback01", "Loopback", protocol=protocol, vendor="anthropic" if backend == "claude" else "openai")
state = EngineStateStore(tmp_path / "engine")
source.credential_ref = state.store_api_key("fixture-key", vendor=source.vendor, protocol=protocol)
service, clock = clock_service(tmp_path, sources=[source])
models = _canonicalize_fixed_test_routes(service)
state.sync_sources(service._bindings(service.store.load()))
client = EngineClient(
EngineConnection(f"http://127.0.0.1:{runner.addresses[0][1]}", "fixture-management", "fixture-gateway"),
timeout=0.1,
)
adapter = CLIProxyEngineAdapter(supervisor=SimpleNamespace(client=lambda: client), state_store=state)
adapter.ensure_installed = service.adapter.ensure_installed
service.adapter = adapter
service._engine_synced = True
gateway = ModelHubTurnGateway(service)
base_url, token = await gateway.endpoint(
backend, process_scope="/repo", turn_id="turn_recovered",
requested_model_id=models.get(backend, "shared-model"),
resolved_model_id="shared-model", source_id=source.id,
)
try:
async with aiohttp.ClientSession(trust_env=False) as downstream:
async with downstream.post(
f"{base_url}/v1/{endpoint}",
headers={"Authorization": f"Bearer {token}"},
json={"model": "shared-model", "stream": streaming},
) as response:
assert response.status == 200
assert await response.read() == (metadata + output + terminal if streaming else buffered)
assert len(calls) == 3
assert calls[0] == calls[1] == calls[2]
assert clock.delays == [30, 60]
assert adapter._active_transports == 0
assert gateway.correlation.recovery_snapshot("turn_recovered") == []
assert (source.id not in service.recovery._sources) is verified
assert sum(event["kind"] == "recover" for event in service.events.list()) == int(verified)
finally:
await gateway.close()
await runner.cleanup()
asyncio.run(run())
@pytest.mark.parametrize("protocol", WIRE)
@pytest.mark.parametrize("ending", ["cancel_owner", "slow_owner"])
def test_real_gateway_has_one_half_open_owner_and_interruptible_waiters(tmp_path, protocol, ending):
"""MH-RETRY-CONCURRENCY-001: real HTTP owners and waiters release only their slot."""
async def run():
connected, release, owner_closed, waiting = (asyncio.Event() for _ in range(4))
calls = []
buffered = WIRE[protocol][3]
async def respond(request):
calls.append(await request.json())
if len(calls) == 1:
connected.set()
try:
await release.wait()
finally:
owner_closed.set()
return web.Response(body=buffered, content_type="application/json")
app = web.Application()
app.router.add_post("/{path:.*}", respond)
runner = web.AppRunner(app, handler_cancellation=True)
await runner.setup()
await web.TCPSite(runner, "127.0.0.1", 0).start()
backend, endpoint = {
"openai_responses": ("codex", "responses"),
"openai_chat": ("opencode", "chat/completions"),
"anthropic": ("claude", "messages"),
}[protocol]
source = _source("src_concurrent1", "Concurrent", protocol=protocol)
state = EngineStateStore(tmp_path / "engine")
source.credential_ref = state.store_api_key("fixture-key", vendor=source.vendor, protocol=protocol)
service, clock = clock_service(tmp_path, sources=[source])
models = _canonicalize_fixed_test_routes(service)
await fail(service, source)
clock.advance(1)
state.sync_sources(service._bindings(service.store.load()))
client = EngineClient(
EngineConnection(f"http://127.0.0.1:{runner.addresses[0][1]}", "fixture-management", "fixture-gateway"),
timeout=0.1,
)
adapter = CLIProxyEngineAdapter(supervisor=SimpleNamespace(client=lambda: client), state_store=state)
adapter.ensure_installed = service.adapter.ensure_installed
service.adapter = adapter
service._engine_synced = True
async def wait_until_changed(delay):
waiting.set()
await asyncio.Future()
service.recovery.sleep = wait_until_changed
gateway = ModelHubTurnGateway(service)
model = models.get(backend, "shared-model")
base_url, first_token = await gateway.endpoint(
backend, process_scope="owner", turn_id="turn_owner",
requested_model_id=model, resolved_model_id="shared-model", source_id=source.id,
)
_, second_token = await gateway.endpoint(
backend, process_scope="waiter", turn_id="turn_waiter",
requested_model_id=model, resolved_model_id="shared-model", source_id=source.id,
)
tasks = []
try:
async with aiohttp.ClientSession(trust_env=False) as downstream:
def post(token):
return asyncio.create_task(downstream.post(
f"{base_url}/v1/{endpoint}",
headers={"Authorization": f"Bearer {token}"},
json={"model": "shared-model", "stream": False},
))
owner = post(first_token)
tasks.append(owner)
await asyncio.wait_for(connected.wait(), 2)
generation = service.recovery._sources[source.id].owner
assert generation is not None
waiter = post(second_token)
tasks.append(waiter)
await asyncio.wait_for(waiting.wait(), 2)
assert len(calls) == 1 and not waiter.done()
chain = service.agent_chain(backend, model)
assert chain["chain"][0]["recovery"] == "in_flight"
assert chain["chain"][0]["runnable"] is False
assert chain["supply_state"] == "waiting"
_assert_valid("agent-chain.schema.json", chain)
snapshot = gateway.correlation.recovery_snapshot("turn_waiter")
if backend == "opencode":
# Shared-server requests have no exact Turn owner yet.
assert snapshot == []
else:
assert len(snapshot) == 1 and snapshot[0]["phase"] == "waiting"
if ending == "cancel_owner":
owner.cancel()
with pytest.raises(asyncio.CancelledError):
await owner
await asyncio.wait_for(owner_closed.wait(), 2)
response = await asyncio.wait_for(waiter, 2)
assert response.status == 200 and await response.read() == buffered
assert len(calls) == 2
else:
clock.advance(121)
service.recovery.notify()
response = await asyncio.wait_for(waiter, 2)
assert response.status == (400 if backend == "codex" else 424)
assert "Retry-After" not in response.headers
assert await response.json() == {
"type": "error",
"error": {
"type": RECOVERY_EXHAUSTED_CODE,
"code": RECOVERY_EXHAUSTED_CODE,
"message": RECOVERY_EXHAUSTED_MESSAGE,
# The failure this request waited out names the ending, not the request.
"cause": "network",
},
}
assert len(calls) == 1 and not owner.done()
assert service.recovery._sources[source.id].owner == generation
release.set()
response = await asyncio.wait_for(owner, 2)
assert response.status == 200 and await response.read() == buffered
await gateway._drain_turn_requests("turn_owner")
await gateway._drain_turn_requests("turn_waiter")
assert adapter._active_transports == 0
assert source.id not in service.recovery._sources
assert gateway.correlation.recovery_snapshot("turn_waiter") == []
assert sum(event["kind"] == "recover" for event in service.events.list()) == 1
finally:
release.set()
for task in tasks:
task.cancel()
await asyncio.gather(*tasks, return_exceptions=True)
await gateway.close()
await runner.cleanup()
asyncio.run(run())
def test_permissive_success_releases_admission_without_verified_recovery(tmp_path):
async def run():
service, clock = clock_service(tmp_path, outcomes=[
_outcome(RawOutcomeKind.SUCCESS, source_id="src_recovery01", stream_started=True),
])
source = service.store.load().sources[0]
source.verification_pending = "vp_fixture"
await fail(service, source)
clock.advance(1)
result = await service.resolve_with_recovery(backend="codex", model_id="shared-model", request={})
assert result.outcome.kind is RawOutcomeKind.SUCCESS
entry = service.recovery._sources[source.id]
assert entry.owner is None and entry.failures == 1
assert service.agent_chain("codex", "shared-model")["chain"][0]["recovery"] == "eligible"
assert not any(event["kind"] == "recover" for event in service.events.list())
asyncio.run(run())
@pytest.mark.parametrize("action_first", [False, True])
def test_draft_preview_does_not_clear_live_health_and_mixed_chain_waits_for_the_healing_hop(
tmp_path, action_first,
):
"""MH-RETRY-MIXED-001: an action owed elsewhere cannot end the wait for a healing hop."""
async def run():
cooling, action = _source("src_primary01", "Primary"), _source("src_backup001", "Backup")
service, clock = clock_service(
tmp_path,
sources=[action, cooling] if action_first else [cooling, action],
outcomes=[_outcome(RawOutcomeKind.SUCCESS, source_id=cooling.id, stream_started=True)],
)
await fail(service, cooling)
draft = copy.deepcopy(service.store.load())
draft.sources[1 if action_first else 0].credential_ref = "cred_draft"
assert cooling.id not in service.recovery_annotations(draft)
assert cooling.id in service.recovery_annotations(service.store.load())
action.state = ModelHubSourceStateConfig(
status="needs_action", detail_key="models.source.needs_action.credential_revoked",
)
assert service.agent_chain("codex", "shared-model")["supply_state"] == "interrupted"
result = await service.resolve_with_recovery(backend="codex", model_id="shared-model", request={})
assert result.source_id == cooling.id and result.outcome.kind is RawOutcomeKind.SUCCESS
assert clock.delays == [1]
assert [source_id for source_id, *_ in service.adapter.invocations] == [cooling.id]
asyncio.run(run())
def test_a_request_that_attempted_keeps_exhausted_after_a_peer_takes_the_slot(tmp_path):
"""MH-RETRY-PROVENANCE-001: attempts from an earlier walk are still this request's own."""
async def run():
service, clock = clock_service(tmp_path, outcomes=[
_outcome(RawOutcomeKind.HTTP_ERROR, source_id="src_recovery01", status=500, code="server_error"),
])
source_id = service.store.load().sources[0].id
wait_for_eligibility = service.recovery.sleep
async def a_peer_claims_on_eligibility(seconds):
await wait_for_eligibility(seconds)
# A concurrent request wins the half-open slot and keeps it.
source = next(item for item in service.store.load().sources if item.id == source_id)
service.recovery.claim(source, service._reserve_settlement_generation(source_id))
service.recovery.sleep = a_peer_claims_on_eligibility
with pytest.raises(ModelHubError) as ended:
await service.resolve_with_recovery(backend="codex", model_id="shared-model", request={})
assert ended.value.code == RECOVERY_EXHAUSTED_CODE
assert ended.value.turn_outcome.outcome == "exhausted"
assert len(service.adapter.invocations) == 1
asyncio.run(run())
@pytest.mark.parametrize("read", ["agent_chain", "agent_chains", "get_agent_sources", "list_agents"])
def test_public_live_reads_capture_config_and_health_once(tmp_path, read):
async def run():
service, _clock = clock_service(tmp_path)
await fail(service, service.store.load().sources[0])
counts = {"config": 0, "health": 0}
load, annotations = service.store.load, service.recovery.annotations
def capture_config():
counts["config"] += 1
return load()
def capture_health(config):
counts["health"] += 1
return annotations(config)
service.store.load = capture_config
service.recovery.annotations = capture_health
args = {
"agent_chain": ("codex", "shared-model"),
"agent_chains": ("codex",),
"get_agent_sources": ("codex",),
"list_agents": (),
}[read]
result = getattr(service, read)(*args)
assert result
assert counts == {"config": 1, "health": 1}
asyncio.run(run())
def test_half_open_projection_cannot_hide_action_or_native_blockers(tmp_path):
async def run():
service, clock = clock_service(tmp_path)
source = service.store.load().sources[0]
await fail(service, source)
clock.advance(1)
generation = service._reserve_settlement_generation(source.id)
assert service.recovery.claim(source, generation)
chain = service.agent_chain("codex", "shared-model")
assert chain["chain"][0]["recovery"] == "in_flight"
_assert_valid("agent-chain.schema.json", chain)
for mutation in [
{"runnable": True},
{"health": "needs_action", "reason": "models.source.needs_action.credential_revoked"},
{"reason": "source_missing"},
{"channel": "native_cli", "reason": "native_cli_unavailable"},
{"recovery": "eligible"},
]:
invalid = copy.deepcopy(chain)
invalid["chain"][0].update(mutation)
with pytest.raises(AssertionError):
_assert_valid("agent-chain.schema.json", invalid)
source.state = ModelHubSourceStateConfig(
status="needs_action", detail_key="models.source.needs_action.credential_revoked",
)
blocked = service.agent_chain("codex", "shared-model")
assert "recovery" not in blocked["chain"][0]
assert blocked["supply_state"] == "interrupted"
_assert_valid("agent-chain.schema.json", blocked)
for phase in ("eligible", "in_flight"):
blocked["chain"][0]["recovery"] = phase
with pytest.raises(AssertionError):
_assert_valid("agent-chain.schema.json", blocked)
asyncio.run(run())
@pytest.mark.parametrize("kind,suffix", [
(RawOutcomeKind.NETWORK_ERROR, "network"), (RawOutcomeKind.TIMEOUT, "timeout"),
])
def test_selected_source_probe_does_not_claim_or_schedule_recovery(tmp_path, kind, suffix):
async def run():
service, clock = clock_service(tmp_path, outcomes=[_outcome(kind, source_id="src_recovery01")])
before = copy.deepcopy(service.store.load().to_payload())
result = await service.probe_source("src_recovery01", {"model": "shared-model"})
assert result["reachable"] is False
assert result["error"] == f"models.source.cooldown.{suffix}"
assert len(service.adapter.invocations) == 1
assert service.recovery._sources == {} and clock.delays == []
assert service.store.load().to_payload() == before
assert service.events.list() == []
asyncio.run(run())
def test_routing_blocker_drops_the_blocked_grants_quota_reading(tmp_path):
"""MH-QUOTA-018: A grant routing marks needs_action is re-read, not shown with its pre-block quota."""
async def run():
service, _clock = clock_service(tmp_path)
source = service.store.config.sources[0]
source.kind, source.billing = "subscription", "monthly"
reads = []
async def subscription_quota(source_id, vendor, credential_ref):
reads.append(source_id)
return {"plan": None, "windows": []}
service.adapter.subscription_quota = subscription_quota
assert (await service.quota_summary())["sources"][0]["state"] == "ok"
await fail(service, service.store.load().sources[0], "credential_revoked")
assert service.store.load().sources[0].state.status == "needs_action"
await service.quota_summary()
assert reads == [source.id, source.id]
asyncio.run(run())