Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
"""Recovery ownership and persistence boundaries from the first exact-head review."""
from __future__ import annotations
import asyncio
import copy
import json
import time
from contextlib import asynccontextmanager
from dataclasses import replace
from datetime import datetime, timedelta, timezone
from types import SimpleNamespace
import aiohttp
from aiohttp import web
import pytest
from config.v2_config import ModelHubConfig, ModelHubRouteConfig, ModelHubRouteHopConfig, ModelHubSourceStateConfig
from core.handlers.model_hub.adapter import RawOutcomeKind
from core.handlers.model_hub.retry import RECOVERY_EXHAUSTED_CODE, RECOVERY_EXHAUSTED_MESSAGE
from core.handlers.model_hub.service import ModelHubError, ModelHubService
from core.handlers.model_hub.stream_wire import ProtocolSSEState
from core.handlers.model_hub import turn_gateway
from core.handlers.model_hub.turn_gateway import ModelHubTurnGateway
from core.run_settlement import SETTLED_BY_TERMINAL_RESULT
from modules.agents.model_hub import ModelHubRuntimeRouter, bind_launch
from tests.test_model_hub_l3 import _assert_valid, _canonicalize_fixed_test_routes, _outcome, _source
from tests.test_model_hub_retry_policy import WIRE, clock_service, fail
from vibe.model_hub_runtime import adapter as engine_adapter_module
from vibe.model_hub_runtime import client as engine_client_module
from vibe.model_hub_runtime.adapter import CLIProxyEngineAdapter
from vibe.model_hub_runtime.client import EngineClient, EngineConnection
from vibe.model_hub_runtime.state import EngineStateStore
from vibe.model_hub_runtime.supervisor import EngineUnavailableError
BACKENDS = {
"codex": ("responses", "openai_responses"),
"claude": ("messages", "anthropic"),
"opencode": ("chat/completions", "openai_chat"),
}
class FreshStore:
"""Every read reconstructs persisted data; failed writes cannot mutate it."""
def __init__(self, config):
self.persisted = copy.deepcopy(config.to_payload())
self.write_error = False
def load(self):
return ModelHubConfig.from_payload(copy.deepcopy(self.persisted))
def save(self, config):
if self.write_error:
if isinstance(self.write_error, BaseException):
raise self.write_error
raise OSError("fixture: read-only configuration")
self.persisted = copy.deepcopy(config.to_payload())
def mutate(self, operation):
config = self.load()
if operation(config):
self.save(config)
def configured_service(tmp_path, **kwargs):
service, clock = clock_service(tmp_path, **kwargs)
models = _canonicalize_fixed_test_routes(service)
models["opencode"] = "shared-model"
return service, clock, models
@asynccontextmanager
async def gateway_client(service, backend, model):
gateway = ModelHubTurnGateway(service)
source = service.store.load().sources[0]
base, token = await gateway.endpoint(
backend, process_scope="lifecycle", turn_id="turn_lifecycle",
requested_model_id=model, resolved_model_id="shared-model", source_id=source.id,
)
try:
async with aiohttp.ClientSession(trust_env=False) as client:
yield gateway, client, f"{base}/v1/{BACKENDS[backend][0]}", {"Authorization": f"Bearer {token}"}
finally:
await gateway.close()
@asynccontextmanager
async def loopback_engine(tmp_path, service, respond, *, models=None):
"""Use the production client/admission/stream observer against local HTTP."""
config = service.store.load()
state = EngineStateStore(tmp_path / "engine")
for source in config.sources:
source.credential_ref = state.store_api_key(
"fixture", vendor=source.vendor, protocol=source.protocol, base_url=source.base_url,
)
service.store.save(config)
state.sync_sources(service._bindings(config))
app = web.Application()
app.router.add_post("/{path:.*}", respond)
if models is not None:
app.router.add_get("/v1/models", models)
runner = web.AppRunner(app, handler_cancellation=True)
await runner.setup()
await web.TCPSite(runner, "127.0.0.1", 0).start()
engine_client = EngineClient(
EngineConnection(f"http://127.0.0.1:{runner.addresses[0][1]}", "fixture-management", "fixture-gateway"),
)
adapter = CLIProxyEngineAdapter(supervisor=SimpleNamespace(client=lambda: engine_client), state_store=state)
adapter.ensure_installed = service.adapter.ensure_installed
service.adapter = adapter
service._engine_synced = True
try:
yield adapter, engine_client
finally:
await runner.cleanup()
@pytest.mark.parametrize("stream", [False, True])
def test_unavailable_production_client_releases_unadmitted_half_open_claim(tmp_path, stream):
"""MH-RETRY-OWNERSHIP-001: completed engine-down never fires on_admitted."""
async def run():
service, clock, models = configured_service(tmp_path)
source = service.store.load().sources[0]
calls = []
async def respond(request):
calls.append(await request.json())
return web.Response(body=WIRE["openai_responses"][3], content_type="application/json")
def unavailable():
raise EngineUnavailableError("fixture: no running engine client")
async with loopback_engine(tmp_path, service, respond) as (adapter, engine_client):
await fail(service, source)
clock.advance(1)
adapter.supervisor.client = unavailable
observed = []
snapshots = []
with pytest.raises(ModelHubError, match="engine_down"):
await service.resolve_with_recovery(
backend="codex", model_id=models["codex"], request={}, stream=stream,
attempt_observer=lambda *args: observed.append(args),
recovery_observer=snapshots.append,
)
assert observed == [] # Not an admitted request, and not another failed Source attempt.
assert not any(snapshot and snapshot["attempt_count"] for snapshot in snapshots)
assert adapter._active_transports == 0 and not service._mutation_lock.locked()
entry = service.recovery._sources[source.id]
assert entry.owner is None and entry.failures == 1
assert service.agent_chain("codex", models["codex"])["chain"][0]["recovery"] == "eligible"
# The next real request can acquire the same half-open admission.
adapter.supervisor.client = lambda: engine_client
async with gateway_client(service, "codex", models["codex"]) as (_gateway, client, url, headers):
async with client.post(url, headers=headers, json={"model": "shared-model"}) as response:
assert response.status == 200
assert await response.read() == WIRE["openai_responses"][3]
assert len(calls) == 1 and adapter._active_transports == 0
assert source.id not in service.recovery._sources
asyncio.run(run())
@pytest.mark.parametrize("stream", [False, True])
def test_an_unnamed_error_on_a_half_open_attempt_releases_the_claim_and_spares_the_source(tmp_path, stream):
"""Ambiguous evidence writes no Source health (decision 2026-10-11), and it must not keep the half-open slot."""
async def run():
service, clock, models = configured_service(tmp_path)
source = service.store.load().sources[0]
responses = [
web.Response(
status=200,
content_type="text/event-stream",
body=b'event: error\ndata: {"type":"error","code":"relay_specific_error",'
b'"message":"upstream protocol error","param":null,"sequence_number":0}\n\n',
)
if stream
else web.json_response({"error": {"type": "relay_specific_error", "message": "upstream protocol error"}}),
web.Response(body=WIRE["openai_responses"][3], content_type="application/json"),
]
async def respond(_request):
return responses.pop(0)
async with loopback_engine(tmp_path, service, respond):
await fail(service, source)
clock.advance(1)
before = copy.deepcopy(service.store.load().to_payload())
with pytest.raises(ModelHubError):
await service.resolve_with_recovery(
backend="codex", model_id=models["codex"], request={}, stream=stream,
)
entry = service.recovery._sources[source.id]
assert entry.owner is None and entry.failures == 1
assert service.store.load().to_payload() == before
assert service.agent_chain("codex", models["codex"])["chain"][0]["recovery"] == "eligible"
# The next real request can acquire the same half-open admission.
async with gateway_client(service, "codex", models["codex"]) as (_gateway, client, url, headers):
async with client.post(url, headers=headers, json={"model": "shared-model"}) as response:
assert response.status == 200
assert await response.read() == WIRE["openai_responses"][3]
assert responses == []
asyncio.run(run())
@pytest.mark.parametrize("backend", ["codex", "claude"])
@pytest.mark.parametrize("diagnostic,delay", [
("503 server error", 30), ("429 rate limit", 60), ("quota exhausted", 300), ("connection failed", 0),
])
def test_native_failure_never_enters_http_recovery_streak(tmp_path, backend, diagnostic, delay):
"""MH-RETRY-NATIVE-002: native success has no Hub handle to clear a streak."""
async def run():
source = _source(
"src_native001", "Native", channel="native_cli", status="active",
vendor="anthropic" if backend == "claude" else "openai",
)
service, clock, models = configured_service(tmp_path, sources=[source])
source.models[0].id = models[backend]
service.store.config.agents[backend].routes[models[backend]] = ModelHubRouteConfig(
hops=(ModelHubRouteHopConfig(source.id, models[backend]),),
)
router = ModelHubRuntimeRouter(
service=service, native_cli_ready=lambda _backend: True, overlay_path=tmp_path / "overlay.json",
)
for _ in range(3):
launch = await router.resolve(backend, models[backend])
assert launch.channel == "native_cli"
context = SimpleNamespace(platform_specific={})
bind_launch(context, launch)
assert await router.record_native_failure(context, diagnostic) is bool(delay)
current = service.store.load().sources[0]
if delay:
assert current.state.retry_at == (clock.now() + timedelta(seconds=delay)).isoformat()
assert service.recovery.annotations(service.store.load()) == {}
assert service.recovery._sources == {}
clock.advance(delay)
assert service.agent_chain(backend, models[backend])["chain"][0]["runnable"]
assert service.adapter.invocations == []
assert not any(event["kind"] == "recover" for event in service.events.list())
asyncio.run(run())
@pytest.mark.parametrize("backend", BACKENDS)
@pytest.mark.parametrize("stream", [False, True])
@pytest.mark.parametrize("write_failure", ["io", "recovery_warning"])
def test_verified_response_survives_failed_recovery_persistence_and_fresh_reads(tmp_path, backend, stream, write_failure):
"""MH-RETRY-PERSISTENCE-001: stored cooldown cannot resurrect after recovery."""
async def run():
service, clock, models = configured_service(tmp_path)
metadata, output, terminal, buffered = WIRE[BACKENDS[backend][1]]
body = metadata + output + terminal if stream else buffered
async def respond(_request):
return web.Response(body=body, content_type="text/event-stream" if stream else "application/json")
async with loopback_engine(tmp_path, service, respond) as (adapter, _engine_client):
service.store = FreshStore(service.store.load())
source = service.store.load().sources[0]
await fail(service, source, "server_error")
clock.advance(30)
before = copy.deepcopy(service.store.persisted)
service.store.write_error = True if write_failure == "io" else ValueError("fixture: recovery warnings")
async with gateway_client(service, backend, models[backend]) as (_gateway, client, url, headers):
async with client.post(url, headers=headers, json={"model": "shared-model", "stream": stream}) as response:
assert response.status == 200
assert await response.read() == body
assert adapter._active_transports == 0
assert service.store.persisted == before # A fresh disk view still has the old cooldown.
for _ in range(3):
chain = service.agent_chain(backend, models[backend])
assert chain["chain"][0]["health"] == "healthy"
assert chain["chain"][0]["runnable"] is True
assert "recovery" not in chain["chain"][0]
assert sum(event["kind"] == "recover" for event in service.events.list()) == 1
# A later real failure starts at the first delay, even if the stored old
# cooldown could not be retired. No further caller loses its response.
await fail(service, service.store.load().sources[0], "server_error")
assert service.recovery._sources[source.id].failures == 1
assert service.agent_chain(backend, models[backend])["supply_state"] == "waiting"
asyncio.run(run())
@pytest.mark.parametrize("change", [
"none", "new_deadline", "new_reason", "needs_action", "error", "credential", "verification",
"endpoint", "deleted", "retired_model", "disk_cleared",
])
def test_recovered_disk_observation_is_exact_and_never_masks_new_authority(tmp_path, change):
async def run():
service, clock, models = configured_service(tmp_path)
service.store = FreshStore(service.store.load())
source = service.store.load().sources[0]
await fail(service, source, "server_error")
clock.advance(30)
generation = service._reserve_settlement_generation(source.id)
service.store.write_error = True
await service._record_recovery_success(source.id, generation, backend="codex", model_id=models["codex"])
assert service.agent_chain("codex", models["codex"])["chain"][0]["health"] == "healthy"
config = service.store.load()
current = config.sources[0]
if change == "new_deadline":
current.state.retry_at = (clock.now() + timedelta(seconds=60)).isoformat()
elif change == "new_reason":
current.state.detail_key = "models.source.cooldown.rate_limited"
elif change in {"needs_action", "error"}:
current.state = ModelHubSourceStateConfig(
status=change,
detail_key="models.source.error.unclassified" if change == "error"
else "models.source.needs_action.credential_revoked",
)
elif change == "credential":
current.credential_ref = "cred_replaced"
elif change == "verification":
ModelHubService._mark_source_unverified(current)
elif change == "endpoint":
current.base_url = "https://replacement.invalid"
elif change == "deleted":
config.sources = []
for agent in config.agents.values():
agent.sources.order = []
agent.routes = {}
elif change == "retired_model":
current.models[0].retired = True
elif change == "disk_cleared":
current.state = ModelHubSourceStateConfig(status="standby")
service.store.persisted = copy.deepcopy(config.to_payload())
chain = service.agent_chain("codex", models["codex"])
_assert_valid("agent-chain.schema.json", chain)
if change == "deleted":
assert chain["chain"] == [] and chain["supply_state"] == "interrupted"
assert service.recovery._sources == service.recovery._retired_cooldowns == {}
return
hop = chain["chain"][0]
if change in {"none", "disk_cleared"}:
assert hop["health"] == "healthy" and hop["runnable"]
assert "recovery" not in hop
elif change == "new_deadline":
assert hop["health"] == "cooldown" and not hop["runnable"]
assert chain["supply_state"] == "waiting"
elif change in {"new_reason", "credential", "verification", "endpoint"}:
assert hop["recovery"] == "eligible" # Not proven healthy for this new observation.
else:
assert not hop["runnable"] and chain["supply_state"] == "interrupted"
# Re-delivery of the old success cannot retire a replacement identity.
if change in {"credential", "verification", "endpoint"}:
await service._record_recovery_success(source.id, generation, backend="codex", model_id=models["codex"])
assert service.agent_chain("codex", models["codex"])["chain"][0]["recovery"] == "eligible"
assert sum(event["kind"] == "recover" for event in service.events.list()) == 1
if change == "disk_cleared":
# Even the identical old payload is new once its retirement was
# observed on disk; the tombstone is not a permanent Source flag.
config.sources[0].state = replace(source.state, status="cooldown", retry_at=clock.now().isoformat(),
detail_key="models.source.cooldown.server_error")
service.store.persisted = config.to_payload()
assert service.agent_chain("codex", models["codex"])["chain"][0]["recovery"] == "eligible"
asyncio.run(run())
def test_recovered_annotation_cannot_retire_a_different_preview_cooldown(tmp_path):
async def run():
service, clock, models = configured_service(tmp_path)
service.store = FreshStore(service.store.load())
source = service.store.load().sources[0]
await fail(service, source, "server_error")
clock.advance(30)
service.store.write_error = True
await service._record_recovery_success(
source.id, service._reserve_settlement_generation(source.id),
backend="codex", model_id=models["codex"],
)
draft = service.store.load()
draft.sources[0].state.retry_at = (clock.now() + timedelta(seconds=300)).isoformat()
assert service._agent_chain(draft, "codex", models["codex"])["supply_state"] == "waiting"
assert service.agent_chain("codex", models["codex"])["chain"][0]["health"] == "healthy"
asyncio.run(run())
@pytest.mark.parametrize("backend", BACKENDS)
@pytest.mark.parametrize("status,code,attempts", [(503, None, 3), (429, None, 2), (429, "quota_exhausted", 1)])
def test_failure_persistence_error_keeps_temporary_admission_and_original_terminal(tmp_path, backend, status, code, attempts):
async def run():
service, _clock, models = configured_service(tmp_path, outcomes=[
_outcome(RawOutcomeKind.HTTP_ERROR, source_id="src_recovery01", status=status, code=code)
for _ in range(attempts)
])
service.store = FreshStore(service.store.load())
before = copy.deepcopy(service.store.persisted)
service.store.write_error = True
reason = code or ("server_error" if status == 503 else "rate_limited")
async with gateway_client(service, backend, models[backend]) as (gateway, client, url, headers):
async with client.post(url, headers=headers, json={"model": "shared-model"}) as response:
assert response.status == (400 if backend == "codex" else 424)
ending = (await response.json())["error"]
# The ending names the failure recovery waited out, whichever way the window closed.
assert (ending["code"], ending["cause"]) == (RECOVERY_EXHAUSTED_CODE, reason)
projection = gateway.correlation.terminal_projection("turn_lifecycle", backend=backend)
if backend == "opencode":
assert projection is None # Shared-server HTTP does not prove an exact Turn.
else:
assert projection is not None
assert [blocker.reason for blocker in projection.supply_facts.blockers] == [reason]
gateway.correlation.settle("turn_lifecycle", settled_by=SETTLED_BY_TERMINAL_RESULT)
record = service.provenance.get("turn_lifecycle")
assert [attempt["http_status"] for attempt in record["failed_attempts"]] == [status] * attempts
assert len(service.adapter.invocations) == attempts
assert service.store.persisted == before
chain = service.agent_chain(backend, models[backend])
assert chain["supply_state"] == "waiting"
assert chain["chain"][0]["health"] == "cooldown"
_assert_valid("agent-chain.schema.json", chain)
with pytest.raises(ModelHubError) as terminal:
await service.resolve(backend=backend, model_id=models[backend], request={})
assert [blocker.reason for blocker in terminal.value.blockers] == ["cooldown"]
asyncio.run(run())
def test_a_usage_cap_cools_its_source_as_quota_until_the_upstream_reset(tmp_path):
"""MH-RETRY-ADVICE-002: a subscription's usage cap waits for its own reset, not a 60 s rate limit."""
async def run():
service, clock, models = configured_service(tmp_path)
# A ChatGPT subscription past its weekly cap, as the engine relays it:
# the upstream body unchanged and no Retry-After header.
resets_in = 475224
resets_at = int(time.time()) + resets_in
body = json.dumps({"error": {
"type": "usage_limit_reached", "message": "The usage limit has been reached",
"plan_type": "prolite", "resets_at": resets_at, "eligible_promo": None,
"limit_window_minutes": 10080, "resets_in_seconds": resets_in,
}}, separators=(",", ":"))
calls = []
async def respond(request):
calls.append(await request.json())
return web.Response(status=429, text=body, content_type="application/json")
async with loopback_engine(tmp_path, service, respond):
async with gateway_client(service, "codex", models["codex"]) as (_gateway, client, url, headers):
async with client.post(url, headers=headers, json={"model": "shared-model"}) as response:
ending = (await response.json())["error"]
assert (ending["code"], ending["cause"]) == (RECOVERY_EXHAUSTED_CODE, "quota_exhausted")
state = service.store.load().sources[0].state
assert (state.status, state.detail_key) == ("cooldown", "models.source.cooldown.quota_exhausted")
assert state.retry_at == datetime.fromtimestamp(resets_at, timezone.utc).isoformat()
# The cap is not probed again inside the turn's recovery window.
assert len(calls) == 1 and clock.delays == []
asyncio.run(run())
@pytest.mark.parametrize("stream", [False, True])
@pytest.mark.parametrize("backend", ["claude", "codex"])
@pytest.mark.parametrize("windows,advice,expected", [
({"5h": 18000}, {}, 18000),
({"7d": 604800}, {}, 604800),
({"5h": 18000, "7d": 604800}, {"Retry-After": "60"}, 604800),
({"5h": 18000}, {"Retry-After": "24000", "Retry-After-Ms": "25000000"}, 25000),
({"5h": 18000}, {"Retry-After": "25000", "Retry-After-Ms": "24000000"}, 25000),
({"5h": 20}, {}, 300), # A short upstream reset cannot shorten the quota ladder.
])
def test_claude_cap_headers_drive_persisted_recovery_without_early_probes(
tmp_path, monkeypatch, caplog, stream, backend, windows, advice, expected,
):
"""MH-RETRY-ADVICE-004: shared Claude caps survive settlement/restart without short-ladder probes or raw headers."""
async def run():
source = _source("src_claudecap", "Claude", vendor="anthropic", protocol="anthropic")
service, clock, models = configured_service(tmp_path, sources=[source])
receipt = clock.now()
deadline = receipt + timedelta(seconds=expected)
class ReceiptClock(datetime):
@classmethod
def now(cls, tz=None):
return clock.now()
monkeypatch.setattr(engine_client_module, "datetime", ReceiptClock)
raw_headers = {
"aNtHrOpIc-Ratelimit-Unified-Status": "rejected",
"Authorization": "private-upstream-credential",
"Set-Cookie": "private-upstream-cookie",
"Organization-Id": "private-upstream-org",
"Anthropic-Ratelimit-Unified-Unknown": "private-unknown-value",
**advice,
}
for window, delay in windows.items():
raw_headers[f"Anthropic-Ratelimit-Unified-{window}-Status"] = "rejected"
raw_headers[f"Anthropic-Ratelimit-Unified-{window}-Reset"] = str(int(receipt.timestamp()) + delay)
calls = []
async def respond(request):
calls.append(await request.json())
if clock.now() < deadline:
return web.json_response({"error": {"type": "rate_limit_error", "message": "Rate limited"}},
status=429, headers=raw_headers)
metadata, output, terminal, buffered = WIRE[BACKENDS[backend][1]]
return web.Response(body=metadata + output + terminal if stream else buffered,
content_type="text/event-stream" if stream else "application/json")
async with loopback_engine(tmp_path, service, respond):
service.store = FreshStore(service.store.load())
async with gateway_client(service, backend, models[backend]) as (gateway, client, url, headers):
async with client.post(url, headers=headers, json={"model": "shared-model", "stream": stream}) as response:
body = await response.json()
assert body["error"]["cause"] == "quota_exhausted"
assert body["error"]["code"] == RECOVERY_EXHAUSTED_CODE
assert not any(name.lower() in response.headers for name in raw_headers)
gateway.correlation.settle("turn_lifecycle", settled_by=SETTLED_BY_TERMINAL_RESULT)
record = service.provenance.get("turn_lifecycle")
assert record["failed_attempts"] == [{
"source_id": source.id, "configured_model_id": "shared-model", "channel": "hub",
"reason": "quota_exhausted", "http_status": 429, "upstream_detail": "Rate limited",
}]
stored = service.store.load().sources[0].state
assert (stored.status, stored.detail_key, stored.retry_at) == (
"cooldown", "models.source.cooldown.quota_exhausted", deadline.isoformat(),
)
assert len(calls) == 1 and clock.delays == []
assert service.events.list()[0]["reason"] == "quota_exhausted"
surfaces = json.dumps([
body, record, service.list_events(), service.list_sources(), service.store.persisted, caplog.text,
])
assert "anthropic-ratelimit" not in surfaces.lower()
for secret in ("private-upstream-credential", "private-upstream-cookie", "private-upstream-org", "private-unknown-value"):
assert secret not in surfaces
# A new policy restores eligibility from fresh persisted config,
# never from this request's header projection or live retry streak.
service.recovery = clock.policy()
clock.advance(expected - 121)
with pytest.raises(ModelHubError):
await service.resolve_with_recovery(backend=backend, model_id=models[backend], request={})
assert len(calls) == 1 and clock.delays == []
chain = service.agent_chain(backend, models[backend])
assert chain["supply_state"] == "waiting"
assert chain["chain"][0]["retry_at"] == deadline.isoformat()
assert chain["chain"][0]["runnable"] is False
_assert_valid("agent-chain.schema.json", chain)
# A concurrent later-completing short failure cannot pull it forward.
await fail(service, service.store.load().sources[0], "rate_limited")
assert service.store.load().sources[0].state.retry_at == deadline.isoformat()
assert service.store.load().sources[0].state.detail_key == "models.source.cooldown.quota_exhausted"
clock.advance(121)
assert service.store.load().sources[0].state.status == "cooldown"
async with gateway_client(service, backend, models[backend]) as (_gateway, client, url, headers):
async with client.post(url, headers=headers, json={"model": "shared-model", "stream": stream}) as response:
assert response.status == 200
await response.read()
assert len(calls) == 2
assert service.store.load().sources[0].state.status == "standby"
asyncio.run(run())
@pytest.mark.parametrize("engine_refuses", [True, False], ids=["engine", "upstream"])
def test_a_401_the_engine_itself_gave_spares_the_source(tmp_path, engine_refuses):
"""MH-RUNTIME-015: the engine's own 401 is told from an upstream's by a witness no upstream answers (#2755).
The bodies are identical, as an upstream JSON error passes through the
engine unchanged. Only the engine-local listing refusing the same key makes
the 401 the engine's: the turn ends as the engine being down, the Source
keeps its health, and the next admission checks the engine again.
"""
async def run():
service, _clock, models = configured_service(tmp_path)
before = service.store.load().sources[0].state
async def respond(request):
await request.read()
return web.json_response({"error": "Invalid API key"}, status=401)
async def listing(_request):
if engine_refuses:
return web.json_response({"error": "Invalid API key"}, status=401)
return web.json_response({"object": "list", "data": []})
rechecks = []
async with loopback_engine(tmp_path, service, respond, models=listing) as (adapter, _engine):
adapter.supervisor.recheck_engine = lambda: rechecks.append(True)
async with gateway_client(service, "codex", models["codex"]) as (_gateway, client, url, headers):
async with client.post(url, headers=headers, json={"model": "shared-model"}) as response:
assert response.status >= 400
state = service.store.load().sources[0].state
if engine_refuses:
assert state == before
assert rechecks == [True]
else:
assert (state.status, state.detail_key) == ("needs_action", "models.source.needs_action.credential_revoked")
assert rechecks == []
asyncio.run(run())
_SPEND_LIMIT = ("needs_action", "models.source.needs_action.spend_limit_reached")
@pytest.mark.parametrize(("error", "settled"), [
# Codex's own client turns this 429 type into "upgrade to Plus" (openai/codex
# codex-api/src/api_bridge.rs); only a plan change ends it.
(
{"type": "usage_not_included", "message": "To use Codex with your ChatGPT plan, upgrade to Plus."},
("needs_action", "models.source.needs_action.plan_excludes_product"),
),
# OpenAI's error-code guide: a billing 429 names its cause in the code, the
# type may stay the generic insufficient_quota, and retrying restores none.
(
{"type": "insufficient_quota", "code": "credit_balance_exhausted", "message": "No prepaid credits remain."},
("needs_action", "models.source.needs_action.balance_exhausted"),
),
({"type": "insufficient_quota", "code": "organization_spend_limit_exceeded", "message": "Spend limit."}, _SPEND_LIMIT),
({"type": "insufficient_quota", "code": "project_spend_limit_exceeded", "message": "Spend limit."}, _SPEND_LIMIT),
({"type": "insufficient_quota", "code": "organization_usage_limit_exceeded", "message": "Usage limit."}, _SPEND_LIMIT),
# A request-class type beside the code does not decide it.
({"type": "invalid_request_error", "code": "project_spend_limit_exceeded", "message": "Spend limit."}, _SPEND_LIMIT),
# The generic type alone names no cause, so it keeps cooling as quota.
(
{"type": "insufficient_quota", "message": "You exceeded your current quota."},
("cooldown", "models.source.cooldown.quota_exhausted"),
),
], ids=lambda value: value.get("code", value.get("type")) if isinstance(value, dict) else None)
def test_a_plan_or_billing_429_settles_its_source_by_the_code_that_names_it(tmp_path, error, settled):
"""MH-RETRY-ADVICE-003: a 429 naming a plan or billing cause is not a 60 s rate limit (#2530)."""
async def run():
service, _clock, models = configured_service(tmp_path)
body = json.dumps({"error": error}, separators=(",", ":"))
calls = []
async def respond(request):
calls.append(await request.json())
return web.Response(status=429, text=body, content_type="application/json")
async with loopback_engine(tmp_path, service, respond):
async with gateway_client(service, "codex", models["codex"]) as (_gateway, client, url, headers):
async with client.post(url, headers=headers, json={"model": "shared-model"}) as response:
assert response.status >= 400
state = service.store.load().sources[0].state
assert (state.status, state.detail_key) == settled
assert (state.retry_at is None) is (settled[0] == "needs_action")
assert len(calls) == 1
asyncio.run(run())
@pytest.mark.parametrize("persisted", [False, True])
@pytest.mark.parametrize("reason,delay", [
("server_error", 30), ("rate_limited", 60), ("quota_exhausted", 300), ("network", 1),
])
@pytest.mark.parametrize("transition", ["none", "needs_action", "error", "retired_model", "recovered"])
def test_terminal_supply_projection_uses_effective_transient_facts_and_stronger_blockers(
tmp_path, persisted, reason, delay, transition,
):
async def run():
service, clock, models = configured_service(tmp_path)
service.store = FreshStore(service.store.load())
service.store.write_error = not persisted
source = service.store.load().sources[0]
await fail(service, source, reason)
config = service.store.load()
if transition in {"needs_action", "error"}:
config.sources[0].state = ModelHubSourceStateConfig(
status=transition,
detail_key="models.source.error.unclassified" if transition == "error"
else "models.source.needs_action.credential_revoked",
)
elif transition == "retired_model":
config.sources[0].models[0].retired = True
service.store.persisted = config.to_payload()
if transition == "recovered":
clock.advance(delay)
service.store.write_error = True
await service._record_recovery_success(
source.id, service._reserve_settlement_generation(source.id),
backend="codex", model_id=models["codex"],
)
chain = service.agent_chain("codex", models["codex"])
assert chain["supply_state"] == "ok" and chain["chain"][0]["health"] == "healthy"
_config, resolution = service._inspect_terminal_chain(backend="codex", model_id=models["codex"])
assert resolution.candidate_hops[0].cooldown_reason is None
return
with pytest.raises(ModelHubError) as terminal:
await service.resolve(backend="codex", model_id=models["codex"], request={})
expected = {
"none": reason, "needs_action": "credential_revoked",
"error": "unclassified_error", "retired_model": "model_unsupported",
}[transition]
facts = terminal.value.turn_outcome.supply_facts
assert [blocker.reason for blocker in facts.blockers] == [expected]
assert facts.supply_state == ("waiting" if transition == "none" else "interrupted")
assert bool(facts.retry_at) is (transition == "none")
assert [blocker.reason for blocker in terminal.value.blockers] == [
("network" if reason == "network" else "cooldown") if transition == "none" else expected,
]
asyncio.run(run())
def test_persisted_cooldown_without_failure_evidence_does_not_invent_server_error(tmp_path):
async def run():
service, clock, models = configured_service(tmp_path)
source = service.store.load().sources[0]
source.state = ModelHubSourceStateConfig(
status="cooldown", retry_at=(clock.now() + timedelta(seconds=30)).isoformat(),
)
with pytest.raises(ModelHubError) as terminal:
await service.resolve(backend="codex", model_id=models["codex"], request={})
assert [blocker.reason for blocker in terminal.value.blockers] == ["cooldown"]
assert terminal.value.turn_outcome.supply_facts.blockers == ()
asyncio.run(run())
@pytest.mark.parametrize("backend", BACKENDS)
@pytest.mark.parametrize("wake", ["owner_success", "runnable_edit", "blocked_edit"])
def test_expiry_remains_a_closed_terminal_when_live_supply_changes(tmp_path, backend, wake):
"""MH-RETRY-EXPIRY-001: an exhausted waiter is not a fresh route failure."""
async def run():
service, clock, models = configured_service(tmp_path)
source = service.store.load().sources[0]
await fail(service, source)
clock.advance(1)
generation = service._reserve_settlement_generation(source.id)
assert service.recovery.claim(source, generation)
entered, release = asyncio.Event(), asyncio.Event()
async def held_wait(_seconds):
entered.set()
await release.wait()
service.recovery.sleep = held_wait
async with gateway_client(service, backend, models[backend]) as (_gateway, client, url, headers):
task = asyncio.create_task(client.post(url, headers=headers, json={"model": "shared-model"}))
try:
await asyncio.wait_for(entered.wait(), 2)
clock.advance(121)
if wake == "owner_success":
await service._record_recovery_success(
source.id, generation, backend=backend, model_id=models[backend],
)
elif wake == "runnable_edit":
source.credential_ref = "cred_replaced"
service.recovery.reconcile(service.store.load())
else:
source.state = ModelHubSourceStateConfig(
status="needs_action", detail_key="models.source.needs_action.credential_revoked",
)
release.set()
response = await asyncio.wait_for(task, 2)
assert response.status == (400 if backend == "codex" else 424)
assert "Retry-After" not in response.headers
assert await response.json() == {
"type": "error",
"error": {
"type": RECOVERY_EXHAUSTED_CODE, "code": RECOVERY_EXHAUSTED_CODE,
"message": RECOVERY_EXHAUSTED_MESSAGE, "cause": "network",
},
}
assert service.adapter.invocations == []
finally:
release.set()
task.cancel()
await asyncio.gather(task, return_exceptions=True)
asyncio.run(run())
@pytest.mark.parametrize("backend", BACKENDS)
@pytest.mark.parametrize("change", ["none", "owner_success", "blocked_edit"])
def test_window_expiry_during_engine_preparation_preserves_admission_cause(tmp_path, backend, change):
async def run():
service, clock, models = configured_service(tmp_path)
source = service.store.load().sources[0]
await fail(service, source)
original_prepare = service._prepare_engine_for_demand
async def expire_during_preparation(**kwargs):
clock.advance(121)
if change == "owner_success":
await service._record_recovery_success(
source.id, service._latest_source_attempt_generation[source.id],
backend=backend, model_id=models[backend],
)
elif change == "blocked_edit":
source.state = ModelHubSourceStateConfig(
status="needs_action", detail_key="models.source.needs_action.credential_revoked",
)
await original_prepare(**kwargs)
service._prepare_engine_for_demand = expire_during_preparation
async with gateway_client(service, backend, models[backend]) as (gateway, client, url, headers):
async with client.post(url, headers=headers, json={"model": "shared-model"}) as response:
assert response.status == (400 if backend == "codex" else 424)
assert (await response.json())["error"]["code"] == RECOVERY_EXHAUSTED_CODE
assert "Retry-After" not in response.headers
projection = gateway.correlation.terminal_projection("turn_lifecycle", backend=backend)
if backend == "opencode":
assert projection is None
else:
# The wire retains admission expiry while the canonical
# projection still reports current supply, not a stale outage.
assert projection.supply_facts.supply_state == (
"interrupted" if change == "blocked_edit" else "waiting"
)
assert [blocker.reason for blocker in projection.supply_facts.blockers] == (
["credential_revoked"] if change == "blocked_edit" else []
)
assert clock.delays == [1] and service.adapter.invocations == []
assert not service._mutation_lock.locked()
asyncio.run(run())
@pytest.mark.parametrize("backend", BACKENDS)
@pytest.mark.parametrize("ending", ["temporary", "invalid", "auth", "engine", "post_output"])
def test_admitted_terminal_classification_is_not_replaced_by_elapsed_window(tmp_path, backend, ending):
async def run():
service, clock, models = configured_service(tmp_path)
metadata, output, _terminal, _buffered = WIRE[BACKENDS[backend][1]]
calls = []
async def respond(request):
calls.append(await request.json())
if len(calls) == 1:
return web.json_response({"error": {"type": "server_error"}}, status=503)
clock.advance(121) # A connected inference may outlive admission.
if ending == "post_output":
error = b'data: {"type":"error","error":{"type":"server_error","message":"fixture"}}\n\n'
return web.Response(body=metadata + output + error, content_type="text/event-stream")
status, code = {
"temporary": (503, "server_error"),
"invalid": (400, "invalid_request_error"),
"auth": (403, "invalid_api_key"),
"engine": (502, "engine_down"),
}[ending]
return web.json_response({"error": {"type": code}}, status=status)
async with loopback_engine(tmp_path, service, respond) as (adapter, engine_client):
if ending == "engine":
client_reads = []
def client_or_late_engine_failure():
client_reads.append(True)
if len(client_reads) == 1:
return engine_client
clock.advance(121)
raise EngineUnavailableError("fixture: engine disappeared before admission")
adapter.supervisor.client = client_or_late_engine_failure
async with gateway_client(service, backend, models[backend]) as (_gateway, client, url, headers):
async with client.post(
url, headers=headers, json={"model": "shared-model", "stream": ending == "post_output"},
) as response:
body = await response.read()
if ending == "temporary":
assert response.status == (400 if backend == "codex" else 424)
assert RECOVERY_EXHAUSTED_CODE.encode() in body
else:
assert RECOVERY_EXHAUSTED_CODE.encode() not in body
if ending == "post_output":
assert response.status == 200 and output in body
elif ending == "invalid":
assert response.status == 400
assert adapter._active_transports == 0
assert len(calls) == (1 if ending == "engine" else 2) and clock.delays == [30]
asyncio.run(run())
@pytest.mark.parametrize("backend", BACKENDS)
def test_expiry_after_mixed_real_failures_keeps_the_action_blocker_terminal(tmp_path, backend):
"""MH-RETRY-EXPIRY-002: elapsed time cannot erase an admitted hard failure."""
async def run():
first, second = _source("src_primary01", "Primary"), _source("src_backup001", "Backup")
service, clock, models = configured_service(tmp_path, sources=[first, second], outcomes=[
_outcome(RawOutcomeKind.NETWORK_ERROR, source_id=first.id),
_outcome(RawOutcomeKind.NETWORK_ERROR, source_id=second.id),
_outcome(RawOutcomeKind.HTTP_ERROR, source_id=first.id, status=403),
_outcome(RawOutcomeKind.HTTP_ERROR, source_id=second.id, status=503),
])
original_invoke = service.adapter.invoke
admissions = []
async def late_last_attempt(*args, **kwargs):
on_admitted = kwargs["on_admitted"]
def admitted():
admissions.append((args[0], args[1]))
on_admitted()
kwargs["on_admitted"] = admitted
handle = await original_invoke(*args, **kwargs)
if len(service.adapter.invocations) == 4:
clock.advance(121)
return handle
service.adapter.invoke = late_last_attempt
async with gateway_client(service, backend, models[backend]) as (gateway, client, url, headers):
async with client.post(url, headers=headers, json={"model": "shared-model"}) as response:
assert response.status == 503
payload = await response.json()
assert payload["error"]["code"] == payload["error"]["type"] == "mapping_target_unavailable"
projection = gateway.correlation.terminal_projection("turn_lifecycle", backend=backend)
if backend == "opencode":
assert projection is None
assert service.provenance.get("turn_lifecycle") is None
else:
assert projection is not None and projection.outcome == "exhausted"
assert projection.supply_facts.supply_state == "interrupted"
assert [(blocker.source, blocker.reason) for blocker in projection.supply_facts.blockers] == [
("Primary", "credential_revoked"), ("Backup", "server_error"),
]
gateway.correlation.settle("turn_lifecycle", settled_by=SETTLED_BY_TERMINAL_RESULT)
record = service.provenance.get("turn_lifecycle")
assert record["outcome"] == "exhausted"
assert record["failed_attempts"] == [
{"source_id": source_id, "configured_model_id": "shared-model", "channel": "hub",
"reason": reason, **({"http_status": status} if status else {})}
for source_id, reason, status in [
(first.id, "network", None), (second.id, "network", None),
(first.id, "credential_revoked", 403), (second.id, "server_error", 503),
]
]
assert len(service.adapter.invocations) == 4 and clock.delays == [1]
assert admissions == [(source_id, "shared-model") for source_id in [first.id, second.id] * 2]
assert service.agent_chain(backend, models[backend])["supply_state"] == "interrupted"
asyncio.run(run())
@pytest.mark.parametrize("backend", ["codex", "claude"])
def test_waiters_without_an_attempt_record_blocked_supply_not_a_protocol_error(tmp_path, backend):
"""MH-RETRY-PROVENANCE-001: losing the half-open slot is blocked supply, not a gateway fault."""
async def run():
service, clock, models = configured_service(tmp_path)
service.store = FreshStore(service.store.load())
source = service.store.load().sources[0]
await fail(service, source, "server_error")
clock.advance(31)
claim = service.recovery.claim
peers = []
def a_peer_claims_first(candidate, generation):
# Another waiter wins the eligible slot between inspection and claim.
if not peers:
peers.append(claim(candidate, service._reserve_settlement_generation(candidate.id)))
return claim(candidate, generation)
service.recovery.claim = a_peer_claims_first
async with gateway_client(service, backend, models[backend]) as (gateway, client, url, headers):
async with client.post(url, headers=headers, json={"model": "shared-model"}) as response:
assert response.status == (400 if backend == "codex" else 424)
gateway.correlation.settle("turn_lifecycle", settled_by=SETTLED_BY_TERMINAL_RESULT)
record = service.provenance.get("turn_lifecycle")
assert peers == [True] and service.adapter.invocations == []
assert (record["outcome"], record["model_supply_state"]) == ("no_candidate", "waiting")
assert record["terminal_error"] is None and record["failed_attempts"] == []
asyncio.run(run())
@pytest.mark.parametrize("backend", ["codex", "claude"])
def test_a_waiter_ending_beside_the_slot_owner_leaves_no_turn_projection(tmp_path, backend):
"""MH-RETRY-PROVENANCE-003: the gateway commits a peer-scoped ending only for its own request."""
async def run():
service, clock, models = configured_service(tmp_path, outcomes=[
_outcome(RawOutcomeKind.SUCCESS, source_id="src_recovery01"),
])
source = service.store.load().sources[0]
await fail(service, source, "server_error")
clock.advance(31)
release = asyncio.Event()
original_invoke = service.adapter.invoke
async def owner_waits_upstream(*args, **kwargs):
handle = await original_invoke(*args, **kwargs)
await release.wait()
return handle
service.adapter.invoke = owner_waits_upstream
async with gateway_client(service, backend, models[backend]) as (gateway, client, url, headers):
async def post():
async with client.post(url, headers=headers, json={"model": "shared-model"}) as response:
return response.status
owner = asyncio.create_task(post())
while not service.adapter.invocations:
await asyncio.sleep(0.01)
assert await post() == (400 if backend == "codex" else 424)
release.set()
assert await owner == 200
assert gateway.correlation.terminal_projection("turn_lifecycle", backend=backend) is None
gateway.correlation.settle("turn_lifecycle", settled_by=SETTLED_BY_TERMINAL_RESULT)
record = service.provenance.get("turn_lifecycle")
assert record["outcome"] == "served" and record["model_supply_state"] is None
asyncio.run(run())
KEEPALIVE = b": keepalive\n\n"
def test_claude_stream_rejected_within_its_first_byte_budget_keeps_the_status(tmp_path):
"""MH-RUNTIME-012: a rejection after a few seconds still reaches Claude as HTTP 400.
Claude Code downgrades an unsupported request feature (a beta, or
thinking.display) and retries only on an HTTP 400 naming it; a rejection
inside a committed 200 stream ends the turn instead.
"""
rejection = "thinking.adaptive.display: Input should be 'summarized', 'omitted'"
async def run() -> None:
source = _source("src_rejects1", "Rejects", vendor="anthropic", protocol="anthropic")
service, _clock, models = configured_service(tmp_path, sources=[source])
calls = []
async def respond(request: web.Request) -> web.Response:
calls.append(await request.json())
# Slower than a fast Source, well within Claude's first-byte window.
await asyncio.sleep(2.5)
return web.json_response(
{"type": "error", "error": {"type": "invalid_request_error", "message": rejection}}, status=400,
)
async with loopback_engine(tmp_path, service, respond) as (adapter, _engine_client):
async with gateway_client(service, "claude", models["claude"]) as (_gateway, client, url, headers):
async with client.post(
url, headers=headers, json={"model": "shared-model", "stream": True},
) as response:
body = await response.text()
assert response.status == 400, body
assert rejection in json.loads(body)["error"]["message"]
assert adapter._active_transports == 0
assert len(calls) == 1
asyncio.run(run())
@pytest.mark.parametrize("backend", BACKENDS)
@pytest.mark.parametrize("ending", ["served", "recovered", "exhausted", "disconnect"])
def test_slow_stream_resolution_keeps_claude_connected(monkeypatch, tmp_path, backend, ending):
"""MH-RUNTIME-010: Claude sees headers and keepalives while the Hub resolves.
Claude Code aborts a stream whose headers or bytes stay silent past its
watchdogs, so its streams commit early and carry every ending inside the
stream. Codex and OpenCode key recovery on the HTTP status and keep it.
"""
monkeypatch.setattr(turn_gateway, "_EARLY_STREAM_COMMIT_SECONDS", 0.05)
monkeypatch.setattr(turn_gateway, "_STREAM_KEEPALIVE_SECONDS", 0.02)
exhausted_event = b"event: error\ndata: " + json.dumps(
{"type": "error", "error": {"type": "api_error", "message": RECOVERY_EXHAUSTED_MESSAGE}},
separators=(",", ":"),
).encode() + b"\n\n"
async def run() -> None:
endpoint, protocol = BACKENDS[backend]
source = _source(
"src_patient1", "Patient", vendor="anthropic" if backend == "claude" else "openai", protocol=protocol,
)
service, clock, models = configured_service(tmp_path, sources=[source])
metadata, output, terminal, _buffered = WIRE[protocol]
held, release, upstream_cancelled = asyncio.Event(), asyncio.Event(), asyncio.Event()
calls = []
async def respond(request: web.Request) -> web.Response:
calls.append(await request.json())
if len(calls) == 1:
# An upstream that sends nothing until its first model output.
held.set()
try:
await release.wait()
except asyncio.CancelledError:
upstream_cancelled.set()
raise
elif ending == "exhausted":
clock.advance(121)
if ending == "served" or (ending == "recovered" and len(calls) == 2):
return web.Response(body=metadata + output + terminal, content_type="text/event-stream")
return web.json_response({"error": {"type": "server_error"}}, status=503)
try:
async with loopback_engine(tmp_path, service, respond) as (adapter, _engine_client):
async with gateway_client(service, backend, models[backend]) as (_gateway, client, url, headers):
post = asyncio.create_task(
client.post(url, headers=headers, json={"model": "shared-model", "stream": True}),
)
try:
await asyncio.wait_for(held.wait(), timeout=2)
if backend == "claude":
response = await asyncio.wait_for(post, timeout=2)
assert response.status == 200
assert response.content_type == "text/event-stream"
first = response.content.readexactly(len(KEEPALIVE))
assert await asyncio.wait_for(first, timeout=2) == KEEPALIVE
else:
done, _pending = await asyncio.wait({post}, timeout=0.2)
assert not done, "a status-keyed stream committed before resolution ended"
if ending == "disconnect":
if backend == "claude":
response.close()
else:
post.cancel()
await asyncio.wait_for(upstream_cancelled.wait(), timeout=2)
else:
release.set()
if backend != "claude":
response = await asyncio.wait_for(post, timeout=2)
body = await asyncio.wait_for(response.read(), timeout=2)
while body.startswith(KEEPALIVE):
body = body[len(KEEPALIVE):]
if ending != "exhausted":
assert response.status == 200
assert body == metadata + output + terminal
elif backend == "claude":
assert response.status == 200
assert body == exhausted_event
else:
assert response.status == (400 if backend == "codex" else 424)
assert RECOVERY_EXHAUSTED_CODE.encode() in body
finally:
if not post.done():
post.cancel()
await asyncio.gather(post, return_exceptions=True)
assert adapter._active_transports == 0
finally:
release.set()
assert len(calls) == (1 if ending in {"served", "disconnect"} else 2)
assert clock.delays == ([] if ending in {"served", "disconnect"} else [30])
asyncio.run(run())
def _large_output(protocol: str) -> bytes:
"""One model-output frame carrying 1 MiB of text in the protocol's shape."""
text = "x" * (1 << 20)
if protocol == "anthropic":
payload = {"type": "content_block_delta", "delta": {"type": "text_delta", "text": text}}
return b"event: content_block_delta\ndata: " + json.dumps(payload).encode() + b"\n\n"
payload = {"type": "response.output_text.delta", "delta": text}
return b"event: response.output_text.delta\ndata: " + json.dumps(payload).encode() + b"\n\n"
IDLE_BOUNDED_STALLS = ("before_headers", "before_output", "after_output", "heartbeat", "backpressure")
# Large enough to fill every buffer between the engine and a caller that stops reading.
BACKPRESSURE_FRAMES = 24
@pytest.mark.parametrize(
"backend,vendor,source_protocol,stall",
[
*(
(backend, vendor, BACKENDS[backend][1], stall)
for backend, vendor in (("codex", "openai"), ("claude", "anthropic"))
for stall in IDLE_BOUNDED_STALLS
),
# Healthy streams that send nothing for minutes: xAI sends a whole tool
# call at once, the engine's translators drop the upstream heartbeat,
# and Chat has no measured cadence.
("codex", "xai", "openai_responses", "unbounded"),
("codex", "anthropic", "anthropic", "unbounded"),
("claude", "openai", "openai_responses", "unbounded"),
("opencode", "openai", "openai_chat", "unbounded"),
],
)
def test_a_silent_engine_stream_is_cancelled_and_retried_at_its_idle_bound(
monkeypatch, tmp_path, backend, vendor, source_protocol, stall,
):
"""MH-RUNTIME-013: a stream silent past the idle bound ends there, not at the caller's bound.
The engine sets no timeout once its upstream is connected, so a relay that
stays alive while its own upstream is stuck sends nothing until the caller
gives up. The gateway cancels that engine request at its idle bound: before
model output it retries without the caller seeing anything; after output it
ends the stream inside the protocol. Any byte, such as an Anthropic ping,
keeps a slow stream alive, and a caller that stops reading pauses the bound.
"""
bound = 0.3
monkeypatch.setattr(engine_client_module, "_STREAM_IDLE_TIMEOUT_SECONDS", bound)
async def run() -> None:
_endpoint, protocol = BACKENDS[backend]
source = _source("src_silent01", "Silent", vendor=vendor, protocol=source_protocol)
service, clock, models = configured_service(tmp_path, sources=[source])
metadata, output, terminal, _buffered = WIRE[protocol]
if stall == "backpressure":
output = _large_output(protocol) * BACKPRESSURE_FRAMES
calls: list[dict] = []
stalled_for: list[float] = []
async def respond(request: web.Request) -> web.StreamResponse:
calls.append(await request.json())
stalls = len(calls) == 1 and stall not in {"heartbeat", "backpressure"}
started = time.monotonic()
try:
if stalls and stall == "before_headers":
await asyncio.Event().wait()
response = web.StreamResponse(headers={"Content-Type": "text/event-stream"})
await response.prepare(request)
await response.write(metadata)
if stall == "heartbeat":
for _ in range(4):
await asyncio.sleep(bound / 2)
await response.write(b": keep-alive\n\n")
if stall == "unbounded":
await asyncio.sleep(bound * 3)
if stalls and stall == "before_output":
started = time.monotonic()
await asyncio.Event().wait()
await response.write(output)
if stalls and stall == "after_output":
started = time.monotonic()
await asyncio.Event().wait()
await response.write(terminal)
await response.write_eof()
return response
except asyncio.CancelledError:
stalled_for.append(time.monotonic() - started)
raise
async with loopback_engine(tmp_path, service, respond) as (adapter, _engine_client):
async with gateway_client(service, backend, models[backend]) as (_gateway, client, url, headers):
async def exchange() -> tuple[int, bytes]:
async with client.post(
url, headers=headers, json={"model": "shared-model", "stream": True},
) as response:
if stall == "backpressure":
# The engine keeps sending while the caller reads nothing.
await asyncio.sleep(bound * 3)
return response.status, await response.read()
# Without the idle bound the stalled engine request never ends.
status, body = await asyncio.wait_for(exchange(), timeout=5)
assert status == 200
assert adapter._active_transports == 0
if stall in {"heartbeat", "unbounded", "backpressure"}:
keepalives = b": keep-alive\n\n" * 4 if stall == "heartbeat" else b""
assert body == metadata + keepalives + output + terminal
assert len(calls) == 1 and stalled_for == []
return
# The engine request itself was cancelled, once, at the bound.
assert len(stalled_for) == 1 and bound * 0.9 <= stalled_for[0] < bound + 2
if stall == "after_output":
# Output already reached the caller, so the stream ends in-protocol instead of replaying.
assert len(calls) == 1
assert body.startswith(metadata + output)
ending = ProtocolSSEState(protocol)
ending.observe(body[len(metadata + output):])
assert ending.terminal_outcome == "failed_terminal"
else:
assert len(calls) == 2
assert body == metadata + output + terminal
assert clock.delays == [1]
asyncio.run(run())
@pytest.mark.parametrize(
"origin,vendor,source_protocol,bound",
[
# An unmeasured Source keeps Vibey's caller-owned bound.
("vibey", "xai", "openai_responses", "caller"),
# A measured Anthropic stream keeps its shorter inter-byte bound after
# the first response byte; Vibey's larger bound applies only before it.
("vibey", "anthropic", "anthropic", "idle"),
# A native caller keeps its own bound, and an unmeasured Source stays unbounded here.
("codex", "xai", "openai_responses", None),
],
)
def test_vibey_hands_its_silence_bound_to_the_engine_read(
monkeypatch, tmp_path, origin, vendor, source_protocol, bound,
):
"""MH-RUNTIME-013: before model output only the engine read sees the upstream, so it holds Vibey's bound.
The gateway withholds the response, heartbeats included, until the first
model output. Vibey therefore gives the engine its 600-second caller-owned
pre-response patience; a measured Source resumes its shorter 120-second
inter-byte bound after the first response byte.
"""
idle, caller = 0.3, 0.8
monkeypatch.setattr(engine_client_module, "_STREAM_IDLE_TIMEOUT_SECONDS", idle)
monkeypatch.setattr(engine_adapter_module, "PROVIDER_SILENCE_TIMEOUT_S", caller)
async def run() -> None:
source = _source("src_silent01", "Silent", vendor=vendor, protocol=source_protocol)
service, _clock, _models = configured_service(tmp_path, sources=[source])
metadata, output, terminal, _buffered = WIRE[source_protocol]
stalled_for: list[float] = []
async def respond(request: web.Request) -> web.StreamResponse:
await request.json()
response = web.StreamResponse(headers={"Content-Type": "text/event-stream"})
await response.prepare(request)
await response.write(metadata)
started = time.monotonic()
try:
await asyncio.sleep(caller * 2)
except asyncio.CancelledError:
stalled_for.append(time.monotonic() - started)
raise
await response.write(output)
await response.write(terminal)
await response.write_eof()
return response
async with loopback_engine(tmp_path, service, respond) as (adapter, _engine_client):
handle = await asyncio.wait_for(
adapter.invoke(source.id, "shared-model", {"stream": True}, True, origin), timeout=5,
)
body = b"".join([chunk async for chunk in handle.stream]) if handle.stream is not None else b""
outcome = await asyncio.wait_for(handle.outcome(), timeout=5)
await asyncio.sleep(0.05)
if bound is None:
assert outcome.kind == RawOutcomeKind.SUCCESS
assert body == metadata + output + terminal
assert stalled_for == []
return
assert outcome.kind == RawOutcomeKind.TIMEOUT
assert body == b""
expected = caller if bound == "caller" else idle
assert len(stalled_for) == 1 and expected * 0.9 <= stalled_for[0] < min(expected + 0.4, caller * 2)
asyncio.run(run())
@pytest.mark.parametrize("vendor,protocol", [("anthropic", "anthropic"), ("openai", "openai_responses")])
@pytest.mark.parametrize("origin", ["vibey", "native"])
@pytest.mark.parametrize("phase", ["headers", "first_byte", "split_prelude"])
def test_measured_stream_prelude_uses_one_caller_budget_until_first_byte(
monkeypatch, tmp_path, vendor, protocol, origin, phase,
):
"""MH-RUNTIME-013: slow relay preludes use caller patience, not a fresh budget per phase.
Vibey can wait beyond measured idle for headers or the first body byte;
native callers cannot. Splitting the delay across headers and body must
not reset the budget. The existing idle test covers later byte gaps.
"""
idle, caller = 0.3, 0.8
delay = (idle + caller) / 2
monkeypatch.setattr(engine_client_module, "_STREAM_IDLE_TIMEOUT_SECONDS", idle)
monkeypatch.setattr(engine_adapter_module, "PROVIDER_SILENCE_TIMEOUT_S", caller)
async def run() -> None:
source = _source("src_slowheaders", "Slow prelude", vendor=vendor, protocol=protocol)
service, _clock, _models = configured_service(tmp_path, sources=[source])
metadata, output, terminal, _buffered = WIRE[protocol]
async def respond(request: web.Request) -> web.StreamResponse:
await request.json()
if phase in {"headers", "split_prelude"}:
await asyncio.sleep(delay)
response = web.StreamResponse(headers={"Content-Type": "text/event-stream"})
await response.prepare(request)
if phase in {"first_byte", "split_prelude"}:
await asyncio.sleep(delay)
await response.write(metadata + output + terminal)
await response.write_eof()
return response
caller_origin = origin if origin == "vibey" else "claude" if protocol == "anthropic" else "codex"
async with loopback_engine(tmp_path, service, respond) as (adapter, _engine_client):
handle = await asyncio.wait_for(
adapter.invoke(source.id, "shared-model", {"stream": True}, True, caller_origin),
timeout=3,
)
body = b"".join([chunk async for chunk in handle.stream]) if handle.stream is not None else b""
outcome = await asyncio.wait_for(handle.outcome(), timeout=3)
assert adapter._active_transports == 0
if origin == "vibey" and phase != "split_prelude":
assert outcome.kind == RawOutcomeKind.SUCCESS
assert body == metadata + output + terminal
else:
assert outcome.kind == RawOutcomeKind.TIMEOUT
assert body == b""
asyncio.run(run())