Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
from __future__ import annotations
import asyncio
import inspect
from types import SimpleNamespace
from unittest.mock import AsyncMock, patch
import aiohttp
import pytest
from core.handlers.model_hub.adapter import RawOutcomeKind
from core.handlers.model_hub.turn_gateway import ModelHubTurnGateway
from core.run_settlement import (
SETTLED_BY_BACKEND_REFRESH, SETTLED_BY_NO_TERMINAL_RESULT,
SETTLED_BY_STOPPED, SETTLED_BY_TERMINAL_RESULT,
)
from modules.agents.model_hub import ModelHubRuntimeRouter
from modules.agents.codex.event_handler import CodexEventHandler
from tests.test_model_hub_l3 import (
FakeStreamResponse,
LiveInvokeHandle,
NeverResolvingCloseHandle,
_canonicalize_fixed_test_routes,
_outcome,
_prepared_gateway_request,
_service,
_source,
)
TERMINAL = (
b'event: response.completed\ndata: {"type":"response.completed",'
b'"response":{"status":"completed","output":[],"usage":'
b'{"input_tokens":1,"output_tokens":1}}}\n\n'
)
async def _settle(runtime, turn_id, settled_by=SETTLED_BY_TERMINAL_RESULT):
completion = runtime.settle_turn(
turn_id, settled_by=settled_by, ts="2026-09-06T00:00:00+00:00"
)
if inspect.isawaitable(completion):
await completion
class HeldTerminalHandle(LiveInvokeHandle):
def __init__(self, source_id, chunks=(TERMINAL,)):
super().__init__(
_outcome(RawOutcomeKind.SUCCESS, status=200, source_id=source_id, stream_started=True),
chunks,
)
self.at_end = asyncio.Event()
self.release = asyncio.Event()
async def _iterate(self, chunks):
for chunk in chunks:
yield chunk
self.at_end.set()
await self.release.wait()
@pytest.mark.parametrize("early", [True, False], ids=["terminal-frame", "gateway-finished"])
def test_loopback_first_error_then_success_retained_on_reused_credentials(tmp_path, early):
async def exercise():
source = _source("src_retention01", "Retention")
handle = HeldTerminalHandle(source.id)
service = _service(
tmp_path,
sources=[source],
outcomes=[_outcome(RawOutcomeKind.HTTP_ERROR, status=404, code="model_not_found", source_id=source.id)],
)
model = _canonicalize_fixed_test_routes(service)["codex"]
gateway = ModelHubTurnGateway(service)
runtime = ModelHubRuntimeRouter(service=service, turn_gateway=gateway, overlay_path=tmp_path / "overlay.json")
first = _prepared_gateway_request(
gateway, turn_id="first-error", requested_model=model, source_id=source.id, stream=True
)
completion = None
try:
await gateway._handle_request(first)
await _settle(runtime, "first-error")
assert service.provenance.get("first-error")["terminal_error"]["upstream_error_code"] == "model_not_found"
service.adapter.live_handles.append(handle)
base, token = await gateway.endpoint(
"codex", process_scope="/repo", turn_id="second-success", requested_model_id=model,
resolved_model_id="shared-model", source_id=source.id,
)
assert first.headers["Authorization"] == f"Bearer {token}"
async with aiohttp.ClientSession(trust_env=False) as client:
async with client.post(
f"{base}/v1/responses", headers={"Authorization": f"Bearer {token}"},
json={"model": "shared-model", "input": "synthetic", "stream": True},
) as response:
frame = await asyncio.wait_for(response.content.readuntil(b"\n\n"), 5)
assert b"response.completed" in frame
await asyncio.wait_for(handle.at_end.wait(), 5)
if early:
completion = asyncio.create_task(_settle(runtime, "second-success"))
await asyncio.sleep(0)
handle.release.set()
await asyncio.wait_for(response.read(), 5)
if completion is None:
completion = asyncio.create_task(_settle(runtime, "second-success"))
await asyncio.wait_for(completion, 5)
latest = service.provenance.latest_for_model("codex", model)
assert latest["turn_id"] == "second-success"
assert latest["outcome"] == "served"
assert latest["terminal_error"] is None
assert len(service.adapter.invocations) == 2
finally:
handle.release.set()
if completion is not None:
await asyncio.gather(completion, return_exceptions=True)
await gateway.close()
asyncio.run(exercise())
def test_codex_will_retry_keeps_same_turn_open_until_success_settles(tmp_path):
async def exercise():
source = _source("src_retryturn1", "Retry")
handle = HeldTerminalHandle(source.id)
service = _service(tmp_path, sources=[source], outcomes=[
_outcome(RawOutcomeKind.HTTP_ERROR, status=404, code="model_not_found", source_id=source.id)
])
model = _canonicalize_fixed_test_routes(service)["codex"]
gateway = ModelHubTurnGateway(service)
runtime = ModelHubRuntimeRouter(service=service, turn_gateway=gateway, overlay_path=tmp_path / "overlay.json")
request = _prepared_gateway_request(
gateway, turn_id="retried", requested_model=model, source_id=source.id, stream=True
)
recorder = AsyncMock()
handler = CodexEventHandler(SimpleNamespace(_record_model_hub_native_failure=recorder))
running = completion = None
try:
await gateway._handle_request(request)
await handler.handle_notification(
"error", {"error": {"message": "synthetic retry"}, "willRetry": True, "turnId": "retried"},
SimpleNamespace(context=SimpleNamespace(platform_specific={"turn_token": "retried"})),
)
recorder.assert_not_awaited()
assert service.provenance.get("retried") is None
retry = _prepared_gateway_request(
gateway, turn_id="retried", requested_model=model, source_id=source.id, stream=True
)
assert request.headers == retry.headers
service.adapter.live_handles.append(handle)
with patch("core.handlers.model_hub.turn_gateway.web.StreamResponse", return_value=FakeStreamResponse()):
running = asyncio.create_task(gateway._handle_request(retry))
await asyncio.wait_for(handle.at_end.wait(), 5)
completion = runtime.settle_turn(
"retried", settled_by=SETTLED_BY_TERMINAL_RESULT, ts="2026-09-06T00:00:00Z"
)
assert completion is not None and not completion.done()
handle.release.set()
await asyncio.wait_for(completion, 5)
await running
record = service.provenance.get("retried")
assert record["outcome"] == "served"
assert record["terminal_error"] is None
assert record["served"]["source_id"] == source.id
assert len(service.adapter.invocations) == 2
finally:
handle.release.set()
await asyncio.gather(*(task for task in (running, completion) if task is not None), return_exceptions=True)
await gateway.close()
asyncio.run(exercise())
@pytest.mark.parametrize("malformed", [False, True], ids=["valid", "malformed"])
def test_request_admitted_before_completion_keeps_identity_through_body_parse(tmp_path, malformed):
async def exercise():
source = _source("src_parsedturn", "Delayed body")
service = _service(tmp_path, sources=[source], live_handles=[
LiveInvokeHandle(_outcome(RawOutcomeKind.SUCCESS, status=200, source_id=source.id), (TERMINAL,))
])
model = _canonicalize_fixed_test_routes(service)["codex"]
gateway = ModelHubTurnGateway(service)
runtime = ModelHubRuntimeRouter(service=service, turn_gateway=gateway, overlay_path=tmp_path / "overlay.json")
request = _prepared_gateway_request(gateway, turn_id="parsed", requested_model=model, source_id=source.id, stream=True)
original_json = request.json
parsing = asyncio.Event()
release = asyncio.Event()
async def parse(**_kwargs):
parsing.set()
await release.wait()
return {} if malformed else await original_json()
request.json = parse
completion = None
running = None
try:
with patch("core.handlers.model_hub.turn_gateway.web.StreamResponse", return_value=FakeStreamResponse()):
running = asyncio.create_task(gateway._handle_request(request))
await asyncio.wait_for(parsing.wait(), 5)
completion = asyncio.create_task(_settle(runtime, "parsed"))
await asyncio.sleep(0)
assert not completion.done()
# A later request may still route, but must not claim the closed turn.
token = request.headers["Authorization"].removeprefix("Bearer ")
with gateway.correlation.gateway_terminalizer(backend="codex", token=token) as late:
assert late.turn_id is None
release.set()
await asyncio.wait_for(running, 5)
await asyncio.wait_for(completion, 5)
record = service.provenance.get("parsed")
assert record["outcome"] == ("failed_terminal" if malformed else "served")
if malformed:
assert record["terminal_error"]["reason"] == "invalid_parameter"
assert service.adapter.invocations == []
finally:
release.set()
await asyncio.gather(*(task for task in (running, completion) if task is not None), return_exceptions=True)
await gateway.close()
asyncio.run(exercise())
@pytest.mark.parametrize("exact_request", [False, True], ids=["untracked-only", "also-exact"])
@pytest.mark.parametrize("reason", [SETTLED_BY_TERMINAL_RESULT, SETTLED_BY_STOPPED])
def test_out_of_window_request_releases_finalization_ownership(tmp_path, exact_request, reason):
async def exercise():
source = _source("src_oldroute01", "Old route")
old_handle = HeldTerminalHandle(source.id)
exact_handle = HeldTerminalHandle(source.id)
service = _service(tmp_path, sources=[source], live_handles=[old_handle, exact_handle])
old_model = _canonicalize_fixed_test_routes(service)["codex"]
agent = service.store.load().agents["codex"]
new_model = next(model for model in agent.routes if model != old_model)
agent.routes[new_model] = agent.routes[old_model]
# This checks ownership, not deadline expiry; normal cleanup must finish.
gateway = ModelHubTurnGateway(service)
runtime = ModelHubRuntimeRouter(service=service, turn_gateway=gateway, overlay_path=tmp_path / "overlay.json")
old_request = _prepared_gateway_request(
gateway, turn_id="old-route", requested_model=old_model, source_id=source.id, stream=True
)
await _settle(runtime, "old-route")
new_request = _prepared_gateway_request(
gateway, turn_id="new-route", requested_model=new_model, source_id=source.id, stream=True
)
assert old_request.headers != new_request.headers
old_task = exact_task = completion = None
try:
with patch("core.handlers.model_hub.turn_gateway.web.StreamResponse", side_effect=lambda **_kwargs: FakeStreamResponse()):
old_task = asyncio.create_task(gateway._handle_request(old_request))
await asyncio.wait_for(old_handle.at_end.wait(), 5)
if exact_request:
exact_task = asyncio.create_task(gateway._handle_request(new_request))
await asyncio.wait_for(exact_handle.at_end.wait(), 5)
completion = runtime.settle_turn("new-route", settled_by=reason, ts="2026-09-06T00:00:00+00:00")
if exact_request:
assert completion is not None
assert not completion.done()
if reason == SETTLED_BY_TERMINAL_RESULT:
exact_handle.release.set()
await asyncio.wait_for(asyncio.shield(completion), 5)
await asyncio.gather(exact_task, return_exceptions=True)
record = service.provenance.get("new-route")
assert record["outcome"] == ("served" if reason == SETTLED_BY_TERMINAL_RESULT else "canceled")
assert record["requested_model_id"] == new_model
else:
assert completion is None
record = service.provenance.get("new-route")
if reason == SETTLED_BY_STOPPED:
# Existing no-request Stop history is retained, without
# attributing any of the out-of-window request's facts.
assert record["outcome"] == "canceled"
assert record["canceled_attempt"] is None
assert record["served"] is None
assert record["terminal_error"] is None
assert record["failed_attempts"] == []
else:
assert record is None
assert not old_task.done(), "finalization must neither wait for nor cancel the untracked request"
old_handle.release.set()
await asyncio.wait_for(old_task, 5)
assert service.provenance.get("old-route") is None
assert len(service.adapter.invocations) == (2 if exact_request else 1)
finally:
old_handle.release.set()
exact_handle.release.set()
await asyncio.gather(*(task for task in (old_task, exact_task, completion) if task is not None), return_exceptions=True)
await gateway.close()
asyncio.run(exercise())
def test_body_claim_releases_request_already_snapshotted_by_finalization(tmp_path):
async def exercise():
source = _source("src_bodyclaim01", "Body claim")
handle = HeldTerminalHandle(source.id)
service = _service(tmp_path, sources=[source], live_handles=[handle])
old_model = _canonicalize_fixed_test_routes(service)["codex"]
agent = service.store.load().agents["codex"]
new_model = next(model for model in agent.routes if model != old_model)
gateway = ModelHubTurnGateway(service)
runtime = ModelHubRuntimeRouter(service=service, turn_gateway=gateway, overlay_path=tmp_path / "overlay.json")
request = _prepared_gateway_request(
gateway, turn_id="old-route", requested_model=old_model, source_id=source.id, stream=True
)
await _settle(runtime, "old-route")
_prepared_gateway_request(gateway, turn_id="new-route", requested_model=new_model, source_id=source.id, stream=True)
original_json = request.json
parsing = asyncio.Event()
release = asyncio.Event()
draining = asyncio.Event()
original_drain = gateway._drain_turn_requests
async def parse(**kwargs):
parsing.set()
await release.wait()
return await original_json(**kwargs)
async def drain(*args, **kwargs):
draining.set()
await original_drain(*args, **kwargs)
request.json = parse
running = completion = None
try:
with patch("core.handlers.model_hub.turn_gateway.web.StreamResponse", return_value=FakeStreamResponse()), patch.object(
gateway, "_drain_turn_requests", side_effect=drain
):
running = asyncio.create_task(gateway._handle_request(request))
await asyncio.wait_for(parsing.wait(), 5)
completion = runtime.settle_turn(
"new-route", settled_by=SETTLED_BY_TERMINAL_RESULT, ts="2026-09-06T00:00:00+00:00"
)
await asyncio.wait_for(draining.wait(), 5)
release.set()
await asyncio.wait_for(handle.at_end.wait(), 5)
await asyncio.wait_for(asyncio.shield(completion), 5)
assert not running.done(), "released attribution must wake the existing drain without canceling transport"
assert service.provenance.get("new-route") is None
handle.release.set()
await asyncio.wait_for(running, 5)
assert service.provenance.get("old-route") is None
finally:
release.set()
handle.release.set()
await asyncio.gather(*(task for task in (running, completion) if task is not None), return_exceptions=True)
await gateway.close()
asyncio.run(exercise())
@pytest.mark.parametrize(
"reason,backend_rejection,expected",
[
(SETTLED_BY_STOPPED, False, "canceled"),
(SETTLED_BY_NO_TERMINAL_RESULT, False, "failed_terminal"),
(SETTLED_BY_BACKEND_REFRESH, False, "failed_terminal"),
(SETTLED_BY_TERMINAL_RESULT, True, "failed_terminal"),
],
)
def test_finalization_preserves_stop_and_backend_failure_before_late_success(tmp_path, reason, backend_rejection, expected):
async def exercise():
source = _source("src_stoppedturn", "Late success")
handle = HeldTerminalHandle(source.id)
service = _service(tmp_path, sources=[source], live_handles=[handle])
model = _canonicalize_fixed_test_routes(service)["codex"]
gateway = ModelHubTurnGateway(service)
runtime = ModelHubRuntimeRouter(service=service, turn_gateway=gateway, overlay_path=tmp_path / "overlay.json")
request = _prepared_gateway_request(gateway, turn_id="terminated", requested_model=model, source_id=source.id, stream=True)
running = None
completion = None
try:
with patch("core.handlers.model_hub.turn_gateway.web.StreamResponse", return_value=FakeStreamResponse()):
running = asyncio.create_task(gateway._handle_request(request))
await asyncio.wait_for(handle.at_end.wait(), 5)
if backend_rejection:
gateway.correlation.fail_hub_attempt("terminated")
completion = asyncio.create_task(_settle(runtime, "terminated", reason))
await asyncio.sleep(0)
handle.release.set()
await asyncio.gather(running, return_exceptions=True)
await asyncio.wait_for(completion, 5)
record = service.provenance.get("terminated")
assert record["outcome"] == expected
assert record["served"] is None
if backend_rejection:
assert record["terminal_error"]["reason"] == "backend_error"
elif reason != SETTLED_BY_STOPPED:
assert record["terminal_error"]["reason"] == "stream_interrupted"
else:
assert record["canceled_attempt"]["source_id"] == source.id
finally:
handle.release.set()
await asyncio.gather(*(task for task in (running, completion) if task is not None), return_exceptions=True)
await gateway.close()
asyncio.run(exercise())
def test_finalization_deadline_cancels_owned_request_without_inventing_success(tmp_path):
async def exercise():
source = _source("src_deadline01", "Deadline")
service = _service(tmp_path, sources=[source])
model = _canonicalize_fixed_test_routes(service)["codex"]
gateway = ModelHubTurnGateway(service, transport_timeout=0.02)
runtime = ModelHubRuntimeRouter(service=service, turn_gateway=gateway, overlay_path=tmp_path / "overlay.json")
request = _prepared_gateway_request(gateway, turn_id="deadline", requested_model=model, source_id=source.id, stream=True)
entered = asyncio.Event()
async def parse(**_kwargs):
entered.set()
await asyncio.Event().wait()
request.json = parse
running = asyncio.create_task(gateway._handle_request(request))
try:
await asyncio.wait_for(entered.wait(), 5)
await asyncio.wait_for(_settle(runtime, "deadline"), 1)
await asyncio.gather(running, return_exceptions=True)
record = service.provenance.get("deadline")
assert record is None or record["served"] is None
assert service.adapter.invocations == []
assert not gateway._turn_requests
assert not gateway._turn_completions
finally:
running.cancel()
await asyncio.gather(running, return_exceptions=True)
await gateway.close()
asyncio.run(exercise())
def test_completion_is_owned_once_through_repeated_waiter_cancellation(tmp_path):
async def exercise():
source = _source("src_cancelwait", "Canceled waiter")
handle = HeldTerminalHandle(source.id)
service = _service(tmp_path, sources=[source], live_handles=[handle])
model = _canonicalize_fixed_test_routes(service)["codex"]
gateway = ModelHubTurnGateway(service)
runtime = ModelHubRuntimeRouter(service=service, turn_gateway=gateway, overlay_path=tmp_path / "overlay.json")
request = _prepared_gateway_request(gateway, turn_id="owned", requested_model=model, source_id=source.id, stream=True)
puts = []
original_put = service.provenance.put
def put(record):
puts.append(record)
original_put(record)
service.provenance.put = put
running = None
waiter = None
try:
with patch("core.handlers.model_hub.turn_gateway.web.StreamResponse", return_value=FakeStreamResponse()):
running = asyncio.create_task(gateway._handle_request(request))
await asyncio.wait_for(handle.at_end.wait(), 5)
completion = runtime.settle_turn("owned", settled_by=SETTLED_BY_TERMINAL_RESULT, ts="2026-09-06T00:00:00Z")
assert completion is runtime.settle_turn("owned", settled_by=SETTLED_BY_STOPPED, ts="later")
async def wait():
await asyncio.shield(completion)
for _ in range(2):
waiter = asyncio.create_task(wait())
await asyncio.sleep(0)
waiter.cancel()
await asyncio.gather(waiter, return_exceptions=True)
assert not completion.done()
handle.release.set()
await asyncio.wait_for(completion, 5)
await running
await _settle(runtime, "owned")
assert len(puts) == 1
assert puts[0]["outcome"] == "served"
finally:
handle.release.set()
await asyncio.gather(*(task for task in (running, waiter) if task is not None), return_exceptions=True)
await gateway.close()
asyncio.run(exercise())
def test_finalization_waits_for_every_request_after_its_attempt_is_finished(tmp_path):
async def exercise():
source = _source("src_retention02", "Concurrent requests")
service = _service(tmp_path, sources=[source], live_handles=[
LiveInvokeHandle(_outcome(RawOutcomeKind.SUCCESS, status=200, source_id=source.id), (TERMINAL,))
for _ in range(2)
])
model = _canonicalize_fixed_test_routes(service)["codex"]
gateway = ModelHubTurnGateway(service)
runtime = ModelHubRuntimeRouter(service=service, turn_gateway=gateway, overlay_path=tmp_path / "overlay.json")
at_eof = [asyncio.Event(), asyncio.Event()]
release = [asyncio.Event(), asyncio.Event()]
class HeldEOF(FakeStreamResponse):
def __init__(self, index):
super().__init__()
self.index = index
async def write_eof(self):
at_eof[self.index].set()
await release[self.index].wait()
requests = []
completion = None
try:
with patch("core.handlers.model_hub.turn_gateway.web.StreamResponse", side_effect=[HeldEOF(0), HeldEOF(1)]):
for _ in range(2):
request = _prepared_gateway_request(
gateway, turn_id="concurrent", requested_model=model, source_id=source.id, stream=True
)
requests.append(asyncio.create_task(gateway._handle_request(request)))
await asyncio.wait_for(asyncio.gather(*(event.wait() for event in at_eof)), 5)
completion = asyncio.create_task(_settle(runtime, "concurrent"))
await asyncio.sleep(0)
assert not completion.done()
release[0].set()
await requests[0]
assert not completion.done()
release[1].set()
await asyncio.gather(*requests)
await asyncio.wait_for(completion, 5)
assert service.provenance.get("concurrent")["outcome"] == "served"
finally:
for event in release:
event.set()
await asyncio.gather(*requests, return_exceptions=True)
if completion is not None:
await asyncio.gather(completion, return_exceptions=True)
await gateway.close()
asyncio.run(exercise())
@pytest.mark.parametrize("no_candidate", [False, True], ids=["malformed", "no-candidate"])
def test_noninvoking_request_must_exit_before_finalization(tmp_path, no_candidate):
from config.v2_config import ModelHubRouteConfig
async def exercise():
source = _source("src_noinvoke01", "Non-invoking request")
service = _service(tmp_path, sources=[source])
model = _canonicalize_fixed_test_routes(service)["codex"]
gateway = ModelHubTurnGateway(service)
runtime = ModelHubRuntimeRouter(service=service, turn_gateway=gateway, overlay_path=tmp_path / "overlay.json")
request = _prepared_gateway_request(gateway, turn_id="noninvoking", requested_model=model, source_id=source.id, stream=True)
if no_candidate:
service.store.config.agents["codex"].routes[model] = ModelHubRouteConfig()
service.store.config.agents["codex"].sources.order = []
else:
request.json.return_value = {}
response_ready = asyncio.Event()
release = asyncio.Event()
run_request = gateway._run_request_turn
async def exiting(*args, **kwargs):
response = await run_request(*args, **kwargs)
response_ready.set()
await release.wait()
return response
gateway._run_request_turn = exiting
running = asyncio.create_task(gateway._handle_request(request))
completion = None
try:
await asyncio.wait_for(response_ready.wait(), 5)
completion = asyncio.create_task(_settle(runtime, "noninvoking"))
await asyncio.sleep(0)
assert not completion.done()
assert service.provenance.get("noninvoking") is None
release.set()
await running
await asyncio.wait_for(completion, 5)
record = service.provenance.get("noninvoking")
assert record["outcome"] == ("no_candidate" if no_candidate else "failed_terminal")
assert service.adapter.invocations == []
finally:
release.set()
await asyncio.gather(*(task for task in (running, completion) if task is not None), return_exceptions=True)
await gateway.close()
asyncio.run(exercise())
@pytest.mark.parametrize("ending", ["stop", "retire", "close"])
def test_committed_gateway_facts_survive_completion_cleanup(tmp_path, ending):
async def exercise():
source = _source("src_committed1", "Committed attempt")
service = _service(tmp_path, sources=[source], live_handles=[
LiveInvokeHandle(_outcome(RawOutcomeKind.SUCCESS, status=200, source_id=source.id), (TERMINAL,))
])
model = _canonicalize_fixed_test_routes(service)["codex"]
gateway = ModelHubTurnGateway(service)
runtime = ModelHubRuntimeRouter(service=service, turn_gateway=gateway, overlay_path=tmp_path / "overlay.json")
request = _prepared_gateway_request(gateway, turn_id="committed", requested_model=model, source_id=source.id, stream=True)
at_eof = asyncio.Event()
release = asyncio.Event()
class HeldEOF(FakeStreamResponse):
async def write_eof(self):
at_eof.set()
await release.wait()
running = None
completion = None
try:
with patch("core.handlers.model_hub.turn_gateway.web.StreamResponse", return_value=HeldEOF()):
running = asyncio.create_task(gateway._handle_request(request))
await asyncio.wait_for(at_eof.wait(), 5)
completion = runtime.settle_turn(
"committed", settled_by=SETTLED_BY_STOPPED if ending == "stop" else SETTLED_BY_TERMINAL_RESULT,
ts="2026-09-06T00:00:00Z",
)
assert completion is not None
if ending == "retire":
gateway.correlation.retire_scope("codex", "/repo")
elif ending == "close":
await gateway.close()
release.set()
await asyncio.gather(running, return_exceptions=True)
await asyncio.wait_for(completion, 5)
assert service.provenance.get("committed")["outcome"] == "served"
assert not gateway._turn_requests
assert not gateway._turn_completions
finally:
release.set()
await asyncio.gather(*(task for task in (running, completion) if task is not None), return_exceptions=True)
await gateway.close()
asyncio.run(exercise())
@pytest.mark.parametrize("ending", ["retire", "overlap", "untracked"])
def test_ambiguous_owned_request_is_never_promoted_to_attributed_success(tmp_path, ending):
async def exercise():
source = _source("src_ambiguous1", "Ambiguous turn")
handle = HeldTerminalHandle(source.id)
service = _service(tmp_path, sources=[source], live_handles=[handle])
model = _canonicalize_fixed_test_routes(service)["codex"]
gateway = ModelHubTurnGateway(service)
runtime = ModelHubRuntimeRouter(service=service, turn_gateway=gateway, overlay_path=tmp_path / "overlay.json")
request = _prepared_gateway_request(gateway, turn_id="ambiguous", requested_model=model, source_id=source.id, stream=True)
running = None
completion = None
try:
with patch("core.handlers.model_hub.turn_gateway.web.StreamResponse", return_value=FakeStreamResponse()):
running = asyncio.create_task(gateway._handle_request(request))
await asyncio.wait_for(handle.at_end.wait(), 5)
completion = runtime.settle_turn("ambiguous", settled_by=SETTLED_BY_TERMINAL_RESULT, ts="2026-09-06T00:00:00Z")
if ending == "retire":
gateway.correlation.retire_scope("codex", "/repo")
else:
gateway.correlation.credentials("codex", "/repo", "peer" if ending == "overlap" else None)
handle.release.set()
await running
await asyncio.wait_for(completion, 5)
assert service.provenance.get("ambiguous") is None
finally:
handle.release.set()
await asyncio.gather(*(task for task in (running, completion) if task is not None), return_exceptions=True)
await gateway.close()
asyncio.run(exercise())
@pytest.mark.parametrize("failed", [False, True])
def test_native_only_and_untracked_turns_finalize_without_async_work(tmp_path, failed):
source = _source("src_nativeonly", "Native", channel="native_cli")
service = _service(tmp_path, sources=[source])
gateway = ModelHubTurnGateway(service)
runtime = ModelHubRuntimeRouter(service=service, turn_gateway=gateway, overlay_path=tmp_path / "overlay.json")
gateway.correlation.begin_native_attempt(
backend="codex", process_scope="/repo", turn_id="native", requested_model_id="shared-model",
source_id=source.id, resolved_model_id="shared-model", via_mapping=False,
)
if failed:
gateway.correlation.fail_native_attempt("native", reason="rate_limited")
assert runtime.settle_turn("native", settled_by=SETTLED_BY_TERMINAL_RESULT, ts="2026-09-06T00:00:00Z") is None
assert service.provenance.get("native")["outcome"] == ("exhausted" if failed else "served")
gateway.correlation.credentials("opencode", "shared", "opencode")
assert runtime.settle_turn("opencode", settled_by=SETTLED_BY_TERMINAL_RESULT, ts="2026-09-06T00:00:00Z") is None
assert service.provenance.get("opencode") is None
assert not gateway._turn_completions
def test_fallback_requests_keep_owned_identity_until_the_final_attempt_finishes(tmp_path):
async def exercise():
first = _source("src_fallback01", "First")
second = _source("src_fallback02", "Second")
handle = HeldTerminalHandle(second.id)
service = _service(tmp_path, sources=[first, second], outcomes=[
_outcome(RawOutcomeKind.HTTP_ERROR, status=429, source_id=first.id)
])
model = _canonicalize_fixed_test_routes(service)["codex"]
gateway = ModelHubTurnGateway(service)
runtime = ModelHubRuntimeRouter(service=service, turn_gateway=gateway, overlay_path=tmp_path / "overlay.json")
original_invoke = service.adapter.invoke
async def invoke(source_id, *args, **kwargs):
if source_id == second.id:
service.adapter.live_handles.append(handle)
return await original_invoke(source_id, *args, **kwargs)
service.adapter.invoke = invoke
request = _prepared_gateway_request(gateway, turn_id="fallback", requested_model=model, source_id=first.id, stream=True)
running = None
completion = None
try:
with patch("core.handlers.model_hub.turn_gateway.web.StreamResponse", return_value=FakeStreamResponse()):
running = asyncio.create_task(gateway._handle_request(request))
await asyncio.wait_for(handle.at_end.wait(), 5)
# Same-turn retry preparation remains idempotent, not overlapping.
assert gateway.correlation.credentials("codex", "/repo", "fallback")
completion = runtime.settle_turn("fallback", settled_by=SETTLED_BY_TERMINAL_RESULT, ts="2026-09-06T00:00:00Z")
assert completion is not None
handle.release.set()
await running
await asyncio.wait_for(completion, 5)
record = service.provenance.get("fallback")
assert record["outcome"] == "served"
assert record["served"]["source_id"] == second.id
assert [attempt["source_id"] for attempt in record["failed_attempts"]] == [first.id]
assert len(service.adapter.invocations) == 2
finally:
handle.release.set()
await asyncio.gather(*(task for task in (running, completion) if task is not None), return_exceptions=True)
await gateway.close()
asyncio.run(exercise())
def test_finalization_abandons_stuck_teardown_without_late_history_mutation(tmp_path):
async def exercise():
source = _source("src_stuckdrain", "Stuck teardown")
handle = NeverResolvingCloseHandle(_outcome(RawOutcomeKind.SUCCESS, source_id=source.id))
service = _service(tmp_path, sources=[source], live_handles=[handle])
model = _canonicalize_fixed_test_routes(service)["codex"]
gateway = ModelHubTurnGateway(service, transport_timeout=0.02)
runtime = ModelHubRuntimeRouter(service=service, turn_gateway=gateway, overlay_path=tmp_path / "overlay.json")
request = _prepared_gateway_request(gateway, turn_id="stuck", requested_model=model, source_id=source.id, stream=True)
running = None
try:
with patch("core.handlers.model_hub.turn_gateway.web.StreamResponse", return_value=FakeStreamResponse()):
running = asyncio.create_task(gateway._handle_request(request))
await asyncio.wait_for(handle.started.wait(), 5)
await asyncio.wait_for(_settle(runtime, "stuck"), 1)
record = service.provenance.get("stuck")
assert record["outcome"] == "failed_terminal"
assert record["terminal_error"]["reason"] == "engine_down"
assert gateway.resource_leak_records
assert not gateway._turn_requests
assert not gateway._turn_completions
handle.release_close.set()
await asyncio.wait_for(handle.closed.wait(), 1)
await asyncio.gather(running, return_exceptions=True)
assert service.provenance.get("stuck") == record
finally:
handle.release_close.set()
if running is not None:
await asyncio.gather(running, return_exceptions=True)
await gateway.close()
asyncio.run(exercise())
def test_buffered_completion_uses_the_existing_synchronous_settlement_path(tmp_path):
async def exercise():
source = _source("src_buffered01", "Buffered")
service = _service(tmp_path, sources=[source], live_handles=[
LiveInvokeHandle(_outcome(RawOutcomeKind.SUCCESS, status=200, source_id=source.id), (b'{"output":[]}',))
])
model = _canonicalize_fixed_test_routes(service)["codex"]
gateway = ModelHubTurnGateway(service)
runtime = ModelHubRuntimeRouter(service=service, turn_gateway=gateway, overlay_path=tmp_path / "overlay.json")
request = _prepared_gateway_request(gateway, turn_id="buffered", requested_model=model, source_id=source.id, stream=False)
try:
response = await gateway._handle_request(request)
assert response.status == 200
assert runtime.settle_turn("buffered", settled_by=SETTLED_BY_TERMINAL_RESULT, ts="2026-09-06T00:00:00Z") is None
assert service.provenance.get("buffered")["outcome"] == "served"
finally:
await gateway.close()
asyncio.run(exercise())