Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
from __future__ import annotations
import ast
import asyncio
import copy
import inspect
import json
import re
import socket
import tempfile
import threading
from collections import deque
from concurrent.futures import ThreadPoolExecutor
from dataclasses import replace
from contextlib import contextmanager
from datetime import datetime, timedelta, timezone
from pathlib import Path
from types import SimpleNamespace
from typing import cast
from unittest.mock import AsyncMock, Mock, patch
import aiohttp
import jwt
import pytest
import yaml
from aiohttp import web
from jsonschema import Draft7Validator, FormatChecker
from sqlalchemy import create_engine, delete, select
from config.v2_config import (
ModelHubAgentSourcesConfig,
ModelHubAgentSupplyConfig,
ModelHubBackendModelConfig,
ModelHubConfig,
ModelHubModelConfig,
ModelHubRouteConfig,
ModelHubRouteHopConfig,
ModelHubSourceConfig,
ModelHubSourceStateConfig,
model_hub_fixed_menu_ids,
)
from core.handlers.model_hub.adapter import (
DiscoveredModel,
EngineEnsureResult,
EngineHealth,
EngineStatus,
RawCallOutcome,
RawOutcomeKind,
MODEL_PROTOCOLS,
SourceBinding,
)
from core.handlers.model_hub.classification import ResolutionDecision, classify_outcome
from core.handlers.model_hub.events import (
BoundedEventLog,
EVENT_REASON_AUTHORITY,
EventKind,
EventReason,
build_resolution_event,
event_reason_label,
)
from core.handlers.model_hub.provenance import (
BoundedProvenanceStore,
ENGINE_DOWN_TURN_OUTCOME,
ExactHopBlocker,
TURN_OUTCOME_RENDERING_AUTHORITY,
TurnOutcomeProductionError,
TurnCorrelationRegistry,
exact_hop_blockers,
produce_turn_outcome,
project_turn_outcome_copy,
render_turn_outcome_copy,
)
from core.handlers.model_hub.request import ModelHubRequest
from core.handlers.model_hub.revocations import CredentialRevocationJournal
from core.handlers.model_hub.service import (
ModelHubError,
ModelHubService,
ResolvedInvocation,
project_opencode_public_model,
seeded_source_name,
)
from core.handlers.model_hub.turn_gateway import (
ModelHubTurnGateway,
_RenderedTurnOutcome,
render_protocol_terminal_event,
)
from core.handlers.model_hub.stream_wire import ProtocolSSEState, ProtocolUsageReport
from core.handlers.model_hub.usage import BoundedUsageLedger, UsageWriter, _ledger_executor
from core.run_settlement import (
SETTLED_BY_BACKEND_DISABLED,
SETTLED_BY_NO_TERMINAL_RESULT,
SETTLED_BY_STOPPED,
SETTLED_BY_TERMINAL_RESULT,
)
from core.services.dispatch import TurnDispatchOutcome
from core.session_turns import SessionTurnManager
from modules.agents.model_hub import (
ModelHubLaunch,
ModelHubRuntimeRouter,
_localized_launch_error,
bind_launch,
bind_turn_mode,
opencode_model_catalog_for_overlay,
)
from storage.models import agent_sessions, messages, metadata
from vibe.i18n import t as i18n_t
from vibe.model_hub_runtime.adapter import (
_AuthenticationEvidence,
_AuthRecord,
CLIProxyEngineAdapter,
hub_subscription_serving_protocol,
_HUB_SUBSCRIPTION_PROTOCOLS,
_OAUTH_ENDPOINTS,
_parse_protocol_authenticated_evidence,
_probe_oauth_protocol_response,
_probe_protocol_response,
_PROTOCOL_OBSERVATION_TAXONOMY,
_ProtocolEvidence,
_ProtocolObservationShape,
_ProtocolProof,
)
from vibe.model_hub_runtime.api_key_vendors import (
api_key_vendor_catalog,
catalog_api_key_vendor_label,
)
from vibe.model_hub_runtime.client import EngineClientError, probe_models
from vibe.model_hub_runtime.state import EngineStateStore
CONTRACTS = Path(__file__).parents[1] / "docs" / "plans" / "model-hub-contracts"
MODEL_HUB_FIXTURES = Path(__file__).parent / "fixtures" / "model_hub"
TERMINAL_SETTLEMENT_BOUNDARIES = json.loads(
(MODEL_HUB_FIXTURES / "terminal_settlement_boundaries.json").read_text(encoding="utf-8")
)["cases"]
RELEASED_V5_PERMISSION_DENIED = json.loads(
(MODEL_HUB_FIXTURES / "released_v5_permission_denied.json").read_text(encoding="utf-8")
)
E64_SETTLEMENT_BOUNDARIES = json.loads(
(MODEL_HUB_FIXTURES / "e64_settlement_boundaries.json").read_text(encoding="utf-8")
)
NOW = datetime(2026, 7, 29, 12, 0, tzinfo=timezone.utc)
# A pin on a model-free Auto probe's interface; Gemini's native pin has only its model-list witness
# (covered by test_model_hub_google and the AUTH-SETUP-112 scenario).
CATALOG_API_KEY_VENDOR_PROTOCOL_CASES = tuple(
pytest.param(entry.id, entry.protocol, id=entry.id)
for entry in api_key_vendor_catalog()
if entry.protocol in MODEL_PROTOCOLS and entry.protocol != "google"
)
def _assert_valid(schema_name: str, payload: dict) -> None:
schema = json.loads((CONTRACTS / schema_name).read_text(encoding="utf-8"))
errors = sorted(
Draft7Validator(
schema,
format_checker=FormatChecker(),
).iter_errors(payload),
key=lambda error: list(error.path),
)
assert not errors, [error.message for error in errors]
def _turn_outcome_matrix() -> dict[str, str]:
lines = (
Path(__file__).parents[1] / "docs/plans/model-hub.md"
).read_text(encoding="utf-8").splitlines()
heading = next(
index
for index, line in enumerate(lines)
if line.startswith("**Turn-outcome copy matrix")
)
table = next(
index
for index in range(heading + 1, len(lines))
if lines[index].startswith("| Decision |")
)
rows: dict[str, str] = {}
for line in lines[table + 2 :]:
if not line.startswith("|"):
break
cells = [cell.strip() for cell in line.strip("|").split("|")]
rows[cells[0].strip("`")] = cells[1].strip("`")
return rows
def test_turn_outcome_rendering_authority_covers_matrix_and_locales() -> None:
assert _turn_outcome_matrix() == {
decision: rule.outcome
for decision, rule in TURN_OUTCOME_RENDERING_AUTHORITY.items()
}
assert len(
{
(rule.outcome, rule.discriminator)
for rule in TURN_OUTCOME_RENDERING_AUTHORITY.values()
}
) == len(TURN_OUTCOME_RENDERING_AUTHORITY)
projected_keys = {
key
for rule in TURN_OUTCOME_RENDERING_AUTHORITY.values()
for key in (
*(key for _variant, key in rule.copy_keys),
rule.upstream_failure_key,
rule.upstream_failure_without_status_key,
rule.connection_failure_key,
)
if key is not None
}
locale_launch_keys = None
for locale in ("en", "zh"):
payload = json.loads(
(Path(__file__).parents[1] / f"vibe/i18n/{locale}.json").read_text(
encoding="utf-8"
)
)
launch_keys = {
f"modelHub.launch.{key}" for key in payload["modelHub"]["launch"]
}
locale_launch_keys = locale_launch_keys or launch_keys
assert launch_keys == locale_launch_keys
assert launch_keys == {
key for key in projected_keys if key.startswith("modelHub.launch.")
}
assert all(i18n_t(key, locale) != key for key in projected_keys)
@pytest.mark.parametrize(
("protocol", "expected_shape"),
[
("anthropic", "anthropic"),
("openai_responses", "responses"),
("openai_chat", "chat"),
],
)
def test_terminal_event_renderer_uses_native_protocol_shape(
protocol: str,
expected_shape: str,
) -> None:
event = render_protocol_terminal_event(
protocol,
"modelHub.launch.retry",
"Retry directly.",
next_sequence_number=0,
)
assert "model_hub_terminal" not in str(event)
if expected_shape == "anthropic":
assert event == {
"type": "error",
"error": {"type": "api_error", "message": "Retry directly."},
}
assert isinstance(event["type"], str)
assert isinstance(event["error"], dict)
assert isinstance(event["error"]["type"], str)
assert isinstance(event["error"]["message"], str)
elif expected_shape == "responses":
assert event == {
"type": "error",
"code": "modelHub.launch.retry",
"message": "Retry directly.",
"param": None,
"sequence_number": 0,
}
assert isinstance(event["type"], str)
assert isinstance(event["code"], str)
assert isinstance(event["message"], str)
assert event["param"] is None or isinstance(event["param"], str)
assert isinstance(event["sequence_number"], int)
else:
assert event == {
"object": "chat.completion.chunk",
"type": "error",
"error": {
"type": "server_error",
"code": "modelHub.launch.retry",
"message": "Retry directly.",
},
"choices": [],
}
assert isinstance(event["object"], str)
assert isinstance(event["type"], str)
assert isinstance(event["error"], dict)
assert isinstance(event["error"]["type"], str)
assert isinstance(event["error"]["code"], str)
assert isinstance(event["error"]["message"], str)
assert isinstance(event["choices"], list)
def test_responses_terminal_event_discards_partial_sequence_before_injection() -> None:
wire = ProtocolSSEState("openai_responses")
wire.observe(b'data: {"type":"response.output_text.delta","sequence_number":7}\n\n')
wire.observe(b'data: {"type":"response.output_text.delta","sequence_number":8}')
assert wire.next_sequence_number == 8
assert wire.invalidate_partial_frame() == b"\ndata: {}\n\n"
assert wire.next_sequence_number == 8
event = render_protocol_terminal_event(
"openai_responses",
"modelHub.launch.retry",
"Retry directly.",
next_sequence_number=wire.next_sequence_number,
)
assert event["sequence_number"] == 8
def test_responses_terminal_event_continues_sequence_on_cr_only_frames() -> None:
wire = ProtocolSSEState("openai_responses")
wire.observe(
b'data: {"type":"response.output_text.delta","sequence_number":7}\r\r'
b'data: {"type":"response.output_text.delta","sequence_number":8}\r\r'
)
assert wire.invalidate_partial_frame() == b""
assert wire.next_sequence_number == 9
event = render_protocol_terminal_event(
"openai_responses",
"modelHub.launch.retry",
"Retry directly.",
next_sequence_number=wire.next_sequence_number,
)
assert event["sequence_number"] == 9
def test_gateway_handle_termination_has_one_settlement_owner() -> None:
path = Path(__file__).parents[1] / "core/handlers/model_hub/turn_gateway.py"
tree = ast.parse(path.read_text(encoding="utf-8"))
owner = next(
node
for node in ast.walk(tree)
if isinstance(node, ast.AsyncFunctionDef)
and node.name == "_settle_consumed_handle"
)
functions = {
node.name: node
for node in ast.walk(tree)
if isinstance(node, ast.AsyncFunctionDef)
}
def is_call(node: ast.AST, name: str) -> bool:
return (
isinstance(node, ast.Call)
and isinstance(node.func, ast.Attribute)
and node.func.attr == name
)
all_outcome_calls = {
node.lineno for node in ast.walk(tree) if is_call(node, "outcome")
}
owner_outcome_calls = {
node.lineno for node in ast.walk(owner) if is_call(node, "outcome")
}
all_close_calls = {
node.lineno for node in ast.walk(tree) if is_call(node, "close_stream")
}
owner_close_calls = {
node.lineno for node in ast.walk(owner) if is_call(node, "close_stream")
}
all_settlement_calls = {
node.lineno
for node in ast.walk(tree)
if is_call(node, "settle_handle_outcome")
}
owner_settlement_calls = {
node.lineno
for node in ast.walk(owner)
if is_call(node, "settle_handle_outcome")
}
settlement_nodes = [
node
for node in ast.walk(owner)
if is_call(node, "settle_handle_outcome")
]
assert owner_outcome_calls
assert owner_close_calls
assert owner_settlement_calls
assert all_outcome_calls == owner_outcome_calls
assert all_close_calls == owner_close_calls
assert max(owner_close_calls) < min(owner_outcome_calls)
assert all_settlement_calls == owner_settlement_calls
assert all(
any(keyword.arg == "termination_origin" for keyword in node.keywords)
for node in settlement_nodes
)
assert all(
any(keyword.arg == "record_attempt" for keyword in node.keywords)
for node in settlement_nodes
)
cancel_handlers = [
(function.name, handler)
for function in functions.values()
for node in ast.walk(function)
if isinstance(node, ast.Try)
for handler in node.handlers
if isinstance(handler.type, ast.Attribute)
and isinstance(handler.type.value, ast.Name)
and handler.type.value.id == "asyncio"
and handler.type.attr == "CancelledError"
]
assert [name for name, _handler in cancel_handlers] == ["_handle_request"]
request_runner = functions["_run_request_turn"]
registered_closers = [
node
for node in ast.walk(request_runner)
if is_call(node, "push_async_callback")
]
assert len(registered_closers) == 1
assert isinstance(registered_closers[0].args[0], ast.Attribute)
assert registered_closers[0].args[0].attr == "close_stream"
def _terminal_resolution_facts(
config: ModelHubConfig,
*,
model: str = "shared-model",
supply_status: str,
structural_reason: str | None = None,
blocker_reason: str | None = None,
next_hop: tuple[str, str] | None = None,
) -> SimpleNamespace:
source = config.sources[0] if config.sources else None
inspections = (
(
SimpleNamespace(
runnable=False,
source=source,
source_id=(source.id if source is not None else None),
model_id=model,
reason=blocker_reason,
temporary_blocker=(source is not None and source.state.status == "cooldown"),
retry_at=source.state.retry_at if source is not None else None,
backoff=False,
recovery=None,
cooldown=(source is not None and source.state.status == "cooldown"),
cooldown_reason=None,
),
)
if blocker_reason is not None or (source is not None and source.state.status == "cooldown")
else ()
)
candidates = (
(
SimpleNamespace(
source_id=next_hop[0],
model_id=next_hop[1],
),
)
if next_hop is not None
else ()
)
return SimpleNamespace(
backend="claude",
requested_model=model,
target_model=model,
matching_sources=tuple(config.sources),
inspected_hops=inspections,
candidate_hops=candidates,
supply_status=supply_status,
structural_blocker_reason=structural_reason,
)
@pytest.mark.parametrize(
("decision", "variant", "expected_key"),
tuple(
(decision, variant, key)
for decision, rule in TURN_OUTCOME_RENDERING_AUTHORITY.items()
for variant, key in rule.copy_keys
),
)
def test_every_turn_outcome_matrix_variant_projects_or_stays_silent(
decision: str,
variant: str,
expected_key: str | None,
) -> None:
producer_kwargs = {}
if variant in {"waiting", "interrupted"}:
if decision == "turn.no_candidate.unconfigured":
config = _config([])
else:
source = _source(
"src_matrix001",
"Matrix source",
status=("cooldown" if variant == "waiting" else "needs_action"),
retry_at=(
(NOW + timedelta(minutes=5)).isoformat()
if variant == "waiting"
else None
),
)
if variant == "interrupted":
source.state.detail_key = (
"models.source.needs_action.credential_revoked"
)
config = _config([source])
resolution = _terminal_resolution_facts(
config,
supply_status=variant,
structural_reason=(
"route_unconfigured"
if decision == "turn.no_candidate.unconfigured"
else None
),
blocker_reason=(
"credential_revoked" if variant == "interrupted" else None
),
)
producer_kwargs = {"config": config, "resolution": resolution}
if decision == "turn.streamed_fallback":
producer_kwargs["attempted_hop"] = (
"src_attempted01",
"shared-model",
)
producer_kwargs["source_transition_persisted"] = True
elif variant == "transition_unpersisted":
producer_kwargs = {"source_transition_persisted": False}
elif variant == "upstream_detail":
producer_kwargs = {"upstream_detail": "Upstream rejected this request."}
elif variant == "waiting_without_retry":
source = _source("src_matrix_ready", "Recovered source")
config = _config([source])
resolution = _terminal_resolution_facts(
config,
supply_status="degraded",
next_hop=(source.id, "shared-model"),
)
producer_kwargs = {"config": config, "resolution": resolution}
elif variant == "next_current":
source = _source("src_matrix002", "Next source")
config = _config([source])
resolution = _terminal_resolution_facts(
config,
supply_status="degraded",
next_hop=(source.id, "shared-model"),
)
producer_kwargs = {
"config": config,
"resolution": resolution,
"attempted_hop": ("src_attempted02", "shared-model"),
"source_transition_persisted": True,
}
projection = produce_turn_outcome(
decision,
stream_started=variant == "stream_started",
**producer_kwargs,
)
copy = project_turn_outcome_copy(projection)
assert (copy.key if copy is not None else None) == expected_key
rendered = render_turn_outcome_copy(projection, "en")
assert (rendered is None) == (expected_key is None)
def test_request_nonfallback_renders_upstream_detail_verbatim() -> None:
detail = "Claude Code 2.1.261 does not support this model; {model} needs 2.1.280."
projection = produce_turn_outcome(
"turn.request_nonfallback",
upstream_detail=detail,
)
assert projection.upstream_detail == detail
for language in ("en", "zh"):
rendered = render_turn_outcome_copy(projection, language)
assert rendered is not None and rendered.endswith(detail)
assert render_turn_outcome_copy(
produce_turn_outcome("turn.request_nonfallback", upstream_detail=""),
"en",
) == i18n_t("modelHub.launch.request_incompatible", "en")
def test_upstream_detail_is_ignored_outside_request_nonfallback() -> None:
projection = produce_turn_outcome(
"turn.engine_down",
upstream_detail="engine said something",
)
assert projection.upstream_detail is None
def test_turn_outcome_producer_rejects_missing_streamed_fallback_facts() -> None:
with pytest.raises(TurnOutcomeProductionError, match="missing"):
produce_turn_outcome("turn.streamed_fallback")
def test_recovered_exhaustion_projects_waiting_without_a_past_retry_time() -> None:
source = _source("src_recovered01", "Recovered source")
config = _config([source])
resolution = _terminal_resolution_facts(
config,
supply_status="degraded",
next_hop=(source.id, "shared-model"),
)
projection = produce_turn_outcome(
"turn.exhausted",
config=config,
resolution=resolution,
)
copy = project_turn_outcome_copy(projection)
assert projection.outcome == "exhausted"
assert projection.supply_facts is not None
assert projection.supply_facts.supply_state == "waiting"
assert projection.supply_facts.retry_at == ""
assert copy is not None
assert copy.key == "modelHub.launch.waiting_without_retry"
def test_recovered_no_candidate_projects_waiting_without_retry_time() -> None:
source = _source("src_recovered02", "Recovered source")
config = _config([source])
resolution = _terminal_resolution_facts(
config,
supply_status="degraded",
next_hop=(source.id, "shared-model"),
)
projection = produce_turn_outcome(
"turn.no_candidate.blocked",
config=config,
resolution=resolution,
)
copy = project_turn_outcome_copy(projection)
assert projection.supply_facts is not None
assert projection.supply_facts.supply_state == "waiting"
assert projection.supply_facts.retry_at == ""
assert copy is not None
assert copy.key == "modelHub.launch.waiting_without_retry"
def test_route_unconfigured_launch_copy_exists_in_each_backend_locale() -> None:
config = _config([])
resolution = _terminal_resolution_facts(
config,
model="menu-model",
supply_status="interrupted",
structural_reason="route_unconfigured",
)
failure = ModelHubError(
"no_candidate",
turn_outcome=produce_turn_outcome(
"turn.no_candidate.unconfigured",
config=config,
resolution=resolution,
),
)
for locale in ("en", "zh"):
rendered = _localized_launch_error(
SimpleNamespace(config=SimpleNamespace(language=locale)),
"claude",
"menu-model",
failure,
)
assert rendered.detail == i18n_t(
"modelHub.launch.route_unconfigured",
locale,
model="menu-model",
)
def test_structural_blocker_copy_uses_its_reason_instead_of_source_status() -> None:
source = _source(
"src_blocker01",
"Exact source",
)
config = _config([source])
config.agents["claude"].routes["shared-model"] = ModelHubRouteConfig(
hops=(ModelHubRouteHopConfig(source.id, "removed-model"),)
)
resolution = _terminal_resolution_facts(
config,
supply_status="interrupted",
blocker_reason="model_unsupported",
)
failure = ModelHubError(
"no_candidate",
turn_outcome=produce_turn_outcome(
"turn.no_candidate.blocked",
config=config,
resolution=resolution,
),
)
rendered = _localized_launch_error(
SimpleNamespace(config=SimpleNamespace(language="en")),
"claude",
"menu-model",
failure,
)
assert event_reason_label("model_unsupported", "en") in rendered.detail
assert "standby" not in rendered.detail
class MemoryStore:
def __init__(self, config: ModelHubConfig):
self.config = config
self.recovery_warning = False
self.requested_models = {
"claude": "shared-model",
"codex": "shared-model",
"opencode": "shared-model",
}
def load(self) -> ModelHubConfig:
if self.recovery_warning:
return ModelHubConfig.from_payload(self.config.to_payload())
return self.config
def save(self, config: ModelHubConfig) -> None:
if self.recovery_warning:
raise ValueError("Config was loaded with recovery warnings")
self.config = config
def requested_model(self, backend: str) -> str:
return self.requested_models.get(backend, "")
def mutate(self, mutator):
config = copy.deepcopy(self.config)
if mutator(config):
self.save(config)
class _EngineObservation:
"""The adapter-side half of `InvokeHandle` these fakes all answer the same way.
Every fake below stands in for an engine that reports nothing about a body
until the gateway reads it, which is the state the contract calls `None`.
A fake that hands over a real observation sets `_observed` and thereby says
so explicitly, instead of being the only one that remembered the member.
"""
_observed = None
@property
def observed(self):
return self._observed
class InvokeHandle(_EngineObservation):
def __init__(self, outcome: RawCallOutcome):
self._outcome = outcome
@property
def stream(self):
return None
@property
def outcome_available(self) -> bool:
return True
async def close_stream(self) -> None:
return None
async def outcome(self) -> RawCallOutcome:
return self._outcome
class LiveInvokeHandle(_EngineObservation):
def __init__(
self,
outcome: RawCallOutcome,
chunks: tuple[bytes, ...],
observed: ProtocolSSEState | None = None,
):
self._outcome = outcome
self._observed = observed
self._stream = self._iterate(chunks)
@staticmethod
async def _iterate(chunks: tuple[bytes, ...]):
for chunk in chunks:
yield chunk
@property
def stream(self):
return self._stream
@property
def outcome_available(self) -> bool:
return True
async def close_stream(self) -> None:
await self._stream.aclose()
async def outcome(self) -> RawCallOutcome:
return self._outcome
class BlockingLiveInvokeHandle(_EngineObservation):
def __init__(self, outcome: RawCallOutcome):
self._outcome = outcome
self.started = asyncio.Event()
self.cancelled = asyncio.Event()
self._release = asyncio.Event()
self._available = False
self._stream = self._iterate()
async def _iterate(self):
self.started.set()
try:
await self._release.wait()
except asyncio.CancelledError:
self.cancelled.set()
raise
self._available = True
yield b"data: {}\n\n"
@property
def stream(self):
return self._stream
@property
def outcome_available(self) -> bool:
return self._available
async def close_stream(self) -> None:
await self._stream.aclose()
def release(self) -> None:
self._release.set()
async def outcome(self) -> RawCallOutcome:
return self._outcome
class MidStreamBlockingInvokeHandle(BlockingLiveInvokeHandle):
async def _iterate(self):
yield b"data: {\"type\":\"response.output_text.delta\"}\n\n"
self.started.set()
try:
await self._release.wait()
except asyncio.CancelledError:
self.cancelled.set()
raise
self._available = True
yield b"data: [DONE]\n\n"
class BrokenUpstreamInvokeHandle(_EngineObservation):
def __init__(self, outcome: RawCallOutcome):
self._outcome = outcome
self._stream = self._iterate()
@staticmethod
async def _iterate():
if False:
yield b""
raise ConnectionResetError("upstream disconnected")
@property
def stream(self):
return self._stream
@property
def outcome_available(self) -> bool:
return True
async def close_stream(self) -> None:
await self._stream.aclose()
async def outcome(self) -> RawCallOutcome:
return self._outcome
@contextmanager
def _occupied_ledger_writer():
"""Hold the one thread ledger writes run on, so a queued write cannot start.
That is the window where cancelling whoever awaits a write used to cancel the
write itself, and it is only reachable while the writing thread is busy.
"""
occupied = threading.Event()
release = threading.Event()
def occupy() -> None:
occupied.set()
release.wait(5)
occupant = _ledger_executor().submit(occupy)
assert occupied.wait(5)
try:
yield
finally:
release.set()
occupant.result(timeout=5)
class FakeStreamResponse:
def __init__(
self,
*,
prepare_error: BaseException | None = None,
write_error: BaseException | None = None,
eof_error: BaseException | None = None,
eof_reached: asyncio.Event | None = None,
) -> None:
self.prepare_error = prepare_error
self.write_error = write_error
self.eof_error = eof_error
self.eof_reached = eof_reached
self.writes: list[bytes] = []
self.eof_called = False
async def prepare(self, _request) -> None:
if self.prepare_error is not None:
raise self.prepare_error
return None
async def write(self, _chunk: bytes) -> None:
self.writes.append(_chunk)
if self.write_error is not None:
raise self.write_error
return None
async def write_eof(self) -> None:
self.eof_called = True
if self.eof_reached is not None:
self.eof_reached.set()
if self.eof_error is not None:
raise self.eof_error
return None
class DeferredLifecycleHandle(_EngineObservation):
def __init__(self, outcome: RawCallOutcome):
self._outcome = outcome
self._available = False
self._stream = self._iterate()
self.close_calls = 0
self.outcome_calls = 0
async def _iterate(self):
try:
yield b"data: {}\n\n"
finally:
self._available = True
@property
def stream(self):
return self._stream
@property
def outcome_available(self) -> bool:
return self._available
async def close_stream(self) -> None:
if self.close_calls:
return
self.close_calls += 1
await self._stream.aclose()
async def outcome(self) -> RawCallOutcome:
self.outcome_calls += 1
if not self._available:
await asyncio.Event().wait()
return self._outcome
class ObservedUnsettledHandle(_EngineObservation):
"""A live handle whose prelude proved billing but not a terminal outcome."""
def __init__(self, observed: ProtocolSSEState):
self._observed = observed
self._stream = self._iterate()
self.close_calls = 0
@staticmethod
async def _iterate():
if False:
yield b""
@property
def stream(self):
return self._stream
@property
def outcome_available(self) -> bool:
return False
async def close_stream(self) -> None:
self.close_calls += 1
await self._stream.aclose()
async def outcome(self) -> RawCallOutcome:
await asyncio.Event().wait()
raise AssertionError("unreachable")
class RepeatedCancellationHandle(_EngineObservation):
def __init__(self, outcome: RawCallOutcome, blocked_phase: str):
self._outcome = outcome
self._blocked_phase = blocked_phase
self.started = asyncio.Event()
self.phase_started = asyncio.Event()
self.release_phase = asyncio.Event()
self.close_calls = 0
self.outcome_calls = 0
self._available = False
self._stream = self._iterate()
async def _iterate(self):
yield b"data: [DONE]\n\n"
self.started.set()
await asyncio.Event().wait()
@property
def stream(self):
return self._stream
@property
def outcome_available(self) -> bool:
return self._available
async def close_stream(self) -> None:
self.close_calls += 1
if self.close_calls > 1:
return
if self._blocked_phase == "resource_teardown":
self.phase_started.set()
await self.release_phase.wait()
self._available = True
await self._stream.aclose()
async def outcome(self) -> RawCallOutcome:
self.outcome_calls += 1
if self._blocked_phase == "handle_outcome":
self.phase_started.set()
await self.release_phase.wait()
return self._outcome
class NeverResolvingCloseHandle(_EngineObservation):
def __init__(self, outcome: RawCallOutcome):
self._outcome = outcome
self.started = asyncio.Event()
self.close_started = asyncio.Event()
self.release_close = asyncio.Event()
self.closed = asyncio.Event()
self._stream = self._iterate()
async def _iterate(self):
yield b"data: [DONE]\n\n"
self.started.set()
await asyncio.Event().wait()
@property
def stream(self):
return self._stream
@property
def outcome_available(self) -> bool:
return False
async def close_stream(self) -> None:
self.close_started.set()
while not self.release_close.is_set():
try:
await self.release_close.wait()
except asyncio.CancelledError:
continue
self.closed.set()
async def outcome(self) -> RawCallOutcome:
return self._outcome
def _prepared_gateway_request(
gateway: ModelHubTurnGateway,
*,
turn_id: str,
requested_model: str,
source_id: str,
stream: bool,
) -> SimpleNamespace:
token = gateway.correlation.prepare_gateway_turn(
backend="codex",
token=gateway.correlation.credentials("codex", "/repo", turn_id),
requested_model_id=requested_model,
resolved_model_id="shared-model",
source_id=source_id,
via_mapping=False,
)
return SimpleNamespace(
match_info={"backend": "codex", "endpoint": "responses"},
headers={"Authorization": f"Bearer {token}"},
json=AsyncMock(
return_value={
"model": "shared-model",
"input": "ping",
"stream": stream,
}
),
)
class ProbeAdapter:
def __init__(
self,
outcomes: list[RawCallOutcome],
live_handles: list[LiveInvokeHandle | BlockingLiveInvokeHandle] | None = None,
):
self.outcomes = deque(outcomes)
self.live_handles = deque(live_handles or [])
self.invocations: list[tuple[str, str, str]] = []
self.requests: list[ModelHubRequest] = []
self.refreshable_credential_refs: set[str] = set()
self.capability_queries: list[str] = []
async def ensure_installed(
self,
*,
force: bool = False,
offline: bool = False,
) -> EngineEnsureResult:
return EngineEnsureResult(status=await self.status(), changed=force)
async def sync_sources(self, _bindings) -> None:
return None
async def revoke_credential(self, _credential_ref: str) -> None:
return None
async def credential_supports_refresh(self, credential_ref: str) -> bool:
self.capability_queries.append(credential_ref)
return credential_ref in self.refreshable_credential_refs
async def invoke(self, source_id, model_id, request, _stream, origin, *, on_admitted=None):
if on_admitted is not None:
on_admitted()
self.invocations.append((source_id, model_id, origin))
self.requests.append(request)
if self.live_handles:
return self.live_handles.popleft()
return InvokeHandle(self.outcomes.popleft())
async def status(self) -> EngineStatus:
return EngineStatus(
health=EngineHealth.OK,
installed_version="test",
verified=True,
listen_host="127.0.0.1",
listen_port=15220,
last_check_iso=NOW.isoformat(),
)
def _source(
source_id: str,
label: str,
*,
channel: str = "hub",
status: str = "standby",
retry_at: str | None = None,
vendor: str = "openai",
protocol: str = "openai_responses",
model_id: str = "shared-model",
) -> ModelHubSourceConfig:
return ModelHubSourceConfig(
id=source_id,
kind="subscription" if channel == "native_cli" else "api_key",
vendor=vendor,
display_name=label,
protocol=protocol,
supply_channel=channel,
billing="monthly" if channel == "native_cli" else "metered",
state=ModelHubSourceStateConfig(
status=status,
retry_at=retry_at,
detail_key=("models.source.cooldown.rate_limited" if status == "cooldown" else None),
),
models=[
ModelHubModelConfig(
id=model_id,
provenance="discovered",
)
],
credential_ref=None if channel == "native_cli" else f"cred_{source_id}",
)
def _config(sources: list[ModelHubSourceConfig]) -> ModelHubConfig:
agents = {
backend: ModelHubAgentSupplyConfig.default(backend, mode="hub") for backend in ("claude", "codex", "opencode")
}
for backend, agent in agents.items():
eligible = [source for source in sources if ModelHubConfig.source_eligible_for_backend(source, backend)]
agent.sources = ModelHubAgentSourcesConfig(
order=[source.id for source in eligible],
)
requested_model = "shared-model"
agent.routes[requested_model] = ModelHubRouteConfig(
hops=tuple(
ModelHubRouteHopConfig(
source_id=source.id,
model_id=source.models[0].id,
)
for source in eligible
)
)
agents["opencode"].models = [
ModelHubBackendModelConfig(
id="shared-model",
native_protocol="openai_responses",
)
]
agents["opencode"].menu.checked = ["shared-model"]
return ModelHubConfig(sources=sources, agents=agents)
def _outcome(
kind: RawOutcomeKind,
*,
status: int | None = None,
code: str | None = None,
message: str | None = None,
source_id: str = "src_primary01",
stream_started: bool = False,
usage: ProtocolUsageReport | None = None,
) -> RawCallOutcome:
return RawCallOutcome(
kind=kind,
http_status=status,
error_code=code,
redacted_message=message,
stream_started=stream_started,
model_id="shared-model",
source_id=source_id,
usage=usage,
)
def _service(
tmp_path: Path,
*,
sources: list[ModelHubSourceConfig],
outcomes: list[RawCallOutcome] | None = None,
live_handles: list[LiveInvokeHandle | BlockingLiveInvokeHandle] | None = None,
recovery_window_seconds: float = 0,
) -> ModelHubService:
store = MemoryStore(_config(sources))
service = ModelHubService(
store=store,
adapter=ProbeAdapter(outcomes or [], live_handles),
events=BoundedEventLog(tmp_path / "events.json"),
provenance=BoundedProvenanceStore(tmp_path / "provenance.json"),
usage=BoundedUsageLedger(tmp_path / "usage.json", now=lambda: NOW),
revocations=CredentialRevocationJournal(tmp_path / "revocations.json"),
now=lambda: NOW,
requested_model_override=store.requested_model,
)
# This fixture characterizes a single resolver walk. Recovery behavior opts
# into its own clocked policy; ordinary production requests default to 120s.
service.recovery.window_seconds = recovery_window_seconds
service.recovery.monotonic = lambda: (service.now() - NOW).total_seconds()
service.recovery.jitter = lambda low, high: low
return service
def _canonicalize_fixed_test_routes(
service: ModelHubService,
) -> dict[str, str]:
selected = {}
config = service.store.load()
for backend in ("claude", "codex"):
menu_ids = model_hub_fixed_menu_ids(backend)
selected[backend] = menu_ids[0]
shared_route = config.agents[backend].routes.get(
"shared-model",
ModelHubRouteConfig(),
)
config.agents[backend].routes = {
model_id: (shared_route if model_id == selected[backend] else ModelHubRouteConfig())
for model_id in menu_ids
}
service.store.requested_models[backend] = selected[backend]
return selected
def _begin_hub_attempt(
registry: TurnCorrelationRegistry,
*,
turn_id: str,
scope: str = "/repo",
) -> str | None:
token = registry.credentials("codex", scope, turn_id)
exact_turn = registry.begin_gateway_request(
backend="codex",
token=token,
requested_model_id="shared-model",
)
registry.begin_attempt(
exact_turn,
source_id="src_primary01",
resolved_model_id="shared-model",
channel="hub",
via_mapping=False,
)
return exact_turn
def test_process_credentials_record_only_exact_turns(tmp_path: Path) -> None:
store = BoundedProvenanceStore(tmp_path / "provenance.json")
registry = TurnCorrelationRegistry(store)
token = registry.credentials("codex", "/repo", "turn_exact")
assert token == registry.credentials("codex", "/repo", "turn_exact")
assert token != registry.credentials("codex", "/other", "turn_other")
assert registry.authenticates("codex", token) is True
assert registry.authenticates("claude", token) is False
assert registry.credentials("claude", "session-a", "turn_a") != registry.credentials(
"claude",
"session-b",
"turn_b",
)
exact_turn = registry.begin_gateway_request(
backend="codex",
token=token,
requested_model_id="shared-model",
)
registry.begin_attempt(
exact_turn,
source_id="src_primary01",
resolved_model_id="shared-model",
channel="hub",
via_mapping=False,
)
success = _outcome(RawOutcomeKind.SUCCESS)
registry.finish_attempt(
exact_turn,
outcome=success,
decision=classify_outcome(success),
)
registry.settle(
"turn_exact",
settled_by=SETTLED_BY_TERMINAL_RESULT,
ts=NOW.isoformat(),
)
record = store.get("turn_exact")
assert record is not None
assert record["outcome"] == "served"
_assert_valid("turn-provenance.schema.json", record)
def test_stopped_settlement_cannot_erase_committed_served_history(
tmp_path: Path,
caplog: pytest.LogCaptureFixture,
) -> None:
store = BoundedProvenanceStore(tmp_path / "stop-after-served.json")
registry = TurnCorrelationRegistry(store)
turn_id = "turn_stop_after_served"
exact_turn = _begin_hub_attempt(registry, turn_id=turn_id)
success = _outcome(RawOutcomeKind.SUCCESS)
registry.finish_attempt(
exact_turn,
outcome=success,
decision=classify_outcome(success),
)
registry.record_turn_outcome(turn_id, produce_turn_outcome("turn.served"))
with caplog.at_level("INFO", logger="core.handlers.model_hub.provenance"):
registry.settle(
turn_id,
settled_by=SETTLED_BY_STOPPED,
ts=NOW.isoformat(),
)
record = store.get(turn_id)
assert record is not None
assert record["outcome"] == "served"
assert record["served"]["source_id"] == "src_primary01"
assert record["canceled_attempt"] is None
assert "Ignored stopped settlement" in caplog.text
_assert_valid("turn-provenance.schema.json", record)
@pytest.mark.parametrize(
("previous_decision", "variant"),
[
(decision, variant)
for decision, rule in TURN_OUTCOME_RENDERING_AUTHORITY.items()
if rule.outcome in {"no_candidate", "exhausted"}
for variant, _key in rule.copy_keys
],
)
@pytest.mark.parametrize("ending", ["served", "exhausted", "failed_terminal", "canceled"])
def test_admitted_retry_supersedes_earlier_supply_failure(tmp_path, previous_decision, variant, ending):
store = BoundedProvenanceStore(tmp_path / "records.json")
registry = TurnCorrelationRegistry(store)
turn_id = _begin_hub_attempt(registry, turn_id="turn-retry")
previous_failure = _outcome(RawOutcomeKind.HTTP_ERROR, status=503)
registry.finish_attempt(turn_id, outcome=previous_failure, decision=classify_outcome(previous_failure))
source = _source(
"src_recovered01", "Recovering",
status="cooldown" if variant == "waiting" else "standby",
retry_at=(NOW + timedelta(seconds=30)).isoformat() if variant == "waiting" else None,
)
config = _config([] if previous_decision == "turn.no_candidate.unconfigured" else [source])
projection = produce_turn_outcome(
previous_decision,
config=config,
resolution=_terminal_resolution_facts(
config,
supply_status="degraded" if variant == "waiting_without_retry" else variant,
structural_reason="route_unconfigured" if previous_decision == "turn.no_candidate.unconfigured" else None,
next_hop=(source.id, "shared-model") if variant == "waiting_without_retry" else None,
),
)
if projection.outcome == "no_candidate":
registry.mark_gateway_no_candidate(
turn_id, projection.supply_facts.supply_state,
blockers=(ExactHopBlocker(source.id, "shared-model", "server_error"),),
)
registry.record_turn_outcome(turn_id, projection)
registry.begin_attempt(
turn_id, source_id=source.id, resolved_model_id="shared-model", channel="hub", via_mapping=False,
request_id="recovered-request",
)
if ending != "canceled":
outcome = _outcome(
RawOutcomeKind.SUCCESS if ending == "served" else RawOutcomeKind.HTTP_ERROR,
status={"served": 200, "exhausted": 503, "failed_terminal": 400}[ending],
code="invalid_parameter" if ending == "failed_terminal" else None,
source_id=source.id,
)
registry.finish_attempt(
turn_id, outcome=outcome, decision=classify_outcome(outcome), request_id="recovered-request",
)
settled_by = SETTLED_BY_STOPPED if ending == "canceled" else SETTLED_BY_TERMINAL_RESULT
registry.close_turn_admission(turn_id, settled_by=settled_by)
registry.settle(turn_id, settled_by=settled_by)
record = store.get(turn_id)
assert record["outcome"] == ending
assert record["model_supply_state"] is None
assert record["blockers"] == []
assert record["failed_attempts"][0]["source_id"] == previous_failure.source_id
assert len(record["failed_attempts"]) == (2 if ending == "exhausted" else 1)
if ending == "canceled":
assert record["canceled_attempt"]["source_id"] == source.id
_assert_valid("turn-provenance.schema.json", record)
def test_stopped_settlement_cannot_erase_committed_protocol_failure(
tmp_path: Path,
) -> None:
fixture = E64_SETTLEMENT_BOUNDARIES["stopped_after_terminal"]
store = BoundedProvenanceStore(tmp_path / "stop-after-protocol.json")
registry = TurnCorrelationRegistry(store)
turn_id = "turn_stop_after_protocol"
exact_turn = _begin_hub_attempt(registry, turn_id=turn_id)
failure = _outcome(RawOutcomeKind.PROTOCOL_ERROR, stream_started=True)
registry.finish_attempt(
exact_turn,
outcome=failure,
decision=classify_outcome(failure),
)
registry.settle(
turn_id,
settled_by=SETTLED_BY_STOPPED,
ts=NOW.isoformat(),
)
record = store.get(turn_id)
assert record is not None
assert record["outcome"] == fixture["expected_outcome"]
assert record["terminal_error"]["reason"] == fixture["terminal_reason"]
assert record["canceled_attempt"] is None
_assert_valid("turn-provenance.schema.json", record)
def test_released_v5_permission_denied_records_degrade_at_read_boundary(
tmp_path: Path,
) -> None:
provenance_path = tmp_path / "provenance.json"
events_path = tmp_path / "events.json"
provenance_path.write_text(
json.dumps(RELEASED_V5_PERMISSION_DENIED["provenance"]),
encoding="utf-8",
)
events_path.write_text(
json.dumps(RELEASED_V5_PERMISSION_DENIED["resolution_events"]),
encoding="utf-8",
)
record = BoundedProvenanceStore(provenance_path).get("turn_01k9x6db")
event = BoundedEventLog(events_path).list()[0]
degraded_reason = RELEASED_V5_PERMISSION_DENIED["degraded_reason"]
assert record is not None
assert record["failed_attempts"][0]["reason"] == degraded_reason
assert event["reason"] == degraded_reason
_assert_valid("turn-provenance.schema.json", record)
_assert_valid("resolution-event.schema.json", event)
assert RELEASED_V5_PERMISSION_DENIED["provenance"][0]["failed_attempts"][0]["reason"] == "permission_denied"
assert RELEASED_V5_PERMISSION_DENIED["resolution_events"][0]["reason"] == "permission_denied"
def test_retired_process_scope_revokes_token_and_fails_closed(
tmp_path: Path,
) -> None:
store = BoundedProvenanceStore(tmp_path / "retired-scope.json")
registry = TurnCorrelationRegistry(store)
token = registry.credentials("codex", "/repo", "turn_evicted")
launch = registry.prepare_gateway_turn(
backend="codex",
token=token,
requested_model_id="shared-model",
resolved_model_id="shared-model",
source_id="src_primary01",
via_mapping=False,
)
assert launch != token
turn_id = registry.begin_gateway_request(
backend="codex",
token=token,
requested_model_id="shared-model",
)
registry.begin_attempt(
turn_id,
source_id="src_primary01",
resolved_model_id="shared-model",
channel="hub",
via_mapping=False,
)
registry.retire_scope("codex", "/repo")
# Every credential the scope minted is revoked, not only the one that
# names the process: a launch credential is as good as a token.
assert registry.authenticates("codex", token) is False
assert registry.authenticates("codex", launch) is False
replacement = registry.credentials("codex", "/repo", "turn_replacement")
assert replacement != token
assert registry.authenticates("codex", replacement) is True
registry.settle(
"turn_evicted",
settled_by=SETTLED_BY_NO_TERMINAL_RESULT,
ts=NOW.isoformat(),
)
assert store.get("turn_evicted") is None
registry.settle(
"turn_replacement",
settled_by=SETTLED_BY_NO_TERMINAL_RESULT,
ts=NOW.isoformat(),
)
registry.retire_scope("codex", "/repo")
assert registry._scopes == {}
assert registry._credentials == {}
def test_terminal_failure_survives_exact_scope_retirement(
tmp_path: Path,
) -> None:
store = BoundedProvenanceStore(tmp_path / "terminal-retirement.json")
registry = TurnCorrelationRegistry(store)
registry.begin_native_attempt(
backend="claude",
process_scope="session:/repo",
turn_id="turn_startup_failure",
requested_model_id="claude-opus",
source_id="src_native01",
resolved_model_id="claude-opus",
via_mapping=False,
)
registry.fail_native_attempt(
"turn_startup_failure",
reason="unclassified_error",
)
registry.retire_scope(
"claude",
"session:/repo",
terminal_turn_id="turn_startup_failure",
)
registry.settle(
"turn_startup_failure",
settled_by=SETTLED_BY_TERMINAL_RESULT,
ts=NOW.isoformat(),
)
record = store.get("turn_startup_failure")
assert record is not None
assert record["outcome"] == "exhausted"
assert record["failed_attempts"][0]["reason"] == "unclassified_error"
_assert_valid("turn-provenance.schema.json", record)
def test_terminal_retirement_does_not_restore_ambiguous_scope(
tmp_path: Path,
) -> None:
store = BoundedProvenanceStore(tmp_path / "ambiguous-retirement.json")
registry = TurnCorrelationRegistry(store)
registry.begin_native_attempt(
backend="claude",
process_scope="session:/repo",
turn_id="turn_first",
requested_model_id="claude-opus",
source_id="src_native01",
resolved_model_id="claude-opus",
via_mapping=False,
)
registry.begin_native_attempt(
backend="claude",
process_scope="session:/repo",
turn_id="turn_second",
requested_model_id="claude-opus",
source_id="src_native01",
resolved_model_id="claude-opus",
via_mapping=False,
)
registry.fail_native_attempt("turn_first", reason="unclassified_error")
registry.retire_scope(
"claude",
"session:/repo",
terminal_turn_id="turn_first",
)
registry.settle(
"turn_first",
settled_by=SETTLED_BY_TERMINAL_RESULT,
ts=NOW.isoformat(),
)
assert store.get("turn_first") is None
def _routing_registry(tmp_path: Path, label: str) -> tuple[TurnCorrelationRegistry, str]:
store = BoundedProvenanceStore(tmp_path / f"routing-{label}.json")
registry = TurnCorrelationRegistry(store)
return registry, registry.credentials("claude", "session:/repo", "turn_route01")
def _prepare_route(
registry: TurnCorrelationRegistry,
token: str,
*,
turn_id: str,
requested: str,
resolved: str,
source_id: str = "src_primary01",
) -> str:
"""Prepare a route and return the credential its launch must use.
Mirrors `ModelHubTurnGateway.endpoint`: the scope credential goes in, the
credential bound to this route comes back, and that is the one the
launched process authenticates with.
"""
return registry.prepare_gateway_turn(
backend="claude",
token=token,
turn_id=turn_id,
requested_model_id=requested,
resolved_model_id=resolved,
source_id=source_id,
via_mapping=True,
)
def _resolution(registry: TurnCorrelationRegistry, token: str, model: str) -> str | None:
return registry.claim_gateway_request(
backend="claude",
token=token,
prepared_turn_id=None,
gateway_model_id=model,
).caller_model_id
@pytest.mark.parametrize(
"arrives",
[
pytest.param("after_its_turn_settled", id="after_its_turn_settled"),
pytest.param("while_another_model_is_live", id="while_another_model_is_live"),
],
)
def test_gateway_keeps_a_live_process_routable_outside_its_turn_windows(
tmp_path: Path,
arrives: str,
) -> None:
"""Routing is a property of the process scope, not of an open turn window.
A launched CLI keeps issuing gateway requests between the turns Avibe
dispatches — tool loops, agent-initiated continuations, transport retries.
Those requests carry the upstream model id, which only the route the
process was launched on can translate back into the caller model the
resolver routes from, so routing tied to an open turn window makes every
one of them unroutable.
"""
registry, token = _routing_registry(tmp_path, arrives)
route_token = _prepare_route(
registry,
token,
turn_id="turn_route01",
requested="caller-model",
resolved="hub-model",
)
registry.settle(
"turn_route01",
settled_by=SETTLED_BY_TERMINAL_RESULT,
ts=NOW.isoformat(),
)
if arrives == "while_another_model_is_live":
# A dispatched turn on a different model is the common case: the CLI
# keeps finishing work for the previous one. Its route answers for the
# model it prepared and must say nothing about any other.
registry.credentials("claude", "session:/repo", "turn_route02")
_prepare_route(
registry,
token,
turn_id="turn_route02",
requested="other-caller",
resolved="other-hub-model",
)
assert _resolution(registry, route_token, "hub-model") == "caller-model"
# Reach follows the route this launch was made on, so a model that route
# does not name stays unroutable rather than becoming reachable by naming
# it — including one another launch on the same process can reach.
assert _resolution(registry, route_token, "never-routed") is None
assert _resolution(registry, route_token, "other-hub-model") is None
@pytest.mark.parametrize(
("scenario", "rival_caller", "rival_source"),
[
pytest.param("two_caller_models", "rival-caller", "src_primary01", id="two_caller_models"),
pytest.param("two_sources", "caller-model", "src_backup001", id="two_sources"),
],
)
def test_gateway_answers_each_route_that_aliases_one_upstream_model(
tmp_path: Path,
scenario: str,
rival_caller: str,
rival_source: str,
) -> None:
"""A request is routed by the credential it arrives on, not by its model.
Two menu entries may resolve to one upstream model — deliberately, as two
price tiers or two Sources for the same model — and one process scope
launches both. The wire carries the upstream id alone, so a credential
naming only the process cannot say which of them a request belongs to, and
the routing table has to either guess or refuse. Binding the credential to
the route removes the question: each launch answers for its own route, and
that stays true once the turns that prepared them have settled.
"""
registry, token = _routing_registry(tmp_path, scenario)
registry.credentials("claude", "session:/repo", "turn_route02")
first = _prepare_route(
registry,
token,
turn_id="turn_route01",
requested="caller-model",
resolved="hub-model",
source_id="src_primary01",
)
rival = _prepare_route(
registry,
token,
turn_id="turn_route02",
requested=rival_caller,
resolved="hub-model",
source_id=rival_source,
)
assert first != rival
for turn_id in (None, "turn_route01", "turn_route02"):
if turn_id is not None:
registry.settle(
turn_id,
settled_by=SETTLED_BY_TERMINAL_RESULT,
ts=NOW.isoformat(),
)
assert _resolution(registry, first, "hub-model") == "caller-model"
assert _resolution(registry, rival, "hub-model") == rival_caller
# The scope credential names the process and no route, so it answers
# for neither rather than picking one.
assert _resolution(registry, token, "hub-model") is None
def test_gateway_routing_dies_with_the_process_scope(tmp_path: Path) -> None:
"""A retired scope revokes its credentials, so its routes stop answering.
Every credential the scope minted, not only the one that names the process
itself: a launch credential outliving its scope would keep a dead
runtime's routes answering on a token nothing can revoke.
"""
registry, token = _routing_registry(tmp_path, "retired")
route_token = _prepare_route(
registry,
token,
turn_id="turn_route01",
requested="caller-model",
resolved="hub-model",
)
registry.settle(
"turn_route01",
settled_by=SETTLED_BY_TERMINAL_RESULT,
ts=NOW.isoformat(),
)
assert _resolution(registry, route_token, "hub-model") == "caller-model"
registry.retire_scope("claude", "session:/repo")
assert registry.authenticates("claude", token) is False
assert registry.authenticates("claude", route_token) is False
assert _resolution(registry, route_token, "hub-model") is None
@pytest.mark.parametrize(
("settled_resolved", "live_resolved"),
[
pytest.param("hub-settled", "hub-live", id="distinct_upstream_models"),
pytest.param("hub-shared", "hub-shared", id="one_aliased_upstream_model"),
],
)
def test_gateway_attributes_a_request_only_to_a_turn_that_claims_its_model(
tmp_path: Path,
settled_resolved: str,
live_resolved: str,
) -> None:
"""Serving a scope-routed request must not cost the live turn its record.
Routing and attribution share one entry point, so an out-of-turn request is
resolved while some unrelated turn is the only open window. Binding it there
would tell that turn a model it never asked for arrived on its token, which
marks it ambiguous — and an ambiguous trace makes `settle` drop the record
for the request the turn goes on to make itself.
A live turn's claim is its whole route, not the upstream model the route
ends at, so the aliased case holds too: two menu entries resolving to one
upstream model are two routes, and the live one owns only its own.
"""
store = BoundedProvenanceStore(tmp_path / "routing-attribution.json")
registry = TurnCorrelationRegistry(store)
token = registry.credentials("claude", "session:/repo", "turn_settled01")
settled_token = _prepare_route(
registry,
token,
turn_id="turn_settled01",
requested="caller-settled",
resolved=settled_resolved,
)
registry.settle(
"turn_settled01",
settled_by=SETTLED_BY_TERMINAL_RESULT,
ts=NOW.isoformat(),
)
assert registry.credentials("claude", "session:/repo", "turn_live01") == token
live_token = _prepare_route(
registry,
token,
turn_id="turn_live01",
requested="caller-live",
resolved=live_resolved,
)
with registry.gateway_terminalizer(
backend="claude",
token=settled_token,
) as out_of_turn:
assert out_of_turn.resolution_model(settled_resolved) == "caller-settled"
# Its own turn has already settled, so there is nothing to attribute to.
assert out_of_turn.turn_id is None
with registry.gateway_terminalizer(backend="claude", token=live_token) as live:
assert live.resolution_model(live_resolved) == "caller-live"
assert live.turn_id == "turn_live01"
registry.settle(
"turn_live01",
settled_by=SETTLED_BY_TERMINAL_RESULT,
ts=NOW.isoformat(),
)
record = store.get("turn_live01")
assert record is not None
assert record["terminal_error"] == {
"source_id": "src_primary01",
"configured_model_id": live_resolved,
"channel": "hub",
"reason": "protocol_error",
"stream_started": False,
}
# Both are the user's own cancellation: a Stop, or turning the backend off.
@pytest.mark.parametrize("settled_by", [SETTLED_BY_STOPPED, SETTLED_BY_BACKEND_DISABLED])
def test_a_canceled_turn_reports_the_attempt_it_waited_on_longest(
tmp_path: Path,
settled_by: str,
) -> None:
"""A turn holding several requests reports the one in flight the longest.
The settlement record has one attempt slot, but a live process issues
concurrent requests on one turn and each carries its own identity — a
request that fell back to a backup Source is not the same attempt as a peer
still holding the primary. The slot has to name one of them, and the oldest
entry is the only stable answer: entries are removed as their requests
settle, so whatever is still first arrived before all the others and no
later request can displace it.
"""
store = BoundedProvenanceStore(tmp_path / "routing-oldest.json")
registry = TurnCorrelationRegistry(store)
token = registry.credentials("claude", "session:/repo", "turn_live01")
live_token = _prepare_route(
registry,
token,
turn_id="turn_live01",
requested="caller-live",
resolved="hub-live",
)
with registry.gateway_terminalizer(backend="claude", token=live_token) as first:
assert first.resolution_model("hub-live") == "caller-live"
# This request fell back to a backup hop and is awaiting upstream.
first.begin_attempt(
source_id="src_backup001",
resolved_model_id="hub-live",
channel="hub",
via_mapping=True,
)
with registry.gateway_terminalizer(
backend="claude",
token=live_token,
) as second:
assert second.resolution_model("hub-live") == "caller-live"
# Cancelled while both are open, before either has settled.
registry.settle(
"turn_live01",
settled_by=settled_by,
ts=NOW.isoformat(),
)
record = store.get("turn_live01")
assert record is not None
assert record["outcome"] == "canceled"
assert record["canceled_attempt"] == {
"source_id": "src_backup001",
"configured_model_id": "hub-live",
"channel": "hub",
}
@pytest.mark.parametrize("peer", ["open_at_commit", "closed_before_commit", "opened_before_commit"])
@pytest.mark.parametrize("peer_result", ["served", "exhausted"])
@pytest.mark.parametrize("loser_exit", ["no_candidate", "gateway_exit"])
def test_a_request_ending_beside_an_open_peer_leaves_the_turn_to_that_peer(
tmp_path: Path,
loser_exit: str,
peer_result: str,
peer: str,
) -> None:
"""MH-RETRY-PROVENANCE-003: one request's exit is wholly its own or wholly the turn's.
A waiter that lost the half-open slot ends beside the request that won it.
Its staged exit commits as one write: beside an open peer it leaves the
turn to that peer; as the last request to end it is the turn's ending.
"""
store = BoundedProvenanceStore(tmp_path / "routing-peer.json")
registry = TurnCorrelationRegistry(store)
token = registry.credentials("claude", "session:/repo", "turn_peer01")
live_token = _prepare_route(
registry,
token,
turn_id="turn_peer01",
requested="caller-live",
resolved="hub-live",
)
result = (
_outcome(RawOutcomeKind.SUCCESS)
if peer_result == "served"
else _outcome(RawOutcomeKind.HTTP_ERROR, status=500, code="server_error")
)
projection = produce_turn_outcome("turn.engine_down")
def open_peer():
terminalizer = registry.gateway_terminalizer(backend="claude", token=live_token)
assert terminalizer.resolution_model("hub-live") == "caller-live"
terminalizer.begin_attempt(
source_id="src_primary01", resolved_model_id="hub-live", channel="hub", via_mapping=True,
)
return terminalizer
winner = open_peer() if peer != "opened_before_commit" else None
with registry.gateway_terminalizer(backend="claude", token=live_token) as loser:
assert loser.resolution_model("hub-live") == "caller-live"
if loser_exit == "no_candidate":
loser.mark_no_candidate("waiting")
else:
loser.fail("protocol_error")
if peer == "closed_before_commit":
winner.finish_attempt(outcome=result, decision=classify_outcome(result))
winner.__exit__(None, None, None)
elif peer == "opened_before_commit":
winner = open_peer()
loser.record_turn_outcome(projection)
if peer != "closed_before_commit":
# The loser left the turn to its peer: no projection of its own remains.
assert registry.terminal_projection("turn_peer01", backend="claude") is None
winner.finish_attempt(outcome=result, decision=classify_outcome(result))
winner.__exit__(None, None, None)
registry.settle("turn_peer01", settled_by=SETTLED_BY_TERMINAL_RESULT, ts=NOW.isoformat())
record = store.get("turn_peer01")
assert record is not None
if peer == "closed_before_commit":
# The loser ended last, so its whole exit is the turn's.
assert record["outcome"] == ("no_candidate" if loser_exit == "no_candidate" else "failed_terminal")
assert (record["model_supply_state"] is not None) is (loser_exit == "no_candidate")
else:
assert record["outcome"] == peer_result
assert record["terminal_error"] is None
assert record["model_supply_state"] is None
def test_an_unclaimed_request_leaves_no_attempt_on_the_live_turn(
tmp_path: Path,
) -> None:
"""A turn that issued no gateway request must settle as having made none.
The terminalizer opens the live turn's attempt on the way in, before any
model id has been read — `fail` runs before routing and needs a turn to
fail. So a request routed from scope history has to give that attempt back
on its way out: left armed, it settles the live turn as having canceled or
interrupted a Hub attempt that only ever belonged to the settled turn.
"""
store = BoundedProvenanceStore(tmp_path / "routing-phantom.json")
registry = TurnCorrelationRegistry(store)
token = registry.credentials("claude", "session:/repo", "turn_settled01")
settled_token = _prepare_route(
registry,
token,
turn_id="turn_settled01",
requested="caller-settled",
resolved="hub-settled",
)
registry.settle(
"turn_settled01",
settled_by=SETTLED_BY_TERMINAL_RESULT,
ts=NOW.isoformat(),
)
assert registry.credentials("claude", "session:/repo", "turn_live01") == token
_prepare_route(
registry,
token,
turn_id="turn_live01",
requested="caller-live",
resolved="hub-live",
)
with registry.gateway_terminalizer(
backend="claude",
token=settled_token,
) as out_of_turn:
assert out_of_turn.resolution_model("hub-settled") == "caller-settled"
registry.settle(
"turn_live01",
settled_by=SETTLED_BY_NO_TERMINAL_RESULT,
ts=NOW.isoformat(),
)
assert store.get("turn_live01") is None
def test_an_unclaimed_request_leaves_an_attempt_in_flight_untouched(
tmp_path: Path,
) -> None:
"""A request records what it attempted, whoever else is on its turn.
An attempt identity describes one request, and a live process issues
concurrent ones on one turn. So a delayed request for a settled turn's
route arrives while the live turn's own request is awaiting an upstream
result, and it touches the turn's pending state twice: on the way in to
arm a launch identity, and on the way out to give it back. Either move,
aimed at the turn rather than at the request, erases the attempt actually
in flight — and `finish_attempt` reconstructs nothing from an absent one,
so the live turn settles having served a model with no record of which
Source and model served it.
Stated as the surviving provenance rather than as the two moves, so any
later route to the same erasure fails here.
"""
store = BoundedProvenanceStore(tmp_path / "routing-concurrent.json")
registry = TurnCorrelationRegistry(store)
token = registry.credentials("claude", "session:/repo", "turn_settled01")
settled_token = _prepare_route(
registry,
token,
turn_id="turn_settled01",
requested="caller-settled",
resolved="hub-settled",
)
registry.settle(
"turn_settled01",
settled_by=SETTLED_BY_TERMINAL_RESULT,
ts=NOW.isoformat(),
)
assert registry.credentials("claude", "session:/repo", "turn_live01") == token
live_token = _prepare_route(
registry,
token,
turn_id="turn_live01",
requested="caller-live",
resolved="hub-live",
source_id="src_live0001",
)
with registry.gateway_terminalizer(backend="claude", token=live_token) as live:
assert live.resolution_model("hub-live") == "caller-live"
live.begin_attempt(
source_id="src_live0001",
resolved_model_id="hub-live",
channel="hub",
via_mapping=True,
)
# The live request is now awaiting upstream. A delayed request for the
# settled turn's route lands on the same process and same live turn.
with registry.gateway_terminalizer(
backend="claude",
token=settled_token,
) as delayed:
assert delayed.resolution_model("hub-settled") == "caller-settled"
success = _outcome(RawOutcomeKind.SUCCESS, source_id="src_live0001")
live.finish_attempt(outcome=success, decision=classify_outcome(success))
registry.settle(
"turn_live01",
settled_by=SETTLED_BY_TERMINAL_RESULT,
ts=NOW.isoformat(),
)
record = store.get("turn_live01")
assert record is not None
assert record["served"] == {
"source_id": "src_live0001",
"configured_model_id": "hub-live",
"channel": "hub",
}
def test_a_request_gives_back_only_its_own_launch_identity(
tmp_path: Path,
) -> None:
"""Giving back a launch identity must not take a real attempt with it.
The reverse arrival order of the test above: a delayed request arms its
launch identity first, and only then does the live request begin its own
attempt. The delayed one is now stale, and it names the same Source and
model as the live attempt whenever the live request went out on its
primary hop — which is the ordinary case. So a give-back cannot be decided
by what a launch identity looks like, only by whose it is; by value the two
are indistinguishable, and the live turn loses the provenance of a request
it really did make.
"""
store = BoundedProvenanceStore(tmp_path / "routing-stale-prep.json")
registry = TurnCorrelationRegistry(store)
token = registry.credentials("claude", "session:/repo", "turn_settled01")
settled_token = _prepare_route(
registry,
token,
turn_id="turn_settled01",
requested="caller-settled",
resolved="hub-settled",
)
registry.settle(
"turn_settled01",
settled_by=SETTLED_BY_TERMINAL_RESULT,
ts=NOW.isoformat(),
)
assert registry.credentials("claude", "session:/repo", "turn_live01") == token
live_token = _prepare_route(
registry,
token,
turn_id="turn_live01",
requested="caller-live",
resolved="hub-live",
source_id="src_live0001",
)
# The delayed request arms its launch identity first.
with registry.gateway_terminalizer(
backend="claude",
token=settled_token,
) as delayed:
# Only then does the live request begin its own attempt, on the primary
# hop — the same Source and model the identity above named.
with registry.gateway_terminalizer(backend="claude", token=live_token) as live:
assert live.resolution_model("hub-live") == "caller-live"
live.begin_attempt(
source_id="src_live0001",
resolved_model_id="hub-live",
channel="hub",
via_mapping=True,
)
assert delayed.resolution_model("hub-settled") == "caller-settled"
success = _outcome(RawOutcomeKind.SUCCESS, source_id="src_live0001")
live.finish_attempt(outcome=success, decision=classify_outcome(success))
registry.settle(
"turn_live01",
settled_by=SETTLED_BY_TERMINAL_RESULT,
ts=NOW.isoformat(),
)
record = store.get("turn_live01")
assert record is not None
assert record["served"] == {
"source_id": "src_live0001",
"configured_model_id": "hub-live",
"channel": "hub",
}
def test_gateway_serves_a_request_that_arrives_after_its_turn_settled(
tmp_path: Path,
) -> None:
"""The 409 this fixes, end to end: same request, before and after settlement.
``service.adapter.invocations`` is the evidence that the caller model was
the routing input — only the route keyed by it carries this hop.
"""
async def exercise() -> None:
primary = _source(
"src_primary01",
"Primary",
vendor="anthropic",
protocol="anthropic",
model_id="hub-model",
)
service = _service(
tmp_path,
sources=[primary],
outcomes=[_outcome(RawOutcomeKind.SUCCESS, source_id=primary.id)],
)
service.store.config.agents["claude"].routes["caller-model"] = ModelHubRouteConfig(
hops=(ModelHubRouteHopConfig(primary.id, "hub-model"),)
)
gateway = ModelHubTurnGateway(service)
base_url, token = await gateway.endpoint(
"claude",
process_scope="session:/repo",
turn_id="turn_between_windows",
requested_model_id="caller-model",
resolved_model_id="hub-model",
source_id=primary.id,
)
gateway.correlation.settle(
"turn_between_windows",
settled_by=SETTLED_BY_TERMINAL_RESULT,
ts=NOW.isoformat(),
)
try:
async with aiohttp.ClientSession(trust_env=False) as client:
response = await client.post(
f"{base_url}/v1/messages",
json={
"model": "hub-model",
"max_tokens": 1,
"messages": [{"role": "user", "content": "ping"}],
"stream": False,
},
headers={"Authorization": f"Bearer {token}"},
)
assert response.status == 200
await response.read()
finally:
await gateway.close()
assert service.adapter.invocations == [(primary.id, "hub-model", "claude")]
asyncio.run(exercise())
def test_gateway_models_endpoint_serves_authenticated_backend_catalog(
tmp_path: Path,
) -> None:
async def exercise() -> None:
service = _service(
tmp_path,
sources=[_source("src_primary01", "Primary")],
)
gateway = ModelHubTurnGateway(service)
base_url, token = await gateway.endpoint(
"codex",
process_scope="/repo",
turn_id="turn_models_catalog",
requested_model_id="shared-model",
resolved_model_id="shared-model",
source_id="src_primary01",
)
expected = [
model["id"]
for model in service.backend_catalog_models("codex")
if model["routeable"] is True
]
try:
async with aiohttp.ClientSession(trust_env=False) as client:
unauthorized = await client.get(f"{base_url}/v1/models")
assert unauthorized.status == 401
response = await client.get(
f"{base_url}/v1/models",
headers={"Authorization": f"Bearer {token}"},
)
assert response.status == 200
payload = await response.json()
assert [model["id"] for model in payload["data"]] == expected
assert payload["has_more"] is False
assert payload["first_id"] == expected[0]
assert payload["last_id"] == expected[-1]
finally:
await gateway.close()
asyncio.run(exercise())
def test_gateway_port_cannot_be_bound_by_another_socket(tmp_path: Path) -> None:
"""Windows SO_REUSEADDR would let any local socket share the gateway's port and its traffic."""
async def exercise() -> None:
service = _service(tmp_path, sources=[_source("src_primary01", "Primary")])
gateway = ModelHubTurnGateway(service)
base_url, _token = await gateway.endpoint(
"codex",
process_scope="/repo",
turn_id="turn_exclusive_port",
requested_model_id="shared-model",
resolved_model_id="shared-model",
source_id="src_primary01",
)
port = int(base_url.rsplit(":", 1)[1].split("/", 1)[0])
try:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as intruder:
intruder.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
with pytest.raises(OSError):
intruder.bind(("127.0.0.1", port))
finally:
await gateway.close()
asyncio.run(exercise())
def test_runtime_resolution_sees_controller_reconcile_after_remote_catalog_refresh(
monkeypatch,
tmp_path: Path,
) -> None:
from core.controller import Controller
from vibe import backend_model_catalog
async def exercise() -> None:
model_id = "gpt-runtime-refresh"
source = _source(
"src_refresh001",
"Refresh provider",
model_id=model_id,
)
service = _service(tmp_path, sources=[source])
for backend in ("claude", "codex"):
fixed_agent = service.store.config.agents[backend]
fixed_agent.routes = {
model.id: ModelHubRouteConfig() for model in fixed_agent.models
}
agent = service.store.config.agents["codex"]
snapshot = {
"codex": {
"generation": "stable-upstream-generation",
"models": [{"id": model.id} for model in agent.models],
}
}
monkeypatch.setattr(
service,
"_builtin_snapshots",
lambda _backends: snapshot,
)
await service.reconcile_builtin_models(("codex",), notify=False)
snapshot["codex"]["models"] = [
*snapshot["codex"]["models"],
{"id": model_id, "display_name": "Runtime refresh"},
]
snapshot["codex"]["generation"] = "refreshed-upstream-generation"
controller = Controller.__new__(Controller)
controller.model_hub_service = service
controller._loop = asyncio.get_running_loop()
controller._model_hub_snapshot_refresh_pending = threading.Event()
controller._model_hub_snapshot_reconcile_task = None
monkeypatch.setattr(
backend_model_catalog,
"_REMOTE_REFRESH_COMPLETED",
controller._model_hub_snapshot_refresh_completed,
)
monkeypatch.setattr(
backend_model_catalog,
"refresh_remote_catalog_now",
lambda _url: {},
)
backend_model_catalog._refresh_remote_catalog_worker()
await asyncio.sleep(0)
reconcile_task = controller._model_hub_snapshot_reconcile_task
if reconcile_task is not None:
await reconcile_task
gateway = ModelHubTurnGateway(service)
router = ModelHubRuntimeRouter(service=service, turn_gateway=gateway)
try:
launch = await router.resolve(
"codex",
model_id,
process_scope="/repo",
turn_id="turn_after_catalog_refresh",
)
finally:
await gateway.close()
assert launch.requested_model == model_id
assert launch.target_model == model_id
assert launch.source_id == source.id
assert model_id in {
model.id for model in service.store.config.agents["codex"].models
}
asyncio.run(exercise())
@pytest.mark.parametrize(
("path", "body", "status", "reason"),
[
("responses", "{", 400, "invalid_parameter"),
("responses", "[]", 400, "invalid_parameter"),
("responses", '{"stream":false}', 400, "invalid_parameter"),
("unsupported", '{"model":"shared-model"}', 404, "protocol_error"),
],
)
def test_authenticated_gateway_validation_failure_is_correlated(
tmp_path: Path,
path: str,
body: str,
status: int,
reason: str,
) -> None:
async def exercise() -> None:
service = _service(
tmp_path,
sources=[_source("src_primary01", "Primary")],
)
gateway = ModelHubTurnGateway(service)
base_url, token = await gateway.endpoint(
"codex",
process_scope="/repo",
turn_id="turn_invalid_gateway_request",
requested_model_id="shared-model",
resolved_model_id="shared-model",
source_id="src_primary01",
)
try:
async with aiohttp.ClientSession(trust_env=False) as client:
response = await client.post(
f"{base_url}/v1/{path}",
data=body,
headers={
"Authorization": f"Bearer {token}",
"Content-Type": "application/json",
},
)
assert response.status == status
await response.read()
finally:
await gateway.close()
gateway.correlation.settle(
"turn_invalid_gateway_request",
settled_by=SETTLED_BY_TERMINAL_RESULT,
ts=NOW.isoformat(),
)
record = service.provenance.get("turn_invalid_gateway_request")
assert record is not None
assert record["outcome"] == "failed_terminal"
assert record["terminal_error"] == {
"source_id": "src_primary01",
"configured_model_id": "shared-model",
"channel": "hub",
"reason": reason,
"stream_started": False,
}
_assert_valid("turn-provenance.schema.json", record)
asyncio.run(exercise())
def test_opencode_gateway_round_trips_aliased_tool_names(tmp_path: Path) -> None:
async def exercise() -> None:
handle = LiveInvokeHandle(
_outcome(RawOutcomeKind.SUCCESS, stream_started=True),
(
b'data: {"choices":[{"delta":{"tool_calls":[{"function":'
b'{"name":"avibe_todo_write","arguments":"{}"}}]}}]}\n\n',
b"data: [DONE]\n\n",
),
)
service = _service(
tmp_path,
sources=[
_source(
"src_primary01",
"Anthropic relay",
vendor="custom",
protocol="anthropic",
)
],
live_handles=[handle],
)
gateway = ModelHubTurnGateway(service)
base_url, token = await gateway.endpoint(
"opencode",
process_scope="opencode:shared-server",
)
try:
async with aiohttp.ClientSession(trust_env=False) as client:
response = await client.post(
f"{base_url}/v1/chat/completions",
json={
"model": "shared-model",
"messages": [{"role": "user", "content": "update the list"}],
"tools": [
{
"type": "function",
"function": {
"name": "todowrite",
"parameters": {"type": "object"},
},
}
],
"stream": True,
},
headers={"Authorization": f"Bearer {token}"},
)
assert response.status == 200
body = await response.read()
finally:
await gateway.close()
adapter = service.adapter
assert isinstance(adapter, ProbeAdapter)
forwarded = adapter.requests[0]
assert forwarded["tools"][0]["function"]["name"] == "avibe_todo_write"
assert b'"name":"todowrite"' in body
assert b'"name":"avibe_todo_write"' not in body
asyncio.run(exercise())
def test_gateway_terminalizer_records_pre_observer_engine_down_before_return(
tmp_path: Path,
) -> None:
async def exercise() -> None:
service = _service(
tmp_path,
sources=[_source("src_primary01", "Primary")],
)
async def fail_sync(_bindings) -> None:
raise RuntimeError("engine unavailable")
service.adapter.sync_sources = fail_sync
gateway = ModelHubTurnGateway(service)
base_url, token = await gateway.endpoint(
"codex",
process_scope="/repo",
turn_id="turn_pre_observer_engine_down",
requested_model_id="shared-model",
resolved_model_id="shared-model",
source_id="src_primary01",
)
try:
async with aiohttp.ClientSession(trust_env=False) as client:
response = await client.post(
f"{base_url}/v1/responses",
json={
"model": "shared-model",
"input": "ping",
"stream": False,
},
headers={"Authorization": f"Bearer {token}"},
)
assert response.status == 503
payload = await response.json()
assert payload["error"]["message"] == i18n_t(
"modelHub.errors.engine_down",
"en",
)
trace = gateway.correlation._traces["turn_pre_observer_engine_down"]
assert trace.pending_attempt is None
assert trace.terminal_error == {
"source_id": None,
"configured_model_id": None,
"channel": None,
"reason": "engine_down",
"stream_started": False,
}
finally:
await gateway.close()
gateway.correlation.settle(
"turn_pre_observer_engine_down",
settled_by=SETTLED_BY_TERMINAL_RESULT,
ts=NOW.isoformat(),
)
record = service.provenance.get("turn_pre_observer_engine_down")
assert record is not None
assert record["outcome"] == "failed_terminal"
assert record["terminal_error"] == {
"source_id": None,
"configured_model_id": None,
"channel": None,
"reason": "engine_down",
"stream_started": False,
}
_assert_valid("turn-provenance.schema.json", record)
asyncio.run(exercise())
@pytest.mark.parametrize("entry", ["process", "gateway"])
def test_no_candidate_provenance_carries_exact_hop_blockers_from_each_entry(
tmp_path: Path,
entry: str,
) -> None:
turn_id = f"turn_blocked_{entry}"
store = BoundedProvenanceStore(tmp_path / f"blocked-{entry}.json")
registry = TurnCorrelationRegistry(store)
blockers = (
ExactHopBlocker(
source_id="src_primary01",
model_id="shared-model",
reason="model_unsupported",
),
ExactHopBlocker(
source_id="src_backup001",
model_id="shared-model",
reason="credential_revoked",
),
)
if entry == "process":
registry.mark_no_candidate(
backend="claude",
process_scope="/repo",
turn_id=turn_id,
requested_model_id="shared-model",
supply_state="interrupted",
blockers=blockers,
)
else:
token = registry.prepare_gateway_turn(
backend="claude",
token=registry.credentials("claude", "/repo", turn_id),
requested_model_id="shared-model",
resolved_model_id="shared-model",
source_id="src_primary01",
via_mapping=False,
)
with registry.gateway_terminalizer(
backend="claude",
token=token,
) as terminalizer:
terminalizer.mark_no_candidate("interrupted", blockers)
registry.settle(
turn_id,
settled_by=SETTLED_BY_TERMINAL_RESULT,
ts=NOW.isoformat(),
)
record = store.get(turn_id)
assert record is not None
assert record["outcome"] == "no_candidate"
assert record["blockers"] == [
{
"source_id": "src_primary01",
"model_id": "shared-model",
"reason": "model_unsupported",
},
{
"source_id": "src_backup001",
"model_id": "shared-model",
"reason": "credential_revoked",
},
]
_assert_valid("turn-provenance.schema.json", record)
def test_runtime_no_candidate_projects_live_exact_hop_blockers(
tmp_path: Path,
) -> None:
source = _source("src_blocked01", "Blocked")
source.state = ModelHubSourceStateConfig(
status="needs_action",
detail_key="models.source.needs_action.credential_revoked",
)
service = _service(tmp_path, sources=[source])
gateway = ModelHubTurnGateway(service)
router = ModelHubRuntimeRouter(service=service, turn_gateway=gateway)
with pytest.raises(ModelHubError) as exc_info:
asyncio.run(
router.resolve(
"claude",
"shared-model",
process_scope="/repo",
turn_id="turn_runtime_blocked",
)
)
assert exc_info.value.supply_state == "interrupted"
router.settle_turn(
"turn_runtime_blocked",
settled_by=SETTLED_BY_TERMINAL_RESULT,
ts=NOW.isoformat(),
)
record = service.provenance.get("turn_runtime_blocked")
assert record is not None
assert record["outcome"] == "no_candidate"
assert record["blockers"] == [
{
"source_id": source.id,
"model_id": "shared-model",
"reason": "credential_revoked",
}
]
_assert_valid("turn-provenance.schema.json", record)
def test_runtime_launch_carries_claude_catalog_capabilities(tmp_path: Path) -> None:
source = _source("src_catalogcap", "Catalog capability")
service = _service(tmp_path, sources=[source])
agent = service.store.config.agents["claude"]
agent.models = [
ModelHubBackendModelConfig(
id="shared-model",
context_window=128_000,
max_output_tokens=32_000,
supports_tools=True,
supports_reasoning=True,
reasoning_efforts=["low", "high"],
)
]
agent.routes = {
"shared-model": ModelHubRouteConfig(
hops=(ModelHubRouteHopConfig(source.id, "shared-model"),)
)
}
gateway = SimpleNamespace(
endpoint=AsyncMock(
return_value=("http://127.0.0.1:19000/claude", "gateway-token")
)
)
router = ModelHubRuntimeRouter(service=service, turn_gateway=gateway)
launch = asyncio.run(router.resolve("claude", "shared-model"))
assert launch.context_window == 128_000
assert launch.max_output_tokens == 32_000
assert launch.supports_tools is True
assert launch.supports_reasoning is True
assert launch.reasoning_efforts == ("low", "high")
def test_native_cli_launch_carries_claude_catalog_capabilities(tmp_path: Path) -> None:
source = _source(
"src_nativecatalog",
"Native catalog capability",
channel="native_cli",
vendor="anthropic",
protocol="anthropic",
)
service = _service(tmp_path, sources=[source])
agent = service.store.config.agents["claude"]
agent.models = [
ModelHubBackendModelConfig(
id="shared-model",
context_window=128_000,
max_output_tokens=32_000,
supports_tools=True,
supports_reasoning=True,
reasoning_efforts=["low", "max"],
)
]
agent.routes = {
"shared-model": ModelHubRouteConfig(
hops=(ModelHubRouteHopConfig(source.id, "shared-model"),)
)
}
router = ModelHubRuntimeRouter(
service=service,
native_cli_ready=lambda _backend: True,
)
launch = asyncio.run(router.resolve("claude", "shared-model"))
assert launch.channel == "native_cli"
assert launch.context_window == 128_000
assert launch.max_output_tokens == 32_000
assert launch.supports_tools is True
assert launch.supports_reasoning is True
assert launch.reasoning_efforts == ("low", "max")
def test_runtime_launch_suppresses_efforts_when_reasoning_is_disabled(
tmp_path: Path,
) -> None:
source = _source("src_noreasoning", "No reasoning")
service = _service(tmp_path, sources=[source])
agent = service.store.config.agents["claude"]
agent.models = [
ModelHubBackendModelConfig(
id="shared-model",
supports_reasoning=False,
reasoning_efforts=["low", "high"],
)
]
agent.routes = {
"shared-model": ModelHubRouteConfig(
hops=(ModelHubRouteHopConfig(source.id, "shared-model"),)
)
}
router = ModelHubRuntimeRouter(
service=service,
turn_gateway=SimpleNamespace(
endpoint=AsyncMock(
return_value=("http://127.0.0.1:19000/claude", "gateway-token")
)
),
)
launch = asyncio.run(router.resolve("claude", "shared-model"))
assert launch.supports_reasoning is False
assert launch.reasoning_efforts == ()
def test_runtime_no_candidate_reinspects_the_full_chain_for_terminal_facts(
tmp_path: Path,
) -> None:
hub = _source(
"src_precheckhub",
"Hub cooling",
status="cooldown",
retry_at=(NOW + timedelta(hours=1)).isoformat(),
)
native = _source("src_prechecknative", "Native ready", channel="native_cli")
service = _service(tmp_path, sources=[hub, native])
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
service.store.config.agents["codex"].routes[requested_model] = ModelHubRouteConfig(
hops=(
ModelHubRouteHopConfig(hub.id, "shared-model"),
ModelHubRouteHopConfig(native.id, "shared-model"),
)
)
with pytest.raises(ModelHubError) as raised:
asyncio.run(
service.resolve(
backend="codex",
model_id=requested_model,
request=ModelHubRequest(
{"model": requested_model, "input": "ping"},
protocol="openai_responses",
),
supply_channel="hub",
)
)
projection = raised.value.turn_outcome
assert projection is not None
assert projection.supply_facts is not None
assert projection.supply_facts.supply_state == "waiting"
assert raised.value.blockers == exact_hop_blockers(
service._inspect_terminal_chain(backend="codex", model_id=requested_model)[1]
)
def test_gateway_no_candidate_projects_live_exact_hop_blockers(
tmp_path: Path,
) -> None:
async def exercise() -> None:
source = _source("src_blocked02", "Blocked")
source.state = ModelHubSourceStateConfig(
status="needs_action",
detail_key="models.source.needs_action.credential_revoked",
)
service = _service(tmp_path, sources=[source])
service.store.config.agents["codex"].routes["shared-model"] = ModelHubRouteConfig(
hops=(ModelHubRouteHopConfig(source.id, "removed-model"),)
)
gateway = ModelHubTurnGateway(service)
base_url, token = await gateway.endpoint(
"codex",
process_scope="/repo",
turn_id="turn_gateway_blocked",
requested_model_id="shared-model",
resolved_model_id="removed-model",
source_id=source.id,
)
try:
async with aiohttp.ClientSession(trust_env=False) as client:
response = await client.post(
f"{base_url}/v1/responses",
json={"model": "removed-model", "input": "ping", "stream": False},
headers={"Authorization": f"Bearer {token}"},
)
assert response.status == 409
await response.read()
finally:
await gateway.close()
gateway.correlation.settle(
"turn_gateway_blocked",
settled_by=SETTLED_BY_TERMINAL_RESULT,
ts=NOW.isoformat(),
)
record = service.provenance.get("turn_gateway_blocked")
assert record is not None
assert record["outcome"] == "no_candidate"
assert record["blockers"] == [
{
"source_id": source.id,
"model_id": "removed-model",
"reason": "credential_revoked",
}
]
_assert_valid("turn-provenance.schema.json", record)
asyncio.run(exercise())
def test_gateway_preserves_exhausted_provenance_after_all_hops_fallback(
tmp_path: Path,
) -> None:
async def exercise() -> None:
first = _source("src_primary01", "Primary")
second = _source("src_backup001", "Backup")
service = _service(
tmp_path,
sources=[first, second],
outcomes=[
_outcome(
RawOutcomeKind.HTTP_ERROR,
status=429,
source_id=first.id,
),
_outcome(
RawOutcomeKind.HTTP_ERROR,
status=503,
source_id=second.id,
),
],
)
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
service.store.config.agents["codex"].routes[requested_model] = ModelHubRouteConfig(
hops=(
ModelHubRouteHopConfig(first.id, "shared-model"),
ModelHubRouteHopConfig(second.id, "shared-model"),
)
)
gateway = ModelHubTurnGateway(service)
base_url, token = await gateway.endpoint(
"codex",
process_scope="/repo",
turn_id="turn_gateway_exhausted",
requested_model_id=requested_model,
resolved_model_id="shared-model",
source_id=first.id,
)
try:
async with aiohttp.ClientSession(trust_env=False) as client:
response = await client.post(
f"{base_url}/v1/responses",
json={"model": "shared-model", "input": "ping", "stream": False},
headers={"Authorization": f"Bearer {token}"},
)
assert response.status == 503
payload = await response.json()
assert payload["error"]["code"] == "mapping_target_unavailable"
cooled = service.store.load().sources
assert payload["error"]["message"] == i18n_t(
"modelHub.launch.waiting",
"en",
model=requested_model,
source=", ".join(source.display_name for source in cooled),
retry_at=min(source.state.retry_at or "" for source in cooled),
)
finally:
await gateway.close()
gateway.correlation.settle(
"turn_gateway_exhausted",
settled_by=SETTLED_BY_TERMINAL_RESULT,
ts=NOW.isoformat(),
)
record = service.provenance.get("turn_gateway_exhausted")
assert record is not None
assert record["outcome"] == "exhausted"
assert record["terminal_error"] is None
assert [attempt["source_id"] for attempt in record["failed_attempts"]] == [
first.id,
second.id,
]
_assert_valid("turn-provenance.schema.json", record)
asyncio.run(exercise())
@pytest.mark.parametrize("already_cooling", [False, True])
@pytest.mark.parametrize("ending", ["recover", "fail", "config_change", "cancel", "extend_recover", "admitted_cancel"])
@pytest.mark.parametrize(
("backend", "endpoint", "protocol"),
[
("codex", "responses", "openai_responses"),
("claude", "messages", "anthropic"),
("opencode", "chat/completions", "openai_chat"),
],
)
def test_gateway_cooldown_uses_one_cancellable_recovery_window(
tmp_path: Path,
already_cooling: bool,
ending: str,
backend: str,
endpoint: str,
protocol: str,
) -> None:
"""MH-RUNTIME-009: one pending request owns all bounded recovery admission."""
async def exercise() -> None:
source = _source(
"src_retrycool01",
"Recovering",
status="cooldown" if already_cooling else "standby",
retry_at=(NOW + timedelta(seconds=30)).isoformat().replace("+00:00", "Z") if already_cooling else None,
vendor="anthropic" if backend == "claude" else "openai",
protocol=protocol,
)
failure = _outcome(RawOutcomeKind.HTTP_ERROR, status=503, source_id=source.id)
final = failure if ending == "fail" else _outcome(RawOutcomeKind.SUCCESS, source_id=source.id)
service = _service(
tmp_path,
sources=[source],
outcomes=([failure] if not already_cooling else []) + ([final] * 4 if ending == "fail" else [final]),
recovery_window_seconds=120,
)
fixed = _canonicalize_fixed_test_routes(service)
model = fixed[backend] if backend in fixed else "shared-model"
payload = {"model": "shared-model", "stream": False}
if endpoint == "responses":
payload["input"] = "ping"
else:
payload["messages"] = [{"role": "user", "content": "ping"}]
clock = {"now": NOW}
service.now = lambda: clock["now"]
gateway = ModelHubTurnGateway(service, now=lambda: clock["now"])
waiting = asyncio.Event()
release = asyncio.Event()
wait_ended = asyncio.Event()
admitted = asyncio.Event()
upstream_canceled = asyncio.Event()
delays = []
async def wait_for_recovery(delay: float) -> None:
delays.append(delay)
waiting.set()
try:
await release.wait()
clock["now"] += timedelta(seconds=delay)
finally:
wait_ended.set()
async def invoke_until_stopped(source_id, model_id, request, stream, origin, *, on_admitted=None):
on_admitted()
service.adapter.invocations.append((source_id, model_id, origin))
admitted.set()
try:
await asyncio.Future()
finally:
upstream_canceled.set()
base_url, token = await gateway.endpoint(
backend,
process_scope="/repo",
turn_id="turn_cooldown_retry",
requested_model_id=model,
resolved_model_id="shared-model",
source_id=source.id,
)
try:
with patch.object(service.recovery, "sleep", side_effect=wait_for_recovery):
async with aiohttp.ClientSession(trust_env=False) as client:
request = asyncio.create_task(client.post(
f"{base_url}/v1/{endpoint}",
json=payload,
headers={"Authorization": f"Bearer {token}"},
))
try:
await asyncio.wait_for(waiting.wait(), timeout=2)
assert not request.done()
assert len(service.adapter.invocations) == (0 if already_cooling else 1)
if ending == "cancel":
request.cancel()
with pytest.raises(asyncio.CancelledError):
await request
await asyncio.wait_for(wait_ended.wait(), timeout=2)
elif ending == "admitted_cancel":
service.adapter.invoke = invoke_until_stopped
release.set()
await asyncio.wait_for(admitted.wait(), timeout=2)
gateway.correlation.close_turn_admission(
"turn_cooldown_retry", settled_by=SETTLED_BY_STOPPED,
)
request.cancel()
with pytest.raises(asyncio.CancelledError):
await request
await asyncio.wait_for(upstream_canceled.wait(), timeout=2)
else:
if ending == "config_change":
service.store.config.sources[0].state = ModelHubSourceStateConfig(
status="needs_action",
detail_key="models.source.needs_action.credential_revoked",
)
elif ending == "extend_recover":
service.store.config.sources[0].state.retry_at = (
NOW + timedelta(seconds=60)
).isoformat().replace("+00:00", "Z")
release.set()
response = await asyncio.wait_for(request, timeout=2)
await response.read()
assert response.status == {
"recover": 200, "fail": 400 if backend == "codex" else 424,
"config_change": 409, "extend_recover": 200,
}[ending]
if ending == "fail":
assert await response.json() == {
"type": "error",
"error": {
"type": "model_hub_recovery_exhausted",
"code": "model_hub_recovery_exhausted",
"message": "Automatic recovery has ended. Try again or choose another model.",
"cause": "server_error",
},
}
assert "Retry-After" not in response.headers
assert delays == (
[30.0, 30.0 if already_cooling else 60.0] if ending == "fail"
else [30.0, 30.0] if ending == "extend_recover" else [30.0]
)
expected = (0 if already_cooling else 1) + (
ending in {"recover", "fail", "extend_recover", "admitted_cancel"}
) + (ending == "fail")
assert len(service.adapter.invocations) == expected
finally:
if not request.done():
request.cancel()
await asyncio.gather(request, return_exceptions=True)
finally:
release.set()
await gateway.close()
if ending in {"extend_recover", "admitted_cancel"} and backend != "opencode":
gateway.correlation.settle(
"turn_cooldown_retry",
settled_by=SETTLED_BY_STOPPED if ending == "admitted_cancel" else SETTLED_BY_TERMINAL_RESULT,
)
record = service.provenance.get("turn_cooldown_retry")
assert record["outcome"] == ("canceled" if ending == "admitted_cancel" else "served")
assert record["model_supply_state"] is None
assert record["blockers"] == []
assert len(record["failed_attempts"]) == (0 if already_cooling else 1)
if ending == "admitted_cancel":
assert record["canceled_attempt"]["source_id"] == source.id
asyncio.run(exercise())
@pytest.mark.parametrize("suffix", ["Z", "+00:00", ""])
def test_gateway_parses_supported_cooldown_timestamps_on_python310(tmp_path, suffix):
class Python310Datetime(datetime):
@classmethod
def fromisoformat(cls, value):
if value.endswith("Z"):
raise ValueError("Python 3.10 requires a numeric UTC offset")
return super().fromisoformat(value)
retry_at = (NOW + timedelta(seconds=30)).replace(tzinfo=None).isoformat() + suffix
source = _source(
"src_timestamp01", "Cooling", status="cooldown", retry_at=retry_at,
)
service = _service(tmp_path, sources=[source])
model = _canonicalize_fixed_test_routes(service)["codex"]
gateway = ModelHubTurnGateway(service, now=lambda: NOW)
with patch("core.handlers.model_hub.resolver.datetime", Python310Datetime):
with pytest.raises(ModelHubError) as failed:
asyncio.run(service.resolve(backend="codex", model_id=model, request={}, supply_channel="hub"))
assert gateway._cooldown_retry_delay(failed.value.turn_outcome) == 30
@pytest.mark.parametrize("reverse_route", [False, True])
@pytest.mark.parametrize(
("early", "late", "delay"),
[
("2026-07-29T17:00:30.125+05:00", "2026-07-29T12:01:00Z", 30.125),
("2026-07-29T12:00:30Z", "2026-07-29T05:01:00-07:00", 30),
("2026-07-29T12:00:30Z", "2026-07-29T12:00:30.125Z", 30),
],
)
def test_cooldown_selection_uses_instants_through_probe_and_gateway(tmp_path, reverse_route, early, late, delay):
async def exercise():
first = _source("src_earliest01", "First recovery", status="cooldown", retry_at=early)
second = _source("src_later0001", "Later recovery", status="cooldown", retry_at=late)
service = _service(
tmp_path,
sources=[second, first] if reverse_route else [first, second],
outcomes=[_outcome(RawOutcomeKind.SUCCESS, source_id=first.id)],
)
model = _canonicalize_fixed_test_routes(service)["codex"]
clock = {"now": NOW}
service.now = lambda: clock["now"]
with pytest.raises(ModelHubError) as probe:
await service.probe_agent("codex", model)
assert probe.value.code == "probe_no_candidate"
expected_retry_at = (NOW + timedelta(seconds=delay)).isoformat()
assert probe.value.data["supply"]["retry_at"] == expected_retry_at
with pytest.raises(ModelHubError) as failed:
await service.resolve(backend="codex", model_id=model, request={}, supply_channel="hub")
assert failed.value.turn_outcome.supply_facts.retry_at == expected_retry_at
gateway = ModelHubTurnGateway(service, now=lambda: clock["now"])
assert gateway._cooldown_retry_delay(failed.value.turn_outcome) == delay
delays = []
async def wait_for_recovery(seconds):
delays.append(seconds)
clock["now"] += timedelta(seconds=seconds)
base_url, token = await gateway.endpoint(
"codex", process_scope="/repo", turn_id="turn_earliest_recovery",
requested_model_id=model, resolved_model_id="shared-model", source_id=first.id,
)
try:
service.recovery.window_seconds = 120
with patch.object(service.recovery, "sleep", side_effect=wait_for_recovery):
async with aiohttp.ClientSession(trust_env=False) as client:
response = await client.post(
f"{base_url}/v1/responses",
json={"model": "shared-model", "input": "ping", "stream": False},
headers={"Authorization": f"Bearer {token}"},
)
await response.read()
assert response.status == 200
assert delays == [delay]
assert service.adapter.invocations == [(first.id, "shared-model", "codex")]
assert service.store.load().sources[0 if reverse_route else 1].state.retry_at == late
finally:
await gateway.close()
asyncio.run(exercise())
def test_gateway_exhaustion_uses_no_time_copy_when_an_earlier_hop_recovers(
tmp_path: Path,
) -> None:
async def exercise() -> None:
first = _source("src_recovery01", "Recovered while waiting")
second = _source("src_recovery02", "Still cooling")
service = _service(
tmp_path,
sources=[first, second],
outcomes=[
_outcome(
RawOutcomeKind.HTTP_ERROR,
status=429,
source_id=first.id,
),
_outcome(
RawOutcomeKind.HTTP_ERROR,
status=503,
source_id=second.id,
),
],
)
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
service.store.config.agents["codex"].routes[requested_model] = (
ModelHubRouteConfig(
hops=(
ModelHubRouteHopConfig(first.id, "shared-model"),
ModelHubRouteHopConfig(second.id, "shared-model"),
)
)
)
clock = {"now": NOW}
service.now = lambda: clock["now"]
invoke = service.adapter.invoke
async def advance_during_later_attempt(
source_id,
model_id,
request,
stream,
origin,
*,
on_admitted=None,
):
if source_id == second.id:
clock["now"] = NOW + timedelta(minutes=10)
return await invoke(source_id, model_id, request, stream, origin, on_admitted=on_admitted)
service.adapter.invoke = advance_during_later_attempt
gateway = ModelHubTurnGateway(service)
base_url, token = await gateway.endpoint(
"codex",
process_scope="/repo",
turn_id="turn_exhausted_after_recovery",
requested_model_id=requested_model,
resolved_model_id="shared-model",
source_id=first.id,
)
try:
async with aiohttp.ClientSession(trust_env=False) as client:
response = await client.post(
f"{base_url}/v1/responses",
json={"model": "shared-model", "input": "ping", "stream": False},
headers={"Authorization": f"Bearer {token}"},
)
assert response.status == 503
payload = await response.json()
assert payload["error"]["message"] == i18n_t(
"modelHub.launch.waiting_without_retry",
"en",
model=requested_model,
)
finally:
await gateway.close()
asyncio.run(exercise())
def test_gateway_streamed_fallback_settles_source_before_rendering_next_current(
tmp_path: Path,
) -> None:
async def exercise() -> None:
first = _source("src_primary01", "Primary")
second = _source("src_backup001", "Backup")
service = _service(
tmp_path,
sources=[first, second],
outcomes=[
_outcome(
RawOutcomeKind.TIMEOUT,
status=200,
source_id=first.id,
stream_started=True,
),
],
)
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
service.store.config.agents["codex"].routes[requested_model] = (
ModelHubRouteConfig(
hops=(
ModelHubRouteHopConfig(first.id, "shared-model"),
ModelHubRouteHopConfig(second.id, "shared-model"),
)
)
)
gateway = ModelHubTurnGateway(service)
base_url, token = await gateway.endpoint(
"codex",
process_scope="/repo",
turn_id="turn_gateway_streamed_fallback",
requested_model_id=requested_model,
resolved_model_id="shared-model",
source_id=first.id,
)
try:
async with aiohttp.ClientSession(trust_env=False) as client:
response = await client.post(
f"{base_url}/v1/responses",
json={"model": "shared-model", "input": "ping", "stream": False},
headers={"Authorization": f"Bearer {token}"},
)
assert response.status == 502
payload = await response.json()
assert payload["error"] == {
"type": "stream_interrupted",
"code": "stream_interrupted",
"message": i18n_t("modelHub.errors.stream_interrupted", "en"),
}
finally:
await gateway.close()
assert service.adapter.invocations == [
(first.id, "shared-model", "codex")
]
persisted = {source.id: source for source in service.store.load().sources}
assert persisted[first.id].state.status == "standby"
assert persisted[second.id].state.status == "standby"
assert service.agent_chain("codex", requested_model)["current"] == {
"source_id": first.id,
"model_id": "shared-model",
}
gateway.correlation.settle(
"turn_gateway_streamed_fallback",
settled_by=SETTLED_BY_TERMINAL_RESULT,
ts=NOW.isoformat(),
)
record = service.provenance.get("turn_gateway_streamed_fallback")
assert record is not None
assert record["outcome"] == "failed_terminal"
assert record["terminal_error"] == {
"source_id": first.id,
"configured_model_id": "shared-model",
"channel": "hub",
"reason": "stream_interrupted",
"stream_started": True,
"http_status": 200,
"upstream_error_code": None,
}
_assert_valid("turn-provenance.schema.json", record)
asyncio.run(exercise())
def test_gateway_live_settlement_emits_matrix_copy_before_eof(
tmp_path: Path,
) -> None:
async def exercise() -> None:
source = _source("src_streamcopy1", "Stream copy")
backup = _source("src_streamcopy2", "Stream copy backup")
service = _service(
tmp_path,
sources=[source, backup],
live_handles=[
LiveInvokeHandle(
_outcome(
RawOutcomeKind.TIMEOUT,
status=200,
source_id=source.id,
stream_started=True,
),
(
b'data: {"type":"response.output_text.delta",'
b'"sequence_number":7}\n\n',
b"data: partial",
),
)
],
)
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
response = FakeStreamResponse()
gateway = ModelHubTurnGateway(service)
request = _prepared_gateway_request(
gateway,
turn_id="turn_stream_copy",
requested_model=requested_model,
source_id=source.id,
stream=True,
)
with patch(
"core.handlers.model_hub.turn_gateway.web.StreamResponse",
return_value=response,
):
result = await gateway._handle_request(request)
assert result is response
assert response.writes[:2] == [
b'data: {"type":"response.output_text.delta",'
b'"sequence_number":7}\n\n',
b"data: partial",
]
terminal = response.writes[-1]
assert terminal.startswith(b"\ndata: {}\n\nevent: error\ndata: ")
payload = json.loads(
terminal.removeprefix(b"\ndata: {}\n\nevent: error\ndata: ")
)
assert payload["sequence_number"] == 8
assert payload["code"] == "modelHub.errors.stream_interrupted"
assert response.eof_called
asyncio.run(exercise())
@pytest.mark.parametrize(
"case",
TERMINAL_SETTLEMENT_BOUNDARIES,
ids=lambda case: f"{case['forwarded_terminal']}-{case['settlement']}",
)
def test_gateway_emits_at_most_one_wire_terminal(case: dict[str, object]) -> None:
async def exercise() -> None:
gateway = ModelHubTurnGateway(SimpleNamespace()) # type: ignore[arg-type]
response = FakeStreamResponse()
outcomes = {
"none": None,
"silent": _RenderedTurnOutcome(None, None),
"copy": _RenderedTurnOutcome(
"modelHub.errors.engine_down",
i18n_t("modelHub.errors.engine_down", "en"),
),
}
forwarded = case["forwarded_terminal"]
await gateway._write_stream_terminal_copy(
response, # type: ignore[arg-type]
"openai_responses",
outcomes[str(case["settlement"])],
ProtocolSSEState("openai_responses"),
forwarded_terminal=cast(str | None, forwarded), # type: ignore[arg-type]
)
assert bool(response.writes) is case["write_terminal"]
asyncio.run(exercise())
@pytest.mark.parametrize(
("stream_started", "expected_key", "expected_next_current"),
[
(True, "modelHub.launch.retry", True),
(False, "modelHub.launch.waiting_without_retry", False),
],
)
def test_terminal_reinspection_uses_every_channel_in_a_mixed_chain(
tmp_path: Path,
stream_started: bool,
expected_key: str,
expected_next_current: bool,
) -> None:
async def exercise() -> None:
hub = _source("src_mixedhub1", "Hub source")
native = _source(
"src_mixednative1",
"Native source",
channel="native_cli",
)
service = _service(
tmp_path,
sources=[hub, native],
outcomes=[
_outcome(
RawOutcomeKind.HTTP_ERROR,
status=429,
code="rate_limit_error",
source_id=hub.id,
stream_started=stream_started,
)
],
)
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
service.store.config.agents["codex"].routes[requested_model] = (
ModelHubRouteConfig(
hops=(
ModelHubRouteHopConfig(hub.id, "shared-model"),
ModelHubRouteHopConfig(native.id, "shared-model"),
)
)
)
with pytest.raises(ModelHubError) as raised:
await service.resolve(
backend="codex",
model_id=requested_model,
request=ModelHubRequest(
{"model": requested_model, "input": "ping"},
protocol="openai_responses",
),
supply_channel="hub",
)
projection = raised.value.turn_outcome
assert projection is not None
assert projection.next_current_changed is expected_next_current
assert project_turn_outcome_copy(projection).key == expected_key
assert service.adapter.invocations == [
(hub.id, "shared-model", "codex")
]
assert service.agent_chain("codex", requested_model)["current"] == {
"source_id": native.id,
"model_id": "shared-model",
}
asyncio.run(exercise())
def test_preoutput_network_fallback_does_not_mutate_source_health(
tmp_path: Path,
) -> None:
async def exercise() -> None:
first = _source("src_network001", "Network source")
second = _source("src_network002", "Backup source")
service = _service(
tmp_path,
sources=[first, second],
outcomes=[
_outcome(
RawOutcomeKind.NETWORK_ERROR,
source_id=first.id,
),
_outcome(RawOutcomeKind.SUCCESS, source_id=second.id),
],
)
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
resolved = await service.resolve(
backend="codex",
model_id=requested_model,
request=ModelHubRequest(
{"model": requested_model, "input": "ping"},
protocol="openai_responses",
),
supply_channel="hub",
)
assert resolved.source_id == second.id
assert [source.state.status for source in service.store.load().sources] == [
"standby",
"standby",
]
assert service.adapter.invocations == [
(first.id, "shared-model", "codex"),
(second.id, "shared-model", "codex"),
]
asyncio.run(exercise())
@pytest.mark.parametrize("stream", [False, True])
@pytest.mark.parametrize(
("kind", "status", "code", "expected_outcome", "expected_source_status"),
[
(RawOutcomeKind.SUCCESS, 200, None, "served", "standby"),
(
RawOutcomeKind.PROTOCOL_ERROR,
502,
"upstream_protocol_error",
"failed_terminal",
"standby",
),
(
RawOutcomeKind.HTTP_ERROR,
200,
"permission_error",
"failed_terminal",
"standby",
),
(RawOutcomeKind.TIMEOUT, 200, None, "failed_terminal", "standby"),
],
)
def test_gateway_handle_terminal_matrix_always_uses_service_settlement(
tmp_path: Path,
stream: bool,
kind: RawOutcomeKind,
status: int,
code: str | None,
expected_outcome: str,
expected_source_status: str,
) -> None:
async def exercise() -> None:
source = _source("src_livehandle", "Live handle")
outcome = _outcome(
kind,
status=status,
code=code,
source_id=source.id,
stream_started=True,
)
chunks = (
(b"data: {}\n\n",)
if stream
else (b'{"id":"response-live-handle"}',)
)
service = _service(
tmp_path,
sources=[source],
live_handles=[LiveInvokeHandle(outcome, chunks)],
)
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
service.settle_handle_outcome = AsyncMock(
wraps=service.settle_handle_outcome
)
turn_id = f"turn_live_{kind.value}_{stream}"
gateway = ModelHubTurnGateway(service)
base_url, token = await gateway.endpoint(
"codex",
process_scope="/repo",
turn_id=turn_id,
requested_model_id=requested_model,
resolved_model_id="shared-model",
source_id=source.id,
)
try:
async with aiohttp.ClientSession(trust_env=False) as client:
response = await client.post(
f"{base_url}/v1/responses",
json={
"model": "shared-model",
"input": "ping",
"stream": stream,
},
headers={"Authorization": f"Bearer {token}"},
)
payload = await response.read()
if stream or kind is RawOutcomeKind.SUCCESS:
assert response.status == 200
elif code == "permission_error":
assert response.status == 403
else:
assert response.status == 502
if not stream and kind is RawOutcomeKind.PROTOCOL_ERROR:
assert json.loads(payload)["error"]["message"] == i18n_t(
"modelHub.errors.upstream_protocol_error",
"en",
)
assert b"request_incompatible" not in payload
if stream and kind is RawOutcomeKind.PROTOCOL_ERROR:
assert b"event: error" in payload
assert b"upstream_protocol_error" in payload
finally:
await gateway.close()
service.settle_handle_outcome.assert_awaited_once()
assert (
service.settle_handle_outcome.await_args.kwargs["termination_origin"]
== "upstream_terminal"
)
assert service.store.load().sources[0].state.status == expected_source_status
gateway.correlation.settle(
turn_id,
settled_by=SETTLED_BY_TERMINAL_RESULT,
ts=NOW.isoformat(),
)
record = service.provenance.get(turn_id)
assert record is not None
assert record["outcome"] == expected_outcome
_assert_valid("turn-provenance.schema.json", record)
asyncio.run(exercise())
def test_handle_settlement_requires_an_explicit_termination_origin(
tmp_path: Path,
) -> None:
async def exercise() -> None:
source = _source("src_origin01", "Explicit origin")
service = _service(tmp_path, sources=[source])
outcome = _outcome(
RawOutcomeKind.NETWORK_ERROR,
source_id=source.id,
)
resolved = ResolvedInvocation(
backend="codex",
requested_model_id="shared-model",
source_id=source.id,
source_label=source.display_name,
model_id="shared-model",
handle=None,
outcome=None,
)
with pytest.raises(TypeError, match="termination_origin"):
await service.settle_handle_outcome(resolved, outcome) # type: ignore[call-arg]
record_attempt = Mock()
settlement = await service.settle_handle_outcome(
resolved,
None,
termination_origin="downstream_cancel",
record_attempt=record_attempt,
)
assert settlement.outcome is None
assert settlement.decision is None
assert settlement.turn_outcome == produce_turn_outcome("turn.canceled")
assert render_turn_outcome_copy(settlement.turn_outcome, "en") is None
assert service.store.load().sources[0].state.status == "standby"
record_attempt.assert_not_called()
asyncio.run(exercise())
def test_handle_settlement_records_history_before_source_transition_event(
tmp_path: Path,
) -> None:
async def exercise() -> None:
source = _source("src_ordered1", "Ordered settlement")
service = _service(tmp_path, sources=[source])
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
settlement_generation = service._reserve_settlement_generation(source.id)
resolved = ResolvedInvocation(
backend="codex",
requested_model_id=requested_model,
source_id=source.id,
source_label=source.display_name,
model_id="shared-model",
handle=None,
outcome=None,
settlement_generation=settlement_generation,
)
outcome = _outcome(
RawOutcomeKind.HTTP_ERROR,
status=429,
code="rate_limit_error",
source_id=source.id,
stream_started=True,
)
order: list[str] = []
record_event = service._record_event
settle_source = service._settle_fallback_source
def ordered_event(**kwargs) -> None:
order.append("event")
record_event(**kwargs)
async def ordered_source_mutation(*args, **kwargs):
order.append("source_mutation")
return await settle_source(*args, **kwargs)
service._record_event = Mock(side_effect=ordered_event)
service._settle_fallback_source = AsyncMock(
side_effect=ordered_source_mutation
)
await service.settle_handle_outcome(
resolved,
outcome,
termination_origin="upstream_terminal",
record_attempt=lambda _outcome, _decision: order.append("provenance"),
)
assert order == ["provenance", "source_mutation", "event"]
asyncio.run(exercise())
def test_engine_down_settlement_does_not_mutate_source(
tmp_path: Path,
) -> None:
async def exercise() -> None:
source = _source("src_enginedown1", "Engine unavailable")
service = _service(tmp_path, sources=[source])
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
resolved = ResolvedInvocation(
backend="codex",
requested_model_id=requested_model,
source_id=source.id,
source_label=source.display_name,
model_id="shared-model",
handle=None,
outcome=None,
)
outcome = _outcome(
RawOutcomeKind.NETWORK_ERROR,
status=200,
code="engine_down",
message="loopback transport failed",
source_id=source.id,
stream_started=True,
)
record_attempt = Mock()
settlement = await service.settle_handle_outcome(
resolved,
outcome,
termination_origin="upstream_terminal",
record_attempt=record_attempt,
)
assert settlement.decision is not None
assert settlement.decision.error_code == "engine_down"
assert settlement.decision.reason is None
assert settlement.turn_outcome == produce_turn_outcome(
"turn.engine_down",
stream_started=True,
)
assert service.store.load().sources[0].state.status == "standby"
assert service.events.list() == []
record_attempt.assert_called_once()
asyncio.run(exercise())
def test_bare_stream_network_settlement_keeps_source_health_unchanged(
tmp_path: Path,
) -> None:
async def exercise() -> None:
source = _source("src_barenetwork1", "Incomplete stream")
service = _service(tmp_path, sources=[source])
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
resolved = ResolvedInvocation(
backend="codex",
requested_model_id=requested_model,
source_id=source.id,
source_label=source.display_name,
model_id="shared-model",
handle=None,
outcome=None,
)
outcome = _outcome(
RawOutcomeKind.NETWORK_ERROR,
status=200,
source_id=source.id,
stream_started=True,
)
record_attempt = Mock()
settlement = await service.settle_handle_outcome(
resolved,
outcome,
termination_origin="upstream_terminal",
record_attempt=record_attempt,
)
assert settlement.decision is not None
assert settlement.decision.reason == "network"
assert service.store.load().sources[0].state.status == "standby"
assert settlement.turn_outcome is None
assert service.events.list() == []
record_attempt.assert_called_once()
asyncio.run(exercise())
@pytest.mark.parametrize("refreshable", (True, False))
def test_streamed_auth_settlement_uses_exact_credential_capability(
tmp_path: Path,
refreshable: bool,
) -> None:
async def exercise() -> None:
source = _source("src_streamauth1", "Stream auth")
service = _service(tmp_path, sources=[source])
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
assert source.credential_ref is not None
if refreshable:
service.adapter.refreshable_credential_refs.add(source.credential_ref)
settlement_generation = service._reserve_settlement_generation(source.id)
resolved = ResolvedInvocation(
backend="codex",
requested_model_id=requested_model,
source_id=source.id,
source_label=source.display_name,
model_id="shared-model",
handle=None,
outcome=None,
credential_ref=source.credential_ref,
settlement_generation=settlement_generation,
)
outcome = _outcome(
RawOutcomeKind.HTTP_ERROR,
status=401,
code="authentication_error",
source_id=source.id,
stream_started=True,
)
record_attempt = Mock()
settlement = await service.settle_handle_outcome(
resolved,
outcome,
termination_origin="upstream_terminal",
record_attempt=record_attempt,
)
assert settlement.decision is not None
assert settlement.decision.action == "surface"
assert settlement.decision.error_code == "stream_interrupted"
persisted = service.store.load().sources[0]
if refreshable:
assert settlement.decision.reason is None
assert settlement.turn_outcome is None
assert persisted.state.status == "standby"
else:
assert settlement.decision.reason == "credential_revoked"
assert settlement.turn_outcome is not None
assert persisted.state.status == "needs_action"
assert service.adapter.capability_queries == [source.credential_ref]
record_attempt.assert_called_once()
asyncio.run(exercise())
def test_streamed_fallback_reports_an_unpersisted_recovery_transition_honestly(
tmp_path: Path,
) -> None:
async def exercise() -> None:
source = _source("src_recoverytx1", "Recovery warning")
service = _service(tmp_path, sources=[source])
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
service.store.recovery_warning = True
resolved = ResolvedInvocation(
backend="codex",
requested_model_id=requested_model,
source_id=source.id,
source_label=source.display_name,
model_id="shared-model",
handle=None,
outcome=None,
credential_ref=source.credential_ref,
)
outcome = _outcome(
RawOutcomeKind.HTTP_ERROR,
status=429,
code="rate_limit_error",
source_id=source.id,
stream_started=True,
)
record_attempt = Mock()
settlement = await service.settle_handle_outcome(
resolved,
outcome,
termination_origin="upstream_terminal",
record_attempt=record_attempt,
)
projection = settlement.turn_outcome
assert projection is not None
assert projection.source_transition_persisted is False
assert projection.next_current_changed is False
assert projection.supply_facts is None
assert project_turn_outcome_copy(projection).key == (
"modelHub.errors.stream_interrupted"
)
assert service.store.config.sources[0].state.status == "standby"
assert service.events.list() == []
record_attempt.assert_called_once()
asyncio.run(exercise())
def test_streamed_blocker_rejected_by_generation_reports_unpersisted(
tmp_path: Path,
) -> None:
async def exercise() -> None:
source = _source("src_stalegen01", "Stale settlement")
service = _service(tmp_path, sources=[source])
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
older_generation = service._reserve_settlement_generation(source.id)
service._reserve_settlement_generation(source.id)
resolved = ResolvedInvocation(
backend="codex",
requested_model_id=requested_model,
source_id=source.id,
source_label=source.display_name,
model_id="shared-model",
handle=None,
outcome=None,
credential_ref=source.credential_ref,
settlement_generation=older_generation,
)
outcome = _outcome(
RawOutcomeKind.HTTP_ERROR,
status=401,
code="authentication_error",
source_id=source.id,
stream_started=True,
)
settlement = await service.settle_handle_outcome(
resolved,
outcome,
termination_origin="upstream_terminal",
record_attempt=Mock(),
)
projection = settlement.turn_outcome
assert projection is not None
assert projection.source_transition_persisted is False
assert projection.next_current_changed is False
assert projection.supply_facts is None
assert service.store.load().sources[0].state.status == "standby"
assert service.events.list() == []
asyncio.run(exercise())
def test_streamed_cooldown_rejected_by_generation_reports_unpersisted(
tmp_path: Path,
) -> None:
async def exercise() -> None:
source = _source("src_stalecool01", "Stale cooldown")
service = _service(tmp_path, sources=[source])
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
older_generation = service._reserve_settlement_generation(source.id)
service._reserve_settlement_generation(source.id)
resolved = ResolvedInvocation(
backend="codex",
requested_model_id=requested_model,
source_id=source.id,
source_label=source.display_name,
model_id="shared-model",
handle=None,
outcome=None,
credential_ref=source.credential_ref,
settlement_generation=older_generation,
)
outcome = _outcome(
RawOutcomeKind.HTTP_ERROR,
status=429,
code="rate_limit_error",
source_id=source.id,
stream_started=True,
)
record_attempt = Mock()
settlement = await service.settle_handle_outcome(
resolved,
outcome,
termination_origin="upstream_terminal",
record_attempt=record_attempt,
)
projection = settlement.turn_outcome
assert projection is not None
assert projection.source_transition_persisted is False
assert projection.next_current_changed is False
assert projection.supply_facts is None
assert service.store.load().sources[0].state.status == "standby"
assert service.events.list() == []
record_attempt.assert_called_once()
asyncio.run(exercise())
@pytest.mark.parametrize(
("failure_phase", "expected_outcome_calls"),
[("prepare", 0), ("write", 1)],
)
def test_gateway_closes_producer_before_downstream_cancel_settlement(
tmp_path: Path,
failure_phase: str,
expected_outcome_calls: int,
) -> None:
async def exercise() -> None:
source = _source("src_close0001", "Closed producer")
outcome = _outcome(
RawOutcomeKind.NETWORK_ERROR,
source_id=source.id,
stream_started=failure_phase == "write",
)
handle = DeferredLifecycleHandle(outcome)
service = _service(
tmp_path,
sources=[source],
live_handles=[handle],
)
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
response = FakeStreamResponse(
prepare_error=(
ConnectionResetError("downstream prepare failed")
if failure_phase == "prepare"
else None
),
write_error=(
ConnectionResetError("downstream write failed")
if failure_phase == "write"
else None
),
)
gateway = ModelHubTurnGateway(service)
request = _prepared_gateway_request(
gateway,
turn_id=f"turn_close_{failure_phase}",
requested_model=requested_model,
source_id=source.id,
stream=True,
)
service.settle_handle_outcome = AsyncMock(
wraps=service.settle_handle_outcome
)
with patch(
"core.handlers.model_hub.turn_gateway.web.StreamResponse",
return_value=response,
):
with pytest.raises(ConnectionError, match="downstream"):
await asyncio.wait_for(
gateway._handle_request(request),
timeout=1,
)
assert handle.close_calls == 1
assert handle.outcome_calls == expected_outcome_calls
service.settle_handle_outcome.assert_awaited_once()
expected_origin = (
"upstream_terminal" if failure_phase == "write" else "downstream_cancel"
)
assert (
service.settle_handle_outcome.await_args.kwargs["termination_origin"]
== expected_origin
)
if failure_phase == "write":
assert service.store.load().sources[0].state.status == "standby"
assert service.events.list() == []
else:
assert service.store.load().sources[0].state.status == "standby"
assert service.events.list() == []
asyncio.run(exercise())
def test_gateway_close_barrier_preserves_terminal_error_seen_on_failed_write(
tmp_path: Path,
) -> None:
async def exercise() -> None:
source = _source("src_barrier01", "Barrier source")
outcome = _outcome(
RawOutcomeKind.HTTP_ERROR,
status=429,
code="rate_limit_error",
source_id=source.id,
stream_started=True,
)
handle = DeferredLifecycleHandle(outcome)
service = _service(tmp_path, sources=[source], live_handles=[handle])
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
gateway = ModelHubTurnGateway(service)
request = _prepared_gateway_request(
gateway,
turn_id="turn_fact_barrier",
requested_model=requested_model,
source_id=source.id,
stream=True,
)
service.settle_handle_outcome = AsyncMock(wraps=service.settle_handle_outcome)
with patch(
"core.handlers.model_hub.turn_gateway.web.StreamResponse",
return_value=FakeStreamResponse(
write_error=ConnectionResetError("terminal frame write failed")
),
):
with pytest.raises(ConnectionError, match="terminal frame"):
await gateway._handle_request(request)
assert handle.close_calls == 1
assert handle.outcome_calls == 1
call = service.settle_handle_outcome.await_args
assert call.kwargs["termination_origin"] == "upstream_terminal"
assert call.args[1] is outcome
assert service.store.load().sources[0].state.status == "cooldown"
asyncio.run(exercise())
def test_gateway_downstream_cancellation_does_not_mutate_source(
tmp_path: Path,
) -> None:
async def exercise() -> None:
source = _source("src_cancel01", "Canceled downstream")
outcome = _outcome(
RawOutcomeKind.NETWORK_ERROR,
source_id=source.id,
)
handle = BlockingLiveInvokeHandle(outcome)
service = _service(
tmp_path,
sources=[source],
live_handles=[handle],
)
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
settled = asyncio.Event()
settle_handle_outcome = service.settle_handle_outcome
async def settle(*args, **kwargs):
result = await settle_handle_outcome(*args, **kwargs)
settled.set()
return result
service.settle_handle_outcome = AsyncMock(
side_effect=settle,
)
gateway = ModelHubTurnGateway(service)
base_url, token = await gateway.endpoint(
"codex",
process_scope="/repo",
turn_id="turn_downstream_cancel",
requested_model_id=requested_model,
resolved_model_id="shared-model",
source_id=source.id,
)
try:
async with aiohttp.ClientSession(trust_env=False) as client:
response = await client.post(
f"{base_url}/v1/responses",
json={
"model": "shared-model",
"input": "ping",
"stream": True,
},
headers={"Authorization": f"Bearer {token}"},
)
await handle.started.wait()
response.close()
await asyncio.wait_for(handle.cancelled.wait(), timeout=1)
await asyncio.wait_for(settled.wait(), timeout=1)
finally:
await gateway.close()
service.settle_handle_outcome.assert_awaited_once()
assert (
service.settle_handle_outcome.await_args.kwargs["termination_origin"]
== "downstream_cancel"
)
assert service.store.load().sources[0].state.status == "standby"
assert service.events.list() == []
asyncio.run(exercise())
@pytest.mark.parametrize("phase", ["resolve", "first_byte", "stream"])
def test_gateway_cancellation_matrix_settles_once_without_upstream_facts(
tmp_path: Path,
phase: str,
) -> None:
async def exercise() -> None:
source = _source("src_cancelmx1", "Cancellation matrix")
outcome = _outcome(RawOutcomeKind.NETWORK_ERROR, source_id=source.id)
handle = (
MidStreamBlockingInvokeHandle(outcome)
if phase == "stream"
else BlockingLiveInvokeHandle(outcome)
)
service = _service(
tmp_path / phase,
sources=[source],
live_handles=[] if phase == "resolve" else [handle],
)
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
resolve_started = asyncio.Event()
if phase == "resolve":
async def blocked_resolve(**_kwargs):
resolve_started.set()
await asyncio.Event().wait()
service.resolve = AsyncMock(side_effect=blocked_resolve)
service.settle_handle_outcome = AsyncMock(
wraps=service.settle_handle_outcome
)
gateway = ModelHubTurnGateway(service)
turn_id = f"turn_cancel_matrix_{phase}"
request = _prepared_gateway_request(
gateway,
turn_id=turn_id,
requested_model=requested_model,
source_id=source.id,
stream=True,
)
with patch(
"core.handlers.model_hub.turn_gateway.web.StreamResponse",
return_value=FakeStreamResponse(),
):
task = asyncio.create_task(gateway._handle_request(request))
await asyncio.wait_for(
resolve_started.wait() if phase == "resolve" else handle.started.wait(),
timeout=1,
)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(task, timeout=1)
service.settle_handle_outcome.assert_awaited_once()
settlement_call = service.settle_handle_outcome.await_args
assert settlement_call.kwargs["termination_origin"] == "downstream_cancel"
assert settlement_call.args[1] is None
assert service.store.load().sources[0].state.status == "standby"
assert service.events.list() == []
if phase == "resolve":
trace = gateway.correlation._traces[turn_id]
assert trace.pending_attempt is None
gateway.correlation.settle(
turn_id,
settled_by=SETTLED_BY_STOPPED,
ts=NOW.isoformat(),
)
record = service.provenance.get(turn_id)
assert record is not None
assert record["outcome"] == "canceled"
assert record["terminal_error"] is None
if phase == "resolve":
assert record["canceled_attempt"] is None
asyncio.run(exercise())
def test_gateway_cancellation_during_upstream_settlement_preserves_history(
tmp_path: Path,
) -> None:
async def exercise() -> None:
source = _source("src_eofdone1", "EOF completed")
outcome = _outcome(RawOutcomeKind.SUCCESS, source_id=source.id)
handle = LiveInvokeHandle(outcome, (b"data: [DONE]\n\n",))
service = _service(
tmp_path,
sources=[source],
live_handles=[handle],
)
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
settlement_started = asyncio.Event()
release_settlement = asyncio.Event()
settle_handle_outcome = service.settle_handle_outcome
async def blocked_settlement(*args, **kwargs):
settlement_started.set()
await release_settlement.wait()
return await settle_handle_outcome(*args, **kwargs)
service.settle_handle_outcome = AsyncMock(side_effect=blocked_settlement)
gateway = ModelHubTurnGateway(service)
turn_id = "turn_cancel_after_eof"
request = _prepared_gateway_request(
gateway,
turn_id=turn_id,
requested_model=requested_model,
source_id=source.id,
stream=True,
)
eof_reached = asyncio.Event()
with patch(
"core.handlers.model_hub.turn_gateway.web.StreamResponse",
return_value=FakeStreamResponse(eof_reached=eof_reached),
):
task = asyncio.create_task(gateway._handle_request(request))
await asyncio.wait_for(settlement_started.wait(), timeout=1)
task.cancel()
release_settlement.set()
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(task, timeout=1)
service.settle_handle_outcome.assert_awaited_once()
assert (
service.settle_handle_outcome.await_args.kwargs["termination_origin"]
== "upstream_terminal"
)
gateway.correlation.settle(
turn_id,
settled_by=SETTLED_BY_TERMINAL_RESULT,
ts=NOW.isoformat(),
)
record = service.provenance.get(turn_id)
assert record is not None
assert record["outcome"] == "served"
assert service.store.load().sources[0].state.status == "standby"
assert service.events.list() == []
asyncio.run(exercise())
def test_resolver_settles_a_bodyless_attempt_that_beats_cancellation(
tmp_path: Path,
) -> None:
async def exercise() -> None:
source = _source("src_cancelbuf", "Cancelled buffered response")
outcome = _outcome(
RawOutcomeKind.HTTP_ERROR,
status=429,
code="rate_limit_error",
source_id=source.id,
usage=ProtocolUsageReport.of(
input_tokens=321,
cached_input_tokens=0,
output_tokens=7,
),
)
service = _service(tmp_path, sources=[source], outcomes=[outcome])
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
invoke_started = asyncio.Event()
release_invoke = asyncio.Event()
handle_ready = asyncio.Event()
invoke = service.adapter.invoke
async def blocked_invoke(*args, **kwargs):
kwargs.pop("on_admitted")()
invoke_started.set()
await release_invoke.wait()
handle = await invoke(*args, **kwargs)
handle_ready.set()
return handle
service.adapter.invoke = blocked_invoke
task = asyncio.create_task(
service.resolve(
backend="codex",
model_id=requested_model,
request={},
stream=False,
supply_channel="hub",
)
)
await asyncio.wait_for(invoke_started.wait(), timeout=1)
release_invoke.set()
await asyncio.wait_for(handle_ready.wait(), timeout=1)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(task, timeout=1)
metered = _usage_of(service, source.id)
assert metered["requests"] == 1
assert metered["token_reports"] == 1
assert metered["input_tokens"] == 321
assert metered["output_tokens"] == 7
assert service.store.load().sources[0].state.status == "cooldown"
assert [event["reason"] for event in service.events.list()] == ["rate_limited"]
asyncio.run(exercise())
def test_resolver_meters_an_observed_stream_that_beats_cancellation(
tmp_path: Path,
) -> None:
async def exercise() -> None:
source = _source("src_cancelobs", "Cancelled observed response")
observed = ProtocolSSEState("openai_responses")
observed.model_output_started = True
observed.usage = ProtocolUsageReport.of(
input_tokens=144,
cached_input_tokens=32,
output_tokens=9,
)
handle = ObservedUnsettledHandle(observed)
service = _service(tmp_path, sources=[source], live_handles=[handle])
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
invoke_started = asyncio.Event()
release_invoke = asyncio.Event()
handle_ready = asyncio.Event()
invoke = service.adapter.invoke
async def blocked_invoke(*args, **kwargs):
kwargs.pop("on_admitted")()
invoke_started.set()
await release_invoke.wait()
result = await invoke(*args, **kwargs)
handle_ready.set()
return result
service.adapter.invoke = blocked_invoke
task = asyncio.create_task(
service.resolve(
backend="codex",
model_id=requested_model,
request={},
stream=True,
supply_channel="hub",
)
)
await asyncio.wait_for(invoke_started.wait(), timeout=1)
release_invoke.set()
await asyncio.wait_for(handle_ready.wait(), timeout=1)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(task, timeout=1)
metered = _usage_of(service, source.id)
assert metered["requests"] == 1
assert metered["token_reports"] == 1
assert metered["input_tokens"] == 144
assert metered["cached_input_tokens"] == 32
assert metered["output_tokens"] == 9
assert handle.close_calls == 1
assert service.store.load().sources[0].state.status == "standby"
assert service.events.list() == []
asyncio.run(exercise())
@pytest.mark.parametrize(
"blocked_phase",
[
"resource_teardown",
"handle_outcome",
"service_settlement",
"service_transition",
],
)
def test_gateway_repeated_cancellation_drains_settlement_before_reraise(
tmp_path: Path,
blocked_phase: str,
) -> None:
async def exercise() -> None:
source = _source("src_canceldr1", "Repeated cancellation")
outcome = _outcome(
(
RawOutcomeKind.HTTP_ERROR
if blocked_phase == "service_transition"
else RawOutcomeKind.SUCCESS
),
status=(429 if blocked_phase == "service_transition" else None),
code=(
"rate_limit_error"
if blocked_phase == "service_transition"
else None
),
source_id=source.id,
stream_started=True,
)
handle = RepeatedCancellationHandle(outcome, blocked_phase)
service = _service(tmp_path, sources=[source], live_handles=[handle])
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
settle_handle_outcome = service.settle_handle_outcome
async def blocked_settlement(*args, **kwargs):
if blocked_phase == "service_settlement":
handle.phase_started.set()
await handle.release_phase.wait()
return await settle_handle_outcome(*args, **kwargs)
service.settle_handle_outcome = AsyncMock(side_effect=blocked_settlement)
settle_fallback_source = service._settle_fallback_source
async def blocked_transition(*args, **kwargs):
handle.phase_started.set()
await handle.release_phase.wait()
return await settle_fallback_source(*args, **kwargs)
if blocked_phase == "service_transition":
service._settle_fallback_source = AsyncMock(
side_effect=blocked_transition
)
gateway = ModelHubTurnGateway(service)
record_turn_outcome = gateway.correlation.record_turn_outcome
gateway.correlation.record_turn_outcome = Mock(
side_effect=record_turn_outcome
)
request = _prepared_gateway_request(
gateway,
turn_id="turn_repeated_cancel",
requested_model=requested_model,
source_id=source.id,
stream=True,
)
response = FakeStreamResponse()
with patch(
"core.handlers.model_hub.turn_gateway.web.StreamResponse",
return_value=response,
):
task = asyncio.create_task(gateway._handle_request(request))
await asyncio.wait_for(handle.started.wait(), timeout=1)
task.cancel()
await asyncio.wait_for(handle.phase_started.wait(), timeout=1)
await asyncio.sleep(0)
task.cancel()
await asyncio.sleep(0)
task.cancel()
assert not task.done()
handle.release_phase.set()
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(task, timeout=1)
service.settle_handle_outcome.assert_awaited_once()
gateway.correlation.record_turn_outcome.assert_called_once()
assert (
service.settle_handle_outcome.await_args.kwargs["termination_origin"]
== "upstream_terminal"
)
gateway.correlation.settle(
"turn_repeated_cancel",
settled_by=SETTLED_BY_TERMINAL_RESULT,
ts=NOW.isoformat(),
)
record = service.provenance.get("turn_repeated_cancel")
assert record is not None
if blocked_phase == "service_transition":
assert record["outcome"] == "failed_terminal"
assert service.store.load().sources[0].state.status == "cooldown"
assert [event["reason"] for event in service.events.list()] == [
"rate_limited"
]
else:
assert record["outcome"] == "served"
assert record["terminal_error"] is None
assert handle.close_calls >= 1
assert handle.outcome_calls == 1
asyncio.run(exercise())
def test_settlement_timeout_preserves_an_already_committed_upstream_fact(
tmp_path: Path,
) -> None:
async def exercise() -> None:
source = _source("src_t3barrier1", "Committed terminal fact")
outcome = _outcome(
RawOutcomeKind.HTTP_ERROR,
status=429,
code="rate_limit_error",
source_id=source.id,
stream_started=True,
)
handle = LiveInvokeHandle(
outcome,
(b'data: {"error":{"type":"rate_limit_error"}}\n\n',),
)
service = _service(tmp_path, sources=[source], live_handles=[handle])
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
transition_started = asyncio.Event()
never_release = asyncio.Event()
async def blocked_transition(*_args, **_kwargs):
transition_started.set()
await never_release.wait()
service._settle_fallback_source = AsyncMock(side_effect=blocked_transition)
gateway = ModelHubTurnGateway(service, transport_timeout=0.02)
turn_id = "turn_t3_barrier"
request = _prepared_gateway_request(
gateway,
turn_id=turn_id,
requested_model=requested_model,
source_id=source.id,
stream=True,
)
with patch(
"core.handlers.model_hub.turn_gateway.web.StreamResponse",
return_value=FakeStreamResponse(),
):
task = asyncio.create_task(gateway._handle_request(request))
await asyncio.wait_for(transition_started.wait(), timeout=1)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(task, timeout=1)
assert gateway.resource_leak_records == (("settlement", turn_id),)
gateway.correlation.settle(
turn_id,
settled_by=SETTLED_BY_STOPPED,
ts=NOW.isoformat(),
)
record = service.provenance.get(turn_id)
assert record is not None
assert record["outcome"] == "failed_terminal"
assert record["terminal_error"]["reason"] == "stream_interrupted"
assert record["terminal_error"]["source_id"] == source.id
assert service.store.load().sources[0].state.status == "standby"
asyncio.run(exercise())
def test_gateway_abandons_never_resolving_teardown_at_transport_deadline(
tmp_path: Path,
caplog: pytest.LogCaptureFixture,
) -> None:
async def exercise() -> None:
source = _source("src_teardown_timeout", "Teardown timeout")
handle = NeverResolvingCloseHandle(_outcome(RawOutcomeKind.SUCCESS, source_id=source.id))
service = _service(tmp_path, sources=[source], live_handles=[handle])
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
gateway = ModelHubTurnGateway(service, transport_timeout=0.02)
turn_id = "turn_teardown_timeout"
request = _prepared_gateway_request(
gateway,
turn_id=turn_id,
requested_model=requested_model,
source_id=source.id,
stream=True,
)
with patch(
"core.handlers.model_hub.turn_gateway.web.StreamResponse",
return_value=FakeStreamResponse(),
):
task = asyncio.create_task(gateway._handle_request(request))
await asyncio.wait_for(handle.started.wait(), timeout=1)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(task, timeout=1)
assert gateway.resource_leak_records == (("resource_teardown", turn_id),)
gateway.correlation.settle(
turn_id,
settled_by=SETTLED_BY_STOPPED,
ts=NOW.isoformat(),
)
record = service.provenance.get(turn_id)
assert record is not None
assert record["outcome"] == "failed_terminal"
assert record["terminal_error"] == {
"source_id": None,
"configured_model_id": None,
"channel": None,
"reason": "engine_down",
"stream_started": False,
}
assert service.store.load().sources[0].state.status == "standby"
assert service.events.list() == []
handle.release_close.set()
await asyncio.wait_for(handle.closed.wait(), timeout=1)
with caplog.at_level("ERROR", logger="core.handlers.model_hub.turn_gateway"):
asyncio.run(exercise())
assert "Abandoned Model Hub turn resource" in caplog.text
def test_gateway_eof_disconnect_after_upstream_outcome_keeps_upstream_history(
tmp_path: Path,
) -> None:
async def exercise() -> None:
source = _source("src_eofdrop1", "EOF disconnect")
outcome = _outcome(RawOutcomeKind.SUCCESS, source_id=source.id)
handle = LiveInvokeHandle(outcome, (b"data: [DONE]\n\n",))
service = _service(tmp_path, sources=[source], live_handles=[handle])
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
gateway = ModelHubTurnGateway(service)
turn_id = "turn_eof_disconnect"
request = _prepared_gateway_request(
gateway,
turn_id=turn_id,
requested_model=requested_model,
source_id=source.id,
stream=True,
)
service.settle_handle_outcome = AsyncMock(
wraps=service.settle_handle_outcome
)
with patch(
"core.handlers.model_hub.turn_gateway.web.StreamResponse",
return_value=FakeStreamResponse(
eof_error=ConnectionResetError("downstream EOF failed")
),
):
with pytest.raises(ConnectionError, match="downstream EOF failed"):
await gateway._handle_request(request)
assert (
service.settle_handle_outcome.await_args.kwargs["termination_origin"]
== "upstream_terminal"
)
gateway.correlation.settle(
turn_id,
settled_by=SETTLED_BY_TERMINAL_RESULT,
ts=NOW.isoformat(),
)
record = service.provenance.get(turn_id)
assert record is not None
assert record["outcome"] == "served"
assert record["terminal_error"] is None
assert service.store.load().sources[0].state.status == "standby"
asyncio.run(exercise())
def test_gateway_upstream_stream_failure_remains_source_attributable(
tmp_path: Path,
) -> None:
async def exercise() -> None:
source = _source("src_upstream01", "Broken upstream")
outcome = _outcome(
RawOutcomeKind.NETWORK_ERROR,
source_id=source.id,
stream_started=True,
)
handle = BrokenUpstreamInvokeHandle(outcome)
service = _service(tmp_path, sources=[source])
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
service.adapter.live_handles.append(handle)
gateway = ModelHubTurnGateway(service)
request = _prepared_gateway_request(
gateway,
turn_id="turn_upstream_disconnect",
requested_model=requested_model,
source_id=source.id,
stream=True,
)
service.settle_handle_outcome = AsyncMock(
wraps=service.settle_handle_outcome
)
with patch(
"core.handlers.model_hub.turn_gateway.web.StreamResponse",
return_value=FakeStreamResponse(),
):
with pytest.raises(ConnectionResetError, match="upstream disconnected"):
await gateway._handle_request(request)
service.settle_handle_outcome.assert_awaited_once()
assert (
service.settle_handle_outcome.await_args.kwargs["termination_origin"]
== "upstream_terminal"
)
assert service.store.load().sources[0].state.status == "standby"
assert service.events.list() == []
asyncio.run(exercise())
def test_shared_source_transition_event_is_emitted_once_for_concurrent_turns(
tmp_path: Path,
) -> None:
async def exercise() -> None:
source = _source("src_sharedtx1", "Shared transition")
service = _service(tmp_path, sources=[source])
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
settlement_generation = service._reserve_settlement_generation(source.id)
resolved = ResolvedInvocation(
backend="codex",
requested_model_id=requested_model,
source_id=source.id,
source_label=source.display_name,
model_id="shared-model",
handle=None,
outcome=None,
settlement_generation=settlement_generation,
)
outcome = _outcome(
RawOutcomeKind.HTTP_ERROR,
status=429,
code="rate_limit_error",
source_id=source.id,
stream_started=True,
)
record_attempts = [Mock(), Mock()]
await asyncio.gather(
*(
service.settle_handle_outcome(
resolved,
outcome,
termination_origin="upstream_terminal",
record_attempt=record_attempt,
)
for record_attempt in record_attempts
)
)
assert all(record_attempt.call_count == 1 for record_attempt in record_attempts)
assert service.store.load().sources[0].state.status == "cooldown"
assert [event["reason"] for event in service.events.list()] == [
"rate_limited"
]
asyncio.run(exercise())
@pytest.mark.parametrize(
("kind", "expected_status", "expected_provenance"),
[
(RawOutcomeKind.SUCCESS, 200, "served"),
(RawOutcomeKind.NETWORK_ERROR, 502, "failed_terminal"),
],
)
def test_live_handle_settlement_survives_concurrent_source_deletion(
tmp_path: Path,
kind: RawOutcomeKind,
expected_status: int,
expected_provenance: str,
) -> None:
async def exercise() -> None:
source = _source("src_deletedlive1", "Deleted during live call")
outcome = _outcome(
kind,
source_id=source.id,
stream_started=kind is RawOutcomeKind.NETWORK_ERROR,
)
handle = BlockingLiveInvokeHandle(outcome)
service = _service(
tmp_path,
sources=[source],
live_handles=[handle],
)
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
with pytest.raises(ModelHubError) as delete_refusal:
await service.delete_source(source.id)
turn_id = f"turn_deleted_live_{kind.value}"
gateway = ModelHubTurnGateway(service)
base_url, token = await gateway.endpoint(
"codex",
process_scope="/repo",
turn_id=turn_id,
requested_model_id=requested_model,
resolved_model_id="shared-model",
source_id=source.id,
)
try:
async with aiohttp.ClientSession(trust_env=False) as client:
response_task = asyncio.create_task(
client.post(
f"{base_url}/v1/responses",
json={
"model": "shared-model",
"input": "ping",
"stream": False,
},
headers={"Authorization": f"Bearer {token}"},
)
)
await asyncio.wait_for(handle.started.wait(), timeout=1)
await service.delete_source(
source.id,
force=True,
confirmed_remove_hops=delete_refusal.value.data[
"would_remove_hops"
],
confirmed_interruptions=delete_refusal.value.data[
"would_interrupt"
],
)
handle.release()
response = await asyncio.wait_for(response_task, timeout=1)
payload = await response.read()
assert response.status == expected_status
assert b"source_not_found" not in payload
finally:
handle.release()
await gateway.close()
assert service.store.load().sources == []
gateway.correlation.settle(
turn_id,
settled_by=SETTLED_BY_TERMINAL_RESULT,
ts=NOW.isoformat(),
)
record = service.provenance.get(turn_id)
assert record is not None
assert record["outcome"] == expected_provenance
assert service.events.list() == []
asyncio.run(exercise())
def test_gateway_localizes_terminal_permission_denial_without_switching(
tmp_path: Path,
) -> None:
async def exercise() -> None:
source = _source("src_primary01", "Primary")
service = _service(
tmp_path,
sources=[source],
outcomes=[
_outcome(
RawOutcomeKind.HTTP_ERROR,
status=403,
code="permission_error",
source_id=source.id,
)
],
)
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
gateway = ModelHubTurnGateway(service, language_provider=lambda: "zh")
base_url, token = await gateway.endpoint(
"codex",
process_scope="/repo",
turn_id="turn_permission_denied",
requested_model_id=requested_model,
resolved_model_id="shared-model",
source_id=source.id,
)
try:
async with aiohttp.ClientSession(trust_env=False) as client:
response = await client.post(
f"{base_url}/v1/responses",
json={"model": "shared-model", "input": "ping", "stream": False},
headers={"Authorization": f"Bearer {token}"},
)
assert response.status == 403
payload = await response.json()
assert payload["error"] == {
"type": "request_incompatible",
"code": "request_incompatible",
"message": i18n_t("modelHub.launch.request_incompatible", "zh"),
}
finally:
await gateway.close()
assert service.adapter.invocations == [
(source.id, "shared-model", "codex")
]
assert service.store.load().sources[0].state.status == "standby"
gateway.correlation.settle(
"turn_permission_denied",
settled_by=SETTLED_BY_TERMINAL_RESULT,
ts=NOW.isoformat(),
)
record = service.provenance.get("turn_permission_denied")
assert record is not None
assert record["outcome"] == "failed_terminal"
assert record["failed_attempts"] == []
assert record["terminal_error"] == {
"source_id": source.id,
"configured_model_id": "shared-model",
"channel": "hub",
"reason": "invalid_parameter",
"stream_started": False,
"http_status": 403,
"upstream_error_code": "permission_error",
}
_assert_valid("turn-provenance.schema.json", record)
asyncio.run(exercise())
def test_gateway_provenance_retains_pre_mapping_model_identity(
tmp_path: Path,
) -> None:
store = BoundedProvenanceStore(tmp_path / "provenance.json")
registry = TurnCorrelationRegistry(store)
token = registry.credentials("codex", "/repo", "turn_mapped")
registry.prepare_gateway_turn(
backend="codex",
token=token,
requested_model_id="gpt-5",
resolved_model_id="custom-gpt-5",
source_id="src_primary01",
via_mapping=True,
)
exact_turn = registry.begin_gateway_request(
backend="codex",
token=token,
requested_model_id="custom-gpt-5",
)
registry.begin_attempt(
exact_turn,
source_id="src_primary01",
resolved_model_id="custom-gpt-5",
channel="hub",
via_mapping=False,
)
success = _outcome(RawOutcomeKind.SUCCESS)
registry.finish_attempt(
exact_turn,
outcome=success,
decision=classify_outcome(success),
)
registry.settle(
"turn_mapped",
settled_by=SETTLED_BY_TERMINAL_RESULT,
ts=NOW.isoformat(),
)
record = store.get("turn_mapped")
assert record is not None
assert record["requested_model_id"] == "gpt-5"
assert record["served"] == {
"source_id": "src_primary01",
"configured_model_id": "custom-gpt-5",
"channel": "hub",
}
_assert_valid("turn-provenance.schema.json", record)
@pytest.mark.parametrize(
"wire_model_id",
[
pytest.param("alias-a", id="canonical_backend_id"),
pytest.param("model-b", id="legacy_upstream_target"),
],
)
def test_gateway_accepts_canonical_backend_id_and_unique_legacy_target(
tmp_path: Path,
wire_model_id: str,
) -> None:
"""A route answers for both model ids a request on it may name.
The launch is told which id to send, so that one is canonical. A process
started before Avibe made it explicit still sends the resolved upstream
id, and its requests must keep routing: the credential already named the
route, so accepting the older shape resolves an alias rather than
guessing between routes.
"""
store = BoundedProvenanceStore(tmp_path / "provenance.json")
registry = TurnCorrelationRegistry(store)
launch_token = registry.prepare_gateway_turn(
backend="codex",
token=registry.credentials("codex", "/repo", "turn_alias"),
requested_model_id="alias-a",
resolved_model_id="model-b",
source_id="src_primary01",
via_mapping=False,
gateway_request_model_id="alias-a",
)
routing = registry.claim_gateway_request(
backend="codex",
token=launch_token,
prepared_turn_id="turn_alias",
gateway_model_id=wire_model_id,
)
assert routing.caller_model_id == "alias-a"
assert routing.owner_turn_id == "turn_alias"
def test_gateway_uses_persisted_exact_hops_for_failover(
tmp_path: Path,
) -> None:
async def exercise() -> None:
requested_model = "claude-opus-4-5"
primary_model = "claude-opus-4-5-20251101"
backup_model = "claude-opus-4-5-20250929"
primary = _source(
"src_primary01",
"Primary",
vendor="anthropic",
protocol="anthropic",
model_id=primary_model,
)
backup = _source(
"src_backup001",
"Backup",
vendor="anthropic",
protocol="anthropic",
model_id=backup_model,
)
service = _service(
tmp_path,
sources=[primary, backup],
outcomes=[
_outcome(
RawOutcomeKind.HTTP_ERROR,
status=429,
source_id=primary.id,
),
_outcome(
RawOutcomeKind.SUCCESS,
source_id=backup.id,
),
],
)
_canonicalize_fixed_test_routes(service)
service.store.config.agents["claude"].routes[requested_model] = ModelHubRouteConfig(
hops=(
ModelHubRouteHopConfig(primary.id, primary_model),
ModelHubRouteHopConfig(backup.id, backup_model),
)
)
gateway = ModelHubTurnGateway(service)
base_url, token = await gateway.endpoint(
"claude",
process_scope="/repo",
turn_id="turn_native_alias",
requested_model_id=requested_model,
resolved_model_id=primary_model,
source_id=primary.id,
)
try:
async with aiohttp.ClientSession(trust_env=False) as client:
response = await client.post(
f"{base_url}/v1/messages",
json={
"model": primary_model,
"max_tokens": 1,
"messages": [{"role": "user", "content": "ping"}],
"stream": False,
},
headers={"x-api-key": token},
)
assert response.status == 200
await response.read()
finally:
await gateway.close()
assert service.adapter.invocations == [
(primary.id, primary_model, "claude"),
(backup.id, backup_model, "claude"),
]
failover_events = [event for event in service.events.list(limit=20) if event["kind"] in {"cooldown", "switch"}]
assert {event["model_id"] for event in failover_events} == {requested_model}
gateway.correlation.settle(
"turn_native_alias",
settled_by=SETTLED_BY_TERMINAL_RESULT,
ts=NOW.isoformat(),
)
record = service.provenance.get("turn_native_alias")
assert record is not None
assert record["requested_model_id"] == requested_model
assert record["served"]["source_id"] == backup.id
assert record["served"]["configured_model_id"] == backup_model
_assert_valid("turn-provenance.schema.json", record)
asyncio.run(exercise())
def test_gateway_keeps_prepared_route_during_overlapping_turns(
tmp_path: Path,
) -> None:
async def exercise() -> None:
requested_model = "claude-opus-4-8"
target_model = "gpt-5.6-luna"
source = _source(
"src_gptrelay01",
"GPT relay",
model_id=target_model,
)
outcome = _outcome(
RawOutcomeKind.SUCCESS,
status=200,
source_id=source.id,
)
response_body = (
b'{"id":"msg_fixture","type":"message","role":"assistant",'
b'"content":[{"type":"text","text":"ok"}],'
b'"model":"gpt-5.6-luna","stop_reason":"end_turn",'
b'"usage":{"input_tokens":1,"output_tokens":1}}'
)
service = _service(
tmp_path,
sources=[source],
live_handles=[LiveInvokeHandle(outcome, (response_body,))],
)
_canonicalize_fixed_test_routes(service)
service.store.config.agents["claude"].routes[requested_model] = ModelHubRouteConfig(
hops=(ModelHubRouteHopConfig(source.id, target_model),)
)
gateway = ModelHubTurnGateway(service)
first_base_url, first_token = await gateway.endpoint(
"claude",
process_scope="session:shared-claude",
turn_id="turn_overlap_first",
requested_model_id=requested_model,
resolved_model_id=target_model,
source_id=source.id,
)
second_base_url, second_token = await gateway.endpoint(
"claude",
process_scope="session:shared-claude",
turn_id="turn_overlap_second",
requested_model_id=requested_model,
resolved_model_id=target_model,
source_id=source.id,
)
assert first_base_url == second_base_url
assert first_token == second_token
try:
async with aiohttp.ClientSession(trust_env=False) as client:
response = await client.post(
f"{second_base_url}/v1/messages",
json={
"model": target_model,
"max_tokens": 1,
"messages": [{"role": "user", "content": "ping"}],
"stream": False,
},
headers={"x-api-key": second_token},
)
assert response.status == 200
await response.read()
finally:
await gateway.close()
assert service.adapter.invocations == [
(source.id, target_model, "claude"),
]
asyncio.run(exercise())
def test_gateway_model_outside_prepared_turn_fails_closed(
tmp_path: Path,
) -> None:
store = BoundedProvenanceStore(tmp_path / "provenance.json")
registry = TurnCorrelationRegistry(store)
token = registry.credentials("codex", "/repo", "turn_expected")
registry.prepare_gateway_turn(
backend="codex",
token=token,
requested_model_id="gpt-5",
resolved_model_id="custom-gpt-5",
source_id="src_primary01",
via_mapping=True,
)
assert (
registry.begin_gateway_request(
backend="codex",
token=token,
requested_model_id="untracked-model",
)
is None
)
registry.settle(
"turn_expected",
settled_by=SETTLED_BY_TERMINAL_RESULT,
ts=NOW.isoformat(),
)
assert store.get("turn_expected") is None
def test_native_terminal_failure_is_not_recorded_as_served(
tmp_path: Path,
) -> None:
store = BoundedProvenanceStore(tmp_path / "native-failure.json")
registry = TurnCorrelationRegistry(store)
registry.begin_native_attempt(
backend="codex",
process_scope="/repo",
turn_id="turn_native_failure",
requested_model_id="shared-model",
source_id="src_primary01",
resolved_model_id="shared-model",
via_mapping=False,
)
registry.fail_native_attempt(
"turn_native_failure",
reason="quota_exhausted",
)
registry.settle(
"turn_native_failure",
settled_by=SETTLED_BY_TERMINAL_RESULT,
ts=NOW.isoformat(),
)
record = store.get("turn_native_failure")
assert record is not None
assert record["outcome"] == "exhausted"
assert record["served"] is None
assert record["failed_attempts"] == [
{
"source_id": "src_primary01",
"configured_model_id": "shared-model",
"channel": "native_cli",
"reason": "quota_exhausted",
}
]
_assert_valid("turn-provenance.schema.json", record)
def test_fsm_drop_overrides_a_completed_gateway_attempt(tmp_path: Path) -> None:
store = BoundedProvenanceStore(tmp_path / "provenance.json")
registry = TurnCorrelationRegistry(store)
exact_turn = _begin_hub_attempt(registry, turn_id="turn_drop")
success = _outcome(RawOutcomeKind.SUCCESS)
registry.finish_attempt(
exact_turn,
outcome=success,
decision=classify_outcome(success),
)
registry.settle(
"turn_drop",
settled_by=SETTLED_BY_NO_TERMINAL_RESULT,
ts=NOW.isoformat(),
)
record = store.get("turn_drop")
assert record is not None
assert record["outcome"] == "failed_terminal"
assert record["terminal_error"]["reason"] == "stream_interrupted"
assert record["served"] is None
_assert_valid("turn-provenance.schema.json", record)
@pytest.mark.parametrize(
"error_kind,recorded,diagnostic,expected_detail",
[
# A backend with no typed kind (Claude, Codex, OpenCode) reports its failure neutrally.
(None, {}, "API Error: 上游连接已关闭\x07 api_key=sk-proj-detail0secret0value",
"API Error: 上游连接已关闭 api_key=[redacted]"),
("stalled", {"error_kind": "stalled"}, "provider sent no data for 600s", "provider sent no data for 600s"),
# Only a kind's shape is ever kept: text in the kind field is dropped, never persisted.
("Authorization: Bearer sk-proj-kind0secret0value", {}, "", None),
# Newly retained native diagnostics must not disclose transport credentials.
(None, {}, "connection failed to https://alice:[email protected]/v1",
"connection failed to https://example.invalid/v1"),
(None, {}, "Authorization: Basic ZHVtbXk6ZHVtbXk=", "Authorization: Basic [redacted]"),
(None, {}, "request failed: eyJkdW1teSI6MX0.eyJkdW1teSI6MX0.ZHVtbXk", "request failed: [redacted]"),
],
)
def test_backend_terminal_failure_overrides_gateway_success(
tmp_path: Path, error_kind: str | None, recorded: dict, diagnostic: str, expected_detail: str | None
) -> None:
store = BoundedProvenanceStore(tmp_path / "backend-terminal.json")
registry = TurnCorrelationRegistry(store)
exact_turn = _begin_hub_attempt(
registry,
turn_id="turn_backend_terminal",
)
success = _outcome(RawOutcomeKind.SUCCESS)
registry.finish_attempt(
exact_turn,
outcome=success,
decision=classify_outcome(success),
)
registry.fail_hub_attempt(
"turn_backend_terminal",
error_kind=error_kind,
diagnostic=diagnostic,
)
registry.settle(
"turn_backend_terminal",
settled_by=SETTLED_BY_TERMINAL_RESULT,
ts=NOW.isoformat(),
)
record = store.get("turn_backend_terminal")
assert record is not None
assert record["outcome"] == "failed_terminal"
assert record["served"] is None
# The backend saw its turn fail, not the request's phase or an upstream verdict.
assert record["terminal_error"] == {
"source_id": "src_primary01",
"configured_model_id": "shared-model",
"channel": "hub",
"reason": "backend_error",
"stream_started": None,
**recorded,
**({"backend_error_detail": expected_detail} if expected_detail else {}),
}
_assert_valid("turn-provenance.schema.json", record)
if error_kind == "stalled":
# Backend-only callbacks have no phase evidence, so neither boolean is
# valid even when the rest of a record is otherwise unchanged.
contradictory = (
{"stream_started": False}, {"stream_started": True},
{"http_status": 400}, {"http_status": None},
{"upstream_error_code": "invalid_request_error"}, {"upstream_error_code": None},
{"upstream_detail": "not an upstream response"},
)
for fields in contradictory:
invalid = {**record, "terminal_error": {**record["terminal_error"], **fields}}
with pytest.raises(AssertionError):
_assert_valid("turn-provenance.schema.json", invalid)
@pytest.mark.parametrize("later_request", [False, True])
def test_backend_failure_keeps_the_gateway_terminal_error_no_later_request_superseded(
tmp_path: Path, later_request: bool
) -> None:
"""A protocol failure has no copy projection; the gateway's exact record of it still outlives the backend's report."""
store = BoundedProvenanceStore(tmp_path / "gateway-terminal.json")
registry = TurnCorrelationRegistry(store)
exact_turn = _begin_hub_attempt(registry, turn_id="turn_gateway_terminal")
success = _outcome(RawOutcomeKind.SUCCESS)
registry.finish_attempt(exact_turn, outcome=success, decision=classify_outcome(success))
registry.begin_attempt(
exact_turn, source_id="src_backup01", resolved_model_id="shared-model", channel="hub", via_mapping=False,
)
failure = _outcome(RawOutcomeKind.PROTOCOL_ERROR, status=200, source_id="src_backup01")
registry.finish_attempt(exact_turn, outcome=failure, decision=classify_outcome(failure))
gateway_terminal = dict(registry._traces[exact_turn].terminal_error)
assert registry.terminal_projection(exact_turn, backend="codex") is None
if later_request:
registry.begin_attempt(
exact_turn, source_id="src_later001", resolved_model_id="shared-model", channel="hub", via_mapping=False,
)
registry.fail_hub_attempt(exact_turn, diagnostic="Codex exited")
# Nothing after the backend's report replaces the turn's record, not even a later success.
late = _outcome(RawOutcomeKind.SUCCESS, source_id="src_late0001")
registry.begin_attempt(
exact_turn, source_id="src_late0001", resolved_model_id="shared-model", channel="hub", via_mapping=False,
request_id="late",
)
registry.finish_attempt(exact_turn, outcome=late, decision=classify_outcome(late), request_id="late")
registry.settle(exact_turn, settled_by=SETTLED_BY_TERMINAL_RESULT, ts=NOW.isoformat())
record = store.get(exact_turn)
assert record is not None and record["outcome"] == "failed_terminal"
if later_request:
# The request still open when the backend failed is the one it reports on.
assert record["terminal_error"] == {
"source_id": "src_later001", "configured_model_id": "shared-model", "channel": "hub",
"reason": "backend_error", "stream_started": None, "backend_error_detail": "Codex exited",
}
else:
assert gateway_terminal["reason"] == "protocol_error" and gateway_terminal["stream_started"] is False
assert record["terminal_error"] == gateway_terminal
_assert_valid("turn-provenance.schema.json", record)
def test_released_backend_rejection_records_read_unchanged(tmp_path: Path) -> None:
"""Records the earlier bridge wrote stay as written: no reason or phase is re-derived on read."""
released = {
"contract_version": 23,
"turn_id": "turn_released_rejection",
"ts": NOW.isoformat(),
"agent": "vibey",
"requested_model_id": "shared-model",
"outcome": "failed_terminal",
"failed_attempts": [],
"served": None,
"terminal_error": {
"source_id": "src_primary01",
"configured_model_id": "shared-model",
"channel": "hub",
"reason": "protocol_error",
"stream_started": True,
},
"canceled_attempt": None,
"model_supply_state": None,
"blockers": [],
}
path = tmp_path / "provenance.json"
path.write_text(json.dumps([released]), encoding="utf-8")
assert BoundedProvenanceStore(path).get("turn_released_rejection") == released
_assert_valid("turn-provenance.schema.json", released)
def test_lost_terminal_retains_completed_fallback_attempts(
tmp_path: Path,
) -> None:
store = BoundedProvenanceStore(tmp_path / "lost-terminal.json")
registry = TurnCorrelationRegistry(store)
exact_turn = _begin_hub_attempt(
registry,
turn_id="turn_lost_terminal",
)
failure = _outcome(
RawOutcomeKind.HTTP_ERROR,
status=429,
)
registry.finish_attempt(
exact_turn,
outcome=failure,
decision=classify_outcome(failure),
)
registry.settle(
"turn_lost_terminal",
settled_by=SETTLED_BY_NO_TERMINAL_RESULT,
ts=NOW.isoformat(),
)
record = store.get("turn_lost_terminal")
assert record is not None
assert record["outcome"] == "exhausted"
assert record["failed_attempts"][0]["reason"] == "rate_limited"
_assert_valid("turn-provenance.schema.json", record)
def test_ambiguous_mixed_and_opencode_attempts_leave_no_record(
tmp_path: Path,
) -> None:
store = BoundedProvenanceStore(tmp_path / "provenance.json")
registry = TurnCorrelationRegistry(store)
first = _begin_hub_attempt(registry, turn_id="turn_first")
assert first == "turn_first"
registry.credentials("codex", "/repo", "turn_second")
assert (
registry.begin_gateway_request(
backend="codex",
token=registry.credentials("codex", "/repo", "turn_second"),
requested_model_id="shared-model",
)
is None
)
registry.settle("turn_first", settled_by=SETTLED_BY_STOPPED)
registry.settle("turn_second", settled_by=SETTLED_BY_TERMINAL_RESULT)
assert store.get("turn_first") is None
assert store.get("turn_second") is None
mixed = _begin_hub_attempt(
registry,
turn_id="turn_mixed",
scope="/mixed",
)
registry.credentials("codex", "/mixed", None)
success = _outcome(RawOutcomeKind.SUCCESS)
registry.finish_attempt(
mixed,
outcome=success,
decision=classify_outcome(success),
)
registry.settle("turn_mixed", settled_by=SETTLED_BY_TERMINAL_RESULT)
assert store.get("turn_mixed") is None
opencode_token = registry.credentials(
"opencode",
"opencode:shared-server",
"turn_opencode",
)
assert (
registry.begin_gateway_request(
backend="opencode",
token=opencode_token,
requested_model_id="shared-model",
)
is None
)
registry.settle("turn_opencode", settled_by=SETTLED_BY_TERMINAL_RESULT)
assert store.get("turn_opencode") is None
def test_fsm_cancel_drop_and_control_classification(tmp_path: Path) -> None:
async def exercise(
*,
turn_id: str,
mode: str,
) -> dict | None:
store = BoundedProvenanceStore(tmp_path / f"{turn_id}.json")
registry = TurnCorrelationRegistry(store)
runtime = SimpleNamespace(
settle_turn=lambda value, *, settled_by, ts, mode=None: registry.settle(
value,
settled_by=settled_by,
ts=ts,
)
)
controller = SimpleNamespace(
model_hub_runtime=runtime,
command_handler=SimpleNamespace(handle_stop=AsyncMock(return_value=True)),
set_agent_status=lambda _session_id, _status: None,
)
manager = SessionTurnManager(controller)
manager.flush_queue = AsyncMock(return_value=False)
started = asyncio.Event()
hold = asyncio.Event()
async def dispatch(_controller, _context, _text, **_kwargs):
current_turn = manager.model_hub_turn_id_for_task()
assert current_turn == turn_id
exact_turn = _begin_hub_attempt(
registry,
turn_id=current_turn,
)
started.set()
if mode == "cancel":
await hold.wait()
if mode == "served":
success = _outcome(RawOutcomeKind.SUCCESS)
registry.finish_attempt(
exact_turn,
outcome=success,
decision=classify_outcome(success),
)
settlement = SETTLED_BY_TERMINAL_RESULT
else:
settlement = SETTLED_BY_NO_TERMINAL_RESULT
return TurnDispatchOutcome(error=None, settled_by=settlement)
context = SimpleNamespace(
platform="avibe",
platform_specific={
"turn_token": turn_id,
"agent_session_target": {"agent_backend": "codex"},
},
)
with patch(
"core.session_turns.dispatch_turn_with_outcome",
side_effect=dispatch,
):
await manager._run(turn_id, context, "test")
task = manager.in_flight[turn_id].task
await started.wait()
if mode == "cancel":
result = await manager.cancel(turn_id)
assert result["status"] == "cancel_requested"
await asyncio.gather(task, return_exceptions=True)
return store.get(turn_id)
async def run_cases() -> tuple[dict, dict, dict]:
canceled = await exercise(turn_id="turn_cancel", mode="cancel")
dropped = await exercise(turn_id="turn_drop", mode="drop")
served = await exercise(turn_id="turn_control", mode="served")
assert canceled is not None
assert dropped is not None
assert served is not None
return canceled, dropped, served
canceled, dropped, served = asyncio.run(run_cases())
assert canceled["outcome"] == "canceled"
assert canceled["canceled_attempt"]["source_id"] == "src_primary01"
assert dropped["outcome"] == "failed_terminal"
assert dropped["terminal_error"]["reason"] == "stream_interrupted"
assert served["outcome"] == "served"
for record in (canceled, dropped, served):
_assert_valid("turn-provenance.schema.json", record)
def test_fsm_settlement_persists_the_turn_time_mode() -> None:
runtime = SimpleNamespace(settle_turn=Mock())
manager = SessionTurnManager(SimpleNamespace(model_hub_runtime=runtime))
context = SimpleNamespace(platform_specific={"turn_token": "turn_direct"})
bind_launch(
context,
ModelHubLaunch(
backend="codex",
channel="direct",
requested_model="gpt-5",
target_model="gpt-5",
runtime_model="gpt-5",
),
)
manager._settle_model_hub_turn(
context,
SETTLED_BY_TERMINAL_RESULT,
)
runtime.settle_turn.assert_called_once()
assert runtime.settle_turn.call_args.kwargs["mode"] == "direct"
def test_fsm_settlement_persists_model_less_direct_mode() -> None:
runtime = SimpleNamespace(settle_turn=Mock())
manager = SessionTurnManager(SimpleNamespace(model_hub_runtime=runtime))
context = SimpleNamespace(platform_specific={"turn_token": "turn_opencode_default"})
bind_turn_mode(context, "direct")
manager._settle_model_hub_turn(
context,
SETTLED_BY_TERMINAL_RESULT,
)
runtime.settle_turn.assert_called_once()
assert runtime.settle_turn.call_args.kwargs["mode"] == "direct"
def test_settlement_retires_correlation_when_mode_persistence_fails(
tmp_path: Path,
) -> None:
correlation = Mock()
service = _service(
tmp_path,
sources=[_source("src_primary01", "Primary")],
)
service.note_turn_mode = Mock(side_effect=OSError("mode store unavailable"))
router = ModelHubRuntimeRouter(
service=service,
turn_gateway=ModelHubTurnGateway(service, correlation=correlation),
)
# The stamp runs on its own writer thread: its failure is logged there and
# never reaches, or holds up, settlement.
router.settle_turn(
"turn_mode_failure",
settled_by=SETTLED_BY_TERMINAL_RESULT,
ts=NOW.isoformat(),
mode="hub",
)
correlation.settle.assert_called_once_with(
"turn_mode_failure",
settled_by=SETTLED_BY_TERMINAL_RESULT,
ts=NOW.isoformat(),
)
def test_opencode_overlay_projects_menu_identity_to_exact_hop_model(tmp_path: Path) -> None:
source = _source("src_overlay01", "Overlay", model_id="upstream-model")
source.models[0].reasoning_efforts = ["medium"]
config = _config([source])
agent = config.agents["opencode"]
agent.routes.pop("shared-model")
agent.routes["menu-model"] = ModelHubRouteConfig(hops=(ModelHubRouteHopConfig(source.id, "upstream-model"),))
agent.models = [
ModelHubBackendModelConfig(
id="menu-model",
native_protocol="openai_responses",
supports_reasoning=True,
reasoning_efforts=["low", "high"],
)
]
agent.menu.checked = ["menu-model"]
service = _service(tmp_path, sources=[source])
service.store.config = config
gateway = SimpleNamespace(
endpoint=AsyncMock(return_value=("http://127.0.0.1:19000", "gateway-token")),
)
router = ModelHubRuntimeRouter(
service=service,
turn_gateway=gateway,
overlay_path=tmp_path / "overlay.json",
)
overlay = asyncio.run(router.prepare_opencode_overlay())
assert overlay is not None
payload = json.loads(overlay.content)
assert overlay.provider_ids == ("avibe-openai",)
assert payload["enabled_providers"] == ["avibe-openai"]
provider = payload["provider"]["avibe-openai"]
assert provider["name"] == "Avibe · OpenAI"
assert provider["npm"] == "@ai-sdk/openai"
assert provider["options"] == {
"apiKey": "gateway-token",
"baseURL": "http://127.0.0.1:19000/v1",
}
assert provider["models"]["menu-model"]["id"] == "menu-model"
assert provider["models"]["menu-model"]["variants"] == {
"high": {"reasoningEffort": "high"},
"low": {"reasoningEffort": "low"},
}
assert provider["models"]["menu-model"]["reasoning"] is True
assert overlay.launches[0].target_model == "upstream-model"
assert opencode_model_catalog_for_overlay(overlay) == {
"providers": [
{
"id": "avibe-openai",
"models": provider["models"],
}
],
"default": {},
}
def test_opencode_public_models_follow_persisted_config_without_overlay(
tmp_path: Path,
) -> None:
source = _source("src_catalog01", "Catalog", model_id="upstream-model")
source.models[0].display_name = "Current model"
source.models[0].reasoning_efforts = ["low", "high"]
config = _config([source])
agent = config.agents["opencode"]
agent.routes = {
"current-model": ModelHubRouteConfig(
hops=(ModelHubRouteHopConfig(source.id, "upstream-model"),)
)
}
agent.models = [
ModelHubBackendModelConfig(
id="current-model",
native_protocol="openai_responses",
display_name="Current model",
reasoning_efforts=["low", "high"],
)
]
agent.menu.checked = ["current-model"]
service = _service(tmp_path, sources=[source])
service.store.config = config
assert service.opencode_public_models() == {
"current-model": {
"id": "current-model",
"name": "Current model",
"native_protocol": "openai_responses",
"variants": {
"low": {"reasoningEffort": "low"},
"high": {"reasoningEffort": "high"},
},
}
}
service.store.config.agents["opencode"].mode = "direct"
assert service.opencode_public_models() == {}
def test_opencode_public_model_hides_preserved_efforts_when_reasoning_is_disabled(
tmp_path: Path,
) -> None:
source = _source("src_no_reasoning", "No reasoning", model_id="upstream-model")
config = _config([source])
agent = config.agents["opencode"]
agent.models = [
ModelHubBackendModelConfig(
id="no-reasoning",
native_protocol="openai_responses",
supports_reasoning=False,
reasoning_efforts=["low", "high"],
)
]
agent.menu.checked = ["no-reasoning"]
service = _service(tmp_path, sources=[source])
service.store.config = config
assert service.opencode_public_models()["no-reasoning"] == {
"id": "no-reasoning",
"name": "no-reasoning",
"native_protocol": "openai_responses",
"reasoning": False,
}
@pytest.mark.parametrize("protocol", ["anthropic", "openai_responses"])
@pytest.mark.parametrize(("context_window", "max_output_tokens"), [
(None, None), (200_000, None), (None, 8_192), (200_000, 8_192),
])
@pytest.mark.parametrize(("input_modalities", "output_modalities"), [([], []), (["text", "image"], []), ([], ["text"])])
@pytest.mark.parametrize("supports_tools", [None, True, False])
@pytest.mark.parametrize("supports_reasoning", [None, True, False])
def test_every_projected_opencode_model_satisfies_the_opencode_schema(
protocol: str,
context_window: int | None,
max_output_tokens: int | None,
input_modalities: list[str],
output_modalities: list[str],
supports_tools: bool | None,
supports_reasoning: bool | None,
) -> None:
"""MH-OPENCODE-LAUNCH-002: OpenCode validates its whole config on start, so
one model described with only a context window (or only an output cap)
failed every OpenCode launch, for every model and session."""
from core.handlers.model_hub.migration import _OPENCODE_MODEL, _opencode_value_well_typed
model = ModelHubBackendModelConfig(
id="custom-模型",
display_name="Custom 模型",
native_protocol=protocol,
context_window=context_window,
max_output_tokens=max_output_tokens,
input_modalities=input_modalities,
output_modalities=output_modalities,
supports_tools=supports_tools,
supports_reasoning=supports_reasoning,
reasoning_efforts=["low", "none"],
)
projected = project_opencode_public_model(model)
assert _opencode_value_well_typed(_OPENCODE_MODEL, projected), projected
if context_window is None and max_output_tokens is None:
assert "limit" not in projected
else:
# A limit the catalog does not know stays OpenCode's "unknown" 0.
assert projected["limit"] == {"context": context_window or 0, "output": max_output_tokens or 0}
@pytest.mark.parametrize("protocol", ["anthropic", "openai_responses"])
@pytest.mark.parametrize("supports_reasoning", [None, True, False])
@pytest.mark.parametrize("efforts", [[], ["none"], ["low", "medium", "custom-tier"], ["low", "none", "custom-tier"]])
def test_opencode_public_model_projects_only_declared_efforts(
protocol: str, supports_reasoning: bool | None, efforts: list[str],
) -> None:
model = ModelHubBackendModelConfig(
id="model-with-explicit-options",
native_protocol=protocol,
supports_reasoning=supports_reasoning,
reasoning_efforts=efforts,
)
before = list(model.reasoning_efforts)
projected = project_opencode_public_model(model)
expected = {}
if supports_reasoning is not False:
for effort in efforts:
if protocol == "anthropic" and effort == "none":
expected[effort] = {"thinking": {"type": "disabled"}}
else:
expected[effort] = {
"effort" if protocol == "anthropic" else "reasoningEffort": effort,
}
assert projected.get("variants", {}) == expected
assert list(projected.get("variants", {})) == list(expected)
assert model.reasoning_efforts == before
@pytest.mark.parametrize(
("input_modalities", "output_modalities", "expected"),
[
(["text"], [], {"input": ["text"]}),
([], ["text"], {"output": ["text"]}),
(["text", "image"], ["text"], {"input": ["text", "image"], "output": ["text"]}),
([], [], None),
],
)
def test_opencode_public_model_omits_unspecified_modality_directions(
input_modalities: list[str],
output_modalities: list[str],
expected: dict[str, list[str]] | None,
) -> None:
model = ModelHubBackendModelConfig(
id="modalities",
native_protocol="openai_responses",
input_modalities=input_modalities,
output_modalities=output_modalities,
)
projected = project_opencode_public_model(model)
if expected is None:
assert "modalities" not in projected
else:
assert projected["modalities"] == expected
def test_opencode_overlay_provider_ids_are_stable_across_gateway_tokens(
tmp_path: Path,
) -> None:
source = _source("src_overlay10", "Overlay")
service = _service(tmp_path, sources=[source])
endpoint = AsyncMock(
side_effect=(
("http://127.0.0.1:19000/opencode", "gateway-token-one"),
("http://127.0.0.1:19000/opencode", "gateway-token-two"),
)
)
router = ModelHubRuntimeRouter(
service=service,
turn_gateway=SimpleNamespace(endpoint=endpoint),
overlay_path=tmp_path / "overlay.json",
)
first = asyncio.run(router.prepare_opencode_overlay())
second = asyncio.run(router.prepare_opencode_overlay())
assert first is not None
assert second is not None
assert first.provider_ids == second.provider_ids == ("avibe-openai",)
assert set(json.loads(first.content)["provider"]) == {"avibe-openai"}
assert set(json.loads(second.content)["provider"]) == {"avibe-openai"}
assert json.loads(first.content)["provider"]["avibe-openai"]["options"]["apiKey"] == (
"gateway-token-one"
)
assert json.loads(second.content)["provider"]["avibe-openai"]["options"]["apiKey"] == (
"gateway-token-two"
)
def test_opencode_overlay_partitions_every_row_by_native_protocol(tmp_path: Path) -> None:
first = _source(
"src_overlay11",
"First",
vendor="custom",
protocol="anthropic",
model_id="first-model",
)
second = _source(
"src_overlay12",
"Second",
vendor="custom",
protocol="openai_responses",
model_id="second-model",
)
config = _config([first, second])
agent = config.agents["opencode"]
agent.models = [
ModelHubBackendModelConfig(
id="first-model",
native_protocol="anthropic",
supports_reasoning=True,
reasoning_efforts=["high"],
),
ModelHubBackendModelConfig(
id="second-model",
native_protocol="openai_responses",
reasoning_efforts=["high"],
),
ModelHubBackendModelConfig(
id="empty-route-model",
native_protocol="anthropic",
),
]
agent.menu.checked = ["first-model", "second-model", "empty-route-model"]
agent.routes = {
"first-model": ModelHubRouteConfig(hops=(ModelHubRouteHopConfig(first.id, "first-model"),)),
"second-model": ModelHubRouteConfig(hops=(ModelHubRouteHopConfig(second.id, "second-model"),)),
"empty-route-model": ModelHubRouteConfig(),
}
service = _service(tmp_path, sources=[first, second])
service.store.config = config
router = ModelHubRuntimeRouter(
service=service,
turn_gateway=SimpleNamespace(
endpoint=AsyncMock(return_value=("http://127.0.0.1:19000/opencode", "gateway-token")),
),
overlay_path=tmp_path / "overlay.json",
)
overlay = asyncio.run(router.prepare_opencode_overlay())
assert overlay is not None
payload = json.loads(overlay.content)
assert set(payload["provider"]) == {"avibe-openai", "avibe-anthropic"}
assert set(payload["enabled_providers"]) == set(payload["provider"])
assert overlay.provider_ids == ("avibe-anthropic", "avibe-openai")
anthropic = payload["provider"]["avibe-anthropic"]
openai = payload["provider"]["avibe-openai"]
assert anthropic["name"] == "Avibe · Anthropic"
assert anthropic["npm"] == "@ai-sdk/anthropic"
assert anthropic["options"]["baseURL"] == "http://127.0.0.1:19000/opencode/v1"
assert anthropic["models"]["first-model"]["variants"] == {
"high": {"effort": "high"},
}
assert "reasoning" not in anthropic["models"]["first-model"]
assert set(anthropic["models"]) == {"first-model", "empty-route-model"}
assert openai["name"] == "Avibe · OpenAI"
assert openai["npm"] == "@ai-sdk/openai"
assert openai["models"]["second-model"]["variants"] == {
"high": {"reasoningEffort": "high"},
}
assert {launch.target_model for launch in overlay.launches} == {
"first-model",
"second-model",
"empty-route-model",
}
def test_opencode_empty_menu_writes_the_empty_overlay(tmp_path: Path) -> None:
config = _config([])
agent = config.agents["opencode"]
agent.menu.checked = []
service = _service(tmp_path, sources=[])
service.store.config = config
router = ModelHubRuntimeRouter(
service=service,
turn_gateway=SimpleNamespace(
endpoint=AsyncMock(
return_value=("http://127.0.0.1:19000/opencode", "gateway-token")
),
),
overlay_path=tmp_path / "overlay.json",
)
overlay = asyncio.run(router.prepare_opencode_overlay())
assert overlay is not None
assert overlay.provider_ids == ("avibe-openai",)
assert overlay.checked_identifiers == ()
assert overlay.model_provider_ids == ()
assert overlay.launches == ()
assert json.loads(overlay.content) == {
"$schema": "https://opencode.ai/config.json",
"enabled_providers": ["avibe-openai"],
"provider": {
"avibe-openai": {
"models": {},
"name": "Avibe · OpenAI",
"npm": "@ai-sdk/openai",
"options": {
"apiKey": "gateway-token",
"baseURL": "http://127.0.0.1:19000/opencode/v1",
},
}
},
}
assert router.overlay_path.read_bytes() == overlay.content
def test_opencode_overlay_identity_is_stable_under_runtime_perturbations(
tmp_path: Path,
) -> None:
first = _source("src_stable001", "First", model_id="upstream-first")
second = _source("src_stable002", "Second", model_id="upstream-second")
config = _config([first, second])
agent = config.agents["opencode"]
agent.models = [
ModelHubBackendModelConfig(
id="stable-model",
native_protocol="openai_responses",
reasoning_efforts=["high"],
)
]
agent.menu.checked = ["stable-model"]
agent.routes = {
"stable-model": ModelHubRouteConfig(
hops=(
ModelHubRouteHopConfig(first.id, "upstream-first"),
ModelHubRouteHopConfig(second.id, "upstream-second"),
)
)
}
service = _service(tmp_path, sources=[first, second])
service.store.config = config
endpoint = AsyncMock(
return_value=("http://127.0.0.1:19000", "gateway-token-one")
)
router = ModelHubRuntimeRouter(
service=service,
turn_gateway=SimpleNamespace(endpoint=endpoint),
overlay_path=tmp_path / "overlay.json",
)
def projected_content() -> bytes:
overlay = asyncio.run(router.prepare_opencode_overlay())
assert overlay is not None
payload = json.loads(overlay.content)
for provider in payload["provider"].values():
provider["options"]["apiKey"] = "<gateway-token>"
assert overlay.checked_identifiers == ("stable-model",)
return json.dumps(
payload,
ensure_ascii=True,
sort_keys=True,
separators=(",", ":"),
).encode()
baseline = projected_content()
perturbations = (
lambda: agent.sources.order.reverse(),
lambda: setattr(
first,
"state",
ModelHubSourceStateConfig(
status="needs_action",
detail_key="models.source.needs_action.oauth_expired",
),
),
lambda: setattr(
agent.routes["stable-model"],
"hops",
(ModelHubRouteHopConfig(second.id, "upstream-second"),),
),
lambda: config.sources.append(
_source("src_stable003", "Unrouted", model_id="unrouted")
),
lambda: config.sources.pop(),
lambda: None,
)
for perturb in perturbations:
perturb()
assert projected_content() == baseline
endpoint.return_value = (
"http://127.0.0.1:19000",
"gateway-token-two",
)
assert projected_content() == baseline
agent.mode = "direct"
assert asyncio.run(router.prepare_opencode_overlay()) is None
assert [model.id for model in agent.models] == ["stable-model"]
agent.mode = "hub"
assert projected_content() == baseline
def test_opencode_overlay_uses_exact_manual_target_missing_from_inventory(tmp_path: Path) -> None:
source = _source(
"src_overlay03",
"Overlay",
model_id="supported-model",
)
config = _config([source])
agent = config.agents["opencode"]
agent.routes.pop("shared-model")
agent.routes["menu-model"] = ModelHubRouteConfig(
hops=(
ModelHubRouteHopConfig(source.id, "stale-model"),
ModelHubRouteHopConfig(source.id, "supported-model"),
)
)
agent.models = [
ModelHubBackendModelConfig(
id="menu-model",
native_protocol="openai_responses",
)
]
agent.menu.checked = ["menu-model"]
service = _service(tmp_path, sources=[source])
service.store.config = config
router = ModelHubRuntimeRouter(
service=service,
turn_gateway=SimpleNamespace(
endpoint=AsyncMock(return_value=("http://127.0.0.1:19000", "gateway-token")),
),
overlay_path=tmp_path / "overlay.json",
)
overlay = asyncio.run(router.prepare_opencode_overlay())
assert overlay is not None
assert [launch.target_model for launch in overlay.launches] == ["stale-model"]
def test_opencode_overlay_preserves_checked_route_with_stale_exact_hop(
tmp_path: Path,
) -> None:
source = _source(
"src_overlay04",
"Overlay",
model_id="current-model",
)
config = _config([source])
agent = config.agents["opencode"]
agent.routes.pop("shared-model")
agent.routes["menu-model"] = ModelHubRouteConfig(hops=(ModelHubRouteHopConfig(source.id, "stale-model"),))
agent.models = [
ModelHubBackendModelConfig(
id="menu-model",
native_protocol="openai_responses",
)
]
agent.menu.checked = ["menu-model"]
service = _service(tmp_path, sources=[source])
service.store.config = config
router = ModelHubRuntimeRouter(
service=service,
turn_gateway=SimpleNamespace(
endpoint=AsyncMock(return_value=("http://127.0.0.1:19000", "gateway-token")),
),
overlay_path=tmp_path / "overlay.json",
)
overlay = asyncio.run(router.prepare_opencode_overlay())
assert overlay is not None
provider = json.loads(overlay.content)["provider"]["avibe-openai"]
assert provider["models"] == {
"menu-model": {
"id": "menu-model",
"name": "menu-model",
}
}
assert overlay.checked_identifiers == ("menu-model",)
assert overlay.available_identifiers == ("menu-model",)
assert [launch.target_model for launch in overlay.launches] == ["stale-model"]
def test_production_adapter_retargets_api_keys_without_exposing_or_mutating_them(
tmp_path: Path,
) -> None:
state_store = EngineStateStore(tmp_path / "engine-state")
old_ref = state_store.store_api_key(
"test-retarget-secret",
vendor="custom",
protocol="openai_chat",
base_url="https://old-relay.example/v1",
)
oauth_ref = state_store.bind_oauth_credential(
"src_oauthrefresh",
"openai",
"codex-test.json",
)
adapter = CLIProxyEngineAdapter(
supervisor=Mock(),
state_store=state_store,
)
replacement_ref = asyncio.run(
adapter.retarget_api_key_credential(
old_ref,
"custom",
"openai_chat",
"https://new-relay.example/v1?api-version=2026-08-10",
)
)
assert replacement_ref != old_ref
assert state_store.credential_metadata(old_ref)["base_url"] == ("https://old-relay.example/v1")
assert state_store.credential_metadata(replacement_ref)["base_url"] == (
"https://new-relay.example/v1?api-version=2026-08-10"
)
assert state_store.read_api_key(old_ref) == "test-retarget-secret"
assert state_store.read_api_key(replacement_ref) == "test-retarget-secret"
assert asyncio.run(adapter.credential_supports_refresh(old_ref)) is False
assert asyncio.run(adapter.credential_supports_refresh(oauth_ref)) is True
# These existing evidence tests exercise model-free Auto probes, not Google's
# explicitly declared native model-list witness (covered by test_model_hub_google).
_MODEL_FREE_OBSERVATION_PROTOCOLS = ("anthropic", "openai_responses", "openai_chat")
def test_source_observation_reduces_the_order_at_the_first_authenticated_proof(
tmp_path: Path,
) -> None:
base_url = "https://relay.example/v1?api-version=2026-07-23"
state_store = EngineStateStore(tmp_path / "engine-state")
credential_ref = state_store.store_api_key(
"test-observation-key",
vendor="custom",
protocol=_MODEL_FREE_OBSERVATION_PROTOCOLS[-1],
base_url=base_url,
)
adapter = CLIProxyEngineAdapter(
supervisor=Mock(),
state_store=state_store,
)
unsupported = EngineClientError(
"unsupported protocol path",
status_code=404,
)
proven_accepted = _ProtocolEvidence(
protocol=_ProtocolProof.PROVEN,
authentication=_AuthenticationEvidence.ACCEPTED,
)
proven_rejected = _ProtocolEvidence(
protocol=_ProtocolProof.PROVEN,
authentication=_AuthenticationEvidence.REJECTED,
)
proven_unknown = _ProtocolEvidence(
protocol=_ProtocolProof.PROVEN,
authentication=_AuthenticationEvidence.UNKNOWN,
)
generic_request_accepted = _ProtocolEvidence(
protocol=_ProtocolProof.UNPROVEN,
authentication=_AuthenticationEvidence.ACCEPTED,
shape=_ProtocolObservationShape.GENERIC_REQUEST_ERROR,
)
unproven_unknown = _ProtocolEvidence(
protocol=_ProtocolProof.UNPROVEN,
authentication=_AuthenticationEvidence.UNKNOWN,
)
excluded_404 = _ProtocolEvidence(
protocol=_ProtocolProof.UNPROVEN,
authentication=_AuthenticationEvidence.UNKNOWN,
shape=_ProtocolObservationShape.HTTP_404,
)
non_json = _ProtocolEvidence(
protocol=_ProtocolProof.UNPROVEN,
authentication=_AuthenticationEvidence.UNKNOWN,
shape=_ProtocolObservationShape.NON_JSON,
)
hinted_order = tuple(reversed(_MODEL_FREE_OBSERVATION_PROTOCOLS))
async def every_candidate_is_supported(**_kwargs) -> _ProtocolEvidence:
return proven_accepted
with (
patch(
"vibe.model_hub_runtime.adapter._probe_protocol_response",
new=AsyncMock(side_effect=every_candidate_is_supported),
) as protocol_probe,
patch(
"vibe.model_hub_runtime.adapter.probe_models",
new=AsyncMock(return_value=(DiscoveredModel(id="upstream-model"),)),
) as inventory_probe,
):
observed = asyncio.run(
adapter.observe_source(
"custom",
base_url,
credential_ref,
hinted_order,
)
)
assert observed.outcome.value == "observed"
assert observed.protocol == hinted_order[0]
assert [call.kwargs["protocol"] for call in protocol_probe.await_args_list] == [hinted_order[0]]
assert inventory_probe.await_args.kwargs["protocol"] == hinted_order[0]
async def indistinguishable_response(**_kwargs) -> _ProtocolEvidence:
return unproven_unknown
with (
patch(
"vibe.model_hub_runtime.adapter._probe_protocol_response",
new=AsyncMock(side_effect=indistinguishable_response),
) as protocol_probe,
patch(
"vibe.model_hub_runtime.adapter.probe_models",
new=AsyncMock(return_value=(DiscoveredModel(id="upstream-model"),)),
) as inventory_probe,
):
ambiguous = asyncio.run(
adapter.observe_source(
"custom",
base_url,
credential_ref,
hinted_order,
)
)
assert ambiguous.outcome.value == "ambiguous"
assert ambiguous.protocol is None
assert ambiguous.authenticated is None
assert [call.kwargs["protocol"] for call in protocol_probe.await_args_list] == list(hinted_order)
inventory_probe.assert_not_awaited()
pairwise_order = ("openai_chat", "openai_responses")
async def openai_pairwise_elimination(**kwargs) -> _ProtocolEvidence:
if kwargs["protocol"] == "openai_chat":
return generic_request_accepted
if kwargs["protocol"] == "openai_responses":
return excluded_404
raise AssertionError(kwargs["protocol"])
with (
patch(
"vibe.model_hub_runtime.adapter._probe_protocol_response",
new=AsyncMock(side_effect=openai_pairwise_elimination),
) as protocol_probe,
patch(
"vibe.model_hub_runtime.adapter.probe_models",
new=AsyncMock(return_value=(DiscoveredModel(id="deepseek-chat"),)),
) as inventory_probe,
):
observed = asyncio.run(
adapter.observe_source(
"custom",
base_url,
credential_ref,
pairwise_order,
)
)
assert observed.outcome.value == "observed"
assert observed.protocol == "openai_chat"
assert observed.model_ids == ("deepseek-chat",)
assert [call.kwargs["protocol"] for call in protocol_probe.await_args_list] == list(pairwise_order)
assert inventory_probe.await_args.kwargs["protocol"] == "openai_chat"
openai_family = {"openai_responses", "openai_chat"}
async def indistinguishable_openai_family(**kwargs) -> _ProtocolEvidence:
if kwargs["protocol"] in openai_family:
return generic_request_accepted
return unproven_unknown
with (
patch(
"vibe.model_hub_runtime.adapter._probe_protocol_response",
new=AsyncMock(side_effect=indistinguishable_openai_family),
) as protocol_probe,
patch(
"vibe.model_hub_runtime.adapter.probe_models",
new=AsyncMock(return_value=(DiscoveredModel(id="should-not-discover"),)),
) as inventory_probe,
):
ambiguous = asyncio.run(
adapter.observe_source(
"custom",
base_url,
credential_ref,
_MODEL_FREE_OBSERVATION_PROTOCOLS,
)
)
assert ambiguous.outcome.value == "ambiguous"
assert ambiguous.protocol is None
assert ambiguous.authenticated is True
assert [call.kwargs["protocol"] for call in protocol_probe.await_args_list] == list(_MODEL_FREE_OBSERVATION_PROTOCOLS)
inventory_probe.assert_not_awaited()
async def structured_unknown_sibling(**kwargs) -> _ProtocolEvidence:
if kwargs["protocol"] == "openai_chat":
return generic_request_accepted
if kwargs["protocol"] == "openai_responses":
return unproven_unknown
return unproven_unknown
with (
patch(
"vibe.model_hub_runtime.adapter._probe_protocol_response",
new=AsyncMock(side_effect=structured_unknown_sibling),
) as protocol_probe,
patch(
"vibe.model_hub_runtime.adapter.probe_models",
new=AsyncMock(return_value=(DiscoveredModel(id="should-not-discover"),)),
) as inventory_probe,
):
ambiguous = asyncio.run(
adapter.observe_source(
"custom",
base_url,
credential_ref,
_MODEL_FREE_OBSERVATION_PROTOCOLS,
)
)
assert ambiguous.outcome.value == "ambiguous"
assert ambiguous.protocol is None
assert ambiguous.authenticated is True
assert [call.kwargs["protocol"] for call in protocol_probe.await_args_list] == list(_MODEL_FREE_OBSERVATION_PROTOCOLS)
inventory_probe.assert_not_awaited()
async def anthropic_server_error_blocks_openai_pairwise(**kwargs) -> _ProtocolEvidence:
if kwargs["protocol"] == "anthropic":
return proven_unknown
if kwargs["protocol"] == "openai_responses":
return generic_request_accepted
if kwargs["protocol"] == "openai_chat":
return excluded_404
raise AssertionError(kwargs["protocol"])
with (
patch(
"vibe.model_hub_runtime.adapter._probe_protocol_response",
new=AsyncMock(side_effect=anthropic_server_error_blocks_openai_pairwise),
) as protocol_probe,
patch(
"vibe.model_hub_runtime.adapter.probe_models",
new=AsyncMock(return_value=(DiscoveredModel(id="should-not-discover"),)),
) as inventory_probe,
):
ambiguous = asyncio.run(
adapter.observe_source(
"custom",
base_url,
credential_ref,
_MODEL_FREE_OBSERVATION_PROTOCOLS,
)
)
assert ambiguous.outcome.value == "ambiguous"
assert ambiguous.protocol is None
assert ambiguous.authenticated is True
assert [call.kwargs["protocol"] for call in protocol_probe.await_args_list] == list(_MODEL_FREE_OBSERVATION_PROTOCOLS)
inventory_probe.assert_not_awaited()
timeout = EngineClientError("protocol observation timed out", error_type="timeout")
async def anthropic_timeout_blocks_openai_pairwise(**kwargs) -> _ProtocolEvidence:
if kwargs["protocol"] == "anthropic":
raise timeout
if kwargs["protocol"] == "openai_responses":
return generic_request_accepted
if kwargs["protocol"] == "openai_chat":
return excluded_404
raise AssertionError(kwargs["protocol"])
with (
patch(
"vibe.model_hub_runtime.adapter._probe_protocol_response",
new=AsyncMock(side_effect=anthropic_timeout_blocks_openai_pairwise),
) as protocol_probe,
patch(
"vibe.model_hub_runtime.adapter.probe_models",
new=AsyncMock(return_value=(DiscoveredModel(id="should-not-discover"),)),
) as inventory_probe,
):
ambiguous = asyncio.run(
adapter.observe_source(
"custom",
base_url,
credential_ref,
_MODEL_FREE_OBSERVATION_PROTOCOLS,
)
)
assert ambiguous.outcome.value == "ambiguous"
assert ambiguous.protocol is None
assert ambiguous.authenticated is True
assert [call.kwargs["protocol"] for call in protocol_probe.await_args_list] == list(_MODEL_FREE_OBSERVATION_PROTOCOLS)
inventory_probe.assert_not_awaited()
async def anthropic_competes_with_openai_pairwise(**kwargs) -> _ProtocolEvidence:
if kwargs["protocol"] == "anthropic":
return generic_request_accepted
if kwargs["protocol"] == "openai_responses":
return generic_request_accepted
if kwargs["protocol"] == "openai_chat":
return excluded_404
raise AssertionError(kwargs["protocol"])
with (
patch(
"vibe.model_hub_runtime.adapter._probe_protocol_response",
new=AsyncMock(side_effect=anthropic_competes_with_openai_pairwise),
) as protocol_probe,
patch(
"vibe.model_hub_runtime.adapter.probe_models",
new=AsyncMock(return_value=(DiscoveredModel(id="should-not-discover"),)),
) as inventory_probe,
):
ambiguous = asyncio.run(
adapter.observe_source(
"custom",
base_url,
credential_ref,
_MODEL_FREE_OBSERVATION_PROTOCOLS,
)
)
assert ambiguous.outcome.value == "ambiguous"
assert ambiguous.protocol is None
assert ambiguous.authenticated is True
assert [call.kwargs["protocol"] for call in protocol_probe.await_args_list] == list(_MODEL_FREE_OBSERVATION_PROTOCOLS)
inventory_probe.assert_not_awaited()
unproven_rejected = _ProtocolEvidence(
protocol=_ProtocolProof.UNPROVEN,
authentication=_AuthenticationEvidence.REJECTED,
)
async def accepted_then_rejected_without_proof(**kwargs) -> _ProtocolEvidence:
if kwargs["protocol"] == "anthropic":
return generic_request_accepted
if kwargs["protocol"] == "openai_responses":
return unproven_rejected
if kwargs["protocol"] == "openai_chat":
return unproven_unknown
raise AssertionError(kwargs["protocol"])
with (
patch(
"vibe.model_hub_runtime.adapter._probe_protocol_response",
new=AsyncMock(side_effect=accepted_then_rejected_without_proof),
) as protocol_probe,
patch(
"vibe.model_hub_runtime.adapter.probe_models",
new=AsyncMock(return_value=(DiscoveredModel(id="should-not-discover"),)),
) as inventory_probe,
):
ambiguous = asyncio.run(
adapter.observe_source(
"custom",
base_url,
credential_ref,
_MODEL_FREE_OBSERVATION_PROTOCOLS,
)
)
assert ambiguous.outcome.value == "ambiguous"
assert ambiguous.protocol is None
assert ambiguous.authenticated is True
assert [call.kwargs["protocol"] for call in protocol_probe.await_args_list] == list(_MODEL_FREE_OBSERVATION_PROTOCOLS)
inventory_probe.assert_not_awaited()
async def rejected_anthropic_allows_openai_pairwise(**kwargs) -> _ProtocolEvidence:
if kwargs["protocol"] == "anthropic":
return unproven_rejected
if kwargs["protocol"] == "openai_responses":
return excluded_404
if kwargs["protocol"] == "openai_chat":
return generic_request_accepted
raise AssertionError(kwargs["protocol"])
with (
patch(
"vibe.model_hub_runtime.adapter._probe_protocol_response",
new=AsyncMock(side_effect=rejected_anthropic_allows_openai_pairwise),
) as protocol_probe,
patch(
"vibe.model_hub_runtime.adapter.probe_models",
new=AsyncMock(return_value=(DiscoveredModel(id="chat-only-model"),)),
) as inventory_probe,
):
observed = asyncio.run(
adapter.observe_source(
"custom",
base_url,
credential_ref,
_MODEL_FREE_OBSERVATION_PROTOCOLS,
)
)
assert observed.outcome.value == "observed"
assert observed.protocol == "openai_chat"
assert observed.authenticated is True
assert observed.model_ids == ("chat-only-model",)
assert [call.kwargs["protocol"] for call in protocol_probe.await_args_list] == list(_MODEL_FREE_OBSERVATION_PROTOCOLS)
assert inventory_probe.await_args.kwargs["protocol"] == "openai_chat"
transient_non_json = _parse_protocol_authenticated_evidence(
"openai_chat",
502,
"<html>bad gateway</html>",
)
assert transient_non_json == _ProtocolEvidence(
protocol=_ProtocolProof.UNPROVEN,
authentication=_AuthenticationEvidence.UNKNOWN,
)
async def transient_non_json_sibling_blocks_pairwise(**kwargs) -> _ProtocolEvidence:
if kwargs["protocol"] == "anthropic":
return excluded_404
if kwargs["protocol"] == "openai_responses":
return generic_request_accepted
if kwargs["protocol"] == "openai_chat":
return transient_non_json
raise AssertionError(kwargs["protocol"])
with (
patch(
"vibe.model_hub_runtime.adapter._probe_protocol_response",
new=AsyncMock(side_effect=transient_non_json_sibling_blocks_pairwise),
) as protocol_probe,
patch(
"vibe.model_hub_runtime.adapter.probe_models",
new=AsyncMock(return_value=(DiscoveredModel(id="should-not-discover"),)),
) as inventory_probe,
):
ambiguous = asyncio.run(
adapter.observe_source(
"custom",
base_url,
credential_ref,
_MODEL_FREE_OBSERVATION_PROTOCOLS,
)
)
assert ambiguous.outcome.value == "ambiguous"
assert ambiguous.protocol is None
assert ambiguous.authenticated is True
assert [call.kwargs["protocol"] for call in protocol_probe.await_args_list] == list(_MODEL_FREE_OBSERVATION_PROTOCOLS)
inventory_probe.assert_not_awaited()
async def anthropic_wrapperless_then_absent_openai(**kwargs) -> _ProtocolEvidence:
if kwargs["protocol"] == "anthropic":
return generic_request_accepted
if kwargs["protocol"] == "openai_responses":
return excluded_404
if kwargs["protocol"] == "openai_chat":
return non_json
raise AssertionError(kwargs["protocol"])
with (
patch(
"vibe.model_hub_runtime.adapter._probe_protocol_response",
new=AsyncMock(side_effect=anthropic_wrapperless_then_absent_openai),
) as protocol_probe,
patch(
"vibe.model_hub_runtime.adapter.probe_models",
new=AsyncMock(return_value=(DiscoveredModel(id="relay-model"),)),
) as inventory_probe,
):
observed = asyncio.run(
adapter.observe_source(
"custom",
base_url,
credential_ref,
_MODEL_FREE_OBSERVATION_PROTOCOLS,
)
)
assert observed.outcome.value == "observed"
assert observed.protocol == "anthropic"
assert observed.model_ids == ("relay-model",)
assert [call.kwargs["protocol"] for call in protocol_probe.await_args_list] == list(_MODEL_FREE_OBSERVATION_PROTOCOLS)
assert inventory_probe.await_args.kwargs["protocol"] == "anthropic"
async def anthropic_wrapperless_with_rejected_openai(**kwargs) -> _ProtocolEvidence:
if kwargs["protocol"] == "anthropic":
return generic_request_accepted
if kwargs["protocol"] in openai_family:
return unproven_rejected
raise AssertionError(kwargs["protocol"])
with (
patch(
"vibe.model_hub_runtime.adapter._probe_protocol_response",
new=AsyncMock(side_effect=anthropic_wrapperless_with_rejected_openai),
) as protocol_probe,
patch(
"vibe.model_hub_runtime.adapter.probe_models",
new=AsyncMock(
return_value=(DiscoveredModel(id="anthropic-relay-model"),)
),
) as inventory_probe,
):
observed = asyncio.run(
adapter.observe_source(
"custom",
base_url,
credential_ref,
_MODEL_FREE_OBSERVATION_PROTOCOLS,
)
)
assert observed.outcome.value == "observed"
assert observed.protocol == "anthropic"
assert observed.authenticated is True
assert observed.model_ids == ("anthropic-relay-model",)
assert [call.kwargs["protocol"] for call in protocol_probe.await_args_list] == list(_MODEL_FREE_OBSERVATION_PROTOCOLS)
assert inventory_probe.await_args.kwargs["protocol"] == "anthropic"
credential_param_rejected = _parse_protocol_authenticated_evidence(
"openai_responses",
400,
json.dumps(
{
"error": {
"type": "invalid_request_error",
"param": "api_key",
}
}
),
)
assert credential_param_rejected == _ProtocolEvidence(
protocol=_ProtocolProof.UNPROVEN,
authentication=_AuthenticationEvidence.REJECTED,
)
async def credential_param_rejection_blocks_pairwise_proof(**kwargs) -> _ProtocolEvidence:
if kwargs["protocol"] == "anthropic":
return unproven_rejected
if kwargs["protocol"] == "openai_responses":
return credential_param_rejected
if kwargs["protocol"] == "openai_chat":
return excluded_404
raise AssertionError(kwargs["protocol"])
with (
patch(
"vibe.model_hub_runtime.adapter._probe_protocol_response",
new=AsyncMock(side_effect=credential_param_rejection_blocks_pairwise_proof),
) as protocol_probe,
patch(
"vibe.model_hub_runtime.adapter.probe_models",
new=AsyncMock(return_value=(DiscoveredModel(id="should-not-discover"),)),
) as inventory_probe,
):
rejected = asyncio.run(
adapter.observe_source(
"custom",
base_url,
credential_ref,
_MODEL_FREE_OBSERVATION_PROTOCOLS,
)
)
assert rejected.outcome.value == "authentication_failed"
assert rejected.protocol is None
assert rejected.authenticated is False
assert [call.kwargs["protocol"] for call in protocol_probe.await_args_list] == list(_MODEL_FREE_OBSERVATION_PROTOCOLS)
inventory_probe.assert_not_awaited()
with (
patch(
"vibe.model_hub_runtime.adapter._probe_protocol_response",
new=AsyncMock(return_value=generic_request_accepted),
) as protocol_probe,
patch(
"vibe.model_hub_runtime.adapter.probe_models",
new=AsyncMock(return_value=(DiscoveredModel(id="should-not-discover"),)),
) as inventory_probe,
):
observed = asyncio.run(
adapter.observe_source(
"custom",
base_url,
credential_ref,
("anthropic",),
)
)
assert observed.outcome.value == "observed"
assert observed.protocol == "anthropic"
assert observed.authenticated is True
assert [call.kwargs["protocol"] for call in protocol_probe.await_args_list] == ["anthropic"]
assert inventory_probe.await_args.kwargs["protocol"] == "anthropic"
async def shaped_server_failure(**kwargs) -> _ProtocolEvidence:
if kwargs["protocol"] == "anthropic":
body = {"type": "error", "error": {"type": "api_error"}}
else:
body = {"error": {"type": "server_error"}}
evidence = _parse_protocol_authenticated_evidence(
kwargs["protocol"],
500,
json.dumps(body),
)
assert evidence == proven_unknown
return evidence
with (
patch(
"vibe.model_hub_runtime.adapter._probe_protocol_response",
new=AsyncMock(side_effect=shaped_server_failure),
) as protocol_probe,
patch(
"vibe.model_hub_runtime.adapter.probe_models",
new=AsyncMock(return_value=(DiscoveredModel(id="unverified-model"),)),
) as inventory_probe,
):
upstream_error = asyncio.run(
adapter.observe_source(
"custom",
base_url,
credential_ref,
_MODEL_FREE_OBSERVATION_PROTOCOLS,
)
)
assert upstream_error.outcome.value == "adapter_error"
assert upstream_error.reachable is True
assert upstream_error.authenticated is None
assert upstream_error.protocol is None
assert [call.kwargs["protocol"] for call in protocol_probe.await_args_list] == list(_MODEL_FREE_OBSERVATION_PROTOCOLS)
inventory_probe.assert_not_awaited()
_assert_valid(
"observation-result.schema.json",
{
"contract_version": 23,
"outcome": "adapter_error",
"reachable": True,
"authenticated": "unknown",
"protocol": None,
"discovery": "not_attempted",
"models": [],
"model_metadata": [],
},
)
proved_protocol = _MODEL_FREE_OBSERVATION_PROTOCOLS[1]
async def later_protocol_probe(**kwargs) -> _ProtocolEvidence:
if kwargs["protocol"] != proved_protocol:
raise unsupported
return proven_accepted
with (
patch(
"vibe.model_hub_runtime.adapter._probe_protocol_response",
new=AsyncMock(side_effect=later_protocol_probe),
) as protocol_probe,
patch(
"vibe.model_hub_runtime.adapter.probe_models",
new=AsyncMock(return_value=(DiscoveredModel(id="upstream-model"),)),
) as inventory_probe,
):
observed = asyncio.run(
adapter.observe_source(
"custom",
base_url,
credential_ref,
_MODEL_FREE_OBSERVATION_PROTOCOLS,
)
)
assert observed.outcome.value == "observed"
assert observed.protocol == proved_protocol
assert observed.model_ids == ("upstream-model",)
assert [call.kwargs["protocol"] for call in protocol_probe.await_args_list] == list(_MODEL_FREE_OBSERVATION_PROTOCOLS[:2])
assert inventory_probe.await_args.kwargs["protocol"] == proved_protocol
assert inventory_probe.await_args.kwargs["base_url"] == base_url
rejected_protocol = _MODEL_FREE_OBSERVATION_PROTOCOLS[0]
accepted_protocol = _MODEL_FREE_OBSERVATION_PROTOCOLS[1]
async def rejected_then_authenticated(**kwargs) -> _ProtocolEvidence:
if kwargs["protocol"] == rejected_protocol:
return proven_rejected
return proven_accepted
with (
patch(
"vibe.model_hub_runtime.adapter._probe_protocol_response",
new=AsyncMock(side_effect=rejected_then_authenticated),
) as protocol_probe,
patch(
"vibe.model_hub_runtime.adapter.probe_models",
new=AsyncMock(
return_value=(DiscoveredModel(id="later-protocol-model"),)
),
) as inventory_probe,
):
observed = asyncio.run(
adapter.observe_source(
"custom",
base_url,
credential_ref,
_MODEL_FREE_OBSERVATION_PROTOCOLS,
)
)
assert observed.outcome.value == "observed"
assert observed.authenticated is True
assert observed.protocol == accepted_protocol
assert observed.model_ids == ("later-protocol-model",)
assert [call.kwargs["protocol"] for call in protocol_probe.await_args_list] == [
rejected_protocol,
accepted_protocol,
]
assert inventory_probe.await_args.kwargs["protocol"] == accepted_protocol
async def reject_every_candidate(**kwargs) -> _ProtocolEvidence:
return _parse_protocol_authenticated_evidence(
kwargs["protocol"],
401,
json.dumps(
{
"error": {
"type": "invalid_request_error",
"code": "invalid_api_key",
"param": None,
}
}
),
)
with (
patch(
"vibe.model_hub_runtime.adapter._probe_protocol_response",
new=AsyncMock(side_effect=reject_every_candidate),
) as protocol_probe,
patch(
"vibe.model_hub_runtime.adapter.probe_models",
new=AsyncMock(return_value=(DiscoveredModel(id="unreachable-model"),)),
) as inventory_probe,
):
rejected = asyncio.run(
adapter.observe_source(
"custom",
base_url,
credential_ref,
_MODEL_FREE_OBSERVATION_PROTOCOLS,
)
)
assert rejected.outcome.value == "authentication_failed"
assert rejected.reachable is True
assert rejected.authenticated is False
assert rejected.protocol is None
assert [call.kwargs["protocol"] for call in protocol_probe.await_args_list] == list(_MODEL_FREE_OBSERVATION_PROTOCOLS)
inventory_probe.assert_not_awaited()
def test_source_observation_accepts_catalog_pin_and_custom_declaration_without_shape_proof(
tmp_path: Path,
) -> None:
state_store = EngineStateStore(tmp_path / "engine-state")
catalog_ref = state_store.store_api_key(
"test-deepseek-key",
vendor="deepseek",
protocol="openai_chat",
base_url=None,
)
custom_ref = state_store.store_api_key(
"test-custom-key",
vendor="custom",
protocol="openai_chat",
base_url="https://api.deepseek.com",
)
adapter = CLIProxyEngineAdapter(
supervisor=Mock(),
state_store=state_store,
)
generic_request_accepted = _ProtocolEvidence(
protocol=_ProtocolProof.UNPROVEN,
authentication=_AuthenticationEvidence.ACCEPTED,
shape=_ProtocolObservationShape.GENERIC_REQUEST_ERROR,
)
async def identical_three_path_shape(**_kwargs) -> _ProtocolEvidence:
return generic_request_accepted
with (
patch(
"vibe.model_hub_runtime.adapter._probe_protocol_response",
new=AsyncMock(side_effect=identical_three_path_shape),
) as protocol_probe,
patch(
"vibe.model_hub_runtime.adapter.probe_models",
new=AsyncMock(return_value=(DiscoveredModel(id="deepseek-chat"),)),
) as inventory_probe,
):
observed = asyncio.run(
adapter.observe_source(
"deepseek",
None,
catalog_ref,
_MODEL_FREE_OBSERVATION_PROTOCOLS,
)
)
assert observed.outcome.value == "observed"
assert observed.protocol == "openai_chat"
assert observed.model_ids == ("deepseek-chat",)
assert [call.kwargs["protocol"] for call in protocol_probe.await_args_list] == list(_MODEL_FREE_OBSERVATION_PROTOCOLS)
assert inventory_probe.await_args.kwargs["protocol"] == "openai_chat"
with (
patch(
"vibe.model_hub_runtime.adapter._probe_protocol_response",
new=AsyncMock(side_effect=identical_three_path_shape),
) as protocol_probe,
patch(
"vibe.model_hub_runtime.adapter.probe_models",
new=AsyncMock(return_value=(DiscoveredModel(id="should-not-discover"),)),
) as inventory_probe,
):
ambiguous = asyncio.run(
adapter.observe_source(
"custom",
"https://api.deepseek.com",
custom_ref,
_MODEL_FREE_OBSERVATION_PROTOCOLS,
)
)
assert ambiguous.outcome.value == "ambiguous"
assert ambiguous.protocol is None
assert ambiguous.authenticated is True
assert [call.kwargs["protocol"] for call in protocol_probe.await_args_list] == list(_MODEL_FREE_OBSERVATION_PROTOCOLS)
inventory_probe.assert_not_awaited()
with (
patch(
"vibe.model_hub_runtime.adapter._probe_protocol_response",
new=AsyncMock(side_effect=identical_three_path_shape),
) as protocol_probe,
patch(
"vibe.model_hub_runtime.adapter.probe_models",
new=AsyncMock(return_value=(DiscoveredModel(id="deepseek-chat"),)),
) as inventory_probe,
):
observed = asyncio.run(
adapter.observe_source(
"custom",
"https://api.deepseek.com",
custom_ref,
("openai_chat",),
)
)
assert observed.outcome.value == "observed"
assert observed.protocol == "openai_chat"
assert observed.model_ids == ("deepseek-chat",)
assert [call.kwargs["protocol"] for call in protocol_probe.await_args_list] == ["openai_chat"]
assert inventory_probe.await_args.kwargs["protocol"] == "openai_chat"
def test_qwen_catalog_pin_validation_remains_unverified(
tmp_path: Path,
) -> None:
state_store = EngineStateStore(tmp_path / "engine-state")
credential_ref = state_store.store_api_key(
"test-qwen-key",
vendor="qwen",
protocol="openai_chat",
base_url=None,
)
adapter = CLIProxyEngineAdapter(
supervisor=Mock(),
state_store=state_store,
)
async def scenario() -> tuple[list[str], object, dict[str, object]]:
requests: list[str] = []
async def capture_probe(request: web.Request) -> web.Response:
requests.append(request.path)
if request.headers.get("Authorization") != "Bearer test-qwen-key":
return web.json_response({"code": "INVALID_API_KEY"}, status=401)
if request.path == _PROTOCOL_OBSERVATION_TAXONOMY["openai_chat"].request_path:
return web.json_response(
{
"code": "InvalidParameter",
"message": "messages is required",
},
status=400,
)
return web.json_response({}, status=404)
app = web.Application()
app.router.add_post("/{tail:.*}", capture_probe)
runner = web.AppRunner(app)
await runner.setup()
site = web.TCPSite(runner, "127.0.0.1", 0)
await site.start()
assert site._server is not None
port = site._server.sockets[0].getsockname()[1]
origin = f"http://127.0.0.1:{port}"
try:
with (
patch.dict(
"vibe.model_hub_runtime.adapter._OFFICIAL_BASE_URLS",
{"qwen": origin},
clear=False,
),
patch(
# This interface publishes no model listing, so nothing on
# it can speak for the credential the wrapperless
# validation error left unknown.
"vibe.model_hub_runtime.adapter.probe_models",
new=AsyncMock(
side_effect=EngineClientError("no listing", status_code=404)
),
) as inventory_probe,
):
observed = await adapter.observe_source(
"qwen",
None,
credential_ref,
_MODEL_FREE_OBSERVATION_PROTOCOLS,
)
inventory_kwargs = dict(inventory_probe.await_args.kwargs)
finally:
await runner.cleanup()
return requests, observed, inventory_kwargs
requests, observed, inventory_kwargs = asyncio.run(scenario())
assert observed.outcome.value == "ambiguous"
assert observed.protocol is None
assert observed.authenticated is None
assert observed.model_ids == ()
assert requests == [
_PROTOCOL_OBSERVATION_TAXONOMY[protocol].request_path
for protocol in _MODEL_FREE_OBSERVATION_PROTOCOLS
]
assert inventory_kwargs["protocol"] == "openai_chat"
def test_openrouter_catalog_pin_numeric_validation_remains_unverified(
tmp_path: Path,
) -> None:
state_store = EngineStateStore(tmp_path / "engine-state")
credential_ref = state_store.store_api_key(
"test-openrouter-key",
vendor="openrouter",
protocol="openai_chat",
base_url=None,
)
adapter = CLIProxyEngineAdapter(
supervisor=Mock(),
state_store=state_store,
)
async def scenario() -> tuple[list[str], object, dict[str, object]]:
requests: list[str] = []
async def capture_probe(request: web.Request) -> web.Response:
requests.append(request.path)
if request.headers.get("Authorization") != "Bearer test-openrouter-key":
return web.json_response({"code": "INVALID_API_KEY"}, status=401)
if request.path == _PROTOCOL_OBSERVATION_TAXONOMY["openai_chat"].request_path:
return web.json_response(
{
"error": {
"code": 400,
"message": "messages is required",
}
},
status=400,
)
return web.json_response({}, status=404)
app = web.Application()
app.router.add_post("/{tail:.*}", capture_probe)
runner = web.AppRunner(app)
await runner.setup()
site = web.TCPSite(runner, "127.0.0.1", 0)
await site.start()
assert site._server is not None
port = site._server.sockets[0].getsockname()[1]
origin = f"http://127.0.0.1:{port}"
try:
with (
patch.dict(
"vibe.model_hub_runtime.adapter._OFFICIAL_BASE_URLS",
{"openrouter": origin},
clear=False,
),
patch(
# This interface publishes no model listing, so nothing on
# it can speak for the credential the numeric validation
# error left unknown.
"vibe.model_hub_runtime.adapter.probe_models",
new=AsyncMock(
side_effect=EngineClientError("no listing", status_code=404)
),
) as inventory_probe,
):
observed = await adapter.observe_source(
"openrouter",
None,
credential_ref,
_MODEL_FREE_OBSERVATION_PROTOCOLS,
)
inventory_kwargs = dict(inventory_probe.await_args.kwargs)
finally:
await runner.cleanup()
return requests, observed, inventory_kwargs
requests, observed, inventory_kwargs = asyncio.run(scenario())
assert observed.outcome.value == "ambiguous"
assert observed.protocol is None
assert observed.authenticated is None
assert observed.model_ids == ()
assert requests == [
_PROTOCOL_OBSERVATION_TAXONOMY[protocol].request_path
for protocol in _MODEL_FREE_OBSERVATION_PROTOCOLS
]
assert inventory_kwargs["protocol"] == "openai_chat"
def _catalog_owner_status_body(vendor: str, status: int) -> dict[str, object]:
if status == 401 and vendor == "openrouter":
return {
"error": {
"code": 401,
"message": "Credentials are invalid",
}
}
if status == 400 and vendor == "zhipuai":
return {
"error": {
"code": "1214",
"message": "messages is required",
}
}
if status == 400 and vendor == "openrouter":
return {
"error": {
"code": 400,
"message": "messages is required",
}
}
return {
"error": {
"code": f"{vendor}-{status}",
"message": f"{vendor} response body",
}
}
def _run_catalog_pin_observation(
tmp_path: Path,
*,
vendor: str,
protocol: str,
status: int,
body: dict[str, object],
listing: str,
interface_key: str | None = None,
) -> tuple[list[tuple[str, str]], object]:
"""Observe a pinned vendor whose probe answers `status` and whose model
listing behaves as `listing`.
``listing`` names what the interface does with ``GET /v1/models``:
``gated`` answers the stored credential and refuses an uncredentialed
caller, ``open`` answers anybody, ``rejects`` refuses every caller,
``absent`` publishes no listing at all, and ``presence`` requires a
credential without ever reading its value.
``interface_key`` is the credential this interface would actually accept,
which the stored one matches unless a caller says otherwise.
"""
key = f"test-{vendor}-{status}"
accepted_key = key if interface_key is None else interface_key
state_store = EngineStateStore(tmp_path / f"engine-state-{vendor}-{status}")
credential_ref = state_store.store_api_key(
key,
vendor=vendor,
protocol=protocol,
base_url=None,
)
adapter = CLIProxyEngineAdapter(
supervisor=Mock(),
state_store=state_store,
)
def supplied_credential(request: web.Request) -> str:
return request.headers.get("x-api-key") or request.headers.get(
"Authorization", ""
).removeprefix("Bearer ")
async def scenario() -> tuple[list[tuple[str, str]], object]:
requests: list[tuple[str, str]] = []
async def capture_probe(request: web.Request) -> web.Response:
requests.append((request.method, request.path))
# An interface that never reads the credential's value answers the
# model-less probe out of its schema check, which precedes any key
# lookup, so the probe leaves a key it never validated unknown.
if listing != "presence" and supplied_credential(request) != accepted_key:
return web.json_response({"code": "INVALID_API_KEY"}, status=401)
return web.json_response(body, status=status)
async def capture_listing(request: web.Request) -> web.Response:
requests.append((request.method, request.path))
if listing == "absent":
return web.json_response({"error": "unknown route"}, status=404)
if listing == "presence":
refused = not supplied_credential(request)
else:
refused = listing == "rejects" or (
listing == "gated" and supplied_credential(request) != accepted_key
)
if refused:
return web.json_response({"code": "INVALID_API_KEY"}, status=401)
return web.json_response({"data": [{"id": f"{vendor}/listed"}]})
app = web.Application()
app.router.add_get("/v1/models", capture_listing)
app.router.add_post("/{tail:.*}", capture_probe)
runner = web.AppRunner(app)
await runner.setup()
site = web.TCPSite(runner, "127.0.0.1", 0)
await site.start()
assert site._server is not None
port = site._server.sockets[0].getsockname()[1]
origin = f"http://127.0.0.1:{port}"
try:
with patch.dict(
"vibe.model_hub_runtime.adapter._OFFICIAL_BASE_URLS",
{vendor: origin},
clear=False,
):
observed = await adapter.observe_source(
vendor,
None,
credential_ref,
(protocol,),
)
finally:
await runner.cleanup()
return requests, observed
return asyncio.run(scenario())
@pytest.mark.parametrize("listing", ["gated", "open", "rejects", "absent"])
@pytest.mark.parametrize("status", [400, 401])
@pytest.mark.parametrize(("vendor", "protocol"), CATALOG_API_KEY_VENDOR_PROTOCOL_CASES)
def test_catalog_pin_authentication_comes_from_a_credential_gated_listing(
tmp_path: Path,
vendor: str,
protocol: str,
status: int,
listing: str,
) -> None:
"""A pin's authentication is whatever its model listing attests.
The model-less probe cannot speak for the credential: neither its request
error nor a bare authentication status without a shaped identifier decides
anything, for any vendor. So the listing on the pinned interface answers
instead, and only a listing that is gated by the credential attests to it --
an open catalogue answers whoever asks and therefore names no key.
"""
requests, observed = _run_catalog_pin_observation(
tmp_path,
vendor=vendor,
protocol=protocol,
status=status,
body=_catalog_owner_status_body(vendor, status),
listing=listing,
)
if listing == "gated":
assert observed.outcome.value == "observed"
assert observed.protocol == protocol
assert observed.authenticated is True
assert observed.model_ids == (f"{vendor}/listed",)
elif listing == "rejects":
assert observed.outcome.value == "authentication_failed"
assert observed.protocol is None
assert observed.authenticated is False
assert observed.model_ids == ()
else:
assert observed.outcome.value == "ambiguous"
assert observed.protocol is None
assert observed.authenticated is None
assert observed.model_ids == ()
assert requests[0] == ("POST", _PROTOCOL_OBSERVATION_TAXONOMY[protocol].request_path)
assert set(requests[1:]) <= {("GET", "/v1/models")}
@pytest.mark.parametrize(("vendor", "protocol"), CATALOG_API_KEY_VENDOR_PROTOCOL_CASES)
def test_a_listing_gate_on_presence_answers_exactly_like_one_on_the_key(
tmp_path: Path,
vendor: str,
protocol: str,
) -> None:
"""Add-time verification cannot see which of the two an interface is doing.
Observation may make exactly two requests: one carrying the stored
credential and one carrying none. An interface that reads the value answers
a credential it accepts with its catalogue and an uncredentialed caller
with a refusal. An interface that only requires the header to be there
answers *any* credential with the same catalogue and that same refusal --
including one it would never have accepted, since its probe answers out of
a schema check that precedes the key lookup it never performs. Both worlds
hand back an identical pair of responses, so no reading of that pair
separates them; only a third request carrying a different value could, and
an altered credential attests to nothing in either direction, which is why
this ladder has no synthetic credential control.
So the witness admits both, and a credential an interface never validated
is left to the first real call and the existing needs-action path -- the
same place a credential revoked after it was added is caught.
"""
validated = _run_catalog_pin_observation(
tmp_path / "validated",
vendor=vendor,
protocol=protocol,
status=400,
body=_catalog_owner_status_body(vendor, 400),
listing="gated",
)
unread = _run_catalog_pin_observation(
tmp_path / "unread",
vendor=vendor,
protocol=protocol,
status=400,
body=_catalog_owner_status_body(vendor, 400),
listing="presence",
interface_key="a-credential-this-source-does-not-carry",
)
def observation(result: tuple[list[tuple[str, str]], object]) -> tuple[object, ...]:
requests, observed = result
return (
requests,
observed.outcome,
observed.protocol,
observed.authenticated,
observed.model_ids,
)
assert observation(validated) == observation(unread)
assert unread[1].outcome.value == "observed"
assert unread[1].authenticated is True
def test_custom_auto_numeric_auth_failure_message_stays_authentication_failed(
tmp_path: Path,
) -> None:
state_store = EngineStateStore(tmp_path / "engine-state-custom-auto")
adapter = CLIProxyEngineAdapter(
supervisor=Mock(),
state_store=state_store,
)
async def scenario() -> tuple[list[str], object]:
requests: list[str] = []
async def capture_probe(request: web.Request) -> web.Response:
requests.append(request.path)
return web.json_response(
{
"error": {
"code": 400,
"message": "invalid API key",
}
},
status=400,
)
app = web.Application()
app.router.add_post("/{tail:.*}", capture_probe)
runner = web.AppRunner(app)
await runner.setup()
site = web.TCPSite(runner, "127.0.0.1", 0)
await site.start()
assert site._server is not None
port = site._server.sockets[0].getsockname()[1]
origin = f"http://127.0.0.1:{port}"
credential_ref = state_store.store_api_key(
"test-custom-auto-invalid-key",
vendor="custom",
protocol="openai_chat",
base_url=origin,
)
try:
with patch(
"vibe.model_hub_runtime.adapter.probe_models",
new=AsyncMock(return_value=(DiscoveredModel(id="should-not-discover"),)),
) as inventory_probe:
observed = await adapter.observe_source(
"custom",
origin,
credential_ref,
_MODEL_FREE_OBSERVATION_PROTOCOLS,
)
inventory_probe.assert_not_awaited()
finally:
await runner.cleanup()
return requests, observed
requests, observed = asyncio.run(scenario())
assert observed.outcome.value == "authentication_failed"
assert observed.protocol is None
assert observed.authenticated is False
assert observed.model_ids == ()
assert requests == [
_PROTOCOL_OBSERVATION_TAXONOMY[protocol].request_path
for protocol in _MODEL_FREE_OBSERVATION_PROTOCOLS
]
@pytest.mark.parametrize(
("vendor", "base_url", "credential_vendor", "credential_protocol", "protocol_order"),
[
("deepseek", None, "deepseek", "openai_chat", ("openai_chat",)),
(
"custom",
"https://api.deepseek.com",
"custom",
"openai_chat",
("openai_chat",),
),
],
ids=("catalog_pin", "custom_declared"),
)
def test_source_observation_catalog_pin_and_declaration_still_require_authentication(
tmp_path: Path,
vendor: str,
base_url: str | None,
credential_vendor: str,
credential_protocol: str,
protocol_order: tuple[str, ...],
) -> None:
state_store = EngineStateStore(tmp_path / "engine-state")
credential_ref = state_store.store_api_key(
"test-observation-key",
vendor=credential_vendor,
protocol=credential_protocol,
base_url=base_url,
)
adapter = CLIProxyEngineAdapter(
supervisor=Mock(),
state_store=state_store,
)
rejected = _ProtocolEvidence(
protocol=_ProtocolProof.UNPROVEN,
authentication=_AuthenticationEvidence.REJECTED,
)
with (
patch(
"vibe.model_hub_runtime.adapter._probe_protocol_response",
new=AsyncMock(return_value=rejected),
) as protocol_probe,
patch(
"vibe.model_hub_runtime.adapter.probe_models",
new=AsyncMock(return_value=(DiscoveredModel(id="should-not-discover"),)),
) as inventory_probe,
):
failed = asyncio.run(
adapter.observe_source(
vendor,
base_url,
credential_ref,
protocol_order,
)
)
assert failed.outcome.value == "authentication_failed"
assert failed.protocol is None
assert failed.authenticated is False
assert [call.kwargs["protocol"] for call in protocol_probe.await_args_list] == list(protocol_order)
inventory_probe.assert_not_awaited()
@pytest.mark.parametrize(
("status", "initial_authentication", "expected_outcome", "expected_authenticated"),
[
(400, _AuthenticationEvidence.REJECTED, "authentication_failed", False),
(400, _AuthenticationEvidence.UNKNOWN, "ambiguous", None),
(401, _AuthenticationEvidence.REJECTED, "authentication_failed", False),
(400, _AuthenticationEvidence.ACCEPTED, "observed", True),
],
ids=("explicit_rejection", "unverified_validation", "auth_error_rejects", "verified_validation"),
)
def test_custom_declared_observation_preserves_transport_authentication_verdict(
tmp_path: Path,
status: int,
initial_authentication: _AuthenticationEvidence,
expected_outcome: str,
expected_authenticated: bool | None,
) -> None:
base_url = "https://relay.example/v1"
state_store = EngineStateStore(tmp_path / f"engine-state-custom-declared-{status}")
credential_ref = state_store.store_api_key(
f"test-custom-declared-{status}",
vendor="custom",
protocol="openai_chat",
base_url=base_url,
)
adapter = CLIProxyEngineAdapter(
supervisor=Mock(),
state_store=state_store,
)
parser_evidence = _ProtocolEvidence(
protocol=_ProtocolProof.UNPROVEN,
authentication=initial_authentication,
shape=_ProtocolObservationShape.GENERIC_REQUEST_ERROR,
status=status,
)
with (
patch(
"vibe.model_hub_runtime.adapter._probe_protocol_response",
new=AsyncMock(return_value=parser_evidence),
) as protocol_probe,
patch(
# This listing answers with or without a credential, so it attests
# to nobody and cannot repair an unknown verdict.
"vibe.model_hub_runtime.adapter.probe_models",
new=AsyncMock(return_value=(DiscoveredModel(id="declared-model"),)),
) as inventory_probe,
):
observed = asyncio.run(
adapter.observe_source(
"custom",
base_url,
credential_ref,
("openai_chat",),
)
)
assert observed.outcome.value == expected_outcome
assert observed.authenticated is expected_authenticated
assert [call.kwargs["protocol"] for call in protocol_probe.await_args_list] == ["openai_chat"]
if expected_outcome == "observed":
assert observed.protocol == "openai_chat"
assert observed.model_ids == ("declared-model",)
assert inventory_probe.await_args is not None
assert inventory_probe.await_args.kwargs["protocol"] == "openai_chat"
else:
assert observed.protocol is None
# An unknown verdict is the only one the listing may still answer: a
# rejection is already decided, so nothing further is asked.
asked_listing = initial_authentication is _AuthenticationEvidence.UNKNOWN
assert (inventory_probe.await_count > 0) is asked_listing
@pytest.mark.parametrize("stored_key", ["relay-live-key", "relay-wrong-key"])
def test_declared_custom_anthropic_relay_verifies_from_its_models_listing(
tmp_path: Path,
stored_key: str,
) -> None:
"""The reported relay, recorded from a live one: a declared `anthropic`
endpoint that answers the model-less probe with Anthropic's own 400
envelope -- which proves the interface and says nothing about the key --
and gates `GET /v1/models` on that key.
The listing is therefore what separates the working credential from the
wrong one, on an interface whose probe can only ever be a request error.
"""
live_key = "relay-live-key"
state_store = EngineStateStore(tmp_path / f"engine-state-relay-{stored_key}")
adapter = CLIProxyEngineAdapter(supervisor=Mock(), state_store=state_store)
async def scenario() -> tuple[list[tuple[str, str, str]], list[dict], object]:
requests: list[tuple[str, str, str]] = []
probe_bodies: list[dict] = []
def supplied(request: web.Request) -> str:
# The engine sends a custom relay's key as Bearer, so observation does too.
return request.headers.get("Authorization", "").removeprefix("Bearer ")
async def relay_messages(request: web.Request) -> web.Response:
requests.append((request.method, request.path, supplied(request)))
probe_bodies.append(await request.json())
if supplied(request) != live_key:
return web.json_response(
{
"type": "error",
"error": {
"type": "authentication_error",
"message": "invalid x-api-key",
},
},
status=401,
)
return web.json_response(
{
"type": "error",
"error": {
"type": "invalid_request_error",
"message": "model: Field required",
},
},
status=400,
)
async def relay_models(request: web.Request) -> web.Response:
requests.append((request.method, request.path, supplied(request)))
if supplied(request) != live_key:
return web.json_response(
{
"type": "error",
"error": {
"type": "authentication_error",
"message": "invalid x-api-key",
},
},
status=401,
)
return web.json_response(
{"data": [{"id": "claude-sonnet-4-5"}, {"id": "claude-opus-4-6"}]}
)
app = web.Application()
app.router.add_get("/v1/models", relay_models)
app.router.add_post("/{tail:.*}", relay_messages)
runner = web.AppRunner(app)
await runner.setup()
site = web.TCPSite(runner, "127.0.0.1", 0)
await site.start()
assert site._server is not None
origin = f"http://127.0.0.1:{site._server.sockets[0].getsockname()[1]}"
credential_ref = state_store.store_api_key(
stored_key,
vendor="custom",
protocol="anthropic",
base_url=origin,
)
try:
observed = await adapter.observe_source(
"custom",
origin,
credential_ref,
("anthropic",),
)
finally:
await runner.cleanup()
return requests, probe_bodies, observed
requests, probe_bodies, observed = asyncio.run(scenario())
probe_path = _PROTOCOL_OBSERVATION_TAXONOMY["anthropic"].request_path
assert all("model" not in body for body in probe_bodies)
if stored_key == live_key:
assert observed.outcome.value == "observed"
assert observed.protocol == "anthropic"
assert observed.authenticated is True
assert observed.model_ids == ("claude-sonnet-4-5", "claude-opus-4-6")
assert requests == [
("POST", probe_path, live_key),
("GET", "/v1/models", live_key),
# The control carries no credential at all, which is how an
# answered listing is told apart from a published one.
("GET", "/v1/models", ""),
]
else:
assert observed.outcome.value == "authentication_failed"
assert observed.authenticated is False
assert observed.protocol is None
assert observed.model_ids == ()
assert requests == [("POST", probe_path, stored_key)]
def test_auth_setup_executable_scenarios_are_registered() -> None:
root = Path(__file__).parent / "scenarios/auth_setup"
catalog = yaml.safe_load((root / "catalog.yaml").read_text())
registered = {entry["id"]: entry for entry in catalog["scenarios"]}
tree = ast.parse((root / "test_auth_setup_scenarios.py").read_text())
for node in ast.walk(tree):
if not isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):
continue
doc = ast.get_docstring(node) or ""
match = re.match(r"Scenario:\s+(AUTH-SETUP-\d+)\b", doc)
if match:
scenario_id = match.group(1)
assert scenario_id in registered, (scenario_id, node.name)
evidence = registered[scenario_id]["test"]
references = evidence if isinstance(evidence, list) else [evidence]
assert any(reference.endswith("::" + node.name) for reference in references)
@pytest.mark.parametrize("protocol", _MODEL_FREE_OBSERVATION_PROTOCOLS)
@pytest.mark.parametrize("status", [401, 403])
@pytest.mark.parametrize("body", ['{"message":"Regional policy"}', '{"error":{"code":"invalid_api_key"}}'])
def test_authentication_status_interpretation_preserves_evidence_role(
protocol, status, body,
) -> None:
evidence = _parse_protocol_authenticated_evidence(protocol, status, body)
expected = (
_AuthenticationEvidence.REJECTED
if "invalid_api_key" in body
else _AuthenticationEvidence.UNKNOWN
)
assert evidence.authentication is expected
@pytest.mark.parametrize("vendor", ["openai", "codex"])
@pytest.mark.parametrize("error_param", ["input", "model", None])
def test_oauth_observation_uses_the_bound_auth_index_and_requires_response_proof(
tmp_path: Path,
vendor: str,
error_param: str | None,
) -> None:
state_store = EngineStateStore(tmp_path / "engine-state")
credential_ref = state_store.bind_oauth_credential(
"src_oauthproof",
vendor,
"codex-test.json",
)
client = Mock()
api_calls: list[dict] = []
def management_request(method, path, *, query=None, payload=None):
if path == "/auth-files":
return {
"files": [
{
"id": "codex-test",
"auth_index": "auth-index-test",
"name": "codex-test.json",
"provider": "codex",
"id_token": {"chatgpt_account_id": "account-test"},
}
]
}
if path == "/api-call":
api_calls.append(payload)
assert payload["header"]["Authorization"] == "Bearer $TOKEN$"
return {
"status_code": 400,
"header": {},
"body": json.dumps(
{
"error": {
"type": "invalid_request_error",
"param": error_param,
}
}
),
}
if path == "/auth-files/models":
assert query == {"name": "codex-test.json"}
return {"models": [{"id": "gpt-5.6"}]}
raise AssertionError((method, path))
client.management_request.side_effect = management_request
supervisor = Mock()
supervisor.client.return_value = client
adapter = CLIProxyEngineAdapter(
supervisor=supervisor,
state_store=state_store,
)
observed = asyncio.run(
adapter.observe_source(
vendor,
None,
credential_ref,
_MODEL_FREE_OBSERVATION_PROTOCOLS,
)
)
assert observed.outcome.value == ("adapter_error" if error_param == "input" else "ambiguous")
assert observed.authenticated is None
assert observed.protocol is None
assert observed.model_ids == ()
assert len(api_calls) == 1
assert api_calls[0]["auth_index"] == "auth-index-test"
assert api_calls[0]["header"]["Chatgpt-Account-Id"] == "account-test"
assert api_calls[0]["url"].endswith("/responses")
assert json.loads(api_calls[0]["data"]) == dict(
_PROTOCOL_OBSERVATION_TAXONOMY["openai_responses"].request_body
)
def ambiguous_management_request(method, path, *, query=None, payload=None):
if path == "/auth-files":
return management_request(method, path, query=query, payload=payload)
if path == "/api-call":
return {"status_code": unknown_status, "header": {}, "body": unknown_body}
raise AssertionError((method, path))
client.management_request.side_effect = ambiguous_management_request
for unknown_status, unknown_body in (
(200, '{"ok":true}'),
(401, "<html>Request blocked</html>"),
(403, '{"message":"Regional policy"}'),
):
ambiguous = asyncio.run(
adapter.observe_source(
vendor,
None,
credential_ref,
_MODEL_FREE_OBSERVATION_PROTOCOLS,
)
)
assert ambiguous.outcome.value == "ambiguous"
assert ambiguous.protocol is None
assert ambiguous.authenticated is None
def test_oauth_observation_does_not_use_api_key_catalog_pins_without_shape_proof(
tmp_path: Path,
) -> None:
state_store = EngineStateStore(tmp_path / "engine-state")
credential_ref = state_store.bind_oauth_credential(
"src_oauthcatalog",
"openai",
"codex-test.json",
)
client = Mock()
def management_request(method, path, *, query=None, payload=None):
if path == "/auth-files":
return {
"files": [
{
"id": "codex-test",
"auth_index": "auth-index-test",
"name": "codex-test.json",
"provider": "codex",
}
]
}
raise AssertionError((method, path))
client.management_request.side_effect = management_request
supervisor = Mock()
supervisor.client.return_value = client
adapter = CLIProxyEngineAdapter(
supervisor=supervisor,
state_store=state_store,
)
accepted_unproven = _ProtocolEvidence(
protocol=_ProtocolProof.UNPROVEN,
authentication=_AuthenticationEvidence.ACCEPTED,
shape=_ProtocolObservationShape.GENERIC_REQUEST_ERROR,
)
with (
patch(
"vibe.model_hub_runtime.adapter._probe_oauth_protocol_response",
return_value=accepted_unproven,
) as oauth_probe,
patch.object(
adapter,
"discover_models",
new=AsyncMock(return_value=(DiscoveredModel(id="should-not-discover"),)),
) as discover_models,
):
ambiguous = asyncio.run(
adapter.observe_source(
"openai",
None,
credential_ref,
_MODEL_FREE_OBSERVATION_PROTOCOLS,
)
)
assert ambiguous.outcome.value == "ambiguous"
assert ambiguous.protocol is None
assert ambiguous.authenticated is True
discover_models.assert_not_awaited()
assert [call.kwargs["protocol"] for call in oauth_probe.call_args_list] == list(_MODEL_FREE_OBSERVATION_PROTOCOLS)
@pytest.mark.parametrize("vendor", sorted(_HUB_SUBSCRIPTION_PROTOCOLS))
def test_hub_subscription_observation_binds_on_the_engine_declared_protocol(
tmp_path: Path,
vendor: str,
) -> None:
"""A pinned subscription binds without a probe, and asks for nothing else.
The engine holds the credential and serves it on one surface, so there is no
upstream Avibe may synthesize a request against — the completed grant is the
reachability and authentication evidence, and the pin supplies the protocol.
Seeding the whole pin table keeps a vendor added later covered without
editing this test.
"""
state_store = EngineStateStore(tmp_path / "engine-state")
credential_ref = state_store.bind_oauth_credential(
"src_hubsubscript",
vendor,
f"{vendor}-test.json",
)
client = Mock()
def management_request(method, path, *, query=None, payload=None):
if path == "/auth-files":
return {
"files": [
{
"id": f"{vendor}-test",
"auth_index": "auth-index-test",
"name": f"{vendor}-test.json",
"provider": vendor,
}
]
}
raise AssertionError((method, path))
client.management_request.side_effect = management_request
supervisor = Mock()
supervisor.client.return_value = client
adapter = CLIProxyEngineAdapter(
supervisor=supervisor,
state_store=state_store,
)
with (
patch(
"vibe.model_hub_runtime.adapter._probe_oauth_protocol_response",
side_effect=AssertionError("a pinned subscription must not be probed"),
),
patch.object(
adapter,
"discover_models",
new=AsyncMock(return_value=(DiscoveredModel(id=f"{vendor}-model"),)),
) as discover_models,
):
observed = asyncio.run(
adapter.observe_source(
vendor,
None,
credential_ref,
_MODEL_FREE_OBSERVATION_PROTOCOLS,
)
)
pinned = hub_subscription_serving_protocol(vendor)
assert observed.outcome.value == "observed"
assert observed.reachable is True
assert observed.authenticated is True
assert observed.protocol == pinned
assert observed.model_ids == (f"{vendor}-model",)
assert discover_models.await_args.args[:3] == (vendor, pinned, None)
@pytest.mark.parametrize("vendor", sorted(_HUB_SUBSCRIPTION_PROTOCOLS))
def test_hub_subscription_projects_on_its_pin_whatever_the_api_key_pin_is(
tmp_path: Path,
vendor: str,
) -> None:
"""A pinned subscription projects on its own protocol, with no base URL.
The engine holds the credential and serves it on the surface this table
names, so nothing about the projection depends on the api-key catalog: that
describes the other channel, Avibe calling the vendor's public API with a
key it holds, and the two may legitimately disagree for one vendor id.
Seeding the whole pin table asserts every row projects on its own terms
rather than by agreeing with its sibling — and because the projection is
atomic, a row that stopped being admissible would take every other Source
down with it rather than fail alone.
"""
state_store = EngineStateStore(tmp_path / "engine-state")
credential_ref = state_store.bind_oauth_credential(
"src_hubsubscript",
vendor,
f"{vendor}-test.json",
)
protocol = hub_subscription_serving_protocol(vendor)
assert protocol in _MODEL_FREE_OBSERVATION_PROTOCOLS
projected = state_store.sync_sources(
[
SourceBinding(
source_id="src_hubsubscript",
vendor=vendor,
protocol=protocol,
base_url=None,
credential_ref=credential_ref,
allowed_origins=("main",),
model_ids=(f"{vendor}-model",),
)
]
)
assert [(record.vendor, record.protocol, record.base_url) for record in projected] == [
(vendor, protocol, None)
]
@pytest.mark.parametrize("vendor", sorted(_OAUTH_ENDPOINTS))
def test_oauth_source_is_seeded_with_its_vendors_catalog_label(vendor: str) -> None:
"""A vendor id is a routing key, so no Source may be named by one we can name.
Seeded over the whole start table rather than the vendors this change adds:
an OAuth vendor the catalog already lists cannot be admitted and then ship
named `xai` beside an api-key Source of the same vendor reading `xAI`. The
id survives as the seed only for a vendor the catalog does not list, which
is the only name that channel has for it.
"""
label = catalog_api_key_vendor_label(vendor)
assert seeded_source_name(vendor) == (label if label is not None else vendor)
DEEPSEEK_AUTHENTICATION_ERROR_PAYLOAD = {
"error": {
"message": "Authentication Fails, Your api key: **** is invalid",
"type": "authentication_error",
"param": None,
"code": "invalid_request_error",
}
}
DEEPSEEK_MODEL_NOT_FOUND_PAYLOAD = {
"error": {
"type": "invalid_request_error",
"param": None,
"code": "model_not_found",
}
}
ANTHROPIC_RELAY_REQUEST_ERROR_PAYLOAD = {
"error": {
"message": "model is required",
"type": "invalid_request_error",
}
}
@pytest.mark.parametrize(
("protocol", "success_body", "request_error_body", "auth_error_body"),
[
(
"anthropic",
{"type": "message"},
{"type": "error", "error": {"type": "invalid_request_error"}},
{"type": "error", "error": {"type": "authentication_error"}},
),
(
"openai_responses",
{"object": "response"},
{"error": {"type": "invalid_request_error", "param": "input"}},
{
"error": {
"type": "invalid_request_error",
"code": "invalid_api_key",
"param": "input",
}
},
),
(
"openai_chat",
{"object": "chat.completion"},
{"error": {"type": "invalid_request_error", "param": "messages"}},
{
"error": {
"type": "invalid_request_error",
"code": "invalid_api_key",
"param": "messages",
}
},
),
],
)
def test_protocol_evidence_parser_requires_candidate_specific_response_shapes(
protocol: str,
success_body: dict,
request_error_body: dict,
auth_error_body: dict,
) -> None:
assert _parse_protocol_authenticated_evidence(
protocol,
200,
json.dumps(success_body),
) == _ProtocolEvidence(
protocol=_ProtocolProof.PROVEN,
authentication=_AuthenticationEvidence.ACCEPTED,
)
assert _parse_protocol_authenticated_evidence(
protocol,
400,
json.dumps(request_error_body),
) == _ProtocolEvidence(
protocol=_ProtocolProof.PROVEN,
authentication=_AuthenticationEvidence.UNKNOWN,
)
assert _parse_protocol_authenticated_evidence(
protocol,
401,
json.dumps(auth_error_body),
) == _ProtocolEvidence(
protocol=_ProtocolProof.PROVEN,
authentication=_AuthenticationEvidence.REJECTED,
)
assert _parse_protocol_authenticated_evidence(
protocol,
400,
"{}",
) == _ProtocolEvidence(
protocol=_ProtocolProof.UNPROVEN,
authentication=_AuthenticationEvidence.UNKNOWN,
shape=_ProtocolObservationShape.UNSTRUCTURED,
)
@pytest.mark.parametrize(
("status", "body", "expected"),
[
(
400,
ANTHROPIC_RELAY_REQUEST_ERROR_PAYLOAD,
_ProtocolEvidence(
protocol=_ProtocolProof.UNPROVEN,
authentication=_AuthenticationEvidence.UNKNOWN,
shape=_ProtocolObservationShape.GENERIC_REQUEST_ERROR,
),
),
(
401,
{"error": {"type": "authentication_error"}},
_ProtocolEvidence(
protocol=_ProtocolProof.UNPROVEN,
authentication=_AuthenticationEvidence.REJECTED,
),
),
(
400,
{"error": {"type": "future_error"}},
_ProtocolEvidence(
protocol=_ProtocolProof.UNPROVEN,
authentication=_AuthenticationEvidence.UNKNOWN,
),
),
],
)
def test_anthropic_relay_openai_style_error_wrapper_preserves_canonical_semantics(
status: int,
body: dict,
expected: _ProtocolEvidence,
) -> None:
assert _parse_protocol_authenticated_evidence(
"anthropic",
status,
json.dumps(body),
) == expected
@pytest.mark.parametrize(
("protocol", "status", "body"),
[
(
"anthropic",
500,
{"type": "error", "error": {"type": "api_error"}},
),
(
"openai_responses",
500,
{"error": {"type": "server_error"}},
),
(
"openai_chat",
500,
{"error": {"type": "server_error"}},
),
(
"anthropic",
429,
{"type": "error", "error": {"type": "rate_limit_error"}},
),
(
"openai_responses",
429,
{"error": {"type": "rate_limit_exceeded"}},
),
(
"openai_chat",
429,
{"error": {"type": "rate_limit_exceeded"}},
),
(
"anthropic",
400,
{"type": "error", "error": {"type": "future_error"}},
),
(
"openai_responses",
400,
{"error": {"type": "future_error", "param": "input"}},
),
(
"openai_chat",
400,
{"error": {"type": "future_error", "param": "messages"}},
),
],
)
def test_protocol_evidence_table_defaults_shaped_non_auth_rows_to_unknown(
protocol: str,
status: int,
body: dict,
) -> None:
assert _parse_protocol_authenticated_evidence(
protocol,
status,
json.dumps(body),
) == _ProtocolEvidence(
protocol=_ProtocolProof.PROVEN,
authentication=_AuthenticationEvidence.UNKNOWN,
)
@pytest.mark.parametrize(
("protocol", "status", "body"),
[
(
"anthropic",
404,
{"type": "error", "error": {"type": "not_found_error"}},
),
],
)
def test_anthropic_protocol_model_errors_leave_authentication_unknown(
protocol: str,
status: int,
body: dict,
) -> None:
assert _parse_protocol_authenticated_evidence(
protocol,
status,
json.dumps(body),
) == _ProtocolEvidence(
protocol=_ProtocolProof.PROVEN,
authentication=_AuthenticationEvidence.UNKNOWN,
)
@pytest.mark.parametrize(
("protocol", "status", "body"),
[
(
"openai_responses",
404,
{"error": {"type": "invalid_request_error", "code": "model_not_found"}},
),
(
"openai_chat",
404,
{"error": {"type": "invalid_request_error", "code": "model_not_found"}},
),
(
"openai_responses",
400,
DEEPSEEK_MODEL_NOT_FOUND_PAYLOAD,
),
(
"openai_chat",
400,
DEEPSEEK_MODEL_NOT_FOUND_PAYLOAD,
),
(
"openai_responses",
422,
DEEPSEEK_MODEL_NOT_FOUND_PAYLOAD,
),
(
"openai_chat",
422,
DEEPSEEK_MODEL_NOT_FOUND_PAYLOAD,
),
],
)
def test_openai_model_errors_without_family_param_leave_authentication_unknown(
protocol: str,
status: int,
body: dict,
) -> None:
assert _parse_protocol_authenticated_evidence(
protocol,
status,
json.dumps(body),
) == _ProtocolEvidence(
protocol=_ProtocolProof.UNPROVEN,
authentication=_AuthenticationEvidence.UNKNOWN,
shape=_ProtocolObservationShape.GENERIC_REQUEST_ERROR,
)
@pytest.mark.parametrize("protocol", ("openai_responses", "openai_chat"))
def test_openai_request_error_without_family_param_leaves_authentication_unknown(
protocol: str,
) -> None:
assert _parse_protocol_authenticated_evidence(
protocol,
400,
json.dumps(
{
"error": {
"type": "invalid_request_error",
"param": None,
}
}
),
) == _ProtocolEvidence(
protocol=_ProtocolProof.UNPROVEN,
authentication=_AuthenticationEvidence.UNKNOWN,
shape=_ProtocolObservationShape.GENERIC_REQUEST_ERROR,
)
def test_qwen_wrapperless_validation_leaves_authentication_unknown() -> None:
assert _parse_protocol_authenticated_evidence(
"openai_chat",
400,
json.dumps(
{
"code": "InvalidParameter",
"message": "messages is required",
}
),
) == _ProtocolEvidence(
protocol=_ProtocolProof.UNPROVEN,
authentication=_AuthenticationEvidence.UNKNOWN,
shape=_ProtocolObservationShape.GENERIC_REQUEST_ERROR,
)
@pytest.mark.parametrize("protocol", ("openai_responses", "openai_chat"))
def test_openai_family_nested_numeric_request_error_stays_unknown_before_owner_override(
protocol: str,
) -> None:
assert _parse_protocol_authenticated_evidence(
protocol,
400,
json.dumps(
{
"error": {
"code": 400,
"message": "missing required parameter",
}
}
),
) == _ProtocolEvidence(
protocol=_ProtocolProof.UNPROVEN,
authentication=_AuthenticationEvidence.UNKNOWN,
)
assert _parse_protocol_authenticated_evidence(
protocol,
400,
json.dumps(
{
"error": {
"code": 400,
"message": "missing required parameter",
}
}
),
vendor="openrouter",
request_root="https://openrouter.ai/api/v1",
) == _ProtocolEvidence(
protocol=_ProtocolProof.UNPROVEN,
authentication=_AuthenticationEvidence.UNKNOWN,
)
@pytest.mark.parametrize("protocol", ("openai_responses", "openai_chat"))
def test_openai_family_nested_numeric_auth_failure_message_is_rejected(
protocol: str,
) -> None:
assert _parse_protocol_authenticated_evidence(
protocol,
400,
json.dumps(
{
"error": {
"code": 400,
"message": "invalid API key",
}
}
),
) == _ProtocolEvidence(
protocol=_ProtocolProof.UNPROVEN,
authentication=_AuthenticationEvidence.REJECTED,
)
@pytest.mark.parametrize("protocol", _MODEL_FREE_OBSERVATION_PROTOCOLS)
@pytest.mark.parametrize("status", (400, 422))
def test_request_error_with_credential_param_is_rejected(
protocol: str,
status: int,
) -> None:
assert _parse_protocol_authenticated_evidence(
protocol,
status,
json.dumps(
{
"error": {
"type": "invalid_request_error",
"param": "api_key",
}
}
),
) == _ProtocolEvidence(
protocol=_ProtocolProof.UNPROVEN,
authentication=_AuthenticationEvidence.REJECTED,
)
def test_shared_openai_authentication_rejection_does_not_prove_a_protocol() -> None:
body = json.dumps(
{
"error": {
"type": "invalid_request_error",
"code": "invalid_api_key",
"param": None,
}
}
)
for protocol in ("openai_responses", "openai_chat"):
assert _parse_protocol_authenticated_evidence(
protocol,
401,
body,
) == _ProtocolEvidence(
protocol=_ProtocolProof.UNPROVEN,
authentication=_AuthenticationEvidence.REJECTED,
)
@pytest.mark.parametrize("protocol", ("openai_responses", "openai_chat"))
def test_deepseek_authentication_rejection_with_null_param_stays_unproven(
protocol: str,
) -> None:
assert _parse_protocol_authenticated_evidence(
protocol,
401,
json.dumps(DEEPSEEK_AUTHENTICATION_ERROR_PAYLOAD),
) == _ProtocolEvidence(
protocol=_ProtocolProof.UNPROVEN,
authentication=_AuthenticationEvidence.REJECTED,
)
def test_structured_but_unrecognized_openai_error_is_not_exclusion_evidence() -> None:
assert _parse_protocol_authenticated_evidence(
"openai_chat",
400,
json.dumps({"error": {"type": "future_error"}}),
) == _ProtocolEvidence(
protocol=_ProtocolProof.UNPROVEN,
authentication=_AuthenticationEvidence.UNKNOWN,
)
def test_request_error_404_and_non_json_responses_are_pairwise_exclusion_shapes() -> None:
assert _parse_protocol_authenticated_evidence(
"openai_responses",
404,
json.dumps({}),
) == _ProtocolEvidence(
protocol=_ProtocolProof.UNPROVEN,
authentication=_AuthenticationEvidence.UNKNOWN,
shape=_ProtocolObservationShape.HTTP_404,
)
assert _parse_protocol_authenticated_evidence(
"openai_responses",
400,
"<html>route not found</html>",
) == _ProtocolEvidence(
protocol=_ProtocolProof.UNPROVEN,
authentication=_AuthenticationEvidence.UNKNOWN,
shape=_ProtocolObservationShape.NON_JSON,
)
assert _parse_protocol_authenticated_evidence(
"openai_responses",
502,
"<html>bad gateway</html>",
) == _ProtocolEvidence(
protocol=_ProtocolProof.UNPROVEN,
authentication=_AuthenticationEvidence.UNKNOWN,
)
def test_top_level_authentication_rejection_is_classified_without_forging_protocol() -> None:
body = json.dumps({"code": "INVALID_API_KEY", "message": "Invalid API key"})
for protocol in _MODEL_FREE_OBSERVATION_PROTOCOLS:
assert _parse_protocol_authenticated_evidence(
protocol,
401,
body,
) == _ProtocolEvidence(
protocol=_ProtocolProof.UNPROVEN,
authentication=_AuthenticationEvidence.REJECTED,
)
def test_protocol_observation_preserves_query_on_each_distinct_upstream_path() -> None:
query = "api-version=2026-07-23"
async def scenario() -> list[tuple[str, str, dict]]:
requests: list[tuple[str, str, dict]] = []
async def reject_empty_probe(request: web.Request) -> web.Response:
requests.append((request.path, request.query_string, await request.json()))
return web.json_response({"error": {"type": "invalid_request"}}, status=400)
app = web.Application()
app.router.add_post("/{tail:.*}", reject_empty_probe)
runner = web.AppRunner(app)
await runner.setup()
site = web.TCPSite(runner, "127.0.0.1", 0)
await site.start()
assert site._server is not None
port = site._server.sockets[0].getsockname()[1]
try:
for protocol in _MODEL_FREE_OBSERVATION_PROTOCOLS:
await _probe_protocol_response(
vendor="custom",
protocol=protocol,
base_url=f"http://127.0.0.1:{port}/v1?{query}",
secret="test-observation-key",
)
finally:
await runner.cleanup()
return requests
requests = asyncio.run(scenario())
paths = [path for path, _query, _body in requests]
assert len(paths) == len(_MODEL_FREE_OBSERVATION_PROTOCOLS)
assert len(set(paths)) == len(paths)
assert {request_query for _path, request_query, _body in requests} == {query}
# Model-free bodies fail request validation before a relay schedules capacity.
assert dict(zip(_MODEL_FREE_OBSERVATION_PROTOCOLS, (body for _path, _query, body in requests))) == {
"anthropic": {"max_tokens": 0, "messages": []},
"openai_responses": {},
"openai_chat": {},
}
@pytest.mark.parametrize("status", (200, 401))
def test_protocol_observation_classifies_from_the_body_not_the_status(status) -> None:
"""A status code alone neither proves an interface nor authenticates a key."""
async def scenario() -> list:
async def shapeless(_request: web.Request) -> web.Response:
return web.json_response({"status": "ok"}, status=status)
app = web.Application()
app.router.add_post("/{tail:.*}", shapeless)
runner = web.AppRunner(app)
await runner.setup()
site = web.TCPSite(runner, "127.0.0.1", 0)
await site.start()
assert site._server is not None
port = site._server.sockets[0].getsockname()[1]
try:
return [
await _probe_protocol_response(
vendor="custom",
protocol=protocol,
base_url=f"http://127.0.0.1:{port}/v1",
secret="test-observation-key",
)
for protocol in _MODEL_FREE_OBSERVATION_PROTOCOLS
]
finally:
await runner.cleanup()
for evidence in asyncio.run(scenario()):
assert evidence.protocol is _ProtocolProof.UNPROVEN
assert evidence.authentication is _AuthenticationEvidence.UNKNOWN
@pytest.mark.parametrize("status", (200, 401))
@pytest.mark.parametrize(
("vendor", "protocol"),
(("anthropic", "anthropic"), ("openai", "openai_responses"), ("codex", "openai_responses")),
)
def test_oauth_protocol_observation_classifies_from_the_body_not_the_status(vendor, protocol, status) -> None:
"""The engine-held OAuth probe sends the model-free body and reads the shape, not the status."""
class Client:
def __init__(self) -> None:
self.payloads: list[dict] = []
def management_request(self, method, path, *, query=None, payload=None, timeout=None):
self.payloads.append(payload)
return {"status_code": status, "body": json.dumps({"status": "ok"})}
client = Client()
evidence = _probe_oauth_protocol_response(
client=client, # type: ignore[arg-type]
auth=_AuthRecord(
identity="account.json",
auth_index="0",
name="account.json",
provider=vendor,
fingerprint="fp",
),
vendor=vendor,
protocol=protocol,
)
assert [json.loads(payload["data"]) for payload in client.payloads] == [
{"anthropic": {"max_tokens": 0, "messages": []}, "openai_responses": {}}[protocol]
]
assert evidence.protocol is _ProtocolProof.UNPROVEN
assert evidence.authentication is _AuthenticationEvidence.UNKNOWN
def test_protocol_observation_adds_standard_v1_paths_to_a_bare_origin() -> None:
async def scenario() -> list[str]:
requests: list[str] = []
async def capture_probe(request: web.Request) -> web.Response:
requests.append(request.path)
return web.json_response({"error": {"type": "invalid_request"}}, status=400)
app = web.Application()
app.router.add_post("/{tail:.*}", capture_probe)
runner = web.AppRunner(app)
await runner.setup()
site = web.TCPSite(runner, "127.0.0.1", 0)
await site.start()
assert site._server is not None
port = site._server.sockets[0].getsockname()[1]
try:
for protocol in _MODEL_FREE_OBSERVATION_PROTOCOLS:
await _probe_protocol_response(
vendor="custom",
protocol=protocol,
base_url=f"http://127.0.0.1:{port}",
secret="test-observation-key",
)
finally:
await runner.cleanup()
return requests
assert asyncio.run(scenario()) == [
_PROTOCOL_OBSERVATION_TAXONOMY[protocol].request_path
for protocol in _MODEL_FREE_OBSERVATION_PROTOCOLS
]
def test_openai_probe_requests_are_mutually_distinguishable() -> None:
responses = _PROTOCOL_OBSERVATION_TAXONOMY["openai_responses"]
chat = _PROTOCOL_OBSERVATION_TAXONOMY["openai_chat"]
assert responses.request_path != chat.request_path
assert "input" not in responses.request_body
assert "messages" not in chat.request_body
assert _parse_protocol_authenticated_evidence(
"openai_responses",
400,
json.dumps({"error": {"type": "invalid_request_error", "param": "messages"}}),
) == _ProtocolEvidence(
protocol=_ProtocolProof.UNPROVEN,
authentication=_AuthenticationEvidence.UNKNOWN,
)
assert _parse_protocol_authenticated_evidence(
"openai_chat",
400,
json.dumps({"error": {"type": "invalid_request_error", "param": "input"}}),
) == _ProtocolEvidence(
protocol=_ProtocolProof.UNPROVEN,
authentication=_AuthenticationEvidence.UNKNOWN,
)
def test_model_discovery_preserves_query_when_appending_models_path() -> None:
query = "api-version=2026-07-23"
async def scenario() -> tuple[str, str, tuple[DiscoveredModel, ...]]:
request_target: tuple[str, str] | None = None
async def list_models(request: web.Request) -> web.Response:
nonlocal request_target
request_target = (request.path, request.query_string)
return web.json_response({"data": [{"id": "upstream-model"}]})
app = web.Application()
app.router.add_get("/{tail:.*}", list_models)
runner = web.AppRunner(app)
await runner.setup()
site = web.TCPSite(runner, "127.0.0.1", 0)
await site.start()
assert site._server is not None
port = site._server.sockets[0].getsockname()[1]
try:
models = await probe_models(
vendor="custom",
protocol="openai_chat",
base_url=f"http://127.0.0.1:{port}/v1?{query}",
secret="test-observation-key",
)
finally:
await runner.cleanup()
assert request_target is not None
return request_target[0], request_target[1], models
path, request_query, models = asyncio.run(scenario())
assert path == "/v1/models"
assert request_query == query
assert models == (DiscoveredModel(id="upstream-model"),)
def test_model_discovery_adds_standard_v1_path_to_a_bare_origin() -> None:
async def scenario() -> str:
request_path = ""
async def list_models(request: web.Request) -> web.Response:
nonlocal request_path
request_path = request.path
return web.json_response({"data": [{"id": "upstream-model"}]})
app = web.Application()
app.router.add_get("/{tail:.*}", list_models)
runner = web.AppRunner(app)
await runner.setup()
site = web.TCPSite(runner, "127.0.0.1", 0)
await site.start()
assert site._server is not None
port = site._server.sockets[0].getsockname()[1]
try:
await probe_models(
vendor="custom",
protocol="anthropic",
base_url=f"http://127.0.0.1:{port}",
secret="test-observation-key",
)
finally:
await runner.cleanup()
return request_path
assert asyncio.run(scenario()) == "/v1/models"
def test_blocked_exact_hop_emits_one_supply_interruption(tmp_path: Path) -> None:
source = _source("src_blocked01", "Blocked")
source.models.append(ModelHubModelConfig(id="removed-model", provenance="discovered", retired=True))
service = _service(tmp_path, sources=[source])
service.store.config.agents["claude"].routes["shared-model"] = ModelHubRouteConfig(
hops=(ModelHubRouteHopConfig(source.id, "removed-model"),)
)
router = ModelHubRuntimeRouter(service=service)
for _attempt in range(2):
with pytest.raises(ModelHubError):
asyncio.run(router.resolve("claude", "shared-model"))
events = [event for event in service.list_events(limit=20) if event["kind"] == "supply_interrupted"]
assert len(events) == 1
assert events[0]["reason"] == "model_unsupported"
@pytest.mark.parametrize(
("route", "native_ready", "reason"),
[
(ModelHubRouteConfig(), True, "route_unconfigured"),
(
ModelHubRouteConfig(hops=(ModelHubRouteHopConfig("src_missing01", "shared-model"),)),
True,
"source_missing",
),
],
)
def test_supply_interruption_preserves_exact_structural_reason(
tmp_path: Path,
route: ModelHubRouteConfig,
native_ready: bool,
reason: str,
) -> None:
source = _source("src_present01", "Present")
service = _service(tmp_path, sources=[source])
# A listed row with no Source behind it; an unlisted id is no model of this
# backend and records no event at all.
service.store.config.agents["claude"].models.append(ModelHubBackendModelConfig(id="shared-model"))
service.store.config.agents["claude"].sources.order = []
service.store.config.agents["claude"].routes["shared-model"] = route
router = ModelHubRuntimeRouter(
service=service,
native_cli_ready=lambda _backend: native_ready,
)
with pytest.raises(ModelHubError):
asyncio.run(router.resolve("claude", "shared-model"))
event = next(item for item in service.list_events(limit=20) if item["kind"] == "supply_interrupted")
assert event["reason"] == reason
_assert_valid("resolution-event.schema.json", event)
def test_supply_interruption_preserves_native_process_reason(tmp_path: Path) -> None:
source = _source(
"src_native011",
"Native",
channel="native_cli",
vendor="anthropic",
protocol="anthropic",
)
service = _service(tmp_path, sources=[source])
router = ModelHubRuntimeRouter(
service=service,
native_cli_ready=lambda _backend: False,
)
with pytest.raises(ModelHubError):
asyncio.run(router.resolve("claude", "shared-model"))
event = next(item for item in service.list_events(limit=20) if item["kind"] == "supply_interrupted")
assert event["reason"] == "native_cli_unavailable"
_assert_valid("resolution-event.schema.json", event)
def test_same_scope_concurrency_is_absent_and_sequential_control_is_present(
tmp_path: Path,
) -> None:
async def run() -> BoundedProvenanceStore:
store = BoundedProvenanceStore(tmp_path / "concurrency.json")
registry = TurnCorrelationRegistry(store)
runtime = SimpleNamespace(
settle_turn=lambda value, *, settled_by, ts, mode=None: registry.settle(
value,
settled_by=settled_by,
ts=ts,
)
)
controller = SimpleNamespace(
model_hub_runtime=runtime,
command_handler=SimpleNamespace(handle_stop=AsyncMock(return_value=True)),
set_agent_status=lambda _session_id, _status: None,
)
manager = SessionTurnManager(controller)
manager.flush_queue = AsyncMock(return_value=False)
started = {turn_id: asyncio.Event() for turn_id in ("turn_one", "turn_two", "turn_sequential")}
release = {turn_id: asyncio.Event() for turn_id in ("turn_one", "turn_two", "turn_sequential")}
async def dispatch(_controller, context, _text, **_kwargs):
turn_id = manager.model_hub_turn_id_for_task()
assert turn_id is not None
exact_turn = _begin_hub_attempt(
registry,
turn_id=turn_id,
scope="/shared-cwd",
)
started[turn_id].set()
await release[turn_id].wait()
success = _outcome(RawOutcomeKind.SUCCESS)
registry.finish_attempt(
exact_turn,
outcome=success,
decision=classify_outcome(success),
)
return TurnDispatchOutcome(
error=None,
settled_by=SETTLED_BY_TERMINAL_RESULT,
)
def context(turn_id: str):
return SimpleNamespace(
platform="avibe",
platform_specific={
"turn_token": turn_id,
"agent_session_target": {"agent_backend": "codex"},
},
)
with patch(
"core.session_turns.dispatch_turn_with_outcome",
side_effect=dispatch,
):
await manager._run("session-one", context("turn_one"), "one")
first_task = manager.in_flight["session-one"].task
await manager._run("session-two", context("turn_two"), "two")
second_task = manager.in_flight["session-two"].task
await started["turn_one"].wait()
await started["turn_two"].wait()
await manager.cancel("session-one")
release["turn_two"].set()
await asyncio.gather(
first_task,
second_task,
return_exceptions=True,
)
assert store.get("turn_one") is None
assert store.get("turn_two") is None
await manager._run(
"session-sequential",
context("turn_sequential"),
"sequential",
)
sequential_task = manager.in_flight["session-sequential"].task
await started["turn_sequential"].wait()
release["turn_sequential"].set()
await asyncio.gather(sequential_task, return_exceptions=True)
return store
store = asyncio.run(run())
sequential = store.get("turn_sequential")
assert sequential is not None
assert sequential["outcome"] == "served"
_assert_valid("turn-provenance.schema.json", sequential)
def test_chain_projection_and_probe_latency_partition(tmp_path: Path) -> None:
cooling = _source(
"src_cooling01",
"Cooling",
status="cooldown",
retry_at=(NOW + timedelta(minutes=5)).isoformat(),
)
primary = _source("src_primary01", "Primary")
service = _service(
tmp_path,
sources=[cooling, primary],
outcomes=[_outcome(RawOutcomeKind.SUCCESS)],
)
menu_model = _canonicalize_fixed_test_routes(service)["claude"]
chain = service.agent_chain("claude", menu_model)
assert [item["source_id"] for item in chain["chain"]] == [
"src_cooling01",
"src_primary01",
]
assert chain["supply_state"] == "ok"
assert chain["current"] == {
"source_id": "src_primary01",
"model_id": "shared-model",
}
assert all(item["channel"] == "hub" for item in chain["chain"])
assert all(item["reason"] is None for item in chain["chain"])
_assert_valid("agent-chain.schema.json", chain)
probe = asyncio.run(service.probe_agent("claude", menu_model))
assert probe["channel"] == "hub"
assert probe["reachable"] is True
assert probe["source_id"] == "src_primary01"
assert isinstance(probe["latency_ms"], int)
assert probe["error"] is None
_assert_valid("probe-result.schema.json", probe)
rate_service = _service(
tmp_path / "rate",
sources=[_source("src_primary01", "Primary")],
outcomes=[
_outcome(
RawOutcomeKind.HTTP_ERROR,
status=429,
code="rate_limited",
)
],
)
rate_model = _canonicalize_fixed_test_routes(rate_service)["claude"]
rate_limited = asyncio.run(rate_service.probe_agent("claude", rate_model))
assert rate_limited["reachable"] is False
assert isinstance(rate_limited["latency_ms"], int)
assert rate_limited["error"] == "models.source.cooldown.rate_limited"
_assert_valid("probe-result.schema.json", rate_limited)
unclassified_service = _service(
tmp_path / "unclassified",
sources=[_source("src_primary01", "Primary")],
outcomes=[
_outcome(
RawOutcomeKind.HTTP_ERROR,
status=418,
code="unexpected_upstream_failure",
)
],
)
unclassified_model = _canonicalize_fixed_test_routes(unclassified_service)["claude"]
unclassified = asyncio.run(unclassified_service.probe_agent("claude", unclassified_model))
assert unclassified["reachable"] is False
assert isinstance(unclassified["latency_ms"], int)
assert unclassified["error"] == "models.source.error.unclassified"
assert unclassified_service.store.load().sources[0].state.status == "error"
_assert_valid("probe-result.schema.json", unclassified)
request_error_service = _service(
tmp_path / "request-error",
sources=[_source("src_primary01", "Primary")],
outcomes=[
_outcome(
RawOutcomeKind.HTTP_ERROR,
status=400,
code="invalid_parameter",
)
],
)
request_error_model = _canonicalize_fixed_test_routes(request_error_service)["claude"]
request_error = asyncio.run(request_error_service.probe_agent("claude", request_error_model))
assert request_error["reachable"] is False
assert isinstance(request_error["latency_ms"], int)
assert request_error["error"] == "models.source.error.unclassified"
assert request_error_service.store.load().sources[0].state.status == "standby"
assert request_error_service.events.list(limit=10) == []
_assert_valid("probe-result.schema.json", request_error)
anthropic_not_found = _service(
tmp_path / "anthropic-not-found",
sources=[_source("src_primary01", "Primary")],
outcomes=[
_outcome(
RawOutcomeKind.HTTP_ERROR,
status=404,
code="not_found_error",
)
],
)
not_found_model = _canonicalize_fixed_test_routes(anthropic_not_found)["claude"]
not_found = asyncio.run(anthropic_not_found.probe_agent("claude", not_found_model))
assert not_found["reachable"] is False
assert anthropic_not_found.store.load().sources[0].state.status == "standby"
assert anthropic_not_found.events.list(limit=10) == []
_assert_valid("probe-result.schema.json", not_found)
def test_probe_transport_failures_await_ac50_backoff_contract(tmp_path: Path) -> None:
for name, kind in (
("network", RawOutcomeKind.NETWORK_ERROR),
("timeout", RawOutcomeKind.TIMEOUT),
):
outcome = _outcome(kind)
assert outcome.stream_started is False
service = _service(
tmp_path / name,
sources=[_source("src_primary01", "Primary")],
outcomes=[outcome],
)
menu_model = _canonicalize_fixed_test_routes(service)["claude"]
config_before = json.loads(json.dumps(service.store.load().to_payload()))
service.store.save = Mock(wraps=service.store.save)
result = asyncio.run(service.probe_agent("claude", menu_model))
assert result["reachable"] is False
assert result["latency_ms"] is None
assert result["error"] == "models.source.backoff.connection_failed"
_assert_valid("probe-result.schema.json", result)
service.store.save.assert_not_called()
assert service.store.load().to_payload() == config_before
assert service.store.load().sources[0].state.status == "standby"
assert service.store.load().sources[0].state.detail_key is None
chain = service.agent_chain("claude", menu_model)
hop = chain["chain"][0]
assert hop["health"] == "backoff"
assert hop["runnable"] is False
assert hop["reason"] == "models.source.backoff.connection_failed"
assert datetime.fromisoformat(hop["retry_at"]) > NOW
assert chain["current"] is None
assert chain["supply_state"] == "waiting"
assert service.events.list(limit=10)[0]["reason"] == "network"
_assert_valid("agent-chain.schema.json", chain)
def test_probe_401_uses_exact_credential_refresh_capability(tmp_path: Path) -> None:
static_source = _source("src_staticprobe", "Static")
static_service = _service(
tmp_path / "static",
sources=[static_source],
outcomes=[
_outcome(
RawOutcomeKind.HTTP_ERROR,
status=401,
code="unauthorized",
source_id=static_source.id,
)
],
)
static_model = _canonicalize_fixed_test_routes(static_service)["claude"]
static_adapter = cast(ProbeAdapter, static_service.adapter)
static_probe = asyncio.run(static_service.probe_agent("claude", static_model))
assert static_probe["reachable"] is False
assert static_probe["error"] == ("models.source.needs_action.credential_revoked")
assert static_adapter.invocations == [(static_source.id, "shared-model", "claude")]
assert static_adapter.capability_queries == [static_source.credential_ref]
assert static_service.store.load().sources[0].state.detail_key == ("models.source.needs_action.credential_revoked")
refreshable_source = _source("src_refreshprobe", "Refreshable")
refreshable_service = _service(
tmp_path / "refreshable",
sources=[refreshable_source],
outcomes=[
_outcome(
RawOutcomeKind.HTTP_ERROR,
status=401,
code="unauthorized",
source_id=refreshable_source.id,
),
_outcome(
RawOutcomeKind.SUCCESS,
status=200,
source_id=refreshable_source.id,
),
],
)
refreshable_model = _canonicalize_fixed_test_routes(refreshable_service)["claude"]
refreshable_adapter = cast(ProbeAdapter, refreshable_service.adapter)
assert refreshable_source.credential_ref is not None
refreshable_adapter.refreshable_credential_refs.add(refreshable_source.credential_ref)
refreshable_probe = asyncio.run(refreshable_service.probe_agent("claude", refreshable_model))
assert refreshable_probe["reachable"] is True
assert refreshable_adapter.invocations == [
(refreshable_source.id, "shared-model", "claude"),
(refreshable_source.id, "shared-model", "claude"),
]
assert refreshable_adapter.capability_queries == [refreshable_source.credential_ref]
@pytest.mark.parametrize(
("refreshable", "expected_detail"),
(
(False, "models.source.needs_action.credential_revoked"),
(True, "models.source.needs_action.oauth_expired"),
),
)
def test_probe_2xx_native_auth_error_uses_classified_blocker_detail(
tmp_path: Path,
refreshable: bool,
expected_detail: str,
) -> None:
source = _source("src_probe2xxauth", "Native auth envelope")
outcomes = [
_outcome(
RawOutcomeKind.HTTP_ERROR,
status=200,
code="invalid_api_key",
source_id=source.id,
)
]
if refreshable:
outcomes.append(
_outcome(
RawOutcomeKind.HTTP_ERROR,
status=200,
code="invalid_api_key",
source_id=source.id,
)
)
service = _service(tmp_path, sources=[source], outcomes=outcomes)
model = _canonicalize_fixed_test_routes(service)["claude"]
adapter = cast(ProbeAdapter, service.adapter)
if refreshable:
assert source.credential_ref is not None
adapter.refreshable_credential_refs.add(source.credential_ref)
result = asyncio.run(service.probe_agent("claude", model))
assert result["error"] == expected_detail
persisted = service.store.load().sources[0].state
assert persisted.status == "needs_action"
assert persisted.detail_key == expected_detail
@pytest.mark.parametrize(
("backend", "source_protocol", "request_protocol", "request_keys"),
[
(
"claude",
source_protocol,
"anthropic",
{"model", "max_tokens", "messages"},
)
for source_protocol in MODEL_PROTOCOLS
]
+ [
(
"codex",
source_protocol,
"openai_responses",
{"model", "max_output_tokens", "input"},
)
for source_protocol in MODEL_PROTOCOLS
]
+ [
(
"opencode",
source_protocol,
source_protocol,
(
{"model", "max_output_tokens", "input"}
if source_protocol == "openai_responses"
else {"contents", "generationConfig"} if source_protocol == "google"
else {"model", "max_tokens", "messages"}
),
)
for source_protocol in MODEL_PROTOCOLS
],
)
def test_probe_request_matches_live_backend_protocol_matrix(
tmp_path: Path,
backend: str,
source_protocol: str,
request_protocol: str,
request_keys: set[str],
) -> None:
source = _source("src_primary01", "Primary")
source.protocol = source_protocol
if source_protocol == "google":
source.vendor = "custom"
service = _service(
tmp_path,
sources=[source],
outcomes=[_outcome(RawOutcomeKind.SUCCESS)],
)
requested_model = "shared-model"
result = asyncio.run(service.probe_agent(backend, requested_model))
assert result["reachable"] is True
assert service.adapter.invocations == [("src_primary01", "shared-model", backend)]
request = service.adapter.requests[0]
assert request.protocol == request_protocol
assert set(request) == request_keys
def test_native_chain_visibility_and_probe_readiness(tmp_path: Path) -> None:
native = _source(
"src_nativecli1",
"Native CLI",
channel="native_cli",
)
service = _service(tmp_path, sources=[native])
chain = service.agent_chain("codex", "shared-model")
assert chain["chain"] == [
{
"source_id": native.id,
"model_id": "shared-model",
"channel": "native_cli",
"health": "healthy",
"runnable": True,
"reason": None,
"retry_at": None,
}
]
assert chain["current"] == {"source_id": native.id, "model_id": "shared-model"}
assert chain["supply_state"] == "ok"
_assert_valid("agent-chain.schema.json", chain)
probe = asyncio.run(service.probe_agent("codex", "shared-model"))
assert probe == {
"contract_version": 23,
"backend": "codex",
"channel": "native_cli",
"reachable": True,
"source_id": native.id,
"model_id": "shared-model",
"latency_ms": None,
"error": None,
}
assert service.adapter.invocations == []
_assert_valid("probe-result.schema.json", probe)
def test_native_unavailability_is_orthogonal_to_health(
tmp_path: Path,
) -> None:
native = _source(
"src_nativecli1",
"Native CLI",
channel="native_cli",
status="cooldown",
retry_at=(NOW + timedelta(minutes=5)).isoformat(),
)
service = _service(tmp_path, sources=[native])
service.native_source_ready = lambda _backend, _source: False
chain = service.agent_chain("codex", "shared-model")
assert chain["chain"][0]["health"] == "cooldown"
assert chain["chain"][0]["reason"] == "native_cli_unavailable"
assert chain["chain"][0]["runnable"] is False
assert chain["current"] is None
assert chain["supply_state"] == "interrupted"
_assert_valid("agent-chain.schema.json", chain)
def test_native_probe_rechecks_readiness_after_selection(
tmp_path: Path,
) -> None:
native = _source(
"src_nativecli1",
"Native CLI",
channel="native_cli",
)
service = _service(tmp_path, sources=[native])
readiness = iter((True, False))
service.native_source_ready = lambda _backend, _source: next(readiness)
probe = asyncio.run(service.probe_agent("codex", "shared-model"))
assert probe["channel"] == "native_cli"
assert probe["reachable"] is False
assert probe["latency_ms"] is None
assert probe["error"] == "models.probe.native_cli_unavailable"
assert service.adapter.invocations == []
_assert_valid("probe-result.schema.json", probe)
def test_probe_no_candidate_and_direct_mode_are_typed(tmp_path: Path) -> None:
cooling = _source(
"src_cooling01",
"Cooling",
status="cooldown",
retry_at=(NOW + timedelta(minutes=5)).isoformat(),
)
service = _service(tmp_path, sources=[cooling])
with pytest.raises(ModelHubError) as exc_info:
asyncio.run(service.probe_agent("claude", "shared-model"))
assert exc_info.value.code == "probe_no_candidate"
assert exc_info.value.detail == "models.probe.no_candidate.waiting"
assert exc_info.value.data == {
"supply": {
"supply_state": "waiting",
"retry_at": cooling.state.retry_at,
}
}
service.store.config.agents["claude"].mode = "direct"
with pytest.raises(ModelHubError) as chain_error:
service.agent_chain("claude", "shared-model")
with pytest.raises(ModelHubError) as probe_error:
asyncio.run(service.probe_agent("claude", "shared-model"))
for error in (chain_error.value, probe_error.value):
assert error.code == "direct_mode"
assert error.detail == "models.hub.direct_mode"
def test_source_failure_event_is_single_grain_and_retained(tmp_path: Path) -> None:
source = _source("src_primary01", "Shared paid source")
service = _service(
tmp_path,
sources=[source],
outcomes=[
_outcome(
RawOutcomeKind.HTTP_ERROR,
status=402,
code="payment_required",
message="balance exhausted",
)
],
)
menu_model = _canonicalize_fixed_test_routes(service)["claude"]
probe = asyncio.run(service.probe_agent("claude", menu_model))
assert probe["reachable"] is False
events = service.list_events(limit=20)
failures = [event for event in events if event["kind"] == "needs_action"]
assert len(failures) == 1
event = failures[0]
assert event["agent"] == "claude"
assert event["from_source"] == source.id
assert event["severity"] == "action_required"
assert "Shared paid source" in event["human_en"]
assert "Shared paid source" in event["human_zh"]
_assert_valid("resolution-event.schema.json", event)
agents = {item["backend"]: item for item in service.list_agents()}
assert agents["claude"]["supply_status"] == "interrupted"
assert agents["codex"]["supply_status"] == "interrupted"
service.store.config.sources[0].state = ModelHubSourceStateConfig(status="standby")
recovered_agents = {item["backend"]: item for item in service.list_agents()}
assert recovered_agents["claude"]["supply_status"] == "ok"
assert recovered_agents["codex"]["supply_status"] == "ok"
assert service.list_events(limit=20)[0] == event
service.store.config.sources.clear()
assert service.list_events(limit=20)[0] == event
def test_retired_mapping_event_is_rejected_and_channel_switch_retains_subject() -> None:
with pytest.raises(ValueError):
build_resolution_event(
agent="codex",
kind=cast(EventKind, "mapping" + "_applied"),
model_id="target-model",
reason=cast(EventReason, "map" + "ping"),
from_label="requested-model",
now=NOW,
)
channel_switch = build_resolution_event(
agent="system",
kind="channel_switch",
model_id=None,
reason="manual",
from_source="src_primary01",
to_source="src_primary01",
from_label="Primary source",
to_label="Primary source",
now=NOW,
)
for text in (channel_switch.human_en, channel_switch.human_zh):
assert "Primary source" in text
def test_resolution_event_reason_contract_matches_runtime_vocabulary() -> None:
schema = json.loads((CONTRACTS / "resolution-event.schema.json").read_text(encoding="utf-8"))
authority = tuple(EVENT_REASON_AUTHORITY)
assert tuple(schema["properties"]["reason"]["enum"]) == authority
locale_reasons = {
locale: json.loads((Path(__file__).parents[1] / f"vibe/i18n/{locale}.json").read_text(encoding="utf-8"))[
"modelHub"
]["events"]["reason"]
for locale in ("en", "zh")
}
assert all(tuple(reasons) == authority for reasons in locale_reasons.values())
assert all(
event_reason_label(reason, locale) != f"modelHub.events.reason.{reason}"
for locale in locale_reasons
for reason in authority
)
@pytest.mark.parametrize("reason", tuple(EVENT_REASON_AUTHORITY))
def test_every_authoritative_reason_has_an_event_emission_path(reason: EventReason) -> None:
reason_class = EVENT_REASON_AUTHORITY[reason]
fields: dict = {
"agent": "system",
"model_id": None,
"reason": reason,
"now": NOW,
}
if reason_class == "structural":
fields.update(agent="codex", kind="supply_interrupted", model_id="model")
elif reason_class == "self_healing":
fields.update(kind="cooldown", from_source="src_reason01")
elif reason_class == "non_self_healing":
fields.update(kind="needs_action", from_source="src_reason01")
elif reason_class == "request_scoped":
fields.update(
agent="codex",
kind="switch",
model_id="model",
from_source="src_reason01",
to_source="src_reason02",
)
elif reason in {"upstream_tiers", "catalog_tiers"}:
fields.update(
kind="reasoning_efforts_override",
model_id="model",
from_source="src_reason01",
)
elif reason == "recovery":
fields.update(kind="recover", to_source="src_reason01")
else:
fields.update(
kind="channel_switch",
from_source="src_reason01",
to_source="src_reason01",
)
event = build_resolution_event(**fields)
assert event.reason == reason
assert event.human_en
assert event.human_zh
def test_shared_source_cooldown_emits_only_on_state_transition(
tmp_path: Path,
) -> None:
source = _source("src_primary01", "Shared source")
service = _service(tmp_path, sources=[source])
menu_models = _canonicalize_fixed_test_routes(service)
decision = classify_outcome(
_outcome(
RawOutcomeKind.HTTP_ERROR,
status=429,
code="rate_limited",
)
)
async def cool_twice() -> tuple[bool, bool]:
first_generation = service._reserve_settlement_generation(source.id)
_first_reason, first_persisted = await service._settle_fallback_source(
source,
decision,
backend="claude",
model_id=menu_models["claude"],
settlement_generation=first_generation,
)
repeated_generation = service._reserve_settlement_generation(source.id)
_repeated_reason, repeated_persisted = await service._settle_fallback_source(
source,
decision,
backend="codex",
model_id=menu_models["codex"],
settlement_generation=repeated_generation,
)
return first_persisted, repeated_persisted
assert asyncio.run(cool_twice()) == (True, True)
events = service.list_events(limit=20)
assert len(events) == 1
assert events[0]["kind"] == "cooldown"
assert events[0]["agent"] == "claude"
def test_late_shorter_cooldown_keeps_the_longest_concurrent_deadline(
tmp_path: Path,
) -> None:
fixture = E64_SETTLEMENT_BOUNDARIES["concurrent_cooldown"]
source = _source("src_cooldownmax01", "Shared source")
service = _service(tmp_path, sources=[source])
menu_models = _canonicalize_fixed_test_routes(service)
async def cool_in_completion_order() -> None:
for backend, seconds, reason in (
("claude", fixture["first_seconds"], "quota_exhausted"),
("codex", fixture["late_seconds"], "server_error"),
):
generation = service._reserve_settlement_generation(source.id)
await service._settle_fallback_source(
source,
ResolutionDecision(
"fallback",
reason=reason,
cooldown_seconds=seconds,
),
backend=backend,
model_id=menu_models[backend],
settlement_generation=generation,
)
asyncio.run(cool_in_completion_order())
persisted = service.store.load().sources[0]
assert persisted.state.status == "cooldown"
assert persisted.state.retry_at == (
NOW + timedelta(seconds=fixture["expected_seconds"])
).isoformat()
assert persisted.state.detail_key == "models.source.cooldown.quota_exhausted"
def test_late_transient_settlement_preserves_needs_action_state(
tmp_path: Path,
) -> None:
source = _source("src_needsaction01", "Credential source")
service = _service(tmp_path, sources=[source])
menu_models = _canonicalize_fixed_test_routes(service)
transient = classify_outcome(
_outcome(
RawOutcomeKind.HTTP_ERROR,
status=503,
code="api_error",
)
)
async def settle_in_order() -> None:
blocker_generation = service._reserve_settlement_generation(source.id)
await service._settle_fallback_source(
source,
ResolutionDecision("fallback", reason="credential_revoked"),
backend="claude",
model_id=menu_models["claude"],
detail_key="models.source.needs_action.credential_revoked",
settlement_generation=blocker_generation,
)
transient_generation = service._reserve_settlement_generation(source.id)
await service._settle_fallback_source(
source,
transient,
backend="codex",
model_id=menu_models["codex"],
settlement_generation=transient_generation,
)
asyncio.run(settle_in_order())
persisted = service.store.load().sources[0]
assert persisted.state.status == "needs_action"
assert persisted.state.detail_key == "models.source.needs_action.credential_revoked"
def test_late_equal_priority_blocker_preserves_the_newer_reason(
tmp_path: Path,
) -> None:
source = _source("src_blockerorder", "Shared credential")
service = _service(tmp_path, sources=[source])
menu_models = _canonicalize_fixed_test_routes(service)
older_generation = service._reserve_settlement_generation(source.id)
newer_generation = service._reserve_settlement_generation(source.id)
async def settle_in_completion_order() -> tuple[bool, bool]:
_newer_reason, newer_persisted = await service._settle_fallback_source(
source,
ResolutionDecision("fallback", reason="account_banned"),
backend="claude",
model_id=menu_models["claude"],
detail_key="models.source.needs_action.account_banned",
settlement_generation=newer_generation,
)
_older_reason, older_persisted = await service._settle_fallback_source(
source,
ResolutionDecision("fallback", reason="credential_expired"),
backend="codex",
model_id=menu_models["codex"],
detail_key="models.source.needs_action.oauth_expired",
settlement_generation=older_generation,
)
return newer_persisted, older_persisted
assert asyncio.run(settle_in_completion_order()) == (True, False)
persisted = service.store.load().sources[0]
assert persisted.state.status == "needs_action"
assert persisted.state.detail_key == "models.source.needs_action.account_banned"
def test_event_emission_rejects_unknown_source_ids(tmp_path: Path) -> None:
service = _service(
tmp_path,
sources=[_source("src_primary01", "Primary")],
)
service._record_event(
agent="claude",
kind="switch",
model_id="shared-model",
reason="manual",
from_source="src_unknown01",
to_source="src_primary01",
now=NOW,
)
service._record_event(
agent="claude",
kind="switch",
model_id="shared-model",
reason="manual",
from_source="src_primary01",
to_source="src_unknown02",
now=NOW,
)
assert service.list_events(limit=20) == []
@pytest.mark.parametrize(
"fields",
[
{
"agent": "system",
"kind": "supply_interrupted",
"model_id": "shared-model",
"reason": "no_enabled_source",
},
{
"agent": "claude",
"kind": "cooldown",
"model_id": "shared-model",
"reason": "credential_revoked",
"from_source": "src_primary01",
},
{
"agent": "claude",
"kind": "needs_action",
"model_id": "shared-model",
"reason": "network",
"from_source": "src_primary01",
},
{
"agent": "claude",
"kind": "switch",
"model_id": "shared-model",
"reason": "no_eligible_source",
"from_source": "src_primary01",
"to_source": "src_backup001",
},
],
)
def test_event_builder_rejects_cross_grain_shapes(fields: dict) -> None:
with pytest.raises(ValueError):
build_resolution_event(**fields)
def test_provenance_absence_codes_are_distinguishable(
monkeypatch,
tmp_path: Path,
) -> None:
service = _service(
tmp_path,
sources=[_source("src_primary01", "Primary")],
)
monkeypatch.setattr(
service,
"_known_turn",
lambda turn_id: (
("codex", "direct")
if turn_id == "turn_direct"
else (("codex", "hub") if turn_id == "turn_ambiguous" else (None, None))
),
)
service.store.config.agents["codex"].mode = "hub"
with pytest.raises(ModelHubError) as direct:
service.get_turn_provenance("turn_direct")
assert direct.value.code == "provenance_unavailable"
assert direct.value.detail == "models.provenance.direct_mode"
service.store.config.agents["codex"].mode = "direct"
with pytest.raises(ModelHubError) as ambiguous:
service.get_turn_provenance("turn_ambiguous")
assert ambiguous.value.code == "provenance_unavailable"
assert ambiguous.value.detail == "models.provenance.attribution_ambiguous"
with pytest.raises(ModelHubError) as unknown:
service.get_turn_provenance("turn_unknown")
assert unknown.value.code == "turn_not_found"
assert unknown.value.status == 404
def test_turn_mode_marker_is_deleted_with_its_turn_record(
monkeypatch,
tmp_path: Path,
) -> None:
engine = create_engine(f"sqlite+pysqlite:///{tmp_path / 'state.db'}")
metadata.create_all(engine)
now = NOW.isoformat()
with engine.begin() as conn:
conn.execute(
agent_sessions.insert().values(
id="ses_turn_mode",
agent_backend="codex",
agent_variant="codex",
session_anchor="anchor",
native_session_id="",
status="active",
metadata_json="{}",
created_at=now,
updated_at=now,
)
)
conn.execute(
messages.insert().values(
id="msg_turn_mode",
session_id="ses_turn_mode",
platform="avibe",
author="user",
type="user",
source="user",
content_json="{}",
metadata_json=json.dumps({"turn_id": "turn_direct"}),
created_at=now,
updated_at=now,
)
)
service = _service(
tmp_path,
sources=[_source("src_primary01", "Primary")],
)
monkeypatch.setattr(
"core.handlers.model_hub.service.get_cached_sqlite_engine",
lambda: engine,
)
service.note_turn_mode("turn_direct", "direct")
with engine.connect() as conn:
stored_mode = conn.execute(
select(
messages.c.metadata_json,
).where(messages.c.id == "msg_turn_mode")
).scalar_one()
assert json.loads(stored_mode)["model_hub_mode"] == "direct"
with pytest.raises(ModelHubError) as unavailable:
service.get_turn_provenance("turn_direct")
assert unavailable.value.code == "provenance_unavailable"
assert unavailable.value.detail == "models.provenance.direct_mode"
with engine.begin() as conn:
conn.execute(delete(messages).where(messages.c.id == "msg_turn_mode"))
with pytest.raises(ModelHubError) as deleted:
service.get_turn_provenance("turn_direct")
assert deleted.value.code == "turn_not_found"
def test_known_opencode_turn_is_fail_closed_but_not_unknown(
monkeypatch,
tmp_path: Path,
) -> None:
store = BoundedProvenanceStore(tmp_path / "opencode-provenance.json")
registry = TurnCorrelationRegistry(store)
token = registry.credentials(
"opencode",
"opencode:shared-server",
"turn_opencode",
)
assert (
registry.begin_gateway_request(
backend="opencode",
token=token,
requested_model_id="shared-model",
)
is None
)
registry.settle(
"turn_opencode",
settled_by=SETTLED_BY_TERMINAL_RESULT,
)
assert store.get("turn_opencode") is None
service = _service(
tmp_path,
sources=[_source("src_primary01", "Primary")],
)
service.provenance = store
monkeypatch.setattr(
service,
"_known_turn",
lambda turn_id: (("opencode", None) if turn_id == "turn_opencode" else (None, None)),
)
with pytest.raises(ModelHubError) as unavailable:
service.get_turn_provenance("turn_opencode")
assert unavailable.value.code == "provenance_unavailable"
assert unavailable.value.detail == "models.provenance.attribution_ambiguous"
def _usage_of(service: ModelHubService, source_id: str) -> dict:
summary = service.usage_summary(days=30)
matches = [source for source in summary["sources"] if source["source_id"] == source_id]
return matches[0] if matches else {}
def test_gateway_meters_a_buffered_served_turn_from_the_adapter_outcome(
tmp_path: Path,
) -> None:
async def exercise() -> None:
source = _source("src_meterbuf01", "Buffered meter")
body = json.dumps(
{
"id": "resp_buffered",
"usage": {
"input_tokens": 4096,
"input_tokens_details": {"cached_tokens": 3072},
"output_tokens": 128,
},
}
).encode("utf-8")
service = _service(
tmp_path,
sources=[source],
live_handles=[
LiveInvokeHandle(
_outcome(
RawOutcomeKind.SUCCESS,
status=200,
source_id=source.id,
usage=ProtocolUsageReport.of(
input_tokens=4096,
cached_input_tokens=3072,
output_tokens=128,
),
),
(body,),
)
],
)
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
gateway = ModelHubTurnGateway(service)
request = _prepared_gateway_request(
gateway,
turn_id="turn_meter_buffered",
requested_model=requested_model,
source_id=source.id,
stream=False,
)
result = await gateway._handle_request(request)
assert result.status == 200
assert result.body == body
metered = _usage_of(service, source.id)
assert metered["label"] == "Buffered meter"
assert metered["requests"] == 1
assert metered["token_reports"] == 1
assert metered["input_tokens"] == 4096
assert metered["cached_input_tokens"] == 3072
assert metered["output_tokens"] == 128
assert [model["model_id"] for model in metered["models"]] == ["shared-model"]
asyncio.run(exercise())
def test_gateway_spools_and_replays_a_large_buffered_response(
tmp_path: Path,
) -> None:
async def exercise() -> None:
source = _source("src_largebuf01", "Large buffered response")
body = (
b'{"payload":"'
+ b"x" * (512 * 1024)
+ b'","usage":{"input_tokens":4096,"input_tokens_details":'
b'{"cached_tokens":3072},"output_tokens":128}}'
)
service = _service(
tmp_path,
sources=[source],
live_handles=[
LiveInvokeHandle(
_outcome(
RawOutcomeKind.SUCCESS,
status=200,
source_id=source.id,
stream_started=True,
usage=ProtocolUsageReport.of(
input_tokens=4096,
cached_input_tokens=3072,
output_tokens=128,
),
),
(body[:200_000], body[200_000:400_000], body[400_000:]),
)
],
)
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
gateway = ModelHubTurnGateway(service)
request = _prepared_gateway_request(
gateway,
turn_id="turn_large_buffered",
requested_model=requested_model,
source_id=source.id,
stream=False,
)
downstream = FakeStreamResponse()
event_loop_thread = threading.get_ident()
write_threads: list[int] = []
spooled_file = tempfile.SpooledTemporaryFile
class TrackingSpool:
def __init__(self, *args: object, **kwargs: object) -> None:
self._file = spooled_file(*args, **kwargs)
def __enter__(self):
return self
def __exit__(self, *args: object) -> None:
self._file.close()
def write(self, data: bytes) -> int:
write_threads.append(threading.get_ident())
return self._file.write(data)
def __getattr__(self, name: str):
return getattr(self._file, name)
with (
patch(
"core.handlers.model_hub.turn_gateway.web.StreamResponse",
return_value=downstream,
),
patch(
"core.handlers.model_hub.turn_gateway.tempfile.SpooledTemporaryFile",
TrackingSpool,
),
):
result = await gateway._handle_request(request)
assert result is downstream
assert b"".join(downstream.writes) == body
assert downstream.eof_called is True
metered = _usage_of(service, source.id)
assert metered["requests"] == 1
assert metered["token_reports"] == 1
assert metered["input_tokens"] == 4096
assert metered["cached_input_tokens"] == 3072
assert metered["output_tokens"] == 128
assert write_threads
assert event_loop_thread not in write_threads
asyncio.run(exercise())
def test_gateway_drains_a_cancelled_spool_write_before_closing(
tmp_path: Path,
) -> None:
async def exercise() -> None:
source = _source("src_spoolown1", "Owned spool")
handle = LiveInvokeHandle(
_outcome(
RawOutcomeKind.SUCCESS,
source_id=source.id,
stream_started=True,
),
(b'{"output":"ok"}',),
)
service = _service(tmp_path, sources=[source], live_handles=[handle])
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
gateway = ModelHubTurnGateway(service)
request = _prepared_gateway_request(
gateway,
turn_id="turn_owned_spool_write",
requested_model=requested_model,
source_id=source.id,
stream=False,
)
write_started = threading.Event()
release_write = threading.Event()
write_finished = threading.Event()
closed = threading.Event()
close_raced_write: list[bool] = []
spooled_file = tempfile.SpooledTemporaryFile
class TrackingSpool:
def __init__(self, *args: object, **kwargs: object) -> None:
self._file = spooled_file(*args, **kwargs)
def __enter__(self):
return self
def __exit__(self, *args: object) -> None:
close_raced_write.append(not write_finished.is_set())
self._file.close()
closed.set()
def write(self, data: bytes) -> int:
write_started.set()
assert release_write.wait(timeout=1)
result = self._file.write(data)
write_finished.set()
return result
def __getattr__(self, name: str):
return getattr(self._file, name)
with patch(
"core.handlers.model_hub.turn_gateway.tempfile.SpooledTemporaryFile",
TrackingSpool,
):
task = asyncio.create_task(gateway._handle_request(request))
assert await asyncio.to_thread(write_started.wait, 1)
task.cancel()
await asyncio.sleep(0)
assert not closed.is_set()
release_write.set()
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(task, timeout=1)
assert write_finished.is_set()
assert closed.is_set()
assert close_raced_write == [False]
await gateway.close()
asyncio.run(exercise())
def test_gateway_meters_a_streamed_served_turn_from_the_wire(tmp_path: Path) -> None:
async def exercise() -> None:
source = _source("src_meterwire01", "Streamed meter")
service = _service(
tmp_path,
sources=[source],
live_handles=[
LiveInvokeHandle(
_outcome(
RawOutcomeKind.SUCCESS,
status=200,
source_id=source.id,
stream_started=True,
),
(
b'event: response.output_text.delta\ndata: {"type":'
b'"response.output_text.delta","sequence_number":1}\n\n',
b'event: response.completed\ndata: {"type":"response.completed",'
b'"sequence_number":2,"response":{"usage":{"input_tokens":900,'
b'"output_tokens":64}}}\n\n',
),
)
],
)
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
gateway = ModelHubTurnGateway(service)
request = _prepared_gateway_request(
gateway,
turn_id="turn_meter_streamed",
requested_model=requested_model,
source_id=source.id,
stream=True,
)
with patch(
"core.handlers.model_hub.turn_gateway.web.StreamResponse",
return_value=FakeStreamResponse(),
):
await gateway._handle_request(request)
metered = _usage_of(service, source.id)
assert metered["requests"] == 1
assert metered["token_reports"] == 1
assert metered["input_tokens"] == 900
assert metered["output_tokens"] == 64
asyncio.run(exercise())
def test_a_stream_that_forwarded_output_is_metered_without_a_token_report(
tmp_path: Path,
) -> None:
"""Review 4959575659 finding 10: `requests` is what our own code measured.
Model output reached the client, so the call reached the model — a connection
lost after a text delta is still a request that happened. Nobody reported its
tokens, which is exactly what `token_reports` staying at zero records.
"""
async def exercise() -> None:
source = _source("src_meterpart01", "Interrupted stream")
service = _service(
tmp_path,
sources=[source],
live_handles=[
LiveInvokeHandle(
_outcome(
RawOutcomeKind.NETWORK_ERROR,
source_id=source.id,
stream_started=True,
),
(
b'event: response.output_text.delta\ndata: {"type":'
b'"response.output_text.delta","sequence_number":1}\n\n',
),
)
],
)
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
gateway = ModelHubTurnGateway(service)
request = _prepared_gateway_request(
gateway,
turn_id="turn_meter_partial",
requested_model=requested_model,
source_id=source.id,
stream=True,
)
with patch(
"core.handlers.model_hub.turn_gateway.web.StreamResponse",
return_value=FakeStreamResponse(),
):
await gateway._handle_request(request)
metered = _usage_of(service, source.id)
assert metered["requests"] == 1
assert metered["token_reports"] == 0
assert metered["input_tokens"] == 0
assert metered["output_tokens"] == 0
asyncio.run(exercise())
def test_a_buffered_response_cancelled_while_settling_is_still_metered(
tmp_path: Path,
) -> None:
"""Review 4960016618: the boundary meters the turn's report, not one shape's.
Upstream delivered a complete billed response and the client then went away
while the turn was settling. The buffered shape has no wire tracker for the
boundary to read, so a report that lived only in the request frame's local
would vanish exactly when the turn was already billed.
"""
async def exercise() -> None:
source = _source("src_meterbuf01", "Cancelled buffered turn")
service = _service(
tmp_path,
sources=[source],
live_handles=[
LiveInvokeHandle(
_outcome(
RawOutcomeKind.SUCCESS,
status=200,
source_id=source.id,
stream_started=True,
usage=ProtocolUsageReport.of(
input_tokens=704,
cached_input_tokens=0,
output_tokens=21,
),
),
(b'{"usage":{"input_tokens":704,"output_tokens":21}}',),
)
],
)
settling = asyncio.Event()
release = asyncio.Event()
settle_handle_outcome = service.settle_handle_outcome
async def blocked_settlement(*args: object, **kwargs: object):
settling.set()
await release.wait()
return await settle_handle_outcome(*args, **kwargs)
service.settle_handle_outcome = blocked_settlement
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
gateway = ModelHubTurnGateway(service)
request = _prepared_gateway_request(
gateway,
turn_id="turn_meter_buffered_cancel",
requested_model=requested_model,
source_id=source.id,
stream=False,
)
turn = asyncio.create_task(gateway._handle_request(request))
await asyncio.wait_for(settling.wait(), timeout=1)
turn.cancel()
release.set()
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(turn, timeout=1)
metered = _usage_of(service, source.id)
assert metered["requests"] == 1
assert metered["token_reports"] == 1
assert metered["input_tokens"] == 704
assert metered["output_tokens"] == 21
asyncio.run(exercise())
def test_buffered_adapter_facts_survive_cancellation_before_body_consumption(
tmp_path: Path,
) -> None:
async def exercise() -> None:
source = _source("src_meterproj1", "Cancelled projection")
body_started = asyncio.Event()
class BufferedHandle(LiveInvokeHandle):
async def _blocked_body(self):
body_started.set()
await asyncio.Event().wait()
if False:
yield b""
def __init__(self, outcome: RawCallOutcome) -> None:
self._outcome = outcome
self._observed = None
self._stream = self._blocked_body()
service = _service(
tmp_path,
sources=[source],
live_handles=[
BufferedHandle(
_outcome(
RawOutcomeKind.SUCCESS,
status=200,
source_id=source.id,
stream_started=True,
usage=ProtocolUsageReport.of(
input_tokens=901,
cached_input_tokens=0,
output_tokens=22,
),
),
)
],
)
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
gateway = ModelHubTurnGateway(service)
request = _prepared_gateway_request(
gateway,
turn_id="turn_meter_projection_cancel",
requested_model=requested_model,
source_id=source.id,
stream=False,
)
turn = asyncio.create_task(gateway._handle_request(request))
await asyncio.wait_for(body_started.wait(), timeout=1)
turn.cancel()
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(turn, timeout=1)
metered = _usage_of(service, source.id)
assert metered["requests"] == 1
assert metered["token_reports"] == 1
assert metered["input_tokens"] == 901
assert metered["output_tokens"] == 22
asyncio.run(exercise())
def test_a_cancelled_buffered_turn_is_counted_even_when_it_reported_no_tokens(
tmp_path: Path,
) -> None:
"""Review 4964520496: a request is self-measured, so a silent vendor still owes one.
Same cancellation as above with the one thing removed that was carrying it —
the usage block. `requests` is measured by our own code and must not depend on
what upstream chose to report, which is exactly what `token_reports` staying
at zero records.
"""
async def exercise() -> None:
source = _source("src_meterbuf02", "Cancelled silent turn")
service = _service(
tmp_path,
sources=[source],
live_handles=[
LiveInvokeHandle(
_outcome(
RawOutcomeKind.SUCCESS,
status=200,
source_id=source.id,
stream_started=True,
),
(b'{"output":[{"type":"message","content":[]}]}',),
)
],
)
settling = asyncio.Event()
release = asyncio.Event()
settle_handle_outcome = service.settle_handle_outcome
async def blocked_settlement(*args: object, **kwargs: object):
settling.set()
await release.wait()
return await settle_handle_outcome(*args, **kwargs)
service.settle_handle_outcome = blocked_settlement
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
gateway = ModelHubTurnGateway(service)
request = _prepared_gateway_request(
gateway,
turn_id="turn_meter_buffered_silent_cancel",
requested_model=requested_model,
source_id=source.id,
stream=False,
)
turn = asyncio.create_task(gateway._handle_request(request))
await asyncio.wait_for(settling.wait(), timeout=1)
turn.cancel()
release.set()
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(turn, timeout=1)
metered = _usage_of(service, source.id)
assert metered["requests"] == 1
assert metered["token_reports"] == 0
asyncio.run(exercise())
def test_a_cancelled_turn_cannot_take_the_ledger_write_it_queued_with_it(
tmp_path: Path,
) -> None:
"""Review 4964754924: the gateway owns the write, so no ending can cancel it.
Occupy the writing thread and the ledger write is queued but not started —
the window where cancelling the awaiting task also cancels the work. The turn
dies there, and the row still lands, because what the turn owns is the
decision to meter and not the write that carries it out.
"""
async def exercise() -> None:
source = _source("src_meterown01", "Owned write")
service = _service(
tmp_path,
sources=[source],
live_handles=[
LiveInvokeHandle(
_outcome(
RawOutcomeKind.SUCCESS,
status=200,
source_id=source.id,
usage=ProtocolUsageReport.of(
input_tokens=512,
cached_input_tokens=0,
output_tokens=16,
),
),
(b'{"usage":{"input_tokens":512,"output_tokens":16}}',),
)
],
)
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
gateway = ModelHubTurnGateway(service)
request = _prepared_gateway_request(
gateway,
turn_id="turn_meter_owned_write",
requested_model=requested_model,
source_id=source.id,
stream=False,
)
with _occupied_ledger_writer():
turn = asyncio.create_task(gateway._handle_request(request))
while not gateway._usage_writer.unpersisted:
await asyncio.sleep(0.01)
# The scenario is only the scenario while the write is still queued.
assert _usage_of(service, source.id) == {}
turn.cancel()
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(turn, timeout=5)
await gateway.close()
metered = _usage_of(service, source.id)
assert metered["requests"] == 1
assert metered["input_tokens"] == 512
assert metered["output_tokens"] == 16
asyncio.run(exercise())
def test_a_cancelled_resolve_cannot_take_the_ledger_write_it_queued_with_it(
tmp_path: Path,
) -> None:
"""Review 4964894667: the same window at the other metering owner.
A call the resolver consumed itself still reports tokens the vendor billed.
Occupy the writing thread and its row is queued but not started; the caller
is cancelled there, and the row lands anyway. Neither owner keeps a write of
its own, so neither can lose one — `UsageWriter` holds both.
"""
async def exercise() -> None:
source = _source("src_meterown02", "Owned resolve write")
service = _service(
tmp_path,
sources=[source],
outcomes=[
_outcome(
RawOutcomeKind.SUCCESS,
status=200,
source_id=source.id,
usage=ProtocolUsageReport.of(
input_tokens=512,
cached_input_tokens=0,
output_tokens=16,
),
)
],
)
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
with _occupied_ledger_writer():
resolve = asyncio.create_task(
service.resolve(
backend="codex",
model_id=requested_model,
request=ModelHubRequest(
{"model": requested_model, "input": "ping"},
protocol="openai_responses",
),
supply_channel="hub",
)
)
while not service.usage_writer.unpersisted:
await asyncio.sleep(0.01)
# The scenario is only the scenario while the write is still queued.
assert _usage_of(service, source.id) == {}
resolve.cancel()
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(resolve, timeout=5)
assert await service.usage_writer.drain(timeout=5) == 0
metered = _usage_of(service, source.id)
assert metered["requests"] == 1
assert metered["input_tokens"] == 512
assert metered["output_tokens"] == 16
asyncio.run(exercise())
def test_a_burst_of_metering_neither_borrows_the_shared_pool_nor_grows_unbounded(
tmp_path: Path,
) -> None:
"""Review 4965076681: the writes queue on their own thread, a batch at a time.
`record_many` holds the ledger's lock across an fsync, so one shared-pool job
per completed call would park that many workers on a lock that admits one,
and unrelated `asyncio.to_thread` work would wait behind metering for nothing.
Hold the loop's default executor for the whole test and metering still lands:
it was never borrowing from there.
What bounds the queue is in the same run. Calls that arrive while a flush is
on disk are taken together by the next one, so a burst costs transactions in
proportion to how long the disk takes rather than to how hard the hub is
driven — and every row still lands, which dropping them would not do.
"""
async def exercise() -> None:
loop = asyncio.get_running_loop()
shared = ThreadPoolExecutor(max_workers=1)
loop.set_default_executor(shared)
borrowed_started = threading.Event()
give_back = threading.Event()
def borrow() -> None:
borrowed_started.set()
give_back.wait(5)
borrower = loop.run_in_executor(shared, borrow)
assert borrowed_started.wait(5)
source = _source("src_meterburst1", "Burst")
service = _service(tmp_path, sources=[source])
transactions: list[int] = []
fold = service.usage.record_many
def counting(calls) -> None:
transactions.append(len(calls))
fold(calls)
service.usage.record_many = counting
writer = service.usage_writer
def meter() -> None:
writer.record(
source_id=source.id,
model_id="gpt-5-codex",
usage=ProtocolUsageReport.of(
input_tokens=1, cached_input_tokens=0, output_tokens=1
),
at=NOW,
)
with _occupied_ledger_writer():
meter()
# White-box on purpose: the batch boundary is the thing under test,
# and it is only observable once the first flush is holding one.
while not writer._writing:
await asyncio.sleep(0.01)
for _ in range(7):
meter()
assert await writer.drain(timeout=5) == 0
# Eight calls, two trips to disk — the second took every call that piled
# up behind the first, which is the whole of the bound. It carries one row
# rather than seven, because the seven were headed for one row anyway; the
# requests below are what proves that folded and lost are different things.
assert transactions == [1, 1]
metered = _usage_of(service, source.id)
assert metered["requests"] == 8
assert metered["input_tokens"] == 8
give_back.set()
await borrower
shared.shutdown()
asyncio.run(exercise())
def test_no_downstream_ending_after_adoption_can_drop_the_turn_from_the_ledger(
tmp_path: Path,
) -> None:
"""Review 4965076681: the gateway takes the body before it can serve it.
Between adopting the upstream body and reading it, the gateway prepares a
downstream response — and a client that leaves in that gap leaves behind
neither a wire tracker nor a buffered verdict for the boundary to read. The
vendor billed the call either way, so what the ledger must not depend on is
*where* the turn died.
Driven off `FakeStreamResponse`'s own failure points instead of a list of
them: a downstream step that becomes failable later is covered the moment it
can fail, rather than the moment someone remembers this test.
"""
failure_points = [
name for name in inspect.signature(FakeStreamResponse).parameters if name.endswith("_error")
]
assert failure_points
async def exercise(point: str) -> None:
state = tmp_path / point
state.mkdir()
source = _source("src_meterdrop01", "Adopted body")
service = _service(
state,
sources=[source],
live_handles=[
LiveInvokeHandle(
_outcome(RawOutcomeKind.SUCCESS, status=200, source_id=source.id),
(
b'event: response.output_text.delta\ndata: {"type":'
b'"response.output_text.delta","sequence_number":1}\n\n',
),
)
],
)
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
gateway = ModelHubTurnGateway(service)
request = _prepared_gateway_request(
gateway,
turn_id=f"turn_meter_{point}",
requested_model=requested_model,
source_id=source.id,
stream=True,
)
with patch(
"core.handlers.model_hub.turn_gateway.web.StreamResponse",
return_value=FakeStreamResponse(**{point: ConnectionResetError("client left")}),
):
with pytest.raises(ConnectionError):
await asyncio.wait_for(gateway._handle_request(request), timeout=5)
await gateway.close()
assert _usage_of(service, source.id).get("requests") == 1, point
for point in failure_points:
asyncio.run(exercise(point))
def test_the_tokens_the_engine_already_read_survive_every_ending_before_our_first(
tmp_path: Path,
) -> None:
"""MH-USAGE-001, review 4965405530: the engine read this body's head before we did.
The gateway asks the engine for a stream, and the engine only knows there is
one because it read far enough to see the first model output — which for
Anthropic is past `message_start`, the frame carrying the input tokens the
vendor already billed. Those bytes then reach the gateway as a replay it
re-tokenizes itself, so every fact it holds about the call starts existing
only once forwarding starts. `requests` survived that gap because adoption
is a fact of the turn; the token counts had nowhere to come from.
Same enumeration as the sibling above, for the same reason: the property is
that no downstream ending decides what upstream reported, so the endings are
read off the response double rather than listed here.
"""
failure_points = [
name for name in inspect.signature(FakeStreamResponse).parameters if name.endswith("_error")
]
assert failure_points
prelude = (
b'event: message_start\ndata: {"type":"message_start","message":'
b'{"usage":{"input_tokens":900,"cache_read_input_tokens":128}}}\n\n'
)
async def exercise(point: str) -> None:
state = tmp_path / f"prelude_{point}"
state.mkdir()
source = _source("src_meterprel01", "Prelude-billed body")
engine_view = ProtocolSSEState("anthropic")
engine_view.observe(prelude)
assert engine_view.usage is not None
service = _service(
state,
sources=[source],
live_handles=[
LiveInvokeHandle(
_outcome(RawOutcomeKind.SUCCESS, status=200, source_id=source.id),
(prelude,),
observed=engine_view,
)
],
)
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
gateway = ModelHubTurnGateway(service)
request = _prepared_gateway_request(
gateway,
turn_id=f"turn_meter_prelude_{point}",
requested_model=requested_model,
source_id=source.id,
stream=True,
)
with patch(
"core.handlers.model_hub.turn_gateway.web.StreamResponse",
return_value=FakeStreamResponse(**{point: ConnectionResetError("client left")}),
):
with pytest.raises(ConnectionError):
await asyncio.wait_for(gateway._handle_request(request), timeout=5)
await gateway.close()
metered = _usage_of(service, source.id)
assert metered.get("requests") == 1, point
assert metered.get("token_reports") == 1, point
assert metered.get("input_tokens") == 1028, point
for point in failure_points:
asyncio.run(exercise(point))
def test_a_turn_cancelled_before_it_read_the_body_is_still_a_call_that_happened(
tmp_path: Path,
) -> None:
"""The same gap on the buffered path, which the review did not reach.
A non-streaming turn adopts the whole body and reads it in one go, so a
client that disconnects while that read is in flight leaves the boundary
exactly as empty-handed as the prepare gap does. Same adoption, same answer,
and `token_reports` at zero is the ledger saying nobody got to read the
tokens rather than that there were none.
"""
async def exercise() -> None:
source = _source("src_meterdrain01", "Adopted buffered body")
handle = BlockingLiveInvokeHandle(
_outcome(RawOutcomeKind.SUCCESS, status=200, source_id=source.id)
)
service = _service(tmp_path, sources=[source], live_handles=[handle])
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
gateway = ModelHubTurnGateway(service)
request = _prepared_gateway_request(
gateway,
turn_id="turn_meter_cancelled_drain",
requested_model=requested_model,
source_id=source.id,
stream=False,
)
turn = asyncio.create_task(gateway._handle_request(request))
await asyncio.wait_for(handle.started.wait(), timeout=5)
# The scenario is only the scenario while nothing has been read yet.
assert _usage_of(service, source.id) == {}
turn.cancel()
with pytest.raises(asyncio.CancelledError):
await asyncio.wait_for(turn, timeout=5)
await gateway.close()
metered = _usage_of(service, source.id)
assert metered["requests"] == 1
assert metered["token_reports"] == 0
asyncio.run(exercise())
def _unnamed_stream_error(source_id: str, *, stream_started: bool = False) -> RawCallOutcome:
"""A 2xx response whose error frame no machine-code row names, as the engine client reports it."""
return replace(
_outcome(
RawOutcomeKind.HTTP_ERROR,
status=200,
message="upstream returned a protocol error event",
source_id=source_id,
stream_started=stream_started,
),
error_candidates=("relay_specific_error",),
upstream_detail="upstream protocol error",
)
def _assert_source_health_untouched(service: ModelHubService, source_id: str) -> None:
source = next(item for item in service.store.load().sources if item.id == source_id)
assert (source.state.status, source.state.retry_at, source.state.detail_key) == ("standby", None, None)
assert source_id not in service.recovery.annotations(service.store.load())
def test_an_unnamed_stream_error_before_output_falls_back_for_this_turn_only(
tmp_path: Path,
) -> None:
"""Orchestrator decision 2026-10-11: ambiguous evidence never writes Source health."""
async def exercise() -> None:
first = _source("src_primary01", "Primary")
second = _source("src_backup001", "Backup")
service = _service(
tmp_path,
sources=[first, second],
outcomes=[
_unnamed_stream_error(first.id),
_outcome(RawOutcomeKind.SUCCESS, source_id=second.id),
_outcome(RawOutcomeKind.SUCCESS, source_id=first.id),
],
)
model = _canonicalize_fixed_test_routes(service)["codex"]
service.store.config.agents["codex"].routes[model] = ModelHubRouteConfig(
hops=(
ModelHubRouteHopConfig(first.id, "shared-model"),
ModelHubRouteHopConfig(second.id, "shared-model"),
)
)
served = await service.resolve(backend="codex", model_id=model, request={}, supply_channel="hub")
assert served.source_id == second.id
_assert_source_health_untouched(service, first.id)
# The turn's fallback is recorded under its own reason; no cooldown or error event exists.
assert [(event["kind"], event["reason"]) for event in service.events.list()] == [
("switch", "unclassified_error")
]
served = await service.resolve(backend="codex", model_id=model, request={}, supply_channel="hub")
assert served.source_id == first.id
assert [invocation[0] for invocation in service.adapter.invocations] == [first.id, second.id, first.id]
asyncio.run(exercise())
def test_an_unnamed_stream_error_on_the_only_source_ends_the_turn_and_leaves_it_usable(
tmp_path: Path,
) -> None:
async def exercise() -> None:
source = _source("src_primary01", "Primary")
service = _service(
tmp_path,
sources=[source],
outcomes=[_unnamed_stream_error(source.id), _outcome(RawOutcomeKind.SUCCESS, source_id=source.id)],
)
model = _canonicalize_fixed_test_routes(service)["codex"]
with pytest.raises(ModelHubError) as failed:
await service.resolve(backend="codex", model_id=model, request={}, supply_channel="hub")
_assert_source_health_untouched(service, source.id)
assert service.events.list() == []
# No Source is blamed: the request could not complete, and the upstream's own words follow.
assert render_turn_outcome_copy(failed.value.turn_outcome, "en") == " ".join(
(
i18n_t("modelHub.launch.waiting_without_retry", "en", model=model),
i18n_t(
"modelHub.launch.last_upstream_failure_without_status",
"en",
source=source.display_name,
detail="upstream protocol error",
),
)
)
served = await service.resolve(backend="codex", model_id=model, request={}, supply_channel="hub")
assert served.source_id == source.id
asyncio.run(exercise())
def test_gateway_leaves_source_health_alone_for_an_unnamed_stream_error_after_output(
tmp_path: Path,
) -> None:
"""After output a shaped error keeps its class; with no named class it is event-only, never a rejected
request (which the engine client's own words, "upstream returned a protocol error event", once made it)."""
async def exercise() -> None:
source = _source("src_unnamed01", "Unnamed failure")
output = (
b'event: response.output_text.delta\ndata: {"type":'
b'"response.output_text.delta","sequence_number":1}\n\n'
)
service = _service(
tmp_path,
sources=[source],
live_handles=[
LiveInvokeHandle(
_unnamed_stream_error(source.id, stream_started=True),
(
output,
b'event: error\ndata: {"type":"error","code":"relay_specific_error",'
b'"sequence_number":2}\n\n',
),
),
LiveInvokeHandle(
_outcome(RawOutcomeKind.SUCCESS, source_id=source.id, stream_started=True),
(output, b'event: response.completed\ndata: {"type":"response.completed"}\n\n'),
),
],
)
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
gateway = ModelHubTurnGateway(service)
request = _prepared_gateway_request(
gateway,
turn_id="turn_unnamed_failure",
requested_model=requested_model,
source_id=source.id,
stream=True,
)
with patch(
"core.handlers.model_hub.turn_gateway.web.StreamResponse",
return_value=FakeStreamResponse(),
):
await gateway._handle_request(request)
_assert_source_health_untouched(service, source.id)
assert service.events.list() == []
gateway.correlation.settle(
"turn_unnamed_failure", settled_by=SETTLED_BY_TERMINAL_RESULT, ts=NOW.isoformat(),
)
record = service.provenance.get("turn_unnamed_failure")
assert record["terminal_error"]["reason"] == "stream_interrupted"
served = await service.resolve(
backend="codex", model_id=requested_model, request={}, supply_channel="hub",
)
assert served.source_id == source.id
assert [invocation[0] for invocation in service.adapter.invocations] == [source.id, source.id]
asyncio.run(exercise())
def test_gateway_meters_a_failed_turn_that_upstream_already_billed(
tmp_path: Path,
) -> None:
"""A vendor that reported tokens billed us even when the stream failed."""
async def exercise() -> None:
source = _source("src_meterfail01", "Billed failure")
service = _service(
tmp_path,
sources=[source],
live_handles=[
LiveInvokeHandle(
_outcome(
RawOutcomeKind.HTTP_ERROR,
status=500,
source_id=source.id,
stream_started=True,
),
(
b'event: response.output_text.delta\ndata: {"type":'
b'"response.output_text.delta","sequence_number":1}\n\n',
b'event: error\ndata: {"type":"error","code":"server_error",'
b'"sequence_number":2,"usage":{"input_tokens":41,'
b'"output_tokens":0}}\n\n',
),
)
],
)
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
gateway = ModelHubTurnGateway(service)
request = _prepared_gateway_request(
gateway,
turn_id="turn_meter_billed_failure",
requested_model=requested_model,
source_id=source.id,
stream=True,
)
with patch(
"core.handlers.model_hub.turn_gateway.web.StreamResponse",
return_value=FakeStreamResponse(),
):
await gateway._handle_request(request)
metered = _usage_of(service, source.id)
assert metered["requests"] == 1
assert metered["token_reports"] == 1
assert metered["input_tokens"] == 41
# The failure keeps its own surface; metering does not replace it.
assert service.store.load().sources[0].state.status != "standby"
asyncio.run(exercise())
def test_gateway_meters_nothing_for_a_turn_that_never_reached_the_model(
tmp_path: Path,
) -> None:
async def exercise() -> None:
source = _source("src_meternone01", "Unreached", status="needs_action")
service = _service(tmp_path, sources=[source])
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
service.store.config.agents["codex"].routes[requested_model] = ModelHubRouteConfig(
hops=(ModelHubRouteHopConfig(source.id, "removed-model"),)
)
gateway = ModelHubTurnGateway(service)
request = _prepared_gateway_request(
gateway,
turn_id="turn_meter_unreached",
requested_model=requested_model,
source_id=source.id,
stream=False,
)
result = await gateway._handle_request(request)
assert result.status == 409
summary = service.usage_summary(days=30)
assert summary["sources"] == []
assert summary["totals"]["requests"] == 0
asyncio.run(exercise())
def test_a_surfaced_buffered_error_still_meters_the_tokens_it_billed(
tmp_path: Path,
) -> None:
"""The call never hands a body onward, so only the resolver can meter it."""
async def exercise() -> None:
source = _source("src_meterbuferr", "Surfaced billing")
service = _service(
tmp_path,
sources=[source],
outcomes=[
_outcome(
RawOutcomeKind.HTTP_ERROR,
status=400,
code="invalid_request_error",
source_id=source.id,
usage=ProtocolUsageReport.of(
input_tokens=310,
cached_input_tokens=64,
output_tokens=0,
),
)
],
)
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
with pytest.raises(ModelHubError) as raised:
await service.resolve(
backend="codex",
model_id=requested_model,
request=ModelHubRequest(
{"model": requested_model, "input": "ping"},
protocol="openai_responses",
),
supply_channel="hub",
)
assert raised.value.code == "upstream_request_invalid"
metered = _usage_of(service, source.id)
assert metered["requests"] == 1
assert metered["token_reports"] == 1
assert metered["input_tokens"] == 310
assert metered["cached_input_tokens"] == 64
assert metered["last_metered_at"] == NOW.isoformat()
asyncio.run(exercise())
def test_a_billed_failover_hop_is_metered_against_the_source_that_billed_it(
tmp_path: Path,
) -> None:
"""MH-USAGE-002: every hop that reported tokens billed its own Source, not the last one."""
async def exercise() -> None:
first = _source("src_meterhop001", "Billed hop")
second = _source("src_meterhop002", "Serving hop")
service = _service(
tmp_path,
sources=[first, second],
outcomes=[
_outcome(
RawOutcomeKind.HTTP_ERROR,
status=429,
code="rate_limit_error",
source_id=first.id,
usage=ProtocolUsageReport.of(
input_tokens=88,
cached_input_tokens=0,
output_tokens=0,
),
),
_outcome(
RawOutcomeKind.SUCCESS,
status=200,
source_id=second.id,
usage=ProtocolUsageReport.of(
input_tokens=120,
cached_input_tokens=0,
output_tokens=17,
),
),
],
)
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
service.store.config.agents["codex"].routes[requested_model] = ModelHubRouteConfig(
hops=(
ModelHubRouteHopConfig(first.id, "shared-model"),
ModelHubRouteHopConfig(second.id, "shared-model"),
)
)
resolved = await service.resolve(
backend="codex",
model_id=requested_model,
request=ModelHubRequest(
{"model": requested_model, "input": "ping"},
protocol="openai_responses",
),
supply_channel="hub",
)
assert resolved.source_id == second.id
assert _usage_of(service, first.id)["input_tokens"] == 88
assert _usage_of(service, second.id)["input_tokens"] == 120
# One turn, two upstream calls: the unit is the call, so the total says so.
assert service.usage_summary(days=30)["totals"]["requests"] == 2
asyncio.run(exercise())
def test_a_call_that_reached_no_model_is_never_metered(tmp_path: Path) -> None:
"""A rejected credential billed nothing; source health already reports it."""
async def exercise() -> None:
source = _source("src_meterauth01", "Rejected")
service = _service(
tmp_path,
sources=[source],
outcomes=[
_outcome(
RawOutcomeKind.HTTP_ERROR,
status=401,
code="authentication_error",
source_id=source.id,
)
],
)
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
with pytest.raises(ModelHubError):
await service.resolve(
backend="codex",
model_id=requested_model,
request=ModelHubRequest(
{"model": requested_model, "input": "ping"},
protocol="openai_responses",
),
supply_channel="hub",
)
assert service.usage_summary(days=30)["totals"]["requests"] == 0
asyncio.run(exercise())
def test_a_deleted_source_keeps_its_usage_and_reads_as_removed(tmp_path: Path) -> None:
"""MH-USAGE-034: the Usage tab folds a Source by the null label this report gives it.
Deleting a Source lets config go, not the usage it metered: every row stays,
with its counts, under the ID it was metered under, and only the label goes,
because a configured Source always carries its name. A model its live Source
stops listing keeps that Source's label and loses only its own.
"""
async def exercise() -> None:
gone = _source("src_usagegone01", "Deleted supplier")
live = _source("src_usagelive01", "Live supplier")
service = _service(tmp_path, sources=[gone, live])
_canonicalize_fixed_test_routes(service)
metered = [
(gone.id, "glm-5.3", 100),
(gone.id, "kimi-k2", 40),
(live.id, "shared-model", 7),
(live.id, "glm-4.6-air", 3),
]
for source_id, model_id, input_tokens in metered:
service.usage.record(
source_id=source_id,
model_id=model_id,
usage=ProtocolUsageReport.of(input_tokens=input_tokens, cached_input_tokens=0, output_tokens=1),
at=NOW,
)
with pytest.raises(ModelHubError) as guard:
await service.delete_source(gone.id)
await service.delete_source(
gone.id,
force=True,
confirmed_remove_hops=guard.value.data["would_remove_hops"],
confirmed_interruptions=guard.value.data["would_interrupt"],
)
report = service.usage_summary(window="24h")
sources = {source["source_id"]: source for source in report["sources"]}
labels = {
source_id: (source["label"], {model["model_id"]: model["label"] for model in source["models"]})
for source_id, source in sources.items()
}
assert labels == {
gone.id: (None, {"glm-5.3": None, "kimi-k2": None}),
live.id: ("Live supplier", {"shared-model": "shared-model", "glm-4.6-air": None}),
}
assert sources[gone.id]["input_tokens"] == 140
assert report["totals"]["input_tokens"] == 150
assert sum(row["input_tokens"] for bucket in report["buckets"] for row in bucket["rows"]) == 150
asyncio.run(exercise())
def test_a_downstream_disconnect_meters_the_terminal_frame_exactly_once(
tmp_path: Path,
) -> None:
"""The wire read the tokens before the write failed, so the turn was billed."""
async def exercise() -> None:
source = _source("src_meterdrop01", "Dropped client")
service = _service(
tmp_path,
sources=[source],
live_handles=[
LiveInvokeHandle(
_outcome(
RawOutcomeKind.SUCCESS,
status=200,
source_id=source.id,
stream_started=True,
),
(
b'event: response.completed\ndata: {"type":"response.completed",'
b'"sequence_number":1,"response":{"usage":{"input_tokens":512,'
b'"output_tokens":33}}}\n\n',
),
)
],
)
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
gateway = ModelHubTurnGateway(service)
request = _prepared_gateway_request(
gateway,
turn_id="turn_meter_disconnect",
requested_model=requested_model,
source_id=source.id,
stream=True,
)
response = FakeStreamResponse(
write_error=ConnectionResetError("downstream write failed")
)
with patch(
"core.handlers.model_hub.turn_gateway.web.StreamResponse",
return_value=response,
):
with pytest.raises(ConnectionError, match="downstream"):
await asyncio.wait_for(gateway._handle_request(request), timeout=1)
metered = _usage_of(service, source.id)
assert metered["requests"] == 1
assert metered["token_reports"] == 1
assert metered["input_tokens"] == 512
assert metered["output_tokens"] == 33
asyncio.run(exercise())
def test_neither_metering_owner_writes_the_ledger_on_the_event_loop(
tmp_path: Path,
) -> None:
"""MH-USAGE-004: a ledger read-modify-write is file I/O; the loop must not wait."""
async def exercise() -> None:
recording_threads: list[int] = []
def capture(_calls: object) -> None:
recording_threads.append(threading.get_ident())
resolver_source = _source("src_meterloop01", "Resolver owner")
resolver_service = _service(
tmp_path / "resolver",
sources=[resolver_source],
outcomes=[
_outcome(
RawOutcomeKind.SUCCESS,
status=200,
source_id=resolver_source.id,
usage=ProtocolUsageReport.of(
input_tokens=5,
cached_input_tokens=0,
output_tokens=1,
),
)
],
)
resolver_service.usage.record_many = capture
requested_model = _canonicalize_fixed_test_routes(resolver_service)["codex"]
await resolver_service.resolve(
backend="codex",
model_id=requested_model,
request=ModelHubRequest(
{"model": requested_model, "input": "ping"},
protocol="openai_responses",
),
supply_channel="hub",
)
gateway_source = _source("src_meterloop02", "Gateway owner")
gateway_service = _service(
tmp_path / "gateway",
sources=[gateway_source],
live_handles=[
LiveInvokeHandle(
_outcome(
RawOutcomeKind.SUCCESS, status=200, source_id=gateway_source.id
),
(json.dumps({"id": "resp", "usage": {"input_tokens": 9}}).encode(),),
)
],
)
gateway_service.usage.record_many = capture
gateway_model = _canonicalize_fixed_test_routes(gateway_service)["codex"]
gateway = ModelHubTurnGateway(gateway_service)
await gateway._handle_request(
_prepared_gateway_request(
gateway,
turn_id="turn_meter_offloop",
requested_model=gateway_model,
source_id=gateway_source.id,
stream=False,
)
)
assert len(recording_threads) == 2
assert threading.get_ident() not in recording_threads
asyncio.run(exercise())
def test_a_usage_ledger_failure_cannot_change_the_served_turn(tmp_path: Path) -> None:
"""Metering is a report, so a ledger fault must stay invisible downstream."""
async def exercise() -> None:
source = _source("src_meterbust01", "Broken ledger")
body = json.dumps({"id": "resp_ok", "usage": {"input_tokens": 7}}).encode("utf-8")
service = _service(
tmp_path,
sources=[source],
live_handles=[
LiveInvokeHandle(
_outcome(RawOutcomeKind.SUCCESS, status=200, source_id=source.id),
(body,),
)
],
)
service.usage.record_many = Mock(side_effect=OSError("read-only state directory"))
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
gateway = ModelHubTurnGateway(service)
request = _prepared_gateway_request(
gateway,
turn_id="turn_meter_broken_ledger",
requested_model=requested_model,
source_id=source.id,
stream=False,
)
result = await gateway._handle_request(request)
assert result.status == 200
assert result.body == body
service.usage.record_many.assert_called_once()
assert service.store.load().sources[0].state.status == "standby"
asyncio.run(exercise())
def test_a_settlement_that_raises_still_meters_the_call_it_was_settling(
tmp_path: Path,
) -> None:
"""MH-USAGE-010: the vendor billed the call before our bookkeeping got a turn.
Metering used to be each ending's own step, placed after that ending's
bookkeeping, so a settlement that raised on the way through took the row with
it. Both shapes, because the two reach the ending from different frames and
each used to carry its own copy of the order — and `requests == 1` rather than
`>= 1`, because the boundary that catches the raise re-enters the same ending.
"""
async def exercise(stream: bool) -> None:
state = tmp_path / ("stream" if stream else "buffer")
state.mkdir()
source = _source("src_meterraise1", "Broken settlement")
chunks = (
(
b'event: response.completed\ndata: {"type":"response.completed",'
b'"sequence_number":1,"response":{"usage":{"input_tokens":256,'
b'"output_tokens":12}}}\n\n',
)
if stream
else (b'{"usage":{"input_tokens":256,"output_tokens":12}}',)
)
service = _service(
state,
sources=[source],
live_handles=[
LiveInvokeHandle(
_outcome(
RawOutcomeKind.SUCCESS,
status=200,
source_id=source.id,
stream_started=stream,
usage=(
None
if stream
else ProtocolUsageReport.of(
input_tokens=256,
cached_input_tokens=0,
output_tokens=12,
)
),
),
chunks,
)
],
)
async def exploding_settlement(*args: object, **kwargs: object):
raise RuntimeError("settlement exploded")
service.settle_handle_outcome = exploding_settlement
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
gateway = ModelHubTurnGateway(service)
request = _prepared_gateway_request(
gateway,
turn_id=f"turn_meter_raise_{'stream' if stream else 'buffer'}",
requested_model=requested_model,
source_id=source.id,
stream=stream,
)
with patch(
"core.handlers.model_hub.turn_gateway.web.StreamResponse",
return_value=FakeStreamResponse(),
):
with pytest.raises(RuntimeError, match="settlement exploded"):
await asyncio.wait_for(gateway._handle_request(request), timeout=5)
await gateway.close()
metered = _usage_of(service, source.id)
assert metered["requests"] == 1, stream
assert metered["token_reports"] == 1, stream
assert metered["input_tokens"] == 256, stream
assert metered["output_tokens"] == 12, stream
for stream in (False, True):
asyncio.run(exercise(stream))
def test_a_ledger_that_stopped_answering_cannot_hold_the_served_turn_open(
tmp_path: Path,
) -> None:
"""MH-USAGE-011: waiting for the row is an ordering convenience, not a gate.
Metering waits out its own write so a client that opens the usage tab right
after its call already sees that call. Unbounded, the convenience becomes the
turn's critical path: hold the one thread ledger writes run on and the served
response waits behind a disk that has stopped answering. Bounded, the turn is
served and the row stays queued — timed out is not dropped, which is the other
half of the same wait.
"""
async def exercise() -> None:
source = _source("src_meterhang01", "Unresponsive ledger")
body = json.dumps({"usage": {"input_tokens": 64, "output_tokens": 4}}).encode("utf-8")
service = _service(
tmp_path,
sources=[source],
live_handles=[
LiveInvokeHandle(
_outcome(
RawOutcomeKind.SUCCESS,
status=200,
source_id=source.id,
usage=ProtocolUsageReport.of(
input_tokens=64,
cached_input_tokens=0,
output_tokens=4,
),
),
(body,),
)
],
)
service.usage_writer = UsageWriter(service.usage, durability_wait=0.05)
requested_model = _canonicalize_fixed_test_routes(service)["codex"]
gateway = ModelHubTurnGateway(service)
request = _prepared_gateway_request(
gateway,
turn_id="turn_meter_hung_ledger",
requested_model=requested_model,
source_id=source.id,
stream=False,
)
with _occupied_ledger_writer():
result = await asyncio.wait_for(gateway._handle_request(request), timeout=1)
assert result.status == 200
assert result.body == body
# The scenario is only the scenario while the write is still queued,
# and a queued write is still the writer's to finish.
assert service.usage_writer.unpersisted == 1
assert _usage_of(service, source.id) == {}
assert await service.usage_writer.drain(timeout=5) == 0
metered = _usage_of(service, source.id)
assert metered["requests"] == 1
assert metered["input_tokens"] == 64
assert metered["output_tokens"] == 4
asyncio.run(exercise())