Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

42.5 KB
Raw Blame
"""Contract tests for the standalone Model Hub mock upstream."""

from __future__ import annotations

import json
import re
import shutil
import socket
import subprocess
import sys
import time
from pathlib import Path
from typing import Any
from urllib.error import HTTPError, URLError
from urllib.parse import urlsplit
from urllib.request import ProxyHandler, Request, build_opener

import psutil
import pytest
import yaml

from tests.e2e.drivers.mock_llm_upstream import (
    MockLLMUpstream,
    OPENCODE_V4_S3_PASSTHROUGH_FIXTURES,
    OPENCODE_V4_S4_VARIANT_FIXTURES,
)
from tests.e2e.test_model_hub_runtime import _engine_app, _install_engine
from tests.e2e.test_model_hub_sources import _configure_protocol, _create_source


pytestmark = pytest.mark.e2e_model_hub

PROTOCOL_CASES = {
    "anthropic": {
        "path": "/v1/messages",
        "body": {
            "model": "mock-model",
            "max_tokens": 32,
            "messages": [{"role": "user", "content": "hello"}],
        },
        "response_key": "type",
        "response_value": "message",
        "terminal": "message_stop",
        "invalid_param": None,
    },
    "openai_responses": {
        "path": "/v1/responses",
        "body": {"model": "mock-model", "input": "hello"},
        "response_key": "object",
        "response_value": "response",
        "terminal": "response.completed",
        "invalid_param": "input",
    },
    "openai_chat": {
        "path": "/v1/chat/completions",
        "body": {
            "model": "mock-model",
            "messages": [{"role": "user", "content": "hello"}],
        },
        "response_key": "object",
        "response_value": "chat.completion",
        "terminal": "[DONE]",
        "invalid_param": "messages",
    },
}

_NO_PROXY_OPENER = build_opener(ProxyHandler({}))


def _request(
    base_url: str,
    path: str,
    *,
    method: str = "GET",
    body: object | None = None,
    headers: dict[str, str] | None = None,
    timeout: float = 2,
) -> tuple[int, dict[str, str], bytes]:
    data = None
    request_headers = dict(headers or {})
    if body is not None:
        data = json.dumps(body).encode("utf-8")
        request_headers.setdefault("Content-Type", "application/json")
    request = Request(
        f"{base_url}{path}",
        data=data,
        headers=request_headers,
        method=method,
    )
    try:
        response = _NO_PROXY_OPENER.open(request, timeout=timeout)
    except HTTPError as exc:
        response = exc
    with response:
        return response.status, dict(response.headers), response.read()


def _json_request(
    base_url: str,
    path: str,
    *,
    method: str = "GET",
    body: object | None = None,
    headers: dict[str, str] | None = None,
    timeout: float = 2,
) -> tuple[int, dict[str, str], Any]:
    status, response_headers, raw = _request(
        base_url,
        path,
        method=method,
        body=body,
        headers=headers,
        timeout=timeout,
    )
    return status, response_headers, json.loads(raw)


def _configure(base_url: str, **config: object) -> dict[str, Any]:
    status, _, payload = _json_request(
        base_url, "/__control/config", method="POST", body=config
    )
    assert status == 200
    assert payload["ok"] is True
    return payload["config"]


def _top_level_diff(before: dict[str, Any], after: dict[str, Any]) -> dict[str, Any]:
    return {
        "added": {key: after[key] for key in after.keys() - before.keys()},
        "removed": sorted(before.keys() - after.keys()),
        "changed": {
            key: [before[key], after[key]]
            for key in before.keys() & after.keys()
            if before[key] != after[key]
        },
    }


def _send_through_real_engine(
    model_hub_app_factory,
    mock_llm_upstream: MockLLMUpstream,
    *,
    upstream_protocol: str,
    frontend_protocol: str,
    stored_model: str,
    frontend_body: dict[str, Any],
) -> tuple[int, dict[str, Any]]:
    with _engine_app(model_hub_app_factory) as app:
        _install_engine(app)
        _configure_protocol(
            mock_llm_upstream,
            upstream_protocol,
            models=[
                {
                    "id": stored_model,
                    "supported_parameters": ["reasoning"],
                }
            ],
        )
        source, _ = _create_source(
            app,
            mock_llm_upstream,
            protocol=upstream_protocol,
            nonce="scn_0000opencodev4s3",
            extra=(
                {"base_url": f"{mock_llm_upstream.url}/v1"}
                if upstream_protocol == "openai_responses"
                else None
            ),
        )
        source_model = next(
            model for model in source["models"] if model["id"] == stored_model
        )
        assert "high" in source_model["reasoning_efforts"], source_model
        started = app.client.post("/api/models/runtime/start", {})
        started_body = started.json()
        assert started.status == 200, started_body
        listening = started_body["runtime"]["status"]["listening"]

        [config_path] = list(
            (
                app.avibe_home
                / "runtime"
                / "model-hub"
                / "state"
                / "instances"
            ).glob("*/config.yaml")
        )
        config = yaml.safe_load(config_path.read_text(encoding="utf-8"))
        entry_key = (
            "claude-api-key"
            if upstream_protocol == "anthropic"
            else "codex-api-key"
        )
        [credential] = config[entry_key]
        inbound = {
            **frontend_body,
            "model": f"{credential['prefix']}/{stored_model}",
        }
        _configure_protocol(
            mock_llm_upstream,
            upstream_protocol,
            models=[
                {
                    "id": stored_model,
                    "supported_parameters": ["reasoning"],
                }
            ],
        )
        path = (
            "/v1/messages"
            if frontend_protocol == "anthropic"
            else "/v1/responses"
        )
        auth_header = (
            {"X-Api-Key": config["api-keys"][0]}
            if frontend_protocol == "anthropic"
            else {"Authorization": f"Bearer {config['api-keys'][0]}"}
        )
        status, _, response_body = _request(
            f"http://{listening['host']}:{listening['port']}",
            path,
            method="POST",
            body=inbound,
            headers=auth_header,
            timeout=20,
        )
        captured = [
            request
            for request in mock_llm_upstream.requests()
            if request["path"]
            == (
                "/v1/messages"
                if upstream_protocol == "anthropic"
                else "/v1/responses"
            )
        ]
        assert len(captured) == 1, {
            "engine_status": status,
            "engine_body": response_body[:1000].decode("utf-8", errors="replace"),
            "upstream_requests": mock_llm_upstream.requests(),
        }
        return status, captured[0]["body"]


def _opencode_generation_records(app) -> list[dict[str, Any]]:
    """The process record of each OpenCode generation the app runs (#2329)."""

    records = app.avibe_home / "runtime" / "opencode" / "generations"
    return [
        json.loads(path.read_text(encoding="utf-8"))
        for path in sorted(records.glob("*.json"))
        if not path.name.endswith(".overlay.json")
    ]


def _send_through_opencode_gateway_engine(
    model_hub_app_factory,
    mock_llm_upstream: MockLLMUpstream,
    *,
    upstream_protocol: str,
    native_protocol: str,
    stored_model: str,
    variant: dict[str, Any],
    prompt_start_delay_seconds: float = 0,
    require_terminal_result: bool = False,
    inventory_models: list[dict[str, Any]] | None = None,
    automatic_route: bool = False,
    isolated_upstream: MockLLMUpstream | None = None,
) -> tuple[dict[str, Any], dict[str, Any] | None]:
    binary = shutil.which("opencode")
    if binary is None:
        pytest.skip("OpenCode executable is unavailable")
    menu_model = (
        "claude-opus-5"
        if native_protocol == "anthropic"
        else "deepseek-v3.2"
    )
    upstream_path = {
        "anthropic": "/v1/messages",
        "openai_responses": "/v1/responses",
        "openai_chat": "/v1/chat/completions",
    }[upstream_protocol]
    inventory = inventory_models if inventory_models is not None else [
        {"id": stored_model, "supported_parameters": ["reasoning"]},
    ]
    with _engine_app(model_hub_app_factory) as app:
        configured = app.client.post(
            "/api/config",
            {"agents": {"opencode": {"enabled": True, "cli_path": binary}}},
        )
        assert configured.status == 200, configured.json()
        _install_engine(app)
        _configure_protocol(
            mock_llm_upstream,
            upstream_protocol,
            models=inventory,
        )
        source, _ = _create_source(
            app,
            mock_llm_upstream,
            protocol=upstream_protocol,
            nonce="scn_0000opencodev4s4",
            extra=(
                {"base_url": f"{mock_llm_upstream.url}/v1"}
                if upstream_protocol == "openai_responses"
                else None
            ),
        )
        if inventory_models is None:
            source_model = next(model for model in source["models"] if model["id"] == stored_model)
            assert "high" in source_model["reasoning_efforts"], source_model
        else:
            assert {model["id"] for model in source["models"]} == {model["id"] for model in inventory}
        if isolated_upstream is not None:
            _configure_protocol(isolated_upstream, upstream_protocol, models=[{"id": stored_model}])
            _create_source(
                app, isolated_upstream, protocol=upstream_protocol, nonce="scn_routingoutside001",
                extra=({"base_url": f"{isolated_upstream.url}/v1"} if upstream_protocol == "openai_responses" else None),
            )
        sources = app.client.put(
            "/api/models/agents/opencode/sources",
            {"order": [source["id"]]},
        )
        assert sources.status == 200, sources.json()
        current_models = app.client.get("/api/models/agents/opencode/models")
        current_body = current_models.json()
        assert current_models.status == 200, current_body
        baseline = current_body["agent"]["catalog_models"]
        rows = [
            {
                "id": "deepseek-v3.2",
                "origin": "manual",
                "supports_reasoning": True,
                "reasoning_efforts": ["high"],
                "native_protocol": "openai_responses",
            },
            {
                "id": "claude-opus-5",
                "origin": "manual",
                "supports_reasoning": True,
                "reasoning_efforts": ["high"],
                "native_protocol": "anthropic",
            },
        ]
        models = app.client.put(
            "/api/models/agents/opencode/models",
            {"baseline": baseline, "models": rows},
        )
        assert models.status == 200, models.json()
        mode = app.client.patch(
            "/api/models/agents/opencode/mode",
            {"mode": "hub"},
        )
        assert mode.status == 200, mode.json()
        route_path = f"/api/models/agents/opencode/chain?model={menu_model}"
        chain = app.client.get(route_path) if automatic_route else app.client.put(
            route_path, {"hops": [{"source_id": source["id"], "model_id": stored_model}]},
        )
        assert chain.status == 200, chain.json()
        if inventory_models is not None:
            assert chain.json()["chain"]["route_origin"] == ("passthrough" if automatic_route else "manual")
            assert chain.json()["chain"]["manual_override"] == (
                None if automatic_route else {"hops": [{"source_id": source["id"], "model_id": stored_model}]}
            )

        options = app.client.post(
            "/api/opencode/options",
            {"cwd": "~"},
        )
        options_body = options.json()
        assert options.status == 200, options_body
        assert options_body["ok"] is True
        projected_providers = {
            provider["id"]: provider
            for provider in options_body["data"]["models"]["providers"]
        }
        assert set(projected_providers) == {
            "avibe-openai",
            "avibe-anthropic",
        }
        assert set(projected_providers["avibe-openai"]["models"]) == {
            "deepseek-v3.2"
        }
        assert set(projected_providers["avibe-anthropic"]["models"]) == {
            "claude-opus-5"
        }
        assert not _opencode_generation_records(app)

        started = app.client.post("/api/models/runtime/start", {})
        assert started.status == 200, started.json()
        engine_state = app.avibe_home / "runtime" / "model-hub" / "state"
        engine_configs = {path: (path.read_bytes(), path.stat().st_mtime_ns) for path in engine_state.glob("instances/*/config.yaml")}

        agent = app.client.post(
            "/api/agents",
            {
                "name": "opencode-v4-e2e",
                "backend": "opencode",
                "model": menu_model,
                "reasoning_effort": "high",
                "system_prompt": "System fixture",
            },
        )
        assert agent.status == 200, agent.json()
        if prompt_start_delay_seconds > 0:
            plugin_dir = app.runtime_root / ".opencode" / "plugins"
            plugin_dir.mkdir(parents=True)
            (plugin_dir / "delay-prompt.js").write_text(
                "export const DelayPrompt = async () => ({\n"
                '  "chat.message": async () => {\n'
                f"    await Bun.sleep({prompt_start_delay_seconds * 1000});\n"
                "  },\n"
                "});\n",
                encoding="utf-8",
            )
        project = app.client.post(
            "/api/projects",
            {
                "folder_path": str(app.runtime_root),
                "display_name": "OpenCode v4 E2E",
            },
        )
        assert project.status == 201, project.json()
        session = app.client.post(
            "/api/sessions",
            {
                "project_id": project.json()["id"],
                "agent_name": "opencode-v4-e2e",
                "model": menu_model,
                "reasoning_effort": "high",
            },
        )
        assert session.status == 201, session.json()

        _configure_protocol(
            mock_llm_upstream,
            upstream_protocol,
            models=inventory,
        )
        if isolated_upstream is not None:
            isolated_upstream.reset_requests()
        sent = app.client.post(
            f"/api/sessions/{session.json()['id']}/messages",
            {"text": "hello"},
        )
        assert sent.status == 202, sent.json()
        deadline = time.monotonic() + 60
        captured: list[dict[str, Any]] = []
        result: dict[str, Any] | None = None
        while time.monotonic() < deadline:
            captured = [
                request
                for request in mock_llm_upstream.requests()
                if request["path"] == upstream_path
                and "You are a title generator"
                not in json.dumps(request["body"], sort_keys=True)
            ]
            if require_terminal_result:
                transcript = app.client.get(
                    f"/api/sessions/{session.json()['id']}/messages?tail=1"
                )
                assert transcript.status == 200, transcript.json()
                result = next(
                    (
                        row
                        for row in reversed(transcript.json()["messages"])
                        if row["type"] == "result"
                    ),
                    None,
                )
            if captured and (result is not None or not require_terminal_result):
                break
            time.sleep(0.1)
        assert len(captured) == 1, app.diagnostics()
        if inventory_models is not None:
            assert captured[0]["body"]["model"] == stored_model
            current = app.client.get("/api/models/sources")
            assert current.status == 200, current.json()
            current_source = next(row for row in current.json()["sources"] if row["id"] == source["id"])
            assert {row["id"] for row in current_source["models"]} == {row["id"] for row in inventory}
            assert app.client.get(route_path).json()["chain"]["manual_override"] == chain.json()["chain"]["manual_override"]
            assert engine_configs
            assert {path: (path.read_bytes(), path.stat().st_mtime_ns) for path in engine_state.glob("instances/*/config.yaml")} == engine_configs
        if isolated_upstream is not None:
            assert not [row for row in isolated_upstream.requests() if row["path"] == upstream_path]
        if require_terminal_result:
            assert result is not None, app.diagnostics()
            assert result["text"] == "mock response", {
                "result": result,
                "diagnostics": app.diagnostics(),
            }
        overlay = json.loads(
            (
                app.avibe_home
                / "runtime"
                / "model-hub"
                / "opencode-overlay.json"
            ).read_text(encoding="utf-8")
        )
        provider_id = {
            "anthropic": "avibe-anthropic",
            "openai_responses": "avibe-openai",
        }[native_protocol]
        assert overlay["enabled_providers"] == [
            "avibe-openai",
            "avibe-anthropic",
        ]
        assert set(overlay["provider"]) == {
            "avibe-openai",
            "avibe-anthropic",
        }
        assert set(overlay["provider"]["avibe-openai"]["models"]) == {
            "deepseek-v3.2"
        }
        assert set(overlay["provider"]["avibe-anthropic"]["models"]) == {
            "claude-opus-5"
        }
        assert overlay["provider"][provider_id]["models"][menu_model]["variants"][
            "high"
        ] == variant

        # The one OpenCode generation serving the turn, the controller's own child.
        (pid_info,) = _opencode_generation_records(app)
        assert app._controller is not None
        assert psutil.Process(pid_info["pid"]).ppid() == app._controller.pid
        providers_status, _, providers_payload = _json_request(
            f"http://{pid_info['host']}:{pid_info['port']}",
            "/config/providers",
            headers={"x-opencode-directory": str(app.runtime_root)},
            timeout=10,
        )
        assert providers_status == 200, providers_payload
        live_providers = {
            provider["id"]: provider
            for provider in providers_payload["providers"]
        }
        assert set(live_providers) == {
            "avibe-openai",
            "avibe-anthropic",
        }
        assert set(live_providers["avibe-openai"]["models"]) == {
            "deepseek-v3.2"
        }
        assert set(live_providers["avibe-anthropic"]["models"]) == {
            "claude-opus-5"
        }
        return captured[0]["body"], result


@pytest.mark.parametrize("protocol", ["openai_responses", "openai_chat", "anthropic"])
@pytest.mark.parametrize("inventory", [[], [{"id": "unrelated-listed-model"}]], ids=["empty", "incomplete"])
@pytest.mark.parametrize("automatic", [True, False], ids=["passthrough", "manual"])
def test_mh_routing_005_real_opencode_gateway_cpa_preserves_unknown_model_and_source_isolation(
    model_hub_app_factory, mock_llm_upstream, protocol, inventory, automatic,
):
    """MH-ROUTING-005: real backend -> gateway -> pinned CPA forwards an unlisted exact model only to its source."""
    native = "anthropic" if protocol == "anthropic" else "openai_responses"
    menu = "claude-opus-5" if native == "anthropic" else "deepseek-v3.2"
    target = menu if automatic else "vendor/unlisted-route-target"
    variant = {"effort": "high"} if native == "anthropic" else {"reasoningEffort": "high"}
    with MockLLMUpstream() as other:
        captured, result = _send_through_opencode_gateway_engine(
            model_hub_app_factory, mock_llm_upstream, upstream_protocol=protocol, native_protocol=native,
            stored_model=target, variant=variant, inventory_models=inventory, automatic_route=automatic,
            isolated_upstream=other, require_terminal_result=True,
        )
    assert captured["model"] == target
    assert result is not None


@pytest.mark.parametrize(
    "fixture",
    OPENCODE_V4_S3_PASSTHROUGH_FIXTURES.values(),
    ids=OPENCODE_V4_S3_PASSTHROUGH_FIXTURES,
)
def test_opencode_v4_s3_engine_diffs_are_frozen_against_mock_capture(
    model_hub_app_factory,
    mock_llm_upstream,
    fixture: dict[str, Any],
) -> None:
    status, captured_body = _send_through_real_engine(
        model_hub_app_factory,
        mock_llm_upstream,
        upstream_protocol=fixture["protocol"],
        frontend_protocol=fixture["protocol"],
        stored_model=fixture["stored_model"],
        frontend_body=fixture["frontend_body"],
    )

    assert status == 200
    assert captured_body == fixture["upstream_body"]
    assert _top_level_diff(
        fixture["frontend_body"],
        captured_body,
    ) == fixture["expected_top_level_diff"]


@pytest.mark.parametrize(
    "fixture",
    OPENCODE_V4_S4_VARIANT_FIXTURES.values(),
    ids=OPENCODE_V4_S4_VARIANT_FIXTURES,
)
def test_opencode_v4_s4_variant_shapes_are_frozen_against_mock_capture(
    model_hub_app_factory,
    mock_llm_upstream,
    fixture: dict[str, Any],
) -> None:
    captured_body, _ = _send_through_opencode_gateway_engine(
        model_hub_app_factory,
        mock_llm_upstream,
        upstream_protocol=fixture["protocol"],
        native_protocol=fixture["frontend_protocol"],
        stored_model=fixture["stored_model"],
        variant=fixture["variant"],
    )

    assert captured_body["model"] == fixture["stored_model"]
    # What the variant sets must survive; fields OpenCode adds on its own vary by its version.
    assert _contains(captured_body, fixture["upstream_fragment"]), captured_body


def _contains(actual: object, expected: object) -> bool:
    """Whether ``actual`` holds ``expected``: every key of an expected mapping, recursively; anything else equal."""

    if isinstance(expected, dict):
        return isinstance(actual, dict) and all(
            key in actual and _contains(actual[key], value) for key, value in expected.items()
        )
    return actual == expected


def test_opencode_responses_turn_completes_through_openai_chat_upstream(
    model_hub_app_factory,
    mock_llm_upstream,
) -> None:
    fixture = OPENCODE_V4_S4_VARIANT_FIXTURES["responses_to_openai_chat"]

    captured_body, result = _send_through_opencode_gateway_engine(
        model_hub_app_factory,
        mock_llm_upstream,
        upstream_protocol=fixture["protocol"],
        native_protocol=fixture["frontend_protocol"],
        stored_model=fixture["stored_model"],
        variant=fixture["variant"],
        require_terminal_result=True,
    )

    assert captured_body["model"] == fixture["stored_model"]
    assert result is not None
    assert result["text"] == "mock response"
    for key, value in fixture["upstream_fragment"].items():
        assert captured_body.get(key) == value, captured_body


def test_opencode_anthropic_turn_waits_for_accepted_prompt_worker(
    model_hub_app_factory,
    mock_llm_upstream,
) -> None:
    fixture = OPENCODE_V4_S4_VARIANT_FIXTURES["anthropic_same_protocol"]

    captured_body, result = _send_through_opencode_gateway_engine(
        model_hub_app_factory,
        mock_llm_upstream,
        upstream_protocol=fixture["protocol"],
        native_protocol=fixture["frontend_protocol"],
        stored_model=fixture["stored_model"],
        variant=fixture["variant"],
        prompt_start_delay_seconds=6.5,
        require_terminal_result=True,
    )

    assert captured_body["model"] == fixture["stored_model"]
    assert result is not None
    assert result["text"] == "mock response"
    for key, value in fixture["upstream_fragment"].items():
        assert captured_body.get(key) == value, captured_body


@pytest.mark.parametrize("protocol", PROTOCOL_CASES)
def test_mock_upstream_inventory_and_buffered_protocol_shapes(
    mock_llm_upstream, protocol: str
) -> None:
    """Mock contract: all three protocol surfaces are shape-distinct."""

    case = PROTOCOL_CASES[protocol]
    config = _configure(
        mock_llm_upstream.url,
        protocol=protocol,
        models=[
            {
                "id": "relay/model",
                "display_name": "Relay Model",
                "context_length": 42_000,
                "pricing": {"input": "0.01", "output": "0.02"},
            }
        ],
    )
    assert config["models_endpoint"] == "ok"

    status, _, inventory = _json_request(
        mock_llm_upstream.url, "/v1/models"
    )
    assert status == 200
    assert inventory["object"] == "list"
    assert inventory["data"][0]["id"] == "relay/model"
    assert inventory["data"][0]["context_length"] == 42_000
    assert inventory["data"][0]["pricing"]["input"] == "0.01"
    if protocol == "anthropic":
        assert inventory["data"][0]["display_name"] == "Relay Model"
        assert inventory["data"][0]["type"] == "model"
    else:
        assert "display_name" not in inventory["data"][0]
        assert inventory["data"][0]["object"] == "model"

    status, _, payload = _json_request(
        mock_llm_upstream.url,
        case["path"],
        method="POST",
        body={**case["body"], "model": "relay/model"},
    )
    assert status == 200
    assert payload[case["response_key"]] == case["response_value"]
    assert payload["model"] == "relay/model"
    assert payload["usage"]


@pytest.mark.parametrize("protocol", PROTOCOL_CASES)
@pytest.mark.parametrize("surface", ["models", "buffered", "probe"])
@pytest.mark.parametrize("credential", [None, "sk-wrong-fixture", "sk-required-fixture"])
def test_mock_upstream_required_api_key_gates_data_surfaces(
    mock_llm_upstream, protocol: str, surface: str, credential: str | None,
) -> None:
    """An authenticated fixture requires the value, not just a header."""
    case = PROTOCOL_CASES[protocol]
    configured = _configure(
        mock_llm_upstream.url, protocol=protocol,
        required_api_key="sk-required-fixture",
    )
    assert configured["required_api_key"] == "sk-required-fixture"
    headers = {}
    if credential is not None:
        headers = (
            {"X-Api-Key": credential}
            if protocol == "anthropic"
            else {"Authorization": f"Bearer {credential}"}
        )
    status, _, payload = _json_request(
        mock_llm_upstream.url,
        "/v1/models" if surface == "models" else case["path"],
        method="GET" if surface == "models" else "POST",
        body=None if surface == "models" else ({} if surface == "probe" else case["body"]),
        headers=headers,
    )
    if credential != "sk-required-fixture":
        assert status == 401
        assert payload["error"]["type"] == "authentication_error"
    elif surface == "probe":
        assert status == 400  # Authenticated schema rejection is not inference.
        assert payload["error"]["type"] == "invalid_request_error"
    else:
        assert status == 200
        if surface == "models":
            assert payload["data"][0]["id"] == "mock-model"
        else:
            assert payload[case["response_key"]] == case["response_value"]


@pytest.mark.parametrize("protocol", PROTOCOL_CASES)
def test_mock_upstream_required_key_uses_the_configured_protocol_header(
    mock_llm_upstream, protocol: str,
) -> None:
    """The other family's header cannot satisfy this interface's credential."""
    _configure(
        mock_llm_upstream.url, protocol=protocol,
        required_api_key="sk-required-fixture",
    )
    headers = (
        {"Authorization": "Bearer sk-required-fixture"}
        if protocol == "anthropic"
        else {"X-Api-Key": "sk-required-fixture"}
    )
    status, _, _ = _json_request(
        mock_llm_upstream.url, "/v1/models", headers=headers,
    )
    assert status == 401
    # Control stays reachable and clearing the opt-in restores a public catalogue.
    assert _configure(mock_llm_upstream.url, required_api_key=None)["required_api_key"] is None
    assert _json_request(mock_llm_upstream.url, "/v1/models")[0] == 200


@pytest.mark.parametrize("scheme", ["bearer", "either"])
@pytest.mark.parametrize("surface", ["models", "probe", "buffered"])
@pytest.mark.parametrize("header", ["missing", "wrong", "x-api-key", "bearer"])
def test_mock_upstream_bearer_opt_in_preserves_authentication_shape(
    mock_llm_upstream, scheme, surface, header,
) -> None:
    """A custom Anthropic relay can require Bearer, or take the key in either auth header."""
    key = "fixture-static-bearer"
    _configure(
        mock_llm_upstream.url, protocol="anthropic",
        required_api_key=key, required_auth_scheme=scheme,
    )
    headers = {
        "missing": {},
        "wrong": {"Authorization": "Bearer fixture-wrong"},
        "x-api-key": {"X-Api-Key": key},
        "bearer": {"Authorization": f"Bearer {key}"},
    }[header]
    status, _, _ = _json_request(
        mock_llm_upstream.url,
        "/v1/models" if surface == "models" else "/v1/messages",
        method="GET" if surface == "models" else "POST",
        body=None if surface == "models" else (
            {} if surface == "probe" else PROTOCOL_CASES["anthropic"]["body"]
        ),
        headers=headers,
    )
    accepted = header == "bearer" or (scheme == "either" and header == "x-api-key")
    assert status == (401 if not accepted else 400 if surface == "probe" else 200)


@pytest.mark.parametrize("protocol", PROTOCOL_CASES)
@pytest.mark.parametrize(
    ("behavior", "status"),
    [({"auth": "429"}, 429), ({"models_endpoint": "http_500"}, 500)],
)
def test_mock_upstream_required_key_precedes_configured_failures(
    mock_llm_upstream, protocol: str, behavior: dict[str, str], status: int,
) -> None:
    """A valid credential reaches the existing failure modes unchanged."""
    _configure(
        mock_llm_upstream.url, protocol=protocol,
        required_api_key="sk-required-fixture", **behavior,
    )
    headers = (
        {"X-Api-Key": "sk-required-fixture"}
        if protocol == "anthropic"
        else {"Authorization": "Bearer sk-required-fixture"}
    )
    assert _json_request(mock_llm_upstream.url, "/v1/models")[0] == 401
    assert _json_request(mock_llm_upstream.url, "/v1/models", headers=headers)[0] == status


@pytest.mark.parametrize("protocol", PROTOCOL_CASES)
def test_mock_upstream_invalid_probe_evidence_is_family_distinctive(
    mock_llm_upstream, protocol: str
) -> None:
    """Mock contract: invalid probes expose protocol-specific evidence."""

    case = PROTOCOL_CASES[protocol]
    _configure(mock_llm_upstream.url, protocol=protocol)
    status, _, payload = _json_request(
        mock_llm_upstream.url,
        case["path"],
        method="POST",
        body={},
    )
    assert status == 400
    error = payload["error"]
    if protocol == "anthropic":
        assert payload["type"] == "error"
        assert "param" not in error
    else:
        assert error["param"] == case["invalid_param"]


@pytest.mark.parametrize("protocol", PROTOCOL_CASES)
@pytest.mark.parametrize(
    ("stream_behavior", "terminal_present"),
    [("healthy", True), ("interrupt_after_first_output", False)],
)
def test_mock_upstream_sse_healthy_and_interrupted_streams(
    mock_llm_upstream,
    protocol: str,
    stream_behavior: str,
    terminal_present: bool,
) -> None:
    """Mock contract: interruption follows the first model output."""

    case = PROTOCOL_CASES[protocol]
    _configure(
        mock_llm_upstream.url,
        protocol=protocol,
        stream=stream_behavior,
    )
    status, headers, raw = _request(
        mock_llm_upstream.url,
        case["path"],
        method="POST",
        body={**case["body"], "stream": True},
    )
    text = raw.decode("utf-8")
    assert status == 200
    assert headers["Content-Type"].startswith("text/event-stream")
    assert "mock response" in text
    assert (case["terminal"] in text) is terminal_present


@pytest.mark.parametrize(
    ("auth", "status"),
    [
        ("401", 401),
        ("403_banned", 403),
        ("402", 402),
        ("429", 429),
        ("quota_message", 429),
        ("5xx", 503),
    ],
)
def test_mock_upstream_auth_modes_precede_every_protocol_surface(
    mock_llm_upstream, auth: str, status: int
) -> None:
    """Mock contract: auth is evaluated before discovery or inference."""

    _configure(
        mock_llm_upstream.url,
        auth=auth,
        protocol="openai_chat",
        models_endpoint="http_500",
    )
    models_status, _, models_payload = _json_request(
        mock_llm_upstream.url, "/v1/models"
    )
    inference_status, _, inference_payload = _json_request(
        mock_llm_upstream.url,
        "/v1/responses",
        method="POST",
        body={},
    )
    assert models_status == status
    assert inference_status == status
    assert models_payload["error"]["type"] != "not_found_error"
    assert inference_payload["error"]["type"] != "not_found_error"


@pytest.mark.parametrize(
    ("behavior", "status"),
    [("http_404", 404), ("http_500", 500)],
)
def test_mock_upstream_models_endpoint_http_failures_are_isolated(
    mock_llm_upstream, behavior: str, status: int
) -> None:
    """Mock contract amendment: discovery can fail after protocol proof."""

    case = PROTOCOL_CASES["openai_chat"]
    _configure(
        mock_llm_upstream.url,
        auth="ok",
        protocol="openai_chat",
        models_endpoint=behavior,
    )
    models_status, _, _ = _json_request(
        mock_llm_upstream.url, "/v1/models"
    )
    inference_status, _, inference = _json_request(
        mock_llm_upstream.url,
        case["path"],
        method="POST",
        body=case["body"],
    )
    assert models_status == status
    assert inference_status == 200
    assert inference["object"] == "chat.completion"


def test_mock_upstream_models_endpoint_malformed_json_is_isolated(
    mock_llm_upstream,
) -> None:
    """Mock contract amendment: malformed discovery is independently set."""

    _configure(mock_llm_upstream.url, models_endpoint="malformed_json")
    status, headers, raw = _request(
        mock_llm_upstream.url, "/v1/models"
    )
    assert status == 200
    assert headers["Content-Type"].startswith("application/json")
    with pytest.raises(json.JSONDecodeError):
        json.loads(raw)

    status, _, payload = _json_request(
        mock_llm_upstream.url,
        "/v1/chat/completions",
        method="POST",
        body=PROTOCOL_CASES["openai_chat"]["body"],
    )
    assert status == 200
    assert payload["object"] == "chat.completion"


def test_mock_upstream_models_endpoint_timeout_is_isolated(
    mock_llm_upstream,
) -> None:
    """Mock contract amendment: discovery can time out independently."""

    _configure(mock_llm_upstream.url, models_endpoint="timeout")
    with pytest.raises((TimeoutError, URLError)):
        _request(
            mock_llm_upstream.url,
            "/v1/models",
            timeout=0.05,
        )
    status, _, payload = _json_request(
        mock_llm_upstream.url,
        "/v1/chat/completions",
        method="POST",
        body=PROTOCOL_CASES["openai_chat"]["body"],
    )
    assert status == 200
    assert payload["object"] == "chat.completion"


def test_mock_upstream_timeout_teardown_joins_handlers_and_closes_socket(
) -> None:
    """Mock lifecycle: timeout handlers and accepted sockets end at stop."""

    upstream = MockLLMUpstream().start()
    parsed = urlsplit(upstream.url)
    assert parsed.hostname is not None and parsed.port is not None
    address = (parsed.hostname, parsed.port)
    handler_threads = ()
    try:
        _configure(upstream.url, models_endpoint="timeout")
        with pytest.raises((TimeoutError, URLError)):
            _request(upstream.url, "/v1/models", timeout=0.05)

        deadline = time.monotonic() + 2
        while time.monotonic() < deadline:
            handler_threads = upstream.active_handler_threads()
            if handler_threads:
                break
            time.sleep(0.01)
        assert handler_threads
    finally:
        upstream.stop()

    assert all(not thread.is_alive() for thread in handler_threads)
    assert upstream.active_handler_threads() == ()
    with pytest.raises(OSError):
        socket.create_connection(address, timeout=0.2)


def test_mock_upstream_empty_inventory_is_success_not_discovery_failure(
    mock_llm_upstream,
) -> None:
    """Mock contract amendment: an empty successful inventory stays 200."""

    _configure(
        mock_llm_upstream.url,
        auth="ok",
        models_endpoint="ok",
        models=[],
    )
    status, _, payload = _json_request(
        mock_llm_upstream.url, "/v1/models"
    )
    assert status == 200
    assert payload["data"] == []


def test_mock_upstream_control_validation_capture_envelope_and_reset(
    mock_llm_upstream,
) -> None:
    """Mock contract: control config is strict and capture is enveloped."""

    for invalid in (
        {"unknown": True},
        {"auth": "invalid"},
        {"stream": "invalid"},
        {"protocol": "invalid"},
        {"models_endpoint": "invalid"},
        {"required_api_key": ""},
        {"required_api_key": "  "},
        {"required_api_key": 123},
        {"required_api_key": False},
        {"required_api_key": []},
        {"required_auth_scheme": "anything"},
        {"required_auth_scheme": None},
        {"required_auth_scheme": []},
        {"models": [""]},
        {"model_errors": []},
        {"model_errors": {"": "model_not_found"}},
        {"model_errors": {"unknown-model": "server_error"}},
    ):
        status, _, payload = _json_request(
            mock_llm_upstream.url,
            "/__control/config",
            method="POST",
            body=invalid,
        )
        assert status == 400
        assert payload["ok"] is False

    headers = {
        "Authorization": "Bearer test-secret",
        "User-Agent": "model-hub-e2e",
        "X-Uncaptured": "ignored",
    }
    status, _, _ = _json_request(
        mock_llm_upstream.url,
        "/v1/chat/completions",
        method="POST",
        body=PROTOCOL_CASES["openai_chat"]["body"],
        headers=headers,
    )
    assert status == 200
    status, _, captured = _json_request(
        mock_llm_upstream.url, "/__control/requests"
    )
    assert status == 200
    assert set(captured) == {"requests"}
    assert len(captured["requests"]) == 1
    record = captured["requests"][0]
    assert record == {
        "method": "POST",
        "path": "/v1/chat/completions",
        "headers": {
            "authorization": "Bearer test-secret",
            "content-type": "application/json",
            "user-agent": "model-hub-e2e",
        },
        "body": PROTOCOL_CASES["openai_chat"]["body"],
    }

    status, _, payload = _json_request(
        mock_llm_upstream.url,
        "/__control/requests",
        method="DELETE",
    )
    assert status == 200
    assert payload == {"ok": True}
    _, _, captured = _json_request(
        mock_llm_upstream.url, "/__control/requests"
    )
    assert captured == {"requests": []}


@pytest.mark.parametrize("protocol", PROTOCOL_CASES)
@pytest.mark.parametrize("inventory", [[], [{"id": "catalog-only-model"}]])
def test_mh_routing_001_mock_model_errors_are_request_scoped_and_inventory_independent(
    mock_llm_upstream, protocol, inventory,
) -> None:
    """MH-ROUTING-001: truthful inventory never gates capture or request-only errors."""

    missing = "vendor/unknown-model-20260906"
    healthy = "vendor/other-unlisted-model"
    case = PROTOCOL_CASES[protocol]
    original = _configure(
        mock_llm_upstream.url,
        protocol=protocol,
        models=inventory,
        model_errors={missing: "model_not_found"},
    )
    for model, expected_status in [(missing, 404), (healthy, 200)]:
        body = {**case["body"], "model": model}
        if protocol == "openai_responses":
            body["input"] = "Original identifier and request: 中文"
        else:
            body["messages"] = [{"role": "user", "content": "Original identifier and request: 中文"}]
        status, _, payload = _json_request(
            mock_llm_upstream.url, case["path"], method="POST", body=body,
        )
        assert status == expected_status, payload
        if model == missing:
            assert model in payload["error"]["message"]
            if protocol != "anthropic":
                assert payload["error"]["code"] == "model_not_found"
        assert mock_llm_upstream.requests()[-1]["body"] == body

    assert mock_llm_upstream.state.config() == original
    assert _configure(mock_llm_upstream.url, model_errors={})["models"] == inventory
    status, _, _ = _json_request(
        mock_llm_upstream.url,
        case["path"],
        method="POST",
        body={**case["body"], "model": missing},
    )
    assert status == 200


def test_mock_upstream_contract_client_ignores_inherited_proxy(
    monkeypatch,
    mock_llm_upstream,
) -> None:
    """Mock contract: loopback control and data calls never use host proxies."""

    for name in ("HTTP_PROXY", "HTTPS_PROXY", "http_proxy", "https_proxy"):
        monkeypatch.setenv(name, "http://127.0.0.1:1")
    for name in ("NO_PROXY", "no_proxy"):
        monkeypatch.delenv(name, raising=False)

    _configure(mock_llm_upstream.url, protocol="openai_chat")
    status, _, payload = _json_request(
        mock_llm_upstream.url,
        "/v1/chat/completions",
        method="POST",
        body=PROTOCOL_CASES["openai_chat"]["body"],
    )
    assert status == 200
    assert payload["object"] == "chat.completion"

    status, _, captured = _json_request(
        mock_llm_upstream.url, "/__control/requests"
    )
    assert status == 200
    assert [request["path"] for request in captured["requests"]] == [
        "/v1/chat/completions"
    ]


def test_mock_upstream_starts_standalone_with_bare_python() -> None:
    """Mock contract: the CLI imports no Avibe or pytest dependency."""

    driver = Path(__file__).parent / "drivers" / "mock_llm_upstream.py"
    process = subprocess.Popen(
        [sys.executable, str(driver), "--host", "127.0.0.1", "--port", "0"],
        cwd=driver.parents[3],
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        text=True,
    )
    try:
        assert process.stdout is not None
        first_line = process.stdout.readline().strip()
        match = re.search(r"(http://127\.0\.0\.1:\d+)$", first_line)
        assert match, first_line
        status, _, payload = _json_request(match.group(1), "/v1/models")
        assert status == 200
        assert payload["data"][0]["id"] == "mock-model"
    finally:
        process.terminate()
        try:
            process.wait(timeout=5)
        except subprocess.TimeoutExpired:
            process.kill()
            process.wait(timeout=5)