Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

22.9 KB
Raw Blame
"""Exercise projected metadata in the actual CLI, without a managed engine."""

from __future__ import annotations

import asyncio
import json
import os
import shlex
import shutil
import socket
import subprocess
from contextlib import nullcontext
from pathlib import Path

import pytest

from config.v2_config import ModelHubBackendModelConfig
from modules.agents.codex.transport import CodexTransport
from modules.agents.codex.transport import CodexRPCError
from modules.agents.model_hub import (
    ModelHubLaunch,
    build_claude_hub_env,
    build_codex_hub_launch,
    claude_settings_for_launch,
)
from tests.e2e.drivers.model_hub_app import ModelHubTestApp
from tests.e2e.drivers.mock_llm_upstream import MockLLMUpstream
from tests.e2e.test_model_hub_routing import _request_credential
from vibe.backend_model_catalog import (
    CodexHubCatalog,
    _codex_hub_catalog_bytes,
    _codex_hub_catalog_path,
    _publish_codex_hub_catalog,
)
from vibe.claude_config import CLAUDE_SETTING_SOURCES


pytestmark = pytest.mark.e2e_model_hub

# A complete one-pixel PNG, supplied directly to the turn rather than downloaded.
IMAGE_URL = (
    "data:image/png;base64,"
    "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="
)


@pytest.fixture
def rejected_external_proxy():
    # Reserve a loopback port without listening: external HTTP(S) attempts fail
    # at this test-owned transport boundary instead of reaching a real service.
    with socket.socket() as reserved:
        reserved.bind(("127.0.0.1", 0))
        yield f"http://127.0.0.1:{reserved.getsockname()[1]}"


def _isolated_runtime(tmp_path, rejected_external_proxy):
    runtime = ModelHubTestApp(Path(__file__).resolve().parents[2], tmp_path / "runtime")
    runtime.home.mkdir(parents=True)
    Path(runtime.env["TMPDIR"]).mkdir()
    for key in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "all_proxy"):
        runtime.env[key] = rejected_external_proxy
    return runtime


def _loopback_cli(binary, runtime):
    sandbox = shutil.which("sandbox-exec")
    if sandbox is None:
        return str(binary)
    # On macOS also enforce loopback-only egress below the CLI's HTTP stack,
    # including background clients that might not honor standard proxy settings.
    profile = '(version 1) (allow default) (deny network-outbound) (allow network-outbound (remote ip "localhost:*"))'
    wrapper = runtime.home / "loopback-cli"
    wrapper.write_text(
        f"#!/bin/sh\nexec {shlex.quote(sandbox)} -p {shlex.quote(profile)} {shlex.quote(str(binary))} \"$@\"\n",
    )
    wrapper.chmod(0o700)
    return str(wrapper)


@pytest.fixture
def codex_catalog_runtime(tmp_path, rejected_external_proxy):
    binary = shutil.which("codex")
    if binary is None:
        pytest.skip("Codex executable is unavailable")
    runtime = _isolated_runtime(tmp_path, rejected_external_proxy)
    binary = _loopback_cli(binary, runtime)
    Path(runtime.env["CODEX_HOME"]).mkdir()
    node = shutil.which("node")
    if node:
        runtime.env["PATH"] += os.pathsep + str(Path(node).parent)
    # This fixture uses the harness's allowlisted environment, never the live
    # credential stores. The native binary can only discover test-owned config.
    exported = subprocess.run(
        [binary, "debug", "models", "--bundled", "-c", "model_catalog_json=null"],
        cwd=runtime.home,
        env=runtime.env,
        capture_output=True,
        timeout=30,
        check=True,
    )
    return binary, runtime, exported.stdout


async def _codex_turn(binary, runtime, gateway, catalog, *, effort, native_config="", turns=1):
    codex_home = Path(runtime.env["CODEX_HOME"])
    config_path = codex_home / "config.toml"
    config_path.write_text('cli_auth_credentials_store = "file"\n' + native_config)
    catalog_path = runtime.home / "models.json"
    catalog_path.write_bytes(catalog)
    model_id = json.loads(catalog)["models"][0]["slug"]
    launch = ModelHubLaunch(
        backend="codex",
        channel="hub",
        requested_model=model_id,
        target_model="unrelated-upstream-alias",
        runtime_model=model_id,
        gateway_base_url=gateway.url,
        gateway_token="isolated-catalog-fixture",
    )
    args, env = build_codex_hub_launch([], runtime.env, launch, model_catalog_path=catalog_path)
    transport = CodexTransport(
        binary=binary, cwd=str(runtime.home), runtime_args=args, runtime_env=env,
    )
    notifications = []
    completed = asyncio.Event()

    async def notify(method, params):
        notifications.append((method, params))
        if method == "turn/completed":
            completed.set()

    transport.on_notification(notify)
    try:
        await transport.start()
        models = await transport.send_request("model/list", {})
        config = await transport.send_request("config/read", {})
        thread = await transport.send_request(
            "thread/start",
            {"model": model_id, "cwd": str(runtime.home), "approvalPolicy": "never"},
        )
        for _turn in range(turns):
            completed.clear()
            await transport.send_request(
                "turn/start",
                {
                    "threadId": thread["thread"]["id"],
                    "input": [
                        {"type": "text", "text": "Describe this image: 中文"},
                        {"type": "image", "url": IMAGE_URL},
                    ],
                    **({"effort": effort} if effort is not None else {}),
                },
            )
            await asyncio.wait_for(completed.wait(), timeout=30)
    finally:
        await transport.stop()
    assert config_path.read_text().startswith('cli_auth_credentials_store = "file"\n' + native_config)
    bodies = [request["body"] for request in gateway.requests() if request["path"] == "/v1/responses"]
    assert bodies, notifications
    assert [
        params["turn"]["status"] for method, params in notifications if method == "turn/completed"
    ] == ["completed"] * turns
    return bodies, models, notifications, config["config"]


def test_codex_reloads_catalog_after_initialize(codex_catalog_runtime):
    """MH-PROTOCOL-004: a live app-server rereads its catalog for a new thread."""
    binary, runtime, raw_catalog = codex_catalog_runtime
    catalog_path = runtime.home / "reload-catalog.json"
    catalog_path.write_bytes(_codex_hub_catalog_bytes(raw_catalog))

    async def probe():
        transport = CodexTransport(
            binary=binary,
            cwd=str(runtime.home),
            runtime_args=["-c", f"model_catalog_json={json.dumps(str(catalog_path))}"],
            runtime_env=runtime.env,
        )
        try:
            await transport.start()
            catalog_path.unlink()
            with pytest.raises(CodexRPCError, match="failed to load configuration"):
                await transport.send_request(
                    "thread/start",
                    {"cwd": str(runtime.home), "ephemeral": True},
                )
        finally:
            await transport.stop()

    asyncio.run(probe())


@pytest.mark.parametrize("inherit", [True, False])
def test_codex_inherits_catalog_pin(codex_catalog_runtime, monkeypatch, inherit):
    """MH-PROTOCOL-004: only a child pin, not the history spare, saves its catalog."""
    binary, runtime, raw_catalog = codex_catalog_runtime
    # Preserve native model behavior while producing distinct valid catalogs.
    # Choose the consumer's path after sorting, so it cannot win the history
    # spare by chance, regardless of which catalog this Codex version bundles.
    variants = []
    for index in range(5):
        payload = json.loads(raw_catalog)
        payload["models"][0]["display_name"] = f"Retention fixture {index}"
        variants.append(json.dumps(payload).encode())
    variants.sort(key=lambda raw: _codex_hub_catalog_path(_codex_hub_catalog_bytes(raw)).name)
    catalog = _publish_codex_hub_catalog(variants[-1])
    path = catalog.path
    if not inherit:
        monkeypatch.setattr(CodexHubCatalog, "inherited_subprocess_kwargs", lambda self: nullcontext({}))

    async def probe():
        transport = CodexTransport(
            binary=binary,
            cwd=str(runtime.home),
            runtime_args=["-c", f"model_catalog_json={json.dumps(str(path))}"],
            runtime_env=runtime.env,
            model_hub_catalog=catalog,
        )
        try:
            await transport.start()
            # Simulate loss of the parent's descriptor without killing pytest.
            catalog.close()
            transport._model_hub_catalog.close()
            for raw in variants[:-1]:
                with _publish_codex_hub_catalog(raw) as successor:
                    assert successor.path.name < path.name
            assert path.is_file() is inherit
            if inherit:
                thread = await transport.send_request(
                    "thread/start",
                    {"cwd": str(runtime.home), "ephemeral": True},
                )
                assert thread["thread"]["id"]
            else:
                with pytest.raises(CodexRPCError, match="failed to load configuration"):
                    await transport.send_request(
                        "thread/start",
                        {"cwd": str(runtime.home), "ephemeral": True},
                    )
        finally:
            catalog.close()
            await transport.stop()
        assert transport._process.returncode is not None
        # Simulated parent loss already closed every parent pin. The next
        # publication must reclaim the child's catalog once it really exits.
        with _publish_codex_hub_catalog(variants[0]):
            pass
        assert not path.exists()
        assert len(list(path.parent.glob("standard-responses-*.json"))) == 1

    asyncio.run(probe())


@pytest.mark.parametrize("effort", [None, "none", "minimal", "low", "medium", "high", "xhigh", "max"])
def test_codex_unknown_catalog_preserves_supplied_image_and_effort(codex_catalog_runtime, effort):
    """MH-PROTOCOL-004: absence must survive the real catalog/turn consumer."""
    binary, runtime, raw_catalog = codex_catalog_runtime
    model = ModelHubBackendModelConfig.from_payload({"id": "custom-unknown"})
    # Round-trip the released shape so empty lists are covered, not just missing
    # JSON keys. The model's alias deliberately differs from its route target.
    catalog = _codex_hub_catalog_bytes(raw_catalog, [model.to_payload()])
    with MockLLMUpstream() as gateway:
        gateway.configure(protocol="openai_responses")
        bodies, _models, _notifications, _config = asyncio.run(
            _codex_turn(binary, runtime, gateway, catalog, effort=effort),
        )
    body = bodies[0]
    assert body["model"] == model.id
    content = [part for item in body["input"] for part in item.get("content", [])]
    images = [part for part in content if part.get("type") == "input_image"]
    assert [part["image_url"] for part in images] == [IMAGE_URL]
    assert all(part.get("detail") != "original" for part in images)
    assert any("中文" in part.get("text", "") for part in content)
    assert (body.get("reasoning") or {}).get("effort") == effort
    assert not body.get("service_tier")


@pytest.mark.parametrize(
    "metadata,effort,expected_effort,image_expected",
    [
        ({"input_modalities": ["text"], "supports_reasoning": False}, None, "none", False),
        ({"input_modalities": ["text"], "supports_reasoning": False, "reasoning_efforts": ["high"]}, None, "none", False),
        ({"input_modalities": ["text", "image"], "reasoning_efforts": ["low", "high"]}, None, "low", True),
        ({"input_modalities": ["text", "image"], "reasoning_efforts": ["low", "high"]}, "high", "high", True),
        ({"supports_reasoning": True}, "high", "high", True),
    ],
)
def test_codex_explicit_catalog_choices_reach_the_consumer(
    codex_catalog_runtime, metadata, effort, expected_effort, image_expected,
):
    """MH-PROTOCOL-004: explicit negatives and effort lists remain meaningful."""
    binary, runtime, raw_catalog = codex_catalog_runtime
    model = ModelHubBackendModelConfig.from_payload({"id": "explicit-custom", **metadata})
    catalog = _codex_hub_catalog_bytes(raw_catalog, [model.to_payload()])
    with MockLLMUpstream() as gateway:
        gateway.configure(protocol="openai_responses")
        bodies, _models, _notifications, _config = asyncio.run(
            _codex_turn(binary, runtime, gateway, catalog, effort=effort),
        )
    content = [part for item in bodies[0]["input"] for part in item.get("content", [])]
    images = [part for part in content if part.get("type") == "input_image"]
    assert [part["image_url"] for part in images] == ([IMAGE_URL] if image_expected else [])
    assert bodies[0]["reasoning"]["effort"] == expected_effort


@pytest.mark.parametrize("explicit_context", [None, 64_000, 512_000])
@pytest.mark.parametrize("model_kind", ["custom", "native"])
def test_codex_native_context_override_wins_over_catalog_planning(codex_catalog_runtime, explicit_context, model_kind):
    """MH-PROTOCOL-004: the actual context consumer retains native overrides."""
    binary, runtime, raw_catalog = codex_catalog_runtime
    model_id = "context-alias" if model_kind == "custom" else json.loads(raw_catalog)["models"][0]["slug"]
    model = ModelHubBackendModelConfig(id=model_id, context_window=128_000, max_output_tokens=32_000)
    catalog = _codex_hub_catalog_bytes(raw_catalog, [model.to_payload()])
    native_config = 'model = "unrelated-native-default"\n'
    if explicit_context is not None:
        native_config += f"model_context_window = {explicit_context}\nmodel_auto_compact_token_limit = 300000\n"
    with MockLLMUpstream() as gateway:
        gateway.configure(protocol="openai_responses")
        bodies, _models, notifications, config = asyncio.run(
            _codex_turn(binary, runtime, gateway, catalog, effort=None, native_config=native_config),
        )
    usage = [params["tokenUsage"] for method, params in notifications if method == "thread/tokenUsage/updated"]
    percent = json.loads(catalog)["models"][0].get("effective_context_window_percent", 95)
    assert usage, [method for method, _params in notifications]
    if explicit_context is not None:
        assert config["model_context_window"] == explicit_context
        assert config["model_auto_compact_token_limit"] == 300_000
    assert usage[-1]["modelContextWindow"] == (explicit_context or model.context_window) * percent // 100
    assert bodies[0]["model"] == model.id
    # BackendModel output metadata has no Codex output-cap projection.
    assert "max_output_tokens" not in bodies[0]


@pytest.mark.parametrize(
    "native_config,input_tokens,compacts",
    [
        ("", 150_000, True),
        ("model_context_window = 512000\nmodel_auto_compact_token_limit = 300000\n", 150_000, False),
        ("model_context_window = 512000\nmodel_auto_compact_token_limit = 300000\n", 350_000, True),
    ],
)
def test_codex_auto_compaction_consumes_resolved_limits(
    codex_catalog_runtime, monkeypatch, native_config, input_tokens, compacts,
):
    """MH-PROTOCOL-004: compaction follows the active window and explicit threshold."""
    from tests.e2e.drivers import mock_llm_upstream

    binary, runtime, raw_catalog = codex_catalog_runtime
    model = ModelHubBackendModelConfig(id="compact-alias", context_window=128_000)
    catalog = _codex_hub_catalog_bytes(raw_catalog, [model.to_payload()])
    original_frames = mock_llm_upstream._responses_stream_frames
    calls = 0

    def frames_with_large_first_usage(model_id):
        nonlocal calls
        frames = original_frames(model_id)
        calls += 1
        if calls == 1:
            completed = json.loads(frames[-1].split(b"data: ", 1)[1])
            completed["response"]["usage"]["input_tokens"] = input_tokens
            completed["response"]["usage"]["total_tokens"] = input_tokens + 5
            frames[-1] = mock_llm_upstream._sse("response.completed", completed)
        return frames

    monkeypatch.setattr(mock_llm_upstream, "_responses_stream_frames", frames_with_large_first_usage)
    with MockLLMUpstream() as gateway:
        gateway.configure(protocol="openai_responses")
        bodies, _models, notifications, _config = asyncio.run(
            _codex_turn(binary, runtime, gateway, catalog, effort=None, native_config=native_config, turns=2),
        )
    compacted = [
        params for method, params in notifications
        if method == "item/completed" and params.get("item", {}).get("type") == "contextCompaction"
    ]
    assert bool(compacted) is compacts
    assert len(bodies) == (3 if compacts else 2)


@pytest.mark.parametrize("channel", ["hub", "native_cli"])
@pytest.mark.parametrize("settings_scope", ["parent", "project", "local", "catalog"])
def test_claude_consumer_preserves_explicit_limit_settings(tmp_path, rejected_external_proxy, channel, settings_scope):
    """MH-CLAUDE-LAUNCH-001: limits remain separate from transport ownership."""
    import claude_agent_sdk

    binary = Path(claude_agent_sdk.__file__).parent / "_bundled" / "claude"
    if not binary.is_file():
        pytest.skip("The installed Claude SDK has no bundled CLI")
    runtime = _isolated_runtime(tmp_path, rejected_external_proxy)
    binary = _loopback_cli(binary, runtime)
    project = runtime.home / "project"
    project.mkdir(parents=True)
    selected_limits = {
        "CLAUDE_CODE_MAX_CONTEXT_TOKENS": "234567",
        "CLAUDE_CODE_MAX_OUTPUT_TOKENS": "1234",
    }
    fixture_settings = None
    if settings_scope in {"project", "local"}:
        filename = "settings.local.json" if settings_scope == "local" else "settings.json"
        fixture_settings = project / ".claude" / filename
        fixture_settings.parent.mkdir()
        fixture_settings.write_text(json.dumps({"env": selected_limits}))
    elif settings_scope == "parent":
        runtime.env.update(selected_limits)
    runtime.env["CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC"] = "1"
    runtime.env["CLAUDE_CODE_ENTRYPOINT"] = "sdk-py"
    with MockLLMUpstream() as gateway:
        gateway.configure(protocol="anthropic")
        launch = ModelHubLaunch(
            backend="claude",
            channel=channel,
            requested_model="planning-alias",
            target_model="custom-limit-model",
            runtime_model="custom-limit-model",
            gateway_base_url=gateway.url,
            gateway_token="isolated-claude-fixture",
            context_window=128_000,
            max_output_tokens=4096,
        )
        # Native CLI authority is tested with synthetic local credentials, never
        # OAuth/keychain discovery. All filesystem paths belong to the fixture.
        runtime.env["ANTHROPIC_BASE_URL"] = gateway.url
        runtime.env["ANTHROPIC_AUTH_TOKEN"] = "isolated-native-fixture"
        env = build_claude_hub_env(runtime.env, launch)
        result = subprocess.run(
            [
                str(binary), "-p", "Reply with hello 中文.", "--output-format", "json",
                "--model", launch.runtime_model, "--tools", "", "--max-turns", "1",
                "--setting-sources", ",".join(CLAUDE_SETTING_SOURCES),
                "--settings", claude_settings_for_launch("{}", launch),
            ],
            cwd=project,
            env=env,
            capture_output=True,
            text=True,
            timeout=30,
        )
        assert result.returncode == 0, (result.stdout[-2000:], result.stderr[-2000:])
        bodies = [request["body"] for request in gateway.requests() if request["path"] == "/v1/messages"]
    assert bodies
    expected_output = 4096 if settings_scope == "catalog" else 1234
    assert bodies[-1]["model"] == launch.runtime_model
    assert bodies[-1]["max_tokens"] == expected_output
    # Normal Claude compaction currently ignores MAX_CONTEXT_TOKENS; do not
    # certify its modelUsage display as a consumer of this environment key.
    # SessionHandler coverage separately checks delivery of that planning value.
    if fixture_settings is not None:
        assert json.loads(fixture_settings.read_text()) == {"env": selected_limits}


def test_claude_hub_launch_applies_user_settings_but_not_their_connection(tmp_path, rejected_external_proxy):
    """MH-CLAUDE-LAUNCH-002: a Hub turn keeps the user's Claude preferences, not their transport."""
    import claude_agent_sdk

    binary = Path(claude_agent_sdk.__file__).parent / "_bundled" / "claude"
    if not binary.is_file():
        pytest.skip("The installed Claude SDK has no bundled CLI")
    runtime = _isolated_runtime(tmp_path, rejected_external_proxy)
    binary = _loopback_cli(binary, runtime)
    project = runtime.home / "project"
    project.mkdir(parents=True)
    runtime.env["CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC"] = "1"
    runtime.env["CLAUDE_CODE_ENTRYPOINT"] = "sdk-py"
    native_key = "native-user-settings-fixture"
    # Commit/PR attribution is the preference users see in their history. Custom
    # text makes its arrival observable in the prompt the CLI sends.
    attribution = {"commit": "User commit trailer 中文", "pr": "User PR footer 中文"}
    with MockLLMUpstream() as gateway, MockLLMUpstream() as native:
        gateway.configure(protocol="anthropic")
        native.configure(protocol="anthropic")
        user_settings = Path(runtime.env["CLAUDE_CONFIG_DIR"]) / "settings.json"
        user_settings.parent.mkdir()
        user_payload = json.dumps({
            "attribution": attribution,
            "env": {
                "ANTHROPIC_BASE_URL": native.url,
                "ANTHROPIC_AUTH_TOKEN": native_key,
                "ANTHROPIC_API_KEY": native_key,
            },
        }, ensure_ascii=False)
        user_settings.write_text(user_payload)
        launch = ModelHubLaunch(
            backend="claude",
            channel="hub",
            requested_model="hub-model",
            target_model="hub-model",
            runtime_model="hub-model",
            gateway_base_url=gateway.url,
            gateway_token="hub-user-settings-fixture",
        )
        result = subprocess.run(
            [
                str(binary), "-p", "Reply with hello.", "--output-format", "json",
                "--model", launch.runtime_model, "--max-turns", "1",
                "--setting-sources", ",".join(CLAUDE_SETTING_SOURCES),
                "--settings", claude_settings_for_launch("{}", launch),
            ],
            cwd=project,
            env=build_claude_hub_env(runtime.env, launch),
            capture_output=True,
            text=True,
            timeout=30,
        )
        assert result.returncode == 0, (result.stdout[-2000:], result.stderr[-2000:])
        native_calls = [request for request in native.requests() if request["path"] == "/v1/messages"]
        captured = [request for request in gateway.requests() if request["path"] == "/v1/messages"]
    assert native_calls == [], "Claude bypassed Model Hub using user settings"
    assert captured
    assert {_request_credential(request) for request in captured} == {launch.gateway_token}
    prompt = json.dumps([request["body"] for request in captured], ensure_ascii=False)
    assert attribution["commit"] in prompt
    assert attribution["pr"] in prompt
    assert user_settings.read_text() == user_payload