Skip to content

AI-powered virtual executive team — a single coherent executive persona backed by 8 specialist agents (FastAPI + Next.js).

19.5 KB
Raw Blame
"""Provider abstraction smoke tests.

Commit 1 wiring: every call site that used to call ``client.messages.create``
or ``client.messages.stream`` now resolves the backend via
``providers.get_provider(model)``. The registry currently always returns
the Anthropic provider; OpenRouter routing will be added in the next commit.

These tests pin the contract:

1. ``get_provider`` returns the same singleton across calls (cheap to call
   inside per-request paths).
2. The returned object satisfies the ``LLMProvider`` Protocol — it exposes
   ``messages_create`` and ``messages_stream``.
3. ``messages_create`` proxies kwargs through to the underlying SDK with
   no rewriting.
4. ``messages_stream`` returns the SDK's async context manager unchanged
   (so the streaming dispatch in ``executive.py`` keeps working).
"""
from __future__ import annotations

import asyncio
import os
from types import SimpleNamespace
from typing import Any
from unittest.mock import AsyncMock, MagicMock

os.environ.setdefault("ANTHROPIC_API_KEY", "sk-test-not-used")

import pytest  # noqa: E402

from openexecutive.providers import LLMProvider, get_provider  # noqa: E402
from openexecutive.providers import registry as registry_mod  # noqa: E402


@pytest.fixture(autouse=True)
def _reset_singleton() -> Any:
    registry_mod._reset_for_tests()
    yield
    registry_mod._reset_for_tests()


def test_get_provider_returns_same_singleton_across_claude_calls() -> None:
    a = get_provider("claude-sonnet-4-6")
    b = get_provider("claude-opus-4-7")
    # Different Claude model args resolve to the same Anthropic-direct
    # singleton — reconstructing the SDK client per request would burn
    # ~10 ms on every specialist call.
    assert a is b


def test_returned_provider_satisfies_protocol() -> None:
    provider = get_provider("claude-sonnet-4-6")
    assert isinstance(provider, LLMProvider)
    assert hasattr(provider, "messages_create")
    assert hasattr(provider, "messages_stream")


def test_messages_create_proxies_kwargs_unchanged(monkeypatch: pytest.MonkeyPatch) -> None:
    """The Anthropic-shape request body must reach the SDK verbatim — adding
    or rewriting fields here would silently break prompt caching."""
    captured: dict[str, Any] = {}
    fake_message = SimpleNamespace(content=[SimpleNamespace(type="text", text="ok")])

    async def _fake_create(**kwargs: Any) -> Any:
        captured.update(kwargs)
        return fake_message

    monkeypatch.setattr(
        "openexecutive.providers.anthropic_provider.anthropic.AsyncAnthropic",
        lambda **_kw: SimpleNamespace(messages=SimpleNamespace(create=_fake_create)),
    )
    registry_mod._reset_for_tests()

    provider = get_provider("claude-sonnet-4-6")
    payload = {
        "model": "claude-sonnet-4-6",
        "max_tokens": 1024,
        "system": [
            {
                "type": "text",
                "text": "S",
                "cache_control": {"type": "ephemeral"},
            }
        ],
        "messages": [{"role": "user", "content": "hi"}],
    }
    asyncio.run(provider.messages_create(**payload))
    assert captured == payload


def test_messages_stream_returns_sdk_context_manager_unchanged(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    sentinel = MagicMock(name="anthropic-stream-cm")

    monkeypatch.setattr(
        "openexecutive.providers.anthropic_provider.anthropic.AsyncAnthropic",
        lambda **_kw: SimpleNamespace(
            messages=SimpleNamespace(stream=lambda **_kw: sentinel)
        ),
    )
    registry_mod._reset_for_tests()

    provider = get_provider("claude-sonnet-4-6")
    # Note: messages_stream is sync; the SDK returns the CM and the caller
    # uses it in ``async with``. The provider must not wrap it in another
    # layer that would change ``stream.get_final_message`` semantics.
    assert provider.messages_stream(model="claude-sonnet-4-6") is sentinel


def test_messages_create_accepts_per_request_timeout(monkeypatch: pytest.MonkeyPatch) -> None:
    """Per-request timeout keyword is what carries through with the singleton
    pattern; the previous per-client timeout setup is gone."""
    captured: dict[str, Any] = {}

    async def _fake_create(**kwargs: Any) -> Any:
        captured.update(kwargs)
        return SimpleNamespace(content=[])

    monkeypatch.setattr(
        "openexecutive.providers.anthropic_provider.anthropic.AsyncAnthropic",
        lambda **_kw: SimpleNamespace(messages=SimpleNamespace(create=_fake_create)),
    )
    registry_mod._reset_for_tests()

    provider = get_provider("claude-sonnet-4-6")
    asyncio.run(
        provider.messages_create(
            model="claude-sonnet-4-6", max_tokens=10, messages=[], timeout=42.0
        )
    )
    assert captured["timeout"] == 42.0


def test_anthropic_provider_constructs_lazy_only_once(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    construction_count = 0

    def _ctor(**_kw: Any) -> Any:
        nonlocal construction_count
        construction_count += 1
        return SimpleNamespace(messages=SimpleNamespace(create=AsyncMock()))

    monkeypatch.setattr(
        "openexecutive.providers.anthropic_provider.anthropic.AsyncAnthropic", _ctor
    )
    registry_mod._reset_for_tests()
    for _ in range(5):
        get_provider("claude-sonnet-4-6")
    assert construction_count == 1


# --------------------------------------------------------------------------
# OpenRouter routing matrix
# --------------------------------------------------------------------------


def _settings_stub(
    *,
    enabled: bool = False,
    key: str | None = "sk-or-v1-xxx",
    anthropic_key: str | None = "sk-test",
    local_enabled: bool = False,
    local_base_url: str | None = None,
    local_models: list[str] | None = None,
    local_api_key: str | None = None,
    local_timeout_s: float = 300.0,
    local_include_usage_accounting: bool = False,
    local_reasoning_effort: str | None = None,
) -> Any:
    return SimpleNamespace(
        anthropic_api_key=anthropic_key,
        openrouter_enabled=enabled,
        openrouter_api_key=key,
        openrouter_base_url="https://openrouter.ai/api/v1",
        openrouter_app_title="Open Executive",
        openrouter_referer=None,
        openrouter_timeout_s=180.0,
        local_models_enabled=local_enabled,
        local_base_url=local_base_url,
        local_models=local_models or [],
        local_api_key=local_api_key,
        local_timeout_s=local_timeout_s,
        local_include_usage_accounting=local_include_usage_accounting,
        local_reasoning_effort=local_reasoning_effort,
    )


def test_claude_model_with_openrouter_off_routes_to_anthropic(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    monkeypatch.setattr(
        "openexecutive.providers.registry.get_settings",
        lambda: _settings_stub(enabled=False),
    )
    registry_mod._reset_for_tests()
    from openexecutive.providers.anthropic_provider import AnthropicProvider

    provider = get_provider("claude-sonnet-4-6")
    assert isinstance(provider, AnthropicProvider)


def test_claude_model_with_openrouter_on_routes_to_openrouter(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    monkeypatch.setattr(
        "openexecutive.providers.registry.get_settings",
        lambda: _settings_stub(enabled=True),
    )
    registry_mod._reset_for_tests()
    from openexecutive.providers.openrouter_provider import OpenRouterProvider

    provider = get_provider("claude-sonnet-4-6")
    assert isinstance(provider, OpenRouterProvider)


def test_non_claude_model_with_openrouter_on_routes_to_openrouter(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    monkeypatch.setattr(
        "openexecutive.providers.registry.get_settings",
        lambda: _settings_stub(enabled=True),
    )
    registry_mod._reset_for_tests()
    from openexecutive.providers.openrouter_provider import OpenRouterProvider

    provider = get_provider("openai/gpt-5")
    assert isinstance(provider, OpenRouterProvider)


def test_non_claude_model_with_openrouter_off_raises_400(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """No fallback path — non-Claude slugs require OpenRouter to be on."""
    from fastapi import HTTPException

    monkeypatch.setattr(
        "openexecutive.providers.registry.get_settings",
        lambda: _settings_stub(enabled=False),
    )
    registry_mod._reset_for_tests()
    with pytest.raises(HTTPException) as exc_info:
        get_provider("openai/gpt-5")
    assert exc_info.value.status_code == 400
    assert "OPENROUTER_ENABLED" in exc_info.value.detail


def test_anthropic_singleton_distinct_from_openrouter_singleton(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Different model families resolve to different singleton instances —
    not the same object — so misconfiguration can't silently send a
    non-Claude slug to the Anthropic SDK."""
    monkeypatch.setattr(
        "openexecutive.providers.registry.get_settings",
        lambda: _settings_stub(enabled=True),
    )
    registry_mod._reset_for_tests()
    claude_provider = get_provider("claude-sonnet-4-6")
    or_provider = get_provider("openai/gpt-5")
    # Both happen to be the OpenRouter provider when enabled=True, so they
    # ARE the same. But when toggled off, the Claude path returns Anthropic.
    # Verify the swap.
    monkeypatch.setattr(
        "openexecutive.providers.registry.get_settings",
        lambda: _settings_stub(enabled=False),
    )
    registry_mod._reset_for_tests()
    direct_claude = get_provider("claude-sonnet-4-6")
    assert direct_claude is not or_provider
    assert direct_claude is not claude_provider


def test_allowed_models_includes_openrouter_set_only_when_enabled(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from openexecutive.providers import allowed_models
    from openexecutive.providers.registry import (
        ANTHROPIC_DIRECT_MODELS,
        OPENROUTER_MODELS,
    )

    monkeypatch.setattr(
        "openexecutive.providers.registry.get_settings",
        lambda: _settings_stub(enabled=False),
    )
    off = allowed_models()
    assert off == ANTHROPIC_DIRECT_MODELS

    monkeypatch.setattr(
        "openexecutive.providers.registry.get_settings",
        lambda: _settings_stub(enabled=True),
    )
    on = allowed_models()
    assert on == [*ANTHROPIC_DIRECT_MODELS, *OPENROUTER_MODELS]
    # Both sets contain entries (catch the case where one list was emptied).
    assert len(on) > len(off)


# --------------------------------------------------------------------------
# Local / self-hosted model routing
# --------------------------------------------------------------------------


def _local_stub(monkeypatch: pytest.MonkeyPatch, **kwargs: Any) -> None:
    monkeypatch.setattr(
        "openexecutive.providers.registry.get_settings",
        lambda: _settings_stub(
            local_enabled=True,
            local_base_url="http://localhost:11434/v1",
            local_models=["llama3.3", "qwen2.5"],
            **kwargs,
        ),
    )
    registry_mod._reset_for_tests()


def test_local_model_routes_to_local_provider(monkeypatch: pytest.MonkeyPatch) -> None:
    from openexecutive.providers.openai_compatible import OpenAICompatibleProvider
    from openexecutive.providers.openrouter_provider import OpenRouterProvider

    _local_stub(monkeypatch, enabled=False)
    provider = get_provider("llama3.3")
    assert isinstance(provider, OpenAICompatibleProvider)
    # The local backend is the plain base class, never the OpenRouter subclass.
    assert not isinstance(provider, OpenRouterProvider)
    # Cached singleton — same object across calls.
    assert get_provider("qwen2.5") is provider


def test_local_provider_omits_usage_accounting_by_default(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    _local_stub(monkeypatch, enabled=False)
    provider = get_provider("llama3.3")
    assert provider._include_usage_accounting is False  # type: ignore[attr-defined]


def test_local_provider_respects_usage_accounting_opt_in(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """LOCAL_INCLUDE_USAGE_ACCOUNTING=true lets an operator who has confirmed
    their LOCAL_BASE_URL backend actually speaks OpenRouter's request format
    (e.g. a hosted, billed gateway) restore cost tracking."""
    _local_stub(monkeypatch, enabled=False, local_include_usage_accounting=True)
    provider = get_provider("llama3.3")
    assert provider._include_usage_accounting is True  # type: ignore[attr-defined]


def test_local_provider_reasoning_effort_unset_by_default(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    _local_stub(monkeypatch, enabled=False)
    provider = get_provider("llama3.3")
    assert provider._reasoning_effort is None  # type: ignore[attr-defined]


def test_local_provider_passes_configured_reasoning_effort(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """LOCAL_REASONING_EFFORT reaches the local provider."""
    _local_stub(monkeypatch, enabled=False, local_reasoning_effort="low")
    provider = get_provider("llama3.3")
    assert provider._reasoning_effort == "low"  # type: ignore[attr-defined]


def test_local_provider_distinct_from_openrouter_singleton(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """With OpenRouter also on, a local slug and an OpenRouter slug resolve to
    different provider instances — a local request can't leak to OpenRouter."""
    _local_stub(monkeypatch, enabled=True)
    local = get_provider("llama3.3")
    openrouter = get_provider("openai/gpt-5")
    assert local is not openrouter


def test_unlisted_model_does_not_route_local(monkeypatch: pytest.MonkeyPatch) -> None:
    """Local routing claims ONLY its configured slugs; an unlisted non-Claude
    slug with OpenRouter off is still a 400."""
    from fastapi import HTTPException

    _local_stub(monkeypatch, enabled=False)
    with pytest.raises(HTTPException) as exc_info:
        get_provider("some-unlisted-model")
    assert exc_info.value.status_code == 400


def test_allowed_models_includes_local_when_enabled(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from openexecutive.providers import allowed_models
    from openexecutive.providers.registry import ANTHROPIC_DIRECT_MODELS

    _local_stub(monkeypatch, enabled=False)
    assert allowed_models() == [*ANTHROPIC_DIRECT_MODELS, "llama3.3", "qwen2.5"]


def test_anthropic_free_deployment_hides_claude_and_serves_local(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """No Anthropic key + OpenRouter off + local on: the Council UI must not
    offer Claude (unreachable), only the local slugs — which route locally."""
    from openexecutive.providers import allowed_models
    from openexecutive.providers.openai_compatible import OpenAICompatibleProvider

    _local_stub(monkeypatch, enabled=False, anthropic_key=None)
    assert allowed_models() == ["llama3.3", "qwen2.5"]
    assert isinstance(get_provider("llama3.3"), OpenAICompatibleProvider)


def test_claude_without_key_or_openrouter_raises_actionable_400(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """An Anthropic-free deployment that left a model setting pointed at Claude
    gets a clear 400 — not a None-key SDK crash deep in a request."""
    from fastapi import HTTPException

    _local_stub(monkeypatch, enabled=False, anthropic_key=None)
    with pytest.raises(HTTPException) as exc_info:
        get_provider("claude-sonnet-4-6")
    assert exc_info.value.status_code == 400
    assert "ANTHROPIC_API_KEY" in exc_info.value.detail


# --------------------------------------------------------------------------
# model_options_for — Council Provider → Model picker grouping
# --------------------------------------------------------------------------


def test_model_options_match_allowlist_ids_and_order(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """The grouped view must never drift from the PATCH validator's list."""
    from openexecutive.providers import allowed_models_for, model_options_for
    from openexecutive.providers import openrouter_catalog as catalog

    monkeypatch.setattr(catalog, "_loaded_models", None)
    for stub in (
        _settings_stub(enabled=False),
        _settings_stub(enabled=True),
        _settings_stub(
            enabled=True,
            local_enabled=True,
            local_base_url="http://localhost:11434/v1",
            local_models=["qwen/qwen3-8b"],
        ),
    ):
        monkeypatch.setattr(
            "openexecutive.providers.registry.get_settings", lambda s=stub: s
        )
        ids = [o["id"] for o in model_options_for("cso")]
        assert ids == allowed_models_for("cso")


def test_model_options_group_claude_direct_under_anthropic(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from openexecutive.providers import model_options_for

    monkeypatch.setattr(
        "openexecutive.providers.registry.get_settings",
        lambda: _settings_stub(enabled=False),
    )
    by_id = {o["id"]: o for o in model_options_for(None)}
    assert by_id["claude-opus-5-5"] == {
        "id": "claude-opus-5-5",
        "provider": "anthropic",
        "provider_label": "Anthropic",
        "route": "direct",
        "label": "Claude Opus 5.5",
    }
    assert by_id["claude-fable-5-1"]["label"] == "Claude Fable 5.1"
    assert by_id["claude-haiku-4-5"]["label"] == "Claude Haiku 4.5"
    assert by_id["claude-sonnet-5"]["label"] == "Claude Sonnet 5"


def test_model_options_route_and_merge_with_openrouter_on(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """With OpenRouter on, Claude ids route via OpenRouter (mirrors
    get_provider), OpenRouter-only Claude slugs join the same Anthropic group,
    and a catalog duplicate of a direct id is not listed twice."""
    from openexecutive.providers import model_options_for
    from openexecutive.providers import openrouter_catalog as catalog

    monkeypatch.setattr(
        catalog,
        "_loaded_models",
        [
            "anthropic/claude-fable-5.1",  # duplicate of claude-fable-5-1
            "anthropic/claude-opus-4.8",  # OpenRouter-only generation
            "openai/gpt-6",
            "x-ai/grok-4.6",
            "newvendor/model-1",
        ],
    )
    monkeypatch.setattr(
        "openexecutive.providers.registry.get_settings",
        lambda: _settings_stub(enabled=True),
    )
    opts = model_options_for(None)
    ids = [o["id"] for o in opts]
    by_id = {o["id"]: o for o in opts}
    assert "anthropic/claude-fable-5.1" not in ids
    assert by_id["claude-fable-5-1"]["route"] == "openrouter"
    assert by_id["anthropic/claude-opus-4.8"] == {
        "id": "anthropic/claude-opus-4.8",
        "provider": "anthropic",
        "provider_label": "Anthropic",
        "route": "openrouter",
        "label": "Claude Opus 4.8",
    }
    assert by_id["openai/gpt-6"]["provider_label"] == "OpenAI"
    assert by_id["openai/gpt-6"]["label"] == "gpt-6"
    assert by_id["x-ai/grok-4.6"]["provider_label"] == "xAI"
    # Unknown vendor: grouped under its raw prefix rather than dropped.
    assert by_id["newvendor/model-1"]["provider"] == "newvendor"
    assert by_id["newvendor/model-1"]["provider_label"] == "newvendor"


def test_model_options_local_wins_over_vendor_shaped_slug(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from openexecutive.providers import model_options_for

    monkeypatch.setattr(
        "openexecutive.providers.registry.get_settings",
        lambda: _settings_stub(
            enabled=False,
            local_enabled=True,
            local_base_url="http://localhost:11434/v1",
            local_models=["qwen/qwen3-8b", "claude-proxy-1"],
        ),
    )
    by_id = {o["id"]: o for o in model_options_for(None)}
    for slug in ("qwen/qwen3-8b", "claude-proxy-1"):
        assert by_id[slug]["provider"] == "local"
        assert by_id[slug]["route"] == "local"
        assert by_id[slug]["label"] == slug