Skip to content

The agent that grows with you

7.5 KB
Raw Blame
"""Regression tests for OpenAI-direct api_mode recomputation during /model switch.

api.openai.com only accepts the Responses API (codex_responses) for its
reasoning models when tools + reasoning are in play — /v1/chat/completions
returns HTTP 400 ("Function tools with reasoning_effort are not supported").

When a session switches to a GPT-5.x model on api.openai.com while carrying a
stale ``chat_completions`` api_mode from the previous provider (e.g. an
openrouter default), the switch must override the stale value with the
host-mandated ``codex_responses``. Filling only when empty (the earlier fix)
was insufficient: the carried value was a *non-empty but wrong* mode.

This surfaced after the reasoning-unification refactor made the switched
model's reasoning effort actually apply, turning a silently-dropped
reasoning_effort into a live one that OpenAI 400s on the chat_completions path.
"""

from unittest.mock import patch

import pytest

from hermes_cli.model_switch import switch_model
from hermes_cli.providers import host_mandated_api_mode

_MOCK_VALIDATION = {
    "accepted": True,
    "persist": True,
    "recognized": True,
    "message": None,
}

# models.dev as the live catalog serves it: Meta's Model API is keyed ``meta``, the id the
# bundled ``meta-ai`` plugin also claims as an alias. Tests seed the registry instead of
# fetching it, so provider resolution never depends on the network or today's catalog.
_MODELS_DEV_WITH_META = {
    "meta": {"id": "meta", "name": "Meta", "env": ["META_MODEL_API_KEY"],
             "api": "https://api.meta.ai/v1", "models": {}},
}


def _run_openai_switch(
    raw_input: str,
    current_provider: str = "openrouter",
    current_model: str = "anthropic/claude-opus-4.8",
    explicit_provider: str = "openai-api",
    runtime_api_mode: str = "chat_completions",
    runtime_base_url: str = "https://api.openai.com/v1",
    models_dev_registry: dict | None = None,
):
    """Run switch_model with OpenAI-direct mocks and return the result."""
    registry = _MODELS_DEV_WITH_META if models_dev_registry is None else models_dev_registry
    with (
        patch("agent.models_dev.fetch_models_dev", lambda *a, **k: registry),
        patch("hermes_cli.model_switch.resolve_alias", return_value=None),
        patch("hermes_cli.model_switch.list_provider_models", return_value=[]),
        patch(
            "hermes_cli.runtime_provider.resolve_runtime_provider",
            return_value={
                "api_key": "sk-test",
                "base_url": runtime_base_url,
                "api_mode": runtime_api_mode,
            },
        ),
        patch(
            "hermes_cli.models_validate.validate_requested_model",
            return_value=_MOCK_VALIDATION,
        ),
        patch("hermes_cli.model_switch.get_model_info", return_value=None),
        patch("hermes_cli.model_switch.get_model_capabilities", return_value=None),
        patch("hermes_cli.models.detect_provider_for_model", return_value=None),
    ):
        return switch_model(
            raw_input=raw_input,
            current_provider=current_provider,
            current_model=current_model,
            explicit_provider=explicit_provider,
        )


def test_stale_chat_completions_overridden_on_openai_direct():
    """The incident: stale chat_completions on api.openai.com → codex_responses.

    Switching from an openrouter/chat_completions session to gpt-5.6-sol on
    api.openai.com must flip the wire protocol to the Responses API so tools +
    reasoning are preserved, instead of 400ing on chat/completions.
    """
    result = _run_openai_switch(
        raw_input="gpt-5.6-sol",
        current_provider="openrouter",
        current_model="anthropic/claude-opus-4.8",
        explicit_provider="openai-api",
        runtime_api_mode="chat_completions",  # stale value carried over
    )

    assert result.success, f"switch_model failed: {result.error_message}"
    assert result.target_provider == "openai-api"
    assert result.new_model == "gpt-5.6-sol"
    assert result.api_mode == "codex_responses"


def test_generic_endpoint_keeps_explicit_api_mode():
    """A generic (non-host-mandated) endpoint must NOT have its api_mode clobbered.

    Only hosts that mandate one wire protocol override a carried value; a
    generic OpenAI-compatible relay returns None from host_mandated_api_mode,
    so the switch path leaves the resolver's api_mode untouched.
    """
    assert host_mandated_api_mode("https://generic.example.com/v1") is None
    # Lookalike / path-spoof hosts must also NOT be treated as mandated (#32243).
    assert host_mandated_api_mode("https://api.openai.com.attacker.test/v1") is None
    assert host_mandated_api_mode("https://proxy.test/api.openai.com/v1") is None
    # The real endpoints are mandated.
    assert host_mandated_api_mode("https://api.openai.com/v1") == "codex_responses"
    assert host_mandated_api_mode("https://api.anthropic.com") == "anthropic_messages"

@pytest.mark.parametrize("models_dev_registry", [_MODELS_DEV_WITH_META, {}], ids=["catalog", "offline"])
def test_stale_chat_overridden_on_meta_direct(models_dev_registry):
    """Stale chat_completions on api.meta.ai → codex_responses (like openai direct).

    Switching from openrouter/chat_completions to muse-spark on meta's
    endpoint must flip to Responses API for caching (0% vs 93-99%). ``meta`` is an
    alias of the bundled ``meta-ai`` plugin, so the switch lands on ``meta-ai``
    whether or not models.dev (which keys the same API ``meta``) is reachable.
    """
    result = _run_openai_switch(
        models_dev_registry=models_dev_registry,
        raw_input="muse-spark-1.2",
        current_provider="openrouter",
        current_model="anthropic/claude-opus-4.8",
        explicit_provider="meta",
        runtime_api_mode="chat_completions",  # stale
        runtime_base_url="https://api.meta.ai/v1",
    )
    assert result.success, f"switch_model failed: {result.error_message}"
    assert result.target_provider == "meta-ai"
    assert result.new_model == "muse-spark-1.2"
    assert result.api_mode == "codex_responses"


def test_generic_relay_not_clobbered_on_meta_switch():
    """Generic relay does not pick up Meta mandate."""
    assert host_mandated_api_mode("https://generic.example.com/v1") is None
    result = _run_openai_switch(
        raw_input="some-model",
        current_provider="meta",
        current_model="muse-spark-1.2",
        explicit_provider="openrouter",
        runtime_api_mode="chat_completions",
        runtime_base_url="https://generic.example.com/v1",
    )
    # For generic endpoint, resolver api_mode is chat_completions and host_mandated is None,
    # so it stays chat_completions (not forced to codex_responses).
    assert result.success
    assert result.api_mode == "chat_completions"


def test_openai_runtime_codex_app_server_survives_host_mandate():
    """``model.openai_runtime: codex_app_server`` must survive the /model switch (#115169).

    The resolver applies the opt-in after its ladder and hands ``api_mode=codex_app_server``
    to the switch; api.openai.com's host-mandated ``codex_responses`` is a wire-protocol
    correction for stale modes and must not overwrite the app-server runtime selection.
    """
    result = _run_openai_switch(
        raw_input="gpt-5.6-sol",
        current_provider="openrouter",
        current_model="anthropic/claude-opus-4.8",
        explicit_provider="openai-api",
        runtime_api_mode="codex_app_server",
    )
    assert result.success, f"switch_model failed: {result.error_message}"
    assert result.target_provider == "openai-api"
    assert result.api_mode == "codex_app_server"