Skip to content

CubeLoop — a Pythonic, async-native agent harness core designed for high performance, readability, and production-grade persistence.

12.4 KB
Raw Blame
"""OpenAIProvider — capability descriptor wiring.

Each test constructs a provider with a known capability, then asserts on
the provider's internal state. Stream-time assertions come in Task 6.
"""

import pytest

from cubeloop.providers.base import (
    Model,
    ReasoningControl,
    StreamOptions,
    TextContent,
    UserMessage,
)
from cubeloop.providers.capability import (
    CapabilityDescriptor,
    ReasoningCapability,
    TemperatureSpec,
)
from cubeloop.providers.openai import OpenAIProvider


def test_provider_accepts_capability_kwarg():
    cap = CapabilityDescriptor(temperature=TemperatureSpec(mode="ignored"))
    p = OpenAIProvider(api_key="x", base_url="http://example", capability=cap)
    assert p._capability is cap


def test_provider_accepts_model_overrides():
    cap = CapabilityDescriptor()
    overrides = {
        "deepseek-r1": CapabilityDescriptor(
            reasoning=ReasoningCapability(effort_path="reasoning.effort")
        )
    }
    p = OpenAIProvider(
        api_key="x",
        base_url="http://example",
        capability=cap,
        model_capability_overrides=overrides,
    )
    assert p._model_overrides == overrides


def test_resolve_capability_uses_override_when_present():
    base = CapabilityDescriptor()
    override = CapabilityDescriptor(
        reasoning=ReasoningCapability(effort_path="reasoning.effort")
    )
    p = OpenAIProvider(
        api_key="x",
        base_url="http://example",
        capability=base,
        model_capability_overrides={"deepseek-r1": override},
    )
    assert p._resolve_capability("deepseek-r1") is override
    assert p._resolve_capability("llama-3") is base


def test_capability_default_when_kwarg_none():
    p = OpenAIProvider(api_key="x", base_url="http://example")
    # No capability passed -> official OpenAI Chat profile, but _cap_active=False.
    assert isinstance(p._capability, CapabilityDescriptor)
    assert p._capability.reasoning is not None
    assert p._cap_active is False


def test_cap_active_when_capability_passed():
    p = OpenAIProvider(
        api_key="x", base_url="http://example", capability=CapabilityDescriptor()
    )
    assert p._cap_active is True


def test_cap_active_when_only_overrides_passed():
    p = OpenAIProvider(
        api_key="x",
        base_url="http://example",
        model_capability_overrides={"m": CapabilityDescriptor()},
    )
    assert p._cap_active is True


def test_resolve_capability_returns_default_when_inactive():
    """When no capability kwarg was passed, _resolve_capability still returns
    a safe-default CapabilityDescriptor (not used in practice — gated by _cap_active —
    but the API should be predictable)."""
    p = OpenAIProvider(api_key="x", base_url="http://example")
    result = p._resolve_capability("any-model")
    assert isinstance(result, CapabilityDescriptor)
    assert result.reasoning is not None


async def _capture_payload_openai(
    provider: OpenAIProvider,
    model: Model,
    *,
    on_payload=None,
    reasoning: ReasoningControl | None = None,
) -> dict:
    """Run a stream with a fake openai client and return the kwargs sent.

    on_payload: optional caller-side payload mutator passed via StreamOptions.
    """
    captured: dict = {}

    async def listener(kwargs: dict, m: Model) -> None:
        captured.update(kwargs)

    provider._request_listeners.append(listener)

    class _FakeResponse:
        response = None

        def __aiter__(self):
            async def gen():
                return
                yield

            return gen()

    async def fake_create(**kw):
        return _FakeResponse()

    provider._client.chat.completions.create = fake_create  # type: ignore[method-assign]
    stream = await provider.stream(
        model=model,
        messages=[UserMessage(content=[TextContent(text="hi")])],
        options=StreamOptions(
            reasoning=reasoning or ReasoningControl(mode="off"),
            on_payload=on_payload,
        ),
    )
    async for _ in stream:
        pass
    return captured


def _model(id: str = "test-model", **kw) -> Model:
    return Model(
        id=id,
        provider_id="test",
        reasoning=kw.get("reasoning", False),
        context_window=kw.get("context_window", 32000),
        max_tokens=kw.get("max_tokens", 4096),
        temperature=kw.get("temperature", 0.7),
    )


@pytest.mark.asyncio
async def test_openai_chat_default_profile_writes_reasoning_effort() -> None:
    payload = await _capture_payload_openai(
        OpenAIProvider(api_key="x", base_url="http://example"),
        _model(reasoning=True),
        reasoning=ReasoningControl(mode="on", effort="high", summary="none"),
    )

    assert payload["reasoning_effort"] == "high"
    assert "thinking" not in payload


@pytest.mark.asyncio
async def test_openai_chat_off_writes_none_effort() -> None:
    payload = await _capture_payload_openai(
        OpenAIProvider(api_key="x", base_url="http://example"),
        _model(reasoning=True),
        reasoning=ReasoningControl(mode="off", effort="minimal", summary="none"),
    )

    assert payload["reasoning_effort"] == "none"


@pytest.mark.asyncio
async def test_openai_chat_reasoning_model_does_not_get_default_temperature() -> None:
    """Reasoning models on chat completions reject a non-default temperature;
    a legacy caller with no explicit capability must not have one injected."""
    payload = await _capture_payload_openai(
        OpenAIProvider(api_key="x", base_url="http://example"),
        _model(reasoning=True, temperature=0.7),
    )

    assert "temperature" not in payload


@pytest.mark.asyncio
async def test_temperature_ignored_strips_field():
    cap = CapabilityDescriptor(temperature=TemperatureSpec(mode="ignored"))
    p = OpenAIProvider(api_key="x", base_url="http://e", capability=cap)
    payload = await _capture_payload_openai(p, _model())
    assert "temperature" not in payload


@pytest.mark.asyncio
async def test_temperature_fixed_overwrites():
    cap = CapabilityDescriptor(
        temperature=TemperatureSpec(mode="fixed", fixed_value=0.0)
    )
    p = OpenAIProvider(api_key="x", base_url="http://e", capability=cap)
    payload = await _capture_payload_openai(p, _model(temperature=0.7))
    assert payload["temperature"] == 0.0


@pytest.mark.asyncio
async def test_max_tokens_field_renamed():
    cap = CapabilityDescriptor(max_tokens_field="max_completion_tokens")
    p = OpenAIProvider(api_key="x", base_url="http://e", capability=cap)
    payload = await _capture_payload_openai(p, _model())
    assert "max_completion_tokens" in payload
    assert "max_tokens" not in payload


@pytest.mark.asyncio
async def test_legacy_no_capability_does_not_inject_temperature_or_max_tokens():
    """Regression guard: no capability passed -> wire bytes identical to today."""
    p = OpenAIProvider(api_key="x", base_url="http://e")  # no capability
    payload = await _capture_payload_openai(p, _model())
    assert "temperature" not in payload
    assert "max_tokens" not in payload


@pytest.mark.asyncio
async def test_temperature_free_preserves_caller_value_via_on_payload():
    """setdefault must not overwrite a temperature the caller set via on_payload.
    Spec §3.5: capability-active path uses setdefault so caller wins."""
    cap = CapabilityDescriptor(
        temperature=TemperatureSpec(mode="free", min=0.0, max=2.0)
    )
    p = OpenAIProvider(api_key="x", base_url="http://e", capability=cap)

    async def set_caller_temp(kwargs, model):
        kwargs["temperature"] = 0.3
        return kwargs

    payload = await _capture_payload_openai(p, _model(), on_payload=set_caller_temp)
    assert payload["temperature"] == 0.3


@pytest.mark.asyncio
async def test_reasoning_mode_off_payload_merged_qwen():
    cap = CapabilityDescriptor(
        reasoning=ReasoningCapability(
            mode_payloads={
                "off": {"extra_body": {"enable_thinking": False}},
                "on": {"extra_body": {"enable_thinking": True}},
            }
        )
    )
    p = OpenAIProvider(api_key="x", base_url="http://e", capability=cap)
    payload = await _capture_payload_openai(p, _model(reasoning=True))
    assert payload["extra_body"]["enable_thinking"] is False


@pytest.mark.asyncio
async def test_reasoning_mode_on_payload_merged_qwen():
    cap = CapabilityDescriptor(
        reasoning=ReasoningCapability(
            mode_payloads={
                "off": {"extra_body": {"enable_thinking": False}},
                "on": {"extra_body": {"enable_thinking": True}},
            }
        )
    )
    p = OpenAIProvider(api_key="x", base_url="http://e", capability=cap)
    payload = await _capture_payload_openai(
        p,
        _model(reasoning=True),
        reasoning=ReasoningControl(mode="on"),
    )
    assert payload["extra_body"]["enable_thinking"] is True


@pytest.mark.asyncio
async def test_reasoning_effort_written():
    cap = CapabilityDescriptor(
        reasoning=ReasoningCapability(
            effort_path="reasoning_effort",
            effort_values={"medium": "medium"},
        )
    )
    p = OpenAIProvider(api_key="x", base_url="http://e", capability=cap)
    payload = await _capture_payload_openai(
        p,
        _model(reasoning=True),
        reasoning=ReasoningControl(mode="on", effort="medium"),
    )
    assert payload["reasoning_effort"] == "medium"


@pytest.mark.asyncio
async def test_model_override_wins_for_reasoning():
    base = CapabilityDescriptor()
    override = CapabilityDescriptor(
        reasoning=ReasoningCapability(
            mode_payloads={"on": {"reasoning": {"effort": "low"}}},
        )
    )
    p = OpenAIProvider(
        api_key="x",
        base_url="http://e",
        capability=base,
        model_capability_overrides={"deepseek-r1": override},
    )
    payload = await _capture_payload_openai(
        p,
        _model("deepseek-r1", reasoning=True),
        reasoning=ReasoningControl(mode="on"),
    )
    assert payload["reasoning"] == {"effort": "low"}


@pytest.mark.asyncio
async def test_legacy_no_capability_does_not_merge_reasoning_payload():
    """Regression: no capability -> no reasoning_off/on payload write."""
    p = OpenAIProvider(api_key="x", base_url="http://e")  # legacy
    payload = await _capture_payload_openai(p, _model())
    assert "extra_body" not in payload
    assert "reasoning_effort" not in payload


@pytest.mark.asyncio
async def test_reasoning_mode_payload_and_effort_both_applied():
    """Anthropic-style combined case: on_payload writes the thinking block,
    then effort_path writes budget_tokens into the same dict. Order
    matters — the merge must run before the level write so the dict exists."""
    cap = CapabilityDescriptor(
        reasoning=ReasoningCapability(
            mode_payloads={"on": {"thinking": {"type": "enabled"}}},
            effort_path="thinking.budget_tokens",
            effort_values={"medium": 8192},
        )
    )
    p = OpenAIProvider(api_key="x", base_url="http://e", capability=cap)
    payload = await _capture_payload_openai(
        p,
        _model(reasoning=True),
        reasoning=ReasoningControl(mode="on", effort="medium"),
    )
    assert payload["thinking"] == {"type": "enabled", "budget_tokens": 8192}


@pytest.mark.asyncio
async def test_max_tokens_field_renamed_preserves_on_payload_value():
    """When on_payload sets max_tokens explicitly, the value survives the rename.
    Mirrors test_temperature_free_preserves_caller_value_via_on_payload but for
    max_tokens / max_completion_tokens."""
    cap = CapabilityDescriptor(max_tokens_field="max_completion_tokens")
    p = OpenAIProvider(api_key="x", base_url="http://e", capability=cap)

    async def set_caller_max(kwargs, model):
        kwargs["max_tokens"] = 1234
        return kwargs

    payload = await _capture_payload_openai(p, _model(), on_payload=set_caller_max)
    assert payload["max_completion_tokens"] == 1234
    assert "max_tokens" not in payload


@pytest.mark.asyncio
async def test_max_completion_tokens_preserves_on_payload_value():
    """When on_payload sets max_completion_tokens and capability would rename
    max_tokens -> max_completion_tokens, the caller's value must survive."""
    cap = CapabilityDescriptor(max_tokens_field="max_completion_tokens")
    p = OpenAIProvider(api_key="x", base_url="http://e", capability=cap)

    async def set_caller(kwargs, model):
        kwargs["max_completion_tokens"] = 999
        return kwargs

    payload = await _capture_payload_openai(p, _model(), on_payload=set_caller)
    assert payload["max_completion_tokens"] == 999
    assert "max_tokens" not in payload