Skip to content

CubeLoop — a Pythonic, async-native agent harness core designed for high performance, readability, and production-grade persistence.

19.0 KB
Raw Blame
"""Tests for cubepi.errors — classifier heuristics and typed error taxonomy."""

from __future__ import annotations

from types import SimpleNamespace

import pytest

from cubeloop.errors import (
    ContentFiltered,
    ContextLengthExceeded,
    ModelNotFound,
    ProviderAuthFailed,
    ProviderBadRequest,
    ProviderError,
    ProviderUnavailable,
    RateLimited,
    _estimate_input_tokens,
    _retry_after_from,
    _status_of,
    classify_and_raise,
)
from cubeloop.providers.base import Model, TextContent, UserMessage


# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------


def _model(
    provider: str = "openai",
    model_id: str = "gpt-4o",
    context_window: int = 128_000,
) -> Model:
    return Model(id=model_id, provider_id=provider, context_window=context_window)


def _messages(text: str = "hello") -> list:
    return [UserMessage(content=[TextContent(text=text)])]


class _FakeExc(Exception):
    """Fake SDK exception with configurable status_code, message, and headers.

    ``status_code`` sets both the direct attr AND ``.response.status_code`` so
    existing classifier tests work. Pass ``response_status_code`` instead to
    put the code only on ``.response`` (for testing the fallback path).
    """

    def __init__(
        self,
        message: str = "",
        *,
        status_code: int | None = None,
        headers: dict | None = None,
        response_status_code: int | None = None,
    ) -> None:
        super().__init__(message)
        if status_code is not None:
            self.status_code = status_code
        response_code = (
            response_status_code if response_status_code is not None else status_code
        )
        if response_code is not None or headers is not None:
            self.response = SimpleNamespace(
                status_code=response_code,
                headers=headers or {},
            )


# ---------------------------------------------------------------------------
# Classifier tests
# ---------------------------------------------------------------------------


class TestClassifyContextLengthExceeded:
    def test_explicit_context_length_message(self) -> None:
        exc = _FakeExc("This model's maximum context length is 4096 tokens.")
        with pytest.raises(ContextLengthExceeded) as ei:
            classify_and_raise(exc, model=_model())
        assert ei.value.__cause__ is exc

    def test_exceed_pattern(self) -> None:
        exc = _FakeExc("context length exceeded: reduce your input")
        with pytest.raises(ContextLengthExceeded):
            classify_and_raise(exc, model=_model())

    def test_too_many_tokens_pattern(self) -> None:
        exc = _FakeExc("too many tokens in input")
        with pytest.raises(ContextLengthExceeded):
            classify_and_raise(exc, model=_model())

    def test_prompt_is_too_long_pattern(self) -> None:
        exc = _FakeExc("prompt is too long")
        with pytest.raises(ContextLengthExceeded):
            classify_and_raise(exc, model=_model())

    def test_reduce_messages_pattern(self) -> None:
        exc = _FakeExc("Please reduce the messages sent in your request.")
        with pytest.raises(ContextLengthExceeded):
            classify_and_raise(exc, model=_model())

    def test_carries_provider_and_model_fields(self) -> None:
        exc = _FakeExc("maximum context length exceeded", status_code=400)
        with pytest.raises(ContextLengthExceeded) as ei:
            classify_and_raise(exc, model=_model(provider="openai", model_id="gpt-4o"))
        err = ei.value
        assert err.provider == "openai"
        assert err.model == "gpt-4o"
        assert err.status_code == 400

    def test_volcano_invalid_parameter_400_with_oversize_tokens(self) -> None:
        """Volcano ARK sends opaque 400 InvalidParameter when over-context."""
        # Build a message whose chars/4 estimate is >= 95% of context_window (100k).
        # context_window=100_000; 95% threshold = 95_000 tokens = 380_000 chars.
        big_text = "x" * 400_000
        msgs = [UserMessage(content=[TextContent(text=big_text)])]
        exc = _FakeExc("InvalidParameter", status_code=400)
        with pytest.raises(ContextLengthExceeded) as ei:
            classify_and_raise(exc, model=_model(context_window=100_000), messages=msgs)
        assert ei.value.tokens_in is not None
        assert ei.value.context_window == 100_000

    def test_400_below_95_percent_threshold_not_context_length(self) -> None:
        """400 with small messages should NOT be classified as ContextLengthExceeded."""
        msgs = _messages("short prompt")
        exc = _FakeExc("InvalidParameter", status_code=400)
        with pytest.raises(ProviderBadRequest):
            classify_and_raise(exc, model=_model(context_window=100_000), messages=msgs)


class TestClassifyRateLimited:
    def test_429_raises_rate_limited(self) -> None:
        exc = _FakeExc("Too Many Requests", status_code=429)
        with pytest.raises(RateLimited) as ei:
            classify_and_raise(exc, model=_model())
        assert ei.value.status_code == 429

    def test_retry_after_parsed_from_headers(self) -> None:
        exc = _FakeExc(
            "rate limit exceeded",
            status_code=429,
            headers={"retry-after": "42"},
        )
        with pytest.raises(RateLimited) as ei:
            classify_and_raise(exc, model=_model())
        assert ei.value.retry_after == 42.0

    def test_quota_wording_triggers_rate_limited(self) -> None:
        # Pattern: "quota (?:exceed|exhaust|limit|reach)" — matches "quota exceeded"
        exc = _FakeExc("quota exceeded for your account", status_code=200)
        with pytest.raises(RateLimited):
            classify_and_raise(exc, model=_model())

    def test_403_with_quota_wording_raises_rate_limited(self) -> None:
        """Anthropic-style 403 with 'quota limit' text → RateLimited, not AuthFailed."""
        exc = _FakeExc(
            "Your account has reached its quota limit. Please upgrade.",
            status_code=403,
        )
        with pytest.raises(RateLimited):
            classify_and_raise(exc, model=_model())


class TestClassifyProviderAuthFailed:
    def test_401_raises_provider_auth_failed(self) -> None:
        exc = _FakeExc("Unauthorized", status_code=401)
        with pytest.raises(ProviderAuthFailed) as ei:
            classify_and_raise(exc, model=_model())
        assert ei.value.status_code == 401

    def test_403_without_quota_wording_raises_provider_auth_failed(self) -> None:
        exc = _FakeExc("Forbidden", status_code=403)
        with pytest.raises(ProviderAuthFailed):
            classify_and_raise(exc, model=_model())

    def test_carries_provider_context(self) -> None:
        exc = _FakeExc("Invalid API key", status_code=401)
        with pytest.raises(ProviderAuthFailed) as ei:
            classify_and_raise(
                exc, model=_model(provider="anthropic", model_id="claude-3-5-sonnet")
            )
        assert ei.value.provider == "anthropic"


class TestClassifyProviderUnavailable:
    def test_5xx_raises_provider_unavailable(self) -> None:
        exc = _FakeExc("Internal Server Error", status_code=500)
        with pytest.raises(ProviderUnavailable) as ei:
            classify_and_raise(exc, model=_model())
        assert ei.value.status_code == 500

    def test_503_raises_provider_unavailable(self) -> None:
        exc = _FakeExc("Service Unavailable", status_code=503)
        with pytest.raises(ProviderUnavailable):
            classify_and_raise(exc, model=_model())

    def test_timeout_error_raises_provider_unavailable(self) -> None:
        exc = TimeoutError("request timed out")
        with pytest.raises(ProviderUnavailable):
            classify_and_raise(exc, model=_model())

    def test_connection_error_raises_provider_unavailable(self) -> None:
        exc = ConnectionError("failed to connect")
        with pytest.raises(ProviderUnavailable):
            classify_and_raise(exc, model=_model())


class TestClassifyProviderBadRequest:
    def test_generic_400_raises_provider_bad_request(self) -> None:
        exc = _FakeExc("Bad Request", status_code=400)
        with pytest.raises(ProviderBadRequest) as ei:
            classify_and_raise(exc, model=_model())
        assert ei.value.status_code == 400

    def test_404_model_not_found(self) -> None:
        exc = _FakeExc("model not found", status_code=404)
        with pytest.raises(ModelNotFound) as ei:
            classify_and_raise(exc, model=_model())
        assert isinstance(ei.value, ProviderBadRequest)


class TestClassifyUnknown:
    def test_unknown_exception_re_raised_as_is(self) -> None:
        """A plain RuntimeError with no status code propagates unchanged."""
        exc = RuntimeError("boom")
        with pytest.raises(RuntimeError, match="boom") as ei:
            classify_and_raise(exc, model=_model())
        assert ei.value is exc

    def test_already_typed_error_re_raised_unchanged(self) -> None:
        """A ProviderError passed in is re-raised without double-wrapping."""
        original = ContextLengthExceeded(
            "already typed", provider="openai", model="gpt-4o"
        )
        with pytest.raises(ContextLengthExceeded) as ei:
            classify_and_raise(original, model=_model())
        assert ei.value is original


class TestProviderErrorInheritance:
    def test_all_subclasses_are_provider_error(self) -> None:
        for cls in (
            ContextLengthExceeded,
            RateLimited,
            ProviderAuthFailed,
            ProviderUnavailable,
            ProviderBadRequest,
            ModelNotFound,
            ContentFiltered,
        ):
            err = cls("msg", provider="p", model="m")
            assert isinstance(err, ProviderError)
            assert isinstance(err, Exception)

    def test_context_length_carries_token_fields(self) -> None:
        err = ContextLengthExceeded(
            "too long",
            provider="openai",
            model="gpt-4o",
            tokens_in=5000,
            context_window=4096,
        )
        assert err.tokens_in == 5000
        assert err.context_window == 4096

    def test_rate_limited_carries_retry_after(self) -> None:
        err = RateLimited(
            "slow down", provider="openai", model="gpt-4o", retry_after=30.0
        )
        assert err.retry_after == 30.0


# ---------------------------------------------------------------------------
# Coverage fillers — internal helpers exercised directly.
# ---------------------------------------------------------------------------


class TestStatusOf:
    def test_direct_status_code(self) -> None:
        exc = _FakeExc("msg", status_code=400)
        assert _status_of(exc) == 400

    def test_response_fallback(self) -> None:
        """exc has no .status_code but .response.status_code — fallback path."""
        exc = _FakeExc("msg", response_status_code=503)
        assert getattr(exc, "status_code", None) is None
        assert _status_of(exc) == 503

    def test_no_status(self) -> None:
        exc = RuntimeError("no status anywhere")
        assert _status_of(exc) is None


class TestEstimateInputTokens:
    def test_none_returns_none(self) -> None:
        assert _estimate_input_tokens(None) is None

    def test_empty_returns_none(self) -> None:
        assert _estimate_input_tokens([]) is None

    def test_user_message_with_text_content(self) -> None:
        msgs = _messages("hello world")
        est = _estimate_input_tokens(msgs)
        assert est >= 2

    def test_block_with_content_not_text_attr(self) -> None:
        """Block that has .content instead of .text — non-TextContent blocks."""
        block = SimpleNamespace(text=None, content="tool result text")
        msg = type("_Msg", (), {"content": [block]})()
        est = _estimate_input_tokens([msg])
        assert est is not None and est >= 3

    def test_string_content(self) -> None:
        """Message with list[str] content blocks is NOT expected but is handled."""

        # Actually test the elif isinstance(content, str) branch via a mock.
        class _StrMsg:
            content = "plain string content for token estimation"

        est = _estimate_input_tokens([_StrMsg])  # type: ignore[list-item]
        assert est is not None and est >= 7

    def test_zero_total_returns_none(self) -> None:
        """Empty strings produce zero total."""
        msgs = [UserMessage(content=[TextContent(text="")])]
        assert _estimate_input_tokens(msgs) is None


class TestRetryAfterFrom:
    def test_lowercase_retry_after_header(self) -> None:
        exc = _FakeExc("rate limited", status_code=429, headers={"retry-after": "60"})
        assert _retry_after_from(exc) == 60.0

    def test_title_case_retry_after_header(self) -> None:
        exc = _FakeExc("rate limited", status_code=429, headers={"Retry-After": "30"})
        assert _retry_after_from(exc) == 30.0

    def test_no_headers_returns_none(self) -> None:
        exc = _FakeExc("rate limited", status_code=429)
        assert _retry_after_from(exc) is None

    def test_no_retry_after_key_returns_none(self) -> None:
        exc = _FakeExc("rate limited", status_code=429, headers={"x-other": "1"})
        assert _retry_after_from(exc) is None

    def test_non_numeric_value_returns_none(self) -> None:
        exc = _FakeExc(
            "rate limited", status_code=429, headers={"retry-after": "not-a-number"}
        )
        assert _retry_after_from(exc) is None


class TestSDKConnectionErrors:
    """SDK-specific connection/timeout exceptions masquerading as ProviderUnavailable."""

    def test_sdk_connection_error_name_match(self) -> None:
        """openai.APIConnectionError has 'ConnectionError' in class name."""
        exc = type("APIConnectionError", (Exception,), {})("cannot connect")
        with pytest.raises(ProviderUnavailable):
            classify_and_raise(exc, model=_model())

    def test_sdk_timeout_name_match(self) -> None:
        """anthropic.APITimeoutError has 'Timeout' in class name."""
        exc = type("APITimeoutError", (Exception,), {})("timed out")
        with pytest.raises(ProviderUnavailable):
            classify_and_raise(exc, model=_model())


class TestErrorCodeExtraction:
    def test_openai_error_code(self) -> None:
        from cubeloop.errors import extract_error_code

        exc = _FakeExc("nope", status_code=400)
        exc.body = {
            "error": {"code": "model_not_found", "type": "invalid_request_error"}
        }
        assert extract_error_code(exc) == "model_not_found"

    def test_content_filtered_from_message(self) -> None:
        exc = _FakeExc("content policy violation", status_code=400)
        with pytest.raises(ContentFiltered):
            classify_and_raise(exc, model=_model())

    def test_503_does_not_exist_is_unavailable_not_model_not_found(self) -> None:
        exc = _FakeExc("replica does not exist", status_code=503)
        with pytest.raises(ProviderUnavailable):
            classify_and_raise(exc, model=_model())

    def test_generic_not_found_code_without_model_hint_is_bad_request(self) -> None:
        exc = _FakeExc("missing resource", status_code=404)
        exc.body = {"error": {"code": "not_found_error"}}
        with pytest.raises(ProviderBadRequest) as ei:
            classify_and_raise(exc, model=_model())
        assert not isinstance(ei.value, ModelNotFound)

    def test_nested_error_object_and_attr_error(self) -> None:
        from cubeloop.errors import extract_error_code

        nested = _FakeExc("x", status_code=400)
        nested.error = {"error": {"code": "model_not_available"}}
        assert extract_error_code(nested) == "model_not_available"

        class _Obj:
            code = None
            type = "content_filter"
            param = None

        attr = _FakeExc("filtered", status_code=400)
        attr.error = _Obj()
        assert extract_error_code(attr) == "content_filter"

    def test_type_fallback_and_param_model(self) -> None:
        from cubeloop.errors import extract_error_code

        typ = _FakeExc("x", status_code=400)
        typ.error = {"type": "invalid_model"}
        assert extract_error_code(typ) == "invalid_model"

        param = _FakeExc("x", status_code=400)
        param.error = {"type": "invalid_request_error", "param": "model"}
        assert extract_error_code(param) == "invalid_request_error"

        other = _FakeExc("x", status_code=400)
        other.error = {"type": "overloaded_error"}
        assert extract_error_code(other) == "overloaded_error"

    def test_looks_like_model_not_found_gates(self) -> None:
        from cubeloop.errors import _looks_like_model_not_found

        assert _looks_like_model_not_found(status=500, code="model_not_found", msg="x")
        assert _looks_like_model_not_found(
            status=400, code=None, msg="x", param="model"
        )
        assert _looks_like_model_not_found(
            status=404, code="not_found_error", msg="unknown model id"
        )
        assert _looks_like_model_not_found(
            status=400, code=None, msg="invalid model xyz"
        )

    def test_content_filtered_by_code(self) -> None:
        from cubeloop.errors import _looks_like_content_filtered

        assert _looks_like_content_filtered(code="content_filter", msg="nope")

    def test_classify_string_error_branches(self) -> None:
        from cubeloop.errors import classify_string_error

        assert isinstance(
            classify_string_error("maximum context length exceeded"),
            ContextLengthExceeded,
        )
        assert isinstance(
            classify_string_error("slow down", status_code=429), RateLimited
        )
        assert isinstance(
            classify_string_error("nope", status_code=401), ProviderAuthFailed
        )
        assert isinstance(
            classify_string_error("blocked by content_policy"), ContentFiltered
        )
        assert isinstance(
            classify_string_error("unknown model foo", status_code=404), ModelNotFound
        )
        assert isinstance(
            classify_string_error("boom", status_code=503), ProviderUnavailable
        )

    def test_error_from_stream_fields_context_length(self) -> None:
        from cubeloop.errors import error_from_stream_fields

        err = error_from_stream_fields(
            error_message="too long",
            error_type="ContextLengthExceeded",
            tokens_in=99,
            context_window=10,
        )
        assert isinstance(err, ContextLengthExceeded)
        assert err.tokens_in == 99

    def test_annotate_error_event_typed_and_raw(self) -> None:
        from cubeloop.errors import annotate_error_event

        rl = RateLimited("wait", retry_after=1.5, provider="p", model="m")
        fields = annotate_error_event(rl)
        assert fields["error_type"] == "RateLimited"
        assert fields["retry_after"] == 1.5

        cle = ContextLengthExceeded("long", tokens_in=10, context_window=8)
        fields = annotate_error_event(cle)
        assert fields["tokens_in"] == 10

        raw = RuntimeError("nope")
        fields = annotate_error_event(raw, fallback_message="x")
        assert fields["error_message"] == "x"
        assert fields["error_type"] is None