Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

30.8 KB
Raw Blame
"""Tests for OpenAI base provider: message prep, tool parsing, retry, cost tracking."""

import json
from typing import Any, Optional
from unittest.mock import AsyncMock, MagicMock

import pytest

from forge.llm.providers._openai_base import (
    BaseOpenAIChatProvider,
    format_function_def_for_openai,
)
from forge.llm.providers.schema import (
    AssistantChatMessage,
    AssistantFunctionCall,
    AssistantToolCall,
    ChatMessage,
    ChatModelInfo,
    CompletionModelFunction,
    ModelProviderBudget,
    ModelProviderConfiguration,
    ModelProviderCredentials,
    ModelProviderName,
    ModelProviderSettings,
    ModelTokenizer,
    ToolResultMessage,
)
from forge.llm.providers.utils import InvalidFunctionCallError
from forge.models.json_schema import JSONSchema


# ---------------------------------------------------------------------------
# Concrete test provider (since BaseOpenAIChatProvider is abstract-ish)
# ---------------------------------------------------------------------------
class _TestModelName(str):
    """Dummy model name type for testing."""

    pass


TEST_MODEL = _TestModelName("test-model")
TEST_MODEL_GPT5 = _TestModelName("gpt-5-test")
TEST_MODEL_O3 = _TestModelName("o3-mini-test")
TEST_MODEL_LEGACY = _TestModelName("gpt-3.5-turbo")

_MODEL_INFO = ChatModelInfo(
    name=TEST_MODEL,
    provider_name=ModelProviderName.OPENAI,
    prompt_token_cost=10.0 / 1e6,
    completion_token_cost=30.0 / 1e6,
    max_tokens=4096,
    has_function_call_api=True,
)

_GPT5_MODEL_INFO = ChatModelInfo(
    name=TEST_MODEL_GPT5,
    provider_name=ModelProviderName.OPENAI,
    prompt_token_cost=1.25 / 1e6,
    completion_token_cost=10.0 / 1e6,
    max_tokens=400_000,
    has_function_call_api=True,
    supports_reasoning_effort=True,
)


class _DummyCredentials(ModelProviderCredentials):
    api_key: Any = "test-key"

    def get_api_access_kwargs(self) -> dict:
        return {"api_key": "test-key"}


class _DummySettings(ModelProviderSettings):
    credentials: Optional[ModelProviderCredentials] = None  # type: ignore[assignment]
    budget: Optional[ModelProviderBudget] = None


class _TestProvider(BaseOpenAIChatProvider[_TestModelName, _DummySettings]):
    MODELS = {
        TEST_MODEL: _MODEL_INFO,
        TEST_MODEL_GPT5: _GPT5_MODEL_INFO,
        TEST_MODEL_O3: ChatModelInfo(
            name=TEST_MODEL_O3,
            provider_name=ModelProviderName.OPENAI,
            max_tokens=200_000,
            supports_reasoning_effort=True,
        ),
        TEST_MODEL_LEGACY: ChatModelInfo(
            name=TEST_MODEL_LEGACY,
            provider_name=ModelProviderName.OPENAI,
            max_tokens=4096,
        ),
    }
    CHAT_MODELS = MODELS  # type: ignore

    default_settings = _DummySettings(
        name="test_provider",
        description="Test",
        configuration=ModelProviderConfiguration(),
        credentials=_DummyCredentials(),
        budget=ModelProviderBudget(),
    )

    def __init__(self, **kwargs):
        # Skip actual OpenAI client initialization
        self._settings = self.default_settings.model_copy(deep=True)
        self._configuration = self._settings.configuration
        self._credentials = self._settings.credentials
        self._budget = self._settings.budget
        self._logger = MagicMock()
        self._client = MagicMock()

    def get_tokenizer(self, model_name) -> ModelTokenizer:
        tokenizer = MagicMock()
        tokenizer.encode = lambda text: text.split()
        return tokenizer


@pytest.fixture
def provider():
    return _TestProvider()


@pytest.fixture
def search_function():
    return CompletionModelFunction(
        name="web_search",
        description="Search the web",
        parameters={
            "query": JSONSchema(
                type=JSONSchema.Type.STRING,
                description="Search query",
                required=True,
            ),
        },
    )


@pytest.fixture
def write_function():
    return CompletionModelFunction(
        name="write_file",
        description="Write content to a file",
        parameters={
            "path": JSONSchema(
                type=JSONSchema.Type.STRING, description="File path", required=True
            ),
            "content": JSONSchema(
                type=JSONSchema.Type.STRING, description="Content", required=True
            ),
        },
    )


# ---------------------------------------------------------------------------
# format_function_def_for_openai
# ---------------------------------------------------------------------------
class TestFormatFunctionDefForOpenAI:
    def _params(self, result):
        """Extract parameters dict from FunctionDefinition."""
        return result.get("parameters") or {}

    def test_basic_function(self, search_function):
        result = format_function_def_for_openai(search_function)
        assert result["name"] == "web_search"
        assert result.get("description") == "Search the web"
        params = self._params(result)
        assert "properties" in params
        assert "query" in params["properties"]
        assert params["required"] == ["query"]

    def test_multiple_required_params(self, write_function):
        result = format_function_def_for_openai(write_function)
        params = self._params(result)
        assert set(params["required"]) == {"path", "content"}

    def test_optional_params_not_in_required(self):
        fn = CompletionModelFunction(
            name="test",
            description="Test",
            parameters={
                "required_param": JSONSchema(
                    type=JSONSchema.Type.STRING, required=True
                ),
                "optional_param": JSONSchema(
                    type=JSONSchema.Type.STRING, required=False
                ),
            },
        )
        result = format_function_def_for_openai(fn)
        assert self._params(result)["required"] == ["required_param"]

    def test_no_params(self):
        fn = CompletionModelFunction(name="noop", description="No-op", parameters={})
        result = format_function_def_for_openai(fn)
        params = self._params(result)
        assert params["properties"] == {}
        assert params["required"] == []

    def test_parameters_type_is_object(self, search_function):
        result = format_function_def_for_openai(search_function)
        assert self._params(result)["type"] == "object"


# ---------------------------------------------------------------------------
# _get_chat_completion_args: message preparation
# ---------------------------------------------------------------------------
class TestGetChatCompletionArgs:
    def test_basic_messages(self, provider):
        messages = [
            ChatMessage.system("You are helpful"),
            ChatMessage.user("Hello"),
        ]
        prepped, kwargs, parse_kwargs = provider._get_chat_completion_args(
            prompt_messages=messages, model=TEST_MODEL
        )
        assert len(prepped) == 2
        assert prepped[0]["role"] == "system"
        assert prepped[1]["role"] == "user"

    def test_tool_calls_arguments_serialized_as_json_strings(self, provider):
        tc = AssistantToolCall(
            id="call_1",
            type="function",
            function=AssistantFunctionCall(
                name="search", arguments={"query": "test", "limit": 10}
            ),
        )
        messages = [
            AssistantChatMessage(content="Searching", tool_calls=[tc]),
        ]
        prepped, _, _ = provider._get_chat_completion_args(
            prompt_messages=messages, model=TEST_MODEL
        )
        args = prepped[0]["tool_calls"][0]["function"]["arguments"]
        assert isinstance(args, str)
        assert json.loads(args) == {"query": "test", "limit": 10}

    def test_empty_tool_call_arguments_serialized(self, provider):
        tc = AssistantToolCall(
            id="call_1",
            type="function",
            function=AssistantFunctionCall(name="noop", arguments={}),
        )
        messages = [AssistantChatMessage(content="ok", tool_calls=[tc])]
        prepped, _, _ = provider._get_chat_completion_args(
            prompt_messages=messages, model=TEST_MODEL
        )
        args = prepped[0]["tool_calls"][0]["function"]["arguments"]
        assert args == "{}"

    def test_messages_without_tool_calls_unaffected(self, provider):
        messages = [ChatMessage.user("Hello"), ChatMessage.system("Be helpful")]
        prepped, _, _ = provider._get_chat_completion_args(
            prompt_messages=messages, model=TEST_MODEL
        )
        assert "tool_calls" not in prepped[0]
        assert "tool_calls" not in prepped[1]

    def test_tool_result_message_preserved(self, provider):
        messages = [
            ToolResultMessage(tool_call_id="call_1", content="Result data"),
        ]
        prepped, _, _ = provider._get_chat_completion_args(
            prompt_messages=messages, model=TEST_MODEL
        )
        assert prepped[0]["tool_call_id"] == "call_1"
        assert prepped[0]["role"] == "tool"

    def test_exclude_none_removes_absent_fields(self, provider):
        messages = [ChatMessage.user("test")]
        prepped, _, _ = provider._get_chat_completion_args(
            prompt_messages=messages, model=TEST_MODEL
        )
        assert "tool_calls" not in prepped[0]
        assert "tool_call_id" not in prepped[0]

    # ---- Functions / Tools ----

    def test_single_function_forces_tool_choice(self, provider, search_function):
        messages = [ChatMessage.user("search")]
        _, kwargs, _ = provider._get_chat_completion_args(
            prompt_messages=messages,
            model=TEST_MODEL,
            functions=[search_function],
        )
        assert "tools" in kwargs
        assert len(kwargs["tools"]) == 1
        assert kwargs["tool_choice"]["type"] == "function"
        assert kwargs["tool_choice"]["function"]["name"] == "web_search"

    def test_multiple_functions_no_forced_choice(
        self, provider, search_function, write_function
    ):
        messages = [ChatMessage.user("do something")]
        _, kwargs, _ = provider._get_chat_completion_args(
            prompt_messages=messages,
            model=TEST_MODEL,
            functions=[search_function, write_function],
        )
        assert len(kwargs["tools"]) == 2
        assert "tool_choice" not in kwargs

    def test_no_functions_no_tools(self, provider):
        messages = [ChatMessage.user("hi")]
        _, kwargs, _ = provider._get_chat_completion_args(
            prompt_messages=messages, model=TEST_MODEL
        )
        assert "tools" not in kwargs

    # ---- max_output_tokens ----

    def test_newer_model_uses_max_completion_tokens(self, provider):
        messages = [ChatMessage.user("hi")]
        _, kwargs, _ = provider._get_chat_completion_args(
            prompt_messages=messages,
            model=TEST_MODEL_GPT5,
            max_output_tokens=1000,
        )
        assert kwargs["max_completion_tokens"] == 1000
        assert "max_tokens" not in kwargs

    def test_o_series_uses_max_completion_tokens(self, provider):
        messages = [ChatMessage.user("hi")]
        _, kwargs, _ = provider._get_chat_completion_args(
            prompt_messages=messages,
            model=TEST_MODEL_O3,
            max_output_tokens=500,
        )
        assert kwargs["max_completion_tokens"] == 500

    def test_legacy_model_uses_max_tokens(self, provider):
        messages = [ChatMessage.user("hi")]
        _, kwargs, _ = provider._get_chat_completion_args(
            prompt_messages=messages,
            model=TEST_MODEL_LEGACY,
            max_output_tokens=500,
        )
        assert kwargs["max_tokens"] == 500
        assert "max_completion_tokens" not in kwargs

    def test_no_max_output_tokens_omits_both(self, provider):
        messages = [ChatMessage.user("hi")]
        _, kwargs, _ = provider._get_chat_completion_args(
            prompt_messages=messages, model=TEST_MODEL
        )
        assert "max_tokens" not in kwargs
        assert "max_completion_tokens" not in kwargs

    # ---- reasoning_effort ----

    def test_reasoning_effort_for_gpt5(self, provider):
        messages = [ChatMessage.user("think hard")]
        _, kwargs, _ = provider._get_chat_completion_args(
            prompt_messages=messages,
            model=TEST_MODEL_GPT5,
            reasoning_effort="high",
        )
        assert kwargs["reasoning_effort"] == "high"

    def test_reasoning_effort_for_o_series(self, provider):
        messages = [ChatMessage.user("think")]
        _, kwargs, _ = provider._get_chat_completion_args(
            prompt_messages=messages,
            model=TEST_MODEL_O3,
            reasoning_effort="low",
        )
        assert kwargs["reasoning_effort"] == "low"

    def test_reasoning_effort_ignored_for_legacy_model(self, provider):
        messages = [ChatMessage.user("hi")]
        _, kwargs, _ = provider._get_chat_completion_args(
            prompt_messages=messages,
            model=TEST_MODEL_LEGACY,
            reasoning_effort="high",
        )
        assert "reasoning_effort" not in kwargs

    # ---- extra_headers ----

    def test_extra_request_headers_merged(self, provider):
        provider._configuration.extra_request_headers = {"X-Custom": "value"}
        messages = [ChatMessage.user("hi")]
        _, kwargs, _ = provider._get_chat_completion_args(
            prompt_messages=messages, model=TEST_MODEL
        )
        assert kwargs["extra_headers"]["X-Custom"] == "value"

    def test_no_extra_headers_when_empty(self, provider):
        messages = [ChatMessage.user("hi")]
        _, kwargs, _ = provider._get_chat_completion_args(
            prompt_messages=messages, model=TEST_MODEL
        )
        assert "extra_headers" not in kwargs


# ---------------------------------------------------------------------------
# _parse_assistant_tool_calls
# ---------------------------------------------------------------------------
class TestParseAssistantToolCalls:
    def _make_openai_message(self, content="", tool_calls=None):
        msg = MagicMock()
        msg.content = content
        msg.tool_calls = tool_calls
        return msg

    def _make_openai_tool_call(self, id, name, arguments_json):
        tc = MagicMock()
        tc.id = id
        tc.type = "function"
        tc.function = MagicMock()
        tc.function.name = name
        tc.function.arguments = arguments_json
        return tc

    def test_successful_parse(self, provider):
        tc = self._make_openai_tool_call("call_1", "search", '{"query": "test"}')
        msg = self._make_openai_message(tool_calls=[tc])
        tool_calls, errors = provider._parse_assistant_tool_calls(msg)
        assert len(tool_calls) == 1
        assert tool_calls[0].id == "call_1"
        assert tool_calls[0].function.name == "search"
        assert tool_calls[0].function.arguments == {"query": "test"}
        assert errors == []

    def test_multiple_tool_calls(self, provider):
        tcs = [
            self._make_openai_tool_call("call_1", "search", '{"q": "a"}'),
            self._make_openai_tool_call("call_2", "write", '{"path": "/tmp"}'),
        ]
        msg = self._make_openai_message(tool_calls=tcs)
        tool_calls, errors = provider._parse_assistant_tool_calls(msg)
        assert len(tool_calls) == 2
        assert errors == []

    def test_no_tool_calls_returns_empty(self, provider):
        msg = self._make_openai_message(content="Just text", tool_calls=None)
        tool_calls, errors = provider._parse_assistant_tool_calls(msg)
        assert tool_calls == []
        assert errors == []

    def test_malformed_json_produces_error(self, provider):
        tc = self._make_openai_tool_call("call_1", "search", "not valid json{{{")
        msg = self._make_openai_message(tool_calls=[tc])
        tool_calls, errors = provider._parse_assistant_tool_calls(msg)
        assert tool_calls == []
        assert len(errors) == 1
        assert "search" in str(errors[0])

    def test_partial_parse_failure_ignored_when_all_succeed(self, provider):
        """If all tool calls eventually parse, errors are cleared."""
        tcs = [
            self._make_openai_tool_call("call_1", "search", '{"q": "test"}'),
            self._make_openai_tool_call("call_2", "write", '{"p": "x"}'),
        ]
        msg = self._make_openai_message(tool_calls=tcs)
        tool_calls, errors = provider._parse_assistant_tool_calls(msg)
        # Both parsed successfully, so errors should be empty
        assert len(tool_calls) == 2
        assert errors == []

    def test_empty_arguments_parsed(self, provider):
        tc = self._make_openai_tool_call("call_1", "noop", "{}")
        msg = self._make_openai_message(tool_calls=[tc])
        tool_calls, errors = provider._parse_assistant_tool_calls(msg)
        assert len(tool_calls) == 1
        assert tool_calls[0].function.arguments == {}
        assert errors == []


# ---------------------------------------------------------------------------
# _format_parse_errors
# ---------------------------------------------------------------------------
class TestFormatParseErrors:
    def test_invalid_function_call_error_formatted(self, provider, search_function):
        err = InvalidFunctionCallError(
            name="web_search",
            arguments={"wrong_param": "value"},
            message="Missing required field",
        )
        result = provider._format_parse_errors([err], None, [search_function])
        assert "web_search" in result
        assert "Missing required field" in result
        assert "wrong_param" in result
        assert "Expected parameters" in result

    def test_generic_exception_formatted(self, provider):
        err = ValueError("Something went wrong")
        result = provider._format_parse_errors([err], None, None)
        assert "ValueError" in result
        assert "Something went wrong" in result

    def test_empty_arguments_shows_no_arguments(self, provider, search_function):
        err = InvalidFunctionCallError(
            name="web_search", arguments={}, message="Bad call"
        )
        result = provider._format_parse_errors([err], None, [search_function])
        assert "(no arguments)" in result

    def test_no_matching_function_skips_expected_params(self, provider):
        err = InvalidFunctionCallError(
            name="unknown", arguments={"x": 1}, message="Unknown function"
        )
        result = provider._format_parse_errors([err], None, [])
        assert "Expected parameters" not in result

    def test_multiple_errors_joined(self, provider):
        errors = [
            ValueError("Error 1"),
            ValueError("Error 2"),
        ]
        result = provider._format_parse_errors(errors, None, None)
        assert "Error 1" in result
        assert "Error 2" in result


# ---------------------------------------------------------------------------
# create_chat_completion: main flow
# ---------------------------------------------------------------------------
class TestCreateChatCompletion:
    def _make_completion_response(
        self, content: str | None = "Response", tool_calls=None, usage=None
    ):
        """Build a mock ChatCompletion object."""
        message = MagicMock()
        message.content = content
        message.tool_calls = tool_calls

        def _model_dump(**kwargs):
            d = {"role": "assistant", "content": content}
            if kwargs.get("exclude_none"):
                d = {k: v for k, v in d.items() if v is not None}
            return d

        message.model_dump = _model_dump

        choice = MagicMock()
        choice.message = message

        completion = MagicMock()
        completion.choices = [choice]
        completion.usage = usage
        return completion

    def _make_usage(self, prompt_tokens=100, completion_tokens=50):
        usage = MagicMock()
        usage.prompt_tokens = prompt_tokens
        usage.completion_tokens = completion_tokens
        return usage

    @pytest.mark.asyncio
    async def test_successful_completion(self, provider):
        usage = self._make_usage()
        completion = self._make_completion_response(content="Hello!", usage=usage)
        provider._client.chat.completions.create = AsyncMock(return_value=completion)

        result = await provider.create_chat_completion(
            model_prompt=[ChatMessage.user("Hi")],
            model_name=TEST_MODEL,
        )
        assert result.response.content == "Hello!"
        assert result.prompt_tokens_used == 100
        assert result.completion_tokens_used == 50

    @pytest.mark.asyncio
    async def test_completion_with_parser(self, provider):
        usage = self._make_usage()
        completion = self._make_completion_response(
            content="parsed content", usage=usage
        )
        provider._client.chat.completions.create = AsyncMock(return_value=completion)

        def parser(msg: AssistantChatMessage) -> str:
            return msg.content.upper()

        result = await provider.create_chat_completion(
            model_prompt=[ChatMessage.user("Hi")],
            model_name=TEST_MODEL,
            completion_parser=parser,
        )
        assert result.parsed_result == "PARSED CONTENT"

    @pytest.mark.asyncio
    async def test_completion_with_tool_calls(self, provider, search_function):
        tc_mock = MagicMock()
        tc_mock.id = "call_1"
        tc_mock.type = "function"
        tc_mock.function = MagicMock()
        tc_mock.function.name = "web_search"
        tc_mock.function.arguments = '{"query": "test"}'

        usage = self._make_usage()
        completion = self._make_completion_response(
            content="Searching", tool_calls=[tc_mock], usage=usage
        )
        provider._client.chat.completions.create = AsyncMock(return_value=completion)

        result = await provider.create_chat_completion(
            model_prompt=[ChatMessage.user("search for test")],
            model_name=TEST_MODEL,
            functions=[search_function],
        )
        assert result.response.tool_calls is not None
        assert result.response.tool_calls[0].function.name == "web_search"

    @pytest.mark.asyncio
    async def test_parse_error_retries_then_succeeds(self, provider):
        """When parser fails on first attempt, retry and succeed on second."""
        usage = self._make_usage()

        # First response will fail parsing, second will succeed
        completion1 = self._make_completion_response(content="bad", usage=usage)
        completion2 = self._make_completion_response(content="good", usage=usage)

        provider._client.chat.completions.create = AsyncMock(
            side_effect=[completion1, completion2]
        )

        call_count = 0

        def flaky_parser(msg: AssistantChatMessage) -> str:
            nonlocal call_count
            call_count += 1
            if call_count == 1:
                raise ValueError("Parse failed")
            return msg.content

        result = await provider.create_chat_completion(
            model_prompt=[ChatMessage.user("test")],
            model_name=TEST_MODEL,
            completion_parser=flaky_parser,
        )
        assert result.parsed_result == "good"

    @pytest.mark.asyncio
    async def test_parse_error_exhausts_retries_raises(self, provider):
        """When parser always fails, it raises after max retries."""
        usage = self._make_usage()
        completion = self._make_completion_response(content="always bad", usage=usage)
        provider._client.chat.completions.create = AsyncMock(return_value=completion)

        provider._configuration.fix_failed_parse_tries = 2

        def always_fail(msg: AssistantChatMessage):
            raise ValueError("Always fails")

        with pytest.raises(ValueError, match="Always fails"):
            await provider.create_chat_completion(
                model_prompt=[ChatMessage.user("test")],
                model_name=TEST_MODEL,
                completion_parser=always_fail,
            )

    @pytest.mark.asyncio
    async def test_retry_strips_tool_calls_from_message(self, provider):
        """On retry, tool_calls are stripped from the failed assistant message
        to prevent OpenAI 400 errors."""
        tc_mock = MagicMock()
        tc_mock.id = "call_1"
        tc_mock.type = "function"
        tc_mock.function = MagicMock()
        tc_mock.function.name = "search"
        tc_mock.function.arguments = '{"q": "test"}'

        usage = self._make_usage()

        # First response has tool_calls that will fail validation
        bad_completion = self._make_completion_response(
            content="searching", tool_calls=[tc_mock], usage=usage
        )
        # Second response succeeds
        good_completion = self._make_completion_response(content="done", usage=usage)

        call_count = 0

        async def track_calls(**kwargs):
            nonlocal call_count
            call_count += 1
            if call_count == 1:
                return bad_completion
            # Verify the retry messages don't contain tool_calls
            msgs = kwargs.get("messages", [])
            for msg in msgs:
                if msg.get("role") == "assistant" and "tool_calls" in msg:
                    pytest.fail("Retry message should not have tool_calls")
            return good_completion

        provider._client.chat.completions.create = track_calls

        attempt = 0

        def fail_then_succeed(msg: AssistantChatMessage) -> str:
            nonlocal attempt
            attempt += 1
            if attempt == 1:
                raise ValueError("Bad parse")
            return msg.content

        result = await provider.create_chat_completion(
            model_prompt=[ChatMessage.user("test")],
            model_name=TEST_MODEL,
            completion_parser=fail_then_succeed,
        )
        assert result.parsed_result == "done"

    @pytest.mark.asyncio
    async def test_retry_with_null_content_does_not_send_null(self, provider):
        """When GPT-5.4 returns tool_calls with content=None, the retry
        message must have content="" not null/missing, otherwise OpenAI
        returns 400 'expected a string, got null'."""
        tc_mock = MagicMock()
        tc_mock.id = "call_1"
        tc_mock.type = "function"
        tc_mock.function = MagicMock()
        tc_mock.function.name = "write_file"
        tc_mock.function.arguments = "invalid json{{"

        usage = self._make_usage()

        # First response: content=None with tool_calls (GPT-5.4 behavior)
        bad_completion = self._make_completion_response(
            content=None, tool_calls=[tc_mock], usage=usage
        )
        good_completion = self._make_completion_response(content="done", usage=usage)

        call_count = 0

        async def check_retry_messages(**kwargs):
            nonlocal call_count
            call_count += 1
            if call_count == 1:
                return bad_completion
            # On retry, verify assistant message has string content
            msgs = kwargs.get("messages", [])
            for msg in msgs:
                if msg.get("role") == "assistant":
                    assert (
                        msg.get("content") is not None
                    ), "Assistant message content must not be null on retry"
                    assert isinstance(
                        msg["content"], str
                    ), f"Content must be str, got {type(msg['content'])}"
            return good_completion

        provider._client.chat.completions.create = check_retry_messages

        attempt = 0

        def fail_then_succeed(msg):
            nonlocal attempt
            attempt += 1
            if attempt == 1:
                raise ValueError("Parse failed")
            return msg.content

        result = await provider.create_chat_completion(
            model_prompt=[ChatMessage.user("test")],
            model_name=TEST_MODEL,
            completion_parser=fail_then_succeed,
        )
        assert result.parsed_result == "done"
        assert call_count >= 2  # May take multiple retries

    @pytest.mark.asyncio
    async def test_no_usage_defaults_to_zero(self, provider):
        completion = self._make_completion_response(content="hi", usage=None)
        provider._client.chat.completions.create = AsyncMock(return_value=completion)

        result = await provider.create_chat_completion(
            model_prompt=[ChatMessage.user("test")],
            model_name=TEST_MODEL,
        )
        assert result.prompt_tokens_used == 0
        assert result.completion_tokens_used == 0

    @pytest.mark.asyncio
    async def test_cost_tracked_in_budget(self, provider):
        usage = self._make_usage(prompt_tokens=1000, completion_tokens=500)
        completion = self._make_completion_response(content="hi", usage=usage)
        provider._client.chat.completions.create = AsyncMock(return_value=completion)

        initial_cost = provider._budget.total_cost

        await provider.create_chat_completion(
            model_prompt=[ChatMessage.user("test")],
            model_name=TEST_MODEL,
        )

        assert provider._budget.total_cost > initial_cost

    @pytest.mark.asyncio
    async def test_no_budget_cost_is_zero(self, provider):
        provider._budget = None
        usage = self._make_usage()
        completion = self._make_completion_response(content="hi", usage=usage)
        provider._client.chat.completions.create = AsyncMock(return_value=completion)

        result = await provider.create_chat_completion(
            model_prompt=[ChatMessage.user("test")],
            model_name=TEST_MODEL,
        )
        assert result is not None


# ---------------------------------------------------------------------------
# count_message_tokens
# ---------------------------------------------------------------------------
class TestCountMessageTokens:
    def test_single_message_counts_role_and_content(self, provider):
        """Tokenizer splits on whitespace; 'USER: hello world' = 3 tokens."""
        msg = ChatMessage.user("hello world")
        count = provider.count_message_tokens(msg, TEST_MODEL)
        # "USER: hello world" split by spaces = ["USER:", "hello", "world"] = 3
        assert count == 3

    def test_list_of_messages_concatenated(self, provider):
        msgs = [ChatMessage.user("hello"), ChatMessage.system("be helpful")]
        count = provider.count_message_tokens(msgs, TEST_MODEL)
        # Concatenated messages should have more tokens than a single one
        single = provider.count_message_tokens(msgs[0], TEST_MODEL)
        assert count > single

    def test_longer_message_has_more_tokens(self, provider):
        short = ChatMessage.user("hi")
        long = ChatMessage.user("this is a much longer message with many words")
        assert provider.count_message_tokens(long, TEST_MODEL) > (
            provider.count_message_tokens(short, TEST_MODEL)
        )


# ---------------------------------------------------------------------------
# get_token_limit
# ---------------------------------------------------------------------------
class TestGetTokenLimit:
    def test_returns_model_max_tokens(self, provider):
        assert provider.get_token_limit(TEST_MODEL) == 4096
        assert provider.get_token_limit(TEST_MODEL_GPT5) == 400_000