Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

44.2 KB
Raw Blame
"""Test cases for Claude LLM implementation using Anthropic SDK."""

from types import SimpleNamespace

import pytest
from jsonschema import Draft202012Validator

from xagent.core.model.chat.basic.claude import (
    ClaudeLLM,
    _anthropic_input_usage,
    _fix_pydantic_schema_for_claude,
)
from xagent.core.model.chat.types import ChunkType


class _AsyncEventStream:
    def __init__(self, events):
        self._events = iter(events)

    def __aiter__(self):
        return self

    async def __anext__(self):
        try:
            return next(self._events)
        except StopIteration as exc:
            raise StopAsyncIteration from exc


class TestClaudeLLM:
    """Test cases for Claude LLM implementation."""

    @pytest.fixture
    def llm(self, claude_llm_config):
        """Fixture providing Claude LLM instance."""
        return ClaudeLLM(**claude_llm_config)

    @pytest.mark.asyncio
    async def test_auto_only_model_normalizes_required_tool_choice(self, mocker):
        llm = ClaudeLLM(model_name="claude-opus-5-5-20261001", api_key="test-key")
        client = mocker.AsyncMock()
        client.messages.create.return_value = SimpleNamespace(
            stop_reason="end_turn",
            content=[SimpleNamespace(type="text", text="Done")],
            usage=SimpleNamespace(input_tokens=1, output_tokens=1),
        )
        llm._client = client
        tools = [
            {
                "type": "function",
                "function": {
                    "name": "lookup",
                    "description": "Look up a value",
                    "parameters": {"type": "object", "properties": {}},
                },
            }
        ]

        await llm.chat(
            [{"role": "user", "content": "hello"}],
            tools=tools,
            tool_choice="required",
        )

        assert llm.supports_forced_tool_choice is False
        assert client.messages.create.call_args.kwargs["tool_choice"] == {
            "type": "auto"
        }

    @pytest.mark.asyncio
    async def test_auto_only_model_normalizes_stream_tool_choice(self, mocker):
        llm = ClaudeLLM(model_name="claude-opus-5-5", api_key="test-key")
        client = mocker.AsyncMock()
        client.messages.create.return_value = _AsyncEventStream([])
        llm._client = client
        tools = [
            {
                "type": "function",
                "function": {
                    "name": "lookup",
                    "description": "Look up a value",
                    "parameters": {"type": "object", "properties": {}},
                },
            }
        ]

        chunks = [
            chunk
            async for chunk in llm.stream_chat(
                [{"role": "user", "content": "hello"}],
                tools=tools,
                tool_choice="required",
            )
        ]

        assert chunks == []
        assert client.messages.create.call_args.kwargs["tool_choice"] == {
            "type": "auto"
        }

    def test_schema_fix_preserves_property_named_type(self):
        """Claude schema fixer must not treat properties maps as schema objects."""
        schema = {
            "title": "get_workspace_output_filesArgs",
            "type": "object",
            "properties": {
                "type": {},
                "limit": {"type": "integer", "minimum": 1},
            },
        }

        fixed = _fix_pydantic_schema_for_claude(schema)

        Draft202012Validator.check_schema(fixed)
        assert fixed["type"] == "object"
        assert fixed["additionalProperties"] is False
        assert fixed["properties"]["type"] == {
            "type": "object",
            "properties": {},
            "additionalProperties": False,
        }
        assert fixed["properties"]["limit"] == {"type": "integer"}

    @pytest.mark.asyncio
    async def test_stream_chat_yields_tool_call_argument_deltas(self, llm, mocker):
        """Claude input_json_delta events should surface before message stop."""
        mock_client = mocker.AsyncMock()
        events = [
            SimpleNamespace(
                type="content_block_start",
                index=0,
                content_block=SimpleNamespace(
                    type="tool_use",
                    id="toolu_1",
                    name="final_answer",
                ),
            ),
            SimpleNamespace(
                type="content_block_delta",
                index=0,
                delta=SimpleNamespace(
                    type="input_json_delta",
                    partial_json='{"answer":"Hel',
                ),
            ),
            SimpleNamespace(
                type="content_block_delta",
                index=0,
                delta=SimpleNamespace(
                    type="input_json_delta",
                    partial_json='lo"}',
                ),
            ),
            SimpleNamespace(
                type="message_delta",
                delta=SimpleNamespace(stop_reason="tool_use"),
            ),
        ]
        mock_client.messages.create.return_value = _AsyncEventStream(events)
        mocker.patch(
            "xagent.core.model.chat.basic.claude.AsyncAnthropic",
            return_value=mock_client,
        )

        chunks = [
            chunk
            async for chunk in llm.stream_chat(
                [{"role": "user", "content": "answer"}],
                tools=[
                    {
                        "type": "function",
                        "function": {
                            "name": "final_answer",
                            "description": "Return final answer",
                            "parameters": {
                                "type": "object",
                                "properties": {"answer": {"type": "string"}},
                                "required": ["answer"],
                            },
                        },
                    }
                ],
            )
        ]

        tool_chunks = [chunk for chunk in chunks if chunk.type == ChunkType.TOOL_CALL]
        assert [
            chunk.tool_calls[0]["function"]["arguments"] for chunk in tool_chunks
        ] == ['{"answer":"Hel', '{"answer":"Hello"}', '{"answer":"Hello"}']
        assert tool_chunks[-1].finish_reason == "tool_use"

    @pytest.mark.asyncio
    async def test_basic_chat_completion(self, llm, mocker):
        """Test basic chat completion functionality."""
        # Setup mock
        mock_client = mocker.AsyncMock()

        # Create mock response
        mock_text_block = mocker.Mock()
        mock_text_block.type = "text"
        mock_text_block.text = "Hello World"

        # Mock usage with integer values
        mock_usage = mocker.Mock()
        mock_usage.input_tokens = 10
        mock_usage.output_tokens = 5

        mock_response = mocker.Mock()
        mock_response.stop_reason = "stop"
        mock_response.content = [mock_text_block]
        mock_response.usage = mock_usage
        mock_response.model_dump = mocker.Mock(return_value={"content": "Hello World"})

        mock_client.messages.create.return_value = mock_response
        mocker.patch(
            "xagent.core.model.chat.basic.claude.AsyncAnthropic",
            return_value=mock_client,
        )

        messages = [
            {
                "role": "user",
                "content": "Hello! Please respond with just 'Hello World'.",
            },
        ]

        response = await llm.chat(messages)

        # Verify response is a non-empty string
        assert isinstance(response, str)
        assert response == "Hello World"
        print(f"Basic chat response: {response}")

        # Verify the API was called with correct parameters
        mock_client.messages.create.assert_called_once()
        call_args = mock_client.messages.create.call_args
        assert call_args.kwargs["model"] == "claude-3-5-sonnet-20241022"
        assert "temperature" in call_args.kwargs
        assert call_args.kwargs["temperature"] == 0.7

    @pytest.mark.asyncio
    async def test_tool_calling(self, llm, mocker):
        """Test tool calling functionality."""
        # Setup mock
        mock_client = mocker.AsyncMock()

        # Create mock response with tool use
        mock_tool_block = mocker.Mock()
        mock_tool_block.type = "tool_use"
        mock_tool_block.id = "test_tool_id"
        mock_tool_block.name = "get_weather"
        mock_tool_block.input = {"location": "Boston"}

        # Mock usage with integer values
        mock_usage = mocker.Mock()
        mock_usage.input_tokens = 20
        mock_usage.output_tokens = 10

        mock_response = mocker.Mock()
        mock_response.stop_reason = "tool_use"
        mock_response.content = [mock_tool_block]
        mock_response.usage = mock_usage
        mock_response.model_dump = mocker.Mock(return_value={"tool_use": "get_weather"})

        mock_client.messages.create.return_value = mock_response
        mocker.patch(
            "xagent.core.model.chat.basic.claude.AsyncAnthropic",
            return_value=mock_client,
        )

        messages = [
            {"role": "user", "content": "What's the weather like in Boston?"},
        ]

        tools = [
            {
                "type": "function",
                "function": {
                    "name": "get_weather",
                    "description": "Get the current weather for a location",
                    "parameters": {
                        "type": "object",
                        "properties": {
                            "location": {
                                "type": "string",
                                "description": "The city and state, e.g. San Francisco, CA",
                            }
                        },
                        "required": ["location"],
                    },
                },
            }
        ]

        response = await llm.chat(messages, tools=tools)

        # Verify tool call response structure
        assert isinstance(response, dict)
        assert response.get("type") == "tool_call"
        assert "tool_calls" in response

        tool_calls = response["tool_calls"]
        assert len(tool_calls) > 0
        assert tool_calls[0]["function"]["name"] == "get_weather"
        assert tool_calls[0]["id"] == "test_tool_id"
        print(f"Tool calling response: {response}")

        # Verify the API was called with tools
        mock_client.messages.create.assert_called_once()
        call_args = mock_client.messages.create.call_args
        assert "tools" in call_args.kwargs

    @pytest.mark.asyncio
    async def test_system_message_handling(self, llm, mocker):
        """Test that system messages are properly handled."""
        # Setup mock
        mock_client = mocker.AsyncMock()

        mock_text_block = mocker.Mock()
        mock_text_block.type = "text"
        mock_text_block.text = "Response"

        # Mock usage with integer values
        mock_usage = mocker.Mock()
        mock_usage.input_tokens = 10
        mock_usage.output_tokens = 5

        mock_response = mocker.Mock()
        mock_response.stop_reason = "stop"
        mock_response.content = [mock_text_block]
        mock_response.usage = mock_usage

        mock_client.messages.create.return_value = mock_response
        mocker.patch(
            "xagent.core.model.chat.basic.claude.AsyncAnthropic",
            return_value=mock_client,
        )

        messages = [
            {"role": "system", "content": "You are a helpful assistant."},
            {"role": "user", "content": "Hello"},
        ]

        await llm.chat(messages)

        # Verify system message was passed separately
        call_args = mock_client.messages.create.call_args
        assert "system" in call_args.kwargs
        assert call_args.kwargs["system"] == "You are a helpful assistant."

    @pytest.mark.asyncio
    async def test_tool_call_history_is_sent_as_tool_use_and_result(self, llm, mocker):
        """Claude expects each tool_result to follow a matching tool_use block."""
        mock_client = mocker.AsyncMock()

        mock_text_block = mocker.Mock()
        mock_text_block.type = "text"
        mock_text_block.text = "Final answer"

        mock_usage = mocker.Mock()
        mock_usage.input_tokens = 10
        mock_usage.output_tokens = 5

        mock_response = mocker.Mock()
        mock_response.stop_reason = "stop"
        mock_response.content = [mock_text_block]
        mock_response.usage = mock_usage

        mock_client.messages.create.return_value = mock_response
        mocker.patch(
            "xagent.core.model.chat.basic.claude.AsyncAnthropic",
            return_value=mock_client,
        )

        messages = [
            {"role": "user", "content": "Search this"},
            {
                "role": "assistant",
                "content": "",
                "tool_calls": [
                    {
                        "id": "toolu_1",
                        "type": "function",
                        "function": {
                            "name": "web_search",
                            "arguments": '{"query":"x"}',
                        },
                    }
                ],
            },
            {
                "role": "tool",
                "content": "Tool web_search returned: result",
                "tool_call_id": "toolu_1",
            },
        ]

        await llm.chat(messages, tools=[])

        call_args = mock_client.messages.create.call_args
        anthropic_messages = call_args.kwargs["messages"]
        assert anthropic_messages[-2] == {
            "role": "assistant",
            "content": [
                {
                    "type": "tool_use",
                    "id": "toolu_1",
                    "name": "web_search",
                    "input": {"query": "x"},
                }
            ],
        }
        assert anthropic_messages[-1] == {
            "role": "user",
            "content": [
                {
                    "type": "tool_result",
                    "tool_use_id": "toolu_1",
                    "content": "Tool web_search returned: result",
                }
            ],
        }

    @pytest.mark.asyncio
    async def test_context_manager(self, claude_llm_config, mocker):
        """Test async context manager functionality."""
        mock_client = mocker.AsyncMock()

        mock_text_block = mocker.Mock()
        mock_text_block.type = "text"
        mock_text_block.text = "test"

        # Mock usage with integer values
        mock_usage = mocker.Mock()
        mock_usage.input_tokens = 5
        mock_usage.output_tokens = 3

        mock_response = mocker.Mock()
        mock_response.stop_reason = "stop"
        mock_response.content = [mock_text_block]
        mock_response.usage = mock_usage

        mock_client.messages.create.return_value = mock_response
        mocker.patch(
            "xagent.core.model.chat.basic.claude.AsyncAnthropic",
            return_value=mock_client,
        )

        async with ClaudeLLM(**claude_llm_config) as ctx_llm:
            messages = [{"role": "user", "content": "Say 'test'"}]
            response = await ctx_llm.chat(messages)

            assert isinstance(response, str)
            assert response == "test"
            print(f"Context manager response: {response}")

        # Verify the client was properly closed
        assert ctx_llm._client is None

    @pytest.mark.asyncio
    async def test_error_handling_missing_sdk(self, claude_llm_config, mocker):
        """Test error handling when SDK is not installed."""
        # Mock AsyncAnthropic as None
        mocker.patch("xagent.core.model.chat.basic.claude.AsyncAnthropic", None)

        llm = ClaudeLLM(**claude_llm_config)
        messages = [{"role": "user", "content": "Hello"}]

        # Should raise a RuntimeError
        with pytest.raises(RuntimeError, match="anthropic SDK is not installed"):
            await llm.chat(messages)

    @pytest.mark.asyncio
    async def test_error_handling_missing_api_key(
        self, claude_llm_config, monkeypatch, mocker
    ):
        """Test error handling when API key is missing."""
        # Remove all API key environment variables
        monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
        monkeypatch.delenv("CLAUDE_API_KEY", raising=False)

        mocker.patch("xagent.core.model.chat.basic.claude.AsyncAnthropic")

        # Create LLM without API key
        config = claude_llm_config.copy()
        config["api_key"] = None

        llm = ClaudeLLM(**config)
        messages = [{"role": "user", "content": "Hello"}]

        # Should raise a RuntimeError
        with pytest.raises(
            RuntimeError, match="ANTHROPIC_API_KEY or CLAUDE_API_KEY must be set"
        ):
            await llm.chat(messages)

    @pytest.mark.asyncio
    async def test_custom_parameters(self, llm, mocker):
        """Test custom parameters like temperature and max_tokens."""
        # Setup mock
        mock_client = mocker.AsyncMock()

        mock_text_block = mocker.Mock()
        mock_text_block.type = "text"
        mock_text_block.text = "Test response"

        mock_response = mocker.Mock()
        mock_response.stop_reason = "stop"
        mock_response.content = [mock_text_block]
        # Mock usage with integer values
        mock_usage = mocker.Mock()
        mock_usage.input_tokens = 10
        mock_usage.output_tokens = 5

        mock_response.usage = mock_usage

        mock_client.messages.create.return_value = mock_response
        mocker.patch(
            "xagent.core.model.chat.basic.claude.AsyncAnthropic",
            return_value=mock_client,
        )

        messages = [{"role": "user", "content": "Count from 1 to 3."}]

        # Test with custom temperature and max_tokens
        response = await llm.chat(
            messages,
            temperature=0.1,  # Low temperature for more deterministic output
            max_tokens=50,  # Limit response length
        )

        assert isinstance(response, str)
        assert response == "Test response"
        print(f"Custom parameters response: {response}")

        # Verify custom parameters were passed
        call_args = mock_client.messages.create.call_args
        assert call_args.kwargs["temperature"] == 0.1
        assert call_args.kwargs["max_tokens"] == 50

    @pytest.mark.asyncio
    async def test_cleanup(self, claude_llm_config, mocker):
        """Test that client cleanup works properly."""
        mock_client = mocker.AsyncMock()

        mock_text_block = mocker.Mock()
        mock_text_block.type = "text"
        mock_text_block.text = "test"

        mock_response = mocker.Mock()
        mock_response.stop_reason = "stop"
        mock_response.content = [mock_text_block]
        # Mock usage with integer values
        mock_usage = mocker.Mock()
        mock_usage.input_tokens = 10
        mock_usage.output_tokens = 5

        mock_response.usage = mock_usage

        mock_client.messages.create.return_value = mock_response
        mocker.patch(
            "xagent.core.model.chat.basic.claude.AsyncAnthropic",
            return_value=mock_client,
        )

        llm = ClaudeLLM(**claude_llm_config)

        # Initialize client
        llm._ensure_client()

        # Verify client was created
        assert llm._client is not None

        # Close the client
        await llm.close()

        # Verify client is closed (this is mainly to ensure no exceptions are raised)
        assert llm._client is None

    @pytest.mark.asyncio
    async def test_abilities_property(self, claude_llm_config):
        """Test that abilities are correctly set based on model name."""
        # Test vision model (Claude 3 models support vision)
        vision_llm = ClaudeLLM(
            model_name="claude-3-5-sonnet-20241022", api_key="test-key"
        )
        assert "vision" in vision_llm.abilities

        # Test chat and tool_calling abilities
        assert "chat" in vision_llm.abilities
        assert "tool_calling" in vision_llm.abilities

    @pytest.mark.asyncio
    async def test_supports_thinking_mode(self, llm):
        """Test that Claude supports thinking mode."""
        assert llm.supports_thinking_mode is True

    @pytest.mark.asyncio
    async def test_thinking_mode_enabled(self, llm, mocker):
        """Test thinking mode configuration."""
        # Setup mock
        mock_client = mocker.AsyncMock()

        mock_text_block = mocker.Mock()
        mock_text_block.type = "text"
        mock_text_block.text = "Response with thinking"

        mock_response = mocker.Mock()
        mock_response.stop_reason = "stop"
        mock_response.content = [mock_text_block]
        # Mock usage with integer values
        mock_usage = mocker.Mock()
        mock_usage.input_tokens = 10
        mock_usage.output_tokens = 5

        mock_response.usage = mock_usage

        mock_client.messages.create.return_value = mock_response
        mocker.patch(
            "xagent.core.model.chat.basic.claude.AsyncAnthropic",
            return_value=mock_client,
        )

        messages = [{"role": "user", "content": "Think about this"}]

        # Test with thinking mode enabled
        response = await llm.chat(
            messages, thinking={"type": "enabled", "budget_tokens": 20480}
        )

        assert isinstance(response, str)

        # Verify thinking mode was passed
        call_args = mock_client.messages.create.call_args
        assert "thinking" in call_args.kwargs
        assert call_args.kwargs["thinking"]["type"] == "enabled"
        assert call_args.kwargs["thinking"]["budget_tokens"] == 20480

    @pytest.mark.asyncio
    async def test_thinking_mode_disabled(self, llm, mocker):
        """Test thinking mode disabled configuration."""
        # Setup mock
        mock_client = mocker.AsyncMock()

        mock_text_block = mocker.Mock()
        mock_text_block.type = "text"
        mock_text_block.text = "Response without thinking"

        mock_response = mocker.Mock()
        mock_response.stop_reason = "stop"
        mock_response.content = [mock_text_block]
        # Mock usage with integer values
        mock_usage = mocker.Mock()
        mock_usage.input_tokens = 10
        mock_usage.output_tokens = 5

        mock_response.usage = mock_usage

        mock_client.messages.create.return_value = mock_response
        mocker.patch(
            "xagent.core.model.chat.basic.claude.AsyncAnthropic",
            return_value=mock_client,
        )

        messages = [{"role": "user", "content": "Quick response"}]

        # Test with thinking mode explicitly disabled
        response = await llm.chat(messages, thinking={"type": "disabled"})

        assert isinstance(response, str)

        # Verify thinking mode was set to disabled
        call_args = mock_client.messages.create.call_args
        assert "thinking" in call_args.kwargs
        assert call_args.kwargs["thinking"]["type"] == "disabled"

    @pytest.mark.asyncio
    async def test_vision_chat(self, llm, mocker):
        """Test vision chat functionality."""
        # Setup mock
        mock_client = mocker.AsyncMock()

        mock_text_block = mocker.Mock()
        mock_text_block.type = "text"
        mock_text_block.text = "I see an image"

        mock_response = mocker.Mock()
        mock_response.stop_reason = "stop"
        mock_response.content = [mock_text_block]
        # Mock usage with integer values
        mock_usage = mocker.Mock()
        mock_usage.input_tokens = 10
        mock_usage.output_tokens = 5

        mock_response.usage = mock_usage

        mock_client.messages.create.return_value = mock_response
        mocker.patch(
            "xagent.core.model.chat.basic.claude.AsyncAnthropic",
            return_value=mock_client,
        )

        messages = [
            {
                "role": "user",
                "content": [
                    {"type": "text", "text": "What's in this image?"},
                    {
                        "type": "image_url",
                        "image_url": {
                            "url": "data:image/jpeg;base64,/9j/4AAQSkZJRg..."
                        },
                    },
                ],
            }
        ]

        response = await llm.vision_chat(messages)

        assert isinstance(response, str)
        assert response == "I see an image"
        print(f"Vision chat response: {response}")

    @pytest.mark.asyncio
    async def test_tool_choice_auto(self, llm, mocker):
        """Test tool_choice with 'auto' mode."""
        # Setup mock
        mock_client = mocker.AsyncMock()

        mock_text_block = mocker.Mock()
        mock_text_block.type = "text"
        mock_text_block.text = "Response"

        mock_response = mocker.Mock()
        mock_response.stop_reason = "stop"
        mock_response.content = [mock_text_block]
        # Mock usage with integer values
        mock_usage = mocker.Mock()
        mock_usage.input_tokens = 10
        mock_usage.output_tokens = 5

        mock_response.usage = mock_usage

        mock_client.messages.create.return_value = mock_response
        mocker.patch(
            "xagent.core.model.chat.basic.claude.AsyncAnthropic",
            return_value=mock_client,
        )

        messages = [{"role": "user", "content": "Hello"}]
        tools = [
            {
                "type": "function",
                "function": {
                    "name": "test_tool",
                    "description": "A test tool",
                    "parameters": {"type": "object", "properties": {}},
                },
            }
        ]

        await llm.chat(messages, tools=tools, tool_choice="auto")

        # Verify tool_choice was set to auto
        call_args = mock_client.messages.create.call_args
        assert "tool_choice" in call_args.kwargs
        assert call_args.kwargs["tool_choice"]["type"] == "auto"

    @pytest.mark.asyncio
    async def test_tool_choice_any(self, llm, mocker):
        """Test tool_choice with 'any' mode."""
        # Setup mock
        mock_client = mocker.AsyncMock()

        mock_text_block = mocker.Mock()
        mock_text_block.type = "text"
        mock_text_block.text = "Response"

        mock_response = mocker.Mock()
        mock_response.stop_reason = "stop"
        mock_response.content = [mock_text_block]
        # Mock usage with integer values
        mock_usage = mocker.Mock()
        mock_usage.input_tokens = 10
        mock_usage.output_tokens = 5

        mock_response.usage = mock_usage

        mock_client.messages.create.return_value = mock_response
        mocker.patch(
            "xagent.core.model.chat.basic.claude.AsyncAnthropic",
            return_value=mock_client,
        )

        messages = [{"role": "user", "content": "Hello"}]
        tools = [
            {
                "type": "function",
                "function": {
                    "name": "test_tool",
                    "description": "A test tool",
                    "parameters": {"type": "object", "properties": {}},
                },
            }
        ]

        await llm.chat(messages, tools=tools, tool_choice="any")

        # Verify tool_choice was set to any
        call_args = mock_client.messages.create.call_args
        assert "tool_choice" in call_args.kwargs
        assert call_args.kwargs["tool_choice"]["type"] == "any"

    @pytest.mark.asyncio
    async def test_default_max_tokens(self, claude_llm_config, mocker):
        """Test that default max_tokens is set when not provided."""
        # Setup mock
        mock_client = mocker.AsyncMock()

        mock_text_block = mocker.Mock()
        mock_text_block.type = "text"
        mock_text_block.text = "Response"

        mock_response = mocker.Mock()
        mock_response.stop_reason = "stop"
        mock_response.content = [mock_text_block]
        # Mock usage with integer values
        mock_usage = mocker.Mock()
        mock_usage.input_tokens = 10
        mock_usage.output_tokens = 5

        mock_response.usage = mock_usage

        mock_client.messages.create.return_value = mock_response
        mocker.patch(
            "xagent.core.model.chat.basic.claude.AsyncAnthropic",
            return_value=mock_client,
        )

        # Create LLM without default_max_tokens
        config = claude_llm_config.copy()
        config["default_max_tokens"] = None

        llm = ClaudeLLM(**config)

        messages = [{"role": "user", "content": "Hello"}]

        await llm.chat(messages)

        # Verify max_tokens was set to default (4096)
        call_args = mock_client.messages.create.call_args
        assert "max_tokens" in call_args.kwargs
        assert call_args.kwargs["max_tokens"] == 4096

    @pytest.mark.asyncio
    async def test_retryable_errors(self, llm, mocker):
        """Test that Anthropic errors are converted to LLMRetryableError."""
        from anthropic import APIStatusError, APITimeoutError, RateLimitError

        from xagent.core.model.chat.exceptions import LLMRetryableError

        # Mock the client
        mock_client = mocker.AsyncMock()
        mocker.patch(
            "xagent.core.model.chat.basic.claude.AsyncAnthropic",
            return_value=mock_client,
        )

        # Initialize client
        llm._ensure_client()
        llm._client = mock_client

        # Case 1: Timeout
        mock_client.messages.create.side_effect = APITimeoutError(request=mocker.Mock())
        with pytest.raises(LLMRetryableError):
            await llm.chat([{"role": "user", "content": "hi"}])

        # Case 2: Rate Limit
        mock_client.messages.create.side_effect = RateLimitError(
            message="Rate limit", response=mocker.Mock(), body={}
        )
        with pytest.raises(LLMRetryableError):
            await llm.chat([{"role": "user", "content": "hi"}])

        # Case 3: 500 Server Error
        mock_response = mocker.Mock()
        mock_response.status_code = 500
        # Mock usage with integer values
        mock_usage = mocker.Mock()
        mock_usage.input_tokens = 10
        mock_usage.output_tokens = 5

        mock_response.usage = mock_usage
        mock_client.messages.create.side_effect = APIStatusError(
            message="Server Error", response=mock_response, body={}
        )
        with pytest.raises(LLMRetryableError):
            await llm.chat([{"role": "user", "content": "hi"}])

        # Case 4: 400 Bad Request
        mock_response = mocker.Mock()
        mock_response.status_code = 400
        # Mock usage with integer values
        mock_usage = mocker.Mock()
        mock_usage.input_tokens = 10
        mock_usage.output_tokens = 5

        mock_response.usage = mock_usage
        mock_client.messages.create.side_effect = APIStatusError(
            message="Bad Request", response=mock_response, body={}
        )

        with pytest.raises(LLMRetryableError):
            await llm.chat([{"role": "user", "content": "hi"}])

    @pytest.mark.asyncio
    async def test_list_available_models_with_default_base_url(self, mocker):
        """Test listing available models using default base URL (official API)."""
        # Mock httpx response
        mock_response = mocker.MagicMock()
        mock_response.status_code = 200
        mock_response.json.return_value = {
            "data": [
                {
                    "id": "claude-3-5-sonnet-20241022",
                    "display_name": "Claude 3.5 Sonnet",
                    "created": 1234567890,
                    "type": "model",
                },
                {
                    "id": "claude-3-5-haiku-20241022",
                    "display_name": "Claude 3.5 Haiku",
                    "created": 1234567891,
                    "type": "model",
                },
            ],
        }

        mock_async_client = mocker.AsyncMock()
        mock_async_client.get.return_value = mock_response
        mock_async_client.__aenter__.return_value = mock_async_client
        mock_async_client.__aexit__.return_value = None

        mocker.patch("httpx.AsyncClient", return_value=mock_async_client)

        # Call without base_url - should use official API
        models = await ClaudeLLM.list_available_models("test-api-key")

        # Verify results
        assert len(models) == 2
        # Models are sorted by created date (newest first)
        # claude-3-5-haiku: created=1234567891, claude-3-5-sonnet: created=1234567890
        assert models[0]["id"] == "claude-3-5-haiku-20241022"
        assert models[1]["id"] == "claude-3-5-sonnet-20241022"

        # Verify the API was called with official base URL
        mock_async_client.get.assert_called_once()
        call_args = mock_async_client.get.call_args
        assert "api.anthropic.com/v1/models" in call_args[0][0]
        assert call_args[1]["headers"]["x-api-key"] == "test-api-key"

    @pytest.mark.asyncio
    async def test_list_available_models_with_custom_base_url(self, mocker):
        """Test listing available models using custom base URL."""
        # Mock httpx response
        mock_response = mocker.MagicMock()
        mock_response.status_code = 200
        mock_response.json.return_value = {
            "data": [
                {
                    "id": "custom-claude-model",
                    "display_name": "Custom Claude",
                    "created": 1234567890,
                    "type": "model",
                },
            ],
        }

        mock_async_client = mocker.AsyncMock()
        mock_async_client.get.return_value = mock_response
        mock_async_client.__aenter__.return_value = mock_async_client
        mock_async_client.__aexit__.return_value = None

        mocker.patch("httpx.AsyncClient", return_value=mock_async_client)

        # Call with custom base_url
        custom_base_url = "https://custom-proxy.com"
        models = await ClaudeLLM.list_available_models(
            "test-api-key", base_url=custom_base_url
        )

        # Verify results
        assert len(models) == 1
        assert models[0]["id"] == "custom-claude-model"

        # Verify the API was called with custom base URL
        mock_async_client.get.assert_called_once()
        call_args = mock_async_client.get.call_args
        assert "custom-proxy.com/v1/models" in call_args[0][0]

    @pytest.mark.asyncio
    async def test_list_available_models_unauthorized(self, mocker):
        """Test listing models with invalid API key."""
        import httpx

        # Mock httpx to raise 401 error
        mock_response = mocker.MagicMock()
        mock_response.status_code = 401
        mock_response.text = "Unauthorized"

        error = httpx.HTTPStatusError(
            "Unauthorized", request=mocker.MagicMock(), response=mock_response
        )

        mock_async_client = mocker.AsyncMock()
        mock_async_client.get.side_effect = error
        mock_async_client.__aenter__.return_value = mock_async_client
        mock_async_client.__aexit__.return_value = None

        mocker.patch("httpx.AsyncClient", return_value=mock_async_client)

        # Should raise ValueError for invalid API key
        with pytest.raises(ValueError, match="Invalid Anthropic API key"):
            await ClaudeLLM.list_available_models("invalid-key")

    @pytest.mark.asyncio
    async def test_output_config_json_schema(self, llm, mocker):
        """Test output_config with json_schema format."""
        # Setup mock
        mock_client = mocker.AsyncMock()

        mock_text_block = mocker.Mock()
        mock_text_block.type = "text"
        mock_text_block.text = '{"joke": "Why did the chicken cross the road?", "punchline": "To get to the other side!"}'

        mock_response = mocker.Mock()
        mock_response.stop_reason = "stop"
        mock_response.content = [mock_text_block]
        # Mock usage with integer values
        mock_usage = mocker.Mock()
        mock_usage.input_tokens = 20
        mock_usage.output_tokens = 15

        mock_response.usage = mock_usage

        mock_client.messages.create.return_value = mock_response
        mocker.patch(
            "xagent.core.model.chat.basic.claude.AsyncAnthropic",
            return_value=mock_client,
        )

        messages = [{"role": "user", "content": "Tell me a short joke."}]

        # Test with output_config using json_schema format
        output_config = {
            "format": {
                "type": "json_schema",
                "schema": {
                    "type": "object",
                    "properties": {
                        "joke": {
                            "type": "string",
                            "description": "The text of the joke.",
                        },
                        "punchline": {
                            "type": "string",
                            "description": "The punchline of the joke.",
                        },
                    },
                    "required": ["joke", "punchline"],
                    "additionalProperties": False,
                },
            }
        }

        response = await llm.chat(messages, output_config=output_config)

        assert isinstance(response, str)
        # Verify the response contains the expected JSON
        assert "joke" in response
        assert "punchline" in response

        # Verify the API was called with output_config
        mock_client.messages.create.assert_called_once()
        call_args = mock_client.messages.create.call_args
        assert "output_config" in call_args.kwargs
        assert call_args.kwargs["output_config"]["format"]["type"] == "json_schema"

    @pytest.mark.asyncio
    async def test_strict_tool_mode(self, llm, mocker):
        """Test strict mode for tool calling."""
        # Setup mock
        mock_client = mocker.AsyncMock()

        mock_tool_block = mocker.Mock()
        mock_tool_block.type = "tool_use"
        mock_tool_block.id = "test_tool_id"
        mock_tool_block.name = "search_flights"
        mock_tool_block.input = {"destination": "Paris", "date": "2025-03-15"}

        mock_usage = mocker.Mock()
        mock_usage.input_tokens = 25
        mock_usage.output_tokens = 10

        mock_response = mocker.Mock()
        mock_response.stop_reason = "tool_use"
        mock_response.content = [mock_tool_block]
        mock_response.usage = mock_usage

        mock_client.messages.create.return_value = mock_response
        mocker.patch(
            "xagent.core.model.chat.basic.claude.AsyncAnthropic",
            return_value=mock_client,
        )

        messages = [
            {"role": "user", "content": "Search for flights to Paris next month"}
        ]

        # Test with strict tool mode
        tools = [
            {
                "type": "function",
                "function": {
                    "name": "search_flights",
                    "description": "Search for flights",
                    "parameters": {
                        "type": "object",
                        "properties": {
                            "destination": {"type": "string"},
                            "date": {"type": "string", "format": "date"},
                        },
                        "required": ["destination", "date"],
                        "additionalProperties": False,
                    },
                },
                "strict": True,  # Enable strict mode
            }
        ]

        response = await llm.chat(messages, tools=tools)

        # Verify tool call response
        assert isinstance(response, dict)
        assert response.get("type") == "tool_call"

        # Verify the API was called with strict mode enabled
        call_args = mock_client.messages.create.call_args
        assert "tools" in call_args.kwargs
        anthropic_tools = call_args.kwargs["tools"]
        assert len(anthropic_tools) == 1
        assert anthropic_tools[0]["strict"] is True
        assert anthropic_tools[0]["name"] == "search_flights"


class TestClaudePromptCacheUsage:
    """Anthropic reports cache reads/writes outside input_tokens; we re-add them."""

    @pytest.fixture
    def llm(self, claude_llm_config):
        return ClaudeLLM(**claude_llm_config)

    def test_anthropic_input_usage_normalization(self):
        usage = SimpleNamespace(
            input_tokens=10,
            cache_read_input_tokens=80,
            cache_creation_input_tokens=30,
        )
        assert _anthropic_input_usage(usage) == (120, 80, 30)

        # Missing cache fields (older SDK / cache disabled) fall back to 0.
        assert _anthropic_input_usage(SimpleNamespace(input_tokens=10)) == (10, 0, 0)

        # Non-numeric values (e.g. Mock attributes in tests) must not raise.
        bad = SimpleNamespace(
            input_tokens=10,
            cache_read_input_tokens=object(),
            cache_creation_input_tokens=None,
        )
        assert _anthropic_input_usage(bad) == (10, 0, 0)

        # Dict-shaped payloads are handled like the shared cache helper.
        assert _anthropic_input_usage(
            {
                "input_tokens": 10,
                "cache_read_input_tokens": 80,
                "cache_creation_input_tokens": 30,
            }
        ) == (120, 80, 30)
        assert _anthropic_input_usage({}) == (0, 0, 0)

    @pytest.mark.asyncio
    async def test_chat_records_normalized_cache_usage(self, llm, mocker):
        from xagent.core.model.chat.token_context import TokenContextManager

        mock_text_block = mocker.Mock()
        mock_text_block.type = "text"
        mock_text_block.text = "ok"

        mock_response = mocker.Mock()
        mock_response.stop_reason = "stop"
        mock_response.content = [mock_text_block]
        mock_response.usage = SimpleNamespace(
            input_tokens=10,
            output_tokens=5,
            cache_read_input_tokens=80,
            cache_creation_input_tokens=30,
        )
        mock_response.model_dump = mocker.Mock(return_value={"content": "ok"})

        mock_client = mocker.AsyncMock()
        mock_client.messages.create.return_value = mock_response
        mocker.patch(
            "xagent.core.model.chat.basic.claude.AsyncAnthropic",
            return_value=mock_client,
        )

        with TokenContextManager() as manager:
            await llm.chat([{"role": "user", "content": "hi"}])
            usage = manager.get_usage()

        assert usage.input_tokens == 120
        assert usage.output_tokens == 5
        inp = next(d for d in usage.details if d["type"] == "input")
        assert inp["tokens"] == 120
        assert inp["cached_tokens"] == 80
        assert inp["cache_write_tokens"] == 30

    @pytest.mark.asyncio
    async def test_stream_chat_records_cache_usage_from_message_start(
        self, llm, mocker
    ):
        from xagent.core.model.chat.token_context import TokenContextManager

        events = [
            SimpleNamespace(
                type="message_start",
                message=SimpleNamespace(
                    usage=SimpleNamespace(
                        input_tokens=10,
                        cache_read_input_tokens=40,
                        cache_creation_input_tokens=15,
                    )
                ),
            ),
            SimpleNamespace(
                type="content_block_start",
                index=0,
                content_block=SimpleNamespace(type="text", text=""),
            ),
            SimpleNamespace(
                type="content_block_delta",
                index=0,
                delta=SimpleNamespace(type="text_delta", text="ok"),
            ),
            SimpleNamespace(
                type="message_delta",
                delta=SimpleNamespace(stop_reason="end_turn"),
                usage=SimpleNamespace(output_tokens=5),
            ),
        ]

        mock_client = mocker.AsyncMock()
        mock_client.messages.create.return_value = _AsyncEventStream(events)
        mocker.patch(
            "xagent.core.model.chat.basic.claude.AsyncAnthropic",
            return_value=mock_client,
        )

        with TokenContextManager() as manager:
            chunks = [
                chunk
                async for chunk in llm.stream_chat([{"role": "user", "content": "hi"}])
            ]
            usage = manager.get_usage()

        assert usage.input_tokens == 65
        assert usage.output_tokens == 5
        inp = next(d for d in usage.details if d["type"] == "input")
        assert inp["tokens"] == 65
        assert inp["cached_tokens"] == 40
        assert inp["cache_write_tokens"] == 15

        # The yielded usage chunk carries the message_start input counts so
        # downstream consumers (runtime usage/trace) see cache telemetry.
        usage_chunk = next(c for c in chunks if c.type == ChunkType.USAGE)
        assert usage_chunk.usage == {
            "input_tokens": 65,
            "output_tokens": 5,
            "total_tokens": 70,
            "cached_input_tokens": 40,
            "cache_write_input_tokens": 15,
        }