Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

38.3 KB
Raw Blame
import asyncio
from unittest.mock import AsyncMock, MagicMock, patch

import aiohttp
import pytest

from xagent.core.model.chat.basic.xinference import XinferenceLLM, _create_async_client
from xagent.core.model.chat.error import retry_on
from xagent.core.model.chat.exceptions import LLMTimeoutError
from xagent.core.model.chat.routing_context import (
    xinference_interactive_requests,
    xinference_routing_key,
)
from xagent.core.model.chat.timeout_config import TimeoutConfig
from xagent.core.model.chat.types import ChunkType


class TestXinferenceLLM:
    @pytest.mark.asyncio
    @pytest.mark.parametrize("stream", [False, True])
    @pytest.mark.parametrize("supports_priority", [False, True])
    async def test_only_interactive_requests_get_supported_chat_priority(
        self, monkeypatch, stream, supports_priority
    ):
        description = {"model_type": "LLM", "model_ability": ["chat"]}
        if supports_priority:
            description["supports_chat_priority"] = True
        describe = AsyncMock(return_value=description)
        monkeypatch.setattr(
            "xagent.core.model.public_xinference.PublicAsyncXinferenceClient.describe_model",
            describe,
        )
        requests = []

        async def chat(self, **kwargs):
            requests.append(kwargs["generate_config"].copy())
            if stream:

                async def result():
                    yield {"choices": [{"delta": {"content": "OK"}}]}

                return result()
            return {"content": "OK"}

        monkeypatch.setattr(
            "xagent.core.model.public_xinference.AsyncRESTfulChatModelHandle.chat",
            chat,
        )
        llm = XinferenceLLM(model_name="model", base_url="https://xinference.example")

        async def call(**kwargs):
            messages = [{"role": "user", "content": "Reply OK"}]
            if stream:
                assert [
                    c.delta async for c in llm.stream_chat(messages=messages, **kwargs)
                ] == ["OK"]
            else:
                await llm.chat(messages=messages, **kwargs)

        try:
            with xinference_interactive_requests(True):
                await call()
                await call(priority=0)
            await call()
        finally:
            await llm.close()
        describe.assert_awaited_once_with("model")
        assert requests[0].get("priority_hint") == (-1 if supports_priority else None)
        assert requests[1]["priority"] == 0
        assert "priority_hint" not in requests[1]
        assert "priority_hint" not in requests[2]

    @pytest.mark.asyncio
    @pytest.mark.parametrize("supports_affinity", [False, True])
    async def test_chat_uses_discovered_session_affinity(
        self, monkeypatch, supports_affinity: bool
    ) -> None:
        description = {"model_type": "LLM", "model_ability": ["chat"]}
        if supports_affinity:
            description["supports_chat_session_affinity"] = True

        def response(body):
            return MagicMock(
                status=200,
                json=AsyncMock(return_value=body),
                wait_for_close=AsyncMock(),
            )

        get = AsyncMock(return_value=response(description))
        post = AsyncMock(return_value=response({"content": "OK"}))
        monkeypatch.setattr(aiohttp.ClientSession, "get", get)
        monkeypatch.setattr(aiohttp.ClientSession, "post", post)
        llm = XinferenceLLM(model_name="model", base_url="https://xinference.example")
        token = xinference_routing_key.set("task-key")
        try:
            for _ in range(2):
                await llm.chat(messages=[{"role": "user", "content": "Reply OK"}])
        finally:
            xinference_routing_key.reset(token)
            await llm.close()
        get.assert_awaited_once()
        assert get.await_args.args == ("https://xinference.example/v1/models/model",)
        assert post.await_count == 2
        for call in post.await_args_list:
            if supports_affinity:
                assert call.kwargs["json"]["routing_key"] == "task-key"
            else:
                assert "routing_key" not in call.kwargs["json"]

    @pytest.mark.asyncio
    async def test_stream_chat_does_not_block_event_loop_before_first_token(
        self,
    ) -> None:
        release_stream = asyncio.Event()

        class BlockingModelHandle:
            async def chat(self, **kwargs: object):
                async def generate():
                    await release_stream.wait()
                    yield {
                        "choices": [
                            {
                                "delta": {"content": "ok"},
                                "finish_reason": None,
                            }
                        ]
                    }

                return generate()

        llm = XinferenceLLM(
            model_name="qwen3.8",
            timeout_config=TimeoutConfig(
                first_token_timeout=1,
                token_interval_timeout=1,
            ),
        )
        llm._client = MagicMock()
        llm._model_handle = BlockingModelHandle()

        heartbeat_completed = asyncio.Event()

        async def heartbeat() -> None:
            await asyncio.sleep(0)
            heartbeat_completed.set()

        heartbeat_task = asyncio.create_task(heartbeat())

        async def release_later() -> None:
            await asyncio.wait_for(heartbeat_completed.wait(), timeout=0.5)
            release_stream.set()

        release_task = asyncio.create_task(release_later())
        try:
            chunks = [
                chunk
                async for chunk in llm.stream_chat(
                    messages=[{"role": "user", "content": "hello"}]
                )
            ]
        finally:
            release_stream.set()
            await release_task

        await heartbeat_task
        assert heartbeat_completed.is_set()
        assert [chunk.delta for chunk in chunks] == ["ok"]

    @pytest.mark.asyncio
    @pytest.mark.parametrize("separate_usage_chunk", [False, True])
    async def test_stream_chat_requests_and_surfaces_usage(
        self, separate_usage_chunk: bool
    ) -> None:
        captured_generate_config: dict[str, object] = {}
        usage = {
            "prompt_tokens": 100,
            "completion_tokens": 5,
            "total_tokens": 105,
            "prompt_tokens_details": {"cached_tokens": 60},
        }

        class ModelHandle:
            async def chat(self, **kwargs: object):
                generate_config = kwargs["generate_config"]
                assert isinstance(generate_config, dict)
                captured_generate_config.update(generate_config)

                async def generate():
                    yield {
                        "choices": [
                            {
                                "delta": {"content": ""},
                                "finish_reason": "stop",
                            }
                        ],
                        "usage": usage,
                    }
                    if separate_usage_chunk:
                        yield {
                            "choices": [],
                            "usage": usage,
                        }

                return generate()

        llm = XinferenceLLM(model_name="qwen3.8")
        llm._client = MagicMock()
        llm._model_handle = ModelHandle()

        with patch(
            "xagent.core.model.chat.basic.xinference.add_token_usage"
        ) as add_usage:
            chunks = [
                chunk
                async for chunk in llm.stream_chat(
                    messages=[{"role": "user", "content": "hello"}]
                )
            ]

        assert captured_generate_config["stream_options"] == {"include_usage": True}
        assert [chunk.type for chunk in chunks] == [ChunkType.END, ChunkType.USAGE]
        assert chunks[-1].usage == usage
        add_usage.assert_called_once_with(
            input_tokens=100,
            output_tokens=5,
            model="qwen3.8",
            model_id="",
            call_type="stream_chat",
            cached_input_tokens=60,
        )

    @pytest.mark.asyncio
    async def test_stream_chat_enforces_timeout_while_waiting_for_first_token(
        self,
    ) -> None:
        release_stream = asyncio.Event()

        class BlockingModelHandle:
            async def chat(self, **kwargs: object):
                async def generate():
                    await release_stream.wait()
                    yield {
                        "choices": [
                            {
                                "delta": {"content": "late"},
                                "finish_reason": None,
                            }
                        ]
                    }

                return generate()

        llm = XinferenceLLM(
            model_name="qwen3.8",
            timeout_config=TimeoutConfig(
                first_token_timeout=0.02,
                token_interval_timeout=0.02,
            ),
        )
        llm._client = MagicMock()
        llm._model_handle = BlockingModelHandle()

        try:
            with pytest.raises(LLMTimeoutError, match="First token timeout"):
                async for _ in llm.stream_chat(
                    messages=[{"role": "user", "content": "hello"}]
                ):
                    pass
        finally:
            release_stream.set()

    @pytest.mark.asyncio
    async def test_chat_timeout_is_retryable(self) -> None:
        class BlockingModelHandle:
            async def chat(self, **kwargs: object):
                await asyncio.Event().wait()

        llm = XinferenceLLM(model_name="qwen3.8", timeout=0.02)
        llm._client = MagicMock()
        llm._model_handle = BlockingModelHandle()

        with pytest.raises(LLMTimeoutError, match="Xinference chat timeout") as exc:
            await llm.chat(messages=[{"role": "user", "content": "hello"}])

        assert retry_on(exc.value)

    @pytest.mark.asyncio
    @pytest.mark.parametrize("blocking_phase", ["get_model", "chat"])
    async def test_stream_first_token_timeout_bounds_request_initiation(
        self, blocking_phase: str
    ) -> None:
        class BlockingClient:
            async def get_chat_model(self, model_uid: str):
                await asyncio.Event().wait()

        class BlockingModelHandle:
            async def chat(self, **kwargs: object):
                await asyncio.Event().wait()

        llm = XinferenceLLM(
            model_name="qwen3.8",
            timeout_config=TimeoutConfig(
                first_token_timeout=0.02,
                token_interval_timeout=1,
            ),
        )
        if blocking_phase == "get_model":
            llm._client = BlockingClient()
        else:
            llm._client = MagicMock()
            llm._model_handle = BlockingModelHandle()

        with pytest.raises(LLMTimeoutError, match="First token timeout"):
            async for _ in llm.stream_chat(
                messages=[{"role": "user", "content": "hello"}]
            ):
                pass

    @pytest.mark.asyncio
    async def test_stream_chat_enforces_token_interval_timeout_and_closes_stream(
        self,
    ) -> None:
        class TrackingStream:
            def __init__(self) -> None:
                self._first = True
                self.closed = False

            def __aiter__(self):
                return self

            async def __anext__(self):
                if self._first:
                    self._first = False
                    return {
                        "choices": [
                            {
                                "delta": {"content": "first"},
                                "finish_reason": None,
                            }
                        ]
                    }
                await asyncio.Event().wait()

            async def aclose(self) -> None:
                self.closed = True

        stream = TrackingStream()

        class ModelHandle:
            async def chat(self, **kwargs: object):
                return stream

        llm = XinferenceLLM(
            model_name="qwen3.8",
            timeout_config=TimeoutConfig(
                first_token_timeout=1,
                token_interval_timeout=0.02,
            ),
        )
        llm._client = MagicMock()
        llm._model_handle = ModelHandle()

        chunks = []
        with pytest.raises(LLMTimeoutError, match="Token interval timeout"):
            async for chunk in llm.stream_chat(
                messages=[{"role": "user", "content": "hello"}]
            ):
                chunks.append(chunk)

        assert [chunk.delta for chunk in chunks] == ["first"]
        assert stream.closed

    @pytest.mark.asyncio
    async def test_stream_chat_keeps_first_deadline_until_payload(self) -> None:
        class DelayedPayloadStream:
            def __init__(self) -> None:
                self._index = 0

            def __aiter__(self):
                return self

            async def __anext__(self):
                self._index += 1
                if self._index == 1:
                    return {
                        "choices": [
                            {
                                "delta": {"role": "assistant", "content": ""},
                                "finish_reason": None,
                            }
                        ]
                    }
                if self._index == 2:
                    await asyncio.sleep(0.05)
                    return {
                        "choices": [
                            {
                                "delta": {"content": "first payload"},
                                "finish_reason": None,
                            }
                        ]
                    }
                raise StopAsyncIteration

            async def aclose(self) -> None:
                pass

        class ModelHandle:
            async def chat(self, **kwargs: object):
                return DelayedPayloadStream()

        llm = XinferenceLLM(
            model_name="qwen3.8",
            timeout_config=TimeoutConfig(
                first_token_timeout=0.2,
                token_interval_timeout=0.02,
            ),
        )
        llm._client = MagicMock()
        llm._model_handle = ModelHandle()

        chunks = [
            chunk
            async for chunk in llm.stream_chat(
                messages=[{"role": "user", "content": "hello"}]
            )
        ]

        assert [chunk.delta for chunk in chunks] == ["first payload"]

    @pytest.mark.asyncio
    async def test_stream_chat_preserves_transport_timeout_cause(self) -> None:
        transport_error = asyncio.TimeoutError("SDK read timed out")

        class TransportTimeoutStream:
            def __aiter__(self):
                return self

            async def __anext__(self):
                raise transport_error

            async def aclose(self) -> None:
                pass

        class ModelHandle:
            async def chat(self, **kwargs: object):
                return TransportTimeoutStream()

        llm = XinferenceLLM(
            model_name="qwen3.8",
            timeout_config=TimeoutConfig(
                first_token_timeout=1,
                token_interval_timeout=1,
            ),
        )
        llm._client = MagicMock()
        llm._model_handle = ModelHandle()

        with pytest.raises(LLMTimeoutError, match="transport timeout") as exc:
            async for _ in llm.stream_chat(
                messages=[{"role": "user", "content": "hello"}]
            ):
                pass

        assert exc.value.__cause__ is transport_error

    @pytest.mark.asyncio
    async def test_stream_chat_outer_close_closes_inner_iterator(self) -> None:
        class TrackingStream:
            def __init__(self) -> None:
                self.closed = 0

            def __aiter__(self):
                return self

            async def __anext__(self):
                return {
                    "choices": [
                        {
                            "delta": {"content": "first"},
                            "finish_reason": None,
                        }
                    ]
                }

            async def aclose(self) -> None:
                self.closed += 1

        stream = TrackingStream()

        class ModelHandle:
            async def chat(self, **kwargs: object):
                return stream

        llm = XinferenceLLM(model_name="qwen3.8")
        llm._client = MagicMock()
        llm._model_handle = ModelHandle()
        outer = llm.stream_chat(messages=[{"role": "user", "content": "hello"}])

        chunk = await outer.__anext__()
        await outer.aclose()

        assert chunk.delta == "first"
        assert stream.closed == 1

    @pytest.mark.asyncio
    @pytest.mark.parametrize(
        ("api_key", "expected_authorization"),
        [(None, None), ("secret", "Bearer secret")],
    )
    async def test_async_client_skips_sync_auth_probe(
        self, api_key: str | None, expected_authorization: str | None
    ) -> None:
        with patch(
            "requests.Session.get",
            side_effect=AssertionError("synchronous auth probe must not run"),
        ):
            client = _create_async_client("https://xinference.example.com", api_key)

        try:
            assert client._headers.get("Authorization") == expected_authorization
        finally:
            await client.close()

    def test_parse_stream_chunk_accumulates_tool_arguments_by_index(self) -> None:
        llm = XinferenceLLM(model_name="qwen3.5")
        accumulated_tool_calls: dict[str, dict] = {}

        first_chunk = llm._parse_stream_chunk(
            {
                "choices": [
                    {
                        "delta": {
                            "tool_calls": [
                                {
                                    "index": 0,
                                    "id": "call_1",
                                    "type": "function",
                                    "function": {
                                        "name": "auto_decision",
                                        "arguments": "{",
                                    },
                                }
                            ]
                        },
                        "finish_reason": None,
                    }
                ]
            },
            accumulated_tool_calls,
        )
        second_chunk = llm._parse_stream_chunk(
            {
                "choices": [
                    {
                        "delta": {
                            "tool_calls": [
                                {
                                    "index": 0,
                                    "function": {
                                        "arguments": '"action":"react"}',
                                    },
                                }
                            ]
                        },
                        "finish_reason": None,
                    }
                ]
            },
            accumulated_tool_calls,
        )
        final_chunk = llm._parse_stream_chunk(
            {
                "choices": [
                    {
                        "delta": {"content": ""},
                        "finish_reason": "tool_calls",
                    }
                ]
            },
            accumulated_tool_calls,
        )

        assert first_chunk is not None
        assert first_chunk.type == ChunkType.TOOL_CALL
        assert second_chunk is not None
        assert second_chunk.tool_calls[0]["function"]["arguments"] == (
            '{"action":"react"}'
        )
        assert final_chunk is not None
        assert final_chunk.finish_reason == "tool_calls"
        assert final_chunk.tool_calls == [
            {
                "index": 0,
                "id": "call_1",
                "type": "function",
                "function": {
                    "name": "auto_decision",
                    "arguments": '{"action":"react"}',
                },
            }
        ]

    def test_parse_stream_chunk_does_not_merge_mismatched_tool_call_index(
        self,
    ) -> None:
        llm = XinferenceLLM(model_name="qwen3.5")
        accumulated_tool_calls: dict[str, dict] = {}

        llm._parse_stream_chunk(
            {
                "choices": [
                    {
                        "delta": {
                            "tool_calls": [
                                {
                                    "index": 0,
                                    "id": "call_1",
                                    "type": "function",
                                    "function": {
                                        "name": "auto_decision",
                                        "arguments": "{",
                                    },
                                }
                            ]
                        },
                        "finish_reason": None,
                    }
                ]
            },
            accumulated_tool_calls,
        )

        chunk = llm._parse_stream_chunk(
            {
                "choices": [
                    {
                        "delta": {
                            "tool_calls": [
                                {
                                    "index": 1,
                                    "function": {
                                        "arguments": '"action":"react"}',
                                    },
                                }
                            ]
                        },
                        "finish_reason": None,
                    }
                ]
            },
            accumulated_tool_calls,
        )

        assert chunk is not None
        assert chunk.tool_calls == [
            {
                "index": 0,
                "id": "call_1",
                "type": "function",
                "function": {
                    "name": "auto_decision",
                    "arguments": "{",
                },
            }
        ]

    def test_parse_stream_chunk_handles_null_tool_call_function(self) -> None:
        llm = XinferenceLLM(model_name="qwen3.5")
        accumulated_tool_calls: dict[str, dict] = {}

        chunk = llm._parse_stream_chunk(
            {
                "choices": [
                    {
                        "delta": {
                            "tool_calls": [
                                {
                                    "index": 0,
                                    "id": "call_1",
                                    "type": "function",
                                    "function": None,
                                }
                            ]
                        },
                        "finish_reason": None,
                    }
                ]
            },
            accumulated_tool_calls,
        )

        assert chunk is not None
        assert chunk.tool_calls == [
            {
                "index": 0,
                "id": "call_1",
                "type": "function",
                "function": {
                    "name": "",
                    "arguments": "",
                },
            }
        ]

    @pytest.mark.asyncio
    @patch("xagent.core.model.chat.basic.xinference._create_async_client")
    async def test_list_available_models_handles_dict_response(
        self, mock_client_factory: MagicMock
    ) -> None:
        mock_client = MagicMock()
        mock_client.list_models = AsyncMock(
            return_value={
                "qwen-chat-uid": {
                    "model_name": "Qwen3-8B-Instruct",
                    "model_type": "LLM",
                    "model_ability": ["chat", "vision", "tool_calling"],
                    "model_description": "Qwen chat model",
                },
                "whisper-uid": {
                    "model_name": "whisper-large-v3",
                    "model_type": "audio",
                    "model_ability": ["audio2text"],
                    "model_description": "ASR model",
                },
            }
        )
        mock_client.close = AsyncMock()
        mock_client_factory.return_value = mock_client

        models = await XinferenceLLM.list_available_models(
            base_url="http://localhost:9997", api_key="test-key"
        )

        assert len(models) == 2
        assert models[0] == {
            "id": "Qwen3-8B-Instruct",
            "model_uid": "qwen-chat-uid",
            "model_type": "LLM",
            "model_ability": ["chat", "vision", "tool_calling"],
            "abilities": ["chat", "vision", "tool_calling"],
            "description": "Qwen chat model",
        }
        assert models[1] == {
            "id": "whisper-large-v3",
            "model_uid": "whisper-uid",
            "model_type": "audio",
            "model_ability": ["asr"],
            "abilities": ["asr"],
            "description": "ASR model",
        }
        mock_client.close.assert_awaited_once()

    @pytest.mark.asyncio
    @patch("xagent.core.model.chat.basic.xinference._create_async_client")
    async def test_list_available_models_preserves_embedding_ability(
        self, mock_client_factory: MagicMock
    ) -> None:
        mock_client = MagicMock()
        mock_client.list_models = AsyncMock(
            return_value={
                "embedding-uid": {
                    "model_name": "Qwen3-Embedding-8B",
                    "model_type": "embedding",
                    "model_ability": ["embedding"],
                    "model_description": "Embedding model",
                }
            }
        )
        mock_client.close = AsyncMock()
        mock_client_factory.return_value = mock_client

        models = await XinferenceLLM.list_available_models(
            base_url="http://localhost:9997", api_key="test-key"
        )

        assert models == [
            {
                "id": "Qwen3-Embedding-8B",
                "model_uid": "embedding-uid",
                "model_type": "embedding",
                "model_ability": ["embedding"],
                "abilities": ["embedding"],
                "description": "Embedding model",
            }
        ]

    @pytest.mark.asyncio
    @patch("xagent.core.model.chat.basic.xinference._create_async_client")
    async def test_list_available_models_handles_legacy_list_response(
        self, mock_client_factory: MagicMock
    ) -> None:
        mock_client = MagicMock()
        mock_client.list_models = AsyncMock(
            return_value=[
                {
                    "id": "legacy-chat-uid",
                    "model_name": "legacy-chat",
                    "model_type": "LLM",
                    "model_ability": ["chat"],
                    "model_description": "Legacy chat model",
                }
            ]
        )
        mock_client.close = AsyncMock()
        mock_client_factory.return_value = mock_client

        models = await XinferenceLLM.list_available_models(
            base_url="http://localhost:9997", api_key="test-key"
        )

        assert models == [
            {
                "id": "legacy-chat",
                "model_uid": "legacy-chat-uid",
                "model_type": "LLM",
                "model_ability": ["chat"],
                "abilities": ["chat"],
                "description": "Legacy chat model",
            }
        ]

    @pytest.mark.asyncio
    @patch("xagent.core.model.chat.basic.xinference._create_async_client")
    async def test_list_available_models_remains_responsive_and_closes_client(
        self, mock_client_factory: MagicMock
    ) -> None:
        discovery_started = asyncio.Event()
        release_discovery = asyncio.Event()

        async def list_models():
            discovery_started.set()
            await release_discovery.wait()
            return {}

        mock_client = MagicMock()
        mock_client.list_models = list_models
        mock_client.close = AsyncMock()
        mock_client_factory.return_value = mock_client

        discovery_task = asyncio.create_task(
            XinferenceLLM.list_available_models("http://localhost:9997")
        )
        try:
            # These guards bound a hang, not model-discovery latency.
            await asyncio.wait_for(discovery_started.wait(), timeout=30)
            heartbeat = asyncio.create_task(asyncio.sleep(0))
            await asyncio.wait_for(heartbeat, timeout=30)
            assert not discovery_task.done()
        finally:
            release_discovery.set()
            models = await asyncio.wait_for(discovery_task, timeout=30)

        assert models == []
        mock_client.close.assert_awaited_once()

    @pytest.mark.asyncio
    @patch(
        "xagent.core.model.chat.basic.xinference.asyncio.sleep", new_callable=AsyncMock
    )
    @patch("xagent.core.model.chat.basic.xinference._create_async_client")
    async def test_list_available_models_closes_client_after_errors(
        self, mock_client_factory: MagicMock, _mock_sleep: AsyncMock
    ) -> None:
        mock_client = MagicMock()
        mock_client.list_models = AsyncMock(side_effect=RuntimeError("boom"))
        mock_client.close = AsyncMock()
        mock_client_factory.return_value = mock_client

        with pytest.raises(RuntimeError, match="Cannot connect to Xinference"):
            await XinferenceLLM.list_available_models("http://localhost:9997")

        assert mock_client_factory.call_count == 3
        assert mock_client.close.await_count == 3

    @pytest.mark.asyncio
    @patch(
        "xagent.core.model.chat.basic.xinference._MODEL_DISCOVERY_TIMEOUT_SECONDS",
        0.01,
    )
    @patch(
        "xagent.core.model.chat.basic.xinference.asyncio.sleep", new_callable=AsyncMock
    )
    @patch("xagent.core.model.chat.basic.xinference._create_async_client")
    async def test_list_available_models_bounds_and_cleans_up_slow_discovery(
        self, mock_client_factory: MagicMock, _mock_sleep: AsyncMock
    ) -> None:
        async def list_models():
            await asyncio.Event().wait()

        mock_client = MagicMock()
        mock_client.list_models = list_models
        mock_client.close = AsyncMock()
        mock_client_factory.return_value = mock_client

        with pytest.raises(RuntimeError, match="Cannot connect to Xinference") as exc:
            await XinferenceLLM.list_available_models("http://localhost:9997")

        assert isinstance(exc.value.__cause__, LLMTimeoutError)
        assert mock_client_factory.call_count == 3
        assert mock_client.close.await_count == 3


class TestProcessChatResponse:
    """Tests for ``XinferenceLLM._process_chat_response``.

    Reasoning-capable models served via Xinference (``qwen3-thinking``,
    ``deepseek-r1``, etc.) can return a response whose ``content`` is empty
    while ``reasoning_content`` carries the partial answer — most commonly
    when ``max_tokens`` truncates the generation before the final answer is
    produced. The adapter must surface those responses as text instead of
    raising ``Invalid Xinference response``.
    """

    def _make_llm(self) -> XinferenceLLM:
        return XinferenceLLM(model_name="qwen3-thinking")

    def test_plain_text_response_is_returned_as_text(self) -> None:
        llm = self._make_llm()
        result = llm._process_chat_response(
            {
                "choices": [
                    {
                        "message": {
                            "role": "assistant",
                            "content": "Hello there",
                        },
                        "finish_reason": "stop",
                    }
                ],
                "usage": {"prompt_tokens": 1, "completion_tokens": 2},
            }
        )

        assert result["type"] == "text"
        assert result["content"] == "Hello there"
        assert "reasoning_content" not in result

    def test_text_response_with_reasoning_content_attaches_reasoning(self) -> None:
        llm = self._make_llm()
        result = llm._process_chat_response(
            {
                "choices": [
                    {
                        "message": {
                            "role": "assistant",
                            "content": "42",
                            "reasoning_content": "Need to compute 6 * 7",
                        },
                        "finish_reason": "stop",
                    }
                ]
            }
        )

        assert result["type"] == "text"
        assert result["content"] == "42"
        assert result["reasoning_content"] == "Need to compute 6 * 7"
        assert result["reasoning"] == "Need to compute 6 * 7"

    def test_empty_content_with_reasoning_falls_back_to_reasoning(self) -> None:
        """Reproduces the bug where ``max_tokens`` truncates a reasoning
        model: ``content=""`` but ``reasoning_content`` is populated and
        ``finish_reason="length"``. The adapter must NOT raise; it should
        treat the reasoning text as the response content.
        """
        llm = self._make_llm()
        result = llm._process_chat_response(
            {
                "id": "chat-1",
                "object": "chat.completion",
                "model": "qwen3-thinking",
                "choices": [
                    {
                        "index": 0,
                        "message": {
                            "role": "assistant",
                            "content": "",
                            "reasoning_content": "Here",
                        },
                        "finish_reason": "length",
                    }
                ],
                "usage": {"prompt_tokens": 11, "completion_tokens": 1},
            }
        )

        assert result["type"] == "text"
        assert result["content"] == "Here"
        assert result["reasoning_content"] == "Here"
        assert result["reasoning"] == "Here"

    def test_tool_call_response_attaches_reasoning_when_present(self) -> None:
        llm = self._make_llm()
        tool_calls = [
            {
                "id": "call_1",
                "type": "function",
                "function": {"name": "do_thing", "arguments": "{}"},
            }
        ]
        result = llm._process_chat_response(
            {
                "choices": [
                    {
                        "message": {
                            "role": "assistant",
                            "content": "",
                            "tool_calls": tool_calls,
                            "reasoning_content": "I should call do_thing",
                        },
                        "finish_reason": "tool_calls",
                    }
                ]
            }
        )

        assert result["type"] == "tool_call"
        assert result["tool_calls"] == tool_calls
        assert result["reasoning_content"] == "I should call do_thing"

    def test_empty_response_without_reasoning_still_raises(self) -> None:
        """If the response has neither ``content``, ``tool_calls`` nor
        ``reasoning_content``, the adapter must keep raising so callers
        can surface the underlying provider issue.
        """
        llm = self._make_llm()

        with pytest.raises(RuntimeError, match="Invalid Xinference response"):
            llm._process_chat_response(
                {
                    "choices": [
                        {
                            "message": {"role": "assistant", "content": ""},
                            "finish_reason": "stop",
                        }
                    ]
                }
            )

    def test_finish_reason_stop_with_only_reasoning_still_raises(self) -> None:
        """The reasoning-content fallback is scoped to truncated responses
        (``finish_reason="length"``) only.

        If a provider returns ``finish_reason="stop"`` with empty content
        and a populated reasoning trace, the model is claiming to be done
        without producing a final answer -- that is a real model failure
        and the adapter must surface it instead of silently promoting the
        scratchpad to the assistant message.
        """
        llm = self._make_llm()

        with pytest.raises(RuntimeError, match="Invalid Xinference response"):
            llm._process_chat_response(
                {
                    "choices": [
                        {
                            "message": {
                                "role": "assistant",
                                "content": "",
                                "reasoning_content": "I should answer the user.",
                            },
                            "finish_reason": "stop",
                        }
                    ]
                }
            )

    def test_whitespace_only_reasoning_with_length_finish_still_raises(
        self,
    ) -> None:
        """A whitespace-only reasoning trace must NOT be treated as a
        usable answer even when ``finish_reason="length"``.

        Otherwise a provider returning ``reasoning_content="   "`` would
        surface a blank string as the assistant message.
        """
        llm = self._make_llm()

        with pytest.raises(RuntimeError, match="Invalid Xinference response"):
            llm._process_chat_response(
                {
                    "choices": [
                        {
                            "message": {
                                "role": "assistant",
                                "content": "",
                                "reasoning_content": "   \n  ",
                            },
                            "finish_reason": "length",
                        }
                    ]
                }
            )


class TestXinferencePromptCacheUsage:
    """Dict-shaped usage payloads surface cached input tokens."""

    def test_process_chat_response_records_cached_tokens(self) -> None:
        from xagent.core.model.chat.token_context import TokenContextManager

        llm = XinferenceLLM(model_name="qwen3-thinking")

        with TokenContextManager() as manager:
            llm._process_chat_response(
                {
                    "choices": [
                        {
                            "message": {"role": "assistant", "content": "ok"},
                            "finish_reason": "stop",
                        }
                    ],
                    "usage": {
                        "prompt_tokens": 100,
                        "completion_tokens": 5,
                        "prompt_tokens_details": {"cached_tokens": 60},
                    },
                }
            )
            usage = manager.get_usage()

        assert usage.input_tokens == 100
        inp = next(d for d in usage.details if d["type"] == "input")
        assert inp["cached_tokens"] == 60