Skip to content

Build and run agents you can see, understand and trust.

60.9 KB
Raw Blame
# -*- coding: utf-8 -*-
# pylint: disable=protected-access
"""Unit tests for OpenAIResponseModel with mocked API responses.

Tests cover both non-streaming and streaming modes.
OpenAI Responses API uses event-based streaming with response.completed.
"""
import asyncio
from typing import Any, cast
import unittest
from unittest import IsolatedAsyncioTestCase
from unittest.mock import AsyncMock, MagicMock

import httpx
from openai import APIError, AsyncOpenAI
from openai.types.responses import Response, ResponseOutputMessage
from pydantic import BaseModel

from utils import AnyString, AnyValue

from agentscope.agent import Agent, ModelConfig
from agentscope.message import (
    AssistantMsg,
    Base64Source,
    DataBlock,
    Msg,
    TextBlock,
    ToolCallBlock,
    ToolResultBlock,
    ToolResultState,
    ThinkingBlock,
)
from agentscope.model import ChatResponse, ChatUsage, OpenAIResponseModel
from agentscope.credential import OpenAICredential
from agentscope.tool import ToolChoice

A = AnyString()


class _MockReasoningSummary(BaseModel):
    """Mock an OpenAI reasoning summary item."""

    text: str
    type: str = "summary_text"


class _MockReasoningItem(BaseModel):
    """Mock an OpenAI reasoning output item."""

    id: str
    summary: list[_MockReasoningSummary]
    type: str = "reasoning"
    content: list[dict[str, Any]] | None = None
    encrypted_content: str | None = None
    status: str | None = None


# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------


def _make_model(stream: bool = False) -> Any:
    return OpenAIResponseModel(
        credential=OpenAICredential(api_key="test"),
        model="o4-mini",
        stream=stream,
        context_size=200_000,
    )


def _mock_completion(
    text: Any = None,
    function_calls: Any = None,
    reasoning_summary: Any = None,
    reasoning_id: str = "rs_test123",
    response_id: str = "resp-openai-1",
    reasoning_output_item: Any = None,
) -> MagicMock:
    """Build a mock non-streaming Responses API response."""
    output = []

    if reasoning_output_item is not None:
        output.append(reasoning_output_item)

    elif reasoning_summary is not None:
        summary_texts = (
            reasoning_summary
            if isinstance(reasoning_summary, list)
            else [reasoning_summary]
            if reasoning_summary is not None
            else []
        )
        reasoning_item = _MockReasoningItem(
            id=reasoning_id,
            summary=[
                _MockReasoningSummary(text=summary_text)
                for summary_text in summary_texts
            ],
        )
        output.append(reasoning_item)

    if text:
        msg_item = MagicMock()
        msg_item.type = "message"
        part = MagicMock()
        part.type = "output_text"
        part.text = text
        msg_item.content = [part]
        output.append(msg_item)

    if function_calls:
        for fc in function_calls:
            fc_item = MagicMock()
            fc_item.type = "function_call"
            fc_item.id = fc["id"]
            fc_item.call_id = fc["call_id"]
            fc_item.name = fc["name"]
            fc_item.arguments = fc["arguments"]
            output.append(fc_item)

    resp = MagicMock()
    resp.id = response_id
    resp.output = output
    resp.usage = MagicMock()
    resp.usage.input_tokens = 10
    resp.usage.output_tokens = 5
    resp.usage.input_tokens_details = None
    return resp


def _make_event(event_type: str, **kwargs: Any) -> MagicMock:
    """Build a mock Responses API streaming event."""
    if event_type in ("response.output_text.delta", "response.refusal.delta"):
        kwargs.setdefault("item_id", "msg_test123")
        kwargs.setdefault("content_index", 0)
    event = MagicMock()
    event.type = event_type
    for key, val in kwargs.items():
        setattr(event, key, val)
    # Default: no response attribute
    if "response" not in kwargs:
        event.response = None
    return event


def _mock_message(
    item_id: str,
    parts: list[tuple[str, str]],
) -> ResponseOutputMessage:
    """Build SDK message parts for text/refusal parser comparisons."""
    return ResponseOutputMessage.model_validate(
        {
            "id": item_id,
            "type": "message",
            "role": "assistant",
            "status": "completed",
            "content": [
                {"type": kind, "text": text, "annotations": []}
                if kind == "output_text"
                else {"type": kind, "refusal": text}
                for kind, text in parts
            ],
        },
    )


class _MockAsyncEventStream:
    """Mock async iterator over Response events."""

    def __init__(self, events: list) -> None:
        self._events = events
        self._index = 0
        self.exited = False
        self.response = MagicMock()

    async def __aenter__(self) -> "_MockAsyncEventStream":
        return self

    async def __aexit__(self, *args: Any) -> None:
        self.exited = True

    def __aiter__(self) -> "_MockAsyncEventStream":
        return self

    async def __anext__(self) -> Any:
        if self._index >= len(self._events):
            raise StopAsyncIteration
        event = self._events[self._index]
        self._index += 1
        return event


# ---------------------------------------------------------------------------
# Non-streaming tests
# ---------------------------------------------------------------------------


class TestOpenAIResponseNonStream(IsolatedAsyncioTestCase):
    """Tests for OpenAIResponseModel in non-streaming mode."""

    def setUp(self) -> None:
        self.model = _make_model(stream=False)
        self.mock_client = MagicMock()
        self.model.client = self.mock_client

    async def test_thinking_enable_nullable(self) -> None:
        """False requests no reasoning, even with a stored effort value."""
        self.assertIsNone(OpenAIResponseModel.Parameters().thinking_enable)
        mock_create = AsyncMock(return_value=_mock_completion(text="Hi"))
        self.mock_client.responses.create = mock_create

        for params, disables_thinking in (
            ({}, False),
            ({"thinking_enable": None}, False),
            ({"thinking_enable": False}, True),
            ({"thinking_enable": True}, False),
        ):
            for effort in (None, "none", "high"):
                with self.subTest(params=params, effort=effort):
                    self.model.parameters = OpenAIResponseModel.Parameters(
                        **params,
                        reasoning_effort=effort,
                    )
                    await self.model([])

                    expected = {
                        "model": "o4-mini",
                        "input": [],
                        "stream": False,
                    }
                    expected_effort = "none" if disables_thinking else effort
                    if expected_effort is not None:
                        expected["reasoning"] = {"effort": expected_effort}
                    self.assertEqual(mock_create.call_args.kwargs, expected)

    async def test_text_response(self) -> None:
        """Non-stream text response returns a single ChatResponse."""
        mock_create = AsyncMock(
            return_value=_mock_completion(text="Hello!"),
        )
        self.mock_client.responses.create = mock_create

        result = await self.model([])

        self.assertEqual(
            (result.is_last, result.content),
            (
                True,
                [TextBlock.model_construct(id=A, created_at=A, text="Hello!")],
            ),
        )
        self.assertEqual(result.id, "resp-openai-1")

    async def test_tool_call_response(
        self,
    ) -> None:
        """Parsing a tool-call response stores call_id as ToolCallBlock.id."""
        mock_create = AsyncMock(
            return_value=_mock_completion(
                function_calls=[
                    {
                        "id": "fc_abc",
                        "call_id": "call-1",
                        "name": "get_weather",
                        "arguments": '{"city":"BJ"}',
                    },
                ],
            ),
        )
        self.mock_client.responses.create = mock_create

        result = await self.model([])

        self.assertEqual(
            (result.is_last, result.content),
            (
                True,
                [
                    ToolCallBlock.model_construct(
                        id="call-1",
                        created_at=A,
                        name="get_weather",
                        input='{"city":"BJ"}',
                    ),
                ],
            ),
        )

    async def test_native_multimodal_tool_result_request(self) -> None:
        """The model sends native multimodal function-call output."""
        mock_create = AsyncMock(
            return_value=_mock_completion(text="verified"),
        )
        self.mock_client.responses.create = mock_create
        messages = [
            AssistantMsg(
                name="assistant",
                content=[
                    ToolCallBlock(
                        id="call-image",
                        name="inspect_image",
                        input="{}",
                    ),
                    ToolResultBlock(
                        id="call-image",
                        name="inspect_image",
                        output=[
                            TextBlock(text="marker-before"),
                            DataBlock(
                                source=Base64Source(
                                    data="aW1hZ2U=",
                                    media_type="image/png",
                                ),
                            ),
                            TextBlock(text="marker-after"),
                        ],
                        state=ToolResultState.SUCCESS,
                    ),
                ],
            ),
        ]

        await self.model(messages)

        self.assertListEqual(
            mock_create.await_args.kwargs["input"],
            [
                {
                    "type": "function_call",
                    "call_id": "call-image",
                    "name": "inspect_image",
                    "arguments": "{}",
                },
                {
                    "type": "function_call_output",
                    "call_id": "call-image",
                    "output": [
                        {"type": "input_text", "text": "marker-before"},
                        {
                            "type": "input_image",
                            "image_url": "data:image/png;base64,aW1hZ2U=",
                        },
                        {"type": "input_text", "text": "marker-after"},
                    ],
                },
            ],
        )

    async def test_reasoning_response(
        self,
    ) -> None:
        """Non-stream reasoning summary plus text returns both block types."""
        mock_create = AsyncMock(
            return_value=_mock_completion(
                reasoning_summary="Thinking step...",
                text="Answer",
                reasoning_id="rs_abc999",
            ),
        )
        self.mock_client.responses.create = mock_create

        result = await self.model([])

        self.assertEqual(
            (result.is_last, result.content),
            (
                True,
                [
                    ThinkingBlock.model_construct(
                        id=A,
                        created_at=A,
                        thinking="Thinking step...",
                        reasoning_item_id="rs_abc999",
                        reasoning_item_raw={
                            "id": "rs_abc999",
                            "summary": [
                                {
                                    "text": "Thinking step...",
                                    "type": "summary_text",
                                },
                            ],
                            "type": "reasoning",
                        },
                    ),
                    TextBlock.model_construct(
                        id=A,
                        created_at=A,
                        text="Answer",
                    ),
                ],
            ),
        )

    async def test_reasoning_raw_item_round_trip(self) -> None:
        """Encrypted reasoning metadata survives parsing and formatting."""
        reasoning_item_raw = {
            "id": "rs_encrypted",
            "summary": [
                {
                    "text": "Thinking step...",
                    "type": "summary_text",
                },
            ],
            "type": "reasoning",
            "content": [],
            "encrypted_content": "encrypted_payload",
            "status": "completed",
        }
        reasoning_item = _MockReasoningItem.model_validate(
            reasoning_item_raw,
        )
        mock_create = AsyncMock(
            return_value=_mock_completion(
                text="Answer",
                reasoning_output_item=reasoning_item,
            ),
        )
        self.mock_client.responses.create = mock_create

        result = await self.model([])

        self.assertEqual(
            (result.is_last, result.content),
            (
                True,
                [
                    ThinkingBlock.model_construct(
                        id=A,
                        created_at=A,
                        thinking="Thinking step...",
                        reasoning_item_id="rs_encrypted",
                        reasoning_item_raw=reasoning_item_raw,
                    ),
                    TextBlock.model_construct(
                        id=A,
                        created_at=A,
                        text="Answer",
                    ),
                ],
            ),
        )

        msg = AssistantMsg(
            name="assistant",
            content=result.content,
        )
        restored_msg = Msg.model_validate(msg.model_dump())
        self.assertEqual(restored_msg.content, result.content)

        formatted = await self.model.formatter.format(
            [restored_msg],
        )
        self.assertListEqual(
            formatted,
            [
                reasoning_item_raw,
                {
                    "role": "assistant",
                    "content": [
                        {
                            "type": "output_text",
                            "text": "Answer",
                        },
                    ],
                },
            ],
        )

    async def test_reasoning_content_with_optional_summary(self) -> None:
        """Raw reasoning remains visible and separate from its summary."""
        for summary in ([], [_MockReasoningSummary(text="Summary")]):
            with self.subTest(summary=summary):
                item = _MockReasoningItem(
                    id="rs_content",
                    summary=summary,
                    content=[
                        {
                            "type": "reasoning_text",
                            "text": "Raw reasoning",
                        },
                    ],
                )
                self.mock_client.responses.create = AsyncMock(
                    return_value=_mock_completion(
                        text="Answer",
                        reasoning_output_item=item,
                    ),
                )
                result = await self.model([])
                texts = (["Summary"] if summary else []) + ["Raw reasoning"]
                self.assertEqual(
                    (result.is_last, result.content),
                    (
                        True,
                        [
                            *[
                                ThinkingBlock.model_construct(
                                    id=A,
                                    created_at=A,
                                    thinking=text,
                                    reasoning_item_id=item.id,
                                    reasoning_item_raw=item.model_dump(
                                        exclude_none=True,
                                    ),
                                )
                                for text in texts
                            ],
                            TextBlock.model_construct(
                                id=A,
                                created_at=A,
                                text="Answer",
                            ),
                        ],
                    ),
                )

    async def test_reasoning_preserves_part_boundaries(self) -> None:
        """Each summary and reasoning content part remains separate."""
        item = _MockReasoningItem(
            id="rs_parts",
            summary=[
                _MockReasoningSummary(text="Summary one"),
                _MockReasoningSummary(text="Summary two"),
            ],
            content=[
                {"type": "reasoning_text", "text": "Reasoning one"},
                {"type": "reasoning_text", "text": "Reasoning two"},
            ],
        )
        self.mock_client.responses.create = AsyncMock(
            return_value=_mock_completion(
                reasoning_output_item=item,
            ),
        )

        result = await self.model([])
        raw = item.model_dump(exclude_none=True)

        self.assertEqual(
            (result.is_last, result.content),
            (
                True,
                [
                    ThinkingBlock.model_construct(
                        id=A,
                        created_at=A,
                        thinking=text,
                        reasoning_item_id=item.id,
                        reasoning_item_raw=raw,
                    )
                    for text in (
                        "Summary one",
                        "Summary two",
                        "Reasoning one",
                        "Reasoning two",
                    )
                ],
            ),
        )

    async def test_reasoning_raw_item_excludes_none_fields(self) -> None:
        """Optional null SDK fields are not stored for history replay."""
        reasoning_item = _MockReasoningItem.model_validate(
            {
                "id": "rs_without_nulls",
                "summary": [],
                "type": "reasoning",
                "encrypted_content": "encrypted_payload",
            },
        )
        mock_create = AsyncMock(
            return_value=_mock_completion(
                reasoning_output_item=reasoning_item,
            ),
        )
        self.mock_client.responses.create = mock_create

        result = await self.model([])

        self.assertEqual(
            result.content,
            [
                ThinkingBlock.model_construct(
                    id=A,
                    created_at=A,
                    thinking="",
                    reasoning_item_id="rs_without_nulls",
                    reasoning_item_raw={
                        "id": "rs_without_nulls",
                        "summary": [],
                        "type": "reasoning",
                        "encrypted_content": "encrypted_payload",
                    },
                ),
            ],
        )

    async def test_empty_reasoning_summary_response(
        self,
    ) -> None:
        """Non-stream empty reasoning summary still preserves its item id."""
        mock_create = AsyncMock(
            return_value=_mock_completion(
                reasoning_summary=[],
                text="Answer",
                reasoning_id="rs_empty",
            ),
        )
        self.mock_client.responses.create = mock_create

        result = await self.model([])

        self.assertEqual(
            (result.is_last, result.content),
            (
                True,
                [
                    ThinkingBlock.model_construct(
                        id=A,
                        created_at=A,
                        thinking="",
                        reasoning_item_id="rs_empty",
                        reasoning_item_raw={
                            "id": "rs_empty",
                            "summary": [],
                            "type": "reasoning",
                        },
                    ),
                    TextBlock.model_construct(
                        id=A,
                        created_at=A,
                        text="Answer",
                    ),
                ],
            ),
        )


class TestOpenAIResponseModelParameters(unittest.TestCase):
    """Tests for OpenAIResponseModel.Parameters."""

    def test_thinking_enable_stored_on_model(self) -> None:
        """thinking_enable is accessible through model.parameters."""
        model = OpenAIResponseModel(
            credential=OpenAICredential(api_key="test"),
            model="o4-mini",
            stream=False,
            context_size=200_000,
            parameters=OpenAIResponseModel.Parameters(thinking_enable=True),
        )
        self.assertTrue(model.parameters.thinking_enable)

    def test_reasoning_effort_stored_on_model(self) -> None:
        """reasoning_effort is accessible through model.parameters."""
        model = OpenAIResponseModel(
            credential=OpenAICredential(api_key="test"),
            model="o4-mini",
            stream=False,
            context_size=200_000,
            parameters=OpenAIResponseModel.Parameters(
                reasoning_effort="high",
            ),
        )
        self.assertEqual(model.parameters.reasoning_effort, "high")


# ---------------------------------------------------------------------------
# Streaming tests
# ---------------------------------------------------------------------------


class TestOpenAIResponseStream(IsolatedAsyncioTestCase):
    """Tests for OpenAIResponseModel in streaming mode."""

    def setUp(self) -> None:
        self.model = _make_model(stream=True)
        self.mock_client = MagicMock()
        self.model.client = self.mock_client

    async def test_stream_text_and_refusal_part_boundaries(self) -> None:
        """Mixed parts match non-streaming without repeating done text."""
        for output in (
            [_mock_message("msg_one", [("refusal", "Cannot help.")])],
            [
                _mock_message(
                    "msg_one",
                    [
                        ("output_text", "Before."),
                        ("refusal", "Cannot help."),
                        ("output_text", "After."),
                    ],
                ),
            ],
            [
                _mock_message("msg_one", [("output_text", "First.")]),
                _mock_message("msg_two", [("refusal", "Cannot help.")]),
            ],
            [
                _mock_message(
                    "msg_one",
                    [("output_text", "First."), ("output_text", "Second.")],
                ),
            ],
        ):
            with self.subTest(output=output):
                completed = _mock_completion()
                completed.output = output
                completed.usage.input_tokens_details = MagicMock(
                    cached_tokens=3,
                )
                parts = [
                    (item.id, index, part.type, part.text)
                    if part.type == "output_text"
                    else (item.id, index, part.type, part.refusal)
                    for item in output
                    for index, part in enumerate(item.content)
                ]
                expected = ChatResponse(
                    id=A,
                    created_at=A,
                    is_last=True,
                    content=[
                        TextBlock.model_construct(
                            id=A,
                            created_at=A,
                            text=text,
                        )
                        for _, _, _, text in parts
                    ],
                    usage=ChatUsage(
                        input_tokens=10,
                        output_tokens=5,
                        cache_input_tokens=3,
                        time=cast(float, AnyValue()),
                    ),
                )
                self.model.stream = False
                self.mock_client.responses.create = AsyncMock(
                    return_value=completed,
                )
                self.assertEqual(await self.model([]), expected)
                self.mock_client.responses.create.assert_awaited_once()

                events = [
                    _make_event(
                        f"response.{kind}.delta",
                        item_id=item_id,
                        content_index=index,
                        delta=text[:2] if fragment == 0 else text[2:],
                    )
                    for fragment in range(2)
                    for item_id, index, kind, text in parts
                ]
                events.extend(
                    _make_event(
                        f"response.{kind}.done",
                        item_id=item_id,
                        content_index=index,
                        **(
                            {"text": text}
                            if kind == "output_text"
                            else {"refusal": text}
                        ),
                    )
                    for item_id, index, kind, text in parts
                )
                events.append(
                    _make_event("response.completed", response=completed),
                )
                self.model.stream = True
                self.mock_client.responses.create = AsyncMock(
                    return_value=_MockAsyncEventStream(events),
                )
                responses = [r async for r in await self.model([])]
                self.assertEqual(
                    responses[:-1],
                    [
                        ChatResponse(
                            id=A,
                            created_at=A,
                            is_last=False,
                            content=[
                                TextBlock.model_construct(
                                    id=A,
                                    created_at=A,
                                    text=text[:2]
                                    if fragment == 0
                                    else text[2:],
                                ),
                            ],
                        )
                        for fragment in range(2)
                        for _, _, _, text in parts
                    ],
                )
                self.assertEqual(responses[-1], expected)
                self.mock_client.responses.create.assert_awaited_once()
                self.assertEqual(
                    AssistantMsg(
                        name="assistant",
                        content=responses[-1].content,
                    ).get_text_content(),
                    "\n".join(text for _, _, _, text in parts),
                )

    async def test_stream_errors_propagate(self) -> None:
        """Failed stream events raise APIError instead of a partial reply."""
        body = {"code": "server_error", "message": "Generation failed."}
        error = _make_event("error", message="Generation failed.")
        error.model_dump.return_value = body
        for failure in (
            error,
            _make_event("response.failed", response=MagicMock(error=error)),
        ):
            with self.subTest(event_type=failure.type):
                self.mock_client.responses.create = AsyncMock(
                    return_value=_MockAsyncEventStream(
                        [
                            _make_event(
                                "response.output_text.delta",
                                delta="Partial",
                            ),
                            failure,
                        ],
                    ),
                )
                gen = await self.model([])
                await anext(gen)
                with self.assertRaises(APIError) as ctx:
                    await anext(gen)
                self.assertDictEqual(
                    {
                        "message": ctx.exception.message,
                        "body": ctx.exception.body,
                    },
                    {"message": "Generation failed.", "body": body},
                )

    async def test_stream_text(self) -> None:
        """Stream text yields deltas then final with full content."""
        completed_resp = MagicMock()
        completed_resp.id = "resp-1"
        completed_resp.output = []
        completed_resp.usage = MagicMock()
        completed_resp.usage.input_tokens = 10
        completed_resp.usage.output_tokens = 5
        completed_resp.usage.input_tokens_details = None

        events = [
            _make_event(
                "response.output_text.delta",
                delta="Hello",
                response=MagicMock(id="resp-1"),
            ),
            _make_event(
                "response.output_text.delta",
                delta=" world",
            ),
            _make_event("response.completed", response=completed_resp),
        ]
        stream = _MockAsyncEventStream(events)
        mock_create = AsyncMock(return_value=stream)
        self.mock_client.responses.create = mock_create

        gen = await self.model([])
        responses = [r async for r in gen]

        self.assertTrue(stream.exited)

        self.assertListEqual(
            [(r.is_last, r.content) for r in responses],
            [
                (
                    False,
                    [
                        TextBlock.model_construct(
                            id=A,
                            created_at=A,
                            text="Hello",
                        ),
                    ],
                ),
                (
                    False,
                    [
                        TextBlock.model_construct(
                            id=A,
                            created_at=A,
                            text=" world",
                        ),
                    ],
                ),
                (
                    True,
                    [
                        TextBlock.model_construct(
                            id=A,
                            created_at=A,
                            text="Hello world",
                        ),
                    ],
                ),
            ],
        )

    async def test_stream_reasoning_and_text(
        self,
    ) -> None:
        """Stream reasoning and text deltas then final with
        reasoning_item_id."""
        reasoning_item_raw = {
            "id": "rs_123",
            "summary": [],
            "type": "reasoning",
        }
        reasoning_item = _MockReasoningItem.model_validate(
            reasoning_item_raw,
        )

        completed_resp = MagicMock()
        completed_resp.id = "resp-2"
        completed_resp.output = [reasoning_item]
        completed_resp.usage = MagicMock()
        completed_resp.usage.input_tokens = 10
        completed_resp.usage.output_tokens = 5
        completed_resp.usage.input_tokens_details = None

        events = [
            _make_event(
                "response.reasoning_summary_text.delta",
                delta="Thinking",
                item_id="rs_123",
                summary_index=0,
                response=MagicMock(id="resp-2"),
            ),
            _make_event(
                "response.output_text.delta",
                delta="Answer",
            ),
            _make_event("response.completed", response=completed_resp),
        ]
        mock_create = AsyncMock(
            return_value=_MockAsyncEventStream(events),
        )
        self.mock_client.responses.create = mock_create

        gen = await self.model([])
        responses = [r async for r in gen]

        self.assertListEqual(
            [(r.is_last, r.content) for r in responses],
            [
                (
                    False,
                    [
                        ThinkingBlock.model_construct(
                            id=A,
                            created_at=A,
                            thinking="Thinking",
                        ),
                    ],
                ),
                (
                    False,
                    [
                        TextBlock.model_construct(
                            id=A,
                            created_at=A,
                            text="Answer",
                        ),
                    ],
                ),
                # ``reasoning_item_id`` is only known at
                # ``response.completed``; it is emitted as a dedicated
                # carrier delta chunk (empty thinking text) that the base
                # accumulator merges onto the existing ``ThinkingBlock``.
                (
                    False,
                    [
                        ThinkingBlock.model_construct(
                            id=A,
                            created_at=A,
                            thinking="",
                            reasoning_item_id="rs_123",
                            reasoning_item_raw=reasoning_item_raw,
                        ),
                    ],
                ),
                (
                    True,
                    [
                        ThinkingBlock.model_construct(
                            id=A,
                            created_at=A,
                            thinking="Thinking",
                            reasoning_item_id="rs_123",
                            reasoning_item_raw=reasoning_item_raw,
                        ),
                        TextBlock.model_construct(
                            id=A,
                            created_at=A,
                            text="Answer",
                        ),
                    ],
                ),
            ],
        )

    async def test_stream_reasoning_text_with_optional_summary(self) -> None:
        """Raw and summary deltas stay separate without done repeats."""
        parts = {
            "reasoning_summary_text": ("Summary ", "text"),
            "reasoning_text": ("Raw ", "reasoning"),
        }
        for kinds in (
            ("reasoning_text",),
            ("reasoning_summary_text", "reasoning_text"),
        ):
            with self.subTest(kinds=kinds):
                item = _MockReasoningItem(
                    id="rs_text",
                    summary=[_MockReasoningSummary(text="Summary text")]
                    if len(kinds) == 2
                    else [],
                    content=[
                        {"type": "reasoning_text", "text": "Raw reasoning"},
                    ],
                )
                raw = item.model_dump(exclude_none=True)
                events = [
                    _make_event(
                        f"response.{kind}.delta",
                        item_id=item.id,
                        delta=parts[kind][index],
                        **(
                            {"summary_index": 0}
                            if kind == "reasoning_summary_text"
                            else {"content_index": 0}
                        ),
                    )
                    for index in range(2)
                    for kind in kinds
                ]
                events.extend(
                    _make_event(
                        f"response.{kind}.done",
                        item_id=item.id,
                        text="".join(parts[kind]),
                    )
                    for kind in kinds
                )
                events.extend(
                    [
                        _make_event(
                            "response.output_text.delta",
                            delta="Answer",
                        ),
                        _make_event(
                            "response.completed",
                            response=_mock_completion(
                                reasoning_output_item=item,
                            ),
                        ),
                    ],
                )
                self.mock_client.responses.create = AsyncMock(
                    return_value=_MockAsyncEventStream(events),
                )
                responses = [r async for r in await self.model([])]
                answer = TextBlock.model_construct(
                    id=A,
                    created_at=A,
                    text="Answer",
                )
                self.assertEqual(
                    (responses[0].is_last, responses[0].content),
                    (
                        False,
                        [
                            ThinkingBlock.model_construct(
                                id=A,
                                created_at=A,
                                thinking=parts[kinds[0]][0],
                            ),
                        ],
                    ),
                )
                self.assertEqual(
                    (responses[-1].is_last, responses[-1].content),
                    (
                        True,
                        [
                            *[
                                ThinkingBlock.model_construct(
                                    id=A,
                                    created_at=A,
                                    thinking="".join(parts[kind]),
                                    reasoning_item_id=item.id,
                                    reasoning_item_raw=raw,
                                )
                                for kind in kinds
                            ],
                            answer,
                        ],
                    ),
                )

    async def test_stream_reasoning_preserves_part_boundaries(self) -> None:
        """Reasoning stream indices identify independent thinking blocks."""
        item = _MockReasoningItem(
            id="rs_parts",
            summary=[
                _MockReasoningSummary(text="Summary one"),
                _MockReasoningSummary(text="Summary two"),
            ],
            content=[
                {"type": "reasoning_text", "text": "Reasoning one"},
                {"type": "reasoning_text", "text": "Reasoning two"},
            ],
        )
        events = [
            _make_event(
                "response.reasoning_summary_text.delta",
                item_id=item.id,
                summary_index=index,
                delta=part.text,
            )
            for index, part in enumerate(item.summary)
        ]
        events.extend(
            _make_event(
                "response.reasoning_text.delta",
                item_id=item.id,
                content_index=index,
                delta=part["text"],
            )
            for index, part in enumerate(item.content or [])
        )
        events.append(
            _make_event(
                "response.completed",
                response=_mock_completion(reasoning_output_item=item),
            ),
        )
        self.mock_client.responses.create = AsyncMock(
            return_value=_MockAsyncEventStream(events),
        )

        responses = [response async for response in await self.model([])]
        raw = item.model_dump(exclude_none=True)

        self.assertEqual(
            (responses[-1].is_last, responses[-1].content),
            (
                True,
                [
                    ThinkingBlock.model_construct(
                        id=A,
                        created_at=A,
                        thinking=text,
                        reasoning_item_id=item.id,
                        reasoning_item_raw=raw,
                    )
                    for text in (
                        "Summary one",
                        "Summary two",
                        "Reasoning one",
                        "Reasoning two",
                    )
                ],
            ),
        )

    async def test_stream_preserves_multiple_reasoning_items(self) -> None:
        """Streaming keeps every reasoning item's encrypted payload."""
        first_raw = {
            "id": "rs_first",
            "summary": [],
            "type": "reasoning",
            "encrypted_content": "encrypted_first",
        }
        second_raw = {
            "id": "rs_second",
            "summary": [],
            "type": "reasoning",
            "encrypted_content": "encrypted_second",
        }
        completed_resp = MagicMock()
        completed_resp.id = "resp-multiple-reasoning"
        completed_resp.output = [
            _MockReasoningItem.model_validate(first_raw),
            _MockReasoningItem.model_validate(second_raw),
        ]
        completed_resp.usage = MagicMock()
        completed_resp.usage.input_tokens = 10
        completed_resp.usage.output_tokens = 5
        completed_resp.usage.input_tokens_details = None

        events = [
            _make_event(
                "response.reasoning_summary_text.delta",
                delta="First",
                item_id="rs_first",
                summary_index=0,
            ),
            _make_event(
                "response.reasoning_summary_text.delta",
                delta="Second",
                item_id="rs_second",
                summary_index=0,
            ),
            _make_event("response.completed", response=completed_resp),
        ]
        self.mock_client.responses.create = AsyncMock(
            return_value=_MockAsyncEventStream(events),
        )

        gen = await self.model([])
        responses = [response async for response in gen]

        self.assertEqual(
            (responses[-1].is_last, responses[-1].content),
            (
                True,
                [
                    ThinkingBlock.model_construct(
                        id=A,
                        created_at=A,
                        thinking="First",
                        reasoning_item_id="rs_first",
                        reasoning_item_raw=first_raw,
                    ),
                    ThinkingBlock.model_construct(
                        id=A,
                        created_at=A,
                        thinking="Second",
                        reasoning_item_id="rs_second",
                        reasoning_item_raw=second_raw,
                    ),
                ],
            ),
        )

        formatted = await self.model.formatter.format(
            [
                AssistantMsg(
                    name="assistant",
                    content=responses[-1].content,
                ),
            ],
        )
        self.assertListEqual(formatted, [first_raw, second_raw])

    async def test_stream_empty_reasoning_summary_keeps_reasoning_item_id(
        self,
    ) -> None:
        """Stream empty reasoning summary still preserves its item id."""
        reasoning_item_raw = {
            "id": "rs_empty",
            "summary": [],
            "type": "reasoning",
            "encrypted_content": "encrypted_stream_payload",
        }
        reasoning_item = _MockReasoningItem.model_validate(
            reasoning_item_raw,
        )

        msg_item = MagicMock()
        msg_item.type = "message"

        completed_resp = MagicMock()
        completed_resp.id = "resp-empty"
        completed_resp.output = [reasoning_item, msg_item]
        completed_resp.usage = MagicMock()
        completed_resp.usage.input_tokens = 10
        completed_resp.usage.output_tokens = 5
        completed_resp.usage.input_tokens_details = None

        events = [
            _make_event(
                "response.output_text.delta",
                delta="Answer",
                response=MagicMock(id="resp-empty"),
            ),
            _make_event("response.completed", response=completed_resp),
        ]
        mock_create = AsyncMock(
            return_value=_MockAsyncEventStream(events),
        )
        self.mock_client.responses.create = mock_create

        gen = await self.model([])
        responses = [r async for r in gen]

        self.assertListEqual(
            [(r.is_last, r.content) for r in responses],
            [
                (
                    False,
                    [
                        TextBlock.model_construct(
                            id=A,
                            created_at=A,
                            text="Answer",
                        ),
                    ],
                ),
                (
                    False,
                    [
                        ThinkingBlock.model_construct(
                            id=A,
                            created_at=A,
                            thinking="",
                            reasoning_item_id="rs_empty",
                            reasoning_item_raw=reasoning_item_raw,
                        ),
                    ],
                ),
                (
                    True,
                    [
                        TextBlock.model_construct(
                            id=A,
                            created_at=A,
                            text="Answer",
                        ),
                        ThinkingBlock.model_construct(
                            id=A,
                            created_at=A,
                            thinking="",
                            reasoning_item_id="rs_empty",
                            reasoning_item_raw=reasoning_item_raw,
                        ),
                    ],
                ),
            ],
        )

    async def test_stream_function_call(
        self,
    ) -> None:
        """Stream function-call events use call_id as ToolCallBlock.id."""
        fc_item = MagicMock()
        fc_item.type = "function_call"
        fc_item.id = "fc_1"
        fc_item.call_id = "call-1"
        fc_item.name = "search"

        completed_resp = MagicMock()
        completed_resp.id = "resp-3"
        completed_resp.output = []
        completed_resp.usage = MagicMock()
        completed_resp.usage.input_tokens = 10
        completed_resp.usage.output_tokens = 5
        completed_resp.usage.input_tokens_details = None

        events = [
            _make_event(
                "response.output_item.added",
                item=fc_item,
                response=MagicMock(id="resp-3"),
            ),
            _make_event(
                "response.function_call_arguments.delta",
                item_id="fc_1",
                delta='{"q":',
            ),
            _make_event(
                "response.function_call_arguments.delta",
                item_id="fc_1",
                delta='"test"}',
            ),
            _make_event("response.completed", response=completed_resp),
        ]
        mock_create = AsyncMock(
            return_value=_MockAsyncEventStream(events),
        )
        self.mock_client.responses.create = mock_create

        gen = await self.model([])
        responses = [r async for r in gen]

        self.assertListEqual(
            [(r.is_last, r.content) for r in responses],
            [
                (
                    False,
                    [
                        ToolCallBlock.model_construct(
                            id="call-1",
                            created_at=A,
                            name="search",
                            input='{"q":',
                        ),
                    ],
                ),
                (
                    False,
                    [
                        ToolCallBlock.model_construct(
                            id="call-1",
                            created_at=A,
                            name="search",
                            input='"test"}',
                        ),
                    ],
                ),
                (
                    True,
                    [
                        ToolCallBlock.model_construct(
                            id="call-1",
                            created_at=A,
                            name="search",
                            input='{"q":"test"}',
                        ),
                    ],
                ),
            ],
        )

    async def test_stream_function_call_without_arguments(self) -> None:
        """A function call with no arguments still streams as a {} block.

        Empty-argument calls may emit no argument delta or an empty one, so
        the block used to be dropped entirely instead of matching the
        non-streaming ``input or "{}"`` result.
        """
        fc_item = MagicMock()
        fc_item.type = "function_call"
        fc_item.id = "fc_1"
        fc_item.call_id = "call-1"
        fc_item.name = "get_current_time"
        fc_item.arguments = ""

        completed_resp = MagicMock()
        completed_resp.id = "resp-4"
        completed_resp.output = []
        completed_resp.usage = MagicMock()
        completed_resp.usage.input_tokens = 10
        completed_resp.usage.output_tokens = 5
        completed_resp.usage.input_tokens_details = None

        non_stream_model = _make_model(stream=False)
        non_stream_model.client = self.mock_client
        non_stream_resp = _mock_completion(
            function_calls=[
                {
                    "id": "fc_1",
                    "call_id": "call-1",
                    "name": "get_current_time",
                    "arguments": "",
                },
            ],
            response_id="resp-4",
        )

        cases = {
            "no_delta": [],
            "empty_delta": [
                _make_event(
                    "response.function_call_arguments.delta",
                    item_id="fc_1",
                    delta="",
                ),
            ],
        }
        expected_block = ToolCallBlock.model_construct(
            id="call-1",
            created_at=A,
            name="get_current_time",
            input="{}",
        )
        for name, argument_events in cases.items():
            with self.subTest(name=name):
                events = [
                    _make_event(
                        "response.output_item.added",
                        item=fc_item,
                        response=MagicMock(id="resp-4"),
                    ),
                    *argument_events,
                    _make_event(
                        "response.function_call_arguments.done",
                        item_id="fc_1",
                        arguments="",
                    ),
                    _make_event(
                        "response.completed",
                        response=completed_resp,
                    ),
                ]
                self.mock_client.responses.create = AsyncMock(
                    side_effect=[
                        _MockAsyncEventStream(events),
                        non_stream_resp,
                    ],
                )

                gen = await self.model([])
                responses = [r async for r in gen]
                non_stream = await non_stream_model([])

                self.assertListEqual(
                    [(r.is_last, r.content) for r in responses],
                    [
                        (False, [expected_block]),
                        (True, [expected_block]),
                    ],
                )
                self.assertListEqual(
                    [
                        (block.id, block.name, block.input)
                        for block in (
                            responses[-1].content[0],
                            non_stream.content[0],
                        )
                    ],
                    [("call-1", "get_current_time", "{}")] * 2,
                )


# ---------------------------------------------------------------------------
# _format_tools tests
# ---------------------------------------------------------------------------

_FT_TOOLS = [
    {
        "type": "function",
        "function": {
            "name": "get_weather",
            "description": "Get the weather",
            "parameters": {
                "type": "object",
                "properties": {"city": {"type": "string"}},
                "required": ["city"],
            },
        },
    },
    {
        "type": "function",
        "function": {
            "name": "get_time",
            "description": "Get the time",
            "parameters": {
                "type": "object",
                "properties": {"timezone": {"type": "string"}},
                "required": ["timezone"],
            },
        },
    },
]

_FT_TOOLS_RESPONSE = [
    {
        "type": "function",
        "name": "get_weather",
        "description": "Get the weather",
        "parameters": {
            "type": "object",
            "properties": {"city": {"type": "string"}},
            "required": ["city"],
        },
    },
    {
        "type": "function",
        "name": "get_time",
        "description": "Get the time",
        "parameters": {
            "type": "object",
            "properties": {"timezone": {"type": "string"}},
            "required": ["timezone"],
        },
    },
]


class TestOpenAIResponseFormatTools(unittest.TestCase):
    """Tests for OpenAIResponseModel._format_tools."""

    def setUp(self) -> None:
        self.model = _make_model()

    def test_auto_mode(self) -> None:
        """Auto mode converts tools and sets choice to 'auto'."""
        fmt_tools, fmt_choice = self.model._format_tools(
            _FT_TOOLS,
            ToolChoice(mode="auto"),
        )
        self.assertEqual(fmt_tools, _FT_TOOLS_RESPONSE)
        self.assertEqual(fmt_choice, "auto")

    def test_none_mode(self) -> None:
        """None mode converts tools and sets choice to 'none'."""
        fmt_tools, fmt_choice = self.model._format_tools(
            _FT_TOOLS,
            ToolChoice(mode="none"),
        )
        self.assertEqual(fmt_tools, _FT_TOOLS_RESPONSE)
        self.assertEqual(fmt_choice, "none")

    def test_required_mode(self) -> None:
        """Required mode converts tools and sets choice to 'required'."""
        fmt_tools, fmt_choice = self.model._format_tools(
            _FT_TOOLS,
            ToolChoice(mode="required"),
        )
        self.assertEqual(fmt_tools, _FT_TOOLS_RESPONSE)
        self.assertEqual(fmt_choice, "required")

    def test_str_mode_force_call(self) -> None:
        """String mode forces a function call for the named tool."""
        fmt_tools, fmt_choice = self.model._format_tools(
            _FT_TOOLS,
            ToolChoice(mode="get_weather"),
        )
        self.assertEqual(fmt_tools, _FT_TOOLS_RESPONSE)
        self.assertEqual(
            fmt_choice,
            {"type": "function", "name": "get_weather"},
        )

    def test_tools_filtered(self) -> None:
        """ToolChoice with tools list keeps the full tools schema and
        narrows the callable subset via ``allowed_tools`` to preserve
        prompt cache hits."""
        fmt_tools, fmt_choice = self.model._format_tools(
            _FT_TOOLS,
            ToolChoice(mode="auto", tools=["get_weather"]),
        )
        self.assertListEqual(fmt_tools, _FT_TOOLS_RESPONSE)
        self.assertEqual(
            fmt_choice,
            {
                "type": "allowed_tools",
                "mode": "auto",
                "tools": [{"type": "function", "name": "get_weather"}],
            },
        )

    def test_no_tool_choice(self) -> None:
        """Tools are converted when tool_choice is None."""
        fmt_tools, fmt_choice = self.model._format_tools(_FT_TOOLS, None)
        self.assertEqual(fmt_tools, _FT_TOOLS_RESPONSE)
        self.assertIsNone(fmt_choice)


class TestNonStreamingFailure(IsolatedAsyncioTestCase):
    """Test failed generations through the real SDK without network access."""

    async def asyncSetUp(self) -> None:
        """Create an offline model returning a failed Responses object."""
        self.payload: dict[str, Any] = {
            "id": "resp_failed_offline",
            "object": "response",
            "created_at": 0,
            "model": "o4-mini",
            "status": "failed",
            "error": {
                "code": "server_error",
                "message": "Generation failed.",
            },
            "output": [],
            "parallel_tool_calls": True,
            "tool_choice": "auto",
            "tools": [],
            "usage": None,
        }

        def _handler(_request: httpx.Request) -> httpx.Response:
            Response.model_validate(self.payload)
            return httpx.Response(200, json=self.payload)

        self.model = OpenAIResponseModel(
            credential=OpenAICredential(api_key="offline-test"),
            model="o4-mini",
            stream=False,
            max_retries=0,
            client_kwargs={
                "http_client": httpx.AsyncClient(
                    transport=httpx.MockTransport(_handler),
                ),
                "max_retries": 0,
            },
        )

    async def asyncTearDown(self) -> None:
        """Close the SDK and HTTP client."""
        await self.model.client.close()

    async def test_failed_generation_raises_with_error_details(self) -> None:
        """An HTTP 200 failed generation must not become a completed reply."""
        with self.assertRaisesRegex(
            RuntimeError,
            "resp_failed_offline.*server_error.*Generation",
        ):
            await self.model([])

    async def test_completed_generation_returns_output(self) -> None:
        """Successful SDK responses still return their generated text."""
        self.payload["status"] = "completed"
        self.payload["error"] = None
        self.payload["output"] = [
            _mock_message(
                "msg_complete",
                [("output_text", "Answer")],
            ).model_dump(),
        ]
        result = await self.model([])
        self.assertEqual(result.content[0].text, "Answer")

    async def test_failed_generation_without_error_details(self) -> None:
        """A failed status is sufficient even when the error field is null."""
        self.payload["error"] = None
        with self.assertRaisesRegex(
            RuntimeError,
            "resp_failed_offline.*failed",
        ):
            await self.model([])

    async def test_failed_generation_does_not_return_partial_output(
        self,
    ) -> None:
        """Partial output must not hide a failed generation."""
        self.payload["output"] = [
            {
                "id": "msg_partial",
                "type": "message",
                "role": "assistant",
                "status": "incomplete",
                "content": [
                    {
                        "type": "output_text",
                        "text": "Partial",
                        "annotations": [],
                    },
                ],
            },
        ]
        with self.assertRaisesRegex(RuntimeError, "server_error"):
            await self.model([])

    async def test_failed_generation_uses_agent_fallback(self) -> None:
        """The configured fallback runs when the primary generation fails."""
        fallback = _make_model(stream=False)
        fallback.client.responses.create = AsyncMock(
            return_value=_mock_completion(text="Fallback answer"),
        )
        try:
            agent = Agent(
                name="test",
                system_prompt="",
                model=self.model,
                model_config=ModelConfig(fallback_model=fallback),
            )
            result = await agent._call_model(messages=[], tools=[])
            self.assertEqual(result.content[0].text, "Fallback answer")
        finally:
            await fallback.client.close()


class _TrackedBody(httpx.AsyncByteStream):
    """HTTP body with observable cleanup."""

    def __init__(self) -> None:
        """Initialize the body state."""
        self.closed = False
        self.stop = asyncio.Event()

    async def __aiter__(self) -> Any:
        """Yield one SSE event and wait."""
        yield (
            b'data: {"type":"response.output_text.delta",'
            b'"sequence_number":0,"item_id":"msg_1","output_index":0,'
            b'"content_index":0,"delta":"hello"}\n\n'
        )
        await self.stop.wait()

    async def aclose(self) -> None:
        """Close and unblock the body."""
        self.closed = True
        self.stop.set()


class TestPublicModelStreamClose(IsolatedAsyncioTestCase):
    """Test public stream cleanup."""

    async def test_aclose_closes_offline_httpx_response(self) -> None:
        """Test cleanup through the OpenAI SDK and HTTPX transport."""
        body = _TrackedBody()

        def _handler(_request: httpx.Request) -> httpx.Response:
            return httpx.Response(
                200,
                headers={"content-type": "text/event-stream"},
                stream=body,
            )

        transport = httpx.MockTransport(_handler)
        client = AsyncOpenAI(
            api_key="offline-test",
            http_client=httpx.AsyncClient(transport=transport),
        )
        model = _make_model(stream=True)
        original_client = model.client
        model.client = client
        stream = None
        try:
            stream = await model([])
            first = await anext(stream)
            self.assertEqual(first.content[0].text, "hello")
            self.assertFalse(body.closed)
            await stream.aclose()
            self.assertTrue(body.closed)
        finally:
            if stream is not None:
                await stream.aclose()
            await body.aclose()
            await client.close()
            await original_client.close()