Skip to content

Build and run agents you can see, understand and trust.

47.5 KB
Raw Blame
# -*- coding: utf-8 -*-
# pylint: disable=redefined-builtin
"""Test the team pipeline."""
import asyncio
from typing import Any, AsyncGenerator
from unittest.async_case import IsolatedAsyncioTestCase
from unittest.mock import patch

from utils import AnyString, MockModel

from agentscope.agent import Agent, InjectionConfig
from agentscope.event import (
    AgentEvent,
    ConfirmResult,
    ExternalExecutionResultEvent,
    UserConfirmResultEvent,
    UserInterruptEvent,
)
from agentscope.message import TextBlock, ToolCallBlock, UserMsg
from agentscope.model import ChatResponse
from agentscope.permission import (
    PermissionBehavior,
    PermissionContext,
    PermissionDecision,
)
from agentscope.pipeline import TeamMember, TeamPipeline
from agentscope.tool import ToolBase, ToolChunk, Toolkit


class AskTool(ToolBase):
    """A tool that asks the user before running."""

    name: str = "ask_tool"
    description: str = "A tool that needs confirmation"
    input_schema: dict[str, Any] = {
        "type": "object",
        "properties": {"input": {"type": "string"}},
        "required": ["input"],
    }
    is_concurrency_safe: bool = True
    is_read_only: bool = False

    async def check_permissions(  # pylint: disable=unused-argument
        self,
        tool_input: dict[str, Any],
        context: PermissionContext,
    ) -> PermissionDecision:
        """Always ask."""
        return PermissionDecision(
            behavior=PermissionBehavior.ASK,
            message="Needs confirmation",
        )

    async def call(self, input: str) -> ToolChunk:
        """Echo the input."""
        return ToolChunk(content=[TextBlock(text=f"ask result: {input}")])


class SlowTool(ToolBase):
    """A tool that runs until cancelled, flagging when it starts."""

    name: str = "slow_tool"
    description: str = "A tool that never finishes"
    input_schema: dict[str, Any] = {"type": "object", "properties": {}}
    is_concurrency_safe: bool = True
    is_read_only: bool = True

    def __init__(self) -> None:
        """Create the start flag per tool."""
        super().__init__()
        self.started = asyncio.Event()

    async def check_permissions(  # pylint: disable=unused-argument
        self,
        tool_input: dict[str, Any],
        context: PermissionContext,
    ) -> PermissionDecision:
        """Always allow."""
        return PermissionDecision(
            behavior=PermissionBehavior.ALLOW,
            message="Allowed",
        )

    async def call(self) -> ToolChunk:
        """Flag the start and wait forever."""
        self.started.set()
        await asyncio.Event().wait()
        return ToolChunk(content=[TextBlock(text="never")])


def _tool_call(
    id: str,
    input: str,
    name: str = "TeamAssign",
) -> list[ChatResponse]:
    """A streamed model reply that calls one tool."""
    block = ToolCallBlock(id=id, name=name, input=input)
    return [
        ChatResponse(content=[block], is_last=False, usage=None),
        ChatResponse(content=[block], is_last=True, usage=None),
    ]


def _text(text: str) -> list[ChatResponse]:
    """A streamed model reply with plain text."""
    return [
        ChatResponse(
            content=[TextBlock(text=text)],
            is_last=False,
            usage=None,
        ),
        ChatResponse(content=[TextBlock(text=text)], is_last=True, usage=None),
    ]


def _agent(name: str, tools: list[ToolBase] | None = None) -> Agent:
    """An agent driven by its own mock model."""
    return Agent(
        name=name,
        system_prompt=f"You are {name}.",
        model=MockModel(),
        toolkit=Toolkit(tools=tools),
        injection_config=InjectionConfig(inject_runtime_state=False),
    )


class TeamPipelineTest(IsolatedAsyncioTestCase):
    """The team pipeline test case."""

    async def asyncSetUp(self) -> None:
        """Build a leader with a researcher it can delegate to."""
        self.leader = _agent("leader")
        self.researcher = _agent("researcher", [AskTool()])
        self.pipeline = TeamPipeline(
            leader=self.leader,
            members=[
                TeamMember(agent=self.researcher, description="Researches"),
            ],
        )
        self.query = UserMsg(name="user", content="What is A?")

    async def _run(self, inputs: Any) -> list[tuple[str, str]]:
        """Drain one run into (event type, owner) pairs."""
        events = [_ async for _ in self.pipeline.reply_stream(inputs)]
        agents = [
            self.leader,
            *[_.agent for _ in self.pipeline.members.values()],
        ]
        owners = {_.state.reply_id: _.name for _ in agents}
        return [(_.type.value, owners[_.reply_id]) for _ in events]

    async def test_delegation_round_trip(self) -> None:
        """The researcher's final reply becomes the leader's tool result,
        its intermediate events are streamed, and its context is reset once
        the leader's reply ends."""
        self.leader.model.set_responses(
            [
                _tool_call(
                    "call-1",
                    '{"member": "researcher", "prompt": "Find A"}',
                ),
                _text("A is 42."),
            ],
        )
        self.researcher.model.set_responses([_text("Found: A = 42")])

        events = await self._run(self.query)

        self.assertListEqual(
            events,
            [
                ("REPLY_START", "leader"),
                ("MODEL_CALL_START", "leader"),
                ("TOOL_CALL_START", "leader"),
                ("TOOL_CALL_DELTA", "leader"),
                ("TOOL_CALL_END", "leader"),
                ("MODEL_CALL_END", "leader"),
                ("TOOL_RESULT_START", "leader"),
                ("REPLY_START", "researcher"),
                ("MODEL_CALL_START", "researcher"),
                ("TEXT_BLOCK_START", "researcher"),
                ("TEXT_BLOCK_DELTA", "researcher"),
                ("TEXT_BLOCK_END", "researcher"),
                ("MODEL_CALL_END", "researcher"),
                ("REPLY_END", "researcher"),
                ("TOOL_RESULT_TEXT_DELTA", "leader"),
                ("TOOL_RESULT_END", "leader"),
                ("MODEL_CALL_START", "leader"),
                ("TEXT_BLOCK_START", "leader"),
                ("TEXT_BLOCK_DELTA", "leader"),
                ("TEXT_BLOCK_END", "leader"),
                ("MODEL_CALL_END", "leader"),
                ("REPLY_END", "leader"),
            ],
        )
        self.assertListEqual(
            [_.model_dump() for _ in self.leader.state.context],
            [
                {
                    "id": AnyString(),
                    "created_at": AnyString(),
                    "finished_at": AnyString(),
                    "finished_reason": None,
                    "structured_output": None,
                    "error": None,
                    "metadata": {},
                    "name": "user",
                    "role": "user",
                    "usage": None,
                    "content": [
                        {
                            "type": "text",
                            "created_at": AnyString(),
                            "finished_at": None,
                            "id": AnyString(),
                            "text": "What is A?",
                        },
                    ],
                },
                {
                    "id": AnyString(),
                    "created_at": AnyString(),
                    "finished_at": None,
                    "finished_reason": None,
                    "structured_output": None,
                    "error": None,
                    "metadata": {},
                    "name": "leader",
                    "role": "assistant",
                    "usage": None,
                    "content": [
                        {
                            "type": "tool_call",
                            "created_at": AnyString(),
                            "finished_at": None,
                            "id": "call-1",
                            "name": "TeamAssign",
                            "input": '{"member": "researcher", '
                            '"prompt": "Find A"}',
                            "state": "finished",
                            "suggested_rules": [],
                        },
                        {
                            "type": "tool_result",
                            "created_at": AnyString(),
                            "finished_at": None,
                            "id": "call-1",
                            "name": "TeamAssign",
                            "output": [
                                {
                                    "type": "text",
                                    "created_at": AnyString(),
                                    "finished_at": None,
                                    "id": AnyString(),
                                    "text": "Found: A = 42",
                                },
                            ],
                            "state": "success",
                            "metadata": {},
                        },
                        {
                            "type": "text",
                            "created_at": AnyString(),
                            "finished_at": None,
                            "id": AnyString(),
                            "text": "A is 42.",
                        },
                    ],
                },
            ],
        )
        self.assertListEqual(self.researcher.state.context, [])

    async def test_member_model_failure_clears_parked_assignment(self) -> None:
        """Close a failed assignment and let the same member work next turn."""
        failure = RuntimeError("simulated transient member model failure")
        self.leader.model.set_responses(
            [
                _tool_call(
                    "call-1",
                    '{"member": "researcher", "prompt": "Find A"}',
                ),
                _tool_call(
                    "call-2",
                    '{"member": "researcher", "prompt": "Retry A"}',
                ),
                _text("A is 42."),
            ],
        )
        self.researcher.model.set_responses(
            [failure, _text("recovered member response")],
        )

        with self.assertRaises(RuntimeError) as caught:
            await self._run(self.query)

        self.assertIs(caught.exception, failure)
        self.assertEqual(
            self.leader.state.get_awaiting_tool_calls("leader"),
            [],
        )
        self.assertEqual(
            [
                block.model_dump()
                for block in self.leader.state.context[-1].get_content_blocks(
                    "tool_result",
                )
            ],
            [
                {
                    "type": "tool_result",
                    "id": "call-1",
                    "name": "TeamAssign",
                    "output": (
                        "<system-reminder>The tool call has been interrupted "
                        "by the user.</system-reminder>"
                    ),
                    "state": "interrupted",
                    "metadata": {},
                    "created_at": AnyString(),
                    "finished_at": None,
                },
            ],
        )

        events = await self._run(UserMsg(name="user", content="try again"))
        self.assertEqual(
            [event for event in events if event[0] == "REPLY_END"],
            [("REPLY_END", "researcher"), ("REPLY_END", "leader")],
        )
        self.assertEqual(
            [
                block.model_dump()
                for block in self.leader.state.context[-1].get_content_blocks(
                    "tool_result",
                )
            ],
            [
                {
                    "type": "tool_result",
                    "id": "call-2",
                    "name": "TeamAssign",
                    "output": [
                        {
                            "type": "text",
                            "text": "recovered member response",
                            "id": AnyString(),
                            "created_at": AnyString(),
                            "finished_at": None,
                        },
                    ],
                    "state": "success",
                    "metadata": {},
                    "created_at": AnyString(),
                    "finished_at": None,
                },
            ],
        )
        self.assertEqual(
            (self.leader.model.cnt, self.researcher.model.cnt),
            (3, 2),
        )
        self.assertEqual(self.researcher.state.context, [])

    async def test_invalid_inputs_preserve_parked_team(self) -> None:
        """Rejected inputs must leave the pending confirmation resumable."""
        self.leader.model.set_responses(
            [
                _tool_call(
                    "call-1",
                    '{"member": "researcher", "prompt": "Find A"}',
                ),
                _text("A is 42."),
            ],
        )
        self.researcher.model.set_responses(
            [
                _tool_call("call-r1", '{"input": "A"}', "ask_tool"),
                _text("Found A"),
            ],
        )
        await self._run(self.query)
        reply_id = self.researcher.state.reply_id
        states = [self.leader.state, self.researcher.state]
        before = [state.model_dump() for state in states]
        invalid_inputs = [
            UserConfirmResultEvent(
                reply_id="stale-reply-id",
                confirm_results=[],
            ),
            UserConfirmResultEvent(
                reply_id=reply_id,
                confirm_results=[
                    ConfirmResult(
                        confirmed=True,
                        tool_call=ToolCallBlock(
                            id="unknown-call",
                            name="ask_tool",
                            input='{"input": "A"}',
                        ),
                    ),
                ],
            ),
            ExternalExecutionResultEvent(
                reply_id=reply_id,
                execution_results=[],
            ),
            self.query,
        ]
        for inputs in invalid_inputs:
            with self.subTest(inputs=inputs):
                with self.assertRaises(ValueError):
                    await self._run(inputs)
                self.assertEqual(
                    [state.model_dump() for state in states],
                    before,
                )

        calls = self.researcher.state.get_awaiting_tool_calls("researcher")
        events = await self._run(
            UserConfirmResultEvent(
                reply_id=reply_id,
                confirm_results=[
                    ConfirmResult(
                        confirmed=True,
                        tool_call=calls[0],
                    ),
                ],
            ),
        )
        self.assertEqual(events[-1], ("REPLY_END", "leader"))

    async def test_cleanup_failure_preserves_member_error(self) -> None:
        """A failing interrupt must neither recurse nor replace the error."""
        failure = RuntimeError("member model failed")
        self.researcher.model.set_responses([failure])
        self.leader.model.set_responses(
            [
                _tool_call(
                    "call-1",
                    '{"member": "researcher", "prompt": "Find A"}',
                ),
            ],
        )
        original_reply_stream = self.leader.reply_stream
        interrupts = []

        async def reply_stream(
            inputs: Any,
        ) -> AsyncGenerator[AgentEvent, None]:
            """Simulate a reply middleware failing before interrupt cleanup."""
            if isinstance(inputs, UserInterruptEvent):
                interrupts.append(inputs.reply_id)
                raise RuntimeError("middleware unavailable")
            async for event in original_reply_stream(inputs):
                yield event

        with (
            patch.object(self.leader, "reply_stream", reply_stream),
            self.assertLogs("as", level="WARNING"),
        ):
            with self.assertRaises(RuntimeError) as caught:
                await self._run(self.query)

        self.assertIs(caught.exception, failure)
        self.assertEqual(interrupts, [self.leader.state.reply_id])

    async def test_resumed_member_failure_clears_assignment(self) -> None:
        """A model failure after confirmation also releases the leader."""
        failure = RuntimeError("resumed member failed")
        self.leader.model.set_responses(
            [
                _tool_call(
                    "call-1",
                    '{"member": "researcher", "prompt": "Find A"}',
                ),
                _text("next turn"),
            ],
        )
        self.researcher.model.set_responses(
            [
                _tool_call("call-r1", '{"input": "A"}', "ask_tool"),
                failure,
            ],
        )
        await self._run(self.query)
        calls = self.researcher.state.get_awaiting_tool_calls("researcher")
        with self.assertRaises(RuntimeError) as caught:
            await self._run(
                UserConfirmResultEvent(
                    reply_id=self.researcher.state.reply_id,
                    confirm_results=[
                        ConfirmResult(
                            confirmed=True,
                            tool_call=calls[0],
                        ),
                    ],
                ),
            )
        self.assertIs(caught.exception, failure)
        self.assertEqual(
            self.leader.state.get_awaiting_tool_calls("leader"),
            [],
        )
        events = await self._run(UserMsg(name="user", content="try again"))
        self.assertEqual(events[-1], ("REPLY_END", "leader"))

    async def test_member_hitl_round_trip(self) -> None:
        """A researcher parked on confirmation ends the stream with the
        leader parked on the delegation; the confirmation is routed back
        and the leader continues with the researcher's reply."""
        self.pipeline.reset_members = False
        self.leader.model.set_responses(
            [
                _tool_call(
                    "call-1",
                    '{"member": "researcher", "prompt": "Find A"}',
                ),
                _text("A is 42."),
            ],
        )
        self.researcher.model.set_responses(
            [
                _tool_call("call-r1", '{"input": "A"}', name="ask_tool"),
                _text("Found: A = 42"),
            ],
        )

        events = await self._run(self.query)

        self.assertListEqual(
            events[6:],
            [
                ("TOOL_RESULT_START", "leader"),
                ("REPLY_START", "researcher"),
                ("MODEL_CALL_START", "researcher"),
                ("TOOL_CALL_START", "researcher"),
                ("TOOL_CALL_DELTA", "researcher"),
                ("TOOL_CALL_END", "researcher"),
                ("MODEL_CALL_END", "researcher"),
                ("REQUIRE_USER_CONFIRM", "researcher"),
            ],
        )
        self.assertListEqual(
            [
                (_.name, _.state)
                for _ in self.leader.state.get_awaiting_tool_calls("leader")
            ],
            [("TeamAssign", "submitted")],
        )
        self.assertListEqual(
            [
                (_.name, _.state)
                for _ in self.researcher.state.get_awaiting_tool_calls(
                    "researcher",
                )
            ],
            [("ask_tool", "asking")],
        )

        events = await self._run(
            UserConfirmResultEvent(
                reply_id=self.researcher.state.reply_id,
                confirm_results=[
                    ConfirmResult(
                        confirmed=True,
                        tool_call=self.researcher.state.context[-1].content[0],
                    ),
                ],
            ),
        )

        self.assertListEqual(
            events,
            [
                ("TOOL_RESULT_START", "researcher"),
                ("TOOL_RESULT_TEXT_DELTA", "researcher"),
                ("TOOL_RESULT_END", "researcher"),
                ("MODEL_CALL_START", "researcher"),
                ("TEXT_BLOCK_START", "researcher"),
                ("TEXT_BLOCK_DELTA", "researcher"),
                ("TEXT_BLOCK_END", "researcher"),
                ("MODEL_CALL_END", "researcher"),
                ("REPLY_END", "researcher"),
                ("TOOL_RESULT_TEXT_DELTA", "leader"),
                ("TOOL_RESULT_END", "leader"),
                ("MODEL_CALL_START", "leader"),
                ("TEXT_BLOCK_START", "leader"),
                ("TEXT_BLOCK_DELTA", "leader"),
                ("TEXT_BLOCK_END", "leader"),
                ("MODEL_CALL_END", "leader"),
                ("REPLY_END", "leader"),
            ],
        )
        self.assertListEqual(
            [_.model_dump() for _ in self.researcher.state.context],
            [
                {
                    "id": AnyString(),
                    "created_at": AnyString(),
                    "finished_at": AnyString(),
                    "finished_reason": None,
                    "structured_output": None,
                    "error": None,
                    "metadata": {},
                    "name": "user",
                    "role": "user",
                    "usage": None,
                    "content": [
                        {
                            "type": "text",
                            "created_at": AnyString(),
                            "finished_at": None,
                            "id": AnyString(),
                            "text": "Find A",
                        },
                    ],
                },
                {
                    "id": AnyString(),
                    "created_at": AnyString(),
                    "finished_at": None,
                    "finished_reason": None,
                    "structured_output": None,
                    "error": None,
                    "metadata": {},
                    "name": "researcher",
                    "role": "assistant",
                    "usage": None,
                    "content": [
                        {
                            "type": "tool_call",
                            "created_at": AnyString(),
                            "finished_at": None,
                            "id": "call-r1",
                            "name": "ask_tool",
                            "input": '{"input": "A"}',
                            "state": "finished",
                            "suggested_rules": [
                                {
                                    "tool_name": "ask_tool",
                                    "rule_content": None,
                                    "behavior": PermissionBehavior.ALLOW,
                                    "source": "suggested",
                                },
                            ],
                        },
                        {
                            "type": "tool_result",
                            "created_at": AnyString(),
                            "finished_at": None,
                            "id": "call-r1",
                            "name": "ask_tool",
                            "output": [
                                {
                                    "type": "text",
                                    "created_at": AnyString(),
                                    "finished_at": None,
                                    "id": AnyString(),
                                    "text": "ask result: A",
                                },
                            ],
                            "state": "success",
                            "metadata": {},
                        },
                        {
                            "type": "text",
                            "created_at": AnyString(),
                            "finished_at": None,
                            "id": AnyString(),
                            "text": "Found: A = 42",
                        },
                    ],
                },
            ],
        )
        self.assertEqual(
            self.leader.state.context[-1].get_text_content(),
            "A is 42.",
        )

    async def test_concurrent_delegations(self) -> None:
        """Delegations to different members in one round run together and
        land in the leader's context in call order."""
        coder = _agent("coder")
        self.pipeline = TeamPipeline(
            leader=self.leader,
            members=[
                TeamMember(agent=self.researcher, description="Researches"),
                TeamMember(agent=coder, description="Codes"),
            ],
        )
        self.leader.model.set_responses(
            [
                [
                    ChatResponse(
                        content=[
                            ToolCallBlock(
                                id="call-1",
                                name="TeamAssign",
                                input='{"member": "researcher", '
                                '"prompt": "Find A"}',
                            ),
                            ToolCallBlock(
                                id="call-2",
                                name="TeamAssign",
                                input='{"member": "coder", '
                                '"prompt": "Write A"}',
                            ),
                        ],
                        is_last=True,
                        usage=None,
                    ),
                ],
                _text("Both done."),
            ],
        )
        self.researcher.model.set_responses([_text("Found: A = 42")])
        coder.model.set_responses([_text("Wrote a.py")])

        events = await self._run(self.query)

        self.assertListEqual(
            sorted(owner for kind, owner in events if kind == "REPLY_START"),
            ["coder", "leader", "researcher"],
        )
        self.assertListEqual(
            [
                (_.id, _.name, _.output[0].text, _.state)
                for _ in self.leader.state.context[-1].get_content_blocks(
                    "tool_result",
                )
            ],
            [
                ("call-1", "TeamAssign", "Found: A = 42", "success"),
                ("call-2", "TeamAssign", "Wrote a.py", "success"),
            ],
        )
        self.assertEqual(
            self.leader.state.context[-1].get_text_content(),
            "Both done.",
        )

    async def test_interrupt_parked_member(self) -> None:
        """An interrupt closes the parked researcher and the leader's
        delegation alike, and resets the researcher afterwards."""
        self.leader.model.set_responses(
            [
                _tool_call(
                    "call-1",
                    '{"member": "researcher", "prompt": "Find A"}',
                ),
            ],
        )
        self.researcher.model.set_responses(
            [_tool_call("call-r1", '{"input": "A"}', name="ask_tool")],
        )
        await self._run(self.query)

        events = await self._run(
            UserInterruptEvent(reply_id=self.researcher.state.reply_id),
        )

        self.assertListEqual(
            events,
            [
                ("TOOL_RESULT_START", "researcher"),
                ("TOOL_RESULT_TEXT_DELTA", "researcher"),
                ("TOOL_RESULT_END", "researcher"),
                ("REPLY_END", "researcher"),
                ("TOOL_RESULT_TEXT_DELTA", "leader"),
                ("TOOL_RESULT_END", "leader"),
                ("REPLY_END", "leader"),
            ],
        )
        self.assertListEqual(
            [
                (_.id, _.state)
                for _ in self.leader.state.context[-1].get_content_blocks(
                    "tool_result",
                )
            ],
            [("call-1", "interrupted")],
        )
        self.assertListEqual(self.researcher.state.context, [])

    async def test_repaired_assignment_input(self) -> None:
        """An assignment input that only parses after repair still reaches
        the member instead of leaving the leader parked."""
        self.leader.model.set_responses(
            [
                _tool_call(
                    "call-1",
                    "{'member': 'researcher', 'prompt': 'Find A'}",
                ),
                _text("A is 42."),
            ],
        )
        self.researcher.model.set_responses([_text("Found: A = 42")])

        await self._run(self.query)

        self.assertDictEqual(
            self.leader.state.context[-1].model_dump(),
            {
                "id": AnyString(),
                "created_at": AnyString(),
                "finished_at": None,
                "finished_reason": None,
                "structured_output": None,
                "error": None,
                "metadata": {},
                "name": "leader",
                "role": "assistant",
                "usage": None,
                "content": [
                    {
                        "type": "tool_call",
                        "created_at": AnyString(),
                        "finished_at": None,
                        "id": "call-1",
                        "name": "TeamAssign",
                        "input": '{"member": "researcher", '
                        '"prompt": "Find A"}',
                        "state": "finished",
                        "suggested_rules": [],
                    },
                    {
                        "type": "tool_result",
                        "created_at": AnyString(),
                        "finished_at": None,
                        "id": "call-1",
                        "name": "TeamAssign",
                        "output": [
                            {
                                "type": "text",
                                "created_at": AnyString(),
                                "finished_at": None,
                                "id": AnyString(),
                                "text": "Found: A = 42",
                            },
                        ],
                        "state": "success",
                        "metadata": {},
                    },
                    {
                        "type": "text",
                        "created_at": AnyString(),
                        "finished_at": None,
                        "id": AnyString(),
                        "text": "A is 42.",
                    },
                ],
            },
        )

    async def test_one_member_parks_in_a_round(self) -> None:
        """With the researcher parked, the coder's result is fed right away
        and the leader stays parked until the researcher resumes."""
        coder = _agent("coder")
        self.pipeline = TeamPipeline(
            leader=self.leader,
            members=[
                TeamMember(agent=self.researcher, description="Researches"),
                TeamMember(agent=coder, description="Codes"),
            ],
        )
        self.leader.model.set_responses(
            [
                [
                    ChatResponse(
                        content=[
                            ToolCallBlock(
                                id="call-1",
                                name="TeamAssign",
                                input='{"member": "researcher", '
                                '"prompt": "Find A"}',
                            ),
                            ToolCallBlock(
                                id="call-2",
                                name="TeamAssign",
                                input='{"member": "coder", '
                                '"prompt": "Write A"}',
                            ),
                        ],
                        is_last=True,
                        usage=None,
                    ),
                ],
                _text("Both done."),
            ],
        )
        self.researcher.model.set_responses(
            [
                _tool_call("call-r1", '{"input": "A"}', name="ask_tool"),
                _text("Found: A = 42"),
            ],
        )
        coder.model.set_responses([_text("Wrote a.py")])

        await self._run(self.query)

        self.assertListEqual(
            [
                (_.id, _.state)
                for _ in self.leader.state.get_awaiting_tool_calls("leader")
            ],
            [("call-1", "submitted")],
        )

        await self._run(
            UserConfirmResultEvent(
                reply_id=self.researcher.state.reply_id,
                confirm_results=[
                    ConfirmResult(
                        confirmed=True,
                        tool_call=self.researcher.state.context[-1].content[0],
                    ),
                ],
            ),
        )

        self.assertDictEqual(
            self.leader.state.context[-1].model_dump(),
            {
                "id": AnyString(),
                "created_at": AnyString(),
                "finished_at": None,
                "finished_reason": None,
                "structured_output": None,
                "error": None,
                "metadata": {},
                "name": "leader",
                "role": "assistant",
                "usage": None,
                "content": [
                    {
                        "type": "tool_call",
                        "created_at": AnyString(),
                        "finished_at": None,
                        "id": "call-1",
                        "name": "TeamAssign",
                        "input": '{"member": "researcher", '
                        '"prompt": "Find A"}',
                        "state": "finished",
                        "suggested_rules": [],
                    },
                    {
                        "type": "tool_call",
                        "created_at": AnyString(),
                        "finished_at": None,
                        "id": "call-2",
                        "name": "TeamAssign",
                        "input": '{"member": "coder", "prompt": "Write A"}',
                        "state": "finished",
                        "suggested_rules": [],
                    },
                    {
                        "type": "tool_result",
                        "created_at": AnyString(),
                        "finished_at": None,
                        "id": "call-2",
                        "name": "TeamAssign",
                        "output": [
                            {
                                "type": "text",
                                "created_at": AnyString(),
                                "finished_at": None,
                                "id": AnyString(),
                                "text": "Wrote a.py",
                            },
                        ],
                        "state": "success",
                        "metadata": {},
                    },
                    {
                        "type": "tool_result",
                        "created_at": AnyString(),
                        "finished_at": None,
                        "id": "call-1",
                        "name": "TeamAssign",
                        "output": [
                            {
                                "type": "text",
                                "created_at": AnyString(),
                                "finished_at": None,
                                "id": AnyString(),
                                "text": "Found: A = 42",
                            },
                        ],
                        "state": "success",
                        "metadata": {},
                    },
                    {
                        "type": "text",
                        "created_at": AnyString(),
                        "finished_at": None,
                        "id": AnyString(),
                        "text": "Both done.",
                    },
                ],
            },
        )

    async def test_same_member_assignments_queue(self) -> None:
        """A second assignment to a parked researcher waits for the first
        one to finish, then runs, and both results land in call order."""
        self.leader.model.set_responses(
            [
                [
                    ChatResponse(
                        content=[
                            ToolCallBlock(
                                id="call-1",
                                name="TeamAssign",
                                input='{"member": "researcher", '
                                '"prompt": "Find A"}',
                            ),
                            ToolCallBlock(
                                id="call-2",
                                name="TeamAssign",
                                input='{"member": "researcher", '
                                '"prompt": "Find B"}',
                            ),
                        ],
                        is_last=True,
                        usage=None,
                    ),
                ],
                _text("Both found."),
            ],
        )
        self.researcher.model.set_responses(
            [
                _tool_call("call-r1", '{"input": "A"}', name="ask_tool"),
                _text("Found A"),
                _text("Found B"),
            ],
        )

        events = await self._run(self.query)

        self.assertEqual(events.count(("REPLY_START", "researcher")), 1)
        self.assertListEqual(
            [
                (_.id, _.state)
                for _ in self.leader.state.get_awaiting_tool_calls("leader")
            ],
            [("call-1", "submitted"), ("call-2", "submitted")],
        )

        # The researcher starts a new reply for the queued call, so the
        # events are drained without mapping owners
        _ = [
            _
            async for _ in self.pipeline.reply_stream(
                UserConfirmResultEvent(
                    reply_id=self.researcher.state.reply_id,
                    confirm_results=[
                        ConfirmResult(
                            confirmed=True,
                            tool_call=self.researcher.state.context[
                                -1
                            ].content[0],
                        ),
                    ],
                ),
            )
        ]

        self.assertDictEqual(
            self.leader.state.context[-1].model_dump(),
            {
                "id": AnyString(),
                "created_at": AnyString(),
                "finished_at": None,
                "finished_reason": None,
                "structured_output": None,
                "error": None,
                "metadata": {},
                "name": "leader",
                "role": "assistant",
                "usage": None,
                "content": [
                    {
                        "type": "tool_call",
                        "created_at": AnyString(),
                        "finished_at": None,
                        "id": "call-1",
                        "name": "TeamAssign",
                        "input": '{"member": "researcher", '
                        '"prompt": "Find A"}',
                        "state": "finished",
                        "suggested_rules": [],
                    },
                    {
                        "type": "tool_call",
                        "created_at": AnyString(),
                        "finished_at": None,
                        "id": "call-2",
                        "name": "TeamAssign",
                        "input": '{"member": "researcher", '
                        '"prompt": "Find B"}',
                        "state": "finished",
                        "suggested_rules": [],
                    },
                    {
                        "type": "tool_result",
                        "created_at": AnyString(),
                        "finished_at": None,
                        "id": "call-1",
                        "name": "TeamAssign",
                        "output": [
                            {
                                "type": "text",
                                "created_at": AnyString(),
                                "finished_at": None,
                                "id": AnyString(),
                                "text": "Found A",
                            },
                        ],
                        "state": "success",
                        "metadata": {},
                    },
                    {
                        "type": "tool_result",
                        "created_at": AnyString(),
                        "finished_at": None,
                        "id": "call-2",
                        "name": "TeamAssign",
                        "output": [
                            {
                                "type": "text",
                                "created_at": AnyString(),
                                "finished_at": None,
                                "id": AnyString(),
                                "text": "Found B",
                            },
                        ],
                        "state": "success",
                        "metadata": {},
                    },
                    {
                        "type": "text",
                        "created_at": AnyString(),
                        "finished_at": None,
                        "id": AnyString(),
                        "text": "Both found.",
                    },
                ],
            },
        )

    async def _cancel(self, inputs: Any, tool: SlowTool) -> list[str]:
        """Cancel the run once the slow tool starts, returning the types of
        the events streamed before and after the cancellation."""
        events = []

        async def consume() -> None:
            async for evt in self.pipeline.reply_stream(inputs):
                events.append(evt.type.value)

        task = asyncio.create_task(consume())
        await tool.started.wait()
        task.cancel()
        await task
        return events

    async def test_cancel_running_member(self) -> None:
        """Cancelling while the researcher runs interrupts it and the leader
        parked on the assignment, so the team takes the next query."""
        slow = SlowTool()
        self.researcher = _agent("researcher", [slow])
        self.pipeline = TeamPipeline(
            leader=self.leader,
            members=[
                TeamMember(agent=self.researcher, description="Researches"),
            ],
        )
        self.leader.model.set_responses(
            [
                _tool_call(
                    "call-1",
                    '{"member": "researcher", "prompt": "Find A"}',
                ),
                _text("Hi again."),
            ],
        )
        self.researcher.model.set_responses(
            [_tool_call("call-r1", "{}", name="slow_tool")],
        )

        events = await self._cancel(self.query, slow)

        self.assertListEqual(
            events[-7:],
            [
                "TOOL_RESULT_START",
                "TOOL_RESULT_TEXT_DELTA",
                "TOOL_RESULT_END",
                "REPLY_END",
                "TOOL_RESULT_TEXT_DELTA",
                "TOOL_RESULT_END",
                "REPLY_END",
            ],
        )
        self.assertDictEqual(
            self.leader.state.context[-1].model_dump(),
            {
                "id": AnyString(),
                "created_at": AnyString(),
                "finished_at": None,
                "finished_reason": None,
                "structured_output": None,
                "error": None,
                "metadata": {},
                "name": "leader",
                "role": "assistant",
                "usage": None,
                "content": [
                    {
                        "type": "tool_call",
                        "created_at": AnyString(),
                        "finished_at": None,
                        "id": "call-1",
                        "name": "TeamAssign",
                        "input": '{"member": "researcher", '
                        '"prompt": "Find A"}',
                        "state": "finished",
                        "suggested_rules": [],
                    },
                    {
                        "type": "tool_result",
                        "created_at": AnyString(),
                        "finished_at": None,
                        "id": "call-1",
                        "name": "TeamAssign",
                        "output": "<system-reminder>The tool call has been "
                        "interrupted by the user.</system-reminder>",
                        "state": "interrupted",
                        "metadata": {},
                    },
                ],
            },
        )
        self.assertListEqual(self.researcher.state.context, [])

        await self._run(UserMsg(name="user", content="Hi"))

        self.assertEqual(
            self.leader.state.context[-1].get_text_content(),
            "Hi again.",
        )

    async def test_cancel_resumed_member(self) -> None:
        """Cancelling a researcher resumed from HITL ends the leader as
        interrupted instead of letting it reason on."""
        slow = SlowTool()
        self.researcher = _agent("researcher", [AskTool(), slow])
        self.pipeline = TeamPipeline(
            leader=self.leader,
            members=[
                TeamMember(agent=self.researcher, description="Researches"),
            ],
        )
        self.leader.model.set_responses(
            [
                _tool_call(
                    "call-1",
                    '{"member": "researcher", "prompt": "Find A"}',
                ),
            ],
        )
        self.researcher.model.set_responses(
            [
                _tool_call("call-r1", '{"input": "A"}', name="ask_tool"),
                _tool_call("call-r2", "{}", name="slow_tool"),
            ],
        )
        await self._run(self.query)

        await self._cancel(
            UserConfirmResultEvent(
                reply_id=self.researcher.state.reply_id,
                confirm_results=[
                    ConfirmResult(
                        confirmed=True,
                        tool_call=self.researcher.state.context[-1].content[0],
                    ),
                ],
            ),
            slow,
        )

        self.assertDictEqual(
            self.leader.state.context[-1].model_dump(),
            {
                "id": AnyString(),
                "created_at": AnyString(),
                "finished_at": None,
                "finished_reason": None,
                "structured_output": None,
                "error": None,
                "metadata": {},
                "name": "leader",
                "role": "assistant",
                "usage": None,
                "content": [
                    {
                        "type": "tool_call",
                        "created_at": AnyString(),
                        "finished_at": None,
                        "id": "call-1",
                        "name": "TeamAssign",
                        "input": '{"member": "researcher", '
                        '"prompt": "Find A"}',
                        "state": "finished",
                        "suggested_rules": [],
                    },
                    {
                        "type": "tool_result",
                        "created_at": AnyString(),
                        "finished_at": None,
                        "id": "call-1",
                        "name": "TeamAssign",
                        "output": "<system-reminder>The tool call has been "
                        "interrupted by the user.</system-reminder>",
                        "state": "interrupted",
                        "metadata": {},
                    },
                ],
            },
        )
        self.assertListEqual(self.researcher.state.context, [])