Skip to content

Enterprise agent platform for building, deploying, and operating AI teammates.

20.1 KB
Raw Blame
"""Inc.3 — concurrent batch execution + ordered backfill (design §4.2.2, §5.5).

``_run_concurrent_batch`` runs a segment of concurrency-safe tool calls under a
Semaphore via ``asyncio.gather`` and back-fills their results into the context
in the original tool-call order. Invariants pinned here:

- I1: ``add_tool_result`` order == input order, even when tools finish out of
  order.
- I2: every ``tool_call_id`` gets exactly one result (including failures).
- real concurrency: the batch overlaps (peak == batch size) rather than running
  serially, and the Semaphore caps the peak.
- exception isolation: one failing tool yields an error result for that call
  while the rest succeed.
"""

from __future__ import annotations

import asyncio

import pytest

from tests.core.agent.concurrency_harness import (
    ConcurrencyTracker,
    FakeRuntime,
    FakeTool,
    RecordingContext,
    make_react,
    make_tool_call,
)
from xagent.core.agent import PatternRuntime, ToolCallInterrupted


async def test_ordered_backfill_despite_out_of_order_completion() -> None:
    names = ["s1", "s2", "s3"]
    tracker = ConcurrencyTracker()
    gates = {name: asyncio.Event() for name in names}
    tools = [
        FakeTool(name, concurrency_safe=True, gate=gates[name], tracker=tracker)
        for name in names
    ]
    pattern = make_react(parallel=True, max_concurrency=3)
    runtime = FakeRuntime()
    context = RecordingContext()
    batch = [
        {**make_tool_call(name), "invocation_id": f"invocation-{name}"}
        for name in names
    ]

    task = asyncio.create_task(
        pattern._run_concurrent_batch(batch, tools, runtime, context)
    )
    # Wait until all three are in-flight, then release them in REVERSE order so
    # completion order is the opposite of input order.
    while tracker.active < 3:
        await asyncio.sleep(0)
    for index, name in enumerate(["s3", "s2", "s1"], start=1):
        gates[name].set()
        while len(tracker.leave_order) < index:
            await asyncio.sleep(0)
    await task

    # Completion was reverse, but backfill preserves input order (I1).
    assert tracker.leave_order == ["s3", "s2", "s1"]
    assert [r["tool_name"] for r in context.tool_results] == names
    assert [r["tool_call_id"] for r in context.tool_results] == [
        tc["id"] for tc in batch
    ]
    assert [r["invocation_id"] for r in context.tool_results] == [
        tc["invocation_id"] for tc in batch
    ]


async def test_every_tool_call_id_gets_exactly_one_result() -> None:
    names = ["s1", "s2", "s3"]
    tools = [FakeTool(name, concurrency_safe=True) for name in names]
    pattern = make_react(parallel=True, max_concurrency=3)
    context = RecordingContext()
    batch = [make_tool_call(name) for name in names]

    await pattern._run_concurrent_batch(batch, tools, FakeRuntime(), context)

    ids = [r["tool_call_id"] for r in context.tool_results]
    assert sorted(ids) == sorted(tc["id"] for tc in batch)
    assert len(ids) == len(set(ids)) == 3


async def test_batch_runs_concurrently() -> None:
    names = ["s1", "s2", "s3"]
    tracker = ConcurrencyTracker()
    tools = [
        FakeTool(name, concurrency_safe=True, delay=0.02, tracker=tracker)
        for name in names
    ]
    pattern = make_react(parallel=True, max_concurrency=3)
    context = RecordingContext()
    batch = [make_tool_call(name) for name in names]

    await pattern._run_concurrent_batch(batch, tools, FakeRuntime(), context)

    # All three overlapped (a serial run would never exceed 1).
    assert tracker.peak == 3


async def test_semaphore_caps_concurrency() -> None:
    names = ["s1", "s2", "s3", "s4"]
    tracker = ConcurrencyTracker()
    tools = [
        FakeTool(name, concurrency_safe=True, delay=0.02, tracker=tracker)
        for name in names
    ]
    pattern = make_react(parallel=True, max_concurrency=2)
    context = RecordingContext()
    batch = [make_tool_call(name) for name in names]

    await pattern._run_concurrent_batch(batch, tools, FakeRuntime(), context)

    assert tracker.peak <= 2
    assert len(context.tool_results) == 4


async def test_exception_isolation_within_batch() -> None:
    tools = [
        FakeTool("s1", concurrency_safe=True),
        FakeTool("boom", concurrency_safe=True, raises=RuntimeError("kaboom")),
        FakeTool("s3", concurrency_safe=True),
    ]
    pattern = make_react(parallel=True, max_concurrency=3)
    context = RecordingContext()
    batch = [make_tool_call(name) for name in ["s1", "boom", "s3"]]

    results = await pattern._run_concurrent_batch(batch, tools, FakeRuntime(), context)

    # All three back-filled in order; the middle one is an error result.
    assert [r["tool_name"] for r in context.tool_results] == ["s1", "boom", "s3"]
    assert results[0]["success"] is True
    assert results[1]["success"] is False
    assert "kaboom" in results[1]["error"]
    assert results[2]["success"] is True


async def test_infra_callback_failure_marks_ledger_terminal() -> None:
    # If an infra callback (on_tool_start) raises after the "running" record is
    # written, the ledger must reach a terminal state. The consecutive-count
    # walks only count {completed, failed}, so a record stuck at "running" would
    # be silently skipped and undercount the repeated-tool-decision triggers.
    class _BoomOnStart(FakeRuntime):
        async def on_tool_start(self, *, tool_call: dict) -> None:
            raise RuntimeError("trace backend down")

    tools = [FakeTool("s1", concurrency_safe=True)]
    pattern = make_react(parallel=True)
    call = make_tool_call("s1")
    original_call = dict(call)

    with pytest.raises(RuntimeError, match="trace backend down"):
        await pattern._execute_tool_safely(call, tools, _BoomOnStart())

    assert pattern._record_for_tool_call_id(call["id"]).status == "failed"
    assert pattern._record_for_tool_call_id(call["id"]).invocation_id
    assert call == original_call


async def test_concurrent_batch_assigns_ids_to_id_less_calls() -> None:
    # An id-less batch must receive distinct fallback and invocation identities
    # before scheduling without mutating caller-owned input. The prepared
    # objects then remain continuous through ledger writes and result backfill.
    tools = [
        FakeTool("s1", concurrency_safe=True),
        FakeTool("s2", concurrency_safe=True),
    ]
    pattern = make_react(parallel=True, max_concurrency=2)
    runtime = FakeRuntime()
    # An active ReAct step makes _with_runtime_step return a *copy*, so stamping
    # the id after the transform (rather than before) would miss the original
    # batch dict that backfill/reorder iterate over.
    runtime.active_react_step_id = "step-x"
    context = RecordingContext()
    batch = [make_tool_call("s1", id=""), make_tool_call("s2", id="")]
    original_batch = [dict(call) for call in batch]

    await pattern._run_concurrent_batch(batch, tools, runtime, context)

    # Exactly one terminal record per call (no orphan stuck at "running").
    assert len(pattern.tool_ledger) == 2
    assert all(record.status == "completed" for record in pattern.tool_ledger.values())
    # Context tool_call_ids are non-empty and match the provider-id index in
    # input order (I2 + I3).
    ctx_ids = [r["tool_call_id"] for r in context.tool_results]
    assert all(ctx_ids)
    assert ctx_ids == list(pattern.tool_call_id_index.keys())
    ctx_invocation_ids = [r["invocation_id"] for r in context.tool_results]
    assert all(ctx_invocation_ids)
    assert len(set(ctx_invocation_ids)) == 2
    assert ctx_invocation_ids == [
        record.invocation_id for record in pattern.tool_ledger.values()
    ]
    assert batch == original_batch


async def test_pending_entry_prepares_legacy_concurrent_calls_once_for_all_consumers() -> (
    None
):
    waiting_result = {
        "success": False,
        "status": "waiting_for_user",
        "interaction_id": "needs-input",
        "message": "Continue?",
        "interactions": [],
    }
    tools = [
        FakeTool("first", concurrency_safe=True, result=waiting_result),
        FakeTool("second", concurrency_safe=True, result=waiting_result),
    ]
    pattern = make_react(parallel=True, max_concurrency=2)
    original_calls = [
        {"name": "first", "args": {}},
        {"name": "second", "args": {}},
    ]
    pattern.pending_tool_calls = original_calls
    context = RecordingContext()

    result = await pattern._execute_pending_tool_calls(
        context=context,
        tools=tools,
        llm=None,
        runtime=FakeRuntime(),
    )

    assert result is not None and result["status"] == "waiting_for_user"
    assert original_calls == [
        {"name": "first", "args": {}},
        {"name": "second", "args": {}},
    ]
    context_ids = [entry["tool_call_id"] for entry in context.tool_results]
    request_ids = [
        request["tool_call_id"]
        for request in pattern.waiting_for_user_request["requests"]
    ]
    assert all(context_ids)
    assert len(set(context_ids)) == 2
    assert request_ids == context_ids
    assert [
        record.tool_call_id for record in pattern.tool_ledger.values()
    ] == context_ids
    assert [entry["invocation_id"] for entry in context.tool_results] == [None, None]
    assert [
        request["invocation_id"]
        for request in pattern.waiting_for_user_request["requests"]
    ] == [None, None]
    assert [record.invocation_id for record in pattern.tool_ledger.values()] == [
        None,
        None,
    ]


async def test_concurrent_batch_propagates_infra_callback_failure() -> None:
    # An infra callback failure (on_tool_start) is a real exception, not a tool
    # failure. The serial path lets it propagate and halt the turn; the
    # concurrent path must do the same instead of mis-reporting infrastructure
    # breakage to the model as a tool-failure result. The exception is re-raised
    # after successful neighbors are back-filled so an explicit retry cannot
    # repeat work that already completed.
    class _BoomOnFirst(FakeRuntime):
        async def on_tool_start(self, *, tool_call: dict) -> None:
            if tool_call["name"] == "boom":
                raise RuntimeError("trace down")
            await super().on_tool_start(tool_call=tool_call)

    tools = [
        FakeTool("s1", concurrency_safe=True),
        FakeTool("boom", concurrency_safe=True),
        FakeTool("s3", concurrency_safe=True),
    ]
    pattern = make_react(parallel=True, max_concurrency=3)
    context = RecordingContext()
    batch = [make_tool_call(name) for name in ["s1", "boom", "s3"]]

    with pytest.raises(RuntimeError, match="trace down"):
        await pattern._run_concurrent_batch(batch, tools, _BoomOnFirst(), context)

    assert [result["tool_name"] for result in context.tool_results] == ["s1", "s3"]
    # The failing call still reaches a terminal ledger state (I3 walk).
    assert pattern._record_for_tool_call_id(batch[1]["id"]).status == "failed"


async def test_mixed_infra_failure_and_interrupt_reconciles_batch_before_raise() -> (
    None
):
    class _BoomOnStart(PatternRuntime):
        async def on_tool_start(self, *, tool_call: dict) -> None:
            if tool_call["name"] == "boom":
                raise RuntimeError("trace down")
            await super().on_tool_start(tool_call=tool_call)

    blocked = asyncio.Event()
    tools = [
        FakeTool("done", concurrency_safe=True),
        FakeTool("boom", concurrency_safe=True),
        FakeTool("paused", concurrency_safe=True, gate=blocked),
    ]
    pattern = make_react(parallel=True, max_concurrency=3)
    context = RecordingContext()
    batch = [make_tool_call(name) for name in ("done", "boom", "paused")]
    pattern.pending_tool_calls = list(batch)
    runtime = _BoomOnStart(execution_id="mixed-batch-failure")

    task = asyncio.create_task(
        pattern._run_concurrent_batch(batch, tools, runtime, context)
    )
    while not tools[0].calls or not tools[2].calls:
        await asyncio.sleep(0)
    runtime.request_interrupt("pause mixed batch")

    with pytest.raises(RuntimeError, match="trace down"):
        await task

    assert [result["tool_name"] for result in context.tool_results] == ["done"]
    assert [call["name"] for call in pattern.pending_tool_calls] == [
        "boom",
        "paused",
    ]
    assert pattern._record_for_tool_call_id(batch[0]["id"]).status == "completed"
    assert pattern._record_for_tool_call_id(batch[1]["id"]).status == "failed"
    assert pattern._record_for_tool_call_id(batch[2]["id"]).status == "interrupted"


async def test_interrupt_filter_uses_batch_position_when_ids_repeat() -> None:
    blocked = asyncio.Event()
    tools = [
        FakeTool("done", concurrency_safe=True),
        FakeTool("paused", concurrency_safe=True, gate=blocked),
    ]
    pattern = make_react(parallel=True, max_concurrency=2)
    context = RecordingContext()
    batch = [
        make_tool_call("done", id="provider-duplicate"),
        make_tool_call("paused", id="provider-duplicate"),
    ]
    pattern.pending_tool_calls = list(batch)
    runtime = PatternRuntime(execution_id="duplicate-provider-ids")

    task = asyncio.create_task(
        pattern._run_concurrent_batch(batch, tools, runtime, context)
    )
    while not tools[0].calls or not tools[1].calls:
        await asyncio.sleep(0)
    runtime.request_interrupt("pause duplicate ids")

    with pytest.raises(ToolCallInterrupted, match="pause duplicate ids"):
        await task

    assert [result["tool_name"] for result in context.tool_results] == ["done"]
    [pending_call] = pattern.pending_tool_calls
    assert pending_call == batch[1]


async def test_interrupt_keeps_only_unfinished_legacy_calls_pending() -> None:
    # Calls restored from a pre-identity checkpoint carry no invocation id.
    # Every preparation layer must hand back the same prepared object, or the
    # batch's identity-based reconciliation cannot drop the completed call.
    blocked = asyncio.Event()
    tools = [
        FakeTool("done", concurrency_safe=True),
        FakeTool("paused", concurrency_safe=True, gate=blocked),
    ]
    pattern = make_react(parallel=True, max_concurrency=2)
    context = RecordingContext()
    pattern.pending_tool_calls = [
        {"id": "legacy-done", "name": "done", "args": {}},
        {"id": "legacy-paused", "name": "paused", "args": {}},
    ]
    runtime = PatternRuntime(execution_id="legacy-interrupt")

    task = asyncio.create_task(
        pattern._execute_pending_tool_calls(
            context=context, tools=tools, llm=None, runtime=runtime
        )
    )
    while not tools[0].calls or not tools[1].calls:
        await asyncio.sleep(0)
    runtime.request_interrupt("pause legacy batch")

    with pytest.raises(ToolCallInterrupted, match="pause legacy batch"):
        await task

    assert [result["tool_name"] for result in context.tool_results] == ["done"]
    [pending_call] = pattern.pending_tool_calls
    assert pending_call["id"] == "legacy-paused"
    assert pending_call["invocation_id"] is None
    assert pattern._prepare_tool_call_identity(pending_call) is pending_call


# --- Inc.4: tool_ledger ordering after a concurrent batch (I3) -------------


def test_reorder_ledger_for_batch_restores_input_order() -> None:
    # Simulate interleaved completion writing batch records out of order, with
    # a pre-existing record before the batch that must keep its position.
    pattern = make_react(parallel=True)
    pattern._record_tool_call(
        make_tool_call("earlier", id="e0"), status="completed", result={"success": True}
    )
    batch = [
        make_tool_call("s1", id="b1"),
        make_tool_call("s2", id="b2"),
        make_tool_call("s3", id="b3"),
    ]
    for tool_call in (batch[2], batch[0], batch[1]):  # scrambled order
        pattern._record_tool_call(
            tool_call, status="completed", result={"success": True}
        )
    assert list(pattern.tool_call_id_index.keys()) == ["e0", "b3", "b1", "b2"]

    pattern._reorder_ledger_for_batch(batch)

    assert list(pattern.tool_call_id_index.keys()) == ["e0", "b1", "b2", "b3"]
    assert [record.tool_call_id for record in pattern.tool_ledger.values()] == [
        "e0",
        "b1",
        "b2",
        "b3",
    ]


def test_reorder_moves_ids_reused_from_an_earlier_step_to_the_tail() -> None:
    # Providers without ids get tool_call_0..N synthesized per response, so a
    # later concurrent batch reuses ids an earlier serial step already used.
    pattern = make_react(parallel=True)
    for index, status in enumerate(["completed", "completed", "failed"]):
        pattern._record_tool_call(
            make_tool_call("web_search", id=f"tool_call_{index}"),
            status=status,
            result={"success": status == "completed"},
        )
    batch = [
        make_tool_call("web_search", id="tool_call_0"),
        make_tool_call("web_search", id="tool_call_1"),
    ]
    for tool_call in reversed(batch):
        pattern._record_tool_call(
            tool_call, status="completed", result={"success": True}
        )

    pattern._reorder_ledger_for_batch(batch)

    projected = pattern._provider_compatible_tool_records()
    assert [record.tool_call_id for record in projected] == [
        "tool_call_2",
        "tool_call_0",
        "tool_call_1",
    ]
    assert [record.invocation_id for record in projected[1:]] == [
        tool_call["invocation_id"] for tool_call in batch
    ]
    group = pattern._tool_decision_group_for_name("web_search")
    assert len(pattern._consecutive_successful_tool_group_records(group)) == 2


async def test_concurrent_batch_keeps_ledger_in_input_order() -> None:
    names = ["s1", "s2", "s3"]
    tracker = ConcurrencyTracker()
    gates = {name: asyncio.Event() for name in names}
    tools = [
        FakeTool(name, concurrency_safe=True, gate=gates[name], tracker=tracker)
        for name in names
    ]
    pattern = make_react(parallel=True, max_concurrency=3)
    context = RecordingContext()
    batch = [make_tool_call(name) for name in names]

    task = asyncio.create_task(
        pattern._run_concurrent_batch(batch, tools, FakeRuntime(), context)
    )
    while tracker.active < 3:
        await asyncio.sleep(0)
    for index, name in enumerate(["s3", "s2", "s1"], start=1):  # reverse completion
        gates[name].set()
        while len(tracker.leave_order) < index:
            await asyncio.sleep(0)
    await task

    assert [tc["id"] for tc in batch] == list(pattern.tool_call_id_index.keys())
    assert [tc["invocation_id"] for tc in batch] == list(pattern.tool_ledger.keys())


async def test_concurrent_batch_updates_consecutive_group_count() -> None:
    group = "search"
    names = ["s1", "s2", "s3"]
    tools = [
        FakeTool(name, concurrency_safe=True, decision_group=group) for name in names
    ]
    pattern = make_react(parallel=True, max_concurrency=3)
    pattern._tool_decision_groups_by_name = pattern._tool_decision_groups_for_tools(
        tools
    )
    context = RecordingContext()
    batch = [make_tool_call(name) for name in names]

    await pattern._run_concurrent_batch(batch, tools, FakeRuntime(), context)

    assert len(pattern._consecutive_successful_tool_group_records(group)) == 3


async def test_mcp_error_result_stops_consecutive_successful_group_count() -> None:
    group = "search"
    tool = FakeTool("s1", concurrency_safe=True, decision_group=group)
    pattern = make_react(parallel=True)
    pattern._tool_decision_groups_by_name = pattern._tool_decision_groups_for_tools(
        [tool]
    )
    pattern._record_tool_call(
        make_tool_call("s1"),
        status="completed",
        result={"content": [{"text": "failed"}], "is_error": True},
    )

    assert len(pattern._consecutive_successful_tool_group_records(group)) == 0


async def test_concurrent_batch_with_failure_counts_work_calls() -> None:
    tools = [
        FakeTool("s1", concurrency_safe=True),
        FakeTool("boom", concurrency_safe=True, raises=RuntimeError("x")),
        FakeTool("s3", concurrency_safe=True),
    ]
    pattern = make_react(parallel=True, max_concurrency=3)
    pattern._tool_decision_groups_by_name = pattern._tool_decision_groups_for_tools(
        tools
    )
    context = RecordingContext()
    batch = [make_tool_call(name) for name in ["s1", "boom", "s3"]]

    await pattern._run_concurrent_batch(batch, tools, FakeRuntime(), context)

    # completed + failed both count as work calls.
    assert pattern._consecutive_work_tool_call_count() == 3