AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(copilot): Execute parallel tool calls concurrently (#12165)
Otto0b151f64e8e6
diff --git a/autogpt_platform/backend/backend/copilot/parallel_tool_calls_test.py b/autogpt_platform/backend/backend/copilot/parallel_tool_calls_test.pynew file mode 100644index 0000000000..9916d4e884--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/parallel_tool_calls_test.py@@ -0,0 +1,272 @@+"""Tests for parallel tool call execution in CoPilot.++These tests mock _yield_tool_call to avoid importing the full copilot stack+which requires Prisma, DB connections, etc.+"""++import asyncio+import time+from typing import Any, cast++import pytest+++async def test_parallel_tool_calls_run_concurrently():+ """Multiple tool calls should complete in ~max(delays), not sum(delays)."""+ # Import here to allow module-level mocking if needed+ from backend.copilot.response_model import (+ StreamToolInputAvailable,+ StreamToolOutputAvailable,+ )+ from backend.copilot.service import _execute_tool_calls_parallel++ n_tools = 3+ delay_per_tool = 0.2+ tool_calls = [+ {+ "id": f"call_{i}",+ "type": "function",+ "function": {"name": f"tool_{i}", "arguments": "{}"},+ }+ for i in range(n_tools)+ ]++ # Minimal session mock+ class FakeSession:+ session_id = "test"+ user_id = "test"++ def __init__(self):+ self.messages = []++ original_yield = None++ async def fake_yield(tc_list, idx, sess, lock=None):+ yield StreamToolInputAvailable(+ toolCallId=tc_list[idx]["id"],+ toolName=tc_list[idx]["function"]["name"],+ input={},+ )+ await asyncio.sleep(delay_per_tool)+ yield StreamToolOutputAvailable(+ toolCallId=tc_list[idx]["id"],+ toolName=tc_list[idx]["function"]["name"],+ output="{}",+ )++ import backend.copilot.service as svc++ original_yield = svc._yield_tool_call+ svc._yield_tool_call = fake_yield+ try:+ start = time.monotonic()+ events = []+ async for event in _execute_tool_calls_parallel(+ tool_calls, cast(Any, FakeSession())+ ):+ events.append(event)+ elapsed = time.monotonic() - start+ finally:+ svc._yield_tool_call = original_yield++ assert len(events) == n_tools * 2+ # Parallel: should take ~delay, not ~n*delay+ assert elapsed < delay_per_tool * (+ n_tools - 0.5+ ), f"Took {elapsed:.2f}s, expected parallel (~{delay_per_tool}s)"+++async def test_single_tool_call_works():+ """Single tool call should work identically."""+ from backend.copilot.response_model import (+ StreamToolInputAvailable,+ StreamToolOutputAvailable,+ )+ from backend.copilot.service import _execute_tool_calls_parallel++ tool_calls = [+ {+ "id": "call_0",+ "type": "function",+ "function": {"name": "t", "arguments": "{}"},+ }+ ]++ class FakeSession:+ session_id = "test"+ user_id = "test"++ def __init__(self):+ self.messages = []++ async def fake_yield(tc_list, idx, sess, lock=None):+ yield StreamToolInputAvailable(toolCallId="call_0", toolName="t", input={})+ yield StreamToolOutputAvailable(toolCallId="call_0", toolName="t", output="{}")++ import backend.copilot.service as svc++ orig = svc._yield_tool_call+ svc._yield_tool_call = fake_yield+ try:+ events = [+ e+ async for e in _execute_tool_calls_parallel(+ tool_calls, cast(Any, FakeSession())+ )+ ]+ finally:+ svc._yield_tool_call = orig++ assert len(events) == 2+++async def test_retryable_error_propagates():+ """Retryable errors should be raised after all tools finish."""+ from backend.copilot.response_model import StreamToolOutputAvailable+ from backend.copilot.service import _execute_tool_calls_parallel++ tool_calls = [+ {+ "id": f"call_{i}",+ "type": "function",+ "function": {"name": f"t_{i}", "arguments": "{}"},+ }+ for i in range(2)+ ]++ class FakeSession:+ session_id = "test"+ user_id = "test"++ def __init__(self):+ self.messages = []++ async def fake_yield(tc_list, idx, sess, lock=None):+ if idx == 1:+ raise KeyError("bad")+ from backend.copilot.response_model import StreamToolInputAvailable++ yield StreamToolInputAvailable(+ toolCallId=tc_list[idx]["id"], toolName="t_0", input={}+ )+ await asyncio.sleep(0.05)+ yield StreamToolOutputAvailable(+ toolCallId=tc_list[idx]["id"], toolName="t_0", output="{}"+ )++ import backend.copilot.service as svc++ orig = svc._yield_tool_call+ svc._yield_tool_call = fake_yield+ try:+ events = []+ with pytest.raises(KeyError):+ async for event in _execute_tool_calls_parallel(+ tool_calls, cast(Any, FakeSession())+ ):+ events.append(event)+ # First tool's events should still be yielded+ assert any(isinstance(e, StreamToolOutputAvailable) for e in events)+ finally:+ svc._yield_tool_call = orig+++async def test_session_lock_shared():+ """All parallel tools should receive the same lock instance."""+ from backend.copilot.response_model import (+ StreamToolInputAvailable,+ StreamToolOutputAvailable,+ )+ from backend.copilot.service import _execute_tool_calls_parallel++ tool_calls = [+ {+ "id": f"call_{i}",+ "type": "function",+ "function": {"name": f"t_{i}", "arguments": "{}"},+ }+ for i in range(3)+ ]++ class FakeSession:+ session_id = "test"+ user_id = "test"++ def __init__(self):+ self.messages = []++ observed_locks = []++ async def fake_yield(tc_list, idx, sess, lock=None):+ observed_locks.append(lock)+ yield StreamToolInputAvailable(+ toolCallId=tc_list[idx]["id"], toolName=f"t_{idx}", input={}+ )+ yield StreamToolOutputAvailable(+ toolCallId=tc_list[idx]["id"], toolName=f"t_{idx}", output="{}"+ )++ import backend.copilot.service as svc++ orig = svc._yield_tool_call+ svc._yield_tool_call = fake_yield+ try:+ async for _ in _execute_tool_calls_parallel(+ tool_calls, cast(Any, FakeSession())+ ):+ pass+ finally:+ svc._yield_tool_call = orig++ assert len(observed_locks) == 3+ assert observed_locks[0] is observed_locks[1] is observed_locks[2]+ assert isinstance(observed_locks[0], asyncio.Lock)+++async def test_cancellation_cleans_up():+ """Generator close should cancel in-flight tasks."""+ from backend.copilot.response_model import StreamToolInputAvailable+ from backend.copilot.service import _execute_tool_calls_parallel++ tool_calls = [+ {+ "id": f"call_{i}",+ "type": "function",+ "function": {"name": f"t_{i}", "arguments": "{}"},+ }+ for i in range(2)+ ]++ class FakeSession:+ session_id = "test"+ user_id = "test"++ def __init__(self):+ self.messages = []++ started = asyncio.Event()++ async def fake_yield(tc_list, idx, sess, lock=None):+ yield StreamToolInputAvailable(+ toolCallId=tc_list[idx]["id"], toolName=f"t_{idx}", input={}+ )+ started.set()+ await asyncio.sleep(10) # simulate long-running++ import backend.copilot.service as svc++ orig = svc._yield_tool_call+ svc._yield_tool_call = fake_yield+ try:+ gen = _execute_tool_calls_parallel(tool_calls, cast(Any, FakeSession()))+ await gen.__anext__() # get first event+ await started.wait()+ await gen.aclose() # close generator+ finally:+ svc._yield_tool_call = orig+ # If we get here without hanging, cleanup workeddiff --git a/autogpt_platform/backend/backend/copilot/service.py b/autogpt_platform/backend/backend/copilot/service.pyindex a6a0bcb9bb..914e565ba6 100644--- a/autogpt_platform/backend/backend/copilot/service.py+++ b/autogpt_platform/backend/backend/copilot/service.py@@ -1232,23 +1232,10 @@ async def _stream_chat_chunks(},)- # Yield all accumulated tool calls after the stream is complete- # This ensures all tool call arguments have been fully received- for idx, tool_call in enumerate(tool_calls):- try:- async for tc in _yield_tool_call(tool_calls, idx, session):- yield tc- except (orjson.JSONDecodeError, KeyError, TypeError) as e:- logger.error(- f"Failed to parse tool call {idx}: {e}",- exc_info=True,- extra={"tool_call": tool_call},- )- yield StreamError(- errorText=f"Invalid tool call arguments for tool {tool_call.get('function', {}).get('name', 'unknown')}: {e}",- )- # Re-raise to trigger retry logic in the parent function- raise+ # Execute all accumulated tool calls in parallel+ # Events are yielded as they arrive from each concurrent tool+ async for event in _execute_tool_calls_parallel(tool_calls, session):+ yield eventtotal_time = (time_module.perf_counter() - stream_chunks_start) * 1000logger.info(@@ -1326,10 +1313,91 @@ async def _stream_chat_chunks(return+async def _with_optional_lock(+ lock: asyncio.Lock | None,+ coro_fn: Any,+) -> Any:+ """Run *coro_fn()* under *lock* when provided, otherwise run directly."""+ if lock:+ async with lock:+ return await coro_fn()+ return await coro_fn()+++async def _execute_tool_calls_parallel(+ tool_calls: list[dict[str, Any]],+ session: ChatSession,+) -> AsyncGenerator[StreamBaseResponse, None]:+ """Execute all tool calls concurrently, yielding stream events as they arrive.++ Each tool runs as an ``asyncio.Task``, pushing events into a shared queue.+ A ``session_lock`` serialises session-state mutations (long-running tool+ bookkeeping, ``run_agent`` counters).+ """+ queue: asyncio.Queue[StreamBaseResponse | None] = asyncio.Queue()+ session_lock = asyncio.Lock()+ n_tools = len(tool_calls)+ retryable_errors: list[Exception] = []++ async def _run_tool(idx: int) -> None:+ tool_name = tool_calls[idx].get("function", {}).get("name", "unknown")+ tool_call_id = tool_calls[idx].get("id", f"unknown_{idx}")+ try:+ async for event in _yield_tool_call(tool_calls, idx, session, session_lock):+ await queue.put(event)+ except (orjson.JSONDecodeError, KeyError, TypeError) as e:+ logger.error(+ f"Failed to parse tool call {idx} ({tool_name}): {e}",+ exc_info=True,+ )+ retryable_errors.append(e)+ except Exception as e:+ # Infrastructure / setup errors — emit an error output so the+ # client always sees a terminal event and doesn't hang.+ logger.error(f"Tool call {idx} ({tool_name}) failed: {e}", exc_info=True)+ await queue.put(+ StreamToolOutputAvailable(+ toolCallId=tool_call_id,+ toolName=tool_name,+ output=ErrorResponse(+ message=f"Tool execution failed: {e!s}",+ error=type(e).__name__,+ session_id=session.session_id,+ ).model_dump_json(),+ success=False,+ )+ )+ finally:+ await queue.put(None) # sentinel++ tasks = [asyncio.create_task(_run_tool(idx)) for idx in range(n_tools)]+ try:+ finished = 0+ while finished < n_tools:+ event = await queue.get()+ if event is None:+ finished += 1+ else:+ yield event+ if retryable_errors:+ if len(retryable_errors) > 1:+ logger.warning(+ f"{len(retryable_errors)} tool calls had retryable errors; "+ f"re-raising first to trigger retry"+ )+ raise retryable_errors[0]+ finally:+ for t in tasks:+ if not t.done():+ t.cancel()+ await asyncio.gather(*tasks, return_exceptions=True)++async def _yield_tool_call(tool_calls: list[dict[str, Any]],yield_idx: int,session: ChatSession,+ session_lock: asyncio.Lock | None = None,) -> AsyncGenerator[StreamBaseResponse, None]:"""Yield a tool call and its execution result.@@ -1427,8 +1495,7 @@ async def _yield_tool_call("check back in a few minutes.")- # Track appended messages for rollback on failure- assistant_message: ChatMessage | None = None+ # Track appended message for rollback on failurepending_message: ChatMessage | None = None# Wrap session save and task creation in try-except to release lock on failure@@ -1443,22 +1510,24 @@ async def _yield_tool_call(operation_id=operation_id,)- # Attach the tool_call to the current turn's assistant message- # (or create one if this is a tool-only response with no text).- session.add_tool_call_to_current_turn(tool_calls[yield_idx])+ # Attach tool_call and save pending result — lock serialises+ # concurrent session mutations during parallel execution.+ async def _save_pending() -> None:+ nonlocal pending_message+ session.add_tool_call_to_current_turn(tool_calls[yield_idx])+ pending_message = ChatMessage(+ role="tool",+ content=OperationPendingResponse(+ message=pending_msg,+ operation_id=operation_id,+ tool_name=tool_name,+ ).model_dump_json(),+ tool_call_id=tool_call_id,+ )+ session.messages.append(pending_message)+ await upsert_chat_session(session)- # Then save pending tool result- pending_message = ChatMessage(- role="tool",- content=OperationPendingResponse(- message=pending_msg,- operation_id=operation_id,- tool_name=tool_name,- ).model_dump_json(),- tool_call_id=tool_call_id,- )- session.messages.append(pending_message)- await upsert_chat_session(session)+ await _with_optional_lock(session_lock, _save_pending)logger.info(f"Saved pending operation {operation_id} (task_id={task_id}) "f"for tool {tool_name} in session {session.session_id}"@@ -1482,19 +1551,13 @@ async def _yield_tool_call(# Associate the asyncio task with the stream registry taskawait stream_registry.set_task_asyncio_task(task_id, bg_task)except Exception as e:- # Roll back appended messages to prevent data corruption on subsequent saves- if (- pending_message- and session.messages- and session.messages[-1] == pending_message- ):- session.messages.pop()- if (- assistant_message- and session.messages- and session.messages[-1] == assistant_message- ):- session.messages.pop()+ # Roll back appended messages — use identity-based removal so+ # it works even when other parallel tools have appended after us.+ async def _rollback() -> None:+ if pending_message and pending_message in session.messages:+ session.messages.remove(pending_message)++ await _with_optional_lock(session_lock, _rollback)# Release the Redis lock since the background task won't be spawnedawait _mark_operation_completed(tool_call_id)