AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend/copilot): prevent duplicate block execution from pre-launch arg mismatch (#12632)
Zamil Majdy8aae7751dcf7
diff --git a/autogpt_platform/backend/backend/copilot/sdk/service.py b/autogpt_platform/backend/backend/copilot/sdk/service.pyindex 64ff20b399..dca5c8683b 100644--- a/autogpt_platform/backend/backend/copilot/sdk/service.py+++ b/autogpt_platform/backend/backend/copilot/sdk/service.py@@ -81,11 +81,9 @@ from .env import build_sdk_env # noqa: F401 — re-export for backward compatfrom .response_adapter import SDKResponseAdapterfrom .security_hooks import create_security_hooksfrom .tool_adapter import (- cancel_pending_tool_tasks,create_copilot_mcp_server,get_copilot_tool_names,get_sdk_disallowed_tools,- pre_launch_tool_call,reset_stash_event,reset_tool_failure_counters,set_execution_context,@@ -1335,28 +1333,17 @@ async def _run_stream_attempt(ended_with_stream_error = Truebreak- # Parallel tool execution: pre-launch every ToolUseBlock as an- # asyncio.Task the moment its AssistantMessage arrives. The SDK- # sends one AssistantMessage per tool call when issuing parallel- # calls, so each message is pre-launched independently. The MCP- # handlers will await the already-running task instead of executing- # fresh, making all concurrent tool calls run in parallel.- #- # Also determine if the message is a tool-only batch (all content+ # Determine if the message is a tool-only batch (all content# items are ToolUseBlocks) — such messages have no text output yet,# so we skip the wait_for_stash flush below.+ #+ # Note: parallel execution of tools is handled natively by the+ # SDK CLI via readOnlyHint annotations on tool definitions.is_tool_only = Falseif isinstance(sdk_msg, AssistantMessage) and sdk_msg.content:- is_tool_only = True- # NOTE: Pre-launches are sequential (each await completes- # file-ref expansion before the next starts). This is fine- # since expansion is typically sub-ms; a future optimisation- # could gather all pre-launches concurrently.- for tool_use in sdk_msg.content:- if isinstance(tool_use, ToolUseBlock):- await pre_launch_tool_call(tool_use.name, tool_use.input)- else:- is_tool_only = False+ is_tool_only = all(+ isinstance(item, ToolUseBlock) for item in sdk_msg.content+ )# Race-condition fix: SDK hooks (PostToolUse) are# executed asynchronously via start_soon() — the next@@ -2047,10 +2034,6 @@ async def stream_chat_completion_sdk(if not isinstance(event, StreamHeartbeat):events_yielded += 1yield event- # Cancel any pre-launched tasks that were never dispatched- # by the SDK (e.g. edge-case SDK behaviour changes). Symmetric- # with the three error-path await cancel_pending_tool_tasks() calls.- await cancel_pending_tool_tasks()break # Stream completed — exit retry loopexcept asyncio.CancelledError:logger.warning(@@ -2059,9 +2042,6 @@ async def stream_chat_completion_sdk(attempt + 1,_MAX_STREAM_ATTEMPTS,)- # Cancel any pre-launched tasks so they don't continue executing- # against a rolled-back or abandoned session.- await cancel_pending_tool_tasks()raiseexcept _HandledStreamError as exc:# _run_stream_attempt already yielded a StreamError and@@ -2093,8 +2073,6 @@ async def stream_chat_completion_sdk(retryable=True,)ended_with_stream_error = True- # Cancel any pre-launched tasks from the failed attempt.- await cancel_pending_tool_tasks()breakexcept Exception as e:stream_err = e@@ -2111,9 +2089,6 @@ async def stream_chat_completion_sdk(exc_info=True,)session.messages = session.messages[:pre_attempt_msg_count]- # Cancel any pre-launched tasks from the failed attempt so they- # don't continue executing against the rolled-back session.- await cancel_pending_tool_tasks()if events_yielded > 0:# Events were already sent to the frontend and cannot be# unsent. Retrying would produce duplicate/inconsistentdiff --git a/autogpt_platform/backend/backend/copilot/sdk/tool_adapter.py b/autogpt_platform/backend/backend/copilot/sdk/tool_adapter.pyindex 10e97a31f1..2c6378e640 100644--- a/autogpt_platform/backend/backend/copilot/sdk/tool_adapter.py+++ b/autogpt_platform/backend/backend/copilot/sdk/tool_adapter.py@@ -14,6 +14,7 @@ from contextvars import ContextVarfrom typing import TYPE_CHECKING, Anyfrom claude_agent_sdk import create_sdk_mcp_server, tool+from mcp.types import ToolAnnotationsfrom backend.copilot.context import (_current_permissions,@@ -53,14 +54,6 @@ _MCP_MAX_CHARS = 500_000MCP_SERVER_NAME = "copilot"MCP_TOOL_PREFIX = f"mcp__{MCP_SERVER_NAME}__"-# Map from tool_name -> Queue of pre-launched (task, args) pairs.-# Initialised per-session in set_execution_context() so concurrent sessions-# never share the same dict.-_TaskQueueItem = tuple[asyncio.Task[dict[str, Any]], dict[str, Any]]-_tool_task_queues: ContextVar[dict[str, asyncio.Queue[_TaskQueueItem]] | None] = (- ContextVar("_tool_task_queues", default=None)-)-# Stash for MCP tool outputs before the SDK potentially truncates them.# Keyed by tool_name → full output string. Consumed (popped) by the# response adapter when it builds StreamToolOutputAvailable.@@ -115,7 +108,6 @@ def set_execution_context(_current_permissions.set(permissions)_pending_tool_outputs.set({})_stash_event.set(asyncio.Event())- _tool_task_queues.set({})_consecutive_tool_failures.set({})@@ -132,48 +124,6 @@ def reset_stash_event() -> None:event.clear()-async def cancel_pending_tool_tasks() -> None:- """Cancel all queued pre-launched tasks for the current execution context.-- Call this when a stream attempt aborts (error, cancellation) to prevent- pre-launched tasks from continuing to execute against a rolled-back session.- Tasks that are already done are skipped; in-flight tasks are cancelled and- awaited so that any cleanup (``finally`` blocks, DB rollbacks) completes- before the next retry starts.- """- queues = _tool_task_queues.get()- if not queues:- return- cancelled_tasks: list[asyncio.Task] = []- for tool_name, queue in list(queues.items()):- cancelled = 0- while not queue.empty():- task, _args = queue.get_nowait()- if not task.done():- task.cancel()- cancelled_tasks.append(task)- cancelled += 1- if cancelled:- logger.debug(- "Cancelled %d pre-launched task(s) for tool '%s'", cancelled, tool_name- )- queues.clear()- # Await all cancelled tasks so their cleanup (finally blocks, DB rollbacks)- # completes before the next retry attempt starts new pre-launches.- # Use a timeout to prevent hanging indefinitely if a task's cleanup is stuck.- if cancelled_tasks:- try:- await asyncio.wait_for(- asyncio.gather(*cancelled_tasks, return_exceptions=True),- timeout=5.0,- )- except TimeoutError:- logger.warning(- "Timed out waiting for %d cancelled task(s) to clean up",- len(cancelled_tasks),- )--def reset_tool_failure_counters() -> None:"""Reset all tool-level circuit breaker counters.@@ -249,10 +199,6 @@ async def wait_for_stash(timeout: float = 2.0) -> bool:Uses ``asyncio.Event.wait()`` so it returns the instant the hook signals —the timeout is purely a safety net for the case where the hook never fires.Returns ``True`` if the stash signal was received, ``False`` on timeout.-- The 2.0 s default was chosen to accommodate slower tool startup in cloud- sandboxes while still failing fast when the hook genuinely will not fire.- With the parallel pre-launch path, hooks typically fire well under 1 ms."""event = _stash_event.get(None)if event is None:@@ -271,95 +217,13 @@ async def wait_for_stash(timeout: float = 2.0) -> bool:return False-async def pre_launch_tool_call(tool_name: str, args: dict[str, Any]) -> None:- """Pre-launch a tool as a background task so parallel calls run concurrently.-- Called when an AssistantMessage with ToolUseBlocks is received, before the- SDK dispatches the MCP tool/call requests. The tool_handler will await the- pre-launched task instead of executing fresh.-- The tool_name may include an MCP prefix (e.g. ``mcp__copilot__run_block``);- the prefix is stripped automatically before looking up the tool.-- Ordering guarantee: the Claude Agent SDK dispatches MCP ``tools/call`` requests- in the same order as the ToolUseBlocks appear in the AssistantMessage.- Pre-launched tasks are queued FIFO per tool name, so the N-th handler for a- given tool name dequeues the N-th pre-launched task — result and args always- correspond when the SDK preserves order (which it does in the current SDK).- """- queues = _tool_task_queues.get()- if queues is None:- return-- # Strip the MCP server prefix (e.g. "mcp__copilot__") to get the bare tool name.- # Use removeprefix so tool names that themselves contain "__" are handled correctly.- bare_name = tool_name.removeprefix(MCP_TOOL_PREFIX)-- base_tool = TOOL_REGISTRY.get(bare_name)- if base_tool is None:- return-- user_id, session = get_execution_context()- if session is None:- return-- # Expand @@agptfile: references before launching the task.- # The _truncating wrapper (which normally handles expansion) runs AFTER- # pre_launch_tool_call — the pre-launched task would otherwise receive raw- # @@agptfile: tokens and fail to resolve them inside _execute_tool_sync.- # Use _build_input_schema (same path as _truncating) for schema-aware expansion.- input_schema: dict[str, Any] | None- try:- input_schema = _build_input_schema(base_tool)- except Exception:- input_schema = None # schema unavailable — skip schema-aware expansion- try:- args = await expand_file_refs_in_args(- args, user_id, session, input_schema=input_schema- )- except FileRefExpansionError as exc:- logger.warning(- "pre_launch_tool_call: @@agptfile expansion failed for %s: %s — skipping pre-launch",- bare_name,- exc,- )- return-- task = asyncio.create_task(_execute_tool_sync(base_tool, user_id, session, args))- # Log unhandled exceptions so "Task exception was never retrieved" warnings- # do not pollute stderr when a task is pre-launched but never dequeued.- task.add_done_callback(- lambda t, name=bare_name: (- logger.warning(- "Pre-launched task for %s raised unhandled: %s",- name,- t.exception(),- )- if not t.cancelled() and t.exception()- else None- )- )-- if bare_name not in queues:- queues[bare_name] = asyncio.Queue[_TaskQueueItem]()- # Store (task, args) so the handler can log a warning if the SDK dispatches- # calls in a different order than the ToolUseBlocks appeared in the message.- queues[bare_name].put_nowait((task, args))--async def _execute_tool_sync(base_tool: BaseTool,user_id: str | None,session: ChatSession,args: dict[str, Any],) -> dict[str, Any]:- """Execute a tool synchronously and return MCP-formatted response.-- Note: ``@@agptfile:`` expansion should be performed by the caller before- invoking this function. For the normal (non-parallel) path it is handled- by the ``_truncating`` wrapper; for the pre-launched parallel path it is- handled in :func:`pre_launch_tool_call` before the task is created.- """+ """Execute a tool synchronously and return MCP-formatted response."""effective_id = f"sdk-{uuid.uuid4().hex[:12]}"result = await base_tool.execute(user_id=user_id,@@ -455,83 +319,7 @@ def create_tool_handler(base_tool: BaseTool):"""async def tool_handler(args: dict[str, Any]) -> dict[str, Any]:- """Execute the wrapped tool and return MCP-formatted response.-- If a pre-launched task exists (from parallel tool pre-launch in the- message loop), await it instead of executing fresh.- """- queues = _tool_task_queues.get()- if queues and base_tool.name in queues:- queue = queues[base_tool.name]- if not queue.empty():- task, launch_args = queue.get_nowait()- # Sanity-check: warn if the args don't match — this can happen- # if the SDK dispatches tool calls in a different order than the- # ToolUseBlocks appeared in the AssistantMessage (unlikely but- # could occur in future SDK versions or with SDK bugs).- # We compare full values (not just keys) so that two run_block- # calls with different block_id values are caught even though- # both have the same key set.- if launch_args != args:- logger.warning(- "Pre-launched task for %s: arg mismatch "- "(launch_keys=%s, call_keys=%s) — cancelling "- "pre-launched task and falling back to direct execution",- base_tool.name,- (- sorted(launch_args.keys())- if isinstance(launch_args, dict)- else type(launch_args).__name__- ),- (- sorted(args.keys())- if isinstance(args, dict)- else type(args).__name__- ),- )- if not task.done():- task.cancel()- # Await cancellation to prevent duplicate concurrent- # execution for blocks with side effects.- try:- await task- except (asyncio.CancelledError, Exception):- pass- # Fall through to the direct-execution path below.- else:- # Args match — await the pre-launched task.- try:- result = await task- except asyncio.CancelledError:- # Re-raise: CancelledError may be propagating from the- # outer streaming loop being cancelled — swallowing it- # would mask the cancellation and prevent proper cleanup.- logger.warning(- "Pre-launched tool %s was cancelled — re-raising",- base_tool.name,- )- raise- except Exception as e:- logger.error(- "Pre-launched tool %s failed: %s",- base_tool.name,- e,- exc_info=True,- )- return _mcp_error(- f"Failed to execute {base_tool.name}. "- "Check server logs for details."- )-- # Pre-truncate the result so the _truncating wrapper (which- # wraps this handler) receives an already-within-budget- # value. _truncating handles stashing — we must NOT stash- # here or the output will be appended twice to the FIFO- # queue and pop_pending_tool_output would return a duplicate- # entry on the second call for the same tool.- return truncate(result, _MCP_MAX_CHARS)-- # No pre-launched task — execute directly (fallback for non-parallel calls).+ """Execute the wrapped tool and return MCP-formatted response."""user_id, session = get_execution_context()if session is None:@@ -648,9 +436,19 @@ def _text_from_mcp_result(result: dict[str, Any]) -> str:)+_PARALLEL_ANNOTATION = ToolAnnotations(readOnlyHint=True)++def create_copilot_mcp_server(*, use_e2b: bool = False):"""Create an in-process MCP server configuration for CoPilot tools.+ All tools are annotated with ``readOnlyHint=True`` so the SDK CLI+ dispatches concurrent tool calls in parallel rather than sequentially.+ This is a deliberate override: even side-effect tools use the hint+ because the MCP tools are already individually sandboxed and the+ pre-launch duplicate-execution bug (SECRT-2204) is worse than+ sequential dispatch.+When *use_e2b* is True, five additional MCP file tools are registeredthat route directly to the E2B sandbox filesystem, and the caller shoulddisable the corresponding SDK built-in tools via@@ -718,24 +516,35 @@ def create_copilot_mcp_server(*, use_e2b: bool = False):for tool_name, base_tool in TOOL_REGISTRY.items():handler = create_tool_handler(base_tool)schema = _build_input_schema(base_tool)+ # All tools annotated readOnlyHint=True to enable parallel dispatch.+ # The SDK CLI uses this hint to dispatch concurrent tool calls in+ # parallel rather than sequentially. Side-effect safety is ensured+ # by the tool implementations themselves (idempotency, credentials).decorated = tool(tool_name,base_tool.description,schema,+ annotations=_PARALLEL_ANNOTATION,)(_truncating(handler, tool_name, input_schema=schema))sdk_tools.append(decorated)# E2B file tools replace SDK built-in Read/Write/Edit/Glob/Grep.if use_e2b:for name, desc, schema, handler in E2B_FILE_TOOLS:- decorated = tool(name, desc, schema)(_truncating(handler, name))+ decorated = tool(+ name,+ desc,+ schema,+ annotations=_PARALLEL_ANNOTATION,+ )(_truncating(handler, name))sdk_tools.append(decorated)- # Read tool for SDK-truncated tool results (always needed).+ # Read tool for SDK-truncated tool results (always needed, read-only).read_tool = tool(_READ_TOOL_NAME,_READ_TOOL_DESCRIPTION,_READ_TOOL_SCHEMA,+ annotations=_PARALLEL_ANNOTATION,)(_truncating(_read_file_handler, _READ_TOOL_NAME))sdk_tools.append(read_tool)diff --git a/autogpt_platform/backend/backend/copilot/sdk/tool_adapter_test.py b/autogpt_platform/backend/backend/copilot/sdk/tool_adapter_test.pyindex f1a2647331..107c302844 100644--- a/autogpt_platform/backend/backend/copilot/sdk/tool_adapter_test.py+++ b/autogpt_platform/backend/backend/copilot/sdk/tool_adapter_test.py@@ -1,23 +1,21 @@-"""Tests for tool_adapter helpers: truncation, stash, context vars, parallel pre-launch."""+"""Tests for tool_adapter: truncation, stash, context vars, readOnlyHint annotations."""import asyncio-from unittest.mock import AsyncMock, MagicMock, patch+from unittest.mock import AsyncMock, MagicMockimport pytest+from mcp.types import ToolAnnotationsfrom backend.copilot.context import get_sdk_cwdfrom backend.copilot.response_model import StreamToolOutputAvailable-from backend.copilot.sdk.file_ref import FileRefExpansionErrorfrom backend.util.truncate import truncatefrom .tool_adapter import (_MCP_MAX_CHARS,SDK_DISALLOWED_TOOLS,_text_from_mcp_result,- cancel_pending_tool_tasks,create_tool_handler,pop_pending_tool_output,- pre_launch_tool_call,reset_stash_event,set_execution_context,stash_pending_tool_output,@@ -245,7 +243,7 @@ class TestTruncationAndStashIntegration:# ----------------------------------------------------------------------------# Parallel pre-launch infrastructure+# create_tool_handler (direct execution, no pre-launch)# ---------------------------------------------------------------------------@@ -278,501 +276,333 @@ def _init_ctx(session=None):)-class TestPreLaunchToolCall:- """Tests for pre_launch_tool_call and the queue-based parallel dispatch."""+class TestCreateToolHandler:+ """Tests for create_tool_handler — direct tool execution."""@pytest.fixture(autouse=True)def _init(self):_init_ctx(session=_make_mock_session())@pytest.mark.asyncio- async def test_unknown_tool_is_silently_ignored(self):- """pre_launch_tool_call does nothing for tools not in TOOL_REGISTRY."""- # Should not raise even if the tool name is completely unknown- await pre_launch_tool_call("nonexistent_tool", {})-- @pytest.mark.asyncio- async def test_mcp_prefix_stripped_before_registry_lookup(self):- """mcp__copilot__run_block is looked up as 'run_block'."""- mock_tool = _make_mock_tool("run_block")- with patch(- "backend.copilot.sdk.tool_adapter.TOOL_REGISTRY",- {"run_block": mock_tool},- ):- await pre_launch_tool_call("mcp__copilot__run_block", {"block_id": "b1"})-- # The task was enqueued — mock_tool.execute should be called once- # (may not complete immediately but should start)- await asyncio.sleep(0) # yield to event loop- mock_tool.execute.assert_awaited_once()+ async def test_handler_executes_tool_directly(self):+ """Handler executes the tool and returns MCP-formatted result."""+ mock_tool = _make_mock_tool("run_block", output="direct result")- @pytest.mark.asyncio- async def test_bare_tool_name_without_prefix(self):- """Tool names without __ separator are looked up as-is."""- mock_tool = _make_mock_tool("run_block")- with patch(- "backend.copilot.sdk.tool_adapter.TOOL_REGISTRY",- {"run_block": mock_tool},- ):- await pre_launch_tool_call("run_block", {"block_id": "b1"})+ handler = create_tool_handler(mock_tool)+ result = await handler({"block_id": "b1"})- await asyncio.sleep(0)+ assert result["isError"] is False+ text = result["content"][0]["text"]+ assert "direct result" in textmock_tool.execute.assert_awaited_once()@pytest.mark.asyncio- async def test_task_enqueued_fifo_for_same_tool(self):- """Two pre-launched calls for the same tool name are enqueued FIFO."""- results = []-- async def slow_execute(*args, **kwargs):- results.append(len(results))- return StreamToolOutputAvailable(- toolCallId="id",- output=str(len(results) - 1),- toolName="t",- success=True,- )-- mock_tool = _make_mock_tool("t")- mock_tool.execute = AsyncMock(side_effect=slow_execute)-- with patch(- "backend.copilot.sdk.tool_adapter.TOOL_REGISTRY",- {"t": mock_tool},- ):- await pre_launch_tool_call("t", {"n": 1})- await pre_launch_tool_call("t", {"n": 2})- await asyncio.sleep(0)-- assert mock_tool.execute.await_count == 2+ async def test_handler_returns_error_on_no_session(self):+ """When session is None, handler returns MCP error."""+ mock_tool = _make_mock_tool("run_block")+ set_execution_context(user_id="u", session=None, sandbox=None) # type: ignore[arg-type]- @pytest.mark.asyncio- async def test_file_ref_expansion_failure_skips_pre_launch(self):- """When @@agptfile: expansion fails, pre_launch_tool_call skips the task.-- The handler should then fall back to direct execution (which will also- fail with a proper MCP error via _truncating's own expansion).- """- mock_tool = _make_mock_tool("run_block", output="should-not-execute")-- with (- patch(- "backend.copilot.sdk.tool_adapter.TOOL_REGISTRY",- {"run_block": mock_tool},- ),- patch(- "backend.copilot.sdk.tool_adapter.expand_file_refs_in_args",- AsyncMock(side_effect=FileRefExpansionError("@@agptfile:missing.txt")),- ),- ):- # Should not raise — expansion failure is handled gracefully- await pre_launch_tool_call("run_block", {"text": "@@agptfile:missing.txt"})- await asyncio.sleep(0)-- # No task was pre-launched — execute was not called- mock_tool.execute.assert_not_awaited()---class TestCreateToolHandlerParallel:- """Tests for create_tool_handler using pre-launched tasks."""+ handler = create_tool_handler(mock_tool)+ result = await handler({"block_id": "b1"})- @pytest.fixture(autouse=True)- def _init(self):- _init_ctx(session=_make_mock_session())+ assert result["isError"] is True+ assert "session" in result["content"][0]["text"].lower()@pytest.mark.asyncio- async def test_handler_uses_prelaunched_task(self):- """Handler pops and awaits the pre-launched task rather than re-executing."""- mock_tool = _make_mock_tool("run_block", output="pre-launched result")-- with patch(- "backend.copilot.sdk.tool_adapter.TOOL_REGISTRY",- {"run_block": mock_tool},- ):- await pre_launch_tool_call("run_block", {"block_id": "b1"})- await asyncio.sleep(0) # let task start+ async def test_handler_returns_error_on_exception(self):+ """Exception from tool execution is caught and returned as MCP error."""+ mock_tool = _make_mock_tool("run_block")+ mock_tool.execute = AsyncMock(side_effect=RuntimeError("block exploded"))- handler = create_tool_handler(mock_tool)- result = await handler({"block_id": "b1"})+ handler = create_tool_handler(mock_tool)+ result = await handler({"block_id": "b1"})- assert result["isError"] is False- text = result["content"][0]["text"]- assert "pre-launched result" in text- # Should only have been called once (the pre-launched task), not twice- mock_tool.execute.assert_awaited_once()+ assert result["isError"] is True+ assert "Failed to execute run_block" in result["content"][0]["text"]@pytest.mark.asyncio- async def test_handler_does_not_double_stash_for_prelaunched_task(self):- """Pre-launched task result must NOT be stashed by tool_handler directly.+ async def test_handler_executes_once_per_call(self):+ """Each handler call executes the tool exactly once — no duplicate execution."""+ mock_tool = _make_mock_tool("run_block", output="single-execution")- The _truncating wrapper wraps tool_handler and handles stashing after- tool_handler returns. If tool_handler also stashed, the output would be- appended twice to the FIFO queue and pop_pending_tool_output would return- a duplicate on the second call.+ handler = create_tool_handler(mock_tool)+ await handler({"block_id": "b1"})+ await handler({"block_id": "b2"})- This test calls tool_handler directly (without _truncating) and asserts- that nothing was stashed — confirming stashing is deferred to _truncating.- """- mock_tool = _make_mock_tool("run_block", output="stash-me")+ assert mock_tool.execute.await_count == 2- with patch(- "backend.copilot.sdk.tool_adapter.TOOL_REGISTRY",- {"run_block": mock_tool},- ):- await pre_launch_tool_call("run_block", {"block_id": "b1"})- await asyncio.sleep(0)- handler = create_tool_handler(mock_tool)- result = await handler({"block_id": "b1"})+# ---------------------------------------------------------------------------+# Regression tests: bugs fixed by removing pre-launch mechanism+#+# Each test class includes a _buggy_handler fixture that reproduces the old+# pre-launch implementation inline. Tests run against BOTH the buggy handler+# (xfail — proves the bug exists) and the current clean handler (must pass).+# ---------------------------------------------------------------------------- assert result["isError"] is False- assert "stash-me" in result["content"][0]["text"]- # tool_handler must NOT stash — _truncating (which wraps handler) does it.- # Calling pop here (without going through _truncating) should return None.- not_stashed = pop_pending_tool_output("run_block")- assert not_stashed is None, (- "tool_handler must not stash directly — _truncating handles stashing "- "to prevent double-stash in the FIFO queue"- )- @pytest.mark.asyncio- async def test_handler_falls_back_when_queue_empty(self):- """When no pre-launched task exists, handler executes directly."""- mock_tool = _make_mock_tool("run_block", output="direct result")+def _make_execute_fn(tool_name: str = "run_block"):+ """Return (execute_fn, call_log) — execute_fn records every call."""+ call_log: list[dict] = []- # Don't call pre_launch_tool_call — queue is empty- handler = create_tool_handler(mock_tool)- result = await handler({"block_id": "b1"})+ async def execute_fn(*args, **kwargs):+ call_log.append(kwargs)+ return StreamToolOutputAvailable(+ toolCallId=f"id-{len(call_log)}",+ output=f"result-{len(call_log)}",+ toolName=tool_name,+ success=True,+ )- assert result["isError"] is False- text = result["content"][0]["text"]- assert "direct result" in text- mock_tool.execute.assert_awaited_once()+ return execute_fn, call_log- @pytest.mark.asyncio- async def test_handler_cancelled_error_propagates(self):- """CancelledError from a pre-launched task is re-raised to preserve cancellation semantics."""- mock_tool = _make_mock_tool("run_block")- mock_tool.execute = AsyncMock(side_effect=asyncio.CancelledError())- with patch(- "backend.copilot.sdk.tool_adapter.TOOL_REGISTRY",- {"run_block": mock_tool},- ):- await pre_launch_tool_call("run_block", {"block_id": "b1"})- await asyncio.sleep(0)+async def _buggy_prelaunch_handler(mock_tool, pre_launch_args, dispatch_args):+ """Simulate the OLD buggy pre-launch flow.- handler = create_tool_handler(mock_tool)- with pytest.raises(asyncio.CancelledError):- await handler({"block_id": "b1"})+ 1. pre_launch_tool_call fires _execute_tool_sync with pre_launch_args+ 2. SDK dispatches handler with dispatch_args+ 3. Handler compares args — on mismatch, cancels + re-executes (BUG)- @pytest.mark.asyncio- async def test_handler_exception_returns_mcp_error(self):- """Exception from a pre-launched task is caught and returned as MCP error."""- mock_tool = _make_mock_tool("run_block")- mock_tool.execute = AsyncMock(side_effect=RuntimeError("block exploded"))+ Returns the handler result.+ """+ from backend.copilot.sdk.tool_adapter import _execute_tool_sync- with patch(- "backend.copilot.sdk.tool_adapter.TOOL_REGISTRY",- {"run_block": mock_tool},- ):- await pre_launch_tool_call("run_block", {"block_id": "b1"})- await asyncio.sleep(0)+ user_id, session = "user-1", _make_mock_session()- handler = create_tool_handler(mock_tool)- result = await handler({"block_id": "b1"})+ # Step 1: pre-launch fires immediately (speculative)+ task = asyncio.create_task(+ _execute_tool_sync(mock_tool, user_id, session, pre_launch_args)+ )+ await asyncio.sleep(0) # let task start++ # Step 2: SDK dispatches with (potentially different) args+ if pre_launch_args != dispatch_args:+ # Arg mismatch path: cancel pre-launched task + re-execute+ if not task.done():+ task.cancel()+ try:+ await task+ except (asyncio.CancelledError, Exception):+ pass+ # Fall through to direct execution (duplicate!)+ return await _execute_tool_sync(mock_tool, user_id, session, dispatch_args)+ else:+ return await task+++class TestBug1DuplicateExecution:+ """Bug 1 (SECRT-2204): arg mismatch causes duplicate execution.++ Pre-launch fires with raw args, SDK dispatches with normalised args.+ Mismatch → cancel (too late) + re-execute → 2 API calls.+ """- assert result["isError"] is True- assert "Failed to execute run_block" in result["content"][0]["text"]+ @pytest.fixture(autouse=True)+ def _init(self):+ _init_ctx(session=_make_mock_session())+ @pytest.mark.xfail(reason="Old pre-launch code causes duplicate execution")@pytest.mark.asyncio- async def test_two_same_tool_calls_dispatched_in_order(self):- """Two pre-launched tasks for the same tool are consumed in FIFO order."""- call_order = []-- async def execute_with_tag(*args, **kwargs):- tag = kwargs.get("block_id", "?")- call_order.append(tag)- return StreamToolOutputAvailable(- toolCallId="id", output=f"out-{tag}", toolName="run_block", success=True- )-+ async def test_old_code_duplicates_on_arg_mismatch(self):+ """OLD CODE: pre-launch with args A, dispatch with args B → 2 calls."""+ execute_fn, call_log = _make_execute_fn()mock_tool = _make_mock_tool("run_block")- mock_tool.execute = AsyncMock(side_effect=execute_with_tag)-- with patch(- "backend.copilot.sdk.tool_adapter.TOOL_REGISTRY",- {"run_block": mock_tool},- ):- await pre_launch_tool_call("run_block", {"block_id": "first"})- await pre_launch_tool_call("run_block", {"block_id": "second"})- await asyncio.sleep(0)-- handler = create_tool_handler(mock_tool)- r1 = await handler({"block_id": "first"})- r2 = await handler({"block_id": "second"})-- assert "out-first" in r1["content"][0]["text"]- assert "out-second" in r2["content"][0]["text"]- assert call_order == [- "first",- "second",- ], f"Expected FIFO dispatch order but got {call_order}"+ mock_tool.execute = AsyncMock(side_effect=execute_fn)- @pytest.mark.asyncio- async def test_arg_mismatch_falls_back_to_direct_execution(self):- """When pre-launched args differ from SDK args, handler cancels pre-launched- task and falls back to direct execution with the correct args."""- mock_tool = _make_mock_tool("run_block", output="direct-result")-- with patch(- "backend.copilot.sdk.tool_adapter.TOOL_REGISTRY",- {"run_block": mock_tool},- ):- # Pre-launch with args {"block_id": "wrong"}- await pre_launch_tool_call("run_block", {"block_id": "wrong"})- await asyncio.sleep(0)-- # SDK dispatches with different args- handler = create_tool_handler(mock_tool)- result = await handler({"block_id": "correct"})+ pre_launch_args = {"block_id": "b1", "input_data": {"title": "Test"}}+ dispatch_args = {+ "block_id": "b1",+ "input_data": {"title": "Test", "priority": None},+ }- assert result["isError"] is False- # The tool was called twice: once by pre-launch (wrong args), once by- # direct fallback (correct args). The result should come from the- # direct execution path.- assert mock_tool.execute.await_count == 2+ await _buggy_prelaunch_handler(mock_tool, pre_launch_args, dispatch_args)++ # BUG: pre-launch executed once + fallback executed again = 2+ assert len(call_log) == 1, (+ f"Expected 1 execution but got {len(call_log)} — "+ f"duplicate execution bug!"+ )@pytest.mark.asyncio- async def test_no_session_falls_back_gracefully(self):- """When session is None and no pre-launched task, handler returns MCP error."""+ async def test_current_code_no_duplicate(self):+ """FIXED: handler executes exactly once regardless of arg shape."""+ execute_fn, call_log = _make_execute_fn()mock_tool = _make_mock_tool("run_block")- # session=None means get_execution_context returns (user_id, None)- set_execution_context(user_id="u", session=None, sandbox=None) # type: ignore[arg-type]+ mock_tool.execute = AsyncMock(side_effect=execute_fn)handler = create_tool_handler(mock_tool)- result = await handler({"block_id": "b1"})+ await handler({"block_id": "b1", "input_data": {"title": "Test"}})- assert result["isError"] is True- assert "session" in result["content"][0]["text"].lower()+ assert len(call_log) == 1, f"Expected 1 execution but got {len(call_log)}"-# ----------------------------------------------------------------------------# cancel_pending_tool_tasks-# ---------------------------------------------------------------------------+class TestBug2FIFODesync:+ """Bug 2: FIFO desync when security hook denies a tool.--class TestCancelPendingToolTasks:- """Tests for cancel_pending_tool_tasks — the stream-abort cleanup helper."""+ Pre-launch queues [task_A, task_B]. Tool A denied (no MCP dispatch).+ Tool B's handler dequeues task_A → returns wrong result.+ """@pytest.fixture(autouse=True)def _init(self):_init_ctx(session=_make_mock_session())+ @pytest.mark.xfail(reason="Old FIFO queue returns wrong result on denial")@pytest.mark.asyncio- async def test_cancels_queued_tasks(self):- """Queued tasks are cancelled and the queue is cleared."""- ran = False+ async def test_old_code_fifo_desync_on_denial(self):+ """OLD CODE: denied tool's task stays in queue, next tool gets wrong result."""+ from backend.copilot.sdk.tool_adapter import _execute_tool_sync++ call_log: list[str] = []- async def never_run(*_args, **_kwargs):- nonlocal ran- await asyncio.sleep(10) # long enough to still be pending- ran = True+ async def tagged_execute(*args, **kwargs):+ tag = kwargs.get("block_id", "?")+ call_log.append(tag)+ return StreamToolOutputAvailable(+ toolCallId="id",+ output=f"result-for-{tag}",+ toolName="run_block",+ success=True,+ )mock_tool = _make_mock_tool("run_block")- mock_tool.execute = AsyncMock(side_effect=never_run)+ mock_tool.execute = AsyncMock(side_effect=tagged_execute)+ user_id, session = "user-1", _make_mock_session()- with patch(- "backend.copilot.sdk.tool_adapter.TOOL_REGISTRY",- {"run_block": mock_tool},- ):- await pre_launch_tool_call("run_block", {"block_id": "b1"})- await asyncio.sleep(0) # let task start- await cancel_pending_tool_tasks()- await asyncio.sleep(0) # let cancellation propagate+ # Simulate old FIFO queue+ queue: asyncio.Queue = asyncio.Queue()- assert not ran, "Task should have been cancelled before completing"+ # Pre-launch for tool A and tool B+ task_a = asyncio.create_task(+ _execute_tool_sync(mock_tool, user_id, session, {"block_id": "A"})+ )+ task_b = asyncio.create_task(+ _execute_tool_sync(mock_tool, user_id, session, {"block_id": "B"})+ )+ queue.put_nowait(task_a)+ queue.put_nowait(task_b)+ await asyncio.sleep(0) # let both tasks run++ # Tool A is DENIED by security hook — no MCP dispatch, no dequeue+ # Tool B's handler dequeues from FIFO → gets task_A!+ dequeued_task = queue.get_nowait()+ result = await dequeued_task+ result_text = result["content"][0]["text"]++ # BUG: handler for B got task_A's result+ assert "result-for-B" in result_text, (+ f"Expected result for B but got: {result_text} — "+ f"FIFO desync: B got A's result!"+ )@pytest.mark.asyncio- async def test_noop_when_no_tasks_queued(self):- """cancel_pending_tool_tasks does not raise when queues are empty."""- await cancel_pending_tool_tasks() # should not raise+ async def test_current_code_no_fifo_desync(self):+ """FIXED: each handler call executes independently, no shared queue."""+ call_log: list[str] = []- @pytest.mark.asyncio- async def test_handler_does_not_find_cancelled_task(self):- """After cancel, tool_handler falls back to direct execution."""- mock_tool = _make_mock_tool("run_block", output="direct-fallback")-- with patch(- "backend.copilot.sdk.tool_adapter.TOOL_REGISTRY",- {"run_block": mock_tool},- ):- await pre_launch_tool_call("run_block", {"block_id": "b1"})- await asyncio.sleep(0)- await cancel_pending_tool_tasks()-- # Queue is now empty — handler should execute directly- handler = create_tool_handler(mock_tool)- result = await handler({"block_id": "b1"})+ async def tagged_execute(*args, **kwargs):+ tag = kwargs.get("block_id", "?")+ call_log.append(tag)+ return StreamToolOutputAvailable(+ toolCallId="id",+ output=f"result-for-{tag}",+ toolName="run_block",+ success=True,+ )- assert result["isError"] is False- assert "direct-fallback" in result["content"][0]["text"]+ mock_tool = _make_mock_tool("run_block")+ mock_tool.execute = AsyncMock(side_effect=tagged_execute)+ handler = create_tool_handler(mock_tool)-# ----------------------------------------------------------------------------# Concurrent / parallel pre-launch scenarios-# ---------------------------------------------------------------------------+ # Tool A denied (never called). Tool B dispatched normally.+ result_b = await handler({"block_id": "B"})++ assert "result-for-B" in result_b["content"][0]["text"]+ assert call_log == ["B"]-class TestAllParallelToolsPrelaunchedIndependently:- """Simulate SDK sending N separate AssistantMessages for the same tool concurrently."""+class TestBug3CancelRace:+ """Bug 3: cancel race — task completes before cancel arrives.++ Pre-launch fires fast HTTP call (< 1s). By the time handler detects+ mismatch and calls task.cancel(), the API call already completed.+ Side effect (Linear issue created) is irreversible.+ """@pytest.fixture(autouse=True)def _init(self):_init_ctx(session=_make_mock_session())+ @pytest.mark.xfail(reason="Old code: cancel arrives after task completes")@pytest.mark.asyncio- async def test_all_parallel_tools_prelaunched_independently(self):- """5 pre-launches for the same tool all enqueue independently and run concurrently.-- Each task sleeps for PER_TASK_S seconds. If they ran sequentially the total- wall time would be ~5*PER_TASK_S. Running concurrently it should finish in- roughly PER_TASK_S (plus scheduling overhead).- """- PER_TASK_S = 0.05- N = 5- started: list[int] = []- finished: list[int] = []-- async def slow_execute(*args, **kwargs):- idx = len(started)- started.append(idx)- await asyncio.sleep(PER_TASK_S)- finished.append(idx)+ async def test_old_code_cancel_arrives_too_late(self):+ """OLD CODE: fast task completes before cancel, side effect persists."""+ side_effects: list[str] = []++ async def fast_execute_with_side_effect(*args, **kwargs):+ # Side effect happens immediately (like an HTTP POST to Linear)+ side_effects.append("created-issue")return StreamToolOutputAvailable(- toolCallId=f"id-{idx}",- output=f"result-{idx}",- toolName="bash_exec",+ toolCallId="id",+ output="issue-created",+ toolName="run_block",success=True,)- mock_tool = _make_mock_tool("bash_exec")- mock_tool.execute = AsyncMock(side_effect=slow_execute)-- with patch(- "backend.copilot.sdk.tool_adapter.TOOL_REGISTRY",- {"bash_exec": mock_tool},- ):- for i in range(N):- await pre_launch_tool_call("bash_exec", {"cmd": f"echo {i}"})-- # Measure only the concurrent execution window, not pre-launch overhead.- # Starting the timer here avoids false failures on slow CI runners where- # the pre_launch_tool_call setup takes longer than the concurrent sleep.- t0 = asyncio.get_running_loop().time()- await asyncio.sleep(PER_TASK_S * 2)- elapsed = asyncio.get_running_loop().time() - t0-- assert mock_tool.execute.await_count == N- assert len(finished) == N- # Wall time of the sleep window should be well under N * PER_TASK_S- # (sequential would be ~0.25s; concurrent finishes in ~PER_TASK_S = 0.05s)- assert elapsed < N * PER_TASK_S, (- f"Expected concurrent execution (<{N * PER_TASK_S:.2f}s) "- f"but sleep window took {elapsed:.2f}s"- )+ mock_tool = _make_mock_tool("run_block")+ mock_tool.execute = AsyncMock(side_effect=fast_execute_with_side_effect)+ # Pre-launch fires immediately+ pre_launch_args = {"block_id": "b1"}+ dispatch_args = {"block_id": "b1", "extra": "normalised"}-class TestHandlerReturnsResultFromCorrectPrelaunchedTask:- """Pop pre-launched tasks in order and verify each returns its own result."""+ await _buggy_prelaunch_handler(mock_tool, pre_launch_args, dispatch_args)- @pytest.fixture(autouse=True)- def _init(self):- _init_ctx(session=_make_mock_session())+ # BUG: side effect happened TWICE (pre-launch + fallback)+ assert len(side_effects) == 1, (+ f"Expected 1 side effect but got {len(side_effects)} — "+ f"cancel race: pre-launch completed before cancel!"+ )@pytest.mark.asyncio- async def test_handler_returns_result_from_correct_prelaunched_task(self):- """Two pre-launches for the same tool: first handler gets first result, second gets second."""+ async def test_current_code_single_side_effect(self):+ """FIXED: no speculative execution, exactly 1 side effect per call."""+ side_effects: list[str] = []- async def execute_with_cmd(*args, **kwargs):- cmd = kwargs.get("cmd", "?")+ async def execute_with_side_effect(*args, **kwargs):+ side_effects.append("created-issue")return StreamToolOutputAvailable(toolCallId="id",- output=f"output-for-{cmd}",- toolName="bash_exec",+ output="issue-created",+ toolName="run_block",success=True,)- mock_tool = _make_mock_tool("bash_exec")- mock_tool.execute = AsyncMock(side_effect=execute_with_cmd)-- with patch(- "backend.copilot.sdk.tool_adapter.TOOL_REGISTRY",- {"bash_exec": mock_tool},- ):- await pre_launch_tool_call("bash_exec", {"cmd": "alpha"})- await pre_launch_tool_call("bash_exec", {"cmd": "beta"})- await asyncio.sleep(0) # let both tasks start-- handler = create_tool_handler(mock_tool)- r1 = await handler({"cmd": "alpha"})- r2 = await handler({"cmd": "beta"})-- text1 = r1["content"][0]["text"]- text2 = r2["content"][0]["text"]- assert "output-for-alpha" in text1, f"Expected alpha result, got: {text1}"- assert "output-for-beta" in text2, f"Expected beta result, got: {text2}"- assert mock_tool.execute.await_count == 2+ mock_tool = _make_mock_tool("run_block")+ mock_tool.execute = AsyncMock(side_effect=execute_with_side_effect)+ handler = create_tool_handler(mock_tool)+ await handler({"block_id": "b1"})-class TestFiveConcurrentPrelaunchAllComplete:- """Pre-launch 5 tasks; consume all 5 via handlers; assert all succeed."""+ assert len(side_effects) == 1- @pytest.fixture(autouse=True)- def _init(self):- _init_ctx(session=_make_mock_session())- @pytest.mark.asyncio- async def test_five_concurrent_prelaunch_all_complete(self):- """All 5 pre-launched tasks complete and return successful results."""- N = 5- call_count = 0-- async def counting_execute(*args, **kwargs):- nonlocal call_count- call_count += 1- n = call_count- return StreamToolOutputAvailable(- toolCallId=f"id-{n}",- output=f"done-{n}",- toolName="bash_exec",- success=True,- )+# ---------------------------------------------------------------------------+# readOnlyHint annotations+# ---------------------------------------------------------------------------+++class TestReadOnlyAnnotations:+ """Tests that all tools get readOnlyHint=True for parallel dispatch."""++ def test_parallel_annotation_constant(self):+ """_PARALLEL_ANNOTATION is a ToolAnnotations with readOnlyHint=True."""+ from .tool_adapter import _PARALLEL_ANNOTATION- mock_tool = _make_mock_tool("bash_exec")- mock_tool.execute = AsyncMock(side_effect=counting_execute)-- with patch(- "backend.copilot.sdk.tool_adapter.TOOL_REGISTRY",- {"bash_exec": mock_tool},- ):- for i in range(N):- await pre_launch_tool_call("bash_exec", {"cmd": f"task-{i}"})-- await asyncio.sleep(0) # let all tasks start-- handler = create_tool_handler(mock_tool)- results = []- for i in range(N):- results.append(await handler({"cmd": f"task-{i}"}))-- assert (- mock_tool.execute.await_count == N- ), f"Expected {N} execute calls, got {mock_tool.execute.await_count}"- for i, result in enumerate(results):- assert result["isError"] is False, f"Result {i} should not be an error"- text = result["content"][0]["text"]- assert "done-" in text, f"Result {i} missing expected output: {text}"+ assert isinstance(_PARALLEL_ANNOTATION, ToolAnnotations)+ assert _PARALLEL_ANNOTATION.readOnlyHint is True# ---------------------------------------------------------------------------