AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(copilot): Wait for agent execution completion (#12147)
Ottoed729ddbe294
diff --git a/autogpt_platform/backend/backend/copilot/tools/agent_output.py b/autogpt_platform/backend/backend/copilot/tools/agent_output.pyindex fe4767d09e..4ccf323aca 100644--- a/autogpt_platform/backend/backend/copilot/tools/agent_output.py+++ b/autogpt_platform/backend/backend/copilot/tools/agent_output.py@@ -5,7 +5,7 @@ import refrom datetime import datetime, timedelta, timezonefrom typing import Any-from pydantic import BaseModel, field_validator+from pydantic import BaseModel, Field, field_validatorfrom backend.api.features.library.model import LibraryAgentfrom backend.copilot.model import ChatSession@@ -13,6 +13,7 @@ from backend.data.db_accessors import execution_db, library_dbfrom backend.data.execution import ExecutionStatus, GraphExecution, GraphExecutionMetafrom .base import BaseTool+from .execution_utils import TERMINAL_STATUSES, wait_for_executionfrom .models import (AgentOutputResponse,ErrorResponse,@@ -33,6 +34,7 @@ class AgentOutputInput(BaseModel):store_slug: str = ""execution_id: str = ""run_time: str = "latest"+ wait_if_running: int = Field(default=0, ge=0, le=300)@field_validator("agent_name",@@ -116,6 +118,11 @@ class AgentOutputTool(BaseTool):Select which run to retrieve using:- execution_id: Specific execution ID- run_time: 'latest' (default), 'yesterday', 'last week', or ISO date 'YYYY-MM-DD'++ Wait for completion (optional):+ - wait_if_running: Max seconds to wait if execution is still running (0-300).+ If the execution is running/queued, waits up to this many seconds for completion.+ Returns current status on timeout. If already finished, returns immediately."""@property@@ -145,6 +152,13 @@ class AgentOutputTool(BaseTool):"Time filter: 'latest', 'yesterday', 'last week', or 'YYYY-MM-DD'"),},+ "wait_if_running": {+ "type": "integer",+ "description": (+ "Max seconds to wait if execution is still running (0-300). "+ "If running, waits for completion. Returns current state on timeout."+ ),+ },},"required": [],}@@ -224,10 +238,14 @@ class AgentOutputTool(BaseTool):execution_id: str | None,time_start: datetime | None,time_end: datetime | None,+ include_running: bool = False,) -> tuple[GraphExecution | None, list[GraphExecutionMeta], str | None]:"""Fetch execution(s) based on filters.Returns (single_execution, available_executions_meta, error_message).++ Args:+ include_running: If True, also look for running/queued executions (for waiting)"""exec_db = execution_db()@@ -242,11 +260,25 @@ class AgentOutputTool(BaseTool):return None, [], f"Execution '{execution_id}' not found"return execution, [], None- # Get completed executions with time filters+ # Determine which statuses to query+ statuses = [ExecutionStatus.COMPLETED]+ if include_running:+ statuses.extend(+ [+ ExecutionStatus.RUNNING,+ ExecutionStatus.QUEUED,+ ExecutionStatus.INCOMPLETE,+ ExecutionStatus.REVIEW,+ ExecutionStatus.FAILED,+ ExecutionStatus.TERMINATED,+ ]+ )++ # Get executions with time filtersexecutions = await exec_db.get_graph_executions(graph_id=graph_id,user_id=user_id,- statuses=[ExecutionStatus.COMPLETED],+ statuses=statuses,created_time_gte=time_start,created_time_lte=time_end,limit=10,@@ -313,10 +345,33 @@ class AgentOutputTool(BaseTool):for e in available_executions[:5]]- message = f"Found execution outputs for agent '{agent.name}'"+ # Build appropriate message based on execution status+ if execution.status == ExecutionStatus.COMPLETED:+ message = f"Found execution outputs for agent '{agent.name}'"+ elif execution.status == ExecutionStatus.FAILED:+ message = f"Execution for agent '{agent.name}' failed"+ elif execution.status == ExecutionStatus.TERMINATED:+ message = f"Execution for agent '{agent.name}' was terminated"+ elif execution.status == ExecutionStatus.REVIEW:+ message = (+ f"Execution for agent '{agent.name}' is awaiting human review. "+ "The user needs to approve it before it can continue."+ )+ elif execution.status in (+ ExecutionStatus.RUNNING,+ ExecutionStatus.QUEUED,+ ExecutionStatus.INCOMPLETE,+ ):+ message = (+ f"Execution for agent '{agent.name}' is still {execution.status.value}. "+ "Results may be incomplete. Use wait_if_running to wait for completion."+ )+ else:+ message = f"Found execution for agent '{agent.name}' (status: {execution.status.value})"+if len(available_executions) > 1:message += (- f". Showing latest of {len(available_executions)} matching executions."+ f" Showing latest of {len(available_executions)} matching executions.")return AgentOutputResponse(@@ -431,13 +486,17 @@ class AgentOutputTool(BaseTool):# Parse time expressiontime_start, time_end = parse_time_expression(input_data.run_time)- # Fetch execution(s)+ # Check if we should wait for running executions+ wait_timeout = input_data.wait_if_running++ # Fetch execution(s) - include running if we're going to waitexecution, available_executions, exec_error = await self._get_execution(user_id=user_id,graph_id=agent.graph_id,execution_id=input_data.execution_id or None,time_start=time_start,time_end=time_end,+ include_running=wait_timeout > 0,)if exec_error:@@ -446,4 +505,17 @@ class AgentOutputTool(BaseTool):session_id=session_id,)+ # If we have an execution that's still running and we should wait+ if execution and wait_timeout > 0 and execution.status not in TERMINAL_STATUSES:+ logger.info(+ f"Execution {execution.id} is {execution.status}, "+ f"waiting up to {wait_timeout}s for completion"+ )+ execution = await wait_for_execution(+ user_id=user_id,+ graph_id=agent.graph_id,+ execution_id=execution.id,+ timeout_seconds=wait_timeout,+ )+return self._build_response(agent, execution, available_executions, session_id)diff --git a/autogpt_platform/backend/backend/copilot/tools/execution_utils.py b/autogpt_platform/backend/backend/copilot/tools/execution_utils.pynew file mode 100644index 0000000000..e460b9e4ca--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/tools/execution_utils.py@@ -0,0 +1,186 @@+"""Shared utilities for execution waiting and status handling."""++import asyncio+import logging+from typing import Any++from backend.data.db_accessors import execution_db+from backend.data.execution import (+ AsyncRedisExecutionEventBus,+ ExecutionStatus,+ GraphExecution,+ GraphExecutionEvent,+)++logger = logging.getLogger(__name__)++# Terminal statuses that indicate execution is complete+TERMINAL_STATUSES = frozenset(+ {+ ExecutionStatus.COMPLETED,+ ExecutionStatus.FAILED,+ ExecutionStatus.TERMINATED,+ }+)++# Statuses where execution is paused but not finished (e.g. human-in-the-loop)+PAUSED_STATUSES = frozenset(+ {+ ExecutionStatus.REVIEW,+ }+)++# Statuses that mean "stop waiting" (terminal or paused)+STOP_WAITING_STATUSES = TERMINAL_STATUSES | PAUSED_STATUSES++_POST_SUBSCRIBE_RECHECK_DELAY = 0.1 # seconds to wait for subscription to establish+++async def wait_for_execution(+ user_id: str,+ graph_id: str,+ execution_id: str,+ timeout_seconds: int,+) -> GraphExecution | None:+ """+ Wait for an execution to reach a terminal or paused status using Redis pubsub.++ Handles the race condition between checking status and subscribing by+ re-checking the DB after the subscription is established.++ Args:+ user_id: User ID+ graph_id: Graph ID+ execution_id: Execution ID to wait for+ timeout_seconds: Max seconds to wait++ Returns:+ The execution with current status, or None if not found+ """+ exec_db = execution_db()++ # Quick check — maybe it's already done+ execution = await exec_db.get_graph_execution(+ user_id=user_id,+ execution_id=execution_id,+ include_node_executions=False,+ )+ if not execution:+ return None++ if execution.status in STOP_WAITING_STATUSES:+ logger.debug(+ f"Execution {execution_id} already in stop-waiting state: "+ f"{execution.status}"+ )+ return execution++ logger.info(+ f"Waiting up to {timeout_seconds}s for execution {execution_id} "+ f"(current status: {execution.status})"+ )++ event_bus = AsyncRedisExecutionEventBus()+ channel_key = f"{user_id}/{graph_id}/{execution_id}"++ # Mutable container so _subscribe_and_wait can surface the task even if+ # asyncio.wait_for cancels the coroutine before it returns.+ task_holder: list[asyncio.Task] = []++ try:+ result = await asyncio.wait_for(+ _subscribe_and_wait(+ event_bus, channel_key, user_id, execution_id, exec_db, task_holder+ ),+ timeout=timeout_seconds,+ )+ return result+ except asyncio.TimeoutError:+ logger.info(f"Timeout waiting for execution {execution_id}")+ except Exception as e:+ logger.error(f"Error waiting for execution: {e}", exc_info=True)+ finally:+ for task in task_holder:+ if not task.done():+ task.cancel()+ try:+ await task+ except asyncio.CancelledError:+ pass+ await event_bus.close()++ # Return current state on timeout/error+ return await exec_db.get_graph_execution(+ user_id=user_id,+ execution_id=execution_id,+ include_node_executions=False,+ )+++async def _subscribe_and_wait(+ event_bus: AsyncRedisExecutionEventBus,+ channel_key: str,+ user_id: str,+ execution_id: str,+ exec_db: Any,+ task_holder: list[asyncio.Task],+) -> GraphExecution | None:+ """+ Subscribe to execution events and wait for a terminal/paused status.++ Appends the consumer task to ``task_holder`` so the caller can clean it up+ even if this coroutine is cancelled by ``asyncio.wait_for``.++ To avoid the race condition where the execution completes between the+ initial DB check and the Redis subscription, we:+ 1. Start listening (which subscribes internally)+ 2. Re-check the DB after subscription is active+ 3. If still running, wait for pubsub events+ """+ listen_iter = event_bus.listen_events(channel_key).__aiter__()++ done = asyncio.Event()+ result_execution: GraphExecution | None = None++ async def _consume() -> None:+ nonlocal result_execution+ try:+ async for event in listen_iter:+ if isinstance(event, GraphExecutionEvent):+ logger.debug(f"Received execution update: {event.status}")+ if event.status in STOP_WAITING_STATUSES:+ result_execution = await exec_db.get_graph_execution(+ user_id=user_id,+ execution_id=execution_id,+ include_node_executions=False,+ )+ done.set()+ return+ except Exception as e:+ logger.error(f"Error in execution consumer: {e}", exc_info=True)+ done.set()++ consume_task = asyncio.create_task(_consume())+ task_holder.append(consume_task)++ # Give the subscription a moment to establish, then re-check DB+ await asyncio.sleep(_POST_SUBSCRIBE_RECHECK_DELAY)++ execution = await exec_db.get_graph_execution(+ user_id=user_id,+ execution_id=execution_id,+ include_node_executions=False,+ )+ if execution and execution.status in STOP_WAITING_STATUSES:+ return execution++ # Wait for the pubsub consumer to find a terminal event+ await done.wait()+ return result_execution+++def get_execution_outputs(execution: GraphExecution | None) -> dict[str, Any] | None:+ """Extract outputs from an execution, or return None."""+ if execution is None:+ return None+ return execution.outputsdiff --git a/autogpt_platform/backend/backend/copilot/tools/run_agent.py b/autogpt_platform/backend/backend/copilot/tools/run_agent.pyindex 46af6fbcb0..ef26c5722e 100644--- a/autogpt_platform/backend/backend/copilot/tools/run_agent.py+++ b/autogpt_platform/backend/backend/copilot/tools/run_agent.py@@ -9,6 +9,7 @@ from backend.copilot.config import ChatConfigfrom backend.copilot.model import ChatSessionfrom backend.copilot.tracking import track_agent_run_success, track_agent_scheduledfrom backend.data.db_accessors import graph_db, library_db, user_db+from backend.data.execution import ExecutionStatusfrom backend.data.graph import GraphModelfrom backend.data.model import CredentialsMetaInputfrom backend.executor import utils as execution_utils@@ -20,12 +21,15 @@ from backend.util.timezone_utils import ()from .base import BaseTool+from .execution_utils import get_execution_outputs, wait_for_executionfrom .helpers import get_inputs_from_schemafrom .models import (AgentDetails,AgentDetailsResponse,+ AgentOutputResponse,ErrorResponse,ExecutionOptions,+ ExecutionOutputInfo,ExecutionStartedResponse,InputValidationErrorResponse,SetupInfo,@@ -66,6 +70,7 @@ class RunAgentInput(BaseModel):schedule_name: str = ""cron: str = ""timezone: str = "UTC"+ wait_for_result: int = Field(default=0, ge=0, le=300)@field_validator("username_agent_slug",@@ -147,6 +152,14 @@ class RunAgentTool(BaseTool):"type": "string","description": "IANA timezone for schedule (default: UTC)",},+ "wait_for_result": {+ "type": "integer",+ "description": (+ "Max seconds to wait for execution to complete (0-300). "+ "If >0, blocks until the execution finishes or times out. "+ "Returns execution outputs when complete."+ ),+ },},"required": [],}@@ -341,6 +354,7 @@ class RunAgentTool(BaseTool):graph=graph,graph_credentials=graph_credentials,inputs=params.inputs,+ wait_for_result=params.wait_for_result,)except NotFoundError as e:@@ -424,8 +438,9 @@ class RunAgentTool(BaseTool):graph: GraphModel,graph_credentials: dict[str, CredentialsMetaInput],inputs: dict[str, Any],+ wait_for_result: int = 0,) -> ToolResponseBase:- """Execute an agent immediately."""+ """Execute an agent immediately, optionally waiting for completion."""session_id = session.session_id# Check rate limits@@ -462,6 +477,91 @@ class RunAgentTool(BaseTool):)library_agent_link = f"/library/agents/{library_agent.id}"++ # If wait_for_result is requested, wait for execution to complete+ if wait_for_result > 0:+ logger.info(+ f"Waiting up to {wait_for_result}s for execution {execution.id}"+ )+ completed = await wait_for_execution(+ user_id=user_id,+ graph_id=library_agent.graph_id,+ execution_id=execution.id,+ timeout_seconds=wait_for_result,+ )++ if completed and completed.status == ExecutionStatus.COMPLETED:+ outputs = get_execution_outputs(completed)+ return AgentOutputResponse(+ message=(+ f"Agent '{library_agent.name}' completed successfully. "+ f"View at {library_agent_link}."+ ),+ session_id=session_id,+ agent_name=library_agent.name,+ agent_id=library_agent.graph_id,+ library_agent_id=library_agent.id,+ library_agent_link=library_agent_link,+ execution=ExecutionOutputInfo(+ execution_id=execution.id,+ status=completed.status.value,+ started_at=completed.started_at,+ ended_at=completed.ended_at,+ outputs=outputs or {},+ ),+ )+ elif completed and completed.status == ExecutionStatus.FAILED:+ error_detail = completed.stats.error if completed.stats else None+ return ErrorResponse(+ message=(+ f"Agent '{library_agent.name}' execution failed. "+ f"View details at {library_agent_link}."+ ),+ session_id=session_id,+ error=error_detail,+ )+ elif completed and completed.status == ExecutionStatus.TERMINATED:+ error_detail = completed.stats.error if completed.stats else None+ return ErrorResponse(+ message=(+ f"Agent '{library_agent.name}' execution was terminated. "+ f"View details at {library_agent_link}."+ ),+ session_id=session_id,+ error=error_detail,+ )+ elif completed and completed.status == ExecutionStatus.REVIEW:+ return ExecutionStartedResponse(+ message=(+ f"Agent '{library_agent.name}' is awaiting human review. "+ f"Check at {library_agent_link}."+ ),+ session_id=session_id,+ execution_id=execution.id,+ graph_id=library_agent.graph_id,+ graph_name=library_agent.name,+ library_agent_id=library_agent.id,+ library_agent_link=library_agent_link,+ status=ExecutionStatus.REVIEW.value,+ )+ else:+ status = completed.status.value if completed else "unknown"+ return ExecutionStartedResponse(+ message=(+ f"Agent '{library_agent.name}' is still {status} after "+ f"{wait_for_result}s. Check results later at "+ f"{library_agent_link}. "+ f"Use view_agent_output with wait_if_running to check again."+ ),+ session_id=session_id,+ execution_id=execution.id,+ graph_id=library_agent.graph_id,+ graph_name=library_agent.name,+ library_agent_id=library_agent.id,+ library_agent_link=library_agent_link,+ status=status,+ )+return ExecutionStartedResponse(message=(f"Agent '{library_agent.name}' execution started successfully. "