AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(backend/executor): Implement cascading stop for nested graph executions (#11277)
Zamil Majdy4922f8885120
diff --git a/autogpt_platform/backend/backend/blocks/agent.py b/autogpt_platform/backend/backend/blocks/agent.pyindex 68fef45ada..b412a1b3a6 100644--- a/autogpt_platform/backend/backend/blocks/agent.py+++ b/autogpt_platform/backend/backend/blocks/agent.py@@ -65,7 +65,13 @@ class AgentExecutorBlock(Block):categories={BlockCategory.AGENT},)- async def run(self, input_data: Input, **kwargs) -> BlockOutput:+ async def run(+ self,+ input_data: Input,+ *,+ graph_exec_id: str,+ **kwargs,+ ) -> BlockOutput:from backend.executor import utils as execution_utils@@ -75,6 +81,7 @@ class AgentExecutorBlock(Block):user_id=input_data.user_id,inputs=input_data.inputs,nodes_input_masks=input_data.nodes_input_masks,+ parent_graph_exec_id=graph_exec_id,)logger = execution_utils.LogMetadata(diff --git a/autogpt_platform/backend/backend/data/execution.py b/autogpt_platform/backend/backend/data/execution.pyindex 6c40e55a31..922672acd0 100644--- a/autogpt_platform/backend/backend/data/execution.py+++ b/autogpt_platform/backend/backend/data/execution.py@@ -346,6 +346,7 @@ class GraphExecutionWithNodes(GraphExecution):self,user_context: "UserContext",compiled_nodes_input_masks: Optional[NodesInputMasks] = None,+ parent_graph_exec_id: Optional[str] = None,):return GraphExecutionEntry(user_id=self.user_id,@@ -354,6 +355,7 @@ class GraphExecutionWithNodes(GraphExecution):graph_exec_id=self.id,nodes_input_masks=compiled_nodes_input_masks,user_context=user_context,+ parent_graph_exec_id=parent_graph_exec_id,)@@ -632,6 +634,25 @@ async def get_graph_execution()+async def get_child_graph_executions(+ parent_exec_id: str,+) -> list[GraphExecutionMeta]:+ """+ Get all child executions of a parent execution.++ Args:+ parent_exec_id: Parent graph execution ID++ Returns:+ List of child graph executions+ """+ children = await AgentGraphExecution.prisma().find_many(+ where={"parentGraphExecutionId": parent_exec_id, "isDeleted": False}+ )++ return [GraphExecutionMeta.from_db(child) for child in children]++async def create_graph_execution(graph_id: str,graph_version: int,@@ -641,6 +662,7 @@ async def create_graph_execution(preset_id: Optional[str] = None,credential_inputs: Optional[Mapping[str, CredentialsMetaInput]] = None,nodes_input_masks: Optional[NodesInputMasks] = None,+ parent_graph_exec_id: Optional[str] = None,) -> GraphExecutionWithNodes:"""Create a new AgentGraphExecution record.@@ -677,6 +699,7 @@ async def create_graph_execution(},"userId": user_id,"agentPresetId": preset_id,+ "parentGraphExecutionId": parent_graph_exec_id,},include=GRAPH_EXECUTION_INCLUDE_WITH_NODES,)@@ -1007,6 +1030,7 @@ class GraphExecutionEntry(BaseModel):graph_version: intnodes_input_masks: Optional[NodesInputMasks] = Noneuser_context: UserContext+ parent_graph_exec_id: Optional[str] = Noneclass NodeExecutionEntry(BaseModel):diff --git a/autogpt_platform/backend/backend/executor/database.py b/autogpt_platform/backend/backend/executor/database.pyindex 929a502d7d..df581e0de4 100644--- a/autogpt_platform/backend/backend/executor/database.py+++ b/autogpt_platform/backend/backend/executor/database.py@@ -7,6 +7,7 @@ from backend.data.credit import UsageTransactionMetadata, get_user_credit_modelfrom backend.data.execution import (create_graph_execution,get_block_error_stats,+ get_child_graph_executions,get_execution_kv_data,get_graph_execution_meta,get_graph_executions,@@ -121,6 +122,7 @@ class DatabaseManager(AppService):return cast(Callable[Concatenate[object, P], R], expose(f))# Executions+ get_child_graph_executions = _(get_child_graph_executions)get_graph_executions = _(get_graph_executions)get_graph_executions_count = _(get_graph_executions_count)get_graph_execution_meta = _(get_graph_execution_meta)@@ -234,6 +236,7 @@ class DatabaseManagerAsyncClient(AppServiceClient):return DatabaseManagercreate_graph_execution = d.create_graph_execution+ get_child_graph_executions = d.get_child_graph_executionsget_connected_output_nodes = d.get_connected_output_nodesget_latest_node_execution = d.get_latest_node_executionget_graph = d.get_graphdiff --git a/autogpt_platform/backend/backend/executor/manager.py b/autogpt_platform/backend/backend/executor/manager.pyindex 2abedcf133..cfe0cc252a 100644--- a/autogpt_platform/backend/backend/executor/manager.py+++ b/autogpt_platform/backend/backend/executor/manager.py@@ -1335,6 +1335,9 @@ class ExecutionManager(AppProcess):return self._run_clientdef run(self):+ logger.info(+ f"[{self.service_name}] 🆔 Pod assigned executor_id: {self.executor_id}"+ )logger.info(f"[{self.service_name}] ⏳ Spawn max-{self.pool_size} workers...")pool_size_gauge.set(self.pool_size)@@ -1495,10 +1498,37 @@ class ExecutionManager(AppProcess):graph_exec_id = graph_exec_entry.graph_exec_iduser_id = graph_exec_entry.user_idgraph_id = graph_exec_entry.graph_id+ parent_graph_exec_id = graph_exec_entry.parent_graph_exec_id+logger.info(- f"[{self.service_name}] Received RUN for graph_exec_id={graph_exec_id}, user_id={user_id}"+ f"[{self.service_name}] Received RUN for graph_exec_id={graph_exec_id}, user_id={user_id}, executor_id={self.executor_id}"+ + (f", parent={parent_graph_exec_id}" if parent_graph_exec_id else ""))+ # Check if parent execution is already terminated (prevents orphaned child executions)+ if parent_graph_exec_id:+ try:+ parent_exec = get_db_client().get_graph_execution_meta(+ execution_id=parent_graph_exec_id,+ user_id=user_id,+ )+ if parent_exec and parent_exec.status == ExecutionStatus.TERMINATED:+ logger.info(+ f"[{self.service_name}] Skipping execution {graph_exec_id} - parent {parent_graph_exec_id} is TERMINATED"+ )+ # Mark this child as terminated since parent was stopped+ get_db_client().update_graph_execution_stats(+ graph_exec_id=graph_exec_id,+ status=ExecutionStatus.TERMINATED,+ )+ _ack_message(reject=False, requeue=False)+ return+ except Exception as e:+ logger.warning(+ f"[{self.service_name}] Could not check parent status for {graph_exec_id}: {e}"+ )+ # Continue execution if parent check fails (don't block on errors)+# Check user rate limit before processingtry:# Only check executions from the last 24 hours for performance@@ -1546,7 +1576,7 @@ class ExecutionManager(AppProcess):# Either someone else has it or Redis is unavailableif current_owner is not None:logger.warning(- f"[{self.service_name}] Graph {graph_exec_id} already running on pod {current_owner}"+ f"[{self.service_name}] Graph {graph_exec_id} already running on pod {current_owner}, current executor_id={self.executor_id}")_ack_message(reject=True, requeue=False)else:@@ -1561,7 +1591,7 @@ class ExecutionManager(AppProcess):self._execution_locks[graph_exec_id] = cluster_locklogger.info(- f"[{self.service_name}] Acquired cluster lock for {graph_exec_id} with executor {self.executor_id}"+ f"[{self.service_name}] Successfully acquired cluster lock for {graph_exec_id}, executor_id={self.executor_id}")cancel_event = threading.Event()@@ -1598,6 +1628,9 @@ class ExecutionManager(AppProcess):finally:# Release the cluster-wide execution lockif graph_exec_id in self._execution_locks:+ logger.info(+ f"[{self.service_name}] Releasing cluster lock for {graph_exec_id}, executor_id={self.executor_id}"+ )self._execution_locks[graph_exec_id].release()del self._execution_locks[graph_exec_id]self._cleanup_completed_runs()diff --git a/autogpt_platform/backend/backend/executor/utils.py b/autogpt_platform/backend/backend/executor/utils.pyindex 445a9ca7d8..ac3ec9110e 100644--- a/autogpt_platform/backend/backend/executor/utils.py+++ b/autogpt_platform/backend/backend/executor/utils.py@@ -25,6 +25,7 @@ from backend.data.db import prismafrom backend.data.dynamic_fields import merge_execution_inputfrom backend.data.execution import (ExecutionStatus,+ GraphExecutionMeta,GraphExecutionStats,GraphExecutionWithNodes,NodesInputMasks,@@ -621,20 +622,74 @@ class CancelExecutionEvent(BaseModel):graph_exec_id: str+async def _get_child_executions(parent_exec_id: str) -> list["GraphExecutionMeta"]:+ """+ Get all child executions of a parent execution using the execution_db pattern.++ Args:+ parent_exec_id: Parent graph execution ID++ Returns:+ List of child graph executions+ """+ from backend.data.db import prisma++ if prisma.is_connected():+ edb = execution_db+ else:+ edb = get_database_manager_async_client()++ return await edb.get_child_graph_executions(parent_exec_id)++async def stop_graph_execution(user_id: str,graph_exec_id: str,wait_timeout: float = 15.0,+ cascade: bool = True,):"""+ Stop a graph execution and optionally all its child executions.+Mechanism:- 1. Set the cancel event- 2. Graph executor's cancel handler thread detects the event, terminates workers,+ 1. Set the cancel event for this execution+ 2. If cascade=True, recursively stop all child executions+ 3. Graph executor's cancel handler thread detects the event, terminates workers,reinitializes worker pool, and returns.- 3. Update execution statuses in DB and set `error` outputs to `"TERMINATED"`.+ 4. Update execution statuses in DB and set `error` outputs to `"TERMINATED"`.++ Args:+ user_id: User ID who owns the execution+ graph_exec_id: Graph execution ID to stop+ wait_timeout: Maximum time to wait for execution to stop (seconds)+ cascade: If True, recursively stop all child executions"""queue_client = await get_async_execution_queue()db = execution_db if prisma.is_connected() else get_database_manager_async_client()++ # First, find and stop all child executions if cascading+ if cascade:+ children = await _get_child_executions(graph_exec_id)+ logger.info(+ f"Stopping {len(children)} child executions of execution {graph_exec_id}"+ )++ # Stop all children in parallel (recursively, with cascading enabled)+ if children:+ await asyncio.gather(+ *[+ stop_graph_execution(+ user_id=user_id,+ graph_exec_id=child.id,+ wait_timeout=wait_timeout,+ cascade=True, # Recursively cascade to grandchildren+ )+ for child in children+ ],+ return_exceptions=True, # Don't fail parent stop if child stop fails+ )++ # Now stop this executionawait queue_client.publish_message(routing_key="",message=CancelExecutionEvent(graph_exec_id=graph_exec_id).model_dump_json(),@@ -698,6 +753,7 @@ async def add_graph_execution(graph_version: Optional[int] = None,graph_credentials_inputs: Optional[Mapping[str, CredentialsMetaInput]] = None,nodes_input_masks: Optional[NodesInputMasks] = None,+ parent_graph_exec_id: Optional[str] = None,) -> GraphExecutionWithNodes:"""Adds a graph execution to the queue and returns the execution entry.@@ -711,6 +767,7 @@ async def add_graph_execution(graph_credentials_inputs: Credentials inputs to use in the execution.Keys should map to the keys generated by `GraphModel.aggregate_credentials_inputs`.nodes_input_masks: Node inputs to use in the execution.+ parent_graph_exec_id: The ID of the parent graph execution (for nested executions).Returns:GraphExecutionEntry: The entry for the graph execution.Raises:@@ -745,11 +802,13 @@ async def add_graph_execution(nodes_input_masks=nodes_input_masks,starting_nodes_input=starting_nodes_input,preset_id=preset_id,+ parent_graph_exec_id=parent_graph_exec_id,)graph_exec_entry = graph_exec.to_graph_execution_entry(user_context=await get_user_context(user_id),compiled_nodes_input_masks=compiled_nodes_input_masks,+ parent_graph_exec_id=parent_graph_exec_id,)logger.info(f"Created graph execution #{graph_exec.id} for graph "diff --git a/autogpt_platform/backend/backend/executor/utils_test.py b/autogpt_platform/backend/backend/executor/utils_test.pyindex 64f40bbcb6..2946aa4156 100644--- a/autogpt_platform/backend/backend/executor/utils_test.py+++ b/autogpt_platform/backend/backend/executor/utils_test.py@@ -379,6 +379,7 @@ async def test_add_graph_execution_is_repeatable(mocker: MockerFixture):nodes_input_masks=nodes_input_masks,starting_nodes_input=starting_nodes_input,preset_id=preset_id,+ parent_graph_exec_id=None,)# Set up the graph execution mock to have properties we can extractdiff --git a/autogpt_platform/backend/migrations/20251028000000_add_parent_graph_execution_tracking/migration.sql b/autogpt_platform/backend/migrations/20251028000000_add_parent_graph_execution_tracking/migration.sqlnew file mode 100644index 0000000000..ff1f93239c--- /dev/null+++ b/autogpt_platform/backend/migrations/20251028000000_add_parent_graph_execution_tracking/migration.sql@@ -0,0 +1,11 @@+-- Add parent execution tracking for nested agent graphs+-- This enables cascading stop operations and prevents orphaned child executions++-- AlterTable+ALTER TABLE "AgentGraphExecution" ADD COLUMN "parentGraphExecutionId" TEXT;++-- CreateIndex+CREATE INDEX "AgentGraphExecution_parentGraphExecutionId_idx" ON "AgentGraphExecution"("parentGraphExecutionId");++-- AddForeignKey+ALTER TABLE "AgentGraphExecution" ADD CONSTRAINT "AgentGraphExecution_parentGraphExecutionId_fkey" FOREIGN KEY ("parentGraphExecutionId") REFERENCES "AgentGraphExecution"("id") ON DELETE SET NULL ON UPDATE CASCADE;diff --git a/autogpt_platform/backend/schema.prisma b/autogpt_platform/backend/schema.prismaindex b340796f32..e8755d99ab 100644--- a/autogpt_platform/backend/schema.prisma+++ b/autogpt_platform/backend/schema.prisma@@ -383,6 +383,11 @@ model AgentGraphExecution {stats Json?+ // Parent-child execution tracking for nested agent graphs+ parentGraphExecutionId String?+ ParentExecution AgentGraphExecution? @relation("ParentChildExecution", fields: [parentGraphExecutionId], references: [id], onDelete: SetNull)+ ChildExecutions AgentGraphExecution[] @relation("ParentChildExecution")+// Sharing fieldsisShared Boolean @default(false)shareToken String? @unique@@ -393,6 +398,7 @@ model AgentGraphExecution {@@index([createdAt])@@index([agentPresetId])@@index([shareToken])+ @@index([parentGraphExecutionId])}// This model describes the execution of an AgentNode.