Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

feat(backend/executor): Implement cascading stop for nested graph executions (#11277)

Zamil Majdy4922f8885120
diff --git a/autogpt_platform/backend/backend/blocks/agent.py b/autogpt_platform/backend/backend/blocks/agent.py
index 68fef45ada..b412a1b3a6 100644
--- a/autogpt_platform/backend/backend/blocks/agent.py
+++ b/autogpt_platform/backend/backend/blocks/agent.py
@@ -65,7 +65,13 @@ class AgentExecutorBlock(Block):
categories={BlockCategory.AGENT},
)
- async def run(self, input_data: Input, **kwargs) -> BlockOutput:
+ async def run(
+ self,
+ input_data: Input,
+ *,
+ graph_exec_id: str,
+ **kwargs,
+ ) -> BlockOutput:
from backend.executor import utils as execution_utils
@@ -75,6 +81,7 @@ class AgentExecutorBlock(Block):
user_id=input_data.user_id,
inputs=input_data.inputs,
nodes_input_masks=input_data.nodes_input_masks,
+ parent_graph_exec_id=graph_exec_id,
)
logger = execution_utils.LogMetadata(
diff --git a/autogpt_platform/backend/backend/data/execution.py b/autogpt_platform/backend/backend/data/execution.py
index 6c40e55a31..922672acd0 100644
--- a/autogpt_platform/backend/backend/data/execution.py
+++ b/autogpt_platform/backend/backend/data/execution.py
@@ -346,6 +346,7 @@ class GraphExecutionWithNodes(GraphExecution):
self,
user_context: "UserContext",
compiled_nodes_input_masks: Optional[NodesInputMasks] = None,
+ parent_graph_exec_id: Optional[str] = None,
):
return GraphExecutionEntry(
user_id=self.user_id,
@@ -354,6 +355,7 @@ class GraphExecutionWithNodes(GraphExecution):
graph_exec_id=self.id,
nodes_input_masks=compiled_nodes_input_masks,
user_context=user_context,
+ parent_graph_exec_id=parent_graph_exec_id,
)
@@ -632,6 +634,25 @@ async def get_graph_execution(
)
+async def get_child_graph_executions(
+ parent_exec_id: str,
+) -> list[GraphExecutionMeta]:
+ """
+ Get all child executions of a parent execution.
+
+ Args:
+ parent_exec_id: Parent graph execution ID
+
+ Returns:
+ List of child graph executions
+ """
+ children = await AgentGraphExecution.prisma().find_many(
+ where={"parentGraphExecutionId": parent_exec_id, "isDeleted": False}
+ )
+
+ return [GraphExecutionMeta.from_db(child) for child in children]
+
+
async def create_graph_execution(
graph_id: str,
graph_version: int,
@@ -641,6 +662,7 @@ async def create_graph_execution(
preset_id: Optional[str] = None,
credential_inputs: Optional[Mapping[str, CredentialsMetaInput]] = None,
nodes_input_masks: Optional[NodesInputMasks] = None,
+ parent_graph_exec_id: Optional[str] = None,
) -> GraphExecutionWithNodes:
"""
Create a new AgentGraphExecution record.
@@ -677,6 +699,7 @@ async def create_graph_execution(
},
"userId": user_id,
"agentPresetId": preset_id,
+ "parentGraphExecutionId": parent_graph_exec_id,
},
include=GRAPH_EXECUTION_INCLUDE_WITH_NODES,
)
@@ -1007,6 +1030,7 @@ class GraphExecutionEntry(BaseModel):
graph_version: int
nodes_input_masks: Optional[NodesInputMasks] = None
user_context: UserContext
+ parent_graph_exec_id: Optional[str] = None
class NodeExecutionEntry(BaseModel):
diff --git a/autogpt_platform/backend/backend/executor/database.py b/autogpt_platform/backend/backend/executor/database.py
index 929a502d7d..df581e0de4 100644
--- a/autogpt_platform/backend/backend/executor/database.py
+++ b/autogpt_platform/backend/backend/executor/database.py
@@ -7,6 +7,7 @@ from backend.data.credit import UsageTransactionMetadata, get_user_credit_model
from backend.data.execution import (
create_graph_execution,
get_block_error_stats,
+ get_child_graph_executions,
get_execution_kv_data,
get_graph_execution_meta,
get_graph_executions,
@@ -121,6 +122,7 @@ class DatabaseManager(AppService):
return cast(Callable[Concatenate[object, P], R], expose(f))
# Executions
+ get_child_graph_executions = _(get_child_graph_executions)
get_graph_executions = _(get_graph_executions)
get_graph_executions_count = _(get_graph_executions_count)
get_graph_execution_meta = _(get_graph_execution_meta)
@@ -234,6 +236,7 @@ class DatabaseManagerAsyncClient(AppServiceClient):
return DatabaseManager
create_graph_execution = d.create_graph_execution
+ get_child_graph_executions = d.get_child_graph_executions
get_connected_output_nodes = d.get_connected_output_nodes
get_latest_node_execution = d.get_latest_node_execution
get_graph = d.get_graph
diff --git a/autogpt_platform/backend/backend/executor/manager.py b/autogpt_platform/backend/backend/executor/manager.py
index 2abedcf133..cfe0cc252a 100644
--- a/autogpt_platform/backend/backend/executor/manager.py
+++ b/autogpt_platform/backend/backend/executor/manager.py
@@ -1335,6 +1335,9 @@ class ExecutionManager(AppProcess):
return self._run_client
def run(self):
+ logger.info(
+ f"[{self.service_name}] 🆔 Pod assigned executor_id: {self.executor_id}"
+ )
logger.info(f"[{self.service_name}] ⏳ Spawn max-{self.pool_size} workers...")
pool_size_gauge.set(self.pool_size)
@@ -1495,10 +1498,37 @@ class ExecutionManager(AppProcess):
graph_exec_id = graph_exec_entry.graph_exec_id
user_id = graph_exec_entry.user_id
graph_id = graph_exec_entry.graph_id
+ parent_graph_exec_id = graph_exec_entry.parent_graph_exec_id
+
logger.info(
- f"[{self.service_name}] Received RUN for graph_exec_id={graph_exec_id}, user_id={user_id}"
+ f"[{self.service_name}] Received RUN for graph_exec_id={graph_exec_id}, user_id={user_id}, executor_id={self.executor_id}"
+ + (f", parent={parent_graph_exec_id}" if parent_graph_exec_id else "")
)
+ # Check if parent execution is already terminated (prevents orphaned child executions)
+ if parent_graph_exec_id:
+ try:
+ parent_exec = get_db_client().get_graph_execution_meta(
+ execution_id=parent_graph_exec_id,
+ user_id=user_id,
+ )
+ if parent_exec and parent_exec.status == ExecutionStatus.TERMINATED:
+ logger.info(
+ f"[{self.service_name}] Skipping execution {graph_exec_id} - parent {parent_graph_exec_id} is TERMINATED"
+ )
+ # Mark this child as terminated since parent was stopped
+ get_db_client().update_graph_execution_stats(
+ graph_exec_id=graph_exec_id,
+ status=ExecutionStatus.TERMINATED,
+ )
+ _ack_message(reject=False, requeue=False)
+ return
+ except Exception as e:
+ logger.warning(
+ f"[{self.service_name}] Could not check parent status for {graph_exec_id}: {e}"
+ )
+ # Continue execution if parent check fails (don't block on errors)
+
# Check user rate limit before processing
try:
# Only check executions from the last 24 hours for performance
@@ -1546,7 +1576,7 @@ class ExecutionManager(AppProcess):
# Either someone else has it or Redis is unavailable
if current_owner is not None:
logger.warning(
- f"[{self.service_name}] Graph {graph_exec_id} already running on pod {current_owner}"
+ f"[{self.service_name}] Graph {graph_exec_id} already running on pod {current_owner}, current executor_id={self.executor_id}"
)
_ack_message(reject=True, requeue=False)
else:
@@ -1561,7 +1591,7 @@ class ExecutionManager(AppProcess):
self._execution_locks[graph_exec_id] = cluster_lock
logger.info(
- f"[{self.service_name}] Acquired cluster lock for {graph_exec_id} with executor {self.executor_id}"
+ f"[{self.service_name}] Successfully acquired cluster lock for {graph_exec_id}, executor_id={self.executor_id}"
)
cancel_event = threading.Event()
@@ -1598,6 +1628,9 @@ class ExecutionManager(AppProcess):
finally:
# Release the cluster-wide execution lock
if graph_exec_id in self._execution_locks:
+ logger.info(
+ f"[{self.service_name}] Releasing cluster lock for {graph_exec_id}, executor_id={self.executor_id}"
+ )
self._execution_locks[graph_exec_id].release()
del self._execution_locks[graph_exec_id]
self._cleanup_completed_runs()
diff --git a/autogpt_platform/backend/backend/executor/utils.py b/autogpt_platform/backend/backend/executor/utils.py
index 445a9ca7d8..ac3ec9110e 100644
--- a/autogpt_platform/backend/backend/executor/utils.py
+++ b/autogpt_platform/backend/backend/executor/utils.py
@@ -25,6 +25,7 @@ from backend.data.db import prisma
from backend.data.dynamic_fields import merge_execution_input
from backend.data.execution import (
ExecutionStatus,
+ GraphExecutionMeta,
GraphExecutionStats,
GraphExecutionWithNodes,
NodesInputMasks,
@@ -621,20 +622,74 @@ class CancelExecutionEvent(BaseModel):
graph_exec_id: str
+async def _get_child_executions(parent_exec_id: str) -> list["GraphExecutionMeta"]:
+ """
+ Get all child executions of a parent execution using the execution_db pattern.
+
+ Args:
+ parent_exec_id: Parent graph execution ID
+
+ Returns:
+ List of child graph executions
+ """
+ from backend.data.db import prisma
+
+ if prisma.is_connected():
+ edb = execution_db
+ else:
+ edb = get_database_manager_async_client()
+
+ return await edb.get_child_graph_executions(parent_exec_id)
+
+
async def stop_graph_execution(
user_id: str,
graph_exec_id: str,
wait_timeout: float = 15.0,
+ cascade: bool = True,
):
"""
+ Stop a graph execution and optionally all its child executions.
+
Mechanism:
- 1. Set the cancel event
- 2. Graph executor's cancel handler thread detects the event, terminates workers,
+ 1. Set the cancel event for this execution
+ 2. If cascade=True, recursively stop all child executions
+ 3. Graph executor's cancel handler thread detects the event, terminates workers,
reinitializes worker pool, and returns.
- 3. Update execution statuses in DB and set `error` outputs to `"TERMINATED"`.
+ 4. Update execution statuses in DB and set `error` outputs to `"TERMINATED"`.
+
+ Args:
+ user_id: User ID who owns the execution
+ graph_exec_id: Graph execution ID to stop
+ wait_timeout: Maximum time to wait for execution to stop (seconds)
+ cascade: If True, recursively stop all child executions
"""
queue_client = await get_async_execution_queue()
db = execution_db if prisma.is_connected() else get_database_manager_async_client()
+
+ # First, find and stop all child executions if cascading
+ if cascade:
+ children = await _get_child_executions(graph_exec_id)
+ logger.info(
+ f"Stopping {len(children)} child executions of execution {graph_exec_id}"
+ )
+
+ # Stop all children in parallel (recursively, with cascading enabled)
+ if children:
+ await asyncio.gather(
+ *[
+ stop_graph_execution(
+ user_id=user_id,
+ graph_exec_id=child.id,
+ wait_timeout=wait_timeout,
+ cascade=True, # Recursively cascade to grandchildren
+ )
+ for child in children
+ ],
+ return_exceptions=True, # Don't fail parent stop if child stop fails
+ )
+
+ # Now stop this execution
await queue_client.publish_message(
routing_key="",
message=CancelExecutionEvent(graph_exec_id=graph_exec_id).model_dump_json(),
@@ -698,6 +753,7 @@ async def add_graph_execution(
graph_version: Optional[int] = None,
graph_credentials_inputs: Optional[Mapping[str, CredentialsMetaInput]] = None,
nodes_input_masks: Optional[NodesInputMasks] = None,
+ parent_graph_exec_id: Optional[str] = None,
) -> GraphExecutionWithNodes:
"""
Adds a graph execution to the queue and returns the execution entry.
@@ -711,6 +767,7 @@ async def add_graph_execution(
graph_credentials_inputs: Credentials inputs to use in the execution.
Keys should map to the keys generated by `GraphModel.aggregate_credentials_inputs`.
nodes_input_masks: Node inputs to use in the execution.
+ parent_graph_exec_id: The ID of the parent graph execution (for nested executions).
Returns:
GraphExecutionEntry: The entry for the graph execution.
Raises:
@@ -745,11 +802,13 @@ async def add_graph_execution(
nodes_input_masks=nodes_input_masks,
starting_nodes_input=starting_nodes_input,
preset_id=preset_id,
+ parent_graph_exec_id=parent_graph_exec_id,
)
graph_exec_entry = graph_exec.to_graph_execution_entry(
user_context=await get_user_context(user_id),
compiled_nodes_input_masks=compiled_nodes_input_masks,
+ parent_graph_exec_id=parent_graph_exec_id,
)
logger.info(
f"Created graph execution #{graph_exec.id} for graph "
diff --git a/autogpt_platform/backend/backend/executor/utils_test.py b/autogpt_platform/backend/backend/executor/utils_test.py
index 64f40bbcb6..2946aa4156 100644
--- a/autogpt_platform/backend/backend/executor/utils_test.py
+++ b/autogpt_platform/backend/backend/executor/utils_test.py
@@ -379,6 +379,7 @@ async def test_add_graph_execution_is_repeatable(mocker: MockerFixture):
nodes_input_masks=nodes_input_masks,
starting_nodes_input=starting_nodes_input,
preset_id=preset_id,
+ parent_graph_exec_id=None,
)
# Set up the graph execution mock to have properties we can extract
diff --git a/autogpt_platform/backend/migrations/20251028000000_add_parent_graph_execution_tracking/migration.sql b/autogpt_platform/backend/migrations/20251028000000_add_parent_graph_execution_tracking/migration.sql
new file mode 100644
index 0000000000..ff1f93239c
--- /dev/null
+++ b/autogpt_platform/backend/migrations/20251028000000_add_parent_graph_execution_tracking/migration.sql
@@ -0,0 +1,11 @@
+-- Add parent execution tracking for nested agent graphs
+-- This enables cascading stop operations and prevents orphaned child executions
+
+-- AlterTable
+ALTER TABLE "AgentGraphExecution" ADD COLUMN "parentGraphExecutionId" TEXT;
+
+-- CreateIndex
+CREATE INDEX "AgentGraphExecution_parentGraphExecutionId_idx" ON "AgentGraphExecution"("parentGraphExecutionId");
+
+-- AddForeignKey
+ALTER TABLE "AgentGraphExecution" ADD CONSTRAINT "AgentGraphExecution_parentGraphExecutionId_fkey" FOREIGN KEY ("parentGraphExecutionId") REFERENCES "AgentGraphExecution"("id") ON DELETE SET NULL ON UPDATE CASCADE;
diff --git a/autogpt_platform/backend/schema.prisma b/autogpt_platform/backend/schema.prisma
index b340796f32..e8755d99ab 100644
--- a/autogpt_platform/backend/schema.prisma
+++ b/autogpt_platform/backend/schema.prisma
@@ -383,6 +383,11 @@ model AgentGraphExecution {
stats Json?
+ // Parent-child execution tracking for nested agent graphs
+ parentGraphExecutionId String?
+ ParentExecution AgentGraphExecution? @relation("ParentChildExecution", fields: [parentGraphExecutionId], references: [id], onDelete: SetNull)
+ ChildExecutions AgentGraphExecution[] @relation("ParentChildExecution")
+
// Sharing fields
isShared Boolean @default(false)
shareToken String? @unique
@@ -393,6 +398,7 @@ model AgentGraphExecution {
@@index([createdAt])
@@index([agentPresetId])
@@index([shareToken])
+ @@index([parentGraphExecutionId])
}
// This model describes the execution of an AgentNode.