AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(backend): Avoid loading all node executions on large continuos agent (#9667)
Zamil Majdyf01b31873fab
diff --git a/autogpt_platform/backend/backend/data/execution.py b/autogpt_platform/backend/backend/data/execution.pyindex 2ced77d4c9..794284eee0 100644--- a/autogpt_platform/backend/backend/data/execution.py+++ b/autogpt_platform/backend/backend/data/execution.py@@ -10,6 +10,7 @@ from prisma.models import (AgentNodeExecution,AgentNodeExecutionInputOutput,)+from prisma.types import AgentNodeExecutionUpdateInput, AgentNodeExecutionWhereInputfrom pydantic import BaseModelfrom backend.data.block import BlockData, BlockInput, CompletedBlockOutput@@ -292,13 +293,19 @@ async def update_graph_execution_start_time(graph_exec_id: str) -> ExecutionResuasync def update_graph_execution_stats(graph_exec_id: str,status: ExecutionStatus,- stats: GraphExecutionStats,+ stats: GraphExecutionStats | None = None,) -> ExecutionResult:- data = stats.model_dump()- if isinstance(data["error"], Exception):+ data = stats.model_dump() if stats else {}+ if isinstance(data.get("error"), Exception):data["error"] = str(data["error"])res = await AgentGraphExecution.prisma().update(- where={"id": graph_exec_id},+ where={+ "id": graph_exec_id,+ "OR": [+ {"executionStatus": ExecutionStatus.RUNNING},+ {"executionStatus": ExecutionStatus.QUEUED},+ ],+ },data={"executionStatus": status,"stats": Json(data),@@ -320,6 +327,17 @@ async def update_node_execution_stats(node_exec_id: str, stats: NodeExecutionSta)+async def update_execution_status_batch(+ node_exec_ids: list[str],+ status: ExecutionStatus,+ stats: dict[str, Any] | None = None,+):+ await AgentNodeExecution.prisma().update_many(+ where={"id": {"in": node_exec_ids}},+ data=_get_update_status_data(status, None, stats),+ )++async def update_execution_status(node_exec_id: str,status: ExecutionStatus,@@ -329,20 +347,9 @@ async def update_execution_status(if status == ExecutionStatus.QUEUED and execution_data is None:raise ValueError("Execution data must be provided when queuing an execution.")- now = datetime.now(tz=timezone.utc)- data = {- **({"executionStatus": status}),- **({"queuedTime": now} if status == ExecutionStatus.QUEUED else {}),- **({"startedTime": now} if status == ExecutionStatus.RUNNING else {}),- **({"endedTime": now} if status == ExecutionStatus.FAILED else {}),- **({"endedTime": now} if status == ExecutionStatus.COMPLETED else {}),- **({"executionData": Json(execution_data)} if execution_data else {}),- **({"stats": Json(stats)} if stats else {}),- }-res = await AgentNodeExecution.prisma().update(where={"id": node_exec_id},- data=data, # type: ignore+ data=_get_update_status_data(status, execution_data, stats),include=EXECUTION_RESULT_INCLUDE,)if not res:@@ -351,6 +358,29 @@ async def update_execution_status(return ExecutionResult.from_db(res)+def _get_update_status_data(+ status: ExecutionStatus,+ execution_data: BlockInput | None = None,+ stats: dict[str, Any] | None = None,+) -> AgentNodeExecutionUpdateInput:+ now = datetime.now(tz=timezone.utc)+ update_data: AgentNodeExecutionUpdateInput = {"executionStatus": status}++ if status == ExecutionStatus.QUEUED:+ update_data["queuedTime"] = now+ elif status == ExecutionStatus.RUNNING:+ update_data["startedTime"] = now+ elif status in (ExecutionStatus.FAILED, ExecutionStatus.COMPLETED):+ update_data["endedTime"] = now++ if execution_data:+ update_data["executionData"] = Json(execution_data)+ if stats:+ update_data["stats"] = Json(stats)++ return update_data++async def delete_execution(graph_exec_id: str, user_id: str, soft_delete: bool = True) -> None:@@ -368,41 +398,29 @@ async def delete_execution()-async def get_execution_results(graph_exec_id: str) -> list[ExecutionResult]:+async def get_execution_results(+ graph_exec_id: str,+ block_ids: list[str] | None = None,+ statuses: list[ExecutionStatus] | None = None,+ limit: int | None = None,+) -> list[ExecutionResult]:+ where_clause: AgentNodeExecutionWhereInput = {+ "agentGraphExecutionId": graph_exec_id,+ }+ if block_ids:+ where_clause["AgentNode"] = {"is": {"agentBlockId": {"in": block_ids}}}+ if statuses:+ where_clause["OR"] = [{"executionStatus": status} for status in statuses]+executions = await AgentNodeExecution.prisma().find_many(- where={"agentGraphExecutionId": graph_exec_id},+ where=where_clause,include=EXECUTION_RESULT_INCLUDE,- order=[- {"queuedTime": "asc"},- {"addedTime": "asc"}, # Fallback: Incomplete execs has no queuedTime.- ],+ take=limit,)res = [ExecutionResult.from_db(execution) for execution in executions]return res-async def get_executions_in_timerange(- user_id: str, start_time: str, end_time: str-) -> list[ExecutionResult]:- try:- executions = await AgentGraphExecution.prisma().find_many(- where={- "startedAt": {- "gte": datetime.fromisoformat(start_time),- "lte": datetime.fromisoformat(end_time),- },- "userId": user_id,- "isDeleted": False,- },- include=GRAPH_EXECUTION_INCLUDE,- )- return [ExecutionResult.from_graph(execution) for execution in executions]- except Exception as e:- raise DatabaseError(- f"Failed to get executions in timerange {start_time} to {end_time} for user {user_id}: {e}"- ) from e--LIST_SPLIT = "_$_"DICT_SPLIT = "_#_"OBJC_SPLIT = "_@_"@@ -545,7 +563,10 @@ async def get_latest_execution(node_id: str, graph_eid: str) -> ExecutionResult"agentGraphExecutionId": graph_eid,"executionStatus": {"not": ExecutionStatus.INCOMPLETE}, # type: ignore},- order={"queuedTime": "desc"},+ order=[+ {"queuedTime": "desc"},+ {"addedTime": "desc"},+ ],include=EXECUTION_RESULT_INCLUDE,)if not execution:diff --git a/autogpt_platform/backend/backend/data/graph.py b/autogpt_platform/backend/backend/data/graph.pyindex 693ca14023..8fa7a9cf07 100644--- a/autogpt_platform/backend/backend/data/graph.py+++ b/autogpt_platform/backend/backend/data/graph.py@@ -23,7 +23,7 @@ from backend.util import type as type_utilsfrom .block import Block, BlockInput, BlockSchema, BlockType, get_block, get_blocksfrom .db import BaseDbModel, transactionfrom .execution import ExecutionResult, ExecutionStatus-from .includes import AGENT_GRAPH_INCLUDE, AGENT_NODE_INCLUDE+from .includes import AGENT_GRAPH_INCLUDE, AGENT_NODE_INCLUDE, GRAPH_EXECUTION_INCLUDEfrom .integrations import Webhooklogger = logging.getLogger(__name__)@@ -216,10 +216,13 @@ class GraphExecution(GraphExecutionMeta):graph_exec = GraphExecutionMeta.from_db(_graph_exec)- node_executions = [- ExecutionResult.from_db(ne, _graph_exec.userId)- for ne in _graph_exec.AgentNodeExecutions- ]+ node_executions = sorted(+ [+ ExecutionResult.from_db(ne, _graph_exec.userId)+ for ne in _graph_exec.AgentNodeExecutions+ ],+ key=lambda ne: (ne.queue_time is None, ne.queue_time or ne.add_time),+ )inputs = {**{@@ -658,20 +661,13 @@ async def get_execution_meta(return GraphExecutionMeta.from_db(execution) if execution else None-async def get_execution(user_id: str, execution_id: str) -> GraphExecution | None:+async def get_execution(+ user_id: str,+ execution_id: str,+) -> GraphExecution | None:execution = await AgentGraphExecution.prisma().find_first(where={"id": execution_id, "isDeleted": False, "userId": user_id},- include={- "AgentNodeExecutions": {- "include": {"AgentNode": True, "Input": True, "Output": True},- "order_by": [- {"queuedTime": "asc"},- { # Fallback: Incomplete execs has no queuedTime.- "addedTime": "asc"- },- ],- },- },+ include=GRAPH_EXECUTION_INCLUDE,)return GraphExecution.from_db(execution) if execution else Nonediff --git a/autogpt_platform/backend/backend/data/includes.py b/autogpt_platform/backend/backend/data/includes.pyindex badf7804f2..331c292054 100644--- a/autogpt_platform/backend/backend/data/includes.py+++ b/autogpt_platform/backend/backend/data/includes.py@@ -18,6 +18,8 @@ EXECUTION_RESULT_INCLUDE: prisma.types.AgentNodeExecutionInclude = {"AgentGraphExecution": True,}+MAX_NODE_EXECUTIONS_FETCH = 1000+GRAPH_EXECUTION_INCLUDE: prisma.types.AgentGraphExecutionInclude = {"AgentNodeExecutions": {"include": {@@ -25,10 +27,17 @@ GRAPH_EXECUTION_INCLUDE: prisma.types.AgentGraphExecutionInclude = {"Output": True,"AgentNode": True,"AgentGraphExecution": True,- }+ },+ "order_by": [+ {"queuedTime": "desc"},+ # Fallback: Incomplete execs has no queuedTime.+ {"addedTime": "desc"},+ ],+ "take": MAX_NODE_EXECUTIONS_FETCH, # Avoid loading excessive node executions.}}+INTEGRATION_WEBHOOK_INCLUDE: prisma.types.IntegrationWebhookInclude = {"AgentNodes": {"include": AGENT_NODE_INCLUDE} # type: ignore}diff --git a/autogpt_platform/backend/backend/executor/database.py b/autogpt_platform/backend/backend/executor/database.pyindex e906e5e674..765ad3c200 100644--- a/autogpt_platform/backend/backend/executor/database.py+++ b/autogpt_platform/backend/backend/executor/database.py@@ -8,6 +8,7 @@ from backend.data.execution import (get_incomplete_executions,get_latest_execution,update_execution_status,+ update_execution_status_batch,update_graph_execution_start_time,update_graph_execution_stats,update_node_execution_stats,@@ -69,6 +70,7 @@ class DatabaseManager(AppService):get_incomplete_executions = exposed_run_and_wait(get_incomplete_executions)get_latest_execution = exposed_run_and_wait(get_latest_execution)update_execution_status = exposed_run_and_wait(update_execution_status)+ update_execution_status_batch = exposed_run_and_wait(update_execution_status_batch)update_graph_execution_start_time = exposed_run_and_wait(update_graph_execution_start_time)diff --git a/autogpt_platform/backend/backend/executor/manager.py b/autogpt_platform/backend/backend/executor/manager.pyindex cbe9b986fa..fdadfe04a5 100644--- a/autogpt_platform/backend/backend/executor/manager.py+++ b/autogpt_platform/backend/backend/executor/manager.py@@ -783,7 +783,10 @@ class Executor:metadata = cls.db_client.get_graph_metadata(graph_exec.graph_id, graph_exec.graph_version)- outputs = cls.db_client.get_execution_results(graph_exec.graph_exec_id)+ outputs = cls.db_client.get_execution_results(+ graph_exec.graph_exec_id,+ block_ids=[AgentOutputBlock().id],+ )named_outputs = [{@@ -791,7 +794,6 @@ class Executor:for key, value in output.output_data.items()}for output in outputs- if output.block_id == AgentOutputBlock().id]event = NotificationEventDTO(@@ -985,29 +987,36 @@ class ExecutionManager(AppService):3. Update execution statuses in DB and set `error` outputs to `"TERMINATED"`."""if graph_exec_id not in self.active_graph_runs:- raise Exception(+ logger.warning(f"Graph execution #{graph_exec_id} not active/running: ""possibly already completed/cancelled.")-- future, cancel_event = self.active_graph_runs[graph_exec_id]- if cancel_event.is_set():- return-- cancel_event.set()- future.result()-- # Update the status of the unfinished node executions- node_execs = self.db_client.get_execution_results(graph_exec_id)+ else:+ future, cancel_event = self.active_graph_runs[graph_exec_id]+ if not cancel_event.is_set():+ cancel_event.set()+ future.result()++ # Update the status of the graph & node executions+ self.db_client.update_graph_execution_stats(+ graph_exec_id,+ ExecutionStatus.TERMINATED,+ )+ node_execs = self.db_client.get_execution_results(+ graph_exec_id=graph_exec_id,+ statuses=[+ ExecutionStatus.QUEUED,+ ExecutionStatus.RUNNING,+ ExecutionStatus.INCOMPLETE,+ ],+ )+ self.db_client.update_execution_status_batch(+ [node_exec.node_exec_id for node_exec in node_execs],+ ExecutionStatus.TERMINATED,+ )for node_exec in node_execs:- if node_exec.status not in (- ExecutionStatus.COMPLETED,- ExecutionStatus.FAILED,- ):- exec_update = self.db_client.update_execution_status(- node_exec.node_exec_id, ExecutionStatus.TERMINATED- )- self.db_client.send_execution_update(exec_update)+ node_exec.status = ExecutionStatus.TERMINATED+ self.db_client.send_execution_update(node_exec)def _validate_node_input_credentials(self, graph: GraphModel, user_id: str):"""Checks all credentials for all nodes of the graph"""diff --git a/autogpt_platform/backend/backend/server/routers/v1.py b/autogpt_platform/backend/backend/server/routers/v1.pyindex 9d69be1403..43b2ffd309 100644--- a/autogpt_platform/backend/backend/server/routers/v1.py+++ b/autogpt_platform/backend/backend/server/routers/v1.py@@ -640,12 +640,8 @@ async def get_graph_execution(graph_exec_id: str,user_id: Annotated[str, Depends(get_user_id)],) -> graph_db.GraphExecution:- graph = await graph_db.get_graph(graph_id, user_id=user_id)- if not graph:- raise HTTPException(status_code=404, detail=f"Graph #{graph_id} not found.")-result = await graph_db.get_execution(execution_id=graph_exec_id, user_id=user_id)- if not result:+ if not result or result.graph_id != graph_id:raise HTTPException(status_code=404, detail=f"Graph execution #{graph_exec_id} not found.")