AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(backend): Continue instead of retrying aborted/broken agent execution (#9903)
Zamil Majdyd7077b516105
diff --git a/autogpt_platform/backend/backend/data/execution.py b/autogpt_platform/backend/backend/data/execution.pyindex 3ca36f09c1..7eeb000bd8 100644--- a/autogpt_platform/backend/backend/data/execution.py+++ b/autogpt_platform/backend/backend/data/execution.py@@ -208,18 +208,6 @@ class GraphExecutionWithNodes(GraphExecution):graph_id=self.graph_id,graph_version=self.graph_version or 0,graph_exec_id=self.id,- start_node_execs=[- NodeExecutionEntry(- user_id=self.user_id,- graph_exec_id=node_exec.graph_exec_id,- graph_id=node_exec.graph_id,- node_exec_id=node_exec.node_exec_id,- node_id=node_exec.node_id,- block_id=node_exec.block_id,- data=node_exec.input_data,- )- for node_exec in self.node_executions- ],node_credentials_input_map={}, # FIXME)@@ -280,6 +268,17 @@ class NodeExecutionResult(BaseModel):end_time=_node_exec.endedTime,)+ def to_node_execution_entry(self) -> "NodeExecutionEntry":+ return NodeExecutionEntry(+ user_id=self.user_id,+ graph_exec_id=self.graph_exec_id,+ graph_id=self.graph_id,+ node_exec_id=self.node_exec_id,+ node_id=self.node_id,+ block_id=self.block_id,+ data=self.input_data,+ )+# --------------------- Model functions --------------------- #@@ -616,8 +615,9 @@ async def delete_graph_execution()-async def get_node_execution_results(+async def get_node_executions(graph_exec_id: str,+ node_id: str | None = None,block_ids: list[str] | None = None,statuses: list[ExecutionStatus] | None = None,limit: int | None = None,@@ -625,6 +625,8 @@ async def get_node_execution_results(where_clause: AgentNodeExecutionWhereInput = {"agentGraphExecutionId": graph_exec_id,}+ if node_id:+ where_clause["agentNodeId"] = node_idif block_ids:where_clause["Node"] = {"is": {"agentBlockId": {"in": block_ids}}}if statuses:@@ -681,20 +683,6 @@ async def get_latest_node_execution(return NodeExecutionResult.from_db(execution)-async def get_incomplete_node_executions(- node_id: str, graph_eid: str-) -> list[NodeExecutionResult]:- executions = await AgentNodeExecution.prisma().find_many(- where={- "agentNodeId": node_id,- "agentGraphExecutionId": graph_eid,- "executionStatus": ExecutionStatus.INCOMPLETE,- },- include=EXECUTION_RESULT_INCLUDE,- )- return [NodeExecutionResult.from_db(execution) for execution in executions]--# ----------------- Execution Infrastructure ----------------- #@@ -703,7 +691,6 @@ class GraphExecutionEntry(BaseModel):graph_exec_id: strgraph_id: strgraph_version: int- start_node_execs: list["NodeExecutionEntry"]node_credentials_input_map: Optional[dict[str, dict[str, CredentialsMetaInput]]]diff --git a/autogpt_platform/backend/backend/executor/database.py b/autogpt_platform/backend/backend/executor/database.pyindex 953e391c94..1791720fae 100644--- a/autogpt_platform/backend/backend/executor/database.py+++ b/autogpt_platform/backend/backend/executor/database.py@@ -6,9 +6,9 @@ from backend.data.credit import UsageTransactionMetadata, get_user_credit_modelfrom backend.data.execution import (create_graph_execution,get_graph_execution,- get_incomplete_node_executions,+ get_graph_execution_meta,get_latest_node_execution,- get_node_execution_results,+ get_node_executions,update_graph_execution_start_time,update_graph_execution_stats,update_node_execution_stats,@@ -86,9 +86,9 @@ class DatabaseManager(AppService):# Executionsget_graph_execution = _(get_graph_execution)+ get_graph_execution_meta = _(get_graph_execution_meta)create_graph_execution = _(create_graph_execution)- get_node_execution_results = _(get_node_execution_results)- get_incomplete_node_executions = _(get_incomplete_node_executions)+ get_node_executions = _(get_node_executions)get_latest_node_execution = _(get_latest_node_execution)update_node_execution_status = _(update_node_execution_status)update_node_execution_status_batch = _(update_node_execution_status_batch)@@ -142,9 +142,9 @@ class DatabaseManagerClient(AppServiceClient):# Executionsget_graph_execution = _(d.get_graph_execution)+ get_graph_execution_meta = _(d.get_graph_execution_meta)create_graph_execution = _(d.create_graph_execution)- get_node_execution_results = _(d.get_node_execution_results)- get_incomplete_node_executions = _(d.get_incomplete_node_executions)+ get_node_executions = _(d.get_node_executions)get_latest_node_execution = _(d.get_latest_node_execution)update_node_execution_status = _(d.update_node_execution_status)update_node_execution_status_batch = _(d.update_node_execution_status_batch)diff --git a/autogpt_platform/backend/backend/executor/manager.py b/autogpt_platform/backend/backend/executor/manager.pyindex 48b89a8b48..fa0fa8fdd1 100644--- a/autogpt_platform/backend/backend/executor/manager.py+++ b/autogpt_platform/backend/backend/executor/manager.py@@ -373,8 +373,10 @@ def _enqueue_next_nodes(# If link is static, there could be some incomplete executions waiting for it.# Load and complete the input missing input data, and try to re-enqueue them.- for iexec in db_client.get_incomplete_node_executions(- next_node_id, graph_exec_id+ for iexec in db_client.get_node_executions(+ node_id=next_node_id,+ graph_exec_id=graph_exec_id,+ statuses=[ExecutionStatus.INCOMPLETE],):idata = iexec.input_dataineid = iexec.node_exec_id@@ -566,16 +568,35 @@ class Executor:node_eid="*",block_name="-",)- exec_meta = cls.db_client.update_graph_execution_start_time(- graph_exec.graph_exec_id++ exec_meta = cls.db_client.get_graph_execution_meta(+ user_id=graph_exec.user_id,+ execution_id=graph_exec.graph_exec_id,)if exec_meta is None:- logger.warning(- f"Skipped graph execution {graph_exec.graph_exec_id}, the graph execution is not found."+ log_metadata.warning(+ f"Skipped graph execution #{graph_exec.graph_exec_id}, the graph execution is not found."+ )+ return++ if exec_meta.status == ExecutionStatus.QUEUED:+ log_metadata.info(f"⚙️ Starting graph execution #{graph_exec.graph_exec_id}")+ exec_meta.status = ExecutionStatus.RUNNING+ send_execution_update(+ cls.db_client.update_graph_execution_start_time(+ graph_exec.graph_exec_id+ )+ )+ elif exec_meta.status == ExecutionStatus.RUNNING:+ log_metadata.info(+ f"⚙️ Graph execution #{graph_exec.graph_exec_id} is already running, continuing where it left off."+ )+ else:+ log_metadata.warning(+ f"Skipped graph execution {graph_exec.graph_exec_id}, the graph execution status is `{exec_meta.status}`.")return- send_execution_update(exec_meta)timing_info, (exec_stats, status, error) = cls._on_graph_execution(graph_exec, cancel, log_metadata)@@ -653,7 +674,6 @@ class Executor:ExecutionStatus: The final status of the graph execution.Exception | None: The error that occurred during the execution, if any."""- log_metadata.info(f"Start graph execution {graph_exec.graph_exec_id}")execution_stats = GraphExecutionStats()execution_status = ExecutionStatus.RUNNINGerror = None@@ -684,8 +704,11 @@ class Executor:)queue = ExecutionQueue[NodeExecutionEntry]()- for node_exec in graph_exec.start_node_execs:- queue.add(node_exec)+ for node_exec in cls.db_client.get_node_executions(+ graph_exec.graph_exec_id,+ statuses=[ExecutionStatus.RUNNING, ExecutionStatus.QUEUED],+ ):+ queue.add(node_exec.to_node_execution_entry())running_executions: dict[str, AsyncResult] = {}@@ -805,24 +828,21 @@ class Executor:execution.wait(3)log_metadata.info(f"Finished graph execution {graph_exec.graph_exec_id}")+ execution_status = ExecutionStatus.COMPLETEDexcept Exception as e:error = e- finally:- if error:- log_metadata.error(- f"Failed graph execution {graph_exec.graph_exec_id}: {error}"- )- execution_status = ExecutionStatus.FAILED- else:- execution_status = ExecutionStatus.COMPLETED+ log_metadata.error(+ f"Failed graph execution {graph_exec.graph_exec_id}: {error}"+ )+ execution_status = ExecutionStatus.FAILED+ finally:if not cancel.is_set():finished = Truecancel.set()cancel_thread.join()clean_exec_files(graph_exec.graph_exec_id)-return execution_stats, execution_status, error@classmethod@@ -834,7 +854,7 @@ class Executor:metadata = cls.db_client.get_graph_metadata(graph_exec.graph_id, graph_exec.graph_version)- outputs = cls.db_client.get_node_execution_results(+ outputs = cls.db_client.get_node_executions(graph_exec.graph_exec_id,block_ids=[AgentOutputBlock().id],)@@ -1062,7 +1082,7 @@ class ExecutionManager(AppProcess):if hasattr(self, "executor"):log(f"{prefix} ⏳ Shutting down GraphExec pool...")- self.executor.shutdown(cancel_futures=True, wait=True)+ self.executor.shutdown(cancel_futures=False, wait=True)log(f"{prefix} ⏳ Disconnecting Redis...")redis.disconnect()@@ -1087,7 +1107,9 @@ def get_notification_service() -> "NotificationManagerClient":return get_service_client(NotificationManagerClient)-def send_execution_update(entry: GraphExecution | NodeExecutionResult):+def send_execution_update(entry: GraphExecution | NodeExecutionResult | None):+ if entry is None:+ returnreturn get_execution_event_bus().publish(entry)diff --git a/autogpt_platform/backend/backend/server/external/routes/v1.py b/autogpt_platform/backend/backend/server/external/routes/v1.pyindex 2ea8688654..15fe8dc7ee 100644--- a/autogpt_platform/backend/backend/server/external/routes/v1.py+++ b/autogpt_platform/backend/backend/server/external/routes/v1.py@@ -122,7 +122,7 @@ async def get_graph_execution_results(if not graph:raise HTTPException(status_code=404, detail=f"Graph #{graph_id} not found.")- results = await execution_db.get_node_execution_results(graph_exec_id)+ results = await execution_db.get_node_executions(graph_exec_id)last_result = results[-1] if results else Noneexecution_status = (last_result.status if last_result else AgentExecutionStatus.INCOMPLETEdiff --git a/autogpt_platform/backend/backend/server/routers/v1.py b/autogpt_platform/backend/backend/server/routers/v1.pyindex 53b994f491..883615c08e 100644--- a/autogpt_platform/backend/backend/server/routers/v1.py+++ b/autogpt_platform/backend/backend/server/routers/v1.py@@ -663,7 +663,7 @@ async def _cancel_execution(graph_exec_id: str):)node_execs = [node_exec.model_copy(update={"status": execution_db.ExecutionStatus.TERMINATED})- for node_exec in await execution_db.get_node_execution_results(+ for node_exec in await execution_db.get_node_executions(graph_exec_id=graph_exec_id,statuses=[execution_db.ExecutionStatus.QUEUED,