AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(backend/executor): Avoid executor premature termination on inflight agent execution (#10552)
Zamil Majdyf9b255fb7abe
diff --git a/autogpt_platform/backend/backend/app.py b/autogpt_platform/backend/backend/app.pyindex 0ba906bc95..596962ae0b 100644--- a/autogpt_platform/backend/backend/app.py+++ b/autogpt_platform/backend/backend/app.py@@ -43,11 +43,11 @@ def main(**kwargs):run_processes(DatabaseManager().set_log_level("warning"),- ExecutionManager(),Scheduler(),NotificationManager(),WebsocketServer(),AgentServer(),+ ExecutionManager(),**kwargs,)diff --git a/autogpt_platform/backend/backend/blocks/agent.py b/autogpt_platform/backend/backend/blocks/agent.pyindex d430367d89..c777084597 100644--- a/autogpt_platform/backend/backend/blocks/agent.py+++ b/autogpt_platform/backend/backend/blocks/agent.py@@ -199,5 +199,5 @@ class AgentExecutorBlock(Block):wait_timeout=3600,)logger.info(f"Execution {log_id} stopped successfully.")- except Exception as e:- logger.error(f"Failed to stop execution {log_id}: {e}")+ except TimeoutError as e:+ logger.error(f"Execution {log_id} stop timed out: {e}")diff --git a/autogpt_platform/backend/backend/executor/manager.py b/autogpt_platform/backend/backend/executor/manager.pyindex 4713b7d066..c61b9cb0a6 100644--- a/autogpt_platform/backend/backend/executor/manager.py+++ b/autogpt_platform/backend/backend/executor/manager.py@@ -2,7 +2,6 @@ import asyncioimport loggingimport multiprocessingimport os-import sysimport threadingimport timefrom collections import defaultdict@@ -57,6 +56,7 @@ from backend.data.execution import ()from backend.data.graph import Link, Nodefrom backend.executor.utils import (+ GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS,GRAPH_EXECUTION_CANCEL_QUEUE_NAME,GRAPH_EXECUTION_QUEUE_NAME,CancelExecutionEvent,@@ -66,7 +66,6 @@ from backend.executor.utils import (execution_usage_cost,get_async_execution_event_bus,get_execution_event_bus,- get_execution_queue,parse_execution_output,validate_exec,)@@ -522,7 +521,7 @@ class Executor:else:# CancelledError or SystemExitlog_metadata.warning(- f"Interuption error on node execution {node_exec.node_exec_id}: {type(e).__name__}"+ f"Interruption error on node execution {node_exec.node_exec_id}: {type(e).__name__}")status = ExecutionStatus.TERMINATED@@ -714,6 +713,13 @@ class Executor:log_metadata: LogMetadata,execution_stats: GraphExecutionStats,) -> ExecutionStatus:++ # Agent execution is uninterrupted.+ import signal++ signal.signal(signal.SIGINT, signal.SIG_IGN)+ signal.signal(signal.SIGTERM, signal.SIG_IGN)+"""Returns:dict: The execution statistics of the graph execution.@@ -892,11 +898,11 @@ class Executor:return execution_status- except BaseException as exc:+ except BaseException as e:error = (- exc- if isinstance(exc, Exception)- else Exception(f"{exc.__class__.__name__}: {exc}")+ e+ if isinstance(e, Exception)+ else Exception(f"{e.__class__.__name__}: {e}"))known_errors = (InsufficientBalanceError,)@@ -1081,42 +1087,57 @@ class ExecutionManager(AppProcess):def __init__(self):super().__init__()self.pool_size = settings.config.num_graph_workers- self.running = Trueself.active_graph_runs: dict[str, tuple[Future, threading.Event]] = {}+ self._stop_consuming = None+ self._executor = None+ self._cancel_client = None+ self._run_client = None++ @property+ def stop_consuming(self) -> threading.Event:+ if self._stop_consuming is None:+ self._stop_consuming = threading.Event()+ return self._stop_consuming++ @property+ def executor(self) -> ProcessPoolExecutor:+ if self._executor is None:+ self._executor = ProcessPoolExecutor(+ max_workers=self.pool_size,+ initializer=Executor.on_graph_executor_start,+ )+ return self._executordef run(self):+ logger.info(f"[{self.service_name}] ⏳ Spawn max-{self.pool_size} workers...")+pool_size_gauge.set(self.pool_size)- active_runs_gauge.set(0)- utilization_gauge.set(0)+ self._update_prompt_metrics()- self.metrics_server = threading.Thread(- target=start_http_server,- args=(settings.config.execution_manager_port,),+ threading.Thread(+ target=lambda: self._consume_execution_cancel(),daemon=True,- )- self.metrics_server.start()- logger.info(f"[{self.service_name}] Starting execution manager...")- self._run()+ ).start()- def _run(self):- logger.info(f"[{self.service_name}] ⏳ Spawn max-{self.pool_size} workers...")- self.executor = ProcessPoolExecutor(- max_workers=self.pool_size,- initializer=Executor.on_graph_executor_start,- )+ threading.Thread(+ target=lambda: self._consume_execution_run(),+ daemon=True,+ ).start()threading.Thread(- target=lambda: self._consume_execution_cancel(),+ target=start_http_server,+ args=(settings.config.execution_manager_port,),daemon=True,).start()- self._consume_execution_run()+ while not self.stop_consuming.is_set():+ time.sleep(1e5)@continuous_retry()def _consume_execution_cancel(self):- cancel_client = SyncRabbitMQ(create_execution_queue_config())- cancel_client.connect()- cancel_channel = cancel_client.get_channel()+ self._cancel_client = SyncRabbitMQ(create_execution_queue_config())+ self._cancel_client.connect()+ cancel_channel = self._cancel_client.get_channel()logger.info(f"[{self.service_name}] ⏳ Starting cancel message consumer...")cancel_channel.basic_consume(queue=GRAPH_EXECUTION_CANCEL_QUEUE_NAME,@@ -1128,15 +1149,19 @@ class ExecutionManager(AppProcess):@continuous_retry()def _consume_execution_run(self):-# Long-running executions are handled by:# 1. Long consumer timeout (x-consumer-timeout) allows long running agent# 2. Enhanced connection settings (5 retries, 1s delay) for quick reconnection# 3. Process monitoring ensures failed executors release messages back to queue+ if self.stop_consuming.is_set():+ logger.info(+ f"[{self.service_name}] Stop reconnecting execution consumer since the service is cleaned up."+ )+ return- run_client = SyncRabbitMQ(create_execution_queue_config())- run_client.connect()- run_channel = run_client.get_channel()+ self._run_client = SyncRabbitMQ(create_execution_queue_config())+ self._run_client.connect()+ run_channel = self._run_client.get_channel()run_channel.basic_qos(prefetch_count=self.pool_size)# Configure consumer for long-running graph executions@@ -1150,15 +1175,27 @@ class ExecutionManager(AppProcess):run_channel.confirm_delivery()logger.info(f"[{self.service_name}] ⏳ Starting to consume run messages...")- run_channel.start_consuming()- raise RuntimeError(f"❌ run message consumer is stopped: {run_channel}")++ # Continue consuming messages until stop flag is set+ # This keeps the connection alive but rejects new messages in _handle_run_message+ while not self.stop_consuming.is_set():+ try:+ run_channel.connection.process_data_events(time_limit=1)+ except Exception as e:+ if self.stop_consuming.is_set():+ # Expected during shutdown+ break+ logger.error(f"[{self.service_name}] Error processing events: {e}")+ raise++ logger.info(f"[{self.service_name}] ✅ Run message consumer stopped gracefully")@error_logged(swallow=True)def _handle_cancel_message(self,- channel: BlockingChannel,- method: Basic.Deliver,- properties: BasicProperties,+ _channel: BlockingChannel,+ _method: Basic.Deliver,+ _properties: BasicProperties,body: bytes,):"""@@ -1191,11 +1228,19 @@ class ExecutionManager(AppProcess):self,channel: BlockingChannel,method: Basic.Deliver,- properties: BasicProperties,+ _properties: BasicProperties,body: bytes,):delivery_tag = method.delivery_tag+ # Check if we're shutting down - reject new messages but keep connection alive+ if self.stop_consuming.is_set():+ logger.info(+ f"[{self.service_name}] Rejecting new execution during shutdown"+ )+ channel.basic_nack(delivery_tag, requeue=True)+ return+# Check if we can accept more runsself._cleanup_completed_runs()if len(self.active_graph_runs) >= self.pool_size:@@ -1205,7 +1250,9 @@ class ExecutionManager(AppProcess):try:graph_exec_entry = GraphExecutionEntry.model_validate_json(body)except Exception as e:- logger.error(f"[{self.service_name}] Could not parse run message: {e}")+ logger.error(+ f"[{self.service_name}] Could not parse run message: {e}, body={body}"+ )channel.basic_nack(delivery_tag, requeue=False)return@@ -1227,8 +1274,7 @@ class ExecutionManager(AppProcess):Executor.on_graph_execution, graph_exec_entry, cancel_event)self.active_graph_runs[graph_exec_id] = (future, cancel_event)- active_runs_gauge.set(len(self.active_graph_runs))- utilization_gauge.set(len(self.active_graph_runs) / self.pool_size)+ self._update_prompt_metrics()def _on_run_done(f: Future):logger.info(f"[{self.service_name}] Run completed for {graph_exec_id}")@@ -1263,7 +1309,7 @@ class ExecutionManager(AppProcess):future.add_done_callback(_on_run_done)- def _cleanup_completed_runs(self, log=logger.info) -> list[str]:+ def _cleanup_completed_runs(self) -> list[str]:"""Remove completed futures from active_graph_runs and update metrics"""completed_runs = []for graph_exec_id, (future, _) in self.active_graph_runs.items():@@ -1271,40 +1317,90 @@ class ExecutionManager(AppProcess):completed_runs.append(graph_exec_id)for geid in completed_runs:- log(f"[{self.service_name}] ✅ Cleaned up completed run {geid}")+ logger.info(f"[{self.service_name}] ✅ Cleaned up completed run {geid}")self.active_graph_runs.pop(geid, None)- # Update metrics+ self._update_prompt_metrics()+ return completed_runs++ def _update_prompt_metrics(self):active_count = len(self.active_graph_runs)active_runs_gauge.set(active_count)- utilization_gauge.set(active_count / self.pool_size)-- return completed_runs+ if self._stop_consuming and self._stop_consuming.is_set():+ utilization_gauge.set(1.0)+ else:+ utilization_gauge.set(active_count / self.pool_size)def cleanup(self):- super().cleanup()- self._on_cleanup()-- def _on_cleanup(self, log=logger.info):+ """Override cleanup to implement graceful shutdown with active execution waiting."""prefix = f"[{self.service_name}][on_graph_executor_stop {os.getpid()}]"- log(f"{prefix} ⏳ Shutting down service loop...")- self.running = False+ logger.info(f"{prefix} 🧹 Starting graceful shutdown...")- log(f"{prefix} ⏳ Shutting down RabbitMQ channel...")- get_execution_queue().get_channel().stop_consuming()+ # Signal the consumer thread to stop (thread-safe)+ self.stop_consuming.set()+ logger.info(f"{prefix} ✅ Signaled execution message consumer to stop")- if hasattr(self, "executor"):- log(f"{prefix} ⏳ Shutting down GraphExec pool...")- self.executor.shutdown(cancel_futures=True, wait=False)+ # Wait for active executions to complete+ if self.active_graph_runs:+ logger.info(+ f"{prefix} ⏳ Waiting for {len(self.active_graph_runs)} active executions to complete..."+ )- log(f"{prefix} ⏳ Cleaning up active graph runs...")- self._cleanup_completed_runs(log=log)+ max_wait = GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS+ wait_interval = 5+ waited = 0- log(f"{prefix} ⏳ Disconnecting Redis...")- redis.disconnect()+ while waited < max_wait:+ self._cleanup_completed_runs()+ if not self.active_graph_runs:+ logger.info(f"{prefix} ✅ All active executions completed")+ break+ else:+ ids = [k.split("-")[0] for k in self.active_graph_runs.keys()]+ logger.info(+ f"{prefix} ⏳ Still waiting for {len(self.active_graph_runs)} executions: {ids}"+ )- log(f"{prefix} ✅ Finished GraphExec cleanup")- sys.exit(0)+ time.sleep(wait_interval)+ waited += wait_interval++ if self.active_graph_runs:+ logger.error(+ f"{prefix} ⚠️ {len(self.active_graph_runs)} executions still running after {max_wait}s"+ )+ else:+ logger.info(f"{prefix} ✅ All executions completed gracefully")++ # NOW shutdown executor pool after all executions and cleanup are complete+ if self._executor:+ logger.info(f"{prefix} ⏳ Shutting down GraphExec pool...")+ try:+ # All active executions are done, safe to shutdown workers+ self._executor.shutdown(cancel_futures=True, wait=False)+ logger.info(f"{prefix} ✅ Executor shutdown completed")+ except Exception as e:+ logger.error(f"{prefix} ⚠️ Error during executor shutdown: {e}")++ # Clean up RabbitMQ connections+ if self._cancel_client:+ logger.info(f"{prefix} ⏳ Disconnecting cancel RabbitMQ client...")+ try:+ self._cancel_client.disconnect()+ logger.info(f"{prefix} ✅ Cancel RabbitMQ client disconnected")+ except Exception as e:+ logger.error(+ f"{prefix} ⚠️ Error disconnecting cancel RabbitMQ client: {e}"+ )++ if self._run_client:+ logger.info(f"{prefix} ⏳ Disconnecting run RabbitMQ client...")+ try:+ self._run_client.disconnect()+ logger.info(f"{prefix} ✅ Run RabbitMQ client disconnected")+ except Exception as e:+ logger.error(f"{prefix} ⚠️ Error disconnecting run RabbitMQ client: {e}")++ logger.info(f"{prefix} ✅ Finished GraphExec cleanup")# ------- UTILITIES ------- #@@ -1429,11 +1525,3 @@ def increment_execution_count(user_id: str) -> int:if counter == 1:r.expire(k, settings.config.execution_counter_expiration_time)return counter---def llprint(message: str):- """- Low-level print/log helper function for use in signal handlers.- Regular log/print statements are not allowed in signal handlers.- """- os.write(sys.stdout.fileno(), (message + "\n").encode())diff --git a/autogpt_platform/backend/backend/executor/scheduler.py b/autogpt_platform/backend/backend/executor/scheduler.pyindex 2e5ccd00f4..2d6a85a52a 100644--- a/autogpt_platform/backend/backend/executor/scheduler.py+++ b/autogpt_platform/backend/backend/executor/scheduler.py@@ -259,8 +259,8 @@ class Scheduler(AppService):def cleanup(self):super().cleanup()- logger.info("⏳ Shutting down scheduler...")if self.scheduler:+ logger.info("⏳ Shutting down scheduler...")self.scheduler.shutdown(wait=False)@exposediff --git a/autogpt_platform/backend/backend/executor/utils.py b/autogpt_platform/backend/backend/executor/utils.pyindex 82d4191f72..65994cff0d 100644--- a/autogpt_platform/backend/backend/executor/utils.py+++ b/autogpt_platform/backend/backend/executor/utils.py@@ -655,6 +655,11 @@ GRAPH_EXECUTION_CANCEL_EXCHANGE = Exchange()GRAPH_EXECUTION_CANCEL_QUEUE_NAME = "graph_execution_cancel_queue"+# Graceful shutdown timeout constants+# Agent executions can run for up to 1 day, so we need a graceful shutdown period+# that allows long-running executions to complete naturally+GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS = 24 * 60 * 60 # 1 day to complete active executions+def create_execution_queue_config() -> RabbitMQConfig:"""@@ -675,7 +680,8 @@ def create_execution_queue_config() -> RabbitMQConfig:# Solution: Disable consumer timeout entirely - let graphs run indefinitely# Safety: Heartbeat mechanism now handles dead consumer detection instead# Use case: Graph executions that take hours to complete (AI model training, etc.)- "x-consumer-timeout": (7 * 24 * 60 * 60 * 1000), # 7 days in milliseconds+ "x-consumer-timeout": GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS+ * 1000,},)cancel_queue = Queue(@@ -817,16 +823,17 @@ async def add_graph_execution(graph_inputs=inputs or {},nodes_input_masks=nodes_input_masks,)-- graph_exec = await edb.create_graph_execution(- user_id=user_id,- graph_id=graph_id,- graph_version=graph.version,- starting_nodes_input=starting_nodes_input,- preset_id=preset_id,- )+ graph_exec = Nonetry:+ graph_exec = await edb.create_graph_execution(+ user_id=user_id,+ graph_id=graph_id,+ graph_version=graph.version,+ starting_nodes_input=starting_nodes_input,+ preset_id=preset_id,+ )+queue = await get_async_execution_queue()graph_exec_entry = graph_exec.to_graph_execution_entry()if nodes_input_masks:@@ -842,7 +849,14 @@ async def add_graph_execution(return graph_execexcept BaseException as e:- logger.error(f"Unable to publish graph #{graph_id} exec #{graph_exec.id}: {e}")+ err = str(e) or type(e).__name__+ if not graph_exec:+ logger.error(f"Graph execution #{graph_id} failed: {err}")+ raise++ logger.error(+ f"Unable to publish graph #{graph_id} exec #{graph_exec.id}: {err}"+ )await edb.update_node_execution_status_batch([node_exec.node_exec_id for node_exec in graph_exec.node_executions],ExecutionStatus.FAILED,@@ -850,7 +864,7 @@ async def add_graph_execution(await edb.update_graph_execution_stats(graph_exec_id=graph_exec.id,status=ExecutionStatus.FAILED,- stats=GraphExecutionStats(error=str(e)),+ stats=GraphExecutionStats(error=err),)raisediff --git a/autogpt_platform/backend/backend/util/decorator_test.py b/autogpt_platform/backend/backend/util/decorator_test.pyindex cdc1cc5b75..07598c59bc 100644--- a/autogpt_platform/backend/backend/util/decorator_test.py+++ b/autogpt_platform/backend/backend/util/decorator_test.py@@ -3,6 +3,7 @@ import timeimport pytestfrom backend.util.decorator import async_error_logged, error_logged, time_measured+from backend.util.retry import continuous_retry@time_measured@@ -72,3 +73,26 @@ def test_async_error_decorator_swallow_false():with pytest.raises(ValueError, match="This async error should NOT be swallowed"):asyncio.run(run_test())+++def test_continuous_retry_basic():+ """Test that continuous_retry decorator retries on exception."""++ class MockManager:+ def __init__(self):+ self.call_count = 0++ @continuous_retry(retry_delay=0.01)+ def failing_method(self):+ self.call_count += 1+ if self.call_count <= 2:+ # Fail on first two calls+ raise RuntimeError("Simulated failure")+ return "success"++ mock_manager = MockManager()++ # Should retry and eventually succeed+ result = mock_manager.failing_method()+ assert result == "success"+ assert mock_manager.call_count == 3 # Failed twice, succeeded on thirddiff --git a/autogpt_platform/backend/backend/util/process.py b/autogpt_platform/backend/backend/util/process.pyindex 3f61e999f5..1460ee9f99 100644--- a/autogpt_platform/backend/backend/util/process.py+++ b/autogpt_platform/backend/backend/util/process.py@@ -73,19 +73,30 @@ class AppProcess(ABC):set_service_name(self.service_name)logger.info(f"[{self.service_name}] Starting...")self.run()- except (KeyboardInterrupt, SystemExit) as e:- logger.warning(f"[{self.service_name}] Terminated: {e}; quitting...")+ except BaseException as e:+ logger.warning(+ f"[{self.service_name}] Termination request: {type(e).__name__}; executing cleanup."+ )finally:- if not self.cleaned_up:- self.cleanup()- self.cleaned_up = True+ self.cleanup()logger.info(f"[{self.service_name}] Terminated.")+ @staticmethod+ def llprint(message: str):+ """+ Low-level print/log helper function for use in signal handlers.+ Regular log/print statements are not allowed in signal handlers.+ """+ os.write(sys.stdout.fileno(), (message + "\n").encode())+def _self_terminate(self, signum: int, frame):if not self.cleaned_up:- self.cleanup()self.cleaned_up = True- sys.exit(0)+ sys.exit(0)+ else:+ self.llprint(+ f"[{self.service_name}] Received exit signal {signum}, but cleanup is already underway."+ )# Methods that are executed OUTSIDE the process #