AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend/executor): Make executor continuously running and retrying message consumption (#9999)
Zamil Majdy793d056d81ca
diff --git a/autogpt_platform/backend/backend/executor/manager.py b/autogpt_platform/backend/backend/executor/manager.pyindex 4e47770f78..48d14edd5a 100644--- a/autogpt_platform/backend/backend/executor/manager.py+++ b/autogpt_platform/backend/backend/executor/manager.py@@ -67,7 +67,7 @@ from backend.util.decorator import error_logged, time_measuredfrom backend.util.file import clean_exec_filesfrom backend.util.logging import TruncatedLogger, configure_loggingfrom backend.util.process import AppProcess, set_service_name-from backend.util.retry import func_retry+from backend.util.retry import continuous_retry, func_retryfrom backend.util.service import get_service_clientfrom backend.util.settings import Settings@@ -938,8 +938,6 @@ class ExecutionManager(AppProcess):self.pool_size = settings.config.num_graph_workersself.running = Trueself.active_graph_runs: dict[str, tuple[Future, threading.Event]] = {}- signal.signal(signal.SIGTERM, lambda sig, frame: self._on_sigterm())- signal.signal(signal.SIGINT, lambda sig, frame: self._on_sigterm())def run(self):pool_size_gauge.set(self.pool_size)@@ -965,22 +963,29 @@ class ExecutionManager(AppProcess):logger.info(f"[{self.service_name}] ⏳ Connecting to Redis...")redis.connect()+ threading.Thread(+ target=lambda: self._consume_execution_cancel(),+ daemon=True,+ ).start()++ self._consume_execution_run()++ @continuous_retry()+ def _consume_execution_cancel(self):cancel_client = SyncRabbitMQ(create_execution_queue_config())cancel_client.connect()cancel_channel = cancel_client.get_channel()logger.info(f"[{self.service_name}] ⏳ Starting cancel message consumer...")- threading.Thread(- target=lambda: (- cancel_channel.basic_consume(- queue=GRAPH_EXECUTION_CANCEL_QUEUE_NAME,- on_message_callback=self._handle_cancel_message,- auto_ack=True,- ),- cancel_channel.start_consuming(),- ),- daemon=True,- ).start()+ cancel_channel.basic_consume(+ queue=GRAPH_EXECUTION_CANCEL_QUEUE_NAME,+ on_message_callback=self._handle_cancel_message,+ auto_ack=True,+ )+ cancel_channel.start_consuming()+ raise RuntimeError(f"❌ cancel message consumer is stopped: {cancel_channel}")+ @continuous_retry()+ def _consume_execution_run(self):run_client = SyncRabbitMQ(create_execution_queue_config())run_client.connect()run_channel = run_client.get_channel()@@ -992,6 +997,7 @@ class ExecutionManager(AppProcess):)logger.info(f"[{self.service_name}] ⏳ Starting to consume run messages...")run_channel.start_consuming()+ raise RuntimeError(f"❌ run message consumer is stopped: {run_channel}")def _handle_cancel_message(self,@@ -1090,10 +1096,6 @@ class ExecutionManager(AppProcess):super().cleanup()self._on_cleanup()- def _on_sigterm(self):- llprint(f"[{self.service_name}] ⚠️ GraphExec SIGTERM received")- self._on_cleanup(log=llprint)-def _on_cleanup(self, log=logger.info):prefix = f"[{self.service_name}][on_graph_executor_stop {os.getpid()}]"log(f"{prefix} ⏳ Shutting down service loop...")@@ -1110,7 +1112,7 @@ class ExecutionManager(AppProcess):redis.disconnect()log(f"{prefix} ✅ Finished GraphExec cleanup")- exit(0)+ sys.exit(0)# ------- UTILITIES ------- #diff --git a/autogpt_platform/backend/backend/util/retry.py b/autogpt_platform/backend/backend/util/retry.pyindex 01fc69d7df..dc20056b98 100644--- a/autogpt_platform/backend/backend/util/retry.py+++ b/autogpt_platform/backend/backend/util/retry.py@@ -2,6 +2,7 @@ import asyncioimport loggingimport osimport threading+import timefrom functools import wrapsfrom uuid import uuid4@@ -80,3 +81,24 @@ func_retry = retry(stop=stop_after_attempt(5),wait=wait_exponential(multiplier=1, min=1, max=30),)+++def continuous_retry(*, retry_delay: float = 1.0):+ def decorator(func):+ @wraps(func)+ def wrapper(*args, **kwargs):+ while True:+ try:+ return func(*args, **kwargs)+ except Exception as exc:+ logger.exception(+ "%s failed with %s — retrying in %.2f s",+ func.__name__,+ exc,+ retry_delay,+ )+ time.sleep(retry_delay)++ return wrapper++ return decorator