AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend): Fix executor running RabbitMQ operations on closed/closing connection (#10578)
Zamil Majdy40601f1616f1
diff --git a/autogpt_platform/backend/backend/data/rabbitmq.py b/autogpt_platform/backend/backend/data/rabbitmq.pyindex e50b86b0d4..bdf2090083 100644--- a/autogpt_platform/backend/backend/data/rabbitmq.py+++ b/autogpt_platform/backend/backend/data/rabbitmq.py@@ -140,6 +140,7 @@ class SyncRabbitMQ(RabbitMQBase):socket_timeout=SOCKET_TIMEOUT,connection_attempts=CONNECTION_ATTEMPTS,retry_delay=RETRY_DELAY,+ heartbeat=300, # 5 minute timeout (heartbeats sent every 2.5 min))self._connection = pika.BlockingConnection(parameters)@@ -244,6 +245,7 @@ class AsyncRabbitMQ(RabbitMQBase):password=self.password,virtualhost=self.config.vhost.lstrip("/"),blocked_connection_timeout=BLOCKED_CONNECTION_TIMEOUT,+ heartbeat=300, # 5 minute timeout (heartbeats sent every 2.5 min))self._channel = await self._connection.channel()await self._channel.set_qos(prefetch_count=1)diff --git a/autogpt_platform/backend/backend/executor/manager.py b/autogpt_platform/backend/backend/executor/manager.pyindex cdb4d33a46..bb147e394d 100644--- a/autogpt_platform/backend/backend/executor/manager.py+++ b/autogpt_platform/backend/backend/executor/manager.py@@ -1257,7 +1257,7 @@ class ExecutionManager(AppProcess):def _handle_run_message(self,- channel: BlockingChannel,+ _channel: BlockingChannel,method: Basic.Deliver,_properties: BasicProperties,body: bytes,@@ -1267,6 +1267,9 @@ class ExecutionManager(AppProcess):@func_retrydef _ack_message(reject: bool = False):"""Acknowledge or reject the message based on execution status."""++ # Connection can be lost, so always get a fresh channel+ channel = self.run_client.get_channel()if reject:channel.connection.add_callback_threadsafe(lambda: channel.basic_nack(delivery_tag, requeue=True)@@ -1416,9 +1419,12 @@ class ExecutionManager(AppProcess):try:run_channel = self.run_client.get_channel()run_channel.connection.add_callback_threadsafe(- lambda: self.run_client.disconnect()+ lambda: run_channel.stop_consuming())self.run_thread.join()+ run_channel.connection.add_callback_threadsafe(+ lambda: self.run_client.disconnect()+ )logger.info(f"{prefix} ✅ Run client disconnected")except Exception as e:logger.error(f"{prefix} ⚠️ Error disconnecting run client: {type(e)} {e}")@@ -1427,9 +1433,12 @@ class ExecutionManager(AppProcess):try:cancel_channel = self.cancel_client.get_channel()cancel_channel.connection.add_callback_threadsafe(- lambda: self.cancel_client.disconnect()+ lambda: cancel_channel.stop_consuming())self.cancel_thread.join()+ cancel_channel.connection.add_callback_threadsafe(+ lambda: self.cancel_client.disconnect()+ )logger.info(f"{prefix} ✅ Cancel client disconnected")except Exception as e:logger.error(f"{prefix} ⚠️ Error disconnecting cancel client: {type(e)} {e}")