AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend/executor): deliver a graph-run cancel to every pod, not one of them (#14685)
Reinier van der Leer062687127ff9
diff --git a/autogpt_platform/backend/backend/data/diagnostics.py b/autogpt_platform/backend/backend/data/diagnostics.pyindex 70bd967c59..4ac59df319 100644--- a/autogpt_platform/backend/backend/data/diagnostics.py+++ b/autogpt_platform/backend/backend/data/diagnostics.py@@ -9,6 +9,7 @@ from datetime import datetime, timedelta, timezonefrom typing import List, Optionalfrom croniter import croniter+from pika.exceptions import ChannelClosedByBrokerfrom prisma.enums import AgentExecutionStatusfrom prisma.models import AgentGraph, AgentGraphExecution, LibraryAgent, Userfrom pydantic import BaseModel@@ -18,8 +19,8 @@ from backend.data.execution import get_graph_executions, get_graph_executions_cofrom backend.data.rabbitmq import SyncRabbitMQfrom backend.executor.utils import (GRAPH_EXECUTION_CANCEL_EXCHANGE,- GRAPH_EXECUTION_CANCEL_QUEUE_NAME,GRAPH_EXECUTION_QUEUE_NAME,+ LEGACY_GRAPH_EXECUTION_CANCEL_QUEUE_NAME,CancelExecutionEvent,create_execution_queue_config,)@@ -583,10 +584,16 @@ def get_rabbitmq_queue_depth() -> int:def get_rabbitmq_cancel_queue_depth() -> int:"""- Get the number of messages in the RabbitMQ cancel queue.+ Get the number of messages left on the retired fleet-wide cancel queue.++ Cancels now fan out to a per-pod queue each ExecutionManager declares for+ itself, and those always have a consumer attached, so the only backlog this+ can still show is the retired queue draining. An environment created after+ the split never had that queue: absent means nothing is stuck, not an error.Returns:- Number of messages in cancel queue, or -1 if error+ Number of messages left on the retired queue, 0 if it is gone, or -1 on+ an unexpected failure."""try:# Create a temporary connection to query the queue@@ -597,9 +604,14 @@ def get_rabbitmq_cancel_queue_depth() -> int:try:# Use passive queue_declare to get queue info without modifying itif rabbitmq._channel:- method_frame = rabbitmq._channel.queue_declare(- queue=GRAPH_EXECUTION_CANCEL_QUEUE_NAME, passive=True- )+ try:+ method_frame = rabbitmq._channel.queue_declare(+ queue=LEGACY_GRAPH_EXECUTION_CANCEL_QUEUE_NAME, passive=True+ )+ except ChannelClosedByBroker as e:+ if e.reply_code != 404:+ raise+ return 0else:raise RuntimeError("RabbitMQ channel not initialized")diff --git a/autogpt_platform/backend/backend/data/diagnostics_test.py b/autogpt_platform/backend/backend/data/diagnostics_test.pyindex fc52070411..2ee8b622c6 100644--- a/autogpt_platform/backend/backend/data/diagnostics_test.py+++ b/autogpt_platform/backend/backend/data/diagnostics_test.py@@ -4,6 +4,7 @@ from datetime import datetime, timedelta, timezonefrom unittest.mock import AsyncMock, MagicMock, patchimport pytest+from pika.exceptions import ChannelClosedByBrokerfrom backend.data.diagnostics import (_calculate_total_runs,@@ -267,6 +268,29 @@ def test_rabbitmq_cancel_queue_depth_success():assert result == 5+def test_rabbitmq_cancel_queue_depth_is_zero_when_the_retired_queue_is_gone():+ """An environment created after the per-pod split never declared the+ fleet-wide queue; absent means nothing is stuck, not an error."""+ mock_channel = MagicMock()+ mock_channel.queue_declare.side_effect = ChannelClosedByBroker(404, "NOT_FOUND")+ mock_rabbitmq = MagicMock()+ mock_rabbitmq._channel = mock_channel++ with (+ patch(+ "backend.data.diagnostics.create_execution_queue_config",+ return_value=MagicMock(),+ ),+ patch(+ "backend.data.diagnostics.SyncRabbitMQ",+ return_value=mock_rabbitmq,+ ),+ ):+ result = get_rabbitmq_cancel_queue_depth()++ assert result == 0++def test_rabbitmq_cancel_queue_depth_error():"""Test RabbitMQ cancel queue depth returns -1 on error."""mock_rabbitmq = MagicMock()diff --git a/autogpt_platform/backend/backend/data/rabbitmq.py b/autogpt_platform/backend/backend/data/rabbitmq.pyindex 524e21748a..e7f2de242d 100644--- a/autogpt_platform/backend/backend/data/rabbitmq.py+++ b/autogpt_platform/backend/backend/data/rabbitmq.py@@ -3,6 +3,7 @@ import loggingfrom abc import ABC, abstractmethodfrom enum import Enumfrom typing import Awaitable, Optional+from uuid import uuid4import aio_pikaimport pika@@ -75,6 +76,83 @@ class RabbitMQConfig(BaseModel):queues: list[Queue]+def declare_broadcast_queue(+ channel: "pika.adapters.blocking_connection.BlockingChannel",+ exchange: Exchange,+ instance_id: str,+) -> str:+ """Give this process its own queue on a fanout exchange, and return its name.++ A fanout reaches every process only when every process owns a queue:+ consumers sharing one queue are round-robined, so a broadcast lands on one+ arbitrary instance. Exclusive and auto-delete, so the queue dies with the+ connection that declared it — which is why callers re-declare on reconnect.+ """+ queue_name = f"{exchange.name}.instance.{instance_id}.{uuid4().hex[:8]}"+ channel.queue_declare(+ queue=queue_name, durable=False, exclusive=True, auto_delete=True+ )+ channel.queue_bind(queue=queue_name, exchange=exchange.name, routing_key="")+ return queue_name+++# Only waits for the last old-image instance to drain, which is a rollout-scale+# event; costs one passive declare per instance per interval.+SHARED_QUEUE_REAP_INTERVAL_SECONDS = 5 * 60+++def start_shared_queue_reaper(+ channel: "pika.adapters.blocking_connection.BlockingChannel", queue_name: str+) -> None:+ """Delete a retired fleet-wide queue once no instance is draining it.++ Runs now and every few minutes after, on the consumer's own connection:+ a pika ``BlockingConnection`` is not thread-safe, and ``call_later`` fires+ from inside ``start_consuming``. An old-image instance that reconnects+ declares that queue again, so the pass repeats while this consumer lives.+ """+ reap_shared_queue(channel, queue_name)+ try:+ channel.connection.call_later(+ SHARED_QUEUE_REAP_INTERVAL_SECONDS,+ lambda: start_shared_queue_reaper(channel, queue_name),+ )+ except Exception as e:+ logger.debug(f"{queue_name} reaper not re-armed: {e}")+++def reap_shared_queue(+ channel: "pika.adapters.blocking_connection.BlockingChannel", queue_name: str+) -> bool:+ """Delete the shared queue if it exists and nothing consumes it.++ The consumer count is the rollout gate: while an old-image instance is+ still draining that queue, deleting it would take its broadcasts away.+ Runs on a scratch channel because a 404 from the passive declare closes+ the channel it arrives on, and the caller's is carrying the consumer.+ """+ try:+ scratch = channel.connection.channel()+ except Exception:+ logger.warning(f"Could not open a channel to reap {queue_name}")+ return False+ try:+ queue = scratch.queue_declare(queue=queue_name, passive=True).method+ if queue.consumer_count:+ return False+ # Check-then-act, because RabbitMQ refuses `if_unused` on a quorum queue:+ # an instance that re-consumes in the gap re-declares it on reconnect.+ scratch.queue_delete(queue=queue_name)+ logger.info(f"Deleted retired queue {queue_name}")+ return True+ except Exception as e:+ logger.debug(f"{queue_name} not reaped: {e}")+ return False+ finally:+ if scratch.is_open:+ scratch.close()++class RabbitMQBase(ABC):"""Base class for RabbitMQ connections with shared configuration"""diff --git a/autogpt_platform/backend/backend/data/rabbitmq_test.py b/autogpt_platform/backend/backend/data/rabbitmq_test.pyindex 9eefa7ddba..c0d82e1a14 100644--- a/autogpt_platform/backend/backend/data/rabbitmq_test.py+++ b/autogpt_platform/backend/backend/data/rabbitmq_test.py@@ -20,8 +20,11 @@ from backend.data.rabbitmq import (ExchangeType,Queue,RabbitMQConfig,+ declare_broadcast_queue,+ reap_shared_queue,)from backend.executor.utils import (+ GRAPH_EXECUTION_CANCEL_EXCHANGE,GRAPH_EXECUTION_EXCHANGE,GRAPH_EXECUTION_QUEUE_NAME,GRAPH_EXECUTION_ROUTING_KEY,@@ -60,12 +63,67 @@ def test_graph_execution_queue_is_quorum() -> None:assert run.exchange is GRAPH_EXECUTION_EXCHANGE-def test_graph_execution_cancel_queue_is_quorum() -> None:- """Cancel queue must also be quorum — losing cancellations on a node- flap is just as bad as losing runs."""+def test_graph_execution_config_declares_no_cancel_queue() -> None:+ """Cancels fan out to a per-pod exclusive queue each consumer declares+ itself; a queue here would be shared by the whole fleet again, and+ RabbitMQ would round-robin each cancel to one arbitrary pod."""cfg = create_execution_queue_config()- cancel = next(q for q in cfg.queues if q.name.endswith("cancel_queue_v2"))- assert cancel.arguments == {"x-queue-type": "quorum"}+ assert GRAPH_EXECUTION_CANCEL_EXCHANGE in cfg.exchanges+ assert [q.name for q in cfg.queues] == [GRAPH_EXECUTION_QUEUE_NAME]+++class TestDeclareBroadcastQueue:+ def test_queue_is_exclusive_auto_delete_and_bound_to_the_exchange(self) -> None:+ channel = MagicMock()+ name = declare_broadcast_queue(+ channel, GRAPH_EXECUTION_CANCEL_EXCHANGE, "executor-1"+ )+ assert name.startswith(f"{GRAPH_EXECUTION_CANCEL_EXCHANGE.name}.instance.")+ channel.queue_declare.assert_called_once_with(+ queue=name, durable=False, exclusive=True, auto_delete=True+ )+ channel.queue_bind.assert_called_once_with(+ queue=name, exchange=GRAPH_EXECUTION_CANCEL_EXCHANGE.name, routing_key=""+ )++ def test_each_instance_gets_its_own_queue(self) -> None:+ names = {+ declare_broadcast_queue(+ MagicMock(), GRAPH_EXECUTION_CANCEL_EXCHANGE, instance_id+ )+ for instance_id in ("executor-1", "executor-2", "executor-1")+ }+ assert len(names) == 3+++class TestReapSharedQueue:+ def test_leaves_a_queue_that_an_old_instance_still_drains(self) -> None:+ """Deleting it then would take that instance's broadcasts away."""+ channel = MagicMock()+ scratch = channel.connection.channel.return_value+ scratch.queue_declare.return_value.method.consumer_count = 1++ assert reap_shared_queue(channel, "old_queue") is False+ scratch.queue_delete.assert_not_called()+ scratch.close.assert_called_once()++ def test_deletes_a_queue_nothing_consumes(self) -> None:+ channel = MagicMock()+ scratch = channel.connection.channel.return_value+ scratch.queue_declare.return_value.method.consumer_count = 0++ assert reap_shared_queue(channel, "old_queue") is True+ scratch.queue_declare.assert_called_once_with(queue="old_queue", passive=True)+ scratch.queue_delete.assert_called_once_with(queue="old_queue")+ scratch.close.assert_called_once()++ def test_a_missing_queue_is_not_an_error(self) -> None:+ channel = MagicMock()+ scratch = channel.connection.channel.return_value+ scratch.queue_declare.side_effect = RuntimeError("NOT_FOUND")++ assert reap_shared_queue(channel, "old_queue") is False+ scratch.queue_delete.assert_not_called()def test_copilot_execution_queue_is_quorum_with_consumer_timeout() -> None:diff --git a/autogpt_platform/backend/backend/executor/cancel_fanout_e2e_test.py b/autogpt_platform/backend/backend/executor/cancel_fanout_e2e_test.pynew file mode 100644index 0000000000..cbe6336ad5--- /dev/null+++ b/autogpt_platform/backend/backend/executor/cancel_fanout_e2e_test.py@@ -0,0 +1,172 @@+"""A graph-run cancel must reach every ExecutionManager pod, on a live broker.++Skips when no RabbitMQ is reachable; CI always runs one, on a per-shard vhost.+"""++from __future__ import annotations++import asyncio+import socket+from contextlib import suppress+from uuid import uuid4++import pytest++from backend.data.rabbitmq import (+ SyncRabbitMQ,+ declare_broadcast_queue,+ reap_shared_queue,+)+from backend.executor.utils import (+ GRAPH_EXECUTION_CANCEL_EXCHANGE,+ CancelExecutionEvent,+ create_execution_queue_config,+)+from backend.util.settings import Settings+++def _has_live_rabbit() -> bool:+ s = Settings()+ try:+ with socket.create_connection(+ (s.config.rabbitmq_host, s.config.rabbitmq_port), timeout=1.0+ ):+ return True+ except Exception: # noqa: BLE001 - any connect failure → skip+ return False+++rabbit_only = pytest.mark.skipif(+ not _has_live_rabbit(), reason="no RabbitMQ reachable; skip live-broker test"+)+++@rabbit_only+async def test_a_cancel_reaches_every_pod_not_just_one() -> None:+ """Two pods, one cancel: both must see it.++ Each pod gets its own connection, as two ExecutionManager pods have.+ Sharing one queue instead makes the broker round-robin the cancel to+ exactly one of them, so the pod running the graph never stops.+ """+ graph_exec_id = str(uuid4())+ pods = [SyncRabbitMQ(create_execution_queue_config()) for _ in range(2)]+ queues: list[str] = []+ seen = [False, False]+ try:+ for pod in pods:+ pod.connect()+ queues.append(+ declare_broadcast_queue(+ pod.get_channel(),+ GRAPH_EXECUTION_CANCEL_EXCHANGE,+ f"pod-{uuid4().hex[:6]}",+ )+ )++ publisher = pods[0]+ publisher.publish_message(+ routing_key="",+ message=CancelExecutionEvent(graph_exec_id=graph_exec_id).model_dump_json(),+ exchange=GRAPH_EXECUTION_CANCEL_EXCHANGE,+ )++ for _ in range(50):+ seen = [+ got or _saw_cancel(pod, queue, graph_exec_id)+ for got, pod, queue in zip(seen, pods, queues)+ ]+ if all(seen):+ break+ await asyncio.sleep(0.1)+ finally:+ for pod in pods:+ pod.disconnect()++ assert seen == [True, True], (+ f"{seen.count(True)} of {len(pods)} pods received the cancel; "+ "a fanout bound to one shared queue delivers to exactly one consumer"+ )+++@rabbit_only+async def test_the_retired_queue_is_reaped_only_once_nothing_drains_it() -> None:+ """The retired fleet-wide queue goes away on its own, with no operator step.++ Deleting it while an old-image pod is still draining it would take that+ pod's cancels away, so the consumer count is the gate.+ """+ legacy = f"graph_execution_cancel_queue_v2_test_{uuid4().hex[:8]}"+ new_pod = SyncRabbitMQ(create_execution_queue_config())+ old_pod = SyncRabbitMQ(create_execution_queue_config())+ try:+ new_pod.connect()+ old_pod.connect()+ # bound before the reap: this exchange is auto-delete, so the last+ # binding leaving would drop the exchange itself+ declare_broadcast_queue(+ new_pod.get_channel(), GRAPH_EXECUTION_CANCEL_EXCHANGE, "new-pod"+ )++ old_channel = old_pod.get_channel()+ old_channel.queue_declare(+ queue=legacy, durable=True, arguments={"x-queue-type": "quorum"}+ )+ # bound the way declare_infrastructure binds it: `routing_key or name`+ old_channel.queue_bind(+ queue=legacy,+ exchange=GRAPH_EXECUTION_CANCEL_EXCHANGE.name,+ routing_key=legacy,+ )+ old_channel.basic_consume(+ queue=legacy, on_message_callback=lambda *_: None, auto_ack=True+ )+ old_channel.connection.process_data_events(time_limit=1)++ assert reap_shared_queue(new_pod.get_channel(), legacy) is False+ assert _queue_exists(new_pod, legacy), "reaped a queue an old pod still drains"++ old_pod.disconnect() # the last old-image pod finishes draining and goes++ # the broker drops the consumer count asynchronously, and this reads it+ # over a different connection, so poll rather than assert on one pass+ reaped = False+ for _ in range(50):+ reaped = reap_shared_queue(new_pod.get_channel(), legacy)+ if reaped:+ break+ await asyncio.sleep(0.1)++ assert reaped, "the retired queue outlived its last consumer"+ assert not _queue_exists(new_pod, legacy)+ finally:+ if new_pod.is_ready: # a failed run must not leave it bound to the fanout+ with suppress(Exception):+ new_pod.get_channel().queue_delete(queue=legacy)+ for pod in (old_pod, new_pod):+ pod.disconnect()+++def _queue_exists(pod: SyncRabbitMQ, queue_name: str) -> bool:+ """Ask the broker, on a scratch channel: a 404 closes the channel it hits."""+ scratch = pod.get_channel().connection.channel()+ try:+ scratch.queue_declare(queue=queue_name, passive=True)+ return True+ except Exception: # noqa: BLE001 - 404 is the answer we are after+ return False+ finally:+ if scratch.is_open:+ scratch.close()+++def _saw_cancel(pod: SyncRabbitMQ, queue_name: str, graph_exec_id: str) -> bool:+ """Drain this pod's queue, reporting whether our cancel was among it."""+ channel = pod.get_channel()+ found = False+ while True:+ method, _properties, body = channel.basic_get(queue=queue_name, auto_ack=True)+ if method is None:+ return found+ event = CancelExecutionEvent.model_validate_json(body.decode())+ found = found or event.graph_exec_id == graph_exec_iddiff --git a/autogpt_platform/backend/backend/executor/manager.py b/autogpt_platform/backend/backend/executor/manager.pyindex 571a7735b6..00a3b26453 100644--- a/autogpt_platform/backend/backend/executor/manager.py+++ b/autogpt_platform/backend/backend/executor/manager.py@@ -41,7 +41,11 @@ from backend.data.model import (NodeExecutionStats,OAuth2Credentials,)-from backend.data.rabbitmq import SyncRabbitMQ+from backend.data.rabbitmq import (+ SyncRabbitMQ,+ declare_broadcast_queue,+ start_shared_queue_reaper,+)from backend.data.redis_helpers import incr_with_ttl_syncfrom backend.executor.cost_tracking import (drain_pending_cost_logs,@@ -96,10 +100,11 @@ from .cluster_lock import ClusterLockfrom .simulator import get_dry_run_credentials, prepare_dry_run, simulate_blockfrom .utils import (GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS,- GRAPH_EXECUTION_CANCEL_QUEUE_NAME,+ GRAPH_EXECUTION_CANCEL_EXCHANGE,GRAPH_EXECUTION_EXCHANGE,GRAPH_EXECUTION_QUEUE_NAME,GRAPH_EXECUTION_ROUTING_KEY,+ LEGACY_GRAPH_EXECUTION_CANCEL_QUEUE_NAME,CancelExecutionEvent,ExecutionOutputEntry,LogMetadata,@@ -1656,12 +1661,25 @@ class ExecutionManager(AppProcess):self.cancel_client.disconnect()self.cancel_client.connect()cancel_channel = self.cancel_client.get_channel()+ # Declared here rather than once at startup: an exclusive queue dies+ # with the connection that made it, and this method is the reconnect.+ # It is also declared before the reaper runs, because this exchange is+ # auto-delete and losing its last binding would drop the exchange.+ cancel_queue_name = declare_broadcast_queue(+ cancel_channel, GRAPH_EXECUTION_CANCEL_EXCHANGE, self.executor_id+ )+ start_shared_queue_reaper(+ cancel_channel, LEGACY_GRAPH_EXECUTION_CANCEL_QUEUE_NAME+ )cancel_channel.basic_consume(- queue=GRAPH_EXECUTION_CANCEL_QUEUE_NAME,+ queue=cancel_queue_name,on_message_callback=self._handle_cancel_message,auto_ack=True,)- logger.info(f"[{self.service_name}] ⏳ Starting cancel message consumer...")+ logger.info(+ f"[{self.service_name}] ⏳ Starting cancel message consumer "+ f"on {cancel_queue_name}..."+ )cancel_channel.start_consuming()if not self.stop_consuming.is_set() or self.active_graph_runs:raise RuntimeError(diff --git a/autogpt_platform/backend/backend/executor/manager_cancel_consumer_test.py b/autogpt_platform/backend/backend/executor/manager_cancel_consumer_test.pynew file mode 100644index 0000000000..8b29d5bf49--- /dev/null+++ b/autogpt_platform/backend/backend/executor/manager_cancel_consumer_test.py@@ -0,0 +1,26 @@+"""The cancel consumer must consume the per-pod queue it just declared."""++from unittest.mock import MagicMock++from backend.executor.manager import ExecutionManager+from backend.executor.utils import GRAPH_EXECUTION_CANCEL_EXCHANGE+++def test_cancel_consumer_consumes_the_queue_it_just_declared() -> None:+ """Pins the consumer to the per-pod queue the declare helper returned.++ Re-pinning ``basic_consume`` to a fixed name restores the original defect —+ one shared queue, so the broker hands each cancel to one arbitrary pod —+ and no other test in the suite would notice.+ """+ manager = ExecutionManager()+ channel = MagicMock()+ manager._cancel_client = MagicMock(is_ready=True)+ manager._cancel_client.get_channel.return_value = channel+ channel.start_consuming.side_effect = lambda: manager.stop_consuming.set()++ ExecutionManager._consume_execution_cancel.__wrapped__(manager)++ declared = channel.queue_declare.call_args.kwargs["queue"]+ assert declared.startswith(f"{GRAPH_EXECUTION_CANCEL_EXCHANGE.name}.instance.")+ assert channel.basic_consume.call_args.kwargs["queue"] == declareddiff --git a/autogpt_platform/backend/backend/executor/utils.py b/autogpt_platform/backend/backend/executor/utils.pyindex 173d62adb4..fa4367ab0f 100644--- a/autogpt_platform/backend/backend/executor/utils.py+++ b/autogpt_platform/backend/backend/executor/utils.py@@ -991,7 +991,11 @@ GRAPH_EXECUTION_CANCEL_EXCHANGE = Exchange(durable=True,auto_delete=True,)-GRAPH_EXECUTION_CANCEL_QUEUE_NAME = "graph_execution_cancel_queue_v2"+# Pre-2026-09 topology: one durable queue bound to the fanout, consumed by every+# ExecutionManager pod, so RabbitMQ round-robined each cancel to a single+# arbitrary pod. Old-image pods keep draining it through a rollout; each new pod+# deletes it once none is left, so no operator step is needed on any install.+LEGACY_GRAPH_EXECUTION_CANCEL_QUEUE_NAME = "graph_execution_cancel_queue_v2"# Graceful shutdown timeout constants# Agent executions can run for up to 1 day, so we need a graceful shutdown period@@ -1024,18 +1028,14 @@ def create_execution_queue_config() -> RabbitMQConfig:"x-consumer-timeout": GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS * 1000,},)- cancel_queue = Queue(- name=GRAPH_EXECUTION_CANCEL_QUEUE_NAME,- exchange=GRAPH_EXECUTION_CANCEL_EXCHANGE,- routing_key="", # not used for FANOUT- durable=True,- auto_delete=False,- arguments={"x-queue-type": "quorum"},- )+ # No cancel queue here: it is per-pod, exclusive to the consuming connection,+ # and declared by the consumer itself through ``declare_broadcast_queue``.+ # A queue in this config is declared by every holder, publishers included,+ # which is how one queue came to serve the whole fleet.return RabbitMQConfig(vhost=Config().rabbitmq_vhost,exchanges=[GRAPH_EXECUTION_EXCHANGE, GRAPH_EXECUTION_CANCEL_EXCHANGE],- queues=[run_queue, cancel_queue],+ queues=[run_queue],)