Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend/copilot): deliver a Stop to every executor pod, not one of them (#14684)

Reinier van der Leer445686df7388
diff --git a/autogpt_platform/backend/backend/copilot/executor/cancel_fanout_e2e_test.py b/autogpt_platform/backend/backend/copilot/executor/cancel_fanout_e2e_test.py
new file mode 100644
index 0000000000..fa48b23cc8
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/executor/cancel_fanout_e2e_test.py
@@ -0,0 +1,162 @@
+"""The cancel fanout must reach every executor pod, against a live broker.
+
+Skips when no RabbitMQ is reachable; CI always runs one, on a per-shard vhost.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import socket
+from contextlib import suppress
+from uuid import uuid4
+
+import pytest
+
+from backend.copilot.executor import utils
+from backend.copilot.executor.utils import (
+ COPILOT_CANCEL_EXCHANGE,
+ CancelCoPilotEvent,
+ create_copilot_queue_config,
+ declare_pod_cancel_queue,
+ enqueue_cancel_task,
+ reap_legacy_cancel_queue,
+)
+from backend.data.rabbitmq import SyncRabbitMQ
+from backend.util.settings import Settings
+
+
+def _has_live_rabbit() -> bool:
+ s = Settings()
+ try:
+ with socket.create_connection(
+ (s.config.rabbitmq_host, s.config.rabbitmq_port), timeout=1.0
+ ):
+ return True
+ except Exception: # noqa: BLE001 - any connect failure → skip
+ return False
+
+
+rabbit_only = pytest.mark.skipif(
+ not _has_live_rabbit(), reason="no RabbitMQ reachable; skip live-broker test"
+)
+
+
+@rabbit_only
+async def test_a_cancel_reaches_every_pod_not_just_one() -> None:
+ """Two pods, one cancel: both must see it.
+
+ Each pod gets its own connection, as two executor pods have. Sharing one
+ queue instead makes the broker round-robin the cancel to exactly one of
+ them, so the pod holding the session never stops.
+ """
+ session_id = str(uuid4())
+ pods = [SyncRabbitMQ(create_copilot_queue_config()) for _ in range(2)]
+ queues: list[str] = []
+ seen = [False, False]
+ try:
+ for pod in pods:
+ pod.connect()
+ queues.append(
+ declare_pod_cancel_queue(pod.get_channel(), f"pod-{uuid4().hex[:6]}")
+ )
+
+ await enqueue_cancel_task(session_id)
+
+ for _ in range(50):
+ seen = [
+ got or _saw_cancel(pod, queue, session_id)
+ for got, pod, queue in zip(seen, pods, queues)
+ ]
+ if all(seen):
+ break
+ await asyncio.sleep(0.1)
+ finally:
+ for pod in pods:
+ pod.disconnect()
+
+ assert seen == [True, True], (
+ f"{seen.count(True)} of {len(pods)} pods received the cancel; "
+ "a fanout bound to one shared queue delivers to exactly one consumer"
+ )
+
+
+@rabbit_only
+async def test_the_retired_queue_is_reaped_only_once_nothing_drains_it(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """The retired fleet-wide queue goes away on its own, with no operator step.
+
+ Deleting it while an old-image pod is still draining it would take that
+ pod's cancels away, so the consumer count is the gate. Runs against a
+ scratch name: the reaper deletes whatever ``LEGACY_...`` points at.
+ """
+ legacy = f"copilot_cancel_queue_v2_test_{uuid4().hex[:8]}"
+ monkeypatch.setattr(utils, "LEGACY_COPILOT_CANCEL_QUEUE_NAME", legacy)
+
+ new_pod = SyncRabbitMQ(create_copilot_queue_config())
+ old_pod = SyncRabbitMQ(create_copilot_queue_config())
+ try:
+ new_pod.connect()
+ old_pod.connect()
+ declare_pod_cancel_queue(new_pod.get_channel(), "new-pod")
+
+ old_channel = old_pod.get_channel()
+ old_channel.queue_declare(
+ queue=legacy, durable=True, arguments={"x-queue-type": "quorum"}
+ )
+ # bound the way declare_infrastructure binds it: `routing_key or name`
+ old_channel.queue_bind(
+ queue=legacy, exchange=COPILOT_CANCEL_EXCHANGE.name, routing_key=legacy
+ )
+ old_channel.basic_consume(
+ queue=legacy, on_message_callback=lambda *_: None, auto_ack=True
+ )
+ old_channel.connection.process_data_events(time_limit=1)
+
+ assert reap_legacy_cancel_queue(new_pod.get_channel()) is False
+ assert _queue_exists(new_pod, legacy), "reaped a queue an old pod still drains"
+
+ old_pod.disconnect() # the last old-image pod finishes draining and goes
+
+ # the broker drops the consumer count asynchronously, and this reads it
+ # over a different connection, so poll rather than assert on one pass
+ reaped = False
+ for _ in range(50):
+ reaped = reap_legacy_cancel_queue(new_pod.get_channel())
+ if reaped:
+ break
+ await asyncio.sleep(0.1)
+
+ assert reaped, "the retired queue outlived its last consumer"
+ assert not _queue_exists(new_pod, legacy)
+ finally:
+ if new_pod.is_ready: # a failed run must not leave it bound to the fanout
+ with suppress(Exception):
+ new_pod.get_channel().queue_delete(queue=legacy)
+ for pod in (old_pod, new_pod):
+ pod.disconnect()
+
+
+def _queue_exists(pod: SyncRabbitMQ, queue_name: str) -> bool:
+ """Ask the broker, on a scratch channel: a 404 closes the channel it hits."""
+ scratch = pod.get_channel().connection.channel()
+ try:
+ scratch.queue_declare(queue=queue_name, passive=True)
+ return True
+ except Exception: # noqa: BLE001 - 404 is the answer we are after
+ return False
+ finally:
+ if scratch.is_open:
+ scratch.close()
+
+
+def _saw_cancel(pod: SyncRabbitMQ, queue_name: str, session_id: str) -> bool:
+ """Drain this pod's queue, reporting whether our cancel was among it."""
+ channel = pod.get_channel()
+ found = False
+ while True:
+ method, _properties, body = channel.basic_get(queue=queue_name, auto_ack=True)
+ if method is None:
+ return found
+ event = CancelCoPilotEvent.model_validate_json(body.decode())
+ found = found or event.session_id == session_id
diff --git a/autogpt_platform/backend/backend/copilot/executor/manager.py b/autogpt_platform/backend/backend/copilot/executor/manager.py
index 3d9c325dbc..07c096be3d 100644
--- a/autogpt_platform/backend/backend/copilot/executor/manager.py
+++ b/autogpt_platform/backend/backend/copilot/executor/manager.py
@@ -32,13 +32,14 @@ from backend.util.settings import Settings
from .processor import execute_copilot_turn, init_worker
from .utils import (
- COPILOT_CANCEL_QUEUE_NAME,
COPILOT_EXECUTION_QUEUE_NAME,
GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS,
CancelCoPilotEvent,
CoPilotExecutionEntry,
create_copilot_queue_config,
+ declare_pod_cancel_queue,
get_session_lock_key,
+ start_legacy_cancel_queue_reaper,
)
logger = TruncatedLogger(logging.getLogger(__name__), prefix="[CoPilotExecutor]")
@@ -291,12 +292,16 @@ class CoPilotExecutor(AppProcess):
return
cancel_channel = self.cancel_client.get_channel()
+ # Declared here rather than once at startup: an exclusive queue dies
+ # with the connection that made it, and this method is the reconnect.
+ cancel_queue_name = declare_pod_cancel_queue(cancel_channel, self.executor_id)
cancel_channel.basic_consume(
- queue=COPILOT_CANCEL_QUEUE_NAME,
+ queue=cancel_queue_name,
on_message_callback=self._handle_cancel_message,
auto_ack=True,
)
- logger.info("Starting to consume cancel messages...")
+ logger.info(f"Starting to consume cancel messages on {cancel_queue_name}...")
+ start_legacy_cancel_queue_reaper(cancel_channel)
cancel_channel.start_consuming()
if not self.stop_consuming.is_set() or self.active_tasks:
raise RuntimeError("Cancel message consumer stopped unexpectedly")
diff --git a/autogpt_platform/backend/backend/copilot/executor/manager_test.py b/autogpt_platform/backend/backend/copilot/executor/manager_test.py
index d61a788b47..515a0aa684 100644
--- a/autogpt_platform/backend/backend/copilot/executor/manager_test.py
+++ b/autogpt_platform/backend/backend/copilot/executor/manager_test.py
@@ -1,4 +1,4 @@
-"""Tests for the engine-switch continuation dispatch in the copilot manager.
+"""Tests for the copilot manager: the engine-switch dispatch and the cancel consumer.
The dispatch is the handoff between a finished baseline turn and the
server-initiated SDK continuation turn — the highest-risk link in the
@@ -16,6 +16,7 @@ from unittest.mock import AsyncMock, MagicMock, call, patch
import pytest
from backend.copilot.engine_switch import CONTINUATION_MESSAGE, SwitchRequest
+from backend.copilot.executor.utils import COPILOT_CANCEL_QUEUE_PREFIX
from .manager import (
_SWITCH_DISPATCH_ATTEMPTS,
@@ -258,3 +259,23 @@ def test_cleanup_closes_codex_pool_between_consumers_and_workers():
executor.cleanup()
assert lifecycle == ["consumer_stop", "codex_pool_close", "worker_cleanup"]
+
+
+def test_cancel_consumer_consumes_the_queue_it_just_declared():
+ """Pins the consumer to the per-pod queue the declare helper returned.
+
+ Re-pinning ``basic_consume`` to a fixed name restores the original defect —
+ one shared queue, so the broker hands each cancel to one arbitrary pod —
+ and no other test in the suite would notice.
+ """
+ executor = CoPilotExecutor()
+ channel = MagicMock()
+ executor._cancel_client = MagicMock(is_ready=True)
+ executor._cancel_client.get_channel.return_value = channel
+ channel.start_consuming.side_effect = lambda: executor.stop_consuming.set()
+
+ CoPilotExecutor._consume_cancel.__wrapped__(executor)
+
+ declared = channel.queue_declare.call_args.kwargs["queue"]
+ assert declared.startswith(COPILOT_CANCEL_QUEUE_PREFIX)
+ assert channel.basic_consume.call_args.kwargs["queue"] == declared
diff --git a/autogpt_platform/backend/backend/copilot/executor/utils.py b/autogpt_platform/backend/backend/copilot/executor/utils.py
index 739a79e27c..2ecf1a737c 100644
--- a/autogpt_platform/backend/backend/copilot/executor/utils.py
+++ b/autogpt_platform/backend/backend/copilot/executor/utils.py
@@ -1,12 +1,14 @@
-"""RabbitMQ queue configuration for CoPilot executor.
+"""RabbitMQ topology for the CoPilot executor.
-Defines two exchanges and queues following the graph executor pattern:
-- 'copilot_execution' (DIRECT) for chat generation tasks
-- 'copilot_cancel' (FANOUT) for cancellation requests
+- 'copilot_execution' (DIRECT) for chat generation tasks, one shared queue so
+ the fleet shares the work.
+- 'copilot_cancel' (FANOUT) for cancellation requests, one queue per pod so
+ every pod sees every cancel and the one holding the session acts on it.
"""
import logging
-from typing import Any
+from typing import TYPE_CHECKING, Any
+from uuid import uuid4
from pydantic import BaseModel
@@ -33,6 +35,9 @@ from backend.data.rabbitmq import Exchange, ExchangeType, Queue, RabbitMQConfig
from backend.util.logging import TruncatedLogger, is_structured_logging_enabled
from backend.util.settings import Config
+if TYPE_CHECKING:
+ from pika.adapters.blocking_connection import BlockingChannel
+
logger = logging.getLogger(__name__)
@@ -101,7 +106,16 @@ COPILOT_CANCEL_EXCHANGE = Exchange(
durable=True,
auto_delete=False,
)
-COPILOT_CANCEL_QUEUE_NAME = "copilot_cancel_queue_v2"
+# Pre-2026-09 topology: one durable queue bound to the fanout and consumed by
+# every pod, so RabbitMQ round-robined each cancel to a single arbitrary pod.
+# Old-image pods keep draining it through a rollout; the reaper below deletes it
+# once none is left, so no operator step is needed on any install.
+LEGACY_COPILOT_CANCEL_QUEUE_NAME = "copilot_cancel_queue_v2"
+COPILOT_CANCEL_QUEUE_PREFIX = "copilot_cancel.pod"
+
+# Only waits for the last old-image pod to drain, which is a rollout-scale
+# event; costs one passive declare per pod per interval.
+LEGACY_CANCEL_QUEUE_REAP_INTERVAL_SECONDS = 5 * 60
def get_session_lock_key(session_id: str) -> str:
@@ -123,14 +137,9 @@ GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS = COPILOT_CONSUMER_TIMEOUT_SECONDS
def create_copilot_queue_config() -> RabbitMQConfig:
- """Create RabbitMQ configuration for CoPilot executor.
+ """Declare both exchanges and the shared run queue.
- Defines two exchanges and queues:
- - 'copilot_execution' (DIRECT) for chat generation tasks
- - 'copilot_cancel' (FANOUT) for cancellation requests
-
- Returns:
- RabbitMQConfig with exchanges and queues defined
+ The cancel queue is deliberately absent; see the comment below.
"""
run_queue = Queue(
name=COPILOT_EXECUTION_QUEUE_NAME,
@@ -166,21 +175,86 @@ def create_copilot_queue_config() -> RabbitMQConfig:
"x-consumer-timeout": COPILOT_CONSUMER_TIMEOUT_SECONDS * 1000,
},
)
- cancel_queue = Queue(
- name=COPILOT_CANCEL_QUEUE_NAME,
- exchange=COPILOT_CANCEL_EXCHANGE,
- routing_key="", # not used for FANOUT
- durable=True,
- auto_delete=False,
- arguments={"x-queue-type": "quorum"},
- )
+ # No cancel queue here: it is per-pod, exclusive to the consuming
+ # connection, and declared by the consumer itself in
+ # ``declare_pod_cancel_queue``. Declaring it in the shared config would
+ # bind one queue for the whole fleet again, and every other holder of this
+ # config (the API, which only publishes) would own a queue nobody drains.
return RabbitMQConfig(
vhost=Config().rabbitmq_vhost,
exchanges=[COPILOT_EXECUTION_EXCHANGE, COPILOT_CANCEL_EXCHANGE],
- queues=[run_queue, cancel_queue],
+ queues=[run_queue],
)
+def declare_pod_cancel_queue(channel: "BlockingChannel", executor_id: str) -> str:
+ """Give this pod its own queue on the cancel fanout and return its name.
+
+ A fanout reaches every pod only when every pod owns a queue: consumers on
+ one shared queue get round-robined, so a cancel lands on one arbitrary pod
+ and the pod actually running that session never hears it. Exclusive and
+ auto-delete, so the queue dies with the connection that declared it.
+ """
+ queue_name = f"{COPILOT_CANCEL_QUEUE_PREFIX}.{executor_id}.{uuid4().hex[:8]}"
+ channel.queue_declare(
+ queue=queue_name, durable=False, exclusive=True, auto_delete=True
+ )
+ channel.queue_bind(
+ queue=queue_name, exchange=COPILOT_CANCEL_EXCHANGE.name, routing_key=""
+ )
+ return queue_name
+
+
+def start_legacy_cancel_queue_reaper(channel: "BlockingChannel") -> None:
+ """Delete the retired fleet-wide cancel queue once no pod is draining it.
+
+ Runs now and every few minutes after, on the consumer's own connection:
+ a pika ``BlockingConnection`` is not thread-safe, and ``call_later`` fires
+ from inside ``start_consuming``. An old-image pod that reconnects declares
+ that queue again, so the pass repeats for as long as this consumer lives.
+ """
+ reap_legacy_cancel_queue(channel)
+ try:
+ channel.connection.call_later(
+ LEGACY_CANCEL_QUEUE_REAP_INTERVAL_SECONDS,
+ lambda: start_legacy_cancel_queue_reaper(channel),
+ )
+ except Exception as e:
+ logger.debug(f"Legacy cancel queue reaper not re-armed: {e}")
+
+
+def reap_legacy_cancel_queue(channel: "BlockingChannel") -> bool:
+ """Delete the legacy cancel queue if it exists and nothing consumes it.
+
+ The consumer count is the rollout gate: while an old-image pod still drains
+ that queue, deleting it would take its cancels away. Runs on a scratch
+ channel because a 404 from the passive declare closes the channel it
+ arrives on, and the caller's is carrying the consumer.
+ """
+ try:
+ scratch = channel.connection.channel()
+ except Exception:
+ logger.warning("Could not open a channel to reap the legacy cancel queue")
+ return False
+ try:
+ queue = scratch.queue_declare(
+ queue=LEGACY_COPILOT_CANCEL_QUEUE_NAME, passive=True
+ ).method
+ if queue.consumer_count:
+ return False
+ # Check-then-act, because RabbitMQ refuses `if_unused` on a quorum queue:
+ # a pod that re-consumes in the gap re-declares it when it reconnects.
+ scratch.queue_delete(queue=LEGACY_COPILOT_CANCEL_QUEUE_NAME)
+ logger.info(f"Deleted retired queue {LEGACY_COPILOT_CANCEL_QUEUE_NAME}")
+ return True
+ except Exception as e:
+ logger.debug(f"{LEGACY_COPILOT_CANCEL_QUEUE_NAME} not reaped: {e}")
+ return False
+ finally:
+ if scratch.is_open:
+ scratch.close()
+
+
# ============ Message Models ============ #
@@ -624,8 +698,6 @@ async def schedule_chat_turn(
"""
# Deferred so the executor module stays a leaf for the queue dataclasses
# (only the chat HTTP path persists user messages this way).
- from uuid import uuid4
-
from backend.copilot.model import ChatMessage, append_and_save_message
from backend.copilot.prompting import VOICE_TURN_PREFIX
from backend.copilot.service import strip_server_injected_tags
diff --git a/autogpt_platform/backend/backend/copilot/executor/utils_test.py b/autogpt_platform/backend/backend/copilot/executor/utils_test.py
index db0129f222..1148112954 100644
--- a/autogpt_platform/backend/backend/copilot/executor/utils_test.py
+++ b/autogpt_platform/backend/backend/copilot/executor/utils_test.py
@@ -7,6 +7,7 @@ import pytest
from backend.copilot.executor import utils
from backend.copilot.executor.utils import (
+ COPILOT_CANCEL_EXCHANGE,
COPILOT_EXECUTION_EXCHANGE,
COPILOT_EXECUTION_QUEUE_NAME,
COPILOT_EXECUTION_ROUTING_KEY,
@@ -14,6 +15,7 @@ from backend.copilot.executor.utils import (
CoPilotExecutionEntry,
CoPilotLogMetadata,
create_copilot_queue_config,
+ declare_pod_cancel_queue,
)
from backend.copilot.permissions import CopilotPermissions
from backend.copilot.prompting import VOICE_TURN_TAG
@@ -127,7 +129,7 @@ class TestCreateCopilotQueueConfig:
def test_returns_valid_config(self):
config = create_copilot_queue_config()
assert len(config.exchanges) == 2
- assert len(config.queues) == 2
+ assert len(config.queues) == 1
def test_execution_queue_properties(self):
config = create_copilot_queue_config()
@@ -138,13 +140,32 @@ class TestCreateCopilotQueueConfig:
assert exec_queue.exchange == COPILOT_EXECUTION_EXCHANGE
assert exec_queue.routing_key == COPILOT_EXECUTION_ROUTING_KEY
- def test_cancel_queue_uses_fanout(self):
+ def test_config_declares_no_shared_cancel_queue(self):
+ """A queue in this config is declared by every holder, the publishing
+ API included, and shared by every consumer — which is what made the
+ fanout deliver each cancel to one pod."""
config = create_copilot_queue_config()
- cancel_queue = next(
- q for q in config.queues if q.name != COPILOT_EXECUTION_QUEUE_NAME
+ assert COPILOT_CANCEL_EXCHANGE in config.exchanges
+ assert all(q.exchange != COPILOT_CANCEL_EXCHANGE for q in config.queues)
+
+
+class TestDeclarePodCancelQueue:
+ def test_queue_is_exclusive_auto_delete_and_bound_to_the_fanout(self):
+ channel = MagicMock()
+ name = declare_pod_cancel_queue(channel, "executor-1")
+ channel.queue_declare.assert_called_once_with(
+ queue=name, durable=False, exclusive=True, auto_delete=True
)
- assert cancel_queue.exchange is not None
- assert cancel_queue.exchange.type.value == "fanout"
+ channel.queue_bind.assert_called_once_with(
+ queue=name, exchange=COPILOT_CANCEL_EXCHANGE.name, routing_key=""
+ )
+
+ def test_each_pod_gets_its_own_queue(self):
+ names = {
+ declare_pod_cancel_queue(MagicMock(), executor_id)
+ for executor_id in ("executor-1", "executor-2", "executor-1")
+ }
+ assert len(names) == 3
class TestCoPilotLogMetadata:
diff --git a/autogpt_platform/backend/backend/data/rabbitmq_test.py b/autogpt_platform/backend/backend/data/rabbitmq_test.py
index 93cec9fa83..9eefa7ddba 100644
--- a/autogpt_platform/backend/backend/data/rabbitmq_test.py
+++ b/autogpt_platform/backend/backend/data/rabbitmq_test.py
@@ -83,10 +83,11 @@ def test_copilot_execution_queue_is_quorum_with_consumer_timeout() -> None:
assert timeout_ms >= 60 * 60 * 1000 # at least 1 hour
-def test_copilot_cancel_queue_is_quorum() -> None:
+def test_copilot_config_declares_no_cancel_queue() -> None:
+ """Copilot cancels fan out to a per-pod exclusive queue the consumer
+ declares itself; a queue here would be shared by the whole fleet again."""
cfg = create_copilot_queue_config()
- cancel = next(q for q in cfg.queues if q.name.endswith("cancel_queue_v2"))
- assert cancel.arguments == {"x-queue-type": "quorum"}
+ assert [q.name for q in cfg.queues] == [COPILOT_EXECUTION_QUEUE_NAME]
# ---------- AsyncRabbitMQ.publish_message: mock-driven behaviour ----------