AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(backend): Redis Cluster client support (#12900)
Zamil Majdy2879528308da
diff --git a/.github/workflows/platform-backend-ci.yml b/.github/workflows/platform-backend-ci.ymlindex 083b228962..5e47afb8dc 100644--- a/.github/workflows/platform-backend-ci.yml+++ b/.github/workflows/platform-backend-ci.yml@@ -119,10 +119,12 @@ jobs:runs-on: ubuntu-latestservices:- redis:- image: redis:latest- ports:- - 6379:6379+ # Redis is provisioned as a real 3-shard cluster below via docker+ # run (see the "Start Redis Cluster" step). GHA services can't+ # override the image CMD or stand up multi-container clusters, so+ # that setup is inlined — it mirrors the topology of the local dev+ # compose stack (autogpt_platform/docker-compose.platform.yml) and+ # prod helm chart.rabbitmq:image: rabbitmq:4.1.4ports:@@ -166,6 +168,68 @@ jobs:with:python-version: ${{ matrix.python-version }}+ - name: Start Redis Cluster (3 shards)+ run: |+ # 3-master Redis Cluster matching the local compose stack+ # (autogpt_platform/docker-compose.platform.yml) and prod. Each+ # shard runs in its own container on a dedicated bridge network,+ # announces its compose-style hostname for intra-network clients,+ # and publishes 1700N on the GHA host so tests can reach every+ # shard via localhost. The backend's ``_address_remap`` rewrites+ # every CLUSTER SLOTS reply to localhost:<announced-port>, which+ # picks the right published port per shard.+ #+ # Not reusing docker-compose.platform.yml directly because compose+ # validates the full file even when only some services are ``up``,+ # and that file references services (db/kong/...) defined in a+ # sibling compose file — pulling both in would needlessly couple+ # CI to the full local-dev stack.+ docker network create redis-cluster-ci+ for i in 0 1 2; do+ port=$((17000 + i))+ bus=$((27000 + i))+ docker run -d --name redis-$i --network redis-cluster-ci \+ --network-alias redis-$i \+ -p $port:$port \+ redis:7 \+ redis-server --port $port \+ --cluster-enabled yes \+ --cluster-config-file nodes.conf \+ --cluster-node-timeout 5000 \+ --cluster-require-full-coverage no \+ --cluster-announce-hostname redis-$i \+ --cluster-announce-port $port \+ --cluster-announce-bus-port $bus \+ --cluster-preferred-endpoint-type hostname+ done+ # Wait for each shard to accept commands.+ for i in 0 1 2; do+ port=$((17000 + i))+ for _ in $(seq 1 30); do+ docker exec redis-$i redis-cli -p $port ping 2>/dev/null | grep -q PONG && break+ sleep 1+ done+ done+ # Form the cluster from an init container on the same network so+ # --cluster-preferred-endpoint-type hostname resolves redis-0/1/2.+ docker run --rm --network redis-cluster-ci redis:7 \+ redis-cli --cluster create \+ redis-0:17000 redis-1:17001 redis-2:17002 \+ --cluster-replicas 0 --cluster-yes+ # Confirm convergence.+ for _ in $(seq 1 30); do+ state=$(docker exec redis-0 redis-cli -p 17000 cluster info | awk -F: '/^cluster_state:/ {print $2}' | tr -d '[:cntrl:]')+ if [ "$state" = "ok" ]; then+ echo "Redis Cluster ready (3 shards, state=ok)"+ docker exec redis-0 redis-cli -p 17000 cluster nodes+ exit 0+ fi+ sleep 1+ done+ echo "Redis Cluster failed to reach ok state" >&2+ docker exec redis-0 redis-cli -p 17000 cluster info >&2 || true+ exit 1+- name: Setup Supabaseuses: supabase/setup-cli@v1with:@@ -286,8 +350,13 @@ jobs:SUPABASE_SERVICE_ROLE_KEY: ${{ steps.supabase.outputs.SERVICE_ROLE_KEY }}JWT_VERIFY_KEY: ${{ steps.supabase.outputs.JWT_SECRET }}REDIS_HOST: "localhost"- REDIS_PORT: "6379"+ REDIS_PORT: "17000"ENCRYPTION_KEY: "dvziYgz0KSK8FENhju0ZYi8-fRTfAdlz6YLhdB_jhNw=" # DO NOT USE IN PRODUCTION!!+ # Opt-in: lets backend/data/e2e_redis_restart_test.py spin up its+ # own isolated 3-shard cluster (ports 27110–27112) and exercise+ # ``docker restart <shard>`` mid-stream. Off locally so a+ # contributor's ``poetry run test`` doesn't pay the ~15s cost.+ E2E_RESTART_ISOLATED: "1"- name: Upload coverage reports to Codecovif: ${{ !cancelled() }}diff --git a/autogpt_platform/autogpt_libs/autogpt_libs/rate_limit/__init__.py b/autogpt_platform/autogpt_libs/autogpt_libs/rate_limit/__init__.pydeleted file mode 100644index e69de29bb2..0000000000diff --git a/autogpt_platform/autogpt_libs/autogpt_libs/rate_limit/config.py b/autogpt_platform/autogpt_libs/autogpt_libs/rate_limit/config.pydeleted file mode 100644index beafc14f24..0000000000--- a/autogpt_platform/autogpt_libs/autogpt_libs/rate_limit/config.py+++ /dev/null@@ -1,33 +0,0 @@-from typing import Optional--from pydantic import Field-from pydantic_settings import BaseSettings, SettingsConfigDict---class RateLimitSettings(BaseSettings):- redis_host: str = Field(- default="redis://localhost:6379",- description="Redis host",- validation_alias="REDIS_HOST",- )-- redis_port: str = Field(- default="6379", description="Redis port", validation_alias="REDIS_PORT"- )-- redis_password: Optional[str] = Field(- default=None,- description="Redis password",- validation_alias="REDIS_PASSWORD",- )-- requests_per_minute: int = Field(- default=60,- description="Maximum number of requests allowed per minute per API key",- validation_alias="RATE_LIMIT_REQUESTS_PER_MINUTE",- )-- model_config = SettingsConfigDict(case_sensitive=True, extra="ignore")---RATE_LIMIT_SETTINGS = RateLimitSettings()diff --git a/autogpt_platform/autogpt_libs/autogpt_libs/rate_limit/limiter.py b/autogpt_platform/autogpt_libs/autogpt_libs/rate_limit/limiter.pydeleted file mode 100644index f86434d60b..0000000000--- a/autogpt_platform/autogpt_libs/autogpt_libs/rate_limit/limiter.py+++ /dev/null@@ -1,51 +0,0 @@-import time-from typing import Tuple--from redis import Redis--from .config import RATE_LIMIT_SETTINGS---class RateLimiter:- def __init__(- self,- redis_host: str = RATE_LIMIT_SETTINGS.redis_host,- redis_port: str = RATE_LIMIT_SETTINGS.redis_port,- redis_password: str | None = RATE_LIMIT_SETTINGS.redis_password,- requests_per_minute: int = RATE_LIMIT_SETTINGS.requests_per_minute,- ):- self.redis = Redis(- host=redis_host,- port=int(redis_port),- password=redis_password,- decode_responses=True,- )- self.window = 60- self.max_requests = requests_per_minute-- async def check_rate_limit(self, api_key_id: str) -> Tuple[bool, int, int]:- """- Check if request is within rate limits.-- Args:- api_key_id: The API key identifier to check-- Returns:- Tuple of (is_allowed, remaining_requests, reset_time)- """- now = time.time()- window_start = now - self.window- key = f"ratelimit:{api_key_id}:1min"-- pipe = self.redis.pipeline()- pipe.zremrangebyscore(key, 0, window_start)- pipe.zadd(key, {str(now): now})- pipe.zcount(key, window_start, now)- pipe.expire(key, self.window)-- _, _, request_count, _ = pipe.execute()-- remaining = max(0, self.max_requests - request_count)- reset_time = int(now + self.window)-- return request_count <= self.max_requests, remaining, reset_timediff --git a/autogpt_platform/autogpt_libs/autogpt_libs/rate_limit/middleware.py b/autogpt_platform/autogpt_libs/autogpt_libs/rate_limit/middleware.pydeleted file mode 100644index 496697d8b1..0000000000--- a/autogpt_platform/autogpt_libs/autogpt_libs/rate_limit/middleware.py+++ /dev/null@@ -1,32 +0,0 @@-from fastapi import HTTPException, Request-from starlette.middleware.base import RequestResponseEndpoint--from .limiter import RateLimiter---async def rate_limit_middleware(request: Request, call_next: RequestResponseEndpoint):- """FastAPI middleware for rate limiting API requests."""- limiter = RateLimiter()-- if not request.url.path.startswith("/api"):- return await call_next(request)-- api_key = request.headers.get("Authorization")- if not api_key:- return await call_next(request)-- api_key = api_key.replace("Bearer ", "")-- is_allowed, remaining, reset_time = await limiter.check_rate_limit(api_key)-- if not is_allowed:- raise HTTPException(- status_code=429, detail="Rate limit exceeded. Please try again later."- )-- response = await call_next(request)- response.headers["X-RateLimit-Limit"] = str(limiter.max_requests)- response.headers["X-RateLimit-Remaining"] = str(remaining)- response.headers["X-RateLimit-Reset"] = str(reset_time)-- return responsediff --git a/autogpt_platform/autogpt_libs/autogpt_libs/utils/synchronize.py b/autogpt_platform/autogpt_libs/autogpt_libs/utils/synchronize.pyindex 348ae4d78d..52c5b7d203 100644--- a/autogpt_platform/autogpt_libs/autogpt_libs/utils/synchronize.py+++ b/autogpt_platform/autogpt_libs/autogpt_libs/utils/synchronize.py@@ -1,13 +1,16 @@import asynciofrom contextlib import asynccontextmanager-from typing import TYPE_CHECKING, Any+from typing import TYPE_CHECKING, Any, Unionfrom expiringdict import ExpiringDictif TYPE_CHECKING:from redis.asyncio import Redis as AsyncRedis+ from redis.asyncio.cluster import RedisCluster as AsyncRedisClusterfrom redis.asyncio.lock import Lock as AsyncRedisLock+ AsyncRedisLike = Union[AsyncRedis, AsyncRedisCluster]+class AsyncRedisKeyedMutex:"""@@ -17,7 +20,7 @@ class AsyncRedisKeyedMutex:in case the key is not unlocked for a specified duration, to prevent memory leaks."""- def __init__(self, redis: "AsyncRedis", timeout: int | None = 60):+ def __init__(self, redis: "AsyncRedisLike", timeout: int | None = 60):self.redis = redisself.timeout = timeoutself.locks: dict[Any, "AsyncRedisLock"] = ExpiringDict(diff --git a/autogpt_platform/backend/backend/api/conn_manager.py b/autogpt_platform/backend/backend/api/conn_manager.pyindex 52e0f50f69..6b89c7851c 100644--- a/autogpt_platform/backend/backend/api/conn_manager.py+++ b/autogpt_platform/backend/backend/api/conn_manager.py@@ -1,14 +1,44 @@import asyncio-from typing import Dict, Set+import json+import logging+import time+from typing import Awaitable, Callable, Dict, Optional, Set-from fastapi import WebSocket+from fastapi import WebSocket, WebSocketDisconnect+from redis.asyncio import Redis as AsyncRedis+from redis.asyncio.client import PubSub as AsyncPubSub+from redis.exceptions import MovedError, RedisError, ResponseError+from starlette.websockets import WebSocketState-from backend.api.model import NotificationPayload, WSMessage, WSMethod+from backend.api.model import WSMessage, WSMethod+from backend.data import redis_client as redis+from backend.data.event_bus import _assert_no_wildcardfrom backend.data.execution import (ExecutionEventType,- GraphExecutionEvent,- NodeExecutionEvent,+ exec_channel,+ get_graph_execution_meta,+ graph_all_channel,)+from backend.data.notification_bus import NotificationEvent+from backend.util.settings import Settings++logger = logging.getLogger(__name__)+_settings = Settings()+++def _is_ws_close_race(exc: BaseException, websocket: WebSocket) -> bool:+ """A SPUBLISH→WS send racing with WS close — benign, drop quietly."""+ if isinstance(exc, WebSocketDisconnect):+ return True+ if (+ getattr(websocket, "application_state", None) == WebSocketState.DISCONNECTED+ or getattr(websocket, "client_state", None) == WebSocketState.DISCONNECTED+ ):+ return True+ if isinstance(exc, RuntimeError) and "close message has been sent" in str(exc):+ return True+ return False+_EVENT_TYPE_TO_METHOD_MAP: dict[ExecutionEventType, WSMethod] = {ExecutionEventType.GRAPH_EXEC_UPDATE: WSMethod.GRAPH_EXECUTION_EVENT,@@ -16,128 +46,379 @@ _EVENT_TYPE_TO_METHOD_MAP: dict[ExecutionEventType, WSMethod] = {}+def event_bus_channel(channel_key: str) -> str:+ """Prefix a channel key with the execution event bus name."""+ return f"{_settings.config.execution_event_bus_name}/{channel_key}"+++def _notification_bus_channel(user_id: str) -> str:+ """Return the full sharded channel name for a user's notifications."""+ return f"{_settings.config.notification_event_bus_name}/{user_id}"+++MessageHandler = Callable[[Optional[bytes | str]], Awaitable[None]]+++def _is_moved_error(exc: BaseException) -> bool:+ """A MOVED redirect — slot migration mid-stream; pump should reconnect."""+ if isinstance(exc, MovedError):+ return True+ if isinstance(exc, ResponseError) and str(exc).startswith("MOVED "):+ return True+ return False+++# Reconnect tunables for shard-failover during pubsub.listen().+_PUMP_RECONNECT_DEADLINE_S = 60.0+_PUMP_RECONNECT_BACKOFF_INITIAL_S = 0.5+_PUMP_RECONNECT_BACKOFF_MAX_S = 8.0+++class _Subscription:+ """One SSUBSCRIBE lifecycle bound to a WebSocket, pinned to the owning shard."""++ def __init__(self, full_channel: str) -> None:+ _assert_no_wildcard(full_channel)+ self.full_channel = full_channel+ self._client: AsyncRedis | None = None+ self._pubsub: AsyncPubSub | None = None+ self._task: asyncio.Task | None = None++ async def start(self, on_message: MessageHandler) -> None:+ await self._open_pubsub()+ self._task = asyncio.create_task(self._pump(on_message))++ async def _open_pubsub(self) -> None:+ """(Re)establish the sharded pubsub connection + SSUBSCRIBE."""+ self._client = await redis.connect_sharded_pubsub_async(self.full_channel)+ self._pubsub = self._client.pubsub()+ await self._pubsub.execute_command("SSUBSCRIBE", self.full_channel)+ # redis-py 6.x async PubSub.listen() exits when ``channels`` is+ # empty; raw SSUBSCRIBE doesn't populate it, so do it ourselves.+ self._pubsub.channels[self.full_channel] = None # type: ignore[index]++ async def _close_pubsub_quietly(self) -> None:+ """Best-effort teardown before reconnect — never raises."""+ if self._pubsub is not None:+ try:+ await self._pubsub.aclose()+ except Exception:+ pass+ self._pubsub = None+ if self._client is not None:+ try:+ await self._client.aclose()+ except Exception:+ pass+ self._client = None++ async def _pump(self, on_message: MessageHandler) -> None:+ if self._pubsub is None:+ return+ backoff = _PUMP_RECONNECT_BACKOFF_INITIAL_S+ deadline = time.monotonic() + _PUMP_RECONNECT_DEADLINE_S+ while True:+ pubsub = self._pubsub+ if pubsub is None:+ return+ needs_reconnect = False+ try:+ async for message in pubsub.listen():+ msg_type = message.get("type")+ # Server-pushed sunsubscribe: slot ownership changed and+ # Redis revoked our SSUBSCRIBE without dropping the TCP.+ # Treat as a reconnect trigger so we re-resolve the shard.+ if msg_type == "sunsubscribe":+ needs_reconnect = True+ break+ if msg_type not in ("smessage", "message", "pmessage"):+ continue+ # Successful read resets the reconnect budget.+ backoff = _PUMP_RECONNECT_BACKOFF_INITIAL_S+ deadline = time.monotonic() + _PUMP_RECONNECT_DEADLINE_S+ try:+ await on_message(message.get("data"))+ except Exception:+ logger.exception(+ "Websocket message-handler failed for channel %s",+ self.full_channel,+ )+ if not needs_reconnect:+ # listen() exited cleanly (channels emptied) — pump is done.+ return+ except asyncio.CancelledError:+ raise+ except (ConnectionError, RedisError) as exc:+ if isinstance(exc, ResponseError) and not _is_moved_error(exc):+ logger.exception(+ "Pubsub pump crashed on non-retryable ResponseError for %s",+ self.full_channel,+ )+ return+ if time.monotonic() > deadline:+ logger.exception(+ "Pubsub pump giving up after reconnect deadline for %s",+ self.full_channel,+ )+ return+ logger.warning(+ "Pubsub pump reconnecting for %s after %s: %s",+ self.full_channel,+ type(exc).__name__,+ exc,+ )+ except Exception:+ logger.exception("Pubsub pump crashed for %s", self.full_channel)+ return++ # Either a retryable error was raised, or the server pushed a+ # sunsubscribe — close the stale pubsub and reopen against the+ # (possibly migrated) shard.+ await self._close_pubsub_quietly()+ await asyncio.sleep(backoff)+ backoff = min(backoff * 2, _PUMP_RECONNECT_BACKOFF_MAX_S)+ try:+ await self._open_pubsub()+ except (ConnectionError, RedisError) as reopen_exc:+ logger.warning(+ "Pubsub pump reopen failed for %s: %s",+ self.full_channel,+ reopen_exc,+ )+ # Loop again — deadline check will eventually exit.+ continue++ async def stop(self) -> None:+ if self._task is not None:+ self._task.cancel()+ try:+ await self._task+ except (asyncio.CancelledError, Exception):+ pass+ self._task = None+ if self._pubsub is not None:+ try:+ await self._pubsub.execute_command("SUNSUBSCRIBE", self.full_channel)+ except Exception:+ logger.warning(+ "SUNSUBSCRIBE failed for %s", self.full_channel, exc_info=True+ )+ try:+ await self._pubsub.aclose()+ except Exception:+ pass+ self._pubsub = None+ if self._client is not None:+ try:+ await self._client.aclose()+ except Exception:+ pass+ self._client = None++class ConnectionManager:def __init__(self):self.active_connections: Set[WebSocket] = set()+ # channel_key → sockets subscribed (public channel keys, not raw Redis channels)self.subscriptions: Dict[str, Set[WebSocket]] = {}- self.user_connections: Dict[str, Set[WebSocket]] = {}+ # websocket → {channel_key: _Subscription}+ self._ws_subs: Dict[WebSocket, Dict[str, _Subscription]] = {}+ # websocket → notification subscription+ self._ws_notifications: Dict[WebSocket, _Subscription] = {}async def connect_socket(self, websocket: WebSocket, *, user_id: str):await websocket.accept()self.active_connections.add(websocket)- if user_id not in self.user_connections:- self.user_connections[user_id] = set()- self.user_connections[user_id].add(websocket)+ self._ws_subs.setdefault(websocket, {})+ await self._start_notification_subscription(websocket, user_id=user_id)- def disconnect_socket(self, websocket: WebSocket, *, user_id: str):+ async def disconnect_socket(self, websocket: WebSocket, *, user_id: str):self.active_connections.discard(websocket)- for subscribers in self.subscriptions.values():+ # Stop SSUBSCRIBE pumps before dropping bookkeeping to avoid leaks.+ subs = self._ws_subs.pop(websocket, {})+ for sub in subs.values():+ await sub.stop()+ notif_sub = self._ws_notifications.pop(websocket, None)+ if notif_sub is not None:+ await notif_sub.stop()+ for channel_key, subscribers in list(self.subscriptions.items()):subscribers.discard(websocket)- user_conns = self.user_connections.get(user_id)- if user_conns is not None:- user_conns.discard(websocket)- if not user_conns:- self.user_connections.pop(user_id, None)+ if not subscribers:+ self.subscriptions.pop(channel_key, None)async def subscribe_graph_exec(self, *, user_id: str, graph_exec_id: str, websocket: WebSocket) -> str:- return await self._subscribe(- _graph_exec_channel_key(user_id, graph_exec_id=graph_exec_id), websocket+ # Hash-tagged channel needs graph_id; resolve once per subscribe.+ meta = await get_graph_execution_meta(user_id, graph_exec_id)+ if meta is None:+ raise ValueError(+ f"graph_exec #{graph_exec_id} not found for user #{user_id}"+ )+ channel_key = graph_exec_channel_key(user_id, graph_exec_id=graph_exec_id)+ full_channel = event_bus_channel(+ exec_channel(user_id, meta.graph_id, graph_exec_id))+ await self._open_subscription(websocket, channel_key, full_channel)+ return channel_keyasync def subscribe_graph_execs(self, *, user_id: str, graph_id: str, websocket: WebSocket) -> str:- return await self._subscribe(- _graph_execs_channel_key(user_id, graph_id=graph_id), websocket- )+ channel_key = _graph_execs_channel_key(user_id, graph_id=graph_id)+ full_channel = event_bus_channel(graph_all_channel(user_id, graph_id))+ await self._open_subscription(websocket, channel_key, full_channel)+ return channel_keyasync def unsubscribe_graph_exec(self, *, user_id: str, graph_exec_id: str, websocket: WebSocket) -> str | None:- return await self._unsubscribe(- _graph_exec_channel_key(user_id, graph_exec_id=graph_exec_id), websocket- )+ channel_key = graph_exec_channel_key(user_id, graph_exec_id=graph_exec_id)+ return await self._close_subscription(websocket, channel_key)async def unsubscribe_graph_execs(self, *, user_id: str, graph_id: str, websocket: WebSocket) -> str | None:- return await self._unsubscribe(- _graph_execs_channel_key(user_id, graph_id=graph_id), websocket- )+ channel_key = _graph_execs_channel_key(user_id, graph_id=graph_id)+ return await self._close_subscription(websocket, channel_key)- async def send_execution_update(- self, exec_event: GraphExecutionEvent | NodeExecutionEvent- ) -> int:- graph_exec_id = (- exec_event.id- if isinstance(exec_event, GraphExecutionEvent)- else exec_event.graph_exec_id- )+ async def _open_subscription(+ self, websocket: WebSocket, channel_key: str, full_channel: str+ ) -> None:+ self.subscriptions.setdefault(channel_key, set()).add(websocket)+ per_ws = self._ws_subs.setdefault(websocket, {})+ if channel_key in per_ws:+ return+ sub = _Subscription(full_channel)- n_sent = 0-- channels: set[str] = {- # Send update to listeners for this graph execution- _graph_exec_channel_key(exec_event.user_id, graph_exec_id=graph_exec_id)- }- if isinstance(exec_event, GraphExecutionEvent):- # Send update to listeners for all executions of this graph- channels.add(- _graph_execs_channel_key(- exec_event.user_id, graph_id=exec_event.graph_id- )+ async def on_message(data: Optional[bytes | str]) -> None:+ await self._forward_exec_event(websocket, channel_key, data)++ await sub.start(on_message)+ per_ws[channel_key] = sub++ async def _close_subscription(+ self, websocket: WebSocket, channel_key: str+ ) -> str | None:+ subscribers = self.subscriptions.get(channel_key)+ if subscribers is None:+ return None+ subscribers.discard(websocket)+ if not subscribers:+ self.subscriptions.pop(channel_key, None)+ per_ws = self._ws_subs.get(websocket)+ if per_ws and channel_key in per_ws:+ sub = per_ws.pop(channel_key)+ await sub.stop()+ return channel_key++ async def _forward_exec_event(+ self,+ websocket: WebSocket,+ channel_key: str,+ raw_payload: Optional[bytes | str],+ ) -> None:+ if raw_payload is None:+ return+ # Unwrap the `_EventPayloadWrapper` envelope, then re-wrap as a WS message.+ try:+ wrapper = (+ raw_payload.decode()+ if isinstance(raw_payload, (bytes, bytearray))+ else raw_payload+ )+ except Exception:+ logger.warning(+ "Failed to decode pubsub payload on %s", channel_key, exc_info=True)+ return- for channel in channels.intersection(self.subscriptions.keys()):+ try:+ parsed = json.loads(wrapper)+ event_data = parsed.get("payload")+ if not isinstance(event_data, dict):+ return+ event_type = event_data.get("event_type")+ method = _EVENT_TYPE_TO_METHOD_MAP.get(ExecutionEventType(event_type))+ if method is None:+ returnmessage = WSMessage(- method=_EVENT_TYPE_TO_METHOD_MAP[exec_event.event_type],- channel=channel,- data=exec_event.model_dump(),+ method=method,+ channel=channel_key,+ data=event_data,).model_dump_json()- for connection in self.subscriptions[channel]:- await connection.send_text(message)- n_sent += 1+ await websocket.send_text(message)+ except Exception as e:+ if _is_ws_close_race(e, websocket):+ logger.debug("Dropped exec event on closed WS for %s", channel_key)+ return+ logger.exception("Failed to forward exec event on %s", channel_key)- return n_sent+ async def _start_notification_subscription(+ self, websocket: WebSocket, *, user_id: str+ ) -> None:+ full_channel = _notification_bus_channel(user_id)+ sub = _Subscription(full_channel)- async def send_notification(- self, *, user_id: str, payload: NotificationPayload- ) -> int:- """Send a notification to all websocket connections belonging to a user."""+ async def on_message(data: Optional[bytes | str]) -> None:+ await self._forward_notification(websocket, user_id, data)++ try:+ await sub.start(on_message)+ except Exception:+ logger.exception(+ "Failed to open notification SSUBSCRIBE for user=%s", user_id+ )+ return+ self._ws_notifications[websocket] = sub++ async def _forward_notification(+ self,+ websocket: WebSocket,+ user_id: str,+ raw_payload: Optional[bytes | str],+ ) -> None:+ if raw_payload is None:+ return+ try:+ wrapper_json = (+ raw_payload.decode()+ if isinstance(raw_payload, (bytes, bytearray))+ else raw_payload+ )+ parsed = json.loads(wrapper_json)+ inner = parsed.get("payload") if isinstance(parsed, dict) else None+ if not isinstance(inner, dict):+ return+ event = NotificationEvent.model_validate(inner)+ except Exception:+ logger.warning(+ "Failed to parse notification payload for user=%s",+ user_id,+ exc_info=True,+ )+ return+ # Defense in depth against cross-user payloads.+ if event.user_id != user_id:+ returnmessage = WSMessage(method=WSMethod.NOTIFICATION,- data=payload.model_dump(),+ data=event.payload.model_dump(),).model_dump_json()-- connections = tuple(self.user_connections.get(user_id, set()))- if not connections:- return 0-- await asyncio.gather(- *(connection.send_text(message) for connection in connections),- return_exceptions=True,- )-- return len(connections)-- async def _subscribe(self, channel_key: str, websocket: WebSocket) -> str:- if channel_key not in self.subscriptions:- self.subscriptions[channel_key] = set()- self.subscriptions[channel_key].add(websocket)- return channel_key-- async def _unsubscribe(self, channel_key: str, websocket: WebSocket) -> str | None:- if channel_key in self.subscriptions:- self.subscriptions[channel_key].discard(websocket)- if not self.subscriptions[channel_key]:- del self.subscriptions[channel_key]- return channel_key- return None+ try:+ await websocket.send_text(message)+ except Exception as e:+ if _is_ws_close_race(e, websocket):+ logger.debug("Dropped notification on closed WS for user=%s", user_id)+ return+ logger.warning(+ "Failed to deliver notification to WS for user=%s",+ user_id,+ exc_info=True,+ )-def _graph_exec_channel_key(user_id: str, *, graph_exec_id: str) -> str:+def graph_exec_channel_key(user_id: str, *, graph_exec_id: str) -> str:return f"{user_id}|graph_exec#{graph_exec_id}"diff --git a/autogpt_platform/backend/backend/api/conn_manager_integration_test.py b/autogpt_platform/backend/backend/api/conn_manager_integration_test.pynew file mode 100644index 0000000000..c7b7c7534f--- /dev/null+++ b/autogpt_platform/backend/backend/api/conn_manager_integration_test.py@@ -0,0 +1,386 @@+"""ConnectionManager integration over the live 3-shard Redis cluster:+SSUBSCRIBE → SPUBLISH → WebSocket forwarding with no Redis mocks. Skips+when the cluster is unreachable."""++import asyncio+import json+from datetime import datetime, timezone+from unittest.mock import AsyncMock+from uuid import uuid4++import pytest+from fastapi import WebSocket++import backend.data.redis_client as redis_client+from backend.api.conn_manager import (+ ConnectionManager,+ _graph_execs_channel_key,+ event_bus_channel,+ graph_exec_channel_key,+)+from backend.api.model import WSMethod+from backend.data.execution import (+ ExecutionStatus,+ GraphExecutionEvent,+ GraphExecutionMeta,+ NodeExecutionEvent,+ exec_channel,+ graph_all_channel,+)+++def _has_live_cluster() -> bool:+ try:+ c = redis_client.connect()+ except Exception: # noqa: BLE001 — any connect failure → skip+ return False+ try:+ c.close()+ except Exception:+ pass+ return True+++pytestmark = pytest.mark.skipif(+ not _has_live_cluster(),+ reason="local redis cluster not reachable; skip conn_manager integration",+)+++def _meta(user_id: str, graph_id: str, graph_exec_id: str) -> GraphExecutionMeta:+ """Build a minimal GraphExecutionMeta for ``subscribe_graph_exec`` to use."""+ return GraphExecutionMeta(+ id=graph_exec_id,+ user_id=user_id,+ graph_id=graph_id,+ graph_version=1,+ inputs=None,+ credential_inputs=None,+ nodes_input_masks=None,+ preset_id=None,+ status=ExecutionStatus.RUNNING,+ started_at=datetime.now(tz=timezone.utc),+ ended_at=None,+ stats=GraphExecutionMeta.Stats(),+ )+++def _node_event_payload(+ *, user_id: str, graph_id: str, graph_exec_id: str, marker: str+) -> bytes:+ """Wire-format a NodeExecutionEvent the way RedisExecutionEventBus would."""+ inner = NodeExecutionEvent(+ user_id=user_id,+ graph_id=graph_id,+ graph_version=1,+ graph_exec_id=graph_exec_id,+ node_exec_id=f"node-exec-{marker}",+ node_id="node-1",+ block_id="block-1",+ status=ExecutionStatus.COMPLETED,+ input_data={"in": marker},+ output_data={"out": [marker]},+ add_time=datetime.now(tz=timezone.utc),+ queue_time=None,+ start_time=datetime.now(tz=timezone.utc),+ end_time=datetime.now(tz=timezone.utc),+ ).model_dump(mode="json")+ return json.dumps({"payload": inner}).encode()+++def _graph_event_payload(+ *, user_id: str, graph_id: str, graph_exec_id: str, marker: str+) -> bytes:+ inner = GraphExecutionEvent(+ id=graph_exec_id,+ user_id=user_id,+ graph_id=graph_id,+ graph_version=1,+ preset_id=None,+ status=ExecutionStatus.COMPLETED,+ started_at=datetime.now(tz=timezone.utc),+ ended_at=datetime.now(tz=timezone.utc),+ stats=GraphExecutionEvent.Stats(+ cost=0,+ duration=1.0,+ node_exec_time=0.5,+ node_exec_count=1,+ ),+ inputs={"x": marker},+ credential_inputs=None,+ nodes_input_masks=None,+ outputs={"y": [marker]},+ ).model_dump(mode="json")+ return json.dumps({"payload": inner}).encode()+++async def _wait_until(predicate, timeout: float = 5.0, interval: float = 0.05) -> bool:+ """Poll ``predicate()`` until truthy or timeout — used to wait for pubsub."""+ deadline = asyncio.get_event_loop().time() + timeout+ while asyncio.get_event_loop().time() < deadline:+ if predicate():+ return True+ await asyncio.sleep(interval)+ return False+++async def test_two_clients_get_independent_ssubscribes_on_right_shards(+ monkeypatch,+) -> None:+ """Two WS clients on different graph_exec_ids each receive ONLY their+ own publish, even when the channels land on different shards."""+ user_id = "user-conn-int-1"+ graph_a = f"graph-a-{uuid4().hex[:8]}"+ graph_b = f"graph-b-{uuid4().hex[:8]}"+ exec_a = f"exec-a-{uuid4().hex[:8]}"+ exec_b = f"exec-b-{uuid4().hex[:8]}"++ # Stub Prisma lookup so tests don't need a DB.+ async def _fake_meta(_uid, gex_id):+ return _meta(user_id, graph_a if gex_id == exec_a else graph_b, gex_id)++ monkeypatch.setattr("backend.api.conn_manager.get_graph_execution_meta", _fake_meta)++ cm = ConnectionManager()+ ws_a: AsyncMock = AsyncMock(spec=WebSocket)+ ws_b: AsyncMock = AsyncMock(spec=WebSocket)+ sent_a: list[str] = []+ sent_b: list[str] = []+ ws_a.send_text = AsyncMock(side_effect=lambda m: sent_a.append(m))+ ws_b.send_text = AsyncMock(side_effect=lambda m: sent_b.append(m))++ redis_client.get_redis.cache_clear()+ cluster = redis_client.get_redis()++ try:+ await cm.subscribe_graph_exec(+ user_id=user_id, graph_exec_id=exec_a, websocket=ws_a+ )+ await cm.subscribe_graph_exec(+ user_id=user_id, graph_exec_id=exec_b, websocket=ws_b+ )+ # Let SSUBSCRIBE settle on each shard.+ await asyncio.sleep(0.2)++ # Publish to each per-exec channel.+ chan_a = event_bus_channel(exec_channel(user_id, graph_a, exec_a))+ chan_b = event_bus_channel(exec_channel(user_id, graph_b, exec_b))+ cluster.spublish(+ chan_a,+ _node_event_payload(+ user_id=user_id,+ graph_id=graph_a,+ graph_exec_id=exec_a,+ marker="A",+ ).decode(),+ )+ cluster.spublish(+ chan_b,+ _node_event_payload(+ user_id=user_id,+ graph_id=graph_b,+ graph_exec_id=exec_b,+ marker="B",+ ).decode(),+ )++ delivered = await _wait_until(lambda: sent_a and sent_b, timeout=5.0)+ assert delivered, f"timeout: sent_a={sent_a!r} sent_b={sent_b!r}"++ msg_a = json.loads(sent_a[0])+ msg_b = json.loads(sent_b[0])+ assert msg_a["channel"] == graph_exec_channel_key(user_id, graph_exec_id=exec_a)+ assert msg_b["channel"] == graph_exec_channel_key(user_id, graph_exec_id=exec_b)+ assert msg_a["data"]["graph_exec_id"] == exec_a+ assert msg_b["data"]["graph_exec_id"] == exec_b+ # No cross-talk: each socket got exactly one message.+ assert len(sent_a) == 1 and len(sent_b) == 1+ finally:+ await cm.disconnect_socket(ws_a, user_id=user_id)+ await cm.disconnect_socket(ws_b, user_id=user_id)+ redis_client.disconnect()+++async def test_aggregate_channel_receives_per_exec_publishes(monkeypatch) -> None:+ """A subscriber on the ``graph_execs`` aggregate channel must receive the+ GraphExecutionEvent published to the ``/all`` channel — even though+ per-exec events go to a different channel."""+ user_id = "user-conn-int-2"+ graph_id = f"graph-{uuid4().hex[:8]}"+ exec_id = f"exec-{uuid4().hex[:8]}"++ async def _fake_meta(_uid, gex_id):+ return _meta(user_id, graph_id, gex_id)++ monkeypatch.setattr("backend.api.conn_manager.get_graph_execution_meta", _fake_meta)++ cm = ConnectionManager()+ ws_agg: AsyncMock = AsyncMock(spec=WebSocket)+ ws_per: AsyncMock = AsyncMock(spec=WebSocket)+ sent_agg: list[str] = []+ sent_per: list[str] = []+ ws_agg.send_text = AsyncMock(side_effect=lambda m: sent_agg.append(m))+ ws_per.send_text = AsyncMock(side_effect=lambda m: sent_per.append(m))++ redis_client.get_redis.cache_clear()+ cluster = redis_client.get_redis()++ try:+ await cm.subscribe_graph_execs(+ user_id=user_id, graph_id=graph_id, websocket=ws_agg+ )+ await cm.subscribe_graph_exec(+ user_id=user_id, graph_exec_id=exec_id, websocket=ws_per+ )+ await asyncio.sleep(0.2)++ # The eventbus publishes the same event to both channels — replicate.+ chan_per = event_bus_channel(exec_channel(user_id, graph_id, exec_id))+ chan_all = event_bus_channel(graph_all_channel(user_id, graph_id))+ payload = _graph_event_payload(+ user_id=user_id,+ graph_id=graph_id,+ graph_exec_id=exec_id,+ marker="agg",+ ).decode()+ cluster.spublish(chan_per, payload)+ cluster.spublish(chan_all, payload)++ delivered = await _wait_until(lambda: sent_agg and sent_per, timeout=5.0)+ assert delivered, f"sent_agg={sent_agg!r} sent_per={sent_per!r}"+ agg_msg = json.loads(sent_agg[0])+ per_msg = json.loads(sent_per[0])+ # Aggregate subscriber's channel key is the per-graph executions key.+ assert agg_msg["channel"] == _graph_execs_channel_key(+ user_id, graph_id=graph_id+ )+ assert per_msg["channel"] == graph_exec_channel_key(+ user_id, graph_exec_id=exec_id+ )+ assert agg_msg["method"] == WSMethod.GRAPH_EXECUTION_EVENT.value+ finally:+ await cm.disconnect_socket(ws_agg, user_id=user_id)+ await cm.disconnect_socket(ws_per, user_id=user_id)+ redis_client.disconnect()+++async def test_disconnect_unsubscribes_and_drops_future_publishes(monkeypatch) -> None:+ """After ``disconnect_socket`` runs, a subsequent SPUBLISH must NOT reach+ the dead websocket — exercises the SUNSUBSCRIBE + bookkeeping cleanup."""+ user_id = "user-conn-int-3"+ graph_id = f"graph-{uuid4().hex[:8]}"+ exec_id = f"exec-{uuid4().hex[:8]}"++ async def _fake_meta(_uid, gex_id):+ return _meta(user_id, graph_id, gex_id)++ monkeypatch.setattr("backend.api.conn_manager.get_graph_execution_meta", _fake_meta)++ cm = ConnectionManager()+ ws: AsyncMock = AsyncMock(spec=WebSocket)+ sent: list[str] = []+ ws.send_text = AsyncMock(side_effect=lambda m: sent.append(m))++ redis_client.get_redis.cache_clear()+ cluster = redis_client.get_redis()+ chan = event_bus_channel(exec_channel(user_id, graph_id, exec_id))+ payload = _node_event_payload(+ user_id=user_id, graph_id=graph_id, graph_exec_id=exec_id, marker="live"+ ).decode()++ try:+ await cm.subscribe_graph_exec(+ user_id=user_id, graph_exec_id=exec_id, websocket=ws+ )+ await asyncio.sleep(0.15)++ # First publish — must reach the socket.+ cluster.spublish(chan, payload)+ delivered = await _wait_until(lambda: bool(sent), timeout=5.0)+ assert delivered+ assert len(sent) == 1++ # Disconnect → SUNSUBSCRIBE + bookkeeping cleared.+ await cm.disconnect_socket(ws, user_id=user_id)+ # Pump cancellation may drain in-flight messages; wait for it.+ await asyncio.sleep(0.2)++ # Channel bookkeeping must be gone.+ assert (+ graph_exec_channel_key(user_id, graph_exec_id=exec_id)+ not in cm.subscriptions+ )+ assert ws not in cm._ws_subs++ # Second publish — must NOT reach the (already-disconnected) socket.+ cluster.spublish(+ chan,+ _node_event_payload(+ user_id=user_id,+ graph_id=graph_id,+ graph_exec_id=exec_id,+ marker="post-disconnect",+ ).decode(),+ )+ await asyncio.sleep(0.5)+ # Still only the one pre-disconnect message.+ assert len(sent) == 1+ finally:+ redis_client.disconnect()+++async def test_slow_consumer_receives_all_events_without_loss(monkeypatch) -> None:+ """Burst-publish many SPUBLISHes; assert every one reaches the subscriber+ in order — guards against drops/reorderings in the pubsub pump."""+ user_id = "user-conn-int-4"+ graph_id = f"graph-{uuid4().hex[:8]}"+ exec_id = f"exec-{uuid4().hex[:8]}"+ n_events = 100++ async def _fake_meta(_uid, gex_id):+ return _meta(user_id, graph_id, gex_id)++ monkeypatch.setattr("backend.api.conn_manager.get_graph_execution_meta", _fake_meta)++ cm = ConnectionManager()+ ws: AsyncMock = AsyncMock(spec=WebSocket)+ sent: list[str] = []+ ws.send_text = AsyncMock(side_effect=lambda m: sent.append(m))++ redis_client.get_redis.cache_clear()+ cluster = redis_client.get_redis()+ chan = event_bus_channel(exec_channel(user_id, graph_id, exec_id))++ try:+ await cm.subscribe_graph_exec(+ user_id=user_id, graph_exec_id=exec_id, websocket=ws+ )+ await asyncio.sleep(0.2)++ # Burst-publish n_events without yielding to the pump.+ for i in range(n_events):+ cluster.spublish(+ chan,+ _node_event_payload(+ user_id=user_id,+ graph_id=graph_id,+ graph_exec_id=exec_id,+ marker=f"m{i}",+ ).decode(),+ )++ delivered = await _wait_until(+ lambda: len(sent) >= n_events, timeout=15.0, interval=0.1+ )+ assert delivered, f"only delivered {len(sent)}/{n_events}"++ # Validate ordering — Redis pub/sub is FIFO per channel.+ markers = [json.loads(m)["data"]["input_data"]["in"] for m in sent[:n_events]]+ assert markers == [f"m{i}" for i in range(n_events)]+ finally:+ await cm.disconnect_socket(ws, user_id=user_id)+ redis_client.disconnect()diff --git a/autogpt_platform/backend/backend/api/conn_manager_test.py b/autogpt_platform/backend/backend/api/conn_manager_test.pyindex 71dbc0ffee..9d428ca78c 100644--- a/autogpt_platform/backend/backend/api/conn_manager_test.py+++ b/autogpt_platform/backend/backend/api/conn_manager_test.py@@ -1,16 +1,20 @@-from datetime import datetime, timezone-from unittest.mock import AsyncMock+import asyncio+import json+from unittest.mock import AsyncMock, MagicMock, patchimport pytestfrom fastapi import WebSocket-from backend.api.conn_manager import ConnectionManager-from backend.api.model import NotificationPayload, WSMessage, WSMethod-from backend.data.execution import (- ExecutionStatus,- GraphExecutionEvent,- NodeExecutionEvent,+from backend.api.conn_manager import (+ ConnectionManager,+ _graph_execs_channel_key,+ _notification_bus_channel,+ _Subscription,+ event_bus_channel,+ graph_exec_channel_key,)+from backend.api.model import WSMethod+from backend.data.execution import GraphExecutionMeta@pytest.fixture@@ -25,43 +29,64 @@ def mock_websocket() -> AsyncMock:return websocket+def _meta(graph_id: str = "test_graph") -> MagicMock:+ meta = MagicMock(spec=GraphExecutionMeta)+ meta.id = "graph-exec-1"+ meta.user_id = "user-1"+ meta.graph_id = graph_id+ return meta++@pytest.mark.asyncioasync def test_connect(connection_manager: ConnectionManager, mock_websocket: AsyncMock) -> None:- await connection_manager.connect_socket(mock_websocket, user_id="user-1")+ # Stub the SSUBSCRIBE the test has no Redis for.+ with patch.object(+ connection_manager, "_start_notification_subscription", AsyncMock()+ ):+ await connection_manager.connect_socket(mock_websocket, user_id="user-1")assert mock_websocket in connection_manager.active_connections- assert mock_websocket in connection_manager.user_connections["user-1"]mock_websocket.accept.assert_called_once()-def test_disconnect(+async def test_disconnect(connection_manager: ConnectionManager, mock_websocket: AsyncMock) -> None:connection_manager.active_connections.add(mock_websocket)connection_manager.subscriptions["test_channel_42"] = {mock_websocket}- connection_manager.user_connections["user-1"] = {mock_websocket}- connection_manager.disconnect_socket(mock_websocket, user_id="user-1")+ await connection_manager.disconnect_socket(mock_websocket, user_id="user-1")assert mock_websocket not in connection_manager.active_connections- assert mock_websocket not in connection_manager.subscriptions["test_channel_42"]- assert "user-1" not in connection_manager.user_connections+ assert "test_channel_42" not in connection_manager.subscriptions@pytest.mark.asyncioasync def test_subscribe(connection_manager: ConnectionManager, mock_websocket: AsyncMock) -> None:- await connection_manager.subscribe_graph_exec(- user_id="user-1",- graph_exec_id="graph-exec-1",- websocket=mock_websocket,- )+ fake_sub = MagicMock()+ fake_sub.start = AsyncMock()+ fake_sub.stop = AsyncMock()+ with (+ patch(+ "backend.api.conn_manager.get_graph_execution_meta",+ AsyncMock(return_value=_meta()),+ ),+ patch("backend.api.conn_manager._Subscription", return_value=fake_sub),+ ):+ await connection_manager.subscribe_graph_exec(+ user_id="user-1",+ graph_exec_id="graph-exec-1",+ websocket=mock_websocket,+ )assert (mock_websocketin connection_manager.subscriptions["user-1|graph_exec#graph-exec-1"])+ fake_sub.start.assert_awaited_once()@pytest.mark.asyncio@@ -69,7 +94,10 @@ async def test_unsubscribe(connection_manager: ConnectionManager, mock_websocket: AsyncMock) -> None:channel_key = "user-1|graph_exec#graph-exec-1"+ fake_sub = MagicMock()+ fake_sub.stop = AsyncMock()connection_manager.subscriptions[channel_key] = {mock_websocket}+ connection_manager._ws_subs[mock_websocket] = {channel_key: fake_sub}await connection_manager.unsubscribe_graph_exec(user_id="user-1",@@ -77,155 +105,957 @@ async def test_unsubscribe(websocket=mock_websocket,)- assert "test_graph" not in connection_manager.subscriptions+ assert channel_key not in connection_manager.subscriptions+ fake_sub.stop.assert_awaited_once()+++# ---------- Channel-name helpers ----------+++def test_event_bus_channel_prefixes_settings_name() -> None:+ """Every SSUBSCRIBE must go through the bus-namespaced channel."""+ full = event_bus_channel("{u/g}/exec/x")+ # Must start with the configured bus name, then slash, then key.+ assert full.endswith("/{u/g}/exec/x")+ assert "/" in full+++def test_notification_bus_channel_prefixes_settings_name() -> None:+ """Notification channels are ``<bus>/<user_id>`` — no wildcard."""+ full = _notification_bus_channel("user-42")+ assert full.endswith("/user-42")+ assert "*" not in full+++def test_graph_exec_channel_key_shape() -> None:+ assert graph_exec_channel_key("u", graph_exec_id="e") == "u|graph_exec#e"+++def test_graph_execs_channel_key_shape() -> None:+ assert _graph_execs_channel_key("u", graph_id="g") == "u|graph#g|executions"+++# ---------- _Subscription lifecycle ----------+++async def test_subscription_start_issues_ssubscribe_and_primes_channels_map() -> None:+ """start() must SSUBSCRIBE and prime pubsub.channels — redis-py 6.x+ `listen()` exits immediately if `channels` is empty."""+ fake_pubsub = MagicMock()+ fake_pubsub.execute_command = AsyncMock()+ fake_pubsub.channels = {}++ async def _listen():+ # Block so the pump doesn't exit immediately.+ await asyncio.sleep(10)+ if False:+ yield++ fake_pubsub.listen = _listen++ fake_client = MagicMock()+ fake_client.pubsub = MagicMock(return_value=fake_pubsub)++ sub = _Subscription("{u/g}/exec/x")+ with patch(+ "backend.api.conn_manager.redis.connect_sharded_pubsub_async",+ AsyncMock(return_value=fake_client),+ ):+ await sub.start(AsyncMock())+ # SSUBSCRIBE was issued against the full channel.+ fake_pubsub.execute_command.assert_awaited_with("SSUBSCRIBE", "{u/g}/exec/x")+ # channels map primed so listen() doesn't early-exit.+ assert "{u/g}/exec/x" in fake_pubsub.channels+ # stop() must tear everything down cleanly.+ fake_pubsub.aclose = AsyncMock()+ fake_client.aclose = AsyncMock()+ await sub.stop()+ fake_pubsub.aclose.assert_awaited_once()+ fake_client.aclose.assert_awaited_once()+++async def test_subscription_rejects_wildcard_channel_before_redis() -> None:+ """Constructor must guard against wildcard channels — SSUBSCRIBE can't pattern."""+ with pytest.raises(ValueError):+ _Subscription("{u/g}/exec/*")+++async def test_subscription_pump_ignores_non_message_types_and_forwards_smessage() -> (+ None+):+ """_pump() must skip subscribe confirmations and forward ``smessage`` payloads."""+ fake_pubsub = MagicMock()+ fake_pubsub.execute_command = AsyncMock()+ fake_pubsub.channels = {}+ fake_pubsub.aclose = AsyncMock()++ async def _listen():+ yield {"type": "ssubscribe", "data": 1}+ yield {"type": "smessage", "data": b'{"payload":"one"}'}+ yield {"type": "psubscribe", "data": 1}+ yield {"type": "smessage", "data": b'{"payload":"two"}'}++ fake_pubsub.listen = _listen+ fake_client = MagicMock()+ fake_client.pubsub = MagicMock(return_value=fake_pubsub)+ fake_client.aclose = AsyncMock()++ received: list = []++ async def on_message(data) -> None:+ received.append(data)++ sub = _Subscription("{u/g}/exec/x")+ with patch(+ "backend.api.conn_manager.redis.connect_sharded_pubsub_async",+ AsyncMock(return_value=fake_client),+ ):+ await sub.start(on_message)+ await asyncio.sleep(0.05) # let pump drain+ await sub.stop()++ assert received == [b'{"payload":"one"}', b'{"payload":"two"}']+++async def test_subscription_pump_swallows_handler_exceptions() -> None:+ """A raising handler must not crash the pump — next message still delivers."""+ fake_pubsub = MagicMock()+ fake_pubsub.execute_command = AsyncMock()+ fake_pubsub.channels = {}+ fake_pubsub.aclose = AsyncMock()++ async def _listen():+ yield {"type": "smessage", "data": b"boom"}+ yield {"type": "smessage", "data": b"ok"}++ fake_pubsub.listen = _listen+ fake_client = MagicMock()+ fake_client.pubsub = MagicMock(return_value=fake_pubsub)+ fake_client.aclose = AsyncMock()++ calls: list = []++ async def on_message(data) -> None:+ calls.append(data)+ if data == b"boom":+ raise RuntimeError("handler failed")++ sub = _Subscription("{u/g}/exec/x")+ with patch(+ "backend.api.conn_manager.redis.connect_sharded_pubsub_async",+ AsyncMock(return_value=fake_client),+ ):+ await sub.start(on_message)+ await asyncio.sleep(0.05)+ await sub.stop()++ assert calls == [b"boom", b"ok"]+++async def test_subscription_pump_logs_and_exits_on_listen_crash() -> None:+ """A `pubsub.listen()` crash must log + exit cleanly, not propagate."""+ fake_pubsub = MagicMock()+ fake_pubsub.execute_command = AsyncMock()+ fake_pubsub.channels = {}+ fake_pubsub.aclose = AsyncMock()++ async def _listen():+ raise RuntimeError("listen() exploded")+ yield # pragma: no cover — unreachable++ fake_pubsub.listen = _listen++ fake_client = MagicMock()+ fake_client.pubsub = MagicMock(return_value=fake_pubsub)+ fake_client.aclose = AsyncMock()++ sub = _Subscription("{u/g}/exec/x")+ with patch(+ "backend.api.conn_manager.redis.connect_sharded_pubsub_async",+ AsyncMock(return_value=fake_client),+ ):+ await sub.start(AsyncMock())+ # Let the pump actually crash + log.+ if sub._task is not None:+ await sub._task+ # stop() is still safe to run.+ await sub.stop()+++async def test_subscription_pump_noop_when_pubsub_missing() -> None:+ """Defensive: _pump returns fast if _pubsub was cleared before scheduling."""+ sub = _Subscription("{u/g}/exec/x")+ # Directly invoke _pump with no pubsub — must not raise.+ await sub._pump(AsyncMock())+++async def test_subscription_stop_swallows_sunsubscribe_errors() -> None:+ """A broken cluster must not break graceful teardown."""+ fake_pubsub = MagicMock()++ async def _listen():+ await asyncio.sleep(10)+ if False:+ yield++ fake_pubsub.execute_command = AsyncMock(+ side_effect=[None, RuntimeError("SUNSUB broke")]+ )+ fake_pubsub.channels = {}+ fake_pubsub.listen = _listen+ fake_pubsub.aclose = AsyncMock(side_effect=RuntimeError("close broke"))++ fake_client = MagicMock()+ fake_client.pubsub = MagicMock(return_value=fake_pubsub)+ fake_client.aclose = AsyncMock(side_effect=RuntimeError("client close broke"))++ sub = _Subscription("{u/g}/exec/x")+ with patch(+ "backend.api.conn_manager.redis.connect_sharded_pubsub_async",+ AsyncMock(return_value=fake_client),+ ):+ await sub.start(AsyncMock())+ # Must not raise even if every teardown call blows up.+ await sub.stop()@pytest.mark.asyncio-async def test_send_graph_execution_result(+async def test_subscription_pump_reconnects_on_connection_error() -> None:+ """A shard-failover ConnectionError mid-listen must trigger reopen + resubscribe."""+ # First pubsub: listen() raises ConnectionError on iteration.+ pubsub_a = MagicMock()+ pubsub_a.execute_command = AsyncMock()+ pubsub_a.channels = {}+ pubsub_a.aclose = AsyncMock()++ async def _listen_a():+ raise ConnectionError("Connection refused")+ yield # pragma: no cover — unreachable++ pubsub_a.listen = _listen_a++ # Second pubsub: listen() yields a real message after reconnect.+ pubsub_b = MagicMock()+ pubsub_b.execute_command = AsyncMock()+ pubsub_b.channels = {}+ pubsub_b.aclose = AsyncMock()++ async def _listen_b():+ yield {"type": "smessage", "data": b'{"payload":"after-reconnect"}'}++ pubsub_b.listen = _listen_b++ client_a = MagicMock()+ client_a.pubsub = MagicMock(return_value=pubsub_a)+ client_a.aclose = AsyncMock()++ client_b = MagicMock()+ client_b.pubsub = MagicMock(return_value=pubsub_b)+ client_b.aclose = AsyncMock()++ received: list = []++ async def on_message(data) -> None:+ received.append(data)++ sub = _Subscription("{u/g}/exec/x")+ # Drive backoff to ~zero so the test runs fast.+ with (+ patch(+ "backend.api.conn_manager.redis.connect_sharded_pubsub_async",+ AsyncMock(side_effect=[client_a, client_b]),+ ),+ patch("backend.api.conn_manager._PUMP_RECONNECT_BACKOFF_INITIAL_S", 0.001),+ patch("backend.api.conn_manager._PUMP_RECONNECT_BACKOFF_MAX_S", 0.001),+ ):+ await sub.start(on_message)+ # Give the pump enough event-loop turns to fail, reconnect, deliver.+ for _ in range(50):+ await asyncio.sleep(0.01)+ if received:+ break+ await sub.stop()++ assert received == [b'{"payload":"after-reconnect"}']+ # Both pubsubs got an SSUBSCRIBE — the reconnect re-issued the command.+ pubsub_a.execute_command.assert_any_await("SSUBSCRIBE", "{u/g}/exec/x")+ pubsub_b.execute_command.assert_any_await("SSUBSCRIBE", "{u/g}/exec/x")+++async def test_subscription_pump_reconnects_on_moved_error() -> None:+ """A mid-stream cluster slot migration must trigger reopen + resubscribe."""+ from redis.exceptions import MovedError++ # First pubsub: listen() raises MovedError on iteration (slot migrated).+ pubsub_a = MagicMock()+ pubsub_a.execute_command = AsyncMock()+ pubsub_a.channels = {}+ pubsub_a.aclose = AsyncMock()++ async def _listen_a():+ raise MovedError("12182 127.0.0.1:7001")+ yield # pragma: no cover — unreachable++ pubsub_a.listen = _listen_a++ # Second pubsub: listen() yields a real message after reconnect.+ pubsub_b = MagicMock()+ pubsub_b.execute_command = AsyncMock()+ pubsub_b.channels = {}+ pubsub_b.aclose = AsyncMock()++ async def _listen_b():+ yield {"type": "smessage", "data": b'{"payload":"after-moved"}'}++ pubsub_b.listen = _listen_b++ client_a = MagicMock()+ client_a.pubsub = MagicMock(return_value=pubsub_a)+ client_a.aclose = AsyncMock()++ client_b = MagicMock()+ client_b.pubsub = MagicMock(return_value=pubsub_b)+ client_b.aclose = AsyncMock()++ received: list = []++ async def on_message(data) -> None:+ received.append(data)++ sub = _Subscription("{u/g}/exec/x")++ snapshot: dict = {}++ async def on_message_snap(data) -> None:+ snapshot["client"] = sub._client+ snapshot["pubsub"] = sub._pubsub+ received.append(data)++ connect_mock = AsyncMock(side_effect=[client_a, client_b])+ with (+ patch(+ "backend.api.conn_manager.redis.connect_sharded_pubsub_async",+ connect_mock,+ ),+ patch("backend.api.conn_manager._PUMP_RECONNECT_BACKOFF_INITIAL_S", 0.001),+ patch("backend.api.conn_manager._PUMP_RECONNECT_BACKOFF_MAX_S", 0.001),+ ):+ await sub.start(on_message_snap)+ for _ in range(50):+ await asyncio.sleep(0.01)+ if received:+ break+ await sub.stop()++ # (a) reconnect happened — connect_sharded_pubsub_async called twice+ assert connect_mock.await_count == 2+ # (b) post-reconnect message reached the handler+ assert received == [b'{"payload":"after-moved"}']+ # (c) both pubsubs got SSUBSCRIBE — the second one re-resolved the shard+ pubsub_a.execute_command.assert_any_await("SSUBSCRIBE", "{u/g}/exec/x")+ pubsub_b.execute_command.assert_any_await("SSUBSCRIBE", "{u/g}/exec/x")+ # subscription's _client / _pubsub pointed to the new connection at delivery time+ assert snapshot["client"] is client_b+ assert snapshot["pubsub"] is pubsub_b+ # The pump must have closed the stale pubsub before reopening.+ pubsub_a.aclose.assert_awaited()+++async def test_subscription_pump_handles_server_sunsubscribe() -> None:+ """A server-pushed sunsubscribe (slot ownership change) must trigger reopen."""+ # First pubsub: listen() yields one server-side sunsubscribe, then closes.+ pubsub_a = MagicMock()+ pubsub_a.execute_command = AsyncMock()+ pubsub_a.channels = {}+ pubsub_a.aclose = AsyncMock()++ async def _listen_a():+ yield {"type": "sunsubscribe", "channel": b"{u/g}/exec/x", "data": 0}++ pubsub_a.listen = _listen_a++ # Second pubsub: listen() yields a real smessage after reconnect.+ pubsub_b = MagicMock()+ pubsub_b.execute_command = AsyncMock()+ pubsub_b.channels = {}+ pubsub_b.aclose = AsyncMock()++ async def _listen_b():+ yield {"type": "smessage", "data": b'{"payload":"after-sunsub"}'}++ pubsub_b.listen = _listen_b++ client_a = MagicMock()+ client_a.pubsub = MagicMock(return_value=pubsub_a)+ client_a.aclose = AsyncMock()+ client_b = MagicMock()+ client_b.pubsub = MagicMock(return_value=pubsub_b)+ client_b.aclose = AsyncMock()++ received: list = []++ async def on_message(data) -> None:+ received.append(data)++ sub = _Subscription("{u/g}/exec/x")++ snapshot: dict = {}++ async def on_message_snap(data) -> None:+ snapshot["client"] = sub._client+ snapshot["pubsub"] = sub._pubsub+ received.append(data)++ connect_mock = AsyncMock(side_effect=[client_a, client_b])+ with (+ patch(+ "backend.api.conn_manager.redis.connect_sharded_pubsub_async",+ connect_mock,+ ),+ patch("backend.api.conn_manager._PUMP_RECONNECT_BACKOFF_INITIAL_S", 0.001),+ patch("backend.api.conn_manager._PUMP_RECONNECT_BACKOFF_MAX_S", 0.001),+ ):+ await sub.start(on_message_snap)+ for _ in range(50):+ await asyncio.sleep(0.01)+ if received:+ break+ await sub.stop()++ # (a) reconnect happened+ assert connect_mock.await_count == 2+ # (b) post-reconnect smessage reached the handler+ assert received == [b'{"payload":"after-sunsub"}']+ # (c) the new connection re-issued SSUBSCRIBE for the channel+ pubsub_b.execute_command.assert_any_await("SSUBSCRIBE", "{u/g}/exec/x")+ # subscription's _client / _pubsub pointed to the new connection at delivery time+ assert snapshot["client"] is client_b+ assert snapshot["pubsub"] is pubsub_b+ # The stale pubsub got closed before reopen.+ pubsub_a.aclose.assert_awaited()+++async def test_subscription_pump_gives_up_after_reconnect_deadline() -> None:+ """If reconnects keep failing past the deadline, the pump must exit (not loop forever)."""+ # Every listen() raises; every reopen also raises ConnectionError.+ pubsub = MagicMock()+ pubsub.execute_command = AsyncMock()+ pubsub.channels = {}+ pubsub.aclose = AsyncMock()++ async def _listen():+ raise ConnectionError("boom")+ yield # pragma: no cover++ pubsub.listen = _listen++ client = MagicMock()+ client.pubsub = MagicMock(return_value=pubsub)+ client.aclose = AsyncMock()++ sub = _Subscription("{u/g}/exec/x")+ with (+ patch(+ "backend.api.conn_manager.redis.connect_sharded_pubsub_async",+ AsyncMock(return_value=client),+ ),+ # Force the deadline to trip on the very first retry.+ patch("backend.api.conn_manager._PUMP_RECONNECT_DEADLINE_S", 0.0),+ patch("backend.api.conn_manager._PUMP_RECONNECT_BACKOFF_INITIAL_S", 0.0),+ ):+ await sub.start(AsyncMock())+ if sub._task is not None:+ await asyncio.wait_for(sub._task, timeout=2.0)+ await sub.stop()+++# ---------- ConnectionManager subscribe reuse / missing-meta handling ----------+++async def test_subscribe_graph_exec_raises_when_meta_missing(connection_manager: ConnectionManager, mock_websocket: AsyncMock) -> None:- channel_key = "user-1|graph_exec#graph-exec-1"+ """If graph_id can't be resolved, subscribe must reject — no silent SSUBSCRIBE."""+ with patch(+ "backend.api.conn_manager.get_graph_execution_meta",+ AsyncMock(return_value=None),+ ):+ with pytest.raises(ValueError):+ await connection_manager.subscribe_graph_exec(+ user_id="user-1",+ graph_exec_id="unknown-exec",+ websocket=mock_websocket,+ )+++async def test_subscribe_graph_exec_reuses_existing_subscription(+ connection_manager: ConnectionManager, mock_websocket: AsyncMock+) -> None:+ """A second subscribe on the same (ws, channel) must not open a second pump."""+ fake_sub = MagicMock()+ fake_sub.start = AsyncMock()+ fake_sub.stop = AsyncMock()++ with (+ patch(+ "backend.api.conn_manager.get_graph_execution_meta",+ AsyncMock(return_value=_meta()),+ ),+ patch("backend.api.conn_manager._Subscription", return_value=fake_sub),+ ):+ await connection_manager.subscribe_graph_exec(+ user_id="user-1",+ graph_exec_id="graph-exec-1",+ websocket=mock_websocket,+ )+ await connection_manager.subscribe_graph_exec(+ user_id="user-1",+ graph_exec_id="graph-exec-1",+ websocket=mock_websocket,+ )++ fake_sub.start.assert_awaited_once()+++async def test_subscribe_graph_execs_opens_aggregate_channel(+ connection_manager: ConnectionManager, mock_websocket: AsyncMock+) -> None:+ """subscribe_graph_execs must SSUBSCRIBE on the aggregate ``/all`` channel."""+ fake_sub = MagicMock()+ fake_sub.start = AsyncMock()++ captured: list[str] = []++ def _capture(full_channel: str) -> MagicMock:+ captured.append(full_channel)+ return fake_sub++ with patch("backend.api.conn_manager._Subscription", side_effect=_capture):+ await connection_manager.subscribe_graph_execs(+ user_id="user-1",+ graph_id="graph-1",+ websocket=mock_websocket,+ )++ assert captured, "must construct a Subscription"+ # Aggregate channel uses {user_id/graph_id} hash tag + /all suffix.+ assert captured[0].endswith("{user-1/graph-1}/all")+ assert "user-1|graph#graph-1|executions" in connection_manager.subscriptions+++async def test_unsubscribe_graph_execs_tears_down(+ connection_manager: ConnectionManager, mock_websocket: AsyncMock+) -> None:+ """unsubscribe_graph_execs must drop bookkeeping and stop the subscription."""+ channel_key = "user-1|graph#graph-1|executions"+ fake_sub = MagicMock()+ fake_sub.stop = AsyncMock()connection_manager.subscriptions[channel_key] = {mock_websocket}- result = GraphExecutionEvent(- id="graph-exec-1",+ connection_manager._ws_subs[mock_websocket] = {channel_key: fake_sub}++ await connection_manager.unsubscribe_graph_execs(+ user_id="user-1",+ graph_id="graph-1",+ websocket=mock_websocket,+ )++ assert channel_key not in connection_manager.subscriptions+ fake_sub.stop.assert_awaited_once()+++async def test_unsubscribe_unknown_channel_returns_none(+ connection_manager: ConnectionManager, mock_websocket: AsyncMock+) -> None:+ """Unsubscribing from a channel that was never subscribed must no-op."""+ result = await connection_manager.unsubscribe_graph_exec(user_id="user-1",- graph_id="test_graph",- graph_version=1,- preset_id=None,- status=ExecutionStatus.COMPLETED,- started_at=datetime.now(tz=timezone.utc),- ended_at=datetime.now(tz=timezone.utc),- stats=GraphExecutionEvent.Stats(- cost=0,- duration=1.2,- node_exec_time=0.5,- node_exec_count=2,+ graph_exec_id="never-subscribed",+ websocket=mock_websocket,+ )+ assert result is None+++async def test_disconnect_stops_all_pumps_and_notification_sub(+ connection_manager: ConnectionManager, mock_websocket: AsyncMock+) -> None:+ """disconnect must await stop() on every exec sub + the notification sub."""+ exec_sub_a = MagicMock()+ exec_sub_a.stop = AsyncMock()+ exec_sub_b = MagicMock()+ exec_sub_b.stop = AsyncMock()+ notif_sub = MagicMock()+ notif_sub.stop = AsyncMock()++ channel_a = "user-1|graph_exec#a"+ channel_b = "user-1|graph_exec#b"++ connection_manager.active_connections.add(mock_websocket)+ connection_manager._ws_subs[mock_websocket] = {+ channel_a: exec_sub_a,+ channel_b: exec_sub_b,+ }+ connection_manager._ws_notifications[mock_websocket] = notif_sub+ connection_manager.subscriptions[channel_a] = {mock_websocket}+ connection_manager.subscriptions[channel_b] = {mock_websocket}++ await connection_manager.disconnect_socket(mock_websocket, user_id="user-1")++ exec_sub_a.stop.assert_awaited_once()+ exec_sub_b.stop.assert_awaited_once()+ notif_sub.stop.assert_awaited_once()+ assert mock_websocket not in connection_manager._ws_subs+ assert mock_websocket not in connection_manager._ws_notifications+ assert channel_a not in connection_manager.subscriptions+ assert channel_b not in connection_manager.subscriptions+++async def test_two_connections_get_independent_subscriptions(+ connection_manager: ConnectionManager,+) -> None:+ """Each WS must get its own _Subscription (its own SSUBSCRIBE)."""+ ws_a: AsyncMock = AsyncMock(spec=WebSocket)+ ws_b: AsyncMock = AsyncMock(spec=WebSocket)+ calls: list[MagicMock] = []++ def _factory(full_channel: str):+ m = MagicMock()+ m.start = AsyncMock()+ m.stop = AsyncMock()+ calls.append(m)+ return m++ with (+ patch(+ "backend.api.conn_manager.get_graph_execution_meta",+ AsyncMock(return_value=_meta()),),- inputs={- "input_1": "some input value :)",- "input_2": "some *other* input value",- },- credential_inputs=None,- nodes_input_masks=None,- outputs={- "the_output": ["some output value"],- "other_output": ["sike there was another output"],- },+ patch("backend.api.conn_manager._Subscription", side_effect=_factory),+ ):+ await connection_manager.subscribe_graph_exec(+ user_id="user-1", graph_exec_id="graph-exec-1", websocket=ws_a+ )+ await connection_manager.subscribe_graph_exec(+ user_id="user-1", graph_exec_id="graph-exec-1", websocket=ws_b+ )++ assert len(calls) == 2+ assert calls[0] is not calls[1]+ # Both websockets are on the same channel set.+ subs = connection_manager.subscriptions["user-1|graph_exec#graph-exec-1"]+ assert ws_a in subs and ws_b in subs+++# ---------- _forward_exec_event ----------+++async def test_forward_exec_event_unwraps_wrapper_and_sends_ws_message(+ connection_manager: ConnectionManager, mock_websocket: AsyncMock+) -> None:+ """Must strip the ``_EventPayloadWrapper`` envelope before re-sending."""+ channel_key = "user-1|graph_exec#x"+ inner = {+ "event_type": "node_execution_update",+ "user_id": "user-1",+ "graph_id": "g",+ "graph_version": 1,+ "graph_exec_id": "x",+ "node_exec_id": "n",+ "node_id": "nid",+ "block_id": "bid",+ "status": "COMPLETED",+ }+ wrapper = json.dumps({"payload": inner})++ await connection_manager._forward_exec_event(+ mock_websocket, channel_key, wrapper.encode())- await connection_manager.send_execution_update(result)+ mock_websocket.send_text.assert_called_once()+ sent = json.loads(mock_websocket.send_text.call_args[0][0])+ assert sent["method"] == WSMethod.NODE_EXECUTION_EVENT.value+ assert sent["channel"] == channel_key+ assert sent["data"]["graph_exec_id"] == "x"+++async def test_forward_exec_event_accepts_str_payload(+ connection_manager: ConnectionManager, mock_websocket: AsyncMock+) -> None:+ """Decode path handles str payload identically to bytes."""+ inner = {+ "event_type": "graph_execution_update",+ "id": "x",+ "user_id": "user-1",+ "graph_id": "g",+ "graph_version": 1,+ "status": "COMPLETED",+ }+ wrapper = json.dumps({"payload": inner})+ await connection_manager._forward_exec_event(mock_websocket, "chan", wrapper)+ mock_websocket.send_text.assert_called_once()+- mock_websocket.send_text.assert_called_once_with(- WSMessage(- method=WSMethod.GRAPH_EXECUTION_EVENT,- channel="user-1|graph_exec#graph-exec-1",- data=result.model_dump(),- ).model_dump_json()+async def test_forward_exec_event_noop_on_none_payload(+ connection_manager: ConnectionManager, mock_websocket: AsyncMock+) -> None:+ await connection_manager._forward_exec_event(mock_websocket, "chan", None)+ mock_websocket.send_text.assert_not_called()+++async def test_forward_exec_event_drops_unknown_event_type(+ connection_manager: ConnectionManager, mock_websocket: AsyncMock+) -> None:+ """Unknown event_type → silent drop, no WS send."""+ wrapper = json.dumps({"payload": {"event_type": "error_comms_update"}})+ await connection_manager._forward_exec_event(+ mock_websocket, "chan", wrapper.encode())+ mock_websocket.send_text.assert_not_called()@pytest.mark.asyncio-async def test_send_node_execution_result(+async def test_forward_exec_event_drops_non_dict_payload(connection_manager: ConnectionManager, mock_websocket: AsyncMock) -> None:- channel_key = "user-1|graph_exec#graph-exec-1"- connection_manager.subscriptions[channel_key] = {mock_websocket}- result = NodeExecutionEvent(- user_id="user-1",- graph_id="test_graph",- graph_version=1,- graph_exec_id="graph-exec-1",- node_exec_id="test_node_exec_id",- node_id="test_node_id",- block_id="test_block_id",- status=ExecutionStatus.COMPLETED,- input_data={"input1": "value1"},- output_data={"output1": ["result1"]},- add_time=datetime.now(tz=timezone.utc),- queue_time=None,- start_time=datetime.now(tz=timezone.utc),- end_time=datetime.now(tz=timezone.utc),+ """Bad envelope (payload not dict) must drop silently, no crash."""+ wrapper = json.dumps({"payload": "not-a-dict"})+ await connection_manager._forward_exec_event(+ mock_websocket, "chan", wrapper.encode())+ mock_websocket.send_text.assert_not_called()- await connection_manager.send_execution_update(result)- mock_websocket.send_text.assert_called_once_with(- WSMessage(- method=WSMethod.NODE_EXECUTION_EVENT,- channel="user-1|graph_exec#graph-exec-1",- data=result.model_dump(),- ).model_dump_json()+async def test_forward_exec_event_swallows_malformed_json(+ connection_manager: ConnectionManager, mock_websocket: AsyncMock+) -> None:+ """Garbage payload must not propagate — it just logs + drops."""+ await connection_manager._forward_exec_event(mock_websocket, "chan", b"not json")+ mock_websocket.send_text.assert_not_called()+++async def test_forward_exec_event_logs_ws_close_race_at_debug(+ connection_manager: ConnectionManager, mock_websocket: AsyncMock+) -> None:+ """SPUBLISH racing WS close → debug, not exception."""+ inner = {+ "event_type": "graph_execution_update",+ "id": "x",+ "user_id": "user-1",+ "graph_id": "g",+ "graph_version": 1,+ "status": "COMPLETED",+ }+ wrapper = json.dumps({"payload": inner})+ mock_websocket.send_text = AsyncMock(+ side_effect=RuntimeError(+ 'Cannot call "send" once a close message has been sent.'+ ))+ with patch("backend.api.conn_manager.logger") as mock_logger:+ await connection_manager._forward_exec_event(+ mock_websocket, "chan", wrapper.encode()+ )+ mock_logger.exception.assert_not_called()+ mock_logger.debug.assert_called_once()+ assert "Dropped exec event on closed WS" in mock_logger.debug.call_args[0][0]@pytest.mark.asyncio-async def test_send_execution_result_user_mismatch(+async def test_forward_exec_event_logs_real_send_error_at_exception(connection_manager: ConnectionManager, mock_websocket: AsyncMock) -> None:- channel_key = "user-1|graph_exec#graph-exec-1"- connection_manager.subscriptions[channel_key] = {mock_websocket}- result = NodeExecutionEvent(- user_id="user-2",- graph_id="test_graph",- graph_version=1,- graph_exec_id="graph-exec-1",- node_exec_id="test_node_exec_id",- node_id="test_node_id",- block_id="test_block_id",- status=ExecutionStatus.COMPLETED,- input_data={"input1": "value1"},- output_data={"output1": ["result1"]},- add_time=datetime.now(tz=timezone.utc),- queue_time=None,- start_time=datetime.now(tz=timezone.utc),- end_time=datetime.now(tz=timezone.utc),+ """A non-close-race RuntimeError must still log at exception level."""+ inner = {+ "event_type": "graph_execution_update",+ "id": "x",+ "user_id": "user-1",+ "graph_id": "g",+ "graph_version": 1,+ "status": "COMPLETED",+ }+ wrapper = json.dumps({"payload": inner})+ mock_websocket.send_text = AsyncMock(side_effect=RuntimeError("something else"))+ # Force application_state to a sentinel that is NOT DISCONNECTED.+ mock_websocket.application_state = None+ mock_websocket.client_state = None+ with patch("backend.api.conn_manager.logger") as mock_logger:+ await connection_manager._forward_exec_event(+ mock_websocket, "chan", wrapper.encode()+ )+ mock_logger.exception.assert_called_once()+++# ---------- _start_notification_subscription / _forward_notification ----------+++async def test_start_notification_subscription_stores_sub(+ connection_manager: ConnectionManager, mock_websocket: AsyncMock+) -> None:+ fake_sub = MagicMock()+ fake_sub.start = AsyncMock()+ with patch("backend.api.conn_manager._Subscription", return_value=fake_sub):+ await connection_manager._start_notification_subscription(+ mock_websocket, user_id="user-1"+ )+ assert connection_manager._ws_notifications[mock_websocket] is fake_sub+ fake_sub.start.assert_awaited_once()+++async def test_start_notification_subscription_swallows_start_errors(+ connection_manager: ConnectionManager, mock_websocket: AsyncMock+) -> None:+ """A failed SSUBSCRIBE must not crash connect — the WS stays usable."""+ fake_sub = MagicMock()+ fake_sub.start = AsyncMock(side_effect=RuntimeError("SSUBSCRIBE failed"))+ with patch("backend.api.conn_manager._Subscription", return_value=fake_sub):+ await connection_manager._start_notification_subscription(+ mock_websocket, user_id="user-1"+ )+ # On failure the sub must NOT be stored (bookkeeping clean).+ assert mock_websocket not in connection_manager._ws_notifications+++async def test_forward_notification_delivers_to_owning_websocket(+ connection_manager: ConnectionManager, mock_websocket: AsyncMock+) -> None:+ """The notification wrapper → WSMessage round-trip must reach the owning WS."""+ inner = {+ "user_id": "user-1",+ "payload": {"type": "info", "event": "hi"},+ }+ wrapper = json.dumps({"payload": inner})++ await connection_manager._forward_notification(+ mock_websocket, "user-1", wrapper.encode())- await connection_manager.send_execution_update(result)+ mock_websocket.send_text.assert_called_once()+ sent = json.loads(mock_websocket.send_text.call_args[0][0])+ assert sent["method"] == WSMethod.NOTIFICATION.value+ assert sent["data"] == {"type": "info", "event": "hi"}+++async def test_forward_notification_rejects_cross_user_payload(+ connection_manager: ConnectionManager, mock_websocket: AsyncMock+) -> None:+ """Defense in depth: a payload for a different user must be dropped."""+ inner = {+ "user_id": "user-2",+ "payload": {"type": "info", "event": "sneaky"},+ }+ wrapper = json.dumps({"payload": inner})++ await connection_manager._forward_notification(+ mock_websocket, "user-1", wrapper.encode()+ )mock_websocket.send_text.assert_not_called()@pytest.mark.asyncio-async def test_send_execution_result_no_subscribers(+async def test_forward_notification_noop_on_none(connection_manager: ConnectionManager, mock_websocket: AsyncMock) -> None:- result = NodeExecutionEvent(- user_id="user-1",- graph_id="test_graph",- graph_version=1,- graph_exec_id="test_exec_id",- node_exec_id="test_node_exec_id",- node_id="test_node_id",- block_id="test_block_id",- status=ExecutionStatus.COMPLETED,- input_data={"input1": "value1"},- output_data={"output1": ["result1"]},- add_time=datetime.now(),- queue_time=None,- start_time=datetime.now(),- end_time=datetime.now(),+ await connection_manager._forward_notification(mock_websocket, "user-1", None)+ mock_websocket.send_text.assert_not_called()+++async def test_forward_notification_swallows_bad_envelope(+ connection_manager: ConnectionManager, mock_websocket: AsyncMock+) -> None:+ """Malformed notification payload must not blow up or send anything."""+ await connection_manager._forward_notification(+ mock_websocket, "user-1", b"not json")+ mock_websocket.send_text.assert_not_called()- await connection_manager.send_execution_update(result)+async def test_forward_notification_drops_non_dict_inner(+ connection_manager: ConnectionManager, mock_websocket: AsyncMock+) -> None:+ wrapper = json.dumps({"payload": "not-a-dict"})+ await connection_manager._forward_notification(+ mock_websocket, "user-1", wrapper.encode()+ )mock_websocket.send_text.assert_not_called()@pytest.mark.asyncio-async def test_send_notification(+async def test_forward_notification_swallows_websocket_send_failure(connection_manager: ConnectionManager, mock_websocket: AsyncMock) -> None:- connection_manager.user_connections["user-1"] = {mock_websocket}+ """A flaky websocket must not propagate the failure to the pump."""+ mock_websocket.send_text = AsyncMock(side_effect=RuntimeError("ws closed"))+ inner = {+ "user_id": "user-1",+ "payload": {"type": "info", "event": "hi"},+ }+ wrapper = json.dumps({"payload": inner})+ # Must not raise.+ await connection_manager._forward_notification(+ mock_websocket, "user-1", wrapper.encode()+ )+- await connection_manager.send_notification(- user_id="user-1", payload=NotificationPayload(type="info", event="hey")+async def test_forward_notification_logs_ws_close_race_at_debug(+ connection_manager: ConnectionManager, mock_websocket: AsyncMock+) -> None:+ """SPUBLISH racing WS close on notification path → debug, not warning."""+ mock_websocket.send_text = AsyncMock(+ side_effect=RuntimeError(+ 'Cannot call "send" once a close message has been sent.'+ ))+ inner = {+ "user_id": "user-1",+ "payload": {"type": "info", "event": "hi"},+ }+ wrapper = json.dumps({"payload": inner})+ with patch("backend.api.conn_manager.logger") as mock_logger:+ await connection_manager._forward_notification(+ mock_websocket, "user-1", wrapper.encode()+ )+ mock_logger.warning.assert_not_called()+ mock_logger.debug.assert_called_once()+ assert "Dropped notification on closed WS" in mock_logger.debug.call_args[0][0]- mock_websocket.send_text.assert_called_once()- sent_message = mock_websocket.send_text.call_args[0][0]- expected_message = WSMessage(- method=WSMethod.NOTIFICATION,- data={"type": "info", "event": "hey"},- ).model_dump_json()- assert sent_message == expected_message++async def test_connect_socket_starts_notification_subscription(+ connection_manager: ConnectionManager, mock_websocket: AsyncMock+) -> None:+ """connect_socket must call _start_notification_subscription once."""+ with patch.object(+ connection_manager, "_start_notification_subscription", AsyncMock()+ ) as mock_start:+ await connection_manager.connect_socket(mock_websocket, user_id="user-1")+ mock_start.assert_awaited_once_with(mock_websocket, user_id="user-1")diff --git a/autogpt_platform/backend/backend/api/features/chat/routes.py b/autogpt_platform/backend/backend/api/features/chat/routes.pyindex ae208d1c0f..7d1bf9c00b 100644--- a/autogpt_platform/backend/backend/api/features/chat/routes.py+++ b/autogpt_platform/backend/backend/api/features/chat/routes.py@@ -320,8 +320,11 @@ async def list_sessions(redis = await get_redis_async()pipe = redis.pipeline(transaction=False)for session in sessions:+ # Use the canonical helper so the hash-tag braces match every+ # other writer; building the key inline drops the braces and+ # silently misses every running session on cluster mode.pipe.hget(- f"{config.session_meta_prefix}{session.session_id}",+ stream_registry.get_session_meta_key(session.session_id),"status",)statuses = await pipe.execute()diff --git a/autogpt_platform/backend/backend/api/rest_api.py b/autogpt_platform/backend/backend/api/rest_api.pyindex abe261b725..87190410e8 100644--- a/autogpt_platform/backend/backend/api/rest_api.py+++ b/autogpt_platform/backend/backend/api/rest_api.py@@ -41,6 +41,7 @@ import backend.api.features.workspace.routes as workspace_routesimport backend.data.blockimport backend.data.dbimport backend.data.graph+import backend.data.redis_clientimport backend.data.userimport backend.integrations.webhooks.utilsimport backend.util.service@@ -95,6 +96,8 @@ async def lifespan_context(app: fastapi.FastAPI):verify_auth_settings()await backend.data.db.connect()+ # Eager connect to fail-fast if Redis is unreachable.+ await backend.data.redis_client.get_redis_async()# Configure thread pool for FastAPI sync operation performance# CRITICAL: FastAPI automatically runs ALL sync functions in this thread pool:@@ -146,7 +149,18 @@ async def lifespan_context(app: fastapi.FastAPI):except Exception as e:logger.warning(f"Error shutting down workspace storage: {e}")- await backend.data.db.disconnect()+ # Each cleanup is wrapped so one failure doesn't block the rest. The+ # Redis close in particular silences asyncio's "Unclosed ClusterNode"+ # GC warning at interpreter shutdown.+ try:+ await backend.data.redis_client.disconnect_async()+ except Exception:+ logger.warning("redis_client.disconnect_async failed", exc_info=True)++ try:+ await backend.data.db.disconnect()+ except Exception:+ logger.warning("db.disconnect failed", exc_info=True)def custom_generate_unique_id(route: APIRoute):diff --git a/autogpt_platform/backend/backend/api/ws_api.py b/autogpt_platform/backend/backend/api/ws_api.pyindex e254d4b4db..196b021d27 100644--- a/autogpt_platform/backend/backend/api/ws_api.py+++ b/autogpt_platform/backend/backend/api/ws_api.py@@ -1,4 +1,3 @@-import asyncioimport loggingfrom contextlib import asynccontextmanagerfrom typing import Protocol@@ -17,14 +16,12 @@ from backend.api.model import (WSSubscribeGraphExecutionsRequest,)from backend.api.utils.cors import build_cors_params-from backend.data.execution import AsyncRedisExecutionEventBus-from backend.data.notification_bus import AsyncRedisNotificationEventBus+from backend.data import db, redis_clientfrom backend.data.user import DEFAULT_USER_IDfrom backend.monitoring.instrumentation import (instrument_fastapi,update_websocket_connections,)-from backend.util.retry import continuous_retryfrom backend.util.service import AppProcessfrom backend.util.settings import AppEnvironment, Config, Settings@@ -34,10 +31,24 @@ settings = Settings()@asynccontextmanagerasync def lifespan(app: FastAPI):- manager = get_connection_manager()- fut = asyncio.create_task(event_broadcaster(manager))- fut.add_done_callback(lambda _: logger.info("Event broadcaster stopped"))- yield+ # Prisma is needed to resolve graph_id from graph_exec_id on subscribe.+ await db.connect()+ # Eager connect to fail-fast if Redis is unreachable.+ await redis_client.get_redis_async()+ try:+ yield+ finally:+ # Each cleanup is wrapped so one failure doesn't block the rest. The+ # Redis close silences asyncio's "Unclosed ClusterNode" GC warning at+ # interpreter shutdown.+ try:+ await redis_client.disconnect_async()+ except Exception:+ logger.warning("redis_client.disconnect_async failed", exc_info=True)+ try:+ await db.disconnect()+ except Exception:+ logger.warning("db.disconnect failed", exc_info=True)docs_url = "/docs" if settings.config.app_env == AppEnvironment.LOCAL else None@@ -61,31 +72,6 @@ def get_connection_manager():return _connection_manager-@continuous_retry()-async def event_broadcaster(manager: ConnectionManager):- execution_bus = AsyncRedisExecutionEventBus()- notification_bus = AsyncRedisNotificationEventBus()-- try:-- async def execution_worker():- async for event in execution_bus.listen("*"):- await manager.send_execution_update(event)-- async def notification_worker():- async for notification in notification_bus.listen("*"):- await manager.send_notification(- user_id=notification.user_id,- payload=notification.payload,- )-- await asyncio.gather(execution_worker(), notification_worker())- finally:- # Ensure PubSub connections are closed on any exit to prevent leaks- await execution_bus.close()- await notification_bus.close()--async def authenticate_websocket(websocket: WebSocket) -> str:if not settings.config.enable_auth:return DEFAULT_USER_ID@@ -297,6 +283,21 @@ async def websocket_router().model_dump_json())continue+ except ValueError as e:+ logger.warning(+ "Subscription rejected for user #%s on '%s': %s",+ user_id,+ message.method.value,+ e,+ )+ await websocket.send_text(+ WSMessage(+ method=WSMethod.ERROR,+ success=False,+ error=str(e),+ ).model_dump_json()+ )+ continueexcept Exception as e:logger.error(f"Error while handling '{message.method.value}' message "@@ -321,9 +322,13 @@ async def websocket_router()except WebSocketDisconnect:- manager.disconnect_socket(websocket, user_id=user_id)logger.debug("WebSocket client disconnected")+ except Exception:+ logger.exception(f"Unexpected error in websocket_router for user #{user_id}")finally:+ # Always release subscription pumps + Redis connections, regardless of how+ # the loop exited — otherwise non-WebSocketDisconnect failures leak both.+ await manager.disconnect_socket(websocket, user_id=user_id)update_websocket_connections(user_id, -1)diff --git a/autogpt_platform/backend/backend/api/ws_api_test.py b/autogpt_platform/backend/backend/api/ws_api_test.pyindex edab1bbded..ccfa76088c 100644--- a/autogpt_platform/backend/backend/api/ws_api_test.py+++ b/autogpt_platform/backend/backend/api/ws_api_test.py@@ -44,9 +44,12 @@ def test_websocket_server_uses_cors_helper(mocker) -> None:"backend.api.ws_api.build_cors_params", return_value=cors_params)- with override_config(- settings, "backend_cors_allow_origins", cors_params["allow_origins"]- ), override_config(settings, "app_env", AppEnvironment.LOCAL):+ with (+ override_config(+ settings, "backend_cors_allow_origins", cors_params["allow_origins"]+ ),+ override_config(settings, "app_env", AppEnvironment.LOCAL),+ ):WebsocketServer().run()build_cors.assert_called_once_with(@@ -65,9 +68,12 @@ def test_websocket_server_uses_cors_helper(mocker) -> None:def test_websocket_server_blocks_localhost_in_production(mocker) -> None:mocker.patch("backend.api.ws_api.uvicorn.run")- with override_config(- settings, "backend_cors_allow_origins", ["http://localhost:3000"]- ), override_config(settings, "app_env", AppEnvironment.PRODUCTION):+ with (+ override_config(+ settings, "backend_cors_allow_origins", ["http://localhost:3000"]+ ),+ override_config(settings, "app_env", AppEnvironment.PRODUCTION),+ ):with pytest.raises(ValueError):WebsocketServer().run()@@ -290,7 +296,232 @@ async def test_handle_unsubscribe_missing_data(message=message,)- mock_manager._unsubscribe.assert_not_called()+ mock_manager.unsubscribe_graph_exec.assert_not_called()mock_websocket.send_text.assert_called_once()assert '"method":"error"' in mock_websocket.send_text.call_args[0][0]assert '"success":false' in mock_websocket.send_text.call_args[0][0]+++# ---------- Per-graph subscribe branch ----------+++async def test_handle_subscribe_graph_execs_branch(+ mock_websocket: AsyncMock, mock_manager: AsyncMock+) -> None:+ """The SUBSCRIBE_GRAPH_EXECS branch must route to subscribe_graph_execs,+ not subscribe_graph_exec — regression guard for the aggregate channel."""+ message = WSMessage(+ method=WSMethod.SUBSCRIBE_GRAPH_EXECS,+ data={"graph_id": "graph-abc"},+ )+ mock_manager.subscribe_graph_execs.return_value = (+ "user-1|graph#graph-abc|executions"+ )++ await handle_subscribe(+ connection_manager=cast(ConnectionManager, mock_manager),+ websocket=cast(WebSocket, mock_websocket),+ user_id="user-1",+ message=message,+ )++ mock_manager.subscribe_graph_execs.assert_called_once_with(+ user_id="user-1",+ graph_id="graph-abc",+ websocket=mock_websocket,+ )+ mock_manager.subscribe_graph_exec.assert_not_called()+ mock_websocket.send_text.assert_called_once()+ assert (+ '"method":"subscribe_graph_executions"'+ in mock_websocket.send_text.call_args[0][0]+ )+ assert '"success":true' in mock_websocket.send_text.call_args[0][0]+++async def test_handle_subscribe_rejects_unrelated_method(+ mock_websocket: AsyncMock, mock_manager: AsyncMock+) -> None:+ """handle_subscribe must raise for methods that aren't SUBSCRIBE_*."""+ import pytest as _pytest++ message = WSMessage(+ method=WSMethod.HEARTBEAT,+ data={"graph_exec_id": "x"},+ )++ with _pytest.raises(ValueError):+ await handle_subscribe(+ connection_manager=cast(ConnectionManager, mock_manager),+ websocket=cast(WebSocket, mock_websocket),+ user_id="user-1",+ message=message,+ )+++# ---------- authenticate_websocket branches ----------+++async def test_authenticate_websocket_missing_token_closes_4001(mocker) -> None:+ from backend.api.ws_api import authenticate_websocket++ mocker.patch.object(settings.config, "enable_auth", True)+ ws = AsyncMock(spec=WebSocket)+ ws.query_params = {}++ user_id = await authenticate_websocket(ws)++ ws.close.assert_awaited_once()+ assert ws.close.call_args.kwargs["code"] == 4001+ assert user_id == ""+++async def test_authenticate_websocket_invalid_token_closes_4003(mocker) -> None:+ from backend.api.ws_api import authenticate_websocket++ mocker.patch.object(settings.config, "enable_auth", True)+ mocker.patch(+ "backend.api.ws_api.parse_jwt_token", side_effect=ValueError("bad token")+ )+ ws = AsyncMock(spec=WebSocket)+ ws.query_params = {"token": "abc"}++ user_id = await authenticate_websocket(ws)++ ws.close.assert_awaited_once()+ assert ws.close.call_args.kwargs["code"] == 4003+ assert user_id == ""+++async def test_authenticate_websocket_missing_sub_closes_4002(mocker) -> None:+ from backend.api.ws_api import authenticate_websocket++ mocker.patch.object(settings.config, "enable_auth", True)+ mocker.patch("backend.api.ws_api.parse_jwt_token", return_value={"not_sub": "x"})+ ws = AsyncMock(spec=WebSocket)+ ws.query_params = {"token": "abc"}++ user_id = await authenticate_websocket(ws)++ ws.close.assert_awaited_once()+ assert ws.close.call_args.kwargs["code"] == 4002+ assert user_id == ""+++async def test_authenticate_websocket_happy_path_returns_sub(mocker) -> None:+ from backend.api.ws_api import authenticate_websocket++ mocker.patch.object(settings.config, "enable_auth", True)+ mocker.patch("backend.api.ws_api.parse_jwt_token", return_value={"sub": "user-X"})+ ws = AsyncMock(spec=WebSocket)+ ws.query_params = {"token": "abc"}++ user_id = await authenticate_websocket(ws)++ assert user_id == "user-X"+++async def test_authenticate_websocket_auth_disabled_returns_default(mocker) -> None:+ from backend.api.ws_api import authenticate_websocket++ mocker.patch.object(settings.config, "enable_auth", False)+ ws = AsyncMock(spec=WebSocket)+ ws.query_params = {}++ user_id = await authenticate_websocket(ws)++ assert user_id == DEFAULT_USER_ID+++# ---------- get_connection_manager singleton ----------+++def test_get_connection_manager_singleton() -> None:+ """Repeated calls must return the same ConnectionManager — the WS router+ depends on a single process-wide subscription table."""+ import backend.api.ws_api as ws_api++ ws_api._connection_manager = None+ a = ws_api.get_connection_manager()+ b = ws_api.get_connection_manager()+ assert a is b+ assert isinstance(a, ConnectionManager)+++# ---------- Lifespan: Prisma connect/disconnect ----------+++async def test_lifespan_connects_and_disconnects_prisma(mocker) -> None:+ """Lifespan must both connect() and disconnect() db — the subscribe path+ resolves graph_id via Prisma so a missing connect() is the regression bug."""+ from fastapi import FastAPI++ from backend.api.ws_api import lifespan++ mock_db = mocker.patch("backend.api.ws_api.db")+ mock_db.connect = AsyncMock()+ mock_db.disconnect = AsyncMock()++ dummy_app = FastAPI()+ async with lifespan(dummy_app):+ mock_db.connect.assert_awaited_once()+ mock_db.disconnect.assert_not_called()+ mock_db.disconnect.assert_awaited_once()+++async def test_lifespan_still_disconnects_on_exception(mocker) -> None:+ """If the app raises inside the yield, Prisma must still disconnect."""+ from fastapi import FastAPI++ from backend.api.ws_api import lifespan++ mock_db = mocker.patch("backend.api.ws_api.db")+ mock_db.connect = AsyncMock()+ mock_db.disconnect = AsyncMock()++ dummy_app = FastAPI()++ class _Boom(Exception):+ pass++ with pytest.raises(_Boom):+ async with lifespan(dummy_app):+ raise _Boom()++ mock_db.disconnect.assert_awaited_once()+++# ---------- Health endpoint ----------+++def test_health_endpoint_returns_ok() -> None:+ # TestClient triggers lifespan — stub it out so Prisma isn't hit.+ from contextlib import asynccontextmanager++ from fastapi.testclient import TestClient++ import backend.api.ws_api as ws_api++ @asynccontextmanager+ async def _noop_lifespan(app):+ yield++ # Replace the app-level lifespan temporarily.+ real_router_lifespan = ws_api.app.router.lifespan_context+ ws_api.app.router.lifespan_context = _noop_lifespan+ try:+ with TestClient(ws_api.app) as client:+ r = client.get("/")+ assert r.status_code == 200+ assert r.json() == {"status": "healthy"}+ finally:+ ws_api.app.router.lifespan_context = real_router_lifespandiff --git a/autogpt_platform/backend/backend/copilot/executor/utils.py b/autogpt_platform/backend/backend/copilot/executor/utils.pyindex de1681b55c..f386d90adf 100644--- a/autogpt_platform/backend/backend/copilot/executor/utils.py+++ b/autogpt_platform/backend/backend/copilot/executor/utils.py@@ -71,7 +71,9 @@ COPILOT_EXECUTION_EXCHANGE = Exchange(durable=True,auto_delete=False,)-COPILOT_EXECUTION_QUEUE_NAME = "copilot_execution_queue"+# ``_v2`` suffix marks the classic→quorum rollover; old-image consumers+# drain the unsuffixed queue. Orphans cleaned up in a follow-up PR.+COPILOT_EXECUTION_QUEUE_NAME = "copilot_execution_queue_v2"COPILOT_EXECUTION_ROUTING_KEY = "copilot.run"COPILOT_CANCEL_EXCHANGE = Exchange(@@ -80,7 +82,7 @@ COPILOT_CANCEL_EXCHANGE = Exchange(durable=True,auto_delete=False,)-COPILOT_CANCEL_QUEUE_NAME = "copilot_cancel_queue"+COPILOT_CANCEL_QUEUE_NAME = "copilot_cancel_queue_v2"def get_session_lock_key(session_id: str) -> str:@@ -118,6 +120,9 @@ def create_copilot_queue_config() -> RabbitMQConfig:durable=True,auto_delete=False,arguments={+ # Quorum (not classic mirrored) for leader election + stronger+ # replication across RabbitMQ 4.x cluster nodes.+ "x-queue-type": "quorum",# Consumer timeout matches the pod graceful-shutdown window so a# rolling deploy never forces redelivery of a turn that the pod# is still legitimately finishing.@@ -131,7 +136,7 @@ def create_copilot_queue_config() -> RabbitMQConfig:# limit), apply a policy:## rabbitmqctl set_policy copilot-consumer-timeout \- # "^copilot_execution_queue$" \+ # "^copilot_execution_queue_v2$" \# '{"consumer-timeout": 21600000}' \# --apply-to queues#@@ -139,8 +144,7 @@ def create_copilot_queue_config() -> RabbitMQConfig:# to match the code's value the policy is redundant for new# pods and can be removed after a stable deploy if desired —# but it's harmless to leave in place.- "x-consumer-timeout": COPILOT_CONSUMER_TIMEOUT_SECONDS- * 1000,+ "x-consumer-timeout": COPILOT_CONSUMER_TIMEOUT_SECONDS * 1000,},)cancel_queue = Queue(@@ -149,6 +153,7 @@ def create_copilot_queue_config() -> RabbitMQConfig:routing_key="", # not used for FANOUTdurable=True,auto_delete=False,+ arguments={"x-queue-type": "quorum"},)return RabbitMQConfig(vhost="/",diff --git a/autogpt_platform/backend/backend/copilot/graphiti/falkordb_driver_test.py b/autogpt_platform/backend/backend/copilot/graphiti/falkordb_driver_test.pyindex fc78765f16..f6022092c5 100644--- a/autogpt_platform/backend/backend/copilot/graphiti/falkordb_driver_test.py+++ b/autogpt_platform/backend/backend/copilot/graphiti/falkordb_driver_test.py@@ -1,9 +1,22 @@+from unittest.mock import MagicMock++import pytest+from .falkordb_driver import AutoGPTFalkorDriver-def test_build_fulltext_query_uses_unquoted_group_ids_for_falkordb() -> None:- driver = AutoGPTFalkorDriver()+def driver() -> AutoGPTFalkorDriver:+ # ``build_fulltext_query`` is a pure string-builder that never touches+ # the FalkorDB client; injecting a mock avoids the eager Redis probe+ # that the upstream ``FalkorDriver.__init__`` runs against+ # ``localhost:6379``.+ return AutoGPTFalkorDriver(falkor_db=MagicMock())++def test_build_fulltext_query_uses_unquoted_group_ids_for_falkordb(+ driver: AutoGPTFalkorDriver,+) -> None:query = driver.build_fulltext_query("Sarah",group_ids=["user_883cc9da-fe37-4863-839b-acba022bf3ef"],@@ -13,18 +26,18 @@ def test_build_fulltext_query_uses_unquoted_group_ids_for_falkordb() -> None:assert '"user_883cc9da-fe37-4863-839b-acba022bf3ef"' not in query-def test_build_fulltext_query_joins_multiple_group_ids_with_or() -> None:- driver = AutoGPTFalkorDriver()-+def test_build_fulltext_query_joins_multiple_group_ids_with_or(+ driver: AutoGPTFalkorDriver,+) -> None:query = driver.build_fulltext_query("Sarah", group_ids=["user_a", "user_b"])assert query == "(@group_id:user_a|user_b) (Sarah)"-def test_stopwords_only_query_returns_group_filter_only() -> None:+def test_stopwords_only_query_returns_group_filter_only(+ driver: AutoGPTFalkorDriver,+) -> None:"""Line 25: sanitized_query is empty (all stopwords) but group_ids present."""- driver = AutoGPTFalkorDriver()-# "the" is a common stopword — the query should reduce to just the group filter.query = driver.build_fulltext_query("the",@@ -34,10 +47,10 @@ def test_stopwords_only_query_returns_group_filter_only() -> None:assert query == "(@group_id:user_abc)"-def test_query_without_group_ids_returns_parenthesized_query() -> None:+def test_query_without_group_ids_returns_parenthesized_query(+ driver: AutoGPTFalkorDriver,+) -> None:"""Line 27: sanitized_query has content but no group_ids provided."""- driver = AutoGPTFalkorDriver()-query = driver.build_fulltext_query("Sarah", group_ids=None)assert query == "(Sarah)"diff --git a/autogpt_platform/backend/backend/copilot/pending_messages.py b/autogpt_platform/backend/backend/copilot/pending_messages.pyindex 089e45644b..0bb546c510 100644--- a/autogpt_platform/backend/backend/copilot/pending_messages.py+++ b/autogpt_platform/backend/backend/copilot/pending_messages.py@@ -33,28 +33,17 @@ from backend.data.redis_helpers import capped_rpush, capped_rpush_if_hash_fieldlogger = logging.getLogger(__name__)-# Per-session cap. Higher values risk a runaway consumer; lower values-# risk dropping user input under heavy typing. 10 was chosen as a-# reasonable ceiling — a user typing faster than the copilot can drain-# between tool rounds is already an unusual usage pattern.+# Per-session cap; typing faster than the copilot drains is already unusual.MAX_PENDING_MESSAGES = 10-# Redis key + TTL. The buffer is ephemeral: if a turn completes or the-# executor dies, the pending messages should either have been drained-# already or are safe to drop (the user can resend).+# Ephemeral buffer: undrained messages are safe to drop at TTL expiry._PENDING_KEY_PREFIX = "copilot:pending:"_PENDING_CHANNEL_PREFIX = "copilot:pending:notify:"_PENDING_TTL_SECONDS = 3600 # 1 hour — matches stream_ttl default-# Secondary queue that carries drained-but-awaiting-persist PendingMessages-# from the MCP tool wrapper (which drains the primary buffer and injects-# into tool output for the LLM) to sdk/service.py's _dispatch_response-# handler for StreamToolOutputAvailable, which pops and persists them as a-# separate user row chronologically after the tool_result row. This is the-# hand-off between "Claude saw the follow-up mid-turn" (wrapper) and "UI-# renders a user bubble for it" (service). Rollback path re-queues into-# the PRIMARY buffer so the next turn-start drain picks them up if the-# user-row persist fails.+# Secondary queue: carries drained-but-awaiting-persist PendingMessages from+# the tool wrapper (which injects them into tool output) to sdk/service.py+# (which persists a user row after the tool_result row)._PERSIST_QUEUE_KEY_PREFIX = "copilot:pending-persist:"# Payload sent on the pub/sub notify channel. Subscribers treat any@@ -65,13 +54,8 @@ _NOTIFY_PAYLOAD = "1"class PendingMessageContext(BaseModel):"""Structured page context attached to a pending message.- Default ``extra='ignore'`` (pydantic's default): unknown keys from- the loose HTTP-level ``StreamChatRequest.context: dict[str, str]``- are silently dropped rather than raising ``ValidationError`` on- forward-compat additions. The strict ``extra='forbid'`` mode was- removed after sentry r3105553772 — strict validation at this- boundary only added a 500 footgun; the upstream request model is- already schemaless so strict mode protects nothing.+ Unknown keys are silently dropped: the upstream request model is+ ``dict[str, str]``, so strict validation here only adds 500 footguns."""url: str | None = Field(default=None, max_length=2_000)@@ -84,19 +68,16 @@ class PendingMessage(BaseModel):content: str = Field(min_length=1, max_length=32_000)file_ids: list[str] = Field(default_factory=list, max_length=20)context: PendingMessageContext | None = None- # Wall-clock time (unix seconds, float) the message was queued by the- # user. Used by the turn-start drain to order pending relative to the- # turn's ``current`` message: items typed *before* the current's- # /stream arrival go ahead of it; items typed *after* (race path,- # queued while the /stream HTTP request was still processing) go- # after. Defaults to 0.0 for backward compatibility with entries- # written before this field existed — those sort as "before everything"- # which matches the pre-fix behaviour.+ # Enqueue time (unix seconds) so the turn-start drain can order pending+ # messages relative to the turn's ``current`` message.enqueued_at: float = Field(default_factory=time.time)def _buffer_key(session_id: str) -> str:- return f"{_PENDING_KEY_PREFIX}{session_id}"+ # Hash-tag braces colocate this key with stream_registry's session-meta key+ # on the same Redis Cluster slot, which the gated-rpush Lua script needs+ # (multi-key scripts return CROSSSLOT when KEYS hash to different slots).+ return f"{_PENDING_KEY_PREFIX}{{{session_id}}}"def _notify_channel(session_id: str) -> str:@@ -104,12 +85,7 @@ def _notify_channel(session_id: str) -> str:def _decode_redis_item(item: Any) -> str:- """Decode a redis-py list item to a str.-- redis-py returns ``bytes`` when ``decode_responses=False`` and ``str``- when ``decode_responses=True``. This helper handles both so callers- don't have to repeat the isinstance guard.- """+ """Decode a redis-py list item to str (handles ``bytes`` and ``str``)."""return item.decode("utf-8") if isinstance(item, bytes) else str(item)@@ -117,22 +93,11 @@ async def push_pending_message(session_id: str,message: PendingMessage,) -> int:- """Append a pending message to the session's buffer.-- Returns the new buffer length. Enforces ``MAX_PENDING_MESSAGES`` by- trimming from the left (oldest) — the newest message always wins if- the user has been typing faster than the copilot can drain.-- Delegates to :func:`backend.data.redis_helpers.capped_rpush` so RPUSH- + LTRIM + EXPIRE + LLEN run atomically (MULTI/EXEC) in one round- trip; a concurrent drain (LPOP) can no longer observe the list- temporarily over ``MAX_PENDING_MESSAGES``.-- Note on durability: if the executor turn crashes after a push but before- the drain window runs, the message remains in Redis until the TTL expires- (``_PENDING_TTL_SECONDS``, currently 1 hour). It is delivered on the- next turn that drains the buffer. If no turn runs within the TTL the- message is silently dropped; the user may resend it.+ """Append a pending message to the session's buffer, capped at+ ``MAX_PENDING_MESSAGES`` (oldest trimmed). Returns the new buffer length.++ The buffer survives consumer crashes until ``_PENDING_TTL_SECONDS``+ expires; messages not drained within that window are dropped."""redis = await get_redis_async()key = _buffer_key(session_id)@@ -146,10 +111,14 @@ async def push_pending_message(ttl_seconds=_PENDING_TTL_SECONDS,)- # Fire-and-forget notify. Subscribers use this as a wake-up hint;- # the buffer itself is authoritative so a lost notify is harmless.+ # Fire-and-forget wake-up hint via sharded pub/sub (SPUBLISH routes to+ # one shard vs classic PUBLISH's cluster-bus broadcast). Use+ # execute_command because redis-py 6.x AsyncRedisCluster has no+ # spublish() wrapper.try:- await redis.publish(_notify_channel(session_id), _NOTIFY_PAYLOAD)+ await redis.execute_command(+ "SPUBLISH", _notify_channel(session_id), _NOTIFY_PAYLOAD+ )except Exception as e: # pragma: no coverlogger.warning("pending_messages: publish failed for %s: %s", session_id, e)@@ -189,8 +158,12 @@ async def push_pending_message_if_session_running()return None+ # Match push_pending_message: SPUBLISH via execute_command so it works on+ # both Redis and AsyncRedisCluster (the cluster client has no publish()).try:- await redis.publish(_notify_channel(session_id), _NOTIFY_PAYLOAD)+ await redis.execute_command(+ "SPUBLISH", _notify_channel(session_id), _NOTIFY_PAYLOAD+ )except Exception as e: # pragma: no coverlogger.warning("pending_messages: publish failed for %s: %s", session_id, e)@@ -212,13 +185,8 @@ async def drain_pending_messages(session_id: str) -> list[PendingMessage]:redis = await get_redis_async()key = _buffer_key(session_id)- # Redis LPOP with count (Redis 6.2+) returns None for missing key,- # empty list if we somehow race an empty key, or the popped items.- # Draining MAX_PENDING_MESSAGES at once is safe because the push side- # uses RPUSH + LTRIM(-MAX_PENDING_MESSAGES, -1) to cap the list to that- # same value, so the list can never hold more items than we drain here.- # If the cap is raised on the push side, raise the drain count here too- # (or switch to a loop drain).+ # LPOP with count drains everything in one round-trip; the push side+ # caps the list at MAX_PENDING_MESSAGES so nothing is left behind.lpop_result = await redis.lpop(key, MAX_PENDING_MESSAGES) # type: ignore[assignment]if not lpop_result:return []@@ -282,24 +250,17 @@ async def peek_pending_messages(session_id: str) -> list[PendingMessage]:async def clear_pending_messages_unsafe(session_id: str) -> None:- """Drop the session's pending buffer — **not** the normal turn cleanup.-- The ``_unsafe`` suffix warns: reaching for this at turn end drops queued- follow-ups on the floor instead of running them (the bug fixed by commit- b64be73). The atomic ``LPOP`` drain at turn start is the primary consumer;- anything pushed after the drain window belongs to the next turn by- definition. Retained only as an operator/debug escape hatch for manually- clearing a stuck session and as a fixture in the unit tests.+ """Drop the session's pending buffer — operator/debug escape hatch.++ The ``_unsafe`` suffix warns: normal turn cleanup uses the atomic LPOP+ drain; this bypass drops queued follow-ups on the floor."""redis = await get_redis_async()await redis.delete(_buffer_key(session_id))-# Per-message and total-block caps for inline tool-boundary injection.-# Per-message keeps a single long paste from dominating; the total cap-# keeps the follow-up block small relative to the 100 KB MCP truncation-# boundary so tool output always stays the larger share of the wrapper-# return value.+# Per-message + total caps keep the follow-up block bounded relative to the+# 100 KB MCP tool-output truncation boundary._FOLLOWUP_CONTENT_MAX_CHARS = 2_000_FOLLOWUP_TOTAL_MAX_CHARS = 6_000@@ -314,17 +275,9 @@ async def stash_pending_for_persist() -> None:"""Enqueue drained PendingMessages for UI-row persistence.- Writes each message as a JSON payload to- ``copilot:pending-persist:{session_id}``. The SDK service's- tool-result dispatch handler LPOPs this queue right after appending- the tool_result row to ``session.messages``, so the resulting user- row lands at the correct chronological position (after the tool- output the follow-up was drained against).-- Fire-and-forget on Redis failures: a stash failure means Claude- still saw the follow-up in tool output (the injection step ran- first), so the only consequence is a missing UI bubble. Logged- so it can be spotted.+ The SDK service LPOPs this right after appending the tool_result row so+ the user bubble lands after the tool output. Stash failures are logged+ but not raised — the only consequence is a missing UI bubble."""if not messages:return@@ -377,8 +330,7 @@ async def drain_pending_for_persist(session_id: str) -> list[PendingMessage]:)except (json.JSONDecodeError, ValidationError, TypeError, ValueError) as e:logger.warning(- "pending_messages: dropping malformed persist-queue entry "- "for %s: %s",+ "pending_messages: dropping malformed persist-queue entry for %s: %s",session_id,e,)diff --git a/autogpt_platform/backend/backend/copilot/pending_messages_test.py b/autogpt_platform/backend/backend/copilot/pending_messages_test.pyindex c997d7d9cf..5447c758c2 100644--- a/autogpt_platform/backend/backend/copilot/pending_messages_test.py+++ b/autogpt_platform/backend/backend/copilot/pending_messages_test.py@@ -60,6 +60,16 @@ class _FakeRedis:self.published.append((channel, payload))return 1+ async def execute_command(self, *args: Any) -> Any:+ # Minimal handler for the sharded SPUBLISH call made by+ # push_pending_message. Routing semantics are irrelevant here —+ # we just record the publish for assertion.+ if args and args[0] == "SPUBLISH":+ _, channel, payload = args+ self.published.append((channel, payload))+ return 1+ raise NotImplementedError(f"fake execute_command does not handle {args[0]!r}")+async def lpop(self, key: str, count: int) -> list[str | bytes] | None:lst = self.lists.get(key)if not lst:@@ -326,7 +336,7 @@ async def test_drain_skips_malformed_entries(fake_redis: _FakeRedis,) -> None:# Seed the fake with a mix of valid and malformed payloads- fake_redis.lists["copilot:pending:bad"] = [+ fake_redis.lists[pm_module._buffer_key("bad")] = [json.dumps({"content": "valid"}),"{not valid json",json.dumps({"content": "also valid", "file_ids": ["a"]}),@@ -347,7 +357,7 @@ async def test_drain_decodes_bytes_payloads(branch in ``drain_pending_messages`` so a regression there doesn'tslip past CI."""- fake_redis.lists["copilot:pending:bytes_sess"] = [+ fake_redis.lists[pm_module._buffer_key("bytes_sess")] = [json.dumps({"content": "from bytes"}).encode("utf-8"),]drained = await drain_pending_messages("bytes_sess")@@ -362,14 +372,14 @@ async def test_peek_decodes_bytes_payloads("""``peek_pending_messages`` uses the same ``_decode_redis_item`` helperas the drain path. Seed with bytes to guard against regression."""- fake_redis.lists["copilot:pending:peek_bytes_sess"] = [+ fake_redis.lists[pm_module._buffer_key("peek_bytes_sess")] = [json.dumps({"content": "peeked from bytes"}).encode("utf-8"),]peeked = await peek_pending_messages("peek_bytes_sess")assert len(peeked) == 1assert peeked[0].content == "peeked from bytes"# peek must NOT consume the item- assert fake_redis.lists["copilot:pending:peek_bytes_sess"] != []+ assert fake_redis.lists[pm_module._buffer_key("peek_bytes_sess")] != []# ── Concurrency ─────────────────────────────────────────────────────@@ -445,7 +455,7 @@ async def test_peek_pending_messages_decodes_bytes_payloads(fake_redis: _FakeRedis,) -> None:"""peek_pending_messages decodes bytes entries the same way drain does."""- fake_redis.lists["copilot:pending:peek_bytes"] = [+ fake_redis.lists[pm_module._buffer_key("peek_bytes")] = [json.dumps({"content": "from bytes"}).encode("utf-8"),]peeked = await peek_pending_messages("peek_bytes")@@ -458,7 +468,7 @@ async def test_peek_pending_messages_skips_malformed_entries(fake_redis: _FakeRedis,) -> None:"""Malformed entries are skipped and valid ones are returned."""- fake_redis.lists["copilot:pending:peek_bad"] = [+ fake_redis.lists[pm_module._buffer_key("peek_bad")] = [json.dumps({"content": "valid peek"}),"{bad json",json.dumps({"content": "also valid peek"}),@@ -486,7 +496,7 @@ async def test_stash_for_persist_enqueues_and_drain_pops_in_order(# Stored under the distinct persist key, NOT the primary buffer.assert "copilot:pending-persist:sess-persist" in fake_redis.lists- assert "copilot:pending:sess-persist" not in fake_redis.lists+ assert pm_module._buffer_key("sess-persist") not in fake_redis.listsdrained = await drain_pending_for_persist("sess-persist")assert len(drained) == 2@@ -612,3 +622,59 @@ async def test_drain_and_format_for_injection_swallows_redis_error(@pytest.mark.asyncioasync def test_drain_and_format_for_injection_missing_session_id() -> None:assert await drain_and_format_for_injection("", log_prefix="[TEST]") == ""+++# ── Cluster-slot colocation regression ───────────────────────────────+# The gated-rpush Lua script in `capped_rpush_if_hash_field` touches both+# the session-meta hash (`stream_registry._get_session_meta_key`) and the+# pending buffer list (`_buffer_key`) atomically. Redis Cluster requires+# every key referenced by a multi-key Lua script to hash to the same slot,+# so both keys must share a hash tag (the `{...}` substring Redis uses for+# slot calculation). Without this, the EVAL returns `CROSSSLOT keys in+# request` once cluster mode is active.+++def _redis_keyslot(key: str) -> int:+ """Compute the Redis Cluster slot for ``key`` using CRC16-XMODEM mod 16384.++ Mirrors the algorithm in redis-py's ``RedisCluster.keyslot`` and the+ Redis spec — extracts the first ``{...}`` substring as the hash tag,+ falls back to the whole key when no tag is present.+ """+ start = key.find("{")+ if start != -1:+ end = key.find("}", start + 1)+ if end > start + 1:+ key = key[start + 1 : end]+ crc = 0+ poly = 0x1021+ for byte in key.encode():+ crc ^= byte << 8+ for _ in range(8):+ crc = ((crc << 1) ^ poly) & 0xFFFF if crc & 0x8000 else (crc << 1) & 0xFFFF+ return crc % 16384+++def test_buffer_and_session_meta_keys_share_cluster_slot() -> None:+ """Regression: pending-buffer key + session-meta key must hash to the+ same Redis Cluster slot, otherwise the gated-rpush Lua script returns+ CROSSSLOT once cluster mode is enabled."""+ # Late import so the test doesn't pull stream_registry's heavy module+ # graph (it transitively wires the AppService client) at file load.+ from backend.copilot.stream_registry import _get_session_meta_key++ for session_id in [+ "sess-abcdef-123",+ "0eb0aae8-6926-4b50-97af-72840841dc70",+ "x",+ ]:+ buf = pm_module._buffer_key(session_id)+ meta = _get_session_meta_key(session_id)+ assert "{" in buf and "}" in buf, f"_buffer_key missing hash tag: {buf!r}"+ assert (+ "{" in meta and "}" in meta+ ), f"_get_session_meta_key missing hash tag: {meta!r}"+ assert _redis_keyslot(buf) == _redis_keyslot(meta), (+ f"CROSSSLOT regression: {buf!r} (slot {_redis_keyslot(buf)}) "+ f"!= {meta!r} (slot {_redis_keyslot(meta)})"+ )diff --git a/autogpt_platform/backend/backend/copilot/rate_limit.py b/autogpt_platform/backend/backend/copilot/rate_limit.pyindex ade137ed06..33f9809d34 100644--- a/autogpt_platform/backend/backend/copilot/rate_limit.py+++ b/autogpt_platform/backend/backend/copilot/rate_limit.py@@ -47,15 +47,14 @@ from pydantic import BaseModel, Fieldfrom redis.exceptions import RedisErrorfrom backend.data.db_accessors import user_db-from backend.data.redis_client import get_redis_async+from backend.data.redis_client import AsyncRedisClient, get_redis_asyncfrom backend.data.user import get_user_by_idfrom backend.util.cache import cachedlogger = logging.getLogger(__name__)-# Redis key prefixes. Bumped from "copilot:usage" (token-based) to-# "copilot:cost" on the token→cost migration so stale counters do not-# get misinterpreted as microdollars (which would dramatically under-count).+# "copilot:cost" (not the legacy "copilot:usage") so stale token-based+# counters are not misread as microdollars._USAGE_KEY_PREFIX = "copilot:cost"@@ -447,25 +446,16 @@ async def reset_daily_usage(user_id: str, daily_cost_limit: int = 0) -> bool:try:redis = await get_redis_async()- # Use a MULTI/EXEC transaction so that DELETE (daily) and DECRBY- # (weekly) either both execute or neither does. This prevents the- # scenario where the daily counter is cleared but the weekly- # counter is not decremented — which would let the caller refund- # credits even though the daily limit was already reset.d_key = _daily_key(user_id, now=now)w_key = _weekly_key(user_id, now=now) if daily_cost_limit > 0 else None- pipe = redis.pipeline(transaction=True)- pipe.delete(d_key)- if w_key is not None:- pipe.decrby(w_key, daily_cost_limit)- results = await pipe.execute()-- # Clamp negative weekly counter to 0 (best-effort; not critical).+ # Daily and weekly keys hash to different cluster slots, so cross-key+ # MULTI/EXEC is not available. Issue the writes sequentially — the+ # failure mode (daily deleted, weekly not decremented) is a+ # best-effort refund budget that the read path already tolerates.+ await redis.delete(d_key)if w_key is not None:- new_val = results[1] # DECRBY result- if new_val < 0:- await redis.set(w_key, 0, keepttl=True)+ await _decr_counter_floor_zero(redis, w_key, daily_cost_limit)logger.info("Reset daily usage for user %s", user_id[:8])return True@@ -555,30 +545,18 @@ async def record_cost_usage(logger.info("Recording copilot spend: %d microdollars", cost_microdollars)now = datetime.now(UTC)+ d_key = _daily_key(user_id, now=now)+ w_key = _weekly_key(user_id, now=now)+ daily_ttl = max(int((_daily_reset_time(now=now) - now).total_seconds()), 1)+ weekly_ttl = max(int((_weekly_reset_time(now=now) - now).total_seconds()), 1)try:redis = await get_redis_async()- # Use MULTI/EXEC so each INCRBY/EXPIRE pair is atomic — guarantees- # the TTL is set even if the connection drops mid-pipeline, so- # counters can never survive past their date-based rotation window.- pipe = redis.pipeline(transaction=True)-- # Daily counter (expires at next midnight UTC)- d_key = _daily_key(user_id, now=now)- pipe.incrby(d_key, cost_microdollars)- seconds_until_daily_reset = int(- (_daily_reset_time(now=now) - now).total_seconds()- )- pipe.expire(d_key, max(seconds_until_daily_reset, 1))-- # Weekly counter (expires end of week)- w_key = _weekly_key(user_id, now=now)- pipe.incrby(w_key, cost_microdollars)- seconds_until_weekly_reset = int(- (_weekly_reset_time(now=now) - now).total_seconds()- )- pipe.expire(w_key, max(seconds_until_weekly_reset, 1))-- await pipe.execute()+ # Daily and weekly keys hash to different cluster slots — cross-slot+ # MULTI/EXEC is not supported, so each counter gets its own+ # single-key transaction. Per-counter INCRBY+EXPIRE atomicity is the+ # invariant that matters; the two counters are independent budgets.+ await _incr_counter_atomic(redis, d_key, cost_microdollars, daily_ttl)+ await _incr_counter_atomic(redis, w_key, cost_microdollars, weekly_ttl)except (RedisError, ConnectionError, OSError):logger.warning("Redis unavailable for recording cost usage (microdollars=%d)",@@ -586,30 +564,56 @@ async def record_cost_usage()+async def _incr_counter_atomic(+ redis: AsyncRedisClient, key: str, delta: int, ttl_seconds: int+) -> None:+ """INCRBY + EXPIRE on a single key inside a MULTI/EXEC transaction."""+ pipe = redis.pipeline(transaction=True)+ pipe.incrby(key, delta)+ pipe.expire(key, ttl_seconds)+ await pipe.execute()+++# Atomic DECRBY + floor-to-zero so a concurrent INCRBY from record_cost_usage+# cannot be lost. DELETE on underflow also avoids leaving a zero-valued key+# with no TTL, which the non-atomic set-with-keepttl variant could do.+_DECR_FLOOR_ZERO_SCRIPT = """+local value = redis.call("DECRBY", KEYS[1], ARGV[1])+if value < 0 then+ redis.call("DEL", KEYS[1])+ return 0+end+return value+"""+++async def _decr_counter_floor_zero(+ redis: AsyncRedisClient, key: str, delta: int+) -> None:+ """Atomically DECRBY ``delta`` on ``key`` and DEL on underflow.++ DEL on underflow avoids leaving a zero-valued key without a TTL, so the+ next INCRBY in ``record_cost_usage`` re-seeds both the value and the+ expiry in one shot.+ """+ await redis.eval(_DECR_FLOOR_ZERO_SCRIPT, 1, key, delta)++class _UserNotFoundError(Exception):"""Raised when a user record is missing or has no subscription tier.- Used internally by ``_fetch_user_tier`` to signal a cache-miss condition:- by raising instead of returning ``DEFAULT_TIER``, we prevent the ``@cached``- decorator from storing the fallback value. This avoids a race condition- where a non-existent user's DEFAULT_TIER is cached, then the user is- created with a higher tier but receives the stale cached FREE tier for- up to 5 minutes.+ Raising (rather than returning ``DEFAULT_TIER``) prevents ``@cached``+ from persisting the fallback, which would otherwise keep serving FREE+ for up to the TTL after the user's real tier is set."""@cached(maxsize=1000, ttl_seconds=300, shared_cache=True)async def _fetch_user_tier(user_id: str) -> SubscriptionTier:- """Fetch the user's rate-limit tier from the database (cached via Redis).-- Uses ``shared_cache=True`` so that tier changes propagate across all pods- immediately when the cache entry is invalidated (via ``cache_delete``).+ """Fetch the user's rate-limit tier, cached across pods.- Only successful DB lookups of existing users with a valid tier are cached.- Raises ``_UserNotFoundError`` when the user is missing or has no tier, so- the ``@cached`` decorator does **not** store a fallback value. This- prevents a race condition where a non-existent user's ``DEFAULT_TIER`` is- cached and then persists after the user is created with a higher tier.+ Only successful lookups are cached. Missing users raise+ ``_UserNotFoundError`` so ``@cached`` never stores the fallback."""try:user = await user_db().get_user_by_id(user_id)@@ -651,20 +655,10 @@ get_user_tier.cache_delete = _fetch_user_tier.cache_delete # type: ignore[attr-async def set_user_tier(user_id: str, tier: SubscriptionTier) -> None:"""Persist the user's rate-limit tier to the database.- Invalidates every cache that keys off the user's subscription tier so the- change is visible immediately: this function's own ``get_user_tier``, the- shared ``get_user_by_id`` (which exposes ``user.subscription_tier``), and- ``get_pending_subscription_change`` (since an admin override can invalidate- a cached ``cancel_at_period_end`` or schedule-based pending state).-- If the user has an active Stripe subscription whose current price does not- match ``tier``, Stripe will keep billing the old price and the next- ``customer.subscription.updated`` webhook will overwrite the DB tier back- to whatever Stripe has. Proper reconciliation (cancelling or modifying the- Stripe subscription when an admin overrides the tier) is out of scope for- this PR — it changes the admin contract and needs its own test coverage.- For now we emit a ``WARNING`` so drift surfaces via Sentry until that- follow-up lands.+ Invalidates the caches that expose ``subscription_tier`` so the change+ takes effect immediately. If the user has an active Stripe subscription+ on a mismatched price, emits a WARNING; Stripe remains the billing+ source of truth and the next webhook will reconcile the DB tier.Raises:prisma.errors.RecordNotFoundError: If the user does not exist.@@ -674,21 +668,13 @@ async def set_user_tier(user_id: str, tier: SubscriptionTier) -> None:data={"subscriptionTier": tier.value},)get_user_tier.cache_delete(user_id) # type: ignore[attr-defined]- # Local import required: backend.data.credit imports backend.copilot.rate_limit- # (via get_user_tier in credit.py's _invalidate_user_tier_caches), so a- # top-level ``from backend.data.credit import ...`` here would create a- # circular import at module-load time.+ # Local import: backend.data.credit imports from this module.from backend.data.credit import get_pending_subscription_changeget_user_by_id.cache_delete(user_id) # type: ignore[attr-defined]get_pending_subscription_change.cache_delete(user_id) # type: ignore[attr-defined]- # The DB write above is already committed; the drift check is best-effort- # diagnostic logging. Fire-and-forget so admin bulk ops don't wait on a- # Stripe roundtrip. The inner helper wraps its body in a timeout + broad- # except so background task errors still surface via logs rather than as- # "task exception never retrieved" warnings. Cancellation on request- # shutdown is acceptable — the drift warning is non-load-bearing.+ # Fire-and-forget drift check so admin bulk ops don't wait on Stripe.asyncio.ensure_future(_drift_check_background(user_id, tier))@@ -711,8 +697,6 @@ async def _drift_check_background(user_id: str, tier: SubscriptionTier) -> None:tier.value,)except asyncio.CancelledError:- # Request may have completed and the event loop is cancelling tasks —- # the drift log is non-critical, so accept cancellation silently.raiseexcept Exception:logger.exception(@@ -726,19 +710,9 @@ async def _drift_check_background(user_id: str, tier: SubscriptionTier) -> None:async def _warn_if_stripe_subscription_drifts(user_id: str, new_tier: SubscriptionTier) -> None:- """Emit a WARNING when an admin tier override leaves an active Stripe sub on a- mismatched price.-- The warning is diagnostic only: Stripe remains the billing source of truth,- so the next ``customer.subscription.updated`` webhook will reset the DB- tier. Surfacing the drift here lets ops catch admin overrides that bypass- the intended Checkout / Portal cancel flows before users notice surprise- charges.- """- # Local imports: see note in ``set_user_tier`` about the credit <-> rate_limit- # circular. These helpers (``_get_active_subscription``,- # ``get_subscription_price_id``) live in credit.py alongside the rest of- # the Stripe billing code.+ """Emit a WARNING when an admin tier override leaves an active Stripe+ subscription on a mismatched price."""+ # Local import: breaks a credit <-> rate_limit circular at module load.from backend.data.credit import _get_active_subscription, get_subscription_price_idtry:@@ -753,10 +727,8 @@ async def _warn_if_stripe_subscription_drifts(returnprice = items[0].pricecurrent_price_id = price if isinstance(price, str) else price.id- # The LaunchDarkly-backed price lookup must live inside this try/except:- # an LD SDK failure (network, token revoked) here would otherwise- # propagate past set_user_tier's already-committed DB write and turn a- # best-effort diagnostic into a 500 on admin tier writes.+ # Inside the try/except: an LD SDK failure here must not turn a+ # best-effort diagnostic into a 500 after the DB write committed.expected_price_id = await get_subscription_price_id(new_tier)except Exception:logger.debug(@@ -838,12 +810,15 @@ async def reset_user_usage(user_id: str, *, reset_weekly: bool = False) -> None:the admin believing the counters were zeroed when they were not."""now = datetime.now(UTC)- keys_to_delete = [_daily_key(user_id, now=now)]- if reset_weekly:- keys_to_delete.append(_weekly_key(user_id, now=now))+ d_key = _daily_key(user_id, now=now)+ w_key = _weekly_key(user_id, now=now) if reset_weekly else Nonetry:redis = await get_redis_async()- await redis.delete(*keys_to_delete)+ # Daily and weekly keys hash to different cluster slots — multi-key+ # DELETE would raise CROSSSLOT, so issue separate calls.+ await redis.delete(d_key)+ if w_key is not None:+ await redis.delete(w_key)except (RedisError, ConnectionError, OSError):logger.warning("Redis unavailable for resetting user usage")raisediff --git a/autogpt_platform/backend/backend/copilot/rate_limit_test.py b/autogpt_platform/backend/backend/copilot/rate_limit_test.pyindex 0c62518610..7cd9b2ec5f 100644--- a/autogpt_platform/backend/backend/copilot/rate_limit_test.py+++ b/autogpt_platform/backend/backend/copilot/rate_limit_test.py@@ -1243,18 +1243,9 @@ class TestTierLimitsRespected:class TestResetDailyUsage:- @staticmethod- def _make_pipeline_mock(decrby_result: int = 0) -> MagicMock:- """Create a pipeline mock that returns [delete_result, decrby_result]."""- pipe = MagicMock()- pipe.execute = AsyncMock(return_value=[1, decrby_result])- return pipe-@pytest.mark.asyncioasync def test_deletes_daily_key(self):- mock_pipe = self._make_pipeline_mock(decrby_result=0)mock_redis = AsyncMock()- mock_redis.pipeline = lambda **_kw: mock_pipewith patch("backend.copilot.rate_limit.get_redis_async",@@ -1263,30 +1254,12 @@ class TestResetDailyUsage:result = await reset_daily_usage(_USER, daily_cost_limit=10000)assert result is True- mock_pipe.delete.assert_called_once()-- @pytest.mark.asyncio- async def test_reduces_weekly_usage_via_decrby(self):- """Weekly counter should be reduced via DECRBY in the pipeline."""- mock_pipe = self._make_pipeline_mock(decrby_result=35000)- mock_redis = AsyncMock()- mock_redis.pipeline = lambda **_kw: mock_pipe-- with patch(- "backend.copilot.rate_limit.get_redis_async",- return_value=mock_redis,- ):- await reset_daily_usage(_USER, daily_cost_limit=10000)-- mock_pipe.decrby.assert_called_once()- mock_redis.set.assert_not_called() # 35000 > 0, no clamp needed+ mock_redis.delete.assert_called_once()@pytest.mark.asyncio- async def test_clamps_negative_weekly_to_zero(self):- """If DECRBY goes negative, SET to 0 (outside the pipeline)."""- mock_pipe = self._make_pipeline_mock(decrby_result=-5000)+ async def test_reduces_weekly_usage_via_eval(self):+ """Weekly counter should be decremented via the atomic Lua script."""mock_redis = AsyncMock()- mock_redis.pipeline = lambda **_kw: mock_pipewith patch("backend.copilot.rate_limit.get_redis_async",@@ -1294,16 +1267,22 @@ class TestResetDailyUsage:):await reset_daily_usage(_USER, daily_cost_limit=10000)- mock_pipe.decrby.assert_called_once()- mock_redis.set.assert_called_once()+ # The Lua script handles both decrement and floor-to-zero in a single+ # call — no separate SET is expected for the clamp branch any more.+ # Pin the call shape so a regression that targets the wrong key or+ # delta (e.g. the daily key, or a sign-flip) fails loudly.+ mock_redis.eval.assert_called_once()+ eval_args = mock_redis.eval.call_args.args+ # eval(script, numkeys, KEYS[1], ARGV[1])+ assert eval_args[1] == 1+ assert eval_args[2] == _weekly_key(_USER)+ assert int(eval_args[3]) == 10000+ mock_redis.set.assert_not_called()@pytest.mark.asyncioasync def test_no_weekly_reduction_when_daily_limit_zero(self):"""When daily_cost_limit is 0, weekly counter should not be touched."""- mock_pipe = self._make_pipeline_mock()- mock_pipe.execute = AsyncMock(return_value=[1]) # only delete resultmock_redis = AsyncMock()- mock_redis.pipeline = lambda **_kw: mock_pipewith patch("backend.copilot.rate_limit.get_redis_async",@@ -1311,8 +1290,8 @@ class TestResetDailyUsage:):await reset_daily_usage(_USER, daily_cost_limit=0)- mock_pipe.delete.assert_called_once()- mock_pipe.decrby.assert_not_called()+ mock_redis.delete.assert_called_once()+ mock_redis.eval.assert_not_called()@pytest.mark.asyncioasync def test_returns_false_when_redis_unavailable(self):@@ -1324,6 +1303,23 @@ class TestResetDailyUsage:assert result is False+ @pytest.mark.asyncio+ async def test_decr_counter_floor_zero_invokes_lua_script(self):+ """The atomic DECRBY+floor helper routes through redis.eval with the+ expected single-key, single-arg call shape."""+ from backend.copilot.rate_limit import (+ _DECR_FLOOR_ZERO_SCRIPT,+ _decr_counter_floor_zero,+ )++ mock_redis = AsyncMock()++ await _decr_counter_floor_zero(mock_redis, "weekly:user1", 42)++ mock_redis.eval.assert_called_once_with(+ _DECR_FLOOR_ZERO_SCRIPT, 1, "weekly:user1", 42+ )+# ---------------------------------------------------------------------------# Tier-limit enforcement (integration-style)@@ -1781,8 +1777,9 @@ class TestResetUserUsage:"backend.copilot.rate_limit.get_redis_async", return_value=mock_redis):await reset_user_usage("user-1", reset_weekly=True)- args = mock_redis.delete.call_args[0]- assert len(args) == 2 # both daily and weekly keys+ # Daily and weekly keys hash to different cluster slots, so they are+ # deleted via two separate DELETE calls (not a single multi-key one).+ assert mock_redis.delete.call_count == 2@pytest.mark.asyncioasync def test_raises_on_redis_failure(self):diff --git a/autogpt_platform/backend/backend/copilot/stream_registry.py b/autogpt_platform/backend/backend/copilot/stream_registry.pyindex a70e9fc4dc..99b263069b 100644--- a/autogpt_platform/backend/backend/copilot/stream_registry.py+++ b/autogpt_platform/backend/backend/copilot/stream_registry.py@@ -90,8 +90,14 @@ class ActiveSession:def _get_session_meta_key(session_id: str) -> str:- """Get Redis key for session metadata (keyed by session_id)."""- return f"{config.session_meta_prefix}{session_id}"+ """Get Redis key for session metadata (keyed by session_id).++ Hash-tag braces colocate this key with ``pending_messages._buffer_key``+ on the same Redis Cluster slot — the gated-rpush Lua script touches both+ keys atomically and would CROSSSLOT-fail if they hashed to different+ shards.+ """+ return f"{config.session_meta_prefix}{{{session_id}}}"def get_session_meta_key(session_id: str) -> str:diff --git a/autogpt_platform/backend/backend/copilot/tools/execution_utils.py b/autogpt_platform/backend/backend/copilot/tools/execution_utils.pyindex e460b9e4ca..038fbfcff2 100644--- a/autogpt_platform/backend/backend/copilot/tools/execution_utils.py+++ b/autogpt_platform/backend/backend/copilot/tools/execution_utils.py@@ -10,6 +10,7 @@ from backend.data.execution import (ExecutionStatus,GraphExecution,GraphExecutionEvent,+ exec_channel,)logger = logging.getLogger(__name__)@@ -81,7 +82,7 @@ async def wait_for_execution()event_bus = AsyncRedisExecutionEventBus()- channel_key = f"{user_id}/{graph_id}/{execution_id}"+ channel_key = exec_channel(user_id, graph_id, execution_id)# Mutable container so _subscribe_and_wait can surface the task even if# asyncio.wait_for cancels the coroutine before it returns.diff --git a/autogpt_platform/backend/backend/data/e2e_redis_rabbit_test.py b/autogpt_platform/backend/backend/data/e2e_redis_rabbit_test.pynew file mode 100644index 0000000000..f9e2d86d2c--- /dev/null+++ b/autogpt_platform/backend/backend/data/e2e_redis_rabbit_test.py@@ -0,0 +1,498 @@+"""End-to-end coverage of the data-layer APIs over the live 3-shard Redis+cluster + RabbitMQ broker. Tests skip when their infra is unreachable.+Container-restart scenarios live in `e2e_redis_restart_test.py`."""++from __future__ import annotations++import asyncio+import json+import time+from datetime import datetime, timezone+from uuid import uuid4++import pytest++import backend.data.redis_client as redis_client+from backend.api.model import NotificationPayload+from backend.data.execution import (+ AsyncRedisExecutionEventBus,+ ExecutionStatus,+ NodeExecutionEvent,+ exec_channel,+ graph_all_channel,+)+from backend.data.notification_bus import (+ AsyncRedisNotificationEventBus,+ NotificationEvent,+)+from backend.data.rabbitmq import AsyncRabbitMQ+from backend.executor.utils import (+ GRAPH_EXECUTION_EXCHANGE,+ GRAPH_EXECUTION_QUEUE_NAME,+ create_execution_queue_config,+)+++def _has_live_cluster() -> bool:+ try:+ c = redis_client.connect()+ except Exception: # noqa: BLE001 — any connect failure → skip+ return False+ try:+ c.close()+ except Exception:+ pass+ return True+++def _has_live_rabbit() -> bool:+ """Probe the rabbitmq host:port from settings; skip if unreachable."""+ import socket++ from backend.util.settings import Settings++ s = Settings()+ try:+ with socket.create_connection(+ (s.config.rabbitmq_host, s.config.rabbitmq_port), timeout=1.0+ ):+ return True+ except Exception: # noqa: BLE001 - any connect failure → skip the test+ return False+++cluster_only = pytest.mark.skipif(+ not _has_live_cluster(),+ reason="local redis cluster not reachable; skip e2e integration",+)+rabbit_only = pytest.mark.skipif(+ not _has_live_rabbit(),+ reason="local rabbitmq not reachable; skip e2e integration",+)+++def _make_node_event(*, user_id: str, graph_id: str, gex_id: str, marker: str):+ return NodeExecutionEvent(+ user_id=user_id,+ graph_id=graph_id,+ graph_version=1,+ graph_exec_id=gex_id,+ node_exec_id=f"node-exec-{marker}",+ node_id="node-1",+ block_id="block-1",+ status=ExecutionStatus.COMPLETED,+ input_data={"in": marker},+ output_data={"out": [marker]},+ add_time=datetime.now(tz=timezone.utc),+ queue_time=None,+ start_time=datetime.now(tz=timezone.utc),+ end_time=datetime.now(tz=timezone.utc),+ )+++# ---------- Scenario 1: cluster cache round-trip across slots ----------+++@cluster_only+def test_cluster_cache_roundtrip_across_three_slots() -> None:+ """A list-graphs-style cache flow: SET keys with hash tags that land on+ different shards, GET them back. Validates the basic cluster-routing+ contract end-to-end."""+ redis_client.get_redis.cache_clear()+ cluster = redis_client.get_redis()+ keys = []+ try:+ # Pick keys that hash to different slots — try until 3 distinct shards.+ seen: set[tuple[str, int]] = set()+ for i in range(2000):+ key = f"e2e:cache:{i}"+ node = cluster.get_node_from_key(key)+ owner = (node.host, node.port)+ if owner in seen:+ continue+ seen.add(owner)+ keys.append(key)+ if len(seen) >= 3:+ break+ assert len(keys) >= 3++ for i, k in enumerate(keys):+ cluster.setex(k, 60, f"v-{i}")+ for i, k in enumerate(keys):+ assert cluster.get(k) == f"v-{i}"+ finally:+ for k in keys:+ try:+ cluster.delete(k)+ except Exception:+ pass+ redis_client.disconnect()+++# ---------- Scenarios 2 & 3: graph execution event streams ----------+++@cluster_only+async def test_graph_execution_events_complete_under_ten_seconds() -> None:+ """A listener subscribes to the per-exec channel; the producer publishes+ one node event. The listener must observe it in under 10 seconds —+ pins the latency contract end-to-end through SPUBLISH/SSUBSCRIBE."""+ redis_client._async_clients.clear()+ user_id = f"u-e2e-{uuid4().hex[:8]}"+ graph_id = f"g-{uuid4().hex[:8]}"+ gex_id = f"x-{uuid4().hex[:8]}"++ publisher = AsyncRedisExecutionEventBus()+ subscriber = AsyncRedisExecutionEventBus()+ received: list[str] = []++ async def _consume() -> None:+ async for evt in subscriber.listen_events(+ exec_channel(user_id, graph_id, gex_id)+ ):+ received.append(getattr(evt, "node_exec_id", "graph"))+ return++ task = asyncio.create_task(_consume())+ await asyncio.sleep(0.3)++ start = time.monotonic()+ try:+ await publisher.publish_event(+ _make_node_event(+ user_id=user_id, graph_id=graph_id, gex_id=gex_id, marker="m1"+ ),+ exec_channel(user_id, graph_id, gex_id),+ )+ await asyncio.wait_for(task, timeout=10.0)+ finally:+ if not task.done():+ task.cancel()+ await subscriber.close()+ await redis_client.disconnect_async()++ elapsed = time.monotonic() - start+ assert elapsed < 10.0, f"event roundtrip took {elapsed:.2f}s, expected < 10s"+ assert received == ["node-exec-m1"]+++@cluster_only+async def test_two_concurrent_graphs_no_cross_talk() -> None:+ """Two graphs execute in parallel; two listeners on different per-exec+ channels each receive only their own events."""+ redis_client._async_clients.clear()+ user_id = f"u-e2e-{uuid4().hex[:8]}"+ g1, g2 = f"g1-{uuid4().hex[:8]}", f"g2-{uuid4().hex[:8]}"+ e1, e2 = f"e1-{uuid4().hex[:8]}", f"e2-{uuid4().hex[:8]}"++ publisher = AsyncRedisExecutionEventBus()+ sub_a = AsyncRedisExecutionEventBus()+ sub_b = AsyncRedisExecutionEventBus()++ async def _listen_one(bus, channel_key: str, sink: list, want: int) -> None:+ async for evt in bus.listen_events(channel_key):+ sink.append(getattr(evt, "node_exec_id", "graph"))+ if len(sink) >= want:+ return++ sink_a: list[str] = []+ sink_b: list[str] = []+ t_a = asyncio.create_task(+ _listen_one(sub_a, exec_channel(user_id, g1, e1), sink_a, want=3)+ )+ t_b = asyncio.create_task(+ _listen_one(sub_b, exec_channel(user_id, g2, e2), sink_b, want=3)+ )+ await asyncio.sleep(0.3)++ try:+ for i in range(3):+ await publisher.publish_event(+ _make_node_event(+ user_id=user_id, graph_id=g1, gex_id=e1, marker=f"a{i}"+ ),+ exec_channel(user_id, g1, e1),+ )+ await publisher.publish_event(+ _make_node_event(+ user_id=user_id, graph_id=g2, gex_id=e2, marker=f"b{i}"+ ),+ exec_channel(user_id, g2, e2),+ )++ await asyncio.wait_for(asyncio.gather(t_a, t_b), timeout=10.0)+ assert sink_a == ["node-exec-a0", "node-exec-a1", "node-exec-a2"]+ assert sink_b == ["node-exec-b0", "node-exec-b1", "node-exec-b2"]+ finally:+ await sub_a.close()+ await sub_b.close()+ await redis_client.disconnect_async()+++# ---------- Scenario 4: aggregate /all channel for graph executions ----------+++@cluster_only+async def test_three_executions_land_on_aggregate_channel() -> None:+ """Subscribe to the aggregate ``/all`` channel; trigger 3 different+ executions of the same graph; assert all 3 land on the aggregate."""+ redis_client._async_clients.clear()+ user_id = f"u-e2e-{uuid4().hex[:8]}"+ graph_id = f"g-{uuid4().hex[:8]}"+ exec_ids = [f"x{i}-{uuid4().hex[:6]}" for i in range(3)]++ publisher = AsyncRedisExecutionEventBus()+ subscriber = AsyncRedisExecutionEventBus()+ received: list[str] = []++ async def _listen_all() -> None:+ async for evt in subscriber.listen_events(graph_all_channel(user_id, graph_id)):+ received.append(getattr(evt, "graph_exec_id", "?"))+ if len(received) >= 3:+ return++ task = asyncio.create_task(_listen_all())+ await asyncio.sleep(0.3)++ try:+ for ex in exec_ids:+ await publisher.publish_event(+ _make_node_event(+ user_id=user_id, graph_id=graph_id, gex_id=ex, marker=ex+ ),+ graph_all_channel(user_id, graph_id),+ )++ await asyncio.wait_for(task, timeout=10.0)+ # Order of receipt may vary slightly under load — check set membership.+ assert set(received) == set(exec_ids)+ finally:+ await subscriber.close()+ await redis_client.disconnect_async()+++# ---------- Scenarios 5 & 6: copilot/notification per-user channels ----------+++@cluster_only+async def test_copilot_cancel_signal_via_sharded_pubsub() -> None:+ """A subscriber on a per-session channel receives an SPUBLISH cancel+ signal — the primitive the copilot executor uses for graceful cancel."""+ redis_client._async_clients.clear()+ session_id = f"sess-{uuid4().hex[:8]}"+ channel = "{copilot/" + session_id + "}/cancel"++ client = await redis_client.connect_sharded_pubsub_async(channel)+ pubsub = client.pubsub()+ received: list[str] = []+ try:+ await pubsub.execute_command("SSUBSCRIBE", channel)+ # Prime the channels map so listen() doesn't early-exit (see _Subscription).+ pubsub.channels[channel] = None # type: ignore[index]++ async def _pump() -> None:+ async for msg in pubsub.listen():+ if msg.get("type") == "smessage":+ received.append(msg["data"])+ return++ listener = asyncio.create_task(_pump())+ await asyncio.sleep(0.2)++ cluster = await redis_client.get_redis_async()+ await cluster.execute_command("SPUBLISH", channel, "cancel")++ await asyncio.wait_for(listener, timeout=5.0)+ assert received == ["cancel"]+ finally:+ try:+ await pubsub.execute_command("SUNSUBSCRIBE", channel)+ except Exception:+ pass+ await pubsub.aclose()+ await client.aclose()+ await redis_client.disconnect_async()+++@cluster_only+async def test_notification_fan_out_per_user_channel() -> None:+ """Per-user SSUBSCRIBE: a publish on the user's notification channel+ reaches the user's listener and only that listener."""+ redis_client._async_clients.clear()+ user_id = f"u-notif-{uuid4().hex[:8]}"+ other_user_id = f"u-other-{uuid4().hex[:8]}"++ publisher = AsyncRedisNotificationEventBus()+ listener_user = AsyncRedisNotificationEventBus()+ listener_other = AsyncRedisNotificationEventBus()++ user_received: list[str] = []+ other_received: list[str] = []+ notif_for_user = NotificationEvent(+ user_id=user_id,+ payload=NotificationPayload(type="info", event="balance-low"),+ )+ notif_for_other = NotificationEvent(+ user_id=other_user_id,+ payload=NotificationPayload(type="info", event="other"),+ )++ async def _listen_one(bus: AsyncRedisNotificationEventBus, uid: str, sink: list):+ async for evt in bus.listen(uid):+ sink.append(evt.user_id)+ return++ t_user = asyncio.create_task(_listen_one(listener_user, user_id, user_received))+ t_other = asyncio.create_task(+ _listen_one(listener_other, other_user_id, other_received)+ )+ await asyncio.sleep(0.3)++ try:+ await publisher.publish(notif_for_user)+ await publisher.publish(notif_for_other)+ await asyncio.wait_for(asyncio.gather(t_user, t_other), timeout=10.0)+ assert user_received == [user_id]+ assert other_received == [other_user_id]+ finally:+ await listener_user.close()+ await listener_other.close()+ await publisher.close()+ await redis_client.disconnect_async()+++# ---------- Scenario 7: idle WS connection 60s ----------+++@cluster_only+async def test_idle_subscriber_60s_then_receives_publish() -> None:+ """An SSUBSCRIBE that sits idle past one health-check interval must+ still deliver a subsequent SPUBLISH (uses HEALTH_CHECK_INTERVAL+5s)."""+ redis_client._async_clients.clear()+ channel = "{idle-e2e}/exec/" + uuid4().hex[:8]+ client = await redis_client.connect_sharded_pubsub_async(channel)+ pubsub = client.pubsub()+ try:+ await pubsub.execute_command("SSUBSCRIBE", channel)+ pubsub.channels[channel] = None # type: ignore[index]+ # Drain ssubscribe confirm.+ async for _msg in pubsub.listen():+ break++ idle_seconds = redis_client.HEALTH_CHECK_INTERVAL + 5+ await asyncio.sleep(idle_seconds)++ cluster = await redis_client.get_redis_async()+ await cluster.execute_command("SPUBLISH", channel, "hello-after-idle")++ async for msg in pubsub.listen():+ if msg.get("type") == "smessage":+ assert msg["data"] == "hello-after-idle"+ return+ finally:+ try:+ await pubsub.execute_command("SUNSUBSCRIBE", channel)+ except Exception:+ pass+ await pubsub.aclose()+ await client.aclose()+ await redis_client.disconnect_async()+++# ---------- Scenario 8: graph_execution_queue_v2 publish + consume ----------+++@rabbit_only+async def test_graph_execution_queue_publish_and_consume() -> None:+ """End-to-end on a test-scoped quorum queue: publish via AsyncRabbitMQ+ → consume → payload round-trips intact. Uses a unique routing key so+ the live executor consumer (if any) doesn't race for the message."""+ from backend.data.rabbitmq import Exchange, ExchangeType, Queue, RabbitMQConfig++ test_queue_name = f"e2e_test_{uuid4().hex[:8]}_v2"+ test_routing_key = f"e2e.test.{uuid4().hex[:8]}"+ test_exchange = Exchange(+ name=GRAPH_EXECUTION_EXCHANGE.name,+ type=ExchangeType.DIRECT,+ durable=True,+ )+ test_queue = Queue(+ name=test_queue_name,+ durable=True,+ # Quorum queues reject auto_delete; we delete the queue explicitly+ # in the finally block instead.+ auto_delete=False,+ exchange=test_exchange,+ routing_key=test_routing_key,+ arguments={"x-queue-type": "quorum"},+ )+ cfg = RabbitMQConfig(vhost="/", exchanges=[test_exchange], queues=[test_queue])++ publisher = AsyncRabbitMQ(cfg)+ await publisher.connect()+ consumer = AsyncRabbitMQ(cfg)+ await consumer.connect()++ payload = json.dumps(+ {"graph_exec_id": f"e2e-{uuid4().hex[:8]}", "marker": "round-trip"}+ )++ try:+ channel = await consumer.get_channel()+ queue_obj = await channel.get_queue(test_queue_name)++ await publisher.publish_message(+ routing_key=test_routing_key,+ message=payload,+ exchange=test_exchange,+ )++ # Poll get() — quorum queue must surface the publish within 5s.+ deadline = time.monotonic() + 5.0+ msg = None+ while time.monotonic() < deadline:+ msg = await queue_obj.get(no_ack=True, fail=False)+ if msg is not None:+ break+ await asyncio.sleep(0.05)+ assert msg is not None, "publish never reached the quorum queue"+ assert msg.body.decode() == payload+ finally:+ # Best-effort delete in case auto_delete didn't trigger.+ try:+ channel = await consumer.get_channel()+ await channel.queue_delete(test_queue_name, if_unused=False, if_empty=False)+ except Exception:+ pass+ await publisher.disconnect()+ await consumer.disconnect()+++@rabbit_only+async def test_graph_execution_queue_uses_quorum_via_real_broker() -> None:+ """Live-broker check that `graph_execution_queue_v2` is declared as+ quorum — passive re-declare with `x-queue-type=quorum` must not raise."""+ cfg = create_execution_queue_config()+ client = AsyncRabbitMQ(cfg)+ await client.connect() # declares everything in cfg+ try:+ channel = await client.get_channel()+ # Re-declare passively — must NOT raise PRECONDITION_FAILED if the+ # type matches, would raise if quorum was lost.+ q = await channel.declare_queue(+ name=GRAPH_EXECUTION_QUEUE_NAME,+ durable=True,+ arguments={"x-queue-type": "quorum"},+ passive=True,+ )+ assert q.name == GRAPH_EXECUTION_QUEUE_NAME+ finally:+ await client.disconnect()diff --git a/autogpt_platform/backend/backend/data/e2e_redis_restart_test.py b/autogpt_platform/backend/backend/data/e2e_redis_restart_test.pynew file mode 100644index 0000000000..cad8210e24--- /dev/null+++ b/autogpt_platform/backend/backend/data/e2e_redis_restart_test.py@@ -0,0 +1,313 @@+"""Sharded pubsub reconnect across a real `docker restart` of a shard,+against a private 3-shard cluster on isolated host ports. Gated on+`E2E_REDIS_CLUSTER_RESTART=1` + `docker` on PATH, marked `pytest.mark.slow`."""++from __future__ import annotations++import asyncio+import importlib+import os+import shutil+import socket+import subprocess+import time+from uuid import uuid4++import pytest++# Disjoint from the dev-compose ports (17000-17002) so both stacks coexist.+ISOLATED_PROJECT = "redis-restart-test"+ISOLATED_PORTS = (27110, 27111, 27112)+ISOLATED_BUS_PORTS = (37110, 37111, 37112)+++def _docker_available() -> bool:+ return shutil.which("docker") is not None+++def _isolated_enabled() -> bool:+ return os.getenv("E2E_REDIS_CLUSTER_RESTART", "").lower() in ("1", "true", "yes")+++cluster_restart_only = pytest.mark.skipif(+ not (_docker_available() and _isolated_enabled()),+ reason=(+ "isolated docker cluster restart e2e: requires docker + E2E_REDIS_CLUSTER_RESTART=1"+ ),+)+++def _run(cmd: list[str], *, timeout: float = 60.0) -> subprocess.CompletedProcess[str]:+ return subprocess.run(+ cmd,+ capture_output=True,+ text=True,+ timeout=timeout,+ check=False,+ )+++def _wait_port(port: int, *, deadline_s: float = 60.0) -> None:+ deadline = time.monotonic() + deadline_s+ while time.monotonic() < deadline:+ try:+ with socket.create_connection(("127.0.0.1", port), timeout=1.0):+ return+ except OSError:+ time.sleep(0.5)+ raise TimeoutError(f"port {port} never opened within {deadline_s:.0f}s")+++def _start_isolated_cluster() -> None:+ """Spin up a private 3-shard cluster via raw `docker run` + one-shot+ `redis-cli --cluster create`."""+ network = f"{ISOLATED_PROJECT}-net"+ _run(["docker", "network", "create", network]) # may exist; ignore exit+ for i, (port, bus) in enumerate(zip(ISOLATED_PORTS, ISOLATED_BUS_PORTS)):+ name = f"{ISOLATED_PROJECT}-redis-{i}"+ _run(["docker", "rm", "-f", name])+ rc = _run(+ [+ "docker",+ "run",+ "-d",+ "--name",+ name,+ "--network",+ network,+ "--network-alias",+ f"redis-{i}",+ "-p",+ f"{port}:{port}",+ "redis:7",+ "redis-server",+ "--port",+ str(port),+ "--cluster-enabled",+ "yes",+ "--cluster-config-file",+ "nodes.conf",+ "--cluster-node-timeout",+ "5000",+ "--cluster-require-full-coverage",+ "no",+ "--cluster-announce-hostname",+ f"redis-{i}",+ "--cluster-announce-port",+ str(port),+ "--cluster-announce-bus-port",+ str(bus),+ "--cluster-preferred-endpoint-type",+ "hostname",+ ]+ )+ if rc.returncode != 0:+ raise RuntimeError(f"docker run redis-{i} failed: {rc.stderr}")+ for port in ISOLATED_PORTS:+ _wait_port(port)+ rc = _run(+ [+ "docker",+ "run",+ "--rm",+ "--network",+ network,+ "redis:7",+ "redis-cli",+ "--cluster",+ "create",+ f"redis-0:{ISOLATED_PORTS[0]}",+ f"redis-1:{ISOLATED_PORTS[1]}",+ f"redis-2:{ISOLATED_PORTS[2]}",+ "--cluster-replicas",+ "0",+ "--cluster-yes",+ ]+ )+ if rc.returncode != 0:+ raise RuntimeError(f"cluster create failed: {rc.stderr}")+ deadline = time.monotonic() + 30+ while time.monotonic() < deadline:+ info = _run(+ [+ "docker",+ "exec",+ f"{ISOLATED_PROJECT}-redis-0",+ "redis-cli",+ "-p",+ str(ISOLATED_PORTS[0]),+ "cluster",+ "info",+ ]+ )+ if "cluster_state:ok" in info.stdout:+ return+ time.sleep(0.5)+ raise TimeoutError("isolated cluster never reached cluster_state:ok")+++def _wait_cluster_ok(timeout_s: float = 30.0) -> bool:+ deadline = time.monotonic() + timeout_s+ while time.monotonic() < deadline:+ info = _run(+ [+ "docker",+ "exec",+ f"{ISOLATED_PROJECT}-redis-0",+ "redis-cli",+ "-p",+ str(ISOLATED_PORTS[0]),+ "cluster",+ "info",+ ]+ )+ if "cluster_state:ok" in info.stdout:+ return True+ time.sleep(0.5)+ return False+++def _teardown_isolated_cluster() -> None:+ for i in range(3):+ _run(["docker", "rm", "-f", f"{ISOLATED_PROJECT}-redis-{i}"])+ _run(["docker", "network", "rm", f"{ISOLATED_PROJECT}-net"])++[email protected](scope="module")+def isolated_cluster():+ """Module-scoped: tests share one cluster lifecycle."""+ _start_isolated_cluster()+ try:+ yield+ finally:+ _teardown_isolated_cluster()+++@cluster_restart_only+async def test_subscriber_survives_shard_restart(isolated_cluster, monkeypatch) -> None:+ """Subscriber must receive a post-`docker restart` SPUBLISH after+ reopening the sharded-pubsub client (the broker drops the socket on+ restart; production's `with_pubsub` loop reconnects the same way)."""+ # Must override REDIS_CLUSTER_HOST/PORT too — those take precedence+ # over REDIS_HOST/PORT and a stray .env would point us at the dev cluster.+ monkeypatch.setenv("REDIS_HOST", "127.0.0.1")+ monkeypatch.setenv("REDIS_PORT", str(ISOLATED_PORTS[0]))+ monkeypatch.setenv("REDIS_CLUSTER_HOST", "127.0.0.1")+ monkeypatch.setenv("REDIS_CLUSTER_PORT", str(ISOLATED_PORTS[0]))+ monkeypatch.setenv("REDIS_USE_ANNOUNCED_ADDRESS", "false")+ monkeypatch.delenv("REDIS_PASSWORD", raising=False)++ import backend.data.redis_client as rc++ importlib.reload(rc)++ # Restart whichever container owns the keyslot, not a guess.+ cluster = rc.get_redis()+ target_tag = f"restart-{uuid4().hex[:8]}"+ channel = "{" + target_tag + "}/restart-test"+ owner = cluster.get_node_from_key(channel)+ port_to_idx = {p: i for i, p in enumerate(ISOLATED_PORTS)}+ target_idx = port_to_idx.get(owner.port)+ assert (+ target_idx is not None+ ), f"owner port {owner.port} not in known set {ISOLATED_PORTS}"+ target_container = f"{ISOLATED_PROJECT}-redis-{target_idx}"++ client = await rc.connect_sharded_pubsub_async(channel)+ pubsub = client.pubsub()+ await pubsub.execute_command("SSUBSCRIBE", channel)+ pubsub.channels[channel] = None # type: ignore[index]++ received: list[str] = []++ async def _drain_one() -> str | None:+ try:+ async for msg in pubsub.listen():+ if msg.get("type") == "smessage":+ return msg["data"]+ except Exception:+ return None+ return None++ try:+ async_cluster = await rc.get_redis_async()+ await async_cluster.execute_command("SPUBLISH", channel, "before-restart")++ first = await asyncio.wait_for(_drain_one(), timeout=6.0)+ received.append(first or "")+ assert received == [+ "before-restart"+ ], f"pre-restart publish did not arrive: {received}"++ # Restart the shard that owns the slot.+ rc_restart = _run(["docker", "restart", "--time", "1", target_container])+ assert rc_restart.returncode == 0, rc_restart.stderr++ assert _wait_cluster_ok(+ timeout_s=30+ ), "isolated cluster never re-converged to state=ok after restart"+ # Hold a small grace window for shard's gossip to settle.+ await asyncio.sleep(1.0)++ # Old socket is dead — open a fresh sharded-pubsub connection.+ try:+ await pubsub.aclose()+ except Exception:+ pass+ try:+ await client.aclose()+ except Exception:+ pass+ rc._async_clients.clear()++ client2 = await rc.connect_sharded_pubsub_async(channel)+ pubsub2 = client2.pubsub()+ try:+ await pubsub2.execute_command("SSUBSCRIBE", channel)+ pubsub2.channels[channel] = None # type: ignore[index]++ # Drain the SSUBSCRIBE confirm.+ async for _msg in pubsub2.listen():+ break++ async def _drain_after() -> str | None:+ async for msg in pubsub2.listen():+ if msg.get("type") == "smessage":+ return msg["data"]+ return None++ async_cluster_2 = await rc.get_redis_async()+ await async_cluster_2.execute_command("SPUBLISH", channel, "after-restart")++ data = await asyncio.wait_for(_drain_after(), timeout=15.0)+ assert (+ data == "after-restart"+ ), f"subscriber did not receive post-restart event (got {data!r})"+ finally:+ try:+ await pubsub2.execute_command("SUNSUBSCRIBE", channel)+ except Exception:+ pass+ try:+ await pubsub2.aclose()+ except Exception:+ pass+ await client2.aclose()+ finally:+ try:+ await pubsub.aclose()+ except Exception:+ pass+ try:+ await client.aclose()+ except Exception:+ pass+ await rc.disconnect_async()+ # Undo monkeypatched env BEFORE reloading so subsequent tests see the+ # original REDIS_HOST/PORT — otherwise the module captures the+ # isolated cluster's port (27110) which is torn down right after this+ # test, and any later test that touches redis hangs on conn_retry.+ monkeypatch.undo()+ importlib.reload(rc)diff --git a/autogpt_platform/backend/backend/data/event_bus.py b/autogpt_platform/backend/backend/data/event_bus.pyindex 614fb158b2..6b11090a15 100644--- a/autogpt_platform/backend/backend/data/event_bus.py+++ b/autogpt_platform/backend/backend/data/event_bus.py@@ -1,7 +1,15 @@import asyncioimport loggingfrom abc import ABC, abstractmethod-from typing import Any, AsyncGenerator, Generator, Generic, Optional, TypeVar+from typing import (+ TYPE_CHECKING,+ Any,+ AsyncGenerator,+ Generator,+ Generic,+ Optional,+ TypeVar,+)from pydantic import BaseModelfrom redis.asyncio.client import PubSub as AsyncPubSub@@ -11,6 +19,9 @@ from backend.data import redis_client as redisfrom backend.util import jsonfrom backend.util.settings import Settings+if TYPE_CHECKING:+ from redis.asyncio import Redis as AsyncRedis+logger = logging.getLogger(__name__)config = Settings().config@@ -18,6 +29,15 @@ config = Settings().configM = TypeVar("M", bound=BaseModel)+def _assert_no_wildcard(channel_key: str) -> None:+ """Sharded pub/sub has no pattern-subscribe; fail fast on wildcards."""+ if "*" in channel_key:+ raise ValueError(+ f"channel_key {channel_key!r} contains a wildcard; sharded pub/sub "+ "(SSUBSCRIBE) requires exact channel names."+ )++class BaseRedisEventBus(Generic[M], ABC):Model: type[M]@@ -71,8 +91,8 @@ class BaseRedisEventBus(Generic[M], ABC):return message, channel_namedef _deserialize_message(self, msg: Any, channel_key: str) -> M | None:- message_type = "pmessage" if "*" in channel_key else "message"- if msg["type"] != message_type:+ # Accept sharded (smessage) and classic (message/pmessage) deliveries.+ if msg["type"] not in ("smessage", "message", "pmessage"):return Nonetry:logger.debug(f"[{channel_key}] Consuming an event from Redis {msg['data']}")@@ -80,12 +100,8 @@ class BaseRedisEventBus(Generic[M], ABC):except Exception as e:logger.error(f"Failed to parse event result from Redis {msg} {e}")- def _get_pubsub_channel(- self, connection: redis.Redis | redis.AsyncRedis, channel_key: str- ) -> tuple[PubSub | AsyncPubSub, str]:- full_channel_name = f"{self.event_bus_name}/{channel_key}"- pubsub = connection.pubsub()- return pubsub, full_channel_name+ def _build_channel_name(self, channel_key: str) -> str:+ return f"{self.event_bus_name}/{channel_key}"class _EventPayloadWrapper(BaseModel, Generic[M]):@@ -98,88 +114,97 @@ class _EventPayloadWrapper(BaseModel, Generic[M]):class RedisEventBus(BaseRedisEventBus[M], ABC):- @property- def connection(self) -> redis.Redis:- return redis.get_redis()-def publish_event(self, event: M, channel_key: str):- """- Publish an event to Redis. Gracefully handles connection failures- by logging the error instead of raising exceptions.- """+ """Publish via SPUBLISH; swallow failures so Redis blips don't crash callers."""+ _assert_no_wildcard(channel_key)try:message, full_channel_name = self._serialize_message(event, channel_key)- self.connection.publish(full_channel_name, message)+ cluster = redis.get_redis()+ cluster.execute_command("SPUBLISH", full_channel_name, message)except Exception:- logger.exception(- f"Failed to publish event to Redis channel {channel_key}. "- "Event bus operation will continue without Redis connectivity."- )+ logger.exception(f"Failed to publish event to Redis channel {channel_key}")def listen_events(self, channel_key: str) -> Generator[M, None, None]:- pubsub, full_channel_name = self._get_pubsub_channel(- self.connection, channel_key- )- assert isinstance(pubsub, PubSub)-- if "*" in channel_key:- pubsub.psubscribe(full_channel_name)- else:- pubsub.subscribe(full_channel_name)+ _assert_no_wildcard(channel_key)+ full_channel_name = self._build_channel_name(channel_key)- for message in pubsub.listen():- if event := self._deserialize_message(message, channel_key):- yield event+ cluster = redis.get_redis()+ pubsub: PubSub = cluster.pubsub()+ try:+ pubsub.ssubscribe(full_channel_name)+ for message in pubsub.listen():+ if event := self._deserialize_message(message, channel_key):+ yield event+ finally:+ try:+ pubsub.sunsubscribe(full_channel_name)+ except Exception:+ logger.warning(+ "Failed to SUNSUBSCRIBE from %s", full_channel_name, exc_info=True+ )+ try:+ pubsub.close()+ except Exception:+ logger.warning(+ "Failed to close sharded pubsub for %s",+ full_channel_name,+ exc_info=True,+ )class AsyncRedisEventBus(BaseRedisEventBus[M], ABC):- def __init__(self):- self._pubsub: AsyncPubSub | None = None-- @property- async def connection(self) -> redis.AsyncRedis:- return await redis.get_redis_async()-async def close(self) -> None:- """Close the PubSub connection if it exists."""- if self._pubsub is not None:- try:- await self._pubsub.close()- except Exception:- logger.warning("Failed to close PubSub connection", exc_info=True)- finally:- self._pubsub = None+ """No-op kept for backward compatibility.- async def publish_event(self, event: M, channel_key: str):- """- Publish an event to Redis. Gracefully handles connection failures- by logging the error instead of raising exceptions.+ Earlier revisions of this class stored the per-listen pubsub on the+ instance, requiring an external close. ``listen_events`` now owns its+ own client/pubsub locally so concurrent calls on a singleton (e.g.+ ``_webhook_event_bus``) cannot clobber each other's connection."""+ return None++ async def publish_event(self, event: M, channel_key: str):+ """Publish via SPUBLISH; swallow failures so Redis blips don't crash callers."""+ _assert_no_wildcard(channel_key)try:message, full_channel_name = self._serialize_message(event, channel_key)- connection = await self.connection- await connection.publish(full_channel_name, message)+ cluster = await redis.get_redis_async()+ # redis-py 6.x async cluster has no spublish(); execute_command handles MOVED.+ await cluster.execute_command("SPUBLISH", full_channel_name, message)except Exception:- logger.exception(- f"Failed to publish event to Redis channel {channel_key}. "- "Event bus operation will continue without Redis connectivity."- )+ logger.exception(f"Failed to publish event to Redis channel {channel_key}")async def listen_events(self, channel_key: str) -> AsyncGenerator[M, None]:- pubsub, full_channel_name = self._get_pubsub_channel(- await self.connection, channel_key+ _assert_no_wildcard(channel_key)+ full_channel_name = self._build_channel_name(channel_key)++ # Sharded pub/sub only delivers on the keyslot-owning shard, so pin+ # a plain AsyncRedis to that node. Both client and pubsub stay+ # generator-local — concurrent listen_events on the same instance+ # (e.g. the singleton _webhook_event_bus) must not share state.+ client: "AsyncRedis" = await redis.connect_sharded_pubsub_async(+ full_channel_name)- assert isinstance(pubsub, AsyncPubSub)- self._pubsub = pubsub-- if "*" in channel_key:- await pubsub.psubscribe(full_channel_name)- else:- await pubsub.subscribe(full_channel_name)-- async for message in pubsub.listen():- if event := self._deserialize_message(message, channel_key):- yield event+ pubsub: AsyncPubSub = client.pubsub()+ try:+ await pubsub.execute_command("SSUBSCRIBE", full_channel_name)+ # redis-py 6.x async PubSub.listen() exits when ``channels`` is+ # empty; raw SSUBSCRIBE doesn't populate it, so do it ourselves.+ pubsub.channels[full_channel_name] = None # type: ignore[index]+ async for message in pubsub.listen():+ if event := self._deserialize_message(message, channel_key):+ yield event+ finally:+ try:+ await pubsub.aclose()+ except Exception:+ logger.warning("Failed to close PubSub connection", exc_info=True)+ try:+ await client.aclose()+ except Exception:+ logger.warning(+ "Failed to close shard-pinned Redis connection", exc_info=True+ )async def wait_for_event(self, channel_key: str, timeout: Optional[float] = Nonediff --git a/autogpt_platform/backend/backend/data/event_bus_test.py b/autogpt_platform/backend/backend/data/event_bus_test.pyindex 93ecc0ba2a..3318ec2164 100644--- a/autogpt_platform/backend/backend/data/event_bus_test.py+++ b/autogpt_platform/backend/backend/data/event_bus_test.py@@ -1,25 +1,26 @@-"""-Tests for event_bus graceful degradation when Redis is unavailable.-"""+"""Tests for event_bus publish/listen paths."""-from unittest.mock import AsyncMock, patch+import asyncio+from unittest.mock import AsyncMock, MagicMock, patchimport pytestfrom pydantic import BaseModel-from backend.data.event_bus import AsyncRedisEventBus+from backend.data.event_bus import (+ AsyncRedisEventBus,+ RedisEventBus,+ _assert_no_wildcard,+)-class TestEvent(BaseModel):- """Test event model."""+class SampleEvent(BaseModel):+ """Minimal event model used by the tests below."""message: str-class TestNotificationBus(AsyncRedisEventBus[TestEvent]):- """Test implementation of AsyncRedisEventBus."""-- Model = TestEvent+class _BusUnderTest(AsyncRedisEventBus[SampleEvent]):+ Model = SampleEvent@propertydef event_bus_name(self) -> str:@@ -28,11 +29,10 @@ class TestNotificationBus(AsyncRedisEventBus[TestEvent]):@pytest.mark.asyncioasync def test_publish_event_handles_connection_failure_gracefully():- """Test that publish_event logs exception instead of raising when Redis is unavailable."""- bus = TestNotificationBus()- event = TestEvent(message="test message")+ """publish_event must log and swallow when the cluster client is down."""+ bus = _BusUnderTest()+ event = SampleEvent(message="test message")- # Mock get_redis_async to raise connection errorwith patch("backend.data.event_bus.redis.get_redis_async",side_effect=ConnectionError("Authentication required."),@@ -42,15 +42,487 @@ async def test_publish_event_handles_connection_failure_gracefully():@pytest.mark.asyncio-async def test_publish_event_works_with_redis_available():- """Test that publish_event works normally when Redis is available."""- bus = TestNotificationBus()- event = TestEvent(message="test message")+async def test_publish_event_spublishes_via_cluster_client():+ """publish_event routes a single SPUBLISH through the cluster client."""+ bus = _BusUnderTest()+ event = SampleEvent(message="test message")- # Mock successful Redis connection- mock_redis = AsyncMock()- mock_redis.publish = AsyncMock()+ mock_cluster = MagicMock()+ mock_cluster.execute_command = AsyncMock()- with patch("backend.data.event_bus.redis.get_redis_async", return_value=mock_redis):+ with patch(+ "backend.data.event_bus.redis.get_redis_async", return_value=mock_cluster+ ):await bus.publish_event(event, "test_channel")- mock_redis.publish.assert_called_once()++ mock_cluster.execute_command.assert_awaited_once()+ assert mock_cluster.execute_command.await_args[0][0] == "SPUBLISH"+++async def test_publish_event_rejects_wildcard_channel():+ """A channel_key containing ``*`` must raise — no silent no-op."""+ bus = _BusUnderTest()+ with patch("backend.data.event_bus.redis.get_redis_async") as get_cluster:+ with pytest.raises(ValueError):+ await bus.publish_event(SampleEvent(message="m"), "user/*/exec")+ # The cluster client must never be reached for a wildcard channel.+ get_cluster.assert_not_called()+++def test_assert_no_wildcard_guard():+ """The standalone guard must reject any ``*``-containing channel."""+ with pytest.raises(ValueError):+ _assert_no_wildcard("user/*/exec")+ # Concrete channels must pass.+ _assert_no_wildcard("execution_event/user-1/graph-1/exec-1")+++# Live SSUBSCRIBE round-trip; skipped when no cluster is reachable.+++def _has_live_cluster() -> bool:+ from backend.data import redis_client++ try:+ c = redis_client.connect()+ except Exception: # noqa: BLE001 - any connect failure → skip the test+ return False+ try:+ c.close()+ except Exception:+ pass+ return True+++ not _has_live_cluster(),+ reason="local redis cluster not reachable; skip SSUBSCRIBE integration",+)+async def test_ssubscribe_end_to_end_async():+ """SPUBLISH on one AsyncRedisEventBus reaches SSUBSCRIBE on another."""+ import asyncio++ from backend.data import redis_client++ redis_client.get_redis.cache_clear()+ redis_client._async_clients.clear()++ publisher = _BusUnderTest()+ subscriber = _BusUnderTest()+ channel_key = "pr12900:event_bus:integration"++ received: list[SampleEvent] = []++ async def consume() -> None:+ async for ev in subscriber.listen_events(channel_key):+ received.append(ev)+ return++ task = asyncio.create_task(consume())+ # Let SSUBSCRIBE settle; races drop the publish otherwise.+ await asyncio.sleep(0.3)+ try:+ await publisher.publish_event(SampleEvent(message="hello-ssub"), channel_key)+ await asyncio.wait_for(task, timeout=5.0)+ finally:+ if not task.done():+ task.cancel()+ await subscriber.close()+ await redis_client.disconnect_async()++ assert received and received[0].message == "hello-ssub"+++ not _has_live_cluster(),+ reason="local redis cluster not reachable; skip execution-bus integration",+)+async def test_execution_bus_listen_and_listen_graph_both_deliver():+ """Per-exec and per-graph channels both receive every execution event."""+ import asyncio+ from datetime import datetime, timezone++ from backend.data import redis_client+ from backend.data.execution import (+ AsyncRedisExecutionEventBus,+ ExecutionStatus,+ GraphExecutionEvent,+ )++ redis_client.get_redis.cache_clear()+ redis_client._async_clients.clear()++ user_id = "user-it"+ graph_id = "graph-it"+ exec_id = "exec-it"+ now = datetime.now(tz=timezone.utc)++ event = GraphExecutionEvent(+ id=exec_id,+ user_id=user_id,+ graph_id=graph_id,+ graph_version=1,+ preset_id=None,+ status=ExecutionStatus.COMPLETED,+ started_at=now,+ ended_at=now,+ stats=GraphExecutionEvent.Stats(+ cost=0, duration=0.1, node_exec_time=0.1, node_exec_count=1+ ),+ inputs={},+ credential_inputs=None,+ nodes_input_masks=None,+ outputs={},+ )++ single = AsyncRedisExecutionEventBus()+ all_execs = AsyncRedisExecutionEventBus()+ publisher = AsyncRedisExecutionEventBus()++ received_single: list = []+ received_all: list = []++ async def _listen_single() -> None:+ async for ev in single.listen(user_id, graph_id, exec_id):+ received_single.append(ev)+ return++ async def _listen_all() -> None:+ async for ev in all_execs.listen_graph(user_id, graph_id):+ received_all.append(ev)+ return++ t1 = asyncio.create_task(_listen_single())+ t2 = asyncio.create_task(_listen_all())+ await asyncio.sleep(0.3)++ try:+ await publisher.publish(event)+ await asyncio.wait_for(asyncio.gather(t1, t2), timeout=5.0)+ finally:+ for t in (t1, t2):+ if not t.done():+ t.cancel()+ await single.close()+ await all_execs.close()+ await publisher.close()+ await redis_client.disconnect_async()++ assert received_single and received_single[0].id == exec_id+ assert received_all and received_all[0].id == exec_id+++async def test_listen_events_rejects_wildcard_channel():+ """listen_events on a wildcard channel must raise before touching Redis."""+ bus = _BusUnderTest()+ with pytest.raises(ValueError):+ async for _ in bus.listen_events("user/*/exec"):+ break+++# ---------- Serialization + size guard ----------+++def test_serialize_message_tags_full_channel_name():+ """_serialize_message returns the ``<bus>/<key>`` full channel name."""+ bus = _BusUnderTest()+ _, full = bus._serialize_message(SampleEvent(message="x"), "chan")+ assert full == "test_event_bus/chan"+++def test_serialize_message_truncates_oversized_payload(monkeypatch):+ """If the payload exceeds max_message_size_limit, it's replaced with an+ ``error_comms_update`` payload rather than crashing the cluster."""+ import backend.data.event_bus as event_bus++ bus = _BusUnderTest()+ # Cap tiny to force truncation.+ monkeypatch.setattr(event_bus.config, "max_message_size_limit", 50)+ message, _ = bus._serialize_message(SampleEvent(message="x" * 1000), "chan")+ assert "error_comms_update" in message+ assert "Payload too large" in message+++def test_deserialize_message_rejects_non_pubsub_types():+ """Non ``smessage|message|pmessage`` deliveries deserialize to None."""+ bus = _BusUnderTest()+ assert bus._deserialize_message({"type": "ssubscribe", "data": 1}, "c") is None+ assert bus._deserialize_message({"type": "subscribe", "data": 1}, "c") is None+++def test_deserialize_message_swallows_bad_json():+ """Corrupted payloads must not raise — they return None (logged elsewhere)."""+ bus = _BusUnderTest()+ assert (+ bus._deserialize_message({"type": "smessage", "data": "not-json"}, "c") is None+ )+++def test_deserialize_message_parses_smessage():+ """Happy-path ``smessage`` yields the inner event model."""+ bus = _BusUnderTest()+ wrapped = '{"payload":{"message":"hi"}}'+ parsed = bus._deserialize_message({"type": "smessage", "data": wrapped}, "chan")+ assert parsed is not None and parsed.message == "hi"+++# ---------- Sync RedisEventBus ----------+++class _SyncBusUnderTest(RedisEventBus[SampleEvent]):+ Model = SampleEvent++ @property+ def event_bus_name(self) -> str:+ return "test_event_bus"+++def test_sync_publish_event_spublish_only():+ """Sync publish_event must issue a single SPUBLISH (no classic fallback)."""+ bus = _SyncBusUnderTest()+ cluster = MagicMock()+ cluster.execute_command = MagicMock()++ with patch("backend.data.event_bus.redis.get_redis", return_value=cluster):+ bus.publish_event(SampleEvent(message="m"), "chan")++ cluster.execute_command.assert_called_once()+ assert cluster.execute_command.call_args.args[0] == "SPUBLISH"+++def test_sync_publish_event_rejects_wildcard():+ bus = _SyncBusUnderTest()+ with patch("backend.data.event_bus.redis.get_redis") as mock_get:+ with pytest.raises(ValueError):+ bus.publish_event(SampleEvent(message="m"), "user/*/exec")+ mock_get.assert_not_called()+++def test_sync_publish_event_swallows_connection_errors():+ """publish_event must never raise to callers — logs + drops on failure."""+ bus = _SyncBusUnderTest()+ with patch(+ "backend.data.event_bus.redis.get_redis",+ side_effect=ConnectionError("no redis"),+ ):+ # Should NOT raise.+ bus.publish_event(SampleEvent(message="m"), "chan")+++def test_sync_listen_events_rejects_wildcard():+ bus = _SyncBusUnderTest()+ with pytest.raises(ValueError):+ next(iter(bus.listen_events("user/*/exec")))+++def test_sync_listen_events_ssubscribes_and_yields_decoded_events():+ """Sync listen_events: SSUBSCRIBE on the full channel, decode smessage payloads."""+ bus = _SyncBusUnderTest()++ fake_pubsub = MagicMock()+ fake_pubsub.ssubscribe = MagicMock()+ fake_pubsub.sunsubscribe = MagicMock()+ fake_pubsub.close = MagicMock()+ fake_pubsub.listen = MagicMock(+ return_value=iter(+ [+ {"type": "ssubscribe", "data": 1},+ {"type": "smessage", "data": '{"payload":{"message":"one"}}'},+ ]+ )+ )++ cluster = MagicMock()+ cluster.pubsub = MagicMock(return_value=fake_pubsub)++ with patch("backend.data.event_bus.redis.get_redis", return_value=cluster):+ gen = bus.listen_events("chan")+ first = next(iter(gen))++ assert first.message == "one"+ fake_pubsub.ssubscribe.assert_called_once_with("test_event_bus/chan")+++def test_sync_listen_events_teardown_swallows_sunsubscribe_errors():+ """Teardown must not propagate SUNSUBSCRIBE/close failures."""+ bus = _SyncBusUnderTest()++ fake_pubsub = MagicMock()+ fake_pubsub.ssubscribe = MagicMock()+ fake_pubsub.sunsubscribe = MagicMock(side_effect=RuntimeError("SUNSUB broke"))+ fake_pubsub.close = MagicMock(side_effect=RuntimeError("close broke"))+ fake_pubsub.listen = MagicMock(return_value=iter([]))+ cluster = MagicMock()+ cluster.pubsub = MagicMock(return_value=fake_pubsub)++ with patch("backend.data.event_bus.redis.get_redis", return_value=cluster):+ # Exhausting the generator runs the ``finally`` teardown.+ list(bus.listen_events("chan"))++ fake_pubsub.sunsubscribe.assert_called_once()+ fake_pubsub.close.assert_called_once()+++# ---------- Async close() teardown ----------+++async def test_async_close_is_noop():+ """close() is a backward-compat no-op now that listen_events owns its own state."""+ bus = _BusUnderTest()+ # Repeated calls must not crash; pubsub/client are generator-locals.+ await bus.close()+ await bus.close()+++async def test_async_listen_events_swallows_aclose_errors():+ """Broken pubsub.aclose / client.aclose must not propagate to the caller."""+ bus = _BusUnderTest()++ fake_pubsub = MagicMock()+ fake_pubsub.execute_command = AsyncMock()+ fake_pubsub.channels = {}+ fake_pubsub.aclose = AsyncMock(side_effect=RuntimeError("pubsub broke"))++ async def _listen():+ return+ yield # pragma: no cover — unreachable++ fake_pubsub.listen = _listen++ fake_client = MagicMock()+ fake_client.pubsub = MagicMock(return_value=fake_pubsub)+ fake_client.aclose = AsyncMock(side_effect=RuntimeError("client broke"))++ with patch(+ "backend.data.event_bus.redis.connect_sharded_pubsub_async",+ AsyncMock(return_value=fake_client),+ ):+ async for _ in bus.listen_events("chan"):+ pass # pragma: no cover — never yields++ # Both aclose attempts must have run despite raising.+ fake_pubsub.aclose.assert_awaited_once()+ fake_client.aclose.assert_awaited_once()+++async def test_async_listen_events_concurrent_does_not_share_state():+ """Two concurrent listens on the same bus must keep their pubsub/client local."""+ bus = _BusUnderTest()++ pubsubs: list[MagicMock] = []+ clients: list[MagicMock] = []+ started = asyncio.Event()+ proceed = asyncio.Event()++ def _make_pair() -> tuple[MagicMock, MagicMock]:+ pubsub = MagicMock()+ pubsub.execute_command = AsyncMock()+ pubsub.channels = {}+ pubsub.aclose = AsyncMock()++ async def _listen():+ started.set()+ await proceed.wait()+ return+ yield # pragma: no cover — unreachable++ pubsub.listen = _listen++ client = MagicMock()+ client.pubsub = MagicMock(return_value=pubsub)+ client.aclose = AsyncMock()+ pubsubs.append(pubsub)+ clients.append(client)+ return pubsub, client++ async def _factory(_chan: str):+ _, client = _make_pair()+ return client++ with patch(+ "backend.data.event_bus.redis.connect_sharded_pubsub_async",+ AsyncMock(side_effect=_factory),+ ):++ async def _run():+ async for _ in bus.listen_events("chan"):+ pass # pragma: no cover — never yields++ task_a = asyncio.create_task(_run())+ task_b = asyncio.create_task(_run())+ # Wait for both pumps to be parked inside listen() before unblocking.+ await started.wait()+ # Yield once more so the second task also enters listen().+ await asyncio.sleep(0)+ proceed.set()+ await asyncio.gather(task_a, task_b)++ # Each listen must have closed its OWN pubsub/client exactly once. If+ # either was closed twice or zero times, the singleton race is back.+ assert len(pubsubs) == 2+ for pubsub in pubsubs:+ pubsub.aclose.assert_awaited_once()+ for client in clients:+ client.aclose.assert_awaited_once()+++async def test_async_wait_for_event_returns_none_on_timeout():+ """wait_for_event must coerce asyncio.TimeoutError → None."""+ import asyncio as _asyncio++ bus = _BusUnderTest()++ async def _never(self, channel_key):+ await _asyncio.sleep(10)+ yield # pragma: no cover — unreachable++ with patch.object(_BusUnderTest, "listen_events", _never):+ result = await bus.wait_for_event("chan", timeout=0.01)++ assert result is None+++# The listen_events async happy path is covered by the live-cluster integration+# test above; this one exercises the close-on-exception fallback.+async def test_async_listen_events_closes_on_exception():+ """If the pump raises, close() must still run to release the shard-pinned client."""+ bus = _BusUnderTest()++ fake_pubsub = MagicMock()+ fake_pubsub.execute_command = AsyncMock()+ fake_pubsub.channels = {}+ fake_pubsub.aclose = AsyncMock()++ class _Boom(Exception):+ pass++ async def _listen():+ raise _Boom()+ yield # pragma: no cover — unreachable++ fake_pubsub.listen = _listen++ fake_client = MagicMock()+ fake_client.pubsub = MagicMock(return_value=fake_pubsub)+ fake_client.aclose = AsyncMock()++ with patch(+ "backend.data.event_bus.redis.connect_sharded_pubsub_async",+ AsyncMock(return_value=fake_client),+ ):+ with pytest.raises(_Boom):+ async for _ in bus.listen_events("chan"):+ pass++ # close() must have fired (both aclose calls).+ fake_pubsub.aclose.assert_awaited_once()+ fake_client.aclose.assert_awaited_once()diff --git a/autogpt_platform/backend/backend/data/execution.py b/autogpt_platform/backend/backend/data/execution.pyindex cd50d7df3c..e89e467f45 100644--- a/autogpt_platform/backend/backend/data/execution.py+++ b/autogpt_platform/backend/backend/data/execution.py@@ -570,7 +570,7 @@ async def get_graph_executions(# Build properly typed order clause# Prisma wants specific typed dicts for each field, so we construct them explicitlyorder_clause: AgentGraphExecutionOrderByInput- match (order_by):+ match order_by:case "startedAt":order_clause = {"startedAt": order_direction,@@ -1337,6 +1337,22 @@ ExecutionEvent = Annotated[]+# Hash-tagged channels keep per-exec and per-graph keys on the same shard,+# so one SSUBSCRIBE connection can watch both.+++def _graph_scope_tag(user_id: str, graph_id: str) -> str:+ return "{" + f"{user_id}/{graph_id}" + "}"+++def exec_channel(user_id: str, graph_id: str, graph_exec_id: str) -> str:+ return f"{_graph_scope_tag(user_id, graph_id)}/exec/{graph_exec_id}"+++def graph_all_channel(user_id: str, graph_id: str) -> str:+ return f"{_graph_scope_tag(user_id, graph_id)}/all"++class RedisExecutionEventBus(RedisEventBus[ExecutionEvent]):Model = ExecutionEvent # type: ignore@@ -1352,16 +1368,20 @@ class RedisExecutionEventBus(RedisEventBus[ExecutionEvent]):def _publish_node_exec_update(self, res: NodeExecutionResult):event = NodeExecutionEvent.model_validate(res.model_dump())- self._publish(event, f"{res.user_id}/{res.graph_id}/{res.graph_exec_id}")+ self._publish(event, res.user_id, res.graph_id, res.graph_exec_id)def _publish_graph_exec_update(self, res: GraphExecution):event = GraphExecutionEvent.model_validate(res.model_dump())- self._publish(event, f"{res.user_id}/{res.graph_id}/{res.id}")+ self._publish(event, res.user_id, res.graph_id, res.id)- def _publish(self, event: ExecutionEvent, channel: str):- """- truncate inputs and outputs to avoid large payloads- """+ def _publish(+ self,+ event: ExecutionEvent,+ user_id: str,+ graph_id: str,+ graph_exec_id: str,+ ):+ """Truncate oversized payloads, then publish to per-exec + per-graph channels."""limit = config.max_message_size_limit // 2if isinstance(event, GraphExecutionEvent):event.inputs = truncate(event.inputs, limit)@@ -1370,12 +1390,22 @@ class RedisExecutionEventBus(RedisEventBus[ExecutionEvent]):event.input_data = truncate(event.input_data, limit)event.output_data = truncate(event.output_data, limit)- super().publish_event(event, channel)+ # Publisher fans out: per-exec and per-graph watchers.+ super().publish_event(event, exec_channel(user_id, graph_id, graph_exec_id))+ super().publish_event(event, graph_all_channel(user_id, graph_id))def listen(- self, user_id: str, graph_id: str = "*", graph_exec_id: str = "*"+ self, user_id: str, graph_id: str, graph_exec_id: str) -> Generator[ExecutionEvent, None, None]:- for event in self.listen_events(f"{user_id}/{graph_id}/{graph_exec_id}"):+ """Stream events for a specific graph execution."""+ for event in self.listen_events(exec_channel(user_id, graph_id, graph_exec_id)):+ yield event++ def listen_graph(+ self, user_id: str, graph_id: str+ ) -> Generator[ExecutionEvent, None, None]:+ """Stream every event for every execution of ``graph_id``."""+ for event in self.listen_events(graph_all_channel(user_id, graph_id)):yield event@@ -1395,7 +1425,7 @@ class AsyncRedisExecutionEventBus(AsyncRedisEventBus[ExecutionEvent]):async def _publish_node_exec_update(self, res: NodeExecutionResult):event = NodeExecutionEvent.model_validate(res.model_dump())- await self._publish(event, f"{res.user_id}/{res.graph_id}/{res.graph_exec_id}")+ await self._publish(event, res.user_id, res.graph_id, res.graph_exec_id)async def _publish_graph_exec_update(self, res: GraphExecutionMeta):# GraphExecutionEvent requires inputs and outputs fields that GraphExecutionMeta doesn't have@@ -1404,12 +1434,16 @@ class AsyncRedisExecutionEventBus(AsyncRedisEventBus[ExecutionEvent]):event_data.setdefault("inputs", {})event_data.setdefault("outputs", {})event = GraphExecutionEvent.model_validate(event_data)- await self._publish(event, f"{res.user_id}/{res.graph_id}/{res.id}")+ await self._publish(event, res.user_id, res.graph_id, res.id)- async def _publish(self, event: ExecutionEvent, channel: str):- """- truncate inputs and outputs to avoid large payloads- """+ async def _publish(+ self,+ event: ExecutionEvent,+ user_id: str,+ graph_id: str,+ graph_exec_id: str,+ ):+ """Truncate oversized payloads, then publish to per-exec + per-graph channels."""limit = config.max_message_size_limit // 2if isinstance(event, GraphExecutionEvent):event.inputs = truncate(event.inputs, limit)@@ -1418,12 +1452,25 @@ class AsyncRedisExecutionEventBus(AsyncRedisEventBus[ExecutionEvent]):event.input_data = truncate(event.input_data, limit)event.output_data = truncate(event.output_data, limit)- await super().publish_event(event, channel)+ await super().publish_event(+ event, exec_channel(user_id, graph_id, graph_exec_id)+ )+ await super().publish_event(event, graph_all_channel(user_id, graph_id))async def listen(- self, user_id: str, graph_id: str = "*", graph_exec_id: str = "*"+ self, user_id: str, graph_id: str, graph_exec_id: str+ ) -> AsyncGenerator[ExecutionEvent, None]:+ """Stream events for a specific graph execution."""+ async for event in self.listen_events(+ exec_channel(user_id, graph_id, graph_exec_id)+ ):+ yield event++ async def listen_graph(+ self, user_id: str, graph_id: str) -> AsyncGenerator[ExecutionEvent, None]:- async for event in self.listen_events(f"{user_id}/{graph_id}/{graph_exec_id}"):+ """Stream every event for every execution of ``graph_id``."""+ async for event in self.listen_events(graph_all_channel(user_id, graph_id)):yield event@@ -1682,11 +1729,11 @@ async def create_shared_execution_files(created += 1except UniqueViolationError:logger.debug(- f"Skipping shared file record for {file_id}: " f"record already exists"+ f"Skipping shared file record for {file_id}: record already exists")except ForeignKeyViolationError:logger.debug(- f"Skipping shared file record for {file_id}: " f"file does not exist"+ f"Skipping shared file record for {file_id}: file does not exist")return createddiff --git a/autogpt_platform/backend/backend/data/execution_event_bus_test.py b/autogpt_platform/backend/backend/data/execution_event_bus_test.pynew file mode 100644index 0000000000..78cbbca9f2--- /dev/null+++ b/autogpt_platform/backend/backend/data/execution_event_bus_test.py@@ -0,0 +1,387 @@+"""Tests for the sharded channel builders + publish/listen paths on+``AsyncRedisExecutionEventBus`` / ``RedisExecutionEventBus``.++These tests are intentionally Prisma-free: they exercise only the in-process+event-routing layer, using mocks for the Redis cluster client. The live+SSUBSCRIBE round-trip is covered by the integration test in+``event_bus_test.py``.+"""++from datetime import datetime, timezone+from unittest.mock import AsyncMock, MagicMock, patch++import pytest++from backend.data.execution import (+ AsyncRedisExecutionEventBus,+ ExecutionEventType,+ ExecutionStatus,+ GraphExecutionEvent,+ NodeExecutionEvent,+ RedisExecutionEventBus,+ _graph_scope_tag,+ exec_channel,+ graph_all_channel,+)++# ---------- Hash-tagged channel builders ----------+++def test_graph_scope_tag_uses_hash_tag_syntax():+ """Hash-tagged tag must look like ``{user/graph}`` so per-exec + per-graph+ channels hash to the same Redis Cluster keyslot."""+ assert _graph_scope_tag("u", "g") == "{u/g}"+++def test_exec_channel_nests_scope_tag():+ """Per-exec channel: ``{user/graph}/exec/<exec_id>``."""+ assert exec_channel("u", "g", "e") == "{u/g}/exec/e"+++def test_graph_all_channel_nests_scope_tag():+ """Aggregate channel: ``{user/graph}/all`` — keyslot-aligned with per-exec."""+ assert graph_all_channel("u", "g") == "{u/g}/all"+++def test_exec_and_graph_channels_share_hash_tag():+ """Invariant: both channels *must* share the ``{user/graph}`` prefix.+ If this breaks, SSUBSCRIBE for per-exec and aggregate routes to different+ shards and the per-graph listener loses some events."""+ exec_ch = exec_channel("u", "g", "e")+ graph_ch = graph_all_channel("u", "g")+ assert exec_ch.startswith("{u/g}")+ assert graph_ch.startswith("{u/g}")+++# ---------- NodeExecutionEvent publish → exec channel only ----------+++def _sample_node_event() -> NodeExecutionEvent:+ now = datetime.now(tz=timezone.utc)+ return NodeExecutionEvent(+ user_id="u",+ graph_id="g",+ graph_version=1,+ graph_exec_id="e",+ node_exec_id="ne",+ node_id="nid",+ block_id="bid",+ status=ExecutionStatus.COMPLETED,+ input_data={"a": 1},+ output_data={"o": [1]},+ add_time=now,+ queue_time=None,+ start_time=now,+ end_time=now,+ )+++def _sample_graph_event() -> GraphExecutionEvent:+ now = datetime.now(tz=timezone.utc)+ return GraphExecutionEvent(+ id="e",+ user_id="u",+ graph_id="g",+ graph_version=1,+ preset_id=None,+ status=ExecutionStatus.COMPLETED,+ started_at=now,+ ended_at=now,+ stats=GraphExecutionEvent.Stats(+ cost=0, duration=0.1, node_exec_time=0.1, node_exec_count=1+ ),+ inputs={},+ credential_inputs=None,+ nodes_input_masks=None,+ outputs={},+ )+++async def test_async_publish_node_sends_to_both_channels():+ """Node events fan out to BOTH per-exec and aggregate channels so the+ per-graph WS subscriber sees every node update, not just graph-level ones.+ """+ bus = AsyncRedisExecutionEventBus()+ sent_channels: list[str] = []++ async def _capture(self, event, channel_key):+ sent_channels.append(channel_key)++ with patch.object(+ AsyncRedisExecutionEventBus.__mro__[1], "publish_event", _capture+ ):+ await bus._publish_node_exec_update(_sample_node_event())++ assert sent_channels == [+ exec_channel("u", "g", "e"),+ graph_all_channel("u", "g"),+ ]+++async def test_async_publish_graph_sends_to_both_channels():+ bus = AsyncRedisExecutionEventBus()+ sent_channels: list[str] = []++ async def _capture(self, event, channel_key):+ sent_channels.append(channel_key)++ with patch.object(+ AsyncRedisExecutionEventBus.__mro__[1], "publish_event", _capture+ ):+ await bus._publish_graph_exec_update(_sample_graph_event())++ assert sent_channels == [+ exec_channel("u", "g", "e"),+ graph_all_channel("u", "g"),+ ]+++async def test_async_publish_routes_via_type_dispatch():+ """publish() dispatches on the model type — not on status or event_type."""+ bus = AsyncRedisExecutionEventBus()++ with (+ patch.object(bus, "_publish_graph_exec_update", AsyncMock()) as graph_pub,+ patch.object(bus, "_publish_node_exec_update", AsyncMock()) as node_pub,+ ):+ await bus.publish(_sample_graph_event())+ await bus.publish(_sample_node_event())++ graph_pub.assert_awaited_once()+ node_pub.assert_awaited_once()+++async def test_async_publish_truncates_oversized_payload(monkeypatch):+ """Payload truncation applies before sending — size exceeded → replacement."""+ import backend.data.execution as execution++ bus = AsyncRedisExecutionEventBus()+ # Force tiny limit so ``truncate`` rewrites the payload.+ monkeypatch.setattr(execution.config, "max_message_size_limit", 10)++ cluster = MagicMock()+ cluster.execute_command = AsyncMock()+ with patch("backend.data.event_bus.redis.get_redis_async", return_value=cluster):+ await bus.publish(_sample_node_event())++ # Called twice: per-exec and per-graph channel.+ assert cluster.execute_command.await_count == 2+++async def test_async_listen_uses_exec_channel():+ """listen() must subscribe to the per-exec hash-tagged channel."""+ bus = AsyncRedisExecutionEventBus()++ captured: list[str] = []++ async def _listen_events(self, channel_key):+ captured.append(channel_key)+ # Return an empty async-generator so the ``async for`` exits cleanly.+ if False:+ yield # pragma: no cover++ with patch.object(AsyncRedisExecutionEventBus, "listen_events", _listen_events):+ async for _ in bus.listen("u", "g", "e"):+ break # pragma: no cover — generator is empty++ assert captured == [exec_channel("u", "g", "e")]+++async def test_async_listen_graph_uses_all_channel():+ """listen_graph() must subscribe to the aggregate hash-tagged channel."""+ bus = AsyncRedisExecutionEventBus()++ captured: list[str] = []++ async def _listen_events(self, channel_key):+ captured.append(channel_key)+ if False:+ yield # pragma: no cover++ with patch.object(AsyncRedisExecutionEventBus, "listen_events", _listen_events):+ async for _ in bus.listen_graph("u", "g"):+ break # pragma: no cover — generator is empty++ assert captured == [graph_all_channel("u", "g")]+++# ---------- Sync RedisExecutionEventBus (smaller surface; covers branching) ----------+++def test_sync_listen_uses_exec_channel():+ bus = RedisExecutionEventBus()++ captured: list[str] = []++ def _listen_events(self, channel_key):+ captured.append(channel_key)+ return iter([])++ with patch.object(RedisExecutionEventBus, "listen_events", _listen_events):+ list(bus.listen("u", "g", "e"))++ assert captured == [exec_channel("u", "g", "e")]+++def test_sync_listen_graph_uses_all_channel():+ bus = RedisExecutionEventBus()++ captured: list[str] = []++ def _listen_events(self, channel_key):+ captured.append(channel_key)+ return iter([])++ with patch.object(RedisExecutionEventBus, "listen_events", _listen_events):+ list(bus.listen_graph("u", "g"))++ assert captured == [graph_all_channel("u", "g")]+++def test_sync_publish_node_sends_to_both_channels():+ """Sync publish path also fans out to per-exec + per-graph."""+ bus = RedisExecutionEventBus()+ sent: list[str] = []++ def _capture(self, event, channel_key):+ sent.append(channel_key)++ with patch.object(RedisExecutionEventBus.__mro__[1], "publish_event", _capture):+ bus._publish_node_exec_update(_sample_node_event().model_copy())++ assert sent == [+ exec_channel("u", "g", "e"),+ graph_all_channel("u", "g"),+ ]+++def test_event_type_is_literal_on_events():+ """event_type is a discriminator literal, not dynamic — the WS fan-out+ relies on ``ExecutionEventType(event_type)`` being stable."""+ node = _sample_node_event()+ graph = _sample_graph_event()+ assert node.event_type == ExecutionEventType.NODE_EXEC_UPDATE+ assert graph.event_type == ExecutionEventType.GRAPH_EXEC_UPDATE+++# ---------- Sync publish dispatch + listen yields ----------+++def test_sync_publish_dispatches_on_model_type():+ """Sync ``publish()`` routes GraphExecution and NodeExecutionResult to+ their respective helpers — regression guard on the type-dispatch branch."""+ from backend.data.execution import GraphExecution, NodeExecutionResult++ bus = RedisExecutionEventBus()++ graph_like = MagicMock(spec=GraphExecution)+ node_like = MagicMock(spec=NodeExecutionResult)++ with (+ patch.object(bus, "_publish_graph_exec_update") as graph_pub,+ patch.object(bus, "_publish_node_exec_update") as node_pub,+ ):+ bus.publish(graph_like)+ bus.publish(node_like)++ graph_pub.assert_called_once_with(graph_like)+ node_pub.assert_called_once_with(node_like)+++def test_sync_publish_graph_exec_update_rebuilds_event():+ """Sync ``_publish_graph_exec_update`` validates the input into a+ GraphExecutionEvent before delegating to ``_publish`` — don't let a raw+ GraphExecution slip through the type-discriminated listener."""+ bus = RedisExecutionEventBus()+ graph_event = _sample_graph_event()++ with patch.object(bus, "_publish") as mock_publish:+ # Feed back the event itself (it's a GraphExecution subclass) to avoid+ # needing a full Graph fixture.+ bus._publish_graph_exec_update(graph_event)++ mock_publish.assert_called_once()+ args = mock_publish.call_args.args+ # The first arg is a GraphExecutionEvent (validated copy).+ assert args[0].event_type == ExecutionEventType.GRAPH_EXEC_UPDATE+ # Channel-routing args match the input.+ assert args[1:] == ("u", "g", "e")+++def test_sync_publish_node_exec_update_rebuilds_event():+ """Sync ``_publish_node_exec_update`` validates to NodeExecutionEvent."""+ bus = RedisExecutionEventBus()+ node_event = _sample_node_event()++ with patch.object(bus, "_publish") as mock_publish:+ bus._publish_node_exec_update(node_event)++ mock_publish.assert_called_once()+ args = mock_publish.call_args.args+ assert args[0].event_type == ExecutionEventType.NODE_EXEC_UPDATE+ assert args[1:] == ("u", "g", "e")+++def test_sync_publish_graph_truncates_inputs_and_outputs(monkeypatch):+ """Sync ``_publish`` must truncate GraphExecutionEvent.inputs/outputs when+ the payload exceeds the cap — protects Redis from oversized frames."""+ import backend.data.execution as execution++ bus = RedisExecutionEventBus()+ monkeypatch.setattr(execution.config, "max_message_size_limit", 4)++ event = _sample_graph_event()+ event.inputs = {"long": "x" * 10_000}+ event.outputs = {"long": ["y" * 10_000]}++ with patch("backend.data.event_bus.redis.get_redis", return_value=MagicMock()):+ bus._publish(event, "u", "g", "e")++ # After _publish runs, inputs/outputs have been truncated in-place.+ import json as _json++ assert len(_json.dumps(event.inputs)) < 1000+ assert len(_json.dumps(event.outputs)) < 1000+++def test_sync_listen_yields_events_from_generator():+ """Sync ``listen()`` must yield through every event produced by the+ underlying ``listen_events`` generator."""+ bus = RedisExecutionEventBus()+ node_ev = _sample_node_event()++ def _listen_events(self, channel_key):+ yield node_ev++ with patch.object(RedisExecutionEventBus, "listen_events", _listen_events):+ got = list(bus.listen("u", "g", "e"))++ assert got == [node_ev]+++def test_sync_listen_graph_yields_events_from_generator():+ bus = RedisExecutionEventBus()+ graph_ev = _sample_graph_event()++ def _listen_events(self, channel_key):+ yield graph_ev++ with patch.object(RedisExecutionEventBus, "listen_events", _listen_events):+ got = list(bus.listen_graph("u", "g"))++ assert got == [graph_ev]+++def test_execution_bus_name_matches_settings():+ """Both sync and async buses must read the same configured bus name — the+ WS subscriber depends on this for channel naming."""+ assert (+ RedisExecutionEventBus().event_bus_name+ == AsyncRedisExecutionEventBus().event_bus_name+ )diff --git a/autogpt_platform/backend/backend/data/integrations.py b/autogpt_platform/backend/backend/data/integrations.pyindex 02db1d005b..a88b8891e3 100644--- a/autogpt_platform/backend/backend/data/integrations.py+++ b/autogpt_platform/backend/backend/data/integrations.py@@ -1,5 +1,5 @@import logging-from typing import AsyncGenerator, Literal, Optional, overload+from typing import Literal, Optional, overloadfrom prisma.models import AgentNode, AgentPreset, IntegrationWebhookfrom prisma.types import (@@ -354,18 +354,10 @@ async def publish_webhook_event(event: WebhookEvent):)-async def listen_for_webhook_events(- webhook_id: str, event_type: Optional[str] = None-) -> AsyncGenerator[WebhookEvent, None]:- async for event in _webhook_event_bus.listen_events(- f"{webhook_id}/{event_type or '*'}"- ):- yield event--async def wait_for_webhook_event(- webhook_id: str, event_type: Optional[str] = None, timeout: Optional[float] = None+ webhook_id: str, event_type: str, timeout: Optional[float] = None) -> WebhookEvent | None:+ # Concrete event_type required: sharded pub/sub has no pattern support.return await _webhook_event_bus.wait_for_event(- f"{webhook_id}/{event_type or '*'}", timeout+ f"{webhook_id}/{event_type}", timeout)diff --git a/autogpt_platform/backend/backend/data/notification_bus.py b/autogpt_platform/backend/backend/data/notification_bus.pyindex 00b57516a9..f20e571ced 100644--- a/autogpt_platform/backend/backend/data/notification_bus.py+++ b/autogpt_platform/backend/backend/data/notification_bus.py@@ -33,8 +33,7 @@ class AsyncRedisNotificationEventBus(AsyncRedisEventBus[NotificationEvent]):async def publish(self, event: NotificationEvent) -> None:await self.publish_event(event, event.user_id)- async def listen(- self, user_id: str = "*"- ) -> AsyncGenerator[NotificationEvent, None]:+ async def listen(self, user_id: str) -> AsyncGenerator[NotificationEvent, None]:+ """Stream notifications for a specific user."""async for event in self.listen_events(user_id):yield eventdiff --git a/autogpt_platform/backend/backend/data/notification_bus_test.py b/autogpt_platform/backend/backend/data/notification_bus_test.pynew file mode 100644index 0000000000..b55b692947--- /dev/null+++ b/autogpt_platform/backend/backend/data/notification_bus_test.py@@ -0,0 +1,72 @@+"""Tests for AsyncRedisNotificationEventBus.++Covers the tiny delegation surface: publish → publish_event(user_id), listen+→ listen_events(user_id), and the payload serializer that ensures extra+fields survive the Redis round-trip.+"""++from unittest.mock import AsyncMock, patch++import pytest++from backend.api.model import NotificationPayload+from backend.data.notification_bus import (+ AsyncRedisNotificationEventBus,+ NotificationEvent,+)+++def test_notification_event_serializes_payload_including_extras():+ """``NotificationPayload`` allows extra fields; the bus serializer must+ preserve them. Dropping extras breaks feature payloads like CopilotCompletion."""+ payload = NotificationPayload(type="info", event="hey", extra_field="survive me")+ event = NotificationEvent(user_id="u", payload=payload)+ dumped = event.model_dump()+ assert dumped["payload"]["type"] == "info"+ assert dumped["payload"]["event"] == "hey"+ assert dumped["payload"]["extra_field"] == "survive me"+++async def test_publish_calls_publish_event_with_user_id_channel():+ """publish(event) → publish_event(event, channel_key=event.user_id)."""+ bus = AsyncRedisNotificationEventBus()+ event = NotificationEvent(+ user_id="user-42", payload=NotificationPayload(type="info", event="hi")+ )++ with patch.object(+ AsyncRedisNotificationEventBus, "publish_event", AsyncMock()+ ) as mock_pub:+ await bus.publish(event)++ mock_pub.assert_awaited_once()+ args = mock_pub.await_args.args+ # Pydantic may pass the event as a positional; regardless, user_id is the channel.+ assert args[-1] == "user-42"+++async def test_listen_delegates_to_listen_events_for_user():+ """listen(user_id) must subscribe on the per-user channel."""+ bus = AsyncRedisNotificationEventBus()++ captured: list[str] = []++ async def _listen_events(self, channel_key):+ captured.append(channel_key)+ if False:+ yield # pragma: no cover++ with patch.object(AsyncRedisNotificationEventBus, "listen_events", _listen_events):+ async for _ in bus.listen("user-42"):+ break # pragma: no cover — generator empty++ assert captured == ["user-42"]+++def test_event_bus_name_is_configured() -> None:+ """The notification bus uses a distinct namespace from the execution bus,+ so WS exec channels and notification channels never collide."""+ bus = AsyncRedisNotificationEventBus()+ assert bus.event_bus_name # non-empty, configured via Settingsdiff --git a/autogpt_platform/backend/backend/data/rabbitmq_test.py b/autogpt_platform/backend/backend/data/rabbitmq_test.pynew file mode 100644index 0000000000..6f666148ca--- /dev/null+++ b/autogpt_platform/backend/backend/data/rabbitmq_test.py@@ -0,0 +1,278 @@+"""Quorum-queue config assertions + mock-driven publish behaviour for+`AsyncRabbitMQ`. Live-broker scenarios live in `e2e_redis_rabbit_test.py`."""++from __future__ import annotations++from unittest.mock import AsyncMock, MagicMock, patch++import aio_pika+import pytest++from backend.copilot.executor.utils import (+ COPILOT_EXECUTION_EXCHANGE,+ COPILOT_EXECUTION_QUEUE_NAME,+ COPILOT_EXECUTION_ROUTING_KEY,+ create_copilot_queue_config,+)+from backend.data.rabbitmq import (+ AsyncRabbitMQ,+ Exchange,+ ExchangeType,+ Queue,+ RabbitMQConfig,+)+from backend.executor.utils import (+ GRAPH_EXECUTION_EXCHANGE,+ GRAPH_EXECUTION_QUEUE_NAME,+ GRAPH_EXECUTION_ROUTING_KEY,+ create_execution_queue_config,+)++# ---------- Quorum queue config: classic→quorum rollover guard ----------+++def test_graph_execution_queue_is_quorum() -> None:+ """Run queue must declare `x-queue-type=quorum` to survive a single+ broker-node outage (AUTOGPT-SERVER-8ST/SV/SW)."""+ cfg = create_execution_queue_config()+ run = next(q for q in cfg.queues if q.name == GRAPH_EXECUTION_QUEUE_NAME)+ assert run.arguments is not None+ assert run.arguments.get("x-queue-type") == "quorum"+ # _v2 suffix marks the rollover so the old-image consumer keeps draining+ # the unsuffixed classic queue during a rolling deploy.+ assert run.name.endswith("_v2")+ assert run.durable is True+ assert run.exchange is GRAPH_EXECUTION_EXCHANGE+++def test_graph_execution_cancel_queue_is_quorum() -> None:+ """Cancel queue must also be quorum — losing cancellations on a node+ flap is just as bad as losing runs."""+ cfg = create_execution_queue_config()+ cancel = next(q for q in cfg.queues if q.name.endswith("cancel_queue_v2"))+ assert cancel.arguments == {"x-queue-type": "quorum"}+++def test_copilot_execution_queue_is_quorum_with_consumer_timeout() -> None:+ """Copilot run queue must be quorum + carry a long consumer timeout+ matching the pod's graceful-shutdown window."""+ cfg = create_copilot_queue_config()+ run = next(q for q in cfg.queues if q.name == COPILOT_EXECUTION_QUEUE_NAME)+ assert run.arguments is not None+ assert run.arguments.get("x-queue-type") == "quorum"+ # Timeout must be in milliseconds and substantially larger than the+ # default 30-minute timeout so a 6-hour copilot turn doesn't get+ # cancelled by RabbitMQ mid-execution.+ timeout_ms = run.arguments.get("x-consumer-timeout")+ assert isinstance(timeout_ms, int)+ assert timeout_ms >= 60 * 60 * 1000 # at least 1 hour+++def test_copilot_cancel_queue_is_quorum() -> None:+ cfg = create_copilot_queue_config()+ cancel = next(q for q in cfg.queues if q.name.endswith("cancel_queue_v2"))+ assert cancel.arguments == {"x-queue-type": "quorum"}+++# ---------- AsyncRabbitMQ.publish_message: mock-driven behaviour ----------+++def _make_async_client(+ *, exchange_publish: AsyncMock | None = None+) -> tuple[AsyncRabbitMQ, MagicMock, MagicMock]:+ """Build an AsyncRabbitMQ wired to mock connection/channel/exchange.++ Returns the client, the mock channel, and the mock exchange so tests can+ assert on per-call arguments and tweak side_effects mid-flight.+ """+ cfg = RabbitMQConfig(+ vhost="/",+ exchanges=[+ Exchange(name="test_exchange", type=ExchangeType.DIRECT, durable=True)+ ],+ queues=[+ Queue(+ name="test_queue",+ exchange=Exchange(+ name="test_exchange", type=ExchangeType.DIRECT, durable=True+ ),+ routing_key="rk",+ arguments={"x-queue-type": "quorum"},+ )+ ],+ )+ client = AsyncRabbitMQ(cfg)++ fake_exchange = MagicMock()+ fake_exchange.publish = exchange_publish or AsyncMock()++ fake_channel = MagicMock()+ fake_channel.is_closed = False+ fake_channel.get_exchange = AsyncMock(return_value=fake_exchange)+ fake_channel.default_exchange = fake_exchange++ fake_connection = MagicMock()+ fake_connection.is_closed = False++ client._connection = fake_connection+ client._channel = fake_channel+ return client, fake_channel, fake_exchange+++async def test_publish_100_messages_to_quorum_queue_all_confirmed() -> None:+ """A healthy quorum queue publish path must confirm 100/100 publishes+ with no NACKs."""+ client, _, fake_exchange = _make_async_client()+ exchange = Exchange(name="test_exchange", type=ExchangeType.DIRECT)+ for i in range(100):+ await client.publish_message(+ routing_key="rk", message=f"msg-{i}", exchange=exchange+ )+ assert fake_exchange.publish.await_count == 100+ # Every call carried a persistent message — durable on the broker side.+ for call in fake_exchange.publish.await_args_list:+ msg = call.args[0]+ assert isinstance(msg, aio_pika.Message)+ assert msg.delivery_mode == aio_pika.DeliveryMode.PERSISTENT+++async def test_publish_retries_on_delivery_error_then_raises() -> None:+ """Broker-side NACK (DeliveryError) must trigger ``func_retry`` and then+ raise gracefully if every retry fails — never crash the publisher loop."""+ publish = AsyncMock(+ side_effect=aio_pika.exceptions.DeliveryError(message=None, frame=None)+ )+ client, _, fake_exchange = _make_async_client(exchange_publish=publish)+ exchange = Exchange(name="test_exchange", type=ExchangeType.DIRECT)++ with pytest.raises(aio_pika.exceptions.DeliveryError):+ await client.publish_message(+ routing_key="rk", message="will-nack", exchange=exchange+ )+ # ``func_retry`` is configured for 5 attempts in retry.py — assert the+ # publisher attempted at least once but bounded retries.+ assert fake_exchange.publish.await_count >= 1+ assert fake_exchange.publish.await_count <= 10 # generous upper bound+++async def test_publish_retries_after_one_transient_failure() -> None:+ """A single transient DeliveryError must NOT propagate — ``func_retry``+ retries and the second call succeeds."""+ publish = AsyncMock(+ side_effect=[+ aio_pika.exceptions.DeliveryError(message=None, frame=None),+ None, # second attempt succeeds+ ]+ )+ client, _, fake_exchange = _make_async_client(exchange_publish=publish)+ exchange = Exchange(name="test_exchange", type=ExchangeType.DIRECT)+ await client.publish_message(+ routing_key="rk", message="recovers", exchange=exchange+ )+ assert fake_exchange.publish.await_count == 2+++async def test_publish_reconnects_on_channel_invalid_state() -> None:+ """ChannelInvalidStateError must clear the channel and trigger a+ reconnect-and-retry — the publish_message wrapper handles this+ explicitly (see the except-clause in rabbitmq.py)."""+ publish = AsyncMock(+ side_effect=[+ aio_pika.exceptions.ChannelInvalidStateError("channel dead"),+ None,+ ]+ )+ client, fake_channel, fake_exchange = _make_async_client(exchange_publish=publish)+ exchange = Exchange(name="test_exchange", type=ExchangeType.DIRECT)++ # Patch connect() so the reconnect path doesn't try to hit a real broker.+ async def _fake_connect():+ # After reconnect the channel must be valid again.+ client._channel = fake_channel+ return None++ with patch.object(client, "connect", side_effect=_fake_connect):+ await client.publish_message(+ routing_key="rk", message="reconnects", exchange=exchange+ )+ # Two publish attempts: the failing one + the post-reconnect retry.+ assert fake_exchange.publish.await_count == 2+++# ---------- Dual-deploy: legacy classic + new quorum publisher in parallel ----------+++async def test_dual_deploy_publishes_to_legacy_and_new_queues_in_parallel() -> None:+ """Rolling-deploy window: old-image producer publishes to classic queue,+ new-image to `_v2` quorum queue — both must succeed independently."""+ legacy_client, _, legacy_exchange = _make_async_client()+ new_client, _, new_exchange = _make_async_client()++ legacy_routing = "copilot.run" # legacy producers used the same routing key+ new_routing = COPILOT_EXECUTION_ROUTING_KEY++ legacy_exch = Exchange(name="copilot_execution", type=ExchangeType.DIRECT)+ new_exch = Exchange(name=COPILOT_EXECUTION_EXCHANGE.name, type=ExchangeType.DIRECT)++ # Interleave 10 publishes from each producer — order doesn't matter.+ for i in range(10):+ await legacy_client.publish_message(+ routing_key=legacy_routing, message=f"legacy-{i}", exchange=legacy_exch+ )+ await new_client.publish_message(+ routing_key=new_routing, message=f"new-{i}", exchange=new_exch+ )++ assert legacy_exchange.publish.await_count == 10+ assert new_exchange.publish.await_count == 10++ # Each publisher's routing key landed on its own exchange — no crosstalk.+ for call in legacy_exchange.publish.await_args_list:+ assert call.kwargs.get("routing_key") == legacy_routing+ for call in new_exchange.publish.await_args_list:+ assert call.kwargs.get("routing_key") == new_routing+++async def test_dual_deploy_legacy_failure_does_not_affect_new_queue() -> None:+ """Legacy classic queue NACKing (AUTOGPT-SERVER-8ST) must not break+ publishes on the new `_v2` quorum queue."""+ legacy_publish = AsyncMock(+ side_effect=aio_pika.exceptions.DeliveryError(message=None, frame=None)+ )+ legacy_client, _, _ = _make_async_client(exchange_publish=legacy_publish)+ new_client, _, new_exchange = _make_async_client()++ legacy_exch = Exchange(name="copilot_execution", type=ExchangeType.DIRECT)+ new_exch = Exchange(name=COPILOT_EXECUTION_EXCHANGE.name, type=ExchangeType.DIRECT)++ # Legacy raises after retries — caller must catch it.+ with pytest.raises(aio_pika.exceptions.DeliveryError):+ await legacy_client.publish_message(+ routing_key="copilot.run", message="legacy-fail", exchange=legacy_exch+ )+ # New publisher continues to work — 5 successful publishes.+ for i in range(5):+ await new_client.publish_message(+ routing_key=COPILOT_EXECUTION_ROUTING_KEY,+ message=f"new-ok-{i}",+ exchange=new_exch,+ )+ assert new_exchange.publish.await_count == 5+++# ---------- Configuration sanity for downstream queues ----------+++def test_graph_execution_routing_key_constants() -> None:+ """Routing key + exchange wiring must stay aligned — guards against the+ classic→quorum migration accidentally also changing the routing key."""+ cfg = create_execution_queue_config()+ run = next(q for q in cfg.queues if q.name == GRAPH_EXECUTION_QUEUE_NAME)+ assert run.routing_key == GRAPH_EXECUTION_ROUTING_KEY+ assert GRAPH_EXECUTION_EXCHANGE in cfg.exchangesdiff --git a/autogpt_platform/backend/backend/data/redis_client.py b/autogpt_platform/backend/backend/data/redis_client.pyindex e3675370e5..048f63ea69 100644--- a/autogpt_platform/backend/backend/data/redis_client.py+++ b/autogpt_platform/backend/backend/data/redis_client.py@@ -1,85 +1,205 @@+import asyncioimport loggingimport osfrom dotenv import load_dotenvfrom redis import Redisfrom redis.asyncio import Redis as AsyncRedis+from redis.asyncio.cluster import ClusterNode as AsyncClusterNode+from redis.asyncio.cluster import RedisCluster as AsyncRedisCluster+from redis.cluster import ClusterNode, RedisCluster-from backend.util.cache import cached, thread_cached+from backend.util.cache import cachedfrom backend.util.retry import conn_retryload_dotenv()-HOST = os.getenv("REDIS_HOST", "localhost")-PORT = int(os.getenv("REDIS_PORT", "6379"))+# Prefer the cluster env vars so the cluster-only image can co-exist with+# old-image pods still reading REDIS_HOST during a rollout.+HOST = os.getenv("REDIS_CLUSTER_HOST") or os.getenv("REDIS_HOST", "localhost")+PORT = int(os.getenv("REDIS_CLUSTER_PORT") or os.getenv("REDIS_PORT", "6379"))PASSWORD = os.getenv("REDIS_PASSWORD", None)-# Default socket timeouts so a wedged Redis endpoint can't hang callers-# indefinitely — long-running code paths (cluster_lock refresh in particular)-# rely on these to fail-fast instead of blocking on no-response TCP. Override-# via env if a specific deployment needs a different budget.-#-# 30s matches the convention in ``backend.data.rabbitmq`` and leaves ~6x-# headroom over the largest ``xread(block=5000)`` wait in stream_registry.-# The connect timeout is shorter (5s) because initial connects should be-# fast; a slow connect usually means the endpoint is genuinely unreachable.+# Fail-fast on a wedged endpoint instead of blocking on no-response TCP.SOCKET_TIMEOUT = float(os.getenv("REDIS_SOCKET_TIMEOUT", "30"))SOCKET_CONNECT_TIMEOUT = float(os.getenv("REDIS_SOCKET_CONNECT_TIMEOUT", "5"))-# How often redis-py sends a PING on idle connections to detect half-open-# sockets; cheap and avoids waiting for the OS TCP keepalive (~2h default).+# PING on idle sockets to detect half-open connections without waiting for+# the OS TCP keepalive (~2h default).HEALTH_CHECK_INTERVAL = int(os.getenv("REDIS_HEALTH_CHECK_INTERVAL", "30"))+# Skip the HOST-pinning remap when each shard's announced hostname resolves+# directly (e.g. compose DNS names redis-0/redis-1/redis-2).+USE_ANNOUNCED_ADDRESS = os.getenv("REDIS_USE_ANNOUNCED_ADDRESS", "").lower() in (+ "1",+ "true",+ "yes",+)+logger = logging.getLogger(__name__)+# Aliases so call-sites don't care which class this is.+RedisClient = RedisCluster+AsyncRedisClient = AsyncRedisCluster+++def _address_remap(addr: tuple[str, int]) -> tuple[str, int]:+ """Pin each shard to the seed `HOST`, keep its announced port.++ Set `REDIS_USE_ANNOUNCED_ADDRESS=true` when the announced shard FQDNs+ resolve directly (e.g. each pod has its own DNS).+ """+ if USE_ANNOUNCED_ADDRESS:+ return addr+ _, port = addr+ return HOST, port+@conn_retry("Redis", "Acquiring connection")-def connect() -> Redis:- c = Redis(- host=HOST,- port=PORT,+def connect() -> RedisClient:+ c = RedisCluster(+ startup_nodes=[ClusterNode(HOST, PORT)],password=PASSWORD,decode_responses=True,socket_timeout=SOCKET_TIMEOUT,socket_connect_timeout=SOCKET_CONNECT_TIMEOUT,socket_keepalive=True,health_check_interval=HEALTH_CHECK_INTERVAL,+ address_remap=_address_remap,)- c.ping()+ # Close on PING failure so retries don't leak ClusterNodes (AUTOGPT-SERVER-8T1).+ try:+ c.ping()+ except Exception:+ try:+ c.close()+ except Exception:+ pass+ raisereturn c@conn_retry("Redis", "Releasing connection")def disconnect():get_redis().close()+ get_redis.cache_clear()@cached(ttl_seconds=3600)-def get_redis() -> Redis:+def get_redis() -> RedisClient:return connect()@conn_retry("AsyncRedis", "Acquiring connection")-async def connect_async() -> AsyncRedis:- c = AsyncRedis(- host=HOST,- port=PORT,+async def connect_async() -> AsyncRedisClient:+ c = AsyncRedisCluster(+ startup_nodes=[AsyncClusterNode(HOST, PORT)],password=PASSWORD,decode_responses=True,socket_timeout=SOCKET_TIMEOUT,socket_connect_timeout=SOCKET_CONNECT_TIMEOUT,socket_keepalive=True,health_check_interval=HEALTH_CHECK_INTERVAL,+ address_remap=_address_remap,)- await c.ping()+ # Close on PING failure so retries don't leak ClusterNodes (AUTOGPT-SERVER-8V6/8V4/8V3).+ try:+ await c.ping()+ except Exception:+ try:+ await c.close()+ except Exception:+ pass+ raisereturn c+# One AsyncRedisCluster per event loop: the client binds to the loop it was+# first awaited on, so a module-level singleton breaks across test loops.+_async_clients: dict[int, AsyncRedisCluster] = {}++@conn_retry("AsyncRedis", "Releasing connection")async def disconnect_async():- c = await get_redis_async()- await c.close()+ loop = asyncio.get_running_loop()+ c = _async_clients.pop(id(loop), None)+ if c is not None:+ await c.close()+++async def get_redis_async() -> AsyncRedisClient:+ loop = asyncio.get_running_loop()+ client = _async_clients.get(id(loop))+ if client is None:+ client = await connect_async()+ _async_clients[id(loop)] = client+ return client+++# Sharded pub/sub only delivers on the keyslot-owning shard; subscribers+# need a plain (Async)Redis connection pinned to that node.+++def resolve_shard_for_channel(channel: str) -> tuple[str, int]:+ """Return the ``(host, port)`` of the shard that owns the channel's keyslot.++ Applies the configured ``_address_remap`` so callers connect through the+ same address the cluster client uses.+ """+ cluster = get_redis()+ node = cluster.get_node_from_key(channel)+ if node is None:+ raise RuntimeError(f"No cluster node owns the keyslot for channel {channel!r}")+ return _address_remap((node.host, node.port))+++@conn_retry("RedisShardedPubSub", "Acquiring connection")+def connect_sharded_pubsub(channel: str) -> Redis:+ """Open a plain ``Redis`` connection pinned to the channel's owning shard."""+ host, port = resolve_shard_for_channel(channel)+ # socket_timeout=None: pubsub reads block indefinitely; a spurious+ # read timeout forces a reconnect whose PING races with subscribe-mode.+ c = Redis(+ host=host,+ port=port,+ password=PASSWORD,+ decode_responses=True,+ socket_timeout=None,+ socket_connect_timeout=SOCKET_CONNECT_TIMEOUT,+ socket_keepalive=True,+ health_check_interval=HEALTH_CHECK_INTERVAL,+ )+ try:+ c.ping()+ except Exception:+ try:+ c.close()+ except Exception:+ pass+ raise+ return c-@thread_cached-async def get_redis_async() -> AsyncRedis:- return await connect_async()+@conn_retry("AsyncRedisShardedPubSub", "Acquiring connection")+async def connect_sharded_pubsub_async(channel: str) -> AsyncRedis:+ """Async variant of :func:`connect_sharded_pubsub`."""+ host, port = resolve_shard_for_channel(channel)+ # socket_timeout=None: see ``connect_sharded_pubsub``.+ c = AsyncRedis(+ host=host,+ port=port,+ password=PASSWORD,+ decode_responses=True,+ socket_timeout=None,+ socket_connect_timeout=SOCKET_CONNECT_TIMEOUT,+ socket_keepalive=True,+ health_check_interval=HEALTH_CHECK_INTERVAL,+ )+ try:+ await c.ping()+ except Exception:+ try:+ await c.close()+ except Exception:+ pass+ raise+ return cdiff --git a/autogpt_platform/backend/backend/data/redis_client_test.py b/autogpt_platform/backend/backend/data/redis_client_test.pynew file mode 100644index 0000000000..ef2a6a1e25--- /dev/null+++ b/autogpt_platform/backend/backend/data/redis_client_test.py@@ -0,0 +1,599 @@+"""Unit tests for the cluster-only Redis client in ``redis_client``.++Patches the redis-py constructors + ``ping()`` so no real Redis is needed.+"""++from unittest.mock import AsyncMock, MagicMock, patch++import pytest+from redis.asyncio.cluster import RedisCluster as AsyncRedisCluster+from redis.cluster import RedisCluster++import backend.data.redis_client as redis_client++[email protected](autouse=True)+def _reset_module_caches() -> None:+ """Flush cached singletons between tests so each test sees a fresh connect."""+ redis_client.get_redis.cache_clear()+ redis_client._async_clients.clear()+++def test_connect_builds_redis_cluster() -> None:+ with patch.object(redis_client, "RedisCluster", autospec=True) as mock_cluster:+ mock_cluster.return_value = MagicMock(spec=RedisCluster)+ client = redis_client.connect()++ mock_cluster.assert_called_once()+ kwargs = mock_cluster.call_args.kwargs+ assert kwargs["password"] == redis_client.PASSWORD+ assert kwargs["decode_responses"] is True+ assert kwargs["socket_timeout"] == redis_client.SOCKET_TIMEOUT+ assert kwargs["socket_connect_timeout"] == redis_client.SOCKET_CONNECT_TIMEOUT+ assert kwargs["socket_keepalive"] is True+ assert kwargs["health_check_interval"] == redis_client.HEALTH_CHECK_INTERVAL+ assert kwargs["address_remap"] is redis_client._address_remap+ startup = kwargs["startup_nodes"]+ assert len(startup) == 1+ # ClusterNode resolves "localhost" → "127.0.0.1" internally; both are+ # valid representations of the configured host.+ assert startup[0].host in {redis_client.HOST, "127.0.0.1"}+ assert startup[0].port == redis_client.PORT+ client.ping.assert_called_once()+++def test_address_remap_pins_host_and_preserves_port() -> None:+ """Default remap rewrites announced shard host to the configured seed."""+ with patch.object(redis_client, "USE_ANNOUNCED_ADDRESS", False):+ assert redis_client._address_remap(("any-other-host", 6380)) == (+ redis_client.HOST,+ 6380,+ )+++def test_address_remap_passthrough_when_use_announced_address() -> None:+ """When announced addresses resolve directly, remap leaves them alone."""+ with patch.object(redis_client, "USE_ANNOUNCED_ADDRESS", True):+ assert redis_client._address_remap(("redis-1", 17001)) == ("redis-1", 17001)+++async def test_connect_async_builds_async_redis_cluster() -> None:+ with patch.object(redis_client, "AsyncRedisCluster", autospec=True) as mock_cluster:+ fake = MagicMock(spec=AsyncRedisCluster)+ fake.ping = AsyncMock()+ mock_cluster.return_value = fake+ client = await redis_client.connect_async()++ mock_cluster.assert_called_once()+ kwargs = mock_cluster.call_args.kwargs+ assert kwargs["password"] == redis_client.PASSWORD+ assert kwargs["decode_responses"] is True+ assert kwargs["socket_timeout"] == redis_client.SOCKET_TIMEOUT+ assert kwargs["socket_connect_timeout"] == redis_client.SOCKET_CONNECT_TIMEOUT+ assert kwargs["socket_keepalive"] is True+ assert kwargs["health_check_interval"] == redis_client.HEALTH_CHECK_INTERVAL+ assert kwargs["address_remap"] is redis_client._address_remap+ startup = kwargs["startup_nodes"]+ assert len(startup) == 1+ assert startup[0].host in {redis_client.HOST, "127.0.0.1"}+ assert startup[0].port == redis_client.PORT+ client.ping.assert_awaited_once()+++def test_get_redis_caches_connect() -> None:+ with patch.object(redis_client, "connect", autospec=True) as mock_connect:+ mock_connect.return_value = MagicMock(spec=RedisCluster)+ client_a = redis_client.get_redis()+ client_b = redis_client.get_redis()++ assert client_a is client_b+ mock_connect.assert_called_once()+++async def test_get_redis_async_caches_connect() -> None:+ with patch.object(redis_client, "connect_async", autospec=True) as mock_conn:+ fake = MagicMock(spec=AsyncRedisCluster)+ mock_conn.return_value = fake+ a = await redis_client.get_redis_async()+ b = await redis_client.get_redis_async()++ assert a is b+ mock_conn.assert_called_once()+++def test_disconnect_closes_cached_client() -> None:+ with patch.object(redis_client, "connect", autospec=True) as mock_connect:+ fake = MagicMock(spec=RedisCluster)+ mock_connect.return_value = fake+ redis_client.get_redis()+ redis_client.disconnect()++ fake.close.assert_called_once()+++async def test_disconnect_async_closes_cached_client() -> None:+ with patch.object(redis_client, "connect_async", autospec=True) as mock_connect:+ fake = MagicMock(spec=AsyncRedisCluster)+ fake.close = AsyncMock()+ mock_connect.return_value = fake+ await redis_client.get_redis_async()+ await redis_client.disconnect_async()++ fake.close.assert_awaited_once()+ assert redis_client._async_clients == {}+++async def test_disconnect_async_no_cached_client_is_noop() -> None:+ with patch.object(redis_client, "connect_async", autospec=True) as mock_connect:+ await redis_client.disconnect_async()+ mock_connect.assert_not_called()+++# Sharded pub/sub end-to-end against the local 3-shard compose cluster.+# Skipped when no cluster is reachable so CI without docker doesn't flap.+++def _has_live_cluster() -> bool:+ try:+ c = redis_client.connect()+ except Exception: # noqa: BLE001 — any connect failure → skip the test+ return False+ try:+ c.close()+ except Exception:+ pass+ return True+++ not _has_live_cluster(),+ reason="local redis cluster not reachable; skip sharded pub/sub integration",+)+def test_sharded_pubsub_end_to_end_sync() -> None:+ """SPUBLISH → SSUBSCRIBE round-trip via the sync cluster client. Uses+ per-node `get_message` because redis-py 6.x's+ `ClusterPubSub.get_sharded_message(ignore_subscribe_messages=True)`+ drops every message, not just the subscribe confirmation."""+ redis_client.get_redis.cache_clear()+ cluster = redis_client.get_redis()+ channel = "pr12900:sharded-pubsub:integration"+ ps = cluster.pubsub()+ try:+ ps.ssubscribe(channel)+ assert cluster.spublish(channel, "hello") >= 1++ # Exactly one node is subscribed (the keyslot owner); read from it.+ assert len(ps.node_pubsub_mapping) == 1+ node_ps = next(iter(ps.node_pubsub_mapping.values()))+ # First message is the ssubscribe confirmation, second is our payload.+ confirm = node_ps.get_message(timeout=2.0)+ assert confirm is not None and confirm["type"] == "ssubscribe"+ received = node_ps.get_message(timeout=5.0)+ assert received is not None and received["type"] == "smessage"+ assert received["data"] == "hello"+ finally:+ try:+ ps.sunsubscribe(channel)+ except Exception:+ pass+ ps.close()+ redis_client.disconnect()+++ not _has_live_cluster(),+ reason="local redis cluster not reachable; skip sharded pub/sub integration",+)+async def test_sharded_spublish_end_to_end_async() -> None:+ """Async cluster client routes SPUBLISH via ``execute_command``+ because redis-py 6.x has no async ``spublish()`` wrapper."""+ redis_client._async_clients.clear()+ cluster = await redis_client.get_redis_async()+ try:+ res = await cluster.execute_command(+ "SPUBLISH", "pr12900:sharded-pubsub:async", "ping"+ )+ # No subscribers — delivered count is 0, but the command must succeed+ # (i.e. not raise MOVED/ASK or routing errors).+ assert isinstance(res, int)+ finally:+ await redis_client.disconnect_async()+++# ---------- Sharded pub/sub: unit tests with mocks ----------+++def test_connect_sharded_pubsub_pins_host_and_disables_socket_timeout() -> None:+ """`socket_timeout=None` on the pubsub socket: a spurious read timeout+ forces a reconnect whose PING races with subscribe-mode."""+ with (+ patch.object(+ redis_client,+ "resolve_shard_for_channel",+ return_value=("shard-host", 7001),+ ),+ patch.object(redis_client, "Redis", autospec=True) as mock_redis,+ ):+ fake_client = MagicMock()+ mock_redis.return_value = fake_client+ client = redis_client.connect_sharded_pubsub("chan")++ mock_redis.assert_called_once()+ kwargs = mock_redis.call_args.kwargs+ # Pinned to the shard's remapped address.+ assert kwargs["host"] == "shard-host"+ assert kwargs["port"] == 7001+ # socket_timeout MUST be None for pubsub — see docstring in redis_client.py.+ assert kwargs["socket_timeout"] is None+ # Idle keepalive + health-check still intact.+ assert kwargs["socket_keepalive"] is True+ assert kwargs["health_check_interval"] == redis_client.HEALTH_CHECK_INTERVAL+ # connect() must PING before returning.+ client.ping.assert_called_once()+++async def test_connect_sharded_pubsub_async_disables_socket_timeout() -> None:+ """Async sibling of ``test_connect_sharded_pubsub_pins_host...``. Same+ invariant: socket_timeout=None."""+ with (+ patch.object(+ redis_client,+ "resolve_shard_for_channel",+ return_value=("shard-host", 7001),+ ),+ patch.object(redis_client, "AsyncRedis", autospec=True) as mock_redis,+ ):+ fake_client = MagicMock()+ fake_client.ping = AsyncMock()+ mock_redis.return_value = fake_client+ client = await redis_client.connect_sharded_pubsub_async("chan")++ kwargs = mock_redis.call_args.kwargs+ assert kwargs["host"] == "shard-host"+ assert kwargs["port"] == 7001+ assert kwargs["socket_timeout"] is None+ assert kwargs["socket_keepalive"] is True+ assert kwargs["health_check_interval"] == redis_client.HEALTH_CHECK_INTERVAL+ client.ping.assert_awaited_once()+++def test_resolve_shard_for_channel_applies_address_remap() -> None:+ """The resolver must run ``_address_remap`` on the announced address so+ callers connect through the same address the cluster client uses."""+ cluster = MagicMock()+ node = MagicMock()+ node.host = "announced-host"+ node.port = 17001+ cluster.get_node_from_key.return_value = node++ with (+ patch.object(redis_client, "get_redis", return_value=cluster),+ patch.object(redis_client, "USE_ANNOUNCED_ADDRESS", False),+ ):+ host, port = redis_client.resolve_shard_for_channel("chan")++ # Remap pins the host to the seed, keeps the announced port.+ assert host == redis_client.HOST+ assert port == 17001+++def test_resolve_shard_for_channel_raises_when_no_node_owns_keyslot() -> None:+ """Missing cluster node → explicit RuntimeError, not a silent None deref."""+ cluster = MagicMock()+ cluster.get_node_from_key.return_value = None++ with patch.object(redis_client, "get_redis", return_value=cluster):+ with pytest.raises(RuntimeError, match="No cluster node"):+ redis_client.resolve_shard_for_channel("chan")+++def test_resolve_shard_for_channel_passthrough_with_announced_flag() -> None:+ """When REDIS_USE_ANNOUNCED_ADDRESS is on, resolver returns the announced+ address verbatim — no HOST override."""+ cluster = MagicMock()+ node = MagicMock()+ node.host = "redis-2"+ node.port = 17002+ cluster.get_node_from_key.return_value = node++ with (+ patch.object(redis_client, "get_redis", return_value=cluster),+ patch.object(redis_client, "USE_ANNOUNCED_ADDRESS", True),+ ):+ host, port = redis_client.resolve_shard_for_channel("chan")++ assert (host, port) == ("redis-2", 17002)+++def test_health_check_interval_is_30s_default() -> None:+ """Idle PING interval must be <=30s so half-open pubsub sockets don't+ wait for the OS TCP keepalive (~2h)."""+ assert redis_client.HEALTH_CHECK_INTERVAL <= 30+++def test_connect_sets_health_check_interval() -> None:+ """The cluster client must propagate health_check_interval to each node+ pool — otherwise idle cluster sockets go stale."""+ with patch.object(redis_client, "RedisCluster", autospec=True) as mock_cluster:+ mock_cluster.return_value = MagicMock(spec=RedisCluster)+ redis_client.connect()+ kwargs = mock_cluster.call_args.kwargs+ assert kwargs["health_check_interval"] == redis_client.HEALTH_CHECK_INTERVAL+ assert kwargs["health_check_interval"] > 0+++# ---------- K8s same-port shard collapse regression (AUTOGPT-SERVER-8SX) ----------+++def test_k8s_shard_collapse_with_announced_address_off_routes_all_to_seed() -> None:+ """In K8s every shard serves on port 6379 behind the seed service, so the+ default `_address_remap` collapses all shards to `(HOST, 6379)` — the+ AUTOGPT-SERVER-8SX bug. Fix: `REDIS_USE_ANNOUNCED_ADDRESS=true`."""+ cluster = MagicMock()+ # 3 shards, each owning a distinct hash slot, but every pod serves on+ # 6379 in K8s — exactly the production topology.+ nodes_by_channel = {+ "{ch-a}/x": MagicMock(host="redis-cluster-redis-0", port=6379),+ "{ch-b}/y": MagicMock(host="redis-cluster-redis-1", port=6379),+ "{ch-c}/z": MagicMock(host="redis-cluster-redis-2", port=6379),+ }+ cluster.get_node_from_key.side_effect = lambda c: nodes_by_channel[c]++ with (+ patch.object(redis_client, "get_redis", return_value=cluster),+ patch.object(redis_client, "USE_ANNOUNCED_ADDRESS", False),+ patch.object(redis_client, "HOST", "redis-dev-seed"),+ ):+ endpoints = {+ channel: redis_client.resolve_shard_for_channel(channel)+ for channel in nodes_by_channel+ }++ # The bug: every shard resolves to the same seed:port endpoint.+ assert len(set(endpoints.values())) == 1, (+ f"Expected the K8s shard-collapse bug, got {endpoints!r}. "+ "If this test is failing it means _address_remap behaviour changed "+ "and the AUTOGPT-SERVER-8SX regression note in this file needs review."+ )+ assert all(ep == ("redis-dev-seed", 6379) for ep in endpoints.values())+++def test_k8s_shard_collapse_fixed_with_announced_address_on() -> None:+ """With `REDIS_USE_ANNOUNCED_ADDRESS=true`, each shard's announced FQDN+ passes through, so distinct slots resolve to distinct endpoints."""+ cluster = MagicMock()+ nodes_by_channel = {+ "{ch-a}/x": MagicMock(host="redis-cluster-redis-0", port=6379),+ "{ch-b}/y": MagicMock(host="redis-cluster-redis-1", port=6379),+ "{ch-c}/z": MagicMock(host="redis-cluster-redis-2", port=6379),+ }+ cluster.get_node_from_key.side_effect = lambda c: nodes_by_channel[c]++ with (+ patch.object(redis_client, "get_redis", return_value=cluster),+ patch.object(redis_client, "USE_ANNOUNCED_ADDRESS", True),+ patch.object(redis_client, "HOST", "redis-dev-seed"),+ ):+ endpoints = {+ channel: redis_client.resolve_shard_for_channel(channel)+ for channel in nodes_by_channel+ }++ # Each shard maps to a distinct endpoint — sharded pubsub can route+ # SSUBSCRIBE to the slot owner.+ assert len(set(endpoints.values())) == 3+ assert endpoints["{ch-a}/x"] == ("redis-cluster-redis-0", 6379)+ assert endpoints["{ch-b}/y"] == ("redis-cluster-redis-1", 6379)+ assert endpoints["{ch-c}/z"] == ("redis-cluster-redis-2", 6379)+++def test_local_compose_remap_keeps_distinct_ports_per_shard() -> None:+ """Local docker-compose announces distinct ports per shard, so the+ `(host, port)` tuple stays distinct even with `HOST` pinned to seed."""+ cluster = MagicMock()+ nodes_by_channel = {+ "{ch-a}/x": MagicMock(host="redis-0", port=17000),+ "{ch-b}/y": MagicMock(host="redis-1", port=17001),+ "{ch-c}/z": MagicMock(host="redis-2", port=17002),+ }+ cluster.get_node_from_key.side_effect = lambda c: nodes_by_channel[c]++ with (+ patch.object(redis_client, "get_redis", return_value=cluster),+ patch.object(redis_client, "USE_ANNOUNCED_ADDRESS", False),+ patch.object(redis_client, "HOST", "localhost"),+ ):+ endpoints = {+ channel: redis_client.resolve_shard_for_channel(channel)+ for channel in nodes_by_channel+ }++ # Distinct ports → distinct endpoints even after remap pins the host.+ assert len(set(endpoints.values())) == 3+ assert endpoints["{ch-a}/x"] == ("localhost", 17000)+ assert endpoints["{ch-b}/y"] == ("localhost", 17001)+ assert endpoints["{ch-c}/z"] == ("localhost", 17002)+++# ---------- Sharded pub/sub: multi-shard integration on the live cluster ----------+++def _channel_owner(channel: str) -> tuple[str, int]:+ """Resolve the slot owner for ``channel`` via the live cluster client."""+ cluster = redis_client.get_redis()+ node = cluster.get_node_from_key(channel)+ assert node is not None, f"no slot owner for {channel!r}"+ return node.host, node.port+++def _channels_on_distinct_shards(n: int = 3) -> list[str]:+ """Build N hash-tagged channels each mapping to a distinct shard."""+ seen: dict[tuple[str, int], str] = {}+ for tag_id in range(2000):+ chan = "{u" + str(tag_id) + "/g}/exec/e"+ owner = _channel_owner(chan)+ seen.setdefault(owner, chan)+ if len(seen) >= n:+ break+ assert len(seen) >= n, f"could only cover {len(seen)} shards"+ return list(seen.values())[:n]+++ not _has_live_cluster(),+ reason="local redis cluster not reachable; skip multi-shard integration",+)+def test_resolve_shard_for_channel_lands_on_distinct_shards() -> None:+ """3 hash-tagged channels resolve to 3 different shards (slot-distribution)."""+ redis_client.get_redis.cache_clear()+ try:+ channels = _channels_on_distinct_shards(3)+ endpoints = {ch: redis_client.resolve_shard_for_channel(ch) for ch in channels}+ # Three channels → three distinct (host, port) endpoints.+ assert len(set(endpoints.values())) == 3+ finally:+ redis_client.disconnect()+++ not _has_live_cluster(),+ reason="local redis cluster not reachable; skip multi-shard integration",+)+def test_sharded_pubsub_concurrent_subscribers_on_three_shards() -> None:+ """SSUBSCRIBE on three channels owned by three different shards, then+ SPUBLISH to each — every payload must land on its subscriber."""+ redis_client.get_redis.cache_clear()+ cluster = redis_client.get_redis()+ try:+ channels = _channels_on_distinct_shards(3)+ # Subscribe via the cluster client so redis-py's per-node pubsub+ # mapping handles the sharded routing for us.+ ps = cluster.pubsub()+ try:+ for ch in channels:+ ps.ssubscribe(ch)+ # The cluster client opens one node-pubsub per shard owner — three+ # channels on three shards must produce three distinct node clients.+ assert len(ps.node_pubsub_mapping) == 3, (+ "Expected SSUBSCRIBE on 3 channels owned by 3 distinct shards "+ f"to open 3 node-pubsubs, got {len(ps.node_pubsub_mapping)}"+ )+ # Publish to each channel and verify each reaches the right node.+ for i, ch in enumerate(channels):+ assert cluster.spublish(ch, f"payload-{i}") >= 1+ # Drain ssubscribe confirmations + smessages from every node.+ received: dict[str, str] = {}+ for node_ps in ps.node_pubsub_mapping.values():+ # First message per node is the ssubscribe confirm; subsequent+ # smessages carry the test payloads.+ for _ in range(4): # confirm + at most 1 payload per shard+ msg = node_ps.get_message(timeout=2.0)+ if msg is None:+ break+ if msg["type"] == "smessage":+ received[msg["channel"]] = msg["data"]+ for i, ch in enumerate(channels):+ assert ch in received, f"channel {ch!r} got no message"+ assert received[ch] == f"payload-{i}"+ finally:+ for ch in channels:+ try:+ ps.sunsubscribe(ch)+ except Exception:+ pass+ ps.close()+ finally:+ redis_client.disconnect()+++ not _has_live_cluster(),+ reason="local redis cluster not reachable; skip multi-shard integration",+)+def test_sharded_pubsub_idle_subscriber_survives_health_check_window() -> None:+ """An SSUBSCRIBE connection must survive an idle window longer than+ `HEALTH_CHECK_INTERVAL` — uses `+5s` to provoke at least one health check."""+ import time as _time++ redis_client.get_redis.cache_clear()+ cluster = redis_client.get_redis()+ channel = "{idle-test}/exec/e"+ client = redis_client.connect_sharded_pubsub(channel)+ ps = client.pubsub()+ try:+ ps.ssubscribe(channel)+ confirm = ps.get_message(timeout=2.0)+ assert confirm is not None and confirm["type"] == "ssubscribe"++ # Idle window — must exceed health_check_interval at least once.+ idle_seconds = redis_client.HEALTH_CHECK_INTERVAL + 5+ _time.sleep(idle_seconds)++ # After idling, publish + receive should still work.+ assert cluster.spublish(channel, "post-idle") >= 1+ msg = ps.get_message(timeout=5.0)+ assert msg is not None and msg["type"] == "smessage"+ assert msg["data"] == "post-idle"+ finally:+ try:+ ps.sunsubscribe(channel)+ except Exception:+ pass+ ps.close()+ client.close()+ redis_client.disconnect()+++ not _has_live_cluster(),+ reason="local redis cluster not reachable; skip multi-shard integration",+)+def test_sharded_pubsub_reconnect_after_forced_disconnect() -> None:+ """Subscriber reconnect after a forced disconnect — close socket, open+ a fresh one, and verify new SPUBLISH events still arrive."""+ redis_client.get_redis.cache_clear()+ cluster = redis_client.get_redis()+ channel = "{reconnect-test}/exec/e"++ # Round 1: subscribe, receive one payload, then close everything.+ client = redis_client.connect_sharded_pubsub(channel)+ ps = client.pubsub()+ try:+ ps.ssubscribe(channel)+ ps.get_message(timeout=2.0) # ssubscribe confirmation+ assert cluster.spublish(channel, "before-restart") >= 1+ msg = ps.get_message(timeout=5.0)+ assert msg is not None and msg["data"] == "before-restart"+ finally:+ try:+ ps.sunsubscribe(channel)+ except Exception:+ pass+ ps.close()+ client.close()++ # Round 2: a fresh subscriber on the same channel — same routing,+ # different socket. This exercises the reconnect-and-resubscribe path+ # the conn_manager runs after a network blip.+ client2 = redis_client.connect_sharded_pubsub(channel)+ ps2 = client2.pubsub()+ try:+ ps2.ssubscribe(channel)+ ps2.get_message(timeout=2.0)+ assert cluster.spublish(channel, "after-restart") >= 1+ msg = ps2.get_message(timeout=5.0)+ assert msg is not None and msg["data"] == "after-restart"+ finally:+ try:+ ps2.sunsubscribe(channel)+ except Exception:+ pass+ ps2.close()+ client2.close()+ redis_client.disconnect()diff --git a/autogpt_platform/backend/backend/data/redis_helpers.py b/autogpt_platform/backend/backend/data/redis_helpers.pyindex f082d45e38..b549804259 100644--- a/autogpt_platform/backend/backend/data/redis_helpers.py+++ b/autogpt_platform/backend/backend/data/redis_helpers.py@@ -22,8 +22,7 @@ this module can cover.from typing import Any, cast-from redis import Redis-from redis.asyncio import Redis as AsyncRedis+from backend.data.redis_client import AsyncRedisClient, RedisClient# ---------------------------------------------------------------------------# Lua scripts — registered centrally so there is exactly ONE authoritative@@ -70,7 +69,7 @@ return redis.call('LLEN', KEYS[2])async def incr_with_ttl(- redis: AsyncRedis,+ redis: AsyncRedisClient,key: str,ttl_seconds: int,*,@@ -106,7 +105,7 @@ async def incr_with_ttl(def incr_with_ttl_sync(- redis: Redis,+ redis: RedisClient,key: str,ttl_seconds: int,*,@@ -124,7 +123,7 @@ def incr_with_ttl_sync(async def capped_rpush(- redis: AsyncRedis,+ redis: AsyncRedisClient,key: str,value: str,*,@@ -151,7 +150,7 @@ async def capped_rpush(async def capped_rpush_if_hash_field(- redis: AsyncRedis,+ redis: AsyncRedisClient,*,hash_key: str,hash_field: str,@@ -185,7 +184,7 @@ async def capped_rpush_if_hash_field(async def hash_compare_and_set(- redis: AsyncRedis,+ redis: AsyncRedisClient,key: str,field: str,*,diff --git a/autogpt_platform/backend/backend/executor/billing.py b/autogpt_platform/backend/backend/executor/billing.pyindex e8979ccba3..1969fc4efa 100644--- a/autogpt_platform/backend/backend/executor/billing.py+++ b/autogpt_platform/backend/backend/executor/billing.py@@ -64,9 +64,12 @@ async def clear_insufficient_funds_notifications(user_id: str) -> int:redis_client = await redis.get_redis_async()pattern = f"{INSUFFICIENT_FUNDS_NOTIFIED_PREFIX}:{user_id}:*"keys = [key async for key in redis_client.scan_iter(match=pattern)]- if keys:- return await redis_client.delete(*keys)- return 0+ # Keys here span multiple graph IDs and therefore multiple cluster+ # slots — a bulk DELETE would raise CROSSSLOT, so delete per key.+ deleted = 0+ for key in keys:+ deleted += await redis_client.delete(key)+ return deletedexcept Exception as e:logger.warning(f"Failed to clear insufficient funds notification flags for user "diff --git a/autogpt_platform/backend/backend/executor/cluster_lock.py b/autogpt_platform/backend/backend/executor/cluster_lock.pyindex 9fe8b744c4..de9305350a 100644--- a/autogpt_platform/backend/backend/executor/cluster_lock.py+++ b/autogpt_platform/backend/backend/executor/cluster_lock.py@@ -7,16 +7,12 @@ import timefrom typing import TYPE_CHECKING, Any, castif TYPE_CHECKING:- from redis import Redis- from redis.asyncio import Redis as AsyncRedis+ from backend.data.redis_client import AsyncRedisClient, RedisClientlogger = logging.getLogger(__name__)-# Lua CAS release: only delete the key if the stored value still matches our-# owner_id. Returns 1 on delete, 0 on no-op. This makes release() safe against-# the race where an external caller (e.g. mark_session_completed's force-release)-# deletes our key and a new owner acquires it before our release() fires — without-# the CAS guard, release() would wipe the successor's valid lock.+# CAS release: DEL only when the stored owner still matches — guards against+# wiping a successor's lock after an external force-release._RELEASE_LUA = ("if redis.call('get', KEYS[1]) == ARGV[1] then ""return redis.call('del', KEYS[1]) "@@ -27,7 +23,9 @@ _RELEASE_LUA = (class ClusterLock:"""Simple Redis-based distributed lock for preventing duplicate execution."""- def __init__(self, redis: "Redis", key: str, owner_id: str, timeout: int = 300):+ def __init__(+ self, redis: "RedisClient", key: str, owner_id: str, timeout: int = 300+ ):self.redis = redisself.key = keyself.owner_id = owner_id@@ -150,7 +148,7 @@ class AsyncClusterLock:"""Async Redis-based distributed lock for preventing duplicate execution."""def __init__(- self, redis: "AsyncRedis", key: str, owner_id: str, timeout: int = 300+ self, redis: "AsyncRedisClient", key: str, owner_id: str, timeout: int = 300):self.redis = redisself.key = keydiff --git a/autogpt_platform/backend/backend/executor/cluster_lock_test.py b/autogpt_platform/backend/backend/executor/cluster_lock_test.pyindex 5491c51cad..137209e410 100644--- a/autogpt_platform/backend/backend/executor/cluster_lock_test.py+++ b/autogpt_platform/backend/backend/executor/cluster_lock_test.py@@ -21,15 +21,21 @@ logger = logging.getLogger(__name__)@pytest.fixturedef redis_client():- """Get Redis client for testing using same config as backend."""- from backend.data.redis_client import HOST, PASSWORD, PORT+ """Get Redis client for testing using same config as backend.- # Use same config as backend but without decode_responses since ClusterLock needs raw bytes- client = redis.Redis(- host=HOST,- port=PORT,+ Uses ``RedisCluster`` (not plain ``Redis``) so tests exercise the same+ cluster-aware routing as prod — a plain client against a sharded+ cluster bounces on ``MOVED`` for any key hashing to a non-owned slot.+ """+ from redis.cluster import ClusterNode, RedisCluster++ from backend.data.redis_client import HOST, PASSWORD, PORT, _address_remap++ client = RedisCluster(+ startup_nodes=[ClusterNode(HOST, PORT)],password=PASSWORD,decode_responses=False, # ClusterLock needs raw bytes for ownership verification+ address_remap=_address_remap,)# Clean up any existing test keys@@ -333,7 +339,7 @@ class TestClusterLockErrorHandling:def test_redis_connection_failure_on_acquire(self, lock_key, owner_id):"""Test graceful handling when Redis is unavailable during acquisition."""- # Use invalid Redis connection+ # INTENTIONAL: plain Redis client here is the test subject — validates cluster_lock rejects non-cluster deploys.bad_redis = redis.Redis(host="invalid_host", port=1234, socket_connect_timeout=1)@@ -353,7 +359,7 @@ class TestClusterLockErrorHandling:# Acquire normallyassert lock.try_acquire() == owner_id- # Replace Redis client with failing one+ # INTENTIONAL: plain Redis client here is the test subject — validates cluster_lock rejects non-cluster deploys.lock.redis = redis.Redis(host="invalid_host", port=1234, socket_connect_timeout=1)@@ -508,6 +514,7 @@ class TestClusterLockRealWorldScenarios:# Simulate Redis becoming unavailableoriginal_redis = lock.redis+ # INTENTIONAL: plain Redis client here is the test subject — validates cluster_lock rejects non-cluster deploys.lock.redis = redis.Redis(host="invalid_host",port=1234,diff --git a/autogpt_platform/backend/backend/executor/manager_insufficient_funds_test.py b/autogpt_platform/backend/backend/executor/manager_insufficient_funds_test.pyindex ddbb4e0e1c..64cab1507a 100644--- a/autogpt_platform/backend/backend/executor/manager_insufficient_funds_test.py+++ b/autogpt_platform/backend/backend/executor/manager_insufficient_funds_test.py@@ -237,8 +237,8 @@ async def test_clear_insufficient_funds_notifications(server: SpinTestServer):f"{INSUFFICIENT_FUNDS_NOTIFIED_PREFIX}:{user_id}:graph-3",]mock_redis_client.scan_iter.return_value = async_iter(mock_keys)- # delete is awaited, so use AsyncMock- mock_redis_client.delete = AsyncMock(return_value=3)+ # Each per-key DELETE returns 1 (one key deleted per call).+ mock_redis_client.delete = AsyncMock(return_value=1)# Clear notificationsresult = await clear_insufficient_funds_notifications(user_id)@@ -247,11 +247,15 @@ async def test_clear_insufficient_funds_notifications(server: SpinTestServer):expected_pattern = f"{INSUFFICIENT_FUNDS_NOTIFIED_PREFIX}:{user_id}:*"mock_redis_client.scan_iter.assert_called_once_with(match=expected_pattern)- # Verify delete was called with all keys- mock_redis_client.delete.assert_called_once_with(*mock_keys)+ # Keys span multiple graph IDs → different cluster slots, so DELETE+ # is called once per key (not a single bulk DELETE).+ assert mock_redis_client.delete.call_count == len(mock_keys)+ assert {+ call.args[0] for call in mock_redis_client.delete.call_args_list+ } == set(mock_keys)- # Verify return value- assert result == 3+ # Verify return value — sum of per-key DELETEs+ assert result == len(mock_keys)@pytest.mark.asyncio(loop_scope="session")diff --git a/autogpt_platform/backend/backend/executor/utils.py b/autogpt_platform/backend/backend/executor/utils.pyindex e14e410edb..fb28bb4cfe 100644--- a/autogpt_platform/backend/backend/executor/utils.py+++ b/autogpt_platform/backend/backend/executor/utils.py@@ -902,7 +902,9 @@ GRAPH_EXECUTION_EXCHANGE = Exchange(durable=True,auto_delete=False,)-GRAPH_EXECUTION_QUEUE_NAME = "graph_execution_queue"+# ``_v2`` suffix marks the classic→quorum rollover; old-image consumers+# drain the unsuffixed queue. Orphans cleaned up in a follow-up PR.+GRAPH_EXECUTION_QUEUE_NAME = "graph_execution_queue_v2"GRAPH_EXECUTION_ROUTING_KEY = "graph_execution.run"GRAPH_EXECUTION_CANCEL_EXCHANGE = Exchange(@@ -911,7 +913,7 @@ GRAPH_EXECUTION_CANCEL_EXCHANGE = Exchange(durable=True,auto_delete=True,)-GRAPH_EXECUTION_CANCEL_QUEUE_NAME = "graph_execution_cancel_queue"+GRAPH_EXECUTION_CANCEL_QUEUE_NAME = "graph_execution_cancel_queue_v2"# Graceful shutdown timeout constants# Agent executions can run for up to 1 day, so we need a graceful shutdown period@@ -932,14 +934,16 @@ def create_execution_queue_config() -> RabbitMQConfig:durable=True,auto_delete=False,arguments={- # x-consumer-timeout (1 week)+ # Quorum (not classic mirrored) for leader election + stronger+ # replication across RabbitMQ 4.x cluster nodes.+ "x-queue-type": "quorum",+ # x-consumer-timeout (24h)# Problem: Default 30-minute consumer timeout kills long-running graph executions# Original error: "Consumer acknowledgement timed out after 1800000 ms (30 minutes)"# Solution: Disable consumer timeout entirely - let graphs run indefinitely# Safety: Heartbeat mechanism now handles dead consumer detection instead# Use case: Graph executions that take hours to complete (AI model training, etc.)- "x-consumer-timeout": GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS- * 1000,+ "x-consumer-timeout": GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS * 1000,},)cancel_queue = Queue(@@ -948,6 +952,7 @@ def create_execution_queue_config() -> RabbitMQConfig:routing_key="", # not used for FANOUTdurable=True,auto_delete=False,+ arguments={"x-queue-type": "quorum"},)return RabbitMQConfig(vhost="/",diff --git a/autogpt_platform/backend/backend/notifications/notifications.py b/autogpt_platform/backend/backend/notifications/notifications.pyindex 674b0d0223..d0c476f75a 100644--- a/autogpt_platform/backend/backend/notifications/notifications.py+++ b/autogpt_platform/backend/backend/notifications/notifications.py@@ -52,55 +52,67 @@ NOTIFICATION_EXCHANGE = Exchange(name="notifications", type=ExchangeType.TOPIC)DEAD_LETTER_EXCHANGE = Exchange(name="dead_letter", type=ExchangeType.TOPIC)EXCHANGES = [NOTIFICATION_EXCHANGE, DEAD_LETTER_EXCHANGE]+# ``_v2`` suffix marks the classic→quorum rollover: old classic queues keep+# draining via old-image consumers; orphans are cleaned up by a follow-up PR.+IMMEDIATE_NOTIFICATIONS_QUEUE = "immediate_notifications_v2"+ADMIN_NOTIFICATIONS_QUEUE = "admin_notifications_v2"+SUMMARY_NOTIFICATIONS_QUEUE = "summary_notifications_v2"+BATCH_NOTIFICATIONS_QUEUE = "batch_notifications_v2"+FAILED_NOTIFICATIONS_QUEUE = "failed_notifications_v2"+def create_notification_config() -> RabbitMQConfig:"""Create RabbitMQ configuration for notifications"""-queues = [# Main notification queuesQueue(- name="immediate_notifications",+ name=IMMEDIATE_NOTIFICATIONS_QUEUE,exchange=NOTIFICATION_EXCHANGE,routing_key="notification.immediate.#",arguments={+ "x-queue-type": "quorum","x-dead-letter-exchange": DEAD_LETTER_EXCHANGE.name,"x-dead-letter-routing-key": "failed.immediate",},),Queue(- name="admin_notifications",+ name=ADMIN_NOTIFICATIONS_QUEUE,exchange=NOTIFICATION_EXCHANGE,routing_key="notification.admin.#",arguments={+ "x-queue-type": "quorum","x-dead-letter-exchange": DEAD_LETTER_EXCHANGE.name,"x-dead-letter-routing-key": "failed.admin",},),# Summary notification queuesQueue(- name="summary_notifications",+ name=SUMMARY_NOTIFICATIONS_QUEUE,exchange=NOTIFICATION_EXCHANGE,routing_key="notification.summary.#",arguments={+ "x-queue-type": "quorum","x-dead-letter-exchange": DEAD_LETTER_EXCHANGE.name,"x-dead-letter-routing-key": "failed.summary",},),# Batch QueueQueue(- name="batch_notifications",+ name=BATCH_NOTIFICATIONS_QUEUE,exchange=NOTIFICATION_EXCHANGE,routing_key="notification.batch.#",arguments={+ "x-queue-type": "quorum","x-dead-letter-exchange": DEAD_LETTER_EXCHANGE.name,"x-dead-letter-routing-key": "failed.batch",},),- # Failed notifications queue+ # DLQ destination — quorum so dead letters survive a broker restart.Queue(- name="failed_notifications",+ name=FAILED_NOTIFICATIONS_QUEUE,exchange=DEAD_LETTER_EXCHANGE,routing_key="failed.#",+ arguments={"x-queue-type": "quorum"},),]@@ -715,14 +727,15 @@ class NotificationManager(AppService):try:# Try to render the email to check its sizetemplate = self.email_sender._get_template(event.type)- _, test_message = (- await self.email_sender.formatter.format_email(- base_template=template.base_template,- subject_template=template.subject_template,- content_template=template.body_template,- data={"notifications": chunk},- unsubscribe_link=f"{self.email_sender.formatter.env.globals.get('base_url', '')}/profile/settings",- )+ (+ _,+ test_message,+ ) = await self.email_sender.formatter.format_email(+ base_template=template.base_template,+ subject_template=template.subject_template,+ content_template=template.body_template,+ data={"notifications": chunk},+ unsubscribe_link=f"{self.email_sender.formatter.env.globals.get('base_url', '')}/profile/settings",)if len(test_message) < MAX_EMAIL_SIZE:@@ -1049,10 +1062,10 @@ class NotificationManager(AppService):# Set prefetch to prevent overwhelming the serviceawait channel.set_qos(prefetch_count=10)- immediate_queue = await channel.get_queue("immediate_notifications")- batch_queue = await channel.get_queue("batch_notifications")- admin_queue = await channel.get_queue("admin_notifications")- summary_queue = await channel.get_queue("summary_notifications")+ immediate_queue = await channel.get_queue(IMMEDIATE_NOTIFICATIONS_QUEUE)+ batch_queue = await channel.get_queue(BATCH_NOTIFICATIONS_QUEUE)+ admin_queue = await channel.get_queue(ADMIN_NOTIFICATIONS_QUEUE)+ summary_queue = await channel.get_queue(SUMMARY_NOTIFICATIONS_QUEUE)# Create consumer tasks for each queue - running in parallelconsumer_tasks = [@@ -1060,28 +1073,28 @@ class NotificationManager(AppService):self._consume_queue(queue=immediate_queue,process_func=self._process_immediate,- queue_name="immediate_notifications",+ queue_name=IMMEDIATE_NOTIFICATIONS_QUEUE,)),asyncio.create_task(self._consume_queue(queue=admin_queue,process_func=self._process_admin_message,- queue_name="admin_notifications",+ queue_name=ADMIN_NOTIFICATIONS_QUEUE,)),asyncio.create_task(self._consume_queue(queue=batch_queue,process_func=self._process_batch,- queue_name="batch_notifications",+ queue_name=BATCH_NOTIFICATIONS_QUEUE,)),asyncio.create_task(self._consume_queue(queue=summary_queue,process_func=self._process_summary,- queue_name="summary_notifications",+ queue_name=SUMMARY_NOTIFICATIONS_QUEUE,)),]diff --git a/autogpt_platform/backend/backend/util/cache.py b/autogpt_platform/backend/backend/util/cache.pyindex 8f55d49fdc..85126c2a83 100644--- a/autogpt_platform/backend/backend/util/cache.py+++ b/autogpt_platform/backend/backend/util/cache.py@@ -21,7 +21,7 @@ from dataclasses import dataclassfrom functools import cache, wrapsfrom typing import Any, Callable, ParamSpec, Protocol, TypeVar, cast, runtime_checkable-from redis import ConnectionPool, Redis+from redis.cluster import ClusterNode, RedisClusterfrom backend.util.retry import conn_retryfrom backend.util.settings import Settings@@ -45,32 +45,26 @@ _HMAC_SIG_LEN = 32@cache-def _get_cache_pool() -> ConnectionPool:- """Get or create a connection pool for cache operations (lazy, thread-safe)."""- return ConnectionPool(- host=settings.config.redis_host,- port=settings.config.redis_port,+@conn_retry("Redis", "Acquiring cache connection")+def _get_redis() -> RedisCluster:+ """Lazily-initialized shared-cache Redis Cluster client (singleton)."""+ # Local import breaks the redis_client <-> cache module cycle.+ from backend.data.redis_client import _address_remap++ c = RedisCluster(+ startup_nodes=[+ ClusterNode(settings.config.redis_host, settings.config.redis_port)+ ],password=settings.config.redis_password or None,decode_responses=False, # Binary mode for picklemax_connections=50,socket_keepalive=True,socket_connect_timeout=5,retry_on_timeout=True,+ address_remap=_address_remap,)---@cache-@conn_retry("Redis", "Acquiring cache connection")-def _get_redis() -> Redis:- """- Get the lazily-initialized Redis client for shared cache operations.- Uses @cache for thread-safe singleton behavior - connection is only- established when first accessed, allowing services that only use- in-memory caching to work without Redis configuration.- """- r = Redis(connection_pool=_get_cache_pool())- r.ping() # Verify connection- return r+ c.ping()+ return cclass _MissingType:@@ -402,12 +396,19 @@ def cached(def cache_clear(pattern: str | None = None) -> None:"""Clear cache entries. If pattern provided, clear matching entries."""if shared_cache:- if pattern:- # Clear entries matching pattern- keys = list(_get_redis().scan_iter(f"cache:{func_name}:{pattern}"))- else:- # Clear all cache keys- keys = list(_get_redis().scan_iter(f"cache:{func_name}:*"))+ # SCAN must fan out across every primary on a cluster — without+ # target_nodes the scan only walks the shard the client is+ # bound to, leaving stale keys on the other shards.+ match_pattern = (+ f"cache:{func_name}:{pattern}"+ if pattern+ else f"cache:{func_name}:*"+ )+ keys = list(+ _get_redis().scan_iter(+ match_pattern, target_nodes=RedisCluster.PRIMARIES+ )+ )if keys:pipeline = _get_redis().pipeline()@@ -425,7 +426,12 @@ def cached(def cache_info() -> dict[str, int | None]:if shared_cache:- cache_keys = list(_get_redis().scan_iter(f"cache:{func_name}:*"))+ cache_keys = list(+ _get_redis().scan_iter(+ f"cache:{func_name}:*",+ target_nodes=RedisCluster.PRIMARIES,+ )+ )return {"size": len(cache_keys),"maxsize": None, # Redis manages its own sizediff --git a/autogpt_platform/backend/backend/util/metrics.py b/autogpt_platform/backend/backend/util/metrics.pyindex 3348dd46d1..a0667804a0 100644--- a/autogpt_platform/backend/backend/util/metrics.py+++ b/autogpt_platform/backend/backend/util/metrics.py@@ -55,6 +55,21 @@ _AMQP_KEYWORDS = [_AMQP_INDICATORS = ["aio_pika", "aiormq", "amqp", "pika", "rabbitmq"]+# Pika reconnect noise: `conn_retry` already handles these. Drop only the+# four known signatures so genuine pika ERRORs (auth, declare failures) still+# reach Sentry. (AUTOGPT-SERVER-6JC/6JD/6JE/6JF.)+_PIKA_RECONNECT_LOGGERS = {+ "pika.adapters.utils.io_services_utils",+ "pika.adapters.blocking_connection",+ "pika.adapters.base_connection",+}+_PIKA_RECONNECT_SIGNATURES = (+ "streamlosterror",+ "transport indicated eof",+ "socket eof",+ "connection_lost",+)+def _before_send(event, hint):"""Filter out expected/transient errors from Sentry to reduce noise."""@@ -94,11 +109,7 @@ def _before_send(event, hint):):return None- # Prisma UniqueViolationError — always caught and handled in our codebase.- # These arise from concurrent create operations racing on unique constraints- # (workspace files, credits, library folders, store listings, chat messages).- # Every call site has an except handler; the global FastAPI handler also- # catches them and returns 400. Safe to drop unconditionally.+ # Prisma UniqueViolationError is always handled (FastAPI returns 400).if exc_type and exc_type.__name__ == "UniqueViolationError":return None@@ -119,7 +130,11 @@ def _before_send(event, hint):log_msg = (logentry.get("formatted") or logentry.get("message") or event.get("message"))- if event.get("logger") and log_msg:+ logger_name = event.get("logger")+ if logger_name in _PIKA_RECONNECT_LOGGERS and log_msg:+ if any(sig in log_msg.lower() for sig in _PIKA_RECONNECT_SIGNATURES):+ return None+ if logger_name and log_msg:msg = log_msg.lower()noisy_log_patterns = ["amqpconnection",diff --git a/autogpt_platform/backend/backend/util/metrics_test.py b/autogpt_platform/backend/backend/util/metrics_test.pynew file mode 100644index 0000000000..feb5994e31--- /dev/null+++ b/autogpt_platform/backend/backend/util/metrics_test.py@@ -0,0 +1,107 @@+"""Tests for the Sentry ``before_send`` filter in ``metrics.py``.++The filter exists to keep noise out of Sentry without dropping load-bearing+errors. We pin the "drop" list at the call-site level so a regression that+changes ``_before_send`` semantics — e.g. accidentally dropping all pika+ERRORs, or letting StreamLostError through — surfaces as a test failure+rather than a Sentry alert flood.+"""++from __future__ import annotations++from backend.util.metrics import (+ _PIKA_RECONNECT_LOGGERS,+ _PIKA_RECONNECT_SIGNATURES,+ _before_send,+)+++def _log_event(logger: str, message: str) -> dict:+ return {+ "logger": logger,+ "logentry": {"formatted": message, "message": message},+ "level": "error",+ }+++# ---------- pika reconnect noise → dropped ----------+++def test_pika_streamlost_error_dropped() -> None:+ """AUTOGPT-SERVER-6JC: ``StreamLostError: Transport indicated EOF`` from+ ``pika.adapters.blocking_connection`` is benign reconnect noise."""+ for logger in _PIKA_RECONNECT_LOGGERS:+ evt = _log_event(logger, "StreamLostError: Transport indicated EOF")+ assert _before_send(evt, hint={}) is None, logger+++def test_pika_socket_eof_dropped() -> None:+ """AUTOGPT-SERVER-6JD: ``Socket EOF`` from+ ``pika.adapters.utils.io_services_utils`` reconnect path."""+ evt = _log_event(+ "pika.adapters.utils.io_services_utils",+ "Socket EOF on fd=12",+ )+ assert _before_send(evt, hint={}) is None+++def test_pika_connection_lost_dropped() -> None:+ """AUTOGPT-SERVER-6JE: ``connection_lost`` callback firing during a+ rolling deploy."""+ evt = _log_event(+ "pika.adapters.base_connection",+ "connection_lost: Stream connection lost: ConnectionResetError(...)",+ )+ assert _before_send(evt, hint={}) is None+++def test_pika_transport_eof_dropped() -> None:+ """AUTOGPT-SERVER-6JF: ``Transport indicated EOF`` standalone string."""+ evt = _log_event("pika.adapters.blocking_connection", "Transport indicated EOF")+ assert _before_send(evt, hint={}) is None+++# ---------- pika ERRORs that must still get through ----------+++def test_pika_authentication_failure_kept() -> None:+ """A real auth failure on the AMQP connection is load-bearing and must+ NOT be filtered out by the reconnect-noise rule."""+ evt = _log_event(+ "pika.adapters.blocking_connection",+ "ProbableAuthenticationError: Server closed connection",+ )+ assert _before_send(evt, hint={}) is not None+++def test_pika_channel_declare_error_kept() -> None:+ """PRECONDITION_FAILED on a queue declare (e.g. quorum-type mismatch) is+ a real bug, not reconnect noise — must be kept."""+ evt = _log_event(+ "pika.adapters.blocking_connection",+ "PRECONDITION_FAILED - inequivalent arg 'x-queue-type' for queue 'foo'",+ )+ assert _before_send(evt, hint={}) is not None+++def test_non_pika_logger_with_streamlost_kept() -> None:+ """Reconnect signatures are only suppressed for the three known pika+ loggers; any other logger emitting the same string is kept (e.g. a+ custom wrapper that re-raises)."""+ evt = _log_event(+ "backend.data.rabbitmq",+ "StreamLostError: Transport indicated EOF",+ )+ assert _before_send(evt, hint={}) is not None+++def test_pika_reconnect_signatures_cover_all_four_known_patterns() -> None:+ """Sanity check: the signatures list still covers all four AUTOGPT-+ SERVER-6JC/6JD/6JE/6JF patterns from the prod Sentry issues."""+ expected = {+ "streamlosterror",+ "transport indicated eof",+ "socket eof",+ "connection_lost",+ }+ assert expected == set(_PIKA_RECONNECT_SIGNATURES)diff --git a/autogpt_platform/backend/backend/util/service.py b/autogpt_platform/backend/backend/util/service.pyindex 1139ecbee9..48cefa26c0 100644--- a/autogpt_platform/backend/backend/util/service.py+++ b/autogpt_platform/backend/backend/util/service.py@@ -33,6 +33,7 @@ from pydantic import BaseModel, TypeAdapter, create_modelfrom sentry_sdk.api import capture_exception as _sentry_capture_exceptionimport backend.util.exceptions as exceptions+from backend.data import redis_clientfrom backend.monitoring.instrumentation import instrument_fastapifrom backend.util.json import to_dictfrom backend.util.metrics import sentry_init@@ -408,10 +409,24 @@ class AppService(BaseAppService, ABC):await db.disconnect()```"""- # Startup - this runs before Uvicorn starts accepting connections+ # Startup - this runs before Uvicorn starts accepting connections.+ # Eager connect so we fail-fast if Redis is unreachable, mirroring+ # the db.connect()/db.disconnect() pattern subclasses use below.+ await redis_client.get_redis_async()yield+ # Close the cluster client so asyncio's GC doesn't emit "Unclosed+ # ClusterNode" warnings at interpreter shutdown. Wrapped so a wedged+ # socket close doesn't block subclass-level db.disconnect calls.+ try:+ await redis_client.disconnect_async()+ except Exception:+ logger.warning(+ f"[{self.service_name}] redis_client.disconnect_async failed",+ exc_info=True,+ )+# Shutdown - this runs when FastAPI/Uvicorn shuts downlogger.info(f"[{self.service_name}] ✅ FastAPI has finished")diff --git a/autogpt_platform/backend/backend/util/settings.py b/autogpt_platform/backend/backend/util/settings.pyindex 5c831b2a34..43b0af6bc3 100644--- a/autogpt_platform/backend/backend/util/settings.py+++ b/autogpt_platform/backend/backend/util/settings.py@@ -4,7 +4,14 @@ import refrom enum import Enumfrom typing import Any, Dict, Generic, List, Set, Tuple, Type, TypeVar-from pydantic import BaseModel, Field, PrivateAttr, ValidationInfo, field_validator+from pydantic import (+ AliasChoices,+ BaseModel,+ Field,+ PrivateAttr,+ ValidationInfo,+ field_validator,+)from pydantic_settings import (BaseSettings,JsonConfigSettingsSource,@@ -302,14 +309,18 @@ class Config(UpdateTrackingModel["Config"], BaseSettings):description="The pool size for the scheduler database connection pool",)+ # Prefer the cluster env var so the new image can co-exist with old-image+ # pods still reading the unsuffixed RABBITMQ_HOST during a rollout.rabbitmq_host: str = Field(default="localhost",description="The host for the RabbitMQ server",+ validation_alias=AliasChoices("RABBITMQ_CLUSTER_HOST", "RABBITMQ_HOST"),)rabbitmq_port: int = Field(default=5672,description="The port for the RabbitMQ server",+ validation_alias=AliasChoices("RABBITMQ_CLUSTER_PORT", "RABBITMQ_PORT"),)rabbitmq_vhost: str = Field(@@ -317,14 +328,19 @@ class Config(UpdateTrackingModel["Config"], BaseSettings):description="The vhost for the RabbitMQ server",)+ # Same rollover pattern as rabbitmq_host; REDIS_CLUSTER_HOST must win so+ # cache.py's RedisCluster client reaches the sharded cluster, not the+ # pre-migration standalone Redis.redis_host: str = Field(default="localhost",description="The host for the Redis server",+ validation_alias=AliasChoices("REDIS_CLUSTER_HOST", "REDIS_HOST"),)redis_port: int = Field(default=6379,description="The port for the Redis server",+ validation_alias=AliasChoices("REDIS_CLUSTER_PORT", "REDIS_PORT"),)redis_password: str = Field(diff --git a/autogpt_platform/backend/pyproject.toml b/autogpt_platform/backend/pyproject.tomlindex eb9c26c5dd..8555fb2c9a 100644--- a/autogpt_platform/backend/pyproject.toml+++ b/autogpt_platform/backend/pyproject.toml@@ -164,6 +164,7 @@ addopts = "-p no:syrupy"markers = ["supplementary: tests kept for coverage but superseded by integration tests","integration: end-to-end tests that require a live database (skipped in CI)",+ "slow: tests that take more than a few seconds (e.g. spin up an isolated docker stack)",]filterwarnings = ["ignore:'audioop' is deprecated:DeprecationWarning:discord.player",diff --git a/autogpt_platform/docker-compose.platform.yml b/autogpt_platform/docker-compose.platform.ymlindex 1b3ff8338f..d792c2a3ca 100644--- a/autogpt_platform/docker-compose.platform.yml+++ b/autogpt_platform/docker-compose.platform.yml@@ -16,7 +16,13 @@ x-backend-env:NOTIFICATIONMANAGER_HOST: notification_serverCLAMAV_SERVICE_HOST: clamavDB_HOST: db- REDIS_HOST: redis+ REDIS_HOST: redis-0+ REDIS_PORT: "17000"+ # The 3-container local cluster announces distinct compose hostnames per+ # shard (redis-0/1/2), each resolvable via Docker DNS inside the compose+ # network. Bypass the HOST-pinning address_remap so CLUSTER SLOTS uses the+ # announced addresses directly — see ``backend.data.redis_client``.+ REDIS_USE_ANNOUNCED_ADDRESS: "true"RABBITMQ_HOST: rabbitmqGRAPHITI_FALKORDB_HOST: falkordbGRAPHITI_FALKORDB_PORT: "6379"@@ -34,6 +40,14 @@ x-backend-env-files: &backend-env-files- path: backend/.env # User overrides (optional)required: false+# Shared base for the three Redis Cluster shards + one-shot init sidecar.+# The anchor absorbs image + network so each service body collapses to its+# hostname + command + (for the seed) volume + healthcheck.+x-redis-node: &redis-node+ image: redis:7+ networks:+ - app-network+services:migrate:build:@@ -71,17 +85,56 @@ services:retries: 3start_period: 5s- redis:- image: redis:latest+ # Three-container 3-master Redis Cluster for local dev. Stock ``redis:7``+ # on each shard runs its default entrypoint; the one-shot ``redis-init``+ # sidecar runs ``redis-cli --cluster create`` once and exits. No volumes+ # — local dev treats the cluster as cache-only, so every ``docker compose+ # up`` starts fresh. (Persisting only the seed volume across restarts is+ # a trap: the other shards come back with new IDs and the seed's+ # ``nodes.conf`` pins stale peers that cluster gossip can't heal.) Each+ # shard announces its own compose hostname via+ # ``--cluster-announce-hostname`` so both in-compose clients (Docker DNS)+ # and host-native clients (handled by the backend's ``address_remap``)+ # receive a consistent address on CLUSTER SLOTS. Ports 17000/17001/17002+ # sit well clear of FalkorDB on host 6380.+ redis-0:+ <<: *redis-node+ hostname: redis-0+ command: redis-server --port 17000 --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 --cluster-require-full-coverage no --cluster-announce-hostname redis-0 --cluster-announce-port 17000 --cluster-announce-bus-port 27000 --cluster-preferred-endpoint-type hostnameports:- - "6379:6379"- networks:- - app-network+ - "17000:17000"healthcheck:- test: ["CMD", "redis-cli", "ping"]+ test:+ [+ "CMD-SHELL",+ "redis-cli -p 17000 ping && redis-cli -p 17000 cluster info | grep -q 'cluster_state:ok'",+ ]interval: 10stimeout: 5s- retries: 5+ retries: 30+ redis-1:+ <<: *redis-node+ hostname: redis-1+ command: redis-server --port 17001 --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 --cluster-require-full-coverage no --cluster-announce-hostname redis-1 --cluster-announce-port 17001 --cluster-announce-bus-port 27001 --cluster-preferred-endpoint-type hostname+ ports:+ - "17001:17001"+ redis-2:+ <<: *redis-node+ hostname: redis-2+ command: redis-server --port 17002 --cluster-enabled yes --cluster-config-file nodes.conf --cluster-node-timeout 5000 --cluster-require-full-coverage no --cluster-announce-hostname redis-2 --cluster-announce-port 17002 --cluster-announce-bus-port 27002 --cluster-preferred-endpoint-type hostname+ ports:+ - "17002:17002"+ # The ``|| cluster create`` short-circuit makes this idempotent within the+ # lifetime of a single compose project: a re-``up`` without ``down`` leaves+ # the already-formed cluster untouched. A ``down`` + ``up`` wipes every+ # shard (no volumes) and re-creates from scratch.+ redis-init:+ <<: *redis-node+ depends_on: [redis-0, redis-1, redis-2]+ restart: "no"+ command: >-+ sh -c "redis-cli -h redis-0 -p 17000 cluster info 2>/dev/null | grep -q cluster_state:ok ||+ redis-cli --cluster create redis-0:17000 redis-1:17001 redis-2:17002 --cluster-replicas 0 --cluster-yes"falkordb:image: falkordb/falkordb:latestports:@@ -130,7 +183,7 @@ services:target: autogpt_platform/backend/action: rebuilddepends_on:- redis:+ redis-0:condition: service_healthydb:condition: service_healthy@@ -165,7 +218,7 @@ services:target: autogpt_platform/backend/action: rebuilddepends_on:- redis:+ redis-0:condition: service_healthyrabbitmq:condition: service_healthy@@ -200,7 +253,7 @@ services:target: autogpt_platform/backend/action: rebuilddepends_on:- redis:+ redis-0:condition: service_healthyrabbitmq:condition: service_healthy@@ -240,7 +293,7 @@ services:depends_on:db:condition: service_healthy- redis:+ redis-0:condition: service_healthymigrate:condition: service_completed_successfully@@ -302,7 +355,7 @@ services:depends_on:db:condition: service_healthy- redis:+ redis-0:condition: service_healthyrabbitmq:condition: service_healthydiff --git a/autogpt_platform/docker-compose.yml b/autogpt_platform/docker-compose.ymlindex f7b4b105fc..94e356941d 100644--- a/autogpt_platform/docker-compose.yml+++ b/autogpt_platform/docker-compose.yml@@ -31,11 +31,29 @@ services:file: ./docker-compose.platform.ymlservice: migrate- redis:+ redis-0:<<: *agpt-servicesextends:file: ./docker-compose.platform.yml- service: redis+ service: redis-0++ redis-1:+ <<: *agpt-services+ extends:+ file: ./docker-compose.platform.yml+ service: redis-1++ redis-2:+ <<: *agpt-services+ extends:+ file: ./docker-compose.platform.yml+ service: redis-2++ redis-init:+ <<: *agpt-services+ extends:+ file: ./docker-compose.platform.yml+ service: redis-initfalkordb:<<: *agpt-services@@ -174,7 +192,10 @@ services:- auth- db- studio- - redis+ - redis-0+ - redis-1+ - redis-2+ - redis-init- rabbitmq- clamav- falkordbdiff --git a/autogpt_platform/frontend/src/playwright/pages/copilot.page.ts b/autogpt_platform/frontend/src/playwright/pages/copilot.page.tsindex d67e20ef6e..b22a25cffe 100644--- a/autogpt_platform/frontend/src/playwright/pages/copilot.page.ts+++ b/autogpt_platform/frontend/src/playwright/pages/copilot.page.ts@@ -14,11 +14,21 @@ export class CopilotPage extends BasePage {}async dismissNotificationPrompt(): Promise<void> {- // Notification permission prompt is optional — only shown on first visit- const notNowButton = this.page.getByRole("button", { name: "Not now" });- if (await notNowButton.isVisible({ timeout: 3000 })) {- await notNowButton.click();+ // The dialog mounts after React hydrates + reads localStorage; on slow CI+ // it can appear AFTER the call returned, leaving a fixed overlay that+ // blocks the chat input. Wait for it to either mount or stay absent.+ const dialog = this.page.getByRole("dialog").filter({+ has: this.page.getByText("Stay in the loop"),+ });+ try {+ await dialog.waitFor({ state: "visible", timeout: 3000 });+ } catch {+ return;}+ await this.page+ .getByRole("button", { name: "Not now" })+ .click({ timeout: 3000 });+ await dialog.waitFor({ state: "hidden", timeout: 5000 });}async createSessionViaApi(): Promise<string> {