Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend/copilot/rate_limit): fail-closed on Redis unavailable to prevent USD-cap bypass (#12990)

Zamil Majdy2c840ea495d4
diff --git a/autogpt_platform/backend/backend/api/features/chat/routes.py b/autogpt_platform/backend/backend/api/features/chat/routes.py
index 550dc99b84..58be141819 100644
--- a/autogpt_platform/backend/backend/api/features/chat/routes.py
+++ b/autogpt_platform/backend/backend/api/features/chat/routes.py
@@ -31,6 +31,7 @@ from backend.copilot.model import (
)
from backend.copilot.pending_message_helpers import (
QueuePendingMessageResponse,
+ StreamRegistryUnavailable,
is_turn_in_flight,
queue_pending_for_http,
)
@@ -38,6 +39,7 @@ from backend.copilot.pending_messages import peek_pending_messages
from backend.copilot.rate_limit import (
CoPilotUsagePublic,
RateLimitExceeded,
+ RateLimitUnavailable,
acquire_reset_lock,
check_rate_limit,
get_daily_reset_count,
@@ -901,6 +903,11 @@ def _empty_ui_message_stream_response() -> StreamingResponse:
responses={
404: {"description": "Session not found or access denied"},
429: {"description": "Cost rate-limit or call-frequency cap exceeded"},
+ 503: {
+ "description": "Chat service degraded (Redis unavailable for rate "
+ "limit or stream registry); client should honour the Retry-After "
+ "header before retrying."
+ },
},
)
async def stream_chat_post(
@@ -944,11 +951,24 @@ async def stream_chat_post(
)
session = await _validate_and_get_session(session_id, user_id)
- if (
- request.is_user_message
- and request.message
- and await is_turn_in_flight(session_id)
- ):
+ try:
+ turn_in_flight = (
+ request.is_user_message
+ and request.message
+ and await is_turn_in_flight(session_id)
+ )
+ except StreamRegistryUnavailable as exc:
+ # Same fail-closed mapping as the RateLimitUnavailable branch below:
+ # the pre-flight chain runs is_turn_in_flight BEFORE check_rate_limit,
+ # so a Redis brown-out at this step would otherwise surface as a raw
+ # 500 instead of the polished 503 + Retry-After.
+ raise HTTPException(
+ status_code=503,
+ detail="Chat service degraded, retry shortly",
+ headers={"Retry-After": "30"},
+ ) from exc
+
+ if turn_in_flight:
try:
await queue_pending_for_http(
session_id=session_id,
@@ -994,6 +1014,16 @@ async def stream_chat_post(
)
except RateLimitExceeded as e:
raise HTTPException(status_code=429, detail=str(e)) from e
+ except RateLimitUnavailable as e:
+ # Fail-closed on Redis brown-out: the user may already be at or
+ # past their USD cap and we cannot prove otherwise. 503 + a short
+ # Retry-After is the right UX (transient outage, retry shortly),
+ # not 429 ("you hit your limit").
+ raise HTTPException(
+ status_code=503,
+ detail="Rate limit service degraded, retry shortly",
+ headers={"Retry-After": "30"},
+ ) from e
# Enrich message with file metadata if file_ids are provided.
# Also sanitise file_ids so only validated, workspace-scoped IDs are
@@ -1225,6 +1255,10 @@ async def stream_chat_post(
404: {"description": "Session not found or access denied"},
409: {"description": "Session has no active turn to receive pending messages"},
429: {"description": "Call-frequency cap exceeded"},
+ 503: {
+ "description": "Chat service degraded (Redis unavailable); "
+ "client should honour the Retry-After header before retrying."
+ },
},
)
async def queue_pending_message(
@@ -1234,7 +1268,15 @@ async def queue_pending_message(
):
"""Queue a follow-up message while the session has an active turn."""
await _validate_and_get_session(session_id, user_id)
- if not await is_turn_in_flight(session_id):
+ try:
+ turn_in_flight = await is_turn_in_flight(session_id)
+ except StreamRegistryUnavailable as exc:
+ raise HTTPException(
+ status_code=503,
+ detail="Chat service degraded, retry shortly",
+ headers={"Retry-After": "30"},
+ ) from exc
+ if not turn_in_flight:
raise HTTPException(
status_code=409,
detail="Session has no active turn. Start a new turn with POST /stream.",
diff --git a/autogpt_platform/backend/backend/api/features/chat/routes_test.py b/autogpt_platform/backend/backend/api/features/chat/routes_test.py
index 7aba4b7e97..51a59e9bce 100644
--- a/autogpt_platform/backend/backend/api/features/chat/routes_test.py
+++ b/autogpt_platform/backend/backend/api/features/chat/routes_test.py
@@ -375,6 +375,71 @@ def test_stream_chat_429_includes_reset_time(mocker: pytest_mock.MockerFixture):
assert "Resets in" in detail
+def test_stream_chat_returns_503_with_retry_after_when_rate_limit_unavailable(
+ mocker: pytest_mock.MockerFixture,
+):
+ """Redis brown-out must NOT bypass the per-user USD cap.
+
+ When ``check_rate_limit`` raises :class:`RateLimitUnavailable` the
+ endpoint must respond 503 with ``Retry-After``, not 429 (different UX:
+ transient outage, not "you hit your limit") and not 200 (which would
+ silently let the user blast the LLM during the outage)."""
+ from backend.copilot.rate_limit import RateLimitUnavailable
+
+ _mock_stream_internals(mocker)
+ mocker.patch.object(chat_routes.config, "daily_cost_limit_microdollars", 10000)
+ mocker.patch.object(chat_routes.config, "weekly_cost_limit_microdollars", 50000)
+ # Patch the limit-resolution helper so the test does not exercise the
+ # real LaunchDarkly / tier-lookup path — keeps the assertion focused on
+ # the RateLimitUnavailable → 503 mapping.
+ mocker.patch(
+ "backend.api.features.chat.routes.get_global_rate_limits",
+ new_callable=AsyncMock,
+ return_value=(10_000, 50_000, SubscriptionTier.BASIC),
+ )
+ mocker.patch(
+ "backend.api.features.chat.routes.check_rate_limit",
+ side_effect=RateLimitUnavailable(),
+ )
+
+ response = client.post(
+ "/sessions/sess-1/stream",
+ json={"message": "hello"},
+ )
+ assert response.status_code == 503
+ assert response.headers.get("Retry-After") == "30"
+ assert "degraded" in response.json()["detail"].lower()
+
+
+def test_stream_chat_returns_503_when_stream_registry_unavailable(
+ mocker: pytest_mock.MockerFixture,
+):
+ """``is_turn_in_flight`` runs BEFORE ``check_rate_limit`` in the
+ pre-flight chain. A Redis brown-out at this step (e.g. ``hgetall`` on
+ the session-meta key fails with ``RedisClusterException``) must be
+ mapped to the same 503 + Retry-After response, NOT bubble as a raw
+ HTTP 500 with internal Redis error in the body. Cap-bypass cannot
+ happen (LLM is not invoked), but the UX must be polished."""
+ from backend.copilot.pending_message_helpers import StreamRegistryUnavailable
+
+ _mock_stream_internals(mocker)
+ # Force the is_turn_in_flight branch to fail-closed by raising the
+ # typed unavailability exception (helper-level Redis-error mapping is
+ # exercised in pending_message_helpers_test).
+ mocker.patch(
+ "backend.api.features.chat.routes.is_turn_in_flight",
+ side_effect=StreamRegistryUnavailable(),
+ )
+
+ response = client.post(
+ "/sessions/sess-1/stream",
+ json={"message": "hello"},
+ )
+ assert response.status_code == 503
+ assert response.headers.get("Retry-After") == "30"
+ assert "degraded" in response.json()["detail"].lower()
+
+
# ─── Usage endpoint ───────────────────────────────────────────────────
@@ -750,6 +815,28 @@ def test_queue_pending_message_without_active_turn_returns_409(
assert response.status_code == 409
+def test_queue_pending_message_returns_503_when_stream_registry_unavailable(
+ mocker: pytest_mock.MockerFixture,
+) -> None:
+ """Redis brown-out on the pre-flight ``is_turn_in_flight`` check must
+ return 503 + Retry-After, not bubble as 500."""
+ from backend.copilot.pending_message_helpers import StreamRegistryUnavailable
+
+ _mock_stream_queue_internals(mocker)
+ mocker.patch(
+ "backend.api.features.chat.routes.is_turn_in_flight",
+ side_effect=StreamRegistryUnavailable(),
+ )
+
+ response = client.post(
+ "/sessions/sess-1/messages/pending",
+ json={"message": "hi"},
+ )
+
+ assert response.status_code == 503
+ assert response.headers.get("Retry-After") == "30"
+
+
def test_queue_pending_message_race_after_active_check_returns_409(
mocker: pytest_mock.MockerFixture,
) -> None:
diff --git a/autogpt_platform/backend/backend/copilot/baseline/service.py b/autogpt_platform/backend/backend/copilot/baseline/service.py
index fb59fb5e4c..49fb290689 100644
--- a/autogpt_platform/backend/backend/copilot/baseline/service.py
+++ b/autogpt_platform/backend/backend/copilot/baseline/service.py
@@ -98,7 +98,6 @@ from backend.copilot.transcript import (
validate_transcript,
)
from backend.copilot.transcript_builder import TranscriptBuilder
-from backend.data.db_accessors import chat_db
from backend.util import json as util_json
from backend.util.exceptions import NotFoundError
from backend.util.prompt import (
diff --git a/autogpt_platform/backend/backend/copilot/pending_message_helpers.py b/autogpt_platform/backend/backend/copilot/pending_message_helpers.py
index 59547408ca..ca901ae216 100644
--- a/autogpt_platform/backend/backend/copilot/pending_message_helpers.py
+++ b/autogpt_platform/backend/backend/copilot/pending_message_helpers.py
@@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Callable
from fastapi import HTTPException
from pydantic import BaseModel
+from redis.exceptions import RedisClusterException, RedisError
from backend.copilot.model import ChatMessage, upsert_chat_session
from backend.copilot.pending_messages import (
@@ -43,6 +44,17 @@ PENDING_CALL_WINDOW_SECONDS = 60
_PENDING_CALL_KEY_PREFIX = "copilot:pending:calls:"
+class StreamRegistryUnavailable(Exception):
+ """The active-session lookup in Redis failed — the route should fail
+ closed with HTTP 503 instead of bubbling the raw Redis error as 500.
+
+ Mirrors :class:`backend.copilot.rate_limit.RateLimitUnavailable` so the
+ chat-route pre-flight chain (``is_turn_in_flight`` → ``check_rate_limit``)
+ maps any Redis brown-out to the same polished 503 + ``Retry-After``
+ response, regardless of which step tripped first.
+ """
+
+
async def is_turn_in_flight(session_id: str) -> bool:
"""Return ``True`` when a copilot turn is actively running for *session_id*.
@@ -50,8 +62,24 @@ async def is_turn_in_flight(session_id: str) -> bool:
second message arriving while an earlier turn is still executing gets
queued into the pending buffer instead of racing the in-flight turn on
the cluster lock.
+
+ Raises :class:`StreamRegistryUnavailable` when the underlying Redis
+ lookup fails. The HTTP layer must catch this and map it to 503 — letting
+ the raw Redis error bubble would surface as an unhandled 500 with
+ internal cluster details in the body.
"""
- active = await get_active_session_meta(session_id)
+ try:
+ active = await get_active_session_meta(session_id)
+ except (RedisError, RedisClusterException, ConnectionError, OSError) as exc:
+ # RedisClusterException covers SlotNotCoveredError raised during a
+ # GKE rolling restart (it does NOT inherit from RedisError, only
+ # from Exception). Same fail-closed posture as ``check_rate_limit``.
+ logger.warning(
+ "is_turn_in_flight: stream registry unavailable for session=%s: %s",
+ session_id,
+ exc,
+ )
+ raise StreamRegistryUnavailable() from exc
return active is not None and active.status == "running"
diff --git a/autogpt_platform/backend/backend/copilot/pending_message_helpers_test.py b/autogpt_platform/backend/backend/copilot/pending_message_helpers_test.py
index 254a128c25..915a1c4ce6 100644
--- a/autogpt_platform/backend/backend/copilot/pending_message_helpers_test.py
+++ b/autogpt_platform/backend/backend/copilot/pending_message_helpers_test.py
@@ -10,10 +10,12 @@ from backend.copilot import pending_message_helpers as helpers_module
from backend.copilot.pending_message_helpers import (
PENDING_CALL_LIMIT,
QueuePendingMessageResponse,
+ StreamRegistryUnavailable,
check_pending_call_rate,
combine_pending_with_current,
drain_pending_safe,
insert_pending_before_last,
+ is_turn_in_flight,
persist_session_safe,
queue_pending_for_http,
)
@@ -49,6 +51,70 @@ async def test_check_pending_call_rate_fails_open_on_redis_error(
assert result == 0
+# ── is_turn_in_flight: fail-closed on Redis errors ────────────────────
+
+
+async def test_is_turn_in_flight_returns_false_when_no_active_session(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr(
+ helpers_module,
+ "get_active_session_meta",
+ AsyncMock(return_value=None),
+ )
+ assert await is_turn_in_flight("sess-1") is False
+
+
+async def test_is_turn_in_flight_returns_true_when_running(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ active = MagicMock()
+ active.status = "running"
+ monkeypatch.setattr(
+ helpers_module,
+ "get_active_session_meta",
+ AsyncMock(return_value=active),
+ )
+ assert await is_turn_in_flight("sess-1") is True
+
+
+async def test_is_turn_in_flight_raises_on_redis_error(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Redis brown-out must NOT bubble as an unhandled 500. The helper
+ raises a typed ``StreamRegistryUnavailable`` so the chat-route
+ pre-flight chain can map it to 503 + Retry-After (matching the
+ ``RateLimitUnavailable`` mapping for the next pre-flight step)."""
+ monkeypatch.setattr(
+ helpers_module,
+ "get_active_session_meta",
+ AsyncMock(side_effect=ConnectionError("redis down")),
+ )
+ with pytest.raises(StreamRegistryUnavailable):
+ await is_turn_in_flight("sess-1")
+
+
+async def test_is_turn_in_flight_raises_on_redis_cluster_exception(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """``RedisClusterException`` (e.g. ``SlotNotCoveredError`` during a
+ GKE rolling restart) does NOT inherit from ``RedisError`` — verify it
+ is caught explicitly by the same fail-closed branch."""
+ from redis.exceptions import RedisClusterException
+
+ monkeypatch.setattr(
+ helpers_module,
+ "get_active_session_meta",
+ AsyncMock(side_effect=RedisClusterException("slot not covered")),
+ )
+ with pytest.raises(StreamRegistryUnavailable):
+ await is_turn_in_flight("sess-1")
+
+
# ── queue_pending_for_http: gate-then-bump ordering ───────────────────
diff --git a/autogpt_platform/backend/backend/copilot/rate_limit.py b/autogpt_platform/backend/backend/copilot/rate_limit.py
index f0d244b57b..3a29270f28 100644
--- a/autogpt_platform/backend/backend/copilot/rate_limit.py
+++ b/autogpt_platform/backend/backend/copilot/rate_limit.py
@@ -3,8 +3,22 @@
Uses Redis fixed-window counters to track per-user USD spend (stored as
microdollars, matching ``PlatformCostLog.cost_microdollars``) with
configurable daily and weekly limits. Daily windows reset at midnight UTC;
-weekly windows reset at ISO week boundary (Monday 00:00 UTC). Fails open
-when Redis is unavailable to avoid blocking users.
+weekly windows reset at ISO week boundary (Monday 00:00 UTC).
+
+Failure-mode policy:
+
+* Enforcement path (:func:`check_rate_limit`) **fails closed** — if Redis
+ is unreachable we raise :class:`RateLimitUnavailable` so the API layer
+ returns HTTP 503. A brown-out must not let a user bypass their
+ daily / weekly USD cap.
+* Observability paths (:func:`get_usage_status`, the reset-count
+ read/write helpers, the recording path :func:`record_cost_usage`) keep
+ fail-open / best-effort semantics — losing a usage gauge or a single
+ cost increment is preferable to 500-ing the request, and the
+ authoritative cap is re-checked on the next turn.
+* Reset paths (:func:`reset_user_usage`, :func:`reset_daily_usage`,
+ :func:`acquire_reset_lock`) re-raise / return ``False`` so billed reset
+ operations cannot silently no-op.
Storing microdollars rather than tokens means the counter already reflects
real model pricing (including cache discounts and provider surcharges), so
@@ -44,7 +58,7 @@ from enum import Enum
from prisma.models import User as PrismaUser
from pydantic import BaseModel, Field
-from redis.exceptions import RedisError
+from redis.exceptions import RedisClusterException, RedisError
from backend.data.db_accessors import user_db
from backend.data.redis_client import AsyncRedisClient, get_redis_async
@@ -433,6 +447,18 @@ class RateLimitExceeded(Exception):
)
+class RateLimitUnavailable(Exception):
+ """Rate limit state is currently unreachable — request rejected to
+ prevent USD-cap bypass. Maps to HTTP 503 in the API layer.
+
+ Distinct from :class:`RateLimitExceeded` (HTTP 429): the user is not
+ over their cap, but Redis is down so we cannot prove they are under it
+ either. Failing closed avoids the brown-out bypass where a user could
+ blast the LLM during a Redis outage and exceed their daily/weekly USD
+ allowance by hundreds of dollars.
+ """
+
+
async def get_usage_status(
user_id: str,
daily_cost_limit: int,
@@ -463,7 +489,9 @@ async def get_usage_status(
)
daily_used = int(daily_raw or 0)
weekly_used = int(weekly_raw or 0)
- except (RedisError, ConnectionError, OSError):
+ except (RedisError, RedisClusterException, ConnectionError, OSError, ValueError):
+ # ValueError: corrupt non-numeric counter (partial write / wrong-type
+ # SET) — same fail-open semantics, returns zeros.
logger.warning("Redis unavailable for usage status, returning zeros")
return CoPilotUsageStatus(
@@ -487,15 +515,18 @@ async def check_rate_limit(
daily_cost_limit: int,
weekly_cost_limit: int,
) -> None:
- """Check if user is within rate limits. Raises RateLimitExceeded if not.
+ """Check if user is within rate limits.
+
+ Raises :class:`RateLimitExceeded` when the user is over their cap, and
+ :class:`RateLimitUnavailable` when Redis is unreachable so the caller
+ must fail closed (HTTP 503) — the daily/weekly USD caps are real money
+ and cannot be bypassed by a Redis brown-out.
This is a pre-turn soft check. The authoritative usage counter is updated
by ``record_cost_usage()`` after the turn completes. Under concurrency,
two parallel turns may both pass this check against the same snapshot.
This is acceptable because cost-based limits are approximate by nature
(the exact cost is unknown until after generation).
-
- Fails open: if Redis is unavailable, allows the request.
"""
# Short-circuit: when both limits are 0 (unlimited) skip the Redis
# round-trip entirely.
@@ -511,9 +542,22 @@ async def check_rate_limit(
)
daily_used = int(daily_raw or 0)
weekly_used = int(weekly_raw or 0)
- except (RedisError, ConnectionError, OSError):
- logger.warning("Redis unavailable for rate limit check, allowing request")
- return
+ except (
+ RedisError,
+ RedisClusterException,
+ ConnectionError,
+ OSError,
+ ValueError,
+ ) as exc:
+ # RedisClusterException covers SlotNotCoveredError raised during a
+ # GKE rolling restart (it does NOT inherit from RedisError, only
+ # from Exception, so it would otherwise bubble up as a 500 — which
+ # is exactly the brown-out scenario this PR is meant to handle).
+ # ValueError covers a corrupt non-numeric counter value (partial
+ # write or wrong-type SET). Same spirit either way: we cannot prove
+ # the user is under their cap, so fail closed.
+ logger.warning("Rate limit state unreadable, rejecting request: %s", exc)
+ raise RateLimitUnavailable() from exc
if daily_cost_limit > 0 and daily_used >= daily_cost_limit:
raise RateLimitExceeded("daily", _daily_reset_time(now=now))
@@ -556,7 +600,7 @@ async def reset_daily_usage(user_id: str, daily_cost_limit: int = 0) -> bool:
logger.info("Reset daily usage for user %s", user_id[:8])
return True
- except (RedisError, ConnectionError, OSError):
+ except (RedisError, RedisClusterException, ConnectionError, OSError):
logger.warning("Redis unavailable for resetting daily usage")
return False
@@ -571,7 +615,7 @@ async def acquire_reset_lock(user_id: str, ttl_seconds: int = 10) -> bool:
redis = await get_redis_async()
key = f"{_RESET_LOCK_PREFIX}:{user_id}"
return bool(await redis.set(key, "1", nx=True, ex=ttl_seconds))
- except (RedisError, ConnectionError, OSError) as exc:
+ except (RedisError, RedisClusterException, ConnectionError, OSError) as exc:
logger.warning("Redis unavailable for reset lock, rejecting reset: %s", exc)
return False
@@ -581,7 +625,7 @@ async def release_reset_lock(user_id: str) -> None:
try:
redis = await get_redis_async()
await redis.delete(f"{_RESET_LOCK_PREFIX}:{user_id}")
- except (RedisError, ConnectionError, OSError):
+ except (RedisError, RedisClusterException, ConnectionError, OSError):
pass # Lock will expire via TTL
@@ -598,7 +642,9 @@ async def get_daily_reset_count(user_id: str) -> int | None:
key = f"{_RESET_COUNT_PREFIX}:{user_id}:{now.strftime('%Y-%m-%d')}"
val = await redis.get(key)
return int(val or 0)
- except (RedisError, ConnectionError, OSError):
+ except (RedisError, RedisClusterException, ConnectionError, OSError, ValueError):
+ # ValueError: corrupt non-numeric counter — fail-closed for billed
+ # resets (returning None makes the caller refuse the billed reset).
logger.warning("Redis unavailable for reading daily reset count")
return None
@@ -614,7 +660,7 @@ async def increment_daily_reset_count(user_id: str) -> None:
seconds_until_reset = int((_daily_reset_time(now=now) - now).total_seconds())
pipe.expire(key, max(seconds_until_reset, 1))
await pipe.execute()
- except (RedisError, ConnectionError, OSError):
+ except (RedisError, RedisClusterException, ConnectionError, OSError):
logger.warning("Redis unavailable for tracking reset count")
@@ -654,7 +700,7 @@ async def record_cost_usage(
# invariant that matters; the two counters are independent budgets.
await _incr_counter_atomic(redis, d_key, cost_microdollars, daily_ttl)
await _incr_counter_atomic(redis, w_key, cost_microdollars, weekly_ttl)
- except (RedisError, ConnectionError, OSError):
+ except (RedisError, RedisClusterException, ConnectionError, OSError):
logger.warning(
"Redis unavailable for recording cost usage (microdollars=%d)",
cost_microdollars,
@@ -939,7 +985,7 @@ async def reset_user_usage(user_id: str, *, reset_weekly: bool = False) -> None:
await redis.delete(d_key)
if w_key is not None:
await redis.delete(w_key)
- except (RedisError, ConnectionError, OSError):
+ except (RedisError, RedisClusterException, ConnectionError, OSError):
logger.warning("Redis unavailable for resetting user usage")
raise
diff --git a/autogpt_platform/backend/backend/copilot/rate_limit_test.py b/autogpt_platform/backend/backend/copilot/rate_limit_test.py
index e5033ac30e..019da08088 100644
--- a/autogpt_platform/backend/backend/copilot/rate_limit_test.py
+++ b/autogpt_platform/backend/backend/copilot/rate_limit_test.py
@@ -4,7 +4,7 @@ from datetime import UTC, datetime, timedelta
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
-from redis.exceptions import RedisError
+from redis.exceptions import RedisClusterException, RedisError
from .rate_limit import (
_DEFAULT_TIER_MULTIPLIERS,
@@ -13,6 +13,7 @@ from .rate_limit import (
TIER_MULTIPLIERS,
CoPilotUsageStatus,
RateLimitExceeded,
+ RateLimitUnavailable,
SubscriptionTier,
UsageWindow,
_daily_key,
@@ -216,16 +217,96 @@ class TestCheckRateLimit:
assert exc_info.value.window == "weekly"
@pytest.mark.asyncio
- async def test_allows_when_redis_unavailable(self):
- """Fail-open: allow requests when Redis is down."""
+ async def test_raises_unavailable_when_redis_connection_error(self):
+ """Fail-closed: ConnectionError must surface as RateLimitUnavailable so
+ the API layer can map it to HTTP 503 instead of silently letting the
+ request through and bypassing the per-user USD cap."""
with patch(
"backend.copilot.rate_limit.get_redis_async",
side_effect=ConnectionError("Redis down"),
):
- # Should not raise
- await check_rate_limit(
- _USER, daily_cost_limit=10000, weekly_cost_limit=50000
- )
+ with pytest.raises(RateLimitUnavailable):
+ await check_rate_limit(
+ _USER, daily_cost_limit=10000, weekly_cost_limit=50000
+ )
+
+ @pytest.mark.asyncio
+ async def test_raises_unavailable_when_redis_redis_error(self):
+ """Fail-closed for redis-py RedisError (covers cluster-down /
+ CROSSSLOT-style failures during a brown-out)."""
+ mock_redis = AsyncMock()
+ mock_redis.get = AsyncMock(side_effect=RedisError("cluster down"))
+ with patch(
+ "backend.copilot.rate_limit.get_redis_async",
+ return_value=mock_redis,
+ ):
+ with pytest.raises(RateLimitUnavailable):
+ await check_rate_limit(
+ _USER, daily_cost_limit=10000, weekly_cost_limit=50000
+ )
+
+ @pytest.mark.asyncio
+ async def test_raises_unavailable_when_redis_cluster_exception(self):
+ """Fail-closed for RedisClusterException — covers SlotNotCoveredError
+ raised during a GKE rolling restart when slot coverage is briefly
+ incomplete. This subclass does NOT inherit from RedisError, so it
+ must be caught explicitly."""
+ try:
+ from redis.exceptions import SlotNotCoveredError as _ClusterExc
+ except ImportError:
+ _ClusterExc = RedisClusterException
+ mock_redis = AsyncMock()
+ mock_redis.get = AsyncMock(side_effect=_ClusterExc("slot 1234 uncovered"))
+ with patch(
+ "backend.copilot.rate_limit.get_redis_async",
+ return_value=mock_redis,
+ ):
+ with pytest.raises(RateLimitUnavailable):
+ await check_rate_limit(
+ _USER, daily_cost_limit=10000, weekly_cost_limit=50000
+ )
+
+ @pytest.mark.asyncio
+ async def test_raises_unavailable_when_os_error(self):
+ """Fail-closed for OSError (DNS / TCP-level failures during a node
+ rolling-restart)."""
+ with patch(
+ "backend.copilot.rate_limit.get_redis_async",
+ side_effect=OSError("ECONNREFUSED"),
+ ):
+ with pytest.raises(RateLimitUnavailable):
+ await check_rate_limit(
+ _USER, daily_cost_limit=10000, weekly_cost_limit=50000
+ )
+
+ @pytest.mark.asyncio
+ async def test_raises_unavailable_when_counter_is_corrupt(self):
+ """Fail-closed when a Redis counter holds a non-numeric value — the
+ ``int(...)`` cast would otherwise raise ValueError and surface as a
+ 500, but the same fail-closed reasoning applies (we cannot prove the
+ user is under their cap)."""
+ mock_redis = AsyncMock()
+ mock_redis.get = AsyncMock(side_effect=["not-a-number", "200"])
+ with patch(
+ "backend.copilot.rate_limit.get_redis_async",
+ return_value=mock_redis,
+ ):
+ with pytest.raises(RateLimitUnavailable):
+ await check_rate_limit(
+ _USER, daily_cost_limit=10000, weekly_cost_limit=50000
+ )
+
+ @pytest.mark.asyncio
+ async def test_skips_redis_and_does_not_raise_unavailable_when_unlimited(self):
+ """When both limits are 0 (unlimited) we must not even attempt Redis,
+ so a brown-out cannot 503 unlimited users."""
+ with patch(
+ "backend.copilot.rate_limit.get_redis_async",
+ side_effect=ConnectionError("Redis down"),
+ ) as mock_get:
+ # Should not raise — short-circuited before touching Redis.
+ await check_rate_limit(_USER, daily_cost_limit=0, weekly_cost_limit=0)
+ mock_get.assert_not_called()
@pytest.mark.asyncio
async def test_skips_check_when_limit_is_zero(self):
diff --git a/autogpt_platform/backend/backend/copilot/sdk/service.py b/autogpt_platform/backend/backend/copilot/sdk/service.py
index 6cc9878bb6..17d6706f55 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/service.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/service.py
@@ -40,7 +40,6 @@ from langsmith.integrations.claude_agent_sdk import configure_claude_agent_sdk
from opentelemetry import trace as otel_trace
from pydantic import BaseModel
-from backend.data.db_accessors import chat_db
from backend.data.redis_client import get_redis_async
from backend.executor.cluster_lock import AsyncClusterLock
from backend.util.exceptions import NotFoundError
diff --git a/autogpt_platform/frontend/src/app/api/openapi.json b/autogpt_platform/frontend/src/app/api/openapi.json
index 197bf29322..62fb7eb80f 100644
--- a/autogpt_platform/frontend/src/app/api/openapi.json
+++ b/autogpt_platform/frontend/src/app/api/openapi.json
@@ -2493,7 +2493,10 @@
}
}
},
- "429": { "description": "Call-frequency cap exceeded" }
+ "429": { "description": "Call-frequency cap exceeded" },
+ "503": {
+ "description": "Chat service degraded (Redis unavailable); client should honour the Retry-After header before retrying."
+ }
}
}
},
@@ -2600,6 +2603,9 @@
},
"429": {
"description": "Cost rate-limit or call-frequency cap exceeded"
+ },
+ "503": {
+ "description": "Chat service degraded (Redis unavailable for rate limit or stream registry); client should honour the Retry-After header before retrying."
}
}
}