AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend): LD context caching + tier reconcile RPC routing (#13339)
Nicholas Tindle6a62ea1ea8cc
diff --git a/autogpt_platform/backend/backend/copilot/rate_limit.py b/autogpt_platform/backend/backend/copilot/rate_limit.pyindex b12b237280..12147a54da 100644--- a/autogpt_platform/backend/backend/copilot/rate_limit.py+++ b/autogpt_platform/backend/backend/copilot/rate_limit.py@@ -62,7 +62,7 @@ from prisma.models import User as PrismaUserfrom pydantic import BaseModel, Fieldfrom redis.exceptions import RedisClusterException, RedisError-from backend.data.db_accessors import user_db+from backend.data.db_accessors import credit_db, user_dbfrom backend.data.redis_client import AsyncRedisClient, get_redis_asyncfrom backend.data.user import get_user_by_idfrom backend.util.cache import cached@@ -955,6 +955,11 @@ async def _maybe_reconcile_stripe_tier(user_id: str) -> bool:all fan out simultaneously. The key is deleted on transient errors sothe next request can retry. Returns True when a subscription was foundand synced (tier was updated in DB + cache cleared).++ The DB work routes through ``credit_db()`` so the reconcile also works+ from Prisma-less processes (scheduler-server, copilot-executor), where+ a direct ``backend.data.credit`` call would raise+ ``ClientNotConnectedError``."""gate_key = f"{_STRIPE_RECONCILE_PREFIX}{user_id}"try:@@ -968,9 +973,7 @@ async def _maybe_reconcile_stripe_tier(user_id: str) -> bool:return Falsetry:- from backend.data.credit import reconcile_stripe_tier_for_user # avoid circular-- return await reconcile_stripe_tier_for_user(user_id)+ return await credit_db().reconcile_stripe_tier_for_user(user_id)except Exception as exc:logger.warning("stripe_reconcile: check failed for %s: %s", user_id[:8], exc)try:diff --git a/autogpt_platform/backend/backend/copilot/rate_limit_test.py b/autogpt_platform/backend/backend/copilot/rate_limit_test.pyindex f21bf5bb11..4024f4b51f 100644--- a/autogpt_platform/backend/backend/copilot/rate_limit_test.py+++ b/autogpt_platform/backend/backend/copilot/rate_limit_test.py@@ -9,6 +9,7 @@ from redis.exceptions import RedisClusterException, RedisErrorfrom .rate_limit import (_DEFAULT_TIER_MULTIPLIERS,_DEFAULT_TIER_WORKSPACE_STORAGE_MB,+ _STRIPE_RECONCILE_PREFIX,DEFAULT_TIER,TIER_MULTIPLIERS,CoPilotUsagePublic,@@ -22,6 +23,7 @@ from .rate_limit import (_fetch_cost_limits_flag,_fetch_tier_multipliers_flag,_fetch_workspace_storage_limits_flag,+ _maybe_reconcile_stripe_tier,_weekly_key,_weekly_reset_time,acquire_reset_lock,@@ -1378,6 +1380,105 @@ class TestGetUserTier:assert tier2 == SubscriptionTier.PRO+# ---------------------------------------------------------------------------+# _maybe_reconcile_stripe_tier+# ---------------------------------------------------------------------------+++class TestMaybeReconcileStripeTier:+ """The lazy Stripe reconcile must resolve its DB work via the accessor+ layer (``credit_db()``) so it works from Prisma-less processes+ (scheduler-server, copilot-executor). A direct ``backend.data.credit``+ call raises ``ClientNotConnectedError`` there, which is swallowed —+ silently disabling tier reconciliation in those processes.+ """++ def _mock_redis(self, gate_acquired: bool = True):+ redis = AsyncMock()+ redis.set = AsyncMock(return_value=gate_acquired)+ redis.delete = AsyncMock()+ return redis++ def _mock_credit_db(self, result: bool = True, raises: Exception | None = None):+ accessor = AsyncMock()+ if raises is not None:+ accessor.reconcile_stripe_tier_for_user = AsyncMock(side_effect=raises)+ else:+ accessor.reconcile_stripe_tier_for_user = AsyncMock(return_value=result)+ return accessor++ @pytest.mark.asyncio+ async def test_reconcile_resolves_db_call_via_accessor(self):+ """The reconcile call must go through credit_db(), not a direct import."""+ redis = self._mock_redis()+ accessor = self._mock_credit_db(result=True)+ with (+ patch(+ "backend.copilot.rate_limit.get_redis_async",+ AsyncMock(return_value=redis),+ ),+ patch("backend.copilot.rate_limit.credit_db", return_value=accessor),+ ):+ result = await _maybe_reconcile_stripe_tier(_USER)+ assert result is True+ accessor.reconcile_stripe_tier_for_user.assert_awaited_once_with(_USER)++ @pytest.mark.asyncio+ async def test_gate_already_held_skips_reconcile(self):+ """When the Redis NX gate is already set, no DB call is made."""+ redis = self._mock_redis(gate_acquired=False)+ accessor = self._mock_credit_db()+ with (+ patch(+ "backend.copilot.rate_limit.get_redis_async",+ AsyncMock(return_value=redis),+ ),+ patch("backend.copilot.rate_limit.credit_db", return_value=accessor),+ ):+ result = await _maybe_reconcile_stripe_tier(_USER)+ assert result is False+ accessor.reconcile_stripe_tier_for_user.assert_not_awaited()++ @pytest.mark.asyncio+ async def test_reconcile_failure_returns_false_and_releases_gate(self):+ """A failing reconcile is swallowed and the NX gate is released for retry."""+ redis = self._mock_redis()+ accessor = self._mock_credit_db(raises=RuntimeError("RPC unavailable"))+ with (+ patch(+ "backend.copilot.rate_limit.get_redis_async",+ AsyncMock(return_value=redis),+ ),+ patch("backend.copilot.rate_limit.credit_db", return_value=accessor),+ ):+ result = await _maybe_reconcile_stripe_tier(_USER)+ assert result is False+ redis.delete.assert_awaited_once_with(f"{_STRIPE_RECONCILE_PREFIX}{_USER}")++ @pytest.mark.asyncio+ async def test_get_user_tier_survives_reconcile_failure(self):+ """Reconcile failure stays non-fatal for rate limiting: get_user_tier+ still resolves the DB-confirmed NO_TIER instead of raising."""+ get_user_tier.cache_clear() # type: ignore[attr-defined]+ mock_user = MagicMock()+ mock_user.subscription_tier = None+ mock_db = AsyncMock()+ mock_db.get_user_by_id = AsyncMock(return_value=mock_user)+ redis = self._mock_redis()+ accessor = self._mock_credit_db(raises=RuntimeError("RPC unavailable"))+ with (+ patch("backend.copilot.rate_limit.user_db", return_value=mock_db),+ patch(+ "backend.copilot.rate_limit.get_redis_async",+ AsyncMock(return_value=redis),+ ),+ patch("backend.copilot.rate_limit.credit_db", return_value=accessor),+ ):+ tier = await get_user_tier(_USER)+ assert tier == SubscriptionTier.NO_TIER+ accessor.reconcile_stripe_tier_for_user.assert_awaited_once_with(_USER)++# ---------------------------------------------------------------------------# set_user_tier# ---------------------------------------------------------------------------diff --git a/autogpt_platform/backend/backend/data/db_manager.py b/autogpt_platform/backend/backend/data/db_manager.pyindex bd950d666a..9c47ab6f39 100644--- a/autogpt_platform/backend/backend/data/db_manager.py+++ b/autogpt_platform/backend/backend/data/db_manager.py@@ -46,7 +46,11 @@ from backend.data.block import (get_blocks_needing_optimization,update_block_optimized_description,)-from backend.data.credit import UsageTransactionMetadata, get_user_credit_model+from backend.data.credit import (+ UsageTransactionMetadata,+ get_user_credit_model,+ reconcile_stripe_tier_for_user,+)from backend.data.execution import (create_graph_execution,get_block_error_stats,@@ -374,6 +378,10 @@ class DatabaseManager(AppService):# ============ Subscription Reconciliation ============ #reconcile_all_stripe_tiers = _(reconcile_all_stripe_tiers)+ # Per-user lazy reconcile, exposed so Prisma-less processes+ # (scheduler-server, copilot-executor) can self-heal a stale NO_TIER+ # row via db_accessors.credit_db() instead of crashing on direct Prisma.+ reconcile_stripe_tier_for_user = _(reconcile_stripe_tier_for_user)# ============ Platform Linking ============ #find_server_link_owner = _(platform_linking_db.find_server_link_owner)@@ -629,6 +637,7 @@ class DatabaseManagerAsyncClient(AppServiceClient):# ============ Subscription Reconciliation ============ #reconcile_all_stripe_tiers = d.reconcile_all_stripe_tiers+ reconcile_stripe_tier_for_user = d.reconcile_stripe_tier_for_user# ============ Platform Linking ============ #find_server_link_owner = d.find_server_link_ownerdiff --git a/autogpt_platform/backend/backend/util/feature_flag.py b/autogpt_platform/backend/backend/util/feature_flag.pyindex 1d0adb40e5..7d49d6d3d2 100644--- a/autogpt_platform/backend/backend/util/feature_flag.py+++ b/autogpt_platform/backend/backend/util/feature_flag.py@@ -195,47 +195,77 @@ def shutdown_launchdarkly() -> None:logger.info("LaunchDarkly client closed successfully")-@cached(maxsize=1000, ttl_seconds=86400) # 1000 entries, 24 hours TTLasync def _fetch_user_context_data(user_id: str) -> Context:"""Fetch user context for LaunchDarkly from Supabase.+ Successful lookups are cached for 24h (see+ ``_fetch_supabase_user_context``). Failed lookups are NOT cached: the+ degraded anonymous fallback is built outside the cache so the next+ evaluation retries Supabase instead of pinning this process to an+ email-less context for a full TTL — which would make its+ email/role-targeted flag evaluations silently diverge from peer+ processes. The degraded path costs one failed Supabase call per+ evaluation; bounded, and acceptable versus a 24h-poisoned cache.+Args:user_id: The user ID to fetch data forReturns:LaunchDarkly Context object"""- builder = Context.builder(user_id).kind("user").anonymous(True)-try:uuid.UUID(user_id)except ValueError:# Non-UUID key (e.g. "system") — skip Supabase lookup, return anonymous context.- return builder.build()+ return _anonymous_context(user_id)try:- from backend.util.clients import get_supabase-- # If we have user data, update context- response = get_supabase().auth.admin.get_user_by_id(user_id)- if response and response.user:- user = response.user- builder.anonymous(False)- if user.role:- builder.set("role", user.role)- # It's weird, I know, but it is what it is.- builder.set("custom", {"role": user.role})- if user.email:- builder.set("email", user.email)- builder.set("email_domain", user.email.split("@")[-1])- if user.created_at:- # ISO-8601 string — LD supports RFC3339 date targeting on- # this attribute (e.g. cohort users by signup window).- builder.set("created_at", user.created_at.isoformat())-+ return await _fetch_supabase_user_context(user_id)except Exception as e:- logger.warning(f"Failed to fetch user context for {user_id}: {e}")+ logger.warning(+ f"Failed to fetch user context for {user_id}: {e} — "+ "falling back to an uncached anonymous context; flag "+ "evaluations for this user may be degraded until the lookup "+ "succeeds"+ )+ return _anonymous_context(user_id)+++def _anonymous_context(user_id: str) -> Context:+ """Build a minimal anonymous LD context carrying only the user key."""+ return Context.builder(user_id).kind("user").anonymous(True).build()+++@cached(maxsize=1000, ttl_seconds=86400) # 1000 entries, 24 hours TTL+async def _fetch_supabase_user_context(user_id: str) -> Context:+ """+ Build the full LaunchDarkly context for ``user_id`` from Supabase.++ Raises on Supabase lookup failure: ``@cached`` never stores results+ of calls that raise, so a degraded context can't be cached here —+ the caller handles the fallback outside the cache.+ """+ from backend.util.clients import get_supabase++ builder = Context.builder(user_id).kind("user").anonymous(True)++ # If we have user data, update context+ response = get_supabase().auth.admin.get_user_by_id(user_id)+ if response and response.user:+ user = response.user+ builder.anonymous(False)+ if user.role:+ builder.set("role", user.role)+ # It's weird, I know, but it is what it is.+ builder.set("custom", {"role": user.role})+ if user.email:+ builder.set("email", user.email)+ builder.set("email_domain", user.email.split("@")[-1])+ if user.created_at:+ # ISO-8601 string — LD supports RFC3339 date targeting on+ # this attribute (e.g. cohort users by signup window).+ builder.set("created_at", user.created_at.isoformat())return builder.build()diff --git a/autogpt_platform/backend/backend/util/feature_flag_test.py b/autogpt_platform/backend/backend/util/feature_flag_test.pyindex 660cc3ca67..e030a19257 100644--- a/autogpt_platform/backend/backend/util/feature_flag_test.py+++ b/autogpt_platform/backend/backend/util/feature_flag_test.py@@ -1,4 +1,5 @@import datetime+import loggingimport uuidimport pytest@@ -206,3 +207,89 @@ class TestUserContext:assert "created_at" not in ctx.custom_attributesassert ctx.get("email") == "[email protected]"supabase.auth.admin.get_user_by_id.assert_called_once_with(user_id)+++class TestUserContextCacheDegradation:+ """A failed Supabase lookup must not poison the 24h context cache.++ If the degraded anonymous (email-less) context were cached, one+ Supabase blip would make this process evaluate email/role-targeted+ flags differently from its peers for a full day, silently.+ """++ @staticmethod+ def _stub_failing_supabase(mocker):+ supabase = mocker.MagicMock()+ supabase.auth.admin.get_user_by_id.side_effect = ConnectionError(+ "supabase unreachable"+ )+ mocker.patch("backend.util.clients.get_supabase", return_value=supabase)+ return supabase++ @pytest.mark.asyncio+ async def test_degraded_anonymous_context_is_not_cached(self, mocker):+ supabase = self._stub_failing_supabase(mocker)+ user_id = str(uuid.uuid4())++ first = await _fetch_user_context_data(user_id)+ second = await _fetch_user_context_data(user_id)++ assert first.anonymous is True+ assert second.anonymous is True+ assert supabase.auth.admin.get_user_by_id.call_count == 2++ @pytest.mark.asyncio+ async def test_successful_context_is_cached_across_calls(self, mocker):+ supabase = TestUserContext._stub_supabase(mocker, created_at=None)+ user_id = str(uuid.uuid4())++ first = await _fetch_user_context_data(user_id)+ second = await _fetch_user_context_data(user_id)++ assert first.get("email") == "[email protected]"+ assert second.get("email") == "[email protected]"+ assert supabase.auth.admin.get_user_by_id.call_count == 1++ @pytest.mark.asyncio+ async def test_context_lookup_recovers_after_transient_failure(self, mocker):+ user = mocker.MagicMock(role="authenticated", email="[email protected]", created_at=None)+ response = mocker.MagicMock(user=user)+ supabase = mocker.MagicMock()+ supabase.auth.admin.get_user_by_id.side_effect = [+ ConnectionError("supabase blip"),+ response,+ ]+ mocker.patch("backend.util.clients.get_supabase", return_value=supabase)+ user_id = str(uuid.uuid4())++ degraded = await _fetch_user_context_data(user_id)+ recovered = await _fetch_user_context_data(user_id)++ assert degraded.anonymous is True+ assert degraded.get("email") is None+ assert recovered.anonymous is False+ assert recovered.get("email") == "[email protected]"++ @pytest.mark.asyncio+ async def test_degraded_lookup_logs_degradation_warning(self, mocker, caplog):+ self._stub_failing_supabase(mocker)+ user_id = str(uuid.uuid4())++ with caplog.at_level(logging.WARNING, logger="backend.util.feature_flag"):+ await _fetch_user_context_data(user_id)++ warnings = [+ record.getMessage()+ for record in caplog.records+ if record.levelno >= logging.WARNING+ ]+ assert any(user_id in message and "degraded" in message for message in warnings)++ @pytest.mark.asyncio+ async def test_non_uuid_key_skips_supabase_lookup(self, mocker):+ get_supabase = mocker.patch("backend.util.clients.get_supabase")++ ctx = await _fetch_user_context_data("system")++ assert ctx.anonymous is True+ get_supabase.assert_not_called()