Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend): share the subscription lookup cache across API workers (#15412)

Nicholas Tindle78fcb7c45942
diff --git a/autogpt_platform/backend/backend/conftest.py b/autogpt_platform/backend/backend/conftest.py
index ba7eaec3d9..91f058d7a7 100644
--- a/autogpt_platform/backend/backend/conftest.py
+++ b/autogpt_platform/backend/backend/conftest.py
@@ -4,11 +4,13 @@ import os
from contextlib import asynccontextmanager
from multiprocessing import get_all_start_methods, get_context, set_forkserver_preload
from threading import Thread
+from uuid import uuid4
import pytest
import pytest_asyncio
from dotenv import load_dotenv
+from backend.data import subscription_lookup_cache
from backend.util.logging import configure_logging
load_dotenv()
@@ -64,6 +66,24 @@ def no_force_all_flags(monkeypatch: pytest.MonkeyPatch):
monkeypatch.delenv("NEXT_PUBLIC_FORCE_ALL_FLAGS", raising=False)
[email protected](autouse=True)
+def isolated_subscription_lookup_cache(monkeypatch: pytest.MonkeyPatch):
+ """The subscription lookup cache lives in Redis, which outlives a test and
+ is shared by parallel runs; give every test keys of its own, and a cache
+ that hasn't backed off after an earlier test's Redis failure."""
+ monkeypatch.setattr(
+ subscription_lookup_cache, "KEY_PREFIX", f"test:{uuid4().hex}:subscription"
+ )
+ monkeypatch.setattr(subscription_lookup_cache._backoff, "until", 0.0)
+ monkeypatch.setattr(subscription_lookup_cache, "_fallback", {})
+ monkeypatch.setattr(subscription_lookup_cache, "_in_flight", {})
+ monkeypatch.setattr(
+ subscription_lookup_cache,
+ "_unconfirmed",
+ subscription_lookup_cache._UnconfirmedInvalidations(),
+ )
+
+
@pytest.fixture
def test_user_id() -> str:
"""Test user ID fixture."""
diff --git a/autogpt_platform/backend/backend/data/credit.py b/autogpt_platform/backend/backend/data/credit.py
index 12b4843ac5..e591968e89 100644
--- a/autogpt_platform/backend/backend/data/credit.py
+++ b/autogpt_platform/backend/backend/data/credit.py
@@ -42,6 +42,11 @@ from backend.data.subscription_checkout import (
expire_other_subscription_checkouts,
subscription_checkout_lock,
)
+from backend.data.subscription_lookup_cache import (
+ ActiveSubscription,
+ invalidate_active_subscription_cache,
+ read_active_subscription,
+)
from backend.data.subscription_trial_stripe import reconcile_trial_subscription
from backend.data.user import get_user_by_id, get_user_email_by_id
from backend.notifications.queue import queue_notification_async
@@ -1629,6 +1634,23 @@ async def _cancel_customer_subscriptions(
# filter at a time (no OR), and we explicitly want to skip canceled/incomplete/
# past_due subs rather than filter them out client-side via status="all".
seen_ids: set[str] = set()
+ try:
+ await _cancel_listed_subscriptions(
+ customer_id, seen_ids, exclude_sub_id, at_period_end, cancellation_comment
+ )
+ finally:
+ if seen_ids:
+ await invalidate_active_subscription_cache(customer_id)
+ return len(seen_ids)
+
+
+async def _cancel_listed_subscriptions(
+ customer_id: str,
+ seen_ids: set[str],
+ exclude_sub_id: str | None,
+ at_period_end: bool,
+ cancellation_comment: str | None,
+) -> None:
for status in ("active", "trialing"):
subscriptions = await stripe_call(
stripe.Subscription.list_async,
@@ -1676,7 +1698,6 @@ async def _cancel_customer_subscriptions(
sub_id,
**_cancel_params(cancellation_comment, trial=False),
)
- return len(seen_ids)
def _cancel_params(
@@ -1759,10 +1780,12 @@ async def get_proration_credit_cents(user_id: str, monthly_cost_cents: int) -> i
# Proration only applies to a paid, active subscription — a trialing
# sub has no paid time to prorate. Matches the previous active-only
# query (status="active") while sharing the cached lookup.
- if sub is None or sub.get("status") != "active":
+ if sub is None or sub.status != "active":
+ return 0
+ period_start = sub.current_period_start
+ period_end = sub.current_period_end
+ if period_start is None or period_end is None:
return 0
- period_start: int = sub["current_period_start"]
- period_end: int = sub["current_period_end"]
now = int(time.time())
total_seconds = period_end - period_start
remaining_seconds = max(period_end - now, 0)
@@ -1820,10 +1843,9 @@ async def _get_active_subscription(customer_id: str) -> stripe.Subscription | No
return None
-@cached(ttl_seconds=15, maxsize=2048, cache_none=True)
async def _get_active_subscription_cached(
customer_id: str,
-) -> stripe.Subscription | None:
+) -> ActiveSubscription | None:
"""Short-TTL per-customer cache over :func:`_get_active_subscription`.
``get_subscription_status`` resolves the active subscription up to three
@@ -1834,20 +1856,17 @@ async def _get_active_subscription_cached(
per request, and again on every page view. A brief cache collapses them
into one lookup per customer per window.
- Read-only display helpers use this. Subscription mutation flows keep calling
- :func:`_get_active_subscription` directly so they always act on fresh Stripe
- state; the 15-second window here only affects display fields (billing cycle,
- next-invoice date, proration estimate), which the frontend re-fetches and
- which self-correct well inside the tolerances already used elsewhere (the
- pending-change lookup is cached for 30s).
+ The cache lives in Redis and is versioned per customer, so a subscription
+ change (``invalidate_active_subscription_cache``) is seen by every API
+ worker on its next read rather than only by the one that made it. Redis
+ trouble falls through to Stripe within a short deadline (see
+ ``backend.data.subscription_lookup_cache``). Subscription mutation flows
+ keep calling :func:`_get_active_subscription` directly so they always act
+ on fresh Stripe state.
"""
- return await _get_active_subscription(customer_id)
-
-
-def invalidate_active_subscription_cache(customer_id: str) -> None:
- """Drop the cached lookup after changing the subscription in Stripe, so the
- status returned right after shows the new period instead of the old one."""
- _get_active_subscription_cached.cache_delete(customer_id)
+ return await read_active_subscription(
+ customer_id, lambda: _get_active_subscription(customer_id)
+ )
async def get_user_billing_cycle(user_id: str) -> BillingCycle | None:
@@ -1870,15 +1889,9 @@ async def get_user_billing_cycle(user_id: str) -> BillingCycle | None:
"get_user_billing_cycle: Stripe lookup failed for user %s", user_id
)
return None
- if sub is None:
- return None
- items = sub["items"].data
- if not items:
- return None
- price = items[0].price
- current_price_id = price if isinstance(price, str) else price.id
- if not current_price_id:
+ if sub is None or not sub.price_id:
return None
+ current_price_id = sub.price_id
priceable = (
SubscriptionTier.BASIC,
@@ -1920,8 +1933,7 @@ async def get_active_subscription_period_end(user_id: str) -> int | None:
return None
if sub is None:
return None
- period_end = sub.current_period_end
- return int(period_end) if period_end else None
+ return sub.current_period_end or None
# Substrings Stripe uses in InvalidRequestError messages when the schedule is
@@ -2248,6 +2260,7 @@ async def modify_stripe_subscription_for_tier(
return True
finally:
get_pending_subscription_change.cache_delete(user_id)
+ await invalidate_active_subscription_cache(user.stripe_customer_id)
async def release_pending_subscription_schedule(user_id: str) -> bool:
@@ -2317,6 +2330,7 @@ async def release_pending_subscription_schedule(user_id: str) -> bool:
finally:
if did_anything:
get_pending_subscription_change.cache_delete(user_id)
+ await invalidate_active_subscription_cache(user.stripe_customer_id)
return did_anything
@@ -2778,8 +2792,18 @@ async def sync_subscription_from_stripe(
``track_lifecycle=False`` is for the periodic tier sweep, which would
otherwise fan out one Stripe call per trial; the daily lifecycle sweep
covers those users.
+
+ The payload is Stripe's latest word on the subscription, so afterwards
+ the cached lookup the billing page reads is dropped on every worker. Only
+ afterwards: saving the tier can change Stripe too (a rejected trial is
+ canceled), and a read in between would cache the state before that.
"""
- await _sync_subscription_tier_from_stripe(stripe_subscription)
+ customer_id = stripe_subscription.get("customer")
+ try:
+ await _sync_subscription_tier_from_stripe(stripe_subscription)
+ finally:
+ if isinstance(customer_id, str) and customer_id:
+ await invalidate_active_subscription_cache(customer_id)
if track_lifecycle:
schedule_posthog_lifecycle_sync(
stripe_customer_id=stripe_subscription.get("customer")
@@ -3363,6 +3387,11 @@ async def handle_subscription_payment_success(invoice: dict) -> None:
if not sub_id:
# Non-subscription invoices (one-off invoices, etc.) — no credit grant.
return
+ # A paid invoice can start a new period (a renewal, a plan's first
+ # charge). Nothing below changes Stripe, so dropping the billing page's
+ # cached lookup now is as good as after; it doesn't wait for the
+ # customer.subscription.updated webhook that usually follows.
+ await invalidate_active_subscription_cache(customer_id)
user = await User.prisma().find_first(where={"stripeCustomerId": customer_id})
if not user:
logger.warning(
diff --git a/autogpt_platform/backend/backend/data/credit_subscription_test.py b/autogpt_platform/backend/backend/data/credit_subscription_test.py
index 2a14f688d5..0de3c61cf8 100644
--- a/autogpt_platform/backend/backend/data/credit_subscription_test.py
+++ b/autogpt_platform/backend/backend/data/credit_subscription_test.py
@@ -47,19 +47,6 @@ def _clear_cache(fn: _CacheClearable) -> None:
fn.cache_clear()
[email protected](autouse=True)
-def _clear_active_subscription_cache():
- """Isolate the short-TTL active-subscription cache between tests.
-
- ``get_user_billing_cycle`` / ``get_active_subscription_period_end`` /
- ``get_proration_credit_cents`` share ``_get_active_subscription_cached``;
- clear it so a subscription mocked in one test cannot leak into the next
- through the cache.
- """
- _clear_cache(_get_active_subscription_cached)
- yield
-
-
@pytest.fixture(autouse=True)
def _no_pending_update():
"""The payment failures here are plans that lapsed, not a change Stripe
@@ -982,7 +969,8 @@ async def test_active_subscription_lookup_is_cached_within_window():
) as raw:
first = await _get_active_subscription_cached("cus_cache")
second = await _get_active_subscription_cached("cus_cache")
- assert first is second is mock_sub
+ assert first == second
+ assert first is not None and first.status == "active"
raw.assert_awaited_once()
diff --git a/autogpt_platform/backend/backend/data/credit_trial_subscription_test.py b/autogpt_platform/backend/backend/data/credit_trial_subscription_test.py
index bb8b92d38f..665b517ed6 100644
--- a/autogpt_platform/backend/backend/data/credit_trial_subscription_test.py
+++ b/autogpt_platform/backend/backend/data/credit_trial_subscription_test.py
@@ -8,7 +8,6 @@ import pytest
import stripe
from prisma.enums import SubscriptionTier
-from backend.data import credit_subscription_test
from backend.data.credit import (
_get_active_subscription_cached,
get_pending_subscription_change,
@@ -17,10 +16,6 @@ from backend.data.credit import (
)
from backend.data.credit_subscription_test import _clear_cache, _make_user
-_clear_active_subscription_cache = (
- credit_subscription_test._clear_active_subscription_cache
-)
-
@pytest.mark.asyncio
async def test_new_plan_ends_cancel_pending_trial_without_invoice():
@@ -177,19 +172,24 @@ async def test_get_pending_subscription_change_reports_paid_cancel(metadata):
async def test_invalidating_the_subscription_lookup_refetches_only_that_customer():
"""A subscription changed in place is read fresh on the next status call;
other customers keep their cached lookup."""
- trialing = {"id": "sub_trial", "status": "trialing"}
- converted = {"id": "sub_trial", "status": "active"}
- other = {"id": "sub_other", "status": "active"}
+ trialing = {"id": "sub_trial", "status": "trialing", "current_period_end": 1}
+ converted = {"id": "sub_trial", "status": "active", "current_period_end": 2}
+ other = {"id": "sub_other", "status": "active", "current_period_end": 3}
+
+ async def period_end(customer_id: str) -> int | None:
+ sub = await _get_active_subscription_cached(customer_id)
+ return sub.current_period_end if sub else None
+
with patch(
"backend.data.credit._get_active_subscription",
new_callable=AsyncMock,
side_effect=[trialing, other, converted],
) as lookup:
- assert await _get_active_subscription_cached("cus_trial") is trialing
- assert await _get_active_subscription_cached("cus_other") is other
+ assert await period_end("cus_trial") == 1
+ assert await period_end("cus_other") == 3
- invalidate_active_subscription_cache("cus_trial")
+ await invalidate_active_subscription_cache("cus_trial")
- assert await _get_active_subscription_cached("cus_trial") is converted
- assert await _get_active_subscription_cached("cus_other") is other
+ assert await period_end("cus_trial") == 2
+ assert await period_end("cus_other") == 3
assert lookup.await_count == 3
diff --git a/autogpt_platform/backend/backend/data/subscription_lookup_cache.py b/autogpt_platform/backend/backend/data/subscription_lookup_cache.py
new file mode 100644
index 0000000000..b9fcfb8c07
--- /dev/null
+++ b/autogpt_platform/backend/backend/data/subscription_lookup_cache.py
@@ -0,0 +1,388 @@
+"""The active-subscription lookup cache, shared by every API worker.
+
+The billing status endpoint reads a customer's active subscription several
+times per request, on almost every page load, so the lookup is cached briefly
+in Redis. Each customer has a random version token; every subscription change
+replaces it, so the next read on any worker misses and asks Stripe again. An
+entry is written only while the token it was read under is still current, so
+a read that was in flight when a change landed can't put the old answer back.
+A missing token (expired, or Redis lost its data) is never treated as a
+version: the reader establishes a fresh one before it asks Stripe.
+
+Within a process, concurrent lookups for one customer and token share one
+Stripe call. A lookup stores its answer only while it is still the current one
+for its key, so an answer read before a change this process made is returned
+to the callers that were already waiting for it, and never kept.
+
+Redis is only an optimisation here. Every operation has a short hard deadline,
+and after a failure this process skips Redis for a few seconds, so an outage
+or a hung connection costs a lookup at most one deadline, and never blocks or
+fails the billing change around it. Meanwhile answers are kept in process for
+a few seconds, to keep Stripe traffic bounded. A Redis that isn't a cluster
+(the platform always runs one) gets this fallback all the time.
+"""
+
+import asyncio
+import logging
+import time
+from typing import Any, Awaitable, Callable, Coroutine, TypeVar
+from uuid import uuid4
+
+import stripe
+from pydantic import BaseModel, ValidationError
+
+from backend.data.redis_client import AsyncRedisClient, get_redis_async
+
+logger = logging.getLogger(__name__)
+
+T = TypeVar("T")
+Fetch = Callable[[], Awaitable[stripe.Subscription | None]]
+
+SUBSCRIPTION_LOOKUP_TTL_SECONDS = 15
+# Bounds connecting as well as the command: the shared client retries a
+# failed connection for minutes on its own.
+_REDIS_DEADLINE_SECONDS = 0.3
+_REDIS_BACKOFF_SECONDS = 5.0
+# How long this process reuses an answer while Redis can't be used; also how
+# long another worker can show the old state after a change in that time.
+_FALLBACK_TTL_SECONDS = 5.0
+_FALLBACK_MAXSIZE = 2048
+_UNCONFIRMED_MAXSIZE = 2048
+_VERSION_TTL_SECONDS = 24 * 60 * 60
+KEY_PREFIX = "active_subscription"
+# The in-flight key for reads while Redis can't be used; tokens are hex.
+_NO_REDIS = "no-redis"
+
+# KEYS: version, lookup. ARGV: version read, entry, TTL.
+_WRITE_IF_CURRENT = """
+if redis.call('GET', KEYS[1]) == ARGV[1] then
+ return redis.call('SET', KEYS[2], ARGV[2], 'EX', ARGV[3])
+end
+return false
+"""
+
+
+class ActiveSubscription(BaseModel):
+ """What the billing display reads from the active or trialing subscription.
+
+ Cached instead of the Stripe object, which carries the API key it was
+ fetched with.
+ """
+
+ status: str | None = None
+ current_period_start: int | None = None
+ current_period_end: int | None = None
+ price_id: str | None = None
+
+
+class _CachedLookup(BaseModel):
+ version: str
+ subscription: ActiveSubscription | None
+
+
+class _RedisUnavailable(Exception):
+ pass
+
+
+class _Backoff:
+ """Process-local: skip Redis for a while after it failed."""
+
+ def __init__(self) -> None:
+ self.until = 0.0
+
+ def active(self) -> bool:
+ return time.monotonic() < self.until
+
+ def trip(self) -> None:
+ if not self.active():
+ logger.warning(
+ "Subscription lookup cache unavailable; reading Stripe directly"
+ f" for {_REDIS_BACKOFF_SECONDS:.0f}s"
+ )
+ self.until = time.monotonic() + _REDIS_BACKOFF_SECONDS
+
+
+_backoff = _Backoff()
+# Lookups running in this process, by (event loop, customer, version).
+_in_flight: dict[
+ tuple[asyncio.AbstractEventLoop, str, str],
+ asyncio.Task[ActiveSubscription | None],
+] = {}
+# Answers kept while Redis can't be used: customer -> (expires at, answer).
+_fallback: dict[str, tuple[float, ActiveSubscription | None]] = {}
+
+
+class _UnconfirmedInvalidations:
+ """Customers whose invalidation this process couldn't record in Redis.
+
+ A write already on its way when the change landed is stored under the
+ old, unchanged token, so until any such entry has expired this process
+ reads the customer as if Redis were out. A mark is never dropped while it
+ is live: with no room left, every customer is covered until the latest
+ mark would have expired.
+ """
+
+ def __init__(self) -> None:
+ self.until: dict[str, float] = {}
+ self.everyone_until = 0.0
+
+ def add(self, customer_id: str) -> None:
+ now = time.monotonic()
+ # Outlives any entry written now: a write in flight finishes within
+ # the Redis deadline, and entries live for the lookup TTL.
+ until = now + SUBSCRIPTION_LOOKUP_TTL_SECONDS + 1
+ self._purge(now)
+ # Re-inserted at the end, so insertion order stays expiry order.
+ self.until.pop(customer_id, None)
+ if len(self.until) >= _UNCONFIRMED_MAXSIZE:
+ self.everyone_until = max(self.everyone_until, until)
+ return
+ self.until[customer_id] = until
+
+ def clear(self, customer_id: str) -> None:
+ self.until.pop(customer_id, None)
+
+ def covers(self, customer_id: str) -> bool:
+ now = time.monotonic()
+ if now < self.everyone_until:
+ return True
+ until = self.until.get(customer_id)
+ return until is not None and until > now
+
+ def _purge(self, now: float) -> None:
+ for customer_id, until in list(self.until.items()):
+ if until > now:
+ break
+ del self.until[customer_id]
+
+
+_unconfirmed = _UnconfirmedInvalidations()
+
+
+def summarize_subscription(
+ subscription: stripe.Subscription | dict | None,
+) -> ActiveSubscription | None:
+ if subscription is None:
+ return None
+ items = (subscription.get("items") or {}).get("data") or []
+ price = items[0].get("price") if items else None
+ return ActiveSubscription(
+ status=subscription.get("status"),
+ current_period_start=subscription.get("current_period_start"),
+ current_period_end=subscription.get("current_period_end"),
+ price_id=price if isinstance(price, str) else (price or {}).get("id"),
+ )
+
+
+async def read_active_subscription(
+ customer_id: str, fetch: Fetch
+) -> ActiveSubscription | None:
+ """The cached lookup, or ``fetch``'s answer when it isn't cached."""
+ if _is_unconfirmed(customer_id):
+ return await _read_without_redis(customer_id, fetch)
+ try:
+ version, cached = await _read_entry(customer_id)
+ if version is None:
+ # Established before the in-flight key is chosen, so a read after
+ # Redis lost the token never joins a lookup started before.
+ version = await _establish_version(customer_id)
+ except _RedisUnavailable:
+ return await _read_without_redis(customer_id, fetch)
+ if cached is not None and cached.version == version:
+ return cached.subscription
+ return await _shared(
+ customer_id, version, lambda: _fetch_and_store(customer_id, version, fetch)
+ )
+
+
+async def invalidate_active_subscription_cache(customer_id: str) -> None:
+ """Make every worker read the customer's subscription fresh from Stripe.
+
+ Call it after the change has landed in Stripe, never before: a read
+ started earlier could otherwise cache the old state under the new token.
+ Never raises and returns within the Redis deadline, because the change it
+ follows has already been made.
+ """
+ _forget_locally(customer_id)
+ try:
+ await _call_redis(
+ lambda redis: redis.set(
+ _version_key(customer_id), uuid4().hex, ex=_VERSION_TTL_SECONDS
+ ),
+ skip_during_backoff=False,
+ )
+ except _RedisUnavailable:
+ # The old token still vouches for entries written under it.
+ _unconfirmed.add(customer_id)
+ logger.warning(
+ f"Could not invalidate the subscription lookup for {customer_id};"
+ f" other workers can show it stale for {SUBSCRIPTION_LOOKUP_TTL_SECONDS}s"
+ )
+ else:
+ # From now on the script refuses anything read under the old token.
+ _unconfirmed.clear(customer_id)
+
+
+async def _read_entry(customer_id: str) -> tuple[str | None, _CachedLookup | None]:
+ keys = (_version_key(customer_id), _lookup_key(customer_id))
+ version, raw = await _call_redis(lambda redis: redis.mget(keys))
+ return _text(version), _parse_lookup(_text(raw))
+
+
+async def _establish_version(customer_id: str) -> str:
+ """A fresh token for a customer who has none, or the one another reader
+ created first."""
+ token = uuid4().hex
+ key = _version_key(customer_id)
+ created = await _call_redis(
+ lambda redis: redis.set(key, token, nx=True, ex=_VERSION_TTL_SECONDS)
+ )
+ if created:
+ return token
+ current = _text(await _call_redis(lambda redis: redis.get(key)))
+ if current:
+ return current
+ # Expired again in between: a token of our own only costs a miss.
+ await _call_redis(lambda redis: redis.set(key, token, ex=_VERSION_TTL_SECONDS))
+ return token
+
+
+async def _fetch_and_store(
+ customer_id: str, version: str, fetch: Fetch
+) -> ActiveSubscription | None:
+ # Another lookup may have stored it while this one was being started.
+ try:
+ current, cached = await _read_entry(customer_id)
+ except _RedisUnavailable:
+ current, cached = None, None
+ if current == version and cached is not None and cached.version == version:
+ return cached.subscription
+ subscription = summarize_subscription(await fetch())
+ if not _still_current(customer_id, version) or _is_unconfirmed(customer_id):
+ return subscription
+ entry = _CachedLookup(version=version, subscription=subscription)
+ keys_and_args = (
+ _version_key(customer_id),
+ _lookup_key(customer_id),
+ version,
+ entry.model_dump_json(),
+ SUBSCRIPTION_LOOKUP_TTL_SECONDS,
+ )
+ try:
+ await _call_redis(
+ lambda redis: redis.eval(_WRITE_IF_CURRENT, 2, *keys_and_args)
+ )
+ except _RedisUnavailable:
+ pass
+ return subscription
+
+
+async def _read_without_redis(
+ customer_id: str, fetch: Fetch
+) -> ActiveSubscription | None:
+ kept = _fallback.get(customer_id)
+ if kept is not None and kept[0] > time.monotonic():
+ return kept[1]
+ return await _shared(
+ customer_id, _NO_REDIS, lambda: _fetch_and_keep(customer_id, fetch)
+ )
+
+
+async def _fetch_and_keep(customer_id: str, fetch: Fetch) -> ActiveSubscription | None:
+ subscription = summarize_subscription(await fetch())
+ if not _still_current(customer_id, _NO_REDIS):
+ return subscription
+ _fallback[customer_id] = (time.monotonic() + _FALLBACK_TTL_SECONDS, subscription)
+ if len(_fallback) > _FALLBACK_MAXSIZE:
+ # Dropping a kept answer only costs another Stripe call.
+ del _fallback[next(iter(_fallback))]
+ return subscription
+
+
+def _is_unconfirmed(customer_id: str) -> bool:
+ return _unconfirmed.covers(customer_id)
+
+
+async def _shared(
+ customer_id: str,
+ version: str,
+ lookup: Callable[[], Coroutine[Any, Any, ActiveSubscription | None]],
+) -> ActiveSubscription | None:
+ """Run ``lookup`` once for every concurrent caller with the same key. The
+ version is part of the key, so a read after a change never joins one
+ started before it."""
+ key = (asyncio.get_running_loop(), customer_id, version)
+ task = _in_flight.get(key)
+ if task is None:
+ task = asyncio.create_task(lookup())
+ _in_flight[key] = task
+ task.add_done_callback(lambda done: _drop_in_flight(key, done))
+ # One caller giving up must not cancel the lookup the others wait on.
+ return await asyncio.shield(task)
+
+
+def _still_current(customer_id: str, version: str) -> bool:
+ """Whether the running lookup still owns its in-flight key: invalidating
+ in this process drops it, and then its answer must not be kept."""
+ key = (asyncio.get_running_loop(), customer_id, version)
+ return _in_flight.get(key) is asyncio.current_task()
+
+
+def _drop_in_flight(
+ key: tuple[asyncio.AbstractEventLoop, str, str],
+ task: asyncio.Task[ActiveSubscription | None],
+) -> None:
+ if _in_flight.get(key) is task:
+ del _in_flight[key]
+
+
+def _forget_locally(customer_id: str) -> None:
+ """Stop this process reusing what it read before a change: its fallback
+ answer, and lookups still running (whose version may not have changed if
+ Redis couldn't be told)."""
+ _fallback.pop(customer_id, None)
+ for key in [key for key in _in_flight if key[1] == customer_id]:
+ del _in_flight[key]
+
+
+def _version_key(customer_id: str) -> str:
+ # The hash tag keeps both keys on one cluster slot, for MGET and the script.
+ return f"{KEY_PREFIX}:{{{customer_id}}}:version"
+
+
+def _lookup_key(customer_id: str) -> str:
+ return f"{KEY_PREFIX}:{{{customer_id}}}:lookup"
+
+
+def _text(value: str | bytes | None) -> str | None:
+ # The shared client decodes responses; its type hints don't say so.
+ return value.decode() if isinstance(value, bytes) else value
+
+
+def _parse_lookup(raw: str | None) -> _CachedLookup | None:
+ if not raw:
+ return None
+ try:
+ return _CachedLookup.model_validate_json(raw)
+ except ValidationError:
+ return None
+
+
+async def _call_redis(
+ operation: Callable[[AsyncRedisClient], Awaitable[T]],
+ *,
+ skip_during_backoff: bool = True,
+) -> T:
+ if skip_during_backoff and _backoff.active():
+ raise _RedisUnavailable()
+ try:
+ return await asyncio.wait_for(
+ _with_client(operation), timeout=_REDIS_DEADLINE_SECONDS
+ )
+ except Exception as exc:
+ _backoff.trip()
+ raise _RedisUnavailable() from exc
+
+
+async def _with_client(operation: Callable[[AsyncRedisClient], Awaitable[T]]) -> T:
+ return await operation(await get_redis_async())
diff --git a/autogpt_platform/backend/backend/data/subscription_lookup_cache_interleaving_test.py b/autogpt_platform/backend/backend/data/subscription_lookup_cache_interleaving_test.py
new file mode 100644
index 0000000000..3a56f89779
--- /dev/null
+++ b/autogpt_platform/backend/backend/data/subscription_lookup_cache_interleaving_test.py
@@ -0,0 +1,212 @@
+"""Randomized interleavings of the subscription lookup cache.
+
+The invariant that matters to a customer: once this worker has changed a
+subscription and invalidated the lookup, no read that starts afterwards on
+this worker returns the subscription as it was before the change. Each seed
+interleaves reads, changes, Redis outages and blips (failed writes, replies
+lost after the write applied, lost data), slow Stripe answers and slow
+script writes, on a fake clock, with the in-process bounds shrunk so they
+are under pressure. Nothing sleeps; the clock only moves when told to.
+"""
+
+import asyncio
+import random
+
+import pytest
+from redis.exceptions import ConnectionError as RedisConnectionError
+
+from backend.data import subscription_lookup_cache as cache
+
+SEEDS = range(300)
+STEPS = 80
+CUSTOMERS = ("cus_a", "cus_b", "cus_c", "cus_d")
+BASE = 1_800_000_000
+# Real Redis finishes or drops a command within the client deadline.
+DEADLINE = 0.3
+
+
+class Clock:
+ def __init__(self) -> None:
+ self.now = 1000.0
+
+ def monotonic(self) -> float:
+ return self.now
+
+
+class Holds:
+ """Answers held until the run releases them; Redis ones are dropped once
+ the clock passes the client deadline, as the real client would."""
+
+ def __init__(self, clock: Clock, rng: random.Random, chance: float) -> None:
+ self.clock = clock
+ self.rng = rng
+ self.chance = chance
+ self.held: list[tuple[float, asyncio.Future[None]]] = []
+
+ async def maybe_hold(self) -> None:
+ if self.rng.random() >= self.chance:
+ return
+ future = asyncio.get_running_loop().create_future()
+ self.held.append((self.clock.now, future))
+ await future
+
+ def release_one(self) -> None:
+ waiting = [future for _, future in self.held if not future.done()]
+ if waiting:
+ self.rng.choice(waiting).set_result(None)
+
+ def release_all(self) -> None:
+ for _, future in self.held:
+ if not future.done():
+ future.set_result(None)
+
+ def drop_older_than(self, limit: float) -> None:
+ for started, future in self.held:
+ if not future.done() and self.clock.now - started > limit:
+ future.set_exception(TimeoutError("past the client deadline"))
+
+
+class FakeRedis:
+ """The commands the cache uses, with TTLs on the fake clock."""
+
+ def __init__(self, clock: Clock, holds: Holds) -> None:
+ self.clock = clock
+ self.holds = holds
+ self.data: dict[str, tuple[str, float]] = {}
+ self.mode = "up"
+
+ def _read(self, key: str) -> str | None:
+ stored = self.data.get(key)
+ if stored is None or stored[1] <= self.clock.now:
+ return None
+ return stored[0]
+
+ def _reachable(self) -> None:
+ if self.mode == "down":
+ raise RedisConnectionError("down")
+
+ async def mget(self, keys):
+ self._reachable()
+ return [self._read(key) for key in keys]
+
+ async def get(self, key):
+ self._reachable()
+ return self._read(key)
+
+ async def set(self, key, value, nx=False, ex=None):
+ self._reachable()
+ if self.mode == "set-fails":
+ raise RedisConnectionError("blip")
+ if nx and self._read(key) is not None:
+ return None
+ self.data[key] = (value, self.clock.now + ex)
+ if self.mode == "reply-lost":
+ raise RedisConnectionError("reply lost after the write applied")
+ return True
+
+ async def eval(self, script, numkeys, version_key, lookup_key, version, entry, ttl):
+ self._reachable()
+ await self.holds.maybe_hold()
+ if self._read(version_key) != version:
+ return None
+ self.data[lookup_key] = (entry, self.clock.now + ttl)
+ return True
+
+
+class Run:
+ def __init__(self, seed: int, monkeypatch: pytest.MonkeyPatch) -> None:
+ self.seed = seed
+ self.rng = random.Random(seed)
+ self.clock = Clock()
+ self.stripe_holds = Holds(self.clock, self.rng, 0.3)
+ self.redis_holds = Holds(self.clock, self.rng, 0.3)
+ self.redis = FakeRedis(self.clock, self.redis_holds)
+ self.state = dict.fromkeys(CUSTOMERS, 0)
+ self.completed = dict.fromkeys(CUSTOMERS, 0)
+ self.violations: list[str] = []
+ self.tasks: list[asyncio.Task[None]] = []
+
+ async def get_redis():
+ return self.redis
+
+ monkeypatch.setattr(cache, "time", self.clock)
+ monkeypatch.setattr(cache, "get_redis_async", get_redis)
+ monkeypatch.setattr(cache, "_REDIS_DEADLINE_SECONDS", 3600)
+ monkeypatch.setattr(cache, "_FALLBACK_MAXSIZE", 1)
+ monkeypatch.setattr(cache, "_UNCONFIRMED_MAXSIZE", 1)
+ monkeypatch.setattr(cache, "_backoff", cache._Backoff())
+ monkeypatch.setattr(cache, "_unconfirmed", cache._UnconfirmedInvalidations())
+ monkeypatch.setattr(cache, "_fallback", {})
+ monkeypatch.setattr(cache, "_in_flight", {})
+
+ def fetch(self, customer_id: str):
+ async def ask_stripe():
+ value = self.state[customer_id]
+ await self.stripe_holds.maybe_hold()
+ return {
+ "status": "active",
+ "current_period_start": BASE,
+ "current_period_end": BASE + value,
+ }
+
+ return ask_stripe
+
+ async def read(self, customer_id: str) -> None:
+ required = self.completed[customer_id]
+ subscription = await cache.read_active_subscription(
+ customer_id, self.fetch(customer_id)
+ )
+ seen = (subscription.current_period_end or 0) - BASE if subscription else -1
+ if seen < required:
+ self.violations.append(
+ f"seed {self.seed}: {customer_id} read {seen} after change {required}"
+ )
+
+ async def change(self, customer_id: str) -> None:
+ self.state[customer_id] += 1
+ value = self.state[customer_id]
+ await cache.invalidate_active_subscription_cache(customer_id)
+ self.completed[customer_id] = max(self.completed[customer_id], value)
+
+ def step(self) -> None:
+ roll = self.rng.random()
+ customer_id = self.rng.choice(CUSTOMERS)
+ if roll < 0.35:
+ self.tasks.append(asyncio.create_task(self.read(customer_id)))
+ elif roll < 0.5:
+ self.tasks.append(asyncio.create_task(self.change(customer_id)))
+ elif roll < 0.6:
+ self.redis.mode = self.rng.choice(
+ ["up", "up", "down", "set-fails", "reply-lost"]
+ )
+ elif roll < 0.63:
+ self.redis.data.clear()
+ elif roll < 0.8:
+ self.rng.choice([self.stripe_holds, self.redis_holds]).release_one()
+ else:
+ self.clock.now += self.rng.choice([0.1, 0.5, 2, 6, 16, 20])
+ self.redis_holds.drop_older_than(DEADLINE)
+
+ async def play(self) -> None:
+ for _ in range(STEPS):
+ self.step()
+ for _ in range(self.rng.randint(1, 6)):
+ await asyncio.sleep(0)
+ self.redis.mode = "up"
+ while not all(task.done() for task in self.tasks):
+ self.stripe_holds.release_all()
+ self.redis_holds.release_all()
+ await asyncio.sleep(0)
+ for task in self.tasks:
+ task.result()
+
+
+async def test_no_read_after_a_change_returns_the_old_subscription(monkeypatch):
+ violations: list[str] = []
+ for seed in SEEDS:
+ with monkeypatch.context() as patched:
+ run = Run(seed, patched)
+ await asyncio.wait_for(run.play(), 30)
+ violations.extend(run.violations)
+ assert violations == []
diff --git a/autogpt_platform/backend/backend/data/subscription_lookup_cache_test.py b/autogpt_platform/backend/backend/data/subscription_lookup_cache_test.py
new file mode 100644
index 0000000000..f849dfe5bd
--- /dev/null
+++ b/autogpt_platform/backend/backend/data/subscription_lookup_cache_test.py
@@ -0,0 +1,599 @@
+"""The billing page's subscription lookup stays fresh across API workers, and
+Redis trouble never slows or blocks a billing change.
+
+Each "worker" is its own copy of ``backend.data.credit`` and of the lookup
+cache module, so it holds its own process-local state, the way separate
+uvicorn processes or pods do; both share the real Redis of the test stack and
+one fake Stripe.
+"""
+
+import asyncio
+import importlib.util
+import time
+from contextlib import asynccontextmanager
+from datetime import UTC, datetime, timedelta
+from types import ModuleType
+from unittest.mock import AsyncMock, MagicMock, patch
+from uuid import uuid4
+
+import pytest
+import pytest_asyncio
+import stripe
+from prisma.enums import SubscriptionTier
+from redis.exceptions import ConnectionError as RedisConnectionError
+from redis.exceptions import RedisClusterException
+
+from backend.data import credit as worker_a
+from backend.data import subscription_lookup_cache as cache_a
+from backend.data import subscription_trial_conversion as conversion
+from backend.data.redis_client import get_redis_async
+from backend.data.subscription_trial import TrialState
+
+pytest_plugins = ("backend.data.subscription_trial_fixtures",)
+
+TRIAL_END = int((datetime.now(UTC) + timedelta(days=7)).timestamp())
+PAID_PERIOD_END = int((datetime.now(UTC) + timedelta(days=31)).timestamp())
+WAIT = 5 # seconds; no wait in these tests may hang the suite
+
+
+def _load_copy(name: str) -> ModuleType:
+ spec = importlib.util.find_spec(name)
+ assert spec is not None and spec.loader is not None
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+def _load_worker() -> ModuleType:
+ """A second, independently loaded credit module with its own cache module."""
+ cache = _load_copy("backend.data.subscription_lookup_cache")
+ cache.KEY_PREFIX = cache_a.KEY_PREFIX
+ worker = _load_copy("backend.data.credit")
+ worker.read_active_subscription = cache.read_active_subscription
+ worker.invalidate_active_subscription_cache = (
+ cache.invalidate_active_subscription_cache
+ )
+ return worker
+
+
+class FakeStripe:
+ """One customer's subscription, as Stripe would answer for it."""
+
+ def __init__(self, subscription: dict):
+ self.subscription = subscription
+ self.lookups = 0
+ self.reached = asyncio.Event()
+
+ def current(self) -> stripe.Subscription:
+ return stripe.Subscription.construct_from(self.subscription, "test-key")
+
+ def hold_next_lookup(
+ self, monkeypatch: pytest.MonkeyPatch, worker: ModuleType
+ ) -> asyncio.Event:
+ """Answer ``worker``'s next lookup with the subscription as it is now,
+ once released: a slow answer that arrives after a change made
+ meanwhile. Later lookups ask the fake Stripe as usual.
+
+ Held per lookup, not per task: on Python 3.11 ``asyncio.wait_for``
+ (inside ``stripe_call``) runs each SDK call in a task of its own."""
+ before = self.current()
+ release = asyncio.Event()
+ ask_stripe = worker._get_active_subscription
+ held = False
+
+ async def lookup(customer_id: str):
+ nonlocal held
+ if held:
+ return await ask_stripe(customer_id)
+ held = True
+ self.lookups += 1
+ self.reached.set()
+ await asyncio.wait_for(release.wait(), WAIT)
+ return before
+
+ monkeypatch.setattr(worker, "_get_active_subscription", lookup)
+ return release
+
+ async def list_async(self, *, customer: str, status: str, limit: int):
+ if status == "active":
+ # _get_active_subscription asks for "active" first on every lookup.
+ self.lookups += 1
+ # Stripe takes a moment to answer, so concurrent lookups overlap.
+ await asyncio.sleep(0.02)
+ answer = self.current()
+ matches = (
+ [answer]
+ if customer == answer["customer"] and status == answer["status"]
+ else []
+ )
+ return stripe.ListObject.construct_from(
+ {"data": matches, "has_more": False}, "test-key"
+ )
+
+ async def retrieve_async(self, subscription_id: str):
+ return self.current()
+
+ async def modify_async(self, subscription_id: str, **changes):
+ if changes.get("trial_end") == "now":
+ self.subscription = {
+ **self.subscription,
+ "status": "active",
+ "cancel_at_period_end": False,
+ "current_period_start": int(datetime.now(UTC).timestamp()),
+ "current_period_end": PAID_PERIOD_END,
+ }
+ return self.current()
+
+
+def trial(trial: TrialState) -> TrialState:
+ """A cancel-pending trial on a customer no other test touches."""
+ now = datetime.now(UTC)
+ return trial.model_copy(
+ update={
+ "customer_id": f"cus_{uuid4().hex}",
+ "subscription_id": "sub_1",
+ "status": "trialing",
+ "card_verified_at": now - timedelta(days=2),
+ "started_at": now - timedelta(days=2),
+ "ends_at": datetime.fromtimestamp(TRIAL_END, UTC),
+ "consumed_at": now - timedelta(days=2),
+ "cancel_at_period_end": True,
+ }
+ )
+
+
+def fake_stripe(trial: TrialState, subscription: dict) -> FakeStripe:
+ return FakeStripe(
+ {
+ **subscription,
+ "customer": trial.customer_id,
+ "cancel_at_period_end": True,
+ "trial_end": TRIAL_END,
+ "current_period_start": TRIAL_END - 7 * 24 * 60 * 60,
+ "current_period_end": TRIAL_END,
+ }
+ )
+
+
+def save_tier() -> AsyncMock:
+ """Where the paid plan is saved to the user; the DB write isn't under test."""
+ return AsyncMock()
+
+
+@pytest_asyncio.fixture
+async def worker_b(trial: TrialState, fake_stripe: FakeStripe, save_tier: AsyncMock):
+ user = MagicMock(stripe_customer_id=trial.customer_id)
+ lookup_user = AsyncMock(return_value=user)
+ worker = _load_worker()
+
+ @asynccontextmanager
+ async def lock(user_id: str):
+ yield
+
+ with (
+ patch.object(stripe.Subscription, "list_async", fake_stripe.list_async),
+ patch.object(stripe.Subscription, "retrieve_async", fake_stripe.retrieve_async),
+ patch.object(stripe.Subscription, "modify_async", fake_stripe.modify_async),
+ patch.object(worker_a, "get_user_by_id", lookup_user),
+ patch.object(worker, "get_user_by_id", lookup_user),
+ patch.object(worker_a, "_sync_subscription_tier_from_stripe", save_tier),
+ patch.object(worker_a, "schedule_posthog_lifecycle_sync", MagicMock()),
+ patch.object(conversion, "subscription_checkout_lock", lock),
+ patch.object(
+ conversion,
+ "subscription_card_can_be_charged",
+ AsyncMock(return_value=True),
+ ),
+ patch.object(conversion, "expire_other_subscription_checkouts", AsyncMock()),
+ patch.object(conversion, "other_plan_is_live", AsyncMock(return_value=False)),
+ ):
+ yield worker
+
+
+@pytest_asyncio.fixture
+async def redis_up():
+ """These tests prove what the cache does, so a Redis the cache can't use
+ must fail them, not let every lookup fall through to Stripe and pass."""
+ redis = await asyncio.wait_for(get_redis_async(), WAIT)
+ assert await asyncio.wait_for(redis.ping(), WAIT)
+ return redis
+
+
+async def _cached_period_end(redis, customer_id: str) -> int | None:
+ raw = await asyncio.wait_for(redis.get(cache_a._lookup_key(customer_id)), WAIT)
+ assert raw, "the lookup was not written to Redis"
+ entry = cache_a._CachedLookup.model_validate_json(raw)
+ assert entry.subscription is not None
+ return entry.subscription.current_period_end
+
+
+async def test_one_workers_lookup_serves_every_worker(
+ trial, fake_stripe, worker_b, redis_up
+):
+ assert await worker_b.get_active_subscription_period_end("user-1") == TRIAL_END
+ assert fake_stripe.lookups == 1
+ assert await _cached_period_end(redis_up, trial.customer_id) == TRIAL_END
+
+ assert await worker_b.get_active_subscription_period_end("user-1") == TRIAL_END
+ assert await worker_a.get_active_subscription_period_end("user-1") == TRIAL_END
+ assert fake_stripe.lookups == 1
+
+
+async def test_a_plan_bought_on_one_worker_shows_its_renewal_on_another(
+ trial, fake_stripe, worker_b, save_tier, redis_up
+):
+ """Trial → cancel → "Subscribe to Pro": the status read that follows the
+ purchase lands on a worker that read the trial moments before."""
+ assert await worker_b.get_active_subscription_period_end("user-1") == TRIAL_END
+
+ await conversion.convert_cancel_pending_trial(trial, SubscriptionTier.PRO)
+ save_tier.assert_awaited_once()
+
+ assert (
+ await worker_b.get_active_subscription_period_end("user-1") == PAID_PERIOD_END
+ )
+ assert fake_stripe.lookups == 2
+ assert await _cached_period_end(redis_up, trial.customer_id) == PAID_PERIOD_END
+ # The refreshed lookup is shared again.
+ assert (
+ await worker_a.get_active_subscription_period_end("user-1") == PAID_PERIOD_END
+ )
+ assert fake_stripe.lookups == 2
+
+
+async def test_a_webhook_on_one_worker_refreshes_the_lookup_on_another(
+ trial, fake_stripe, worker_b, redis_up
+):
+ """A change made in Stripe (the dashboard, a renewal) reaches us only as a
+ webhook, on whichever worker Stripe's request lands."""
+ assert await worker_b.get_active_subscription_period_end("user-1") == TRIAL_END
+
+ converted = await fake_stripe.modify_async("sub_1", trial_end="now")
+ await worker_a.sync_subscription_from_stripe(dict(converted))
+
+ assert (
+ await worker_b.get_active_subscription_period_end("user-1") == PAID_PERIOD_END
+ )
+ assert fake_stripe.lookups == 2
+
+
+async def test_a_read_in_flight_during_the_purchase_does_not_cache_the_trial(
+ trial, fake_stripe, worker_b, redis_up, monkeypatch
+):
+ """A page load whose Stripe answer predates the purchase, but arrives after
+ it, must neither be served to later reads nor overwrite the fresh entry."""
+ release = fake_stripe.hold_next_lookup(monkeypatch, worker_b)
+ in_flight = asyncio.create_task(
+ worker_b.get_active_subscription_period_end("user-1")
+ )
+ await asyncio.wait_for(fake_stripe.reached.wait(), WAIT)
+
+ await conversion.convert_cancel_pending_trial(trial, SubscriptionTier.PRO)
+ assert (
+ await worker_a.get_active_subscription_period_end("user-1") == PAID_PERIOD_END
+ )
+ release.set()
+ assert await asyncio.wait_for(in_flight, WAIT) == TRIAL_END
+
+ assert await _cached_period_end(redis_up, trial.customer_id) == PAID_PERIOD_END
+ assert (
+ await worker_b.get_active_subscription_period_end("user-1") == PAID_PERIOD_END
+ )
+ assert fake_stripe.lookups == 2
+
+
+async def test_a_lookup_read_before_redis_lost_its_data_is_not_served(
+ trial, fake_stripe, worker_b, redis_up, monkeypatch
+):
+ """Trial read → purchase → Redis restarts empty → the slow trial answer
+ arrives. With no token left, nothing may vouch for that answer."""
+ release = fake_stripe.hold_next_lookup(monkeypatch, worker_b)
+ in_flight = asyncio.create_task(
+ worker_b.get_active_subscription_period_end("user-1")
+ )
+ await asyncio.wait_for(fake_stripe.reached.wait(), WAIT)
+ await conversion.convert_cancel_pending_trial(trial, SubscriptionTier.PRO)
+ for key in (
+ cache_a._version_key(trial.customer_id),
+ cache_a._lookup_key(trial.customer_id),
+ ):
+ await asyncio.wait_for(redis_up.delete(key), WAIT)
+
+ release.set()
+ assert await asyncio.wait_for(in_flight, WAIT) == TRIAL_END
+ assert not await asyncio.wait_for(
+ redis_up.get(cache_a._lookup_key(trial.customer_id)), WAIT
+ )
+
+ assert (
+ await worker_a.get_active_subscription_period_end("user-1") == PAID_PERIOD_END
+ )
+ assert (
+ await worker_b.get_active_subscription_period_end("user-1") == PAID_PERIOD_END
+ )
+ assert fake_stripe.lookups == 2
+
+
+async def test_a_read_after_redis_lost_its_data_does_not_join_an_older_lookup(
+ trial, fake_stripe, worker_b, redis_up, monkeypatch
+):
+ """A lookup started before a purchase on another worker, then Redis loses
+ every key: a read on this worker after that asks Stripe itself instead of
+ waiting for the old lookup's answer."""
+ release = fake_stripe.hold_next_lookup(monkeypatch, worker_b)
+ in_flight = asyncio.create_task(
+ worker_b.get_active_subscription_period_end("user-1")
+ )
+ await asyncio.wait_for(fake_stripe.reached.wait(), WAIT)
+ await conversion.convert_cancel_pending_trial(trial, SubscriptionTier.PRO)
+ for key in (
+ cache_a._version_key(trial.customer_id),
+ cache_a._lookup_key(trial.customer_id),
+ ):
+ await asyncio.wait_for(redis_up.delete(key), WAIT)
+
+ try:
+ after = await asyncio.wait_for(
+ worker_b.get_active_subscription_period_end("user-1"), 2
+ )
+ finally:
+ release.set()
+ assert after == PAID_PERIOD_END
+ assert await asyncio.wait_for(in_flight, WAIT) == TRIAL_END
+
+
+async def test_concurrent_misses_on_a_worker_share_one_stripe_call(
+ fake_stripe, worker_b, redis_up
+):
+ ends = await asyncio.wait_for(
+ asyncio.gather(
+ *(worker_b.get_active_subscription_period_end("user-1") for _ in range(60))
+ ),
+ WAIT,
+ )
+ assert set(ends) == {TRIAL_END}
+ assert fake_stripe.lookups == 1
+
+
+async def test_the_tier_is_saved_before_the_lookup_is_dropped(
+ fake_stripe, worker_b, save_tier
+):
+ """Saving the tier can change Stripe too (a rejected trial is canceled), so
+ the lookup is dropped only afterwards, and even when saving fails."""
+ calls: list[str] = []
+
+ async def failing_save(*args) -> None:
+ calls.append("save")
+ raise RuntimeError("the save failed")
+
+ async def invalidate(customer_id: str) -> None:
+ calls.append("invalidate")
+
+ save_tier.side_effect = failing_save
+ with patch.object(worker_a, "invalidate_active_subscription_cache", invalidate):
+ with pytest.raises(RuntimeError):
+ await worker_a.sync_subscription_from_stripe(fake_stripe.subscription)
+ assert calls == ["save", "invalidate"]
+
+
+class _HangingRedis:
+ async def mget(self, *args, **kwargs):
+ await asyncio.Event().wait()
+
+ async def set(self, *args, **kwargs):
+ await asyncio.Event().wait()
+
+
+async def _never_connects():
+ await asyncio.Event().wait()
+
+
+ "outage",
+ [
+ pytest.param(
+ lambda: AsyncMock(side_effect=_never_connects), id="connect-hangs"
+ ),
+ pytest.param(
+ lambda: AsyncMock(return_value=_HangingRedis()), id="command-hangs"
+ ),
+ pytest.param(
+ lambda: AsyncMock(side_effect=RedisConnectionError("refused")),
+ id="refused",
+ ),
+ pytest.param(
+ lambda: AsyncMock(
+ side_effect=RedisClusterException(
+ "Cluster mode is not enabled on this node"
+ )
+ ),
+ id="standalone-redis",
+ ),
+ ],
+)
+async def test_a_redis_outage_never_slows_or_blocks_billing(
+ trial, fake_stripe, worker_b, save_tier, outage
+):
+ """With Redis hanging or unusable, lookups answer from Stripe within the
+ cache's deadline, and a charge still saves the paid plan."""
+ with patch.object(cache_a, "get_redis_async", outage()):
+ started = time.monotonic()
+ ends = await asyncio.wait_for(
+ asyncio.gather(
+ *(
+ worker_a.get_active_subscription_period_end("user-1")
+ for _ in range(60)
+ )
+ ),
+ WAIT,
+ )
+ for _ in range(100):
+ ends.append(await worker_a.get_active_subscription_period_end("user-1"))
+ assert set(ends) == {TRIAL_END}
+ assert time.monotonic() - started < 1.5
+ # Stripe traffic stays bounded while Redis can't be used.
+ assert fake_stripe.lookups == 1
+
+ started = time.monotonic()
+ await asyncio.wait_for(
+ conversion.convert_cancel_pending_trial(trial, SubscriptionTier.PRO),
+ WAIT,
+ )
+ assert time.monotonic() - started < 2
+ save_tier.assert_awaited_once()
+ assert save_tier.await_args.args[0]["status"] == "active"
+
+ # The change this process made is not hidden by what it kept.
+ assert (
+ await worker_a.get_active_subscription_period_end("user-1")
+ == PAID_PERIOD_END
+ )
+
+
+async def test_an_answer_read_before_a_change_is_not_kept_during_an_outage(
+ trial, fake_stripe, worker_b, save_tier, monkeypatch
+):
+ """Redis is out: a lookup started before this worker's purchase answers
+ after it. Its callers get the old answer, but this worker must not keep
+ it and serve it for the next few seconds."""
+ monkeypatch.setattr(
+ cache_a,
+ "get_redis_async",
+ AsyncMock(side_effect=RedisConnectionError("refused")),
+ )
+ release = fake_stripe.hold_next_lookup(monkeypatch, worker_a)
+ in_flight = asyncio.create_task(
+ worker_a.get_active_subscription_period_end("user-1")
+ )
+ await asyncio.wait_for(fake_stripe.reached.wait(), WAIT)
+
+ await conversion.convert_cancel_pending_trial(trial, SubscriptionTier.PRO)
+ assert (
+ await worker_a.get_active_subscription_period_end("user-1") == PAID_PERIOD_END
+ )
+ release.set()
+ assert await asyncio.wait_for(in_flight, WAIT) == TRIAL_END
+
+ assert (
+ await worker_a.get_active_subscription_period_end("user-1") == PAID_PERIOD_END
+ )
+ assert fake_stripe.lookups == 2
+
+
+class _BlippingRedis:
+ """The real Redis, except that the next script call can be held on its way
+ and SETs can be made to fail."""
+
+ def __init__(self, redis):
+ self.redis = redis
+ self.fail_sets = False
+ self.release_eval: asyncio.Event | None = None
+ self.eval_reached = asyncio.Event()
+
+ async def mget(self, *args, **kwargs):
+ return await self.redis.mget(*args, **kwargs)
+
+ async def get(self, *args, **kwargs):
+ return await self.redis.get(*args, **kwargs)
+
+ async def set(self, *args, **kwargs):
+ if self.fail_sets:
+ raise RedisConnectionError("blip")
+ return await self.redis.set(*args, **kwargs)
+
+ async def eval(self, *args):
+ if self.release_eval is not None:
+ release, self.release_eval = self.release_eval, None
+ self.eval_reached.set()
+ await asyncio.wait_for(release.wait(), WAIT)
+ return await self.redis.eval(*args)
+
+
+async def test_a_write_racing_an_unrecorded_invalidation_is_not_served_later(
+ trial, fake_stripe, worker_b, redis_up, monkeypatch
+):
+ """A lookup's write is on its way to Redis when this worker changes the
+ subscription, and Redis blips so the new token isn't stored: the write
+ lands under the old token. Once Redis is back, this worker must not
+ serve it."""
+ monkeypatch.setattr(cache_a, "_REDIS_DEADLINE_SECONDS", WAIT)
+ redis = _BlippingRedis(redis_up)
+ monkeypatch.setattr(cache_a, "get_redis_async", AsyncMock(return_value=redis))
+ redis.release_eval = release = asyncio.Event()
+ first = asyncio.create_task(worker_a.get_active_subscription_period_end("user-1"))
+ await asyncio.wait_for(redis.eval_reached.wait(), WAIT)
+
+ await fake_stripe.modify_async("sub_1", trial_end="now")
+ redis.fail_sets = True
+ await cache_a.invalidate_active_subscription_cache(trial.customer_id)
+ redis.fail_sets = False
+ assert (
+ await worker_a.get_active_subscription_period_end("user-1") == PAID_PERIOD_END
+ )
+
+ release.set()
+ assert await asyncio.wait_for(first, WAIT) == TRIAL_END
+ # The old answer did land in Redis under the unchanged token.
+ assert await _cached_period_end(redis_up, trial.customer_id) == TRIAL_END
+
+ monkeypatch.setattr(cache_a._backoff, "until", 0.0)
+ assert (
+ await worker_a.get_active_subscription_period_end("user-1") == PAID_PERIOD_END
+ )
+
+
+class _Clock:
+ def __init__(self) -> None:
+ self.now = 1000.0
+
+ def monotonic(self) -> float:
+ return self.now
+
+
+def redis_down(monkeypatch):
+ monkeypatch.setattr(
+ cache_a,
+ "get_redis_async",
+ AsyncMock(side_effect=RedisConnectionError("down")),
+ )
+
+
+async def test_a_live_mark_survives_any_number_of_failed_invalidations(redis_down):
+ """A mark is what stops this worker serving an entry written under the
+ old token; other customers' failures must never push it out."""
+ await cache_a.invalidate_active_subscription_cache("cus_changed")
+ for i in range(2100):
+ await cache_a.invalidate_active_subscription_cache(f"cus_other_{i}")
+ assert cache_a._is_unconfirmed("cus_changed")
+
+
+async def test_a_renewed_mark_outlives_expired_ones(redis_down, monkeypatch):
+ """Expired marks must not crowd out a live one, whatever their order."""
+ clock = _Clock()
+ monkeypatch.setattr(cache_a, "time", clock)
+ monkeypatch.setattr(cache_a, "_UNCONFIRMED_MAXSIZE", 4, raising=False)
+ monkeypatch.setattr(cache_a, "_FALLBACK_MAXSIZE", 4)
+ for customer in ("cus_changed", "cus_1", "cus_2", "cus_3"):
+ await cache_a.invalidate_active_subscription_cache(customer)
+ clock.now += 60
+
+ await cache_a.invalidate_active_subscription_cache("cus_changed")
+ await cache_a.invalidate_active_subscription_cache("cus_4")
+ assert cache_a._is_unconfirmed("cus_changed")
diff --git a/autogpt_platform/backend/backend/data/subscription_trial_conversion.py b/autogpt_platform/backend/backend/data/subscription_trial_conversion.py
index 0c13d29255..162367fd14 100644
--- a/autogpt_platform/backend/backend/data/subscription_trial_conversion.py
+++ b/autogpt_platform/backend/backend/data/subscription_trial_conversion.py
@@ -11,10 +11,7 @@ import stripe
from prisma.enums import SubscriptionTier
from pydantic import BaseModel
-from backend.data.credit import (
- invalidate_active_subscription_cache,
- sync_subscription_from_stripe,
-)
+from backend.data.credit import sync_subscription_from_stripe
from backend.data.stripe_authentication import requires_authentication
from backend.data.stripe_client import stripe_call
from backend.data.subscription_checkout import (
@@ -24,6 +21,7 @@ from backend.data.subscription_checkout import (
other_plan_is_live,
subscription_checkout_lock,
)
+from backend.data.subscription_lookup_cache import invalidate_active_subscription_cache
from backend.data.subscription_trial import TrialState, get_subscription_trial
from backend.data.subscription_trial_payment import (
CONVERSION_PRICE_KEY,
@@ -189,18 +187,22 @@ async def _confirm_with_bank(
async def _save_conversion(trial: TrialState, converted: stripe.Subscription) -> None:
- invalidate_active_subscription_cache(trial.customer_id)
- if converted.get("status") != "active":
- await sync_subscription_from_stripe(dict(converted))
- raise TrialConversionRefused(TRIAL_ENDED)
+ # The tier is saved first; dropping the billing page's cached lookup is
+ # bounded, never raises, and comes after so it can't delay the plan.
try:
- await sync_subscription_from_stripe(dict(converted))
- except Exception:
- # The card is charged and the plan is live in Stripe; its webhooks
- # save it. Failing now would tell a paying customer it failed.
- logger.exception(
- f"Trial {trial.id} converted in Stripe but the follow-up sync failed"
- )
+ if converted.get("status") != "active":
+ await sync_subscription_from_stripe(dict(converted))
+ raise TrialConversionRefused(TRIAL_ENDED)
+ try:
+ await sync_subscription_from_stripe(dict(converted))
+ except Exception:
+ # The card is charged and the plan is live in Stripe; its webhooks
+ # save it. Failing now would tell a paying customer it failed.
+ logger.exception(
+ f"Trial {trial.id} converted in Stripe but the follow-up sync failed"
+ )
+ finally:
+ await invalidate_active_subscription_cache(trial.customer_id)
def _is_cancel_pending(trial: TrialState) -> bool:
diff --git a/autogpt_platform/backend/backend/data/subscription_trial_conversion_test.py b/autogpt_platform/backend/backend/data/subscription_trial_conversion_test.py
index c6b9e2e6cc..50d00cdb9a 100644
--- a/autogpt_platform/backend/backend/data/subscription_trial_conversion_test.py
+++ b/autogpt_platform/backend/backend/data/subscription_trial_conversion_test.py
@@ -491,18 +491,9 @@ async def test_ended_plans_do_not_block_the_conversion(pending_trial, live, boun
boundaries.sync.assert_awaited_once_with(dict(boundaries.converted))
-def subscription_lookup():
- """The short-lived active-subscription lookup the status route reads."""
- lookup = credit._get_active_subscription_cached
- lookup.cache_delete("cus_1")
- yield lookup
- lookup.cache_delete("cus_1")
-
-
@pytest.mark.asyncio
async def test_conversion_refreshes_the_cached_subscription_lookup(
- pending_trial, live, boundaries, subscription_lookup
+ pending_trial, live, boundaries
):
"""The status returned right after shows the paid period, not the trial's."""
trialing = stripe.Subscription.construct_from(live, "test-key")
@@ -511,9 +502,35 @@ async def test_conversion_refreshes_the_cached_subscription_lookup(
"_get_active_subscription",
AsyncMock(side_effect=[trialing, boundaries.converted]),
):
- assert await subscription_lookup("cus_1") is trialing
+ before = await credit._get_active_subscription_cached("cus_1")
+ assert before is not None and before.status == "trialing"
await conversion.convert_cancel_pending_trial(pending_trial, PRO)
- assert await subscription_lookup("cus_1") is boundaries.converted
+ after = await credit._get_active_subscription_cached("cus_1")
+ assert after is not None and after.status == "active"
+
+
[email protected]("sync_fails", [False, True])
+async def test_the_paid_plan_is_saved_before_the_lookup_is_dropped(
+ pending_trial, boundaries, sync_fails
+):
+ """The charge has gone through: saving the plan comes first, and dropping
+ the cached lookup follows even when the save fails."""
+
+ async def failing_sync(*args, **kwargs):
+ boundaries.calls.append("sync")
+ raise RuntimeError("database down")
+
+ if sync_fails:
+ boundaries.sync.side_effect = failing_sync
+
+ async def invalidate(customer_id: str) -> None:
+ boundaries.calls.append(f"invalidate:{customer_id}")
+
+ with patch.object(conversion, "invalidate_active_subscription_cache", invalidate):
+ assert await conversion.convert_cancel_pending_trial(pending_trial, PRO) is None
+
+ assert boundaries.calls[-4:] == ["modify", "sync", "invalidate:cus_1", "unlock"]
@pytest.mark.asyncio