Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend): stop a slow read hiding a pending subscription change (#15427)

Nicholas Tindle76383e6b948a
diff --git a/autogpt_platform/backend/backend/conftest.py b/autogpt_platform/backend/backend/conftest.py
index 3e94b15180..fcfe17caec 100644
--- a/autogpt_platform/backend/backend/conftest.py
+++ b/autogpt_platform/backend/backend/conftest.py
@@ -13,7 +13,10 @@ import pytest
import pytest_asyncio
from dotenv import load_dotenv
-from backend.data import subscription_lookup_cache
+from backend.data import versioned_cache
+from backend.data.subscription_lookup_cache import ACTIVE_SUBSCRIPTIONS
+from backend.data.subscription_lookup_unconfirmed import UnconfirmedInvalidations
+from backend.data.subscription_pending_cache import PENDING_CHANGES
from backend.util.logging import configure_logging
load_dotenv()
@@ -133,21 +136,21 @@ def no_force_all_flags(monkeypatch: pytest.MonkeyPatch):
@pytest.fixture(autouse=True)
-def isolated_subscription_lookup_cache(monkeypatch: pytest.MonkeyPatch):
- """The subscription lookup cache lives in Redis, which outlives a test and
- is shared by parallel runs; give every test keys of its own, and a cache
- that hasn't backed off after an earlier test's Redis failure."""
- monkeypatch.setattr(
- subscription_lookup_cache, "KEY_PREFIX", f"test:{uuid4().hex}:subscription"
- )
- monkeypatch.setattr(subscription_lookup_cache._backoff, "until", 0.0)
- monkeypatch.setattr(subscription_lookup_cache, "_fallback", {})
- monkeypatch.setattr(subscription_lookup_cache, "_in_flight", {})
- monkeypatch.setattr(
- subscription_lookup_cache,
- "_unconfirmed",
- subscription_lookup_cache._UnconfirmedInvalidations(),
- )
+def isolated_versioned_caches(monkeypatch: pytest.MonkeyPatch):
+ """Versioned caches live in Redis, which outlives a test and is shared by
+ parallel runs: give every test keys of its own, empty process-local state,
+ and a Redis that hasn't backed off after an earlier test's failure."""
+ run = uuid4().hex
+ monkeypatch.setattr(versioned_cache._backoff, "until", 0.0)
+ for cache in (ACTIVE_SUBSCRIPTIONS, PENDING_CHANGES):
+ monkeypatch.setattr(cache, "prefix", f"test:{run}:{cache.prefix}")
+ monkeypatch.setattr(cache, "_in_flight", {})
+ monkeypatch.setattr(cache, "_fallback", {})
+ monkeypatch.setattr(
+ cache,
+ "_unconfirmed",
+ UnconfirmedInvalidations(lifetime=cache.ttl_seconds + 1),
+ )
@pytest.fixture
diff --git a/autogpt_platform/backend/backend/copilot/rate_limit.py b/autogpt_platform/backend/backend/copilot/rate_limit.py
index 67a883593e..c775b94927 100644
--- a/autogpt_platform/backend/backend/copilot/rate_limit.py
+++ b/autogpt_platform/backend/backend/copilot/rate_limit.py
@@ -1087,11 +1087,7 @@ async def set_user_tier(user_id: str, tier: SubscriptionTier) -> None:
data={"subscriptionTier": tier.value},
)
get_user_tier.cache_delete(user_id) # type: ignore[attr-defined]
- # Local import: backend.data.credit imports from this module.
- from backend.data.credit import get_pending_subscription_change
-
get_user_by_id.cache_delete(user_id) # type: ignore[attr-defined]
- get_pending_subscription_change.cache_delete(user_id) # type: ignore[attr-defined]
schedule_posthog_lifecycle_sync(user_id)
# Fire-and-forget drift check so admin bulk ops don't wait on Stripe.
diff --git a/autogpt_platform/backend/backend/data/credit.py b/autogpt_platform/backend/backend/data/credit.py
index e591968e89..42e3bf3f57 100644
--- a/autogpt_platform/backend/backend/data/credit.py
+++ b/autogpt_platform/backend/backend/data/credit.py
@@ -47,6 +47,11 @@ from backend.data.subscription_lookup_cache import (
invalidate_active_subscription_cache,
read_active_subscription,
)
+from backend.data.subscription_pending_cache import (
+ PENDING_CHANGES,
+ PendingChange,
+ invalidate_pending_subscription_change,
+)
from backend.data.subscription_trial_stripe import reconcile_trial_subscription
from backend.data.user import get_user_by_id, get_user_email_by_id
from backend.notifications.queue import queue_notification_async
@@ -1585,6 +1590,18 @@ async def set_subscription_tier(
schedule_posthog_lifecycle_sync(user_id)
+async def invalidate_billing_lookups(customer_id: str) -> None:
+ """Drop the customer's cached billing lookups on every worker: the active
+ subscription and any pending change. Every path that changes the
+ customer's subscription in Stripe calls this after the change; it never
+ raises. Both at once, so a Redis outage costs the caller one deadline,
+ not two in a row."""
+ await asyncio.gather(
+ invalidate_active_subscription_cache(customer_id),
+ invalidate_pending_subscription_change(customer_id),
+ )
+
+
def invalidate_subscription_caches(user_id: str) -> None:
get_user_by_id.cache_delete(user_id)
# Also invalidate the rate-limit tier cache so CoPilot picks up the new
@@ -1592,12 +1609,9 @@ def invalidate_subscription_caches(user_id: str) -> None:
from backend.copilot.rate_limit import get_user_tier # local import avoids circular
get_user_tier.cache_delete(user_id) # type: ignore[attr-defined]
- # Invalidate the pending-change cache too — an admin tier override or the
- # webhook-driven phase transition means any cached pending-change state
- # (schedule, cancel_at_period_end) is likely stale. Without this the
- # billing page can show a pending change for up to 30s after the tier
- # has already flipped.
- get_pending_subscription_change.cache_delete(user_id)
+ # The billing lookups need nothing here: they are read from the Stripe
+ # subscription alone, and every path that changes it in Stripe
+ # (including the payment-failure cancel) calls invalidate_billing_lookups.
# Stripe stamps ``cancellation_details.reason = "cancellation_requested"`` on
@@ -1640,7 +1654,7 @@ async def _cancel_customer_subscriptions(
)
finally:
if seen_ids:
- await invalidate_active_subscription_cache(customer_id)
+ await invalidate_billing_lookups(customer_id)
return len(seen_ids)
@@ -1743,7 +1757,6 @@ async def cancel_stripe_subscription(user_id: str) -> bool:
customer_id, at_period_end=True
)
if cancelled_count > 0:
- get_pending_subscription_change.cache_delete(user_id)
current_tier = user.subscription_tier or SubscriptionTier.NO_TIER
_track_billing_event(
PostHogEvent.SUBSCRIPTION_CANCELLATION_SCHEDULED,
@@ -1828,7 +1841,7 @@ def is_tier_downgrade(current: SubscriptionTier, target: SubscriptionTier) -> bo
class PendingChangeUnknown(Exception):
"""Raised when pending-change state cannot be determined (e.g. LaunchDarkly
- price-id lookup failed). Propagates past the @cached wrapper so the next
+ price-id lookup failed). Propagates past the pending-change cache so the next
request retries instead of serving a stale `None` for the TTL window."""
@@ -2259,8 +2272,7 @@ async def modify_stripe_subscription_for_tier(
)
return True
finally:
- get_pending_subscription_change.cache_delete(user_id)
- await invalidate_active_subscription_cache(user.stripe_customer_id)
+ await invalidate_billing_lookups(user.stripe_customer_id)
async def release_pending_subscription_schedule(user_id: str) -> bool:
@@ -2329,12 +2341,10 @@ async def release_pending_subscription_schedule(user_id: str) -> bool:
)
finally:
if did_anything:
- get_pending_subscription_change.cache_delete(user_id)
- await invalidate_active_subscription_cache(user.stripe_customer_id)
+ await invalidate_billing_lookups(user.stripe_customer_id)
return did_anything
-@cached(ttl_seconds=30, maxsize=512, cache_none=True, shared_cache=True)
async def get_pending_subscription_change(
user_id: str,
) -> tuple[SubscriptionTier, datetime, BillingCycle | None] | None:
@@ -2343,34 +2353,55 @@ async def get_pending_subscription_change(
Reflects both Subscription Schedule phase transitions (paid→paid downgrade)
and ``cancel_at_period_end=True`` (paid→BASIC cancel).
- Cached for 30 seconds per user_id. *Why the cache exists:* this function
+ Cached for 30 seconds per Stripe customer, for every API worker
+ (``PENDING_CHANGES``, a versioned cache). *Why the cache exists:* this function
runs on every dashboard/home fetch and would otherwise fire
2× Subscription.list + 1× Schedule.retrieve per page load. A busy user
polling the billing page would quickly brush up against Stripe's per-API
rate limits; the 30s TTL absorbs dashboard polling while being short
enough that the UI reconciles quickly after a downgrade / cancel action.
- *Invalidation contract.* Every call-site that mutates Stripe state which
- could change the pending-change answer MUST call
- ``get_pending_subscription_change.cache_delete(user_id)`` so the UI never
- shows a stale pending badge after a user-visible action. Current
- invalidators (keep this list in sync when adding new mutators):
-
- - ``set_subscription_tier`` — admin or webhook-driven tier flip.
+ *Invalidation contract.* The answer is read from the customer's Stripe
+ subscription alone (``cancel_at_period_end`` and its schedule), never from
+ the DB tier. Every call-site that changes those in Stripe MUST await
+ ``invalidate_billing_lookups(customer_id)`` after the change,
+ so the UI never shows a stale pending badge after a user-visible action;
+ a read already in flight then can't store its old answer. Current
+ invalidators (keep this list in sync when adding new mutators), each
+ through ``invalidate_billing_lookups``, which drops the active-subscription
+ lookup too:
+
+ - ``sync_subscription_from_stripe`` — ``finally`` block: every
+ subscription and schedule webhook, Stripe-side changes (billing portal,
+ dashboard) and trial transitions, whether or not the tier changed.
+ - ``_cancel_customer_subscriptions`` — ``finally`` block: user cancels,
+ stale-subscription cleanup, and the payment-failure cancel.
- ``modify_stripe_subscription_for_tier`` — ``finally`` block (covers
upgrade path clear + downgrade-schedule create + any partial failure).
- ``release_pending_subscription_schedule`` — ``finally`` block when a
schedule release OR ``cancel_at_period_end`` clear succeeded.
- - ``cancel_stripe_subscription`` — after scheduling period-end cancel.
- - ``sync_subscription_from_stripe`` — webhook entry point.
- - ``set_user_tier`` (``backend.copilot.rate_limit``) — admin tier override
- invalidates any cached pending state keyed off the old tier.
+ - ``handle_subscription_payment_failure`` — after paying a past-due
+ invoice from the balance; ``handle_subscription_payment_success``.
+ - The trial conversion (``_save_conversion``), after saving the plan.
+
+ ``PendingChangeUnknown`` and Stripe errors reach every caller and are not
+ cached, so the next request retries.
"""
user = await get_user_by_id(user_id)
- if not user.stripe_customer_id:
+ customer_id = user.stripe_customer_id
+ if not customer_id:
# Short-circuit for users with no Stripe customer (admin-granted tiers,
# BASIC-only users): skip the Stripe API calls entirely.
return None
+ pending = await PENDING_CHANGES.read(
+ customer_id, lambda: _fetch_pending_subscription_change(customer_id)
+ )
+ if pending is None:
+ return None
+ return pending.tier, pending.effective_at, pending.billing_cycle
+
+
+async def _fetch_pending_subscription_change(customer_id: str) -> PendingChange | None:
priceable = (
SubscriptionTier.BASIC,
@@ -2404,7 +2435,7 @@ async def get_pending_subscription_change(
"Stripe price lookup failed; pending-change state cannot be determined"
)
- sub = await _get_active_subscription(user.stripe_customer_id)
+ sub = await _get_active_subscription(customer_id)
if sub is None:
return None
if sub.get("status") == "trialing" and (sub.get("metadata") or {}).get(
@@ -2417,14 +2448,18 @@ async def get_pending_subscription_change(
return None
effective_at = datetime.fromtimestamp(period_end, tz=timezone.utc)
if sub.cancel_at_period_end:
- return SubscriptionTier.NO_TIER, effective_at, None
+ return PendingChange(tier=SubscriptionTier.NO_TIER, effective_at=effective_at)
if not sub.schedule:
return None
schedule_id = sub.schedule if isinstance(sub.schedule, str) else sub.schedule.id
schedule = await stripe_call(
stripe.SubscriptionSchedule.retrieve_async, schedule_id
)
- return _next_phase_tier_and_start(schedule, price_to_tier, price_to_cycle)
+ next_phase = _next_phase_tier_and_start(schedule, price_to_tier, price_to_cycle)
+ if next_phase is None:
+ return None
+ tier, starts_at, cycle = next_phase
+ return PendingChange(tier=tier, effective_at=starts_at, billing_cycle=cycle)
def _next_phase_tier_and_start(
@@ -2794,16 +2829,18 @@ async def sync_subscription_from_stripe(
covers those users.
The payload is Stripe's latest word on the subscription, so afterwards
- the cached lookup the billing page reads is dropped on every worker. Only
- afterwards: saving the tier can change Stripe too (a rejected trial is
- canceled), and a read in between would cache the state before that.
+ the cached lookups the billing page reads (the active subscription and
+ any pending change) are dropped on every worker, whether or not the tier
+ changed: a cancellation or downgrade scheduled in Stripe changes neither.
+ Only afterwards: saving the tier can change Stripe too (a rejected trial
+ is canceled), and a read in between would cache the state before that.
"""
customer_id = stripe_subscription.get("customer")
try:
await _sync_subscription_tier_from_stripe(stripe_subscription)
finally:
if isinstance(customer_id, str) and customer_id:
- await invalidate_active_subscription_cache(customer_id)
+ await invalidate_billing_lookups(customer_id)
if track_lifecycle:
schedule_posthog_lifecycle_sync(
stripe_customer_id=stripe_subscription.get("customer")
@@ -2992,9 +3029,6 @@ async def _sync_subscription_tier_from_stripe(stripe_subscription: dict) -> None
"billing_cycle": billing_cycle,
},
)
- # Tier changed — bust any cached pending-change view so the next
- # dashboard fetch reflects the new state immediately.
- get_pending_subscription_change.cache_delete(user.id)
async def sync_subscription_schedule_from_stripe(stripe_schedule: dict) -> None:
@@ -3300,6 +3334,10 @@ async def handle_subscription_payment_failure(invoice: dict) -> None:
user.id,
invoice_id,
)
+ finally:
+ # A past-due subscription is active again, also when the call
+ # timed out after Stripe applied it.
+ await invalidate_billing_lookups(customer_id)
logger.info(
"handle_subscription_payment_failure: deducted %d cents from balance"
" for user %s; Stripe invoice %s paid, sub %s intact, tier preserved",
@@ -3389,9 +3427,9 @@ async def handle_subscription_payment_success(invoice: dict) -> None:
return
# A paid invoice can start a new period (a renewal, a plan's first
# charge). Nothing below changes Stripe, so dropping the billing page's
- # cached lookup now is as good as after; it doesn't wait for the
+ # cached lookups now is as good as after; it doesn't wait for the
# customer.subscription.updated webhook that usually follows.
- await invalidate_active_subscription_cache(customer_id)
+ await invalidate_billing_lookups(customer_id)
user = await User.prisma().find_first(where={"stripeCustomerId": customer_id})
if not user:
logger.warning(
diff --git a/autogpt_platform/backend/backend/data/credit_subscription_test.py b/autogpt_platform/backend/backend/data/credit_subscription_test.py
index 0de3c61cf8..4924792b77 100644
--- a/autogpt_platform/backend/backend/data/credit_subscription_test.py
+++ b/autogpt_platform/backend/backend/data/credit_subscription_test.py
@@ -184,7 +184,10 @@ async def test_sync_subscription_from_stripe_tracks_upgrade():
return_value=MagicMock(capture=track_mock),
),
patch("backend.util.posthog_client._environment", return_value="test"),
- patch.object(get_pending_subscription_change, "cache_delete"),
+ patch(
+ "backend.data.credit.invalidate_pending_subscription_change",
+ new_callable=AsyncMock,
+ ),
):
await sync_subscription_from_stripe(stripe_sub)
@@ -442,16 +445,16 @@ async def test_sync_subscription_from_stripe_cancelled_applies_no_tier_storage_l
"ENTERPRISE": 15 * 1024,
},
),
- patch.object(
- get_pending_subscription_change,
- "cache_delete",
+ patch(
+ "backend.data.credit.invalidate_pending_subscription_change",
+ new_callable=AsyncMock,
) as mock_pending_cache_delete,
):
await sync_subscription_from_stripe(stripe_sub)
result = await get_workspace_storage_limit_bytes("user-1")
assert result == 250 * 1024 * 1024
- mock_pending_cache_delete.assert_called_once_with("user-1")
+ mock_pending_cache_delete.assert_awaited_once_with("cus_123")
@pytest.mark.asyncio
@@ -618,7 +621,10 @@ async def test_cancel_stripe_subscription_tracks_cancellation():
return_value=MagicMock(capture=track_mock),
),
patch("backend.util.posthog_client._environment", return_value="test"),
- patch.object(get_pending_subscription_change, "cache_delete"),
+ patch(
+ "backend.data.credit.invalidate_pending_subscription_change",
+ new_callable=AsyncMock,
+ ),
):
result = await cancel_stripe_subscription("user-1")
@@ -1888,9 +1894,15 @@ async def test_handle_subscription_payment_failure_invoice_pay_error_does_not_ra
"backend.data.credit.stripe.Invoice.pay_async",
side_effect=stripe_mod.StripeError("network error"),
),
+ patch(
+ "backend.data.credit.invalidate_billing_lookups", new_callable=AsyncMock
+ ) as invalidate,
):
# Must not raise — the pay failure is only logged as a warning
await handle_subscription_payment_failure(invoice)
+ # A timed-out call may still have been applied in Stripe, so the cached
+ # billing lookups are dropped either way.
+ invalidate.assert_awaited_once_with("cus_123")
@pytest.mark.asyncio
@@ -3143,7 +3155,10 @@ async def test_modify_stripe_subscription_skips_emit_when_db_flip_fails():
return_value=MagicMock(capture=track_mock),
),
patch("backend.util.posthog_client._environment", return_value="test"),
- patch.object(get_pending_subscription_change, "cache_delete"),
+ patch(
+ "backend.data.credit.invalidate_pending_subscription_change",
+ new_callable=AsyncMock,
+ ),
):
result = await modify_stripe_subscription_for_tier(
"user-1", SubscriptionTier.BUSINESS
@@ -3415,8 +3430,6 @@ async def test_get_pending_subscription_change_cancel_at_period_end():
"""cancel_at_period_end=True maps to pending NO_TIER at current_period_end."""
import time as time_mod
- _clear_cache(get_pending_subscription_change)
-
now = int(time_mod.time())
period_end = now + 10 * 24 * 3600
mock_sub = stripe.Subscription.construct_from(
@@ -3473,8 +3486,6 @@ async def test_get_pending_subscription_change_from_schedule():
"""A schedule whose next phase uses the PRO price maps to pending_tier=PRO."""
import time as time_mod
- _clear_cache(get_pending_subscription_change)
-
now = int(time_mod.time())
period_end = now + 10 * 24 * 3600
mock_sub = stripe.Subscription.construct_from(
@@ -3557,8 +3568,6 @@ async def test_get_pending_subscription_change_yearly_next_phase_maps_to_tier():
correct tier — yearly subscribers must see their pending downgrade in UI."""
import time as time_mod
- _clear_cache(get_pending_subscription_change)
-
now = int(time_mod.time())
period_end = now + 10 * 24 * 3600
mock_sub = stripe.Subscription.construct_from(
@@ -3646,8 +3655,6 @@ async def test_get_pending_subscription_change_from_schedule_to_basic():
"""A schedule whose next phase uses the BASIC price maps to pending_tier=BASIC."""
import time as time_mod
- _clear_cache(get_pending_subscription_change)
-
now = int(time_mod.time())
period_end = now + 10 * 24 * 3600
mock_sub = stripe.Subscription.construct_from(
@@ -3731,8 +3738,6 @@ async def test_get_pending_subscription_change_none_when_no_schedule_or_cancel()
"""Returns None when neither a schedule nor cancel_at_period_end is set."""
import time as time_mod
- _clear_cache(get_pending_subscription_change)
-
now = int(time_mod.time())
mock_sub = stripe.Subscription.construct_from(
{
@@ -3785,8 +3790,6 @@ async def test_get_pending_subscription_change_same_tier_yearly_to_monthly_repor
would otherwise look like a no-op or a confusing same-tier "downgrade")."""
import time as time_mod
- _clear_cache(get_pending_subscription_change)
-
now = int(time_mod.time())
period_end = now + 200 * 24 * 3600
mock_sub = stripe.Subscription.construct_from(
@@ -4489,8 +4492,6 @@ async def test_get_pending_subscription_change_raises_when_price_lookups_fail():
@cached wrapper doesn't store None and hide pending changes for 30s."""
from backend.data.credit import PendingChangeUnknown
- _clear_cache(get_pending_subscription_change)
-
mock_user = MagicMock()
mock_user.stripe_customer_id = "cus_abc"
@@ -4519,7 +4520,6 @@ async def test_release_pending_subscription_schedule_invalidates_cache_on_partia
"""If schedule.release succeeds but cancel_at_period_end clear fails, the
cache must still be invalidated — otherwise the UI shows a stale pending
banner for up to 30s even though the schedule was actually released."""
- _clear_cache(get_pending_subscription_change)
mock_user = MagicMock()
mock_user.stripe_customer_id = "cus_abc"
@@ -4559,14 +4559,15 @@ async def test_release_pending_subscription_schedule_invalidates_cache_on_partia
new_callable=AsyncMock,
side_effect=stripe.APIConnectionError("transient Stripe error"),
),
- patch.object(
- get_pending_subscription_change, "cache_delete"
+ patch(
+ "backend.data.credit.invalidate_pending_subscription_change",
+ new_callable=AsyncMock,
) as mock_cache_delete,
):
with pytest.raises(stripe.APIConnectionError):
await release_pending_subscription_schedule("user-partial")
- mock_cache_delete.assert_called_once_with("user-partial")
+ mock_cache_delete.assert_awaited_once_with("cus_abc")
# ─── TOP_UP vs GRANT routing ─────────────────────────────────────────────
diff --git a/autogpt_platform/backend/backend/data/credit_trial_subscription_test.py b/autogpt_platform/backend/backend/data/credit_trial_subscription_test.py
index 665b517ed6..40688c3c80 100644
--- a/autogpt_platform/backend/backend/data/credit_trial_subscription_test.py
+++ b/autogpt_platform/backend/backend/data/credit_trial_subscription_test.py
@@ -14,7 +14,7 @@ from backend.data.credit import (
invalidate_active_subscription_cache,
sync_subscription_from_stripe,
)
-from backend.data.credit_subscription_test import _clear_cache, _make_user
+from backend.data.credit_subscription_test import _make_user
@pytest.mark.asyncio
@@ -96,7 +96,6 @@ async def test_new_plan_ends_cancel_pending_trial_without_invoice():
async def _pending_change_for(subscription: dict):
- _clear_cache(get_pending_subscription_change)
mock_list = MagicMock()
mock_list.data = [stripe.Subscription.construct_from(subscription, "k")]
with (
diff --git a/autogpt_platform/backend/backend/data/subscription_lookup_cache.py b/autogpt_platform/backend/backend/data/subscription_lookup_cache.py
index b9fcfb8c07..0f8e27fb25 100644
--- a/autogpt_platform/backend/backend/data/subscription_lookup_cache.py
+++ b/autogpt_platform/backend/backend/data/subscription_lookup_cache.py
@@ -1,65 +1,20 @@
-"""The active-subscription lookup cache, shared by every API worker.
+"""The active-subscription lookup the billing page reads, cached for every
+API worker.
The billing status endpoint reads a customer's active subscription several
times per request, on almost every page load, so the lookup is cached briefly
-in Redis. Each customer has a random version token; every subscription change
-replaces it, so the next read on any worker misses and asks Stripe again. An
-entry is written only while the token it was read under is still current, so
-a read that was in flight when a change landed can't put the old answer back.
-A missing token (expired, or Redis lost its data) is never treated as a
-version: the reader establishes a fresh one before it asks Stripe.
-
-Within a process, concurrent lookups for one customer and token share one
-Stripe call. A lookup stores its answer only while it is still the current one
-for its key, so an answer read before a change this process made is returned
-to the callers that were already waiting for it, and never kept.
-
-Redis is only an optimisation here. Every operation has a short hard deadline,
-and after a failure this process skips Redis for a few seconds, so an outage
-or a hung connection costs a lookup at most one deadline, and never blocks or
-fails the billing change around it. Meanwhile answers are kept in process for
-a few seconds, to keep Stripe traffic bounded. A Redis that isn't a cluster
-(the platform always runs one) gets this fallback all the time.
+in Redis (see :mod:`backend.data.versioned_cache` for how a change reaches
+every worker on its next read).
"""
-import asyncio
-import logging
-import time
-from typing import Any, Awaitable, Callable, Coroutine, TypeVar
-from uuid import uuid4
+from typing import Awaitable, Callable
import stripe
-from pydantic import BaseModel, ValidationError
-
-from backend.data.redis_client import AsyncRedisClient, get_redis_async
+from pydantic import BaseModel
-logger = logging.getLogger(__name__)
-
-T = TypeVar("T")
-Fetch = Callable[[], Awaitable[stripe.Subscription | None]]
+from backend.data.versioned_cache import VersionedCache
SUBSCRIPTION_LOOKUP_TTL_SECONDS = 15
-# Bounds connecting as well as the command: the shared client retries a
-# failed connection for minutes on its own.
-_REDIS_DEADLINE_SECONDS = 0.3
-_REDIS_BACKOFF_SECONDS = 5.0
-# How long this process reuses an answer while Redis can't be used; also how
-# long another worker can show the old state after a change in that time.
-_FALLBACK_TTL_SECONDS = 5.0
-_FALLBACK_MAXSIZE = 2048
-_UNCONFIRMED_MAXSIZE = 2048
-_VERSION_TTL_SECONDS = 24 * 60 * 60
-KEY_PREFIX = "active_subscription"
-# The in-flight key for reads while Redis can't be used; tokens are hex.
-_NO_REDIS = "no-redis"
-
-# KEYS: version, lookup. ARGV: version read, entry, TTL.
-_WRITE_IF_CURRENT = """
-if redis.call('GET', KEYS[1]) == ARGV[1] then
- return redis.call('SET', KEYS[2], ARGV[2], 'EX', ARGV[3])
-end
-return false
-"""
class ActiveSubscription(BaseModel):
@@ -75,88 +30,12 @@ class ActiveSubscription(BaseModel):
price_id: str | None = None
-class _CachedLookup(BaseModel):
- version: str
- subscription: ActiveSubscription | None
-
-
-class _RedisUnavailable(Exception):
- pass
-
-
-class _Backoff:
- """Process-local: skip Redis for a while after it failed."""
-
- def __init__(self) -> None:
- self.until = 0.0
-
- def active(self) -> bool:
- return time.monotonic() < self.until
-
- def trip(self) -> None:
- if not self.active():
- logger.warning(
- "Subscription lookup cache unavailable; reading Stripe directly"
- f" for {_REDIS_BACKOFF_SECONDS:.0f}s"
- )
- self.until = time.monotonic() + _REDIS_BACKOFF_SECONDS
-
-
-_backoff = _Backoff()
-# Lookups running in this process, by (event loop, customer, version).
-_in_flight: dict[
- tuple[asyncio.AbstractEventLoop, str, str],
- asyncio.Task[ActiveSubscription | None],
-] = {}
-# Answers kept while Redis can't be used: customer -> (expires at, answer).
-_fallback: dict[str, tuple[float, ActiveSubscription | None]] = {}
-
-
-class _UnconfirmedInvalidations:
- """Customers whose invalidation this process couldn't record in Redis.
-
- A write already on its way when the change landed is stored under the
- old, unchanged token, so until any such entry has expired this process
- reads the customer as if Redis were out. A mark is never dropped while it
- is live: with no room left, every customer is covered until the latest
- mark would have expired.
- """
-
- def __init__(self) -> None:
- self.until: dict[str, float] = {}
- self.everyone_until = 0.0
-
- def add(self, customer_id: str) -> None:
- now = time.monotonic()
- # Outlives any entry written now: a write in flight finishes within
- # the Redis deadline, and entries live for the lookup TTL.
- until = now + SUBSCRIPTION_LOOKUP_TTL_SECONDS + 1
- self._purge(now)
- # Re-inserted at the end, so insertion order stays expiry order.
- self.until.pop(customer_id, None)
- if len(self.until) >= _UNCONFIRMED_MAXSIZE:
- self.everyone_until = max(self.everyone_until, until)
- return
- self.until[customer_id] = until
-
- def clear(self, customer_id: str) -> None:
- self.until.pop(customer_id, None)
-
- def covers(self, customer_id: str) -> bool:
- now = time.monotonic()
- if now < self.everyone_until:
- return True
- until = self.until.get(customer_id)
- return until is not None and until > now
-
- def _purge(self, now: float) -> None:
- for customer_id, until in list(self.until.items()):
- if until > now:
- break
- del self.until[customer_id]
-
-
-_unconfirmed = _UnconfirmedInvalidations()
+# Keyed by Stripe customer id.
+ACTIVE_SUBSCRIPTIONS: VersionedCache[ActiveSubscription | None] = VersionedCache(
+ "active_subscription",
+ ActiveSubscription | None,
+ ttl_seconds=SUBSCRIPTION_LOOKUP_TTL_SECONDS,
+)
def summarize_subscription(
@@ -175,214 +54,18 @@ def summarize_subscription(
async def read_active_subscription(
- customer_id: str, fetch: Fetch
-) -> ActiveSubscription | None:
- """The cached lookup, or ``fetch``'s answer when it isn't cached."""
- if _is_unconfirmed(customer_id):
- return await _read_without_redis(customer_id, fetch)
- try:
- version, cached = await _read_entry(customer_id)
- if version is None:
- # Established before the in-flight key is chosen, so a read after
- # Redis lost the token never joins a lookup started before.
- version = await _establish_version(customer_id)
- except _RedisUnavailable:
- return await _read_without_redis(customer_id, fetch)
- if cached is not None and cached.version == version:
- return cached.subscription
- return await _shared(
- customer_id, version, lambda: _fetch_and_store(customer_id, version, fetch)
- )
-
-
-async def invalidate_active_subscription_cache(customer_id: str) -> None:
- """Make every worker read the customer's subscription fresh from Stripe.
-
- Call it after the change has landed in Stripe, never before: a read
- started earlier could otherwise cache the old state under the new token.
- Never raises and returns within the Redis deadline, because the change it
- follows has already been made.
- """
- _forget_locally(customer_id)
- try:
- await _call_redis(
- lambda redis: redis.set(
- _version_key(customer_id), uuid4().hex, ex=_VERSION_TTL_SECONDS
- ),
- skip_during_backoff=False,
- )
- except _RedisUnavailable:
- # The old token still vouches for entries written under it.
- _unconfirmed.add(customer_id)
- logger.warning(
- f"Could not invalidate the subscription lookup for {customer_id};"
- f" other workers can show it stale for {SUBSCRIPTION_LOOKUP_TTL_SECONDS}s"
- )
- else:
- # From now on the script refuses anything read under the old token.
- _unconfirmed.clear(customer_id)
-
-
-async def _read_entry(customer_id: str) -> tuple[str | None, _CachedLookup | None]:
- keys = (_version_key(customer_id), _lookup_key(customer_id))
- version, raw = await _call_redis(lambda redis: redis.mget(keys))
- return _text(version), _parse_lookup(_text(raw))
-
-
-async def _establish_version(customer_id: str) -> str:
- """A fresh token for a customer who has none, or the one another reader
- created first."""
- token = uuid4().hex
- key = _version_key(customer_id)
- created = await _call_redis(
- lambda redis: redis.set(key, token, nx=True, ex=_VERSION_TTL_SECONDS)
- )
- if created:
- return token
- current = _text(await _call_redis(lambda redis: redis.get(key)))
- if current:
- return current
- # Expired again in between: a token of our own only costs a miss.
- await _call_redis(lambda redis: redis.set(key, token, ex=_VERSION_TTL_SECONDS))
- return token
-
-
-async def _fetch_and_store(
- customer_id: str, version: str, fetch: Fetch
-) -> ActiveSubscription | None:
- # Another lookup may have stored it while this one was being started.
- try:
- current, cached = await _read_entry(customer_id)
- except _RedisUnavailable:
- current, cached = None, None
- if current == version and cached is not None and cached.version == version:
- return cached.subscription
- subscription = summarize_subscription(await fetch())
- if not _still_current(customer_id, version) or _is_unconfirmed(customer_id):
- return subscription
- entry = _CachedLookup(version=version, subscription=subscription)
- keys_and_args = (
- _version_key(customer_id),
- _lookup_key(customer_id),
- version,
- entry.model_dump_json(),
- SUBSCRIPTION_LOOKUP_TTL_SECONDS,
- )
- try:
- await _call_redis(
- lambda redis: redis.eval(_WRITE_IF_CURRENT, 2, *keys_and_args)
- )
- except _RedisUnavailable:
- pass
- return subscription
-
-
-async def _read_without_redis(
- customer_id: str, fetch: Fetch
-) -> ActiveSubscription | None:
- kept = _fallback.get(customer_id)
- if kept is not None and kept[0] > time.monotonic():
- return kept[1]
- return await _shared(
- customer_id, _NO_REDIS, lambda: _fetch_and_keep(customer_id, fetch)
- )
-
-
-async def _fetch_and_keep(customer_id: str, fetch: Fetch) -> ActiveSubscription | None:
- subscription = summarize_subscription(await fetch())
- if not _still_current(customer_id, _NO_REDIS):
- return subscription
- _fallback[customer_id] = (time.monotonic() + _FALLBACK_TTL_SECONDS, subscription)
- if len(_fallback) > _FALLBACK_MAXSIZE:
- # Dropping a kept answer only costs another Stripe call.
- del _fallback[next(iter(_fallback))]
- return subscription
-
-
-def _is_unconfirmed(customer_id: str) -> bool:
- return _unconfirmed.covers(customer_id)
-
-
-async def _shared(
customer_id: str,
- version: str,
- lookup: Callable[[], Coroutine[Any, Any, ActiveSubscription | None]],
+ fetch: Callable[[], Awaitable[stripe.Subscription | dict | None]],
) -> ActiveSubscription | None:
- """Run ``lookup`` once for every concurrent caller with the same key. The
- version is part of the key, so a read after a change never joins one
- started before it."""
- key = (asyncio.get_running_loop(), customer_id, version)
- task = _in_flight.get(key)
- if task is None:
- task = asyncio.create_task(lookup())
- _in_flight[key] = task
- task.add_done_callback(lambda done: _drop_in_flight(key, done))
- # One caller giving up must not cancel the lookup the others wait on.
- return await asyncio.shield(task)
-
-
-def _still_current(customer_id: str, version: str) -> bool:
- """Whether the running lookup still owns its in-flight key: invalidating
- in this process drops it, and then its answer must not be kept."""
- key = (asyncio.get_running_loop(), customer_id, version)
- return _in_flight.get(key) is asyncio.current_task()
-
-
-def _drop_in_flight(
- key: tuple[asyncio.AbstractEventLoop, str, str],
- task: asyncio.Task[ActiveSubscription | None],
-) -> None:
- if _in_flight.get(key) is task:
- del _in_flight[key]
-
-
-def _forget_locally(customer_id: str) -> None:
- """Stop this process reusing what it read before a change: its fallback
- answer, and lookups still running (whose version may not have changed if
- Redis couldn't be told)."""
- _fallback.pop(customer_id, None)
- for key in [key for key in _in_flight if key[1] == customer_id]:
- del _in_flight[key]
-
-
-def _version_key(customer_id: str) -> str:
- # The hash tag keeps both keys on one cluster slot, for MGET and the script.
- return f"{KEY_PREFIX}:{{{customer_id}}}:version"
-
-
-def _lookup_key(customer_id: str) -> str:
- return f"{KEY_PREFIX}:{{{customer_id}}}:lookup"
-
-
-def _text(value: str | bytes | None) -> str | None:
- # The shared client decodes responses; its type hints don't say so.
- return value.decode() if isinstance(value, bytes) else value
-
-
-def _parse_lookup(raw: str | None) -> _CachedLookup | None:
- if not raw:
- return None
- try:
- return _CachedLookup.model_validate_json(raw)
- except ValidationError:
- return None
+ """The cached lookup, or ``fetch``'s answer when it isn't cached."""
+ async def lookup() -> ActiveSubscription | None:
+ return summarize_subscription(await fetch())
-async def _call_redis(
- operation: Callable[[AsyncRedisClient], Awaitable[T]],
- *,
- skip_during_backoff: bool = True,
-) -> T:
- if skip_during_backoff and _backoff.active():
- raise _RedisUnavailable()
- try:
- return await asyncio.wait_for(
- _with_client(operation), timeout=_REDIS_DEADLINE_SECONDS
- )
- except Exception as exc:
- _backoff.trip()
- raise _RedisUnavailable() from exc
+ return await ACTIVE_SUBSCRIPTIONS.read(customer_id, lookup)
-async def _with_client(operation: Callable[[AsyncRedisClient], Awaitable[T]]) -> T:
- return await operation(await get_redis_async())
+async def invalidate_active_subscription_cache(customer_id: str) -> None:
+ """Make every worker read the customer's subscription fresh from Stripe.
+ Call it after the change has landed in Stripe; it never raises."""
+ await ACTIVE_SUBSCRIPTIONS.invalidate(customer_id)
diff --git a/autogpt_platform/backend/backend/data/subscription_lookup_cache_test.py b/autogpt_platform/backend/backend/data/subscription_lookup_cache_test.py
index f849dfe5bd..c0b6a52052 100644
--- a/autogpt_platform/backend/backend/data/subscription_lookup_cache_test.py
+++ b/autogpt_platform/backend/backend/data/subscription_lookup_cache_test.py
@@ -9,6 +9,7 @@ one fake Stripe.
import asyncio
import importlib.util
+import json
import time
from contextlib import asynccontextmanager
from datetime import UTC, datetime, timedelta
@@ -26,11 +27,15 @@ from redis.exceptions import RedisClusterException
from backend.data import credit as worker_a
from backend.data import subscription_lookup_cache as cache_a
from backend.data import subscription_trial_conversion as conversion
+from backend.data import versioned_cache
from backend.data.redis_client import get_redis_async
+from backend.data.subscription_lookup_unconfirmed import UnconfirmedInvalidations
from backend.data.subscription_trial import TrialState
pytest_plugins = ("backend.data.subscription_trial_fixtures",)
+ACTIVE = cache_a.ACTIVE_SUBSCRIPTIONS
+
TRIAL_END = int((datetime.now(UTC) + timedelta(days=7)).timestamp())
PAID_PERIOD_END = int((datetime.now(UTC) + timedelta(days=31)).timestamp())
WAIT = 5 # seconds; no wait in these tests may hang the suite
@@ -47,7 +52,7 @@ def _load_copy(name: str) -> ModuleType:
def _load_worker() -> ModuleType:
"""A second, independently loaded credit module with its own cache module."""
cache = _load_copy("backend.data.subscription_lookup_cache")
- cache.KEY_PREFIX = cache_a.KEY_PREFIX
+ cache.ACTIVE_SUBSCRIPTIONS.prefix = ACTIVE.prefix
worker = _load_copy("backend.data.credit")
worker.read_active_subscription = cache.read_active_subscription
worker.invalidate_active_subscription_cache = (
@@ -194,20 +199,25 @@ async def worker_b(trial: TrialState, fake_stripe: FakeStripe, save_tier: AsyncM
@pytest_asyncio.fixture
-async def redis_up():
+async def redis_up(monkeypatch):
"""These tests prove what the cache does, so a Redis the cache can't use
- must fail them, not let every lookup fall through to Stripe and pass."""
+ must fail them, not let every lookup fall through to Stripe and pass.
+
+ They assert what is cached, not how fast: a busy CI runner can take
+ longer than the production deadline for one Redis call, which would
+ rightly send that read to Stripe and miscount the lookups."""
+ monkeypatch.setattr(versioned_cache, "_REDIS_DEADLINE_SECONDS", WAIT)
redis = await asyncio.wait_for(get_redis_async(), WAIT)
assert await asyncio.wait_for(redis.ping(), WAIT)
return redis
async def _cached_period_end(redis, customer_id: str) -> int | None:
- raw = await asyncio.wait_for(redis.get(cache_a._lookup_key(customer_id)), WAIT)
+ raw = await asyncio.wait_for(redis.get(ACTIVE._entry_key(customer_id)), WAIT)
assert raw, "the lookup was not written to Redis"
- entry = cache_a._CachedLookup.model_validate_json(raw)
- assert entry.subscription is not None
- return entry.subscription.current_period_end
+ subscription = json.loads(raw)["value"]
+ assert subscription is not None
+ return subscription["current_period_end"]
@pytest.mark.asyncio
@@ -302,15 +312,15 @@ async def test_a_lookup_read_before_redis_lost_its_data_is_not_served(
await asyncio.wait_for(fake_stripe.reached.wait(), WAIT)
await conversion.convert_cancel_pending_trial(trial, SubscriptionTier.PRO)
for key in (
- cache_a._version_key(trial.customer_id),
- cache_a._lookup_key(trial.customer_id),
+ ACTIVE._version_key(trial.customer_id),
+ ACTIVE._entry_key(trial.customer_id),
):
await asyncio.wait_for(redis_up.delete(key), WAIT)
release.set()
assert await asyncio.wait_for(in_flight, WAIT) == TRIAL_END
assert not await asyncio.wait_for(
- redis_up.get(cache_a._lookup_key(trial.customer_id)), WAIT
+ redis_up.get(ACTIVE._entry_key(trial.customer_id)), WAIT
)
assert (
@@ -336,8 +346,8 @@ async def test_a_read_after_redis_lost_its_data_does_not_join_an_older_lookup(
await asyncio.wait_for(fake_stripe.reached.wait(), WAIT)
await conversion.convert_cancel_pending_trial(trial, SubscriptionTier.PRO)
for key in (
- cache_a._version_key(trial.customer_id),
- cache_a._lookup_key(trial.customer_id),
+ ACTIVE._version_key(trial.customer_id),
+ ACTIVE._entry_key(trial.customer_id),
):
await asyncio.wait_for(redis_up.delete(key), WAIT)
@@ -387,48 +397,68 @@ async def test_the_tier_is_saved_before_the_lookup_is_dropped(
assert calls == ["save", "invalidate"]
+class _Outage:
+ """Stands in for ``get_redis_async``, which every Redis operation calls
+ first, so ``attempts`` counts each time the cache tries Redis.
+
+ ``serial_waits`` counts the deadline waits a caller pays one after the
+ other: a hanging attempt that starts while no other is hanging opens a
+ new wait; attempts made alongside it share that one."""
+
+ def __init__(self, kind: str) -> None:
+ self.kind = kind
+ self.attempts = 0
+ self.serial_waits = 0
+ self._hanging = 0
+
+ async def __call__(self) -> "_HangingRedis":
+ self.attempts += 1
+ if self.kind == "connect-hangs":
+ await self.hang()
+ if self.kind == "refused":
+ raise RedisConnectionError("refused")
+ if self.kind == "standalone-redis":
+ raise RedisClusterException("Cluster mode is not enabled on this node")
+ return _HangingRedis(self)
+
+ async def hang(self) -> None:
+ if self._hanging == 0:
+ self.serial_waits += 1
+ self._hanging += 1
+ try:
+ await asyncio.Event().wait()
+ finally:
+ self._hanging -= 1
+
+
class _HangingRedis:
+ def __init__(self, outage: _Outage) -> None:
+ self.outage = outage
+
async def mget(self, *args, **kwargs):
- await asyncio.Event().wait()
+ await self.outage.hang()
async def set(self, *args, **kwargs):
- await asyncio.Event().wait()
-
-
-async def _never_connects():
- await asyncio.Event().wait()
+ await self.outage.hang()
@pytest.mark.asyncio
@pytest.mark.parametrize(
- "outage",
- [
- pytest.param(
- lambda: AsyncMock(side_effect=_never_connects), id="connect-hangs"
- ),
- pytest.param(
- lambda: AsyncMock(return_value=_HangingRedis()), id="command-hangs"
- ),
- pytest.param(
- lambda: AsyncMock(side_effect=RedisConnectionError("refused")),
- id="refused",
- ),
- pytest.param(
- lambda: AsyncMock(
- side_effect=RedisClusterException(
- "Cluster mode is not enabled on this node"
- )
- ),
- id="standalone-redis",
- ),
- ],
+ "kind", ["connect-hangs", "command-hangs", "refused", "standalone-redis"]
)
async def test_a_redis_outage_never_slows_or_blocks_billing(
- trial, fake_stripe, worker_b, save_tier, outage
+ trial, fake_stripe, worker_b, save_tier, kind
):
- """With Redis hanging or unusable, lookups answer from Stripe within the
- cache's deadline, and a charge still saves the paid plan."""
- with patch.object(cache_a, "get_redis_async", outage()):
+ """With Redis hanging or unusable, lookups answer from Stripe after at
+ most one deadline, then skip Redis entirely for the backoff, and a charge
+ still saves the paid plan.
+
+ Counts Redis attempts rather than timing them; the wall clock is only a
+ backstop, far below what paying a deadline per read would take."""
+ outage = _Outage(kind)
+ hangs = kind in ("connect-hangs", "command-hangs")
+ backstop = 10 * versioned_cache._REDIS_DEADLINE_SECONDS
+ with patch.object(versioned_cache, "get_redis_async", outage):
started = time.monotonic()
ends = await asyncio.wait_for(
asyncio.gather(
@@ -439,19 +469,35 @@ async def test_a_redis_outage_never_slows_or_blocks_billing(
),
WAIT,
)
+ first_wave = outage.attempts
+ # Each concurrent read tries Redis at most once before the backoff,
+ # and when Redis hangs every read pays that one deadline at the same
+ # time, not one after another.
+ assert 1 <= first_wave <= 60
+ assert outage.serial_waits == (1 if hangs else 0)
for _ in range(100):
ends.append(await worker_a.get_active_subscription_period_end("user-1"))
+ # The backoff holds: no read in its window tries Redis again.
+ assert outage.attempts == first_wave
assert set(ends) == {TRIAL_END}
- assert time.monotonic() - started < 1.5
# Stripe traffic stays bounded while Redis can't be used.
assert fake_stripe.lookups == 1
+ assert time.monotonic() - started < backstop
+ before = outage.attempts
+ waits_before = outage.serial_waits
started = time.monotonic()
await asyncio.wait_for(
conversion.convert_cancel_pending_trial(trial, SubscriptionTier.PRO),
WAIT,
)
- assert time.monotonic() - started < 2
+ # Only invalidations, which never skip Redis, try it:
+ # invalidate_billing_lookups drops both caches at once, after the tier
+ # sync and again after saving the conversion, so the purchase pays at
+ # most two deadlines in a row.
+ assert outage.attempts - before == 4
+ assert outage.serial_waits - waits_before == (2 if hangs else 0)
+ assert time.monotonic() - started < backstop
save_tier.assert_awaited_once()
assert save_tier.await_args.args[0]["status"] == "active"
@@ -470,7 +516,7 @@ async def test_an_answer_read_before_a_change_is_not_kept_during_an_outage(
after it. Its callers get the old answer, but this worker must not keep
it and serve it for the next few seconds."""
monkeypatch.setattr(
- cache_a,
+ versioned_cache,
"get_redis_async",
AsyncMock(side_effect=RedisConnectionError("refused")),
)
@@ -530,9 +576,11 @@ async def test_a_write_racing_an_unrecorded_invalidation_is_not_served_later(
subscription, and Redis blips so the new token isn't stored: the write
lands under the old token. Once Redis is back, this worker must not
serve it."""
- monkeypatch.setattr(cache_a, "_REDIS_DEADLINE_SECONDS", WAIT)
+ monkeypatch.setattr(versioned_cache, "_REDIS_DEADLINE_SECONDS", WAIT)
redis = _BlippingRedis(redis_up)
- monkeypatch.setattr(cache_a, "get_redis_async", AsyncMock(return_value=redis))
+ monkeypatch.setattr(
+ versioned_cache, "get_redis_async", AsyncMock(return_value=redis)
+ )
redis.release_eval = release = asyncio.Event()
first = asyncio.create_task(worker_a.get_active_subscription_period_end("user-1"))
await asyncio.wait_for(redis.eval_reached.wait(), WAIT)
@@ -550,7 +598,7 @@ async def test_a_write_racing_an_unrecorded_invalidation_is_not_served_later(
# The old answer did land in Redis under the unchanged token.
assert await _cached_period_end(redis_up, trial.customer_id) == TRIAL_END
- monkeypatch.setattr(cache_a._backoff, "until", 0.0)
+ monkeypatch.setattr(versioned_cache._backoff, "until", 0.0)
assert (
await worker_a.get_active_subscription_period_end("user-1") == PAID_PERIOD_END
)
@@ -567,7 +615,7 @@ class _Clock:
@pytest.fixture
def redis_down(monkeypatch):
monkeypatch.setattr(
- cache_a,
+ versioned_cache,
"get_redis_async",
AsyncMock(side_effect=RedisConnectionError("down")),
)
@@ -580,20 +628,25 @@ async def test_a_live_mark_survives_any_number_of_failed_invalidations(redis_dow
await cache_a.invalidate_active_subscription_cache("cus_changed")
for i in range(2100):
await cache_a.invalidate_active_subscription_cache(f"cus_other_{i}")
- assert cache_a._is_unconfirmed("cus_changed")
+ assert ACTIVE._is_unconfirmed("cus_changed")
@pytest.mark.asyncio
async def test_a_renewed_mark_outlives_expired_ones(redis_down, monkeypatch):
"""Expired marks must not crowd out a live one, whatever their order."""
clock = _Clock()
- monkeypatch.setattr(cache_a, "time", clock)
- monkeypatch.setattr(cache_a, "_UNCONFIRMED_MAXSIZE", 4, raising=False)
- monkeypatch.setattr(cache_a, "_FALLBACK_MAXSIZE", 4)
+ monkeypatch.setattr(versioned_cache, "time", clock)
+ monkeypatch.setattr(
+ ACTIVE,
+ "_unconfirmed",
+ UnconfirmedInvalidations(
+ lifetime=cache_a.SUBSCRIPTION_LOOKUP_TTL_SECONDS + 1, maxsize=4
+ ),
+ )
for customer in ("cus_changed", "cus_1", "cus_2", "cus_3"):
await cache_a.invalidate_active_subscription_cache(customer)
clock.now += 60
await cache_a.invalidate_active_subscription_cache("cus_changed")
await cache_a.invalidate_active_subscription_cache("cus_4")
- assert cache_a._is_unconfirmed("cus_changed")
+ assert ACTIVE._is_unconfirmed("cus_changed")
diff --git a/autogpt_platform/backend/backend/data/subscription_lookup_unconfirmed.py b/autogpt_platform/backend/backend/data/subscription_lookup_unconfirmed.py
new file mode 100644
index 0000000000..1e0ece0af7
--- /dev/null
+++ b/autogpt_platform/backend/backend/data/subscription_lookup_unconfirmed.py
@@ -0,0 +1,50 @@
+"""Invalidations this process couldn't record in Redis.
+
+A versioned cache entry is trusted while the token it was written under is
+current. When an invalidation fails to replace the token, a write already on
+its way when the change landed is stored under the old, unchanged token, so
+until any such entry has expired this process must read the key as if Redis
+were out. These marks say which keys, and until when.
+"""
+
+UNCONFIRMED_MAXSIZE = 2048
+
+
+class UnconfirmedInvalidations:
+ """Live marks, never dropped: with no room left, every key is covered
+ until the latest mark would have expired.
+
+ Callers pass the time (monotonic seconds), so the clock stays theirs.
+ """
+
+ def __init__(self, *, lifetime: float, maxsize: int = UNCONFIRMED_MAXSIZE):
+ self.lifetime = lifetime
+ self.maxsize = maxsize
+ self.until: dict[str, float] = {}
+ self.everyone_until = 0.0
+
+ def add(self, key: str, now: float) -> None:
+ until = now + self.lifetime
+ self._purge(now)
+ # Re-inserted at the end: every mark has the same lifetime, so
+ # insertion order stays expiry order.
+ self.until.pop(key, None)
+ if len(self.until) >= self.maxsize:
+ self.everyone_until = max(self.everyone_until, until)
+ return
+ self.until[key] = until
+
+ def clear(self, key: str) -> None:
+ self.until.pop(key, None)
+
+ def covers(self, key: str, now: float) -> bool:
+ if now < self.everyone_until:
+ return True
+ until = self.until.get(key)
+ return until is not None and until > now
+
+ def _purge(self, now: float) -> None:
+ for key, until in list(self.until.items()):
+ if until > now:
+ break
+ del self.until[key]
diff --git a/autogpt_platform/backend/backend/data/subscription_pending_cache.py b/autogpt_platform/backend/backend/data/subscription_pending_cache.py
new file mode 100644
index 0000000000..40a4499c2a
--- /dev/null
+++ b/autogpt_platform/backend/backend/data/subscription_pending_cache.py
@@ -0,0 +1,48 @@
+"""The pending subscription change the billing page shows, cached for every
+API worker.
+
+A queued downgrade (a Subscription Schedule phase) or cancellation
+(``cancel_at_period_end``) is read on every dashboard and billing page load
+and takes two Subscription lists plus a Schedule retrieve, so it is cached
+briefly; see :mod:`backend.data.versioned_cache` for how a change reaches
+every worker on its next read.
+
+Rolling out from the previous cache (a shared ``@cached`` with a plain
+delete, under other keys): until every pod runs this code, an old pod's
+invalidation doesn't replace the token new pods read, so a slow read on a
+new pod that started before such a change can still store the old answer.
+That entry lives for the TTL from when it was written, so the bound is
+30 seconds after the last such write, not after the change.
+"""
+
+from datetime import datetime
+from typing import Literal
+
+from prisma.enums import SubscriptionTier
+from pydantic import BaseModel
+
+from backend.data.versioned_cache import VersionedCache
+
+PENDING_CHANGE_TTL_SECONDS = 30
+
+
+class PendingChange(BaseModel):
+ tier: SubscriptionTier
+ effective_at: datetime
+ billing_cycle: Literal["monthly", "yearly"] | None = None
+
+
+# Keyed by Stripe customer id: the pending change is read from the customer's
+# Stripe subscription alone, so every change to it comes from Stripe and
+# reaches sync_subscription_from_stripe, which knows the customer.
+PENDING_CHANGES: VersionedCache[PendingChange | None] = VersionedCache(
+ "pending_subscription_change",
+ PendingChange | None,
+ ttl_seconds=PENDING_CHANGE_TTL_SECONDS,
+)
+
+
+async def invalidate_pending_subscription_change(customer_id: str) -> None:
+ """Make every worker read the customer's pending change fresh from Stripe.
+ Call it after the change has landed in Stripe; it never raises."""
+ await PENDING_CHANGES.invalidate(customer_id)
diff --git a/autogpt_platform/backend/backend/data/subscription_pending_cache_test.py b/autogpt_platform/backend/backend/data/subscription_pending_cache_test.py
new file mode 100644
index 0000000000..9303d144c8
--- /dev/null
+++ b/autogpt_platform/backend/backend/data/subscription_pending_cache_test.py
@@ -0,0 +1,236 @@
+"""The billing page's pending subscription change stays fresh across API
+workers.
+
+Each "worker" is its own copy of ``backend.data.credit`` and of the pending
+change cache, so it holds its own process-local state; both share the real
+Redis of the test stack and one fake Stripe.
+"""
+
+import asyncio
+import importlib.util
+from datetime import UTC, datetime, timedelta
+from types import ModuleType
+from unittest.mock import AsyncMock, MagicMock, patch
+from uuid import uuid4
+
+import pytest
+import pytest_asyncio
+import stripe
+from prisma.enums import SubscriptionTier
+
+from backend.data import credit as worker_a
+from backend.data import versioned_cache
+from backend.data.redis_client import get_redis_async
+from backend.data.subscription_pending_cache import PENDING_CHANGES
+from backend.util.exceptions import InsufficientBalanceError
+
+PERIOD_END = int((datetime.now(UTC) + timedelta(days=20)).timestamp())
+WAIT = 5 # seconds; no wait in these tests may hang the suite
+
+
+def _load_copy(name: str) -> ModuleType:
+ spec = importlib.util.find_spec(name)
+ assert spec is not None and spec.loader is not None
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+def _load_worker() -> ModuleType:
+ """A second, independently loaded credit module with its own cache."""
+ cache = _load_copy("backend.data.subscription_pending_cache")
+ cache.PENDING_CHANGES.prefix = PENDING_CHANGES.prefix
+ worker = _load_copy("backend.data.credit")
+ worker.PENDING_CHANGES = cache.PENDING_CHANGES
+ worker.invalidate_pending_subscription_change = (
+ cache.invalidate_pending_subscription_change
+ )
+ return worker
+
+
+class FakeStripe:
+ """One customer's paid subscription, as Stripe would answer for it."""
+
+ def __init__(self, customer_id: str) -> None:
+ self.subscription = {
+ "id": "sub_pro",
+ "object": "subscription",
+ "customer": customer_id,
+ "status": "active",
+ "cancel_at_period_end": False,
+ "schedule": None,
+ "metadata": {},
+ "current_period_start": PERIOD_END - 30 * 24 * 60 * 60,
+ "current_period_end": PERIOD_END,
+ "items": {
+ "object": "list",
+ "data": [{"id": "si_1", "price": {"id": "price_PRO_monthly"}}],
+ "has_more": False,
+ },
+ }
+ self.lookups = 0
+ self.reached = asyncio.Event()
+
+ def current(self) -> stripe.Subscription:
+ return stripe.Subscription.construct_from(self.subscription, "test-key")
+
+ async def list_async(self, *, customer: str, status: str, limit: int):
+ if status == "active":
+ self.lookups += 1
+ answer = self.current()
+ matches = [answer] if status == answer["status"] else []
+ return stripe.ListObject.construct_from(
+ {"object": "list", "data": matches, "has_more": False}, "test-key"
+ )
+
+ async def modify_async(self, subscription_id: str, **changes):
+ self.subscription = {**self.subscription, **changes}
+ return self.current()
+
+ async def cancel_async(self, subscription_id: str, **params):
+ self.subscription = {**self.subscription, "status": "canceled"}
+ return self.current()
+
+ def hold_next_lookup(
+ self, monkeypatch: pytest.MonkeyPatch, worker: ModuleType
+ ) -> asyncio.Event:
+ """Answer ``worker``'s next subscription lookup as it is now, once
+ released: a slow answer that arrives after a change made meanwhile."""
+ before = self.current()
+ release = asyncio.Event()
+ ask_stripe = worker._get_active_subscription
+ held = False
+
+ async def lookup(customer_id: str):
+ nonlocal held
+ if held:
+ return await ask_stripe(customer_id)
+ held = True
+ self.reached.set()
+ await asyncio.wait_for(release.wait(), WAIT)
+ return before
+
+ monkeypatch.setattr(worker, "_get_active_subscription", lookup)
+ return release
+
+
+async def _price_id(tier: SubscriptionTier, cycle: str = "monthly") -> str:
+ return f"price_{tier.value}_{cycle}"
+
+
+def user_id() -> str:
+ return f"user-{uuid4().hex}"
+
+
+def fake_stripe() -> FakeStripe:
+ return FakeStripe(f"cus_{uuid4().hex}")
+
+
+@pytest_asyncio.fixture
+async def worker_b(user_id: str, fake_stripe: FakeStripe, monkeypatch):
+ # These tests assert what is cached, not how fast: a busy CI runner can
+ # take longer than the production deadline for one Redis call.
+ monkeypatch.setattr(versioned_cache, "_REDIS_DEADLINE_SECONDS", WAIT)
+ redis = await asyncio.wait_for(get_redis_async(), WAIT)
+ assert await asyncio.wait_for(redis.ping(), WAIT), "these tests need Redis"
+ user = MagicMock(
+ id=user_id,
+ stripe_customer_id=fake_stripe.subscription["customer"],
+ subscription_tier=SubscriptionTier.PRO,
+ )
+ lookup_user = AsyncMock(return_value=user)
+ worker = _load_worker()
+ with (
+ patch.object(stripe.Subscription, "list_async", fake_stripe.list_async),
+ patch.object(stripe.Subscription, "modify_async", fake_stripe.modify_async),
+ patch.object(stripe.Subscription, "cancel_async", fake_stripe.cancel_async),
+ patch.object(worker_a, "get_user_by_id", lookup_user),
+ patch.object(worker, "get_user_by_id", lookup_user),
+ patch.object(worker_a, "get_subscription_price_id", _price_id),
+ patch.object(worker, "get_subscription_price_id", _price_id),
+ patch.object(worker_a, "_track_billing_event", MagicMock()),
+ # The tier write and the analytics sync are not what's under test.
+ patch.object(worker_a, "_sync_subscription_tier_from_stripe", AsyncMock()),
+ patch.object(worker_a, "schedule_posthog_lifecycle_sync", MagicMock()),
+ ):
+ yield worker
+
+
+async def test_a_read_in_flight_during_a_cancellation_does_not_hide_it(
+ user_id, fake_stripe, worker_b, monkeypatch
+):
+ """A page load on one worker reads the plan before the user cancels on
+ another; its slow answer must not wipe out the "cancels on" badge."""
+ release = fake_stripe.hold_next_lookup(monkeypatch, worker_b)
+ in_flight = asyncio.create_task(worker_b.get_pending_subscription_change(user_id))
+ await asyncio.wait_for(fake_stripe.reached.wait(), WAIT)
+
+ assert await worker_a.cancel_stripe_subscription(user_id) is True
+ cancelling = await worker_a.get_pending_subscription_change(user_id)
+ assert cancelling is not None and cancelling[0] == SubscriptionTier.NO_TIER
+
+ release.set()
+ assert await asyncio.wait_for(in_flight, WAIT) is None
+
+ for worker in (worker_a, worker_b):
+ pending = await worker.get_pending_subscription_change(user_id)
+ assert pending is not None
+ assert pending[0] == SubscriptionTier.NO_TIER
+ assert int(pending[1].timestamp()) == PERIOD_END
+
+
+async def test_a_cancellation_made_in_stripe_shows_on_every_worker(
+ user_id, fake_stripe, worker_b
+):
+ """A cancellation from Stripe's billing portal reaches us only as a
+ webhook, and leaves the tier as it is until the period ends."""
+ assert await worker_b.get_pending_subscription_change(user_id) is None
+
+ cancelled = await fake_stripe.modify_async("sub_pro", cancel_at_period_end=True)
+ await worker_a.sync_subscription_from_stripe(dict(cancelled))
+
+ pending = await worker_b.get_pending_subscription_change(user_id)
+ assert pending is not None and pending[0] == SubscriptionTier.NO_TIER
+ assert fake_stripe.lookups == 2
+
+
+async def test_a_renewal_the_balance_cant_cover_clears_the_pending_change(
+ user_id, fake_stripe, worker_b
+):
+ """A cancellation is scheduled, then the renewal fails and the balance
+ can't cover it, so we end the plan in Stripe now: the "cancels on" badge
+ must go on every worker, not linger until it expires."""
+ fake_stripe.subscription["cancel_at_period_end"] = True
+ pending = await worker_b.get_pending_subscription_change(user_id)
+ assert pending is not None and pending[0] == SubscriptionTier.NO_TIER
+
+ users = MagicMock()
+ users.prisma.return_value.find_first = AsyncMock(
+ return_value=MagicMock(id=user_id, subscriptionTier=SubscriptionTier.PRO)
+ )
+ balance = MagicMock()
+ balance._add_transaction = AsyncMock(
+ side_effect=InsufficientBalanceError("no balance", user_id, 0, 5000)
+ )
+ renewal = {
+ "id": "in_renewal",
+ "customer": fake_stripe.subscription["customer"],
+ "amount_due": 5000,
+ "billing_reason": "subscription_cycle",
+ "subscription": "sub_pro",
+ }
+ with (
+ patch.object(worker_a, "User", users),
+ patch.object(worker_a, "UserCredit", MagicMock(return_value=balance)),
+ patch.object(worker_a, "set_subscription_tier", AsyncMock()) as set_tier,
+ ):
+ await worker_a.handle_subscription_payment_failure(renewal)
+
+ assert fake_stripe.subscription["status"] == "canceled"
+ set_tier.assert_awaited_once_with(user_id, SubscriptionTier.NO_TIER)
+ assert await worker_b.get_pending_subscription_change(user_id) is None
diff --git a/autogpt_platform/backend/backend/data/subscription_trial_conversion.py b/autogpt_platform/backend/backend/data/subscription_trial_conversion.py
index 162367fd14..4bef25e1da 100644
--- a/autogpt_platform/backend/backend/data/subscription_trial_conversion.py
+++ b/autogpt_platform/backend/backend/data/subscription_trial_conversion.py
@@ -11,7 +11,10 @@ import stripe
from prisma.enums import SubscriptionTier
from pydantic import BaseModel
-from backend.data.credit import sync_subscription_from_stripe
+from backend.data.credit import (
+ invalidate_billing_lookups,
+ sync_subscription_from_stripe,
+)
from backend.data.stripe_authentication import requires_authentication
from backend.data.stripe_client import stripe_call
from backend.data.subscription_checkout import (
@@ -21,7 +24,6 @@ from backend.data.subscription_checkout import (
other_plan_is_live,
subscription_checkout_lock,
)
-from backend.data.subscription_lookup_cache import invalidate_active_subscription_cache
from backend.data.subscription_trial import TrialState, get_subscription_trial
from backend.data.subscription_trial_payment import (
CONVERSION_PRICE_KEY,
@@ -202,7 +204,7 @@ async def _save_conversion(trial: TrialState, converted: stripe.Subscription) ->
f"Trial {trial.id} converted in Stripe but the follow-up sync failed"
)
finally:
- await invalidate_active_subscription_cache(trial.customer_id)
+ await invalidate_billing_lookups(trial.customer_id)
def _is_cancel_pending(trial: TrialState) -> bool:
diff --git a/autogpt_platform/backend/backend/data/subscription_trial_conversion_test.py b/autogpt_platform/backend/backend/data/subscription_trial_conversion_test.py
index 50d00cdb9a..16fd37728a 100644
--- a/autogpt_platform/backend/backend/data/subscription_trial_conversion_test.py
+++ b/autogpt_platform/backend/backend/data/subscription_trial_conversion_test.py
@@ -527,7 +527,7 @@ async def test_the_paid_plan_is_saved_before_the_lookup_is_dropped(
async def invalidate(customer_id: str) -> None:
boundaries.calls.append(f"invalidate:{customer_id}")
- with patch.object(conversion, "invalidate_active_subscription_cache", invalidate):
+ with patch.object(conversion, "invalidate_billing_lookups", invalidate):
assert await conversion.convert_cancel_pending_trial(pending_trial, PRO) is None
assert boundaries.calls[-4:] == ["modify", "sync", "invalidate:cus_1", "unlock"]
diff --git a/autogpt_platform/backend/backend/data/versioned_cache.py b/autogpt_platform/backend/backend/data/versioned_cache.py
new file mode 100644
index 0000000000..5a348383b0
--- /dev/null
+++ b/autogpt_platform/backend/backend/data/versioned_cache.py
@@ -0,0 +1,315 @@
+"""A short-TTL cache shared by every API worker, safe to invalidate.
+
+For lookups that are read on almost every page load but must show a change
+right after it is made (billing state read from Stripe). Each key has a
+random version token in Redis; every change replaces it, so the next read on
+any worker misses. An entry is written only while the token it was read under
+is still current, so a read that was in flight when a change landed can't put
+the old answer back. A missing token (expired, or Redis lost its data) is
+never treated as a version: the reader establishes a fresh one before it
+fetches, and before it picks its in-flight key.
+
+Within a process, concurrent reads of one key and token share one fetch. A
+read stores its answer only while it is still the current one for its key, so
+an answer read before a change this process made goes to the callers already
+waiting for it and is never kept.
+
+Redis is only an optimisation. Every operation has a short hard deadline, and
+after a failure this process skips Redis for a few seconds, so an outage or a
+hung connection costs a read at most one deadline and never blocks or fails
+the change around it. Meanwhile answers are kept in process for a few seconds
+to bound the traffic to the source; under churn across more keys than that
+memo holds, it degrades to a fetch per read, still shared per key. When an
+invalidation can't replace the token, this process reads the key as if Redis
+were out until anything written under the old token has expired. A Redis
+that isn't a cluster (the platform always runs one) gets the fallback all the
+time.
+"""
+
+import asyncio
+import logging
+import time
+from typing import Any, Awaitable, Callable, Coroutine, Generic, TypeVar
+from uuid import uuid4
+
+from pydantic import BaseModel, TypeAdapter, ValidationError
+
+from backend.data.redis_client import AsyncRedisClient, get_redis_async
+from backend.data.subscription_lookup_unconfirmed import UnconfirmedInvalidations
+
+logger = logging.getLogger(__name__)
+
+T = TypeVar("T")
+V = TypeVar("V")
+Fetch = Callable[[], Coroutine[Any, Any, V]]
+
+# Bounds connecting as well as the command: the shared client retries a
+# failed connection for minutes on its own.
+_REDIS_DEADLINE_SECONDS = 0.3
+_REDIS_BACKOFF_SECONDS = 5.0
+# How long a process reuses an answer while Redis can't be used; also how long
+# another worker can show the old state after a change in that time.
+_FALLBACK_TTL_SECONDS = 5.0
+_FALLBACK_MAXSIZE = 2048
+_VERSION_TTL_SECONDS = 24 * 60 * 60
+# The in-flight key for reads while Redis can't be used; tokens are hex.
+_NO_REDIS = "no-redis"
+
+# KEYS: version, entry. ARGV: version read, entry, TTL.
+_WRITE_IF_CURRENT = """
+if redis.call('GET', KEYS[1]) == ARGV[1] then
+ return redis.call('SET', KEYS[2], ARGV[2], 'EX', ARGV[3])
+end
+return false
+"""
+
+
+class _Entry(BaseModel):
+ version: str
+ value: Any
+
+
+class _RedisUnavailable(Exception):
+ pass
+
+
+class _Backoff:
+ """Process-local, for every cache: skip Redis for a while after it failed."""
+
+ def __init__(self) -> None:
+ self.until = 0.0
+
+ def active(self) -> bool:
+ return time.monotonic() < self.until
+
+ def trip(self) -> None:
+ if not self.active():
+ logger.warning(
+ "Versioned cache Redis unavailable; reading sources directly"
+ f" for {_REDIS_BACKOFF_SECONDS:.0f}s"
+ )
+ self.until = time.monotonic() + _REDIS_BACKOFF_SECONDS
+
+
+_backoff = _Backoff()
+
+
+class VersionedCache(Generic[V]):
+ """``prefix`` names the Redis keys; ``value_type`` is what ``fetch``
+ returns, stored as JSON."""
+
+ def __init__(self, prefix: str, value_type: Any, *, ttl_seconds: int) -> None:
+ self.prefix = prefix
+ self.ttl_seconds = ttl_seconds
+ self._adapter: TypeAdapter[V] = TypeAdapter(value_type)
+ # Reads running in this process, by (event loop, key, version).
+ self._in_flight: dict[
+ tuple[asyncio.AbstractEventLoop, str, str], asyncio.Task[V]
+ ] = {}
+ # Answers kept while Redis can't be used: key -> (expires at, answer).
+ self._fallback: dict[str, tuple[float, V]] = {}
+ # A mark outlives any entry written meanwhile: a write in flight
+ # finishes within the Redis deadline, and entries live for the TTL.
+ self._unconfirmed = UnconfirmedInvalidations(lifetime=ttl_seconds + 1)
+
+ async def read(self, key: str, fetch: Fetch[V]) -> V:
+ """The cached value, or ``fetch``'s answer when it isn't cached.
+ ``fetch``'s exceptions reach every caller sharing it, and nothing is
+ cached for them."""
+ if self._is_unconfirmed(key):
+ return await self._read_without_redis(key, fetch)
+ try:
+ version, cached = await self._read_entry(key)
+ if version is None:
+ # Established before the in-flight key is chosen, so a read
+ # after Redis lost the token never joins one started before.
+ version = await self._establish_version(key)
+ except _RedisUnavailable:
+ return await self._read_without_redis(key, fetch)
+ if cached is not None and cached[0] == version:
+ return cached[1]
+ return await self._shared(
+ key, version, lambda: self._fetch_and_store(key, version, fetch)
+ )
+
+ async def invalidate(self, key: str) -> None:
+ """Make every worker fetch ``key`` fresh on its next read.
+
+ Call it after the change has landed at the source, never before: a
+ read started earlier could otherwise cache the old state under the
+ new token. Never raises and returns within the Redis deadline,
+ because the change it follows has already been made.
+ """
+ self._forget_locally(key)
+ try:
+ await _call_redis(
+ lambda redis: redis.set(
+ self._version_key(key), uuid4().hex, ex=_VERSION_TTL_SECONDS
+ ),
+ skip_during_backoff=False,
+ )
+ except _RedisUnavailable:
+ # The old token still vouches for entries written under it.
+ self._unconfirmed.add(key, time.monotonic())
+ logger.warning(
+ f"Could not invalidate {self.prefix} for {key}; other workers"
+ f" can show it stale for {self.ttl_seconds}s"
+ )
+ else:
+ # From now on the script refuses anything read under the old token.
+ self._unconfirmed.clear(key)
+
+ async def _read_entry(self, key: str) -> tuple[str | None, tuple[str, V] | None]:
+ keys = (self._version_key(key), self._entry_key(key))
+ version, raw = await _call_redis(lambda redis: redis.mget(keys))
+ return _text(version), self._parse_entry(_text(raw))
+
+ async def _establish_version(self, key: str) -> str:
+ """A fresh token for a key that has none, or the one another reader
+ created first."""
+ token = uuid4().hex
+ version_key = self._version_key(key)
+ created = await _call_redis(
+ lambda redis: redis.set(
+ version_key, token, nx=True, ex=_VERSION_TTL_SECONDS
+ )
+ )
+ if created:
+ return token
+ current = _text(await _call_redis(lambda redis: redis.get(version_key)))
+ if current:
+ return current
+ # Expired again in between: a token of our own only costs a miss.
+ await _call_redis(
+ lambda redis: redis.set(version_key, token, ex=_VERSION_TTL_SECONDS)
+ )
+ return token
+
+ async def _fetch_and_store(self, key: str, version: str, fetch: Fetch[V]) -> V:
+ # Another read may have stored it while this one was being started.
+ try:
+ current, cached = await self._read_entry(key)
+ except _RedisUnavailable:
+ current, cached = None, None
+ if current == version and cached is not None and cached[0] == version:
+ return cached[1]
+ value = await fetch()
+ if not self._still_current(key, version) or self._is_unconfirmed(key):
+ return value
+ entry = _Entry(
+ version=version, value=self._adapter.dump_python(value, mode="json")
+ )
+ keys_and_args = (
+ self._version_key(key),
+ self._entry_key(key),
+ version,
+ entry.model_dump_json(),
+ self.ttl_seconds,
+ )
+ try:
+ await _call_redis(
+ lambda redis: redis.eval(_WRITE_IF_CURRENT, 2, *keys_and_args)
+ )
+ except _RedisUnavailable:
+ pass
+ return value
+
+ async def _read_without_redis(self, key: str, fetch: Fetch[V]) -> V:
+ kept = self._fallback.get(key)
+ if kept is not None and kept[0] > time.monotonic():
+ return kept[1]
+ return await self._shared(
+ key, _NO_REDIS, lambda: self._fetch_and_keep(key, fetch)
+ )
+
+ async def _fetch_and_keep(self, key: str, fetch: Fetch[V]) -> V:
+ value = await fetch()
+ if not self._still_current(key, _NO_REDIS):
+ return value
+ self._fallback[key] = (time.monotonic() + _FALLBACK_TTL_SECONDS, value)
+ if len(self._fallback) > _FALLBACK_MAXSIZE:
+ # Dropping a kept answer only costs another fetch.
+ del self._fallback[next(iter(self._fallback))]
+ return value
+
+ def _is_unconfirmed(self, key: str) -> bool:
+ return self._unconfirmed.covers(key, time.monotonic())
+
+ async def _shared(
+ self, key: str, version: str, read: Callable[[], Coroutine[Any, Any, V]]
+ ) -> V:
+ """Run ``read`` once for every concurrent caller with the same key.
+ The version is part of the key, so a read after a change never joins
+ one started before it."""
+ flight = (asyncio.get_running_loop(), key, version)
+ task = self._in_flight.get(flight)
+ if task is None:
+ task = asyncio.create_task(read())
+ self._in_flight[flight] = task
+ task.add_done_callback(lambda done: self._drop_in_flight(flight, done))
+ # One caller giving up must not cancel the read the others wait on.
+ return await asyncio.shield(task)
+
+ def _still_current(self, key: str, version: str) -> bool:
+ """Whether the running read still owns its in-flight key: invalidating
+ in this process drops it, and then its answer must not be kept."""
+ flight = (asyncio.get_running_loop(), key, version)
+ return self._in_flight.get(flight) is asyncio.current_task()
+
+ def _drop_in_flight(
+ self,
+ flight: tuple[asyncio.AbstractEventLoop, str, str],
+ task: asyncio.Task[V],
+ ) -> None:
+ if self._in_flight.get(flight) is task:
+ del self._in_flight[flight]
+
+ def _forget_locally(self, key: str) -> None:
+ """Stop this process reusing what it read before a change: its
+ fallback answer, and reads still running (whose version may not have
+ changed if Redis couldn't be told)."""
+ self._fallback.pop(key, None)
+ for flight in [flight for flight in self._in_flight if flight[1] == key]:
+ del self._in_flight[flight]
+
+ def _version_key(self, key: str) -> str:
+ # The hash tag keeps both keys on one cluster slot, for MGET and the
+ # script.
+ return f"{self.prefix}:{{{key}}}:version"
+
+ def _entry_key(self, key: str) -> str:
+ return f"{self.prefix}:{{{key}}}:lookup"
+
+ def _parse_entry(self, raw: str | None) -> tuple[str, V] | None:
+ if not raw:
+ return None
+ try:
+ entry = _Entry.model_validate_json(raw)
+ return entry.version, self._adapter.validate_python(entry.value)
+ except ValidationError:
+ return None
+
+
+def _text(value: str | bytes | None) -> str | None:
+ # The shared client decodes responses; its type hints don't say so.
+ return value.decode() if isinstance(value, bytes) else value
+
+
+async def _call_redis(
+ operation: Callable[[AsyncRedisClient], Awaitable[T]],
+ *,
+ skip_during_backoff: bool = True,
+) -> T:
+ if skip_during_backoff and _backoff.active():
+ raise _RedisUnavailable()
+ try:
+ return await asyncio.wait_for(
+ _with_client(operation), timeout=_REDIS_DEADLINE_SECONDS
+ )
+ except Exception as exc:
+ _backoff.trip()
+ raise _RedisUnavailable() from exc
+
+
+async def _with_client(operation: Callable[[AsyncRedisClient], Awaitable[T]]) -> T:
+ return await operation(await get_redis_async())
diff --git a/autogpt_platform/backend/backend/data/subscription_lookup_cache_interleaving_test.py b/autogpt_platform/backend/backend/data/versioned_cache_interleaving_test.py
similarity index 58%
rename from autogpt_platform/backend/backend/data/subscription_lookup_cache_interleaving_test.py
rename to autogpt_platform/backend/backend/data/versioned_cache_interleaving_test.py
index 3a56f89779..6a0f8c3d94 100644
--- a/autogpt_platform/backend/backend/data/subscription_lookup_cache_interleaving_test.py
+++ b/autogpt_platform/backend/backend/data/versioned_cache_interleaving_test.py
@@ -1,8 +1,9 @@
-"""Randomized interleavings of the subscription lookup cache.
+"""Randomized interleavings of the versioned billing caches: the active
+subscription lookup and the pending subscription change.
The invariant that matters to a customer: once this worker has changed a
-subscription and invalidated the lookup, no read that starts afterwards on
-this worker returns the subscription as it was before the change. Each seed
+subscription and invalidated the cache, no read that starts afterwards on
+this worker returns the state as it was before the change. Each seed
interleaves reads, changes, Redis outages and blips (failed writes, replies
lost after the write applied, lost data), slow Stripe answers and slow
script writes, on a fake clock, with the in-process bounds shrunk so they
@@ -11,11 +12,25 @@ are under pressure. Nothing sleeps; the clock only moves when told to.
import asyncio
import random
+from datetime import UTC, datetime
+from unittest.mock import MagicMock
import pytest
+from prisma.enums import SubscriptionTier
from redis.exceptions import ConnectionError as RedisConnectionError
-from backend.data import subscription_lookup_cache as cache
+from backend.data import credit, versioned_cache
+from backend.data.subscription_lookup_cache import (
+ ACTIVE_SUBSCRIPTIONS,
+ invalidate_active_subscription_cache,
+ read_active_subscription,
+)
+from backend.data.subscription_lookup_unconfirmed import UnconfirmedInvalidations
+from backend.data.subscription_pending_cache import (
+ PENDING_CHANGES,
+ PendingChange,
+ invalidate_pending_subscription_change,
+)
SEEDS = range(300)
STEPS = 80
@@ -114,7 +129,7 @@ class FakeRedis:
class Run:
- def __init__(self, seed: int, monkeypatch: pytest.MonkeyPatch) -> None:
+ def __init__(self, seed: int, target: str, monkeypatch: pytest.MonkeyPatch) -> None:
self.seed = seed
self.rng = random.Random(seed)
self.clock = Clock()
@@ -125,48 +140,57 @@ class Run:
self.completed = dict.fromkeys(CUSTOMERS, 0)
self.violations: list[str] = []
self.tasks: list[asyncio.Task[None]] = []
+ self.cache, self.read_value, self.invalidate = TARGETS[target]
async def get_redis():
return self.redis
- monkeypatch.setattr(cache, "time", self.clock)
- monkeypatch.setattr(cache, "get_redis_async", get_redis)
- monkeypatch.setattr(cache, "_REDIS_DEADLINE_SECONDS", 3600)
- monkeypatch.setattr(cache, "_FALLBACK_MAXSIZE", 1)
- monkeypatch.setattr(cache, "_UNCONFIRMED_MAXSIZE", 1)
- monkeypatch.setattr(cache, "_backoff", cache._Backoff())
- monkeypatch.setattr(cache, "_unconfirmed", cache._UnconfirmedInvalidations())
- monkeypatch.setattr(cache, "_fallback", {})
- monkeypatch.setattr(cache, "_in_flight", {})
-
- def fetch(self, customer_id: str):
- async def ask_stripe():
- value = self.state[customer_id]
- await self.stripe_holds.maybe_hold()
- return {
- "status": "active",
- "current_period_start": BASE,
- "current_period_end": BASE + value,
- }
-
- return ask_stripe
-
- async def read(self, customer_id: str) -> None:
- required = self.completed[customer_id]
- subscription = await cache.read_active_subscription(
- customer_id, self.fetch(customer_id)
+ async def fetch_pending(customer_id: str) -> PendingChange:
+ value = await self.answer(customer_id)
+ return PendingChange(
+ tier=SubscriptionTier.NO_TIER,
+ effective_at=datetime.fromtimestamp(BASE + value, UTC),
+ )
+
+ monkeypatch.setattr(versioned_cache, "time", self.clock)
+ monkeypatch.setattr(versioned_cache, "get_redis_async", get_redis)
+ monkeypatch.setattr(versioned_cache, "_REDIS_DEADLINE_SECONDS", 3600)
+ monkeypatch.setattr(versioned_cache, "_FALLBACK_MAXSIZE", 1)
+ monkeypatch.setattr(versioned_cache, "_backoff", versioned_cache._Backoff())
+
+ async def user_by_id(user_id: str) -> MagicMock:
+ # Each key is both the user id and their Stripe customer id.
+ return MagicMock(stripe_customer_id=user_id)
+
+ monkeypatch.setattr(credit, "_fetch_pending_subscription_change", fetch_pending)
+ monkeypatch.setattr(credit, "get_user_by_id", user_by_id)
+ monkeypatch.setattr(
+ self.cache,
+ "_unconfirmed",
+ UnconfirmedInvalidations(lifetime=self.cache.ttl_seconds + 1, maxsize=1),
)
- seen = (subscription.current_period_end or 0) - BASE if subscription else -1
+ monkeypatch.setattr(self.cache, "_fallback", {})
+ monkeypatch.setattr(self.cache, "_in_flight", {})
+
+ async def answer(self, key: str) -> int:
+ """What the source says now, delivered after a possible delay."""
+ value = self.state[key]
+ await self.stripe_holds.maybe_hold()
+ return value
+
+ async def read(self, key: str) -> None:
+ required = self.completed[key]
+ seen = await self.read_value(self, key)
if seen < required:
self.violations.append(
- f"seed {self.seed}: {customer_id} read {seen} after change {required}"
+ f"seed {self.seed}: {key} read {seen} after change {required}"
)
- async def change(self, customer_id: str) -> None:
- self.state[customer_id] += 1
- value = self.state[customer_id]
- await cache.invalidate_active_subscription_cache(customer_id)
- self.completed[customer_id] = max(self.completed[customer_id], value)
+ async def change(self, key: str) -> None:
+ self.state[key] += 1
+ value = self.state[key]
+ await self.invalidate(key)
+ self.completed[key] = max(self.completed[key], value)
def step(self) -> None:
roll = self.rng.random()
@@ -201,12 +225,45 @@ class Run:
task.result()
+async def read_active(run: Run, customer_id: str) -> int:
+ async def ask_stripe():
+ value = await run.answer(customer_id)
+ return {
+ "status": "active",
+ "current_period_start": BASE,
+ "current_period_end": BASE + value,
+ }
+
+ subscription = await read_active_subscription(customer_id, ask_stripe)
+ return (subscription.current_period_end or 0) - BASE if subscription else -1
+
+
+async def read_pending(run: Run, user_id: str) -> int:
+ pending = await credit.get_pending_subscription_change(user_id)
+ return int(pending[1].timestamp()) - BASE if pending else -1
+
+
+TARGETS = {
+ "active-subscription": (
+ ACTIVE_SUBSCRIPTIONS,
+ read_active,
+ invalidate_active_subscription_cache,
+ ),
+ "pending-change": (
+ PENDING_CHANGES,
+ read_pending,
+ invalidate_pending_subscription_change,
+ ),
+}
+
+
@pytest.mark.asyncio
-async def test_no_read_after_a_change_returns_the_old_subscription(monkeypatch):
[email protected]("target", list(TARGETS))
+async def test_no_read_after_a_change_returns_the_old_state(monkeypatch, target):
violations: list[str] = []
for seed in SEEDS:
with monkeypatch.context() as patched:
- run = Run(seed, patched)
+ run = Run(seed, target, patched)
await asyncio.wait_for(run.play(), 30)
violations.extend(run.violations)
assert violations == []