Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend/copilot): read a trial user's trial once per chat message, not four times (#15467)

Nicholas Tindlee49524177643
diff --git a/autogpt_platform/backend/backend/api/features/chat/routes.py b/autogpt_platform/backend/backend/api/features/chat/routes.py
index 02dd581b5d..51df0a5cc6 100644
--- a/autogpt_platform/backend/backend/api/features/chat/routes.py
+++ b/autogpt_platform/backend/backend/api/features/chat/routes.py
@@ -156,6 +156,7 @@ from backend.copilot.transports import (
is_deployment_chat_available,
save_default_chat_route,
)
+from backend.copilot.trial_reads import sharing_trial_reads
from backend.copilot.turn_tracing import MODEL_OUTPUT_CHUNKS
from backend.data.credit import UsageTransactionMetadata, get_user_credit_model
from backend.data.redis_client import get_redis_async
@@ -1726,6 +1727,9 @@ def _empty_ui_message_stream_response(
},
},
)
+# The paywall and the rate-limit checks below each read a trial user's
+# subscription trial; inside this handler they share one read.
+@sharing_trial_reads
async def stream_chat_post(
session_id: str,
request: StreamChatRequest,
diff --git a/autogpt_platform/backend/backend/copilot/rate_limit.py b/autogpt_platform/backend/backend/copilot/rate_limit.py
index c775b94927..e115e78596 100644
--- a/autogpt_platform/backend/backend/copilot/rate_limit.py
+++ b/autogpt_platform/backend/backend/copilot/rate_limit.py
@@ -64,9 +64,11 @@ from pydantic import BaseModel, Field
from redis.exceptions import RedisClusterException, RedisError
from backend.copilot.trial_cost_context import record_attributed_trial_cost
+from backend.copilot.trial_reads import shared_trial_read
from backend.data.db_accessors import credit_db, user_db
from backend.data.posthog_lifecycle_sync import schedule_posthog_lifecycle_sync
from backend.data.redis_client import AsyncRedisClient, get_redis_async
+from backend.data.subscription_trial import TrialState
from backend.data.user import get_user_by_id
from backend.util.cache import cached
from backend.util.exceptions import UserPaywalledError
@@ -569,7 +571,7 @@ async def get_remaining_usd_budget(
"""
trial = None
if await _fetch_user_tier(user_id) == SubscriptionTier.TRIAL:
- trial = await credit_db().get_subscription_trial(user_id)
+ trial = await _subscription_trial(user_id)
if trial is None or not trial.active:
return 0.0
now = datetime.now(UTC)
@@ -693,7 +695,7 @@ async def check_rate_limit(
"""
now = datetime.now(UTC)
if await _fetch_user_tier(user_id) == SubscriptionTier.TRIAL:
- trial = await credit_db().get_subscription_trial(user_id)
+ trial = await _subscription_trial(user_id)
if trial is None or not trial.active:
raise RateLimitExceeded("trial", now)
if trial.cost_microdollars >= trial.offer.total_cost_limit:
@@ -964,6 +966,13 @@ async def _fetch_user_tier(user_id: str) -> SubscriptionTier:
raise _UserNotFoundError(user_id)
+async def _subscription_trial(user_id: str) -> TrialState | None:
+ """The user's trial; one read serves every check inside ``trial_reads_shared``."""
+ return await shared_trial_read(
+ user_id, lambda: credit_db().get_subscription_trial(user_id)
+ )
+
+
_STRIPE_RECONCILE_PREFIX = "stripe_reconcile:"
_STRIPE_RECONCILE_TTL = 300 # seconds — matches the tier cache TTL
@@ -1034,7 +1043,7 @@ async def get_user_tier(user_id: str) -> SubscriptionTier:
tier_from_db = False
if tier == SubscriptionTier.TRIAL:
- trial = await credit_db().get_subscription_trial(user_id)
+ trial = await _subscription_trial(user_id)
return tier if trial and trial.active else SubscriptionTier.NO_TIER
if tier != SubscriptionTier.NO_TIER:
return tier
@@ -1296,7 +1305,7 @@ async def get_global_rate_limits(
# falls back to BASIC limits so testers retain access.
tier = await get_user_tier(user_id)
if tier == SubscriptionTier.TRIAL:
- trial = await credit_db().get_subscription_trial(user_id)
+ trial = await _subscription_trial(user_id)
if trial is None or not trial.active:
return 0, 0, SubscriptionTier.NO_TIER
return trial.offer.daily_cost_limit, trial.offer.weekly_cost_limit, tier
@@ -1425,7 +1434,7 @@ async def is_user_paywalled(user_id: str) -> bool:
)
tier = SubscriptionTier.NO_TIER
if tier == SubscriptionTier.TRIAL:
- trial = await credit_db().get_subscription_trial(user_id)
+ trial = await _subscription_trial(user_id)
return trial is None or not trial.active
if tier != SubscriptionTier.NO_TIER:
return False
diff --git a/autogpt_platform/backend/backend/copilot/trial_reads.py b/autogpt_platform/backend/backend/copilot/trial_reads.py
new file mode 100644
index 0000000000..5a922e57d6
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/trial_reads.py
@@ -0,0 +1,91 @@
+"""One subscription-trial read per request instead of one per check.
+
+The chat route's paywall and its rate-limit checks each read the user's
+trial, and every read is a database round trip (about 110 ms in prod). Inside
+``trial_reads_shared()`` those checks share one read per user; outside it,
+every call reads the database.
+
+The sharing is scoped to one block of one request, never across requests, so
+nothing can go stale: a trial's state is not expected to change between
+checks a few milliseconds apart. A task started inside the block inherits the
+block, but stops sharing when the block ends.
+"""
+
+import asyncio
+import contextvars
+import functools
+from collections.abc import Awaitable, Callable, Iterator
+from contextlib import contextmanager
+from typing import ParamSpec, TypeVar
+
+from backend.data.subscription_trial import TrialState
+
+_P = ParamSpec("_P")
+_R = TypeVar("_R")
+
+
+class _SharedReads:
+ """The reads a ``trial_reads_shared`` block shares while it is open."""
+
+ def __init__(self) -> None:
+ self.reads: dict[str, asyncio.Future[TrialState | None]] = {}
+ self.open = True
+
+
+_shared_reads: contextvars.ContextVar[_SharedReads | None] = contextvars.ContextVar(
+ "copilot_shared_trial_reads", default=None
+)
+
+
+@contextmanager
+def trial_reads_shared() -> Iterator[None]:
+ """Let the trial lookups inside the block share one read per user."""
+ shared = _SharedReads()
+ token = _shared_reads.set(shared)
+ try:
+ yield
+ finally:
+ # Tasks started in the block copied the context and still see
+ # ``shared``; closing it sends their later lookups to the database.
+ shared.open = False
+ _shared_reads.reset(token)
+
+
+def sharing_trial_reads(
+ handler: Callable[_P, Awaitable[_R]],
+) -> Callable[_P, Awaitable[_R]]:
+ """Run each call of *handler* inside ``trial_reads_shared()``.
+
+ For a request handler: the scope ends when it returns, before any response
+ is streamed. ``functools.wraps`` keeps the signature FastAPI resolves.
+ """
+
+ @functools.wraps(handler)
+ async def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> _R:
+ with trial_reads_shared():
+ return await handler(*args, **kwargs)
+
+ return wrapper
+
+
+async def shared_trial_read(
+ user_id: str, read: Callable[[], Awaitable[TrialState | None]]
+) -> TrialState | None:
+ """``read()`` the user's trial, or join the read already made in this block."""
+ shared = _shared_reads.get()
+ if shared is None or not shared.open:
+ return await read()
+ pending = shared.reads.get(user_id)
+ if pending is None:
+ pending = asyncio.ensure_future(read())
+ pending.add_done_callback(_retrieve_exception)
+ shared.reads[user_id] = pending
+ # Shielded: one caller giving up must not cancel the read the others wait on.
+ return await asyncio.shield(pending)
+
+
+def _retrieve_exception(read: asyncio.Future[TrialState | None]) -> None:
+ # If every caller gave up, nobody awaits a failed read; retrieving its
+ # error here keeps asyncio from logging it as never retrieved.
+ if not read.cancelled():
+ read.exception()
diff --git a/autogpt_platform/backend/backend/copilot/trial_reads_test.py b/autogpt_platform/backend/backend/copilot/trial_reads_test.py
new file mode 100644
index 0000000000..9f0608dec5
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/trial_reads_test.py
@@ -0,0 +1,208 @@
+"""One trial read per ``trial_reads_shared`` block, and none shared outside it."""
+
+import asyncio
+import inspect
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from backend.api.features.chat.routes import StreamChatRequest, stream_chat_post
+from backend.copilot.rate_limit import (
+ SubscriptionTier,
+ check_rate_limit,
+ enforce_payment_paywall,
+ get_global_rate_limits,
+)
+from backend.copilot.trial_reads import (
+ shared_trial_read,
+ sharing_trial_reads,
+ trial_reads_shared,
+)
+from backend.data.subscription_trial import TrialState
+
+
+def _trial(*, active: bool = True, cost: int = 0) -> MagicMock:
+ trial = MagicMock(spec=TrialState)
+ trial.active = active
+ trial.cost_microdollars = cost
+ trial.offer = MagicMock(
+ daily_cost_limit=1_000_000,
+ weekly_cost_limit=5_000_000,
+ total_cost_limit=10_000_000,
+ )
+ trial.ends_at = None
+ return trial
+
+
+async def test_outside_a_block_every_call_reads():
+ read = AsyncMock(return_value=_trial())
+ await shared_trial_read("u1", read)
+ await shared_trial_read("u1", read)
+ assert read.await_count == 2
+
+
+async def test_inside_a_block_a_user_is_read_once():
+ trial = _trial()
+ read = AsyncMock(return_value=trial)
+ other = AsyncMock(return_value=None)
+ with trial_reads_shared():
+ assert await shared_trial_read("u1", read) is trial
+ assert await shared_trial_read("u1", read) is trial
+ assert await shared_trial_read("u2", other) is None
+ assert read.await_count == 1
+ assert other.await_count == 1
+ await shared_trial_read("u1", read)
+ assert read.await_count == 2
+
+
+async def test_concurrent_checks_wait_on_the_same_read():
+ started = 0
+ release = asyncio.Event()
+
+ async def read() -> MagicMock:
+ nonlocal started
+ started += 1
+ await release.wait()
+ return _trial()
+
+ with trial_reads_shared():
+ waiters = [asyncio.create_task(shared_trial_read("u1", read)) for _ in range(3)]
+ await asyncio.sleep(0)
+ release.set()
+ results = await asyncio.gather(*waiters)
+ assert started == 1
+ assert results[0] is results[1] is results[2]
+
+
+async def test_one_caller_giving_up_does_not_cancel_the_shared_read():
+ release = asyncio.Event()
+ trial = _trial()
+
+ async def read() -> MagicMock:
+ await release.wait()
+ return trial
+
+ with trial_reads_shared():
+ first = asyncio.create_task(shared_trial_read("u1", read))
+ second = asyncio.create_task(shared_trial_read("u1", read))
+ await asyncio.sleep(0)
+ first.cancel()
+ release.set()
+ assert await second is trial
+ with pytest.raises(asyncio.CancelledError):
+ await first
+
+
+async def test_a_task_started_in_a_block_reads_afresh_once_it_ends():
+ read = AsyncMock(return_value=_trial())
+ block_ended = asyncio.Event()
+
+ async def background() -> None:
+ await shared_trial_read("u1", read)
+ await block_ended.wait()
+ await shared_trial_read("u1", read)
+
+ with trial_reads_shared():
+ task = asyncio.create_task(background())
+ await shared_trial_read("u1", read)
+ block_ended.set()
+ await task
+ assert read.await_count == 2
+
+
+async def test_a_wrapped_handler_shares_one_read_per_call():
+ read = AsyncMock(return_value=_trial())
+
+ async def handler(user_id: str, *, limit: int = 1) -> None:
+ await shared_trial_read(user_id, read)
+ await shared_trial_read(user_id, read)
+
+ wrapped = sharing_trial_reads(handler)
+ await wrapped("u1")
+ await wrapped("u1")
+
+ assert read.await_count == 2
+ assert inspect.signature(wrapped) == inspect.signature(handler)
+
+
+class _StopTheTurn(Exception):
+ pass
+
+
+async def test_the_stream_route_runs_its_checks_in_one_block():
+ read = AsyncMock(return_value=_trial())
+
+ async def first_checks(session_id: str, user_id: str) -> None:
+ await shared_trial_read(user_id, read)
+ await shared_trial_read(user_id, read)
+ raise _StopTheTurn
+
+ with (
+ patch(
+ "backend.api.features.chat.routes._validate_and_get_writable_session",
+ new=first_checks,
+ ),
+ pytest.raises(_StopTheTurn),
+ ):
+ await stream_chat_post(
+ "session-1", StreamChatRequest(message="hi"), "u1", MagicMock()
+ )
+
+ assert read.await_count == 1
+
+
+async def test_a_failed_read_fails_every_check_that_shares_it():
+ read = AsyncMock(side_effect=ConnectionError("db down"))
+ with trial_reads_shared():
+ for _ in range(2):
+ with pytest.raises(ConnectionError):
+ await shared_trial_read("u1", read)
+ assert read.await_count == 1
+
+
+async def test_the_chat_routes_trial_checks_read_the_trial_once():
+ """Paywall, limits and limit check each read a trial user's trial: four
+ database round trips before, one inside the route's block."""
+
+ async def run_checks() -> None:
+ await enforce_payment_paywall("trial-user")
+ daily, weekly, _ = await get_global_rate_limits("trial-user", 1, 1)
+ await check_rate_limit("trial-user", daily, weekly)
+
+ store = MagicMock()
+ store.get_subscription_trial = AsyncMock(return_value=_trial())
+ redis = MagicMock()
+ redis.get = AsyncMock(return_value=None)
+ with (
+ patch(
+ "backend.copilot.rate_limit._fetch_user_tier",
+ new=AsyncMock(return_value=SubscriptionTier.TRIAL),
+ ),
+ patch(
+ "backend.copilot.rate_limit._fetch_cost_limits_flag",
+ new=AsyncMock(return_value=None),
+ ),
+ patch("backend.copilot.rate_limit.credit_db", return_value=store),
+ patch(
+ "backend.copilot.rate_limit.get_redis_async",
+ new=AsyncMock(return_value=redis),
+ ),
+ ):
+ await run_checks()
+ unshared = store.get_subscription_trial.await_count
+ store.get_subscription_trial.reset_mock()
+ with trial_reads_shared():
+ await run_checks()
+ shared = store.get_subscription_trial.await_count
+
+ assert unshared == 4
+ assert shared == 1