Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(billing): return 429 when billing rate limits a request (#125460)

Roy Cohenfd37495c55a2
diff --git a/ee/api/test/test_billing.py b/ee/api/test/test_billing.py
index 13284b0e790c..4a4fb442b998 100644
--- a/ee/api/test/test_billing.py
+++ b/ee/api/test/test_billing.py
@@ -2881,6 +2881,46 @@ class TestBillingUsageAndSpendAPI(APILicensedTest):
self.assertNotIn("start_date is invalid", response.content.decode())
+def _billing_rate_limited_response() -> Response:
+ response = Response()
+ response.status_code = status.HTTP_429_TOO_MANY_REQUESTS
+ response.headers["Retry-After"] = "30"
+ response._content = json.dumps({"detail": "Request was throttled. Expected available in 30 seconds."}).encode()
+ response.raw = MagicMock()
+ return response
+
+
+@patch(
+ "ee.billing.billing_manager.http_session.get", side_effect=lambda *args, **kwargs: _billing_rate_limited_response()
+)
+class TestBillingRateLimitedAPI(APILicensedTest):
+ def setUp(self):
+ super().setUp()
+ self.organization_membership.level = OrganizationMembership.Level.ADMIN
+ self.organization_membership.save()
+
+ def _assert_rate_limited(self, response: Any) -> None:
+ self.assertEqual(response.status_code, status.HTTP_429_TOO_MANY_REQUESTS)
+ self.assertEqual(response.json()["code"], "billing_rate_limited")
+ self.assertEqual(response.headers["Retry-After"], "30")
+
+ def test_billing_list_returns_429(self, _mock_get):
+ self._assert_rate_limited(self.client.get("/api/billing/"))
+
+ def test_billing_usage_returns_429(self, _mock_get):
+ self._assert_rate_limited(self.client.get("/api/billing/usage/?start_date=2025-01-01"))
+
+ def test_billing_invoices_returns_429(self, _mock_get):
+ self._assert_rate_limited(self.client.get("/api/billing/get_invoices"))
+
+ def test_billing_429_is_not_captured(self, _mock_get):
+ with patch("posthog.exceptions.capture_exception", return_value=None) as mock_capture_exception:
+ response = self.client.get("/api/billing/")
+
+ self.assertEqual(response.status_code, status.HTTP_429_TOO_MANY_REQUESTS)
+ mock_capture_exception.assert_not_called()
+
+
class TestBillingPeriodAPI(APILicensedTest):
def test_member_can_read_synced_organization_period(self):
self.organization_membership.level = OrganizationMembership.Level.MEMBER
diff --git a/ee/billing/billing_manager.py b/ee/billing/billing_manager.py
index 885e3e6f8a7b..f47fd0952179 100644
--- a/ee/billing/billing_manager.py
+++ b/ee/billing/billing_manager.py
@@ -1,5 +1,6 @@
import hmac
import json
+import math
import time
import hashlib
from datetime import UTC, datetime, timedelta
@@ -20,7 +21,7 @@ import jwt
import requests
import structlog
from requests import JSONDecodeError
-from rest_framework.exceptions import NotAuthenticated, NotFound, PermissionDenied
+from rest_framework.exceptions import NotAuthenticated, NotFound, PermissionDenied, Throttled
from posthog.cloud_utils import get_cached_instance_license
from posthog.dataclasses import frozen
@@ -346,7 +347,8 @@ def _raise_for_organization_error(res: requests.Response, *, map_not_found: bool
class BillingServiceResponseError(Exception):
- """Billing answered with a status code the caller does not accept."""
+ """Billing answered with a status code the caller does not accept, other than 429,
+ which raises BillingServiceRateLimited."""
def __init__(self, status_code: int, body: Any) -> None:
# The message and the body keep the positions callers already read: see
@@ -356,8 +358,32 @@ class BillingServiceResponseError(Exception):
self.body = body
+class BillingServiceRateLimited(Throttled):
+ # DRF sets this in Throttled.__init__ and its handler turns it into Retry-After; the stubs omit it.
+ wait: Optional[float]
+
+ default_code = "billing_rate_limited"
+ default_detail = "Billing is receiving too many requests for this organization. Try again shortly."
+
+
+def _retry_after_seconds(res: requests.Response) -> Optional[float]:
+ try:
+ seconds = float(res.headers.get("Retry-After", ""))
+ except ValueError:
+ return None
+ # DRF rounds the wait up with math.ceil, which raises on nan and infinity.
+ return seconds if math.isfinite(seconds) and seconds >= 0 else None
+
+
def handle_billing_service_error(res: requests.Response, valid_codes=(200, 201, 404, 401)) -> None:
if res.status_code not in valid_codes:
+ if res.status_code == 429:
+ wait = _retry_after_seconds(res)
+ # A streamed response (see _get_csv) keeps its pooled connection until it is closed.
+ res.close()
+ # Keyword arguments only, so args stays empty: the legacy handlers in ee/api/billing.py turn an
+ # exception with more than two args into a 400.
+ raise BillingServiceRateLimited(wait=wait)
logger.error(f"Billing service returned bad status code: {res.status_code}, body: {res.text}")
try:
body: Any = res.json()
diff --git a/ee/billing/test/test_billing_manager.py b/ee/billing/test/test_billing_manager.py
index fedc29a1b18b..63e16e41e63a 100644
--- a/ee/billing/test/test_billing_manager.py
+++ b/ee/billing/test/test_billing_manager.py
@@ -32,6 +32,7 @@ from ee.billing.billing_manager import (
BILLING_PROVIDER_WEBHOOK_SIGNATURE_VERSION,
BILLING_PROVIDER_WEBHOOK_TIMESTAMP_HEADER,
BillingManager,
+ BillingServiceRateLimited,
BillingServiceResponseError,
FundingStatusUnavailable,
OrganizationFundingStatus,
@@ -40,6 +41,7 @@ from ee.billing.billing_manager import (
_get_user_organization_role,
_parse_funding_status,
build_billing_token,
+ handle_billing_service_error,
http_session,
)
from ee.billing.billing_types import BillingProvider, BillingStatus, Product
@@ -984,6 +986,58 @@ class TestBillingSession(SimpleTestCase):
self.assertEqual(len(http_session.cookies), 0)
+def _response(status_code: int, headers: dict[str, str] | None = None, body: Any = None) -> requests.Response:
+ response = requests.Response()
+ response.status_code = status_code
+ response.headers.update(headers or {})
+ response._content = b"" if body is None else json.dumps(body).encode()
+ response.raw = MagicMock()
+ return response
+
+
+class TestHandleBillingServiceError(SimpleTestCase):
+ def test_handle_billing_service_error_raises_rate_limited_on_429(self):
+ res = _response(429, headers={"Retry-After": "30"}, body={"detail": "x"})
+
+ with self.assertRaises(BillingServiceRateLimited) as context:
+ handle_billing_service_error(res)
+
+ assert context.exception.status_code == 429
+ assert context.exception.wait == 30
+ assert context.exception.args == ()
+
+ def test_rate_limited_response_is_closed(self):
+ res = _response(429, headers={"Retry-After": "30"})
+
+ with patch.object(res, "close", wraps=res.close) as close, self.assertRaises(BillingServiceRateLimited):
+ handle_billing_service_error(res)
+
+ close.assert_called_once()
+
+ @parameterized.expand(
+ [
+ ("missing", {}),
+ ("not_a_number", {"Retry-After": "soon"}),
+ ("nan", {"Retry-After": "nan"}),
+ ("negative", {"Retry-After": "-5"}),
+ ]
+ )
+ def test_rate_limited_without_retry_after_has_no_wait(self, _name, headers):
+ with self.assertRaises(BillingServiceRateLimited) as context:
+ handle_billing_service_error(_response(429, headers=headers))
+
+ assert context.exception.wait is None
+
+ def test_other_bad_statuses_still_raise_response_error(self):
+ res = _response(500, body={"detail": "boom"})
+
+ with self.assertRaises(BillingServiceResponseError) as context:
+ handle_billing_service_error(res)
+
+ assert context.exception.status_code == 500
+ assert context.exception.body == {"detail": "boom"}
+
+
class TestBillingProviderWebhookSigning(SimpleTestCase):
def setUp(self):
self.license = SimpleNamespace(key="license_id::license_secret")
diff --git a/ee/partners/stripe/api/provisioning/billing.py b/ee/partners/stripe/api/provisioning/billing.py
index 2b65c02b3157..fae58385e771 100644
--- a/ee/partners/stripe/api/provisioning/billing.py
+++ b/ee/partners/stripe/api/provisioning/billing.py
@@ -20,6 +20,7 @@ from posthog.models.user import User
from ee.billing.billing_manager import (
BillingManagedByPartnerError,
BillingManager,
+ BillingServiceRateLimited,
BillingServiceResponseError,
build_billing_token,
)
@@ -74,6 +75,9 @@ def _activate_billing_with_spt(team: Team, user: User, spt_token: str) -> bool:
except BillingManagedByPartnerError:
logger.info("stripe_provisioning.spt_billing_refused_partner_managed", **log_context)
return False
+ except BillingServiceRateLimited:
+ logger.warning("stripe_provisioning.spt_billing_rate_limited", **log_context)
+ return False
except BillingServiceResponseError as error:
capture_exception(
Exception(f"Billing SPT activation failed: {error.status_code}"),
diff --git a/ee/partners/stripe/api/provisioning/test/test_resources.py b/ee/partners/stripe/api/provisioning/test/test_resources.py
index 8001563c11a4..8f482ea003ae 100644
--- a/ee/partners/stripe/api/provisioning/test/test_resources.py
+++ b/ee/partners/stripe/api/provisioning/test/test_resources.py
@@ -1,6 +1,6 @@
from datetime import timedelta
-from unittest.mock import patch
+from unittest.mock import MagicMock, patch
from django.utils import timezone
@@ -171,6 +171,41 @@ class TestResources(StripeProvisioningTestBase):
"error": {"code": "requires_payment_credentials", "message": "Billing activation failed"},
}
+ def test_rate_limited_spt_activation_logs_a_warning_instead_of_an_error(self) -> None:
+ token = self._get_bearer_token()
+ billing_response = requests.Response()
+ billing_response.status_code = 429
+ billing_response.headers["Retry-After"] = "30"
+ billing_response.raw = MagicMock()
+
+ with (
+ patch(
+ "ee.partners.stripe.api.provisioning.billing.get_cached_instance_license",
+ return_value=License(key="12345::67890"),
+ ),
+ patch("ee.partners.stripe.api.provisioning.billing._team_has_active_billing", return_value=False),
+ patch("ee.billing.billing_manager.http_session.post", return_value=billing_response),
+ patch("ee.partners.stripe.api.provisioning.billing.capture_exception") as capture,
+ patch("ee.partners.stripe.api.provisioning.billing.logger") as logger,
+ ):
+ res = self._post_signed_with_bearer(
+ RESOURCES_URL,
+ data={
+ "service_id": "analytics",
+ "payment_credentials": {"type": "stripe_payment_token", "stripe_payment_token": "spt_1"},
+ },
+ token=token,
+ )
+
+ assert res.status_code == 400
+ assert res.json()["error"]["message"] == "Billing activation failed"
+ capture.assert_not_called()
+ logger.warning.assert_called_once_with(
+ "stripe_provisioning.spt_billing_rate_limited",
+ team_id=self.team.id,
+ org_id=str(self.team.organization_id),
+ )
+
@parameterized.expand(
[
("partner_billed", None, 400, []),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/row_tracking.py b/products/warehouse_sources/backend/temporal/data_imports/row_tracking.py
index 94cc495db019..804f7e314d60 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/row_tracking.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/row_tracking.py
@@ -287,7 +287,8 @@ def _rows_synced_in_billing_period(
# Billing answers a request it could not finish in time with a 408, and a request it could not
# serve with a 5xx. Both clear on a retry, and the check runs once per sync, so a few seconds of
# retry is cheaper than a sync that skips the limit.
-_TRANSIENT_BILLING_STATUS_CODES = frozenset({408, 429, 500, 502, 503, 504})
+# A 429 is not retried: a retry within seconds lands in the same rate-limit window and only adds counted requests.
+_TRANSIENT_BILLING_STATUS_CODES = frozenset({408, 500, 502, 503, 504})
def _is_transient_billing_error(error: BaseException) -> bool:
@@ -435,6 +436,11 @@ async def will_hit_billing_limit(team_id: int, source: "ExternalDataSource", log
return False
except Exception as e:
+ if getattr(e, "status_code", None) == 429:
+ await logger.awarning(f"BillingLimits: billing service rate limited the check, failing open: {e}")
+
+ return False
+
if _is_transient_billing_error(e):
# Billing refused with a status that clears on its own, and the retries above are
# spent. The check already fails open, so this is infrastructure noise rather than a
diff --git a/products/warehouse_sources/backend/temporal/data_imports/tests/test_row_tracking.py b/products/warehouse_sources/backend/temporal/data_imports/tests/test_row_tracking.py
index 1174263dbbd6..d46030feacf4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/tests/test_row_tracking.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/tests/test_row_tracking.py
@@ -567,6 +567,31 @@ class TestRowTracking(BaseTest):
assert mock_get_billing.call_count == 3
mock_capture_exception.assert_not_called()
+ @pytest.mark.asyncio
+ async def test_row_tracking_does_not_retry_a_rate_limited_billing_check(self):
+ from ee.billing.billing_manager import handle_billing_service_error
+
+ source = await self._create_source()
+
+ def _raise_billing_rate_limited(*args, **kwargs):
+ response = requests.Response()
+ response.status_code = 429
+ response.raw = mock.MagicMock()
+ handle_billing_service_error(response)
+
+ with (
+ mock.patch("ee.billing.billing_manager.BillingManager.get_billing") as mock_get_billing,
+ mock.patch(
+ "products.warehouse_sources.backend.temporal.data_imports.row_tracking.capture_exception"
+ ) as mock_capture_exception,
+ ):
+ mock_get_billing.side_effect = _raise_billing_rate_limited
+
+ assert await self._run(source, 10) is False
+
+ assert mock_get_billing.call_count == 1
+ mock_capture_exception.assert_not_called()
+
@pytest.mark.asyncio
async def test_row_tracking_fails_open_on_request_error_without_capturing_exception(self):
# A network blip (e.g. a proxy timeout) reaching the billing service is a transient