:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(billing): return 429 when billing rate limits a request (#125460)
Roy Cohenfd37495c55a2
diff --git a/ee/api/test/test_billing.py b/ee/api/test/test_billing.pyindex 13284b0e790c..4a4fb442b998 100644--- a/ee/api/test/test_billing.py+++ b/ee/api/test/test_billing.py@@ -2881,6 +2881,46 @@ class TestBillingUsageAndSpendAPI(APILicensedTest):self.assertNotIn("start_date is invalid", response.content.decode())+def _billing_rate_limited_response() -> Response:+ response = Response()+ response.status_code = status.HTTP_429_TOO_MANY_REQUESTS+ response.headers["Retry-After"] = "30"+ response._content = json.dumps({"detail": "Request was throttled. Expected available in 30 seconds."}).encode()+ response.raw = MagicMock()+ return response+++@patch(+ "ee.billing.billing_manager.http_session.get", side_effect=lambda *args, **kwargs: _billing_rate_limited_response()+)+class TestBillingRateLimitedAPI(APILicensedTest):+ def setUp(self):+ super().setUp()+ self.organization_membership.level = OrganizationMembership.Level.ADMIN+ self.organization_membership.save()++ def _assert_rate_limited(self, response: Any) -> None:+ self.assertEqual(response.status_code, status.HTTP_429_TOO_MANY_REQUESTS)+ self.assertEqual(response.json()["code"], "billing_rate_limited")+ self.assertEqual(response.headers["Retry-After"], "30")++ def test_billing_list_returns_429(self, _mock_get):+ self._assert_rate_limited(self.client.get("/api/billing/"))++ def test_billing_usage_returns_429(self, _mock_get):+ self._assert_rate_limited(self.client.get("/api/billing/usage/?start_date=2025-01-01"))++ def test_billing_invoices_returns_429(self, _mock_get):+ self._assert_rate_limited(self.client.get("/api/billing/get_invoices"))++ def test_billing_429_is_not_captured(self, _mock_get):+ with patch("posthog.exceptions.capture_exception", return_value=None) as mock_capture_exception:+ response = self.client.get("/api/billing/")++ self.assertEqual(response.status_code, status.HTTP_429_TOO_MANY_REQUESTS)+ mock_capture_exception.assert_not_called()++class TestBillingPeriodAPI(APILicensedTest):def test_member_can_read_synced_organization_period(self):self.organization_membership.level = OrganizationMembership.Level.MEMBERdiff --git a/ee/billing/billing_manager.py b/ee/billing/billing_manager.pyindex 885e3e6f8a7b..f47fd0952179 100644--- a/ee/billing/billing_manager.py+++ b/ee/billing/billing_manager.py@@ -1,5 +1,6 @@import hmacimport json+import mathimport timeimport hashlibfrom datetime import UTC, datetime, timedelta@@ -20,7 +21,7 @@ import jwtimport requestsimport structlogfrom requests import JSONDecodeError-from rest_framework.exceptions import NotAuthenticated, NotFound, PermissionDenied+from rest_framework.exceptions import NotAuthenticated, NotFound, PermissionDenied, Throttledfrom posthog.cloud_utils import get_cached_instance_licensefrom posthog.dataclasses import frozen@@ -346,7 +347,8 @@ def _raise_for_organization_error(res: requests.Response, *, map_not_found: boolclass BillingServiceResponseError(Exception):- """Billing answered with a status code the caller does not accept."""+ """Billing answered with a status code the caller does not accept, other than 429,+ which raises BillingServiceRateLimited."""def __init__(self, status_code: int, body: Any) -> None:# The message and the body keep the positions callers already read: see@@ -356,8 +358,32 @@ class BillingServiceResponseError(Exception):self.body = body+class BillingServiceRateLimited(Throttled):+ # DRF sets this in Throttled.__init__ and its handler turns it into Retry-After; the stubs omit it.+ wait: Optional[float]++ default_code = "billing_rate_limited"+ default_detail = "Billing is receiving too many requests for this organization. Try again shortly."+++def _retry_after_seconds(res: requests.Response) -> Optional[float]:+ try:+ seconds = float(res.headers.get("Retry-After", ""))+ except ValueError:+ return None+ # DRF rounds the wait up with math.ceil, which raises on nan and infinity.+ return seconds if math.isfinite(seconds) and seconds >= 0 else None++def handle_billing_service_error(res: requests.Response, valid_codes=(200, 201, 404, 401)) -> None:if res.status_code not in valid_codes:+ if res.status_code == 429:+ wait = _retry_after_seconds(res)+ # A streamed response (see _get_csv) keeps its pooled connection until it is closed.+ res.close()+ # Keyword arguments only, so args stays empty: the legacy handlers in ee/api/billing.py turn an+ # exception with more than two args into a 400.+ raise BillingServiceRateLimited(wait=wait)logger.error(f"Billing service returned bad status code: {res.status_code}, body: {res.text}")try:body: Any = res.json()diff --git a/ee/billing/test/test_billing_manager.py b/ee/billing/test/test_billing_manager.pyindex fedc29a1b18b..63e16e41e63a 100644--- a/ee/billing/test/test_billing_manager.py+++ b/ee/billing/test/test_billing_manager.py@@ -32,6 +32,7 @@ from ee.billing.billing_manager import (BILLING_PROVIDER_WEBHOOK_SIGNATURE_VERSION,BILLING_PROVIDER_WEBHOOK_TIMESTAMP_HEADER,BillingManager,+ BillingServiceRateLimited,BillingServiceResponseError,FundingStatusUnavailable,OrganizationFundingStatus,@@ -40,6 +41,7 @@ from ee.billing.billing_manager import (_get_user_organization_role,_parse_funding_status,build_billing_token,+ handle_billing_service_error,http_session,)from ee.billing.billing_types import BillingProvider, BillingStatus, Product@@ -984,6 +986,58 @@ class TestBillingSession(SimpleTestCase):self.assertEqual(len(http_session.cookies), 0)+def _response(status_code: int, headers: dict[str, str] | None = None, body: Any = None) -> requests.Response:+ response = requests.Response()+ response.status_code = status_code+ response.headers.update(headers or {})+ response._content = b"" if body is None else json.dumps(body).encode()+ response.raw = MagicMock()+ return response+++class TestHandleBillingServiceError(SimpleTestCase):+ def test_handle_billing_service_error_raises_rate_limited_on_429(self):+ res = _response(429, headers={"Retry-After": "30"}, body={"detail": "x"})++ with self.assertRaises(BillingServiceRateLimited) as context:+ handle_billing_service_error(res)++ assert context.exception.status_code == 429+ assert context.exception.wait == 30+ assert context.exception.args == ()++ def test_rate_limited_response_is_closed(self):+ res = _response(429, headers={"Retry-After": "30"})++ with patch.object(res, "close", wraps=res.close) as close, self.assertRaises(BillingServiceRateLimited):+ handle_billing_service_error(res)++ close.assert_called_once()++ @parameterized.expand(+ [+ ("missing", {}),+ ("not_a_number", {"Retry-After": "soon"}),+ ("nan", {"Retry-After": "nan"}),+ ("negative", {"Retry-After": "-5"}),+ ]+ )+ def test_rate_limited_without_retry_after_has_no_wait(self, _name, headers):+ with self.assertRaises(BillingServiceRateLimited) as context:+ handle_billing_service_error(_response(429, headers=headers))++ assert context.exception.wait is None++ def test_other_bad_statuses_still_raise_response_error(self):+ res = _response(500, body={"detail": "boom"})++ with self.assertRaises(BillingServiceResponseError) as context:+ handle_billing_service_error(res)++ assert context.exception.status_code == 500+ assert context.exception.body == {"detail": "boom"}++class TestBillingProviderWebhookSigning(SimpleTestCase):def setUp(self):self.license = SimpleNamespace(key="license_id::license_secret")diff --git a/ee/partners/stripe/api/provisioning/billing.py b/ee/partners/stripe/api/provisioning/billing.pyindex 2b65c02b3157..fae58385e771 100644--- a/ee/partners/stripe/api/provisioning/billing.py+++ b/ee/partners/stripe/api/provisioning/billing.py@@ -20,6 +20,7 @@ from posthog.models.user import Userfrom ee.billing.billing_manager import (BillingManagedByPartnerError,BillingManager,+ BillingServiceRateLimited,BillingServiceResponseError,build_billing_token,)@@ -74,6 +75,9 @@ def _activate_billing_with_spt(team: Team, user: User, spt_token: str) -> bool:except BillingManagedByPartnerError:logger.info("stripe_provisioning.spt_billing_refused_partner_managed", **log_context)return False+ except BillingServiceRateLimited:+ logger.warning("stripe_provisioning.spt_billing_rate_limited", **log_context)+ return Falseexcept BillingServiceResponseError as error:capture_exception(Exception(f"Billing SPT activation failed: {error.status_code}"),diff --git a/ee/partners/stripe/api/provisioning/test/test_resources.py b/ee/partners/stripe/api/provisioning/test/test_resources.pyindex 8001563c11a4..8f482ea003ae 100644--- a/ee/partners/stripe/api/provisioning/test/test_resources.py+++ b/ee/partners/stripe/api/provisioning/test/test_resources.py@@ -1,6 +1,6 @@from datetime import timedelta-from unittest.mock import patch+from unittest.mock import MagicMock, patchfrom django.utils import timezone@@ -171,6 +171,41 @@ class TestResources(StripeProvisioningTestBase):"error": {"code": "requires_payment_credentials", "message": "Billing activation failed"},}+ def test_rate_limited_spt_activation_logs_a_warning_instead_of_an_error(self) -> None:+ token = self._get_bearer_token()+ billing_response = requests.Response()+ billing_response.status_code = 429+ billing_response.headers["Retry-After"] = "30"+ billing_response.raw = MagicMock()++ with (+ patch(+ "ee.partners.stripe.api.provisioning.billing.get_cached_instance_license",+ return_value=License(key="12345::67890"),+ ),+ patch("ee.partners.stripe.api.provisioning.billing._team_has_active_billing", return_value=False),+ patch("ee.billing.billing_manager.http_session.post", return_value=billing_response),+ patch("ee.partners.stripe.api.provisioning.billing.capture_exception") as capture,+ patch("ee.partners.stripe.api.provisioning.billing.logger") as logger,+ ):+ res = self._post_signed_with_bearer(+ RESOURCES_URL,+ data={+ "service_id": "analytics",+ "payment_credentials": {"type": "stripe_payment_token", "stripe_payment_token": "spt_1"},+ },+ token=token,+ )++ assert res.status_code == 400+ assert res.json()["error"]["message"] == "Billing activation failed"+ capture.assert_not_called()+ logger.warning.assert_called_once_with(+ "stripe_provisioning.spt_billing_rate_limited",+ team_id=self.team.id,+ org_id=str(self.team.organization_id),+ )+@parameterized.expand([("partner_billed", None, 400, []),diff --git a/products/warehouse_sources/backend/temporal/data_imports/row_tracking.py b/products/warehouse_sources/backend/temporal/data_imports/row_tracking.pyindex 94cc495db019..804f7e314d60 100644--- a/products/warehouse_sources/backend/temporal/data_imports/row_tracking.py+++ b/products/warehouse_sources/backend/temporal/data_imports/row_tracking.py@@ -287,7 +287,8 @@ def _rows_synced_in_billing_period(# Billing answers a request it could not finish in time with a 408, and a request it could not# serve with a 5xx. Both clear on a retry, and the check runs once per sync, so a few seconds of# retry is cheaper than a sync that skips the limit.-_TRANSIENT_BILLING_STATUS_CODES = frozenset({408, 429, 500, 502, 503, 504})+# A 429 is not retried: a retry within seconds lands in the same rate-limit window and only adds counted requests.+_TRANSIENT_BILLING_STATUS_CODES = frozenset({408, 500, 502, 503, 504})def _is_transient_billing_error(error: BaseException) -> bool:@@ -435,6 +436,11 @@ async def will_hit_billing_limit(team_id: int, source: "ExternalDataSource", logreturn Falseexcept Exception as e:+ if getattr(e, "status_code", None) == 429:+ await logger.awarning(f"BillingLimits: billing service rate limited the check, failing open: {e}")++ return False+if _is_transient_billing_error(e):# Billing refused with a status that clears on its own, and the retries above are# spent. The check already fails open, so this is infrastructure noise rather than adiff --git a/products/warehouse_sources/backend/temporal/data_imports/tests/test_row_tracking.py b/products/warehouse_sources/backend/temporal/data_imports/tests/test_row_tracking.pyindex 1174263dbbd6..d46030feacf4 100644--- a/products/warehouse_sources/backend/temporal/data_imports/tests/test_row_tracking.py+++ b/products/warehouse_sources/backend/temporal/data_imports/tests/test_row_tracking.py@@ -567,6 +567,31 @@ class TestRowTracking(BaseTest):assert mock_get_billing.call_count == 3mock_capture_exception.assert_not_called()+ @pytest.mark.asyncio+ async def test_row_tracking_does_not_retry_a_rate_limited_billing_check(self):+ from ee.billing.billing_manager import handle_billing_service_error++ source = await self._create_source()++ def _raise_billing_rate_limited(*args, **kwargs):+ response = requests.Response()+ response.status_code = 429+ response.raw = mock.MagicMock()+ handle_billing_service_error(response)++ with (+ mock.patch("ee.billing.billing_manager.BillingManager.get_billing") as mock_get_billing,+ mock.patch(+ "products.warehouse_sources.backend.temporal.data_imports.row_tracking.capture_exception"+ ) as mock_capture_exception,+ ):+ mock_get_billing.side_effect = _raise_billing_rate_limited++ assert await self._run(source, 10) is False++ assert mock_get_billing.call_count == 1+ mock_capture_exception.assert_not_called()+@pytest.mark.asyncioasync def test_row_tracking_fails_open_on_request_error_without_capturing_exception(self):# A network blip (e.g. a proxy timeout) reaching the billing service is a transient