:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-imports): retry modal billing report rate limits with backoff (#125271)
Tom Owers82122291e28a
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/modal/modal.py b/products/warehouse_sources/backend/temporal/data_imports/sources/modal/modal.pyindex 3dbbbf951401..706f74864759 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/modal/modal.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/modal/modal.py@@ -1,4 +1,4 @@-from collections.abc import Iterator+from collections.abc import Callable, Iteratorfrom contextlib import contextmanagerfrom datetime import UTC, datetime, timedeltafrom typing import TYPE_CHECKING@@ -11,11 +11,15 @@ from grpclib import GRPCError, Statusfrom modal._utils.async_utils import synchronizerfrom modal.client import _Clientfrom modal.environments import list_environments-from modal.exception import AuthError, NotFoundError, PermissionDeniedError+from modal.exception import AuthError, NotFoundError, PermissionDeniedError, ResourceExhaustedError+from modal.types import BillingReportItemfrom modal_proto import api_pb2from posthog.dataclasses import frozen+from products.warehouse_sources.backend.temporal.data_imports.sources.common.interruptible_wait import (+ interruptible_wait,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import schema_for_resourcefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponsefrom products.warehouse_sources.backend.temporal.data_imports.sources.modal.settings import (@@ -171,6 +175,39 @@ def validate_credentials(config: ModalSourceConfig) -> tuple[bool, str | None]:return True, None+# Modal's own gRPC client already retries DEADLINE_EXCEEDED/UNAVAILABLE/CANCELLED/INTERNAL/UNKNOWN+# transport errors (see modal._utils.grpc_utils.RETRYABLE_GRPC_STATUS_CODES), but deliberately+# excludes RESOURCE_EXHAUSTED, so a workspace-level rate limit on the billing-report RPC reaches us+# directly as ResourceExhaustedError. Riding it out here with a short backoff avoids failing the+# whole import activity, which would otherwise rebuild the gRPC client and resume from the last+# saved window anyway.+MAX_REPORT_RETRY_ATTEMPTS = 4+MAX_REPORT_BACKOFF_SECONDS = 30+++def _report_with_rate_limit_retry(+ workspace: "modal.Workspace",+ *,+ start: datetime,+ end: datetime,+ resolution: str,+ tag_names: list[str],+ safe_point: Callable[[], None],+) -> list[BillingReportItem]:+ attempt = 0+ while True:+ try:+ return workspace.billing.report(start=start, end=end, resolution=resolution, tag_names=tag_names)+ except ResourceExhaustedError:+ attempt += 1+ if attempt >= MAX_REPORT_RETRY_ATTEMPTS:+ raise+ # Every row before this window is already yielded and its cursor staged, so a worker+ # shutdown during the backoff can hand the run off from here without losing rows.+ if interruptible_wait(min(2 * attempt, MAX_REPORT_BACKOFF_SECONDS), safe_point=safe_point):+ raise++def modal_source(config: ModalSourceConfig,inputs: SourceInputs,@@ -216,8 +253,13 @@ def billing_report_source(workspace = modal.Workspace.from_context(client=client)while start < end:window_end = min(start + table.window_size, end)- report = workspace.billing.report(- start=start, end=window_end, resolution=table.resolution, tag_names=["*"]+ report = _report_with_rate_limit_retry(+ workspace,+ start=start,+ end=window_end,+ resolution=table.resolution,+ tag_names=["*"],+ safe_point=resumable_source_manager.safe_point,)ordered = sorted(report, key=lambda item: as_utc(item.interval_start))rows: list[dict[str, object]] = [diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/modal/tests/test_modal.py b/products/warehouse_sources/backend/temporal/data_imports/sources/modal/tests/test_modal.pyindex 120798e5b37e..9c8468abcf3f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/modal/tests/test_modal.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/modal/tests/test_modal.py@@ -19,6 +19,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.schfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponsefrom products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.modal import ModalSourceConfigfrom products.warehouse_sources.backend.temporal.data_imports.sources.modal.modal import (+ MAX_REPORT_RETRY_ATTEMPTS,ModalAuthenticationError,ModalPermissionError,ModalResumeConfig,@@ -195,6 +196,56 @@ def test_retry_resumes_after_saved_window(assert report.call_args_list[-1].kwargs["end"] == NOW+def test_rides_out_rate_limit_on_report(+ config: ModalSourceConfig,+ inputs: SourceInputs,+ manager: MagicMock,+ create_client: MagicMock,+ report: MagicMock,+) -> None:+ inputs.schema_name = "billing_report_hourly"+ inputs.db_incremental_field_last_value = "2026-03-15T10:00:00Z"+ rows = [billing_item(datetime(2026, 3, 15, 10, tzinfo=UTC))]+ report.side_effect = [ResourceExhaustedError("rate limit"), ResourceExhaustedError("rate limit"), rows]++ with patch(+ "products.warehouse_sources.backend.temporal.data_imports.sources.modal.modal.interruptible_wait",+ return_value=False,+ ) as wait:+ result = list(source_items(modal_source(config, inputs, manager)))++ # Two rate-limit rejections were retried in-process, then the same window was served — no data lost.+ assert report.call_count == 3+ assert [item["object_id"] for item in result[0]] == ["ap-test"]+ # Backoff grows per attempt per `min(2 * attempt, 30)`: 2s after the 1st rejection, 4s after the 2nd.+ assert [c.args for c in wait.call_args_list] == [(2,), (4,)]+ create_client.return_value.__exit__.assert_called_once()+++def test_persistent_rate_limit_is_reraised_for_temporal_to_retry(+ config: ModalSourceConfig,+ inputs: SourceInputs,+ manager: MagicMock,+ create_client: MagicMock,+ report: MagicMock,+) -> None:+ inputs.schema_name = "billing_report_hourly"+ inputs.db_incremental_field_last_value = "2026-03-15T10:00:00Z"+ report.side_effect = ResourceExhaustedError("rate limit")++ with patch(+ "products.warehouse_sources.backend.temporal.data_imports.sources.modal.modal.interruptible_wait",+ return_value=False,+ ) as wait:+ with pytest.raises(ResourceExhaustedError, match="rate limit"):+ list(source_items(modal_source(config, inputs, manager)))++ assert report.call_count == MAX_REPORT_RETRY_ATTEMPTS+ assert [c.args for c in wait.call_args_list] == [(2,), (4,), (6,)]+ manager.save_state.assert_not_called()+ create_client.return_value.__exit__.assert_called_once()++@pytest.mark.parametrize("during_creation", [False, True])@pytest.mark.parametrize(("error", "mapped_type", "message"),