Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-imports): retry modal billing report rate limits with backoff (#125271)

Tom Owers82122291e28a
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/modal/modal.py b/products/warehouse_sources/backend/temporal/data_imports/sources/modal/modal.py
index 3dbbbf951401..706f74864759 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/modal/modal.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/modal/modal.py
@@ -1,4 +1,4 @@
-from collections.abc import Iterator
+from collections.abc import Callable, Iterator
from contextlib import contextmanager
from datetime import UTC, datetime, timedelta
from typing import TYPE_CHECKING
@@ -11,11 +11,15 @@ from grpclib import GRPCError, Status
from modal._utils.async_utils import synchronizer
from modal.client import _Client
from modal.environments import list_environments
-from modal.exception import AuthError, NotFoundError, PermissionDeniedError
+from modal.exception import AuthError, NotFoundError, PermissionDeniedError, ResourceExhaustedError
+from modal.types import BillingReportItem
from modal_proto import api_pb2
from posthog.dataclasses import frozen
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.interruptible_wait import (
+ interruptible_wait,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import schema_for_resource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.modal.settings import (
@@ -171,6 +175,39 @@ def validate_credentials(config: ModalSourceConfig) -> tuple[bool, str | None]:
return True, None
+# Modal's own gRPC client already retries DEADLINE_EXCEEDED/UNAVAILABLE/CANCELLED/INTERNAL/UNKNOWN
+# transport errors (see modal._utils.grpc_utils.RETRYABLE_GRPC_STATUS_CODES), but deliberately
+# excludes RESOURCE_EXHAUSTED, so a workspace-level rate limit on the billing-report RPC reaches us
+# directly as ResourceExhaustedError. Riding it out here with a short backoff avoids failing the
+# whole import activity, which would otherwise rebuild the gRPC client and resume from the last
+# saved window anyway.
+MAX_REPORT_RETRY_ATTEMPTS = 4
+MAX_REPORT_BACKOFF_SECONDS = 30
+
+
+def _report_with_rate_limit_retry(
+ workspace: "modal.Workspace",
+ *,
+ start: datetime,
+ end: datetime,
+ resolution: str,
+ tag_names: list[str],
+ safe_point: Callable[[], None],
+) -> list[BillingReportItem]:
+ attempt = 0
+ while True:
+ try:
+ return workspace.billing.report(start=start, end=end, resolution=resolution, tag_names=tag_names)
+ except ResourceExhaustedError:
+ attempt += 1
+ if attempt >= MAX_REPORT_RETRY_ATTEMPTS:
+ raise
+ # Every row before this window is already yielded and its cursor staged, so a worker
+ # shutdown during the backoff can hand the run off from here without losing rows.
+ if interruptible_wait(min(2 * attempt, MAX_REPORT_BACKOFF_SECONDS), safe_point=safe_point):
+ raise
+
+
def modal_source(
config: ModalSourceConfig,
inputs: SourceInputs,
@@ -216,8 +253,13 @@ def billing_report_source(
workspace = modal.Workspace.from_context(client=client)
while start < end:
window_end = min(start + table.window_size, end)
- report = workspace.billing.report(
- start=start, end=window_end, resolution=table.resolution, tag_names=["*"]
+ report = _report_with_rate_limit_retry(
+ workspace,
+ start=start,
+ end=window_end,
+ resolution=table.resolution,
+ tag_names=["*"],
+ safe_point=resumable_source_manager.safe_point,
)
ordered = sorted(report, key=lambda item: as_utc(item.interval_start))
rows: list[dict[str, object]] = [
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/modal/tests/test_modal.py b/products/warehouse_sources/backend/temporal/data_imports/sources/modal/tests/test_modal.py
index 120798e5b37e..9c8468abcf3f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/modal/tests/test_modal.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/modal/tests/test_modal.py
@@ -19,6 +19,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.sch
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.modal import ModalSourceConfig
from products.warehouse_sources.backend.temporal.data_imports.sources.modal.modal import (
+ MAX_REPORT_RETRY_ATTEMPTS,
ModalAuthenticationError,
ModalPermissionError,
ModalResumeConfig,
@@ -195,6 +196,56 @@ def test_retry_resumes_after_saved_window(
assert report.call_args_list[-1].kwargs["end"] == NOW
+def test_rides_out_rate_limit_on_report(
+ config: ModalSourceConfig,
+ inputs: SourceInputs,
+ manager: MagicMock,
+ create_client: MagicMock,
+ report: MagicMock,
+) -> None:
+ inputs.schema_name = "billing_report_hourly"
+ inputs.db_incremental_field_last_value = "2026-03-15T10:00:00Z"
+ rows = [billing_item(datetime(2026, 3, 15, 10, tzinfo=UTC))]
+ report.side_effect = [ResourceExhaustedError("rate limit"), ResourceExhaustedError("rate limit"), rows]
+
+ with patch(
+ "products.warehouse_sources.backend.temporal.data_imports.sources.modal.modal.interruptible_wait",
+ return_value=False,
+ ) as wait:
+ result = list(source_items(modal_source(config, inputs, manager)))
+
+ # Two rate-limit rejections were retried in-process, then the same window was served — no data lost.
+ assert report.call_count == 3
+ assert [item["object_id"] for item in result[0]] == ["ap-test"]
+ # Backoff grows per attempt per `min(2 * attempt, 30)`: 2s after the 1st rejection, 4s after the 2nd.
+ assert [c.args for c in wait.call_args_list] == [(2,), (4,)]
+ create_client.return_value.__exit__.assert_called_once()
+
+
+def test_persistent_rate_limit_is_reraised_for_temporal_to_retry(
+ config: ModalSourceConfig,
+ inputs: SourceInputs,
+ manager: MagicMock,
+ create_client: MagicMock,
+ report: MagicMock,
+) -> None:
+ inputs.schema_name = "billing_report_hourly"
+ inputs.db_incremental_field_last_value = "2026-03-15T10:00:00Z"
+ report.side_effect = ResourceExhaustedError("rate limit")
+
+ with patch(
+ "products.warehouse_sources.backend.temporal.data_imports.sources.modal.modal.interruptible_wait",
+ return_value=False,
+ ) as wait:
+ with pytest.raises(ResourceExhaustedError, match="rate limit"):
+ list(source_items(modal_source(config, inputs, manager)))
+
+ assert report.call_count == MAX_REPORT_RETRY_ATTEMPTS
+ assert [c.args for c in wait.call_args_list] == [(2,), (4,), (6,)]
+ manager.save_state.assert_not_called()
+ create_client.return_value.__exit__.assert_called_once()
+
+
@pytest.mark.parametrize("during_creation", [False, True])
@pytest.mark.parametrize(
("error", "mapped_type", "message"),