Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(anthropic): wait out a long retry-after on claude code report syncs (#119255)

posthog[bot]6b7088c671df
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/anthropic/anthropic.py b/products/warehouse_sources/backend/temporal/data_imports/sources/anthropic/anthropic.py
index ce69085f16da..0b9d080014de 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/anthropic/anthropic.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/anthropic/anthropic.py
@@ -48,11 +48,12 @@ ENTITY_PAGE_SIZE = 1000
# a transient blip without letting a genuinely broken endpoint retry for long.
MAX_RETRY_ATTEMPTS = 8
# The usage/cost/analytics report endpoints share one organization-level rate limit on Anthropic's
-# Admin API, which answers 429 with no `Retry-After`. The `anthropic-ratelimit-requests-reset`
-# instant is often already stale when we read it, so the client falls back to exponential backoff.
-# A per-minute limit needs a budget that outlasts the window, so give the report endpoints more
-# attempts and a higher backoff ceiling than the entity lists — each wait still stays capped, so a
-# sync never stalls indefinitely.
+# Admin API. Some of them, such as the Claude Code usage report, answer 429 with a `Retry-After` of
+# up to an hour, which is the time until the budget is full again. The budget refills continuously,
+# so the report clients cap each server delay at the shared limit and wait it out, instead of
+# failing the request at once. A 429 without a usable delay falls back to exponential backoff.
+# Give the report endpoints more attempts and a higher backoff ceiling than the entity lists. Each
+# wait stays capped, so a sync never stalls indefinitely.
REPORT_MAX_RETRY_ATTEMPTS = 12
REPORT_RETRY_BACKOFF_MAX_SECONDS = 300.0
# The waits of the report attempts above add up to about 1,100 seconds, which is more than the
@@ -852,6 +853,7 @@ def anthropic_source(
if is_report_endpoint:
client_config["retry_backoff_max_seconds"] = REPORT_RETRY_BACKOFF_MAX_SECONDS
client_config["retry_budget_seconds"] = REPORT_RETRY_BUDGET_SECONDS
+ client_config["cap_retry_after"] = True
resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/anthropic/tests/test_anthropic.py b/products/warehouse_sources/backend/temporal/data_imports/sources/anthropic/tests/test_anthropic.py
index 49e3d211216c..623bf837645f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/anthropic/tests/test_anthropic.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/anthropic/tests/test_anthropic.py
@@ -366,6 +366,20 @@ class TestClaudeCodeDayFanOut:
assert result.params("page") == ["P2"]
assert result.params("starting_at") == [today.isoformat()]
+ def test_waits_out_a_rate_limit_that_asks_for_an_hour(self) -> None:
+ today = datetime.now(UTC).date()
+ result = _driver().run(
+ "claude_code_analytics",
+ [
+ ScriptedResponse(status=429, headers={"Retry-After": "3600"}),
+ _cc_page([_cc_record()], has_more=False, next_page=None),
+ ],
+ incremental_field="date",
+ db_incremental_field_last_value=_midnight(today),
+ )
+ assert result.raised is None
+ assert len(result.rows) == 1
+
class TestRetiredEndpoint:
def test_a_dropped_endpoint_fails_non_retryably(self) -> None:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/__init__.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/__init__.py
index e11fe8cf0e26..868d953677e4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/__init__.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/__init__.py
@@ -360,6 +360,7 @@ def create_resources(
allow_redirects=client_config.get("allow_redirects", True),
request_timeout=client_config.get("request_timeout"),
capture=client_config.get("capture", True),
+ cap_retry_after=client_config.get("cap_retry_after", False),
**_retry_limit_overrides(client_config),
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/rest_client.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/rest_client.py
index b7f1f0287cad..5cb6aefb6d26 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/rest_client.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/rest_client.py
@@ -237,8 +237,7 @@ def _seconds_until_anthropic_replenish(value: str) -> Optional[float]:
# the parser that turns its value into "seconds from now until the budget replenishes".
_RATE_LIMIT_RESET_HEADERS: tuple[tuple[str, Callable[[str], Optional[float]]], ...] = (
# Anthropic rate limits per organization and documents this RFC 3339 instant as when the request
- # budget replenishes. Its Admin API (the usage/cost report endpoints an Anthropic source pages
- # through) answers 429 without a ``Retry-After``, so this is the only delay it advertises.
+ # budget replenishes. Some Admin API report endpoints send it on a 429 without a ``Retry-After``.
("anthropic-ratelimit-requests-reset", _seconds_until_anthropic_replenish),
# Sentry signals its rate-limit window with a UNIX epoch timestamp rather than ``Retry-After``,
# and Sentry's flat / fan-out endpoints (e.g. ``project_users``) sync through this client too.
@@ -290,11 +289,15 @@ def _parse_retry_after(response: Response) -> Optional[float]:
return None
+def _retry_after_limit(client: Any) -> float:
+ limit = getattr(client, "_retry_after_max", None)
+ return max_retry_after_seconds() if limit is None else limit
+
+
def _retry_after_exceeds_limit(client: Any, exc: Optional[BaseException]) -> bool:
if not isinstance(exc, RESTClientRetryableError) or exc.retry_after is None:
return False
- limit = getattr(client, "_retry_after_max", None)
- return exc.retry_after > (max_retry_after_seconds() if limit is None else limit)
+ return exc.retry_after > _retry_after_limit(client)
def _stop_after_client_attempts(state: RetryCallState) -> bool:
@@ -381,6 +384,7 @@ class RESTClient:
capture: bool = True,
retry_budget_seconds: Optional[float] = None,
retry_after_max_seconds: Optional[float] = None,
+ cap_retry_after: bool = False,
) -> None:
self.base_url = base_url or ""
self.headers = headers or {}
@@ -394,6 +398,10 @@ class RESTClient:
# does not fail the run.
self._retry_budget = retry_budget_seconds
self._retry_after_max = retry_after_max_seconds
+ # When True, a server delay above the limit becomes a wait of the limit, and the request
+ # does not fail at once. Use it for a vendor whose advertised delay is longer than the
+ # time the client must wait before a retry can succeed.
+ self._cap_retry_after = cap_retry_after
# Per-request (connect, read) timeout in seconds handed to ``session.send``. Left None,
# the session's own default applies, and after that the tracked adapter's
# ``default_request_timeout()``. Sources talking to a customer-controlled host should set
@@ -674,6 +682,8 @@ class RESTClient:
# the percent-encoded form an `api_key` query credential takes in the URL, so scheme/
# host/path-only is what keeps an encoded credential out of the persisted `latest_error`.
retry_after = _parse_retry_after(response)
+ if retry_after is not None and self._cap_retry_after:
+ retry_after = min(retry_after, _retry_after_limit(self))
message = f"HTTP {response.status_code} for {_safe_url(response.url)}"
error = RESTClientRetryableError(self._redact(message), retry_after=retry_after)
if _retry_after_exceeds_limit(self, error):
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/typing.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/typing.py
index df9983c64c31..c989a985ba2f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/typing.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/typing.py
@@ -221,6 +221,9 @@ class ClientConfig(TypedDict, total=False):
# `DATA_WAREHOUSE_SOURCE_*` settings apply. Raise both for a vendor with a longer rate limit window.
retry_budget_seconds: float
retry_after_max_seconds: float
+ # When True, a server-provided delay above `retry_after_max_seconds` becomes a wait of that
+ # limit, and the request does not fail at once. Left unset, such a request fails without a wait.
+ cap_retry_after: bool
# SSRF host-pinning. When set (even to an empty list), every outgoing request URL —
# including paginator next-page links and seeded resume URLs — must resolve to one of
# these hosts; the base_url host is always implicitly allowed. Off-host URLs are rejected