Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(anthropic): retry claude code syncs after the rate limit window (#121372)

posthog[bot]5696f8a6a0c3
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/anthropic/anthropic.py b/products/warehouse_sources/backend/temporal/data_imports/sources/anthropic/anthropic.py
index 0b9d080014de..7b60c40c1cc3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/anthropic/anthropic.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/anthropic/anthropic.py
@@ -49,11 +49,11 @@ ENTITY_PAGE_SIZE = 1000
MAX_RETRY_ATTEMPTS = 8
# The usage/cost/analytics report endpoints share one organization-level rate limit on Anthropic's
# Admin API. Some of them, such as the Claude Code usage report, answer 429 with a `Retry-After` of
-# up to an hour, which is the time until the budget is full again. The budget refills continuously,
-# so the report clients cap each server delay at the shared limit and wait it out, instead of
-# failing the request at once. A 429 without a usable delay falls back to exponential backoff.
-# Give the report endpoints more attempts and a higher backoff ceiling than the entity lists. Each
-# wait stays capped, so a sync never stalls indefinitely.
+# up to an hour, and a retry before that time also gets a 429. Such a request fails at once with the
+# server delay, and the import activity retries after that delay, so no worker waits out the hour.
+# A 429 without a usable delay falls back to exponential backoff. Give the report endpoints more
+# attempts and a higher backoff ceiling than the entity lists. Each wait stays capped, so a sync
+# never stalls indefinitely.
REPORT_MAX_RETRY_ATTEMPTS = 12
REPORT_RETRY_BACKOFF_MAX_SECONDS = 300.0
# The waits of the report attempts above add up to about 1,100 seconds, which is more than the
@@ -853,7 +853,6 @@ def anthropic_source(
if is_report_endpoint:
client_config["retry_backoff_max_seconds"] = REPORT_RETRY_BACKOFF_MAX_SECONDS
client_config["retry_budget_seconds"] = REPORT_RETRY_BUDGET_SECONDS
- client_config["cap_retry_after"] = True
resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/anthropic/tests/test_anthropic.py b/products/warehouse_sources/backend/temporal/data_imports/sources/anthropic/tests/test_anthropic.py
index 623bf837645f..bc4b372fa6e7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/anthropic/tests/test_anthropic.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/anthropic/tests/test_anthropic.py
@@ -32,6 +32,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.anthropic.
)
from products.warehouse_sources.backend.temporal.data_imports.sources.anthropic.source import AnthropicSource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import error_message_matches
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import (
+ RESTClientRetryableError,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
ScriptedResponse,
SourceDriver,
@@ -366,19 +369,17 @@ class TestClaudeCodeDayFanOut:
assert result.params("page") == ["P2"]
assert result.params("starting_at") == [today.isoformat()]
- def test_waits_out_a_rate_limit_that_asks_for_an_hour(self) -> None:
+ def test_fails_at_once_with_the_delay_when_a_rate_limit_asks_for_an_hour(self) -> None:
today = datetime.now(UTC).date()
result = _driver().run(
"claude_code_analytics",
- [
- ScriptedResponse(status=429, headers={"Retry-After": "3600"}),
- _cc_page([_cc_record()], has_more=False, next_page=None),
- ],
+ [ScriptedResponse(status=429, headers={"Retry-After": "3600"})],
incremental_field="date",
db_incremental_field_last_value=_midnight(today),
)
- assert result.raised is None
- assert len(result.rows) == 1
+ assert isinstance(result.raised, RESTClientRetryableError)
+ assert result.raised.retry_after == 3600
+ assert len(result.requests) == 1
class TestRetiredEndpoint:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/__init__.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/__init__.py
index 868d953677e4..e11fe8cf0e26 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/__init__.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/__init__.py
@@ -360,7 +360,6 @@ def create_resources(
allow_redirects=client_config.get("allow_redirects", True),
request_timeout=client_config.get("request_timeout"),
capture=client_config.get("capture", True),
- cap_retry_after=client_config.get("cap_retry_after", False),
**_retry_limit_overrides(client_config),
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/rest_client.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/rest_client.py
index d1f2ac4d8ab8..83d3377f27c5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/rest_client.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/rest_client.py
@@ -307,15 +307,11 @@ def _parse_retry_after(response: Response) -> Optional[float]:
return None
-def _retry_after_limit(client: Any) -> float:
- limit = getattr(client, "_retry_after_max", None)
- return max_retry_after_seconds() if limit is None else limit
-
-
def _retry_after_exceeds_limit(client: Any, exc: Optional[BaseException]) -> bool:
if not isinstance(exc, RESTClientRetryableError) or exc.retry_after is None:
return False
- return exc.retry_after > _retry_after_limit(client)
+ limit = getattr(client, "_retry_after_max", None)
+ return exc.retry_after > (max_retry_after_seconds() if limit is None else limit)
def _stop_after_client_attempts(state: RetryCallState) -> bool:
@@ -402,7 +398,6 @@ class RESTClient:
capture: bool = True,
retry_budget_seconds: Optional[float] = None,
retry_after_max_seconds: Optional[float] = None,
- cap_retry_after: bool = False,
) -> None:
self.base_url = base_url or ""
self.headers = headers or {}
@@ -416,10 +411,6 @@ class RESTClient:
# does not fail the run.
self._retry_budget = retry_budget_seconds
self._retry_after_max = retry_after_max_seconds
- # When True, a server delay above the limit becomes a wait of the limit, and the request
- # does not fail at once. Use it for a vendor whose advertised delay is longer than the
- # time the client must wait before a retry can succeed.
- self._cap_retry_after = cap_retry_after
# Per-request (connect, read) timeout in seconds handed to ``session.send``. Left None,
# the session's own default applies, and after that the tracked adapter's
# ``default_request_timeout()``. Sources talking to a customer-controlled host should set
@@ -704,8 +695,6 @@ class RESTClient:
# the percent-encoded form an `api_key` query credential takes in the URL, so scheme/
# host/path-only is what keeps an encoded credential out of the persisted `latest_error`.
retry_after = _parse_retry_after(response)
- if retry_after is not None and self._cap_retry_after:
- retry_after = min(retry_after, _retry_after_limit(self))
message = f"HTTP {response.status_code} for {_safe_url(response.url)}"
error = RESTClientRetryableError(self._redact(message), retry_after=retry_after)
if _retry_after_exceeds_limit(self, error):
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/typing.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/typing.py
index c989a985ba2f..df9983c64c31 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/typing.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/typing.py
@@ -221,9 +221,6 @@ class ClientConfig(TypedDict, total=False):
# `DATA_WAREHOUSE_SOURCE_*` settings apply. Raise both for a vendor with a longer rate limit window.
retry_budget_seconds: float
retry_after_max_seconds: float
- # When True, a server-provided delay above `retry_after_max_seconds` becomes a wait of that
- # limit, and the request does not fail at once. Left unset, such a request fails without a wait.
- cap_retry_after: bool
# SSRF host-pinning. When set (even to an empty list), every outgoing request URL —
# including paginator next-page links and seeded resume URLs — must resolve to one of
# these hosts; the base_url host is always implicitly allowed. Off-host URLs are rejected
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py
index 3e5f78314798..2cd021021bce 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py
@@ -28,7 +28,7 @@ from posthog.exceptions_capture import capture_exception
from posthog.models.integration import UndecryptedIntegrationSecretError
from posthog.sync import database_sync_to_async_pool
from posthog.temporal.common.activity_context import current_activity_attempt
-from posthog.temporal.common.errors import NonReportableError
+from posthog.temporal.common.errors import NonReportableApplicationError, NonReportableError
from posthog.temporal.common.heartbeat import LivenessHeartbeater as Heartbeater
from posthog.temporal.common.logger import get_logger
from posthog.temporal.common.shutdown import DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT, ShutdownMonitor, WorkerShuttingDownError
@@ -325,6 +325,10 @@ if TYPE_CHECKING:
# page, putting break-even near 700 rows. This floor sits above it with margin.
MIN_WAREHOUSE_PARENT_ROWS = 1_000
+# The longest server-provided delay that the next import attempt waits for. Anthropic's Claude Code
+# report asks for about an hour. The cap stops a wrong header from pushing a run back by days.
+MAX_SERVER_RETRY_DELAY = dt.timedelta(hours=2)
+
def _parent_unusable_reason(parent: ExternalDataSchema | None) -> str | None:
"""Why a fan-out child can't read this parent from the warehouse, or None when it can.
@@ -1197,6 +1201,14 @@ async def _handle_import_error(
if isinstance(error, RESTClientRetryableError):
await logger.awarning(error_msg)
await logger.adebug("REST client exhausted its retries - re-raising for Temporal retry")
+ if error.retry_after:
+ # Defer the next Temporal attempt past the server's window instead of holding a worker in it.
+ raise NonReportableApplicationError(
+ error_msg,
+ type=RESTClientRetryableError.__name__,
+ # Clamp the float first: timedelta raises OverflowError for inf or huge header values.
+ next_retry_delay=dt.timedelta(seconds=min(error.retry_after, MAX_SERVER_RETRY_DELAY.total_seconds())),
+ ) from error
raise error
# The web pods and the data-import workers deploy separately, so a table that ships in one
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py
index 7c081010dbde..4fc3f5dfa14c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py
@@ -19,6 +19,7 @@ import deltalake.exceptions
from jsonpath_ng.exceptions import JsonPathParserError
from parameterized import parameterized
from requests.exceptions import HTTPError, ProxyError
+from temporalio.exceptions import ApplicationError
from posthog.credentials_service.errors import (
CredentialsServiceUnreachableError,
@@ -852,13 +853,25 @@ async def test_http_401_is_reraised_for_activity_retry():
@pytest.mark.asyncio
-async def test_rest_client_retryable_error_logged_as_warning_without_source_opt_in():
+ "retry_after, expected_delay",
+ [
+ (None, None),
+ (3600.0, timedelta(hours=1)),
+ (86400.0, module.MAX_SERVER_RETRY_DELAY),
+ (float("inf"), module.MAX_SERVER_RETRY_DELAY),
+ (1e15, module.MAX_SERVER_RETRY_DELAY),
+ ],
+)
+async def test_rest_client_retryable_error_logged_as_warning_without_source_opt_in(
+ retry_after: float | None, expected_delay: timedelta | None
+):
# RESTClientRetryableError only escapes the shared REST engine's tenacity retry loop once its
# own attempts (rate limits, transient 5xx, connection resets/timeouts) are exhausted. It must
# be honored by type even when the source's get_retryable_errors doesn't list the message, so
# every REST-based source gets this benign, self-recovering failure logged as a warning instead
# of minting error-tracking noise.
- error = RESTClientRetryableError("HTTP 429 for https://api.example.com/v3/orders/")
+ error = RESTClientRetryableError("HTTP 429 for https://api.example.com/v3/orders/", retry_after=retry_after)
source = mock.MagicMock(spec=SimpleSource)
source.get_non_retryable_errors.return_value = {}
source.get_retryable_errors.return_value = set()
@@ -869,9 +882,15 @@ async def test_rest_client_retryable_error_logged_as_warning_without_source_opt_
logger.adebug = mock.AsyncMock()
with mock.patch.object(module.SourceRegistry, "get_source", return_value=source):
- with pytest.raises(RESTClientRetryableError):
+ with pytest.raises(NonReportableError) as raised:
await module._handle_import_error(mock.MagicMock(), logger, error)
+ if expected_delay is None:
+ assert raised.value is error
+ else:
+ assert isinstance(raised.value, ApplicationError)
+ assert raised.value.type == "RESTClientRetryableError"
+ assert raised.value.next_retry_delay == expected_delay
logger.awarning.assert_awaited_once()
logger.aexception.assert_not_awaited()