:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(anthropic): retry claude code syncs after the rate limit window (#121372)
posthog[bot]5696f8a6a0c3
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/anthropic/anthropic.py b/products/warehouse_sources/backend/temporal/data_imports/sources/anthropic/anthropic.pyindex 0b9d080014de..7b60c40c1cc3 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/anthropic/anthropic.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/anthropic/anthropic.py@@ -49,11 +49,11 @@ ENTITY_PAGE_SIZE = 1000MAX_RETRY_ATTEMPTS = 8# The usage/cost/analytics report endpoints share one organization-level rate limit on Anthropic's# Admin API. Some of them, such as the Claude Code usage report, answer 429 with a `Retry-After` of-# up to an hour, which is the time until the budget is full again. The budget refills continuously,-# so the report clients cap each server delay at the shared limit and wait it out, instead of-# failing the request at once. A 429 without a usable delay falls back to exponential backoff.-# Give the report endpoints more attempts and a higher backoff ceiling than the entity lists. Each-# wait stays capped, so a sync never stalls indefinitely.+# up to an hour, and a retry before that time also gets a 429. Such a request fails at once with the+# server delay, and the import activity retries after that delay, so no worker waits out the hour.+# A 429 without a usable delay falls back to exponential backoff. Give the report endpoints more+# attempts and a higher backoff ceiling than the entity lists. Each wait stays capped, so a sync+# never stalls indefinitely.REPORT_MAX_RETRY_ATTEMPTS = 12REPORT_RETRY_BACKOFF_MAX_SECONDS = 300.0# The waits of the report attempts above add up to about 1,100 seconds, which is more than the@@ -853,7 +853,6 @@ def anthropic_source(if is_report_endpoint:client_config["retry_backoff_max_seconds"] = REPORT_RETRY_BACKOFF_MAX_SECONDSclient_config["retry_budget_seconds"] = REPORT_RETRY_BUDGET_SECONDS- client_config["cap_retry_after"] = Trueresume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else Nonediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/anthropic/tests/test_anthropic.py b/products/warehouse_sources/backend/temporal/data_imports/sources/anthropic/tests/test_anthropic.pyindex 623bf837645f..bc4b372fa6e7 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/anthropic/tests/test_anthropic.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/anthropic/tests/test_anthropic.py@@ -32,6 +32,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.anthropic.)from products.warehouse_sources.backend.temporal.data_imports.sources.anthropic.source import AnthropicSourcefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.base import error_message_matches+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import (+ RESTClientRetryableError,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (ScriptedResponse,SourceDriver,@@ -366,19 +369,17 @@ class TestClaudeCodeDayFanOut:assert result.params("page") == ["P2"]assert result.params("starting_at") == [today.isoformat()]- def test_waits_out_a_rate_limit_that_asks_for_an_hour(self) -> None:+ def test_fails_at_once_with_the_delay_when_a_rate_limit_asks_for_an_hour(self) -> None:today = datetime.now(UTC).date()result = _driver().run("claude_code_analytics",- [- ScriptedResponse(status=429, headers={"Retry-After": "3600"}),- _cc_page([_cc_record()], has_more=False, next_page=None),- ],+ [ScriptedResponse(status=429, headers={"Retry-After": "3600"})],incremental_field="date",db_incremental_field_last_value=_midnight(today),)- assert result.raised is None- assert len(result.rows) == 1+ assert isinstance(result.raised, RESTClientRetryableError)+ assert result.raised.retry_after == 3600+ assert len(result.requests) == 1class TestRetiredEndpoint:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/__init__.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/__init__.pyindex 868d953677e4..e11fe8cf0e26 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/__init__.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/__init__.py@@ -360,7 +360,6 @@ def create_resources(allow_redirects=client_config.get("allow_redirects", True),request_timeout=client_config.get("request_timeout"),capture=client_config.get("capture", True),- cap_retry_after=client_config.get("cap_retry_after", False),**_retry_limit_overrides(client_config),)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/rest_client.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/rest_client.pyindex d1f2ac4d8ab8..83d3377f27c5 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/rest_client.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/rest_client.py@@ -307,15 +307,11 @@ def _parse_retry_after(response: Response) -> Optional[float]:return None-def _retry_after_limit(client: Any) -> float:- limit = getattr(client, "_retry_after_max", None)- return max_retry_after_seconds() if limit is None else limit--def _retry_after_exceeds_limit(client: Any, exc: Optional[BaseException]) -> bool:if not isinstance(exc, RESTClientRetryableError) or exc.retry_after is None:return False- return exc.retry_after > _retry_after_limit(client)+ limit = getattr(client, "_retry_after_max", None)+ return exc.retry_after > (max_retry_after_seconds() if limit is None else limit)def _stop_after_client_attempts(state: RetryCallState) -> bool:@@ -402,7 +398,6 @@ class RESTClient:capture: bool = True,retry_budget_seconds: Optional[float] = None,retry_after_max_seconds: Optional[float] = None,- cap_retry_after: bool = False,) -> None:self.base_url = base_url or ""self.headers = headers or {}@@ -416,10 +411,6 @@ class RESTClient:# does not fail the run.self._retry_budget = retry_budget_secondsself._retry_after_max = retry_after_max_seconds- # When True, a server delay above the limit becomes a wait of the limit, and the request- # does not fail at once. Use it for a vendor whose advertised delay is longer than the- # time the client must wait before a retry can succeed.- self._cap_retry_after = cap_retry_after# Per-request (connect, read) timeout in seconds handed to ``session.send``. Left None,# the session's own default applies, and after that the tracked adapter's# ``default_request_timeout()``. Sources talking to a customer-controlled host should set@@ -704,8 +695,6 @@ class RESTClient:# the percent-encoded form an `api_key` query credential takes in the URL, so scheme/# host/path-only is what keeps an encoded credential out of the persisted `latest_error`.retry_after = _parse_retry_after(response)- if retry_after is not None and self._cap_retry_after:- retry_after = min(retry_after, _retry_after_limit(self))message = f"HTTP {response.status_code} for {_safe_url(response.url)}"error = RESTClientRetryableError(self._redact(message), retry_after=retry_after)if _retry_after_exceeds_limit(self, error):diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/typing.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/typing.pyindex c989a985ba2f..df9983c64c31 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/typing.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/typing.py@@ -221,9 +221,6 @@ class ClientConfig(TypedDict, total=False):# `DATA_WAREHOUSE_SOURCE_*` settings apply. Raise both for a vendor with a longer rate limit window.retry_budget_seconds: floatretry_after_max_seconds: float- # When True, a server-provided delay above `retry_after_max_seconds` becomes a wait of that- # limit, and the request does not fail at once. Left unset, such a request fails without a wait.- cap_retry_after: bool# SSRF host-pinning. When set (even to an empty list), every outgoing request URL —# including paginator next-page links and seeded resume URLs — must resolve to one of# these hosts; the base_url host is always implicitly allowed. Off-host URLs are rejecteddiff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.pyindex 3e5f78314798..2cd021021bce 100644--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/import_data_sync.py@@ -28,7 +28,7 @@ from posthog.exceptions_capture import capture_exceptionfrom posthog.models.integration import UndecryptedIntegrationSecretErrorfrom posthog.sync import database_sync_to_async_poolfrom posthog.temporal.common.activity_context import current_activity_attempt-from posthog.temporal.common.errors import NonReportableError+from posthog.temporal.common.errors import NonReportableApplicationError, NonReportableErrorfrom posthog.temporal.common.heartbeat import LivenessHeartbeater as Heartbeaterfrom posthog.temporal.common.logger import get_loggerfrom posthog.temporal.common.shutdown import DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT, ShutdownMonitor, WorkerShuttingDownError@@ -325,6 +325,10 @@ if TYPE_CHECKING:# page, putting break-even near 700 rows. This floor sits above it with margin.MIN_WAREHOUSE_PARENT_ROWS = 1_000+# The longest server-provided delay that the next import attempt waits for. Anthropic's Claude Code+# report asks for about an hour. The cap stops a wrong header from pushing a run back by days.+MAX_SERVER_RETRY_DELAY = dt.timedelta(hours=2)+def _parent_unusable_reason(parent: ExternalDataSchema | None) -> str | None:"""Why a fan-out child can't read this parent from the warehouse, or None when it can.@@ -1197,6 +1201,14 @@ async def _handle_import_error(if isinstance(error, RESTClientRetryableError):await logger.awarning(error_msg)await logger.adebug("REST client exhausted its retries - re-raising for Temporal retry")+ if error.retry_after:+ # Defer the next Temporal attempt past the server's window instead of holding a worker in it.+ raise NonReportableApplicationError(+ error_msg,+ type=RESTClientRetryableError.__name__,+ # Clamp the float first: timedelta raises OverflowError for inf or huge header values.+ next_retry_delay=dt.timedelta(seconds=min(error.retry_after, MAX_SERVER_RETRY_DELAY.total_seconds())),+ ) from errorraise error# The web pods and the data-import workers deploy separately, so a table that ships in onediff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.pyindex 7c081010dbde..4fc3f5dfa14c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/tests/test_import_data_sync.py@@ -19,6 +19,7 @@ import deltalake.exceptionsfrom jsonpath_ng.exceptions import JsonPathParserErrorfrom parameterized import parameterizedfrom requests.exceptions import HTTPError, ProxyError+from temporalio.exceptions import ApplicationErrorfrom posthog.credentials_service.errors import (CredentialsServiceUnreachableError,@@ -852,13 +853,25 @@ async def test_http_401_is_reraised_for_activity_retry():@pytest.mark.asyncio-async def test_rest_client_retryable_error_logged_as_warning_without_source_opt_in():+ "retry_after, expected_delay",+ [+ (None, None),+ (3600.0, timedelta(hours=1)),+ (86400.0, module.MAX_SERVER_RETRY_DELAY),+ (float("inf"), module.MAX_SERVER_RETRY_DELAY),+ (1e15, module.MAX_SERVER_RETRY_DELAY),+ ],+)+async def test_rest_client_retryable_error_logged_as_warning_without_source_opt_in(+ retry_after: float | None, expected_delay: timedelta | None+):# RESTClientRetryableError only escapes the shared REST engine's tenacity retry loop once its# own attempts (rate limits, transient 5xx, connection resets/timeouts) are exhausted. It must# be honored by type even when the source's get_retryable_errors doesn't list the message, so# every REST-based source gets this benign, self-recovering failure logged as a warning instead# of minting error-tracking noise.- error = RESTClientRetryableError("HTTP 429 for https://api.example.com/v3/orders/")+ error = RESTClientRetryableError("HTTP 429 for https://api.example.com/v3/orders/", retry_after=retry_after)source = mock.MagicMock(spec=SimpleSource)source.get_non_retryable_errors.return_value = {}source.get_retryable_errors.return_value = set()@@ -869,9 +882,15 @@ async def test_rest_client_retryable_error_logged_as_warning_without_source_opt_logger.adebug = mock.AsyncMock()with mock.patch.object(module.SourceRegistry, "get_source", return_value=source):- with pytest.raises(RESTClientRetryableError):+ with pytest.raises(NonReportableError) as raised:await module._handle_import_error(mock.MagicMock(), logger, error)+ if expected_delay is None:+ assert raised.value is error+ else:+ assert isinstance(raised.value, ApplicationError)+ assert raised.value.type == "RESTClientRetryableError"+ assert raised.value.next_retry_delay == expected_delaylogger.awarning.assert_awaited_once()logger.aexception.assert_not_awaited()