:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(trigger_dev): honor rate-limit reset on 429 retries (#128680)
Tom Owersfe0e8cf466fa
diff --git a/products/warehouse_sources/sources/trigger_dev/tests/test_trigger_dev.py b/products/warehouse_sources/sources/trigger_dev/tests/test_trigger_dev.pyindex a61d3a081746..b99cbc25d0f6 100644--- a/products/warehouse_sources/sources/trigger_dev/tests/test_trigger_dev.py+++ b/products/warehouse_sources/sources/trigger_dev/tests/test_trigger_dev.py@@ -1,7 +1,9 @@+import timefrom datetime import UTC, date, datetimefrom typing import Anyimport pytest+import time_machinefrom unittest.mock import MagicMock, patchimport requests@@ -322,6 +324,60 @@ class TestFetchPageRetries:trigger_dev._fetch_page(session, f"{BASE_URL}/api/v1/runs", {}, MagicMock())assert session.get.call_count == 1+ def test_rate_limit_wait_comes_from_reset_header_not_blind_backoff(self) -> None:+ # A 429 with a usable `x-ratelimit-reset` must wait for the bucket to actually refill+ # instead of the short blind exponential backoff, which can retry before it does.+ bad = MagicMock()+ bad.status_code = 429+ bad.headers = {"x-ratelimit-reset": str(int(time.time() * 1000) + 45_000)}+ good = MagicMock()+ good.status_code = 200+ good.ok = True+ good.json.return_value = {"data": []}+ session = MagicMock()+ session.get.side_effect = [bad, good]+ sleep = MagicMock()++ with patch.object(trigger_dev._fetch_page.retry, "sleep", sleep): # type: ignore[attr-defined]+ trigger_dev._fetch_page(session, f"{BASE_URL}/api/v1/runs", {}, MagicMock())++ sleep.assert_called_once()+ assert sleep.call_args[0][0] == pytest.approx(45.0, abs=1.0)+++class TestRateLimitWaitSeconds:+ @pytest.fixture(autouse=True)+ def _freeze_time(self):+ with time_machine.travel("2026-01-01T00:00:00Z", tick=False):+ yield++ @parameterized.expand(+ [+ ("resets_soon", 45_000, 45.0),+ ("already_past", -90_000, 0.0),+ ]+ )+ def test_wait_matches_time_until_reset(self, _name: str, offset_ms: int, expected: float) -> None:+ response = MagicMock()+ response.headers = {"x-ratelimit-reset": str(int(time.time() * 1000) + offset_ms)}+ assert trigger_dev._rate_limit_wait_seconds(response) == pytest.approx(expected, abs=0.01)++ def test_far_future_reset_is_capped(self) -> None:+ response = MagicMock()+ response.headers = {"x-ratelimit-reset": "9999999999999"}+ assert trigger_dev._rate_limit_wait_seconds(response) == trigger_dev.MAX_RATE_LIMIT_WAIT_SECONDS++ @parameterized.expand(+ [+ ("missing_header", {}),+ ("non_numeric", {"x-ratelimit-reset": "soon"}),+ ]+ )+ def test_unusable_header_falls_back_to_zero(self, _name: str, headers: dict[str, str]) -> None:+ response = MagicMock()+ response.headers = headers+ assert trigger_dev._rate_limit_wait_seconds(response) == 0.0+class TestSourceResponseSortMode:@parameterized.expand(diff --git a/products/warehouse_sources/sources/trigger_dev/trigger_dev.py b/products/warehouse_sources/sources/trigger_dev/trigger_dev.pyindex d50d6a65c7a5..786a8d997169 100644--- a/products/warehouse_sources/sources/trigger_dev/trigger_dev.py+++ b/products/warehouse_sources/sources/trigger_dev/trigger_dev.py@@ -1,3 +1,4 @@+import timeimport dataclassesfrom collections.abc import Iteratorfrom datetime import UTC, date, datetime, timedelta@@ -7,7 +8,7 @@ from urllib.parse import urlencode, urlsplitimport requestsfrom dateutil import parser as dateutil_parserfrom structlog.types import FilteringBoundLogger-from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitterfrom products.warehouse_sources.sources.sdk import (ResumableSourceManager,@@ -24,9 +25,44 @@ TRIGGER_DEV_DEFAULT_BASE_URL = "https://api.trigger.dev"# times out. Schedules and queues are small config tables, so this ceiling is far above any real size.MAX_CLASSIC_PAGES = 10_000+# Upper bound on how long we'll honor the bucket's reported reset time, so a bogus or far-future+# header can't pin a worker.+MAX_RATE_LIMIT_WAIT_SECONDS = 120++_EXPONENTIAL_WAIT = wait_exponential_jitter(initial=1, max=30)+class TriggerDevRetryableError(Exception):- pass+ def __init__(self, message: str, retry_after: float = 0) -> None:+ super().__init__(message)+ # Seconds to wait before retrying, from a 429's `x-ratelimit-reset`; 0 when absent or+ # unusable, which falls back to exponential backoff.+ self.retry_after = retry_after+++def _rate_limit_wait_seconds(response: requests.Response) -> float:+ """Seconds until Trigger.dev's token bucket resets, from the 429's `x-ratelimit-reset` header.++ Trigger.dev documents this header as a unix timestamp in milliseconds. Its own SDK moved from+ blind exponential backoff to honoring it for the same reason we do: a 429 means the bucket is+ empty, and a blind backoff either retries before it refills or waits longer than necessary.+ """+ raw = response.headers.get("x-ratelimit-reset")+ if raw is None:+ return 0.0+ try:+ reset_at_seconds = float(raw) / 1000+ except (TypeError, ValueError):+ return 0.0+ return max(0.0, min(reset_at_seconds - time.time(), MAX_RATE_LIMIT_WAIT_SECONDS))+++def _retry_wait(retry_state: RetryCallState) -> float:+ # Honor Trigger.dev's rate-limit reset once; otherwise back off exponentially.+ exc = retry_state.outcome.exception() if retry_state.outcome else None+ if isinstance(exc, TriggerDevRetryableError) and exc.retry_after:+ return exc.retry_after+ return _EXPONENTIAL_WAIT(retry_state)@dataclasses.dataclass # nosemgrep: prefer-frozen-dataclasses -- pre-existing; new field shifts the backlog finding into diff@@ -122,15 +158,20 @@ def _should_stop_desc(items: list[dict[str, Any]], incremental_field: str | None)),stop=stop_after_attempt(5),- wait=wait_exponential_jitter(initial=1, max=30),+ wait=_retry_wait,reraise=True,)def _fetch_page(session: requests.Session, url: str, headers: dict[str, str], logger: FilteringBoundLogger) -> dict:response = session.get(url, headers=headers, timeout=60)# Trigger.dev rate-limits per API key with a token bucket and standard x-ratelimit-* headers;- # back off and retry on 429 and transient 5xx.- if response.status_code == 429 or response.status_code >= 500:+ # back off and retry on 429 (honoring the bucket's reset time) and transient 5xx.+ if response.status_code == 429:+ raise TriggerDevRetryableError(+ f"Trigger.dev API error (retryable): status={response.status_code}, url={url}",+ retry_after=_rate_limit_wait_seconds(response),+ )+ if response.status_code >= 500:raise TriggerDevRetryableError(f"Trigger.dev API error (retryable): status={response.status_code}, url={url}")if not response.ok: