Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(trigger_dev): honor rate-limit reset on 429 retries (#128680)

Tom Owersfe0e8cf466fa
diff --git a/products/warehouse_sources/sources/trigger_dev/tests/test_trigger_dev.py b/products/warehouse_sources/sources/trigger_dev/tests/test_trigger_dev.py
index a61d3a081746..b99cbc25d0f6 100644
--- a/products/warehouse_sources/sources/trigger_dev/tests/test_trigger_dev.py
+++ b/products/warehouse_sources/sources/trigger_dev/tests/test_trigger_dev.py
@@ -1,7 +1,9 @@
+import time
from datetime import UTC, date, datetime
from typing import Any
import pytest
+import time_machine
from unittest.mock import MagicMock, patch
import requests
@@ -322,6 +324,60 @@ class TestFetchPageRetries:
trigger_dev._fetch_page(session, f"{BASE_URL}/api/v1/runs", {}, MagicMock())
assert session.get.call_count == 1
+ def test_rate_limit_wait_comes_from_reset_header_not_blind_backoff(self) -> None:
+ # A 429 with a usable `x-ratelimit-reset` must wait for the bucket to actually refill
+ # instead of the short blind exponential backoff, which can retry before it does.
+ bad = MagicMock()
+ bad.status_code = 429
+ bad.headers = {"x-ratelimit-reset": str(int(time.time() * 1000) + 45_000)}
+ good = MagicMock()
+ good.status_code = 200
+ good.ok = True
+ good.json.return_value = {"data": []}
+ session = MagicMock()
+ session.get.side_effect = [bad, good]
+ sleep = MagicMock()
+
+ with patch.object(trigger_dev._fetch_page.retry, "sleep", sleep): # type: ignore[attr-defined]
+ trigger_dev._fetch_page(session, f"{BASE_URL}/api/v1/runs", {}, MagicMock())
+
+ sleep.assert_called_once()
+ assert sleep.call_args[0][0] == pytest.approx(45.0, abs=1.0)
+
+
+class TestRateLimitWaitSeconds:
+ @pytest.fixture(autouse=True)
+ def _freeze_time(self):
+ with time_machine.travel("2026-01-01T00:00:00Z", tick=False):
+ yield
+
+ @parameterized.expand(
+ [
+ ("resets_soon", 45_000, 45.0),
+ ("already_past", -90_000, 0.0),
+ ]
+ )
+ def test_wait_matches_time_until_reset(self, _name: str, offset_ms: int, expected: float) -> None:
+ response = MagicMock()
+ response.headers = {"x-ratelimit-reset": str(int(time.time() * 1000) + offset_ms)}
+ assert trigger_dev._rate_limit_wait_seconds(response) == pytest.approx(expected, abs=0.01)
+
+ def test_far_future_reset_is_capped(self) -> None:
+ response = MagicMock()
+ response.headers = {"x-ratelimit-reset": "9999999999999"}
+ assert trigger_dev._rate_limit_wait_seconds(response) == trigger_dev.MAX_RATE_LIMIT_WAIT_SECONDS
+
+ @parameterized.expand(
+ [
+ ("missing_header", {}),
+ ("non_numeric", {"x-ratelimit-reset": "soon"}),
+ ]
+ )
+ def test_unusable_header_falls_back_to_zero(self, _name: str, headers: dict[str, str]) -> None:
+ response = MagicMock()
+ response.headers = headers
+ assert trigger_dev._rate_limit_wait_seconds(response) == 0.0
+
class TestSourceResponseSortMode:
@parameterized.expand(
diff --git a/products/warehouse_sources/sources/trigger_dev/trigger_dev.py b/products/warehouse_sources/sources/trigger_dev/trigger_dev.py
index d50d6a65c7a5..786a8d997169 100644
--- a/products/warehouse_sources/sources/trigger_dev/trigger_dev.py
+++ b/products/warehouse_sources/sources/trigger_dev/trigger_dev.py
@@ -1,3 +1,4 @@
+import time
import dataclasses
from collections.abc import Iterator
from datetime import UTC, date, datetime, timedelta
@@ -7,7 +8,7 @@ from urllib.parse import urlencode, urlsplit
import requests
from dateutil import parser as dateutil_parser
from structlog.types import FilteringBoundLogger
-from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.sources.sdk import (
ResumableSourceManager,
@@ -24,9 +25,44 @@ TRIGGER_DEV_DEFAULT_BASE_URL = "https://api.trigger.dev"
# times out. Schedules and queues are small config tables, so this ceiling is far above any real size.
MAX_CLASSIC_PAGES = 10_000
+# Upper bound on how long we'll honor the bucket's reported reset time, so a bogus or far-future
+# header can't pin a worker.
+MAX_RATE_LIMIT_WAIT_SECONDS = 120
+
+_EXPONENTIAL_WAIT = wait_exponential_jitter(initial=1, max=30)
+
class TriggerDevRetryableError(Exception):
- pass
+ def __init__(self, message: str, retry_after: float = 0) -> None:
+ super().__init__(message)
+ # Seconds to wait before retrying, from a 429's `x-ratelimit-reset`; 0 when absent or
+ # unusable, which falls back to exponential backoff.
+ self.retry_after = retry_after
+
+
+def _rate_limit_wait_seconds(response: requests.Response) -> float:
+ """Seconds until Trigger.dev's token bucket resets, from the 429's `x-ratelimit-reset` header.
+
+ Trigger.dev documents this header as a unix timestamp in milliseconds. Its own SDK moved from
+ blind exponential backoff to honoring it for the same reason we do: a 429 means the bucket is
+ empty, and a blind backoff either retries before it refills or waits longer than necessary.
+ """
+ raw = response.headers.get("x-ratelimit-reset")
+ if raw is None:
+ return 0.0
+ try:
+ reset_at_seconds = float(raw) / 1000
+ except (TypeError, ValueError):
+ return 0.0
+ return max(0.0, min(reset_at_seconds - time.time(), MAX_RATE_LIMIT_WAIT_SECONDS))
+
+
+def _retry_wait(retry_state: RetryCallState) -> float:
+ # Honor Trigger.dev's rate-limit reset once; otherwise back off exponentially.
+ exc = retry_state.outcome.exception() if retry_state.outcome else None
+ if isinstance(exc, TriggerDevRetryableError) and exc.retry_after:
+ return exc.retry_after
+ return _EXPONENTIAL_WAIT(retry_state)
@dataclasses.dataclass # nosemgrep: prefer-frozen-dataclasses -- pre-existing; new field shifts the backlog finding into diff
@@ -122,15 +158,20 @@ def _should_stop_desc(items: list[dict[str, Any]], incremental_field: str | None
)
),
stop=stop_after_attempt(5),
- wait=wait_exponential_jitter(initial=1, max=30),
+ wait=_retry_wait,
reraise=True,
)
def _fetch_page(session: requests.Session, url: str, headers: dict[str, str], logger: FilteringBoundLogger) -> dict:
response = session.get(url, headers=headers, timeout=60)
# Trigger.dev rate-limits per API key with a token bucket and standard x-ratelimit-* headers;
- # back off and retry on 429 and transient 5xx.
- if response.status_code == 429 or response.status_code >= 500:
+ # back off and retry on 429 (honoring the bucket's reset time) and transient 5xx.
+ if response.status_code == 429:
+ raise TriggerDevRetryableError(
+ f"Trigger.dev API error (retryable): status={response.status_code}, url={url}",
+ retry_after=_rate_limit_wait_seconds(response),
+ )
+ if response.status_code >= 500:
raise TriggerDevRetryableError(f"Trigger.dev API error (retryable): status={response.status_code}, url={url}")
if not response.ok: