Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(blueshift): retry 429/5xx on the campaign stats endpoint (#128593)

Tom Owers349d88eeb3c7
diff --git a/products/warehouse_sources/sources/blueshift/blueshift.py b/products/warehouse_sources/sources/blueshift/blueshift.py
index 9abba33d6317..55c6de39d891 100644
--- a/products/warehouse_sources/sources/blueshift/blueshift.py
+++ b/products/warehouse_sources/sources/blueshift/blueshift.py
@@ -4,6 +4,9 @@ from datetime import UTC, date, datetime, time, timedelta
from typing import Any, ClassVar, Optional
from urllib.parse import urlencode
+import requests
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+
from products.warehouse_sources.sources.blueshift.settings import (
BLUESHIFT_ENDPOINTS,
BLUESHIFT_REGION_URLS,
@@ -27,11 +30,22 @@ from products.warehouse_sources.sources.sdk import (
make_tracked_session,
parse_datetime_value,
rest_api_resource,
+ retry_with_deadline,
+ suspend_adapter_retries,
validate_via_probe,
)
STATS_REQUEST_TIMEOUT = 120
_ISO_UTC = "%Y-%m-%dT%H:%M:%S.000Z"
+# The v1 stats endpoint has no documented rate limit, but it 429s under heavy per-day polling
+# (one request per UTC day of backfill). It's fetched with a raw `session.get`, not
+# `rest_api_resource`, so it needs its own retry-with-backoff before giving up, same as every
+# other hand-rolled source transport in this tree.
+MAX_STATS_RETRY_ATTEMPTS = 5
+
+
+class BlueshiftRetryableError(Exception):
+ pass
@frozen
@@ -107,6 +121,23 @@ def _stats_window(day: date) -> dict[str, str]:
return {"start_time": start.strftime(_ISO_UTC), "end_time": end.strftime(_ISO_UTC)}
+@retry_with_deadline(
+ retry=retry_if_exception_type((BlueshiftRetryableError, requests.ReadTimeout, requests.ConnectionError)),
+ stop=stop_after_attempt(MAX_STATS_RETRY_ATTEMPTS),
+ wait=wait_exponential_jitter(initial=1, max=30),
+ reraise=True,
+)
+def _fetch_stats_day(session: requests.Session, url: str, headers: dict[str, str], day: date) -> dict[str, Any]:
+ # This function owns the retries. Leaving the session's own adapter retry on too would multiply
+ # the tries and the waits of both layers, and the adapter's waits have no safe point.
+ with suspend_adapter_retries():
+ response = session.get(url, params=_stats_window(day), headers=headers, timeout=STATS_REQUEST_TIMEOUT)
+ if response.status_code == 429 or response.status_code >= 500:
+ raise BlueshiftRetryableError(f"Blueshift API error (retryable): status={response.status_code}, url={url}")
+ response.raise_for_status()
+ return response.json()
+
+
def _campaign_stats_rows(
region: str,
api_key: str,
@@ -117,6 +148,8 @@ def _campaign_stats_rows(
) -> Iterator[list[dict[str, Any]]]:
"""Yield the per-campaign performance summary for each UTC day, oldest first, through today."""
config = BLUESHIFT_ENDPOINTS["campaign_stats"]
+ assert config.data_selector is not None
+ data_selector = config.data_selector
auth_value = _basic_auth_value(api_key)
session = make_tracked_session(redact_values=(api_key, auth_value))
headers = {"Authorization": auth_value, "Accept": "application/json"}
@@ -142,9 +175,8 @@ def _campaign_stats_rows(
day = today - timedelta(days=STATS_BACKFILL_DAYS)
while day <= today:
- response = session.get(url, params=_stats_window(day), headers=headers, timeout=STATS_REQUEST_TIMEOUT)
- response.raise_for_status()
- campaigns = response.json().get(config.data_selector) or []
+ body = _fetch_stats_day(session, url, headers, day)
+ campaigns = body.get(data_selector) or []
next_day = day + timedelta(days=1)
if next_day <= today:
resumable_source_manager.save_state(BlueshiftResumeConfig(next_date=next_day.isoformat()))
diff --git a/products/warehouse_sources/sources/blueshift/source.py b/products/warehouse_sources/sources/blueshift/source.py
index 49e52af1db74..bfb75e4b3417 100644
--- a/products/warehouse_sources/sources/blueshift/source.py
+++ b/products/warehouse_sources/sources/blueshift/source.py
@@ -114,6 +114,15 @@ Use a **user API key** from **Account settings > API keys** in Blueshift. Pick t
"403 Client Error: Forbidden for url": "The Blueshift API key can't read one of the tables you sync. Use a key with access to that data, or deselect the table.",
}
+ def get_retryable_errors(self) -> set[str]:
+ # The campaign_stats daily walk fetches with a raw `session.get`, not `rest_api_resource`,
+ # so it retries a 429/5xx itself and re-raises as "Blueshift API error (retryable): ..."
+ # once that budget is exhausted. `make_tracked_session`'s own DEFAULT_RETRY can also
+ # exhaust first on a dropped connection or read timeout, surfacing as urllib3's
+ # "Max retries exceeded with url: ...". Temporal retries the whole activity either way, so
+ # the failure is transient and self-recovering, not a bug to report.
+ return {"Max retries exceeded with url", "Blueshift API error (retryable)"}
+
def get_schemas(
self,
config: BlueshiftSourceConfig,
diff --git a/products/warehouse_sources/sources/blueshift/tests/test_blueshift.py b/products/warehouse_sources/sources/blueshift/tests/test_blueshift.py
index 36cf862eb0a5..5b70de269e37 100644
--- a/products/warehouse_sources/sources/blueshift/tests/test_blueshift.py
+++ b/products/warehouse_sources/sources/blueshift/tests/test_blueshift.py
@@ -8,7 +8,12 @@ import time_machine
from parameterized import parameterized
from products.warehouse_sources.sources.blueshift._config import BlueshiftSourceConfig
-from products.warehouse_sources.sources.blueshift.blueshift import BlueshiftResumeConfig, BlueshiftStatsCursor
+from products.warehouse_sources.sources.blueshift.blueshift import (
+ MAX_STATS_RETRY_ATTEMPTS,
+ BlueshiftResumeConfig,
+ BlueshiftRetryableError,
+ BlueshiftStatsCursor,
+)
from products.warehouse_sources.sources.blueshift.source import BlueshiftSource
from products.warehouse_sources.sources.sdk import SourceCursorManager
from products.warehouse_sources.sources.sdk.testing import ScriptedResponse, SourceDriver, always, scripted_network
@@ -169,6 +174,43 @@ class TestBlueshiftCampaignStats:
assert result.requests[0].param("start_time") == "2025-10-15T00:00:00.000Z"
assert len(result.requests) == 366
+ @time_machine.travel(NOW, tick=False)
+ def test_a_rate_limited_day_is_retried_rather_than_failing_the_run(self) -> None:
+ calls = {"n": 0}
+
+ def answer(_request: Any) -> ScriptedResponse:
+ calls["n"] += 1
+ if calls["n"] == 1:
+ return ScriptedResponse(status=429, json={})
+ return ScriptedResponse(json={"campaigns": []})
+
+ result = _driver().run(
+ "campaign_stats",
+ answer,
+ incremental_field="date",
+ source_cursor=_no_stored_cursor(),
+ db_incremental_field_last_value=date(2026, 10, 15),
+ )
+
+ assert result.raised is None
+ # One extra request over the plain day-count: the first day's 429 is retried once, then
+ # every day (including the retried one) answers cleanly.
+ assert len(result.requests) == 9
+
+ @time_machine.travel(NOW, tick=False)
+ def test_a_sustained_rate_limit_exhausts_retries_and_raises_the_marker(self) -> None:
+ result = _driver().run(
+ "campaign_stats",
+ always(ScriptedResponse(status=429, json={})),
+ incremental_field="date",
+ source_cursor=_no_stored_cursor(),
+ db_incremental_field_last_value=date(2026, 10, 15),
+ )
+
+ assert isinstance(result.raised, BlueshiftRetryableError)
+ assert "Blueshift API error (retryable): status=429" in str(result.raised)
+ assert len(result.requests) == MAX_STATS_RETRY_ATTEMPTS
+
@time_machine.travel(NOW, tick=False)
def test_rejected_key_fails_the_run_and_keeps_the_key_out_of_logs(self) -> None:
result = _driver().run(
@@ -220,3 +262,21 @@ class TestBlueshiftValidateCredentials:
assert valid is expected_valid
assert (message is None) is expected_valid
+
+
+class TestBlueshiftRetryableErrors:
+ @parameterized.expand(
+ [
+ (
+ "exhausted_rate_limit",
+ "Blueshift API error (retryable): status=429, url=https://api.getblueshift.com/api/v1/campaigns.json",
+ ),
+ (
+ "exhausted_5xx",
+ "Blueshift API error (retryable): status=503, url=https://api.getblueshift.com/api/v1/campaigns.json",
+ ),
+ ]
+ )
+ def test_retryable_errors_match_exhausted_retry(self, _name: str, observed_error: str) -> None:
+ retryable_errors = BlueshiftSource().get_retryable_errors()
+ assert any(key in observed_error for key in retryable_errors)
diff --git a/products/warehouse_sources/sources/sdk/__init__.py b/products/warehouse_sources/sources/sdk/__init__.py
index 66e85f639162..650522dd2c23 100644
--- a/products/warehouse_sources/sources/sdk/__init__.py
+++ b/products/warehouse_sources/sources/sdk/__init__.py
@@ -116,6 +116,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.htt
request_deadline,
retry_with_deadline,
stop_at_request_deadline,
+ suspend_adapter_retries,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http.observer import redact_request_urls
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http.transport import (
@@ -504,6 +505,7 @@ __all__ = [
"str_to_optional_list",
"strip_oauth2_permanent_marker",
"submit_with_context",
+ "suspend_adapter_retries",
"unreachable_vendor_message",
"update_sync_type_config_keys",
"uses_brotli_compression",