:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(blueshift): retry 429/5xx on the campaign stats endpoint (#128593)
Tom Owers349d88eeb3c7
diff --git a/products/warehouse_sources/sources/blueshift/blueshift.py b/products/warehouse_sources/sources/blueshift/blueshift.pyindex 9abba33d6317..55c6de39d891 100644--- a/products/warehouse_sources/sources/blueshift/blueshift.py+++ b/products/warehouse_sources/sources/blueshift/blueshift.py@@ -4,6 +4,9 @@ from datetime import UTC, date, datetime, time, timedeltafrom typing import Any, ClassVar, Optionalfrom urllib.parse import urlencode+import requests+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter+from products.warehouse_sources.sources.blueshift.settings import (BLUESHIFT_ENDPOINTS,BLUESHIFT_REGION_URLS,@@ -27,11 +30,22 @@ from products.warehouse_sources.sources.sdk import (make_tracked_session,parse_datetime_value,rest_api_resource,+ retry_with_deadline,+ suspend_adapter_retries,validate_via_probe,)STATS_REQUEST_TIMEOUT = 120_ISO_UTC = "%Y-%m-%dT%H:%M:%S.000Z"+# The v1 stats endpoint has no documented rate limit, but it 429s under heavy per-day polling+# (one request per UTC day of backfill). It's fetched with a raw `session.get`, not+# `rest_api_resource`, so it needs its own retry-with-backoff before giving up, same as every+# other hand-rolled source transport in this tree.+MAX_STATS_RETRY_ATTEMPTS = 5+++class BlueshiftRetryableError(Exception):+ pass@frozen@@ -107,6 +121,23 @@ def _stats_window(day: date) -> dict[str, str]:return {"start_time": start.strftime(_ISO_UTC), "end_time": end.strftime(_ISO_UTC)}+@retry_with_deadline(+ retry=retry_if_exception_type((BlueshiftRetryableError, requests.ReadTimeout, requests.ConnectionError)),+ stop=stop_after_attempt(MAX_STATS_RETRY_ATTEMPTS),+ wait=wait_exponential_jitter(initial=1, max=30),+ reraise=True,+)+def _fetch_stats_day(session: requests.Session, url: str, headers: dict[str, str], day: date) -> dict[str, Any]:+ # This function owns the retries. Leaving the session's own adapter retry on too would multiply+ # the tries and the waits of both layers, and the adapter's waits have no safe point.+ with suspend_adapter_retries():+ response = session.get(url, params=_stats_window(day), headers=headers, timeout=STATS_REQUEST_TIMEOUT)+ if response.status_code == 429 or response.status_code >= 500:+ raise BlueshiftRetryableError(f"Blueshift API error (retryable): status={response.status_code}, url={url}")+ response.raise_for_status()+ return response.json()++def _campaign_stats_rows(region: str,api_key: str,@@ -117,6 +148,8 @@ def _campaign_stats_rows() -> Iterator[list[dict[str, Any]]]:"""Yield the per-campaign performance summary for each UTC day, oldest first, through today."""config = BLUESHIFT_ENDPOINTS["campaign_stats"]+ assert config.data_selector is not None+ data_selector = config.data_selectorauth_value = _basic_auth_value(api_key)session = make_tracked_session(redact_values=(api_key, auth_value))headers = {"Authorization": auth_value, "Accept": "application/json"}@@ -142,9 +175,8 @@ def _campaign_stats_rows(day = today - timedelta(days=STATS_BACKFILL_DAYS)while day <= today:- response = session.get(url, params=_stats_window(day), headers=headers, timeout=STATS_REQUEST_TIMEOUT)- response.raise_for_status()- campaigns = response.json().get(config.data_selector) or []+ body = _fetch_stats_day(session, url, headers, day)+ campaigns = body.get(data_selector) or []next_day = day + timedelta(days=1)if next_day <= today:resumable_source_manager.save_state(BlueshiftResumeConfig(next_date=next_day.isoformat()))diff --git a/products/warehouse_sources/sources/blueshift/source.py b/products/warehouse_sources/sources/blueshift/source.pyindex 49e52af1db74..bfb75e4b3417 100644--- a/products/warehouse_sources/sources/blueshift/source.py+++ b/products/warehouse_sources/sources/blueshift/source.py@@ -114,6 +114,15 @@ Use a **user API key** from **Account settings > API keys** in Blueshift. Pick t"403 Client Error: Forbidden for url": "The Blueshift API key can't read one of the tables you sync. Use a key with access to that data, or deselect the table.",}+ def get_retryable_errors(self) -> set[str]:+ # The campaign_stats daily walk fetches with a raw `session.get`, not `rest_api_resource`,+ # so it retries a 429/5xx itself and re-raises as "Blueshift API error (retryable): ..."+ # once that budget is exhausted. `make_tracked_session`'s own DEFAULT_RETRY can also+ # exhaust first on a dropped connection or read timeout, surfacing as urllib3's+ # "Max retries exceeded with url: ...". Temporal retries the whole activity either way, so+ # the failure is transient and self-recovering, not a bug to report.+ return {"Max retries exceeded with url", "Blueshift API error (retryable)"}+def get_schemas(self,config: BlueshiftSourceConfig,diff --git a/products/warehouse_sources/sources/blueshift/tests/test_blueshift.py b/products/warehouse_sources/sources/blueshift/tests/test_blueshift.pyindex 36cf862eb0a5..5b70de269e37 100644--- a/products/warehouse_sources/sources/blueshift/tests/test_blueshift.py+++ b/products/warehouse_sources/sources/blueshift/tests/test_blueshift.py@@ -8,7 +8,12 @@ import time_machinefrom parameterized import parameterizedfrom products.warehouse_sources.sources.blueshift._config import BlueshiftSourceConfig-from products.warehouse_sources.sources.blueshift.blueshift import BlueshiftResumeConfig, BlueshiftStatsCursor+from products.warehouse_sources.sources.blueshift.blueshift import (+ MAX_STATS_RETRY_ATTEMPTS,+ BlueshiftResumeConfig,+ BlueshiftRetryableError,+ BlueshiftStatsCursor,+)from products.warehouse_sources.sources.blueshift.source import BlueshiftSourcefrom products.warehouse_sources.sources.sdk import SourceCursorManagerfrom products.warehouse_sources.sources.sdk.testing import ScriptedResponse, SourceDriver, always, scripted_network@@ -169,6 +174,43 @@ class TestBlueshiftCampaignStats:assert result.requests[0].param("start_time") == "2025-10-15T00:00:00.000Z"assert len(result.requests) == 366+ @time_machine.travel(NOW, tick=False)+ def test_a_rate_limited_day_is_retried_rather_than_failing_the_run(self) -> None:+ calls = {"n": 0}++ def answer(_request: Any) -> ScriptedResponse:+ calls["n"] += 1+ if calls["n"] == 1:+ return ScriptedResponse(status=429, json={})+ return ScriptedResponse(json={"campaigns": []})++ result = _driver().run(+ "campaign_stats",+ answer,+ incremental_field="date",+ source_cursor=_no_stored_cursor(),+ db_incremental_field_last_value=date(2026, 10, 15),+ )++ assert result.raised is None+ # One extra request over the plain day-count: the first day's 429 is retried once, then+ # every day (including the retried one) answers cleanly.+ assert len(result.requests) == 9++ @time_machine.travel(NOW, tick=False)+ def test_a_sustained_rate_limit_exhausts_retries_and_raises_the_marker(self) -> None:+ result = _driver().run(+ "campaign_stats",+ always(ScriptedResponse(status=429, json={})),+ incremental_field="date",+ source_cursor=_no_stored_cursor(),+ db_incremental_field_last_value=date(2026, 10, 15),+ )++ assert isinstance(result.raised, BlueshiftRetryableError)+ assert "Blueshift API error (retryable): status=429" in str(result.raised)+ assert len(result.requests) == MAX_STATS_RETRY_ATTEMPTS+@time_machine.travel(NOW, tick=False)def test_rejected_key_fails_the_run_and_keeps_the_key_out_of_logs(self) -> None:result = _driver().run(@@ -220,3 +262,21 @@ class TestBlueshiftValidateCredentials:assert valid is expected_validassert (message is None) is expected_valid+++class TestBlueshiftRetryableErrors:+ @parameterized.expand(+ [+ (+ "exhausted_rate_limit",+ "Blueshift API error (retryable): status=429, url=https://api.getblueshift.com/api/v1/campaigns.json",+ ),+ (+ "exhausted_5xx",+ "Blueshift API error (retryable): status=503, url=https://api.getblueshift.com/api/v1/campaigns.json",+ ),+ ]+ )+ def test_retryable_errors_match_exhausted_retry(self, _name: str, observed_error: str) -> None:+ retryable_errors = BlueshiftSource().get_retryable_errors()+ assert any(key in observed_error for key in retryable_errors)diff --git a/products/warehouse_sources/sources/sdk/__init__.py b/products/warehouse_sources/sources/sdk/__init__.pyindex 66e85f639162..650522dd2c23 100644--- a/products/warehouse_sources/sources/sdk/__init__.py+++ b/products/warehouse_sources/sources/sdk/__init__.py@@ -116,6 +116,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.httrequest_deadline,retry_with_deadline,stop_at_request_deadline,+ suspend_adapter_retries,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.http.observer import redact_request_urlsfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.http.transport import (@@ -504,6 +505,7 @@ __all__ = ["str_to_optional_list","strip_oauth2_permanent_marker","submit_with_context",+ "suspend_adapter_retries","unreachable_vendor_message","update_sync_type_config_keys","uses_brotli_compression",