Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): skip the capped wait when retry-after passes the deadline (#120485)

Tom Owers7c518184e95d
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/http/tests/test_request_deadline.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/http/tests/test_request_deadline.py
index cc9c8403a186..454332c58974 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/http/tests/test_request_deadline.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/http/tests/test_request_deadline.py
@@ -91,12 +91,12 @@ class _Run:
error: BaseException | None
-def _run(send: Send, script: Sequence[Answer]) -> _Run:
+def _run(send: Send, script: Sequence[Answer], budget: float = BUDGET) -> _Run:
vendor = _StallingVendor(script)
error: BaseException | None = None
with (
override_settings(
- DATA_WAREHOUSE_SOURCE_RETRY_BUDGET_SECONDS=BUDGET,
+ DATA_WAREHOUSE_SOURCE_RETRY_BUDGET_SECONDS=budget,
DATA_WAREHOUSE_SOURCE_READ_TIMEOUT_SECONDS=READ_TIMEOUT,
),
fake_environment(vendor) as network,
@@ -162,6 +162,14 @@ class TestRequestDeadline:
assert run.vendor.tries[3].read_timeout == pytest.approx(READ_TIMEOUT, abs=SLACK)
+ def test_a_retry_after_past_the_deadline_gets_no_capped_wait(self) -> None:
+ # The budget leaves room for the 120 s cap on a wait, but not for the day that the server asks for.
+ run = _run(_through_tracked_session, [_status(429, 86400)] * 3, budget=300)
+
+ assert isinstance(run.error, requests.HTTPError)
+ assert len(run.vendor.tries) == 1
+ assert run.elapsed == pytest.approx(0, abs=SLACK)
+
def test_a_caller_deadline_bounds_the_adapter_retries(self) -> None:
def send() -> None:
with request_deadline(time.monotonic() + 50):
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/http/transport.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/http/transport.py
index c22f967b3208..5f1293bcb063 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/http/transport.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/http/transport.py
@@ -174,7 +174,8 @@ class BoundedRetry(Retry):
- All tries of one request end by one deadline. In `TrackedHTTPAdapter` it is the request
deadline (see `request_deadline`). Outside it, it is `retry_budget_seconds()` after the first
failure. No wait starts that would end at or after the deadline: the last response or error
- goes to the caller.
+ goes to the caller. A `Retry-After` counts with the delay that the server asked for, before
+ the cap, so a server that asks for a day gets its 429 back at once and no capped wait.
- A wait ends early when the worker starts to shut down, so the caller can hand the run off.
"""
@@ -206,15 +207,17 @@ class BoundedRetry(Retry):
deadline = _request_deadline.get()
if deadline is None:
deadline = now + retry_budget_seconds() if self._retry_deadline is None else self._retry_deadline
- if now + retry._next_wait(response) >= deadline:
+ if now + retry._requested_wait(response) >= deadline:
raise MaxRetryError(_pool, url or "", error or ResponseError("retry budget spent"))
retry._retry_deadline = deadline
return retry
- def _next_wait(self, response: Any) -> float:
- # The same choice as `Retry.sleep`, which urllib3 calls on the object that `increment` returns.
+ def _requested_wait(self, response: Any) -> float:
+ # The same choice as `Retry.sleep`, which urllib3 calls on the object that `increment` returns,
+ # but with the delay the server asked for and not the capped one. A capped wait ends before the
+ # server is ready, so a retry after it fails again and only holds the worker.
if self.respect_retry_after_header and response is not None:
- retry_after = self.get_retry_after(response)
+ retry_after = super().get_retry_after(response)
if retry_after:
return retry_after
return self.get_backoff_time()
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt
index fe778d40d5f0..8da25244d710 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt
@@ -1,167 +1,166 @@
Adjust fail:safe-point
Adyen fail:safe-point
-Airbrake fail:rate-limit-budget,stall-budget
-AlphaVantage fail:rate-limit-budget,stall-budget
-AmazonAds fail:rate-limit-budget,stall-budget
+Airbrake fail:stall-budget
+AlphaVantage fail:stall-budget
+AmazonAds fail:stall-budget
AmazonS3 not-checkable:schemas
-Amplitude fail:rate-limit-budget,stall-budget
+Amplitude fail:stall-budget
Anthropic fail:safe-point
Apollo fail:rate-limit-budget,stall-budget
AppLovin fail:safe-point
AppStoreConnect not-checkable:no-request
Appdynamics fail:stall-budget
-Appfigures fail:rate-limit-budget,stall-budget
-Appfollow fail:rate-limit-budget,stall-budget
+Appfigures fail:stall-budget
+Appfollow fail:stall-budget
AppleSearchAds not-checkable:no-request
-AppsFlyer fail:rate-limit-budget,safe-point,stall-budget
+AppsFlyer fail:safe-point,stall-budget
Appsignal fail:safe-point,stall-budget
Argocd fail:stall-budget
Attentive not-checkable:database
-Automox fail:rate-limit-budget,stall-budget
-Aviator fail:rate-limit-budget,stall-budget
-Awin fail:rate-limit-budget,stall-budget
+Automox fail:stall-budget
+Aviator fail:stall-budget
+Awin fail:stall-budget
AwsBudgets fail:rate-limit-budget
AwsCostExplorer fail:rate-limit-budget,safe-point,stall-budget
AwsOrganizations fail:stall-budget
AwsWaf fail:stall-budget
AzureCostManagement not-checkable:no-request
-AzureDevOps fail:rate-limit-budget,stall-budget
+AzureDevOps fail:stall-budget
Baserow not-checkable:schemas
BigMailer fail:safe-point
BigQuery not-checkable:driver:google.cloud
BingAds not-checkable:database
-Bitbucket fail:rate-limit-budget,safe-point,stall-budget
+Bitbucket fail:safe-point,stall-budget
Braintree fail:stall-budget
Breezometer not-checkable:no-request
-Bugsnag fail:rate-limit-budget,safe-point,stall-budget
-BuildBetter fail:rate-limit-budget
+Bugsnag fail:safe-point,stall-budget
CalCom fail:safe-point
CanvasLms fail:safe-point
-CareQualityCommission fail:rate-limit-budget,stall-budget
+CareQualityCommission fail:stall-budget
Chatwoot not-checkable:database
-Checkmarx fail:rate-limit-budget,stall-budget
-Cimis fail:rate-limit-budget,stall-budget
+Checkmarx fail:stall-budget
+Cimis fail:stall-budget
CircleCI fail:rate-limit-budget,safe-point,stall-budget
CircleciInsights fail:rate-limit-budget,safe-point,stall-budget
CiscoDuo not-checkable:no-request
Clari fail:stall-budget
ClickHouse not-checkable:schemas
-ClickhouseCloud fail:rate-limit-budget,stall-budget
+ClickhouseCloud fail:stall-budget
Coda fail:safe-point
-Codacy fail:rate-limit-budget,stall-budget
-Codecov fail:rate-limit-budget,safe-point,stall-budget
+Codacy fail:stall-budget
+Codecov fail:safe-point,stall-budget
Cody fail:stall-budget
-CoinApi fail:rate-limit-budget,stall-budget
+CoinApi fail:stall-budget
CoinMarketCap fail:safe-point
-Commercetools fail:rate-limit-budget,stall-budget
-Concord fail:rate-limit-budget,stall-budget
-ConfluentCloud fail:rate-limit-budget,stall-budget
+Commercetools fail:stall-budget
+Concord fail:stall-budget
+ConfluentCloud fail:stall-budget
Convex not-checkable:schemas
Coralogix fail:stall-budget
CosmosDB not-checkable:schemas
-Coupa fail:rate-limit-budget,stall-budget
+Coupa fail:stall-budget
Coveralls not-checkable:no-request
-CratesIO fail:rate-limit-budget,stall-budget
-Cronitor fail:rate-limit-budget,stall-budget
+CratesIO fail:stall-budget
+Cronitor fail:stall-budget
Cursor fail:safe-point,stall-budget
Custom not-checkable:schemas
-CustomerIO fail:rate-limit-budget,stall-budget
-Customerly fail:rate-limit-budget,stall-budget
+CustomerIO fail:stall-budget
+Customerly fail:stall-budget
DagsterCloud fail:stall-budget
DataForSEO fail:stall-budget
Databricks not-checkable:driver:databricks
-Datadog fail:rate-limit-budget,safe-point,stall-budget
-Datahub fail:rate-limit-budget,stall-budget
-Dbt fail:rate-limit-budget,stall-budget
-Decagon fail:rate-limit-budget,stall-budget
+Datadog fail:safe-point,stall-budget
+Datahub fail:stall-budget
+Dbt fail:stall-budget
+Decagon fail:stall-budget
Deepsource fail:rate-limit-budget,stall-budget
Depot not-checkable:no-request
Discourse fail:safe-point
-Dixa fail:rate-limit-budget,safe-point,stall-budget
-Dockerhub fail:rate-limit-budget,safe-point,stall-budget
+Dixa fail:safe-point,stall-budget
+Dockerhub fail:safe-point,stall-budget
Docusign not-checkable:no-request
-Doppler fail:rate-limit-budget,stall-budget
-DropboxSign fail:rate-limit-budget,stall-budget
+Doppler fail:stall-budget
+DropboxSign fail:stall-budget
Dynamics365 not-checkable:no-request
DynamoDB not-checkable:schemas
-Easypost fail:rate-limit-budget,stall-budget
+Easypost fail:stall-budget
EcbDataPortal fail:safe-point
Elasticsearch not-checkable:schemas
-Everhour fail:rate-limit-budget,stall-budget
-ExchangeRatesApi fail:rate-limit-budget,stall-budget
-Fastly fail:rate-limit-budget,safe-point,stall-budget
+Everhour fail:stall-budget
+ExchangeRatesApi fail:stall-budget
+Fastly fail:safe-point,stall-budget
Featurebase not-checkable:database
Finage fail:stall-budget
-FinancialModelling fail:rate-limit-budget,safe-point,stall-budget
-Finnhub fail:rate-limit-budget,stall-budget
-Finnworlds fail:rate-limit-budget,stall-budget
+FinancialModelling fail:safe-point,stall-budget
+Finnhub fail:stall-budget
+Finnworlds fail:stall-budget
Firebase not-checkable:schemas
Flagsmith fail:safe-point,stall-budget
-Formbricks fail:rate-limit-budget,safe-point,stall-budget
+Formbricks fail:safe-point,stall-budget
Framer not-checkable:no-request
-Freshsales fail:rate-limit-budget,stall-budget
-Frill fail:rate-limit-budget,stall-budget
+Freshsales fail:stall-budget
+Frill fail:stall-budget
G2 fail:safe-point
-GNews fail:rate-limit-budget,stall-budget
+GNews fail:stall-budget
GainsightPx fail:safe-point
GcpCloudBilling not-checkable:no-request
-Gerrit fail:rate-limit-budget,stall-budget
+Gerrit fail:stall-budget
Ghost not-checkable:no-request
-GitLab fail:rate-limit-budget,stall-budget
-Gitea fail:rate-limit-budget,stall-budget
-Gitguardian fail:rate-limit-budget,safe-point,stall-budget
+GitLab fail:stall-budget
+Gitea fail:stall-budget
+Gitguardian fail:safe-point,stall-budget
Github not-checkable:schemas
-Gladly fail:rate-limit-budget,stall-budget
-Gong fail:rate-limit-budget,stall-budget
+Gladly fail:stall-budget
+Gong fail:stall-budget
GoogleAds not-checkable:driver:google.ads
GoogleAnalytics not-checkable:database
GooglePageSpeedInsights fail:stall-budget
GooglePlayConsole not-checkable:no-request
GoogleSearchConsole not-checkable:database
GoogleSheets not-checkable:schemas
-Gorgias fail:rate-limit-budget,stall-budget
+Gorgias fail:stall-budget
Grafana fail:stall-budget
GrafanaIRM not-checkable:no-request
-Gridly fail:rate-limit-budget,stall-budget
+Gridly fail:stall-budget
Groq fail:safe-point
-Harvey fail:rate-limit-budget,stall-budget
+Harvey fail:stall-budget
Hatchet not-checkable:no-request
-Healthchecks fail:rate-limit-budget,stall-budget
+Healthchecks fail:stall-budget
Helicone fail:stall-budget
HelpScout not-checkable:database
Hitpay fail:safe-point
-Honeybadger fail:rate-limit-budget,safe-point,stall-budget
-Honeycomb fail:rate-limit-budget,stall-budget
+Honeybadger fail:safe-point,stall-budget
+Honeycomb fail:stall-budget
Hubspot not-checkable:database
-Hyperspell fail:rate-limit-budget,safe-point,stall-budget
-IP2Whois fail:rate-limit-budget,stall-budget
-Inngest fail:rate-limit-budget,stall-budget
+Hyperspell fail:safe-point,stall-budget
+IP2Whois fail:stall-budget
+Inngest fail:stall-budget
Instagram not-checkable:database
-Instana fail:rate-limit-budget,safe-point,stall-budget
+Instana fail:safe-point,stall-budget
Intercom not-checkable:database
-JamfPro fail:rate-limit-budget,stall-budget
+JamfPro fail:stall-budget
Jellyfish fail:stall-budget
-Jenkins fail:rate-limit-budget,stall-budget
-JfrogArtifactory fail:rate-limit-budget,stall-budget
-Jumpcloud fail:rate-limit-budget,stall-budget
+Jenkins fail:stall-budget
+JfrogArtifactory fail:stall-budget
+Jumpcloud fail:stall-budget
Kafka not-checkable:driver:confluent_kafka
Kandji fail:safe-point
KapaAI fail:safe-point
-Kernel fail:rate-limit-budget,safe-point,stall-budget
+Kernel fail:safe-point,stall-budget
Kestra not-checkable:no-request
Klaus fail:rate-limit-budget,stall-budget
Klaviyo fail:rate-limit-budget,safe-point,stall-budget
KongKonnect fail:stall-budget
-Kubecost fail:rate-limit-budget,stall-budget
-Lacework fail:rate-limit-budget,safe-point,stall-budget
-LangSmith fail:rate-limit-budget,stall-budget
+Kubecost fail:stall-budget
+Lacework fail:safe-point,stall-budget
+LangSmith fail:stall-budget
Langfuse fail:rate-limit-budget,stall-budget
Linear not-checkable:database
-Linearb fail:rate-limit-budget,stall-budget
+Linearb fail:stall-budget
LinkedinAds not-checkable:database
Linkrunner fail:rate-limit-budget,stall-budget
-LlamaCloud fail:rate-limit-budget,safe-point,stall-budget
-Lob fail:rate-limit-budget,stall-budget
+LlamaCloud fail:safe-point,stall-budget
+Lob fail:stall-budget
Logicmonitor not-checkable:no-request
LogzIO fail:stall-budget
Lovable fail:safe-point
@@ -171,11 +170,11 @@ MailerSend fail:safe-point
Mailgun fail:rate-limit-budget,stall-budget
Matomo fail:stall-budget
Mem0 fail:safe-point
-Mention fail:rate-limit-budget,safe-point,stall-budget
+Mention fail:safe-point,stall-budget
MetaAds not-checkable:database
-Metaplane fail:rate-limit-budget,stall-budget
+Metaplane fail:stall-budget
Meteostat fail:safe-point
-MistralAI fail:rate-limit-budget,stall-budget
+MistralAI fail:stall-budget
Mixpanel fail:rate-limit-budget,stall-budget
MoEngage fail:safe-point
Modal not-checkable:driver:grpclib
@@ -185,114 +184,114 @@ MonteCarlo fail:safe-point,stall-budget
Motherduck not-checkable:driver:duckdb
MyHours fail:safe-point
MySQL not-checkable:driver:psycopg
-NebiusAI fail:rate-limit-budget,stall-budget
+NebiusAI fail:stall-budget
Neo4j not-checkable:schemas
Neon not-checkable:driver:psycopg
NewRelic fail:safe-point,stall-budget
-NewYorkTimes fail:rate-limit-budget,stall-budget
+NewYorkTimes fail:stall-budget
Northflank fail:rate-limit-budget,stall-budget
Notion fail:rate-limit-budget,safe-point,stall-budget
NpmRegistry fail:safe-point
-Nuget fail:rate-limit-budget,stall-budget
+Nuget fail:stall-budget
OctopusDeploy fail:stall-budget
OpenExchangeRates fail:stall-budget
OpenMeteo not-checkable:no-request
-OpenRouter fail:rate-limit-budget,stall-budget
+OpenRouter fail:stall-budget
OpenWeather not-checkable:no-request
Optimizely fail:safe-point
Ortto fail:safe-point,stall-budget
-PabblySubscriptionsBilling fail:rate-limit-budget,safe-point,stall-budget
-Packagist fail:rate-limit-budget,stall-budget
+PabblySubscriptionsBilling fail:safe-point,stall-budget
+Packagist fail:stall-budget
Pardot not-checkable:database
PayFit fail:stall-budget
-Pendo fail:rate-limit-budget,stall-budget
-Persona fail:rate-limit-budget,stall-budget
+Pendo fail:stall-budget
+Persona fail:stall-budget
PinterestAds not-checkable:database
Pipedrive not-checkable:database
Pipeliner fail:safe-point
Plaid fail:stall-budget
PlanetScaleMySQL not-checkable:driver:psycopg
PlanetScalePostgres not-checkable:driver:psycopg
-PlatformSh fail:rate-limit-budget,stall-budget
+PlatformSh fail:stall-budget
Plunk fail:safe-point
Postgres not-checkable:driver:psycopg
PowerBiAdmin fail:safe-point
-Printify fail:rate-limit-budget,stall-budget
+Printify fail:stall-budget
ProofpointTap not-checkable:no-request
-PulumiCloud fail:rate-limit-budget,stall-budget
-PyPI fail:rate-limit-budget,stall-budget
-Pylon fail:rate-limit-budget,safe-point,stall-budget
+PulumiCloud fail:stall-budget
+PyPI fail:stall-budget
+Pylon fail:safe-point,stall-budget
QualysVmdr fail:rate-limit-budget,stall-budget
Railway fail:stall-budget
RedditAds not-checkable:database
Redshift not-checkable:driver:psycopg
-Render fail:rate-limit-budget,stall-budget
-Replicate fail:rate-limit-budget,safe-point,stall-budget
+Render fail:stall-budget
+Replicate fail:safe-point,stall-budget
RevenueCat fail:safe-point
-Rollbar fail:rate-limit-budget,stall-budget
-Rubygems fail:rate-limit-budget,stall-budget
-RunPod fail:rate-limit-budget,safe-point,stall-budget
+Rollbar fail:stall-budget
+Rubygems fail:stall-budget
+RunPod fail:safe-point,stall-budget
SFTP not-checkable:transport
-SageHR fail:rate-limit-budget,safe-point,stall-budget
+SageHR fail:safe-point,stall-budget
Salesforce not-checkable:database
-Segment fail:rate-limit-budget,stall-budget
+Segment fail:stall-budget
Sevdesk fail:safe-point
-Shippo fail:rate-limit-budget,safe-point,stall-budget
+Shippo fail:safe-point,stall-budget
Shopify fail:stall-budget
SigNoz fail:stall-budget
Singular fail:stall-budget
Slack not-checkable:schemas
-Smaily fail:rate-limit-budget,safe-point,stall-budget
+Smaily fail:safe-point,stall-budget
SnapchatAds not-checkable:database
Snowflake not-checkable:driver:snowflake
-Snowplow fail:rate-limit-budget,stall-budget
-Sonarqube fail:rate-limit-budget,stall-budget
-SonatypeNexus fail:rate-limit-budget,stall-budget
+Snowplow fail:stall-budget
+Sonarqube fail:stall-budget
+SonatypeNexus fail:stall-budget
Sourcegraph fail:stall-budget
Spacelift fail:stall-budget
-SplitIo fail:rate-limit-budget,stall-budget
-SplunkObservabilityCloud fail:rate-limit-budget,stall-budget
-Squadcast fail:rate-limit-budget,stall-budget
-Square fail:rate-limit-budget,safe-point,stall-budget
-Squarespace fail:rate-limit-budget,stall-budget
-Statuscake fail:rate-limit-budget,stall-budget
+SplitIo fail:stall-budget
+SplunkObservabilityCloud fail:stall-budget
+Squadcast fail:stall-budget
+Square fail:safe-point,stall-budget
+Squarespace fail:stall-budget
+Statuscake fail:stall-budget
Stripe not-checkable:database
Stytch fail:stall-budget
SumoLogic fail:safe-point,stall-budget
Supabase not-checkable:driver:psycopg
-Swarmia fail:rate-limit-budget,safe-point,stall-budget
-Taboola fail:rate-limit-budget,stall-budget
-Tailscale fail:rate-limit-budget,safe-point,stall-budget
+Swarmia fail:safe-point,stall-budget
+Taboola fail:stall-budget
+Tailscale fail:safe-point,stall-budget
TawkTo fail:stall-budget
-Teamcity fail:rate-limit-budget,stall-budget
+Teamcity fail:stall-budget
TemporalIO not-checkable:driver:temporalio.client
TenableVulnerabilityManagement fail:stall-budget
-TerraformCloud fail:rate-limit-budget,safe-point,stall-budget
-Testrail fail:rate-limit-budget,stall-budget
+TerraformCloud fail:safe-point,stall-budget
+Testrail fail:stall-budget
TikTokAds not-checkable:database
-TravisCI fail:rate-limit-budget,stall-budget
-TriggerDev fail:rate-limit-budget,stall-budget
+TravisCI fail:stall-budget
+TriggerDev fail:stall-budget
Trino not-checkable:schemas
TrunkIo fail:safe-point
TwitterAds not-checkable:database
-Ubidots fail:rate-limit-budget,safe-point,stall-budget
+Ubidots fail:safe-point,stall-budget
Unleash fail:safe-point
-Unstructured fail:rate-limit-budget,stall-budget
+Unstructured fail:stall-budget
Uptimerobot fail:stall-budget
-Usersnap fail:rate-limit-budget,stall-budget
-Vapi fail:rate-limit-budget,stall-budget
+Usersnap fail:stall-budget
+Vapi fail:stall-budget
Veracode fail:stall-budget
-Vercel fail:rate-limit-budget,stall-budget
+Vercel fail:stall-budget
Vitally fail:safe-point
WeightsAndBiases fail:stall-budget
WikipediaPageviews fail:safe-point
-Wordpress fail:rate-limit-budget,stall-budget
+Wordpress fail:stall-budget
Wrike not-checkable:no-request
Writesonic fail:rate-limit-budget,safe-point,stall-budget
YouTubeAnalytics not-checkable:database
ZapSign fail:safe-point
-ZapierSupportedStorage fail:rate-limit-budget,safe-point,stall-budget
-Zenduty fail:rate-limit-budget,safe-point,stall-budget
-Zep fail:rate-limit-budget,stall-budget
+ZapierSupportedStorage fail:safe-point,stall-budget
+Zenduty fail:safe-point,stall-budget
+Zep fail:stall-budget
shared-path:rest-client-paginate fail:safe-point
shared-path:rest-framework-wrapped-resource fail:safe-point