:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): skip the capped wait when retry-after passes the deadline (#120485)
Tom Owers7c518184e95d
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/http/tests/test_request_deadline.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/http/tests/test_request_deadline.pyindex cc9c8403a186..454332c58974 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/http/tests/test_request_deadline.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/http/tests/test_request_deadline.py@@ -91,12 +91,12 @@ class _Run:error: BaseException | None-def _run(send: Send, script: Sequence[Answer]) -> _Run:+def _run(send: Send, script: Sequence[Answer], budget: float = BUDGET) -> _Run:vendor = _StallingVendor(script)error: BaseException | None = Nonewith (override_settings(- DATA_WAREHOUSE_SOURCE_RETRY_BUDGET_SECONDS=BUDGET,+ DATA_WAREHOUSE_SOURCE_RETRY_BUDGET_SECONDS=budget,DATA_WAREHOUSE_SOURCE_READ_TIMEOUT_SECONDS=READ_TIMEOUT,),fake_environment(vendor) as network,@@ -162,6 +162,14 @@ class TestRequestDeadline:assert run.vendor.tries[3].read_timeout == pytest.approx(READ_TIMEOUT, abs=SLACK)+ def test_a_retry_after_past_the_deadline_gets_no_capped_wait(self) -> None:+ # The budget leaves room for the 120 s cap on a wait, but not for the day that the server asks for.+ run = _run(_through_tracked_session, [_status(429, 86400)] * 3, budget=300)++ assert isinstance(run.error, requests.HTTPError)+ assert len(run.vendor.tries) == 1+ assert run.elapsed == pytest.approx(0, abs=SLACK)+def test_a_caller_deadline_bounds_the_adapter_retries(self) -> None:def send() -> None:with request_deadline(time.monotonic() + 50):diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/http/transport.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/http/transport.pyindex c22f967b3208..5f1293bcb063 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/http/transport.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/http/transport.py@@ -174,7 +174,8 @@ class BoundedRetry(Retry):- All tries of one request end by one deadline. In `TrackedHTTPAdapter` it is the requestdeadline (see `request_deadline`). Outside it, it is `retry_budget_seconds()` after the firstfailure. No wait starts that would end at or after the deadline: the last response or error- goes to the caller.+ goes to the caller. A `Retry-After` counts with the delay that the server asked for, before+ the cap, so a server that asks for a day gets its 429 back at once and no capped wait.- A wait ends early when the worker starts to shut down, so the caller can hand the run off."""@@ -206,15 +207,17 @@ class BoundedRetry(Retry):deadline = _request_deadline.get()if deadline is None:deadline = now + retry_budget_seconds() if self._retry_deadline is None else self._retry_deadline- if now + retry._next_wait(response) >= deadline:+ if now + retry._requested_wait(response) >= deadline:raise MaxRetryError(_pool, url or "", error or ResponseError("retry budget spent"))retry._retry_deadline = deadlinereturn retry- def _next_wait(self, response: Any) -> float:- # The same choice as `Retry.sleep`, which urllib3 calls on the object that `increment` returns.+ def _requested_wait(self, response: Any) -> float:+ # The same choice as `Retry.sleep`, which urllib3 calls on the object that `increment` returns,+ # but with the delay the server asked for and not the capped one. A capped wait ends before the+ # server is ready, so a retry after it fails again and only holds the worker.if self.respect_retry_after_header and response is not None:- retry_after = self.get_retry_after(response)+ retry_after = super().get_retry_after(response)if retry_after:return retry_afterreturn self.get_backoff_time()diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txtindex fe778d40d5f0..8da25244d710 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt@@ -1,167 +1,166 @@Adjust fail:safe-pointAdyen fail:safe-point-Airbrake fail:rate-limit-budget,stall-budget-AlphaVantage fail:rate-limit-budget,stall-budget-AmazonAds fail:rate-limit-budget,stall-budget+Airbrake fail:stall-budget+AlphaVantage fail:stall-budget+AmazonAds fail:stall-budgetAmazonS3 not-checkable:schemas-Amplitude fail:rate-limit-budget,stall-budget+Amplitude fail:stall-budgetAnthropic fail:safe-pointApollo fail:rate-limit-budget,stall-budgetAppLovin fail:safe-pointAppStoreConnect not-checkable:no-requestAppdynamics fail:stall-budget-Appfigures fail:rate-limit-budget,stall-budget-Appfollow fail:rate-limit-budget,stall-budget+Appfigures fail:stall-budget+Appfollow fail:stall-budgetAppleSearchAds not-checkable:no-request-AppsFlyer fail:rate-limit-budget,safe-point,stall-budget+AppsFlyer fail:safe-point,stall-budgetAppsignal fail:safe-point,stall-budgetArgocd fail:stall-budgetAttentive not-checkable:database-Automox fail:rate-limit-budget,stall-budget-Aviator fail:rate-limit-budget,stall-budget-Awin fail:rate-limit-budget,stall-budget+Automox fail:stall-budget+Aviator fail:stall-budget+Awin fail:stall-budgetAwsBudgets fail:rate-limit-budgetAwsCostExplorer fail:rate-limit-budget,safe-point,stall-budgetAwsOrganizations fail:stall-budgetAwsWaf fail:stall-budgetAzureCostManagement not-checkable:no-request-AzureDevOps fail:rate-limit-budget,stall-budget+AzureDevOps fail:stall-budgetBaserow not-checkable:schemasBigMailer fail:safe-pointBigQuery not-checkable:driver:google.cloudBingAds not-checkable:database-Bitbucket fail:rate-limit-budget,safe-point,stall-budget+Bitbucket fail:safe-point,stall-budgetBraintree fail:stall-budgetBreezometer not-checkable:no-request-Bugsnag fail:rate-limit-budget,safe-point,stall-budget-BuildBetter fail:rate-limit-budget+Bugsnag fail:safe-point,stall-budgetCalCom fail:safe-pointCanvasLms fail:safe-point-CareQualityCommission fail:rate-limit-budget,stall-budget+CareQualityCommission fail:stall-budgetChatwoot not-checkable:database-Checkmarx fail:rate-limit-budget,stall-budget-Cimis fail:rate-limit-budget,stall-budget+Checkmarx fail:stall-budget+Cimis fail:stall-budgetCircleCI fail:rate-limit-budget,safe-point,stall-budgetCircleciInsights fail:rate-limit-budget,safe-point,stall-budgetCiscoDuo not-checkable:no-requestClari fail:stall-budgetClickHouse not-checkable:schemas-ClickhouseCloud fail:rate-limit-budget,stall-budget+ClickhouseCloud fail:stall-budgetCoda fail:safe-point-Codacy fail:rate-limit-budget,stall-budget-Codecov fail:rate-limit-budget,safe-point,stall-budget+Codacy fail:stall-budget+Codecov fail:safe-point,stall-budgetCody fail:stall-budget-CoinApi fail:rate-limit-budget,stall-budget+CoinApi fail:stall-budgetCoinMarketCap fail:safe-point-Commercetools fail:rate-limit-budget,stall-budget-Concord fail:rate-limit-budget,stall-budget-ConfluentCloud fail:rate-limit-budget,stall-budget+Commercetools fail:stall-budget+Concord fail:stall-budget+ConfluentCloud fail:stall-budgetConvex not-checkable:schemasCoralogix fail:stall-budgetCosmosDB not-checkable:schemas-Coupa fail:rate-limit-budget,stall-budget+Coupa fail:stall-budgetCoveralls not-checkable:no-request-CratesIO fail:rate-limit-budget,stall-budget-Cronitor fail:rate-limit-budget,stall-budget+CratesIO fail:stall-budget+Cronitor fail:stall-budgetCursor fail:safe-point,stall-budgetCustom not-checkable:schemas-CustomerIO fail:rate-limit-budget,stall-budget-Customerly fail:rate-limit-budget,stall-budget+CustomerIO fail:stall-budget+Customerly fail:stall-budgetDagsterCloud fail:stall-budgetDataForSEO fail:stall-budgetDatabricks not-checkable:driver:databricks-Datadog fail:rate-limit-budget,safe-point,stall-budget-Datahub fail:rate-limit-budget,stall-budget-Dbt fail:rate-limit-budget,stall-budget-Decagon fail:rate-limit-budget,stall-budget+Datadog fail:safe-point,stall-budget+Datahub fail:stall-budget+Dbt fail:stall-budget+Decagon fail:stall-budgetDeepsource fail:rate-limit-budget,stall-budgetDepot not-checkable:no-requestDiscourse fail:safe-point-Dixa fail:rate-limit-budget,safe-point,stall-budget-Dockerhub fail:rate-limit-budget,safe-point,stall-budget+Dixa fail:safe-point,stall-budget+Dockerhub fail:safe-point,stall-budgetDocusign not-checkable:no-request-Doppler fail:rate-limit-budget,stall-budget-DropboxSign fail:rate-limit-budget,stall-budget+Doppler fail:stall-budget+DropboxSign fail:stall-budgetDynamics365 not-checkable:no-requestDynamoDB not-checkable:schemas-Easypost fail:rate-limit-budget,stall-budget+Easypost fail:stall-budgetEcbDataPortal fail:safe-pointElasticsearch not-checkable:schemas-Everhour fail:rate-limit-budget,stall-budget-ExchangeRatesApi fail:rate-limit-budget,stall-budget-Fastly fail:rate-limit-budget,safe-point,stall-budget+Everhour fail:stall-budget+ExchangeRatesApi fail:stall-budget+Fastly fail:safe-point,stall-budgetFeaturebase not-checkable:databaseFinage fail:stall-budget-FinancialModelling fail:rate-limit-budget,safe-point,stall-budget-Finnhub fail:rate-limit-budget,stall-budget-Finnworlds fail:rate-limit-budget,stall-budget+FinancialModelling fail:safe-point,stall-budget+Finnhub fail:stall-budget+Finnworlds fail:stall-budgetFirebase not-checkable:schemasFlagsmith fail:safe-point,stall-budget-Formbricks fail:rate-limit-budget,safe-point,stall-budget+Formbricks fail:safe-point,stall-budgetFramer not-checkable:no-request-Freshsales fail:rate-limit-budget,stall-budget-Frill fail:rate-limit-budget,stall-budget+Freshsales fail:stall-budget+Frill fail:stall-budgetG2 fail:safe-point-GNews fail:rate-limit-budget,stall-budget+GNews fail:stall-budgetGainsightPx fail:safe-pointGcpCloudBilling not-checkable:no-request-Gerrit fail:rate-limit-budget,stall-budget+Gerrit fail:stall-budgetGhost not-checkable:no-request-GitLab fail:rate-limit-budget,stall-budget-Gitea fail:rate-limit-budget,stall-budget-Gitguardian fail:rate-limit-budget,safe-point,stall-budget+GitLab fail:stall-budget+Gitea fail:stall-budget+Gitguardian fail:safe-point,stall-budgetGithub not-checkable:schemas-Gladly fail:rate-limit-budget,stall-budget-Gong fail:rate-limit-budget,stall-budget+Gladly fail:stall-budget+Gong fail:stall-budgetGoogleAds not-checkable:driver:google.adsGoogleAnalytics not-checkable:databaseGooglePageSpeedInsights fail:stall-budgetGooglePlayConsole not-checkable:no-requestGoogleSearchConsole not-checkable:databaseGoogleSheets not-checkable:schemas-Gorgias fail:rate-limit-budget,stall-budget+Gorgias fail:stall-budgetGrafana fail:stall-budgetGrafanaIRM not-checkable:no-request-Gridly fail:rate-limit-budget,stall-budget+Gridly fail:stall-budgetGroq fail:safe-point-Harvey fail:rate-limit-budget,stall-budget+Harvey fail:stall-budgetHatchet not-checkable:no-request-Healthchecks fail:rate-limit-budget,stall-budget+Healthchecks fail:stall-budgetHelicone fail:stall-budgetHelpScout not-checkable:databaseHitpay fail:safe-point-Honeybadger fail:rate-limit-budget,safe-point,stall-budget-Honeycomb fail:rate-limit-budget,stall-budget+Honeybadger fail:safe-point,stall-budget+Honeycomb fail:stall-budgetHubspot not-checkable:database-Hyperspell fail:rate-limit-budget,safe-point,stall-budget-IP2Whois fail:rate-limit-budget,stall-budget-Inngest fail:rate-limit-budget,stall-budget+Hyperspell fail:safe-point,stall-budget+IP2Whois fail:stall-budget+Inngest fail:stall-budgetInstagram not-checkable:database-Instana fail:rate-limit-budget,safe-point,stall-budget+Instana fail:safe-point,stall-budgetIntercom not-checkable:database-JamfPro fail:rate-limit-budget,stall-budget+JamfPro fail:stall-budgetJellyfish fail:stall-budget-Jenkins fail:rate-limit-budget,stall-budget-JfrogArtifactory fail:rate-limit-budget,stall-budget-Jumpcloud fail:rate-limit-budget,stall-budget+Jenkins fail:stall-budget+JfrogArtifactory fail:stall-budget+Jumpcloud fail:stall-budgetKafka not-checkable:driver:confluent_kafkaKandji fail:safe-pointKapaAI fail:safe-point-Kernel fail:rate-limit-budget,safe-point,stall-budget+Kernel fail:safe-point,stall-budgetKestra not-checkable:no-requestKlaus fail:rate-limit-budget,stall-budgetKlaviyo fail:rate-limit-budget,safe-point,stall-budgetKongKonnect fail:stall-budget-Kubecost fail:rate-limit-budget,stall-budget-Lacework fail:rate-limit-budget,safe-point,stall-budget-LangSmith fail:rate-limit-budget,stall-budget+Kubecost fail:stall-budget+Lacework fail:safe-point,stall-budget+LangSmith fail:stall-budgetLangfuse fail:rate-limit-budget,stall-budgetLinear not-checkable:database-Linearb fail:rate-limit-budget,stall-budget+Linearb fail:stall-budgetLinkedinAds not-checkable:databaseLinkrunner fail:rate-limit-budget,stall-budget-LlamaCloud fail:rate-limit-budget,safe-point,stall-budget-Lob fail:rate-limit-budget,stall-budget+LlamaCloud fail:safe-point,stall-budget+Lob fail:stall-budgetLogicmonitor not-checkable:no-requestLogzIO fail:stall-budgetLovable fail:safe-point@@ -171,11 +170,11 @@ MailerSend fail:safe-pointMailgun fail:rate-limit-budget,stall-budgetMatomo fail:stall-budgetMem0 fail:safe-point-Mention fail:rate-limit-budget,safe-point,stall-budget+Mention fail:safe-point,stall-budgetMetaAds not-checkable:database-Metaplane fail:rate-limit-budget,stall-budget+Metaplane fail:stall-budgetMeteostat fail:safe-point-MistralAI fail:rate-limit-budget,stall-budget+MistralAI fail:stall-budgetMixpanel fail:rate-limit-budget,stall-budgetMoEngage fail:safe-pointModal not-checkable:driver:grpclib@@ -185,114 +184,114 @@ MonteCarlo fail:safe-point,stall-budgetMotherduck not-checkable:driver:duckdbMyHours fail:safe-pointMySQL not-checkable:driver:psycopg-NebiusAI fail:rate-limit-budget,stall-budget+NebiusAI fail:stall-budgetNeo4j not-checkable:schemasNeon not-checkable:driver:psycopgNewRelic fail:safe-point,stall-budget-NewYorkTimes fail:rate-limit-budget,stall-budget+NewYorkTimes fail:stall-budgetNorthflank fail:rate-limit-budget,stall-budgetNotion fail:rate-limit-budget,safe-point,stall-budgetNpmRegistry fail:safe-point-Nuget fail:rate-limit-budget,stall-budget+Nuget fail:stall-budgetOctopusDeploy fail:stall-budgetOpenExchangeRates fail:stall-budgetOpenMeteo not-checkable:no-request-OpenRouter fail:rate-limit-budget,stall-budget+OpenRouter fail:stall-budgetOpenWeather not-checkable:no-requestOptimizely fail:safe-pointOrtto fail:safe-point,stall-budget-PabblySubscriptionsBilling fail:rate-limit-budget,safe-point,stall-budget-Packagist fail:rate-limit-budget,stall-budget+PabblySubscriptionsBilling fail:safe-point,stall-budget+Packagist fail:stall-budgetPardot not-checkable:databasePayFit fail:stall-budget-Pendo fail:rate-limit-budget,stall-budget-Persona fail:rate-limit-budget,stall-budget+Pendo fail:stall-budget+Persona fail:stall-budgetPinterestAds not-checkable:databasePipedrive not-checkable:databasePipeliner fail:safe-pointPlaid fail:stall-budgetPlanetScaleMySQL not-checkable:driver:psycopgPlanetScalePostgres not-checkable:driver:psycopg-PlatformSh fail:rate-limit-budget,stall-budget+PlatformSh fail:stall-budgetPlunk fail:safe-pointPostgres not-checkable:driver:psycopgPowerBiAdmin fail:safe-point-Printify fail:rate-limit-budget,stall-budget+Printify fail:stall-budgetProofpointTap not-checkable:no-request-PulumiCloud fail:rate-limit-budget,stall-budget-PyPI fail:rate-limit-budget,stall-budget-Pylon fail:rate-limit-budget,safe-point,stall-budget+PulumiCloud fail:stall-budget+PyPI fail:stall-budget+Pylon fail:safe-point,stall-budgetQualysVmdr fail:rate-limit-budget,stall-budgetRailway fail:stall-budgetRedditAds not-checkable:databaseRedshift not-checkable:driver:psycopg-Render fail:rate-limit-budget,stall-budget-Replicate fail:rate-limit-budget,safe-point,stall-budget+Render fail:stall-budget+Replicate fail:safe-point,stall-budgetRevenueCat fail:safe-point-Rollbar fail:rate-limit-budget,stall-budget-Rubygems fail:rate-limit-budget,stall-budget-RunPod fail:rate-limit-budget,safe-point,stall-budget+Rollbar fail:stall-budget+Rubygems fail:stall-budget+RunPod fail:safe-point,stall-budgetSFTP not-checkable:transport-SageHR fail:rate-limit-budget,safe-point,stall-budget+SageHR fail:safe-point,stall-budgetSalesforce not-checkable:database-Segment fail:rate-limit-budget,stall-budget+Segment fail:stall-budgetSevdesk fail:safe-point-Shippo fail:rate-limit-budget,safe-point,stall-budget+Shippo fail:safe-point,stall-budgetShopify fail:stall-budgetSigNoz fail:stall-budgetSingular fail:stall-budgetSlack not-checkable:schemas-Smaily fail:rate-limit-budget,safe-point,stall-budget+Smaily fail:safe-point,stall-budgetSnapchatAds not-checkable:databaseSnowflake not-checkable:driver:snowflake-Snowplow fail:rate-limit-budget,stall-budget-Sonarqube fail:rate-limit-budget,stall-budget-SonatypeNexus fail:rate-limit-budget,stall-budget+Snowplow fail:stall-budget+Sonarqube fail:stall-budget+SonatypeNexus fail:stall-budgetSourcegraph fail:stall-budgetSpacelift fail:stall-budget-SplitIo fail:rate-limit-budget,stall-budget-SplunkObservabilityCloud fail:rate-limit-budget,stall-budget-Squadcast fail:rate-limit-budget,stall-budget-Square fail:rate-limit-budget,safe-point,stall-budget-Squarespace fail:rate-limit-budget,stall-budget-Statuscake fail:rate-limit-budget,stall-budget+SplitIo fail:stall-budget+SplunkObservabilityCloud fail:stall-budget+Squadcast fail:stall-budget+Square fail:safe-point,stall-budget+Squarespace fail:stall-budget+Statuscake fail:stall-budgetStripe not-checkable:databaseStytch fail:stall-budgetSumoLogic fail:safe-point,stall-budgetSupabase not-checkable:driver:psycopg-Swarmia fail:rate-limit-budget,safe-point,stall-budget-Taboola fail:rate-limit-budget,stall-budget-Tailscale fail:rate-limit-budget,safe-point,stall-budget+Swarmia fail:safe-point,stall-budget+Taboola fail:stall-budget+Tailscale fail:safe-point,stall-budgetTawkTo fail:stall-budget-Teamcity fail:rate-limit-budget,stall-budget+Teamcity fail:stall-budgetTemporalIO not-checkable:driver:temporalio.clientTenableVulnerabilityManagement fail:stall-budget-TerraformCloud fail:rate-limit-budget,safe-point,stall-budget-Testrail fail:rate-limit-budget,stall-budget+TerraformCloud fail:safe-point,stall-budget+Testrail fail:stall-budgetTikTokAds not-checkable:database-TravisCI fail:rate-limit-budget,stall-budget-TriggerDev fail:rate-limit-budget,stall-budget+TravisCI fail:stall-budget+TriggerDev fail:stall-budgetTrino not-checkable:schemasTrunkIo fail:safe-pointTwitterAds not-checkable:database-Ubidots fail:rate-limit-budget,safe-point,stall-budget+Ubidots fail:safe-point,stall-budgetUnleash fail:safe-point-Unstructured fail:rate-limit-budget,stall-budget+Unstructured fail:stall-budgetUptimerobot fail:stall-budget-Usersnap fail:rate-limit-budget,stall-budget-Vapi fail:rate-limit-budget,stall-budget+Usersnap fail:stall-budget+Vapi fail:stall-budgetVeracode fail:stall-budget-Vercel fail:rate-limit-budget,stall-budget+Vercel fail:stall-budgetVitally fail:safe-pointWeightsAndBiases fail:stall-budgetWikipediaPageviews fail:safe-point-Wordpress fail:rate-limit-budget,stall-budget+Wordpress fail:stall-budgetWrike not-checkable:no-requestWritesonic fail:rate-limit-budget,safe-point,stall-budgetYouTubeAnalytics not-checkable:databaseZapSign fail:safe-point-ZapierSupportedStorage fail:rate-limit-budget,safe-point,stall-budget-Zenduty fail:rate-limit-budget,safe-point,stall-budget-Zep fail:rate-limit-budget,stall-budget+ZapierSupportedStorage fail:safe-point,stall-budget+Zenduty fail:safe-point,stall-budget+Zep fail:stall-budgetshared-path:rest-client-paginate fail:safe-pointshared-path:rest-framework-wrapped-resource fail:safe-point