Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): bound every source-owned retry loop by one deadline (#121000)

Tom Owers07231dcea11a
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/alpha_vantage/alpha_vantage.py b/products/warehouse_sources/backend/temporal/data_imports/sources/alpha_vantage/alpha_vantage.py
index 703045045d63..1458d9b3f137 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/alpha_vantage/alpha_vantage.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/alpha_vantage/alpha_vantage.py
@@ -7,14 +7,17 @@ from typing import Any
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.backend.temporal.data_imports.sources.alpha_vantage.settings import (
ALPHA_VANTAGE_ENDPOINTS,
AlphaVantageEndpointConfig,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.datetime_utils import parse_datetime_value
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
ALPHA_VANTAGE_BASE_URL = "https://www.alphavantage.co/query"
@@ -106,7 +109,7 @@ def _raise_for_envelope(body: dict[str, Any]) -> None:
raise AlphaVantageAPIError(f"Alpha Vantage API error [rate_limit_or_premium]: {body['Information']}")
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((AlphaVantageRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
# Alpha Vantage's free per-minute throttle resets on a ~60s window, so back off long enough to
@@ -149,7 +152,7 @@ def _raise_for_csv_envelope(csv_text: str) -> None:
raise AlphaVantageAPIError(f"Alpha Vantage API error [rate_limit_or_premium]: {joined}")
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((AlphaVantageRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=2, max=60),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_ads/amazon_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_ads/amazon_ads.py
index 73d947130916..9b948ec34392 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_ads/amazon_ads.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/amazon_ads/amazon_ads.py
@@ -20,7 +20,11 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.amazon_ads
AmazonAdsEndpointConfig,
AmazonAdsReportConfig,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ request_deadline,
+ stop_at_request_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.sync_window import SyncWindow
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -242,7 +246,7 @@ class AmazonAdsClient:
# Built per client rather than as a decorator, so a test can swap the wait policy out.
self._retrying = Retrying(
retry=retry_if_exception_type((AmazonAdsRetryableError, requests.ReadTimeout, requests.ConnectionError)),
- stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
+ stop=stop_after_attempt(MAX_RETRY_ATTEMPTS) | stop_at_request_deadline(),
wait=wait_exponential_jitter(initial=2, max=120),
reraise=True,
)
@@ -307,7 +311,8 @@ class AmazonAdsClient:
extra_headers: Optional[dict[str, str]] = None,
http: Optional[requests.Session] = None,
) -> requests.Response:
- return self._retrying(self._send, method, path, profile_id, body, extra_headers, http)
+ with request_deadline():
+ return self._retrying(self._send, method, path, profile_id, body, extra_headers, http)
def list_profiles(self) -> list[dict[str, Any]]:
body = self.request("GET", "/v2/profiles").json()
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/amplitude/amplitude.py b/products/warehouse_sources/backend/temporal/data_imports/sources/amplitude/amplitude.py
index 6ca8594b1725..6395215bad6c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/amplitude/amplitude.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/amplitude/amplitude.py
@@ -11,7 +11,7 @@ from urllib.parse import urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.backend.temporal.data_imports.sources.amplitude.settings import (
AMPLITUDE_ENDPOINTS,
@@ -21,7 +21,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.amplitude.
EVENTS_EXPORT_WINDOW_HOURS,
AmplitudeEndpointConfig,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -132,7 +135,7 @@ def validate_credentials(api_key: str, secret_key: str, region: str) -> tuple[bo
return False, f"Unexpected response from Amplitude (status {response.status_code})."
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((AmplitudeRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(RETRY_MAX_ATTEMPTS),
wait=wait_exponential_jitter(initial=2, max=60),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/apollo/apollo.py b/products/warehouse_sources/backend/temporal/data_imports/sources/apollo/apollo.py
index 55254033cbd7..fb4e73ac6ea2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/apollo/apollo.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/apollo/apollo.py
@@ -6,10 +6,13 @@ from typing import Any, Optional
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.backend.temporal.data_imports.sources.apollo.settings import APOLLO_ENDPOINTS
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -117,7 +120,7 @@ def get_rows(
if resume_config is not None:
logger.debug(f"Apollo: resuming {endpoint} from page {page}")
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((ApolloRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
wait=_wait_apollo,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/appfigures/appfigures.py b/products/warehouse_sources/backend/temporal/data_imports/sources/appfigures/appfigures.py
index 66299e7bafc6..fefea7c39b9a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/appfigures/appfigures.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/appfigures/appfigures.py
@@ -5,7 +5,7 @@ from typing import Any, Optional
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.dataclasses import frozen
@@ -13,7 +13,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.appfigures
APPFIGURES_ENDPOINTS,
AppfiguresEndpointConfig,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -94,7 +97,7 @@ def _is_page_limit_response(response: requests.Response) -> bool:
return "must be less than or equal to 10000" in text.lower()
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((AppfiguresRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/appsflyer/appsflyer.py b/products/warehouse_sources/backend/temporal/data_imports/sources/appsflyer/appsflyer.py
index 075e4d35f082..2cb25b748d9d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/appsflyer/appsflyer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/appsflyer/appsflyer.py
@@ -8,7 +8,7 @@ from urllib.parse import quote, urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.dataclasses import frozen
@@ -17,7 +17,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.appsflyer.
AppsFlyerEndpointConfig,
AppsFlyerReportKind,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
APPSFLYER_BASE_URL = "https://hq1.appsflyer.com"
@@ -248,7 +251,7 @@ def _request_windows(kind: AppsFlyerReportKind, start: date, end: date) -> Itera
window_start = window_end + timedelta(days=1)
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((AppsFlyerRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
wait=wait_exponential_jitter(initial=5, max=120),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/appsignal/appsignal.py b/products/warehouse_sources/backend/temporal/data_imports/sources/appsignal/appsignal.py
index 58f459944756..cde541a610bc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/appsignal/appsignal.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/appsignal/appsignal.py
@@ -6,7 +6,7 @@ from urllib.parse import quote
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.dataclasses import frozen
@@ -14,7 +14,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.appsignal.
APPSIGNAL_ENDPOINTS,
AppsignalEndpointConfig,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http.url_utils import (
redact_literal_values,
scrub_url,
@@ -115,7 +118,7 @@ def _raise_for_status_scrubbed(response: requests.Response) -> None:
)
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((AppsignalRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRIES),
wait=wait_exponential_jitter(initial=1, max=60),
@@ -147,7 +150,7 @@ def _fetch_json(
return response.json()
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((AppsignalRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRIES),
wait=wait_exponential_jitter(initial=1, max=60),
@@ -485,7 +488,7 @@ def _resolve_walk_start(
return start
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((AppsignalRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRIES),
wait=wait_exponential_jitter(initial=1, max=60),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/automox/automox.py b/products/warehouse_sources/backend/temporal/data_imports/sources/automox/automox.py
index fcbcf88db4ca..8f485d2f136f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/automox/automox.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/automox/automox.py
@@ -8,7 +8,7 @@ import requests
import structlog
from dateutil import parser as dateutil_parser
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.dataclasses import frozen
@@ -17,7 +17,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.automox.se
AutomoxEndpointConfig,
AutomoxFanOutConfig,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -174,7 +177,7 @@ def _build_url(path: str, params: dict[str, Any]) -> str:
return f"{AUTOMOX_BASE_URL}{path}?{urlencode(query)}"
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((AutomoxRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aviator/aviator.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aviator/aviator.py
index 346897d411a6..3d6a81b45070 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aviator/aviator.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aviator/aviator.py
@@ -5,7 +5,7 @@ from typing import Any, Optional
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.batcher import Batcher
from products.warehouse_sources.backend.temporal.data_imports.sources.aviator.settings import (
@@ -15,7 +15,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.aviator.se
from products.warehouse_sources.backend.temporal.data_imports.sources.common.boundary_checkpoint import (
BoundaryCheckpoint,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -66,7 +69,7 @@ def _get_headers(api_token: str) -> dict[str, str]:
}
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
AviatorRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/awin/awin.py b/products/warehouse_sources/backend/temporal/data_imports/sources/awin/awin.py
index b98caf993afc..8b429110a834 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/awin/awin.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/awin/awin.py
@@ -5,7 +5,7 @@ from urllib.parse import urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.dataclasses import frozen
@@ -15,7 +15,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.awin.setti
MAX_WINDOW_DAYS,
AwinEndpointConfig,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -62,7 +65,7 @@ def _get_headers(api_token: str) -> dict[str, str]:
}
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
AwinRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/aws_organizations.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/aws_organizations.py
index 7f6e8dd87c4b..451965fb5f11 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/aws_organizations.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_organizations/aws_organizations.py
@@ -10,7 +10,7 @@ from botocore.auth import SigV4Auth
from botocore.awsrequest import AWSRequest
from botocore.credentials import Credentials
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.backend.temporal.data_imports.sources.aws_organizations.settings import (
AWS_ORGANIZATIONS_ENDPOINTS,
@@ -30,7 +30,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.aws_organi
TARGET_PREFIXES,
AwsOrganizationsEndpointConfig,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http.transport import BoundedRetry
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -215,7 +218,7 @@ class AwsOrganizationsClient:
# Throttling arrives as HTTP 400 with the code in the body, which the tracked transport's
# status-code retries can't see, so there is no compounding here.
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((AwsOrganizationsRetryableError, requests.ConnectionError, requests.Timeout)),
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
wait=wait_exponential_jitter(initial=RETRY_INITIAL_WAIT_SECONDS, max=RETRY_MAX_WAIT_SECONDS),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/aws_waf.py b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/aws_waf.py
index b0e61b513020..12acc74af1e8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/aws_waf.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/aws_waf/aws_waf.py
@@ -9,7 +9,7 @@ import requests
from botocore.auth import SigV4Auth
from botocore.awsrequest import AWSRequest
from botocore.credentials import Credentials
-from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from urllib3.exceptions import InvalidHeader
from urllib3.util.retry import Retry
@@ -41,7 +41,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.aws_waf.se
AwsService,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.datetime_utils import parse_datetime_value
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import probe_status_message
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.awswaf import AwsWafSourceConfig
@@ -150,7 +153,7 @@ class AwsWafClient:
def close(self) -> None:
self.session.close()
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((AwsWafRetryableError, requests.ConnectionError, requests.Timeout)),
stop=stop_after_attempt(5),
wait=retry_wait,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/azure_devops/azure_devops.py b/products/warehouse_sources/backend/temporal/data_imports/sources/azure_devops/azure_devops.py
index 49e9434bc1bb..7abbbd18a7eb 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/azure_devops/azure_devops.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/azure_devops/azure_devops.py
@@ -7,7 +7,7 @@ from urllib.parse import quote, urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.dataclasses import frozen
@@ -16,7 +16,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.azure_devo
AZURE_DEVOPS_ENDPOINTS,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import UNVERSIONED_API_VERSION
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -338,7 +341,7 @@ def get_rows(
org = _validate_organization(organization)
wire_version = wire_api_version(api_version)
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((AzureDevOpsRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
wait=wait_exponential_jitter(initial=2, max=120),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bitbucket/bitbucket.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bitbucket/bitbucket.py
index 4a3b2fbee431..248447146cf0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bitbucket/bitbucket.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bitbucket/bitbucket.py
@@ -7,14 +7,17 @@ from urllib.parse import parse_qsl, quote, urlencode, urlsplit, urlunsplit
import requests
from dateutil import parser as dateutil_parser
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.backend.temporal.data_imports.sources.bitbucket.settings import (
BITBUCKET_ENDPOINTS,
BitbucketEndpointConfig,
SecondLevelFanOut,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -122,7 +125,7 @@ def _ensure_bitbucket_url(url: str) -> str:
return url
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
BitbucketRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bugsnag/bugsnag.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bugsnag/bugsnag.py
index e1a3f6a5a972..978346fb88a6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bugsnag/bugsnag.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bugsnag/bugsnag.py
@@ -7,7 +7,7 @@ from urllib.parse import parse_qs, quote, urlencode, urlparse, urlunparse
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.batcher import Batcher
from products.warehouse_sources.backend.temporal.data_imports.sources.bugsnag.settings import (
@@ -15,7 +15,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.bugsnag.se
BugsnagEndpointConfig,
BugsnagScope,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -104,7 +107,7 @@ def _next_offset_url(url: str, page_len: int) -> str | None:
return urlunparse(parsed._replace(query=urlencode(query, doseq=True)))
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((BugsnagRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=60),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/care_quality_commission/care_quality_commission.py b/products/warehouse_sources/backend/temporal/data_imports/sources/care_quality_commission/care_quality_commission.py
index 2087cac5c899..44d75cb06ffa 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/care_quality_commission/care_quality_commission.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/care_quality_commission/care_quality_commission.py
@@ -36,7 +36,7 @@ from urllib.parse import urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.batcher import Batcher
from products.warehouse_sources.backend.temporal.data_imports.sources.care_quality_commission.settings import (
@@ -46,7 +46,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.care_quali
from products.warehouse_sources.backend.temporal.data_imports.sources.common.boundary_checkpoint import (
BoundaryCheckpoint,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -84,7 +87,7 @@ def _build_url(path: str, params: dict[str, Any]) -> str:
return f"{url}?{urlencode(filtered)}"
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((CQCRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=60),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/checkmarx/checkmarx.py b/products/warehouse_sources/backend/temporal/data_imports/sources/checkmarx/checkmarx.py
index 55559aa55590..1c266e7be7d5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/checkmarx/checkmarx.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/checkmarx/checkmarx.py
@@ -8,7 +8,7 @@ from urllib.parse import quote
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.dataclasses import frozen
@@ -19,7 +19,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.checkmarx.
CheckmarxFanOutConfig,
RegionHosts,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -134,7 +137,7 @@ class CheckmarxAuth:
return self._token
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
CheckmarxRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cimis/cimis.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cimis/cimis.py
index d22247f059c9..360a4ee57a1a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cimis/cimis.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cimis/cimis.py
@@ -6,7 +6,7 @@ from zoneinfo import ZoneInfo
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.backend.temporal.data_imports.sources.cimis.settings import (
CIMIS_BASE_URL,
@@ -17,7 +17,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.cimis.sett
DEFAULT_HOURLY_DATA_ITEMS,
CimisEndpointConfig,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
REQUEST_TIMEOUT = 60
@@ -113,7 +116,7 @@ def _flatten_record(record: dict[str, Any]) -> dict[str, Any]:
return flat
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((CimisRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/circleci/circleci.py b/products/warehouse_sources/backend/temporal/data_imports/sources/circleci/circleci.py
index 596ffd31f8ec..4d655fdf913d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/circleci/circleci.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/circleci/circleci.py
@@ -4,12 +4,15 @@ from urllib.parse import quote, urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.dataclasses import frozen
from products.warehouse_sources.backend.temporal.data_imports.sources.circleci.settings import CIRCLECI_ENDPOINTS
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -211,7 +214,7 @@ def _make_fetch_page(
# tracking hold; redact_values masks the token regardless of header-name denylists.
session = make_tracked_session(redact_values=(api_token,))
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((CircleCIRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRIES),
wait=_retry_wait,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/circleci_insights/circleci_insights.py b/products/warehouse_sources/backend/temporal/data_imports/sources/circleci_insights/circleci_insights.py
index 219749b40af8..fb58caa977c7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/circleci_insights/circleci_insights.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/circleci_insights/circleci_insights.py
@@ -7,7 +7,7 @@ from urllib.parse import quote, urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.backend.temporal.data_imports.sources.circleci_insights.settings import (
CIRCLECI_INSIGHTS_ENDPOINTS,
@@ -15,7 +15,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.circleci_i
REPORTING_WINDOWS,
CircleciInsightsEndpointConfig,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -200,7 +203,7 @@ def _make_fetch_page(api_token: str, logger: FilteringBoundLogger) -> FetchPageF
# tracking hold; redact_values masks the token regardless of header-name denylists.
session = make_tracked_session(redact_values=(api_token,))
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((CircleciInsightsRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRIES),
wait=_retry_wait,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clari/clari.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clari/clari.py
index 1ebb227df94c..6ac682617ecf 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clari/clari.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clari/clari.py
@@ -6,7 +6,7 @@ from urllib.parse import quote, urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.dataclasses import frozen
@@ -15,7 +15,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.clari.sett
ACTIVITY_TYPES,
CLARI_BASE_URL,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -85,7 +88,7 @@ def validate_credentials(api_key: str) -> bool:
def _make_fetcher(session: requests.Session, logger: FilteringBoundLogger):
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((ClariRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
wait=wait_exponential_jitter(initial=2, max=90),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clickhouse_cloud/clickhouse_cloud.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clickhouse_cloud/clickhouse_cloud.py
index 4ce4ea8e40c6..45f94362d713 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clickhouse_cloud/clickhouse_cloud.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clickhouse_cloud/clickhouse_cloud.py
@@ -6,14 +6,17 @@ from urllib.parse import urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.backend.temporal.data_imports.sources.clickhouse_cloud.settings import (
CLICKHOUSE_CLOUD_ENDPOINTS,
USAGE_COST_MAX_WINDOW_DAYS,
ClickhouseCloudEndpointConfig,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -78,7 +81,7 @@ def _build_url(path: str, params: dict[str, Any] | None = None) -> str:
return f"{url}?{urlencode({k: str(v) for k, v in params.items() if v is not None})}"
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
ClickhouseCloudRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/codacy/codacy.py b/products/warehouse_sources/backend/temporal/data_imports/sources/codacy/codacy.py
index a461cac081a8..165470ad640b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/codacy/codacy.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/codacy/codacy.py
@@ -5,7 +5,7 @@ from urllib.parse import urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.backend.temporal.data_imports.sources.codacy.settings import (
CODACY_ENDPOINTS,
@@ -13,7 +13,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.codacy.set
METRICS_PERIOD,
CodacyEndpointConfig,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
CODACY_BASE_URL = "https://api.codacy.com/api/v3"
@@ -33,7 +36,7 @@ def _get_headers(api_token: str) -> dict[str, str]:
}
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
CodacyRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/codecov/codecov.py b/products/warehouse_sources/backend/temporal/data_imports/sources/codecov/codecov.py
index 9a11def19a4c..1881b1b5bed2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/codecov/codecov.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/codecov/codecov.py
@@ -7,7 +7,7 @@ from urllib.parse import quote, urlencode
import requests
from dateutil import parser as dateutil_parser
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.backend.temporal.data_imports.sources.codecov.settings import (
CODECOV_ENDPOINTS,
@@ -15,7 +15,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.codecov.se
CodecovParentConfig,
CodecovResponseShape,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -171,7 +174,7 @@ def parse_repositories(repositories: str | None) -> list[str]:
return [name.strip() for name in repositories.split(",") if name.strip()]
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
CodecovRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cody/cody.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cody/cody.py
index bde5ac72f34c..1328fec03268 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cody/cody.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cody/cody.py
@@ -9,7 +9,7 @@ from urllib.parse import urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from urllib3.util.retry import Retry
from products.warehouse_sources.backend.temporal.data_imports.sources.cody.settings import (
@@ -17,7 +17,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.cody.setti
REPORTS_PATH,
CodyEndpointConfig,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -85,7 +88,7 @@ def _build_url(
return f"{CODY_BASE_URL}{config.path}?{urlencode(params)}"
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
CodyRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/coin_api/coin_api.py b/products/warehouse_sources/backend/temporal/data_imports/sources/coin_api/coin_api.py
index 2a4082a6a65c..3d9f413ef5ff 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/coin_api/coin_api.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/coin_api/coin_api.py
@@ -6,7 +6,7 @@ from urllib.parse import urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.dataclasses import frozen
@@ -14,7 +14,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.coin_api.s
COIN_API_ENDPOINTS,
CoinApiEndpointConfig,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -225,7 +228,7 @@ def get_rows(
session = make_tracked_session(redact_values=(api_key,) if api_key else ())
headers = _headers(api_key)
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((CoinApiRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
wait=wait_exponential_jitter(initial=2, max=120),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/commercetools/commercetools.py b/products/warehouse_sources/backend/temporal/data_imports/sources/commercetools/commercetools.py
index 5ea8887dc1a5..4c79925e8592 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/commercetools/commercetools.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/commercetools/commercetools.py
@@ -7,13 +7,16 @@ from urllib.parse import urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.backend.temporal.data_imports.sources.commercetools.settings import (
COMMERCETOOLS_ENDPOINTS,
CommercetoolsEndpointConfig,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -143,7 +146,7 @@ def get_rows(
)
offset = 0
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((CommercetoolsRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
wait=wait_exponential_jitter(initial=1, max=60),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/concord/concord.py b/products/warehouse_sources/backend/temporal/data_imports/sources/concord/concord.py
index 79dc9d7e7da8..b236644c183a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/concord/concord.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/concord/concord.py
@@ -6,13 +6,16 @@ from urllib.parse import urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.batcher import Batcher
from products.warehouse_sources.backend.temporal.data_imports.sources.common.boundary_checkpoint import (
BoundaryCheckpoint,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.concord.settings import (
@@ -91,7 +94,7 @@ def _build_url(base_url: str, path: str, params: dict[str, Any] | None = None) -
return f"{url}?{urlencode(params, doseq=True)}"
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((ConcordRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/confluent_cloud/confluent_cloud.py b/products/warehouse_sources/backend/temporal/data_imports/sources/confluent_cloud/confluent_cloud.py
index 05a8ba436b3b..34b16dfbf3bd 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/confluent_cloud/confluent_cloud.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/confluent_cloud/confluent_cloud.py
@@ -6,9 +6,12 @@ from urllib.parse import urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.sync_window import SyncWindow
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -81,7 +84,7 @@ def _coerce_datetime(value: Any) -> datetime | None:
return None
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((ConfluentCloudRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=60),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/coralogix/coralogix.py b/products/warehouse_sources/backend/temporal/data_imports/sources/coralogix/coralogix.py
index 8382a120d9f1..f3dbaf23957a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/coralogix/coralogix.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/coralogix/coralogix.py
@@ -6,9 +6,12 @@ from typing import Any, Optional
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.coralogix.settings import (
@@ -145,7 +148,7 @@ def _normalize_row(result: dict[str, Any]) -> dict[str, Any]:
return row
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
CoralogixRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/coupa/coupa.py b/products/warehouse_sources/backend/temporal/data_imports/sources/coupa/coupa.py
index 1b80d8c4394e..9575ab20ab1d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/coupa/coupa.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/coupa/coupa.py
@@ -6,9 +6,12 @@ from urllib.parse import urlencode, urlparse
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.coupa.settings import COUPA_ENDPOINTS, PAGE_SIZE
@@ -130,7 +133,7 @@ def get_rows(
base_url = normalize_host(instance_url)
token = _mint_token(session, instance_url, client_id, client_secret, config.scope)
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((CoupaRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
wait=wait_exponential_jitter(initial=2, max=90),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/crates_io/crates_io.py b/products/warehouse_sources/backend/temporal/data_imports/sources/crates_io/crates_io.py
index 15c7aa323e31..06890a634d2c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/crates_io/crates_io.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/crates_io/crates_io.py
@@ -6,9 +6,12 @@ from urllib.parse import quote, urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.crates_io.settings import (
CRATES_IO_ENDPOINTS,
@@ -133,7 +136,7 @@ def _crate_url(crate: str) -> str:
return f"{CRATES_IO_BASE_URL}/crates/{quote(crate, safe='')}"
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((CratesIORetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cronitor/cronitor.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cronitor/cronitor.py
index a32be4996f54..743bcd61cdc9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cronitor/cronitor.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cronitor/cronitor.py
@@ -7,11 +7,14 @@ from urllib.parse import quote, urlencode, urlsplit, urlunsplit
import requests
from dateutil import parser
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.dataclasses import frozen
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.cronitor.settings import (
@@ -81,7 +84,7 @@ def _coerce_epoch(value: Any) -> int | None:
return None
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((CronitorRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cursor/cursor.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cursor/cursor.py
index 4739a209f8d9..6d9bf3e4e35d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cursor/cursor.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cursor/cursor.py
@@ -8,12 +8,15 @@ from typing import Any
import orjson
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from urllib3.util.retry import Retry
from posthog.dataclasses import frozen
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -113,7 +116,7 @@ def _is_plan_rejection(response: requests.Response) -> bool:
return isinstance(message, str) and _ENTERPRISE_PLAN_MARKER in message.lower()
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
CursorRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/customer_io/api_client.py b/products/warehouse_sources/backend/temporal/data_imports/sources/customer_io/api_client.py
index 7c02a9b08803..c40c5c46ef51 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/customer_io/api_client.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/customer_io/api_client.py
@@ -10,14 +10,17 @@ from typing import Any
import requests
import structlog
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
ExternalWebhookInfo,
WebhookCreationResult,
WebhookDeletionResult,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.customer_io.constants import (
CIO_AUTO_WEBHOOK_NAME,
CIO_EU_BASE_URL,
@@ -115,7 +118,7 @@ def validate_credentials(api_key: str, region: str | None) -> tuple[bool, str |
return True, None
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((ListEndpointRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/customerly/customerly.py b/products/warehouse_sources/backend/temporal/data_imports/sources/customerly/customerly.py
index f0b414a9ee5f..0b334d923dc6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/customerly/customerly.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/customerly/customerly.py
@@ -5,9 +5,12 @@ from urllib.parse import urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.customerly.settings import (
@@ -61,7 +64,7 @@ def _is_auth_error_body(body: str) -> bool:
def _make_fetch_page(headers: dict[str, str], logger: FilteringBoundLogger):
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((CustomerlyRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRIES),
wait=wait_exponential_jitter(initial=1, max=60),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/dagster_cloud/dagster_cloud.py b/products/warehouse_sources/backend/temporal/data_imports/sources/dagster_cloud/dagster_cloud.py
index 6c568a1c2932..f38244af3451 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/dagster_cloud/dagster_cloud.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/dagster_cloud/dagster_cloud.py
@@ -5,11 +5,14 @@ from typing import Any, Optional
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.dataclasses import frozen
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.dagster_cloud.queries import VALIDATION_QUERY
@@ -194,7 +197,7 @@ def _validate_response(response: requests.Response, url: str) -> dict[str, Any]:
return payload
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(DagsterCloudRetryableError),
stop=stop_after_attempt(DAGSTER_CLOUD_MAX_RETRY_ATTEMPTS),
wait=wait_exponential_jitter(initial=1, max=60),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/datadog/datadog.py b/products/warehouse_sources/backend/temporal/data_imports/sources/datadog/datadog.py
index 926f5e63a201..0db6ee27a5c8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/datadog/datadog.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/datadog/datadog.py
@@ -6,9 +6,12 @@ from urllib.parse import parse_qsl, quote, urlencode, urlparse, urlunparse
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import schema_for_resource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -257,7 +260,7 @@ def _make_fetcher(
session: requests.Session,
logger: FilteringBoundLogger,
) -> Callable[..., Any]:
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((DatadogRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/dataforseo/dataforseo.py b/products/warehouse_sources/backend/temporal/data_imports/sources/dataforseo/dataforseo.py
index 6d1e7f89c4f7..3273ab1b965e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/dataforseo/dataforseo.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/dataforseo/dataforseo.py
@@ -4,11 +4,14 @@ from typing import Any
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.dataclasses import frozen
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.dataforseo.settings import (
@@ -131,7 +134,7 @@ def _raise_for_body_status(status_code: Any, status_message: Any) -> None:
raise DataForSEOAPIError(f"DataForSEO API error [{status_code}]: {status_message}")
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((DataForSEORetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRIES),
wait=wait_exponential_jitter(initial=2, max=60),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/datahub/datahub.py b/products/warehouse_sources/backend/temporal/data_imports/sources/datahub/datahub.py
index 250e83824353..05cedca69a62 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/datahub/datahub.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/datahub/datahub.py
@@ -8,12 +8,15 @@ from urllib.parse import urlparse
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.cloud_utils import is_cloud
from posthog.dataclasses import frozen
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import _is_host_safe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -266,7 +269,7 @@ def _read_capped_bytes(response: requests.Response, max_bytes: int) -> bytes:
return raw
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((DatahubRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/dbt/dbt.py b/products/warehouse_sources/backend/temporal/data_imports/sources/dbt/dbt.py
index dab1e6bdd66d..aa60f8a7c10c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/dbt/dbt.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/dbt/dbt.py
@@ -6,11 +6,14 @@ from urllib.parse import urlencode, urlparse
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.dataclasses import frozen
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import _is_host_safe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -187,7 +190,7 @@ def _raise_for_graphql_errors(payload: dict[str, Any]) -> None:
raise Exception(f"dbt Discovery API error: {messages}")
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(DbtRetryableError),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
@@ -355,7 +358,7 @@ def _coerce_datetime(value: Any) -> datetime | None:
return None
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
DbtRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/decagon/decagon.py b/products/warehouse_sources/backend/temporal/data_imports/sources/decagon/decagon.py
index 49025b0d0e55..c4e4f6e541b8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/decagon/decagon.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/decagon/decagon.py
@@ -7,9 +7,12 @@ from typing import Any, Optional
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.decagon.settings import (
@@ -400,7 +403,7 @@ class _PageFetcher:
self._throttle = RequestThrottle()
self._optional_params_dropped = not config.optional_params
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((DecagonRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
# The 1 rps limit means a 429 needs a generous backoff, not a quick retry.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/deepsource/deepsource.py b/products/warehouse_sources/backend/temporal/data_imports/sources/deepsource/deepsource.py
index caac9ae2174e..a0307a20ecf3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/deepsource/deepsource.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/deepsource/deepsource.py
@@ -4,9 +4,12 @@ from typing import Any
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.deepsource.queries import (
@@ -81,7 +84,7 @@ def _make_session(api_token: str) -> requests.Session:
)
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(DeepsourceRetryableError),
stop=stop_after_attempt(DEEPSOURCE_MAX_RETRY_ATTEMPTS),
wait=_wait_strategy,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/dixa/dixa.py b/products/warehouse_sources/backend/temporal/data_imports/sources/dixa/dixa.py
index f98a64f0bd46..b6b8e17269df 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/dixa/dixa.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/dixa/dixa.py
@@ -7,11 +7,14 @@ from urllib.parse import quote, urlencode, urljoin, urlparse
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.dataclasses import frozen
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.dixa.settings import (
@@ -303,7 +306,7 @@ def get_rows(
config = DIXA_ENDPOINTS[endpoint]
session = _get_session(api_token)
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((DixaRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
wait=wait_exponential_jitter(initial=5, max=120),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/dockerhub/dockerhub.py b/products/warehouse_sources/backend/temporal/data_imports/sources/dockerhub/dockerhub.py
index c9b71a48464a..b3f3939a0439 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/dockerhub/dockerhub.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/dockerhub/dockerhub.py
@@ -7,11 +7,14 @@ from urllib.parse import quote, urlencode, urlsplit
import orjson
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.dataclasses import frozen
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.dockerhub.settings import (
@@ -102,7 +105,7 @@ def _tags_url(namespace: str, repository: str) -> str:
return f"{DOCKERHUB_BASE_URL}/v2/namespaces/{quote(namespace)}/repositories/{quote(repository)}/tags?{query}"
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((DockerhubRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
@@ -162,7 +165,7 @@ def _request_json(
return response.json()
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((DockerhubRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
@@ -186,7 +189,7 @@ def _fetch_page(
return data["results"], None
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((DockerhubRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/doppler/doppler.py b/products/warehouse_sources/backend/temporal/data_imports/sources/doppler/doppler.py
index 5d3be8c226e3..46d8eeb496a6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/doppler/doppler.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/doppler/doppler.py
@@ -6,9 +6,12 @@ from urllib.parse import quote, urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.doppler.settings import (
@@ -71,7 +74,7 @@ def _coerce_watermark(value: Any) -> datetime | None:
return None
-@retry(
+@retry_with_deadline(
# Doppler enforces plan-tiered per-minute read limits and returns 429 with a retry-after
# header when exceeded; transient 5xx are retryable too.
retry=retry_if_exception_type((DopplerRetryableError, requests.ReadTimeout, requests.ConnectionError)),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/dropbox_sign/dropbox_sign.py b/products/warehouse_sources/backend/temporal/data_imports/sources/dropbox_sign/dropbox_sign.py
index 3918ef189d50..636b4bc6ce72 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/dropbox_sign/dropbox_sign.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/dropbox_sign/dropbox_sign.py
@@ -5,10 +5,13 @@ from typing import Any
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.batcher import Batcher
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.dropbox_sign.settings import (
@@ -50,7 +53,7 @@ def validate_credentials(api_key: str) -> bool:
return False
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((DropboxSignRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/easypost/easypost.py b/products/warehouse_sources/backend/temporal/data_imports/sources/easypost/easypost.py
index 25532d5939d8..cc67dbda79f0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/easypost/easypost.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/easypost/easypost.py
@@ -4,11 +4,14 @@ from typing import Any
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.dataclasses import frozen
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.easypost.settings import EASYPOST_ENDPOINTS
@@ -57,7 +60,7 @@ def _format_datetime(value: datetime) -> str:
return _ensure_utc(value).strftime("%Y-%m-%dT%H:%M:%SZ")
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((EasypostRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/everhour/everhour.py b/products/warehouse_sources/backend/temporal/data_imports/sources/everhour/everhour.py
index 239b15d5a906..503bfdfd8807 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/everhour/everhour.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/everhour/everhour.py
@@ -7,9 +7,12 @@ from urllib.parse import urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.everhour.settings import (
@@ -111,7 +114,7 @@ def _row_key(item: dict[str, Any], dedupe_keys: list[str]) -> tuple[Any, ...]:
return tuple(item.get(key) for key in dedupe_keys)
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((EverhourRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRIES),
wait=wait_exponential_jitter(initial=1, max=60),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/exchange_rates_api/exchange_rates_api.py b/products/warehouse_sources/backend/temporal/data_imports/sources/exchange_rates_api/exchange_rates_api.py
index a8624f838255..18d735de83b0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/exchange_rates_api/exchange_rates_api.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/exchange_rates_api/exchange_rates_api.py
@@ -6,9 +6,12 @@ from urllib.parse import urlencode, urlsplit, urlunsplit
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.exchange_rates_api.settings import (
@@ -193,7 +196,7 @@ def get_rows(
base = base_currency or DEFAULT_BASE_CURRENCY
common_params: dict[str, Any] = {"access_key": access_key}
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((ExchangeRatesApiRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
wait=wait_exponential_jitter(initial=2, max=120),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/fastly/fastly.py b/products/warehouse_sources/backend/temporal/data_imports/sources/fastly/fastly.py
index e8c37738e2ff..7be6b2448171 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/fastly/fastly.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/fastly/fastly.py
@@ -6,11 +6,14 @@ from urllib.parse import urlencode, urljoin, urlsplit
import requests
from dateutil import parser as date_parser
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.dataclasses import frozen
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.fastly.settings import (
@@ -97,7 +100,7 @@ def _build_url(base_url: str, params: dict[str, Any], safe: str = "") -> str:
return f"{base_url}?{urlencode(params, safe=safe)}"
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
FastlyRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/finage/finage.py b/products/warehouse_sources/backend/temporal/data_imports/sources/finage/finage.py
index 4dcd8e85bdd1..50543bb00672 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/finage/finage.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/finage/finage.py
@@ -5,12 +5,15 @@ from typing import Any, Optional
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from urllib3.util.retry import Retry
from posthog.dataclasses import frozen
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.finage.settings import (
FINAGE_ENDPOINTS,
@@ -164,7 +167,7 @@ def _is_iso_date(value: Any) -> bool:
return False
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((FinageRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/financial_modelling/financial_modelling.py b/products/warehouse_sources/backend/temporal/data_imports/sources/financial_modelling/financial_modelling.py
index 335a9d7fe1f9..63a5426d3f1d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/financial_modelling/financial_modelling.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/financial_modelling/financial_modelling.py
@@ -6,12 +6,15 @@ from urllib.parse import urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.dataclasses import frozen
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.batcher import Batcher
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.financial_modelling.settings import (
@@ -62,7 +65,7 @@ def _build_url(path: str, params: dict[str, Any], api_key: str) -> str:
return f"{FINANCIAL_MODELLING_BASE_URL}/{path}?{query}"
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((FinancialModellingRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
wait=wait_exponential_jitter(initial=1, max=60),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/finnhub/finnhub.py b/products/warehouse_sources/backend/temporal/data_imports/sources/finnhub/finnhub.py
index 93d85fe8a4bf..c19216842755 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/finnhub/finnhub.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/finnhub/finnhub.py
@@ -5,9 +5,12 @@ from urllib.parse import urlsplit
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.sync_window import SyncWindow
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.finnhub.settings import (
@@ -85,7 +88,7 @@ def _to_date(value: Any) -> date:
raise ValueError(f"Cannot coerce incremental value {value!r} to a date")
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((FinnhubRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(6),
wait=wait_exponential_jitter(initial=2, max=60),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/finnworlds/finnworlds.py b/products/warehouse_sources/backend/temporal/data_imports/sources/finnworlds/finnworlds.py
index 829f355db972..c296acf686b8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/finnworlds/finnworlds.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/finnworlds/finnworlds.py
@@ -5,9 +5,12 @@ from urllib.parse import urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.finnworlds.settings import (
FINNWORLDS_ENDPOINTS,
@@ -101,7 +104,7 @@ def _redact_key(text: str) -> str:
return _KEY_RE.sub(r"\1REDACTED", text)
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((FinnworldsRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/flagsmith/flagsmith.py b/products/warehouse_sources/backend/temporal/data_imports/sources/flagsmith/flagsmith.py
index c1201a95df1f..91cf15193f79 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/flagsmith/flagsmith.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/flagsmith/flagsmith.py
@@ -7,10 +7,13 @@ from urllib.parse import parse_qsl, urlencode, urlparse
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt
+from tenacity import retry_if_exception_type, stop_after_attempt
from urllib3.util.retry import Retry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.flagsmith.settings import (
@@ -277,7 +280,7 @@ def _wait_strategy(retry_state: Any) -> float:
return min(2.0**retry_state.attempt_number, MAX_RETRY_WAIT_SECONDS)
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((FlagsmithRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
wait=_wait_strategy,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/formbricks/formbricks.py b/products/warehouse_sources/backend/temporal/data_imports/sources/formbricks/formbricks.py
index 36fedbeac4b5..d2f71ae49a0b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/formbricks/formbricks.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/formbricks/formbricks.py
@@ -9,9 +9,12 @@ from urllib.parse import parse_qsl, urlencode, urlparse
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import _is_host_safe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -224,7 +227,7 @@ def _advance_skip(url: str) -> str:
return parsed._replace(query=urlencode(params)).geturl()
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((FormbricksRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRIES),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/freshsales/freshsales.py b/products/warehouse_sources/backend/temporal/data_imports/sources/freshsales/freshsales.py
index 5cb9e431e704..15e9cecf7a5d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/freshsales/freshsales.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/freshsales/freshsales.py
@@ -11,11 +11,14 @@ from requests.exceptions import (
RequestException,
)
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.dataclasses import frozen
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.freshsales.settings import (
@@ -104,7 +107,7 @@ def _build_page_url(
return f"{path}?{urlencode(params)}"
-@retry(
+@retry_with_deadline(
# Only transport hiccups and explicit 429/5xx are retried — HTTPError (a RequestException
# subclass raised for 4xx by raise_for_status) must fail fast, not retry.
retry=retry_if_exception_type((FreshsalesRetryableError, RequestsConnectionError, ReadTimeout)),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/frill/frill.py b/products/warehouse_sources/backend/temporal/data_imports/sources/frill/frill.py
index 2b0bb1cb5b7a..cc25614b43af 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/frill/frill.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/frill/frill.py
@@ -4,9 +4,12 @@ from typing import Any, Optional
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.frill.settings import (
@@ -66,7 +69,7 @@ def _handle_response(response: requests.Response, url: str, logger: FilteringBou
return data
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((FrillRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRIES),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gerrit/gerrit.py b/products/warehouse_sources/backend/temporal/data_imports/sources/gerrit/gerrit.py
index 6117e73fd61d..576e1146647f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gerrit/gerrit.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/gerrit/gerrit.py
@@ -10,9 +10,12 @@ from urllib.parse import quote, urlencode, urlparse
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import _is_host_safe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -435,7 +438,7 @@ def get_rows(
offset = resume.offset
logger.debug(f"Gerrit: resuming {endpoint} from offset {offset}")
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((GerritRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRIES),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gitea/gitea.py b/products/warehouse_sources/backend/temporal/data_imports/sources/gitea/gitea.py
index e4f3a19fbdb3..0182cf33897f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gitea/gitea.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/gitea/gitea.py
@@ -9,7 +9,7 @@ import pyarrow as pa
import requests
from asgiref.sync import async_to_sync
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.dataclasses import frozen
@@ -19,7 +19,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.bas
WebhookDeletionResult,
WebhookSyncResult,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
@@ -273,7 +276,7 @@ def _get_item_mapper(endpoint: str) -> Callable[[dict[str, Any]], dict[str, Any]
return None
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
GiteaRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gitguardian/gitguardian.py b/products/warehouse_sources/backend/temporal/data_imports/sources/gitguardian/gitguardian.py
index 3267943b9fbd..49a50ca53d78 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gitguardian/gitguardian.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/gitguardian/gitguardian.py
@@ -6,11 +6,14 @@ from urllib.parse import quote, urlencode, urljoin, urlsplit
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.dataclasses import frozen
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.gitguardian.settings import (
@@ -80,7 +83,7 @@ def _build_url(base_url: str, path: str, params: dict[str, Any]) -> str:
return f"{base_url}{path}?{urlencode(params)}"
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
GitGuardianRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gitlab/gitlab.py b/products/warehouse_sources/backend/temporal/data_imports/sources/gitlab/gitlab.py
index a8a8577a7672..480cfc92493a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gitlab/gitlab.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/gitlab/gitlab.py
@@ -8,10 +8,13 @@ from urllib.parse import quote, urlencode, urlparse
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.batcher import Batcher
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import _is_host_safe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -386,7 +389,7 @@ def get_rows(
logger.warning("GitLab: ignoring resume URL whose host does not match the configured host")
url = initial_url
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((GitLabRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRIES),
wait=_retry_wait,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gladly/gladly.py b/products/warehouse_sources/backend/temporal/data_imports/sources/gladly/gladly.py
index 74ce843e884d..aeff6288a346 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gladly/gladly.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/gladly/gladly.py
@@ -11,11 +11,14 @@ from urllib.parse import quote, urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.dataclasses import frozen
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.gladly.settings import (
@@ -328,7 +331,7 @@ def get_rows(
)
return
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((GladlyRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
wait=wait_exponential_jitter(initial=2, max=90),
@@ -517,7 +520,7 @@ def _voice_transcript_rows(
logger: FilteringBoundLogger,
resumable_source_manager: ResumableSourceManager[GladlyResumeConfig],
) -> Iterator[list[dict[str, Any]]]:
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((GladlyRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
wait=wait_exponential_jitter(initial=2, max=90),
@@ -594,7 +597,7 @@ def _report_rows(
db_incremental_field_last_value: Any = None,
schema_has_ever_synced: bool = False,
) -> Iterator[list[dict[str, Any]]]:
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((GladlyRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
wait=wait_exponential_jitter(initial=2, max=90),
@@ -637,7 +640,7 @@ def _report_rows(
if not inject_row_id:
required_columns.add(config.primary_key)
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type(GladlyReportUnavailableError),
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
wait=wait_exponential_jitter(initial=2, max=90),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gnews/gnews.py b/products/warehouse_sources/backend/temporal/data_imports/sources/gnews/gnews.py
index b0e469e6d864..193d7ce566a0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gnews/gnews.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/gnews/gnews.py
@@ -6,10 +6,13 @@ from urllib.parse import urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.batcher import Batcher
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.gnews.settings import (
@@ -87,7 +90,7 @@ def _build_url(config: GNewsEndpointConfig, params: dict[str, Any]) -> str:
return f"{GNEWS_BASE_URL}{config.path}?{urlencode(params)}"
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
GNewsRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gong/gong.py b/products/warehouse_sources/backend/temporal/data_imports/sources/gong/gong.py
index 7cff9455a222..6f7f1b729ef6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gong/gong.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/gong/gong.py
@@ -8,11 +8,14 @@ from urllib.parse import urlencode
import requests
from dateutil import parser as dateutil_parser
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.dataclasses import frozen
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.gong.settings import (
@@ -164,7 +167,7 @@ def get_rows(
config = GONG_ENDPOINTS[endpoint]
headers = _get_headers(access_key, access_key_secret)
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((GongRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=60),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/google_pagespeed_insights/google_pagespeed_insights.py b/products/warehouse_sources/backend/temporal/data_imports/sources/google_pagespeed_insights/google_pagespeed_insights.py
index 3bcb1f973ac0..eee88026d2da 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/google_pagespeed_insights/google_pagespeed_insights.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/google_pagespeed_insights/google_pagespeed_insights.py
@@ -7,10 +7,13 @@ from urllib.parse import urlencode, urlparse
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from urllib3.util.retry import Retry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.google_pagespeed_insights.settings import (
PAGESPEED_ENDPOINTS,
@@ -144,7 +147,7 @@ def _redact_key(text: str) -> str:
# `RequestException`, not subclasses, so they must be listed explicitly: a connection broken
# mid-response (server hung up before the body was fully streamed) or a body that fails to decode is
# transient and safe to retry on the large PageSpeed payloads, not a reason to fail the whole sync.
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
PageSpeedRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gorgias/gorgias.py b/products/warehouse_sources/backend/temporal/data_imports/sources/gorgias/gorgias.py
index fff7685c5d18..4ea4f65e7720 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gorgias/gorgias.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/gorgias/gorgias.py
@@ -8,9 +8,12 @@ from typing import Any, Optional
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.gorgias.settings import (
@@ -231,7 +234,7 @@ def get_rows(
if resume_config:
logger.debug(f"Gorgias: resuming {endpoint} from saved cursor")
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type(
(GorgiasRetryableError, requests.ReadTimeout, requests.ConnectionError),
),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/grafana/grafana.py b/products/warehouse_sources/backend/temporal/data_imports/sources/grafana/grafana.py
index 8e94f0b7473b..495d2c4a3398 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/grafana/grafana.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/grafana/grafana.py
@@ -10,12 +10,15 @@ from urllib.parse import quote, urlencode, urlparse
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from urllib3.util.retry import Retry
from posthog.dataclasses import frozen
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import _is_host_safe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -374,7 +377,7 @@ def _check_host(host: str, team_id: int) -> None:
def _make_fetch(session: requests.Session, headers: dict[str, str], logger: FilteringBoundLogger):
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((GrafanaRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRIES),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gridly/gridly.py b/products/warehouse_sources/backend/temporal/data_imports/sources/gridly/gridly.py
index 2945b4d1298f..f44505d8ee8a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gridly/gridly.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/gridly/gridly.py
@@ -7,9 +7,12 @@ from urllib.parse import quote, urlsplit
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http.observer import redact_request_urls
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -54,7 +57,7 @@ def _parse_total_count(value: str | None) -> int | None:
return None
-@retry(
+@retry_with_deadline(
# ChunkedEncodingError is a mid-stream connection break (the server truncated a chunked
# response body); it's transient like ConnectionError/ReadTimeout, not a ConnectionError subclass.
retry=retry_if_exception_type(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/harvey/harvey.py b/products/warehouse_sources/backend/temporal/data_imports/sources/harvey/harvey.py
index 3f184fee0e3b..7156e87a2536 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/harvey/harvey.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/harvey/harvey.py
@@ -6,11 +6,14 @@ from urllib.parse import quote, urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.dataclasses import frozen
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.harvey.settings import (
@@ -162,7 +165,7 @@ def check_endpoint_access(api_key: str, region: str | None, endpoint: str) -> st
return None
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
HarveyRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/healthchecks/healthchecks.py b/products/warehouse_sources/backend/temporal/data_imports/sources/healthchecks/healthchecks.py
index 5f9abd79f781..30c555dfa255 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/healthchecks/healthchecks.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/healthchecks/healthchecks.py
@@ -6,9 +6,12 @@ from urllib.parse import urlencode, urlparse
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.healthchecks.settings import (
@@ -107,7 +110,7 @@ def _check_key(check: dict[str, Any]) -> str | None:
return check.get("uuid") or check.get("unique_key")
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
HealthchecksRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/helicone/helicone.py b/products/warehouse_sources/backend/temporal/data_imports/sources/helicone/helicone.py
index 19e74a31c11a..ecefa0a86c16 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/helicone/helicone.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/helicone/helicone.py
@@ -5,9 +5,12 @@ from typing import Any, Optional
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.helicone.settings import (
@@ -85,7 +88,7 @@ def _extract_data(body: Any, url: str) -> list[Any]:
return []
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((HeliconeRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(RETRY_MAX_ATTEMPTS),
wait=wait_exponential_jitter(initial=2, max=60),
@@ -98,7 +101,7 @@ def _post(
return _parse_response(response, url, logger)
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((HeliconeRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(RETRY_MAX_ATTEMPTS),
wait=wait_exponential_jitter(initial=2, max=60),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/honeybadger/honeybadger.py b/products/warehouse_sources/backend/temporal/data_imports/sources/honeybadger/honeybadger.py
index 9460378d2a35..b449dab6b8e9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/honeybadger/honeybadger.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/honeybadger/honeybadger.py
@@ -7,9 +7,12 @@ from urllib.parse import urlencode, urljoin, urlparse
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.honeybadger.settings import (
@@ -76,7 +79,7 @@ def _to_unix_timestamp(value: Any) -> int:
return int(datetime.fromisoformat(str(value).replace("Z", "+00:00")).timestamp())
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
HoneybadgerRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/honeycomb/honeycomb.py b/products/warehouse_sources/backend/temporal/data_imports/sources/honeycomb/honeycomb.py
index cb23a676ba2b..a7eabcaaac87 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/honeycomb/honeycomb.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/honeycomb/honeycomb.py
@@ -6,9 +6,12 @@ from urllib.parse import urlencode, urljoin, urlsplit
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.honeycomb.settings import (
@@ -94,7 +97,7 @@ def _base_url(region: str) -> str:
return HONEYCOMB_BASE_URLS.get(region, HONEYCOMB_BASE_URLS["us"])
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((HoneycombRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=60),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/hyperspell/hyperspell.py b/products/warehouse_sources/backend/temporal/data_imports/sources/hyperspell/hyperspell.py
index 751544d57b8a..dd00b529b110 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/hyperspell/hyperspell.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/hyperspell/hyperspell.py
@@ -5,11 +5,14 @@ from urllib.parse import quote, urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.dataclasses import frozen
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.hyperspell.settings import (
@@ -112,7 +115,7 @@ def validate_credentials(
return False, f"Hyperspell API returned an unexpected status code: {response.status_code}"
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((HyperspellRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/inngest/inngest.py b/products/warehouse_sources/backend/temporal/data_imports/sources/inngest/inngest.py
index af143aed7de0..7391366e6e15 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/inngest/inngest.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/inngest/inngest.py
@@ -7,9 +7,12 @@ from urllib.parse import quote
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.sync_window import SyncWindow
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -57,7 +60,7 @@ def _get_headers(signing_key: str, environment: str | None) -> dict[str, str]:
return headers
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
InngestRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instana/instana.py b/products/warehouse_sources/backend/temporal/data_imports/sources/instana/instana.py
index 3e70325adcfd..9b40975399e6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instana/instana.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/instana/instana.py
@@ -7,11 +7,14 @@ from urllib.parse import quote, urlencode, urlparse
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.dataclasses import frozen
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import _is_host_safe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -205,7 +208,7 @@ def _fetch(
logger: FilteringBoundLogger,
json_body: Any = None,
) -> Any:
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((InstanaRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ip2whois/ip2whois.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ip2whois/ip2whois.py
index 98d546ac56f3..9915fcae6862 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ip2whois/ip2whois.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ip2whois/ip2whois.py
@@ -5,9 +5,12 @@ from urllib.parse import urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.ip2whois.settings import IP2WHOIS_ENDPOINTS
@@ -108,7 +111,7 @@ def _safe_json(response: requests.Response) -> dict[str, Any]:
return body if isinstance(body, dict) else {}
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((IP2WhoisRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/jamf_pro/jamf_pro.py b/products/warehouse_sources/backend/temporal/data_imports/sources/jamf_pro/jamf_pro.py
index 47c33329bbfb..b6e6a091087b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/jamf_pro/jamf_pro.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/jamf_pro/jamf_pro.py
@@ -10,11 +10,14 @@ from urllib.parse import quote, urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.dataclasses import frozen
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import _is_host_safe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -543,7 +546,7 @@ def get_rows(
token_manager = JamfProTokenManager(make_tracked_session(capture=False), host, credentials)
session = make_tracked_session(capture=config.capture_samples)
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((JamfProRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRIES),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/jellyfish/jellyfish.py b/products/warehouse_sources/backend/temporal/data_imports/sources/jellyfish/jellyfish.py
index a3511366fff3..d8314d2719e6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/jellyfish/jellyfish.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/jellyfish/jellyfish.py
@@ -6,10 +6,13 @@ from urllib.parse import urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from urllib3.util.retry import Retry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.jellyfish.settings import (
@@ -141,7 +144,7 @@ def _retry_wait(retry_state: Any) -> float:
return wait_exponential_jitter(initial=1, max=30)(retry_state)
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(JellyfishRetryableError, JellyfishRateLimitError, requests.ReadTimeout, requests.ConnectionError)
),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/jenkins/jenkins.py b/products/warehouse_sources/backend/temporal/data_imports/sources/jenkins/jenkins.py
index 58faae133d4e..bc93635dc356 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/jenkins/jenkins.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/jenkins/jenkins.py
@@ -8,13 +8,16 @@ from urllib.parse import urlparse
import urllib3
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.batcher import Batcher
from products.warehouse_sources.backend.temporal.data_imports.sources.common.boundary_checkpoint import (
BoundaryCheckpoint,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.jenkins.settings import JENKINS_ENDPOINTS
@@ -134,7 +137,7 @@ def _is_job_container(job: dict[str, Any]) -> bool:
return cls.endswith("Folder") or "MultiBranch" in cls or "OrganizationFolder" in cls
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
JenkinsRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/jfrog_artifactory/jfrog_artifactory.py b/products/warehouse_sources/backend/temporal/data_imports/sources/jfrog_artifactory/jfrog_artifactory.py
index 001753e8fec3..9d067c3fd682 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/jfrog_artifactory/jfrog_artifactory.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/jfrog_artifactory/jfrog_artifactory.py
@@ -8,11 +8,14 @@ from urllib.parse import urlparse
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.cloud_utils import is_cloud
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.jfrog_artifactory.settings import (
@@ -237,7 +240,7 @@ def build_xray_violations_request(
}
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
JfrogArtifactoryRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/jumpcloud/jumpcloud.py b/products/warehouse_sources/backend/temporal/data_imports/sources/jumpcloud/jumpcloud.py
index c0a48f80f6b0..eb8c1a0e7276 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/jumpcloud/jumpcloud.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/jumpcloud/jumpcloud.py
@@ -7,9 +7,12 @@ from urllib.parse import quote, urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.jumpcloud.settings import (
@@ -113,7 +116,7 @@ def _retry_wait(retry_state: RetryCallState) -> float:
return wait_exponential_jitter(initial=1, max=30)(retry_state)
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((JumpcloudRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRIES),
wait=_retry_wait,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/kernel/kernel.py b/products/warehouse_sources/backend/temporal/data_imports/sources/kernel/kernel.py
index 938d446c9e32..cc6e8ac6ce8d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/kernel/kernel.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/kernel/kernel.py
@@ -9,10 +9,13 @@ from urllib.parse import quote, urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.batcher import Batcher
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.kernel.settings import (
BROWSER_TELEMETRY_EVENTS,
@@ -156,7 +159,7 @@ def _next_page(headers: Any, current_offset: int, page_len: int) -> tuple[bool,
return has_more, current_offset + page_len
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
KernelRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/klaus/klaus.py b/products/warehouse_sources/backend/temporal/data_imports/sources/klaus/klaus.py
index 94b7c982d7bb..22fcf54f8862 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/klaus/klaus.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/klaus/klaus.py
@@ -6,9 +6,12 @@ from typing import Any, Optional
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.klaus.settings import (
@@ -101,7 +104,7 @@ def _retry_wait(retry_state: RetryCallState) -> float:
return wait_exponential_jitter(initial=10, max=MAX_RETRY_WAIT_SECONDS)(retry_state)
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
KlausRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/kong_konnect/kong_konnect.py b/products/warehouse_sources/backend/temporal/data_imports/sources/kong_konnect/kong_konnect.py
index 2707d8c78f3d..f0e4d15ea962 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/kong_konnect/kong_konnect.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/kong_konnect/kong_konnect.py
@@ -6,9 +6,12 @@ from urllib.parse import parse_qs, urlparse
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.kong_konnect.settings import (
@@ -116,7 +119,7 @@ def _build_body(start: str, end: str, offset: int, size: int) -> dict[str, Any]:
}
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
KongKonnectRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/kubecost/kubecost.py b/products/warehouse_sources/backend/temporal/data_imports/sources/kubecost/kubecost.py
index 8ff38de4252e..8d430d41fbfb 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/kubecost/kubecost.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/kubecost/kubecost.py
@@ -6,9 +6,12 @@ from urllib.parse import urlparse
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.sync_window import SyncWindow
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -184,7 +187,7 @@ def get_rows(
session = _get_session(api_key)
url = f"{base_url}{config.path}"
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((KubecostRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
wait=wait_exponential_jitter(initial=2, max=90),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/lacework/lacework.py b/products/warehouse_sources/backend/temporal/data_imports/sources/lacework/lacework.py
index 67ac7d05cecd..4c5421ce58fd 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/lacework/lacework.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/lacework/lacework.py
@@ -7,9 +7,12 @@ from urllib.parse import urlencode, urlparse
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.lacework.settings import (
@@ -154,7 +157,7 @@ class LaceworkClient:
# sample scrubbers, so keep auth calls out of HTTP sample capture entirely.
self._auth_session = make_tracked_session(redact_values=(secret_key,), capture=False)
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((LaceworkRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRIES),
wait=_retry_wait,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/langfuse/langfuse.py b/products/warehouse_sources/backend/temporal/data_imports/sources/langfuse/langfuse.py
index 6a169535e90c..0803692b1dee 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/langfuse/langfuse.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/langfuse/langfuse.py
@@ -10,11 +10,14 @@ from urllib.parse import quote, urlparse
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from urllib3.util.retry import Retry
from products.warehouse_sources.backend.temporal.data_imports.sources.common.datetime_utils import parse_datetime_value
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import _is_host_safe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -408,7 +411,7 @@ def get_rows(
# MAX_RETRY_AFTER_SECONDS) is the only retry layer.
session = make_tracked_session(retry=Retry(total=0))
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((LangfuseRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRIES),
wait=_retry_wait,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/langsmith/langsmith.py b/products/warehouse_sources/backend/temporal/data_imports/sources/langsmith/langsmith.py
index 2bb0b2cc0e55..ac0dab805f81 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/langsmith/langsmith.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/langsmith/langsmith.py
@@ -10,7 +10,7 @@ from urllib.parse import quote, urlencode, urlparse
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.cloud_utils import is_cloud
@@ -18,7 +18,10 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.bat
from products.warehouse_sources.backend.temporal.data_imports.sources.common.boundary_checkpoint import (
BoundaryCheckpoint,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import _is_host_safe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -360,7 +363,7 @@ def validate_credentials(api_key: str, host: str | None, team_id: int | None = N
)
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
LangSmithRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/linearb/linearb.py b/products/warehouse_sources/backend/temporal/data_imports/sources/linearb/linearb.py
index 2f033dc99655..04e3ff07d4de 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/linearb/linearb.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/linearb/linearb.py
@@ -6,9 +6,12 @@ from typing import Any
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.linearb.settings import (
@@ -51,7 +54,7 @@ def _sanitize_metric_key(key: str) -> str:
return _NON_COLUMN_SAFE.sub("_", key).strip("_")
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
LinearbRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/linkrunner/linkrunner.py b/products/warehouse_sources/backend/temporal/data_imports/sources/linkrunner/linkrunner.py
index de86776f6466..268041e48008 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/linkrunner/linkrunner.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/linkrunner/linkrunner.py
@@ -5,13 +5,16 @@ from typing import Any, Optional
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.batcher import Batcher
from products.warehouse_sources.backend.temporal.data_imports.sources.common.boundary_checkpoint import (
BoundaryCheckpoint,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.retry_limits import max_retry_after_seconds
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -106,7 +109,7 @@ def _stop_retrying(retry_state: RetryCallState) -> bool:
return isinstance(exc, LinkrunnerRateLimitError) and exc.retry_after > max_retry_after_seconds()
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
LinkrunnerRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/llama_cloud/llama_cloud.py b/products/warehouse_sources/backend/temporal/data_imports/sources/llama_cloud/llama_cloud.py
index ad0c42751b63..d554ff1b2857 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/llama_cloud/llama_cloud.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/llama_cloud/llama_cloud.py
@@ -5,12 +5,15 @@ from typing import Any, Optional
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.backend.temporal.data_imports.sources.common.datetime_utils import (
coerce_datetime_to_utc,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.llama_cloud.settings import (
@@ -91,7 +94,7 @@ def _format_incremental_value(config: LlamaCloudEndpointConfig, value: Any) -> s
return _format_datetime(value)
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
LlamaCloudRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/lob/lob.py b/products/warehouse_sources/backend/temporal/data_imports/sources/lob/lob.py
index bc4729976e07..055c399e26cf 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/lob/lob.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/lob/lob.py
@@ -6,9 +6,12 @@ from urllib.parse import parse_qsl, urlsplit
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.lob.settings import (
@@ -123,7 +126,7 @@ def validate_credentials(api_key: str) -> tuple[bool, int | None]:
return response.status_code == 200, response.status_code
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((LobRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/logz_io/logz_io.py b/products/warehouse_sources/backend/temporal/data_imports/sources/logz_io/logz_io.py
index 0f9bac684789..5d35b6c94854 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/logz_io/logz_io.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/logz_io/logz_io.py
@@ -8,9 +8,12 @@ from typing import Any, Optional
import orjson
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.logz_io.settings import (
@@ -72,7 +75,7 @@ def _format_timestamp(value: Any) -> str:
return dt.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(LogzIORetryableError, requests.ReadTimeout, requests.ConnectionError, requests.exceptions.ChunkedEncodingError)
),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/luma/luma.py b/products/warehouse_sources/backend/temporal/data_imports/sources/luma/luma.py
index f9598164eb93..ba47ca40db7b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/luma/luma.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/luma/luma.py
@@ -4,9 +4,12 @@ from typing import Any, Optional
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.retry_limits import max_retry_after_seconds
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -67,7 +70,7 @@ def _retry_wait(retry_state: RetryCallState) -> float:
# Luma holds a rate-limited key for one minute, so the waits must add up to more than that.
# Eight attempts wait about 1+2+4+8+16+30+30 = 91 seconds in total.
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((LumaRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(8),
wait=_retry_wait,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/mailgun.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/mailgun.py
index 81ecabe4342f..537be3892177 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/mailgun.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/mailgun.py
@@ -13,7 +13,7 @@ import requests
import structlog
from asgiref.sync import async_to_sync
from structlog.types import FilteringBoundLogger
-from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
@@ -22,7 +22,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.bas
WebhookDeletionResult,
WebhookSyncResult,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
@@ -257,7 +260,7 @@ def _normalize_row(config: MailgunEndpointConfig, domain: Optional[str], item: d
return row
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((MailgunRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRIES),
wait=_retry_wait,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/matomo/matomo.py b/products/warehouse_sources/backend/temporal/data_imports/sources/matomo/matomo.py
index 2c83fd6997ce..1d85bf3b9afe 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/matomo/matomo.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/matomo/matomo.py
@@ -6,11 +6,14 @@ from urllib.parse import urlparse
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.dataclasses import frozen
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.matomo.settings import (
@@ -114,7 +117,7 @@ def get_rows(
session = _get_session(api_token)
api_url = f"{normalize_host(host)}/index.php"
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((MatomoRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
wait=wait_exponential_jitter(initial=2, max=90),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mention/mention.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mention/mention.py
index 3caea3f576c7..a1a4e8e74096 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mention/mention.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mention/mention.py
@@ -6,11 +6,14 @@ from urllib.parse import urlencode, urljoin
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.dataclasses import frozen
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.mention.settings import MENTION_ENDPOINTS
@@ -102,7 +105,7 @@ def _more_url(payload: dict[str, Any]) -> Optional[str]:
return resolved
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((MentionRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/metaplane/metaplane.py b/products/warehouse_sources/backend/temporal/data_imports/sources/metaplane/metaplane.py
index 6c496d920ef9..ceef96e9a21f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/metaplane/metaplane.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/metaplane/metaplane.py
@@ -5,9 +5,12 @@ from typing import Any, Optional
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.metaplane.settings import METAPLANE_ENDPOINTS
@@ -80,7 +83,7 @@ def validate_credentials(api_key: str) -> bool:
return False
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((MetaplaneRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mistral_ai/mistral_ai.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mistral_ai/mistral_ai.py
index e73514fdecfd..406d8c7fa2c5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mistral_ai/mistral_ai.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mistral_ai/mistral_ai.py
@@ -5,9 +5,12 @@ from typing import Any, Optional
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.mistral_ai.settings import (
@@ -101,7 +104,7 @@ def _build_base_params(
return params
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
MistralAIRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mixpanel/mixpanel.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mixpanel/mixpanel.py
index cee8bac2f3d4..ad09d97f9401 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mixpanel/mixpanel.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mixpanel/mixpanel.py
@@ -9,10 +9,13 @@ from typing import Any, Optional
import orjson
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from urllib3.util.retry import Retry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.mixpanel.settings import (
@@ -237,7 +240,7 @@ def _check_response(response: requests.Response, url: str, logger: FilteringBoun
return response
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((MixpanelRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=_retry_wait,
@@ -311,10 +314,11 @@ def _stream_export_day(
resume relies on)."""
params = {"from_date": from_date, "to_date": from_date, "project_id": project_id}
for attempt in range(1, STREAM_MAX_ATTEMPTS + 1):
+ # Outside the try: when `_request` gave up, its own retries are spent, so re-fetching
+ # here would multiply the stall by `STREAM_MAX_ATTEMPTS`.
+ response_cm = _request("GET", url, username=username, secret=secret, logger=logger, params=params, stream=True)
try:
- with _request(
- "GET", url, username=username, secret=secret, logger=logger, params=params, stream=True
- ) as response:
+ with response_cm as response:
batch: list[dict[str, Any]] = []
for line in response.iter_lines():
if not line:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mixpanel/test_mixpanel.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mixpanel/test_mixpanel.py
index 58a02a0d6d5f..7ffb118e1707 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mixpanel/test_mixpanel.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mixpanel/test_mixpanel.py
@@ -403,6 +403,37 @@ class TestExportStreamRetry:
assert mock_request.call_count == 1
mock_sleep.assert_not_called()
+ @parameterized.expand(
+ [
+ ("read_timeout", requests.ReadTimeout("stalled")),
+ ("connection_error", requests.ConnectionError("refused")),
+ ]
+ )
+ def test_does_not_refetch_when_request_gave_up(self, _name: str, error: Exception) -> None:
+ manager = FakeManager()
+ day = date(2024, 1, 1)
+ with (
+ patch.object(mp, "_request", side_effect=error) as mock_request,
+ patch.object(mp.time, "sleep") as mock_sleep,
+ ):
+ with pytest.raises(type(error)):
+ list(
+ mp._iter_export(
+ "us",
+ "u",
+ "s",
+ "123",
+ LOGGER,
+ manager, # type: ignore[arg-type]
+ start_date=day,
+ end_date=day,
+ api_version=MIXPANEL_API_VERSION_V1,
+ )
+ )
+
+ assert mock_request.call_count == 1
+ mock_sleep.assert_not_called()
+
class TestEngageIterator:
def test_paginates_with_session_id_and_saves_state(self) -> None:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/monday/monday.py b/products/warehouse_sources/backend/temporal/data_imports/sources/monday/monday.py
index a2451221a0f0..8c1d022c1cd9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/monday/monday.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/monday/monday.py
@@ -4,9 +4,12 @@ from typing import Any, Optional
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.monday.settings import MONDAY_ENDPOINTS
@@ -312,7 +315,7 @@ def get_rows(
) -> Iterator[list[dict[str, Any]]]:
session = _get_session(api_token, api_version)
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((MondayRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
wait=wait_exponential_jitter(initial=5, max=120),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/monte_carlo/monte_carlo.py b/products/warehouse_sources/backend/temporal/data_imports/sources/monte_carlo/monte_carlo.py
index 4a2ecc0edd53..7e538fd99666 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/monte_carlo/monte_carlo.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/monte_carlo/monte_carlo.py
@@ -6,9 +6,12 @@ from typing import Any, Optional
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.monte_carlo.settings import (
@@ -92,7 +95,7 @@ def _parse_incremental_value(value: Any) -> datetime | None:
return None
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
MonteCarloRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/nebius_ai/nebius_ai.py b/products/warehouse_sources/backend/temporal/data_imports/sources/nebius_ai/nebius_ai.py
index c3cf76f097e7..de84ddb49e9c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/nebius_ai/nebius_ai.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/nebius_ai/nebius_ai.py
@@ -6,9 +6,12 @@ from urllib.parse import quote, urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.nebius_ai.settings import (
@@ -89,7 +92,7 @@ def validate_credentials(api_key: str) -> tuple[bool, str | None]:
return False, f"Nebius AI API returned status {response.status_code}, try reconnecting shortly."
-@retry(
+@retry_with_deadline(
# ChunkedEncodingError is a mid-stream connection break, transient like ConnectionError/ReadTimeout.
retry=retry_if_exception_type(
(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/new_relic/new_relic.py b/products/warehouse_sources/backend/temporal/data_imports/sources/new_relic/new_relic.py
index 7013880c6f2e..db47e8e73213 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/new_relic/new_relic.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/new_relic/new_relic.py
@@ -6,9 +6,12 @@ from typing import Any, Optional
import requests
import structlog
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.new_relic.settings import (
@@ -160,7 +163,7 @@ def _get_session(api_key: str) -> requests.Session:
return make_tracked_session(headers={"API-Key": api_key}, redact_values=(api_key,))
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
NewRelicRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/new_york_times/new_york_times.py b/products/warehouse_sources/backend/temporal/data_imports/sources/new_york_times/new_york_times.py
index 989988a3347a..5da716fac298 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/new_york_times/new_york_times.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/new_york_times/new_york_times.py
@@ -7,9 +7,12 @@ from urllib.parse import urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.new_york_times.settings import (
@@ -84,7 +87,7 @@ def _sanitize_url(url: str) -> str:
return url.split("?", 1)[0]
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
NewYorkTimesRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/northflank.py b/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/northflank.py
index 5339f3f1c28e..42cda34fc1ee 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/northflank.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/northflank/northflank.py
@@ -4,10 +4,13 @@ from urllib.parse import urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from urllib3.util.retry import Retry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.northflank.settings import NORTHFLANK_ENDPOINTS
@@ -128,7 +131,7 @@ def _make_fetch_page(api_token: str, logger: FilteringBoundLogger) -> FetchPageF
# bypassing this connector's capped rate-limit backoff.
session = make_tracked_session(retry=Retry(total=0), redact_values=(api_token,))
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((NorthflankRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRIES),
wait=_retry_wait,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/nuget/nuget.py b/products/warehouse_sources/backend/temporal/data_imports/sources/nuget/nuget.py
index 622b63b1e2ae..3db4f2ed022c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/nuget/nuget.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/nuget/nuget.py
@@ -7,9 +7,12 @@ from urllib.parse import quote, urlencode
import requests
from dateutil import parser as dateutil_parser
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.nuget.settings import (
@@ -99,7 +102,7 @@ def _resolve_resource(service_index: dict[str, Any], preferred_types: tuple[str,
raise ValueError(f"NuGet service index has no resource of type {preferred_types[0]}")
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(NugetRetryableError, requests.ReadTimeout, requests.ConnectionError, requests.exceptions.ChunkedEncodingError)
),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/octopus_deploy/octopus_deploy.py b/products/warehouse_sources/backend/temporal/data_imports/sources/octopus_deploy/octopus_deploy.py
index de0753759da1..0c2ccfbd87f9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/octopus_deploy/octopus_deploy.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/octopus_deploy/octopus_deploy.py
@@ -9,10 +9,13 @@ from urllib.parse import urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from urllib3.util.retry import Retry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import _is_host_safe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -193,7 +196,7 @@ def _retry_wait(retry_state: RetryCallState) -> float:
return wait_exponential_jitter(initial=1, max=30)(retry_state)
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((OctopusDeployRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRIES),
wait=_retry_wait,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/open_exchange_rates.py b/products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/open_exchange_rates.py
index cf11a0242e9c..a9cb79b1858c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/open_exchange_rates.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/open_exchange_rates/open_exchange_rates.py
@@ -6,11 +6,14 @@ from urllib.parse import urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.exceptions_capture import capture_exception
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.open_exchange_rates.settings import (
@@ -226,7 +229,7 @@ def get_rows(
session = make_tracked_session(headers=_auth_headers(app_id), redact_values=(app_id,) if app_id else ())
base = base_currency or DEFAULT_BASE_CURRENCY
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type(
(OpenExchangeRatesRetryableError, requests.ReadTimeout, requests.ConnectionError)
),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openrouter/openrouter.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openrouter/openrouter.py
index 1f87fbdf00c7..01ddfcb9c668 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openrouter/openrouter.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openrouter/openrouter.py
@@ -6,9 +6,12 @@ from urllib.parse import urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.openrouter.settings import (
@@ -51,7 +54,7 @@ def _build_url(path: str, params: Optional[dict[str, Any]] = None) -> str:
return url
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
OpenRouterRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ortto/ortto.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ortto/ortto.py
index 4b66889d2461..3da1ddfcf3ac 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ortto/ortto.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ortto/ortto.py
@@ -4,9 +4,12 @@ from typing import Any, Optional
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.ortto.settings import (
@@ -72,7 +75,7 @@ def get_rows(
session = _get_session(api_key)
base_url = _base_url(region)
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((OrttoRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
wait=wait_exponential_jitter(initial=2, max=90),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pabbly_subscriptions_billing/pabbly_subscriptions_billing.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pabbly_subscriptions_billing/pabbly_subscriptions_billing.py
index 54a6db84a295..75020343fdce 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pabbly_subscriptions_billing/pabbly_subscriptions_billing.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pabbly_subscriptions_billing/pabbly_subscriptions_billing.py
@@ -4,9 +4,12 @@ from typing import Any, Optional
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.pabbly_subscriptions_billing.settings import (
@@ -56,7 +59,7 @@ def _is_no_data_message(message: str) -> bool:
return lowered.startswith("no ") and "found" in lowered
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((PabblyRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/packagist/packagist.py b/products/warehouse_sources/backend/temporal/data_imports/sources/packagist/packagist.py
index ab3888c9f280..4a10d2e17e89 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/packagist/packagist.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/packagist/packagist.py
@@ -7,9 +7,12 @@ from urllib.parse import quote, urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.packagist.settings import (
@@ -112,7 +115,7 @@ def _advisories_url(packages: list[str]) -> str:
return f"{PACKAGIST_BASE_URL}/api/security-advisories/?{urlencode([('packages[]', p) for p in packages])}"
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((PackagistRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/payfit.py b/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/payfit.py
index 0d38baa2ef4d..12787e9db490 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/payfit.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/payfit/payfit.py
@@ -4,9 +4,12 @@ from typing import Any
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.payfit.settings import (
@@ -60,7 +63,7 @@ def _make_introspection_session(api_key: str) -> requests.Session:
)
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((PayFitRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
@@ -101,7 +104,7 @@ def get_company_id(api_key: str) -> str:
return company_id
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((PayFitRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
@@ -143,7 +146,7 @@ def _fetch_page(
return rows, next_token or None
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((PayFitRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/pendo.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/pendo.py
index 83633713bc80..980bd8e71411 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/pendo.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pendo/pendo.py
@@ -4,9 +4,12 @@ from typing import Any, Optional
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.pendo.settings import (
@@ -51,7 +54,7 @@ def _get_headers(integration_key: str) -> dict[str, str]:
}
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((PendoRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRIES),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/persona/persona.py b/products/warehouse_sources/backend/temporal/data_imports/sources/persona/persona.py
index 39c0c23fe071..4c3376ef1778 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/persona/persona.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/persona/persona.py
@@ -7,11 +7,14 @@ from urllib.parse import urljoin, urlparse
import requests
from dateutil import parser as date_parser
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.batcher import Batcher
from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import UNVERSIONED_API_VERSION
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.persona.settings import (
@@ -148,7 +151,7 @@ def validate_credentials(api_key: str, api_version: str) -> int:
return 0
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
PersonaRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/plaid.py b/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/plaid.py
index 432bb5baace2..e3644621850d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/plaid.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/plaid/plaid.py
@@ -6,9 +6,12 @@ from typing import Any, Optional
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.plaid.settings import PLAID_ENDPOINTS
@@ -91,7 +94,7 @@ def get_rows(
base_url = _base_url(environment)
credentials = {"client_id": client_id, "secret": secret, "access_token": access_token}
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((PlaidRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
wait=wait_exponential_jitter(initial=5, max=120),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/printify/printify.py b/products/warehouse_sources/backend/temporal/data_imports/sources/printify/printify.py
index 7a67e3355ddc..e44cc2d8e983 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/printify/printify.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/printify/printify.py
@@ -5,9 +5,12 @@ from urllib.parse import urlsplit
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.printify.settings import (
@@ -45,7 +48,7 @@ def _headers(api_key: str) -> dict[str, str]:
}
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((PrintifyRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pulumi_cloud/pulumi_cloud.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pulumi_cloud/pulumi_cloud.py
index af37fe7fcb3c..007db25b7f79 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pulumi_cloud/pulumi_cloud.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pulumi_cloud/pulumi_cloud.py
@@ -8,9 +8,12 @@ from urllib.parse import quote
import orjson
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.pulumi_cloud.settings import (
@@ -79,7 +82,7 @@ def _as_unix_seconds(value: Any) -> int | None:
return None
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
PulumiCloudRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pylon/pylon.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pylon/pylon.py
index dbde79a3046d..8d07fe464102 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pylon/pylon.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pylon/pylon.py
@@ -6,9 +6,12 @@ from urllib.parse import urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.pylon.settings import (
@@ -78,7 +81,7 @@ def _build_url(base_url: str, params: dict[str, Any]) -> str:
return f"{base_url}?{urlencode(params)}"
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((PylonRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=2, max=60),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pypi/pypi.py b/products/warehouse_sources/backend/temporal/data_imports/sources/pypi/pypi.py
index 620e4107db1f..2b62abd7ba98 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pypi/pypi.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/pypi/pypi.py
@@ -5,9 +5,12 @@ from urllib.parse import quote
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.pypi.settings import (
PYPI_ENDPOINTS,
@@ -83,7 +86,7 @@ def _project_url(package: str) -> str:
return f"{PYPI_BASE_URL}/pypi/{quote(package, safe='')}/json"
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((PyPIRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/qualys_vmdr/qualys_vmdr.py b/products/warehouse_sources/backend/temporal/data_imports/sources/qualys_vmdr/qualys_vmdr.py
index 394efec41f75..424269c33c20 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/qualys_vmdr/qualys_vmdr.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/qualys_vmdr/qualys_vmdr.py
@@ -12,11 +12,14 @@ from xml.etree.ElementTree import Element
import requests
import defusedxml.ElementTree as DET
from structlog.types import FilteringBoundLogger
-from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.security.url_validation import is_url_allowed
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.qualys_vmdr.settings import (
@@ -212,7 +215,7 @@ def _parse_xml(text: str) -> Element:
return root
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
QualysVmdrRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/railway/railway.py b/products/warehouse_sources/backend/temporal/data_imports/sources/railway/railway.py
index f6d728b1692d..2daaea1fb302 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/railway/railway.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/railway/railway.py
@@ -5,9 +5,12 @@ from typing import Any, Optional
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.railway.settings import (
@@ -65,7 +68,7 @@ def _raise_for_graphql_errors(payload: dict[str, Any]) -> None:
raise Exception(f"Railway GraphQL error: {messages}")
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
RailwayRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/render/render.py b/products/warehouse_sources/backend/temporal/data_imports/sources/render/render.py
index a42b6f933dcc..e6d1cef89b86 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/render/render.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/render/render.py
@@ -6,10 +6,13 @@ from urllib.parse import urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.backend.temporal.data_imports.sources.common import source_helpers
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.render.settings import (
@@ -101,7 +104,7 @@ def validate_credentials(api_key: str) -> tuple[bool, str | None]:
return False, PROBE_FAILED_MESSAGE
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
RenderRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/replicate.py b/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/replicate.py
index c0cb7946073a..0e45f366c08e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/replicate.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/replicate/replicate.py
@@ -6,9 +6,12 @@ from urllib.parse import urlencode, urlsplit, urlunsplit
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.replicate.settings import (
@@ -85,7 +88,7 @@ def validate_credentials(api_key: str) -> bool:
return False
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
ReplicateRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/rollbar/rollbar.py b/products/warehouse_sources/backend/temporal/data_imports/sources/rollbar/rollbar.py
index 019cae865468..70f490b12670 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/rollbar/rollbar.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/rollbar/rollbar.py
@@ -5,9 +5,12 @@ from urllib.parse import urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.rollbar.settings import ROLLBAR_ENDPOINTS
@@ -97,7 +100,7 @@ def get_rows(
config = ROLLBAR_ENDPOINTS[endpoint]
session = _get_session(access_token)
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((RollbarRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
wait=wait_exponential_jitter(initial=1, max=60),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/rubygems/rubygems.py b/products/warehouse_sources/backend/temporal/data_imports/sources/rubygems/rubygems.py
index b48151ff072f..bfd00f93b45d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/rubygems/rubygems.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/rubygems/rubygems.py
@@ -5,9 +5,12 @@ from urllib.parse import quote
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.rubygems.settings import (
RUBYGEMS_ENDPOINTS,
@@ -74,7 +77,7 @@ def _versions_url(gem: str) -> str:
return f"{RUBYGEMS_BASE_URL}/versions/{quote(gem, safe='')}.json"
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((RubyGemsRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/runpod/runpod.py b/products/warehouse_sources/backend/temporal/data_imports/sources/runpod/runpod.py
index e8cc8a155a2e..ec2c5da8d7fe 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/runpod/runpod.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/runpod/runpod.py
@@ -13,9 +13,12 @@ from urllib.parse import urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.runpod.settings import (
@@ -74,7 +77,7 @@ def _build_url(path: str, params: dict[str, Any]) -> str:
return f"{url}?{encoded}" if encoded else url
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
RunPodRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/sage_hr/sage_hr.py b/products/warehouse_sources/backend/temporal/data_imports/sources/sage_hr/sage_hr.py
index 3f3a74979ebe..0325ef334bb3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/sage_hr/sage_hr.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/sage_hr/sage_hr.py
@@ -7,9 +7,12 @@ from urllib.parse import urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.sync_window import SyncWindow
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -93,7 +96,7 @@ def _extract_next_page(payload: dict[str, Any]) -> int | None:
return next_page if isinstance(next_page, int) else None
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((SageHRRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/segment/segment.py b/products/warehouse_sources/backend/temporal/data_imports/sources/segment/segment.py
index 09453492a233..f27a132e5310 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/segment/segment.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/segment/segment.py
@@ -5,9 +5,12 @@ from urllib.parse import quote
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.segment.settings import (
@@ -55,7 +58,7 @@ def _build_url(base_url: str, path: str, cursor: str | None) -> str:
return f"{base_url}{path}?{'&'.join(params)}"
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((SegmentRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/shippo/shippo.py b/products/warehouse_sources/backend/temporal/data_imports/sources/shippo/shippo.py
index 984aa900bc0c..f57f2619184d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/shippo/shippo.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/shippo/shippo.py
@@ -6,9 +6,12 @@ from urllib.parse import parse_qs, urlencode, urlsplit
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.shippo.settings import SHIPPO_ENDPOINTS
@@ -91,7 +94,7 @@ def _build_url(path: str, params: dict[str, Any]) -> str:
return f"{SHIPPO_BASE_URL}{path}/?{urlencode(clean_params)}"
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((ShippoRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRIES),
wait=wait_exponential_jitter(initial=5, max=90),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/shopify.py b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/shopify.py
index 93fc880cc619..05bc37d373c8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/shopify.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/shopify/shopify.py
@@ -10,13 +10,16 @@ import requests
import structlog
from requests.exceptions import ChunkedEncodingError
from structlog.types import FilteringBoundLogger
-from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from tenacity.nap import sleep as retry_sleep
from posthog.dataclasses import frozen
from posthog.exceptions_capture import capture_exception
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http.transport import (
default_request_timeout,
)
@@ -399,7 +402,7 @@ def iter_shopify_pages(
else:
stop.wait(seconds)
- @retry(
+ @retry_with_deadline(
# Transient network failures (proxy/egress hiccups, connect/read timeouts) surface from
# `sess.post` as requests ConnectionError/Timeout — e.g. a 504 from the egress proxy tunnel.
# They're retryable like a 5xx, so reissue the request with backoff instead of failing the
@@ -665,7 +668,7 @@ def _access_token_auth_error_message(error_code: str | None) -> str:
return SHOPIFY_ACCESS_TOKEN_AUTH_ERROR
-@retry(
+@retry_with_deadline(
# A transient TLS/connection drop on the token endpoint (e.g. SSL EOF, proxy/egress hiccup,
# connect/read timeout) surfaces from `post` as requests ConnectionError/Timeout — SSLError
# is a ConnectionError. A connection dropped mid-response surfaces as ChunkedEncodingError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/signoz/signoz.py b/products/warehouse_sources/backend/temporal/data_imports/sources/signoz/signoz.py
index 14919a350443..11b9acd9a26a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/signoz/signoz.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/signoz/signoz.py
@@ -6,9 +6,12 @@ from typing import Any, Optional
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import _is_host_safe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.sync_window import SyncWindow
@@ -285,7 +288,7 @@ def get_rows(
# can't reliably strip, so they must never reach the shared sample bucket.
session = make_tracked_session(headers=_get_headers(api_key), redact_values=(api_key,), capture=False)
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((SigNozRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRIES),
wait=_retry_wait,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/singular/singular.py b/products/warehouse_sources/backend/temporal/data_imports/sources/singular/singular.py
index 003a9071e51b..cebb22f5e001 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/singular/singular.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/singular/singular.py
@@ -12,7 +12,11 @@ from tenacity import Retrying, retry_if_exception_type, stop_after_attempt, wait
from posthog.dataclasses import frozen
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ request_deadline,
+ stop_at_request_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.singular.settings import (
@@ -175,7 +179,7 @@ class SingularClient:
# The transport retries only GETs, so the report POST needs its own bounded retry.
self._create_retrying = Retrying(
retry=retry_if_exception_type((SingularRetryableError, requests.ReadTimeout, requests.ConnectionError)),
- stop=stop_after_attempt(CREATE_MAX_ATTEMPTS),
+ stop=stop_after_attempt(CREATE_MAX_ATTEMPTS) | stop_at_request_deadline(),
wait=wait_exponential_jitter(initial=5, max=60),
reraise=True,
)
@@ -195,7 +199,8 @@ class SingularClient:
def create_report(self, query: SingularReportQuery, day: dt.date) -> str:
try:
- value = self._create_retrying(self._post_report, query, day)
+ with request_deadline():
+ value = self._create_retrying(self._post_report, query, day)
except SingularRateLimitError as error:
# Singular does not document how it reports a used-up daily row quota. A 429 that
# outlasts the retries is the closest signal, and waiting longer inside one run cannot
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/smaily/smaily.py b/products/warehouse_sources/backend/temporal/data_imports/sources/smaily/smaily.py
index 3fc624c9c310..ae44a0f47d92 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/smaily/smaily.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/smaily/smaily.py
@@ -5,9 +5,12 @@ from typing import Any, Optional
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.smaily.settings import (
@@ -66,7 +69,7 @@ def _make_session(username: str, password: str) -> requests.Session:
return session
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((SmailyRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/snowplow/snowplow.py b/products/warehouse_sources/backend/temporal/data_imports/sources/snowplow/snowplow.py
index 820b4bacaa70..14ec2cf0fd53 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/snowplow/snowplow.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/snowplow/snowplow.py
@@ -5,9 +5,12 @@ from typing import Any, Optional
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.sync_window import SyncWindow
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -102,7 +105,7 @@ class SnowplowClient:
def _org_base_url(self) -> str:
return f"{SNOWPLOW_BASE_URL}/organizations/{self._organization_id}"
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type(
(
SnowplowRetryableError,
@@ -139,7 +142,7 @@ class SnowplowClient:
raise SnowplowAuthError("Snowplow API authentication failed: the token response had no access token.")
return token
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type(
(
SnowplowRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/sonarqube/sonarqube.py b/products/warehouse_sources/backend/temporal/data_imports/sources/sonarqube/sonarqube.py
index 9587fc3efb00..53e1e6fc1a8e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/sonarqube/sonarqube.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/sonarqube/sonarqube.py
@@ -8,10 +8,13 @@ from urllib.parse import urlencode, urlparse
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.batcher import Batcher
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.sonarqube.settings import (
@@ -209,7 +212,7 @@ def _read_bounded(response: requests.Response) -> bytes:
return b"".join(chunks)
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((SonarqubeRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/sonatype_nexus/sonatype_nexus.py b/products/warehouse_sources/backend/temporal/data_imports/sources/sonatype_nexus/sonatype_nexus.py
index 16e1a02cb309..389b9b4c0957 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/sonatype_nexus/sonatype_nexus.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/sonatype_nexus/sonatype_nexus.py
@@ -7,11 +7,14 @@ from urllib.parse import quote, urlencode, urlparse
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.cloud_utils import is_cloud
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.sonatype_nexus.settings import (
@@ -167,7 +170,7 @@ def _build_url(base_url: str, params: dict[str, Any]) -> str:
return f"{base_url}?{urlencode(params)}"
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
SonatypeNexusRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/sourcegraph/sourcegraph.py b/products/warehouse_sources/backend/temporal/data_imports/sources/sourcegraph/sourcegraph.py
index 9fb389a10b47..0d381ba6a4a3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/sourcegraph/sourcegraph.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/sourcegraph/sourcegraph.py
@@ -9,9 +9,12 @@ from typing import Any, Optional
import requests
import structlog
from structlog.types import FilteringBoundLogger
-from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import _is_host_safe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -193,7 +196,7 @@ def _retry_wait(retry_state: RetryCallState) -> float:
return wait_exponential_jitter(initial=1, max=30)(retry_state)
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((SourcegraphRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRIES),
wait=_retry_wait,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/spacelift.py b/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/spacelift.py
index f7207637ef2f..e22fbd8de6ee 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/spacelift.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/spacelift/spacelift.py
@@ -7,9 +7,12 @@ from typing import Any, Optional
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.spacelift.settings import (
@@ -261,7 +264,7 @@ class SpaceliftClient:
def close(self) -> None:
self._session.close()
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type(SpaceliftRetryableError),
stop=stop_after_attempt(SPACELIFT_MAX_RETRY_ATTEMPTS),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/split_io/split_io.py b/products/warehouse_sources/backend/temporal/data_imports/sources/split_io/split_io.py
index 57e0061764fc..3c33fa5bbf25 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/split_io/split_io.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/split_io/split_io.py
@@ -5,9 +5,12 @@ from urllib.parse import parse_qs, urlencode, urlsplit, urlunsplit
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt
+from tenacity import retry_if_exception_type, stop_after_attempt
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.split_io.settings import (
@@ -124,7 +127,7 @@ def _wait_strategy(retry_state: Any) -> float:
return min(2.0**retry_state.attempt_number, MAX_RETRY_WAIT_SECONDS)
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((SplitIoRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
wait=_wait_strategy,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/splunk_observability_cloud/splunk_observability_cloud.py b/products/warehouse_sources/backend/temporal/data_imports/sources/splunk_observability_cloud/splunk_observability_cloud.py
index 6cca005aaa8c..3a6835a09a2f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/splunk_observability_cloud/splunk_observability_cloud.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/splunk_observability_cloud/splunk_observability_cloud.py
@@ -8,9 +8,12 @@ from urllib.parse import urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.splunk_observability_cloud.settings import (
@@ -86,7 +89,7 @@ def _ms_to_datetime(value: Any) -> datetime | None:
return datetime.fromtimestamp(int(value) / 1000, tz=UTC)
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
SplunkObservabilityCloudRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/squadcast/squadcast.py b/products/warehouse_sources/backend/temporal/data_imports/sources/squadcast/squadcast.py
index 0efadce5d4f5..6f43cb7d5bf7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/squadcast/squadcast.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/squadcast/squadcast.py
@@ -7,9 +7,12 @@ from urllib.parse import urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.squadcast.settings import (
@@ -151,7 +154,7 @@ class SquadcastClient:
assert self._access_token is not None
return self._access_token
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((SquadcastRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(RETRY_ATTEMPTS),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/square/square.py b/products/warehouse_sources/backend/temporal/data_imports/sources/square/square.py
index 2219acd80b28..be183e490c32 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/square/square.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/square/square.py
@@ -6,11 +6,14 @@ from urllib.parse import urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.dataclasses import frozen
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.square.settings import (
@@ -190,7 +193,7 @@ def get_rows(
if cursor:
logger.debug(f"Square: resuming {endpoint} from saved cursor")
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((SquareRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/squarespace/squarespace.py b/products/warehouse_sources/backend/temporal/data_imports/sources/squarespace/squarespace.py
index c615e0cbd541..1e45081d434d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/squarespace/squarespace.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/squarespace/squarespace.py
@@ -5,9 +5,12 @@ from typing import Any, Optional
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.squarespace.settings import (
@@ -164,7 +167,7 @@ def get_rows(
if cursor:
logger.debug(f"Squarespace: resuming {endpoint} from saved cursor")
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((SquarespaceRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/statuscake/statuscake.py b/products/warehouse_sources/backend/temporal/data_imports/sources/statuscake/statuscake.py
index e64ed522049b..f52d31513a53 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/statuscake/statuscake.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/statuscake/statuscake.py
@@ -6,9 +6,12 @@ from urllib.parse import urlencode, urlsplit
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.statuscake.settings import (
@@ -119,7 +122,7 @@ def _parse_rfc3339(value: Any) -> Optional[datetime]:
return parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=UTC)
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((StatusCakeRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(_MAX_ATTEMPTS),
wait=wait_exponential_jitter(initial=2, max=90),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/stytch.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/stytch.py
index 2e8888239f87..1d6a2cb993cd 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/stytch.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stytch/stytch.py
@@ -5,9 +5,12 @@ from typing import Any, Optional
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.stytch.settings import (
@@ -101,7 +104,7 @@ def build_users_search_body(
return body
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((StytchRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRIES),
wait=wait_exponential_jitter(initial=1, max=60),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/sumo_logic/sumo_logic.py b/products/warehouse_sources/backend/temporal/data_imports/sources/sumo_logic/sumo_logic.py
index 2f60f9a885fb..58f33f888443 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/sumo_logic/sumo_logic.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/sumo_logic/sumo_logic.py
@@ -8,9 +8,12 @@ from urllib.parse import urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.sumo_logic.settings import (
@@ -108,7 +111,7 @@ def validate_credentials(deployment: Optional[str], access_id: str, access_key:
def _make_fetch(session: requests.Session, logger: FilteringBoundLogger) -> Any:
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((SumoLogicRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(RETRY_ATTEMPTS),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/swarmia/swarmia.py b/products/warehouse_sources/backend/temporal/data_imports/sources/swarmia/swarmia.py
index a3be550d4777..56c46e8cf7fc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/swarmia/swarmia.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/swarmia/swarmia.py
@@ -9,9 +9,12 @@ from urllib.parse import urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.swarmia.settings import (
@@ -162,7 +165,7 @@ def _make_session(api_key: str) -> requests.Session:
return make_tracked_session(redact_values=(api_key,), capture=False)
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
SwarmiaRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/taboola/taboola.py b/products/warehouse_sources/backend/temporal/data_imports/sources/taboola/taboola.py
index 29aa9219d414..cbb71d6b6e7f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/taboola/taboola.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/taboola/taboola.py
@@ -6,9 +6,12 @@ from urllib.parse import quote, urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.taboola.settings import (
@@ -107,7 +110,7 @@ def get_rows(
# sync outright. Don't reuse `fetch`'s retry for this: it also retries on
# TaboolaRetryableError, and re-mint failures already propagate into that retry via the
# 401 branch below, so decorating the mint itself too would let the two retries compound.
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type(TaboolaRetryableError),
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
wait=wait_exponential_jitter(initial=2, max=90),
@@ -119,7 +122,7 @@ def get_rows(
token = mint_initial_token()
account_base = f"{TABOOLA_API_BASE_URL}/{_encode_path_segment(account_id)}"
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((TaboolaRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
wait=wait_exponential_jitter(initial=2, max=90),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tailscale/tailscale.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tailscale/tailscale.py
index 0a4210ad49c8..166cda717e67 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tailscale/tailscale.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tailscale/tailscale.py
@@ -8,9 +8,12 @@ from urllib.parse import quote
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.tailscale.settings import (
@@ -149,7 +152,7 @@ def _retry_wait(retry_state: RetryCallState) -> float:
return wait_exponential_jitter(initial=1, max=30)(retry_state)
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((TailscaleRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRIES),
wait=_retry_wait,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tawk_to/tawk_to.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tawk_to/tawk_to.py
index c16faaec773f..e56634e3ae75 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tawk_to/tawk_to.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tawk_to/tawk_to.py
@@ -4,9 +4,12 @@ from typing import Any, Optional
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.tawk_to.settings import (
@@ -60,7 +63,7 @@ def _post(
body: dict[str, Any],
logger: FilteringBoundLogger,
) -> dict[str, Any]:
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((TawkToRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRIES),
wait=wait_exponential_jitter(initial=1, max=60),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/teamcity/teamcity.py b/products/warehouse_sources/backend/temporal/data_imports/sources/teamcity/teamcity.py
index 3c5a82955786..c110b09254a8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/teamcity/teamcity.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/teamcity/teamcity.py
@@ -8,11 +8,14 @@ from urllib.parse import urlencode, urljoin, urlparse
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.cloud_utils import is_cloud
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import _is_host_safe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -184,7 +187,7 @@ def _build_list_url(
return f"{api_base}{path}?{urlencode(params)}"
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((TeamCityRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/tenable_vulnerability_management.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/tenable_vulnerability_management.py
index 156ef520aef5..b17e819d8dc4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/tenable_vulnerability_management.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tenable_vulnerability_management/tenable_vulnerability_management.py
@@ -7,9 +7,12 @@ from urllib.parse import urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.tenable_vulnerability_management.settings import (
@@ -115,7 +118,7 @@ def _to_since_date(value: Any) -> str | None:
return (datetime.fromtimestamp(timestamp, tz=UTC).date() - timedelta(days=1)).isoformat()
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
TenableRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/terraform_cloud/terraform_cloud.py b/products/warehouse_sources/backend/temporal/data_imports/sources/terraform_cloud/terraform_cloud.py
index 19603d8d8fac..f88918eac52a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/terraform_cloud/terraform_cloud.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/terraform_cloud/terraform_cloud.py
@@ -6,9 +6,12 @@ from urllib.parse import quote, urlencode, urljoin
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.terraform_cloud.settings import (
@@ -56,7 +59,7 @@ def _make_session(api_token: str) -> requests.Session:
)
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
TerraformCloudRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/testrail/testrail.py b/products/warehouse_sources/backend/temporal/data_imports/sources/testrail/testrail.py
index b406d81cb45c..d3ec3bbfd1c8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/testrail/testrail.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/testrail/testrail.py
@@ -7,9 +7,12 @@ from urllib.parse import urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.testrail.settings import (
@@ -130,7 +133,7 @@ def _to_epoch(value: Any) -> int:
return int(str(value))
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((TestrailRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=2, max=60),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt
index 09796fd8fabc..f61de89f700f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/source_contract_baseline.txt
@@ -1,293 +1,162 @@
Adjust fail:safe-point
Adyen fail:safe-point
-AlphaVantage fail:stall-budget
-AmazonAds fail:stall-budget
AmazonS3 not-checkable:schemas
-Amplitude fail:stall-budget
Anthropic fail:safe-point
-Apollo fail:rate-limit-budget,stall-budget
AppLovin fail:safe-point
AppStoreConnect not-checkable:no-request
-Appfigures fail:stall-budget
AppleSearchAds not-checkable:no-request
-AppsFlyer fail:safe-point,stall-budget
-Appsignal fail:safe-point,stall-budget
+AppsFlyer fail:safe-point
+Appsignal fail:safe-point
Attentive not-checkable:database
-Automox fail:stall-budget
-Aviator fail:stall-budget
-Awin fail:stall-budget
AwsCostExplorer fail:safe-point
-AwsOrganizations fail:stall-budget
-AwsWaf fail:stall-budget
AzureBlob not-checkable:schemas
AzureCostManagement not-checkable:no-request
-AzureDevOps fail:stall-budget
Baserow not-checkable:schemas
BigMailer fail:safe-point
BigQuery not-checkable:driver:google.cloud
BingAds not-checkable:database
-Bitbucket fail:safe-point,stall-budget
+Bitbucket fail:safe-point
Breezometer not-checkable:no-request
-Bugsnag fail:safe-point,stall-budget
+Bugsnag fail:safe-point
CalCom fail:safe-point
CanvasLms fail:safe-point
-CareQualityCommission fail:stall-budget
Chatwoot not-checkable:database
-Checkmarx fail:stall-budget
-Cimis fail:stall-budget
-CircleCI fail:rate-limit-budget,safe-point,stall-budget
-CircleciInsights fail:rate-limit-budget,safe-point,stall-budget
+CircleCI fail:safe-point
+CircleciInsights fail:safe-point
CiscoDuo not-checkable:no-request
-Clari fail:stall-budget
ClickHouse not-checkable:schemas
-ClickhouseCloud fail:stall-budget
CloudflareR2 not-checkable:schemas
Coda fail:safe-point
-Codacy fail:stall-budget
-Codecov fail:safe-point,stall-budget
-Cody fail:stall-budget
-CoinApi fail:stall-budget
+Codecov fail:safe-point
CoinMarketCap fail:safe-point
-Commercetools fail:stall-budget
-Concord fail:stall-budget
-ConfluentCloud fail:stall-budget
Convex not-checkable:schemas
-Coralogix fail:stall-budget
CosmosDB not-checkable:schemas
-Coupa fail:stall-budget
Coveralls not-checkable:no-request
-CratesIO fail:stall-budget
-Cronitor fail:stall-budget
-Cursor fail:safe-point,stall-budget
+Cursor fail:safe-point
Custom not-checkable:schemas
-CustomerIO fail:stall-budget
-Customerly fail:stall-budget
-DagsterCloud fail:stall-budget
-DataForSEO fail:stall-budget
Databricks not-checkable:driver:databricks
-Datadog fail:safe-point,stall-budget
-Datahub fail:stall-budget
-Dbt fail:stall-budget
-Decagon fail:stall-budget
-Deepsource fail:rate-limit-budget,stall-budget
+Datadog fail:safe-point
Depot not-checkable:no-request
Discourse fail:safe-point
-Dixa fail:safe-point,stall-budget
-Dockerhub fail:safe-point,stall-budget
+Dixa fail:safe-point
+Dockerhub fail:safe-point
Docusign not-checkable:no-request
-Doppler fail:stall-budget
-DropboxSign fail:stall-budget
Dynamics365 not-checkable:no-request
DynamoDB not-checkable:schemas
-Easypost fail:stall-budget
EcbDataPortal fail:safe-point
Elasticsearch not-checkable:schemas
-Everhour fail:stall-budget
-ExchangeRatesApi fail:stall-budget
-Fastly fail:safe-point,stall-budget
+Fastly fail:safe-point
Featurebase not-checkable:database
-Finage fail:stall-budget
-FinancialModelling fail:safe-point,stall-budget
-Finnhub fail:stall-budget
-Finnworlds fail:stall-budget
+FinancialModelling fail:safe-point
Firebase not-checkable:schemas
-Flagsmith fail:safe-point,stall-budget
-Formbricks fail:safe-point,stall-budget
+Flagsmith fail:safe-point
+Formbricks fail:safe-point
Framer not-checkable:no-request
-Freshsales fail:stall-budget
-Frill fail:stall-budget
G2 fail:safe-point
-GNews fail:stall-budget
GainsightPx fail:safe-point
GcpCloudBilling not-checkable:no-request
-Gerrit fail:stall-budget
Ghost not-checkable:no-request
-GitLab fail:stall-budget
-Gitea fail:stall-budget
-Gitguardian fail:safe-point,stall-budget
+Gitguardian fail:safe-point
Github not-checkable:schemas
-Gladly fail:stall-budget
-Gong fail:stall-budget
GoogleAds not-checkable:driver:google.ads
GoogleAnalytics not-checkable:database
GoogleCloudStorage not-checkable:schemas
-GooglePageSpeedInsights fail:stall-budget
GooglePlayConsole not-checkable:no-request
GoogleSearchConsole not-checkable:database
GoogleSheets not-checkable:schemas
-Gorgias fail:stall-budget
-Grafana fail:stall-budget
GrafanaIRM not-checkable:no-request
-Gridly fail:stall-budget
Groq fail:safe-point
-Harvey fail:stall-budget
Hatchet not-checkable:no-request
-Healthchecks fail:stall-budget
-Helicone fail:stall-budget
HelpScout not-checkable:database
Hitpay fail:safe-point
-Honeybadger fail:safe-point,stall-budget
-Honeycomb fail:stall-budget
+Honeybadger fail:safe-point
Hubspot not-checkable:database
-Hyperspell fail:safe-point,stall-budget
-IP2Whois fail:stall-budget
-Inngest fail:stall-budget
+Hyperspell fail:safe-point
Instagram not-checkable:database
-Instana fail:safe-point,stall-budget
+Instana fail:safe-point
Intercom not-checkable:database
-JamfPro fail:stall-budget
-Jellyfish fail:stall-budget
-Jenkins fail:stall-budget
-JfrogArtifactory fail:stall-budget
-Jumpcloud fail:stall-budget
Kafka not-checkable:driver:confluent_kafka
Kandji fail:safe-point
KapaAI fail:safe-point
-Kernel fail:safe-point,stall-budget
+Kernel fail:safe-point
Kestra not-checkable:no-request
-Klaus fail:rate-limit-budget,stall-budget
Klaviyo fail:safe-point
-KongKonnect fail:stall-budget
-Kubecost fail:stall-budget
-Lacework fail:safe-point,stall-budget
-LangSmith fail:stall-budget
-Langfuse fail:rate-limit-budget,stall-budget
+Lacework fail:safe-point
Linear not-checkable:database
-Linearb fail:stall-budget
LinkedinAds not-checkable:database
-Linkrunner fail:stall-budget
-LlamaCloud fail:safe-point,stall-budget
-Lob fail:stall-budget
+LlamaCloud fail:safe-point
Logicmonitor not-checkable:no-request
-LogzIO fail:stall-budget
Lovable fail:safe-point
-Luma fail:rate-limit-budget,safe-point,stall-budget
+Luma fail:safe-point
MSSQL not-checkable:driver:pymssql
MailerSend fail:safe-point
-Mailgun fail:rate-limit-budget,stall-budget
-Matomo fail:stall-budget
Mem0 fail:safe-point
-Mention fail:safe-point,stall-budget
+Mention fail:safe-point
MetaAds not-checkable:database
-Metaplane fail:stall-budget
Meteostat fail:safe-point
-MistralAI fail:stall-budget
-Mixpanel fail:rate-limit-budget,stall-budget
MoEngage fail:safe-point
Modal not-checkable:driver:grpclib
-Monday fail:stall-budget
MongoDB not-checkable:driver:pymongo
-MonteCarlo fail:safe-point,stall-budget
+MonteCarlo fail:safe-point
Motherduck not-checkable:driver:duckdb
MyHours fail:safe-point
MySQL not-checkable:driver:psycopg
-NebiusAI fail:stall-budget
Neo4j not-checkable:schemas
Neon not-checkable:driver:psycopg
-NewRelic fail:safe-point,stall-budget
-NewYorkTimes fail:stall-budget
-Northflank fail:rate-limit-budget,stall-budget
+NewRelic fail:safe-point
Notion fail:safe-point
NpmRegistry fail:safe-point
-Nuget fail:stall-budget
-OctopusDeploy fail:stall-budget
-OpenExchangeRates fail:stall-budget
OpenMeteo not-checkable:no-request
-OpenRouter fail:stall-budget
OpenWeather not-checkable:no-request
Optimizely fail:safe-point
-Ortto fail:safe-point,stall-budget
-PabblySubscriptionsBilling fail:safe-point,stall-budget
-Packagist fail:stall-budget
+Ortto fail:safe-point
+PabblySubscriptionsBilling fail:safe-point
Pardot not-checkable:database
-PayFit fail:stall-budget
-Pendo fail:stall-budget
-Persona fail:stall-budget
PinterestAds not-checkable:database
Pipedrive not-checkable:database
Pipeliner fail:safe-point
-Plaid fail:stall-budget
PlanetScaleMySQL not-checkable:driver:psycopg
PlanetScalePostgres not-checkable:driver:psycopg
Plunk fail:safe-point
Postgres not-checkable:driver:psycopg
PowerBiAdmin fail:safe-point
-Printify fail:stall-budget
ProofpointTap not-checkable:no-request
-PulumiCloud fail:stall-budget
-PyPI fail:stall-budget
-Pylon fail:safe-point,stall-budget
-QualysVmdr fail:rate-limit-budget,stall-budget
-Railway fail:stall-budget
+Pylon fail:safe-point
RedditAds not-checkable:database
Redshift not-checkable:driver:psycopg
-Render fail:stall-budget
-Replicate fail:safe-point,stall-budget
+Replicate fail:safe-point
RevenueCat fail:safe-point
-Rollbar fail:stall-budget
-Rubygems fail:stall-budget
-RunPod fail:safe-point,stall-budget
+RunPod fail:safe-point
SFTP not-checkable:transport
-SageHR fail:safe-point,stall-budget
+SageHR fail:safe-point
Salesforce not-checkable:database
-Segment fail:stall-budget
Sevdesk fail:safe-point
-Shippo fail:safe-point,stall-budget
-Shopify fail:stall-budget
-SigNoz fail:stall-budget
-Singular fail:stall-budget
+Shippo fail:safe-point
Slack not-checkable:schemas
-Smaily fail:safe-point,stall-budget
+Smaily fail:safe-point
SnapchatAds not-checkable:database
Snowflake not-checkable:driver:snowflake
-Snowplow fail:stall-budget
-Sonarqube fail:stall-budget
-SonatypeNexus fail:stall-budget
-Sourcegraph fail:stall-budget
-Spacelift fail:stall-budget
-SplitIo fail:stall-budget
-SplunkObservabilityCloud fail:stall-budget
-Squadcast fail:stall-budget
-Square fail:safe-point,stall-budget
-Squarespace fail:stall-budget
-Statuscake fail:stall-budget
+Square fail:safe-point
Stripe not-checkable:database
-Stytch fail:stall-budget
-SumoLogic fail:safe-point,stall-budget
+SumoLogic fail:safe-point
Supabase not-checkable:driver:psycopg
-Swarmia fail:safe-point,stall-budget
-Taboola fail:stall-budget
-Tailscale fail:safe-point,stall-budget
-TawkTo fail:stall-budget
-Teamcity fail:stall-budget
+Swarmia fail:safe-point
+Tailscale fail:safe-point
TemporalIO not-checkable:driver:temporalio.client
-TenableVulnerabilityManagement fail:stall-budget
-TerraformCloud fail:safe-point,stall-budget
-Testrail fail:stall-budget
+TerraformCloud fail:safe-point
TikTokAds not-checkable:database
-TravisCI fail:stall-budget
-TriggerDev fail:stall-budget
Trino not-checkable:schemas
TrunkIo fail:safe-point
TwitterAds not-checkable:database
-Ubidots fail:safe-point,stall-budget
+Ubidots fail:safe-point
Unleash fail:safe-point
-Unstructured fail:stall-budget
-Uptimerobot fail:stall-budget
-Usersnap fail:stall-budget
-Vapi fail:stall-budget
-Veracode fail:stall-budget
-Vercel fail:stall-budget
Vitally fail:safe-point
-WeightsAndBiases fail:stall-budget
WikipediaPageviews fail:safe-point
-Wordpress fail:stall-budget
Wrike not-checkable:no-request
-Writesonic fail:rate-limit-budget,safe-point,stall-budget
+Writesonic fail:safe-point
YouTubeAnalytics not-checkable:database
ZapSign fail:safe-point
-ZapierSupportedStorage fail:safe-point,stall-budget
-Zenduty fail:safe-point,stall-budget
-Zep fail:stall-budget
+ZapierSupportedStorage fail:safe-point
+Zenduty fail:safe-point
shared-path:rest-client-paginate fail:safe-point
shared-path:rest-framework-wrapped-resource fail:safe-point
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/travis_ci/travis_ci.py b/products/warehouse_sources/backend/temporal/data_imports/sources/travis_ci/travis_ci.py
index 596582644055..066102665183 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/travis_ci/travis_ci.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/travis_ci/travis_ci.py
@@ -5,9 +5,12 @@ from urllib.parse import urlencode, urlsplit
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.travis_ci.settings import TRAVIS_CI_ENDPOINTS
@@ -113,7 +116,7 @@ def validate_credentials(api_token: str) -> tuple[bool, str | None]:
return False, message
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((TravisCIRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRIES),
wait=wait_exponential_jitter(initial=1, max=60),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/trigger_dev/trigger_dev.py b/products/warehouse_sources/backend/temporal/data_imports/sources/trigger_dev/trigger_dev.py
index dba6122ff4ba..ea90ccfad2e6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/trigger_dev/trigger_dev.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/trigger_dev/trigger_dev.py
@@ -7,9 +7,12 @@ from urllib.parse import urlencode, urlsplit
import requests
from dateutil import parser as dateutil_parser
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.trigger_dev.settings import (
@@ -112,7 +115,7 @@ def _should_stop_desc(items: list[dict[str, Any]], incremental_field: str | None
return any(_is_older_than_cutoff(item.get(incremental_field), cutoff) for item in items if item)
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
TriggerDevRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ubidots/ubidots.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ubidots/ubidots.py
index 4fe89f58557e..3f46a627fadc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ubidots/ubidots.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ubidots/ubidots.py
@@ -8,10 +8,13 @@ from urllib.parse import urlencode, urlparse, urlunparse
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import UNVERSIONED_API_VERSION
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.ubidots.settings import (
@@ -144,7 +147,7 @@ def _start_timestamp_ms(value: Any) -> Optional[int]:
return None
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((UbidotsRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
@@ -321,7 +324,7 @@ def _read_capped_body(response: requests.Response) -> bytes:
return b"".join(chunks)
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((UbidotsRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/unstructured.py b/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/unstructured.py
index a29f0db96955..e2b80701de0b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/unstructured.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/unstructured/unstructured.py
@@ -32,9 +32,12 @@ from urllib.parse import urlparse
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import _is_host_safe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
@@ -110,7 +113,7 @@ def _drop_sensitive_fields(rows: list[dict[str, Any]], drop_fields: tuple[str, .
return [{k: v for k, v in row.items() if k not in drop_fields} for row in rows]
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
UnstructuredRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/uptimerobot/uptimerobot.py b/products/warehouse_sources/backend/temporal/data_imports/sources/uptimerobot/uptimerobot.py
index ce3542b6f0b5..64e3758133dc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/uptimerobot/uptimerobot.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/uptimerobot/uptimerobot.py
@@ -6,9 +6,12 @@ from typing import Any, Optional
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.uptimerobot.settings import (
@@ -107,7 +110,7 @@ def _to_unix_timestamp(value: Any) -> int | None:
return None
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((UptimeRobotRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/usersnap/usersnap.py b/products/warehouse_sources/backend/temporal/data_imports/sources/usersnap/usersnap.py
index f748646ae3ed..fe479af053be 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/usersnap/usersnap.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/usersnap/usersnap.py
@@ -8,9 +8,12 @@ from urllib.parse import urlencode
import jwt
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.usersnap.settings import (
@@ -89,7 +92,7 @@ def validate_credentials(jwt_secret: str, jwt_id: str) -> bool:
return False
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
UsersnapRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/vapi/vapi.py b/products/warehouse_sources/backend/temporal/data_imports/sources/vapi/vapi.py
index a95c1ddab329..f4c8400601d3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/vapi/vapi.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/vapi/vapi.py
@@ -7,10 +7,13 @@ from urllib.parse import urlencode
import pyarrow as pa
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.batcher import Batcher
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.vapi.settings import (
@@ -81,7 +84,7 @@ def _make_session(api_key: str) -> requests.Session:
)
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
VapiRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/veracode/veracode.py b/products/warehouse_sources/backend/temporal/data_imports/sources/veracode/veracode.py
index c35e47010855..7e0ede3d0950 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/veracode/veracode.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/veracode/veracode.py
@@ -10,10 +10,13 @@ from urllib.parse import urlencode, urlparse
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from urllib3.util.retry import Retry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.veracode.settings import (
@@ -144,7 +147,7 @@ def _fetch_page_once(session: requests.Session, url: str, logger: FilteringBound
return response.json()
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
VeracodeRetryableError,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/vercel/vercel.py b/products/warehouse_sources/backend/temporal/data_imports/sources/vercel/vercel.py
index 0abf4515bef5..1db21a2afbf3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/vercel/vercel.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/vercel/vercel.py
@@ -14,12 +14,15 @@ from urllib.parse import urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.exceptions_capture import capture_exception
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.batcher import Batcher
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.vercel.settings import (
@@ -114,7 +117,7 @@ def _build_url(path: str, params: dict[str, Any]) -> str:
return f"{VERCEL_BASE_URL}{path}?{urlencode(params)}"
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((VercelRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
@@ -416,7 +419,7 @@ def _billing_window_start(
return max(start, cap)
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((VercelRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/weights_and_biases/weights_and_biases.py b/products/warehouse_sources/backend/temporal/data_imports/sources/weights_and_biases/weights_and_biases.py
index 1f6154ffcf24..8a26f33ef67f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/weights_and_biases/weights_and_biases.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/weights_and_biases/weights_and_biases.py
@@ -8,11 +8,14 @@ from urllib.parse import urlsplit
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from posthog.security.url_validation import is_url_allowed
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.weights_and_biases.settings import WANDB_ENDPOINTS
@@ -550,7 +553,7 @@ def get_rows(
session = _get_session(api_key)
url = _graphql_url(host)
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((WeightsAndBiasesRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRY_ATTEMPTS),
# Rate limits are 50-200 req/min per key depending on plan, so back off generously.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/wordpress/wordpress.py b/products/warehouse_sources/backend/temporal/data_imports/sources/wordpress/wordpress.py
index a7754f6386ad..ed437b55d02b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/wordpress/wordpress.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/wordpress/wordpress.py
@@ -8,10 +8,13 @@ from urllib.parse import urlencode, urlparse
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.batcher import Batcher
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import _is_host_safe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import (
RESTClientNonRetryableError,
@@ -360,7 +363,7 @@ def get_rows(
logger.warning("WordPress: ignoring resume URL whose host does not match the configured host")
url = initial_url
- @retry(
+ @retry_with_deadline(
retry=retry_if_exception_type((WordpressRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(MAX_RETRIES),
wait=_retry_wait,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/writesonic/writesonic.py b/products/warehouse_sources/backend/temporal/data_imports/sources/writesonic/writesonic.py
index ee2a885f790b..8283b295bf8d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/writesonic/writesonic.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/writesonic/writesonic.py
@@ -7,10 +7,13 @@ from urllib.parse import urljoin
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from urllib3.util.retry import Retry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.writesonic.settings import (
@@ -175,7 +178,7 @@ def _check_response(response: requests.Response, url: str, logger: FilteringBoun
return response
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((WritesonicRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=_retry_wait,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/zapier_supported_storage/zapier_supported_storage.py b/products/warehouse_sources/backend/temporal/data_imports/sources/zapier_supported_storage/zapier_supported_storage.py
index 7b40b9b02b5a..f29049b9e00a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/zapier_supported_storage/zapier_supported_storage.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/zapier_supported_storage/zapier_supported_storage.py
@@ -5,10 +5,13 @@ from urllib.parse import urlencode, urlsplit
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.batcher import Batcher
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.zapier_supported_storage.settings import (
ZAPIER_SUPPORTED_STORAGE_ENDPOINTS,
@@ -41,7 +44,7 @@ def _stringify_value(value: Any) -> str | None:
return json.dumps(value)
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(ZapierSupportedStorageRetryableError, requests.ReadTimeout, requests.ConnectionError)
),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/zenduty/zenduty.py b/products/warehouse_sources/backend/temporal/data_imports/sources/zenduty/zenduty.py
index a3926197d4b1..fd2e341216a5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/zenduty/zenduty.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/zenduty/zenduty.py
@@ -5,9 +5,12 @@ from urllib.parse import urlencode, urlsplit
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.zenduty.settings import (
@@ -76,7 +79,7 @@ def _extract_items_and_next(data: Any) -> tuple[list[dict[str, Any]], str | None
return [], None
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type((ZendutyRetryableError, requests.ReadTimeout, requests.ConnectionError)),
stop=stop_after_attempt(5),
wait=wait_exponential_jitter(initial=1, max=30),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/zep/zep.py b/products/warehouse_sources/backend/temporal/data_imports/sources/zep/zep.py
index 1b9781030489..d8a9df58a499 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/zep/zep.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/zep/zep.py
@@ -6,9 +6,12 @@ from urllib.parse import quote, urlencode
import requests
from structlog.types import FilteringBoundLogger
-from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
+from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ make_tracked_session,
+ retry_with_deadline,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
from products.warehouse_sources.backend.temporal.data_imports.sources.zep.settings import (
@@ -52,7 +55,7 @@ def _build_url(path: str, params: dict[str, Any]) -> str:
return f"{url}?{urlencode(params)}"
-@retry(
+@retry_with_deadline(
retry=retry_if_exception_type(
(
ZepRetryableError,