:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(warehouse-sources): keep syncs enabled on transient errors (#124319)
Tom Owers77b1c2d02521
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/settings.pyindex 491545300b15..96dbcc34be67 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/settings.py@@ -299,10 +299,11 @@ INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {AUTH_ERROR = "Ahrefs authentication failed. Check your API key in Account settings > API keys."ACCESS_ERROR = "Ahrefs denied access. Check your plan, project access, and verified ownership for page data."QUOTA_ERROR = "Ahrefs API units are exhausted. Increase your API key or workspace limit, or wait for renewal."-# A matched sync error turns the table's sync off, so renewed units alone don't start it again.-QUOTA_SYNC_ERROR = (- "Your Ahrefs API units ran out, so this sync is off. Raise your API key or workspace unit limit, "- "or wait for the units to renew, then re-enable the sync."+# The units renew each period, so the schema stays enabled and keeps retrying. Disabling it would+# leave the sync off after the units come back.+QUOTA_RETRY_ERROR = (+ "Your Ahrefs API units ran out, so this sync can't finish. It tries again on its own and works again "+ "once you raise your API key or workspace unit limit, or the units renew.")PROJECT_NOT_FOUND_ERROR = ("Ahrefs returned no target for the Site Audit project. Check the project ID and that the project has a target URL."@@ -311,11 +312,11 @@ PROJECT_NOT_FOUND_ERROR = (NON_RETRYABLE_ERRORS: dict[str, str | None] = {"ahrefs_invalid_api_key": AUTH_ERROR,"ahrefs_access_denied": ACCESS_ERROR,- "ahrefs_quota_exceeded": QUOTA_SYNC_ERROR,"ahrefs_project_not_found": PROJECT_NOT_FOUND_ERROR,"401 Client Error": AUTH_ERROR,"403 Client Error": ACCESS_ERROR,- "402 Client Error": QUOTA_SYNC_ERROR,}+RETRYABLE_QUOTA_ERRORS: frozenset[str] = frozenset({"ahrefs_quota_exceeded", "402 Client Error"})+CREDENTIAL_ERRORS: dict[str, str | None] = {**NON_RETRYABLE_ERRORS, "ahrefs_quota_exceeded": QUOTA_ERROR}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/source.pyindex 6fbe3ec97e84..410fec26b696 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/source.py@@ -24,6 +24,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.ahrefs.setMAX_PAGE_ROWS,MAX_REFDOMAIN_ROWS,NON_RETRYABLE_ERRORS,+ QUOTA_RETRY_ERROR,+ RETRYABLE_QUOTA_ERRORS,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, SimpleSourcefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (@@ -55,6 +57,12 @@ class AhrefsSource(SimpleSource[AhrefsSourceConfig]):def get_non_retryable_errors(self) -> dict[str, str | None]:return NON_RETRYABLE_ERRORS+ def get_retryable_errors(self) -> set[str]:+ return set(RETRYABLE_QUOTA_ERRORS)++ def get_retry_exhausted_errors(self) -> dict[str, str]:+ return dict.fromkeys(RETRYABLE_QUOTA_ERRORS, QUOTA_RETRY_ERROR)+def get_canonical_descriptions(self) -> CanonicalDescriptions:return CANONICAL_DESCRIPTIONSdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/tests/test_ahrefs.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/tests/test_ahrefs.pyindex 8a54d5837bdd..10cdd34bd706 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/tests/test_ahrefs.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ahrefs/tests/test_ahrefs.py@@ -19,6 +19,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.ahrefs.setREFDOMAIN_FIELDS,)from products.warehouse_sources.backend.temporal.data_imports.sources.ahrefs.source import AhrefsSource+from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import error_message_matchesfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import (RESTClientRetryableError,)@@ -298,18 +299,18 @@ def test_page_sample_cap(count: int) -> None:@pytest.mark.parametrize(- ("status", "body", "message", "sync_message"),+ ("status", "body"),[- (400, {"error": "API units limit exceeded"}, "API units are exhausted", "then re-enable the sync"),- (402, {"error": "Payment required"}, "API units are exhausted", "then re-enable the sync"),+ (400, {"error": "API units limit exceeded"}),+ (402, {"error": "Payment required"}),],)-def test_credential_and_sync_error_mapping(status: int, body: object, message: str, sync_message: str) -> None:+def test_exhausted_units_keep_the_schema_enabled(status: int, body: object) -> None:answer = ScriptedResponse(status=status, json=body)with scripted_network([answer]) as network:valid, error = validate_credentials("test-key")assert valid is False- assert error is not None and message in error and sync_message not in error+ assert error is not None and "API units are exhausted" in errorassert len(network.requests_log) == 1source = AhrefsSource()@@ -317,8 +318,11 @@ def test_credential_and_sync_error_mapping(status: int, body: object, message: s"site_audit_issues", [answer])assert isinstance(result.raised, ValueError)- mapped = source.get_non_retryable_errors()[str(result.raised)]- assert mapped is not None and sync_message in mapped+ raised = str(result.raised)+ assert error_message_matches(raised, source.get_retryable_errors())+ assert not error_message_matches(raised, source.get_non_retryable_errors())+ exhausted = [m for key, m in source.get_retry_exhausted_errors().items() if error_message_matches(raised, [key])]+ assert exhausted and "tries again on its own" in exhausted[0]assert len(result.requests) == 1diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/clerk.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/clerk.pyindex 4d1df1761e23..853a65adf302 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/clerk.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/clerk.py@@ -310,7 +310,15 @@ def validate_credentials(secret_key: str) -> tuple[bool, str | None]:# Clerk error codes meaning "this account has not switched the feature on". Paired with the# statuses Clerk returns them under: 402 carries no body code, so the status alone is the signal.-_FEATURE_DISABLED_CODES = frozenset({"billing_not_enabled", "feature_not_enabled"})+_FEATURE_DISABLED_CODES = frozenset(+ {+ "billing_not_enabled",+ "feature_not_enabled",+ "organization_not_enabled_in_instance",+ # Enterprise SSO connections need email address sign-in on the instance.+ "feature_requires_email_address_enabled",+ }+)# A gated list endpoint (the Restrictions allow-list and block-list) answers 404 `resource_not_found`# instead of a 4xx feature code when its feature is off. This check only runs for endpoints that carry@@ -323,7 +331,7 @@ def _is_feature_disabled(response: Optional[Response]) -> bool:return Falseif response.status_code == 402:return True- if response.status_code not in (400, 403, 404):+ if response.status_code not in (400, 403, 404, 422):return Falsetry:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/settings.pyindex 9a27309829cc..7a2c0558903b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/settings.py@@ -115,11 +115,20 @@ CLERK_ENDPOINTS: dict[str, ClerkEndpointConfig] = {# endpoints give. Skip zero rows instead of failing the schema every run.gated_feature="Satellite domains",),+ # Clerk answers 422 feature_requires_email_address_enabled for the SSO connection lists on+ # instances without email address sign-in. Every table starts enabled, so most accounts never+ # chose these tables. Skip zero rows instead of disabling the schema."saml_connections": ClerkEndpointConfig(- name="saml_connections", path="/saml_connections", is_wrapped_response=True+ name="saml_connections",+ path="/saml_connections",+ is_wrapped_response=True,+ gated_feature="Enterprise SSO",),"enterprise_connections": ClerkEndpointConfig(- name="enterprise_connections", path="/enterprise_connections", is_wrapped_response=True+ name="enterprise_connections",+ path="/enterprise_connections",+ is_wrapped_response=True,+ gated_feature="Enterprise SSO",),"oauth_applications": ClerkEndpointConfig(name="oauth_applications",@@ -140,7 +149,10 @@ CLERK_ENDPOINTS: dict[str, ClerkEndpointConfig] = {gated_feature="Machine (M2M) authentication",),# /api_keys and /m2m_tokens cap `limit` at 100 rather than 500.- "api_keys": ClerkEndpointConfig(name="api_keys", path="/api_keys", is_wrapped_response=True),+ # Clerk answers 403 feature_not_enabled on instances without API keys. Skip zero rows there.+ "api_keys": ClerkEndpointConfig(+ name="api_keys", path="/api_keys", is_wrapped_response=True, gated_feature="API keys"+ ),"m2m_tokens": ClerkEndpointConfig(name="m2m_tokens",path="/m2m_tokens",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/tests/test_clerk.py b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/tests/test_clerk.pyindex e40f447b84c9..089ddf336ee4 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/tests/test_clerk.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/tests/test_clerk.py@@ -313,6 +313,15 @@ class TestClerkFeatureGatedEndpoints:("machines", 404, {"errors": [{"code": "resource_not_found"}]}),# m2m_tokens fans out over /machines, so it meets the same 404 on its parent fetch.("m2m_tokens", 404, {"errors": [{"code": "resource_not_found"}]}),+ # Organizations off: the lists answer 403 with a feature code instead of a 404.+ ("organizations", 403, {"errors": [{"code": "organization_not_enabled_in_instance"}]}),+ ("organization_domains", 403, {"errors": [{"code": "organization_not_enabled_in_instance"}]}),+ ("organization_roles", 403, {"errors": [{"code": "organization_not_enabled_in_instance"}]}),+ ("organization_permissions", 403, {"errors": [{"code": "organization_not_enabled_in_instance"}]}),+ ("api_keys", 403, {"errors": [{"code": "feature_not_enabled"}]}),+ # Instances without email address sign-in answer 422 for the SSO connection lists.+ ("saml_connections", 422, {"errors": [{"code": "feature_requires_email_address_enabled"}]}),+ ("enterprise_connections", 422, {"errors": [{"code": "feature_requires_email_address_enabled"}]}),],)def test_feature_not_enabled_syncs_no_rows_instead_of_failing(@@ -334,6 +343,13 @@ class TestClerkFeatureGatedEndpoints:("allowlist_identifiers", 404, {"errors": [{"code": "invalid_request"}]}),# A table with no feature gate never swallows an error.("users", 402, {"errors": [{"code": "payment_required"}]}),+ ("users", 403, {"errors": [{"code": "organization_not_enabled_in_instance"}]}),+ # A 422 without the feature code is a bad request, not a feature-off signal.+ ("saml_connections", 422, {"errors": [{"code": "form_param_invalid"}]}),+ # API keys with the feature on still reject an unfiltered list, which the curated copy covers.+ ("api_keys", 400, {"errors": [{"code": "bad_request"}]}),+ # A permission denial on a gated table stays a failure.+ ("organizations", 403, {"errors": [{"code": "authorization_invalid"}]}),],)def test_other_errors_still_fail_the_sync(self, endpoint: str, status_code: int, body: dict[str, Any]) -> None:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/cloudflare.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/cloudflare.pyindex 31d00bdaf893..bc1f55ad6fea 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/cloudflare.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/cloudflare.py@@ -116,6 +116,9 @@ query DnsAnalyticsReport($zoneTag: string, $since: Time, $until: Time) {"""DNS_ANALYTICS_ALL_ZONES_FAILED = "Cloudflare refused the DNS analytics query for every zone"+# Cloudflare's text for a token without the Zone Analytics read permission. The REST report this+# replaced needed the same permission and skipped such zones, so many tokens never had it.+DNS_ANALYTICS_PERMISSION_DENIED = "does not have permission"def _dns_analytics_rows(zone_id: str, groups: list[dict[str, Any]]) -> list[dict[str, Any]]:@@ -154,7 +157,9 @@ def _dns_analytics_report(api_token: str) -> Iterator[list[dict[str, Any]]]:window = {"since": _to_rfc3339(until - DNS_ANALYTICS_WINDOW), "until": _to_rfc3339(until)}zones_read = 0- first_error: Optional[str] = None+ # A permission denial only means the token can't read this table, like a 403 on the REST+ # fan-out. Any other error on every zone means the query itself is broken.+ first_failure: Optional[str] = Nonefor zones in client.paginate(_PARENT_PATHS[ZONES_PARENT],params={"per_page": PAGE_SIZE},@@ -178,7 +183,8 @@ def _dns_analytics_report(api_token: str) -> Iterator[list[dict[str, Any]]]:error = _graphql_error(body)if error is not None:logger.info("cloudflare.dns_analytics_zone_skipped", zone_id=zone_id, error=error)- first_error = first_error or error+ if first_failure is None and DNS_ANALYTICS_PERMISSION_DENIED not in error:+ first_failure = errorcontinuezones_read += 1zone_results = ((body.get("data") or {}).get("viewer") or {}).get("zones") or []@@ -187,8 +193,8 @@ def _dns_analytics_report(api_token: str) -> Iterator[list[dict[str, Any]]]:if rows:yield rows- if first_error is not None and zones_read == 0:- raise ValueError(f"{DNS_ANALYTICS_ALL_ZONES_FAILED}: {first_error}")+ if first_failure is not None and zones_read == 0:+ raise ValueError(f"{DNS_ANALYTICS_ALL_ZONES_FAILED}: {first_failure}")# A logpush job's `destination_conf` is a URI whose query string can carry thediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/settings.pyindex 4a420df79ce1..37bc4493090f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/settings.py@@ -295,11 +295,14 @@ CLOUDFLARE_ENDPOINTS: dict[str, CloudflareEndpointConfig] = {cursor_path="result_info.cursor",data_selector="result.buckets",),+ # Workers KV and D1 answer 401 (code 10000), not 403, to a token without their read+ # permission. The account list already succeeded with the same token, so skip that account."kv_namespaces": CloudflareEndpointConfig(name="kv_namespaces",path="/accounts/{account_id}/storage/kv/namespaces",parent=ACCOUNTS_PARENT,parent_key="_account_id",+ extra_skip_status_codes=(401,),),"d1_databases": CloudflareEndpointConfig(name="d1_databases",@@ -307,6 +310,7 @@ CLOUDFLARE_ENDPOINTS: dict[str, CloudflareEndpointConfig] = {primary_keys=("uuid",),parent=ACCOUNTS_PARENT,parent_key="_account_id",+ extra_skip_status_codes=(401,),),"stream_usage": CloudflareEndpointConfig(name="stream_usage",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/source.pyindex e8b8cd132936..a5f3bb41f9d1 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/source.py@@ -10,7 +10,6 @@ from products.warehouse_sources.backend.facade.source_config import ()from products.warehouse_sources.backend.temporal.data_imports.sources.cloudflare.cloudflare import (CLOUDFLARE_BASE_URL,- DNS_ANALYTICS_ALL_ZONES_FAILED,cloudflare_source,validate_credentials as validate_cloudflare_credentials,)@@ -199,7 +198,6 @@ class CloudflareSource(SimpleSource[CloudflareSourceConfig]):f"403 Client Error: Forbidden for url: {CLOUDFLARE_BASE_URL}/zones?": "Your Cloudflare API token can't list your zones. Check that the token is still active and has the Zone Read permission, then re-enable the sync.",f"403 Client Error: Forbidden for url: {CLOUDFLARE_BASE_URL}/accounts?": "Your Cloudflare API token can't list your accounts. Check that the token is still active and has the Account Settings Read permission, then re-enable the sync.","403 Client Error: Forbidden for url: https://api.cloudflare.com": "Cloudflare denied access. Please check that your API token has read permissions for this resource.",- DNS_ANALYTICS_ALL_ZONES_FAILED: "Your Cloudflare API token can't read analytics for any of your zones. Add the Zone Analytics read permission to the token, then re-enable the sync.",}@propertydiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/tests/test_cloudflare_client.py b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/tests/test_cloudflare_client.pyindex bba6faa9b4e7..1d47b8e40db1 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/tests/test_cloudflare_client.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/cloudflare/tests/test_cloudflare_client.py@@ -250,10 +250,18 @@ class TestEndpointConfigConsistency:class TestAccountFanout:- @pytest.mark.parametrize("status_code", [403, 404])- def test_skips_account_the_token_cannot_read(self, status_code: int) -> None:+ @pytest.mark.parametrize(+ "endpoint, status_code",+ [+ ("kv_namespaces", 403),+ ("kv_namespaces", 404),+ ("kv_namespaces", 401),+ ("d1_databases", 401),+ ],+ )+ def test_skips_account_the_token_cannot_read(self, endpoint: str, status_code: int) -> None:result = _driver().run(- "kv_namespaces",+ endpoint,[_response([{"id": "a1"}, {"id": "a2"}], total_pages=1),_error_response(status_code),@@ -333,19 +341,41 @@ class TestDnsAnalyticsReport:assert (graphql_request.method, graphql_request.path) == ("POST", "/client/v4/graphql")assert graphql_request.json()["variables"]["zoneTag"] == "z2"- def test_fails_when_every_zone_is_refused(self) -> None:+ def test_token_without_analytics_permission_syncs_no_rows(self) -> None:+ denied = "Actor 'com.cloudflare.api.token.abc' does not have permission 'com.cloudflare.api.account.zone.analytics.read' for zone z1"+ result = _driver().run(+ "dns_analytics_report",+ [+ _response([{"id": "z1"}, {"id": "z2"}], total_pages=1),+ _graphql_error(denied),+ _graphql_error(denied),+ ],+ )++ assert result.raised is None+ assert result.rows == []++ @pytest.mark.parametrize(+ "second_zone_error",+ [+ "unknown field dnsAnalyticsAdaptiveGroups",+ "Actor 'com.cloudflare.api.token.abc' does not have permission 'com.cloudflare.api.account.zone.analytics.read' for zone z2",+ ],+ )+ def test_broken_query_on_every_zone_fails_without_disabling(self, second_zone_error: str) -> None:result = _driver().run("dns_analytics_report",[_response([{"id": "z1"}, {"id": "z2"}], total_pages=1),_graphql_error("unknown field dnsAnalyticsAdaptiveGroups"),- _graphql_error("unknown field dnsAnalyticsAdaptiveGroups"),+ _graphql_error(second_zone_error),],)assert isinstance(result.raised, ValueError)assert DNS_ANALYTICS_ALL_ZONES_FAILED in str(result.raised)assert "unknown field dnsAnalyticsAdaptiveGroups" in str(result.raised)+ assert not error_message_matches(str(result.raised), CloudflareSource().get_non_retryable_errors().keys())class TestLogpushDestinationRedaction:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/linear/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/linear/source.pyindex e0a95a9fdc69..c1fbc4276ced 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/linear/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/linear/source.py@@ -94,9 +94,9 @@ class LinearSource(ResumableSource[LinearSourceConfig, LinearResumeConfig], OAut# Retrying can never resolve this — the integration won't reappear — so stop and# ask the user to reconnect. Matched as a substring; the trailing integration id varies."Integration not found": "The linked Linear integration no longer exists. Please reconnect your Linear account.",- # The Linear OAuth app isn't configured on this PostHog instance (missing client id/secret),- # so the source can't refresh its access token. Deterministic — retrying never resolves it.- "Linear app not configured": "The Linear app is not configured on this PostHog instance. Please contact support.",+ # "Linear app not configured" is not here on purpose. It means the worker that ran the job+ # has no Linear client credentials, which is a PostHog fault that the customer can't fix,+ # so it must not disable the schema.}def _get_access_token(self, config: LinearSourceConfig, team_id: int) -> str:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/linear/test_linear.py b/products/warehouse_sources/backend/temporal/data_imports/sources/linear/test_linear.pyindex f27c3d6f2215..27d4c0511f06 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/linear/test_linear.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/linear/test_linear.py@@ -327,8 +327,6 @@ class TestLinearSourceNonRetryableErrors:("deleted_integration", "Integration not found: 165665"),("auth_401", "401 Client Error: Unauthorized for url: https://api.linear.app/graphql"),("forbidden_403", "403 Client Error: Forbidden for url: https://api.linear.app/graphql"),- # NotImplementedError raised by OauthIntegration when the instance lacks Linear client id/secret.- ("app_not_configured", "Linear app not configured"),])def test_non_retryable_errors_match(self, _name: str, observed_error: str) -> None:@@ -344,6 +342,8 @@ class TestLinearSourceNonRetryableErrors:"read_timeout","Linear: transient network error - HTTPSConnectionPool(host='api.linear.app', port=443): Read timed out. (read timeout=60)",),+ # Raised when the worker has no Linear client credentials: a PostHog fault, not the customer's.+ ("app_not_configured", "NotImplementedError: Linear app not configured"),])def test_non_retryable_errors_does_not_match_transient(self, _name: str, observed_error: str) -> None:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/meta_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/meta_ads.pyindex 1a2c16aed44d..c8c9060f08d2 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/meta_ads.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/meta_ads.py@@ -1,3 +1,4 @@+import reimport jsonimport timeimport typing@@ -1279,6 +1280,67 @@ def _make_paginated_api_request(yield from _iter_time_range_pagination(url, params, time_range, resume_config, resumable_source_manager)+# The values Meta's Insights API accepts for `action_attribution_windows`, as its code 100 error+# lists them. The setting is free text, so a value outside this set would fail every stats sync.+META_ATTRIBUTION_WINDOWS = frozenset(+ {+ "1d_view",+ "7d_view",+ "28d_view",+ "1d_click",+ "7d_click",+ "28d_click",+ "1d_ev",+ "dda",+ "default",+ "7d_view_first_conversion",+ "28d_view_first_conversion",+ "7d_view_all_conversions",+ "28d_view_all_conversions",+ "skan_view",+ "skan_click",+ "skan_click_second_postback",+ "skan_view_second_postback",+ "skan_click_third_postback",+ "skan_view_third_postback",+ "incrementality",+ "incrementality_first_conversion",+ "incrementality_all_conversions",+ "custom",+ "1d_sequenced",+ "7d_sequenced",+ "28d_sequenced",+ }+)++# Spelled-out day windows such as "7 day click" or "28-days view".+_SPELLED_DAY_WINDOW = re.compile(r"\b(\d+)[\s_-]*(?:days?|d)[\s_-]*(click|view)s?\b")++# Every accepted window is made of these characters, so any other character separates two values.+_ATTRIBUTION_WINDOW_SEPARATOR = re.compile(r"[^a-z0-9_]+")+++def _normalize_attribution_windows(raw: str) -> list[str]:+ """Parse the free-text windows setting into the values Meta accepts, in order and without repeats.++ Values Meta does not accept are dropped, because Meta rejects the whole Insights request for+ one of them, and that would stop every stats sync on the source.+ """+ text = _SPELLED_DAY_WINDOW.sub(r"\1d_\2", raw.lower())+ windows: list[str] = []+ dropped: list[str] = []+ for token in _ATTRIBUTION_WINDOW_SEPARATOR.split(text):+ if not token:+ continue+ if token not in META_ATTRIBUTION_WINDOWS:+ dropped.append(token)+ elif token not in windows:+ windows.append(token)+ if dropped:+ logger.warning("meta_ads_dropped_invalid_attribution_windows", dropped=dropped)+ return windows++def _attribution_params(config: MetaAdsSourceConfig) -> dict[str, str]:"""Attribution settings to add to an Insights request, only when the source configures them.@@ -1287,7 +1349,8 @@ def _attribution_params(config: MetaAdsSourceConfig) -> dict[str, str]:PostHog's spend and conversion numbers with what Ads Manager shows.- ``action_attribution_windows``: a comma-separated list in config (e.g. ``7d_click,1d_view``),- sent JSON-encoded the way Meta's Graph API accepts a list parameter.+ normalized by ``_normalize_attribution_windows`` and sent JSON-encoded the way Meta's Graph+ API accepts a list parameter. Omitted when no configured value is one Meta accepts.- ``use_unified_attribution_setting``: a string flag (``"true"``/``"false"``) sent verbatim;any other value (including the "use Meta's default" empty option) leaves it unset."""@@ -1295,7 +1358,7 @@ def _attribution_params(config: MetaAdsSourceConfig) -> dict[str, str]:windows_raw = getattr(config, "action_attribution_windows", None)if windows_raw:- windows = [window.strip() for window in windows_raw.split(",") if window.strip()]+ windows = _normalize_attribution_windows(windows_raw)if windows:params["action_attribution_windows"] = json.dumps(windows)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/test_meta_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/test_meta_ads.pyindex 80bb31066561..5303ff50cb99 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/test_meta_ads.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/test_meta_ads.py@@ -2414,25 +2414,45 @@ class TestBreakdownStatsRequests:assert result.requests[0].param("time_range") is Noneassert result.requests[0].path.endswith("/act_123/adcreatives")- def test_attribution_settings_omitted_when_unset(self, monkeypatch) -> None:+ @pytest.mark.parametrize("action_attribution_windows", [None, "", "first_touch, 30d_click"])+ def test_attribution_settings_omitted_when_unset(self, monkeypatch, action_attribution_windows: str | None) -> None:# Backward compatibility: an existing connection that never set attribution config must# produce the same Insights request as before — Meta then applies its own default window.- result = self._capture_request(monkeypatch, MetaAdsResource.CampaignStats)+ # A setting with no value Meta accepts falls back the same way, because Meta rejects the+ # whole request for one unknown window.+ result = self._capture_request(+ monkeypatch, MetaAdsResource.CampaignStats, action_attribution_windows=action_attribution_windows+ )assert result.requests[0].param("action_attribution_windows") is Noneassert result.requests[0].param("use_unified_attribution_setting") is None- def test_attribution_settings_threaded_into_stats_request(self, monkeypatch) -> None:+ @pytest.mark.parametrize(+ "action_attribution_windows, expected",+ [+ ("7d_click, 1d_view", ["7d_click", "1d_view"]),+ ("7D_Click;1d_VIEW", ["7d_click", "1d_view"]),+ ("7d_click 1d_view", ["7d_click", "1d_view"]),+ ('["7d_click", "28d_view"]', ["7d_click", "28d_view"]),+ ("7 day click, 1-day view, 28 days clicks", ["7d_click", "1d_view", "28d_click"]),+ ("7d_click, 7d_click", ["7d_click"]),+ ("dda, 7d_view_first_conversion, skan_click", ["dda", "7d_view_first_conversion", "skan_click"]),+ ("7d_click, last_touch, 2d_click", ["7d_click"]),+ ],+ )+ def test_attribution_settings_threaded_into_stats_request(+ self, monkeypatch, action_attribution_windows: str, expected: list[str]+ ) -> None:result = self._capture_request(monkeypatch,MetaAdsResource.CampaignStats,- action_attribution_windows="7d_click, 1d_view",+ action_attribution_windows=action_attribution_windows,use_unified_attribution_setting="true",)- # Windows are sent JSON-encoded (Meta's list-parameter shape) with surrounding whitespace- # trimmed; the unified-setting flag is passed through verbatim.- assert json.loads(result.requests[0].param("action_attribution_windows") or "null") == ["7d_click", "1d_view"]+ # Windows are sent JSON-encoded (Meta's list-parameter shape), normalized to the values Meta+ # accepts, with unknown ones dropped; the unified-setting flag is passed through verbatim.+ assert json.loads(result.requests[0].param("action_attribution_windows") or "null") == expectedassert result.requests[0].param("use_unified_attribution_setting") == "true"def test_attribution_default_sentinel_leaves_unified_setting_unset(self, monkeypatch) -> None:diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/source.pyindex ffa7f524a674..35c9368c3e21 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/source.py@@ -89,6 +89,32 @@ _VALIDATE_CONNECTION_HINTS: list[tuple[str, str]] = [("Unknown database", "Database does not exist. Check the database name is correct."),]+_CANT_CONNECT_ERROR = "Can't connect to MySQL server on"++# The OS details of a 2003 error that do not change between attempts: the server refused the port,+# no route reaches it, or its name does not exist.+_UNREACHABLE_HOST_CONNECT_DETAILS = (+ "Connection refused",+ "No route to host",+ "Network is unreachable",+ "Name or service not known",+ "nodename nor servname provided",+)++_UNREACHABLE_HOST_ERROR = (+ "PostHog couldn't connect to your MySQL server. Check that the host and port are "+ "correct and that the database is reachable from the public internet, with PostHog's "+ "egress IP addresses allowed through your firewall. For a database that can't be "+ "exposed publicly, use the SSH tunnel option, then re-enable the sync."+)++_CANT_CONNECT_EXHAUSTED_MESSAGE = (+ "PostHog couldn't connect to your MySQL server, and it kept failing on every retry. Check that "+ "the server is running and reachable from the public internet, and that PostHog's egress IP "+ "addresses are allowed through your firewall. For a database that can't be exposed publicly, "+ "use the SSH tunnel option. This sync is still enabled and will run again on its next schedule."+)+# Error 1045 is the same failure the Postgres, Supabase, and Neon sources already word this# way. Keeping one wording means a wrong password reads the same whichever database it is._INVALID_CREDENTIALS_ERROR = (@@ -305,19 +331,13 @@ class MySQLSource(# pymysql wraps a TLS failure in the generic 2003 error below, so these must come first.**CERTIFICATE_ERRORS,**TLS_CONFIGURATION_ERRORS,- # pymysql collapses every connect-level failure (wrong host/port, closed firewall,- # unreachable host, a connect timeout that outlasts the in-process retry budget) into- # error 2003, "Can't connect to MySQL server on '<host>' (<os detail>)". Non-retryable- # for the same reason as the Postgres source's connect-timeout entry: a persistently- # unreachable host won't recover on retry. Give the actionable reachability guidance the- # raw driver tuple lacks; the volatile host/os-detail are excluded from the match. The- # create-time `_VALIDATE_CONNECTION_HINTS` above stay more granular.- "Can't connect to MySQL server on": (- "PostHog couldn't connect to your MySQL server. Check that the host and port are "- "correct and that the database is reachable from the public internet, with PostHog's "- "egress IP addresses allowed through your firewall. For a database that can't be "- "exposed publicly, use the SSH tunnel option, then re-enable the sync."- ),+ # pymysql collapses every connect-level failure into error 2003, "Can't connect to MySQL+ # server on '<host>' (<os detail>)". Only the OS details that name a wrong host, port or+ # route stop the sync. A timeout, reset or temporary DNS failure is in the same error+ # code, but the next run often succeeds with no change, so those stay retryable through+ # `_CANT_CONNECT_ERROR` in `get_retryable_errors`. The create-time+ # `_VALIDATE_CONNECTION_HINTS` above stay more granular.+ **dict.fromkeys(_UNREACHABLE_HOST_CONNECT_DETAILS, _UNREACHABLE_HOST_ERROR),"No primary key defined for table": ("This table needs a primary key to sync incrementally, but none is set. Choose a primary ""key for the table in its sync settings, or switch it to full table replication, then "@@ -582,6 +602,7 @@ class MySQLSource(# stable phrase here as a backstop — Temporal's own activity retry lands on the newly promoted# primary once the reparent completes.return {+ _CANT_CONNECT_ERROR,"Lost connection to MySQL server during query","Too many connections","Can't create a new thread",@@ -616,6 +637,10 @@ class MySQLSource("grpc: the client connection is closing",}+ def get_retry_exhausted_errors(self) -> dict[str, str]:+ # Without this, the job stores pymysql's raw tuple, which repeats the customer's host.+ return {_CANT_CONNECT_ERROR: _CANT_CONNECT_EXHAUSTED_MESSAGE}+def reconcile_schema_metadata(self,source: "ExternalDataSource",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/tests/test_mysql.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/tests/test_mysql.pyindex fa7b724a92fa..30a6447b5519 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/tests/test_mysql.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/tests/test_mysql.py@@ -2448,23 +2448,40 @@ class TestMySQLSourceNonRetryableErrors:return MySQLSource()@pytest.mark.parametrize(- "error_msg",+ "error_msg, stops_the_sync",[- "(2003, \"Can't connect to MySQL server on 'db.example.com' (timed out)\")",- "(2003, \"Can't connect to MySQL server on 'db.example.com' ([Errno 111] Connection refused)\")",+ ("(2003, \"Can't connect to MySQL server on 'db.example.com' (timed out)\")", False),+ ("(2003, \"Can't connect to MySQL server on 'db.example.com' ([Errno 110] Connection timed out)\")", False),+ (+ "(2003, \"Can't connect to MySQL server on 'db.example.com' ([Errno 104] Connection reset by peer)\")",+ False,+ ),+ ("(2003, \"Can't connect to MySQL server on 'db.example.com' ([Errno 111] Connection refused)\")", True),+ ("(2003, \"Can't connect to MySQL server on 'db.example.com' ([Errno 113] No route to host)\")", True),+ (+ "(2003, \"Can't connect to MySQL server on 'nope.example.com' ([Errno -2] Name or service not known)\")",+ True,+ ),],)- def test_cannot_connect_surfaces_actionable_message(self, source, error_msg):- # A persistent connect failure is non-retryable, but the customer must get reachability- # guidance rather than the raw pymysql (2003, ...) tuple. Guards against reverting the- # umbrella message back to None.- non_retryable = source.get_non_retryable_errors()+ def test_cannot_connect_surfaces_actionable_message(self, source, error_msg, stops_the_sync):+ # A refused port or an unknown host stays wrong on every attempt, so it stops the sync. A+ # timeout or reset often clears by the next run, so it stays retryable. Both must replace+ # pymysql's raw (2003, ...) tuple, which repeats the customer's host.+ non_retryable = {**Any_Source_Errors, **source.get_non_retryable_errors()}+ assert error_message_matches(error_msg, non_retryable) is stops_the_sync+ if stops_the_sync:+ messages = source.get_non_retryable_errors()+ else:+ assert error_message_matches(error_msg, source.get_retryable_errors())+ messages = source.get_retry_exhausted_errors()friendly = next(- (message for pattern, message in non_retryable.items() if pattern in error_msg),+ (message for pattern, message in messages.items() if error_message_matches(error_msg, [pattern])),None,)assert friendly is not None, f"Connect failure should surface a friendly message: {error_msg}"assert "SSH tunnel" in friendly+ assert "example.com" not in friendlydef test_tidb_cloud_access_denied_surfaces_actionable_message(self, source):# TiDB Cloud's ER_ACCESS_DENIED_ERROR wording doesn't contain the standard MySQLdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/openai_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/openai_ads.pyindex 9869aac2370e..86faf97ce673 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/openai_ads.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/openai_ads.py@@ -1,12 +1,12 @@import reimport jsonimport dataclasses-from collections.abc import Callable, Iterator+from collections.abc import Callable, Iterable, Iteratorfrom datetime import UTC, date, datetime, timedelta-from typing import Any, Optional+from typing import Any, Optional, TypeVarfrom dateutil import parser-from requests import Request, Response+from requests import HTTPError, Request, Responsefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_sessionfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (@@ -44,6 +44,15 @@ NO_ACCESS_ERROR = (UNREACHABLE_ERROR = ("Couldn't reach OpenAI Ads to validate your API key. This is usually temporary, so try again in a moment.")+SERVER_ERROR_MESSAGE = (+ "OpenAI Ads had a temporary server error. This can also happen for a few minutes after you create an "+ "API key. The next sync will try again automatically."+)++# `requests` writes this prefix for every 403. The source's non-retryable 403 entry matches it.+_FORBIDDEN_PREFIX = "403 Client Error: Forbidden for url:"++_T = TypeVar("_T")# OpenAI Ads keeps daily delivery reporting for the most recent 365 days and rejects an earlier# `since` with a 400 (https://developers.openai.com/ads/reporting). The window is one day shorter@@ -116,6 +125,28 @@ class _ListPaginator(BasePaginator):return "_ListPaginator()"+class OpenAIAdsServerError(Exception):+ """A 403 whose error body has the code `server_error`."""+++def _retry_forbidden_server_errors(pages: Iterable[_T]) -> Iterator[_T]:+ """Re-raise a 403 with the body code `server_error` as a retryable error.++ OpenAI Ads sends this 403 for a server fault, and for a new API key for some minutes after+ it is created. The same key then works. The error text must not keep the `requests` 403+ prefix, because the source's non-retryable 403 entry matches it and disables the schema.+ """+ try:+ yield from pages+ except HTTPError as error:+ message = str(error)+ status = error.response.status_code if error.response is not None else None+ if status != 403 or "code=server_error" not in message:+ raise+ # Built from the message that is already redacted, not from the response URL.+ raise OpenAIAdsServerError(message.replace(_FORBIDDEN_PREFIX, "Server error (HTTP 403) for", 1)) from error++def _headers() -> dict[str, str]:# Auth (Bearer) is supplied via the framework auth config so its value is redacted from logs# and raised errors; only the non-secret accept header is set here.@@ -202,8 +233,8 @@ def _make_client(api_key: str) -> RESTClient:def _account_currency(api_key: str) -> str:- for page in _make_client(api_key).paginate(- path="/v1/ad_account", paginator=SinglePagePaginator(), data_selector="$"+ for page in _retry_forbidden_server_errors(+ _make_client(api_key).paginate(path="/v1/ad_account", paginator=SinglePagePaginator(), data_selector="$")):for account in page:currency = account.get("currency_code")@@ -279,7 +310,7 @@ def openai_ads_source(# Multi-level fan-out with query-param parents: hand-rolled over the tracked, retrying# RESTClient. Not resumable — a retry re-walks the hierarchy and full refresh replaces.client = _make_client(api_key)- return _source_response(config, lambda: _fan_out_rows(client, endpoint))+ return _source_response(config, lambda: _retry_forbidden_server_errors(_fan_out_rows(client, endpoint)))resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None@@ -357,4 +388,6 @@ def openai_ads_source(initial_paginator_state=initial_paginator_state,)- return _source_response(config, lambda: resource, column_hints=resource.column_hints)+ return _source_response(+ config, lambda: _retry_forbidden_server_errors(resource), column_hints=resource.column_hints+ )diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/source.pyindex a53633b3d8e2..ebc86e016e8c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/source.py@@ -22,6 +22,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_from products.warehouse_sources.backend.temporal.data_imports.sources.openai_ads.openai_ads import (INVALID_CREDENTIALS_ERROR,NO_ACCESS_ERROR,+ SERVER_ERROR_MESSAGE,OpenAIAdsResumeConfig,openai_ads_source,validate_credentials as validate_openai_ads_credentials,@@ -106,6 +107,9 @@ Create an API key in the Settings tab of [OpenAI Ads Manager](https://ads.openai"code=server_error",}+ def get_retry_exhausted_errors(self) -> dict[str, str]:+ return {"code=server_error": SERVER_ERROR_MESSAGE}+def get_schemas(self,config: OpenAIAdsSourceConfig,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/tests/test_openai_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/tests/test_openai_ads.pyindex a1ea3d9b0db8..4fa195ff4c38 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/tests/test_openai_ads.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/openai_ads/tests/test_openai_ads.py@@ -6,6 +6,7 @@ import time_machinefrom parameterized import parameterized+from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import error_message_matchesfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (ScriptedResponse,SourceDriver,@@ -151,6 +152,29 @@ class TestInsights:assert result.paths == ["/v1/ad_account"]+class TestForbiddenResponses:+ @parameterized.expand(+ [+ ("entity_list", "campaigns", "server_error", False),+ ("fan_out", "ads", "server_error", False),+ ("insights_currency_lookup", "campaign_insights", "server_error", False),+ ("permission_code", "campaigns", "permission_denied", True),+ ("no_error_code", "ads", None, True),+ ]+ )+ def test_403_disables_the_schema_only_without_a_server_error_code(+ self, _name: str, endpoint: str, code: str | None, disables: bool+ ) -> None:+ source = OpenAIAdsSource()+ body = {"error": {"code": code}} if code is not None else {}+ result = _driver().run(endpoint, always(ScriptedResponse(status=403, json=body)))++ assert result.raised is not None+ error = str(result.raised)+ assert error_message_matches(error, source.get_non_retryable_errors()) is disables+ assert error_message_matches(error, source.get_retryable_errors()) is not disables++class TestValidateCredentials:@parameterized.expand([diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.pyindex d5ef0c93e079..9bb0a4ebf780 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py@@ -367,8 +367,8 @@ _CONNECTION_DROPPED_ERROR_SUBSTRINGS = (# the fetch succeeds again, so a fresh connect after backoff typically recovers. Match the full# "failed to retrieve database credentials" wording, NOT the bare "(ECIRCUITBREAKER)" code —# Supavisor reuses the same code for a different condition ("too many authentication failures",- # tripped by repeated bad credentials rather than pooler bookkeeping), which must stay- # non-retryable and is matched separately in source.py's `get_non_retryable_errors`.+ # tripped by repeated failed sign-ins). That block is not retried in-process, because retrying+ # during the block only adds attempts. source.py keeps it retryable across activity attempts."(ecircuitbreaker) failed to retrieve database credentials",# pgcat (a Rust Postgres pooler) refuses to hand out a backend when every server in the pool is# currently banned/down — a failed health check bans a server and pgcat auto-unbans it after@@ -615,7 +615,8 @@ def _is_dropped_or_connect_timeout(error: BaseException) -> bool:`offset_chunking` reconnect. The schema-discovery path retries drops, connection-limit refusals,server-startup refusals, and recovery conflicts too (via `_is_dropped_or_connection_limit`) butdeliberately keeps failing fast on connect-time *timeouts*, where a timeout usually means an- unreachable host / unconfigured firewall (see `PostgresErrors` and `get_non_retryable_errors`).+ unreachable host / unconfigured firewall (see `PostgresErrors`). A fast failure there is still+ retried by the activity, because `get_retryable_errors` lists the timeout."""return (_is_connection_dropped_error(error)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.pyindex cfee777cb0db..9c37424902b3 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py@@ -524,6 +524,31 @@ _SERIALIZATION_CONFLICT_EXHAUSTED_MESSAGE = ("next schedule.")+# psycopg raises this only while it opens a connection. A schema that synced an hour earlier often+# times out once and then syncs again with no change to the source, so a timeout alone is not proof+# that the host is unreachable for good.+_CONNECT_TIMEOUT_ERROR = "connection timeout expired"+_CONNECT_TIMEOUT_EXHAUSTED_MESSAGE = (+ "PostHog couldn't connect to your database before the connection timed out, and it did so again "+ "on every retry. Check that the database is running and reachable from the public internet, and "+ "that PostHog's egress IP addresses are allowed through your firewall. For a database that can't "+ "be exposed publicly, use the SSH tunnel option. This sync is still enabled and will run again on "+ "its next schedule."+)++# Supavisor blocks new connections for a short time after too many failed sign-ins to a project.+# The failures can come from any client of that project, so the block does not prove that PostHog's+# credentials are wrong. When they are wrong, the next attempt after the block clears gets+# "password authentication failed", which stops the sync.+_POOLER_AUTH_CIRCUIT_BREAKER_ERROR = "too many authentication failures"+_POOLER_AUTH_CIRCUIT_BREAKER_EXHAUSTED_MESSAGE = (+ "Your database connection pooler temporarily blocked new connections after too many failed "+ "sign-in attempts, and the block was still active after every retry. Another client with an old "+ "username or password can cause this block. Check the credentials of each client that connects "+ "to this database, including this source. This sync is still enabled and will run again on its "+ "next schedule."+)+@SourceRegistry.registerclass PostgresSource(@@ -900,18 +925,6 @@ class PostgresSource("require a pooler-specific username such as postgres.<project-ref>. Check your ""credentials, then re-enable the sync."),- # Supabase/Supavisor trips its circuit breaker after repeated bad credentials and refuses- # new connections with "FATAL: (ECIRCUITBREAKER) too many authentication failures, new- # connections are temporarily blocked". The block only clears once the failing attempts- # stop, so it's deterministic until the customer fixes the credentials — retrying just- # re-hits the block. Distinct from the transient credential-fetch variant of the same- # code, which postgres.py keeps retrying (see `_CONNECTION_DROPPED_ERROR_SUBSTRINGS`).- "too many authentication failures": (- "Your database connection pooler is blocking new connections after too many failed "- 'sign-in attempts ("too many authentication failures"). This usually means the '- "username or password is wrong. Check your credentials, wait for the block to clear, "- "then re-enable the sync."- ),# A Postgres server configured with `pam` auth in pg_hba.conf rejects bad credentials with# "FATAL: PAM authentication failed for user <user>" instead of PostgreSQL's# "password authentication failed for user", so the password key above doesn't@@ -1193,18 +1206,6 @@ class PostgresSource(# retry until the customer fixes the network path."Connection timed out": None,"password authentication failed connection": None,- # psycopg raises ConnectionTimeout ("connection timeout expired") only while establishing- # a connection. The read path retries it in-process first (see- # `_is_dropped_or_connect_timeout`); reaching here means every reconnect timed out, i.e. a- # persistently unreachable host — usually a firewall dropping PostHog's egress IPs, an- # IPv6-only host, or a wrong host/port. Stays non-retryable; give the actionable guidance- # the bare driver text lacks (mirrors the validate-path message for "timeout expired").- "connection timeout expired": (- "PostHog couldn't connect to your database before the connection timed out. Check that "- "the database is reachable from the public internet and that PostHog's egress IP "- "addresses are allowed through your firewall. For a database that can't be exposed "- "publicly, use the SSH tunnel option, then re-enable the sync."- ),# TLS ALPN alert (RFC 7301 "no_application_protocol", alert 120) sent by the server# during the TLS handshake. libpq (Postgres 17+) offers the "postgresql" ALPN protocol;# an endpoint that negotiates ALPN but doesn't accept it rejects the handshake outright.@@ -1734,6 +1735,9 @@ class PostgresSource(# retry gives up opening the port forward to the database behind the bastion. See its# definition above for why that stays retryable rather than disabling the schema.#+ # `_CONNECT_TIMEOUT_ERROR` and `_POOLER_AUTH_CIRCUIT_BREAKER_ERROR` are temporary refusals at+ # connect time. See their definitions above for why neither one disables the schema.+ ## "TransactionRetryWithProtoRefreshError" is CockroachDB's serialization-conflict wrapper# (its wire protocol is Postgres-compatible, so customers configure it as a plain Postgres# source — see the TABLESAMPLE workarounds in postgres.py). It covers every conflict reason@@ -1754,6 +1758,8 @@ class PostgresSource(HOST_RESOLUTION_TIMEOUT_ERROR,TEMPORARY_HOST_RESOLUTION_ERROR,_SSH_FORWARD_FAILED_ERROR,+ _CONNECT_TIMEOUT_ERROR,+ _POOLER_AUTH_CIRCUIT_BREAKER_ERROR,}def get_retry_exhausted_errors(self) -> dict[str, str]:@@ -1778,6 +1784,8 @@ class PostgresSource(HOST_RESOLUTION_TIMEOUT_ERROR: HOST_RESOLUTION_EXHAUSTED_MESSAGE,TEMPORARY_HOST_RESOLUTION_ERROR: HOST_RESOLUTION_EXHAUSTED_MESSAGE,_SSH_FORWARD_FAILED_ERROR: _SSH_FORWARD_FAILED_EXHAUSTED_MESSAGE,+ _CONNECT_TIMEOUT_ERROR: _CONNECT_TIMEOUT_EXHAUSTED_MESSAGE,+ _POOLER_AUTH_CIRCUIT_BREAKER_ERROR: _POOLER_AUTH_CIRCUIT_BREAKER_EXHAUSTED_MESSAGE,}def reconcile_schema_metadata(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.pyindex 64fbb6dddc27..4cc62b43e6dd 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py@@ -441,6 +441,18 @@ class TestPostgresSourceNonRetryableErrors:# it accepts connections once recovery completes — so unlike the permanent# hot-standby-disabled refusal it must stay out of NonRetryableErrors.'connection failed: connection to server at "127.0.0.1", port 5432 failed: FATAL: the database system is not yet accepting connections DETAIL: Consistent recovery state has not been yet reached.',+ # A connect that timed out on every in-process attempt. The same schema often syncs again+ # on the next run with no change to the source.+ "ConnectionTimeout: connection timeout expired",+ # A pooler wake timeout on one address and connect timeouts on the others. Both are+ # temporary, so the timeout lines must not disable the sync either.+ 'OperationalError: connection failed: connection to server at "198.51.100.7", port 5432 failed: '+ "FATAL: Failed to connect to database: authentication did not complete within 5000ms\n"+ "Multiple connection attempts failed. All failures were:\n"+ "- host: 'db.example.com', port: '5432', hostaddr: '198.51.100.8': connection timeout expired",+ # Supavisor blocks a project for a short time after failed sign-ins from any client, so the+ # block alone does not prove that this source's credentials are wrong.+ 'connection failed: connection to server at "10.0.0.1", port 5432 failed: FATAL: (ECIRCUITBREAKER) too many authentication failures, new connections are temporarily blocked',],)def test_transient_connection_errors_are_retryable(self, source, error_msg):@@ -489,10 +501,6 @@ class TestPostgresSourceNonRetryableErrors:# hit a plan limit. Account-level state only the customer can lift, so retrying re-hits# the same refusal — must not keep retrying. Host/port are invented, not a real value.'connection failed: connection to server at "db.example.com", port 5432 failed: Your account has restrictions: planLimitReached. Please contact your provider to resolve account restrictions.',- # Supabase/Supavisor trips its circuit breaker after repeated bad credentials. The block- # stays until the failing attempts stop, so it's permanent until the customer fixes the- # credentials — distinct from the transient credential-fetch variant of the same code.- 'connection failed: connection to server at "10.0.0.1", port 5432 failed: FATAL: (ECIRCUITBREAKER) too many authentication failures, new connections are temporarily blocked',# The target database has datallowconn off, or a managed provider paused/suspended it# (SQLSTATE 57P03). Permanent until the customer restores it, so it must not keep retrying.# Distinct from the transient "not yet accepting connections" startup refusal above (which@@ -560,21 +568,6 @@ class TestPostgresSourceNonRetryableErrors:assert matches and matches[0] is not Noneassert "reactivate the branch" in matches[0].lower()- def test_connect_timeout_surfaces_actionable_message(self, source):- # A persistently timing-out connect stays non-retryable, but must surface firewall/reachability- # guidance rather than the bare "connection timeout expired" driver text. Mirror the finalizer's- # first-match selection (external_data_job.py) so a future reorder that shadows it with an- # earlier None-valued timeout key is caught.- error_msg = "connection timeout expired"- matches = [- friendly- for pattern, friendly in source.get_non_retryable_errors().items()- if error_message_matches(error_msg, [pattern])- ]- assert matches, "connect timeout must be classified non-retryable"- assert matches[0] is not None, "connect timeout must surface an actionable message, not raw driver text"- assert "firewall" in matches[0].lower()-@pytest.mark.parametrize("error_msg",[@@ -773,6 +766,15 @@ class TestPostgresSourceNonRetryableErrors:"canceling statement due to user request","canceled",),+ (+ "ConnectionTimeout: connection timeout expired",+ "timed out",+ ),+ (+ 'OperationalError: connection failed: connection to server at "db.example.com" (10.0.0.1), port 5432 failed: '+ "FATAL: (ECIRCUITBREAKER) too many authentication failures, new connections are temporarily blocked",+ "blocked new connections",+ ),("restart transaction: TransactionRetryWithProtoRefreshError: ReadWithinUncertaintyIntervalError: ""read at time 100.000000000,0",@@ -1163,39 +1165,23 @@ class TestPostgresSourceNonRetryableErrors:assert friendly, "PAM authentication failure should surface an actionable message"assert "credentials" in friendly[0]- def test_circuit_breaker_too_many_auth_failures_is_non_retryable(self, source):- # Supavisor's "(ECIRCUITBREAKER)" code also covers a distinct, transient pooler-bookkeeping- # condition ("failed to retrieve database credentials", see postgres.py's- # `_CONNECTION_DROPPED_ERROR_SUBSTRINGS`) — confirm that variant does NOT trip this key, so- # the two "(ECIRCUITBREAKER)" wordings can't be confused for each other.- non_retryable = source.get_non_retryable_errors()- pooler_bookkeeping_msg = (- 'connection failed: connection to server at "10.0.0.1", port 5432 failed: '- "FATAL: (ECIRCUITBREAKER) failed to retrieve database credentials after multiple "- "attempts, new connections are temporarily blocked"- )- assert not any(pattern in pooler_bookkeeping_msg for pattern in non_retryable), (- "the transient pooler-bookkeeping ECIRCUITBREAKER wording must stay retryable"- )-- auth_failure_msg = (- 'connection failed: connection to server at "18.176.230.146", port 5432 failed: '- "FATAL: (ECIRCUITBREAKER) too many authentication failures, new connections are "- "temporarily blocked"- )- assert "too many authentication failures" in non_retryable- assert any(pattern in auth_failure_msg for pattern in non_retryable)-- def test_circuit_breaker_too_many_auth_failures_returns_friendly_message(self, source):- non_retryable = source.get_non_retryable_errors()+ def test_circuit_breaker_with_a_rejected_password_still_stops_the_sync(self, source):+ # The block itself stays retryable, but a rejected password reported on another address+ # must still stop the sync and point the customer at the credentials.error_msg = (- 'connection failed: connection to server at "18.176.230.146", port 5432 failed: '+ 'connection failed: connection to server at "10.0.0.1", port 5432 failed: '"FATAL: (ECIRCUITBREAKER) too many authentication failures, new connections are "- "temporarily blocked"+ "temporarily blocked\nMultiple connection attempts failed. All failures were:\n"+ "- host: 'db.example.com', port: '5432', hostaddr: '10.0.0.2': connection failed: "+ 'FATAL: password authentication failed for user "postgres"')- friendly = [reason for pattern, reason in non_retryable.items() if pattern in error_msg and reason]- assert friendly, "the auth-failure circuit breaker should surface an actionable message"- assert "credentials" in friendly[0]+ matches = [+ friendly+ for pattern, friendly in source.get_non_retryable_errors().items()+ if error_message_matches(error_msg, [pattern])+ ]+ assert matches and matches[0] is not None+ assert "password" in matches[0].lower()def test_supavisor_enotfound_tenant_user_uses_new_key(self, source):# The older tenant/user patterns don't cover the newer "(ENOTFOUND) tenant/user" wording,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/snapchat_ads/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/snapchat_ads/source.pyindex b547879d1f00..c92cf0f753a2 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/snapchat_ads/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/snapchat_ads/source.py@@ -3,7 +3,7 @@ from typing import Optional, castimport requestsfrom posthog.exceptions_capture import capture_exception-from posthog.models.integration import ERROR_TOKEN_REFRESH_FAILED, OauthIntegration+from posthog.models.integration import ERROR_TOKEN_REFRESH_FAILED, OauthIntegration, refresh_backoff_activefrom products.warehouse_sources.backend.facade.codebase_detection import detectfrom products.warehouse_sources.backend.facade.source_config import (@@ -42,6 +42,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.snapchat_afrom products.warehouse_sources.backend.temporal.data_imports.sources.snapchat_ads.utils import list_ad_accountsfrom products.warehouse_sources.backend.types import ExternalDataSourceType+# Raised when the refresh token is dead (an unbroken invalid_grant streak), so only a reconnect helps.+SNAPCHAT_REAUTHORIZATION_REQUIRED_ERROR = "Snapchat Ads authorization can no longer be refreshed"+SNAPCHAT_RECONNECT_MESSAGE = "Snapchat Ads authentication failed. Please reconnect your Snapchat account."+@SourceRegistry.registerclass SnapchatAdsSource(ResumableSource[SnapchatAdsSourceConfig, SnapchatResumeConfig], OAuthMixin):@@ -73,14 +77,26 @@ class SnapchatAdsSource(ResumableSource[SnapchatAdsSourceConfig, SnapchatResumeCdef get_non_retryable_errors(self) -> dict[str, str | None]:# Snapchat's Marketing API surfaces these as requests HTTPError "<code> Client Error".- # They're all permanent for a given config — a deleted/inaccessible ad account (404),- # revoked auth (401), or insufficient permissions (403) — so retrying cannot recover.+ # They're permanent for a given config — a deleted/inaccessible ad account (404) or+ # insufficient permissions (403) — so retrying cannot recover. A revoked grant shows up+ # when the refresh token is rejected, not as a 401, see `source_for_pipeline`.return {- "401 Client Error": "Snapchat Ads authentication failed. Please reconnect your Snapchat account.",+ SNAPCHAT_REAUTHORIZATION_REQUIRED_ERROR: SNAPCHAT_RECONNECT_MESSAGE,"403 Client Error": "Snapchat Ads access forbidden. Please check your account permissions.","404 Client Error": "Snapchat Ads resource not found. Please check that the ad account still exists and is accessible.",}+ def get_retryable_errors(self) -> set[str]:+ # The access token lives for one hour. A 401 can come from a token that expired before the+ # refresh, or from Snapchat when it rejects valid tokens of many accounts at once. The next+ # attempt refreshes an expired token, so a 401 must not disable the schema.+ return {"401 Client Error"}++ def get_retry_exhausted_errors(self) -> dict[str, str]:+ return {+ "401 Client Error": "Snapchat Ads did not accept the access token. The next sync tries again automatically. If this error continues, reconnect your Snapchat account.",+ }+@propertydef get_source_config(self) -> SourceConfig:return SourceConfig(@@ -222,6 +238,17 @@ class SnapchatAdsSource(ResumableSource[SnapchatAdsSourceConfig, SnapchatResumeC) -> SourceResponse:integration = self.get_oauth_integration(config.snapchat_integration_id, inputs.team_id)+ if integration.config.get("refresh_terminal"):+ raise ValueError(f"{SNAPCHAT_REAUTHORIZATION_REQUIRED_ERROR} for job {inputs.job_id}")++ # The refresh sweep can fall behind, and an expired token gets a 401 on every request. A+ # failed refresh is recorded on the integration and the run continues with the old token.+ oauth = OauthIntegration(integration)+ if not refresh_backoff_active(integration) and oauth.access_token_expired():+ oauth.refresh_access_token()+ if integration.config.get("refresh_terminal"):+ raise ValueError(f"{SNAPCHAT_REAUTHORIZATION_REQUIRED_ERROR} for job {inputs.job_id}")+if not integration.access_token:raise ValueError(f"Snapchat Ads access token not found for job {inputs.job_id}")diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/snapchat_ads/tests/test_snapchat_ads.py b/products/warehouse_sources/backend/temporal/data_imports/sources/snapchat_ads/tests/test_snapchat_ads.pyindex f6ab8568928f..eb8ef2b039f4 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/snapchat_ads/tests/test_snapchat_ads.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/snapchat_ads/tests/test_snapchat_ads.py@@ -1,3 +1,4 @@+import timefrom collections.abc import Callable, Iteratorfrom datetime import datetimefrom typing import Any, Optional, cast@@ -7,15 +8,22 @@ from unittest.mock import MagicMock, patchfrom parameterized import parameterized+from posthog.models.integration import Integration, OauthIntegration++from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import error_message_matchesfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.integration_accounts import (IntegrationAccountListingError,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (ScriptedResponse,+ SourceDriver,always,scripted_network,)+from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.snapchatads import (+ SnapchatAdsSourceConfig,+)from products.warehouse_sources.backend.temporal.data_imports.sources.snapchat_ads.settings import (SNAPCHAT_ADS_CONFIG,EndpointType,@@ -311,7 +319,6 @@ class TestNonRetryableErrors:[# Real requests HTTPError strings from Snapchat's Marketing API."404 Client Error: Not Found for url: https://adsapi.snapchat.com/v1/adaccounts/abc/stats",- "401 Client Error: Unauthorized for url: https://adsapi.snapchat.com/v1/adaccounts/abc/stats","403 Client Error: Forbidden for url: https://adsapi.snapchat.com/v1/adaccounts/abc/stats",])@@ -324,6 +331,9 @@ class TestNonRetryableErrors:["500 Server Error: Internal Server Error for url: https://adsapi.snapchat.com/v1/stats","429 Client Error: Too Many Requests for url: https://adsapi.snapchat.com/v1/stats",+ # An expired access token, or a burst of rejections on Snapchat's side. The next attempt+ # refreshes the token, so this must not disable the schema.+ "401 Client Error: Unauthorized for url: https://adsapi.snapchat.com/v1/adaccounts/abc/stats",])def test_transient_errors_stay_retryable(self, error_msg: str) -> None:@@ -540,6 +550,90 @@ class TestBreakdownEndpointConfig:]+def _snapchat_integration(*, refreshed_seconds_ago: int, **config: Any) -> Integration:+ return Integration(+ kind="snapchat",+ config={"expires_in": 3600, "refreshed_at": int(time.time()) - refreshed_seconds_ago, **config},+ sensitive_config={"access_token": "old-token", "refresh_token": "refresh-token"},+ )+++class TestSourceForPipelineTokenRefresh:+ @parameterized.expand(+ [+ ("fresh_token", 60, {}, "new-token", False, "old-token"),+ ("expired_token", 7200, {}, "new-token", True, "new-token"),+ ("expired_token_refresh_fails", 7200, {}, None, True, "old-token"),+ (+ "expired_token_in_refresh_backoff",+ 7200,+ {"refresh_next_attempt_at": 2**40},+ "new-token",+ False,+ "old-token",+ ),+ ]+ )+ def test_sync_sends_a_refreshed_token(+ self,+ _name: str,+ refreshed_seconds_ago: int,+ config: dict[str, Any],+ minted_token: str | None,+ expect_refresh: bool,+ expected_token: str,+ ) -> None:+ integration = _snapchat_integration(refreshed_seconds_ago=refreshed_seconds_ago, **config)+ refreshes: list[bool] = []++ def fake_refresh(oauth: OauthIntegration) -> None:+ refreshes.append(True)+ if minted_token is not None:+ oauth.integration.sensitive_config["access_token"] = minted_token+ oauth.integration.config["refreshed_at"] = int(time.time())++ source = SnapchatAdsSource()+ with (+ patch.object(source, "get_oauth_integration", return_value=integration),+ patch.object(OauthIntegration, "refresh_access_token", fake_refresh),+ ):+ result = SourceDriver(+ source, SnapchatAdsSourceConfig(snapchat_integration_id=1, ad_account_id="acct-example")+ ).run("campaigns", always(ScriptedResponse(json={"request_status": "SUCCESS", "campaigns": []})))++ assert result.raised is None+ assert bool(refreshes) is expect_refresh+ assert {r.headers["authorization"] for r in result.requests} == {f"Bearer {expected_token}"}++ @parameterized.expand(+ [+ ("already_terminal", {"refresh_terminal": True}, False),+ ("terminal_after_refresh", {}, True),+ ]+ )+ def test_dead_refresh_token_disables_the_schema(+ self, _name: str, config: dict[str, Any], terminal_after_refresh: bool+ ) -> None:+ integration = _snapchat_integration(refreshed_seconds_ago=7200, **config)++ def fake_refresh(oauth: OauthIntegration) -> None:+ if terminal_after_refresh:+ oauth.integration.config["refresh_terminal"] = True++ source = SnapchatAdsSource()+ with (+ patch.object(source, "get_oauth_integration", return_value=integration),+ patch.object(OauthIntegration, "refresh_access_token", fake_refresh),+ ):+ result = SourceDriver(+ source, SnapchatAdsSourceConfig(snapchat_integration_id=1, ad_account_id="acct-example")+ ).run("campaigns", always(ScriptedResponse(json={"request_status": "SUCCESS", "campaigns": []})))++ assert result.raised is not None+ assert result.requests == []+ assert error_message_matches(str(result.raised), source.get_non_retryable_errors())++class TestValidateCredentials:@parameterized.expand([diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/snowflake/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/snowflake/source.pyindex 7cb2da6e9a6b..2325100f9938 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/snowflake/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/snowflake/source.py@@ -64,6 +64,8 @@ _UNENCRYPTED_KEY_WITH_PASSPHRASE_MESSAGE = ("Remove the passphrase, or paste your encrypted private key, then {action}")+_RESOURCE_MONITOR_QUOTA_ERROR = "cannot be resumed because resource monitor"+# Shown when `validate_credentials` hits a transient connect blip (see `get_retryable_errors`). The# sync path retries such a blip quietly, but interactive validation has nothing to retry# automatically, so it tells the user it was a brief blip and to try again rather than capturing it@@ -84,6 +86,7 @@ _MFA_ENFORCED_MESSAGE = ()SnowflakeErrors = {+ _RESOURCE_MONITOR_QUOTA_ERROR: "The Snowflake warehouse can't resume because its resource monitor has used its credit quota. Raise the quota or wait for it to reset, then try again.","No active warehouse selected in the current session": "No active warehouse is available for this connection. Check that the configured warehouse exists, is running, and that the connecting role has USAGE on it, then try again.","or attempt to login with another role": "Role specified doesn't exist or is not authorized","Incorrect username or password was specified": "Incorrect username or password was specified",@@ -325,10 +328,6 @@ class SnowflakeSource(SQLSource[SnowflakeSourceConfig], ResumableSource[Snowflak# warehouse is missing/suspended or the role lacks USAGE on it — it just leaves the session# warehouse unset. Retrying can never succeed until the customer fixes the grant or warehouse."No active warehouse selected in the current session": "No active warehouse is available for this connection. Check that the configured warehouse exists, is running, and that the connecting role has USAGE on it, then resync.",- # Snowflake error 090073 (22000): the warehouse can't resume because the customer's resource- # monitor used its credit quota. Only the customer can fix it, so retrying never succeeds.- # The query id and names are volatile, so we match the stable phrase.- "cannot be resumed because resource monitor": "Your Snowflake warehouse can't resume because its resource monitor has used its credit quota. Raise the quota or wait for it to reset, then resync.",# Snowflake error 090105 (22000): the session has no current database, so the first query# that references an unqualified object (our `information_schema` listing query) fails. The# connector doesn't fail at connect time when the configured database is missing or the role@@ -483,6 +482,17 @@ class SnowflakeSource(SQLSource[SnowflakeSourceConfig], ResumableSource[Snowflak# opens a new connection and re-executes from scratch, so this is self-recovering rather# than a bug. The query id in the message is volatile, so we match the stable phrase."SQL execution was cancelled by the client due to a timeout",+ # Snowflake error 090073 (22000): the warehouse can't resume because the customer's resource+ # monitor used its credit quota. Monitors reset on the schedule the customer set (daily,+ # weekly or monthly), and the customer can raise the quota at any time. Disabling the+ # schema would leave the sync off after the quota returns, so keep it enabled and retry.+ # The query id and names are volatile, so we match the stable phrase.+ _RESOURCE_MONITOR_QUOTA_ERROR,+ }++ def get_retry_exhausted_errors(self) -> dict[str, str]:+ return {+ _RESOURCE_MONITOR_QUOTA_ERROR: "Your Snowflake warehouse can't resume because its resource monitor has used its credit quota. This sync tries again on its own and works again once you raise the quota or it resets.",}def reconcile_schema_metadata(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/snowflake/tests/test_snowflake.py b/products/warehouse_sources/backend/temporal/data_imports/sources/snowflake/tests/test_snowflake.pyindex ccc53c880fc7..af3dcafca2ac 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/snowflake/tests/test_snowflake.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/snowflake/tests/test_snowflake.py@@ -1020,7 +1020,7 @@ class TestSnowflakeSourceNonRetryableErrors:assert is_non_retryable, f"No-active-warehouse error should be non-retryable: {error_msg}"@pytest.mark.parametrize(- "error_msg,expected_non_retryable",+ "error_msg,expected_quota",[("090073 (22000): 01c00000-0000-0000-0000-000000000000: Warehouse 'EXAMPLE_WH' cannot be resumed "@@ -1032,20 +1032,19 @@ class TestSnowflakeSourceNonRetryableErrors:True,),("090001 (22000): 01c00000-0000-0000-0000-000000000000: Warehouse 'EXAMPLE_WH' cannot be resumed.", False),- (- "000606 (57P03): 01c00000-0000-0000-0000-000000000000: Statement reached its statement or warehouse timeout.",- False,- ),],)- def test_resource_monitor_quota_is_non_retryable_only_for_that_error(- self, source, error_msg, expected_non_retryable- ):- non_retryable = source.get_non_retryable_errors()- matches = [message for pattern, message in non_retryable.items() if pattern in error_msg]- assert bool(matches) is expected_non_retryable- if expected_non_retryable:- assert all(message is not None and "resource monitor" in message for message in matches)+ def test_resource_monitor_quota_keeps_the_schema_enabled(self, source, error_msg, expected_quota):+ assert error_message_matches(error_msg, source.get_retryable_errors()) is expected_quota+ assert not error_message_matches(error_msg, source.get_non_retryable_errors())+ exhausted = [+ message+ for key, message in source.get_retry_exhausted_errors().items()+ if error_message_matches(error_msg, [key])+ ]+ assert bool(exhausted) is expected_quota+ if expected_quota:+ assert "tries again on its own" in exhausted[0]@pytest.mark.parametrize("error_msg",@@ -1350,6 +1349,19 @@ class TestSnowflakeValidateCredentials:assert message is not None and "passphrase" in messagemock_capture.assert_not_called()+ def test_resource_monitor_quota_names_the_quota_not_a_network_blip(self, source):+ error = ProgrammingError(+ msg="090073 (22000): 01c00000-0000-0000-0000-000000000000: Warehouse 'EXAMPLE_WH' cannot be resumed "+ "because resource monitor 'EXAMPLE_MONITOR' has exceeded its quota.",+ errno=90073,+ )+ with patch.object(source, "get_schemas", side_effect=error):+ ok, message = source.validate_credentials(_make_config("password"), team_id=1)++ assert ok is False+ assert message is not None and "resource monitor" in message+ assert "brief" not in message+def test_unencrypted_key_with_passphrase_returns_friendly_message_without_capture(self, source):# A passphrase supplied for an unencrypted key — a TypeError raised while parsing, so it's a# user config error that should surface an actionable message, not be captured.