:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): classify auth and outage errors correctly for 8 sources (#120451)
Tom Owersac8189791b11
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/appdynamics/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/appdynamics/source.pyindex ae950669952d..fe0b8d3f7d86 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/appdynamics/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/appdynamics/source.py@@ -11,6 +11,7 @@ from products.warehouse_sources.backend.facade.source_config import (SourceFieldSelectConfigOption,)from products.warehouse_sources.backend.temporal.data_imports.sources.appdynamics.appdynamics import (+ OAUTH_FAILED_MESSAGE,AppdynamicsAuth,AppdynamicsResumeConfig,appdynamics_source,@@ -193,7 +194,7 @@ You can create an API client in your controller under **Administration → API Creturn {"401 Client Error": "Your AppDynamics credentials are invalid or expired. Please update your credentials and reconnect.","403 Client Error": "Your AppDynamics user or API client is missing read access to application data. Grant the required roles and try again.",- "AppDynamics OAuth token request failed": "Your AppDynamics API client credentials were rejected. Check your API client name, client secret, and account name, then reconnect.",+ OAUTH_FAILED_MESSAGE: "Your AppDynamics API client credentials were rejected. Check your API client name, client secret, and account name, then reconnect.","Too many metric paths configured": None,"Too many event types configured": None,"Too many AppDynamics applications": None,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/culture_amp/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/culture_amp/source.pyindex 26412156b880..a16c83cbbed9 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/culture_amp/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/culture_amp/source.py@@ -13,6 +13,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.canCanonicalDescriptions,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import (+ OAUTH2_PERMANENT_ERROR_MARKER,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import (SourceSchema,@@ -50,6 +53,7 @@ class CultureAmpSource(ResumableSource[CultureAmpSourceConfig, CultureAmpResumeCdef get_non_retryable_errors(self) -> dict[str, str | None]:return {+ OAUTH2_PERMANENT_ERROR_MARKER: "Culture Amp rejected the client ID or client secret. Check both values, then reconnect.","400 Client Error: Bad Request for url: https://api.cultureamp.com/v1/oauth2/token": "Culture Amp rejected the token request. Please check your client ID, client secret, and account ID, and that the credentials have the required permissions.","401 Client Error: Unauthorized for url: https://api.cultureamp.com/v1/oauth2/token": "Culture Amp authentication failed. Please check your client ID and client secret.","403 Client Error: Forbidden for url: https://api.cultureamp.com": "Culture Amp denied access. Please check that your API credentials have the required permissions for this dataset.",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/ip2whois/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/ip2whois/source.pyindex 67e13ee2f658..cad0f252929f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/ip2whois/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/ip2whois/source.py@@ -113,7 +113,8 @@ Each sync looks up every configured domain once and replaces the table with the# Account-level API errors returned in the JSON error envelope (exhausted monthly quota,# disabled account, and anything else that isn't specific to a single domain) are raised as# IP2WhoisAPIError. These are deterministic within a run, so fail fast rather than retry.- "IP2WHOIS API error": "Your IP2WHOIS request was rejected by the API — the account may be disabled or the monthly lookup quota exhausted. Check your IP2WHOIS dashboard, then reconnect or resync.",+ "IP2WHOIS API error [": "Your IP2WHOIS request was rejected by the API. The account may be disabled or the monthly lookup quota used up. Check your IP2WHOIS dashboard, then reconnect or resync.",+ "IP2WHOIS API error: unexpected status": "Your IP2WHOIS request was rejected by the API. The account may be disabled or the monthly lookup quota used up. Check your IP2WHOIS dashboard, then reconnect or resync.",}def get_schemas(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/marketo/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/marketo/source.pyindex 6ae91b027141..8292b23288ef 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/marketo/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/marketo/source.py@@ -55,8 +55,15 @@ class MarketoSource(ResumableSource[MarketoSourceConfig, MarketoResumeConfig]):return CANONICAL_DESCRIPTIONSdef get_non_retryable_errors(self) -> dict[str, str | None]:+ auth_message = (+ "PostHog could not authenticate with Marketo. Check the client ID and secret on your Marketo custom service, "+ "and make sure the service is still active."+ )return {- "Marketo authentication failed": "PostHog could not authenticate with Marketo. Check the client ID and secret on your Marketo custom service, and make sure the service is still active.",+ "Marketo authentication failed: status=400": auth_message,+ "Marketo authentication failed: status=401": auth_message,+ "Marketo authentication failed: status=403": auth_message,+ "Marketo authentication failed: no access token": auth_message,"Marketo API error 601": "Your Marketo access token is invalid. Reconnect the source with a fresh client ID and secret.","Marketo API error 603": "Your Marketo custom service does not have permission for this data. Add the missing API role permissions in Marketo and try again.","Marketo API error 607": "Your Marketo instance hit its daily API quota. The sync will run again once the quota resets.",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/sigma_computing/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/sigma_computing/source.pyindex 2a0ef0d2081c..cf6aada0f270 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/sigma_computing/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/sigma_computing/source.py@@ -15,6 +15,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.canCanonicalDescriptions,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import (+ OAUTH2_PERMANENT_ERROR_MARKER,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import (SourceSchema,@@ -77,6 +80,7 @@ class SigmaComputingSource(ResumableSource[SigmaComputingSourceConfig, SigmaCompdef get_non_retryable_errors(self) -> dict[str, str | None]:return {+ OAUTH2_PERMANENT_ERROR_MARKER: "Sigma rejected your API client credentials. Check the client ID and secret, then reconnect.","Sigma rejected the API client credentials": "Sigma rejected your API client credentials. Please check the client ID and secret and reconnect.","401 Client Error": "Sigma rejected the access token. Please check the client ID and secret, then reconnect.","403 Client Error": "Your Sigma API client does not have permission for this resource.",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/similarweb/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/similarweb/source.pyindex 832f045c0144..7c18be219fc1 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/similarweb/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/similarweb/source.py@@ -66,7 +66,7 @@ class SimilarwebSource(ResumableSource[SimilarwebSourceConfig, SimilarwebResumeCdef get_non_retryable_errors(self) -> dict[str, str | None]:return {- "403 Client Error: Forbidden for url": (+ "Similarweb API returned status 403": ("Similarweb rejected the API key, or the account is out of data credits. ""Check both on the API management page in your Similarweb account."),diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_error_contract.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_error_contract.pyindex 1c2c38b67e6e..ee4eb5d952bf 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_error_contract.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_error_contract.py@@ -194,6 +194,8 @@ ONE_AUTH_STATUS_CLASSIFIED: dict[str, str] = {"Rewardful": "401","SharePoint": "401","Shopify": "401",+ # A 401 means no data for one domain, so the source skips it on purpose.+ "Similarweb": "403","SpotlerCRM": "403","TMDb": "401","TrunkIo": "401",@@ -239,19 +241,9 @@ AUTH_CLASSIFIED_ELSEWHERE: dict[str, str] = {"UsBea": "error-body",}-# Sources whose real 401 and 403 errors the pipeline retries until the retry budget runs out, and then-# shows as the raw error. Each entry is a defect to fix.-AUTH_FAILURE_NOT_CLASSIFIED: dict[str, str] = {- # The shared OAuth2 token exchange raises `OAuth2AuthRequestError`, and no key matches its- # `OAUTH2_PERMANENT_ERROR_MARKER`.- "CultureAmp": "oauth2-token",- "SigmaComputing": "oauth2-token",- "Workday": "oauth2-token",- "Zoom": "oauth2-token",- # The source raises `Similarweb API returned status 403 for <url>`, and its key expects the- # `requests` wording `403 Client Error: Forbidden for url`. A 401 means no data for one domain, on purpose.- "Similarweb": "key-mismatch",-}+# Sources whose real 401 and 403 errors the pipeline retries until the retry budget runs out.+# Each entry is a defect to fix.+AUTH_FAILURE_NOT_CLASSIFIED: dict[str, str] = {}# Sources whose real error for a transient failure matches a key on purpose. The value is every such failure.TRANSIENT_FAILURE_NON_RETRYABLE_ON_PURPOSE: dict[str, frozenset[str]] = {@@ -263,16 +255,9 @@ TRANSIENT_FAILURE_NON_RETRYABLE_ON_PURPOSE: dict[str, frozenset[str]] = {"Singular": frozenset({"429"}),}-# Sources whose real error for a transient failure matches a key, so that a short outage at the vendor-# disables the sync. Each entry is a defect to fix: make the key narrower. The value is every such failure.-TRANSIENT_FAILURE_CLASSIFIED_NON_RETRYABLE: dict[str, frozenset[str]] = {- # The key `AppDynamics OAuth token request failed` also matches the retryable token error.- "Appdynamics": frozenset({"429", "500"}),- # The key `IP2WHOIS API error` also matches `IP2WHOIS API error (retryable)`.- "IP2Whois": frozenset({"429", "500"}),- # The key `Marketo authentication failed` also matches a 429 or a 5xx from the identity endpoint.- "Marketo": frozenset({"429", "500"}),-}+# Sources whose real error for a transient failure matches a key, so that a short outage disables the sync.+# Each entry is a defect to fix: make the key narrower. The value is every such failure.+TRANSIENT_FAILURE_CLASSIFIED_NON_RETRYABLE: dict[str, frozenset[str]] = {}@frozen@@ -471,3 +456,21 @@ def test_transient_failures_stay_retryable(name: str) -> None:)def test_allowlists_name_only_extracting_sources(allowlist: dict[str, Any]) -> None:assert sorted(allowlist.keys() - set(EXTRACTING_SOURCE_NAMES)) == []+++ "name,message",+ [+ (+ "Appdynamics",+ "AppDynamics OAuth token request failed. Check your API client name, client secret, and account name.",+ ),+ ("IP2Whois", "IP2WHOIS API error [10000]: Invalid API key."),+ ("IP2Whois", "IP2WHOIS API error: unexpected status 418"),+ ("Marketo", "Marketo authentication failed: status=401, body={}"),+ ("Marketo", "Marketo authentication failed: status=401, path=/rest/v1/leads.json, body={}"),+ ("Marketo", "Marketo authentication failed: no access token in response ({})"),+ ],+)+def test_narrowed_keys_still_match_permanent_errors(name: str, message: str) -> None:+ assert error_message_matches(message, ALL_SOURCES[name].get_non_retryable_errors())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/workday/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/workday/source.pyindex 0ed2088daac0..2a797956c390 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/workday/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/workday/source.py@@ -13,6 +13,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.canCanonicalDescriptions,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import (+ OAUTH2_PERMANENT_ERROR_MARKER,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import (SourceSchema,@@ -123,6 +126,7 @@ Your hostname and tenant are the first two parts of your Workday URL — for `htdef get_non_retryable_errors(self) -> dict[str, str | None]:return {+ OAUTH2_PERMANENT_ERROR_MARKER: "Workday rejected your API client credentials. Check the client ID, secret and refresh token, then reconnect.",TOKEN_ERROR: "Workday rejected your API client credentials. Please check the client ID, secret and refresh token, then reconnect.","401 Client Error": "Workday rejected the access token. Please reconnect the source with a fresh refresh token.","403 Client Error": "Your Workday API client is not authorized for this resource. Please check its scopes and domain security policies.",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/zoom/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/zoom/source.pyindex 12e58fab378f..8d0a8ead4442 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/zoom/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/zoom/source.py@@ -13,6 +13,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.canCanonicalDescriptions,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import (+ OAUTH2_PERMANENT_ERROR_MARKER,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import SourceSchemafrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse@@ -110,6 +113,7 @@ class ZoomSource(ResumableSource[ZoomSourceConfig, ZoomResumeConfig]):def get_non_retryable_errors(self) -> dict[str, str | None]:return {+ OAUTH2_PERMANENT_ERROR_MARKER: "Zoom rejected the client ID or client secret. Check the account ID, client ID and client secret, then reconnect.","401 Client Error": "Zoom authentication failed. Please check your account ID, client ID, and client secret.","Invalid access token": "Zoom authentication failed. Please reconnect the source.","403 Client Error": "Your Zoom app is missing the required scopes. Grant user/meeting/webinar read scopes and reconnect.",