:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-imports): stop retrying a postgres row-level security denial (#120389)
Tom Owers4138fd9a025e
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.pyindex 71582df8f681..559c9b2882fd 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py@@ -2853,6 +2853,15 @@ def _has_duplicate_primary_keys(# follows fails on the same denial — already surfaced as non-retryable with an actionable# message via `get_non_retryable_errors`. Propagate rather than capture it as noise.raise+ except psycopg.errors.RaiseException:+ # SQLSTATE P0001, raised only by a plain `RAISE EXCEPTION` inside a PL/pgSQL function —+ # never by Postgres itself. Since this source only reads, it's almost always a row-level+ # security policy or computed-column function blocking the SELECT, and the extraction+ # that follows hits the same raise. Swallowing it as "no duplicate keys" would be a false+ # negative and double the error-tracking noise. Propagate so it surfaces once, through+ # the same non-retryable classification (where the message names an access denial) as the+ # extraction failure; a transient custom raise stays retryable either way.+ raiseexcept psycopg.OperationalError:# A connection-level failure here (e.g. a foreign-data-wrapper server refusing a new# connection with "too many connections") means the probe never ran — swallowing it asdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.pyindex 5c9e58bcbb0f..562b5f540565 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py@@ -139,6 +139,17 @@ _ROLE_CANNOT_LOG_IN_ERROR = ("user, then re-enable the sync.")+# A custom PL/pgSQL `RAISE EXCEPTION` (most often a row-level security policy) denying PostHog's+# sync connection, which carries no authenticated application user. The message is customer-authored+# free text, so this stays deliberately generic rather than naming the specific check that failed.+_CUSTOM_ACCESS_DENIAL_ERROR = (+ "A database function used by one or more of the tables you selected to sync denied the read "+ '(PostgreSQL reported a custom "RAISE EXCEPTION" naming an access denial). This is often a '+ "row-level security policy that expects an authenticated application user, which PostHog's "+ "sync connection doesn't have. Update the policy or function to allow PostHog's connecting "+ "role, or remove the affected tables from the sync, then re-enable the sync."+)+# A DNS-resolution failure for the database host, surfaced by libpq ("could not translate host# name") or the raw socket wording ("Name or service not known" / "No address associated with# hostname"). Already non-retryable, but the bare driver text gives the customer nothing to act on,@@ -1012,6 +1023,18 @@ class PostgresSource(# Match that message substring so the role-lacks-SELECT case is caught at both layers# and we stop retrying instead of re-reading into the same denial every attempt."InsufficientPrivilege": None,+ # A row-level security policy (or other PL/pgSQL function) can deny a read with a+ # custom `RAISE EXCEPTION` (SQLSTATE P0001, psycopg's `RaiseException`) instead of+ # Postgres's own `InsufficientPrivilege`. The message is customer-authored free text+ # with no fixed wording, so matching the bare exception class would also catch a+ # transient custom raise (a lock wait, a maintenance window) and wrongly auto-disable+ # a healthy sync — match only phrasing that names an access denial. These substrings+ # are also the raw psycopg message, so the same keys stop the in-activity retry loop+ # in `_handle_import_error` (no Temporal wrapping/class name needed there), not just+ # future scheduled runs.+ "access denied": _CUSTOM_ACCESS_DENIAL_ERROR,+ "not authorized": _CUSTOM_ACCESS_DENIAL_ERROR,+ "unauthorized": _CUSTOM_ACCESS_DENIAL_ERROR,# A view selected for sync calls a function the connecting role can't execute (SQLSTATE# 42501) — most often a view that decrypts secrets via Supabase Vault / pgsodium# (`crypto_aead_det_decrypt`). Distinct from the table/view SELECT denial below: grantingdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.pyindex 52e571251003..ce6f0b58d352 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py@@ -1620,6 +1620,49 @@ class TestPostgresSourceNonRetryableErrors:assert first_match is not None, "Recursive RLS policy error should surface an actionable message"assert "BYPASSRLS" in first_match+ @pytest.mark.parametrize(+ "error_msg",+ [+ # Raw psycopg message (what the activity-level check sees via str(e)) — no class+ # name. Unlike InsufficientPrivilege's SQLSTATE 42501 text, this wording is+ # customer-authored RAISE text, but "access denied"/"unauthorized" is common enough+ # phrasing for a hand-rolled access check to match directly, without a class name.+ "Access denied: No user ID provided",+ "unauthorized: missing tenant claim",+ # Temporal-wrapped message (what the workflow-level check sees) — carries the class+ # name, matched the same way regardless.+ "RaiseException: Access denied: No user ID provided",+ ],+ )+ def test_custom_access_denial_raise_is_non_retryable(self, source, error_msg):+ # Matching is case-insensitive in production (`error_message_matches`), unlike the+ # customer-authored sentence-case wording a hand-rolled RAISE typically uses.+ non_retryable = source.get_non_retryable_errors()+ assert error_message_matches(error_msg, non_retryable.keys()), (+ f"Custom access-denial RAISE EXCEPTION should be non-retryable: {error_msg}"+ )++ def test_custom_access_denial_raise_returns_friendly_message(self, source):+ non_retryable = source.get_non_retryable_errors()+ error_msg = "RaiseException: Access denied: No user ID provided"+ friendly = [+ reason+ for pattern, reason in non_retryable.items()+ if error_message_matches(error_msg, [pattern]) and reason+ ]+ assert friendly, "Custom access-denial RAISE EXCEPTION should surface an actionable message"+ assert "row-level security" in friendly[0]++ def test_unrelated_custom_raise_exception_stays_retryable(self, source):+ # A custom RAISE EXCEPTION that doesn't name an access denial (e.g. a lock wait or a+ # maintenance-window check) must stay retryable — classifying every RaiseException would+ # wrongly auto-disable a sync over a transient condition instead of a permanent one.+ error_msg = "RaiseException: table is temporarily locked for maintenance, try again later"+ non_retryable = source.get_non_retryable_errors()+ assert not error_message_matches(error_msg, non_retryable.keys()), (+ f"Unrelated custom RAISE EXCEPTION should stay retryable: {error_msg}"+ )+@pytest.mark.parametrize("error_msg",[@@ -8758,6 +8801,14 @@ class TestHasDuplicatePrimaryKeys:psycopg.errors.InsufficientPrivilege("permission denied for table orders"),psycopg.errors.InsufficientPrivilege,),+ # A row-level security policy (or other PL/pgSQL function) blocked the read with a+ # custom RAISE (SQLSTATE P0001). Already non-retryable via get_non_retryable_errors,+ # so the probe must propagate it rather than capture it.+ (+ "raise_exception",+ psycopg.errors.RaiseException("Access denied: No user ID provided"),+ psycopg.errors.RaiseException,+ ),])def test_reraises_without_capturing(self, _name, side_effect, expected_exception):