Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-imports): stop retrying a postgres row-level security denial (#120389)

Tom Owers4138fd9a025e
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py
index 71582df8f681..559c9b2882fd 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py
@@ -2853,6 +2853,15 @@ def _has_duplicate_primary_keys(
# follows fails on the same denial — already surfaced as non-retryable with an actionable
# message via `get_non_retryable_errors`. Propagate rather than capture it as noise.
raise
+ except psycopg.errors.RaiseException:
+ # SQLSTATE P0001, raised only by a plain `RAISE EXCEPTION` inside a PL/pgSQL function —
+ # never by Postgres itself. Since this source only reads, it's almost always a row-level
+ # security policy or computed-column function blocking the SELECT, and the extraction
+ # that follows hits the same raise. Swallowing it as "no duplicate keys" would be a false
+ # negative and double the error-tracking noise. Propagate so it surfaces once, through
+ # the same non-retryable classification (where the message names an access denial) as the
+ # extraction failure; a transient custom raise stays retryable either way.
+ raise
except psycopg.OperationalError:
# A connection-level failure here (e.g. a foreign-data-wrapper server refusing a new
# connection with "too many connections") means the probe never ran — swallowing it as
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py
index 5c9e58bcbb0f..562b5f540565 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py
@@ -139,6 +139,17 @@ _ROLE_CANNOT_LOG_IN_ERROR = (
"user, then re-enable the sync."
)
+# A custom PL/pgSQL `RAISE EXCEPTION` (most often a row-level security policy) denying PostHog's
+# sync connection, which carries no authenticated application user. The message is customer-authored
+# free text, so this stays deliberately generic rather than naming the specific check that failed.
+_CUSTOM_ACCESS_DENIAL_ERROR = (
+ "A database function used by one or more of the tables you selected to sync denied the read "
+ '(PostgreSQL reported a custom "RAISE EXCEPTION" naming an access denial). This is often a '
+ "row-level security policy that expects an authenticated application user, which PostHog's "
+ "sync connection doesn't have. Update the policy or function to allow PostHog's connecting "
+ "role, or remove the affected tables from the sync, then re-enable the sync."
+)
+
# A DNS-resolution failure for the database host, surfaced by libpq ("could not translate host
# name") or the raw socket wording ("Name or service not known" / "No address associated with
# hostname"). Already non-retryable, but the bare driver text gives the customer nothing to act on,
@@ -1012,6 +1023,18 @@ class PostgresSource(
# Match that message substring so the role-lacks-SELECT case is caught at both layers
# and we stop retrying instead of re-reading into the same denial every attempt.
"InsufficientPrivilege": None,
+ # A row-level security policy (or other PL/pgSQL function) can deny a read with a
+ # custom `RAISE EXCEPTION` (SQLSTATE P0001, psycopg's `RaiseException`) instead of
+ # Postgres's own `InsufficientPrivilege`. The message is customer-authored free text
+ # with no fixed wording, so matching the bare exception class would also catch a
+ # transient custom raise (a lock wait, a maintenance window) and wrongly auto-disable
+ # a healthy sync — match only phrasing that names an access denial. These substrings
+ # are also the raw psycopg message, so the same keys stop the in-activity retry loop
+ # in `_handle_import_error` (no Temporal wrapping/class name needed there), not just
+ # future scheduled runs.
+ "access denied": _CUSTOM_ACCESS_DENIAL_ERROR,
+ "not authorized": _CUSTOM_ACCESS_DENIAL_ERROR,
+ "unauthorized": _CUSTOM_ACCESS_DENIAL_ERROR,
# A view selected for sync calls a function the connecting role can't execute (SQLSTATE
# 42501) — most often a view that decrypts secrets via Supabase Vault / pgsodium
# (`crypto_aead_det_decrypt`). Distinct from the table/view SELECT denial below: granting
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py
index 52e571251003..ce6f0b58d352 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py
@@ -1620,6 +1620,49 @@ class TestPostgresSourceNonRetryableErrors:
assert first_match is not None, "Recursive RLS policy error should surface an actionable message"
assert "BYPASSRLS" in first_match
+ @pytest.mark.parametrize(
+ "error_msg",
+ [
+ # Raw psycopg message (what the activity-level check sees via str(e)) — no class
+ # name. Unlike InsufficientPrivilege's SQLSTATE 42501 text, this wording is
+ # customer-authored RAISE text, but "access denied"/"unauthorized" is common enough
+ # phrasing for a hand-rolled access check to match directly, without a class name.
+ "Access denied: No user ID provided",
+ "unauthorized: missing tenant claim",
+ # Temporal-wrapped message (what the workflow-level check sees) — carries the class
+ # name, matched the same way regardless.
+ "RaiseException: Access denied: No user ID provided",
+ ],
+ )
+ def test_custom_access_denial_raise_is_non_retryable(self, source, error_msg):
+ # Matching is case-insensitive in production (`error_message_matches`), unlike the
+ # customer-authored sentence-case wording a hand-rolled RAISE typically uses.
+ non_retryable = source.get_non_retryable_errors()
+ assert error_message_matches(error_msg, non_retryable.keys()), (
+ f"Custom access-denial RAISE EXCEPTION should be non-retryable: {error_msg}"
+ )
+
+ def test_custom_access_denial_raise_returns_friendly_message(self, source):
+ non_retryable = source.get_non_retryable_errors()
+ error_msg = "RaiseException: Access denied: No user ID provided"
+ friendly = [
+ reason
+ for pattern, reason in non_retryable.items()
+ if error_message_matches(error_msg, [pattern]) and reason
+ ]
+ assert friendly, "Custom access-denial RAISE EXCEPTION should surface an actionable message"
+ assert "row-level security" in friendly[0]
+
+ def test_unrelated_custom_raise_exception_stays_retryable(self, source):
+ # A custom RAISE EXCEPTION that doesn't name an access denial (e.g. a lock wait or a
+ # maintenance-window check) must stay retryable — classifying every RaiseException would
+ # wrongly auto-disable a sync over a transient condition instead of a permanent one.
+ error_msg = "RaiseException: table is temporarily locked for maintenance, try again later"
+ non_retryable = source.get_non_retryable_errors()
+ assert not error_message_matches(error_msg, non_retryable.keys()), (
+ f"Unrelated custom RAISE EXCEPTION should stay retryable: {error_msg}"
+ )
+
@pytest.mark.parametrize(
"error_msg",
[
@@ -8758,6 +8801,14 @@ class TestHasDuplicatePrimaryKeys:
psycopg.errors.InsufficientPrivilege("permission denied for table orders"),
psycopg.errors.InsufficientPrivilege,
),
+ # A row-level security policy (or other PL/pgSQL function) blocked the read with a
+ # custom RAISE (SQLSTATE P0001). Already non-retryable via get_non_retryable_errors,
+ # so the probe must propagate it rather than capture it.
+ (
+ "raise_exception",
+ psycopg.errors.RaiseException("Access denied: No user ID provided"),
+ psycopg.errors.RaiseException,
+ ),
]
)
def test_reraises_without_capturing(self, _name, side_effect, expected_exception):