:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-imports): stop retrying postgres FOREACH-over-null-array errors (#125641)
Tom Owers8f728c4c9ae7
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.pyindex 824bdbd752ca..f57857ee3e23 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py@@ -1692,6 +1692,20 @@ class PostgresSource("division in that column or view definition (for example with a CASE or NULLIF), or ""remove it from the sync, then re-enable the sync."),+ # A view, computed column, or RLS policy in the customer's own schema calls a PL/pgSQL+ # function that does `FOREACH var IN ARRAY <expr>`, where `<expr>` evaluates to NULL for+ # at least one row (NullValueNotAllowed, SQLSTATE 22004) — Postgres's FOREACH never+ # accepts a null array. We only ever run `SELECT ... FROM <relation>`; the function lives+ # in the customer's own schema and runs on every row, so it fails identically on every+ # retry. Match the stable error text and exclude the volatile CONTEXT line naming the+ # function and line number.+ "FOREACH expression must not be null": (+ "A view, column, or policy you're syncing calls a function in your database that "+ "loops over an array with FOREACH, and that array is null for at least one row "+ '("FOREACH expression must not be null"). Guard the function against a null array '+ "(for example with COALESCE(expr, '{}')), or remove the affected table or view from "+ "the sync, then re-enable the sync."+ ),# A view or computed column in the customer's own schema calls a PL/pgSQL function that# enforces its own access check and raises a custom RAISE EXCEPTION when it finds no# user context — commonly a Supabase helper keyed off auth.uid(), which Supabase'sdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.pyindex e7b0f38031db..176e0767c87b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py@@ -1824,6 +1824,28 @@ class TestPostgresSourceNonRetryableErrors:assert friendly, "pg_stat_statements not loaded error should surface an actionable message"assert "shared_preload_libraries" in friendly[0]+ @pytest.mark.parametrize(+ "error_msg",+ [+ # Raw psycopg message (what the activity-level check sees via str(e)) — a view/RLS+ # policy's own PL/pgSQL function doing FOREACH over an array that is null for a row.+ 'FOREACH expression must not be null\nCONTEXT: PL/pgSQL function ledger.holdout_hash(text,text) line 4 at FOREACH over array\nSQL function "is_holdout_domain" statement 1',+ # Temporal-wrapped message (what the workflow-level check sees) — carries the class name.+ "NullValueNotAllowed: FOREACH expression must not be null",+ ],+ )+ def test_foreach_over_null_array_is_non_retryable(self, source, error_msg):+ non_retryable = source.get_non_retryable_errors()+ is_non_retryable = any(pattern in error_msg for pattern in non_retryable.keys())+ assert is_non_retryable, f"FOREACH-over-null-array error should be non-retryable: {error_msg}"++ def test_foreach_over_null_array_returns_friendly_message(self, source):+ non_retryable = source.get_non_retryable_errors()+ error_msg = "FOREACH expression must not be null"+ friendly = [reason for pattern, reason in non_retryable.items() if pattern in error_msg and reason]+ assert friendly, "FOREACH-over-null-array error should surface an actionable message"+ assert "FOREACH" in friendly[0]+@pytest.mark.parametrize("error_msg",[