:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(postgres): stop retrying cdc syncs whose pooler can't find the tenant (#117631)
Tom Owers22da9520cb4c
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/errors.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/errors.pyindex 66c7cda9a8fa..c2f273438bfc 100644--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/errors.py+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/errors.py@@ -28,6 +28,7 @@ class CDCErrorCategory(enum.StrEnum):CONNECTION_FAILED = "connection_failed"HOST_UNREACHABLE = "host_unreachable"QUOTA_EXCEEDED = "quota_exceeded"+ POOLER_TENANT_NOT_FOUND = "pooler_tenant_not_found"SSH_TUNNEL_FAILED = "ssh_tunnel_failed"SLOT_MISSING = "slot_missing"SLOT_NOT_CONFIGURED = "slot_not_configured"@@ -115,6 +116,11 @@ _CATEGORY_DEFAULTS: dict[CDCErrorCategory, tuple[str, bool]] = {"change data capture.",False,),+ CDCErrorCategory.POOLER_TENANT_NOT_FOUND: (+ "Your connection pooler couldn't find your database project or user. Check that the project "+ "is active and the pooler username is correct, then re-enable change data capture.",+ False,+ ),CDCErrorCategory.SSH_TUNNEL_FAILED: ("Could not connect to your SSH tunnel — PostHog couldn't open a session to the SSH gateway. ""Check that the SSH host and port point to a reachable SSH server (not the database port), "diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_errors.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_errors.pyindex 1eb3b244e9aa..5f22e67019dd 100644--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_errors.py+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_errors.py@@ -34,6 +34,7 @@ class TestCDCErrorInfo:(CDCErrorCategory.CONNECTION_FAILED, True),(CDCErrorCategory.HOST_UNREACHABLE, False),(CDCErrorCategory.QUOTA_EXCEEDED, False),+ (CDCErrorCategory.POOLER_TENANT_NOT_FOUND, False),(CDCErrorCategory.SLOT_MISSING, False),(CDCErrorCategory.SLOT_NOT_CONFIGURED, False),(CDCErrorCategory.PUBLICATION_MISSING, False),diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/errors.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/errors.pyindex 7aa339bee112..0e35b61b11eb 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/errors.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/errors.py@@ -66,6 +66,17 @@ _IP_NOT_ALLOWED_MARKERS = (# wall. Match the stable quota phrase; it names no volatile host/IP/port detail._QUOTA_EXCEEDED_MARKER = "exceeded the quota"+# A Supabase/Supavisor pooler that can't resolve the tenant behind the pooler username, in its+# older ("Tenant or user not found") and newer ("(ENOTFOUND) tenant/user <user> not found")+# wordings, or whose upstream DNS record is gone ("{:error, :nxdomain}"). The project is paused or+# deleted, or the pooler username is wrong, so every reconnect re-hits it. Mirrors the+# non-retryable treatment on the batch path (PostgresSource.get_non_retryable_errors).+_POOLER_TENANT_NOT_FOUND_MARKERS = (+ "tenant or user not found",+ "(enotfound) tenant/user",+ "{:error, :nxdomain}",+)+# sshtunnel raises BaseSSHTunnelForwarderError("Could not establish session to SSH gateway") when it# can't open a session to the bastion — the SSH host/port is wrong or unreachable, the bastion is# down, or its firewall blocks PostHog's IPs. Deterministic for the configured tunnel, so it's@@ -142,6 +153,8 @@ def classify_postgres_cdc_error(exc: BaseException) -> CDCErrorCategory | None:return CDCErrorCategory.HOST_UNREACHABLEif _QUOTA_EXCEEDED_MARKER in message:return CDCErrorCategory.QUOTA_EXCEEDED+ if any(marker in message for marker in _POOLER_TENANT_NOT_FOUND_MARKERS):+ return CDCErrorCategory.POOLER_TENANT_NOT_FOUND# psycopg raises ConnectionTimeout only from connect() itself, and the reader already# retries it in-process first (stream_reader._open_streaming_connection wraps every# connect in _connect_with_dropped_retry, which widens its predicate to connect-timediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/tests/test_errors.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/tests/test_errors.pyindex d41fcc467a26..ca202fb44854 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/tests/test_errors.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/tests/test_errors.py@@ -57,6 +57,24 @@ class TestClassifyPostgresCDCError:),CDCErrorCategory.QUOTA_EXCEEDED,),+ (+ "pooler_enotfound_tenant_is_non_retryable",+ psycopg.OperationalError(+ 'connection to server at "203.0.113.1", port 5432 failed: '+ "FATAL: (ENOTFOUND) tenant/user postgres.exampleref not found"+ ),+ CDCErrorCategory.POOLER_TENANT_NOT_FOUND,+ ),+ (+ "pooler_legacy_tenant_wording_is_non_retryable",+ psycopg.OperationalError("FATAL: Tenant or user not found"),+ CDCErrorCategory.POOLER_TENANT_NOT_FOUND,+ ),+ (+ "pooler_nxdomain_is_non_retryable",+ psycopg.OperationalError("FATAL: Failed to connect to database: {:error, :nxdomain}"),+ CDCErrorCategory.POOLER_TENANT_NOT_FOUND,+ ),("network_unreachable_is_non_retryable_host",psycopg.OperationalError(