Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(postgres): stop retrying cdc syncs whose pooler can't find the tenant (#117631)

Tom Owers22da9520cb4c
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/errors.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/errors.py
index 66c7cda9a8fa..c2f273438bfc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/errors.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/errors.py
@@ -28,6 +28,7 @@ class CDCErrorCategory(enum.StrEnum):
CONNECTION_FAILED = "connection_failed"
HOST_UNREACHABLE = "host_unreachable"
QUOTA_EXCEEDED = "quota_exceeded"
+ POOLER_TENANT_NOT_FOUND = "pooler_tenant_not_found"
SSH_TUNNEL_FAILED = "ssh_tunnel_failed"
SLOT_MISSING = "slot_missing"
SLOT_NOT_CONFIGURED = "slot_not_configured"
@@ -115,6 +116,11 @@ _CATEGORY_DEFAULTS: dict[CDCErrorCategory, tuple[str, bool]] = {
"change data capture.",
False,
),
+ CDCErrorCategory.POOLER_TENANT_NOT_FOUND: (
+ "Your connection pooler couldn't find your database project or user. Check that the project "
+ "is active and the pooler username is correct, then re-enable change data capture.",
+ False,
+ ),
CDCErrorCategory.SSH_TUNNEL_FAILED: (
"Could not connect to your SSH tunnel — PostHog couldn't open a session to the SSH gateway. "
"Check that the SSH host and port point to a reachable SSH server (not the database port), "
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_errors.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_errors.py
index 1eb3b244e9aa..5f22e67019dd 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_errors.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_errors.py
@@ -34,6 +34,7 @@ class TestCDCErrorInfo:
(CDCErrorCategory.CONNECTION_FAILED, True),
(CDCErrorCategory.HOST_UNREACHABLE, False),
(CDCErrorCategory.QUOTA_EXCEEDED, False),
+ (CDCErrorCategory.POOLER_TENANT_NOT_FOUND, False),
(CDCErrorCategory.SLOT_MISSING, False),
(CDCErrorCategory.SLOT_NOT_CONFIGURED, False),
(CDCErrorCategory.PUBLICATION_MISSING, False),
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/errors.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/errors.py
index 7aa339bee112..0e35b61b11eb 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/errors.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/errors.py
@@ -66,6 +66,17 @@ _IP_NOT_ALLOWED_MARKERS = (
# wall. Match the stable quota phrase; it names no volatile host/IP/port detail.
_QUOTA_EXCEEDED_MARKER = "exceeded the quota"
+# A Supabase/Supavisor pooler that can't resolve the tenant behind the pooler username, in its
+# older ("Tenant or user not found") and newer ("(ENOTFOUND) tenant/user <user> not found")
+# wordings, or whose upstream DNS record is gone ("{:error, :nxdomain}"). The project is paused or
+# deleted, or the pooler username is wrong, so every reconnect re-hits it. Mirrors the
+# non-retryable treatment on the batch path (PostgresSource.get_non_retryable_errors).
+_POOLER_TENANT_NOT_FOUND_MARKERS = (
+ "tenant or user not found",
+ "(enotfound) tenant/user",
+ "{:error, :nxdomain}",
+)
+
# sshtunnel raises BaseSSHTunnelForwarderError("Could not establish session to SSH gateway") when it
# can't open a session to the bastion — the SSH host/port is wrong or unreachable, the bastion is
# down, or its firewall blocks PostHog's IPs. Deterministic for the configured tunnel, so it's
@@ -142,6 +153,8 @@ def classify_postgres_cdc_error(exc: BaseException) -> CDCErrorCategory | None:
return CDCErrorCategory.HOST_UNREACHABLE
if _QUOTA_EXCEEDED_MARKER in message:
return CDCErrorCategory.QUOTA_EXCEEDED
+ if any(marker in message for marker in _POOLER_TENANT_NOT_FOUND_MARKERS):
+ return CDCErrorCategory.POOLER_TENANT_NOT_FOUND
# psycopg raises ConnectionTimeout only from connect() itself, and the reader already
# retries it in-process first (stream_reader._open_streaming_connection wraps every
# connect in _connect_with_dropped_retry, which widens its predicate to connect-time
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/tests/test_errors.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/tests/test_errors.py
index d41fcc467a26..ca202fb44854 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/tests/test_errors.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/tests/test_errors.py
@@ -57,6 +57,24 @@ class TestClassifyPostgresCDCError:
),
CDCErrorCategory.QUOTA_EXCEEDED,
),
+ (
+ "pooler_enotfound_tenant_is_non_retryable",
+ psycopg.OperationalError(
+ 'connection to server at "203.0.113.1", port 5432 failed: '
+ "FATAL: (ENOTFOUND) tenant/user postgres.exampleref not found"
+ ),
+ CDCErrorCategory.POOLER_TENANT_NOT_FOUND,
+ ),
+ (
+ "pooler_legacy_tenant_wording_is_non_retryable",
+ psycopg.OperationalError("FATAL: Tenant or user not found"),
+ CDCErrorCategory.POOLER_TENANT_NOT_FOUND,
+ ),
+ (
+ "pooler_nxdomain_is_non_retryable",
+ psycopg.OperationalError("FATAL: Failed to connect to database: {:error, :nxdomain}"),
+ CDCErrorCategory.POOLER_TENANT_NOT_FOUND,
+ ),
(
"network_unreachable_is_non_retryable_host",
psycopg.OperationalError(