Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(postgres): stop retrying a supavisor user that does not exist (#128120)

Tom Owersfbea56891371
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py
index 36fda6b2a774..664c9e03522d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py
@@ -865,6 +865,18 @@ class PostgresSource(
"deleted, or the pooler username/host is wrong. Check that your database is active "
"and the connection details are correct, then re-enable the sync."
),
+ # The sync-time twins of the permanent "(EAUTHQUERY)" outcomes in `PostgresErrors`. The
+ # transient "(eauthquery)" reconnect in postgres.py also matches them, so without these
+ # keys a dropped user retries on every run. Supavisor then trips its "(ECIRCUITBREAKER)"
+ # credential-fetch block, and the raw text leads with that line but still lists this one.
+ "user not found in the database": (
+ "Your database doesn't have the user this source connects as. Update the user for this "
+ "source, then re-enable the sync."
+ ),
+ "unsupported or invalid secret format": (
+ "Your connection pooler can't check this user's password because of how your database "
+ "stores it. Reset the user's password in your database, then re-enable the sync."
+ ),
# Supabase's Supavisor pooler reports a permanent DNS failure resolving its upstream
# backend as "FATAL: Failed to connect to database: {:error, :nxdomain}" — Erlang's
# resolver returning "no such domain", unlike the sibling ":etimedout"/":econnrefused"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py
index f867ce66fdfd..25bcdcb0d45c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py
@@ -468,6 +468,9 @@ class TestPostgresSourceNonRetryableErrors:
# Supavisor blocks a project for a short time after failed sign-ins from any client, so the
# block alone does not prove that this source's credentials are wrong.
'connection failed: connection to server at "10.0.0.1", port 5432 failed: FATAL: (ECIRCUITBREAKER) too many authentication failures, new connections are temporarily blocked',
+ # The one "(EAUTHQUERY)" outcome that clears on its own, so a key on the bare code must
+ # not disable it along with the missing-user and secret-format outcomes.
+ 'connection failed: connection to server at "10.0.0.1", port 5432 failed: FATAL: (EAUTHQUERY) auth_query secret check timed out',
],
)
def test_transient_connection_errors_are_retryable(self, source, error_msg):
@@ -528,6 +531,17 @@ class TestPostgresSourceNonRetryableErrors:
# so libpq can't load the file `tls_files` wrote for it. Permanent until the customer
# fixes the field. Host and temp path are invented, not real values.
'connection failed: connection to server at "203.0.113.7", port 5432 failed: could not read root certificate file "/tmp/source-tls-xxxxxxxx/ca.crt": no certificate or crl found',
+ # Supavisor's auth_query finds no such role. Repeated misses trip its credential-fetch
+ # circuit breaker, so the first address can report that block while a later one still
+ # names the missing user. Hosts and IPs are invented, not real values.
+ 'connection failed: connection to server at "203.0.113.20", port 5432 failed: FATAL: '
+ "(ECIRCUITBREAKER) failed to retrieve database credentials after multiple attempts, new "
+ "connections are temporarily blocked\n"
+ "Multiple connection attempts failed. All failures were:\n"
+ "- host: 'pooler.example.com', port: '5432', hostaddr: '203.0.113.21': connection failed: "
+ 'connection to server at "203.0.113.21", port 5432 failed: FATAL: (EAUTHQUERY) user not '
+ "found in the database",
+ 'connection failed: connection to server at "203.0.113.20", port 5432 failed: FATAL: (EAUTHQUERY) unsupported or invalid secret format',
],
)
def test_permanent_connection_errors_are_non_retryable(self, source, error_msg):