Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(postgres): retry transient proxy-to-backend connect failures (#119961)

Tom Owers494cb1136b58
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py
index 1e44a9452d7f..2ce0f5314313 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py
@@ -383,6 +383,15 @@ _CONNECTION_DROPPED_ERROR_SUBSTRINGS = (
# "no error details available" suffix is the signal — libpq only omits the message in this local,
# pre-handshake case, so it never matches a genuine (and permanent) server-reported rejection.
"connection is bad: no error details available",
+ # A TCP proxy/load balancer fronting the actual Postgres backend (common on providers that put
+ # a public endpoint in front of a private-network database) reports a failed connect to its own
+ # upstream as a bare FATAL error: "FATAL: failed to connect to upstream <ip>:<port>". The proxy
+ # itself answered fine; the backend behind it was momentarily unreachable (a restart, failover,
+ # or scaling event on the provider's private network) — the same transient class as Supavisor's
+ # "{:error, :econnrefused}"/"{:error, :etimedout}" above, just worded by a different proxy. A
+ # fresh reconnect recovers once the backend is reachable again. Match the stable phrase and
+ # exclude the volatile upstream ip:port.
+ "failed to connect to upstream",
)
# Supavisor (Supabase's connection pooler) doesn't surface a dropped upstream connection with a
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py
index 536747951357..293b4ad70fdf 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py
@@ -2577,6 +2577,14 @@ class TestIsConnectionDroppedError:
'to server at "safekeeper-1.cell-1.us-east-1.aws.neon.tech" (203.0.113.10), port 6401 '
"failed: server closed the connection unexpectedly"
),
+ # A TCP proxy fronting the actual backend (a public endpoint over a private-network
+ # database) reports a failed connect to its own upstream as a bare FATAL error. The
+ # proxy itself answered; the backend behind it was momentarily unreachable. Transient —
+ # the reconnect must catch it rather than letting it escape and fail the whole sync.
+ psycopg.OperationalError(
+ 'connection failed: connection to server at "203.0.113.5", port 5432 failed: '
+ "FATAL: failed to connect to upstream 10.0.0.2:5432"
+ ),
],
)
def test_connection_dropped_errors_are_detected(self, error):