:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(postgres): stop retrying a Neon SSH-tunnel SNI rejection (#123524)
Tom Owers4ce89a780049
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.pyindex 4d8332097868..5773fa0fbcec 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py@@ -267,6 +267,21 @@ PostgresErrors = {"IP address instead of a hostname. Update the host to the hostname your database ""provider gave you and try again."),+ # Twin of the `get_non_retryable_errors` (sync-time) key below: Neon's proxy routes a+ # connection using the Postgres "endpoint ID" carried in the TLS SNI extension, and SNI is+ # dropped whenever the host is dialed as a bare IP literal. `pinned_host_kwargs` preserves an+ # IP literal as-is, so this fires both for a host entered as a raw IP and for an SSH tunnel,+ # which always connects through its own loopback bind address. Map it here too so credential+ # validation returns an actionable message instead of capturing this expected user/upstream+ # condition.+ "Endpoint ID is not specified": (+ "Your database provider (Neon) couldn't route the connection because it didn't receive "+ 'routing information ("Endpoint ID is not specified"). This happens when the host is '+ "configured as an IP address instead of a hostname, or when connecting through an SSH "+ "tunnel, which can't carry that information to Neon. Update the host to the hostname your "+ "database provider gave you, or connect directly instead of through an SSH tunnel, then "+ "try again."+ ),# Some poolers (for example Supabase's transaction pooler on port 6543) reject bad credentials# during the SASL/SCRAM exchange with "FATAL: SASL authentication failed" instead of libpq's# "password authentication failed for user", so none of the password keys above substring-match@@ -843,6 +858,22 @@ class PostgresSource("configured as an IP address instead of a hostname. Update the host for this ""source to the hostname your database provider gave you, then re-enable the sync."),+ # Neon's proxy routes a connection using the Postgres "endpoint ID" carried in the TLS+ # SNI extension. SNI is dropped whenever the host is dialed as a bare IP literal.+ # `pinned_host_kwargs` preserves an IP literal as-is, so this fires both for a host+ # entered as a raw IP and for an SSH tunnel, which always connects through its own+ # loopback bind address (`_require_loopback` in mixins.py) regardless of the hostname+ # configured for the source. Deterministic for this host/transport combination, so+ # retrying just re-hits the same rejection. `PostgresErrors` already handles this on+ # the validation path; map it here too so a sync that hits it stops retrying.+ "Endpoint ID is not specified": (+ "Your database provider (Neon) couldn't route the connection because it didn't "+ 'receive routing information ("Endpoint ID is not specified"). This happens when '+ "the host is configured as an IP address instead of a hostname, or when connecting "+ "through an SSH tunnel, which can't carry that information to Neon. Update the "+ "host for this source to the hostname your database provider gave you, or connect "+ "directly instead of through an SSH tunnel, then re-enable the sync."+ ),"error received from server in SCRAM exchange: Wrong password": _INVALID_CREDENTIALS_ERROR,# The server (commonly Supabase's Supavisor transaction pooler on port 6543) rejects the# SASL/SCRAM credential exchange with "FATAL: SASL authentication failed" instead ofdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.pyindex 2503f0e07810..ac2ece4171c4 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py@@ -1363,6 +1363,24 @@ class TestPostgresSourceNonRetryableErrors:assert friendly, "SNI hostname routing rejection should surface an actionable message"assert "requires connecting via" in friendly[0]+ def test_neon_endpoint_id_not_specified_is_non_retryable(self, source):+ non_retryable = source.get_non_retryable_errors()+ error_msg = (+ 'connection failed: connection to server at "127.0.0.1", port 35975 failed: ERROR: '+ "Endpoint ID is not specified. Either please upgrade the postgres client library "+ "(libpq) for SNI support or pass the endpoint ID (first part of the domain name) as "+ "a parameter: '?options=endpoint%3D<endpoint-id>'. See more at https://neon.com/sni\n"+ 'connection to server at "127.0.0.1", port 35975 failed: ERROR: connection is '+ "insecure (try using `sslmode=require`)"+ )+ assert "Endpoint ID is not specified" in non_retryable+ is_non_retryable = any(pattern in error_msg for pattern in non_retryable.keys())+ assert is_non_retryable, f"Neon endpoint ID rejection should be non-retryable: {error_msg}"+ friendly = [reason for pattern, reason in non_retryable.items() if pattern in error_msg and reason]+ assert friendly, "Neon endpoint ID rejection should surface an actionable message"+ assert "SSH tunnel" in friendly[0]+ assert "hostname" in friendly[0]+def test_plaintext_refusal_alone_is_non_retryable(self, source):non_retryable = source.get_non_retryable_errors()error_msg = (@@ -5663,6 +5681,23 @@ class TestValidateCredentialsErrorMapping:"Could not connect to the database on the host and port given. Check the host and port are ""correct, and that PostHog's IP addresses are allowed through your firewall.",),+ # Neon's proxy routes by the "endpoint ID" carried in the TLS SNI extension, which is+ # dropped whenever the host is dialed as a bare IP literal — exactly what an SSH tunnel+ # does, since it always connects through its own loopback bind address.+ (+ 'connection failed: connection to server at "127.0.0.1", port 35975 failed: ERROR: '+ "Endpoint ID is not specified. Either please upgrade the postgres client library "+ "(libpq) for SNI support or pass the endpoint ID (first part of the domain name) as "+ "a parameter: '?options=endpoint%3D<endpoint-id>'. See more at https://neon.com/sni\n"+ 'connection to server at "127.0.0.1", port 35975 failed: ERROR: connection is '+ "insecure (try using `sslmode=require`)",+ "Your database provider (Neon) couldn't route the connection because it didn't "+ 'receive routing information ("Endpoint ID is not specified"). This happens when '+ "the host is configured as an IP address instead of a hostname, or when connecting "+ "through an SSH tunnel, which can't carry that information to Neon. Update the "+ "host to the hostname your database provider gave you, or connect directly instead "+ "of through an SSH tunnel, then try again.",+ ),# Unmapped errors fall back to the generic message.("some brand new failure",