Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(postgres): stop retrying a Neon SSH-tunnel SNI rejection (#123524)

Tom Owers4ce89a780049
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py
index 4d8332097868..5773fa0fbcec 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py
@@ -267,6 +267,21 @@ PostgresErrors = {
"IP address instead of a hostname. Update the host to the hostname your database "
"provider gave you and try again."
),
+ # Twin of the `get_non_retryable_errors` (sync-time) key below: Neon's proxy routes a
+ # connection using the Postgres "endpoint ID" carried in the TLS SNI extension, and SNI is
+ # dropped whenever the host is dialed as a bare IP literal. `pinned_host_kwargs` preserves an
+ # IP literal as-is, so this fires both for a host entered as a raw IP and for an SSH tunnel,
+ # which always connects through its own loopback bind address. Map it here too so credential
+ # validation returns an actionable message instead of capturing this expected user/upstream
+ # condition.
+ "Endpoint ID is not specified": (
+ "Your database provider (Neon) couldn't route the connection because it didn't receive "
+ 'routing information ("Endpoint ID is not specified"). This happens when the host is '
+ "configured as an IP address instead of a hostname, or when connecting through an SSH "
+ "tunnel, which can't carry that information to Neon. Update the host to the hostname your "
+ "database provider gave you, or connect directly instead of through an SSH tunnel, then "
+ "try again."
+ ),
# Some poolers (for example Supabase's transaction pooler on port 6543) reject bad credentials
# during the SASL/SCRAM exchange with "FATAL: SASL authentication failed" instead of libpq's
# "password authentication failed for user", so none of the password keys above substring-match
@@ -843,6 +858,22 @@ class PostgresSource(
"configured as an IP address instead of a hostname. Update the host for this "
"source to the hostname your database provider gave you, then re-enable the sync."
),
+ # Neon's proxy routes a connection using the Postgres "endpoint ID" carried in the TLS
+ # SNI extension. SNI is dropped whenever the host is dialed as a bare IP literal.
+ # `pinned_host_kwargs` preserves an IP literal as-is, so this fires both for a host
+ # entered as a raw IP and for an SSH tunnel, which always connects through its own
+ # loopback bind address (`_require_loopback` in mixins.py) regardless of the hostname
+ # configured for the source. Deterministic for this host/transport combination, so
+ # retrying just re-hits the same rejection. `PostgresErrors` already handles this on
+ # the validation path; map it here too so a sync that hits it stops retrying.
+ "Endpoint ID is not specified": (
+ "Your database provider (Neon) couldn't route the connection because it didn't "
+ 'receive routing information ("Endpoint ID is not specified"). This happens when '
+ "the host is configured as an IP address instead of a hostname, or when connecting "
+ "through an SSH tunnel, which can't carry that information to Neon. Update the "
+ "host for this source to the hostname your database provider gave you, or connect "
+ "directly instead of through an SSH tunnel, then re-enable the sync."
+ ),
"error received from server in SCRAM exchange: Wrong password": _INVALID_CREDENTIALS_ERROR,
# The server (commonly Supabase's Supavisor transaction pooler on port 6543) rejects the
# SASL/SCRAM credential exchange with "FATAL: SASL authentication failed" instead of
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py
index 2503f0e07810..ac2ece4171c4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py
@@ -1363,6 +1363,24 @@ class TestPostgresSourceNonRetryableErrors:
assert friendly, "SNI hostname routing rejection should surface an actionable message"
assert "requires connecting via" in friendly[0]
+ def test_neon_endpoint_id_not_specified_is_non_retryable(self, source):
+ non_retryable = source.get_non_retryable_errors()
+ error_msg = (
+ 'connection failed: connection to server at "127.0.0.1", port 35975 failed: ERROR: '
+ "Endpoint ID is not specified. Either please upgrade the postgres client library "
+ "(libpq) for SNI support or pass the endpoint ID (first part of the domain name) as "
+ "a parameter: '?options=endpoint%3D<endpoint-id>'. See more at https://neon.com/sni\n"
+ 'connection to server at "127.0.0.1", port 35975 failed: ERROR: connection is '
+ "insecure (try using `sslmode=require`)"
+ )
+ assert "Endpoint ID is not specified" in non_retryable
+ is_non_retryable = any(pattern in error_msg for pattern in non_retryable.keys())
+ assert is_non_retryable, f"Neon endpoint ID rejection should be non-retryable: {error_msg}"
+ friendly = [reason for pattern, reason in non_retryable.items() if pattern in error_msg and reason]
+ assert friendly, "Neon endpoint ID rejection should surface an actionable message"
+ assert "SSH tunnel" in friendly[0]
+ assert "hostname" in friendly[0]
+
def test_plaintext_refusal_alone_is_non_retryable(self, source):
non_retryable = source.get_non_retryable_errors()
error_msg = (
@@ -5663,6 +5681,23 @@ class TestValidateCredentialsErrorMapping:
"Could not connect to the database on the host and port given. Check the host and port are "
"correct, and that PostHog's IP addresses are allowed through your firewall.",
),
+ # Neon's proxy routes by the "endpoint ID" carried in the TLS SNI extension, which is
+ # dropped whenever the host is dialed as a bare IP literal — exactly what an SSH tunnel
+ # does, since it always connects through its own loopback bind address.
+ (
+ 'connection failed: connection to server at "127.0.0.1", port 35975 failed: ERROR: '
+ "Endpoint ID is not specified. Either please upgrade the postgres client library "
+ "(libpq) for SNI support or pass the endpoint ID (first part of the domain name) as "
+ "a parameter: '?options=endpoint%3D<endpoint-id>'. See more at https://neon.com/sni\n"
+ 'connection to server at "127.0.0.1", port 35975 failed: ERROR: connection is '
+ "insecure (try using `sslmode=require`)",
+ "Your database provider (Neon) couldn't route the connection because it didn't "
+ 'receive routing information ("Endpoint ID is not specified"). This happens when '
+ "the host is configured as an IP address instead of a hostname, or when connecting "
+ "through an SSH tunnel, which can't carry that information to Neon. Update the "
+ "host to the hostname your database provider gave you, or connect directly instead "
+ "of through an SSH tunnel, then try again.",
+ ),
# Unmapped errors fall back to the generic message.
(
"some brand new failure",