Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(postgres): point ssh tunnel connect drops at the tunnel (#128151)

Tom Owers057dac496bb2
diff --git a/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py b/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py
index 72a4e7683d12..ec7ee36fa020 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/external_data_job.py
@@ -272,6 +272,17 @@ TRANSIENT_CONNECT_DROPPED_MESSAGE = (
"allows PostHog's IP addresses; the next sync runs on schedule."
)
+# A tunneled source connects to the local end of the SSH tunnel, which `SSHTunnel.get_tunnel` binds
+# to loopback, so libpq names that address instead of the customer's host. The local end accepts the
+# connection and closes it when the SSH server cannot reach the database behind it, so an IP
+# allowlist is not the cause. Wording mirrors `_SSH_FORWARD_FAILED_GUIDANCE` in the Postgres source.
+LIBPQ_SSH_TUNNEL_CONNECT_FAILURE_MARKER = f'{LIBPQ_CONNECT_FAILURE_MARKER} "127.0.0.1"'
+
+TRANSIENT_SSH_TUNNEL_CONNECT_DROPPED_MESSAGE = (
+ "Your SSH server couldn't reach your database through the tunnel. Check the database host and "
+ "port set for the tunnel; the next sync runs on schedule."
+)
+
TRANSIENT_POOLER_MESSAGE = (
"Your database's connection pooler couldn't reach your database and refused the connection. "
"Check that the database is running; the next sync runs on schedule."
@@ -478,6 +489,8 @@ def _transient_error_message(internal_error: str) -> str | None:
if message == TRANSIENT_SOURCE_CONNECTION_MESSAGE and error_message_matches(
internal_error, [LIBPQ_CONNECT_FAILURE_MARKER]
):
+ if error_message_matches(internal_error, [LIBPQ_SSH_TUNNEL_CONNECT_FAILURE_MARKER]):
+ return TRANSIENT_SSH_TUNNEL_CONNECT_DROPPED_MESSAGE
return TRANSIENT_CONNECT_DROPPED_MESSAGE
return message
diff --git a/products/warehouse_sources/backend/temporal/data_imports/tests/test_external_data_job_activities.py b/products/warehouse_sources/backend/temporal/data_imports/tests/test_external_data_job_activities.py
index 4f2882b9f627..a9a1b9f3049c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/tests/test_external_data_job_activities.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/tests/test_external_data_job_activities.py
@@ -26,6 +26,7 @@ from products.warehouse_sources.backend.temporal.data_imports.external_data_job
TRANSIENT_POOLER_MESSAGE,
TRANSIENT_SOURCE_CONNECTION_MESSAGE,
TRANSIENT_SOURCE_ERROR_MESSAGE,
+ TRANSIENT_SSH_TUNNEL_CONNECT_DROPPED_MESSAGE,
TRANSIENT_VENDOR_UNAVAILABLE_MESSAGE,
UNEXPECTED_ERROR_MESSAGE,
UpdateExternalDataJobStatusInputs,
@@ -400,6 +401,23 @@ def test_read_only_transaction_disables_the_schema_only_when_the_source_raised_i
"SSL SYSCALL error: EOF detected",
TRANSIENT_CONNECT_DROPPED_MESSAGE,
),
+ # An SSH-tunneled source connects to the local end of the tunnel, so libpq names loopback.
+ # The tunnel closes the connection when the SSH server cannot reach the database.
+ (
+ "postgres_ssh_tunnel_closed_at_connect",
+ ExternalDataSourceType.POSTGRES,
+ 'connection failed: connection to server at "127.0.0.1", port 40123 failed: '
+ "server closed the connection unexpectedly",
+ TRANSIENT_SSH_TUNNEL_CONNECT_DROPPED_MESSAGE,
+ ),
+ # A loopback host does not claim failures that are not a connect-time drop.
+ (
+ "postgres_ssh_tunnel_connection_limit",
+ ExternalDataSourceType.POSTGRES,
+ 'connection failed: connection to server at "127.0.0.1", port 40123 failed: '
+ "FATAL: sorry, too many clients already",
+ _CONNECTION_LIMIT_EXHAUSTED_MESSAGE,
+ ),
# The same drop on an established connection, which keeps the idle-connection copy.
(
"postgres_server_closed_mid_stream",