Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(postgres): translate ssh handshake eof in shared tunnel helper (#118217)

Tom Owersfcfa7fd3bbb2
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/mixins.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/mixins.py
index d92160b2b0ea..0546ef39e777 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/mixins.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/mixins.py
@@ -613,6 +613,15 @@ def _check_direct_host(config, team_id: int | None) -> None:
# classification it reaches today, about 14 seconds later.
_SSH_TUNNEL_CONNECT_BACKOFF_SECONDS: tuple[float, ...] = (2.0, 4.0, 8.0)
+# paramiko raises a bare, message-less EOFError when the SSH gateway accepts the TCP connection
+# but closes it during the handshake itself — a non-SSH service on the port, a bastion refusing
+# PostHog's IPs, or a proxy resetting the stream. sshtunnel doesn't wrap it (it only translates
+# *auth* failures into BaseSSHTunnelForwarderError), so it escapes with an empty `str()`, matching
+# no non-retryable rule and retrying forever. Translate it into this stable, classifiable message
+# instead. Postgres registers it in `PostgresSource.get_non_retryable_errors`; other sources using
+# this mixin at least get a readable message in place of a bare "EOFError".
+SSH_HANDSHAKE_EOF_ERROR = "SSH gateway closed the connection during the SSH handshake"
+
def _release_failed_forwarder(forwarder: SSHTunnelForwarder) -> None:
"""Release the sockets and threads a failed `start` left behind.
@@ -673,6 +682,11 @@ def _connected_ssh_tunnel(ssh_tunnel: SSHTunnel, config, *, ssh_host: str) -> Ge
)
time.sleep(_SSH_TUNNEL_CONNECT_BACKOFF_SECONDS[attempt])
continue
+ except EOFError as e:
+ # Not one of the retried gateway-connect failures above, and retrying a dead
+ # handshake would just re-dial a port that isn't speaking SSH.
+ _release_failed_forwarder(forwarder)
+ raise Exception(SSH_HANDSHAKE_EOF_ERROR) from e
yield tunnel
return
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/test_mixins.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test_mixins.py
index 5b9173bfe9cf..65d5d4e5596b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/test_mixins.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test_mixins.py
@@ -23,6 +23,7 @@ from products.warehouse_sources.backend.temporal.data_imports.external_data_job
from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import error_message_matches
from products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import (
_SSH_TUNNEL_CONNECT_BACKOFF_SECONDS,
+ SSH_HANDSHAKE_EOF_ERROR,
HostNotAllowedError,
OAuthMixin,
SSHTunnelMixin,
@@ -617,6 +618,28 @@ class TestSSHTunnelConnectRetry(SimpleTestCase):
assert get_tunnel.call_count == attempts
assert mock_sleep.call_count == attempts - 1
+ @parameterized.expand([("open_ssh_tunnel",), ("factory",)])
+ def test_bare_handshake_eof_is_translated_and_not_retried(self, entrypoint: str):
+ # paramiko raises a bare EOFError (no message) when the gateway closes the connection
+ # during the handshake itself, instead of sshtunnel's own BaseSSHTunnelForwarderError.
+ # Left uncaught, it escapes as an unclassifiable, message-less exception that retries
+ # forever; it must come out translated, non-retried, and with the original as its cause.
+ forwarder = self._forwarder(fails=True)
+ forwarder.__enter__.side_effect = EOFError()
+ with (
+ patch(f"{_MIXINS_MODULE}.SSHTunnel") as mock_ssh,
+ patch(f"{_MIXINS_MODULE}.time.sleep") as mock_sleep,
+ patch(f"{_MIXINS_MODULE}.logger"),
+ ):
+ get_tunnel = mock_ssh.from_config.return_value.get_tunnel
+ get_tunnel.return_value = forwarder
+ with pytest.raises(Exception, match=SSH_HANDSHAKE_EOF_ERROR) as exc_info:
+ with self._tunnel_cm(entrypoint, self._config()):
+ pass
+ assert isinstance(exc_info.value.__cause__, EOFError)
+ assert get_tunnel.call_count == 1
+ mock_sleep.assert_not_called()
+
@parameterized.expand([("open_ssh_tunnel",), ("factory",)])
def test_failure_after_the_tunnel_is_up_does_not_reconnect(self, entrypoint: str):
# Reopening here would restart a sync that had already begun reading rows.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py
index 6ec0ad0bfbd8..72a2a9eddf56 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py
@@ -60,6 +60,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.helpers
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.cursor import SourceCursorManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import (
+ SSH_HANDSHAKE_EOF_ERROR as _SSH_HANDSHAKE_EOF_ERROR,
open_ssh_tunnel,
pinned_host_kwargs,
)
@@ -3622,16 +3623,6 @@ def _get_table(
return Table(name=table_name, parents=(schema,), columns=columns, type=table_type)
-# paramiko raises a bare, message-less EOFError from `start_client` when the SSH gateway accepts
-# the TCP connection but closes it during the SSH handshake — a non-SSH service on the port, a
-# bastion refusing PostHog's IPs, or a proxy that resets the stream. sshtunnel doesn't wrap it (it
-# only translates *auth* failures into BaseSSHTunnelForwarderError), so it escapes with an empty
-# `str()`, matching no non-retryable rule and retrying forever. `_tunnel_with_handshake_translation`
-# turns it into this stable, classifiable message (see `PostgresSource.get_non_retryable_errors`) —
-# same gateway-configuration class as a wrapped "Could not establish session to SSH gateway".
-_SSH_HANDSHAKE_EOF_ERROR = "SSH gateway closed the connection during the SSH handshake"
-
-
@contextmanager
def _tunnel_with_handshake_translation(
tunnel: Callable[[], _GeneratorContextManager[tuple[str, int]]],