:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(postgres): translate ssh handshake eof in shared tunnel helper (#118217)
Tom Owersfcfa7fd3bbb2
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/mixins.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/mixins.pyindex d92160b2b0ea..0546ef39e777 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/mixins.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/mixins.py@@ -613,6 +613,15 @@ def _check_direct_host(config, team_id: int | None) -> None:# classification it reaches today, about 14 seconds later._SSH_TUNNEL_CONNECT_BACKOFF_SECONDS: tuple[float, ...] = (2.0, 4.0, 8.0)+# paramiko raises a bare, message-less EOFError when the SSH gateway accepts the TCP connection+# but closes it during the handshake itself — a non-SSH service on the port, a bastion refusing+# PostHog's IPs, or a proxy resetting the stream. sshtunnel doesn't wrap it (it only translates+# *auth* failures into BaseSSHTunnelForwarderError), so it escapes with an empty `str()`, matching+# no non-retryable rule and retrying forever. Translate it into this stable, classifiable message+# instead. Postgres registers it in `PostgresSource.get_non_retryable_errors`; other sources using+# this mixin at least get a readable message in place of a bare "EOFError".+SSH_HANDSHAKE_EOF_ERROR = "SSH gateway closed the connection during the SSH handshake"+def _release_failed_forwarder(forwarder: SSHTunnelForwarder) -> None:"""Release the sockets and threads a failed `start` left behind.@@ -673,6 +682,11 @@ def _connected_ssh_tunnel(ssh_tunnel: SSHTunnel, config, *, ssh_host: str) -> Ge)time.sleep(_SSH_TUNNEL_CONNECT_BACKOFF_SECONDS[attempt])continue+ except EOFError as e:+ # Not one of the retried gateway-connect failures above, and retrying a dead+ # handshake would just re-dial a port that isn't speaking SSH.+ _release_failed_forwarder(forwarder)+ raise Exception(SSH_HANDSHAKE_EOF_ERROR) from eyield tunnelreturndiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/test_mixins.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test_mixins.pyindex 5b9173bfe9cf..65d5d4e5596b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/test_mixins.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test_mixins.py@@ -23,6 +23,7 @@ from products.warehouse_sources.backend.temporal.data_imports.external_data_jobfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.base import error_message_matchesfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import (_SSH_TUNNEL_CONNECT_BACKOFF_SECONDS,+ SSH_HANDSHAKE_EOF_ERROR,HostNotAllowedError,OAuthMixin,SSHTunnelMixin,@@ -617,6 +618,28 @@ class TestSSHTunnelConnectRetry(SimpleTestCase):assert get_tunnel.call_count == attemptsassert mock_sleep.call_count == attempts - 1+ @parameterized.expand([("open_ssh_tunnel",), ("factory",)])+ def test_bare_handshake_eof_is_translated_and_not_retried(self, entrypoint: str):+ # paramiko raises a bare EOFError (no message) when the gateway closes the connection+ # during the handshake itself, instead of sshtunnel's own BaseSSHTunnelForwarderError.+ # Left uncaught, it escapes as an unclassifiable, message-less exception that retries+ # forever; it must come out translated, non-retried, and with the original as its cause.+ forwarder = self._forwarder(fails=True)+ forwarder.__enter__.side_effect = EOFError()+ with (+ patch(f"{_MIXINS_MODULE}.SSHTunnel") as mock_ssh,+ patch(f"{_MIXINS_MODULE}.time.sleep") as mock_sleep,+ patch(f"{_MIXINS_MODULE}.logger"),+ ):+ get_tunnel = mock_ssh.from_config.return_value.get_tunnel+ get_tunnel.return_value = forwarder+ with pytest.raises(Exception, match=SSH_HANDSHAKE_EOF_ERROR) as exc_info:+ with self._tunnel_cm(entrypoint, self._config()):+ pass+ assert isinstance(exc_info.value.__cause__, EOFError)+ assert get_tunnel.call_count == 1+ mock_sleep.assert_not_called()+@parameterized.expand([("open_ssh_tunnel",), ("factory",)])def test_failure_after_the_tunnel_is_up_does_not_reconnect(self, entrypoint: str):# Reopening here would restart a sync that had already begun reading rows.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.pyindex 6ec0ad0bfbd8..72a2a9eddf56 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py@@ -60,6 +60,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.helpers)from products.warehouse_sources.backend.temporal.data_imports.sources.common.cursor import SourceCursorManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import (+ SSH_HANDSHAKE_EOF_ERROR as _SSH_HANDSHAKE_EOF_ERROR,open_ssh_tunnel,pinned_host_kwargs,)@@ -3622,16 +3623,6 @@ def _get_table(return Table(name=table_name, parents=(schema,), columns=columns, type=table_type)-# paramiko raises a bare, message-less EOFError from `start_client` when the SSH gateway accepts-# the TCP connection but closes it during the SSH handshake — a non-SSH service on the port, a-# bastion refusing PostHog's IPs, or a proxy that resets the stream. sshtunnel doesn't wrap it (it-# only translates *auth* failures into BaseSSHTunnelForwarderError), so it escapes with an empty-# `str()`, matching no non-retryable rule and retrying forever. `_tunnel_with_handshake_translation`-# turns it into this stable, classifiable message (see `PostgresSource.get_non_retryable_errors`) —-# same gateway-configuration class as a wrapped "Could not establish session to SSH gateway".-_SSH_HANDSHAKE_EOF_ERROR = "SSH gateway closed the connection during the SSH handshake"--@contextmanagerdef _tunnel_with_handshake_translation(tunnel: Callable[[], _GeneratorContextManager[tuple[str, int]]],