:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(postgres): classify a failed SSH port forward as non-retryable (#123123)
Tom Owers86b784131ce3
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.pyindex 427d1a01f217..4d8332097868 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py@@ -413,11 +413,25 @@ _SSH_GATEWAY_UNREACHABLE_SYNC_MESSAGE = f"{_SSH_GATEWAY_UNREACHABLE_GUIDANCE}, t# sshtunnel raises this once the SSH session is up but no forwarded tunnel came up with it: the# bastion could not open a channel to the database host and port, or it refuses port forwarding# altogether. Its own wording names neither, so it leaves the user nothing to act on.+#+# Unlike `_SSH_GATEWAY_SESSION_ERROR` above (the bastion itself — usually static infrastructure),+# what sits on the other end of this channel is the customer's database: the same kind of endpoint+# `ForeignServerUnreachableError` and `_CONNECTION_DROPPED_ERROR_SUBSTRINGS` already treat as+# capable of a brief, self-recovering outage (a restart, a failover, a maintenance window).+# `_connected_ssh_tunnel`'s own ~14 second retry only rules out the shortest blips, so this stays+# retryable rather than disabling an otherwise healthy sync._SSH_FORWARD_FAILED_ERROR = "An error occurred while opening tunnels."-_SSH_FORWARD_FAILED_MESSAGE = (+_SSH_FORWARD_FAILED_GUIDANCE = ("PostHog signed in to your SSH server but couldn't reach the database through it. Check that ""the database host and port are reachable from the SSH server, and that the server allows port "- "forwarding."+ "forwarding"+)+_SSH_FORWARD_FAILED_MESSAGE = f"{_SSH_FORWARD_FAILED_GUIDANCE}."+# The schema stays enabled for this class, so this only replaces the raw driver text once every+# retry is spent — see `get_retry_exhausted_errors`.+_SSH_FORWARD_FAILED_EXHAUSTED_MESSAGE = (+ f"{_SSH_FORWARD_FAILED_GUIDANCE}, and this kept happening on every retry. This sync is still "+ "enabled and will run again on its next schedule.")# A source past the SSL cutoff connects with sslmode=require, so a server built without SSL support@@ -1663,6 +1677,10 @@ class PostgresSource(# connection mid-query, or an admin cancel — canceled it. Neither says anything about this# query or schema, and a retry runs on a fresh connection, so it's self-recovering like the# other entries here rather than a verdict to stop retrying.+ #+ # `_SSH_FORWARD_FAILED_ERROR` reaches here once `_connected_ssh_tunnel`'s own ~14 second+ # retry gives up opening the port forward to the database behind the bastion. See its+ # definition above for why that stays retryable rather than disabling the schema.return {CLIENT_DEADLINE_ERROR,"canceling statement due to user request",@@ -1673,6 +1691,7 @@ class PostgresSource("conflict with recovery",HOST_RESOLUTION_TIMEOUT_ERROR,TEMPORARY_HOST_RESOLUTION_ERROR,+ _SSH_FORWARD_FAILED_ERROR,}def get_retry_exhausted_errors(self) -> dict[str, str]:@@ -1695,6 +1714,7 @@ class PostgresSource("canceling statement due to user request": _SPURIOUS_CANCEL_EXHAUSTED_MESSAGE,HOST_RESOLUTION_TIMEOUT_ERROR: HOST_RESOLUTION_EXHAUSTED_MESSAGE,TEMPORARY_HOST_RESOLUTION_ERROR: HOST_RESOLUTION_EXHAUSTED_MESSAGE,+ _SSH_FORWARD_FAILED_ERROR: _SSH_FORWARD_FAILED_EXHAUSTED_MESSAGE,}def reconcile_schema_metadata(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.pyindex 9cafd330c234..1a0ad66c4906 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/test_postgres.py@@ -608,6 +608,27 @@ class TestPostgresSourceNonRetryableErrors:assert matches[0] is not None, "an unreachable SSH gateway must surface an actionable message"assert "re-enable the sync" in matches[0].lower()+ def test_ssh_forward_failure_stays_retryable_and_explains_once_exhausted(self, source):+ # Unlike the gateway-session error above, what sits behind this channel is the customer's+ # database, which can have a brief, self-recovering outage the same as a direct connection+ # can. `_connected_ssh_tunnel`'s own ~14 second retry only rules out the shortest blips, so+ # this must stay retryable rather than disable an otherwise healthy sync, and still explain+ # itself once the activity's own retries are spent.+ error_msg = "HandlerSSHTunnelForwarderError: An error occurred while opening tunnels."+ assert error_message_matches(error_msg, source.get_retryable_errors())+ assert not error_message_matches(error_msg, source.get_non_retryable_errors().keys())++ message = next(+ (+ friendly+ for pattern, friendly in source.get_retry_exhausted_errors().items()+ if error_message_matches(error_msg, [pattern])+ ),+ None,+ )+ assert message is not None, "a failed SSH port forward must surface an actionable message once exhausted"+ assert "still enabled" in message.lower()+@pytest.mark.parametrize(("error_msg", "reason_code", "expected_word"),[