:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(warehouse-sources): store ssh tunnel auth under one key on update (#125502)
Tom Owersadaa83326f0e
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_source/helpers.py b/products/warehouse_sources/backend/presentation/views/external_data_source/helpers.pyindex a15374e4b534..ac37bbe6d5ba 100644--- a/products/warehouse_sources/backend/presentation/views/external_data_source/helpers.py+++ b/products/warehouse_sources/backend/presentation/views/external_data_source/helpers.py@@ -3,12 +3,15 @@from __future__ import annotationsimport re+import dataclassesfrom collections.abc import Iterable, Mappingfrom typing import TYPE_CHECKING, Anyfrom urllib.parse import unquote, urlsplitfrom django.db.models import Q, QuerySet+from rest_framework.exceptions import ValidationError+from posthog.dataclasses import frozenfrom posthog.exceptions_capture import capture_exceptionfrom posthog.ph_client import feature_enabled_or_false@@ -616,6 +619,69 @@ def has_preserved_credentials(return False+# The SSH tunnel fields that pick which SSH server receives the tunnel's SSH credentials.+_SSH_TUNNEL_ENDPOINT_FIELDS = ("host", "port")+_SSH_TUNNEL_SECRET_FIELDS = ("password", "passphrase", "private_key")+# Migration 0807 stored the tunnel auth under this key. The config parser reads it before+# the canonical `auth` key, so a stale block under it shadows the auth an update supplies.+_LEGACY_SSH_TUNNEL_AUTH_KEY = "auth_type"+++@frozen+class _SshTunnelParts:+ # The tunnel fields other than its auth: enabled, host, port, host_key and the rest.+ settings: dict[str, Any]+ # Read from `auth` or the legacy key. It holds the SSH secrets, so repr must not show it.+ auth: dict[str, Any] = dataclasses.field(repr=False)+++def _split_ssh_tunnel_auth(tunnel: dict[str, Any], *, reject_conflict: bool) -> _SshTunnelParts:+ auth_blocks = [tunnel[key] for key in ("auth", _LEGACY_SSH_TUNNEL_AUTH_KEY) if tunnel.get(key)]+ if any(not isinstance(block, dict) for block in auth_blocks):+ raise ValidationError({"job_inputs": {"ssh_tunnel": "SSH tunnel auth must be an object."}})+ if reject_conflict and len(auth_blocks) == 2 and auth_blocks[0] != auth_blocks[1]:+ raise ValidationError(+ {"job_inputs": {"ssh_tunnel": "Send the SSH tunnel auth under `auth` only, not also under `auth_type`."}}+ )+ return _SshTunnelParts(+ settings={key: value for key, value in tunnel.items() if key not in ("auth", _LEGACY_SSH_TUNNEL_AUTH_KEY)},+ auth=auth_blocks[0] if auth_blocks else {},+ )+++def merge_ssh_tunnel(existing: Any, incoming: Any) -> dict[str, Any]:+ """Merge a partial SSH tunnel update onto the stored tunnel, with the auth under `auth` only.++ The API response omits the SSH secrets, so an update that keeps the endpoint keeps any secret+ it does not re-supply. An update that changes the host or port must supply a new SSH secret,+ and no stored secret carries over. Otherwise the credential check would send the stored SSH+ password to a server that the editor picked.+ """+ if not isinstance(incoming, dict):+ raise ValidationError({"job_inputs": {"ssh_tunnel": "Must be an object."}})+ incoming_parts = _split_ssh_tunnel_auth(incoming, reject_conflict=True)+ if not isinstance(existing, dict) or not existing:+ return (+ {**incoming_parts.settings, "auth": incoming_parts.auth} if incoming_parts.auth else incoming_parts.settings+ )++ existing_parts = _split_ssh_tunnel_auth(existing, reject_conflict=False)+ endpoint_changed = any(+ key in incoming_parts.settings+ and connection_target_changed(existing_parts.settings.get(key), incoming_parts.settings[key])+ for key in _SSH_TUNNEL_ENDPOINT_FIELDS+ )+ if endpoint_changed and not any(incoming_parts.auth.get(key) for key in _SSH_TUNNEL_SECRET_FIELDS):+ raise ValidationError("Changing the SSH tunnel host or port requires re-entering your SSH credentials.")++ merged_auth = {**incoming_parts.auth} if incoming_parts.auth else {**existing_parts.auth}+ if incoming_parts.auth and not endpoint_changed:+ for key in _SSH_TUNNEL_SECRET_FIELDS:+ if existing_parts.auth.get(key) and not incoming_parts.auth.get(key):+ merged_auth[key] = existing_parts.auth[key]+ return {**existing_parts.settings, **incoming_parts.settings, "auth": merged_auth}++def get_direct_connection_metadata(*,source_impl: Any,diff --git a/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py b/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.pyindex 48f3285dd3bf..f360c4657e69 100644--- a/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py+++ b/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py@@ -513,9 +513,6 @@ class ExternalDataSourceSerializers(UserAccessControlSerializerMixin, serializerif existing_job_inputs.get(key) and not incoming_job_inputs.get(key):new_job_inputs[key] = existing_job_inputs[key]- # SSH tunnel is a nested config - deep-merge it so partial updates preserve existing fields- existing_ssh_tunnel = existing_job_inputs.get("ssh_tunnel")-# Nested containers (e.g. Stripe `auth_method`, Snowflake `auth_type`, Billomat `registered_app`)# need a deep-merge that preserves sensitive fields not explicitly provided. The shallow merge# above would otherwise wipe redacted credentials nested inside these containers. Same container@@ -565,40 +562,13 @@ class ExternalDataSourceSerializers(UserAccessControlSerializerMixin, serializernew_job_inputs.pop(key, None)new_job_inputs[incoming_key] = merged_group+ # SSH tunnel is a nested config, so deep-merge it to keep the stored fields a partial update omits.incoming_ssh_tunnel = incoming_job_inputs.get("ssh_tunnel")- if existing_ssh_tunnel and incoming_ssh_tunnel is not None:- ssh_tunnel_host_changed = "host" in incoming_ssh_tunnel and incoming_ssh_tunnel[- "host"- ] != existing_ssh_tunnel.get("host")-- # Deep-merge: start with existing, overlay incoming top-level keys- merged_ssh_tunnel = {**existing_ssh_tunnel, **incoming_ssh_tunnel}-- # Check both 'auth' (new format) and 'auth_type' (legacy format from migration 0807)- existing_auth = (- (existing_ssh_tunnel or {}).get("auth") or (existing_ssh_tunnel or {}).get("auth_type") or {}- )- incoming_auth = (- (incoming_ssh_tunnel or {}).get("auth") or (incoming_ssh_tunnel or {}).get("auth_type") or {}+ if incoming_ssh_tunnel is not None:+ new_job_inputs["ssh_tunnel"] = helpers.merge_ssh_tunnel(+ existing_job_inputs.get("ssh_tunnel"), incoming_ssh_tunnel)- if ssh_tunnel_host_changed and not incoming_auth:- raise ValidationError("Changing the SSH tunnel host requires re-entering your SSH credentials.")-- if not incoming_auth:- # No auth in incoming request - preserve entire existing auth- merged_ssh_tunnel["auth"] = {**existing_auth}- else:- # Merge auth, preserving sensitive fields not explicitly provided- merged_auth = {**incoming_auth}- if not ssh_tunnel_host_changed:- for key in ("password", "passphrase", "private_key"):- if existing_auth.get(key) and not incoming_auth.get(key):- merged_auth[key] = existing_auth[key]- merged_ssh_tunnel["auth"] = merged_auth-- new_job_inputs["ssh_tunnel"] = merged_ssh_tunnel-is_valid, errors = source.validate_config(new_job_inputs)if not is_valid:raise ValidationError(f"Invalid source config: {', '.join(errors)}")diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_source.py b/products/warehouse_sources/backend/tests/api/test_external_data_source.pyindex 729f57525e1c..883552c83881 100644--- a/products/warehouse_sources/backend/tests/api/test_external_data_source.py+++ b/products/warehouse_sources/backend/tests/api/test_external_data_source.py@@ -74,6 +74,7 @@ from products.warehouse_sources.backend.presentation.views.external_data_source.get_nonsensitive_and_sensitive_field_names,get_oauth_integration_kinds,has_preserved_credentials,+ merge_ssh_tunnel,restore_declared_field_names,strip_sensitive_from_dict,)@@ -8640,11 +8641,20 @@ class TestExternalDataSource(APIBaseTest):source.refresh_from_db()assert source.job_inputs["ssh_tunnel"]["host"] == "ssh.example.com"+ @parameterized.expand(+ [+ ("canonical_auth", "auth", {"type": "password"}),+ # Migration 0807 wrote the auth under `auth_type`, keyed by `selection`, with blank unused secrets.+ ("legacy_auth_type", "auth_type", {"selection": "password", "passphrase": "", "private_key": ""}),+ ]+ )@patch("products.warehouse_sources.backend.temporal.data_imports.sources.postgres.source.PostgresSource.validate_credentials",return_value=(True, None),)- def test_update_source_with_ssh_tunnel_host_change_and_auth_succeeds(self, mock_validate_credentials):+ def test_update_source_with_ssh_tunnel_host_change_and_auth_succeeds(+ self, _name: str, stored_auth_key: str, stored_auth: dict, mock_validate_credentials+ ):source = ExternalDataSource.objects.create(team_id=self.team.pk,source_id=str(uuid.uuid4()),@@ -8665,8 +8675,8 @@ class TestExternalDataSource(APIBaseTest):"enabled": "True","host": "ssh.example.com","port": "22",- "auth": {- "type": "password",+ stored_auth_key: {+ **stored_auth,"username": "sshuser","password": "old_ssh_password",},@@ -8698,7 +8708,10 @@ class TestExternalDataSource(APIBaseTest):source.refresh_from_db()assert source.job_inputs["ssh_tunnel"]["host"] == "new-ssh.example.com"assert source.job_inputs["ssh_tunnel"]["auth"]["password"] == "new_ssh_password"+ assert "auth_type" not in source.job_inputs["ssh_tunnel"]mock_validate_credentials.assert_called_once()+ probed_tunnel = mock_validate_credentials.call_args.args[0].ssh_tunnel+ assert (probed_tunnel.host, probed_tunnel.auth.password) == ("new-ssh.example.com", "new_ssh_password")@patch("products.warehouse_sources.backend.temporal.data_imports.sources.postgres.source.PostgresSource.validate_credentials",@@ -8738,12 +8751,11 @@ class TestExternalDataSource(APIBaseTest):f"/api/environments/{self.team.pk}/external_data_sources/{source.pk}/",data={"job_inputs": {- # Changing any SSH tunnel field forces re-entry of the database password.- "password": "db_password",+ "database": "renamed_db","ssh_tunnel": {"enabled": True,"host": "ssh.example.com",- "port": 2222,+ "port": 22,},},},@@ -14975,6 +14987,55 @@ class TestHasPreservedCredentials(SimpleTestCase):assert preserved is True+class TestMergeSshTunnel(SimpleTestCase):+ STORED_LEGACY = {+ "enabled": "True",+ "host": "ssh.example.com",+ "port": "22",+ "auth_type": {"selection": "password", "username": "sshuser", "password": "stored_ssh_password"},+ }++ def test_retarget_with_new_auth_drops_the_legacy_auth(self) -> None:+ merged = merge_ssh_tunnel(+ self.STORED_LEGACY,+ {"host": "new-ssh.example.com", "auth": {"selection": "password", "username": "u", "password": "new"}},+ )++ assert merged == {+ "enabled": "True",+ "host": "new-ssh.example.com",+ "port": "22",+ "auth": {"selection": "password", "username": "u", "password": "new"},+ }++ def test_same_endpoint_keeps_the_stored_secret_under_auth(self) -> None:+ merged = merge_ssh_tunnel(+ self.STORED_LEGACY, {"host": "ssh.example.com", "port": 22, "auth": {"selection": "password"}}+ )++ assert merged["auth"] == {"selection": "password", "password": "stored_ssh_password"}+ assert "auth_type" not in merged++ @parameterized.expand(+ [+ ("host_without_auth", {"host": "new-ssh.example.com"}),+ ("host_with_auth_but_no_secret", {"host": "new-ssh.example.com", "auth": {"username": "sshuser"}}),+ ("port_without_secret", {"port": 2222, "auth": {"selection": "password", "username": "sshuser"}}),+ (+ "conflicting_auth_and_auth_type",+ {+ "auth": {"selection": "password", "password": "new"},+ "auth_type": {"selection": "password", "password": "other"},+ },+ ),+ ("auth_not_an_object", {"auth": "password"}),+ ]+ )+ def test_rejects_update(self, _name: str, incoming: dict) -> None:+ with self.assertRaises(ValidationError):+ merge_ssh_tunnel(self.STORED_LEGACY, incoming)++class TestGithubMultiRepoPatch(APIBaseTest):def _create_github_source(self, job_inputs: dict) -> ExternalDataSource:return ExternalDataSource.objects.create(