Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(warehouse-sources): store ssh tunnel auth under one key on update (#125502)

Tom Owersadaa83326f0e
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_source/helpers.py b/products/warehouse_sources/backend/presentation/views/external_data_source/helpers.py
index a15374e4b534..ac37bbe6d5ba 100644
--- a/products/warehouse_sources/backend/presentation/views/external_data_source/helpers.py
+++ b/products/warehouse_sources/backend/presentation/views/external_data_source/helpers.py
@@ -3,12 +3,15 @@
from __future__ import annotations
import re
+import dataclasses
from collections.abc import Iterable, Mapping
from typing import TYPE_CHECKING, Any
from urllib.parse import unquote, urlsplit
from django.db.models import Q, QuerySet
+from rest_framework.exceptions import ValidationError
+
from posthog.dataclasses import frozen
from posthog.exceptions_capture import capture_exception
from posthog.ph_client import feature_enabled_or_false
@@ -616,6 +619,69 @@ def has_preserved_credentials(
return False
+# The SSH tunnel fields that pick which SSH server receives the tunnel's SSH credentials.
+_SSH_TUNNEL_ENDPOINT_FIELDS = ("host", "port")
+_SSH_TUNNEL_SECRET_FIELDS = ("password", "passphrase", "private_key")
+# Migration 0807 stored the tunnel auth under this key. The config parser reads it before
+# the canonical `auth` key, so a stale block under it shadows the auth an update supplies.
+_LEGACY_SSH_TUNNEL_AUTH_KEY = "auth_type"
+
+
+@frozen
+class _SshTunnelParts:
+ # The tunnel fields other than its auth: enabled, host, port, host_key and the rest.
+ settings: dict[str, Any]
+ # Read from `auth` or the legacy key. It holds the SSH secrets, so repr must not show it.
+ auth: dict[str, Any] = dataclasses.field(repr=False)
+
+
+def _split_ssh_tunnel_auth(tunnel: dict[str, Any], *, reject_conflict: bool) -> _SshTunnelParts:
+ auth_blocks = [tunnel[key] for key in ("auth", _LEGACY_SSH_TUNNEL_AUTH_KEY) if tunnel.get(key)]
+ if any(not isinstance(block, dict) for block in auth_blocks):
+ raise ValidationError({"job_inputs": {"ssh_tunnel": "SSH tunnel auth must be an object."}})
+ if reject_conflict and len(auth_blocks) == 2 and auth_blocks[0] != auth_blocks[1]:
+ raise ValidationError(
+ {"job_inputs": {"ssh_tunnel": "Send the SSH tunnel auth under `auth` only, not also under `auth_type`."}}
+ )
+ return _SshTunnelParts(
+ settings={key: value for key, value in tunnel.items() if key not in ("auth", _LEGACY_SSH_TUNNEL_AUTH_KEY)},
+ auth=auth_blocks[0] if auth_blocks else {},
+ )
+
+
+def merge_ssh_tunnel(existing: Any, incoming: Any) -> dict[str, Any]:
+ """Merge a partial SSH tunnel update onto the stored tunnel, with the auth under `auth` only.
+
+ The API response omits the SSH secrets, so an update that keeps the endpoint keeps any secret
+ it does not re-supply. An update that changes the host or port must supply a new SSH secret,
+ and no stored secret carries over. Otherwise the credential check would send the stored SSH
+ password to a server that the editor picked.
+ """
+ if not isinstance(incoming, dict):
+ raise ValidationError({"job_inputs": {"ssh_tunnel": "Must be an object."}})
+ incoming_parts = _split_ssh_tunnel_auth(incoming, reject_conflict=True)
+ if not isinstance(existing, dict) or not existing:
+ return (
+ {**incoming_parts.settings, "auth": incoming_parts.auth} if incoming_parts.auth else incoming_parts.settings
+ )
+
+ existing_parts = _split_ssh_tunnel_auth(existing, reject_conflict=False)
+ endpoint_changed = any(
+ key in incoming_parts.settings
+ and connection_target_changed(existing_parts.settings.get(key), incoming_parts.settings[key])
+ for key in _SSH_TUNNEL_ENDPOINT_FIELDS
+ )
+ if endpoint_changed and not any(incoming_parts.auth.get(key) for key in _SSH_TUNNEL_SECRET_FIELDS):
+ raise ValidationError("Changing the SSH tunnel host or port requires re-entering your SSH credentials.")
+
+ merged_auth = {**incoming_parts.auth} if incoming_parts.auth else {**existing_parts.auth}
+ if incoming_parts.auth and not endpoint_changed:
+ for key in _SSH_TUNNEL_SECRET_FIELDS:
+ if existing_parts.auth.get(key) and not incoming_parts.auth.get(key):
+ merged_auth[key] = existing_parts.auth[key]
+ return {**existing_parts.settings, **incoming_parts.settings, "auth": merged_auth}
+
+
def get_direct_connection_metadata(
*,
source_impl: Any,
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py b/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py
index 48f3285dd3bf..f360c4657e69 100644
--- a/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py
+++ b/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py
@@ -513,9 +513,6 @@ class ExternalDataSourceSerializers(UserAccessControlSerializerMixin, serializer
if existing_job_inputs.get(key) and not incoming_job_inputs.get(key):
new_job_inputs[key] = existing_job_inputs[key]
- # SSH tunnel is a nested config - deep-merge it so partial updates preserve existing fields
- existing_ssh_tunnel = existing_job_inputs.get("ssh_tunnel")
-
# Nested containers (e.g. Stripe `auth_method`, Snowflake `auth_type`, Billomat `registered_app`)
# need a deep-merge that preserves sensitive fields not explicitly provided. The shallow merge
# above would otherwise wipe redacted credentials nested inside these containers. Same container
@@ -565,40 +562,13 @@ class ExternalDataSourceSerializers(UserAccessControlSerializerMixin, serializer
new_job_inputs.pop(key, None)
new_job_inputs[incoming_key] = merged_group
+ # SSH tunnel is a nested config, so deep-merge it to keep the stored fields a partial update omits.
incoming_ssh_tunnel = incoming_job_inputs.get("ssh_tunnel")
- if existing_ssh_tunnel and incoming_ssh_tunnel is not None:
- ssh_tunnel_host_changed = "host" in incoming_ssh_tunnel and incoming_ssh_tunnel[
- "host"
- ] != existing_ssh_tunnel.get("host")
-
- # Deep-merge: start with existing, overlay incoming top-level keys
- merged_ssh_tunnel = {**existing_ssh_tunnel, **incoming_ssh_tunnel}
-
- # Check both 'auth' (new format) and 'auth_type' (legacy format from migration 0807)
- existing_auth = (
- (existing_ssh_tunnel or {}).get("auth") or (existing_ssh_tunnel or {}).get("auth_type") or {}
- )
- incoming_auth = (
- (incoming_ssh_tunnel or {}).get("auth") or (incoming_ssh_tunnel or {}).get("auth_type") or {}
+ if incoming_ssh_tunnel is not None:
+ new_job_inputs["ssh_tunnel"] = helpers.merge_ssh_tunnel(
+ existing_job_inputs.get("ssh_tunnel"), incoming_ssh_tunnel
)
- if ssh_tunnel_host_changed and not incoming_auth:
- raise ValidationError("Changing the SSH tunnel host requires re-entering your SSH credentials.")
-
- if not incoming_auth:
- # No auth in incoming request - preserve entire existing auth
- merged_ssh_tunnel["auth"] = {**existing_auth}
- else:
- # Merge auth, preserving sensitive fields not explicitly provided
- merged_auth = {**incoming_auth}
- if not ssh_tunnel_host_changed:
- for key in ("password", "passphrase", "private_key"):
- if existing_auth.get(key) and not incoming_auth.get(key):
- merged_auth[key] = existing_auth[key]
- merged_ssh_tunnel["auth"] = merged_auth
-
- new_job_inputs["ssh_tunnel"] = merged_ssh_tunnel
-
is_valid, errors = source.validate_config(new_job_inputs)
if not is_valid:
raise ValidationError(f"Invalid source config: {', '.join(errors)}")
diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_source.py b/products/warehouse_sources/backend/tests/api/test_external_data_source.py
index 729f57525e1c..883552c83881 100644
--- a/products/warehouse_sources/backend/tests/api/test_external_data_source.py
+++ b/products/warehouse_sources/backend/tests/api/test_external_data_source.py
@@ -74,6 +74,7 @@ from products.warehouse_sources.backend.presentation.views.external_data_source.
get_nonsensitive_and_sensitive_field_names,
get_oauth_integration_kinds,
has_preserved_credentials,
+ merge_ssh_tunnel,
restore_declared_field_names,
strip_sensitive_from_dict,
)
@@ -8640,11 +8641,20 @@ class TestExternalDataSource(APIBaseTest):
source.refresh_from_db()
assert source.job_inputs["ssh_tunnel"]["host"] == "ssh.example.com"
+ @parameterized.expand(
+ [
+ ("canonical_auth", "auth", {"type": "password"}),
+ # Migration 0807 wrote the auth under `auth_type`, keyed by `selection`, with blank unused secrets.
+ ("legacy_auth_type", "auth_type", {"selection": "password", "passphrase": "", "private_key": ""}),
+ ]
+ )
@patch(
"products.warehouse_sources.backend.temporal.data_imports.sources.postgres.source.PostgresSource.validate_credentials",
return_value=(True, None),
)
- def test_update_source_with_ssh_tunnel_host_change_and_auth_succeeds(self, mock_validate_credentials):
+ def test_update_source_with_ssh_tunnel_host_change_and_auth_succeeds(
+ self, _name: str, stored_auth_key: str, stored_auth: dict, mock_validate_credentials
+ ):
source = ExternalDataSource.objects.create(
team_id=self.team.pk,
source_id=str(uuid.uuid4()),
@@ -8665,8 +8675,8 @@ class TestExternalDataSource(APIBaseTest):
"enabled": "True",
"host": "ssh.example.com",
"port": "22",
- "auth": {
- "type": "password",
+ stored_auth_key: {
+ **stored_auth,
"username": "sshuser",
"password": "old_ssh_password",
},
@@ -8698,7 +8708,10 @@ class TestExternalDataSource(APIBaseTest):
source.refresh_from_db()
assert source.job_inputs["ssh_tunnel"]["host"] == "new-ssh.example.com"
assert source.job_inputs["ssh_tunnel"]["auth"]["password"] == "new_ssh_password"
+ assert "auth_type" not in source.job_inputs["ssh_tunnel"]
mock_validate_credentials.assert_called_once()
+ probed_tunnel = mock_validate_credentials.call_args.args[0].ssh_tunnel
+ assert (probed_tunnel.host, probed_tunnel.auth.password) == ("new-ssh.example.com", "new_ssh_password")
@patch(
"products.warehouse_sources.backend.temporal.data_imports.sources.postgres.source.PostgresSource.validate_credentials",
@@ -8738,12 +8751,11 @@ class TestExternalDataSource(APIBaseTest):
f"/api/environments/{self.team.pk}/external_data_sources/{source.pk}/",
data={
"job_inputs": {
- # Changing any SSH tunnel field forces re-entry of the database password.
- "password": "db_password",
+ "database": "renamed_db",
"ssh_tunnel": {
"enabled": True,
"host": "ssh.example.com",
- "port": 2222,
+ "port": 22,
},
},
},
@@ -14975,6 +14987,55 @@ class TestHasPreservedCredentials(SimpleTestCase):
assert preserved is True
+class TestMergeSshTunnel(SimpleTestCase):
+ STORED_LEGACY = {
+ "enabled": "True",
+ "host": "ssh.example.com",
+ "port": "22",
+ "auth_type": {"selection": "password", "username": "sshuser", "password": "stored_ssh_password"},
+ }
+
+ def test_retarget_with_new_auth_drops_the_legacy_auth(self) -> None:
+ merged = merge_ssh_tunnel(
+ self.STORED_LEGACY,
+ {"host": "new-ssh.example.com", "auth": {"selection": "password", "username": "u", "password": "new"}},
+ )
+
+ assert merged == {
+ "enabled": "True",
+ "host": "new-ssh.example.com",
+ "port": "22",
+ "auth": {"selection": "password", "username": "u", "password": "new"},
+ }
+
+ def test_same_endpoint_keeps_the_stored_secret_under_auth(self) -> None:
+ merged = merge_ssh_tunnel(
+ self.STORED_LEGACY, {"host": "ssh.example.com", "port": 22, "auth": {"selection": "password"}}
+ )
+
+ assert merged["auth"] == {"selection": "password", "password": "stored_ssh_password"}
+ assert "auth_type" not in merged
+
+ @parameterized.expand(
+ [
+ ("host_without_auth", {"host": "new-ssh.example.com"}),
+ ("host_with_auth_but_no_secret", {"host": "new-ssh.example.com", "auth": {"username": "sshuser"}}),
+ ("port_without_secret", {"port": 2222, "auth": {"selection": "password", "username": "sshuser"}}),
+ (
+ "conflicting_auth_and_auth_type",
+ {
+ "auth": {"selection": "password", "password": "new"},
+ "auth_type": {"selection": "password", "password": "other"},
+ },
+ ),
+ ("auth_not_an_object", {"auth": "password"}),
+ ]
+ )
+ def test_rejects_update(self, _name: str, incoming: dict) -> None:
+ with self.assertRaises(ValidationError):
+ merge_ssh_tunnel(self.STORED_LEGACY, incoming)
+
+
class TestGithubMultiRepoPatch(APIBaseTest):
def _create_github_source(self, job_inputs: dict) -> ExternalDataSource:
return ExternalDataSource.objects.create(