:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(batch-exports): check and pin databricks hosts at connect time (#122000)
Ross6b56ccf161ea
diff --git a/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py b/products/batch_exports/backend/temporal/destinations/databricks_batch_export.pyindex 3f77faefad6c..071d4355e37e 100644--- a/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py+++ b/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py@@ -5,6 +5,7 @@ import socketimport typing as timport asyncioimport datetime as dt+import ipaddressimport contextlibimport dataclassesimport collections.abc@@ -24,7 +25,15 @@ from structlog.contextvars import bind_contextvarsfrom temporalio import activity, exceptions, workflowfrom temporalio.common import RetryPolicy+from posthog.cloud_utils import is_cloudfrom posthog.models.integration import DatabricksIntegration, Integration+from posthog.security.oob_domains import matched_oob_domain+from posthog.security.url_validation import (+ INVALID_HOST_MESSAGE,+ UNREACHABLE_HOST_MESSAGE,+ ShapeError,+ validate_external_host,+)from posthog.temporal.common.base import PostHogWorkflowfrom posthog.temporal.common.heartbeat import Heartbeaterfrom posthog.temporal.common.logger import get_logger, get_write_only_logger@@ -319,11 +328,42 @@ class DatabricksClient:that case in ``connect_timeout_seconds`` and, unlike wrapping ``sql.connect`` in a timeout,leaves no stranded thread — the socket terminates on its own timeout and the SDK call is neverstarted.++ We also resolve and validate the hostname again here, and connect only to the validated+ addresses, to prevent SSRF through DNS rebinding."""def probe() -> None:- with socket.create_connection((self.server_hostname, 443), timeout=self.connect_timeout_seconds):- pass+ if is_cloud() and (oob_domain := matched_oob_domain(self.server_hostname)) is not None:+ # Detection only, so a match is logged and never refused. See posthog/security/crosslang/README.md.+ self.logger.info("security.oob_domain_seen", domain=oob_domain, guard="databricks_probe")++ addresses = [+ str(sockaddr[0])+ for *_, sockaddr in socket.getaddrinfo(self.server_hostname, 443, type=socket.SOCK_STREAM)+ ]+ try:+ validate_external_host(+ self.server_hostname, resolved_ips={ipaddress.ip_address(address) for address in addresses}+ )+ except ShapeError as err:+ raise DatabricksConnectionError(INVALID_HOST_MESSAGE, retryable=False) from err+ except ValueError as err:+ self.logger.warning(+ "Refused to connect to a Databricks host that failed the SSRF check",+ reason=str(err),+ addresses=addresses,+ )+ raise DatabricksConnectionError(UNREACHABLE_HOST_MESSAGE, retryable=False) from err++ errors: list[OSError] = []+ for address in addresses:+ try:+ with socket.create_connection((address, 443), timeout=self.connect_timeout_seconds):+ return+ except OSError as err:+ errors.append(err)+ raise errors[-1]try:await asyncio.to_thread(probe)diff --git a/products/batch_exports/backend/tests/temporal/destinations/databricks/test_client.py b/products/batch_exports/backend/tests/temporal/destinations/databricks/test_client.pyindex a749f281c415..107fb52f43c7 100644--- a/products/batch_exports/backend/tests/temporal/destinations/databricks/test_client.py+++ b/products/batch_exports/backend/tests/temporal/destinations/databricks/test_client.py@@ -204,15 +204,17 @@ class TestConnect:host becomes reachable, then complete the connection."""attempts = {"n": 0}- def reachability_result(*_args, **_kwargs):+ def resolution_result(*_args, **_kwargs):attempts["n"] += 1if attempts["n"] < 3:raise socket.gaierror(socket.EAI_AGAIN, "Temporary failure in name resolution")- return MagicMock()+ return [(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", ("8.8.8.8", 443))]- with patch(- "products.batch_exports.backend.temporal.destinations.databricks_batch_export.socket.create_connection",- side_effect=reachability_result,+ with (+ patch("socket.getaddrinfo", side_effect=resolution_result),+ patch(+ "products.batch_exports.backend.temporal.destinations.databricks_batch_export.socket.create_connection",+ ),):async with client.connect():pass@@ -237,19 +239,16 @@ class TestConnect:):"""A hostname that does not exist is a config error, so we should fail fast with a targetedmessage instead of backing off for minutes."""- create_connection_mock = MagicMock(side_effect=socket.gaierror(errno, "Name or service not known"))+ getaddrinfo_mock = MagicMock(side_effect=socket.gaierror(errno, "Name or service not known"))with (- patch(- "products.batch_exports.backend.temporal.destinations.databricks_batch_export.socket.create_connection",- new=create_connection_mock,- ),+ patch("socket.getaddrinfo", new=getaddrinfo_mock),pytest.raises(DatabricksConnectionError, match=expected_match),):async with client.connect():pass- assert create_connection_mock.call_count == 1+ assert getaddrinfo_mock.call_count == 1mock_sql_connect.sql_connect.assert_not_called()async def test_does_not_retry_invalid_credentials(@@ -268,6 +267,60 @@ class TestConnect:assert mock_sql_connect.sql_connect.call_count == 1+ async def test_probe_connects_to_the_address_it_checked(+ self, client: DatabricksClient, settings, mock_sql_connect: MockDatabricksSqlConnect+ ):+ settings.FORCE_URL_VALIDATION = True+ client.server_hostname = "dbc.example.com"+ addrinfo = [(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", ("8.8.8.8", 443))]++ with (+ patch("socket.getaddrinfo", return_value=addrinfo),+ patch(+ "products.batch_exports.backend.temporal.destinations.databricks_batch_export.socket.create_connection",+ ) as create_connection_mock,+ ):+ async with client.connect():+ pass++ create_connection_mock.assert_called_once_with(("8.8.8.8", 443), timeout=client.connect_timeout_seconds)+ mock_sql_connect.sql_connect.assert_called_once()++ @pytest.mark.parametrize(+ "addresses",+ [+ ["127.0.0.1"],+ ["169.254.169.254"],+ ["8.8.8.8", "10.0.0.5"],+ ],+ )+ async def test_probe_refuses_a_host_that_resolves_internal(+ self,+ client: DatabricksClient,+ settings,+ fast_backoff,+ mock_sql_connect: MockDatabricksSqlConnect,+ addresses: list[str],+ ):+ settings.FORCE_URL_VALIDATION = True+ client.server_hostname = "dbc.example.com"+ addrinfo = [+ (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", (address, 443)) for address in addresses+ ]++ with (+ patch("socket.getaddrinfo", return_value=addrinfo),+ patch(+ "products.batch_exports.backend.temporal.destinations.databricks_batch_export.socket.create_connection",+ ) as create_connection_mock,+ pytest.raises(DatabricksConnectionError, match="Could not reach this host"),+ ):+ async with client.connect():+ pass++ create_connection_mock.assert_not_called()+ mock_sql_connect.sql_connect.assert_not_called()+class TestQueryBuilders:@pytest.mark.parametrize("schema_evolution", [True, False])