Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(batch-exports): check and pin databricks hosts at connect time (#122000)

Ross6b56ccf161ea
diff --git a/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py b/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py
index 3f77faefad6c..071d4355e37e 100644
--- a/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py
+++ b/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py
@@ -5,6 +5,7 @@ import socket
import typing as t
import asyncio
import datetime as dt
+import ipaddress
import contextlib
import dataclasses
import collections.abc
@@ -24,7 +25,15 @@ from structlog.contextvars import bind_contextvars
from temporalio import activity, exceptions, workflow
from temporalio.common import RetryPolicy
+from posthog.cloud_utils import is_cloud
from posthog.models.integration import DatabricksIntegration, Integration
+from posthog.security.oob_domains import matched_oob_domain
+from posthog.security.url_validation import (
+ INVALID_HOST_MESSAGE,
+ UNREACHABLE_HOST_MESSAGE,
+ ShapeError,
+ validate_external_host,
+)
from posthog.temporal.common.base import PostHogWorkflow
from posthog.temporal.common.heartbeat import Heartbeater
from posthog.temporal.common.logger import get_logger, get_write_only_logger
@@ -319,11 +328,42 @@ class DatabricksClient:
that case in ``connect_timeout_seconds`` and, unlike wrapping ``sql.connect`` in a timeout,
leaves no stranded thread — the socket terminates on its own timeout and the SDK call is never
started.
+
+ We also resolve and validate the hostname again here, and connect only to the validated
+ addresses, to prevent SSRF through DNS rebinding.
"""
def probe() -> None:
- with socket.create_connection((self.server_hostname, 443), timeout=self.connect_timeout_seconds):
- pass
+ if is_cloud() and (oob_domain := matched_oob_domain(self.server_hostname)) is not None:
+ # Detection only, so a match is logged and never refused. See posthog/security/crosslang/README.md.
+ self.logger.info("security.oob_domain_seen", domain=oob_domain, guard="databricks_probe")
+
+ addresses = [
+ str(sockaddr[0])
+ for *_, sockaddr in socket.getaddrinfo(self.server_hostname, 443, type=socket.SOCK_STREAM)
+ ]
+ try:
+ validate_external_host(
+ self.server_hostname, resolved_ips={ipaddress.ip_address(address) for address in addresses}
+ )
+ except ShapeError as err:
+ raise DatabricksConnectionError(INVALID_HOST_MESSAGE, retryable=False) from err
+ except ValueError as err:
+ self.logger.warning(
+ "Refused to connect to a Databricks host that failed the SSRF check",
+ reason=str(err),
+ addresses=addresses,
+ )
+ raise DatabricksConnectionError(UNREACHABLE_HOST_MESSAGE, retryable=False) from err
+
+ errors: list[OSError] = []
+ for address in addresses:
+ try:
+ with socket.create_connection((address, 443), timeout=self.connect_timeout_seconds):
+ return
+ except OSError as err:
+ errors.append(err)
+ raise errors[-1]
try:
await asyncio.to_thread(probe)
diff --git a/products/batch_exports/backend/tests/temporal/destinations/databricks/test_client.py b/products/batch_exports/backend/tests/temporal/destinations/databricks/test_client.py
index a749f281c415..107fb52f43c7 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/databricks/test_client.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/databricks/test_client.py
@@ -204,15 +204,17 @@ class TestConnect:
host becomes reachable, then complete the connection."""
attempts = {"n": 0}
- def reachability_result(*_args, **_kwargs):
+ def resolution_result(*_args, **_kwargs):
attempts["n"] += 1
if attempts["n"] < 3:
raise socket.gaierror(socket.EAI_AGAIN, "Temporary failure in name resolution")
- return MagicMock()
+ return [(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", ("8.8.8.8", 443))]
- with patch(
- "products.batch_exports.backend.temporal.destinations.databricks_batch_export.socket.create_connection",
- side_effect=reachability_result,
+ with (
+ patch("socket.getaddrinfo", side_effect=resolution_result),
+ patch(
+ "products.batch_exports.backend.temporal.destinations.databricks_batch_export.socket.create_connection",
+ ),
):
async with client.connect():
pass
@@ -237,19 +239,16 @@ class TestConnect:
):
"""A hostname that does not exist is a config error, so we should fail fast with a targeted
message instead of backing off for minutes."""
- create_connection_mock = MagicMock(side_effect=socket.gaierror(errno, "Name or service not known"))
+ getaddrinfo_mock = MagicMock(side_effect=socket.gaierror(errno, "Name or service not known"))
with (
- patch(
- "products.batch_exports.backend.temporal.destinations.databricks_batch_export.socket.create_connection",
- new=create_connection_mock,
- ),
+ patch("socket.getaddrinfo", new=getaddrinfo_mock),
pytest.raises(DatabricksConnectionError, match=expected_match),
):
async with client.connect():
pass
- assert create_connection_mock.call_count == 1
+ assert getaddrinfo_mock.call_count == 1
mock_sql_connect.sql_connect.assert_not_called()
async def test_does_not_retry_invalid_credentials(
@@ -268,6 +267,60 @@ class TestConnect:
assert mock_sql_connect.sql_connect.call_count == 1
+ async def test_probe_connects_to_the_address_it_checked(
+ self, client: DatabricksClient, settings, mock_sql_connect: MockDatabricksSqlConnect
+ ):
+ settings.FORCE_URL_VALIDATION = True
+ client.server_hostname = "dbc.example.com"
+ addrinfo = [(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", ("8.8.8.8", 443))]
+
+ with (
+ patch("socket.getaddrinfo", return_value=addrinfo),
+ patch(
+ "products.batch_exports.backend.temporal.destinations.databricks_batch_export.socket.create_connection",
+ ) as create_connection_mock,
+ ):
+ async with client.connect():
+ pass
+
+ create_connection_mock.assert_called_once_with(("8.8.8.8", 443), timeout=client.connect_timeout_seconds)
+ mock_sql_connect.sql_connect.assert_called_once()
+
+ @pytest.mark.parametrize(
+ "addresses",
+ [
+ ["127.0.0.1"],
+ ["169.254.169.254"],
+ ["8.8.8.8", "10.0.0.5"],
+ ],
+ )
+ async def test_probe_refuses_a_host_that_resolves_internal(
+ self,
+ client: DatabricksClient,
+ settings,
+ fast_backoff,
+ mock_sql_connect: MockDatabricksSqlConnect,
+ addresses: list[str],
+ ):
+ settings.FORCE_URL_VALIDATION = True
+ client.server_hostname = "dbc.example.com"
+ addrinfo = [
+ (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", (address, 443)) for address in addresses
+ ]
+
+ with (
+ patch("socket.getaddrinfo", return_value=addrinfo),
+ patch(
+ "products.batch_exports.backend.temporal.destinations.databricks_batch_export.socket.create_connection",
+ ) as create_connection_mock,
+ pytest.raises(DatabricksConnectionError, match="Could not reach this host"),
+ ):
+ async with client.connect():
+ pass
+
+ create_connection_mock.assert_not_called()
+ mock_sql_connect.sql_connect.assert_not_called()
+
class TestQueryBuilders:
@pytest.mark.parametrize("schema_evolution", [True, False])