:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(batch-exports): check and pin postgres hosts at connect time (#121761)
Felipe Rosa de Almeida0c65117259bc
diff --git a/posthog/security/url_validation.py b/posthog/security/url_validation.pyindex 2e4c0c9cb899..db8503f1886a 100644--- a/posthog/security/url_validation.py+++ b/posthog/security/url_validation.py@@ -552,13 +552,18 @@ UNREACHABLE_HOST_MESSAGE = ()-def validate_external_host(host: str) -> None:+def validate_external_host(host: str, *, resolved_ips: ResolvedIPs | None = None) -> None:"""Raise ``ValueError`` unless ``host`` is safe for our servers to reach.SSRF guard for a destination that stores a bare host rather than a URL (Postgres,Redshift). Applies the same name and IP rules as ``validate_external_url``, so the twocannot drift apart. Bypassed in local dev and in tests, unless the``FORCE_URL_VALIDATION`` setting is true.++ A caller that resolves ``host`` itself and dials that answer passes it as ``resolved_ips``,+ so the addresses checked are the addresses dialed. Without it, the host is resolved here,+ and a record with a short TTL can answer public for this check and private for the caller's+ own lookup."""# The host could come from untyped config, so check its type first@@ -578,6 +583,6 @@ def validate_external_host(host: str) -> None:# Return the same error message in both cases, to avoid exposing details of our internal# network.- ips = resolve_host_ips(host)+ ips = resolve_host_ips(host) if resolved_ips is None else resolved_ipsif not ips or any(_is_internal_ip(ip) for ip in ips):raise ValueError("Host does not resolve to a valid IP address")diff --git a/products/batch_exports/backend/temporal/destinations/postgres_batch_export.py b/products/batch_exports/backend/temporal/destinations/postgres_batch_export.pyindex 70e513be3fcc..bae7bf367e38 100644--- a/products/batch_exports/backend/temporal/destinations/postgres_batch_export.py+++ b/products/batch_exports/backend/temporal/destinations/postgres_batch_export.py@@ -7,6 +7,7 @@ import typingimport asyncioimport datetime as dtimport tempfile+import ipaddressimport contextlibimport dataclassesimport collections.abc@@ -29,6 +30,14 @@ from posthog.models.integration import (Integration,PostgreSQLIntegration,)+from posthog.psycopg_helpers import is_resolvable_hostname, resolve_psycopg_hostaddr_with_timeout+from posthog.security.url_validation import (+ INVALID_HOST_MESSAGE,+ UNREACHABLE_HOST_MESSAGE,+ ResolvedIPs,+ ShapeError,+ validate_external_host,+)from posthog.temporal.common.base import PostHogWorkflowfrom posthog.temporal.common.heartbeat import Heartbeaterfrom posthog.temporal.common.logger import get_logger, get_write_only_logger@@ -346,9 +355,14 @@ class PostgreSQLClient:error as its `__cause__`. Without it, every `OperationalError` is retried."""max_attempts = 5++ async def connect_to_checked_host(**kwargs: typing.Any) -> psycopg.AsyncConnection:+ host_kwargs = await asyncio.to_thread(self._checked_host_kwargs)+ return await psycopg.AsyncConnection.connect(**{**kwargs, **host_kwargs})+connect: typing.Callable[..., typing.Awaitable[psycopg.AsyncConnection]] = (make_retryable_with_exponential_backoff(- psycopg.AsyncConnection.connect,+ connect_to_checked_host,max_attempts=max_attempts,retryable_exceptions=(psycopg.OperationalError, psycopg.errors.ConnectionTimeout),is_exception_retryable=is_error_retryable or (lambda _: True),@@ -361,7 +375,6 @@ class PostgreSQLClient:user=self.user,password=self.password,dbname=self.database,- host=self.host,port=self.port,sslmode=self.ssl_mode,sslrootcert=ssl_root_cert,@@ -384,6 +397,45 @@ class PostgreSQLClient:self._connection = connectionyield self+ def _checked_host_kwargs(self) -> dict[str, str]:+ """Return the libpq `host`/`hostaddr` that dial only addresses that pass the SSRF check.++ The API checks the host when a destination is saved, but the hostname is what gets stored,+ and libpq resolves it again on every connect. A record with a short TTL can answer public+ for the save and private for the connect, and no egress proxy sees a raw Postgres socket,+ so the check has to run on the addresses this connect dials. `host` repeats the hostname+ once per address because libpq pairs the two lists by position, and TLS checks the server+ certificate against `host`.+ """+ resolved_ips: ResolvedIPs | None = None+ host_kwargs = {"host": self.host}+ if is_resolvable_hostname(self.host):+ addresses = resolve_psycopg_hostaddr_with_timeout(+ self.host,+ self.port,+ self.connection_timeout,+ fail_on_resolution_error=True,+ raise_on_temporary_failure=True,+ )+ if not addresses:+ raise psycopg.OperationalError("Could not resolve database host name")+ resolved_ips = {ipaddress.ip_address(address) for address in addresses}+ host_kwargs = {"host": ",".join([self.host] * len(addresses)), "hostaddr": ",".join(addresses)}++ try:+ validate_external_host(self.host, resolved_ips=resolved_ips)+ except ShapeError as err:+ raise PostgreSQLConnectionError(INVALID_HOST_MESSAGE) from err+ except ValueError as err:+ self.logger.warning(+ "Refused to connect to a host that failed the SSRF check",+ reason=str(err),+ hostaddr=host_kwargs.get("hostaddr"),+ )+ raise PostgreSQLConnectionError(UNREACHABLE_HOST_MESSAGE) from err++ return host_kwargs+@contextlib.asynccontextmanagerasync def _ensure_ssl_root_cert_file(self):if self.ssl_root_cert in ("system", MISSING_CERT_PATH):diff --git a/products/batch_exports/backend/tests/temporal/destinations/postgres/test_postgres_client.py b/products/batch_exports/backend/tests/temporal/destinations/postgres/test_postgres_client.pyindex bb535db17bfe..948095674cc5 100644--- a/products/batch_exports/backend/tests/temporal/destinations/postgres/test_postgres_client.py+++ b/products/batch_exports/backend/tests/temporal/destinations/postgres/test_postgres_client.py@@ -1,7 +1,9 @@+import socketimport typingimport pathlibimport pytest+from unittest import mockimport psycopgfrom psycopg import sql@@ -13,6 +15,7 @@ from products.batch_exports.backend.temporal.destinations.postgres_batch_exportNON_RETRYABLE_ERROR_TYPES,PostgresInsertInputs,PostgreSQLClient,+ PostgreSQLConnectionError,PostgreSQLMergeTimeoutError,PostgreSQLMissingRequiredInputsError,PostgreSQLTransactionError,@@ -69,6 +72,46 @@ def test_missing_required_inputs_error_is_non_retryable():assert PostgreSQLMissingRequiredInputsError.__name__ in NON_RETRYABLE_ERROR_TYPES+def _client_resolving_to(address: str) -> tuple[PostgreSQLClient, list[tuple]]:+ client = PostgreSQLClient(+ user="loader", password="secret", database="db", host="db.example.com", port=5432, ssl_mode="require"+ )+ addrinfo = [(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", (address, 5432))]+ return client, addrinfo+++async def test_connect_dials_the_address_it_checked(settings):+ settings.FORCE_URL_VALIDATION = True+ client, addrinfo = _client_resolving_to("8.8.8.8")+ connection = mock.AsyncMock()+ connection.__aenter__.return_value = connection++ with (+ mock.patch("socket.getaddrinfo", return_value=addrinfo),+ mock.patch.object(psycopg.AsyncConnection, "connect", return_value=connection) as connect,+ ):+ async with client.connect():+ pass++ assert connect.call_args.kwargs["host"] == "db.example.com"+ assert connect.call_args.kwargs["hostaddr"] == "8.8.8.8"+++async def test_connect_refuses_a_host_that_resolves_internal(settings):+ settings.FORCE_URL_VALIDATION = True+ client, addrinfo = _client_resolving_to("169.254.169.254")++ with (+ mock.patch("socket.getaddrinfo", return_value=addrinfo),+ mock.patch.object(psycopg.AsyncConnection, "connect") as connect,+ pytest.raises(PostgreSQLConnectionError),+ ):+ async with client.connect():+ pass++ connect.assert_not_called()++async def test_run_in_retryable_transaction_raises_non_retryable_error_after_max_retries(postgres_config, setup_postgres_test_db):diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_postgres_configuration_errors.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_postgres_configuration_errors.pyindex 0233746559cc..fe3e00d8b8d4 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_postgres_configuration_errors.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_postgres_configuration_errors.py@@ -6,6 +6,7 @@ from unittest import mockimport psycopgfrom posthog.models.integration import Integration+from posthog.security.url_validation import UNREACHABLE_HOST_MESSAGEfrom products.batch_exports.backend.facade.destinations.postgres import PostgreSQLClient, PostgreSQLConnectionErrorfrom products.warehouse_sources.backend.temporal.data_imports.destinations.contracts import DestinationRunContext@@ -253,6 +254,20 @@ class TestWriterClient:assert IPV6_ONLY_HOST_DETAIL in str(caught.value)connect.assert_not_called()+ @pytest.mark.asyncio+ async def test_a_host_that_resolves_internal_fails_without_connecting(self, settings) -> None:+ settings.FORCE_URL_VALIDATION = True++ with (+ mock.patch("socket.getaddrinfo", return_value=_addrinfo("10.0.0.5")),+ mock.patch.object(psycopg.AsyncConnection, "connect") as connect,+ pytest.raises(DestinationConfigurationError) as caught,+ ):+ await _open(_FixedClientWriter(HOST))++ assert UNREACHABLE_HOST_MESSAGE in str(caught.value)+ connect.assert_not_called()+@pytest.mark.parametrize(("_name", "error", "expected_error", "expected_attempts"),[diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/postgres.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/postgres.pyindex 83097bfe3f54..bd59e6610fb9 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/postgres.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/postgres.py@@ -291,23 +291,21 @@ class PostgresDestinationWriter:async def _client(self) -> AsyncIterator[PostgreSQLClient]:client = await self._make_client()- # The pre-check resolves the host only to classify it; `connect()` resolves it again.+ # The pre-check resolves the host only to classify it. `connect()` resolves it again,+ # checks those addresses, and dials only them.host_detail = await host_configuration_error_detail(client.host, client.port)if host_detail is not None:raise DestinationConfigurationError(self._ctx.destination_name, host_detail)- # `connect()` resolves and dials the integration's hostname with no connection-time- # address validation or pinning, same as batch exports' own Postgres destination on- # this same client. An editor-controlled hostname could DNS-rebind to a private address- # between resolution and connect; closing that needs pinning inside `PostgreSQLClient`- # itself (posthog#86986 review discussion), which this product can't reach into, so- # every caller of this shared client is fixed together rather than patched here alone.async with AsyncExitStack() as stack:# Entered apart from the body, so only a failure to connect is classified here, never# an error the caller raises while it holds the connection.try:await stack.enter_async_context(client.connect(is_error_retryable=_is_connect_error_retryable))except PostgreSQLConnectionError as err:+ # The client's host check refused the address. Its message hides which check failed.+ if isinstance(err.__cause__, ValueError):+ raise DestinationConfigurationError(self._ctx.destination_name, str(err)) from errdetail = connect_configuration_error_detail(err.__cause__) if err.__cause__ else Noneif detail is not None:raise DestinationConfigurationError(self._ctx.destination_name, detail) from err