Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(batch-exports): check and pin postgres hosts at connect time (#121761)

Felipe Rosa de Almeida0c65117259bc
diff --git a/posthog/security/url_validation.py b/posthog/security/url_validation.py
index 2e4c0c9cb899..db8503f1886a 100644
--- a/posthog/security/url_validation.py
+++ b/posthog/security/url_validation.py
@@ -552,13 +552,18 @@ UNREACHABLE_HOST_MESSAGE = (
)
-def validate_external_host(host: str) -> None:
+def validate_external_host(host: str, *, resolved_ips: ResolvedIPs | None = None) -> None:
"""Raise ``ValueError`` unless ``host`` is safe for our servers to reach.
SSRF guard for a destination that stores a bare host rather than a URL (Postgres,
Redshift). Applies the same name and IP rules as ``validate_external_url``, so the two
cannot drift apart. Bypassed in local dev and in tests, unless the
``FORCE_URL_VALIDATION`` setting is true.
+
+ A caller that resolves ``host`` itself and dials that answer passes it as ``resolved_ips``,
+ so the addresses checked are the addresses dialed. Without it, the host is resolved here,
+ and a record with a short TTL can answer public for this check and private for the caller's
+ own lookup.
"""
# The host could come from untyped config, so check its type first
@@ -578,6 +583,6 @@ def validate_external_host(host: str) -> None:
# Return the same error message in both cases, to avoid exposing details of our internal
# network.
- ips = resolve_host_ips(host)
+ ips = resolve_host_ips(host) if resolved_ips is None else resolved_ips
if not ips or any(_is_internal_ip(ip) for ip in ips):
raise ValueError("Host does not resolve to a valid IP address")
diff --git a/products/batch_exports/backend/temporal/destinations/postgres_batch_export.py b/products/batch_exports/backend/temporal/destinations/postgres_batch_export.py
index 70e513be3fcc..bae7bf367e38 100644
--- a/products/batch_exports/backend/temporal/destinations/postgres_batch_export.py
+++ b/products/batch_exports/backend/temporal/destinations/postgres_batch_export.py
@@ -7,6 +7,7 @@ import typing
import asyncio
import datetime as dt
import tempfile
+import ipaddress
import contextlib
import dataclasses
import collections.abc
@@ -29,6 +30,14 @@ from posthog.models.integration import (
Integration,
PostgreSQLIntegration,
)
+from posthog.psycopg_helpers import is_resolvable_hostname, resolve_psycopg_hostaddr_with_timeout
+from posthog.security.url_validation import (
+ INVALID_HOST_MESSAGE,
+ UNREACHABLE_HOST_MESSAGE,
+ ResolvedIPs,
+ ShapeError,
+ validate_external_host,
+)
from posthog.temporal.common.base import PostHogWorkflow
from posthog.temporal.common.heartbeat import Heartbeater
from posthog.temporal.common.logger import get_logger, get_write_only_logger
@@ -346,9 +355,14 @@ class PostgreSQLClient:
error as its `__cause__`. Without it, every `OperationalError` is retried.
"""
max_attempts = 5
+
+ async def connect_to_checked_host(**kwargs: typing.Any) -> psycopg.AsyncConnection:
+ host_kwargs = await asyncio.to_thread(self._checked_host_kwargs)
+ return await psycopg.AsyncConnection.connect(**{**kwargs, **host_kwargs})
+
connect: typing.Callable[..., typing.Awaitable[psycopg.AsyncConnection]] = (
make_retryable_with_exponential_backoff(
- psycopg.AsyncConnection.connect,
+ connect_to_checked_host,
max_attempts=max_attempts,
retryable_exceptions=(psycopg.OperationalError, psycopg.errors.ConnectionTimeout),
is_exception_retryable=is_error_retryable or (lambda _: True),
@@ -361,7 +375,6 @@ class PostgreSQLClient:
user=self.user,
password=self.password,
dbname=self.database,
- host=self.host,
port=self.port,
sslmode=self.ssl_mode,
sslrootcert=ssl_root_cert,
@@ -384,6 +397,45 @@ class PostgreSQLClient:
self._connection = connection
yield self
+ def _checked_host_kwargs(self) -> dict[str, str]:
+ """Return the libpq `host`/`hostaddr` that dial only addresses that pass the SSRF check.
+
+ The API checks the host when a destination is saved, but the hostname is what gets stored,
+ and libpq resolves it again on every connect. A record with a short TTL can answer public
+ for the save and private for the connect, and no egress proxy sees a raw Postgres socket,
+ so the check has to run on the addresses this connect dials. `host` repeats the hostname
+ once per address because libpq pairs the two lists by position, and TLS checks the server
+ certificate against `host`.
+ """
+ resolved_ips: ResolvedIPs | None = None
+ host_kwargs = {"host": self.host}
+ if is_resolvable_hostname(self.host):
+ addresses = resolve_psycopg_hostaddr_with_timeout(
+ self.host,
+ self.port,
+ self.connection_timeout,
+ fail_on_resolution_error=True,
+ raise_on_temporary_failure=True,
+ )
+ if not addresses:
+ raise psycopg.OperationalError("Could not resolve database host name")
+ resolved_ips = {ipaddress.ip_address(address) for address in addresses}
+ host_kwargs = {"host": ",".join([self.host] * len(addresses)), "hostaddr": ",".join(addresses)}
+
+ try:
+ validate_external_host(self.host, resolved_ips=resolved_ips)
+ except ShapeError as err:
+ raise PostgreSQLConnectionError(INVALID_HOST_MESSAGE) from err
+ except ValueError as err:
+ self.logger.warning(
+ "Refused to connect to a host that failed the SSRF check",
+ reason=str(err),
+ hostaddr=host_kwargs.get("hostaddr"),
+ )
+ raise PostgreSQLConnectionError(UNREACHABLE_HOST_MESSAGE) from err
+
+ return host_kwargs
+
@contextlib.asynccontextmanager
async def _ensure_ssl_root_cert_file(self):
if self.ssl_root_cert in ("system", MISSING_CERT_PATH):
diff --git a/products/batch_exports/backend/tests/temporal/destinations/postgres/test_postgres_client.py b/products/batch_exports/backend/tests/temporal/destinations/postgres/test_postgres_client.py
index bb535db17bfe..948095674cc5 100644
--- a/products/batch_exports/backend/tests/temporal/destinations/postgres/test_postgres_client.py
+++ b/products/batch_exports/backend/tests/temporal/destinations/postgres/test_postgres_client.py
@@ -1,7 +1,9 @@
+import socket
import typing
import pathlib
import pytest
+from unittest import mock
import psycopg
from psycopg import sql
@@ -13,6 +15,7 @@ from products.batch_exports.backend.temporal.destinations.postgres_batch_export
NON_RETRYABLE_ERROR_TYPES,
PostgresInsertInputs,
PostgreSQLClient,
+ PostgreSQLConnectionError,
PostgreSQLMergeTimeoutError,
PostgreSQLMissingRequiredInputsError,
PostgreSQLTransactionError,
@@ -69,6 +72,46 @@ def test_missing_required_inputs_error_is_non_retryable():
assert PostgreSQLMissingRequiredInputsError.__name__ in NON_RETRYABLE_ERROR_TYPES
+def _client_resolving_to(address: str) -> tuple[PostgreSQLClient, list[tuple]]:
+ client = PostgreSQLClient(
+ user="loader", password="secret", database="db", host="db.example.com", port=5432, ssl_mode="require"
+ )
+ addrinfo = [(socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", (address, 5432))]
+ return client, addrinfo
+
+
+async def test_connect_dials_the_address_it_checked(settings):
+ settings.FORCE_URL_VALIDATION = True
+ client, addrinfo = _client_resolving_to("8.8.8.8")
+ connection = mock.AsyncMock()
+ connection.__aenter__.return_value = connection
+
+ with (
+ mock.patch("socket.getaddrinfo", return_value=addrinfo),
+ mock.patch.object(psycopg.AsyncConnection, "connect", return_value=connection) as connect,
+ ):
+ async with client.connect():
+ pass
+
+ assert connect.call_args.kwargs["host"] == "db.example.com"
+ assert connect.call_args.kwargs["hostaddr"] == "8.8.8.8"
+
+
+async def test_connect_refuses_a_host_that_resolves_internal(settings):
+ settings.FORCE_URL_VALIDATION = True
+ client, addrinfo = _client_resolving_to("169.254.169.254")
+
+ with (
+ mock.patch("socket.getaddrinfo", return_value=addrinfo),
+ mock.patch.object(psycopg.AsyncConnection, "connect") as connect,
+ pytest.raises(PostgreSQLConnectionError),
+ ):
+ async with client.connect():
+ pass
+
+ connect.assert_not_called()
+
+
async def test_run_in_retryable_transaction_raises_non_retryable_error_after_max_retries(
postgres_config, setup_postgres_test_db
):
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_postgres_configuration_errors.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_postgres_configuration_errors.py
index 0233746559cc..fe3e00d8b8d4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_postgres_configuration_errors.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_postgres_configuration_errors.py
@@ -6,6 +6,7 @@ from unittest import mock
import psycopg
from posthog.models.integration import Integration
+from posthog.security.url_validation import UNREACHABLE_HOST_MESSAGE
from products.batch_exports.backend.facade.destinations.postgres import PostgreSQLClient, PostgreSQLConnectionError
from products.warehouse_sources.backend.temporal.data_imports.destinations.contracts import DestinationRunContext
@@ -253,6 +254,20 @@ class TestWriterClient:
assert IPV6_ONLY_HOST_DETAIL in str(caught.value)
connect.assert_not_called()
+ @pytest.mark.asyncio
+ async def test_a_host_that_resolves_internal_fails_without_connecting(self, settings) -> None:
+ settings.FORCE_URL_VALIDATION = True
+
+ with (
+ mock.patch("socket.getaddrinfo", return_value=_addrinfo("10.0.0.5")),
+ mock.patch.object(psycopg.AsyncConnection, "connect") as connect,
+ pytest.raises(DestinationConfigurationError) as caught,
+ ):
+ await _open(_FixedClientWriter(HOST))
+
+ assert UNREACHABLE_HOST_MESSAGE in str(caught.value)
+ connect.assert_not_called()
+
@pytest.mark.parametrize(
("_name", "error", "expected_error", "expected_attempts"),
[
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/postgres.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/postgres.py
index 83097bfe3f54..bd59e6610fb9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/postgres.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/postgres.py
@@ -291,23 +291,21 @@ class PostgresDestinationWriter:
async def _client(self) -> AsyncIterator[PostgreSQLClient]:
client = await self._make_client()
- # The pre-check resolves the host only to classify it; `connect()` resolves it again.
+ # The pre-check resolves the host only to classify it. `connect()` resolves it again,
+ # checks those addresses, and dials only them.
host_detail = await host_configuration_error_detail(client.host, client.port)
if host_detail is not None:
raise DestinationConfigurationError(self._ctx.destination_name, host_detail)
- # `connect()` resolves and dials the integration's hostname with no connection-time
- # address validation or pinning, same as batch exports' own Postgres destination on
- # this same client. An editor-controlled hostname could DNS-rebind to a private address
- # between resolution and connect; closing that needs pinning inside `PostgreSQLClient`
- # itself (posthog#86986 review discussion), which this product can't reach into, so
- # every caller of this shared client is fixed together rather than patched here alone.
async with AsyncExitStack() as stack:
# Entered apart from the body, so only a failure to connect is classified here, never
# an error the caller raises while it holds the connection.
try:
await stack.enter_async_context(client.connect(is_error_retryable=_is_connect_error_retryable))
except PostgreSQLConnectionError as err:
+ # The client's host check refused the address. Its message hides which check failed.
+ if isinstance(err.__cause__, ValueError):
+ raise DestinationConfigurationError(self._ctx.destination_name, str(err)) from err
detail = connect_configuration_error_detail(err.__cause__) if err.__cause__ else None
if detail is not None:
raise DestinationConfigurationError(self._ctx.destination_name, detail) from err