:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(hogql): pass port and source type to managed warehouse host checks (#123103)
posthog[bot]94e8af6293c7
diff --git a/posthog/hogql/direct_sql/postgres_adapter.py b/posthog/hogql/direct_sql/postgres_adapter.pyindex 4982157933e3..45152083db52 100644--- a/posthog/hogql/direct_sql/postgres_adapter.py+++ b/posthog/hogql/direct_sql/postgres_adapter.py@@ -130,7 +130,11 @@ class PostgresAdapter:raise ExposedHogQLError("Invalid direct Postgres connection.")with timings.measure("postgres_source_registry", emit_span=True):- from products.warehouse_sources.backend.facade.source_management import PostgresSource, SourceRegistry+ from products.warehouse_sources.backend.facade.source_management import (+ POSTGRES_HOST_CHECK_SOURCE_TYPE,+ PostgresSource,+ SourceRegistry,+ )from products.warehouse_sources.backend.facade.types import ExternalDataSourceTypepostgres_source = cast(PostgresSource, SourceRegistry.get_source(ExternalDataSourceType.POSTGRES))@@ -144,7 +148,11 @@ class PostgresAdapter:with timings.measure("postgres_host_validation"):valid_host, host_errors = postgres_source.is_database_host_valid(- config.host, team.pk, using_ssh_tunnel=config.ssh_tunnel.enabled if config.ssh_tunnel else False+ config.host,+ team.pk,+ using_ssh_tunnel=config.ssh_tunnel.enabled if config.ssh_tunnel else False,+ port=config.port,+ source_type=POSTGRES_HOST_CHECK_SOURCE_TYPE,)if not valid_host:raise ExposedHogQLError(host_errors or "Invalid Postgres host.")@@ -159,6 +167,7 @@ class PostgresAdapter:with request.timings.measure("postgres_source_validation"):with request.timings.measure("postgres_source_helpers_import"):from products.warehouse_sources.backend.facade.source_management import (+ POSTGRES_HOST_CHECK_SOURCE_TYPE,HostNotAllowedError,TemporaryHostResolutionError,_get_sslmode,@@ -185,7 +194,9 @@ class PostgresAdapter:with request.timings.measure("postgres_tunnel_open", emit_span=True):try:host, port = tunnel_stack.enter_context(- postgres_source.with_ssh_tunnel(source_config, request.team.pk)+ postgres_source.with_ssh_tunnel(+ source_config, request.team.pk, POSTGRES_HOST_CHECK_SOURCE_TYPE+ ))except HostNotAllowedError as error:# `validate_source_config` already ran the host check, but a short-TTL@@ -221,6 +232,7 @@ class PostgresAdapter:port=port,connect_timeout=DIRECT_POSTGRES_CONNECT_TIMEOUT_SECONDS,team_id=request.team.pk,+ source_type=POSTGRES_HOST_CHECK_SOURCE_TYPE,),))diff --git a/posthog/hogql/direct_sql/test/test_trino_adapter.py b/posthog/hogql/direct_sql/test/test_trino_adapter.pyindex 98685e27d9f5..323e1b4817cc 100644--- a/posthog/hogql/direct_sql/test/test_trino_adapter.py+++ b/posthog/hogql/direct_sql/test/test_trino_adapter.py@@ -1,9 +1,15 @@import pytestfrom unittest.mock import MagicMock, patch+from django.test import override_settings+from posthog.hogql.direct_sql.trino_adapter import TrinoAdapter, ensure_read_only_raw_trino_statementfrom posthog.hogql.errors import ExposedHogQLError+from products.managed_warehouse.backend.facade.contracts import ManagedWarehouseTarget++_MIXINS_MODULE = "products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins"+@pytest.mark.parametrize("sql",@@ -128,3 +134,27 @@ def test_cancellation_during_prepare_prevents_starting_the_main_query() -> None:)assert len(statements) == 1assert statements[0].startswith("PREPARE ")+++@override_settings(CLOUD_DEPLOYMENT="US", MANAGED_WAREHOUSE_INGRESS_IPS="10.1.0.1")+def test_validate_source_config_accepts_the_teams_managed_warehouse() -> None:+ # A managed warehouse host resolves to a private load balancer address. The host check+ # exempts it only when the caller names the port and the source type.+ host = "acme.dw.us.postwh.com"+ source = MagicMock(direct_engine="trino")+ team = MagicMock(pk=1)+ config = MagicMock(host=host, port=443)++ with (+ patch("posthog.hogql.direct_sql.trino_adapter.is_direct_capable", return_value=True),+ patch("posthog.hogql.direct_sql.trino_adapter.parse_direct_source_config", return_value=config),+ patch(f"{_MIXINS_MODULE}.socket.getaddrinfo", return_value=[(None, None, None, None, ("10.1.0.1", 0))]),+ patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),+ patch(+ "products.managed_warehouse.backend.managed_ingress.get_managed_warehouse_target",+ return_value=ManagedWarehouseTarget(host=host, ports=frozenset({443})),+ ),+ ):+ _, validated_config = TrinoAdapter().validate_source_config(source, team)++ assert validated_config is configdiff --git a/posthog/hogql/direct_sql/trino_adapter.py b/posthog/hogql/direct_sql/trino_adapter.pyindex a3ec46f1ea7b..1477a997cb5a 100644--- a/posthog/hogql/direct_sql/trino_adapter.py+++ b/posthog/hogql/direct_sql/trino_adapter.py@@ -244,6 +244,7 @@ class TrinoAdapter:def validate_source_config(self, source: ExternalDataSource, team: Team) -> tuple[TrinoSource, TrinoSourceConfig]:from products.warehouse_sources.backend.facade.source_management import (+ MANAGED_WAREHOUSE_TRINO_SOURCE_TYPE, # noqa: PLC0415 — avoids loading every source during HogQL startupSourceRegistry, # noqa: PLC0415 — avoids loading every source during HogQL startup)from products.warehouse_sources.backend.facade.types import (@@ -255,7 +256,9 @@ class TrinoAdapter:trino_source = cast("TrinoSource", SourceRegistry.get_source(ExternalDataSourceType.TRINO))config = parse_direct_source_config(trino_source, source)- is_valid, error = trino_source.is_database_host_valid(config.host, team.pk)+ is_valid, error = trino_source.is_database_host_valid(+ config.host, team.pk, port=config.port, source_type=MANAGED_WAREHOUSE_TRINO_SOURCE_TYPE+ )if not is_valid:raise ExposedHogQLError(error or "Invalid Trino host.")return trino_source, configdiff --git a/posthog/hogql/test/test_direct_postgres_query.py b/posthog/hogql/test/test_direct_postgres_query.pyindex 5a555634322f..211cde78197a 100644--- a/posthog/hogql/test/test_direct_postgres_query.py+++ b/posthog/hogql/test/test_direct_postgres_query.py@@ -36,6 +36,7 @@ from posthog.hogql.query_stats import query_stats_scopefrom posthog.models import Team+from products.managed_warehouse.backend.facade.contracts import ManagedWarehouseTargetfrom products.warehouse_sources.backend.facade.models import DataWarehouseTable, ExternalDataSchema, ExternalDataSourcefrom products.warehouse_sources.backend.facade.source_management import SSL_REQUIRED_AFTER_DATE, HostNotAllowedError@@ -2149,6 +2150,69 @@ class TestDirectPostgresQuery(APIBaseTest):self.assertEqual(mock_connect.call_args.kwargs["sslkey"], "/tmp/no.txt")self.assertEqual(mock_connect.call_args.kwargs["sslrootcert"], "/tmp/no.txt")+ @override_settings(DEBUG=False, TEST=False, CLOUD_DEPLOYMENT="US", MANAGED_WAREHOUSE_INGRESS_IPS="10.1.0.1")+ @patch("posthog.hogql.direct_sql.postgres_adapter.psycopg.connect")+ def test_execute_direct_postgres_query_reaches_the_teams_managed_warehouse(self, mock_connect):+ # A managed warehouse host resolves to a private load balancer address. The host check+ # exempts it only when the caller names the port and the source type.+ host = "acme.dw.us.postwh.com"+ source = ExternalDataSource.objects.create(+ team=self.team,+ source_id="source_id",+ connection_id="connection_id",+ status=ExternalDataSource.Status.COMPLETED,+ source_type="Postgres",+ access_method=ExternalDataSource.AccessMethod.DIRECT,+ prefix="ph3",+ job_inputs={+ "host": host,+ "port": 5432,+ "database": "postgres",+ "user": "postgres",+ "password": "postgres",+ "schema": "ph3",+ },+ )+ DataWarehouseTable.objects.create(+ name="posthog_dashboard",+ format="Parquet",+ team=self.team,+ external_data_source=source,+ url_pattern="direct://postgres",+ columns={+ "id": {"hogql": "IntegerDatabaseField", "clickhouse": "Int64", "valid": True},+ },+ )++ mocked_cursor = MagicMock()+ mocked_cursor.fetchall.return_value = [(1,)]+ column = MagicMock(type_code=23)+ column.name = "id"+ mocked_cursor.description = [column]+ mocked_connection = MagicMock()+ mocked_connection.cursor.return_value.__enter__.return_value = mocked_cursor+ mock_connect.return_value.__enter__.return_value = mocked_connection++ with (+ patch(+ f"{_MIXINS_MODULE}.socket.getaddrinfo",+ return_value=[(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("10.1.0.1", 5432))],+ ),+ patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),+ patch(+ "products.managed_warehouse.backend.managed_ingress.get_managed_warehouse_target",+ return_value=ManagedWarehouseTarget(host=host, ports=frozenset({5432})),+ ),+ ):+ HogQLQueryExecutor(+ query="SELECT id FROM posthog_dashboard LIMIT 1",+ team=self.team,+ connection_id=str(source.id),+ ).execute()++ self.assertEqual(mock_connect.call_args.kwargs["host"], host)+ self.assertEqual(mock_connect.call_args.kwargs["hostaddr"], "10.1.0.1")+def test_hogql_query_for_synced_source_prints_identical_sql_to_pure_direct(self):job_inputs = {"host": "localhost",diff --git a/products/warehouse_sources/backend/facade/source_management.py b/products/warehouse_sources/backend/facade/source_management.pyindex 9f4240ce559b..ed822b52ee95 100644--- a/products/warehouse_sources/backend/facade/source_management.py+++ b/products/warehouse_sources/backend/facade/source_management.py@@ -98,9 +98,11 @@ _LAZY = {"get_primary_key_columns": "sources.postgres.postgres","new_source_requires_ssl": "sources.postgres.postgres","source_requires_ssl": "sources.postgres.postgres",+ "POSTGRES_HOST_CHECK_SOURCE_TYPE": "sources.postgres.source","PostgresSource": "sources.postgres.source","RedshiftSource": "sources.redshift.source","SnowflakeSource": "sources.snowflake.source",+ "MANAGED_WAREHOUSE_TRINO_SOURCE_TYPE": "sources.trino.trino","connect_trino": "sources.trino.trino","is_posthog_managed_trino_host": "sources.trino.trino","trino_error_to_message": "sources.trino.trino",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/mixins.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/mixins.pyindex 845b1b58cfba..4e254d31fe5f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/mixins.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/mixins.py@@ -703,9 +703,16 @@ def _require_loopback(host: str) -> str:return host-def _checked_connect_host(host: str, team_id: int | None, refusal_prefix: str) -> str:+def _checked_connect_host(+ host: str,+ team_id: int | None,+ refusal_prefix: str,+ *,+ port: int | None = None,+ source_type: str | None = None,+) -> str:"""Return the address `resolve_safe_host` approves for `host`, or raise `HostNotAllowedError`."""- resolution = resolve_safe_host(host, team_id)+ resolution = resolve_safe_host(host, team_id, port=port, source_type=source_type)if resolution.connect_host is None:raise HostNotAllowedError(f"{refusal_prefix}: {resolution.error or _INTERNAL_IP_ERROR}")return resolution.connect_host@@ -756,7 +763,7 @@ def pinned_connect_host(host: str, team_id: int | None) -> DialTarget:)-def _check_direct_host(config, team_id: int | None) -> None:+def _check_direct_host(config, team_id: int | None, source_type: str | None = None) -> None:"""Refuse a direct database connection to a host that resolves somewhere internal.The connect-time counterpart to `is_database_host_valid`, which runs at the API layer on@@ -783,7 +790,9 @@ def _check_direct_host(config, team_id: int | None) -> None:The resolve inside `resolve_safe_host` stops at `HOST_RESOLUTION_TIMEOUT_SECONDS`, so a stalledresolver fails fast and retryably."""- _checked_connect_host(config.host, team_id, DATABASE_HOST_NOT_ALLOWED_ERROR)+ _checked_connect_host(+ config.host, team_id, DATABASE_HOST_NOT_ALLOWED_ERROR, port=config.port, source_type=source_type+ )# sshtunnel reports every failure to reach the gateway with the same wording. `_create_tunnels`@@ -874,7 +883,7 @@ def _connected_ssh_tunnel(ssh_tunnel: SSHTunnel, config, *, ssh_host: str) -> Ge@contextmanager-def open_ssh_tunnel(config, team_id: int | None = None) -> Generator[tuple[str, int]]:+def open_ssh_tunnel(config, team_id: int | None = None, source_type: str | None = None) -> Generator[tuple[str, int]]:"""Yield `(host, port)` for a database connection, going through an SSH tunnel if configured."""with _logged_connection(config, team_id):ssh_config = _enabled_ssh_tunnel(config)@@ -887,7 +896,7 @@ def open_ssh_tunnel(config, team_id: int | None = None) -> Generator[tuple[str,yield _require_loopback(tunnel.local_bind_host), tunnel.local_bind_portelse:- _check_direct_host(config, team_id)+ _check_direct_host(config, team_id, source_type)yield config.host, config.port@@ -931,8 +940,10 @@ def make_ssh_tunnel_factory(class SSHTunnelMixin:"""Mixin for sources that support SSH tunnels"""- def with_ssh_tunnel(self, config, team_id: int | None = None) -> _GeneratorContextManager[tuple[str, int]]:- return open_ssh_tunnel(config, team_id)+ def with_ssh_tunnel(+ self, config, team_id: int | None = None, source_type: str | None = None+ ) -> _GeneratorContextManager[tuple[str, int]]:+ return open_ssh_tunnel(config, team_id, source_type)def make_ssh_tunnel_func(self, config, team_id: int | None = None