Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(hogql): pass port and source type to managed warehouse host checks (#123103)

posthog[bot]94e8af6293c7
diff --git a/posthog/hogql/direct_sql/postgres_adapter.py b/posthog/hogql/direct_sql/postgres_adapter.py
index 4982157933e3..45152083db52 100644
--- a/posthog/hogql/direct_sql/postgres_adapter.py
+++ b/posthog/hogql/direct_sql/postgres_adapter.py
@@ -130,7 +130,11 @@ class PostgresAdapter:
raise ExposedHogQLError("Invalid direct Postgres connection.")
with timings.measure("postgres_source_registry", emit_span=True):
- from products.warehouse_sources.backend.facade.source_management import PostgresSource, SourceRegistry
+ from products.warehouse_sources.backend.facade.source_management import (
+ POSTGRES_HOST_CHECK_SOURCE_TYPE,
+ PostgresSource,
+ SourceRegistry,
+ )
from products.warehouse_sources.backend.facade.types import ExternalDataSourceType
postgres_source = cast(PostgresSource, SourceRegistry.get_source(ExternalDataSourceType.POSTGRES))
@@ -144,7 +148,11 @@ class PostgresAdapter:
with timings.measure("postgres_host_validation"):
valid_host, host_errors = postgres_source.is_database_host_valid(
- config.host, team.pk, using_ssh_tunnel=config.ssh_tunnel.enabled if config.ssh_tunnel else False
+ config.host,
+ team.pk,
+ using_ssh_tunnel=config.ssh_tunnel.enabled if config.ssh_tunnel else False,
+ port=config.port,
+ source_type=POSTGRES_HOST_CHECK_SOURCE_TYPE,
)
if not valid_host:
raise ExposedHogQLError(host_errors or "Invalid Postgres host.")
@@ -159,6 +167,7 @@ class PostgresAdapter:
with request.timings.measure("postgres_source_validation"):
with request.timings.measure("postgres_source_helpers_import"):
from products.warehouse_sources.backend.facade.source_management import (
+ POSTGRES_HOST_CHECK_SOURCE_TYPE,
HostNotAllowedError,
TemporaryHostResolutionError,
_get_sslmode,
@@ -185,7 +194,9 @@ class PostgresAdapter:
with request.timings.measure("postgres_tunnel_open", emit_span=True):
try:
host, port = tunnel_stack.enter_context(
- postgres_source.with_ssh_tunnel(source_config, request.team.pk)
+ postgres_source.with_ssh_tunnel(
+ source_config, request.team.pk, POSTGRES_HOST_CHECK_SOURCE_TYPE
+ )
)
except HostNotAllowedError as error:
# `validate_source_config` already ran the host check, but a short-TTL
@@ -221,6 +232,7 @@ class PostgresAdapter:
port=port,
connect_timeout=DIRECT_POSTGRES_CONNECT_TIMEOUT_SECONDS,
team_id=request.team.pk,
+ source_type=POSTGRES_HOST_CHECK_SOURCE_TYPE,
),
)
)
diff --git a/posthog/hogql/direct_sql/test/test_trino_adapter.py b/posthog/hogql/direct_sql/test/test_trino_adapter.py
index 98685e27d9f5..323e1b4817cc 100644
--- a/posthog/hogql/direct_sql/test/test_trino_adapter.py
+++ b/posthog/hogql/direct_sql/test/test_trino_adapter.py
@@ -1,9 +1,15 @@
import pytest
from unittest.mock import MagicMock, patch
+from django.test import override_settings
+
from posthog.hogql.direct_sql.trino_adapter import TrinoAdapter, ensure_read_only_raw_trino_statement
from posthog.hogql.errors import ExposedHogQLError
+from products.managed_warehouse.backend.facade.contracts import ManagedWarehouseTarget
+
+_MIXINS_MODULE = "products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins"
+
@pytest.mark.parametrize(
"sql",
@@ -128,3 +134,27 @@ def test_cancellation_during_prepare_prevents_starting_the_main_query() -> None:
)
assert len(statements) == 1
assert statements[0].startswith("PREPARE ")
+
+
+@override_settings(CLOUD_DEPLOYMENT="US", MANAGED_WAREHOUSE_INGRESS_IPS="10.1.0.1")
+def test_validate_source_config_accepts_the_teams_managed_warehouse() -> None:
+ # A managed warehouse host resolves to a private load balancer address. The host check
+ # exempts it only when the caller names the port and the source type.
+ host = "acme.dw.us.postwh.com"
+ source = MagicMock(direct_engine="trino")
+ team = MagicMock(pk=1)
+ config = MagicMock(host=host, port=443)
+
+ with (
+ patch("posthog.hogql.direct_sql.trino_adapter.is_direct_capable", return_value=True),
+ patch("posthog.hogql.direct_sql.trino_adapter.parse_direct_source_config", return_value=config),
+ patch(f"{_MIXINS_MODULE}.socket.getaddrinfo", return_value=[(None, None, None, None, ("10.1.0.1", 0))]),
+ patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),
+ patch(
+ "products.managed_warehouse.backend.managed_ingress.get_managed_warehouse_target",
+ return_value=ManagedWarehouseTarget(host=host, ports=frozenset({443})),
+ ),
+ ):
+ _, validated_config = TrinoAdapter().validate_source_config(source, team)
+
+ assert validated_config is config
diff --git a/posthog/hogql/direct_sql/trino_adapter.py b/posthog/hogql/direct_sql/trino_adapter.py
index a3ec46f1ea7b..1477a997cb5a 100644
--- a/posthog/hogql/direct_sql/trino_adapter.py
+++ b/posthog/hogql/direct_sql/trino_adapter.py
@@ -244,6 +244,7 @@ class TrinoAdapter:
def validate_source_config(self, source: ExternalDataSource, team: Team) -> tuple[TrinoSource, TrinoSourceConfig]:
from products.warehouse_sources.backend.facade.source_management import (
+ MANAGED_WAREHOUSE_TRINO_SOURCE_TYPE, # noqa: PLC0415 — avoids loading every source during HogQL startup
SourceRegistry, # noqa: PLC0415 — avoids loading every source during HogQL startup
)
from products.warehouse_sources.backend.facade.types import (
@@ -255,7 +256,9 @@ class TrinoAdapter:
trino_source = cast("TrinoSource", SourceRegistry.get_source(ExternalDataSourceType.TRINO))
config = parse_direct_source_config(trino_source, source)
- is_valid, error = trino_source.is_database_host_valid(config.host, team.pk)
+ is_valid, error = trino_source.is_database_host_valid(
+ config.host, team.pk, port=config.port, source_type=MANAGED_WAREHOUSE_TRINO_SOURCE_TYPE
+ )
if not is_valid:
raise ExposedHogQLError(error or "Invalid Trino host.")
return trino_source, config
diff --git a/posthog/hogql/test/test_direct_postgres_query.py b/posthog/hogql/test/test_direct_postgres_query.py
index 5a555634322f..211cde78197a 100644
--- a/posthog/hogql/test/test_direct_postgres_query.py
+++ b/posthog/hogql/test/test_direct_postgres_query.py
@@ -36,6 +36,7 @@ from posthog.hogql.query_stats import query_stats_scope
from posthog.models import Team
+from products.managed_warehouse.backend.facade.contracts import ManagedWarehouseTarget
from products.warehouse_sources.backend.facade.models import DataWarehouseTable, ExternalDataSchema, ExternalDataSource
from products.warehouse_sources.backend.facade.source_management import SSL_REQUIRED_AFTER_DATE, HostNotAllowedError
@@ -2149,6 +2150,69 @@ class TestDirectPostgresQuery(APIBaseTest):
self.assertEqual(mock_connect.call_args.kwargs["sslkey"], "/tmp/no.txt")
self.assertEqual(mock_connect.call_args.kwargs["sslrootcert"], "/tmp/no.txt")
+ @override_settings(DEBUG=False, TEST=False, CLOUD_DEPLOYMENT="US", MANAGED_WAREHOUSE_INGRESS_IPS="10.1.0.1")
+ @patch("posthog.hogql.direct_sql.postgres_adapter.psycopg.connect")
+ def test_execute_direct_postgres_query_reaches_the_teams_managed_warehouse(self, mock_connect):
+ # A managed warehouse host resolves to a private load balancer address. The host check
+ # exempts it only when the caller names the port and the source type.
+ host = "acme.dw.us.postwh.com"
+ source = ExternalDataSource.objects.create(
+ team=self.team,
+ source_id="source_id",
+ connection_id="connection_id",
+ status=ExternalDataSource.Status.COMPLETED,
+ source_type="Postgres",
+ access_method=ExternalDataSource.AccessMethod.DIRECT,
+ prefix="ph3",
+ job_inputs={
+ "host": host,
+ "port": 5432,
+ "database": "postgres",
+ "user": "postgres",
+ "password": "postgres",
+ "schema": "ph3",
+ },
+ )
+ DataWarehouseTable.objects.create(
+ name="posthog_dashboard",
+ format="Parquet",
+ team=self.team,
+ external_data_source=source,
+ url_pattern="direct://postgres",
+ columns={
+ "id": {"hogql": "IntegerDatabaseField", "clickhouse": "Int64", "valid": True},
+ },
+ )
+
+ mocked_cursor = MagicMock()
+ mocked_cursor.fetchall.return_value = [(1,)]
+ column = MagicMock(type_code=23)
+ column.name = "id"
+ mocked_cursor.description = [column]
+ mocked_connection = MagicMock()
+ mocked_connection.cursor.return_value.__enter__.return_value = mocked_cursor
+ mock_connect.return_value.__enter__.return_value = mocked_connection
+
+ with (
+ patch(
+ f"{_MIXINS_MODULE}.socket.getaddrinfo",
+ return_value=[(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("10.1.0.1", 5432))],
+ ),
+ patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),
+ patch(
+ "products.managed_warehouse.backend.managed_ingress.get_managed_warehouse_target",
+ return_value=ManagedWarehouseTarget(host=host, ports=frozenset({5432})),
+ ),
+ ):
+ HogQLQueryExecutor(
+ query="SELECT id FROM posthog_dashboard LIMIT 1",
+ team=self.team,
+ connection_id=str(source.id),
+ ).execute()
+
+ self.assertEqual(mock_connect.call_args.kwargs["host"], host)
+ self.assertEqual(mock_connect.call_args.kwargs["hostaddr"], "10.1.0.1")
+
def test_hogql_query_for_synced_source_prints_identical_sql_to_pure_direct(self):
job_inputs = {
"host": "localhost",
diff --git a/products/warehouse_sources/backend/facade/source_management.py b/products/warehouse_sources/backend/facade/source_management.py
index 9f4240ce559b..ed822b52ee95 100644
--- a/products/warehouse_sources/backend/facade/source_management.py
+++ b/products/warehouse_sources/backend/facade/source_management.py
@@ -98,9 +98,11 @@ _LAZY = {
"get_primary_key_columns": "sources.postgres.postgres",
"new_source_requires_ssl": "sources.postgres.postgres",
"source_requires_ssl": "sources.postgres.postgres",
+ "POSTGRES_HOST_CHECK_SOURCE_TYPE": "sources.postgres.source",
"PostgresSource": "sources.postgres.source",
"RedshiftSource": "sources.redshift.source",
"SnowflakeSource": "sources.snowflake.source",
+ "MANAGED_WAREHOUSE_TRINO_SOURCE_TYPE": "sources.trino.trino",
"connect_trino": "sources.trino.trino",
"is_posthog_managed_trino_host": "sources.trino.trino",
"trino_error_to_message": "sources.trino.trino",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/mixins.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/mixins.py
index 845b1b58cfba..4e254d31fe5f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/mixins.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/mixins.py
@@ -703,9 +703,16 @@ def _require_loopback(host: str) -> str:
return host
-def _checked_connect_host(host: str, team_id: int | None, refusal_prefix: str) -> str:
+def _checked_connect_host(
+ host: str,
+ team_id: int | None,
+ refusal_prefix: str,
+ *,
+ port: int | None = None,
+ source_type: str | None = None,
+) -> str:
"""Return the address `resolve_safe_host` approves for `host`, or raise `HostNotAllowedError`."""
- resolution = resolve_safe_host(host, team_id)
+ resolution = resolve_safe_host(host, team_id, port=port, source_type=source_type)
if resolution.connect_host is None:
raise HostNotAllowedError(f"{refusal_prefix}: {resolution.error or _INTERNAL_IP_ERROR}")
return resolution.connect_host
@@ -756,7 +763,7 @@ def pinned_connect_host(host: str, team_id: int | None) -> DialTarget:
)
-def _check_direct_host(config, team_id: int | None) -> None:
+def _check_direct_host(config, team_id: int | None, source_type: str | None = None) -> None:
"""Refuse a direct database connection to a host that resolves somewhere internal.
The connect-time counterpart to `is_database_host_valid`, which runs at the API layer on
@@ -783,7 +790,9 @@ def _check_direct_host(config, team_id: int | None) -> None:
The resolve inside `resolve_safe_host` stops at `HOST_RESOLUTION_TIMEOUT_SECONDS`, so a stalled
resolver fails fast and retryably.
"""
- _checked_connect_host(config.host, team_id, DATABASE_HOST_NOT_ALLOWED_ERROR)
+ _checked_connect_host(
+ config.host, team_id, DATABASE_HOST_NOT_ALLOWED_ERROR, port=config.port, source_type=source_type
+ )
# sshtunnel reports every failure to reach the gateway with the same wording. `_create_tunnels`
@@ -874,7 +883,7 @@ def _connected_ssh_tunnel(ssh_tunnel: SSHTunnel, config, *, ssh_host: str) -> Ge
@contextmanager
-def open_ssh_tunnel(config, team_id: int | None = None) -> Generator[tuple[str, int]]:
+def open_ssh_tunnel(config, team_id: int | None = None, source_type: str | None = None) -> Generator[tuple[str, int]]:
"""Yield `(host, port)` for a database connection, going through an SSH tunnel if configured."""
with _logged_connection(config, team_id):
ssh_config = _enabled_ssh_tunnel(config)
@@ -887,7 +896,7 @@ def open_ssh_tunnel(config, team_id: int | None = None) -> Generator[tuple[str,
yield _require_loopback(tunnel.local_bind_host), tunnel.local_bind_port
else:
- _check_direct_host(config, team_id)
+ _check_direct_host(config, team_id, source_type)
yield config.host, config.port
@@ -931,8 +940,10 @@ def make_ssh_tunnel_factory(
class SSHTunnelMixin:
"""Mixin for sources that support SSH tunnels"""
- def with_ssh_tunnel(self, config, team_id: int | None = None) -> _GeneratorContextManager[tuple[str, int]]:
- return open_ssh_tunnel(config, team_id)
+ def with_ssh_tunnel(
+ self, config, team_id: int | None = None, source_type: str | None = None
+ ) -> _GeneratorContextManager[tuple[str, int]]:
+ return open_ssh_tunnel(config, team_id, source_type)
def make_ssh_tunnel_func(
self, config, team_id: int | None = None