Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(warehouse-sources): narrow the managed warehouse host exemption (#122358)

Benjamin Knofe-Vider8dc833fa6f37
diff --git a/posthog/settings/base_variables.py b/posthog/settings/base_variables.py
index 02df9f30fe37..9508fd7274ad 100644
--- a/posthog/settings/base_variables.py
+++ b/posthog/settings/base_variables.py
@@ -88,6 +88,11 @@ DUCKGRES_PG_PORT: int = get_from_env("DUCKGRES_PG_PORT", 5432, type_cast=int)
# HOGTOWER_API_URL is set, control-plane calls go to hogtower's /api/v2 instead of duckgres.
HOGTOWER_API_URL: str | None = get_from_env("HOGTOWER_API_URL", optional=True)
HOGTOWER_INTERNAL_SECRET: str | None = get_from_env("HOGTOWER_INTERNAL_SECRET", optional=True)
+# MANAGED_WAREHOUSE_INGRESS_IPS is the comma-separated address set the shared managed-warehouse
+# `dw` load balancer answers on. The warehouse-source host guard exempts a managed warehouse host
+# from the private-address check only when every resolved address is in this set, so an empty value
+# refuses every managed warehouse host. posthog-cloud-infra owns the values.
+MANAGED_WAREHOUSE_INGRESS_IPS: str = get_from_env("MANAGED_WAREHOUSE_INGRESS_IPS", "", type_cast=str)
MANAGED_WAREHOUSE_CREDENTIAL_CACHE_REDIS_CONNECT_TIMEOUT_SECONDS: float = get_from_env(
"MANAGED_WAREHOUSE_CREDENTIAL_CACHE_REDIS_CONNECT_TIMEOUT_SECONDS", 0.5, type_cast=float
)
diff --git a/products/managed_warehouse/backend/facade/api.py b/products/managed_warehouse/backend/facade/api.py
index de605f067539..d5914e895f21 100644
--- a/products/managed_warehouse/backend/facade/api.py
+++ b/products/managed_warehouse/backend/facade/api.py
@@ -21,6 +21,7 @@ from products.managed_warehouse.backend.facade.contracts import (
DuckLakeCatalogConnectionConfig,
ManagedWarehouseBackfillState,
ManagedWarehouseProvisionStatus,
+ ManagedWarehouseTarget,
)
if TYPE_CHECKING:
@@ -44,6 +45,9 @@ __all__ = [
"get_duckgres_query_server_config",
"get_managed_warehouse_trino_password",
"get_org_id_for_team",
+ "get_managed_warehouse_target",
+ "managed_warehouse_domain",
+ "managed_warehouse_ingress_ips",
"get_team_deletion_block_reason",
"get_stored_bucket_config",
"get_stored_warehouse_config",
@@ -99,6 +103,36 @@ def default_bucket_region() -> str:
return common.default_bucket_region()
+def get_managed_warehouse_target(organization_id: str, *, source_type: str) -> ManagedWarehouseTarget | None:
+ """The managed warehouse `organization_id` may reach from a `source_type` source, or None.
+
+ The warehouse-source host guard is the only caller. It decides whether a source's host is
+ exempt from the private-address check, so it needs the control plane's own answer for this
+ organization rather than any stored row.
+ """
+ from products.managed_warehouse.backend.managed_ingress import get_managed_warehouse_target as _get # noqa: PLC0415
+
+ return _get(organization_id, source_type=source_type)
+
+
+def managed_warehouse_ingress_ips() -> tuple[str, ...]:
+ """The address set the managed warehouse ingress answers on, read per call.
+
+ The warehouse-source host guard passes this to `ManagedWarehouseTarget.accepts`, so a settings
+ override in a test takes effect without a reload.
+ """
+ from products.managed_warehouse.backend.managed_ingress import _ingress_ips # noqa: PLC0415
+
+ return _ingress_ips()
+
+
+def managed_warehouse_domain() -> str:
+ """The zone managed warehouse hostnames are served under in this deployment."""
+ from products.managed_warehouse.backend.managed_ingress import managed_warehouse_domain as _domain # noqa: PLC0415
+
+ return _domain()
+
+
def get_org_id_for_team(team_id: int) -> str:
return common._get_org_id_for_team(team_id)
diff --git a/products/managed_warehouse/backend/facade/contracts.py b/products/managed_warehouse/backend/facade/contracts.py
index 77487ce81d55..9ad6204465f0 100644
--- a/products/managed_warehouse/backend/facade/contracts.py
+++ b/products/managed_warehouse/backend/facade/contracts.py
@@ -254,6 +254,63 @@ class ManagedWarehouseSourceJobStatus(StrEnum):
STALE = "stale"
+@frozen(kw_only=True)
+class ManagedWarehouseTarget:
+ """The endpoint an organization's managed warehouse answers on, for the host guard.
+
+ `host` is the control plane's own host string, so a comparison against it is a comparison
+ against the value the ingress was provisioned for rather than anything derived locally.
+ `ports` is the listener set this organization's sources may reach, which keeps a Postgres
+ source off the gateway listener and a Trino source off the pgwire one.
+
+ The guard compares a source's host, port and resolved addresses against this to decide
+ whether the private-address check applies. It is not a connection; it is an authorization
+ decision.
+ """
+
+ host: str
+ ports: frozenset[int]
+
+ def accepts(self, host: str, port: int | None, addresses: list[str] | None, ingress_ips: tuple[str, ...]) -> bool:
+ """Whether this target authorizes a dial to `host` on `port` at `addresses`.
+
+ Every condition is required, and a missing one denies. `addresses` must be non-empty
+ because an empty set is a failed lookup, not an approved destination, and it must be a
+ subset of `ingress_ips` so a record that points elsewhere is refused. The caller supplies
+ the ingress set because a contract cannot read settings without importing Django.
+
+ Both hostnames are compared in their normalized form, so a control-plane host that comes
+ back with a capital letter or a root-zone trailing dot still matches the name the guard
+ normalized.
+ """
+ if _normalize_host(host) != _normalize_host(self.host):
+ return False
+ if port is None or port not in self.ports:
+ return False
+ if not addresses or not ingress_ips:
+ return False
+ allowed = {_canonical_ip(ip) for ip in ingress_ips}
+ return all(_canonical_ip(address) in allowed for address in addresses)
+
+
+def _normalize_host(host: str) -> str:
+ """Lowercase a hostname and drop one root-zone trailing dot, as the guard does."""
+ return host.lower().strip().rstrip(".")
+
+
+def _canonical_ip(value: str) -> str:
+ """Compare addresses as numbers, so an IPv4-mapped IPv6 form matches its IPv4 spelling."""
+ import ipaddress # noqa: PLC0415
+
+ try:
+ parsed = ipaddress.ip_address(value.strip().strip("[]"))
+ except ValueError:
+ return value
+ if isinstance(parsed, ipaddress.IPv6Address) and parsed.ipv4_mapped is not None:
+ return str(parsed.ipv4_mapped)
+ return str(parsed)
+
+
@dataclass(frozen=True, kw_only=True)
class ManagedWarehouseSourceJobUpdate:
team_id: int
diff --git a/products/managed_warehouse/backend/facade/testing.py b/products/managed_warehouse/backend/facade/testing.py
index 0bc0466853db..b0c99bf09e06 100644
--- a/products/managed_warehouse/backend/facade/testing.py
+++ b/products/managed_warehouse/backend/facade/testing.py
@@ -1,9 +1,14 @@
from __future__ import annotations
-from products.managed_warehouse.backend import source_job_state
+from products.managed_warehouse.backend import managed_ingress, source_job_state
from products.managed_warehouse.backend.facade.contracts import ManagedWarehouseSourceJobUpdate
-__all__ = ["record_source_job_state"]
+__all__ = ["NO_MANAGED_WAREHOUSE_TARGET", "record_source_job_state"]
+
+# The warehouse-source host guard's tests assert that a denial is recognised after the cache
+# serializes it. They cannot reach the sentinel through `managed_ingress` without breaking the
+# product's public-interface rule, so the facade re-exports it.
+NO_MANAGED_WAREHOUSE_TARGET = managed_ingress._NO_TARGET
def record_source_job_state(input: ManagedWarehouseSourceJobUpdate) -> None:
diff --git a/products/managed_warehouse/backend/managed_ingress.py b/products/managed_warehouse/backend/managed_ingress.py
new file mode 100644
index 000000000000..742371836b16
--- /dev/null
+++ b/products/managed_warehouse/backend/managed_ingress.py
@@ -0,0 +1,218 @@
+"""Read the managed-warehouse target an organization's warehouse is reachable at.
+
+The warehouse-source host guard needs one fact: may this organization dial its own managed
+warehouse, and at which address. The control plane owns that answer. This module is the only
+place that asks, so the guard stays free of transport details and the read has one cache.
+
+Hogtower answers it through `GET /warehouses/{org}`. Its `connection` block is present only
+while the warehouse is `ready`, and its `host` is `<warehouse_name>.dw.<env>.postwh.com` with
+port 5432. A warehouse that is provisioning, deleting, failed or absent yields no target, so a
+caller that gets None must treat the organization as having no reachable managed warehouse.
+
+Failures are not cached as successes and never raise past this module. They return None, which
+denies the exemption: refusing a managed source is the safe direction, and the guard has no
+useful reply for a control-plane outage.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+from uuid import UUID
+
+from django.core.cache import cache
+
+import requests as http_requests
+import structlog
+
+from products.managed_warehouse.backend.facade.contracts import ManagedWarehouseTarget # noqa: F401
+from products.managed_warehouse.backend.presentation import hogtower
+
+logger = structlog.get_logger(__name__)
+
+# The ports the shared `dw` load balancer publishes, keyed by the source type that legitimately
+# reaches it. Postgres-wire sources speak the 5432 listener. The Trino source speaks the gateway
+# on 443. Arrow Flight on 8815 has no warehouse source, so nothing is exempt there.
+MANAGED_WAREHOUSE_PORTS_BY_SOURCE_TYPE: dict[str, frozenset[int]] = {
+ "postgres": frozenset({5432}),
+ "trino": frozenset({443}),
+}
+
+# The dataclasses name the ports directly so a caller that already knows its source type does not
+# import the mapping above.
+MANAGED_WAREHOUSE_SOURCE_TYPES = frozenset(MANAGED_WAREHOUSE_PORTS_BY_SOURCE_TYPE)
+
+
+# The DNS zone each deployment serves managed warehouses under. It lives here rather than in
+# `presentation.views` because the warehouse-source host guard reads it, and importing a DRF
+# module from a Temporal worker's connect path is how a presentation dependency ends up in a
+# background process. `views.managed_warehouse_domain` keeps its own copy because presentation
+# cannot import a plain backend module without breaking the facade contract, and both assert the
+# same values in tests.
+_MANAGED_WAREHOUSE_DOMAINS = {
+ "US": "us.postwh.com",
+ "EU": "eu.postwh.com",
+ "DEV": "dev.postwh.com",
+}
+
+
+def managed_warehouse_domain() -> str:
+ """The zone managed warehouse hostnames are served under in this deployment."""
+ from django.conf import settings # noqa: PLC0415
+
+ deployment = (getattr(settings, "CLOUD_DEPLOYMENT", None) or "").upper()
+ return _MANAGED_WAREHOUSE_DOMAINS.get(deployment, "test.local")
+
+
+def _ingress_ips() -> tuple[str, ...]:
+ """The configured ingress address set, read per call so a settings override takes effect."""
+ from django.conf import settings # noqa: PLC0415
+
+ configured = getattr(settings, "MANAGED_WAREHOUSE_INGRESS_IPS", "") or ""
+ return tuple(ip.strip() for ip in configured.split(",") if ip.strip())
+
+
+# A ready warehouse's host changes only when an operator renames it, so a positive answer is worth
+# caching. Every other answer is short-lived: a warehouse that just became ready must not stay
+# denied for the whole positive TTL.
+_TARGET_CACHE_TTL_SECONDS = 300
+_DENIAL_CACHE_TTL_SECONDS = 30
+_CONTROL_PLANE_TIMEOUT_SECONDS = 2
+
+
+# The canonical definition is the contract, because the product's public interface exposes the
+# facade and not this module. The caching and the control-plane read live here; the type does not.
+
+
+def _normalize_host(host: str) -> str:
+ """Lowercase a hostname and drop one root-zone trailing dot, as the guard does."""
+ return host.lower().strip().rstrip(".")
+
+
+def _canonical_ip(value: str) -> str:
+ """Compare addresses as numbers, so an IPv4-mapped IPv6 form matches its IPv4 spelling."""
+ import ipaddress # noqa: PLC0415
+
+ try:
+ parsed = ipaddress.ip_address(value.strip().strip("[]"))
+ except ValueError:
+ return value
+ if isinstance(parsed, ipaddress.IPv6Address) and parsed.ipv4_mapped is not None:
+ return str(parsed.ipv4_mapped)
+ return str(parsed)
+
+
+def _cache_key(organization_id: str, source_type: str) -> str:
+ return f"managed_warehouse_target:{organization_id}:{source_type}"
+
+
+def get_managed_warehouse_target(organization_id: str | UUID, *, source_type: str) -> ManagedWarehouseTarget | None:
+ """The managed warehouse `organization_id` may reach from a `source_type` source, or None.
+
+ Returns None for a source type the managed warehouse does not serve, for an organization
+ without a ready warehouse, and when the control plane cannot answer.
+ """
+ ports = MANAGED_WAREHOUSE_PORTS_BY_SOURCE_TYPE.get(source_type)
+ if ports is None:
+ return None
+
+ org_id = str(organization_id)
+ key = _cache_key(org_id, source_type)
+ cached = _cache_get(key)
+ if isinstance(cached, ManagedWarehouseTarget):
+ return cached
+ if cached == _NO_TARGET:
+ return None
+
+ host = _read_ready_host(org_id)
+ if host is None:
+ _cache_denial(key)
+ return None
+
+ target = ManagedWarehouseTarget(host=host, ports=ports)
+ _cache_target(key, target)
+ return target
+
+
+# A sentinel is cached rather than None: Django's cache stores None as a miss, so a denial would
+# otherwise be re-read from the control plane on every dial. It is a string, not `object()`,
+# because the cache pickles its values and an unpickled sentinel is a new object. A denial
+# compared by identity would therefore never hit, and every denied dial during an outage would
+# pay the full control-plane timeout.
+_NO_TARGET = "no-managed-warehouse-target"
+
+
+def _cache_get(key: str) -> object:
+ """The cached answer for `key`, or None when absent.
+
+ A cache fault denies rather than raising: the guard runs inside a connect, and an exception
+ here would replace a clear refusal with an opaque connection error.
+ """
+ try:
+ return cache.get(key)
+ except Exception:
+ logger.exception("managed_warehouse_target_cache_read_failed")
+ return None
+
+
+def _cache_denial(key: str) -> None:
+ try:
+ cache.set(key, _NO_TARGET, _DENIAL_CACHE_TTL_SECONDS)
+ except Exception:
+ logger.exception("managed_warehouse_target_denial_cache_failed")
+
+
+def _cache_target(key: str, target: ManagedWarehouseTarget) -> None:
+ try:
+ cache.set(key, target, _TARGET_CACHE_TTL_SECONDS)
+ except Exception:
+ # A positive answer that cannot be cached still answers this dial.
+ logger.exception("managed_warehouse_target_cache_write_failed")
+
+
+def _read_ready_host(organization_id: str) -> str | None:
+ """The control plane's host for a ready warehouse, or None.
+
+ Import is local so this module's importers do not pull the presentation package, and so a
+ worker that never guards a managed source never loads it.
+ """
+ if not hogtower.is_configured():
+ return None
+
+ try:
+ response = hogtower.request("GET", organization_id, "/warehouse/status", timeout=_CONTROL_PLANE_TIMEOUT_SECONDS)
+ except http_requests.Timeout:
+ logger.warning("managed_warehouse_target_timeout", organization_id=organization_id)
+ return None
+ except http_requests.RequestException:
+ logger.warning("managed_warehouse_target_unreachable", organization_id=organization_id)
+ return None
+ except Exception:
+ # A control-plane response the adapter cannot translate is not a verdict on the
+ # organization. Deny, and keep the failure out of the caller's connect.
+ logger.exception("managed_warehouse_target_control_plane_error", organization_id=organization_id)
+ return None
+
+ if response.status_code != 200 or not isinstance(response.body, dict):
+ return None
+ return _host_from_status(organization_id, response.body)
+
+
+def _host_from_status(organization_id: str, body: dict[str, Any]) -> str | None:
+ """The ready host from a status body, refusing one that names another organization."""
+ response_org = body.get("org_id")
+ if response_org is not None and str(response_org) != str(organization_id):
+ logger.warning(
+ "managed_warehouse_target_org_mismatch",
+ requested_organization_id=str(organization_id),
+ response_organization_id=str(response_org),
+ )
+ return None
+
+ if body.get("state") != "ready":
+ return None
+
+ connection = body.get("connection")
+ if not isinstance(connection, dict):
+ return None
+ host = connection.get("host")
+ return host if isinstance(host, str) and host.strip() else None
diff --git a/products/managed_warehouse/backend/presentation/views.py b/products/managed_warehouse/backend/presentation/views.py
index 315764a4fd7f..90eed3754e9e 100644
--- a/products/managed_warehouse/backend/presentation/views.py
+++ b/products/managed_warehouse/backend/presentation/views.py
@@ -109,6 +109,12 @@ DataOpsVariant = Literal["trino", "duckdb"]
def managed_warehouse_domain() -> str:
+ """The zone managed warehouse hostnames are served under in this deployment.
+
+ The value is shared with `managed_ingress`, which the warehouse-source host guard reads. It
+ is defined here because a presentation module cannot import a plain backend module without
+ breaking the facade contract.
+ """
deployment = (getattr(settings, "CLOUD_DEPLOYMENT", None) or "").upper()
return _MANAGED_WAREHOUSE_DOMAINS.get(deployment, "test.local")
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/mixins.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/mixins.py
index e888fc4d5896..845b1b58cfba 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/mixins.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/mixins.py
@@ -138,20 +138,28 @@ class HostResolution:
addresses: tuple[str, ...] = ()
-def _is_host_safe(host: str, team_id: int) -> tuple[bool, str | None]:
+def _is_host_safe(
+ host: str, team_id: int, *, port: int | None = None, source_type: str | None = None
+) -> tuple[bool, str | None]:
"""Whether a host is safe to connect to. See `resolve_safe_host` for the policy.
+ `port` and `source_type` narrow the managed-warehouse exemption to the listener that source
+ type is served on. A caller that omits them still gets a safe answer: the exemption is then
+ unavailable, which is the refusal direction.
+
Callers that go on to open the connection themselves should use `resolve_safe_host` and
connect to the address it returns, so that the address checked is the address used.
"""
try:
- resolution = resolve_safe_host(host, team_id)
+ resolution = resolve_safe_host(host, team_id, port=port, source_type=source_type)
except TemporaryHostResolutionError as e:
return False, str(e)
return resolution.connect_host is not None, resolution.error
-def resolve_safe_host(host: str, team_id: int | None) -> HostResolution:
+def resolve_safe_host(
+ host: str, team_id: int | None, *, port: int | None = None, source_type: str | None = None
+) -> HostResolution:
"""Resolve a host to the address a connection should actually be made to.
Only enforced on cloud deployments — self-hosted instances are allowed
@@ -174,6 +182,10 @@ def resolve_safe_host(host: str, team_id: int | None) -> HostResolution:
A client that cannot dial a bare address, because it needs the name for SNI or dials several
addresses in turn, resolves the host itself and passes the answer to
`check_resolved_addresses` instead, so the set it validates is the set it dials.
+
+ A managed-warehouse host is resolved and then checked against the address it answers with, so
+ the exemption is never granted on the name alone. `port` and `source_type` select the listener
+ that exemption applies to, and omitting either leaves the host on the ordinary path.
"""
note_oob_domain(host, team_id)
@@ -199,13 +211,32 @@ def resolve_safe_host(host: str, team_id: int | None) -> HostResolution:
except ValueError:
pass
+ resolved_ips = _resolve_host_addresses(host)
+
+ # The managed-warehouse check runs on the addresses this function just resolved, and only
+ # then. A managed host answers with a private address by design, so it can never pass the
+ # public-address check below, and it must never pass on its name alone either.
+ if _managed_warehouse_exemption(host, team_id, port, source_type, resolved_ips) is not None:
+ _log_host_check(host, team_id, "allow", "postwh_managed", None, resolved_ips)
+ routable = prefer_routable_addresses(resolved_ips)
+ return HostResolution(connect_host=routable[0], error=None, addresses=tuple(routable))
+
+ return _check_resolved_ips(host, team_id, resolved_ips)
+
+
+def _resolve_host_addresses(host: str) -> list[str]:
+ """Every address `host` answers with, in resolver order.
+
+ A lookup that cannot answer raises `TemporaryHostResolutionError` so the caller retries; a
+ name that does not exist answers with an empty list, which `_check_resolved_ips` refuses.
+ """
try:
addrinfo = run_with_deadline(
lambda: socket.getaddrinfo(host, None, proto=socket.IPPROTO_TCP),
timeout_seconds=HOST_RESOLUTION_TIMEOUT_SECONDS,
thread_name="host-check-resolve",
)
- resolved_ips = [str(sockaddr[0]) for *_meta, sockaddr in addrinfo]
+ return [str(sockaddr[0]) for *_meta, sockaddr in addrinfo]
except DeadlineExceededError as e:
# A resolver that does not answer is a blip too, and `getaddrinfo` has no timeout of its own.
raise TemporaryHostResolutionError(host) from e
@@ -213,17 +244,22 @@ def resolve_safe_host(host: str, team_id: int | None) -> HostResolution:
# A resolver blip is not a verdict on the host; refusing it would disable the schema.
if is_temporary_resolution_failure(e):
raise TemporaryHostResolutionError(host) from e
- resolved_ips = []
+ return []
except UnicodeError:
# getaddrinfo IDNA-encodes the host, so a malformed hostname (e.g. a DNS label over 63
# bytes) raises UnicodeError ("label too long") instead of gaierror. Either way the host
# can't be resolved — return the actionable message rather than crashing.
- resolved_ips = []
-
- return _check_resolved_ips(host, team_id, resolved_ips)
+ return []
-def check_resolved_addresses(host: str, addresses: Sequence[str], team_id: int | None) -> HostResolution:
+def check_resolved_addresses(
+ host: str,
+ addresses: Sequence[str],
+ team_id: int | None,
+ *,
+ port: int | None = None,
+ source_type: str | None = None,
+) -> HostResolution:
"""Apply the `resolve_safe_host` policy to addresses the caller resolved itself.
For a client that looks `host` up on its own and dials that answer. `resolve_safe_host`
@@ -232,9 +268,13 @@ def check_resolved_addresses(host: str, addresses: Sequence[str], team_id: int |
guarantee: the set it validates is the set it dials, so a record that answers public on one
lookup and private on the next has no second lookup to slip past.
- The same exemptions as `resolve_safe_host` apply, and `addresses` is returned whole so an
- exempt caller still dials what it resolved. An empty `addresses` is a failed lookup and is
+ The same policy as `resolve_safe_host` applies, and `addresses` is returned whole so an
+ approved caller still dials what it resolved. An empty `addresses` is a failed lookup and is
refused, because letting the connection library resolve again would reopen the gap.
+
+ A managed-warehouse host is approved only when the addresses the caller is about to dial are
+ all configured ingress addresses, so an answer the caller resolved itself is checked exactly
+ as one this module resolved would be.
"""
note_oob_domain(host, team_id, stage="oob_domain_resolved")
@@ -247,13 +287,25 @@ def check_resolved_addresses(host: str, addresses: Sequence[str], team_id: int |
_log_host_check(host, team_id, "allow", exempt_stage, None)
return HostResolution(connect_host=host, error=None, addresses=tuple(addresses))
- return _check_resolved_ips(host, team_id, list(addresses))
+ checked = list(addresses)
+ if _managed_warehouse_exemption(host, team_id, port, source_type, checked) is not None:
+ _log_host_check(host, team_id, "allow", "postwh_managed", None, checked)
+ return HostResolution(connect_host=host, error=None, addresses=tuple(checked))
+
+ return _check_resolved_ips(host, team_id, checked)
_resolve_hostaddr_with_timeout = resolve_psycopg_hostaddr_with_timeout
-def pinned_host_kwargs(host: str, *, port: int, connect_timeout: float, team_id: int | None) -> dict[str, str]:
+def pinned_host_kwargs(
+ host: str,
+ *,
+ port: int,
+ connect_timeout: float,
+ team_id: int | None,
+ source_type: str | None = None,
+) -> dict[str, str]:
"""Resolve `host` once, validate the answer, and return the libpq `host`/`hostaddr` pair that
dials exactly those addresses.
@@ -280,7 +332,7 @@ def pinned_host_kwargs(host: str, *, port: int, connect_timeout: float, team_id:
return {"host": host}
addresses = _resolve_hostaddr_with_timeout(host, port, connect_timeout, raise_on_temporary_failure=True) or []
- resolution = check_resolved_addresses(host, addresses, team_id)
+ resolution = check_resolved_addresses(host, addresses, team_id, port=port, source_type=source_type)
if resolution.connect_host is None:
raise HostNotAllowedError(f"{DATABASE_HOST_NOT_ALLOWED_ERROR}: {resolution.error or _INTERNAL_IP_ERROR}")
if not resolution.addresses:
@@ -333,6 +385,11 @@ def bracket_host(host: str) -> str:
return f"[{stripped}]" if parsed.version == 6 else host
+# A DNS-1123 label. Stricter than `_HOST_LABEL`, which admits an underscore because some
+# providers publish records with one. A managed warehouse host is a name PostHog issues, so
+# nothing outside the strict set can be one.
+_DNS_LABEL = re.compile(r"^(?!-)[a-z0-9-]{1,63}(?<!-)\Z")
+
_HOST_LABEL = re.compile(r"^(?!-)[a-z0-9_-]{1,63}(?<!-)\Z")
@@ -372,7 +429,12 @@ def _single_host_error(host: str) -> str | None:
def _host_check_exemption(host: str, team_id: int | None) -> str | None:
- """Return the stage name that exempts this host from the check, or None when it applies."""
+ """Return the stage name that skips the address check entirely, or None when it applies.
+
+ Only stages that do not depend on where the name resolves live here. A managed-warehouse host
+ is checked against the address it resolves to instead, in `_managed_warehouse_exemption`, so
+ that an exemption can never be granted on a name alone.
+ """
if not is_cloud():
return "not_cloud"
@@ -382,14 +444,106 @@ def _host_check_exemption(host: str, team_id: int | None) -> str | None:
if team_id is not None and is_team_allowlisted_for_internal_hosts(team_id):
return "team_allowlist"
- # PostHog-managed DuckLake hosts resolve to internal IPs but are safe. A suffix test is
- # enough only because callers refuse anything that is not one hostname before asking.
- if _normalize_host(host).endswith(".postwh.com"):
- return "postwh_managed"
-
return None
+def _managed_warehouse_exemption(
+ host: str,
+ team_id: int | None,
+ port: int | None,
+ source_type: str | None,
+ addresses: list[str],
+) -> str | None:
+ """Whether `host` is this team's own managed warehouse at `addresses`, or None when it is not.
+
+ The exemption the guard grants here is narrower than "this is a PostHog hostname". Every part
+ of the destination has to agree with what the control plane says the team owns:
+
+ - the host equals the control plane's host for the team's organization, so a guessed tenant
+ label is refused and a source cannot reach a neighbour's warehouse;
+ - the port is one the managed warehouse serves this source type on, so a Postgres source
+ cannot be pointed at the gateway listener;
+ - every resolved address is one of the configured ingress addresses, so a `postwh.com` record
+ that is dangling or has been taken over cannot redirect the connection elsewhere.
+
+ A missing port or source type denies, because the check cannot then confirm the destination
+ is one the managed warehouse serves.
+ """
+ if not is_cloud():
+ return None
+ if port is None or source_type is None:
+ return None
+ if not _is_managed_warehouse_host(host):
+ return None
+
+ from products.managed_warehouse.backend.facade.api import (
+ get_managed_warehouse_target, # noqa: PLC0415
+ managed_warehouse_ingress_ips, # noqa: PLC0415
+ )
+ from products.warehouse_sources.backend.facade.models import MANAGED_WAREHOUSE_SOURCE_PREFIX # noqa: PLC0415
+
+ organization_id = _managed_warehouse_organization_id(team_id)
+ if organization_id is None:
+ return None
+
+ target = get_managed_warehouse_target(organization_id, source_type=source_type)
+ if target is None:
+ return None
+ if not target.accepts(_normalize_host(host), port, addresses, managed_warehouse_ingress_ips()):
+ return None
+
+ logger.info(
+ "data_imports.managed_warehouse_exemption",
+ host=host,
+ port=port,
+ source_type=source_type,
+ team_id=team_id,
+ prefix=MANAGED_WAREHOUSE_SOURCE_PREFIX,
+ )
+ return "postwh_managed"
+
+
+def _is_managed_warehouse_host(host: str) -> bool:
+ """Whether `host` has the shape of a managed warehouse tenant endpoint.
+
+ This is a cheap prefilter in front of the control-plane read, never an authorization on its
+ own. It matches exactly one DNS label under one of the deployment's `dw` zones: the apex, a
+ nested subdomain and a trailing dot all fail.
+ """
+ normalized = _normalize_host(host)
+ if normalized != host:
+ # The comparison in `_managed_warehouse_exemption` uses the normalized form, so a host
+ # that differs from it is refused here rather than normalized twice.
+ return False
+ for suffix in _managed_warehouse_host_suffixes():
+ if not normalized.endswith(suffix):
+ continue
+ label = normalized[: -len(suffix)]
+ if _DNS_LABEL.match(label):
+ return True
+ return False
+
+
+def _managed_warehouse_host_suffixes() -> tuple[str, ...]:
+ """The `dw` zones this deployment serves managed warehouses under."""
+ from products.managed_warehouse.backend.facade.api import managed_warehouse_domain # noqa: PLC0415
+
+ return (f".dw.{managed_warehouse_domain()}",)
+
+
+def _managed_warehouse_organization_id(team_id: int | None) -> str | None:
+ if team_id is None:
+ return None
+ from products.managed_warehouse.backend.facade.api import get_org_id_for_team # noqa: PLC0415
+
+ try:
+ return get_org_id_for_team(team_id)
+ except Exception:
+ # A team lookup that fails denies the exemption rather than failing the connect.
+ logger.exception("data_imports.managed_warehouse_team_lookup_failed", team_id=team_id)
+ return None
+
+
def note_oob_domain(host: str, team_id: int | None, stage: str = "oob_domain") -> None:
"""Record a lookup of an out-of-band interaction domain, without refusing it.
@@ -905,12 +1059,18 @@ class ValidateDatabaseHostMixin:
"""Mixin for database-based sources to validate connection host"""
def is_database_host_valid(
- self, host: str, team_id: int, using_ssh_tunnel: bool = False
+ self,
+ host: str,
+ team_id: int,
+ using_ssh_tunnel: bool = False,
+ *,
+ port: int | None = None,
+ source_type: str | None = None,
) -> tuple[bool, str | None]:
if using_ssh_tunnel:
return True, None
- is_valid, error = _is_host_safe(host, team_id)
+ is_valid, error = _is_host_safe(host, team_id, port=port, source_type=source_type)
if not is_valid and error == _INTERNAL_IP_ERROR:
# Direct (non-tunneled) connection, so reaching a private database through a public
# SSH bastion is a genuine workaround worth pointing at.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_self_managed.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_self_managed.py
index 3a255bffd138..164db21c0586 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_self_managed.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_self_managed.py
@@ -5,10 +5,13 @@ from datetime import UTC, date, datetime
from decimal import Decimal
from pathlib import Path
from tempfile import TemporaryDirectory
-from typing import Any
+from typing import TYPE_CHECKING, Any
import pytest
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, patch
+
+if TYPE_CHECKING:
+ from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.globs import _State
import pyarrow as pa
import structlog
@@ -45,6 +48,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.obj
iter_file_tables,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.object_store.globs import (
+ ClickHouseGlob,
compile_clickhouse_glob,
glob_key_prefix,
glob_listing_prefix,
@@ -289,15 +293,25 @@ class TestClickHouseGlob:
("eight_stars", "*a*a*a*a*a*a*a*a*b.csv"),
]
)
- def test_adversarial_match_time(self, _name: str, pattern: str) -> None:
+ def test_adversarial_match_work_is_bounded(self, _name: str, pattern: str) -> None:
glob = compile_clickhouse_glob(pattern)
- path = "a" * 200 + ".csv"
+ # Short enough that a backtracking matcher returns and fails the assertions below instead of hanging.
+ path = "a" * 20 + ".csv"
+ closure = ClickHouseGlob._epsilon_closure
+ state_counts: list[int] = []
- started = time.perf_counter()
- for _ in range(1000):
+ def counting_closure(self: ClickHouseGlob, states: set[_State]) -> set[_State]:
+ result = closure(self, states)
+ state_counts.append(len(result))
+ return result
+
+ with patch.object(ClickHouseGlob, "_epsilon_closure", counting_closure):
assert glob.matches(path) is False
- assert time.perf_counter() - started < 2
- assert glob.matches("a" * 200 + "b.csv") is True
+
+ # One state set per character, each no larger than the token count, keeps the work linear.
+ assert 0 < len(state_counts) <= len(path) + 1
+ assert max(state_counts) <= len(glob.tokens) + 1
+ assert glob.matches("a" * 20 + "b.csv") is True
@parameterized.expand(
[
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/test_managed_warehouse_exemption.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test_managed_warehouse_exemption.py
new file mode 100644
index 000000000000..a8e219560dcd
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test_managed_warehouse_exemption.py
@@ -0,0 +1,702 @@
+"""Tests for the managed-warehouse host exemption.
+
+The exemption replaces a `.postwh.com` suffix test that skipped the address check on any port,
+for any organization, on any subdomain. Every test here pins one of the conditions that replaced
+it, because a regression in any single one re-opens the same hole:
+
+- the host must be the organization's own control-plane host, not a name of the right shape;
+- the port must be one the managed warehouse serves that source type on;
+- every resolved address must be a configured ingress address;
+- the check must run on the addresses the connection will actually dial.
+
+Each test states the attack it prevents, so a later reader can tell a real regression from a
+value that simply changed.
+"""
+
+import pytest
+from unittest.mock import patch
+
+from django.test import SimpleTestCase, override_settings
+
+from parameterized import parameterized
+
+from products.managed_warehouse.backend.facade.api import get_managed_warehouse_target
+from products.managed_warehouse.backend.facade.contracts import ManagedWarehouseTarget
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import (
+ HostNotAllowedError,
+ _is_managed_warehouse_host,
+ check_resolved_addresses,
+ pinned_host_kwargs,
+ resolve_safe_host,
+)
+
+_MIXINS_MODULE = "products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins"
+_INGRESS_MODULE = "products.managed_warehouse.backend.managed_ingress"
+
+# Two addresses, because the incident observed the shared load balancer answering on both.
+_NLB_A = "10.1.0.1"
+_NLB_B = "10.1.0.2"
+_TENANT_HOST = "acme.dw.us.postwh.com"
+_INGRESS_SET = (_NLB_A, _NLB_B)
+
+
+def _target(host: str = _TENANT_HOST, ports: frozenset[int] = frozenset({5432})) -> ManagedWarehouseTarget:
+ return ManagedWarehouseTarget(host=host, ports=ports)
+
+
+class TestManagedWarehouseAccepts(SimpleTestCase):
+ """`ManagedWarehouseTarget.accepts` is the whole policy, so test it without the guard."""
+
+ @override_settings(MANAGED_WAREHOUSE_INGRESS_IPS=f"{_NLB_A},{_NLB_B}")
+ def test_the_organizations_own_host_on_its_own_port_at_an_ingress_address_is_accepted(self) -> None:
+ assert _target().accepts(_TENANT_HOST, 5432, [_NLB_A], _INGRESS_SET) is True
+
+ @override_settings(MANAGED_WAREHOUSE_INGRESS_IPS=f"{_NLB_A},{_NLB_B}")
+ def test_every_resolved_address_is_allowed(self) -> None:
+ assert _target().accepts(_TENANT_HOST, 5432, [_NLB_A, _NLB_B], _INGRESS_SET) is True
+
+ @override_settings(MANAGED_WAREHOUSE_INGRESS_IPS=f"{_NLB_A},{_NLB_B}")
+ def test_a_neighbouring_tenant_host_is_refused(self) -> None:
+ # The signal the actor used to enumerate tenants: a guessed label that still resolves
+ # to the shared load balancer must not be exempted.
+ assert _target().accepts("zzq7x9neg.dw.us.postwh.com", 5432, [_NLB_A], _INGRESS_SET) is False
+
+ @override_settings(MANAGED_WAREHOUSE_INGRESS_IPS=f"{_NLB_A},{_NLB_B}")
+ def test_a_different_tenant_at_the_same_address_is_refused(self) -> None:
+ # Both tenant names answer with the same ingress address, so the address check alone
+ # cannot separate them. The host comparison is what stops one tenant reaching another.
+ assert _target().accepts("globex.dw.us.postwh.com", 5432, [_NLB_A], _INGRESS_SET) is False
+
+ @override_settings(MANAGED_WAREHOUSE_INGRESS_IPS=f"{_NLB_A},{_NLB_B}")
+ def test_a_port_the_source_type_is_not_served_on_is_refused(self) -> None:
+ # A Postgres source must not be able to reach the gateway listener on the same address.
+ assert _target().accepts(_TENANT_HOST, 443, [_NLB_A], _INGRESS_SET) is False
+
+ @override_settings(MANAGED_WAREHOUSE_INGRESS_IPS=f"{_NLB_A},{_NLB_B}")
+ def test_a_missing_port_is_refused(self) -> None:
+ # `None` must deny. A branch that treats it as "no port to check" restores the
+ # any-port hole the exemption had.
+ assert _target().accepts(_TENANT_HOST, None, [_NLB_A], _INGRESS_SET) is False
+
+ @override_settings(MANAGED_WAREHOUSE_INGRESS_IPS=f"{_NLB_A},{_NLB_B}")
+ def test_an_address_outside_the_ingress_set_is_refused(self) -> None:
+ # A dangling or taken-over `postwh.com` record answers with an address that is not ours.
+ assert _target().accepts(_TENANT_HOST, 5432, ["10.0.0.7"], _INGRESS_SET) is False
+
+ @override_settings(MANAGED_WAREHOUSE_INGRESS_IPS=f"{_NLB_A},{_NLB_B}")
+ def test_one_foreign_address_in_the_set_is_refused(self) -> None:
+ # A mixed answer is the DNS-rebinding shape: one approved address to pass a check, one
+ # the connection could actually reach. Every address must agree.
+ assert _target().accepts(_TENANT_HOST, 5432, [_NLB_A, "10.0.0.7"], _INGRESS_SET) is False
+
+ @override_settings(MANAGED_WAREHOUSE_INGRESS_IPS=f"{_NLB_A},{_NLB_B}")
+ def test_an_empty_address_set_is_refused(self) -> None:
+ # An empty set is a failed lookup, not an approval.
+ assert _target().accepts(_TENANT_HOST, 5432, [], _INGRESS_SET) is False
+
+ def test_an_unconfigured_ingress_set_refuses_everything(self) -> None:
+ # The fail-closed default. A deployment that has not been given the address set refuses
+ # managed warehouse hosts rather than trusting the name. The caller passes the set, so an
+ # empty one is the refusal, not a setting the contract reads.
+ assert _target().accepts(_TENANT_HOST, 5432, [_NLB_A], ()) is False
+
+ @override_settings(MANAGED_WAREHOUSE_INGRESS_IPS=_NLB_A)
+ def test_an_ipv4_mapped_form_matches_its_ipv4_spelling(self) -> None:
+ # The resolver can answer in either form, so a string comparison would refuse a real
+ # address or, worse, accept the wrong one.
+ assert _target().accepts(_TENANT_HOST, 5432, [f"::ffff:{_NLB_A}"], _INGRESS_SET) is True
+
+
+class TestManagedWarehouseHostShape(SimpleTestCase):
+ """The shape check is a prefilter in front of the control-plane read, not an authorization."""
+
+ @parameterized.expand(
+ [
+ ("us", "US", "acme.dw.us.postwh.com", "acme.dw.eu.postwh.com"),
+ ("eu", "EU", "acme.dw.eu.postwh.com", "acme.dw.us.postwh.com"),
+ ("dev", "DEV", "acme.dw.dev.postwh.com", "acme.dw.us.postwh.com"),
+ ]
+ )
+ def test_one_label_under_this_deployments_zone_has_the_shape_and_another_does_not(
+ self, _name: str, deployment: str, own_zone_host: str, other_zone_host: str
+ ) -> None:
+ # The zone is derived from the deployment, so a host under a different environment's zone
+ # must not qualify: a dev deployment must not exempt a production tenant name.
+ with override_settings(CLOUD_DEPLOYMENT=deployment):
+ assert _is_managed_warehouse_host(own_zone_host) is True
+ assert _is_managed_warehouse_host(other_zone_host) is False
+
+ @parameterized.expand(
+ [
+ ("apex", "dw.us.postwh.com"),
+ ("nested", "a.b.dw.us.postwh.com"),
+ ("legacy_no_dw", "acme.us.postwh.com"),
+ ("suffix_only", "something.postwh.com"),
+ ("trailing_dot", "acme.dw.us.postwh.com."),
+ ("uppercase", "ACME.DW.US.POSTWH.COM"),
+ ("host_list", "10.0.0.5,acme.dw.us.postwh.com"),
+ ("host_port", "acme.dw.us.postwh.com:5432"),
+ ("socket_path", "/var/run/acme.dw.us.postwh.com"),
+ ("lookalike_tld", "acme.dw.us.postwh.com.evil.example"),
+ ("leading_hyphen", "-acme.dw.us.postwh.com"),
+ ("trailing_hyphen", "acme-.dw.us.postwh.com"),
+ ("underscore", "acme_.dw.us.postwh.com"),
+ ("non_ascii", "\u212a.dw.us.postwh.com"),
+ ]
+ )
+ @override_settings(CLOUD_DEPLOYMENT="US")
+ def test_everything_else_fails_the_shape(self, _name: str, host: str) -> None:
+ assert _is_managed_warehouse_host(host) is False
+
+
+class TestManagedWarehouseExemptionInTheGuard(SimpleTestCase):
+ """The guard is what an attacker reaches. These are the regression tests for the incident."""
+
+ @override_settings(CLOUD_DEPLOYMENT="US", MANAGED_WAREHOUSE_INGRESS_IPS=f"{_NLB_A},{_NLB_B}")
+ def test_a_guessed_tenant_label_resolving_to_the_load_balancer_is_refused(self) -> None:
+ # The incident: a customer-controlled host that resolves to the managed ingress. Before
+ # the fix the suffix alone exempted it, so the connector dialled the load balancer.
+ with (
+ patch(f"{_MIXINS_MODULE}.socket.getaddrinfo", return_value=[(None, None, None, None, (_NLB_A, 0))]),
+ patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),
+ patch(f"{_MIXINS_MODULE}.logger"),
+ patch(
+ f"{_INGRESS_MODULE}.get_managed_warehouse_target",
+ return_value=_target(),
+ ),
+ ):
+ resolution = resolve_safe_host("zzq7x9neg.dw.us.postwh.com", 999, port=5432, source_type="postgres")
+
+ assert resolution.connect_host is None
+ assert resolution.error is not None
+
+ @override_settings(CLOUD_DEPLOYMENT="US", MANAGED_WAREHOUSE_INGRESS_IPS=f"{_NLB_A},{_NLB_B}")
+ def test_imds_behind_a_postwh_name_is_refused(self) -> None:
+ # The prize the campaign was after: a name that answers with the instance metadata
+ # address must not be exempt, however well formed it looks.
+ with (
+ patch(
+ f"{_MIXINS_MODULE}.socket.getaddrinfo",
+ return_value=[(None, None, None, None, ("169.254.169.254", 0))],
+ ),
+ patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),
+ patch(f"{_MIXINS_MODULE}.logger"),
+ ):
+ resolution = resolve_safe_host("acme.dw.us.postwh.com", 999, port=5432, source_type="postgres")
+
+ assert resolution.connect_host is None
+ assert resolution.error is not None
+
+ @override_settings(CLOUD_DEPLOYMENT="US", MANAGED_WAREHOUSE_INGRESS_IPS=f"{_NLB_A},{_NLB_B}")
+ def test_a_source_type_the_warehouse_does_not_serve_is_refused(self) -> None:
+ # No managed warehouse serves MySQL, so the exemption must not be reachable from that
+ # source even with a matching host, port and address.
+ with (
+ patch(f"{_MIXINS_MODULE}.socket.getaddrinfo", return_value=[(None, None, None, None, (_NLB_A, 0))]),
+ patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),
+ patch(f"{_MIXINS_MODULE}.logger"),
+ ):
+ resolution = resolve_safe_host("acme.dw.us.postwh.com", 999, port=3306, source_type="mysql")
+
+ assert resolution.connect_host is None
+
+ @override_settings(CLOUD_DEPLOYMENT="US", MANAGED_WAREHOUSE_INGRESS_IPS=f"{_NLB_A},{_NLB_B}")
+ def test_a_missing_source_type_is_refused(self) -> None:
+ # A caller that forgets to name its source type loses the exemption, which is the safe
+ # direction, and this is the test that stops it becoming an accidental bypass.
+ with patch(f"{_MIXINS_MODULE}.socket.getaddrinfo", return_value=[(None, None, None, None, (_NLB_A, 0))]):
+ resolution = resolve_safe_host("acme.dw.us.postwh.com", 999, port=5432)
+
+ assert resolution.connect_host is None
+
+ @override_settings(CLOUD_DEPLOYMENT="US", MANAGED_WAREHOUSE_INGRESS_IPS=f"{_NLB_A},{_NLB_B}")
+ def test_a_real_managed_warehouse_host_is_pinned_to_the_ingress_address(self) -> None:
+ # The exemption must still work for the customer it exists for. Pinning matters: the
+ # approved address is the address dialled, so a second lookup cannot answer differently.
+ with (
+ patch(f"{_MIXINS_MODULE}.socket.getaddrinfo", return_value=[(None, None, None, None, (_NLB_A, 0))]),
+ patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),
+ patch(f"{_MIXINS_MODULE}.logger"),
+ patch(
+ f"{_INGRESS_MODULE}.get_managed_warehouse_target",
+ return_value=_target(),
+ ),
+ ):
+ resolution = resolve_safe_host(_TENANT_HOST, 999, port=5432, source_type="postgres")
+
+ assert resolution.error is None
+ assert resolution.addresses == (_NLB_A,)
+ assert resolution.connect_host == _NLB_A
+
+ @override_settings(CLOUD_DEPLOYMENT="US", MANAGED_WAREHOUSE_INGRESS_IPS=f"{_NLB_A},{_NLB_B}")
+ def test_the_checked_addresses_are_the_addresses_returned_for_a_managed_host(self) -> None:
+ # `check_resolved_addresses` is the path a client that resolves the name itself takes.
+ # The addresses it approves must be the ones it hands back.
+ with (
+ patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),
+ patch(f"{_MIXINS_MODULE}.logger"),
+ patch(
+ f"{_INGRESS_MODULE}.get_managed_warehouse_target",
+ return_value=_target(),
+ ),
+ ):
+ resolution = check_resolved_addresses(
+ _TENANT_HOST, [_NLB_A, _NLB_B], 999, port=5432, source_type="postgres"
+ )
+
+ assert resolution.connect_host == _TENANT_HOST
+ assert resolution.addresses == (_NLB_A, _NLB_B)
+
+ @override_settings(CLOUD_DEPLOYMENT="US", MANAGED_WAREHOUSE_INGRESS_IPS=f"{_NLB_A},{_NLB_B}")
+ def test_a_mixed_address_set_is_refused_for_a_managed_host(self) -> None:
+ with (
+ patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),
+ patch(f"{_MIXINS_MODULE}.logger"),
+ patch(
+ f"{_INGRESS_MODULE}.get_managed_warehouse_target",
+ return_value=_target(),
+ ),
+ ):
+ resolution = check_resolved_addresses(
+ _TENANT_HOST, [_NLB_A, "10.0.0.7"], 999, port=5432, source_type="postgres"
+ )
+
+ assert resolution.connect_host is None
+
+ @override_settings(CLOUD_DEPLOYMENT="US", MANAGED_WAREHOUSE_INGRESS_IPS=f"{_NLB_A},{_NLB_B}")
+ def test_an_organization_without_a_warehouse_is_refused(self) -> None:
+ # An organization that has no ready warehouse must not reach the shared ingress at all,
+ # which is what stops the ingress doubling as a probing oracle.
+ with (
+ patch(f"{_MIXINS_MODULE}.socket.getaddrinfo", return_value=[(None, None, None, None, (_NLB_A, 0))]),
+ patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),
+ patch(f"{_MIXINS_MODULE}.logger"),
+ patch(f"{_INGRESS_MODULE}.get_managed_warehouse_target", return_value=None),
+ ):
+ resolution = resolve_safe_host(_TENANT_HOST, 999, port=5432, source_type="postgres")
+
+ assert resolution.connect_host is None
+
+ @override_settings(CLOUD_DEPLOYMENT="US", MANAGED_WAREHOUSE_INGRESS_IPS=f"{_NLB_A},{_NLB_B}")
+ def test_a_team_that_cannot_be_resolved_to_an_organization_is_refused(self) -> None:
+ # `team_id` is optional on several paths. A missing or unresolvable team must deny the
+ # exemption rather than skip the ownership question.
+ with patch(f"{_MIXINS_MODULE}.socket.getaddrinfo", return_value=[(None, None, None, None, (_NLB_A, 0))]):
+ resolution = resolve_safe_host(_TENANT_HOST, None, port=5432, source_type="postgres")
+
+ assert resolution.connect_host is None
+
+ @override_settings(CLOUD_DEPLOYMENT="US", MANAGED_WAREHOUSE_INGRESS_IPS=f"{_NLB_A},{_NLB_B}")
+ def test_a_control_plane_failure_denies_instead_of_raising(self) -> None:
+ # A control-plane outage must not fail the connect with an unexpected exception, and must
+ # not grant the exemption either.
+ with (
+ patch(f"{_MIXINS_MODULE}.socket.getaddrinfo", return_value=[(None, None, None, None, (_NLB_A, 0))]),
+ patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),
+ patch(f"{_MIXINS_MODULE}.logger"),
+ patch(f"{_INGRESS_MODULE}.get_managed_warehouse_target", return_value=None),
+ ):
+ resolution = resolve_safe_host(_TENANT_HOST, 999, port=5432, source_type="postgres")
+
+ assert resolution.connect_host is None
+ assert resolution.error is not None
+
+ @override_settings(CLOUD_DEPLOYMENT="US", MANAGED_WAREHOUSE_INGRESS_IPS=f"{_NLB_A},{_NLB_B}")
+ def test_the_team_allowlist_still_bypasses_the_address_check(self) -> None:
+ # The internal-team exemption is unchanged, and must keep working for team 2 in US.
+ with patch(f"{_MIXINS_MODULE}.logger"):
+ resolution = check_resolved_addresses("db.internal.example.com", [_NLB_A], team_id=2)
+
+ assert resolution.connect_host == "db.internal.example.com"
+
+ @override_settings(CLOUD_DEPLOYMENT="US", MANAGED_WAREHOUSE_INGRESS_IPS=f"{_NLB_A},{_NLB_B}")
+ def test_a_managed_host_is_refused_when_the_ingress_set_is_empty(self) -> None:
+ # Fail closed: shipping the code before the address set must refuse rather than trust.
+ with (
+ patch(f"{_MIXINS_MODULE}.socket.getaddrinfo", return_value=[(None, None, None, None, (_NLB_A, 0))]),
+ patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),
+ patch(f"{_MIXINS_MODULE}.logger"),
+ patch(f"{_INGRESS_MODULE}.get_managed_warehouse_target", return_value=_target()),
+ override_settings(MANAGED_WAREHOUSE_INGRESS_IPS=""),
+ ):
+ resolution = resolve_safe_host(_TENANT_HOST, 999, port=5432, source_type="postgres")
+
+ assert resolution.connect_host is None
+
+
+class TestManagedWarehouseTargetRead(SimpleTestCase):
+ """The control-plane read: what it accepts as a ready target, and what it refuses."""
+
+ @override_settings(MANAGED_WAREHOUSE_INGRESS_IPS=f"{_NLB_A},{_NLB_B}")
+ def test_a_ready_warehouse_yields_its_host(self) -> None:
+ with (
+ patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,
+ patch(f"{_INGRESS_MODULE}.cache"),
+ ):
+ hogtower.is_configured.return_value = True
+ hogtower.request.return_value = _response(
+ {"org_id": "org-1", "state": "ready", "connection": {"host": _TENANT_HOST}}
+ )
+ target = get_managed_warehouse_target("org-1", source_type="postgres")
+
+ assert target is not None
+ assert target.host == _TENANT_HOST
+ assert target.ports == frozenset({5432})
+
+ @parameterized.expand(
+ [
+ ("provisioning", {"state": "provisioning", "connection": {"host": _TENANT_HOST}}),
+ ("deleting", {"state": "deleting", "connection": {"host": _TENANT_HOST}}),
+ ("failed", {"state": "failed", "connection": {"host": _TENANT_HOST}}),
+ ("ready_without_connection", {"state": "ready"}),
+ ("ready_with_blank_host", {"state": "ready", "connection": {"host": " "}}),
+ ]
+ )
+ def test_a_warehouse_that_is_not_ready_yields_no_target(self, _name: str, body: dict) -> None:
+ with (
+ patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,
+ patch(f"{_INGRESS_MODULE}.cache"),
+ ):
+ hogtower.is_configured.return_value = True
+ hogtower.request.return_value = _response({"org_id": "org-1", **body})
+ assert get_managed_warehouse_target("org-1", source_type="postgres") is None
+
+ def test_a_response_naming_another_organization_is_refused(self) -> None:
+ # A control-plane bug or a misrouted proxy must never let one tenant's host answer for
+ # another, which would be a cross-tenant read path.
+ with (
+ patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,
+ patch(f"{_INGRESS_MODULE}.cache"),
+ patch(f"{_INGRESS_MODULE}.logger"),
+ ):
+ hogtower.is_configured.return_value = True
+ hogtower.request.return_value = _response(
+ {"org_id": "org-2", "state": "ready", "connection": {"host": "other.dw.us.postwh.com"}}
+ )
+ assert get_managed_warehouse_target("org-1", source_type="postgres") is None
+
+ def test_a_source_type_the_warehouse_does_not_serve_is_not_read(self) -> None:
+ # No control-plane call at all: the mapping decides before anything leaves the process.
+ with (
+ patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,
+ patch(f"{_INGRESS_MODULE}.cache"),
+ ):
+ assert get_managed_warehouse_target("org-1", source_type="mysql") is None
+ hogtower.request.assert_not_called()
+
+ def test_an_unconfigured_control_plane_yields_no_target(self) -> None:
+ with (
+ patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,
+ patch(f"{_INGRESS_MODULE}.cache"),
+ ):
+ hogtower.is_configured.return_value = False
+ assert get_managed_warehouse_target("org-1", source_type="postgres") is None
+
+ def test_a_control_plane_timeout_is_denied_and_cached_briefly(self) -> None:
+ # An outage must not turn every connect into a full-timeout call to an already-sick
+ # service, so a denial is cached for a short window.
+ import requests
+
+ with (
+ patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,
+ patch(f"{_INGRESS_MODULE}.cache") as cache,
+ patch(f"{_INGRESS_MODULE}.logger"),
+ ):
+ hogtower.is_configured.return_value = True
+ hogtower.request.side_effect = requests.Timeout()
+ assert get_managed_warehouse_target("org-1", source_type="postgres") is None
+
+ assert cache.set.called
+
+ def test_a_cached_target_is_returned_without_a_control_plane_call(self) -> None:
+ cached = _target()
+ with (
+ patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,
+ patch(f"{_INGRESS_MODULE}.cache") as cache,
+ ):
+ cache.get.return_value = cached
+ assert get_managed_warehouse_target("org-1", source_type="postgres") is cached
+ hogtower.request.assert_not_called()
+
+
+def _response(body: dict):
+ # The control-plane read only checks status_code and body, so a small local object stands in
+ # for the translator's response type without importing the presentation layer.
+ return type("Response", (), {"status_code": 200, "body": body})()
+
+
+class TestNoImportOfTheManagedWarehouseProductAtModuleLoad(SimpleTestCase):
+ """The guard must not pull the control-plane read into every worker that loads it.
+
+ `mixins` is imported when the Temporal worker starts. The product's Django app registry
+ (models, apps) loads with `django.setup()` whatever this file does, so the assertion names
+ the modules the guard itself could add: the presentation layer and the control-plane read.
+ A module-level import of those would cost every worker and would fail at startup rather
+ than at the one call site that needs them.
+ """
+
+ def test_importing_the_guard_does_not_load_the_control_plane_path(self) -> None:
+ import sys
+ import subprocess
+
+ code = (
+ "import os, sys, django\n"
+ "os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'posthog.settings')\n"
+ "django.setup()\n"
+ "before = set(sys.modules)\n"
+ "from products.warehouse_sources.backend.temporal.data_imports.sources.common import mixins\n"
+ "added = set(sys.modules) - before\n"
+ "forbidden = sorted(m for m in added if m in {\n"
+ " 'products.managed_warehouse.backend.presentation.views',\n"
+ " 'products.managed_warehouse.backend.presentation.hogtower',\n"
+ " 'products.managed_warehouse.backend.managed_ingress',\n"
+ "})\n"
+ "assert not forbidden, forbidden\n"
+ "print('clean')\n"
+ )
+ result = subprocess.run(
+ [sys.executable, "-c", code], capture_output=True, text=True, cwd=str(_repo_root()), timeout=180
+ )
+ assert result.returncode == 0, result.stderr[-2000:]
+ assert "clean" in result.stdout
+
+
+def _repo_root():
+ from pathlib import Path
+
+ return Path(__file__).resolve().parents[7]
+
+
+class TestPinnedHostKwargsForAManagedWarehouse(SimpleTestCase):
+ """libpq pairs `host` and `hostaddr` positionally, so the pin is what the driver dials.
+
+ The exemption is only worth anything if the approved address is the address connected to.
+ Handing libpq the hostname alone would let it resolve a second time, and a record that
+ answers with the ingress address for the check and something else for the connect would
+ slip through.
+ """
+
+ @override_settings(CLOUD_DEPLOYMENT="US", MANAGED_WAREHOUSE_INGRESS_IPS=f"{_NLB_A},{_NLB_B}")
+ def test_the_hostname_keeps_sni_and_the_address_is_pinned(self) -> None:
+ with (
+ patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),
+ patch(f"{_MIXINS_MODULE}.logger"),
+ patch(f"{_INGRESS_MODULE}.get_managed_warehouse_target", return_value=_target()),
+ patch(f"{_MIXINS_MODULE}.host_lookup_is_skipped", return_value=False),
+ patch(
+ f"{_MIXINS_MODULE}._resolve_hostaddr_with_timeout",
+ return_value=[_NLB_A, _NLB_B],
+ ),
+ ):
+ kwargs = pinned_host_kwargs(_TENANT_HOST, port=5432, connect_timeout=5, team_id=999, source_type="postgres")
+
+ # The name is repeated once per address so libpq can pair them, and it still carries the
+ # SNI and certificate name a pin to a bare address would lose.
+ assert kwargs["host"] == f"{_TENANT_HOST},{_TENANT_HOST}"
+ assert kwargs["hostaddr"] == f"{_NLB_A},{_NLB_B}"
+
+ @override_settings(CLOUD_DEPLOYMENT="US", MANAGED_WAREHOUSE_INGRESS_IPS=f"{_NLB_A},{_NLB_B}")
+ def test_a_managed_host_whose_lookup_fails_is_refused_rather_than_resolved_again(self) -> None:
+ # An empty answer is a failed lookup. Falling back to the bare hostname would hand the
+ # resolution, and the decision, back to libpq.
+ with (
+ patch(f"{_MIXINS_MODULE}._managed_warehouse_organization_id", return_value="org-1"),
+ patch(f"{_MIXINS_MODULE}.logger"),
+ patch(f"{_INGRESS_MODULE}.get_managed_warehouse_target", return_value=_target()),
+ patch(f"{_MIXINS_MODULE}.host_lookup_is_skipped", return_value=False),
+ patch(f"{_MIXINS_MODULE}._resolve_hostaddr_with_timeout", return_value=[]),
+ ):
+ with pytest.raises(HostNotAllowedError):
+ pinned_host_kwargs(_TENANT_HOST, port=5432, connect_timeout=5, team_id=999, source_type="postgres")
+
+
+class TestDenialCacheSurvivesSerialization(SimpleTestCase):
+ """A denial must be recognised after a round trip through the cache backend.
+
+ The cache pickles its values, so an `object()` sentinel comes back as a different object and
+ an identity comparison never matches. A denial that never hits is worse than no cache: every
+ denied dial, and every dial during a control-plane outage, pays the full control-plane
+ timeout against an already-sick service.
+ """
+
+ def test_a_denial_is_recognised_after_a_pickle_round_trip(self) -> None:
+ import pickle
+
+ from products.managed_warehouse.backend.facade.testing import NO_MANAGED_WAREHOUSE_TARGET
+
+ serialized = pickle.dumps(NO_MANAGED_WAREHOUSE_TARGET) # nosemgrep
+ restored = pickle.loads(serialized) # nosemgrep
+ assert restored == NO_MANAGED_WAREHOUSE_TARGET
+
+ def test_a_cached_denial_is_returned_without_a_control_plane_call(self) -> None:
+ from products.managed_warehouse.backend.facade.testing import NO_MANAGED_WAREHOUSE_TARGET
+
+ with (
+ patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,
+ patch(f"{_INGRESS_MODULE}.cache") as cache,
+ ):
+ cache.get.return_value = NO_MANAGED_WAREHOUSE_TARGET
+ assert get_managed_warehouse_target("org-1", source_type="postgres") is None
+ hogtower.request.assert_not_called()
+
+ def test_a_cache_read_fault_denies_instead_of_raising(self) -> None:
+ # The guard runs inside a connect. An exception from the cache would replace a clear
+ # refusal with an opaque connection error.
+ with (
+ patch(f"{_INGRESS_MODULE}.cache") as cache,
+ patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,
+ patch(f"{_INGRESS_MODULE}.logger"),
+ ):
+ cache.get.side_effect = RuntimeError("redis down")
+ hogtower.is_configured.return_value = False
+ assert get_managed_warehouse_target("org-1", source_type="postgres") is None
+
+ def test_a_cache_write_fault_still_answers_the_dial(self) -> None:
+ with (
+ patch(f"{_INGRESS_MODULE}.cache") as cache,
+ patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,
+ patch(f"{_INGRESS_MODULE}.logger"),
+ ):
+ cache.get.return_value = None
+ cache.set.side_effect = RuntimeError("redis down")
+ hogtower.is_configured.return_value = True
+ hogtower.request.return_value = _response(
+ {"org_id": "org-1", "state": "ready", "connection": {"host": _TENANT_HOST}}
+ )
+ target = get_managed_warehouse_target("org-1", source_type="postgres")
+
+ assert target is not None
+ assert target.host == _TENANT_HOST
+
+ def test_a_control_plane_error_the_adapter_cannot_translate_denies(self) -> None:
+ with (
+ patch(f"{_INGRESS_MODULE}.cache"),
+ patch(f"{_INGRESS_MODULE}.hogtower") as hogtower,
+ patch(f"{_INGRESS_MODULE}.logger"),
+ ):
+ hogtower.is_configured.return_value = True
+ hogtower.request.side_effect = KeyError("unmapped route")
+ assert get_managed_warehouse_target("org-1", source_type="postgres") is None
+
+
+class TestHostComparisonIsNormalized(SimpleTestCase):
+ """A control-plane host with a capital or a trailing dot must still match.
+
+ The guard refuses any host that is not already normalized, so it compares the normalized
+ name. Comparing against the control plane's string as stored would refuse that host for as
+ long as the control plane kept answering with it.
+ """
+
+ @parameterized.expand(
+ [
+ ("uppercase", "ACME.DW.US.POSTWH.COM"),
+ ("trailing_dot", "acme.dw.us.postwh.com."),
+ ("mixed", "Acme.Dw.Us.Postwh.Com."),
+ ]
+ )
+ @override_settings(MANAGED_WAREHOUSE_INGRESS_IPS=_NLB_A)
+ def test_a_differently_cased_or_dotted_control_plane_host_still_matches(self, _name: str, stored: str) -> None:
+ target = ManagedWarehouseTarget(host=stored, ports=frozenset({5432}))
+ assert target.accepts(_TENANT_HOST, 5432, [_NLB_A], _INGRESS_SET) is True
+
+ @override_settings(MANAGED_WAREHOUSE_INGRESS_IPS=_NLB_A)
+ def test_a_genuinely_different_host_is_still_refused(self) -> None:
+ # Normalization must not become a fuzzy match.
+ target = ManagedWarehouseTarget(host="globex.dw.us.postwh.com", ports=frozenset({5432}))
+ assert target.accepts(_TENANT_HOST, 5432, [_NLB_A], _INGRESS_SET) is False
+
+
+class TestTrinoConnectPathAgreesWithValidation(SimpleTestCase):
+ """A managed Trino source must not validate and then refuse every sync.
+
+ The taken branch checked the host without a port or source type, so the exemption was
+ unavailable on the connect path even when validation had just approved it.
+ """
+
+ @override_settings(CLOUD_DEPLOYMENT="US", MANAGED_WAREHOUSE_INGRESS_IPS=_NLB_A)
+ def test_the_trino_connect_passes_port_and_source_type(self) -> None:
+ from products.warehouse_sources.backend.temporal.data_imports.sources.trino.trino import TrinoImplementation
+
+ config = _stub_trino_config(port=443)
+ with patch.object(TrinoImplementation, "is_database_host_valid", return_value=(True, None)) as check:
+ with patch(
+ "products.warehouse_sources.backend.temporal.data_imports.sources.trino.trino.connect_trino",
+ side_effect=_no_connection,
+ ):
+ try:
+ with TrinoImplementation().connect(config, team_id=999):
+ pass
+ except _Stop:
+ pass
+
+ check.assert_called_once()
+ kwargs = check.call_args.kwargs
+ assert kwargs.get("port") == 443
+ assert kwargs.get("source_type") == "trino"
+
+
+class _Stop(Exception):
+ pass
+
+
+def _no_connection(*args, **kwargs):
+ raise _Stop()
+
+
+def _stub_trino_config(port: int):
+ from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.trino import (
+ TrinoAuthTypeConfig,
+ TrinoSourceConfig,
+ )
+
+ return TrinoSourceConfig(
+ host=_TENANT_HOST,
+ port=port,
+ catalog="ducklake",
+ auth_type=TrinoAuthTypeConfig(user="root", selection="password", password="secret"),
+ schema="main",
+ use_ssl=True,
+ verify_ssl=True,
+ )
+
+
+class TestTheRealDependenciesResolve(SimpleTestCase):
+ """The guard's imports must resolve without a patch, on the real module path.
+
+ Every other test in this file patches `_managed_warehouse_organization_id`, so a wrong
+ import path inside it would pass them all and then raise an `ImportError` at the call site in
+ production. These tests call the real functions.
+ """
+
+ def test_the_organization_lookup_resolves(self) -> None:
+ from products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import (
+ _managed_warehouse_organization_id,
+ )
+
+ # A nonexistent team must deny, not raise. The point is that the import resolves.
+ assert _managed_warehouse_organization_id(2**31 - 1) is None
+
+ def test_a_missing_team_denies(self) -> None:
+ from products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import (
+ _managed_warehouse_organization_id,
+ )
+
+ with patch(f"{_MIXINS_MODULE}.logger"):
+ assert _managed_warehouse_organization_id(None) is None
+
+ def test_the_domain_helper_resolves_without_the_presentation_layer(self) -> None:
+ # The guard reads the zone from a dependency-free module, so a Temporal worker does not
+ # import the DRF views module to learn a DNS suffix.
+ from products.managed_warehouse.backend.facade.api import managed_warehouse_domain
+
+ with override_settings(CLOUD_DEPLOYMENT="US"):
+ assert managed_warehouse_domain() == "us.postwh.com"
+ with override_settings(CLOUD_DEPLOYMENT="EU"):
+ assert managed_warehouse_domain() == "eu.postwh.com"
+
+ def test_the_views_helper_agrees_with_the_dependency_free_one(self) -> None:
+ from products.managed_warehouse.backend.facade.api import managed_warehouse_domain as dependency_free
+ from products.managed_warehouse.backend.presentation.views import managed_warehouse_domain
+
+ with override_settings(CLOUD_DEPLOYMENT="US"):
+ assert managed_warehouse_domain() == dependency_free() == "us.postwh.com"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/test_mixins.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test_mixins.py
index 71d43b5d2b62..d47333455e9c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/test_mixins.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/test_mixins.py
@@ -112,15 +112,25 @@ class TestIsHostSafe(SimpleTestCase):
@parameterized.expand(
[
- ("postwh_us", "entirely-chief-wildcat.us.postwh.com"),
- ("postwh_eu", "my-db.eu.postwh.com"),
- ("postwh_bare", "something.postwh.com"),
+ ("legacy_us_shape", "entirely-chief-wildcat.us.postwh.com"),
+ ("legacy_eu_shape", "my-db.eu.postwh.com"),
+ ("bare_suffix", "something.postwh.com"),
+ ("managed_shape", "acme.dw.us.postwh.com"),
]
)
- @override_settings(CLOUD_DEPLOYMENT="US")
- def test_allows_postwh_hosts(self, _name: str, host: str):
- valid, _ = _is_host_safe(host, team_id=999)
- assert valid
+ @override_settings(CLOUD_DEPLOYMENT="US", MANAGED_WAREHOUSE_INGRESS_IPS="10.1.0.1")
+ def test_a_postwh_name_alone_earns_no_exemption(self, _name: str, host: str):
+ # The exemption used to be the `.postwh.com` suffix. A name is now only a prefilter: the
+ # organization's own control-plane host and the resolved ingress address both have to
+ # agree, and no control plane is configured here, so every one of these is refused.
+ with patch(
+ f"{_MIXINS_MODULE}.socket.getaddrinfo",
+ return_value=[(None, None, None, None, ("10.1.0.1", 0))],
+ ):
+ valid, error = _is_host_safe(host, team_id=999, port=5432, source_type="postgres")
+
+ assert not valid
+ assert error is not None
@parameterized.expand(
[
@@ -920,7 +930,6 @@ class TestCheckResolvedAddresses(SimpleTestCase):
[
("team_allowlist", "US", "db.internal.example.com", 2),
("not_cloud", None, "db.internal.example.com", 999),
- ("posthog_managed", "US", "warehouse.postwh.com", 999),
]
)
def test_exemptions_skip_the_check_and_keep_the_set(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/mysql.py b/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/mysql.py
index 989813bad268..06e989bf6acc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/mysql.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/mysql/mysql.py
@@ -111,6 +111,11 @@ __all__ = [
"get_connection_metadata",
]
+# The source type the managed-warehouse host exemption is keyed on. No managed warehouse
+# serves MySQL, so the exemption grants nothing here; naming it keeps the call site
+# explicit rather than relying on the missing port to deny.
+_HOST_CHECK_SOURCE_TYPE = "mysql"
+
_IDENTIFIER_QUOTER = BacktickIdentifierQuoter()
_QUERY_BUILDER = SelectQueryBuilder(quoter=_IDENTIFIER_QUOTER)
@@ -717,7 +722,7 @@ def _pinned_addresses(host: str, port: int, connect_timeout: int, team_id: int |
except UnicodeError:
addresses = []
- resolution = check_resolved_addresses(host, addresses, team_id)
+ resolution = check_resolved_addresses(host, addresses, team_id, port=port, source_type=_HOST_CHECK_SOURCE_TYPE)
if resolution.connect_host is None:
raise HostNotAllowedError(
f"{DATABASE_HOST_NOT_ALLOWED_ERROR}: {resolution.error or DATABASE_HOST_NOT_ALLOWED_GUIDANCE}"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py
index 559c9b2882fd..098dc86d20da 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py
@@ -938,6 +938,11 @@ def _is_options_startup_param_unsupported(error: BaseException) -> bool:
# reach a PostgreSQL server speaking the SSL protocol — a wrong port, an HTTP/proxy/edge endpoint,
# or a TCP proxy fronting a paused/deleted database. Don't mislabel it "SSL not supported": let the
# raw message reach `get_non_retryable_errors` for an accurate, non-retryable diagnosis.
+# The source type the managed-warehouse host exemption is keyed on. Kept next to the one
+# connect this module opens so a managed warehouse host is treated the same at connect time
+# as it is at validation time.
+_HOST_CHECK_SOURCE_TYPE = "postgres"
+
_INVALID_SSL_NEGOTIATION_RESPONSE_SUBSTRING = "received invalid response to ssl negotiation"
@@ -959,6 +964,7 @@ def _open_connection(*, team_id: int | None = None, **connect_kwargs: Any) -> ps
port=connect_kwargs.get("port", 5432),
connect_timeout=connect_kwargs.get("connect_timeout", 15),
team_id=team_id,
+ source_type=_HOST_CHECK_SOURCE_TYPE,
)
)
try:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py
index 8416da4b83ba..176a4faa6789 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py
@@ -190,6 +190,10 @@ _DATABASE_NOT_FOUND_VALIDATION_ERROR = (
"correct and try again."
)
+# The source type the managed-warehouse host exemption is keyed on. It names the listener
+# set the exemption applies to, and both the validation and the connect path pass it.
+POSTGRES_HOST_CHECK_SOURCE_TYPE = "postgres"
+
PostgresErrors = {
"password authentication failed for user": _INVALID_CREDENTIALS_VALIDATION_ERROR,
# A proxy/pooler in front of some providers rejects bad credentials during its own
@@ -1985,7 +1989,11 @@ class PostgresSource(
return False, _RAILWAY_INTERNAL_HOST_ERROR
valid_host, host_errors = self.is_database_host_valid(
- config.host, team_id, using_ssh_tunnel=self.ssh_tunnel_enabled(config)
+ config.host,
+ team_id,
+ using_ssh_tunnel=self.ssh_tunnel_enabled(config),
+ port=config.port,
+ source_type=POSTGRES_HOST_CHECK_SOURCE_TYPE,
)
if not valid_host:
return valid_host, host_errors
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/source.py
index 4383802e23c6..14779e3247a9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/source.py
@@ -44,6 +44,12 @@ from products.warehouse_sources.backend.types import ExternalDataSourceType
_TRINO_IMPLEMENTATION = TrinoImplementation()
+# The source type the managed-warehouse host exemption is keyed on. A managed Trino tenant
+# endpoint answers on the gateway listener, not the pgwire one, so the exemption is asked
+# for under this name and grants port 443 only.
+_HOST_CHECK_SOURCE_TYPE = "trino"
+
+
@SourceRegistry.register
class TrinoSource(SQLSource[TrinoSourceConfig], ValidateDatabaseHostMixin):
codebase_detection = detect(
@@ -215,7 +221,9 @@ class TrinoSource(SQLSource[TrinoSourceConfig], ValidateDatabaseHostMixin):
def validate_credentials(
self, config: TrinoSourceConfig, team_id: int, schema_name: Optional[str] = None, api_version: str | None = None
) -> tuple[bool, str | None]:
- is_valid, error = self.is_database_host_valid(config.host, team_id)
+ is_valid, error = self.is_database_host_valid(
+ config.host, team_id, port=config.port, source_type=_HOST_CHECK_SOURCE_TYPE
+ )
if not is_valid:
return False, error
try:
@@ -236,7 +244,9 @@ class TrinoSource(SQLSource[TrinoSourceConfig], ValidateDatabaseHostMixin):
force_refresh: bool = False,
api_version: str | None = None,
) -> list[SourceSchema]:
- is_valid, error = self.is_database_host_valid(config.host, team_id)
+ is_valid, error = self.is_database_host_valid(
+ config.host, team_id, port=config.port, source_type=_HOST_CHECK_SOURCE_TYPE
+ )
if not is_valid:
raise ValueError(error or "Invalid Trino host.")
with trino_failures_as_discovery_error(), connect_trino(config) as connection:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.py b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.py
index 800a518e178c..b297164a7f63 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/trino/trino.py
@@ -167,6 +167,7 @@ TRINO_PERMANENT_SYNC_ERROR_MESSAGES = frozenset(
TRINO_CLUSTER_ACCESS_DENIED_ERROR,
}
)
+MANAGED_WAREHOUSE_TRINO_SOURCE_TYPE = "trino"
# The managed-Trino control plane's own wording when it mints a tenant's compute-pool grant before
# the pool coordinator has finished admitting that tenant. The request was never dispatched, so a
# retry is safe, and the race clears within a few seconds once admission catches up.
@@ -732,7 +733,12 @@ class TrinoImplementation(SQLSourceImplementation[TrinoSourceConfig, Any, Any],
@contextmanager
def connect(self, config: TrinoSourceConfig, *, team_id: int | None = None) -> Iterator[Any]:
if team_id is not None:
- is_valid, error = self.is_database_host_valid(config.host, team_id)
+ # The managed-warehouse exemption is keyed on the source type and the port, so the
+ # connect path asks for it the same way validation does. Without both, a managed
+ # Trino source validates and then refuses every sync.
+ is_valid, error = self.is_database_host_valid(
+ config.host, team_id, port=config.port, source_type=MANAGED_WAREHOUSE_TRINO_SOURCE_TYPE
+ )
if not is_valid:
raise ValueError(error or "Invalid Trino host.")
# A UTC session makes any time zone conversion Trino still does in a filter deterministic.