Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(managed-warehouse): route flagged users to a trino canary host (#127937)

Eric Duong23ab65863ebd
diff --git a/posthog/clickhouse/client/execute_async.py b/posthog/clickhouse/client/execute_async.py
index 92ed6fca50d0..e3d5abcac8df 100644
--- a/posthog/clickhouse/client/execute_async.py
+++ b/posthog/clickhouse/client/execute_async.py
@@ -239,7 +239,7 @@ def execute_process_query(
user: Optional[User] = None
if user_id:
- user = User.objects.only("email", "is_staff").get(pk=user_id)
+ user = User.objects.only("email", "is_staff", "distinct_id").get(pk=user_id)
is_staff_user = user.is_staff
elif sharing_configuration_id:
# A shared-link viewer has no user row, so the identity has to be rebuilt from the share it
diff --git a/posthog/dags/README_DUCKLINGS.md b/posthog/dags/README_DUCKLINGS.md
index 5e06fbd7959a..4c1ca200de59 100644
--- a/posthog/dags/README_DUCKLINGS.md
+++ b/posthog/dags/README_DUCKLINGS.md
@@ -29,6 +29,9 @@ The ClickHouse export is the same, except that it casts each column to the Parqu
An export-only run (`skip_ducklake_registration: true`) writes the same types, so `add_files` can register its files later.
ClickHouse lists the files it exported, because nothing else on the path can list the duckling bucket.
Each export writes under a new file prefix, so a retry never overwrites a file that DuckLake has already registered.
+A Trino backfill connects through the normal Trino hostname.
+Set `use_trino_canary: true` in the run config to connect through the canary hostname instead, which needs `MANAGED_WAREHOUSE_TRINO_CANARY_DOMAIN` to be set. Without it, the run fails.
+The flag `managed-warehouse-trino-canary` does not apply to backfills.
One `add_files` call registers the files of an export and removes the rows they replace in a single DuckLake snapshot:
@@ -192,6 +195,7 @@ class DucklingBackfillConfig:
create_tables_if_missing: bool = True # Auto-create events/persons tables
delete_tables: bool = False # DANGER: Drop and recreate tables
dry_run: bool = False # Preview mode, no writes
+ use_trino_canary: bool = False # Trino path only: connect through the canary hostname
events_parquet_row_group_size_bytes: int = 536_870_912 # Events only; 512 MiB
target_rows_per_file: int = 5_000_000 # Target rows per exported file
max_s3_file_fanout: int = 256 # Maximum before the automatic buffer-budget limit
diff --git a/posthog/dags/events_backfill_to_duckling.py b/posthog/dags/events_backfill_to_duckling.py
index 77066e04e0cb..e5a091cf4ac6 100644
--- a/posthog/dags/events_backfill_to_duckling.py
+++ b/posthog/dags/events_backfill_to_duckling.py
@@ -1036,6 +1036,8 @@ class DucklingBackfillConfig(Config):
create_tables_if_missing: bool = True
delete_tables: bool = False # Danger: drops and recreates tables, losing all data
dry_run: bool = False
+ # Opt-in per run. A backfill never evaluates the canary feature flag.
+ use_trino_canary: bool = False
events_parquet_row_group_size_bytes: int = pydantic.Field(
default=DEFAULT_EVENTS_PARQUET_ROW_GROUP_SIZE_BYTES,
gt=0,
@@ -2611,7 +2613,9 @@ def backfill_through_trino(
partitions_exported = 0
files_registered = 0
register = not (config.dry_run or config.skip_ducklake_registration)
- with connect_backfill_catalog(target.organization_id) if register else nullcontext() as catalog:
+ with (
+ connect_backfill_catalog(target.organization_id, canary=config.use_trino_canary) if register else nullcontext()
+ ) as catalog:
if catalog is not None:
catalog.ensure_table(
table,
diff --git a/posthog/dags/test_events_backfill_to_duckling.py b/posthog/dags/test_events_backfill_to_duckling.py
index baae112a0c68..12fb9111b86d 100644
--- a/posthog/dags/test_events_backfill_to_duckling.py
+++ b/posthog/dags/test_events_backfill_to_duckling.py
@@ -2746,7 +2746,7 @@ class TestBackfillThroughTrino:
catalog, connect_catalog = self._run(run, partition_key, clickhouse, DucklingBackfillConfig())
- connect_catalog.assert_called_with("org-1")
+ connect_catalog.assert_called_with("org-1", canary=False)
assert catalog.tables == [(table, column_types)] * 2
assert [(t, r) for t, _, _, r in catalog.registered] == [(table, replacement)] * 2
first, second = catalog.registered
@@ -2804,6 +2804,20 @@ class TestBackfillThroughTrino:
assert cast_column in insert
assert settings["output_format_parquet_string_as_string"] == 1
+ @parameterized.expand(
+ [
+ ("events", _run_duckling_events_backfill, "2_2026-06-17"),
+ ("persons", _run_duckling_persons_backfill, "2"),
+ ]
+ )
+ @time_machine.travel(datetime(2026, 7, 1, tzinfo=UTC), tick=False)
+ def test_opted_in_runs_register_through_the_canary_host(self, _name, run, partition_key):
+ _, connect_catalog = self._run(
+ run, partition_key, _FakeExportClickHouse(files_per_export=1), DucklingBackfillConfig(use_trino_canary=True)
+ )
+
+ connect_catalog.assert_called_with("org-1", canary=True)
+
@time_machine.travel(datetime(2026, 7, 1, tzinfo=UTC), tick=False)
def test_dry_runs_neither_export_nor_open_the_catalog(self):
clickhouse = _FakeExportClickHouse(files_per_export=2)
diff --git a/posthog/hogql/direct_sql/adapter.py b/posthog/hogql/direct_sql/adapter.py
index ba127f0d5883..96dbac2e5926 100644
--- a/posthog/hogql/direct_sql/adapter.py
+++ b/posthog/hogql/direct_sql/adapter.py
@@ -39,6 +39,8 @@ def parse_direct_source_config(source_impl: _ParseableSource[_ConfigT_co], sourc
@frozen
class DirectQueryPrincipal:
value: str
+ # The query runner decides once per query, so the adapter never evaluates the canary flag itself.
+ trino_canary: bool = False
@dataclasses.dataclass(frozen=False)
diff --git a/posthog/hogql/query.py b/posthog/hogql/query.py
index cb247a898b50..45c05ed95d2e 100644
--- a/posthog/hogql/query.py
+++ b/posthog/hogql/query.py
@@ -203,6 +203,7 @@ class HogQLQueryExecutor:
direct_dialect: Optional[HogQLDialect] = None
connection_id: Optional[str] = None
expected_direct_engine: str | None = None
+ trino_canary: bool = False
send_raw_query: bool = False
user: Optional[User] = None
bypass_warehouse_access_control: bool = False
@@ -760,7 +761,10 @@ class HogQLQueryExecutor:
query_type=self.query_type,
debug=bool(self.debug),
principal=(
- DirectQueryPrincipal(value=f"posthog:sql-editor:team:{self.team.pk}:user:{self.user.pk}")
+ DirectQueryPrincipal(
+ value=f"posthog:sql-editor:team:{self.team.pk}:user:{self.user.pk}",
+ trino_canary=self.trino_canary,
+ )
if isinstance(self.user, User) and self.user.pk is not None
else None
),
diff --git a/posthog/hogql/test/test_direct_duckgres_query.py b/posthog/hogql/test/test_direct_duckgres_query.py
index 2e30469ebe51..80df3eacd90e 100644
--- a/posthog/hogql/test/test_direct_duckgres_query.py
+++ b/posthog/hogql/test/test_direct_duckgres_query.py
@@ -219,9 +219,11 @@ class TestDirectDuckgresQuery(APIBaseTest):
self.resolve_hostaddr = resolver_patcher.start()
self.addCleanup(resolver_patcher.stop)
- @parameterized.expand([(True,), (False,)])
+ @parameterized.expand([(True, False), (False, True)])
@patch("posthoganalytics.feature_enabled", return_value=True)
- def test_managed_source_switches_raw_and_hogql_execution_to_trino(self, raw: bool, _flag: MagicMock) -> None:
+ def test_managed_source_switches_raw_and_hogql_execution_to_trino(
+ self, raw: bool, canary: bool, _flag: MagicMock
+ ) -> None:
_flag.side_effect = lambda flag, *_args, **_kwargs: flag == "managed-warehouse-trino-query"
source = self._dynamic_managed_source()
source.job_inputs = {}
@@ -258,6 +260,7 @@ class TestDirectDuckgresQuery(APIBaseTest):
user=self.user,
connection_id=str(source.id),
send_raw_query=raw,
+ trino_canary=canary,
).execute()
with self.assertRaisesMessage(ExposedHogQLError, "warehouse engine changed"):
HogQLQueryExecutor(
@@ -270,7 +273,9 @@ class TestDirectDuckgresQuery(APIBaseTest):
).execute()
self.assertEqual(response.results, [("row-uuid",)])
connect.assert_called_once_with(
- str(self.team.organization_id), principal=f"posthog:sql-editor:team:{self.team.pk}:user:{self.user.pk}"
+ str(self.team.organization_id),
+ principal=f"posthog:sql-editor:team:{self.team.pk}:user:{self.user.pk}",
+ canary=canary,
)
duckgres.assert_not_called()
self.assertIn('"posthog"."events"', cursor.execute.call_args.args[0])
diff --git a/posthog/hogql_queries/hogql_query_runner.py b/posthog/hogql_queries/hogql_query_runner.py
index 73dacce6bf4e..150d66308cda 100644
--- a/posthog/hogql_queries/hogql_query_runner.py
+++ b/posthog/hogql_queries/hogql_query_runner.py
@@ -42,6 +42,7 @@ from posthog.models.activity_logging.retention import get_activity_log_lookback_
from products.managed_warehouse.backend.facade import query_labels as managed_warehouse_query_labels
from products.managed_warehouse.backend.facade.query_execution import (
execute_managed_trino_query,
+ use_trino_canary,
validate_managed_trino_query,
)
from products.warehouse_sources.backend.facade.types import ManagedWarehouseSQLMode
@@ -67,6 +68,7 @@ class HogQLQueryRunner(AnalyticsQueryRunner[HogQLQueryResponse]):
self._direct_connection_validated = False
self._direct_engine: str | None = None
self._managed_warehouse_sql_mode: ManagedWarehouseSQLMode | None = None
+ self._trino_canary = False
super().__init__(*args, **kwargs)
def _reads_events_route(self) -> bool:
@@ -104,8 +106,21 @@ class HogQLQueryRunner(AnalyticsQueryRunner[HogQLQueryResponse]):
direct_engine = adapter.engine if adapter is not None else None
self._managed_warehouse_sql_mode = managed_warehouse_sql_mode
self._direct_engine = direct_engine
+ self._trino_canary = self._decide_trino_canary(user if user is not None else self.user)
self._direct_connection_validated = True
+ def _decide_trino_canary(self, user: Optional[User]) -> bool:
+ """Decide the Trino lane once per validation. The cache key and the connection both read the result, so
+ a flag change between the two steps cannot store a result under the key of the other lane."""
+ runs_on_managed_trino = self.query.executionTarget == ExecutionTarget.MANAGED_TRINO or (
+ self._managed_warehouse_sql_mode == ManagedWarehouseSQLMode.BUILT_IN and self._direct_engine == "trino"
+ )
+ return (
+ runs_on_managed_trino
+ and user is not None
+ and use_trino_canary(user.distinct_id, str(self.team.organization_id), limit_context=self.limit_context)
+ )
+
def get_cache_payload(self) -> dict:
self._validate_direct_connection()
payload = super().get_cache_payload()
@@ -137,6 +152,12 @@ class HogQLQueryRunner(AnalyticsQueryRunner[HogQLQueryResponse]):
return payload
+ def get_cache_key_variant(self) -> str:
+ variant = super().get_cache_key_variant()
+ self._validate_direct_connection()
+ # The canary lane can return different results, so they never share a cache entry with the normal lane.
+ return f"{variant}_trino_canary" if self._trino_canary else variant
+
def query_status_labels(self) -> list[str] | None:
self._validate_direct_connection()
if self.query.executionTarget == ExecutionTarget.MANAGED_TRINO and self.query.sendRawQuery:
@@ -280,6 +301,7 @@ class HogQLQueryRunner(AnalyticsQueryRunner[HogQLQueryResponse]):
settings=self.settings or HogQLGlobalSettings(),
timings=self.timings,
limit_context=self.limit_context,
+ canary=self._trino_canary,
)
if self.query.sendRawQuery and self.query.connectionId:
@@ -300,6 +322,7 @@ class HogQLQueryRunner(AnalyticsQueryRunner[HogQLQueryResponse]):
settings=self.settings,
send_raw_query=True,
expected_direct_engine=self._direct_engine,
+ trino_canary=self._trino_canary,
)
query = self._parse_query()[0] if self._direct_engine == "trino" else self.to_query()
@@ -344,6 +367,7 @@ class HogQLQueryRunner(AnalyticsQueryRunner[HogQLQueryResponse]):
ch_user=self.ch_user,
settings=self.settings,
expected_direct_engine=self._direct_engine,
+ trino_canary=self._trino_canary,
**context_kwargs,
)
if paginator:
diff --git a/posthog/settings/base_variables.py b/posthog/settings/base_variables.py
index 9508fd7274ad..d38572bb7a50 100644
--- a/posthog/settings/base_variables.py
+++ b/posthog/settings/base_variables.py
@@ -93,6 +93,10 @@ HOGTOWER_INTERNAL_SECRET: str | None = get_from_env("HOGTOWER_INTERNAL_SECRET",
# from the private-address check only when every resolved address is in this set, so an empty value
# refuses every managed warehouse host. posthog-cloud-infra owns the values.
MANAGED_WAREHOUSE_INGRESS_IPS: str = get_from_env("MANAGED_WAREHOUSE_INGRESS_IPS", "", type_cast=str)
+# MANAGED_WAREHOUSE_TRINO_CANARY_DOMAIN is the domain whose hosts reach the canary Trino lane, for example
+# `canary.dw.us.postwh.com`. A canary connection replaces the host of a minted credential with
+# `<database label>.<this domain>`. An empty value turns the canary route off.
+MANAGED_WAREHOUSE_TRINO_CANARY_DOMAIN: str = get_from_env("MANAGED_WAREHOUSE_TRINO_CANARY_DOMAIN", "", type_cast=str)
MANAGED_WAREHOUSE_CREDENTIAL_CACHE_REDIS_CONNECT_TIMEOUT_SECONDS: float = get_from_env(
"MANAGED_WAREHOUSE_CREDENTIAL_CACHE_REDIS_CONNECT_TIMEOUT_SECONDS", 0.5, type_cast=float
)
diff --git a/products/managed_warehouse/backend/README.md b/products/managed_warehouse/backend/README.md
index 1ffb14ab5743..fbd04530474b 100644
--- a/products/managed_warehouse/backend/README.md
+++ b/products/managed_warehouse/backend/README.md
@@ -130,6 +130,32 @@ The data-import copy and registration paths target the same stable DuckLake tabl
The two organization flags also select the Data ops variant. `managed-warehouse-trino-query` gives the Trino variant and wins when both are on. `data-warehouse-scene` alone gives the DuckDB variant. `data_ops_variant` in `presentation/views.py` is the single backend implementation of that rule.
+## Trino canary hostname
+
+`MANAGED_WAREHOUSE_TRINO_CANARY_DOMAIN` (empty by default) names the domain of the canary Trino lane, for example `canary.dw.us.postwh.com`.
+A canary connection keeps the minted credential and replaces the host with `<database label>.<canary domain>`.
+The minted host must be a single label under `dw.<region>.postwh.com`.
+Any other minted host, or an empty canary domain, makes a canary connection fail with an explicit error instead of falling back to the normal host.
+
+An interactive run uses the canary host when the requesting user has the `managed-warehouse-trino-canary` flag, evaluated with `user.distinct_id` and the organization group.
+This covers the SQL editor, the schema browser, and the managed warehouse connection of the SQL editor.
+These runs never use the canary host:
+
+- Alert runs and cache-warming runs, which carry alert and warming query tags.
+- Runs with the export, SQL alert, or PostHog AI limit context. Exports and subscriptions run as their creator and carry no tag, so the limit context identifies them.
+- Schema discovery, data modeling, and model aliases, which never pass the canary option.
+
+A Dagster backfill uses the canary host only when its run config sets `use_trino_canary: true`.
+It does not evaluate the flag.
+
+The query runner decides once per query and uses that decision for the cache key and for the connection.
+A canary result is cached under its own key, so the normal lane never serves it.
+The decision is made again for each runner, so the web request and the Celery task that continues it each agree with themselves.
+
+Before you turn the flag on, run one real query through the canary host.
+The canary host must return `nextUri` and `infoUri` values on the canary host.
+The connection accepts only responses that name the host it dialed, so a response on the normal host fails with "Trino returned a different service credential destination".
+
## Data Ops workflow status
The copy and registration workflows both write their lifecycle to `ManagedWarehouseSourceJob`. Each row identifies the project, source schema, external data job, workflow type, and workflow attempt. The supported states are running, completed, failed, skipped, and stale.
diff --git a/products/managed_warehouse/backend/backfill_trino.py b/products/managed_warehouse/backend/backfill_trino.py
index 77ad053d3852..d461aba71661 100644
--- a/products/managed_warehouse/backend/backfill_trino.py
+++ b/products/managed_warehouse/backend/backfill_trino.py
@@ -107,8 +107,10 @@ class TrinoBackfillCatalog:
@classmethod
@contextlib.contextmanager
- def connect(cls, organization_id: str) -> Iterator[TrinoBackfillCatalog]:
- with connect_managed_warehouse_trino(organization_id, principal=BACKFILL_TRINO_PRINCIPAL) as connection:
+ def connect(cls, organization_id: str, *, canary: bool = False) -> Iterator[TrinoBackfillCatalog]:
+ with connect_managed_warehouse_trino(
+ organization_id, principal=BACKFILL_TRINO_PRINCIPAL, canary=canary
+ ) as connection:
yield cls(connection)
def _table(self, table: str) -> str:
diff --git a/products/managed_warehouse/backend/facade/backfill_trino.py b/products/managed_warehouse/backend/facade/backfill_trino.py
index b74d5c9fa230..183580cf73b0 100644
--- a/products/managed_warehouse/backend/facade/backfill_trino.py
+++ b/products/managed_warehouse/backend/facade/backfill_trino.py
@@ -14,5 +14,7 @@ def backfills_through_trino(organization_id: str) -> bool:
return backfill_trino.backfills_through_trino(organization_id)
-def connect_backfill_catalog(organization_id: str) -> AbstractContextManager[DuckLakeBackfillCatalog]:
- return backfill_trino.TrinoBackfillCatalog.connect(organization_id)
+def connect_backfill_catalog(
+ organization_id: str, *, canary: bool = False
+) -> AbstractContextManager[DuckLakeBackfillCatalog]:
+ return backfill_trino.TrinoBackfillCatalog.connect(organization_id, canary=canary)
diff --git a/products/managed_warehouse/backend/facade/feature_flags.py b/products/managed_warehouse/backend/facade/feature_flags.py
index e8980d00d8ed..3908bcae9d22 100644
--- a/products/managed_warehouse/backend/facade/feature_flags.py
+++ b/products/managed_warehouse/backend/facade/feature_flags.py
@@ -3,5 +3,7 @@ MANAGED_TRINO_QUERY_FLAG = "managed-warehouse-trino-query"
# Either flag gives an organization the managed warehouse.
MANAGED_WAREHOUSE_ACCESS_FLAGS = (MANAGED_TRINO_QUERY_FLAG, DATA_WAREHOUSE_SCENE_FLAG)
DATA_MODELING_SHADOW_FLAG = "managed-warehouse-data-modeling-shadow"
+# Evaluated for the requesting user. A user with the flag reaches Trino through the canary hostname.
+MANAGED_WAREHOUSE_TRINO_CANARY_FLAG = "managed-warehouse-trino-canary"
# Marks an organization whose warehouse has no duckgres, so its data imports and backfills register through Trino.
DUCKLAKE_TRINO_REGISTRATION_FLAG = "managed-warehouse-trino-import-registration"
diff --git a/products/managed_warehouse/backend/facade/query_execution.py b/products/managed_warehouse/backend/facade/query_execution.py
index 87a4b8c638dc..a03c27d2db64 100644
--- a/products/managed_warehouse/backend/facade/query_execution.py
+++ b/products/managed_warehouse/backend/facade/query_execution.py
@@ -38,6 +38,7 @@ def execute_managed_trino_query(
settings: HogQLGlobalSettings,
timings: HogQLTimings,
limit_context: LimitContext | None,
+ canary: bool = False,
) -> HogQLQueryResponse:
from products.managed_warehouse.backend.query_execution import (
execute_managed_trino_query as execute, # noqa: PLC0415 -- query compilation is optional during Django startup
@@ -51,9 +52,20 @@ def execute_managed_trino_query(
settings=settings,
timings=timings,
limit_context=limit_context,
+ canary=canary,
)
+def use_trino_canary(
+ distinct_id: str | None, organization_id: str, *, limit_context: LimitContext | None = None
+) -> bool:
+ from products.managed_warehouse.backend.trino_connection import (
+ use_trino_canary as decide, # noqa: PLC0415 -- keeps the optional Trino connection path off Django startup
+ )
+
+ return decide(distinct_id, organization_id, limit_context=limit_context)
+
+
def get_managed_trino_schema(
query: DatabaseSchemaQuery, *, team: Team, user: User | None
) -> DatabaseSchemaQueryResponse:
diff --git a/products/managed_warehouse/backend/logic/sql_editor_trino.py b/products/managed_warehouse/backend/logic/sql_editor_trino.py
index f9a5a3ce1c10..1cdf27373b53 100644
--- a/products/managed_warehouse/backend/logic/sql_editor_trino.py
+++ b/products/managed_warehouse/backend/logic/sql_editor_trino.py
@@ -63,7 +63,11 @@ class ManagedTrinoAdapter:
debug=request.debug,
cancellation_token=request.cancellation_token,
),
- connect_managed_warehouse_trino(str(request.team.organization_id), principal=request.principal.value),
+ connect_managed_warehouse_trino(
+ str(request.team.organization_id),
+ principal=request.principal.value,
+ canary=request.principal.trino_canary,
+ ),
)
except (ExposedHogQLError, ManagedWarehouseTrinoConnectionUnavailable) as error:
if not isinstance(error, ManagedWarehouseTrinoConnectionUnavailable) and not isinstance(
diff --git a/products/managed_warehouse/backend/query_execution.py b/products/managed_warehouse/backend/query_execution.py
index 3fc891e838ff..db345998691e 100644
--- a/products/managed_warehouse/backend/query_execution.py
+++ b/products/managed_warehouse/backend/query_execution.py
@@ -80,6 +80,7 @@ def execute_managed_trino_query(
settings: HogQLGlobalSettings,
timings: HogQLTimings,
limit_context: LimitContext | None,
+ canary: bool = False,
) -> HogQLQueryResponse:
validate_managed_trino_query(query, team, user)
assert user is not None
@@ -131,7 +132,9 @@ def execute_managed_trino_query(
cancellation_token=cancellation_token,
),
connect_managed_warehouse_trino(
- str(team.organization_id), principal=f"posthog:sql-editor:team:{team.pk}:user:{user.pk}"
+ str(team.organization_id),
+ principal=f"posthog:sql-editor:team:{team.pk}:user:{user.pk}",
+ canary=canary,
),
)
response = HogQLQueryResponse(
diff --git a/products/managed_warehouse/backend/query_schema.py b/products/managed_warehouse/backend/query_schema.py
index c1a44dea518e..585c48085516 100644
--- a/products/managed_warehouse/backend/query_schema.py
+++ b/products/managed_warehouse/backend/query_schema.py
@@ -17,7 +17,7 @@ from posthog.hogql.escape_sql import quote_hogql_identifier
from products.managed_warehouse.backend.facade.contracts import ManagedWarehouseTrinoConnectionUnavailable
from products.managed_warehouse.backend.logic.sql_editor_trino import MANAGED_TRINO_SCHEMA_DISCOVERY_TIMEOUT_SECONDS
from products.managed_warehouse.backend.query_execution import validate_managed_trino_query
-from products.managed_warehouse.backend.trino_connection import connect_managed_warehouse_trino
+from products.managed_warehouse.backend.trino_connection import connect_managed_warehouse_trino, use_trino_canary
from products.warehouse_sources.backend.facade.models import trino_column_to_dwh_column
from products.warehouse_sources.backend.facade.source_management import discover_trino_catalog
@@ -37,7 +37,9 @@ def get_managed_trino_schema(
)
assert user is not None
with connect_managed_warehouse_trino(
- str(team.organization_id), principal=f"posthog:sql-editor:team:{team.pk}:user:{user.pk}"
+ str(team.organization_id),
+ principal=f"posthog:sql-editor:team:{team.pk}:user:{user.pk}",
+ canary=use_trino_canary(user.distinct_id, str(team.organization_id)),
) as connection:
with bounded_trino_cursor(connection, MANAGED_TRINO_SCHEMA_DISCOVERY_TIMEOUT_SECONDS, team_id=team.pk) as (
cursor,
diff --git a/products/managed_warehouse/backend/tests/test_backfill_trino.py b/products/managed_warehouse/backend/tests/test_backfill_trino.py
index 4768ee0e6066..cd44643c90c1 100644
--- a/products/managed_warehouse/backend/tests/test_backfill_trino.py
+++ b/products/managed_warehouse/backend/tests/test_backfill_trino.py
@@ -1,15 +1,21 @@
from __future__ import annotations
+from contextlib import nullcontext
from datetime import date
from typing import TYPE_CHECKING, cast
import pytest
-from unittest.mock import patch
+from unittest.mock import MagicMock, patch
from parameterized import parameterized
from trino.exceptions import Http503Error, TrinoConnectionError, TrinoExternalError, TrinoQueryError, TrinoUserError
-from products.managed_warehouse.backend.backfill_trino import ADD_FILES_MAX_FILES, TrinoBackfillCatalog
+from products.managed_warehouse.backend.backfill_trino import (
+ ADD_FILES_MAX_FILES,
+ BACKFILL_TRINO_PRINCIPAL,
+ TrinoBackfillCatalog,
+)
+from products.managed_warehouse.backend.facade.backfill_trino import connect_backfill_catalog
from products.managed_warehouse.backend.facade.contracts import DuckLakeDayReplacement, DuckLakeTableReplacement
if TYPE_CHECKING:
@@ -62,6 +68,18 @@ def _catalog(
return TrinoBackfillCatalog(cast("Connection", connection)), connection
[email protected]("canary", [False, True])
+def test_the_backfill_connection_uses_the_canary_host_only_when_asked(canary: bool) -> None:
+ with patch(
+ "products.managed_warehouse.backend.backfill_trino.connect_managed_warehouse_trino",
+ return_value=nullcontext(MagicMock(catalog="example_catalog")),
+ ) as connect:
+ with connect_backfill_catalog("org-1", canary=canary):
+ pass
+
+ connect.assert_called_once_with("org-1", principal=BACKFILL_TRINO_PRINCIPAL, canary=canary)
+
+
class TestRegisterFiles:
@parameterized.expand(
[
diff --git a/products/managed_warehouse/backend/tests/test_groups_backfill.py b/products/managed_warehouse/backend/tests/test_groups_backfill.py
index 8eab45c3085f..bff56ac21826 100644
--- a/products/managed_warehouse/backend/tests/test_groups_backfill.py
+++ b/products/managed_warehouse/backend/tests/test_groups_backfill.py
@@ -340,7 +340,7 @@ def test_groups_backfill_registers_through_trino_without_duckgres(
) -> None:
inserts, catalog, connect_catalog, context = _run_groups_through_trino(partition_key, DucklingBackfillConfig())
- connect_catalog.assert_called_once_with("org")
+ connect_catalog.assert_called_once_with("org", canary=False)
assert catalog.tables == [("groups_team_7", GROUPS_TRINO_COLUMN_TYPES)]
[(table, files, registered_replacement)] = catalog.registered
assert (table, registered_replacement) == ("groups_team_7", replacement)
diff --git a/products/managed_warehouse/backend/tests/test_query_execution.py b/products/managed_warehouse/backend/tests/test_query_execution.py
index 79582016f1e7..f4f585363189 100644
--- a/products/managed_warehouse/backend/tests/test_query_execution.py
+++ b/products/managed_warehouse/backend/tests/test_query_execution.py
@@ -1,16 +1,20 @@
-from contextlib import nullcontext
+from collections.abc import Iterator
+from contextlib import contextmanager, nullcontext
from uuid import UUID
import pytest
from unittest.mock import MagicMock, PropertyMock, patch
+from django.test import override_settings
+
from posthog.schema import HogQLQuery, HogQLQueryModifiers, HogQLQueryResponse
from posthog.hogql.constants import HogQLGlobalSettings, LimitContext
from posthog.hogql.errors import ExposedHogQLError
from posthog.hogql.timings import HogQLTimings
-from posthog.clickhouse.query_tagging import AccessMethod, QueryTags
+from posthog.clickhouse.cluster import Workload
+from posthog.clickhouse.query_tagging import AccessMethod, QueryTags, tags_context
from posthog.hogql_queries.hogql_query_runner import HogQLQueryRunner
from posthog.hogql_queries.query_runner import AnalyticsQueryRunner, ExecutionMode
from posthog.models import Organization, Team, User
@@ -21,6 +25,9 @@ from products.managed_warehouse.backend.facade.contracts import (
)
from products.managed_warehouse.backend.facade.query_labels import MANAGED_TRINO_RAW_QUERY_STATUS_LABEL
from products.managed_warehouse.backend.query_execution import execute_managed_trino_query, validate_managed_trino_query
+from products.warehouse_sources.backend.facade.types import ManagedWarehouseSQLMode
+
+CANARY_DOMAIN = "canary.dw.us.postwh.com"
@pytest.fixture
@@ -39,6 +46,141 @@ def test_default_requests_keep_normal_execution_without_evaluating_the_flag(team
assert runner.query.executionTarget is None
+def _canary_runner(
+ team: Team, *, connection: bool, limit_context: LimitContext = LimitContext.QUERY
+) -> HogQLQueryRunner:
+ runner = object.__new__(HogQLQueryRunner)
+ runner.query = HogQLQuery(
+ query="SELECT 1",
+ executionTarget=None if connection else "managed_trino",
+ connectionId="source-1" if connection else None,
+ sendRawQuery=True,
+ )
+ runner.team = team
+ runner.user = User(id=9, distinct_id="user-9")
+ runner.limit_context = limit_context
+ runner._user_access_control = None
+ runner.settings = HogQLGlobalSettings()
+ runner.modifiers = HogQLQueryModifiers()
+ runner.timings = HogQLTimings()
+ runner.is_query_service = False
+ runner.workload = Workload.DEFAULT
+ runner._direct_connection_validated = False
+ runner._direct_engine = None
+ runner._managed_warehouse_sql_mode = None
+ runner._trino_canary = False
+ return runner
+
+
+@contextmanager
+def _canary_runner_boundaries(canary_domain: str = CANARY_DOMAIN) -> Iterator[tuple[MagicMock, MagicMock]]:
+ source = MagicMock(
+ direct_engine="postgres",
+ has_managed_warehouse_prefix=True,
+ managed_warehouse_sql_mode=ManagedWarehouseSQLMode.BUILT_IN,
+ )
+ module = "posthog.hogql_queries.hogql_query_runner"
+ with (
+ override_settings(MANAGED_WAREHOUSE_TRINO_CANARY_DOMAIN=canary_domain),
+ tags_context(trigger=None, feature=None, alert_config_id=None),
+ patch(f"{module}.validate_managed_trino_query"),
+ patch(f"{module}.get_direct_connection_source", return_value=source),
+ patch(f"{module}.get_adapter_for_source", return_value=MagicMock(engine="trino")),
+ patch(f"{module}.tag_contains_user_hogql"),
+ patch(f"{module}.execute_managed_trino_query") as managed_trino,
+ patch(f"{module}.execute_hogql_query") as direct,
+ ):
+ yield managed_trino, direct
+
+
[email protected]("connection", [False, True])
+ "limit_context,flag,expected_variant",
+ [
+ (LimitContext.QUERY, True, "_trino_canary"),
+ (LimitContext.QUERY_ASYNC, True, "_trino_canary"),
+ (LimitContext.QUERY, False, ""),
+ (LimitContext.EXPORT, True, ""),
+ (LimitContext.SQL_ALERT, True, ""),
+ (LimitContext.POSTHOG_AI, True, ""),
+ ],
+)
+def test_canary_results_do_not_share_a_cache_entry_with_the_normal_lane(
+ team: Team, connection: bool, limit_context: LimitContext, flag: bool, expected_variant: str
+) -> None:
+ runner = _canary_runner(team, connection=connection, limit_context=limit_context)
+ with _canary_runner_boundaries(), patch("posthoganalytics.feature_enabled", return_value=flag):
+ assert runner.get_cache_key_variant() == expected_variant
+
+
+def test_queries_off_managed_trino_never_evaluate_the_canary_flag(team: Team) -> None:
+ runner = object.__new__(HogQLQueryRunner)
+ runner.query = HogQLQuery(query="SELECT 1")
+ runner.team = team
+ runner.user = User(id=9, distinct_id="user-9")
+ runner.limit_context = LimitContext.QUERY
+ runner._direct_connection_validated = False
+ with (
+ override_settings(MANAGED_WAREHOUSE_TRINO_CANARY_DOMAIN=CANARY_DOMAIN),
+ patch("posthoganalytics.feature_enabled", side_effect=AssertionError("evaluated a flag")),
+ ):
+ assert runner.get_cache_key_variant() == ""
+
+
[email protected]("connection", [False, True])
[email protected]("first,later", [(True, False), (False, True)])
+def test_the_cache_key_and_the_connection_share_one_canary_decision(
+ team: Team, connection: bool, first: bool, later: bool
+) -> None:
+ runner = _canary_runner(team, connection=connection)
+ with (
+ _canary_runner_boundaries() as (managed_trino, direct),
+ patch("posthog.hogql_queries.hogql_query_runner.use_trino_canary", side_effect=[first, later, later]) as decide,
+ ):
+ variants = {runner.get_cache_key_variant(), runner.get_cache_key_variant()}
+ runner._calculate()
+ assert variants == {"_trino_canary" if first else ""}
+ assert decide.call_count == 1
+ used_canary = direct.call_args.kwargs["trino_canary"] if connection else managed_trino.call_args.kwargs["canary"]
+ assert used_canary is first
+
+
[email protected]("canary", [False, True])
+def test_a_managed_trino_run_connects_the_way_the_runner_decided(team: Team, canary: bool) -> None:
+ user = User(id=9, distinct_id="user-9")
+ cursor = MagicMock()
+ cursor.fetchmany.return_value = [(1,)]
+ cursor.description = [("value", "bigint")]
+ connection = MagicMock()
+ connection.cursor.return_value = cursor
+ with (
+ patch("posthoganalytics.feature_enabled", return_value=True),
+ patch("products.managed_warehouse.backend.query_execution.enable_customer_warehouse_access", return_value=True),
+ patch(
+ "products.managed_warehouse.backend.query_execution.UserAccessControl.has_project_access",
+ new_callable=PropertyMock,
+ return_value=True,
+ ),
+ patch(
+ "products.managed_warehouse.backend.query_execution.connect_managed_warehouse_trino",
+ return_value=nullcontext(connection),
+ ) as connect,
+ ):
+ execute_managed_trino_query(
+ HogQLQuery(query="SELECT 1", executionTarget="managed_trino", sendRawQuery=True),
+ team=team,
+ user=user,
+ user_access_control=None,
+ settings=HogQLGlobalSettings(),
+ timings=HogQLTimings(),
+ limit_context=LimitContext.QUERY,
+ canary=canary,
+ )
+ connect.assert_called_once_with(
+ str(team.organization_id), principal=f"posthog:sql-editor:team:{team.pk}:user:{user.pk}", canary=canary
+ )
+
+
@pytest.mark.parametrize("flag_result", [False, None, RuntimeError("flag unavailable")])
@pytest.mark.parametrize("raw", [False, True])
def test_disabled_target_rejects_cached_requests(team: Team, flag_result: object, raw: bool) -> None:
@@ -131,7 +273,7 @@ def test_live_query_binds_values_and_uses_hosted_service_credentials(
limit_context=limit_context,
)
connect.assert_called_once_with(
- str(team.organization_id), principal=f"posthog:sql-editor:team:{team.pk}:user:{user.pk}"
+ str(team.organization_id), principal=f"posthog:sql-editor:team:{team.pk}:user:{user.pk}", canary=False
)
cursor.execute.assert_called_once_with(
f'SELECT ? AS "value" OFFSET 0 ROWS LIMIT {expected_limit + 1}', ["quoted ' value"]
diff --git a/products/managed_warehouse/backend/tests/test_query_schema.py b/products/managed_warehouse/backend/tests/test_query_schema.py
index 98a1ded119b3..bda790c15e23 100644
--- a/products/managed_warehouse/backend/tests/test_query_schema.py
+++ b/products/managed_warehouse/backend/tests/test_query_schema.py
@@ -4,21 +4,28 @@ from uuid import UUID
import pytest
from unittest.mock import MagicMock, PropertyMock, patch
+from django.test import override_settings
+
from posthog.schema import DatabaseSchemaQuery
from posthog.hogql.errors import ExposedHogQLError
from posthog.api.services.query import process_database_schema_query
-from posthog.clickhouse.query_tagging import AccessMethod, QueryTags
+from posthog.clickhouse.query_tagging import AccessMethod, QueryTags, tags_context
from posthog.exceptions import DatabaseSchemaUnavailable
from posthog.models import Team, User
[email protected]("include_fields,table_names", [(False, None), (True, None), (True, ["analytics.orders"])])
+ "include_fields,table_names,canary",
+ [(False, None, False), (True, None, False), (True, ["analytics.orders"], True)],
+)
@pytest.mark.django_db
-def test_native_schema_uses_warehouse_inventory(include_fields: bool, table_names: list[str] | None) -> None:
+def test_native_schema_uses_warehouse_inventory(
+ include_fields: bool, table_names: list[str] | None, canary: bool
+) -> None:
team = Team(id=7, organization_id=UUID("00000000-0000-0000-0000-000000000001"))
- user = User(id=9)
+ user = User(id=9, distinct_id="user-9")
cursor = MagicMock()
cursor.fetchall.side_effect = [
[("analytics", "orders"), ("posthog", "events_example")],
@@ -28,6 +35,8 @@ def test_native_schema_uses_warehouse_inventory(include_fields: bool, table_name
connection = MagicMock(catalog="example_catalog")
connection.cursor.return_value = cursor
with (
+ override_settings(MANAGED_WAREHOUSE_TRINO_CANARY_DOMAIN="canary.dw.us.postwh.com" if canary else ""),
+ tags_context(trigger=None, feature=None, alert_config_id=None),
patch("posthoganalytics.feature_enabled", return_value=True),
patch("products.managed_warehouse.backend.query_execution.get_query_tags", return_value=QueryTags()),
patch(
@@ -56,7 +65,9 @@ def test_native_schema_uses_warehouse_inventory(include_fields: bool, table_name
else:
assert all(not table.fields for table in response.tables.values())
assert cursor.execute.call_count == 1
- connect.assert_called_once_with(str(team.organization_id), principal="posthog:sql-editor:team:7:user:9")
+ connect.assert_called_once_with(
+ str(team.organization_id), principal="posthog:sql-editor:team:7:user:9", canary=canary
+ )
cursor.close.assert_called_once()
diff --git a/products/managed_warehouse/backend/tests/test_trino_connection.py b/products/managed_warehouse/backend/tests/test_trino_connection.py
index c9f853ff3b8b..46588334a529 100644
--- a/products/managed_warehouse/backend/tests/test_trino_connection.py
+++ b/products/managed_warehouse/backend/tests/test_trino_connection.py
@@ -1,22 +1,32 @@
import os
+import uuid
import base64
from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime, timedelta
from threading import Event
+from urllib.parse import urlsplit
import pytest
from unittest import mock
+from django.test import override_settings
+
import requests
from rest_framework.response import Response
+from posthog.hogql.constants import LimitContext
+
+from posthog.clickhouse.query_tagging import Feature, tags_context
+
from products.managed_warehouse.backend.facade.contracts import ManagedWarehouseTrinoConnectionUnavailable
from products.managed_warehouse.backend.trino_connection import (
connect_managed_warehouse_trino,
resolve_managed_warehouse_trino_connection,
+ use_trino_canary,
)
CREDENTIAL_ID = "svc_0123456789abcdef01234567"
+CANARY_DOMAIN = "canary.dw.us.postwh.com"
def _mint_response(**target_overrides: object) -> Response:
@@ -99,8 +109,18 @@ def test_rejects_invalid_trino_targets(target: dict[str, object]) -> None:
resolve_managed_warehouse_trino_connection("org-1")
+ "canary,canary_domain,host",
+ [
+ (False, "", "tenant.dw.us.postwh.com"),
+ (False, CANARY_DOMAIN, "tenant.dw.us.postwh.com"),
+ (True, CANARY_DOMAIN, "tenant.canary.dw.us.postwh.com"),
+ ],
+)
@pytest.mark.parametrize("renewal_target", ["included", "omitted", "null"])
-def test_polling_renews_the_grant_and_preserves_cancellation_ownership(renewal_target: str) -> None:
+def test_polling_renews_the_grant_and_preserves_cancellation_ownership(
+ renewal_target: str, canary: bool, canary_domain: str, host: str
+) -> None:
minted = _mint_response()
refreshed = _mint_response()
refreshed.data.pop("credential_secret")
@@ -119,16 +139,16 @@ def test_polling_renews_the_grant_and_preserves_cancellation_ownership(renewal_t
return _http_response(
request,
id="query-1",
- infoUri="https://tenant.dw.us.postwh.com/query-1",
- nextUri="https://tenant.dw.us.postwh.com/v1/statement/query-1",
+ infoUri=f"https://{host}/query-1",
+ nextUri=f"https://{host}/v1/statement/query-1",
stats={},
)
if request.method == "GET":
return _http_response(
request,
id="query-1",
- infoUri="https://tenant.dw.us.postwh.com/query-1",
- nextUri="https://tenant.dw.us.postwh.com/v1/statement/query-1",
+ infoUri=f"https://{host}/query-1",
+ nextUri=f"https://{host}/v1/statement/query-1",
columns=[{"name": "value", "type": "bigint", "typeSignature": {"rawType": "bigint", "arguments": []}}],
data=[[1]],
stats={},
@@ -138,18 +158,20 @@ def test_polling_renews_the_grant_and_preserves_cancellation_ownership(renewal_t
return response
with (
+ override_settings(MANAGED_WAREHOUSE_TRINO_CANARY_DOMAIN=canary_domain),
mock.patch(
"products.managed_warehouse.backend.presentation.views._request", side_effect=[minted, refreshed]
) as cp,
mock.patch("products.managed_warehouse.backend.trino_connection._utcnow", return_value=issued_at) as now,
mock.patch("requests.adapters.HTTPAdapter.send", side_effect=send),
):
- with connect_managed_warehouse_trino("org-1") as connection:
+ with connect_managed_warehouse_trino("org-1", canary=canary) as connection:
cursor = connection.cursor()
cursor.execute("SELECT 1")
now.return_value = issued_at + timedelta(minutes=14)
cursor.cancel()
assert cp.call_count == 2
+ assert {urlsplit(r.url).hostname for r in sent} == {host}
assert cp.call_args.args == ("POST", "org-1", "/service-credentials/refresh")
assert cp.call_args.kwargs["json_body"]["credential_id"] == CREDENTIAL_ID
assert cp.call_args.kwargs["json_body"]["rotate_secret"] is False
@@ -199,22 +221,67 @@ def test_failed_refresh_does_not_send_a_request(refresh_response: str) -> None:
send.assert_not_called()
+def test_canary_without_a_configured_domain_is_refused_before_a_credential_is_minted() -> None:
+ with (
+ override_settings(MANAGED_WAREHOUSE_TRINO_CANARY_DOMAIN=""),
+ mock.patch("products.managed_warehouse.backend.presentation.views._request") as cp,
+ pytest.raises(ManagedWarehouseTrinoConnectionUnavailable, match="canary domain is not configured"),
+ ):
+ with connect_managed_warehouse_trino("org-1", canary=True):
+ pass
+ cp.assert_not_called()
+
+
+ "host",
+ [
+ "trino.example.com",
+ "tenant",
+ "nested.tenant.dw.us.postwh.com",
+ "tenant.dw.us.postwh.com.example.com",
+ "tenant.canary.dw.us.postwh.com",
+ ],
+)
+def test_canary_route_needs_a_host_that_the_control_plane_issued_under_a_normal_domain(host: str) -> None:
+ with (
+ override_settings(MANAGED_WAREHOUSE_TRINO_CANARY_DOMAIN=CANARY_DOMAIN),
+ mock.patch(
+ "products.managed_warehouse.backend.presentation.views._request", return_value=_mint_response(host=host)
+ ),
+ mock.patch("requests.adapters.HTTPAdapter.send") as send,
+ pytest.raises(ManagedWarehouseTrinoConnectionUnavailable, match="hosted warehouse host"),
+ ):
+ with connect_managed_warehouse_trino("org-1", canary=True):
+ pass
+ send.assert_not_called()
+
+
@pytest.mark.parametrize(
- "host,port,bypass_proxy",
+ "host,port,bypass_proxy,canary_domain",
[
- ("tenant.dw.us.postwh.com", 443, True),
- ("tenant.dw.eu.postwh.com", 443, True),
- ("TENANT.DW.DEV.POSTWH.COM.", 443, True),
- ("tenant.dw.us.postwh.com", 8443, False),
- ("trino.example.com", 443, False),
- ("nested.tenant.dw.us.postwh.com", 443, False),
- ("dw.us.postwh.com", 443, False),
- ("tenant.dw.us.postwh.com.example.com", 443, False),
+ ("tenant.dw.us.postwh.com", 443, True, ""),
+ ("tenant.dw.eu.postwh.com", 443, True, ""),
+ ("TENANT.DW.DEV.POSTWH.COM.", 443, True, ""),
+ ("tenant.dw.us.postwh.com", 8443, False, ""),
+ ("trino.example.com", 443, False, ""),
+ ("nested.tenant.dw.us.postwh.com", 443, False, ""),
+ ("dw.us.postwh.com", 443, False, ""),
+ ("tenant.dw.us.postwh.com.example.com", 443, False, ""),
+ ("tenant", 443, False, ""),
+ ("tenant.canary.dw.us.postwh.com", 443, False, ""),
+ ("tenant.canary.dw.us.postwh.com", 443, True, CANARY_DOMAIN),
+ ("tenant.canary.dw.us.postwh.com", 8443, False, CANARY_DOMAIN),
+ ("nested.tenant.canary.dw.us.postwh.com", 443, False, CANARY_DOMAIN),
+ ("tenant.canary.dw.eu.postwh.com", 443, False, CANARY_DOMAIN),
+ ("tenant.canary.dw.us.postwh.com.example.com", 443, False, CANARY_DOMAIN),
],
)
-def test_connection_keeps_verified_tls_and_scoped_proxy_bypass(host: str, port: int, bypass_proxy: bool) -> None:
+def test_connection_keeps_verified_tls_and_scoped_proxy_bypass(
+ host: str, port: int, bypass_proxy: bool, canary_domain: str
+) -> None:
proxy = "http://proxy.example.com:4750"
with (
+ override_settings(MANAGED_WAREHOUSE_TRINO_CANARY_DOMAIN=canary_domain),
mock.patch.dict(os.environ, {"HTTPS_PROXY": proxy, "NO_PROXY": ""}, clear=True),
mock.patch(
"products.managed_warehouse.backend.presentation.views._request",
@@ -337,3 +404,52 @@ def test_cancellation_does_not_wait_for_a_blocked_poll() -> None:
release_poll.set()
assert poll.result(timeout=5).status_code == 200
assert cp.call_count == 2
+
+
+ "canary_domain,distinct_id,tags,limit_context,flag_result,evaluates_flag,expected",
+ [
+ (CANARY_DOMAIN, "user-1", {}, None, True, True, True),
+ (CANARY_DOMAIN, "user-1", {}, LimitContext.QUERY, True, True, True),
+ (CANARY_DOMAIN, "user-1", {}, LimitContext.QUERY_ASYNC, True, True, True),
+ (CANARY_DOMAIN, "user-1", {}, None, False, True, False),
+ (CANARY_DOMAIN, "user-1", {}, None, None, True, False),
+ (CANARY_DOMAIN, "user-1", {}, None, RuntimeError("flags unavailable"), True, False),
+ ("", "user-1", {}, None, True, False, False),
+ (CANARY_DOMAIN, None, {}, None, True, False, False),
+ (CANARY_DOMAIN, "", {}, None, True, False, False),
+ (CANARY_DOMAIN, "user-1", {"trigger": "warmingV2"}, None, True, False, False),
+ (CANARY_DOMAIN, "user-1", {"feature": Feature.CACHE_WARMUP}, None, True, False, False),
+ (CANARY_DOMAIN, "user-1", {"alert_config_id": uuid.uuid4()}, None, True, False, False),
+ (CANARY_DOMAIN, "user-1", {"feature": Feature.ALERTING}, None, True, False, False),
+ (CANARY_DOMAIN, "user-1", {}, LimitContext.EXPORT, True, False, False),
+ (CANARY_DOMAIN, "user-1", {}, LimitContext.SQL_ALERT, True, False, False),
+ (CANARY_DOMAIN, "user-1", {}, LimitContext.POSTHOG_AI, True, False, False),
+ ],
+)
+def test_use_trino_canary_needs_a_flagged_user_on_an_interactive_run(
+ canary_domain: str,
+ distinct_id: str | None,
+ tags: dict[str, object],
+ limit_context: LimitContext | None,
+ flag_result: object,
+ evaluates_flag: bool,
+ expected: bool,
+) -> None:
+ clean_tags = {"trigger": None, "feature": None, "alert_config_id": None}
+ with (
+ override_settings(MANAGED_WAREHOUSE_TRINO_CANARY_DOMAIN=canary_domain),
+ tags_context(**{**clean_tags, **tags}),
+ mock.patch(
+ "posthoganalytics.feature_enabled",
+ side_effect=flag_result if isinstance(flag_result, Exception) else None,
+ return_value=flag_result,
+ ) as flag,
+ ):
+ assert use_trino_canary(distinct_id, "org-1", limit_context=limit_context) is expected
+ assert flag.called is evaluates_flag
+ if evaluates_flag:
+ assert flag.call_args.args == ("managed-warehouse-trino-canary", distinct_id)
+ assert flag.call_args.kwargs["groups"] == {"organization": "org-1"}
+ assert flag.call_args.kwargs["only_evaluate_locally"] is False
+ assert flag.call_args.kwargs["send_feature_flag_events"] is False
diff --git a/products/managed_warehouse/backend/trino_connection.py b/products/managed_warehouse/backend/trino_connection.py
index d9c6df27b835..255f50b77a77 100644
--- a/products/managed_warehouse/backend/trino_connection.py
+++ b/products/managed_warehouse/backend/trino_connection.py
@@ -1,7 +1,7 @@
from __future__ import annotations
import re
-from collections.abc import Iterator, MutableMapping
+from collections.abc import Collection, Iterator, MutableMapping
from contextlib import contextmanager
from dataclasses import replace
from datetime import UTC, datetime, timedelta
@@ -9,10 +9,18 @@ from threading import Lock
from typing import TYPE_CHECKING
from urllib.parse import urlsplit
+from django.conf import settings
+
import requests
+import structlog
from requests.auth import HTTPBasicAuth
from requests.utils import resolve_proxies
+from posthog.hogql.constants import LimitContext
+
+from posthog.clickhouse.query_tagging import Feature, get_query_tags
+from posthog.ph_client import feature_enabled_or_false
+
from products.managed_warehouse.backend.facade.contracts import (
ManagedWarehouseTrinoConnection,
ManagedWarehouseTrinoConnectionUnavailable,
@@ -20,28 +28,95 @@ from products.managed_warehouse.backend.facade.contracts import (
ServiceCredentialTrinoConnect,
ServiceCredentialUnavailable,
)
+from products.managed_warehouse.backend.facade.feature_flags import MANAGED_WAREHOUSE_TRINO_CANARY_FLAG
from products.managed_warehouse.backend.service_credentials import mint_service_credential, renew_service_credential
if TYPE_CHECKING:
from trino.dbapi import Connection
+LOGGER = structlog.get_logger(__name__)
+
_CREDENTIAL_REFRESH_MARGIN = timedelta(minutes=2)
_CREDENTIAL_REQUEST_TIMEOUT_SECONDS = 10
+_HOSTED_SERVICE_DOMAINS = frozenset({"dw.us.postwh.com", "dw.eu.postwh.com", "dw.dev.postwh.com"})
+_AUTOMATED_LIMIT_CONTEXTS = frozenset({LimitContext.EXPORT, LimitContext.SQL_ALERT, LimitContext.POSTHOG_AI})
def _utcnow() -> datetime:
return datetime.now(UTC)
+def _canary_domain() -> str:
+ return settings.MANAGED_WAREHOUSE_TRINO_CANARY_DOMAIN.strip().lower().strip(".")
+
+
+def _require_canary_domain() -> str:
+ domain = _canary_domain()
+ if not domain:
+ raise ManagedWarehouseTrinoConnectionUnavailable("The Trino canary domain is not configured")
+ return domain
+
+
+def _is_single_label_under(host: str, domains: Collection[str]) -> bool:
+ label, _, domain = host.lower().removesuffix(".").partition(".")
+ return domain in domains and re.fullmatch(r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?", label) is not None
+
+
def _is_hosted_service_target(host: str) -> bool:
# Tenant endpoints are trusted only when issued by the control plane, never from external-source configuration.
- label, _, domain = host.lower().removesuffix(".").partition(".")
+ canary_domain = _canary_domain()
+ # An empty canary domain must stay out of the set, because a host without a domain would match it.
+ return _is_single_label_under(host, _HOSTED_SERVICE_DOMAINS | ({canary_domain} if canary_domain else set()))
+
+
+def _with_canary_host(connection: ManagedWarehouseTrinoConnection, canary: bool) -> ManagedWarehouseTrinoConnection:
+ if not canary:
+ return connection
+ domain = _require_canary_domain()
+ # Only a host that the control plane issued under a normal domain has a canary equivalent.
+ if not _is_single_label_under(connection.host, _HOSTED_SERVICE_DOMAINS):
+ raise ManagedWarehouseTrinoConnectionUnavailable("The Trino canary route needs a hosted warehouse host")
+ # The canary host keeps the database label of the minted host, so the same service credential applies.
+ label = connection.host.lower().removesuffix(".").partition(".")[0]
+ return replace(connection, host=f"{label}.{domain}")
+
+
+def _is_automated_run(limit_context: LimitContext | None) -> bool:
+ # The alert and warming code sets these tags before it calls any query runner, so they are visible in every
+ # layer. Exports and subscriptions run as their creator and carry no tag, so the limit context identifies them.
+ tags = get_query_tags()
return (
- domain in {"dw.us.postwh.com", "dw.eu.postwh.com", "dw.dev.postwh.com"}
- and re.fullmatch(r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?", label) is not None
+ limit_context in _AUTOMATED_LIMIT_CONTEXTS
+ or tags.alert_config_id is not None
+ or tags.feature in (Feature.ALERTING, Feature.CACHE_WARMUP)
+ or (tags.trigger or "").startswith("warming")
)
+def use_trino_canary(
+ distinct_id: str | None, organization_id: str, *, limit_context: LimitContext | None = None
+) -> bool:
+ """Whether an interactive run for this user connects through the canary hostname.
+
+ The cheap checks come first, so the remote flag evaluation runs only for a user who could reach the canary.
+ A caller decides once per query and reuses the result, because each call evaluates the flag again.
+ """
+ if not _canary_domain() or not distinct_id or _is_automated_run(limit_context):
+ return False
+ try:
+ return feature_enabled_or_false(
+ MANAGED_WAREHOUSE_TRINO_CANARY_FLAG,
+ distinct_id,
+ groups={"organization": organization_id},
+ group_properties={"organization": {"id": organization_id}},
+ only_evaluate_locally=False,
+ send_feature_flag_events=False,
+ )
+ except Exception:
+ LOGGER.warning("Failed to evaluate the Trino canary feature flag", organization_id=organization_id)
+ return False
+
+
def _connection_from_credential(credential: ServiceCredential) -> ManagedWarehouseTrinoConnection:
target = credential.trino_connect
if (
@@ -81,10 +156,13 @@ def resolve_managed_warehouse_trino_connection(
class _TrinoServiceSession(requests.Session):
- def __init__(self, organization_id: str, connection: ManagedWarehouseTrinoConnection) -> None:
+ def __init__(
+ self, organization_id: str, connection: ManagedWarehouseTrinoConnection, *, canary: bool = False
+ ) -> None:
super().__init__()
self._organization_id = organization_id
self._connection = connection
+ self._canary = canary
self._lock = Lock()
# Defer authentication until send so queued requests also check credential expiry.
self.auth = self._defer_authentication
@@ -129,7 +207,8 @@ class _TrinoServiceSession(requests.Session):
credential_secret=connection.password,
timeout_seconds=_CREDENTIAL_REQUEST_TIMEOUT_SECONDS,
)
- if credential.trino_connect is None:
+ reused_target = credential.trino_connect is None
+ if reused_target:
# Readiness can hide dial targets during renewal without invalidating an existing query's grant.
credential = replace(
credential,
@@ -146,6 +225,9 @@ class _TrinoServiceSession(requests.Session):
raise ManagedWarehouseTrinoConnectionUnavailable(
"Could not renew the Trino service credential"
) from error
+ # The control plane returns the normal host again. A reused target already holds the canary host.
+ if not reused_target:
+ refreshed = _with_canary_host(refreshed, self._canary)
if (
refreshed.credential_id != connection.credential_id
or refreshed.host != connection.host
@@ -174,11 +256,15 @@ class _TrinoServiceSession(requests.Session):
@contextmanager
-def connect_managed_warehouse_trino(organization_id: str, *, principal: str = "posthog:trino") -> Iterator[Connection]:
+def connect_managed_warehouse_trino(
+ organization_id: str, *, principal: str = "posthog:trino", canary: bool = False
+) -> Iterator[Connection]:
from trino.dbapi import connect # noqa: PLC0415 -- keeps the optional driver off startup paths
- config = resolve_managed_warehouse_trino_connection(organization_id, principal=principal)
- with _TrinoServiceSession(organization_id, config) as http_session:
+ if canary:
+ _require_canary_domain()
+ config = _with_canary_host(resolve_managed_warehouse_trino_connection(organization_id, principal=principal), canary)
+ with _TrinoServiceSession(organization_id, config, canary=canary) as http_session:
if _is_hosted_service_target(config.host) and config.port == 443:
http_session.trust_env = False
connection = connect(