:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(managed-warehouse): route flagged users to a trino canary host (#127937)
Eric Duong23ab65863ebd
diff --git a/posthog/clickhouse/client/execute_async.py b/posthog/clickhouse/client/execute_async.pyindex 92ed6fca50d0..e3d5abcac8df 100644--- a/posthog/clickhouse/client/execute_async.py+++ b/posthog/clickhouse/client/execute_async.py@@ -239,7 +239,7 @@ def execute_process_query(user: Optional[User] = Noneif user_id:- user = User.objects.only("email", "is_staff").get(pk=user_id)+ user = User.objects.only("email", "is_staff", "distinct_id").get(pk=user_id)is_staff_user = user.is_staffelif sharing_configuration_id:# A shared-link viewer has no user row, so the identity has to be rebuilt from the share itdiff --git a/posthog/dags/README_DUCKLINGS.md b/posthog/dags/README_DUCKLINGS.mdindex 5e06fbd7959a..4c1ca200de59 100644--- a/posthog/dags/README_DUCKLINGS.md+++ b/posthog/dags/README_DUCKLINGS.md@@ -29,6 +29,9 @@ The ClickHouse export is the same, except that it casts each column to the ParquAn export-only run (`skip_ducklake_registration: true`) writes the same types, so `add_files` can register its files later.ClickHouse lists the files it exported, because nothing else on the path can list the duckling bucket.Each export writes under a new file prefix, so a retry never overwrites a file that DuckLake has already registered.+A Trino backfill connects through the normal Trino hostname.+Set `use_trino_canary: true` in the run config to connect through the canary hostname instead, which needs `MANAGED_WAREHOUSE_TRINO_CANARY_DOMAIN` to be set. Without it, the run fails.+The flag `managed-warehouse-trino-canary` does not apply to backfills.One `add_files` call registers the files of an export and removes the rows they replace in a single DuckLake snapshot:@@ -192,6 +195,7 @@ class DucklingBackfillConfig:create_tables_if_missing: bool = True # Auto-create events/persons tablesdelete_tables: bool = False # DANGER: Drop and recreate tablesdry_run: bool = False # Preview mode, no writes+ use_trino_canary: bool = False # Trino path only: connect through the canary hostnameevents_parquet_row_group_size_bytes: int = 536_870_912 # Events only; 512 MiBtarget_rows_per_file: int = 5_000_000 # Target rows per exported filemax_s3_file_fanout: int = 256 # Maximum before the automatic buffer-budget limitdiff --git a/posthog/dags/events_backfill_to_duckling.py b/posthog/dags/events_backfill_to_duckling.pyindex 77066e04e0cb..e5a091cf4ac6 100644--- a/posthog/dags/events_backfill_to_duckling.py+++ b/posthog/dags/events_backfill_to_duckling.py@@ -1036,6 +1036,8 @@ class DucklingBackfillConfig(Config):create_tables_if_missing: bool = Truedelete_tables: bool = False # Danger: drops and recreates tables, losing all datadry_run: bool = False+ # Opt-in per run. A backfill never evaluates the canary feature flag.+ use_trino_canary: bool = Falseevents_parquet_row_group_size_bytes: int = pydantic.Field(default=DEFAULT_EVENTS_PARQUET_ROW_GROUP_SIZE_BYTES,gt=0,@@ -2611,7 +2613,9 @@ def backfill_through_trino(partitions_exported = 0files_registered = 0register = not (config.dry_run or config.skip_ducklake_registration)- with connect_backfill_catalog(target.organization_id) if register else nullcontext() as catalog:+ with (+ connect_backfill_catalog(target.organization_id, canary=config.use_trino_canary) if register else nullcontext()+ ) as catalog:if catalog is not None:catalog.ensure_table(table,diff --git a/posthog/dags/test_events_backfill_to_duckling.py b/posthog/dags/test_events_backfill_to_duckling.pyindex baae112a0c68..12fb9111b86d 100644--- a/posthog/dags/test_events_backfill_to_duckling.py+++ b/posthog/dags/test_events_backfill_to_duckling.py@@ -2746,7 +2746,7 @@ class TestBackfillThroughTrino:catalog, connect_catalog = self._run(run, partition_key, clickhouse, DucklingBackfillConfig())- connect_catalog.assert_called_with("org-1")+ connect_catalog.assert_called_with("org-1", canary=False)assert catalog.tables == [(table, column_types)] * 2assert [(t, r) for t, _, _, r in catalog.registered] == [(table, replacement)] * 2first, second = catalog.registered@@ -2804,6 +2804,20 @@ class TestBackfillThroughTrino:assert cast_column in insertassert settings["output_format_parquet_string_as_string"] == 1+ @parameterized.expand(+ [+ ("events", _run_duckling_events_backfill, "2_2026-06-17"),+ ("persons", _run_duckling_persons_backfill, "2"),+ ]+ )+ @time_machine.travel(datetime(2026, 7, 1, tzinfo=UTC), tick=False)+ def test_opted_in_runs_register_through_the_canary_host(self, _name, run, partition_key):+ _, connect_catalog = self._run(+ run, partition_key, _FakeExportClickHouse(files_per_export=1), DucklingBackfillConfig(use_trino_canary=True)+ )++ connect_catalog.assert_called_with("org-1", canary=True)+@time_machine.travel(datetime(2026, 7, 1, tzinfo=UTC), tick=False)def test_dry_runs_neither_export_nor_open_the_catalog(self):clickhouse = _FakeExportClickHouse(files_per_export=2)diff --git a/posthog/hogql/direct_sql/adapter.py b/posthog/hogql/direct_sql/adapter.pyindex ba127f0d5883..96dbac2e5926 100644--- a/posthog/hogql/direct_sql/adapter.py+++ b/posthog/hogql/direct_sql/adapter.py@@ -39,6 +39,8 @@ def parse_direct_source_config(source_impl: _ParseableSource[_ConfigT_co], sourc@frozenclass DirectQueryPrincipal:value: str+ # The query runner decides once per query, so the adapter never evaluates the canary flag itself.+ trino_canary: bool = False@dataclasses.dataclass(frozen=False)diff --git a/posthog/hogql/query.py b/posthog/hogql/query.pyindex cb247a898b50..45c05ed95d2e 100644--- a/posthog/hogql/query.py+++ b/posthog/hogql/query.py@@ -203,6 +203,7 @@ class HogQLQueryExecutor:direct_dialect: Optional[HogQLDialect] = Noneconnection_id: Optional[str] = Noneexpected_direct_engine: str | None = None+ trino_canary: bool = Falsesend_raw_query: bool = Falseuser: Optional[User] = Nonebypass_warehouse_access_control: bool = False@@ -760,7 +761,10 @@ class HogQLQueryExecutor:query_type=self.query_type,debug=bool(self.debug),principal=(- DirectQueryPrincipal(value=f"posthog:sql-editor:team:{self.team.pk}:user:{self.user.pk}")+ DirectQueryPrincipal(+ value=f"posthog:sql-editor:team:{self.team.pk}:user:{self.user.pk}",+ trino_canary=self.trino_canary,+ )if isinstance(self.user, User) and self.user.pk is not Noneelse None),diff --git a/posthog/hogql/test/test_direct_duckgres_query.py b/posthog/hogql/test/test_direct_duckgres_query.pyindex 2e30469ebe51..80df3eacd90e 100644--- a/posthog/hogql/test/test_direct_duckgres_query.py+++ b/posthog/hogql/test/test_direct_duckgres_query.py@@ -219,9 +219,11 @@ class TestDirectDuckgresQuery(APIBaseTest):self.resolve_hostaddr = resolver_patcher.start()self.addCleanup(resolver_patcher.stop)- @parameterized.expand([(True,), (False,)])+ @parameterized.expand([(True, False), (False, True)])@patch("posthoganalytics.feature_enabled", return_value=True)- def test_managed_source_switches_raw_and_hogql_execution_to_trino(self, raw: bool, _flag: MagicMock) -> None:+ def test_managed_source_switches_raw_and_hogql_execution_to_trino(+ self, raw: bool, canary: bool, _flag: MagicMock+ ) -> None:_flag.side_effect = lambda flag, *_args, **_kwargs: flag == "managed-warehouse-trino-query"source = self._dynamic_managed_source()source.job_inputs = {}@@ -258,6 +260,7 @@ class TestDirectDuckgresQuery(APIBaseTest):user=self.user,connection_id=str(source.id),send_raw_query=raw,+ trino_canary=canary,).execute()with self.assertRaisesMessage(ExposedHogQLError, "warehouse engine changed"):HogQLQueryExecutor(@@ -270,7 +273,9 @@ class TestDirectDuckgresQuery(APIBaseTest):).execute()self.assertEqual(response.results, [("row-uuid",)])connect.assert_called_once_with(- str(self.team.organization_id), principal=f"posthog:sql-editor:team:{self.team.pk}:user:{self.user.pk}"+ str(self.team.organization_id),+ principal=f"posthog:sql-editor:team:{self.team.pk}:user:{self.user.pk}",+ canary=canary,)duckgres.assert_not_called()self.assertIn('"posthog"."events"', cursor.execute.call_args.args[0])diff --git a/posthog/hogql_queries/hogql_query_runner.py b/posthog/hogql_queries/hogql_query_runner.pyindex 73dacce6bf4e..150d66308cda 100644--- a/posthog/hogql_queries/hogql_query_runner.py+++ b/posthog/hogql_queries/hogql_query_runner.py@@ -42,6 +42,7 @@ from posthog.models.activity_logging.retention import get_activity_log_lookback_from products.managed_warehouse.backend.facade import query_labels as managed_warehouse_query_labelsfrom products.managed_warehouse.backend.facade.query_execution import (execute_managed_trino_query,+ use_trino_canary,validate_managed_trino_query,)from products.warehouse_sources.backend.facade.types import ManagedWarehouseSQLMode@@ -67,6 +68,7 @@ class HogQLQueryRunner(AnalyticsQueryRunner[HogQLQueryResponse]):self._direct_connection_validated = Falseself._direct_engine: str | None = Noneself._managed_warehouse_sql_mode: ManagedWarehouseSQLMode | None = None+ self._trino_canary = Falsesuper().__init__(*args, **kwargs)def _reads_events_route(self) -> bool:@@ -104,8 +106,21 @@ class HogQLQueryRunner(AnalyticsQueryRunner[HogQLQueryResponse]):direct_engine = adapter.engine if adapter is not None else Noneself._managed_warehouse_sql_mode = managed_warehouse_sql_modeself._direct_engine = direct_engine+ self._trino_canary = self._decide_trino_canary(user if user is not None else self.user)self._direct_connection_validated = True+ def _decide_trino_canary(self, user: Optional[User]) -> bool:+ """Decide the Trino lane once per validation. The cache key and the connection both read the result, so+ a flag change between the two steps cannot store a result under the key of the other lane."""+ runs_on_managed_trino = self.query.executionTarget == ExecutionTarget.MANAGED_TRINO or (+ self._managed_warehouse_sql_mode == ManagedWarehouseSQLMode.BUILT_IN and self._direct_engine == "trino"+ )+ return (+ runs_on_managed_trino+ and user is not None+ and use_trino_canary(user.distinct_id, str(self.team.organization_id), limit_context=self.limit_context)+ )+def get_cache_payload(self) -> dict:self._validate_direct_connection()payload = super().get_cache_payload()@@ -137,6 +152,12 @@ class HogQLQueryRunner(AnalyticsQueryRunner[HogQLQueryResponse]):return payload+ def get_cache_key_variant(self) -> str:+ variant = super().get_cache_key_variant()+ self._validate_direct_connection()+ # The canary lane can return different results, so they never share a cache entry with the normal lane.+ return f"{variant}_trino_canary" if self._trino_canary else variant+def query_status_labels(self) -> list[str] | None:self._validate_direct_connection()if self.query.executionTarget == ExecutionTarget.MANAGED_TRINO and self.query.sendRawQuery:@@ -280,6 +301,7 @@ class HogQLQueryRunner(AnalyticsQueryRunner[HogQLQueryResponse]):settings=self.settings or HogQLGlobalSettings(),timings=self.timings,limit_context=self.limit_context,+ canary=self._trino_canary,)if self.query.sendRawQuery and self.query.connectionId:@@ -300,6 +322,7 @@ class HogQLQueryRunner(AnalyticsQueryRunner[HogQLQueryResponse]):settings=self.settings,send_raw_query=True,expected_direct_engine=self._direct_engine,+ trino_canary=self._trino_canary,)query = self._parse_query()[0] if self._direct_engine == "trino" else self.to_query()@@ -344,6 +367,7 @@ class HogQLQueryRunner(AnalyticsQueryRunner[HogQLQueryResponse]):ch_user=self.ch_user,settings=self.settings,expected_direct_engine=self._direct_engine,+ trino_canary=self._trino_canary,**context_kwargs,)if paginator:diff --git a/posthog/settings/base_variables.py b/posthog/settings/base_variables.pyindex 9508fd7274ad..d38572bb7a50 100644--- a/posthog/settings/base_variables.py+++ b/posthog/settings/base_variables.py@@ -93,6 +93,10 @@ HOGTOWER_INTERNAL_SECRET: str | None = get_from_env("HOGTOWER_INTERNAL_SECRET",# from the private-address check only when every resolved address is in this set, so an empty value# refuses every managed warehouse host. posthog-cloud-infra owns the values.MANAGED_WAREHOUSE_INGRESS_IPS: str = get_from_env("MANAGED_WAREHOUSE_INGRESS_IPS", "", type_cast=str)+# MANAGED_WAREHOUSE_TRINO_CANARY_DOMAIN is the domain whose hosts reach the canary Trino lane, for example+# `canary.dw.us.postwh.com`. A canary connection replaces the host of a minted credential with+# `<database label>.<this domain>`. An empty value turns the canary route off.+MANAGED_WAREHOUSE_TRINO_CANARY_DOMAIN: str = get_from_env("MANAGED_WAREHOUSE_TRINO_CANARY_DOMAIN", "", type_cast=str)MANAGED_WAREHOUSE_CREDENTIAL_CACHE_REDIS_CONNECT_TIMEOUT_SECONDS: float = get_from_env("MANAGED_WAREHOUSE_CREDENTIAL_CACHE_REDIS_CONNECT_TIMEOUT_SECONDS", 0.5, type_cast=float)diff --git a/products/managed_warehouse/backend/README.md b/products/managed_warehouse/backend/README.mdindex 1ffb14ab5743..fbd04530474b 100644--- a/products/managed_warehouse/backend/README.md+++ b/products/managed_warehouse/backend/README.md@@ -130,6 +130,32 @@ The data-import copy and registration paths target the same stable DuckLake tablThe two organization flags also select the Data ops variant. `managed-warehouse-trino-query` gives the Trino variant and wins when both are on. `data-warehouse-scene` alone gives the DuckDB variant. `data_ops_variant` in `presentation/views.py` is the single backend implementation of that rule.+## Trino canary hostname++`MANAGED_WAREHOUSE_TRINO_CANARY_DOMAIN` (empty by default) names the domain of the canary Trino lane, for example `canary.dw.us.postwh.com`.+A canary connection keeps the minted credential and replaces the host with `<database label>.<canary domain>`.+The minted host must be a single label under `dw.<region>.postwh.com`.+Any other minted host, or an empty canary domain, makes a canary connection fail with an explicit error instead of falling back to the normal host.++An interactive run uses the canary host when the requesting user has the `managed-warehouse-trino-canary` flag, evaluated with `user.distinct_id` and the organization group.+This covers the SQL editor, the schema browser, and the managed warehouse connection of the SQL editor.+These runs never use the canary host:++- Alert runs and cache-warming runs, which carry alert and warming query tags.+- Runs with the export, SQL alert, or PostHog AI limit context. Exports and subscriptions run as their creator and carry no tag, so the limit context identifies them.+- Schema discovery, data modeling, and model aliases, which never pass the canary option.++A Dagster backfill uses the canary host only when its run config sets `use_trino_canary: true`.+It does not evaluate the flag.++The query runner decides once per query and uses that decision for the cache key and for the connection.+A canary result is cached under its own key, so the normal lane never serves it.+The decision is made again for each runner, so the web request and the Celery task that continues it each agree with themselves.++Before you turn the flag on, run one real query through the canary host.+The canary host must return `nextUri` and `infoUri` values on the canary host.+The connection accepts only responses that name the host it dialed, so a response on the normal host fails with "Trino returned a different service credential destination".+## Data Ops workflow statusThe copy and registration workflows both write their lifecycle to `ManagedWarehouseSourceJob`. Each row identifies the project, source schema, external data job, workflow type, and workflow attempt. The supported states are running, completed, failed, skipped, and stale.diff --git a/products/managed_warehouse/backend/backfill_trino.py b/products/managed_warehouse/backend/backfill_trino.pyindex 77ad053d3852..d461aba71661 100644--- a/products/managed_warehouse/backend/backfill_trino.py+++ b/products/managed_warehouse/backend/backfill_trino.py@@ -107,8 +107,10 @@ class TrinoBackfillCatalog:@classmethod@contextlib.contextmanager- def connect(cls, organization_id: str) -> Iterator[TrinoBackfillCatalog]:- with connect_managed_warehouse_trino(organization_id, principal=BACKFILL_TRINO_PRINCIPAL) as connection:+ def connect(cls, organization_id: str, *, canary: bool = False) -> Iterator[TrinoBackfillCatalog]:+ with connect_managed_warehouse_trino(+ organization_id, principal=BACKFILL_TRINO_PRINCIPAL, canary=canary+ ) as connection:yield cls(connection)def _table(self, table: str) -> str:diff --git a/products/managed_warehouse/backend/facade/backfill_trino.py b/products/managed_warehouse/backend/facade/backfill_trino.pyindex b74d5c9fa230..183580cf73b0 100644--- a/products/managed_warehouse/backend/facade/backfill_trino.py+++ b/products/managed_warehouse/backend/facade/backfill_trino.py@@ -14,5 +14,7 @@ def backfills_through_trino(organization_id: str) -> bool:return backfill_trino.backfills_through_trino(organization_id)-def connect_backfill_catalog(organization_id: str) -> AbstractContextManager[DuckLakeBackfillCatalog]:- return backfill_trino.TrinoBackfillCatalog.connect(organization_id)+def connect_backfill_catalog(+ organization_id: str, *, canary: bool = False+) -> AbstractContextManager[DuckLakeBackfillCatalog]:+ return backfill_trino.TrinoBackfillCatalog.connect(organization_id, canary=canary)diff --git a/products/managed_warehouse/backend/facade/feature_flags.py b/products/managed_warehouse/backend/facade/feature_flags.pyindex e8980d00d8ed..3908bcae9d22 100644--- a/products/managed_warehouse/backend/facade/feature_flags.py+++ b/products/managed_warehouse/backend/facade/feature_flags.py@@ -3,5 +3,7 @@ MANAGED_TRINO_QUERY_FLAG = "managed-warehouse-trino-query"# Either flag gives an organization the managed warehouse.MANAGED_WAREHOUSE_ACCESS_FLAGS = (MANAGED_TRINO_QUERY_FLAG, DATA_WAREHOUSE_SCENE_FLAG)DATA_MODELING_SHADOW_FLAG = "managed-warehouse-data-modeling-shadow"+# Evaluated for the requesting user. A user with the flag reaches Trino through the canary hostname.+MANAGED_WAREHOUSE_TRINO_CANARY_FLAG = "managed-warehouse-trino-canary"# Marks an organization whose warehouse has no duckgres, so its data imports and backfills register through Trino.DUCKLAKE_TRINO_REGISTRATION_FLAG = "managed-warehouse-trino-import-registration"diff --git a/products/managed_warehouse/backend/facade/query_execution.py b/products/managed_warehouse/backend/facade/query_execution.pyindex 87a4b8c638dc..a03c27d2db64 100644--- a/products/managed_warehouse/backend/facade/query_execution.py+++ b/products/managed_warehouse/backend/facade/query_execution.py@@ -38,6 +38,7 @@ def execute_managed_trino_query(settings: HogQLGlobalSettings,timings: HogQLTimings,limit_context: LimitContext | None,+ canary: bool = False,) -> HogQLQueryResponse:from products.managed_warehouse.backend.query_execution import (execute_managed_trino_query as execute, # noqa: PLC0415 -- query compilation is optional during Django startup@@ -51,9 +52,20 @@ def execute_managed_trino_query(settings=settings,timings=timings,limit_context=limit_context,+ canary=canary,)+def use_trino_canary(+ distinct_id: str | None, organization_id: str, *, limit_context: LimitContext | None = None+) -> bool:+ from products.managed_warehouse.backend.trino_connection import (+ use_trino_canary as decide, # noqa: PLC0415 -- keeps the optional Trino connection path off Django startup+ )++ return decide(distinct_id, organization_id, limit_context=limit_context)++def get_managed_trino_schema(query: DatabaseSchemaQuery, *, team: Team, user: User | None) -> DatabaseSchemaQueryResponse:diff --git a/products/managed_warehouse/backend/logic/sql_editor_trino.py b/products/managed_warehouse/backend/logic/sql_editor_trino.pyindex f9a5a3ce1c10..1cdf27373b53 100644--- a/products/managed_warehouse/backend/logic/sql_editor_trino.py+++ b/products/managed_warehouse/backend/logic/sql_editor_trino.py@@ -63,7 +63,11 @@ class ManagedTrinoAdapter:debug=request.debug,cancellation_token=request.cancellation_token,),- connect_managed_warehouse_trino(str(request.team.organization_id), principal=request.principal.value),+ connect_managed_warehouse_trino(+ str(request.team.organization_id),+ principal=request.principal.value,+ canary=request.principal.trino_canary,+ ),)except (ExposedHogQLError, ManagedWarehouseTrinoConnectionUnavailable) as error:if not isinstance(error, ManagedWarehouseTrinoConnectionUnavailable) and not isinstance(diff --git a/products/managed_warehouse/backend/query_execution.py b/products/managed_warehouse/backend/query_execution.pyindex 3fc891e838ff..db345998691e 100644--- a/products/managed_warehouse/backend/query_execution.py+++ b/products/managed_warehouse/backend/query_execution.py@@ -80,6 +80,7 @@ def execute_managed_trino_query(settings: HogQLGlobalSettings,timings: HogQLTimings,limit_context: LimitContext | None,+ canary: bool = False,) -> HogQLQueryResponse:validate_managed_trino_query(query, team, user)assert user is not None@@ -131,7 +132,9 @@ def execute_managed_trino_query(cancellation_token=cancellation_token,),connect_managed_warehouse_trino(- str(team.organization_id), principal=f"posthog:sql-editor:team:{team.pk}:user:{user.pk}"+ str(team.organization_id),+ principal=f"posthog:sql-editor:team:{team.pk}:user:{user.pk}",+ canary=canary,),)response = HogQLQueryResponse(diff --git a/products/managed_warehouse/backend/query_schema.py b/products/managed_warehouse/backend/query_schema.pyindex c1a44dea518e..585c48085516 100644--- a/products/managed_warehouse/backend/query_schema.py+++ b/products/managed_warehouse/backend/query_schema.py@@ -17,7 +17,7 @@ from posthog.hogql.escape_sql import quote_hogql_identifierfrom products.managed_warehouse.backend.facade.contracts import ManagedWarehouseTrinoConnectionUnavailablefrom products.managed_warehouse.backend.logic.sql_editor_trino import MANAGED_TRINO_SCHEMA_DISCOVERY_TIMEOUT_SECONDSfrom products.managed_warehouse.backend.query_execution import validate_managed_trino_query-from products.managed_warehouse.backend.trino_connection import connect_managed_warehouse_trino+from products.managed_warehouse.backend.trino_connection import connect_managed_warehouse_trino, use_trino_canaryfrom products.warehouse_sources.backend.facade.models import trino_column_to_dwh_columnfrom products.warehouse_sources.backend.facade.source_management import discover_trino_catalog@@ -37,7 +37,9 @@ def get_managed_trino_schema()assert user is not Nonewith connect_managed_warehouse_trino(- str(team.organization_id), principal=f"posthog:sql-editor:team:{team.pk}:user:{user.pk}"+ str(team.organization_id),+ principal=f"posthog:sql-editor:team:{team.pk}:user:{user.pk}",+ canary=use_trino_canary(user.distinct_id, str(team.organization_id)),) as connection:with bounded_trino_cursor(connection, MANAGED_TRINO_SCHEMA_DISCOVERY_TIMEOUT_SECONDS, team_id=team.pk) as (cursor,diff --git a/products/managed_warehouse/backend/tests/test_backfill_trino.py b/products/managed_warehouse/backend/tests/test_backfill_trino.pyindex 4768ee0e6066..cd44643c90c1 100644--- a/products/managed_warehouse/backend/tests/test_backfill_trino.py+++ b/products/managed_warehouse/backend/tests/test_backfill_trino.py@@ -1,15 +1,21 @@from __future__ import annotations+from contextlib import nullcontextfrom datetime import datefrom typing import TYPE_CHECKING, castimport pytest-from unittest.mock import patch+from unittest.mock import MagicMock, patchfrom parameterized import parameterizedfrom trino.exceptions import Http503Error, TrinoConnectionError, TrinoExternalError, TrinoQueryError, TrinoUserError-from products.managed_warehouse.backend.backfill_trino import ADD_FILES_MAX_FILES, TrinoBackfillCatalog+from products.managed_warehouse.backend.backfill_trino import (+ ADD_FILES_MAX_FILES,+ BACKFILL_TRINO_PRINCIPAL,+ TrinoBackfillCatalog,+)+from products.managed_warehouse.backend.facade.backfill_trino import connect_backfill_catalogfrom products.managed_warehouse.backend.facade.contracts import DuckLakeDayReplacement, DuckLakeTableReplacementif TYPE_CHECKING:@@ -62,6 +68,18 @@ def _catalog(return TrinoBackfillCatalog(cast("Connection", connection)), connection[email protected]("canary", [False, True])+def test_the_backfill_connection_uses_the_canary_host_only_when_asked(canary: bool) -> None:+ with patch(+ "products.managed_warehouse.backend.backfill_trino.connect_managed_warehouse_trino",+ return_value=nullcontext(MagicMock(catalog="example_catalog")),+ ) as connect:+ with connect_backfill_catalog("org-1", canary=canary):+ pass++ connect.assert_called_once_with("org-1", principal=BACKFILL_TRINO_PRINCIPAL, canary=canary)++class TestRegisterFiles:@parameterized.expand([diff --git a/products/managed_warehouse/backend/tests/test_groups_backfill.py b/products/managed_warehouse/backend/tests/test_groups_backfill.pyindex 8eab45c3085f..bff56ac21826 100644--- a/products/managed_warehouse/backend/tests/test_groups_backfill.py+++ b/products/managed_warehouse/backend/tests/test_groups_backfill.py@@ -340,7 +340,7 @@ def test_groups_backfill_registers_through_trino_without_duckgres() -> None:inserts, catalog, connect_catalog, context = _run_groups_through_trino(partition_key, DucklingBackfillConfig())- connect_catalog.assert_called_once_with("org")+ connect_catalog.assert_called_once_with("org", canary=False)assert catalog.tables == [("groups_team_7", GROUPS_TRINO_COLUMN_TYPES)][(table, files, registered_replacement)] = catalog.registeredassert (table, registered_replacement) == ("groups_team_7", replacement)diff --git a/products/managed_warehouse/backend/tests/test_query_execution.py b/products/managed_warehouse/backend/tests/test_query_execution.pyindex 79582016f1e7..f4f585363189 100644--- a/products/managed_warehouse/backend/tests/test_query_execution.py+++ b/products/managed_warehouse/backend/tests/test_query_execution.py@@ -1,16 +1,20 @@-from contextlib import nullcontext+from collections.abc import Iterator+from contextlib import contextmanager, nullcontextfrom uuid import UUIDimport pytestfrom unittest.mock import MagicMock, PropertyMock, patch+from django.test import override_settings+from posthog.schema import HogQLQuery, HogQLQueryModifiers, HogQLQueryResponsefrom posthog.hogql.constants import HogQLGlobalSettings, LimitContextfrom posthog.hogql.errors import ExposedHogQLErrorfrom posthog.hogql.timings import HogQLTimings-from posthog.clickhouse.query_tagging import AccessMethod, QueryTags+from posthog.clickhouse.cluster import Workload+from posthog.clickhouse.query_tagging import AccessMethod, QueryTags, tags_contextfrom posthog.hogql_queries.hogql_query_runner import HogQLQueryRunnerfrom posthog.hogql_queries.query_runner import AnalyticsQueryRunner, ExecutionModefrom posthog.models import Organization, Team, User@@ -21,6 +25,9 @@ from products.managed_warehouse.backend.facade.contracts import ()from products.managed_warehouse.backend.facade.query_labels import MANAGED_TRINO_RAW_QUERY_STATUS_LABELfrom products.managed_warehouse.backend.query_execution import execute_managed_trino_query, validate_managed_trino_query+from products.warehouse_sources.backend.facade.types import ManagedWarehouseSQLMode++CANARY_DOMAIN = "canary.dw.us.postwh.com"@pytest.fixture@@ -39,6 +46,141 @@ def test_default_requests_keep_normal_execution_without_evaluating_the_flag(teamassert runner.query.executionTarget is None+def _canary_runner(+ team: Team, *, connection: bool, limit_context: LimitContext = LimitContext.QUERY+) -> HogQLQueryRunner:+ runner = object.__new__(HogQLQueryRunner)+ runner.query = HogQLQuery(+ query="SELECT 1",+ executionTarget=None if connection else "managed_trino",+ connectionId="source-1" if connection else None,+ sendRawQuery=True,+ )+ runner.team = team+ runner.user = User(id=9, distinct_id="user-9")+ runner.limit_context = limit_context+ runner._user_access_control = None+ runner.settings = HogQLGlobalSettings()+ runner.modifiers = HogQLQueryModifiers()+ runner.timings = HogQLTimings()+ runner.is_query_service = False+ runner.workload = Workload.DEFAULT+ runner._direct_connection_validated = False+ runner._direct_engine = None+ runner._managed_warehouse_sql_mode = None+ runner._trino_canary = False+ return runner+++@contextmanager+def _canary_runner_boundaries(canary_domain: str = CANARY_DOMAIN) -> Iterator[tuple[MagicMock, MagicMock]]:+ source = MagicMock(+ direct_engine="postgres",+ has_managed_warehouse_prefix=True,+ managed_warehouse_sql_mode=ManagedWarehouseSQLMode.BUILT_IN,+ )+ module = "posthog.hogql_queries.hogql_query_runner"+ with (+ override_settings(MANAGED_WAREHOUSE_TRINO_CANARY_DOMAIN=canary_domain),+ tags_context(trigger=None, feature=None, alert_config_id=None),+ patch(f"{module}.validate_managed_trino_query"),+ patch(f"{module}.get_direct_connection_source", return_value=source),+ patch(f"{module}.get_adapter_for_source", return_value=MagicMock(engine="trino")),+ patch(f"{module}.tag_contains_user_hogql"),+ patch(f"{module}.execute_managed_trino_query") as managed_trino,+ patch(f"{module}.execute_hogql_query") as direct,+ ):+ yield managed_trino, direct++[email protected]("connection", [False, True])+ "limit_context,flag,expected_variant",+ [+ (LimitContext.QUERY, True, "_trino_canary"),+ (LimitContext.QUERY_ASYNC, True, "_trino_canary"),+ (LimitContext.QUERY, False, ""),+ (LimitContext.EXPORT, True, ""),+ (LimitContext.SQL_ALERT, True, ""),+ (LimitContext.POSTHOG_AI, True, ""),+ ],+)+def test_canary_results_do_not_share_a_cache_entry_with_the_normal_lane(+ team: Team, connection: bool, limit_context: LimitContext, flag: bool, expected_variant: str+) -> None:+ runner = _canary_runner(team, connection=connection, limit_context=limit_context)+ with _canary_runner_boundaries(), patch("posthoganalytics.feature_enabled", return_value=flag):+ assert runner.get_cache_key_variant() == expected_variant+++def test_queries_off_managed_trino_never_evaluate_the_canary_flag(team: Team) -> None:+ runner = object.__new__(HogQLQueryRunner)+ runner.query = HogQLQuery(query="SELECT 1")+ runner.team = team+ runner.user = User(id=9, distinct_id="user-9")+ runner.limit_context = LimitContext.QUERY+ runner._direct_connection_validated = False+ with (+ override_settings(MANAGED_WAREHOUSE_TRINO_CANARY_DOMAIN=CANARY_DOMAIN),+ patch("posthoganalytics.feature_enabled", side_effect=AssertionError("evaluated a flag")),+ ):+ assert runner.get_cache_key_variant() == ""++[email protected]("connection", [False, True])[email protected]("first,later", [(True, False), (False, True)])+def test_the_cache_key_and_the_connection_share_one_canary_decision(+ team: Team, connection: bool, first: bool, later: bool+) -> None:+ runner = _canary_runner(team, connection=connection)+ with (+ _canary_runner_boundaries() as (managed_trino, direct),+ patch("posthog.hogql_queries.hogql_query_runner.use_trino_canary", side_effect=[first, later, later]) as decide,+ ):+ variants = {runner.get_cache_key_variant(), runner.get_cache_key_variant()}+ runner._calculate()+ assert variants == {"_trino_canary" if first else ""}+ assert decide.call_count == 1+ used_canary = direct.call_args.kwargs["trino_canary"] if connection else managed_trino.call_args.kwargs["canary"]+ assert used_canary is first++[email protected]("canary", [False, True])+def test_a_managed_trino_run_connects_the_way_the_runner_decided(team: Team, canary: bool) -> None:+ user = User(id=9, distinct_id="user-9")+ cursor = MagicMock()+ cursor.fetchmany.return_value = [(1,)]+ cursor.description = [("value", "bigint")]+ connection = MagicMock()+ connection.cursor.return_value = cursor+ with (+ patch("posthoganalytics.feature_enabled", return_value=True),+ patch("products.managed_warehouse.backend.query_execution.enable_customer_warehouse_access", return_value=True),+ patch(+ "products.managed_warehouse.backend.query_execution.UserAccessControl.has_project_access",+ new_callable=PropertyMock,+ return_value=True,+ ),+ patch(+ "products.managed_warehouse.backend.query_execution.connect_managed_warehouse_trino",+ return_value=nullcontext(connection),+ ) as connect,+ ):+ execute_managed_trino_query(+ HogQLQuery(query="SELECT 1", executionTarget="managed_trino", sendRawQuery=True),+ team=team,+ user=user,+ user_access_control=None,+ settings=HogQLGlobalSettings(),+ timings=HogQLTimings(),+ limit_context=LimitContext.QUERY,+ canary=canary,+ )+ connect.assert_called_once_with(+ str(team.organization_id), principal=f"posthog:sql-editor:team:{team.pk}:user:{user.pk}", canary=canary+ )++@pytest.mark.parametrize("flag_result", [False, None, RuntimeError("flag unavailable")])@pytest.mark.parametrize("raw", [False, True])def test_disabled_target_rejects_cached_requests(team: Team, flag_result: object, raw: bool) -> None:@@ -131,7 +273,7 @@ def test_live_query_binds_values_and_uses_hosted_service_credentials(limit_context=limit_context,)connect.assert_called_once_with(- str(team.organization_id), principal=f"posthog:sql-editor:team:{team.pk}:user:{user.pk}"+ str(team.organization_id), principal=f"posthog:sql-editor:team:{team.pk}:user:{user.pk}", canary=False)cursor.execute.assert_called_once_with(f'SELECT ? AS "value" OFFSET 0 ROWS LIMIT {expected_limit + 1}', ["quoted ' value"]diff --git a/products/managed_warehouse/backend/tests/test_query_schema.py b/products/managed_warehouse/backend/tests/test_query_schema.pyindex 98a1ded119b3..bda790c15e23 100644--- a/products/managed_warehouse/backend/tests/test_query_schema.py+++ b/products/managed_warehouse/backend/tests/test_query_schema.py@@ -4,21 +4,28 @@ from uuid import UUIDimport pytestfrom unittest.mock import MagicMock, PropertyMock, patch+from django.test import override_settings+from posthog.schema import DatabaseSchemaQueryfrom posthog.hogql.errors import ExposedHogQLErrorfrom posthog.api.services.query import process_database_schema_query-from posthog.clickhouse.query_tagging import AccessMethod, QueryTags+from posthog.clickhouse.query_tagging import AccessMethod, QueryTags, tags_contextfrom posthog.exceptions import DatabaseSchemaUnavailablefrom posthog.models import Team, User[email protected]("include_fields,table_names", [(False, None), (True, None), (True, ["analytics.orders"])])+ "include_fields,table_names,canary",+ [(False, None, False), (True, None, False), (True, ["analytics.orders"], True)],+)@pytest.mark.django_db-def test_native_schema_uses_warehouse_inventory(include_fields: bool, table_names: list[str] | None) -> None:+def test_native_schema_uses_warehouse_inventory(+ include_fields: bool, table_names: list[str] | None, canary: bool+) -> None:team = Team(id=7, organization_id=UUID("00000000-0000-0000-0000-000000000001"))- user = User(id=9)+ user = User(id=9, distinct_id="user-9")cursor = MagicMock()cursor.fetchall.side_effect = [[("analytics", "orders"), ("posthog", "events_example")],@@ -28,6 +35,8 @@ def test_native_schema_uses_warehouse_inventory(include_fields: bool, table_nameconnection = MagicMock(catalog="example_catalog")connection.cursor.return_value = cursorwith (+ override_settings(MANAGED_WAREHOUSE_TRINO_CANARY_DOMAIN="canary.dw.us.postwh.com" if canary else ""),+ tags_context(trigger=None, feature=None, alert_config_id=None),patch("posthoganalytics.feature_enabled", return_value=True),patch("products.managed_warehouse.backend.query_execution.get_query_tags", return_value=QueryTags()),patch(@@ -56,7 +65,9 @@ def test_native_schema_uses_warehouse_inventory(include_fields: bool, table_nameelse:assert all(not table.fields for table in response.tables.values())assert cursor.execute.call_count == 1- connect.assert_called_once_with(str(team.organization_id), principal="posthog:sql-editor:team:7:user:9")+ connect.assert_called_once_with(+ str(team.organization_id), principal="posthog:sql-editor:team:7:user:9", canary=canary+ )cursor.close.assert_called_once()diff --git a/products/managed_warehouse/backend/tests/test_trino_connection.py b/products/managed_warehouse/backend/tests/test_trino_connection.pyindex c9f853ff3b8b..46588334a529 100644--- a/products/managed_warehouse/backend/tests/test_trino_connection.py+++ b/products/managed_warehouse/backend/tests/test_trino_connection.py@@ -1,22 +1,32 @@import os+import uuidimport base64from concurrent.futures import ThreadPoolExecutorfrom datetime import UTC, datetime, timedeltafrom threading import Event+from urllib.parse import urlsplitimport pytestfrom unittest import mock+from django.test import override_settings+import requestsfrom rest_framework.response import Response+from posthog.hogql.constants import LimitContext++from posthog.clickhouse.query_tagging import Feature, tags_context+from products.managed_warehouse.backend.facade.contracts import ManagedWarehouseTrinoConnectionUnavailablefrom products.managed_warehouse.backend.trino_connection import (connect_managed_warehouse_trino,resolve_managed_warehouse_trino_connection,+ use_trino_canary,)CREDENTIAL_ID = "svc_0123456789abcdef01234567"+CANARY_DOMAIN = "canary.dw.us.postwh.com"def _mint_response(**target_overrides: object) -> Response:@@ -99,8 +109,18 @@ def test_rejects_invalid_trino_targets(target: dict[str, object]) -> None:resolve_managed_warehouse_trino_connection("org-1")+ "canary,canary_domain,host",+ [+ (False, "", "tenant.dw.us.postwh.com"),+ (False, CANARY_DOMAIN, "tenant.dw.us.postwh.com"),+ (True, CANARY_DOMAIN, "tenant.canary.dw.us.postwh.com"),+ ],+)@pytest.mark.parametrize("renewal_target", ["included", "omitted", "null"])-def test_polling_renews_the_grant_and_preserves_cancellation_ownership(renewal_target: str) -> None:+def test_polling_renews_the_grant_and_preserves_cancellation_ownership(+ renewal_target: str, canary: bool, canary_domain: str, host: str+) -> None:minted = _mint_response()refreshed = _mint_response()refreshed.data.pop("credential_secret")@@ -119,16 +139,16 @@ def test_polling_renews_the_grant_and_preserves_cancellation_ownership(renewal_treturn _http_response(request,id="query-1",- infoUri="https://tenant.dw.us.postwh.com/query-1",- nextUri="https://tenant.dw.us.postwh.com/v1/statement/query-1",+ infoUri=f"https://{host}/query-1",+ nextUri=f"https://{host}/v1/statement/query-1",stats={},)if request.method == "GET":return _http_response(request,id="query-1",- infoUri="https://tenant.dw.us.postwh.com/query-1",- nextUri="https://tenant.dw.us.postwh.com/v1/statement/query-1",+ infoUri=f"https://{host}/query-1",+ nextUri=f"https://{host}/v1/statement/query-1",columns=[{"name": "value", "type": "bigint", "typeSignature": {"rawType": "bigint", "arguments": []}}],data=[[1]],stats={},@@ -138,18 +158,20 @@ def test_polling_renews_the_grant_and_preserves_cancellation_ownership(renewal_treturn responsewith (+ override_settings(MANAGED_WAREHOUSE_TRINO_CANARY_DOMAIN=canary_domain),mock.patch("products.managed_warehouse.backend.presentation.views._request", side_effect=[minted, refreshed]) as cp,mock.patch("products.managed_warehouse.backend.trino_connection._utcnow", return_value=issued_at) as now,mock.patch("requests.adapters.HTTPAdapter.send", side_effect=send),):- with connect_managed_warehouse_trino("org-1") as connection:+ with connect_managed_warehouse_trino("org-1", canary=canary) as connection:cursor = connection.cursor()cursor.execute("SELECT 1")now.return_value = issued_at + timedelta(minutes=14)cursor.cancel()assert cp.call_count == 2+ assert {urlsplit(r.url).hostname for r in sent} == {host}assert cp.call_args.args == ("POST", "org-1", "/service-credentials/refresh")assert cp.call_args.kwargs["json_body"]["credential_id"] == CREDENTIAL_IDassert cp.call_args.kwargs["json_body"]["rotate_secret"] is False@@ -199,22 +221,67 @@ def test_failed_refresh_does_not_send_a_request(refresh_response: str) -> None:send.assert_not_called()+def test_canary_without_a_configured_domain_is_refused_before_a_credential_is_minted() -> None:+ with (+ override_settings(MANAGED_WAREHOUSE_TRINO_CANARY_DOMAIN=""),+ mock.patch("products.managed_warehouse.backend.presentation.views._request") as cp,+ pytest.raises(ManagedWarehouseTrinoConnectionUnavailable, match="canary domain is not configured"),+ ):+ with connect_managed_warehouse_trino("org-1", canary=True):+ pass+ cp.assert_not_called()+++ "host",+ [+ "trino.example.com",+ "tenant",+ "nested.tenant.dw.us.postwh.com",+ "tenant.dw.us.postwh.com.example.com",+ "tenant.canary.dw.us.postwh.com",+ ],+)+def test_canary_route_needs_a_host_that_the_control_plane_issued_under_a_normal_domain(host: str) -> None:+ with (+ override_settings(MANAGED_WAREHOUSE_TRINO_CANARY_DOMAIN=CANARY_DOMAIN),+ mock.patch(+ "products.managed_warehouse.backend.presentation.views._request", return_value=_mint_response(host=host)+ ),+ mock.patch("requests.adapters.HTTPAdapter.send") as send,+ pytest.raises(ManagedWarehouseTrinoConnectionUnavailable, match="hosted warehouse host"),+ ):+ with connect_managed_warehouse_trino("org-1", canary=True):+ pass+ send.assert_not_called()++@pytest.mark.parametrize(- "host,port,bypass_proxy",+ "host,port,bypass_proxy,canary_domain",[- ("tenant.dw.us.postwh.com", 443, True),- ("tenant.dw.eu.postwh.com", 443, True),- ("TENANT.DW.DEV.POSTWH.COM.", 443, True),- ("tenant.dw.us.postwh.com", 8443, False),- ("trino.example.com", 443, False),- ("nested.tenant.dw.us.postwh.com", 443, False),- ("dw.us.postwh.com", 443, False),- ("tenant.dw.us.postwh.com.example.com", 443, False),+ ("tenant.dw.us.postwh.com", 443, True, ""),+ ("tenant.dw.eu.postwh.com", 443, True, ""),+ ("TENANT.DW.DEV.POSTWH.COM.", 443, True, ""),+ ("tenant.dw.us.postwh.com", 8443, False, ""),+ ("trino.example.com", 443, False, ""),+ ("nested.tenant.dw.us.postwh.com", 443, False, ""),+ ("dw.us.postwh.com", 443, False, ""),+ ("tenant.dw.us.postwh.com.example.com", 443, False, ""),+ ("tenant", 443, False, ""),+ ("tenant.canary.dw.us.postwh.com", 443, False, ""),+ ("tenant.canary.dw.us.postwh.com", 443, True, CANARY_DOMAIN),+ ("tenant.canary.dw.us.postwh.com", 8443, False, CANARY_DOMAIN),+ ("nested.tenant.canary.dw.us.postwh.com", 443, False, CANARY_DOMAIN),+ ("tenant.canary.dw.eu.postwh.com", 443, False, CANARY_DOMAIN),+ ("tenant.canary.dw.us.postwh.com.example.com", 443, False, CANARY_DOMAIN),],)-def test_connection_keeps_verified_tls_and_scoped_proxy_bypass(host: str, port: int, bypass_proxy: bool) -> None:+def test_connection_keeps_verified_tls_and_scoped_proxy_bypass(+ host: str, port: int, bypass_proxy: bool, canary_domain: str+) -> None:proxy = "http://proxy.example.com:4750"with (+ override_settings(MANAGED_WAREHOUSE_TRINO_CANARY_DOMAIN=canary_domain),mock.patch.dict(os.environ, {"HTTPS_PROXY": proxy, "NO_PROXY": ""}, clear=True),mock.patch("products.managed_warehouse.backend.presentation.views._request",@@ -337,3 +404,52 @@ def test_cancellation_does_not_wait_for_a_blocked_poll() -> None:release_poll.set()assert poll.result(timeout=5).status_code == 200assert cp.call_count == 2+++ "canary_domain,distinct_id,tags,limit_context,flag_result,evaluates_flag,expected",+ [+ (CANARY_DOMAIN, "user-1", {}, None, True, True, True),+ (CANARY_DOMAIN, "user-1", {}, LimitContext.QUERY, True, True, True),+ (CANARY_DOMAIN, "user-1", {}, LimitContext.QUERY_ASYNC, True, True, True),+ (CANARY_DOMAIN, "user-1", {}, None, False, True, False),+ (CANARY_DOMAIN, "user-1", {}, None, None, True, False),+ (CANARY_DOMAIN, "user-1", {}, None, RuntimeError("flags unavailable"), True, False),+ ("", "user-1", {}, None, True, False, False),+ (CANARY_DOMAIN, None, {}, None, True, False, False),+ (CANARY_DOMAIN, "", {}, None, True, False, False),+ (CANARY_DOMAIN, "user-1", {"trigger": "warmingV2"}, None, True, False, False),+ (CANARY_DOMAIN, "user-1", {"feature": Feature.CACHE_WARMUP}, None, True, False, False),+ (CANARY_DOMAIN, "user-1", {"alert_config_id": uuid.uuid4()}, None, True, False, False),+ (CANARY_DOMAIN, "user-1", {"feature": Feature.ALERTING}, None, True, False, False),+ (CANARY_DOMAIN, "user-1", {}, LimitContext.EXPORT, True, False, False),+ (CANARY_DOMAIN, "user-1", {}, LimitContext.SQL_ALERT, True, False, False),+ (CANARY_DOMAIN, "user-1", {}, LimitContext.POSTHOG_AI, True, False, False),+ ],+)+def test_use_trino_canary_needs_a_flagged_user_on_an_interactive_run(+ canary_domain: str,+ distinct_id: str | None,+ tags: dict[str, object],+ limit_context: LimitContext | None,+ flag_result: object,+ evaluates_flag: bool,+ expected: bool,+) -> None:+ clean_tags = {"trigger": None, "feature": None, "alert_config_id": None}+ with (+ override_settings(MANAGED_WAREHOUSE_TRINO_CANARY_DOMAIN=canary_domain),+ tags_context(**{**clean_tags, **tags}),+ mock.patch(+ "posthoganalytics.feature_enabled",+ side_effect=flag_result if isinstance(flag_result, Exception) else None,+ return_value=flag_result,+ ) as flag,+ ):+ assert use_trino_canary(distinct_id, "org-1", limit_context=limit_context) is expected+ assert flag.called is evaluates_flag+ if evaluates_flag:+ assert flag.call_args.args == ("managed-warehouse-trino-canary", distinct_id)+ assert flag.call_args.kwargs["groups"] == {"organization": "org-1"}+ assert flag.call_args.kwargs["only_evaluate_locally"] is False+ assert flag.call_args.kwargs["send_feature_flag_events"] is Falsediff --git a/products/managed_warehouse/backend/trino_connection.py b/products/managed_warehouse/backend/trino_connection.pyindex d9c6df27b835..255f50b77a77 100644--- a/products/managed_warehouse/backend/trino_connection.py+++ b/products/managed_warehouse/backend/trino_connection.py@@ -1,7 +1,7 @@from __future__ import annotationsimport re-from collections.abc import Iterator, MutableMapping+from collections.abc import Collection, Iterator, MutableMappingfrom contextlib import contextmanagerfrom dataclasses import replacefrom datetime import UTC, datetime, timedelta@@ -9,10 +9,18 @@ from threading import Lockfrom typing import TYPE_CHECKINGfrom urllib.parse import urlsplit+from django.conf import settings+import requests+import structlogfrom requests.auth import HTTPBasicAuthfrom requests.utils import resolve_proxies+from posthog.hogql.constants import LimitContext++from posthog.clickhouse.query_tagging import Feature, get_query_tags+from posthog.ph_client import feature_enabled_or_false+from products.managed_warehouse.backend.facade.contracts import (ManagedWarehouseTrinoConnection,ManagedWarehouseTrinoConnectionUnavailable,@@ -20,28 +28,95 @@ from products.managed_warehouse.backend.facade.contracts import (ServiceCredentialTrinoConnect,ServiceCredentialUnavailable,)+from products.managed_warehouse.backend.facade.feature_flags import MANAGED_WAREHOUSE_TRINO_CANARY_FLAGfrom products.managed_warehouse.backend.service_credentials import mint_service_credential, renew_service_credentialif TYPE_CHECKING:from trino.dbapi import Connection+LOGGER = structlog.get_logger(__name__)+_CREDENTIAL_REFRESH_MARGIN = timedelta(minutes=2)_CREDENTIAL_REQUEST_TIMEOUT_SECONDS = 10+_HOSTED_SERVICE_DOMAINS = frozenset({"dw.us.postwh.com", "dw.eu.postwh.com", "dw.dev.postwh.com"})+_AUTOMATED_LIMIT_CONTEXTS = frozenset({LimitContext.EXPORT, LimitContext.SQL_ALERT, LimitContext.POSTHOG_AI})def _utcnow() -> datetime:return datetime.now(UTC)+def _canary_domain() -> str:+ return settings.MANAGED_WAREHOUSE_TRINO_CANARY_DOMAIN.strip().lower().strip(".")+++def _require_canary_domain() -> str:+ domain = _canary_domain()+ if not domain:+ raise ManagedWarehouseTrinoConnectionUnavailable("The Trino canary domain is not configured")+ return domain+++def _is_single_label_under(host: str, domains: Collection[str]) -> bool:+ label, _, domain = host.lower().removesuffix(".").partition(".")+ return domain in domains and re.fullmatch(r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?", label) is not None++def _is_hosted_service_target(host: str) -> bool:# Tenant endpoints are trusted only when issued by the control plane, never from external-source configuration.- label, _, domain = host.lower().removesuffix(".").partition(".")+ canary_domain = _canary_domain()+ # An empty canary domain must stay out of the set, because a host without a domain would match it.+ return _is_single_label_under(host, _HOSTED_SERVICE_DOMAINS | ({canary_domain} if canary_domain else set()))+++def _with_canary_host(connection: ManagedWarehouseTrinoConnection, canary: bool) -> ManagedWarehouseTrinoConnection:+ if not canary:+ return connection+ domain = _require_canary_domain()+ # Only a host that the control plane issued under a normal domain has a canary equivalent.+ if not _is_single_label_under(connection.host, _HOSTED_SERVICE_DOMAINS):+ raise ManagedWarehouseTrinoConnectionUnavailable("The Trino canary route needs a hosted warehouse host")+ # The canary host keeps the database label of the minted host, so the same service credential applies.+ label = connection.host.lower().removesuffix(".").partition(".")[0]+ return replace(connection, host=f"{label}.{domain}")+++def _is_automated_run(limit_context: LimitContext | None) -> bool:+ # The alert and warming code sets these tags before it calls any query runner, so they are visible in every+ # layer. Exports and subscriptions run as their creator and carry no tag, so the limit context identifies them.+ tags = get_query_tags()return (- domain in {"dw.us.postwh.com", "dw.eu.postwh.com", "dw.dev.postwh.com"}- and re.fullmatch(r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?", label) is not None+ limit_context in _AUTOMATED_LIMIT_CONTEXTS+ or tags.alert_config_id is not None+ or tags.feature in (Feature.ALERTING, Feature.CACHE_WARMUP)+ or (tags.trigger or "").startswith("warming"))+def use_trino_canary(+ distinct_id: str | None, organization_id: str, *, limit_context: LimitContext | None = None+) -> bool:+ """Whether an interactive run for this user connects through the canary hostname.++ The cheap checks come first, so the remote flag evaluation runs only for a user who could reach the canary.+ A caller decides once per query and reuses the result, because each call evaluates the flag again.+ """+ if not _canary_domain() or not distinct_id or _is_automated_run(limit_context):+ return False+ try:+ return feature_enabled_or_false(+ MANAGED_WAREHOUSE_TRINO_CANARY_FLAG,+ distinct_id,+ groups={"organization": organization_id},+ group_properties={"organization": {"id": organization_id}},+ only_evaluate_locally=False,+ send_feature_flag_events=False,+ )+ except Exception:+ LOGGER.warning("Failed to evaluate the Trino canary feature flag", organization_id=organization_id)+ return False++def _connection_from_credential(credential: ServiceCredential) -> ManagedWarehouseTrinoConnection:target = credential.trino_connectif (@@ -81,10 +156,13 @@ def resolve_managed_warehouse_trino_connection(class _TrinoServiceSession(requests.Session):- def __init__(self, organization_id: str, connection: ManagedWarehouseTrinoConnection) -> None:+ def __init__(+ self, organization_id: str, connection: ManagedWarehouseTrinoConnection, *, canary: bool = False+ ) -> None:super().__init__()self._organization_id = organization_idself._connection = connection+ self._canary = canaryself._lock = Lock()# Defer authentication until send so queued requests also check credential expiry.self.auth = self._defer_authentication@@ -129,7 +207,8 @@ class _TrinoServiceSession(requests.Session):credential_secret=connection.password,timeout_seconds=_CREDENTIAL_REQUEST_TIMEOUT_SECONDS,)- if credential.trino_connect is None:+ reused_target = credential.trino_connect is None+ if reused_target:# Readiness can hide dial targets during renewal without invalidating an existing query's grant.credential = replace(credential,@@ -146,6 +225,9 @@ class _TrinoServiceSession(requests.Session):raise ManagedWarehouseTrinoConnectionUnavailable("Could not renew the Trino service credential") from error+ # The control plane returns the normal host again. A reused target already holds the canary host.+ if not reused_target:+ refreshed = _with_canary_host(refreshed, self._canary)if (refreshed.credential_id != connection.credential_idor refreshed.host != connection.host@@ -174,11 +256,15 @@ class _TrinoServiceSession(requests.Session):@contextmanager-def connect_managed_warehouse_trino(organization_id: str, *, principal: str = "posthog:trino") -> Iterator[Connection]:+def connect_managed_warehouse_trino(+ organization_id: str, *, principal: str = "posthog:trino", canary: bool = False+) -> Iterator[Connection]:from trino.dbapi import connect # noqa: PLC0415 -- keeps the optional driver off startup paths- config = resolve_managed_warehouse_trino_connection(organization_id, principal=principal)- with _TrinoServiceSession(organization_id, config) as http_session:+ if canary:+ _require_canary_domain()+ config = _with_canary_host(resolve_managed_warehouse_trino_connection(organization_id, principal=principal), canary)+ with _TrinoServiceSession(organization_id, config, canary=canary) as http_session:if _is_hosted_service_target(config.host) and config.port == 443:http_session.trust_env = Falseconnection = connect(