Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(managed-warehouse): stop reporting trino user errors from the shadow build (#118937)

posthog[bot]ee921f44214b
diff --git a/posthog/hogql/printer/test/test_trino_printer.py b/posthog/hogql/printer/test/test_trino_printer.py
index 97df6361055b..c8bb6eabd96d 100644
--- a/posthog/hogql/printer/test/test_trino_printer.py
+++ b/posthog/hogql/printer/test/test_trino_printer.py
@@ -947,6 +947,19 @@ def test_scalar_cte_chains_preserve_results(depth: int) -> None:
assert connection.execute(sql).fetchall() == [(2**depth,)]
+def test_wide_subquery_scalar_cte_references_stay_within_budget() -> None:
+ columns = ", ".join(f"user_id AS c{i}" for i in range(100))
+ references = ", ".join("total" for _ in range(100))
+ sql, _ = prepare_and_print_ast(
+ parse_select(
+ f"WITH wide AS (SELECT {columns} FROM users), (SELECT count() FROM wide) AS total SELECT {references}"
+ ),
+ _context_with_trino_table(),
+ "trino",
+ )
+ assert sql.count('(SELECT count(*) FROM "wide")') == 100
+
+
def test_rejects_excessive_scalar_cte_expansion() -> None:
ctes = ["1 AS c0", *[f"c{i - 1} + c{i - 1} AS c{i}" for i in range(1, 18)]]
with pytest.raises(TrinoLoweringError, match="TRINO_AST_EXPANSION_LIMIT"):
diff --git a/posthog/hogql/test/test_trino_semantics.py b/posthog/hogql/test/test_trino_semantics.py
index 437dcda830f0..09dd3a47c440 100644
--- a/posthog/hogql/test/test_trino_semantics.py
+++ b/posthog/hogql/test/test_trino_semantics.py
@@ -231,9 +231,10 @@ class TestTrinoPersonsSemantics(SimpleTestCase):
PersonsOnEventsMode.DISABLED,
PersonsOnEventsMode.PERSON_ID_NO_OVERRIDE_PROPERTIES_ON_EVENTS,
PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_JOINED,
+ "person_id_override_properties_joined",
):
with self.subTest(mode=mode):
- context = self._context(HogQLQueryModifiers(personsOnEventsMode=mode))
+ context = self._context(HogQLQueryModifiers.model_construct(personsOnEventsMode=mode)) # type: ignore[arg-type] # team modifiers can hold a raw string
with self.assertRaisesRegex(
TrinoLoweringError,
diff --git a/posthog/hogql/transforms/trino/limits.py b/posthog/hogql/transforms/trino/limits.py
index e344885766a3..286a8ae01e9f 100644
--- a/posthog/hogql/transforms/trino/limits.py
+++ b/posthog/hogql/transforms/trino/limits.py
@@ -23,5 +23,8 @@ class TrinoCompilationBudget(TraversingVisitor):
)
def visit(self, node: ast.AST | None) -> None:
+ # Type nodes share one resolved graph, so counting them charges each field reference for its whole scope.
+ if isinstance(node, ast.Type):
+ return
self.consume_node(node)
super().visit(node)
diff --git a/posthog/hogql/transforms/trino/manifest.py b/posthog/hogql/transforms/trino/manifest.py
index b7575bb0d150..237ccca1b72d 100644
--- a/posthog/hogql/transforms/trino/manifest.py
+++ b/posthog/hogql/transforms/trino/manifest.py
@@ -302,7 +302,7 @@ def _validate_pure_inputs(
if persons_mode is not None and persons_mode != PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS:
raise TrinoLoweringError(
"TRINO_PERSONS_ON_EVENTS_MODE_UNSUPPORTED",
- f"personsOnEventsMode={persons_mode.value}",
+ f"personsOnEventsMode={persons_mode}",
detail="Trino compilation supports only personsOnEventsMode=person_id_override_properties_on_events.",
)
diff --git a/posthog/hogql/transforms/trino/test/test_manifest.py b/posthog/hogql/transforms/trino/test/test_manifest.py
index 0b9fe080d221..2fbfde1c3fc9 100644
--- a/posthog/hogql/transforms/trino/test/test_manifest.py
+++ b/posthog/hogql/transforms/trino/test/test_manifest.py
@@ -197,6 +197,11 @@ def test_manifest_relation_can_share_the_internal_unnest_function_name() -> None
{"modifiers": HogQLQueryModifiers(personsOnEventsMode=PersonsOnEventsMode.DISABLED)},
"TRINO_PERSONS_ON_EVENTS_MODE_UNSUPPORTED",
),
+ (
+ "SELECT event FROM events",
+ {"modifiers": HogQLQueryModifiers.model_construct(personsOnEventsMode="disabled")}, # type: ignore[arg-type] # team modifiers can hold a raw string
+ "TRINO_PERSONS_ON_EVENTS_MODE_UNSUPPORTED",
+ ),
],
)
def test_rejects_django_backed_semantics(query: str, kwargs: dict[str, Any], feature_code: str) -> None:
diff --git a/posthog/hogql/transforms/trino/validate.py b/posthog/hogql/transforms/trino/validate.py
index e206ba84fb63..38c7210f8b3e 100644
--- a/posthog/hogql/transforms/trino/validate.py
+++ b/posthog/hogql/transforms/trino/validate.py
@@ -198,7 +198,7 @@ _SUPPORTED_CALLS = frozenset(
def validate_trino_context(context: HogQLContext) -> None:
mode = context.modifiers.personsOnEventsMode
if mode != PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS:
- mode_name = mode.value if mode is not None else "unset"
+ mode_name = str(mode) if mode is not None else "unset"
raise TrinoLoweringError(
"TRINO_PERSONS_ON_EVENTS_MODE_UNSUPPORTED",
f"personsOnEventsMode={mode_name}",
diff --git a/posthog/temporal/data_modeling/activities/materialize_view_managed_warehouse.py b/posthog/temporal/data_modeling/activities/materialize_view_managed_warehouse.py
index 4cee3b3e69b1..b78ff5c3de93 100644
--- a/posthog/temporal/data_modeling/activities/materialize_view_managed_warehouse.py
+++ b/posthog/temporal/data_modeling/activities/materialize_view_managed_warehouse.py
@@ -7,6 +7,8 @@ import dataclasses
from structlog.contextvars import bind_contextvars
from temporalio import activity
+from posthog.hogql.transforms.trino.errors import TrinoLoweringError
+
from posthog.dataclasses import frozen
from posthog.exceptions_capture import capture_exception
from posthog.models import Team
@@ -46,7 +48,12 @@ from products.managed_warehouse.backend.facade.contracts import (
)
from products.managed_warehouse.backend.facade.feature_flags import DATA_MODELING_SHADOW_FLAG
-from ..metrics import UNKNOWN_RUN_MODE, get_node_suspended_metric
+from ..metrics import (
+ UNKNOWN_RUN_MODE,
+ get_managed_warehouse_trino_compile_failure_metric,
+ get_managed_warehouse_trino_missing_object_metric,
+ get_node_suspended_metric,
+)
from .utils import (
CONSECUTIVE_FAILURES_TO_SUSPEND,
bind_data_modeling_log_context,
@@ -149,6 +156,7 @@ def _is_managed_warehouse_shadow_enabled(team: Team) -> bool:
return is_data_modeling_shadow_ready(
organization_id=team.organization_id,
+ team_id=team.pk,
)
@@ -354,14 +362,37 @@ async def _materialize_view_managed_warehouse(
)
return shadow_result
except Exception as e:
+ from products.managed_warehouse.backend.facade.client import get_trino_missing_object_error_name
+
duration = time.monotonic() - start_time
- capture_exception(e, {"sql": sql, "inputs": inputs})
- await logger.awarning(
- "Managed warehouse shadow materialization failed",
- node_name=node.name,
- error=str(e),
- duration_seconds=round(duration, 2),
- )
+ missing_object_error_name = get_trino_missing_object_error_name(e)
+ if isinstance(e, TrinoLoweringError):
+ # Unsupported HogQL is an expected shadow outcome, so it is not an exception to track.
+ get_managed_warehouse_trino_compile_failure_metric(e.feature_code).add(1)
+ await logger.awarning(
+ "Managed warehouse shadow could not compile the view for Trino",
+ node_name=node.name,
+ error_code=e.feature_code,
+ duration_seconds=round(duration, 2),
+ )
+ elif missing_object_error_name is not None:
+ # A column or table that Trino cannot resolve is an expected shadow outcome, so it is not an exception to track.
+ get_managed_warehouse_trino_missing_object_metric(missing_object_error_name).add(1)
+ await logger.awarning(
+ "Managed warehouse shadow query failed in Trino",
+ node_name=node.name,
+ error_name=missing_object_error_name,
+ error=str(e),
+ duration_seconds=round(duration, 2),
+ )
+ else:
+ capture_exception(e, {"sql": sql, "inputs": inputs})
+ await logger.awarning(
+ "Managed warehouse shadow materialization failed",
+ node_name=node.name,
+ error=str(e),
+ duration_seconds=round(duration, 2),
+ )
shadow_result = ManagedWarehouseShadowResult(
row_count=0,
duration_seconds=duration,
diff --git a/posthog/temporal/data_modeling/metrics.py b/posthog/temporal/data_modeling/metrics.py
index 61344104ec58..b2c5312a0013 100644
--- a/posthog/temporal/data_modeling/metrics.py
+++ b/posthog/temporal/data_modeling/metrics.py
@@ -79,6 +79,28 @@ def get_node_suspended_metric(engine: str) -> MetricCounter:
)
+def get_managed_warehouse_trino_compile_failure_metric(error_code: str) -> MetricCounter:
+ return (
+ activity.metric_meter()
+ .with_additional_attributes({"error_code": error_code})
+ .create_counter(
+ "managed_warehouse_shadow_trino_compile_failure",
+ "Number of managed warehouse shadow views that Trino compilation rejected, by error code.",
+ )
+ )
+
+
+def get_managed_warehouse_trino_missing_object_metric(error_name: str) -> MetricCounter:
+ return (
+ activity.metric_meter()
+ .with_additional_attributes({"error_name": error_name})
+ .create_counter(
+ "managed_warehouse_shadow_trino_missing_object",
+ "Number of managed warehouse shadow builds that failed because Trino could not resolve a catalog, schema, table or column, by error name.",
+ )
+ )
+
+
@frozen
class _DualMetricCounters:
managed_warehouse: MetricCounter
diff --git a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py
index 34eb26ee951c..3bb10fddbb24 100644
--- a/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py
+++ b/posthog/temporal/tests/data_modeling/test_materialize_view_activities.py
@@ -18,8 +18,10 @@ import pyarrow as pa
import deltalake
import pyarrow.parquet as pq
from temporalio.exceptions import ApplicationError
+from trino.exceptions import TrinoUserError
from posthog.hogql.resolver import ResolverFactory
+from posthog.hogql.transforms.trino.errors import TrinoLoweringError
from posthog.clickhouse.query_tagging import get_query_tags
from posthog.models import Team, User
@@ -156,13 +158,20 @@ class TestMaterializeViewManagedWarehouseActivity:
flag_enabled and target_ready
)
if flag_enabled:
- is_ready.assert_called_once_with(organization_id=ateam.organization_id)
+ is_ready.assert_called_once_with(organization_id=ateam.organization_id, team_id=ateam.pk)
else:
is_ready.assert_not_called()
@pytest.mark.parametrize(
- "compile_fails,alias_dispatch_fails",
- [(False, False), (True, False), (False, True)],
+ "compile_error,error_tracked,alias_dispatch_fails",
+ [
+ (None, False, False),
+ (ValueError("Trino compilation failed"), True, False),
+ (TrinoLoweringError("TRINO_TABLE_LOCATOR_MISSING", "table"), False, False),
+ (TrinoUserError({"errorName": "COLUMN_NOT_FOUND", "errorType": "USER_ERROR"}), False, False),
+ (TrinoUserError({"errorName": "SYNTAX_ERROR", "errorType": "USER_ERROR"}), True, False),
+ (None, True, True),
+ ],
)
async def test_trino_shadow_records_execution_or_compilation_result(
self,
@@ -172,7 +181,8 @@ class TestMaterializeViewManagedWarehouseActivity:
ajob,
adag,
asaved_query,
- compile_fails: bool,
+ compile_error: Exception | None,
+ error_tracked: bool,
alias_dispatch_fails: bool,
) -> None:
inputs = ManagedWarehouseShadowInputs(
@@ -190,8 +200,8 @@ class TestMaterializeViewManagedWarehouseActivity:
def execute_trino_model(**kwargs: Any) -> DuckLakeTableResult:
kwargs["on_processed_bytes"](300)
kwargs["on_processed_bytes"](45)
- if compile_fails:
- raise ValueError("Trino compilation failed")
+ if compile_error is not None:
+ raise compile_error
return DuckLakeTableResult(schema_name="shadow_models", table_name="test_model", row_count=12)
with (
@@ -210,6 +220,9 @@ class TestMaterializeViewManagedWarehouseActivity:
"posthog.temporal.data_modeling.activities.materialize_view_managed_warehouse.prepare_executable_query",
side_effect=lambda saved_query: setattr(saved_query, "query", executable_query),
),
+ unittest.mock.patch(
+ "posthog.temporal.data_modeling.activities.materialize_view_managed_warehouse.capture_exception"
+ ) as capture,
):
result = await activity_environment.run(materialize_view_managed_warehouse_activity, inputs)
@@ -224,9 +237,10 @@ class TestMaterializeViewManagedWarehouseActivity:
legacy_execute.assert_not_called()
await database_sync_to_async(ajob.refresh_from_db)()
assert ajob.bytes_read == 345
- if compile_fails:
+ assert capture.called is error_tracked
+ if compile_error is not None:
reconcile_aliases.assert_not_awaited()
- assert result.error == "Trino compilation failed"
+ assert result.error == str(compile_error)
assert ajob.status == DataModelingJobStatus.FAILED
else:
reconcile_aliases.assert_awaited_once_with(ateam.pk, str(asaved_query.id))
diff --git a/products/managed_warehouse/backend/facade/api.py b/products/managed_warehouse/backend/facade/api.py
index 31c9bbea72a4..de605f067539 100644
--- a/products/managed_warehouse/backend/facade/api.py
+++ b/products/managed_warehouse/backend/facade/api.py
@@ -127,12 +127,18 @@ def has_provisioned_warehouse(organization_id: str | UUID) -> bool:
def is_data_modeling_shadow_ready(
*,
organization_id: str | UUID,
+ team_id: int,
) -> bool:
+ from products.managed_warehouse.backend.facade.cp_teams import ( # noqa: PLC0415 -- keeps control-plane imports off startup paths
+ get_org_team_membership,
+ )
from products.managed_warehouse.backend.trino_compiler import ( # noqa: PLC0415 -- keeps optional compiler imports off startup paths
get_ready_trino_catalog_name,
)
- return get_ready_trino_catalog_name(str(organization_id)) is not None
+ if get_ready_trino_catalog_name(str(organization_id)) is None:
+ return False
+ return get_org_team_membership(str(organization_id), team_id) is not None
def get_duckgres_query_server_config(organization_id: str) -> DuckgresQueryServerConfig:
diff --git a/products/managed_warehouse/backend/facade/client.py b/products/managed_warehouse/backend/facade/client.py
index f25342261292..a9145384f8cf 100644
--- a/products/managed_warehouse/backend/facade/client.py
+++ b/products/managed_warehouse/backend/facade/client.py
@@ -63,6 +63,7 @@ __all__ = [
"execute_ducklake_query",
"execute_trino_shadow_materialization",
"execute_trino_model",
+ "get_trino_missing_object_error_name",
"make_duckgres_conninfo",
"mint_service_credential",
"prepare_hogql_to_trino_compiler",
@@ -232,6 +233,17 @@ async def request_model_alias_reconciliation(team_id: int, saved_query_id: str |
await request(team_id, saved_query_id)
+TRINO_MISSING_OBJECT_ERROR_NAMES = frozenset(
+ {"CATALOG_NOT_FOUND", "SCHEMA_NOT_FOUND", "TABLE_NOT_FOUND", "COLUMN_NOT_FOUND"}
+)
+
+
+def get_trino_missing_object_error_name(error: BaseException) -> str | None:
+ """The Trino error name when Trino could not resolve a catalog, schema, table or column, else None."""
+ error_name = getattr(error, "error_name", None)
+ return error_name if error_name in TRINO_MISSING_OBJECT_ERROR_NAMES else None
+
+
async def execute_trino_model(
*,
organization_id: str,
diff --git a/products/managed_warehouse/backend/tests/test_api.py b/products/managed_warehouse/backend/tests/test_api.py
index b3ff2eb4e85e..44a328907b0f 100644
--- a/products/managed_warehouse/backend/tests/test_api.py
+++ b/products/managed_warehouse/backend/tests/test_api.py
@@ -3,6 +3,8 @@ from unittest.mock import patch
from django.test import SimpleTestCase
+from parameterized import parameterized
+
from posthog.models import Organization
from products.managed_warehouse.backend.facade.api import (
@@ -67,12 +69,25 @@ def test_stored_warehouse_config_maps_connection_and_catalog_without_returning_t
class TestDataModelingShadowReadiness(SimpleTestCase):
- def test_requires_a_ready_trino_target_without_translation_records(self) -> None:
- for catalog in (None, "managed_catalog"):
- with self.subTest(catalog=catalog):
- with patch(
- "products.managed_warehouse.backend.trino_compiler.get_ready_trino_catalog_name",
- return_value=catalog,
- ) as get_catalog:
- assert is_data_modeling_shadow_ready(organization_id="organization-id") is (catalog is not None)
- get_catalog.assert_called_once_with("organization-id")
+ @parameterized.expand(
+ [
+ ("no_catalog", None, object(), False),
+ ("no_table_mapping", "managed_catalog", None, False),
+ ("ready", "managed_catalog", object(), True),
+ ]
+ )
+ def test_requires_a_ready_trino_target_and_a_table_mapping(
+ self, _name: str, catalog: str | None, membership: object | None, expected: bool
+ ) -> None:
+ with (
+ patch(
+ "products.managed_warehouse.backend.trino_compiler.get_ready_trino_catalog_name",
+ return_value=catalog,
+ ) as get_catalog,
+ patch(
+ "products.managed_warehouse.backend.facade.cp_teams.get_org_team_membership",
+ return_value=membership,
+ ),
+ ):
+ assert is_data_modeling_shadow_ready(organization_id="organization-id", team_id=7) is expected
+ get_catalog.assert_called_once_with("organization-id")
diff --git a/products/managed_warehouse/backend/tests/test_trino_materialization.py b/products/managed_warehouse/backend/tests/test_trino_materialization.py
index 88540c49d3b7..c273a3f3f3e8 100644
--- a/products/managed_warehouse/backend/tests/test_trino_materialization.py
+++ b/products/managed_warehouse/backend/tests/test_trino_materialization.py
@@ -15,6 +15,7 @@ from posthog.hogql.parser import parse_select
from posthog.models import Team
from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery
+from products.data_modeling.backend.facade.system_tables import DATA_MODELING_ALLOWED_SYSTEM_TABLES
from products.managed_warehouse.backend.facade.client import execute_trino_shadow_materialization
from products.managed_warehouse.backend.facade.contracts import (
DuckLakeTableResult,
@@ -90,6 +91,7 @@ class TestTrinoShadowMaterialization(BaseTest):
HogQLQuery.model_validate(self.query),
team=self.team,
bypass_warehouse_access_control=True,
+ allowed_system_tables=DATA_MODELING_ALLOWED_SYSTEM_TABLES,
expansion_mode=TrinoExpansionMode.DJANGO,
select_transform=ANY,
preserve_clickhouse_output_names=True,
diff --git a/products/managed_warehouse/backend/trino_compiler.py b/products/managed_warehouse/backend/trino_compiler.py
index a1526cf8b747..26f2633c4ec2 100644
--- a/products/managed_warehouse/backend/trino_compiler.py
+++ b/products/managed_warehouse/backend/trino_compiler.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+from collections.abc import Collection
from dataclasses import field
from typing import TYPE_CHECKING, Any, cast
@@ -228,6 +229,7 @@ def compile_hogql_to_trino_sql(
limit_top_select: bool = False,
limit_context: LimitContext | None = None,
bypass_warehouse_access_control: bool = False,
+ allowed_system_tables: Collection[str] | None = None,
include_hogql: bool = False,
expansion_mode: TrinoExpansionMode = TrinoExpansionMode.PURE,
catalog_manifest: TrinoCatalogManifest | None = None,
@@ -238,7 +240,8 @@ def compile_hogql_to_trino_sql(
"""Compile HogQL for the ready Trino catalog that serves the team's DuckLake data.
Set ``bypass_warehouse_access_control`` only for trusted internal callers that compile
- without a user. This entry point does not execute the returned SQL or alter query routing.
+ without a user. ``allowed_system_tables`` exposes those scoped system tables to a userless
+ Django-mode compilation. This entry point does not execute the returned SQL or alter query routing.
Set ``include_hogql`` to render normalized HogQL for diagnostics. Pure manifest-backed
compilation is the default; select ``TrinoExpansionMode.DJANGO`` only when the query needs
actions, cohorts, saved queries, variables, filters, or other Django-backed semantics.
@@ -306,6 +309,7 @@ def compile_hogql_to_trino_sql(
modifiers=query_modifiers,
# Preserve an explicit trusted-internal bypass; the public default remains fail-closed.
bypass_warehouse_access_control=bypass_warehouse_access_control,
+ allowed_system_tables=allowed_system_tables,
trigger="trino",
)
placeholders = find_placeholders(parsed)
diff --git a/products/managed_warehouse/backend/trino_materialization.py b/products/managed_warehouse/backend/trino_materialization.py
index d82af1125f9d..b9f0f2231d35 100644
--- a/products/managed_warehouse/backend/trino_materialization.py
+++ b/products/managed_warehouse/backend/trino_materialization.py
@@ -12,6 +12,7 @@ from posthog.hogql.trino_parameters import convert_pyformat_placeholders
from posthog.models import Team
+from products.data_modeling.backend.facade.system_tables import DATA_MODELING_ALLOWED_SYSTEM_TABLES
from products.managed_warehouse.backend.common import ducklake_data_modeling_schema
from products.managed_warehouse.backend.facade.contracts import (
DuckLakeTableResult,
@@ -115,6 +116,7 @@ def _compile(team: Team, query: HogQLQuery, incremental: TrinoIncrementalWrite |
query,
team=team,
bypass_warehouse_access_control=True,
+ allowed_system_tables=DATA_MODELING_ALLOWED_SYSTEM_TABLES,
expansion_mode=TrinoExpansionMode.DJANGO,
select_transform=select_transform,
preserve_clickhouse_output_names=True,