Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(hogql): reuse eligible repeated ctes (#112574)

George Munyoroff4622d8a5ed
diff --git a/docs/published/handbook/engineering/databases/query-performance-optimization.md b/docs/published/handbook/engineering/databases/query-performance-optimization.md
index 94f9292d87f3..f4a103de915a 100644
--- a/docs/published/handbook/engineering/databases/query-performance-optimization.md
+++ b/docs/published/handbook/engineering/databases/query-performance-optimization.md
@@ -181,3 +181,7 @@ Need more granular access to queries than these dashboards provide? Take a look
### How-to fix slow queries
See [ClickHouse manual](https://posthog.com/handbook/engineering/clickhouse/) for tips and tricks.
+
+### Repeated HogQL CTEs
+
+The `hogql-materialize-repeated-ctes` project flag, or `materializeRepeatedCTEs` query modifier, can reuse eligible repeated event aggregates within one execution. It defaults off and additionally requires `HOGQL_MATERIALIZED_CTE_SUPPORTED=true` on deployments whose application ClickHouse profile enables materialized CTEs and the analyzer. Explicit materialization hints take precedence. The initial path requires literal timestamp bounds, deterministic aggregates, and references within the same SELECT without filters or keyed joins; other queries retain their existing compilation. Compare equivalent results, median duration, bytes read, and peak memory over five paired runs before enabling a project, and disable the flag or capability setting if errors, latency, or memory regress.
diff --git a/frontend/src/queries/schema.json b/frontend/src/queries/schema.json
index 98ffe7512d66..645d3545b467 100644
--- a/frontend/src/queries/schema.json
+++ b/frontend/src/queries/schema.json
@@ -29745,6 +29745,10 @@
"enum": ["auto", "legacy_null_as_string", "legacy_null_as_null", "disabled"],
"type": "string"
},
+ "materializeRepeatedCTEs": {
+ "description": "Reuse eligible bounded event aggregates within one query execution. When unset, the server may enable it via the cloud feature flag.",
+ "type": "boolean"
+ },
"materializedColumnsOptimizationMode": {
"enum": ["disabled", "optimized"],
"type": "string"
diff --git a/frontend/src/queries/schema/schema-general.ts b/frontend/src/queries/schema/schema-general.ts
index 7f23c11a6fbe..33e87364a7db 100644
--- a/frontend/src/queries/schema/schema-general.ts
+++ b/frontend/src/queries/schema/schema-general.ts
@@ -566,6 +566,8 @@ export interface HogQLQueryModifiers {
usePreaggregatedTableTransforms?: boolean
usePreaggregatedIntermediateResults?: boolean
optimizeProjections?: boolean
+ /** Reuse eligible bounded event aggregates within one query execution. When unset, the server may enable it via the cloud feature flag. */
+ materializeRepeatedCTEs?: boolean
/** Remove provably redundant casts and nullability wrappers (e.g. `toString(String)`, `assumeNotNull(non_nullable)`, dead `ifNull` fallbacks) using inferred expression types */
typeAwareCastSimplification?: boolean
pushDownPredicates?: boolean
diff --git a/posthog/hogql/modifiers.py b/posthog/hogql/modifiers.py
index 8465ca863914..101291b8b949 100644
--- a/posthog/hogql/modifiers.py
+++ b/posthog/hogql/modifiers.py
@@ -95,6 +95,20 @@ def create_default_modifiers_for_team(
if modifiers.optimizeProjections is None:
modifiers.optimizeProjections = True
+ if modifiers.materializeRepeatedCTEs is None and is_cloud():
+ from posthog.ph_client import (
+ feature_enabled_or_false, # noqa: PLC0415 -- keeps the analytics client off django.setup's import path
+ )
+
+ if feature_enabled_or_false(
+ "hogql-materialize-repeated-ctes",
+ str(team.uuid),
+ groups={"organization": str(team.organization_id), "project": str(team.id)},
+ only_evaluate_locally=True,
+ send_feature_flag_events=False,
+ ):
+ modifiers.materializeRepeatedCTEs = True
+
from products.web_analytics.backend.hogql_queries.cookieless_flag import ( # noqa: PLC0415 - keeps posthog.schema off the django.setup() import path
resolve_cookieless_traffic_is_regular_modifier,
)
diff --git a/posthog/hogql/printer/utils.py b/posthog/hogql/printer/utils.py
index 9d254d752e53..8b4ee4776c15 100644
--- a/posthog/hogql/printer/utils.py
+++ b/posthog/hogql/printer/utils.py
@@ -39,6 +39,7 @@ from posthog.hogql.transforms.json_property_pushdown import (
)
from posthog.hogql.transforms.lazy_tables import resolve_lazy_tables
from posthog.hogql.transforms.logical_property_lowering import lower_property_access
+from posthog.hogql.transforms.materialized_ctes import materialize_repeated_ctes
from posthog.hogql.transforms.metrics_time_bucket_bounds import add_metrics_time_bucket_bounds
from posthog.hogql.transforms.negated_person_filters import rewrite_negated_person_filters
from posthog.hogql.transforms.projection_pushdown import pushdown_projections
@@ -327,6 +328,17 @@ def prepare_ast_for_printing(
# wrongly pruned column fails loudly at compile time instead of emitting broken SQL.
context.cte_database_table_cache.clear()
+ if dialect == "clickhouse" and context.modifiers.materializeRepeatedCTEs:
+ from django.conf import (
+ settings as django_settings, # noqa: PLC0415 -- deployment capability is read only on opt-in compilation
+ )
+
+ if django_settings.HOGQL_MATERIALIZED_CTE_SUPPORTED and (
+ settings is None or settings.enable_analyzer is not False
+ ):
+ with context.timings.measure("materialize_repeated_ctes"):
+ materialize_repeated_ctes(node, context)
+
if dialect == "trino":
with context.timings.measure("trino_structural_lowering"):
# The next resolver pass looks up logical schema keys, which may differ from physical column names.
diff --git a/posthog/hogql/test/test_modifiers.py b/posthog/hogql/test/test_modifiers.py
index 907f25e716fc..402608a21a86 100644
--- a/posthog/hogql/test/test_modifiers.py
+++ b/posthog/hogql/test/test_modifiers.py
@@ -6,6 +6,8 @@ from unittest.mock import patch
from django.conf import settings
from django.test import override_settings
+from parameterized import parameterized
+
from posthog.schema import HogQLQueryModifiers, MaterializationMode, PersonsArgMaxVersion, PersonsOnEventsMode
from posthog.hogql.modifiers import create_default_modifiers_for_team
@@ -61,6 +63,22 @@ class TestModifiers(BaseTest):
)
assert modifiers.personsOnEventsMode == PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS
+ @parameterized.expand([(None, False, None), (None, True, True), (False, True, False)])
+ def test_cte_rollout_respects_explicit_kill_switch(
+ self, explicit: bool | None, flag: bool, expected: bool | None
+ ) -> None:
+ with (
+ patch("posthog.hogql.modifiers.is_cloud", return_value=True),
+ patch(
+ "posthog.ph_client.feature_enabled_or_false",
+ side_effect=lambda key, *args, **kwargs: key == "hogql-materialize-repeated-ctes" and flag,
+ ),
+ ):
+ modifiers = create_default_modifiers_for_team(
+ self.team, HogQLQueryModifiers(materializeRepeatedCTEs=explicit)
+ )
+ assert modifiers.materializeRepeatedCTEs is expected
+
@override_settings(PERSON_ON_EVENTS_OVERRIDE=False, PERSON_ON_EVENTS_V2_OVERRIDE=False)
def test_team_modifiers_override(self):
assert self.team.modifiers is None
diff --git a/posthog/hogql/transforms/materialized_ctes.py b/posthog/hogql/transforms/materialized_ctes.py
new file mode 100644
index 000000000000..a3db7da91220
--- /dev/null
+++ b/posthog/hogql/transforms/materialized_ctes.py
@@ -0,0 +1,236 @@
+from collections import defaultdict
+
+from posthog.hogql import ast
+from posthog.hogql.context import HogQLContext
+from posthog.hogql.database.schema.events import EventsTable
+from posthog.hogql.visitor import TraversingVisitor
+
+from posthog.dataclasses import frozen
+
+
+@frozen
+class _TimestampBounds:
+ lower: bool = False
+ upper: bool = False
+
+
+class CTEReferences(TraversingVisitor):
+ def __init__(self) -> None:
+ super().__init__()
+ self.definitions: dict[int, ast.CTE] = {}
+ self.consumers: dict[int, list[ast.SelectQuery]] = defaultdict(list)
+ self.enclosing: dict[int, list[ast.SelectQuery]] = {}
+ self.selects: list[ast.SelectQuery] = []
+ self.set_depth = 0
+ self.unshared: set[int] = set()
+
+ def visit_field(self, node: ast.Field) -> None:
+ pass
+
+ def visit_cte(self, node: ast.CTE) -> None:
+ if node.expr.type is not None:
+ self.definitions[id(node.expr.type)] = node
+ if self.set_depth:
+ self.unshared.add(id(node.expr.type))
+ super().visit_cte(node)
+
+ def visit_select_set_query(self, node: ast.SelectSetQuery) -> None:
+ # ClickHouse expands WITH independently across set operands, including materialization hints.
+ self.set_depth += 1
+ super().visit_select_set_query(node)
+ self.set_depth -= 1
+
+ def visit_select_query(self, node: ast.SelectQuery) -> None:
+ self.selects.append(node)
+ super().visit_select_query(node)
+ self.selects.pop()
+
+ def visit_join_expr(self, node: ast.JoinExpr) -> None:
+ table_type = node.table.type if node.table is not None else None
+ if isinstance(table_type, ast.CTETableAliasType):
+ table_type = table_type.cte_table_type
+ if isinstance(table_type, ast.CTETableType) and self.selects:
+ self.consumers[id(table_type.select_query_type)].append(self.selects[-1])
+ self.enclosing[id(self.selects[-1])] = self.selects[:-1]
+ super().visit_join_expr(node)
+
+
+class _DeterministicEventsAggregate(TraversingVisitor):
+ _AGGREGATES = frozenset({"count", "countIf", "min", "max", "uniqExact"})
+ _CALLS = _AGGREGATES | frozenset(
+ {
+ "toDateTime",
+ "toDate",
+ "toStartOfDay",
+ "toStartOfWeek",
+ "toStartOfMonth",
+ "toStartOfHour",
+ "toStartOfMinute",
+ "toTimeZone",
+ }
+ )
+
+ def __init__(self, query: ast.SelectQuery, context: HogQLContext) -> None:
+ super().__init__()
+ self.query = query
+ self.context = context
+ self.safe = True
+ self.aggregate = False
+
+ def visit_field(self, node: ast.Field) -> None:
+ field_type = node.type
+ if not isinstance(field_type, ast.FieldType):
+ self.safe = False
+ return
+ table_type = field_type.table_type
+ if table_type is self.query.type:
+ return
+ if not isinstance(table_type, ast.BaseTableType) or not isinstance(
+ table_type.resolve_database_table(self.context), EventsTable
+ ):
+ self.safe = False
+ return
+ # A correlated events reference also names EventsTable: require the same resolved scope.
+ source = self.query.select_from
+ if source is None or source.table is None or table_type is not source.table.type:
+ self.safe = False
+
+ def visit_call(self, node: ast.Call) -> None:
+ if node.name not in self._CALLS or node.params or node.distinct:
+ self.safe = False
+ if node.name in self._AGGREGATES:
+ self.aggregate = True
+ super().visit_call(node)
+
+ def visit_select_query(self, node: ast.SelectQuery) -> None:
+ self.safe = False
+
+ def visit_select_set_query(self, node: ast.SelectSetQuery) -> None:
+ self.safe = False
+
+ def visit_window_function(self, node: ast.WindowFunction) -> None:
+ self.safe = False
+
+ @staticmethod
+ def _strip_alias(node: ast.Expr) -> ast.Expr:
+ while isinstance(node, ast.Alias):
+ node = node.expr
+ return node
+
+ @classmethod
+ def _literal_date(cls, node: ast.Expr) -> bool:
+ node = cls._strip_alias(node)
+ return (
+ isinstance(node, ast.Constant)
+ and node.value is not None
+ or (
+ isinstance(node, ast.Call)
+ and node.name in {"toDate", "toDateTime"}
+ and bool(node.args)
+ and all(isinstance(arg, ast.Constant) and arg.value is not None for arg in node.args)
+ )
+ )
+
+ def _timestamp(self, node: ast.Expr) -> bool:
+ node = self._strip_alias(node)
+ source = self.query.select_from
+ return (
+ isinstance(node, ast.Field)
+ and isinstance(node.type, ast.FieldType)
+ and node.type.name == "timestamp"
+ and source is not None
+ and source.table is not None
+ and node.type.table_type is source.table.type
+ )
+
+ def _bounds(self, node: ast.Expr | None) -> _TimestampBounds:
+ if isinstance(node, ast.And):
+ bounds = [self._bounds(expr) for expr in node.exprs]
+ return _TimestampBounds(
+ lower=any(bound.lower for bound in bounds), upper=any(bound.upper for bound in bounds)
+ )
+ if isinstance(node, ast.BetweenExpr) and not node.negated and self._timestamp(node.expr):
+ return _TimestampBounds(lower=self._literal_date(node.low), upper=self._literal_date(node.high))
+ if isinstance(node, ast.CompareOperation):
+ op = node.op
+ if self._timestamp(node.right) and self._literal_date(node.left):
+ op = {
+ ast.CompareOperationOp.Gt: ast.CompareOperationOp.Lt,
+ ast.CompareOperationOp.GtEq: ast.CompareOperationOp.LtEq,
+ ast.CompareOperationOp.Lt: ast.CompareOperationOp.Gt,
+ ast.CompareOperationOp.LtEq: ast.CompareOperationOp.GtEq,
+ }.get(op, op)
+ elif not (self._timestamp(node.left) and self._literal_date(node.right)):
+ return _TimestampBounds()
+ return _TimestampBounds(
+ lower=op in {ast.CompareOperationOp.Gt, ast.CompareOperationOp.GtEq, ast.CompareOperationOp.Eq},
+ upper=op in {ast.CompareOperationOp.Lt, ast.CompareOperationOp.LtEq, ast.CompareOperationOp.Eq},
+ )
+ return _TimestampBounds()
+
+ def eligible(self) -> bool:
+ query = self.query
+ source = query.select_from
+ bounds = self._bounds(ast.And(exprs=[expr for expr in (query.where, query.prewhere) if expr is not None]))
+ if (
+ source is None
+ or source.table is None
+ or source.next_join is not None
+ or source.table_args
+ or source.sample is not None
+ or not isinstance(source.table.type, ast.BaseTableType)
+ or not isinstance(source.table.type.resolve_database_table(self.context), EventsTable)
+ or query.ctes
+ or query.limit is not None
+ or query.limit_by is not None
+ or query.offset is not None
+ or query.order_by
+ or query.array_join_list
+ or query.window_exprs
+ or query.qualify
+ or query.settings
+ or not bounds.lower
+ or not bounds.upper
+ ):
+ return False
+ for expr in [*query.select, *(query.group_by or []), query.where, query.prewhere, query.having]:
+ self.visit(expr)
+ return self.safe and self.aggregate
+
+
+def _filters(select: ast.SelectQuery) -> bool:
+ if select.where or select.prewhere or select.having or select.qualify:
+ return True
+ join = select.select_from.next_join if select.select_from else None
+ while join is not None:
+ if join.join_type != "CROSS JOIN" or join.constraint is not None:
+ return True
+ join = join.next_join
+ return False
+
+
+def materialize_repeated_ctes(node: ast.AST, context: HogQLContext) -> None:
+ references = CTEReferences()
+ references.visit(node)
+ for type_id, cte in references.definitions.items():
+ consumers = references.consumers[type_id]
+ if (
+ type_id in references.unshared
+ or cte.materialized is not None
+ or cte.recursive
+ or cte.columns
+ or cte.using_key
+ or cte.cte_type != "subquery"
+ or not isinstance(cte.expr, ast.SelectQuery)
+ or len(consumers) < 2
+ or len({id(consumer) for consumer in consumers}) != 1
+ ):
+ continue
+ # Consumer predicates may benefit more from pushdown than from sharing the entire aggregate.
+ # ClickHouse also pushes predicates from enclosing SELECTs down through subqueries.
+ if any(
+ _filters(select) for consumer in consumers for select in [consumer, *references.enclosing[id(consumer)]]
+ ):
+ continue
+ if _DeterministicEventsAggregate(cte.expr, context).eligible():
+ cte.materialized = True
diff --git a/posthog/hogql/transforms/test/__snapshots__/test_materialized_ctes.ambr b/posthog/hogql/transforms/test/__snapshots__/test_materialized_ctes.ambr
new file mode 100644
index 000000000000..2b4ff935380d
--- /dev/null
+++ b/posthog/hogql/transforms/test/__snapshots__/test_materialized_ctes.ambr
@@ -0,0 +1,26 @@
+# serializer version: 1
+# name: TestMaterializedCTEResults.test_reuse_preserves_results_including_empty_data_0
+ '''
+ WITH totals AS MATERIALIZED
+ (SELECT events.event AS event, count() AS n
+ FROM events
+ WHERE and(equals(events.team_id, 99999), greaterOrEquals(events.timestamp, toDateTime(%(hogql_val_0)s, %(hogql_val_1)s)), less(events.timestamp, toDateTime(%(hogql_val_2)s, %(hogql_val_3)s)))
+ GROUP BY events.event)
+ SELECT sum(l.n) AS total,
+ max(r.n) AS maximum
+ FROM totals AS l
+ CROSS JOIN totals AS r
+ LIMIT 100 SETTINGS readonly=2,
+ max_execution_time=60,
+ allow_experimental_object_type=1,
+ max_ast_elements=4000000,
+ max_expanded_ast_elements=4000000,
+ max_bytes_before_external_group_by=0,
+ transform_null_in=1,
+ optimize_min_equality_disjunction_chain_length=4294967295,
+ optimize_rewrite_aggregate_function_with_if=0,
+ optimize_min_inequality_conjunction_chain_length=4294967295,
+ allow_experimental_join_condition=1,
+ use_hive_partitioning=0
+ '''
+# ---
diff --git a/posthog/hogql/transforms/test/__snapshots__/test_materialized_ctes.new_events_schema.ambr b/posthog/hogql/transforms/test/__snapshots__/test_materialized_ctes.new_events_schema.ambr
new file mode 100644
index 000000000000..26d8d670ca1a
--- /dev/null
+++ b/posthog/hogql/transforms/test/__snapshots__/test_materialized_ctes.new_events_schema.ambr
@@ -0,0 +1,28 @@
+# serializer version: 1
+# name: TestMaterializedCTEResults.test_reuse_preserves_results_including_empty_data_0[new_events_schema]
+ '''
+ WITH totals AS MATERIALIZED
+ (SELECT events.event AS event, count() AS n
+ FROM events_json AS events
+ WHERE and(equals(events.team_id, 99999), greaterOrEquals(events.timestamp, toDateTime(%(hogql_val_0)s, %(hogql_val_1)s)), less(events.timestamp, toDateTime(%(hogql_val_2)s, %(hogql_val_3)s)))
+ GROUP BY events.event)
+ SELECT sum(l.n) AS total,
+ max(r.n) AS maximum
+ FROM totals AS l
+ CROSS JOIN totals AS r
+ LIMIT 100 SETTINGS readonly=2,
+ max_execution_time=60,
+ allow_experimental_object_type=1,
+ max_ast_elements=4000000,
+ max_expanded_ast_elements=4000000,
+ max_bytes_before_external_group_by=0,
+ transform_null_in=1,
+ optimize_min_equality_disjunction_chain_length=4294967295,
+ optimize_rewrite_aggregate_function_with_if=0,
+ optimize_min_inequality_conjunction_chain_length=4294967295,
+ allow_experimental_join_condition=1,
+ use_hive_partitioning=0,
+ json_type_escape_dots_in_keys=1,
+ output_format_json_escape_forward_slashes=0
+ '''
+# ---
diff --git a/posthog/hogql/transforms/test/test_materialized_ctes.py b/posthog/hogql/transforms/test/test_materialized_ctes.py
new file mode 100644
index 000000000000..97e087f76c5d
--- /dev/null
+++ b/posthog/hogql/transforms/test/test_materialized_ctes.py
@@ -0,0 +1,139 @@
+from uuid import UUID
+
+import time_machine
+from posthog.test.base import BaseTest, ClickhouseTestMixin, _create_event, flush_persons_and_events
+
+from django.conf import settings
+from django.test import SimpleTestCase, override_settings
+
+from parameterized import parameterized
+
+from posthog.schema import HogQLQueryModifiers
+
+from posthog.hogql.constants import HogQLGlobalSettings
+from posthog.hogql.context import HogQLContext
+from posthog.hogql.database.database import Database
+from posthog.hogql.errors import ImpossibleASTError
+from posthog.hogql.parser import parse_select
+from posthog.hogql.printer import prepare_and_print_ast
+from posthog.hogql.query import execute_hogql_query
+
+from posthog.models.team import Team
+
+_BODY = "SELECT event, count() AS n FROM events WHERE timestamp >= toDateTime('2023-02-01') AND timestamp < toDateTime('2023-02-03') GROUP BY event"
+_QUERY = "WITH totals AS ({body}) SELECT sum(l.n) AS total, max(r.n) AS maximum FROM totals l CROSS JOIN totals r"
+
+
+@override_settings(HOGQL_MATERIALIZED_CTE_SUPPORTED=True)
+class TestMaterializedCTEs(SimpleTestCase):
+ @parameterized.expand(
+ [
+ (_BODY, True),
+ (_BODY.replace("count()", "any(distinct_id)"), False),
+ (_BODY.replace("count()", "groupArray(distinct_id)"), False),
+ (_BODY.replace("count()", "sum(rand())"), False),
+ (_BODY + " LIMIT 10", False),
+ (_BODY.replace("AND timestamp < toDateTime('2023-02-03')", ""), False),
+ (
+ "SELECT event, 1 AS n FROM events WHERE timestamp >= toDateTime('2023-02-01') AND timestamp < toDateTime('2023-02-03')",
+ False,
+ ),
+ ]
+ )
+ def test_only_bounded_deterministic_aggregates_are_reused(self, body: str, materialized: bool) -> None:
+ sql = self._compile(_QUERY.format(body=body))
+ assert ("AS MATERIALIZED" in sql) is materialized
+
+ def _compile(self, query: str, enabled: bool = True, analyzer: bool | None = None) -> str:
+ return prepare_and_print_ast(
+ parse_select(query),
+ dialect="clickhouse",
+ context=HogQLContext(
+ team=Team(id=1, timezone="UTC"),
+ database=Database(timezone="UTC"),
+ restricted_properties=set(),
+ enable_select_queries=True,
+ modifiers=HogQLQueryModifiers(
+ materializeRepeatedCTEs=enabled,
+ useNewEventsSchema=settings.CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA,
+ ),
+ ),
+ settings=HogQLGlobalSettings(enable_analyzer=analyzer),
+ )[0]
+
+ @parameterized.expand(
+ [
+ (_QUERY.format(body=_BODY).replace("AS (", "AS NOT MATERIALIZED (", 1), None),
+ (_QUERY.format(body=_BODY).replace("AS (", "AS MATERIALIZED (", 1), True),
+ (f"WITH totals AS ({_BODY}) SELECT sum(n) FROM totals", False),
+ (_QUERY.format(body=_BODY) + " WHERE l.event = 'example'", False),
+ (_QUERY.format(body=_BODY) + " GROUP BY l.event HAVING l.event = 'example'", False),
+ (
+ f"WITH totals AS ({_BODY}) SELECT total FROM (SELECT l.event AS event, sum(l.n) AS total FROM totals l CROSS JOIN totals r GROUP BY l.event) WHERE event = 'example'",
+ False,
+ ),
+ (
+ f"WITH totals AS ({_BODY}) SELECT total FROM (SELECT sum(l.n) AS total FROM totals l CROSS JOIN totals r)",
+ True,
+ ),
+ (
+ f"WITH totals AS ({_BODY}) SELECT sum(n) FROM totals WHERE event = 'example' UNION ALL SELECT max(n) FROM totals",
+ False,
+ ),
+ (
+ f"WITH totals AS ({_BODY}) SELECT sum(n) FROM totals UNION ALL SELECT n FROM (WITH totals AS (SELECT 1 AS n) SELECT n FROM totals)",
+ False,
+ ),
+ ]
+ )
+ def test_explicit_hints_filters_and_shadowed_scopes(self, query: str, materialized: bool | None) -> None:
+ if materialized is None:
+ with self.assertRaisesRegex(ImpossibleASTError, "NOT MATERIALIZED"):
+ self._compile(query)
+ else:
+ sql = self._compile(query)
+ assert ("AS MATERIALIZED" in sql) is materialized
+ if not materialized:
+ assert sql == self._compile(query, enabled=False)
+
+ @parameterized.expand([(False, True, None), (True, False, None), (True, True, False)])
+ def test_modifier_capability_and_analyzer_are_required(
+ self, enabled: bool, capability: bool, analyzer: bool | None
+ ) -> None:
+ with override_settings(HOGQL_MATERIALIZED_CTE_SUPPORTED=capability):
+ assert "AS MATERIALIZED" not in self._compile(_QUERY.format(body=_BODY), enabled, analyzer)
+
+
+@override_settings(HOGQL_MATERIALIZED_CTE_SUPPORTED=True)
+class TestMaterializedCTEResults(ClickhouseTestMixin, BaseTest):
+ allow_dual_schema_snapshots = True
+
+ def test_property_timestamp_does_not_bound_the_event_scan(self) -> None:
+ query = _QUERY.format(body=_BODY.replace("timestamp", "properties.timestamp"))
+ response = execute_hogql_query(query, self.team, modifiers=HogQLQueryModifiers(materializeRepeatedCTEs=True))
+ assert response.clickhouse is not None
+ assert "AS MATERIALIZED" not in response.clickhouse
+
+ @parameterized.expand([(False,), (True,)])
+ @time_machine.travel("2023-02-03", tick=False)
+ def test_reuse_preserves_results_including_empty_data(self, empty: bool) -> None:
+ if not empty:
+ for event in ["example_a", "example_a", "example_b"]:
+ _create_event(
+ team=self.team,
+ event=event,
+ distinct_id="example_actor",
+ person_id=UUID(int=1),
+ timestamp="2023-02-01T00:00:00Z",
+ )
+ flush_persons_and_events()
+ query = _QUERY.format(body=_BODY)
+ original = execute_hogql_query(query, self.team, modifiers=HogQLQueryModifiers(materializeRepeatedCTEs=False))
+ optimized = execute_hogql_query(query, self.team, modifiers=HogQLQueryModifiers(materializeRepeatedCTEs=True))
+ assert optimized.results == original.results
+ assert optimized.columns == original.columns
+ assert optimized.types == original.types
+ assert optimized.clickhouse is not None
+ assert "AS MATERIALIZED" in optimized.clickhouse
+ if not empty:
+ self.assertQueryMatchesSnapshot(optimized.clickhouse)
diff --git a/posthog/schema.py b/posthog/schema.py
index 416d2769eb16..81f70d57ce01 100644
--- a/posthog/schema.py
+++ b/posthog/schema.py
@@ -6239,6 +6239,13 @@ class HogQLQueryModifiers(BaseModel):
inCohortVia: InCohortVia | None = None
inlineCohortCalculation: InlineCohortCalculation | None = None
materializationMode: MaterializationMode | None = None
+ materializeRepeatedCTEs: bool | None = Field(
+ default=None,
+ description=(
+ "Reuse eligible bounded event aggregates within one query execution. When"
+ " unset, the server may enable it via the cloud feature flag."
+ ),
+ )
materializedColumnsOptimizationMode: MaterializedColumnsOptimizationMode | None = None
mergeFederatedAggregateJoins: bool | None = Field(
default=None,
diff --git a/posthog/settings/__init__.py b/posthog/settings/__init__.py
index d1653923f7a4..b9af458de233 100644
--- a/posthog/settings/__init__.py
+++ b/posthog/settings/__init__.py
@@ -158,6 +158,9 @@ OTEL_SERVICE_NAME: str | None = os.getenv("OTEL_SERVICE_NAME", None)
PROM_PUSHGATEWAY_ADDRESS: str | None = os.getenv("PROM_PUSHGATEWAY_ADDRESS", None)
+# Set only when the application ClickHouse profile supports AS MATERIALIZED and enables its analyzer.
+HOGQL_MATERIALIZED_CTE_SUPPORTED: bool = get_from_env("HOGQL_MATERIALIZED_CTE_SUPPORTED", False, type_cast=str_to_bool)
+
HOGQL_INCREASED_MAX_EXECUTION_TIME: int = get_from_env("HOGQL_INCREASED_MAX_EXECUTION_TIME", 600, type_cast=int)
# Extend and override these settings with EE's ones
diff --git a/products/batch_exports/frontend/generated/api.schemas.ts b/products/batch_exports/frontend/generated/api.schemas.ts
index ce0051bdd3d7..2b655e77ecd6 100644
--- a/products/batch_exports/frontend/generated/api.schemas.ts
+++ b/products/batch_exports/frontend/generated/api.schemas.ts
@@ -832,6 +832,8 @@ export interface HogQLQueryModifiersApi {
inCohortVia?: InCohortViaApi | null
inlineCohortCalculation?: InlineCohortCalculationApi | null
materializationMode?: MaterializationModeApi | null
+ /** Reuse eligible bounded event aggregates within one query execution. When unset, the server may enable it via the cloud feature flag. */
+ materializeRepeatedCTEs?: boolean | null
materializedColumnsOptimizationMode?: MaterializedColumnsOptimizationModeApi | null
/** Merge sibling aggregating LEFT JOINs over federated Postgres tables into one UNION ALL join, so their scans overlap */
mergeFederatedAggregateJoins?: boolean | null
diff --git a/products/batch_exports/frontend/generated/api.zod.ts b/products/batch_exports/frontend/generated/api.zod.ts
index 42ebb894ac41..b6131f4f7552 100644
--- a/products/batch_exports/frontend/generated/api.zod.ts
+++ b/products/batch_exports/frontend/generated/api.zod.ts
@@ -583,6 +583,12 @@ export const BatchExportsCreateBody = /* @__PURE__ */ zod
zod.null(),
])
.optional(),
+ materializeRepeatedCTEs: zod
+ .union([zod.boolean(), zod.null()])
+ .optional()
+ .describe(
+ 'Reuse eligible bounded event aggregates within one query execution. When unset, the server may enable it via the cloud feature flag.'
+ ),
materializedColumnsOptimizationMode: zod
.union([zod.enum(['disabled', 'optimized']), zod.null()])
.optional(),
@@ -1563,6 +1569,12 @@ export const BatchExportsUpdateBody = /* @__PURE__ */ zod
zod.null(),
])
.optional(),
+ materializeRepeatedCTEs: zod
+ .union([zod.boolean(), zod.null()])
+ .optional()
+ .describe(
+ 'Reuse eligible bounded event aggregates within one query execution. When unset, the server may enable it via the cloud feature flag.'
+ ),
materializedColumnsOptimizationMode: zod
.union([zod.enum(['disabled', 'optimized']), zod.null()])
.optional(),
@@ -2302,6 +2314,12 @@ export const BatchExportsPartialUpdateBody = /* @__PURE__ */ zod
zod.null(),
])
.optional(),
+ materializeRepeatedCTEs: zod
+ .union([zod.boolean(), zod.null()])
+ .optional()
+ .describe(
+ 'Reuse eligible bounded event aggregates within one query execution. When unset, the server may enable it via the cloud feature flag.'
+ ),
materializedColumnsOptimizationMode: zod
.union([zod.enum(['disabled', 'optimized']), zod.null()])
.optional(),
@@ -3026,6 +3044,12 @@ export const BatchExportsRunTestStepCreateBody = /* @__PURE__ */ zod
zod.null(),
])
.optional(),
+ materializeRepeatedCTEs: zod
+ .union([zod.boolean(), zod.null()])
+ .optional()
+ .describe(
+ 'Reuse eligible bounded event aggregates within one query execution. When unset, the server may enable it via the cloud feature flag.'
+ ),
materializedColumnsOptimizationMode: zod
.union([zod.enum(['disabled', 'optimized']), zod.null()])
.optional(),
@@ -3769,6 +3793,12 @@ export const BatchExportsRunTestStepNewCreateBody = /* @__PURE__ */ zod
zod.null(),
])
.optional(),
+ materializeRepeatedCTEs: zod
+ .union([zod.boolean(), zod.null()])
+ .optional()
+ .describe(
+ 'Reuse eligible bounded event aggregates within one query execution. When unset, the server may enable it via the cloud feature flag.'
+ ),
materializedColumnsOptimizationMode: zod
.union([zod.enum(['disabled', 'optimized']), zod.null()])
.optional(),
@@ -4229,6 +4259,12 @@ export const FileDownloadBatchExportsCreateBody = /* @__PURE__ */ zod.union([
zod.null(),
])
.optional(),
+ materializeRepeatedCTEs: zod
+ .union([zod.boolean(), zod.null()])
+ .optional()
+ .describe(
+ 'Reuse eligible bounded event aggregates within one query execution. When unset, the server may enable it via the cloud feature flag.'
+ ),
materializedColumnsOptimizationMode: zod
.union([zod.enum(['disabled', 'optimized']), zod.null()])
.optional(),
@@ -4495,6 +4531,12 @@ export const FileDownloadBatchExportsCountRowsCreateBody = /* @__PURE__ */ zod
materializationMode: zod
.union([zod.enum(['auto', 'legacy_null_as_string', 'legacy_null_as_null', 'disabled']), zod.null()])
.optional(),
+ materializeRepeatedCTEs: zod
+ .union([zod.boolean(), zod.null()])
+ .optional()
+ .describe(
+ 'Reuse eligible bounded event aggregates within one query execution. When unset, the server may enable it via the cloud feature flag.'
+ ),
materializedColumnsOptimizationMode: zod
.union([zod.enum(['disabled', 'optimized']), zod.null()])
.optional(),
diff --git a/products/customer_analytics/frontend/generated/api.schemas.ts b/products/customer_analytics/frontend/generated/api.schemas.ts
index c2400a8aa01a..37334e551e12 100644
--- a/products/customer_analytics/frontend/generated/api.schemas.ts
+++ b/products/customer_analytics/frontend/generated/api.schemas.ts
@@ -1762,6 +1762,8 @@ export interface HogQLQueryModifiersApi {
inCohortVia?: InCohortViaApi | null
inlineCohortCalculation?: InlineCohortCalculationApi | null
materializationMode?: MaterializationModeApi | null
+ /** Reuse eligible bounded event aggregates within one query execution. When unset, the server may enable it via the cloud feature flag. */
+ materializeRepeatedCTEs?: boolean | null
materializedColumnsOptimizationMode?: MaterializedColumnsOptimizationModeApi | null
/** Merge sibling aggregating LEFT JOINs over federated Postgres tables into one UNION ALL join, so their scans overlap */
mergeFederatedAggregateJoins?: boolean | null
diff --git a/products/dashboards/frontend/generated/api.schemas.ts b/products/dashboards/frontend/generated/api.schemas.ts
index c1c5017cdbb9..d13c800fc89f 100644
--- a/products/dashboards/frontend/generated/api.schemas.ts
+++ b/products/dashboards/frontend/generated/api.schemas.ts
@@ -2435,6 +2435,8 @@ export interface HogQLQueryModifiersApi {
inCohortVia?: InCohortViaApi | null
inlineCohortCalculation?: InlineCohortCalculationApi | null
materializationMode?: MaterializationModeApi | null
+ /** Reuse eligible bounded event aggregates within one query execution. When unset, the server may enable it via the cloud feature flag. */
+ materializeRepeatedCTEs?: boolean | null
materializedColumnsOptimizationMode?: MaterializedColumnsOptimizationModeApi | null
/** Merge sibling aggregating LEFT JOINs over federated Postgres tables into one UNION ALL join, so their scans overlap */
mergeFederatedAggregateJoins?: boolean | null
diff --git a/products/desktop/packages/api-client/src/generated.ts b/products/desktop/packages/api-client/src/generated.ts
index 629c6c0fdb3a..d193252cd596 100644
--- a/products/desktop/packages/api-client/src/generated.ts
+++ b/products/desktop/packages/api-client/src/generated.ts
@@ -213,6 +213,7 @@ export namespace Schemas {
inCohortVia: InCohortVia | null;
inlineCohortCalculation: InlineCohortCalculation | null;
materializationMode: MaterializationMode | null;
+ materializeRepeatedCTEs: boolean | null;
materializedColumnsOptimizationMode: MaterializedColumnsOptimizationMode | null;
mergeFederatedAggregateJoins: boolean | null;
negatedPersonFiltersNotIn: boolean | null;
diff --git a/products/marketing_analytics/frontend/generated/api.schemas.ts b/products/marketing_analytics/frontend/generated/api.schemas.ts
index 66d2dea31fab..5575a835bc35 100644
--- a/products/marketing_analytics/frontend/generated/api.schemas.ts
+++ b/products/marketing_analytics/frontend/generated/api.schemas.ts
@@ -1719,6 +1719,8 @@ export interface HogQLQueryModifiersApi {
inCohortVia?: InCohortViaApi | null
inlineCohortCalculation?: InlineCohortCalculationApi | null
materializationMode?: MaterializationModeApi | null
+ /** Reuse eligible bounded event aggregates within one query execution. When unset, the server may enable it via the cloud feature flag. */
+ materializeRepeatedCTEs?: boolean | null
materializedColumnsOptimizationMode?: MaterializedColumnsOptimizationModeApi | null
/** Merge sibling aggregating LEFT JOINs over federated Postgres tables into one UNION ALL join, so their scans overlap */
mergeFederatedAggregateJoins?: boolean | null
diff --git a/products/persons/frontend/generated/api.schemas.ts b/products/persons/frontend/generated/api.schemas.ts
index 1ac403ec39a2..86e45a155586 100644
--- a/products/persons/frontend/generated/api.schemas.ts
+++ b/products/persons/frontend/generated/api.schemas.ts
@@ -537,6 +537,8 @@ export interface HogQLQueryModifiersApi {
inCohortVia?: InCohortViaApi | null
inlineCohortCalculation?: InlineCohortCalculationApi | null
materializationMode?: MaterializationModeApi | null
+ /** Reuse eligible bounded event aggregates within one query execution. When unset, the server may enable it via the cloud feature flag. */
+ materializeRepeatedCTEs?: boolean | null
materializedColumnsOptimizationMode?: MaterializedColumnsOptimizationModeApi | null
/** Merge sibling aggregating LEFT JOINs over federated Postgres tables into one UNION ALL join, so their scans overlap */
mergeFederatedAggregateJoins?: boolean | null
diff --git a/products/product_analytics/frontend/generated/api.schemas.ts b/products/product_analytics/frontend/generated/api.schemas.ts
index 0c7129287620..4aeccf63cebf 100644
--- a/products/product_analytics/frontend/generated/api.schemas.ts
+++ b/products/product_analytics/frontend/generated/api.schemas.ts
@@ -1251,6 +1251,8 @@ export interface HogQLQueryModifiersApi {
inCohortVia?: InCohortViaApi | null
inlineCohortCalculation?: InlineCohortCalculationApi | null
materializationMode?: MaterializationModeApi | null
+ /** Reuse eligible bounded event aggregates within one query execution. When unset, the server may enable it via the cloud feature flag. */
+ materializeRepeatedCTEs?: boolean | null
materializedColumnsOptimizationMode?: MaterializedColumnsOptimizationModeApi | null
/** Merge sibling aggregating LEFT JOINs over federated Postgres tables into one UNION ALL join, so their scans overlap */
mergeFederatedAggregateJoins?: boolean | null
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index da8825d83e46..afc1f904d8c6 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -2837,6 +2837,8 @@ export namespace Schemas {
inCohortVia?: InCohortVia | null;
inlineCohortCalculation?: InlineCohortCalculation | null;
materializationMode?: MaterializationMode | null;
+ /** Reuse eligible bounded event aggregates within one query execution. When unset, the server may enable it via the cloud feature flag. */
+ materializeRepeatedCTEs?: boolean | null;
materializedColumnsOptimizationMode?: MaterializedColumnsOptimizationMode | null;
/** Merge sibling aggregating LEFT JOINs over federated Postgres tables into one UNION ALL join, so their scans overlap */
mergeFederatedAggregateJoins?: boolean | null;
diff --git a/services/mcp/src/generated/batch_exports/api.ts b/services/mcp/src/generated/batch_exports/api.ts
index 6a9e0094ce86..73c2f9b1406f 100644
--- a/services/mcp/src/generated/batch_exports/api.ts
+++ b/services/mcp/src/generated/batch_exports/api.ts
@@ -595,6 +595,12 @@ export const BatchExportsCreateBody = () => zod
zod.null(),
])
.optional(),
+ materializeRepeatedCTEs: zod
+ .union([zod.boolean(), zod.null()])
+ .optional()
+ .describe(
+ 'Reuse eligible bounded event aggregates within one query execution. When unset, the server may enable it via the cloud feature flag.'
+ ),
materializedColumnsOptimizationMode: zod
.union([zod.enum(['disabled', 'optimized']), zod.null()])
.optional(),
@@ -1338,6 +1344,12 @@ export const BatchExportsPartialUpdateBody = () => zod
zod.null(),
])
.optional(),
+ materializeRepeatedCTEs: zod
+ .union([zod.boolean(), zod.null()])
+ .optional()
+ .describe(
+ 'Reuse eligible bounded event aggregates within one query execution. When unset, the server may enable it via the cloud feature flag.'
+ ),
materializedColumnsOptimizationMode: zod
.union([zod.enum(['disabled', 'optimized']), zod.null()])
.optional(),
@@ -1812,6 +1824,12 @@ export const FileDownloadBatchExportsCreateBody = () => zod.union([
zod.null(),
])
.optional(),
+ materializeRepeatedCTEs: zod
+ .union([zod.boolean(), zod.null()])
+ .optional()
+ .describe(
+ 'Reuse eligible bounded event aggregates within one query execution. When unset, the server may enable it via the cloud feature flag.'
+ ),
materializedColumnsOptimizationMode: zod
.union([zod.enum(['disabled', 'optimized']), zod.null()])
.optional(),
@@ -2114,6 +2132,12 @@ export const FileDownloadBatchExportsCountRowsCreateBody = () => zod
materializationMode: zod
.union([zod.enum(['auto', 'legacy_null_as_string', 'legacy_null_as_null', 'disabled']), zod.null()])
.optional(),
+ materializeRepeatedCTEs: zod
+ .union([zod.boolean(), zod.null()])
+ .optional()
+ .describe(
+ 'Reuse eligible bounded event aggregates within one query execution. When unset, the server may enable it via the cloud feature flag.'
+ ),
materializedColumnsOptimizationMode: zod
.union([zod.enum(['disabled', 'optimized']), zod.null()])
.optional(),
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/batch-export-create.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/batch-export-create.json
index b30e4d3790c1..e085400de2e2 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/batch-export-create.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/batch-export-create.json
@@ -848,6 +848,17 @@
}
]
},
+ "materializeRepeatedCTEs": {
+ "anyOf": [
+ {
+ "type": "boolean"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "description": "Reuse eligible bounded event aggregates within one query execution. When unset, the server may enable it via the cloud feature flag."
+ },
"mergeFederatedAggregateJoins": {
"anyOf": [
{
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/batch-export-update.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/batch-export-update.json
index 92e3c09c1c56..a9e08f6b4925 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/batch-export-update.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/batch-export-update.json
@@ -847,6 +847,17 @@
}
]
},
+ "materializeRepeatedCTEs": {
+ "anyOf": [
+ {
+ "type": "boolean"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "description": "Reuse eligible bounded event aggregates within one query execution. When unset, the server may enable it via the cloud feature flag."
+ },
"mergeFederatedAggregateJoins": {
"anyOf": [
{
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/file-download-batch-exports-count-rows-create.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/file-download-batch-exports-count-rows-create.json
index 442048d51b07..f6914a50859f 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/file-download-batch-exports-count-rows-create.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/file-download-batch-exports-count-rows-create.json
@@ -323,6 +323,17 @@
}
]
},
+ "materializeRepeatedCTEs": {
+ "anyOf": [
+ {
+ "type": "boolean"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "description": "Reuse eligible bounded event aggregates within one query execution. When unset, the server may enable it via the cloud feature flag."
+ },
"mergeFederatedAggregateJoins": {
"anyOf": [
{
diff --git a/services/mcp/tests/unit/__snapshots__/tool-schemas/file-download-batch-exports-create.json b/services/mcp/tests/unit/__snapshots__/tool-schemas/file-download-batch-exports-create.json
index b07225dde207..5b5f542a9200 100644
--- a/services/mcp/tests/unit/__snapshots__/tool-schemas/file-download-batch-exports-create.json
+++ b/services/mcp/tests/unit/__snapshots__/tool-schemas/file-download-batch-exports-create.json
@@ -552,6 +552,17 @@
}
]
},
+ "materializeRepeatedCTEs": {
+ "anyOf": [
+ {
+ "type": "boolean"
+ },
+ {
+ "type": "null"
+ }
+ ],
+ "description": "Reuse eligible bounded event aggregates within one query execution. When unset, the server may enable it via the cloud feature flag."
+ },
"mergeFederatedAggregateJoins": {
"anyOf": [
{