:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(hogql): reland read-keys export with by-name union pruning (#128821)
Sandy Spicer2f8f63cdac12
diff --git a/docs/published/handbook/engineering/databases/hogql-python.md b/docs/published/handbook/engineering/databases/hogql-python.mdindex f1ba08e65906..f190b39d9f11 100644--- a/docs/published/handbook/engineering/databases/hogql-python.md+++ b/docs/published/handbook/engineering/databases/hogql-python.md@@ -59,6 +59,13 @@ Circular dependencies fail with a query error. Soft saves skip column inferenceQuery preparation shares a 60-second cooperative deadline across parsing, resolution, transformations, and printing.AST visitors check the deadline while walking; native parser calls are checked when they return, rather than interrupted mid-call.+## Disabling projection optimization++Set `HOGQL_PROJECTION_OPTIMIZATION_ENABLED` to `false` in Django admin's instance settings to disable unused-column pruning and exported event-property narrowing for every project.+The setting defaults to `true` and overrides the `optimizeProjections` query modifier when disabled.+Workers pick up the change within about a minute; it affects newly compiled queries, not running queries or cached results.+Disabling these optimizations can increase query time and memory use. Set the instance setting back to `true` to restore the normal per-query behavior.+## Pattern matching during query preparationExpressions inside HogQL placeholders execute in the Python HogVM.diff --git a/posthog/api/test/test_event.py b/posthog/api/test/test_event.pyindex 28f07c1122b6..d09178e6e4ce 100644--- a/posthog/api/test/test_event.py+++ b/posthog/api/test/test_event.py@@ -114,10 +114,10 @@ class TestEvents(ClickhouseTestMixin, APIBaseTest):# personhog, not Postgres, so it's not in this count. Was 16 before passing team=team# into get_restricted_properties_for_team, which lets is_property_access_control_enabled# skip its per-call Team+organization lookup. +1 for the saved-expressions fetch in the- # HogQL database build. +1 for the flag-cache TTL instance setting, cold-cache here but- # TTL-cached per worker in production. +1 for the organization's flag_evaluations mode lookup in- # the HogQL database build.- with self.assertNumQueries(16):+ # HogQL database build. +1 each for the flag-cache TTL, projection kill-switch and+ # properties-narrowing instance settings, cold-cache here but TTL-cached per worker in production. +1 for the+ # organization's flag_evaluations mode lookup in the HogQL database build.+ with self.assertNumQueries(18):response = self.client.get(f"/api/projects/{self.team.id}/events/?event=event_name").json()assert response["results"][0]["event"] == "event_name"@@ -152,10 +152,10 @@ class TestEvents(ClickhouseTestMixin, APIBaseTest):# Group-type-mapping is read via personhog, not Postgres, so it's not in this count.# Was 24 before passing team=team into get_restricted_properties_for_team, which lets# is_property_access_control_enabled skip its per-call Team+organization lookup.- # +1 for the saved-expressions fetch in the HogQL database build. +1 for the flag-cache- # TTL instance setting, cold-cache here but TTL-cached per worker in production. +1 for- # the organization's flag_evaluations mode lookup in the HogQL database build.- expected_queries = 23 if settings.CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA else 24+ # +1 for the saved-expressions fetch in the HogQL database build. +1 each for the flag-cache+ # TTL, projection kill-switch and properties-narrowing instance settings, cold-cache here but+ # TTL-cached per worker in production. +1 for the organization's flag_evaluations mode lookup in the HogQL database build.+ expected_queries = 25 if settings.CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA else 26with self.assertNumQueries(expected_queries):response = self.client.get(diff --git a/posthog/api/test/test_person.py b/posthog/api/test/test_person.pyindex 58073ec7a2c1..e5049cd95680 100644--- a/posthog/api/test/test_person.py+++ b/posthog/api/test/test_person.py@@ -2264,10 +2264,10 @@ class TestPerson(ClickhouseTestMixin, APIBaseTest, QueryMatchingTest):# The property-access-control feature check reuses the request's already-loaded team,# so listing persons no longer pays a per-request Team lookup (was 16). +1 for the# saved-expressions fetch in the HogQL database build. +1 each for the shared-database- # kill-switch and flag-cache TTL instance settings, cold-cache here but TTL-cached per- # worker in production. +1 for the organization's flag_evaluations mode lookup in the HogQL- # database build.- with self.assertNumQueries(17):+ # kill-switch, projection kill-switch, properties-narrowing and flag-cache TTL instance+ # settings, cold-cache here but TTL-cached per worker in production. +1 for the organization's flag_evaluations mode+ # lookup in the HogQL database build.+ with self.assertNumQueries(19):response = self.client.get("/api/person/?limit=10").json()self.assertEqual(len(response["results"]), 9)returned_ids += [x["distinct_ids"][0] for x in response["results"]]diff --git a/posthog/hogql/ACCESS_CONTROL.md b/posthog/hogql/ACCESS_CONTROL.mdindex d597c5875249..5bd904e984ec 100644--- a/posthog/hogql/ACCESS_CONTROL.md+++ b/posthog/hogql/ACCESS_CONTROL.md@@ -191,6 +191,7 @@ They're masked when the query is printed to ClickHouse SQL, so a restricted read- **Explicit reads** (`properties.email`) are replaced with `NULL`, and the resolver refuses to back them with a materialized column — `ClickHousePropertyResolver` in `posthog/hogql/transforms/clickhouse_property_resolution.py`.- **Whole-blob reads** (`SELECT properties` or `SELECT *`) have the restricted keys stripped from the returned JSON via `JSONDropKeysPool(...)` — `ClickHousePrinter._maybe_apply_json_drop_keys()` in `posthog/hogql/printer/clickhouse.py`.- **Unrestricted key reads** (`properties.$browser`, `JSONExtractString(properties, '$browser')`) read the stored String blob without `JSONDropKeysPool`, because dropping keys cannot change their value. A read keeps the mask when its key overlaps a restricted key by dot segment (`customer` against a restricted `customer.ssn`), or when the key is an integer position — `key_read_overlaps_restriction()` in `posthog/hogql/restricted_properties.py`.+- **Narrowed exports.** A subquery whose `properties` column is only read by key exports a map of those keys instead of the blob — `posthog/hogql/transforms/exported_properties.py`. On the legacy table each map value extracts its key from the masked blob. On native event JSON the resolver declines a key that overlaps a restriction, so the subquery exports the masked document — `stored_key_json_text()` in `posthog/hogql/transforms/clickhouse_property_resolution.py`.Group restrictions retain their group type index, so a same-named property on another group type stays readable. The masking also applies to the Postgres-backed `system.groups.group_properties` field.diff --git a/posthog/hogql/printer/utils.py b/posthog/hogql/printer/utils.pyindex 579f964fa97c..3a1b95662b63 100644--- a/posthog/hogql/printer/utils.py+++ b/posthog/hogql/printer/utils.py@@ -335,11 +335,17 @@ def prepare_ast_for_printing(context.modifiers.propertyGroupsMode = PropertyGroupsMode.OPTIMIZEDif context.modifiers.optimizeProjections:- with context.timings.measure("projection_pushdown"):- node = pushdown_projections(node, context)- # Pushdown mutates SelectQueryType.columns, staling cached CTE tables. Drop them so a- # wrongly pruned column fails loudly at compile time instead of emitting broken SQL.- context.cte_database_table_cache.clear()+ from posthog.models.instance_setting import ( # noqa: PLC0415 -- keeps the printer importable without Django+ get_instance_setting,+ )++ if get_instance_setting("HOGQL_PROJECTION_OPTIMIZATION_ENABLED"):+ narrow_properties = dialect == "clickhouse" and get_instance_setting("HOGQL_PROPERTIES_NARROWING_ENABLED")+ with context.timings.measure("projection_pushdown"):+ node = pushdown_projections(node, context, narrow_properties=narrow_properties)+ # Pushdown mutates SelectQueryType.columns, staling cached CTE tables. Drop them so a+ # wrongly pruned column fails loudly at compile time instead of emitting broken SQL.+ context.cte_database_table_cache.clear()if dialect == "clickhouse" and context.modifiers.materializeRepeatedCTEs:from django.conf import (diff --git a/posthog/hogql/transforms/clickhouse_property_resolution.py b/posthog/hogql/transforms/clickhouse_property_resolution.pyindex 716d9ab463d8..4955e9f6155e 100644--- a/posthog/hogql/transforms/clickhouse_property_resolution.py+++ b/posthog/hogql/transforms/clickhouse_property_resolution.py@@ -222,6 +222,21 @@ def resolve_json_subcolumn_source()+def stored_key_json_text(field_type: ast.FieldType, key: str, context: HogQLContext) -> ast.Expr | None:+ """The JSON text of `<blob>.<key>` on the JSON events table, read from the key's own stored column and serialized as+ `JSONExtractRaw(document, key)` returns it from the rebuilt document. None when no single column holds what the+ document shows: the document leaves `$feature_flags` out in favour of its `$feature/<key>` pairs, and a key that a+ restriction touches reads the masked document as text. The read is the `PropertyAccess` that lowering builds, so+ it gets no property type and the value pass resolves it like any other key read.+ """+ if key == "$feature_flags" or property_path_overlaps_restriction(key, field_type.table_type, context):+ return None+ read = ast.PropertyAccess(+ expr=ast.Field(chain=[field_type.name], type=field_type), keys=[key], type=ast.StringType(nullable=True)+ )+ return ast.Call(name="toJSONString", args=[read], type=ast.StringType(nullable=True))++def resolve_property_group_source(field_type: ast.FieldType, property_name: str, context: HogQLContext) -> MaterializedPropertySource | None:diff --git a/posthog/hogql/transforms/exported_properties.py b/posthog/hogql/transforms/exported_properties.pynew file mode 100644index 000000000000..263b115f1ae3--- /dev/null+++ b/posthog/hogql/transforms/exported_properties.py@@ -0,0 +1,317 @@+"""Exported properties narrowing.++A select that exports the events `properties` blob to readers that only read keys of it exports a map of those keys+instead, under the same name:++ SELECT properties.email FROM (SELECT properties FROM events)+ ->+ SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties['email']), ''), 'null'), '^"|"$', '')+ FROM (SELECT map('email', JSONExtractRaw(events.properties, 'email')) AS properties FROM events)++On the native JSON events table the exported blob is a document rebuilt per row from every stored property, and the+reader parses that text again; the map reads one subcolumn per key instead. The column keeps its name, so a select that+passes it through to its own reader needs no change, and a reader looks a key up in the map where it extracted it from+the blob before.++The map holds each key's JSON text as `JSONExtractRaw(blob, key)` returned it. On the native table the property+resolution pass supplies that text from the key's own stored column (`stored_key_json_text`), or declines when no single+column holds what the rebuilt document shows, and a declined key keeps the blob. Neither is a property read: a property+read gets the property-definition type and, on the legacy table, a materialized column, which reads an empty string as+NULL and decodes escapes. The reader's own extract stays around the lookup, so a missing key still reads as `''` and+then NULL.++A blob stays a blob when any reader reads it whole (`SELECT properties`, `JSONHas(properties, k)`, a DISTINCT, GROUP BY+or ORDER BY on the column, a bare alias reference), when the resolution pass declines a key a native reader asks for, or+when any column that carries the same value cannot be narrowed: the columns a blob passes through and the branches of a+UNION ALL form one group, decided together. The decision runs on the reads `ProjectionPushdownOptimizer` collected,+after it pruned.+"""++from typing import TYPE_CHECKING++from posthog.hogql import ast+from posthog.hogql.compilation import compilation_budget+from posthog.hogql.context import HogQLContext+from posthog.hogql.database.models import StringJSONDatabaseField+from posthog.hogql.database.schema.events import EventsTable+from posthog.hogql.database.schema.util.null_sentinels import null_if_sentinels+from posthog.hogql.visitor import clone_expr++if TYPE_CHECKING:+ from posthog.hogql.transforms.projection_pushdown import ProjectionPushdownOptimizer, SubqueryNode++_MAP_TYPE_NAME = "Map(String, Nullable(String))"+++def narrow_exported_properties(optimizer: "ProjectionPushdownOptimizer", context: HogQLContext) -> None:+ """Mutates the optimized query in place."""+ _ExportedPropertiesNarrower(optimizer, context).run()+++def document_key_extract(document: ast.Expr, chain: list[str | int]) -> ast.Expr:+ """The key's raw JSON text with the quotes trimmed, as the printer renders a property read of a text document."""+ extract = ast.Call(+ name="JSONExtractRaw",+ args=[document, *(ast.Constant(value=key) for key in chain)],+ type=ast.StringType(nullable=False),+ )+ return ast.Call(+ name="replaceRegexpAll",+ args=[null_if_sentinels(extract), ast.Constant(value='^"|"$'), ast.Constant(value="")],+ type=ast.StringType(nullable=True),+ )+++def _map_type() -> ast.MapType:+ return ast.MapType(key_type=ast.StringType(nullable=False), value_type=ast.StringType(nullable=True))+++def _key_json_text(source: ast.Field, key: str, context: HogQLContext) -> ast.Expr | None:+ """The key's JSON text as `JSONExtractRaw(blob, key)` returns it, read from the events table (module docstring), or+ None when the native table has no single stored column for it."""+ if context.uses_new_events_schema():+ # Deferred: the resolution pass imports the printer, which imports the pruner and so this module.+ from posthog.hogql.transforms.clickhouse_property_resolution import stored_key_json_text # noqa: PLC0415++ field_type = source.type+ assert isinstance(field_type, ast.FieldType)+ return stored_key_json_text(field_type, key, context)+ return ast.Call(+ name="JSONExtractRaw",+ args=[clone_expr(source, clear_types=False), ast.Constant(value=key)],+ type=ast.StringType(nullable=False),+ )+++def _column_name(expr: ast.Expr) -> str | None:+ if isinstance(expr, ast.Alias):+ return expr.alias+ if isinstance(expr, ast.Field) and expr.chain:+ return str(expr.chain[-1])+ return None+++def _unwrap_aliases(expr: ast.Expr) -> ast.Expr:+ while isinstance(expr, ast.Alias):+ expr = expr.expr+ return expr+++class _ExportedPropertiesNarrower:+ def __init__(self, optimizer: "ProjectionPushdownOptimizer", context: HogQLContext):+ self.optimizer = optimizer+ self.context = context+ self._registered = {id(node) for node in optimizer.subquery_map.values()}+ self._exprs_by_name: dict[int, dict[str, ast.Expr]] = {}++ def run(self) -> None:+ for group in self._column_groups():+ if not all(self._allows_map(node, name) for node, name in group):+ continue+ for node, name in group:+ self._rewrite_reads(node, name)+ if isinstance(node, ast.SelectQuery) and self._events_source(node, name) is not None:+ self._rewrite_export(node, name)++ def _column_groups(self) -> list[list[tuple["SubqueryNode", str]]]:+ """The columns that carry one value between them, so one decision covers them all: a column and the child+ column it passes through, a set query's column and the same column of each branch."""+ nodes = list({id(node): node for node in self.optimizer.subquery_map.values()}.values())+ seen: set[tuple[int, str]] = set()+ groups: list[list[tuple[SubqueryNode, str]]] = []+ budget = compilation_budget.get()+ for node in nodes:+ if budget is not None:+ budget.check_time()+ for name in self._column_names(node):+ if (id(node), name) in seen:+ continue+ seen.add((id(node), name))+ group: list[tuple[SubqueryNode, str]] = []+ queue: list[tuple[SubqueryNode, str]] = [(node, name)]+ while queue:+ member = queue.pop()+ group.append(member)+ for neighbor in self._neighbors(*member):+ key = (id(neighbor[0]), neighbor[1])+ if key not in seen:+ seen.add(key)+ queue.append(neighbor)+ groups.append(group)+ return groups++ def _neighbors(self, node: "SubqueryNode", name: str) -> list[tuple["SubqueryNode", str]]:+ neighbors: list[tuple[SubqueryNode, str]] = []+ if isinstance(node, ast.SelectSetQuery):+ for query in self.optimizer.union_queries(node):+ branch_name = self.optimizer.branch_column_name(node, query, name)+ if branch_name is not None:+ neighbors.append((query, branch_name))+ else:+ passthrough = self._passthrough(node, name)+ if passthrough is not None:+ neighbors.append(passthrough)+ neighbors.extend(self.optimizer.passthrough_parents.get((id(node), name), {}).values())+ set_query = self.optimizer.union_of.get(id(node))+ if set_query is not None:+ set_name = self.optimizer.set_column_name(set_query, node, name)+ if set_name is not None:+ neighbors.append((set_query, set_name))+ return neighbors++ def _column_names(self, node: "SubqueryNode") -> list[str]:+ if isinstance(node, ast.SelectSetQuery):+ return self._column_names(self.optimizer.union_queries(node)[0])+ return [name for name in (_column_name(expr) for expr in node.select) if name is not None]++ def _select_expr(self, node: ast.SelectQuery, name: str) -> ast.Expr | None:+ exprs = self._exprs_by_name.get(id(node))+ if exprs is None:+ exprs = {}+ for expr in node.select:+ column = _column_name(expr)+ if column is not None and column not in exprs:+ exprs[column] = expr+ self._exprs_by_name[id(node)] = exprs+ return exprs.get(name)++ def _source_field(self, node: ast.SelectQuery, name: str) -> ast.Field | None:+ expr = self._select_expr(node, name)+ source = _unwrap_aliases(expr) if expr is not None else None+ if isinstance(source, ast.Field) and isinstance(source.type, ast.FieldType):+ return source+ return None++ def _passthrough(self, node: ast.SelectQuery, name: str) -> tuple["SubqueryNode", str] | None:+ source = self._source_field(node, name)+ if source is None:+ return None+ assert isinstance(source.type, ast.FieldType)+ child = self.optimizer.child_for(source.type.table_type)+ return (child, source.type.name) if child is not None else None++ def _events_source(self, node: ast.SelectQuery, name: str) -> ast.Field | None:+ source = self._source_field(node, name)+ if source is None or not isinstance(source.type, ast.FieldType) or not self._is_events_properties(source.type):+ return None+ return source++ def _allows_map(self, node: "SubqueryNode", name: str) -> bool:+ """Whether this member of a column group can carry the map instead of the blob.++ A select may only pass a `properties` column along, or export it from the events table, when nothing in it+ consumes the column's value: no reader of the column itself (a bare field read, or a clause naming it by alias+ or by position), no DISTINCT, GROUP BY or HAVING, which compare or group by every column or the named ones, no+ set operation but UNION ALL, and the select is neither the query's output nor a recursive CTE. These are the+ ways a ClickHouse select can use a column's value without a field reference, so the check lives here in one+ place rather than in the pruner's read tracking.+ """+ reads = self.optimizer.reads.get((id(node), name))+ if reads is not None and reads.whole:+ return False+ set_query = self.optimizer.union_of.get(id(node))+ if set_query is not None and not self._lines_up_with_set(set_query, node, name):+ return False+ if isinstance(node, ast.SelectSetQuery):+ return all(set_node.set_operator.startswith("UNION ALL") for set_node in node.subsequent_select_queries)+ if id(node) in self.optimizer.roots or id(node) in self.optimizer.recursive_cte_exprs:+ return False+ if id(node) not in self._registered and id(node) not in self.optimizer.union_of:+ # A select inside an expression: its columns are its output+ return False+ if node.distinct or node.group_by or node.group_by_mode is not None or node.having is not None:+ return False+ if id(node) in self.optimizer.strict_set_branches:+ return False+ expr = self._select_expr(node, name)+ if expr is None:+ return False+ if any(expr is column for column in self.optimizer.positional_columns(node)):+ return False+ if expr.type is not None and id(expr.type) in self.optimizer.alias_referenced_types:+ return False+ if self._passthrough(node, name) is not None:+ return True+ source = self._events_source(node, name)+ if source is None:+ return False+ keys = reads.keys if reads is not None else set()+ return all(_key_json_text(source, key, self.context) is not None for key in keys)++ def _lines_up_with_set(self, set_query: ast.SelectSetQuery, node: "SubqueryNode", name: str) -> bool:+ """Whether the branch column sits at the position of the set query's column it feeds. A set operation pairs+ columns by position, so a map exported from a column matched by name at another position would meet the blob+ or another column there."""+ set_name = self.optimizer.set_column_name(set_query, node, name)+ if set_name is None:+ return False+ return self.optimizer.select_position(node, name) == self.optimizer.select_position(set_query, set_name)++ def _is_events_properties(self, field_type: ast.FieldType) -> bool:+ table_type = field_type.table_type+ if isinstance(table_type, ast.TableAliasType):+ table_type = table_type.table_type+ if not isinstance(table_type, ast.TableType) or not isinstance(table_type.table, EventsTable):+ return False+ database_field = field_type.resolve_database_field(self.context)+ return isinstance(database_field, StringJSONDatabaseField) and database_field.name == "properties"++ def _rewrite_reads(self, node: "SubqueryNode", name: str) -> None:+ reads = self.optimizer.reads.get((id(node), name))+ if reads is not None:+ for alias_node, field_node in reads.sites.values():+ self._rewrite_read(alias_node, field_node)++ def _rewrite_read(self, alias_node: ast.Alias, field_node: ast.Field) -> None:+ property_type = field_node.type+ assert isinstance(property_type, ast.PropertyType)+ base_chain = field_node.chain[: len(field_node.chain) - len(property_type.chain)]+ base = ast.Field(chain=list(base_chain) or [property_type.field_type.name], type=property_type.field_type)+ lookup = ast.ArrayAccess(+ array=base, property=ast.Constant(value=property_type.chain[0]), type=ast.StringType(nullable=True)+ )+ new_expr = document_key_extract(lookup, list(property_type.chain[1:]))+ alias_node.expr = new_expr+ if isinstance(alias_node.type, ast.FieldAliasType):+ # The resolver shares this type object with the select scope's column and alias maps+ alias_node.type.type = new_expr.type or ast.UnknownType()++ def _rewrite_export(self, node: ast.SelectQuery, name: str) -> None:+ expr = self._select_expr(node, name)+ source = self._events_source(node, name)+ assert expr is not None and source is not None+ reads = self.optimizer.reads.get((id(node), name))+ map_expr = self._map_expr(source, sorted(reads.keys) if reads is not None else [])+ innermost: ast.Alias | None = None+ current = expr+ while isinstance(current, ast.Alias):+ innermost = current+ current = current.expr+ if innermost is None:+ replacement = ast.Alias(+ alias=name,+ expr=map_expr,+ hidden=True,+ type=ast.FieldAliasType(alias=name, type=_map_type()),+ )+ node.select[node.select.index(expr)] = replacement+ self._exprs_by_name.pop(id(node), None)+ if isinstance(node.type, ast.SelectQueryType):+ node.type.columns[name] = replacement.type or ast.UnknownType()+ return+ innermost.expr = map_expr+ if isinstance(innermost.type, ast.FieldAliasType):+ innermost.type.type = _map_type()++ def _map_expr(self, source: ast.Field, keys: list[str]) -> ast.Expr:+ args: list[ast.Expr] = []+ for key in keys:+ text = _key_json_text(source, key, self.context)+ assert text is not None+ args.append(ast.Constant(value=key))+ args.append(text)+ map_call = ast.Call(name="map", args=args, type=_map_type())+ if keys:+ return map_call+ # Nobody reads a key; `map()` alone has the type Nothing, which a pass-through cannot carry+ return ast.Call(name="accurateCast", args=[map_call, ast.Constant(value=_MAP_TYPE_NAME)], type=_map_type())diff --git a/posthog/hogql/transforms/projection_pushdown.py b/posthog/hogql/transforms/projection_pushdown.pyindex d8cff24f46e0..08a14919821d 100644--- a/posthog/hogql/transforms/projection_pushdown.py+++ b/posthog/hogql/transforms/projection_pushdown.py@@ -1,21 +1,44 @@from collections import defaultdict+from dataclasses import fieldfrom typing import castfrom posthog.hogql import astfrom posthog.hogql.base import _T_ASTfrom posthog.hogql.context import HogQLContext+from posthog.hogql.transforms.exported_properties import narrow_exported_propertiesfrom posthog.hogql.visitor import TraversingVisitor+from posthog.dataclasses import frozen++SubqueryNode = ast.SelectQuery | ast.SelectSetQuery++# Demand flows top-down, but a CTE's reader can be visited after the CTE, so one pass can leave a column's readers+# incomplete. Each pass adds what the last one learned; the walk stops when a pass adds nothing. A query that still+# changes after this many passes is pruned with what was collected and not narrowed.+_MAX_COLLECTION_PASSES = 50+++@frozen(frozen=False)+class ColumnReads:+ """How the readers of one subquery column use it. `whole` is set by any read of the column itself; `keys` are the+ first keys of `column.<key>` reads, with the reads themselves in `sites`, by the hidden alias the resolver wraps+ every field in."""++ whole: bool = False+ keys: set[str] = field(default_factory=set)+ sites: dict[int, tuple[ast.Alias, ast.Field]] = field(default_factory=dict)+class ProjectionPushdownOptimizer(TraversingVisitor):"""- Top-down projection pushdown optimizer that prunes unused asterisk-expanded columns from subqueries.+ Top-down projection pushdown optimizer that prunes unused asterisk-expanded columns from subqueries, and records+ how each subquery column is read so `exported_properties` can export only the keys read of a properties blob.Algorithm Overview:──────────────────- This optimizer makes two top-down passes through the query tree: the first only collects- demands, the second prunes with the complete demand set. A CTE can be consumed by nodes- visited after it — notably sibling UNION branches — so pruning must wait for the full walk.+ This optimizer makes several top-down passes through the query tree: the collecting passes only gather demands,+ repeated until a pass learns nothing new, and the final pass prunes with the complete demand set. A CTE can be+ consumed by nodes visited after it — notably sibling UNION branches — so pruning must wait for the full walk.Each pass runs these phases per query:@@ -23,21 +46,68 @@ class ProjectionPushdownOptimizer(TraversingVisitor):Phase 2 - Collect: Gather column demands from WHERE/GROUP BY/ORDER BY/etcPhase 3 - Propagate: For demanded columns, visit their source to propagate to child queriesPhase 4 - Recurse: Visit child subqueries (repeat phases 1-4)- Phase 5 - Prune: Remove unreferenced asterisk columns from this query (second pass only)+ Phase 5 - Prune: Remove unreferenced asterisk columns from this query (final pass only)++ A column that a query exports as it is (`SELECT properties FROM sub`) is not a read of it: how the column is used+ is its own readers' business, so phase 3 hands their reads down to the child column instead. The reads recorded+ here are the ones made through a field; whether a select's own shape (DISTINCT, a set operation, an ordinal, an+ alias reference) consumes a column's value is `exported_properties`' question, answered from the select itself."""- def __init__(self):+ def __init__(self, context: HogQLContext | None = None):super().__init__()+ self.context = contextself.demands: dict[int, set[str]] = defaultdict(set)- self.subquery_map: dict[int, ast.SelectQuery | ast.SelectSetQuery] = {}- # True during the first pass; gates pruning to the second (see class docstring)+ self.reads: dict[tuple[int, str], ColumnReads] = defaultdict(ColumnReads)+ self.subquery_map: dict[int, SubqueryNode] = {}+ # The select columns that export a child column as it is, by the child column they export+ self.passthrough_parents: dict[tuple[int, str], dict[tuple[int, str], tuple[SubqueryNode, str]]] = defaultdict(+ dict+ )+ self.roots: set[int] = set()+ self.recursive_cte_exprs: set[int] = set()+ self.converged: bool = False+ self.union_of: dict[int, ast.SelectSetQuery] = {}+ # Column names and first positions by select, dropped when the select's columns are pruned+ self._names_by_select: dict[int, list[str | None]] = {}+ self._positions_by_select: dict[int, dict[str, int]] = {}+ # Branches of a set query that compares rows (anything but UNION ALL)+ self.strict_set_branches: set[int] = set()+ # Select leaves of a set query one of whose leaves keeps every column, so the branches keep lining up+ self.aligned_set_leaves: set[int] = set()+ # True during the collecting passes; gates pruning to the final one (see class docstring)self.collecting: bool = False+ self._alias_stack: list[ast.Alias] = []+ # Alias types a clause named directly (`ORDER BY p`), which reads the aliased column's value itself+ self.alias_referenced_types: set[int] = set()- def optimize(self, node: ast.SelectQuery | ast.SelectSetQuery) -> ast.SelectQuery | ast.SelectSetQuery:+ def optimize(self, node: SubqueryNode) -> SubqueryNode:+ self.roots = self._root_ids(node)self.collecting = True- self.visit(node)+ signature: object = None+ for _ in range(_MAX_COLLECTION_PASSES):+ self.visit(node)+ new_signature = self._demand_signature()+ if new_signature == signature:+ self.converged = True+ break+ signature = new_signatureself.collecting = False- return cast(ast.SelectQuery | ast.SelectSetQuery, self.visit(node))+ return cast(SubqueryNode, self.visit(node))++ def _root_ids(self, node: SubqueryNode) -> set[int]:+ if isinstance(node, ast.SelectSetQuery):+ ids = {id(node)}+ for query in self.union_queries(node):+ ids |= self._root_ids(query)+ return ids+ return {id(node)}++ def _demand_signature(self) -> object:+ return (+ tuple(sorted((key, tuple(sorted(names))) for key, names in self.demands.items())),+ tuple(sorted((key, reads.whole, tuple(sorted(reads.keys))) for key, reads in self.reads.items())),+ )def visit_select_query(self, node: ast.SelectQuery) -> ast.SelectQuery:# Phase 1: Register subqueries and CTEs for demand tracking@@ -46,9 +116,16 @@ class ProjectionPushdownOptimizer(TraversingVisitor):if node.select_from:self._register_subqueries(node.select_from)- # Phase 2: Collect column demands from query clauses+ # Phase 2: Collect column demands from query clauses. An asterisk column is demanded by the parent (phase 3),+ # unless every column has to stay, when the child must keep it too.+ keeps_all_columns = self.keeps_all_columns(node)for expr in node.select:- if not self._is_from_asterisk(expr):+ if self._is_from_asterisk(expr) and not keeps_all_columns:+ continue+ passthrough = self._passthrough_source(expr)+ if passthrough is not None:+ self._demand_passthrough(node, expr, passthrough)+ elif not self._is_from_asterisk(expr):self.visit(expr)if node.where:@@ -60,6 +137,8 @@ class ProjectionPushdownOptimizer(TraversingVisitor):self.visit(expr)if node.having:self.visit(node.having)+ if node.qualify:+ self.visit(node.qualify)if node.order_by:for expr in node.order_by:self.visit(expr)@@ -67,6 +146,14 @@ class ProjectionPushdownOptimizer(TraversingVisitor):self.visit(node.limit)if node.offset:self.visit(node.offset)+ if node.limit_by:+ self.visit(node.limit_by)+ if node.array_join_list:+ for expr in node.array_join_list:+ self.visit(expr)+ if node.window_exprs:+ for window_expr in node.window_exprs.values():+ self.visit(window_expr)if node.select_from:self._collect_join_constraint_column_demands(node.select_from)@@ -80,7 +167,7 @@ class ProjectionPushdownOptimizer(TraversingVisitor):if node.select_from:self.visit(node.select_from)- # Phase 5: Prune unreferenced asterisk columns from this query and CTEs (second pass only)+ # Phase 5: Prune unreferenced asterisk columns from this query and CTEs (final pass only)if not self.collecting:self._prune_columns(node)if node.ctes:@@ -96,6 +183,15 @@ class ProjectionPushdownOptimizer(TraversingVisitor):1. Propagate parent demands to all branches uniformly2. Visit each branch to apply pruning"""+ strict = any(not set_node.set_operator.startswith("UNION ALL") for set_node in node.subsequent_select_queries)+ for query in self.union_queries(node):+ self.union_of[id(query)] = node+ if strict:+ self.strict_set_branches.add(id(query))+ leaves = self._select_leaves(node)+ if any(self.keeps_all_columns(leaf) for leaf in leaves):+ self.aligned_set_leaves.update(id(leaf) for leaf in leaves)+# Propagate parent demands to all branches uniformlyself._propagate_demands_to_union_branches(node)@@ -106,6 +202,59 @@ class ProjectionPushdownOptimizer(TraversingVisitor):return node+ def union_queries(self, node: ast.SelectSetQuery) -> list[SubqueryNode]:+ return [node.initial_select_query, *(set_node.select_query for set_node in node.subsequent_select_queries)]++ def _select_leaves(self, node: SubqueryNode) -> list[ast.SelectQuery]:+ if isinstance(node, ast.SelectSetQuery):+ return [leaf for query in self.union_queries(node) for leaf in self._select_leaves(query)]+ return [node]++ def _select_names(self, node: SubqueryNode) -> list[str | None]:+ if isinstance(node, ast.SelectSetQuery):+ return self._select_names(node.initial_select_query)+ names = self._names_by_select.get(id(node))+ if names is None:+ names = [self._get_column_name(expr) for expr in node.select]+ self._names_by_select[id(node)] = names+ return names++ def select_position(self, node: SubqueryNode, name: str) -> int | None:+ """The position of the first select column called `name`, the one a set operation pairs with a sibling's"""+ if isinstance(node, ast.SelectSetQuery):+ return self.select_position(node.initial_select_query, name)+ positions = self._positions_by_select.get(id(node))+ if positions is None:+ positions = {}+ for position, column in enumerate(self._select_names(node)):+ if column is not None and column not in positions:+ positions[column] = position+ self._positions_by_select[id(node)] = positions+ return positions.get(name)++ def branch_column_name(self, set_query: ast.SelectSetQuery, branch: SubqueryNode, name: str) -> str | None:+ """The branch column that feeds the set query's column `name`. The first branch names the result. A branch+ with a column of that name supplies it from that column: the branches of a `SELECT *` union over tables whose+ columns differ in order or count only line up once each keeps the same names, and queries rely on that. A+ branch without the name supplies the column at the same position, as the set operation itself does."""+ position = self.select_position(set_query, name)+ if position is None:+ return None+ if self.select_position(branch, name) is not None:+ return name+ branch_names = self._select_names(branch)+ return branch_names[position] if position < len(branch_names) else None++ def set_column_name(self, set_query: ast.SelectSetQuery, branch: SubqueryNode, branch_name: str) -> str | None:+ """The set query's column that the branch column `branch_name` feeds, the inverse of `branch_column_name`"""+ position = self.select_position(branch, branch_name)+ if position is None:+ return None+ if self.select_position(set_query, branch_name) is not None:+ return branch_name+ set_names = self._select_names(set_query)+ return set_names[position] if position < len(set_names) else None+def _is_from_asterisk(self, expr: ast.Expr) -> bool:"""Check if an expression was expanded from asterisk"""if isinstance(expr, ast.Alias):@@ -114,45 +263,164 @@ class ProjectionPushdownOptimizer(TraversingVisitor):return expr.from_asteriskreturn False+ def _passthrough_source(self, expr: ast.Expr) -> tuple[SubqueryNode, str] | None:+ """The child query and column name behind a select column that exports a subquery or CTE column as it is."""+ source = self._unwrap_aliases(expr)+ if not isinstance(source, ast.Field) or not isinstance(source.type, ast.FieldType):+ return None+ child = self.child_for(source.type.table_type)+ if child is None:+ return None+ return child, source.type.name++ def _unwrap_aliases(self, expr: ast.Expr) -> ast.Expr:+ while isinstance(expr, ast.Alias):+ expr = expr.expr+ return expr++ def compares_rows(self, node: ast.SelectQuery) -> bool:+ """Whether the select's rows are compared whole, so every column's value counts"""+ return node.distinct or id(node) in self.strict_set_branches or node.group_by_mode == "all"++ def positional_columns(self, node: ast.SelectQuery) -> list[ast.Expr]:+ """The select columns a bare integer in ORDER BY, GROUP BY or LIMIT BY names by position (ClickHouse resolves+ them positionally), so their values count and their positions must not move"""+ referencing: list[ast.Expr] = [order.expr for order in node.order_by or []]+ for group_expr in node.group_by or []:+ if isinstance(group_expr, ast.GroupingSet):+ referencing.extend(group_expr.exprs)+ else:+ referencing.append(group_expr)+ if node.limit_by:+ referencing.extend(node.limit_by.exprs)+ columns: list[ast.Expr] = []+ for expr in referencing:+ if isinstance(expr, ast.Constant) and isinstance(expr.value, int) and not isinstance(expr.value, bool):+ if 0 < expr.value <= len(node.select):+ columns.append(node.select[expr.value - 1])+ return columns++ def keeps_all_columns(self, node: ast.SelectQuery) -> bool:+ """Whether dropping a column would change the select's rows, move a position another clause names, or put the+ select out of line with a sibling branch that keeps its columns"""+ return self.compares_rows(node) or bool(self.positional_columns(node)) or id(node) in self.aligned_set_leaves++ def _demand_passthrough(self, node: ast.SelectQuery, expr: ast.Expr, passthrough: tuple[SubqueryNode, str]) -> None:+ child, child_name = passthrough+ self.demands[id(child)].add(child_name)+ col_name = self._get_column_name(expr)+ if col_name is not None:+ self.passthrough_parents[(id(child), child_name)][(id(node), col_name)] = (node, col_name)+def _propagate_demands_to_children(self, node: ast.SelectQuery) -> None:"""Propagate parent demands to child subqueries.When a parent query demands a column from us, we need to visit that column's- source expression to propagate the demand down to our child subqueries.+ source expression to propagate the demand down to our child subqueries. A column+ exported as it is hands the parent's reads of it down to the child column.- If there are no parent demands but we have asterisk columns,- this query defines the output. Visit all asterisk columns to preserve them.+ A query without parent demands is either the query's output, which needs every+ column as it is, or a subquery nobody reads a column of, which only has to keep+ its rows."""demanded_from_this = self.demands.get(id(node))if not demanded_from_this:- for expr in node.select:- if self._is_from_asterisk(expr):- self.visit(expr)+ if id(node) in self.roots or id(node) not in self._registered_ids():+ # The query's output, or a select inside an expression: every column is read as it is+ for expr in node.select:+ self._column_read_whole(node, expr)+ elif not self.collecting:+ # Only the final pass knows that nobody reads the column; a reader may still turn up while collecting+ self._keep_unread_subquery_rows(node)returnfor col_name in demanded_from_this:+ position = self.select_position(node, col_name)+ if position is None:+ continue+ expr = node.select[position]+ passthrough = self._passthrough_source(expr)+ if passthrough is None:+ self.visit(expr)+ continue+ child, child_name = passthrough+ self._demand_passthrough(node, expr, passthrough)+ parent_reads = self.reads.get((id(node), col_name))+ if parent_reads is not None:+ child_reads = self.reads[(id(child), child_name)]+ child_reads.whole = child_reads.whole or parent_reads.whole+ child_reads.keys.update(parent_reads.keys)++ def _keep_unread_subquery_rows(self, node: ast.SelectQuery) -> None:+ if self.keeps_all_columns(node) or id(node) in self.union_of:+ # Every asterisk column stays: a row comparison counts them, and set query branches must line upfor expr in node.select:- if self._get_column_name(expr) == col_name:+ if self._is_from_asterisk(expr):self.visit(expr)- break+ return+ # Nobody reads a column, so the asterisk columns would only be computed and dropped. An explicit column keeps+ # the rows; without one, the cheapest asterisk column does, and a JSON blob is the most expensive to compute.+ explicit_columns = [expr for expr in node.select if not self._is_from_asterisk(expr)]+ asterisk_columns = [expr for expr in node.select if self._is_from_asterisk(expr)]+ if explicit_columns:+ col_name = self._get_column_name(explicit_columns[0])+ if col_name is not None:+ self.demands[id(node)].add(col_name)+ return+ if not asterisk_columns:+ return+ keep = next((expr for expr in asterisk_columns if not self._is_json_blob(expr)), asterisk_columns[0])+ col_name = self._get_column_name(keep)+ if col_name is not None:+ self.demands[id(node)].add(col_name)+ passthrough = self._passthrough_source(keep)+ if passthrough is not None:+ self._demand_passthrough(node, keep, passthrough)+ else:+ self.visit(keep)++ def _registered_ids(self) -> set[int]:+ return {id(query) for query in self.subquery_map.values()}++ def _column_read_whole(self, node: ast.SelectQuery, expr: ast.Expr) -> None:+ """The column is the query's output, so the child column it passes through is read as it is"""+ passthrough = self._passthrough_source(expr)+ if passthrough is None:+ self.visit(expr)+ return+ child, child_name = passthrough+ self._demand_passthrough(node, expr, passthrough)+ self.reads[(id(child), child_name)].whole = True++ def _is_json_blob(self, expr: ast.Expr) -> bool:+ source = self._unwrap_aliases(expr)+ if not isinstance(source, ast.Field) or source.type is None or self.context is None:+ return False+ try:+ return isinstance(source.type.resolve_constant_type(self.context), ast.StringJSONType)+ except Exception:+ return Falsedef _propagate_demands_to_union_branches(self, node: ast.SelectSetQuery) -> None:- """- Propagate demands from parent to all UNION/INTERSECT/EXCEPT branches.-- Since all branches must have identical column structure, we propagate- the same demands to all of them.- """+ """Propagate demands from parent to all UNION/INTERSECT/EXCEPT branches, each to the branch column that feeds+ the demanded column (`branch_column_name`), so every branch keeps the same columns."""demanded_from_this = self.demands.get(id(node))if not demanded_from_this:return- all_queries = [node.initial_select_query] + [sn.select_query for sn in node.subsequent_select_queries]-- for query in all_queries:- if isinstance(query, ast.SelectQuery | ast.SelectSetQuery):- self.demands[id(query)].update(demanded_from_this)+ for query in self.union_queries(node):+ for col_name in demanded_from_this:+ branch_name = self.branch_column_name(node, query, col_name)+ if branch_name is None:+ continue+ self.demands[id(query)].add(branch_name)+ parent_reads = self.reads.get((id(node), col_name))+ if parent_reads is None:+ continue+ branch_reads = self.reads[(id(query), branch_name)]+ branch_reads.whole = branch_reads.whole or parent_reads.whole+ branch_reads.keys.update(parent_reads.keys)def _register_subqueries(self, from_clause: ast.JoinExpr) -> None:"""Register all subqueries in FROM clause before collecting demands"""@@ -175,7 +443,10 @@ class ProjectionPushdownOptimizer(TraversingVisitor):"""Map type to subquery node for demand tracking"""# Map both the type and the inner SelectQueryType if it's an aliasself.subquery_map[id(type_annotation)] = subquery- if isinstance(type_annotation, ast.SelectQueryAliasType) and type_annotation.select_query_type:+ if (+ isinstance(type_annotation, ast.SelectQueryAliasType | ast.SelectViewType)+ and type_annotation.select_query_type+ ):self.subquery_map[id(type_annotation.select_query_type)] = subquerydef _register_union_subquery(@@ -187,7 +458,7 @@ class ProjectionPushdownOptimizer(TraversingVisitor):self.subquery_map[id(type_annotation)] = union_query# Also register with the inner SelectSetQueryType for both aliased and non-aliased cases- if isinstance(type_annotation, ast.SelectQueryAliasType):+ if isinstance(type_annotation, ast.SelectQueryAliasType | ast.SelectViewType):# Aliased: register with the inner select_query_typeif type_annotation.select_query_type:self.subquery_map[id(type_annotation.select_query_type)] = union_query@@ -197,43 +468,76 @@ class ProjectionPushdownOptimizer(TraversingVisitor):if type_annotation.types:self.subquery_map[id(type_annotation.types[0])] = union_query- def _get_subquery(self, table_type: ast.Type) -> ast.SelectQuery | ast.SelectSetQuery | None:+ def _get_subquery(self, table_type: ast.Type) -> SubqueryNode | None:"""Retrieve subquery by type"""return self.subquery_map.get(id(table_type))+ def child_for(self, table_type: ast.Type) -> SubqueryNode | None:+ """The registered subquery, saved view body or CTE query a field's table type refers to, if any. The saved+ view type has to map as well: otherwise nothing ever reads the view body and the final pass prunes it."""+ if isinstance(+ table_type, ast.SelectQueryType | ast.SelectQueryAliasType | ast.SelectSetQueryType | ast.SelectViewType+ ):+ return self._get_subquery(table_type)+ if isinstance(table_type, ast.CTETableType | ast.CTETableAliasType):+ # For CTE types, get the underlying SelectQueryType and demand from it+ cte_select_query_type = (+ table_type.cte_table_type.select_query_type+ if isinstance(table_type, ast.CTETableAliasType)+ else table_type.select_query_type+ )+ return self._get_subquery(cte_select_query_type)+ return None++ def visit_alias(self, node: ast.Alias) -> None:+ self._alias_stack.append(node)+ try:+ super().visit_alias(node)+ finally:+ self._alias_stack.pop()+def visit_field(self, node: ast.Field) -> ast.Field:"""Record demand when field references subquery or CTE column"""field_type = node.type+ if isinstance(field_type, ast.FieldAliasType):+ self.alias_referenced_types.add(id(field_type))+ return node+# A property read (`col.some_key`) still demands its base blob column. Its node type is a# PropertyType wrapping the base FieldType, so unwrap to that base — otherwise the column is# never recorded as demanded and gets pruned from an asterisk-expanded subquery/CTE, leaving a# dangling reference that later blows up property lowering.+ property_type: ast.PropertyType | None = Noneif isinstance(field_type, ast.PropertyType):+ property_type = field_typefield_type = field_type.field_type# Handle both FieldType and ExpressionFieldTypeif not isinstance(field_type, ast.FieldType | ast.ExpressionFieldType):return node- table_type = field_type.table_type-- if isinstance(table_type, ast.SelectQueryType | ast.SelectQueryAliasType | ast.SelectSetQueryType):- subquery = self._get_subquery(table_type)- if subquery:- self.demands[id(subquery)].add(field_type.name)- elif isinstance(table_type, ast.CTETableType | ast.CTETableAliasType):- # For CTE types, get the underlying SelectQueryType and demand from it- cte_select_query_type = (- table_type.cte_table_type.select_query_type- if isinstance(table_type, ast.CTETableAliasType)- else table_type.select_query_type- )- cte_query = self._get_subquery(cte_select_query_type)- if cte_query:- self.demands[id(cte_query)].add(field_type.name)-+ child = self.child_for(field_type.table_type)+ if child is None:+ return node+ self.demands[id(child)].add(field_type.name)+ reads = self.reads[(id(child), field_type.name)]+ if self._is_key_read(property_type, node):+ assert property_type is not None+ reads.keys.add(cast(str, property_type.chain[0]))+ reads.sites[id(self._alias_stack[-1])] = (self._alias_stack[-1], node)+ else:+ reads.whole = Truereturn node+ def _is_key_read(self, property_type: ast.PropertyType | None, node: ast.Field) -> bool:+ """A `column.<key>` read the resolver wrapped in a hidden alias, which `exported_properties` can re-point.+ A read of the JSON text itself, or of a position, needs the blob."""+ if property_type is None or property_type.as_json or not property_type.chain:+ return False+ if not isinstance(property_type.chain[0], str):+ return False+ return bool(self._alias_stack) and self._alias_stack[-1].expr is node+def _collect_join_constraint_column_demands(self, from_clause: ast.JoinExpr) -> None:"""Collect demands from JOIN constraints"""if from_clause.constraint and from_clause.constraint.expr:@@ -246,30 +550,29 @@ class ProjectionPushdownOptimizer(TraversingVisitor):def _prune_columns(self, node: ast.SelectQuery) -> None:"""Prune asterisk-expanded columns that aren't demanded"""demanded = self.demands.get(id(node))- if not demanded:+ if not demanded or self.keeps_all_columns(node):returnpruned_select = []+ dropped_names: set[str] = set()for expr in node.select:if not self._is_from_asterisk(expr):# Keep explicitly written columnspruned_select.append(expr)- else:- # Keep demanded asterisk columns- col_name = self._get_column_name(expr)- if col_name and col_name in demanded:- pruned_select.append(expr)+ continue+ # Keep demanded asterisk columns+ col_name = self._get_column_name(expr)+ if col_name and col_name in demanded:+ pruned_select.append(expr)+ elif col_name:+ dropped_names.add(col_name)if pruned_select:- # Collect the names of asterisk columns we're about to drop- dropped_names: set[str] = set()- for expr in node.select:- if self._is_from_asterisk(expr) and expr not in pruned_select:- col_name = self._get_column_name(expr)- if col_name:- dropped_names.add(col_name)-+ # A name a kept column still exports stays a column of the select's type+ dropped_names -= {self._get_column_name(expr) for expr in pruned_select}node.select = pruned_select+ self._names_by_select.pop(id(node), None)+ self._positions_by_select.pop(id(node), None)# Remove dropped asterisk columns from SelectQueryType.columns.# Without this, stale LazyTableType references on pruned columns# leak through type traversal (e.g. CTETableType → SelectQueryType.columns)@@ -292,10 +595,13 @@ class ProjectionPushdownOptimizer(TraversingVisitor):# Map the CTE's SelectQueryType to its expr for demand trackingif cte.expr.type:self.subquery_map[id(cte.expr.type)] = cte.expr+ if cte.recursive:+ self.recursive_cte_exprs.add(id(cte.expr))def _visit_ctes(self, ctes: dict[str, ast.CTE]) -> None:- """Visit and optimize CTEs"""- for cte in ctes.values():+ """Visit and optimize CTEs. A CTE can only read the ones defined before it, so the last one goes first and+ its demands reach the earlier ones in the same pass."""+ for cte in reversed(list(ctes.values())):if isinstance(cte.expr, ast.SelectQuery | ast.SelectSetQuery):self.visit(cte.expr)@@ -306,9 +612,13 @@ class ProjectionPushdownOptimizer(TraversingVisitor):self._prune_columns(cte.expr)-def pushdown_projections(node: _T_AST, context: HogQLContext) -> _T_AST:- """Prune unused columns from asterisk expansions in subqueries"""+def pushdown_projections(node: _T_AST, context: HogQLContext, *, narrow_properties: bool = False) -> _T_AST:+ """Prune unused columns from asterisk expansions in subqueries. With `narrow_properties`, a subquery that exports+ the events `properties` blob exports only the keys its readers read (see `exported_properties`)."""if not isinstance(node, (ast.SelectQuery, ast.SelectSetQuery)):return node- optimizer = ProjectionPushdownOptimizer()- return cast(_T_AST, optimizer.optimize(node))+ optimizer = ProjectionPushdownOptimizer(context)+ result = optimizer.optimize(node)+ if narrow_properties and optimizer.converged:+ narrow_exported_properties(optimizer, context)+ return cast(_T_AST, result)diff --git a/posthog/hogql/transforms/test/test_exported_properties.py b/posthog/hogql/transforms/test/test_exported_properties.pynew file mode 100644index 000000000000..adef2ac01671--- /dev/null+++ b/posthog/hogql/transforms/test/test_exported_properties.py@@ -0,0 +1,412 @@+from posthog.test.base import (+ APIBaseTest,+ BaseTest,+ ClickhouseTestMixin,+ _create_event,+ flush_persons_and_events,+ materialized,+)++from parameterized import parameterized++from posthog.schema import HogQLQueryModifiers++from posthog.hogql.constants import HogQLDialect+from posthog.hogql.context import HogQLContext+from posthog.hogql.parser import parse_select+from posthog.hogql.printer.utils import prepare_and_print_ast+from posthog.hogql.property_access_types import RestrictedProperty+from posthog.hogql.query import execute_hogql_query++from posthog.clickhouse.client.escape import substitute_params+from posthog.models import PropertyDefinition+from posthog.models.instance_setting import override_instance_config++from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery+from products.event_definitions.backend.models.property_definition import PropertyType++STAR_EVENTS = "(SELECT * FROM events)"+EMAIL_MAP = "map('email', JSONExtractRaw(events.properties, 'email'))"+++def _modifiers(pruning: bool) -> HogQLQueryModifiers:+ return HogQLQueryModifiers(optimizeProjections=pruning)+++class TestExportedProperties(BaseTest):+ maxDiff = None++ def _print(+ self,+ select: str,+ *,+ native: bool = False,+ pruning: bool = True,+ restricted: set[RestrictedProperty] | None = None,+ dialect: HogQLDialect = "clickhouse",+ ) -> str:+ context = HogQLContext(+ team_id=self.team.pk,+ enable_select_queries=True,+ modifiers=_modifiers(pruning),+ use_new_events_schema=native,+ )+ if restricted is not None:+ context.restricted_properties = restricted+ query, _ = prepare_and_print_ast(parse_select(select), context, dialect)+ return substitute_params(query, context.values).replace(f"team_id, {self.team.pk})", "team_id, 420)")++ @parameterized.expand(+ [+ (+ "one_level",+ f"SELECT properties.email FROM {STAR_EVENTS}",+ f"{EMAIL_MAP} AS properties",+ "JSONExtractRaw(properties['email'])",+ ),+ (+ "passed_through_two_levels",+ f"SELECT properties.email FROM (SELECT * FROM {STAR_EVENTS})",+ f"SELECT properties AS properties FROM (SELECT {EMAIL_MAP} AS properties",+ "JSONExtractRaw(properties['email'])",+ ),+ (+ "aliased_passthrough",+ "SELECT p.email FROM (SELECT properties AS p FROM events)",+ f"{EMAIL_MAP} AS p",+ "JSONExtractRaw(p['email'])",+ ),+ (+ "cte",+ f"WITH e AS {STAR_EVENTS} SELECT properties.email FROM e",+ f"WITH e AS (SELECT {EMAIL_MAP} AS properties",+ "JSONExtractRaw(e.properties['email'])",+ ),+ (+ "union_branches",+ "SELECT properties.email FROM (SELECT properties FROM events UNION ALL SELECT properties FROM events)",+ f"{EMAIL_MAP} AS properties FROM events WHERE equals(events.team_id, 420) UNION ALL SELECT {EMAIL_MAP} AS properties",+ "JSONExtractRaw(properties['email'])",+ ),+ (+ "keys_read_at_two_levels",+ f"SELECT properties.email FROM (SELECT * FROM {STAR_EVENTS} WHERE properties.utm_source = 'g')",+ "map('email', JSONExtractRaw(events.properties, 'email'), 'utm_source', JSONExtractRaw(events.properties, 'utm_source')) AS properties",+ "JSONExtractRaw(properties['utm_source'])",+ ),+ (+ "nested_key",+ "SELECT properties.a.b FROM (SELECT properties FROM events)",+ "map('a', JSONExtractRaw(events.properties, 'a')) AS properties",+ "JSONExtractRaw(properties['a'], 'b')",+ ),+ (+ "left_join_on_the_subquery",+ f"SELECT e.properties.email, s.n FROM {STAR_EVENTS} AS e LEFT JOIN (SELECT event AS ev, count() AS n FROM events GROUP BY ev) AS s ON s.ev = e.event",+ f"{EMAIL_MAP} AS properties",+ "JSONExtractRaw(e.properties['email'])",+ ),+ (+ "read_inside_a_lambda",+ "SELECT arrayMap(x -> properties.email, [1]) FROM (SELECT properties FROM events)",+ f"{EMAIL_MAP} AS properties",+ "x -> replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties['email'])",+ ),+ (+ "unread_blob",+ "SELECT count() FROM (SELECT properties FROM events)",+ "accurateCast(map(), 'Map(String, Nullable(String))') AS properties",+ "count()",+ ),+ (+ "eight_chained_ctes",+ f"WITH c1 AS {STAR_EVENTS}, "+ + ", ".join(f"c{n} AS (SELECT * FROM c{n - 1})" for n in range(2, 9))+ + " SELECT properties.email FROM c8",+ f"WITH c1 AS (SELECT {EMAIL_MAP} AS properties",+ "JSONExtractRaw(c8.properties['email'])",+ ),+ ]+ )+ def test_exports_only_the_keys_read(self, _name: str, select: str, exported: str, read: str) -> None:+ printed = self._print(select)+ assert exported in printed, printed+ assert read in printed, printed+ assert "events.properties AS properties" not in printed, printed++ @parameterized.expand(+ [+ ("blob_and_key", "SELECT properties, properties.email FROM (SELECT properties FROM events)"),+ ("json_function_on_the_blob", "SELECT JSONHas(properties, 'x') FROM (SELECT properties FROM events)"),+ ("distinct_inner", "SELECT properties.email FROM (SELECT DISTINCT properties FROM events)"),+ (+ "grouped_inner",+ "SELECT properties.email FROM (SELECT properties, count() AS c FROM events GROUP BY properties)",+ ),+ ("inner_order_by_alias", "SELECT p.email FROM (SELECT properties AS p FROM events ORDER BY p)"),+ (+ "union_distinct",+ "SELECT properties.email FROM (SELECT properties FROM events UNION DISTINCT SELECT properties FROM events)",+ ),+ (+ "intersect",+ "SELECT properties.email FROM (SELECT properties FROM events INTERSECT SELECT properties FROM events)",+ ),+ (+ "distinct_asterisk_between_the_levels",+ "SELECT properties.email FROM (SELECT DISTINCT * FROM (SELECT properties FROM events))",+ ),+ (+ "union_branch_that_cannot_narrow",+ "SELECT properties.email FROM (SELECT properties FROM (SELECT properties FROM events) UNION ALL SELECT DISTINCT properties FROM events)",+ ),+ (+ "positional_order_by_on_a_passthrough",+ "SELECT properties.email FROM (SELECT properties, event FROM (SELECT properties, event FROM events) ORDER BY 1 LIMIT 1)",+ ),+ (+ "positional_order_by_at_the_events_level",+ "SELECT p.email FROM (SELECT properties AS p, event FROM events ORDER BY 1 LIMIT 1)",+ ),+ (+ "grouping_sets_ordinal_on_a_passthrough",+ "SELECT properties.email FROM (SELECT properties, event FROM (SELECT properties, event FROM events) GROUP BY GROUPING SETS ((1)))",+ ),+ (+ "group_by_all_on_a_passthrough",+ "SELECT properties.email FROM (SELECT properties, event FROM (SELECT properties, event FROM events) GROUP BY ALL)",+ ),+ (+ "second_alias_of_the_column_feeds_a_blob_branch",+ "SELECT x.a.email, x.e FROM (SELECT a, b.email AS e FROM (SELECT properties AS a, properties AS b FROM (SELECT properties FROM events)) "+ "UNION ALL SELECT DISTINCT properties AS a, '' AS e FROM events) AS x",+ ),+ (+ "union_branch_column_matched_by_name_at_another_position",+ "SELECT a.email FROM (SELECT properties AS a, properties AS b FROM events UNION ALL SELECT properties AS b, properties AS a FROM events)",+ ),+ ("query_output", "SELECT properties FROM (SELECT properties FROM events)"),+ ("read_by_position", f"SELECT properties.arr[1] FROM {STAR_EVENTS}"),+ ("native_flag_map", "SELECT properties.$feature_flags FROM (SELECT properties FROM events)", True),+ ]+ )+ def test_keeps_the_blob_when_a_reader_needs_it(self, _name: str, select: str, native: bool = False) -> None:+ printed = self._print(select, native=native)+ assert "map(" not in printed, printed++ def test_saved_view_exports_only_the_keys_read(self) -> None:+ DataWarehouseSavedQuery.objects.create(+ team=self.team,+ name="pageviews",+ query={"query": "SELECT event, properties FROM events WHERE event = '$pageview'"},+ columns={+ "event": {"clickhouse": "String", "hogql": "StringDatabaseField"},+ "properties": {"clickhouse": "String", "hogql": "StringJSONDatabaseField"},+ },+ )+ printed = self._print("SELECT properties.email FROM pageviews")+ assert f"{EMAIL_MAP} AS properties" in printed, printed+ assert "JSONExtractRaw(pageviews.properties['email'])" in printed, printed++ @parameterized.expand([("legacy", False), ("native", True)])+ def test_global_kill_switch_overrides_query_modifier(self, _name: str, native: bool) -> None:+ select = f"SELECT properties.email FROM {STAR_EVENTS}"+ unoptimized = self._print(select, native=native, pruning=False)+ with override_instance_config("HOGQL_PROJECTION_OPTIMIZATION_ENABLED", False):+ assert self._print(select, native=native) == unoptimized+ with override_instance_config("HOGQL_PROJECTION_OPTIMIZATION_ENABLED", True):+ optimized = self._print(select, native=native)+ assert "map('email', " in optimized, optimized+ assert "events.distinct_id AS distinct_id" not in optimized, optimized+ assert self._print(select, native=native, pruning=False) == unoptimized++ @parameterized.expand([("legacy", False), ("native", True)])+ def test_narrowing_kill_switch_keeps_the_blob_and_the_pruning(self, _name: str, native: bool) -> None:+ select = f"SELECT properties.email FROM {STAR_EVENTS}"+ with override_instance_config("HOGQL_PROPERTIES_NARROWING_ENABLED", False):+ printed = self._print(select, native=native)+ assert "map('email', " not in printed, printed+ assert "events.distinct_id AS distinct_id" not in printed, printed+ with override_instance_config("HOGQL_PROPERTIES_NARROWING_ENABLED", True):+ assert "map('email', " in self._print(select, native=native)++ def test_native_export_reads_one_subcolumn(self) -> None:+ printed = self._print(f"SELECT properties.utm_source FROM {STAR_EVENTS}", native=True)+ assert "map('utm_source', " in printed, printed+ assert "toJSONString(events.properties.utm_source)" in printed, printed+ assert "JSONExtractKeysAndValuesRaw" not in printed, printed++ @parameterized.expand(+ [+ ("legacy_key", False, "email", "properties.email", True),+ ("legacy_nested", False, "obj.secret", "properties.obj.visible", True),+ ("native_key", True, "email", "properties.email", False),+ ("native_nested", True, "obj.secret", "properties.obj.visible", False),+ ]+ )+ def test_restricted_path_reads_the_masked_blob(+ self, _name: str, native: bool, restricted: str, read: str, narrowed: bool+ ) -> None:+ printed = self._print(+ f"SELECT {read}, properties.utm_source FROM {STAR_EVENTS}",+ native=native,+ restricted={RestrictedProperty(name=restricted, property_type=PropertyDefinition.Type.EVENT)},+ )+ first_key = read.split(".")[1]+ assert ("map(" in printed) is narrowed, printed+ assert "JSONDropKeysPool(" in printed, printed+ assert f"JSONExtractRaw(events.properties, '{first_key}')" not in printed, printed++ def test_string_parser_still_gets_the_text(self) -> None:+ PropertyDefinition.objects.create(+ team=self.team, name="amount", property_type=PropertyType.Numeric, type=PropertyDefinition.Type.EVENT+ )+ printed = self._print(f"SELECT sum(toFloatOrZero(properties.amount)) FROM {STAR_EVENTS}")+ assert "toFloat64OrZero(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties['amount'])" in printed, printed+ assert "'Float64'" not in printed, printed++ def test_hogql_dialect_keeps_the_blob(self) -> None:+ printed = self._print(f"SELECT properties.email FROM {STAR_EVENTS}", dialect="hogql")+ assert "map(" not in printed, printed+++class TestExportedPropertiesExecution(ClickhouseTestMixin, APIBaseTest):+ def setUp(self) -> None:+ super().setUp()+ PropertyDefinition.objects.create(+ team=self.team, name="amount", property_type=PropertyType.Numeric, type=PropertyDefinition.Type.EVENT+ )+ rows = [+ ("2024-01-02T10:00:00", "$pageview", {"utm_source": "google", "amount": 2, "email": "[email protected]"}),+ ("2024-01-02T11:00:00", "$pageview", {"utm_source": "", "amount": "7.5", "email": None}),+ ("2024-01-03T10:00:00", "$pageview", {"amount": None, "nested": {"a": {"b": "deep"}}}),+ ("2024-01-03T11:00:00", "signup", {"utm_source": "bing", "n": 1.50, "t": True}),+ ("2024-01-03T12:00:00", "signup", {"utm_source": 'say "hi"\\now', "email": "[email protected]"}),+ (+ "2024-01-03T13:00:00",+ "signup",+ {"$feature/test": "true", "$feature_flags": {"test": "true"}, "$active_feature_flags": ["test"]},+ ),+ ("2023-12-31T23:59:59", "$pageview", {"utm_source": "google", "amount": 100, "a.b": "dotted"}),+ ("2024-01-04T10:00:00", "$pageview", {}),+ ]+ for index, (timestamp, event, properties) in enumerate(rows):+ _create_event(+ team=self.team, event=event, distinct_id=f"u{index}", timestamp=timestamp, properties=properties+ )+ flush_persons_and_events()++ def _assert_same_rows(self, select: str) -> None:+ narrowed = execute_hogql_query(select, team=self.team, modifiers=_modifiers(True)).results+ plain = execute_hogql_query(select, team=self.team, modifiers=_modifiers(False)).results+ assert narrowed == plain, f"narrowed={narrowed} plain={plain}"+ assert narrowed, "the query must match some rows for the comparison to mean anything"++ @parameterized.expand(+ [+ (+ "keys_and_filter",+ f"SELECT properties.email, event FROM {STAR_EVENTS} WHERE properties.utm_source = 'google' ORDER BY 1, 2",+ ),+ (+ "counts_and_key_reads",+ f"SELECT count(), countIf(coalesce(properties.utm_source, '') != ''), uniq(person_id) FROM {STAR_EVENTS} "+ "WHERE event = '$pageview' AND timestamp >= '2024-01-01'",+ ),+ (+ "grouped_key_read",+ f"SELECT properties.utm_source AS src, count() FROM {STAR_EVENTS} WHERE timestamp >= '2024-01-01' GROUP BY src ORDER BY src",+ ),+ (+ "string_parser_on_numeric_property",+ f"SELECT sum(toFloatOrZero(properties.amount)), arraySort(groupArray(properties.amount)) FROM {STAR_EVENTS} WHERE event = '$pageview'",+ ),+ (+ "escaped_string_value",+ f"SELECT properties.utm_source FROM {STAR_EVENTS} WHERE event = 'signup' ORDER BY 1",+ ),+ (+ "empty_null_number_and_bool_values",+ f"SELECT properties.utm_source, properties.email, properties.n, properties.t FROM {STAR_EVENTS} ORDER BY 1, 2, 3, 4",+ ),+ (+ "nested_and_dotted_keys",+ f"SELECT properties.nested.a.b, properties.`a.b` FROM {STAR_EVENTS} WHERE event = '$pageview' ORDER BY 1, 2",+ ),+ (+ "passed_through_two_levels",+ f"SELECT properties.email FROM (SELECT * FROM {STAR_EVENTS} WHERE properties.utm_source = 'google') ORDER BY 1",+ ),+ (+ "aliased_passthrough",+ "SELECT p.email FROM (SELECT properties AS p FROM events) ORDER BY 1",+ ),+ (+ "cte",+ f"WITH e AS {STAR_EVENTS} SELECT properties.email FROM e ORDER BY 1",+ ),+ (+ "union_branches",+ "SELECT properties.email FROM (SELECT properties FROM events UNION ALL SELECT properties FROM events) ORDER BY 1",+ ),+ (+ "left_join_on_the_subquery",+ f"SELECT e.properties.email, s.n FROM {STAR_EVENTS} AS e LEFT JOIN (SELECT event AS ev, count() AS n FROM events GROUP BY ev) AS s ON s.ev = e.event ORDER BY 1, 2",+ ),+ (+ "unread_blob",+ "SELECT count() FROM (SELECT properties FROM events)",+ ),+ (+ "rebuilt_flag_reads",+ f"SELECT properties.`$feature/test`, properties.$active_feature_flags FROM {STAR_EVENTS} WHERE event = 'signup' ORDER BY 1, 2",+ ),+ (+ "flag_map_read",+ f"SELECT count() FROM {STAR_EVENTS} WHERE properties.$feature_flags IS NULL",+ ),+ (+ "enclosing_query_over_the_narrowed_one",+ f"SELECT src, count() FROM (SELECT properties.utm_source AS src FROM {STAR_EVENTS} WHERE event = '$pageview') "+ "WHERE src != '' GROUP BY src ORDER BY src",+ ),+ (+ "filled_rows_read_as_missing",+ "SELECT d, event, properties.email FROM (SELECT toDate(timestamp) AS d, event, properties FROM events ORDER BY d WITH FILL) "+ "WHERE event = '$pageview' ORDER BY d, event, 3",+ ),+ ]+ )+ def test_narrowing_returns_the_same_rows(self, _name: str, select: str) -> None:+ self._assert_same_rows(select)++ @parameterized.expand(+ [+ (+ "string_empty_value",+ "utm_source",+ False,+ None,+ f"SELECT count() FROM {STAR_EVENTS} WHERE properties.utm_source = ''",+ ),+ (+ "nullable_string_escapes",+ "utm_source",+ True,+ None,+ f"SELECT properties.utm_source FROM {STAR_EVENTS} WHERE event = 'signup' ORDER BY 1",+ ),+ (+ "float_under_a_string_parser",+ "amount",+ False,+ "Float64",+ f"SELECT sum(toFloatOrZero(properties.amount)), arraySort(groupArray(properties.amount)) FROM {STAR_EVENTS} WHERE event = '$pageview'",+ ),+ ]+ )+ def test_narrowing_ignores_a_materialized_column(+ self, _name: str, property_name: str, is_nullable: bool, column_type: str | None, select: str+ ) -> None:+ with materialized("events", property_name, is_nullable=is_nullable, column_type=column_type):+ self._assert_same_rows(select)diff --git a/posthog/hogql/transforms/test/test_projection_pushdown.py b/posthog/hogql/transforms/test/test_projection_pushdown.pyindex 758300578433..3fc90cbf0122 100644--- a/posthog/hogql/transforms/test/test_projection_pushdown.py+++ b/posthog/hogql/transforms/test/test_projection_pushdown.py@@ -1,5 +1,5 @@import pytest-from posthog.test.base import BaseTest+from posthog.test.base import APIBaseTest, BaseTest, ClickhouseTestMixin, _create_event, flush_persons_and_eventsfrom parameterized import parameterized@@ -9,8 +9,11 @@ from posthog.hogql import astfrom posthog.hogql.context import HogQLContextfrom posthog.hogql.parser import parse_selectfrom posthog.hogql.printer import prepare_ast_for_printing+from posthog.hogql.query import execute_hogql_queryfrom posthog.hogql.transforms.projection_pushdown import pushdown_projections+from posthog.models.instance_setting import override_instance_config+class TestProjectionPushdown(BaseTest):maxDiff = None@@ -118,6 +121,101 @@ class TestProjectionPushdown(BaseTest):assert optimized.to_hogql() == self.snapshot+ @parameterized.expand(+ [+ ("asterisk", "SELECT count() FROM (SELECT * FROM events) AS sub", ["uuid"]),+ ("explicit_and_asterisk", "SELECT count() FROM (SELECT *, 1 AS one FROM events) AS sub", ["one"]),+ ("distinct_asterisk", "SELECT count() FROM (SELECT DISTINCT * FROM events) AS sub", None),+ ]+ )+ def test_unread_subquery_keeps_only_what_holds_its_rows(self, _name, query, expected_columns):+ optimized = self._optimize(query)+ inner_query = optimized.select_from.table+ column_names = [self._col_name(col) for col in inner_query.select]+ if expected_columns is None:+ assert len(column_names) > 1+ else:+ assert column_names == expected_columns++ @parameterized.expand(+ [+ ("distinct", "SELECT event FROM (SELECT DISTINCT * FROM events) AS sub"),+ ("union_distinct", "SELECT event FROM (SELECT * FROM events UNION DISTINCT SELECT * FROM events) AS sub"),+ ("intersect", "SELECT event FROM (SELECT * FROM events INTERSECT SELECT * FROM events) AS sub"),+ (+ "group_by_all",+ "SELECT count() FROM (SELECT * FROM (SELECT distinct_id, properties FROM events) GROUP BY ALL) AS sub",+ ),+ (+ "positional_group_by",+ "SELECT count() FROM (SELECT * FROM (SELECT distinct_id, properties FROM events) GROUP BY 1, 2) AS sub",+ ),+ (+ "grouping_sets_ordinals",+ "SELECT count() FROM (SELECT * FROM (SELECT distinct_id, properties FROM events) GROUP BY GROUPING SETS ((1, 2))) AS sub",+ ),+ ]+ )+ def test_row_comparison_keeps_every_column(self, _name, query):+ optimized = self._optimize(query)+ inner_query = optimized.select_from.table+ if isinstance(inner_query, ast.SelectSetQuery):+ inner_query = inner_query.initial_select_query+ assert len(inner_query.select) > 1++ def test_set_query_branches_keep_lining_up(self):+ optimized = self._optimize(+ "SELECT event FROM (SELECT DISTINCT * FROM events UNION ALL SELECT * FROM events) AS sub"+ )+ set_query = optimized.select_from.table+ first = set_query.initial_select_query.select+ second = set_query.subsequent_select_queries[0].select_query.select+ assert len(first) == len(second) > 1++ @parameterized.expand(+ [+ (+ "same_names_in_another_order",+ "SELECT a, count() FROM (SELECT * FROM (SELECT event AS a, distinct_id AS b FROM events) "+ "UNION ALL SELECT * FROM (SELECT distinct_id AS b, event AS a FROM events)) AS sub GROUP BY a",+ [["a"], ["a"]],+ ),+ (+ "explicit_column_after_asterisks_of_different_counts",+ "SELECT region, sum(n) FROM (SELECT *, 'US' AS region FROM (SELECT 1 AS n, event AS a FROM events) "+ "UNION ALL SELECT *, 'EU' AS region FROM (SELECT distinct_id AS b, uuid AS c, 1 AS n FROM events)) "+ "AS sub GROUP BY region",+ [["n", "region"], ["n", "region"]],+ ),+ (+ "name_past_the_end_of_a_shorter_branch",+ "SELECT c FROM (SELECT * FROM (SELECT event AS a, distinct_id AS b, uuid AS c FROM events) "+ "UNION ALL SELECT * FROM (SELECT uuid AS c, event AS a FROM events)) AS sub",+ [["c"], ["c"]],+ ),+ (+ "branch_without_the_name_keeps_the_position",+ "SELECT a FROM (SELECT * FROM (SELECT event AS a, distinct_id AS x FROM events) "+ "UNION ALL SELECT * FROM (SELECT event AS b, distinct_id AS y FROM events)) AS sub",+ [["a"], ["b"]],+ ),+ ]+ )+ def test_set_query_branches_keep_the_same_columns(self, _name, query, expected):+ set_query = self._optimize(query).select_from.table+ branches = [set_query.initial_select_query, *(s.select_query for s in set_query.subsequent_select_queries)]+ assert [[self._col_name(col) for col in branch.select] for branch in branches] == expected++ def test_cte_union_branches_keep_the_same_columns(self):+ optimized = self._optimize(+ "WITH sources AS (SELECT * FROM (SELECT event AS a, distinct_id AS b FROM events) "+ "UNION ALL SELECT * FROM (SELECT distinct_id AS b, event AS a FROM events)) "+ "SELECT a, count() FROM sources GROUP BY a ORDER BY 2 DESC"+ )+ set_query = optimized.ctes["sources"].expr+ branches = [set_query.initial_select_query, *(s.select_query for s in set_query.subsequent_select_queries)]+ assert [[self._col_name(col) for col in branch.select] for branch in branches] == [["a"], ["a"]]+@pytest.mark.usefixtures("unittest_snapshot")def test_prewhere_clause(self):"""PREWHERE clause columns should be preserved"""@@ -631,3 +729,73 @@ class TestProjectionPushdown(BaseTest):assert isinstance(inner, ast.SelectQuery)column_names = {self._col_name(col) for col in inner.select}assert column_names == {"event", "properties"}, f"pruner dropped the property base column: got {column_names}"+++class TestProjectionPushdownExecution(ClickhouseTestMixin, APIBaseTest):+ def setUp(self) -> None:+ super().setUp()+ _create_event(+ team=self.team,+ event="$pageview",+ distinct_id="u0",+ timestamp="2024-01-02T10:00:00",+ properties={"plan": "free"},+ )+ _create_event(+ team=self.team,+ event="signup",+ distinct_id="u1",+ timestamp="2024-01-02T11:00:00",+ properties={"plan": "paid"},+ )+ flush_persons_and_events()++ @parameterized.expand(+ [+ (+ "same_names_in_another_order",+ "SELECT a, count() FROM (SELECT * FROM (SELECT event AS a, distinct_id AS b FROM events) "+ "UNION ALL SELECT * FROM (SELECT distinct_id AS b, event AS a FROM events)) AS sub GROUP BY a ORDER BY a",+ [("$pageview", 2), ("signup", 2)],+ ),+ (+ "explicit_column_after_asterisks_of_different_counts",+ "SELECT region, sum(n) FROM (SELECT *, 'US' AS region FROM (SELECT 1 AS n, event AS a FROM events) "+ "UNION ALL SELECT *, 'EU' AS region FROM (SELECT distinct_id AS b, uuid AS c, 1 AS n FROM events)) "+ "AS sub GROUP BY region ORDER BY region",+ [("EU", 2), ("US", 2)],+ ),+ (+ "name_past_the_end_of_a_shorter_branch",+ "SELECT c FROM (SELECT * FROM (SELECT distinct_id AS a, 1 AS b, event AS c FROM events) "+ "UNION ALL SELECT * FROM (SELECT event AS c, distinct_id AS a FROM events)) AS sub ORDER BY c",+ [("$pageview",), ("$pageview",), ("signup",), ("signup",)],+ ),+ (+ "branch_without_the_name_keeps_the_position",+ "SELECT a FROM (SELECT * FROM (SELECT event AS a, distinct_id AS x FROM events) "+ "UNION ALL SELECT * FROM (SELECT event AS b, distinct_id AS y FROM events)) AS sub ORDER BY a",+ [("$pageview",), ("$pageview",), ("signup",), ("signup",)],+ ),+ (+ "cte_union",+ "WITH sources AS (SELECT * FROM (SELECT event AS a, distinct_id AS b FROM events) "+ "UNION ALL SELECT * FROM (SELECT distinct_id AS b, event AS a FROM events)) "+ "SELECT a, count() FROM sources GROUP BY a ORDER BY a",+ [("$pageview", 2), ("signup", 2)],+ ),+ (+ "properties_through_both_branches",+ "SELECT properties.plan AS plan, count() FROM (SELECT * FROM (SELECT event AS a, properties FROM events) "+ "UNION ALL SELECT * FROM (SELECT event AS a, properties FROM events)) AS sub GROUP BY plan ORDER BY plan",+ [("free", 2), ("paid", 2)],+ ),+ ]+ )+ def test_set_query_branches_return_the_named_columns(self, _name: str, query: str, expected: list[tuple]) -> None:+ for narrowing in (True, False):+ with override_instance_config("HOGQL_PROPERTIES_NARROWING_ENABLED", narrowing):+ response = execute_hogql_query(+ query, team=self.team, modifiers=HogQLQueryModifiers(optimizeProjections=True)+ )+ assert response.results == expected, f"narrowing={narrowing}: {response.results}"diff --git a/posthog/settings/dynamic_settings.py b/posthog/settings/dynamic_settings.pyindex 8f1307e28d8e..a888f7ceea4b 100644--- a/posthog/settings/dynamic_settings.py+++ b/posthog/settings/dynamic_settings.py@@ -20,6 +20,20 @@ CONSTANCE_CONFIG = {"Number of weeks recording performance events will be kept before removing them (for all projects). Storing performance events for a shorter timeframe can help reduce Clickhouse disk usage.",int,),+ "HOGQL_PROJECTION_OPTIMIZATION_ENABLED": (+ get_from_env("HOGQL_PROJECTION_OPTIMIZATION_ENABLED", True, type_cast=str_to_bool),+ "Whether HogQL prunes unused subquery columns and narrows exported event properties. "+ "Disable to skip both optimizations for every project, even when optimizeProjections is true. "+ "Changes reach workers within about a minute and affect newly compiled queries only.",+ bool,+ ),+ "HOGQL_PROPERTIES_NARROWING_ENABLED": (+ get_from_env("HOGQL_PROPERTIES_NARROWING_ENABLED", True, type_cast=str_to_bool),+ "Whether a HogQL subquery exports only the keys read of the events properties blob. "+ "Disable to export the whole blob again while unused subquery columns are still pruned. "+ "Changes reach workers within about a minute and affect newly compiled queries only.",+ bool,+ ),"HOGQL_SHARED_INSIGHT_DATABASE_ENABLED": (get_from_env("HOGQL_SHARED_INSIGHT_DATABASE_ENABLED", True, type_cast=str_to_bool),"Whether insight query runners share one HogQL database across execution, response printing, "@@ -422,6 +436,8 @@ SETTINGS_ALLOWING_API_OVERRIDE = ("GROWTH_ICP_REENRICH_DAILY_CAP","GROWTH_RESCORE_WEBHOOK_SECRET","HOGQL_DEFERRED_REVENUE_VIEWS_ENABLED",+ "HOGQL_PROJECTION_OPTIMIZATION_ENABLED",+ "HOGQL_PROPERTIES_NARROWING_ENABLED","HOGQL_SHARED_INSIGHT_DATABASE_ENABLED","HOGQL_TEAM_FLAG_CACHE_TTL_SECONDS","RECORDINGS_PERFORMANCE_EVENTS_TTL_WEEKS",diff --git a/products/signals/backend/test/test_signal_queries.py b/products/signals/backend/test/test_signal_queries.pyindex 3d662e44637d..42c5c3fe6a60 100644--- a/products/signals/backend/test/test_signal_queries.py+++ b/products/signals/backend/test/test_signal_queries.py@@ -156,8 +156,10 @@ class TestFetchSourceProductsForReports(_SignalEmbeddingsTestBase):self._emit_version(document_id="d3", report_id="rA", source_product="errors", inserted_at=self.base)self._emit_version(document_id="d4", report_id="rB", source_product="surveys", inserted_at=self.base)+ # +1 each for the projection kill-switch and properties-narrowing instance settings the HogQL+ # compile reads, cold-cache here but TTL-cached per worker in production.with (- self.assertNumQueries(FuzzyInt(0, 1)),+ self.assertNumQueries(FuzzyInt(1, 3)),patch("products.signals.backend.signal_metadata.execute_hogql_query", wraps=execute_hogql_query) as query,):result = fetch_source_products_for_reports(