Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(hogql): revert "export only the read keys of properties from subqueries" (#128804)

Sandy Spiceredc8c1089e3d
diff --git a/docs/published/handbook/engineering/databases/hogql-python.md b/docs/published/handbook/engineering/databases/hogql-python.md
index f190b39d9f11..f1ba08e65906 100644
--- a/docs/published/handbook/engineering/databases/hogql-python.md
+++ b/docs/published/handbook/engineering/databases/hogql-python.md
@@ -59,13 +59,6 @@ Circular dependencies fail with a query error. Soft saves skip column inference
Query preparation shares a 60-second cooperative deadline across parsing, resolution, transformations, and printing.
AST visitors check the deadline while walking; native parser calls are checked when they return, rather than interrupted mid-call.
-## Disabling projection optimization
-
-Set `HOGQL_PROJECTION_OPTIMIZATION_ENABLED` to `false` in Django admin's instance settings to disable unused-column pruning and exported event-property narrowing for every project.
-The setting defaults to `true` and overrides the `optimizeProjections` query modifier when disabled.
-Workers pick up the change within about a minute; it affects newly compiled queries, not running queries or cached results.
-Disabling these optimizations can increase query time and memory use. Set the instance setting back to `true` to restore the normal per-query behavior.
-
## Pattern matching during query preparation
Expressions inside HogQL placeholders execute in the Python HogVM.
diff --git a/posthog/api/test/test_event.py b/posthog/api/test/test_event.py
index d13044f3be96..28f07c1122b6 100644
--- a/posthog/api/test/test_event.py
+++ b/posthog/api/test/test_event.py
@@ -114,10 +114,10 @@ class TestEvents(ClickhouseTestMixin, APIBaseTest):
# personhog, not Postgres, so it's not in this count. Was 16 before passing team=team
# into get_restricted_properties_for_team, which lets is_property_access_control_enabled
# skip its per-call Team+organization lookup. +1 for the saved-expressions fetch in the
- # HogQL database build. +1 each for the flag-cache TTL and projection kill-switch instance
- # settings, cold-cache here but TTL-cached per worker in production. +1 for the
- # organization's flag_evaluations mode lookup in the HogQL database build.
- with self.assertNumQueries(17):
+ # HogQL database build. +1 for the flag-cache TTL instance setting, cold-cache here but
+ # TTL-cached per worker in production. +1 for the organization's flag_evaluations mode lookup in
+ # the HogQL database build.
+ with self.assertNumQueries(16):
response = self.client.get(f"/api/projects/{self.team.id}/events/?event=event_name").json()
assert response["results"][0]["event"] == "event_name"
@@ -152,10 +152,10 @@ class TestEvents(ClickhouseTestMixin, APIBaseTest):
# Group-type-mapping is read via personhog, not Postgres, so it's not in this count.
# Was 24 before passing team=team into get_restricted_properties_for_team, which lets
# is_property_access_control_enabled skip its per-call Team+organization lookup.
- # +1 for the saved-expressions fetch in the HogQL database build. +1 each for the flag-cache
- # TTL and projection kill-switch instance settings, cold-cache here but TTL-cached per worker
- # in production. +1 for the organization's flag_evaluations mode lookup in the HogQL database build.
- expected_queries = 24 if settings.CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA else 25
+ # +1 for the saved-expressions fetch in the HogQL database build. +1 for the flag-cache
+ # TTL instance setting, cold-cache here but TTL-cached per worker in production. +1 for
+ # the organization's flag_evaluations mode lookup in the HogQL database build.
+ expected_queries = 23 if settings.CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA else 24
with self.assertNumQueries(expected_queries):
response = self.client.get(
diff --git a/posthog/api/test/test_person.py b/posthog/api/test/test_person.py
index e6f24efdc9e0..58073ec7a2c1 100644
--- a/posthog/api/test/test_person.py
+++ b/posthog/api/test/test_person.py
@@ -2264,10 +2264,10 @@ class TestPerson(ClickhouseTestMixin, APIBaseTest, QueryMatchingTest):
# The property-access-control feature check reuses the request's already-loaded team,
# so listing persons no longer pays a per-request Team lookup (was 16). +1 for the
# saved-expressions fetch in the HogQL database build. +1 each for the shared-database
- # kill-switch, projection kill-switch and flag-cache TTL instance settings, cold-cache here
- # but TTL-cached per worker in production. +1 for the organization's flag_evaluations mode
- # lookup in the HogQL database build.
- with self.assertNumQueries(18):
+ # kill-switch and flag-cache TTL instance settings, cold-cache here but TTL-cached per
+ # worker in production. +1 for the organization's flag_evaluations mode lookup in the HogQL
+ # database build.
+ with self.assertNumQueries(17):
response = self.client.get("/api/person/?limit=10").json()
self.assertEqual(len(response["results"]), 9)
returned_ids += [x["distinct_ids"][0] for x in response["results"]]
diff --git a/posthog/hogql/ACCESS_CONTROL.md b/posthog/hogql/ACCESS_CONTROL.md
index 5bd904e984ec..d597c5875249 100644
--- a/posthog/hogql/ACCESS_CONTROL.md
+++ b/posthog/hogql/ACCESS_CONTROL.md
@@ -191,7 +191,6 @@ They're masked when the query is printed to ClickHouse SQL, so a restricted read
- **Explicit reads** (`properties.email`) are replaced with `NULL`, and the resolver refuses to back them with a materialized column — `ClickHousePropertyResolver` in `posthog/hogql/transforms/clickhouse_property_resolution.py`.
- **Whole-blob reads** (`SELECT properties` or `SELECT *`) have the restricted keys stripped from the returned JSON via `JSONDropKeysPool(...)` — `ClickHousePrinter._maybe_apply_json_drop_keys()` in `posthog/hogql/printer/clickhouse.py`.
- **Unrestricted key reads** (`properties.$browser`, `JSONExtractString(properties, '$browser')`) read the stored String blob without `JSONDropKeysPool`, because dropping keys cannot change their value. A read keeps the mask when its key overlaps a restricted key by dot segment (`customer` against a restricted `customer.ssn`), or when the key is an integer position — `key_read_overlaps_restriction()` in `posthog/hogql/restricted_properties.py`.
-- **Narrowed exports.** A subquery whose `properties` column is only read by key exports a map of those keys instead of the blob — `posthog/hogql/transforms/exported_properties.py`. On the legacy table each map value extracts its key from the masked blob. On native event JSON the resolver declines a key that overlaps a restriction, so the subquery exports the masked document — `stored_key_json_text()` in `posthog/hogql/transforms/clickhouse_property_resolution.py`.
Group restrictions retain their group type index, so a same-named property on another group type stays readable. The masking also applies to the Postgres-backed `system.groups.group_properties` field.
diff --git a/posthog/hogql/printer/utils.py b/posthog/hogql/printer/utils.py
index cdd2adc2b1ac..579f964fa97c 100644
--- a/posthog/hogql/printer/utils.py
+++ b/posthog/hogql/printer/utils.py
@@ -335,16 +335,11 @@ def prepare_ast_for_printing(
context.modifiers.propertyGroupsMode = PropertyGroupsMode.OPTIMIZED
if context.modifiers.optimizeProjections:
- from posthog.models.instance_setting import ( # noqa: PLC0415 -- keeps the printer importable without Django
- get_instance_setting,
- )
-
- if get_instance_setting("HOGQL_PROJECTION_OPTIMIZATION_ENABLED"):
- with context.timings.measure("projection_pushdown"):
- node = pushdown_projections(node, context, narrow_properties=dialect == "clickhouse")
- # Pushdown mutates SelectQueryType.columns, staling cached CTE tables. Drop them so a
- # wrongly pruned column fails loudly at compile time instead of emitting broken SQL.
- context.cte_database_table_cache.clear()
+ with context.timings.measure("projection_pushdown"):
+ node = pushdown_projections(node, context)
+ # Pushdown mutates SelectQueryType.columns, staling cached CTE tables. Drop them so a
+ # wrongly pruned column fails loudly at compile time instead of emitting broken SQL.
+ context.cte_database_table_cache.clear()
if dialect == "clickhouse" and context.modifiers.materializeRepeatedCTEs:
from django.conf import (
diff --git a/posthog/hogql/transforms/clickhouse_property_resolution.py b/posthog/hogql/transforms/clickhouse_property_resolution.py
index 4955e9f6155e..716d9ab463d8 100644
--- a/posthog/hogql/transforms/clickhouse_property_resolution.py
+++ b/posthog/hogql/transforms/clickhouse_property_resolution.py
@@ -222,21 +222,6 @@ def resolve_json_subcolumn_source(
)
-def stored_key_json_text(field_type: ast.FieldType, key: str, context: HogQLContext) -> ast.Expr | None:
- """The JSON text of `<blob>.<key>` on the JSON events table, read from the key's own stored column and serialized as
- `JSONExtractRaw(document, key)` returns it from the rebuilt document. None when no single column holds what the
- document shows: the document leaves `$feature_flags` out in favour of its `$feature/<key>` pairs, and a key that a
- restriction touches reads the masked document as text. The read is the `PropertyAccess` that lowering builds, so
- it gets no property type and the value pass resolves it like any other key read.
- """
- if key == "$feature_flags" or property_path_overlaps_restriction(key, field_type.table_type, context):
- return None
- read = ast.PropertyAccess(
- expr=ast.Field(chain=[field_type.name], type=field_type), keys=[key], type=ast.StringType(nullable=True)
- )
- return ast.Call(name="toJSONString", args=[read], type=ast.StringType(nullable=True))
-
-
def resolve_property_group_source(
field_type: ast.FieldType, property_name: str, context: HogQLContext
) -> MaterializedPropertySource | None:
diff --git a/posthog/hogql/transforms/exported_properties.py b/posthog/hogql/transforms/exported_properties.py
deleted file mode 100644
index bd09722a9f3d..000000000000
--- a/posthog/hogql/transforms/exported_properties.py
+++ /dev/null
@@ -1,291 +0,0 @@
-"""Exported properties narrowing.
-
-A select that exports the events `properties` blob to readers that only read keys of it exports a map of those keys
-instead, under the same name:
-
- SELECT properties.email FROM (SELECT properties FROM events)
- ->
- SELECT replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties['email']), ''), 'null'), '^"|"$', '')
- FROM (SELECT map('email', JSONExtractRaw(events.properties, 'email')) AS properties FROM events)
-
-On the native JSON events table the exported blob is a document rebuilt per row from every stored property, and the
-reader parses that text again; the map reads one subcolumn per key instead. The column keeps its name, so a select that
-passes it through to its own reader needs no change, and a reader looks a key up in the map where it extracted it from
-the blob before.
-
-The map holds each key's JSON text as `JSONExtractRaw(blob, key)` returned it. On the native table the property
-resolution pass supplies that text from the key's own stored column (`stored_key_json_text`), or declines when no single
-column holds what the rebuilt document shows, and a declined key keeps the blob. Neither is a property read: a property
-read gets the property-definition type and, on the legacy table, a materialized column, which reads an empty string as
-NULL and decodes escapes. The reader's own extract stays around the lookup, so a missing key still reads as `''` and
-then NULL.
-
-A blob stays a blob when any reader reads it whole (`SELECT properties`, `JSONHas(properties, k)`, a DISTINCT, GROUP BY
-or ORDER BY on the column, a bare alias reference), when the resolution pass declines a key a native reader asks for, or
-when any column that carries the same value cannot be narrowed: the columns a blob passes through and the branches of a
-UNION ALL form one group, decided together. The decision runs on the reads `ProjectionPushdownOptimizer` collected,
-after it pruned.
-"""
-
-from typing import TYPE_CHECKING
-
-from posthog.hogql import ast
-from posthog.hogql.context import HogQLContext
-from posthog.hogql.database.models import StringJSONDatabaseField
-from posthog.hogql.database.schema.events import EventsTable
-from posthog.hogql.database.schema.util.null_sentinels import null_if_sentinels
-from posthog.hogql.visitor import clone_expr
-
-if TYPE_CHECKING:
- from posthog.hogql.transforms.projection_pushdown import ProjectionPushdownOptimizer, SubqueryNode
-
-_MAP_TYPE_NAME = "Map(String, Nullable(String))"
-
-
-def narrow_exported_properties(optimizer: "ProjectionPushdownOptimizer", context: HogQLContext) -> None:
- """Mutates the optimized query in place."""
- _ExportedPropertiesNarrower(optimizer, context).run()
-
-
-def document_key_extract(document: ast.Expr, chain: list[str | int]) -> ast.Expr:
- """The key's raw JSON text with the quotes trimmed, as the printer renders a property read of a text document."""
- extract = ast.Call(
- name="JSONExtractRaw",
- args=[document, *(ast.Constant(value=key) for key in chain)],
- type=ast.StringType(nullable=False),
- )
- return ast.Call(
- name="replaceRegexpAll",
- args=[null_if_sentinels(extract), ast.Constant(value='^"|"$'), ast.Constant(value="")],
- type=ast.StringType(nullable=True),
- )
-
-
-def _map_type() -> ast.MapType:
- return ast.MapType(key_type=ast.StringType(nullable=False), value_type=ast.StringType(nullable=True))
-
-
-def _key_json_text(source: ast.Field, key: str, context: HogQLContext) -> ast.Expr | None:
- """The key's JSON text as `JSONExtractRaw(blob, key)` returns it, read from the events table (module docstring), or
- None when the native table has no single stored column for it."""
- if context.uses_new_events_schema():
- # Deferred: the resolution pass imports the printer, which imports the pruner and so this module.
- from posthog.hogql.transforms.clickhouse_property_resolution import stored_key_json_text # noqa: PLC0415
-
- field_type = source.type
- assert isinstance(field_type, ast.FieldType)
- return stored_key_json_text(field_type, key, context)
- return ast.Call(
- name="JSONExtractRaw",
- args=[clone_expr(source, clear_types=False), ast.Constant(value=key)],
- type=ast.StringType(nullable=False),
- )
-
-
-def _column_name(expr: ast.Expr) -> str | None:
- if isinstance(expr, ast.Alias):
- return expr.alias
- if isinstance(expr, ast.Field) and expr.chain:
- return str(expr.chain[-1])
- return None
-
-
-def _unwrap_aliases(expr: ast.Expr) -> ast.Expr:
- while isinstance(expr, ast.Alias):
- expr = expr.expr
- return expr
-
-
-class _ExportedPropertiesNarrower:
- def __init__(self, optimizer: "ProjectionPushdownOptimizer", context: HogQLContext):
- self.optimizer = optimizer
- self.context = context
- self._registered = {id(node) for node in optimizer.subquery_map.values()}
-
- def run(self) -> None:
- for group in self._column_groups():
- if not all(self._allows_map(node, name) for node, name in group):
- continue
- for node, name in group:
- self._rewrite_reads(node, name)
- if isinstance(node, ast.SelectQuery) and self._events_source(node, name) is not None:
- self._rewrite_export(node, name)
-
- def _column_groups(self) -> list[list[tuple["SubqueryNode", str]]]:
- """The columns that carry one value between them, so one decision covers them all: a column and the child
- column it passes through, a set query's column and the same column of each branch."""
- nodes = list({id(node): node for node in self.optimizer.subquery_map.values()}.values())
- seen: set[tuple[int, str]] = set()
- groups: list[list[tuple[SubqueryNode, str]]] = []
- for node in nodes:
- for name in self._column_names(node):
- if (id(node), name) in seen:
- continue
- seen.add((id(node), name))
- group: list[tuple[SubqueryNode, str]] = []
- queue: list[tuple[SubqueryNode, str]] = [(node, name)]
- while queue:
- member = queue.pop()
- group.append(member)
- for neighbor in self._neighbors(*member):
- key = (id(neighbor[0]), neighbor[1])
- if key not in seen:
- seen.add(key)
- queue.append(neighbor)
- groups.append(group)
- return groups
-
- def _neighbors(self, node: "SubqueryNode", name: str) -> list[tuple["SubqueryNode", str]]:
- neighbors: list[tuple[SubqueryNode, str]] = []
- if isinstance(node, ast.SelectSetQuery):
- for query in self.optimizer.union_queries(node):
- branch_name = self.optimizer.branch_column_name(node, query, name)
- if branch_name is not None:
- neighbors.append((query, branch_name))
- else:
- passthrough = self._passthrough(node, name)
- if passthrough is not None:
- neighbors.append(passthrough)
- neighbors.extend(self.optimizer.passthrough_parents.get((id(node), name), {}).values())
- set_query = self.optimizer.union_of.get(id(node))
- if set_query is not None:
- set_name = self.optimizer.set_column_name(set_query, node, name)
- if set_name is not None:
- neighbors.append((set_query, set_name))
- return neighbors
-
- def _column_names(self, node: "SubqueryNode") -> list[str]:
- if isinstance(node, ast.SelectSetQuery):
- return self._column_names(self.optimizer.union_queries(node)[0])
- return [name for name in (_column_name(expr) for expr in node.select) if name is not None]
-
- def _select_expr(self, node: ast.SelectQuery, name: str) -> ast.Expr | None:
- return next((expr for expr in node.select if _column_name(expr) == name), None)
-
- def _source_field(self, node: ast.SelectQuery, name: str) -> ast.Field | None:
- expr = self._select_expr(node, name)
- source = _unwrap_aliases(expr) if expr is not None else None
- if isinstance(source, ast.Field) and isinstance(source.type, ast.FieldType):
- return source
- return None
-
- def _passthrough(self, node: ast.SelectQuery, name: str) -> tuple["SubqueryNode", str] | None:
- source = self._source_field(node, name)
- if source is None:
- return None
- assert isinstance(source.type, ast.FieldType)
- child = self.optimizer.child_for(source.type.table_type)
- return (child, source.type.name) if child is not None else None
-
- def _events_source(self, node: ast.SelectQuery, name: str) -> ast.Field | None:
- source = self._source_field(node, name)
- if source is None or not isinstance(source.type, ast.FieldType) or not self._is_events_properties(source.type):
- return None
- return source
-
- def _allows_map(self, node: "SubqueryNode", name: str) -> bool:
- """Whether this member of a column group can carry the map instead of the blob.
-
- A select may only pass a `properties` column along, or export it from the events table, when nothing in it
- consumes the column's value: no reader of the column itself (a bare field read, or a clause naming it by alias
- or by position), no DISTINCT, GROUP BY or HAVING, which compare or group by every column or the named ones, no
- set operation but UNION ALL, and the select is neither the query's output nor a recursive CTE. These are the
- ways a ClickHouse select can use a column's value without a field reference, so the check lives here in one
- place rather than in the pruner's read tracking.
- """
- reads = self.optimizer.reads.get((id(node), name))
- if reads is not None and reads.whole:
- return False
- if isinstance(node, ast.SelectSetQuery):
- return all(set_node.set_operator.startswith("UNION ALL") for set_node in node.subsequent_select_queries)
- if id(node) in self.optimizer.roots or id(node) in self.optimizer.recursive_cte_exprs:
- return False
- if id(node) not in self._registered and id(node) not in self.optimizer.union_of:
- # A select inside an expression: its columns are its output
- return False
- if node.distinct or node.group_by or node.group_by_mode is not None or node.having is not None:
- return False
- if id(node) in self.optimizer.strict_set_branches:
- return False
- expr = self._select_expr(node, name)
- if expr is None:
- return False
- if any(expr is column for column in self.optimizer.positional_columns(node)):
- return False
- if expr.type is not None and id(expr.type) in self.optimizer.alias_referenced_types:
- return False
- if self._passthrough(node, name) is not None:
- return True
- source = self._events_source(node, name)
- if source is None:
- return False
- keys = reads.keys if reads is not None else set()
- return all(_key_json_text(source, key, self.context) is not None for key in keys)
-
- def _is_events_properties(self, field_type: ast.FieldType) -> bool:
- table_type = field_type.table_type
- if isinstance(table_type, ast.TableAliasType):
- table_type = table_type.table_type
- if not isinstance(table_type, ast.TableType) or not isinstance(table_type.table, EventsTable):
- return False
- database_field = field_type.resolve_database_field(self.context)
- return isinstance(database_field, StringJSONDatabaseField) and database_field.name == "properties"
-
- def _rewrite_reads(self, node: "SubqueryNode", name: str) -> None:
- reads = self.optimizer.reads.get((id(node), name))
- if reads is not None:
- for alias_node, field_node in reads.sites.values():
- self._rewrite_read(alias_node, field_node)
-
- def _rewrite_read(self, alias_node: ast.Alias, field_node: ast.Field) -> None:
- property_type = field_node.type
- assert isinstance(property_type, ast.PropertyType)
- base_chain = field_node.chain[: len(field_node.chain) - len(property_type.chain)]
- base = ast.Field(chain=list(base_chain) or [property_type.field_type.name], type=property_type.field_type)
- lookup = ast.ArrayAccess(
- array=base, property=ast.Constant(value=property_type.chain[0]), type=ast.StringType(nullable=True)
- )
- new_expr = document_key_extract(lookup, list(property_type.chain[1:]))
- alias_node.expr = new_expr
- if isinstance(alias_node.type, ast.FieldAliasType):
- # The resolver shares this type object with the select scope's column and alias maps
- alias_node.type.type = new_expr.type or ast.UnknownType()
-
- def _rewrite_export(self, node: ast.SelectQuery, name: str) -> None:
- expr = self._select_expr(node, name)
- source = self._events_source(node, name)
- assert expr is not None and source is not None
- reads = self.optimizer.reads.get((id(node), name))
- map_expr = self._map_expr(source, sorted(reads.keys) if reads is not None else [])
- innermost: ast.Alias | None = None
- current = expr
- while isinstance(current, ast.Alias):
- innermost = current
- current = current.expr
- if innermost is None:
- replacement = ast.Alias(
- alias=name,
- expr=map_expr,
- hidden=True,
- type=ast.FieldAliasType(alias=name, type=_map_type()),
- )
- node.select[node.select.index(expr)] = replacement
- if isinstance(node.type, ast.SelectQueryType):
- node.type.columns[name] = replacement.type or ast.UnknownType()
- return
- innermost.expr = map_expr
- if isinstance(innermost.type, ast.FieldAliasType):
- innermost.type.type = _map_type()
-
- def _map_expr(self, source: ast.Field, keys: list[str]) -> ast.Expr:
- args: list[ast.Expr] = []
- for key in keys:
- text = _key_json_text(source, key, self.context)
- assert text is not None
- args.append(ast.Constant(value=key))
- args.append(text)
- map_call = ast.Call(name="map", args=args, type=_map_type())
- if keys:
- return map_call
- # Nobody reads a key; `map()` alone has the type Nothing, which a pass-through cannot carry
- return ast.Call(name="accurateCast", args=[map_call, ast.Constant(value=_MAP_TYPE_NAME)], type=_map_type())
diff --git a/posthog/hogql/transforms/projection_pushdown.py b/posthog/hogql/transforms/projection_pushdown.py
index 03afdfe34ae6..d8cff24f46e0 100644
--- a/posthog/hogql/transforms/projection_pushdown.py
+++ b/posthog/hogql/transforms/projection_pushdown.py
@@ -1,44 +1,21 @@
from collections import defaultdict
-from dataclasses import field
from typing import cast
from posthog.hogql import ast
from posthog.hogql.base import _T_AST
from posthog.hogql.context import HogQLContext
-from posthog.hogql.transforms.exported_properties import narrow_exported_properties
from posthog.hogql.visitor import TraversingVisitor
-from posthog.dataclasses import frozen
-
-SubqueryNode = ast.SelectQuery | ast.SelectSetQuery
-
-# Demand flows top-down, but a CTE's reader can be visited after the CTE, so one pass can leave a column's readers
-# incomplete. Each pass adds what the last one learned; the walk stops when a pass adds nothing. A query that still
-# changes after this many passes is pruned with what was collected and not narrowed.
-_MAX_COLLECTION_PASSES = 50
-
-
-@frozen(frozen=False)
-class ColumnReads:
- """How the readers of one subquery column use it. `whole` is set by any read of the column itself; `keys` are the
- first keys of `column.<key>` reads, with the reads themselves in `sites`, by the hidden alias the resolver wraps
- every field in."""
-
- whole: bool = False
- keys: set[str] = field(default_factory=set)
- sites: dict[int, tuple[ast.Alias, ast.Field]] = field(default_factory=dict)
-
class ProjectionPushdownOptimizer(TraversingVisitor):
"""
- Top-down projection pushdown optimizer that prunes unused asterisk-expanded columns from subqueries, and records
- how each subquery column is read so `exported_properties` can export only the keys read of a properties blob.
+ Top-down projection pushdown optimizer that prunes unused asterisk-expanded columns from subqueries.
Algorithm Overview:
──────────────────
- This optimizer makes several top-down passes through the query tree: the collecting passes only gather demands,
- repeated until a pass learns nothing new, and the final pass prunes with the complete demand set. A CTE can be
- consumed by nodes visited after it — notably sibling UNION branches — so pruning must wait for the full walk.
+ This optimizer makes two top-down passes through the query tree: the first only collects
+ demands, the second prunes with the complete demand set. A CTE can be consumed by nodes
+ visited after it — notably sibling UNION branches — so pruning must wait for the full walk.
Each pass runs these phases per query:
@@ -46,65 +23,21 @@ class ProjectionPushdownOptimizer(TraversingVisitor):
Phase 2 - Collect: Gather column demands from WHERE/GROUP BY/ORDER BY/etc
Phase 3 - Propagate: For demanded columns, visit their source to propagate to child queries
Phase 4 - Recurse: Visit child subqueries (repeat phases 1-4)
- Phase 5 - Prune: Remove unreferenced asterisk columns from this query (final pass only)
-
- A column that a query exports as it is (`SELECT properties FROM sub`) is not a read of it: how the column is used
- is its own readers' business, so phase 3 hands their reads down to the child column instead. The reads recorded
- here are the ones made through a field; whether a select's own shape (DISTINCT, a set operation, an ordinal, an
- alias reference) consumes a column's value is `exported_properties`' question, answered from the select itself.
+ Phase 5 - Prune: Remove unreferenced asterisk columns from this query (second pass only)
"""
- def __init__(self, context: HogQLContext | None = None):
+ def __init__(self):
super().__init__()
- self.context = context
self.demands: dict[int, set[str]] = defaultdict(set)
- self.reads: dict[tuple[int, str], ColumnReads] = defaultdict(ColumnReads)
- self.subquery_map: dict[int, SubqueryNode] = {}
- # The select columns that export a child column as it is, by the child column they export
- self.passthrough_parents: dict[tuple[int, str], dict[tuple[int, str], tuple[SubqueryNode, str]]] = defaultdict(
- dict
- )
- self.roots: set[int] = set()
- self.recursive_cte_exprs: set[int] = set()
- self.converged: bool = False
- self.union_of: dict[int, ast.SelectSetQuery] = {}
- # Branches of a set query that compares rows (anything but UNION ALL)
- self.strict_set_branches: set[int] = set()
- # Select leaves of a set query one of whose leaves keeps every column, so the branches keep lining up
- self.aligned_set_leaves: set[int] = set()
- # True during the collecting passes; gates pruning to the final one (see class docstring)
+ self.subquery_map: dict[int, ast.SelectQuery | ast.SelectSetQuery] = {}
+ # True during the first pass; gates pruning to the second (see class docstring)
self.collecting: bool = False
- self._alias_stack: list[ast.Alias] = []
- # Alias types a clause named directly (`ORDER BY p`), which reads the aliased column's value itself
- self.alias_referenced_types: set[int] = set()
- def optimize(self, node: SubqueryNode) -> SubqueryNode:
- self.roots = self._root_ids(node)
+ def optimize(self, node: ast.SelectQuery | ast.SelectSetQuery) -> ast.SelectQuery | ast.SelectSetQuery:
self.collecting = True
- signature: object = None
- for _ in range(_MAX_COLLECTION_PASSES):
- self.visit(node)
- new_signature = self._demand_signature()
- if new_signature == signature:
- self.converged = True
- break
- signature = new_signature
+ self.visit(node)
self.collecting = False
- return cast(SubqueryNode, self.visit(node))
-
- def _root_ids(self, node: SubqueryNode) -> set[int]:
- if isinstance(node, ast.SelectSetQuery):
- ids = {id(node)}
- for query in self.union_queries(node):
- ids |= self._root_ids(query)
- return ids
- return {id(node)}
-
- def _demand_signature(self) -> object:
- return (
- tuple(sorted((key, tuple(sorted(names))) for key, names in self.demands.items())),
- tuple(sorted((key, reads.whole, tuple(sorted(reads.keys))) for key, reads in self.reads.items())),
- )
+ return cast(ast.SelectQuery | ast.SelectSetQuery, self.visit(node))
def visit_select_query(self, node: ast.SelectQuery) -> ast.SelectQuery:
# Phase 1: Register subqueries and CTEs for demand tracking
@@ -113,16 +46,9 @@ class ProjectionPushdownOptimizer(TraversingVisitor):
if node.select_from:
self._register_subqueries(node.select_from)
- # Phase 2: Collect column demands from query clauses. An asterisk column is demanded by the parent (phase 3),
- # unless every column has to stay, when the child must keep it too.
- keeps_all_columns = self.keeps_all_columns(node)
+ # Phase 2: Collect column demands from query clauses
for expr in node.select:
- if self._is_from_asterisk(expr) and not keeps_all_columns:
- continue
- passthrough = self._passthrough_source(expr)
- if passthrough is not None:
- self._demand_passthrough(node, expr, passthrough)
- elif not self._is_from_asterisk(expr):
+ if not self._is_from_asterisk(expr):
self.visit(expr)
if node.where:
@@ -134,8 +60,6 @@ class ProjectionPushdownOptimizer(TraversingVisitor):
self.visit(expr)
if node.having:
self.visit(node.having)
- if node.qualify:
- self.visit(node.qualify)
if node.order_by:
for expr in node.order_by:
self.visit(expr)
@@ -143,14 +67,6 @@ class ProjectionPushdownOptimizer(TraversingVisitor):
self.visit(node.limit)
if node.offset:
self.visit(node.offset)
- if node.limit_by:
- self.visit(node.limit_by)
- if node.array_join_list:
- for expr in node.array_join_list:
- self.visit(expr)
- if node.window_exprs:
- for window_expr in node.window_exprs.values():
- self.visit(window_expr)
if node.select_from:
self._collect_join_constraint_column_demands(node.select_from)
@@ -164,7 +80,7 @@ class ProjectionPushdownOptimizer(TraversingVisitor):
if node.select_from:
self.visit(node.select_from)
- # Phase 5: Prune unreferenced asterisk columns from this query and CTEs (final pass only)
+ # Phase 5: Prune unreferenced asterisk columns from this query and CTEs (second pass only)
if not self.collecting:
self._prune_columns(node)
if node.ctes:
@@ -180,15 +96,6 @@ class ProjectionPushdownOptimizer(TraversingVisitor):
1. Propagate parent demands to all branches uniformly
2. Visit each branch to apply pruning
"""
- strict = any(not set_node.set_operator.startswith("UNION ALL") for set_node in node.subsequent_select_queries)
- for query in self.union_queries(node):
- self.union_of[id(query)] = node
- if strict:
- self.strict_set_branches.add(id(query))
- leaves = self._select_leaves(node)
- if any(self.keeps_all_columns(leaf) for leaf in leaves):
- self.aligned_set_leaves.update(id(leaf) for leaf in leaves)
-
# Propagate parent demands to all branches uniformly
self._propagate_demands_to_union_branches(node)
@@ -199,38 +106,6 @@ class ProjectionPushdownOptimizer(TraversingVisitor):
return node
- def union_queries(self, node: ast.SelectSetQuery) -> list[SubqueryNode]:
- return [node.initial_select_query, *(set_node.select_query for set_node in node.subsequent_select_queries)]
-
- def _select_leaves(self, node: SubqueryNode) -> list[ast.SelectQuery]:
- if isinstance(node, ast.SelectSetQuery):
- return [leaf for query in self.union_queries(node) for leaf in self._select_leaves(query)]
- return [node]
-
- def _select_names(self, node: SubqueryNode) -> list[str | None]:
- if isinstance(node, ast.SelectSetQuery):
- return self._select_names(node.initial_select_query)
- return [self._get_column_name(expr) for expr in node.select]
-
- def branch_column_name(self, set_query: ast.SelectSetQuery, branch: SubqueryNode, name: str) -> str | None:
- """The branch column that feeds the set query's column `name`. A set operation matches columns by position
- and the first branch names the result; the resolver already reorders BY NAME branches to line up."""
- set_names = self._select_names(set_query)
- if name not in set_names:
- return None
- position = set_names.index(name)
- branch_names = self._select_names(branch)
- return branch_names[position] if position < len(branch_names) else None
-
- def set_column_name(self, set_query: ast.SelectSetQuery, branch: SubqueryNode, branch_name: str) -> str | None:
- """The set query's column that the branch column `branch_name` feeds"""
- branch_names = self._select_names(branch)
- if branch_name not in branch_names:
- return None
- position = branch_names.index(branch_name)
- set_names = self._select_names(set_query)
- return set_names[position] if position < len(set_names) else None
-
def _is_from_asterisk(self, expr: ast.Expr) -> bool:
"""Check if an expression was expanded from asterisk"""
if isinstance(expr, ast.Alias):
@@ -239,144 +114,28 @@ class ProjectionPushdownOptimizer(TraversingVisitor):
return expr.from_asterisk
return False
- def _passthrough_source(self, expr: ast.Expr) -> tuple[SubqueryNode, str] | None:
- """The child query and column name behind a select column that exports a subquery or CTE column as it is."""
- source = self._unwrap_aliases(expr)
- if not isinstance(source, ast.Field) or not isinstance(source.type, ast.FieldType):
- return None
- child = self.child_for(source.type.table_type)
- if child is None:
- return None
- return child, source.type.name
-
- def _unwrap_aliases(self, expr: ast.Expr) -> ast.Expr:
- while isinstance(expr, ast.Alias):
- expr = expr.expr
- return expr
-
- def compares_rows(self, node: ast.SelectQuery) -> bool:
- """Whether the select's rows are compared whole, so every column's value counts"""
- return node.distinct or id(node) in self.strict_set_branches or node.group_by_mode == "all"
-
- def positional_columns(self, node: ast.SelectQuery) -> list[ast.Expr]:
- """The select columns a bare integer in ORDER BY, GROUP BY or LIMIT BY names by position (ClickHouse resolves
- them positionally), so their values count and their positions must not move"""
- referencing: list[ast.Expr] = [order.expr for order in node.order_by or []]
- for group_expr in node.group_by or []:
- if isinstance(group_expr, ast.GroupingSet):
- referencing.extend(group_expr.exprs)
- else:
- referencing.append(group_expr)
- if node.limit_by:
- referencing.extend(node.limit_by.exprs)
- columns: list[ast.Expr] = []
- for expr in referencing:
- if isinstance(expr, ast.Constant) and isinstance(expr.value, int) and not isinstance(expr.value, bool):
- if 0 < expr.value <= len(node.select):
- columns.append(node.select[expr.value - 1])
- return columns
-
- def keeps_all_columns(self, node: ast.SelectQuery) -> bool:
- """Whether dropping a column would change the select's rows, move a position another clause names, or put the
- select out of line with a sibling branch that keeps its columns"""
- return self.compares_rows(node) or bool(self.positional_columns(node)) or id(node) in self.aligned_set_leaves
-
- def _demand_passthrough(self, node: ast.SelectQuery, expr: ast.Expr, passthrough: tuple[SubqueryNode, str]) -> None:
- child, child_name = passthrough
- self.demands[id(child)].add(child_name)
- col_name = self._get_column_name(expr)
- if col_name is not None:
- self.passthrough_parents[(id(child), child_name)][(id(node), col_name)] = (node, col_name)
-
def _propagate_demands_to_children(self, node: ast.SelectQuery) -> None:
"""
Propagate parent demands to child subqueries.
When a parent query demands a column from us, we need to visit that column's
- source expression to propagate the demand down to our child subqueries. A column
- exported as it is hands the parent's reads of it down to the child column.
+ source expression to propagate the demand down to our child subqueries.
- A query without parent demands is either the query's output, which needs every
- column as it is, or a subquery nobody reads a column of, which only has to keep
- its rows.
+ If there are no parent demands but we have asterisk columns,
+ this query defines the output. Visit all asterisk columns to preserve them.
"""
demanded_from_this = self.demands.get(id(node))
if not demanded_from_this:
- if id(node) in self.roots or id(node) not in self._registered_ids():
- # The query's output, or a select inside an expression: every column is read as it is
- for expr in node.select:
- self._column_read_whole(node, expr)
- elif not self.collecting:
- # Only the final pass knows that nobody reads the column; a reader may still turn up while collecting
- self._keep_unread_subquery_rows(node)
+ for expr in node.select:
+ if self._is_from_asterisk(expr):
+ self.visit(expr)
return
for col_name in demanded_from_this:
for expr in node.select:
- if self._get_column_name(expr) != col_name:
- continue
- passthrough = self._passthrough_source(expr)
- if passthrough is None:
+ if self._get_column_name(expr) == col_name:
self.visit(expr)
break
- child, child_name = passthrough
- self._demand_passthrough(node, expr, passthrough)
- parent_reads = self.reads.get((id(node), col_name))
- if parent_reads is not None:
- child_reads = self.reads[(id(child), child_name)]
- child_reads.whole = child_reads.whole or parent_reads.whole
- child_reads.keys.update(parent_reads.keys)
- break
-
- def _keep_unread_subquery_rows(self, node: ast.SelectQuery) -> None:
- if self.keeps_all_columns(node) or id(node) in self.union_of:
- # Every asterisk column stays: a row comparison counts them, and set query branches must line up
- for expr in node.select:
- if self._is_from_asterisk(expr):
- self.visit(expr)
- return
- # Nobody reads a column, so the asterisk columns would only be computed and dropped. An explicit column keeps
- # the rows; without one, the cheapest asterisk column does, and a JSON blob is the most expensive to compute.
- explicit_columns = [expr for expr in node.select if not self._is_from_asterisk(expr)]
- asterisk_columns = [expr for expr in node.select if self._is_from_asterisk(expr)]
- if explicit_columns:
- col_name = self._get_column_name(explicit_columns[0])
- if col_name is not None:
- self.demands[id(node)].add(col_name)
- return
- if not asterisk_columns:
- return
- keep = next((expr for expr in asterisk_columns if not self._is_json_blob(expr)), asterisk_columns[0])
- col_name = self._get_column_name(keep)
- if col_name is not None:
- self.demands[id(node)].add(col_name)
- passthrough = self._passthrough_source(keep)
- if passthrough is not None:
- self._demand_passthrough(node, keep, passthrough)
- else:
- self.visit(keep)
-
- def _registered_ids(self) -> set[int]:
- return {id(query) for query in self.subquery_map.values()}
-
- def _column_read_whole(self, node: ast.SelectQuery, expr: ast.Expr) -> None:
- """The column is the query's output, so the child column it passes through is read as it is"""
- passthrough = self._passthrough_source(expr)
- if passthrough is None:
- self.visit(expr)
- return
- child, child_name = passthrough
- self._demand_passthrough(node, expr, passthrough)
- self.reads[(id(child), child_name)].whole = True
-
- def _is_json_blob(self, expr: ast.Expr) -> bool:
- source = self._unwrap_aliases(expr)
- if not isinstance(source, ast.Field) or source.type is None or self.context is None:
- return False
- try:
- return isinstance(source.type.resolve_constant_type(self.context), ast.StringJSONType)
- except Exception:
- return False
def _propagate_demands_to_union_branches(self, node: ast.SelectSetQuery) -> None:
"""
@@ -389,18 +148,11 @@ class ProjectionPushdownOptimizer(TraversingVisitor):
if not demanded_from_this:
return
- for query in self.union_queries(node):
- for col_name in demanded_from_this:
- branch_name = self.branch_column_name(node, query, col_name)
- if branch_name is None:
- continue
- self.demands[id(query)].add(branch_name)
- parent_reads = self.reads.get((id(node), col_name))
- if parent_reads is None:
- continue
- branch_reads = self.reads[(id(query), branch_name)]
- branch_reads.whole = branch_reads.whole or parent_reads.whole
- branch_reads.keys.update(parent_reads.keys)
+ all_queries = [node.initial_select_query] + [sn.select_query for sn in node.subsequent_select_queries]
+
+ for query in all_queries:
+ if isinstance(query, ast.SelectQuery | ast.SelectSetQuery):
+ self.demands[id(query)].update(demanded_from_this)
def _register_subqueries(self, from_clause: ast.JoinExpr) -> None:
"""Register all subqueries in FROM clause before collecting demands"""
@@ -423,10 +175,7 @@ class ProjectionPushdownOptimizer(TraversingVisitor):
"""Map type to subquery node for demand tracking"""
# Map both the type and the inner SelectQueryType if it's an alias
self.subquery_map[id(type_annotation)] = subquery
- if (
- isinstance(type_annotation, ast.SelectQueryAliasType | ast.SelectViewType)
- and type_annotation.select_query_type
- ):
+ if isinstance(type_annotation, ast.SelectQueryAliasType) and type_annotation.select_query_type:
self.subquery_map[id(type_annotation.select_query_type)] = subquery
def _register_union_subquery(
@@ -438,7 +187,7 @@ class ProjectionPushdownOptimizer(TraversingVisitor):
self.subquery_map[id(type_annotation)] = union_query
# Also register with the inner SelectSetQueryType for both aliased and non-aliased cases
- if isinstance(type_annotation, ast.SelectQueryAliasType | ast.SelectViewType):
+ if isinstance(type_annotation, ast.SelectQueryAliasType):
# Aliased: register with the inner select_query_type
if type_annotation.select_query_type:
self.subquery_map[id(type_annotation.select_query_type)] = union_query
@@ -448,75 +197,42 @@ class ProjectionPushdownOptimizer(TraversingVisitor):
if type_annotation.types:
self.subquery_map[id(type_annotation.types[0])] = union_query
- def _get_subquery(self, table_type: ast.Type) -> SubqueryNode | None:
+ def _get_subquery(self, table_type: ast.Type) -> ast.SelectQuery | ast.SelectSetQuery | None:
"""Retrieve subquery by type"""
return self.subquery_map.get(id(table_type))
- def child_for(self, table_type: ast.Type) -> SubqueryNode | None:
- """The registered subquery, saved view body or CTE query a field's table type refers to, if any. The saved
- view type has to map as well: otherwise nothing ever reads the view body and the final pass prunes it."""
- if isinstance(
- table_type, ast.SelectQueryType | ast.SelectQueryAliasType | ast.SelectSetQueryType | ast.SelectViewType
- ):
- return self._get_subquery(table_type)
- if isinstance(table_type, ast.CTETableType | ast.CTETableAliasType):
- # For CTE types, get the underlying SelectQueryType and demand from it
- cte_select_query_type = (
- table_type.cte_table_type.select_query_type
- if isinstance(table_type, ast.CTETableAliasType)
- else table_type.select_query_type
- )
- return self._get_subquery(cte_select_query_type)
- return None
-
- def visit_alias(self, node: ast.Alias) -> None:
- self._alias_stack.append(node)
- try:
- super().visit_alias(node)
- finally:
- self._alias_stack.pop()
-
def visit_field(self, node: ast.Field) -> ast.Field:
"""Record demand when field references subquery or CTE column"""
field_type = node.type
- if isinstance(field_type, ast.FieldAliasType):
- self.alias_referenced_types.add(id(field_type))
- return node
-
# A property read (`col.some_key`) still demands its base blob column. Its node type is a
# PropertyType wrapping the base FieldType, so unwrap to that base — otherwise the column is
# never recorded as demanded and gets pruned from an asterisk-expanded subquery/CTE, leaving a
# dangling reference that later blows up property lowering.
- property_type: ast.PropertyType | None = None
if isinstance(field_type, ast.PropertyType):
- property_type = field_type
field_type = field_type.field_type
# Handle both FieldType and ExpressionFieldType
if not isinstance(field_type, ast.FieldType | ast.ExpressionFieldType):
return node
- child = self.child_for(field_type.table_type)
- if child is None:
- return node
- self.demands[id(child)].add(field_type.name)
- reads = self.reads[(id(child), field_type.name)]
- if self._is_key_read(property_type, node):
- assert property_type is not None
- reads.keys.add(cast(str, property_type.chain[0]))
- reads.sites[id(self._alias_stack[-1])] = (self._alias_stack[-1], node)
- else:
- reads.whole = True
- return node
+ table_type = field_type.table_type
- def _is_key_read(self, property_type: ast.PropertyType | None, node: ast.Field) -> bool:
- """A `column.<key>` read the resolver wrapped in a hidden alias, which `exported_properties` can re-point.
- A read of the JSON text itself, or of a position, needs the blob."""
- if property_type is None or property_type.as_json or not property_type.chain:
- return False
- if not isinstance(property_type.chain[0], str):
- return False
- return bool(self._alias_stack) and self._alias_stack[-1].expr is node
+ if isinstance(table_type, ast.SelectQueryType | ast.SelectQueryAliasType | ast.SelectSetQueryType):
+ subquery = self._get_subquery(table_type)
+ if subquery:
+ self.demands[id(subquery)].add(field_type.name)
+ elif isinstance(table_type, ast.CTETableType | ast.CTETableAliasType):
+ # For CTE types, get the underlying SelectQueryType and demand from it
+ cte_select_query_type = (
+ table_type.cte_table_type.select_query_type
+ if isinstance(table_type, ast.CTETableAliasType)
+ else table_type.select_query_type
+ )
+ cte_query = self._get_subquery(cte_select_query_type)
+ if cte_query:
+ self.demands[id(cte_query)].add(field_type.name)
+
+ return node
def _collect_join_constraint_column_demands(self, from_clause: ast.JoinExpr) -> None:
"""Collect demands from JOIN constraints"""
@@ -530,7 +246,7 @@ class ProjectionPushdownOptimizer(TraversingVisitor):
def _prune_columns(self, node: ast.SelectQuery) -> None:
"""Prune asterisk-expanded columns that aren't demanded"""
demanded = self.demands.get(id(node))
- if not demanded or self.keeps_all_columns(node):
+ if not demanded:
return
pruned_select = []
@@ -576,13 +292,10 @@ class ProjectionPushdownOptimizer(TraversingVisitor):
# Map the CTE's SelectQueryType to its expr for demand tracking
if cte.expr.type:
self.subquery_map[id(cte.expr.type)] = cte.expr
- if cte.recursive:
- self.recursive_cte_exprs.add(id(cte.expr))
def _visit_ctes(self, ctes: dict[str, ast.CTE]) -> None:
- """Visit and optimize CTEs. A CTE can only read the ones defined before it, so the last one goes first and
- its demands reach the earlier ones in the same pass."""
- for cte in reversed(list(ctes.values())):
+ """Visit and optimize CTEs"""
+ for cte in ctes.values():
if isinstance(cte.expr, ast.SelectQuery | ast.SelectSetQuery):
self.visit(cte.expr)
@@ -593,13 +306,9 @@ class ProjectionPushdownOptimizer(TraversingVisitor):
self._prune_columns(cte.expr)
-def pushdown_projections(node: _T_AST, context: HogQLContext, *, narrow_properties: bool = False) -> _T_AST:
- """Prune unused columns from asterisk expansions in subqueries. With `narrow_properties`, a subquery that exports
- the events `properties` blob exports only the keys its readers read (see `exported_properties`)."""
+def pushdown_projections(node: _T_AST, context: HogQLContext) -> _T_AST:
+ """Prune unused columns from asterisk expansions in subqueries"""
if not isinstance(node, (ast.SelectQuery, ast.SelectSetQuery)):
return node
- optimizer = ProjectionPushdownOptimizer(context)
- result = optimizer.optimize(node)
- if narrow_properties and optimizer.converged:
- narrow_exported_properties(optimizer, context)
- return cast(_T_AST, result)
+ optimizer = ProjectionPushdownOptimizer()
+ return cast(_T_AST, optimizer.optimize(node))
diff --git a/posthog/hogql/transforms/test/test_exported_properties.py b/posthog/hogql/transforms/test/test_exported_properties.py
deleted file mode 100644
index 38ef0a664b7b..000000000000
--- a/posthog/hogql/transforms/test/test_exported_properties.py
+++ /dev/null
@@ -1,404 +0,0 @@
-from posthog.test.base import (
- APIBaseTest,
- BaseTest,
- ClickhouseTestMixin,
- _create_event,
- flush_persons_and_events,
- materialized,
-)
-
-from parameterized import parameterized
-
-from posthog.schema import HogQLQueryModifiers
-
-from posthog.hogql.constants import HogQLDialect
-from posthog.hogql.context import HogQLContext
-from posthog.hogql.parser import parse_select
-from posthog.hogql.printer.utils import prepare_and_print_ast
-from posthog.hogql.property_access_types import RestrictedProperty
-from posthog.hogql.query import execute_hogql_query
-
-from posthog.clickhouse.client.escape import substitute_params
-from posthog.models import PropertyDefinition
-from posthog.models.instance_setting import override_instance_config
-
-from products.data_modeling.backend.facade.models import DataWarehouseSavedQuery
-from products.event_definitions.backend.models.property_definition import PropertyType
-
-STAR_EVENTS = "(SELECT * FROM events)"
-EMAIL_MAP = "map('email', JSONExtractRaw(events.properties, 'email'))"
-
-
-def _modifiers(pruning: bool) -> HogQLQueryModifiers:
- return HogQLQueryModifiers(optimizeProjections=pruning)
-
-
-class TestExportedProperties(BaseTest):
- maxDiff = None
-
- def _print(
- self,
- select: str,
- *,
- native: bool = False,
- pruning: bool = True,
- restricted: set[RestrictedProperty] | None = None,
- dialect: HogQLDialect = "clickhouse",
- ) -> str:
- context = HogQLContext(
- team_id=self.team.pk,
- enable_select_queries=True,
- modifiers=_modifiers(pruning),
- use_new_events_schema=native,
- )
- if restricted is not None:
- context.restricted_properties = restricted
- query, _ = prepare_and_print_ast(parse_select(select), context, dialect)
- return substitute_params(query, context.values).replace(f"team_id, {self.team.pk})", "team_id, 420)")
-
- @parameterized.expand(
- [
- (
- "one_level",
- f"SELECT properties.email FROM {STAR_EVENTS}",
- f"{EMAIL_MAP} AS properties",
- "JSONExtractRaw(properties['email'])",
- ),
- (
- "passed_through_two_levels",
- f"SELECT properties.email FROM (SELECT * FROM {STAR_EVENTS})",
- f"SELECT properties AS properties FROM (SELECT {EMAIL_MAP} AS properties",
- "JSONExtractRaw(properties['email'])",
- ),
- (
- "aliased_passthrough",
- "SELECT p.email FROM (SELECT properties AS p FROM events)",
- f"{EMAIL_MAP} AS p",
- "JSONExtractRaw(p['email'])",
- ),
- (
- "cte",
- f"WITH e AS {STAR_EVENTS} SELECT properties.email FROM e",
- f"WITH e AS (SELECT {EMAIL_MAP} AS properties",
- "JSONExtractRaw(e.properties['email'])",
- ),
- (
- "union_branches",
- "SELECT properties.email FROM (SELECT properties FROM events UNION ALL SELECT properties FROM events)",
- f"{EMAIL_MAP} AS properties FROM events WHERE equals(events.team_id, 420) UNION ALL SELECT {EMAIL_MAP} AS properties",
- "JSONExtractRaw(properties['email'])",
- ),
- (
- "keys_read_at_two_levels",
- f"SELECT properties.email FROM (SELECT * FROM {STAR_EVENTS} WHERE properties.utm_source = 'g')",
- "map('email', JSONExtractRaw(events.properties, 'email'), 'utm_source', JSONExtractRaw(events.properties, 'utm_source')) AS properties",
- "JSONExtractRaw(properties['utm_source'])",
- ),
- (
- "nested_key",
- "SELECT properties.a.b FROM (SELECT properties FROM events)",
- "map('a', JSONExtractRaw(events.properties, 'a')) AS properties",
- "JSONExtractRaw(properties['a'], 'b')",
- ),
- (
- "left_join_on_the_subquery",
- f"SELECT e.properties.email, s.n FROM {STAR_EVENTS} AS e LEFT JOIN (SELECT event AS ev, count() AS n FROM events GROUP BY ev) AS s ON s.ev = e.event",
- f"{EMAIL_MAP} AS properties",
- "JSONExtractRaw(e.properties['email'])",
- ),
- (
- "read_inside_a_lambda",
- "SELECT arrayMap(x -> properties.email, [1]) FROM (SELECT properties FROM events)",
- f"{EMAIL_MAP} AS properties",
- "x -> replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties['email'])",
- ),
- (
- "unread_blob",
- "SELECT count() FROM (SELECT properties FROM events)",
- "accurateCast(map(), 'Map(String, Nullable(String))') AS properties",
- "count()",
- ),
- (
- "union_branches_matched_by_position",
- "SELECT a.email FROM (SELECT properties AS a, properties AS b FROM events UNION ALL SELECT properties AS b, properties AS a FROM events)",
- f"{EMAIL_MAP} AS b",
- "JSONExtractRaw(a['email'])",
- ),
- (
- "eight_chained_ctes",
- f"WITH c1 AS {STAR_EVENTS}, "
- + ", ".join(f"c{n} AS (SELECT * FROM c{n - 1})" for n in range(2, 9))
- + " SELECT properties.email FROM c8",
- f"WITH c1 AS (SELECT {EMAIL_MAP} AS properties",
- "JSONExtractRaw(c8.properties['email'])",
- ),
- ]
- )
- def test_exports_only_the_keys_read(self, _name: str, select: str, exported: str, read: str) -> None:
- printed = self._print(select)
- assert exported in printed, printed
- assert read in printed, printed
- assert "events.properties AS properties" not in printed, printed
-
- @parameterized.expand(
- [
- ("blob_and_key", "SELECT properties, properties.email FROM (SELECT properties FROM events)"),
- ("json_function_on_the_blob", "SELECT JSONHas(properties, 'x') FROM (SELECT properties FROM events)"),
- ("distinct_inner", "SELECT properties.email FROM (SELECT DISTINCT properties FROM events)"),
- (
- "grouped_inner",
- "SELECT properties.email FROM (SELECT properties, count() AS c FROM events GROUP BY properties)",
- ),
- ("inner_order_by_alias", "SELECT p.email FROM (SELECT properties AS p FROM events ORDER BY p)"),
- (
- "union_distinct",
- "SELECT properties.email FROM (SELECT properties FROM events UNION DISTINCT SELECT properties FROM events)",
- ),
- (
- "intersect",
- "SELECT properties.email FROM (SELECT properties FROM events INTERSECT SELECT properties FROM events)",
- ),
- (
- "distinct_asterisk_between_the_levels",
- "SELECT properties.email FROM (SELECT DISTINCT * FROM (SELECT properties FROM events))",
- ),
- (
- "union_branch_that_cannot_narrow",
- "SELECT properties.email FROM (SELECT properties FROM (SELECT properties FROM events) UNION ALL SELECT DISTINCT properties FROM events)",
- ),
- (
- "positional_order_by_on_a_passthrough",
- "SELECT properties.email FROM (SELECT properties, event FROM (SELECT properties, event FROM events) ORDER BY 1 LIMIT 1)",
- ),
- (
- "positional_order_by_at_the_events_level",
- "SELECT p.email FROM (SELECT properties AS p, event FROM events ORDER BY 1 LIMIT 1)",
- ),
- (
- "grouping_sets_ordinal_on_a_passthrough",
- "SELECT properties.email FROM (SELECT properties, event FROM (SELECT properties, event FROM events) GROUP BY GROUPING SETS ((1)))",
- ),
- (
- "group_by_all_on_a_passthrough",
- "SELECT properties.email FROM (SELECT properties, event FROM (SELECT properties, event FROM events) GROUP BY ALL)",
- ),
- (
- "second_alias_of_the_column_feeds_a_blob_branch",
- "SELECT x.a.email, x.e FROM (SELECT a, b.email AS e FROM (SELECT properties AS a, properties AS b FROM (SELECT properties FROM events)) "
- "UNION ALL SELECT DISTINCT properties AS a, '' AS e FROM events) AS x",
- ),
- ("query_output", "SELECT properties FROM (SELECT properties FROM events)"),
- ("read_by_position", f"SELECT properties.arr[1] FROM {STAR_EVENTS}"),
- ("native_flag_map", "SELECT properties.$feature_flags FROM (SELECT properties FROM events)", True),
- ]
- )
- def test_keeps_the_blob_when_a_reader_needs_it(self, _name: str, select: str, native: bool = False) -> None:
- printed = self._print(select, native=native)
- assert "map(" not in printed, printed
-
- def test_saved_view_exports_only_the_keys_read(self) -> None:
- DataWarehouseSavedQuery.objects.create(
- team=self.team,
- name="pageviews",
- query={"query": "SELECT event, properties FROM events WHERE event = '$pageview'"},
- columns={
- "event": {"clickhouse": "String", "hogql": "StringDatabaseField"},
- "properties": {"clickhouse": "String", "hogql": "StringJSONDatabaseField"},
- },
- )
- printed = self._print("SELECT properties.email FROM pageviews")
- assert f"{EMAIL_MAP} AS properties" in printed, printed
- assert "JSONExtractRaw(pageviews.properties['email'])" in printed, printed
-
- @parameterized.expand([("legacy", False), ("native", True)])
- def test_global_kill_switch_overrides_query_modifier(self, _name: str, native: bool) -> None:
- select = f"SELECT properties.email FROM {STAR_EVENTS}"
- unoptimized = self._print(select, native=native, pruning=False)
- with override_instance_config("HOGQL_PROJECTION_OPTIMIZATION_ENABLED", False):
- assert self._print(select, native=native) == unoptimized
- with override_instance_config("HOGQL_PROJECTION_OPTIMIZATION_ENABLED", True):
- optimized = self._print(select, native=native)
- assert "map('email', " in optimized, optimized
- assert "events.distinct_id AS distinct_id" not in optimized, optimized
- assert self._print(select, native=native, pruning=False) == unoptimized
-
- def test_native_export_reads_one_subcolumn(self) -> None:
- printed = self._print(f"SELECT properties.utm_source FROM {STAR_EVENTS}", native=True)
- assert "map('utm_source', " in printed, printed
- assert "toJSONString(events.properties.utm_source)" in printed, printed
- assert "JSONExtractKeysAndValuesRaw" not in printed, printed
-
- @parameterized.expand(
- [
- ("legacy_key", False, "email", "properties.email", True),
- ("legacy_nested", False, "obj.secret", "properties.obj.visible", True),
- ("native_key", True, "email", "properties.email", False),
- ("native_nested", True, "obj.secret", "properties.obj.visible", False),
- ]
- )
- def test_restricted_path_reads_the_masked_blob(
- self, _name: str, native: bool, restricted: str, read: str, narrowed: bool
- ) -> None:
- printed = self._print(
- f"SELECT {read}, properties.utm_source FROM {STAR_EVENTS}",
- native=native,
- restricted={RestrictedProperty(name=restricted, property_type=PropertyDefinition.Type.EVENT)},
- )
- first_key = read.split(".")[1]
- assert ("map(" in printed) is narrowed, printed
- assert "JSONDropKeysPool(" in printed, printed
- assert f"JSONExtractRaw(events.properties, '{first_key}')" not in printed, printed
-
- def test_string_parser_still_gets_the_text(self) -> None:
- PropertyDefinition.objects.create(
- team=self.team, name="amount", property_type=PropertyType.Numeric, type=PropertyDefinition.Type.EVENT
- )
- printed = self._print(f"SELECT sum(toFloatOrZero(properties.amount)) FROM {STAR_EVENTS}")
- assert "toFloat64OrZero(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(properties['amount'])" in printed, printed
- assert "'Float64'" not in printed, printed
-
- def test_hogql_dialect_keeps_the_blob(self) -> None:
- printed = self._print(f"SELECT properties.email FROM {STAR_EVENTS}", dialect="hogql")
- assert "map(" not in printed, printed
-
-
-class TestExportedPropertiesExecution(ClickhouseTestMixin, APIBaseTest):
- def setUp(self) -> None:
- super().setUp()
- PropertyDefinition.objects.create(
- team=self.team, name="amount", property_type=PropertyType.Numeric, type=PropertyDefinition.Type.EVENT
- )
- rows = [
- ("2024-01-02T10:00:00", "$pageview", {"utm_source": "google", "amount": 2, "email": "[email protected]"}),
- ("2024-01-02T11:00:00", "$pageview", {"utm_source": "", "amount": "7.5", "email": None}),
- ("2024-01-03T10:00:00", "$pageview", {"amount": None, "nested": {"a": {"b": "deep"}}}),
- ("2024-01-03T11:00:00", "signup", {"utm_source": "bing", "n": 1.50, "t": True}),
- ("2024-01-03T12:00:00", "signup", {"utm_source": 'say "hi"\\now', "email": "[email protected]"}),
- (
- "2024-01-03T13:00:00",
- "signup",
- {"$feature/test": "true", "$feature_flags": {"test": "true"}, "$active_feature_flags": ["test"]},
- ),
- ("2023-12-31T23:59:59", "$pageview", {"utm_source": "google", "amount": 100, "a.b": "dotted"}),
- ("2024-01-04T10:00:00", "$pageview", {}),
- ]
- for index, (timestamp, event, properties) in enumerate(rows):
- _create_event(
- team=self.team, event=event, distinct_id=f"u{index}", timestamp=timestamp, properties=properties
- )
- flush_persons_and_events()
-
- def _assert_same_rows(self, select: str) -> None:
- narrowed = execute_hogql_query(select, team=self.team, modifiers=_modifiers(True)).results
- plain = execute_hogql_query(select, team=self.team, modifiers=_modifiers(False)).results
- assert narrowed == plain, f"narrowed={narrowed} plain={plain}"
- assert narrowed, "the query must match some rows for the comparison to mean anything"
-
- @parameterized.expand(
- [
- (
- "keys_and_filter",
- f"SELECT properties.email, event FROM {STAR_EVENTS} WHERE properties.utm_source = 'google' ORDER BY 1, 2",
- ),
- (
- "counts_and_key_reads",
- f"SELECT count(), countIf(coalesce(properties.utm_source, '') != ''), uniq(person_id) FROM {STAR_EVENTS} "
- "WHERE event = '$pageview' AND timestamp >= '2024-01-01'",
- ),
- (
- "grouped_key_read",
- f"SELECT properties.utm_source AS src, count() FROM {STAR_EVENTS} WHERE timestamp >= '2024-01-01' GROUP BY src ORDER BY src",
- ),
- (
- "string_parser_on_numeric_property",
- f"SELECT sum(toFloatOrZero(properties.amount)), arraySort(groupArray(properties.amount)) FROM {STAR_EVENTS} WHERE event = '$pageview'",
- ),
- (
- "escaped_string_value",
- f"SELECT properties.utm_source FROM {STAR_EVENTS} WHERE event = 'signup' ORDER BY 1",
- ),
- (
- "empty_null_number_and_bool_values",
- f"SELECT properties.utm_source, properties.email, properties.n, properties.t FROM {STAR_EVENTS} ORDER BY 1, 2, 3, 4",
- ),
- (
- "nested_and_dotted_keys",
- f"SELECT properties.nested.a.b, properties.`a.b` FROM {STAR_EVENTS} WHERE event = '$pageview' ORDER BY 1, 2",
- ),
- (
- "passed_through_two_levels",
- f"SELECT properties.email FROM (SELECT * FROM {STAR_EVENTS} WHERE properties.utm_source = 'google') ORDER BY 1",
- ),
- (
- "aliased_passthrough",
- "SELECT p.email FROM (SELECT properties AS p FROM events) ORDER BY 1",
- ),
- (
- "cte",
- f"WITH e AS {STAR_EVENTS} SELECT properties.email FROM e ORDER BY 1",
- ),
- (
- "union_branches",
- "SELECT properties.email FROM (SELECT properties FROM events UNION ALL SELECT properties FROM events) ORDER BY 1",
- ),
- (
- "left_join_on_the_subquery",
- f"SELECT e.properties.email, s.n FROM {STAR_EVENTS} AS e LEFT JOIN (SELECT event AS ev, count() AS n FROM events GROUP BY ev) AS s ON s.ev = e.event ORDER BY 1, 2",
- ),
- (
- "unread_blob",
- "SELECT count() FROM (SELECT properties FROM events)",
- ),
- (
- "rebuilt_flag_reads",
- f"SELECT properties.`$feature/test`, properties.$active_feature_flags FROM {STAR_EVENTS} WHERE event = 'signup' ORDER BY 1, 2",
- ),
- (
- "flag_map_read",
- f"SELECT count() FROM {STAR_EVENTS} WHERE properties.$feature_flags IS NULL",
- ),
- (
- "enclosing_query_over_the_narrowed_one",
- f"SELECT src, count() FROM (SELECT properties.utm_source AS src FROM {STAR_EVENTS} WHERE event = '$pageview') "
- "WHERE src != '' GROUP BY src ORDER BY src",
- ),
- (
- "filled_rows_read_as_missing",
- "SELECT d, event, properties.email FROM (SELECT toDate(timestamp) AS d, event, properties FROM events ORDER BY d WITH FILL) "
- "WHERE event = '$pageview' ORDER BY d, event, 3",
- ),
- ]
- )
- def test_narrowing_returns_the_same_rows(self, _name: str, select: str) -> None:
- self._assert_same_rows(select)
-
- @parameterized.expand(
- [
- (
- "string_empty_value",
- "utm_source",
- False,
- None,
- f"SELECT count() FROM {STAR_EVENTS} WHERE properties.utm_source = ''",
- ),
- (
- "nullable_string_escapes",
- "utm_source",
- True,
- None,
- f"SELECT properties.utm_source FROM {STAR_EVENTS} WHERE event = 'signup' ORDER BY 1",
- ),
- (
- "float_under_a_string_parser",
- "amount",
- False,
- "Float64",
- f"SELECT sum(toFloatOrZero(properties.amount)), arraySort(groupArray(properties.amount)) FROM {STAR_EVENTS} WHERE event = '$pageview'",
- ),
- ]
- )
- def test_narrowing_ignores_a_materialized_column(
- self, _name: str, property_name: str, is_nullable: bool, column_type: str | None, select: str
- ) -> None:
- with materialized("events", property_name, is_nullable=is_nullable, column_type=column_type):
- self._assert_same_rows(select)
diff --git a/posthog/hogql/transforms/test/test_projection_pushdown.py b/posthog/hogql/transforms/test/test_projection_pushdown.py
index 6c67c63496f5..758300578433 100644
--- a/posthog/hogql/transforms/test/test_projection_pushdown.py
+++ b/posthog/hogql/transforms/test/test_projection_pushdown.py
@@ -118,57 +118,6 @@ class TestProjectionPushdown(BaseTest):
assert optimized.to_hogql() == self.snapshot
- @parameterized.expand(
- [
- ("asterisk", "SELECT count() FROM (SELECT * FROM events) AS sub", ["uuid"]),
- ("explicit_and_asterisk", "SELECT count() FROM (SELECT *, 1 AS one FROM events) AS sub", ["one"]),
- ("distinct_asterisk", "SELECT count() FROM (SELECT DISTINCT * FROM events) AS sub", None),
- ]
- )
- def test_unread_subquery_keeps_only_what_holds_its_rows(self, _name, query, expected_columns):
- optimized = self._optimize(query)
- inner_query = optimized.select_from.table
- column_names = [self._col_name(col) for col in inner_query.select]
- if expected_columns is None:
- assert len(column_names) > 1
- else:
- assert column_names == expected_columns
-
- @parameterized.expand(
- [
- ("distinct", "SELECT event FROM (SELECT DISTINCT * FROM events) AS sub"),
- ("union_distinct", "SELECT event FROM (SELECT * FROM events UNION DISTINCT SELECT * FROM events) AS sub"),
- ("intersect", "SELECT event FROM (SELECT * FROM events INTERSECT SELECT * FROM events) AS sub"),
- (
- "group_by_all",
- "SELECT count() FROM (SELECT * FROM (SELECT distinct_id, properties FROM events) GROUP BY ALL) AS sub",
- ),
- (
- "positional_group_by",
- "SELECT count() FROM (SELECT * FROM (SELECT distinct_id, properties FROM events) GROUP BY 1, 2) AS sub",
- ),
- (
- "grouping_sets_ordinals",
- "SELECT count() FROM (SELECT * FROM (SELECT distinct_id, properties FROM events) GROUP BY GROUPING SETS ((1, 2))) AS sub",
- ),
- ]
- )
- def test_row_comparison_keeps_every_column(self, _name, query):
- optimized = self._optimize(query)
- inner_query = optimized.select_from.table
- if isinstance(inner_query, ast.SelectSetQuery):
- inner_query = inner_query.initial_select_query
- assert len(inner_query.select) > 1
-
- def test_set_query_branches_keep_lining_up(self):
- optimized = self._optimize(
- "SELECT event FROM (SELECT DISTINCT * FROM events UNION ALL SELECT * FROM events) AS sub"
- )
- set_query = optimized.select_from.table
- first = set_query.initial_select_query.select
- second = set_query.subsequent_select_queries[0].select_query.select
- assert len(first) == len(second) > 1
-
@pytest.mark.usefixtures("unittest_snapshot")
def test_prewhere_clause(self):
"""PREWHERE clause columns should be preserved"""
diff --git a/posthog/settings/dynamic_settings.py b/posthog/settings/dynamic_settings.py
index ff6d25eb4069..8f1307e28d8e 100644
--- a/posthog/settings/dynamic_settings.py
+++ b/posthog/settings/dynamic_settings.py
@@ -20,13 +20,6 @@ CONSTANCE_CONFIG = {
"Number of weeks recording performance events will be kept before removing them (for all projects). Storing performance events for a shorter timeframe can help reduce Clickhouse disk usage.",
int,
),
- "HOGQL_PROJECTION_OPTIMIZATION_ENABLED": (
- get_from_env("HOGQL_PROJECTION_OPTIMIZATION_ENABLED", True, type_cast=str_to_bool),
- "Whether HogQL prunes unused subquery columns and narrows exported event properties. "
- "Disable to skip both optimizations for every project, even when optimizeProjections is true. "
- "Changes reach workers within about a minute and affect newly compiled queries only.",
- bool,
- ),
"HOGQL_SHARED_INSIGHT_DATABASE_ENABLED": (
get_from_env("HOGQL_SHARED_INSIGHT_DATABASE_ENABLED", True, type_cast=str_to_bool),
"Whether insight query runners share one HogQL database across execution, response printing, "
@@ -429,7 +422,6 @@ SETTINGS_ALLOWING_API_OVERRIDE = (
"GROWTH_ICP_REENRICH_DAILY_CAP",
"GROWTH_RESCORE_WEBHOOK_SECRET",
"HOGQL_DEFERRED_REVENUE_VIEWS_ENABLED",
- "HOGQL_PROJECTION_OPTIMIZATION_ENABLED",
"HOGQL_SHARED_INSIGHT_DATABASE_ENABLED",
"HOGQL_TEAM_FLAG_CACHE_TTL_SECONDS",
"RECORDINGS_PERFORMANCE_EVENTS_TTL_WEEKS",
diff --git a/products/signals/backend/test/test_signal_queries.py b/products/signals/backend/test/test_signal_queries.py
index 1a006a9acb3c..3d662e44637d 100644
--- a/products/signals/backend/test/test_signal_queries.py
+++ b/products/signals/backend/test/test_signal_queries.py
@@ -156,10 +156,8 @@ class TestFetchSourceProductsForReports(_SignalEmbeddingsTestBase):
self._emit_version(document_id="d3", report_id="rA", source_product="errors", inserted_at=self.base)
self._emit_version(document_id="d4", report_id="rB", source_product="surveys", inserted_at=self.base)
- # +1 for the projection kill-switch instance setting the HogQL compile reads, cold-cache here
- # but TTL-cached per worker in production.
with (
- self.assertNumQueries(FuzzyInt(1, 2)),
+ self.assertNumQueries(FuzzyInt(0, 1)),
patch("products.signals.backend.signal_metadata.execute_hogql_query", wraps=execute_hogql_query) as query,
):
result = fetch_source_products_for_reports(