:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(hogql): reject grouping by a whole properties document (#129469)
Sandy Spicer7223e77519d6
diff --git a/docs/published/handbook/engineering/databases/hogql-python.md b/docs/published/handbook/engineering/databases/hogql-python.mdindex 4f6e1327813b..d908a03ca121 100644--- a/docs/published/handbook/engineering/databases/hogql-python.md+++ b/docs/published/handbook/engineering/databases/hogql-python.md@@ -210,6 +210,9 @@ Integer extraction keeps ClickHouse's JSON parser on the serialized value, becau`JSONHas`, a key computed per row and a call with no key still rebuild the whole document.Grouping keys and expressions evaluated after grouping use the general property reader so conversions reference the same expression as GROUP BY.This includes window inputs, HAVING, ORDER BY and LIMIT BY; ordinary aggregate inputs and row-level predicates retain direct-path optimizations.+A whole properties document cannot be a grouping key: HogQL rejects `GROUP BY properties` on both events tables, whether the document is named directly, by select-list position, through an alias, inside a tuple key, under GROUP BY ALL, or as a column of a subquery, CTE, view or union.+The native table reads a key as a subcolumn, which ClickHouse does not accept as a function of the grouped document, so a key read after `GROUP BY properties` works on the legacy table only; rejecting the key everywhere keeps a query from changing behavior at the switch.+Group by the keys the query reads, or by `uuid` for one row per event; an expression over the document such as `toString(properties)` is not rejected.When a join constraint needs a lazy join of the joined events table, HogQL wraps that table in a subquery and the outer query reads the subquery's columns.The subquery exports the columns, lazy-join fields and property keys the outer query reads, found by their resolved table binding rather than by alias text.A property key is exported as its stored value and the outer read keeps its own conversion, so a Numeric definition and an explicit string parser read the same export.diff --git a/posthog/hogql/resolver.py b/posthog/hogql/resolver.pyindex c294f9172559..8cce4fd9d1b3 100644--- a/posthog/hogql/resolver.py+++ b/posthog/hogql/resolver.py@@ -73,6 +73,7 @@ from posthog.hogql.resolver_utils import (suggested_field_fix,)from posthog.hogql.transforms.flag_calls_from_flag_evaluations import swap_flag_calls, swaps_flag_calls+from posthog.hogql.transforms.grouped_clauses import raise_on_whole_document_groupingfrom posthog.hogql.transforms.trino.persons import (lower_trino_table,resolve_internal_trino_logical_table,@@ -1199,6 +1200,7 @@ class Resolver(CloningVisitor):if node.group_by:new_node.group_by = [self.visit(expr) for expr in node.group_by]new_node.group_by_mode = node.group_by_mode+ raise_on_whole_document_grouping(new_node, self.context)if node.order_by:new_node.order_by = [self.visit(expr) for expr in node.order_by]if node.interpolate is not None:diff --git a/posthog/hogql/test/test_resolver.py b/posthog/hogql/test/test_resolver.pyindex ff529f0ed20c..0fff46da4f2e 100644--- a/posthog/hogql/test/test_resolver.py+++ b/posthog/hogql/test/test_resolver.py@@ -11,7 +11,7 @@ from django.test import SimpleTestCase, override_settingsfrom parameterized import parameterized-from posthog.schema import HogQLQueryModifiers+from posthog.schema import HogQLQueryModifiers, PersonsOnEventsModeimport posthog.hogql.resolver_utils as resolver_utilsfrom posthog.hogql import ast@@ -43,6 +43,13 @@ from posthog.hogql.test.utils import pretty_dataclassesfrom posthog.hogql.visitor import clone_expr+def _doubled_alias_chain(root: str, depth: int, group_by: str) -> str:+ """A select list where each alias names the previous one twice, so a pass that expands every reference doubles+ its work at every link."""+ columns = [f"{root} AS e0"] + [f"tuple(e{i - 1}, e{i - 1}) AS e{i}" for i in range(1, depth + 1)]+ return f"SELECT {', '.join(columns)}, count() FROM events GROUP BY {group_by}"++class TestSavedViewBounds(SimpleTestCase):def _context(self, views: dict[str, str]) -> HogQLContext:database = Database()@@ -2409,3 +2416,110 @@ class TestResolver(BaseTest):# so the canonical-form guard must not reject their queriesexpr = self._select("SELECT event FROM events WHERE person_id = 'not-a-uuid'")resolve_types(expr, self.context, dialect="postgres")++ @parameterized.expand(+ [+ ("direct", "SELECT properties.$browser, count() FROM events GROUP BY timestamp, properties", None),+ ("ordinal", "SELECT properties, count() FROM events GROUP BY 1", None),+ ("alias", "SELECT properties AS p, count() FROM events GROUP BY p", None),+ ("group_by_all", "SELECT properties, count() FROM events GROUP BY ALL", None),+ ("grouping_sets", "SELECT event, count() FROM events GROUP BY GROUPING SETS ((event), (properties))", None),+ ("tuple_key", "SELECT properties.$browser, count() FROM events GROUP BY (properties, event)", None),+ ("group_by_all_inside_an_expression", "SELECT tuple(properties, count()) FROM events GROUP BY ALL", None),+ ("tuple_alias", "SELECT tuple(properties, event) AS k, count() FROM events GROUP BY k", None),+ ("tuple_under_group_by_all", "SELECT tuple(properties, event), count() FROM events GROUP BY ALL", None),+ (+ "aggregate_alias_under_group_by_all",+ "SELECT count() AS n, if(n > 0, properties, '') AS p FROM events GROUP BY ALL",+ None,+ ),+ (+ "nested_aggregate_alias_under_group_by_all",+ "SELECT (count() AS n) + 1, if(n > 0, properties, '') FROM events GROUP BY ALL",+ None,+ ),+ (+ "document_behind_doubled_aliases_under_group_by_all",+ _doubled_alias_chain("properties", 40, "ALL"),+ None,+ ),+ (+ "nested_tuple_alias",+ "SELECT toString(tuple(properties, event) AS k), count() FROM events GROUP BY k",+ None,+ ),+ ("aliased_ordinal", "SELECT properties, properties.browser, count() FROM events GROUP BY (1 AS k)", None),+ (+ "alias_shadowing_a_column",+ "SELECT properties AS event, properties.x, any(events.event) FROM events GROUP BY event",+ None,+ ),+ (+ "subquery_column",+ "SELECT properties, count() FROM (SELECT properties FROM events) GROUP BY properties",+ None,+ ),+ (+ "cte_alias_column",+ "WITH e AS (SELECT properties AS p FROM events) SELECT p, count() FROM e GROUP BY p",+ None,+ ),+ (+ "nested_union_column",+ "SELECT properties, count() FROM ((SELECT properties FROM events UNION ALL SELECT properties FROM events) UNION ALL SELECT properties FROM events) GROUP BY properties",+ None,+ ),+ ("column_aliased_table", "SELECT p.$browser, count() FROM events AS e(uuid, event, p) GROUP BY p", None),+ (+ "person_properties_on_events",+ "SELECT count() FROM events GROUP BY person.properties",+ PersonsOnEventsMode.PERSON_ID_OVERRIDE_PROPERTIES_ON_EVENTS,+ ),+ ]+ )+ def test_whole_properties_document_cannot_be_a_grouping_key(+ self, _name: str, query: str, poe_mode: PersonsOnEventsMode | None+ ) -> None:+ context = self.context+ if poe_mode is not None:+ modifiers = create_default_modifiers_for_team(self.team, HogQLQueryModifiers(personsOnEventsMode=poe_mode))+ database = Database.create_for(team=self.team, modifiers=modifiers)+ context = HogQLContext(+ database=database, team_id=self.team.pk, enable_select_queries=True, modifiers=modifiers+ )+ with self.assertRaisesMessage(QueryError, "Can't group by the whole properties column"):+ resolve_types(self._select(query), context, dialect="clickhouse")++ @parameterized.expand(+ [+ ("property_key", "SELECT properties.$browser, count() FROM events GROUP BY properties.$browser"),+ (+ "json_function",+ "SELECT JSONExtractString(properties, '$browser'), count() FROM events GROUP BY JSONExtractString(properties, '$browser')",+ ),+ ("to_string", "SELECT toString(properties), count() FROM events GROUP BY toString(properties)"),+ ("ordinal_to_key", "SELECT event, count() FROM events GROUP BY 1"),+ ("aggregate_under_group_by_all", "SELECT any(properties), count() FROM events GROUP BY ALL"),+ ("aggregate_argument", "SELECT any(properties) FROM events GROUP BY event"),+ (+ "aggregate_alias_wrapping_the_document",+ "SELECT count() AS n, if(n > 0, toString(properties), '') FROM events GROUP BY ALL",+ ),+ ("subquery_wrapped", "SELECT s, count() FROM (SELECT toString(properties) AS s FROM events) GROUP BY s"),+ ("tuple_of_keys", "SELECT properties.$browser, count() FROM events GROUP BY (properties.$browser, event)"),+ ("persons_document", "SELECT properties.email, count() FROM persons GROUP BY properties"),+ (+ "column_aliased_properties",+ "SELECT event AS properties, any(events.properties) FROM events GROUP BY properties",+ ),+ ("lambda_argument_named_like_an_alias", "SELECT arrayMap(x -> x, [1]) AS x FROM events GROUP BY ALL"),+ (+ "lambda_local_alias",+ "SELECT event AS p, any(arrayMap(x -> (properties AS p), [1])) FROM events GROUP BY p",+ ),+ ("aggregate_behind_doubled_aliases_under_group_by_all", _doubled_alias_chain("count()", 40, "ALL")),+ ("doubled_aliases_as_the_grouping_key", _doubled_alias_chain("event", 40, "e40")),+ ]+ )+ def test_grouping_by_a_property_key_or_a_wrapped_document_resolves(self, _name: str, query: str) -> None:+ resolve_types(self._select(query), self.context, dialect="clickhouse")diff --git a/posthog/hogql/transforms/grouped_clauses.py b/posthog/hogql/transforms/grouped_clauses.pyindex 793b21f81ab6..7711b7e78af2 100644--- a/posthog/hogql/transforms/grouped_clauses.py+++ b/posthog/hogql/transforms/grouped_clauses.py@@ -3,10 +3,20 @@from collections.abc import Callable, Iteratorfrom contextlib import contextmanagerfrom dataclasses import replace-from typing import Any+from typing import TYPE_CHECKING, Anyfrom posthog.hogql import ast+from posthog.hogql.base import AST+from posthog.hogql.database.models import StringJSONDatabaseField+from posthog.hogql.database.schema.events import EVENTS_TABLE_TYPES+from posthog.hogql.errors import QueryErrorfrom posthog.hogql.functions.mapping import find_hogql_aggregation+from posthog.hogql.visitor import TraversingVisitor++from posthog.clickhouse.events_json import NULL_KEYS_COLUMNS++if TYPE_CHECKING:+ from posthog.hogql.context import HogQLContext# The clauses that see the grouped rows. FROM, WHERE, PREWHERE and ARRAY JOIN see the source rows.GROUPED_CLAUSES = ("select", "group_by", "having", "order_by", "qualify", "window_exprs", "limit_by", "interpolate")@@ -63,3 +73,207 @@ class GroupedClauseTracker:yieldelse:yield+++WHOLE_DOCUMENT_GROUPING_ERROR = (+ "Can't group by the whole properties column. "+ "Group by the properties you need, like properties.$browser, or by uuid for one row per event."+)+++class _AliasExpressions(TraversingVisitor):+ """The expression behind each alias, keyed by the alias type the resolver shares between the alias and every field+ that refers to it, so an alias of the same name in a lambda stays apart."""++ def __init__(self, select: ast.SelectQuery) -> None:+ super().__init__()+ self.by_type: dict[int, ast.Expr] = {}+ super().visit_select_query(select)++ def visit_alias(self, node: ast.Alias) -> None:+ if isinstance(node.type, ast.FieldAliasType):+ self.by_type[id(node.type)] = node.expr+ super().visit_alias(node)++ def visit_field(self, node: ast.Field) -> None:+ """A field holds no alias. The base visitor would walk its type, which the resolver shares between an alias+ and every reference to it, so that walk doubles at every alias referred to twice."""++ def visit_select_query(self, node: ast.SelectQuery) -> None:+ """A nested query is checked on its own, and a key that reads one of its columns is followed through the column+ type rather than through its aliases."""+++def _aliased_expression(expr: ast.Expr, aliases: dict[int, ast.Expr]) -> ast.Expr:+ """The expression behind an alias, or behind a reference to one, which is what ClickHouse groups by.++ Every alias followed on the way is pointed at the result, so a chain of aliases that each name the previous one+ is walked once for the query instead of once per reference.+ """+ followed: list[int] = []+ while True:+ if isinstance(expr, ast.Alias):+ expr = expr.expr+ continue+ if isinstance(expr, ast.Field) and isinstance(expr.type, ast.FieldAliasType):+ aliased = aliases.get(id(expr.type))+ if aliased is not None:+ followed.append(id(expr.type))+ expr = aliased+ continue+ for alias_type_id in followed:+ aliases[alias_type_id] = expr+ return expr+++class _AggregateFinder(TraversingVisitor):+ """Whether an expression holds an aggregate.++ Each node is answered once per query, through a memo shared by every call, because a scan that restarts at every+ node is quadratic in the expression, and an alias that refers to an earlier alias twice doubles the expansion at+ every link of the chain.+ """++ def __init__(self, aliases: dict[int, ast.Expr], memo: dict[int, bool]) -> None:+ super().__init__()+ self.aliases = aliases+ self.memo = memo+ self.found = False++ def visit(self, node: AST | None) -> None:+ if node is None or self.found:+ return+ known = self.memo.get(id(node))+ if known is not None:+ self.found = known+ return+ super().visit(node)+ self.memo[id(node)] = self.found++ def visit_call(self, node: ast.Call) -> None:+ if find_hogql_aggregation(node.name):+ self.found = True+ else:+ super().visit_call(node)++ def visit_field(self, node: ast.Field) -> None:+ aliased = _aliased_expression(node, self.aliases)+ if aliased is not node:+ self.visit(aliased)++ def visit_select_query(self, node: ast.SelectQuery) -> None:+ """A subquery's aggregates group the subquery's rows."""+++def _has_aggregate(expr: ast.Expr, aliases: dict[int, ast.Expr], memo: dict[int, bool]) -> bool:+ finder = _AggregateFinder(aliases, memo)+ finder.visit(expr)+ return finder.found+++class _ImplicitGroupingKeys(TraversingVisitor):+ """Collects the keys GROUP BY ALL takes from a select column: its largest parts that hold no aggregate."""++ def __init__(self, aliases: dict[int, ast.Expr]) -> None:+ super().__init__()+ self.aliases = aliases+ self.aggregate_memo: dict[int, bool] = {}+ self.visited: set[int] = set()+ self.keys: list[ast.Expr] = []++ def visit(self, node: AST | None) -> None:+ if isinstance(node, ast.Expr):+ node = _aliased_expression(node, self.aliases)+ # An alias expression reached through a second reference has given its keys already.+ if id(node) in self.visited:+ return+ self.visited.add(id(node))+ if isinstance(node, ast.Call) and find_hogql_aggregation(node.name):+ return+ if not _has_aggregate(node, self.aliases, self.aggregate_memo):+ self.keys.append(node)+ return+ super().visit(node)+++def _key_parts(key: ast.Expr, aliases: dict[int, ast.Expr], seen: set[int]) -> Iterator[ast.Expr]:+ """A key as ClickHouse groups by it: an alias stands for its expression, and a tuple is one key per element.++ An alias expression reached through a second reference is the same key, so it is skipped: a chain of aliases that+ each name the previous one twice would otherwise double the parts at every link.+ """+ key = _aliased_expression(key, aliases)+ if id(key) in seen:+ return+ seen.add(id(key))+ if isinstance(key, ast.Tuple):+ elements = key.exprs+ elif isinstance(key, ast.Call) and key.name == "tuple":+ elements = key.args+ else:+ yield key+ return+ for element in elements:+ yield from _key_parts(element, aliases, seen)+++def _reads_events_json_document(field_type: ast.Type | None, context: "HogQLContext") -> bool:+ """Whether a field reads a JSON document of the events table, through any alias, subquery, CTE, view or union."""+ while isinstance(field_type, ast.FieldAliasType):+ field_type = field_type.type+ if not isinstance(field_type, ast.FieldType):+ return False+ table_type: ast.Type = field_type.table_type+ if isinstance(table_type, ast.CTETableAliasType):+ table_type = table_type.cte_table_type+ if isinstance(table_type, (ast.SelectQueryAliasType, ast.CTETableType, ast.SelectViewType)):+ table_type = table_type.select_query_type+ while isinstance(table_type, ast.SelectSetQueryType):+ table_type = table_type.types[0]+ if isinstance(table_type, ast.SelectQueryType):+ return _reads_events_json_document(table_type.columns.get(field_type.name), context)+ if not isinstance(table_type, ast.BaseTableType):+ return False+ database_field = field_type.resolve_database_field(context)+ # The native table stores the columns that have a null-keys companion as JSON and reads their keys as subcolumns.+ return (+ isinstance(table_type.resolve_database_table(context), EVENTS_TABLE_TYPES)+ and isinstance(database_field, StringJSONDatabaseField)+ and database_field.name in NULL_KEYS_COLUMNS+ )+++def _grouping_keys(node: ast.SelectQuery) -> list[ast.Expr]:+ """The keys ClickHouse groups by: an ordinal names a select column, and GROUP BY ALL takes the largest+ aggregate-free parts of every select column."""+ aliases = _AliasExpressions(node).by_type+ keys: list[ast.Expr] = []+ if node.group_by_mode == "all":+ implicit = _ImplicitGroupingKeys(aliases)+ for expr in node.select:+ implicit.visit(expr)+ keys = implicit.keys+ else:+ for group in node.group_by or []:+ for key in group.exprs if isinstance(group, ast.GroupingSet) else [group]:+ key = _aliased_expression(key, aliases)+ if isinstance(key, ast.Constant) and type(key.value) is int and 1 <= key.value <= len(node.select):+ key = node.select[key.value - 1]+ keys.append(key)+ seen: set[int] = set()+ return [part for key in keys for part in _key_parts(key, aliases, seen)]+++def raise_on_whole_document_grouping(node: ast.SelectQuery, context: "HogQLContext") -> None:+ """Rejects a JSON document of the events table, `properties` or `person_properties`, as a grouping key.++ The native events table reads a key as a subcolumn, and ClickHouse does not accept the subcolumn+ `properties.$browser` as a function of the key `properties`, so a key read after `GROUP BY properties` fails+ there while the legacy table accepts it. Both tables reject the key so a query cannot work on one and fail on+ the other.+ """+ if not is_grouped(node):+ return+ for key in _grouping_keys(node):+ if isinstance(key, ast.Field) and _reads_events_json_document(key.type, context):+ raise QueryError(WHOLE_DOCUMENT_GROUPING_ERROR, node=key)diff --git a/posthog/hogql/transforms/test/test_exported_properties.py b/posthog/hogql/transforms/test/test_exported_properties.pyindex adef2ac01671..818ab1749e03 100644--- a/posthog/hogql/transforms/test/test_exported_properties.py+++ b/posthog/hogql/transforms/test/test_exported_properties.py@@ -139,10 +139,6 @@ class TestExportedProperties(BaseTest):("blob_and_key", "SELECT properties, properties.email FROM (SELECT properties FROM events)"),("json_function_on_the_blob", "SELECT JSONHas(properties, 'x') FROM (SELECT properties FROM events)"),("distinct_inner", "SELECT properties.email FROM (SELECT DISTINCT properties FROM events)"),- (- "grouped_inner",- "SELECT properties.email FROM (SELECT properties, count() AS c FROM events GROUP BY properties)",- ),("inner_order_by_alias", "SELECT p.email FROM (SELECT properties AS p FROM events ORDER BY p)"),("union_distinct",@@ -168,14 +164,6 @@ class TestExportedProperties(BaseTest):"positional_order_by_at_the_events_level","SELECT p.email FROM (SELECT properties AS p, event FROM events ORDER BY 1 LIMIT 1)",),- (- "grouping_sets_ordinal_on_a_passthrough",- "SELECT properties.email FROM (SELECT properties, event FROM (SELECT properties, event FROM events) GROUP BY GROUPING SETS ((1)))",- ),- (- "group_by_all_on_a_passthrough",- "SELECT properties.email FROM (SELECT properties, event FROM (SELECT properties, event FROM events) GROUP BY ALL)",- ),("second_alias_of_the_column_feeds_a_blob_branch","SELECT x.a.email, x.e FROM (SELECT a, b.email AS e FROM (SELECT properties AS a, properties AS b FROM (SELECT properties FROM events)) "diff --git a/posthog/hogql/transforms/test/test_projection_pushdown.py b/posthog/hogql/transforms/test/test_projection_pushdown.pyindex 3fc90cbf0122..258715434b20 100644--- a/posthog/hogql/transforms/test/test_projection_pushdown.py+++ b/posthog/hogql/transforms/test/test_projection_pushdown.py@@ -144,15 +144,15 @@ class TestProjectionPushdown(BaseTest):("intersect", "SELECT event FROM (SELECT * FROM events INTERSECT SELECT * FROM events) AS sub"),("group_by_all",- "SELECT count() FROM (SELECT * FROM (SELECT distinct_id, properties FROM events) GROUP BY ALL) AS sub",+ "SELECT count() FROM (SELECT * FROM (SELECT distinct_id, event FROM events) GROUP BY ALL) AS sub",),("positional_group_by",- "SELECT count() FROM (SELECT * FROM (SELECT distinct_id, properties FROM events) GROUP BY 1, 2) AS sub",+ "SELECT count() FROM (SELECT * FROM (SELECT distinct_id, event FROM events) GROUP BY 1, 2) AS sub",),("grouping_sets_ordinals",- "SELECT count() FROM (SELECT * FROM (SELECT distinct_id, properties FROM events) GROUP BY GROUPING SETS ((1, 2))) AS sub",+ "SELECT count() FROM (SELECT * FROM (SELECT distinct_id, event FROM events) GROUP BY GROUPING SETS ((1, 2))) AS sub",),])diff --git a/products/data_modeling/backend/test/test_incremental_eligibility.py b/products/data_modeling/backend/test/test_incremental_eligibility.pyindex 7d3068886873..d597e02aaf2c 100644--- a/products/data_modeling/backend/test/test_incremental_eligibility.py+++ b/products/data_modeling/backend/test/test_incremental_eligibility.py@@ -327,7 +327,8 @@ class TestStarExpansion:assert result.key_candidates == ["a"]def test_an_aggregate_alongside_a_star_stays_excluded(self) -> None:- result = self._check("SELECT *, count() AS c FROM events GROUP BY ALL")+ # A star over the events table would group by the whole properties document, which HogQL rejects.+ result = self._check("SELECT *, count() AS c FROM (SELECT event, timestamp FROM events) GROUP BY ALL")assert "timestamp" in result.key_candidatesassert "c" not in result.key_candidates