:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(groups): stop the groups list query from running out of memory (#122179)
Phil Haack5e3bd10cdc7b
diff --git a/posthog/hogql/database/schema/groups.py b/posthog/hogql/database/schema/groups.pyindex bbe8d63f17f1..9802d3d2129b 100644--- a/posthog/hogql/database/schema/groups.py+++ b/posthog/hogql/database/schema/groups.py@@ -56,17 +56,17 @@ GROUPS_TABLE_FIELDS: dict[str, FieldOrTable] = {}-def select_from_groups_table(requested_fields: dict[str, list[str | int]], key_limit: int | None = None):+def select_from_groups_table(+ requested_fields: dict[str, list[str | int]], keys: ast.SelectQuery | ast.SelectSetQuery | None = None+) -> ast.SelectQuery:+ """``keys`` selects ``(index, key)`` pairs. When it is given, the dedup reads only those groups."""select = argmax_select(table_name="raw_groups",select_fields=requested_fields,group_fields=["index", "key"],argmax_field="updated_at",)- if key_limit is not None:- # Two-phase dedup: pick the N group keys cheaply, then argMax the heavy properties for only those keys.- keys = parse_select("SELECT index, key FROM raw_groups GROUP BY index, key")- keys.limit = ast.Constant(value=key_limit)+ if keys is not None:# GLOBAL IN, not IN: groups is Distributed(rand()); a plain IN dedups per-shard over a subset of versions.select.where = ast.CompareOperation(op=ast.CompareOperationOp.GlobalIn,@@ -385,7 +385,13 @@ class GroupsTable(LazyTable):fields: dict[str, FieldOrTable] = GROUPS_TABLE_FIELDSdef lazy_select(self, table_to_add: LazyTableToAdd, context, node):- return select_from_groups_table(table_to_add.fields_accessed, key_limit=_bare_limit_key_count(node))+ key_limit = _bare_limit_key_count(node)+ if key_limit is None:+ return select_from_groups_table(table_to_add.fields_accessed)+ # Two-phase dedup: pick the N group keys cheaply, then argMax the heavy properties for only those keys.+ keys = parse_select("SELECT index, key FROM raw_groups GROUP BY index, key")+ keys.limit = ast.Constant(value=key_limit)+ return select_from_groups_table(table_to_add.fields_accessed, keys=keys)def to_printed_clickhouse(self, context):return "groups"diff --git a/posthog/models/group/util.py b/posthog/models/group/util.pyindex e4e882d76422..f889aa19318c 100644--- a/posthog/models/group/util.py+++ b/posthog/models/group/util.py@@ -235,95 +235,88 @@ def list_groups() -> ListGroupsResult:"""List groups for the public groups API, served from ClickHouse.- Filtering, full-text search, ``created_at`` ordering and keyset pagination all run against the- deduped ``groups`` table, so this never reads the persons Postgres database; results are- eventually consistent. ``search`` matches a substring of the raw properties JSON or the group- key exactly (case-insensitive), preserving the previous endpoint's semantics. The keyset cursor+ Filtering, full-text search, ``created_at`` ordering and keyset pagination all run against+ ``raw_groups``, so this never reads the persons Postgres database; results are eventually+ consistent. ``search`` matches a substring of the latest raw properties JSON or the group key+ exactly (case-insensitive), preserving the previous endpoint's semantics. The keyset cursorbreaks ``created_at`` ties on ``group_key`` (unique within a team and type).++ Dedup runs in two phases, so memory does not grow with the group type's total property volume.+ The first phase picks the page's keys and aggregates only ``created_at`` and the search match+ flag per key. The second phase reads the latest properties for those keys only. A single dedup+ over the ``groups`` table holds the latest properties of every group of the type in memory,+ because the search, cursor and ordering can only apply after the dedup. That runs out of memory+ on large projects."""from posthog.hogql import ast # noqa: PLC0415 — keep HogQL off this widely-imported module's import path+ from posthog.hogql.database.schema.groups import select_from_groups_table # noqa: PLC0415+ from posthog.hogql.parser import parse_expr, parse_select # noqa: PLC0415from posthog.hogql.query import execute_hogql_query # noqa: PLC0415from posthog.models.team import Team # noqa: PLC0415team = Team.objects.get(pk=team_id)- where_exprs: list[ast.Expr] = [- ast.CompareOperation(- left=ast.Field(chain=["index"]),- op=ast.CompareOperationOp.Eq,- right=ast.Constant(value=group_type_index),- )- ]-+ # Row-level filters on the group identity hold for every version of a group, so they can apply+ # before the dedup.+ key_filter: ast.Expr = ast.Constant(value=True)if group_key_contains:- where_exprs.append(- ast.CompareOperation(- op=ast.CompareOperationOp.ILike,- left=ast.Call(name="toString", args=[ast.Field(chain=["key"])]),- right=ast.Constant(value=f"%{_escape_clickhouse_like(group_key_contains)}%"),- )+ key_filter = parse_expr(+ "toString(key) ILIKE {pattern}",+ placeholders={"pattern": ast.Constant(value=f"%{_escape_clickhouse_like(group_key_contains)}%")},)+ # Filters on versioned columns must see only the latest version, so they apply after the dedup.+ having_exprs: list[ast.Expr] = []if search:- where_exprs.append(- ast.Or(- exprs=[- ast.CompareOperation(- op=ast.CompareOperationOp.ILike,- left=ast.Call(name="toString", args=[ast.Field(chain=["properties"])]),- right=ast.Constant(value=f"%{_escape_clickhouse_like(search)}%"),- ),- ast.CompareOperation(- op=ast.CompareOperationOp.Eq,- left=ast.Call(name="lower", args=[ast.Call(name="toString", args=[ast.Field(chain=["key"])])]),- right=ast.Call(name="lower", args=[ast.Constant(value=search)]),- ),- ]+ having_exprs.append(+ parse_expr(+ "argMax(toString(properties) ILIKE {pattern}, updated_at) OR lower(toString(key)) = lower({search})",+ placeholders={+ "pattern": ast.Constant(value=f"%{_escape_clickhouse_like(search)}%"),+ "search": ast.Constant(value=search),+ },))-if cursor_created_at_us > 0:cursor_dt = datetime.datetime.fromtimestamp(cursor_created_at_us / 1_000_000, tz=datetime.UTC)- where_exprs.append(- ast.Or(- exprs=[- ast.CompareOperation(- op=ast.CompareOperationOp.Lt,- left=ast.Field(chain=["created_at"]),- right=ast.Constant(value=cursor_dt),- ),- ast.And(- exprs=[- ast.CompareOperation(- op=ast.CompareOperationOp.Eq,- left=ast.Field(chain=["created_at"]),- right=ast.Constant(value=cursor_dt),- ),- ast.CompareOperation(- op=ast.CompareOperationOp.Lt,- left=ast.Field(chain=["key"]),- right=ast.Constant(value=cursor_group_key),- ),- ]- ),- ]+ having_exprs.append(+ parse_expr(+ "argMax(created_at, updated_at) < {cursor_dt}"+ " OR (argMax(created_at, updated_at) = {cursor_dt} AND key < {cursor_group_key})",+ placeholders={+ "cursor_dt": ast.Constant(value=cursor_dt),+ "cursor_group_key": ast.Constant(value=cursor_group_key),+ },))- query = ast.SelectQuery(- select=[- ast.Field(chain=["key"]),- ast.Field(chain=["created_at"]),- ast.Field(chain=["properties"]),- ],- select_from=ast.JoinExpr(table=ast.Field(chain=["groups"])),- where=ast.And(exprs=where_exprs),- order_by=[- ast.OrderExpr(expr=ast.Field(chain=["created_at"]), order="DESC"),- ast.OrderExpr(expr=ast.Field(chain=["key"]), order="DESC"),- ],- limit=ast.Constant(value=limit + 1),+ page_keys = parse_select(+ """+ SELECT index, key+ FROM raw_groups+ WHERE index = {group_type_index} AND {key_filter}+ GROUP BY index, key+ HAVING {having}+ ORDER BY argMax(created_at, updated_at) DESC, key DESC+ LIMIT {limit}+ """,+ placeholders={+ "group_type_index": ast.Constant(value=group_type_index),+ "key_filter": key_filter,+ "having": ast.And(exprs=having_exprs) if having_exprs else ast.Constant(value=True),+ "limit": ast.Constant(value=limit + 1),+ },+ )+ # Without its own LIMIT, the executor's default row limit can drop the extra row that sets has_more.+ query = parse_select(+ "SELECT key, created_at, properties FROM {latest_groups} ORDER BY created_at DESC, key DESC LIMIT {limit}",+ placeholders={+ "latest_groups": select_from_groups_table(+ {"created_at": ["created_at"], "properties": ["properties"]}, keys=page_keys+ ),+ "limit": ast.Constant(value=limit + 1),+ },)response = execute_hogql_query(query, team=team, query_type="groups_list")diff --git a/products/groups/backend/tests/test_clickhouse_groups.py b/products/groups/backend/tests/test_clickhouse_groups.pyindex 205da471333d..a278d7ca65ed 100644--- a/products/groups/backend/tests/test_clickhouse_groups.py+++ b/products/groups/backend/tests/test_clickhouse_groups.py@@ -2498,16 +2498,31 @@ class TestListGroupsFunction(ClickhouseTestMixin, APIBaseTest):assert result.groups[1].group_key == "org:2"assert result.has_more is True- @time_machine.travel("2021-05-04", tick=False)+ @time_machine.travel("2021-05-03", tick=False)+ def test_has_more_at_default_limit(self):+ for i in range(101):+ raw_create_group_ch(+ team_id=self.team.pk, group_type_index=0, group_key=f"org:{i:03d}", properties={}, created_at=now()+ )++ result = list_groups(team_id=self.team.pk, group_type_index=0)++ assert len(result.groups) == 100+ assert result.has_more is True++ @time_machine.travel("2021-05-06", tick=False)def test_pagination_cursor_returns_next_page(self):- with time_machine.travel("2021-05-01", tick=False):- create_group(team_id=self.team.pk, group_type_index=0, group_key="org:1", properties={})- with time_machine.travel("2021-05-02", tick=False):- create_group(team_id=self.team.pk, group_type_index=0, group_key="org:2", properties={})- with time_machine.travel("2021-05-03", tick=False):- create_group(team_id=self.team.pk, group_type_index=0, group_key="org:3", properties={})+ for day in range(1, 6):+ with time_machine.travel(f"2021-05-0{day}", tick=False):+ create_group(+ team_id=self.team.pk,+ group_type_index=0,+ group_key=f"org:{day}",+ properties={"plan": "pro" if day % 2 else "free"},+ )page1 = list_groups(team_id=self.team.pk, group_type_index=0, limit=2)+ assert [g.group_key for g in page1.groups] == ["org:5", "org:4"]last = page1.groups[-1]page2 = list_groups(@@ -2518,21 +2533,42 @@ class TestListGroupsFunction(ClickhouseTestMixin, APIBaseTest):limit=2,)- assert len(page2.groups) == 1- assert page2.groups[0].group_key == "org:1"- assert page2.has_more is False+ assert [g.group_key for g in page2.groups] == ["org:3", "org:2"]+ assert page2.has_more is True++ first = page1.groups[0]+ with self.capture_select_queries() as queries:+ searched = list_groups(+ team_id=self.team.pk,+ group_type_index=0,+ search="pro",+ cursor_created_at_us=int(first.created_at.timestamp() * 1_000_000),+ cursor_group_key=first.group_key,+ limit=1,+ )++ assert [g.group_key for g in searched.groups] == ["org:3"]+ assert searched.has_more is True+ # Results alone cannot show that both queries stop at limit + 1 keys and that only the+ # property-reading query aggregates whole properties.+ sql = next(query for query in queries if "globalIn(tuple(" in query)+ assert sql.count("LIMIT 2") == 2+ assert sql.count("argMax(groups.group_properties,") == 1@time_machine.travel("2021-05-04", tick=False)def test_pagination_cursor_breaks_created_at_ties_on_group_key(self):- # Three groups created at the same instant: keyset pagination must fall back to the+ # Four groups created at the same instant: keyset pagination must fall back to the# group_key tiebreaker (descending) and neither skip nor duplicate a row across pages.+ # The tie spans more than limit + 1 keys, so the key-selection query's own tiebreak decides+ # which keys reach a page.with time_machine.travel("2021-05-01", tick=False):create_group(team_id=self.team.pk, group_type_index=0, group_key="org:a", properties={})create_group(team_id=self.team.pk, group_type_index=0, group_key="org:b", properties={})create_group(team_id=self.team.pk, group_type_index=0, group_key="org:c", properties={})+ create_group(team_id=self.team.pk, group_type_index=0, group_key="org:d", properties={})page1 = list_groups(team_id=self.team.pk, group_type_index=0, limit=2)- assert [g.group_key for g in page1.groups] == ["org:c", "org:b"]+ assert [g.group_key for g in page1.groups] == ["org:d", "org:c"]assert page1.has_more is Truelast = page1.groups[-1]@@ -2543,7 +2579,7 @@ class TestListGroupsFunction(ClickhouseTestMixin, APIBaseTest):cursor_group_key=last.group_key,limit=2,)- assert [g.group_key for g in page2.groups] == ["org:a"]+ assert [g.group_key for g in page2.groups] == ["org:b", "org:a"]assert page2.has_more is False@time_machine.travel("2021-05-03", tick=False)@@ -2592,13 +2628,13 @@ class TestListGroupsFunction(ClickhouseTestMixin, APIBaseTest):@time_machine.travel("2021-05-03", tick=False)def test_filters_by_group_type_index(self):- create_group(team_id=self.team.pk, group_type_index=0, group_key="org:1", properties={})- create_group(team_id=self.team.pk, group_type_index=1, group_key="company:1", properties={})+ create_group(team_id=self.team.pk, group_type_index=0, group_key="org:1", properties={"name": "Org"})+ # The same key under another type is a different group; its versions must not merge in.+ create_group(team_id=self.team.pk, group_type_index=1, group_key="org:1", properties={"name": "Company"})result = list_groups(team_id=self.team.pk, group_type_index=0)- assert len(result.groups) == 1- assert result.groups[0].group_key == "org:1"+ assert [(g.group_key, g.group_properties) for g in result.groups] == [("org:1", {"name": "Org"})]@time_machine.travel("2021-05-03", tick=False)def test_empty_result(self):@@ -2627,8 +2663,6 @@ class TestListGroupsFunction(ClickhouseTestMixin, APIBaseTest):assert result.groups[0].group_properties == {}def test_returns_latest_properties_after_update(self):- # The groups table dedups by argMax(_timestamp); a newer write for the same (index, key)- # must win and produce a single row — guards the choice of `groups` over `raw_groups`.with time_machine.travel("2021-05-01", tick=False):create_group(team_id=self.team.pk, group_type_index=0, group_key="org:1", properties={"name": "Old"})with time_machine.travel("2021-05-02", tick=False):@@ -2644,6 +2678,9 @@ class TestListGroupsFunction(ClickhouseTestMixin, APIBaseTest):assert len(result.groups) == 1assert result.groups[0].group_properties == {"name": "New"}+ assert list_groups(team_id=self.team.pk, group_type_index=0, search="Old").groups == []+ searched = list_groups(team_id=self.team.pk, group_type_index=0, search="New")+ assert [(g.group_key, g.group_properties) for g in searched.groups] == [("org:1", {"name": "New"})]@time_machine.travel("2021-05-03", tick=False)def test_preserves_complex_property_values(self):