Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(groups): stop the groups list query from running out of memory (#122179)

Phil Haack5e3bd10cdc7b
diff --git a/posthog/hogql/database/schema/groups.py b/posthog/hogql/database/schema/groups.py
index bbe8d63f17f1..9802d3d2129b 100644
--- a/posthog/hogql/database/schema/groups.py
+++ b/posthog/hogql/database/schema/groups.py
@@ -56,17 +56,17 @@ GROUPS_TABLE_FIELDS: dict[str, FieldOrTable] = {
}
-def select_from_groups_table(requested_fields: dict[str, list[str | int]], key_limit: int | None = None):
+def select_from_groups_table(
+ requested_fields: dict[str, list[str | int]], keys: ast.SelectQuery | ast.SelectSetQuery | None = None
+) -> ast.SelectQuery:
+ """``keys`` selects ``(index, key)`` pairs. When it is given, the dedup reads only those groups."""
select = argmax_select(
table_name="raw_groups",
select_fields=requested_fields,
group_fields=["index", "key"],
argmax_field="updated_at",
)
- if key_limit is not None:
- # Two-phase dedup: pick the N group keys cheaply, then argMax the heavy properties for only those keys.
- keys = parse_select("SELECT index, key FROM raw_groups GROUP BY index, key")
- keys.limit = ast.Constant(value=key_limit)
+ if keys is not None:
# GLOBAL IN, not IN: groups is Distributed(rand()); a plain IN dedups per-shard over a subset of versions.
select.where = ast.CompareOperation(
op=ast.CompareOperationOp.GlobalIn,
@@ -385,7 +385,13 @@ class GroupsTable(LazyTable):
fields: dict[str, FieldOrTable] = GROUPS_TABLE_FIELDS
def lazy_select(self, table_to_add: LazyTableToAdd, context, node):
- return select_from_groups_table(table_to_add.fields_accessed, key_limit=_bare_limit_key_count(node))
+ key_limit = _bare_limit_key_count(node)
+ if key_limit is None:
+ return select_from_groups_table(table_to_add.fields_accessed)
+ # Two-phase dedup: pick the N group keys cheaply, then argMax the heavy properties for only those keys.
+ keys = parse_select("SELECT index, key FROM raw_groups GROUP BY index, key")
+ keys.limit = ast.Constant(value=key_limit)
+ return select_from_groups_table(table_to_add.fields_accessed, keys=keys)
def to_printed_clickhouse(self, context):
return "groups"
diff --git a/posthog/models/group/util.py b/posthog/models/group/util.py
index e4e882d76422..f889aa19318c 100644
--- a/posthog/models/group/util.py
+++ b/posthog/models/group/util.py
@@ -235,95 +235,88 @@ def list_groups(
) -> ListGroupsResult:
"""List groups for the public groups API, served from ClickHouse.
- Filtering, full-text search, ``created_at`` ordering and keyset pagination all run against the
- deduped ``groups`` table, so this never reads the persons Postgres database; results are
- eventually consistent. ``search`` matches a substring of the raw properties JSON or the group
- key exactly (case-insensitive), preserving the previous endpoint's semantics. The keyset cursor
+ Filtering, full-text search, ``created_at`` ordering and keyset pagination all run against
+ ``raw_groups``, so this never reads the persons Postgres database; results are eventually
+ consistent. ``search`` matches a substring of the latest raw properties JSON or the group key
+ exactly (case-insensitive), preserving the previous endpoint's semantics. The keyset cursor
breaks ``created_at`` ties on ``group_key`` (unique within a team and type).
+
+ Dedup runs in two phases, so memory does not grow with the group type's total property volume.
+ The first phase picks the page's keys and aggregates only ``created_at`` and the search match
+ flag per key. The second phase reads the latest properties for those keys only. A single dedup
+ over the ``groups`` table holds the latest properties of every group of the type in memory,
+ because the search, cursor and ordering can only apply after the dedup. That runs out of memory
+ on large projects.
"""
from posthog.hogql import ast # noqa: PLC0415 — keep HogQL off this widely-imported module's import path
+ from posthog.hogql.database.schema.groups import select_from_groups_table # noqa: PLC0415
+ from posthog.hogql.parser import parse_expr, parse_select # noqa: PLC0415
from posthog.hogql.query import execute_hogql_query # noqa: PLC0415
from posthog.models.team import Team # noqa: PLC0415
team = Team.objects.get(pk=team_id)
- where_exprs: list[ast.Expr] = [
- ast.CompareOperation(
- left=ast.Field(chain=["index"]),
- op=ast.CompareOperationOp.Eq,
- right=ast.Constant(value=group_type_index),
- )
- ]
-
+ # Row-level filters on the group identity hold for every version of a group, so they can apply
+ # before the dedup.
+ key_filter: ast.Expr = ast.Constant(value=True)
if group_key_contains:
- where_exprs.append(
- ast.CompareOperation(
- op=ast.CompareOperationOp.ILike,
- left=ast.Call(name="toString", args=[ast.Field(chain=["key"])]),
- right=ast.Constant(value=f"%{_escape_clickhouse_like(group_key_contains)}%"),
- )
+ key_filter = parse_expr(
+ "toString(key) ILIKE {pattern}",
+ placeholders={"pattern": ast.Constant(value=f"%{_escape_clickhouse_like(group_key_contains)}%")},
)
+ # Filters on versioned columns must see only the latest version, so they apply after the dedup.
+ having_exprs: list[ast.Expr] = []
if search:
- where_exprs.append(
- ast.Or(
- exprs=[
- ast.CompareOperation(
- op=ast.CompareOperationOp.ILike,
- left=ast.Call(name="toString", args=[ast.Field(chain=["properties"])]),
- right=ast.Constant(value=f"%{_escape_clickhouse_like(search)}%"),
- ),
- ast.CompareOperation(
- op=ast.CompareOperationOp.Eq,
- left=ast.Call(name="lower", args=[ast.Call(name="toString", args=[ast.Field(chain=["key"])])]),
- right=ast.Call(name="lower", args=[ast.Constant(value=search)]),
- ),
- ]
+ having_exprs.append(
+ parse_expr(
+ "argMax(toString(properties) ILIKE {pattern}, updated_at) OR lower(toString(key)) = lower({search})",
+ placeholders={
+ "pattern": ast.Constant(value=f"%{_escape_clickhouse_like(search)}%"),
+ "search": ast.Constant(value=search),
+ },
)
)
-
if cursor_created_at_us > 0:
cursor_dt = datetime.datetime.fromtimestamp(cursor_created_at_us / 1_000_000, tz=datetime.UTC)
- where_exprs.append(
- ast.Or(
- exprs=[
- ast.CompareOperation(
- op=ast.CompareOperationOp.Lt,
- left=ast.Field(chain=["created_at"]),
- right=ast.Constant(value=cursor_dt),
- ),
- ast.And(
- exprs=[
- ast.CompareOperation(
- op=ast.CompareOperationOp.Eq,
- left=ast.Field(chain=["created_at"]),
- right=ast.Constant(value=cursor_dt),
- ),
- ast.CompareOperation(
- op=ast.CompareOperationOp.Lt,
- left=ast.Field(chain=["key"]),
- right=ast.Constant(value=cursor_group_key),
- ),
- ]
- ),
- ]
+ having_exprs.append(
+ parse_expr(
+ "argMax(created_at, updated_at) < {cursor_dt}"
+ " OR (argMax(created_at, updated_at) = {cursor_dt} AND key < {cursor_group_key})",
+ placeholders={
+ "cursor_dt": ast.Constant(value=cursor_dt),
+ "cursor_group_key": ast.Constant(value=cursor_group_key),
+ },
)
)
- query = ast.SelectQuery(
- select=[
- ast.Field(chain=["key"]),
- ast.Field(chain=["created_at"]),
- ast.Field(chain=["properties"]),
- ],
- select_from=ast.JoinExpr(table=ast.Field(chain=["groups"])),
- where=ast.And(exprs=where_exprs),
- order_by=[
- ast.OrderExpr(expr=ast.Field(chain=["created_at"]), order="DESC"),
- ast.OrderExpr(expr=ast.Field(chain=["key"]), order="DESC"),
- ],
- limit=ast.Constant(value=limit + 1),
+ page_keys = parse_select(
+ """
+ SELECT index, key
+ FROM raw_groups
+ WHERE index = {group_type_index} AND {key_filter}
+ GROUP BY index, key
+ HAVING {having}
+ ORDER BY argMax(created_at, updated_at) DESC, key DESC
+ LIMIT {limit}
+ """,
+ placeholders={
+ "group_type_index": ast.Constant(value=group_type_index),
+ "key_filter": key_filter,
+ "having": ast.And(exprs=having_exprs) if having_exprs else ast.Constant(value=True),
+ "limit": ast.Constant(value=limit + 1),
+ },
+ )
+ # Without its own LIMIT, the executor's default row limit can drop the extra row that sets has_more.
+ query = parse_select(
+ "SELECT key, created_at, properties FROM {latest_groups} ORDER BY created_at DESC, key DESC LIMIT {limit}",
+ placeholders={
+ "latest_groups": select_from_groups_table(
+ {"created_at": ["created_at"], "properties": ["properties"]}, keys=page_keys
+ ),
+ "limit": ast.Constant(value=limit + 1),
+ },
)
response = execute_hogql_query(query, team=team, query_type="groups_list")
diff --git a/products/groups/backend/tests/test_clickhouse_groups.py b/products/groups/backend/tests/test_clickhouse_groups.py
index 205da471333d..a278d7ca65ed 100644
--- a/products/groups/backend/tests/test_clickhouse_groups.py
+++ b/products/groups/backend/tests/test_clickhouse_groups.py
@@ -2498,16 +2498,31 @@ class TestListGroupsFunction(ClickhouseTestMixin, APIBaseTest):
assert result.groups[1].group_key == "org:2"
assert result.has_more is True
- @time_machine.travel("2021-05-04", tick=False)
+ @time_machine.travel("2021-05-03", tick=False)
+ def test_has_more_at_default_limit(self):
+ for i in range(101):
+ raw_create_group_ch(
+ team_id=self.team.pk, group_type_index=0, group_key=f"org:{i:03d}", properties={}, created_at=now()
+ )
+
+ result = list_groups(team_id=self.team.pk, group_type_index=0)
+
+ assert len(result.groups) == 100
+ assert result.has_more is True
+
+ @time_machine.travel("2021-05-06", tick=False)
def test_pagination_cursor_returns_next_page(self):
- with time_machine.travel("2021-05-01", tick=False):
- create_group(team_id=self.team.pk, group_type_index=0, group_key="org:1", properties={})
- with time_machine.travel("2021-05-02", tick=False):
- create_group(team_id=self.team.pk, group_type_index=0, group_key="org:2", properties={})
- with time_machine.travel("2021-05-03", tick=False):
- create_group(team_id=self.team.pk, group_type_index=0, group_key="org:3", properties={})
+ for day in range(1, 6):
+ with time_machine.travel(f"2021-05-0{day}", tick=False):
+ create_group(
+ team_id=self.team.pk,
+ group_type_index=0,
+ group_key=f"org:{day}",
+ properties={"plan": "pro" if day % 2 else "free"},
+ )
page1 = list_groups(team_id=self.team.pk, group_type_index=0, limit=2)
+ assert [g.group_key for g in page1.groups] == ["org:5", "org:4"]
last = page1.groups[-1]
page2 = list_groups(
@@ -2518,21 +2533,42 @@ class TestListGroupsFunction(ClickhouseTestMixin, APIBaseTest):
limit=2,
)
- assert len(page2.groups) == 1
- assert page2.groups[0].group_key == "org:1"
- assert page2.has_more is False
+ assert [g.group_key for g in page2.groups] == ["org:3", "org:2"]
+ assert page2.has_more is True
+
+ first = page1.groups[0]
+ with self.capture_select_queries() as queries:
+ searched = list_groups(
+ team_id=self.team.pk,
+ group_type_index=0,
+ search="pro",
+ cursor_created_at_us=int(first.created_at.timestamp() * 1_000_000),
+ cursor_group_key=first.group_key,
+ limit=1,
+ )
+
+ assert [g.group_key for g in searched.groups] == ["org:3"]
+ assert searched.has_more is True
+ # Results alone cannot show that both queries stop at limit + 1 keys and that only the
+ # property-reading query aggregates whole properties.
+ sql = next(query for query in queries if "globalIn(tuple(" in query)
+ assert sql.count("LIMIT 2") == 2
+ assert sql.count("argMax(groups.group_properties,") == 1
@time_machine.travel("2021-05-04", tick=False)
def test_pagination_cursor_breaks_created_at_ties_on_group_key(self):
- # Three groups created at the same instant: keyset pagination must fall back to the
+ # Four groups created at the same instant: keyset pagination must fall back to the
# group_key tiebreaker (descending) and neither skip nor duplicate a row across pages.
+ # The tie spans more than limit + 1 keys, so the key-selection query's own tiebreak decides
+ # which keys reach a page.
with time_machine.travel("2021-05-01", tick=False):
create_group(team_id=self.team.pk, group_type_index=0, group_key="org:a", properties={})
create_group(team_id=self.team.pk, group_type_index=0, group_key="org:b", properties={})
create_group(team_id=self.team.pk, group_type_index=0, group_key="org:c", properties={})
+ create_group(team_id=self.team.pk, group_type_index=0, group_key="org:d", properties={})
page1 = list_groups(team_id=self.team.pk, group_type_index=0, limit=2)
- assert [g.group_key for g in page1.groups] == ["org:c", "org:b"]
+ assert [g.group_key for g in page1.groups] == ["org:d", "org:c"]
assert page1.has_more is True
last = page1.groups[-1]
@@ -2543,7 +2579,7 @@ class TestListGroupsFunction(ClickhouseTestMixin, APIBaseTest):
cursor_group_key=last.group_key,
limit=2,
)
- assert [g.group_key for g in page2.groups] == ["org:a"]
+ assert [g.group_key for g in page2.groups] == ["org:b", "org:a"]
assert page2.has_more is False
@time_machine.travel("2021-05-03", tick=False)
@@ -2592,13 +2628,13 @@ class TestListGroupsFunction(ClickhouseTestMixin, APIBaseTest):
@time_machine.travel("2021-05-03", tick=False)
def test_filters_by_group_type_index(self):
- create_group(team_id=self.team.pk, group_type_index=0, group_key="org:1", properties={})
- create_group(team_id=self.team.pk, group_type_index=1, group_key="company:1", properties={})
+ create_group(team_id=self.team.pk, group_type_index=0, group_key="org:1", properties={"name": "Org"})
+ # The same key under another type is a different group; its versions must not merge in.
+ create_group(team_id=self.team.pk, group_type_index=1, group_key="org:1", properties={"name": "Company"})
result = list_groups(team_id=self.team.pk, group_type_index=0)
- assert len(result.groups) == 1
- assert result.groups[0].group_key == "org:1"
+ assert [(g.group_key, g.group_properties) for g in result.groups] == [("org:1", {"name": "Org"})]
@time_machine.travel("2021-05-03", tick=False)
def test_empty_result(self):
@@ -2627,8 +2663,6 @@ class TestListGroupsFunction(ClickhouseTestMixin, APIBaseTest):
assert result.groups[0].group_properties == {}
def test_returns_latest_properties_after_update(self):
- # The groups table dedups by argMax(_timestamp); a newer write for the same (index, key)
- # must win and produce a single row — guards the choice of `groups` over `raw_groups`.
with time_machine.travel("2021-05-01", tick=False):
create_group(team_id=self.team.pk, group_type_index=0, group_key="org:1", properties={"name": "Old"})
with time_machine.travel("2021-05-02", tick=False):
@@ -2644,6 +2678,9 @@ class TestListGroupsFunction(ClickhouseTestMixin, APIBaseTest):
assert len(result.groups) == 1
assert result.groups[0].group_properties == {"name": "New"}
+ assert list_groups(team_id=self.team.pk, group_type_index=0, search="Old").groups == []
+ searched = list_groups(team_id=self.team.pk, group_type_index=0, search="New")
+ assert [(g.group_key, g.group_properties) for g in searched.groups] == [("org:1", {"name": "New"})]
@time_machine.travel("2021-05-03", tick=False)
def test_preserves_complex_property_values(self):