Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(hogql): support account membership queries (#126672)

Arthur Moreira de Deus96aa08c340e7
diff --git a/docs/published/handbook/engineering/databases/hogql-python.md b/docs/published/handbook/engineering/databases/hogql-python.md
index b37879c78bdb..f190b39d9f11 100644
--- a/docs/published/handbook/engineering/databases/hogql-python.md
+++ b/docs/published/handbook/engineering/databases/hogql-python.md
@@ -143,6 +143,9 @@ Pass the printed SQL and `HogQLContext.values` together to the database driver.
Built-in tables also accept the `posthog` namespace, such as `SELECT id FROM posthog.persons` and `SELECT count() FROM posthog.groups`.
Explicit aliases work with these names, for example `SELECT p.id FROM posthog.persons AS p`.
+Trusted internal ASTs can set `JoinExpr.table_schema` to bind a table without adding it to the public catalog.
+User SQL still resolves tables through the catalog. Never bind table names taken from user expressions.
+
The HogQL database schema is in flux. You will soon be able to explore it in the [PostHog app itself](https://github.com/PostHog/posthog/pull/14591).
The most up to date resource is [hogql/database.py](https://github.com/PostHog/posthog/blob/master/posthog/hogql/database.py) on Github. At the time of writing, these tables were available:
diff --git a/posthog/hogql/ast.py b/posthog/hogql/ast.py
index 6d198f417797..3a45c3b67c82 100644
--- a/posthog/hogql/ast.py
+++ b/posthog/hogql/ast.py
@@ -1200,7 +1200,7 @@ class PivotExpr(Expr):
group_by: Optional[list[Expr]] = None
-@dataclass(kw_only=True, slots=True)
+@dataclass(kw_only=True, slots=True, frozen=False)
class JoinExpr(Expr):
# :TRICKY: When adding new fields, make sure they're handled in visitor.py and resolver.py
type: Optional[TableOrSelectType] = None
@@ -1218,6 +1218,8 @@ class JoinExpr(Expr):
"Field",
]
] = None
+ table_schema: Optional[Table] = None
+ cte_name: Optional[str] = None
table_args: Optional[list[Expr]] = None
alias: Optional[str] = None
column_aliases: Optional[list[str]] = None
diff --git a/posthog/hogql/constants.py b/posthog/hogql/constants.py
index 9e158f2b03b9..24fc86cea8d9 100644
--- a/posthog/hogql/constants.py
+++ b/posthog/hogql/constants.py
@@ -214,6 +214,7 @@ class HogQLGlobalSettings(HogQLQuerySettings):
max_expanded_ast_elements: Optional[int] = 4_000_000
max_bytes_before_external_group_by: Optional[int] = 0 # default value means we don't swap ordering by to disk
enable_analyzer: Optional[bool] = None
+ enable_materialized_cte: Optional[bool] = None
transform_null_in: Optional[bool] = True
# A bugfix workaround that stops clauses that look like
# `or(event = '1', event = '2', event = '3')` from being optimized into `event IN ('1', '2', '3')`
diff --git a/posthog/hogql/database/schema/person_group_membership.py b/posthog/hogql/database/schema/person_group_membership.py
new file mode 100644
index 000000000000..a771a2526864
--- /dev/null
+++ b/posthog/hogql/database/schema/person_group_membership.py
@@ -0,0 +1,42 @@
+from posthog.hogql.context import HogQLContext
+from posthog.hogql.database.models import (
+ DateTimeDatabaseField,
+ FieldOrTable,
+ IntegerDatabaseField,
+ StringDatabaseField,
+ Table,
+)
+
+
+class PersonGroupMembershipTable(Table):
+ description: str = (
+ "Historical distinct-ID membership in groups. Resolve current persons through person_distinct_ids."
+ )
+ fields: dict[str, FieldOrTable] = {
+ "team_id": IntegerDatabaseField(
+ name="team_id", nullable=False, description="Project that owns the membership."
+ ),
+ "group_type_index": IntegerDatabaseField(
+ name="group_type_index", nullable=False, description="Configured account group type index, from 0 to 4."
+ ),
+ "group_key": StringDatabaseField(
+ name="group_key", nullable=False, description="External group key of the account."
+ ),
+ "distinct_id": StringDatabaseField(
+ name="distinct_id",
+ nullable=False,
+ description="Event distinct ID. Join to person_distinct_ids.distinct_id.",
+ ),
+ "first_seen": DateTimeDatabaseField(
+ name="first_seen", nullable=False, description="Earliest account event for this distinct ID, in UTC."
+ ),
+ "last_seen": DateTimeDatabaseField(
+ name="last_seen", nullable=False, description="Latest account event for this distinct ID, in UTC."
+ ),
+ }
+
+ def to_printed_clickhouse(self, context: HogQLContext) -> str:
+ return "person_group_membership"
+
+ def to_printed_hogql(self) -> str:
+ return "posthog.person_group_membership"
diff --git a/posthog/hogql/database/test/__snapshots__/test_person_group_membership.ambr b/posthog/hogql/database/test/__snapshots__/test_person_group_membership.ambr
new file mode 100644
index 000000000000..492116868af3
--- /dev/null
+++ b/posthog/hogql/database/test/__snapshots__/test_person_group_membership.ambr
@@ -0,0 +1,9 @@
+# serializer version: 1
+# name: TestMembershipSchema.test_membership_is_namespaced_and_team_guarded
+ '''
+ SELECT person_group_membership.distinct_id AS distinct_id
+ FROM person_group_membership
+ WHERE and(equals(person_group_membership.team_id, 99999), equals(person_group_membership.group_key, %(hogql_val_0)s))
+ LIMIT 50000
+ '''
+# ---
diff --git a/posthog/hogql/database/test/test_person_group_membership.py b/posthog/hogql/database/test/test_person_group_membership.py
new file mode 100644
index 000000000000..3411f341a9c8
--- /dev/null
+++ b/posthog/hogql/database/test/test_person_group_membership.py
@@ -0,0 +1,52 @@
+from posthog.test.base import QueryMatchingTest
+
+from django.test import SimpleTestCase
+
+from parameterized import parameterized
+
+from posthog.hogql import ast
+from posthog.hogql.context import HogQLContext
+from posthog.hogql.database.database import Database
+from posthog.hogql.database.schema.person_group_membership import PersonGroupMembershipTable
+from posthog.hogql.errors import QueryError
+from posthog.hogql.parser import parse_select
+from posthog.hogql.printer import prepare_and_print_ast
+from posthog.hogql.visitor import clone_expr
+
+from posthog.models import Team
+
+
+class TestMembershipSchema(QueryMatchingTest, SimpleTestCase):
+ def test_membership_is_namespaced_and_team_guarded(self) -> None:
+ database = Database()
+ assert not database.has_table(["posthog", "person_group_membership"])
+ assert not database.has_table(["person_group_membership"])
+ query = parse_select(
+ "SELECT distinct_id FROM posthog.person_group_membership WHERE group_key = 'workspace-demo'"
+ )
+ assert isinstance(query, ast.SelectQuery) and query.select_from is not None
+ query.select_from.table_schema = PersonGroupMembershipTable()
+ query = clone_expr(query, clear_types=True)
+ context = HogQLContext(
+ team_id=123,
+ team=Team(id=123, project_id=123),
+ database=database,
+ enable_select_queries=True,
+ restricted_properties=set(),
+ use_new_events_schema=False,
+ )
+ sql, _ = prepare_and_print_ast(query, context=context, dialect="clickhouse")
+ self.assertQueryMatchesSnapshot(sql)
+ assert "team_id" in sql
+ assert "person_group_membership.team_id, 123" in sql
+
+ @parameterized.expand([("person_group_membership",), ("posthog.person_group_membership",)])
+ def test_membership_cannot_be_queried_by_name(self, table: str) -> None:
+ context = HogQLContext(
+ team_id=123,
+ database=Database(),
+ enable_select_queries=True,
+ restricted_properties=set(),
+ )
+ with self.assertRaises(QueryError):
+ prepare_and_print_ast(parse_select(f"SELECT distinct_id FROM {table}"), context, "clickhouse")
diff --git a/posthog/hogql/resolver.py b/posthog/hogql/resolver.py
index 8090d2f73d14..8f44e67b1995 100644
--- a/posthog/hogql/resolver.py
+++ b/posthog/hogql/resolver.py
@@ -454,6 +454,18 @@ class FieldCollector(TraversingVisitor):
return node
+class CteReferenceBinder(TraversingVisitor):
+ def __init__(self, names: set[str]) -> None:
+ self.names = names
+
+ def visit_join_expr(self, node: ast.JoinExpr) -> None:
+ if isinstance(node.table, ast.Field) and node.table_schema is None:
+ name = ".".join(str(part) for part in node.table.chain)
+ if name in self.names:
+ node.cte_name = name
+ super().visit_join_expr(node)
+
+
class Resolver(CloningVisitor):
"""The Resolver visits an AST and 1) resolves all fields, 2) assigns types to nodes, 3) expands all CTEs."""
@@ -1517,7 +1529,17 @@ class Resolver(CloningVisitor):
if self.dialect == "trino" and node.table_args is not None
else None
)
- cte_table = self.ctes.get(".".join(table_name_chain)) if internal_table_function is None else None
+ cte_table = (
+ self.ctes.get(".".join(table_name_chain))
+ if internal_table_function is None and node.table_schema is None
+ else None
+ )
+ # Only bound references can carry an outer CTE into a generated lazy query.
+ if cte_table is None and node.cte_name is not None:
+ for inherited_scope in reversed(self.scopes):
+ cte_table = (inherited_scope.ctes or {}).get(node.cte_name)
+ if cte_table is not None:
+ break
if cte_table:
assert isinstance(cte_table.expr.type, ast.SelectQueryType | ast.SelectSetQueryType)
# Use CTETableType so that fields are properly qualified with the CTE name when printed
@@ -1543,6 +1565,7 @@ class Resolver(CloningVisitor):
node.type = node_type
node.table = clone_expr(cast(ast.Field, node.table))
node.table.type = cte_table_type
+ node.cte_name = cte_table.name
node.next_join = self.visit(node.next_join)
node.alias = table_alias
@@ -1558,7 +1581,7 @@ class Resolver(CloningVisitor):
or resolve_internal_trino_logical_table(table_name_chain, self.context)
)
else:
- database_table = None
+ database_table = node.table_schema
if database_table is None:
try:
@@ -1602,6 +1625,9 @@ class Resolver(CloningVisitor):
if isinstance(database_table, LazyTable):
if isinstance(database_table, PersonsTable):
+ # Promotion moves predicates before resolution, so preserve their CTE bindings first.
+ if node.constraint is not None:
+ CteReferenceBinder(set(self.ctes)).visit(node.constraint)
# Check for inlineable exprs in the join on the persons table
database_table = database_table.create_new_table_with_filter(node)
node_table_type: ast.TableType | ast.LazyTableType = ast.LazyTableType(table=database_table)
diff --git a/posthog/hogql/test/test_resolver.py b/posthog/hogql/test/test_resolver.py
index f63464724532..ff529f0ed20c 100644
--- a/posthog/hogql/test/test_resolver.py
+++ b/posthog/hogql/test/test_resolver.py
@@ -156,6 +156,26 @@ class TestSavedViewBounds(SimpleTestCase):
resolve_types(parse_select("select * from leaf"), context, "hogql")
+class TestInheritedCtes(SimpleTestCase):
+ @parameterized.expand([(False,), (True,)])
+ def test_only_bound_cte_references_cross_resolver_scopes(self, bound: bool) -> None:
+ context = HogQLContext(database=Database(), enable_select_queries=True)
+ parent = resolve_types(
+ parse_select("WITH shared AS (SELECT 1 AS value) SELECT value FROM shared"), context, "clickhouse"
+ )
+ assert isinstance(parent.type, ast.SelectQueryType)
+ query = clone_expr(parent, clear_types=True) if bound else parse_select("SELECT value FROM shared")
+ assert isinstance(query, ast.SelectQuery)
+ query.ctes = None
+ if bound:
+ resolved = resolve_types(query, context, "clickhouse", scopes=[parent.type])
+ assert isinstance(resolved.type, ast.SelectQueryType)
+ assert isinstance(resolved.type.columns["value"].resolve_constant_type(context), ast.IntegerType)
+ else:
+ with self.assertRaises(QueryError):
+ resolve_types(query, context, "clickhouse", scopes=[parent.type])
+
+
class TestResolver(BaseTest):
maxDiff = None
snapshot: Any
diff --git a/posthog/hogql/transforms/test/test_lazy_tables.py b/posthog/hogql/transforms/test/test_lazy_tables.py
index a4500da80dc6..0892c7e32066 100644
--- a/posthog/hogql/transforms/test/test_lazy_tables.py
+++ b/posthog/hogql/transforms/test/test_lazy_tables.py
@@ -39,6 +39,7 @@ from posthog.hogql.query import execute_hogql_query
from posthog.hogql.test.utils import pretty_print_in_tests
from posthog.models import Organization, PropertyDefinition, Team
+from posthog.schema_enums import PersonsArgMaxVersion
from products.data_tools.backend.models.join import DataWarehouseJoin
@@ -66,7 +67,69 @@ class TestNamespacedLazyTables(SimpleTestCase):
actual = self._compile(query.format(table=f"posthog.{table}"))
self.assertEqual(actual.replace(f"posthog__{table}", table), expected)
- def _compile(self, query: str) -> str:
+ @parameterized.expand(
+ [
+ (name, query, HogQLQueryModifiers(personsArgMaxVersion=version))
+ for name, query in (
+ ("unused", "WITH raw_persons AS (SELECT 7 AS marker) SELECT id FROM persons"),
+ (
+ "used",
+ "WITH raw_persons AS (SELECT 7 AS marker) SELECT persons.id FROM persons CROSS JOIN raw_persons",
+ ),
+ (
+ "materialized",
+ "WITH raw_persons AS MATERIALIZED (SELECT 7 AS marker) SELECT id FROM persons",
+ ),
+ (
+ "filter",
+ "WITH raw_persons AS (SELECT id FROM persons) SELECT id FROM persons WHERE id IN (SELECT id FROM raw_persons)",
+ ),
+ (
+ "promoted_filter",
+ "WITH members AS MATERIALIZED (SELECT id FROM persons) SELECT p.id FROM persons AS p "
+ "INNER JOIN members AS m ON p.id = m.id AND p.id IN (SELECT id FROM members)",
+ ),
+ )
+ for version in (PersonsArgMaxVersion.V1, PersonsArgMaxVersion.V2)
+ ]
+ + [
+ ("groups", "WITH raw_groups AS (SELECT 7 AS marker) SELECT key FROM groups", HogQLQueryModifiers()),
+ (
+ "identities",
+ "WITH raw_person_distinct_ids AS (SELECT 7 AS marker) SELECT person_id FROM person_distinct_ids",
+ HogQLQueryModifiers(),
+ ),
+ (
+ "cohorts",
+ "WITH raw_cohort_people AS (SELECT 7 AS marker) SELECT person_id FROM cohort_people",
+ HogQLQueryModifiers(),
+ ),
+ (
+ "logs",
+ "WITH log_entries AS (SELECT 7 AS marker) SELECT message FROM console_logs_log_entries",
+ HogQLQueryModifiers(),
+ ),
+ (
+ "replay",
+ "WITH raw_session_replay_events AS (SELECT 7 AS marker) SELECT session_id FROM session_replay_events",
+ HogQLQueryModifiers(),
+ ),
+ ]
+ + [
+ (
+ f"sessions_{version.value}",
+ "WITH raw_sessions AS (SELECT 7 AS marker) SELECT session.$session_duration FROM events",
+ HogQLQueryModifiers(sessionTableVersion=version),
+ )
+ for version in (SessionTableVersion.V1, SessionTableVersion.V2, SessionTableVersion.V3)
+ ]
+ )
+ def test_ctes_do_not_shadow_generated_backing_tables(
+ self, _name: str, query: str, modifiers: HogQLQueryModifiers
+ ) -> None:
+ self._compile(query, modifiers)
+
+ def _compile(self, query: str, modifiers: HogQLQueryModifiers | None = None) -> str:
context = HogQLContext(
team_id=1,
team=Team(id=1, organization=Organization()),
@@ -74,9 +137,14 @@ class TestNamespacedLazyTables(SimpleTestCase):
enable_select_queries=True,
use_new_events_schema=False,
apply_events_retention_floor=False,
+ modifiers=modifiers or HogQLQueryModifiers(),
restricted_properties=set(),
)
- with patch("posthog.hogql.transforms.property_types.load_property_metadata", return_value=PropertyMetadata()):
+ with (
+ patch("posthog.hogql.transforms.property_types.load_property_metadata", return_value=PropertyMetadata()),
+ patch("products.cohorts.backend.models.cohort.Cohort.objects") as cohorts,
+ ):
+ cohorts.filter.return_value.exclude.return_value.values_list.return_value = []
return prepare_and_print_ast(parse_select(query), context, "clickhouse")[0]
diff --git a/posthog/hogql/visitor.py b/posthog/hogql/visitor.py
index e8e54736f2b5..69a07c0161ce 100644
--- a/posthog/hogql/visitor.py
+++ b/posthog/hogql/visitor.py
@@ -906,6 +906,8 @@ class CloningVisitor(Visitor[Any]):
end=None if self.clear_locations else node.end,
type=None if self.clear_types else node.type,
table=self.visit(node.table),
+ table_schema=node.table_schema,
+ cte_name=node.cte_name,
table_args=[self.visit(expr) for expr in node.table_args] if node.table_args is not None else None,
next_join=self.visit(node.next_join),
table_final=node.table_final,
diff --git a/products/experiments/backend/hogql_queries/test/__snapshots__/test_experiment_query_builder.ambr b/products/experiments/backend/hogql_queries/test/__snapshots__/test_experiment_query_builder.ambr
index 70d9a12f81b8..f2d11e8424fb 100644
--- a/products/experiments/backend/hogql_queries/test/__snapshots__/test_experiment_query_builder.ambr
+++ b/products/experiments/backend/hogql_queries/test/__snapshots__/test_experiment_query_builder.ambr
@@ -77,7 +77,7 @@
'''
# ---
# name: TestExperimentQueryBuilderSnapshots.test_built_query_matches_snapshot_exposures_precompute_write.1
- '8d3cae031c3db3ed052cc58b07ab25c013527a7324e761853e73006011310c58'
+ '8e6e3c90ada6dae75e8cf2aaa75537d8cb3c04e9c0c4c9bfa887dca6ce80b611'
# ---
# name: TestExperimentQueryBuilderSnapshots.test_built_query_matches_snapshot_funnel_activation_drops_cuped
'''
@@ -342,7 +342,7 @@
'''
# ---
# name: TestExperimentQueryBuilderSnapshots.test_built_query_matches_snapshot_funnel_precompute_write.1
- '99b0aa3d82a91602b532e3fbcde19359a8f566487aac41f3600327eea84aa2dc'
+ 'e3401f5b5536c49e54822c1d831cdf92fc81e82fd36773b30cb96c3a7eff84c9'
# ---
# name: TestExperimentQueryBuilderSnapshots.test_built_query_matches_snapshot_funnel_precomputed
'''
@@ -804,7 +804,7 @@
'''
# ---
# name: TestExperimentQueryBuilderSnapshots.test_built_query_matches_snapshot_mean_precompute_write.1
- '7943f0cc2626c567a5bdfec5c7d9edcab47714f2b71b073441d23d2a4df5fbc8'
+ 'c5ca6f2977715faffdaf77ba6b17040979db027db022ad1c73b0cbf1fb722a46'
# ---
# name: TestExperimentQueryBuilderSnapshots.test_built_query_matches_snapshot_mean_precomputed
'''
@@ -1478,7 +1478,7 @@
'''
# ---
# name: TestExperimentQueryBuilderSnapshots.test_built_query_matches_snapshot_retention_precompute_write.1
- '66967bfa1dbb70d9709f161467a74c51924604649d2e8493dfc73482fd0007d3'
+ '8e613e029fb5165c9f2b0cb575e1b31f575a1efbbbcc16dee53102f09ff98e05'
# ---
# name: TestExperimentQueryBuilderSnapshots.test_built_query_matches_snapshot_retention_precomputed
'''