:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(hogql): support account membership queries (#126672)
Arthur Moreira de Deus96aa08c340e7
diff --git a/docs/published/handbook/engineering/databases/hogql-python.md b/docs/published/handbook/engineering/databases/hogql-python.mdindex b37879c78bdb..f190b39d9f11 100644--- a/docs/published/handbook/engineering/databases/hogql-python.md+++ b/docs/published/handbook/engineering/databases/hogql-python.md@@ -143,6 +143,9 @@ Pass the printed SQL and `HogQLContext.values` together to the database driver.Built-in tables also accept the `posthog` namespace, such as `SELECT id FROM posthog.persons` and `SELECT count() FROM posthog.groups`.Explicit aliases work with these names, for example `SELECT p.id FROM posthog.persons AS p`.+Trusted internal ASTs can set `JoinExpr.table_schema` to bind a table without adding it to the public catalog.+User SQL still resolves tables through the catalog. Never bind table names taken from user expressions.+The HogQL database schema is in flux. You will soon be able to explore it in the [PostHog app itself](https://github.com/PostHog/posthog/pull/14591).The most up to date resource is [hogql/database.py](https://github.com/PostHog/posthog/blob/master/posthog/hogql/database.py) on Github. At the time of writing, these tables were available:diff --git a/posthog/hogql/ast.py b/posthog/hogql/ast.pyindex 6d198f417797..3a45c3b67c82 100644--- a/posthog/hogql/ast.py+++ b/posthog/hogql/ast.py@@ -1200,7 +1200,7 @@ class PivotExpr(Expr):group_by: Optional[list[Expr]] = None-@dataclass(kw_only=True, slots=True)+@dataclass(kw_only=True, slots=True, frozen=False)class JoinExpr(Expr):# :TRICKY: When adding new fields, make sure they're handled in visitor.py and resolver.pytype: Optional[TableOrSelectType] = None@@ -1218,6 +1218,8 @@ class JoinExpr(Expr):"Field",]] = None+ table_schema: Optional[Table] = None+ cte_name: Optional[str] = Nonetable_args: Optional[list[Expr]] = Nonealias: Optional[str] = Nonecolumn_aliases: Optional[list[str]] = Nonediff --git a/posthog/hogql/constants.py b/posthog/hogql/constants.pyindex 9e158f2b03b9..24fc86cea8d9 100644--- a/posthog/hogql/constants.py+++ b/posthog/hogql/constants.py@@ -214,6 +214,7 @@ class HogQLGlobalSettings(HogQLQuerySettings):max_expanded_ast_elements: Optional[int] = 4_000_000max_bytes_before_external_group_by: Optional[int] = 0 # default value means we don't swap ordering by to diskenable_analyzer: Optional[bool] = None+ enable_materialized_cte: Optional[bool] = Nonetransform_null_in: Optional[bool] = True# A bugfix workaround that stops clauses that look like# `or(event = '1', event = '2', event = '3')` from being optimized into `event IN ('1', '2', '3')`diff --git a/posthog/hogql/database/schema/person_group_membership.py b/posthog/hogql/database/schema/person_group_membership.pynew file mode 100644index 000000000000..a771a2526864--- /dev/null+++ b/posthog/hogql/database/schema/person_group_membership.py@@ -0,0 +1,42 @@+from posthog.hogql.context import HogQLContext+from posthog.hogql.database.models import (+ DateTimeDatabaseField,+ FieldOrTable,+ IntegerDatabaseField,+ StringDatabaseField,+ Table,+)+++class PersonGroupMembershipTable(Table):+ description: str = (+ "Historical distinct-ID membership in groups. Resolve current persons through person_distinct_ids."+ )+ fields: dict[str, FieldOrTable] = {+ "team_id": IntegerDatabaseField(+ name="team_id", nullable=False, description="Project that owns the membership."+ ),+ "group_type_index": IntegerDatabaseField(+ name="group_type_index", nullable=False, description="Configured account group type index, from 0 to 4."+ ),+ "group_key": StringDatabaseField(+ name="group_key", nullable=False, description="External group key of the account."+ ),+ "distinct_id": StringDatabaseField(+ name="distinct_id",+ nullable=False,+ description="Event distinct ID. Join to person_distinct_ids.distinct_id.",+ ),+ "first_seen": DateTimeDatabaseField(+ name="first_seen", nullable=False, description="Earliest account event for this distinct ID, in UTC."+ ),+ "last_seen": DateTimeDatabaseField(+ name="last_seen", nullable=False, description="Latest account event for this distinct ID, in UTC."+ ),+ }++ def to_printed_clickhouse(self, context: HogQLContext) -> str:+ return "person_group_membership"++ def to_printed_hogql(self) -> str:+ return "posthog.person_group_membership"diff --git a/posthog/hogql/database/test/__snapshots__/test_person_group_membership.ambr b/posthog/hogql/database/test/__snapshots__/test_person_group_membership.ambrnew file mode 100644index 000000000000..492116868af3--- /dev/null+++ b/posthog/hogql/database/test/__snapshots__/test_person_group_membership.ambr@@ -0,0 +1,9 @@+# serializer version: 1+# name: TestMembershipSchema.test_membership_is_namespaced_and_team_guarded+ '''+ SELECT person_group_membership.distinct_id AS distinct_id+ FROM person_group_membership+ WHERE and(equals(person_group_membership.team_id, 99999), equals(person_group_membership.group_key, %(hogql_val_0)s))+ LIMIT 50000+ '''+# ---diff --git a/posthog/hogql/database/test/test_person_group_membership.py b/posthog/hogql/database/test/test_person_group_membership.pynew file mode 100644index 000000000000..3411f341a9c8--- /dev/null+++ b/posthog/hogql/database/test/test_person_group_membership.py@@ -0,0 +1,52 @@+from posthog.test.base import QueryMatchingTest++from django.test import SimpleTestCase++from parameterized import parameterized++from posthog.hogql import ast+from posthog.hogql.context import HogQLContext+from posthog.hogql.database.database import Database+from posthog.hogql.database.schema.person_group_membership import PersonGroupMembershipTable+from posthog.hogql.errors import QueryError+from posthog.hogql.parser import parse_select+from posthog.hogql.printer import prepare_and_print_ast+from posthog.hogql.visitor import clone_expr++from posthog.models import Team+++class TestMembershipSchema(QueryMatchingTest, SimpleTestCase):+ def test_membership_is_namespaced_and_team_guarded(self) -> None:+ database = Database()+ assert not database.has_table(["posthog", "person_group_membership"])+ assert not database.has_table(["person_group_membership"])+ query = parse_select(+ "SELECT distinct_id FROM posthog.person_group_membership WHERE group_key = 'workspace-demo'"+ )+ assert isinstance(query, ast.SelectQuery) and query.select_from is not None+ query.select_from.table_schema = PersonGroupMembershipTable()+ query = clone_expr(query, clear_types=True)+ context = HogQLContext(+ team_id=123,+ team=Team(id=123, project_id=123),+ database=database,+ enable_select_queries=True,+ restricted_properties=set(),+ use_new_events_schema=False,+ )+ sql, _ = prepare_and_print_ast(query, context=context, dialect="clickhouse")+ self.assertQueryMatchesSnapshot(sql)+ assert "team_id" in sql+ assert "person_group_membership.team_id, 123" in sql++ @parameterized.expand([("person_group_membership",), ("posthog.person_group_membership",)])+ def test_membership_cannot_be_queried_by_name(self, table: str) -> None:+ context = HogQLContext(+ team_id=123,+ database=Database(),+ enable_select_queries=True,+ restricted_properties=set(),+ )+ with self.assertRaises(QueryError):+ prepare_and_print_ast(parse_select(f"SELECT distinct_id FROM {table}"), context, "clickhouse")diff --git a/posthog/hogql/resolver.py b/posthog/hogql/resolver.pyindex 8090d2f73d14..8f44e67b1995 100644--- a/posthog/hogql/resolver.py+++ b/posthog/hogql/resolver.py@@ -454,6 +454,18 @@ class FieldCollector(TraversingVisitor):return node+class CteReferenceBinder(TraversingVisitor):+ def __init__(self, names: set[str]) -> None:+ self.names = names++ def visit_join_expr(self, node: ast.JoinExpr) -> None:+ if isinstance(node.table, ast.Field) and node.table_schema is None:+ name = ".".join(str(part) for part in node.table.chain)+ if name in self.names:+ node.cte_name = name+ super().visit_join_expr(node)++class Resolver(CloningVisitor):"""The Resolver visits an AST and 1) resolves all fields, 2) assigns types to nodes, 3) expands all CTEs."""@@ -1517,7 +1529,17 @@ class Resolver(CloningVisitor):if self.dialect == "trino" and node.table_args is not Noneelse None)- cte_table = self.ctes.get(".".join(table_name_chain)) if internal_table_function is None else None+ cte_table = (+ self.ctes.get(".".join(table_name_chain))+ if internal_table_function is None and node.table_schema is None+ else None+ )+ # Only bound references can carry an outer CTE into a generated lazy query.+ if cte_table is None and node.cte_name is not None:+ for inherited_scope in reversed(self.scopes):+ cte_table = (inherited_scope.ctes or {}).get(node.cte_name)+ if cte_table is not None:+ breakif cte_table:assert isinstance(cte_table.expr.type, ast.SelectQueryType | ast.SelectSetQueryType)# Use CTETableType so that fields are properly qualified with the CTE name when printed@@ -1543,6 +1565,7 @@ class Resolver(CloningVisitor):node.type = node_typenode.table = clone_expr(cast(ast.Field, node.table))node.table.type = cte_table_type+ node.cte_name = cte_table.namenode.next_join = self.visit(node.next_join)node.alias = table_alias@@ -1558,7 +1581,7 @@ class Resolver(CloningVisitor):or resolve_internal_trino_logical_table(table_name_chain, self.context))else:- database_table = None+ database_table = node.table_schemaif database_table is None:try:@@ -1602,6 +1625,9 @@ class Resolver(CloningVisitor):if isinstance(database_table, LazyTable):if isinstance(database_table, PersonsTable):+ # Promotion moves predicates before resolution, so preserve their CTE bindings first.+ if node.constraint is not None:+ CteReferenceBinder(set(self.ctes)).visit(node.constraint)# Check for inlineable exprs in the join on the persons tabledatabase_table = database_table.create_new_table_with_filter(node)node_table_type: ast.TableType | ast.LazyTableType = ast.LazyTableType(table=database_table)diff --git a/posthog/hogql/test/test_resolver.py b/posthog/hogql/test/test_resolver.pyindex f63464724532..ff529f0ed20c 100644--- a/posthog/hogql/test/test_resolver.py+++ b/posthog/hogql/test/test_resolver.py@@ -156,6 +156,26 @@ class TestSavedViewBounds(SimpleTestCase):resolve_types(parse_select("select * from leaf"), context, "hogql")+class TestInheritedCtes(SimpleTestCase):+ @parameterized.expand([(False,), (True,)])+ def test_only_bound_cte_references_cross_resolver_scopes(self, bound: bool) -> None:+ context = HogQLContext(database=Database(), enable_select_queries=True)+ parent = resolve_types(+ parse_select("WITH shared AS (SELECT 1 AS value) SELECT value FROM shared"), context, "clickhouse"+ )+ assert isinstance(parent.type, ast.SelectQueryType)+ query = clone_expr(parent, clear_types=True) if bound else parse_select("SELECT value FROM shared")+ assert isinstance(query, ast.SelectQuery)+ query.ctes = None+ if bound:+ resolved = resolve_types(query, context, "clickhouse", scopes=[parent.type])+ assert isinstance(resolved.type, ast.SelectQueryType)+ assert isinstance(resolved.type.columns["value"].resolve_constant_type(context), ast.IntegerType)+ else:+ with self.assertRaises(QueryError):+ resolve_types(query, context, "clickhouse", scopes=[parent.type])++class TestResolver(BaseTest):maxDiff = Nonesnapshot: Anydiff --git a/posthog/hogql/transforms/test/test_lazy_tables.py b/posthog/hogql/transforms/test/test_lazy_tables.pyindex a4500da80dc6..0892c7e32066 100644--- a/posthog/hogql/transforms/test/test_lazy_tables.py+++ b/posthog/hogql/transforms/test/test_lazy_tables.py@@ -39,6 +39,7 @@ from posthog.hogql.query import execute_hogql_queryfrom posthog.hogql.test.utils import pretty_print_in_testsfrom posthog.models import Organization, PropertyDefinition, Team+from posthog.schema_enums import PersonsArgMaxVersionfrom products.data_tools.backend.models.join import DataWarehouseJoin@@ -66,7 +67,69 @@ class TestNamespacedLazyTables(SimpleTestCase):actual = self._compile(query.format(table=f"posthog.{table}"))self.assertEqual(actual.replace(f"posthog__{table}", table), expected)- def _compile(self, query: str) -> str:+ @parameterized.expand(+ [+ (name, query, HogQLQueryModifiers(personsArgMaxVersion=version))+ for name, query in (+ ("unused", "WITH raw_persons AS (SELECT 7 AS marker) SELECT id FROM persons"),+ (+ "used",+ "WITH raw_persons AS (SELECT 7 AS marker) SELECT persons.id FROM persons CROSS JOIN raw_persons",+ ),+ (+ "materialized",+ "WITH raw_persons AS MATERIALIZED (SELECT 7 AS marker) SELECT id FROM persons",+ ),+ (+ "filter",+ "WITH raw_persons AS (SELECT id FROM persons) SELECT id FROM persons WHERE id IN (SELECT id FROM raw_persons)",+ ),+ (+ "promoted_filter",+ "WITH members AS MATERIALIZED (SELECT id FROM persons) SELECT p.id FROM persons AS p "+ "INNER JOIN members AS m ON p.id = m.id AND p.id IN (SELECT id FROM members)",+ ),+ )+ for version in (PersonsArgMaxVersion.V1, PersonsArgMaxVersion.V2)+ ]+ + [+ ("groups", "WITH raw_groups AS (SELECT 7 AS marker) SELECT key FROM groups", HogQLQueryModifiers()),+ (+ "identities",+ "WITH raw_person_distinct_ids AS (SELECT 7 AS marker) SELECT person_id FROM person_distinct_ids",+ HogQLQueryModifiers(),+ ),+ (+ "cohorts",+ "WITH raw_cohort_people AS (SELECT 7 AS marker) SELECT person_id FROM cohort_people",+ HogQLQueryModifiers(),+ ),+ (+ "logs",+ "WITH log_entries AS (SELECT 7 AS marker) SELECT message FROM console_logs_log_entries",+ HogQLQueryModifiers(),+ ),+ (+ "replay",+ "WITH raw_session_replay_events AS (SELECT 7 AS marker) SELECT session_id FROM session_replay_events",+ HogQLQueryModifiers(),+ ),+ ]+ + [+ (+ f"sessions_{version.value}",+ "WITH raw_sessions AS (SELECT 7 AS marker) SELECT session.$session_duration FROM events",+ HogQLQueryModifiers(sessionTableVersion=version),+ )+ for version in (SessionTableVersion.V1, SessionTableVersion.V2, SessionTableVersion.V3)+ ]+ )+ def test_ctes_do_not_shadow_generated_backing_tables(+ self, _name: str, query: str, modifiers: HogQLQueryModifiers+ ) -> None:+ self._compile(query, modifiers)++ def _compile(self, query: str, modifiers: HogQLQueryModifiers | None = None) -> str:context = HogQLContext(team_id=1,team=Team(id=1, organization=Organization()),@@ -74,9 +137,14 @@ class TestNamespacedLazyTables(SimpleTestCase):enable_select_queries=True,use_new_events_schema=False,apply_events_retention_floor=False,+ modifiers=modifiers or HogQLQueryModifiers(),restricted_properties=set(),)- with patch("posthog.hogql.transforms.property_types.load_property_metadata", return_value=PropertyMetadata()):+ with (+ patch("posthog.hogql.transforms.property_types.load_property_metadata", return_value=PropertyMetadata()),+ patch("products.cohorts.backend.models.cohort.Cohort.objects") as cohorts,+ ):+ cohorts.filter.return_value.exclude.return_value.values_list.return_value = []return prepare_and_print_ast(parse_select(query), context, "clickhouse")[0]diff --git a/posthog/hogql/visitor.py b/posthog/hogql/visitor.pyindex e8e54736f2b5..69a07c0161ce 100644--- a/posthog/hogql/visitor.py+++ b/posthog/hogql/visitor.py@@ -906,6 +906,8 @@ class CloningVisitor(Visitor[Any]):end=None if self.clear_locations else node.end,type=None if self.clear_types else node.type,table=self.visit(node.table),+ table_schema=node.table_schema,+ cte_name=node.cte_name,table_args=[self.visit(expr) for expr in node.table_args] if node.table_args is not None else None,next_join=self.visit(node.next_join),table_final=node.table_final,diff --git a/products/experiments/backend/hogql_queries/test/__snapshots__/test_experiment_query_builder.ambr b/products/experiments/backend/hogql_queries/test/__snapshots__/test_experiment_query_builder.ambrindex 70d9a12f81b8..f2d11e8424fb 100644--- a/products/experiments/backend/hogql_queries/test/__snapshots__/test_experiment_query_builder.ambr+++ b/products/experiments/backend/hogql_queries/test/__snapshots__/test_experiment_query_builder.ambr@@ -77,7 +77,7 @@'''# ---# name: TestExperimentQueryBuilderSnapshots.test_built_query_matches_snapshot_exposures_precompute_write.1- '8d3cae031c3db3ed052cc58b07ab25c013527a7324e761853e73006011310c58'+ '8e6e3c90ada6dae75e8cf2aaa75537d8cb3c04e9c0c4c9bfa887dca6ce80b611'# ---# name: TestExperimentQueryBuilderSnapshots.test_built_query_matches_snapshot_funnel_activation_drops_cuped'''@@ -342,7 +342,7 @@'''# ---# name: TestExperimentQueryBuilderSnapshots.test_built_query_matches_snapshot_funnel_precompute_write.1- '99b0aa3d82a91602b532e3fbcde19359a8f566487aac41f3600327eea84aa2dc'+ 'e3401f5b5536c49e54822c1d831cdf92fc81e82fd36773b30cb96c3a7eff84c9'# ---# name: TestExperimentQueryBuilderSnapshots.test_built_query_matches_snapshot_funnel_precomputed'''@@ -804,7 +804,7 @@'''# ---# name: TestExperimentQueryBuilderSnapshots.test_built_query_matches_snapshot_mean_precompute_write.1- '7943f0cc2626c567a5bdfec5c7d9edcab47714f2b71b073441d23d2a4df5fbc8'+ 'c5ca6f2977715faffdaf77ba6b17040979db027db022ad1c73b0cbf1fb722a46'# ---# name: TestExperimentQueryBuilderSnapshots.test_built_query_matches_snapshot_mean_precomputed'''@@ -1478,7 +1478,7 @@'''# ---# name: TestExperimentQueryBuilderSnapshots.test_built_query_matches_snapshot_retention_precompute_write.1- '66967bfa1dbb70d9709f161467a74c51924604649d2e8493dfc73482fd0007d3'+ '8e613e029fb5165c9f2b0cb575e1b31f575a1efbbbcc16dee53102f09ff98e05'# ---# name: TestExperimentQueryBuilderSnapshots.test_built_query_matches_snapshot_retention_precomputed'''