:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(customer-analytics): query account members from memberships (#126673)
Arthur Moreira de Deusd45756afd1a7
diff --git a/posthog/hogql_queries/actors_query_runner.py b/posthog/hogql_queries/actors_query_runner.pyindex 7812ee8205d0..f4b3dd467a37 100644--- a/posthog/hogql_queries/actors_query_runner.py+++ b/posthog/hogql_queries/actors_query_runner.py@@ -1,10 +1,12 @@import itertoolsfrom collections.abc import Iterable, Iterator, Sequence-from typing import TYPE_CHECKING, Any, Optional+from typing import TYPE_CHECKING, Any, Optional, castif TYPE_CHECKING:from posthog.event_usage import AnalyticsProps+ from products.customer_analytics.backend.facade.queries import AccountUsersQueryRunner+from posthoganalytics import feature_enabledfrom posthog.schema import (@@ -18,6 +20,7 @@ from posthog.schema import (from posthog.hogql import astfrom posthog.hogql.constants import HogQLQuerySettings, LimitContext+from posthog.hogql.context import HogQLContextfrom posthog.hogql.parser import parse_expr, parse_order_exprfrom posthog.hogql.printer import prepare_and_print_astfrom posthog.hogql.property import has_aggregation@@ -70,6 +73,16 @@ class ActorsQueryRunner(AnalyticsQueryRunner[ActorsQueryResponse]):def _child_runners(self) -> Iterable[QueryRunner]:return [self.source_query_runner] if self.source_query_runner is not None else []+ def _get_account_users_source(self) -> "AccountUsersQueryRunner | None":+ if self.query.source is not None and self.query.source.kind == "AccountUsersQuery":+ return cast("AccountUsersQueryRunner", self.source_query_runner)+ return None++ def build_hogql_context(self, **kwargs: object) -> HogQLContext:+ context = super().build_hogql_context(**kwargs)+ source = self._get_account_users_source()+ return source.get_actors_hogql_context(context) if source is not None else context+def _on_user_changed(self) -> None:super()._on_user_changed()self.strategy = self.determine_strategy()@@ -88,6 +101,9 @@ class ActorsQueryRunner(AnalyticsQueryRunner[ActorsQueryResponse]):# hide the change from the base run(), leaving the shared database and access-control# snapshot scoped to the previous user.self.set_user(user)+ account_users_source = self._get_account_users_source()+ if account_users_source is not None:+ account_users_source.validate_execution_mode(execution_mode)return super().run(execution_mode,user,@@ -125,6 +141,9 @@ class ActorsQueryRunner(AnalyticsQueryRunner[ActorsQueryResponse]):return self.source_query_runner.is_session_aggregationdef determine_strategy(self) -> ActorStrategy:+ account_users_source = self._get_account_users_source()+ if account_users_source is not None:+ return account_users_source.get_actors_strategy(self.query, self.paginator)if self.group_type_index is not None:return GroupStrategy(self.group_type_index,@@ -218,6 +237,7 @@ class ActorsQueryRunner(AnalyticsQueryRunner[ActorsQueryResponse]):return column_index_events, self.strategy.get_recordings(matching_events_list)def _calculate_internal(self) -> ActorsQueryResponse:+ account_users_source = self._get_account_users_source()response = self.paginator.execute_hogql_query(query_type="ActorsQuery",query=self.to_query(),@@ -226,6 +246,7 @@ class ActorsQueryRunner(AnalyticsQueryRunner[ActorsQueryResponse]):timings=self.timings,modifiers=self.modifiers,context=self.build_hogql_context(),+ settings=account_users_source.get_actors_query_settings() if account_users_source else None,)input_columns = self.input_columns()missing_actors_count = None@@ -303,6 +324,9 @@ class ActorsQueryRunner(AnalyticsQueryRunner[ActorsQueryResponse]):super().validate()if self.source_query_runner is not None:self.source_query_runner.validate()+ account_users_source = self._get_account_users_source()+ if account_users_source is not None:+ account_users_source.validate_actors_query(self.query)def input_columns(self) -> list[str]:strategy_input_cols = self.strategy.input_columns()@@ -477,6 +501,7 @@ class ActorsQueryRunner(AnalyticsQueryRunner[ActorsQueryResponse]):else:order_by = []+ source_query: ast.SelectQuery | ast.SelectSetQuery | None = Nonewith self.timings.measure("select"):select_query = ast.SelectQuery(select=columns,@@ -484,7 +509,9 @@ class ActorsQueryRunner(AnalyticsQueryRunner[ActorsQueryResponse]):having=having,group_by=group_by if has_any_aggregation else None,order_by=order_by,- settings=HogQLQuerySettings(join_algorithm="auto", optimize_aggregation_in_order=True),+ settings=HogQLQuerySettings(+ join_algorithm="auto", optimize_aggregation_in_order=self._get_account_users_source() is None+ ),)if not self.query.source:select_query.select_from = ast.JoinExpr(table=ast.Field(chain=[self.strategy.origin]))@@ -522,20 +549,21 @@ class ActorsQueryRunner(AnalyticsQueryRunner[ActorsQueryResponse]):):select_query.select.append(ast.Field(chain=[source_distinct_id_column]))- try:- prepare_and_print_ast(- select_query,- context=self.build_hogql_context(- team=self.team,- enable_select_queries=True,- timings=self.timings,- modifiers=self.modifiers,- ),- dialect="clickhouse",- )- return select_query- except Exception:- pass+ if self._get_account_users_source() is None:+ try:+ prepare_and_print_ast(+ select_query,+ context=self.build_hogql_context(+ team=self.team,+ enable_select_queries=True,+ timings=self.timings,+ modifiers=self.modifiers,+ ),+ dialect="clickhouse",+ )+ return select_query+ except Exception:+ passorigin = self.strategy.origin@@ -577,7 +605,7 @@ class ActorsQueryRunner(AnalyticsQueryRunner[ActorsQueryResponse]):alias=source_alias,next_join=ast.JoinExpr(table=ast.Field(chain=[origin]),- join_type="INNER JOIN",+ join_type="GLOBAL INNER JOIN" if self._get_account_users_source() is not None else "INNER JOIN",constraint=ast.JoinConstraint(expr=join_on,constraint_type="ON",@@ -585,7 +613,12 @@ class ActorsQueryRunner(AnalyticsQueryRunner[ActorsQueryResponse]):),)- return select_query+ account_users_source = self._get_account_users_source()+ return (+ account_users_source.prepare_actors_query(select_query, source_query)+ if account_users_source and source_query is not None+ else select_query+ )def to_actors_query(self) -> ast.SelectQuery | ast.SelectSetQuery:return self.to_query()@@ -600,6 +633,9 @@ class ActorsQueryRunner(AnalyticsQueryRunner[ActorsQueryResponse]):return nodedef _get_person_display_name_column(self) -> ast.Expr:+ account_users_source = self._get_account_users_source()+ if account_users_source is not None:+ return account_users_source.get_person_display_name_column()property_keys = self.team.person_display_name_properties or PERSON_DEFAULT_DISPLAY_NAME_PROPERTIESprops = person_display_name_property_exprs(property_keys, "properties")# nosemgrep: hogql-fstring-audit (property_keys from team config is admin-only, not user input)diff --git a/posthog/settings/data_stores.py b/posthog/settings/data_stores.pyindex f944a2753f31..d1baee8d4c87 100644--- a/posthog/settings/data_stores.py+++ b/posthog/settings/data_stores.py@@ -869,6 +869,12 @@ PATCH_EVENT_LIST_MAX_OFFSET_PER_TEAM: set[int] = get_from_env(CLICKHOUSE_EVENT_LIST_MAX_THREADS: int = get_from_env("CLICKHOUSE_EVENT_LIST_MAX_THREADS", 50, type_cast=int)+CLICKHOUSE_ACCOUNT_USERS_MAX_THREADS: int | None = get_from_env(+ "CLICKHOUSE_ACCOUNT_USERS_MAX_THREADS", optional=True, type_cast=int+)+if CLICKHOUSE_ACCOUNT_USERS_MAX_THREADS is not None and CLICKHOUSE_ACCOUNT_USERS_MAX_THREADS <= 0:+ raise ImproperlyConfigured("CLICKHOUSE_ACCOUNT_USERS_MAX_THREADS must be a positive integer.")+WAREHOUSE_SOURCES_QUEUE_PARTITION_DATABASE_URL: str = os.getenv("WAREHOUSE_SOURCES_QUEUE_PARTITION_DATABASE_URL", "")WAREHOUSE_SOURCES_QUEUE_PARTITION_SLACK_WEBHOOK_URL: str = os.getenv("WAREHOUSE_SOURCES_QUEUE_PARTITION_SLACK_WEBHOOK_URL", ""diff --git a/products/customer_analytics/backend/facade/api.py b/products/customer_analytics/backend/facade/api.pyindex bf9e3b57ec94..437f0da31e0a 100644--- a/products/customer_analytics/backend/facade/api.py+++ b/products/customer_analytics/backend/facade/api.py@@ -28,7 +28,7 @@ from django.conf import settingsfrom django.contrib.postgres.aggregates import ArrayAggfrom django.contrib.postgres.fields import ArrayFieldfrom django.core.cache import cache-from django.core.exceptions import ValidationError+from django.core.exceptions import PermissionDenied, ValidationErrorfrom django.db import IntegrityError, transactionfrom django.db.models import (Aggregate,@@ -132,7 +132,6 @@ from products.customer_analytics.backend.logic.account_filters import (parse_email_search,)from products.customer_analytics.backend.logic.account_logo import resolve_logo_domain-from products.customer_analytics.backend.logic.account_member_search import get_account_users_organization_idfrom products.customer_analytics.backend.logic.custom_property_definitions import (apply_option_side_effects,coerce_is_big_number,@@ -399,8 +398,7 @@ def _resolve_accessible_account(def validate_account_users_access(team: Team, user: "User | None", account_id: str) -> None:- """Raise `UserAccessControlError` unless `user` may read the users of this account."""- get_account_users_organization_id(team=team, user=user, account_id=account_id)+ raise PermissionDenied("Run the account-user query synchronously.")def get_account(diff --git a/products/customer_analytics/backend/hogql_queries/account_users_query_runner.py b/products/customer_analytics/backend/hogql_queries/account_users_query_runner.pyindex 95547d94a4bd..1a460fac1094 100644--- a/products/customer_analytics/backend/hogql_queries/account_users_query_runner.py+++ b/products/customer_analytics/backend/hogql_queries/account_users_query_runner.py@@ -1,83 +1,179 @@-from posthog.schema import AccountUsersQuery, HogQLQueryResponse+from typing import cast++from django.conf import settings++from posthog.schema import AccountUsersQuery, ActorsQuery, HogQLQueryResponsefrom posthog.hogql import ast+from posthog.hogql.constants import HogQLGlobalSettings, HogQLQuerySettings+from posthog.hogql.context import HogQLContext+from posthog.hogql.database.models import LazyTableToAdd+from posthog.hogql.database.schema.persons import PersonsTable+from posthog.hogql.errors import QueryErrorfrom posthog.hogql.parser import parse_select-from posthog.hogql.query import execute_hogql_query+from posthog.hogql.visitor import TraversingVisitor-from posthog.hogql_queries.query_runner import AnalyticsQueryRunner-from posthog.models.user import User+from posthog.hogql_queries.actor_strategies import ActorStrategy, PersonStrategy+from posthog.hogql_queries.paginators import HogQLHasMorePaginator+from posthog.hogql_queries.query_runner import AnalyticsQueryRunner, ExecutionMode+from posthog.models import User+from posthog.schema_enums import PersonsArgMaxVersion-from products.access_control.backend.facade.user_access_control import UserAccessControlError-from products.customer_analytics.backend.constants import ACCOUNT_USERS_QUERY_STATUS_LABEL_PREFIX-from products.customer_analytics.backend.logic.account_member_search import (- get_account_users_organization_id,- list_organization_member_distinct_ids,+from products.access_control.backend.facade.api import (+ get_restricted_properties_with_group_type_index_for_team,+ split_restricted_property_names,)+from products.access_control.backend.facade.user_access_control import UserAccessControlError+from products.customer_analytics.backend.logic.account_persons import AccountPersonsReader-class AccountUsersQueryRunner(AnalyticsQueryRunner[HogQLQueryResponse]):- """Persons who use an account. Runs only as an `ActorsQuery` source, which hydrates the persons.+class AccountUsersStrategy(PersonStrategy):+ def filter_conditions(self) -> list[ast.Expr]:+ conditions = PersonStrategy(+ team=self.team,+ user=self.user,+ paginator=self.paginator,+ query=self.query.model_copy(update={"search": None}),+ ).filter_conditions()+ if self.query.search and self.query.search.strip():+ source = self.query.source+ assert isinstance(source, AccountUsersQuery)+ reader = AccountPersonsReader.create(team=self.team, user=self.user, account_id=source.accountId)+ if reader is None:+ raise UserAccessControlError("account", "viewer")+ restrictions = get_restricted_properties_with_group_type_index_for_team(+ user=self.user, team_id=self.team.pk+ )+ condition = reader.get_search_filter(+ self.query.search.strip(), split_restricted_property_names(restrictions).person+ )+ conditions.append(condition)+ return conditions- The person set comes from the PostHog Cloud organization whose ID is the account's external ID,- so it only works in PostHog's own project.- """+class AccountUsersPersonsTable(PersonsTable):+ def lazy_select(+ self, table_to_add: LazyTableToAdd, context: HogQLContext, node: ast.SelectQuery+ ) -> ast.SelectQuery:+ query = cast(ast.SelectQuery, super().lazy_select(table_to_add, context, node))+ query.settings = (query.settings or HogQLQuerySettings()).model_copy(+ update={"optimize_aggregation_in_order": False}+ )+ if self.filter is not None and query.where is not None:+ query.prewhere = query.where+ query.where = None+ return query+++class AccountUsersMembershipReferences(TraversingVisitor):+ def __init__(self, source_query: ast.SelectQuery | ast.SelectSetQuery) -> None:+ super().__init__()+ self.source_query = source_query++ def visit_join_expr(self, node: ast.JoinExpr) -> None:+ if node.table is self.source_query:+ node.table = ast.Field(chain=["__account_members"])+ super().visit_join_expr(node)+++class AccountUsersQueryRunner(AnalyticsQueryRunner[HogQLQueryResponse]):query: AccountUsersQuery+ def validate_execution_mode(self, execution_mode: ExecutionMode) -> None:+ if execution_mode not in {+ ExecutionMode.CALCULATE_BLOCKING_ALWAYS,+ ExecutionMode.RECENT_CACHE_CALCULATE_BLOCKING_IF_STALE,+ ExecutionMode.CACHE_ONLY_NEVER_CALCULATE,+ }:+ raise QueryError(+ "Account-user queries only support synchronous execution. Use refresh='blocking' or 'force_blocking'."+ )++ def get_reader(self) -> AccountPersonsReader:+ reader = AccountPersonsReader.create(team=self.team, user=self.user, account_id=self.query.accountId)+ if reader is None:+ raise UserAccessControlError("account", "viewer")+ return reader+def validate_query_runner_access(self, user: User) -> bool:- get_account_users_organization_id(team=self.team, user=user, account_id=self.query.accountId)+ if AccountPersonsReader.create(team=self.team, user=user, account_id=self.query.accountId) is None:+ raise UserAccessControlError("account", "viewer")return Truedef get_cache_key_variant(self) -> str:- # Anonymous share-link viewers and cache-only reads skip `validate_query_runner_access`. A- # separate key for them misses staff results, and a fresh run then fails the access check.try:- get_account_users_organization_id(team=self.team, user=self.user, account_id=self.query.accountId)+ reader = self.get_reader()except UserAccessControlError:- return f"{super().get_cache_key_variant()}_denied"- return super().get_cache_key_variant()+ return "denied"+ return reader.get_access_key()- def query_status_labels(self) -> list[str]:- return [f"{ACCOUNT_USERS_QUERY_STATUS_LABEL_PREFIX}{self.query.accountId}"]+ def to_query(self) -> ast.SelectQuery:+ self.modifiers.personsArgMaxVersion = PersonsArgMaxVersion.V2+ return self.get_reader().get_source_query()- def _list_distinct_ids(self) -> tuple[str, ...]:- organization_id = get_account_users_organization_id(- team=self.team, user=self.user, account_id=self.query.accountId+ def to_actors_query(self) -> ast.SelectQuery:+ if not settings.HOGQL_MATERIALIZED_CTE_SUPPORTED:+ return self.to_query()+ return cast(+ ast.SelectQuery,+ parse_select(+ "SELECT person_id, distinct_ids, account_first_seen, account_last_seen FROM __account_members"+ ),)- if organization_id is None:- return ()- # Narrowed by the access check above, which rejects any principal that is not a User.- assert isinstance(self.user, User)- return list_organization_member_distinct_ids(team=self.team, user=self.user, organization_id=organization_id)- def to_query(self) -> ast.SelectQuery:- distinct_ids = self._list_distinct_ids()- condition: ast.Expr = (- ast.CompareOperation(- op=ast.CompareOperationOp.In,- left=ast.Field(chain=["distinct_id"]),- right=ast.Tuple(exprs=[ast.Constant(value=distinct_id) for distinct_id in distinct_ids]),- )- if distinct_ids- else ast.Constant(value=False)- )- query = parse_select(- "SELECT DISTINCT person_id FROM person_distinct_ids WHERE {condition}",- placeholders={"condition": condition},- )- assert isinstance(query, ast.SelectQuery)+ def prepare_actors_query(+ self, query: ast.SelectQuery, source_query: ast.SelectQuery | ast.SelectSetQuery+ ) -> ast.SelectQuery:+ if not settings.HOGQL_MATERIALIZED_CTE_SUPPORTED:+ return query+ AccountUsersMembershipReferences(source_query).visit(query)+ query.ctes = {+ **(query.ctes or {}),+ "__account_members": ast.CTE(+ name="__account_members", expr=self.to_query(), cte_type="subquery", materialized=True+ ),+ }return query- def to_actors_query(self) -> ast.SelectQuery:- return self.to_query()+ def get_actors_query_settings(self) -> HogQLGlobalSettings:+ return HogQLGlobalSettings(+ max_threads=settings.CLICKHOUSE_ACCOUNT_USERS_MAX_THREADS,+ enable_materialized_cte=True if settings.HOGQL_MATERIALIZED_CTE_SUPPORTED else None,+ )def _calculate(self) -> HogQLQueryResponse:- return execute_hogql_query(- query_type="AccountUsersQuery",- query=self.to_query(),- team=self.team,- user=self.user,- timings=self.timings,- modifiers=self.modifiers,- limit_context=self.limit_context,+ raise QueryError("Use AccountUsersQuery as an ActorsQuery source.")++ def get_actors_hogql_context(self, context: HogQLContext) -> HogQLContext:+ assert context.database is not None+ root = context.database.tables+ persons = root.get_child(["persons"])+ assert isinstance(persons.table, PersonsTable)+ table = AccountUsersPersonsTable(fields=persons.table.fields, filter=persons.table.filter)+ context.database = context.database.model_copy(+ update={+ "tables": root.model_copy(+ update={"children": {**root.children, "persons": persons.model_copy(update={"table": table})}}+ )+ }+ )+ return context++ def get_actors_strategy(self, query: ActorsQuery, paginator: HogQLHasMorePaginator) -> ActorStrategy:+ return AccountUsersStrategy(team=self.team, query=query, paginator=paginator, user=self.user)++ def validate_actors_query(self, query: ActorsQuery) -> None:+ if not query.select or any(+ column.strip() in {"person", "group", "actor", "session"} for column in query.select+ ):+ raise QueryError("Select explicit person columns for account users.")++ def get_person_display_name_column(self) -> ast.Expr:+ reader = self.get_reader()+ restrictions = get_restricted_properties_with_group_type_index_for_team(user=self.user, team_id=self.team.pk)+ return ast.Tuple(+ exprs=[+ reader.get_display_name(split_restricted_property_names(restrictions).person),+ ast.Call(name="toString", args=[ast.Field(chain=["persons", "id"])]),+ ])diff --git a/products/customer_analytics/backend/hogql_queries/test/__snapshots__/test_account_users_query_runner.ambr b/products/customer_analytics/backend/hogql_queries/test/__snapshots__/test_account_users_query_runner.ambrnew file mode 100644index 000000000000..18216816420c--- /dev/null+++ b/products/customer_analytics/backend/hogql_queries/test/__snapshots__/test_account_users_query_runner.ambr@@ -0,0 +1,123 @@+# serializer version: 1+# name: TestAccountUsersQuery.test_widget_query_supports_display_names_filters_search_sort_and_pages_0+ '''+ SELECT tuple(coalesce(nullIf(toString(persons.properties___email), %(hogql_val_14)s), nullIf(toString(persons.properties___name), %(hogql_val_15)s), nullIf(toString(persons.properties___username), %(hogql_val_16)s), toString(persons.id)), toString(persons.id)),+ persons.properties___plan AS plan,+ source.account_last_seen AS account_last_seen+ FROM+ (SELECT pdi.person_id AS person_id,+ groupUniqArray(101)(membership.distinct_id) AS distinct_ids,+ min(toTimeZone(membership.first_seen, %(hogql_val_1)s)) AS account_first_seen,+ max(toTimeZone(membership.last_seen, %(hogql_val_2)s)) AS account_last_seen+ FROM person_group_membership AS membership GLOBAL+ INNER JOIN+ (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id,+ person_distinct_id2.distinct_id AS distinct_id+ FROM person_distinct_id2+ WHERE and(equals(person_distinct_id2.team_id, 99999), in(person_distinct_id2.distinct_id,+ (SELECT person_group_membership.distinct_id AS distinct_id+ FROM person_group_membership+ WHERE and(equals(person_group_membership.team_id, 99999), equals(person_group_membership.group_type_index, 0), equals(person_group_membership.group_key, %(hogql_val_0)s)))))+ GROUP BY person_distinct_id2.distinct_id+ HAVING equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0)) AS pdi ON equals(membership.distinct_id, pdi.distinct_id)+ WHERE and(equals(membership.team_id, 99999), equals(membership.group_type_index, 0), equals(membership.group_key, %(hogql_val_3)s))+ GROUP BY pdi.person_id) AS source GLOBAL+ INNER JOIN+ (SELECT person.id AS id,+ replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_4)s), ''), 'null'), '^"|"$', '') AS properties___email,+ replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_5)s), ''), 'null'), '^"|"$', '') AS properties___name,+ replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_6)s), ''), 'null'), '^"|"$', '') AS properties___username,+ replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_7)s), ''), 'null'), '^"|"$', '') AS properties___plan+ FROM person PREWHERE in(tuple(person.id, person.version),+ (SELECT person.id AS id,+ max(person.version) AS version+ FROM person+ WHERE and(equals(person.team_id, 99999), in(person.id,+ (SELECT source.person_id AS person_id+ FROM+ (SELECT pdi.person_id AS person_id, groupUniqArray(101)(membership.distinct_id) AS distinct_ids, min(toTimeZone(membership.first_seen, %(hogql_val_9)s)) AS account_first_seen, max(toTimeZone(membership.last_seen, %(hogql_val_10)s)) AS account_last_seen+ FROM person_group_membership AS membership GLOBAL+ INNER JOIN+ (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id AS distinct_id+ FROM person_distinct_id2+ WHERE and(equals(person_distinct_id2.team_id, 99999), in(person_distinct_id2.distinct_id,+ (SELECT person_group_membership.distinct_id AS distinct_id+ FROM person_group_membership+ WHERE and(equals(person_group_membership.team_id, 99999), equals(person_group_membership.group_type_index, 0), equals(person_group_membership.group_key, %(hogql_val_8)s)))))+ GROUP BY person_distinct_id2.distinct_id+ HAVING equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0)) AS pdi ON equals(membership.distinct_id, pdi.distinct_id)+ WHERE and(equals(membership.team_id, 99999), equals(membership.group_type_index, 0), equals(membership.group_key, %(hogql_val_11)s))+ GROUP BY pdi.person_id) AS source)))+ GROUP BY person.id+ HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, %(hogql_val_12)s), person.version), plus(now64(6, %(hogql_val_13)s), toIntervalDay(1))))))+ WHERE equals(person.team_id, 99999) SETTINGS optimize_aggregation_in_order=0) AS persons ON equals(persons.id, source.person_id)+ ORDER BY source.account_last_seen DESC+ LIMIT 2+ OFFSET 0 SETTINGS optimize_aggregation_in_order=0,+ join_algorithm='auto'+ '''+# ---+# name: TestAccountUsersQuery.test_widget_query_supports_display_names_filters_search_sort_and_pages_1+ '''+ WITH __account_members AS MATERIALIZED+ (SELECT pdi.person_id AS person_id, groupUniqArray(101)(membership.distinct_id) AS distinct_ids, min(toTimeZone(membership.first_seen, %(hogql_val_11)s)) AS account_first_seen, max(toTimeZone(membership.last_seen, %(hogql_val_12)s)) AS account_last_seen+ FROM person_group_membership AS membership GLOBAL+ INNER JOIN+ (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id AS distinct_id+ FROM person_distinct_id2+ WHERE and(equals(person_distinct_id2.team_id, 99999), in(person_distinct_id2.distinct_id,+ (SELECT person_group_membership.distinct_id AS distinct_id+ FROM person_group_membership+ WHERE and(equals(person_group_membership.team_id, 99999), equals(person_group_membership.group_type_index, 0), equals(person_group_membership.group_key, %(hogql_val_10)s)))))+ GROUP BY person_distinct_id2.distinct_id+ HAVING equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0)) AS pdi ON equals(membership.distinct_id, pdi.distinct_id)+ WHERE and(equals(membership.team_id, 99999), equals(membership.group_type_index, 0), equals(membership.group_key, %(hogql_val_13)s))+ GROUP BY pdi.person_id)+ SELECT tuple(coalesce(nullIf(toString(persons.properties___email), %(hogql_val_6)s), nullIf(toString(persons.properties___name), %(hogql_val_7)s), nullIf(toString(persons.properties___username), %(hogql_val_8)s), toString(persons.id)), toString(persons.id)),+ persons.properties___plan AS plan,+ toTimeZone(source.account_last_seen, %(hogql_val_9)s) AS account_last_seen+ FROM __account_members AS source GLOBAL+ INNER JOIN+ (SELECT person.id AS id,+ replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '') AS properties___email,+ replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_1)s), ''), 'null'), '^"|"$', '') AS properties___name,+ replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_2)s), ''), 'null'), '^"|"$', '') AS properties___username,+ replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_3)s), ''), 'null'), '^"|"$', '') AS properties___plan+ FROM person PREWHERE in(tuple(person.id, person.version),+ (SELECT person.id AS id,+ max(person.version) AS version+ FROM person+ WHERE and(equals(person.team_id, 99999), in(person.id,+ (SELECT source.person_id AS person_id+ FROM __account_members AS source)))+ GROUP BY person.id+ HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, %(hogql_val_4)s), person.version), plus(now64(6, %(hogql_val_5)s), toIntervalDay(1))))))+ WHERE equals(person.team_id, 99999) SETTINGS optimize_aggregation_in_order=0) AS persons ON equals(persons.id, source.person_id)+ ORDER BY toTimeZone(source.account_last_seen, %(hogql_val_14)s) DESC+ LIMIT 2+ OFFSET 0 SETTINGS optimize_aggregation_in_order=0,+ join_algorithm='auto'+ '''+# ---+# name: TestMembershipSchema.test_source_resolves_current_mappings_and_binds_account_key+ '''+ SELECT pdi.person_id AS person_id,+ groupUniqArray(101)(membership.distinct_id) AS distinct_ids,+ min(toTimeZone(membership.first_seen, %(hogql_val_1)s)) AS account_first_seen,+ max(toTimeZone(membership.last_seen, %(hogql_val_2)s)) AS account_last_seen+ FROM person_group_membership AS membership GLOBAL+ INNER JOIN+ (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id,+ person_distinct_id2.distinct_id AS distinct_id+ FROM person_distinct_id2+ WHERE and(equals(person_distinct_id2.team_id, 99999), in(person_distinct_id2.distinct_id,+ (SELECT person_group_membership.distinct_id AS distinct_id+ FROM person_group_membership+ WHERE and(equals(person_group_membership.team_id, 99999), equals(person_group_membership.group_type_index, 0), equals(person_group_membership.group_key, %(hogql_val_0)s)))))+ GROUP BY person_distinct_id2.distinct_id+ HAVING equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0)) AS pdi ON equals(membership.distinct_id, pdi.distinct_id)+ WHERE and(equals(membership.team_id, 99999), equals(membership.group_type_index, 0), equals(membership.group_key, %(hogql_val_3)s))+ GROUP BY pdi.person_id+ LIMIT 50000+ '''+# ---diff --git a/products/customer_analytics/backend/hogql_queries/test/test_account_users_query_runner.py b/products/customer_analytics/backend/hogql_queries/test/test_account_users_query_runner.pyindex c58bb02b66e2..f074d661e6a6 100644--- a/products/customer_analytics/backend/hogql_queries/test/test_account_users_query_runner.py+++ b/products/customer_analytics/backend/hogql_queries/test/test_account_users_query_runner.py@@ -1,151 +1,376 @@-from posthog.test.base import APIBaseTest, ClickhouseTestMixin, flush_persons_and_events+import json+from datetime import UTC, datetime+from typing import Any++from posthog.test.base import APIBaseTest, ClickhouseTestMixin, QueryMatchingTest, flush_persons_and_eventsfrom unittest.mock import patch+from django.test import SimpleTestCase, override_settings+from parameterized import parameterized-from rest_framework import statusfrom posthog.schema import CacheMissResponse, QueryStatus+from posthog.hogql.context import HogQLContext+from posthog.hogql.database.database import Database+from posthog.hogql.printer import prepare_and_print_ast+from posthog.hogql.query import execute_hogql_query++from posthog.clickhouse.client import sync_executefrom posthog.constants import AvailableFeature+from posthog.hogql_queries.actors_query_runner import ActorsQueryRunnerfrom posthog.hogql_queries.query_runner import ExecutionMode, get_query_runner-from posthog.models import Organization, OrganizationMembership, User+from posthog.models import OrganizationMembership, PropertyDefinition, Team, Userfrom posthog.models.personal_api_key import PersonalAPIKeyfrom posthog.models.utils import generate_random_token_personal, hash_key_valuefrom posthog.test.persons import create_personfrom products.access_control.backend.models.access_control import AccessControl-from products.customer_analytics.backend.constants import ACCOUNT_USERS_QUERY_STATUS_LABEL_PREFIX-from products.customer_analytics.backend.logic.account_member_search import list_organization_member_distinct_ids+from products.access_control.backend.models.property_access_control import PropertyAccessControl+from products.customer_analytics.backend.logic.account_persons import AccountPersonsReader+from products.customer_analytics.backend.models import TeamCustomerAnalyticsConfigfrom products.customer_analytics.backend.test.factories import create_account-FLAG_CHECK = "products.customer_analytics.backend.logic.account_member_search.posthog_feature_flag_enabled"+FLAG_CHECK = "products.customer_analytics.backend.logic.account_persons.posthog_feature_flag_enabled"+++class TestMembershipSchema(QueryMatchingTest, SimpleTestCase):+ def test_source_resolves_current_mappings_and_binds_account_key(self) -> None:+ team = Team(id=123, project_id=123)+ group_key = "workspace' OR 1 = 1 --"+ reader = AccountPersonsReader(team=team, user=User(), group_type_index=0, group_key=group_key)+ context = HogQLContext(+ team_id=team.pk,+ team=team,+ database=Database(),+ enable_select_queries=True,+ restricted_properties=set(),+ use_new_events_schema=False,+ )+ sql, _ = prepare_and_print_ast(reader.get_source_query(), context=context, dialect="clickhouse")+ self.assertQueryMatchesSnapshot(sql)+ assert group_key not in sql+ assert group_key in context.values.values()-class TestAccountUsersQuery(ClickhouseTestMixin, APIBaseTest):+@override_settings(HOGQL_MATERIALIZED_CTE_SUPPORTED=True)+class TestAccountUsersQuery(ClickhouseTestMixin, APIBaseTest, QueryMatchingTest):def setUp(self) -> None:super().setUp()- customer_org = Organization.objects.create(name="Customer")- for email in ("[email protected]", "[email protected]"):- member = User.objects.create(email=email, distinct_id=f"distinct-{email}")- OrganizationMembership.objects.create(organization=customer_org, user=member)- create_person(team=self.team, distinct_ids=[member.distinct_id], properties={"email": email})- create_person(team=self.team, distinct_ids=["outsider"], properties={"email": "[email protected]"})+ self.enterContext(patch(FLAG_CHECK, return_value=True))+ TeamCustomerAnalyticsConfig.objects.update_or_create(team=self.team, defaults={"account_group_type_index": 0})+ self.account = create_account(team_id=self.team.pk, external_id="workspace-demo")+ self.endpoint = f"/api/projects/{self.team.pk}/query/"+ self.first = create_person(+ team=self.team,+ distinct_ids=["member-one", "member-one-alias"],+ properties={"email": "[email protected]", "name": "First member", "plan": "paid", "private_note": "hidden"},+ )+ self.second = create_person(+ team=self.team, distinct_ids=["member-two"], properties={"email": "[email protected]", "plan": "free"}+ )+ create_person(team=self.team, distinct_ids=["non-member"], properties={"email": "[email protected]"})flush_persons_and_events()- self.account = create_account(team_id=self.team.id, external_id=str(customer_org.id))- self.query = {+ self._insert_membership("member-one", "2024-01-01", "2024-01-05")+ self._insert_membership("member-one-alias", "2024-01-02", "2024-01-09")+ self._insert_membership("member-two", "2024-01-03", "2024-01-06")+ self._insert_membership("non-member", "2024-01-01", "2024-01-10", group_key="another-workspace")+ self._insert_membership("non-member", "2024-01-01", "2024-01-10", group_type_index=1)+ self._insert_membership(+ "non-member", "2024-01-01", "2024-01-10", team_id=Team.objects.create(organization=self.organization).pk+ )++ def _get_widget_query(self, **overrides: Any) -> dict[str, Any]:+ return {"kind": "ActorsQuery","source": {"kind": "AccountUsersQuery", "accountId": str(self.account.id)},- "select": ["properties.email"],- "orderBy": ["properties.email DESC"],+ "select": ["person_display_name -- Person", "properties.plan", "source.account_last_seen"],+ "orderBy": ["source.account_last_seen DESC"],+ **overrides,}- self.user.is_staff = True- self.user.save()- def _run(self, headers: dict[str, str] | None = None):+ def _run_widget_query(self, **overrides: Any) -> Any:return self.client.post(- f"/api/projects/{self.team.id}/query/",- {"query": self.query, "refresh": "blocking"},+ self.endpoint,+ {"query": self._get_widget_query(**overrides), "refresh": "blocking"},format="json",- headers=headers,)- def _hide_account_from_user(self) -> None:- self.organization.available_product_features = [{"key": AvailableFeature.ACCESS_CONTROL}]- self.organization.save()- AccessControl.objects.create(- team=self.team,- resource="account",- resource_id=str(self.account.id),- access_level="none",- organization_member=OrganizationMembership.objects.get(user=self.user, organization=self.organization),+ @parameterized.expand([(False, None), (True, 120)])+ def test_widget_query_supports_display_names_filters_search_sort_and_pages(+ self, materialized_cte: bool, max_threads: int | None+ ) -> None:+ self.enterContext(+ self.settings(+ HOGQL_MATERIALIZED_CTE_SUPPORTED=materialized_cte, CLICKHOUSE_ACCOUNT_USERS_MAX_THREADS=max_threads+ ))+ with patch("posthog.hogql_queries.paginators.execute_hogql_query", wraps=execute_hogql_query) as execute_query:+ response = self._run_widget_query(limit=1)+ assert execute_query.call_args.kwargs["settings"].max_threads == max_threads+ assert response.status_code == 200, response.content+ page = response.json()+ assert page["results"][0][0] == {"display_name": "[email protected]", "id": str(self.first.uuid)}+ assert page["hasMore"] and page["limit"] == 1+ runner = get_query_runner(self._get_widget_query(limit=1), self.team, user=self.user)+ assert isinstance(runner, ActorsQueryRunner)+ query = runner.paginator.paginate(runner.to_query())+ context = runner.build_hogql_context(enable_select_queries=True, modifiers=runner.modifiers)+ sql, _ = prepare_and_print_ast(+ query,+ context=context,+ dialect="clickhouse",+ )+ self.assertQueryMatchesSnapshot(sql)+ plan = json.loads(sync_execute("EXPLAIN PLAN indexes=1, json=1 " + sql, context.values)[0][0])+ pending = [plan[0]["Plan"]]+ person_reads = []+ while pending:+ step = pending.pop()+ if step["Node Type"] == "ReadFromMergeTree" and step.get("Description", "").endswith(".person"):+ person_reads.append(step)+ pending.extend(step.get("Plans", []))+ assert person_reads+ assert all(+ any(+ index["Type"] == "PrimaryKey" and "(id in " in index.get("Condition", "")+ for index in step.get("Indexes", [])+ )+ for step in person_reads+ )+ response = self._run_widget_query(limit=1, offset=1)+ assert response.status_code == 200, response.content+ assert response.json()["results"][0][0]["id"] == str(self.second.uuid)+ response = self._run_widget_query(+ search="member-one-alias",+ properties=[{"type": "person", "key": "plan", "operator": "exact", "value": "paid"}],+ orderBy=["properties.plan ASC"],+ )+ assert response.status_code == 200, response.content+ assert len(response.json()["results"]) == 1+ assert response.json()["results"][0][0]["id"] == str(self.first.uuid)- @patch(FLAG_CHECK, return_value=True)- def test_staff_get_the_organization_members_as_persons(self, _flag) -> None:- response = self._run()-- assert response.status_code == status.HTTP_200_OK, response.content- assert response.json()["results"] == [["[email protected]"], ["[email protected]"]]+ @parameterized.expand([("flag_disabled",), ("account_hidden",)])+ def test_widget_cached_results_repeat_access_checks(self, condition: str) -> None:+ response = self._run_widget_query()+ assert response.status_code == 200, response.content+ if condition == "account_hidden":+ self.organization.available_product_features = [{"key": AvailableFeature.ACCESS_CONTROL}]+ self.organization.save()+ AccessControl.objects.create(+ team=self.team,+ resource="account",+ resource_id=str(self.account.id),+ access_level="none",+ organization_member=self.organization_membership,+ )+ with patch(FLAG_CHECK, return_value=condition != "flag_disabled"):+ response = self.client.post(self.endpoint, {"query": self._get_widget_query()}, format="json")+ assert response.status_code == 400, response.content@parameterized.expand([- ("not_staff", False, True, False),- ("flag_off", True, False, False),- ("account_hidden", True, True, True),+ ("no_user",),+ ("different_user",),+ ("person_properties",),+ ("event_properties",),+ ("group_properties",),+ ("resource_access",),])- def test_denies_access_even_when_cached(self, _name, is_staff, flag_on, account_hidden) -> None:- with patch(FLAG_CHECK, return_value=True):- assert self._run().status_code == status.HTTP_200_OK-- self.user.is_staff = is_staff- self.user.save()- if account_hidden:- self._hide_account_from_user()- with patch(FLAG_CHECK, return_value=flag_on):- response = self._run()-- assert response.status_code == status.HTTP_400_BAD_REQUEST, response.content- assert "Access control failure" in response.json()["detail"]-- @patch(FLAG_CHECK, return_value=True)- def test_cache_only_read_without_a_user_misses_staff_results(self, _flag) -> None:- get_query_runner(self.query, self.team, user=self.user).run(+ def test_widget_cache_read_requires_the_same_authorization_context(self, condition: str) -> None:+ if condition.endswith("properties"):+ self.organization.available_product_features = [{"key": AvailableFeature.PROPERTY_ACCESS_CONTROL}]+ self.organization.save()+ self.organization_membership.level = OrganizationMembership.Level.MEMBER+ self.organization_membership.save()+ elif condition == "resource_access":+ self.organization.available_product_features = [{"key": AvailableFeature.ACCESS_CONTROL}]+ self.organization.save()+ get_query_runner(self._get_widget_query(), self.team, user=self.user).run(ExecutionMode.CALCULATE_BLOCKING_ALWAYS, user=self.user)+ user: User | None = self.user+ if condition == "no_user":+ user = None+ elif condition == "different_user":+ user = User.objects.create_and_join(self.organization, "[email protected]", password=None)+ elif condition == "resource_access":+ AccessControl.objects.create(+ team=self.team,+ resource="dashboard",+ access_level="none",+ organization_member=self.organization_membership,+ )+ else:+ property_type = {+ "person_properties": PropertyDefinition.Type.PERSON,+ "event_properties": PropertyDefinition.Type.EVENT,+ "group_properties": PropertyDefinition.Type.GROUP,+ }[condition]+ self._restrict_properties(property_type)+ response = get_query_runner(self._get_widget_query(), self.team, user=user).run(+ ExecutionMode.CACHE_ONLY_NEVER_CALCULATE, user=user+ )+ assert isinstance(response, CacheMissResponse)- response = get_query_runner(self.query, self.team).run(ExecutionMode.CACHE_ONLY_NEVER_CALCULATE)+ def test_widget_display_name_and_search_hide_restricted_properties(self) -> None:+ self._restrict_properties()+ response = self._run_widget_query()+ assert response.status_code == 200, response.content+ assert all(row[0]["display_name"] == row[0]["id"] for row in response.json()["results"])+ response = self._run_widget_query(search="[email protected]")+ assert response.status_code == 200, response.content+ assert response.json()["results"] == []- assert isinstance(response, CacheMissResponse)+ @parameterized.expand(+ [+ ([],),+ (["person"],),+ (["actor"],),+ (["persons.id AS `group`", "group"],),+ (["persons.id AS session", "session"],),+ (["(SELECT count() FROM posthog.person_group_membership)"],),+ ]+ )+ def test_widget_rejects_unsupported_columns(self, columns: list[str]) -> None:+ self._restrict_properties()+ assert self._run_widget_query(select=columns).status_code == 400++ @parameterized.expand(+ [+ (refresh, interface)+ for refresh in ("async", "force_async", "async_except_on_cache_miss", "lazy_async", "legacy")+ for interface in ("generic", "wrapped")+ ]+ )+ def test_async_account_user_requests_are_rejected(self, refresh: str, interface: str) -> None:+ query = self._get_widget_query()+ data: dict[str, Any] = {+ "query": {"kind": "DataTableNode", "source": query} if interface == "wrapped" else query+ }+ if refresh == "legacy":+ data["async"] = True+ else:+ data["refresh"] = refresh+ endpoint = self.endpoint+ with patch("posthog.hogql_queries.query_runner.enqueue_process_query_task") as enqueue:+ response = self.client.post(endpoint, data, format="json")+ assert response.status_code == 400, response.content+ enqueue.assert_not_called()- @parameterized.expand([("staff", True, status.HTTP_200_OK), ("not_staff", False, status.HTTP_403_FORBIDDEN)])- @patch(FLAG_CHECK, return_value=True)- def test_async_result_retrieval_repeats_the_access_check(self, _name, is_staff, expected_status, _flag) -> None:- self.user.is_staff = is_staff- self.user.save()+ @parameterized.expand([(True, 403), (False, 200)])+ def test_account_user_query_ids_are_unavailable(self, account_users: bool, expected: int) -> None:stored = QueryStatus(- id="account-users-query",- team_id=self.team.id,+ id="stored-query",+ team_id=self.team.pk,complete=True,- labels=[f"{ACCOUNT_USERS_QUERY_STATUS_LABEL_PREFIX}{self.account.id}"],- results={"results": [["[email protected]"]]},+ labels=[f"customer-analytics-account-users:{self.account.id}"] if account_users else [],+ results={"results": [["member"]]},)-with patch("posthog.api.query.get_query_status", return_value=stored):- response = self.client.get(f"/api/projects/{self.team.id}/query/account-users-query/")+ response = self.client.get(f"/api/projects/{self.team.pk}/query/stored-query/")+ assert response.status_code == expected, response.content- assert response.status_code == expected_status, response.content+ def _insert_membership(+ self,+ distinct_id: str,+ first_seen: str,+ last_seen: str,+ *,+ group_key: str = "workspace-demo",+ group_type_index: int = 0,+ team_id: int | None = None,+ ) -> None:+ sync_execute(+ "INSERT INTO person_group_membership (team_id, group_type_index, group_key, distinct_id, first_seen, last_seen) VALUES",+ [+ {+ "team_id": self.team.pk if team_id is None else team_id,+ "group_type_index": group_type_index,+ "group_key": group_key,+ "distinct_id": distinct_id,+ "first_seen": datetime.fromisoformat(first_seen).replace(tzinfo=UTC),+ "last_seen": datetime.fromisoformat(last_seen).replace(tzinfo=UTC),+ }+ ],+ )++ @parameterized.expand(+ [(side, interface) for side in ("mapping", "person") for interface in ("widget", "membership")]+ )+ def test_deleted_identity_is_not_returned(self, deleted_side: str, interface: str) -> None:+ if deleted_side == "mapping":+ sync_execute(+ "INSERT INTO person_distinct_id2 (team_id, distinct_id, person_id, is_deleted, version) VALUES",+ [+ {+ "team_id": self.team.pk,+ "distinct_id": "member-two",+ "person_id": self.second.uuid,+ "is_deleted": 1,+ "version": 100,+ }+ ],+ )+ else:+ sync_execute(+ "INSERT INTO person (team_id, id, properties, is_identified, is_deleted, version, created_at) VALUES",+ [+ {+ "team_id": self.team.pk,+ "id": self.second.uuid,+ "properties": "{}",+ "is_identified": 1,+ "is_deleted": 1,+ "version": 100,+ "created_at": datetime(2024, 1, 1, tzinfo=UTC),+ }+ ],+ )+ columns = ["persons.id"] if interface == "widget" else ["source.distinct_ids"]+ response = self._run_widget_query(select=columns, limit=1)+ assert response.status_code == 200, response.content+ page = response.json()+ assert len(page["results"]) == 1+ assert not page["hasMore"]+ if interface == "widget":+ assert page["results"][0][0] == str(self.first.uuid)+ else:+ assert set(page["results"][0][0]) == {"member-one", "member-one-alias"}@parameterized.expand([- ("query_scope_only", ["query:read"], status.HTTP_403_FORBIDDEN),- ("all_scopes", ["query:read", "account:read", "person:read"], status.HTTP_200_OK),+ ("query_account_only", ["query:read", "account:read"], 403, "query"),+ ("query_person_only", ["query:read", "person:read"], 403, "query"),+ ("query_both", ["query:read", "account:read", "person:read"], 200, "query"),])- @patch(FLAG_CHECK, return_value=True)- def test_api_tokens_need_account_and_person_scopes(self, _name, scopes, expected_status, _flag) -> None:+ def test_tokens_require_both_scopes(self, _name: str, scopes: list[str], expected: int, interface: str) -> None:token = generate_random_token_personal()PersonalAPIKey.objects.create(- label="account users query",- user=self.user,- secure_value=hash_key_value(token),- scopes=scopes,- scoped_teams=[],- scoped_organizations=[],+ label="account persons", user=self.user, secure_value=hash_key_value(token), scopes=scopes)+ self.client.logout()+ headers = {"authorization": f"Bearer {token}"}+ response = self.client.post(+ self.endpoint,+ {"query": self._get_widget_query(), "refresh": "blocking"},+ format="json",+ headers=headers,+ )+ assert response.status_code == expected, response.content- response = self._run(headers={"authorization": f"Bearer {token}"})-- assert response.status_code == expected_status, response.content-- def test_eu_lookup_failure_raises_instead_of_returning_no_members(self) -> None:- with (- patch(- "products.customer_analytics.backend.logic.account_member_search.execute_hogql_query",- side_effect=TimeoutError("eu lookup timed out"),- ),- self.assertRaises(TimeoutError),- ):- list_organization_member_distinct_ids(- team=self.team, user=self.user, organization_id=str(Organization.objects.create(name="EU").id)+ def _restrict_properties(self, property_type: int = PropertyDefinition.Type.PERSON) -> None:+ self.organization.available_product_features = [{"key": AvailableFeature.PROPERTY_ACCESS_CONTROL}]+ self.organization.save()+ self.organization_membership.level = OrganizationMembership.Level.MEMBER+ self.organization_membership.save()+ for name in ["email", "name", "private_note"]:+ definition = PropertyDefinition.objects.create(+ team=self.team,+ name=name,+ type=property_type,+ group_type_index=0 if property_type == PropertyDefinition.Type.GROUP else None,)+ PropertyAccessControl.objects.create(team=self.team, property_definition=definition, access_level="none")diff --git a/products/customer_analytics/backend/logic/account_member_search.py b/products/customer_analytics/backend/logic/account_member_search.pyindex a612259ed45c..f3e6a3d16306 100644--- a/products/customer_analytics/backend/logic/account_member_search.py+++ b/products/customer_analytics/backend/logic/account_member_search.py@@ -1,7 +1,4 @@from typing import cast-from uuid import UUID--from django.core.exceptions import ValidationErrorimport structlog@@ -9,7 +6,6 @@ from posthog.hogql import astfrom posthog.hogql.errors import ResolutionError, TableAccessDeniedErrorfrom posthog.hogql.query import execute_hogql_query-from posthog.api.organization_member import organization_members_base_querysetfrom posthog.clickhouse.query_tagging import Feature, Product, tags_contextfrom posthog.constants import INTERNAL_BOT_EMAIL_SUFFIXfrom posthog.models import OrganizationMembership@@ -17,15 +13,11 @@ from posthog.models.team import Teamfrom posthog.models.user import Userfrom posthog.permissions import posthog_feature_flag_enabled-from products.access_control.backend.facade.user_access_control import UserAccessControl, UserAccessControlErrorfrom products.customer_analytics.backend.constants import CUSTOMER_ANALYTICS_CSP_FLAG-from products.customer_analytics.backend.models.account import Accountlogger = structlog.get_logger(__name__)ACCOUNT_MEMBER_SEARCH_MAX_ORGANIZATIONS = 100-# The member list becomes a constant IN list in ClickHouse, which caps how many members one query can carry.-ORGANIZATION_MEMBERS_MAX = 10_000EU_ORGANIZATION_MEMBERS_VIEW = "eu_org_members"@@ -61,7 +53,6 @@ def _query_eu_organization_members(placeholders: dict[str, ast.Expr],query_type: str,log_event: str,- raise_query_failures: bool = False,) -> tuple[str, ...]:try:with tags_context(product=Product.CUSTOMER_ANALYTICS, feature=Feature.QUERY):@@ -77,8 +68,6 @@ def _query_eu_organization_members(return ()except Exception as error:logger.warning(f"{log_event}_failed", team_id=team.id, error_type=type(error).__name__)- if raise_query_failures:- raisereturn ()rows = cast(list[list[object]], response.results or [])@@ -110,53 +99,3 @@ def list_account_external_ids_by_member_email(*, team: Team, user: User, email:if organization_ids:return organization_idsreturn _list_eu_organization_ids(team, user, email)---def list_organization_member_distinct_ids(*, team: Team, user: User, organization_id: str) -> tuple[str, ...]:- """Distinct IDs of the PostHog Cloud users in an organization, from US Postgres or else the EU warehouse view."""- us_distinct_ids = tuple(- organization_members_base_queryset()- .filter(organization_id=organization_id, user__distinct_id__isnull=False)- .order_by("-joined_at")- .values_list("user__distinct_id", flat=True)[:ORGANIZATION_MEMBERS_MAX]- )- if us_distinct_ids:- return us_distinct_ids- return _query_eu_organization_members(- team,- user,- query=f"""- SELECT distinct_id- FROM {EU_ORGANIZATION_MEMBERS_VIEW}- WHERE organization_id = {{organization_id}} AND distinct_id IS NOT NULL- ORDER BY joined_at DESC- LIMIT {ORGANIZATION_MEMBERS_MAX}- """,- placeholders={"organization_id": ast.Constant(value=organization_id)},- query_type="customer_analytics_account_member_distinct_id_lookup",- log_event="account_member_distinct_id_lookup",- raise_query_failures=True,- )---def get_account_users_organization_id(*, team: Team, user: User | None, account_id: str) -> str | None:- """The PostHog Cloud organization behind an account's users, or None when the account has none.-- Raises `UserAccessControlError` unless `user` is flagged staff who can see the account: members are- read from any organization, not the caller's own.- """- if not isinstance(user, User) or not is_account_member_search_enabled(team, user):- raise UserAccessControlError("account", "viewer")- visible_accounts = UserAccessControl(user=user, team=team).filter_queryset_by_access_level(- Account.objects.for_team(team.id)- )- try:- account = visible_accounts.filter(id=account_id).values("external_id").first()- except (ValidationError, ValueError):- account = None- if account is None:- raise UserAccessControlError("account", "viewer", resource_id=account_id)- try:- return str(UUID(account["external_id"])) if account["external_id"] else None- except ValueError:- return Nonediff --git a/products/customer_analytics/backend/logic/account_persons.py b/products/customer_analytics/backend/logic/account_persons.pynew file mode 100644index 000000000000..f3769d48a82b--- /dev/null+++ b/products/customer_analytics/backend/logic/account_persons.py@@ -0,0 +1,188 @@+import json+from hashlib import sha256++from posthog.hogql import ast+from posthog.hogql.database.schema.person_group_membership import PersonGroupMembershipTable+from posthog.hogql.parser import parse_select+from posthog.hogql.visitor import TraversingVisitor++from posthog.hogql_queries.actor_strategies import person_email_property_keys+from posthog.models import Team, User+from posthog.permissions import posthog_feature_flag_enabled++from products.access_control.backend.facade.api import get_restricted_properties_with_group_type_index_for_team+from products.access_control.backend.facade.user_access_control import UserAccessControl+from products.customer_analytics.backend.facade.api import get_accessible_account_id, get_account++ACCOUNT_PERSONS_FLAG = "customer-analytics-account-persons-api"+++class MembershipTableReferences(TraversingVisitor):+ def visit_join_expr(self, node: ast.JoinExpr) -> None:+ if isinstance(node.table, ast.Field) and node.table.chain == ["posthog", "person_group_membership"]:+ node.table_schema = PersonGroupMembershipTable()+ super().visit_join_expr(node)+++class AccountPersonsReader:+ @classmethod+ def create(+ cls,+ *,+ team: Team,+ user: User | None,+ account_id: str,+ user_access_control: UserAccessControl | None = None,+ ) -> "AccountPersonsReader | None":+ if (+ not isinstance(user, User)+ or not user.is_active+ or not posthog_feature_flag_enabled(+ ACCOUNT_PERSONS_FLAG, str(user.distinct_id), organization_id=team.organization_id, team_id=team.pk+ )+ ):+ return None+ access = user_access_control or UserAccessControl(user=user, team=team)+ if not access.check_access_level_for_resource("account", "viewer"):+ return None+ if get_accessible_account_id(team.pk, account_id, access) is None:+ return None+ account = get_account(team.pk, account_id=account_id)+ group_type_index = team.customer_analytics_config.account_group_type_index+ if account is None or group_type_index is None or group_type_index not in range(5):+ return None+ return cls(+ team=team,+ user=user,+ group_type_index=group_type_index,+ group_key=account.external_id,+ account_id=account_id,+ )++ def __init__(+ self, *, team: Team, user: User, group_type_index: int, group_key: str | None, account_id: str = ""+ ) -> None:+ self.account_id = account_id+ self.team = team+ self.user = user+ self.group_type_index = group_type_index+ self.group_key = group_key++ def get_access_key(self) -> str:+ restrictions = get_restricted_properties_with_group_type_index_for_team(user=self.user, team_id=self.team.pk)+ access = UserAccessControl(user=self.user, team=self.team)+ payload = {+ "user_id": self.user.pk,+ "group_type_index": self.group_type_index,+ "group_key": self.group_key,+ "properties": sorted(+ (item.property_type, item.group_type_index if item.group_type_index is not None else -1, item.name)+ for item in restrictions+ ),+ "blocked_objects": {scope: sorted(ids) for scope, ids in access.blocked_resource_ids_by_scope.items()},+ "allowed_objects": {scope: sorted(ids) for scope, ids in access.allowlisted_resource_ids_by_scope.items()},+ "blocked_resources": sorted(access.blocked_resources),+ }+ return sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest()++ @staticmethod+ def parse_membership_query(query: str, placeholders: dict[str, ast.Expr]) -> ast.SelectQuery:+ select = parse_select(query, placeholders=placeholders)+ assert isinstance(select, ast.SelectQuery)+ MembershipTableReferences().visit(select)+ return select++ def get_source_query(self) -> ast.SelectQuery:+ query = self.parse_membership_query(+ """+ SELECT pdi.person_id AS person_id,+ groupUniqArray(101)(membership.distinct_id) AS distinct_ids,+ min(membership.first_seen) AS account_first_seen,+ max(membership.last_seen) AS account_last_seen+ FROM posthog.person_group_membership AS membership+ INNER JOIN (+ SELECT argMax(person_id, version) AS person_id, distinct_id+ FROM raw_person_distinct_ids+ WHERE distinct_id IN (+ SELECT distinct_id FROM posthog.person_group_membership+ WHERE group_type_index = {group_type_index} AND group_key = {group_key}+ )+ GROUP BY distinct_id+ HAVING argMax(is_deleted, version) = 0+ ) AS pdi ON membership.distinct_id = pdi.distinct_id+ WHERE membership.group_type_index = {group_type_index} AND membership.group_key = {group_key}+ GROUP BY pdi.person_id+ """,+ placeholders={+ "group_type_index": ast.Constant(value=self.group_type_index),+ "group_key": ast.Constant(value=self.group_key),+ },+ )+ assert isinstance(query, ast.SelectQuery)+ assert query.select_from is not None and query.select_from.next_join is not None+ query.select_from.next_join.join_type = "GLOBAL INNER JOIN"+ return query++ def get_display_name(self, restricted_properties: frozenset[str]) -> ast.Expr:+ property_keys = self.team.person_display_name_properties or ["email", "name", "username"]+ return ast.Call(+ name="coalesce",+ args=[+ *[+ ast.Call(+ name="nullIf",+ args=[+ ast.Call(name="toString", args=[ast.Field(chain=["properties", key])]),+ ast.Constant(value=""),+ ],+ )+ for key in property_keys+ if key not in restricted_properties+ ],+ ast.Call(name="toString", args=[ast.Field(chain=["persons", "id"])]),+ ],+ )++ def get_search_filter(self, search: str, restricted_properties: frozenset[str]) -> ast.Expr:+ expressions: list[ast.Expr] = [ast.Call(name="toString", args=[ast.Field(chain=["persons", "id"])])]+ expressions.extend(+ ast.Call(name="toString", args=[ast.Field(chain=["properties", key])])+ for key in dict.fromkeys([*person_email_property_keys(self.team), "name"])+ if key not in restricted_properties+ )+ condition = ast.Or(+ exprs=[+ *[+ ast.CompareOperation(+ left=expr,+ op=ast.CompareOperationOp.ILike,+ right=ast.Constant(value=f"%{search}%"),+ )+ for expr in expressions+ ],+ ast.CompareOperation(+ left=ast.Field(chain=["persons", "id"]),+ op=ast.CompareOperationOp.In,+ right=self.parse_membership_query(+ """+ SELECT argMax(person_id, version) AS person_id+ FROM raw_person_distinct_ids+ WHERE distinct_id IN (+ SELECT distinct_id FROM posthog.person_group_membership+ WHERE group_type_index = {group_type_index}+ AND group_key = {group_key}+ AND ilike(distinct_id, {search})+ )+ GROUP BY distinct_id+ HAVING argMax(is_deleted, version) = 0+ """,+ placeholders={+ "group_type_index": ast.Constant(value=self.group_type_index),+ "group_key": ast.Constant(value=self.group_key),+ "search": ast.Constant(value=f"%{search}%"),+ },+ ),+ ),+ ]+ )+ return condition