Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(customer-analytics): query account members from memberships (#126673)

Arthur Moreira de Deusd45756afd1a7
diff --git a/posthog/hogql_queries/actors_query_runner.py b/posthog/hogql_queries/actors_query_runner.py
index 7812ee8205d0..f4b3dd467a37 100644
--- a/posthog/hogql_queries/actors_query_runner.py
+++ b/posthog/hogql_queries/actors_query_runner.py
@@ -1,10 +1,12 @@
import itertools
from collections.abc import Iterable, Iterator, Sequence
-from typing import TYPE_CHECKING, Any, Optional
+from typing import TYPE_CHECKING, Any, Optional, cast
if TYPE_CHECKING:
from posthog.event_usage import AnalyticsProps
+ from products.customer_analytics.backend.facade.queries import AccountUsersQueryRunner
+
from posthoganalytics import feature_enabled
from posthog.schema import (
@@ -18,6 +20,7 @@ from posthog.schema import (
from posthog.hogql import ast
from posthog.hogql.constants import HogQLQuerySettings, LimitContext
+from posthog.hogql.context import HogQLContext
from posthog.hogql.parser import parse_expr, parse_order_expr
from posthog.hogql.printer import prepare_and_print_ast
from posthog.hogql.property import has_aggregation
@@ -70,6 +73,16 @@ class ActorsQueryRunner(AnalyticsQueryRunner[ActorsQueryResponse]):
def _child_runners(self) -> Iterable[QueryRunner]:
return [self.source_query_runner] if self.source_query_runner is not None else []
+ def _get_account_users_source(self) -> "AccountUsersQueryRunner | None":
+ if self.query.source is not None and self.query.source.kind == "AccountUsersQuery":
+ return cast("AccountUsersQueryRunner", self.source_query_runner)
+ return None
+
+ def build_hogql_context(self, **kwargs: object) -> HogQLContext:
+ context = super().build_hogql_context(**kwargs)
+ source = self._get_account_users_source()
+ return source.get_actors_hogql_context(context) if source is not None else context
+
def _on_user_changed(self) -> None:
super()._on_user_changed()
self.strategy = self.determine_strategy()
@@ -88,6 +101,9 @@ class ActorsQueryRunner(AnalyticsQueryRunner[ActorsQueryResponse]):
# hide the change from the base run(), leaving the shared database and access-control
# snapshot scoped to the previous user.
self.set_user(user)
+ account_users_source = self._get_account_users_source()
+ if account_users_source is not None:
+ account_users_source.validate_execution_mode(execution_mode)
return super().run(
execution_mode,
user,
@@ -125,6 +141,9 @@ class ActorsQueryRunner(AnalyticsQueryRunner[ActorsQueryResponse]):
return self.source_query_runner.is_session_aggregation
def determine_strategy(self) -> ActorStrategy:
+ account_users_source = self._get_account_users_source()
+ if account_users_source is not None:
+ return account_users_source.get_actors_strategy(self.query, self.paginator)
if self.group_type_index is not None:
return GroupStrategy(
self.group_type_index,
@@ -218,6 +237,7 @@ class ActorsQueryRunner(AnalyticsQueryRunner[ActorsQueryResponse]):
return column_index_events, self.strategy.get_recordings(matching_events_list)
def _calculate_internal(self) -> ActorsQueryResponse:
+ account_users_source = self._get_account_users_source()
response = self.paginator.execute_hogql_query(
query_type="ActorsQuery",
query=self.to_query(),
@@ -226,6 +246,7 @@ class ActorsQueryRunner(AnalyticsQueryRunner[ActorsQueryResponse]):
timings=self.timings,
modifiers=self.modifiers,
context=self.build_hogql_context(),
+ settings=account_users_source.get_actors_query_settings() if account_users_source else None,
)
input_columns = self.input_columns()
missing_actors_count = None
@@ -303,6 +324,9 @@ class ActorsQueryRunner(AnalyticsQueryRunner[ActorsQueryResponse]):
super().validate()
if self.source_query_runner is not None:
self.source_query_runner.validate()
+ account_users_source = self._get_account_users_source()
+ if account_users_source is not None:
+ account_users_source.validate_actors_query(self.query)
def input_columns(self) -> list[str]:
strategy_input_cols = self.strategy.input_columns()
@@ -477,6 +501,7 @@ class ActorsQueryRunner(AnalyticsQueryRunner[ActorsQueryResponse]):
else:
order_by = []
+ source_query: ast.SelectQuery | ast.SelectSetQuery | None = None
with self.timings.measure("select"):
select_query = ast.SelectQuery(
select=columns,
@@ -484,7 +509,9 @@ class ActorsQueryRunner(AnalyticsQueryRunner[ActorsQueryResponse]):
having=having,
group_by=group_by if has_any_aggregation else None,
order_by=order_by,
- settings=HogQLQuerySettings(join_algorithm="auto", optimize_aggregation_in_order=True),
+ settings=HogQLQuerySettings(
+ join_algorithm="auto", optimize_aggregation_in_order=self._get_account_users_source() is None
+ ),
)
if not self.query.source:
select_query.select_from = ast.JoinExpr(table=ast.Field(chain=[self.strategy.origin]))
@@ -522,20 +549,21 @@ class ActorsQueryRunner(AnalyticsQueryRunner[ActorsQueryResponse]):
):
select_query.select.append(ast.Field(chain=[source_distinct_id_column]))
- try:
- prepare_and_print_ast(
- select_query,
- context=self.build_hogql_context(
- team=self.team,
- enable_select_queries=True,
- timings=self.timings,
- modifiers=self.modifiers,
- ),
- dialect="clickhouse",
- )
- return select_query
- except Exception:
- pass
+ if self._get_account_users_source() is None:
+ try:
+ prepare_and_print_ast(
+ select_query,
+ context=self.build_hogql_context(
+ team=self.team,
+ enable_select_queries=True,
+ timings=self.timings,
+ modifiers=self.modifiers,
+ ),
+ dialect="clickhouse",
+ )
+ return select_query
+ except Exception:
+ pass
origin = self.strategy.origin
@@ -577,7 +605,7 @@ class ActorsQueryRunner(AnalyticsQueryRunner[ActorsQueryResponse]):
alias=source_alias,
next_join=ast.JoinExpr(
table=ast.Field(chain=[origin]),
- join_type="INNER JOIN",
+ join_type="GLOBAL INNER JOIN" if self._get_account_users_source() is not None else "INNER JOIN",
constraint=ast.JoinConstraint(
expr=join_on,
constraint_type="ON",
@@ -585,7 +613,12 @@ class ActorsQueryRunner(AnalyticsQueryRunner[ActorsQueryResponse]):
),
)
- return select_query
+ account_users_source = self._get_account_users_source()
+ return (
+ account_users_source.prepare_actors_query(select_query, source_query)
+ if account_users_source and source_query is not None
+ else select_query
+ )
def to_actors_query(self) -> ast.SelectQuery | ast.SelectSetQuery:
return self.to_query()
@@ -600,6 +633,9 @@ class ActorsQueryRunner(AnalyticsQueryRunner[ActorsQueryResponse]):
return node
def _get_person_display_name_column(self) -> ast.Expr:
+ account_users_source = self._get_account_users_source()
+ if account_users_source is not None:
+ return account_users_source.get_person_display_name_column()
property_keys = self.team.person_display_name_properties or PERSON_DEFAULT_DISPLAY_NAME_PROPERTIES
props = person_display_name_property_exprs(property_keys, "properties")
# nosemgrep: hogql-fstring-audit (property_keys from team config is admin-only, not user input)
diff --git a/posthog/settings/data_stores.py b/posthog/settings/data_stores.py
index f944a2753f31..d1baee8d4c87 100644
--- a/posthog/settings/data_stores.py
+++ b/posthog/settings/data_stores.py
@@ -869,6 +869,12 @@ PATCH_EVENT_LIST_MAX_OFFSET_PER_TEAM: set[int] = get_from_env(
CLICKHOUSE_EVENT_LIST_MAX_THREADS: int = get_from_env("CLICKHOUSE_EVENT_LIST_MAX_THREADS", 50, type_cast=int)
+CLICKHOUSE_ACCOUNT_USERS_MAX_THREADS: int | None = get_from_env(
+ "CLICKHOUSE_ACCOUNT_USERS_MAX_THREADS", optional=True, type_cast=int
+)
+if CLICKHOUSE_ACCOUNT_USERS_MAX_THREADS is not None and CLICKHOUSE_ACCOUNT_USERS_MAX_THREADS <= 0:
+ raise ImproperlyConfigured("CLICKHOUSE_ACCOUNT_USERS_MAX_THREADS must be a positive integer.")
+
WAREHOUSE_SOURCES_QUEUE_PARTITION_DATABASE_URL: str = os.getenv("WAREHOUSE_SOURCES_QUEUE_PARTITION_DATABASE_URL", "")
WAREHOUSE_SOURCES_QUEUE_PARTITION_SLACK_WEBHOOK_URL: str = os.getenv(
"WAREHOUSE_SOURCES_QUEUE_PARTITION_SLACK_WEBHOOK_URL", ""
diff --git a/products/customer_analytics/backend/facade/api.py b/products/customer_analytics/backend/facade/api.py
index bf9e3b57ec94..437f0da31e0a 100644
--- a/products/customer_analytics/backend/facade/api.py
+++ b/products/customer_analytics/backend/facade/api.py
@@ -28,7 +28,7 @@ from django.conf import settings
from django.contrib.postgres.aggregates import ArrayAgg
from django.contrib.postgres.fields import ArrayField
from django.core.cache import cache
-from django.core.exceptions import ValidationError
+from django.core.exceptions import PermissionDenied, ValidationError
from django.db import IntegrityError, transaction
from django.db.models import (
Aggregate,
@@ -132,7 +132,6 @@ from products.customer_analytics.backend.logic.account_filters import (
parse_email_search,
)
from products.customer_analytics.backend.logic.account_logo import resolve_logo_domain
-from products.customer_analytics.backend.logic.account_member_search import get_account_users_organization_id
from products.customer_analytics.backend.logic.custom_property_definitions import (
apply_option_side_effects,
coerce_is_big_number,
@@ -399,8 +398,7 @@ def _resolve_accessible_account(
def validate_account_users_access(team: Team, user: "User | None", account_id: str) -> None:
- """Raise `UserAccessControlError` unless `user` may read the users of this account."""
- get_account_users_organization_id(team=team, user=user, account_id=account_id)
+ raise PermissionDenied("Run the account-user query synchronously.")
def get_account(
diff --git a/products/customer_analytics/backend/hogql_queries/account_users_query_runner.py b/products/customer_analytics/backend/hogql_queries/account_users_query_runner.py
index 95547d94a4bd..1a460fac1094 100644
--- a/products/customer_analytics/backend/hogql_queries/account_users_query_runner.py
+++ b/products/customer_analytics/backend/hogql_queries/account_users_query_runner.py
@@ -1,83 +1,179 @@
-from posthog.schema import AccountUsersQuery, HogQLQueryResponse
+from typing import cast
+
+from django.conf import settings
+
+from posthog.schema import AccountUsersQuery, ActorsQuery, HogQLQueryResponse
from posthog.hogql import ast
+from posthog.hogql.constants import HogQLGlobalSettings, HogQLQuerySettings
+from posthog.hogql.context import HogQLContext
+from posthog.hogql.database.models import LazyTableToAdd
+from posthog.hogql.database.schema.persons import PersonsTable
+from posthog.hogql.errors import QueryError
from posthog.hogql.parser import parse_select
-from posthog.hogql.query import execute_hogql_query
+from posthog.hogql.visitor import TraversingVisitor
-from posthog.hogql_queries.query_runner import AnalyticsQueryRunner
-from posthog.models.user import User
+from posthog.hogql_queries.actor_strategies import ActorStrategy, PersonStrategy
+from posthog.hogql_queries.paginators import HogQLHasMorePaginator
+from posthog.hogql_queries.query_runner import AnalyticsQueryRunner, ExecutionMode
+from posthog.models import User
+from posthog.schema_enums import PersonsArgMaxVersion
-from products.access_control.backend.facade.user_access_control import UserAccessControlError
-from products.customer_analytics.backend.constants import ACCOUNT_USERS_QUERY_STATUS_LABEL_PREFIX
-from products.customer_analytics.backend.logic.account_member_search import (
- get_account_users_organization_id,
- list_organization_member_distinct_ids,
+from products.access_control.backend.facade.api import (
+ get_restricted_properties_with_group_type_index_for_team,
+ split_restricted_property_names,
)
+from products.access_control.backend.facade.user_access_control import UserAccessControlError
+from products.customer_analytics.backend.logic.account_persons import AccountPersonsReader
-class AccountUsersQueryRunner(AnalyticsQueryRunner[HogQLQueryResponse]):
- """Persons who use an account. Runs only as an `ActorsQuery` source, which hydrates the persons.
+class AccountUsersStrategy(PersonStrategy):
+ def filter_conditions(self) -> list[ast.Expr]:
+ conditions = PersonStrategy(
+ team=self.team,
+ user=self.user,
+ paginator=self.paginator,
+ query=self.query.model_copy(update={"search": None}),
+ ).filter_conditions()
+ if self.query.search and self.query.search.strip():
+ source = self.query.source
+ assert isinstance(source, AccountUsersQuery)
+ reader = AccountPersonsReader.create(team=self.team, user=self.user, account_id=source.accountId)
+ if reader is None:
+ raise UserAccessControlError("account", "viewer")
+ restrictions = get_restricted_properties_with_group_type_index_for_team(
+ user=self.user, team_id=self.team.pk
+ )
+ condition = reader.get_search_filter(
+ self.query.search.strip(), split_restricted_property_names(restrictions).person
+ )
+ conditions.append(condition)
+ return conditions
- The person set comes from the PostHog Cloud organization whose ID is the account's external ID,
- so it only works in PostHog's own project.
- """
+class AccountUsersPersonsTable(PersonsTable):
+ def lazy_select(
+ self, table_to_add: LazyTableToAdd, context: HogQLContext, node: ast.SelectQuery
+ ) -> ast.SelectQuery:
+ query = cast(ast.SelectQuery, super().lazy_select(table_to_add, context, node))
+ query.settings = (query.settings or HogQLQuerySettings()).model_copy(
+ update={"optimize_aggregation_in_order": False}
+ )
+ if self.filter is not None and query.where is not None:
+ query.prewhere = query.where
+ query.where = None
+ return query
+
+
+class AccountUsersMembershipReferences(TraversingVisitor):
+ def __init__(self, source_query: ast.SelectQuery | ast.SelectSetQuery) -> None:
+ super().__init__()
+ self.source_query = source_query
+
+ def visit_join_expr(self, node: ast.JoinExpr) -> None:
+ if node.table is self.source_query:
+ node.table = ast.Field(chain=["__account_members"])
+ super().visit_join_expr(node)
+
+
+class AccountUsersQueryRunner(AnalyticsQueryRunner[HogQLQueryResponse]):
query: AccountUsersQuery
+ def validate_execution_mode(self, execution_mode: ExecutionMode) -> None:
+ if execution_mode not in {
+ ExecutionMode.CALCULATE_BLOCKING_ALWAYS,
+ ExecutionMode.RECENT_CACHE_CALCULATE_BLOCKING_IF_STALE,
+ ExecutionMode.CACHE_ONLY_NEVER_CALCULATE,
+ }:
+ raise QueryError(
+ "Account-user queries only support synchronous execution. Use refresh='blocking' or 'force_blocking'."
+ )
+
+ def get_reader(self) -> AccountPersonsReader:
+ reader = AccountPersonsReader.create(team=self.team, user=self.user, account_id=self.query.accountId)
+ if reader is None:
+ raise UserAccessControlError("account", "viewer")
+ return reader
+
def validate_query_runner_access(self, user: User) -> bool:
- get_account_users_organization_id(team=self.team, user=user, account_id=self.query.accountId)
+ if AccountPersonsReader.create(team=self.team, user=user, account_id=self.query.accountId) is None:
+ raise UserAccessControlError("account", "viewer")
return True
def get_cache_key_variant(self) -> str:
- # Anonymous share-link viewers and cache-only reads skip `validate_query_runner_access`. A
- # separate key for them misses staff results, and a fresh run then fails the access check.
try:
- get_account_users_organization_id(team=self.team, user=self.user, account_id=self.query.accountId)
+ reader = self.get_reader()
except UserAccessControlError:
- return f"{super().get_cache_key_variant()}_denied"
- return super().get_cache_key_variant()
+ return "denied"
+ return reader.get_access_key()
- def query_status_labels(self) -> list[str]:
- return [f"{ACCOUNT_USERS_QUERY_STATUS_LABEL_PREFIX}{self.query.accountId}"]
+ def to_query(self) -> ast.SelectQuery:
+ self.modifiers.personsArgMaxVersion = PersonsArgMaxVersion.V2
+ return self.get_reader().get_source_query()
- def _list_distinct_ids(self) -> tuple[str, ...]:
- organization_id = get_account_users_organization_id(
- team=self.team, user=self.user, account_id=self.query.accountId
+ def to_actors_query(self) -> ast.SelectQuery:
+ if not settings.HOGQL_MATERIALIZED_CTE_SUPPORTED:
+ return self.to_query()
+ return cast(
+ ast.SelectQuery,
+ parse_select(
+ "SELECT person_id, distinct_ids, account_first_seen, account_last_seen FROM __account_members"
+ ),
)
- if organization_id is None:
- return ()
- # Narrowed by the access check above, which rejects any principal that is not a User.
- assert isinstance(self.user, User)
- return list_organization_member_distinct_ids(team=self.team, user=self.user, organization_id=organization_id)
- def to_query(self) -> ast.SelectQuery:
- distinct_ids = self._list_distinct_ids()
- condition: ast.Expr = (
- ast.CompareOperation(
- op=ast.CompareOperationOp.In,
- left=ast.Field(chain=["distinct_id"]),
- right=ast.Tuple(exprs=[ast.Constant(value=distinct_id) for distinct_id in distinct_ids]),
- )
- if distinct_ids
- else ast.Constant(value=False)
- )
- query = parse_select(
- "SELECT DISTINCT person_id FROM person_distinct_ids WHERE {condition}",
- placeholders={"condition": condition},
- )
- assert isinstance(query, ast.SelectQuery)
+ def prepare_actors_query(
+ self, query: ast.SelectQuery, source_query: ast.SelectQuery | ast.SelectSetQuery
+ ) -> ast.SelectQuery:
+ if not settings.HOGQL_MATERIALIZED_CTE_SUPPORTED:
+ return query
+ AccountUsersMembershipReferences(source_query).visit(query)
+ query.ctes = {
+ **(query.ctes or {}),
+ "__account_members": ast.CTE(
+ name="__account_members", expr=self.to_query(), cte_type="subquery", materialized=True
+ ),
+ }
return query
- def to_actors_query(self) -> ast.SelectQuery:
- return self.to_query()
+ def get_actors_query_settings(self) -> HogQLGlobalSettings:
+ return HogQLGlobalSettings(
+ max_threads=settings.CLICKHOUSE_ACCOUNT_USERS_MAX_THREADS,
+ enable_materialized_cte=True if settings.HOGQL_MATERIALIZED_CTE_SUPPORTED else None,
+ )
def _calculate(self) -> HogQLQueryResponse:
- return execute_hogql_query(
- query_type="AccountUsersQuery",
- query=self.to_query(),
- team=self.team,
- user=self.user,
- timings=self.timings,
- modifiers=self.modifiers,
- limit_context=self.limit_context,
+ raise QueryError("Use AccountUsersQuery as an ActorsQuery source.")
+
+ def get_actors_hogql_context(self, context: HogQLContext) -> HogQLContext:
+ assert context.database is not None
+ root = context.database.tables
+ persons = root.get_child(["persons"])
+ assert isinstance(persons.table, PersonsTable)
+ table = AccountUsersPersonsTable(fields=persons.table.fields, filter=persons.table.filter)
+ context.database = context.database.model_copy(
+ update={
+ "tables": root.model_copy(
+ update={"children": {**root.children, "persons": persons.model_copy(update={"table": table})}}
+ )
+ }
+ )
+ return context
+
+ def get_actors_strategy(self, query: ActorsQuery, paginator: HogQLHasMorePaginator) -> ActorStrategy:
+ return AccountUsersStrategy(team=self.team, query=query, paginator=paginator, user=self.user)
+
+ def validate_actors_query(self, query: ActorsQuery) -> None:
+ if not query.select or any(
+ column.strip() in {"person", "group", "actor", "session"} for column in query.select
+ ):
+ raise QueryError("Select explicit person columns for account users.")
+
+ def get_person_display_name_column(self) -> ast.Expr:
+ reader = self.get_reader()
+ restrictions = get_restricted_properties_with_group_type_index_for_team(user=self.user, team_id=self.team.pk)
+ return ast.Tuple(
+ exprs=[
+ reader.get_display_name(split_restricted_property_names(restrictions).person),
+ ast.Call(name="toString", args=[ast.Field(chain=["persons", "id"])]),
+ ]
)
diff --git a/products/customer_analytics/backend/hogql_queries/test/__snapshots__/test_account_users_query_runner.ambr b/products/customer_analytics/backend/hogql_queries/test/__snapshots__/test_account_users_query_runner.ambr
new file mode 100644
index 000000000000..18216816420c
--- /dev/null
+++ b/products/customer_analytics/backend/hogql_queries/test/__snapshots__/test_account_users_query_runner.ambr
@@ -0,0 +1,123 @@
+# serializer version: 1
+# name: TestAccountUsersQuery.test_widget_query_supports_display_names_filters_search_sort_and_pages_0
+ '''
+ SELECT tuple(coalesce(nullIf(toString(persons.properties___email), %(hogql_val_14)s), nullIf(toString(persons.properties___name), %(hogql_val_15)s), nullIf(toString(persons.properties___username), %(hogql_val_16)s), toString(persons.id)), toString(persons.id)),
+ persons.properties___plan AS plan,
+ source.account_last_seen AS account_last_seen
+ FROM
+ (SELECT pdi.person_id AS person_id,
+ groupUniqArray(101)(membership.distinct_id) AS distinct_ids,
+ min(toTimeZone(membership.first_seen, %(hogql_val_1)s)) AS account_first_seen,
+ max(toTimeZone(membership.last_seen, %(hogql_val_2)s)) AS account_last_seen
+ FROM person_group_membership AS membership GLOBAL
+ INNER JOIN
+ (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id,
+ person_distinct_id2.distinct_id AS distinct_id
+ FROM person_distinct_id2
+ WHERE and(equals(person_distinct_id2.team_id, 99999), in(person_distinct_id2.distinct_id,
+ (SELECT person_group_membership.distinct_id AS distinct_id
+ FROM person_group_membership
+ WHERE and(equals(person_group_membership.team_id, 99999), equals(person_group_membership.group_type_index, 0), equals(person_group_membership.group_key, %(hogql_val_0)s)))))
+ GROUP BY person_distinct_id2.distinct_id
+ HAVING equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0)) AS pdi ON equals(membership.distinct_id, pdi.distinct_id)
+ WHERE and(equals(membership.team_id, 99999), equals(membership.group_type_index, 0), equals(membership.group_key, %(hogql_val_3)s))
+ GROUP BY pdi.person_id) AS source GLOBAL
+ INNER JOIN
+ (SELECT person.id AS id,
+ replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_4)s), ''), 'null'), '^"|"$', '') AS properties___email,
+ replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_5)s), ''), 'null'), '^"|"$', '') AS properties___name,
+ replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_6)s), ''), 'null'), '^"|"$', '') AS properties___username,
+ replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_7)s), ''), 'null'), '^"|"$', '') AS properties___plan
+ FROM person PREWHERE in(tuple(person.id, person.version),
+ (SELECT person.id AS id,
+ max(person.version) AS version
+ FROM person
+ WHERE and(equals(person.team_id, 99999), in(person.id,
+ (SELECT source.person_id AS person_id
+ FROM
+ (SELECT pdi.person_id AS person_id, groupUniqArray(101)(membership.distinct_id) AS distinct_ids, min(toTimeZone(membership.first_seen, %(hogql_val_9)s)) AS account_first_seen, max(toTimeZone(membership.last_seen, %(hogql_val_10)s)) AS account_last_seen
+ FROM person_group_membership AS membership GLOBAL
+ INNER JOIN
+ (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id AS distinct_id
+ FROM person_distinct_id2
+ WHERE and(equals(person_distinct_id2.team_id, 99999), in(person_distinct_id2.distinct_id,
+ (SELECT person_group_membership.distinct_id AS distinct_id
+ FROM person_group_membership
+ WHERE and(equals(person_group_membership.team_id, 99999), equals(person_group_membership.group_type_index, 0), equals(person_group_membership.group_key, %(hogql_val_8)s)))))
+ GROUP BY person_distinct_id2.distinct_id
+ HAVING equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0)) AS pdi ON equals(membership.distinct_id, pdi.distinct_id)
+ WHERE and(equals(membership.team_id, 99999), equals(membership.group_type_index, 0), equals(membership.group_key, %(hogql_val_11)s))
+ GROUP BY pdi.person_id) AS source)))
+ GROUP BY person.id
+ HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, %(hogql_val_12)s), person.version), plus(now64(6, %(hogql_val_13)s), toIntervalDay(1))))))
+ WHERE equals(person.team_id, 99999) SETTINGS optimize_aggregation_in_order=0) AS persons ON equals(persons.id, source.person_id)
+ ORDER BY source.account_last_seen DESC
+ LIMIT 2
+ OFFSET 0 SETTINGS optimize_aggregation_in_order=0,
+ join_algorithm='auto'
+ '''
+# ---
+# name: TestAccountUsersQuery.test_widget_query_supports_display_names_filters_search_sort_and_pages_1
+ '''
+ WITH __account_members AS MATERIALIZED
+ (SELECT pdi.person_id AS person_id, groupUniqArray(101)(membership.distinct_id) AS distinct_ids, min(toTimeZone(membership.first_seen, %(hogql_val_11)s)) AS account_first_seen, max(toTimeZone(membership.last_seen, %(hogql_val_12)s)) AS account_last_seen
+ FROM person_group_membership AS membership GLOBAL
+ INNER JOIN
+ (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id, person_distinct_id2.distinct_id AS distinct_id
+ FROM person_distinct_id2
+ WHERE and(equals(person_distinct_id2.team_id, 99999), in(person_distinct_id2.distinct_id,
+ (SELECT person_group_membership.distinct_id AS distinct_id
+ FROM person_group_membership
+ WHERE and(equals(person_group_membership.team_id, 99999), equals(person_group_membership.group_type_index, 0), equals(person_group_membership.group_key, %(hogql_val_10)s)))))
+ GROUP BY person_distinct_id2.distinct_id
+ HAVING equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0)) AS pdi ON equals(membership.distinct_id, pdi.distinct_id)
+ WHERE and(equals(membership.team_id, 99999), equals(membership.group_type_index, 0), equals(membership.group_key, %(hogql_val_13)s))
+ GROUP BY pdi.person_id)
+ SELECT tuple(coalesce(nullIf(toString(persons.properties___email), %(hogql_val_6)s), nullIf(toString(persons.properties___name), %(hogql_val_7)s), nullIf(toString(persons.properties___username), %(hogql_val_8)s), toString(persons.id)), toString(persons.id)),
+ persons.properties___plan AS plan,
+ toTimeZone(source.account_last_seen, %(hogql_val_9)s) AS account_last_seen
+ FROM __account_members AS source GLOBAL
+ INNER JOIN
+ (SELECT person.id AS id,
+ replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_0)s), ''), 'null'), '^"|"$', '') AS properties___email,
+ replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_1)s), ''), 'null'), '^"|"$', '') AS properties___name,
+ replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_2)s), ''), 'null'), '^"|"$', '') AS properties___username,
+ replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(person.properties, %(hogql_val_3)s), ''), 'null'), '^"|"$', '') AS properties___plan
+ FROM person PREWHERE in(tuple(person.id, person.version),
+ (SELECT person.id AS id,
+ max(person.version) AS version
+ FROM person
+ WHERE and(equals(person.team_id, 99999), in(person.id,
+ (SELECT source.person_id AS person_id
+ FROM __account_members AS source)))
+ GROUP BY person.id
+ HAVING and(equals(argMax(person.is_deleted, person.version), 0), less(argMax(toTimeZone(person.created_at, %(hogql_val_4)s), person.version), plus(now64(6, %(hogql_val_5)s), toIntervalDay(1))))))
+ WHERE equals(person.team_id, 99999) SETTINGS optimize_aggregation_in_order=0) AS persons ON equals(persons.id, source.person_id)
+ ORDER BY toTimeZone(source.account_last_seen, %(hogql_val_14)s) DESC
+ LIMIT 2
+ OFFSET 0 SETTINGS optimize_aggregation_in_order=0,
+ join_algorithm='auto'
+ '''
+# ---
+# name: TestMembershipSchema.test_source_resolves_current_mappings_and_binds_account_key
+ '''
+ SELECT pdi.person_id AS person_id,
+ groupUniqArray(101)(membership.distinct_id) AS distinct_ids,
+ min(toTimeZone(membership.first_seen, %(hogql_val_1)s)) AS account_first_seen,
+ max(toTimeZone(membership.last_seen, %(hogql_val_2)s)) AS account_last_seen
+ FROM person_group_membership AS membership GLOBAL
+ INNER JOIN
+ (SELECT argMax(person_distinct_id2.person_id, person_distinct_id2.version) AS person_id,
+ person_distinct_id2.distinct_id AS distinct_id
+ FROM person_distinct_id2
+ WHERE and(equals(person_distinct_id2.team_id, 99999), in(person_distinct_id2.distinct_id,
+ (SELECT person_group_membership.distinct_id AS distinct_id
+ FROM person_group_membership
+ WHERE and(equals(person_group_membership.team_id, 99999), equals(person_group_membership.group_type_index, 0), equals(person_group_membership.group_key, %(hogql_val_0)s)))))
+ GROUP BY person_distinct_id2.distinct_id
+ HAVING equals(argMax(person_distinct_id2.is_deleted, person_distinct_id2.version), 0)) AS pdi ON equals(membership.distinct_id, pdi.distinct_id)
+ WHERE and(equals(membership.team_id, 99999), equals(membership.group_type_index, 0), equals(membership.group_key, %(hogql_val_3)s))
+ GROUP BY pdi.person_id
+ LIMIT 50000
+ '''
+# ---
diff --git a/products/customer_analytics/backend/hogql_queries/test/test_account_users_query_runner.py b/products/customer_analytics/backend/hogql_queries/test/test_account_users_query_runner.py
index c58bb02b66e2..f074d661e6a6 100644
--- a/products/customer_analytics/backend/hogql_queries/test/test_account_users_query_runner.py
+++ b/products/customer_analytics/backend/hogql_queries/test/test_account_users_query_runner.py
@@ -1,151 +1,376 @@
-from posthog.test.base import APIBaseTest, ClickhouseTestMixin, flush_persons_and_events
+import json
+from datetime import UTC, datetime
+from typing import Any
+
+from posthog.test.base import APIBaseTest, ClickhouseTestMixin, QueryMatchingTest, flush_persons_and_events
from unittest.mock import patch
+from django.test import SimpleTestCase, override_settings
+
from parameterized import parameterized
-from rest_framework import status
from posthog.schema import CacheMissResponse, QueryStatus
+from posthog.hogql.context import HogQLContext
+from posthog.hogql.database.database import Database
+from posthog.hogql.printer import prepare_and_print_ast
+from posthog.hogql.query import execute_hogql_query
+
+from posthog.clickhouse.client import sync_execute
from posthog.constants import AvailableFeature
+from posthog.hogql_queries.actors_query_runner import ActorsQueryRunner
from posthog.hogql_queries.query_runner import ExecutionMode, get_query_runner
-from posthog.models import Organization, OrganizationMembership, User
+from posthog.models import OrganizationMembership, PropertyDefinition, Team, User
from posthog.models.personal_api_key import PersonalAPIKey
from posthog.models.utils import generate_random_token_personal, hash_key_value
from posthog.test.persons import create_person
from products.access_control.backend.models.access_control import AccessControl
-from products.customer_analytics.backend.constants import ACCOUNT_USERS_QUERY_STATUS_LABEL_PREFIX
-from products.customer_analytics.backend.logic.account_member_search import list_organization_member_distinct_ids
+from products.access_control.backend.models.property_access_control import PropertyAccessControl
+from products.customer_analytics.backend.logic.account_persons import AccountPersonsReader
+from products.customer_analytics.backend.models import TeamCustomerAnalyticsConfig
from products.customer_analytics.backend.test.factories import create_account
-FLAG_CHECK = "products.customer_analytics.backend.logic.account_member_search.posthog_feature_flag_enabled"
+FLAG_CHECK = "products.customer_analytics.backend.logic.account_persons.posthog_feature_flag_enabled"
+
+
+class TestMembershipSchema(QueryMatchingTest, SimpleTestCase):
+ def test_source_resolves_current_mappings_and_binds_account_key(self) -> None:
+ team = Team(id=123, project_id=123)
+ group_key = "workspace' OR 1 = 1 --"
+ reader = AccountPersonsReader(team=team, user=User(), group_type_index=0, group_key=group_key)
+ context = HogQLContext(
+ team_id=team.pk,
+ team=team,
+ database=Database(),
+ enable_select_queries=True,
+ restricted_properties=set(),
+ use_new_events_schema=False,
+ )
+ sql, _ = prepare_and_print_ast(reader.get_source_query(), context=context, dialect="clickhouse")
+ self.assertQueryMatchesSnapshot(sql)
+ assert group_key not in sql
+ assert group_key in context.values.values()
-class TestAccountUsersQuery(ClickhouseTestMixin, APIBaseTest):
+@override_settings(HOGQL_MATERIALIZED_CTE_SUPPORTED=True)
+class TestAccountUsersQuery(ClickhouseTestMixin, APIBaseTest, QueryMatchingTest):
def setUp(self) -> None:
super().setUp()
- customer_org = Organization.objects.create(name="Customer")
- for email in ("[email protected]", "[email protected]"):
- member = User.objects.create(email=email, distinct_id=f"distinct-{email}")
- OrganizationMembership.objects.create(organization=customer_org, user=member)
- create_person(team=self.team, distinct_ids=[member.distinct_id], properties={"email": email})
- create_person(team=self.team, distinct_ids=["outsider"], properties={"email": "[email protected]"})
+ self.enterContext(patch(FLAG_CHECK, return_value=True))
+ TeamCustomerAnalyticsConfig.objects.update_or_create(team=self.team, defaults={"account_group_type_index": 0})
+ self.account = create_account(team_id=self.team.pk, external_id="workspace-demo")
+ self.endpoint = f"/api/projects/{self.team.pk}/query/"
+ self.first = create_person(
+ team=self.team,
+ distinct_ids=["member-one", "member-one-alias"],
+ properties={"email": "[email protected]", "name": "First member", "plan": "paid", "private_note": "hidden"},
+ )
+ self.second = create_person(
+ team=self.team, distinct_ids=["member-two"], properties={"email": "[email protected]", "plan": "free"}
+ )
+ create_person(team=self.team, distinct_ids=["non-member"], properties={"email": "[email protected]"})
flush_persons_and_events()
- self.account = create_account(team_id=self.team.id, external_id=str(customer_org.id))
- self.query = {
+ self._insert_membership("member-one", "2024-01-01", "2024-01-05")
+ self._insert_membership("member-one-alias", "2024-01-02", "2024-01-09")
+ self._insert_membership("member-two", "2024-01-03", "2024-01-06")
+ self._insert_membership("non-member", "2024-01-01", "2024-01-10", group_key="another-workspace")
+ self._insert_membership("non-member", "2024-01-01", "2024-01-10", group_type_index=1)
+ self._insert_membership(
+ "non-member", "2024-01-01", "2024-01-10", team_id=Team.objects.create(organization=self.organization).pk
+ )
+
+ def _get_widget_query(self, **overrides: Any) -> dict[str, Any]:
+ return {
"kind": "ActorsQuery",
"source": {"kind": "AccountUsersQuery", "accountId": str(self.account.id)},
- "select": ["properties.email"],
- "orderBy": ["properties.email DESC"],
+ "select": ["person_display_name -- Person", "properties.plan", "source.account_last_seen"],
+ "orderBy": ["source.account_last_seen DESC"],
+ **overrides,
}
- self.user.is_staff = True
- self.user.save()
- def _run(self, headers: dict[str, str] | None = None):
+ def _run_widget_query(self, **overrides: Any) -> Any:
return self.client.post(
- f"/api/projects/{self.team.id}/query/",
- {"query": self.query, "refresh": "blocking"},
+ self.endpoint,
+ {"query": self._get_widget_query(**overrides), "refresh": "blocking"},
format="json",
- headers=headers,
)
- def _hide_account_from_user(self) -> None:
- self.organization.available_product_features = [{"key": AvailableFeature.ACCESS_CONTROL}]
- self.organization.save()
- AccessControl.objects.create(
- team=self.team,
- resource="account",
- resource_id=str(self.account.id),
- access_level="none",
- organization_member=OrganizationMembership.objects.get(user=self.user, organization=self.organization),
+ @parameterized.expand([(False, None), (True, 120)])
+ def test_widget_query_supports_display_names_filters_search_sort_and_pages(
+ self, materialized_cte: bool, max_threads: int | None
+ ) -> None:
+ self.enterContext(
+ self.settings(
+ HOGQL_MATERIALIZED_CTE_SUPPORTED=materialized_cte, CLICKHOUSE_ACCOUNT_USERS_MAX_THREADS=max_threads
+ )
)
+ with patch("posthog.hogql_queries.paginators.execute_hogql_query", wraps=execute_hogql_query) as execute_query:
+ response = self._run_widget_query(limit=1)
+ assert execute_query.call_args.kwargs["settings"].max_threads == max_threads
+ assert response.status_code == 200, response.content
+ page = response.json()
+ assert page["results"][0][0] == {"display_name": "[email protected]", "id": str(self.first.uuid)}
+ assert page["hasMore"] and page["limit"] == 1
+ runner = get_query_runner(self._get_widget_query(limit=1), self.team, user=self.user)
+ assert isinstance(runner, ActorsQueryRunner)
+ query = runner.paginator.paginate(runner.to_query())
+ context = runner.build_hogql_context(enable_select_queries=True, modifiers=runner.modifiers)
+ sql, _ = prepare_and_print_ast(
+ query,
+ context=context,
+ dialect="clickhouse",
+ )
+ self.assertQueryMatchesSnapshot(sql)
+ plan = json.loads(sync_execute("EXPLAIN PLAN indexes=1, json=1 " + sql, context.values)[0][0])
+ pending = [plan[0]["Plan"]]
+ person_reads = []
+ while pending:
+ step = pending.pop()
+ if step["Node Type"] == "ReadFromMergeTree" and step.get("Description", "").endswith(".person"):
+ person_reads.append(step)
+ pending.extend(step.get("Plans", []))
+ assert person_reads
+ assert all(
+ any(
+ index["Type"] == "PrimaryKey" and "(id in " in index.get("Condition", "")
+ for index in step.get("Indexes", [])
+ )
+ for step in person_reads
+ )
+ response = self._run_widget_query(limit=1, offset=1)
+ assert response.status_code == 200, response.content
+ assert response.json()["results"][0][0]["id"] == str(self.second.uuid)
+ response = self._run_widget_query(
+ search="member-one-alias",
+ properties=[{"type": "person", "key": "plan", "operator": "exact", "value": "paid"}],
+ orderBy=["properties.plan ASC"],
+ )
+ assert response.status_code == 200, response.content
+ assert len(response.json()["results"]) == 1
+ assert response.json()["results"][0][0]["id"] == str(self.first.uuid)
- @patch(FLAG_CHECK, return_value=True)
- def test_staff_get_the_organization_members_as_persons(self, _flag) -> None:
- response = self._run()
-
- assert response.status_code == status.HTTP_200_OK, response.content
- assert response.json()["results"] == [["[email protected]"], ["[email protected]"]]
+ @parameterized.expand([("flag_disabled",), ("account_hidden",)])
+ def test_widget_cached_results_repeat_access_checks(self, condition: str) -> None:
+ response = self._run_widget_query()
+ assert response.status_code == 200, response.content
+ if condition == "account_hidden":
+ self.organization.available_product_features = [{"key": AvailableFeature.ACCESS_CONTROL}]
+ self.organization.save()
+ AccessControl.objects.create(
+ team=self.team,
+ resource="account",
+ resource_id=str(self.account.id),
+ access_level="none",
+ organization_member=self.organization_membership,
+ )
+ with patch(FLAG_CHECK, return_value=condition != "flag_disabled"):
+ response = self.client.post(self.endpoint, {"query": self._get_widget_query()}, format="json")
+ assert response.status_code == 400, response.content
@parameterized.expand(
[
- ("not_staff", False, True, False),
- ("flag_off", True, False, False),
- ("account_hidden", True, True, True),
+ ("no_user",),
+ ("different_user",),
+ ("person_properties",),
+ ("event_properties",),
+ ("group_properties",),
+ ("resource_access",),
]
)
- def test_denies_access_even_when_cached(self, _name, is_staff, flag_on, account_hidden) -> None:
- with patch(FLAG_CHECK, return_value=True):
- assert self._run().status_code == status.HTTP_200_OK
-
- self.user.is_staff = is_staff
- self.user.save()
- if account_hidden:
- self._hide_account_from_user()
- with patch(FLAG_CHECK, return_value=flag_on):
- response = self._run()
-
- assert response.status_code == status.HTTP_400_BAD_REQUEST, response.content
- assert "Access control failure" in response.json()["detail"]
-
- @patch(FLAG_CHECK, return_value=True)
- def test_cache_only_read_without_a_user_misses_staff_results(self, _flag) -> None:
- get_query_runner(self.query, self.team, user=self.user).run(
+ def test_widget_cache_read_requires_the_same_authorization_context(self, condition: str) -> None:
+ if condition.endswith("properties"):
+ self.organization.available_product_features = [{"key": AvailableFeature.PROPERTY_ACCESS_CONTROL}]
+ self.organization.save()
+ self.organization_membership.level = OrganizationMembership.Level.MEMBER
+ self.organization_membership.save()
+ elif condition == "resource_access":
+ self.organization.available_product_features = [{"key": AvailableFeature.ACCESS_CONTROL}]
+ self.organization.save()
+ get_query_runner(self._get_widget_query(), self.team, user=self.user).run(
ExecutionMode.CALCULATE_BLOCKING_ALWAYS, user=self.user
)
+ user: User | None = self.user
+ if condition == "no_user":
+ user = None
+ elif condition == "different_user":
+ user = User.objects.create_and_join(self.organization, "[email protected]", password=None)
+ elif condition == "resource_access":
+ AccessControl.objects.create(
+ team=self.team,
+ resource="dashboard",
+ access_level="none",
+ organization_member=self.organization_membership,
+ )
+ else:
+ property_type = {
+ "person_properties": PropertyDefinition.Type.PERSON,
+ "event_properties": PropertyDefinition.Type.EVENT,
+ "group_properties": PropertyDefinition.Type.GROUP,
+ }[condition]
+ self._restrict_properties(property_type)
+ response = get_query_runner(self._get_widget_query(), self.team, user=user).run(
+ ExecutionMode.CACHE_ONLY_NEVER_CALCULATE, user=user
+ )
+ assert isinstance(response, CacheMissResponse)
- response = get_query_runner(self.query, self.team).run(ExecutionMode.CACHE_ONLY_NEVER_CALCULATE)
+ def test_widget_display_name_and_search_hide_restricted_properties(self) -> None:
+ self._restrict_properties()
+ response = self._run_widget_query()
+ assert response.status_code == 200, response.content
+ assert all(row[0]["display_name"] == row[0]["id"] for row in response.json()["results"])
+ response = self._run_widget_query(search="[email protected]")
+ assert response.status_code == 200, response.content
+ assert response.json()["results"] == []
- assert isinstance(response, CacheMissResponse)
+ @parameterized.expand(
+ [
+ ([],),
+ (["person"],),
+ (["actor"],),
+ (["persons.id AS `group`", "group"],),
+ (["persons.id AS session", "session"],),
+ (["(SELECT count() FROM posthog.person_group_membership)"],),
+ ]
+ )
+ def test_widget_rejects_unsupported_columns(self, columns: list[str]) -> None:
+ self._restrict_properties()
+ assert self._run_widget_query(select=columns).status_code == 400
+
+ @parameterized.expand(
+ [
+ (refresh, interface)
+ for refresh in ("async", "force_async", "async_except_on_cache_miss", "lazy_async", "legacy")
+ for interface in ("generic", "wrapped")
+ ]
+ )
+ def test_async_account_user_requests_are_rejected(self, refresh: str, interface: str) -> None:
+ query = self._get_widget_query()
+ data: dict[str, Any] = {
+ "query": {"kind": "DataTableNode", "source": query} if interface == "wrapped" else query
+ }
+ if refresh == "legacy":
+ data["async"] = True
+ else:
+ data["refresh"] = refresh
+ endpoint = self.endpoint
+ with patch("posthog.hogql_queries.query_runner.enqueue_process_query_task") as enqueue:
+ response = self.client.post(endpoint, data, format="json")
+ assert response.status_code == 400, response.content
+ enqueue.assert_not_called()
- @parameterized.expand([("staff", True, status.HTTP_200_OK), ("not_staff", False, status.HTTP_403_FORBIDDEN)])
- @patch(FLAG_CHECK, return_value=True)
- def test_async_result_retrieval_repeats_the_access_check(self, _name, is_staff, expected_status, _flag) -> None:
- self.user.is_staff = is_staff
- self.user.save()
+ @parameterized.expand([(True, 403), (False, 200)])
+ def test_account_user_query_ids_are_unavailable(self, account_users: bool, expected: int) -> None:
stored = QueryStatus(
- id="account-users-query",
- team_id=self.team.id,
+ id="stored-query",
+ team_id=self.team.pk,
complete=True,
- labels=[f"{ACCOUNT_USERS_QUERY_STATUS_LABEL_PREFIX}{self.account.id}"],
- results={"results": [["[email protected]"]]},
+ labels=[f"customer-analytics-account-users:{self.account.id}"] if account_users else [],
+ results={"results": [["member"]]},
)
-
with patch("posthog.api.query.get_query_status", return_value=stored):
- response = self.client.get(f"/api/projects/{self.team.id}/query/account-users-query/")
+ response = self.client.get(f"/api/projects/{self.team.pk}/query/stored-query/")
+ assert response.status_code == expected, response.content
- assert response.status_code == expected_status, response.content
+ def _insert_membership(
+ self,
+ distinct_id: str,
+ first_seen: str,
+ last_seen: str,
+ *,
+ group_key: str = "workspace-demo",
+ group_type_index: int = 0,
+ team_id: int | None = None,
+ ) -> None:
+ sync_execute(
+ "INSERT INTO person_group_membership (team_id, group_type_index, group_key, distinct_id, first_seen, last_seen) VALUES",
+ [
+ {
+ "team_id": self.team.pk if team_id is None else team_id,
+ "group_type_index": group_type_index,
+ "group_key": group_key,
+ "distinct_id": distinct_id,
+ "first_seen": datetime.fromisoformat(first_seen).replace(tzinfo=UTC),
+ "last_seen": datetime.fromisoformat(last_seen).replace(tzinfo=UTC),
+ }
+ ],
+ )
+
+ @parameterized.expand(
+ [(side, interface) for side in ("mapping", "person") for interface in ("widget", "membership")]
+ )
+ def test_deleted_identity_is_not_returned(self, deleted_side: str, interface: str) -> None:
+ if deleted_side == "mapping":
+ sync_execute(
+ "INSERT INTO person_distinct_id2 (team_id, distinct_id, person_id, is_deleted, version) VALUES",
+ [
+ {
+ "team_id": self.team.pk,
+ "distinct_id": "member-two",
+ "person_id": self.second.uuid,
+ "is_deleted": 1,
+ "version": 100,
+ }
+ ],
+ )
+ else:
+ sync_execute(
+ "INSERT INTO person (team_id, id, properties, is_identified, is_deleted, version, created_at) VALUES",
+ [
+ {
+ "team_id": self.team.pk,
+ "id": self.second.uuid,
+ "properties": "{}",
+ "is_identified": 1,
+ "is_deleted": 1,
+ "version": 100,
+ "created_at": datetime(2024, 1, 1, tzinfo=UTC),
+ }
+ ],
+ )
+ columns = ["persons.id"] if interface == "widget" else ["source.distinct_ids"]
+ response = self._run_widget_query(select=columns, limit=1)
+ assert response.status_code == 200, response.content
+ page = response.json()
+ assert len(page["results"]) == 1
+ assert not page["hasMore"]
+ if interface == "widget":
+ assert page["results"][0][0] == str(self.first.uuid)
+ else:
+ assert set(page["results"][0][0]) == {"member-one", "member-one-alias"}
@parameterized.expand(
[
- ("query_scope_only", ["query:read"], status.HTTP_403_FORBIDDEN),
- ("all_scopes", ["query:read", "account:read", "person:read"], status.HTTP_200_OK),
+ ("query_account_only", ["query:read", "account:read"], 403, "query"),
+ ("query_person_only", ["query:read", "person:read"], 403, "query"),
+ ("query_both", ["query:read", "account:read", "person:read"], 200, "query"),
]
)
- @patch(FLAG_CHECK, return_value=True)
- def test_api_tokens_need_account_and_person_scopes(self, _name, scopes, expected_status, _flag) -> None:
+ def test_tokens_require_both_scopes(self, _name: str, scopes: list[str], expected: int, interface: str) -> None:
token = generate_random_token_personal()
PersonalAPIKey.objects.create(
- label="account users query",
- user=self.user,
- secure_value=hash_key_value(token),
- scopes=scopes,
- scoped_teams=[],
- scoped_organizations=[],
+ label="account persons", user=self.user, secure_value=hash_key_value(token), scopes=scopes
)
+ self.client.logout()
+ headers = {"authorization": f"Bearer {token}"}
+ response = self.client.post(
+ self.endpoint,
+ {"query": self._get_widget_query(), "refresh": "blocking"},
+ format="json",
+ headers=headers,
+ )
+ assert response.status_code == expected, response.content
- response = self._run(headers={"authorization": f"Bearer {token}"})
-
- assert response.status_code == expected_status, response.content
-
- def test_eu_lookup_failure_raises_instead_of_returning_no_members(self) -> None:
- with (
- patch(
- "products.customer_analytics.backend.logic.account_member_search.execute_hogql_query",
- side_effect=TimeoutError("eu lookup timed out"),
- ),
- self.assertRaises(TimeoutError),
- ):
- list_organization_member_distinct_ids(
- team=self.team, user=self.user, organization_id=str(Organization.objects.create(name="EU").id)
+ def _restrict_properties(self, property_type: int = PropertyDefinition.Type.PERSON) -> None:
+ self.organization.available_product_features = [{"key": AvailableFeature.PROPERTY_ACCESS_CONTROL}]
+ self.organization.save()
+ self.organization_membership.level = OrganizationMembership.Level.MEMBER
+ self.organization_membership.save()
+ for name in ["email", "name", "private_note"]:
+ definition = PropertyDefinition.objects.create(
+ team=self.team,
+ name=name,
+ type=property_type,
+ group_type_index=0 if property_type == PropertyDefinition.Type.GROUP else None,
)
+ PropertyAccessControl.objects.create(team=self.team, property_definition=definition, access_level="none")
diff --git a/products/customer_analytics/backend/logic/account_member_search.py b/products/customer_analytics/backend/logic/account_member_search.py
index a612259ed45c..f3e6a3d16306 100644
--- a/products/customer_analytics/backend/logic/account_member_search.py
+++ b/products/customer_analytics/backend/logic/account_member_search.py
@@ -1,7 +1,4 @@
from typing import cast
-from uuid import UUID
-
-from django.core.exceptions import ValidationError
import structlog
@@ -9,7 +6,6 @@ from posthog.hogql import ast
from posthog.hogql.errors import ResolutionError, TableAccessDeniedError
from posthog.hogql.query import execute_hogql_query
-from posthog.api.organization_member import organization_members_base_queryset
from posthog.clickhouse.query_tagging import Feature, Product, tags_context
from posthog.constants import INTERNAL_BOT_EMAIL_SUFFIX
from posthog.models import OrganizationMembership
@@ -17,15 +13,11 @@ from posthog.models.team import Team
from posthog.models.user import User
from posthog.permissions import posthog_feature_flag_enabled
-from products.access_control.backend.facade.user_access_control import UserAccessControl, UserAccessControlError
from products.customer_analytics.backend.constants import CUSTOMER_ANALYTICS_CSP_FLAG
-from products.customer_analytics.backend.models.account import Account
logger = structlog.get_logger(__name__)
ACCOUNT_MEMBER_SEARCH_MAX_ORGANIZATIONS = 100
-# The member list becomes a constant IN list in ClickHouse, which caps how many members one query can carry.
-ORGANIZATION_MEMBERS_MAX = 10_000
EU_ORGANIZATION_MEMBERS_VIEW = "eu_org_members"
@@ -61,7 +53,6 @@ def _query_eu_organization_members(
placeholders: dict[str, ast.Expr],
query_type: str,
log_event: str,
- raise_query_failures: bool = False,
) -> tuple[str, ...]:
try:
with tags_context(product=Product.CUSTOMER_ANALYTICS, feature=Feature.QUERY):
@@ -77,8 +68,6 @@ def _query_eu_organization_members(
return ()
except Exception as error:
logger.warning(f"{log_event}_failed", team_id=team.id, error_type=type(error).__name__)
- if raise_query_failures:
- raise
return ()
rows = cast(list[list[object]], response.results or [])
@@ -110,53 +99,3 @@ def list_account_external_ids_by_member_email(*, team: Team, user: User, email:
if organization_ids:
return organization_ids
return _list_eu_organization_ids(team, user, email)
-
-
-def list_organization_member_distinct_ids(*, team: Team, user: User, organization_id: str) -> tuple[str, ...]:
- """Distinct IDs of the PostHog Cloud users in an organization, from US Postgres or else the EU warehouse view."""
- us_distinct_ids = tuple(
- organization_members_base_queryset()
- .filter(organization_id=organization_id, user__distinct_id__isnull=False)
- .order_by("-joined_at")
- .values_list("user__distinct_id", flat=True)[:ORGANIZATION_MEMBERS_MAX]
- )
- if us_distinct_ids:
- return us_distinct_ids
- return _query_eu_organization_members(
- team,
- user,
- query=f"""
- SELECT distinct_id
- FROM {EU_ORGANIZATION_MEMBERS_VIEW}
- WHERE organization_id = {{organization_id}} AND distinct_id IS NOT NULL
- ORDER BY joined_at DESC
- LIMIT {ORGANIZATION_MEMBERS_MAX}
- """,
- placeholders={"organization_id": ast.Constant(value=organization_id)},
- query_type="customer_analytics_account_member_distinct_id_lookup",
- log_event="account_member_distinct_id_lookup",
- raise_query_failures=True,
- )
-
-
-def get_account_users_organization_id(*, team: Team, user: User | None, account_id: str) -> str | None:
- """The PostHog Cloud organization behind an account's users, or None when the account has none.
-
- Raises `UserAccessControlError` unless `user` is flagged staff who can see the account: members are
- read from any organization, not the caller's own.
- """
- if not isinstance(user, User) or not is_account_member_search_enabled(team, user):
- raise UserAccessControlError("account", "viewer")
- visible_accounts = UserAccessControl(user=user, team=team).filter_queryset_by_access_level(
- Account.objects.for_team(team.id)
- )
- try:
- account = visible_accounts.filter(id=account_id).values("external_id").first()
- except (ValidationError, ValueError):
- account = None
- if account is None:
- raise UserAccessControlError("account", "viewer", resource_id=account_id)
- try:
- return str(UUID(account["external_id"])) if account["external_id"] else None
- except ValueError:
- return None
diff --git a/products/customer_analytics/backend/logic/account_persons.py b/products/customer_analytics/backend/logic/account_persons.py
new file mode 100644
index 000000000000..f3769d48a82b
--- /dev/null
+++ b/products/customer_analytics/backend/logic/account_persons.py
@@ -0,0 +1,188 @@
+import json
+from hashlib import sha256
+
+from posthog.hogql import ast
+from posthog.hogql.database.schema.person_group_membership import PersonGroupMembershipTable
+from posthog.hogql.parser import parse_select
+from posthog.hogql.visitor import TraversingVisitor
+
+from posthog.hogql_queries.actor_strategies import person_email_property_keys
+from posthog.models import Team, User
+from posthog.permissions import posthog_feature_flag_enabled
+
+from products.access_control.backend.facade.api import get_restricted_properties_with_group_type_index_for_team
+from products.access_control.backend.facade.user_access_control import UserAccessControl
+from products.customer_analytics.backend.facade.api import get_accessible_account_id, get_account
+
+ACCOUNT_PERSONS_FLAG = "customer-analytics-account-persons-api"
+
+
+class MembershipTableReferences(TraversingVisitor):
+ def visit_join_expr(self, node: ast.JoinExpr) -> None:
+ if isinstance(node.table, ast.Field) and node.table.chain == ["posthog", "person_group_membership"]:
+ node.table_schema = PersonGroupMembershipTable()
+ super().visit_join_expr(node)
+
+
+class AccountPersonsReader:
+ @classmethod
+ def create(
+ cls,
+ *,
+ team: Team,
+ user: User | None,
+ account_id: str,
+ user_access_control: UserAccessControl | None = None,
+ ) -> "AccountPersonsReader | None":
+ if (
+ not isinstance(user, User)
+ or not user.is_active
+ or not posthog_feature_flag_enabled(
+ ACCOUNT_PERSONS_FLAG, str(user.distinct_id), organization_id=team.organization_id, team_id=team.pk
+ )
+ ):
+ return None
+ access = user_access_control or UserAccessControl(user=user, team=team)
+ if not access.check_access_level_for_resource("account", "viewer"):
+ return None
+ if get_accessible_account_id(team.pk, account_id, access) is None:
+ return None
+ account = get_account(team.pk, account_id=account_id)
+ group_type_index = team.customer_analytics_config.account_group_type_index
+ if account is None or group_type_index is None or group_type_index not in range(5):
+ return None
+ return cls(
+ team=team,
+ user=user,
+ group_type_index=group_type_index,
+ group_key=account.external_id,
+ account_id=account_id,
+ )
+
+ def __init__(
+ self, *, team: Team, user: User, group_type_index: int, group_key: str | None, account_id: str = ""
+ ) -> None:
+ self.account_id = account_id
+ self.team = team
+ self.user = user
+ self.group_type_index = group_type_index
+ self.group_key = group_key
+
+ def get_access_key(self) -> str:
+ restrictions = get_restricted_properties_with_group_type_index_for_team(user=self.user, team_id=self.team.pk)
+ access = UserAccessControl(user=self.user, team=self.team)
+ payload = {
+ "user_id": self.user.pk,
+ "group_type_index": self.group_type_index,
+ "group_key": self.group_key,
+ "properties": sorted(
+ (item.property_type, item.group_type_index if item.group_type_index is not None else -1, item.name)
+ for item in restrictions
+ ),
+ "blocked_objects": {scope: sorted(ids) for scope, ids in access.blocked_resource_ids_by_scope.items()},
+ "allowed_objects": {scope: sorted(ids) for scope, ids in access.allowlisted_resource_ids_by_scope.items()},
+ "blocked_resources": sorted(access.blocked_resources),
+ }
+ return sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest()
+
+ @staticmethod
+ def parse_membership_query(query: str, placeholders: dict[str, ast.Expr]) -> ast.SelectQuery:
+ select = parse_select(query, placeholders=placeholders)
+ assert isinstance(select, ast.SelectQuery)
+ MembershipTableReferences().visit(select)
+ return select
+
+ def get_source_query(self) -> ast.SelectQuery:
+ query = self.parse_membership_query(
+ """
+ SELECT pdi.person_id AS person_id,
+ groupUniqArray(101)(membership.distinct_id) AS distinct_ids,
+ min(membership.first_seen) AS account_first_seen,
+ max(membership.last_seen) AS account_last_seen
+ FROM posthog.person_group_membership AS membership
+ INNER JOIN (
+ SELECT argMax(person_id, version) AS person_id, distinct_id
+ FROM raw_person_distinct_ids
+ WHERE distinct_id IN (
+ SELECT distinct_id FROM posthog.person_group_membership
+ WHERE group_type_index = {group_type_index} AND group_key = {group_key}
+ )
+ GROUP BY distinct_id
+ HAVING argMax(is_deleted, version) = 0
+ ) AS pdi ON membership.distinct_id = pdi.distinct_id
+ WHERE membership.group_type_index = {group_type_index} AND membership.group_key = {group_key}
+ GROUP BY pdi.person_id
+ """,
+ placeholders={
+ "group_type_index": ast.Constant(value=self.group_type_index),
+ "group_key": ast.Constant(value=self.group_key),
+ },
+ )
+ assert isinstance(query, ast.SelectQuery)
+ assert query.select_from is not None and query.select_from.next_join is not None
+ query.select_from.next_join.join_type = "GLOBAL INNER JOIN"
+ return query
+
+ def get_display_name(self, restricted_properties: frozenset[str]) -> ast.Expr:
+ property_keys = self.team.person_display_name_properties or ["email", "name", "username"]
+ return ast.Call(
+ name="coalesce",
+ args=[
+ *[
+ ast.Call(
+ name="nullIf",
+ args=[
+ ast.Call(name="toString", args=[ast.Field(chain=["properties", key])]),
+ ast.Constant(value=""),
+ ],
+ )
+ for key in property_keys
+ if key not in restricted_properties
+ ],
+ ast.Call(name="toString", args=[ast.Field(chain=["persons", "id"])]),
+ ],
+ )
+
+ def get_search_filter(self, search: str, restricted_properties: frozenset[str]) -> ast.Expr:
+ expressions: list[ast.Expr] = [ast.Call(name="toString", args=[ast.Field(chain=["persons", "id"])])]
+ expressions.extend(
+ ast.Call(name="toString", args=[ast.Field(chain=["properties", key])])
+ for key in dict.fromkeys([*person_email_property_keys(self.team), "name"])
+ if key not in restricted_properties
+ )
+ condition = ast.Or(
+ exprs=[
+ *[
+ ast.CompareOperation(
+ left=expr,
+ op=ast.CompareOperationOp.ILike,
+ right=ast.Constant(value=f"%{search}%"),
+ )
+ for expr in expressions
+ ],
+ ast.CompareOperation(
+ left=ast.Field(chain=["persons", "id"]),
+ op=ast.CompareOperationOp.In,
+ right=self.parse_membership_query(
+ """
+ SELECT argMax(person_id, version) AS person_id
+ FROM raw_person_distinct_ids
+ WHERE distinct_id IN (
+ SELECT distinct_id FROM posthog.person_group_membership
+ WHERE group_type_index = {group_type_index}
+ AND group_key = {group_key}
+ AND ilike(distinct_id, {search})
+ )
+ GROUP BY distinct_id
+ HAVING argMax(is_deleted, version) = 0
+ """,
+ placeholders={
+ "group_type_index": ast.Constant(value=self.group_type_index),
+ "group_key": ast.Constant(value=self.group_key),
+ "search": ast.Constant(value=f"%{search}%"),
+ },
+ ),
+ ),
+ ]
+ )
+ return condition