:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(customer-analytics): sort accounts by custom property from value indexes (#129470)
Arthur Moreira de Deusd68cf99872e3
diff --git a/products/customer_analytics/backend/facade/api.py b/products/customer_analytics/backend/facade/api.pyindex 437f0da31e0a..dd6ce4718afe 100644--- a/products/customer_analytics/backend/facade/api.py+++ b/products/customer_analytics/backend/facade/api.py@@ -39,7 +39,6 @@ from django.db.models import (Exists,ExpressionWrapper,F,- FilteredRelation,FloatField,IntegerField,Max,@@ -53,7 +52,7 @@ from django.db.models import (Value,)from django.db.models.fields.json import KeyTextTransform-from django.db.models.functions import Cast, Coalesce+from django.db.models.functions import Cast, Coalesce, Leftfrom django.utils import timezoneimport structlog@@ -188,6 +187,7 @@ from products.customer_analytics.backend.models.custom_property_definition imporNUMERIC_DISPLAY_TYPES,DataType,)+from products.customer_analytics.backend.models.custom_property_value import SORT_VALUE_STR_PREFIX_LENGTHfrom products.customer_analytics.backend.tasks.tasks import send_announcementfrom products.notebooks.backend.facade import (api as notebooks,@@ -3432,7 +3432,6 @@ def _apply_account_table_sort(*,team_id: int,sort: contracts.AccountTableSort | None,- custom_property_display_types: dict[UUID, DisplayType],) -> QuerySet[Account]:if sort is None:return queryset.order_by("-created_at", "-id")@@ -3489,27 +3488,6 @@ def _apply_account_table_sort(queryset = queryset.annotate(_account_table_sort=Subquery(relationship_values, output_field=ArrayField(IntegerField())))- elif sort.kind == contracts.AccountTableSortKind.CUSTOM_PROPERTY:- if sort.definition_id is None:- raise InvalidAccountTableColumn("Custom property sorting requires a definition.")- display_type = custom_property_display_types[sort.definition_id]- value_field = {- DataType.STRING: "value_str",- DataType.NUMERIC: "value_num",- DataType.BOOLEAN: "value_bool",- DataType.DATETIME: "value_datetime",- }[DATA_TYPE_BY_DISPLAY_TYPE[display_type]]- queryset = queryset.annotate(- _custom_property_sort_value=FilteredRelation(- "custom_property_values",- condition=Q(- custom_property_values__team_id=team_id,- custom_property_values__definition_id=sort.definition_id,- custom_property_values__is_deleted=False,- ),- ),- _account_table_sort=F(f"_custom_property_sort_value__{value_field}"),- )order = F("_account_table_sort")primary_order = (@@ -3520,6 +3498,58 @@ def _apply_account_table_sort(return queryset.order_by(primary_order, "id")+def _list_accounts_sorted_by_custom_property(+ queryset: QuerySet[Account],+ *,+ team_id: int,+ sort: contracts.AccountTableSort,+ custom_property_display_types: dict[UUID, DisplayType],+ offset: int,+ limit: int,+) -> list[Account]:+ if sort.definition_id is None:+ raise InvalidAccountTableColumn("Custom property sorting requires a definition.")+ value_field = {+ DataType.STRING: "value_str",+ DataType.NUMERIC: "value_num",+ DataType.BOOLEAN: "value_bool",+ DataType.DATETIME: "value_datetime",+ }[DATA_TYPE_BY_DISPLAY_TYPE[custom_property_display_types[sort.definition_id]]]+ sort_value = F(f"custom_property_values__{value_field}")+ with_value = queryset.filter(+ custom_property_values__team_id=team_id,+ custom_property_values__definition_id=sort.definition_id,+ custom_property_values__is_deleted=False,+ **{f"custom_property_values__{value_field}__isnull": False},+ ).annotate(+ _account_table_sort=(+ Left(sort_value, SORT_VALUE_STR_PREFIX_LENGTH) if value_field == "value_str" else sort_value+ ),+ )+ sort_keys = [F("_account_table_sort"), *([sort_value] if value_field == "value_str" else [])]+ if sort.direction == contracts.AccountTableSortDirection.ASCENDING:+ with_value = with_value.order_by(*(key.asc() for key in sort_keys), "id")+ else:+ with_value = with_value.order_by(*(key.desc() for key in sort_keys), "-id")+ accounts: list[Account] = list(with_value[offset : offset + limit])+ if len(accounts) == limit:+ return accounts++ with_value_count = offset + len(accounts) if accounts or offset == 0 else with_value.count()+ without_value = queryset.exclude(+ Exists(+ CustomPropertyValue.objects.for_team(team_id).filter(+ account_id=OuterRef("pk"),+ definition_id=sort.definition_id,+ is_deleted=False,+ **{f"{value_field}__isnull": False},+ )+ )+ ).order_by("id")+ without_value_offset = max(offset - with_value_count, 0)+ return accounts + list(without_value[without_value_offset : without_value_offset + limit - len(accounts)])++class _PercentileCont(Aggregate):function = "PERCENTILE_CONT"template = "%(function)s(0.5) WITHIN GROUP (ORDER BY %(expressions)s)%(filter)s"@@ -3704,13 +3734,18 @@ def query_accounts_table(include_churned=include_churned,include_ignored=include_ignored,)- queryset = _apply_account_table_sort(- queryset,- team_id=team_id,- sort=sort,- custom_property_display_types=custom_property_display_types,- )- fetched_accounts = list(queryset[offset : offset + limit + 1])+ if sort is not None and sort.kind == contracts.AccountTableSortKind.CUSTOM_PROPERTY:+ fetched_accounts = _list_accounts_sorted_by_custom_property(+ queryset,+ team_id=team_id,+ sort=sort,+ custom_property_display_types=custom_property_display_types,+ offset=offset,+ limit=limit + 1,+ )+ else:+ queryset = _apply_account_table_sort(queryset, team_id=team_id, sort=sort)+ fetched_accounts = list(queryset[offset : offset + limit + 1])has_more = len(fetched_accounts) > limitaccounts = fetched_accounts[:limit]account_ids = [account.id for account in accounts]diff --git a/products/customer_analytics/backend/hogql_queries/test/test_accounts_table_query_runner.py b/products/customer_analytics/backend/hogql_queries/test/test_accounts_table_query_runner.pyindex 02d6fdbdbe25..a340de8e2554 100644--- a/products/customer_analytics/backend/hogql_queries/test/test_accounts_table_query_runner.py+++ b/products/customer_analytics/backend/hogql_queries/test/test_accounts_table_query_runner.py@@ -68,6 +68,7 @@ from products.customer_analytics.backend.models import (CustomPropertyValue,DisplayType,)+from products.customer_analytics.backend.models.custom_property_value import SORT_VALUE_STR_PREFIX_LENGTHfrom products.customer_analytics.backend.test.factories import create_account, create_custom_property_definitionfrom products.notebooks.backend.models import Notebook, ResourceNotebook@@ -1278,6 +1279,15 @@ class TestAccountsTableQueryRunner(BaseTest):CustomPropertyValue.objects.unscoped().create(team=self.team, account=beta, definition=custom_property_definition, value_num=50, is_deleted=True)+ long_text_definition = create_custom_property_definition(team_id=self.team.id, name="Notes")+ shared_prefix = "x" * SORT_VALUE_STR_PREFIX_LENGTH+ lower_id_account, higher_id_account = sorted([alpha, beta], key=lambda account: account.id)+ CustomPropertyValue.objects.unscoped().create(+ team=self.team, account=lower_id_account, definition=long_text_definition, value_str=f"{shared_prefix}b"+ )+ CustomPropertyValue.objects.unscoped().create(+ team=self.team, account=higher_id_account, definition=long_text_definition, value_str=f"{shared_prefix}a"+ )paginated = self._run(AccountsTableQuery(@@ -1309,6 +1319,10 @@ class TestAccountsTableQueryRunner(BaseTest):AccountsTableCustomPropertyColumn(definitionId=str(custom_property_definition.id)),["Beta", "Alpha", "Gamma"],),+ (+ AccountsTableCustomPropertyColumn(definitionId=str(long_text_definition.id)),+ [higher_id_account.name, lower_id_account.name, "Gamma"],+ ),]for column, expected_names in sort_cases:with self.subTest(column_type=type(column).__name__):@@ -1326,6 +1340,25 @@ class TestAccountsTableQueryRunner(BaseTest):)assert [row.name for row in response.results] == expected_names+ custom_property_column = AccountsTableCustomPropertyColumn(definitionId=str(custom_property_definition.id))+ for direction, expected_names in [+ (AccountsTableSortDirection.ASC, ["Beta", "Alpha", "Gamma"]),+ (AccountsTableSortDirection.DESC, ["Alpha", "Beta", "Gamma"]),+ ]:+ pages = [+ self._run(+ AccountsTableQuery(+ columns=[],+ filters=[],+ sort=AccountsTableSort(column=custom_property_column, direction=direction),+ limit=1,+ offset=offset,+ )+ ).results+ for offset in range(4)+ ]+ assert [row.name for page in pages for row in page] == expected_names, direction+def test_rejects_regex_custom_property_filters(self) -> None:definition = create_custom_property_definition(team_id=self.team.id, name="Plan")diff --git a/products/customer_analytics/backend/migrations/0064_custompropertyvalue_active_sort_indexes.py b/products/customer_analytics/backend/migrations/0064_custompropertyvalue_active_sort_indexes.pynew file mode 100644index 000000000000..57a123543a5d--- /dev/null+++ b/products/customer_analytics/backend/migrations/0064_custompropertyvalue_active_sort_indexes.py@@ -0,0 +1,59 @@+import django.db.models.functions.text+from django.db import migrations, models++from posthog.migration_helpers import SafeAddIndexConcurrently+++class Migration(migrations.Migration):+ atomic = False++ dependencies = [+ ("customer_analytics", "0063_teamcustomeranalyticsconfig_properties"),+ ]++ operations = [+ SafeAddIndexConcurrently(+ model_name="custompropertyvalue",+ index=models.Index(+ models.F("team"),+ models.F("definition"),+ django.db.models.functions.text.Left("value_str", 256),+ models.F("account"),+ condition=models.Q(("is_deleted", False), ("value_str__isnull", False)),+ name="cpv_active_sort_str_idx",+ ),+ ),+ SafeAddIndexConcurrently(+ model_name="custompropertyvalue",+ index=models.Index(+ models.F("team"),+ models.F("definition"),+ models.F("value_bool"),+ models.F("account"),+ condition=models.Q(("is_deleted", False), ("value_bool__isnull", False)),+ name="cpv_active_sort_bool_idx",+ ),+ ),+ SafeAddIndexConcurrently(+ model_name="custompropertyvalue",+ index=models.Index(+ models.F("team"),+ models.F("definition"),+ models.F("value_num"),+ models.F("account"),+ condition=models.Q(("is_deleted", False), ("value_num__isnull", False)),+ name="cpv_active_sort_num_idx",+ ),+ ),+ SafeAddIndexConcurrently(+ model_name="custompropertyvalue",+ index=models.Index(+ models.F("team"),+ models.F("definition"),+ models.F("value_datetime"),+ models.F("account"),+ condition=models.Q(("is_deleted", False), ("value_datetime__isnull", False)),+ name="cpv_active_sort_datetime_idx",+ ),+ ),+ ]diff --git a/products/customer_analytics/backend/migrations/max_migration.txt b/products/customer_analytics/backend/migrations/max_migration.txtindex 75d47ef7b9c2..d19f5dafa8bb 100644--- a/products/customer_analytics/backend/migrations/max_migration.txt+++ b/products/customer_analytics/backend/migrations/max_migration.txt@@ -1 +1 @@-0063_teamcustomeranalyticsconfig_properties+0064_custompropertyvalue_active_sort_indexesdiff --git a/products/customer_analytics/backend/models/custom_property_value.py b/products/customer_analytics/backend/models/custom_property_value.pyindex 9c76ed76110a..453f32f0a4dd 100644--- a/products/customer_analytics/backend/models/custom_property_value.py+++ b/products/customer_analytics/backend/models/custom_property_value.py@@ -1,5 +1,6 @@from django.db import models-from django.db.models import Q+from django.db.models import F, Q+from django.db.models.functions import Leftfrom posthog.models.scoping.root_mixin import TeamScopedRootMixinfrom posthog.models.utils import CreatedMetaFields, UUIDModel@@ -8,6 +9,19 @@ from posthog.models.utils import CreatedMetaFields, UUIDModel# service can tell this (retriable) race apart from other integrity errors by name.ACTIVE_VALUE_CONSTRAINT_NAME = "unique_active_custom_property_value"+SORT_VALUE_STR_PREFIX_LENGTH = 256+++def _active_sort_index(value_field: str, name: str, *, sort_expression: Left | None = None) -> models.Index:+ return models.Index(+ F("team"),+ F("definition"),+ sort_expression or F(value_field),+ F("account"),+ condition=Q(is_deleted=False, **{f"{value_field}__isnull": False}),+ name=name,+ )+class CustomPropertyValue(TeamScopedRootMixin, UUIDModel, CreatedMetaFields):team = models.ForeignKey("posthog.Team", on_delete=models.CASCADE, db_constraint=False, related_name="+")@@ -37,6 +51,14 @@ class CustomPropertyValue(TeamScopedRootMixin, UUIDModel, CreatedMetaFields):condition=Q(is_deleted=False),name="cpv_active_team_definition_idx",),+ _active_sort_index(+ "value_str",+ "cpv_active_sort_str_idx",+ sort_expression=Left("value_str", SORT_VALUE_STR_PREFIX_LENGTH),+ ),+ _active_sort_index("value_bool", "cpv_active_sort_bool_idx"),+ _active_sort_index("value_num", "cpv_active_sort_num_idx"),+ _active_sort_index("value_datetime", "cpv_active_sort_datetime_idx"),]constraints = [models.UniqueConstraint(