Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(customer-analytics): sort accounts by custom property from value indexes (#129470)

Arthur Moreira de Deusd68cf99872e3
diff --git a/products/customer_analytics/backend/facade/api.py b/products/customer_analytics/backend/facade/api.py
index 437f0da31e0a..dd6ce4718afe 100644
--- a/products/customer_analytics/backend/facade/api.py
+++ b/products/customer_analytics/backend/facade/api.py
@@ -39,7 +39,6 @@ from django.db.models import (
Exists,
ExpressionWrapper,
F,
- FilteredRelation,
FloatField,
IntegerField,
Max,
@@ -53,7 +52,7 @@ from django.db.models import (
Value,
)
from django.db.models.fields.json import KeyTextTransform
-from django.db.models.functions import Cast, Coalesce
+from django.db.models.functions import Cast, Coalesce, Left
from django.utils import timezone
import structlog
@@ -188,6 +187,7 @@ from products.customer_analytics.backend.models.custom_property_definition impor
NUMERIC_DISPLAY_TYPES,
DataType,
)
+from products.customer_analytics.backend.models.custom_property_value import SORT_VALUE_STR_PREFIX_LENGTH
from products.customer_analytics.backend.tasks.tasks import send_announcement
from products.notebooks.backend.facade import (
api as notebooks,
@@ -3432,7 +3432,6 @@ def _apply_account_table_sort(
*,
team_id: int,
sort: contracts.AccountTableSort | None,
- custom_property_display_types: dict[UUID, DisplayType],
) -> QuerySet[Account]:
if sort is None:
return queryset.order_by("-created_at", "-id")
@@ -3489,27 +3488,6 @@ def _apply_account_table_sort(
queryset = queryset.annotate(
_account_table_sort=Subquery(relationship_values, output_field=ArrayField(IntegerField()))
)
- elif sort.kind == contracts.AccountTableSortKind.CUSTOM_PROPERTY:
- if sort.definition_id is None:
- raise InvalidAccountTableColumn("Custom property sorting requires a definition.")
- display_type = custom_property_display_types[sort.definition_id]
- value_field = {
- DataType.STRING: "value_str",
- DataType.NUMERIC: "value_num",
- DataType.BOOLEAN: "value_bool",
- DataType.DATETIME: "value_datetime",
- }[DATA_TYPE_BY_DISPLAY_TYPE[display_type]]
- queryset = queryset.annotate(
- _custom_property_sort_value=FilteredRelation(
- "custom_property_values",
- condition=Q(
- custom_property_values__team_id=team_id,
- custom_property_values__definition_id=sort.definition_id,
- custom_property_values__is_deleted=False,
- ),
- ),
- _account_table_sort=F(f"_custom_property_sort_value__{value_field}"),
- )
order = F("_account_table_sort")
primary_order = (
@@ -3520,6 +3498,58 @@ def _apply_account_table_sort(
return queryset.order_by(primary_order, "id")
+def _list_accounts_sorted_by_custom_property(
+ queryset: QuerySet[Account],
+ *,
+ team_id: int,
+ sort: contracts.AccountTableSort,
+ custom_property_display_types: dict[UUID, DisplayType],
+ offset: int,
+ limit: int,
+) -> list[Account]:
+ if sort.definition_id is None:
+ raise InvalidAccountTableColumn("Custom property sorting requires a definition.")
+ value_field = {
+ DataType.STRING: "value_str",
+ DataType.NUMERIC: "value_num",
+ DataType.BOOLEAN: "value_bool",
+ DataType.DATETIME: "value_datetime",
+ }[DATA_TYPE_BY_DISPLAY_TYPE[custom_property_display_types[sort.definition_id]]]
+ sort_value = F(f"custom_property_values__{value_field}")
+ with_value = queryset.filter(
+ custom_property_values__team_id=team_id,
+ custom_property_values__definition_id=sort.definition_id,
+ custom_property_values__is_deleted=False,
+ **{f"custom_property_values__{value_field}__isnull": False},
+ ).annotate(
+ _account_table_sort=(
+ Left(sort_value, SORT_VALUE_STR_PREFIX_LENGTH) if value_field == "value_str" else sort_value
+ ),
+ )
+ sort_keys = [F("_account_table_sort"), *([sort_value] if value_field == "value_str" else [])]
+ if sort.direction == contracts.AccountTableSortDirection.ASCENDING:
+ with_value = with_value.order_by(*(key.asc() for key in sort_keys), "id")
+ else:
+ with_value = with_value.order_by(*(key.desc() for key in sort_keys), "-id")
+ accounts: list[Account] = list(with_value[offset : offset + limit])
+ if len(accounts) == limit:
+ return accounts
+
+ with_value_count = offset + len(accounts) if accounts or offset == 0 else with_value.count()
+ without_value = queryset.exclude(
+ Exists(
+ CustomPropertyValue.objects.for_team(team_id).filter(
+ account_id=OuterRef("pk"),
+ definition_id=sort.definition_id,
+ is_deleted=False,
+ **{f"{value_field}__isnull": False},
+ )
+ )
+ ).order_by("id")
+ without_value_offset = max(offset - with_value_count, 0)
+ return accounts + list(without_value[without_value_offset : without_value_offset + limit - len(accounts)])
+
+
class _PercentileCont(Aggregate):
function = "PERCENTILE_CONT"
template = "%(function)s(0.5) WITHIN GROUP (ORDER BY %(expressions)s)%(filter)s"
@@ -3704,13 +3734,18 @@ def query_accounts_table(
include_churned=include_churned,
include_ignored=include_ignored,
)
- queryset = _apply_account_table_sort(
- queryset,
- team_id=team_id,
- sort=sort,
- custom_property_display_types=custom_property_display_types,
- )
- fetched_accounts = list(queryset[offset : offset + limit + 1])
+ if sort is not None and sort.kind == contracts.AccountTableSortKind.CUSTOM_PROPERTY:
+ fetched_accounts = _list_accounts_sorted_by_custom_property(
+ queryset,
+ team_id=team_id,
+ sort=sort,
+ custom_property_display_types=custom_property_display_types,
+ offset=offset,
+ limit=limit + 1,
+ )
+ else:
+ queryset = _apply_account_table_sort(queryset, team_id=team_id, sort=sort)
+ fetched_accounts = list(queryset[offset : offset + limit + 1])
has_more = len(fetched_accounts) > limit
accounts = fetched_accounts[:limit]
account_ids = [account.id for account in accounts]
diff --git a/products/customer_analytics/backend/hogql_queries/test/test_accounts_table_query_runner.py b/products/customer_analytics/backend/hogql_queries/test/test_accounts_table_query_runner.py
index 02d6fdbdbe25..a340de8e2554 100644
--- a/products/customer_analytics/backend/hogql_queries/test/test_accounts_table_query_runner.py
+++ b/products/customer_analytics/backend/hogql_queries/test/test_accounts_table_query_runner.py
@@ -68,6 +68,7 @@ from products.customer_analytics.backend.models import (
CustomPropertyValue,
DisplayType,
)
+from products.customer_analytics.backend.models.custom_property_value import SORT_VALUE_STR_PREFIX_LENGTH
from products.customer_analytics.backend.test.factories import create_account, create_custom_property_definition
from products.notebooks.backend.models import Notebook, ResourceNotebook
@@ -1278,6 +1279,15 @@ class TestAccountsTableQueryRunner(BaseTest):
CustomPropertyValue.objects.unscoped().create(
team=self.team, account=beta, definition=custom_property_definition, value_num=50, is_deleted=True
)
+ long_text_definition = create_custom_property_definition(team_id=self.team.id, name="Notes")
+ shared_prefix = "x" * SORT_VALUE_STR_PREFIX_LENGTH
+ lower_id_account, higher_id_account = sorted([alpha, beta], key=lambda account: account.id)
+ CustomPropertyValue.objects.unscoped().create(
+ team=self.team, account=lower_id_account, definition=long_text_definition, value_str=f"{shared_prefix}b"
+ )
+ CustomPropertyValue.objects.unscoped().create(
+ team=self.team, account=higher_id_account, definition=long_text_definition, value_str=f"{shared_prefix}a"
+ )
paginated = self._run(
AccountsTableQuery(
@@ -1309,6 +1319,10 @@ class TestAccountsTableQueryRunner(BaseTest):
AccountsTableCustomPropertyColumn(definitionId=str(custom_property_definition.id)),
["Beta", "Alpha", "Gamma"],
),
+ (
+ AccountsTableCustomPropertyColumn(definitionId=str(long_text_definition.id)),
+ [higher_id_account.name, lower_id_account.name, "Gamma"],
+ ),
]
for column, expected_names in sort_cases:
with self.subTest(column_type=type(column).__name__):
@@ -1326,6 +1340,25 @@ class TestAccountsTableQueryRunner(BaseTest):
)
assert [row.name for row in response.results] == expected_names
+ custom_property_column = AccountsTableCustomPropertyColumn(definitionId=str(custom_property_definition.id))
+ for direction, expected_names in [
+ (AccountsTableSortDirection.ASC, ["Beta", "Alpha", "Gamma"]),
+ (AccountsTableSortDirection.DESC, ["Alpha", "Beta", "Gamma"]),
+ ]:
+ pages = [
+ self._run(
+ AccountsTableQuery(
+ columns=[],
+ filters=[],
+ sort=AccountsTableSort(column=custom_property_column, direction=direction),
+ limit=1,
+ offset=offset,
+ )
+ ).results
+ for offset in range(4)
+ ]
+ assert [row.name for page in pages for row in page] == expected_names, direction
+
def test_rejects_regex_custom_property_filters(self) -> None:
definition = create_custom_property_definition(team_id=self.team.id, name="Plan")
diff --git a/products/customer_analytics/backend/migrations/0064_custompropertyvalue_active_sort_indexes.py b/products/customer_analytics/backend/migrations/0064_custompropertyvalue_active_sort_indexes.py
new file mode 100644
index 000000000000..57a123543a5d
--- /dev/null
+++ b/products/customer_analytics/backend/migrations/0064_custompropertyvalue_active_sort_indexes.py
@@ -0,0 +1,59 @@
+import django.db.models.functions.text
+from django.db import migrations, models
+
+from posthog.migration_helpers import SafeAddIndexConcurrently
+
+
+class Migration(migrations.Migration):
+ atomic = False
+
+ dependencies = [
+ ("customer_analytics", "0063_teamcustomeranalyticsconfig_properties"),
+ ]
+
+ operations = [
+ SafeAddIndexConcurrently(
+ model_name="custompropertyvalue",
+ index=models.Index(
+ models.F("team"),
+ models.F("definition"),
+ django.db.models.functions.text.Left("value_str", 256),
+ models.F("account"),
+ condition=models.Q(("is_deleted", False), ("value_str__isnull", False)),
+ name="cpv_active_sort_str_idx",
+ ),
+ ),
+ SafeAddIndexConcurrently(
+ model_name="custompropertyvalue",
+ index=models.Index(
+ models.F("team"),
+ models.F("definition"),
+ models.F("value_bool"),
+ models.F("account"),
+ condition=models.Q(("is_deleted", False), ("value_bool__isnull", False)),
+ name="cpv_active_sort_bool_idx",
+ ),
+ ),
+ SafeAddIndexConcurrently(
+ model_name="custompropertyvalue",
+ index=models.Index(
+ models.F("team"),
+ models.F("definition"),
+ models.F("value_num"),
+ models.F("account"),
+ condition=models.Q(("is_deleted", False), ("value_num__isnull", False)),
+ name="cpv_active_sort_num_idx",
+ ),
+ ),
+ SafeAddIndexConcurrently(
+ model_name="custompropertyvalue",
+ index=models.Index(
+ models.F("team"),
+ models.F("definition"),
+ models.F("value_datetime"),
+ models.F("account"),
+ condition=models.Q(("is_deleted", False), ("value_datetime__isnull", False)),
+ name="cpv_active_sort_datetime_idx",
+ ),
+ ),
+ ]
diff --git a/products/customer_analytics/backend/migrations/max_migration.txt b/products/customer_analytics/backend/migrations/max_migration.txt
index 75d47ef7b9c2..d19f5dafa8bb 100644
--- a/products/customer_analytics/backend/migrations/max_migration.txt
+++ b/products/customer_analytics/backend/migrations/max_migration.txt
@@ -1 +1 @@
-0063_teamcustomeranalyticsconfig_properties
+0064_custompropertyvalue_active_sort_indexes
diff --git a/products/customer_analytics/backend/models/custom_property_value.py b/products/customer_analytics/backend/models/custom_property_value.py
index 9c76ed76110a..453f32f0a4dd 100644
--- a/products/customer_analytics/backend/models/custom_property_value.py
+++ b/products/customer_analytics/backend/models/custom_property_value.py
@@ -1,5 +1,6 @@
from django.db import models
-from django.db.models import Q
+from django.db.models import F, Q
+from django.db.models.functions import Left
from posthog.models.scoping.root_mixin import TeamScopedRootMixin
from posthog.models.utils import CreatedMetaFields, UUIDModel
@@ -8,6 +9,19 @@ from posthog.models.utils import CreatedMetaFields, UUIDModel
# service can tell this (retriable) race apart from other integrity errors by name.
ACTIVE_VALUE_CONSTRAINT_NAME = "unique_active_custom_property_value"
+SORT_VALUE_STR_PREFIX_LENGTH = 256
+
+
+def _active_sort_index(value_field: str, name: str, *, sort_expression: Left | None = None) -> models.Index:
+ return models.Index(
+ F("team"),
+ F("definition"),
+ sort_expression or F(value_field),
+ F("account"),
+ condition=Q(is_deleted=False, **{f"{value_field}__isnull": False}),
+ name=name,
+ )
+
class CustomPropertyValue(TeamScopedRootMixin, UUIDModel, CreatedMetaFields):
team = models.ForeignKey("posthog.Team", on_delete=models.CASCADE, db_constraint=False, related_name="+")
@@ -37,6 +51,14 @@ class CustomPropertyValue(TeamScopedRootMixin, UUIDModel, CreatedMetaFields):
condition=Q(is_deleted=False),
name="cpv_active_team_definition_idx",
),
+ _active_sort_index(
+ "value_str",
+ "cpv_active_sort_str_idx",
+ sort_expression=Left("value_str", SORT_VALUE_STR_PREFIX_LENGTH),
+ ),
+ _active_sort_index("value_bool", "cpv_active_sort_bool_idx"),
+ _active_sort_index("value_num", "cpv_active_sort_num_idx"),
+ _active_sort_index("value_datetime", "cpv_active_sort_datetime_idx"),
]
constraints = [
models.UniqueConstraint(