:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(customer-analytics): add membership backfill engine (#123585)
Arthur Moreira de Deus5c8167a63458
diff --git a/.github/new-events-schema-targets.txt b/.github/new-events-schema-targets.txtindex cdbdda95b3a7..68d14640dbac 100644--- a/.github/new-events-schema-targets.txt+++ b/.github/new-events-schema-targets.txt@@ -29,6 +29,7 @@ products/ai_observability/backend/tests/test_facade_api.pyproducts/ai_observability/backend/tests/test_trace_views.pyproducts/cohortsproducts/customer_analytics/backend/hogql_queries+products/customer_analytics/backend/test/test_membership_backfill.pyproducts/error_tracking/backend/hogql_queriesproducts/error_tracking/backend/tests/test_issue_summaries.pyproducts/experimentsdiff --git a/posthog/api/test/dashboards/__snapshots__/test_dashboard.ambr b/posthog/api/test/dashboards/__snapshots__/test_dashboard.ambrindex 415eaaadd35a..2ca626f95177 100644--- a/posthog/api/test/dashboards/__snapshots__/test_dashboard.ambr+++ b/posthog/api/test/dashboards/__snapshots__/test_dashboard.ambr@@ -462,7 +462,8 @@"customer_analytics_teamcustomeranalyticsconfig"."account_group_type_index","customer_analytics_teamcustomeranalyticsconfig"."default_pinned_properties","customer_analytics_teamcustomeranalyticsconfig"."account_track_rules",- "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at"+ "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at",+ "customer_analytics_teamcustomeranalyticsconfig"."properties"FROM "customer_analytics_teamcustomeranalyticsconfig"WHERE "customer_analytics_teamcustomeranalyticsconfig"."team_id" = 99999LIMIT 21@@ -1732,7 +1733,8 @@"customer_analytics_teamcustomeranalyticsconfig"."account_group_type_index","customer_analytics_teamcustomeranalyticsconfig"."default_pinned_properties","customer_analytics_teamcustomeranalyticsconfig"."account_track_rules",- "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at"+ "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at",+ "customer_analytics_teamcustomeranalyticsconfig"."properties"FROM "customer_analytics_teamcustomeranalyticsconfig"WHERE "customer_analytics_teamcustomeranalyticsconfig"."team_id" = 99999LIMIT 21@@ -2673,7 +2675,8 @@"customer_analytics_teamcustomeranalyticsconfig"."account_group_type_index","customer_analytics_teamcustomeranalyticsconfig"."default_pinned_properties","customer_analytics_teamcustomeranalyticsconfig"."account_track_rules",- "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at"+ "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at",+ "customer_analytics_teamcustomeranalyticsconfig"."properties"FROM "customer_analytics_teamcustomeranalyticsconfig"WHERE "customer_analytics_teamcustomeranalyticsconfig"."team_id" = 99999LIMIT 21@@ -3967,7 +3970,8 @@"customer_analytics_teamcustomeranalyticsconfig"."account_group_type_index","customer_analytics_teamcustomeranalyticsconfig"."default_pinned_properties","customer_analytics_teamcustomeranalyticsconfig"."account_track_rules",- "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at"+ "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at",+ "customer_analytics_teamcustomeranalyticsconfig"."properties"FROM "customer_analytics_teamcustomeranalyticsconfig"WHERE "customer_analytics_teamcustomeranalyticsconfig"."team_id" = 99999LIMIT 21@@ -6609,7 +6613,8 @@"customer_analytics_teamcustomeranalyticsconfig"."account_group_type_index","customer_analytics_teamcustomeranalyticsconfig"."default_pinned_properties","customer_analytics_teamcustomeranalyticsconfig"."account_track_rules",- "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at"+ "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at",+ "customer_analytics_teamcustomeranalyticsconfig"."properties"FROM "customer_analytics_teamcustomeranalyticsconfig"WHERE "customer_analytics_teamcustomeranalyticsconfig"."team_id" = 99999LIMIT 21@@ -7899,7 +7904,8 @@"customer_analytics_teamcustomeranalyticsconfig"."account_group_type_index","customer_analytics_teamcustomeranalyticsconfig"."default_pinned_properties","customer_analytics_teamcustomeranalyticsconfig"."account_track_rules",- "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at"+ "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at",+ "customer_analytics_teamcustomeranalyticsconfig"."properties"FROM "customer_analytics_teamcustomeranalyticsconfig"WHERE "customer_analytics_teamcustomeranalyticsconfig"."team_id" = 99999LIMIT 21@@ -9707,7 +9713,8 @@"customer_analytics_teamcustomeranalyticsconfig"."account_group_type_index","customer_analytics_teamcustomeranalyticsconfig"."default_pinned_properties","customer_analytics_teamcustomeranalyticsconfig"."account_track_rules",- "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at"+ "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at",+ "customer_analytics_teamcustomeranalyticsconfig"."properties"FROM "customer_analytics_teamcustomeranalyticsconfig"WHERE "customer_analytics_teamcustomeranalyticsconfig"."team_id" = 99999LIMIT 21@@ -10748,7 +10755,8 @@"customer_analytics_teamcustomeranalyticsconfig"."account_group_type_index","customer_analytics_teamcustomeranalyticsconfig"."default_pinned_properties","customer_analytics_teamcustomeranalyticsconfig"."account_track_rules",- "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at"+ "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at",+ "customer_analytics_teamcustomeranalyticsconfig"."properties"FROM "customer_analytics_teamcustomeranalyticsconfig"WHERE "customer_analytics_teamcustomeranalyticsconfig"."team_id" = 99999LIMIT 21@@ -13425,7 +13433,8 @@"customer_analytics_teamcustomeranalyticsconfig"."account_group_type_index","customer_analytics_teamcustomeranalyticsconfig"."default_pinned_properties","customer_analytics_teamcustomeranalyticsconfig"."account_track_rules",- "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at"+ "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at",+ "customer_analytics_teamcustomeranalyticsconfig"."properties"FROM "customer_analytics_teamcustomeranalyticsconfig"WHERE "customer_analytics_teamcustomeranalyticsconfig"."team_id" = 99999LIMIT 21@@ -14464,7 +14473,8 @@"customer_analytics_teamcustomeranalyticsconfig"."account_group_type_index","customer_analytics_teamcustomeranalyticsconfig"."default_pinned_properties","customer_analytics_teamcustomeranalyticsconfig"."account_track_rules",- "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at"+ "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at",+ "customer_analytics_teamcustomeranalyticsconfig"."properties"FROM "customer_analytics_teamcustomeranalyticsconfig"WHERE "customer_analytics_teamcustomeranalyticsconfig"."team_id" = 99999LIMIT 21@@ -15519,7 +15529,8 @@"customer_analytics_teamcustomeranalyticsconfig"."account_group_type_index","customer_analytics_teamcustomeranalyticsconfig"."default_pinned_properties","customer_analytics_teamcustomeranalyticsconfig"."account_track_rules",- "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at"+ "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at",+ "customer_analytics_teamcustomeranalyticsconfig"."properties"FROM "customer_analytics_teamcustomeranalyticsconfig"WHERE "customer_analytics_teamcustomeranalyticsconfig"."team_id" = 99999LIMIT 21@@ -18126,7 +18137,8 @@"customer_analytics_teamcustomeranalyticsconfig"."account_group_type_index","customer_analytics_teamcustomeranalyticsconfig"."default_pinned_properties","customer_analytics_teamcustomeranalyticsconfig"."account_track_rules",- "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at"+ "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at",+ "customer_analytics_teamcustomeranalyticsconfig"."properties"FROM "customer_analytics_teamcustomeranalyticsconfig"WHERE "customer_analytics_teamcustomeranalyticsconfig"."team_id" = 99999LIMIT 21@@ -19153,7 +19165,8 @@"customer_analytics_teamcustomeranalyticsconfig"."account_group_type_index","customer_analytics_teamcustomeranalyticsconfig"."default_pinned_properties","customer_analytics_teamcustomeranalyticsconfig"."account_track_rules",- "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at"+ "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at",+ "customer_analytics_teamcustomeranalyticsconfig"."properties"FROM "customer_analytics_teamcustomeranalyticsconfig"WHERE "customer_analytics_teamcustomeranalyticsconfig"."team_id" = 99999LIMIT 21@@ -20190,7 +20203,8 @@"customer_analytics_teamcustomeranalyticsconfig"."account_group_type_index","customer_analytics_teamcustomeranalyticsconfig"."default_pinned_properties","customer_analytics_teamcustomeranalyticsconfig"."account_track_rules",- "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at"+ "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at",+ "customer_analytics_teamcustomeranalyticsconfig"."properties"FROM "customer_analytics_teamcustomeranalyticsconfig"WHERE "customer_analytics_teamcustomeranalyticsconfig"."team_id" = 99999LIMIT 21@@ -22962,7 +22976,8 @@"customer_analytics_teamcustomeranalyticsconfig"."account_group_type_index","customer_analytics_teamcustomeranalyticsconfig"."default_pinned_properties","customer_analytics_teamcustomeranalyticsconfig"."account_track_rules",- "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at"+ "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at",+ "customer_analytics_teamcustomeranalyticsconfig"."properties"FROM "customer_analytics_teamcustomeranalyticsconfig"WHERE "customer_analytics_teamcustomeranalyticsconfig"."team_id" = 99999LIMIT 21@@ -24077,7 +24092,8 @@"customer_analytics_teamcustomeranalyticsconfig"."account_group_type_index","customer_analytics_teamcustomeranalyticsconfig"."default_pinned_properties","customer_analytics_teamcustomeranalyticsconfig"."account_track_rules",- "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at"+ "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at",+ "customer_analytics_teamcustomeranalyticsconfig"."properties"FROM "customer_analytics_teamcustomeranalyticsconfig"WHERE "customer_analytics_teamcustomeranalyticsconfig"."team_id" = 99999LIMIT 21@@ -25102,7 +25118,8 @@"customer_analytics_teamcustomeranalyticsconfig"."account_group_type_index","customer_analytics_teamcustomeranalyticsconfig"."default_pinned_properties","customer_analytics_teamcustomeranalyticsconfig"."account_track_rules",- "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at"+ "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at",+ "customer_analytics_teamcustomeranalyticsconfig"."properties"FROM "customer_analytics_teamcustomeranalyticsconfig"WHERE "customer_analytics_teamcustomeranalyticsconfig"."team_id" = 99999LIMIT 21@@ -27740,7 +27757,8 @@"customer_analytics_teamcustomeranalyticsconfig"."account_group_type_index","customer_analytics_teamcustomeranalyticsconfig"."default_pinned_properties","customer_analytics_teamcustomeranalyticsconfig"."account_track_rules",- "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at"+ "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at",+ "customer_analytics_teamcustomeranalyticsconfig"."properties"FROM "customer_analytics_teamcustomeranalyticsconfig"WHERE "customer_analytics_teamcustomeranalyticsconfig"."team_id" = 99999LIMIT 21@@ -28980,7 +28998,8 @@"customer_analytics_teamcustomeranalyticsconfig"."account_group_type_index","customer_analytics_teamcustomeranalyticsconfig"."default_pinned_properties","customer_analytics_teamcustomeranalyticsconfig"."account_track_rules",- "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at"+ "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at",+ "customer_analytics_teamcustomeranalyticsconfig"."properties"FROM "customer_analytics_teamcustomeranalyticsconfig"WHERE "customer_analytics_teamcustomeranalyticsconfig"."team_id" = 99999LIMIT 21@@ -30057,7 +30076,8 @@"customer_analytics_teamcustomeranalyticsconfig"."account_group_type_index","customer_analytics_teamcustomeranalyticsconfig"."default_pinned_properties","customer_analytics_teamcustomeranalyticsconfig"."account_track_rules",- "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at"+ "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at",+ "customer_analytics_teamcustomeranalyticsconfig"."properties"FROM "customer_analytics_teamcustomeranalyticsconfig"WHERE "customer_analytics_teamcustomeranalyticsconfig"."team_id" = 99999LIMIT 21@@ -32704,7 +32724,8 @@"customer_analytics_teamcustomeranalyticsconfig"."account_group_type_index","customer_analytics_teamcustomeranalyticsconfig"."default_pinned_properties","customer_analytics_teamcustomeranalyticsconfig"."account_track_rules",- "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at"+ "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at",+ "customer_analytics_teamcustomeranalyticsconfig"."properties"FROM "customer_analytics_teamcustomeranalyticsconfig"WHERE "customer_analytics_teamcustomeranalyticsconfig"."team_id" = 99999LIMIT 21diff --git a/products/customer_analytics/backend/facade/membership.py b/products/customer_analytics/backend/facade/membership.pynew file mode 100644index 000000000000..1af0e15de4a9--- /dev/null+++ b/products/customer_analytics/backend/facade/membership.py@@ -0,0 +1,3 @@+from products.customer_analytics.backend.logic.membership_backfill import get_membership_readiness++__all__ = ["get_membership_readiness"]diff --git a/products/customer_analytics/backend/logic/feature_gate.py b/products/customer_analytics/backend/logic/feature_gate.pynew file mode 100644index 000000000000..6db229722a48--- /dev/null+++ b/products/customer_analytics/backend/logic/feature_gate.py@@ -0,0 +1,19 @@+import posthoganalytics++from posthog.models.team import Team++from products.customer_analytics.backend.constants import CUSTOMER_ANALYTICS_CSP_FLAG+++def is_customer_analytics_active(team: Team, *, raise_on_unknown: bool = False) -> bool:+ organization_id = str(team.organization_id)+ enabled = posthoganalytics.feature_enabled(+ CUSTOMER_ANALYTICS_CSP_FLAG,+ organization_id,+ groups={"organization": organization_id},+ only_evaluate_locally=False,+ send_feature_flag_events=False,+ )+ if enabled is None and raise_on_unknown:+ raise RuntimeError("Customer analytics flag evaluation is unavailable")+ return bool(enabled)diff --git a/products/customer_analytics/backend/logic/membership_backfill.py b/products/customer_analytics/backend/logic/membership_backfill.pynew file mode 100644index 000000000000..b69e12435e78--- /dev/null+++ b/products/customer_analytics/backend/logic/membership_backfill.py@@ -0,0 +1,797 @@+from collections.abc import Callable, Mapping, Sequence+from dataclasses import replace+from datetime import UTC, date, datetime, timedelta+from enum import StrEnum+from math import ceil+from time import monotonic, time_ns+from typing import cast++from django.conf import settings+from django.db import transaction+from django.db.models import Exists, F, Func, IntegerField, JSONField, OuterRef, Subquery, Value+from django.db.models.expressions import CombinedExpression+from django.db.models.functions import Cast, Coalesce+from django.utils import timezone++from clickhouse_driver import Client+from pydantic import BaseModel, Field++from posthog.hogql.escape_sql import escape_clickhouse_identifier++from posthog.clickhouse.client import sync_execute+from posthog.clickhouse.client.connection import NodeRole, Workload+from posthog.clickhouse.cluster import get_cluster+from posthog.clickhouse.query_tagging import Feature, Product, tags_context+from posthog.dataclasses import frozen+from posthog.models.event.new_events_schema import events_read_table, use_new_events_schema+from posthog.models.person.util import get_persons_mapped_by_distinct_id+from posthog.models.person_group_membership.sql import (+ DISTRIBUTED_PERSON_GROUP_MEMBERSHIP_CONFIG_TABLE,+ PERSON_GROUP_MEMBERSHIP_CONFIG_DICTIONARY,+ PERSON_GROUP_MEMBERSHIP_MAX_GROUP_TYPE_INDEX,+ PERSON_GROUP_MEMBERSHIP_TABLE,+)+from posthog.models.team import Team+from posthog.redis import get_client as get_redis_client++from products.customer_analytics.backend.logic.feature_gate import is_customer_analytics_active+from products.customer_analytics.backend.models import TeamCustomerAnalyticsConfig++PAGE_SIZE = 100+BATCH_SIZE = 500+WINDOW_SIZE = timedelta(days=1)+HISTORICAL_WINDOW_SIZE = timedelta(days=90)+QUERY_SETTINGS = {+ "max_execution_time": 120,+ "max_memory_usage": 1_000_000_000,+ "max_bytes_to_read": 50 * 1024**4,+ "max_result_bytes": 8_000_000,+ "result_overflow_mode": "throw",+ "max_threads": 30,+ "distributed_foreground_insert": 1,+}+++class MembershipReadiness(StrEnum):+ PENDING_CONFIG = "pending_config"+ BACKFILLING = "backfilling"+ CATCHING_UP = "catching_up"+ READY = "ready"+ FAILED = "failed"+ DISABLED = "disabled"+++class MembershipState(BaseModel):+ group_type_index: int = Field(default=0, ge=0, le=PERSON_GROUP_MEMBERSHIP_MAX_GROUP_TYPE_INDEX)+ version: int = Field(default=0, ge=0)+ status: MembershipReadiness = MembershipReadiness.DISABLED+ synced_at: datetime | None = None+ updated_at: datetime | None = None+ error_type: str = Field(default="", max_length=128)+++def get_membership_properties_update(updates: Mapping[str, object]) -> Func:+ membership = CombinedExpression(+ Coalesce(F("properties__membership"), Value({}, output_field=JSONField())),+ "||",+ Value(dict(updates), output_field=JSONField()),+ output_field=JSONField(),+ )+ return Func(F("properties"), Value(["membership"]), membership, function="jsonb_set", output_field=JSONField())+++@frozen+class MembershipTeam:+ team_id: int+ version: int = 0+ account_keys: tuple[str, ...] = ()+ dry_run: bool = False+ cursor: "MembershipCursor | None" = None+++@frozen+class MembershipSyncPage:+ teams: tuple[MembershipTeam, ...]+ next_team_id: int | None+ candidates: int+ enabled: int+ changed: int+++@frozen(frozen=False)+class MembershipCursor:+ team_id: int+ version: int+ group_type_index: int+ status: str = "backfilling"+ account_keys: tuple[str, ...] = ()+ dry_run: bool = False+ historical_start: datetime | None = None+ historical_end: datetime | None = None+ next_window_start: datetime | None = None+ next_window_end: datetime | None = None+ current_group_key: str = ""+ catch_up_end: datetime | None = None+ last_distinct_id: str = ""+ last_group_key: str = ""+ pending_group_keys: tuple[str, ...] = ()+ discovery_window_end: datetime | None = None+ snapshot_end: datetime | None = None+ error_type: str = ""+ accounts_completed: int = 0+ windows_completed: int = 0+ rows_written: int = 0+ rows_found: int = 0+ started_at: datetime | None = None+ last_batch_duration_seconds: float = 0+ lag_seconds: float = 0+++@frozen+class MembershipStep:+ status: str+ rows: int = 0+ lag_seconds: float | None = None+ cursor: MembershipCursor | None = None+++@frozen+class MembershipHistoryBounds:+ start: datetime | None+ end: datetime+++@frozen+class MembershipPlan:+ team_id: int+ group_type_index: int+ historical_start: datetime | None+ historical_end: datetime+ estimated_events: int+ windows: int+++@frozen+class MembershipRow:+ distinct_id: str+ group_key: str+ first_seen: datetime+ last_seen: datetime+++def get_membership_group_type_indices(teams: Sequence[Team]) -> dict[int, int | None]:+ if not teams:+ return {}+ indices = dict(+ TeamCustomerAnalyticsConfig.objects.filter(+ team_id__in={team.parent_team_id or team.id for team in teams}+ ).values_list("team_id", "account_group_type_index")+ )+ active_organizations: dict[str, bool] = {}+ result: dict[int, int | None] = {}+ for team in teams:+ index = indices.get(team.parent_team_id or team.id)+ if index is None or not 0 <= index <= PERSON_GROUP_MEMBERSHIP_MAX_GROUP_TYPE_INDEX:+ result[team.id] = None+ continue+ organization_id = str(team.organization_id)+ if organization_id not in active_organizations:+ active_organizations[organization_id] = is_customer_analytics_active(team, raise_on_unknown=True)+ result[team.id] = index if active_organizations[organization_id] else None+ return result+++def get_membership_group_type_index(team: Team) -> int | None:+ return get_membership_group_type_indices([team])[team.id]+++def get_membership_readiness(team: Team) -> str:+ if not is_customer_analytics_active(team):+ return MembershipReadiness.DISABLED+ index = get_membership_group_type_index(team)+ if index is None:+ return MembershipReadiness.PENDING_CONFIG+ config = TeamCustomerAnalyticsConfig.objects.filter(team_id=team.id).only("properties").first()+ state = MembershipState.model_validate(config.properties.get("membership", {})) if config else None+ if state is None or not state.version or state.group_type_index != index:+ return MembershipReadiness.PENDING_CONFIG+ return state.status+++def get_window_end(start: datetime, end: datetime) -> datetime:+ return min(start + WINDOW_SIZE, end)+++def get_historical_window_start(start: datetime, end: datetime) -> datetime:+ return max(start, end - HISTORICAL_WINDOW_SIZE)+++def get_membership_group_column(group_type_index: int, *, native_json: bool) -> str:+ if group_type_index not in range(PERSON_GROUP_MEMBERSHIP_MAX_GROUP_TYPE_INDEX + 1):+ raise ValueError("Invalid membership group type index")+ column = escape_clickhouse_identifier(f"$group_{group_type_index}")+ return f"properties.{column}" if native_json else column+++def get_membership_batch_sql(*, group_type_index: int, catching_up: bool, native_json: bool = False) -> str:+ group_column = get_membership_group_column(group_type_index, native_json=native_json)+ if not catching_up:+ return f"""+ SELECT distinct_id, %(group_key)s AS group_key, min(timestamp), max(timestamp)+ FROM {events_read_table(native_json)}+ PREWHERE team_id = %(team_id)s+ AND timestamp >= %(start)s AND timestamp < %(end)s+ AND {group_column} = %(group_key)s+ AND person_mode != 'propertyless'+ WHERE distinct_id > %(last_distinct_id)s+ AND (inserted_at IS NULL OR inserted_at < %(snapshot_end)s)+ GROUP BY distinct_id ORDER BY distinct_id LIMIT {BATCH_SIZE}+ """+ return f"""+ SELECT distinct_id, {group_column} AS group_key,+ min(timestamp), max(timestamp)+ FROM {events_read_table(native_json)}+ WHERE team_id = %(team_id)s AND person_mode != 'propertyless'+ AND inserted_at >= %(start)s AND inserted_at < %(end)s+ AND notEmpty(group_key)+ AND (empty(%(account_keys)s) OR group_key IN %(account_keys)s)+ AND (distinct_id, group_key) > (%(last_distinct_id)s, %(last_group_key)s)+ GROUP BY team_id, distinct_id, group_key+ ORDER BY distinct_id, group_key+ LIMIT {BATCH_SIZE}+ """+++class MembershipBackfillService:+ def __init__(self, *, deadline: float | None = None, should_stop: Callable[[], bool] | None = None) -> None:+ self.deadline = deadline+ self.should_stop = should_stop++ def check_attempt(self) -> None:+ if (self.deadline is not None and monotonic() >= self.deadline) or (self.should_stop and self.should_stop()):+ raise RuntimeError("Membership backfill attempt expired")++ def query(+ self, sql: str, params: Mapping[str, object] | list[tuple[object, ...]], *, team_id: int | None = None+ ) -> list[tuple[object, ...]]:+ self.check_attempt()+ query_settings = dict(QUERY_SETTINGS)+ if self.deadline is not None:+ query_settings["max_execution_time"] = min(120, max(1, int(self.deadline - monotonic())))+ with tags_context(product=Product.CUSTOMER_ANALYTICS, feature=Feature.QUERY):+ rows = cast(+ list[tuple[object, ...]],+ sync_execute(sql, params, settings=query_settings, team_id=team_id, workload=Workload.OFFLINE) or [],+ )+ self.check_attempt()+ return rows++ def get_history_bounds(+ self, team_id: int, index: int, end: datetime, account_keys: tuple[str, ...] = ()+ ) -> MembershipHistoryBounds:+ native_json = use_new_events_schema(team_id)+ table = events_read_table(native_json)+ group_column = get_membership_group_column(index, native_json=native_json)+ days = []+ for direction in ("ASC", "DESC"):+ rows = self.query(+ f"""SELECT toDate(timestamp) AS event_date FROM {table}+ WHERE team_id = %(team_id)s+ AND (inserted_at IS NULL OR inserted_at < %(snapshot_end)s)+ AND person_mode != 'propertyless'+ AND notEmpty({group_column})+ AND (empty(%(account_keys)s) OR {group_column} IN %(account_keys)s)+ ORDER BY event_date {direction} LIMIT 1+ SETTINGS optimize_read_in_order = 1""",+ {"team_id": team_id, "snapshot_end": end, "account_keys": list(account_keys)},+ team_id=team_id,+ )+ if not rows:+ return MembershipHistoryBounds(start=None, end=end)+ days.append(datetime.combine(cast(date, rows[0][0]), datetime.min.time(), tzinfo=UTC))+ return MembershipHistoryBounds(start=days[0], end=max(end, days[1] + timedelta(days=1)))++ def plan(self, team: Team, index: int, end: datetime, account_keys: tuple[str, ...] = ()) -> MembershipPlan:+ bounds = self.get_history_bounds(team.id, index, end, account_keys)+ native_json = use_new_events_schema(team.id)+ group_column = get_membership_group_column(index, native_json=native_json)+ start: datetime | None = None+ history_end = end+ estimated_events = 0+ window_end = bounds.end+ while bounds.start is not None and window_end > bounds.start:+ window_start = get_historical_window_start(bounds.start, window_end)+ rows = self.query(+ f"""SELECT minOrNull(timestamp), maxOrNull(timestamp), count() FROM {events_read_table(native_json)}+ PREWHERE team_id = %(team_id)s+ AND timestamp >= %(start)s AND timestamp < %(end)s+ WHERE (inserted_at IS NULL OR inserted_at < %(snapshot_end)s)+ AND person_mode != 'propertyless'+ AND notEmpty({group_column})+ AND (empty(%(account_keys)s) OR {group_column} IN %(account_keys)s)""",+ {+ "team_id": team.id,+ "start": window_start,+ "end": window_end,+ "snapshot_end": end,+ "account_keys": list(account_keys),+ },+ team_id=team.id,+ )+ first_timestamp = cast(datetime | None, rows[0][0])+ if first_timestamp is not None:+ start = first_timestamp.replace(tzinfo=UTC)+ last_timestamp = cast(datetime, rows[0][1]).replace(tzinfo=UTC, microsecond=0)+ history_end = max(history_end, last_timestamp + timedelta(seconds=1))+ estimated_events += cast(int, rows[0][2])+ window_end = window_start+ return MembershipPlan(+ team_id=team.id,+ group_type_index=index,+ historical_start=start,+ historical_end=history_end,+ estimated_events=estimated_events,+ windows=max(0, ceil((history_end - start) / HISTORICAL_WINDOW_SIZE)) if start else 0,+ )++ def sync_page(self, after_team_id: int, *, dry_run: bool) -> MembershipSyncPage:+ self.check_attempt()+ configured = TeamCustomerAnalyticsConfig.objects.filter(+ team_id=Coalesce(OuterRef("parent_team_id"), OuterRef("id")),+ account_group_type_index__isnull=False,+ )+ config_ids = list(+ Team.objects.filter(id__gt=after_team_id)+ .filter(Exists(configured))+ .order_by("id")+ .values_list("id", flat=True)[:PAGE_SIZE]+ )+ registry_rows = self.query(+ f"""SELECT team_id, argMax(tuple(group_type_index, enabled, version), version) AS config+ FROM {DISTRIBUTED_PERSON_GROUP_MEMBERSHIP_CONFIG_TABLE}+ WHERE team_id > %(after)s GROUP BY team_id+ HAVING config.2 = 1 ORDER BY team_id LIMIT {PAGE_SIZE}""",+ {"after": after_team_id},+ )+ team_ids = sorted(set(config_ids) | {cast(int, row[0]) for row in registry_rows})[:PAGE_SIZE]+ current_rows = (+ self.query(+ f"""SELECT team_id, argMax(tuple(group_type_index, enabled, version), version)+ FROM {DISTRIBUTED_PERSON_GROUP_MEMBERSHIP_CONFIG_TABLE}+ WHERE team_id IN %(team_ids)s GROUP BY team_id""",+ {"team_ids": team_ids},+ )+ if team_ids+ else []+ )+ current = {cast(int, row[0]): cast(tuple[int, int, int], row[1]) for row in current_rows}+ teams = {team.id: team for team in Team.objects.filter(id__in=team_ids)}+ indices = get_membership_group_type_indices(list(teams.values()))+ pending = []+ enabled = changed = 0+ for team_id in team_ids:+ self.check_attempt()+ team = teams.get(team_id)+ index = indices.get(team_id)+ enabled += int(index is not None)+ existing = current.get(team_id)+ changed += int(+ (index is not None, index)+ != (bool(existing and existing[1]), existing[0] if existing and existing[1] else None)+ )+ if dry_run:+ continue+ if team is None:+ if existing:+ self.write_registry(team_id, existing[0], False, max(time_ns() // 1000, existing[2] + 1))+ continue+ state = self.sync_team(team, index, existing)+ if index is not None and state.status != MembershipReadiness.READY:+ pending.append(MembershipTeam(team_id=team_id, version=state.version))+ return MembershipSyncPage(+ teams=tuple(pending),+ next_team_id=team_ids[-1] if len(team_ids) == PAGE_SIZE else None,+ candidates=len(team_ids),+ enabled=enabled,+ changed=changed,+ )++ def write_registry(self, team_id: int, index: int, enabled: bool, version: int) -> None:+ self.query(+ f"INSERT INTO {DISTRIBUTED_PERSON_GROUP_MEMBERSHIP_CONFIG_TABLE} (team_id, group_type_index, enabled, version) VALUES",+ [(team_id, index, int(enabled), version)],+ team_id=team_id,+ )++ def configure_team(self, team: Team, index: int | None, existing: tuple[int, int, int] | None) -> MembershipState:+ self.check_attempt()+ with transaction.atomic():+ config, _ = TeamCustomerAnalyticsConfig.objects.select_for_update().get_or_create(team_id=team.id)+ self.check_attempt()+ state = MembershipState.model_validate(config.properties.get("membership", {}))+ reset = index is not None and (+ state.version == 0+ or state.group_type_index != index+ or state.status == MembershipReadiness.DISABLED+ or existing is None+ or existing[0] != index+ or not existing[1]+ )+ disable = index is None and (state.version == 0 or state.status != MembershipReadiness.DISABLED)+ if reset or disable:+ state.version = max(time_ns() // 1000, state.version + 1, existing[2] + 1 if existing else 1)+ state.group_type_index = index if index is not None else state.group_type_index+ state.status = MembershipReadiness.BACKFILLING if reset else MembershipReadiness.DISABLED+ state.error_type = ""+ state.synced_at = None+ self.check_attempt()+ state.updated_at = timezone.now()+ TeamCustomerAnalyticsConfig.objects.filter(team_id=team.id).update(+ properties=get_membership_properties_update(state.model_dump(mode="json"))+ )+ return state++ def sync_registry(+ self, team: Team, state: MembershipState, index: int | None, existing: tuple[int, int, int] | None+ ) -> MembershipState:+ if (+ state.synced_at is None+ or existing is None+ or existing != (state.group_type_index, int(index is not None), state.version)+ ):+ self.write_registry(team.id, state.group_type_index, index is not None, state.version)+ self.check_attempt()+ synced_at = timezone.now()+ updated = TeamCustomerAnalyticsConfig.objects.filter(+ team_id=team.id, properties__membership__version=state.version+ ).update(properties=get_membership_properties_update({"synced_at": synced_at.isoformat()}))+ if updated:+ state.synced_at = synced_at+ return state++ def sync_team(self, team: Team, index: int | None, existing: tuple[int, int, int] | None) -> MembershipState:+ return self.sync_registry(team, self.configure_team(team, index, existing), index, existing)++ def prepare_backfill(self, input: MembershipTeam) -> MembershipTeam:+ self.check_attempt()+ if input.version or input.dry_run:+ return input+ team = Team.objects.get(id=input.team_id)+ index = get_membership_group_type_index(team)+ if index is None:+ return input+ rows = self.query(+ f"""SELECT argMax(tuple(group_type_index, enabled, version), version)+ FROM {DISTRIBUTED_PERSON_GROUP_MEMBERSHIP_CONFIG_TABLE} WHERE team_id = %(team_id)s""",+ {"team_id": team.id},+ team_id=team.id,+ )+ existing = cast(tuple[int, int, int], rows[0][0]) if rows else None+ state = self.configure_team(team, index, existing)+ return replace(input, version=state.version)++ def get_active_state(self, team_id: int, version: int) -> MembershipState | None:+ self.check_attempt()+ configured_index = TeamCustomerAnalyticsConfig.objects.filter(+ team_id=Coalesce(OuterRef("team__parent_team_id"), OuterRef("team_id"))+ ).values("account_group_type_index")[:1]+ config = (+ TeamCustomerAnalyticsConfig.objects.filter(team_id=team_id, properties__membership__version=version)+ .select_related("team")+ .annotate(membership_group_type_index=Cast("properties__membership__group_type_index", IntegerField()))+ .filter(+ membership_group_type_index=Subquery(configured_index),+ membership_group_type_index__lte=PERSON_GROUP_MEMBERSHIP_MAX_GROUP_TYPE_INDEX,+ )+ .exclude(properties__membership__status=MembershipReadiness.DISABLED)+ .first()+ )+ if config is None:+ return None+ state = MembershipState.model_validate(config.properties.get("membership", {}))+ if not state.version or state.status == MembershipReadiness.DISABLED:+ return None+ if not is_customer_analytics_active(config.team, raise_on_unknown=True):+ return None+ return state++ @staticmethod+ def parse_membership_rows(rows: Sequence[tuple[object, ...]]) -> list[MembershipRow]:+ return [+ MembershipRow(+ distinct_id=cast(str, row[0]),+ group_key=cast(str, row[1]),+ first_seen=cast(datetime, row[2]),+ last_seen=cast(datetime, row[3]),+ )+ for row in rows+ ]++ def discover_accounts(self, state: MembershipCursor) -> bool:+ assert state.historical_start is not None and state.historical_end is not None+ if state.discovery_window_end is None:+ state.discovery_window_end = state.historical_end+ if state.discovery_window_end == state.historical_start:+ return False+ start = get_historical_window_start(state.historical_start, state.discovery_window_end)+ native_json = use_new_events_schema(state.team_id)+ group_column = get_membership_group_column(state.group_type_index, native_json=native_json)+ rows = self.query(+ f"""SELECT {group_column} AS group_key FROM {events_read_table(native_json)}+ PREWHERE team_id = %(team_id)s+ AND timestamp >= %(start)s AND timestamp < %(end)s+ WHERE group_key > %(after)s AND person_mode != 'propertyless'+ AND (inserted_at IS NULL OR inserted_at < %(snapshot_end)s)+ GROUP BY group_key ORDER BY group_key LIMIT {PAGE_SIZE}""",+ {+ "team_id": state.team_id,+ "start": start,+ "end": state.discovery_window_end,+ "snapshot_end": state.snapshot_end or state.historical_end,+ "after": state.last_group_key,+ },+ team_id=state.team_id,+ )+ state.pending_group_keys = tuple(+ sorted(set(state.pending_group_keys) | {cast(str, row[0]) for row in rows})[:PAGE_SIZE]+ )+ state.discovery_window_end = start+ self.save_progress(state)+ return start > state.historical_start++ def process_history(self, state: MembershipCursor) -> MembershipStep:+ assert state.historical_end is not None+ snapshot_end = state.snapshot_end or state.historical_end+ if state.historical_start is not None and not state.current_group_key:+ if state.account_keys:+ accounts = tuple(key for key in state.account_keys if key > state.last_group_key)[:1]+ else:+ if self.discover_accounts(state):+ return MembershipStep(status=MembershipReadiness.BACKFILLING)+ accounts = state.pending_group_keys[:1]+ state.pending_group_keys = state.pending_group_keys[1:]+ if accounts:+ state.current_group_key = accounts[0]+ state.next_window_end = state.historical_end+ self.save_progress(state)+ if not state.current_group_key:+ if state.dry_run:+ state.status = "completed"+ return MembershipStep(status=state.status)+ state.status = MembershipReadiness.CATCHING_UP+ self.save_progress(state)+ if not self.dictionary_visible(state.team_id, state.group_type_index):+ return MembershipStep(+ status="waiting_for_dictionary",+ lag_seconds=max(0, (timezone.now() - snapshot_end).total_seconds()),+ )+ state.catch_up_end = timezone.now()+ state.next_window_start = snapshot_end - timedelta(minutes=5)+ state.last_distinct_id = state.last_group_key = ""+ self.save_progress(state)+ return MembershipStep(status=state.status)+ assert state.historical_start is not None and state.next_window_end is not None+ start = get_historical_window_start(state.historical_start, state.next_window_end)+ rows = self.parse_membership_rows(+ self.query(+ get_membership_batch_sql(+ group_type_index=state.group_type_index,+ catching_up=False,+ native_json=use_new_events_schema(state.team_id),+ ),+ {+ "team_id": state.team_id,+ "group_key": state.current_group_key,+ "start": start,+ "end": state.next_window_end,+ "snapshot_end": snapshot_end,+ "last_distinct_id": state.last_distinct_id,+ },+ team_id=state.team_id,+ )+ )+ state.rows_found += len(rows)+ written = self.insert_memberships(state, rows)+ if len(rows) == BATCH_SIZE:+ state.last_distinct_id = rows[-1].distinct_id+ else:+ state.last_distinct_id = ""+ state.next_window_start = state.next_window_end = start+ state.windows_completed += 1+ if start == state.historical_start:+ state.accounts_completed += 1+ state.last_group_key = state.current_group_key+ state.current_group_key = ""+ state.next_window_end = None+ if not state.pending_group_keys:+ state.discovery_window_end = None+ state.status = MembershipReadiness.BACKFILLING+ state.error_type = ""+ self.save_progress(state)+ return MembershipStep(status=state.status, rows=written)++ def create_cursor(self, input: MembershipTeam) -> MembershipCursor | None:+ team = Team.objects.filter(id=input.team_id).first()+ index = get_membership_group_type_index(team) if team else None+ if team is None or index is None:+ return None+ version = input.version+ if not input.dry_run:+ prepared = self.prepare_backfill(input)+ config = self.get_active_state(team.id, prepared.version)+ if config is None:+ return None+ self.sync_registry(team, config, index, None)+ version = config.version+ end = timezone.now()+ bounds = self.get_history_bounds(team.id, index, end, input.account_keys)+ return MembershipCursor(+ team_id=team.id,+ version=version,+ group_type_index=index,+ account_keys=tuple(sorted(set(input.account_keys))),+ dry_run=input.dry_run,+ historical_start=bounds.start,+ historical_end=bounds.end,+ snapshot_end=end,+ next_window_start=bounds.start or end,+ started_at=end,+ )++ def process_batch(self, input: MembershipTeam) -> MembershipStep:+ self.check_attempt()+ state = replace(input.cursor) if input.cursor else self.create_cursor(input)+ if state is None:+ return MembershipStep(status=MembershipReadiness.DISABLED)+ if not state.dry_run and self.get_active_state(state.team_id, state.version) is None:+ return MembershipStep(status=MembershipReadiness.DISABLED, cursor=state)+ result = self.process_cursor(state)+ state.rows_written += result.rows+ return replace(result, cursor=state)++ @classmethod+ def process_activity_batch(+ cls, input: MembershipTeam, *, deadline: float, should_stop: Callable[[], bool]+ ) -> MembershipStep:+ lease = get_redis_client(socket_timeout=2, socket_connect_timeout=2).lock(+ f"customer_analytics:membership_backfill:{input.team_id}", timeout=330, blocking_timeout=120+ )+ if not lease.acquire():+ raise RuntimeError("Another membership backfill attempt is active")+ try:+ service = cls(deadline=deadline, should_stop=lambda: should_stop() or not lease.owned())+ result = service.process_batch(input)+ service.check_attempt()+ return result+ finally:+ if lease.owned():+ lease.release()++ def process_cursor(self, state: MembershipCursor) -> MembershipStep:+ team_id = state.team_id+ assert state.historical_end is not None and state.next_window_start is not None+ if state.catch_up_end is None:+ return self.process_history(state)+ end = state.catch_up_end+ if state.next_window_start >= end:+ if not self.dictionary_visible(team_id, state.group_type_index):+ return MembershipStep(+ status="waiting_for_dictionary", lag_seconds=(timezone.now() - end).total_seconds()+ )+ state.status = MembershipReadiness.READY+ state.error_type = ""+ self.save_progress(state)+ return MembershipStep(status=state.status)+ window_end = get_window_end(state.next_window_start, end)+ rows = self.parse_membership_rows(+ self.query(+ get_membership_batch_sql(+ group_type_index=state.group_type_index,+ catching_up=True,+ native_json=use_new_events_schema(team_id),+ ),+ {+ "team_id": team_id,+ "start": state.next_window_start,+ "end": window_end,+ "last_distinct_id": state.last_distinct_id,+ "last_group_key": state.last_group_key,+ "account_keys": list(state.account_keys),+ },+ team_id=team_id,+ )+ )+ state.rows_found += len(rows)+ written = self.insert_memberships(state, rows)+ if len(rows) == BATCH_SIZE:+ state.last_distinct_id, state.last_group_key = rows[-1].distinct_id, rows[-1].group_key+ else:+ state.next_window_start = window_end+ state.windows_completed += 1+ state.last_distinct_id = state.last_group_key = ""+ state.status = MembershipReadiness.CATCHING_UP+ state.error_type = ""+ self.save_progress(state)+ return MembershipStep(+ status=state.status, rows=written, lag_seconds=max(0, (timezone.now() - window_end).total_seconds())+ )++ def insert_memberships(self, state: MembershipCursor, rows: Sequence[MembershipRow]) -> int:+ if not rows or state.dry_run:+ return 0+ persons = get_persons_mapped_by_distinct_id(state.team_id, list({row.distinct_id for row in rows}))+ admitted = [row for row in rows if row.distinct_id in persons and persons[row.distinct_id].is_identified]+ if not admitted or self.get_active_state(state.team_id, state.version) is None:+ return 0+ self.query(+ f"INSERT INTO {PERSON_GROUP_MEMBERSHIP_TABLE} (team_id, group_type_index, group_key, distinct_id, first_seen, last_seen) VALUES",+ [+ (state.team_id, state.group_type_index, row.group_key, row.distinct_id, row.first_seen, row.last_seen)+ for row in admitted+ ],+ team_id=state.team_id,+ )+ return len(admitted)++ def save_progress(self, state: MembershipCursor) -> None:+ self.check_attempt()+ if state.dry_run or state.account_keys:+ return+ updated = (+ TeamCustomerAnalyticsConfig.objects.filter(+ team_id=state.team_id, properties__membership__version=state.version+ )+ .exclude(properties__membership__status=MembershipReadiness.DISABLED)+ .update(+ properties=get_membership_properties_update(+ {+ "status": MembershipReadiness(state.status),+ "error_type": state.error_type,+ "updated_at": timezone.now().isoformat(),+ }+ )+ )+ )+ if not updated:+ raise RuntimeError("Membership configuration changed during this batch")++ @staticmethod+ def fail(team_id: int, version: int, error_type: str) -> None:+ TeamCustomerAnalyticsConfig.objects.filter(team_id=team_id, properties__membership__version=version).exclude(+ properties__membership__status__in=[MembershipReadiness.READY, MembershipReadiness.DISABLED]+ ).update(+ properties=get_membership_properties_update(+ {+ "status": MembershipReadiness.FAILED,+ "error_type": error_type[:128],+ "updated_at": timezone.now().isoformat(),+ }+ )+ )++ def dictionary_visible(self, team_id: int, index: int) -> bool:+ self.check_attempt()+ sql = """SELECT dictGet(%(dictionary)s, 'enabled', tuple(toInt64(%(team_id)s))),+ dictGet(%(dictionary)s, 'group_type_index', tuple(toInt64(%(team_id)s)))"""+ params = {+ "dictionary": f"{settings.CLICKHOUSE_DATABASE}.{PERSON_GROUP_MEMBERSHIP_CONFIG_DICTIONARY}",+ "team_id": team_id,+ }+ if not settings.CLOUD_DEPLOYMENT and not settings.MULTINODE_CLICKHOUSE:+ return self.query(sql, params, team_id=team_id) == [(1, index)]++ def check(client: Client) -> bool:+ rows = client.execute(+ sql,+ params,+ settings={"max_execution_time": 15},+ )+ return rows == [(1, index)]++ results = (+ get_cluster(cluster=settings.CLICKHOUSE_MIGRATIONS_CLUSTER, data_cluster=settings.CLICKHOUSE_CLUSTER)+ .map_hosts_by_roles(check, [NodeRole.INGESTION_SMALL], concurrency=2, require_hosts=True)+ .result()+ )+ self.check_attempt()+ return bool(results) and all(results.values())diff --git a/products/customer_analytics/backend/logic/usage_spike_notifications.py b/products/customer_analytics/backend/logic/usage_spike_notifications.pyindex 065e46e89825..34e8f1a16f28 100644--- a/products/customer_analytics/backend/logic/usage_spike_notifications.py+++ b/products/customer_analytics/backend/logic/usage_spike_notifications.py@@ -1,12 +1,11 @@from typing import Anyimport structlog-import posthoganalyticsfrom posthog.exceptions_capture import capture_exceptionfrom products.customer_analytics.backend.account_urls import build_account_deeplink-from products.customer_analytics.backend.constants import CUSTOMER_ANALYTICS_CSP_FLAG+from products.customer_analytics.backend.logic.feature_gate import is_customer_analytics_activefrom products.customer_analytics.backend.models import Account, AccountRelationshipfrom products.notifications.backend.facade.api import (NotificationData,@@ -113,16 +112,7 @@ def _find_account(def _is_csp_enabled(account: Account) -> bool:- organization_id = str(account.team.organization_id)- return bool(- posthoganalytics.feature_enabled(- CUSTOMER_ANALYTICS_CSP_FLAG,- organization_id,- groups={"organization": organization_id},- only_evaluate_locally=False,- send_feature_flag_events=False,- )- )+ return is_customer_analytics_active(account.team)def _get_manager_user_ids(account: Account) -> list[int]:diff --git a/products/customer_analytics/backend/management/commands/plan_membership_backfill.py b/products/customer_analytics/backend/management/commands/plan_membership_backfill.pynew file mode 100644index 000000000000..6b98be2b4b46--- /dev/null+++ b/products/customer_analytics/backend/management/commands/plan_membership_backfill.py@@ -0,0 +1,31 @@+import json+from dataclasses import asdict+from time import monotonic+from typing import cast++from django.core.management.base import BaseCommand, CommandError, CommandParser+from django.utils import timezone++from posthog.models.team import Team++from products.customer_analytics.backend.logic.membership_backfill import (+ MembershipBackfillService,+ get_membership_group_type_index,+)+++class Command(BaseCommand):+ help = "Estimate membership backfill events and date windows without writes"++ def add_arguments(self, parser: CommandParser) -> None:+ parser.add_argument("--team-id", type=int, required=True)++ def handle(self, *args: object, **options: object) -> None:+ team = Team.objects.filter(id=cast(int, options["team_id"])).first()+ if team is None:+ raise CommandError("Project does not exist")+ index = get_membership_group_type_index(team)+ if index is None:+ raise CommandError("Customer analytics must be active with a valid account group type")+ plan = MembershipBackfillService(deadline=monotonic() + 300).plan(team, index, timezone.now())+ self.stdout.write(json.dumps(asdict(plan), default=str))diff --git a/products/customer_analytics/backend/migrations/0063_teamcustomeranalyticsconfig_properties.py b/products/customer_analytics/backend/migrations/0063_teamcustomeranalyticsconfig_properties.pynew file mode 100644index 000000000000..489149f9a197--- /dev/null+++ b/products/customer_analytics/backend/migrations/0063_teamcustomeranalyticsconfig_properties.py@@ -0,0 +1,15 @@+from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ ("customer_analytics", "0062_custompropertyvalue_active_team_definition_index"),+ ]++ operations = [+ migrations.AddField(+ model_name="teamcustomeranalyticsconfig",+ name="properties",+ field=models.JSONField(db_default={}, default=dict),+ ),+ ]diff --git a/products/customer_analytics/backend/migrations/max_migration.txt b/products/customer_analytics/backend/migrations/max_migration.txtindex 7448e430a1ce..75d47ef7b9c2 100644--- a/products/customer_analytics/backend/migrations/max_migration.txt+++ b/products/customer_analytics/backend/migrations/max_migration.txt@@ -1 +1 @@-0062_custompropertyvalue_active_team_definition_index+0063_teamcustomeranalyticsconfig_propertiesdiff --git a/products/customer_analytics/backend/models/team_customer_analytics_config.py b/products/customer_analytics/backend/models/team_customer_analytics_config.pyindex 67afbfbf9e13..d41405add2f1 100644--- a/products/customer_analytics/backend/models/team_customer_analytics_config.py+++ b/products/customer_analytics/backend/models/team_customer_analytics_config.py@@ -33,6 +33,7 @@ class TeamCustomerAnalyticsConfig(models.Model):models.JSONField(default=default_account_track_rules), "project", "admin")account_track_rules_enabled_at = models.DateTimeField(null=True, blank=True)+ properties = models.JSONField(default=dict, db_default={})def to_cache_key_dict(self) -> dict:return {diff --git a/products/customer_analytics/backend/test/test_membership_backfill.py b/products/customer_analytics/backend/test/test_membership_backfill.pynew file mode 100644index 000000000000..0c4529c4a71a--- /dev/null+++ b/products/customer_analytics/backend/test/test_membership_backfill.py@@ -0,0 +1,972 @@+from collections.abc import Mapping+from dataclasses import fields+from datetime import UTC, datetime, timedelta+from io import StringIO+from types import SimpleNamespace+from typing import Literal, cast+from uuid import uuid4++import pytest+from posthog.test.base import BaseTest+from unittest.mock import patch++from django.core.management import call_command+from django.test import override_settings+from django.utils import timezone++from parameterized import parameterized++from posthog.api.team.team_config import TeamCustomerAnalyticsConfigSerializer+from posthog.models import Organization, Team+from posthog.models.event.util import create_event, events_only_in_active_schema+from posthog.models.group.util import raw_create_group_ch+from posthog.redis import get_client as get_redis_client+from posthog.test.persons import create_person++from products.customer_analytics.backend.logic.feature_gate import is_customer_analytics_active+from products.customer_analytics.backend.logic.membership_backfill import (+ BATCH_SIZE,+ MembershipBackfillService,+ MembershipCursor,+ MembershipReadiness,+ MembershipRow,+ MembershipState,+ MembershipStep,+ MembershipTeam,+ get_historical_window_start,+ get_membership_batch_sql,+ get_membership_group_type_index,+ get_membership_readiness,+ get_window_end,+)+from products.customer_analytics.backend.models import TeamCustomerAnalyticsConfig++MODULE = "products.customer_analytics.backend.logic.membership_backfill"+START = datetime(2026, 1, 1, tzinfo=UTC)+END = START + timedelta(days=2)++[email protected]("duration", [timedelta(0), timedelta(hours=3), timedelta(days=1), timedelta(days=40)])+def test_date_windows_have_no_gap_or_overlap(duration: timedelta) -> None:+ cursor = START+ ends = []+ while cursor < START + duration:+ end = get_window_end(cursor, START + duration)+ assert timedelta(0) < end - cursor <= timedelta(days=1)+ ends.append(end)+ cursor = end+ assert cursor == START + duration+ assert ends == sorted(set(ends))++[email protected]("days", [0, 1, 90, 91, 270])+def test_historical_windows_cover_history_backwards(days: int) -> None:+ cursor = START + timedelta(days=days)+ while cursor > START:+ start = get_historical_window_start(START, cursor)+ assert timedelta(0) < cursor - start <= timedelta(days=90)+ cursor = start+ assert cursor == START++[email protected]("catching_up", [False, True])[email protected]("native_json", [False, True])[email protected]("group_type_index", [0, 1, 4])+def test_batch_projection_is_scoped_and_bounded(catching_up: bool, native_json: bool, group_type_index: int) -> None:+ sql = get_membership_batch_sql(group_type_index=group_type_index, catching_up=catching_up, native_json=native_json)+ group_column = f"properties.`$group_{group_type_index}`" if native_json else f"`$group_{group_type_index}`"+ assert "JSONExtract" not in sql+ assert "toJSONString" not in sql+ assert "team_id = %(team_id)s" in sql+ assert "person_mode != 'propertyless'" in sql+ if catching_up:+ assert f"{group_column} AS group_key" in sql+ assert "notEmpty(group_key)" in sql+ assert f"LIMIT {BATCH_SIZE}" in sql+ else:+ assert f"{group_column} = %(group_key)s" in sql+ assert "PREWHERE team_id = %(team_id)s" in sql+ assert f"LIMIT {BATCH_SIZE}" in sql+ assert "distinct_id > %(last_distinct_id)s" in sql+ assert "(inserted_at IS NULL OR inserted_at < %(snapshot_end)s)" in sql+ assert "min(timestamp), max(timestamp)" in sql+ assert ("inserted_at >=" in sql) == catching_up+ assert ("FROM events_json" in sql) == native_json++[email protected]("enabled", [False, True, None])+def test_shared_gate_uses_organization_identity(enabled: bool | None) -> None:+ team = Team(id=7, organization_id="00000000-0000-4000-8000-000000000001")+ with patch("posthoganalytics.feature_enabled", return_value=enabled) as gate:+ assert is_customer_analytics_active(team) is bool(enabled)+ gate.assert_called_once_with(+ "customer-analytics-csp",+ str(team.organization_id),+ groups={"organization": str(team.organization_id)},+ only_evaluate_locally=False,+ send_feature_flag_events=False,+ )+++def test_lost_activity_lease_stops_the_attempt() -> None:+ team_id = 17+ key = f"customer_analytics:membership_backfill:{team_id}"++ def process_batch(service: MembershipBackfillService, _input: MembershipTeam) -> MembershipStep:+ get_redis_client(socket_timeout=2, socket_connect_timeout=2).delete(key)+ service.check_attempt()+ return MembershipStep(status="backfilling")++ with (+ patch.object(MembershipBackfillService, "process_batch", autospec=True, side_effect=process_batch),+ pytest.raises(RuntimeError, match="attempt expired"),+ ):+ MembershipBackfillService.process_activity_batch(+ MembershipTeam(team_id=team_id), deadline=float("inf"), should_stop=lambda: False+ )+++class TestMembershipBackfill(BaseTest):+ CLASS_DATA_LEVEL_SETUP = False++ def setUp(self) -> None:+ super().setUp()+ self.config, _ = TeamCustomerAnalyticsConfig.objects.get_or_create(team=self.team)+ self.config.account_group_type_index = 0+ self.config.save(update_fields=["account_group_type_index"])+ self.gate = patch(f"{MODULE}.is_customer_analytics_active", return_value=True)+ self.gate.start()+ self.addCleanup(self.gate.stop)+ self.service = MembershipBackfillService()++ @staticmethod+ def get_state(team_id: int) -> MembershipState:+ config = TeamCustomerAnalyticsConfig.objects.get(team_id=team_id)+ return MembershipState.model_validate(config.properties["membership"])++ def create_state(+ self,+ *,+ historical_start: datetime = START,+ historical_end: datetime = END,+ next_window_start: datetime = START,+ next_window_end: datetime | None = END,+ current_group_key: str = "account-a",+ last_group_key: str = "",+ catch_up_end: datetime | None = None,+ account_keys: tuple[str, ...] = (),+ dry_run: bool = False,+ snapshot_end: datetime | None = None,+ ) -> MembershipCursor:+ self.config.refresh_from_db()+ self.config.properties["membership"] = MembershipState(+ group_type_index=0, version=1, status=MembershipReadiness.BACKFILLING+ ).model_dump(mode="json")+ self.config.save(update_fields=["properties"])+ return MembershipCursor(+ team_id=self.team.id,+ group_type_index=0,+ version=1,+ historical_start=historical_start,+ historical_end=historical_end,+ next_window_start=next_window_start,+ next_window_end=next_window_end,+ current_group_key=current_group_key,+ last_group_key=last_group_key,+ catch_up_end=catch_up_end,+ account_keys=account_keys,+ dry_run=dry_run,+ snapshot_end=snapshot_end,+ )++ def process_batch(self, state: MembershipCursor) -> MembershipStep:+ result = self.service.process_batch(+ MembershipTeam(+ team_id=state.team_id,+ version=state.version,+ account_keys=state.account_keys,+ dry_run=state.dry_run,+ cursor=state,+ )+ )+ if result.cursor:+ for field in fields(state):+ setattr(state, field.name, getattr(result.cursor, field.name))+ return result++ def test_eligibility_uses_shared_gate_and_valid_index(self) -> None:+ for index in (None, -1, 0, 4, 5):+ self.config.account_group_type_index = index+ self.config.save(update_fields=["account_group_type_index"])+ for active in (False, True):+ with patch(f"{MODULE}.is_customer_analytics_active", return_value=active):+ assert get_membership_group_type_index(self.team) == (index if active and index in (0, 4) else None)++ def test_registry_versions_change_only_for_configuration_changes(self) -> None:+ self.config.properties = {"other": {"enabled": True}, "membership": {"future_key": "keep"}}+ self.config.save(update_fields=["properties"])+ with patch.object(self.service, "write_registry"):+ first = self.service.sync_team(self.team, 0, None)+ second = self.service.sync_team(self.team, 0, (0, 1, first.version))+ changed = self.service.sync_team(self.team, 3, (0, 1, first.version))+ disabled = self.service.sync_team(self.team, None, (3, 1, changed.version))+ assert first.version == second.version+ assert changed.version > first.version+ assert changed.group_type_index == 3+ assert disabled.version > changed.version+ assert disabled.status == MembershipReadiness.DISABLED+ self.config.refresh_from_db()+ assert self.config.properties["other"] == {"enabled": True}+ assert self.config.properties["membership"]["future_key"] == "keep"++ def test_registry_write_failure_can_resume(self) -> None:+ with patch.object(self.service, "write_registry", side_effect=RuntimeError("Unavailable")):+ with pytest.raises(RuntimeError):+ self.service.sync_team(self.team, 0, None)+ pending = self.get_state(self.team.id)+ with patch.object(self.service, "write_registry"):+ recovered = self.service.sync_team(self.team, 0, (0, 1, pending.version))+ assert recovered.version == pending.version+ assert self.get_state(self.team.id).synced_at is not None++ @parameterized.expand([("registry",), ("progress",), ("failure",)])+ def test_membership_updates_preserve_other_properties(self, operation: str) -> None:+ state = self.create_state()+ stale_config = TeamCustomerAnalyticsConfig.objects.get(team_id=self.team.id)+ synced_at = timezone.now().isoformat()++ def update_other_properties(*args: object) -> None:+ config = TeamCustomerAnalyticsConfig.objects.get(team_id=self.team.id)+ config.properties["other"] = {"enabled": True}+ config.properties["membership"]["future_key"] = "keep"+ config.properties["membership"]["synced_at"] = synced_at+ config.save(update_fields=["properties"])++ if operation == "registry":+ with patch.object(self.service, "write_registry", side_effect=update_other_properties):+ self.service.sync_team(self.team, 0, (0, 1, state.version))+ else:+ update_other_properties()+ if operation == "progress":+ state.status = MembershipReadiness.CATCHING_UP+ self.service.save_progress(state)+ else:+ self.service.fail(self.team.id, state.version, "Unavailable")++ serializer = TeamCustomerAnalyticsConfigSerializer(+ stale_config, data={"activity_event": {"event": "$pageview"}}, partial=True+ )+ serializer.is_valid(raise_exception=True)+ serializer.save()+ self.config.refresh_from_db()+ assert self.config.properties["other"] == {"enabled": True}+ assert self.config.properties["membership"]["future_key"] == "keep"+ membership = self.get_state(self.team.id)+ assert membership.synced_at is not None+ if operation != "registry":+ assert membership.synced_at.isoformat() == synced_at+ assert (+ membership.status+ == {+ "registry": MembershipReadiness.BACKFILLING,+ "progress": MembershipReadiness.CATCHING_UP,+ "failure": MembershipReadiness.FAILED,+ }[operation]+ )++ def test_coordinator_registry_and_disable(self) -> None:+ disabled_team = Team.objects.create(organization=Organization.objects.create(name="Disabled organization"))+ disabled_config, _ = TeamCustomerAnalyticsConfig.objects.get_or_create(team=disabled_team)+ disabled_config.account_group_type_index = 2+ disabled_config.save(update_fields=["account_group_type_index"])+ unconfigured_team = Team.objects.create(organization=self.organization)+ TeamCustomerAnalyticsConfig.objects.get_or_create(team=unconfigured_team)+ with patch(f"{MODULE}.is_customer_analytics_active", side_effect=lambda team, **_: team.id != disabled_team.id):+ dry_run = self.service.sync_page(0, dry_run=True)+ assert dry_run.teams == ()+ assert "membership" not in TeamCustomerAnalyticsConfig.objects.get(team_id=self.team.id).properties+ page = self.service.sync_page(0, dry_run=False)+ assert [team.team_id for team in page.teams] == [self.team.id]+ assert self.service.query(+ "SELECT team_id FROM distributed_person_group_membership_config GROUP BY team_id HAVING argMax(enabled, version) = 1",+ {},+ ) == [(self.team.id,)]+ self.service.query("SYSTEM RELOAD DICTIONARY person_group_membership_config_dict", {})+ assert self.service.dictionary_visible(self.team.id, 0)+ assert not self.service.dictionary_visible(disabled_team.id, 2)+ self.config.account_group_type_index = None+ self.config.save(update_fields=["account_group_type_index"])+ assert self.service.sync_page(0, dry_run=False).teams == ()+ assert (+ self.service.query(+ "SELECT team_id FROM distributed_person_group_membership_config GROUP BY team_id HAVING argMax(enabled, version) = 1",+ {},+ )+ == []+ )+ self.service.query("SYSTEM RELOAD DICTIONARY person_group_membership_config_dict", {})+ assert not self.service.dictionary_visible(self.team.id, 0)+ assert get_membership_readiness(self.team) == "pending_config"++ def test_coordinator_batches_configuration_reads_and_organization_flags(self) -> None:+ sibling = Team.objects.create(organization=self.organization)+ child = Team.objects.create(organization=self.organization, parent_team=self.team)+ other = Team.objects.create(organization=Organization.objects.create(name="Other organization"))+ for team in (sibling, other):+ config, _ = TeamCustomerAnalyticsConfig.objects.get_or_create(team=team)+ config.account_group_type_index = 4+ config.save(update_fields=["account_group_type_index"])+ with (+ patch.object(self.service, "query", return_value=[]),+ patch(f"{MODULE}.is_customer_analytics_active", return_value=True) as gate,+ self.assertNumQueries(3),+ ):+ page = self.service.sync_page(0, dry_run=True)+ assert page.candidates == page.enabled == 4+ assert gate.call_count == 2+ assert {call.args[0].organization_id for call in gate.call_args_list} == {+ self.organization.id,+ other.organization_id,+ }+ assert not TeamCustomerAnalyticsConfig.objects.filter(team_id=child.id).exists()+ with patch(f"{MODULE}.PAGE_SIZE", 1):+ page = self.service.sync_page(child.id - 1, dry_run=False)+ assert [team.team_id for team in page.teams] == [child.id]+ assert page.next_team_id == child.id+ assert self.get_state(child.id).group_type_index == 0++ @parameterized.expand([("cancelled",), ("expired",)])+ def test_coordinator_stops_before_registry_writes(self, reason: str) -> None:+ stopped = False+ service = MembershipBackfillService(should_stop=lambda: stopped)++ def stop_during_flag_check(*args: object, **kwargs: object) -> bool:+ nonlocal stopped+ if reason == "expired":+ service.deadline = 0+ else:+ stopped = True+ return True++ with (+ patch(f"{MODULE}.is_customer_analytics_active", side_effect=stop_during_flag_check),+ pytest.raises(RuntimeError, match="attempt expired"),+ ):+ service.sync_page(0, dry_run=False)+ self.config.refresh_from_db()+ assert "membership" not in self.config.properties+ assert self.service.query(+ "SELECT count() FROM distributed_person_group_membership_config WHERE team_id = %(team_id)s",+ {"team_id": self.team.id},+ ) == [(0,)]++ @parameterized.expand([("registry",), ("planning",)])+ def test_prepared_version_marks_initial_batch_failed(self, failure: str) -> None:+ prepared = self.service.prepare_backfill(MembershipTeam(team_id=self.team.id))+ assert prepared.version > 0+ assert prepared.version == self.get_state(self.team.id).version+ query = self.service.query++ def fail_query(+ sql: str, params: Mapping[str, object] | list[tuple[object, ...]], *, team_id: int | None = None+ ) -> list[tuple[object, ...]]:+ if (failure == "registry" and sql.startswith("INSERT")) or (+ failure == "planning" and "ORDER BY event_date" in sql+ ):+ raise RuntimeError("Unavailable")+ return query(sql, params, team_id=team_id)++ with patch.object(self.service, "query", side_effect=fail_query):+ with pytest.raises(RuntimeError, match="Unavailable"):+ self.service.process_batch(prepared)+ self.service.fail(self.team.id, prepared.version, "Unavailable")+ assert self.get_state(self.team.id).status == MembershipReadiness.FAILED+ assert self.get_state(self.team.id).version == prepared.version++ def test_unknown_flag_keeps_registry_and_cursor(self) -> None:+ state = self.create_state()+ self.service.write_registry(self.team.id, 0, True, state.version)+ with (+ patch(f"{MODULE}.is_customer_analytics_active", wraps=is_customer_analytics_active),+ patch("posthoganalytics.feature_enabled", return_value=None),+ ):+ with pytest.raises(RuntimeError, match="flag evaluation is unavailable"):+ self.service.sync_page(0, dry_run=False)+ with pytest.raises(RuntimeError, match="flag evaluation is unavailable"):+ self.process_batch(state)+ config = self.get_state(self.team.id)+ assert config.status == MembershipReadiness.BACKFILLING+ assert config.version == state.version+ assert state.next_window_end == END+ assert self.service.query(+ "SELECT argMax(enabled, version) FROM distributed_person_group_membership_config WHERE team_id = %(team_id)s",+ {"team_id": self.team.id},+ ) == [(1,)]++ def test_child_environment_has_its_own_configuration_and_memberships(self) -> None:+ parent = self.create_state()+ child = Team.objects.create(organization=self.organization, parent_team=self.team)+ with patch.object(self.service, "write_registry"):+ child_config = self.service.sync_team(child, 0, None)+ assert self.get_state(child.id).version == child_config.version+ assert get_membership_group_type_index(child) == 0+ with self.assertNumQueries(1):+ active = self.service.get_active_state(child.id, child_config.version)+ assert active is not None and active.version == child_config.version+ child_cursor = MembershipCursor(team_id=child.id, group_type_index=0, version=child_config.version)+ with patch(+ f"{MODULE}.get_persons_mapped_by_distinct_id",+ return_value={"identified": SimpleNamespace(is_identified=True)},+ ):+ assert (+ self.service.insert_memberships(+ child_cursor,+ [MembershipRow(distinct_id="identified", group_key="account-a", first_seen=START, last_seen=END)],+ )+ == 1+ )+ assert self.service.query(+ "SELECT team_id FROM person_group_membership WHERE team_id IN %(team_ids)s GROUP BY team_id",+ {"team_ids": [self.team.id, child.id]},+ ) == [(child.id,)]+ assert self.get_state(self.team.id).version == parent.version+ assert self.get_state(child.id).version == child_config.version++ @parameterized.expand([("cancelled",), ("expired",)])+ def test_stopped_attempt_cannot_write_or_advance(self, reason: str) -> None:+ state = self.create_state()+ stopped = False+ service = MembershipBackfillService(should_stop=lambda: stopped)++ def get_persons(*args: object, **kwargs: object) -> dict[str, SimpleNamespace]:+ nonlocal stopped+ if reason == "cancelled":+ stopped = True+ else:+ service.deadline = 0+ return {"identified": SimpleNamespace(is_identified=True)}++ with (+ patch.object(service, "query", return_value=[("identified", "account-a", START, END)]),+ patch(f"{MODULE}.get_persons_mapped_by_distinct_id", side_effect=get_persons),+ patch(f"{MODULE}.use_new_events_schema", return_value=False),+ pytest.raises(RuntimeError, match="attempt expired"),+ ):+ service.process_batch(MembershipTeam(team_id=self.team.id, version=state.version, cursor=state))+ assert state.next_window_end == END+ assert state.rows_written == 0+ assert self.service.query(+ "SELECT count() FROM person_group_membership WHERE team_id = %(team_id)s", {"team_id": self.team.id}+ ) == [(0,)]++ @parameterized.expand([("version",), ("disabled",)])+ def test_stale_cursor_cannot_replace_newer_configuration(self, change: str) -> None:+ state = self.create_state()+ self.config.refresh_from_db()+ if change == "version":+ self.config.properties["membership"]["version"] = 2+ else:+ self.config.properties["membership"]["status"] = MembershipReadiness.DISABLED+ expected = dict(self.config.properties["membership"])+ self.config.save(update_fields=["properties"])+ with pytest.raises(RuntimeError, match="configuration changed"):+ self.service.save_progress(state)+ assert self.process_batch(state).status == "disabled"+ self.service.fail(self.team.id, state.version, "Unavailable")+ self.config.refresh_from_db()+ assert self.config.properties["membership"] == expected++ def test_identity_failure_does_not_advance_the_cursor(self) -> None:+ state = self.create_state()+ with (+ patch.object(self.service, "query", return_value=[("person-a", "account-a", START, START)]),+ patch(f"{MODULE}.use_new_events_schema", return_value=False),+ patch(f"{MODULE}.get_persons_mapped_by_distinct_id", side_effect=RuntimeError("Unavailable")),+ ):+ with pytest.raises(RuntimeError, match="Unavailable"):+ self.process_batch(state)+ assert state.next_window_start == START+ assert state.last_distinct_id == ""++ def test_insert_admits_only_current_identified_profiles(self) -> None:+ state = self.create_state()+ rows = [+ MembershipRow(distinct_id=key, group_key="account-a", first_seen=START, last_seen=END)+ for key in ("identified", "anonymous", "deleted", "missing")+ ]+ persons = {"identified": SimpleNamespace(is_identified=True), "anonymous": SimpleNamespace(is_identified=False)}+ with (+ patch(f"{MODULE}.get_persons_mapped_by_distinct_id", return_value=persons),+ patch.object(self.service, "query") as query,+ ):+ assert self.service.insert_memberships(state, rows) == 1+ assert query.call_args.args[1] == [(self.team.id, 0, "account-a", "identified", START, END)]++ def test_batch_eligibility_uses_two_reads_and_rechecks_the_flag(self) -> None:+ state = self.create_state(account_keys=("account-a",))+ with (+ patch.object(self.service, "query", side_effect=[[("identified", "account-a", START, END)], []]),+ patch(f"{MODULE}.use_new_events_schema", return_value=False),+ patch(+ f"{MODULE}.get_persons_mapped_by_distinct_id",+ return_value={"identified": SimpleNamespace(is_identified=True)},+ ),+ patch(f"{MODULE}.is_customer_analytics_active", return_value=True) as gate,+ self.assertNumQueries(2),+ ):+ assert self.process_batch(state).rows == 1+ assert gate.call_count == 2++ @parameterized.expand([("index",), ("version",), ("flag",)])+ def test_batch_rechecks_eligibility_before_writing(self, change: str) -> None:+ state = self.create_state(account_keys=("account-a",))+ with patch(f"{MODULE}.is_customer_analytics_active", return_value=True) as gate:++ def get_persons(*args: object, **kwargs: object) -> dict[str, SimpleNamespace]:+ if change == "index":+ self.config.account_group_type_index = 1+ self.config.save(update_fields=["account_group_type_index"])+ elif change == "version":+ self.config.refresh_from_db()+ self.config.properties["membership"]["version"] = state.version + 1+ self.config.save(update_fields=["properties"])+ else:+ gate.return_value = False+ return {"identified": SimpleNamespace(is_identified=True)}++ with (+ patch.object(self.service, "query", return_value=[("identified", "account-a", START, END)]) as query,+ patch(f"{MODULE}.use_new_events_schema", return_value=False),+ patch(f"{MODULE}.get_persons_mapped_by_distinct_id", side_effect=get_persons),+ ):+ assert self.process_batch(state).rows == 0+ query.assert_called_once()++ def test_failed_batch_resumes_from_the_last_committed_window(self) -> None:+ state = self.create_state(+ historical_end=START + timedelta(days=270), next_window_end=START + timedelta(days=270)+ )+ with (+ patch.object(self.service, "query", return_value=[]),+ patch(f"{MODULE}.use_new_events_schema", return_value=False),+ ):+ assert self.process_batch(state).status == "backfilling"+ self.service.fail(self.team.id, state.version, "Unavailable")+ assert self.process_batch(state).status == "backfilling"+ assert state.next_window_end == START + timedelta(days=90)+ assert state.current_group_key == "account-a"+ assert state.error_type == ""++ def test_readiness_waits_for_dictionary_and_catch_up(self) -> None:+ state = self.create_state(+ next_window_start=END, current_group_key="", last_group_key="account-a", next_window_end=None+ )+ with patch.object(self.service, "dictionary_visible", return_value=False):+ assert self.process_batch(state).status == "waiting_for_dictionary"+ assert get_membership_readiness(self.team) == "catching_up"+ with (+ patch.object(self.service, "dictionary_visible", return_value=True),+ patch(f"{MODULE}.timezone.now", return_value=END + timedelta(minutes=10)),+ ):+ assert self.process_batch(state).status == "catching_up"+ assert state.catch_up_end == END + timedelta(minutes=10)+ assert state.next_window_start is not None+ assert state.next_window_start < END+ with (+ patch.object(self.service, "query", return_value=[]),+ patch(f"{MODULE}.use_new_events_schema", return_value=False),+ ):+ self.process_batch(state)+ with patch.object(self.service, "dictionary_visible", return_value=True):+ assert self.process_batch(state).status == "ready"+ assert get_membership_readiness(self.team) == "ready"++ def test_disabled_and_changed_configs_stop_work(self) -> None:+ state = self.create_state()+ for index in (None, 2):+ self.config.account_group_type_index = index+ self.config.save(update_fields=["account_group_type_index"])+ assert self.process_batch(state).status == "disabled"+ assert state.next_window_start == START++ def test_planning_command_stops_at_its_deadline(self) -> None:+ clock = 1000.0+ end = timezone.now()++ def execute(*args: object, **kwargs: object) -> list[tuple[object, ...]]:+ nonlocal clock+ sql = cast(str, args[0])+ if "ORDER BY event_date ASC" in sql:+ return [((end - timedelta(days=3650)).date(),)]+ if "ORDER BY event_date DESC" in sql:+ return [(end.date(),)]+ clock += 301+ return [(None, None, 0)]++ output = StringIO()+ with (+ patch(+ "products.customer_analytics.backend.management.commands.plan_membership_backfill.monotonic",+ return_value=clock,+ ),+ patch(f"{MODULE}.monotonic", side_effect=lambda: clock),+ patch(f"{MODULE}.sync_execute", side_effect=execute) as query,+ pytest.raises(RuntimeError, match="attempt expired"),+ ):+ call_command("plan_membership_backfill", team_id=self.team.id, stdout=output)+ assert query.call_count == 3+ assert output.getvalue() == ""++ @parameterized.expand([(False,), (True,)])+ def test_backfill_sql_admission_and_overlap(self, native_json: bool) -> None:+ start = timezone.now() - timedelta(days=100)+ end = start + timedelta(hours=1)+ state = self.create_state(+ historical_start=start,+ historical_end=end,+ next_window_start=start,+ next_window_end=end,+ snapshot_end=timezone.now() + timedelta(minutes=1),+ )+ create_person(team=self.team, distinct_ids=["identified"], is_identified=True)+ create_person(team=self.team, distinct_ids=["anonymous"], is_identified=False)+ with (+ patch(f"{MODULE}.use_new_events_schema", return_value=native_json),+ override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=native_json),+ events_only_in_active_schema(),+ ):+ events: list[tuple[str, Literal["full", "propertyless"], dict[str, str]]] = [+ ("identified", "full", {"$group_0": "account-a", "$group_1": "other-account"}),+ ("identified", "propertyless", {"$group_0": "personless-account"}),+ ("anonymous", "full", {"$group_0": "account-a"}),+ ("missing", "full", {"$group_0": "account-a"}),+ ("identified", "full", {"$group_1": "wrong-index"}),+ ("identified", "full", {"$group_0": ""}),+ ]+ for distinct_id, mode, properties in events:+ create_event(+ event_uuid=uuid4(),+ event="example_action",+ team=self.team,+ distinct_id=distinct_id,+ person_mode=mode,+ properties=properties,+ timestamp=start + timedelta(minutes=5),+ )+ outliers: list[tuple[int, Literal["full", "propertyless"], dict[str, str]]] = [+ (-3650, "full", {}),+ (-3600, "propertyless", {"$group_0": "account-a"}),+ (3650, "full", {"$group_1": "wrong-index"}),+ ]+ for days, mode, properties in outliers:+ create_event(+ event_uuid=uuid4(),+ event="example_action",+ team=self.team,+ distinct_id="identified",+ person_mode=mode,+ properties=properties,+ timestamp=start + timedelta(days=days),+ )+ snapshot = timezone.now() + timedelta(minutes=1)+ with patch.object(self.service, "query", wraps=self.service.query) as query:+ plan = self.service.plan(self.team, 0, snapshot)+ planning_windows = [call.args[1] for call in query.call_args_list if "minOrNull(timestamp)" in call.args[0]]+ assert len(planning_windows) == 2+ for window in planning_windows:+ assert timedelta(0) < window["end"] - window["start"] <= timedelta(days=90)+ assert window["snapshot_end"] == snapshot+ assert plan.historical_start == start + timedelta(minutes=5)+ assert plan.historical_end == snapshot+ assert plan.estimated_events == 3+ assert plan.windows == 2+ absent_account = self.service.get_history_bounds(self.team.id, 0, snapshot, ("absent-account",))+ assert absent_account.start is None+ assert absent_account.end == snapshot+ self.process_batch(state)+ first = self.service.query(+ "SELECT group_type_index, group_key, distinct_id, min(first_seen), max(last_seen) FROM person_group_membership WHERE team_id = %(team_id)s GROUP BY group_type_index, group_key, distinct_id",+ {"team_id": self.team.id},+ team_id=self.team.id,+ )+ assert len(first) == 1+ assert first[0][:3] == (0, "account-a", "identified")+ state.next_window_start = start+ state.next_window_end = end+ state.current_group_key = "account-a"+ self.service.save_progress(state)+ self.process_batch(state)+ assert (+ self.service.query(+ "SELECT group_type_index, group_key, distinct_id, min(first_seen), max(last_seen) FROM person_group_membership WHERE team_id = %(team_id)s GROUP BY group_type_index, group_key, distinct_id",+ {"team_id": self.team.id},+ team_id=self.team.id,+ )+ == first+ )+ create_event(+ event_uuid=uuid4(),+ event="example_action",+ team=self.team,+ distinct_id="identified",+ person_mode="full",+ properties={"$group_0": "account-a"},+ timestamp=start + timedelta(minutes=10),+ )+ state.catch_up_end = timezone.now() + timedelta(minutes=1)+ state.next_window_start = timezone.now() - timedelta(minutes=1)+ self.service.save_progress(state)+ self.process_batch(state)+ caught_up = self.service.query(+ "SELECT group_key, distinct_id, min(first_seen), max(last_seen) FROM person_group_membership WHERE team_id = %(team_id)s GROUP BY group_key, distinct_id",+ {"team_id": self.team.id},+ team_id=self.team.id,+ )+ assert len(caught_up) == 1+ assert caught_up[0][:2] == ("account-a", "identified")+ assert caught_up[0][2] == first[0][3]+ assert caught_up[0][3] == cast(datetime, first[0][4]) + timedelta(minutes=5)++ @parameterized.expand([(False,), (True,)])+ def test_history_pages_distinct_ids_without_skipping_a_window(self, native_json: bool) -> None:+ end = timezone.now() + timedelta(minutes=1)+ start = end - timedelta(hours=1)+ state = self.create_state(+ historical_start=start, historical_end=end, next_window_start=start, next_window_end=end+ )+ with (+ patch(f"{MODULE}.use_new_events_schema", return_value=native_json),+ patch(f"{MODULE}.BATCH_SIZE", 2),+ override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=native_json),+ events_only_in_active_schema(),+ ):+ for key in ("person-a", "person-b", "person-c"):+ create_person(team=self.team, distinct_ids=[key], is_identified=True)+ create_event(+ event_uuid=uuid4(),+ event="example_action",+ team=self.team,+ distinct_id=key,+ person_mode="full",+ properties={"$group_0": "account-a"},+ timestamp=start + timedelta(minutes=5),+ )+ assert self.process_batch(state).rows == 2+ assert state.last_distinct_id == "person-b"+ assert state.next_window_end == end+ assert state.windows_completed == 0+ with patch(f"{MODULE}.get_persons_mapped_by_distinct_id", side_effect=RuntimeError("Unavailable")):+ with pytest.raises(RuntimeError, match="Unavailable"):+ self.process_batch(state)+ assert state.last_distinct_id == "person-b"+ assert self.process_batch(state).rows == 1+ assert state.windows_completed == 1+ assert state.accounts_completed == 1+ assert state.rows_written == 3+ assert self.service.query(+ "SELECT distinct_id FROM person_group_membership WHERE team_id = %(team_id)s GROUP BY distinct_id ORDER BY distinct_id",+ {"team_id": self.team.id},+ ) == [("person-a",), ("person-b",), ("person-c",)]++ @parameterized.expand([(False,), (True,)])+ def test_initial_snapshot_includes_future_timestamped_events(self, native_json: bool) -> None:+ now = timezone.now()+ snapshot_end = now + timedelta(minutes=10)+ future = now + timedelta(hours=12)+ create_person(team=self.team, distinct_ids=["identified"], is_identified=True)+ with (+ patch(f"{MODULE}.use_new_events_schema", return_value=native_json),+ override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=native_json),+ events_only_in_active_schema(),+ ):+ create_event(+ event_uuid=uuid4(),+ event="example_action",+ team=self.team,+ distinct_id="identified",+ person_mode="full",+ properties={"$group_0": "account-a"},+ timestamp=future,+ )+ plan = self.service.plan(self.team, 0, snapshot_end)+ assert plan.estimated_events == 1+ with patch(f"{MODULE}.timezone.now", return_value=snapshot_end):+ result = self.service.process_batch(MembershipTeam(team_id=self.team.id))+ assert result.rows == 1+ assert result.cursor is not None+ assert result.cursor.snapshot_end == snapshot_end+ assert result.cursor.historical_end is not None+ assert future < result.cursor.historical_end <= future + timedelta(days=1)+ assert result.cursor.accounts_completed == 1+ assert self.service.query(+ "SELECT min(first_seen), max(last_seen) FROM person_group_membership WHERE team_id = %(team_id)s",+ {"team_id": self.team.id},+ ) == [(future, future)]+ with (+ patch.object(self.service, "dictionary_visible", return_value=True),+ patch(f"{MODULE}.timezone.now", return_value=snapshot_end + timedelta(minutes=1)),+ ):+ caught_up = self.service.process_batch(MembershipTeam(team_id=self.team.id, cursor=result.cursor))+ assert caught_up.cursor is not None+ assert caught_up.cursor.next_window_start == snapshot_end - timedelta(minutes=5)++ def test_initial_snapshot_includes_null_insertion_timestamps(self) -> None:+ end = timezone.now() - timedelta(hours=1)+ start = end - timedelta(days=1)+ create_person(team=self.team, distinct_ids=["identified"], is_identified=True)+ with (+ patch(f"{MODULE}.use_new_events_schema", return_value=False),+ override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=False),+ ):+ self.service.query(+ """INSERT INTO sharded_events+ (uuid, event, team_id, distinct_id, timestamp, properties, person_mode, inserted_at) VALUES""",+ [+ (+ uuid4(),+ "example_action",+ self.team.id,+ "identified",+ start,+ '{"$group_0":"account-a"}',+ "full",+ None,+ ),+ (+ uuid4(),+ "example_action",+ self.team.id,+ "identified",+ start,+ '{"$group_0":"late-account"}',+ "full",+ end + timedelta(minutes=1),+ ),+ ],+ )+ plan = self.service.plan(self.team, 0, end)+ assert plan.estimated_events == 1+ assert plan.historical_start == start+ state = self.create_state(+ historical_start=start, historical_end=end, current_group_key="", snapshot_end=end+ )+ assert self.process_batch(state).rows == 1+ assert state.accounts_completed == 1+ assert self.service.query(+ "SELECT group_key, min(first_seen), max(last_seen) FROM person_group_membership "+ "WHERE team_id = %(team_id)s GROUP BY group_key",+ {"team_id": self.team.id},+ ) == [("account-a", start, start)]++ @parameterized.expand([(False,), (True,)])+ def test_history_visits_each_account_through_empty_windows(self, native_json: bool) -> None:+ end = timezone.now()+ start = end - timedelta(days=270)+ state = self.create_state(+ historical_start=start,+ historical_end=end,+ next_window_start=start,+ next_window_end=None,+ current_group_key="",+ )+ create_person(team=self.team, distinct_ids=["identified"], is_identified=True)+ raw_create_group_ch(self.team.id, 1, "wrong-index", {}, start)+ other_team = Team.objects.create(organization=self.organization)+ raw_create_group_ch(other_team.id, 0, "wrong-team", {}, start)+ with (+ patch(f"{MODULE}.use_new_events_schema", return_value=native_json),+ override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=native_json),+ events_only_in_active_schema(),+ patch(f"{MODULE}.PAGE_SIZE", 1),+ ):+ for key, timestamp in (+ ("account-a", start + timedelta(hours=1)),+ ("account-a", end - timedelta(hours=1)),+ ("account-b", end - timedelta(days=100)),+ ):+ create_event(+ event_uuid=uuid4(),+ event="example_action",+ team=self.team,+ distinct_id="identified",+ person_mode="full",+ properties={"$group_0": key},+ timestamp=timestamp,+ )+ state.snapshot_end = timezone.now() + timedelta(minutes=1)+ assert self.process_batch(state).status == "backfilling"+ assert state.discovery_window_end == end - timedelta(days=90)+ assert state.accounts_completed == 0+ with patch.object(self.service, "query", side_effect=RuntimeError("Unavailable")):+ with pytest.raises(RuntimeError, match="Unavailable"):+ self.process_batch(state)+ assert state.discovery_window_end == end - timedelta(days=90)+ for _ in range(20):+ assert self.process_batch(state).status == "backfilling"+ if state.accounts_completed == 2:+ break+ assert state.accounts_completed == 2+ assert state.windows_completed == 6+ assert state.current_group_key == ""+ assert state.last_group_key == "account-b"+ assert self.service.query(+ "SELECT group_key, distinct_id, min(first_seen), max(last_seen) FROM person_group_membership WHERE team_id = %(team_id)s GROUP BY group_key, distinct_id ORDER BY group_key",+ {"team_id": self.team.id},+ team_id=self.team.id,+ ) == [+ ("account-a", "identified", start + timedelta(hours=1), end - timedelta(hours=1)),+ ("account-b", "identified", end - timedelta(days=100), end - timedelta(days=100)),+ ]++ @parameterized.expand([(False,), (True,)])+ def test_selected_accounts_do_not_mark_team_ready(self, dry_run: bool) -> None:+ state = self.create_state(account_keys=("account-a",), dry_run=dry_run)+ state.next_window_start = END+ state.catch_up_end = END+ with patch.object(self.service, "dictionary_visible", return_value=True):+ assert self.process_batch(state).status == "ready"+ assert get_membership_readiness(self.team) == "backfilling"++ @parameterized.expand([(False,), (True,)])+ def test_selected_catch_up_excludes_other_accounts(self, native_json: bool) -> None:+ start = timezone.now() - timedelta(hours=1)+ end = timezone.now() + timedelta(minutes=1)+ state = self.create_state(+ account_keys=("account-a",),+ catch_up_end=end,+ next_window_start=start,+ )+ create_person(team=self.team, distinct_ids=["identified"], is_identified=True)+ with (+ patch(f"{MODULE}.use_new_events_schema", return_value=native_json),+ override_settings(CLICKHOUSE_HOGQL_USE_NEW_EVENTS_SCHEMA=native_json),+ events_only_in_active_schema(),+ ):+ for key in ("account-a", "account-b"):+ create_event(+ event_uuid=uuid4(),+ event="example_action",+ team=self.team,+ distinct_id="identified",+ person_mode="full",+ properties={"$group_0": key},+ )+ assert self.process_batch(state).rows == 1+ assert self.service.query(+ "SELECT group_key FROM person_group_membership WHERE team_id = %(team_id)s GROUP BY group_key",+ {"team_id": self.team.id},+ team_id=self.team.id,+ ) == [("account-a",)]++ def test_dry_run_reads_history_without_membership_or_readiness_writes(self) -> None:+ state = self.create_state(dry_run=True, account_keys=("account-a",))+ with (+ patch.object(self.service, "query", return_value=[("person-a", "account-a", START, START)]) as query,+ patch(f"{MODULE}.use_new_events_schema", return_value=False),+ ):+ result = self.process_batch(state)+ assert result.rows == 0+ assert state.rows_found == 1+ assert state.windows_completed == 1+ assert query.call_count == 1+ assert get_membership_readiness(self.team) == "backfilling"diff --git a/products/product_analytics/backend/tests/api/__snapshots__/test_insight.ambr b/products/product_analytics/backend/tests/api/__snapshots__/test_insight.ambrindex 36365900e570..ac0896a69dee 100644--- a/products/product_analytics/backend/tests/api/__snapshots__/test_insight.ambr+++ b/products/product_analytics/backend/tests/api/__snapshots__/test_insight.ambr@@ -73,7 +73,8 @@"customer_analytics_teamcustomeranalyticsconfig"."account_group_type_index","customer_analytics_teamcustomeranalyticsconfig"."default_pinned_properties","customer_analytics_teamcustomeranalyticsconfig"."account_track_rules",- "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at"+ "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at",+ "customer_analytics_teamcustomeranalyticsconfig"."properties"FROM "customer_analytics_teamcustomeranalyticsconfig"WHERE "customer_analytics_teamcustomeranalyticsconfig"."team_id" = 99999LIMIT 21@@ -1667,7 +1668,8 @@"customer_analytics_teamcustomeranalyticsconfig"."account_group_type_index","customer_analytics_teamcustomeranalyticsconfig"."default_pinned_properties","customer_analytics_teamcustomeranalyticsconfig"."account_track_rules",- "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at"+ "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at",+ "customer_analytics_teamcustomeranalyticsconfig"."properties"FROM "customer_analytics_teamcustomeranalyticsconfig"WHERE "customer_analytics_teamcustomeranalyticsconfig"."team_id" = 99999LIMIT 21@@ -3470,7 +3472,8 @@"customer_analytics_teamcustomeranalyticsconfig"."account_group_type_index","customer_analytics_teamcustomeranalyticsconfig"."default_pinned_properties","customer_analytics_teamcustomeranalyticsconfig"."account_track_rules",- "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at"+ "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at",+ "customer_analytics_teamcustomeranalyticsconfig"."properties"FROM "customer_analytics_teamcustomeranalyticsconfig"WHERE "customer_analytics_teamcustomeranalyticsconfig"."team_id" = 99999LIMIT 21@@ -4248,7 +4251,8 @@"customer_analytics_teamcustomeranalyticsconfig"."account_group_type_index","customer_analytics_teamcustomeranalyticsconfig"."default_pinned_properties","customer_analytics_teamcustomeranalyticsconfig"."account_track_rules",- "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at"+ "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at",+ "customer_analytics_teamcustomeranalyticsconfig"."properties"FROM "customer_analytics_teamcustomeranalyticsconfig"WHERE "customer_analytics_teamcustomeranalyticsconfig"."team_id" = 99999LIMIT 21@@ -6048,7 +6052,8 @@"customer_analytics_teamcustomeranalyticsconfig"."account_group_type_index","customer_analytics_teamcustomeranalyticsconfig"."default_pinned_properties","customer_analytics_teamcustomeranalyticsconfig"."account_track_rules",- "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at"+ "customer_analytics_teamcustomeranalyticsconfig"."account_track_rules_enabled_at",+ "customer_analytics_teamcustomeranalyticsconfig"."properties"FROM "customer_analytics_teamcustomeranalyticsconfig"WHERE "customer_analytics_teamcustomeranalyticsconfig"."team_id" = 99999LIMIT 21