:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(customer-analytics): add membership backfill controls (#124295)
Arthur Moreira de Deuse75ecae4952a
diff --git a/posthog/temporal/schedule.py b/posthog/temporal/schedule.pyindex 318179d19ac7..bf044637a667 100644--- a/posthog/temporal/schedule.py+++ b/posthog/temporal/schedule.py@@ -96,6 +96,7 @@ from products.conversations.backend.temporal.schedule import create_support_replfrom products.customer_analytics.backend.facade.temporal import (create_account_track_rule_coordinator_schedule,create_calendar_sync_coordinator_schedule,+ create_membership_coordinator_schedule,create_ownership_claims_coordinator_schedule,)from products.data_catalog.backend.facade.temporal import create_data_catalog_weekly_digest_schedule@@ -1021,6 +1022,7 @@ schedules = [create_account_track_rule_coordinator_schedule,create_calendar_sync_coordinator_schedule,create_ownership_claims_coordinator_schedule,+ create_membership_coordinator_schedule,create_replay_vision_reconciler_schedule,create_replay_vision_estimates_schedule,cleanup_replay_vision_search_suggestions_schedule,diff --git a/products/customer_analytics/backend/facade/temporal.py b/products/customer_analytics/backend/facade/temporal.pyindex d61e48b859c1..ac8ee3e8d68e 100644--- a/products/customer_analytics/backend/facade/temporal.py+++ b/products/customer_analytics/backend/facade/temporal.py@@ -12,6 +12,7 @@ from products.customer_analytics.backend.temporal.account_track_rules import (create_account_track_rule_coordinator_schedule,)from products.customer_analytics.backend.temporal.calendar_sync import create_calendar_sync_coordinator_schedule+from products.customer_analytics.backend.temporal.membership_backfill import create_membership_coordinator_schedulefrom products.customer_analytics.backend.temporal.ownership_claims import create_ownership_claims_coordinator_schedule__all__ = [@@ -23,5 +24,6 @@ __all__ = ["create_account_track_rule_coordinator_schedule","create_calendar_sync_coordinator_schedule","create_ownership_claims_coordinator_schedule",+ "create_membership_coordinator_schedule","stage_warehouse_account_property_files_activity",]diff --git a/products/customer_analytics/backend/logic/membership_backfill_control.py b/products/customer_analytics/backend/logic/membership_backfill_control.pynew file mode 100644index 000000000000..066ae0fe27fe--- /dev/null+++ b/products/customer_analytics/backend/logic/membership_backfill_control.py@@ -0,0 +1,48 @@+from django.conf import settings++from temporalio.client import Client+from temporalio.common import RetryPolicy, WorkflowIDReusePolicy++from posthog.temporal.common.client import async_connect++from products.customer_analytics.backend.logic.membership_backfill import MembershipTeam+from products.customer_analytics.backend.temporal.membership_backfill import (+ MembershipBackfillWorkflow,+ MembershipRunStatus,+ get_membership_workflow_id,+)+++class MembershipBackfillControl:+ def __init__(self, client: Client) -> None:+ self.client = client++ @classmethod+ async def connect(cls) -> "MembershipBackfillControl":+ return cls(await async_connect())++ async def start(self, team_id: int, *, account_keys: tuple[str, ...], dry_run: bool) -> str:+ handle = await self.client.start_workflow(+ MembershipBackfillWorkflow.run,+ MembershipTeam(team_id=team_id, account_keys=account_keys, dry_run=dry_run),+ id=get_membership_workflow_id(team_id),+ id_reuse_policy=WorkflowIDReusePolicy.ALLOW_DUPLICATE,+ task_queue=settings.VIDEO_EXPORT_TASK_QUEUE,+ retry_policy=RetryPolicy(maximum_attempts=1),+ )+ return handle.first_execution_run_id or ""++ async def get_status(self, team_id: int) -> MembershipRunStatus:+ handle = self.client.get_workflow_handle(get_membership_workflow_id(team_id))+ return await handle.query(MembershipBackfillWorkflow.get_status)++ async def resume(self, team_id: int) -> None:+ status = await self.get_status(team_id)+ if status.status != "failed":+ raise ValueError("Only a failed backfill can resume")+ handle = self.client.get_workflow_handle(get_membership_workflow_id(team_id))+ await handle.signal(MembershipBackfillWorkflow.resume)++ async def cancel(self, team_id: int) -> None:+ handle = self.client.get_workflow_handle(get_membership_workflow_id(team_id))+ await handle.signal(MembershipBackfillWorkflow.cancel)diff --git a/products/customer_analytics/backend/management/commands/backfill_memberships.py b/products/customer_analytics/backend/management/commands/backfill_memberships.pynew file mode 100644index 000000000000..31dc528fb76f--- /dev/null+++ b/products/customer_analytics/backend/management/commands/backfill_memberships.py@@ -0,0 +1,70 @@+import json+import asyncio+from dataclasses import asdict+from typing import cast++from django.core.management.base import BaseCommand, CommandError, CommandParser++from temporalio.exceptions import WorkflowAlreadyStartedError+from temporalio.service import RPCError++from posthog.models.team import Team++from products.customer_analytics.backend.logic.membership_backfill import get_membership_group_type_index+from products.customer_analytics.backend.logic.membership_backfill_control import MembershipBackfillControl+++class Command(BaseCommand):+ help = "Start, inspect, resume, or cancel an account membership backfill"++ def add_arguments(self, parser: CommandParser) -> None:+ commands = parser.add_subparsers(dest="action", required=True)+ for action in ("start", "status", "resume", "cancel"):+ command = commands.add_parser(action)+ command.add_argument("--team-id", type=int, required=True)+ if action == "start":+ scope = command.add_mutually_exclusive_group(required=True)+ scope.add_argument("--all-accounts", action="store_true")+ scope.add_argument("--account", action="append", dest="accounts")+ command.add_argument("--dry-run", action="store_true")++ def handle(self, *args: object, **options: object) -> None:+ team_id = cast(int, options["team_id"])+ action = cast(str, options["action"])+ accounts = tuple(sorted(set(cast(list[str], options.get("accounts") or []))))+ dry_run = bool(options.get("dry_run"))+ if action == "start":+ if any(not key or len(key.encode("utf-8")) > 1024 for key in accounts) or len(accounts) > 100:+ raise CommandError("Supply at most 100 nonempty account keys, each within 1024 bytes")+ team = Team.objects.filter(id=team_id).first()+ if team is None:+ raise CommandError("Project does not exist")+ try:+ index = get_membership_group_type_index(team)+ except RuntimeError as error:+ raise CommandError(str(error)) from error+ if index is None:+ raise CommandError("Customer analytics must be active with a valid account group type")+ try:+ output = asyncio.run(self.get_output(action, team_id, accounts, dry_run=dry_run))+ except WorkflowAlreadyStartedError as error:+ raise CommandError(+ "A backfill is already active for this project. Use status, resume, or cancel"+ ) from error+ except (RPCError, ValueError) as error:+ raise CommandError(str(error)) from error+ self.stdout.write(output)++ @staticmethod+ async def get_output(action: str, team_id: int, accounts: tuple[str, ...], *, dry_run: bool) -> str:+ control = await MembershipBackfillControl.connect()+ if action == "start":+ run_id = await control.start(team_id, account_keys=accounts, dry_run=dry_run)+ return json.dumps({"team_id": team_id, "run_id": run_id, "status": "starting", "dry_run": dry_run})+ if action == "status":+ return json.dumps(asdict(await control.get_status(team_id)), default=str)+ if action == "resume":+ await control.resume(team_id)+ return "Resume requested"+ await control.cancel(team_id)+ return "Cancel requested. The current batch can finish before the backfill stops"diff --git a/products/customer_analytics/backend/metrics.py b/products/customer_analytics/backend/metrics.pyindex 1db8d4e19b2b..20c2ae89ba92 100644--- a/products/customer_analytics/backend/metrics.py+++ b/products/customer_analytics/backend/metrics.py@@ -3,6 +3,24 @@ from prometheus_client import Counter, Gauge, Histogramfrom posthog.otel_metrics import OtelInstrumentFactory++def record_membership_operation(+ operation: str, outcome: str, count: int = 1, *, rows: int = 0, lag_seconds: float | None = None+) -> None:+ client = posthoganalytics.default_client+ if client is None:+ return+ try:+ client.metrics.count(+ "customer_analytics.membership.operations", count, attributes={"operation": operation, "outcome": outcome}+ )+ client.metrics.count("customer_analytics.membership.rows", rows)+ if lag_seconds is not None:+ client.metrics.histogram("customer_analytics.membership.lag", lag_seconds, unit="s")+ except Exception:+ pass++_ACCOUNT_TRACK_RULE_RUNS = Counter("customer_analytics_account_track_rule_runs_total","Account Track Rule runs by trigger and terminal status",diff --git a/products/customer_analytics/backend/metrics_test.py b/products/customer_analytics/backend/metrics_test.pyindex 32ec1b411259..b16fed047cc4 100644--- a/products/customer_analytics/backend/metrics_test.py+++ b/products/customer_analytics/backend/metrics_test.py@@ -1,6 +1,21 @@+import pytestfrom unittest.mock import MagicMock, patch-from products.customer_analytics.backend.metrics import record_account_property_sync_phase_duration+from products.customer_analytics.backend.metrics import (+ record_account_property_sync_phase_duration,+ record_membership_operation,+)++[email protected]("lag_seconds", [None, 0, 12.5])+def test_membership_lag_records_only_measured_values(lag_seconds: float | None) -> None:+ client = MagicMock()+ with patch("products.customer_analytics.backend.metrics.posthoganalytics.default_client", client):+ record_membership_operation("backfill", "catching_up", lag_seconds=lag_seconds)+ if lag_seconds is None:+ client.metrics.histogram.assert_not_called()+ else:+ client.metrics.histogram.assert_called_once_with("customer_analytics.membership.lag", lag_seconds, unit="s")def test_account_property_sync_phase_duration_records_posthog_metric() -> None:diff --git a/products/customer_analytics/backend/temporal/__init__.py b/products/customer_analytics/backend/temporal/__init__.pyindex 9d5703141314..ebeb6eb6760c 100644--- a/products/customer_analytics/backend/temporal/__init__.py+++ b/products/customer_analytics/backend/temporal/__init__.py@@ -16,6 +16,15 @@ from products.customer_analytics.backend.temporal.calendar_sync import (calendar_sync_integration_activity,google_account_email_backfill_activity,)+from products.customer_analytics.backend.temporal.membership_backfill import (+ MembershipBackfillWorkflow,+ MembershipCoordinatorWorkflow,+ backfill_membership_batch_activity,+ fail_membership_backfill_activity,+ observe_membership_backfill_activity,+ prepare_membership_backfill_activity,+ sync_membership_page_activity,+)from products.customer_analytics.backend.temporal.ownership_claims import (OwnershipClaimsCoordinatorWorkflow,OwnershipClaimsSweepWorkflow,@@ -24,6 +33,8 @@ from products.customer_analytics.backend.temporal.ownership_claims import ()WORKFLOWS = [+ MembershipBackfillWorkflow,+ MembershipCoordinatorWorkflow,AccountTrackRuleCoordinatorWorkflow,AccountTrackRuleEvaluationWorkflow,CalendarSyncCoordinatorWorkflow,@@ -33,6 +44,11 @@ WORKFLOWS = [OwnershipClaimsSweepWorkflow,]ACTIVITIES = [+ backfill_membership_batch_activity,+ fail_membership_backfill_activity,+ sync_membership_page_activity,+ observe_membership_backfill_activity,+ prepare_membership_backfill_activity,account_track_rule_collect_configs_activity,account_track_rule_create_scheduled_run_activity,account_track_rule_fail_run_activity,diff --git a/products/customer_analytics/backend/temporal/membership_backfill.py b/products/customer_analytics/backend/temporal/membership_backfill.pynew file mode 100644index 000000000000..299a6c5243bf--- /dev/null+++ b/products/customer_analytics/backend/temporal/membership_backfill.py@@ -0,0 +1,379 @@+import json+from dataclasses import replace+from datetime import timedelta+from threading import Event+from time import monotonic++from temporalio import activity, workflow+from temporalio.client import (+ Client,+ Schedule,+ ScheduleActionStartWorkflow,+ ScheduleIntervalSpec,+ ScheduleOverlapPolicy,+ SchedulePolicy,+ ScheduleSpec,+ ScheduleState,+)+from temporalio.common import RetryPolicy, WorkflowIDReusePolicy+from temporalio.exceptions import ActivityError, ApplicationError, WorkflowAlreadyStartedError++with workflow.unsafe.imports_passed_through():+ from django.conf import settings++ import structlog++ from posthog.dataclasses import frozen+ from posthog.scheduling.jitter import deterministic_offset+ from posthog.sync import database_sync_to_async+ from posthog.temporal.common.heartbeat import Heartbeater+ from posthog.temporal.common.schedule import a_create_schedule, a_schedule_exists, a_update_schedule++ from products.customer_analytics.backend.logic.membership_backfill import (+ MembershipBackfillService,+ MembershipCursor,+ MembershipStep,+ MembershipSyncPage,+ MembershipTeam,+ )+ from products.customer_analytics.backend.metrics import record_membership_operation++COORDINATOR_NAME = "customer-analytics-membership-coordinator"+BACKFILL_NAME = "customer-analytics-membership-backfill"+SCHEDULE_ID = "customer-analytics-membership-coordinator-schedule"+LOGGER = structlog.get_logger(__name__)+++@frozen+class MembershipCoordinatorInput:+ after_team_id: int = 0+ dry_run: bool = True+++@frozen+class MembershipFailure:+ team_id: int+ version: int+ error_type: str+ account_keys: tuple[str, ...] = ()+ dry_run: bool = False+++@frozen+class MembershipObservation:+ input: MembershipTeam+ run_id: str+ event: str+ status: str+ error_type: str = ""+++@frozen+class MembershipRunStatus:+ team_id: int+ run_id: str+ status: str+ dry_run: bool+ account_count: int+ cursor: MembershipCursor | None+ error_type: str = ""+++def get_membership_workflow_id(team_id: int) -> str:+ return f"{BACKFILL_NAME}-{team_id}"+++async def sync_membership_page_activity(input: MembershipCoordinatorInput) -> MembershipSyncPage:+ stopped = Event()+ service = MembershipBackfillService(deadline=monotonic() + 585, should_stop=stopped.is_set)+ try:+ async with Heartbeater():+ page = await database_sync_to_async(service.sync_page, thread_sensitive=False)(+ input.after_team_id, dry_run=input.dry_run+ )+ record_membership_operation("sync", "candidates", page.candidates)+ record_membership_operation("sync", "enabled", page.enabled)+ record_membership_operation("sync", "changed", page.changed)+ return page+ except Exception:+ record_membership_operation("sync", "failed")+ raise+ finally:+ stopped.set()+++async def prepare_membership_backfill_activity(input: MembershipTeam) -> MembershipTeam:+ stopped = Event()+ service = MembershipBackfillService(deadline=monotonic() + 105, should_stop=stopped.is_set)+ try:+ async with Heartbeater():+ return await database_sync_to_async(service.prepare_backfill, thread_sensitive=False)(input)+ finally:+ stopped.set()+++async def backfill_membership_batch_activity(input: MembershipTeam) -> MembershipStep:+ started = monotonic()+ stopped = Event()+ try:+ async with Heartbeater():+ result = await database_sync_to_async(+ MembershipBackfillService.process_activity_batch, thread_sensitive=False+ )(input, deadline=started + 285, should_stop=stopped.is_set)+ if result.cursor:+ result.cursor.last_batch_duration_seconds = monotonic() - started+ if result.lag_seconds is not None:+ result.cursor.lag_seconds = result.lag_seconds+ record_membership_operation("backfill", result.status, rows=result.rows, lag_seconds=result.lag_seconds)+ return result+ except Exception:+ record_membership_operation("backfill", "failed")+ raise+ finally:+ stopped.set()+++async def fail_membership_backfill_activity(input: MembershipFailure) -> None:+ if not input.account_keys and not input.dry_run:+ await database_sync_to_async(MembershipBackfillService.fail, thread_sensitive=False)(+ input.team_id, input.version, input.error_type+ )+++async def observe_membership_backfill_activity(observation: MembershipObservation) -> None:+ input = observation.input+ cursor = input.cursor+ duration = (+ (activity.info().current_attempt_scheduled_time - cursor.started_at).total_seconds()+ if cursor and cursor.started_at+ else 0+ )+ LOGGER.info(+ "customer_analytics.membership_backfill",+ backfill_event=observation.event,+ backfill_run_id=observation.run_id,+ team_id=input.team_id,+ status=observation.status,+ dry_run=input.dry_run,+ account_scope="selected" if input.account_keys else "all",+ account_count=len(input.account_keys),+ accounts_completed=cursor.accounts_completed if cursor else 0,+ windows_completed=cursor.windows_completed if cursor else 0,+ rows_written=cursor.rows_written if cursor else 0,+ rows_found=cursor.rows_found if cursor else 0,+ duration_seconds=max(0, duration),+ batch_duration_seconds=cursor.last_batch_duration_seconds if cursor else 0,+ lag_seconds=cursor.lag_seconds if cursor else 0,+ error_type=observation.error_type,+ )++[email protected](name=BACKFILL_NAME)+class MembershipBackfillWorkflow:+ def __init__(self) -> None:+ self.input = MembershipTeam(team_id=0)+ self.status = "starting"+ self.error_type = ""+ self.resume_requested = False+ self.cancel_requested = False+ self.run_id = ""++ @staticmethod+ def parse_inputs(inputs: list[str]) -> MembershipTeam:+ return MembershipTeam(**json.loads(inputs[0]))++ @workflow.query+ def get_status(self) -> MembershipRunStatus:+ return MembershipRunStatus(+ team_id=self.input.team_id,+ run_id=self.run_id,+ status=self.status,+ dry_run=self.input.dry_run,+ account_count=len(self.input.account_keys),+ cursor=self.input.cursor,+ error_type=self.error_type,+ )++ @workflow.signal+ def resume(self) -> None:+ if self.status == "failed":+ self.resume_requested = True++ @workflow.signal+ def cancel(self) -> None:+ self.cancel_requested = True++ async def observe(self, event: str) -> None:+ try:+ await workflow.execute_activity(+ observe_membership_backfill_activity,+ MembershipObservation(+ input=self.input, run_id=self.run_id, event=event, status=self.status, error_type=self.error_type+ ),+ start_to_close_timeout=timedelta(seconds=30),+ retry_policy=RetryPolicy(maximum_attempts=3),+ )+ except ActivityError:+ workflow.logger.warning("Membership backfill observation failed", extra={"backfill_event": event})++ async def process_batch(self) -> MembershipStep:+ if not self.input.cursor and not self.input.version and not self.input.dry_run:+ self.input = await workflow.execute_activity(+ prepare_membership_backfill_activity,+ self.input,+ start_to_close_timeout=timedelta(minutes=2),+ heartbeat_timeout=timedelta(minutes=1),+ retry_policy=RetryPolicy(maximum_attempts=3),+ )+ result = await workflow.execute_activity(+ backfill_membership_batch_activity,+ self.input,+ start_to_close_timeout=timedelta(minutes=5),+ heartbeat_timeout=timedelta(minutes=1),+ retry_policy=RetryPolicy(maximum_attempts=3),+ )+ self.input = replace(self.input, cursor=result.cursor)+ self.status = result.status+ self.error_type = ""+ await self.observe("progress")+ return replace(result, status="cancelled") if self.cancel_requested else result++ @workflow.run+ async def run(self, input: MembershipTeam) -> str:+ if workflow.info().workflow_id != get_membership_workflow_id(input.team_id):+ raise ApplicationError("Use the team membership workflow ID", non_retryable=True)+ self.input = input+ self.run_id = workflow.info().first_execution_run_id+ self.status = input.cursor.status if input.cursor else "starting"+ if input.cursor is None:+ await self.observe("started")+ waiting_since = None+ for _ in range(100):+ if self.cancel_requested:+ break+ try:+ result = await self.process_batch()+ if result.status == "cancelled":+ break+ if result.status in ("ready", "disabled", "completed"):+ await self.observe("completed")+ return result.status+ if result.status == "waiting_for_dictionary":+ waiting_since = waiting_since or workflow.now()+ if workflow.now() - waiting_since >= timedelta(minutes=30):+ raise ApplicationError("Membership dictionary did not refresh")+ await workflow.wait_condition(lambda: self.cancel_requested, timeout=timedelta(seconds=30))+ else:+ waiting_since = None+ except TimeoutError:+ continue+ except Exception as error:+ self.status = "failed"+ cause = error.__cause__ or error+ self.error_type = (+ cause.type if isinstance(cause, ApplicationError) and cause.type else type(cause).__name__+ )+ try:+ await workflow.execute_activity(+ fail_membership_backfill_activity,+ MembershipFailure(+ team_id=input.team_id,+ version=self.input.cursor.version if self.input.cursor else self.input.version,+ error_type=self.error_type,+ account_keys=input.account_keys,+ dry_run=input.dry_run,+ ),+ start_to_close_timeout=timedelta(minutes=1),+ retry_policy=RetryPolicy(maximum_attempts=3),+ )+ except ActivityError:+ workflow.logger.warning("Membership backfill failure update failed")+ await self.observe("failed")+ await workflow.wait_condition(lambda: self.resume_requested or self.cancel_requested)+ self.resume_requested = False+ waiting_since = None+ if not self.cancel_requested:+ self.status = "resuming"+ await self.observe("resumed")+ if self.cancel_requested:+ self.status = "cancelled"+ await self.observe("cancelled")+ return self.status+ return workflow.continue_as_new(self.input)++[email protected](name=COORDINATOR_NAME)+class MembershipCoordinatorWorkflow:+ @staticmethod+ def parse_inputs(inputs: list[str]) -> MembershipCoordinatorInput:+ return MembershipCoordinatorInput(**json.loads(inputs[0])) if inputs else MembershipCoordinatorInput()++ @workflow.run+ async def run(self, input: MembershipCoordinatorInput) -> None:+ after = input.after_team_id+ for _ in range(50):+ page = await workflow.execute_activity(+ sync_membership_page_activity,+ MembershipCoordinatorInput(after_team_id=after, dry_run=input.dry_run),+ start_to_close_timeout=timedelta(minutes=10),+ heartbeat_timeout=timedelta(minutes=1),+ retry_policy=RetryPolicy(maximum_attempts=3),+ )+ for team in page.teams:+ try:+ await workflow.start_child_workflow(+ MembershipBackfillWorkflow.run,+ team,+ id=get_membership_workflow_id(team.team_id),+ id_reuse_policy=WorkflowIDReusePolicy.ALLOW_DUPLICATE,+ parent_close_policy=workflow.ParentClosePolicy.ABANDON,+ retry_policy=RetryPolicy(maximum_attempts=1),+ )+ except WorkflowAlreadyStartedError:+ pass+ if page.next_team_id is None:+ return+ after = page.next_team_id+ workflow.continue_as_new(MembershipCoordinatorInput(after_team_id=after, dry_run=input.dry_run))+++def create_membership_schedule(state: ScheduleState) -> Schedule:+ return Schedule(+ action=ScheduleActionStartWorkflow(+ COORDINATOR_NAME,+ MembershipCoordinatorInput(dry_run=False),+ id=COORDINATOR_NAME,+ task_queue=settings.VIDEO_EXPORT_TASK_QUEUE,+ execution_timeout=timedelta(hours=24),+ retry_policy=RetryPolicy(maximum_attempts=1),+ ),+ spec=ScheduleSpec(+ intervals=[+ ScheduleIntervalSpec(+ every=timedelta(hours=1), offset=deterministic_offset(SCHEDULE_ID, timedelta(hours=1))+ )+ ]+ ),+ state=state,+ policy=SchedulePolicy(overlap=ScheduleOverlapPolicy.SKIP),+ )+++async def create_membership_coordinator_schedule(client: Client) -> None:+ if await a_schedule_exists(client, SCHEDULE_ID):+ description = await client.get_schedule_handle(SCHEDULE_ID).describe()+ await a_update_schedule(+ client, SCHEDULE_ID, create_membership_schedule(description.schedule.state), keep_paused=True+ )+ else:+ await a_create_schedule(+ client,+ SCHEDULE_ID,+ create_membership_schedule(ScheduleState(paused=True, note="Enable after live ingestion is ready")),+ trigger_immediately=False,+ )diff --git a/products/customer_analytics/backend/temporal/membership_backfill_test.py b/products/customer_analytics/backend/temporal/membership_backfill_test.pynew file mode 100644index 000000000000..c58f6d0b96be--- /dev/null+++ b/products/customer_analytics/backend/temporal/membership_backfill_test.py@@ -0,0 +1,345 @@+import asyncio+import logging+from collections.abc import Callable+from dataclasses import replace+from datetime import UTC, datetime, timedelta+from threading import Event+from uuid import uuid4++import pytest+from unittest.mock import AsyncMock, Mock, patch++from structlog.testing import capture_logs+from temporalio import activity+from temporalio.api.enums.v1 import EventType+from temporalio.client import ScheduleActionStartWorkflow, ScheduleState+from temporalio.converter import DataConverter+from temporalio.exceptions import ApplicationError+from temporalio.testing import ActivityEnvironment, WorkflowEnvironment+from temporalio.worker import Replayer, Worker++from products.customer_analytics.backend.logic.membership_backfill import (+ PAGE_SIZE,+ MembershipBackfillService,+ MembershipCursor,+ MembershipStep,+ MembershipSyncPage,+ MembershipTeam,+)+from products.customer_analytics.backend.temporal.membership_backfill import (+ MembershipBackfillWorkflow,+ MembershipCoordinatorInput,+ MembershipCoordinatorWorkflow,+ MembershipFailure,+ MembershipObservation,+ backfill_membership_batch_activity,+ create_membership_coordinator_schedule,+ create_membership_schedule,+ get_membership_workflow_id,+ observe_membership_backfill_activity,+ prepare_membership_backfill_activity,+ sync_membership_page_activity,+)+++async def test_largest_coordinator_payload_contains_only_bounded_team_references() -> None:+ page = MembershipSyncPage(+ teams=tuple(MembershipTeam(team_id=team_id, version=2**63) for team_id in range(PAGE_SIZE)),+ next_team_id=PAGE_SIZE,+ candidates=PAGE_SIZE,+ enabled=PAGE_SIZE,+ changed=PAGE_SIZE,+ )+ payloads = await DataConverter.default.encode([page])+ assert sum(len(payload.data) for payload in payloads) < 16_384+ assert await DataConverter.default.decode(payloads, [MembershipSyncPage]) == [page]+++def test_schedule_preserves_pause_and_runs_with_writes() -> None:+ state = ScheduleState(paused=True)+ schedule = create_membership_schedule(state)+ assert schedule.state.paused+ assert isinstance(schedule.action, ScheduleActionStartWorkflow)+ assert schedule.action.args == [MembershipCoordinatorInput(dry_run=False)]+ assert not create_membership_schedule(ScheduleState(paused=False)).state.paused+++async def test_schedule_update_keeps_an_operator_pause() -> None:+ client = AsyncMock()+ description = client.get_schedule_handle.return_value.describe.return_value+ description.schedule = create_membership_schedule(ScheduleState(paused=False))+ client.get_schedule_handle = Mock(return_value=client.get_schedule_handle.return_value)+ with patch("products.customer_analytics.backend.temporal.membership_backfill.a_schedule_exists", return_value=True):+ await create_membership_coordinator_schedule(client)+ updater = client.get_schedule_handle.return_value.update.call_args.kwargs["updater"]+ current = Mock(schedule=create_membership_schedule(ScheduleState(paused=True, note="Operator paused")))+ update_input = Mock(description=current)+ update = await updater(update_input)+ assert update.schedule.state.paused+ assert update.schedule.state.note == "Operator paused"+++async def test_retry_overlap_worker_restart_and_replay() -> None:+ logging.getLogger("temporalio.activity").setLevel(logging.INFO)+ logging.getLogger("temporalio.workflow").setLevel(logging.INFO)+ started = asyncio.Event()+ attempts = 0+ team = MembershipTeam(team_id=17, version=1)++ @activity.defn(name="sync_membership_page_activity")+ async def sync_page(_input: MembershipCoordinatorInput) -> MembershipSyncPage:+ return MembershipSyncPage(teams=(team,), next_team_id=None, candidates=1, enabled=1, changed=1)++ @activity.defn(name="backfill_membership_batch_activity")+ async def backfill(_input: MembershipTeam) -> MembershipStep:+ nonlocal attempts+ attempts += 1+ if attempts == 1:+ raise ApplicationError("Retry this batch")+ if attempts == 2:+ started.set()+ return MembershipStep(status="waiting_for_dictionary")+ return MembershipStep(status="ready", rows=1)++ @activity.defn(name="fail_membership_backfill_activity")+ async def fail(_input: MembershipFailure) -> None:+ raise AssertionError("The retried batch must succeed")++ observations: list[str] = []++ @activity.defn(name="observe_membership_backfill_activity")+ async def observe(input: MembershipObservation) -> None:+ observations.append(input.event)++ queue = f"membership-test-{uuid4()}"+ async with await WorkflowEnvironment.start_time_skipping() as env:+ with env.auto_time_skipping_disabled():+ async with Worker(+ env.client,+ task_queue=queue,+ workflows=[MembershipCoordinatorWorkflow, MembershipBackfillWorkflow],+ activities=[sync_page, backfill, fail, observe],+ max_cached_workflows=0,+ ):+ await env.client.execute_workflow(+ MembershipCoordinatorWorkflow.run,+ MembershipCoordinatorInput(dry_run=False),+ id=f"coordinator-{uuid4()}",+ task_queue=queue,+ execution_timeout=timedelta(minutes=5),+ )+ await asyncio.wait_for(started.wait(), timeout=30)+ await env.client.execute_workflow(+ MembershipCoordinatorWorkflow.run,+ MembershipCoordinatorInput(dry_run=False),+ id=f"coordinator-{uuid4()}",+ task_queue=queue,+ execution_timeout=timedelta(minutes=5),+ )+ workflow_id = get_membership_workflow_id(team.team_id)+ handle = env.client.get_workflow_handle(workflow_id)+ handle = env.client.get_workflow_handle(workflow_id, run_id=(await handle.describe()).run_id)+ for _ in range(100):+ history = await handle.fetch_history()+ if any(event.event_type == EventType.EVENT_TYPE_TIMER_STARTED for event in history.events):+ break+ await asyncio.sleep(0.01)+ else:+ raise AssertionError("The workflow did not start its dictionary timer")+ async with Worker(+ env.client,+ task_queue=queue,+ workflows=[MembershipCoordinatorWorkflow, MembershipBackfillWorkflow],+ activities=[sync_page, backfill, fail, observe],+ ):+ await env.sleep(timedelta(seconds=31))+ assert await asyncio.wait_for(handle.result(), timeout=30) == "ready"+ history = await handle.fetch_history()+ assert sum(event.event_type == EventType.EVENT_TYPE_WORKFLOW_EXECUTION_STARTED for event in history.events) == 1+ assert attempts == 3+ assert observations.count("started") == 1+ assert observations[-1] == "completed"+ await Replayer(workflows=[MembershipBackfillWorkflow]).replay_workflow(history)+++ "has_cursor,bookkeeping_fails,observation_fails",+ [(True, False, False), (True, True, True), (False, False, False)],+)+async def test_failed_run_keeps_its_cursor_for_resume_and_can_be_cancelled(+ has_cursor: bool, bookkeeping_fails: bool, observation_fails: bool+) -> None:+ logging.getLogger("temporalio.activity").setLevel(logging.INFO)+ logging.getLogger("temporalio.workflow").setLevel(logging.INFO)+ observations: asyncio.Queue[MembershipObservation] = asyncio.Queue()+ failures: list[MembershipFailure] = []+ account_keys = ("account-a",) if has_cursor else ()+ failed = True++ @activity.defn(name="prepare_membership_backfill_activity")+ async def prepare(input: MembershipTeam) -> MembershipTeam:+ return replace(input, version=7)++ @activity.defn(name="backfill_membership_batch_activity")+ async def backfill(input: MembershipTeam) -> MembershipStep:+ assert (input.cursor.version if input.cursor else input.version) == 7+ if failed:+ raise ApplicationError("Unavailable", type="ExampleUnavailable", non_retryable=True)+ assert input.account_keys == account_keys+ return MembershipStep(status="waiting_for_dictionary", cursor=input.cursor)++ @activity.defn(name="fail_membership_backfill_activity")+ async def fail(input: MembershipFailure) -> None:+ failures.append(input)+ if bookkeeping_fails:+ raise ApplicationError("Unavailable", non_retryable=True)++ @activity.defn(name="observe_membership_backfill_activity")+ async def observe(input: MembershipObservation) -> None:+ await observations.put(input)+ if observation_fails and input.event == "failed":+ raise ApplicationError("Unavailable", non_retryable=True)++ async def wait_for_event(event: str) -> MembershipObservation:+ while True:+ observation = await asyncio.wait_for(observations.get(), timeout=30)+ if observation.event == event:+ return observation++ queue = f"membership-test-{uuid4()}"+ async with await WorkflowEnvironment.start_time_skipping() as env:+ with env.auto_time_skipping_disabled():+ async with Worker(+ env.client,+ task_queue=queue,+ workflows=[MembershipBackfillWorkflow],+ activities=[prepare, backfill, fail, observe],+ ):+ handle = await env.client.start_workflow(+ MembershipBackfillWorkflow.run,+ MembershipTeam(+ team_id=17,+ account_keys=account_keys,+ dry_run=has_cursor,+ cursor=MembershipCursor(+ team_id=17,+ version=7,+ group_type_index=0,+ account_keys=("account-a",),+ dry_run=True,+ historical_end=datetime(2026, 1, 1, tzinfo=UTC),+ windows_completed=3,+ )+ if has_cursor+ else None,+ ),+ id=get_membership_workflow_id(17),+ task_queue=queue,+ execution_timeout=timedelta(minutes=5),+ )+ assert (await wait_for_event("failed")).error_type == "ExampleUnavailable"+ assert (await handle.query(MembershipBackfillWorkflow.get_status)).status == "failed"+ assert failures[0].version == 7+ failed = False+ await handle.signal(MembershipBackfillWorkflow.resume)+ await wait_for_event("resumed")+ await wait_for_event("progress")+ await handle.signal(MembershipBackfillWorkflow.cancel)+ assert await asyncio.wait_for(handle.result(), timeout=30) == "cancelled"+ observation = await wait_for_event("cancelled")+ assert observation.input.account_keys == account_keys+ if has_cursor:+ assert observation.input.cursor is not None+ assert observation.input.cursor.windows_completed == 3+ assert observation.input.cursor.historical_end == datetime(2026, 1, 1, tzinfo=UTC)+ else:+ assert observation.input.version == 7+ await Replayer(workflows=[MembershipBackfillWorkflow]).replay_workflow(await handle.fetch_history())++[email protected]("operation", ["batch", "sync", "prepare"])+async def test_activity_cancellation_stops_the_database_thread(operation: str) -> None:+ loop = asyncio.get_running_loop()+ started = asyncio.Event()+ checked = asyncio.Event()+ release = Event()+ stopped: list[bool] = []++ def wait_for_cancellation(should_stop: Callable[[], bool]) -> None:+ loop.call_soon_threadsafe(started.set)+ if not release.wait(timeout=10):+ raise RuntimeError("Database thread did not finish")+ stopped.append(should_stop())+ loop.call_soon_threadsafe(checked.set)++ def process_batch(_input: MembershipTeam, *, deadline: float, should_stop: Callable[[], bool]) -> MembershipStep:+ wait_for_cancellation(should_stop)+ return MembershipStep(status="backfilling")++ def sync_page(service: MembershipBackfillService, _after: int, *, dry_run: bool) -> MembershipSyncPage:+ assert service.should_stop is not None and service.deadline is not None+ wait_for_cancellation(service.should_stop)+ return MembershipSyncPage(teams=(), next_team_id=None, candidates=0, enabled=0, changed=0)++ def prepare(service: MembershipBackfillService, input: MembershipTeam) -> MembershipTeam:+ assert service.should_stop is not None and service.deadline is not None+ wait_for_cancellation(service.should_stop)+ return input++ env = ActivityEnvironment()++ async def run_activity() -> None:+ if operation == "batch":+ await env.run(backfill_membership_batch_activity, MembershipTeam(team_id=17))+ elif operation == "sync":+ await env.run(sync_membership_page_activity, MembershipCoordinatorInput())+ else:+ await env.run(prepare_membership_backfill_activity, MembershipTeam(team_id=17))++ with (+ patch.object(MembershipBackfillService, "process_activity_batch", side_effect=process_batch),+ patch.object(MembershipBackfillService, "sync_page", autospec=True, side_effect=sync_page),+ patch.object(MembershipBackfillService, "prepare_backfill", autospec=True, side_effect=prepare),+ ):+ task = asyncio.create_task(run_activity())+ try:+ await asyncio.wait_for(started.wait(), timeout=10)+ env.cancel()+ with pytest.raises(asyncio.CancelledError):+ await task+ finally:+ release.set()+ await asyncio.wait_for(checked.wait(), timeout=10)+ assert stopped == [True]+++async def test_monitoring_logs_use_run_counters_without_account_keys() -> None:+ cursor = MembershipCursor(+ team_id=17,+ version=1,+ group_type_index=0,+ accounts_completed=2,+ windows_completed=3,+ rows_written=7,+ rows_found=9,+ )+ observation = MembershipObservation(+ input=MembershipTeam(team_id=17, account_keys=("example-account",), cursor=cursor),+ run_id="example-run",+ event="progress",+ status="backfilling",+ )+ with capture_logs() as logs:+ await ActivityEnvironment().run(observe_membership_backfill_activity, observation)+ assert logs[0]["event"] == "customer_analytics.membership_backfill"+ assert logs[0]["backfill_run_id"] == "example-run"+ assert logs[0]["account_scope"] == "selected"+ assert logs[0]["accounts_completed"] == 2+ assert logs[0]["windows_completed"] == 3+ assert logs[0]["rows_written"] == 7+ assert "example-account" not in str(logs)diff --git a/products/customer_analytics/backend/test/test_backfill_memberships_command.py b/products/customer_analytics/backend/test/test_backfill_memberships_command.pynew file mode 100644index 000000000000..128766ce2eb4--- /dev/null+++ b/products/customer_analytics/backend/test/test_backfill_memberships_command.py@@ -0,0 +1,45 @@+import io+import json++from unittest.mock import AsyncMock, patch++from django.core.management import call_command+from django.test import SimpleTestCase++from parameterized import parameterized++from products.customer_analytics.backend.logic.membership_backfill import MembershipTeam++COMMAND = "products.customer_analytics.backend.management.commands.backfill_memberships"+CONTROL = "products.customer_analytics.backend.logic.membership_backfill_control"+++class TestBackfillMembershipsCommand(SimpleTestCase):+ @parameterized.expand(+ [+ (scope, accounts, dry_run)+ for scope, accounts in (+ (["--all-accounts"], ()),+ (["--account", "b", "--account", "a", "--account", "b"], ("a", "b")),+ )+ for dry_run in (False, True)+ ]+ )+ def test_start_preserves_account_scope(+ self, scope: list[str], expected_accounts: tuple[str, ...], dry_run: bool+ ) -> None:+ client = AsyncMock()+ client.start_workflow.return_value.first_execution_run_id = "example-run"+ output = io.StringIO()+ with (+ patch(f"{COMMAND}.Team.objects.filter"),+ patch(f"{COMMAND}.get_membership_group_type_index", return_value=0),+ patch(f"{CONTROL}.async_connect", return_value=client),+ ):+ call_command("backfill_memberships", "start", "--team-id", "17", *scope, dry_run=dry_run, stdout=output)+ assert json.loads(output.getvalue())["run_id"] == "example-run"+ assert client.start_workflow.call_args.args[1] == MembershipTeam(+ team_id=17,+ account_keys=expected_accounts,+ dry_run=dry_run,+ )