Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(customer-analytics): add membership backfill controls (#124295)

Arthur Moreira de Deuse75ecae4952a
diff --git a/posthog/temporal/schedule.py b/posthog/temporal/schedule.py
index 318179d19ac7..bf044637a667 100644
--- a/posthog/temporal/schedule.py
+++ b/posthog/temporal/schedule.py
@@ -96,6 +96,7 @@ from products.conversations.backend.temporal.schedule import create_support_repl
from products.customer_analytics.backend.facade.temporal import (
create_account_track_rule_coordinator_schedule,
create_calendar_sync_coordinator_schedule,
+ create_membership_coordinator_schedule,
create_ownership_claims_coordinator_schedule,
)
from products.data_catalog.backend.facade.temporal import create_data_catalog_weekly_digest_schedule
@@ -1021,6 +1022,7 @@ schedules = [
create_account_track_rule_coordinator_schedule,
create_calendar_sync_coordinator_schedule,
create_ownership_claims_coordinator_schedule,
+ create_membership_coordinator_schedule,
create_replay_vision_reconciler_schedule,
create_replay_vision_estimates_schedule,
cleanup_replay_vision_search_suggestions_schedule,
diff --git a/products/customer_analytics/backend/facade/temporal.py b/products/customer_analytics/backend/facade/temporal.py
index d61e48b859c1..ac8ee3e8d68e 100644
--- a/products/customer_analytics/backend/facade/temporal.py
+++ b/products/customer_analytics/backend/facade/temporal.py
@@ -12,6 +12,7 @@ from products.customer_analytics.backend.temporal.account_track_rules import (
create_account_track_rule_coordinator_schedule,
)
from products.customer_analytics.backend.temporal.calendar_sync import create_calendar_sync_coordinator_schedule
+from products.customer_analytics.backend.temporal.membership_backfill import create_membership_coordinator_schedule
from products.customer_analytics.backend.temporal.ownership_claims import create_ownership_claims_coordinator_schedule
__all__ = [
@@ -23,5 +24,6 @@ __all__ = [
"create_account_track_rule_coordinator_schedule",
"create_calendar_sync_coordinator_schedule",
"create_ownership_claims_coordinator_schedule",
+ "create_membership_coordinator_schedule",
"stage_warehouse_account_property_files_activity",
]
diff --git a/products/customer_analytics/backend/logic/membership_backfill_control.py b/products/customer_analytics/backend/logic/membership_backfill_control.py
new file mode 100644
index 000000000000..066ae0fe27fe
--- /dev/null
+++ b/products/customer_analytics/backend/logic/membership_backfill_control.py
@@ -0,0 +1,48 @@
+from django.conf import settings
+
+from temporalio.client import Client
+from temporalio.common import RetryPolicy, WorkflowIDReusePolicy
+
+from posthog.temporal.common.client import async_connect
+
+from products.customer_analytics.backend.logic.membership_backfill import MembershipTeam
+from products.customer_analytics.backend.temporal.membership_backfill import (
+ MembershipBackfillWorkflow,
+ MembershipRunStatus,
+ get_membership_workflow_id,
+)
+
+
+class MembershipBackfillControl:
+ def __init__(self, client: Client) -> None:
+ self.client = client
+
+ @classmethod
+ async def connect(cls) -> "MembershipBackfillControl":
+ return cls(await async_connect())
+
+ async def start(self, team_id: int, *, account_keys: tuple[str, ...], dry_run: bool) -> str:
+ handle = await self.client.start_workflow(
+ MembershipBackfillWorkflow.run,
+ MembershipTeam(team_id=team_id, account_keys=account_keys, dry_run=dry_run),
+ id=get_membership_workflow_id(team_id),
+ id_reuse_policy=WorkflowIDReusePolicy.ALLOW_DUPLICATE,
+ task_queue=settings.VIDEO_EXPORT_TASK_QUEUE,
+ retry_policy=RetryPolicy(maximum_attempts=1),
+ )
+ return handle.first_execution_run_id or ""
+
+ async def get_status(self, team_id: int) -> MembershipRunStatus:
+ handle = self.client.get_workflow_handle(get_membership_workflow_id(team_id))
+ return await handle.query(MembershipBackfillWorkflow.get_status)
+
+ async def resume(self, team_id: int) -> None:
+ status = await self.get_status(team_id)
+ if status.status != "failed":
+ raise ValueError("Only a failed backfill can resume")
+ handle = self.client.get_workflow_handle(get_membership_workflow_id(team_id))
+ await handle.signal(MembershipBackfillWorkflow.resume)
+
+ async def cancel(self, team_id: int) -> None:
+ handle = self.client.get_workflow_handle(get_membership_workflow_id(team_id))
+ await handle.signal(MembershipBackfillWorkflow.cancel)
diff --git a/products/customer_analytics/backend/management/commands/backfill_memberships.py b/products/customer_analytics/backend/management/commands/backfill_memberships.py
new file mode 100644
index 000000000000..31dc528fb76f
--- /dev/null
+++ b/products/customer_analytics/backend/management/commands/backfill_memberships.py
@@ -0,0 +1,70 @@
+import json
+import asyncio
+from dataclasses import asdict
+from typing import cast
+
+from django.core.management.base import BaseCommand, CommandError, CommandParser
+
+from temporalio.exceptions import WorkflowAlreadyStartedError
+from temporalio.service import RPCError
+
+from posthog.models.team import Team
+
+from products.customer_analytics.backend.logic.membership_backfill import get_membership_group_type_index
+from products.customer_analytics.backend.logic.membership_backfill_control import MembershipBackfillControl
+
+
+class Command(BaseCommand):
+ help = "Start, inspect, resume, or cancel an account membership backfill"
+
+ def add_arguments(self, parser: CommandParser) -> None:
+ commands = parser.add_subparsers(dest="action", required=True)
+ for action in ("start", "status", "resume", "cancel"):
+ command = commands.add_parser(action)
+ command.add_argument("--team-id", type=int, required=True)
+ if action == "start":
+ scope = command.add_mutually_exclusive_group(required=True)
+ scope.add_argument("--all-accounts", action="store_true")
+ scope.add_argument("--account", action="append", dest="accounts")
+ command.add_argument("--dry-run", action="store_true")
+
+ def handle(self, *args: object, **options: object) -> None:
+ team_id = cast(int, options["team_id"])
+ action = cast(str, options["action"])
+ accounts = tuple(sorted(set(cast(list[str], options.get("accounts") or []))))
+ dry_run = bool(options.get("dry_run"))
+ if action == "start":
+ if any(not key or len(key.encode("utf-8")) > 1024 for key in accounts) or len(accounts) > 100:
+ raise CommandError("Supply at most 100 nonempty account keys, each within 1024 bytes")
+ team = Team.objects.filter(id=team_id).first()
+ if team is None:
+ raise CommandError("Project does not exist")
+ try:
+ index = get_membership_group_type_index(team)
+ except RuntimeError as error:
+ raise CommandError(str(error)) from error
+ if index is None:
+ raise CommandError("Customer analytics must be active with a valid account group type")
+ try:
+ output = asyncio.run(self.get_output(action, team_id, accounts, dry_run=dry_run))
+ except WorkflowAlreadyStartedError as error:
+ raise CommandError(
+ "A backfill is already active for this project. Use status, resume, or cancel"
+ ) from error
+ except (RPCError, ValueError) as error:
+ raise CommandError(str(error)) from error
+ self.stdout.write(output)
+
+ @staticmethod
+ async def get_output(action: str, team_id: int, accounts: tuple[str, ...], *, dry_run: bool) -> str:
+ control = await MembershipBackfillControl.connect()
+ if action == "start":
+ run_id = await control.start(team_id, account_keys=accounts, dry_run=dry_run)
+ return json.dumps({"team_id": team_id, "run_id": run_id, "status": "starting", "dry_run": dry_run})
+ if action == "status":
+ return json.dumps(asdict(await control.get_status(team_id)), default=str)
+ if action == "resume":
+ await control.resume(team_id)
+ return "Resume requested"
+ await control.cancel(team_id)
+ return "Cancel requested. The current batch can finish before the backfill stops"
diff --git a/products/customer_analytics/backend/metrics.py b/products/customer_analytics/backend/metrics.py
index 1db8d4e19b2b..20c2ae89ba92 100644
--- a/products/customer_analytics/backend/metrics.py
+++ b/products/customer_analytics/backend/metrics.py
@@ -3,6 +3,24 @@ from prometheus_client import Counter, Gauge, Histogram
from posthog.otel_metrics import OtelInstrumentFactory
+
+def record_membership_operation(
+ operation: str, outcome: str, count: int = 1, *, rows: int = 0, lag_seconds: float | None = None
+) -> None:
+ client = posthoganalytics.default_client
+ if client is None:
+ return
+ try:
+ client.metrics.count(
+ "customer_analytics.membership.operations", count, attributes={"operation": operation, "outcome": outcome}
+ )
+ client.metrics.count("customer_analytics.membership.rows", rows)
+ if lag_seconds is not None:
+ client.metrics.histogram("customer_analytics.membership.lag", lag_seconds, unit="s")
+ except Exception:
+ pass
+
+
_ACCOUNT_TRACK_RULE_RUNS = Counter(
"customer_analytics_account_track_rule_runs_total",
"Account Track Rule runs by trigger and terminal status",
diff --git a/products/customer_analytics/backend/metrics_test.py b/products/customer_analytics/backend/metrics_test.py
index 32ec1b411259..b16fed047cc4 100644
--- a/products/customer_analytics/backend/metrics_test.py
+++ b/products/customer_analytics/backend/metrics_test.py
@@ -1,6 +1,21 @@
+import pytest
from unittest.mock import MagicMock, patch
-from products.customer_analytics.backend.metrics import record_account_property_sync_phase_duration
+from products.customer_analytics.backend.metrics import (
+ record_account_property_sync_phase_duration,
+ record_membership_operation,
+)
+
+
[email protected]("lag_seconds", [None, 0, 12.5])
+def test_membership_lag_records_only_measured_values(lag_seconds: float | None) -> None:
+ client = MagicMock()
+ with patch("products.customer_analytics.backend.metrics.posthoganalytics.default_client", client):
+ record_membership_operation("backfill", "catching_up", lag_seconds=lag_seconds)
+ if lag_seconds is None:
+ client.metrics.histogram.assert_not_called()
+ else:
+ client.metrics.histogram.assert_called_once_with("customer_analytics.membership.lag", lag_seconds, unit="s")
def test_account_property_sync_phase_duration_records_posthog_metric() -> None:
diff --git a/products/customer_analytics/backend/temporal/__init__.py b/products/customer_analytics/backend/temporal/__init__.py
index 9d5703141314..ebeb6eb6760c 100644
--- a/products/customer_analytics/backend/temporal/__init__.py
+++ b/products/customer_analytics/backend/temporal/__init__.py
@@ -16,6 +16,15 @@ from products.customer_analytics.backend.temporal.calendar_sync import (
calendar_sync_integration_activity,
google_account_email_backfill_activity,
)
+from products.customer_analytics.backend.temporal.membership_backfill import (
+ MembershipBackfillWorkflow,
+ MembershipCoordinatorWorkflow,
+ backfill_membership_batch_activity,
+ fail_membership_backfill_activity,
+ observe_membership_backfill_activity,
+ prepare_membership_backfill_activity,
+ sync_membership_page_activity,
+)
from products.customer_analytics.backend.temporal.ownership_claims import (
OwnershipClaimsCoordinatorWorkflow,
OwnershipClaimsSweepWorkflow,
@@ -24,6 +33,8 @@ from products.customer_analytics.backend.temporal.ownership_claims import (
)
WORKFLOWS = [
+ MembershipBackfillWorkflow,
+ MembershipCoordinatorWorkflow,
AccountTrackRuleCoordinatorWorkflow,
AccountTrackRuleEvaluationWorkflow,
CalendarSyncCoordinatorWorkflow,
@@ -33,6 +44,11 @@ WORKFLOWS = [
OwnershipClaimsSweepWorkflow,
]
ACTIVITIES = [
+ backfill_membership_batch_activity,
+ fail_membership_backfill_activity,
+ sync_membership_page_activity,
+ observe_membership_backfill_activity,
+ prepare_membership_backfill_activity,
account_track_rule_collect_configs_activity,
account_track_rule_create_scheduled_run_activity,
account_track_rule_fail_run_activity,
diff --git a/products/customer_analytics/backend/temporal/membership_backfill.py b/products/customer_analytics/backend/temporal/membership_backfill.py
new file mode 100644
index 000000000000..299a6c5243bf
--- /dev/null
+++ b/products/customer_analytics/backend/temporal/membership_backfill.py
@@ -0,0 +1,379 @@
+import json
+from dataclasses import replace
+from datetime import timedelta
+from threading import Event
+from time import monotonic
+
+from temporalio import activity, workflow
+from temporalio.client import (
+ Client,
+ Schedule,
+ ScheduleActionStartWorkflow,
+ ScheduleIntervalSpec,
+ ScheduleOverlapPolicy,
+ SchedulePolicy,
+ ScheduleSpec,
+ ScheduleState,
+)
+from temporalio.common import RetryPolicy, WorkflowIDReusePolicy
+from temporalio.exceptions import ActivityError, ApplicationError, WorkflowAlreadyStartedError
+
+with workflow.unsafe.imports_passed_through():
+ from django.conf import settings
+
+ import structlog
+
+ from posthog.dataclasses import frozen
+ from posthog.scheduling.jitter import deterministic_offset
+ from posthog.sync import database_sync_to_async
+ from posthog.temporal.common.heartbeat import Heartbeater
+ from posthog.temporal.common.schedule import a_create_schedule, a_schedule_exists, a_update_schedule
+
+ from products.customer_analytics.backend.logic.membership_backfill import (
+ MembershipBackfillService,
+ MembershipCursor,
+ MembershipStep,
+ MembershipSyncPage,
+ MembershipTeam,
+ )
+ from products.customer_analytics.backend.metrics import record_membership_operation
+
+COORDINATOR_NAME = "customer-analytics-membership-coordinator"
+BACKFILL_NAME = "customer-analytics-membership-backfill"
+SCHEDULE_ID = "customer-analytics-membership-coordinator-schedule"
+LOGGER = structlog.get_logger(__name__)
+
+
+@frozen
+class MembershipCoordinatorInput:
+ after_team_id: int = 0
+ dry_run: bool = True
+
+
+@frozen
+class MembershipFailure:
+ team_id: int
+ version: int
+ error_type: str
+ account_keys: tuple[str, ...] = ()
+ dry_run: bool = False
+
+
+@frozen
+class MembershipObservation:
+ input: MembershipTeam
+ run_id: str
+ event: str
+ status: str
+ error_type: str = ""
+
+
+@frozen
+class MembershipRunStatus:
+ team_id: int
+ run_id: str
+ status: str
+ dry_run: bool
+ account_count: int
+ cursor: MembershipCursor | None
+ error_type: str = ""
+
+
+def get_membership_workflow_id(team_id: int) -> str:
+ return f"{BACKFILL_NAME}-{team_id}"
+
+
+async def sync_membership_page_activity(input: MembershipCoordinatorInput) -> MembershipSyncPage:
+ stopped = Event()
+ service = MembershipBackfillService(deadline=monotonic() + 585, should_stop=stopped.is_set)
+ try:
+ async with Heartbeater():
+ page = await database_sync_to_async(service.sync_page, thread_sensitive=False)(
+ input.after_team_id, dry_run=input.dry_run
+ )
+ record_membership_operation("sync", "candidates", page.candidates)
+ record_membership_operation("sync", "enabled", page.enabled)
+ record_membership_operation("sync", "changed", page.changed)
+ return page
+ except Exception:
+ record_membership_operation("sync", "failed")
+ raise
+ finally:
+ stopped.set()
+
+
+async def prepare_membership_backfill_activity(input: MembershipTeam) -> MembershipTeam:
+ stopped = Event()
+ service = MembershipBackfillService(deadline=monotonic() + 105, should_stop=stopped.is_set)
+ try:
+ async with Heartbeater():
+ return await database_sync_to_async(service.prepare_backfill, thread_sensitive=False)(input)
+ finally:
+ stopped.set()
+
+
+async def backfill_membership_batch_activity(input: MembershipTeam) -> MembershipStep:
+ started = monotonic()
+ stopped = Event()
+ try:
+ async with Heartbeater():
+ result = await database_sync_to_async(
+ MembershipBackfillService.process_activity_batch, thread_sensitive=False
+ )(input, deadline=started + 285, should_stop=stopped.is_set)
+ if result.cursor:
+ result.cursor.last_batch_duration_seconds = monotonic() - started
+ if result.lag_seconds is not None:
+ result.cursor.lag_seconds = result.lag_seconds
+ record_membership_operation("backfill", result.status, rows=result.rows, lag_seconds=result.lag_seconds)
+ return result
+ except Exception:
+ record_membership_operation("backfill", "failed")
+ raise
+ finally:
+ stopped.set()
+
+
+async def fail_membership_backfill_activity(input: MembershipFailure) -> None:
+ if not input.account_keys and not input.dry_run:
+ await database_sync_to_async(MembershipBackfillService.fail, thread_sensitive=False)(
+ input.team_id, input.version, input.error_type
+ )
+
+
+async def observe_membership_backfill_activity(observation: MembershipObservation) -> None:
+ input = observation.input
+ cursor = input.cursor
+ duration = (
+ (activity.info().current_attempt_scheduled_time - cursor.started_at).total_seconds()
+ if cursor and cursor.started_at
+ else 0
+ )
+ LOGGER.info(
+ "customer_analytics.membership_backfill",
+ backfill_event=observation.event,
+ backfill_run_id=observation.run_id,
+ team_id=input.team_id,
+ status=observation.status,
+ dry_run=input.dry_run,
+ account_scope="selected" if input.account_keys else "all",
+ account_count=len(input.account_keys),
+ accounts_completed=cursor.accounts_completed if cursor else 0,
+ windows_completed=cursor.windows_completed if cursor else 0,
+ rows_written=cursor.rows_written if cursor else 0,
+ rows_found=cursor.rows_found if cursor else 0,
+ duration_seconds=max(0, duration),
+ batch_duration_seconds=cursor.last_batch_duration_seconds if cursor else 0,
+ lag_seconds=cursor.lag_seconds if cursor else 0,
+ error_type=observation.error_type,
+ )
+
+
[email protected](name=BACKFILL_NAME)
+class MembershipBackfillWorkflow:
+ def __init__(self) -> None:
+ self.input = MembershipTeam(team_id=0)
+ self.status = "starting"
+ self.error_type = ""
+ self.resume_requested = False
+ self.cancel_requested = False
+ self.run_id = ""
+
+ @staticmethod
+ def parse_inputs(inputs: list[str]) -> MembershipTeam:
+ return MembershipTeam(**json.loads(inputs[0]))
+
+ @workflow.query
+ def get_status(self) -> MembershipRunStatus:
+ return MembershipRunStatus(
+ team_id=self.input.team_id,
+ run_id=self.run_id,
+ status=self.status,
+ dry_run=self.input.dry_run,
+ account_count=len(self.input.account_keys),
+ cursor=self.input.cursor,
+ error_type=self.error_type,
+ )
+
+ @workflow.signal
+ def resume(self) -> None:
+ if self.status == "failed":
+ self.resume_requested = True
+
+ @workflow.signal
+ def cancel(self) -> None:
+ self.cancel_requested = True
+
+ async def observe(self, event: str) -> None:
+ try:
+ await workflow.execute_activity(
+ observe_membership_backfill_activity,
+ MembershipObservation(
+ input=self.input, run_id=self.run_id, event=event, status=self.status, error_type=self.error_type
+ ),
+ start_to_close_timeout=timedelta(seconds=30),
+ retry_policy=RetryPolicy(maximum_attempts=3),
+ )
+ except ActivityError:
+ workflow.logger.warning("Membership backfill observation failed", extra={"backfill_event": event})
+
+ async def process_batch(self) -> MembershipStep:
+ if not self.input.cursor and not self.input.version and not self.input.dry_run:
+ self.input = await workflow.execute_activity(
+ prepare_membership_backfill_activity,
+ self.input,
+ start_to_close_timeout=timedelta(minutes=2),
+ heartbeat_timeout=timedelta(minutes=1),
+ retry_policy=RetryPolicy(maximum_attempts=3),
+ )
+ result = await workflow.execute_activity(
+ backfill_membership_batch_activity,
+ self.input,
+ start_to_close_timeout=timedelta(minutes=5),
+ heartbeat_timeout=timedelta(minutes=1),
+ retry_policy=RetryPolicy(maximum_attempts=3),
+ )
+ self.input = replace(self.input, cursor=result.cursor)
+ self.status = result.status
+ self.error_type = ""
+ await self.observe("progress")
+ return replace(result, status="cancelled") if self.cancel_requested else result
+
+ @workflow.run
+ async def run(self, input: MembershipTeam) -> str:
+ if workflow.info().workflow_id != get_membership_workflow_id(input.team_id):
+ raise ApplicationError("Use the team membership workflow ID", non_retryable=True)
+ self.input = input
+ self.run_id = workflow.info().first_execution_run_id
+ self.status = input.cursor.status if input.cursor else "starting"
+ if input.cursor is None:
+ await self.observe("started")
+ waiting_since = None
+ for _ in range(100):
+ if self.cancel_requested:
+ break
+ try:
+ result = await self.process_batch()
+ if result.status == "cancelled":
+ break
+ if result.status in ("ready", "disabled", "completed"):
+ await self.observe("completed")
+ return result.status
+ if result.status == "waiting_for_dictionary":
+ waiting_since = waiting_since or workflow.now()
+ if workflow.now() - waiting_since >= timedelta(minutes=30):
+ raise ApplicationError("Membership dictionary did not refresh")
+ await workflow.wait_condition(lambda: self.cancel_requested, timeout=timedelta(seconds=30))
+ else:
+ waiting_since = None
+ except TimeoutError:
+ continue
+ except Exception as error:
+ self.status = "failed"
+ cause = error.__cause__ or error
+ self.error_type = (
+ cause.type if isinstance(cause, ApplicationError) and cause.type else type(cause).__name__
+ )
+ try:
+ await workflow.execute_activity(
+ fail_membership_backfill_activity,
+ MembershipFailure(
+ team_id=input.team_id,
+ version=self.input.cursor.version if self.input.cursor else self.input.version,
+ error_type=self.error_type,
+ account_keys=input.account_keys,
+ dry_run=input.dry_run,
+ ),
+ start_to_close_timeout=timedelta(minutes=1),
+ retry_policy=RetryPolicy(maximum_attempts=3),
+ )
+ except ActivityError:
+ workflow.logger.warning("Membership backfill failure update failed")
+ await self.observe("failed")
+ await workflow.wait_condition(lambda: self.resume_requested or self.cancel_requested)
+ self.resume_requested = False
+ waiting_since = None
+ if not self.cancel_requested:
+ self.status = "resuming"
+ await self.observe("resumed")
+ if self.cancel_requested:
+ self.status = "cancelled"
+ await self.observe("cancelled")
+ return self.status
+ return workflow.continue_as_new(self.input)
+
+
[email protected](name=COORDINATOR_NAME)
+class MembershipCoordinatorWorkflow:
+ @staticmethod
+ def parse_inputs(inputs: list[str]) -> MembershipCoordinatorInput:
+ return MembershipCoordinatorInput(**json.loads(inputs[0])) if inputs else MembershipCoordinatorInput()
+
+ @workflow.run
+ async def run(self, input: MembershipCoordinatorInput) -> None:
+ after = input.after_team_id
+ for _ in range(50):
+ page = await workflow.execute_activity(
+ sync_membership_page_activity,
+ MembershipCoordinatorInput(after_team_id=after, dry_run=input.dry_run),
+ start_to_close_timeout=timedelta(minutes=10),
+ heartbeat_timeout=timedelta(minutes=1),
+ retry_policy=RetryPolicy(maximum_attempts=3),
+ )
+ for team in page.teams:
+ try:
+ await workflow.start_child_workflow(
+ MembershipBackfillWorkflow.run,
+ team,
+ id=get_membership_workflow_id(team.team_id),
+ id_reuse_policy=WorkflowIDReusePolicy.ALLOW_DUPLICATE,
+ parent_close_policy=workflow.ParentClosePolicy.ABANDON,
+ retry_policy=RetryPolicy(maximum_attempts=1),
+ )
+ except WorkflowAlreadyStartedError:
+ pass
+ if page.next_team_id is None:
+ return
+ after = page.next_team_id
+ workflow.continue_as_new(MembershipCoordinatorInput(after_team_id=after, dry_run=input.dry_run))
+
+
+def create_membership_schedule(state: ScheduleState) -> Schedule:
+ return Schedule(
+ action=ScheduleActionStartWorkflow(
+ COORDINATOR_NAME,
+ MembershipCoordinatorInput(dry_run=False),
+ id=COORDINATOR_NAME,
+ task_queue=settings.VIDEO_EXPORT_TASK_QUEUE,
+ execution_timeout=timedelta(hours=24),
+ retry_policy=RetryPolicy(maximum_attempts=1),
+ ),
+ spec=ScheduleSpec(
+ intervals=[
+ ScheduleIntervalSpec(
+ every=timedelta(hours=1), offset=deterministic_offset(SCHEDULE_ID, timedelta(hours=1))
+ )
+ ]
+ ),
+ state=state,
+ policy=SchedulePolicy(overlap=ScheduleOverlapPolicy.SKIP),
+ )
+
+
+async def create_membership_coordinator_schedule(client: Client) -> None:
+ if await a_schedule_exists(client, SCHEDULE_ID):
+ description = await client.get_schedule_handle(SCHEDULE_ID).describe()
+ await a_update_schedule(
+ client, SCHEDULE_ID, create_membership_schedule(description.schedule.state), keep_paused=True
+ )
+ else:
+ await a_create_schedule(
+ client,
+ SCHEDULE_ID,
+ create_membership_schedule(ScheduleState(paused=True, note="Enable after live ingestion is ready")),
+ trigger_immediately=False,
+ )
diff --git a/products/customer_analytics/backend/temporal/membership_backfill_test.py b/products/customer_analytics/backend/temporal/membership_backfill_test.py
new file mode 100644
index 000000000000..c58f6d0b96be
--- /dev/null
+++ b/products/customer_analytics/backend/temporal/membership_backfill_test.py
@@ -0,0 +1,345 @@
+import asyncio
+import logging
+from collections.abc import Callable
+from dataclasses import replace
+from datetime import UTC, datetime, timedelta
+from threading import Event
+from uuid import uuid4
+
+import pytest
+from unittest.mock import AsyncMock, Mock, patch
+
+from structlog.testing import capture_logs
+from temporalio import activity
+from temporalio.api.enums.v1 import EventType
+from temporalio.client import ScheduleActionStartWorkflow, ScheduleState
+from temporalio.converter import DataConverter
+from temporalio.exceptions import ApplicationError
+from temporalio.testing import ActivityEnvironment, WorkflowEnvironment
+from temporalio.worker import Replayer, Worker
+
+from products.customer_analytics.backend.logic.membership_backfill import (
+ PAGE_SIZE,
+ MembershipBackfillService,
+ MembershipCursor,
+ MembershipStep,
+ MembershipSyncPage,
+ MembershipTeam,
+)
+from products.customer_analytics.backend.temporal.membership_backfill import (
+ MembershipBackfillWorkflow,
+ MembershipCoordinatorInput,
+ MembershipCoordinatorWorkflow,
+ MembershipFailure,
+ MembershipObservation,
+ backfill_membership_batch_activity,
+ create_membership_coordinator_schedule,
+ create_membership_schedule,
+ get_membership_workflow_id,
+ observe_membership_backfill_activity,
+ prepare_membership_backfill_activity,
+ sync_membership_page_activity,
+)
+
+
+async def test_largest_coordinator_payload_contains_only_bounded_team_references() -> None:
+ page = MembershipSyncPage(
+ teams=tuple(MembershipTeam(team_id=team_id, version=2**63) for team_id in range(PAGE_SIZE)),
+ next_team_id=PAGE_SIZE,
+ candidates=PAGE_SIZE,
+ enabled=PAGE_SIZE,
+ changed=PAGE_SIZE,
+ )
+ payloads = await DataConverter.default.encode([page])
+ assert sum(len(payload.data) for payload in payloads) < 16_384
+ assert await DataConverter.default.decode(payloads, [MembershipSyncPage]) == [page]
+
+
+def test_schedule_preserves_pause_and_runs_with_writes() -> None:
+ state = ScheduleState(paused=True)
+ schedule = create_membership_schedule(state)
+ assert schedule.state.paused
+ assert isinstance(schedule.action, ScheduleActionStartWorkflow)
+ assert schedule.action.args == [MembershipCoordinatorInput(dry_run=False)]
+ assert not create_membership_schedule(ScheduleState(paused=False)).state.paused
+
+
+async def test_schedule_update_keeps_an_operator_pause() -> None:
+ client = AsyncMock()
+ description = client.get_schedule_handle.return_value.describe.return_value
+ description.schedule = create_membership_schedule(ScheduleState(paused=False))
+ client.get_schedule_handle = Mock(return_value=client.get_schedule_handle.return_value)
+ with patch("products.customer_analytics.backend.temporal.membership_backfill.a_schedule_exists", return_value=True):
+ await create_membership_coordinator_schedule(client)
+ updater = client.get_schedule_handle.return_value.update.call_args.kwargs["updater"]
+ current = Mock(schedule=create_membership_schedule(ScheduleState(paused=True, note="Operator paused")))
+ update_input = Mock(description=current)
+ update = await updater(update_input)
+ assert update.schedule.state.paused
+ assert update.schedule.state.note == "Operator paused"
+
+
+async def test_retry_overlap_worker_restart_and_replay() -> None:
+ logging.getLogger("temporalio.activity").setLevel(logging.INFO)
+ logging.getLogger("temporalio.workflow").setLevel(logging.INFO)
+ started = asyncio.Event()
+ attempts = 0
+ team = MembershipTeam(team_id=17, version=1)
+
+ @activity.defn(name="sync_membership_page_activity")
+ async def sync_page(_input: MembershipCoordinatorInput) -> MembershipSyncPage:
+ return MembershipSyncPage(teams=(team,), next_team_id=None, candidates=1, enabled=1, changed=1)
+
+ @activity.defn(name="backfill_membership_batch_activity")
+ async def backfill(_input: MembershipTeam) -> MembershipStep:
+ nonlocal attempts
+ attempts += 1
+ if attempts == 1:
+ raise ApplicationError("Retry this batch")
+ if attempts == 2:
+ started.set()
+ return MembershipStep(status="waiting_for_dictionary")
+ return MembershipStep(status="ready", rows=1)
+
+ @activity.defn(name="fail_membership_backfill_activity")
+ async def fail(_input: MembershipFailure) -> None:
+ raise AssertionError("The retried batch must succeed")
+
+ observations: list[str] = []
+
+ @activity.defn(name="observe_membership_backfill_activity")
+ async def observe(input: MembershipObservation) -> None:
+ observations.append(input.event)
+
+ queue = f"membership-test-{uuid4()}"
+ async with await WorkflowEnvironment.start_time_skipping() as env:
+ with env.auto_time_skipping_disabled():
+ async with Worker(
+ env.client,
+ task_queue=queue,
+ workflows=[MembershipCoordinatorWorkflow, MembershipBackfillWorkflow],
+ activities=[sync_page, backfill, fail, observe],
+ max_cached_workflows=0,
+ ):
+ await env.client.execute_workflow(
+ MembershipCoordinatorWorkflow.run,
+ MembershipCoordinatorInput(dry_run=False),
+ id=f"coordinator-{uuid4()}",
+ task_queue=queue,
+ execution_timeout=timedelta(minutes=5),
+ )
+ await asyncio.wait_for(started.wait(), timeout=30)
+ await env.client.execute_workflow(
+ MembershipCoordinatorWorkflow.run,
+ MembershipCoordinatorInput(dry_run=False),
+ id=f"coordinator-{uuid4()}",
+ task_queue=queue,
+ execution_timeout=timedelta(minutes=5),
+ )
+ workflow_id = get_membership_workflow_id(team.team_id)
+ handle = env.client.get_workflow_handle(workflow_id)
+ handle = env.client.get_workflow_handle(workflow_id, run_id=(await handle.describe()).run_id)
+ for _ in range(100):
+ history = await handle.fetch_history()
+ if any(event.event_type == EventType.EVENT_TYPE_TIMER_STARTED for event in history.events):
+ break
+ await asyncio.sleep(0.01)
+ else:
+ raise AssertionError("The workflow did not start its dictionary timer")
+ async with Worker(
+ env.client,
+ task_queue=queue,
+ workflows=[MembershipCoordinatorWorkflow, MembershipBackfillWorkflow],
+ activities=[sync_page, backfill, fail, observe],
+ ):
+ await env.sleep(timedelta(seconds=31))
+ assert await asyncio.wait_for(handle.result(), timeout=30) == "ready"
+ history = await handle.fetch_history()
+ assert sum(event.event_type == EventType.EVENT_TYPE_WORKFLOW_EXECUTION_STARTED for event in history.events) == 1
+ assert attempts == 3
+ assert observations.count("started") == 1
+ assert observations[-1] == "completed"
+ await Replayer(workflows=[MembershipBackfillWorkflow]).replay_workflow(history)
+
+
+ "has_cursor,bookkeeping_fails,observation_fails",
+ [(True, False, False), (True, True, True), (False, False, False)],
+)
+async def test_failed_run_keeps_its_cursor_for_resume_and_can_be_cancelled(
+ has_cursor: bool, bookkeeping_fails: bool, observation_fails: bool
+) -> None:
+ logging.getLogger("temporalio.activity").setLevel(logging.INFO)
+ logging.getLogger("temporalio.workflow").setLevel(logging.INFO)
+ observations: asyncio.Queue[MembershipObservation] = asyncio.Queue()
+ failures: list[MembershipFailure] = []
+ account_keys = ("account-a",) if has_cursor else ()
+ failed = True
+
+ @activity.defn(name="prepare_membership_backfill_activity")
+ async def prepare(input: MembershipTeam) -> MembershipTeam:
+ return replace(input, version=7)
+
+ @activity.defn(name="backfill_membership_batch_activity")
+ async def backfill(input: MembershipTeam) -> MembershipStep:
+ assert (input.cursor.version if input.cursor else input.version) == 7
+ if failed:
+ raise ApplicationError("Unavailable", type="ExampleUnavailable", non_retryable=True)
+ assert input.account_keys == account_keys
+ return MembershipStep(status="waiting_for_dictionary", cursor=input.cursor)
+
+ @activity.defn(name="fail_membership_backfill_activity")
+ async def fail(input: MembershipFailure) -> None:
+ failures.append(input)
+ if bookkeeping_fails:
+ raise ApplicationError("Unavailable", non_retryable=True)
+
+ @activity.defn(name="observe_membership_backfill_activity")
+ async def observe(input: MembershipObservation) -> None:
+ await observations.put(input)
+ if observation_fails and input.event == "failed":
+ raise ApplicationError("Unavailable", non_retryable=True)
+
+ async def wait_for_event(event: str) -> MembershipObservation:
+ while True:
+ observation = await asyncio.wait_for(observations.get(), timeout=30)
+ if observation.event == event:
+ return observation
+
+ queue = f"membership-test-{uuid4()}"
+ async with await WorkflowEnvironment.start_time_skipping() as env:
+ with env.auto_time_skipping_disabled():
+ async with Worker(
+ env.client,
+ task_queue=queue,
+ workflows=[MembershipBackfillWorkflow],
+ activities=[prepare, backfill, fail, observe],
+ ):
+ handle = await env.client.start_workflow(
+ MembershipBackfillWorkflow.run,
+ MembershipTeam(
+ team_id=17,
+ account_keys=account_keys,
+ dry_run=has_cursor,
+ cursor=MembershipCursor(
+ team_id=17,
+ version=7,
+ group_type_index=0,
+ account_keys=("account-a",),
+ dry_run=True,
+ historical_end=datetime(2026, 1, 1, tzinfo=UTC),
+ windows_completed=3,
+ )
+ if has_cursor
+ else None,
+ ),
+ id=get_membership_workflow_id(17),
+ task_queue=queue,
+ execution_timeout=timedelta(minutes=5),
+ )
+ assert (await wait_for_event("failed")).error_type == "ExampleUnavailable"
+ assert (await handle.query(MembershipBackfillWorkflow.get_status)).status == "failed"
+ assert failures[0].version == 7
+ failed = False
+ await handle.signal(MembershipBackfillWorkflow.resume)
+ await wait_for_event("resumed")
+ await wait_for_event("progress")
+ await handle.signal(MembershipBackfillWorkflow.cancel)
+ assert await asyncio.wait_for(handle.result(), timeout=30) == "cancelled"
+ observation = await wait_for_event("cancelled")
+ assert observation.input.account_keys == account_keys
+ if has_cursor:
+ assert observation.input.cursor is not None
+ assert observation.input.cursor.windows_completed == 3
+ assert observation.input.cursor.historical_end == datetime(2026, 1, 1, tzinfo=UTC)
+ else:
+ assert observation.input.version == 7
+ await Replayer(workflows=[MembershipBackfillWorkflow]).replay_workflow(await handle.fetch_history())
+
+
[email protected]("operation", ["batch", "sync", "prepare"])
+async def test_activity_cancellation_stops_the_database_thread(operation: str) -> None:
+ loop = asyncio.get_running_loop()
+ started = asyncio.Event()
+ checked = asyncio.Event()
+ release = Event()
+ stopped: list[bool] = []
+
+ def wait_for_cancellation(should_stop: Callable[[], bool]) -> None:
+ loop.call_soon_threadsafe(started.set)
+ if not release.wait(timeout=10):
+ raise RuntimeError("Database thread did not finish")
+ stopped.append(should_stop())
+ loop.call_soon_threadsafe(checked.set)
+
+ def process_batch(_input: MembershipTeam, *, deadline: float, should_stop: Callable[[], bool]) -> MembershipStep:
+ wait_for_cancellation(should_stop)
+ return MembershipStep(status="backfilling")
+
+ def sync_page(service: MembershipBackfillService, _after: int, *, dry_run: bool) -> MembershipSyncPage:
+ assert service.should_stop is not None and service.deadline is not None
+ wait_for_cancellation(service.should_stop)
+ return MembershipSyncPage(teams=(), next_team_id=None, candidates=0, enabled=0, changed=0)
+
+ def prepare(service: MembershipBackfillService, input: MembershipTeam) -> MembershipTeam:
+ assert service.should_stop is not None and service.deadline is not None
+ wait_for_cancellation(service.should_stop)
+ return input
+
+ env = ActivityEnvironment()
+
+ async def run_activity() -> None:
+ if operation == "batch":
+ await env.run(backfill_membership_batch_activity, MembershipTeam(team_id=17))
+ elif operation == "sync":
+ await env.run(sync_membership_page_activity, MembershipCoordinatorInput())
+ else:
+ await env.run(prepare_membership_backfill_activity, MembershipTeam(team_id=17))
+
+ with (
+ patch.object(MembershipBackfillService, "process_activity_batch", side_effect=process_batch),
+ patch.object(MembershipBackfillService, "sync_page", autospec=True, side_effect=sync_page),
+ patch.object(MembershipBackfillService, "prepare_backfill", autospec=True, side_effect=prepare),
+ ):
+ task = asyncio.create_task(run_activity())
+ try:
+ await asyncio.wait_for(started.wait(), timeout=10)
+ env.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+ finally:
+ release.set()
+ await asyncio.wait_for(checked.wait(), timeout=10)
+ assert stopped == [True]
+
+
+async def test_monitoring_logs_use_run_counters_without_account_keys() -> None:
+ cursor = MembershipCursor(
+ team_id=17,
+ version=1,
+ group_type_index=0,
+ accounts_completed=2,
+ windows_completed=3,
+ rows_written=7,
+ rows_found=9,
+ )
+ observation = MembershipObservation(
+ input=MembershipTeam(team_id=17, account_keys=("example-account",), cursor=cursor),
+ run_id="example-run",
+ event="progress",
+ status="backfilling",
+ )
+ with capture_logs() as logs:
+ await ActivityEnvironment().run(observe_membership_backfill_activity, observation)
+ assert logs[0]["event"] == "customer_analytics.membership_backfill"
+ assert logs[0]["backfill_run_id"] == "example-run"
+ assert logs[0]["account_scope"] == "selected"
+ assert logs[0]["accounts_completed"] == 2
+ assert logs[0]["windows_completed"] == 3
+ assert logs[0]["rows_written"] == 7
+ assert "example-account" not in str(logs)
diff --git a/products/customer_analytics/backend/test/test_backfill_memberships_command.py b/products/customer_analytics/backend/test/test_backfill_memberships_command.py
new file mode 100644
index 000000000000..128766ce2eb4
--- /dev/null
+++ b/products/customer_analytics/backend/test/test_backfill_memberships_command.py
@@ -0,0 +1,45 @@
+import io
+import json
+
+from unittest.mock import AsyncMock, patch
+
+from django.core.management import call_command
+from django.test import SimpleTestCase
+
+from parameterized import parameterized
+
+from products.customer_analytics.backend.logic.membership_backfill import MembershipTeam
+
+COMMAND = "products.customer_analytics.backend.management.commands.backfill_memberships"
+CONTROL = "products.customer_analytics.backend.logic.membership_backfill_control"
+
+
+class TestBackfillMembershipsCommand(SimpleTestCase):
+ @parameterized.expand(
+ [
+ (scope, accounts, dry_run)
+ for scope, accounts in (
+ (["--all-accounts"], ()),
+ (["--account", "b", "--account", "a", "--account", "b"], ("a", "b")),
+ )
+ for dry_run in (False, True)
+ ]
+ )
+ def test_start_preserves_account_scope(
+ self, scope: list[str], expected_accounts: tuple[str, ...], dry_run: bool
+ ) -> None:
+ client = AsyncMock()
+ client.start_workflow.return_value.first_execution_run_id = "example-run"
+ output = io.StringIO()
+ with (
+ patch(f"{COMMAND}.Team.objects.filter"),
+ patch(f"{COMMAND}.get_membership_group_type_index", return_value=0),
+ patch(f"{CONTROL}.async_connect", return_value=client),
+ ):
+ call_command("backfill_memberships", "start", "--team-id", "17", *scope, dry_run=dry_run, stdout=output)
+ assert json.loads(output.getvalue())["run_id"] == "example-run"
+ assert client.start_workflow.call_args.args[1] == MembershipTeam(
+ team_id=17,
+ account_keys=expected_accounts,
+ dry_run=dry_run,
+ )