:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(messaging): run provider imports in temporal (#124426)
Harley Alexander0b517fea8112
diff --git a/posthog/management/commands/start_temporal_worker.py b/posthog/management/commands/start_temporal_worker.pyindex 7a0641f45718..f2a857bc443c 100644--- a/posthog/management/commands/start_temporal_worker.py+++ b/posthog/management/commands/start_temporal_worker.py@@ -264,6 +264,10 @@ from products.managed_warehouse.backend.facade.temporal import (ACTIVITIES as DUCKLAKE_COPY_ACTIVITIES,WORKFLOWS as DUCKLAKE_COPY_WORKFLOWS,)+from products.messaging.backend.facade.temporal import (+ ACTIVITIES as MESSAGING_IMPORT_ACTIVITIES,+ WORKFLOWS as MESSAGING_IMPORT_WORKFLOWS,+)from products.metrics.backend.facade.temporal import (ACTIVITIES as METRICS_ACTIVITIES,WORKFLOWS as METRICS_WORKFLOWS,@@ -471,6 +475,13 @@ _task_queue_specs = [CANVAS_BUILD_WORKFLOWS,CANVAS_BUILD_ACTIVITIES,),+ # Messaging provider imports. MESSAGING_IMPORT_TASK_QUEUE defaults to the general-purpose queue name (so it+ # merges into that fleet until a dedicated worker exists).+ (+ settings.MESSAGING_IMPORT_TASK_QUEUE,+ MESSAGING_IMPORT_WORKFLOWS,+ MESSAGING_IMPORT_ACTIVITIES,+ ),(settings.EXPERIMENTS_RECALCULATION_TASK_QUEUE,EXPERIMENTS_RECALCULATION_WORKFLOWS,diff --git a/posthog/settings/temporal.py b/posthog/settings/temporal.pyindex 7dcb59a92d3b..536ae569164a 100644--- a/posthog/settings/temporal.py+++ b/posthog/settings/temporal.py@@ -363,6 +363,11 @@ SIGNUP_ENRICHMENT_TASK_QUEUE = _set_temporal_task_queue(# dispatching services. Setting it first strands builds on a pollerless queue until the# workflow execution timeout closes them.CANVAS_BUILD_TASK_QUEUE = _set_temporal_task_queue(os.getenv("CANVAS_BUILD_TASK_QUEUE", "general-purpose-task-queue"))+# Messaging provider imports (for example Customer.io suppressions) share the general-purpose fleet. To move them to a+# dedicated worker, deploy a worker polling the new queue before setting this env on the dispatching services.+MESSAGING_IMPORT_TASK_QUEUE = _set_temporal_task_queue(+ os.getenv("MESSAGING_IMPORT_TASK_QUEUE", "general-purpose-task-queue")+)EXPERIMENTS_RECALCULATION_TASK_QUEUE = _set_temporal_task_queue("experiments-recalculation-task-queue")EXPERIMENTS_RECALCULATION_ACTIVITY_TASK_QUEUE = _set_temporal_task_queue("experiments-recalculation-activity-task-queue"diff --git a/products/messaging/backend/facade/temporal.py b/products/messaging/backend/facade/temporal.pynew file mode 100644index 000000000000..06e8102ad098--- /dev/null+++ b/products/messaging/backend/facade/temporal.py@@ -0,0 +1,6 @@+"""Temporal workflows and activities the provider import worker registers."""++from products.messaging.backend.temporal.registry import (+ ACTIVITIES as ACTIVITIES,+ WORKFLOWS as WORKFLOWS,+)diff --git a/products/messaging/backend/provider_imports/__init__.py b/products/messaging/backend/provider_imports/__init__.pynew file mode 100644index 000000000000..e69de29bb2d1diff --git a/products/messaging/backend/provider_imports/base.py b/products/messaging/backend/provider_imports/base.pynew file mode 100644index 000000000000..6817dd8d76ba--- /dev/null+++ b/products/messaging/backend/provider_imports/base.py@@ -0,0 +1,105 @@+"""The contract every provider importer implements.++A provider importer knows one messaging tool: how to check and store its credentials, and how to page through each+kind of data it can import. It knows nothing about Temporal or run state. `runs.py` drives the steps, saves each page+together with the step's progress, and resumes a retried step from the last saved checkpoint.+"""++from abc import ABC, abstractmethod+from collections.abc import Callable, Iterator+from dataclasses import field+from typing import ClassVar++from posthog.dataclasses import frozen++# Where a step continues after a page. Each step defines its own keys. The values are JSON-safe because the+# checkpoint is stored in the run's `progress`.+StepCheckpoint = dict[str, str | int]+++class ImportStepError(Exception):+ """A failure whose message is safe to show the user. The run shows a generic message for any other exception."""+++class ImportCredentialsMissing(ImportStepError):+ """The request has no API key and none is stored. The message names what is missing."""+++class ImportCredentialsConflict(ImportStepError):+ """A key is stored and the request sends another one. The message tells the caller to delete it first."""+++class ImportCredentialsInvalid(ImportStepError):+ """The provider rejected the API key. A retry does not fix this."""+++class ImportProviderUnavailable(ImportStepError):+ """The provider did not respond or returned a server error. A later retry can succeed."""+++class UnknownImportStep(ImportStepError):+ """The provider has no step with this key."""+++@frozen+class ImportStepSpec:+ key: str+ label: str+ description: str+++@frozen+class ImportCredentials:+ api_key: str = field(repr=False)+++@frozen+class ImportContext:+ team_id: int+ created_by_id: int | None+ integration_id: int | None+++@frozen+class SavedPage:+ # How many records the page added or changed in PostHog. Records that already match do not count.+ imported: int+ # Problems with single records that do not fail the step.+ warnings: tuple[str, ...] = ()+++@frozen+class ImportPage:+ # How many source records the page read.+ processed: int+ # Writes the page to PostHog. The runner calls it in the transaction that saves the step's progress, so the page+ # and its counts and checkpoint commit together.+ save: Callable[[], SavedPage]+ # Where to continue when the step runs again after this page. None on the last page of the step.+ checkpoint: StepCheckpoint | None = None+++class ImportProvider(ABC):+ key: ClassVar[str]+ label: ClassVar[str]+ # The steps in the order they run. Each one is safe to run again over records it already imported.+ steps: ClassVar[tuple[ImportStepSpec, ...]]++ @abstractmethod+ def check_credentials(self, team_id: int, api_key: str | None) -> ImportCredentials:+ """Choose the API key to import with and check it with the provider. Without `api_key`, use the stored key.++ This runs before the import claims its run, so it must not store anything. Raises the ImportCredentials*+ errors, or ImportProviderUnavailable when the provider can't answer.+ """++ @abstractmethod+ def store_credentials(self, team_id: int, credentials: ImportCredentials, created_by_id: int | None) -> int:+ """Store checked credentials on first use and return the integration id.++ This runs in the transaction that claims the run, so a request that loses the claim stores nothing.+ """++ @abstractmethod+ def run_step(self, step: str, context: ImportContext, checkpoint: StepCheckpoint | None) -> Iterator[ImportPage]:+ """Fetch one step's pages from `checkpoint`, or from the start when it is None, without saving them."""diff --git a/products/messaging/backend/provider_imports/customerio.py b/products/messaging/backend/provider_imports/customerio.pynew file mode 100644index 000000000000..479c823a2b38--- /dev/null+++ b/products/messaging/backend/provider_imports/customerio.py@@ -0,0 +1,177 @@+import logging+from collections.abc import Iterator+from functools import partial++from products.messaging.backend.models.message_preferences import ALL_MESSAGE_PREFERENCE_CATEGORY_ID+from products.messaging.backend.models.provider_import_run import ImportProviderKey+from products.messaging.backend.provider_imports.base import (+ ImportContext,+ ImportCredentials,+ ImportCredentialsConflict,+ ImportCredentialsInvalid,+ ImportCredentialsMissing,+ ImportPage,+ ImportProvider,+ ImportProviderUnavailable,+ ImportStepSpec,+ SavedPage,+ StepCheckpoint,+ UnknownImportStep,+)+from products.messaging.backend.services import customerio_config+from products.messaging.backend.services.customerio_client import CustomerIOAPIError+from products.messaging.backend.services.customerio_import_service import (+ CUSTOMERIO_SUPPRESSION_TYPES,+ CustomerIOImportService,+ GlobalUnsubscribePage,+ SuppressionPage,+)++logger = logging.getLogger(__name__)++SUPPRESSION_TYPE_ORDER = tuple(CUSTOMERIO_SUPPRESSION_TYPES)+# The key check runs on the request path and tries both regions, so it gets a shorter timeout than the import steps.+CREDENTIAL_CHECK_TIMEOUT_SECONDS = 10+INVALID_CREDENTIALS_MESSAGE = (+ "Customer.io rejected the App API key. Check that it is an App API key, that it has not expired, "+ "and that you copied all of it."+)+++class CustomerIOImportProvider(ImportProvider):+ key = ImportProviderKey.CUSTOMERIO.value+ label = ImportProviderKey.CUSTOMERIO.label+ steps = (+ ImportStepSpec(+ key="categories",+ label="Subscription topics",+ description="Creates a message category for each Customer.io subscription topic.",+ ),+ ImportStepSpec(+ key="suppressions",+ label="Suppressed addresses",+ description=(+ "Adds the addresses that Customer.io's email provider bounced, marked as invalid, "+ "or got a spam report for to the suppression list."+ ),+ ),+ ImportStepSpec(+ key="opt_outs",+ label="Global unsubscribes",+ description="Opts people who unsubscribed from all Customer.io messages out of all marketing messages.",+ ),+ )++ def check_credentials(self, team_id: int, api_key: str | None) -> ImportCredentials:+ try:+ resolved_key = customerio_config.resolve_app_api_key(team_id, api_key)+ except customerio_config.ConfigIncomplete as e:+ raise ImportCredentialsMissing(str(e)) from e+ except customerio_config.ConfigConflict as e:+ raise ImportCredentialsConflict(str(e)) from e+ try:+ connected = CustomerIOImportService(team_id, resolved_key, None).connect(+ timeout=CREDENTIAL_CHECK_TIMEOUT_SECONDS+ )+ except CustomerIOAPIError as e:+ raise ImportProviderUnavailable(_api_error_message(e)) from e+ if not connected:+ raise ImportCredentialsInvalid(INVALID_CREDENTIALS_MESSAGE)+ return ImportCredentials(api_key=resolved_key)++ def store_credentials(self, team_id: int, credentials: ImportCredentials, created_by_id: int | None) -> int:+ return customerio_config.save_app_integration(team_id, credentials.api_key, created_by_id).id++ def run_step(self, step: str, context: ImportContext, checkpoint: StepCheckpoint | None) -> Iterator[ImportPage]:+ try:+ match step:+ case "categories":+ yield from self._fetch_categories(self._connect(context))+ case "suppressions":+ yield from self._fetch_suppressions(self._connect(context), checkpoint)+ case "opt_outs":+ yield from self._fetch_opt_outs(self._connect(context), checkpoint)+ case _:+ raise UnknownImportStep(f"Customer.io has no import step named {step}.")+ except CustomerIOAPIError as e:+ raise ImportProviderUnavailable(_api_error_message(e)) from e++ def _connect(self, context: ImportContext) -> CustomerIOImportService:+ api_key = (+ customerio_config.stored_app_api_key(context.team_id, context.integration_id)+ if context.integration_id is not None+ else None+ )+ if not api_key:+ raise ImportCredentialsMissing(+ "The Customer.io App API key was deleted. Add it again and start a new import."+ )+ service = CustomerIOImportService(context.team_id, api_key, context.created_by_id)+ if not service.connect():+ raise ImportCredentialsInvalid(INVALID_CREDENTIALS_MESSAGE)+ return service++ def _fetch_categories(self, service: CustomerIOImportService) -> Iterator[ImportPage]:+ assert service.client is not None+ topics = service.client.get_subscription_topics()+ yield ImportPage(processed=len(topics), save=partial(_save_categories, service, topics))++ def _fetch_suppressions(+ self, service: CustomerIOImportService, checkpoint: StepCheckpoint | None+ ) -> Iterator[ImportPage]:+ resume_type = checkpoint.get("suppression_type") if checkpoint else None+ start_index = SUPPRESSION_TYPE_ORDER.index(resume_type) if resume_type in SUPPRESSION_TYPE_ORDER else 0+ for index in range(start_index, len(SUPPRESSION_TYPE_ORDER)):+ suppression_type = SUPPRESSION_TYPE_ORDER[index]+ offset = int(checkpoint.get("offset", 0)) if checkpoint and suppression_type == resume_type else 0+ for page in service.fetch_suppression_pages(suppression_type, offset):+ next_checkpoint: StepCheckpoint | None+ if not page.is_last:+ next_checkpoint = {"suppression_type": suppression_type, "offset": page.next_offset}+ elif index + 1 < len(SUPPRESSION_TYPE_ORDER):+ next_checkpoint = {"suppression_type": SUPPRESSION_TYPE_ORDER[index + 1], "offset": 0}+ else:+ next_checkpoint = None+ yield ImportPage(+ processed=page.entries, save=partial(_save_suppressions, service, page), checkpoint=next_checkpoint+ )++ def _fetch_opt_outs(+ self, service: CustomerIOImportService, checkpoint: StepCheckpoint | None+ ) -> Iterator[ImportPage]:+ start = checkpoint.get("start") if checkpoint else None+ for page in service.fetch_global_unsubscribe_pages(start=str(start) if start else None):+ yield ImportPage(+ processed=page.customers,+ save=partial(_save_opt_outs, service, page),+ checkpoint={"start": page.next_start} if page.next_start else None,+ )+++def _save_categories(service: CustomerIOImportService, topics: list[dict]) -> SavedPage:+ service.import_categories(topics)+ errors = service.progress["errors"]+ if not errors:+ return SavedPage(imported=service.progress["categories_created"])+ # The service's errors hold raw exception text, which stays in the server log.+ logger.warning("Customer.io topics failed to import", extra={"team_id": service.team_id, "errors": errors})+ return SavedPage(+ imported=service.progress["categories_created"],+ warnings=(f"{len(errors)} subscription topics couldn't be imported. Run the import again to retry them.",),+ )+++def _save_suppressions(service: CustomerIOImportService, page: SuppressionPage) -> SavedPage:+ return SavedPage(imported=service.save_suppression_page(page))+++def _save_opt_outs(service: CustomerIOImportService, page: GlobalUnsubscribePage) -> SavedPage:+ if not page.emails:+ return SavedPage(imported=0)+ return SavedPage(imported=service.save_opt_outs(list(page.emails), [ALL_MESSAGE_PREFERENCE_CATEGORY_ID]))+++def _api_error_message(error: CustomerIOAPIError) -> str:+ if error.status_code is None:+ return "Customer.io did not respond."+ return f"Customer.io returned HTTP {error.status_code}."diff --git a/products/messaging/backend/provider_imports/registry.py b/products/messaging/backend/provider_imports/registry.pynew file mode 100644index 000000000000..ec380a4a35b6--- /dev/null+++ b/products/messaging/backend/provider_imports/registry.py@@ -0,0 +1,16 @@+from products.messaging.backend.provider_imports.base import ImportProvider+from products.messaging.backend.provider_imports.customerio import CustomerIOImportProvider++# Add a provider here, and its key to ImportProviderKey, to make it importable.+PROVIDERS: dict[str, ImportProvider] = {provider.key: provider for provider in (CustomerIOImportProvider(),)}+++class UnknownImportProvider(Exception):+ pass+++def get_provider(key: str) -> ImportProvider:+ provider = PROVIDERS.get(key)+ if provider is None:+ raise UnknownImportProvider(key)+ return providerdiff --git a/products/messaging/backend/provider_imports/runs.py b/products/messaging/backend/provider_imports/runs.pynew file mode 100644index 000000000000..e345ae83e250--- /dev/null+++ b/products/messaging/backend/provider_imports/runs.py@@ -0,0 +1,276 @@+"""Start provider import runs, run their steps page by page, and record the outcome."""++import logging+from collections.abc import Callable+from datetime import timedelta+from typing import Any+from uuid import UUID++from django.db import IntegrityError, transaction+from django.db.models import QuerySet+from django.utils import timezone++from posthog.dataclasses import frozen+from posthog.event_usage import report_team_action+from posthog.models import Team++from products.messaging.backend.models.provider_import_run import (+ ACTIVE_IMPORT_STATUSES,+ ImportStepStatus,+ ProviderImportRun,+ ProviderImportStatus,+)+from products.messaging.backend.provider_imports.base import (+ ImportContext,+ ImportPage,+ ImportProvider,+ ImportStepError,+ StepCheckpoint,+)+from products.messaging.backend.provider_imports.registry import get_provider+from products.messaging.backend.temporal.client import RUN_EXECUTION_TIMEOUT, start_provider_import_workflow++logger = logging.getLogger(__name__)++# A run still active after the workflow's execution timeout has no workflow left to finish it.+STALE_RUN_AGE = RUN_EXECUTION_TIMEOUT + timedelta(hours=1)+MAX_STEP_WARNINGS = 50+UNEXPECTED_ERROR_MESSAGE = "An unexpected error interrupted this step."+++class ProviderImportAlreadyRunning(Exception):+ pass+++class InvalidImportSteps(Exception):+ pass+++class ImportRunClosed(Exception):+ """The run already ended, for example because its workflow failed to start. Its steps must not import."""+++class StaleImportAttempt(Exception):+ """A newer attempt of the step took over, so this attempt must stop without writing."""+++@frozen+class _StepClaim:+ completed: bool+ checkpoint: StepCheckpoint | None+++def start_run(+ team_id: int, provider_key: str, steps: list[str], api_key: str | None, created_by_id: int | None+) -> ProviderImportRun:+ """Check the credentials, claim a queued run, store the credentials, and start the run's workflow.++ Raises UnknownImportProvider, InvalidImportSteps, ProviderImportAlreadyRunning, the ImportCredentials* errors and+ ImportProviderUnavailable.+ """+ provider = get_provider(provider_key)+ ordered_steps = _order_steps(provider, steps)++ _fail_stale_runs(team_id)+ if _runs(team_id).filter(provider=provider.key, status__in=ACTIVE_IMPORT_STATUSES).exists():+ raise ProviderImportAlreadyRunning(f"A {provider.label} import is already running.")++ credentials = provider.check_credentials(team_id, api_key)+ try:+ with transaction.atomic():+ # Create the run before storing the credentials. A concurrent start then waits on the active-run index+ # and fails, so only the request that owns the run stores its key.+ run = _runs(team_id).create(+ team_id=team_id,+ provider=provider.key,+ steps=ordered_steps,+ created_by_id=created_by_id,+ progress={step: _new_step_progress() for step in ordered_steps},+ )+ run.integration_id = provider.store_credentials(team_id, credentials, created_by_id)+ run.save(update_fields=["integration"])+ except IntegrityError as e:+ raise ProviderImportAlreadyRunning(f"A {provider.label} import is already running.") from e++ try:+ start_provider_import_workflow(team_id=team_id, run_id=run.id, steps=ordered_steps)+ except Exception:+ # A queued run without a workflow would block every later import until it goes stale. If Temporal started+ # the workflow but the response got lost, its steps find the run closed and import nothing.+ _close_run(team_id, run.id)+ raise+ return run+++def run_step(team_id: int, run_id: UUID, step: str, attempt: int) -> None:+ """Run one step from its saved checkpoint, and save each page together with the step's progress.++ `attempt` is the Temporal attempt number. An older attempt can keep running after Temporal starts a newer one,+ so each page write checks that its attempt still owns the step.+ """+ run = _runs(team_id).get(id=run_id)+ provider = get_provider(run.provider)+ claim = _claim_step(team_id, run_id, step, attempt)+ if claim.completed:+ return++ context = ImportContext(team_id=team_id, created_by_id=run.created_by_id, integration_id=run.integration_id)+ try:+ for page in provider.run_step(step, context, claim.checkpoint):+ _save_page(team_id, run_id, step, attempt, page)+ if page.checkpoint is None:+ return+ _update_owned_step(team_id, run_id, step, attempt, _mark_completed)+ except (StaleImportAttempt, ImportRunClosed):+ raise+ except Exception as e:+ if not isinstance(e, ImportStepError):+ logger.exception("Provider import step failed", extra={"run_id": str(run_id), "step": step})+ last_error = str(e)[:500] if isinstance(e, ImportStepError) else UNEXPECTED_ERROR_MESSAGE+ try:+ _update_owned_step(team_id, run_id, step, attempt, lambda progress: progress.update(last_error=last_error))+ except (StaleImportAttempt, ImportRunClosed):+ pass+ raise+++def finish_run(team_id: int, run_id: UUID) -> None:+ """Fail the steps that did not complete, and set the run's final status. A run that already ended stays as it is."""+ _close_run(team_id, run_id)+++def list_runs(team_id: int, limit: int) -> list[ProviderImportRun]:+ _fail_stale_runs(team_id)+ return list(_runs(team_id).order_by("-created_at")[:limit])+++def get_run(team_id: int, run_id: UUID) -> ProviderImportRun | None:+ _fail_stale_runs(team_id)+ return _runs(team_id).filter(id=run_id).first()+++def _runs(team_id: int) -> QuerySet[ProviderImportRun]:+ return ProviderImportRun.objects.for_team(team_id)+++def _order_steps(provider: ImportProvider, steps: list[str]) -> list[str]:+ known = [spec.key for spec in provider.steps]+ unknown = sorted(set(steps) - set(known))+ if unknown:+ raise InvalidImportSteps(f"{provider.label} has no step named {', '.join(unknown)}.")+ if not steps:+ raise InvalidImportSteps("Select at least one thing to import.")+ return [key for key in known if key in steps]+++def _fail_stale_runs(team_id: int) -> None:+ stale_ids = list(+ _runs(team_id)+ .filter(status__in=ACTIVE_IMPORT_STATUSES, created_at__lt=timezone.now() - STALE_RUN_AGE)+ .values_list("id", flat=True)+ )+ for run_id in stale_ids:+ _close_run(team_id, run_id)+++def _close_run(team_id: int, run_id: UUID) -> None:+ with transaction.atomic():+ run = _runs(team_id).select_for_update().get(id=run_id)+ if run.status not in ACTIVE_IMPORT_STATUSES:+ return+ for step in run.steps:+ progress = run.progress.setdefault(step, _new_step_progress())+ if progress["status"] != ImportStepStatus.COMPLETED:+ progress["status"] = ImportStepStatus.FAILED+ completed = all(run.progress[step]["status"] == ImportStepStatus.COMPLETED for step in run.steps)+ run.status = ProviderImportStatus.COMPLETED if completed else ProviderImportStatus.FAILED+ run.finished_at = timezone.now()+ run.save(update_fields=["progress", "status", "finished_at"])++ _report_finished(run)+++def _claim_step(team_id: int, run_id: UUID, step: str, attempt: int) -> _StepClaim:+ with transaction.atomic():+ run = _runs(team_id).select_for_update().get(id=run_id)+ if run.status not in ACTIVE_IMPORT_STATUSES:+ raise ImportRunClosed(f"Import run {run_id} already ended.")+ progress = run.progress.setdefault(step, _new_step_progress())+ if progress["status"] == ImportStepStatus.COMPLETED:+ return _StepClaim(completed=True, checkpoint=None)+ progress.update(status=ImportStepStatus.RUNNING.value, attempt=attempt)+ update_fields = ["progress"]+ if run.status == ProviderImportStatus.QUEUED:+ run.status = ProviderImportStatus.RUNNING+ run.started_at = timezone.now()+ update_fields += ["status", "started_at"]+ run.save(update_fields=update_fields)+ return _StepClaim(completed=False, checkpoint=progress.get("checkpoint"))+++def _save_page(team_id: int, run_id: UUID, step: str, attempt: int, page: ImportPage) -> None:+ with transaction.atomic():+ run, progress = _lock_owned_step(team_id, run_id, step, attempt)+ saved = page.save()+ progress["processed"] += page.processed+ progress["imported"] += saved.imported+ progress["warnings"] = (progress["warnings"] + list(saved.warnings))[:MAX_STEP_WARNINGS]+ progress["checkpoint"] = page.checkpoint+ if page.checkpoint is None:+ _mark_completed(progress)+ run.save(update_fields=["progress"])+++def _update_owned_step(+ team_id: int, run_id: UUID, step: str, attempt: int, change: Callable[[dict[str, Any]], None]+) -> None:+ with transaction.atomic():+ run, progress = _lock_owned_step(team_id, run_id, step, attempt)+ change(progress)+ run.save(update_fields=["progress"])+++def _lock_owned_step(team_id: int, run_id: UUID, step: str, attempt: int) -> tuple[ProviderImportRun, dict[str, Any]]:+ run = _runs(team_id).select_for_update().get(id=run_id)+ if run.status not in ACTIVE_IMPORT_STATUSES:+ raise ImportRunClosed(f"Import run {run_id} ended while step {step} was running.")+ progress = run.progress.setdefault(step, _new_step_progress())+ if progress.get("attempt") != attempt:+ raise StaleImportAttempt(f"Attempt {attempt} of step {step} no longer owns import run {run_id}.")+ return run, progress+++def _new_step_progress() -> dict[str, Any]:+ return {+ "status": ImportStepStatus.PENDING.value,+ "processed": 0,+ "imported": 0,+ "warnings": [],+ "last_error": None,+ "checkpoint": None,+ "attempt": None,+ }+++def _mark_completed(progress: dict[str, Any]) -> None:+ progress.update(status=ImportStepStatus.COMPLETED.value, last_error=None, checkpoint=None)+++def _report_finished(run: ProviderImportRun) -> None:+ try:+ report_team_action(+ Team.objects.get(id=run.team_id),+ "messaging provider import finished",+ {+ "provider": run.provider,+ "status": run.status,+ "steps": run.steps,+ "duration_seconds": int((run.finished_at - run.created_at).total_seconds())+ if run.finished_at+ else None,+ **{f"{step}_status": run.progress[step]["status"] for step in run.steps},+ **{f"{step}_imported": run.progress[step]["imported"] for step in run.steps},+ },+ )+ except Exception:+ logger.exception("Failed to report the provider import outcome")diff --git a/products/messaging/backend/temporal/__init__.py b/products/messaging/backend/temporal/__init__.pynew file mode 100644index 000000000000..e69de29bb2d1diff --git a/products/messaging/backend/temporal/activities.py b/products/messaging/backend/temporal/activities.pynew file mode 100644index 000000000000..4ae20837797b--- /dev/null+++ b/products/messaging/backend/temporal/activities.py@@ -0,0 +1,22 @@+from uuid import UUID++from temporalio import activity++from posthog.temporal.common.heartbeat_sync import HeartbeaterSync+from posthog.temporal.common.utils import asyncify++from products.messaging.backend.provider_imports.runs import finish_run, run_step+from products.messaging.backend.temporal.inputs import ProviderImportRunInputs, ProviderImportStepInputs++[email protected](name="messaging-provider-import.run_step")+@asyncify+def run_provider_import_step_activity(inputs: ProviderImportStepInputs) -> None:+ with HeartbeaterSync():+ run_step(inputs.team_id, UUID(inputs.run_id), inputs.step, activity.info().attempt)++[email protected](name="messaging-provider-import.finish_run")+@asyncify+def finish_provider_import_run_activity(inputs: ProviderImportRunInputs) -> None:+ finish_run(inputs.team_id, UUID(inputs.run_id))diff --git a/products/messaging/backend/temporal/client.py b/products/messaging/backend/temporal/client.pynew file mode 100644index 000000000000..8b9880704080--- /dev/null+++ b/products/messaging/backend/temporal/client.py@@ -0,0 +1,32 @@+"""Start the provider import workflow from Django."""++from datetime import timedelta+from uuid import UUID++from django.conf import settings++from asgiref.sync import async_to_sync+from temporalio.common import RetryPolicy++from posthog.temporal.common.client import async_connect++from products.messaging.backend.temporal.inputs import ProviderImportWorkflowInputs++PROVIDER_IMPORT_WORKFLOW = "messaging-provider-import"++# Bounds the whole run, so a run stranded on a queue with no pollers closes instead of blocking new imports forever.+RUN_EXECUTION_TIMEOUT = timedelta(hours=24)+++@async_to_sync+async def start_provider_import_workflow(team_id: int, run_id: UUID, steps: list[str]) -> None:+ client = await async_connect()+ await client.start_workflow(+ PROVIDER_IMPORT_WORKFLOW,+ ProviderImportWorkflowInputs(team_id=team_id, run_id=str(run_id), steps=steps),+ id=f"{PROVIDER_IMPORT_WORKFLOW}-{run_id}",+ task_queue=settings.MESSAGING_IMPORT_TASK_QUEUE,+ execution_timeout=RUN_EXECUTION_TIMEOUT,+ # Retries live on the activities. A workflow retry would run steps that already completed again.+ retry_policy=RetryPolicy(maximum_attempts=1),+ )diff --git a/products/messaging/backend/temporal/inputs.py b/products/messaging/backend/temporal/inputs.pynew file mode 100644index 000000000000..6345b4c9a624--- /dev/null+++ b/products/messaging/backend/temporal/inputs.py@@ -0,0 +1,21 @@+from posthog.dataclasses import frozen+++@frozen+class ProviderImportWorkflowInputs:+ team_id: int+ run_id: str+ steps: list[str]+++@frozen+class ProviderImportStepInputs:+ team_id: int+ run_id: str+ step: str+++@frozen+class ProviderImportRunInputs:+ team_id: int+ run_id: strdiff --git a/products/messaging/backend/temporal/registry.py b/products/messaging/backend/temporal/registry.pynew file mode 100644index 000000000000..68eea42342d9--- /dev/null+++ b/products/messaging/backend/temporal/registry.py@@ -0,0 +1,16 @@+"""Workflow and activity registration lists for the provider import worker.++Core reaches these through ``facade/temporal.py``.+"""++from products.messaging.backend.temporal.activities import (+ finish_provider_import_run_activity,+ run_provider_import_step_activity,+)+from products.messaging.backend.temporal.workflow import ProviderImportWorkflow++WORKFLOWS = [ProviderImportWorkflow]++ACTIVITIES = [run_provider_import_step_activity, finish_provider_import_run_activity]++__all__ = ["WORKFLOWS", "ACTIVITIES"]diff --git a/products/messaging/backend/temporal/workflow.py b/products/messaging/backend/temporal/workflow.pynew file mode 100644index 000000000000..690568ff8b60--- /dev/null+++ b/products/messaging/backend/temporal/workflow.py@@ -0,0 +1,66 @@+from datetime import timedelta++import temporalio.workflow+from temporalio import workflow+from temporalio.common import RetryPolicy+from temporalio.exceptions import ActivityError++from posthog.temporal.common.base import PostHogWorkflow++with temporalio.workflow.unsafe.imports_passed_through():+ from products.messaging.backend.temporal.activities import (+ finish_provider_import_run_activity,+ run_provider_import_step_activity,+ )+ from products.messaging.backend.temporal.inputs import (+ ProviderImportRunInputs,+ ProviderImportStepInputs,+ ProviderImportWorkflowInputs,+ )++# A step pages through a whole provider list, which can run for hours. The heartbeat timeout finds a dead worker+# quickly, and the step continues from its last saved page on the next attempt.+STEP_TIMEOUT = timedelta(hours=6)+STEP_HEARTBEAT_TIMEOUT = timedelta(minutes=2)+STEP_RETRY_POLICY = RetryPolicy(+ maximum_attempts=5,+ initial_interval=timedelta(seconds=10),+ backoff_coefficient=2.0,+ maximum_interval=timedelta(minutes=5),+ non_retryable_error_types=[+ "ImportCredentialsMissing",+ "ImportCredentialsInvalid",+ "ImportRunClosed",+ "StaleImportAttempt",+ "UnknownImportStep",+ "UnknownImportProvider",+ ],+)+FINISH_RETRY_POLICY = RetryPolicy(maximum_attempts=5, initial_interval=timedelta(seconds=5))++[email protected](name="messaging-provider-import")+class ProviderImportWorkflow(PostHogWorkflow):+ inputs_cls = ProviderImportWorkflowInputs++ @workflow.run+ async def run(self, inputs: ProviderImportWorkflowInputs) -> None:+ for step in inputs.steps:+ try:+ await workflow.execute_activity(+ run_provider_import_step_activity,+ ProviderImportStepInputs(team_id=inputs.team_id, run_id=inputs.run_id, step=step),+ start_to_close_timeout=STEP_TIMEOUT,+ heartbeat_timeout=STEP_HEARTBEAT_TIMEOUT,+ retry_policy=STEP_RETRY_POLICY,+ )+ except ActivityError:+ # The steps import independent data, so one failed step does not stop the others.+ workflow.logger.warning("Provider import step failed", extra={"run_id": inputs.run_id, "step": step})++ await workflow.execute_activity(+ finish_provider_import_run_activity,+ ProviderImportRunInputs(team_id=inputs.team_id, run_id=inputs.run_id),+ start_to_close_timeout=timedelta(minutes=1),+ retry_policy=FINISH_RETRY_POLICY,+ )diff --git a/products/messaging/backend/tests/test_provider_import_workflow.py b/products/messaging/backend/tests/test_provider_import_workflow.pynew file mode 100644index 000000000000..fc6b075c41c7--- /dev/null+++ b/products/messaging/backend/tests/test_provider_import_workflow.py@@ -0,0 +1,55 @@+import uuid+import logging+from datetime import timedelta++import pytest++from temporalio import activity+from temporalio.exceptions import ApplicationError+from temporalio.testing import WorkflowEnvironment+from temporalio.worker import UnsandboxedWorkflowRunner, Worker++from products.messaging.backend.temporal.inputs import (+ ProviderImportRunInputs,+ ProviderImportStepInputs,+ ProviderImportWorkflowInputs,+)+from products.messaging.backend.temporal.workflow import ProviderImportWorkflow+++async def test_a_failed_step_does_not_stop_later_steps_or_the_finish(caplog: pytest.LogCaptureFixture) -> None:+ caplog.set_level(logging.INFO, logger="temporalio.activity")+ caplog.set_level(logging.INFO, logger="temporalio.workflow")+ attempted_steps: list[str] = []+ finished_runs: list[str] = []++ @activity.defn(name="messaging-provider-import.run_step")+ async def run_step(inputs: ProviderImportStepInputs) -> None:+ attempted_steps.append(inputs.step)+ if inputs.step == "categories":+ raise ApplicationError("The key was deleted", type="ImportCredentialsMissing")++ @activity.defn(name="messaging-provider-import.finish_run")+ async def finish_run(inputs: ProviderImportRunInputs) -> None:+ finished_runs.append(inputs.run_id)++ run_id = str(uuid.uuid4())+ async with await WorkflowEnvironment.start_time_skipping() as env:+ async with Worker(+ env.client,+ task_queue="provider-import-test",+ workflows=[ProviderImportWorkflow],+ activities=[run_step, finish_run],+ workflow_runner=UnsandboxedWorkflowRunner(),+ ):+ await env.client.execute_workflow(+ ProviderImportWorkflow.run,+ ProviderImportWorkflowInputs(team_id=1, run_id=run_id, steps=["categories", "suppressions"]),+ id=f"provider-import-test-{run_id}",+ task_queue="provider-import-test",+ execution_timeout=timedelta(minutes=5),+ )++ assert attempted_steps == ["categories", "suppressions"]+ assert finished_runs == [run_id]diff --git a/products/messaging/backend/tests/test_provider_imports.py b/products/messaging/backend/tests/test_provider_imports.pynew file mode 100644index 000000000000..be224aaff244--- /dev/null+++ b/products/messaging/backend/tests/test_provider_imports.py@@ -0,0 +1,237 @@+from datetime import timedelta++import time_machine+from posthog.test.base import BaseTest+from unittest.mock import MagicMock, patch++from django.db import IntegrityError+from django.utils import timezone++from parameterized import parameterized++from products.messaging.backend.models.message_suppression import MessageSuppression+from products.messaging.backend.models.provider_import_run import ProviderImportRun+from products.messaging.backend.provider_imports import runs+from products.messaging.backend.provider_imports.base import ImportProviderUnavailable+from products.messaging.backend.services.customerio_client import CustomerIOAPIError++SERVICE = "products.messaging.backend.services.customerio_import_service"+++@patch(f"{SERVICE}.SUPPRESSION_PAGE_DELAY_SECONDS", 0)+@patch(f"{SERVICE}.SUPPRESSION_PAGE_SIZE", 2)+@patch("products.messaging.backend.provider_imports.runs.start_provider_import_workflow")+@patch(f"{SERVICE}.CustomerIOClient")+class TestProviderImportRuns(BaseTest):+ pages = {+ ("bounces", 0): [{"email": "[email protected]"}, {"email": "[email protected]"}],+ ("bounces", 2): [{"email": "[email protected]"}],+ ("spam_reports", 0): [{"email": "[email protected]"}],+ }++ def _start(self, steps: list[str]) -> ProviderImportRun:+ return runs.start_run(self.team.id, "customerio", steps, "app-key", self.user.id)++ def _suppressed(self) -> set[str]:+ return set(MessageSuppression.objects.for_team(self.team.id).values_list("identifier", flat=True))++ def test_suppressions_step_resumes_from_the_last_saved_page_and_does_not_rerun_once_completed(+ self, mock_client_class: MagicMock, _start_workflow: MagicMock+ ) -> None:+ failures = {("bounces", 2): CustomerIOAPIError("Customer.io API error", status_code=503)}++ def get_suppressions(suppression_type: str, limit: int, offset: int) -> list[dict]:+ if error := failures.pop((suppression_type, offset), None):+ raise error+ return self.pages.get((suppression_type, offset), [])++ mock_client = mock_client_class.return_value+ mock_client.get_suppressions.side_effect = get_suppressions+ run = self._start(["suppressions"])++ with self.assertRaises(ImportProviderUnavailable):+ runs.run_step(self.team.id, run.id, "suppressions", attempt=1)++ run.refresh_from_db()+ assert run.status == "running"+ assert run.progress["suppressions"]["checkpoint"] == {"suppression_type": "bounces", "offset": 2}+ assert run.progress["suppressions"]["last_error"] == "Customer.io returned HTTP 503."++ mock_client.get_suppressions.reset_mock()+ runs.run_step(self.team.id, run.id, "suppressions", attempt=2)++ assert [(call.args[0], call.kwargs["offset"]) for call in mock_client.get_suppressions.call_args_list] == [+ ("bounces", 2),+ ("invalid_emails", 0),+ ("spam_reports", 0),+ ]+ run.refresh_from_db()+ assert run.progress["suppressions"] == {+ "status": "completed",+ "processed": 4,+ "imported": 4,+ "warnings": [],+ "last_error": None,+ "checkpoint": None,+ "attempt": 2,+ }+ assert self._suppressed() == {"[email protected]", "[email protected]", "[email protected]", "[email protected]"}++ mock_client.get_suppressions.reset_mock()+ runs.run_step(self.team.id, run.id, "suppressions", attempt=3)++ mock_client.get_suppressions.assert_not_called()++ def test_an_older_attempt_stops_without_writing_once_a_newer_attempt_takes_over(+ self, mock_client_class: MagicMock, _start_workflow: MagicMock+ ) -> None:+ run = self._start(["suppressions"])+ newer_attempt_started = False++ def get_suppressions(suppression_type: str, limit: int, offset: int) -> list[dict]:+ nonlocal newer_attempt_started+ if (suppression_type, offset) == ("bounces", 2) and not newer_attempt_started:+ newer_attempt_started = True+ runs.run_step(self.team.id, run.id, "suppressions", attempt=2)+ return self.pages.get((suppression_type, offset), [])++ mock_client_class.return_value.get_suppressions.side_effect = get_suppressions++ with self.assertRaises(runs.StaleImportAttempt):+ runs.run_step(self.team.id, run.id, "suppressions", attempt=1)++ run.refresh_from_db()+ assert run.progress["suppressions"]["attempt"] == 2+ assert run.progress["suppressions"]["status"] == "completed"+ assert run.progress["suppressions"]["processed"] == 4+ assert self._suppressed() == {"[email protected]", "[email protected]", "[email protected]", "[email protected]"}++ def test_an_attempt_stops_without_writing_once_its_run_closes(+ self, mock_client_class: MagicMock, _start_workflow: MagicMock+ ) -> None:+ run = self._start(["suppressions"])++ def get_suppressions(suppression_type: str, limit: int, offset: int) -> list[dict]:+ if (suppression_type, offset) == ("bounces", 2):+ runs.finish_run(self.team.id, run.id)+ return self.pages.get((suppression_type, offset), [])++ mock_client_class.return_value.get_suppressions.side_effect = get_suppressions++ with self.assertRaises(runs.ImportRunClosed):+ runs.run_step(self.team.id, run.id, "suppressions", attempt=1)++ run.refresh_from_db()+ assert run.status == "failed"+ assert run.progress["suppressions"]["checkpoint"] == {"suppression_type": "bounces", "offset": 2}+ assert self._suppressed() == {"[email protected]", "[email protected]"}++ def test_category_warnings_never_carry_raw_exception_text(+ self, mock_client_class: MagicMock, _start_workflow: MagicMock+ ) -> None:+ mock_client_class.return_value.get_subscription_topics.return_value = [+ {"id": 1, "identifier": "topic_1", "name": "Newsletter"},+ {"id": 2, "identifier": "topic_2", "name": "Product updates"},+ ]+ run = self._start(["categories"])++ with patch(+ f"{SERVICE}.MessageCategory.objects.update_or_create",+ side_effect=IntegrityError("Key (key)=(internal-detail) already exists."),+ ):+ runs.run_step(self.team.id, run.id, "categories", attempt=1)++ run.refresh_from_db()+ assert run.progress["categories"]["warnings"] == [+ "2 subscription topics couldn't be imported. Run the import again to retry them."+ ]++ @parameterized.expand(+ [+ ("provider_error", "get_suppressions", CustomerIOAPIError("error for [email protected]", status_code=503)),+ ("database_error", "save", IntegrityError("Key (identifier)=([email protected]) already exists.")),+ ]+ )+ def test_step_errors_shown_to_users_never_carry_raw_exception_text(+ self,+ mock_client_class: MagicMock,+ _start_workflow: MagicMock,+ _name: str,+ failing_call: str,+ error: Exception,+ ) -> None:+ mock_client = mock_client_class.return_value+ mock_client.get_suppressions.side_effect = lambda suppression_type, limit, offset: [{"email": "[email protected]"}]+ if failing_call == "get_suppressions":+ mock_client.get_suppressions.side_effect = error+ run = self._start(["suppressions"])++ with (+ patch(f"{SERVICE}.CustomerIOImportService.save_suppression_page", side_effect=error),+ self.assertRaises(Exception),+ ):+ runs.run_step(self.team.id, run.id, "suppressions", attempt=1)++ run.refresh_from_db()+ assert run.progress["suppressions"]["last_error"] is not None+ assert "[email protected]" not in run.progress["suppressions"]["last_error"]++ def test_steps_of_a_run_whose_workflow_failed_to_start_import_nothing(+ self, mock_client_class: MagicMock, start_workflow: MagicMock+ ) -> None:+ start_workflow.side_effect = RuntimeError("Temporal did not answer")+ with self.assertRaises(RuntimeError):+ self._start(["suppressions"])+ run = ProviderImportRun.objects.for_team(self.team.id).get()++ with self.assertRaises(runs.ImportRunClosed):+ runs.run_step(self.team.id, run.id, "suppressions", attempt=1)++ mock_client_class.return_value.get_suppressions.assert_not_called()+ assert ProviderImportRun.objects.for_team(self.team.id).filter(status="failed").count() == 1++ @parameterized.expand(+ [+ ("every_step_completed", ["completed", "completed"], "completed", ["completed", "completed"]),+ ("a_step_still_running", ["completed", "running"], "failed", ["completed", "failed"]),+ ("a_step_never_started", ["pending", "completed"], "failed", ["failed", "completed"]),+ ]+ )+ def test_finish_sets_the_run_status_from_its_steps_once(+ self,+ mock_client_class: MagicMock,+ _start_workflow: MagicMock,+ _name: str,+ step_statuses: list[str],+ expected_run_status: str,+ expected_step_statuses: list[str],+ ) -> None:+ run = self._start(["suppressions", "opt_outs"])+ for step, step_status in zip(run.steps, step_statuses):+ run.progress[step]["status"] = step_status+ run.save()++ with patch("products.messaging.backend.provider_imports.runs.report_team_action") as report:+ runs.finish_run(self.team.id, run.id)+ runs.finish_run(self.team.id, run.id)++ run.refresh_from_db()+ assert run.status == expected_run_status+ assert [run.progress[step]["status"] for step in run.steps] == expected_step_statuses+ assert run.finished_at is not None+ assert report.call_count == 1++ @time_machine.travel("2026-03-02T12:00:00Z", tick=False)+ def test_reading_an_expired_run_fails_it_and_its_unfinished_steps(+ self, mock_client_class: MagicMock, _start_workflow: MagicMock+ ) -> None:+ run = self._start(["suppressions"])+ ProviderImportRun.objects.for_team(self.team.id).filter(id=run.id).update(+ created_at=timezone.now() - runs.STALE_RUN_AGE - timedelta(minutes=1)+ )++ expired = runs.get_run(self.team.id, run.id)++ assert expired is not None+ assert expired.status == "failed"+ assert expired.progress["suppressions"]["status"] == "failed"diff --git a/products/messaging/turbo.json b/products/messaging/turbo.jsonindex 5a1fb406a1b9..92243a70f3cf 100644--- a/products/messaging/turbo.json+++ b/products/messaging/turbo.json@@ -7,6 +7,7 @@"backend/presentation/**","backend/routes.py","backend/tasks/**",+ "backend/temporal/**","backend/models/**","backend/migrations/**"],