Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(messaging): run provider imports in temporal (#124426)

Harley Alexander0b517fea8112
diff --git a/posthog/management/commands/start_temporal_worker.py b/posthog/management/commands/start_temporal_worker.py
index 7a0641f45718..f2a857bc443c 100644
--- a/posthog/management/commands/start_temporal_worker.py
+++ b/posthog/management/commands/start_temporal_worker.py
@@ -264,6 +264,10 @@ from products.managed_warehouse.backend.facade.temporal import (
ACTIVITIES as DUCKLAKE_COPY_ACTIVITIES,
WORKFLOWS as DUCKLAKE_COPY_WORKFLOWS,
)
+from products.messaging.backend.facade.temporal import (
+ ACTIVITIES as MESSAGING_IMPORT_ACTIVITIES,
+ WORKFLOWS as MESSAGING_IMPORT_WORKFLOWS,
+)
from products.metrics.backend.facade.temporal import (
ACTIVITIES as METRICS_ACTIVITIES,
WORKFLOWS as METRICS_WORKFLOWS,
@@ -471,6 +475,13 @@ _task_queue_specs = [
CANVAS_BUILD_WORKFLOWS,
CANVAS_BUILD_ACTIVITIES,
),
+ # Messaging provider imports. MESSAGING_IMPORT_TASK_QUEUE defaults to the general-purpose queue name (so it
+ # merges into that fleet until a dedicated worker exists).
+ (
+ settings.MESSAGING_IMPORT_TASK_QUEUE,
+ MESSAGING_IMPORT_WORKFLOWS,
+ MESSAGING_IMPORT_ACTIVITIES,
+ ),
(
settings.EXPERIMENTS_RECALCULATION_TASK_QUEUE,
EXPERIMENTS_RECALCULATION_WORKFLOWS,
diff --git a/posthog/settings/temporal.py b/posthog/settings/temporal.py
index 7dcb59a92d3b..536ae569164a 100644
--- a/posthog/settings/temporal.py
+++ b/posthog/settings/temporal.py
@@ -363,6 +363,11 @@ SIGNUP_ENRICHMENT_TASK_QUEUE = _set_temporal_task_queue(
# dispatching services. Setting it first strands builds on a pollerless queue until the
# workflow execution timeout closes them.
CANVAS_BUILD_TASK_QUEUE = _set_temporal_task_queue(os.getenv("CANVAS_BUILD_TASK_QUEUE", "general-purpose-task-queue"))
+# Messaging provider imports (for example Customer.io suppressions) share the general-purpose fleet. To move them to a
+# dedicated worker, deploy a worker polling the new queue before setting this env on the dispatching services.
+MESSAGING_IMPORT_TASK_QUEUE = _set_temporal_task_queue(
+ os.getenv("MESSAGING_IMPORT_TASK_QUEUE", "general-purpose-task-queue")
+)
EXPERIMENTS_RECALCULATION_TASK_QUEUE = _set_temporal_task_queue("experiments-recalculation-task-queue")
EXPERIMENTS_RECALCULATION_ACTIVITY_TASK_QUEUE = _set_temporal_task_queue(
"experiments-recalculation-activity-task-queue"
diff --git a/products/messaging/backend/facade/temporal.py b/products/messaging/backend/facade/temporal.py
new file mode 100644
index 000000000000..06e8102ad098
--- /dev/null
+++ b/products/messaging/backend/facade/temporal.py
@@ -0,0 +1,6 @@
+"""Temporal workflows and activities the provider import worker registers."""
+
+from products.messaging.backend.temporal.registry import (
+ ACTIVITIES as ACTIVITIES,
+ WORKFLOWS as WORKFLOWS,
+)
diff --git a/products/messaging/backend/provider_imports/__init__.py b/products/messaging/backend/provider_imports/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/messaging/backend/provider_imports/base.py b/products/messaging/backend/provider_imports/base.py
new file mode 100644
index 000000000000..6817dd8d76ba
--- /dev/null
+++ b/products/messaging/backend/provider_imports/base.py
@@ -0,0 +1,105 @@
+"""The contract every provider importer implements.
+
+A provider importer knows one messaging tool: how to check and store its credentials, and how to page through each
+kind of data it can import. It knows nothing about Temporal or run state. `runs.py` drives the steps, saves each page
+together with the step's progress, and resumes a retried step from the last saved checkpoint.
+"""
+
+from abc import ABC, abstractmethod
+from collections.abc import Callable, Iterator
+from dataclasses import field
+from typing import ClassVar
+
+from posthog.dataclasses import frozen
+
+# Where a step continues after a page. Each step defines its own keys. The values are JSON-safe because the
+# checkpoint is stored in the run's `progress`.
+StepCheckpoint = dict[str, str | int]
+
+
+class ImportStepError(Exception):
+ """A failure whose message is safe to show the user. The run shows a generic message for any other exception."""
+
+
+class ImportCredentialsMissing(ImportStepError):
+ """The request has no API key and none is stored. The message names what is missing."""
+
+
+class ImportCredentialsConflict(ImportStepError):
+ """A key is stored and the request sends another one. The message tells the caller to delete it first."""
+
+
+class ImportCredentialsInvalid(ImportStepError):
+ """The provider rejected the API key. A retry does not fix this."""
+
+
+class ImportProviderUnavailable(ImportStepError):
+ """The provider did not respond or returned a server error. A later retry can succeed."""
+
+
+class UnknownImportStep(ImportStepError):
+ """The provider has no step with this key."""
+
+
+@frozen
+class ImportStepSpec:
+ key: str
+ label: str
+ description: str
+
+
+@frozen
+class ImportCredentials:
+ api_key: str = field(repr=False)
+
+
+@frozen
+class ImportContext:
+ team_id: int
+ created_by_id: int | None
+ integration_id: int | None
+
+
+@frozen
+class SavedPage:
+ # How many records the page added or changed in PostHog. Records that already match do not count.
+ imported: int
+ # Problems with single records that do not fail the step.
+ warnings: tuple[str, ...] = ()
+
+
+@frozen
+class ImportPage:
+ # How many source records the page read.
+ processed: int
+ # Writes the page to PostHog. The runner calls it in the transaction that saves the step's progress, so the page
+ # and its counts and checkpoint commit together.
+ save: Callable[[], SavedPage]
+ # Where to continue when the step runs again after this page. None on the last page of the step.
+ checkpoint: StepCheckpoint | None = None
+
+
+class ImportProvider(ABC):
+ key: ClassVar[str]
+ label: ClassVar[str]
+ # The steps in the order they run. Each one is safe to run again over records it already imported.
+ steps: ClassVar[tuple[ImportStepSpec, ...]]
+
+ @abstractmethod
+ def check_credentials(self, team_id: int, api_key: str | None) -> ImportCredentials:
+ """Choose the API key to import with and check it with the provider. Without `api_key`, use the stored key.
+
+ This runs before the import claims its run, so it must not store anything. Raises the ImportCredentials*
+ errors, or ImportProviderUnavailable when the provider can't answer.
+ """
+
+ @abstractmethod
+ def store_credentials(self, team_id: int, credentials: ImportCredentials, created_by_id: int | None) -> int:
+ """Store checked credentials on first use and return the integration id.
+
+ This runs in the transaction that claims the run, so a request that loses the claim stores nothing.
+ """
+
+ @abstractmethod
+ def run_step(self, step: str, context: ImportContext, checkpoint: StepCheckpoint | None) -> Iterator[ImportPage]:
+ """Fetch one step's pages from `checkpoint`, or from the start when it is None, without saving them."""
diff --git a/products/messaging/backend/provider_imports/customerio.py b/products/messaging/backend/provider_imports/customerio.py
new file mode 100644
index 000000000000..479c823a2b38
--- /dev/null
+++ b/products/messaging/backend/provider_imports/customerio.py
@@ -0,0 +1,177 @@
+import logging
+from collections.abc import Iterator
+from functools import partial
+
+from products.messaging.backend.models.message_preferences import ALL_MESSAGE_PREFERENCE_CATEGORY_ID
+from products.messaging.backend.models.provider_import_run import ImportProviderKey
+from products.messaging.backend.provider_imports.base import (
+ ImportContext,
+ ImportCredentials,
+ ImportCredentialsConflict,
+ ImportCredentialsInvalid,
+ ImportCredentialsMissing,
+ ImportPage,
+ ImportProvider,
+ ImportProviderUnavailable,
+ ImportStepSpec,
+ SavedPage,
+ StepCheckpoint,
+ UnknownImportStep,
+)
+from products.messaging.backend.services import customerio_config
+from products.messaging.backend.services.customerio_client import CustomerIOAPIError
+from products.messaging.backend.services.customerio_import_service import (
+ CUSTOMERIO_SUPPRESSION_TYPES,
+ CustomerIOImportService,
+ GlobalUnsubscribePage,
+ SuppressionPage,
+)
+
+logger = logging.getLogger(__name__)
+
+SUPPRESSION_TYPE_ORDER = tuple(CUSTOMERIO_SUPPRESSION_TYPES)
+# The key check runs on the request path and tries both regions, so it gets a shorter timeout than the import steps.
+CREDENTIAL_CHECK_TIMEOUT_SECONDS = 10
+INVALID_CREDENTIALS_MESSAGE = (
+ "Customer.io rejected the App API key. Check that it is an App API key, that it has not expired, "
+ "and that you copied all of it."
+)
+
+
+class CustomerIOImportProvider(ImportProvider):
+ key = ImportProviderKey.CUSTOMERIO.value
+ label = ImportProviderKey.CUSTOMERIO.label
+ steps = (
+ ImportStepSpec(
+ key="categories",
+ label="Subscription topics",
+ description="Creates a message category for each Customer.io subscription topic.",
+ ),
+ ImportStepSpec(
+ key="suppressions",
+ label="Suppressed addresses",
+ description=(
+ "Adds the addresses that Customer.io's email provider bounced, marked as invalid, "
+ "or got a spam report for to the suppression list."
+ ),
+ ),
+ ImportStepSpec(
+ key="opt_outs",
+ label="Global unsubscribes",
+ description="Opts people who unsubscribed from all Customer.io messages out of all marketing messages.",
+ ),
+ )
+
+ def check_credentials(self, team_id: int, api_key: str | None) -> ImportCredentials:
+ try:
+ resolved_key = customerio_config.resolve_app_api_key(team_id, api_key)
+ except customerio_config.ConfigIncomplete as e:
+ raise ImportCredentialsMissing(str(e)) from e
+ except customerio_config.ConfigConflict as e:
+ raise ImportCredentialsConflict(str(e)) from e
+ try:
+ connected = CustomerIOImportService(team_id, resolved_key, None).connect(
+ timeout=CREDENTIAL_CHECK_TIMEOUT_SECONDS
+ )
+ except CustomerIOAPIError as e:
+ raise ImportProviderUnavailable(_api_error_message(e)) from e
+ if not connected:
+ raise ImportCredentialsInvalid(INVALID_CREDENTIALS_MESSAGE)
+ return ImportCredentials(api_key=resolved_key)
+
+ def store_credentials(self, team_id: int, credentials: ImportCredentials, created_by_id: int | None) -> int:
+ return customerio_config.save_app_integration(team_id, credentials.api_key, created_by_id).id
+
+ def run_step(self, step: str, context: ImportContext, checkpoint: StepCheckpoint | None) -> Iterator[ImportPage]:
+ try:
+ match step:
+ case "categories":
+ yield from self._fetch_categories(self._connect(context))
+ case "suppressions":
+ yield from self._fetch_suppressions(self._connect(context), checkpoint)
+ case "opt_outs":
+ yield from self._fetch_opt_outs(self._connect(context), checkpoint)
+ case _:
+ raise UnknownImportStep(f"Customer.io has no import step named {step}.")
+ except CustomerIOAPIError as e:
+ raise ImportProviderUnavailable(_api_error_message(e)) from e
+
+ def _connect(self, context: ImportContext) -> CustomerIOImportService:
+ api_key = (
+ customerio_config.stored_app_api_key(context.team_id, context.integration_id)
+ if context.integration_id is not None
+ else None
+ )
+ if not api_key:
+ raise ImportCredentialsMissing(
+ "The Customer.io App API key was deleted. Add it again and start a new import."
+ )
+ service = CustomerIOImportService(context.team_id, api_key, context.created_by_id)
+ if not service.connect():
+ raise ImportCredentialsInvalid(INVALID_CREDENTIALS_MESSAGE)
+ return service
+
+ def _fetch_categories(self, service: CustomerIOImportService) -> Iterator[ImportPage]:
+ assert service.client is not None
+ topics = service.client.get_subscription_topics()
+ yield ImportPage(processed=len(topics), save=partial(_save_categories, service, topics))
+
+ def _fetch_suppressions(
+ self, service: CustomerIOImportService, checkpoint: StepCheckpoint | None
+ ) -> Iterator[ImportPage]:
+ resume_type = checkpoint.get("suppression_type") if checkpoint else None
+ start_index = SUPPRESSION_TYPE_ORDER.index(resume_type) if resume_type in SUPPRESSION_TYPE_ORDER else 0
+ for index in range(start_index, len(SUPPRESSION_TYPE_ORDER)):
+ suppression_type = SUPPRESSION_TYPE_ORDER[index]
+ offset = int(checkpoint.get("offset", 0)) if checkpoint and suppression_type == resume_type else 0
+ for page in service.fetch_suppression_pages(suppression_type, offset):
+ next_checkpoint: StepCheckpoint | None
+ if not page.is_last:
+ next_checkpoint = {"suppression_type": suppression_type, "offset": page.next_offset}
+ elif index + 1 < len(SUPPRESSION_TYPE_ORDER):
+ next_checkpoint = {"suppression_type": SUPPRESSION_TYPE_ORDER[index + 1], "offset": 0}
+ else:
+ next_checkpoint = None
+ yield ImportPage(
+ processed=page.entries, save=partial(_save_suppressions, service, page), checkpoint=next_checkpoint
+ )
+
+ def _fetch_opt_outs(
+ self, service: CustomerIOImportService, checkpoint: StepCheckpoint | None
+ ) -> Iterator[ImportPage]:
+ start = checkpoint.get("start") if checkpoint else None
+ for page in service.fetch_global_unsubscribe_pages(start=str(start) if start else None):
+ yield ImportPage(
+ processed=page.customers,
+ save=partial(_save_opt_outs, service, page),
+ checkpoint={"start": page.next_start} if page.next_start else None,
+ )
+
+
+def _save_categories(service: CustomerIOImportService, topics: list[dict]) -> SavedPage:
+ service.import_categories(topics)
+ errors = service.progress["errors"]
+ if not errors:
+ return SavedPage(imported=service.progress["categories_created"])
+ # The service's errors hold raw exception text, which stays in the server log.
+ logger.warning("Customer.io topics failed to import", extra={"team_id": service.team_id, "errors": errors})
+ return SavedPage(
+ imported=service.progress["categories_created"],
+ warnings=(f"{len(errors)} subscription topics couldn't be imported. Run the import again to retry them.",),
+ )
+
+
+def _save_suppressions(service: CustomerIOImportService, page: SuppressionPage) -> SavedPage:
+ return SavedPage(imported=service.save_suppression_page(page))
+
+
+def _save_opt_outs(service: CustomerIOImportService, page: GlobalUnsubscribePage) -> SavedPage:
+ if not page.emails:
+ return SavedPage(imported=0)
+ return SavedPage(imported=service.save_opt_outs(list(page.emails), [ALL_MESSAGE_PREFERENCE_CATEGORY_ID]))
+
+
+def _api_error_message(error: CustomerIOAPIError) -> str:
+ if error.status_code is None:
+ return "Customer.io did not respond."
+ return f"Customer.io returned HTTP {error.status_code}."
diff --git a/products/messaging/backend/provider_imports/registry.py b/products/messaging/backend/provider_imports/registry.py
new file mode 100644
index 000000000000..ec380a4a35b6
--- /dev/null
+++ b/products/messaging/backend/provider_imports/registry.py
@@ -0,0 +1,16 @@
+from products.messaging.backend.provider_imports.base import ImportProvider
+from products.messaging.backend.provider_imports.customerio import CustomerIOImportProvider
+
+# Add a provider here, and its key to ImportProviderKey, to make it importable.
+PROVIDERS: dict[str, ImportProvider] = {provider.key: provider for provider in (CustomerIOImportProvider(),)}
+
+
+class UnknownImportProvider(Exception):
+ pass
+
+
+def get_provider(key: str) -> ImportProvider:
+ provider = PROVIDERS.get(key)
+ if provider is None:
+ raise UnknownImportProvider(key)
+ return provider
diff --git a/products/messaging/backend/provider_imports/runs.py b/products/messaging/backend/provider_imports/runs.py
new file mode 100644
index 000000000000..e345ae83e250
--- /dev/null
+++ b/products/messaging/backend/provider_imports/runs.py
@@ -0,0 +1,276 @@
+"""Start provider import runs, run their steps page by page, and record the outcome."""
+
+import logging
+from collections.abc import Callable
+from datetime import timedelta
+from typing import Any
+from uuid import UUID
+
+from django.db import IntegrityError, transaction
+from django.db.models import QuerySet
+from django.utils import timezone
+
+from posthog.dataclasses import frozen
+from posthog.event_usage import report_team_action
+from posthog.models import Team
+
+from products.messaging.backend.models.provider_import_run import (
+ ACTIVE_IMPORT_STATUSES,
+ ImportStepStatus,
+ ProviderImportRun,
+ ProviderImportStatus,
+)
+from products.messaging.backend.provider_imports.base import (
+ ImportContext,
+ ImportPage,
+ ImportProvider,
+ ImportStepError,
+ StepCheckpoint,
+)
+from products.messaging.backend.provider_imports.registry import get_provider
+from products.messaging.backend.temporal.client import RUN_EXECUTION_TIMEOUT, start_provider_import_workflow
+
+logger = logging.getLogger(__name__)
+
+# A run still active after the workflow's execution timeout has no workflow left to finish it.
+STALE_RUN_AGE = RUN_EXECUTION_TIMEOUT + timedelta(hours=1)
+MAX_STEP_WARNINGS = 50
+UNEXPECTED_ERROR_MESSAGE = "An unexpected error interrupted this step."
+
+
+class ProviderImportAlreadyRunning(Exception):
+ pass
+
+
+class InvalidImportSteps(Exception):
+ pass
+
+
+class ImportRunClosed(Exception):
+ """The run already ended, for example because its workflow failed to start. Its steps must not import."""
+
+
+class StaleImportAttempt(Exception):
+ """A newer attempt of the step took over, so this attempt must stop without writing."""
+
+
+@frozen
+class _StepClaim:
+ completed: bool
+ checkpoint: StepCheckpoint | None
+
+
+def start_run(
+ team_id: int, provider_key: str, steps: list[str], api_key: str | None, created_by_id: int | None
+) -> ProviderImportRun:
+ """Check the credentials, claim a queued run, store the credentials, and start the run's workflow.
+
+ Raises UnknownImportProvider, InvalidImportSteps, ProviderImportAlreadyRunning, the ImportCredentials* errors and
+ ImportProviderUnavailable.
+ """
+ provider = get_provider(provider_key)
+ ordered_steps = _order_steps(provider, steps)
+
+ _fail_stale_runs(team_id)
+ if _runs(team_id).filter(provider=provider.key, status__in=ACTIVE_IMPORT_STATUSES).exists():
+ raise ProviderImportAlreadyRunning(f"A {provider.label} import is already running.")
+
+ credentials = provider.check_credentials(team_id, api_key)
+ try:
+ with transaction.atomic():
+ # Create the run before storing the credentials. A concurrent start then waits on the active-run index
+ # and fails, so only the request that owns the run stores its key.
+ run = _runs(team_id).create(
+ team_id=team_id,
+ provider=provider.key,
+ steps=ordered_steps,
+ created_by_id=created_by_id,
+ progress={step: _new_step_progress() for step in ordered_steps},
+ )
+ run.integration_id = provider.store_credentials(team_id, credentials, created_by_id)
+ run.save(update_fields=["integration"])
+ except IntegrityError as e:
+ raise ProviderImportAlreadyRunning(f"A {provider.label} import is already running.") from e
+
+ try:
+ start_provider_import_workflow(team_id=team_id, run_id=run.id, steps=ordered_steps)
+ except Exception:
+ # A queued run without a workflow would block every later import until it goes stale. If Temporal started
+ # the workflow but the response got lost, its steps find the run closed and import nothing.
+ _close_run(team_id, run.id)
+ raise
+ return run
+
+
+def run_step(team_id: int, run_id: UUID, step: str, attempt: int) -> None:
+ """Run one step from its saved checkpoint, and save each page together with the step's progress.
+
+ `attempt` is the Temporal attempt number. An older attempt can keep running after Temporal starts a newer one,
+ so each page write checks that its attempt still owns the step.
+ """
+ run = _runs(team_id).get(id=run_id)
+ provider = get_provider(run.provider)
+ claim = _claim_step(team_id, run_id, step, attempt)
+ if claim.completed:
+ return
+
+ context = ImportContext(team_id=team_id, created_by_id=run.created_by_id, integration_id=run.integration_id)
+ try:
+ for page in provider.run_step(step, context, claim.checkpoint):
+ _save_page(team_id, run_id, step, attempt, page)
+ if page.checkpoint is None:
+ return
+ _update_owned_step(team_id, run_id, step, attempt, _mark_completed)
+ except (StaleImportAttempt, ImportRunClosed):
+ raise
+ except Exception as e:
+ if not isinstance(e, ImportStepError):
+ logger.exception("Provider import step failed", extra={"run_id": str(run_id), "step": step})
+ last_error = str(e)[:500] if isinstance(e, ImportStepError) else UNEXPECTED_ERROR_MESSAGE
+ try:
+ _update_owned_step(team_id, run_id, step, attempt, lambda progress: progress.update(last_error=last_error))
+ except (StaleImportAttempt, ImportRunClosed):
+ pass
+ raise
+
+
+def finish_run(team_id: int, run_id: UUID) -> None:
+ """Fail the steps that did not complete, and set the run's final status. A run that already ended stays as it is."""
+ _close_run(team_id, run_id)
+
+
+def list_runs(team_id: int, limit: int) -> list[ProviderImportRun]:
+ _fail_stale_runs(team_id)
+ return list(_runs(team_id).order_by("-created_at")[:limit])
+
+
+def get_run(team_id: int, run_id: UUID) -> ProviderImportRun | None:
+ _fail_stale_runs(team_id)
+ return _runs(team_id).filter(id=run_id).first()
+
+
+def _runs(team_id: int) -> QuerySet[ProviderImportRun]:
+ return ProviderImportRun.objects.for_team(team_id)
+
+
+def _order_steps(provider: ImportProvider, steps: list[str]) -> list[str]:
+ known = [spec.key for spec in provider.steps]
+ unknown = sorted(set(steps) - set(known))
+ if unknown:
+ raise InvalidImportSteps(f"{provider.label} has no step named {', '.join(unknown)}.")
+ if not steps:
+ raise InvalidImportSteps("Select at least one thing to import.")
+ return [key for key in known if key in steps]
+
+
+def _fail_stale_runs(team_id: int) -> None:
+ stale_ids = list(
+ _runs(team_id)
+ .filter(status__in=ACTIVE_IMPORT_STATUSES, created_at__lt=timezone.now() - STALE_RUN_AGE)
+ .values_list("id", flat=True)
+ )
+ for run_id in stale_ids:
+ _close_run(team_id, run_id)
+
+
+def _close_run(team_id: int, run_id: UUID) -> None:
+ with transaction.atomic():
+ run = _runs(team_id).select_for_update().get(id=run_id)
+ if run.status not in ACTIVE_IMPORT_STATUSES:
+ return
+ for step in run.steps:
+ progress = run.progress.setdefault(step, _new_step_progress())
+ if progress["status"] != ImportStepStatus.COMPLETED:
+ progress["status"] = ImportStepStatus.FAILED
+ completed = all(run.progress[step]["status"] == ImportStepStatus.COMPLETED for step in run.steps)
+ run.status = ProviderImportStatus.COMPLETED if completed else ProviderImportStatus.FAILED
+ run.finished_at = timezone.now()
+ run.save(update_fields=["progress", "status", "finished_at"])
+
+ _report_finished(run)
+
+
+def _claim_step(team_id: int, run_id: UUID, step: str, attempt: int) -> _StepClaim:
+ with transaction.atomic():
+ run = _runs(team_id).select_for_update().get(id=run_id)
+ if run.status not in ACTIVE_IMPORT_STATUSES:
+ raise ImportRunClosed(f"Import run {run_id} already ended.")
+ progress = run.progress.setdefault(step, _new_step_progress())
+ if progress["status"] == ImportStepStatus.COMPLETED:
+ return _StepClaim(completed=True, checkpoint=None)
+ progress.update(status=ImportStepStatus.RUNNING.value, attempt=attempt)
+ update_fields = ["progress"]
+ if run.status == ProviderImportStatus.QUEUED:
+ run.status = ProviderImportStatus.RUNNING
+ run.started_at = timezone.now()
+ update_fields += ["status", "started_at"]
+ run.save(update_fields=update_fields)
+ return _StepClaim(completed=False, checkpoint=progress.get("checkpoint"))
+
+
+def _save_page(team_id: int, run_id: UUID, step: str, attempt: int, page: ImportPage) -> None:
+ with transaction.atomic():
+ run, progress = _lock_owned_step(team_id, run_id, step, attempt)
+ saved = page.save()
+ progress["processed"] += page.processed
+ progress["imported"] += saved.imported
+ progress["warnings"] = (progress["warnings"] + list(saved.warnings))[:MAX_STEP_WARNINGS]
+ progress["checkpoint"] = page.checkpoint
+ if page.checkpoint is None:
+ _mark_completed(progress)
+ run.save(update_fields=["progress"])
+
+
+def _update_owned_step(
+ team_id: int, run_id: UUID, step: str, attempt: int, change: Callable[[dict[str, Any]], None]
+) -> None:
+ with transaction.atomic():
+ run, progress = _lock_owned_step(team_id, run_id, step, attempt)
+ change(progress)
+ run.save(update_fields=["progress"])
+
+
+def _lock_owned_step(team_id: int, run_id: UUID, step: str, attempt: int) -> tuple[ProviderImportRun, dict[str, Any]]:
+ run = _runs(team_id).select_for_update().get(id=run_id)
+ if run.status not in ACTIVE_IMPORT_STATUSES:
+ raise ImportRunClosed(f"Import run {run_id} ended while step {step} was running.")
+ progress = run.progress.setdefault(step, _new_step_progress())
+ if progress.get("attempt") != attempt:
+ raise StaleImportAttempt(f"Attempt {attempt} of step {step} no longer owns import run {run_id}.")
+ return run, progress
+
+
+def _new_step_progress() -> dict[str, Any]:
+ return {
+ "status": ImportStepStatus.PENDING.value,
+ "processed": 0,
+ "imported": 0,
+ "warnings": [],
+ "last_error": None,
+ "checkpoint": None,
+ "attempt": None,
+ }
+
+
+def _mark_completed(progress: dict[str, Any]) -> None:
+ progress.update(status=ImportStepStatus.COMPLETED.value, last_error=None, checkpoint=None)
+
+
+def _report_finished(run: ProviderImportRun) -> None:
+ try:
+ report_team_action(
+ Team.objects.get(id=run.team_id),
+ "messaging provider import finished",
+ {
+ "provider": run.provider,
+ "status": run.status,
+ "steps": run.steps,
+ "duration_seconds": int((run.finished_at - run.created_at).total_seconds())
+ if run.finished_at
+ else None,
+ **{f"{step}_status": run.progress[step]["status"] for step in run.steps},
+ **{f"{step}_imported": run.progress[step]["imported"] for step in run.steps},
+ },
+ )
+ except Exception:
+ logger.exception("Failed to report the provider import outcome")
diff --git a/products/messaging/backend/temporal/__init__.py b/products/messaging/backend/temporal/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/messaging/backend/temporal/activities.py b/products/messaging/backend/temporal/activities.py
new file mode 100644
index 000000000000..4ae20837797b
--- /dev/null
+++ b/products/messaging/backend/temporal/activities.py
@@ -0,0 +1,22 @@
+from uuid import UUID
+
+from temporalio import activity
+
+from posthog.temporal.common.heartbeat_sync import HeartbeaterSync
+from posthog.temporal.common.utils import asyncify
+
+from products.messaging.backend.provider_imports.runs import finish_run, run_step
+from products.messaging.backend.temporal.inputs import ProviderImportRunInputs, ProviderImportStepInputs
+
+
[email protected](name="messaging-provider-import.run_step")
+@asyncify
+def run_provider_import_step_activity(inputs: ProviderImportStepInputs) -> None:
+ with HeartbeaterSync():
+ run_step(inputs.team_id, UUID(inputs.run_id), inputs.step, activity.info().attempt)
+
+
[email protected](name="messaging-provider-import.finish_run")
+@asyncify
+def finish_provider_import_run_activity(inputs: ProviderImportRunInputs) -> None:
+ finish_run(inputs.team_id, UUID(inputs.run_id))
diff --git a/products/messaging/backend/temporal/client.py b/products/messaging/backend/temporal/client.py
new file mode 100644
index 000000000000..8b9880704080
--- /dev/null
+++ b/products/messaging/backend/temporal/client.py
@@ -0,0 +1,32 @@
+"""Start the provider import workflow from Django."""
+
+from datetime import timedelta
+from uuid import UUID
+
+from django.conf import settings
+
+from asgiref.sync import async_to_sync
+from temporalio.common import RetryPolicy
+
+from posthog.temporal.common.client import async_connect
+
+from products.messaging.backend.temporal.inputs import ProviderImportWorkflowInputs
+
+PROVIDER_IMPORT_WORKFLOW = "messaging-provider-import"
+
+# Bounds the whole run, so a run stranded on a queue with no pollers closes instead of blocking new imports forever.
+RUN_EXECUTION_TIMEOUT = timedelta(hours=24)
+
+
+@async_to_sync
+async def start_provider_import_workflow(team_id: int, run_id: UUID, steps: list[str]) -> None:
+ client = await async_connect()
+ await client.start_workflow(
+ PROVIDER_IMPORT_WORKFLOW,
+ ProviderImportWorkflowInputs(team_id=team_id, run_id=str(run_id), steps=steps),
+ id=f"{PROVIDER_IMPORT_WORKFLOW}-{run_id}",
+ task_queue=settings.MESSAGING_IMPORT_TASK_QUEUE,
+ execution_timeout=RUN_EXECUTION_TIMEOUT,
+ # Retries live on the activities. A workflow retry would run steps that already completed again.
+ retry_policy=RetryPolicy(maximum_attempts=1),
+ )
diff --git a/products/messaging/backend/temporal/inputs.py b/products/messaging/backend/temporal/inputs.py
new file mode 100644
index 000000000000..6345b4c9a624
--- /dev/null
+++ b/products/messaging/backend/temporal/inputs.py
@@ -0,0 +1,21 @@
+from posthog.dataclasses import frozen
+
+
+@frozen
+class ProviderImportWorkflowInputs:
+ team_id: int
+ run_id: str
+ steps: list[str]
+
+
+@frozen
+class ProviderImportStepInputs:
+ team_id: int
+ run_id: str
+ step: str
+
+
+@frozen
+class ProviderImportRunInputs:
+ team_id: int
+ run_id: str
diff --git a/products/messaging/backend/temporal/registry.py b/products/messaging/backend/temporal/registry.py
new file mode 100644
index 000000000000..68eea42342d9
--- /dev/null
+++ b/products/messaging/backend/temporal/registry.py
@@ -0,0 +1,16 @@
+"""Workflow and activity registration lists for the provider import worker.
+
+Core reaches these through ``facade/temporal.py``.
+"""
+
+from products.messaging.backend.temporal.activities import (
+ finish_provider_import_run_activity,
+ run_provider_import_step_activity,
+)
+from products.messaging.backend.temporal.workflow import ProviderImportWorkflow
+
+WORKFLOWS = [ProviderImportWorkflow]
+
+ACTIVITIES = [run_provider_import_step_activity, finish_provider_import_run_activity]
+
+__all__ = ["WORKFLOWS", "ACTIVITIES"]
diff --git a/products/messaging/backend/temporal/workflow.py b/products/messaging/backend/temporal/workflow.py
new file mode 100644
index 000000000000..690568ff8b60
--- /dev/null
+++ b/products/messaging/backend/temporal/workflow.py
@@ -0,0 +1,66 @@
+from datetime import timedelta
+
+import temporalio.workflow
+from temporalio import workflow
+from temporalio.common import RetryPolicy
+from temporalio.exceptions import ActivityError
+
+from posthog.temporal.common.base import PostHogWorkflow
+
+with temporalio.workflow.unsafe.imports_passed_through():
+ from products.messaging.backend.temporal.activities import (
+ finish_provider_import_run_activity,
+ run_provider_import_step_activity,
+ )
+ from products.messaging.backend.temporal.inputs import (
+ ProviderImportRunInputs,
+ ProviderImportStepInputs,
+ ProviderImportWorkflowInputs,
+ )
+
+# A step pages through a whole provider list, which can run for hours. The heartbeat timeout finds a dead worker
+# quickly, and the step continues from its last saved page on the next attempt.
+STEP_TIMEOUT = timedelta(hours=6)
+STEP_HEARTBEAT_TIMEOUT = timedelta(minutes=2)
+STEP_RETRY_POLICY = RetryPolicy(
+ maximum_attempts=5,
+ initial_interval=timedelta(seconds=10),
+ backoff_coefficient=2.0,
+ maximum_interval=timedelta(minutes=5),
+ non_retryable_error_types=[
+ "ImportCredentialsMissing",
+ "ImportCredentialsInvalid",
+ "ImportRunClosed",
+ "StaleImportAttempt",
+ "UnknownImportStep",
+ "UnknownImportProvider",
+ ],
+)
+FINISH_RETRY_POLICY = RetryPolicy(maximum_attempts=5, initial_interval=timedelta(seconds=5))
+
+
[email protected](name="messaging-provider-import")
+class ProviderImportWorkflow(PostHogWorkflow):
+ inputs_cls = ProviderImportWorkflowInputs
+
+ @workflow.run
+ async def run(self, inputs: ProviderImportWorkflowInputs) -> None:
+ for step in inputs.steps:
+ try:
+ await workflow.execute_activity(
+ run_provider_import_step_activity,
+ ProviderImportStepInputs(team_id=inputs.team_id, run_id=inputs.run_id, step=step),
+ start_to_close_timeout=STEP_TIMEOUT,
+ heartbeat_timeout=STEP_HEARTBEAT_TIMEOUT,
+ retry_policy=STEP_RETRY_POLICY,
+ )
+ except ActivityError:
+ # The steps import independent data, so one failed step does not stop the others.
+ workflow.logger.warning("Provider import step failed", extra={"run_id": inputs.run_id, "step": step})
+
+ await workflow.execute_activity(
+ finish_provider_import_run_activity,
+ ProviderImportRunInputs(team_id=inputs.team_id, run_id=inputs.run_id),
+ start_to_close_timeout=timedelta(minutes=1),
+ retry_policy=FINISH_RETRY_POLICY,
+ )
diff --git a/products/messaging/backend/tests/test_provider_import_workflow.py b/products/messaging/backend/tests/test_provider_import_workflow.py
new file mode 100644
index 000000000000..fc6b075c41c7
--- /dev/null
+++ b/products/messaging/backend/tests/test_provider_import_workflow.py
@@ -0,0 +1,55 @@
+import uuid
+import logging
+from datetime import timedelta
+
+import pytest
+
+from temporalio import activity
+from temporalio.exceptions import ApplicationError
+from temporalio.testing import WorkflowEnvironment
+from temporalio.worker import UnsandboxedWorkflowRunner, Worker
+
+from products.messaging.backend.temporal.inputs import (
+ ProviderImportRunInputs,
+ ProviderImportStepInputs,
+ ProviderImportWorkflowInputs,
+)
+from products.messaging.backend.temporal.workflow import ProviderImportWorkflow
+
+
+async def test_a_failed_step_does_not_stop_later_steps_or_the_finish(caplog: pytest.LogCaptureFixture) -> None:
+ caplog.set_level(logging.INFO, logger="temporalio.activity")
+ caplog.set_level(logging.INFO, logger="temporalio.workflow")
+ attempted_steps: list[str] = []
+ finished_runs: list[str] = []
+
+ @activity.defn(name="messaging-provider-import.run_step")
+ async def run_step(inputs: ProviderImportStepInputs) -> None:
+ attempted_steps.append(inputs.step)
+ if inputs.step == "categories":
+ raise ApplicationError("The key was deleted", type="ImportCredentialsMissing")
+
+ @activity.defn(name="messaging-provider-import.finish_run")
+ async def finish_run(inputs: ProviderImportRunInputs) -> None:
+ finished_runs.append(inputs.run_id)
+
+ run_id = str(uuid.uuid4())
+ async with await WorkflowEnvironment.start_time_skipping() as env:
+ async with Worker(
+ env.client,
+ task_queue="provider-import-test",
+ workflows=[ProviderImportWorkflow],
+ activities=[run_step, finish_run],
+ workflow_runner=UnsandboxedWorkflowRunner(),
+ ):
+ await env.client.execute_workflow(
+ ProviderImportWorkflow.run,
+ ProviderImportWorkflowInputs(team_id=1, run_id=run_id, steps=["categories", "suppressions"]),
+ id=f"provider-import-test-{run_id}",
+ task_queue="provider-import-test",
+ execution_timeout=timedelta(minutes=5),
+ )
+
+ assert attempted_steps == ["categories", "suppressions"]
+ assert finished_runs == [run_id]
diff --git a/products/messaging/backend/tests/test_provider_imports.py b/products/messaging/backend/tests/test_provider_imports.py
new file mode 100644
index 000000000000..be224aaff244
--- /dev/null
+++ b/products/messaging/backend/tests/test_provider_imports.py
@@ -0,0 +1,237 @@
+from datetime import timedelta
+
+import time_machine
+from posthog.test.base import BaseTest
+from unittest.mock import MagicMock, patch
+
+from django.db import IntegrityError
+from django.utils import timezone
+
+from parameterized import parameterized
+
+from products.messaging.backend.models.message_suppression import MessageSuppression
+from products.messaging.backend.models.provider_import_run import ProviderImportRun
+from products.messaging.backend.provider_imports import runs
+from products.messaging.backend.provider_imports.base import ImportProviderUnavailable
+from products.messaging.backend.services.customerio_client import CustomerIOAPIError
+
+SERVICE = "products.messaging.backend.services.customerio_import_service"
+
+
+@patch(f"{SERVICE}.SUPPRESSION_PAGE_DELAY_SECONDS", 0)
+@patch(f"{SERVICE}.SUPPRESSION_PAGE_SIZE", 2)
+@patch("products.messaging.backend.provider_imports.runs.start_provider_import_workflow")
+@patch(f"{SERVICE}.CustomerIOClient")
+class TestProviderImportRuns(BaseTest):
+ pages = {
+ ("bounces", 0): [{"email": "[email protected]"}, {"email": "[email protected]"}],
+ ("bounces", 2): [{"email": "[email protected]"}],
+ ("spam_reports", 0): [{"email": "[email protected]"}],
+ }
+
+ def _start(self, steps: list[str]) -> ProviderImportRun:
+ return runs.start_run(self.team.id, "customerio", steps, "app-key", self.user.id)
+
+ def _suppressed(self) -> set[str]:
+ return set(MessageSuppression.objects.for_team(self.team.id).values_list("identifier", flat=True))
+
+ def test_suppressions_step_resumes_from_the_last_saved_page_and_does_not_rerun_once_completed(
+ self, mock_client_class: MagicMock, _start_workflow: MagicMock
+ ) -> None:
+ failures = {("bounces", 2): CustomerIOAPIError("Customer.io API error", status_code=503)}
+
+ def get_suppressions(suppression_type: str, limit: int, offset: int) -> list[dict]:
+ if error := failures.pop((suppression_type, offset), None):
+ raise error
+ return self.pages.get((suppression_type, offset), [])
+
+ mock_client = mock_client_class.return_value
+ mock_client.get_suppressions.side_effect = get_suppressions
+ run = self._start(["suppressions"])
+
+ with self.assertRaises(ImportProviderUnavailable):
+ runs.run_step(self.team.id, run.id, "suppressions", attempt=1)
+
+ run.refresh_from_db()
+ assert run.status == "running"
+ assert run.progress["suppressions"]["checkpoint"] == {"suppression_type": "bounces", "offset": 2}
+ assert run.progress["suppressions"]["last_error"] == "Customer.io returned HTTP 503."
+
+ mock_client.get_suppressions.reset_mock()
+ runs.run_step(self.team.id, run.id, "suppressions", attempt=2)
+
+ assert [(call.args[0], call.kwargs["offset"]) for call in mock_client.get_suppressions.call_args_list] == [
+ ("bounces", 2),
+ ("invalid_emails", 0),
+ ("spam_reports", 0),
+ ]
+ run.refresh_from_db()
+ assert run.progress["suppressions"] == {
+ "status": "completed",
+ "processed": 4,
+ "imported": 4,
+ "warnings": [],
+ "last_error": None,
+ "checkpoint": None,
+ "attempt": 2,
+ }
+
+ mock_client.get_suppressions.reset_mock()
+ runs.run_step(self.team.id, run.id, "suppressions", attempt=3)
+
+ mock_client.get_suppressions.assert_not_called()
+
+ def test_an_older_attempt_stops_without_writing_once_a_newer_attempt_takes_over(
+ self, mock_client_class: MagicMock, _start_workflow: MagicMock
+ ) -> None:
+ run = self._start(["suppressions"])
+ newer_attempt_started = False
+
+ def get_suppressions(suppression_type: str, limit: int, offset: int) -> list[dict]:
+ nonlocal newer_attempt_started
+ if (suppression_type, offset) == ("bounces", 2) and not newer_attempt_started:
+ newer_attempt_started = True
+ runs.run_step(self.team.id, run.id, "suppressions", attempt=2)
+ return self.pages.get((suppression_type, offset), [])
+
+ mock_client_class.return_value.get_suppressions.side_effect = get_suppressions
+
+ with self.assertRaises(runs.StaleImportAttempt):
+ runs.run_step(self.team.id, run.id, "suppressions", attempt=1)
+
+ run.refresh_from_db()
+ assert run.progress["suppressions"]["attempt"] == 2
+ assert run.progress["suppressions"]["status"] == "completed"
+ assert run.progress["suppressions"]["processed"] == 4
+
+ def test_an_attempt_stops_without_writing_once_its_run_closes(
+ self, mock_client_class: MagicMock, _start_workflow: MagicMock
+ ) -> None:
+ run = self._start(["suppressions"])
+
+ def get_suppressions(suppression_type: str, limit: int, offset: int) -> list[dict]:
+ if (suppression_type, offset) == ("bounces", 2):
+ runs.finish_run(self.team.id, run.id)
+ return self.pages.get((suppression_type, offset), [])
+
+ mock_client_class.return_value.get_suppressions.side_effect = get_suppressions
+
+ with self.assertRaises(runs.ImportRunClosed):
+ runs.run_step(self.team.id, run.id, "suppressions", attempt=1)
+
+ run.refresh_from_db()
+ assert run.status == "failed"
+ assert run.progress["suppressions"]["checkpoint"] == {"suppression_type": "bounces", "offset": 2}
+ assert self._suppressed() == {"[email protected]", "[email protected]"}
+
+ def test_category_warnings_never_carry_raw_exception_text(
+ self, mock_client_class: MagicMock, _start_workflow: MagicMock
+ ) -> None:
+ mock_client_class.return_value.get_subscription_topics.return_value = [
+ {"id": 1, "identifier": "topic_1", "name": "Newsletter"},
+ {"id": 2, "identifier": "topic_2", "name": "Product updates"},
+ ]
+ run = self._start(["categories"])
+
+ with patch(
+ f"{SERVICE}.MessageCategory.objects.update_or_create",
+ side_effect=IntegrityError("Key (key)=(internal-detail) already exists."),
+ ):
+ runs.run_step(self.team.id, run.id, "categories", attempt=1)
+
+ run.refresh_from_db()
+ assert run.progress["categories"]["warnings"] == [
+ "2 subscription topics couldn't be imported. Run the import again to retry them."
+ ]
+
+ @parameterized.expand(
+ [
+ ("provider_error", "get_suppressions", CustomerIOAPIError("error for [email protected]", status_code=503)),
+ ("database_error", "save", IntegrityError("Key (identifier)=([email protected]) already exists.")),
+ ]
+ )
+ def test_step_errors_shown_to_users_never_carry_raw_exception_text(
+ self,
+ mock_client_class: MagicMock,
+ _start_workflow: MagicMock,
+ _name: str,
+ failing_call: str,
+ error: Exception,
+ ) -> None:
+ mock_client = mock_client_class.return_value
+ mock_client.get_suppressions.side_effect = lambda suppression_type, limit, offset: [{"email": "[email protected]"}]
+ if failing_call == "get_suppressions":
+ mock_client.get_suppressions.side_effect = error
+ run = self._start(["suppressions"])
+
+ with (
+ patch(f"{SERVICE}.CustomerIOImportService.save_suppression_page", side_effect=error),
+ self.assertRaises(Exception),
+ ):
+ runs.run_step(self.team.id, run.id, "suppressions", attempt=1)
+
+ run.refresh_from_db()
+ assert run.progress["suppressions"]["last_error"] is not None
+ assert "[email protected]" not in run.progress["suppressions"]["last_error"]
+
+ def test_steps_of_a_run_whose_workflow_failed_to_start_import_nothing(
+ self, mock_client_class: MagicMock, start_workflow: MagicMock
+ ) -> None:
+ start_workflow.side_effect = RuntimeError("Temporal did not answer")
+ with self.assertRaises(RuntimeError):
+ self._start(["suppressions"])
+ run = ProviderImportRun.objects.for_team(self.team.id).get()
+
+ with self.assertRaises(runs.ImportRunClosed):
+ runs.run_step(self.team.id, run.id, "suppressions", attempt=1)
+
+ mock_client_class.return_value.get_suppressions.assert_not_called()
+ assert ProviderImportRun.objects.for_team(self.team.id).filter(status="failed").count() == 1
+
+ @parameterized.expand(
+ [
+ ("every_step_completed", ["completed", "completed"], "completed", ["completed", "completed"]),
+ ("a_step_still_running", ["completed", "running"], "failed", ["completed", "failed"]),
+ ("a_step_never_started", ["pending", "completed"], "failed", ["failed", "completed"]),
+ ]
+ )
+ def test_finish_sets_the_run_status_from_its_steps_once(
+ self,
+ mock_client_class: MagicMock,
+ _start_workflow: MagicMock,
+ _name: str,
+ step_statuses: list[str],
+ expected_run_status: str,
+ expected_step_statuses: list[str],
+ ) -> None:
+ run = self._start(["suppressions", "opt_outs"])
+ for step, step_status in zip(run.steps, step_statuses):
+ run.progress[step]["status"] = step_status
+ run.save()
+
+ with patch("products.messaging.backend.provider_imports.runs.report_team_action") as report:
+ runs.finish_run(self.team.id, run.id)
+ runs.finish_run(self.team.id, run.id)
+
+ run.refresh_from_db()
+ assert run.status == expected_run_status
+ assert [run.progress[step]["status"] for step in run.steps] == expected_step_statuses
+ assert run.finished_at is not None
+ assert report.call_count == 1
+
+ @time_machine.travel("2026-03-02T12:00:00Z", tick=False)
+ def test_reading_an_expired_run_fails_it_and_its_unfinished_steps(
+ self, mock_client_class: MagicMock, _start_workflow: MagicMock
+ ) -> None:
+ run = self._start(["suppressions"])
+ ProviderImportRun.objects.for_team(self.team.id).filter(id=run.id).update(
+ created_at=timezone.now() - runs.STALE_RUN_AGE - timedelta(minutes=1)
+ )
+
+ expired = runs.get_run(self.team.id, run.id)
+
+ assert expired is not None
+ assert expired.status == "failed"
+ assert expired.progress["suppressions"]["status"] == "failed"
diff --git a/products/messaging/turbo.json b/products/messaging/turbo.json
index 5a1fb406a1b9..92243a70f3cf 100644
--- a/products/messaging/turbo.json
+++ b/products/messaging/turbo.json
@@ -7,6 +7,7 @@
"backend/presentation/**",
"backend/routes.py",
"backend/tasks/**",
+ "backend/temporal/**",
"backend/models/**",
"backend/migrations/**"
],