Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(workflows): import push device tokens in the background (#129105)

Daniel Marchuk013440b86e08
diff --git a/products/workflows/backend/facade/contracts.py b/products/workflows/backend/facade/contracts.py
index 6777f40b158c..5aea64085d3a 100644
--- a/products/workflows/backend/facade/contracts.py
+++ b/products/workflows/backend/facade/contracts.py
@@ -21,6 +21,8 @@ from products.workflows.backend.facade.enums import (
HogFlowScheduleStatus,
HogFlowTemplateExitCondition,
HogFlowTemplateScope,
+ PushTokenImportRowFailureReason,
+ PushTokenImportStatus,
SetupStepKey,
WorkflowImportNoteLevel,
WorkflowProposalStatus,
@@ -1165,3 +1167,39 @@ class WorkflowImportRecord:
notes: tuple[dict[str, Any], ...]
updated_at: datetime
hog_flow_created_by_id: int | None = None
+
+
+class PushTokenImportInvalid(Exception):
+ """The upload cannot be used. The message is shown to the person who sent it."""
+
+
+class PushTokenImportNotFound(LookupError):
+ pass
+
+
+@frozen
+class PushTokenImportRowFailure:
+ line: int
+ distinct_id: str
+ device_token_suffix: str
+ reason: PushTokenImportRowFailureReason
+ detail: str
+
+
+@frozen
+class PushTokenImportRecord:
+ id: UUID
+ integration_id: int
+ status: PushTokenImportStatus
+ create_missing_persons: bool
+ total_devices: int
+ processed_devices: int
+ imported_devices: int
+ failed_devices: int
+ unknown_person_devices: int
+ duplicate_lines: int
+ failures: tuple[PushTokenImportRowFailure, ...]
+ can_import_unknown_persons: bool
+ error: str
+ created_at: datetime
+ finished_at: datetime | None
diff --git a/products/workflows/backend/facade/push_token_import.py b/products/workflows/backend/facade/push_token_import.py
new file mode 100644
index 000000000000..bbd64add6969
--- /dev/null
+++ b/products/workflows/backend/facade/push_token_import.py
@@ -0,0 +1,29 @@
+"""Uploaded files of push device tokens, imported into a push channel in the background."""
+
+from products.workflows.backend.facade.contracts import (
+ PushTokenImportInvalid,
+ PushTokenImportNotFound,
+ PushTokenImportRecord,
+ PushTokenImportRowFailure,
+)
+from products.workflows.backend.services.push_token_import import (
+ MAX_FILE_BYTES,
+ cancel_push_token_import,
+ get_push_token_import,
+ import_unknown_persons,
+ latest_push_token_import,
+ start_push_token_import,
+)
+
+__all__ = [
+ "MAX_FILE_BYTES",
+ "PushTokenImportInvalid",
+ "PushTokenImportNotFound",
+ "PushTokenImportRecord",
+ "PushTokenImportRowFailure",
+ "cancel_push_token_import",
+ "get_push_token_import",
+ "import_unknown_persons",
+ "latest_push_token_import",
+ "start_push_token_import",
+]
diff --git a/products/workflows/backend/services/push_token_import.py b/products/workflows/backend/services/push_token_import.py
new file mode 100644
index 000000000000..087478df2857
--- /dev/null
+++ b/products/workflows/backend/services/push_token_import.py
@@ -0,0 +1,677 @@
+"""Uploaded files of push device tokens, imported into one push channel by a background task."""
+
+import io
+import csv
+import json
+import time
+import uuid
+import zlib
+from datetime import datetime, timedelta
+from typing import TypedDict
+
+from django.db import IntegrityError, transaction
+from django.utils import timezone
+
+from structlog import get_logger
+
+from posthog.api.integration_push_tokens import (
+ MAX_DISTINCT_ID_BYTES,
+ MAX_SUBSCRIPTIONS_PER_REQUEST,
+ TOKEN_SUFFIX_LENGTH,
+ PushTokenImportFailure,
+ PushTokenImportFailureReason,
+ import_push_tokens,
+ push_channel_app_id,
+)
+from posthog.dataclasses import frozen
+from posthog.helpers.encrypted_fields import EncryptedFieldMixin
+from posthog.integrations import IntegrationManager, IntegrationNotFound, IntegrationRecord
+from posthog.models.team.team import Team
+from posthog.models.user import User
+from posthog.ph_client import ph_scoped_capture
+from posthog.storage import object_storage
+
+from products.workflows.backend.facade.contracts import (
+ PushTokenImportInvalid,
+ PushTokenImportNotFound,
+ PushTokenImportRecord,
+ PushTokenImportRowFailure,
+)
+from products.workflows.backend.facade.enums import PushTokenImportRowFailureReason, PushTokenImportStatus
+from products.workflows.backend.models.push_token_import import PushTokenImport
+
+logger = get_logger(__name__)
+
+MAX_FILE_BYTES = 20 * 1024 * 1024
+BATCH_SIZE = MAX_SUBSCRIPTIONS_PER_REQUEST
+# One task run sends this many batches and then queues the next run, so a worker restart loses little.
+BATCHES_PER_TASK = 20
+# The result lists this many failed lines. The counts cover every line.
+MAX_STORED_FAILURES = 1000
+# Devices of people who don't exist yet stay stored this long, so they can be imported with new persons.
+UNKNOWN_PERSON_DEVICES_KEPT_FOR = timedelta(hours=24)
+# Counted from the upload, so it covers the time the import itself takes.
+EXPIRY_MARGIN = timedelta(hours=1)
+# A task run takes seconds, so this much quiet means the worker stopped, for example in a deploy.
+STALL_CHECK_EVERY = timedelta(minutes=5)
+DELETE_RETRY_AFTER = timedelta(hours=1)
+NOT_STORED_ATTEMPTS = 3
+NOT_STORED_RETRY_SECONDS = 2
+STALLED_AFTER = timedelta(minutes=5)
+MAX_RESUMES = 3
+
+# Blank lines are a byte each, so this bounds the work a file within the size limit can cause.
+MAX_FILE_LINES = 400_000
+MAX_DEVICE_TOKEN_LENGTH = 4096
+_DISTINCT_ID_COLUMN = "distinct_id"
+_DEVICE_TOKEN_COLUMN = "device_token"
+_STORAGE_FOLDER = "workflows_push_token_imports"
+_PUSH_KINDS = ("apns", "firebase")
+
+
+class StagedDevice(TypedDict):
+ line: int
+ distinct_id: str
+ device_token: str
+
+
+@frozen
+class ParsedPushTokenFile:
+ devices: list[StagedDevice]
+ rejected: list[PushTokenImportRowFailure]
+ rejected_lines: int
+ duplicate_lines: int
+
+
+@frozen
+class _Columns:
+ distinct_id: int
+ device_token: int
+
+
+class _DeviceCipher(EncryptedFieldMixin):
+ pass
+
+
+def parse_push_token_csv(content: bytes) -> ParsedPushTokenFile:
+ """Reads a CSV with a header row into devices. Raises `PushTokenImportInvalid` when the file can't be used."""
+ # Decoding while reading keeps a second, wider copy of the whole file out of memory.
+ reader = csv.reader(io.TextIOWrapper(io.BytesIO(content), encoding="utf-8-sig", newline=""), strict=True)
+ devices: list[StagedDevice] = []
+ rejected: list[PushTokenImportRowFailure] = []
+ rejected_lines = 0
+ seen: set[tuple[str, str]] = set()
+ duplicate_lines = 0
+ has_data = False
+ columns: _Columns | None = None
+ try:
+ for cells in reader:
+ # A quoted value can span lines, so the record's line is where it started.
+ line = reader.line_num - sum(cell.count("\n") for cell in cells)
+ if reader.line_num > MAX_FILE_LINES:
+ raise PushTokenImportInvalid(
+ f"This file has more than {MAX_FILE_LINES:,} lines. "
+ "Split it into smaller files and import them one at a time."
+ )
+ if columns is None:
+ columns = _columns(cells)
+ continue
+ if all(not cell.strip() for cell in cells):
+ continue
+ has_data = True
+ distinct_id = _cell(cells, columns.distinct_id)
+ device_token = _cell(cells, columns.device_token)
+ if problem := _row_problem(distinct_id, device_token):
+ rejected_lines += 1
+ if len(rejected) < MAX_STORED_FAILURES:
+ rejected.append(
+ PushTokenImportRowFailure(
+ line=line,
+ distinct_id=distinct_id.replace("\x00", ""),
+ device_token_suffix=device_token[-TOKEN_SUFFIX_LENGTH:].replace("\x00", ""),
+ reason=PushTokenImportRowFailureReason.INVALID_ROW,
+ detail=problem,
+ )
+ )
+ continue
+ key = (distinct_id, device_token)
+ if key in seen:
+ duplicate_lines += 1
+ continue
+ seen.add(key)
+ devices.append({"line": line, "distinct_id": distinct_id, "device_token": device_token})
+ except UnicodeDecodeError:
+ raise PushTokenImportInvalid('This file isn\'t saved as UTF-8. Save it as "CSV UTF-8" and upload it again.')
+ except csv.Error as error:
+ raise PushTokenImportInvalid(f"Line {reader.line_num} of the file can't be read: {error}.")
+ if columns is None:
+ raise PushTokenImportInvalid("The file is empty.")
+ if not has_data:
+ raise PushTokenImportInvalid("The file has no devices in it, only the column names.")
+ return ParsedPushTokenFile(
+ devices=devices, rejected=rejected, rejected_lines=rejected_lines, duplicate_lines=duplicate_lines
+ )
+
+
+def _columns(header: list[str]) -> _Columns:
+ columns = [cell.strip().lower() for cell in header]
+ if _DISTINCT_ID_COLUMN not in columns or _DEVICE_TOKEN_COLUMN not in columns:
+ raise PushTokenImportInvalid(
+ "The first line of the file must name the columns, and it needs a distinct_id column and a device_token column."
+ )
+ return _Columns(distinct_id=columns.index(_DISTINCT_ID_COLUMN), device_token=columns.index(_DEVICE_TOKEN_COLUMN))
+
+
+def start_push_token_import(
+ *,
+ team: Team,
+ user: User | None,
+ integration_id: int,
+ content: bytes,
+ create_missing_persons: bool,
+) -> PushTokenImportRecord:
+ _push_channel(team.id, integration_id)
+ if len(content) > MAX_FILE_BYTES:
+ raise PushTokenImportInvalid(
+ f"This file is larger than {MAX_FILE_BYTES // (1024 * 1024)} MB. "
+ "Split it into smaller files and import them one at a time."
+ )
+ parsed = parse_push_token_csv(content)
+ if not parsed.devices:
+ raise PushTokenImportInvalid(
+ "None of the lines in the file can be imported. Fix them and upload the file again."
+ )
+ return _create_import(
+ team=team,
+ user=user,
+ integration_id=integration_id,
+ devices=parsed.devices,
+ create_missing_persons=create_missing_persons,
+ rejected=parsed.rejected,
+ rejected_lines=parsed.rejected_lines,
+ duplicate_lines=parsed.duplicate_lines,
+ )
+
+
+def import_unknown_persons(*, team: Team, user: User | None, import_id: uuid.UUID) -> PushTokenImportRecord:
+ """Starts a second import of the devices whose distinct ID had no person, creating those persons."""
+ previous = _get(team.id, import_id)
+ if not _can_import_unknown_persons(previous):
+ raise PushTokenImportInvalid(
+ "These devices can't be imported again. Upload the file again to import them with new persons."
+ )
+ storage_key = previous.storage_key
+ unknown_lines = set(previous.unknown_person_lines)
+ devices = [device for device in _read_devices(storage_key) if device["line"] in unknown_lines]
+ if not devices:
+ raise PushTokenImportInvalid(
+ "These devices can't be imported again. Upload the file again to import them with new persons."
+ )
+ # Only one import per channel runs at a time, so a second click fails here instead of importing twice.
+ record = _create_import(
+ team=team,
+ user=user,
+ integration_id=previous.integration_id,
+ devices=devices,
+ create_missing_persons=True,
+ rejected=[],
+ rejected_lines=0,
+ duplicate_lines=0,
+ )
+ # Cleared before the delete, so a failed delete can't let these devices be imported a second time.
+ PushTokenImport.objects.for_team(team.id).filter(id=import_id).update(
+ unknown_person_lines=[], updated_at=timezone.now()
+ )
+ _release_devices(team.id, import_id, storage_key)
+ return record
+
+
+def get_push_token_import(team_id: int, import_id: uuid.UUID) -> PushTokenImportRecord:
+ return _record(_get(team_id, import_id))
+
+
+def latest_push_token_import(team_id: int, integration_id: int) -> PushTokenImportRecord | None:
+ latest = PushTokenImport.objects.for_team(team_id).filter(integration_id=integration_id).order_by("-created_at")
+ found = latest.first()
+ return _record(found) if found else None
+
+
+def cancel_push_token_import(team_id: int, import_id: uuid.UUID) -> PushTokenImportRecord:
+ """Stops a running import after the batch it is sending. Devices already sent stay imported.
+
+ The task sets `finished_at` once that batch is counted, so the import reads as stopping until then.
+ """
+ PushTokenImport.objects.for_team(team_id).filter(id=import_id, status=PushTokenImportStatus.RUNNING).update(
+ status=PushTokenImportStatus.CANCELLED, updated_at=timezone.now()
+ )
+ return get_push_token_import(team_id, import_id)
+
+
+def run_push_token_import_batches(team_id: int, import_id: uuid.UUID) -> bool:
+ """Sends the next batches of a running import. Returns True when batches are left for another run."""
+ job = PushTokenImport.objects.for_team(team_id).filter(id=import_id).first()
+ if job is None:
+ return False
+ if job.status != PushTokenImportStatus.RUNNING:
+ _end_stopped(job)
+ return False
+ try:
+ integration = _push_channel(team_id, job.integration_id)
+ except PushTokenImportInvalid as error:
+ fail_push_token_import(team_id, import_id, str(error))
+ return False
+ api_token = Team.objects.only("api_token").get(id=team_id).api_token
+ devices = _read_devices(job.storage_key)
+ if not devices:
+ fail_push_token_import(
+ team_id,
+ import_id,
+ "PostHog couldn't read the stored devices, so the import stopped. Upload the file again.",
+ )
+ return False
+
+ for _ in range(BATCHES_PER_TASK):
+ start = job.processed_devices
+ if start >= len(devices):
+ _finish(job)
+ return False
+ batch = devices[start : start + BATCH_SIZE]
+ imported, failed = _send_batch(
+ integration,
+ batch,
+ api_token,
+ create_missing_persons=job.create_missing_persons,
+ # A batch sent again after a worker restart reuses its event IDs, so its copies can be deduplicated.
+ import_id=uuid.uuid5(job.id, str(start)),
+ timestamp=job.created_at,
+ )
+ for device, failure in failed:
+ line = device["line"]
+ if failure.reason == PushTokenImportFailureReason.UNKNOWN_PERSON:
+ job.unknown_person_lines.append(line)
+ if len(job.failures) < MAX_STORED_FAILURES:
+ job.failures.append(
+ _failure_json(
+ PushTokenImportRowFailure(
+ line=line,
+ distinct_id=failure.distinct_id,
+ device_token_suffix=failure.device_token_suffix,
+ reason=PushTokenImportRowFailureReason(failure.reason),
+ detail=failure.detail,
+ )
+ )
+ )
+ job.processed_devices = start + len(batch)
+ job.imported_devices += imported
+ job.failed_devices += len(failed)
+ # Matching the progress read at the start means a cancel or a second worker stops this one.
+ saved = (
+ PushTokenImport.objects.for_team(team_id)
+ .filter(id=import_id, status=PushTokenImportStatus.RUNNING, processed_devices=start)
+ .update(
+ processed_devices=job.processed_devices,
+ imported_devices=job.imported_devices,
+ failed_devices=job.failed_devices,
+ failures=job.failures,
+ unknown_person_lines=job.unknown_person_lines,
+ updated_at=timezone.now(),
+ )
+ )
+ if not saved:
+ # A stop that lands mid-batch leaves the batch's devices stored, so the stopped import still counts them.
+ PushTokenImport.objects.for_team(team_id).filter(
+ id=import_id, status=PushTokenImportStatus.CANCELLED, processed_devices=start
+ ).update(
+ processed_devices=job.processed_devices,
+ imported_devices=job.imported_devices,
+ failed_devices=job.failed_devices,
+ failures=job.failures,
+ unknown_person_lines=job.unknown_person_lines,
+ updated_at=timezone.now(),
+ )
+ job.refresh_from_db()
+ _end_stopped(job)
+ return False
+ if job.processed_devices >= len(devices):
+ _finish(job)
+ return False
+ return True
+
+
+def _send_batch(
+ integration: IntegrationRecord,
+ batch: list[StagedDevice],
+ api_token: str,
+ *,
+ create_missing_persons: bool,
+ import_id: uuid.UUID,
+ timestamp: datetime,
+) -> tuple[int, list[tuple[StagedDevice, PushTokenImportFailure]]]:
+ """Sends one batch, and sends again the devices ingestion did not store, so a short outage loses nothing."""
+ pending = batch
+ imported = 0
+ failed: list[tuple[StagedDevice, PushTokenImportFailure]] = []
+ for attempt in range(NOT_STORED_ATTEMPTS):
+ if attempt:
+ time.sleep(NOT_STORED_RETRY_SECONDS * attempt)
+ result = import_push_tokens(
+ integration,
+ [{"distinct_id": device["distinct_id"], "device_token": device["device_token"]} for device in pending],
+ api_token,
+ create_missing_persons=create_missing_persons,
+ import_id=import_id,
+ timestamp=timestamp,
+ )
+ imported += result.imported
+ retry: list[StagedDevice] = []
+ for failure in result.failed:
+ device = pending[failure.index]
+ if failure.reason == PushTokenImportFailureReason.NOT_STORED and attempt + 1 < NOT_STORED_ATTEMPTS:
+ retry.append(device)
+ else:
+ failed.append((device, failure))
+ if not retry:
+ break
+ pending = retry
+ return imported, failed
+
+
+def fail_push_token_import(
+ team_id: int, import_id: uuid.UUID, error: str, *, unless_updated_since: datetime | None = None
+) -> bool:
+ """Returns False when the import was not running, or made progress after `unless_updated_since`."""
+ running = PushTokenImport.objects.for_team(team_id).filter(id=import_id, status=PushTokenImportStatus.RUNNING)
+ if unless_updated_since is not None:
+ running = running.filter(updated_at=unless_updated_since)
+ updated = running.update(
+ status=PushTokenImportStatus.FAILED, error=error, finished_at=timezone.now(), updated_at=timezone.now()
+ )
+ job = PushTokenImport.objects.for_team(team_id).filter(id=import_id).first()
+ if job is not None:
+ _discard_devices_of_stopped(job)
+ if updated:
+ _report_finished(job)
+ return bool(updated)
+
+
+def check_stalled_push_token_import(team_id: int, import_id: uuid.UUID) -> bool:
+ """Resumes an import that stopped making progress, then fails it. Returns True while it still runs."""
+ job = PushTokenImport.objects.for_team(team_id).filter(id=import_id).first()
+ if job is None or job.finished_at is not None:
+ return False
+ if timezone.now() - job.updated_at < STALLED_AFTER:
+ return True
+ if job.status == PushTokenImportStatus.CANCELLED:
+ _end_stopped(job)
+ return False
+ logger.warning("push_token_import_stalled", team_id=team_id, import_id=str(import_id))
+ if job.resumed_times < MAX_RESUMES:
+ # Progress is saved after each batch and a batch sent again keeps its event IDs, so resuming is safe.
+ resumed = (
+ PushTokenImport.objects.for_team(team_id)
+ .filter(id=import_id, status=PushTokenImportStatus.RUNNING, updated_at=job.updated_at)
+ .update(resumed_times=job.resumed_times + 1, updated_at=timezone.now())
+ )
+ if resumed:
+ # The task module imports this one, so a module-level import would be circular.
+ from products.workflows.backend.tasks.push_token_import import run_push_token_import # noqa: PLC0415
+
+ run_push_token_import.delay(team_id=team_id, import_id=str(import_id))
+ return True
+ failed = fail_push_token_import(
+ team_id,
+ import_id,
+ f"The import stopped after {job.processed_devices:,} of {job.total_devices:,} devices. "
+ "Upload the file again to import the rest. Devices already imported are not stored twice.",
+ unless_updated_since=job.updated_at,
+ )
+ return not failed
+
+
+def expire_unknown_person_devices(team_id: int, import_id: uuid.UUID) -> bool:
+ """Deletes the devices an import kept. Returns True when the import still runs, so the check comes back later."""
+ job = PushTokenImport.objects.for_team(team_id).filter(id=import_id).exclude(storage_key="").first()
+ if job is None:
+ return False
+ if job.status == PushTokenImportStatus.RUNNING:
+ return True
+ _release_devices(team_id, import_id, job.storage_key)
+ return False
+
+
+def _create_import(
+ *,
+ team: Team,
+ user: User | None,
+ integration_id: int,
+ devices: list[StagedDevice],
+ create_missing_persons: bool,
+ rejected: list[PushTokenImportRowFailure],
+ rejected_lines: int,
+ duplicate_lines: int,
+) -> PushTokenImportRecord:
+ # The task module imports this one, so a module-level import would be circular.
+ from products.workflows.backend.tasks.push_token_import import ( # noqa: PLC0415
+ check_push_token_import_stalled,
+ expire_push_token_import_devices,
+ run_push_token_import,
+ )
+
+ import_id = uuid.uuid4()
+ storage_key = f"{_STORAGE_FOLDER}/team-{team.id}/{import_id}.json"
+ object_storage.write(storage_key, _encrypt_devices(devices))
+ try:
+ with transaction.atomic():
+ job = PushTokenImport.objects.for_team(team.id).create(
+ id=import_id,
+ team_id=team.id,
+ integration_id=integration_id,
+ create_missing_persons=create_missing_persons,
+ total_devices=len(devices),
+ failed_devices=rejected_lines,
+ duplicate_lines=duplicate_lines,
+ failures=[_failure_json(failure) for failure in rejected[:MAX_STORED_FAILURES]],
+ storage_key=storage_key,
+ created_by=user,
+ )
+ except IntegrityError:
+ _delete_object(storage_key)
+ raise PushTokenImportInvalid(
+ "An import into this channel is already running. Wait for it to finish, or stop it, and try again."
+ )
+ except Exception:
+ # No import row exists yet, so nothing else would ever delete the stored devices.
+ _delete_object(storage_key)
+ raise
+ try:
+ run_push_token_import.delay(team_id=team.id, import_id=str(job.id))
+ check_push_token_import_stalled.apply_async(
+ kwargs={"team_id": team.id, "import_id": str(job.id)}, countdown=int(STALL_CHECK_EVERY.total_seconds())
+ )
+ # Scheduled with the import, so devices kept for unknown people can't outlive it if a later step fails.
+ expire_push_token_import_devices.apply_async(
+ kwargs={"team_id": team.id, "import_id": str(job.id)},
+ countdown=int((UNKNOWN_PERSON_DEVICES_KEPT_FOR + EXPIRY_MARGIN).total_seconds()),
+ )
+ except Exception:
+ fail_push_token_import(team.id, job.id, "The import couldn't start. Try again.")
+ raise
+ return _record(job)
+
+
+def _finish(job: PushTokenImport) -> None:
+ keep_unknown = bool(job.unknown_person_lines) and not job.create_missing_persons
+ storage_key = job.storage_key
+ finished = (
+ PushTokenImport.objects.for_team(job.team_id)
+ .filter(id=job.id, status=PushTokenImportStatus.RUNNING)
+ .update(
+ status=PushTokenImportStatus.COMPLETED,
+ finished_at=timezone.now(),
+ updated_at=timezone.now(),
+ )
+ )
+ if not finished:
+ return
+ job.refresh_from_db()
+ if not keep_unknown:
+ _release_devices(job.team_id, job.id, storage_key)
+ _report_finished(job)
+
+
+def _end_stopped(job: PushTokenImport) -> None:
+ """Marks a stopped import as ended once no batch of it is being sent any more."""
+ if job.status == PushTokenImportStatus.CANCELLED and job.finished_at is None:
+ if (
+ PushTokenImport.objects.for_team(job.team_id)
+ .filter(id=job.id, finished_at__isnull=True)
+ .update(finished_at=timezone.now(), updated_at=timezone.now())
+ ):
+ job.refresh_from_db()
+ _report_finished(job)
+ _discard_devices_of_stopped(job)
+
+
+def _discard_devices_of_stopped(job: PushTokenImport) -> None:
+ if job.status == PushTokenImportStatus.RUNNING or not job.storage_key or _can_import_unknown_persons(job):
+ return
+ _release_devices(job.team_id, job.id, job.storage_key)
+
+
+def _report_finished(job: PushTokenImport) -> None:
+ user = User.objects.filter(id=job.created_by_id).only("distinct_id").first() if job.created_by_id else None
+ if user is None:
+ return
+ try:
+ with ph_scoped_capture() as capture:
+ capture(
+ distinct_id=str(user.distinct_id),
+ event="workflows push token import finished",
+ properties={
+ "status": job.status,
+ "devices": job.total_devices,
+ "imported": job.imported_devices,
+ "failed": job.failed_devices,
+ "unknown_persons": len(job.unknown_person_lines),
+ "create_missing_persons": job.create_missing_persons,
+ },
+ )
+ except Exception:
+ logger.exception("push_token_import_report_failed", import_id=str(job.id))
+
+
+def _push_channel(team_id: int, integration_id: int) -> IntegrationRecord:
+ try:
+ integration = IntegrationManager.kinds(*_PUSH_KINDS).get(team_id, integration_id)
+ except IntegrationNotFound:
+ raise PushTokenImportInvalid("This push channel no longer exists.")
+ if not push_channel_app_id(integration):
+ raise PushTokenImportInvalid("This push channel has no app set up, so devices can't be imported into it.")
+ return integration
+
+
+def _get(team_id: int, import_id: uuid.UUID) -> PushTokenImport:
+ job = PushTokenImport.objects.for_team(team_id).filter(id=import_id).first()
+ if job is None:
+ raise PushTokenImportNotFound(str(import_id))
+ return job
+
+
+def _can_import_unknown_persons(job: PushTokenImport) -> bool:
+ return (
+ job.status == PushTokenImportStatus.COMPLETED
+ and not job.create_missing_persons
+ and bool(job.unknown_person_lines)
+ and bool(job.storage_key)
+ )
+
+
+def _read_devices(storage_key: str) -> list[StagedDevice]:
+ content = object_storage.read_bytes(storage_key, missing_ok=True) if storage_key else None
+ return json.loads(zlib.decompress(_DeviceCipher().f.decrypt(content))) if content else []
+
+
+def _encrypt_devices(devices: list[StagedDevice]) -> bytes:
+ # Compressing first keeps the copies the cipher makes of a full-size file small.
+ return _DeviceCipher().f.encrypt(zlib.compress(json.dumps(devices, separators=(",", ":")).encode(), 1))
+
+
+def _release_devices(team_id: int, import_id: uuid.UUID, storage_key: str) -> None:
+ """Deletes the stored devices, and forgets the key only once they are gone, so a failed delete is tried again."""
+ if not _delete_object(storage_key):
+ # The task module imports this one, so a module-level import would be circular.
+ from products.workflows.backend.tasks.push_token_import import expire_push_token_import_devices # noqa: PLC0415
+
+ expire_push_token_import_devices.apply_async(
+ kwargs={"team_id": team_id, "import_id": str(import_id)}, countdown=int(DELETE_RETRY_AFTER.total_seconds())
+ )
+ return
+ PushTokenImport.objects.for_team(team_id).filter(id=import_id, storage_key=storage_key).update(
+ storage_key="", updated_at=timezone.now()
+ )
+
+
+def _delete_object(storage_key: str) -> bool:
+ try:
+ object_storage.delete(storage_key)
+ except Exception:
+ logger.exception("push_token_import_cleanup_failed", storage_key=storage_key)
+ return False
+ return True
+
+
+def _cell(cells: list[str], column: int) -> str:
+ return cells[column].strip() if column < len(cells) else ""
+
+
+def _row_problem(distinct_id: str, device_token: str) -> str | None:
+ if "\x00" in distinct_id or "\x00" in device_token:
+ return "The line has a NUL character in it"
+ if not distinct_id:
+ return "No distinct ID"
+ if not device_token:
+ return "No device token"
+ if len(distinct_id.encode()) > MAX_DISTINCT_ID_BYTES:
+ return f"Distinct ID is longer than {MAX_DISTINCT_ID_BYTES} bytes"
+ if len(device_token) > MAX_DEVICE_TOKEN_LENGTH:
+ return f"Device token is longer than {MAX_DEVICE_TOKEN_LENGTH} characters"
+ return None
+
+
+def _failure_json(failure: PushTokenImportRowFailure) -> dict[str, str | int]:
+ return {
+ "line": failure.line,
+ "distinct_id": failure.distinct_id,
+ "device_token_suffix": failure.device_token_suffix,
+ "reason": str(failure.reason),
+ "detail": failure.detail,
+ }
+
+
+def _record(job: PushTokenImport) -> PushTokenImportRecord:
+ return PushTokenImportRecord(
+ id=job.id,
+ integration_id=job.integration_id,
+ status=PushTokenImportStatus(job.status),
+ create_missing_persons=job.create_missing_persons,
+ total_devices=job.total_devices,
+ processed_devices=job.processed_devices,
+ imported_devices=job.imported_devices,
+ failed_devices=job.failed_devices,
+ unknown_person_devices=len(job.unknown_person_lines),
+ duplicate_lines=job.duplicate_lines,
+ failures=tuple(
+ PushTokenImportRowFailure(
+ line=int(failure["line"]),
+ distinct_id=str(failure["distinct_id"]),
+ device_token_suffix=str(failure["device_token_suffix"]),
+ reason=PushTokenImportRowFailureReason(failure["reason"]),
+ detail=str(failure["detail"]),
+ )
+ for failure in sorted(job.failures, key=lambda failure: int(failure["line"]))
+ ),
+ can_import_unknown_persons=_can_import_unknown_persons(job),
+ error=job.error,
+ created_at=job.created_at,
+ finished_at=job.finished_at,
+ )
diff --git a/products/workflows/backend/tasks/__init__.py b/products/workflows/backend/tasks/__init__.py
index c57aedd65e4e..4ab1316c0d5a 100644
--- a/products/workflows/backend/tasks/__init__.py
+++ b/products/workflows/backend/tasks/__init__.py
@@ -4,6 +4,7 @@ from . import (
email_sending_tiers,
hog_flows,
people_import,
+ push_token_import,
ses_account_reputation,
)
@@ -13,5 +14,6 @@ __all__ = [
"email_sending_tiers",
"hog_flows",
"people_import",
+ "push_token_import",
"ses_account_reputation",
]
diff --git a/products/workflows/backend/tasks/push_token_import.py b/products/workflows/backend/tasks/push_token_import.py
new file mode 100644
index 000000000000..e73f51ac3732
--- /dev/null
+++ b/products/workflows/backend/tasks/push_token_import.py
@@ -0,0 +1,72 @@
+from uuid import UUID
+
+from celery import Task, shared_task
+from structlog import get_logger
+
+from posthog.scoping_audit import skip_team_scope_audit
+from posthog.tasks.utils import CeleryQueue
+
+from products.workflows.backend.services.push_token_import import (
+ EXPIRY_MARGIN,
+ STALL_CHECK_EVERY,
+ check_stalled_push_token_import,
+ expire_unknown_person_devices,
+ fail_push_token_import,
+ run_push_token_import_batches,
+)
+
+logger = get_logger(__name__)
+
+MAX_RETRIES = 3
+MAINTENANCE_RETRIES = 8
+
+
+@shared_task(bind=True, ignore_result=True, queue=CeleryQueue.DEFAULT.value, max_retries=MAX_RETRIES)
+@skip_team_scope_audit
+def run_push_token_import(self: Task, *, team_id: int, import_id: str) -> None:
+ try:
+ more = run_push_token_import_batches(team_id, UUID(import_id))
+ except Exception as error:
+ logger.warning("push_token_import_batch_failed", team_id=team_id, import_id=import_id, error=str(error))
+ if self.request.retries < MAX_RETRIES:
+ raise self.retry(countdown=30 * (self.request.retries + 1))
+ fail_push_token_import(
+ team_id,
+ UUID(import_id),
+ "PostHog couldn't save the devices, so the import stopped. Upload the file again to import the rest. "
+ "Devices already imported are not stored twice.",
+ )
+ return
+ if more:
+ run_push_token_import.delay(team_id=team_id, import_id=import_id)
+
+
+# A check that dies does not schedule the next one, so an error has to retry it rather than end the chain.
+@shared_task(
+ ignore_result=True,
+ queue=CeleryQueue.DEFAULT.value,
+ autoretry_for=(Exception,),
+ retry_backoff=30,
+ max_retries=MAINTENANCE_RETRIES,
+)
+@skip_team_scope_audit
+def check_push_token_import_stalled(*, team_id: int, import_id: str) -> None:
+ if check_stalled_push_token_import(team_id, UUID(import_id)):
+ check_push_token_import_stalled.apply_async(
+ kwargs={"team_id": team_id, "import_id": import_id}, countdown=int(STALL_CHECK_EVERY.total_seconds())
+ )
+
+
+@shared_task(
+ ignore_result=True,
+ queue=CeleryQueue.DEFAULT.value,
+ autoretry_for=(Exception,),
+ retry_backoff=30,
+ max_retries=MAINTENANCE_RETRIES,
+)
+@skip_team_scope_audit
+def expire_push_token_import_devices(*, team_id: int, import_id: str) -> None:
+ if expire_unknown_person_devices(team_id, UUID(import_id)):
+ expire_push_token_import_devices.apply_async(
+ kwargs={"team_id": team_id, "import_id": import_id}, countdown=int(EXPIRY_MARGIN.total_seconds())
+ )
diff --git a/products/workflows/backend/test/test_push_token_import.py b/products/workflows/backend/test/test_push_token_import.py
new file mode 100644
index 000000000000..436359852bc3
--- /dev/null
+++ b/products/workflows/backend/test/test_push_token_import.py
@@ -0,0 +1,332 @@
+from datetime import timedelta
+from typing import Any
+from uuid import UUID
+
+import time_machine
+from posthog.test.base import APIBaseTest, _create_person, flush_persons_and_events
+from unittest.mock import patch
+
+from django.db import OperationalError
+from django.test import SimpleTestCase
+from django.utils import timezone
+
+from parameterized import parameterized
+
+from posthog.api.capture import CaptureInternalResult
+from posthog.models.integration import Integration
+from posthog.models.organization import OrganizationMembership
+
+from products.workflows.backend.facade.contracts import PushTokenImportInvalid, PushTokenImportRecord
+from products.workflows.backend.facade.enums import PushTokenImportRowFailureReason, PushTokenImportStatus
+from products.workflows.backend.models.push_token_import import PushTokenImport
+from products.workflows.backend.services import push_token_import
+from products.workflows.backend.services.push_token_import import (
+ cancel_push_token_import,
+ check_stalled_push_token_import,
+ get_push_token_import,
+ import_unknown_persons,
+ parse_push_token_csv,
+ run_push_token_import_batches,
+ start_push_token_import,
+)
+
+APNS_TOKEN = "a1" * 32
+
+
+class _MemoryStorage:
+ def __init__(self) -> None:
+ self.objects: dict[str, bytes] = {}
+
+ def write(self, key: str, content: bytes) -> None:
+ self.objects[key] = content
+
+ def read_bytes(self, key: str, *, missing_ok: bool = False) -> bytes | None:
+ return self.objects.get(key)
+
+ def delete(self, key: str) -> None:
+ self.objects.pop(key, None)
+
+
+class TestParsePushTokenCsv(SimpleTestCase):
+ @parameterized.expand(
+ [
+ ("not_utf8", "distinct_id,device_token\nuser-é,abc\n".encode("latin-1"), "isn't saved as UTF-8"),
+ ("missing_column", b"distinct_id,token\nuser-1,abc\n", "needs a distinct_id column"),
+ ("only_header", b"distinct_id,device_token\n\n , \n", "only the column names"),
+ ("empty", b"", "The file is empty"),
+ ("broken_quotes", b'distinct_id,device_token\n"user-1,abc\n', "can't be read"),
+ ("too_many_lines", b"distinct_id,device_token\nuser-1,abc" + b"\n" * 400_001, "more than 400,000 lines"),
+ ]
+ )
+ def test_rejects_a_file_it_cannot_use(self, _name: str, content: bytes, message: str) -> None:
+ with self.assertRaisesMessage(PushTokenImportInvalid, message):
+ parse_push_token_csv(content)
+
+ def test_reads_devices_with_the_line_each_came_from(self) -> None:
+ content = (
+ "Name,Distinct_ID , DEVICE_TOKEN\n"
+ '"Ada\nLovelace",user-1,tok-1\n'
+ "\n"
+ "Bob,user-2,tok-2\n"
+ "Bob again,user-2,tok-2\n"
+ "No id,,tok-3\n"
+ f"Long,user-4,{'x' * 4097}\n"
+ "Ünï 🎉,user-🎉,tok-5\n"
+ "Nul,,tok\x00\n"
+ f"Emoji,{'🎉' * 51},tok-6\n"
+ ).encode()
+
+ parsed = parse_push_token_csv(content)
+
+ assert parsed.devices == [
+ {"line": 2, "distinct_id": "user-1", "device_token": "tok-1"},
+ {"line": 5, "distinct_id": "user-2", "device_token": "tok-2"},
+ {"line": 9, "distinct_id": "user-🎉", "device_token": "tok-5"},
+ ]
+ assert [(failure.line, failure.detail) for failure in parsed.rejected] == [
+ (7, "No distinct ID"),
+ (8, "Device token is longer than 4096 characters"),
+ (10, "The line has a NUL character in it"),
+ (11, "Distinct ID is longer than 200 bytes"),
+ ]
+ assert parsed.duplicate_lines == 1
+
+
+class _PushTokenImportTestCase(APIBaseTest):
+ def setUp(self) -> None:
+ super().setUp()
+ self.organization_membership.level = OrganizationMembership.Level.ADMIN
+ self.organization_membership.save()
+ self.apns = Integration.objects.create(
+ team=self.team,
+ kind="apns",
+ integration_id="TEAM.com.example.app",
+ config={"bundle_id": "com.example.app", "team_id": "TEAM", "key_id": "KEY"},
+ )
+ self.storage = _MemoryStorage()
+ patchers: list[Any] = [
+ patch.object(push_token_import, "object_storage", self.storage),
+ patch.object(push_token_import, "BATCH_SIZE", 2),
+ patch.object(push_token_import, "BATCHES_PER_TASK", 1),
+ patch("products.workflows.backend.tasks.push_token_import.run_push_token_import.delay"),
+ patch("products.workflows.backend.tasks.push_token_import.check_push_token_import_stalled.apply_async"),
+ patch.object(push_token_import, "ph_scoped_capture"),
+ ]
+ for patcher in patchers:
+ patcher.start()
+ self.addCleanup(patcher.stop)
+ expire = patch(
+ "products.workflows.backend.tasks.push_token_import.expire_push_token_import_devices.apply_async"
+ )
+ self.schedule_expiry = expire.start()
+ self.addCleanup(expire.stop)
+ self.captured: list[list[dict[str, Any]]] = []
+ self.drop_once: set[str] = set()
+ capture = patch("posthog.api.integration_push_tokens.capture_batch_internal", side_effect=self._capture)
+ capture.start()
+ self.addCleanup(capture.stop)
+
+ def _capture(self, *, events: list[dict[str, Any]], **kwargs: Any) -> CaptureInternalResult:
+ self.captured.append(events)
+ result = CaptureInternalResult(status_code=200)
+ for event in events:
+ dropped = event["distinct_id"] in self.drop_once
+ self.drop_once.discard(event["distinct_id"])
+ (result.dropped if dropped else result.ok).append(event["event_uuid"])
+ return result
+
+ def _upload(self, lines: list[str], *, create_missing_persons: bool = False) -> PushTokenImportRecord:
+ return start_push_token_import(
+ team=self.team,
+ user=self.user,
+ integration_id=self.apns.id,
+ content="\n".join(["distinct_id,device_token", *lines]).encode(),
+ create_missing_persons=create_missing_persons,
+ )
+
+ def _run_to_end(self, import_id: UUID) -> PushTokenImportRecord:
+ while run_push_token_import_batches(self.team.id, import_id):
+ pass
+ return get_push_token_import(self.team.id, import_id)
+
+
+class TestPushTokenImportJob(_PushTokenImportTestCase):
+ def test_imports_the_file_in_batches_and_reports_each_failed_line(self) -> None:
+ for distinct_id in ["user-1", "user-2", "user-3"]:
+ _create_person(team=self.team, distinct_ids=[distinct_id])
+ flush_persons_and_events()
+
+ started = self._upload(
+ [
+ f"user-1,{APNS_TOKEN}",
+ f"user-2,{APNS_TOKEN}",
+ ",missing-id",
+ "user-3,ExponentPushToken[abc]",
+ f"nobody,{APNS_TOKEN}",
+ f"user-3,{'b2' * 32}",
+ ]
+ )
+ assert started.status == PushTokenImportStatus.RUNNING
+ kept_for = push_token_import.UNKNOWN_PERSON_DEVICES_KEPT_FOR + push_token_import.EXPIRY_MARGIN
+ assert self.schedule_expiry.call_args.kwargs["countdown"] == int(kept_for.total_seconds())
+
+ result = self._run_to_end(started.id)
+
+ assert result.status == PushTokenImportStatus.COMPLETED
+ assert [len(batch) for batch in self.captured] == [2, 1]
+ assert (result.total_devices, result.imported_devices, result.failed_devices) == (5, 3, 3)
+ assert sorted((failure.line, failure.reason) for failure in result.failures) == [
+ (4, PushTokenImportRowFailureReason.INVALID_ROW),
+ (5, PushTokenImportRowFailureReason.TOKEN_MISMATCH),
+ (6, PushTokenImportRowFailureReason.UNKNOWN_PERSON),
+ ]
+ assert result.unknown_person_devices == 1
+ assert result.can_import_unknown_persons is True
+
+ def test_imports_the_devices_of_unknown_people_once_with_new_persons(self) -> None:
+ _create_person(team=self.team, distinct_ids=["user-1"])
+ flush_persons_and_events()
+ first = self._run_to_end(self._upload([f"user-1,{APNS_TOKEN}", f"new-1,{APNS_TOKEN}", f"new-2,{'c3' * 32}"]).id)
+ self.captured.clear()
+
+ retry = import_unknown_persons(team=self.team, user=self.user, import_id=first.id)
+ with self.assertRaises(PushTokenImportInvalid):
+ import_unknown_persons(team=self.team, user=self.user, import_id=first.id)
+
+ result = self._run_to_end(retry.id)
+ assert result.create_missing_persons is True
+ assert (result.total_devices, result.imported_devices) == (2, 2)
+ assert sorted(event["distinct_id"] for batch in self.captured for event in batch) == ["new-1", "new-2"]
+ assert get_push_token_import(self.team.id, first.id).can_import_unknown_persons is False
+ assert self.storage.objects == {}
+
+ def test_a_channel_runs_one_import_at_a_time_and_a_cancelled_import_stops_sending(self) -> None:
+ first = self._upload([f"user-1,{APNS_TOKEN}"])
+ with self.assertRaisesRegex(PushTokenImportInvalid, "already running"):
+ self._upload([f"user-2,{APNS_TOKEN}"])
+
+ cancelled = cancel_push_token_import(self.team.id, first.id)
+ result = self._run_to_end(first.id)
+
+ assert (cancelled.status, cancelled.finished_at) == (PushTokenImportStatus.CANCELLED, None)
+ assert result.status == PushTokenImportStatus.CANCELLED
+ assert result.finished_at is not None
+ assert self.captured == []
+ assert self.storage.objects == {}
+ assert self._upload([f"user-2,{APNS_TOKEN}"]).status == PushTokenImportStatus.RUNNING
+
+ def test_a_stop_that_lands_during_a_batch_still_counts_that_batch(self) -> None:
+ import_id = self._upload(
+ [f"a,{APNS_TOKEN}", f"b,{APNS_TOKEN}", f"c,{APNS_TOKEN}"], create_missing_persons=True
+ ).id
+
+ def stop_while_sending(**kwargs: Any) -> CaptureInternalResult:
+ cancel_push_token_import(self.team.id, import_id)
+ return self._capture(**kwargs)
+
+ with patch("posthog.api.integration_push_tokens.capture_batch_internal", side_effect=stop_while_sending):
+ result = self._run_to_end(import_id)
+
+ assert (result.status, result.processed_devices, result.imported_devices) == (
+ PushTokenImportStatus.CANCELLED,
+ 2,
+ 2,
+ )
+ assert result.finished_at is not None
+ assert self.storage.objects == {}
+
+ @patch.object(push_token_import.time, "sleep")
+ def test_a_device_ingestion_did_not_store_is_sent_again_with_its_event_id(self, _sleep: Any) -> None:
+ self.drop_once = {"b"}
+ import_id = self._upload([f"a,{APNS_TOKEN}", f"b,{APNS_TOKEN}"], create_missing_persons=True).id
+
+ result = self._run_to_end(import_id)
+
+ assert (result.imported_devices, result.failed_devices) == (2, 0)
+ assert [event["distinct_id"] for event in self.captured[1]] == ["b"]
+ assert self.captured[1][0]["event_uuid"] == self.captured[0][1]["event_uuid"]
+
+ def test_a_failed_delete_keeps_the_devices_known_so_cleanup_runs_again(self) -> None:
+ import_id = self._upload([f"a,{APNS_TOKEN}"], create_missing_persons=True).id
+ with (
+ patch.object(self.storage, "delete", side_effect=RuntimeError("storage down")),
+ patch(
+ "products.workflows.backend.tasks.push_token_import.expire_push_token_import_devices.apply_async"
+ ) as retry_cleanup,
+ ):
+ self._run_to_end(import_id)
+
+ assert PushTokenImport.objects.for_team(self.team.id).get(id=import_id).storage_key != ""
+ retry_cleanup.assert_called_once()
+ assert retry_cleanup.call_args.kwargs["countdown"] == int(push_token_import.DELETE_RETRY_AFTER.total_seconds())
+
+ def test_a_batch_sent_again_after_a_crash_reuses_its_event_ids(self) -> None:
+ lines = [f"a,{APNS_TOKEN}", f"b,{APNS_TOKEN}", f"c,{APNS_TOKEN}"]
+ import_id = self._upload(lines, create_missing_persons=True).id
+ assert run_push_token_import_batches(self.team.id, import_id) is True
+ first_events = [(event["event_uuid"], event["timestamp"]) for event in self.captured[0]]
+ PushTokenImport.objects.for_team(self.team.id).filter(id=import_id).update(processed_devices=0)
+ self.captured.clear()
+
+ run_push_token_import_batches(self.team.id, import_id)
+
+ assert [(event["event_uuid"], event["timestamp"]) for event in self.captured[0]] == first_events
+
+ def test_an_import_whose_stored_devices_are_gone_fails_instead_of_completing(self) -> None:
+ import_id = self._upload([f"a,{APNS_TOKEN}"], create_missing_persons=True).id
+ self.storage.objects.clear()
+
+ result = self._run_to_end(import_id)
+
+ assert result.status == PushTokenImportStatus.FAILED
+ assert "couldn't read the stored devices" in result.error
+ assert self.captured == []
+
+ def test_a_failed_delete_does_not_let_unknown_people_be_imported_twice(self) -> None:
+ first = self._run_to_end(self._upload([f"new-1,{APNS_TOKEN}"]).id)
+ with patch.object(self.storage, "delete", side_effect=RuntimeError("storage down")):
+ retry = import_unknown_persons(team=self.team, user=self.user, import_id=first.id)
+ self._run_to_end(retry.id)
+
+ with self.assertRaises(PushTokenImportInvalid):
+ import_unknown_persons(team=self.team, user=self.user, import_id=first.id)
+
+ def test_a_failed_row_write_deletes_the_stored_devices(self) -> None:
+ with patch.object(PushTokenImport.objects, "for_team", side_effect=OperationalError("database down")):
+ with self.assertRaises(OperationalError):
+ self._upload([f"a,{APNS_TOKEN}"])
+
+ assert self.storage.objects == {}
+
+ def test_progress_during_the_last_stall_check_keeps_the_import_running(self) -> None:
+ import_id = self._upload([f"user-1,{APNS_TOKEN}"]).id
+ PushTokenImport.objects.for_team(self.team.id).filter(id=import_id).update(resumed_times=3)
+ stale = timezone.now() - timedelta(hours=1)
+ PushTokenImport.objects.for_team(self.team.id).filter(id=import_id).update(updated_at=stale)
+
+ assert (
+ push_token_import.fail_push_token_import(
+ self.team.id, import_id, "stopped", unless_updated_since=stale - timedelta(seconds=1)
+ )
+ is False
+ )
+
+ job = PushTokenImport.objects.for_team(self.team.id).get(id=import_id)
+ assert job.status == PushTokenImportStatus.RUNNING
+
+ def test_an_import_that_stops_making_progress_is_resumed_then_failed(self) -> None:
+ import_id = self._upload([f"user-1,{APNS_TOKEN}"]).id
+ assert check_stalled_push_token_import(self.team.id, import_id) is True
+
+ with patch("products.workflows.backend.tasks.push_token_import.run_push_token_import.delay") as resume:
+ for minutes in [6, 12, 18]:
+ with time_machine.travel(timezone.now() + timedelta(minutes=minutes), tick=False):
+ assert check_stalled_push_token_import(self.team.id, import_id) is True
+ assert resume.call_count == 3
+ with time_machine.travel(timezone.now() + timedelta(minutes=24), tick=False):
+ assert check_stalled_push_token_import(self.team.id, import_id) is False
+
+ job = PushTokenImport.objects.for_team(self.team.id).get(id=import_id)
+ assert job.status == PushTokenImportStatus.FAILED
+ assert "stopped after 0 of 1 devices" in job.error
+ assert self.storage.objects == {}