:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(workflows): import push device tokens in the background (#129105)
Daniel Marchuk013440b86e08
diff --git a/products/workflows/backend/facade/contracts.py b/products/workflows/backend/facade/contracts.pyindex 6777f40b158c..5aea64085d3a 100644--- a/products/workflows/backend/facade/contracts.py+++ b/products/workflows/backend/facade/contracts.py@@ -21,6 +21,8 @@ from products.workflows.backend.facade.enums import (HogFlowScheduleStatus,HogFlowTemplateExitCondition,HogFlowTemplateScope,+ PushTokenImportRowFailureReason,+ PushTokenImportStatus,SetupStepKey,WorkflowImportNoteLevel,WorkflowProposalStatus,@@ -1165,3 +1167,39 @@ class WorkflowImportRecord:notes: tuple[dict[str, Any], ...]updated_at: datetimehog_flow_created_by_id: int | None = None+++class PushTokenImportInvalid(Exception):+ """The upload cannot be used. The message is shown to the person who sent it."""+++class PushTokenImportNotFound(LookupError):+ pass+++@frozen+class PushTokenImportRowFailure:+ line: int+ distinct_id: str+ device_token_suffix: str+ reason: PushTokenImportRowFailureReason+ detail: str+++@frozen+class PushTokenImportRecord:+ id: UUID+ integration_id: int+ status: PushTokenImportStatus+ create_missing_persons: bool+ total_devices: int+ processed_devices: int+ imported_devices: int+ failed_devices: int+ unknown_person_devices: int+ duplicate_lines: int+ failures: tuple[PushTokenImportRowFailure, ...]+ can_import_unknown_persons: bool+ error: str+ created_at: datetime+ finished_at: datetime | Nonediff --git a/products/workflows/backend/facade/push_token_import.py b/products/workflows/backend/facade/push_token_import.pynew file mode 100644index 000000000000..bbd64add6969--- /dev/null+++ b/products/workflows/backend/facade/push_token_import.py@@ -0,0 +1,29 @@+"""Uploaded files of push device tokens, imported into a push channel in the background."""++from products.workflows.backend.facade.contracts import (+ PushTokenImportInvalid,+ PushTokenImportNotFound,+ PushTokenImportRecord,+ PushTokenImportRowFailure,+)+from products.workflows.backend.services.push_token_import import (+ MAX_FILE_BYTES,+ cancel_push_token_import,+ get_push_token_import,+ import_unknown_persons,+ latest_push_token_import,+ start_push_token_import,+)++__all__ = [+ "MAX_FILE_BYTES",+ "PushTokenImportInvalid",+ "PushTokenImportNotFound",+ "PushTokenImportRecord",+ "PushTokenImportRowFailure",+ "cancel_push_token_import",+ "get_push_token_import",+ "import_unknown_persons",+ "latest_push_token_import",+ "start_push_token_import",+]diff --git a/products/workflows/backend/services/push_token_import.py b/products/workflows/backend/services/push_token_import.pynew file mode 100644index 000000000000..087478df2857--- /dev/null+++ b/products/workflows/backend/services/push_token_import.py@@ -0,0 +1,677 @@+"""Uploaded files of push device tokens, imported into one push channel by a background task."""++import io+import csv+import json+import time+import uuid+import zlib+from datetime import datetime, timedelta+from typing import TypedDict++from django.db import IntegrityError, transaction+from django.utils import timezone++from structlog import get_logger++from posthog.api.integration_push_tokens import (+ MAX_DISTINCT_ID_BYTES,+ MAX_SUBSCRIPTIONS_PER_REQUEST,+ TOKEN_SUFFIX_LENGTH,+ PushTokenImportFailure,+ PushTokenImportFailureReason,+ import_push_tokens,+ push_channel_app_id,+)+from posthog.dataclasses import frozen+from posthog.helpers.encrypted_fields import EncryptedFieldMixin+from posthog.integrations import IntegrationManager, IntegrationNotFound, IntegrationRecord+from posthog.models.team.team import Team+from posthog.models.user import User+from posthog.ph_client import ph_scoped_capture+from posthog.storage import object_storage++from products.workflows.backend.facade.contracts import (+ PushTokenImportInvalid,+ PushTokenImportNotFound,+ PushTokenImportRecord,+ PushTokenImportRowFailure,+)+from products.workflows.backend.facade.enums import PushTokenImportRowFailureReason, PushTokenImportStatus+from products.workflows.backend.models.push_token_import import PushTokenImport++logger = get_logger(__name__)++MAX_FILE_BYTES = 20 * 1024 * 1024+BATCH_SIZE = MAX_SUBSCRIPTIONS_PER_REQUEST+# One task run sends this many batches and then queues the next run, so a worker restart loses little.+BATCHES_PER_TASK = 20+# The result lists this many failed lines. The counts cover every line.+MAX_STORED_FAILURES = 1000+# Devices of people who don't exist yet stay stored this long, so they can be imported with new persons.+UNKNOWN_PERSON_DEVICES_KEPT_FOR = timedelta(hours=24)+# Counted from the upload, so it covers the time the import itself takes.+EXPIRY_MARGIN = timedelta(hours=1)+# A task run takes seconds, so this much quiet means the worker stopped, for example in a deploy.+STALL_CHECK_EVERY = timedelta(minutes=5)+DELETE_RETRY_AFTER = timedelta(hours=1)+NOT_STORED_ATTEMPTS = 3+NOT_STORED_RETRY_SECONDS = 2+STALLED_AFTER = timedelta(minutes=5)+MAX_RESUMES = 3++# Blank lines are a byte each, so this bounds the work a file within the size limit can cause.+MAX_FILE_LINES = 400_000+MAX_DEVICE_TOKEN_LENGTH = 4096+_DISTINCT_ID_COLUMN = "distinct_id"+_DEVICE_TOKEN_COLUMN = "device_token"+_STORAGE_FOLDER = "workflows_push_token_imports"+_PUSH_KINDS = ("apns", "firebase")+++class StagedDevice(TypedDict):+ line: int+ distinct_id: str+ device_token: str+++@frozen+class ParsedPushTokenFile:+ devices: list[StagedDevice]+ rejected: list[PushTokenImportRowFailure]+ rejected_lines: int+ duplicate_lines: int+++@frozen+class _Columns:+ distinct_id: int+ device_token: int+++class _DeviceCipher(EncryptedFieldMixin):+ pass+++def parse_push_token_csv(content: bytes) -> ParsedPushTokenFile:+ """Reads a CSV with a header row into devices. Raises `PushTokenImportInvalid` when the file can't be used."""+ # Decoding while reading keeps a second, wider copy of the whole file out of memory.+ reader = csv.reader(io.TextIOWrapper(io.BytesIO(content), encoding="utf-8-sig", newline=""), strict=True)+ devices: list[StagedDevice] = []+ rejected: list[PushTokenImportRowFailure] = []+ rejected_lines = 0+ seen: set[tuple[str, str]] = set()+ duplicate_lines = 0+ has_data = False+ columns: _Columns | None = None+ try:+ for cells in reader:+ # A quoted value can span lines, so the record's line is where it started.+ line = reader.line_num - sum(cell.count("\n") for cell in cells)+ if reader.line_num > MAX_FILE_LINES:+ raise PushTokenImportInvalid(+ f"This file has more than {MAX_FILE_LINES:,} lines. "+ "Split it into smaller files and import them one at a time."+ )+ if columns is None:+ columns = _columns(cells)+ continue+ if all(not cell.strip() for cell in cells):+ continue+ has_data = True+ distinct_id = _cell(cells, columns.distinct_id)+ device_token = _cell(cells, columns.device_token)+ if problem := _row_problem(distinct_id, device_token):+ rejected_lines += 1+ if len(rejected) < MAX_STORED_FAILURES:+ rejected.append(+ PushTokenImportRowFailure(+ line=line,+ distinct_id=distinct_id.replace("\x00", ""),+ device_token_suffix=device_token[-TOKEN_SUFFIX_LENGTH:].replace("\x00", ""),+ reason=PushTokenImportRowFailureReason.INVALID_ROW,+ detail=problem,+ )+ )+ continue+ key = (distinct_id, device_token)+ if key in seen:+ duplicate_lines += 1+ continue+ seen.add(key)+ devices.append({"line": line, "distinct_id": distinct_id, "device_token": device_token})+ except UnicodeDecodeError:+ raise PushTokenImportInvalid('This file isn\'t saved as UTF-8. Save it as "CSV UTF-8" and upload it again.')+ except csv.Error as error:+ raise PushTokenImportInvalid(f"Line {reader.line_num} of the file can't be read: {error}.")+ if columns is None:+ raise PushTokenImportInvalid("The file is empty.")+ if not has_data:+ raise PushTokenImportInvalid("The file has no devices in it, only the column names.")+ return ParsedPushTokenFile(+ devices=devices, rejected=rejected, rejected_lines=rejected_lines, duplicate_lines=duplicate_lines+ )+++def _columns(header: list[str]) -> _Columns:+ columns = [cell.strip().lower() for cell in header]+ if _DISTINCT_ID_COLUMN not in columns or _DEVICE_TOKEN_COLUMN not in columns:+ raise PushTokenImportInvalid(+ "The first line of the file must name the columns, and it needs a distinct_id column and a device_token column."+ )+ return _Columns(distinct_id=columns.index(_DISTINCT_ID_COLUMN), device_token=columns.index(_DEVICE_TOKEN_COLUMN))+++def start_push_token_import(+ *,+ team: Team,+ user: User | None,+ integration_id: int,+ content: bytes,+ create_missing_persons: bool,+) -> PushTokenImportRecord:+ _push_channel(team.id, integration_id)+ if len(content) > MAX_FILE_BYTES:+ raise PushTokenImportInvalid(+ f"This file is larger than {MAX_FILE_BYTES // (1024 * 1024)} MB. "+ "Split it into smaller files and import them one at a time."+ )+ parsed = parse_push_token_csv(content)+ if not parsed.devices:+ raise PushTokenImportInvalid(+ "None of the lines in the file can be imported. Fix them and upload the file again."+ )+ return _create_import(+ team=team,+ user=user,+ integration_id=integration_id,+ devices=parsed.devices,+ create_missing_persons=create_missing_persons,+ rejected=parsed.rejected,+ rejected_lines=parsed.rejected_lines,+ duplicate_lines=parsed.duplicate_lines,+ )+++def import_unknown_persons(*, team: Team, user: User | None, import_id: uuid.UUID) -> PushTokenImportRecord:+ """Starts a second import of the devices whose distinct ID had no person, creating those persons."""+ previous = _get(team.id, import_id)+ if not _can_import_unknown_persons(previous):+ raise PushTokenImportInvalid(+ "These devices can't be imported again. Upload the file again to import them with new persons."+ )+ storage_key = previous.storage_key+ unknown_lines = set(previous.unknown_person_lines)+ devices = [device for device in _read_devices(storage_key) if device["line"] in unknown_lines]+ if not devices:+ raise PushTokenImportInvalid(+ "These devices can't be imported again. Upload the file again to import them with new persons."+ )+ # Only one import per channel runs at a time, so a second click fails here instead of importing twice.+ record = _create_import(+ team=team,+ user=user,+ integration_id=previous.integration_id,+ devices=devices,+ create_missing_persons=True,+ rejected=[],+ rejected_lines=0,+ duplicate_lines=0,+ )+ # Cleared before the delete, so a failed delete can't let these devices be imported a second time.+ PushTokenImport.objects.for_team(team.id).filter(id=import_id).update(+ unknown_person_lines=[], updated_at=timezone.now()+ )+ _release_devices(team.id, import_id, storage_key)+ return record+++def get_push_token_import(team_id: int, import_id: uuid.UUID) -> PushTokenImportRecord:+ return _record(_get(team_id, import_id))+++def latest_push_token_import(team_id: int, integration_id: int) -> PushTokenImportRecord | None:+ latest = PushTokenImport.objects.for_team(team_id).filter(integration_id=integration_id).order_by("-created_at")+ found = latest.first()+ return _record(found) if found else None+++def cancel_push_token_import(team_id: int, import_id: uuid.UUID) -> PushTokenImportRecord:+ """Stops a running import after the batch it is sending. Devices already sent stay imported.++ The task sets `finished_at` once that batch is counted, so the import reads as stopping until then.+ """+ PushTokenImport.objects.for_team(team_id).filter(id=import_id, status=PushTokenImportStatus.RUNNING).update(+ status=PushTokenImportStatus.CANCELLED, updated_at=timezone.now()+ )+ return get_push_token_import(team_id, import_id)+++def run_push_token_import_batches(team_id: int, import_id: uuid.UUID) -> bool:+ """Sends the next batches of a running import. Returns True when batches are left for another run."""+ job = PushTokenImport.objects.for_team(team_id).filter(id=import_id).first()+ if job is None:+ return False+ if job.status != PushTokenImportStatus.RUNNING:+ _end_stopped(job)+ return False+ try:+ integration = _push_channel(team_id, job.integration_id)+ except PushTokenImportInvalid as error:+ fail_push_token_import(team_id, import_id, str(error))+ return False+ api_token = Team.objects.only("api_token").get(id=team_id).api_token+ devices = _read_devices(job.storage_key)+ if not devices:+ fail_push_token_import(+ team_id,+ import_id,+ "PostHog couldn't read the stored devices, so the import stopped. Upload the file again.",+ )+ return False++ for _ in range(BATCHES_PER_TASK):+ start = job.processed_devices+ if start >= len(devices):+ _finish(job)+ return False+ batch = devices[start : start + BATCH_SIZE]+ imported, failed = _send_batch(+ integration,+ batch,+ api_token,+ create_missing_persons=job.create_missing_persons,+ # A batch sent again after a worker restart reuses its event IDs, so its copies can be deduplicated.+ import_id=uuid.uuid5(job.id, str(start)),+ timestamp=job.created_at,+ )+ for device, failure in failed:+ line = device["line"]+ if failure.reason == PushTokenImportFailureReason.UNKNOWN_PERSON:+ job.unknown_person_lines.append(line)+ if len(job.failures) < MAX_STORED_FAILURES:+ job.failures.append(+ _failure_json(+ PushTokenImportRowFailure(+ line=line,+ distinct_id=failure.distinct_id,+ device_token_suffix=failure.device_token_suffix,+ reason=PushTokenImportRowFailureReason(failure.reason),+ detail=failure.detail,+ )+ )+ )+ job.processed_devices = start + len(batch)+ job.imported_devices += imported+ job.failed_devices += len(failed)+ # Matching the progress read at the start means a cancel or a second worker stops this one.+ saved = (+ PushTokenImport.objects.for_team(team_id)+ .filter(id=import_id, status=PushTokenImportStatus.RUNNING, processed_devices=start)+ .update(+ processed_devices=job.processed_devices,+ imported_devices=job.imported_devices,+ failed_devices=job.failed_devices,+ failures=job.failures,+ unknown_person_lines=job.unknown_person_lines,+ updated_at=timezone.now(),+ )+ )+ if not saved:+ # A stop that lands mid-batch leaves the batch's devices stored, so the stopped import still counts them.+ PushTokenImport.objects.for_team(team_id).filter(+ id=import_id, status=PushTokenImportStatus.CANCELLED, processed_devices=start+ ).update(+ processed_devices=job.processed_devices,+ imported_devices=job.imported_devices,+ failed_devices=job.failed_devices,+ failures=job.failures,+ unknown_person_lines=job.unknown_person_lines,+ updated_at=timezone.now(),+ )+ job.refresh_from_db()+ _end_stopped(job)+ return False+ if job.processed_devices >= len(devices):+ _finish(job)+ return False+ return True+++def _send_batch(+ integration: IntegrationRecord,+ batch: list[StagedDevice],+ api_token: str,+ *,+ create_missing_persons: bool,+ import_id: uuid.UUID,+ timestamp: datetime,+) -> tuple[int, list[tuple[StagedDevice, PushTokenImportFailure]]]:+ """Sends one batch, and sends again the devices ingestion did not store, so a short outage loses nothing."""+ pending = batch+ imported = 0+ failed: list[tuple[StagedDevice, PushTokenImportFailure]] = []+ for attempt in range(NOT_STORED_ATTEMPTS):+ if attempt:+ time.sleep(NOT_STORED_RETRY_SECONDS * attempt)+ result = import_push_tokens(+ integration,+ [{"distinct_id": device["distinct_id"], "device_token": device["device_token"]} for device in pending],+ api_token,+ create_missing_persons=create_missing_persons,+ import_id=import_id,+ timestamp=timestamp,+ )+ imported += result.imported+ retry: list[StagedDevice] = []+ for failure in result.failed:+ device = pending[failure.index]+ if failure.reason == PushTokenImportFailureReason.NOT_STORED and attempt + 1 < NOT_STORED_ATTEMPTS:+ retry.append(device)+ else:+ failed.append((device, failure))+ if not retry:+ break+ pending = retry+ return imported, failed+++def fail_push_token_import(+ team_id: int, import_id: uuid.UUID, error: str, *, unless_updated_since: datetime | None = None+) -> bool:+ """Returns False when the import was not running, or made progress after `unless_updated_since`."""+ running = PushTokenImport.objects.for_team(team_id).filter(id=import_id, status=PushTokenImportStatus.RUNNING)+ if unless_updated_since is not None:+ running = running.filter(updated_at=unless_updated_since)+ updated = running.update(+ status=PushTokenImportStatus.FAILED, error=error, finished_at=timezone.now(), updated_at=timezone.now()+ )+ job = PushTokenImport.objects.for_team(team_id).filter(id=import_id).first()+ if job is not None:+ _discard_devices_of_stopped(job)+ if updated:+ _report_finished(job)+ return bool(updated)+++def check_stalled_push_token_import(team_id: int, import_id: uuid.UUID) -> bool:+ """Resumes an import that stopped making progress, then fails it. Returns True while it still runs."""+ job = PushTokenImport.objects.for_team(team_id).filter(id=import_id).first()+ if job is None or job.finished_at is not None:+ return False+ if timezone.now() - job.updated_at < STALLED_AFTER:+ return True+ if job.status == PushTokenImportStatus.CANCELLED:+ _end_stopped(job)+ return False+ logger.warning("push_token_import_stalled", team_id=team_id, import_id=str(import_id))+ if job.resumed_times < MAX_RESUMES:+ # Progress is saved after each batch and a batch sent again keeps its event IDs, so resuming is safe.+ resumed = (+ PushTokenImport.objects.for_team(team_id)+ .filter(id=import_id, status=PushTokenImportStatus.RUNNING, updated_at=job.updated_at)+ .update(resumed_times=job.resumed_times + 1, updated_at=timezone.now())+ )+ if resumed:+ # The task module imports this one, so a module-level import would be circular.+ from products.workflows.backend.tasks.push_token_import import run_push_token_import # noqa: PLC0415++ run_push_token_import.delay(team_id=team_id, import_id=str(import_id))+ return True+ failed = fail_push_token_import(+ team_id,+ import_id,+ f"The import stopped after {job.processed_devices:,} of {job.total_devices:,} devices. "+ "Upload the file again to import the rest. Devices already imported are not stored twice.",+ unless_updated_since=job.updated_at,+ )+ return not failed+++def expire_unknown_person_devices(team_id: int, import_id: uuid.UUID) -> bool:+ """Deletes the devices an import kept. Returns True when the import still runs, so the check comes back later."""+ job = PushTokenImport.objects.for_team(team_id).filter(id=import_id).exclude(storage_key="").first()+ if job is None:+ return False+ if job.status == PushTokenImportStatus.RUNNING:+ return True+ _release_devices(team_id, import_id, job.storage_key)+ return False+++def _create_import(+ *,+ team: Team,+ user: User | None,+ integration_id: int,+ devices: list[StagedDevice],+ create_missing_persons: bool,+ rejected: list[PushTokenImportRowFailure],+ rejected_lines: int,+ duplicate_lines: int,+) -> PushTokenImportRecord:+ # The task module imports this one, so a module-level import would be circular.+ from products.workflows.backend.tasks.push_token_import import ( # noqa: PLC0415+ check_push_token_import_stalled,+ expire_push_token_import_devices,+ run_push_token_import,+ )++ import_id = uuid.uuid4()+ storage_key = f"{_STORAGE_FOLDER}/team-{team.id}/{import_id}.json"+ object_storage.write(storage_key, _encrypt_devices(devices))+ try:+ with transaction.atomic():+ job = PushTokenImport.objects.for_team(team.id).create(+ id=import_id,+ team_id=team.id,+ integration_id=integration_id,+ create_missing_persons=create_missing_persons,+ total_devices=len(devices),+ failed_devices=rejected_lines,+ duplicate_lines=duplicate_lines,+ failures=[_failure_json(failure) for failure in rejected[:MAX_STORED_FAILURES]],+ storage_key=storage_key,+ created_by=user,+ )+ except IntegrityError:+ _delete_object(storage_key)+ raise PushTokenImportInvalid(+ "An import into this channel is already running. Wait for it to finish, or stop it, and try again."+ )+ except Exception:+ # No import row exists yet, so nothing else would ever delete the stored devices.+ _delete_object(storage_key)+ raise+ try:+ run_push_token_import.delay(team_id=team.id, import_id=str(job.id))+ check_push_token_import_stalled.apply_async(+ kwargs={"team_id": team.id, "import_id": str(job.id)}, countdown=int(STALL_CHECK_EVERY.total_seconds())+ )+ # Scheduled with the import, so devices kept for unknown people can't outlive it if a later step fails.+ expire_push_token_import_devices.apply_async(+ kwargs={"team_id": team.id, "import_id": str(job.id)},+ countdown=int((UNKNOWN_PERSON_DEVICES_KEPT_FOR + EXPIRY_MARGIN).total_seconds()),+ )+ except Exception:+ fail_push_token_import(team.id, job.id, "The import couldn't start. Try again.")+ raise+ return _record(job)+++def _finish(job: PushTokenImport) -> None:+ keep_unknown = bool(job.unknown_person_lines) and not job.create_missing_persons+ storage_key = job.storage_key+ finished = (+ PushTokenImport.objects.for_team(job.team_id)+ .filter(id=job.id, status=PushTokenImportStatus.RUNNING)+ .update(+ status=PushTokenImportStatus.COMPLETED,+ finished_at=timezone.now(),+ updated_at=timezone.now(),+ )+ )+ if not finished:+ return+ job.refresh_from_db()+ if not keep_unknown:+ _release_devices(job.team_id, job.id, storage_key)+ _report_finished(job)+++def _end_stopped(job: PushTokenImport) -> None:+ """Marks a stopped import as ended once no batch of it is being sent any more."""+ if job.status == PushTokenImportStatus.CANCELLED and job.finished_at is None:+ if (+ PushTokenImport.objects.for_team(job.team_id)+ .filter(id=job.id, finished_at__isnull=True)+ .update(finished_at=timezone.now(), updated_at=timezone.now())+ ):+ job.refresh_from_db()+ _report_finished(job)+ _discard_devices_of_stopped(job)+++def _discard_devices_of_stopped(job: PushTokenImport) -> None:+ if job.status == PushTokenImportStatus.RUNNING or not job.storage_key or _can_import_unknown_persons(job):+ return+ _release_devices(job.team_id, job.id, job.storage_key)+++def _report_finished(job: PushTokenImport) -> None:+ user = User.objects.filter(id=job.created_by_id).only("distinct_id").first() if job.created_by_id else None+ if user is None:+ return+ try:+ with ph_scoped_capture() as capture:+ capture(+ distinct_id=str(user.distinct_id),+ event="workflows push token import finished",+ properties={+ "status": job.status,+ "devices": job.total_devices,+ "imported": job.imported_devices,+ "failed": job.failed_devices,+ "unknown_persons": len(job.unknown_person_lines),+ "create_missing_persons": job.create_missing_persons,+ },+ )+ except Exception:+ logger.exception("push_token_import_report_failed", import_id=str(job.id))+++def _push_channel(team_id: int, integration_id: int) -> IntegrationRecord:+ try:+ integration = IntegrationManager.kinds(*_PUSH_KINDS).get(team_id, integration_id)+ except IntegrationNotFound:+ raise PushTokenImportInvalid("This push channel no longer exists.")+ if not push_channel_app_id(integration):+ raise PushTokenImportInvalid("This push channel has no app set up, so devices can't be imported into it.")+ return integration+++def _get(team_id: int, import_id: uuid.UUID) -> PushTokenImport:+ job = PushTokenImport.objects.for_team(team_id).filter(id=import_id).first()+ if job is None:+ raise PushTokenImportNotFound(str(import_id))+ return job+++def _can_import_unknown_persons(job: PushTokenImport) -> bool:+ return (+ job.status == PushTokenImportStatus.COMPLETED+ and not job.create_missing_persons+ and bool(job.unknown_person_lines)+ and bool(job.storage_key)+ )+++def _read_devices(storage_key: str) -> list[StagedDevice]:+ content = object_storage.read_bytes(storage_key, missing_ok=True) if storage_key else None+ return json.loads(zlib.decompress(_DeviceCipher().f.decrypt(content))) if content else []+++def _encrypt_devices(devices: list[StagedDevice]) -> bytes:+ # Compressing first keeps the copies the cipher makes of a full-size file small.+ return _DeviceCipher().f.encrypt(zlib.compress(json.dumps(devices, separators=(",", ":")).encode(), 1))+++def _release_devices(team_id: int, import_id: uuid.UUID, storage_key: str) -> None:+ """Deletes the stored devices, and forgets the key only once they are gone, so a failed delete is tried again."""+ if not _delete_object(storage_key):+ # The task module imports this one, so a module-level import would be circular.+ from products.workflows.backend.tasks.push_token_import import expire_push_token_import_devices # noqa: PLC0415++ expire_push_token_import_devices.apply_async(+ kwargs={"team_id": team_id, "import_id": str(import_id)}, countdown=int(DELETE_RETRY_AFTER.total_seconds())+ )+ return+ PushTokenImport.objects.for_team(team_id).filter(id=import_id, storage_key=storage_key).update(+ storage_key="", updated_at=timezone.now()+ )+++def _delete_object(storage_key: str) -> bool:+ try:+ object_storage.delete(storage_key)+ except Exception:+ logger.exception("push_token_import_cleanup_failed", storage_key=storage_key)+ return False+ return True+++def _cell(cells: list[str], column: int) -> str:+ return cells[column].strip() if column < len(cells) else ""+++def _row_problem(distinct_id: str, device_token: str) -> str | None:+ if "\x00" in distinct_id or "\x00" in device_token:+ return "The line has a NUL character in it"+ if not distinct_id:+ return "No distinct ID"+ if not device_token:+ return "No device token"+ if len(distinct_id.encode()) > MAX_DISTINCT_ID_BYTES:+ return f"Distinct ID is longer than {MAX_DISTINCT_ID_BYTES} bytes"+ if len(device_token) > MAX_DEVICE_TOKEN_LENGTH:+ return f"Device token is longer than {MAX_DEVICE_TOKEN_LENGTH} characters"+ return None+++def _failure_json(failure: PushTokenImportRowFailure) -> dict[str, str | int]:+ return {+ "line": failure.line,+ "distinct_id": failure.distinct_id,+ "device_token_suffix": failure.device_token_suffix,+ "reason": str(failure.reason),+ "detail": failure.detail,+ }+++def _record(job: PushTokenImport) -> PushTokenImportRecord:+ return PushTokenImportRecord(+ id=job.id,+ integration_id=job.integration_id,+ status=PushTokenImportStatus(job.status),+ create_missing_persons=job.create_missing_persons,+ total_devices=job.total_devices,+ processed_devices=job.processed_devices,+ imported_devices=job.imported_devices,+ failed_devices=job.failed_devices,+ unknown_person_devices=len(job.unknown_person_lines),+ duplicate_lines=job.duplicate_lines,+ failures=tuple(+ PushTokenImportRowFailure(+ line=int(failure["line"]),+ distinct_id=str(failure["distinct_id"]),+ device_token_suffix=str(failure["device_token_suffix"]),+ reason=PushTokenImportRowFailureReason(failure["reason"]),+ detail=str(failure["detail"]),+ )+ for failure in sorted(job.failures, key=lambda failure: int(failure["line"]))+ ),+ can_import_unknown_persons=_can_import_unknown_persons(job),+ error=job.error,+ created_at=job.created_at,+ finished_at=job.finished_at,+ )diff --git a/products/workflows/backend/tasks/__init__.py b/products/workflows/backend/tasks/__init__.pyindex c57aedd65e4e..4ab1316c0d5a 100644--- a/products/workflows/backend/tasks/__init__.py+++ b/products/workflows/backend/tasks/__init__.py@@ -4,6 +4,7 @@ from . import (email_sending_tiers,hog_flows,people_import,+ push_token_import,ses_account_reputation,)@@ -13,5 +14,6 @@ __all__ = ["email_sending_tiers","hog_flows","people_import",+ "push_token_import","ses_account_reputation",]diff --git a/products/workflows/backend/tasks/push_token_import.py b/products/workflows/backend/tasks/push_token_import.pynew file mode 100644index 000000000000..e73f51ac3732--- /dev/null+++ b/products/workflows/backend/tasks/push_token_import.py@@ -0,0 +1,72 @@+from uuid import UUID++from celery import Task, shared_task+from structlog import get_logger++from posthog.scoping_audit import skip_team_scope_audit+from posthog.tasks.utils import CeleryQueue++from products.workflows.backend.services.push_token_import import (+ EXPIRY_MARGIN,+ STALL_CHECK_EVERY,+ check_stalled_push_token_import,+ expire_unknown_person_devices,+ fail_push_token_import,+ run_push_token_import_batches,+)++logger = get_logger(__name__)++MAX_RETRIES = 3+MAINTENANCE_RETRIES = 8+++@shared_task(bind=True, ignore_result=True, queue=CeleryQueue.DEFAULT.value, max_retries=MAX_RETRIES)+@skip_team_scope_audit+def run_push_token_import(self: Task, *, team_id: int, import_id: str) -> None:+ try:+ more = run_push_token_import_batches(team_id, UUID(import_id))+ except Exception as error:+ logger.warning("push_token_import_batch_failed", team_id=team_id, import_id=import_id, error=str(error))+ if self.request.retries < MAX_RETRIES:+ raise self.retry(countdown=30 * (self.request.retries + 1))+ fail_push_token_import(+ team_id,+ UUID(import_id),+ "PostHog couldn't save the devices, so the import stopped. Upload the file again to import the rest. "+ "Devices already imported are not stored twice.",+ )+ return+ if more:+ run_push_token_import.delay(team_id=team_id, import_id=import_id)+++# A check that dies does not schedule the next one, so an error has to retry it rather than end the chain.+@shared_task(+ ignore_result=True,+ queue=CeleryQueue.DEFAULT.value,+ autoretry_for=(Exception,),+ retry_backoff=30,+ max_retries=MAINTENANCE_RETRIES,+)+@skip_team_scope_audit+def check_push_token_import_stalled(*, team_id: int, import_id: str) -> None:+ if check_stalled_push_token_import(team_id, UUID(import_id)):+ check_push_token_import_stalled.apply_async(+ kwargs={"team_id": team_id, "import_id": import_id}, countdown=int(STALL_CHECK_EVERY.total_seconds())+ )+++@shared_task(+ ignore_result=True,+ queue=CeleryQueue.DEFAULT.value,+ autoretry_for=(Exception,),+ retry_backoff=30,+ max_retries=MAINTENANCE_RETRIES,+)+@skip_team_scope_audit+def expire_push_token_import_devices(*, team_id: int, import_id: str) -> None:+ if expire_unknown_person_devices(team_id, UUID(import_id)):+ expire_push_token_import_devices.apply_async(+ kwargs={"team_id": team_id, "import_id": import_id}, countdown=int(EXPIRY_MARGIN.total_seconds())+ )diff --git a/products/workflows/backend/test/test_push_token_import.py b/products/workflows/backend/test/test_push_token_import.pynew file mode 100644index 000000000000..436359852bc3--- /dev/null+++ b/products/workflows/backend/test/test_push_token_import.py@@ -0,0 +1,332 @@+from datetime import timedelta+from typing import Any+from uuid import UUID++import time_machine+from posthog.test.base import APIBaseTest, _create_person, flush_persons_and_events+from unittest.mock import patch++from django.db import OperationalError+from django.test import SimpleTestCase+from django.utils import timezone++from parameterized import parameterized++from posthog.api.capture import CaptureInternalResult+from posthog.models.integration import Integration+from posthog.models.organization import OrganizationMembership++from products.workflows.backend.facade.contracts import PushTokenImportInvalid, PushTokenImportRecord+from products.workflows.backend.facade.enums import PushTokenImportRowFailureReason, PushTokenImportStatus+from products.workflows.backend.models.push_token_import import PushTokenImport+from products.workflows.backend.services import push_token_import+from products.workflows.backend.services.push_token_import import (+ cancel_push_token_import,+ check_stalled_push_token_import,+ get_push_token_import,+ import_unknown_persons,+ parse_push_token_csv,+ run_push_token_import_batches,+ start_push_token_import,+)++APNS_TOKEN = "a1" * 32+++class _MemoryStorage:+ def __init__(self) -> None:+ self.objects: dict[str, bytes] = {}++ def write(self, key: str, content: bytes) -> None:+ self.objects[key] = content++ def read_bytes(self, key: str, *, missing_ok: bool = False) -> bytes | None:+ return self.objects.get(key)++ def delete(self, key: str) -> None:+ self.objects.pop(key, None)+++class TestParsePushTokenCsv(SimpleTestCase):+ @parameterized.expand(+ [+ ("not_utf8", "distinct_id,device_token\nuser-é,abc\n".encode("latin-1"), "isn't saved as UTF-8"),+ ("missing_column", b"distinct_id,token\nuser-1,abc\n", "needs a distinct_id column"),+ ("only_header", b"distinct_id,device_token\n\n , \n", "only the column names"),+ ("empty", b"", "The file is empty"),+ ("broken_quotes", b'distinct_id,device_token\n"user-1,abc\n', "can't be read"),+ ("too_many_lines", b"distinct_id,device_token\nuser-1,abc" + b"\n" * 400_001, "more than 400,000 lines"),+ ]+ )+ def test_rejects_a_file_it_cannot_use(self, _name: str, content: bytes, message: str) -> None:+ with self.assertRaisesMessage(PushTokenImportInvalid, message):+ parse_push_token_csv(content)++ def test_reads_devices_with_the_line_each_came_from(self) -> None:+ content = (+ "Name,Distinct_ID , DEVICE_TOKEN\n"+ '"Ada\nLovelace",user-1,tok-1\n'+ "\n"+ "Bob,user-2,tok-2\n"+ "Bob again,user-2,tok-2\n"+ "No id,,tok-3\n"+ f"Long,user-4,{'x' * 4097}\n"+ "Ünï 🎉,user-🎉,tok-5\n"+ "Nul,,tok\x00\n"+ f"Emoji,{'🎉' * 51},tok-6\n"+ ).encode()++ parsed = parse_push_token_csv(content)++ assert parsed.devices == [+ {"line": 2, "distinct_id": "user-1", "device_token": "tok-1"},+ {"line": 5, "distinct_id": "user-2", "device_token": "tok-2"},+ {"line": 9, "distinct_id": "user-🎉", "device_token": "tok-5"},+ ]+ assert [(failure.line, failure.detail) for failure in parsed.rejected] == [+ (7, "No distinct ID"),+ (8, "Device token is longer than 4096 characters"),+ (10, "The line has a NUL character in it"),+ (11, "Distinct ID is longer than 200 bytes"),+ ]+ assert parsed.duplicate_lines == 1+++class _PushTokenImportTestCase(APIBaseTest):+ def setUp(self) -> None:+ super().setUp()+ self.organization_membership.level = OrganizationMembership.Level.ADMIN+ self.organization_membership.save()+ self.apns = Integration.objects.create(+ team=self.team,+ kind="apns",+ integration_id="TEAM.com.example.app",+ config={"bundle_id": "com.example.app", "team_id": "TEAM", "key_id": "KEY"},+ )+ self.storage = _MemoryStorage()+ patchers: list[Any] = [+ patch.object(push_token_import, "object_storage", self.storage),+ patch.object(push_token_import, "BATCH_SIZE", 2),+ patch.object(push_token_import, "BATCHES_PER_TASK", 1),+ patch("products.workflows.backend.tasks.push_token_import.run_push_token_import.delay"),+ patch("products.workflows.backend.tasks.push_token_import.check_push_token_import_stalled.apply_async"),+ patch.object(push_token_import, "ph_scoped_capture"),+ ]+ for patcher in patchers:+ patcher.start()+ self.addCleanup(patcher.stop)+ expire = patch(+ "products.workflows.backend.tasks.push_token_import.expire_push_token_import_devices.apply_async"+ )+ self.schedule_expiry = expire.start()+ self.addCleanup(expire.stop)+ self.captured: list[list[dict[str, Any]]] = []+ self.drop_once: set[str] = set()+ capture = patch("posthog.api.integration_push_tokens.capture_batch_internal", side_effect=self._capture)+ capture.start()+ self.addCleanup(capture.stop)++ def _capture(self, *, events: list[dict[str, Any]], **kwargs: Any) -> CaptureInternalResult:+ self.captured.append(events)+ result = CaptureInternalResult(status_code=200)+ for event in events:+ dropped = event["distinct_id"] in self.drop_once+ self.drop_once.discard(event["distinct_id"])+ (result.dropped if dropped else result.ok).append(event["event_uuid"])+ return result++ def _upload(self, lines: list[str], *, create_missing_persons: bool = False) -> PushTokenImportRecord:+ return start_push_token_import(+ team=self.team,+ user=self.user,+ integration_id=self.apns.id,+ content="\n".join(["distinct_id,device_token", *lines]).encode(),+ create_missing_persons=create_missing_persons,+ )++ def _run_to_end(self, import_id: UUID) -> PushTokenImportRecord:+ while run_push_token_import_batches(self.team.id, import_id):+ pass+ return get_push_token_import(self.team.id, import_id)+++class TestPushTokenImportJob(_PushTokenImportTestCase):+ def test_imports_the_file_in_batches_and_reports_each_failed_line(self) -> None:+ for distinct_id in ["user-1", "user-2", "user-3"]:+ _create_person(team=self.team, distinct_ids=[distinct_id])+ flush_persons_and_events()++ started = self._upload(+ [+ f"user-1,{APNS_TOKEN}",+ f"user-2,{APNS_TOKEN}",+ ",missing-id",+ "user-3,ExponentPushToken[abc]",+ f"nobody,{APNS_TOKEN}",+ f"user-3,{'b2' * 32}",+ ]+ )+ assert started.status == PushTokenImportStatus.RUNNING+ kept_for = push_token_import.UNKNOWN_PERSON_DEVICES_KEPT_FOR + push_token_import.EXPIRY_MARGIN+ assert self.schedule_expiry.call_args.kwargs["countdown"] == int(kept_for.total_seconds())++ result = self._run_to_end(started.id)++ assert result.status == PushTokenImportStatus.COMPLETED+ assert [len(batch) for batch in self.captured] == [2, 1]+ assert (result.total_devices, result.imported_devices, result.failed_devices) == (5, 3, 3)+ assert sorted((failure.line, failure.reason) for failure in result.failures) == [+ (4, PushTokenImportRowFailureReason.INVALID_ROW),+ (5, PushTokenImportRowFailureReason.TOKEN_MISMATCH),+ (6, PushTokenImportRowFailureReason.UNKNOWN_PERSON),+ ]+ assert result.unknown_person_devices == 1+ assert result.can_import_unknown_persons is True++ def test_imports_the_devices_of_unknown_people_once_with_new_persons(self) -> None:+ _create_person(team=self.team, distinct_ids=["user-1"])+ flush_persons_and_events()+ first = self._run_to_end(self._upload([f"user-1,{APNS_TOKEN}", f"new-1,{APNS_TOKEN}", f"new-2,{'c3' * 32}"]).id)+ self.captured.clear()++ retry = import_unknown_persons(team=self.team, user=self.user, import_id=first.id)+ with self.assertRaises(PushTokenImportInvalid):+ import_unknown_persons(team=self.team, user=self.user, import_id=first.id)++ result = self._run_to_end(retry.id)+ assert result.create_missing_persons is True+ assert (result.total_devices, result.imported_devices) == (2, 2)+ assert sorted(event["distinct_id"] for batch in self.captured for event in batch) == ["new-1", "new-2"]+ assert get_push_token_import(self.team.id, first.id).can_import_unknown_persons is False+ assert self.storage.objects == {}++ def test_a_channel_runs_one_import_at_a_time_and_a_cancelled_import_stops_sending(self) -> None:+ first = self._upload([f"user-1,{APNS_TOKEN}"])+ with self.assertRaisesRegex(PushTokenImportInvalid, "already running"):+ self._upload([f"user-2,{APNS_TOKEN}"])++ cancelled = cancel_push_token_import(self.team.id, first.id)+ result = self._run_to_end(first.id)++ assert (cancelled.status, cancelled.finished_at) == (PushTokenImportStatus.CANCELLED, None)+ assert result.status == PushTokenImportStatus.CANCELLED+ assert result.finished_at is not None+ assert self.captured == []+ assert self.storage.objects == {}+ assert self._upload([f"user-2,{APNS_TOKEN}"]).status == PushTokenImportStatus.RUNNING++ def test_a_stop_that_lands_during_a_batch_still_counts_that_batch(self) -> None:+ import_id = self._upload(+ [f"a,{APNS_TOKEN}", f"b,{APNS_TOKEN}", f"c,{APNS_TOKEN}"], create_missing_persons=True+ ).id++ def stop_while_sending(**kwargs: Any) -> CaptureInternalResult:+ cancel_push_token_import(self.team.id, import_id)+ return self._capture(**kwargs)++ with patch("posthog.api.integration_push_tokens.capture_batch_internal", side_effect=stop_while_sending):+ result = self._run_to_end(import_id)++ assert (result.status, result.processed_devices, result.imported_devices) == (+ PushTokenImportStatus.CANCELLED,+ 2,+ 2,+ )+ assert result.finished_at is not None+ assert self.storage.objects == {}++ @patch.object(push_token_import.time, "sleep")+ def test_a_device_ingestion_did_not_store_is_sent_again_with_its_event_id(self, _sleep: Any) -> None:+ self.drop_once = {"b"}+ import_id = self._upload([f"a,{APNS_TOKEN}", f"b,{APNS_TOKEN}"], create_missing_persons=True).id++ result = self._run_to_end(import_id)++ assert (result.imported_devices, result.failed_devices) == (2, 0)+ assert [event["distinct_id"] for event in self.captured[1]] == ["b"]+ assert self.captured[1][0]["event_uuid"] == self.captured[0][1]["event_uuid"]++ def test_a_failed_delete_keeps_the_devices_known_so_cleanup_runs_again(self) -> None:+ import_id = self._upload([f"a,{APNS_TOKEN}"], create_missing_persons=True).id+ with (+ patch.object(self.storage, "delete", side_effect=RuntimeError("storage down")),+ patch(+ "products.workflows.backend.tasks.push_token_import.expire_push_token_import_devices.apply_async"+ ) as retry_cleanup,+ ):+ self._run_to_end(import_id)++ assert PushTokenImport.objects.for_team(self.team.id).get(id=import_id).storage_key != ""+ retry_cleanup.assert_called_once()+ assert retry_cleanup.call_args.kwargs["countdown"] == int(push_token_import.DELETE_RETRY_AFTER.total_seconds())++ def test_a_batch_sent_again_after_a_crash_reuses_its_event_ids(self) -> None:+ lines = [f"a,{APNS_TOKEN}", f"b,{APNS_TOKEN}", f"c,{APNS_TOKEN}"]+ import_id = self._upload(lines, create_missing_persons=True).id+ assert run_push_token_import_batches(self.team.id, import_id) is True+ first_events = [(event["event_uuid"], event["timestamp"]) for event in self.captured[0]]+ PushTokenImport.objects.for_team(self.team.id).filter(id=import_id).update(processed_devices=0)+ self.captured.clear()++ run_push_token_import_batches(self.team.id, import_id)++ assert [(event["event_uuid"], event["timestamp"]) for event in self.captured[0]] == first_events++ def test_an_import_whose_stored_devices_are_gone_fails_instead_of_completing(self) -> None:+ import_id = self._upload([f"a,{APNS_TOKEN}"], create_missing_persons=True).id+ self.storage.objects.clear()++ result = self._run_to_end(import_id)++ assert result.status == PushTokenImportStatus.FAILED+ assert "couldn't read the stored devices" in result.error+ assert self.captured == []++ def test_a_failed_delete_does_not_let_unknown_people_be_imported_twice(self) -> None:+ first = self._run_to_end(self._upload([f"new-1,{APNS_TOKEN}"]).id)+ with patch.object(self.storage, "delete", side_effect=RuntimeError("storage down")):+ retry = import_unknown_persons(team=self.team, user=self.user, import_id=first.id)+ self._run_to_end(retry.id)++ with self.assertRaises(PushTokenImportInvalid):+ import_unknown_persons(team=self.team, user=self.user, import_id=first.id)++ def test_a_failed_row_write_deletes_the_stored_devices(self) -> None:+ with patch.object(PushTokenImport.objects, "for_team", side_effect=OperationalError("database down")):+ with self.assertRaises(OperationalError):+ self._upload([f"a,{APNS_TOKEN}"])++ assert self.storage.objects == {}++ def test_progress_during_the_last_stall_check_keeps_the_import_running(self) -> None:+ import_id = self._upload([f"user-1,{APNS_TOKEN}"]).id+ PushTokenImport.objects.for_team(self.team.id).filter(id=import_id).update(resumed_times=3)+ stale = timezone.now() - timedelta(hours=1)+ PushTokenImport.objects.for_team(self.team.id).filter(id=import_id).update(updated_at=stale)++ assert (+ push_token_import.fail_push_token_import(+ self.team.id, import_id, "stopped", unless_updated_since=stale - timedelta(seconds=1)+ )+ is False+ )++ job = PushTokenImport.objects.for_team(self.team.id).get(id=import_id)+ assert job.status == PushTokenImportStatus.RUNNING++ def test_an_import_that_stops_making_progress_is_resumed_then_failed(self) -> None:+ import_id = self._upload([f"user-1,{APNS_TOKEN}"]).id+ assert check_stalled_push_token_import(self.team.id, import_id) is True++ with patch("products.workflows.backend.tasks.push_token_import.run_push_token_import.delay") as resume:+ for minutes in [6, 12, 18]:+ with time_machine.travel(timezone.now() + timedelta(minutes=minutes), tick=False):+ assert check_stalled_push_token_import(self.team.id, import_id) is True+ assert resume.call_count == 3+ with time_machine.travel(timezone.now() + timedelta(minutes=24), tick=False):+ assert check_stalled_push_token_import(self.team.id, import_id) is False++ job = PushTokenImport.objects.for_team(self.team.id).get(id=import_id)+ assert job.status == PushTokenImportStatus.FAILED+ assert "stopped after 0 of 1 devices" in job.error+ assert self.storage.objects == {}