:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(messaging): keep customer.io suppression dates and bounce diagnostics (#124198)
Harley Alexander2b9734345017
diff --git a/products/messaging/backend/services/customerio_import_service.py b/products/messaging/backend/services/customerio_import_service.pyindex d727a9940bed..3eb071e18c47 100644--- a/products/messaging/backend/services/customerio_import_service.py+++ b/products/messaging/backend/services/customerio_import_service.py@@ -3,11 +3,13 @@ import csvimport jsonimport timeimport logging+from datetime import UTC, datetimefrom typing import Any, Optionalfrom django.db import transactionfrom django.utils import timezone+from posthog.dataclasses import frozenfrom posthog.models.scoping.manager import resolve_effective_team_idfrom products.messaging.backend.models.message_category import MessageCategory, MessageCategoryType@@ -35,6 +37,14 @@ SUPPRESSION_PAGE_SIZE = 1000SUPPRESSION_PAGE_DELAY_SECONDS = 0.2+@frozen+class _ImportedSuppression:+ # When Customer.io suppressed the address, or None when the entry has no time.+ suppressed_at: datetime | None+ # The bounce status code and Mailgun's reason, for example "550 Mailbox full". None for spam reports.+ diagnostic: str | None++class CustomerIOImportService:def __init__(self, team_id: int, api_key: Optional[str], created_by_id: Optional[int]):self.team_id = team_id@@ -436,29 +446,31 @@ class CustomerIOImportService:self.progress["errors"].append(f"Error importing Customer.io {suppression_type}: {str(e)[:200]}")break- emails = sorted(- {- email.strip().lower()- for entry in page- if isinstance(entry, dict) and isinstance(email := entry.get("email"), str) and "@" in email- }- )- if emails:- self.progress["suppressions_imported"] += self._add_suppressions(team_id, emails, source, reason)+ imported = {+ email.strip().lower(): _imported_suppression(entry)+ for entry in page+ if isinstance(entry, dict) and isinstance(email := entry.get("email"), str) and "@" in email+ }+ if imported:+ self.progress["suppressions_imported"] += self._add_suppressions(team_id, imported, source, reason)if len(page) < SUPPRESSION_PAGE_SIZE:breakoffset += len(page)time.sleep(SUPPRESSION_PAGE_DELAY_SECONDS)- def _add_suppressions(self, team_id: int, emails: list[str], source: SuppressionSource, reason: str) -> int:+ def _add_suppressions(+ self, team_id: int, imported: dict[str, _ImportedSuppression], source: SuppressionSource, reason: str+ ) -> int:"""Suppress the addresses that have no suppression row yet, and return how many it added.An existing row is left alone. It can be a manual entry, an address someone removed from the list on- purpose, or a row the bounce feedback already manages.+ purpose, or a row the bounce feedback already manages. The one exception is a row an earlier import of+ this list created without Customer.io's date and diagnostic, which gets them filled in."""suppressions = MessageSuppression.objects.for_team(team_id, canonical=True)- existing = set(suppressions.filter(identifier__in=emails).values_list("identifier", flat=True))+ existing = {row.identifier: row for row in suppressions.filter(identifier__in=list(imported))}now = timezone.now()+ is_bounce = source == SuppressionSource.BOUNCEnew_rows = [MessageSuppression(team_id=team_id,@@ -466,14 +478,29 @@ class CustomerIOImportService:source=source,reason=reason,suppressed=True,- suppressed_at=now,+ suppressed_at=details.suppressed_at or now,+ last_bounce_at=details.suppressed_at if is_bounce else None,+ last_bounce_diagnostic=details.diagnostic,created_by_id=self.created_by_id,)- for email in emails+ for email, details in sorted(imported.items())if email not in existing]# ignore_conflicts covers an address the bounce feedback inserts between the read above and this write.suppressions.bulk_create(new_rows, batch_size=500, ignore_conflicts=True)++ backfilled = []+ for email, row in existing.items():+ details = imported[email]+ if row.reason != reason or row.deleted or row.last_bounce_diagnostic or not details.suppressed_at:+ continue+ row.suppressed_at = details.suppressed_at+ row.last_bounce_at = details.suppressed_at if is_bounce else row.last_bounce_at+ row.last_bounce_diagnostic = details.diagnostic+ backfilled.append(row)+ suppressions.bulk_update(+ backfilled, ["suppressed_at", "last_bounce_at", "last_bounce_diagnostic"], batch_size=500+ )return len(new_rows)def _bulk_update_all_preferences(self, emails: list[str], category_ids: list[str]) -> None:@@ -541,3 +568,15 @@ class CustomerIOImportService:def get_progress(self) -> dict[str, Any]:return self.progress+++def _imported_suppression(entry: dict[str, Any]) -> _ImportedSuppression:+ """The date and bounce diagnostic Customer.io returns for one suppressed address."""+ created = entry.get("created")+ suppressed_at = (+ datetime.fromtimestamp(created, tz=UTC) if isinstance(created, int | float) and created > 0 else None+ )+ parts = [+ part.strip() for part in (entry.get("status"), entry.get("reason")) if isinstance(part, str) and part.strip()+ ]+ return _ImportedSuppression(suppressed_at=suppressed_at, diagnostic=" ".join(parts)[:1000] or None)diff --git a/products/messaging/backend/services/test_customerio_import_service.py b/products/messaging/backend/services/test_customerio_import_service.pyindex 85ef2c497cf2..904c9ae85a67 100644--- a/products/messaging/backend/services/test_customerio_import_service.py+++ b/products/messaging/backend/services/test_customerio_import_service.py@@ -307,10 +307,39 @@ [email protected],cio_6,"{""topics"": {""topic_1"": true, ""topic_2"": true}}"MessageSuppression.objects.for_team(self.team.id).create(team=self.team, identifier="[email protected]", suppressed=False, deleted=True)+ MessageSuppression.objects.for_team(self.team.id).create(+ team=self.team,+ identifier="[email protected]",+ source=SuppressionSource.BOUNCE,+ reason="Imported from Customer.io (bounced)",+ suppressed=True,+ )+ bounced_at, earlier_at, spam_at = 1700000000, 1700100000, 1700200000pages = {- ("bounces", 0): [{"email": " [email protected] "}, {"email": "[email protected]"}],- ("bounces", 2): [{"email": "[email protected]"}],- ("spam_reports", 0): [{"email": "[email protected]"}],+ ("bounces", 0): [+ {+ "email": " [email protected] ",+ "created": bounced_at,+ "status": "550",+ "reason": "Recipient address rejected: user unknown",+ },+ {+ "email": "[email protected]",+ "created": bounced_at,+ "status": "550",+ "reason": "Recipient address rejected: user unknown",+ },+ ],+ ("bounces", 2): [+ {+ "email": "[email protected]",+ "created": bounced_at,+ "status": "550",+ "reason": "Recipient address rejected: user unknown",+ },+ {"email": "[email protected]", "created": earlier_at, "status": "550", "reason": "Mailbox full"},+ ],+ ("spam_reports", 0): [{"email": "[email protected]", "created": spam_at, "status": "", "reason": ""}],}def get_suppressions(suppression_type: str, limit: int, offset: int) -> list[dict]:@@ -338,7 +367,26 @@ [email protected],cio_6,"{""topics"": {""topic_1"": true, ""topic_2"": true}}""[email protected]": (SuppressionSource.COMPLAINT, True, False),"[email protected]": (SuppressionSource.MANUAL, True, False),"[email protected]": (SuppressionSource.BOUNCE, False, True),+ "[email protected]": (SuppressionSource.BOUNCE, True, False),}+ details = {+ row.identifier: (+ row.suppressed_at and int(row.suppressed_at.timestamp()),+ row.last_bounce_at and int(row.last_bounce_at.timestamp()),+ row.last_bounce_diagnostic,+ )+ for row in MessageSuppression.objects.for_team(self.team.id).filter(+ identifier__in=["[email protected]", "[email protected]", "[email protected]", "[email protected]"]+ )+ }+ assert details["[email protected]"] == (+ bounced_at,+ bounced_at,+ "550 Recipient address rejected: user unknown",+ )+ assert details["[email protected]"] == (spam_at, None, None)+ assert details["[email protected]"] == (earlier_at, earlier_at, "550 Mailbox full")+ assert details["[email protected]"][2] is None@patch("products.messaging.backend.services.customerio_import_service.SUPPRESSION_PAGE_DELAY_SECONDS", 0)@patch("products.messaging.backend.services.customerio_import_service.CustomerIOClient")