Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

refactor(messaging): split c.io import paging from saving (#124424)

Harley Alexander950a6a49daec
diff --git a/products/messaging/backend/services/customerio_client.py b/products/messaging/backend/services/customerio_client.py
index 7d5237554f0f..3c008e44ae5b 100644
--- a/products/messaging/backend/services/customerio_client.py
+++ b/products/messaging/backend/services/customerio_client.py
@@ -8,7 +8,10 @@ logger = logging.getLogger(__name__)
class CustomerIOAPIError(Exception):
- pass
+ def __init__(self, message: str, status_code: int | None = None):
+ super().__init__(message)
+ # The HTTP status Customer.io returned, or None when the request got no response.
+ self.status_code = status_code
class CustomerIOClient:
@@ -58,7 +61,7 @@ class CustomerIOClient:
logger.exception(f"API request failed. {error_msg}. {status_info}")
else:
logger.exception(f"API request failed. {error_msg}")
- raise CustomerIOAPIError(error_msg)
+ raise CustomerIOAPIError(error_msg, status_code=e.response.status_code if e.response is not None else None)
except requests.exceptions.RequestException as e:
raise CustomerIOAPIError(f"Request failed: {e}")
diff --git a/products/messaging/backend/services/customerio_config.py b/products/messaging/backend/services/customerio_config.py
index 81753f46f8ae..0a82f72cfbe7 100644
--- a/products/messaging/backend/services/customerio_config.py
+++ b/products/messaging/backend/services/customerio_config.py
@@ -23,7 +23,8 @@ class ConfigIncomplete(Exception):
pass
-def import_from_customerio(team_id: int, app_api_key: str | None, created_by_id: int | None) -> dict[str, Any]:
+def resolve_app_api_key(team_id: int, app_api_key: str | None) -> str:
+ """The App API key to import with: the one sent, or the stored one when none is sent."""
integration = Integration.objects.filter(team_id=team_id, kind="customerio-app").first()
api_key = app_api_key or (integration.sensitive_config.get("app_api_key") if integration else None)
@@ -33,6 +34,12 @@ def import_from_customerio(team_id: int, app_api_key: str | None, created_by_id:
if integration and app_api_key:
raise ConfigConflict("Integration already exists. Delete it first to use a different key.")
+ return api_key
+
+
+def save_app_integration(team_id: int, api_key: str, created_by_id: int | None) -> Integration:
+ """Store the App API key on first use and link it to the sync config. Returns the stored integration."""
+ integration = Integration.objects.filter(team_id=team_id, kind="customerio-app").first()
if not integration:
integration = Integration.objects.create(
team_id=team_id,
@@ -45,6 +52,18 @@ def import_from_customerio(team_id: int, app_api_key: str | None, created_by_id:
config, _ = OptOutSyncConfig.objects.get_or_create(team_id=team_id)
config.app_integration = integration
config.save(update_fields=["app_integration"])
+ return integration
+
+
+def stored_app_api_key(team_id: int, integration_id: int) -> str | None:
+ integration = Integration.objects.filter(team_id=team_id, kind="customerio-app", id=integration_id).first()
+ return integration.sensitive_config.get("app_api_key") if integration else None
+
+
+def import_from_customerio(team_id: int, app_api_key: str | None, created_by_id: int | None) -> dict[str, Any]:
+ api_key = resolve_app_api_key(team_id, app_api_key)
+ save_app_integration(team_id, api_key, created_by_id)
+ config = OptOutSyncConfig.objects.get(team_id=team_id)
# Run import synchronously
result = CustomerIOImportService(team_id=team_id, api_key=api_key, created_by_id=created_by_id).import_api_data()
diff --git a/products/messaging/backend/services/customerio_import_service.py b/products/messaging/backend/services/customerio_import_service.py
index 3eb071e18c47..8c22f7bd0cc5 100644
--- a/products/messaging/backend/services/customerio_import_service.py
+++ b/products/messaging/backend/services/customerio_import_service.py
@@ -3,6 +3,7 @@ import csv
import json
import time
import logging
+from collections.abc import Iterator
from datetime import UTC, datetime
from typing import Any, Optional
@@ -20,7 +21,7 @@ from products.messaging.backend.models.message_preferences import (
)
from products.messaging.backend.models.message_suppression import MessageSuppression, SuppressionSource
-from .customerio_client import CustomerIOClient
+from .customerio_client import CustomerIOAPIError, CustomerIOClient
logger = logging.getLogger(__name__)
@@ -45,6 +46,26 @@ class _ImportedSuppression:
diagnostic: str | None
+@frozen
+class SuppressionPage:
+ suppression_type: str
+ # How many entries Customer.io returned, including any without a usable address.
+ entries: int
+ addresses: dict[str, _ImportedSuppression]
+ # The offset of the next page in the same suppression list.
+ next_offset: int
+ is_last: bool
+
+
+@frozen
+class GlobalUnsubscribePage:
+ # How many customers Customer.io returned, including any without an email address.
+ customers: int
+ emails: tuple[str, ...]
+ # The cursor of the next page, or None on the last page.
+ next_start: str | None
+
+
class CustomerIOImportService:
def __init__(self, team_id: int, api_key: Optional[str], created_by_id: Optional[int]):
self.team_id = team_id
@@ -66,6 +87,30 @@ class CustomerIOImportService:
"details": "",
}
+ def connect(self, timeout: int = 30) -> bool:
+ """Find the region the App API key belongs to and keep a client for it.
+
+ Returns False when every region rejects the key. Raises CustomerIOAPIError when a region fails in another
+ way, for example a timeout or a server error, because the key can still be valid there.
+ """
+ if not self.api_key:
+ return False
+ unavailable: CustomerIOAPIError | None = None
+ for region in ("us", "eu"):
+ client = CustomerIOClient(self.api_key, region=region, timeout=timeout)
+ try:
+ client.get_subscription_topics()
+ except CustomerIOAPIError as e:
+ if e.status_code not in (401, 403):
+ unavailable = e
+ continue
+ self.client = client
+ return True
+ self.client = None
+ if unavailable is not None:
+ raise unavailable
+ return False
+
def import_api_data(self) -> dict[str, Any]:
"""Import categories, globally unsubscribed users and suppressed addresses via API"""
if not self.api_key:
@@ -92,7 +137,7 @@ class CustomerIOImportService:
topics = self.client.get_subscription_topics()
self.progress["topics_found"] = len(topics)
- self._import_categories(topics)
+ self.import_categories(topics)
# Step 3: Import addresses Customer.io's email provider suppressed. They go before the global
# unsubscribes because those can run to hundreds of pages, past the request timeout, and each page
@@ -325,7 +370,7 @@ class CustomerIOImportService:
return preferences_count
- def _import_categories(self, topics: list[dict[str, Any]]) -> None:
+ def import_categories(self, topics: list[dict[str, Any]]) -> None:
"""Import Customer.io topics as message categories"""
for topic in topics:
try:
@@ -371,52 +416,35 @@ class CustomerIOImportService:
if not self.client:
return
- start = None
batch_num = 0
total_processed = 0
- global_opt_out_ids = [ALL_MESSAGE_PREFERENCE_CATEGORY_ID]
+ pages = self.fetch_global_unsubscribe_pages(start=None)
while True:
batch_num += 1
self.progress["current_batch"] = batch_num
try:
- # Fetch batch of globally unsubscribed customers
- response = self.client.get_globally_unsubscribed_customers(limit=1000, start=start)
- identifiers = response.get("identifiers", [])
-
- if not identifiers:
+ page = next(pages, None)
+ if page is None:
break
- self.progress["customers_in_current_batch"] = len(identifiers)
+ self.progress["customers_in_current_batch"] = page.customers
self.progress["details"] = (
- f"Processing batch {batch_num} of globally unsubscribed users ({len(identifiers)} users)..."
+ f"Processing batch {batch_num} of globally unsubscribed users ({page.customers} users)..."
)
+ # Add to our global set of processed users
+ self.all_processed_users.update(page.emails)
- # Collect emails for this batch
- emails_to_process = []
- for customer_info in identifiers:
- email = customer_info.get("email")
- if email:
- emails_to_process.append(email)
- # Add to our global set of processed users
- self.all_processed_users.add(email)
-
- if emails_to_process:
+ if page.emails:
# Bulk update: opt out these users from all marketing
- self._bulk_update_all_preferences(emails_to_process, global_opt_out_ids)
+ self._bulk_update_all_preferences(list(page.emails), [ALL_MESSAGE_PREFERENCE_CATEGORY_ID])
- batch_processed = len(emails_to_process)
+ batch_processed = len(page.emails)
total_processed += batch_processed
self.progress["globally_unsubscribed_count"] += batch_processed
self.progress["customers_processed"] += batch_processed
- self.progress["preferences_updated"] += batch_processed * len(global_opt_out_ids)
-
- # Check for next page
- next_cursor = response.get("next")
- if not next_cursor or next_cursor == "":
- break
- start = next_cursor
+ self.progress["preferences_updated"] += batch_processed
except Exception as e:
self.progress["errors"].append(
@@ -426,6 +454,28 @@ class CustomerIOImportService:
self.progress["details"] = f"Processed {total_processed} globally unsubscribed users"
+ def fetch_global_unsubscribe_pages(self, start: str | None) -> Iterator[GlobalUnsubscribePage]:
+ """Page through Customer.io's globally unsubscribed customers, starting at the `start` cursor.
+
+ The caller writes each page, so it decides what a failed write does to the rest of the import.
+ """
+ assert self.client is not None
+ while True:
+ response = self.client.get_globally_unsubscribed_customers(limit=1000, start=start)
+ identifiers = response.get("identifiers", [])
+ if not identifiers:
+ return
+
+ next_cursor = response.get("next") or None
+ yield GlobalUnsubscribePage(
+ customers=len(identifiers),
+ emails=tuple(email for customer in identifiers if (email := customer.get("email"))),
+ next_start=next_cursor,
+ )
+ if not next_cursor:
+ return
+ start = next_cursor
+
def _import_suppressions(self) -> None:
"""Add Customer.io's bounced, invalid and spam-reported addresses to the suppression list.
@@ -435,28 +485,50 @@ class CustomerIOImportService:
if not self.client:
return
- # bulk_create skips the save() that moves a child environment's rows to its parent project.
- team_id = resolve_effective_team_id(self.team_id)
- for suppression_type, (source, reason) in CUSTOMERIO_SUPPRESSION_TYPES.items():
- offset = 0
+ for suppression_type in CUSTOMERIO_SUPPRESSION_TYPES:
+ pages = self.fetch_suppression_pages(suppression_type, offset=0)
while True:
try:
- page = self.client.get_suppressions(suppression_type, limit=SUPPRESSION_PAGE_SIZE, offset=offset)
+ page = next(pages, None)
except Exception as e:
self.progress["errors"].append(f"Error importing Customer.io {suppression_type}: {str(e)[:200]}")
break
+ if page is None:
+ break
+ self.progress["suppressions_imported"] += self.save_suppression_page(page)
+
+ def fetch_suppression_pages(self, suppression_type: str, offset: int) -> Iterator[SuppressionPage]:
+ """Page through one Customer.io suppression list, starting at `offset`.
- imported = {
+ The caller saves each page with `save_suppression_page`, so it decides what a failed request does to the
+ rest of the import.
+ """
+ assert self.client is not None
+ while True:
+ entries = self.client.get_suppressions(suppression_type, limit=SUPPRESSION_PAGE_SIZE, offset=offset)
+ offset += len(entries)
+ yield SuppressionPage(
+ suppression_type=suppression_type,
+ entries=len(entries),
+ addresses={
email.strip().lower(): _imported_suppression(entry)
- for entry in page
+ for entry in entries
if isinstance(entry, dict) and isinstance(email := entry.get("email"), str) and "@" in email
- }
- if imported:
- self.progress["suppressions_imported"] += self._add_suppressions(team_id, imported, source, reason)
- if len(page) < SUPPRESSION_PAGE_SIZE:
- break
- offset += len(page)
- time.sleep(SUPPRESSION_PAGE_DELAY_SECONDS)
+ },
+ next_offset=offset,
+ is_last=len(entries) < SUPPRESSION_PAGE_SIZE,
+ )
+ if len(entries) < SUPPRESSION_PAGE_SIZE:
+ return
+ time.sleep(SUPPRESSION_PAGE_DELAY_SECONDS)
+
+ def save_suppression_page(self, page: SuppressionPage) -> int:
+ """Suppress the page's addresses and return how many new rows it added."""
+ if not page.addresses:
+ return 0
+ source, reason = CUSTOMERIO_SUPPRESSION_TYPES[page.suppression_type]
+ # bulk_create skips the save() that moves a child environment's rows to its parent project.
+ return self._add_suppressions(resolve_effective_team_id(self.team_id), page.addresses, source, reason)
def _add_suppressions(
self, team_id: int, imported: dict[str, _ImportedSuppression], source: SuppressionSource, reason: str
@@ -506,50 +578,63 @@ class CustomerIOImportService:
def _bulk_update_all_preferences(self, emails: list[str], category_ids: list[str]) -> None:
"""Bulk update preferences to opt out users from ALL categories"""
try:
- with transaction.atomic():
- # Fetch existing preferences
- existing_prefs = {
- pref.identifier: pref
- for pref in MessageRecipientPreference.objects.filter(team_id=self.team_id, identifier__in=emails)
- }
+ self.save_opt_outs(emails, category_ids)
+ except Exception as e:
+ logger.exception(f"Error bulk updating all preferences: {e}")
+ # Don't fall back to individual updates - it would make performance worse
+ # Instead, record the batch failure and continue
+ self.progress["errors"].append(f"Failed to update batch of {len(emails)} users: {str(e)[:200]}")
- to_create = []
- to_update = []
+ def save_opt_outs(self, emails: list[str], category_ids: list[str]) -> int:
+ """Opt the recipients out of the categories, keeping their other preferences.
- for email in emails:
- if email in existing_prefs:
- # Update existing preference
- pref = existing_prefs[email]
- for category_id in category_ids:
- pref.preferences[str(category_id)] = PreferenceStatus.OPTED_OUT.value
- to_update.append(pref)
- else:
- # Create new preference with all categories opted out
- preferences_dict = {
- str(category_id): PreferenceStatus.OPTED_OUT.value for category_id in category_ids
- }
- to_create.append(
- MessageRecipientPreference(
- team_id=self.team_id,
- identifier=email,
- preferences=preferences_dict,
- )
+ Returns how many recipients were not already opted out of every category.
+ """
+ changed = 0
+ with transaction.atomic():
+ # Fetch existing preferences
+ existing_prefs = {
+ pref.identifier: pref
+ for pref in MessageRecipientPreference.objects.filter(team_id=self.team_id, identifier__in=emails)
+ }
+
+ to_create = []
+ to_update = []
+
+ for email in emails:
+ if email in existing_prefs:
+ # Update existing preference
+ pref = existing_prefs[email]
+ if any(
+ pref.preferences.get(str(category_id)) != PreferenceStatus.OPTED_OUT.value
+ for category_id in category_ids
+ ):
+ changed += 1
+ for category_id in category_ids:
+ pref.preferences[str(category_id)] = PreferenceStatus.OPTED_OUT.value
+ to_update.append(pref)
+ else:
+ # Create new preference with all categories opted out
+ preferences_dict = {
+ str(category_id): PreferenceStatus.OPTED_OUT.value for category_id in category_ids
+ }
+ changed += 1
+ to_create.append(
+ MessageRecipientPreference(
+ team_id=self.team_id,
+ identifier=email,
+ preferences=preferences_dict,
)
+ )
- # Bulk operations
- if to_create:
- MessageRecipientPreference.objects.bulk_create(to_create, batch_size=500)
+ # Bulk operations
+ if to_create:
+ MessageRecipientPreference.objects.bulk_create(to_create, batch_size=500)
- if to_update:
- MessageRecipientPreference.objects.bulk_update(
- to_update, ["preferences", "updated_at"], batch_size=500
- )
+ if to_update:
+ MessageRecipientPreference.objects.bulk_update(to_update, ["preferences", "updated_at"], batch_size=500)
- except Exception as e:
- logger.exception(f"Error bulk updating all preferences: {e}")
- # Don't fall back to individual updates - it would make performance worse
- # Instead, record the batch failure and continue
- self.progress["errors"].append(f"Failed to update batch of {len(emails)} users: {str(e)[:200]}")
+ return changed
def _load_topic_mapping(self) -> None:
"""Load topic mapping from existing categories"""
diff --git a/products/messaging/backend/services/test_customerio_import_service_unit.py b/products/messaging/backend/services/test_customerio_import_service_unit.py
index 8d0b47d92531..31eb8bf4bb12 100644
--- a/products/messaging/backend/services/test_customerio_import_service_unit.py
+++ b/products/messaging/backend/services/test_customerio_import_service_unit.py
@@ -2,8 +2,10 @@ from unittest.mock import MagicMock, patch
from django.test import SimpleTestCase
+import requests
from parameterized import parameterized
+from .customerio_client import CustomerIOAPIError, CustomerIOClient
from .customerio_import_service import CustomerIOImportService
@@ -122,3 +124,39 @@ class TestCustomerIOImportServiceUnit(SimpleTestCase):
assert result["status"] == "failed"
assert len(result["errors"]) == 1
assert "API key is required" in result["errors"][0]
+
+ @parameterized.expand(
+ [
+ ("us_key", {"us": 200}, CustomerIOClient.US_BASE_URL),
+ ("eu_key", {"us": 401, "eu": 200}, CustomerIOClient.EU_BASE_URL),
+ ("rejected_everywhere", {"us": 401, "eu": 403}, False),
+ ("us_down", {"us": 503, "eu": 401}, CustomerIOAPIError),
+ ("us_unreachable", {"us": None, "eu": 401}, CustomerIOAPIError),
+ ]
+ )
+ def test_connect_tells_a_rejected_key_from_an_outage(
+ self, _name: str, statuses: dict[str, int | None], expected: str | bool | type[Exception]
+ ) -> None:
+ def request(method: str, url: str, **kwargs: object) -> MagicMock:
+ region = "eu" if url.startswith(CustomerIOClient.EU_BASE_URL) else "us"
+ status_code = statuses[region]
+ if status_code is None:
+ raise requests.exceptions.ConnectTimeout()
+ response = MagicMock(status_code=status_code)
+ response.json.return_value = {"topics": []}
+ if status_code >= 400:
+ response.raise_for_status.side_effect = requests.exceptions.HTTPError(response=response)
+ return response
+
+ with patch(
+ "products.messaging.backend.services.customerio_client.requests.Session.request", side_effect=request
+ ):
+ if isinstance(expected, type):
+ with self.assertRaises(expected):
+ self.service.connect()
+ elif expected is False:
+ assert self.service.connect() is False
+ assert self.service.client is None
+ else:
+ assert self.service.connect() is True
+ assert self.service.client is not None and self.service.client.BASE_URL == expected