:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
refactor(messaging): split c.io import paging from saving (#124424)
Harley Alexander950a6a49daec
diff --git a/products/messaging/backend/services/customerio_client.py b/products/messaging/backend/services/customerio_client.pyindex 7d5237554f0f..3c008e44ae5b 100644--- a/products/messaging/backend/services/customerio_client.py+++ b/products/messaging/backend/services/customerio_client.py@@ -8,7 +8,10 @@ logger = logging.getLogger(__name__)class CustomerIOAPIError(Exception):- pass+ def __init__(self, message: str, status_code: int | None = None):+ super().__init__(message)+ # The HTTP status Customer.io returned, or None when the request got no response.+ self.status_code = status_codeclass CustomerIOClient:@@ -58,7 +61,7 @@ class CustomerIOClient:logger.exception(f"API request failed. {error_msg}. {status_info}")else:logger.exception(f"API request failed. {error_msg}")- raise CustomerIOAPIError(error_msg)+ raise CustomerIOAPIError(error_msg, status_code=e.response.status_code if e.response is not None else None)except requests.exceptions.RequestException as e:raise CustomerIOAPIError(f"Request failed: {e}")diff --git a/products/messaging/backend/services/customerio_config.py b/products/messaging/backend/services/customerio_config.pyindex 81753f46f8ae..0a82f72cfbe7 100644--- a/products/messaging/backend/services/customerio_config.py+++ b/products/messaging/backend/services/customerio_config.py@@ -23,7 +23,8 @@ class ConfigIncomplete(Exception):pass-def import_from_customerio(team_id: int, app_api_key: str | None, created_by_id: int | None) -> dict[str, Any]:+def resolve_app_api_key(team_id: int, app_api_key: str | None) -> str:+ """The App API key to import with: the one sent, or the stored one when none is sent."""integration = Integration.objects.filter(team_id=team_id, kind="customerio-app").first()api_key = app_api_key or (integration.sensitive_config.get("app_api_key") if integration else None)@@ -33,6 +34,12 @@ def import_from_customerio(team_id: int, app_api_key: str | None, created_by_id:if integration and app_api_key:raise ConfigConflict("Integration already exists. Delete it first to use a different key.")+ return api_key+++def save_app_integration(team_id: int, api_key: str, created_by_id: int | None) -> Integration:+ """Store the App API key on first use and link it to the sync config. Returns the stored integration."""+ integration = Integration.objects.filter(team_id=team_id, kind="customerio-app").first()if not integration:integration = Integration.objects.create(team_id=team_id,@@ -45,6 +52,18 @@ def import_from_customerio(team_id: int, app_api_key: str | None, created_by_id:config, _ = OptOutSyncConfig.objects.get_or_create(team_id=team_id)config.app_integration = integrationconfig.save(update_fields=["app_integration"])+ return integration+++def stored_app_api_key(team_id: int, integration_id: int) -> str | None:+ integration = Integration.objects.filter(team_id=team_id, kind="customerio-app", id=integration_id).first()+ return integration.sensitive_config.get("app_api_key") if integration else None+++def import_from_customerio(team_id: int, app_api_key: str | None, created_by_id: int | None) -> dict[str, Any]:+ api_key = resolve_app_api_key(team_id, app_api_key)+ save_app_integration(team_id, api_key, created_by_id)+ config = OptOutSyncConfig.objects.get(team_id=team_id)# Run import synchronouslyresult = CustomerIOImportService(team_id=team_id, api_key=api_key, created_by_id=created_by_id).import_api_data()diff --git a/products/messaging/backend/services/customerio_import_service.py b/products/messaging/backend/services/customerio_import_service.pyindex 3eb071e18c47..8c22f7bd0cc5 100644--- a/products/messaging/backend/services/customerio_import_service.py+++ b/products/messaging/backend/services/customerio_import_service.py@@ -3,6 +3,7 @@ import csvimport jsonimport timeimport logging+from collections.abc import Iteratorfrom datetime import UTC, datetimefrom typing import Any, Optional@@ -20,7 +21,7 @@ from products.messaging.backend.models.message_preferences import ()from products.messaging.backend.models.message_suppression import MessageSuppression, SuppressionSource-from .customerio_client import CustomerIOClient+from .customerio_client import CustomerIOAPIError, CustomerIOClientlogger = logging.getLogger(__name__)@@ -45,6 +46,26 @@ class _ImportedSuppression:diagnostic: str | None+@frozen+class SuppressionPage:+ suppression_type: str+ # How many entries Customer.io returned, including any without a usable address.+ entries: int+ addresses: dict[str, _ImportedSuppression]+ # The offset of the next page in the same suppression list.+ next_offset: int+ is_last: bool+++@frozen+class GlobalUnsubscribePage:+ # How many customers Customer.io returned, including any without an email address.+ customers: int+ emails: tuple[str, ...]+ # The cursor of the next page, or None on the last page.+ next_start: str | None++class CustomerIOImportService:def __init__(self, team_id: int, api_key: Optional[str], created_by_id: Optional[int]):self.team_id = team_id@@ -66,6 +87,30 @@ class CustomerIOImportService:"details": "",}+ def connect(self, timeout: int = 30) -> bool:+ """Find the region the App API key belongs to and keep a client for it.++ Returns False when every region rejects the key. Raises CustomerIOAPIError when a region fails in another+ way, for example a timeout or a server error, because the key can still be valid there.+ """+ if not self.api_key:+ return False+ unavailable: CustomerIOAPIError | None = None+ for region in ("us", "eu"):+ client = CustomerIOClient(self.api_key, region=region, timeout=timeout)+ try:+ client.get_subscription_topics()+ except CustomerIOAPIError as e:+ if e.status_code not in (401, 403):+ unavailable = e+ continue+ self.client = client+ return True+ self.client = None+ if unavailable is not None:+ raise unavailable+ return False+def import_api_data(self) -> dict[str, Any]:"""Import categories, globally unsubscribed users and suppressed addresses via API"""if not self.api_key:@@ -92,7 +137,7 @@ class CustomerIOImportService:topics = self.client.get_subscription_topics()self.progress["topics_found"] = len(topics)- self._import_categories(topics)+ self.import_categories(topics)# Step 3: Import addresses Customer.io's email provider suppressed. They go before the global# unsubscribes because those can run to hundreds of pages, past the request timeout, and each page@@ -325,7 +370,7 @@ class CustomerIOImportService:return preferences_count- def _import_categories(self, topics: list[dict[str, Any]]) -> None:+ def import_categories(self, topics: list[dict[str, Any]]) -> None:"""Import Customer.io topics as message categories"""for topic in topics:try:@@ -371,52 +416,35 @@ class CustomerIOImportService:if not self.client:return- start = Nonebatch_num = 0total_processed = 0- global_opt_out_ids = [ALL_MESSAGE_PREFERENCE_CATEGORY_ID]+ pages = self.fetch_global_unsubscribe_pages(start=None)while True:batch_num += 1self.progress["current_batch"] = batch_numtry:- # Fetch batch of globally unsubscribed customers- response = self.client.get_globally_unsubscribed_customers(limit=1000, start=start)- identifiers = response.get("identifiers", [])-- if not identifiers:+ page = next(pages, None)+ if page is None:break- self.progress["customers_in_current_batch"] = len(identifiers)+ self.progress["customers_in_current_batch"] = page.customersself.progress["details"] = (- f"Processing batch {batch_num} of globally unsubscribed users ({len(identifiers)} users)..."+ f"Processing batch {batch_num} of globally unsubscribed users ({page.customers} users)...")+ # Add to our global set of processed users+ self.all_processed_users.update(page.emails)- # Collect emails for this batch- emails_to_process = []- for customer_info in identifiers:- email = customer_info.get("email")- if email:- emails_to_process.append(email)- # Add to our global set of processed users- self.all_processed_users.add(email)-- if emails_to_process:+ if page.emails:# Bulk update: opt out these users from all marketing- self._bulk_update_all_preferences(emails_to_process, global_opt_out_ids)+ self._bulk_update_all_preferences(list(page.emails), [ALL_MESSAGE_PREFERENCE_CATEGORY_ID])- batch_processed = len(emails_to_process)+ batch_processed = len(page.emails)total_processed += batch_processedself.progress["globally_unsubscribed_count"] += batch_processedself.progress["customers_processed"] += batch_processed- self.progress["preferences_updated"] += batch_processed * len(global_opt_out_ids)-- # Check for next page- next_cursor = response.get("next")- if not next_cursor or next_cursor == "":- break- start = next_cursor+ self.progress["preferences_updated"] += batch_processedexcept Exception as e:self.progress["errors"].append(@@ -426,6 +454,28 @@ class CustomerIOImportService:self.progress["details"] = f"Processed {total_processed} globally unsubscribed users"+ def fetch_global_unsubscribe_pages(self, start: str | None) -> Iterator[GlobalUnsubscribePage]:+ """Page through Customer.io's globally unsubscribed customers, starting at the `start` cursor.++ The caller writes each page, so it decides what a failed write does to the rest of the import.+ """+ assert self.client is not None+ while True:+ response = self.client.get_globally_unsubscribed_customers(limit=1000, start=start)+ identifiers = response.get("identifiers", [])+ if not identifiers:+ return++ next_cursor = response.get("next") or None+ yield GlobalUnsubscribePage(+ customers=len(identifiers),+ emails=tuple(email for customer in identifiers if (email := customer.get("email"))),+ next_start=next_cursor,+ )+ if not next_cursor:+ return+ start = next_cursor+def _import_suppressions(self) -> None:"""Add Customer.io's bounced, invalid and spam-reported addresses to the suppression list.@@ -435,28 +485,50 @@ class CustomerIOImportService:if not self.client:return- # bulk_create skips the save() that moves a child environment's rows to its parent project.- team_id = resolve_effective_team_id(self.team_id)- for suppression_type, (source, reason) in CUSTOMERIO_SUPPRESSION_TYPES.items():- offset = 0+ for suppression_type in CUSTOMERIO_SUPPRESSION_TYPES:+ pages = self.fetch_suppression_pages(suppression_type, offset=0)while True:try:- page = self.client.get_suppressions(suppression_type, limit=SUPPRESSION_PAGE_SIZE, offset=offset)+ page = next(pages, None)except Exception as e:self.progress["errors"].append(f"Error importing Customer.io {suppression_type}: {str(e)[:200]}")break+ if page is None:+ break+ self.progress["suppressions_imported"] += self.save_suppression_page(page)++ def fetch_suppression_pages(self, suppression_type: str, offset: int) -> Iterator[SuppressionPage]:+ """Page through one Customer.io suppression list, starting at `offset`.- imported = {+ The caller saves each page with `save_suppression_page`, so it decides what a failed request does to the+ rest of the import.+ """+ assert self.client is not None+ while True:+ entries = self.client.get_suppressions(suppression_type, limit=SUPPRESSION_PAGE_SIZE, offset=offset)+ offset += len(entries)+ yield SuppressionPage(+ suppression_type=suppression_type,+ entries=len(entries),+ addresses={email.strip().lower(): _imported_suppression(entry)- for entry in page+ for entry in entriesif isinstance(entry, dict) and isinstance(email := entry.get("email"), str) and "@" in email- }- if imported:- self.progress["suppressions_imported"] += self._add_suppressions(team_id, imported, source, reason)- if len(page) < SUPPRESSION_PAGE_SIZE:- break- offset += len(page)- time.sleep(SUPPRESSION_PAGE_DELAY_SECONDS)+ },+ next_offset=offset,+ is_last=len(entries) < SUPPRESSION_PAGE_SIZE,+ )+ if len(entries) < SUPPRESSION_PAGE_SIZE:+ return+ time.sleep(SUPPRESSION_PAGE_DELAY_SECONDS)++ def save_suppression_page(self, page: SuppressionPage) -> int:+ """Suppress the page's addresses and return how many new rows it added."""+ if not page.addresses:+ return 0+ source, reason = CUSTOMERIO_SUPPRESSION_TYPES[page.suppression_type]+ # bulk_create skips the save() that moves a child environment's rows to its parent project.+ return self._add_suppressions(resolve_effective_team_id(self.team_id), page.addresses, source, reason)def _add_suppressions(self, team_id: int, imported: dict[str, _ImportedSuppression], source: SuppressionSource, reason: str@@ -506,50 +578,63 @@ class CustomerIOImportService:def _bulk_update_all_preferences(self, emails: list[str], category_ids: list[str]) -> None:"""Bulk update preferences to opt out users from ALL categories"""try:- with transaction.atomic():- # Fetch existing preferences- existing_prefs = {- pref.identifier: pref- for pref in MessageRecipientPreference.objects.filter(team_id=self.team_id, identifier__in=emails)- }+ self.save_opt_outs(emails, category_ids)+ except Exception as e:+ logger.exception(f"Error bulk updating all preferences: {e}")+ # Don't fall back to individual updates - it would make performance worse+ # Instead, record the batch failure and continue+ self.progress["errors"].append(f"Failed to update batch of {len(emails)} users: {str(e)[:200]}")- to_create = []- to_update = []+ def save_opt_outs(self, emails: list[str], category_ids: list[str]) -> int:+ """Opt the recipients out of the categories, keeping their other preferences.- for email in emails:- if email in existing_prefs:- # Update existing preference- pref = existing_prefs[email]- for category_id in category_ids:- pref.preferences[str(category_id)] = PreferenceStatus.OPTED_OUT.value- to_update.append(pref)- else:- # Create new preference with all categories opted out- preferences_dict = {- str(category_id): PreferenceStatus.OPTED_OUT.value for category_id in category_ids- }- to_create.append(- MessageRecipientPreference(- team_id=self.team_id,- identifier=email,- preferences=preferences_dict,- )+ Returns how many recipients were not already opted out of every category.+ """+ changed = 0+ with transaction.atomic():+ # Fetch existing preferences+ existing_prefs = {+ pref.identifier: pref+ for pref in MessageRecipientPreference.objects.filter(team_id=self.team_id, identifier__in=emails)+ }++ to_create = []+ to_update = []++ for email in emails:+ if email in existing_prefs:+ # Update existing preference+ pref = existing_prefs[email]+ if any(+ pref.preferences.get(str(category_id)) != PreferenceStatus.OPTED_OUT.value+ for category_id in category_ids+ ):+ changed += 1+ for category_id in category_ids:+ pref.preferences[str(category_id)] = PreferenceStatus.OPTED_OUT.value+ to_update.append(pref)+ else:+ # Create new preference with all categories opted out+ preferences_dict = {+ str(category_id): PreferenceStatus.OPTED_OUT.value for category_id in category_ids+ }+ changed += 1+ to_create.append(+ MessageRecipientPreference(+ team_id=self.team_id,+ identifier=email,+ preferences=preferences_dict,)+ )- # Bulk operations- if to_create:- MessageRecipientPreference.objects.bulk_create(to_create, batch_size=500)+ # Bulk operations+ if to_create:+ MessageRecipientPreference.objects.bulk_create(to_create, batch_size=500)- if to_update:- MessageRecipientPreference.objects.bulk_update(- to_update, ["preferences", "updated_at"], batch_size=500- )+ if to_update:+ MessageRecipientPreference.objects.bulk_update(to_update, ["preferences", "updated_at"], batch_size=500)- except Exception as e:- logger.exception(f"Error bulk updating all preferences: {e}")- # Don't fall back to individual updates - it would make performance worse- # Instead, record the batch failure and continue- self.progress["errors"].append(f"Failed to update batch of {len(emails)} users: {str(e)[:200]}")+ return changeddef _load_topic_mapping(self) -> None:"""Load topic mapping from existing categories"""diff --git a/products/messaging/backend/services/test_customerio_import_service_unit.py b/products/messaging/backend/services/test_customerio_import_service_unit.pyindex 8d0b47d92531..31eb8bf4bb12 100644--- a/products/messaging/backend/services/test_customerio_import_service_unit.py+++ b/products/messaging/backend/services/test_customerio_import_service_unit.py@@ -2,8 +2,10 @@ from unittest.mock import MagicMock, patchfrom django.test import SimpleTestCase+import requestsfrom parameterized import parameterized+from .customerio_client import CustomerIOAPIError, CustomerIOClientfrom .customerio_import_service import CustomerIOImportService@@ -122,3 +124,39 @@ class TestCustomerIOImportServiceUnit(SimpleTestCase):assert result["status"] == "failed"assert len(result["errors"]) == 1assert "API key is required" in result["errors"][0]++ @parameterized.expand(+ [+ ("us_key", {"us": 200}, CustomerIOClient.US_BASE_URL),+ ("eu_key", {"us": 401, "eu": 200}, CustomerIOClient.EU_BASE_URL),+ ("rejected_everywhere", {"us": 401, "eu": 403}, False),+ ("us_down", {"us": 503, "eu": 401}, CustomerIOAPIError),+ ("us_unreachable", {"us": None, "eu": 401}, CustomerIOAPIError),+ ]+ )+ def test_connect_tells_a_rejected_key_from_an_outage(+ self, _name: str, statuses: dict[str, int | None], expected: str | bool | type[Exception]+ ) -> None:+ def request(method: str, url: str, **kwargs: object) -> MagicMock:+ region = "eu" if url.startswith(CustomerIOClient.EU_BASE_URL) else "us"+ status_code = statuses[region]+ if status_code is None:+ raise requests.exceptions.ConnectTimeout()+ response = MagicMock(status_code=status_code)+ response.json.return_value = {"topics": []}+ if status_code >= 400:+ response.raise_for_status.side_effect = requests.exceptions.HTTPError(response=response)+ return response++ with patch(+ "products.messaging.backend.services.customerio_client.requests.Session.request", side_effect=request+ ):+ if isinstance(expected, type):+ with self.assertRaises(expected):+ self.service.connect()+ elif expected is False:+ assert self.service.connect() is False+ assert self.service.client is None+ else:+ assert self.service.connect() is True+ assert self.service.client is not None and self.service.client.BASE_URL == expected