:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(messaging): preserve customer.io global unsubscribes (#123479)
Harley Alexander63a811107eec
diff --git a/nodejs/src/cdp/services/messaging/recipient-preferences.service.test.ts b/nodejs/src/cdp/services/messaging/recipient-preferences.service.test.tsindex aee71553405d..90358f3f4c4f 100644--- a/nodejs/src/cdp/services/messaging/recipient-preferences.service.test.ts+++ b/nodejs/src/cdp/services/messaging/recipient-preferences.service.test.ts@@ -256,8 +256,12 @@ describe('RecipientPreferencesService', () => {expect(mockRecipientsManagerGet).not.toHaveBeenCalled()})- it('should return true if recipient is opted out of all marketing messaging', async () => {- const action = createEmailAction('[email protected]', '123e4567-e89b-12d3-a456-426614174000')+ it.each([+ ['an existing category', '123e4567-e89b-12d3-a456-426614174000'],+ ['no category', ''],+ ['a category created after the opt-out', '123e4567-e89b-12d3-a456-426614174001'],+ ])('should block globally opted-out recipients for %s', async (_name, categoryId) => {+ const action = createEmailAction('[email protected]', categoryId)const invocation = createFunctionStepInvocation(action)const recipient = createRecipient('[email protected]', {'123e4567-e89b-12d3-a456-426614174000': 'OPTED_IN', // Opted in for this category@@ -265,8 +269,6 @@ describe('RecipientPreferencesService', () => {})mockRecipientsManagerGet.mockResolvedValue(recipient)- mockRecipientsManagerGetPreference.mockReturnValue('OPTED_IN')- mockRecipientsManagerGetAllMarketingMessagingPreference.mockReturnValue('OPTED_OUT')const result = await service.shouldSkipAction(invocation, action)diff --git a/products/messaging/backend/presentation/views/message_categories.py b/products/messaging/backend/presentation/views/message_categories.pyindex 80b4e77b226d..e34c65fcbddf 100644--- a/products/messaging/backend/presentation/views/message_categories.py+++ b/products/messaging/backend/presentation/views/message_categories.py@@ -280,7 +280,7 @@ class MessageCategoryViewSet(def import_preferences_csv(self, request, **kwargs):"""Import customer preferences from CSV file- Expected CSV columns: id, email, cio_subscription_preferences+ Expected CSV columns: id, email, unsubscribed, cio_subscription_preferences"""csv_file = request.FILES.get("csv_file")diff --git a/products/messaging/backend/services/customerio_import_service.py b/products/messaging/backend/services/customerio_import_service.pyindex 14c9cb165564..6f219b8c9a29 100644--- a/products/messaging/backend/services/customerio_import_service.py+++ b/products/messaging/backend/services/customerio_import_service.py@@ -7,7 +7,11 @@ from typing import Any, Optionalfrom django.db import transactionfrom products.messaging.backend.models.message_category import MessageCategory, MessageCategoryType-from products.messaging.backend.models.message_preferences import MessageRecipientPreference, PreferenceStatus+from products.messaging.backend.models.message_preferences import (+ ALL_MESSAGE_PREFERENCE_CATEGORY_ID,+ MessageRecipientPreference,+ PreferenceStatus,+)from .customerio_client import CustomerIOClient@@ -59,11 +63,6 @@ class CustomerIOImportService:assert self.client is not None # We validated credentials abovetopics = self.client.get_subscription_topics()- if not topics:- self.progress["errors"].append("No subscription topics found in Customer.io")- self.progress["status"] = "completed"- return self.progress-self.progress["topics_found"] = len(topics)self._import_categories(topics)@@ -99,11 +98,6 @@ class CustomerIOImportService:# Load topic mapping from existing categoriesself._load_topic_mapping()- if not self.topic_mapping:- csv_progress["status"] = "failed"- csv_progress["details"] = "No categories found. Please run API import first."- return csv_progress-# Read CSV contentif hasattr(csv_file, "read"):content = csv_file.read()@@ -115,6 +109,11 @@ class CustomerIOImportService:# Parse CSVcsv_reader = csv.DictReader(io.StringIO(content))+ if not self.topic_mapping and "unsubscribed" not in (csv_reader.fieldnames or []):+ csv_progress["status"] = "failed"+ csv_progress["details"] = "No categories found. Please run API import first."+ return csv_progress+# Process in batchesbatch_size = 1000current_batch = []@@ -136,7 +135,7 @@ class CustomerIOImportService:csv_progress["users_with_optouts"] += 1elif result["status"] == "success":csv_progress["users_skipped"] += 1- elif result["status"] == "error":+ if result.get("error"):csv_progress["parse_errors"] += 1csv_progress["failed_imports"].append({"email": result.get("email", "unknown"), "error": result["error"]}@@ -170,28 +169,37 @@ class CustomerIOImportService:def _process_csv_row(self, row: dict) -> dict:"""Process a single CSV row and return the result"""- email = row.get("email", "").strip()- cio_id = row.get("id", "").strip() # Get Customer.io ID- preferences_json = row.get("cio_subscription_preferences", "").strip()+ email = (row.get("email") or "").strip()+ cio_id = (row.get("id") or "").strip() # Get Customer.io ID+ preferences_json = (row.get("cio_subscription_preferences") or "").strip()if not email:# Use Customer.io ID if email is missingidentifier = f"Customer.io ID: {cio_id}" if cio_id else "unknown"return {"status": "error", "email": identifier, "error": "Missing email"}+ globally_unsubscribed = (row.get("unsubscribed") or "").strip().lower() in {"true", "1"}+ opted_out_categories = [ALL_MESSAGE_PREFERENCE_CATEGORY_ID] if globally_unsubscribed else []+if not preferences_json:- return {"status": "success", "email": email, "opted_out_categories": []}+ return {"status": "success", "email": email, "opted_out_categories": opted_out_categories}+ error = ""try:# Parse JSON preferencesprefs = json.loads(preferences_json)topics = prefs.get("topics", {})# Collect opted-out categories (where value is false)- opted_out_categories = []-for topic_key, is_subscribed in topics.items():if is_subscribed is False: # Only process opt-outs+ if not self.topic_mapping and not globally_unsubscribed:+ return {+ "status": "error",+ "email": email,+ "error": "No categories found. Please run API import first.",+ }+# Extract topic ID (handle both "topic_1" and "1" formats)topic_id = topic_key.replace("topic_", "")@@ -204,12 +212,22 @@ class CustomerIOImportService:# Unknown topic, but don't fail the whole rowlogger.warning(f"Unknown topic ID '{topic_key}' for {email}")- return {"status": "success", "email": email, "opted_out_categories": opted_out_categories}-except json.JSONDecodeError as e:- return {"status": "error", "email": email, "error": f"Invalid JSON: {str(e)[:100]}"}+ error = f"Invalid JSON: {str(e)[:100]}"except Exception as e:- return {"status": "error", "email": email, "error": f"Processing error: {str(e)[:100]}"}+ error = f"Processing error: {str(e)[:100]}"++ if error:+ return {+ "status": "success" if globally_unsubscribed else "error",+ "email": email,+ "opted_out_categories": opted_out_categories,+ "error": f"Global opt-out imported; topic preferences not imported: {error}. Fix the topic preferences and upload the CSV again."+ if globally_unsubscribed+ else error,+ }++ return {"status": "success", "email": email, "opted_out_categories": opted_out_categories}def _save_csv_batch(self, batch: list[tuple[str, list[str]]]) -> int:"""Save a batch of CSV preferences to database"""@@ -316,14 +334,14 @@ class CustomerIOImportService:self.progress["errors"].append(f"Failed to import topic {topic_id}: {str(e)}")def _process_globally_unsubscribed_users(self) -> None:- """Process users who are globally unsubscribed (opted out of ALL categories)"""- if not self.topic_mapping or not self.client:+ """Record Customer.io global unsubscribes as an opt-out of all marketing, keeping each topic choice"""+ if not self.client:returnstart = Nonebatch_num = 0total_processed = 0- all_category_ids = list(set(self.topic_mapping.values()))+ global_opt_out_ids = [ALL_MESSAGE_PREFERENCE_CATEGORY_ID]while True:batch_num += 1@@ -352,15 +370,14 @@ class CustomerIOImportService:self.all_processed_users.add(email)if emails_to_process:- # Bulk update: opt out these users from ALL categories- self._bulk_update_all_preferences(emails_to_process, all_category_ids)+ # Bulk update: opt out these users from all marketing+ self._bulk_update_all_preferences(emails_to_process, global_opt_out_ids)batch_processed = len(emails_to_process)total_processed += batch_processedself.progress["globally_unsubscribed_count"] += batch_processedself.progress["customers_processed"] += batch_processed- # Each user gets opted out of all categories- self.progress["preferences_updated"] += batch_processed * len(all_category_ids)+ self.progress["preferences_updated"] += batch_processed * len(global_opt_out_ids)# Check for next pagenext_cursor = response.get("next")diff --git a/products/messaging/backend/services/test_customerio_import_service.py b/products/messaging/backend/services/test_customerio_import_service.pyindex b106e3b699a4..0aa4f01d489e 100644--- a/products/messaging/backend/services/test_customerio_import_service.py+++ b/products/messaging/backend/services/test_customerio_import_service.py@@ -3,8 +3,14 @@ from io import StringIOfrom posthog.test.base import BaseTestfrom unittest.mock import MagicMock, patch+from parameterized import parameterized+from products.messaging.backend.models.message_category import MessageCategory, MessageCategoryType-from products.messaging.backend.models.message_preferences import MessageRecipientPreference, PreferenceStatus+from products.messaging.backend.models.message_preferences import (+ ALL_MESSAGE_PREFERENCE_CATEGORY_ID,+ MessageRecipientPreference,+ PreferenceStatus,+)from .customerio_import_service import CustomerIOImportService@@ -15,6 +21,80 @@ class TestCustomerIOImportService(BaseTest):self.api_key = "test-api-key"self.service = CustomerIOImportService(self.team.id, self.api_key, self.user.id)+ @parameterized.expand(+ [+ ("true", "", True, True),+ (" TRUE ", "invalid json", True, True),+ ("false", "", False, True),+ ("", "", False, True),+ ("true", "", True, False),+ ("1", "", True, True),+ ("1", "", True, False),+ ("0", "", False, True),+ ]+ )+ def test_csv_import_global_unsubscribe(+ self, unsubscribed: str, topics: str, globally_opted_out: bool, has_categories: bool+ ) -> None:+ if has_categories:+ MessageCategory.objects.create(team=self.team, key="customerio_topic_1", name="Newsletter")+ recipient = MessageRecipientPreference.objects.create(+ team=self.team,+ identifier="[email protected]",+ preferences={"existing_category": PreferenceStatus.OPTED_IN.value},+ )+ csv_content = (+ f"email,unsubscribed,cio_subscription_preferences\[email protected],{unsubscribed},{topics}\n"+ )++ result = self.service.process_preferences_csv(StringIO(csv_content))++ assert result["status"] == "completed"+ assert result["parse_errors"] == (1 if topics == "invalid json" else 0)+ if topics == "invalid json":+ assert result["failed_imports"][0]["email"] == "[email protected]"+ assert "Global opt-out imported" in result["failed_imports"][0]["error"]+ recipient.refresh_from_db()+ assert recipient.get_preference("existing_category") == PreferenceStatus.OPTED_IN+ assert recipient.get_preference(ALL_MESSAGE_PREFERENCE_CATEGORY_ID) == (+ PreferenceStatus.OPTED_OUT if globally_opted_out else PreferenceStatus.NO_PREFERENCE+ )++ def test_csv_short_row_preserves_pending_global_optouts(self) -> None:+ csv_content = (+ "email,cio_subscription_preferences,unsubscribed\[email protected],,true\[email protected],\n"+ )++ result = self.service.process_preferences_csv(StringIO(csv_content))++ assert result["status"] == "completed"+ assert result["users_with_optouts"] == 1+ preference = MessageRecipientPreference.objects.get(team=self.team, identifier="[email protected]")+ assert preference.get_preference(ALL_MESSAGE_PREFERENCE_CATEGORY_ID) == PreferenceStatus.OPTED_OUT++ def test_csv_reports_topic_optouts_without_categories(self) -> None:+ csv_content = (+ 'email,unsubscribed,cio_subscription_preferences\[email protected],false,{"topics":{"topic_1":false}}\n'+ )++ result = self.service.process_preferences_csv(StringIO(csv_content))++ assert result["parse_errors"] == 1+ assert result["users_skipped"] == 0+ assert result["failed_imports"][0]["email"] == "[email protected]"+ assert "No categories found" in result["failed_imports"][0]["error"]++ def test_csv_global_unsubscribe_preserves_topic_choices(self) -> None:+ category = MessageCategory.objects.create(team=self.team, key="customerio_topic_1", name="Newsletter")+ csv_content = 'email,unsubscribed,cio_subscription_preferences\[email protected],true,{"topics":{"topic_1":false}}\n'++ result = self.service.process_preferences_csv(StringIO(csv_content))++ assert result["parse_errors"] == 0+ preference = MessageRecipientPreference.objects.get(team=self.team, identifier="[email protected]")+ assert preference.get_preference(ALL_MESSAGE_PREFERENCE_CATEGORY_ID) == PreferenceStatus.OPTED_OUT+ assert preference.get_preference(str(category.id)) == PreferenceStatus.OPTED_OUT+def test_process_preferences_csv_complete_flow(self):"""Test complete CSV processing flow with batching"""# Create categories first@@ -99,6 +179,13 @@ [email protected],cio_6,"{""topics"": {""topic_1"": true, ""topic_2"": true}}"},]+ newsletter = MessageCategory.objects.create(team=self.team, key="customerio_topic_1", name="Newsletter")+ MessageRecipientPreference.objects.create(+ team=self.team,+ identifier="[email protected]",+ preferences={str(newsletter.id): PreferenceStatus.OPTED_IN.value},+ )+# Mock globally unsubscribed customersmock_client.get_globally_unsubscribed_customers.side_effect = [{@@ -124,7 +211,7 @@ [email protected],cio_6,"{""topics"": {""topic_1"": true, ""topic_2"": true}}"assert result["topics_found"] == 2assert result["categories_created"] == 2assert result["globally_unsubscribed_count"] == 3- assert result["preferences_updated"] == 6 # 3 users * 2 categories+ assert result["preferences_updated"] == 3# Check categories were createdcat1 = MessageCategory.objects.get(team=self.team, key="customerio_topic_1")@@ -137,8 +224,10 @@ [email protected],cio_6,"{""topics"": {""topic_1"": true, ""topic_2"": true}}"# Check globally unsubscribed preferencespref1 = MessageRecipientPreference.objects.get(team_id=self.team.id, identifier="[email protected]")- assert pref1.preferences[str(cat1.id)] == PreferenceStatus.OPTED_OUT.value- assert pref1.preferences[str(cat2.id)] == PreferenceStatus.OPTED_OUT.value+ assert cat1.id == newsletter.id+ assert pref1.get_preference(str(cat1.id)) == PreferenceStatus.OPTED_IN+ assert pref1.get_preference(str(cat2.id)) == PreferenceStatus.NO_PREFERENCE+ assert pref1.get_preference(ALL_MESSAGE_PREFERENCE_CATEGORY_ID) == PreferenceStatus.OPTED_OUTdef test_save_csv_batch_with_existing_preferences(self):"""Test batch saving when some users already have preferences"""@@ -180,6 +269,22 @@ [email protected],cio_6,"{""topics"": {""topic_1"": true, ""topic_2"": true}}"new_pref = MessageRecipientPreference.objects.get(team_id=self.team.id, identifier="[email protected]")assert new_pref.preferences[str(cat1.id)] == PreferenceStatus.OPTED_OUT.value+ @patch("products.messaging.backend.services.customerio_import_service.CustomerIOClient")+ def test_api_import_global_unsubscribe_without_topics(self, mock_client_class) -> None:+ mock_client = mock_client_class.return_value+ mock_client.validate_credentials.return_value = True+ mock_client.get_subscription_topics.return_value = []+ mock_client.get_globally_unsubscribed_customers.return_value = {+ "identifiers": [{"email": "[email protected]"}],+ }++ result = self.service.import_api_data()++ assert result["status"] == "completed"+ assert result["globally_unsubscribed_count"] == 1+ preference = MessageRecipientPreference.objects.get(team=self.team, identifier="[email protected]")+ assert preference.get_preference(ALL_MESSAGE_PREFERENCE_CATEGORY_ID) == PreferenceStatus.OPTED_OUT+def test_process_csv_without_categories(self):"""Test CSV processing when no categories exist"""csv_content = """email,id,cio_subscription_preferencesdiff --git a/products/messaging/frontend/generated/api.ts b/products/messaging/frontend/generated/api.tsindex 37b659d0838b..3ebd3092e7f4 100644--- a/products/messaging/frontend/generated/api.ts+++ b/products/messaging/frontend/generated/api.ts@@ -196,7 +196,7 @@ export const getMessagingCategoriesImportPreferencesCsvCreateUrl = (projectId: s/*** Import customer preferences from CSV file- * Expected CSV columns: id, email, cio_subscription_preferences+ * Expected CSV columns: id, email, unsubscribed, cio_subscription_preferences*/export const messagingCategoriesImportPreferencesCsvCreate = async (projectId: string,diff --git a/products/messaging/frontend/generated/api.zod.ts b/products/messaging/frontend/generated/api.zod.tsindex a465ae49a882..8bebf94f5fae 100644--- a/products/messaging/frontend/generated/api.zod.ts+++ b/products/messaging/frontend/generated/api.zod.ts@@ -162,7 +162,7 @@ export const MessagingCategoriesImportFromCustomerioCreateBody = /* @__PURE__ *//*** Import customer preferences from CSV file- * Expected CSV columns: id, email, cio_subscription_preferences+ * Expected CSV columns: id, email, unsubscribed, cio_subscription_preferences*/export const messagingCategoriesImportPreferencesCsvCreateBodyKeyMax = 64