Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(messaging): preserve customer.io global unsubscribes (#123479)

Harley Alexander63a811107eec
diff --git a/nodejs/src/cdp/services/messaging/recipient-preferences.service.test.ts b/nodejs/src/cdp/services/messaging/recipient-preferences.service.test.ts
index aee71553405d..90358f3f4c4f 100644
--- a/nodejs/src/cdp/services/messaging/recipient-preferences.service.test.ts
+++ b/nodejs/src/cdp/services/messaging/recipient-preferences.service.test.ts
@@ -256,8 +256,12 @@ describe('RecipientPreferencesService', () => {
expect(mockRecipientsManagerGet).not.toHaveBeenCalled()
})
- it('should return true if recipient is opted out of all marketing messaging', async () => {
- const action = createEmailAction('[email protected]', '123e4567-e89b-12d3-a456-426614174000')
+ it.each([
+ ['an existing category', '123e4567-e89b-12d3-a456-426614174000'],
+ ['no category', ''],
+ ['a category created after the opt-out', '123e4567-e89b-12d3-a456-426614174001'],
+ ])('should block globally opted-out recipients for %s', async (_name, categoryId) => {
+ const action = createEmailAction('[email protected]', categoryId)
const invocation = createFunctionStepInvocation(action)
const recipient = createRecipient('[email protected]', {
'123e4567-e89b-12d3-a456-426614174000': 'OPTED_IN', // Opted in for this category
@@ -265,8 +269,6 @@ describe('RecipientPreferencesService', () => {
})
mockRecipientsManagerGet.mockResolvedValue(recipient)
- mockRecipientsManagerGetPreference.mockReturnValue('OPTED_IN')
- mockRecipientsManagerGetAllMarketingMessagingPreference.mockReturnValue('OPTED_OUT')
const result = await service.shouldSkipAction(invocation, action)
diff --git a/products/messaging/backend/presentation/views/message_categories.py b/products/messaging/backend/presentation/views/message_categories.py
index 80b4e77b226d..e34c65fcbddf 100644
--- a/products/messaging/backend/presentation/views/message_categories.py
+++ b/products/messaging/backend/presentation/views/message_categories.py
@@ -280,7 +280,7 @@ class MessageCategoryViewSet(
def import_preferences_csv(self, request, **kwargs):
"""
Import customer preferences from CSV file
- Expected CSV columns: id, email, cio_subscription_preferences
+ Expected CSV columns: id, email, unsubscribed, cio_subscription_preferences
"""
csv_file = request.FILES.get("csv_file")
diff --git a/products/messaging/backend/services/customerio_import_service.py b/products/messaging/backend/services/customerio_import_service.py
index 14c9cb165564..6f219b8c9a29 100644
--- a/products/messaging/backend/services/customerio_import_service.py
+++ b/products/messaging/backend/services/customerio_import_service.py
@@ -7,7 +7,11 @@ from typing import Any, Optional
from django.db import transaction
from products.messaging.backend.models.message_category import MessageCategory, MessageCategoryType
-from products.messaging.backend.models.message_preferences import MessageRecipientPreference, PreferenceStatus
+from products.messaging.backend.models.message_preferences import (
+ ALL_MESSAGE_PREFERENCE_CATEGORY_ID,
+ MessageRecipientPreference,
+ PreferenceStatus,
+)
from .customerio_client import CustomerIOClient
@@ -59,11 +63,6 @@ class CustomerIOImportService:
assert self.client is not None # We validated credentials above
topics = self.client.get_subscription_topics()
- if not topics:
- self.progress["errors"].append("No subscription topics found in Customer.io")
- self.progress["status"] = "completed"
- return self.progress
-
self.progress["topics_found"] = len(topics)
self._import_categories(topics)
@@ -99,11 +98,6 @@ class CustomerIOImportService:
# Load topic mapping from existing categories
self._load_topic_mapping()
- if not self.topic_mapping:
- csv_progress["status"] = "failed"
- csv_progress["details"] = "No categories found. Please run API import first."
- return csv_progress
-
# Read CSV content
if hasattr(csv_file, "read"):
content = csv_file.read()
@@ -115,6 +109,11 @@ class CustomerIOImportService:
# Parse CSV
csv_reader = csv.DictReader(io.StringIO(content))
+ if not self.topic_mapping and "unsubscribed" not in (csv_reader.fieldnames or []):
+ csv_progress["status"] = "failed"
+ csv_progress["details"] = "No categories found. Please run API import first."
+ return csv_progress
+
# Process in batches
batch_size = 1000
current_batch = []
@@ -136,7 +135,7 @@ class CustomerIOImportService:
csv_progress["users_with_optouts"] += 1
elif result["status"] == "success":
csv_progress["users_skipped"] += 1
- elif result["status"] == "error":
+ if result.get("error"):
csv_progress["parse_errors"] += 1
csv_progress["failed_imports"].append(
{"email": result.get("email", "unknown"), "error": result["error"]}
@@ -170,28 +169,37 @@ class CustomerIOImportService:
def _process_csv_row(self, row: dict) -> dict:
"""Process a single CSV row and return the result"""
- email = row.get("email", "").strip()
- cio_id = row.get("id", "").strip() # Get Customer.io ID
- preferences_json = row.get("cio_subscription_preferences", "").strip()
+ email = (row.get("email") or "").strip()
+ cio_id = (row.get("id") or "").strip() # Get Customer.io ID
+ preferences_json = (row.get("cio_subscription_preferences") or "").strip()
if not email:
# Use Customer.io ID if email is missing
identifier = f"Customer.io ID: {cio_id}" if cio_id else "unknown"
return {"status": "error", "email": identifier, "error": "Missing email"}
+ globally_unsubscribed = (row.get("unsubscribed") or "").strip().lower() in {"true", "1"}
+ opted_out_categories = [ALL_MESSAGE_PREFERENCE_CATEGORY_ID] if globally_unsubscribed else []
+
if not preferences_json:
- return {"status": "success", "email": email, "opted_out_categories": []}
+ return {"status": "success", "email": email, "opted_out_categories": opted_out_categories}
+ error = ""
try:
# Parse JSON preferences
prefs = json.loads(preferences_json)
topics = prefs.get("topics", {})
# Collect opted-out categories (where value is false)
- opted_out_categories = []
-
for topic_key, is_subscribed in topics.items():
if is_subscribed is False: # Only process opt-outs
+ if not self.topic_mapping and not globally_unsubscribed:
+ return {
+ "status": "error",
+ "email": email,
+ "error": "No categories found. Please run API import first.",
+ }
+
# Extract topic ID (handle both "topic_1" and "1" formats)
topic_id = topic_key.replace("topic_", "")
@@ -204,12 +212,22 @@ class CustomerIOImportService:
# Unknown topic, but don't fail the whole row
logger.warning(f"Unknown topic ID '{topic_key}' for {email}")
- return {"status": "success", "email": email, "opted_out_categories": opted_out_categories}
-
except json.JSONDecodeError as e:
- return {"status": "error", "email": email, "error": f"Invalid JSON: {str(e)[:100]}"}
+ error = f"Invalid JSON: {str(e)[:100]}"
except Exception as e:
- return {"status": "error", "email": email, "error": f"Processing error: {str(e)[:100]}"}
+ error = f"Processing error: {str(e)[:100]}"
+
+ if error:
+ return {
+ "status": "success" if globally_unsubscribed else "error",
+ "email": email,
+ "opted_out_categories": opted_out_categories,
+ "error": f"Global opt-out imported; topic preferences not imported: {error}. Fix the topic preferences and upload the CSV again."
+ if globally_unsubscribed
+ else error,
+ }
+
+ return {"status": "success", "email": email, "opted_out_categories": opted_out_categories}
def _save_csv_batch(self, batch: list[tuple[str, list[str]]]) -> int:
"""Save a batch of CSV preferences to database"""
@@ -316,14 +334,14 @@ class CustomerIOImportService:
self.progress["errors"].append(f"Failed to import topic {topic_id}: {str(e)}")
def _process_globally_unsubscribed_users(self) -> None:
- """Process users who are globally unsubscribed (opted out of ALL categories)"""
- if not self.topic_mapping or not self.client:
+ """Record Customer.io global unsubscribes as an opt-out of all marketing, keeping each topic choice"""
+ if not self.client:
return
start = None
batch_num = 0
total_processed = 0
- all_category_ids = list(set(self.topic_mapping.values()))
+ global_opt_out_ids = [ALL_MESSAGE_PREFERENCE_CATEGORY_ID]
while True:
batch_num += 1
@@ -352,15 +370,14 @@ class CustomerIOImportService:
self.all_processed_users.add(email)
if emails_to_process:
- # Bulk update: opt out these users from ALL categories
- self._bulk_update_all_preferences(emails_to_process, all_category_ids)
+ # Bulk update: opt out these users from all marketing
+ self._bulk_update_all_preferences(emails_to_process, global_opt_out_ids)
batch_processed = len(emails_to_process)
total_processed += batch_processed
self.progress["globally_unsubscribed_count"] += batch_processed
self.progress["customers_processed"] += batch_processed
- # Each user gets opted out of all categories
- self.progress["preferences_updated"] += batch_processed * len(all_category_ids)
+ self.progress["preferences_updated"] += batch_processed * len(global_opt_out_ids)
# Check for next page
next_cursor = response.get("next")
diff --git a/products/messaging/backend/services/test_customerio_import_service.py b/products/messaging/backend/services/test_customerio_import_service.py
index b106e3b699a4..0aa4f01d489e 100644
--- a/products/messaging/backend/services/test_customerio_import_service.py
+++ b/products/messaging/backend/services/test_customerio_import_service.py
@@ -3,8 +3,14 @@ from io import StringIO
from posthog.test.base import BaseTest
from unittest.mock import MagicMock, patch
+from parameterized import parameterized
+
from products.messaging.backend.models.message_category import MessageCategory, MessageCategoryType
-from products.messaging.backend.models.message_preferences import MessageRecipientPreference, PreferenceStatus
+from products.messaging.backend.models.message_preferences import (
+ ALL_MESSAGE_PREFERENCE_CATEGORY_ID,
+ MessageRecipientPreference,
+ PreferenceStatus,
+)
from .customerio_import_service import CustomerIOImportService
@@ -15,6 +21,80 @@ class TestCustomerIOImportService(BaseTest):
self.api_key = "test-api-key"
self.service = CustomerIOImportService(self.team.id, self.api_key, self.user.id)
+ @parameterized.expand(
+ [
+ ("true", "", True, True),
+ (" TRUE ", "invalid json", True, True),
+ ("false", "", False, True),
+ ("", "", False, True),
+ ("true", "", True, False),
+ ("1", "", True, True),
+ ("1", "", True, False),
+ ("0", "", False, True),
+ ]
+ )
+ def test_csv_import_global_unsubscribe(
+ self, unsubscribed: str, topics: str, globally_opted_out: bool, has_categories: bool
+ ) -> None:
+ if has_categories:
+ MessageCategory.objects.create(team=self.team, key="customerio_topic_1", name="Newsletter")
+ recipient = MessageRecipientPreference.objects.create(
+ team=self.team,
+ identifier="[email protected]",
+ preferences={"existing_category": PreferenceStatus.OPTED_IN.value},
+ )
+ csv_content = (
+ f"email,unsubscribed,cio_subscription_preferences\[email protected],{unsubscribed},{topics}\n"
+ )
+
+ result = self.service.process_preferences_csv(StringIO(csv_content))
+
+ assert result["status"] == "completed"
+ assert result["parse_errors"] == (1 if topics == "invalid json" else 0)
+ if topics == "invalid json":
+ assert result["failed_imports"][0]["email"] == "[email protected]"
+ assert "Global opt-out imported" in result["failed_imports"][0]["error"]
+ recipient.refresh_from_db()
+ assert recipient.get_preference("existing_category") == PreferenceStatus.OPTED_IN
+ assert recipient.get_preference(ALL_MESSAGE_PREFERENCE_CATEGORY_ID) == (
+ PreferenceStatus.OPTED_OUT if globally_opted_out else PreferenceStatus.NO_PREFERENCE
+ )
+
+ def test_csv_short_row_preserves_pending_global_optouts(self) -> None:
+ csv_content = (
+ "email,cio_subscription_preferences,unsubscribed\[email protected],,true\[email protected],\n"
+ )
+
+ result = self.service.process_preferences_csv(StringIO(csv_content))
+
+ assert result["status"] == "completed"
+ assert result["users_with_optouts"] == 1
+ preference = MessageRecipientPreference.objects.get(team=self.team, identifier="[email protected]")
+ assert preference.get_preference(ALL_MESSAGE_PREFERENCE_CATEGORY_ID) == PreferenceStatus.OPTED_OUT
+
+ def test_csv_reports_topic_optouts_without_categories(self) -> None:
+ csv_content = (
+ 'email,unsubscribed,cio_subscription_preferences\[email protected],false,{"topics":{"topic_1":false}}\n'
+ )
+
+ result = self.service.process_preferences_csv(StringIO(csv_content))
+
+ assert result["parse_errors"] == 1
+ assert result["users_skipped"] == 0
+ assert result["failed_imports"][0]["email"] == "[email protected]"
+ assert "No categories found" in result["failed_imports"][0]["error"]
+
+ def test_csv_global_unsubscribe_preserves_topic_choices(self) -> None:
+ category = MessageCategory.objects.create(team=self.team, key="customerio_topic_1", name="Newsletter")
+ csv_content = 'email,unsubscribed,cio_subscription_preferences\[email protected],true,{"topics":{"topic_1":false}}\n'
+
+ result = self.service.process_preferences_csv(StringIO(csv_content))
+
+ assert result["parse_errors"] == 0
+ preference = MessageRecipientPreference.objects.get(team=self.team, identifier="[email protected]")
+ assert preference.get_preference(ALL_MESSAGE_PREFERENCE_CATEGORY_ID) == PreferenceStatus.OPTED_OUT
+ assert preference.get_preference(str(category.id)) == PreferenceStatus.OPTED_OUT
+
def test_process_preferences_csv_complete_flow(self):
"""Test complete CSV processing flow with batching"""
# Create categories first
@@ -99,6 +179,13 @@ [email protected],cio_6,"{""topics"": {""topic_1"": true, ""topic_2"": true}}"
},
]
+ newsletter = MessageCategory.objects.create(team=self.team, key="customerio_topic_1", name="Newsletter")
+ MessageRecipientPreference.objects.create(
+ team=self.team,
+ identifier="[email protected]",
+ preferences={str(newsletter.id): PreferenceStatus.OPTED_IN.value},
+ )
+
# Mock globally unsubscribed customers
mock_client.get_globally_unsubscribed_customers.side_effect = [
{
@@ -124,7 +211,7 @@ [email protected],cio_6,"{""topics"": {""topic_1"": true, ""topic_2"": true}}"
assert result["topics_found"] == 2
assert result["categories_created"] == 2
assert result["globally_unsubscribed_count"] == 3
- assert result["preferences_updated"] == 6 # 3 users * 2 categories
+ assert result["preferences_updated"] == 3
# Check categories were created
cat1 = MessageCategory.objects.get(team=self.team, key="customerio_topic_1")
@@ -137,8 +224,10 @@ [email protected],cio_6,"{""topics"": {""topic_1"": true, ""topic_2"": true}}"
# Check globally unsubscribed preferences
pref1 = MessageRecipientPreference.objects.get(team_id=self.team.id, identifier="[email protected]")
- assert pref1.preferences[str(cat1.id)] == PreferenceStatus.OPTED_OUT.value
- assert pref1.preferences[str(cat2.id)] == PreferenceStatus.OPTED_OUT.value
+ assert cat1.id == newsletter.id
+ assert pref1.get_preference(str(cat1.id)) == PreferenceStatus.OPTED_IN
+ assert pref1.get_preference(str(cat2.id)) == PreferenceStatus.NO_PREFERENCE
+ assert pref1.get_preference(ALL_MESSAGE_PREFERENCE_CATEGORY_ID) == PreferenceStatus.OPTED_OUT
def test_save_csv_batch_with_existing_preferences(self):
"""Test batch saving when some users already have preferences"""
@@ -180,6 +269,22 @@ [email protected],cio_6,"{""topics"": {""topic_1"": true, ""topic_2"": true}}"
new_pref = MessageRecipientPreference.objects.get(team_id=self.team.id, identifier="[email protected]")
assert new_pref.preferences[str(cat1.id)] == PreferenceStatus.OPTED_OUT.value
+ @patch("products.messaging.backend.services.customerio_import_service.CustomerIOClient")
+ def test_api_import_global_unsubscribe_without_topics(self, mock_client_class) -> None:
+ mock_client = mock_client_class.return_value
+ mock_client.validate_credentials.return_value = True
+ mock_client.get_subscription_topics.return_value = []
+ mock_client.get_globally_unsubscribed_customers.return_value = {
+ "identifiers": [{"email": "[email protected]"}],
+ }
+
+ result = self.service.import_api_data()
+
+ assert result["status"] == "completed"
+ assert result["globally_unsubscribed_count"] == 1
+ preference = MessageRecipientPreference.objects.get(team=self.team, identifier="[email protected]")
+ assert preference.get_preference(ALL_MESSAGE_PREFERENCE_CATEGORY_ID) == PreferenceStatus.OPTED_OUT
+
def test_process_csv_without_categories(self):
"""Test CSV processing when no categories exist"""
csv_content = """email,id,cio_subscription_preferences
diff --git a/products/messaging/frontend/generated/api.ts b/products/messaging/frontend/generated/api.ts
index 37b659d0838b..3ebd3092e7f4 100644
--- a/products/messaging/frontend/generated/api.ts
+++ b/products/messaging/frontend/generated/api.ts
@@ -196,7 +196,7 @@ export const getMessagingCategoriesImportPreferencesCsvCreateUrl = (projectId: s
/**
* Import customer preferences from CSV file
- * Expected CSV columns: id, email, cio_subscription_preferences
+ * Expected CSV columns: id, email, unsubscribed, cio_subscription_preferences
*/
export const messagingCategoriesImportPreferencesCsvCreate = async (
projectId: string,
diff --git a/products/messaging/frontend/generated/api.zod.ts b/products/messaging/frontend/generated/api.zod.ts
index a465ae49a882..8bebf94f5fae 100644
--- a/products/messaging/frontend/generated/api.zod.ts
+++ b/products/messaging/frontend/generated/api.zod.ts
@@ -162,7 +162,7 @@ export const MessagingCategoriesImportFromCustomerioCreateBody = /* @__PURE__ */
/**
* Import customer preferences from CSV file
- * Expected CSV columns: id, email, cio_subscription_preferences
+ * Expected CSV columns: id, email, unsubscribed, cio_subscription_preferences
*/
export const messagingCategoriesImportPreferencesCsvCreateBodyKeyMax = 64