:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(workflows): fill customer.io import gaps for topics, subjects and webhooks (#127083)
Daniel Marchukd54ac0a52e2b
diff --git a/products/workflows/backend/facade/contracts.py b/products/workflows/backend/facade/contracts.pyindex 592ae8378fe8..d4f1ffe457b2 100644--- a/products/workflows/backend/facade/contracts.py+++ b/products/workflows/backend/facade/contracts.py@@ -1119,9 +1119,10 @@ class ImportContent:@dataclass(frozen=False)class SnippetRoom:- """How many more bytes of snippet content one import request may inline, shared by every automation in it.+ """How many more bytes of snippet and subject content one import request may inline, shared by every automation.- Snippets are inlined by copy, so a short template that repeats a large one could grow without bound.+ Snippets and `{{ message.subject }}` are inlined by copy, so a short template that repeats a large one could grow+ without bound."""remaining: int = 5_000_000diff --git a/products/workflows/backend/services/customerio_import/images.py b/products/workflows/backend/services/customerio_import/images.pyindex 2547c3766c5e..1830906bb11b 100644--- a/products/workflows/backend/services/customerio_import/images.py+++ b/products/workflows/backend/services/customerio_import/images.py@@ -54,8 +54,9 @@ def rehost_images(ImportNote(level=WorkflowImportNoteLevel.NEEDS_INPUT,message=f"{count} email {_images(count)} still {_load(count)} from Customer.io's servers, "- "because this import can't write to the media library: the user needs editor access to it, "- "and an API key the uploaded_media:write scope. Import as someone who can, or upload "+ "because this import can't write to the media library. The person importing needs editor "+ "access to it, and their API key or PostHog MCP connection needs the uploaded_media:write "+ "scope. Reconnect the MCP and allow uploading media, then import again, or upload "f"{_them(count)} before you close your Customer.io account.",),),diff --git a/products/workflows/backend/services/customerio_import/translator.py b/products/workflows/backend/services/customerio_import/translator.pyindex 781e47b7c3d2..3da650c1b122 100644--- a/products/workflows/backend/services/customerio_import/translator.py+++ b/products/workflows/backend/services/customerio_import/translator.py@@ -11,6 +11,7 @@ import secretsfrom collections import dequefrom collections.abc import Iteratorfrom email.utils import parseaddr+from html import escapefrom typing import Anyfrom posthog.dataclasses import frozen@@ -48,6 +49,10 @@ _DAY_NAMES = {}_WEEKDAYS = ["monday", "tuesday", "wednesday", "thursday", "friday"]_LAYOUT_CONTENT = re.compile(r"\{\{-?\s*content\s*-?\}\}")+# Only a bare subject or one with a `default`, so other filters such as `upcase` keep the unsupported note.+_MESSAGE_SUBJECT = re.compile(r"\{\{-?\s*message\.subject\s*(?:\|\s*default:\s*(?:'[^']*'|\"[^\"]*\")\s*)?-?\}\}")+# Never resolves (RFC 6761), so a draft enabled before the URL is set can't reach a real host.+_PLACEHOLDER_WEBHOOK_URL = "https://webhook-url-not-imported.invalid"_LIQUID_TAG = re.compile(r"\{\{.{0,500}?\}\}|\{%.{0,500}?%\}", re.DOTALL)_SANS_SERIF = "font-family: Arial, Helvetica, sans-serif;"_STATIC_SEGMENT_TYPES = {"static", "manual"}@@ -455,17 +460,18 @@ class _Builder:def _email(self, source: dict[str, Any], action: dict[str, Any]) -> dict[str, Any]:template = self._template(source, action)recipient = template.get("recipient") or source.get("recipient") or "{{customer.email}}"- html = self._liquid(self._with_layout(template, action), action, "email body")+ subject = template.get("subject") or source.get("subject") or ""+ body = self._with_subject(self._with_layout(template, action), _html_subject(subject))email: dict[str, Any] = {"to": {"email": self._liquid(recipient, action, "recipient"), "name": ""},"from": self._sender(template, action),"replyTo": self._address(template, "reply_to_identity_id", "reply_to_id", "reply_to").email,- "subject": self._liquid(template.get("subject") or source.get("subject") or "", action, "subject"),+ "subject": self._liquid(subject, action, "subject"),"preheader": self._liquid(template.get("preheader_text") or "", action, "preheader"),- "html": html,+ "html": self._liquid(body, action, "email body"),}if template.get("body_plain"):- email["text"] = self._liquid(template["body_plain"], action, "plain-text body")+ email["text"] = self._liquid(self._with_subject(template["body_plain"], subject), action, "plain-text body")if not template:# A draft needs a subject and a body to save. The note about the missing content asks for the real ones.email["subject"] = email["subject"] or action["name"]@@ -492,6 +498,14 @@ class _Builder:config["tracking_enabled"] = Falsereturn {"type": "function_email", "config": config}+ def _with_subject(self, body: str, subject: str) -> str:+ """The body with Customer.io's `{{ message.subject }}` replaced by the subject, while the copies fit the room."""+ count = len(_MESSAGE_SUBJECT.findall(body)) if subject else 0+ if not count or count * len(subject) > self.snippet_room.remaining:+ return body+ self.snippet_room.remaining -= count * len(subject)+ return _MESSAGE_SUBJECT.sub(lambda _: subject, body)+def _with_layout(self, template: dict[str, Any], action: dict[str, Any]) -> str:body = template.get("body") or ""layout_id = template.get("layout_id")@@ -545,6 +559,9 @@ class _Builder:def _category(self, source: dict[str, Any], config: dict[str, Any], action: dict[str, Any]) -> None:topic_id = source.get("subscription_topic_id")+ if not topic_id and not source.get("send_to_unsubscribed"):+ # An email step without its own topic sends under the automation's topic, unless it ignores opt-outs.+ topic_id = self.campaign.get("subscription_topic_id")if topic_id:category = self.options.topic_categories.get(str(topic_id))if category is not None:@@ -581,8 +598,9 @@ class _Builder:)else:secret_headers[name] = value+ url = self._liquid(template.get("url") or "", action, "URL") or _PLACEHOLDER_WEBHOOK_URLinputs: dict[str, Any] = {- "url": {"value": self._liquid(template.get("url") or "", action, "URL"), "templating": "liquid"},+ "url": {"value": url, "templating": "liquid"},"method": {"value": str(template.get("request_method") or "POST").upper()},"headers": {"value": headers, "templating": "liquid"},}@@ -599,7 +617,12 @@ class _Builder:action,)if not template.get("url"):- self.note(NoteLevel.NEEDS_INPUT, "Set the webhook URL.", action)+ host = source.get("hostname")+ self.note(+ NoteLevel.NEEDS_INPUT,+ f"Set the webhook URL. Customer.io sends this request to {host}." if host else "Set the webhook URL.",+ action,+ )return {"type": "function", "config": {"template_id": "template-webhook", "inputs": inputs}}def _attribute_update(self, source: dict[str, Any], base: dict[str, Any]) -> dict[str, Any] | None:@@ -1005,6 +1028,17 @@ def _plain_name(name: Any) -> str:return text[:400]+def _html_subject(subject: str) -> str:+ """The subject escaped for HTML, with its Liquid tags left as they are."""+ escaped: list[str] = []+ position = 0+ for tag in _LIQUID_TAG.finditer(subject):+ escaped += [escape(subject[position : tag.start()]), tag.group()]+ position = tag.end()+ escaped.append(escape(subject[position:]))+ return "".join(escaped)++def _strip_comments(html: str) -> str:"""Drop plain HTML comments, keeping Outlook's conditional ones (`<!--[if mso]>`, `<!--<![endif]-->`)."""parts: list[str] = []diff --git a/products/workflows/backend/tests/test_customerio_import.py b/products/workflows/backend/tests/test_customerio_import.pyindex 2dea26f52422..86fe5d5fa782 100644--- a/products/workflows/backend/tests/test_customerio_import.py+++ b/products/workflows/backend/tests/test_customerio_import.py@@ -107,7 +107,8 @@ def _onboarding_campaign() -> CampaignSource:return CampaignSource(campaign=campaign, actions=actions, templates=templates)-def _sync_email_template() -> None:+def _sync_templates() -> None:+ sync_template_to_db(MOCK_NODE_TEMPLATES[0])sync_template_to_db({**MOCK_NODE_TEMPLATES[0],@@ -124,7 +125,7 @@ _OPTIONS = ImportOptions(snippets={"footer": '<a href="{% unsubscribe_url %}">Unclass TestCustomerioCampaignTranslation(BaseTest):def setUp(self) -> None:super().setUp()- _sync_email_template()+ _sync_templates()def test_translated_campaign_saves_as_a_draft_with_the_same_paths(self) -> None:translated = translate_campaign(_onboarding_campaign(), _OPTIONS)@@ -270,22 +271,43 @@ class TestCustomerioCampaignTranslation(BaseTest):assert {"from": "trigger_node", "to": "cio_99", "type": "continue"} in workflow["edges"]- def test_segment_trigger_without_a_cohort_still_saves_and_asks_for_one(self) -> None:+ def test_draft_without_a_segment_cohort_or_webhook_content_still_saves_and_asks_for_them(self) -> None:source = _onboarding_campaign()- campaign = {**source.campaign, "type": "seg_attr", "trigger": {"and": [{"or": [{"segment": {"id": 7}}]}]}}+ campaign = {+ **source.campaign,+ "type": "seg_attr",+ "trigger": {"and": [{"or": [{"segment": {"id": 7}}]}]},+ "edges": [*source.campaign["edges"], {"from": "7", "to": "10", "type": "continue"}],+ "workflow_action_ids": [*source.campaign["workflow_action_ids"], "10"],+ }+ actions = [+ *source.actions,+ {"id": "10", "type": "webhook_action", "name": "CRM", "template_id": "15", "hostname": "crm.example.com"},+ ]translated = translate_campaign(- CampaignSource(campaign=campaign, actions=source.actions, templates=source.templates), _OPTIONS+ CampaignSource(campaign=campaign, actions=actions, templates=source.templates), _OPTIONS)assert validated_draft(self.team, translated.workflow, EventSource.API) is not Noneassert any(note.level == NoteLevel.NEEDS_INPUT and "segment 7" in note.message for note in translated.notes)+ assert any(+ note.step_id == "cio_10"+ and "Set the webhook URL. Customer.io sends this request to crm.example.com." == note.message+ for note in translated.notes+ )def test_email_takes_its_layout_sender_and_category(self) -> None:source = _onboarding_campaign()- campaign = {**source.campaign, "filters": _encode([{"type": "segments", "segments": [9]}])}+ campaign = {+ **source.campaign,+ "filters": _encode([{"type": "segments", "segments": [9]}]),+ "subscription_topic_id": "4",+ }actions = [{**action, "subscription_topic_id": 3, "name": "Welcome {{ customer.first_name }} !"}if action["id"] == "1"+ else {**action, "send_to_unsubscribed": True}+ if action["id"] == "7"else actionfor action in source.actions]@@ -293,6 +315,11 @@ class TestCustomerioCampaignTranslation(BaseTest):**source.templates,"11": {**source.templates["11"], "layout_id": 5, "from_identity_id": "77", "reply_to": "[email protected]"},"12": {**source.templates["12"], "from": "Ops <[email protected]>", "layout_id": 6},+ "13": {+ **source.templates["13"],+ "subject": "Tips & tricks for {{ customer.company }}",+ "body": "<title>{{message.subject | default: ''}}</title><h1>{{ message.subject | upcase }}</h1>",+ },}options = ImportOptions(snippets={},@@ -303,7 +330,10 @@ class TestCustomerioCampaignTranslation(BaseTest):},sender_addresses={"77": "Team <[email protected]>"},senders={"[email protected]": ImportSender(integration_id=12, email="[email protected]", name="PostHog")},- topic_categories={"3": ImportCategory(id="cat-1", category_type="marketing")},+ topic_categories={+ "3": ImportCategory(id="cat-1", category_type="marketing"),+ "4": ImportCategory(id="cat-2", category_type="marketing"),+ },segments={"9": ImportSegment(name="Beta list", type="static")},)translated = translate_campaign(@@ -323,6 +353,12 @@ class TestCustomerioCampaignTranslation(BaseTest):assert email["from"] == {"integrationId": 12, "email": "[email protected]", "name": "Team"}assert email["replyTo"] == "[email protected]"assert (welcome["message_category_id"], welcome["message_category_type"]) == ("cat-1", "marketing")+ assert actions_by_id["cio_5"]["config"]["message_category_id"] == "cat-2"+ nudge = actions_by_id["cio_7"]["config"]+ assert (nudge.get("message_category_id"), nudge["message_category_type"]) == (None, "transactional")+ nudge_html = nudge["inputs"]["email"]["value"]["html"]+ assert "<title>Tips & tricks for {{ person.properties.company }}</title>" in nudge_html+ assert "<h1>{{ message.subject | upcase }}</h1>" in nudge_htmlactivated = actions_by_id["cio_5"]["config"]["inputs"]["email"]["value"]assert activated["from"]["email"] == "[email protected]"assert activated["html"].startswith('<div style="font-family: Arial, Helvetica, sans-serif;">')@@ -340,6 +376,7 @@ class TestCustomerioCampaignTranslation(BaseTest):)assert any(text.startswith('Only people in segment "Beta list" (9) can enter') for _, _, text in messages)assert not any(step == "cio_1" and "sender" in text for _, step, text in messages)+ assert any(step == "cio_7" and "`message`" in text for _, step, text in messages)def test_language_split_keeps_only_the_default_message(self) -> None:campaign = {@@ -508,11 +545,24 @@ class TestCustomerioLiquid(SimpleTestCase):assert rewritten.text == "0123456" + tag * 2assert rewritten.missing_snippets == ("big",)+ def test_stops_inlining_the_subject_past_the_budget(self) -> None:+ source = _onboarding_campaign()+ templates = {**source.templates, "12": {**source.templates["12"], "body": "{{ message.subject }}" * 3}}+ translated = translate_campaign(+ CampaignSource(campaign=source.campaign, actions=source.actions, templates=templates),+ _OPTIONS,+ SnippetRoom(remaining=20),+ )++ steps = {action["id"]: action for action in translated.workflow["actions"]}+ assert "message.subject" in steps["cio_5"]["config"]["inputs"]["email"]["value"]["html"]+ assert any(note.step_id == "cio_5" and "`message`" in note.message for note in translated.notes)+class TestWorkflowImportAPI(APIBaseTest):def setUp(self) -> None:super().setUp()- _sync_email_template()+ _sync_templates()flag = patch("posthoganalytics.feature_enabled", return_value=True)flag.start()self.addCleanup(flag.stop)diff --git a/products/workflows/skills/migrating-from-customer-io/SKILL.md b/products/workflows/skills/migrating-from-customer-io/SKILL.mdindex 9897f0b666a4..960044940a5a 100644--- a/products/workflows/skills/migrating-from-customer-io/SKILL.md+++ b/products/workflows/skills/migrating-from-customer-io/SKILL.md@@ -44,7 +44,11 @@ The same notes appear in the PostHog app on the imported workflow, and `workflow## Senders and categories-An email gets the PostHog email sender with the same address as its Customer.io sender. When no PostHog sender has that address, a `needs_input` note names it, so the user can add it as a sender. An email's subscription topic becomes the PostHog message category the opt-out import made from that topic. When the topics were not imported, a note says so.+An email gets the PostHog email sender with the same address as its Customer.io sender. When no PostHog sender has that address, a `needs_input` note names it, so the user can add it as a sender. An email's subscription topic, or the automation's topic when the email has none, becomes the PostHog message category the opt-out import made from that topic. When the topics were not imported, a note says so.++## Images++The import copies email images hosted by Customer.io into the project's media library, so they keep loading after the Customer.io account closes. That needs the `uploaded_media:write` scope. If a note says images still load from Customer.io's servers, ask the user to reconnect the PostHog MCP and allow uploading media, then import again.## Segments and cohorts