Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(workflows): fill customer.io import gaps for topics, subjects and webhooks (#127083)

Daniel Marchukd54ac0a52e2b
diff --git a/products/workflows/backend/facade/contracts.py b/products/workflows/backend/facade/contracts.py
index 592ae8378fe8..d4f1ffe457b2 100644
--- a/products/workflows/backend/facade/contracts.py
+++ b/products/workflows/backend/facade/contracts.py
@@ -1119,9 +1119,10 @@ class ImportContent:
@dataclass(frozen=False)
class SnippetRoom:
- """How many more bytes of snippet content one import request may inline, shared by every automation in it.
+ """How many more bytes of snippet and subject content one import request may inline, shared by every automation.
- Snippets are inlined by copy, so a short template that repeats a large one could grow without bound.
+ Snippets and `{{ message.subject }}` are inlined by copy, so a short template that repeats a large one could grow
+ without bound.
"""
remaining: int = 5_000_000
diff --git a/products/workflows/backend/services/customerio_import/images.py b/products/workflows/backend/services/customerio_import/images.py
index 2547c3766c5e..1830906bb11b 100644
--- a/products/workflows/backend/services/customerio_import/images.py
+++ b/products/workflows/backend/services/customerio_import/images.py
@@ -54,8 +54,9 @@ def rehost_images(
ImportNote(
level=WorkflowImportNoteLevel.NEEDS_INPUT,
message=f"{count} email {_images(count)} still {_load(count)} from Customer.io's servers, "
- "because this import can't write to the media library: the user needs editor access to it, "
- "and an API key the uploaded_media:write scope. Import as someone who can, or upload "
+ "because this import can't write to the media library. The person importing needs editor "
+ "access to it, and their API key or PostHog MCP connection needs the uploaded_media:write "
+ "scope. Reconnect the MCP and allow uploading media, then import again, or upload "
f"{_them(count)} before you close your Customer.io account.",
),
),
diff --git a/products/workflows/backend/services/customerio_import/translator.py b/products/workflows/backend/services/customerio_import/translator.py
index 781e47b7c3d2..3da650c1b122 100644
--- a/products/workflows/backend/services/customerio_import/translator.py
+++ b/products/workflows/backend/services/customerio_import/translator.py
@@ -11,6 +11,7 @@ import secrets
from collections import deque
from collections.abc import Iterator
from email.utils import parseaddr
+from html import escape
from typing import Any
from posthog.dataclasses import frozen
@@ -48,6 +49,10 @@ _DAY_NAMES = {
}
_WEEKDAYS = ["monday", "tuesday", "wednesday", "thursday", "friday"]
_LAYOUT_CONTENT = re.compile(r"\{\{-?\s*content\s*-?\}\}")
+# Only a bare subject or one with a `default`, so other filters such as `upcase` keep the unsupported note.
+_MESSAGE_SUBJECT = re.compile(r"\{\{-?\s*message\.subject\s*(?:\|\s*default:\s*(?:'[^']*'|\"[^\"]*\")\s*)?-?\}\}")
+# Never resolves (RFC 6761), so a draft enabled before the URL is set can't reach a real host.
+_PLACEHOLDER_WEBHOOK_URL = "https://webhook-url-not-imported.invalid"
_LIQUID_TAG = re.compile(r"\{\{.{0,500}?\}\}|\{%.{0,500}?%\}", re.DOTALL)
_SANS_SERIF = "font-family: Arial, Helvetica, sans-serif;"
_STATIC_SEGMENT_TYPES = {"static", "manual"}
@@ -455,17 +460,18 @@ class _Builder:
def _email(self, source: dict[str, Any], action: dict[str, Any]) -> dict[str, Any]:
template = self._template(source, action)
recipient = template.get("recipient") or source.get("recipient") or "{{customer.email}}"
- html = self._liquid(self._with_layout(template, action), action, "email body")
+ subject = template.get("subject") or source.get("subject") or ""
+ body = self._with_subject(self._with_layout(template, action), _html_subject(subject))
email: dict[str, Any] = {
"to": {"email": self._liquid(recipient, action, "recipient"), "name": ""},
"from": self._sender(template, action),
"replyTo": self._address(template, "reply_to_identity_id", "reply_to_id", "reply_to").email,
- "subject": self._liquid(template.get("subject") or source.get("subject") or "", action, "subject"),
+ "subject": self._liquid(subject, action, "subject"),
"preheader": self._liquid(template.get("preheader_text") or "", action, "preheader"),
- "html": html,
+ "html": self._liquid(body, action, "email body"),
}
if template.get("body_plain"):
- email["text"] = self._liquid(template["body_plain"], action, "plain-text body")
+ email["text"] = self._liquid(self._with_subject(template["body_plain"], subject), action, "plain-text body")
if not template:
# A draft needs a subject and a body to save. The note about the missing content asks for the real ones.
email["subject"] = email["subject"] or action["name"]
@@ -492,6 +498,14 @@ class _Builder:
config["tracking_enabled"] = False
return {"type": "function_email", "config": config}
+ def _with_subject(self, body: str, subject: str) -> str:
+ """The body with Customer.io's `{{ message.subject }}` replaced by the subject, while the copies fit the room."""
+ count = len(_MESSAGE_SUBJECT.findall(body)) if subject else 0
+ if not count or count * len(subject) > self.snippet_room.remaining:
+ return body
+ self.snippet_room.remaining -= count * len(subject)
+ return _MESSAGE_SUBJECT.sub(lambda _: subject, body)
+
def _with_layout(self, template: dict[str, Any], action: dict[str, Any]) -> str:
body = template.get("body") or ""
layout_id = template.get("layout_id")
@@ -545,6 +559,9 @@ class _Builder:
def _category(self, source: dict[str, Any], config: dict[str, Any], action: dict[str, Any]) -> None:
topic_id = source.get("subscription_topic_id")
+ if not topic_id and not source.get("send_to_unsubscribed"):
+ # An email step without its own topic sends under the automation's topic, unless it ignores opt-outs.
+ topic_id = self.campaign.get("subscription_topic_id")
if topic_id:
category = self.options.topic_categories.get(str(topic_id))
if category is not None:
@@ -581,8 +598,9 @@ class _Builder:
)
else:
secret_headers[name] = value
+ url = self._liquid(template.get("url") or "", action, "URL") or _PLACEHOLDER_WEBHOOK_URL
inputs: dict[str, Any] = {
- "url": {"value": self._liquid(template.get("url") or "", action, "URL"), "templating": "liquid"},
+ "url": {"value": url, "templating": "liquid"},
"method": {"value": str(template.get("request_method") or "POST").upper()},
"headers": {"value": headers, "templating": "liquid"},
}
@@ -599,7 +617,12 @@ class _Builder:
action,
)
if not template.get("url"):
- self.note(NoteLevel.NEEDS_INPUT, "Set the webhook URL.", action)
+ host = source.get("hostname")
+ self.note(
+ NoteLevel.NEEDS_INPUT,
+ f"Set the webhook URL. Customer.io sends this request to {host}." if host else "Set the webhook URL.",
+ action,
+ )
return {"type": "function", "config": {"template_id": "template-webhook", "inputs": inputs}}
def _attribute_update(self, source: dict[str, Any], base: dict[str, Any]) -> dict[str, Any] | None:
@@ -1005,6 +1028,17 @@ def _plain_name(name: Any) -> str:
return text[:400]
+def _html_subject(subject: str) -> str:
+ """The subject escaped for HTML, with its Liquid tags left as they are."""
+ escaped: list[str] = []
+ position = 0
+ for tag in _LIQUID_TAG.finditer(subject):
+ escaped += [escape(subject[position : tag.start()]), tag.group()]
+ position = tag.end()
+ escaped.append(escape(subject[position:]))
+ return "".join(escaped)
+
+
def _strip_comments(html: str) -> str:
"""Drop plain HTML comments, keeping Outlook's conditional ones (`<!--[if mso]>`, `<!--<![endif]-->`)."""
parts: list[str] = []
diff --git a/products/workflows/backend/tests/test_customerio_import.py b/products/workflows/backend/tests/test_customerio_import.py
index 2dea26f52422..86fe5d5fa782 100644
--- a/products/workflows/backend/tests/test_customerio_import.py
+++ b/products/workflows/backend/tests/test_customerio_import.py
@@ -107,7 +107,8 @@ def _onboarding_campaign() -> CampaignSource:
return CampaignSource(campaign=campaign, actions=actions, templates=templates)
-def _sync_email_template() -> None:
+def _sync_templates() -> None:
+ sync_template_to_db(MOCK_NODE_TEMPLATES[0])
sync_template_to_db(
{
**MOCK_NODE_TEMPLATES[0],
@@ -124,7 +125,7 @@ _OPTIONS = ImportOptions(snippets={"footer": '<a href="{% unsubscribe_url %}">Un
class TestCustomerioCampaignTranslation(BaseTest):
def setUp(self) -> None:
super().setUp()
- _sync_email_template()
+ _sync_templates()
def test_translated_campaign_saves_as_a_draft_with_the_same_paths(self) -> None:
translated = translate_campaign(_onboarding_campaign(), _OPTIONS)
@@ -270,22 +271,43 @@ class TestCustomerioCampaignTranslation(BaseTest):
assert {"from": "trigger_node", "to": "cio_99", "type": "continue"} in workflow["edges"]
- def test_segment_trigger_without_a_cohort_still_saves_and_asks_for_one(self) -> None:
+ def test_draft_without_a_segment_cohort_or_webhook_content_still_saves_and_asks_for_them(self) -> None:
source = _onboarding_campaign()
- campaign = {**source.campaign, "type": "seg_attr", "trigger": {"and": [{"or": [{"segment": {"id": 7}}]}]}}
+ campaign = {
+ **source.campaign,
+ "type": "seg_attr",
+ "trigger": {"and": [{"or": [{"segment": {"id": 7}}]}]},
+ "edges": [*source.campaign["edges"], {"from": "7", "to": "10", "type": "continue"}],
+ "workflow_action_ids": [*source.campaign["workflow_action_ids"], "10"],
+ }
+ actions = [
+ *source.actions,
+ {"id": "10", "type": "webhook_action", "name": "CRM", "template_id": "15", "hostname": "crm.example.com"},
+ ]
translated = translate_campaign(
- CampaignSource(campaign=campaign, actions=source.actions, templates=source.templates), _OPTIONS
+ CampaignSource(campaign=campaign, actions=actions, templates=source.templates), _OPTIONS
)
assert validated_draft(self.team, translated.workflow, EventSource.API) is not None
assert any(note.level == NoteLevel.NEEDS_INPUT and "segment 7" in note.message for note in translated.notes)
+ assert any(
+ note.step_id == "cio_10"
+ and "Set the webhook URL. Customer.io sends this request to crm.example.com." == note.message
+ for note in translated.notes
+ )
def test_email_takes_its_layout_sender_and_category(self) -> None:
source = _onboarding_campaign()
- campaign = {**source.campaign, "filters": _encode([{"type": "segments", "segments": [9]}])}
+ campaign = {
+ **source.campaign,
+ "filters": _encode([{"type": "segments", "segments": [9]}]),
+ "subscription_topic_id": "4",
+ }
actions = [
{**action, "subscription_topic_id": 3, "name": "Welcome {{ customer.first_name }} !"}
if action["id"] == "1"
+ else {**action, "send_to_unsubscribed": True}
+ if action["id"] == "7"
else action
for action in source.actions
]
@@ -293,6 +315,11 @@ class TestCustomerioCampaignTranslation(BaseTest):
**source.templates,
"11": {**source.templates["11"], "layout_id": 5, "from_identity_id": "77", "reply_to": "[email protected]"},
"12": {**source.templates["12"], "from": "Ops <[email protected]>", "layout_id": 6},
+ "13": {
+ **source.templates["13"],
+ "subject": "Tips & tricks for {{ customer.company }}",
+ "body": "<title>{{message.subject | default: ''}}</title><h1>{{ message.subject | upcase }}</h1>",
+ },
}
options = ImportOptions(
snippets={},
@@ -303,7 +330,10 @@ class TestCustomerioCampaignTranslation(BaseTest):
},
sender_addresses={"77": "Team <[email protected]>"},
senders={"[email protected]": ImportSender(integration_id=12, email="[email protected]", name="PostHog")},
- topic_categories={"3": ImportCategory(id="cat-1", category_type="marketing")},
+ topic_categories={
+ "3": ImportCategory(id="cat-1", category_type="marketing"),
+ "4": ImportCategory(id="cat-2", category_type="marketing"),
+ },
segments={"9": ImportSegment(name="Beta list", type="static")},
)
translated = translate_campaign(
@@ -323,6 +353,12 @@ class TestCustomerioCampaignTranslation(BaseTest):
assert email["from"] == {"integrationId": 12, "email": "[email protected]", "name": "Team"}
assert email["replyTo"] == "[email protected]"
assert (welcome["message_category_id"], welcome["message_category_type"]) == ("cat-1", "marketing")
+ assert actions_by_id["cio_5"]["config"]["message_category_id"] == "cat-2"
+ nudge = actions_by_id["cio_7"]["config"]
+ assert (nudge.get("message_category_id"), nudge["message_category_type"]) == (None, "transactional")
+ nudge_html = nudge["inputs"]["email"]["value"]["html"]
+ assert "<title>Tips &amp; tricks for {{ person.properties.company }}</title>" in nudge_html
+ assert "<h1>{{ message.subject | upcase }}</h1>" in nudge_html
activated = actions_by_id["cio_5"]["config"]["inputs"]["email"]["value"]
assert activated["from"]["email"] == "[email protected]"
assert activated["html"].startswith('<div style="font-family: Arial, Helvetica, sans-serif;">')
@@ -340,6 +376,7 @@ class TestCustomerioCampaignTranslation(BaseTest):
)
assert any(text.startswith('Only people in segment "Beta list" (9) can enter') for _, _, text in messages)
assert not any(step == "cio_1" and "sender" in text for _, step, text in messages)
+ assert any(step == "cio_7" and "`message`" in text for _, step, text in messages)
def test_language_split_keeps_only_the_default_message(self) -> None:
campaign = {
@@ -508,11 +545,24 @@ class TestCustomerioLiquid(SimpleTestCase):
assert rewritten.text == "0123456" + tag * 2
assert rewritten.missing_snippets == ("big",)
+ def test_stops_inlining_the_subject_past_the_budget(self) -> None:
+ source = _onboarding_campaign()
+ templates = {**source.templates, "12": {**source.templates["12"], "body": "{{ message.subject }}" * 3}}
+ translated = translate_campaign(
+ CampaignSource(campaign=source.campaign, actions=source.actions, templates=templates),
+ _OPTIONS,
+ SnippetRoom(remaining=20),
+ )
+
+ steps = {action["id"]: action for action in translated.workflow["actions"]}
+ assert "message.subject" in steps["cio_5"]["config"]["inputs"]["email"]["value"]["html"]
+ assert any(note.step_id == "cio_5" and "`message`" in note.message for note in translated.notes)
+
class TestWorkflowImportAPI(APIBaseTest):
def setUp(self) -> None:
super().setUp()
- _sync_email_template()
+ _sync_templates()
flag = patch("posthoganalytics.feature_enabled", return_value=True)
flag.start()
self.addCleanup(flag.stop)
diff --git a/products/workflows/skills/migrating-from-customer-io/SKILL.md b/products/workflows/skills/migrating-from-customer-io/SKILL.md
index 9897f0b666a4..960044940a5a 100644
--- a/products/workflows/skills/migrating-from-customer-io/SKILL.md
+++ b/products/workflows/skills/migrating-from-customer-io/SKILL.md
@@ -44,7 +44,11 @@ The same notes appear in the PostHog app on the imported workflow, and `workflow
## Senders and categories
-An email gets the PostHog email sender with the same address as its Customer.io sender. When no PostHog sender has that address, a `needs_input` note names it, so the user can add it as a sender. An email's subscription topic becomes the PostHog message category the opt-out import made from that topic. When the topics were not imported, a note says so.
+An email gets the PostHog email sender with the same address as its Customer.io sender. When no PostHog sender has that address, a `needs_input` note names it, so the user can add it as a sender. An email's subscription topic, or the automation's topic when the email has none, becomes the PostHog message category the opt-out import made from that topic. When the topics were not imported, a note says so.
+
+## Images
+
+The import copies email images hosted by Customer.io into the project's media library, so they keep loading after the Customer.io account closes. That needs the `uploaded_media:write` scope. If a note says images still load from Customer.io's servers, ask the user to reconnect the PostHog MCP and allow uploading media, then import again.
## Segments and cohorts