:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(workflows): open the current email html when the stored wrap design is stale (#129747)
ruby childs7c0f0ba7000a
diff --git a/frontend/src/scenes/hog-functions/email-templater/emailTemplaterLogic.test.ts b/frontend/src/scenes/hog-functions/email-templater/emailTemplaterLogic.test.tsindex 42916bcae9cb..50c0c5e85b44 100644--- a/frontend/src/scenes/hog-functions/email-templater/emailTemplaterLogic.test.ts+++ b/frontend/src/scenes/hog-functions/email-templater/emailTemplaterLogic.test.ts@@ -14,6 +14,7 @@ import {EditorRef,EmailTemplate,EmailTemplaterLogicProps,+ buildHtmlWrapDesign,buildPersonPropertyMergeValue,emailTemplaterLogic,} from './emailTemplaterLogic'@@ -39,6 +40,14 @@ function makeProps(overrides?: Partial<EmailTemplaterLogicProps>): EmailTemplateconst ADVANCED_FIELDS = EMAIL_TYPE_SUPPORTED_FIELDS.native_email.filter((f) => f.isAdvancedField)const NON_ADVANCED_FIELDS = EMAIL_TYPE_SUPPORTED_FIELDS.native_email.filter((f) => !f.isAdvancedField)+const renderAround = (block: string): string =>+ `<html><head><style>#u_body a { color: #00e; }</style></head><body class="clean-body u_body"><!--[if mso]><table><tr><td><![endif]--><table id="u_body"><tbody><tr><td><div class="u-row">${block}</div></td></tr></tbody></table></body></html>`+const RENDER_OF_OLD = renderAround('<p>old</p>')+const RENDER_WITH_FOOTER = renderAround('<p>old</p><p>P.S.</p>')+const RENDERED_PAGE = renderAround('<p>page</p>')+const RENDERED_PAGE_WITH_FOOTER = `<div>${RENDERED_PAGE}<p>footer</p></div>`+const RENDER_WITH_IMAGE = renderAround('<p>old</p><img src="https://example.com/offer.png">')+describe('emailTemplaterLogic', () => {let logic: ReturnType<typeof emailTemplaterLogic.build>@@ -299,6 +308,28 @@ describe('emailTemplaterLogic', () => {}))})++ it('applying a template whose stored wrap is stale loads its current html', () => {+ const loadDesign = jest.fn()+ logic = emailTemplaterLogic(makeProps())+ logic.mount()+ logic.actions.setEmailEditorRef({+ editor: { loadDesign, addEventListener: jest.fn() },+ } as unknown as EditorRef)+ logic.actions.onEmailEditorReady()+ loadDesign.mockClear()++ logic.actions.applyTemplate({+ id: 'template-3',+ name: 'Newsletter',+ content: {+ templating: 'liquid',+ email: { subject: 'News', html: '<p>new</p>', design: buildHtmlWrapDesign('<p>old</p>') },+ },+ } as MessageTemplate)++ expect(loadDesign).toHaveBeenCalledWith(buildHtmlWrapDesign('<p>new</p>'))+ })})describe('html-only emails', () => {@@ -321,6 +352,32 @@ describe('emailTemplaterLogic', () => {})})+ it.each([+ ['the html was rewritten', '<p>old</p>', '<p>new</p>', '<p>new</p>'],+ ['copy was appended to the block', '<p>old</p>', '<p>old</p><p>footer</p>', '<p>old</p><p>footer</p>'],+ ['copy was appended inside its Unlayer render', '<p>old</p>', RENDER_WITH_FOOTER, RENDER_WITH_FOOTER],+ [+ 'the block itself is an Unlayer page that got a footer',+ RENDERED_PAGE,+ RENDERED_PAGE_WITH_FOOTER,+ RENDERED_PAGE_WITH_FOOTER,+ ],+ ['an image was added inside its Unlayer render', '<p>old</p>', RENDER_WITH_IMAGE, RENDER_WITH_IMAGE],+ ['the html is the Unlayer render of the wrap', '<p>old</p>', RENDER_OF_OLD, '<p>old</p>'],+ ])('when %s, the canvas loads a wrap of the right html', (_name, blockHtml, html, expectedBlockHtml) => {+ const loadDesign = jest.fn()+ logic = emailTemplaterLogic(+ makeProps({ value: { ...DEFAULT_EMAIL_TEMPLATE, design: buildHtmlWrapDesign(blockHtml), html } })+ )+ logic.mount()+ logic.actions.setEmailEditorRef({+ editor: { loadDesign, addEventListener: jest.fn() },+ } as unknown as EditorRef)+ logic.actions.onEmailEditorReady()++ expect(loadDesign).toHaveBeenCalledWith(buildHtmlWrapDesign(expectedBlockHtml))+ })+it('does not load anything when there is neither design nor html', () => {const loadDesign = jest.fn()logic = emailTemplaterLogic(makeProps({ value: { ...DEFAULT_EMAIL_TEMPLATE, design: null, html: '' } }))diff --git a/frontend/src/scenes/hog-functions/email-templater/emailTemplaterLogic.tsx b/frontend/src/scenes/hog-functions/email-templater/emailTemplaterLogic.tsxindex 074288c3cd6b..9070d66d8b8c 100644--- a/frontend/src/scenes/hog-functions/email-templater/emailTemplaterLogic.tsx+++ b/frontend/src/scenes/hog-functions/email-templater/emailTemplaterLogic.tsx@@ -169,6 +169,84 @@ export function buildHtmlWrapDesign(html: string): JSONTemplate {} as unknown as JSONTemplate}+const UNLAYER_BODY_CLASS = /\bu_body\b/+const UNLAYER_LAYOUT_TAGS = new Set(['!doctype', 'html', 'body', 'table', 'tbody', 'tr', 'td', 'div'])+const TAG_NAME = /<\/?\s*([a-z!][a-z0-9]*)/y+const SKIPPED_ELEMENT_ENDS = new Map([+ ['head', '</head>'],+ ['style', '</style>'],+])++function tagNameAt(html: string, position: number): string | null {+ TAG_NAME.lastIndex = position+ return TAG_NAME.exec(html)?.[1] ?? null+}++function positionAfterLayoutMarkup(html: string, position: number): number {+ if (html.startsWith('<!--', position)) {+ const end = html.indexOf('-->', position + 4)+ return end === -1 ? -1 : end + 3+ }+ const tag = tagNameAt(html, position)+ const end = html.indexOf('>', position)+ if (tag === null || end === -1) {+ return -1+ }+ const closing = SKIPPED_ELEMENT_ENDS.get(tag)+ if (closing !== undefined) {+ const closingAt = html.indexOf(closing, end)+ return closingAt === -1 ? -1 : closingAt + closing.length+ }+ return UNLAYER_LAYOUT_TAGS.has(tag) ? end + 1 : -1+}++function isOnlyUnlayerLayout(markup: string): boolean {+ const html = markup.toLowerCase()+ let position = 0+ while (position < html.length) {+ if (html[position] === '<') {+ position = positionAfterLayoutMarkup(html, position)+ if (position === -1) {+ return false+ }+ } else if (html.startsWith(' ', position)) {+ position += 6+ } else if (/\s/.test(html[position])) {+ position += 1+ } else {+ return false+ }+ }+ return true+}++function isUnlayerRenderOf(html: string, blockHtml: string): boolean {+ const start = html.indexOf(blockHtml)+ if (start === -1) {+ return false+ }+ const aroundBlock = html.slice(0, start) + html.slice(start + blockHtml.length)+ return UNLAYER_BODY_CLASS.test(aroundBlock) && isOnlyUnlayerLayout(aroundBlock)+}++export function editorDesignForEmail(value: Pick<EmailTemplate, 'html' | 'design'>): JSONTemplate | null {+ const { html, design } = value+ if (!design) {+ return html ? buildHtmlWrapDesign(html) : null+ }+ const wrappedHtml = design.body?.rows?.[0]?.columns?.[0]?.contents?.[0]?.values?.html+ if (+ html &&+ typeof wrappedHtml === 'string' &&+ wrappedHtml !== html &&+ objectsEqual(design, buildHtmlWrapDesign(wrappedHtml)) &&+ !isUnlayerRenderOf(html, wrappedHtml)+ ) {+ return buildHtmlWrapDesign(html)+ }+ return design+}+// URL reflection for the fullscreen editor (?editor=email), so back, Escape, and deep links work.const EMAIL_EDITOR_URL_PARAM = 'editor'const EMAIL_EDITOR_URL_VALUE = 'email'@@ -701,13 +779,10 @@ export const emailTemplaterLogic = kea<emailTemplaterLogicType>([// The editor read its merge tags when it started, so any added since then are pushed in.cache.lastMergeTags = buildMergeTags(values.personPropertyDefinitions, props.type, props.variables)values.emailEditorRef?.editor?.setMergeTags?.(cache.lastMergeTags)- if (props.value?.design) {- cache.lastEditorDesign = props.value.design- values.emailEditorRef?.editor?.loadDesign(props.value.design)- } else if (props.value?.html) {- const wrapped = buildHtmlWrapDesign(props.value.html)- cache.lastEditorDesign = wrapped- values.emailEditorRef?.editor?.loadDesign(wrapped)+ const design = props.value ? editorDesignForEmail(props.value) : null+ if (design) {+ cache.lastEditorDesign = design+ values.emailEditorRef?.editor?.loadDesign(design)}},@@ -812,10 +887,10 @@ export const emailTemplaterLogic = kea<emailTemplaterLogicType>([design: emailTemplateContent.design ?? null,})- // Load the design into the editor if it's ready and has a design- if (values.isEmailEditorReady && emailTemplateContent.design) {- cache.lastEditorDesign = emailTemplateContent.design- values.emailEditorRef?.editor?.loadDesign(emailTemplateContent.design)+ const design = editorDesignForEmail(emailTemplateContent)+ if (values.isEmailEditorReady && design) {+ cache.lastEditorDesign = design+ values.emailEditorRef?.editor?.loadDesign(design)}},@@ -984,7 +1059,7 @@ export const emailTemplaterLogic = kea<emailTemplaterLogicType>([values.isEmailEditorReady &&!cache.pendingDesignEdit) {- const design = props.value.design ?? (props.value.html ? buildHtmlWrapDesign(props.value.html) : null)+ const design = editorDesignForEmail(props.value)if (design &&!objectsEqual(design, cache.lastEditorDesign) &&diff --git a/products/cdp/backend/presentation/hog_functions.py b/products/cdp/backend/presentation/hog_functions.pyindex d4745ab1e49b..19205c4f6ca3 100644--- a/products/cdp/backend/presentation/hog_functions.py+++ b/products/cdp/backend/presentation/hog_functions.py@@ -19,5 +19,6 @@ from products.cdp.backend.validation import (COHORT_NOT_READY_CODE as COHORT_NOT_READY_CODE,build_html_wrap_design as build_html_wrap_design,generate_template_bytecode as generate_template_bytecode,+ html_wrap_design_needs_refresh as html_wrap_design_needs_refresh,validate_realtime_cohorts as validate_realtime_cohorts,)diff --git a/products/cdp/backend/test/test_validation.py b/products/cdp/backend/test/test_validation.pyindex 81ad0df619de..d21fed82f971 100644--- a/products/cdp/backend/test/test_validation.py+++ b/products/cdp/backend/test/test_validation.py@@ -19,6 +19,7 @@ from products.cdp.backend.validation import (InputsSchemaItemSerializer,InputsSchemaSerializer,RecordAliasRewriter,+ build_html_wrap_design,compile_hog,generate_template_bytecode,reserved_functions_used,@@ -89,6 +90,16 @@ def create_example_inputs():}+def _render_around(block_html: str) -> str:+ return (+ "<!DOCTYPE html><html><head><style>#u_body a { color: #00e; }</style></head>"+ '<body class="clean-body u_body"><!--[if mso]><table><tr><td><![endif]-->'+ '<table id="u_body"><tbody><tr><td><div class="u-row-container"><div class="u-row">'+ f"{block_html}"+ "</div></div></td></tr></tbody></table></body></html>"+ )++class TestHogFunctionValidation(ClickhouseTestMixin, APIBaseTest, QueryMatchingTest):filters_context: dict = {}@@ -503,6 +514,84 @@ class TestHogFunctionValidation(ClickhouseTestMixin, APIBaseTest, QueryMatchingTassert validated["email"]["value"].get("design") == expected_design+ @parameterized.expand(+ [+ ("plain_html", "<p>Old copy</p>", "<p>New copy</p>"),+ ("copy_appended_to_the_block", "<p>Old copy</p>", "<p>Old copy</p><p>New footer</p>"),+ ("block_wrapped_in_new_markup", "<p>Old copy</p>", '<div style="padding: 8px"><p>Old copy</p></div>'),+ (+ "copy_appended_inside_its_unlayer_render",+ "<p>Old copy</p>",+ _render_around("<p>Old copy</p><p>P.S.</p>"),+ ),+ (+ "block_is_an_unlayer_page_that_got_a_footer",+ _render_around("<p>Page</p>"),+ f"<div>{_render_around('<p>Page</p>')}<p>Footer</p></div>",+ ),+ (+ "unrelated_body_class_is_not_the_unlayer_page",+ "<p>Old copy</p>",+ '<div class="menu_body"><p>Old copy</p></div>',+ ),+ (+ "image_added_inside_its_unlayer_render",+ "<p>Old copy</p>",+ _render_around('<p>Old copy</p><img src="https://example.com/offer.png">'),+ ),+ (+ "link_added_inside_its_unlayer_render",+ "<p>Old copy</p>",+ _render_around('<p>Old copy</p><a href="https://example.com"></a>'),+ ),+ ("unterminated_tags_around_the_block", "<p>Old copy</p>", "<p>Old copy</p>" + "<div u_body" * 1000),+ (+ "full_render_replaced_by_another",+ '<html><body class="clean-body u_body"><p>Old copy</p></body></html>',+ '<html><body class="clean-body u_body"><p>New copy</p></body></html>',+ ),+ ]+ )+ def test_stale_html_wrap_design_is_rebuilt_from_new_html(self, _name, old_html, new_html):+ inputs_schema = [{"key": "email", "type": "native_email", "required": True, "templating": "liquid"}]+ value = {+ "from": "[email protected]",+ "to": "[email protected]",+ "subject": "hi",+ "html": new_html,+ "design": build_html_wrap_design(old_html),+ }++ validated = validate_inputs(inputs_schema, {"email": {"value": value}})++ assert validated["email"]["value"]["design"] == build_html_wrap_design(new_html)++ def test_wrap_design_edited_in_the_visual_editor_is_kept(self):+ inputs_schema = [{"key": "email", "type": "native_email", "required": True, "templating": "liquid"}]+ edited_design = build_html_wrap_design("<p>Inner block</p>")+ edited_design["body"]["values"] = {"backgroundColor": "#ffffff"}+ value = {+ "from": "[email protected]",+ "to": "[email protected]",+ "subject": "hi",+ "html": '<html><body class="clean-body u_body"><p>Edited block</p></body></html>',+ "design": edited_design,+ }++ validated = validate_inputs(inputs_schema, {"email": {"value": value}})++ assert validated["email"]["value"]["design"] == edited_design++ def test_wrap_design_kept_when_the_html_is_its_unlayer_render(self):+ html = _render_around("<p>Inner block</p>")+ inputs_schema = [{"key": "email", "type": "native_email", "required": True, "templating": "liquid"}]+ design = build_html_wrap_design("<p>Inner block</p>")+ value = {"from": "[email protected]", "to": "[email protected]", "subject": "hi", "html": html, "design": design}++ validated = validate_inputs(inputs_schema, {"email": {"value": value}})++ assert validated["email"]["value"]["design"] == build_html_wrap_design("<p>Inner block</p>")+def test_html_only_email_wrap_is_deterministic(self):# Callers resend the same html-only value on every save; a fresh design each time would# register as a content change in revision and draft-diff equality checks.diff --git a/products/cdp/backend/validation.py b/products/cdp/backend/validation.pyindex 1dec2c491433..0345fd88dea2 100644--- a/products/cdp/backend/validation.py+++ b/products/cdp/backend/validation.py@@ -242,6 +242,67 @@ def build_html_wrap_design(html: str) -> dict:}+_UNLAYER_BODY_CLASS = re.compile(r"\bu_body\b")+_UNLAYER_LAYOUT_TAGS = frozenset({"!doctype", "html", "body", "table", "tbody", "tr", "td", "div"})+_TAG_NAME = re.compile(r"</?\s*([a-z!][a-z0-9]*)")+_SKIPPED_ELEMENT_ENDS = {"head": "</head>", "style": "</style>"}+++def _position_after_layout_markup(html: str, position: int) -> int:+ if html.startswith("<!--", position):+ end = html.find("-->", position + 4)+ return -1 if end == -1 else end + 3+ tag = _TAG_NAME.match(html, position)+ end = html.find(">", position)+ if tag is None or end == -1:+ return -1+ closing = _SKIPPED_ELEMENT_ENDS.get(tag.group(1))+ if closing is not None:+ end = html.find(closing, end)+ return -1 if end == -1 else end + len(closing)+ return end + 1 if tag.group(1) in _UNLAYER_LAYOUT_TAGS else -1+++def _is_only_unlayer_layout(html: str) -> bool:+ html = html.lower()+ position = 0+ while position < len(html):+ if html[position] == "<":+ position = _position_after_layout_markup(html, position)+ if position == -1:+ return False+ elif html.startswith(" ", position):+ position += 6+ elif html[position].isspace():+ position += 1+ else:+ return False+ return True+++def _is_unlayer_render_of(html: str, block_html: str) -> bool:+ start = html.find(block_html)+ if start == -1:+ return False+ around_block = html[:start] + html[start + len(block_html) :]+ return bool(_UNLAYER_BODY_CLASS.search(around_block)) and _is_only_unlayer_layout(around_block)+++def html_wrap_design_needs_refresh(html: str, design: Any) -> bool:+ if not design:+ return True+ try:+ wrapped_html = design["body"]["rows"][0]["columns"][0]["contents"][0]["values"]["html"]+ except (KeyError, IndexError, TypeError):+ return False+ return (+ isinstance(wrapped_html, str)+ and wrapped_html != html+ and design == build_html_wrap_design(wrapped_html)+ and not _is_unlayer_render_of(html, wrapped_html)+ )++def register_supported_function(name: str) -> None:PRODUCT_ASYNC_FUNCTIONS.add(name)@@ -929,7 +990,11 @@ class InputsItemSerializer(serializers.Serializer):_validate_email_sender_override(from_value, self.context)- if isinstance(value.get("html"), str) and value["html"] and not value.get("design"):+ if (+ isinstance(value.get("html"), str)+ and value["html"]+ and html_wrap_design_needs_refresh(value["html"], value.get("design"))+ ):# Programmatically authored emails often supply html without a design, which the# visual editor can't open. Wrap it so every stored email has an editable design.value = {**value, "design": build_html_wrap_design(value["html"])}diff --git a/products/messaging/backend/presentation/views/message_templates.py b/products/messaging/backend/presentation/views/message_templates.pyindex a7204319d9cc..146a98888f5a 100644--- a/products/messaging/backend/presentation/views/message_templates.py+++ b/products/messaging/backend/presentation/views/message_templates.py@@ -40,7 +40,7 @@ from posthog.models.tag import tagifyfrom posthog.permissions import is_authenticated_via_project_secret_api_key, legacy_scope_object_usedfrom posthog.rate_limit import PersonalOrProjectSecretApiKeyRateThrottle, ProjectSecretApiKeyTeamRateThrottle-from products.cdp.backend.presentation.hog_functions import build_html_wrap_design+from products.cdp.backend.presentation.hog_functions import build_html_wrap_design, html_wrap_design_needs_refreshfrom products.messaging.backend.facade.api import (UnlayerNotConfiguredError,UnlayerRenderError,@@ -353,7 +353,7 @@ class MessageTemplateSerializer(serializers.Serializer):# Programmatically authored templates often supply html without a design, which the# visual editor can't open. Wrap the html in a single custom HTML block; the stored# html stays untouched, so nothing about the sent email changes.- if email and email.get("html") and not email.get("design"):+ if email and email.get("html") and html_wrap_design_needs_refresh(email["html"], email.get("design")):email["design"] = build_html_wrap_design(email["html"])# Design-only saves get their html rendered server-side (the send path uses html# verbatim). A submitted html is trusted as-is — that's the visual editor's own export.diff --git a/products/messaging/backend/tests/api/test_message_templates.py b/products/messaging/backend/tests/api/test_message_templates.pyindex f545bfc90b47..d65dd2ec31ff 100644--- a/products/messaging/backend/tests/api/test_message_templates.py+++ b/products/messaging/backend/tests/api/test_message_templates.py@@ -10,6 +10,7 @@ from rest_framework import statusfrom posthog.models import Organization, Team+from products.cdp.backend.presentation.hog_functions import build_html_wrap_designfrom products.messaging.backend.models.message_category import MessageCategoryfrom products.messaging.backend.models.message_template import MessageTemplatefrom products.messaging.backend.unlayer import UnlayerNotConfiguredError, UnlayerRenderError@@ -248,6 +249,39 @@ class TestMessageTemplatesAPI(APIBaseTest):assert contents[0]["type"] == "html"assert contents[0]["values"]["html"] == "<p>Hello</p>"+ @parameterized.expand(+ [+ ("stale_wrap_is_rebuilt", "<p>New copy</p>", "<p>New copy</p>"),+ ("wrap_with_appended_copy_is_rebuilt", "<p>Old copy</p><p>Footer</p>", "<p>Old copy</p><p>Footer</p>"),+ (+ "wrap_with_its_unlayer_render_is_kept",+ '<html><body class="clean-body u_body"><div class="u-row"><p>Old copy</p></div></body></html>',+ "<p>Old copy</p>",+ ),+ ]+ )+ def test_update_with_html_and_wrap_design(self, _name, html, expected_block_html):+ self.message_template.content = {+ "email": {"subject": "Hi", "html": "<p>Old copy</p>", "design": build_html_wrap_design("<p>Old copy</p>")}+ }+ self.message_template.save()++ response = self.client.patch(+ f"/api/projects/{self.team.id}/messaging_templates/{self.message_template.id}/",+ data={+ "content": {+ "email": {"subject": "Hi", "html": html, "design": build_html_wrap_design("<p>Old copy</p>")}+ }+ },+ format="json",+ )++ assert response.status_code == status.HTTP_200_OK, response.json()+ self.message_template.refresh_from_db()+ email = self.message_template.content["email"]+ assert email["html"] == html+ assert email["design"] == build_html_wrap_design(expected_block_html)+def test_create_email_template_without_email_content_succeeds(self):response = self.client.post(f"/api/environments/{self.team.id}/messaging_templates/",