Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(cdp): tag every email link the UTM option should reach (#127110)

Michael Reichenbachcd2d2a724893
diff --git a/nodejs/src/cdp/cdp-services.ts b/nodejs/src/cdp/cdp-services.ts
index 79187db4a904..6e3eb56bb824 100644
--- a/nodejs/src/cdp/cdp-services.ts
+++ b/nodejs/src/cdp/cdp-services.ts
@@ -179,6 +179,7 @@ export type CdpCoreServicesConfig = Pick<
| 'EMAIL_TEAM_SENDING_CAP_MODE'
| 'EMAIL_TEAM_SENDING_CAP_HOURLY_BY_TIER'
| 'EMAIL_TEAM_SENDING_CAP_DAILY_BY_TIER'
+ | 'CDP_EMAIL_UTM_LINK_SCANNER_TEAMS'
| 'CDP_GOOGLE_ADWORDS_DEVELOPER_TOKEN'
| 'CONVERSATIONS_TICKETS_JWT_SECRET'
| 'CUSTOMER_ANALYTICS_ACCOUNTS_JWT_SECRET'
@@ -477,6 +478,7 @@ export function createCdpCoreServices(
teamEmailCapMode: parseTeamEmailCapMode(config.EMAIL_TEAM_SENDING_CAP_MODE),
teamEmailTierHourlyCaps: parseTierCaps(config.EMAIL_TEAM_SENDING_CAP_HOURLY_BY_TIER),
teamEmailTierDailyCaps: parseTierCaps(config.EMAIL_TEAM_SENDING_CAP_DAILY_BY_TIER),
+ utmLinkScannerTeams: config.CDP_EMAIL_UTM_LINK_SCANNER_TEAMS,
},
deps.integrationManager,
teamWorkflowsConfigService,
diff --git a/nodejs/src/cdp/config.ts b/nodejs/src/cdp/config.ts
index ac96bee83881..5e70db94dc93 100644
--- a/nodejs/src/cdp/config.ts
+++ b/nodejs/src/cdp/config.ts
@@ -191,6 +191,9 @@ export type CdpConfig = ClickhouseConfig & {
// Comma-separated caps indexed by tier. Both lists must be the same length.
EMAIL_TEAM_SENDING_CAP_HOURLY_BY_TIER: string
EMAIL_TEAM_SENDING_CAP_DAILY_BY_TIER: string
+ // Comma-separated team ids, or "*", whose UTM link tagging runs the scanner in helpers/utm.ts.
+ // Every other team runs the previous scanner in helpers/utm-legacy.ts.
+ CDP_EMAIL_UTM_LINK_SCANNER_TEAMS: string
// Destination migration diffing
DESTINATION_MIGRATION_DIFFING_ENABLED: boolean
@@ -428,6 +431,7 @@ export function getDefaultCdpConfig(): CdpConfig {
EMAIL_TEAM_SENDING_CAP_MODE: 'off',
EMAIL_TEAM_SENDING_CAP_HOURLY_BY_TIER: '50,200,600,2000,6000,20000,60000,200000',
EMAIL_TEAM_SENDING_CAP_DAILY_BY_TIER: '100,1000,3000,10000,30000,100000,300000,1000000',
+ CDP_EMAIL_UTM_LINK_SCANNER_TEAMS: '',
// Destination migration diffing
DESTINATION_MIGRATION_DIFFING_ENABLED: false,
diff --git a/nodejs/src/cdp/services/messaging/email.service.test.ts b/nodejs/src/cdp/services/messaging/email.service.test.ts
index 836d3ca98b86..1778ab03d56c 100644
--- a/nodejs/src/cdp/services/messaging/email.service.test.ts
+++ b/nodejs/src/cdp/services/messaging/email.service.test.ts
@@ -1989,6 +1989,33 @@ describe('EmailService', () => {
)
})
+ it.each([
+ { case: 'an empty list', list: (): string => '', fixedScanner: false },
+ { case: 'a list with this team', list: (): string => `${team.id}`, fixedScanner: true },
+ { case: 'a list with another team', list: (): string => `${team.id + 1}`, fixedScanner: false },
+ { case: 'every team', list: (): string => '*', fixedScanner: true },
+ ])('picks the UTM link scanner from $case', async ({ list, fixedScanner }) => {
+ service['sesConfig'].utmLinkScannerTeams = list()
+ invocation.queueParameters = createEmailParams({
+ from: { integrationId: 1 },
+ html: '<body><a href=https://example.com/pricing>Click me</a></body>',
+ })
+ invocation.hogFunction.metadata = {
+ tracking_enabled: false,
+ utm_tags_enabled: true,
+ hog_flow_name: 'Spring sale',
+ hog_flow_action_name: 'Welcome',
+ }
+ const result = await service.executeSendEmail(invocation)
+ expect(result.error).toBeUndefined()
+ const sentCommand = sendEmailSpy.mock.calls[0][0] as { input: any }
+ expect(sentCommand.input.Content.Simple.Body.Html.Data).toEqual(
+ fixedScanner
+ ? '<body><a href="https://example.com/pricing?utm_source=posthog&amp;utm_medium=email&amp;utm_campaign=Spring%20sale&amp;utm_content=Welcome">Click me</a></body>'
+ : '<body><a href=https://example.com/pricing>Click me</a></body>'
+ )
+ })
+
it('falls back to the tracked configuration set when no untracked set is configured, still untracked HTML', async () => {
const result = await service.executeSendEmail(invocation)
expect(result.error).toBeUndefined()
diff --git a/nodejs/src/cdp/services/messaging/email.service.ts b/nodejs/src/cdp/services/messaging/email.service.ts
index 7bb4a99cce01..2951dc03f412 100644
--- a/nodejs/src/cdp/services/messaging/email.service.ts
+++ b/nodejs/src/cdp/services/messaging/email.service.ts
@@ -14,6 +14,7 @@ import {
} from '~/cdp/types'
import { createAddLogFunction, logEntry } from '~/cdp/utils'
import { createInvocationResult } from '~/cdp/utils/invocation-utils'
+import { buildIntegerMatcher } from '~/common/config/matchers'
import { logger } from '~/common/utils/logger'
import { TRANSACTIONAL_EMAIL_EVENT } from '../hogflows/hogflow-sensitivity'
@@ -356,6 +357,7 @@ export interface EmailServiceConfig {
teamEmailCapMode?: TeamEmailCapMode
teamEmailTierHourlyCaps?: number[]
teamEmailTierDailyCaps?: number[]
+ utmLinkScannerTeams?: string
}
/**
@@ -753,7 +755,8 @@ export class EmailService {
)
)
),
- this.siteUrl
+ this.siteUrl,
+ buildIntegerMatcher(this.sesConfig.utmLinkScannerTeams, true)(invocation.teamId)
),
}
: paramsWithReplyTo
diff --git a/nodejs/src/cdp/services/messaging/helpers/utm-legacy.ts b/nodejs/src/cdp/services/messaging/helpers/utm-legacy.ts
new file mode 100644
index 000000000000..0d26f2cf0422
--- /dev/null
+++ b/nodejs/src/cdp/services/messaging/helpers/utm-legacy.ts
@@ -0,0 +1,118 @@
+import { decodeHtmlEntitiesInHref } from '../email-tracking.service'
+import type { UtmTags } from './utm'
+
+// The previous link scanner, unchanged. Teams outside CDP_EMAIL_UTM_LINK_SCANNER_TEAMS still run it, so the
+// scanner in utm.ts can roll out team by team. Delete this file once every team runs the scanner in utm.ts.
+
+// One attribute per match, so `href` and `data-ph-no-utm` count only as attribute names, never as text
+// inside another attribute's value or as part of a longer name like `data-href`.
+const ATTRIBUTE_REGEX = /\s([^\s"'<>/=]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/g
+
+const hostOf = (url: string): string | null => {
+ try {
+ return new URL(url).host
+ } catch {
+ return null
+ }
+}
+
+/**
+ * Appends each UTM tag the link does not already carry. Returns null when the link stays as is:
+ * not an absolute http(s) URL, a link to PostHog itself (unsubscribe and preference pages), an
+ * unrendered template tag, or a link that already has every tag.
+ */
+const addUtmTagsToUrl = (url: string, tags: UtmTags, siteHost: string | null): string | null => {
+ if (url.includes('{{') || url.includes('{%')) {
+ return null
+ }
+ let parsed: URL
+ try {
+ parsed = new URL(url)
+ } catch {
+ return null
+ }
+ if ((parsed.protocol !== 'http:' && parsed.protocol !== 'https:') || parsed.host === siteHost) {
+ return null
+ }
+ const missing = Object.entries(tags).filter(([key, value]) => value && !parsed.searchParams.has(key))
+ if (missing.length === 0) {
+ return null
+ }
+ // Append to the original string rather than serializing the URL, so the rest of the link stays byte for byte.
+ const hashIndex = url.indexOf('#')
+ const base = hashIndex === -1 ? url : url.slice(0, hashIndex)
+ const hash = hashIndex === -1 ? '' : url.slice(hashIndex)
+ const separator = !base.includes('?') ? '?' : base.endsWith('?') || base.endsWith('&') ? '' : '&'
+ const query = missing.map(([key, value]) => `${key}=${encodeURIComponent(value)}`).join('&')
+ return `${base}${separator}${query}${hash}`
+}
+
+/** Index of the `>` that closes the tag opened before `from`, skipping quoted values, or -1 if it never closes. */
+const findTagEnd = (html: string, from: number): number => {
+ let quote: string | null = null
+ for (let i = from; i < html.length; i++) {
+ const char = html[i]
+ if (quote) {
+ if (char === quote) {
+ quote = null
+ }
+ } else if (char === '"' || char === "'") {
+ quote = char
+ } else if (char === '>') {
+ return i
+ }
+ }
+ return -1
+}
+
+const tagAnchor = (anchor: string, tags: UtmTags, siteHost: string | null): string => {
+ const attributes = [...anchor.matchAll(ATTRIBUTE_REGEX)]
+ if (attributes.some((attribute) => attribute[1].toLowerCase() === 'data-ph-no-utm')) {
+ return anchor
+ }
+ const href = attributes.find((attribute) => attribute[1].toLowerCase() === 'href')
+ const value = href ? (href[2] ?? href[3]) : undefined
+ if (!href || value === undefined) {
+ return anchor
+ }
+ const tagged = addUtmTagsToUrl(decodeHtmlEntitiesInHref(value), tags, siteHost)
+ if (tagged === null) {
+ return anchor
+ }
+ const quote = href[2] !== undefined ? '"' : "'"
+ const encoded = tagged
+ .replace(/&/g, '&amp;')
+ .replace(quote === '"' ? /"/g : /'/g, quote === '"' ? '&quot;' : '&#39;')
+ const start = href.index! + href[0].indexOf('=')
+ return anchor.slice(0, start) + `=${quote}${encoded}${quote}` + anchor.slice(href.index! + href[0].length)
+}
+
+// One forward pass rather than a regex, so a body full of unclosed `<a` tags costs linear time instead
+// of rescanning the rest of the email for each one and stalling the worker.
+export const addUtmTagsToEmailLegacy = (html: string, tags: UtmTags, siteUrl: string): string => {
+ const siteHost = hostOf(siteUrl)
+ const lower = html.toLowerCase()
+ let output = ''
+ let cursor = 0
+ while (cursor < html.length) {
+ const start = lower.indexOf('<a', cursor)
+ if (start === -1) {
+ break
+ }
+ const after = lower[start + 2]
+ if (after !== undefined && !/[\s>/]/.test(after)) {
+ // `<abbr>`, `<area>` and other tags that only start with "a".
+ output += html.slice(cursor, start + 2)
+ cursor = start + 2
+ continue
+ }
+ const end = findTagEnd(html, start + 2)
+ if (end === -1) {
+ // The tag never closes, so it is not a link, and the rest of the email is left as is.
+ break
+ }
+ output += html.slice(cursor, start) + tagAnchor(html.slice(start, end + 1), tags, siteHost)
+ cursor = end + 1
+ }
+ return output + html.slice(cursor)
+}
diff --git a/nodejs/src/cdp/services/messaging/helpers/utm.test.ts b/nodejs/src/cdp/services/messaging/helpers/utm.test.ts
index c54085f50584..50eaec6bf639 100644
--- a/nodejs/src/cdp/services/messaging/helpers/utm.test.ts
+++ b/nodejs/src/cdp/services/messaging/helpers/utm.test.ts
@@ -5,7 +5,7 @@ const SITE_URL = 'https://us.posthog.com'
const QUERY = 'utm_source=posthog&amp;utm_medium=email&amp;utm_campaign=Spring%20sale'
describe('addUtmTagsToEmail', () => {
- it.each([
+ it.each<{ case: string; html: string; expected: string; fixedScanner?: boolean }>([
{
case: 'a plain link',
html: '<a href="https://example.com/pricing">x</a>',
@@ -72,8 +72,50 @@ describe('addUtmTagsToEmail', () => {
html: '<a href="https://example.com/{{ person.id }}">x</a>',
expected: '<a href="https://example.com/{{ person.id }}">x</a>',
},
- ])('tags $case', ({ html, expected }) => {
- expect(addUtmTagsToEmail(html, TAGS, SITE_URL)).toEqual(expected)
+ {
+ case: 'an unquoted href',
+ html: '<a href=https://example.com/pricing>x</a>',
+ expected: `<a href="https://example.com/pricing?${QUERY}">x</a>`,
+ },
+ {
+ case: 'an unquoted href with a query',
+ html: '<a href=https://example.com/pricing?plan=pro>x</a>',
+ expected: `<a href="https://example.com/pricing?plan=pro&amp;${QUERY}">x</a>`,
+ },
+ {
+ case: 'a quoted and an unquoted href with the fixed scanner on for the team',
+ html: '<a href="https://example.com/a">x</a><a href=https://example.com/b>y</a>',
+ expected: `<a href="https://example.com/a?${QUERY}">x</a><a href="https://example.com/b?${QUERY}">y</a>`,
+ fixedScanner: true,
+ },
+ {
+ case: "a quoted and an unquoted href with the fixed scanner off for the team, which keeps master's scanner",
+ html: '<a href="https://example.com/a">x</a><a href=https://example.com/b>y</a>',
+ expected: `<a href="https://example.com/a?${QUERY}">x</a><a href=https://example.com/b>y</a>`,
+ fixedScanner: false,
+ },
+ {
+ case: 'an unquoted href with a non-breaking space, which HTML keeps in the value',
+ html: '<a href=https://example.com/summer sale>x</a>',
+ expected: `<a href="https://example.com/summer sale?${QUERY}">x</a>`,
+ },
+ {
+ case: 'an href with a trailing space',
+ html: '<a href="https://example.com/pricing ">x</a>',
+ expected: `<a href="https://example.com/pricing?${QUERY}">x</a>`,
+ },
+ {
+ case: 'an href ending in a non-breaking space, which is part of the URL',
+ html: '<a href="https://example.com/pricing&#160;">x</a>',
+ expected: `<a href="https://example.com/pricing\u00a0?${QUERY}">x</a>`,
+ },
+ {
+ case: 'links after text that grows when lowercased',
+ html: '<p>İstanbul, İzmir, İzmit</p><a data-ph-no-utm href="https://example.com/a">x</a><a href="https://example.com/b">y</a>',
+ expected: `<p>İstanbul, İzmir, İzmit</p><a data-ph-no-utm href="https://example.com/a">x</a><a href="https://example.com/b?${QUERY}">y</a>`,
+ },
+ ])('tags $case', ({ html, expected, fixedScanner = true }) => {
+ expect(addUtmTagsToEmail(html, TAGS, SITE_URL, fixedScanner)).toEqual(expected)
})
it.each([
diff --git a/nodejs/src/cdp/services/messaging/helpers/utm.ts b/nodejs/src/cdp/services/messaging/helpers/utm.ts
index ac341e164bcd..98fa3581a9e0 100644
--- a/nodejs/src/cdp/services/messaging/helpers/utm.ts
+++ b/nodejs/src/cdp/services/messaging/helpers/utm.ts
@@ -2,6 +2,7 @@ import { HogFunctionInvocationGlobalsWithInputs } from '~/cdp/types'
import { LiquidRenderer } from '~/cdp/utils/liquid'
import { decodeHtmlEntitiesInHref } from '../email-tracking.service'
+import { addUtmTagsToEmailLegacy } from './utm-legacy'
export type UtmTags = {
utm_source: string
@@ -14,7 +15,7 @@ const UTM_KEYS = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_content'] as
// One attribute per match, so `href` and `data-ph-no-utm` count only as attribute names, never as text
// inside another attribute's value or as part of a longer name like `data-href`.
-const ATTRIBUTE_REGEX = /\s([^\s"'<>/=]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/g
+const ATTRIBUTE_REGEX = /\s([^\s"'<>/=]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\t\n\f\r >]+)))?/g
/**
* Renders each custom value as Liquid with the recipient's data. A value that fails to render is
@@ -65,12 +66,27 @@ const hostOf = (url: string): string | null => {
}
}
+const isStrippedByUrlParser = (char: string): boolean => char <= ' '
+
+const trimUrlWhitespace = (href: string): string => {
+ let start = 0
+ let end = href.length
+ while (start < end && isStrippedByUrlParser(href[start])) {
+ start++
+ }
+ while (end > start && isStrippedByUrlParser(href[end - 1])) {
+ end--
+ }
+ return href.slice(start, end)
+}
+
/**
* Appends each UTM tag the link does not already carry. Returns null when the link stays as is:
* not an absolute http(s) URL, a link to PostHog itself (unsubscribe and preference pages), an
* unrendered template tag, or a link that already has every tag.
*/
-export const addUtmTagsToUrl = (url: string, tags: UtmTags, siteHost: string | null): string | null => {
+export const addUtmTagsToUrl = (href: string, tags: UtmTags, siteHost: string | null): string | null => {
+ const url = trimUrlWhitespace(href)
if (url.includes('{{') || url.includes('{%')) {
return null
}
@@ -120,7 +136,7 @@ const tagAnchor = (anchor: string, tags: UtmTags, siteHost: string | null): stri
return anchor
}
const href = attributes.find((attribute) => attribute[1].toLowerCase() === 'href')
- const value = href ? (href[2] ?? href[3]) : undefined
+ const value = href ? (href[2] ?? href[3] ?? href[4]) : undefined
if (!href || value === undefined) {
return anchor
}
@@ -128,7 +144,7 @@ const tagAnchor = (anchor: string, tags: UtmTags, siteHost: string | null): stri
if (tagged === null) {
return anchor
}
- const quote = href[2] !== undefined ? '"' : "'"
+ const quote = href[3] !== undefined ? "'" : '"'
const encoded = tagged
.replace(/&/g, '&amp;')
.replace(quote === '"' ? /"/g : /'/g, quote === '"' ? '&quot;' : '&#39;')
@@ -136,19 +152,27 @@ const tagAnchor = (anchor: string, tags: UtmTags, siteHost: string | null): stri
return anchor.slice(0, start) + `=${quote}${encoded}${quote}` + anchor.slice(href.index! + href[0].length)
}
+const findAnchorStart = (html: string, from: number): number => {
+ const anchorStart = /<a/gi
+ anchorStart.lastIndex = from
+ return anchorStart.exec(html)?.index ?? -1
+}
+
// One forward pass rather than a regex, so a body full of unclosed `<a` tags costs linear time instead
// of rescanning the rest of the email for each one and stalling the worker.
-export const addUtmTagsToEmail = (html: string, tags: UtmTags, siteUrl: string): string => {
+export const addUtmTagsToEmail = (html: string, tags: UtmTags, siteUrl: string, fixedScanner: boolean): string => {
+ if (!fixedScanner) {
+ return addUtmTagsToEmailLegacy(html, tags, siteUrl)
+ }
const siteHost = hostOf(siteUrl)
- const lower = html.toLowerCase()
let output = ''
let cursor = 0
while (cursor < html.length) {
- const start = lower.indexOf('<a', cursor)
+ const start = findAnchorStart(html, cursor)
if (start === -1) {
break
}
- const after = lower[start + 2]
+ const after = html[start + 2]
if (after !== undefined && !/[\s>/]/.test(after)) {
// `<abbr>`, `<area>` and other tags that only start with "a".
output += html.slice(cursor, start + 2)