:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(cdp): tag every email link the UTM option should reach (#127110)
Michael Reichenbachcd2d2a724893
diff --git a/nodejs/src/cdp/cdp-services.ts b/nodejs/src/cdp/cdp-services.tsindex 79187db4a904..6e3eb56bb824 100644--- a/nodejs/src/cdp/cdp-services.ts+++ b/nodejs/src/cdp/cdp-services.ts@@ -179,6 +179,7 @@ export type CdpCoreServicesConfig = Pick<| 'EMAIL_TEAM_SENDING_CAP_MODE'| 'EMAIL_TEAM_SENDING_CAP_HOURLY_BY_TIER'| 'EMAIL_TEAM_SENDING_CAP_DAILY_BY_TIER'+ | 'CDP_EMAIL_UTM_LINK_SCANNER_TEAMS'| 'CDP_GOOGLE_ADWORDS_DEVELOPER_TOKEN'| 'CONVERSATIONS_TICKETS_JWT_SECRET'| 'CUSTOMER_ANALYTICS_ACCOUNTS_JWT_SECRET'@@ -477,6 +478,7 @@ export function createCdpCoreServices(teamEmailCapMode: parseTeamEmailCapMode(config.EMAIL_TEAM_SENDING_CAP_MODE),teamEmailTierHourlyCaps: parseTierCaps(config.EMAIL_TEAM_SENDING_CAP_HOURLY_BY_TIER),teamEmailTierDailyCaps: parseTierCaps(config.EMAIL_TEAM_SENDING_CAP_DAILY_BY_TIER),+ utmLinkScannerTeams: config.CDP_EMAIL_UTM_LINK_SCANNER_TEAMS,},deps.integrationManager,teamWorkflowsConfigService,diff --git a/nodejs/src/cdp/config.ts b/nodejs/src/cdp/config.tsindex ac96bee83881..5e70db94dc93 100644--- a/nodejs/src/cdp/config.ts+++ b/nodejs/src/cdp/config.ts@@ -191,6 +191,9 @@ export type CdpConfig = ClickhouseConfig & {// Comma-separated caps indexed by tier. Both lists must be the same length.EMAIL_TEAM_SENDING_CAP_HOURLY_BY_TIER: stringEMAIL_TEAM_SENDING_CAP_DAILY_BY_TIER: string+ // Comma-separated team ids, or "*", whose UTM link tagging runs the scanner in helpers/utm.ts.+ // Every other team runs the previous scanner in helpers/utm-legacy.ts.+ CDP_EMAIL_UTM_LINK_SCANNER_TEAMS: string// Destination migration diffingDESTINATION_MIGRATION_DIFFING_ENABLED: boolean@@ -428,6 +431,7 @@ export function getDefaultCdpConfig(): CdpConfig {EMAIL_TEAM_SENDING_CAP_MODE: 'off',EMAIL_TEAM_SENDING_CAP_HOURLY_BY_TIER: '50,200,600,2000,6000,20000,60000,200000',EMAIL_TEAM_SENDING_CAP_DAILY_BY_TIER: '100,1000,3000,10000,30000,100000,300000,1000000',+ CDP_EMAIL_UTM_LINK_SCANNER_TEAMS: '',// Destination migration diffingDESTINATION_MIGRATION_DIFFING_ENABLED: false,diff --git a/nodejs/src/cdp/services/messaging/email.service.test.ts b/nodejs/src/cdp/services/messaging/email.service.test.tsindex 836d3ca98b86..1778ab03d56c 100644--- a/nodejs/src/cdp/services/messaging/email.service.test.ts+++ b/nodejs/src/cdp/services/messaging/email.service.test.ts@@ -1989,6 +1989,33 @@ describe('EmailService', () => {)})+ it.each([+ { case: 'an empty list', list: (): string => '', fixedScanner: false },+ { case: 'a list with this team', list: (): string => `${team.id}`, fixedScanner: true },+ { case: 'a list with another team', list: (): string => `${team.id + 1}`, fixedScanner: false },+ { case: 'every team', list: (): string => '*', fixedScanner: true },+ ])('picks the UTM link scanner from $case', async ({ list, fixedScanner }) => {+ service['sesConfig'].utmLinkScannerTeams = list()+ invocation.queueParameters = createEmailParams({+ from: { integrationId: 1 },+ html: '<body><a href=https://example.com/pricing>Click me</a></body>',+ })+ invocation.hogFunction.metadata = {+ tracking_enabled: false,+ utm_tags_enabled: true,+ hog_flow_name: 'Spring sale',+ hog_flow_action_name: 'Welcome',+ }+ const result = await service.executeSendEmail(invocation)+ expect(result.error).toBeUndefined()+ const sentCommand = sendEmailSpy.mock.calls[0][0] as { input: any }+ expect(sentCommand.input.Content.Simple.Body.Html.Data).toEqual(+ fixedScanner+ ? '<body><a href="https://example.com/pricing?utm_source=posthog&utm_medium=email&utm_campaign=Spring%20sale&utm_content=Welcome">Click me</a></body>'+ : '<body><a href=https://example.com/pricing>Click me</a></body>'+ )+ })+it('falls back to the tracked configuration set when no untracked set is configured, still untracked HTML', async () => {const result = await service.executeSendEmail(invocation)expect(result.error).toBeUndefined()diff --git a/nodejs/src/cdp/services/messaging/email.service.ts b/nodejs/src/cdp/services/messaging/email.service.tsindex 7bb4a99cce01..2951dc03f412 100644--- a/nodejs/src/cdp/services/messaging/email.service.ts+++ b/nodejs/src/cdp/services/messaging/email.service.ts@@ -14,6 +14,7 @@ import {} from '~/cdp/types'import { createAddLogFunction, logEntry } from '~/cdp/utils'import { createInvocationResult } from '~/cdp/utils/invocation-utils'+import { buildIntegerMatcher } from '~/common/config/matchers'import { logger } from '~/common/utils/logger'import { TRANSACTIONAL_EMAIL_EVENT } from '../hogflows/hogflow-sensitivity'@@ -356,6 +357,7 @@ export interface EmailServiceConfig {teamEmailCapMode?: TeamEmailCapModeteamEmailTierHourlyCaps?: number[]teamEmailTierDailyCaps?: number[]+ utmLinkScannerTeams?: string}/**@@ -753,7 +755,8 @@ export class EmailService {))),- this.siteUrl+ this.siteUrl,+ buildIntegerMatcher(this.sesConfig.utmLinkScannerTeams, true)(invocation.teamId)),}: paramsWithReplyTodiff --git a/nodejs/src/cdp/services/messaging/helpers/utm-legacy.ts b/nodejs/src/cdp/services/messaging/helpers/utm-legacy.tsnew file mode 100644index 000000000000..0d26f2cf0422--- /dev/null+++ b/nodejs/src/cdp/services/messaging/helpers/utm-legacy.ts@@ -0,0 +1,118 @@+import { decodeHtmlEntitiesInHref } from '../email-tracking.service'+import type { UtmTags } from './utm'++// The previous link scanner, unchanged. Teams outside CDP_EMAIL_UTM_LINK_SCANNER_TEAMS still run it, so the+// scanner in utm.ts can roll out team by team. Delete this file once every team runs the scanner in utm.ts.++// One attribute per match, so `href` and `data-ph-no-utm` count only as attribute names, never as text+// inside another attribute's value or as part of a longer name like `data-href`.+const ATTRIBUTE_REGEX = /\s([^\s"'<>/=]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/g++const hostOf = (url: string): string | null => {+ try {+ return new URL(url).host+ } catch {+ return null+ }+}++/**+ * Appends each UTM tag the link does not already carry. Returns null when the link stays as is:+ * not an absolute http(s) URL, a link to PostHog itself (unsubscribe and preference pages), an+ * unrendered template tag, or a link that already has every tag.+ */+const addUtmTagsToUrl = (url: string, tags: UtmTags, siteHost: string | null): string | null => {+ if (url.includes('{{') || url.includes('{%')) {+ return null+ }+ let parsed: URL+ try {+ parsed = new URL(url)+ } catch {+ return null+ }+ if ((parsed.protocol !== 'http:' && parsed.protocol !== 'https:') || parsed.host === siteHost) {+ return null+ }+ const missing = Object.entries(tags).filter(([key, value]) => value && !parsed.searchParams.has(key))+ if (missing.length === 0) {+ return null+ }+ // Append to the original string rather than serializing the URL, so the rest of the link stays byte for byte.+ const hashIndex = url.indexOf('#')+ const base = hashIndex === -1 ? url : url.slice(0, hashIndex)+ const hash = hashIndex === -1 ? '' : url.slice(hashIndex)+ const separator = !base.includes('?') ? '?' : base.endsWith('?') || base.endsWith('&') ? '' : '&'+ const query = missing.map(([key, value]) => `${key}=${encodeURIComponent(value)}`).join('&')+ return `${base}${separator}${query}${hash}`+}++/** Index of the `>` that closes the tag opened before `from`, skipping quoted values, or -1 if it never closes. */+const findTagEnd = (html: string, from: number): number => {+ let quote: string | null = null+ for (let i = from; i < html.length; i++) {+ const char = html[i]+ if (quote) {+ if (char === quote) {+ quote = null+ }+ } else if (char === '"' || char === "'") {+ quote = char+ } else if (char === '>') {+ return i+ }+ }+ return -1+}++const tagAnchor = (anchor: string, tags: UtmTags, siteHost: string | null): string => {+ const attributes = [...anchor.matchAll(ATTRIBUTE_REGEX)]+ if (attributes.some((attribute) => attribute[1].toLowerCase() === 'data-ph-no-utm')) {+ return anchor+ }+ const href = attributes.find((attribute) => attribute[1].toLowerCase() === 'href')+ const value = href ? (href[2] ?? href[3]) : undefined+ if (!href || value === undefined) {+ return anchor+ }+ const tagged = addUtmTagsToUrl(decodeHtmlEntitiesInHref(value), tags, siteHost)+ if (tagged === null) {+ return anchor+ }+ const quote = href[2] !== undefined ? '"' : "'"+ const encoded = tagged+ .replace(/&/g, '&')+ .replace(quote === '"' ? /"/g : /'/g, quote === '"' ? '"' : ''')+ const start = href.index! + href[0].indexOf('=')+ return anchor.slice(0, start) + `=${quote}${encoded}${quote}` + anchor.slice(href.index! + href[0].length)+}++// One forward pass rather than a regex, so a body full of unclosed `<a` tags costs linear time instead+// of rescanning the rest of the email for each one and stalling the worker.+export const addUtmTagsToEmailLegacy = (html: string, tags: UtmTags, siteUrl: string): string => {+ const siteHost = hostOf(siteUrl)+ const lower = html.toLowerCase()+ let output = ''+ let cursor = 0+ while (cursor < html.length) {+ const start = lower.indexOf('<a', cursor)+ if (start === -1) {+ break+ }+ const after = lower[start + 2]+ if (after !== undefined && !/[\s>/]/.test(after)) {+ // `<abbr>`, `<area>` and other tags that only start with "a".+ output += html.slice(cursor, start + 2)+ cursor = start + 2+ continue+ }+ const end = findTagEnd(html, start + 2)+ if (end === -1) {+ // The tag never closes, so it is not a link, and the rest of the email is left as is.+ break+ }+ output += html.slice(cursor, start) + tagAnchor(html.slice(start, end + 1), tags, siteHost)+ cursor = end + 1+ }+ return output + html.slice(cursor)+}diff --git a/nodejs/src/cdp/services/messaging/helpers/utm.test.ts b/nodejs/src/cdp/services/messaging/helpers/utm.test.tsindex c54085f50584..50eaec6bf639 100644--- a/nodejs/src/cdp/services/messaging/helpers/utm.test.ts+++ b/nodejs/src/cdp/services/messaging/helpers/utm.test.ts@@ -5,7 +5,7 @@ const SITE_URL = 'https://us.posthog.com'const QUERY = 'utm_source=posthog&utm_medium=email&utm_campaign=Spring%20sale'describe('addUtmTagsToEmail', () => {- it.each([+ it.each<{ case: string; html: string; expected: string; fixedScanner?: boolean }>([{case: 'a plain link',html: '<a href="https://example.com/pricing">x</a>',@@ -72,8 +72,50 @@ describe('addUtmTagsToEmail', () => {html: '<a href="https://example.com/{{ person.id }}">x</a>',expected: '<a href="https://example.com/{{ person.id }}">x</a>',},- ])('tags $case', ({ html, expected }) => {- expect(addUtmTagsToEmail(html, TAGS, SITE_URL)).toEqual(expected)+ {+ case: 'an unquoted href',+ html: '<a href=https://example.com/pricing>x</a>',+ expected: `<a href="https://example.com/pricing?${QUERY}">x</a>`,+ },+ {+ case: 'an unquoted href with a query',+ html: '<a href=https://example.com/pricing?plan=pro>x</a>',+ expected: `<a href="https://example.com/pricing?plan=pro&${QUERY}">x</a>`,+ },+ {+ case: 'a quoted and an unquoted href with the fixed scanner on for the team',+ html: '<a href="https://example.com/a">x</a><a href=https://example.com/b>y</a>',+ expected: `<a href="https://example.com/a?${QUERY}">x</a><a href="https://example.com/b?${QUERY}">y</a>`,+ fixedScanner: true,+ },+ {+ case: "a quoted and an unquoted href with the fixed scanner off for the team, which keeps master's scanner",+ html: '<a href="https://example.com/a">x</a><a href=https://example.com/b>y</a>',+ expected: `<a href="https://example.com/a?${QUERY}">x</a><a href=https://example.com/b>y</a>`,+ fixedScanner: false,+ },+ {+ case: 'an unquoted href with a non-breaking space, which HTML keeps in the value',+ html: '<a href=https://example.com/summer sale>x</a>',+ expected: `<a href="https://example.com/summer sale?${QUERY}">x</a>`,+ },+ {+ case: 'an href with a trailing space',+ html: '<a href="https://example.com/pricing ">x</a>',+ expected: `<a href="https://example.com/pricing?${QUERY}">x</a>`,+ },+ {+ case: 'an href ending in a non-breaking space, which is part of the URL',+ html: '<a href="https://example.com/pricing ">x</a>',+ expected: `<a href="https://example.com/pricing\u00a0?${QUERY}">x</a>`,+ },+ {+ case: 'links after text that grows when lowercased',+ html: '<p>İstanbul, İzmir, İzmit</p><a data-ph-no-utm href="https://example.com/a">x</a><a href="https://example.com/b">y</a>',+ expected: `<p>İstanbul, İzmir, İzmit</p><a data-ph-no-utm href="https://example.com/a">x</a><a href="https://example.com/b?${QUERY}">y</a>`,+ },+ ])('tags $case', ({ html, expected, fixedScanner = true }) => {+ expect(addUtmTagsToEmail(html, TAGS, SITE_URL, fixedScanner)).toEqual(expected)})it.each([diff --git a/nodejs/src/cdp/services/messaging/helpers/utm.ts b/nodejs/src/cdp/services/messaging/helpers/utm.tsindex ac341e164bcd..98fa3581a9e0 100644--- a/nodejs/src/cdp/services/messaging/helpers/utm.ts+++ b/nodejs/src/cdp/services/messaging/helpers/utm.ts@@ -2,6 +2,7 @@ import { HogFunctionInvocationGlobalsWithInputs } from '~/cdp/types'import { LiquidRenderer } from '~/cdp/utils/liquid'import { decodeHtmlEntitiesInHref } from '../email-tracking.service'+import { addUtmTagsToEmailLegacy } from './utm-legacy'export type UtmTags = {utm_source: string@@ -14,7 +15,7 @@ const UTM_KEYS = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_content'] as// One attribute per match, so `href` and `data-ph-no-utm` count only as attribute names, never as text// inside another attribute's value or as part of a longer name like `data-href`.-const ATTRIBUTE_REGEX = /\s([^\s"'<>/=]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/g+const ATTRIBUTE_REGEX = /\s([^\s"'<>/=]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\t\n\f\r >]+)))?/g/*** Renders each custom value as Liquid with the recipient's data. A value that fails to render is@@ -65,12 +66,27 @@ const hostOf = (url: string): string | null => {}}+const isStrippedByUrlParser = (char: string): boolean => char <= ' '++const trimUrlWhitespace = (href: string): string => {+ let start = 0+ let end = href.length+ while (start < end && isStrippedByUrlParser(href[start])) {+ start+++ }+ while (end > start && isStrippedByUrlParser(href[end - 1])) {+ end--+ }+ return href.slice(start, end)+}+/*** Appends each UTM tag the link does not already carry. Returns null when the link stays as is:* not an absolute http(s) URL, a link to PostHog itself (unsubscribe and preference pages), an* unrendered template tag, or a link that already has every tag.*/-export const addUtmTagsToUrl = (url: string, tags: UtmTags, siteHost: string | null): string | null => {+export const addUtmTagsToUrl = (href: string, tags: UtmTags, siteHost: string | null): string | null => {+ const url = trimUrlWhitespace(href)if (url.includes('{{') || url.includes('{%')) {return null}@@ -120,7 +136,7 @@ const tagAnchor = (anchor: string, tags: UtmTags, siteHost: string | null): strireturn anchor}const href = attributes.find((attribute) => attribute[1].toLowerCase() === 'href')- const value = href ? (href[2] ?? href[3]) : undefined+ const value = href ? (href[2] ?? href[3] ?? href[4]) : undefinedif (!href || value === undefined) {return anchor}@@ -128,7 +144,7 @@ const tagAnchor = (anchor: string, tags: UtmTags, siteHost: string | null): striif (tagged === null) {return anchor}- const quote = href[2] !== undefined ? '"' : "'"+ const quote = href[3] !== undefined ? "'" : '"'const encoded = tagged.replace(/&/g, '&').replace(quote === '"' ? /"/g : /'/g, quote === '"' ? '"' : ''')@@ -136,19 +152,27 @@ const tagAnchor = (anchor: string, tags: UtmTags, siteHost: string | null): strireturn anchor.slice(0, start) + `=${quote}${encoded}${quote}` + anchor.slice(href.index! + href[0].length)}+const findAnchorStart = (html: string, from: number): number => {+ const anchorStart = /<a/gi+ anchorStart.lastIndex = from+ return anchorStart.exec(html)?.index ?? -1+}+// One forward pass rather than a regex, so a body full of unclosed `<a` tags costs linear time instead// of rescanning the rest of the email for each one and stalling the worker.-export const addUtmTagsToEmail = (html: string, tags: UtmTags, siteUrl: string): string => {+export const addUtmTagsToEmail = (html: string, tags: UtmTags, siteUrl: string, fixedScanner: boolean): string => {+ if (!fixedScanner) {+ return addUtmTagsToEmailLegacy(html, tags, siteUrl)+ }const siteHost = hostOf(siteUrl)- const lower = html.toLowerCase()let output = ''let cursor = 0while (cursor < html.length) {- const start = lower.indexOf('<a', cursor)+ const start = findAnchorStart(html, cursor)if (start === -1) {break}- const after = lower[start + 2]+ const after = html[start + 2]if (after !== undefined && !/[\s>/]/.test(after)) {// `<abbr>`, `<area>` and other tags that only start with "a".output += html.slice(cursor, start + 2)