Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

53.8 KB
Raw Blame
import { createMockJobQueue } from '~/tests/helpers/mocks/job-queue.mock'
import { mockProducerObserver } from '~/tests/helpers/mocks/producer.mock'
import { mockFetch, mockInternalFetch } from '~/tests/helpers/mocks/request.mock'

import { Server } from 'http'
import { register } from 'prom-client'
import supertest from 'supertest'
import express from 'ultimate-express'

import { FixtureHogFlowBuilder } from '~/cdp/_tests/builders/hogflow.builder'
import { createExampleInvocation, insertHogFunction, insertIntegration } from '~/cdp/_tests/fixtures'
import { hideHogFlowsFromReplica, insertHogFlow } from '~/cdp/_tests/fixtures-hogflows'
import { CdpApi } from '~/cdp/cdp-api'
import { CyclotronJobInvocationHogFunction, HogFunctionType } from '~/cdp/types'
import { setupExpressApp } from '~/common/api/router'
import { defaultConfig } from '~/common/config/config'
import { KAFKA_APP_METRICS_2, KAFKA_EMAIL_STATUS, KAFKA_LOG_ENTRIES } from '~/common/config/kafka-topics'
import { closeHub, createHub } from '~/common/utils/db/hub'
import { PostgresUse } from '~/common/utils/db/postgres'
import * as envUtils from '~/common/utils/env-utils'
import { parseJSON } from '~/common/utils/json-parse'
import { UUIDT } from '~/common/utils/utils'
import { createCdpConsumerDeps } from '~/tests/helpers/cdp'
import { waitForExpect } from '~/tests/helpers/expectations'
import { createTestTeamFixture } from '~/tests/helpers/sql'

import { Hub, Team } from '../../../types'
import { EmailSuppressionService } from './email-suppression.service'
import {
    METRIC_NAME_TO_EVENT_NAME,
    PIXEL_GIF,
    UNSIGNED_REDIRECT_SUNSET,
    addTrackingToEmail,
    decodeHtmlEntitiesInHref,
    resolveEmailEngagementDistinctId,
} from './email-tracking.service'
import { SesWebhookHandler } from './helpers/ses'
import { EmailTrackingCodeSigner, TRACKING_CODE_HEADER_NAME } from './helpers/tracking-code'

describe('EmailTrackingService', () => {
    let hub: Hub
    let team: Team

    const signer = new EmailTrackingCodeSigner(defaultConfig.ENCRYPTION_SALT_KEYS, defaultConfig.CDP_EMAIL_TRACKING_URL)

    beforeEach(async () => {
        hub = await createHub({})
        team = (await createTestTeamFixture(hub.postgres)).team

        mockFetch.mockClear()
        mockInternalFetch.mockClear()
        mockProducerObserver.resetKafkaProducer()
    })

    afterEach(async () => {
        await closeHub(hub)
    })

    describe('addTrackingToEmail', () => {
        const invocation = {
            functionId: 'fn-1',
            id: 'inv-1',
            teamId: 1,
        } as any

        const extractTarget = (html: string): string => {
            const match = html.match(/href="[^"]*target=([^"&]+)/)
            if (!match) {
                throw new Error(`no tracking href in: ${html}`)
            }
            return decodeURIComponent(match[1])
        }

        it.each([
            [
                'named entity &',
                '<body><a href="https://example.com/?foo=bar&amp;baz=bop">x</a></body>',
                'https://example.com/?foo=bar&baz=bop',
            ],
            [
                'decimal numeric entity &#38;',
                '<body><a href="https://example.com/?a=1&#38;b=2">x</a></body>',
                'https://example.com/?a=1&b=2',
            ],
            [
                'hex numeric entity &#x26;',
                '<body><a href="https://example.com/?a=1&#x26;b=2">x</a></body>',
                'https://example.com/?a=1&b=2',
            ],
            [
                'plain URL with no entities',
                '<body><a href="https://example.com/path">x</a></body>',
                'https://example.com/path',
            ],
        ])('decodes %s in the redirect target', (_name, html, expected) => {
            expect(extractTarget(addTrackingToEmail(html, invocation, signer))).toBe(expected)
        })

        // The pixel and redirect URLs are the only tracking carrier for a provider whose opens and
        // clicks we record ourselves, so a version missing here means those engagement metrics never
        // split by version however the send path is configured.
        it('carries the sending version into the pixel and redirect codes', () => {
            const flowInvocation = { ...invocation, hogFlow: { id: 'flow-1', version: 7 } } as any
            const html = '<body><a href="https://example.com/">x</a></body>'

            const out = addTrackingToEmail(html, flowInvocation, signer)
            const codes = [...out.matchAll(/ph_id=([A-Za-z0-9_.-]+)/g)].map((match) => match[1])

            expect(codes).toHaveLength(2) // one redirect, one pixel
            for (const code of codes) {
                expect(signer.parse(code)?.workflowVersion).toBe(7)
            }
        })

        it('mints no version for a hog function send, which has no workflow', () => {
            const out = addTrackingToEmail('<body><a href="https://example.com/">x</a></body>', invocation, signer)
            const code = out.match(/ph_id=([A-Za-z0-9_.-]+)/)![1]

            expect(signer.parse(code)?.workflowVersion).toBeUndefined()
        })

        it('skips literal javascript: hrefs', () => {
            const html = '<body><a href="javascript:alert(1)">x</a></body>'
            const out = addTrackingToEmail(html, invocation, signer)
            expect(out).toContain('href="javascript:alert(1)"')
            expect(out).not.toContain('target=')
        })

        it('skips entity-encoded javascript: hrefs after decoding', () => {
            const html = '<body><a href="java&#x73;cript:alert(1)">x</a></body>'
            const out = addTrackingToEmail(html, invocation, signer)
            expect(out).toContain('href="java&#x73;cript:alert(1)"')
            expect(out).not.toContain('target=')
        })

        const sensitiveInvocation = { ...invocation, hogFunction: { metadata: { sensitive: true } } }

        it.each([
            ['clicktracking="off"', '<body><a href="https://example.com" clicktracking="off">x</a></body>', invocation],
            ['data-ph-no-track', '<body><a href="https://example.com" data-ph-no-track>x</a></body>', invocation],
            ['a sensitive template', '<body><a href="https://example.com">x</a></body>', sensitiveInvocation],
        ])('leaves the href untouched when the anchor opts out via %s', (_name, html, inv) => {
            const out = addTrackingToEmail(html, inv, signer)
            expect(out).toContain('href="https://example.com"')
            expect(out).not.toContain('target=')
        })

        it('still wraps the link when the opt-out marker is on a child, not the anchor tag', () => {
            const html = '<body><a href="https://example.com"><span data-ph-no-track>x</span></a></body>'
            const out = addTrackingToEmail(html, invocation, signer)
            expect(out).toContain('target=')
        })

        describe('ses mode', () => {
            const sesTrack = (html: string): string => addTrackingToEmail(html, invocation, signer, false, 'ses')

            it('leaves hrefs pointing at the destination and tags each anchor by position', () => {
                const out = sesTrack(
                    '<body><a href="https://example.com/a">a</a><a href="https://example.com/b">b</a></body>'
                )
                // A wrapper here would bury the destination inside a per-send URL that SES then
                // reports verbatim as click.link, which is what makes per-link counts impossible.
                expect(out).not.toContain('/public/m/redirect')
                expect(out).toContain('<a href="https://example.com/a" ses:tags="phl:0">')
                expect(out).toContain('<a href="https://example.com/b" ses:tags="phl:1">')
            })

            it.each([
                [
                    'clicktracking="off"',
                    '<body><a href="https://example.com" clicktracking="off">x</a></body>',
                    invocation,
                ],
                ['data-ph-no-track', '<body><a href="https://example.com" data-ph-no-track>x</a></body>', invocation],
                ['a sensitive template', '<body><a href="https://example.com">x</a></body>', sensitiveInvocation],
            ])('translates the %s opt-out into ses:no-track', (_name, html, inv) => {
                // SES honors only its own attribute, so without this the anchor is still rewritten
                // to awstrack.me and app deeplinks stop resolving.
                const out = addTrackingToEmail(html, inv, signer, false, 'ses')
                expect(out).toContain('ses:no-track')
                expect(out).not.toContain('ses:tags')
                // Opting links out of click tracking keeps the open pixel.
                expect(out).toContain('ph_id=')
            })

            it('merges into an author-supplied ses:tags value instead of emitting a second attribute', () => {
                const out = sesTrack('<body><a href="https://example.com" ses:tags="campaign:spring">x</a></body>')
                expect(out).toContain('ses:tags="campaign:spring;phl:0"')
            })

            it('keeps anchor positions stable when an earlier link opts out', () => {
                const out = sesTrack(
                    '<body><a href="https://example.com/a" data-ph-no-track>a</a><a href="https://example.com/b">b</a></body>'
                )
                expect(out).toContain('ses:tags="phl:1"')
            })
        })

        const invocationWithDistinctId = {
            functionId: 'fn-1',
            id: 'inv-1',
            teamId: 1,
            state: { globals: { event: { distinct_id: 'leaky-id' } } },
        } as any
        const phIdOf = (html: string): string => html.match(/ph_id=([A-Za-z0-9._-]+)/)![1]

        it.each([
            { captureEngagement: undefined, distinctId: 'leaky-id' },
            { captureEngagement: true, distinctId: 'leaky-id' },
            { captureEngagement: false, distinctId: undefined },
        ])(
            'mints the pixel and redirect codes in dev/test with the distinct_id the send choice $captureEngagement allows',
            ({ captureEngagement, distinctId }) => {
                const out = addTrackingToEmail(
                    '<body><a href="https://example.com">x</a></body>',
                    {
                        ...invocationWithDistinctId,
                        state: {
                            ...invocationWithDistinctId.state,
                            send: captureEngagement === undefined ? undefined : { tracking: { captureEngagement } },
                        },
                    },
                    signer
                )
                const codes = [...out.matchAll(/ph_id=([A-Za-z0-9_.-]+)/g)].map((match) => signer.parse(match[1]))
                expect(codes).toHaveLength(2)
                for (const code of codes) {
                    expect(code).toMatchObject({ functionId: 'fn-1', invocationId: 'inv-1', distinctId })
                }
            }
        )

        it('omits distinct_id from the public tracking URLs in production (Referer-leak guard)', () => {
            const devSpy = jest.spyOn(envUtils, 'isDevEnv').mockReturnValue(false)
            const testSpy = jest.spyOn(envUtils, 'isTestEnv').mockReturnValue(false)
            try {
                const out = addTrackingToEmail(
                    '<body><a href="https://example.com">x</a></body>',
                    invocationWithDistinctId,
                    signer
                )
                expect(signer.parse(phIdOf(out))?.distinctId).toBeUndefined()
            } finally {
                devSpy.mockRestore()
                testSpy.mockRestore()
            }
        })
    })

    describe('decodeHtmlEntitiesInHref', () => {
        it.each([
            ['https://example.com/?a=1&amp;b=2', 'https://example.com/?a=1&b=2'],
            ['https://example.com/?a=1&#38;b=2', 'https://example.com/?a=1&b=2'],
            ['https://example.com/?a=1&#x26;b=2', 'https://example.com/?a=1&b=2'],
            ['https://example.com/?a=1&amp;b=2&amp;c=3', 'https://example.com/?a=1&b=2&c=3'],
            ['https://example.com/path', 'https://example.com/path'],
            // Non-entity ampersands (legacy unencoded HTML) pass through untouched.
            ['https://example.com/?a=1&b=2', 'https://example.com/?a=1&b=2'],
            // Out-of-range code points (> 0x10FFFF) must not throw RangeError;
            // the entity is left as-is.
            ['https://example.com/?x=&#x200000;', 'https://example.com/?x=&#x200000;'],
            ['https://example.com/?x=&#2097152;', 'https://example.com/?x=&#2097152;'],
        ])('decodes %s to %s', (input, expected) => {
            expect(decodeHtmlEntitiesInHref(input)).toBe(expected)
        })
    })

    describe('api', () => {
        let api: CdpApi
        let app: express.Application
        let hogFunction: HogFunctionType
        const invocationId = 'invocation-id'
        let server: Server

        beforeEach(async () => {
            api = new CdpApi(hub, createCdpConsumerDeps(hub), {
                hogQueue: createMockJobQueue(),
                hogflowQueue: createMockJobQueue(),
            })
            app = setupExpressApp()
            app.use('/', api.router())
            server = app.listen(0, () => {})

            hogFunction = await insertHogFunction(hub.postgres, team.id)
        })

        afterEach(() => {
            server.close()
        })

        // In production, opens/clicks come from SES webhooks. In dev/test (which jest runs as)
        // there is no SES, so the pixel/redirect handlers themselves emit the metric — these
        // tests run in test env and exercise that path.
        describe('handleEmailTrackingRedirect', () => {
            const trackingCode = (): string =>
                signer.generate({ functionId: hogFunction.id, id: invocationId, teamId: team.id })

            const mintedLinkPath = (target: string): string => {
                const link = new URL(
                    signer.redirectUrl({ functionId: hogFunction.id, id: invocationId, teamId: team.id }, target)
                )
                return `${link.pathname}${link.search}`
            }

            const realNow = Date.now
            let clock: jest.SpyInstance
            // Shifts the clock instead of freezing it, because waitForExpect reads Date.now for its timeout.
            const setClock = (at: number): void => {
                const offset = at - realNow()
                clock.mockImplementation(() => realNow() + offset)
            }

            beforeEach(() => {
                clock = jest.spyOn(Date, 'now')
                setClock(UNSIGNED_REDIRECT_SUNSET - 60_000)
            })

            afterEach(() => {
                clock.mockRestore()
            })

            const expectLinkClickedMetrics = async (count: number): Promise<void> => {
                await waitForExpect(() => {
                    const messages = mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
                    expect(messages).toHaveLength(count)
                    for (const message of messages) {
                        expect(message.value).toMatchObject({
                            team_id: team.id,
                            metric_name: 'email_link_clicked',
                            metric_kind: 'email',
                        })
                    }
                })
            }

            it('should redirect a signed link after the unsigned link sunset and record an email_link_clicked metric', async () => {
                setClock(UNSIGNED_REDIRECT_SUNSET)
                const res = await supertest(app).get(mintedLinkPath('https://example.com/?a=1&b=2'))
                expect(res.status).toBe(302)
                expect(res.headers.location).toBe('https://example.com/?a=1&b=2')

                await expectLinkClickedMetrics(1)
            })

            it.each([
                [
                    'has no tracking code',
                    () => '/public/m/redirect?target=https://other.example/',
                    'https://other.example/',
                    0,
                ],
                [
                    'has no target signature',
                    () => `/public/m/redirect?ph_id=${trackingCode()}&target=https://other.example/`,
                    'https://other.example/',
                    1,
                ],
                [
                    'has its target swapped after signing',
                    () =>
                        mintedLinkPath('https://example.com/').replace(/target=[^&]+/, 'target=https://other.example/'),
                    'https://other.example/',
                    1,
                ],
                [
                    'carries markup in its target',
                    // The URL parser percent-encodes markup in a query or a hierarchical path, but keeps it raw in
                    // the opaque path of a mailto: URL, so only this target depends on the page escaping it.
                    () =>
                        `/public/m/redirect?target=${encodeURIComponent('mailto:"><script>alert(1)</script>@example.com')}`,
                    'mailto:&quot;&gt;&lt;script&gt;alert(1)&lt;/script&gt;@example.com',
                    0,
                ],
                [
                    'points at an email address',
                    () => '/public/m/redirect?target=mailto:[email protected]',
                    'mailto:[email protected]',
                    0,
                ],
            ])(
                'should show a confirmation page instead of redirecting when the link %s',
                async (_name, path, expectedHref, expectedMetrics) => {
                    const res = await supertest(app).get(path())
                    expect(res.status).toBe(200)
                    expect(res.headers.location).toBeUndefined()
                    expect(res.headers).toMatchObject({
                        'content-type': 'text/html; charset=utf-8',
                        'x-content-type-options': 'nosniff',
                        'cache-control': 'no-store',
                    })
                    expect(res.headers['content-security-policy']).toContain("default-src 'none'")
                    expect(res.text).toContain(`href="${expectedHref}"`)
                    expect(res.text).not.toContain('<script')

                    await expectLinkClickedMetrics(expectedMetrics)
                }
            )

            it('should return 404 for an unsigned link after the unsigned link sunset', async () => {
                setClock(UNSIGNED_REDIRECT_SUNSET)
                const res = await supertest(app).get('/public/m/redirect?target=https://other.example/')
                expect(res.status).toBe(404)
            })

            it.each([
                ['the target is not provided', () => `/public/m/redirect?ph_id=${trackingCode()}`],
                ['an unsigned target has a script scheme', () => '/public/m/redirect?target=javascript:alert(1)'],
                ['an unsigned target opens an app', () => '/public/m/redirect?target=sms:5555550100'],
                [
                    'an unsigned target puts a username before its host',
                    () => '/public/m/redirect?target=https://[email protected]/',
                ],
                [
                    'the target is repeated',
                    () => '/public/m/redirect?target=https://a.example/&target=https://b.example/',
                ],
            ])('should return 404 if %s', async (_name, path) => {
                const res = await supertest(app).get(path())
                expect(res.status).toBe(404)

                const messages = mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
                expect(messages).toHaveLength(0)
            })
        })

        describe('email tracking pixel', () => {
            it('should return a gif image and record an email_opened metric', async () => {
                const phId = signer.generate({
                    functionId: hogFunction.id,
                    id: invocationId,
                    teamId: team.id,
                })
                const res = await supertest(app).get(`/public/m/pixel?ph_id=${phId}`)
                expect(res.status).toBe(200)
                expect(res.headers['content-type']).toBe('image/gif')
                expect(res.body).toEqual(PIXEL_GIF)

                await waitForExpect(() => {
                    const messages = mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
                    expect(messages).toHaveLength(1)
                    expect(messages[0].value).toMatchObject({
                        team_id: team.id,
                        metric_name: 'email_opened',
                        metric_kind: 'email',
                    })
                })
            })

            it('counts a large audience, where most new people do not change the HyperLogLog', async () => {
                const parentRunId = new UUIDT().toString()
                const service = api['emailTrackingService'] as any
                let opened = 0
                for (let index = 0; index < 5000; index++) {
                    const added = await service.newEngagedPeople(parentRunId, `person-${index}`, 'email_opened', 'ses')
                    opened += added[0]?.count ?? 0
                }
                expect(opened).toBeGreaterThan(4900)
                expect(opened).toBeLessThan(5100)
            })

            it('still records the open when Redis stalls', async () => {
                const parentRunId = new UUIDT().toString()
                const service = api['emailTrackingService'] as any
                jest.spyOn(service.engagementRedis, 'useClient').mockReturnValue(new Promise(() => {}))
                const code = signer.generate({
                    functionId: hogFunction.id,
                    id: new UUIDT().toString(),
                    teamId: team.id,
                    parentRunId,
                })

                const response = await supertest(app).get(`/public/m/pixel?ph_id=${code}`)

                expect(response.status).toBe(200)
                await waitForExpect(() => {
                    const names = mockProducerObserver
                        .getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
                        .filter(({ value }) => value.app_source_id === parentRunId)
                        .map(({ value }) => value.metric_name)
                    expect(names).toEqual(['email_opened'])
                })
            })

            it('counts a person once per run however often they open or click', async () => {
                const parentRunId = new UUIDT().toString()
                const phId = (id: string): string =>
                    signer.generate({
                        functionId: hogFunction.id,
                        id,
                        teamId: team.id,
                        parentRunId,
                    })
                const first = phId(new UUIDT().toString())
                const second = phId(new UUIDT().toString())

                for (const code of [first, first, second]) {
                    await supertest(app).get(`/public/m/pixel?ph_id=${code}`)
                }
                await supertest(app).get(`/public/m/redirect?ph_id=${first}&target=https://example.com`)
                await supertest(app).get(`/public/m/redirect?ph_id=${first}&target=https://example.com`)

                await waitForExpect(() => {
                    const counts: Record<string, number> = {}
                    for (const { value } of mockProducerObserver.getProducedKafkaMessagesForTopic(
                        KAFKA_APP_METRICS_2
                    )) {
                        if (value.app_source_id === parentRunId) {
                            counts[value.metric_name as string] =
                                (counts[value.metric_name as string] ?? 0) + (value.count as number)
                        }
                    }
                    expect(counts).toEqual({
                        email_opened: 3,
                        email_opened_by_person: 2,
                        email_link_clicked: 2,
                        email_clicked_by_person: 1,
                    })
                })
            })

            it('should return a 200 even if the tracking code is invalid', async () => {
                const res = await supertest(app).get(`/public/m/pixel?ph_id=invalid-tracking-code`)
                expect(res.status).toBe(200)
                expect(res.headers['content-type']).toBe('image/gif')
                expect(res.body).toEqual(PIXEL_GIF)

                const messages = mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
                expect(messages).toHaveLength(0)
            })
        })

        describe('SES webhook log entries', () => {
            // The route enforces a real SNS signature; verifying it needs AWS's private key, so we
            // stub the check and let a posted SNS envelope flow through the real handler + service.
            let verifySignatureSpy: jest.SpyInstance

            beforeEach(() => {
                verifySignatureSpy = jest
                    .spyOn(SesWebhookHandler.prototype as any, 'verifySnsSignature')
                    .mockResolvedValue(true)
            })

            afterEach(() => {
                verifySignatureSpy.mockRestore()
            })

            const postBounce = async ({
                functionId,
                parentRunId,
                workflowVersion,
                transactionalMessageId,
                headerCode,
            }: {
                functionId: string
                parentRunId?: string
                workflowVersion?: number
                // Sends the bounce as SES reports it for a transactional email with this message id.
                transactionalMessageId?: string
                headerCode?: string
            }): Promise<supertest.Response> => {
                const trackingCode = signer.generate(
                    {
                        functionId,
                        id: transactionalMessageId ?? invocationId,
                        teamId: team.id,
                        parentRunId,
                        workflowVersion,
                    },
                    false,
                    workflowVersion !== undefined
                )
                const sesRecord = {
                    eventType: 'Bounce',
                    mail: {
                        timestamp: '2024-01-01T00:00:00.000Z',
                        source: '[email protected]',
                        messageId: 'ses-message-id',
                        destination: ['[email protected]'],
                        headers: [{ name: TRACKING_CODE_HEADER_NAME, value: headerCode ?? trackingCode }],
                        ...(transactionalMessageId
                            ? {
                                  commonHeaders: { to: ['[email protected]'] },
                                  tags: { ph_email_status: ['true'] },
                              }
                            : {}),
                    },
                    bounce: {
                        bounceType: 'Permanent',
                        bouncedRecipients: [{ emailAddress: '[email protected]' }],
                        timestamp: '2024-01-01T00:00:00.000Z',
                    },
                }
                const envelope = {
                    Type: 'Notification',
                    MessageId: 'sns-message-id',
                    TopicArn: 'arn:aws:sns:us-east-1:123456789012:ses-events',
                    Message: JSON.stringify(sesRecord),
                    Timestamp: '2024-01-01T00:00:00.000Z',
                    SignatureVersion: '1',
                    Signature: 'stubbed',
                    SigningCertURL: 'https://sns.us-east-1.amazonaws.com/cert.pem',
                }
                return await supertest(app)
                    .post('/public/m/ses_webhook')
                    .set('Content-Type', 'text/plain')
                    .send(JSON.stringify(envelope))
            }

            it.each([
                ['on the replica', false, false],
                ['only on the primary, during replica lag', true, false],
                ['on the replica, while the hog function lookup fails', false, true],
                ['only on the primary, while the hog function lookup fails', true, true],
            ])('writes a hog_flow log entry for a bounce that resolves to a flow %s', async (_, lagging, fail) => {
                const hogFlow = await insertHogFlow(
                    hub.postgres,
                    new FixtureHogFlowBuilder().withTeamId(team.id).build()
                )
                if (lagging) {
                    hideHogFlowsFromReplica(hub.postgres, [hogFlow.id])
                }
                if (fail) {
                    jest.spyOn(api['emailTrackingService']['hogFunctionManager'], 'getHogFunctions').mockRejectedValue(
                        new Error('hog function lookup failed')
                    )
                }

                const res = await postBounce({ functionId: hogFlow.id })
                expect(res.status).toBe(200)

                await waitForExpect(() => {
                    const logs = mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_LOG_ENTRIES)
                    expect(logs).toHaveLength(1)
                    expect(logs[0].value).toMatchObject({
                        team_id: team.id,
                        log_source: 'hog_flow',
                        log_source_id: hogFlow.id,
                        instance_id: invocationId,
                        level: 'error',
                    })
                    expect(logs[0].value.message).toContain('Permanent bounce to [email protected]')
                })

                const metrics = mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
                // Permanent bounces emit the rollup plus the hard-only sub-metric
                expect(metrics.map((m) => m.value.metric_name)).toEqual(['email_bounced', 'email_bounced_hard'])
                expect(metrics[0].value).toMatchObject({
                    team_id: team.id,
                    app_source: 'hog_flow',
                    app_source_id: hogFlow.id,
                    metric_name: 'email_bounced',
                    metric_kind: 'email',
                })
            })

            it('counts a failed primary lookup as a lookup failure for a bounce on a flow only the primary has', async () => {
                const hogFlow = await insertHogFlow(
                    hub.postgres,
                    new FixtureHogFlowBuilder().withTeamId(team.id).build()
                )
                hideHogFlowsFromReplica(hub.postgres, [hogFlow.id])
                jest.spyOn(
                    api['emailTrackingService']['hogFlowManager'],
                    'getHogFlowsCheckingPrimaryOnMiss'
                ).mockRejectedValue(new Error('primary lookup failed'))
                const lookupFailures = async (): Promise<number> =>
                    (await register.getSingleMetric('email_tracking_errors_total')!.get()).values.find(
                        (v) => v.labels.error_type === 'hog_flow_lookup_failed' && v.labels.source === 'ses'
                    )?.value ?? 0
                const before = await lookupFailures()

                const res = await postBounce({ functionId: hogFlow.id })
                expect(res.status).toBe(200)

                expect(await lookupFailures()).toBeGreaterThan(before)
                expect(mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_LOG_ENTRIES)).toHaveLength(0)
            })

            it('records the bounced status of a transactional email', async () => {
                const hogFlow = await insertHogFlow(
                    hub.postgres,
                    new FixtureHogFlowBuilder().withTeamId(team.id).build()
                )
                const messageId = new UUIDT().toString()

                const res = await postBounce({ functionId: hogFlow.id, transactionalMessageId: messageId })
                expect(res.status).toBe(200)

                // The webhook flushes the status row before it answers.
                expect(mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_EMAIL_STATUS)).toEqual([
                    expect.objectContaining({
                        key: messageId,
                        value: expect.objectContaining({
                            team_id: team.id,
                            message_id: messageId,
                            status: 'bounced',
                            reason: 'hard',
                        }),
                    }),
                ])
            })

            it('records the metric but writes no log entry and reads no workflow from the primary when the bounce resolves to a hog_function', async () => {
                const query = jest.spyOn(hub.postgres, 'query')

                const res = await postBounce({ functionId: hogFunction.id })
                expect(res.status).toBe(200)

                await waitForExpect(() => {
                    const metrics = mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
                    // Permanent bounces emit the rollup plus the hard-only sub-metric
                    expect(metrics.map((m) => m.value.metric_name)).toEqual(['email_bounced', 'email_bounced_hard'])
                    expect(metrics[0].value).toMatchObject({
                        team_id: team.id,
                        metric_name: 'email_bounced',
                    })
                })

                const logs = mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_LOG_ENTRIES)
                expect(logs).toHaveLength(0)
                expect(
                    query.mock.calls.filter(
                        ([target, , , tag]) => target === PostgresUse.COMMON_WRITE && tag === 'fetchHogFlows'
                    )
                ).toHaveLength(0)
            })

            it('attributes the bounce to the version that sent it, not the version live when it lands', async () => {
                // The workflow has been republished since the send. Reading the version off the flow
                // manager here would blame v5 for v2's bounce — which is precisely the comparison
                // ("did the new version bounce more?") the versioned series exists to answer.
                const hogFlow = await insertHogFlow(hub.postgres, {
                    ...new FixtureHogFlowBuilder().withTeamId(team.id).build(),
                    version: 5,
                })

                const res = await postBounce({ functionId: hogFlow.id, workflowVersion: 2 })
                expect(res.status).toBe(200)

                await waitForExpect(() => {
                    const metrics = mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_APP_METRICS_2)
                    const versioned = metrics.filter((m) => m.value.app_source === 'hog_flow_version')
                    // Permanent bounces emit the rollup plus the hard-only sub-metric, so both mirror.
                    expect(versioned.map((m) => m.value.app_source_id)).toEqual([`${hogFlow.id}/2`, `${hogFlow.id}/2`])
                    // Mirrored, not moved — the version-agnostic series every existing reader uses
                    // still carries the same two rows.
                    expect(
                        metrics.filter((m) => m.value.app_source === 'hog_flow').map((m) => m.value.metric_name)
                    ).toEqual(['email_bounced', 'email_bounced_hard'])
                })
            })

            it.each([
                { captureEngagement: true, teamCaptures: true, captured: 1 },
                { captureEngagement: undefined, teamCaptures: true, captured: 1 },
                { captureEngagement: false, teamCaptures: true, captured: 0 },
                { captureEngagement: true, teamCaptures: false, captured: 0 },
            ])(
                "captures the bounce of a sent email by the send's choice $captureEngagement and the team switch $teamCaptures",
                async ({ captureEngagement, teamCaptures, captured }) => {
                    await hub.postgres.query(
                        PostgresUse.COMMON_WRITE,
                        `INSERT INTO workflows_teamworkflowsconfig
                            (team_id, capture_workflows_engagement_events, email_tracking_consent_mode,
                             email_sending_suspension_reason, ses_tenant_sending_status, email_sending_tier)
                         VALUES ($1, $2, 'off', '', '', 0)`,
                        [team.id, teamCaptures],
                        'test-set-engagement-capture'
                    )
                    const hogFlow = await insertHogFlow(
                        hub.postgres,
                        new FixtureHogFlowBuilder().withTeamId(team.id).build()
                    )
                    const integration = await insertIntegration(hub.postgres, team.id, {
                        kind: 'email',
                        config: {
                            email: '[email protected]',
                            name: 'Sender',
                            domain: 'posthog-test.com',
                            verified: true,
                            provider: 'ses',
                        },
                    })
                    const emailService = api['hogExecutorAsync']['deps'].emailService
                    const sesSend = jest
                        .spyOn(emailService.sesV2Client!, 'send')
                        .mockResolvedValue({ MessageId: 'ses-message-id' } as never)
                    const invocation = createExampleInvocation({
                        id: hogFlow.id,
                        team_id: team.id,
                    })
                    invocation.state.globals.event.distinct_id = 'user-1'
                    if (captureEngagement !== undefined) {
                        invocation.state.send = { tracking: { captureEngagement } }
                    }
                    invocation.queueParameters = {
                        type: 'email',
                        to: { email: '[email protected]', name: 'User' },
                        from: { integrationId: integration.id },
                        subject: 'Reset your password',
                        text: 'Hi',
                        html: '<p>Hi</p>',
                    }

                    expect((await emailService.executeSendEmail(invocation)).error).toBeUndefined()
                    const headers: { Name: string; Value: string }[] = (sesSend.mock.calls[0][0] as any).input.Content
                        .Simple.Headers
                    const headerCode = headers.find((header) => header.Name === TRACKING_CODE_HEADER_NAME)!.Value
                    mockInternalFetch.mockClear()

                    const res = await postBounce({ functionId: hogFlow.id, headerCode })
                    expect(res.status).toBe(200)

                    const capturedEvents = mockInternalFetch.mock.calls.map(
                        ([, options]) => parseJSON(String(options?.body)).event
                    )
                    expect(capturedEvents.filter((event) => event === '$workflows_email_bounced')).toHaveLength(
                        captured
                    )
                }
            )

            it('keys the log entry under parentRunId for batch-triggered runs', async () => {
                const hogFlow = await insertHogFlow(
                    hub.postgres,
                    new FixtureHogFlowBuilder().withTeamId(team.id).build()
                )
                const parentRunId = 'batch-run-id'

                const res = await postBounce({ functionId: hogFlow.id, parentRunId })
                expect(res.status).toBe(200)

                await waitForExpect(() => {
                    const logs = mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_LOG_ENTRIES)
                    expect(logs).toHaveLength(1)
                    expect(logs[0].value).toMatchObject({
                        log_source: 'hog_flow',
                        log_source_id: parentRunId,
                        instance_id: invocationId,
                    })
                })
            })
        })
    })

    describe('SES webhook writes to the suppression list', () => {
        // EmailSuppressionService reads its threshold from `hub.EMAIL_SUPPRESSION_*` (via CdpConfig),
        // not directly from process.env. The outer beforeEach recreates `hub` per test, so overriding
        // here doesn't leak across tests — no restore in afterEach needed. Threshold=1 keeps the test
        // to a single POST — the counter arithmetic is not what this test is guarding, the write-path
        // wiring is.
        let api: CdpApi
        let app: express.Application
        let server: Server
        let verifySignatureSpy: jest.SpyInstance

        beforeEach(() => {
            hub.EMAIL_SUPPRESSION_TRANSIENT_BOUNCE_THRESHOLD = 1

            api = new CdpApi(hub, createCdpConsumerDeps(hub), {
                hogQueue: createMockJobQueue(),
                hogflowQueue: createMockJobQueue(),
            })
            app = setupExpressApp()
            app.use('/', api.router())
            server = app.listen(0, () => {})

            verifySignatureSpy = jest
                .spyOn(SesWebhookHandler.prototype as any, 'verifySnsSignature')
                .mockResolvedValue(true)
        })

        afterEach(() => {
            server.close()
            verifySignatureSpy.mockRestore()
        })

        const postTransientBounce = async (functionId: string, emailAddress: string): Promise<supertest.Response> => {
            const trackingCode = signer.generate({ functionId, id: 'invocation-id', teamId: team.id })
            const sesRecord = {
                eventType: 'Bounce',
                mail: {
                    timestamp: '2024-01-01T00:00:00.000Z',
                    source: '[email protected]',
                    messageId: 'ses-message-id',
                    destination: [emailAddress],
                    headers: [{ name: TRACKING_CODE_HEADER_NAME, value: trackingCode }],
                },
                bounce: {
                    bounceType: 'Transient',
                    bouncedRecipients: [
                        { emailAddress, diagnosticCode: 'smtp; 421 4.2.1 mailbox temporarily unavailable' },
                    ],
                    timestamp: '2024-01-01T00:00:00.000Z',
                },
            }
            const envelope = {
                Type: 'Notification',
                MessageId: 'sns-message-id',
                TopicArn: 'arn:aws:sns:us-east-1:123456789012:ses-events',
                Message: JSON.stringify(sesRecord),
                Timestamp: '2024-01-01T00:00:00.000Z',
                SignatureVersion: '1',
                Signature: 'stubbed',
                SigningCertURL: 'https://sns.us-east-1.amazonaws.com/cert.pem',
            }
            return await supertest(app)
                .post('/public/m/ses_webhook')
                .set('Content-Type', 'text/plain')
                .send(JSON.stringify(envelope))
        }

        const postPermanentBounce = async (functionId: string, emailAddress: string): Promise<supertest.Response> => {
            const trackingCode = signer.generate({ functionId, id: 'invocation-id', teamId: team.id })
            const sesRecord = {
                eventType: 'Bounce',
                mail: {
                    timestamp: '2024-01-01T00:00:00.000Z',
                    source: '[email protected]',
                    messageId: 'ses-message-id',
                    destination: [emailAddress],
                    headers: [{ name: TRACKING_CODE_HEADER_NAME, value: trackingCode }],
                },
                bounce: {
                    bounceType: 'Permanent',
                    bouncedRecipients: [{ emailAddress, diagnosticCode: 'smtp; 550 5.1.1 user unknown' }],
                    timestamp: '2024-01-01T00:00:00.000Z',
                },
            }
            const envelope = {
                Type: 'Notification',
                MessageId: 'sns-message-id',
                TopicArn: 'arn:aws:sns:us-east-1:123456789012:ses-events',
                Message: JSON.stringify(sesRecord),
                Timestamp: '2024-01-01T00:00:00.000Z',
                SignatureVersion: '1',
                Signature: 'stubbed',
                SigningCertURL: 'https://sns.us-east-1.amazonaws.com/cert.pem',
            }
            return await supertest(app)
                .post('/public/m/ses_webhook')
                .set('Content-Type', 'text/plain')
                .send(JSON.stringify(envelope))
        }

        const postComplaint = async (functionId: string, emailAddress: string): Promise<supertest.Response> => {
            const trackingCode = signer.generate({ functionId, id: 'invocation-id', teamId: team.id })
            const sesRecord = {
                eventType: 'Complaint',
                mail: {
                    timestamp: '2024-01-01T00:00:00.000Z',
                    source: '[email protected]',
                    messageId: 'ses-message-id',
                    destination: [emailAddress],
                    headers: [{ name: TRACKING_CODE_HEADER_NAME, value: trackingCode }],
                },
                complaint: {
                    complainedRecipients: [{ emailAddress }],
                    complaintFeedbackType: 'abuse',
                    timestamp: '2024-01-01T00:00:00.000Z',
                },
            }
            const envelope = {
                Type: 'Notification',
                MessageId: 'sns-message-id',
                TopicArn: 'arn:aws:sns:us-east-1:123456789012:ses-events',
                Message: JSON.stringify(sesRecord),
                Timestamp: '2024-01-01T00:00:00.000Z',
                SignatureVersion: '1',
                Signature: 'stubbed',
                SigningCertURL: 'https://sns.us-east-1.amazonaws.com/cert.pem',
            }
            return await supertest(app)
                .post('/public/m/ses_webhook')
                .set('Content-Type', 'text/plain')
                .send(JSON.stringify(envelope))
        }

        it('inserts a suppression row and marks it suppressed after a Transient bounce webhook', async () => {
            const hogFlow = await insertHogFlow(hub.postgres, new FixtureHogFlowBuilder().withTeamId(team.id).build())
            const email = '[email protected]'

            const res = await postTransientBounce(hogFlow.id, email)
            expect(res.status).toBe(200)

            // handleSesWebhook awaits the suppression write inline, so the row is present
            // as soon as the 200 returns — no waitForExpect needed.
            const result = await hub.postgres.query<{
                identifier: string
                source: string
                suppressed: boolean
                transient_bounce_count: number
                deleted: boolean
            }>(
                PostgresUse.COMMON_READ,
                `SELECT identifier, source, suppressed, transient_bounce_count, deleted
                 FROM posthog_messagesuppression
                 WHERE team_id = $1 AND identifier = $2`,
                [team.id, email],
                'test-read-suppression'
            )
            expect(result.rows).toEqual([
                {
                    identifier: email,
                    source: 'BOUNCE',
                    suppressed: true,
                    transient_bounce_count: 1,
                    deleted: false,
                },
            ])
        })

        it('inserts a suppressed row for a Permanent bounce webhook', async () => {
            const hogFlow = await insertHogFlow(hub.postgres, new FixtureHogFlowBuilder().withTeamId(team.id).build())
            const email = '[email protected]'

            const res = await postPermanentBounce(hogFlow.id, email)
            expect(res.status).toBe(200)

            const result = await hub.postgres.query<{
                identifier: string
                source: string
                suppressed: boolean
                transient_bounce_count: number
                last_bounce_diagnostic: string | null
                deleted: boolean
            }>(
                PostgresUse.COMMON_READ,
                `SELECT identifier, source, suppressed, transient_bounce_count, last_bounce_diagnostic, deleted
                 FROM posthog_messagesuppression
                 WHERE team_id = $1 AND identifier = $2`,
                [team.id, email],
                'test-read-hard-bounce-suppression'
            )
            expect(result.rows).toEqual([
                {
                    identifier: email,
                    source: 'BOUNCE',
                    suppressed: true,
                    transient_bounce_count: 0,
                    last_bounce_diagnostic: 'smtp; 550 5.1.1 user unknown',
                    deleted: false,
                },
            ])
        })

        it('inserts a suppressed row for a Complaint webhook', async () => {
            const hogFlow = await insertHogFlow(hub.postgres, new FixtureHogFlowBuilder().withTeamId(team.id).build())
            const email = '[email protected]'

            const res = await postComplaint(hogFlow.id, email)
            expect(res.status).toBe(200)

            const result = await hub.postgres.query<{
                identifier: string
                source: string
                suppressed: boolean
                transient_bounce_count: number
                deleted: boolean
            }>(
                PostgresUse.COMMON_READ,
                `SELECT identifier, source, suppressed, transient_bounce_count, deleted
                 FROM posthog_messagesuppression
                 WHERE team_id = $1 AND identifier = $2`,
                [team.id, email],
                'test-read-complaint-suppression'
            )
            expect(result.rows).toEqual([
                {
                    identifier: email,
                    source: 'COMPLAINT',
                    suppressed: true,
                    transient_bounce_count: 0,
                    deleted: false,
                },
            ])
        })

        // The webhook write and the pre-send read are covered separately. This joins them end to end:
        // a real SES envelope over the HTTP route, then the check every email step runs before sending.
        // It fails if a Permanent bounce stops marking the row, which would keep sending transactional
        // email to a dead address, or if a soft bounce or complaint starts marking it, which would drop
        // invoices and password resets for a working address.
        it.each([
            ['a transient bounce', postTransientBounce, false],
            ['a complaint', postComplaint, false],
            ['a permanent bounce', postPermanentBounce, true],
        ] as const)(
            '%s suppresses marketing email, and blocks transactional email only for a hard bounce',
            async (_label, postWebhook, blocksTransactional) => {
                const hogFlow = await insertHogFlow(
                    hub.postgres,
                    new FixtureHogFlowBuilder().withTeamId(team.id).build()
                )
                const email = '[email protected]'

                const res = await postWebhook(hogFlow.id, email)
                expect(res.status).toBe(200)

                const suppression = new EmailSuppressionService(hub.postgres, {
                    transientBounceThreshold: hub.EMAIL_SUPPRESSION_TRANSIENT_BOUNCE_THRESHOLD,
                })

                expect(await suppression.isSuppressed(team.id, email)).toBe(true)
                expect(await suppression.isSuppressed(team.id, email, { transactional: true })).toBe(
                    blocksTransactional
                )
            }
        )
    })

    describe('resolveEmailEngagementDistinctId', () => {
        const buildInvocation = (
            globals: Partial<NonNullable<CyclotronJobInvocationHogFunction['state']['globals']>>
        ): CyclotronJobInvocationHogFunction =>
            ({
                state: { globals: globals as any },
            }) as CyclotronJobInvocationHogFunction

        it.each([
            {
                name: 'uses event.distinct_id (native for event-triggered; backfilled by the worker for batch)',
                globals: { event: { distinct_id: 'user-from-event' } },
                expected: 'user-from-event',
            },
            {
                // Empty event.distinct_id means no distinct_id resolved upstream; we must NOT derive
                // one from globals.person — person.id is the uuid (phantom person) and person.distinct_id
                // is the same source already folded into event.distinct_id.
                name: 'ignores globals.person and returns undefined when event.distinct_id is empty',
                globals: { event: { distinct_id: '' }, person: { id: 'person-uuid', distinct_id: 'person-distinct' } },
                expected: undefined,
            },
            {
                name: 'returns undefined when there is no event',
                globals: {},
                expected: undefined,
            },
        ])('$name', ({ globals, expected }) => {
            expect(resolveEmailEngagementDistinctId(buildInvocation(globals as any))).toBe(expected)
        })
    })

    describe('METRIC_NAME_TO_EVENT_NAME allowlist', () => {
        it('maps every email metric we want to surface and excludes internal ones', () => {
            // Adding entries here changes the set of events customers can build insights on top of —
            // treat as a public-API change. Removing entries leaves customers with broken insights.
            expect(METRIC_NAME_TO_EVENT_NAME).toEqual({
                email_sent: '$workflows_email_sent',
                email_failed: '$workflows_email_failed',
                email_delivered: '$workflows_email_delivered',
                email_opened: '$workflows_email_opened',
                email_link_clicked: '$workflows_email_link_clicked',
                email_bounced: '$workflows_email_bounced',
                email_blocked: '$workflows_email_blocked',
            })
        })
    })
})