Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

38.2 KB
Raw Blame
import { DateTime } from 'luxon'
import { Counter } from 'prom-client'
import express from 'ultimate-express'

import { HogFlow } from '~/cdp/schema/hogflow'
import {
    CyclotronJobInvocationHogFlow,
    CyclotronJobInvocationHogFunction,
    HogFunctionType,
    LogEntry,
    LogEntryLevel,
    MinimalAppMetric,
} from '~/cdp/types'
import { ModifiedRequest } from '~/common/api/router'
import { RedisV2 } from '~/common/redis/redis-v2'
import { isDevEnv, isTestEnv } from '~/common/utils/env-utils'
import { parseJSON } from '~/common/utils/json-parse'
import { logger, serializeError } from '~/common/utils/logger'

import { CapturedEventsService } from '../captured-events/captured-events.service'
import { HogFlowManagerService } from '../hogflows/hogflow-manager.service'
import { HogFunctionManagerService } from '../managers/hog-function-manager.service'
import { TeamWorkflowsConfigService } from '../managers/team-workflows-config.service'
import { HogFunctionMonitoringService } from '../monitoring/hog-function-monitoring.service'
import { EmailStatusService } from './email-status.service'
import { EmailSuppressionService } from './email-suppression.service'
import { escapeHtml } from './helpers/escape-html'
import { SES_LINK_INDEX_TAG, SesWebhookHandler } from './helpers/ses'
import { EmailTrackingCodeSigner, trackingCodeFormatCounter } from './helpers/tracking-code'

export const PIXEL_GIF = Buffer.from('R0lGODlhAQABAAAAACH5BAEKAAEALAAAAAABAAEAAAICTAEAOw==', 'base64')
const LINK_REGEX =
    /<a\b[^>]*\bhref\s*=\s*(?:"(?!javascript:)([^"]*)"|'(?!javascript:)([^']*)'|(?!javascript:)([^'">\s]+))[^>]*>([\s\S]*?)<\/a>/gi

// Anchors carrying either opt-out marker are left unwrapped (no click-tracking redirect).
const LINK_TRACKING_OPT_OUT_REGEX = /\bclicktracking\s*=\s*["']?off["']?|\bdata-ph-no-track\b/i

// Matches an existing `ses:tags` attribute so our link tag can be merged into its value.
const SES_TAGS_ATTR_REGEX = /\bses:tags\s*=\s*(?:"([^"]*)"|'([^']*)'|([^'">\s]+))/i
const SES_NO_TRACK_ATTR_REGEX = /\bses:no-track\b/i

/**
 * How anchors in the email body are prepared for click attribution.
 *
 * `redirect` rewrites each href to our own `/public/m/redirect` endpoint, which records the click
 * before forwarding. Used on the maildev path, where no provider webhook exists.
 *
 * `ses` leaves hrefs pointing at their real destination and instead annotates each anchor with the
 * attributes SES reads when it rewrites links for its own click tracking. SES is the sole click
 * source on this path, so wrapping the href as well would only bury the destination inside a
 * per-send URL that the Click event then reports verbatim.
 */
export type EmailTrackingMode = 'redirect' | 'ses'

const addAttributeToOpeningTag = (openingTag: string, attribute: string): string => {
    return openingTag.replace(/\s*\/?>$/, ` ${attribute}>`)
}

const withSesLinkIndexTag = (openingTag: string, linkIndex: number): string => {
    const tag = `${SES_LINK_INDEX_TAG}:${linkIndex}`
    const existing = openingTag.match(SES_TAGS_ATTR_REGEX)
    if (!existing) {
        return addAttributeToOpeningTag(openingTag, `ses:tags="${tag}"`)
    }
    // An HTML parser keeps only the first of two same-named attributes, so emitting a second
    // `ses:tags` would silently discard whichever set loses. Merge into the author's existing
    // value instead so both their tags and ours reach SES. SES separates pairs with `;` and
    // allows only `0-9 A-Z a-z - _` in a value, so any other separator makes one malformed tag.
    const value = existing[1] ?? existing[2] ?? existing[3] ?? ''
    return openingTag.replace(SES_TAGS_ATTR_REGEX, () => `ses:tags="${value ? `${value};` : ''}${tag}"`)
}

const trackingEventsCounter = new Counter({
    name: 'email_tracking_events_total',
    help: 'Total number of email tracking events received',
    labelNames: ['event_type', 'source'],
})

const emailTrackingErrorsCounter = new Counter({
    name: 'email_tracking_errors_total',
    help: 'Total number of email tracking processing errors',
    labelNames: ['error_type', 'source'],
})

// Skips here are expected (e.g. SES webhook for a hog_function rather than a hog_flow),
// kept separate so alerts on `email_tracking_errors_total` don't fire on normal traffic.
const emailTrackingLogSkipsCounter = new Counter({
    name: 'email_tracking_log_skips_total',
    help: 'Total number of email tracking log entries skipped (expected, non-error)',
    labelNames: ['reason', 'source'],
})

const ENGAGEMENT_PERSON_METRICS: Partial<Record<MinimalAppMetric['metric_name'], MinimalAppMetric['metric_name'][]>> = {
    email_opened: ['email_opened_by_person'],
    email_link_clicked: ['email_opened_by_person', 'email_clicked_by_person'],
}
// An open after this counts its person again. Almost all opens and clicks come in the first days.
const ENGAGEMENT_PEOPLE_TTL_SECONDS = 30 * 24 * 60 * 60
// A slow Redis costs a person count, never an open or click webhook.
const ENGAGEMENT_REDIS_TIMEOUT_MS = 500
// PFADD returns 0 for many new people once a HyperLogLog fills, so the growth of PFCOUNT is what counts. Each pair of
// keys is a HyperLogLog and the total already reported. Returns how much each total grew.
const ENGAGEMENT_PEOPLE_SCRIPT = `
local grew = {}
for index = 1, #KEYS, 2 do
    redis.call('PFADD', KEYS[index], ARGV[1])
    local total = redis.call('PFCOUNT', KEYS[index])
    local reported = tonumber(redis.call('GET', KEYS[index + 1]) or '0')
    redis.call('SET', KEYS[index + 1], math.max(total, reported), 'EX', ARGV[2])
    redis.call('EXPIRE', KEYS[index], ARGV[2])
    grew[#grew + 1] = math.max(total - reported, 0)
end
return grew
`

// Allowlist of metrics that surface as PostHog events when engagement-event capture is enabled.
// Metrics not in this map are deliberately ignored (no `$messaging_${metricName}` fallback) so a new
// internal metric can't silently leak into customers' event streams.
export const METRIC_NAME_TO_EVENT_NAME: Partial<Record<MinimalAppMetric['metric_name'], string>> = {
    email_sent: '$workflows_email_sent',
    email_failed: '$workflows_email_failed',
    email_delivered: '$workflows_email_delivered',
    email_opened: '$workflows_email_opened',
    email_link_clicked: '$workflows_email_link_clicked',
    email_bounced: '$workflows_email_bounced',
    email_blocked: '$workflows_email_blocked',
}

/**
 * Resolve the identifier to attribute engagement events to. `event.distinct_id` is the canonical
 * source for both flows: event-triggered runs carry it from the trigger event, and batch/scheduled
 * runs have it backfilled by the cyclotron worker from the person's resolved distinct_id (see the
 * `getCyclotronPerson` backfill in cdp-cyclotron-worker-hogflow.consumer.ts). We deliberately do
 * NOT derive it from `globals.person` — `person.id` is the person UUID, which was never ingested as
 * a distinct_id and would mint a phantom person, and `person.distinct_id` is the same source already
 * folded into `event.distinct_id` upstream. If there's no distinct_id we skip capture rather than
 * emit an unattributable event.
 */
export const resolveEmailEngagementDistinctId = (
    invocation: Pick<CyclotronJobInvocationHogFunction, 'state'>
): string | undefined => {
    return invocation.state?.globals?.event?.distinct_id || undefined
}

// The workflow version that is sending this message, for the tracking code minted below. A flow's
// email runs as a hog function invocation built by spreading the flow invocation, so `hogFlow` is
// present at runtime even though the type is the narrower hog function shape. A hog function send
// has none, and its engagement lands in the version-agnostic series alone.
export const resolveEmailSendingVersion = (invocation: CyclotronJobInvocationHogFunction): number | undefined => {
    return 'hogFlow' in invocation
        ? (invocation as unknown as CyclotronJobInvocationHogFlow).hogFlow.version
        : undefined
}

export const resolveEmailTrackingDistinctId = (
    invocation: Pick<CyclotronJobInvocationHogFunction, 'state' | 'hogFunction'>
): string | undefined => {
    return invocation.state?.send?.tracking.captureEngagement === false
        ? undefined
        : resolveEmailEngagementDistinctId(invocation)
}

// HTML attribute values arrive entity-encoded (e.g. `&amp;`, `&#38;`). Decode before
// percent-encoding for the tracking redirect, otherwise `?a=1&amp;b=2` round-trips
// through `target=` as a literal `&amp;` and breaks the destination page's query string.
const HTML_ENTITY_REGEX = /&(?:(amp|lt|gt|quot|apos)|#(\d+)|#x([0-9a-fA-F]+));/g
const NAMED_ENTITIES: Record<string, string> = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" }

export const decodeHtmlEntitiesInHref = (value: string): string => {
    return value.replace(HTML_ENTITY_REGEX, (_match, named, dec, hex) => {
        if (named) {
            return NAMED_ENTITIES[named]
        }
        const code = dec ? parseInt(dec, 10) : parseInt(hex, 16)
        // `String.fromCodePoint` throws RangeError above 0x10FFFF — `Number.isFinite` alone
        // wouldn't catch e.g. `&#x200000;` since `parseInt` happily returns a finite value.
        return code >= 0 && code <= 0x10ffff ? String.fromCodePoint(code) : _match
    })
}

export const addTrackingToEmail = (
    html: string,
    invocation: CyclotronJobInvocationHogFunction,
    signer: EmailTrackingCodeSigner,
    isTest = false,
    mode: EmailTrackingMode = 'redirect'
): string => {
    // Only carry distinct_id in the in-email pixel/redirect URLs in dev/test, where those handlers
    // record metrics. In production they don't (SES webhooks own open/click attribution via the
    // signed header), so embedding distinct_id in the public `ph_id` would be unused — and worse,
    // a tracked-link click could leak the recipient identifier to the destination via the Referer.
    const distinctId = isDevEnv() || isTestEnv() ? resolveEmailTrackingDistinctId(invocation) : undefined
    // The pixel and redirect codes carry the sending version too. SES-attributed opens and clicks
    // read the version off the signed header instead, but a provider we track ourselves has these
    // URLs as its only carrier, and without the version its engagement cannot split by version.
    const trackingInvocation = { ...invocation, distinctId, workflowVersion: resolveEmailSendingVersion(invocation) }
    const trackingUrl = signer.pixelUrl(trackingInvocation, isTest)

    // Incremented for every anchor LINK_REGEX matches, including ones that turn out to be opted
    // out or unsafe, so the index stays tied to document position instead of shifting with how
    // many anchors happened to be trackable. The same template therefore yields the same indices
    // on every send, which is what makes counts comparable across recipients.
    let anchorIndex = 0
    // A sensitive email's links can carry a login code or a reset token, so none of them goes through
    // a click-tracking redirect. Opens and deliveries are still tracked.
    const sensitive = invocation.hogFunction?.metadata?.sensitive === true

    html = html.replace(LINK_REGEX, (m, d, s, u) => {
        const linkIndex = anchorIndex++
        const href = decodeHtmlEntitiesInHref(d || s || u || '')
        // LINK_REGEX skips literal `javascript:` hrefs, but an attacker could entity-encode
        // the scheme (e.g. `java&#x73;cript:`) to slip past it; re-check after decoding.
        if (/^\s*javascript:/i.test(href)) {
            return m
        }
        // Per-link opt-out. Leaves the href on its own domain so mobile universal links /
        // app deeplinks resolve - routing through our cross-domain redirect breaks them.
        // `clicktracking="off"` is the ESP de-facto standard; `data-ph-no-track` is our name.
        // Match only the opening <a> tag so a marker in the link's inner HTML (a child
        // element's attribute or literal link text) can't silently disable tracking.
        const openingTag = m.match(/^<a\b[^>]*>/i)?.[0] ?? ''
        const optedOut = sensitive || LINK_TRACKING_OPT_OUT_REGEX.test(openingTag)

        if (mode === 'ses') {
            // SES rewrites every http(s) anchor once click tracking is on, and it honors only its
            // own `ses:no-track`. Translating the opt-out into that attribute is what actually
            // keeps a universal link or app deeplink off awstrack.me; leaving the href alone (the
            // `redirect` mode's opt-out) does nothing here.
            if (optedOut) {
                return SES_NO_TRACK_ATTR_REGEX.test(openingTag)
                    ? m
                    : addAttributeToOpeningTag(openingTag, 'ses:no-track') + m.slice(openingTag.length)
            }
            return withSesLinkIndexTag(openingTag, linkIndex) + m.slice(openingTag.length)
        }

        if (optedOut) {
            return m
        }
        const tracked = signer.redirectUrl(trackingInvocation, href, isTest)

        // replace just the href in the original tag to preserve other attributes
        return m.replace(/\bhref\s*=\s*(?:"[^"]*"|'[^']*'|[^'">\s]+)/i, `href="${tracked}"`)
    })

    html = html.replace('</body>', `<img src="${trackingUrl}" style="display: none;" /></body>`)

    return html
}

const REDIRECT_CONFIRMATION_PAGE_HEADERS = {
    'Content-Type': 'text/html; charset=utf-8',
    'X-Content-Type-Options': 'nosniff',
    'Cache-Control': 'no-store',
    // The page runs no script, and blocking frames stops another site from overlaying the link.
    'Content-Security-Policy':
        "default-src 'none'; style-src 'unsafe-inline'; base-uri 'none'; form-action 'none'; frame-ancestors 'none'",
    // The page URL carries the tracking code, so the click must not send it to the destination.
    'Referrer-Policy': 'no-referrer',
}

// The redirect mode wrapped every href except `javascript:`, so a legacy link can carry any other
// scheme or a relative href. Only web, email and phone links get the page. Every other target returns
// 404, so the page never renders a link that runs script in this origin or opens an arbitrary app.
const CONFIRMABLE_PROTOCOLS = new Set(['http:', 'https:', 'mailto:', 'tel:'])

// Only the local maildev path still wraps links, and it signs them, so every unsigned link in a real
// inbox predates the signature. After this instant an unsigned link returns 404, which closes the
// window in which anyone can make this endpoint show a destination of their choice.
export const UNSIGNED_REDIRECT_SUNSET = Date.parse('2027-02-11T00:00:00Z')

// Emails sent before redirect targets were signed still sit in inboxes, so an unsigned link gets a
// page that names the destination instead of a 404. The recipient then decides whether to continue,
// which keeps this endpoint from silently forwarding anyone to a URL that no email ever contained.
const renderRedirectConfirmationPage = (target: string): string | null => {
    let url: URL
    try {
        url = new URL(target)
    } catch {
        return null
    }
    if (!CONFIRMABLE_PROTOCOLS.has(url.protocol)) {
        return null
    }
    // `https://[email protected]/` goes to evil.example, but a reader trusts the prefix.
    if (url.username || url.password) {
        return null
    }
    const href = escapeHtml(url.href)
    // A mailto: or tel: URL has no host, so the page names the whole target instead.
    const destination = escapeHtml(url.host || url.href)
    return `<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Confirm the link</title>
<style>body{font-family:ui-sans-serif,system-ui,-apple-system,sans-serif;max-width:560px;margin:4rem auto;padding:0 1rem;line-height:1.5;color:#111}.url{word-break:break-all;color:#555}</style>
</head>
<body>
<h1>Confirm the link</h1>
<p>This link goes to <strong>${destination}</strong>. Continue only if you trust it.</p>
<p class="url">${href}</p>
<p><a href="${href}" rel="noreferrer">Continue to ${destination}</a></p>
</body>
</html>
`
}

export class EmailTrackingService {
    private sesWebhookHandler: SesWebhookHandler

    constructor(
        private hogFunctionManager: HogFunctionManagerService,
        private hogFlowManager: HogFlowManagerService,
        private hogFunctionMonitoringService: HogFunctionMonitoringService,
        private capturedEventsService: CapturedEventsService,
        private teamWorkflowsConfigService: TeamWorkflowsConfigService,
        private trackingCodeSigner: EmailTrackingCodeSigner,
        private emailSuppressionService: EmailSuppressionService,
        private emailStatusService: EmailStatusService | null = null,
        private engagementRedis: RedisV2 | null = null
    ) {
        const allowedTopicArns = (process.env.SES_ALLOWED_SNS_TOPIC_ARNS ?? '').split(',')
        this.sesWebhookHandler = new SesWebhookHandler(this.trackingCodeSigner, allowedTopicArns)
    }

    public async trackMetric({
        functionId,
        invocationId,
        actionId,
        parentRunId,
        distinctId,
        metricName,
        source,
        properties,
        timestamp,
        instanceIdOverride,
        deferFlush = false,
        workflowVersion,
        verifiedTeamId,
    }: {
        functionId?: string
        invocationId?: string
        actionId?: string
        parentRunId?: string
        distinctId?: string
        metricName: MinimalAppMetric['metric_name']
        source: 'direct' | 'ses'
        properties?: Record<string, unknown>
        timestamp?: string
        instanceIdOverride?: string
        // Skips the per-call Kafka flush so a caller handling several metrics can queue them all
        // and pay a single broker round-trip. The caller owns flushing when it sets this.
        deferFlush?: boolean
        workflowVersion?: number
        // Team id from a signature-verified tracking code. Lets the metric survive the deletion
        // of its workflow: sends are recorded at send time, so dropping the delayed complaint and
        // bounce feedback here would make the team's rates read cleaner than they are.
        verifiedTeamId?: string
    }): Promise<void> {
        if (!functionId || !invocationId) {
            logger.error('[EmailTrackingService] trackMetric: Invalid custom ID', {
                functionId,
                invocationId,
                metricName,
            })
            emailTrackingErrorsCounter.inc({ error_type: 'invalid_custom_id', source })
            return
        }

        // The function ID could be one or the other so we load both
        const [hogFunction, hogFlow] = await Promise.all([
            this.hogFunctionManager.getHogFunction(functionId).catch(() => null),
            this.getFeedbackHogFlows([functionId], source)
                .then((hogFlows) => hogFlows[functionId])
                .catch(() => null),
        ])

        let teamId = hogFunction?.team_id ?? hogFlow?.team_id
        let appSourceId = hogFunction?.id ?? hogFlow?.id

        if (!teamId || !appSourceId) {
            const fallbackTeamId = Number(verifiedTeamId)
            if (Number.isInteger(fallbackTeamId) && fallbackTeamId > 0) {
                // The workflow row is gone (deleting a finished campaign is routine housekeeping),
                // but its sends were recorded at send time, so dropping the delayed complaint and
                // bounce feedback would make the team read cleaner than it is to everything that
                // sums team totals, including the sending tiers. The tracking code is signed, so
                // its team and function ids are trustworthy without the row.
                logger.warn('[EmailTrackingService] trackMetric: recording for a deleted function or flow', {
                    functionId,
                    invocationId,
                    source,
                })
                emailTrackingErrorsCounter.inc({ error_type: 'hog_function_or_flow_deleted', source })
                teamId = fallbackTeamId
                appSourceId = functionId
            } else {
                logger.error('[EmailTrackingService] trackMetric: Hog function or flow not found', {
                    functionId,
                    invocationId,
                    source,
                })
                emailTrackingErrorsCounter.inc({ error_type: 'hog_function_or_flow_not_found', source })
                return
            }
        }

        this.hogFunctionMonitoringService.queueAppMetric(
            {
                team_id: teamId,
                // Mirror email.service.ts's `parentRunId ?? functionId` so batch-triggered
                // runs get their webhook metrics attributed to the batch run, not the workflow.
                app_source_id: parentRunId ?? appSourceId,
                instance_id: instanceIdOverride || actionId || invocationId,
                metric_name: metricName,
                metric_kind: 'email',
                count: 1,
                // The version comes off the tracking code minted at send time, never from `hogFlow`
                // above — that's the currently published version, which for an engagement event
                // arriving after a republish would blame the new version for the old one's sends.
                app_source_version:
                    hogFlow && workflowVersion !== undefined ? { id: hogFlow.id, version: workflowVersion } : undefined,
            },
            hogFlow ? 'hog_flow' : 'hog_function'
        )

        if (parentRunId) {
            for (const { metric, count } of await this.newEngagedPeople(
                parentRunId,
                invocationId,
                metricName,
                source
            )) {
                this.hogFunctionMonitoringService.queueAppMetric(
                    {
                        team_id: teamId,
                        app_source_id: parentRunId,
                        instance_id: instanceIdOverride || actionId || invocationId,
                        metric_name: metric,
                        metric_kind: 'email',
                        count,
                    },
                    hogFlow ? 'hog_flow' : 'hog_function'
                )
            }
        }

        const eventName = METRIC_NAME_TO_EVENT_NAME[metricName]
        if (eventName && distinctId && (await this.teamWorkflowsConfigService.shouldCaptureEngagementEvents(teamId))) {
            await this.capturedEventsService.queueEvent({
                team_id: teamId,
                event: eventName,
                distinct_id: distinctId,
                timestamp,
                properties: {
                    $workflow_id: appSourceId,
                    $workflow_action_id: actionId,
                    ...(workflowVersion !== undefined ? { $workflow_version: workflowVersion } : {}),
                    ...properties,
                },
            })
            await this.capturedEventsService.flush()
        }

        if (!deferFlush) {
            await this.hogFunctionMonitoringService.flush()
        }

        trackingEventsCounter.inc({ event_type: metricName, source })
        logger.debug('[EmailTrackingService] trackMetric: Email tracking event', {
            functionId,
            invocationId,
            metricName,
        })
    }

    public async trackLogs(
        entries: {
            functionId?: string
            invocationId?: string
            parentRunId?: string
            level: LogEntryLevel
            message: string
        }[]
    ): Promise<void> {
        if (entries.length === 0) {
            return
        }

        const uniqueFunctionIds = Array.from(
            new Set(entries.map((e) => e.functionId).filter((id): id is string => Boolean(id)))
        )

        let hogFlows: Record<string, HogFlow | null> = {}
        try {
            hogFlows = await this.getFeedbackHogFlows(uniqueFunctionIds, 'ses')
        } catch (error) {
            logger.error('[EmailTrackingService] trackLogs: Failed to load hog flows', { error })
            emailTrackingErrorsCounter.inc({ error_type: 'hog_flow_lookup_failed', source: 'ses' })
            return
        }

        const now = DateTime.utc()
        const logEntries: LogEntry[] = []
        for (const entry of entries) {
            if (!entry.functionId || !entry.invocationId) {
                logger.error('[EmailTrackingService] trackLogs: Invalid custom ID', {
                    functionId: entry.functionId,
                    invocationId: entry.invocationId,
                })
                emailTrackingErrorsCounter.inc({ error_type: 'invalid_custom_id', source: 'ses' })
                continue
            }

            const hogFlow = hogFlows[entry.functionId]
            if (!hogFlow) {
                emailTrackingLogSkipsCounter.inc({ reason: 'non_flow', source: 'ses' })
                continue
            }

            logEntries.push({
                team_id: hogFlow.team_id,
                log_source: 'hog_flow',
                // Batch-triggered runs log under the batch run id (parentRunId); mirror the metrics path.
                log_source_id: entry.parentRunId ?? hogFlow.id,
                instance_id: entry.invocationId,
                timestamp: now,
                level: entry.level,
                message: entry.message,
            })
        }

        if (logEntries.length === 0) {
            return
        }

        this.hogFunctionMonitoringService.queueLogs(logEntries, 'hog_flow')
        await this.hogFunctionMonitoringService.flush()
    }

    public async handleSesWebhook(req: ModifiedRequest): Promise<{ status: number; message?: string }> {
        if (!req.body) {
            return { status: 403, message: 'Missing request body' }
        }

        try {
            const {
                status,
                body,
                metrics,
                logEntries,
                transientBounceRecipients,
                hardBounceRecipients,
                complainedRecipients,
                deliveredRecipients,
                emailStatusUpdates,
            } = await this.sesWebhookHandler.handleWebhook({
                body: parseJSON(req.body),
                headers: req.headers,
                verifySignature: true,
            })

            // A single SNS notification can yield several metrics (a Click emits its rollup plus a
            // per-link row, a Bounce its rollup plus a per-type row). Flushing inside each call would
            // make those serial Kafka round-trips, so defer and flush once for the batch.
            for (const metric of metrics || []) {
                await this.trackMetric({
                    functionId: metric.functionId,
                    invocationId: metric.invocationId,
                    actionId: metric.actionId,
                    parentRunId: metric.parentRunId,
                    distinctId: metric.distinctId,
                    metricName: metric.metricName,
                    source: 'ses',
                    properties: metric.properties,
                    timestamp: metric.timestamp,
                    instanceIdOverride: metric.instanceIdOverride,
                    deferFlush: true,
                    workflowVersion: metric.workflowVersion,
                    verifiedTeamId: metric.teamId,
                })
            }
            if (metrics?.length) {
                await this.hogFunctionMonitoringService.flush()
            }

            // Wrapped so a failure here doesn't skip the suppression writes below.
            try {
                await this.trackLogs(
                    (logEntries || []).map((entry) => ({
                        functionId: entry.functionId,
                        invocationId: entry.invocationId,
                        parentRunId: entry.parentRunId,
                        level: entry.level,
                        message: entry.message,
                    }))
                )
            } catch (error) {
                logger.error('[EmailTrackingService] handleSesWebhook: Failed to track logs', { error })
                emailTrackingErrorsCounter.inc({ error_type: 'track_logs_failed', source: 'ses' })
            }

            // Best-effort like the writes around it: queue and flush never throw.
            if (this.emailStatusService && emailStatusUpdates?.length) {
                for (const update of emailStatusUpdates) {
                    const teamId = parseInt(update.teamId, 10)
                    if (Number.isNaN(teamId)) {
                        continue
                    }
                    this.emailStatusService.queue({
                        teamId,
                        messageId: update.invocationId,
                        status: update.status,
                        reason: update.reason,
                        statusAt: new Date(update.timestamp),
                    })
                }
                await this.emailStatusService.flush()
            }

            // Feed bounces, complaints and successful deliveries into the suppression list. Wrapped
            // so a failure here never affects the webhook's 200 response to SNS. Deliveries are
            // processed first so a delivery + bounce in the same batch nets out conservatively
            // (count resets, then the fresh bounce re-counts from a clean slate).
            try {
                for (const { teamId, emailAddresses, timestamp, clearsGlobalHardBounces } of deliveredRecipients ||
                    []) {
                    const parsedTeamId = teamId ? parseInt(teamId, 10) : NaN
                    if (parsedTeamId && !isNaN(parsedTeamId)) {
                        await this.emailSuppressionService.recordDeliveries(parsedTeamId, emailAddresses, timestamp, {
                            clearGlobal: clearsGlobalHardBounces ?? false,
                        })
                    }
                }
                for (const { teamId, emailAddresses, diagnostic } of transientBounceRecipients || []) {
                    const parsedTeamId = teamId ? parseInt(teamId, 10) : NaN
                    if (parsedTeamId && !isNaN(parsedTeamId)) {
                        await this.emailSuppressionService.recordTransientBounces(
                            parsedTeamId,
                            emailAddresses,
                            diagnostic
                        )
                    }
                }
                for (const {
                    teamId,
                    emailAddresses,
                    diagnostic,
                    nonexistentAddresses,
                    bounceTimestamp,
                } of hardBounceRecipients ?? []) {
                    const parsedTeamId = teamId ? parseInt(teamId, 10) : NaN
                    if (parsedTeamId && !isNaN(parsedTeamId)) {
                        await this.emailSuppressionService.recordHardBounces(parsedTeamId, emailAddresses, diagnostic)
                        if (nonexistentAddresses) {
                            await this.emailSuppressionService.recordGlobalHardBounces(
                                nonexistentAddresses,
                                bounceTimestamp
                            )
                        }
                    }
                }
                for (const { teamId, emailAddresses, feedbackType } of complainedRecipients || []) {
                    const parsedTeamId = teamId ? parseInt(teamId, 10) : NaN
                    if (parsedTeamId && !isNaN(parsedTeamId)) {
                        await this.emailSuppressionService.recordComplaints(parsedTeamId, emailAddresses, feedbackType)
                    }
                }
            } catch (error) {
                logger.error('[EmailTrackingService] Failed to update suppression list', { error })
                emailTrackingErrorsCounter.inc({ error_type: 'suppression_update_failed', source: 'ses' })
            }

            return { status, message: body as string }
        } catch (error) {
            emailTrackingErrorsCounter.inc({ error_type: error.name || 'unknown' })
            logger.error('[EmailService] handleWebhook: SES webhook error', { error: serializeError(error) })
            throw error
        }
    }

    private async getFeedbackHogFlows(
        ids: HogFlow['id'][],
        source: 'direct' | 'ses'
    ): Promise<Record<HogFlow['id'], HogFlow | null>> {
        const [onReplica, hogFunctions] = await Promise.all([
            this.hogFlowManager.getHogFlows(ids),
            this.hogFunctionManager.getHogFunctions(ids).catch((error): Record<string, HogFunctionType | null> => {
                logger.warn('[EmailTrackingService] Failed to load hog functions', { error })
                return {}
            }),
        ])
        const unknown = ids.filter((id) => !onReplica[id] && !hogFunctions[id])
        const onPrimary: Record<HogFlow['id'], HogFlow | null> =
            unknown.length > 0
                ? await this.hogFlowManager.getHogFlowsCheckingPrimaryOnMiss(unknown).catch((error) => {
                      logger.error('[EmailTrackingService] Failed to load hog flows from the primary', {
                          error,
                          ids: unknown,
                      })
                      emailTrackingErrorsCounter.inc({ error_type: 'hog_flow_lookup_failed', source })
                      return {}
                  })
                : {}
        return Object.fromEntries(ids.map((id) => [id, onReplica[id] ?? onPrimary[id] ?? null]))
    }

    // How many people this open or click adds to each per-person metric of its run. One email goes to one person, so
    // the email's invocation id stands for the person. A click also counts as an open, because the open pixel is often
    // blocked. A HyperLogLog stays small for any audience and is about 1% off. When Redis fails, the counts read low.
    private async newEngagedPeople(
        parentRunId: string,
        invocationId: string,
        metricName: MinimalAppMetric['metric_name'],
        source: 'direct' | 'ses'
    ): Promise<{ metric: MinimalAppMetric['metric_name']; count: number }[]> {
        const metrics = ENGAGEMENT_PERSON_METRICS[metricName]
        if (!metrics || !this.engagementRedis) {
            return []
        }
        const keys = metrics.flatMap((metric) => {
            const key = `@posthog/email-engagement-people/{${parentRunId}}/${metric}`
            return [key, `${key}/reported`]
        })
        let timer: NodeJS.Timeout | undefined
        const grew = await Promise.race([
            this.engagementRedis.useClient(
                { name: 'email-engagement-people', failOpen: true },
                (client) =>
                    client.eval(
                        ENGAGEMENT_PEOPLE_SCRIPT,
                        keys.length,
                        ...keys,
                        invocationId,
                        ENGAGEMENT_PEOPLE_TTL_SECONDS
                    ) as Promise<number[]>
            ),
            new Promise<null>((resolve) => {
                timer = setTimeout(() => resolve(null), ENGAGEMENT_REDIS_TIMEOUT_MS)
            }),
        ])
        clearTimeout(timer)
        if (!grew) {
            emailTrackingErrorsCounter.inc({ error_type: 'engagement_people_not_counted', source })
            return []
        }
        return metrics.map((metric, index) => ({ metric, count: grew[index] })).filter(({ count }) => count > 0)
    }

    private parseTrackingParams(query: Record<string, any>): {
        functionId?: string
        invocationId?: string
        actionId?: string
        parentRunId?: string
        distinctId?: string
        workflowVersion?: number
    } {
        // Support both combined ph_id format and legacy separate params
        if (query.ph_id) {
            const parsed = this.trackingCodeSigner.parse(query.ph_id as string)
            if (parsed) {
                trackingCodeFormatCounter.inc({ format: parsed.format, source: 'tracking' })
            }
            return {
                functionId: parsed?.functionId,
                invocationId: parsed?.invocationId,
                actionId: parsed?.actionId,
                parentRunId: parsed?.parentRunId,
                distinctId: parsed?.distinctId,
                workflowVersion: parsed?.workflowVersion,
            }
        }
        return {
            functionId: query.ph_fn_id as string | undefined,
            invocationId: query.ph_inv_id as string | undefined,
        }
    }

    // NOTE: this is somewhat naieve. We should expand with UA checking for things like apple's tracking prevention etc.
    // In production, opens are tracked via SES webhooks — recording here would double-count.
    // In dev/test (where maildev replaces SES and no webhooks come back), we fire the
    // engagement event from the pixel handler so local testing produces real events.
    public handleEmailTrackingPixel(req: ModifiedRequest, res: express.Response): void {
        res.status(200).set('Content-Type', 'image/gif').send(PIXEL_GIF)

        if (isDevEnv() || isTestEnv()) {
            const params = this.parseTrackingParams(req.query as Record<string, any>)
            void this.trackMetric({ ...params, metricName: 'email_opened', source: 'direct' }).catch((error) => {
                logger.error('[EmailTrackingService] handleEmailTrackingPixel: trackMetric failed', { error })
            })
        }
    }

    // Same rationale as handleEmailTrackingPixel: skip in production (SES webhooks own
    // click tracking), but emit in dev/test where maildev never produces webhooks.
    public handleEmailTrackingRedirect(req: ModifiedRequest, res: express.Response): void {
        const { target, ph_id, ph_sig } = req.query

        if (typeof target !== 'string' || !target) {
            res.status(404).send('Not found')
            return
        }

        const isSignedTarget =
            typeof ph_id === 'string' &&
            typeof ph_sig === 'string' &&
            this.trackingCodeSigner.verifyRedirectTarget(ph_id, target, ph_sig)

        if (isSignedTarget) {
            res.redirect(target)
        } else {
            const page = Date.now() < UNSIGNED_REDIRECT_SUNSET ? renderRedirectConfirmationPage(target) : null
            if (!page) {
                res.status(404).send('Not found')
                return
            }
            res.status(200).set(REDIRECT_CONFIRMATION_PAGE_HEADERS).send(page)
        }

        if (isDevEnv() || isTestEnv()) {
            const params = this.parseTrackingParams(req.query as Record<string, any>)
            void this.trackMetric({ ...params, metricName: 'email_link_clicked', source: 'direct' }).catch((error) => {
                logger.error('[EmailTrackingService] handleEmailTrackingRedirect: trackMetric failed', { error })
            })
        }
    }
}