Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

49.2 KB
Raw Blame
import crypto from 'node:crypto'
import addressparser from 'nodemailer/lib/addressparser'
import { Counter } from 'prom-client'
import { z } from 'zod'

import { MinimalAppMetric } from '~/cdp/types'
import { parseJSON } from '~/common/utils/json-parse'
import { logger } from '~/common/utils/logger'
import { fetch } from '~/common/utils/request'

import type { EmailStatus, EmailStatusReason } from '../email-status.service'
import { unquoteMailbox } from './suppressed-recipients'
import { EmailTrackingCodeSigner, TRACKING_CODE_HEADER_NAME, trackingCodeFormatCounter } from './tracking-code'

/**
 * ---------- SNS envelope types ----------
 * If raw_message_delivery=false (default), SNS wraps your message in this envelope.
 */
const SnsEnvelopeSchema = z.object({
    Type: z.enum(['SubscriptionConfirmation', 'Notification', 'UnsubscribeConfirmation']),
    MessageId: z.string(),
    Token: z.string().optional(),
    TopicArn: z.string(),
    Subject: z.string().optional(),
    Message: z.string(), // either SES event JSON (Notification) or human-readable confirmation text
    // Top-level on (Un)SubscribeConfirmation payloads and part of their signed string, so it must
    // survive schema parsing for signature verification to succeed.
    SubscribeURL: z.string().optional(),
    Timestamp: z.string(),
    SignatureVersion: z.enum(['1']),
    Signature: z.string(),
    SigningCertURL: z.string().url(),
    UnsubscribeURL: z.string().url().optional(),
})

export type SnsEnvelope = z.infer<typeof SnsEnvelopeSchema>

/**
 * ---------- SES event types ----------
 * AWS posts an array of records in the "Message" (or directly as body if raw_message_delivery=true).
 * We model the common fields and create specific event detail types.
 */
const SesMailSchema = z.object({
    timestamp: z.string(),
    source: z.string(), // From address
    messageId: z.string(),
    destination: z.array(z.string()),
    headersTruncated: z.boolean().optional(),
    headers: z.array(z.object({ name: z.string(), value: z.string() })).optional(),
    commonHeaders: z
        // A `to` of another shape only turns off the status update, never the whole record.
        .object({ subject: z.string().optional(), to: z.array(z.string()).nullish().catch(undefined) })
        .passthrough()
        .optional(),
    tags: z.record(z.string(), z.array(z.string())).optional(), // your message tags: { user_id: ["u_123"] }
})

const SesCommonEventBase = z.object({
    eventType: z.enum([
        'Send',
        'Reject',
        'Bounce',
        'Complaint',
        'Delivery',
        'Open',
        'Click',
        'RenderingFailure',
        'DeliveryDelay',
    ]),
    mail: SesMailSchema,
})

const SesOpenEventSchema = SesCommonEventBase.extend({
    eventType: z.literal('Open'),
    open: z.object({
        ipAddress: z.string().optional(),
        userAgent: z.string().optional(),
        timestamp: z.string(),
    }),
})

const SesClickEventSchema = SesCommonEventBase.extend({
    eventType: z.literal('Click'),
    click: z.object({
        ipAddress: z.string().optional(),
        link: z.string(),
        // Echoes the `ses:tags` we set on each anchor at send time, as {name: [value]}.
        // Some SES Click events send `null` here instead of omitting the field.
        linkTags: z.record(z.string(), z.array(z.string())).nullish(),
        userAgent: z.string().optional(),
        timestamp: z.string(),
    }),
})

const SesDeliveryEventSchema = SesCommonEventBase.extend({
    eventType: z.literal('Delivery'),
    delivery: z.object({
        processingTimeMillis: z.number().optional(),
        smtpResponse: z.string().optional(),
        reportingMTA: z.string().optional(),
        timestamp: z.string(),
        recipients: z.array(z.string()).optional(),
    }),
})

const SesBounceEventSchema = SesCommonEventBase.extend({
    eventType: z.literal('Bounce'),
    bounce: z.object({
        bounceType: z.enum(['Undetermined', 'Permanent', 'Transient']),
        bounceSubType: z.string().optional(),
        bouncedRecipients: z.array(
            z.object({
                emailAddress: z.string(),
                action: z.string().optional(),
                status: z.string().optional(),
                diagnosticCode: z.string().optional(),
            })
        ),
        timestamp: z.string(),
        reportingMTA: z.string().optional(),
    }),
})

// 5.1.1 (unknown user) and 5.1.10 (null MX). The diagnostic match is anchored after the reply code, because its
// free text can quote sender-chosen values such as a domain named 5.1.1.example.com.
const ADDRESS_DOES_NOT_EXIST_STATUS = /^5\.1\.(1|10)$/
const ADDRESS_DOES_NOT_EXIST_DIAGNOSTIC = /^(smtp;\s*)?5\d\d[\s-]+5\.1\.(1|10)(\s|$)/i

// SES files sender-side rejections such as DMARC failures under Permanent/General too, so General needs the status.
export function nonexistentRecipients(bounce: z.infer<typeof SesBounceEventSchema>['bounce']): string[] {
    if (bounce.bounceType !== 'Permanent') {
        return []
    }
    return bounce.bouncedRecipients
        .filter(
            (r) =>
                bounce.bounceSubType === 'NoEmail' ||
                (bounce.bounceSubType === 'General' &&
                    (r.status
                        ? ADDRESS_DOES_NOT_EXIST_STATUS.test(r.status.trim())
                        : ADDRESS_DOES_NOT_EXIST_DIAGNOSTIC.test((r.diagnosticCode ?? '').trim())))
        )
        .map((r) => r.emailAddress)
}

const SesComplaintEventSchema = SesCommonEventBase.extend({
    eventType: z.literal('Complaint'),
    complaint: z.object({
        complainedRecipients: z.array(z.object({ emailAddress: z.string() })),
        timestamp: z.string(),
        complaintFeedbackType: z.string().optional(),
        userAgent: z.string().optional(),
        feedbackId: z.string().optional(),
    }),
})

const SesRenderingFailureSchema = SesCommonEventBase.extend({
    eventType: z.literal('RenderingFailure'),
    renderingFailure: z.object({
        errorMessage: z.string(),
        templateName: z.string().optional(),
    }),
})

const SesSendEventSchema = SesCommonEventBase.extend({ eventType: z.literal('Send') })
const SesRejectEventSchema = SesCommonEventBase.extend({
    eventType: z.literal('Reject'),
    reject: z.object({ reason: z.string().optional() }).optional(),
})

// Acknowledged so SNS doesn't retry forever; details aren't surfaced.
const SesDeliveryDelayEventSchema = SesCommonEventBase.extend({ eventType: z.literal('DeliveryDelay') })

const SesEventRecordSchema = z.union([
    SesOpenEventSchema,
    SesClickEventSchema,
    SesDeliveryEventSchema,
    SesBounceEventSchema,
    SesComplaintEventSchema,
    SesRenderingFailureSchema,
    SesSendEventSchema,
    SesRejectEventSchema,
    SesDeliveryDelayEventSchema,
])

const SesEventBatchSchema = z.array(SesEventRecordSchema)

export type SesEventRecord = z.infer<typeof SesEventRecordSchema>

// email_sent is recorded synchronously in email.service.ts when the email is sent,
// so we don't record it again from SES Send webhooks to avoid double counting.
const BOUNCE_TYPE_TO_METRIC_NAME: Record<'Permanent' | 'Transient' | 'Undetermined', MinimalAppMetric['metric_name']> =
    {
        Permanent: 'email_bounced_hard',
        Transient: 'email_bounced_transient',
        Undetermined: 'email_bounced_undetermined',
    }

// The words senders use, rather than the provider's. This lands on a customer-facing event, where
// "Permanent" reads as a severity rather than as the reason the address will never accept mail.
const BOUNCE_TYPE_TO_EVENT_PROPERTY: Record<'Permanent' | 'Transient' | 'Undetermined', 'hard' | 'soft' | 'unknown'> = {
    Permanent: 'hard',
    Transient: 'soft',
    Undetermined: 'unknown',
}

const EVENT_TYPE_TO_METRIC_NAME: Partial<Record<SesEventRecord['eventType'], MinimalAppMetric['metric_name']>> = {
    Open: 'email_opened',
    Click: 'email_link_clicked',
    Delivery: 'email_delivered',
    Bounce: 'email_bounced',
    // A Complaint is a recipient's "report spam" relayed through the provider's feedback loop.
    // It stays recorded under email_blocked because complaint counts have always been stored
    // there; the UI surfaces the metric as "Marked as spam".
    Complaint: 'email_blocked',
    RenderingFailure: 'email_failed',
    Reject: 'email_failed',
}

export type SesEventLogLine = {
    level: 'warn' | 'error'
    message: string
}

// Records with no usable tracking code can't be attributed to a workflow, so they're dropped. A
// silent drop hides systemic loss, so count what's dropped, by event type and reason. `reason`
// separates two different failures with different owners: `missing` (no carrier value at all, for
// example a configuration set that stops emitting the original headers or tags) from `invalid` (a
// carrier value was present but did not parse, for example a signed header left over after a
// signing key rotated out).
const sesUnattributedEventsCounter = new Counter({
    name: 'email_tracking_unattributed_total',
    help: 'SES webhook records dropped because their tracking code was missing or unparseable',
    labelNames: ['event_type', 'reason'],
})

// Kept off `email_tracking_errors_total` because the SES webhook errors alert sums that counter, and an
// SNS message that is not an SES event is acknowledged, not a fault.
const sesIgnoredNotificationsCounter = new Counter({
    name: 'ses_webhook_ignored_notifications_total',
    help: 'SNS notifications on the SES webhook acknowledged without processing',
    labelNames: ['reason'],
})

// SES link tag naming an anchor's ordinal position in the email body. Set on each `<a>` at send
// time and echoed back on the Click event, which is what lets two anchors pointing at the same URL
// (a header logo and a footer link, say) be counted separately. SES restricts tag names and values
// to alphanumerics, hyphens and underscores, so an integer index is the only stable identifier
// cheap enough to carry here.
export const SES_LINK_INDEX_TAG = 'phl'

// SES message tag set at send time on emails whose template is marked sensitive, and echoed back on
// every event for that message. Such an email's subject and link paths can carry a login code or a
// reset token, so its events keep no subject and only each clicked link's origin. `$link_index` still
// tells the links apart.
export const SES_SENSITIVE_TAG = 'ph_sensitive'

// SES message tag set at send time on transactional emails, the only emails whose status is recorded,
// and echoed back on every event for that message. SES allows only ASCII letters, digits, underscores
// and dashes in a tag, so the value is 'true' as for SES_SENSITIVE_TAG.
export const SES_EMAIL_STATUS_TAG = 'ph_email_status'

export type SesEmailStatusUpdate = {
    teamId: string
    invocationId: string
    status: EmailStatus
    reason?: EmailStatusReason
    timestamp: string
}

const normalizeAddress = (address: string): string => unquoteMailbox(address).trim().toLowerCase()

// SES reports each entry as written at send time, so a display name can wrap the address.
const mailboxAddresses = (entries: string[]): Set<string> =>
    new Set(
        entries.flatMap((entry) =>
            addressparser(entry, { flatten: true }).map((parsed) => normalizeAddress(parsed.address))
        )
    )

const toHeaderAddresses = (rec: SesEventRecord): Set<string> => mailboxAddresses(rec.mail.commonHeaders?.to ?? [])

const sesRecipientsNotSentToCounter = new Counter({
    name: 'email_tracking_recipients_not_sent_to_total',
    help: 'Bounce and complaint recipients left out of suppression because the email was not sent to them',
    labelNames: ['event_type'],
})

// SES reports a bounce or complaint recipient from the receiving server's report, which can name any address.
// Only an address this send went to may change suppression state.
const sentToRecipients = <T extends { emailAddress: string }>(rec: SesEventRecord, recipients: T[]): T[] => {
    const sentTo = mailboxAddresses(rec.mail.destination)
    const kept = recipients.filter((r) => sentTo.has(normalizeAddress(r.emailAddress)))
    if (kept.length < recipients.length) {
        sesRecipientsNotSentToCounter.inc({ event_type: rec.eventType }, recipients.length - kept.length)
    }
    return kept
}

/**
 * The status change a Delivery, Bounce, Complaint or Reject record makes to the email it reports on, or the
 * open or click an Open or Click record adds to it, or null.
 *
 * The status describes the email to its To recipient. SES reports cc and bcc copies under the same
 * message, so a record counts only when one of its recipients is in the To header.
 */
const resolveEmailStatusUpdate = (
    rec: SesEventRecord
): Pick<SesEmailStatusUpdate, 'status' | 'reason' | 'timestamp'> | null => {
    const to = toHeaderAddresses(rec)
    const reachesTo = (addresses: string[]): boolean => addresses.some((address) => to.has(normalizeAddress(address)))
    switch (rec.eventType) {
        case 'Delivery':
            return reachesTo(rec.delivery.recipients ?? [])
                ? { status: 'delivered', timestamp: rec.delivery.timestamp }
                : null
        case 'Bounce':
            return reachesTo(rec.bounce.bouncedRecipients.map((r) => r.emailAddress))
                ? {
                      status: 'bounced',
                      reason: BOUNCE_TYPE_TO_EVENT_PROPERTY[rec.bounce.bounceType],
                      timestamp: rec.bounce.timestamp,
                  }
                : null
        case 'Complaint':
            // A "not-spam" feedback type corrects an earlier spam report, so it is not a complaint.
            return rec.complaint.complaintFeedbackType?.toLowerCase() !== 'not-spam' &&
                reachesTo(rec.complaint.complainedRecipients.map((r) => r.emailAddress))
                ? { status: 'complained', timestamp: rec.complaint.timestamp }
                : null
        case 'Reject':
            // SES refuses the whole message, so the email fails for its To recipient too.
            return { status: 'failed', reason: 'provider_rejected', timestamp: rec.mail.timestamp }
        case 'Open':
            return { status: 'opened', timestamp: rec.open.timestamp }
        case 'Click':
            return { status: 'clicked', timestamp: rec.click.timestamp }
        default:
            return null
    }
}

const sensitiveLinkOrigin = (link: string): string => {
    try {
        const url = new URL(link)
        return url.host ? `${url.protocol}//${url.host}` : url.protocol
    } catch {
        return ''
    }
}

// Path of our own click-tracking redirect, which used to wrap every anchor before SES saw it.
const TRACKING_REDIRECT_PATH = '/public/m/redirect'

/**
 * The destination the recipient actually asked for.
 *
 * SES reports the link as it appeared in the HTML, before SES rewrote it. Messages sent while we
 * still wrapped anchors in our own redirect therefore report that wrapper, whose `ph_id` differs on
 * every send, so the raw value is both unaggregatable and a carrier for a signed tracking code that
 * should not be stored on an event. Unwrap it back to `target`. Anything else is passed through.
 */
export const resolveClickDestination = (link: string): string => {
    try {
        const url = new URL(link)
        if (!url.pathname.endsWith(TRACKING_REDIRECT_PATH)) {
            return link
        }
        return url.searchParams.get('target') || link
    } catch {
        return link
    }
}

// Bounds how much of a URL rides in the app_metrics2 sort key. Long enough to stay recognizable in
// the UI, short enough that a pathological query-less URL can't bloat the index.
const MAX_LINK_URL_LENGTH = 200

/**
 * Collapses a clicked URL to the identity we count by.
 *
 * The query string and fragment are dropped because merge tags and per-recipient UTM values make
 * them differ on every send, which would scatter one link's clicks across thousands of buckets. Two
 * anchors that genuinely point at the same path stay distinguishable through the link index, which
 * is carried alongside this value.
 */
// Templates routinely build hrefs like `/unsubscribe/{{token}}` or `/users/12345`, so a path segment
// is often a per-recipient identifier rather than part of the page's identity. Collapsing those is
// what keeps the metrics sort key bounded by pages rather than by audience size, and keeps recipient
// tokens out of a store the whole team can read.
//
// A token is recognized by its runs of unbroken alphanumerics rather than by the segment as a whole,
// because tokens arrive glued together with punctuation: a JWT is dot-separated, and base64url
// tokens carry `-` and `_`. Testing runs also stops long slugs from being mistaken for tokens, since
// `how-to-set-up-feature-flags` is only ever short words once split.
const UUID_SEGMENT = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i
const TOKEN_RUN_SEPARATORS = /[.\-_~]/
const HEX_RUN = /^[0-9a-f]{12,}$/i
const NUMERIC_RUN = /^[0-9]{6,}$/
const SINGLE_CASE_WORD = /^([a-z]+|[A-Z]+)$/
// Below this, mixed-case runs are still plausibly words (`Introduction`, `CHANGELOG`).
const MIN_RANDOM_RUN_LENGTH = 16
const OPAQUE_PATH_SEGMENT_PLACEHOLDER = '*'

const isOpaqueRun = (run: string): boolean =>
    HEX_RUN.test(run) || NUMERIC_RUN.test(run) || (run.length >= MIN_RANDOM_RUN_LENGTH && !SINGLE_CASE_WORD.test(run))

const isOpaquePathSegment = (segment: string): boolean => {
    // Percent-encoding would otherwise hide a token's shape behind `%`, which none of the runs match.
    let decoded = segment
    try {
        decoded = decodeURIComponent(segment)
    } catch {
        // Malformed escapes are left as-is; the raw segment is still worth testing.
    }
    return UUID_SEGMENT.test(decoded) || decoded.split(TOKEN_RUN_SEPARATORS).some(isOpaqueRun)
}

const redactOpaquePathSegments = (pathname: string): string =>
    pathname
        .split('/')
        .map((segment) => (isOpaquePathSegment(segment) ? OPAQUE_PATH_SEGMENT_PLACEHOLDER : segment))
        .join('/')

export const normalizeClickUrl = (link: string): string => {
    try {
        const url = new URL(link)
        // `new URL` also parses hostless schemes (mailto:, tel:, sms:), where `host` is empty and
        // rebuilding as `scheme://host+path` would invent an authority — `mailto:x` becoming
        // `mailto://x`. Only http(s)-shaped links have a host to normalize; the rest pass through.
        if (!url.host) {
            return link.slice(0, MAX_LINK_URL_LENGTH)
        }
        const path = redactOpaquePathSegments(url.pathname)
        return `${url.protocol}//${url.host}${path}`.replace(/\/$/, '').slice(0, MAX_LINK_URL_LENGTH)
    } catch {
        return link.slice(0, MAX_LINK_URL_LENGTH)
    }
}

// Separator between the parts of a per-link metric's `instance_id`. The frontend splits on this to
// recover (action id, link index, url); see LINK_INSTANCE_SEPARATOR in workflowMetricsSummaryLogic.
// A URL may legitimately contain this character, so parsers must limit themselves to the first two
// separators and treat the remainder as the URL.
export const LINK_INSTANCE_SEPARATOR = '|'

export const buildLinkInstanceId = (actionId: string, linkIndex: string, normalizedUrl: string): string =>
    [actionId, linkIndex, normalizedUrl].join(LINK_INSTANCE_SEPARATOR)

const MAX_SES_FIELD_LENGTH = 1024

// Strip control chars, neutralize rich-log bracket tokens, and cap length.
const sanitizeSesField = (value: string, max = MAX_SES_FIELD_LENGTH): string => {
    const cleaned = value
        .replace(/[\u0000-\u001f\u007f]/g, ' ')
        .replace(/\[/g, '(')
        .replace(/\]/g, ')')
    return cleaned.length > max ? cleaned.slice(0, max) + '…' : cleaned
}

// Only warn/error events become log entries. Info-level events (Send, Delivery, Open, Click)
// are tracked as metrics only - they're analytics signal, not debugging signal.
export const formatSesEventLogs = (rec: SesEventRecord): SesEventLogLine[] => {
    switch (rec.eventType) {
        case 'Bounce': {
            const bounceType = rec.bounce.bounceType
            const level: SesEventLogLine['level'] = bounceType === 'Permanent' ? 'error' : 'warn'
            return rec.bounce.bouncedRecipients.map((r) => {
                const diag = r.diagnosticCode ? sanitizeSesField(r.diagnosticCode) : ''
                // SES typically inlines the SMTP status inside diagnosticCode already
                // (e.g. "smtp; 550 5.1.1 user unknown"), so skip the suffix when present.
                const statusSuffix = r.status && !diag.includes(r.status) ? ` (${sanitizeSesField(r.status)})` : ''
                const diagPart = diag ? `, ${diag}` : ''
                return {
                    level,
                    message: `${bounceType} bounce to ${sanitizeSesField(r.emailAddress)}${diagPart}${statusSuffix}`,
                }
            })
        }
        case 'Complaint': {
            const feedback = rec.complaint.complaintFeedbackType
                ? `, feedback type: ${sanitizeSesField(rec.complaint.complaintFeedbackType)}`
                : ''
            return rec.complaint.complainedRecipients.map((r) => ({
                level: 'warn',
                message: `Complaint from ${sanitizeSesField(r.emailAddress)}${feedback}`,
            }))
        }
        case 'RenderingFailure': {
            const tmpl = rec.renderingFailure.templateName
                ? ` for template ${sanitizeSesField(rec.renderingFailure.templateName)}`
                : ''
            return [
                {
                    level: 'error',
                    message: `Rendering failure${tmpl}: ${sanitizeSesField(rec.renderingFailure.errorMessage)}`,
                },
            ]
        }
        case 'Reject': {
            const reason = rec.reject?.reason ? `: ${sanitizeSesField(rec.reject.reason)}` : ''
            return [
                {
                    level: 'error',
                    message: `Message rejected by SES${reason}`,
                },
            ]
        }
        default:
            return []
    }
}

// Only text that does not open like JSON is quoted, because a malformed SES event carries recipients.
// Any AWS account can send it when the topic allowlist is empty, so it is sanitized like other SES fields.
const plainTextPrefix = (message: string): string | undefined =>
    /^\s*[{[]/.test(message) ? undefined : sanitizeSesField(message, 200)

export class SesWebhookHandler {
    certCache: Record<string, Promise<string> | undefined> = {}
    private allowedTopicArns: Set<string>

    constructor(
        private trackingCodeSigner: EmailTrackingCodeSigner,
        // Empty set means no restriction (dev/test); prod is expected to configure the workflow SES
        // topic ARN.
        allowedTopicArns: string[] = []
    ) {
        this.allowedTopicArns = new Set(allowedTopicArns.map((arn) => arn.trim()).filter(Boolean))
    }

    private isTopicAllowed(topicArn: string | undefined): boolean {
        if (this.allowedTopicArns.size === 0) {
            return true
        }
        return typeof topicArn === 'string' && this.allowedTopicArns.has(topicArn)
    }

    private async fetchText(url: string): Promise<string> {
        const response = await fetch(url)
        if (response.status >= 400) {
            throw new Error(`Failed to fetch ${url}: ${response.status}`)
        }
        return await response.text()
    }

    private async fetchCert(url: string): Promise<string> {
        // Validate that the URL is from AWS SNS
        if (!this.isValidSnsCertUrl(url)) {
            throw new Error(`Invalid SNS certificate URL: ${url}`)
        }

        if (this.certCache[url]) {
            return await this.certCache[url]!
        }
        this.certCache[url] = this.fetchText(url)
        return this.certCache[url]!
    }

    private isValidSnsCertUrl(url: string): boolean {
        try {
            const parsedUrl = new URL(url)

            // Must be HTTPS
            if (parsedUrl.protocol !== 'https:') {
                return false
            }

            // Must be from sns.{region}.amazonaws.com
            const hostname = parsedUrl.hostname
            if (!hostname.match(/^sns\.[a-z0-9-]+\.amazonaws\.com$/)) {
                return false
            }

            // Must end with .pem
            if (!parsedUrl.pathname.endsWith('.pem')) {
                return false
            }

            return true
        } catch {
            return false
        }
    }

    private isValidSnsSubscribeUrl(url: string): boolean {
        try {
            const parsedUrl = new URL(url)

            if (parsedUrl.protocol !== 'https:') {
                return false
            }

            // Must be from sns.{region}.amazonaws.com
            if (!parsedUrl.hostname.match(/^sns\.[a-z0-9-]+\.amazonaws\.com$/)) {
                return false
            }

            return true
        } catch {
            return false
        }
    }

    /**
     * Parse incoming body accounting for SNS raw vs envelope
     */
    private parseIncomingBody(
        body: unknown
    ): { mode: 'raw'; records: SesEventRecord[] } | { mode: 'sns'; envelope: SnsEnvelope } {
        // If it's already an object with "Type", it's probably the SNS envelope (raw=false)
        if (body && typeof body === 'object' && 'Type' in (body as any)) {
            return { mode: 'sns', envelope: SnsEnvelopeSchema.parse(body) }
        }

        // raw_message_delivery=true → body is already the SES array
        const records = SesEventBatchSchema.parse(body)
        return { mode: 'raw', records }
    }

    private parseNotificationRecords(envelope: SnsEnvelope): SesEventRecord[] {
        let message: unknown
        try {
            message = parseJSON(envelope.Message)
        } catch {
            sesIgnoredNotificationsCounter.inc({ reason: 'non_json_message' })
            logger.warn('[SesWebhookHandler] Ignoring Notification whose Message is not JSON', {
                topicArn: envelope.TopicArn,
                messageId: envelope.MessageId,
                messageLength: envelope.Message.length,
                messagePrefix: plainTextPrefix(envelope.Message),
            })
            return []
        }
        return [SesEventRecordSchema.parse(message)]
    }

    /**
     * Verify SNS signature
     * Best practice: verify the signature unless you're behind a trusted ALB / private VPC.
     */
    private async verifySnsSignature(envelope: SnsEnvelope): Promise<boolean> {
        try {
            // 1) Fetch cert
            const cert = await this.fetchCert(envelope.SigningCertURL)
            // 2) Build string to sign (per SNS docs)
            const stringToSign = this.buildStringToSign(envelope)
            // 3) Verify
            const verifier = crypto.createVerify('RSA-SHA1') // SNS SignatureVersion=1 uses SHA1
            verifier.update(stringToSign, 'utf8')
            return verifier.verify(cert, envelope.Signature, 'base64')
        } catch {
            return false
        }
    }

    /**
     * Build string to sign for SNS signature verification
     */
    private buildStringToSign(m: SnsEnvelope): string {
        // Follows AWS SNS docs for SignatureVersion=1
        // For Notification:
        //   "Message\n{Message}\nMessageId\n{MessageId}\nSubject\n{Subject}\nTimestamp\n{Timestamp}\nTopicArn\n{TopicArn}\nType\n{Type}\n"
        // Subject line omitted if not present.
        const lines: string[] = []
        const pushKV = (k: string, v?: string) => {
            if (v !== undefined) {
                lines.push(k)
                lines.push(v)
            }
        }
        if (m.Type === 'Notification') {
            pushKV('Message', m.Message)
            pushKV('MessageId', m.MessageId)
            pushKV('Subject', m.Subject)
            pushKV('Timestamp', m.Timestamp)
            pushKV('TopicArn', m.TopicArn)
            pushKV('Type', m.Type)
        } else if (m.Type === 'SubscriptionConfirmation' || m.Type === 'UnsubscribeConfirmation') {
            pushKV('Message', m.Message)
            pushKV('MessageId', m.MessageId)
            pushKV('SubscribeURL', m.SubscribeURL)
            pushKV('Timestamp', m.Timestamp)
            pushKV('Token', m.Token!)
            pushKV('TopicArn', m.TopicArn)
            pushKV('Type', m.Type)
        }
        return lines.join('\n') + '\n'
    }

    async handleWebhook(opts: {
        body: any
        headers: Record<string, string | string[] | undefined>
        verifySignature?: boolean
    }): Promise<{
        status: number
        body: unknown
        metrics?: {
            functionId?: string
            invocationId?: string
            actionId?: string
            parentRunId?: string
            distinctId?: string
            teamId?: string
            metricName: MinimalAppMetric['metric_name']
            // Replaces the instance_id trackMetric would derive, for metrics keyed by something
            // other than the action or invocation (see buildLinkInstanceId).
            instanceIdOverride?: string
            properties?: Record<string, any>
            timestamp?: string
            workflowVersion?: number
        }[]
        logEntries?: {
            functionId?: string
            invocationId?: string
            actionId?: string
            parentRunId?: string
            teamId?: string
            level: SesEventLogLine['level']
            message: string
        }[]
        // Soft (Transient) bounces — fed into the suppression list's consecutive-bounce counter.
        transientBounceRecipients?: {
            teamId?: string
            emailAddresses: string[]
            diagnostic?: string
        }[]
        // Hard (Permanent) bounces — recorded in the suppression list so future sends to the address
        // are blocked. Suppression enforcement replaces the legacy opt-out-on-bounce write.
        hardBounceRecipients?: {
            teamId?: string
            emailAddresses: string[]
            diagnostic?: string
            // Addresses the bounce proves do not exist, for the global hard-bounce list.
            nonexistentAddresses?: string[]
            bounceTimestamp?: string
        }[]
        // Spam complaints — recorded in the suppression list so future sends to the address are
        // blocked. A complaint is the recipient asking us to stop, and provider complaint-rate
        // thresholds are far tighter than bounce thresholds.
        complainedRecipients?: {
            teamId?: string
            emailAddresses: string[]
            feedbackType?: string
        }[]
        // Successful deliveries — reset the suppression counter so transient outages don't accumulate.
        // Timestamp is threaded through so the reset ignores an out-of-order delivery from an older send
        // arriving after a newer bounce.
        deliveredRecipients?: {
            teamId?: string
            emailAddresses: string[]
            timestamp?: string
            clearsGlobalHardBounces?: boolean
        }[]
        // Status changes of transactional emails, for the email_status table.
        emailStatusUpdates?: SesEmailStatusUpdate[]
    }> {
        // Payloads carry recipients and clicked links, including reset links from sensitive emails,
        // so the logs keep identifiers only.
        const parsed = this.parseIncomingBody(opts.body)
        logger.info('[SesWebhookHandler] handleWebhook', { mode: parsed.mode })

        // When verification is required the message must be a signed SNS envelope. Raw deliveries
        // carry no signature, and prod uses envelope delivery, so reject them to block forged events.
        if (opts.verifySignature) {
            if (!('envelope' in parsed)) {
                return { status: 403, body: { error: 'Unsigned raw delivery not allowed' } }
            }
            logger.info('[SesWebhookHandler] verifying signature', {
                type: parsed.envelope.Type,
                messageId: parsed.envelope.MessageId,
            })
            const ok = await this.verifySnsSignature(parsed.envelope)
            logger.info('[SesWebhookHandler] signature verified', { ok })
            if (!ok) {
                return { status: 403, body: { error: 'Invalid SNS signature' } }
            }
        }

        // Restrict accepted SNS envelopes (Notification, SubscriptionConfirmation, UnsubscribeConfirmation)
        // to the configured TopicArn allowlist. SNS signature verification proves the message came from
        // AWS, but not from a specific topic — this narrows accepted sources to the workflow SES topic.
        if ('envelope' in parsed && !this.isTopicAllowed(parsed.envelope.TopicArn)) {
            logger.warn('[SesWebhookHandler] Rejecting event from disallowed TopicArn', {
                topicArn: parsed.envelope.TopicArn,
            })
            return { status: 403, body: { error: 'SNS TopicArn not allowed' } }
        }

        // Handle confirmation flow
        if (parsed.mode === 'sns' && parsed.envelope.Type === 'SubscriptionConfirmation') {
            logger.info('[SesWebhookHandler] confirming subscription', { envelope: parsed.envelope })
            // Confirm by visiting the top-level SubscribeURL field; Message is human-readable text here.
            const subscribeUrl = parsed.envelope.SubscribeURL
            if (subscribeUrl) {
                if (!this.isValidSnsSubscribeUrl(subscribeUrl)) {
                    logger.warn('[SesWebhookHandler] Invalid SubscribeURL, rejecting', {
                        url: subscribeUrl,
                    })
                    return { status: 403, body: { error: 'Invalid SubscribeURL' } }
                }
                await this.fetchText(subscribeUrl)
            } else {
                logger.warn('[SesWebhookHandler] SubscriptionConfirmation without SubscribeURL - cannot confirm')
                return { status: 400, body: { error: 'Missing SubscribeURL' } }
            }
            return { status: 200, body: { ok: true } }
        }

        if (parsed.mode === 'sns' && parsed.envelope.Type === 'UnsubscribeConfirmation') {
            logger.info('[SesWebhookHandler] confirming unsubscribe', { envelope: parsed.envelope })
            return { status: 200, body: { ok: true } }
        }

        const records = parsed.mode === 'raw' ? parsed.records : this.parseNotificationRecords(parsed.envelope)

        const metrics: {
            functionId?: string
            invocationId?: string
            actionId?: string
            parentRunId?: string
            distinctId?: string
            teamId?: string
            metricName: MinimalAppMetric['metric_name']
            instanceIdOverride?: string
            properties?: Record<string, any>
            timestamp?: string
            workflowVersion?: number
        }[] = []
        const logEntries: {
            functionId?: string
            invocationId?: string
            actionId?: string
            parentRunId?: string
            teamId?: string
            level: SesEventLogLine['level']
            message: string
        }[] = []
        const transientBounceRecipients: {
            teamId?: string
            emailAddresses: string[]
            diagnostic?: string
        }[] = []
        const hardBounceRecipients: {
            teamId?: string
            emailAddresses: string[]
            diagnostic?: string
            nonexistentAddresses?: string[]
            bounceTimestamp?: string
        }[] = []
        const complainedRecipients: {
            teamId?: string
            emailAddresses: string[]
            feedbackType?: string
        }[] = []
        const deliveredRecipients: {
            teamId?: string
            emailAddresses: string[]
            timestamp?: string
            clearsGlobalHardBounces?: boolean
        }[] = []
        // A signature only proves AWS sent the event, from any account's topic. The global hard-bounce list
        // affects every team, so it only takes events from a configured topic.
        const fromTrustedTopic = 'envelope' in parsed && this.allowedTopicArns.size > 0
        const emailStatusUpdates: SesEmailStatusUpdate[] = []

        for (const rec of records) {
            logger.info('[SesWebhookHandler] processing record', {
                eventType: rec.eventType,
                messageId: rec.mail.messageId,
            })
            // Prefer the custom MIME header (carries the full signed code including distinct_id,
            // unbounded). Fall back to the SES EmailTag for messages sent before the header carrier
            // was added, or where the configuration set hasn't enabled `IncludeOriginalHeaders`.
            const headerValue = rec.mail.headers?.find(
                (h) => h.name.toLowerCase() === TRACKING_CODE_HEADER_NAME.toLowerCase()
            )?.value
            const tagValue = rec.mail.tags?.ph_id?.[0]
            const rawCode = headerValue ?? tagValue ?? ''
            const parsedCode = this.trackingCodeSigner.parse(rawCode)
            if (parsedCode) {
                trackingCodeFormatCounter.inc({ format: parsedCode.format, source: 'ses' })
            }
            const { functionId, invocationId, teamId, actionId, parentRunId, distinctId, isTest, workflowVersion } =
                parsedCode || {}

            if (!functionId && !invocationId) {
                // A present but unparseable code is a different incident from a wholly absent one, so
                // record which occurred. An empty `rawCode` means neither carrier held a value.
                const reason = rawCode === '' ? 'missing' : 'invalid'
                sesUnattributedEventsCounter.inc({ event_type: rec.eventType, reason })
                logger.error('[SesWebhookHandler] handleWebhook: No functionId or invocationId found', {
                    eventType: rec.eventType,
                    messageId: rec.mail.messageId,
                    reason,
                })
                continue
            }

            const metricName = EVENT_TYPE_TO_METRIC_NAME[rec.eventType]
            // Test sends (from the editor's "Run test") are not production activity, so we skip
            // their delivery/open/click metrics — otherwise a draft/never-enabled workflow shows
            // email activity in its Metrics tab. Bounce-driven opt-outs below still apply.
            if (metricName && !isTest) {
                const isSensitive = rec.mail.tags?.[SES_SENSITIVE_TAG]?.[0] === 'true'
                const properties: Record<string, any> = {
                    $email_to: rec.mail.destination?.[0],
                    ...(isSensitive ? {} : { $email_subject: rec.mail.commonHeaders?.subject }),
                }

                // Each SES event detail carries its own timestamp (open.timestamp, click.timestamp, etc.)
                // — prefer those over the webhook receipt time so the event reflects when the action
                // actually happened, not when AWS got around to delivering the notification.
                let timestamp: string | undefined
                if ('open' in rec && rec.open) {
                    timestamp = rec.open.timestamp
                } else if ('click' in rec && rec.click) {
                    timestamp = rec.click.timestamp
                    const destination = resolveClickDestination(rec.click.link)
                    properties.$link_url = isSensitive ? sensitiveLinkOrigin(destination) : destination
                    const linkIndex = rec.click.linkTags?.[SES_LINK_INDEX_TAG]?.[0]
                    if (linkIndex !== undefined) {
                        properties.$link_index = linkIndex
                    }
                } else if ('delivery' in rec && rec.delivery) {
                    timestamp = rec.delivery.timestamp
                } else if ('bounce' in rec && rec.bounce) {
                    timestamp = rec.bounce.timestamp
                    // Whether the address will ever accept mail again. A hard bounce is also written
                    // to the suppression list below; a soft one is not, so the two need telling apart
                    // to read a bounce rate at all.
                    properties.$bounce_type = BOUNCE_TYPE_TO_EVENT_PROPERTY[rec.bounce.bounceType]
                    if (rec.bounce.bounceSubType) {
                        // The provider's own reason (e.g. "MailboxFull", "NoEmail", "Suppressed").
                        properties.$bounce_sub_type = rec.bounce.bounceSubType
                    }
                } else if ('complaint' in rec && rec.complaint) {
                    timestamp = rec.complaint.timestamp
                    if (rec.complaint.complaintFeedbackType) {
                        // The provider's classification of the report (e.g. "abuse", "fraud").
                        properties.$complaint_feedback_type = rec.complaint.complaintFeedbackType
                    }
                }
                timestamp = timestamp ?? rec.mail.timestamp

                metrics.push({
                    functionId,
                    invocationId,
                    actionId,
                    parentRunId,
                    distinctId,
                    teamId,
                    metricName,
                    properties,
                    timestamp,
                    workflowVersion,
                })

                // Per-link companion row for the workflow Metrics tab's link breakdown. Emitted only
                // when the send came from a workflow step: without an action id the key would fall
                // back to the invocation id, producing a row per send per link rather than an
                // aggregate, and the breakdown is per-action anyway. It carries its own metric name
                // so the existing `email_link_clicked` totals and trends can't double-count it.
                if (rec.eventType === 'Click' && actionId) {
                    metrics.push({
                        functionId,
                        invocationId,
                        actionId,
                        parentRunId,
                        distinctId,
                        teamId,
                        metricName: 'email_link_clicked_by_link',
                        instanceIdOverride: buildLinkInstanceId(
                            actionId,
                            rec.click.linkTags?.[SES_LINK_INDEX_TAG]?.[0] ?? '',
                            isSensitive
                                ? sensitiveLinkOrigin(resolveClickDestination(rec.click.link))
                                : normalizeClickUrl(resolveClickDestination(rec.click.link))
                        ),
                        timestamp,
                        workflowVersion,
                    })
                }

                // email_bounced stays the catch-all rollup; each bounce additionally emits a
                // per-type sub-metric (hard + transient + undetermined = email_bounced). AWS's
                // account bounce rate — what the email reputation feature calibrates against —
                // counts only hard (Permanent) bounces, so reputation reads email_bounced_hard;
                // the others exist for diagnosis (transient spikes are a leading indicator).
                if (rec.eventType === 'Bounce') {
                    metrics.push({
                        functionId,
                        invocationId,
                        actionId,
                        parentRunId,
                        distinctId,
                        teamId,
                        metricName: BOUNCE_TYPE_TO_METRIC_NAME[rec.bounce.bounceType],
                        properties,
                        timestamp,
                        workflowVersion,
                    })
                }
            }

            // Allowlist actionId before interpolating into the [Action:…] rich-log token,
            // since actionId comes from the attacker-influenceable ph_id tag.
            const safeActionId = actionId && /^[a-zA-Z0-9_-]+$/.test(actionId) ? actionId : undefined
            // Skip test sends here too (mirrors the metrics gate above) so a "Run test" bounce
            // doesn't write a misleading failure into the workflow's invocation logs.
            for (const line of isTest ? [] : formatSesEventLogs(rec)) {
                const prefix = safeActionId ? `[Action:${safeActionId}] ` : ''
                logEntries.push({
                    functionId,
                    invocationId,
                    actionId,
                    parentRunId,
                    teamId,
                    level: line.level,
                    message: `${prefix}${line.message}`,
                })
            }

            // State-changing writes (opt-out, suppression, delivery counter reset) require a signed
            // tracking code. The signed header carries the HMAC we mint on send; the unsigned SES-tag
            // carrier is only used for engagement metrics/log entries above.
            const codeIsTrusted = parsedCode?.format === 'signed'

            // Suppression writes (below) skip test sends — editor "Run test" traffic must not be
            // able to perturb production suppression state by targeting a bad recipient.
            const suppressionAllowed = teamId && codeIsTrusted && !isTest

            // Record permanent bounces in the suppression list so future sends to the address are
            // blocked. The SMTP diagnostic is threaded through for operator visibility on the row.
            if (suppressionAllowed && rec.eventType === 'Bounce' && rec.bounce.bounceType === 'Permanent') {
                const bounced = sentToRecipients(rec, rec.bounce.bouncedRecipients)
                if (bounced.length > 0) {
                    const diagnostic = bounced.find((r) => r.diagnosticCode)?.diagnosticCode
                    const nonexistentAddresses = fromTrustedTopic
                        ? nonexistentRecipients({ ...rec.bounce, bouncedRecipients: bounced })
                        : []
                    hardBounceRecipients.push({
                        teamId,
                        emailAddresses: bounced.map((r) => r.emailAddress),
                        diagnostic,
                        ...(nonexistentAddresses.length > 0
                            ? { nonexistentAddresses, bounceTimestamp: rec.bounce.timestamp }
                            : {}),
                    })
                }
            }

            // Count soft (Transient) bounces toward suppression. These are recipient-side failures
            // (server unreachable, mailbox full, greylisting); one is harmless but a persistent run
            // of them means the address can't receive mail.
            if (suppressionAllowed && rec.eventType === 'Bounce' && rec.bounce.bounceType === 'Transient') {
                const bounced = sentToRecipients(rec, rec.bounce.bouncedRecipients)
                if (bounced.length > 0) {
                    const diagnostic = bounced.find((r) => r.diagnosticCode)?.diagnosticCode
                    transientBounceRecipients.push({
                        teamId,
                        emailAddresses: bounced.map((r) => r.emailAddress),
                        diagnostic,
                    })
                }
            }

            // Record complaints in the suppression list so future sends to the address are blocked.
            // The recipient reported the message as spam — one complaint suppresses immediately,
            // like a hard bounce. Skip a "not-spam" feedback type, because that is a correction:
            // the recipient or their provider states the message is not spam, so it must not
            // suppress (and would otherwise resurrect a manually removed row). The value comes from
            // an external provider, so compare case-insensitively.
            if (
                suppressionAllowed &&
                rec.eventType === 'Complaint' &&
                rec.complaint.complaintFeedbackType?.toLowerCase() !== 'not-spam'
            ) {
                const complained = sentToRecipients(rec, rec.complaint.complainedRecipients)
                if (complained.length > 0) {
                    complainedRecipients.push({
                        teamId,
                        emailAddresses: complained.map((r) => r.emailAddress),
                        feedbackType: rec.complaint.complaintFeedbackType,
                    })
                }
            }

            // Successful delivery resets an address's soft-bounce counter — but only if newer than the
            // last-recorded bounce (checked at the SQL layer), so an out-of-order delivery from an
            // older send can't erase a fresh bounce.
            if (suppressionAllowed && rec.eventType === 'Delivery') {
                const emails = rec.delivery.recipients ?? rec.mail.destination ?? []
                if (emails.length > 0) {
                    deliveredRecipients.push({
                        teamId,
                        emailAddresses: emails,
                        timestamp: rec.delivery.timestamp,
                        ...(fromTrustedTopic ? { clearsGlobalHardBounces: true } : {}),
                    })
                }
            }

            // A status write is state-changing too, so it needs the signed code like the writes above.
            if (suppressionAllowed && invocationId && rec.mail.tags?.[SES_EMAIL_STATUS_TAG]?.[0] === 'true') {
                const update = resolveEmailStatusUpdate(rec)
                if (update) {
                    emailStatusUpdates.push({ teamId, invocationId, ...update })
                }
            }
        }

        return {
            status: 200,
            body: { ok: true },
            metrics,
            logEntries,
            transientBounceRecipients,
            hardBounceRecipients,
            complainedRecipients,
            deliveredRecipients,
            emailStatusUpdates,
        }
    }
}