:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import crypto from 'node:crypto'
import addressparser from 'nodemailer/lib/addressparser'
import { Counter } from 'prom-client'
import { z } from 'zod'
import { MinimalAppMetric } from '~/cdp/types'
import { parseJSON } from '~/common/utils/json-parse'
import { logger } from '~/common/utils/logger'
import { fetch } from '~/common/utils/request'
import type { EmailStatus, EmailStatusReason } from '../email-status.service'
import { unquoteMailbox } from './suppressed-recipients'
import { EmailTrackingCodeSigner, TRACKING_CODE_HEADER_NAME, trackingCodeFormatCounter } from './tracking-code'
/**
* ---------- SNS envelope types ----------
* If raw_message_delivery=false (default), SNS wraps your message in this envelope.
*/
const SnsEnvelopeSchema = z.object({
Type: z.enum(['SubscriptionConfirmation', 'Notification', 'UnsubscribeConfirmation']),
MessageId: z.string(),
Token: z.string().optional(),
TopicArn: z.string(),
Subject: z.string().optional(),
Message: z.string(), // either SES event JSON (Notification) or human-readable confirmation text
// Top-level on (Un)SubscribeConfirmation payloads and part of their signed string, so it must
// survive schema parsing for signature verification to succeed.
SubscribeURL: z.string().optional(),
Timestamp: z.string(),
SignatureVersion: z.enum(['1']),
Signature: z.string(),
SigningCertURL: z.string().url(),
UnsubscribeURL: z.string().url().optional(),
})
export type SnsEnvelope = z.infer<typeof SnsEnvelopeSchema>
/**
* ---------- SES event types ----------
* AWS posts an array of records in the "Message" (or directly as body if raw_message_delivery=true).
* We model the common fields and create specific event detail types.
*/
const SesMailSchema = z.object({
timestamp: z.string(),
source: z.string(), // From address
messageId: z.string(),
destination: z.array(z.string()),
headersTruncated: z.boolean().optional(),
headers: z.array(z.object({ name: z.string(), value: z.string() })).optional(),
commonHeaders: z
// A `to` of another shape only turns off the status update, never the whole record.
.object({ subject: z.string().optional(), to: z.array(z.string()).nullish().catch(undefined) })
.passthrough()
.optional(),
tags: z.record(z.string(), z.array(z.string())).optional(), // your message tags: { user_id: ["u_123"] }
})
const SesCommonEventBase = z.object({
eventType: z.enum([
'Send',
'Reject',
'Bounce',
'Complaint',
'Delivery',
'Open',
'Click',
'RenderingFailure',
'DeliveryDelay',
]),
mail: SesMailSchema,
})
const SesOpenEventSchema = SesCommonEventBase.extend({
eventType: z.literal('Open'),
open: z.object({
ipAddress: z.string().optional(),
userAgent: z.string().optional(),
timestamp: z.string(),
}),
})
const SesClickEventSchema = SesCommonEventBase.extend({
eventType: z.literal('Click'),
click: z.object({
ipAddress: z.string().optional(),
link: z.string(),
// Echoes the `ses:tags` we set on each anchor at send time, as {name: [value]}.
// Some SES Click events send `null` here instead of omitting the field.
linkTags: z.record(z.string(), z.array(z.string())).nullish(),
userAgent: z.string().optional(),
timestamp: z.string(),
}),
})
const SesDeliveryEventSchema = SesCommonEventBase.extend({
eventType: z.literal('Delivery'),
delivery: z.object({
processingTimeMillis: z.number().optional(),
smtpResponse: z.string().optional(),
reportingMTA: z.string().optional(),
timestamp: z.string(),
recipients: z.array(z.string()).optional(),
}),
})
const SesBounceEventSchema = SesCommonEventBase.extend({
eventType: z.literal('Bounce'),
bounce: z.object({
bounceType: z.enum(['Undetermined', 'Permanent', 'Transient']),
bounceSubType: z.string().optional(),
bouncedRecipients: z.array(
z.object({
emailAddress: z.string(),
action: z.string().optional(),
status: z.string().optional(),
diagnosticCode: z.string().optional(),
})
),
timestamp: z.string(),
reportingMTA: z.string().optional(),
}),
})
// 5.1.1 (unknown user) and 5.1.10 (null MX). The diagnostic match is anchored after the reply code, because its
// free text can quote sender-chosen values such as a domain named 5.1.1.example.com.
const ADDRESS_DOES_NOT_EXIST_STATUS = /^5\.1\.(1|10)$/
const ADDRESS_DOES_NOT_EXIST_DIAGNOSTIC = /^(smtp;\s*)?5\d\d[\s-]+5\.1\.(1|10)(\s|$)/i
// SES files sender-side rejections such as DMARC failures under Permanent/General too, so General needs the status.
export function nonexistentRecipients(bounce: z.infer<typeof SesBounceEventSchema>['bounce']): string[] {
if (bounce.bounceType !== 'Permanent') {
return []
}
return bounce.bouncedRecipients
.filter(
(r) =>
bounce.bounceSubType === 'NoEmail' ||
(bounce.bounceSubType === 'General' &&
(r.status
? ADDRESS_DOES_NOT_EXIST_STATUS.test(r.status.trim())
: ADDRESS_DOES_NOT_EXIST_DIAGNOSTIC.test((r.diagnosticCode ?? '').trim())))
)
.map((r) => r.emailAddress)
}
const SesComplaintEventSchema = SesCommonEventBase.extend({
eventType: z.literal('Complaint'),
complaint: z.object({
complainedRecipients: z.array(z.object({ emailAddress: z.string() })),
timestamp: z.string(),
complaintFeedbackType: z.string().optional(),
userAgent: z.string().optional(),
feedbackId: z.string().optional(),
}),
})
const SesRenderingFailureSchema = SesCommonEventBase.extend({
eventType: z.literal('RenderingFailure'),
renderingFailure: z.object({
errorMessage: z.string(),
templateName: z.string().optional(),
}),
})
const SesSendEventSchema = SesCommonEventBase.extend({ eventType: z.literal('Send') })
const SesRejectEventSchema = SesCommonEventBase.extend({
eventType: z.literal('Reject'),
reject: z.object({ reason: z.string().optional() }).optional(),
})
// Acknowledged so SNS doesn't retry forever; details aren't surfaced.
const SesDeliveryDelayEventSchema = SesCommonEventBase.extend({ eventType: z.literal('DeliveryDelay') })
const SesEventRecordSchema = z.union([
SesOpenEventSchema,
SesClickEventSchema,
SesDeliveryEventSchema,
SesBounceEventSchema,
SesComplaintEventSchema,
SesRenderingFailureSchema,
SesSendEventSchema,
SesRejectEventSchema,
SesDeliveryDelayEventSchema,
])
const SesEventBatchSchema = z.array(SesEventRecordSchema)
export type SesEventRecord = z.infer<typeof SesEventRecordSchema>
// email_sent is recorded synchronously in email.service.ts when the email is sent,
// so we don't record it again from SES Send webhooks to avoid double counting.
const BOUNCE_TYPE_TO_METRIC_NAME: Record<'Permanent' | 'Transient' | 'Undetermined', MinimalAppMetric['metric_name']> =
{
Permanent: 'email_bounced_hard',
Transient: 'email_bounced_transient',
Undetermined: 'email_bounced_undetermined',
}
// The words senders use, rather than the provider's. This lands on a customer-facing event, where
// "Permanent" reads as a severity rather than as the reason the address will never accept mail.
const BOUNCE_TYPE_TO_EVENT_PROPERTY: Record<'Permanent' | 'Transient' | 'Undetermined', 'hard' | 'soft' | 'unknown'> = {
Permanent: 'hard',
Transient: 'soft',
Undetermined: 'unknown',
}
const EVENT_TYPE_TO_METRIC_NAME: Partial<Record<SesEventRecord['eventType'], MinimalAppMetric['metric_name']>> = {
Open: 'email_opened',
Click: 'email_link_clicked',
Delivery: 'email_delivered',
Bounce: 'email_bounced',
// A Complaint is a recipient's "report spam" relayed through the provider's feedback loop.
// It stays recorded under email_blocked because complaint counts have always been stored
// there; the UI surfaces the metric as "Marked as spam".
Complaint: 'email_blocked',
RenderingFailure: 'email_failed',
Reject: 'email_failed',
}
export type SesEventLogLine = {
level: 'warn' | 'error'
message: string
}
// Records with no usable tracking code can't be attributed to a workflow, so they're dropped. A
// silent drop hides systemic loss, so count what's dropped, by event type and reason. `reason`
// separates two different failures with different owners: `missing` (no carrier value at all, for
// example a configuration set that stops emitting the original headers or tags) from `invalid` (a
// carrier value was present but did not parse, for example a signed header left over after a
// signing key rotated out).
const sesUnattributedEventsCounter = new Counter({
name: 'email_tracking_unattributed_total',
help: 'SES webhook records dropped because their tracking code was missing or unparseable',
labelNames: ['event_type', 'reason'],
})
// Kept off `email_tracking_errors_total` because the SES webhook errors alert sums that counter, and an
// SNS message that is not an SES event is acknowledged, not a fault.
const sesIgnoredNotificationsCounter = new Counter({
name: 'ses_webhook_ignored_notifications_total',
help: 'SNS notifications on the SES webhook acknowledged without processing',
labelNames: ['reason'],
})
// SES link tag naming an anchor's ordinal position in the email body. Set on each `<a>` at send
// time and echoed back on the Click event, which is what lets two anchors pointing at the same URL
// (a header logo and a footer link, say) be counted separately. SES restricts tag names and values
// to alphanumerics, hyphens and underscores, so an integer index is the only stable identifier
// cheap enough to carry here.
export const SES_LINK_INDEX_TAG = 'phl'
// SES message tag set at send time on emails whose template is marked sensitive, and echoed back on
// every event for that message. Such an email's subject and link paths can carry a login code or a
// reset token, so its events keep no subject and only each clicked link's origin. `$link_index` still
// tells the links apart.
export const SES_SENSITIVE_TAG = 'ph_sensitive'
// SES message tag set at send time on transactional emails, the only emails whose status is recorded,
// and echoed back on every event for that message. SES allows only ASCII letters, digits, underscores
// and dashes in a tag, so the value is 'true' as for SES_SENSITIVE_TAG.
export const SES_EMAIL_STATUS_TAG = 'ph_email_status'
export type SesEmailStatusUpdate = {
teamId: string
invocationId: string
status: EmailStatus
reason?: EmailStatusReason
timestamp: string
}
const normalizeAddress = (address: string): string => unquoteMailbox(address).trim().toLowerCase()
// SES reports each entry as written at send time, so a display name can wrap the address.
const mailboxAddresses = (entries: string[]): Set<string> =>
new Set(
entries.flatMap((entry) =>
addressparser(entry, { flatten: true }).map((parsed) => normalizeAddress(parsed.address))
)
)
const toHeaderAddresses = (rec: SesEventRecord): Set<string> => mailboxAddresses(rec.mail.commonHeaders?.to ?? [])
const sesRecipientsNotSentToCounter = new Counter({
name: 'email_tracking_recipients_not_sent_to_total',
help: 'Bounce and complaint recipients left out of suppression because the email was not sent to them',
labelNames: ['event_type'],
})
// SES reports a bounce or complaint recipient from the receiving server's report, which can name any address.
// Only an address this send went to may change suppression state.
const sentToRecipients = <T extends { emailAddress: string }>(rec: SesEventRecord, recipients: T[]): T[] => {
const sentTo = mailboxAddresses(rec.mail.destination)
const kept = recipients.filter((r) => sentTo.has(normalizeAddress(r.emailAddress)))
if (kept.length < recipients.length) {
sesRecipientsNotSentToCounter.inc({ event_type: rec.eventType }, recipients.length - kept.length)
}
return kept
}
/**
* The status change a Delivery, Bounce, Complaint or Reject record makes to the email it reports on, or the
* open or click an Open or Click record adds to it, or null.
*
* The status describes the email to its To recipient. SES reports cc and bcc copies under the same
* message, so a record counts only when one of its recipients is in the To header.
*/
const resolveEmailStatusUpdate = (
rec: SesEventRecord
): Pick<SesEmailStatusUpdate, 'status' | 'reason' | 'timestamp'> | null => {
const to = toHeaderAddresses(rec)
const reachesTo = (addresses: string[]): boolean => addresses.some((address) => to.has(normalizeAddress(address)))
switch (rec.eventType) {
case 'Delivery':
return reachesTo(rec.delivery.recipients ?? [])
? { status: 'delivered', timestamp: rec.delivery.timestamp }
: null
case 'Bounce':
return reachesTo(rec.bounce.bouncedRecipients.map((r) => r.emailAddress))
? {
status: 'bounced',
reason: BOUNCE_TYPE_TO_EVENT_PROPERTY[rec.bounce.bounceType],
timestamp: rec.bounce.timestamp,
}
: null
case 'Complaint':
// A "not-spam" feedback type corrects an earlier spam report, so it is not a complaint.
return rec.complaint.complaintFeedbackType?.toLowerCase() !== 'not-spam' &&
reachesTo(rec.complaint.complainedRecipients.map((r) => r.emailAddress))
? { status: 'complained', timestamp: rec.complaint.timestamp }
: null
case 'Reject':
// SES refuses the whole message, so the email fails for its To recipient too.
return { status: 'failed', reason: 'provider_rejected', timestamp: rec.mail.timestamp }
case 'Open':
return { status: 'opened', timestamp: rec.open.timestamp }
case 'Click':
return { status: 'clicked', timestamp: rec.click.timestamp }
default:
return null
}
}
const sensitiveLinkOrigin = (link: string): string => {
try {
const url = new URL(link)
return url.host ? `${url.protocol}//${url.host}` : url.protocol
} catch {
return ''
}
}
// Path of our own click-tracking redirect, which used to wrap every anchor before SES saw it.
const TRACKING_REDIRECT_PATH = '/public/m/redirect'
/**
* The destination the recipient actually asked for.
*
* SES reports the link as it appeared in the HTML, before SES rewrote it. Messages sent while we
* still wrapped anchors in our own redirect therefore report that wrapper, whose `ph_id` differs on
* every send, so the raw value is both unaggregatable and a carrier for a signed tracking code that
* should not be stored on an event. Unwrap it back to `target`. Anything else is passed through.
*/
export const resolveClickDestination = (link: string): string => {
try {
const url = new URL(link)
if (!url.pathname.endsWith(TRACKING_REDIRECT_PATH)) {
return link
}
return url.searchParams.get('target') || link
} catch {
return link
}
}
// Bounds how much of a URL rides in the app_metrics2 sort key. Long enough to stay recognizable in
// the UI, short enough that a pathological query-less URL can't bloat the index.
const MAX_LINK_URL_LENGTH = 200
/**
* Collapses a clicked URL to the identity we count by.
*
* The query string and fragment are dropped because merge tags and per-recipient UTM values make
* them differ on every send, which would scatter one link's clicks across thousands of buckets. Two
* anchors that genuinely point at the same path stay distinguishable through the link index, which
* is carried alongside this value.
*/
// Templates routinely build hrefs like `/unsubscribe/{{token}}` or `/users/12345`, so a path segment
// is often a per-recipient identifier rather than part of the page's identity. Collapsing those is
// what keeps the metrics sort key bounded by pages rather than by audience size, and keeps recipient
// tokens out of a store the whole team can read.
//
// A token is recognized by its runs of unbroken alphanumerics rather than by the segment as a whole,
// because tokens arrive glued together with punctuation: a JWT is dot-separated, and base64url
// tokens carry `-` and `_`. Testing runs also stops long slugs from being mistaken for tokens, since
// `how-to-set-up-feature-flags` is only ever short words once split.
const UUID_SEGMENT = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i
const TOKEN_RUN_SEPARATORS = /[.\-_~]/
const HEX_RUN = /^[0-9a-f]{12,}$/i
const NUMERIC_RUN = /^[0-9]{6,}$/
const SINGLE_CASE_WORD = /^([a-z]+|[A-Z]+)$/
// Below this, mixed-case runs are still plausibly words (`Introduction`, `CHANGELOG`).
const MIN_RANDOM_RUN_LENGTH = 16
const OPAQUE_PATH_SEGMENT_PLACEHOLDER = '*'
const isOpaqueRun = (run: string): boolean =>
HEX_RUN.test(run) || NUMERIC_RUN.test(run) || (run.length >= MIN_RANDOM_RUN_LENGTH && !SINGLE_CASE_WORD.test(run))
const isOpaquePathSegment = (segment: string): boolean => {
// Percent-encoding would otherwise hide a token's shape behind `%`, which none of the runs match.
let decoded = segment
try {
decoded = decodeURIComponent(segment)
} catch {
// Malformed escapes are left as-is; the raw segment is still worth testing.
}
return UUID_SEGMENT.test(decoded) || decoded.split(TOKEN_RUN_SEPARATORS).some(isOpaqueRun)
}
const redactOpaquePathSegments = (pathname: string): string =>
pathname
.split('/')
.map((segment) => (isOpaquePathSegment(segment) ? OPAQUE_PATH_SEGMENT_PLACEHOLDER : segment))
.join('/')
export const normalizeClickUrl = (link: string): string => {
try {
const url = new URL(link)
// `new URL` also parses hostless schemes (mailto:, tel:, sms:), where `host` is empty and
// rebuilding as `scheme://host+path` would invent an authority — `mailto:x` becoming
// `mailto://x`. Only http(s)-shaped links have a host to normalize; the rest pass through.
if (!url.host) {
return link.slice(0, MAX_LINK_URL_LENGTH)
}
const path = redactOpaquePathSegments(url.pathname)
return `${url.protocol}//${url.host}${path}`.replace(/\/$/, '').slice(0, MAX_LINK_URL_LENGTH)
} catch {
return link.slice(0, MAX_LINK_URL_LENGTH)
}
}
// Separator between the parts of a per-link metric's `instance_id`. The frontend splits on this to
// recover (action id, link index, url); see LINK_INSTANCE_SEPARATOR in workflowMetricsSummaryLogic.
// A URL may legitimately contain this character, so parsers must limit themselves to the first two
// separators and treat the remainder as the URL.
export const LINK_INSTANCE_SEPARATOR = '|'
export const buildLinkInstanceId = (actionId: string, linkIndex: string, normalizedUrl: string): string =>
[actionId, linkIndex, normalizedUrl].join(LINK_INSTANCE_SEPARATOR)
const MAX_SES_FIELD_LENGTH = 1024
// Strip control chars, neutralize rich-log bracket tokens, and cap length.
const sanitizeSesField = (value: string, max = MAX_SES_FIELD_LENGTH): string => {
const cleaned = value
.replace(/[\u0000-\u001f\u007f]/g, ' ')
.replace(/\[/g, '(')
.replace(/\]/g, ')')
return cleaned.length > max ? cleaned.slice(0, max) + '…' : cleaned
}
// Only warn/error events become log entries. Info-level events (Send, Delivery, Open, Click)
// are tracked as metrics only - they're analytics signal, not debugging signal.
export const formatSesEventLogs = (rec: SesEventRecord): SesEventLogLine[] => {
switch (rec.eventType) {
case 'Bounce': {
const bounceType = rec.bounce.bounceType
const level: SesEventLogLine['level'] = bounceType === 'Permanent' ? 'error' : 'warn'
return rec.bounce.bouncedRecipients.map((r) => {
const diag = r.diagnosticCode ? sanitizeSesField(r.diagnosticCode) : ''
// SES typically inlines the SMTP status inside diagnosticCode already
// (e.g. "smtp; 550 5.1.1 user unknown"), so skip the suffix when present.
const statusSuffix = r.status && !diag.includes(r.status) ? ` (${sanitizeSesField(r.status)})` : ''
const diagPart = diag ? `, ${diag}` : ''
return {
level,
message: `${bounceType} bounce to ${sanitizeSesField(r.emailAddress)}${diagPart}${statusSuffix}`,
}
})
}
case 'Complaint': {
const feedback = rec.complaint.complaintFeedbackType
? `, feedback type: ${sanitizeSesField(rec.complaint.complaintFeedbackType)}`
: ''
return rec.complaint.complainedRecipients.map((r) => ({
level: 'warn',
message: `Complaint from ${sanitizeSesField(r.emailAddress)}${feedback}`,
}))
}
case 'RenderingFailure': {
const tmpl = rec.renderingFailure.templateName
? ` for template ${sanitizeSesField(rec.renderingFailure.templateName)}`
: ''
return [
{
level: 'error',
message: `Rendering failure${tmpl}: ${sanitizeSesField(rec.renderingFailure.errorMessage)}`,
},
]
}
case 'Reject': {
const reason = rec.reject?.reason ? `: ${sanitizeSesField(rec.reject.reason)}` : ''
return [
{
level: 'error',
message: `Message rejected by SES${reason}`,
},
]
}
default:
return []
}
}
// Only text that does not open like JSON is quoted, because a malformed SES event carries recipients.
// Any AWS account can send it when the topic allowlist is empty, so it is sanitized like other SES fields.
const plainTextPrefix = (message: string): string | undefined =>
/^\s*[{[]/.test(message) ? undefined : sanitizeSesField(message, 200)
export class SesWebhookHandler {
certCache: Record<string, Promise<string> | undefined> = {}
private allowedTopicArns: Set<string>
constructor(
private trackingCodeSigner: EmailTrackingCodeSigner,
// Empty set means no restriction (dev/test); prod is expected to configure the workflow SES
// topic ARN.
allowedTopicArns: string[] = []
) {
this.allowedTopicArns = new Set(allowedTopicArns.map((arn) => arn.trim()).filter(Boolean))
}
private isTopicAllowed(topicArn: string | undefined): boolean {
if (this.allowedTopicArns.size === 0) {
return true
}
return typeof topicArn === 'string' && this.allowedTopicArns.has(topicArn)
}
private async fetchText(url: string): Promise<string> {
const response = await fetch(url)
if (response.status >= 400) {
throw new Error(`Failed to fetch ${url}: ${response.status}`)
}
return await response.text()
}
private async fetchCert(url: string): Promise<string> {
// Validate that the URL is from AWS SNS
if (!this.isValidSnsCertUrl(url)) {
throw new Error(`Invalid SNS certificate URL: ${url}`)
}
if (this.certCache[url]) {
return await this.certCache[url]!
}
this.certCache[url] = this.fetchText(url)
return this.certCache[url]!
}
private isValidSnsCertUrl(url: string): boolean {
try {
const parsedUrl = new URL(url)
// Must be HTTPS
if (parsedUrl.protocol !== 'https:') {
return false
}
// Must be from sns.{region}.amazonaws.com
const hostname = parsedUrl.hostname
if (!hostname.match(/^sns\.[a-z0-9-]+\.amazonaws\.com$/)) {
return false
}
// Must end with .pem
if (!parsedUrl.pathname.endsWith('.pem')) {
return false
}
return true
} catch {
return false
}
}
private isValidSnsSubscribeUrl(url: string): boolean {
try {
const parsedUrl = new URL(url)
if (parsedUrl.protocol !== 'https:') {
return false
}
// Must be from sns.{region}.amazonaws.com
if (!parsedUrl.hostname.match(/^sns\.[a-z0-9-]+\.amazonaws\.com$/)) {
return false
}
return true
} catch {
return false
}
}
/**
* Parse incoming body accounting for SNS raw vs envelope
*/
private parseIncomingBody(
body: unknown
): { mode: 'raw'; records: SesEventRecord[] } | { mode: 'sns'; envelope: SnsEnvelope } {
// If it's already an object with "Type", it's probably the SNS envelope (raw=false)
if (body && typeof body === 'object' && 'Type' in (body as any)) {
return { mode: 'sns', envelope: SnsEnvelopeSchema.parse(body) }
}
// raw_message_delivery=true → body is already the SES array
const records = SesEventBatchSchema.parse(body)
return { mode: 'raw', records }
}
private parseNotificationRecords(envelope: SnsEnvelope): SesEventRecord[] {
let message: unknown
try {
message = parseJSON(envelope.Message)
} catch {
sesIgnoredNotificationsCounter.inc({ reason: 'non_json_message' })
logger.warn('[SesWebhookHandler] Ignoring Notification whose Message is not JSON', {
topicArn: envelope.TopicArn,
messageId: envelope.MessageId,
messageLength: envelope.Message.length,
messagePrefix: plainTextPrefix(envelope.Message),
})
return []
}
return [SesEventRecordSchema.parse(message)]
}
/**
* Verify SNS signature
* Best practice: verify the signature unless you're behind a trusted ALB / private VPC.
*/
private async verifySnsSignature(envelope: SnsEnvelope): Promise<boolean> {
try {
// 1) Fetch cert
const cert = await this.fetchCert(envelope.SigningCertURL)
// 2) Build string to sign (per SNS docs)
const stringToSign = this.buildStringToSign(envelope)
// 3) Verify
const verifier = crypto.createVerify('RSA-SHA1') // SNS SignatureVersion=1 uses SHA1
verifier.update(stringToSign, 'utf8')
return verifier.verify(cert, envelope.Signature, 'base64')
} catch {
return false
}
}
/**
* Build string to sign for SNS signature verification
*/
private buildStringToSign(m: SnsEnvelope): string {
// Follows AWS SNS docs for SignatureVersion=1
// For Notification:
// "Message\n{Message}\nMessageId\n{MessageId}\nSubject\n{Subject}\nTimestamp\n{Timestamp}\nTopicArn\n{TopicArn}\nType\n{Type}\n"
// Subject line omitted if not present.
const lines: string[] = []
const pushKV = (k: string, v?: string) => {
if (v !== undefined) {
lines.push(k)
lines.push(v)
}
}
if (m.Type === 'Notification') {
pushKV('Message', m.Message)
pushKV('MessageId', m.MessageId)
pushKV('Subject', m.Subject)
pushKV('Timestamp', m.Timestamp)
pushKV('TopicArn', m.TopicArn)
pushKV('Type', m.Type)
} else if (m.Type === 'SubscriptionConfirmation' || m.Type === 'UnsubscribeConfirmation') {
pushKV('Message', m.Message)
pushKV('MessageId', m.MessageId)
pushKV('SubscribeURL', m.SubscribeURL)
pushKV('Timestamp', m.Timestamp)
pushKV('Token', m.Token!)
pushKV('TopicArn', m.TopicArn)
pushKV('Type', m.Type)
}
return lines.join('\n') + '\n'
}
async handleWebhook(opts: {
body: any
headers: Record<string, string | string[] | undefined>
verifySignature?: boolean
}): Promise<{
status: number
body: unknown
metrics?: {
functionId?: string
invocationId?: string
actionId?: string
parentRunId?: string
distinctId?: string
teamId?: string
metricName: MinimalAppMetric['metric_name']
// Replaces the instance_id trackMetric would derive, for metrics keyed by something
// other than the action or invocation (see buildLinkInstanceId).
instanceIdOverride?: string
properties?: Record<string, any>
timestamp?: string
workflowVersion?: number
}[]
logEntries?: {
functionId?: string
invocationId?: string
actionId?: string
parentRunId?: string
teamId?: string
level: SesEventLogLine['level']
message: string
}[]
// Soft (Transient) bounces — fed into the suppression list's consecutive-bounce counter.
transientBounceRecipients?: {
teamId?: string
emailAddresses: string[]
diagnostic?: string
}[]
// Hard (Permanent) bounces — recorded in the suppression list so future sends to the address
// are blocked. Suppression enforcement replaces the legacy opt-out-on-bounce write.
hardBounceRecipients?: {
teamId?: string
emailAddresses: string[]
diagnostic?: string
// Addresses the bounce proves do not exist, for the global hard-bounce list.
nonexistentAddresses?: string[]
bounceTimestamp?: string
}[]
// Spam complaints — recorded in the suppression list so future sends to the address are
// blocked. A complaint is the recipient asking us to stop, and provider complaint-rate
// thresholds are far tighter than bounce thresholds.
complainedRecipients?: {
teamId?: string
emailAddresses: string[]
feedbackType?: string
}[]
// Successful deliveries — reset the suppression counter so transient outages don't accumulate.
// Timestamp is threaded through so the reset ignores an out-of-order delivery from an older send
// arriving after a newer bounce.
deliveredRecipients?: {
teamId?: string
emailAddresses: string[]
timestamp?: string
clearsGlobalHardBounces?: boolean
}[]
// Status changes of transactional emails, for the email_status table.
emailStatusUpdates?: SesEmailStatusUpdate[]
}> {
// Payloads carry recipients and clicked links, including reset links from sensitive emails,
// so the logs keep identifiers only.
const parsed = this.parseIncomingBody(opts.body)
logger.info('[SesWebhookHandler] handleWebhook', { mode: parsed.mode })
// When verification is required the message must be a signed SNS envelope. Raw deliveries
// carry no signature, and prod uses envelope delivery, so reject them to block forged events.
if (opts.verifySignature) {
if (!('envelope' in parsed)) {
return { status: 403, body: { error: 'Unsigned raw delivery not allowed' } }
}
logger.info('[SesWebhookHandler] verifying signature', {
type: parsed.envelope.Type,
messageId: parsed.envelope.MessageId,
})
const ok = await this.verifySnsSignature(parsed.envelope)
logger.info('[SesWebhookHandler] signature verified', { ok })
if (!ok) {
return { status: 403, body: { error: 'Invalid SNS signature' } }
}
}
// Restrict accepted SNS envelopes (Notification, SubscriptionConfirmation, UnsubscribeConfirmation)
// to the configured TopicArn allowlist. SNS signature verification proves the message came from
// AWS, but not from a specific topic — this narrows accepted sources to the workflow SES topic.
if ('envelope' in parsed && !this.isTopicAllowed(parsed.envelope.TopicArn)) {
logger.warn('[SesWebhookHandler] Rejecting event from disallowed TopicArn', {
topicArn: parsed.envelope.TopicArn,
})
return { status: 403, body: { error: 'SNS TopicArn not allowed' } }
}
// Handle confirmation flow
if (parsed.mode === 'sns' && parsed.envelope.Type === 'SubscriptionConfirmation') {
logger.info('[SesWebhookHandler] confirming subscription', { envelope: parsed.envelope })
// Confirm by visiting the top-level SubscribeURL field; Message is human-readable text here.
const subscribeUrl = parsed.envelope.SubscribeURL
if (subscribeUrl) {
if (!this.isValidSnsSubscribeUrl(subscribeUrl)) {
logger.warn('[SesWebhookHandler] Invalid SubscribeURL, rejecting', {
url: subscribeUrl,
})
return { status: 403, body: { error: 'Invalid SubscribeURL' } }
}
await this.fetchText(subscribeUrl)
} else {
logger.warn('[SesWebhookHandler] SubscriptionConfirmation without SubscribeURL - cannot confirm')
return { status: 400, body: { error: 'Missing SubscribeURL' } }
}
return { status: 200, body: { ok: true } }
}
if (parsed.mode === 'sns' && parsed.envelope.Type === 'UnsubscribeConfirmation') {
logger.info('[SesWebhookHandler] confirming unsubscribe', { envelope: parsed.envelope })
return { status: 200, body: { ok: true } }
}
const records = parsed.mode === 'raw' ? parsed.records : this.parseNotificationRecords(parsed.envelope)
const metrics: {
functionId?: string
invocationId?: string
actionId?: string
parentRunId?: string
distinctId?: string
teamId?: string
metricName: MinimalAppMetric['metric_name']
instanceIdOverride?: string
properties?: Record<string, any>
timestamp?: string
workflowVersion?: number
}[] = []
const logEntries: {
functionId?: string
invocationId?: string
actionId?: string
parentRunId?: string
teamId?: string
level: SesEventLogLine['level']
message: string
}[] = []
const transientBounceRecipients: {
teamId?: string
emailAddresses: string[]
diagnostic?: string
}[] = []
const hardBounceRecipients: {
teamId?: string
emailAddresses: string[]
diagnostic?: string
nonexistentAddresses?: string[]
bounceTimestamp?: string
}[] = []
const complainedRecipients: {
teamId?: string
emailAddresses: string[]
feedbackType?: string
}[] = []
const deliveredRecipients: {
teamId?: string
emailAddresses: string[]
timestamp?: string
clearsGlobalHardBounces?: boolean
}[] = []
// A signature only proves AWS sent the event, from any account's topic. The global hard-bounce list
// affects every team, so it only takes events from a configured topic.
const fromTrustedTopic = 'envelope' in parsed && this.allowedTopicArns.size > 0
const emailStatusUpdates: SesEmailStatusUpdate[] = []
for (const rec of records) {
logger.info('[SesWebhookHandler] processing record', {
eventType: rec.eventType,
messageId: rec.mail.messageId,
})
// Prefer the custom MIME header (carries the full signed code including distinct_id,
// unbounded). Fall back to the SES EmailTag for messages sent before the header carrier
// was added, or where the configuration set hasn't enabled `IncludeOriginalHeaders`.
const headerValue = rec.mail.headers?.find(
(h) => h.name.toLowerCase() === TRACKING_CODE_HEADER_NAME.toLowerCase()
)?.value
const tagValue = rec.mail.tags?.ph_id?.[0]
const rawCode = headerValue ?? tagValue ?? ''
const parsedCode = this.trackingCodeSigner.parse(rawCode)
if (parsedCode) {
trackingCodeFormatCounter.inc({ format: parsedCode.format, source: 'ses' })
}
const { functionId, invocationId, teamId, actionId, parentRunId, distinctId, isTest, workflowVersion } =
parsedCode || {}
if (!functionId && !invocationId) {
// A present but unparseable code is a different incident from a wholly absent one, so
// record which occurred. An empty `rawCode` means neither carrier held a value.
const reason = rawCode === '' ? 'missing' : 'invalid'
sesUnattributedEventsCounter.inc({ event_type: rec.eventType, reason })
logger.error('[SesWebhookHandler] handleWebhook: No functionId or invocationId found', {
eventType: rec.eventType,
messageId: rec.mail.messageId,
reason,
})
continue
}
const metricName = EVENT_TYPE_TO_METRIC_NAME[rec.eventType]
// Test sends (from the editor's "Run test") are not production activity, so we skip
// their delivery/open/click metrics — otherwise a draft/never-enabled workflow shows
// email activity in its Metrics tab. Bounce-driven opt-outs below still apply.
if (metricName && !isTest) {
const isSensitive = rec.mail.tags?.[SES_SENSITIVE_TAG]?.[0] === 'true'
const properties: Record<string, any> = {
$email_to: rec.mail.destination?.[0],
...(isSensitive ? {} : { $email_subject: rec.mail.commonHeaders?.subject }),
}
// Each SES event detail carries its own timestamp (open.timestamp, click.timestamp, etc.)
// — prefer those over the webhook receipt time so the event reflects when the action
// actually happened, not when AWS got around to delivering the notification.
let timestamp: string | undefined
if ('open' in rec && rec.open) {
timestamp = rec.open.timestamp
} else if ('click' in rec && rec.click) {
timestamp = rec.click.timestamp
const destination = resolveClickDestination(rec.click.link)
properties.$link_url = isSensitive ? sensitiveLinkOrigin(destination) : destination
const linkIndex = rec.click.linkTags?.[SES_LINK_INDEX_TAG]?.[0]
if (linkIndex !== undefined) {
properties.$link_index = linkIndex
}
} else if ('delivery' in rec && rec.delivery) {
timestamp = rec.delivery.timestamp
} else if ('bounce' in rec && rec.bounce) {
timestamp = rec.bounce.timestamp
// Whether the address will ever accept mail again. A hard bounce is also written
// to the suppression list below; a soft one is not, so the two need telling apart
// to read a bounce rate at all.
properties.$bounce_type = BOUNCE_TYPE_TO_EVENT_PROPERTY[rec.bounce.bounceType]
if (rec.bounce.bounceSubType) {
// The provider's own reason (e.g. "MailboxFull", "NoEmail", "Suppressed").
properties.$bounce_sub_type = rec.bounce.bounceSubType
}
} else if ('complaint' in rec && rec.complaint) {
timestamp = rec.complaint.timestamp
if (rec.complaint.complaintFeedbackType) {
// The provider's classification of the report (e.g. "abuse", "fraud").
properties.$complaint_feedback_type = rec.complaint.complaintFeedbackType
}
}
timestamp = timestamp ?? rec.mail.timestamp
metrics.push({
functionId,
invocationId,
actionId,
parentRunId,
distinctId,
teamId,
metricName,
properties,
timestamp,
workflowVersion,
})
// Per-link companion row for the workflow Metrics tab's link breakdown. Emitted only
// when the send came from a workflow step: without an action id the key would fall
// back to the invocation id, producing a row per send per link rather than an
// aggregate, and the breakdown is per-action anyway. It carries its own metric name
// so the existing `email_link_clicked` totals and trends can't double-count it.
if (rec.eventType === 'Click' && actionId) {
metrics.push({
functionId,
invocationId,
actionId,
parentRunId,
distinctId,
teamId,
metricName: 'email_link_clicked_by_link',
instanceIdOverride: buildLinkInstanceId(
actionId,
rec.click.linkTags?.[SES_LINK_INDEX_TAG]?.[0] ?? '',
isSensitive
? sensitiveLinkOrigin(resolveClickDestination(rec.click.link))
: normalizeClickUrl(resolveClickDestination(rec.click.link))
),
timestamp,
workflowVersion,
})
}
// email_bounced stays the catch-all rollup; each bounce additionally emits a
// per-type sub-metric (hard + transient + undetermined = email_bounced). AWS's
// account bounce rate — what the email reputation feature calibrates against —
// counts only hard (Permanent) bounces, so reputation reads email_bounced_hard;
// the others exist for diagnosis (transient spikes are a leading indicator).
if (rec.eventType === 'Bounce') {
metrics.push({
functionId,
invocationId,
actionId,
parentRunId,
distinctId,
teamId,
metricName: BOUNCE_TYPE_TO_METRIC_NAME[rec.bounce.bounceType],
properties,
timestamp,
workflowVersion,
})
}
}
// Allowlist actionId before interpolating into the [Action:…] rich-log token,
// since actionId comes from the attacker-influenceable ph_id tag.
const safeActionId = actionId && /^[a-zA-Z0-9_-]+$/.test(actionId) ? actionId : undefined
// Skip test sends here too (mirrors the metrics gate above) so a "Run test" bounce
// doesn't write a misleading failure into the workflow's invocation logs.
for (const line of isTest ? [] : formatSesEventLogs(rec)) {
const prefix = safeActionId ? `[Action:${safeActionId}] ` : ''
logEntries.push({
functionId,
invocationId,
actionId,
parentRunId,
teamId,
level: line.level,
message: `${prefix}${line.message}`,
})
}
// State-changing writes (opt-out, suppression, delivery counter reset) require a signed
// tracking code. The signed header carries the HMAC we mint on send; the unsigned SES-tag
// carrier is only used for engagement metrics/log entries above.
const codeIsTrusted = parsedCode?.format === 'signed'
// Suppression writes (below) skip test sends — editor "Run test" traffic must not be
// able to perturb production suppression state by targeting a bad recipient.
const suppressionAllowed = teamId && codeIsTrusted && !isTest
// Record permanent bounces in the suppression list so future sends to the address are
// blocked. The SMTP diagnostic is threaded through for operator visibility on the row.
if (suppressionAllowed && rec.eventType === 'Bounce' && rec.bounce.bounceType === 'Permanent') {
const bounced = sentToRecipients(rec, rec.bounce.bouncedRecipients)
if (bounced.length > 0) {
const diagnostic = bounced.find((r) => r.diagnosticCode)?.diagnosticCode
const nonexistentAddresses = fromTrustedTopic
? nonexistentRecipients({ ...rec.bounce, bouncedRecipients: bounced })
: []
hardBounceRecipients.push({
teamId,
emailAddresses: bounced.map((r) => r.emailAddress),
diagnostic,
...(nonexistentAddresses.length > 0
? { nonexistentAddresses, bounceTimestamp: rec.bounce.timestamp }
: {}),
})
}
}
// Count soft (Transient) bounces toward suppression. These are recipient-side failures
// (server unreachable, mailbox full, greylisting); one is harmless but a persistent run
// of them means the address can't receive mail.
if (suppressionAllowed && rec.eventType === 'Bounce' && rec.bounce.bounceType === 'Transient') {
const bounced = sentToRecipients(rec, rec.bounce.bouncedRecipients)
if (bounced.length > 0) {
const diagnostic = bounced.find((r) => r.diagnosticCode)?.diagnosticCode
transientBounceRecipients.push({
teamId,
emailAddresses: bounced.map((r) => r.emailAddress),
diagnostic,
})
}
}
// Record complaints in the suppression list so future sends to the address are blocked.
// The recipient reported the message as spam — one complaint suppresses immediately,
// like a hard bounce. Skip a "not-spam" feedback type, because that is a correction:
// the recipient or their provider states the message is not spam, so it must not
// suppress (and would otherwise resurrect a manually removed row). The value comes from
// an external provider, so compare case-insensitively.
if (
suppressionAllowed &&
rec.eventType === 'Complaint' &&
rec.complaint.complaintFeedbackType?.toLowerCase() !== 'not-spam'
) {
const complained = sentToRecipients(rec, rec.complaint.complainedRecipients)
if (complained.length > 0) {
complainedRecipients.push({
teamId,
emailAddresses: complained.map((r) => r.emailAddress),
feedbackType: rec.complaint.complaintFeedbackType,
})
}
}
// Successful delivery resets an address's soft-bounce counter — but only if newer than the
// last-recorded bounce (checked at the SQL layer), so an out-of-order delivery from an
// older send can't erase a fresh bounce.
if (suppressionAllowed && rec.eventType === 'Delivery') {
const emails = rec.delivery.recipients ?? rec.mail.destination ?? []
if (emails.length > 0) {
deliveredRecipients.push({
teamId,
emailAddresses: emails,
timestamp: rec.delivery.timestamp,
...(fromTrustedTopic ? { clearsGlobalHardBounces: true } : {}),
})
}
}
// A status write is state-changing too, so it needs the signed code like the writes above.
if (suppressionAllowed && invocationId && rec.mail.tags?.[SES_EMAIL_STATUS_TAG]?.[0] === 'true') {
const update = resolveEmailStatusUpdate(rec)
if (update) {
emailStatusUpdates.push({ teamId, invocationId, ...update })
}
}
}
return {
status: 200,
body: { ok: true },
metrics,
logEntries,
transientBounceRecipients,
hardBounceRecipients,
complainedRecipients,
deliveredRecipients,
emailStatusUpdates,
}
}
}