Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(cdp): ack SES notifications whose message is not JSON (#129881)

Michael Reichenbachc869fc549eeb
diff --git a/nodejs/src/cdp/services/messaging/email-tracking.service.ts b/nodejs/src/cdp/services/messaging/email-tracking.service.ts
index 736a4a901b80..200b8cafb9f0 100644
--- a/nodejs/src/cdp/services/messaging/email-tracking.service.ts
+++ b/nodejs/src/cdp/services/messaging/email-tracking.service.ts
@@ -15,7 +15,7 @@ import { ModifiedRequest } from '~/common/api/router'
import { RedisV2 } from '~/common/redis/redis-v2'
import { isDevEnv, isTestEnv } from '~/common/utils/env-utils'
import { parseJSON } from '~/common/utils/json-parse'
-import { logger } from '~/common/utils/logger'
+import { logger, serializeError } from '~/common/utils/logger'
import { CapturedEventsService } from '../captured-events/captured-events.service'
import { HogFlowManagerService } from '../hogflows/hogflow-manager.service'
@@ -676,7 +676,7 @@ export class EmailTrackingService {
return { status, message: body as string }
} catch (error) {
emailTrackingErrorsCounter.inc({ error_type: error.name || 'unknown' })
- logger.error('[EmailService] handleWebhook: SES webhook error', { error })
+ logger.error('[EmailService] handleWebhook: SES webhook error', { error: serializeError(error) })
throw error
}
}
diff --git a/nodejs/src/cdp/services/messaging/helpers/ses.test.ts b/nodejs/src/cdp/services/messaging/helpers/ses.test.ts
index fe5a1c6bd744..4366541e15fd 100644
--- a/nodejs/src/cdp/services/messaging/helpers/ses.test.ts
+++ b/nodejs/src/cdp/services/messaging/helpers/ses.test.ts
@@ -725,27 +725,30 @@ describe('SesWebhookHandler', () => {
fetchSpy.mockRestore()
})
- it('confirms a signed SubscriptionConfirmation end to end (SubscribeURL is part of the signed string)', async () => {
+ // Signs like AWS SNS SignatureVersion=1: alphabetical key/value lines of the fields each
+ // envelope type signs, and serves the matching certificate.
+ const SIGNED_FIELDS: Record<string, string[]> = {
+ Notification: ['Message', 'MessageId', 'Timestamp', 'TopicArn', 'Type'],
+ SubscriptionConfirmation: ['Message', 'MessageId', 'SubscribeURL', 'Timestamp', 'Token', 'TopicArn', 'Type'],
+ }
+ const signEnvelope = (envelope: Record<string, any>): jest.SpyInstance => {
const { generateKeyPairSync, createSign } = jest.requireActual<typeof import('node:crypto')>('node:crypto')
const { publicKey, privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 })
const publicKeyPem = publicKey.export({ type: 'spki', format: 'pem' }).toString()
-
- const envelope = buildConfirmationEnvelope(
- 'https://sns.us-east-1.amazonaws.com/?Action=ConfirmSubscription&TopicArn=arn:aws:sns:us-east-1:123456789012:ses-topic&Token=token-123'
- )
- // String to sign per AWS SNS SignatureVersion=1 for SubscriptionConfirmation:
- // alphabetical key/value lines incl. the top-level SubscribeURL.
- const stringToSign =
- ['Message', 'MessageId', 'SubscribeURL', 'Timestamp', 'Token', 'TopicArn', 'Type']
- .map((k) => `${k}\n${envelope[k]}`)
- .join('\n') + '\n'
+ const stringToSign = SIGNED_FIELDS[envelope.Type].map((k) => `${k}\n${envelope[k]}`).join('\n') + '\n'
const sign = createSign('RSA-SHA1')
sign.update(stringToSign, 'utf8')
envelope.Signature = sign.sign(privateKey, 'base64')
-
- const fetchSpy = jest
+ return jest
.spyOn(handler as any, 'fetchText')
.mockImplementation((url) => Promise.resolve((url as string).endsWith('.pem') ? publicKeyPem : ''))
+ }
+
+ it('confirms a signed SubscriptionConfirmation end to end (SubscribeURL is part of the signed string)', async () => {
+ const envelope = buildConfirmationEnvelope(
+ 'https://sns.us-east-1.amazonaws.com/?Action=ConfirmSubscription&TopicArn=arn:aws:sns:us-east-1:123456789012:ses-topic&Token=token-123'
+ )
+ const fetchSpy = signEnvelope(envelope)
const result = await handler.handleWebhook({ body: envelope, headers: {}, verifySignature: true })
expect(result.status).toBe(200)
@@ -753,6 +756,71 @@ describe('SesWebhookHandler', () => {
fetchSpy.mockRestore()
})
+ describe('a Notification whose Message is not JSON', () => {
+ const plainText = 'Successfully validated SNS topic for Amazon SES event publishing.'
+ const plainTextNotification = (message = plainText): Record<string, any> => ({
+ Type: 'Notification',
+ MessageId: 'sns-msg-1',
+ TopicArn: 'arn:aws:sns:us-east-1:123456789012:ses-topic',
+ Message: message,
+ Timestamp: '2025-10-03T12:10:00Z',
+ SignatureVersion: '1',
+ Signature: 'forged',
+ SigningCertURL: 'https://sns.us-east-1.amazonaws.com/cert.pem',
+ })
+ const ignoredCount = async (): Promise<number> => {
+ const metric = await register.getSingleMetric('ses_webhook_ignored_notifications_total')?.get()
+ return metric?.values.find((v) => v.labels.reason === 'non_json_message')?.value ?? 0
+ }
+
+ it.each([
+ { label: 'plain text', message: plainText, logged: plainText },
+ { label: 'control character', message: '\u001b[31mforged\nline', logged: ' (31mforged line' },
+ {
+ label: 'truncated SES JSON',
+ message: '{"eventType":"Bounce","mail":{"destination":["[email protected]"',
+ logged: undefined,
+ },
+ ])(
+ 'is acknowledged without metrics when signed, logging its $label prefix only if not JSON-shaped',
+ async ({ message, logged }) => {
+ const warnSpy = jest.spyOn(logger, 'warn')
+ const envelope = plainTextNotification(message)
+ const fetchSpy = signEnvelope(envelope)
+ const before = await ignoredCount()
+
+ const result = await handler.handleWebhook({ body: envelope, headers: {}, verifySignature: true })
+
+ expect(result.status).toBe(200)
+ expect(result.metrics).toEqual([])
+ expect(await ignoredCount()).toBe(before + 1)
+ expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('not JSON'), {
+ topicArn: envelope.TopicArn,
+ messageId: envelope.MessageId,
+ messageLength: message.length,
+ messagePrefix: logged,
+ })
+ fetchSpy.mockRestore()
+ warnSpy.mockRestore()
+ }
+ )
+
+ it('is rejected when its signature does not verify', async () => {
+ const fetchSpy = jest.spyOn(handler as any, 'fetchText').mockResolvedValue('not a certificate')
+ const before = await ignoredCount()
+
+ const result = await handler.handleWebhook({
+ body: plainTextNotification(),
+ headers: {},
+ verifySignature: true,
+ })
+
+ expect(result.status).toBe(403)
+ expect(await ignoredCount()).toBe(before)
+ fetchSpy.mockRestore()
+ })
+ })
+
it.each([
['non-SNS SubscribeURL', 'https://evil.lhr.life/latest/meta-data/iam/security-credentials/role'],
['HTTP SubscribeURL', 'http://sns.us-east-1.amazonaws.com/subscribe'],
diff --git a/nodejs/src/cdp/services/messaging/helpers/ses.ts b/nodejs/src/cdp/services/messaging/helpers/ses.ts
index ecddcfd4ef1c..6bbafadffa33 100644
--- a/nodejs/src/cdp/services/messaging/helpers/ses.ts
+++ b/nodejs/src/cdp/services/messaging/helpers/ses.ts
@@ -234,6 +234,14 @@ const sesUnattributedEventsCounter = new Counter({
labelNames: ['event_type', 'reason'],
})
+// Kept off `email_tracking_errors_total` because the SES webhook errors alert sums that counter, and an
+// SNS message that is not an SES event is acknowledged, not a fault.
+const sesIgnoredNotificationsCounter = new Counter({
+ name: 'ses_webhook_ignored_notifications_total',
+ help: 'SNS notifications on the SES webhook acknowledged without processing',
+ labelNames: ['reason'],
+})
+
// SES link tag naming an anchor's ordinal position in the email body. Set on each `<a>` at send
// time and echoed back on the Click event, which is what lets two anchors pointing at the same URL
// (a header logo and a footer link, say) be counted separately. SES restricts tag names and values
@@ -479,6 +487,11 @@ export const formatSesEventLogs = (rec: SesEventRecord): SesEventLogLine[] => {
}
}
+// Only text that does not open like JSON is quoted, because a malformed SES event carries recipients.
+// Any AWS account can send it when the topic allowlist is empty, so it is sanitized like other SES fields.
+const plainTextPrefix = (message: string): string | undefined =>
+ /^\s*[{[]/.test(message) ? undefined : sanitizeSesField(message, 200)
+
export class SesWebhookHandler {
certCache: Record<string, Promise<string> | undefined> = {}
private allowedTopicArns: Set<string>
@@ -570,19 +583,10 @@ export class SesWebhookHandler {
*/
private parseIncomingBody(
body: unknown
- ):
- | { mode: 'raw' | 'sns'; records: SesEventRecord[] }
- | { mode: 'sns'; envelope: SnsEnvelope; records?: SesEventRecord[] } {
+ ): { mode: 'raw'; records: SesEventRecord[] } | { mode: 'sns'; envelope: SnsEnvelope } {
// If it's already an object with "Type", it's probably the SNS envelope (raw=false)
if (body && typeof body === 'object' && 'Type' in (body as any)) {
- const env = SnsEnvelopeSchema.parse(body)
- if (env.Type === 'Notification') {
- const inner = parseJSON(env.Message)
- const record = SesEventRecordSchema.parse(inner)
- return { mode: 'sns', envelope: env, records: [record] }
- }
- // For non-Notification, return envelope; caller decides how to handle
- return { mode: 'sns', envelope: env }
+ return { mode: 'sns', envelope: SnsEnvelopeSchema.parse(body) }
}
// raw_message_delivery=true → body is already the SES array
@@ -590,6 +594,23 @@ export class SesWebhookHandler {
return { mode: 'raw', records }
}
+ private parseNotificationRecords(envelope: SnsEnvelope): SesEventRecord[] {
+ let message: unknown
+ try {
+ message = parseJSON(envelope.Message)
+ } catch {
+ sesIgnoredNotificationsCounter.inc({ reason: 'non_json_message' })
+ logger.warn('[SesWebhookHandler] Ignoring Notification whose Message is not JSON', {
+ topicArn: envelope.TopicArn,
+ messageId: envelope.MessageId,
+ messageLength: envelope.Message.length,
+ messagePrefix: plainTextPrefix(envelope.Message),
+ })
+ return []
+ }
+ return [SesEventRecordSchema.parse(message)]
+ }
+
/**
* Verify SNS signature
* Best practice: verify the signature unless you're behind a trusted ALB / private VPC.
@@ -743,7 +764,7 @@ export class SesWebhookHandler {
}
// Handle confirmation flow
- if (parsed.mode === 'sns' && 'envelope' in parsed && parsed.envelope?.Type === 'SubscriptionConfirmation') {
+ if (parsed.mode === 'sns' && parsed.envelope.Type === 'SubscriptionConfirmation') {
logger.info('[SesWebhookHandler] confirming subscription', { envelope: parsed.envelope })
// Confirm by visiting the top-level SubscribeURL field; Message is human-readable text here.
const subscribeUrl = parsed.envelope.SubscribeURL
@@ -762,13 +783,13 @@ export class SesWebhookHandler {
return { status: 200, body: { ok: true } }
}
- if (parsed.mode === 'sns' && 'envelope' in parsed && parsed.envelope?.Type === 'UnsubscribeConfirmation') {
+ if (parsed.mode === 'sns' && parsed.envelope.Type === 'UnsubscribeConfirmation') {
logger.info('[SesWebhookHandler] confirming unsubscribe', { envelope: parsed.envelope })
return { status: 200, body: { ok: true } }
}
- // Process SES events
- const records = parsed.mode === 'raw' ? parsed.records : parsed.records!
+ const records = parsed.mode === 'raw' ? parsed.records : this.parseNotificationRecords(parsed.envelope)
+
const metrics: {
functionId?: string
invocationId?: string