:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(cdp): ack SES notifications whose message is not JSON (#129881)
Michael Reichenbachc869fc549eeb
diff --git a/nodejs/src/cdp/services/messaging/email-tracking.service.ts b/nodejs/src/cdp/services/messaging/email-tracking.service.tsindex 736a4a901b80..200b8cafb9f0 100644--- a/nodejs/src/cdp/services/messaging/email-tracking.service.ts+++ b/nodejs/src/cdp/services/messaging/email-tracking.service.ts@@ -15,7 +15,7 @@ import { ModifiedRequest } from '~/common/api/router'import { RedisV2 } from '~/common/redis/redis-v2'import { isDevEnv, isTestEnv } from '~/common/utils/env-utils'import { parseJSON } from '~/common/utils/json-parse'-import { logger } from '~/common/utils/logger'+import { logger, serializeError } from '~/common/utils/logger'import { CapturedEventsService } from '../captured-events/captured-events.service'import { HogFlowManagerService } from '../hogflows/hogflow-manager.service'@@ -676,7 +676,7 @@ export class EmailTrackingService {return { status, message: body as string }} catch (error) {emailTrackingErrorsCounter.inc({ error_type: error.name || 'unknown' })- logger.error('[EmailService] handleWebhook: SES webhook error', { error })+ logger.error('[EmailService] handleWebhook: SES webhook error', { error: serializeError(error) })throw error}}diff --git a/nodejs/src/cdp/services/messaging/helpers/ses.test.ts b/nodejs/src/cdp/services/messaging/helpers/ses.test.tsindex fe5a1c6bd744..4366541e15fd 100644--- a/nodejs/src/cdp/services/messaging/helpers/ses.test.ts+++ b/nodejs/src/cdp/services/messaging/helpers/ses.test.ts@@ -725,27 +725,30 @@ describe('SesWebhookHandler', () => {fetchSpy.mockRestore()})- it('confirms a signed SubscriptionConfirmation end to end (SubscribeURL is part of the signed string)', async () => {+ // Signs like AWS SNS SignatureVersion=1: alphabetical key/value lines of the fields each+ // envelope type signs, and serves the matching certificate.+ const SIGNED_FIELDS: Record<string, string[]> = {+ Notification: ['Message', 'MessageId', 'Timestamp', 'TopicArn', 'Type'],+ SubscriptionConfirmation: ['Message', 'MessageId', 'SubscribeURL', 'Timestamp', 'Token', 'TopicArn', 'Type'],+ }+ const signEnvelope = (envelope: Record<string, any>): jest.SpyInstance => {const { generateKeyPairSync, createSign } = jest.requireActual<typeof import('node:crypto')>('node:crypto')const { publicKey, privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 })const publicKeyPem = publicKey.export({ type: 'spki', format: 'pem' }).toString()-- const envelope = buildConfirmationEnvelope(- 'https://sns.us-east-1.amazonaws.com/?Action=ConfirmSubscription&TopicArn=arn:aws:sns:us-east-1:123456789012:ses-topic&Token=token-123'- )- // String to sign per AWS SNS SignatureVersion=1 for SubscriptionConfirmation:- // alphabetical key/value lines incl. the top-level SubscribeURL.- const stringToSign =- ['Message', 'MessageId', 'SubscribeURL', 'Timestamp', 'Token', 'TopicArn', 'Type']- .map((k) => `${k}\n${envelope[k]}`)- .join('\n') + '\n'+ const stringToSign = SIGNED_FIELDS[envelope.Type].map((k) => `${k}\n${envelope[k]}`).join('\n') + '\n'const sign = createSign('RSA-SHA1')sign.update(stringToSign, 'utf8')envelope.Signature = sign.sign(privateKey, 'base64')-- const fetchSpy = jest+ return jest.spyOn(handler as any, 'fetchText').mockImplementation((url) => Promise.resolve((url as string).endsWith('.pem') ? publicKeyPem : ''))+ }++ it('confirms a signed SubscriptionConfirmation end to end (SubscribeURL is part of the signed string)', async () => {+ const envelope = buildConfirmationEnvelope(+ 'https://sns.us-east-1.amazonaws.com/?Action=ConfirmSubscription&TopicArn=arn:aws:sns:us-east-1:123456789012:ses-topic&Token=token-123'+ )+ const fetchSpy = signEnvelope(envelope)const result = await handler.handleWebhook({ body: envelope, headers: {}, verifySignature: true })expect(result.status).toBe(200)@@ -753,6 +756,71 @@ describe('SesWebhookHandler', () => {fetchSpy.mockRestore()})+ describe('a Notification whose Message is not JSON', () => {+ const plainText = 'Successfully validated SNS topic for Amazon SES event publishing.'+ const plainTextNotification = (message = plainText): Record<string, any> => ({+ Type: 'Notification',+ MessageId: 'sns-msg-1',+ TopicArn: 'arn:aws:sns:us-east-1:123456789012:ses-topic',+ Message: message,+ Timestamp: '2025-10-03T12:10:00Z',+ SignatureVersion: '1',+ Signature: 'forged',+ SigningCertURL: 'https://sns.us-east-1.amazonaws.com/cert.pem',+ })+ const ignoredCount = async (): Promise<number> => {+ const metric = await register.getSingleMetric('ses_webhook_ignored_notifications_total')?.get()+ return metric?.values.find((v) => v.labels.reason === 'non_json_message')?.value ?? 0+ }++ it.each([+ { label: 'plain text', message: plainText, logged: plainText },+ { label: 'control character', message: '\u001b[31mforged\nline', logged: ' (31mforged line' },+ {+ label: 'truncated SES JSON',+ message: '{"eventType":"Bounce","mail":{"destination":["[email protected]"',+ logged: undefined,+ },+ ])(+ 'is acknowledged without metrics when signed, logging its $label prefix only if not JSON-shaped',+ async ({ message, logged }) => {+ const warnSpy = jest.spyOn(logger, 'warn')+ const envelope = plainTextNotification(message)+ const fetchSpy = signEnvelope(envelope)+ const before = await ignoredCount()++ const result = await handler.handleWebhook({ body: envelope, headers: {}, verifySignature: true })++ expect(result.status).toBe(200)+ expect(result.metrics).toEqual([])+ expect(await ignoredCount()).toBe(before + 1)+ expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('not JSON'), {+ topicArn: envelope.TopicArn,+ messageId: envelope.MessageId,+ messageLength: message.length,+ messagePrefix: logged,+ })+ fetchSpy.mockRestore()+ warnSpy.mockRestore()+ }+ )++ it('is rejected when its signature does not verify', async () => {+ const fetchSpy = jest.spyOn(handler as any, 'fetchText').mockResolvedValue('not a certificate')+ const before = await ignoredCount()++ const result = await handler.handleWebhook({+ body: plainTextNotification(),+ headers: {},+ verifySignature: true,+ })++ expect(result.status).toBe(403)+ expect(await ignoredCount()).toBe(before)+ fetchSpy.mockRestore()+ })+ })+it.each([['non-SNS SubscribeURL', 'https://evil.lhr.life/latest/meta-data/iam/security-credentials/role'],['HTTP SubscribeURL', 'http://sns.us-east-1.amazonaws.com/subscribe'],diff --git a/nodejs/src/cdp/services/messaging/helpers/ses.ts b/nodejs/src/cdp/services/messaging/helpers/ses.tsindex ecddcfd4ef1c..6bbafadffa33 100644--- a/nodejs/src/cdp/services/messaging/helpers/ses.ts+++ b/nodejs/src/cdp/services/messaging/helpers/ses.ts@@ -234,6 +234,14 @@ const sesUnattributedEventsCounter = new Counter({labelNames: ['event_type', 'reason'],})+// Kept off `email_tracking_errors_total` because the SES webhook errors alert sums that counter, and an+// SNS message that is not an SES event is acknowledged, not a fault.+const sesIgnoredNotificationsCounter = new Counter({+ name: 'ses_webhook_ignored_notifications_total',+ help: 'SNS notifications on the SES webhook acknowledged without processing',+ labelNames: ['reason'],+})+// SES link tag naming an anchor's ordinal position in the email body. Set on each `<a>` at send// time and echoed back on the Click event, which is what lets two anchors pointing at the same URL// (a header logo and a footer link, say) be counted separately. SES restricts tag names and values@@ -479,6 +487,11 @@ export const formatSesEventLogs = (rec: SesEventRecord): SesEventLogLine[] => {}}+// Only text that does not open like JSON is quoted, because a malformed SES event carries recipients.+// Any AWS account can send it when the topic allowlist is empty, so it is sanitized like other SES fields.+const plainTextPrefix = (message: string): string | undefined =>+ /^\s*[{[]/.test(message) ? undefined : sanitizeSesField(message, 200)+export class SesWebhookHandler {certCache: Record<string, Promise<string> | undefined> = {}private allowedTopicArns: Set<string>@@ -570,19 +583,10 @@ export class SesWebhookHandler {*/private parseIncomingBody(body: unknown- ):- | { mode: 'raw' | 'sns'; records: SesEventRecord[] }- | { mode: 'sns'; envelope: SnsEnvelope; records?: SesEventRecord[] } {+ ): { mode: 'raw'; records: SesEventRecord[] } | { mode: 'sns'; envelope: SnsEnvelope } {// If it's already an object with "Type", it's probably the SNS envelope (raw=false)if (body && typeof body === 'object' && 'Type' in (body as any)) {- const env = SnsEnvelopeSchema.parse(body)- if (env.Type === 'Notification') {- const inner = parseJSON(env.Message)- const record = SesEventRecordSchema.parse(inner)- return { mode: 'sns', envelope: env, records: [record] }- }- // For non-Notification, return envelope; caller decides how to handle- return { mode: 'sns', envelope: env }+ return { mode: 'sns', envelope: SnsEnvelopeSchema.parse(body) }}// raw_message_delivery=true → body is already the SES array@@ -590,6 +594,23 @@ export class SesWebhookHandler {return { mode: 'raw', records }}+ private parseNotificationRecords(envelope: SnsEnvelope): SesEventRecord[] {+ let message: unknown+ try {+ message = parseJSON(envelope.Message)+ } catch {+ sesIgnoredNotificationsCounter.inc({ reason: 'non_json_message' })+ logger.warn('[SesWebhookHandler] Ignoring Notification whose Message is not JSON', {+ topicArn: envelope.TopicArn,+ messageId: envelope.MessageId,+ messageLength: envelope.Message.length,+ messagePrefix: plainTextPrefix(envelope.Message),+ })+ return []+ }+ return [SesEventRecordSchema.parse(message)]+ }+/*** Verify SNS signature* Best practice: verify the signature unless you're behind a trusted ALB / private VPC.@@ -743,7 +764,7 @@ export class SesWebhookHandler {}// Handle confirmation flow- if (parsed.mode === 'sns' && 'envelope' in parsed && parsed.envelope?.Type === 'SubscriptionConfirmation') {+ if (parsed.mode === 'sns' && parsed.envelope.Type === 'SubscriptionConfirmation') {logger.info('[SesWebhookHandler] confirming subscription', { envelope: parsed.envelope })// Confirm by visiting the top-level SubscribeURL field; Message is human-readable text here.const subscribeUrl = parsed.envelope.SubscribeURL@@ -762,13 +783,13 @@ export class SesWebhookHandler {return { status: 200, body: { ok: true } }}- if (parsed.mode === 'sns' && 'envelope' in parsed && parsed.envelope?.Type === 'UnsubscribeConfirmation') {+ if (parsed.mode === 'sns' && parsed.envelope.Type === 'UnsubscribeConfirmation') {logger.info('[SesWebhookHandler] confirming unsubscribe', { envelope: parsed.envelope })return { status: 200, body: { ok: true } }}- // Process SES events- const records = parsed.mode === 'raw' ? parsed.records : parsed.records!+ const records = parsed.mode === 'raw' ? parsed.records : this.parseNotificationRecords(parsed.envelope)+const metrics: {functionId?: stringinvocationId?: string