:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(workflows): suppress hard-bounced addresses across teams (#128089)
Harley Alexandere0df36436e30
diff --git a/nodejs/src/cdp/cdp-services.ts b/nodejs/src/cdp/cdp-services.tsindex 03a5fdb36ef5..f2e268ea6873 100644--- a/nodejs/src/cdp/cdp-services.ts+++ b/nodejs/src/cdp/cdp-services.ts@@ -176,6 +176,7 @@ export type CdpCoreServicesConfig = Pick<| 'SES_TRACKED_CONFIGURATION_SET'| 'SES_UNTRACKED_CONFIGURATION_SET'| 'EMAIL_SUPPRESSION_TRANSIENT_BOUNCE_THRESHOLD'+ | 'EMAIL_SUPPRESSION_GLOBAL_HARD_BOUNCE_ENFORCE'| 'EMAIL_TEAM_SENDING_CAP_MODE'| 'EMAIL_TEAM_SENDING_CAP_HOURLY_BY_TIER'| 'EMAIL_TEAM_SENDING_CAP_DAILY_BY_TIER'@@ -454,6 +455,7 @@ export function createCdpCoreServices(// choke point regardless of whether the invocation came from a workflow action or a hog function.const emailSuppressionService = new EmailSuppressionService(deps.postgres, {transientBounceThreshold: config.EMAIL_SUPPRESSION_TRANSIENT_BOUNCE_THRESHOLD,+ globalHardBounceEnforce: config.EMAIL_SUPPRESSION_GLOBAL_HARD_BOUNCE_ENFORCE,})const recipientsManager = new RecipientsManagerService(deps.postgres)const personsManager = new PersonsManagerService(deps.teamManager, deps.personRepository, config.SITE_URL)diff --git a/nodejs/src/cdp/config.ts b/nodejs/src/cdp/config.tsindex 5e70db94dc93..38d36652a100 100644--- a/nodejs/src/cdp/config.ts+++ b/nodejs/src/cdp/config.ts@@ -180,6 +180,9 @@ export type CdpConfig = ClickhouseConfig & {// Consecutive soft bounces before an address is auto-suppressed. Tunable without a deploy.EMAIL_SUPPRESSION_TRANSIENT_BOUNCE_THRESHOLD: number+ // When false, the global hard-bounce list only counts the sends it would block.+ EMAIL_SUPPRESSION_GLOBAL_HARD_BOUNCE_ENFORCE: boolean+// Trust-tiered per-team workflow email caps ("team warming"). A team's tier, stored by the// Django side, picks an hourly and a daily cap from these tables. See// WORKFLOWS_EMAIL_TIER_* in posthog/settings/web.py, which must stay in step with them.@@ -425,6 +428,7 @@ export function getDefaultCdpConfig(): CdpConfig {SES_UNTRACKED_CONFIGURATION_SET: '',SES_ALLOWED_SNS_TOPIC_ARNS: '',EMAIL_SUPPRESSION_TRANSIENT_BOUNCE_THRESHOLD: 5,+ EMAIL_SUPPRESSION_GLOBAL_HARD_BOUNCE_ENFORCE: false,// Ships dark: tiers are computed and stored first, then observed in shadow mode, and only// then enforced. Match the defaults in posthog/settings/web.py.diff --git a/nodejs/src/cdp/services/messaging/email-suppression.service.test.ts b/nodejs/src/cdp/services/messaging/email-suppression.service.test.tsindex 9ded72ae7bf8..41dd48b9ae96 100644--- a/nodejs/src/cdp/services/messaging/email-suppression.service.test.ts+++ b/nodejs/src/cdp/services/messaging/email-suppression.service.test.ts@@ -358,6 +358,133 @@ describe('EmailSuppressionService', () => {})})+ describe('global hard-bounce list', () => {+ const newService = (globalHardBounceEnforce: boolean): EmailSuppressionService =>+ new EmailSuppressionService(hub.postgres, { ...emailSuppressionConfigFromEnv(), globalHardBounceEnforce })++ // The list is instance-global and outlives each test's team, so every test uses its own address.+ const bounceForAnotherTeam = async (email: string): Promise<number> => {+ const otherTeamId = await createTeam(hub.postgres, team.organization_id)+ const svc = newService(true)+ await svc.recordHardBounces(otherTeamId, [email], 'smtp; 550 5.1.1 user unknown')+ await svc.recordGlobalHardBounces([email])+ return otherTeamId+ }++ it.each([+ ['shadow', false, false, undefined],+ ['enforce', true, true, 'BOUNCE'],+ ] as const)(+ 'in %s mode, another team bouncing the address blocks it: %s',+ async (mode, enforce, blocked, source) => {+ const email = `gone-${mode}-${team.id}@example.com`+ await bounceForAnotherTeam(email)++ const svc = newService(enforce)+ expect(await svc.isSuppressed(team.id, email, { transactional: true })).toBe(blocked)+ expect((await readRow(email))?.source).toBe(source)+ }+ )++ it.each([+ ['the team removed the address', 'removed'],+ ['a later delivery cleared the entry', 'delivered'],+ ] as const)('does not block when %s', async (_label, step) => {+ const email = `back-${step}-${team.id}@example.com`+ const otherTeamId = await bounceForAnotherTeam(email)++ if (step === 'removed') {+ await hub.postgres.query(+ PostgresUse.COMMON_WRITE,+ `INSERT INTO posthog_messagesuppression+ (id, team_id, identifier, source, reason, transient_bounce_count,+ suppressed, suppressed_at, deleted, created_at, updated_at)+ VALUES (gen_random_uuid(), $1, $2, 'BOUNCE', NULL, 0, false, NULL, true, NOW(), NOW())`,+ [team.id, email],+ 'test-insert-removed-suppression'+ )+ } else {+ // Bounces stamp last_bounced_at with NOW(), so a delivery has to be newer than that.+ await newService(true).recordDeliveries(+ otherTeamId,+ [email],+ new Date(Date.now() + 60_000).toISOString(),+ {+ clearGlobal: true,+ }+ )+ }++ expect(await newService(true).isSuppressed(team.id, email)).toBe(false)+ })++ it("blocks transactional email when the team's own entry only blocks other email", async () => {+ const email = `complained-${team.id}@example.com`+ await bounceForAnotherTeam(email)+ const svc = newService(true)+ await svc.recordComplaints(team.id, [email], 'abuse')++ expect(await svc.isSuppressed(team.id, email, { transactional: true })).toBe(true)+ })++ it('clears the entry for a delivery that happened after the bounce but arrived first', async () => {+ const email = `late-bounce-${team.id}@example.com`+ const svc = newService(true)+ const bounceAt = new Date(Date.now() - 120_000).toISOString()+ const deliveredAt = new Date(Date.now() - 60_000).toISOString()+ await svc.recordGlobalHardBounces([email], bounceAt)+ await svc.recordDeliveries(team.id, [email], deliveredAt, { clearGlobal: true })++ expect(await newService(true).isSuppressed(team.id, email)).toBe(false)+ })++ it.each([+ ['no other event', false],+ ['a delayed soft bounce for an earlier send', true],+ ])(+ "releases a team's copied block once a delivery clears the global entry after %s",+ async (_label, softBounce) => {+ const email = `recovered-${softBounce ? 'soft' : 'plain'}-${team.id}@example.com`+ const otherTeamId = await bounceForAnotherTeam(email)+ expect(await newService(true).isSuppressed(team.id, email)).toBe(true)+ expect(await newService(true).isSuppressed(team.id, email)).toBe(true)+ if (softBounce) {+ await newService(true).recordTransientBounces(team.id, [email], 'smtp; 421 try again later')+ }++ await newService(true).recordDeliveries(+ otherTeamId,+ [email],+ new Date(Date.now() + 60_000).toISOString(),+ {+ clearGlobal: true,+ }+ )++ expect(await newService(true).isSuppressed(team.id, email, { transactional: true })).toBe(false)+ expect(await readRow(email)).toMatchObject({ suppressed: false, deleted: false })+ }+ )++ it('does not undo a removal that a lagging read missed', async () => {+ const email = `removed-${team.id}@example.com`+ await bounceForAnotherTeam(email)+ await hub.postgres.query(+ PostgresUse.COMMON_WRITE,+ `INSERT INTO posthog_messagesuppression+ (id, team_id, identifier, source, reason, transient_bounce_count,+ suppressed, suppressed_at, deleted, created_at, updated_at)+ VALUES (gen_random_uuid(), $1, $2, 'BOUNCE', NULL, 0, false, NULL, true, NOW(), NOW())`,+ [team.id, email],+ 'test-insert-removed-suppression'+ )++ await (newService(true) as any).upsertHardBounces(team.id, [email], 'copied', { keepRemoved: true })++ expect(await readRow(email)).toMatchObject({ suppressed: false, deleted: true })+ })+ })+describe('cache invalidation', () => {it("a write for one team does not evict another team's cached entries", async () => {// Guards against a regression from markForRefresh(touchedKeys) back to clear() — clear()diff --git a/nodejs/src/cdp/services/messaging/email-suppression.service.ts b/nodejs/src/cdp/services/messaging/email-suppression.service.tsindex e7aeab8e9c14..45ac2b3e90ed 100644--- a/nodejs/src/cdp/services/messaging/email-suppression.service.ts+++ b/nodejs/src/cdp/services/messaging/email-suppression.service.ts@@ -7,7 +7,7 @@ import { isValidTeamId } from '~/common/utils/team-manager'const cdpEmailSuppressionTotal = new Counter({name: 'cdp_email_suppression_total',- help: 'Email suppression-list outcomes. `suppressed_hit` = a send skipped because the recipient is on the list; `transient_bounce` = a soft-bounce counter increment; `hard_bounce` = an address suppressed immediately after a permanent bounce; `complaint` = an address suppressed immediately after a spam complaint.',+ help: 'Email suppression-list outcomes. `suppressed_hit` = a send skipped because the recipient is on the list; `transient_bounce` = a soft-bounce counter increment; `hard_bounce` = an address suppressed immediately after a permanent bounce; `complaint` = an address suppressed immediately after a spam complaint; `global_hard_bounce` = an address added to the global hard-bounce list; `global_hard_bounce_cleared` = an address removed from that list after a delivery; `global_hard_bounce_hit` = a send skipped because of that list; `global_hard_bounce_shadow_hit` = a send that list would skip once enforced.',labelNames: ['result'],})@@ -20,6 +20,7 @@ const toKey = (teamId: number, identifier: string): string => `${teamId}:${identexport type EmailSuppressionConfig = {// Consecutive soft bounces before auto-suppression.transientBounceThreshold: number+ globalHardBounceEnforce?: boolean}// Test helper — production paths take config from `CdpConfig`; tests that still mutate process.env@@ -38,10 +39,20 @@ export type SuppressionCheckOptions = {transactional?: boolean}+// `unlisted`: no active row on the team's list. `allowsTransactional`: an active row that lets transactional+// email through. `copy`: a row this service copied from the global list, which only holds while the global entry does.+type GlobalCandidate = {+ key: string+ teamId: number+ identifier: string+ kind: 'unlisted' | 'allowsTransactional' | 'copy'+}+// An active suppression. It blocks transactional email only when the address hard-bounced.-type SuppressionEntry = { blocksTransactional: boolean }+type SuppressionEntry = { blocksTransactional: boolean; global?: 'shadow' | 'enforce' }const HARD_BOUNCE_REASON = 'Auto-suppressed after a hard bounce'+const GLOBAL_HARD_BOUNCE_DIAGNOSTIC = "This address doesn't exist. An earlier email to it hard-bounced."// Reasons that mark a dead address on rows written before `hard_bounced_at` existed: our own hard-bounce// rows, and the two Customer.io lists the importer maps to BOUNCE. Both writers now set the column, so@@ -78,6 +89,7 @@ const LEGACY_HARD_BOUNCE_REASONS = [*/export class EmailSuppressionService {private readonly threshold: number+ private readonly globalHardBounceEnforce: boolean// A key's active suppression, or false when it has none.private readonly lazyLoader: LazyLoader<SuppressionEntry | false>@@ -86,6 +98,7 @@ export class EmailSuppressionService {config: EmailSuppressionConfig) {this.threshold = config.transientBounceThreshold+ this.globalHardBounceEnforce = config.globalHardBounceEnforce ?? falsethis.lazyLoader = new LazyLoader<SuppressionEntry | false>({name: 'email_suppression',loader: async (keys) => await this.loadSuppressed(keys),@@ -108,9 +121,13 @@ export class EmailSuppressionService {}try {const entry = (await this.lazyLoader.get(toKey(teamId, identifier))) ?? false+ if (entry !== false && entry.global === 'shadow') {+ cdpEmailSuppressionTotal.inc({ result: 'global_hard_bounce_shadow_hit' })+ return false+ }const suppressed = entry !== false && (!transactional || entry.blocksTransactional)if (suppressed) {- cdpEmailSuppressionTotal.inc({ result: 'suppressed_hit' })+ cdpEmailSuppressionTotal.inc({ result: entry.global ? 'global_hard_bounce_hit' : 'suppressed_hit' })}return suppressed} catch (error) {@@ -143,6 +160,7 @@ export class EmailSuppressionService {this.threshold,diag,LEGACY_HARD_BOUNCE_REASONS,+ GLOBAL_HARD_BOUNCE_DIAGNOSTIC,]identifiers.forEach((identifier, i) => {const p = params.length + 1 + i@@ -162,7 +180,12 @@ export class EmailSuppressionService {ON CONFLICT (team_id, identifier) DO UPDATE SETtransient_bounce_count = posthog_messagesuppression.transient_bounce_count + 1,last_bounce_at = NOW(),- last_bounce_diagnostic = EXCLUDED.last_bounce_diagnostic,+ -- The diagnostic marks a copy of a global hard bounce, which a delivery must still be able to release.+ last_bounce_diagnostic = CASE+ WHEN posthog_messagesuppression.source = 'BOUNCE'+ AND posthog_messagesuppression.last_bounce_diagnostic = $6+ THEN posthog_messagesuppression.last_bounce_diagnostic+ ELSE EXCLUDED.last_bounce_diagnostic END,-- Never downgrade or re-key a manual entry; only auto (BOUNCE) rows can cross the threshold.suppressed = CASEWHEN posthog_messagesuppression.source = 'MANUAL' THEN posthog_messagesuppression.suppressed@@ -216,6 +239,22 @@ export class EmailSuppressionService {return}+ try {+ await this.upsertHardBounces(teamId, identifiers, diagnostic)+ cdpEmailSuppressionTotal.inc({ result: 'hard_bounce' }, identifiers.length)+ // Only invalidate the keys we touched — cross-team entries stay warm.+ this.lazyLoader.markForRefresh(identifiers.map((identifier) => toKey(teamId, identifier)))+ } catch (error) {+ logger.error('[EmailSuppression] Failed to record hard bounces', { teamId, error })+ }+ }++ private async upsertHardBounces(+ teamId: number,+ identifiers: string[],+ diagnostic?: string,+ { keepRemoved = false }: { keepRemoved?: boolean } = {}+ ): Promise<void> {const diag = diagnostic ? diagnostic.slice(0, DIAGNOSTIC_MAX_LENGTH) : null// Params: teamId, reason, diag, then one identifier per row.const valueClauses: string[] = []@@ -260,15 +299,36 @@ export class EmailSuppressionService {WHEN posthog_messagesuppression.source = 'MANUAL' THEN posthog_messagesuppression.deletedELSE false END,updated_at = NOW()+ ${keepRemoved ? 'WHERE NOT posthog_messagesuppression.deleted' : ''}`+ await this.postgres.query(PostgresUse.COMMON_WRITE, query, params, 'recordHardBounces')+ }++ /** Only for a bounce that proves the address does not exist, or one team could block valid addresses for all. */+ public async recordGlobalHardBounces(emails: string[], bounceTimestamp?: string): Promise<void> {+ const identifiers = Array.from(new Set(emails.map(normalizeIdentifier).filter(Boolean)))+ if (identifiers.length === 0) {+ return+ }+ // Store when SES saw the bounce, because the bounce and delivery webhooks can arrive in either order.+ const valueClauses = identifiers.map(+ (_, i) => `($${i + 2}, COALESCE($1::timestamptz, NOW()), COALESCE($1::timestamptz, NOW()))`+ )try {- await this.postgres.query(PostgresUse.COMMON_WRITE, query, params, 'recordHardBounces')- cdpEmailSuppressionTotal.inc({ result: 'hard_bounce' }, identifiers.length)- // Only invalidate the keys we touched — cross-team entries stay warm.- this.lazyLoader.markForRefresh(identifiers.map((identifier) => toKey(teamId, identifier)))+ await this.postgres.query(+ PostgresUse.COMMON_WRITE,+ `INSERT INTO posthog_emailhardbounce (email, first_bounced_at, last_bounced_at)+ VALUES ${valueClauses.join(', ')}+ ON CONFLICT (email) DO UPDATE SET+ first_bounced_at = LEAST(posthog_emailhardbounce.first_bounced_at, EXCLUDED.first_bounced_at),+ last_bounced_at = GREATEST(posthog_emailhardbounce.last_bounced_at, EXCLUDED.last_bounced_at)`,+ [bounceTimestamp ?? null, ...identifiers],+ 'recordGlobalHardBounces'+ )+ cdpEmailSuppressionTotal.inc({ result: 'global_hard_bounce' }, identifiers.length)} catch (error) {- logger.error('[EmailSuppression] Failed to record hard bounces', { teamId, error })+ logger.error('[EmailSuppression] Failed to record global hard bounces', { error })}}@@ -356,7 +416,12 @@ export class EmailSuppressionService {* erase a fresh bounce for a newer send. Reset only when the delivery is newer than the last* recorded bounce.*/- public async recordDeliveries(teamId: number, emails: string[], deliveryTimestamp?: string): Promise<void> {+ public async recordDeliveries(+ teamId: number,+ emails: string[],+ deliveryTimestamp?: string,+ { clearGlobal = false }: { clearGlobal?: boolean } = {}+ ): Promise<void> {const identifiers = Array.from(new Set(emails.map(normalizeIdentifier).filter(Boolean)))if (identifiers.length === 0) {return@@ -387,6 +452,29 @@ export class EmailSuppressionService {} catch (error) {logger.error('[EmailSuppression] Failed to reset bounce counts on delivery', { teamId, error })}++ if (!clearGlobal) {+ return+ }++ // A delivery proves the address exists. The timestamp guard stops a late, older delivery from+ // clearing a fresh bounce.+ try {+ const result = await this.postgres.query(+ PostgresUse.COMMON_WRITE,+ `DELETE FROM posthog_emailhardbounce+ WHERE email = ANY($1)+ AND $2::timestamptz IS NOT NULL+ AND last_bounced_at < $2::timestamptz`,+ [identifiers, deliveryTimestamp ?? null],+ 'clearGlobalHardBounces'+ )+ if (result.rowCount) {+ cdpEmailSuppressionTotal.inc({ result: 'global_hard_bounce_cleared' }, result.rowCount)+ }+ } catch (error) {+ logger.error('[EmailSuppression] Failed to clear global hard bounces on delivery', { teamId, error })+ }}private async loadSuppressed(keys: string[]): Promise<Record<string, SuppressionEntry | false>> {@@ -413,11 +501,20 @@ export class EmailSuppressionService {}const entries = new Map<string, SuppressionEntry>()+ // A removed row is the team's override of the global list, so it is never a candidate.+ const candidates: GlobalCandidate[] = []for (const [teamId, identifiers] of byTeam) {// The reason check covers rows that hard-bounced before `hard_bounced_at` existed.- const result = await this.postgres.query<{ identifier: string; blocks_transactional: boolean }>(+ const result = await this.postgres.query<{+ identifier: string+ deleted: boolean+ is_global_copy: boolean+ blocks_transactional: boolean+ }>(PostgresUse.COMMON_READ,`SELECT identifier,+ deleted,+ COALESCE(source = 'BOUNCE' AND last_bounce_diagnostic = $4, false) AS is_global_copy,-- A soft-bounce row carries no reason until it crosses the threshold, and-- NULL = ANY(...) is NULL, so COALESCE keeps the column a real boolean.COALESCE(hard_bounced_at IS NOT NULL OR (source = 'BOUNCE' AND reason = ANY($3)), false)@@ -425,17 +522,122 @@ export class EmailSuppressionService {FROM posthog_messagesuppressionWHERE team_id = $1AND identifier = ANY($2)- AND suppressed = true- AND deleted = false`,- [teamId, identifiers, LEGACY_HARD_BOUNCE_REASONS],+ AND (deleted = true OR suppressed = true)`,+ [teamId, identifiers, LEGACY_HARD_BOUNCE_REASONS, GLOBAL_HARD_BOUNCE_DIAGNOSTIC],'loadSuppressed')+ const listed = new Set<string>()for (const row of result.rows) {- entries.set(toKey(teamId, row.identifier), { blocksTransactional: row.blocks_transactional })+ listed.add(row.identifier)+ if (row.deleted) {+ continue+ }+ const key = toKey(teamId, row.identifier)+ entries.set(key, { blocksTransactional: row.blocks_transactional })+ if (row.is_global_copy) {+ candidates.push({ key, teamId, identifier: row.identifier, kind: 'copy' })+ } else if (!row.blocks_transactional) {+ candidates.push({ key, teamId, identifier: row.identifier, kind: 'allowsTransactional' })+ }+ }+ for (const identifier of identifiers) {+ if (!listed.has(identifier)) {+ candidates.push({ key: toKey(teamId, identifier), teamId, identifier, kind: 'unlisted' })+ }}}+ if (candidates.length > 0) {+ await this.applyGlobalHardBounces(candidates, entries)+ }+// Default every requested key to false so the LazyLoader caches negatives too.return Object.fromEntries(parsed.map((p) => [p.key, entries.get(p.key) ?? false]))}++ private async applyGlobalHardBounces(+ candidates: GlobalCandidate[],+ entries: Map<string, SuppressionEntry>+ ): Promise<void> {+ let bounced: Set<string>+ try {+ const result = await this.postgres.query<{ email: string }>(+ PostgresUse.COMMON_READ,+ `SELECT email FROM posthog_emailhardbounce WHERE email = ANY($1)`,+ [Array.from(new Set(candidates.map((c) => c.identifier)))],+ 'loadGlobalHardBounces'+ )+ bounced = new Set(result.rows.map((row) => row.email))+ } catch (error) {+ // Fail open, like the rest of the pre-send check, and keep the team's own list.+ logger.error('[EmailSuppression] Failed to check global hard bounces', { error })+ return+ }++ const hitsByTeam = new Map<number, string[]>()+ const releasesByTeam = new Map<number, string[]>()+ for (const { key, teamId, identifier, kind } of candidates) {+ if (kind === 'copy') {+ if (!bounced.has(identifier)) {+ // A delivery cleared the global entry, which proves the address exists.+ entries.delete(key)+ releasesByTeam.set(teamId, [...(releasesByTeam.get(teamId) ?? []), identifier])+ } else if (!this.globalHardBounceEnforce) {+ entries.set(key, { blocksTransactional: false, global: 'shadow' })+ }+ continue+ }+ if (!bounced.has(identifier)) {+ continue+ }+ if (kind === 'allowsTransactional') {+ // The team's own list already blocks other email, so only transactional email changes.+ if (this.globalHardBounceEnforce) {+ entries.set(key, { blocksTransactional: true })+ }+ continue+ }+ if (!this.globalHardBounceEnforce) {+ entries.set(key, { blocksTransactional: false, global: 'shadow' })+ continue+ }+ entries.set(key, { blocksTransactional: true, global: 'enforce' })+ hitsByTeam.set(teamId, [...(hitsByTeam.get(teamId) ?? []), identifier])+ }++ // The copy lets the team see and remove the address. The read can lag the primary, so it never+ // touches a row the team removed meanwhile.+ for (const [teamId, identifiers] of hitsByTeam) {+ try {+ await this.upsertHardBounces(teamId, identifiers, GLOBAL_HARD_BOUNCE_DIAGNOSTIC, { keepRemoved: true })+ } catch (error) {+ logger.error('[EmailSuppression] Failed to copy global hard bounces to the team list', {+ teamId,+ error,+ })+ }+ }++ // The WHERE clause re-checks the row is still an untouched copy, because a real bounce can land+ // between the read and this write.+ for (const [teamId, identifiers] of releasesByTeam) {+ try {+ await this.postgres.query(+ PostgresUse.COMMON_WRITE,+ `UPDATE posthog_messagesuppression+ SET suppressed = false, hard_bounced_at = NULL, reason = NULL,+ last_bounce_diagnostic = NULL, updated_at = NOW()+ WHERE team_id = $1+ AND identifier = ANY($2)+ AND source = 'BOUNCE'+ AND last_bounce_diagnostic = $3+ AND deleted = false`,+ [teamId, identifiers, GLOBAL_HARD_BOUNCE_DIAGNOSTIC],+ 'releaseGlobalHardBounceCopies'+ )+ } catch (error) {+ logger.error('[EmailSuppression] Failed to release copied global hard bounces', { teamId, error })+ }+ }+ }}diff --git a/nodejs/src/cdp/services/messaging/email-tracking.service.ts b/nodejs/src/cdp/services/messaging/email-tracking.service.tsindex 6bcea0f6056f..4e6cfc312e01 100644--- a/nodejs/src/cdp/services/messaging/email-tracking.service.ts+++ b/nodejs/src/cdp/services/messaging/email-tracking.service.ts@@ -579,10 +579,13 @@ export class EmailTrackingService {// processed first so a delivery + bounce in the same batch nets out conservatively// (count resets, then the fresh bounce re-counts from a clean slate).try {- for (const { teamId, emailAddresses, timestamp } of deliveredRecipients || []) {+ for (const { teamId, emailAddresses, timestamp, clearsGlobalHardBounces } of deliveredRecipients ||+ []) {const parsedTeamId = teamId ? parseInt(teamId, 10) : NaNif (parsedTeamId && !isNaN(parsedTeamId)) {- await this.emailSuppressionService.recordDeliveries(parsedTeamId, emailAddresses, timestamp)+ await this.emailSuppressionService.recordDeliveries(parsedTeamId, emailAddresses, timestamp, {+ clearGlobal: clearsGlobalHardBounces ?? false,+ })}}for (const { teamId, emailAddresses, diagnostic } of transientBounceRecipients || []) {@@ -595,10 +598,22 @@ export class EmailTrackingService {)}}- for (const { teamId, emailAddresses, diagnostic } of hardBounceRecipients || []) {+ for (const {+ teamId,+ emailAddresses,+ diagnostic,+ nonexistentAddresses,+ bounceTimestamp,+ } of hardBounceRecipients ?? []) {const parsedTeamId = teamId ? parseInt(teamId, 10) : NaNif (parsedTeamId && !isNaN(parsedTeamId)) {await this.emailSuppressionService.recordHardBounces(parsedTeamId, emailAddresses, diagnostic)+ if (nonexistentAddresses) {+ await this.emailSuppressionService.recordGlobalHardBounces(+ nonexistentAddresses,+ bounceTimestamp+ )+ }}}for (const { teamId, emailAddresses, feedbackType } of complainedRecipients || []) {diff --git a/nodejs/src/cdp/services/messaging/helpers/ses.test.ts b/nodejs/src/cdp/services/messaging/helpers/ses.test.tsindex 1578818dab63..fe5a1c6bd744 100644--- a/nodejs/src/cdp/services/messaging/helpers/ses.test.ts+++ b/nodejs/src/cdp/services/messaging/helpers/ses.test.ts@@ -3,7 +3,7 @@ import { register } from 'prom-client'import { defaultConfig } from '~/common/config/config'import { logger } from '~/common/utils/logger'-import { SesWebhookHandler, normalizeClickUrl, resolveClickDestination } from './ses'+import { SesWebhookHandler, nonexistentRecipients, normalizeClickUrl, resolveClickDestination } from './ses'import { EmailTrackingCodeSigner } from './tracking-code'// Hardcoded (not imported) so a change to the header constant fails this test.@@ -1106,6 +1106,50 @@ describe('SesWebhookHandler', () => {SigningCertURL: 'https://sns.us-east-1.amazonaws.com/cert.pem',})+ const globalBounce = {+ eventType: 'Bounce',+ mail: baseMail,+ bounce: {+ bounceType: 'Permanent',+ bounceSubType: 'General',+ bouncedRecipients: [+ { emailAddress: '[email protected]', status: '5.1.1' },+ { emailAddress: '[email protected]', status: '5.1.1' },+ ],+ timestamp: '2025-10-03T12:04:00Z',+ },+ }+ const globalDelivery = {+ eventType: 'Delivery',+ mail: baseMail,+ delivery: { timestamp: '2025-10-03T12:05:00Z', recipients: ['[email protected]'] },+ }++ it.each([+ ['an allowlisted topic', ['arn:aws:sns:us-east-1:123456789012:allowed-topic'], true, ['[email protected]']],+ ['an empty allowlist', [], true, undefined],+ ['a raw delivery', ['arn:aws:sns:us-east-1:123456789012:allowed-topic'], false, undefined],+ ] as const)(+ 'writes the global hard-bounce list only for sent-to addresses from %s',+ async (_label, topics, enveloped, expected) => {+ const restricted = new SesWebhookHandler(signer, [...topics])+ const wrap = (record: object): object =>+ enveloped ? buildEnvelope('arn:aws:sns:us-east-1:123456789012:allowed-topic', record) : [record]+ const bounce = await restricted.handleWebhook({+ body: wrap(globalBounce),+ headers: {},+ verifySignature: false,+ })+ const delivery = await restricted.handleWebhook({+ body: wrap(globalDelivery),+ headers: {},+ verifySignature: false,+ })+ expect(bounce.hardBounceRecipients?.[0].nonexistentAddresses).toEqual(expected)+ expect(delivery.deliveredRecipients?.[0].clearsGlobalHardBounces).toEqual(expected ? true : undefined)+ }+ )+it('rejects a Notification whose TopicArn is not on the allowlist', async () => {const restricted = new SesWebhookHandler(signer, ['arn:aws:sns:us-east-1:123456789012:allowed-topic'])const envelope = buildEnvelope('arn:aws:sns:us-east-1:999999999999:other-topic', {@@ -1217,4 +1261,31 @@ describe('SesWebhookHandler', () => {expect(result.metrics?.length).toBeGreaterThan(0)})})+ describe('nonexistentRecipients', () => {+ // A sender-side rejection going global would let one team's broken DNS block valid addresses for all.+ it.each([+ ['Permanent', 'NoEmail', undefined, undefined, true],+ ['Permanent', 'General', '5.1.1', 'smtp; 550 5.1.1 user unknown', true],+ ['Permanent', 'General', '5.1.10', 'smtp; 556 5.1.10 null MX', true],+ ['Permanent', 'General', undefined, 'smtp; 550 5.1.1 user unknown', true],+ ['Permanent', 'General', '5.7.1', 'smtp; 550 5.7.1 rejected by DMARC policy', false],+ ['Permanent', 'General', '5.7.26', 'smtp; 550 5.7.26 rejected for 5.1.1.example.com', false],+ ['Permanent', 'General', undefined, 'smtp; 550 5.7.26 rejected for 5.1.1.example.com', false],+ ['Permanent', 'General', undefined, 'smtp; 554 sender domain blocked', false],+ ['Permanent', 'Suppressed', '5.1.1', undefined, false],+ ['Permanent', 'OnAccountSuppressionList', undefined, undefined, false],+ ['Transient', 'General', '5.1.1', 'smtp; 550 5.1.1 user unknown', false],+ ] as const)(+ '%s/%s bounce with status %s counts: %s',+ (bounceType, bounceSubType, status, diagnosticCode, expected) => {+ const recipients = nonexistentRecipients({+ bounceType,+ bounceSubType,+ bouncedRecipients: [{ emailAddress: '[email protected]', status, diagnosticCode }],+ timestamp: '2026-01-01T00:00:00.000Z',+ })+ expect(recipients).toEqual(expected ? ['[email protected]'] : [])+ }+ )+ })})diff --git a/nodejs/src/cdp/services/messaging/helpers/ses.ts b/nodejs/src/cdp/services/messaging/helpers/ses.tsindex 952e7922912e..ecddcfd4ef1c 100644--- a/nodejs/src/cdp/services/messaging/helpers/ses.ts+++ b/nodejs/src/cdp/services/messaging/helpers/ses.ts@@ -121,6 +121,28 @@ const SesBounceEventSchema = SesCommonEventBase.extend({}),})+// 5.1.1 (unknown user) and 5.1.10 (null MX). The diagnostic match is anchored after the reply code, because its+// free text can quote sender-chosen values such as a domain named 5.1.1.example.com.+const ADDRESS_DOES_NOT_EXIST_STATUS = /^5\.1\.(1|10)$/+const ADDRESS_DOES_NOT_EXIST_DIAGNOSTIC = /^(smtp;\s*)?5\d\d[\s-]+5\.1\.(1|10)(\s|$)/i++// SES files sender-side rejections such as DMARC failures under Permanent/General too, so General needs the status.+export function nonexistentRecipients(bounce: z.infer<typeof SesBounceEventSchema>['bounce']): string[] {+ if (bounce.bounceType !== 'Permanent') {+ return []+ }+ return bounce.bouncedRecipients+ .filter(+ (r) =>+ bounce.bounceSubType === 'NoEmail' ||+ (bounce.bounceSubType === 'General' &&+ (r.status+ ? ADDRESS_DOES_NOT_EXIST_STATUS.test(r.status.trim())+ : ADDRESS_DOES_NOT_EXIST_DIAGNOSTIC.test((r.diagnosticCode ?? '').trim())))+ )+ .map((r) => r.emailAddress)+}+const SesComplaintEventSchema = SesCommonEventBase.extend({eventType: z.literal('Complaint'),complaint: z.object({@@ -664,6 +686,9 @@ export class SesWebhookHandler {teamId?: stringemailAddresses: string[]diagnostic?: string+ // Addresses the bounce proves do not exist, for the global hard-bounce list.+ nonexistentAddresses?: string[]+ bounceTimestamp?: string}[]// Spam complaints — recorded in the suppression list so future sends to the address are// blocked. A complaint is the recipient asking us to stop, and provider complaint-rate@@ -680,6 +705,7 @@ export class SesWebhookHandler {teamId?: stringemailAddresses: string[]timestamp?: string+ clearsGlobalHardBounces?: boolean}[]// Status changes of transactional emails, for the email_status table.emailStatusUpdates?: SesEmailStatusUpdate[]@@ -774,6 +800,8 @@ export class SesWebhookHandler {teamId?: stringemailAddresses: string[]diagnostic?: string+ nonexistentAddresses?: string[]+ bounceTimestamp?: string}[] = []const complainedRecipients: {teamId?: string@@ -784,7 +812,11 @@ export class SesWebhookHandler {teamId?: stringemailAddresses: string[]timestamp?: string+ clearsGlobalHardBounces?: boolean}[] = []+ // A signature only proves AWS sent the event, from any account's topic. The global hard-bounce list+ // affects every team, so it only takes events from a configured topic.+ const fromTrustedTopic = 'envelope' in parsed && this.allowedTopicArns.size > 0const emailStatusUpdates: SesEmailStatusUpdate[] = []for (const rec of records) {@@ -958,7 +990,20 @@ export class SesWebhookHandler {if (suppressionAllowed && rec.eventType === 'Bounce' && rec.bounce.bounceType === 'Permanent') {const emails = rec.bounce.bouncedRecipients.map((r) => r.emailAddress)const diagnostic = rec.bounce.bouncedRecipients.find((r) => r.diagnosticCode)?.diagnosticCode- hardBounceRecipients.push({ teamId, emailAddresses: emails, diagnostic })+ // SES reports the DSN's final recipient, which the receiving server chooses. Only an address+ // this send went to can go on the global list.+ const sentTo = new Set(rec.mail.destination.map(normalizeAddress))+ const nonexistentAddresses = fromTrustedTopic+ ? nonexistentRecipients(rec.bounce).filter((email) => sentTo.has(normalizeAddress(email)))+ : []+ hardBounceRecipients.push({+ teamId,+ emailAddresses: emails,+ diagnostic,+ ...(nonexistentAddresses.length > 0+ ? { nonexistentAddresses, bounceTimestamp: rec.bounce.timestamp }+ : {}),+ })}// Count soft (Transient) bounces toward suppression. These are recipient-side failures@@ -995,7 +1040,12 @@ export class SesWebhookHandler {if (suppressionAllowed && rec.eventType === 'Delivery') {const emails = rec.delivery.recipients ?? rec.mail.destination ?? []if (emails.length > 0) {- deliveredRecipients.push({ teamId, emailAddresses: emails, timestamp: rec.delivery.timestamp })+ deliveredRecipients.push({+ teamId,+ emailAddresses: emails,+ timestamp: rec.delivery.timestamp,+ ...(fromTrustedTopic ? { clearsGlobalHardBounces: true } : {}),+ })}}