:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(cdp): record the queued and sent status of each transactional email (#124736)
Harley Alexanderef2369a75734
diff --git a/nodejs/src/cdp/cdp-api.serial.test.ts b/nodejs/src/cdp/cdp-api.serial.test.tsindex 233988af94a2..419637585c47 100644--- a/nodejs/src/cdp/cdp-api.serial.test.ts+++ b/nodejs/src/cdp/cdp-api.serial.test.ts@@ -2148,7 +2148,14 @@ describe('CDP API', () => {.set(headers).send(payload)+ let produceOutput: jest.SpyInstance+ const emailStatusRows = (): Record<string, unknown>[] =>+ produceOutput.mock.calls+ .filter(([output]) => output === 'email_status')+ .map(([, message]) => parseJSON(message.value.toString()))+beforeEach(async () => {+ produceOutput = jest.spyOn(api['emailStatusService']['outputs'], 'produce')mockQueueInvocations = jest.fn().mockResolvedValue(undefined)api['hogflowQueue'] = { queueInvocations: mockQueueInvocations } as anyhogFlow = await insertHogFlow(transactionalFlow('api', 'active'))@@ -2186,6 +2193,9 @@ describe('CDP API', () => {expect.objectContaining({ app_source_id: hogFlow.id, metric_name: 'triggered', count: 1 }),'hog_flow')+ expect(emailStatusRows()).toEqual([+ expect.objectContaining({ team_id: team.id, message_id: res.body.invocation_id, status: 'queued' }),+ ])})describe('with an idempotency key', () => {@@ -2211,6 +2221,9 @@ describe('CDP API', () => {replayed: true,})expect(mockQueueInvocations).toHaveBeenCalledTimes(1)+ expect(emailStatusRows()).toEqual([+ expect.objectContaining({ message_id: first.body.invocation_id, status: 'queued' }),+ ])})it('rejects the key for a different request', async () => {@@ -2333,6 +2346,7 @@ describe('CDP API', () => {expect(refused.body).toEqual({ error: code })expect(await outcomeCount(code)).toEqual(refusedBefore + 1)expect(mockQueueInvocations).not.toHaveBeenCalled()+ expect(emailStatusRows()).toEqual([])const retry = await send(hogFlow.id, headers, keyed)diff --git a/nodejs/src/cdp/cdp-api.ts b/nodejs/src/cdp/cdp-api.tsindex 225ea19ba3c3..e3bd345a759e 100644--- a/nodejs/src/cdp/cdp-api.ts+++ b/nodejs/src/cdp/cdp-api.ts@@ -64,6 +64,7 @@ import { GroupsManagerService } from './services/managers/groups-manager.serviceimport { HogFunctionManagerService } from './services/managers/hog-function-manager.service'import { TeamWorkflowsConfigService } from './services/managers/team-workflows-config.service'import { EMAIL_QUEUE_PRIORITY } from './services/messaging/email-priority'+import { EmailStatusService } from './services/messaging/email-status.service'import { EmailSuppressionService } from './services/messaging/email-suppression.service'import { EmailTrackingService } from './services/messaging/email-tracking.service'import { EmailValidationService } from './services/messaging/email-validation.service'@@ -202,6 +203,7 @@ export class CdpApi {private emailValidationService: EmailValidationServiceprivate teamWorkflowsConfigService: TeamWorkflowsConfigServiceprivate hogFunctionMonitoringService: HogFunctionMonitoringService+ private emailStatusService: EmailStatusServiceprivate resumeBatchJwt: ScopedServiceJwtconstructor(@@ -226,6 +228,7 @@ export class CdpApi {this.hogWatcher = services.hogWatcherthis.hogWatcherMirror = services.hogWatcherMirrorthis.invocationResultsService = services.invocationResultsService+ this.emailStatusService = services.emailStatusService// API-only services. The hog-transformer's monitoring service reuses the same// resolved outputs registry as the core CDP services — no separate construction.@@ -1260,7 +1263,12 @@ export class CdpApi {if (idempotencyKey) {await idempotencyKeys.markQueued(team.id, idempotencyKey, fingerprint, invocation.id)}- await this.invocationResultsService.invocationResultsRowsService.flush()+ this.emailStatusService.queue({ teamId: team.id, messageId: invocation.id, status: 'queued' })+ // Flushed together, and bounded, so the best-effort status row can't hold up the 202.+ await Promise.all([+ this.invocationResultsService.invocationResultsRowsService.flush(),+ this.emailStatusService.flush(),+ ])// The run skips the trigger pipeline that counts runs for event-triggered workflows.this.hogFunctionMonitoringService.queueAppMetric({diff --git a/nodejs/src/cdp/cdp-services.ts b/nodejs/src/cdp/cdp-services.tsindex 47643b470597..a61771e3e788 100644--- a/nodejs/src/cdp/cdp-services.ts+++ b/nodejs/src/cdp/cdp-services.ts@@ -34,6 +34,7 @@ import { IntegrationManagerService } from './services/managers/integration-managimport { PersonsManagerService } from './services/managers/persons-manager.service'import { RecipientsManagerService } from './services/managers/recipients-manager.service'import { TeamWorkflowsConfigService } from './services/managers/team-workflows-config.service'+import { EmailStatusService } from './services/messaging/email-status.service'import { EmailSuppressionService } from './services/messaging/email-suppression.service'import { EmailValidationService } from './services/messaging/email-validation.service'import { EmailService, parseTeamEmailCapMode, parseTierCaps } from './services/messaging/email.service'@@ -118,6 +119,8 @@ export interface CdpCoreServices {emailService: EmailService/** Buffers rendered-email asset rows and bulk-flushes them at the batch boundary. */messageAssetsService: MessageAssetsService+ /** Buffers transactional email status rows. `invocationResultsService.flush()` produces them. */+ emailStatusService: EmailStatusService}export type CdpCoreServicesConfig = Pick<@@ -188,6 +191,9 @@ export type CdpCoreServicesConfig = Pick<| 'HOG_INVOCATION_RESULTS_ENABLED'| 'MESSAGE_ASSETS_TOPIC'| 'MESSAGE_ASSETS_PRODUCER'+ | 'EMAIL_STATUS_TOPIC'+ | 'EMAIL_STATUS_PRODUCER'+ | 'EMAIL_STATUS_ENABLED'| 'CDP_WAREHOUSE_SOURCE_WEBHOOKS_TOPIC'| 'CDP_WAREHOUSE_SOURCE_WEBHOOKS_PRODUCER'| 'CDP_DLQ_ENABLED'@@ -435,6 +441,7 @@ export function createCdpCoreServices(const teamWorkflowsConfigService = new TeamWorkflowsConfigService(deps.postgres, deps.pubSub)const outputs = createCdpOutputsRegistry().build(deps.cdpProducerRegistry, config)const messageAssetsService = new MessageAssetsService(outputs)+ const emailStatusService = new EmailStatusService(outputs, config)// Constructed here (rather than below with the other messaging services) so it can be threaded// into EmailService — the pre-send suppression check lives there so every send path shares one// choke point regardless of whether the invocation came from a workflow action or a hog function.@@ -472,6 +479,7 @@ export function createCdpCoreServices(trackingCodeSigner,emailSuppressionService,recipientsManager,+ emailStatusService,messageAssetsService,workflowEmailRateLimiter,teamEmailRateLimiter@@ -576,7 +584,8 @@ export function createCdpCoreServices(warehouseWebhooksService,capturedEventsService,messageAssetsService,- conversionWatchersService+ conversionWatchersService,+ emailStatusService)const nativeDestinationExecutorService = new NativeDestinationExecutorService(config)@@ -612,5 +621,6 @@ export function createCdpCoreServices(outputs,emailService,messageAssetsService,+ emailStatusService,}}diff --git a/nodejs/src/cdp/config.ts b/nodejs/src/cdp/config.tsindex dd7198cb4a09..6d798251fc7d 100644--- a/nodejs/src/cdp/config.ts+++ b/nodejs/src/cdp/config.ts@@ -1,6 +1,7 @@import {KAFKA_APP_METRICS_2,KAFKA_CDP_EVENTS_DLQ,+ KAFKA_EMAIL_STATUS,KAFKA_EVENTS_JSON,KAFKA_HOG_INVOCATION_RESULTS,KAFKA_LOG_ENTRIES,@@ -133,6 +134,10 @@ export type CdpConfig = ClickhouseConfig & {// "Assets" tab.MESSAGE_ASSETS_TOPIC: stringMESSAGE_ASSETS_PRODUCER: CdpProducerName+ // One row per status change of a transactional email, for the email_status ClickHouse table.+ EMAIL_STATUS_TOPIC: string+ EMAIL_STATUS_PRODUCER: CdpProducerName+ EMAIL_STATUS_ENABLED: booleanHOG_INVOCATION_RERUN_MAX_COUNT: number// How many rerun wrapper jobs the worker dequeues per cyclotron-v2 poll.// Kept small by default — each job runs a full ClickHouse query per page.@@ -374,6 +379,11 @@ export function getDefaultCdpConfig(): CdpConfig {// Same cyclotron Warpstream cluster as hog_invocation_results — ClickHouse// consumes message_assets from the warpstream_cyclotron named collection.MESSAGE_ASSETS_PRODUCER: WARPSTREAM_CYCLOTRON_PRODUCER,+ EMAIL_STATUS_TOPIC: KAFKA_EMAIL_STATUS,+ // ClickHouse consumes email_status from the warpstream_cyclotron named collection.+ EMAIL_STATUS_PRODUCER: WARPSTREAM_CYCLOTRON_PRODUCER,+ // Off by default in production, so that writes start only once the email_status table exists.+ EMAIL_STATUS_ENABLED: isDevEnv() || isTestEnv(),// Hard cap on rows a single rerun wrapper job will drain. Mirrors the// Django serializer's HOG_INVOCATION_RERUN_MAX_COUNT (same env var).HOG_INVOCATION_RERUN_MAX_COUNT: 10000,diff --git a/nodejs/src/cdp/outputs/outputs.ts b/nodejs/src/cdp/outputs/outputs.tsindex 6cda4e049958..48ad7789d397 100644--- a/nodejs/src/cdp/outputs/outputs.ts+++ b/nodejs/src/cdp/outputs/outputs.ts@@ -12,3 +12,6 @@ export type WarehouseSourceWebhooksOutput = typeof WAREHOUSE_SOURCE_WEBHOOKS_OUTexport const CDP_EVENTS_DLQ_OUTPUT = 'cdp_events_dlq' as constexport type CdpEventsDlqOutput = typeof CDP_EVENTS_DLQ_OUTPUT++export const EMAIL_STATUS_OUTPUT = 'email_status' as const+export type EmailStatusOutput = typeof EMAIL_STATUS_OUTPUTdiff --git a/nodejs/src/cdp/outputs/registry.ts b/nodejs/src/cdp/outputs/registry.tsindex 94b64ebbfaf9..7c62868954ae 100644--- a/nodejs/src/cdp/outputs/registry.ts+++ b/nodejs/src/cdp/outputs/registry.ts@@ -6,7 +6,7 @@ import {} from '~/common/outputs'import { IngestionOutputsBuilder } from '~/common/outputs/ingestion-outputs-builder'-import { CDP_EVENTS_DLQ_OUTPUT, WAREHOUSE_SOURCE_WEBHOOKS_OUTPUT } from './outputs'+import { CDP_EVENTS_DLQ_OUTPUT, EMAIL_STATUS_OUTPUT, WAREHOUSE_SOURCE_WEBHOOKS_OUTPUT } from './outputs'/*** Outputs the CDP deployments write to. Each output's topic + producer are@@ -18,6 +18,7 @@ import { CDP_EVENTS_DLQ_OUTPUT, WAREHOUSE_SOURCE_WEBHOOKS_OUTPUT } from './outpu* `clickhouse_app_metrics2` schema).* - `WAREHOUSE_SOURCE_WEBHOOKS_OUTPUT` — warehouse source webhook payloads.* - `CDP_EVENTS_DLQ_OUTPUT` — events that matched a function but produced no invocation.+ * - `EMAIL_STATUS_OUTPUT` — status changes of transactional emails.*/export function createCdpOutputsRegistry() {return new IngestionOutputsBuilder()@@ -45,4 +46,8 @@ export function createCdpOutputsRegistry() {topicKey: 'CDP_EVENTS_DLQ_TOPIC',producerKey: 'CDP_EVENTS_DLQ_PRODUCER',})+ .register(EMAIL_STATUS_OUTPUT, {+ topicKey: 'EMAIL_STATUS_TOPIC',+ producerKey: 'EMAIL_STATUS_PRODUCER',+ })}diff --git a/nodejs/src/cdp/services/hog-executor.service.test.ts b/nodejs/src/cdp/services/hog-executor.service.test.tsindex 2ae11dcd4a1e..994f90a42ca5 100644--- a/nodejs/src/cdp/services/hog-executor.service.test.ts+++ b/nodejs/src/cdp/services/hog-executor.service.test.ts@@ -88,7 +88,8 @@ describe('Hog Executor', () => {hub.SITE_URL,new EmailTrackingCodeSigner(hub.ENCRYPTION_SALT_KEYS, hub.CDP_EMAIL_TRACKING_URL),new EmailSuppressionService(hub.postgres, emailSuppressionConfigFromEnv()),- new RecipientsManagerService(hub.postgres)+ new RecipientsManagerService(hub.postgres),+ null)const recipientTokensService = new RecipientTokensService(hub.ENCRYPTION_SALT_KEYS, hub.SITE_URL)return new HogExecutorAsyncService(diff --git a/nodejs/src/cdp/services/hogflows/actions/hog_function.test.ts b/nodejs/src/cdp/services/hogflows/actions/hog_function.test.tsindex 60e53e0ed5f7..a268665367ac 100644--- a/nodejs/src/cdp/services/hogflows/actions/hog_function.test.ts+++ b/nodejs/src/cdp/services/hogflows/actions/hog_function.test.ts@@ -87,7 +87,8 @@ describe('HogFunctionHandler', () => {hub.SITE_URL,new EmailTrackingCodeSigner(hub.ENCRYPTION_SALT_KEYS, hub.CDP_EMAIL_TRACKING_URL),new EmailSuppressionService(hub.postgres, emailSuppressionConfigFromEnv()),- new RecipientsManagerService(hub.postgres)+ new RecipientsManagerService(hub.postgres),+ null)mockHogFunctionExecutor = new HogExecutorAsyncService(new HogExecutorService({ executionTimeoutMs: hub.CDP_WATCHER_HOG_COST_TIMING_UPPER_MS }, hogInputsService),diff --git a/nodejs/src/cdp/services/hogflows/hogflow-executor.service.test.ts b/nodejs/src/cdp/services/hogflows/hogflow-executor.service.test.tsindex 5f6df00a7697..6b467b61d305 100644--- a/nodejs/src/cdp/services/hogflows/hogflow-executor.service.test.ts+++ b/nodejs/src/cdp/services/hogflows/hogflow-executor.service.test.ts@@ -99,7 +99,8 @@ describe('Hogflow Executor', () => {hub.SITE_URL,new EmailTrackingCodeSigner(hub.ENCRYPTION_SALT_KEYS, hub.CDP_EMAIL_TRACKING_URL),emailSuppressionService,- new RecipientsManagerService(hub.postgres)+ new RecipientsManagerService(hub.postgres),+ null)const recipientTokensService = new RecipientTokensService(hub.ENCRYPTION_SALT_KEYS, hub.SITE_URL)const hogExecutor = new HogExecutorAsyncService(diff --git a/nodejs/src/cdp/services/invocation-results.service.test.ts b/nodejs/src/cdp/services/invocation-results.service.test.tsindex 00ab9ed588f3..c6bc739339b1 100644--- a/nodejs/src/cdp/services/invocation-results.service.test.ts+++ b/nodejs/src/cdp/services/invocation-results.service.test.ts@@ -2,6 +2,7 @@ import { CyclotronJobInvocationResult } from '../types'import { CapturedEventsService } from './captured-events/captured-events.service'import { ConversionWatchersService } from './conversion-watchers/conversion-watchers.service'import { InvocationResultsService } from './invocation-results.service'+import { EmailStatusService } from './messaging/email-status.service'import { MessageAssetsService } from './messaging/message-assets.service'import { HogFunctionMonitoringService } from './monitoring/hog-function-monitoring.service'import { HogInvocationResultsService } from './monitoring/hog-invocation-results.service'@@ -14,6 +15,7 @@ describe('InvocationResultsService', () => {let capturedEventsService: jest.Mocked<CapturedEventsService>let messageAssetsService: jest.Mocked<MessageAssetsService>let conversionWatchersService: jest.Mocked<ConversionWatchersService>+ let emailStatusService: jest.Mocked<EmailStatusService>let service: InvocationResultsServiceconst results = [@@ -52,13 +54,18 @@ describe('InvocationResultsService', () => {flush: jest.fn().mockResolvedValue(undefined),} as unknown as jest.Mocked<ConversionWatchersService>+ emailStatusService = {+ flush: jest.fn().mockResolvedValue(undefined),+ } as unknown as jest.Mocked<EmailStatusService>+service = new InvocationResultsService(monitoringService,invocationResultsRowsService,warehouseWebhooksService,capturedEventsService,messageAssetsService,- conversionWatchersService+ conversionWatchersService,+ emailStatusService)})@@ -114,6 +121,7 @@ describe('InvocationResultsService', () => {expect(warehouseWebhooksService.flush).toHaveBeenCalledTimes(1)expect(capturedEventsService.flush).toHaveBeenCalledTimes(1)expect(messageAssetsService.flush).toHaveBeenCalledTimes(1)+ expect(emailStatusService.flush).toHaveBeenCalledTimes(1)})it('flushes all in parallel (all start before any finish)', async () => {diff --git a/nodejs/src/cdp/services/invocation-results.service.ts b/nodejs/src/cdp/services/invocation-results.service.tsindex 200fdb0798ba..b11ae9cc3a24 100644--- a/nodejs/src/cdp/services/invocation-results.service.ts+++ b/nodejs/src/cdp/services/invocation-results.service.ts@@ -3,6 +3,7 @@ import { instrumentFn } from '~/common/tracing/tracing-utils'import { CyclotronJobInvocationResult } from '../types'import { CapturedEventsService } from './captured-events/captured-events.service'import { ConversionWatchersService } from './conversion-watchers/conversion-watchers.service'+import { EmailStatusService } from './messaging/email-status.service'import { MessageAssetsService } from './messaging/message-assets.service'import { HogFunctionMonitoringService } from './monitoring/hog-function-monitoring.service'import { HogInvocationResultsService } from './monitoring/hog-invocation-results.service'@@ -32,6 +33,8 @@ export interface ResultSink {* Assets tab* - `ConversionWatchersService` — per-run conversion watchers, which outlive the* run so a late conversion stays observable+ * - `EmailStatusService` — status rows of transactional emails that the+ * email service queues at send time** Callers interact with this one service instead of coordinating queue/flush* calls across the five individually. `queueInvocationResultsAndFlush` is the@@ -48,7 +51,8 @@ export class InvocationResultsService {public readonly warehouseWebhooksService: WarehouseWebhooksService,public readonly capturedEventsService: CapturedEventsService,public readonly messageAssetsService: MessageAssetsService,- public readonly conversionWatchersService: ConversionWatchersService+ public readonly conversionWatchersService: ConversionWatchersService,+ public readonly emailStatusService: EmailStatusService) {this.sinks = [monitoringService,@@ -57,6 +61,7 @@ export class InvocationResultsService {capturedEventsService,messageAssetsService,conversionWatchersService,+ emailStatusService,]}diff --git a/nodejs/src/cdp/services/messaging/email-status.service.test.ts b/nodejs/src/cdp/services/messaging/email-status.service.test.tsnew file mode 100644index 000000000000..97e6f4017823--- /dev/null+++ b/nodejs/src/cdp/services/messaging/email-status.service.test.ts@@ -0,0 +1,87 @@+import { register } from 'prom-client'++import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'+import { UUIDT } from '~/common/utils/utils'++import { EmailStatusOutput } from '../../outputs/outputs'+import { EmailStatus, EmailStatusService, buildEmailStatusRow } from './email-status.service'++const failedWrites = async (status: EmailStatus): Promise<number> =>+ (await register.getSingleMetric('cdp_email_status_write_failed')!.get()).values.find(+ (v) => v.labels.status === status+ )?.value ?? 0++describe('EmailStatusService', () => {+ const createdAt = Date.UTC(2026, 0, 2, 3, 4, 5, 678)+ const messageId = new UUIDT(createdAt).toString()++ it('builds a row whose created_at comes from the message id and whose version leads with the status rank', () => {+ const statusAt = new Date(Date.UTC(2026, 0, 3, 10, 0, 0, 123))++ expect(+ buildEmailStatusRow({ teamId: 2, messageId, status: 'failed', reason: 'provider_error', statusAt })+ ).toEqual({+ team_id: 2,+ message_id: messageId,+ status: 'failed',+ // 2 * 2^48 + status_at in ms, as a decimal string for ClickHouse's UInt64.+ version: (2n * 2n ** 48n + BigInt(statusAt.getTime())).toString(),+ created_at: '2026-01-02 03:04:05.678000',+ status_at: '2026-01-03 10:00:00.123000',+ reason: 'provider_error',+ })+ })++ it('ranks every status above the previous one even when it is timestamped earlier', () => {+ const statuses: EmailStatus[] = ['queued', 'failed', 'sent', 'delivered', 'bounced', 'complained']+ const versions = statuses.map((status, index) =>+ BigInt(+ buildEmailStatusRow({+ teamId: 2,+ messageId,+ status,+ // Each later status carries an earlier time, as a late webhook or a clock skew would.+ statusAt: new Date(createdAt + (statuses.length - index) * 60_000),+ }).version+ )+ )++ expect(versions).toEqual([...versions].sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)))+ expect(new Set(versions).size).toEqual(statuses.length)+ })++ it('returns from a flush whose produce stalls once the wait runs out', async () => {+ const produce = jest.fn(() => new Promise<void>(() => {}))+ const service = new EmailStatusService(+ { produce } as unknown as IngestionOutputs<EmailStatusOutput>,+ { EMAIL_STATUS_ENABLED: true },+ 10+ )++ service.queue({ teamId: 2, messageId, status: 'sent' })++ await expect(service.flush()).resolves.toBeUndefined()+ expect(produce).toHaveBeenCalledTimes(1)+ })++ it.each([+ ['the produce fails', messageId, () => Promise.reject(new Error('broker down'))],+ ['the message id is not a UUID', 'not-a-uuid', () => Promise.resolve()],+ ])('drops and counts a row when %s, and still writes the others', async (_desc, badMessageId, produceBad) => {+ const otherMessageId = new UUIDT().toString()+ const produce = jest.fn((_output, message: { key: Buffer }) =>+ message.key.toString() === badMessageId ? produceBad() : Promise.resolve()+ )+ const service = new EmailStatusService({ produce } as unknown as IngestionOutputs<EmailStatusOutput>, {+ EMAIL_STATUS_ENABLED: true,+ })+ const failedBefore = await failedWrites('delivered')++ service.queue({ teamId: 2, messageId: badMessageId, status: 'delivered' })+ service.queue({ teamId: 2, messageId: otherMessageId, status: 'delivered' })+ await expect(service.flush()).resolves.toBeUndefined()++ expect(await failedWrites('delivered')).toEqual(failedBefore + 1)+ expect(produce.mock.calls.map(([, message]) => message.key.toString())).toContain(otherMessageId)+ })+})diff --git a/nodejs/src/cdp/services/messaging/email-status.service.ts b/nodejs/src/cdp/services/messaging/email-status.service.tsnew file mode 100644index 000000000000..ce4f19d13c55--- /dev/null+++ b/nodejs/src/cdp/services/messaging/email-status.service.ts@@ -0,0 +1,176 @@+import { Counter } from 'prom-client'+import { setTimeout as delay } from 'timers/promises'++import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'+import { logger } from '~/common/utils/logger'++import { EMAIL_STATUS_OUTPUT, EmailStatusOutput } from '../../outputs/outputs'+import { CyclotronJobInvocationHogFunction } from '../../types'+import { TRANSACTIONAL_EMAIL_EVENT } from '../hogflows/hogflow-sensitivity'++const counterEmailStatusRowsProduced = new Counter({+ name: 'cdp_email_status_rows_produced',+ help: 'Transactional email status rows produced to the email_status Kafka topic.',+ labelNames: ['status'],+})++const counterEmailStatusWriteFailed = new Counter({+ name: 'cdp_email_status_write_failed',+ help: 'Transactional email status rows dropped because they could not be built or produced. The send is never affected.',+ labelNames: ['status'],+})++export type EmailStatus = 'queued' | 'failed' | 'sent' | 'delivered' | 'bounced' | 'complained'++export type EmailStatusReason =+ // A failed send. The provider rejected the email, or the send could not be made.+ 'provider_error'++// The rank is the high part of `version`, so a message's status only moves forward in the+// ReplacingMergeTree, whatever order the rows arrive in. `failed` ranks below `sent` so that a+// successful rerun of the same invocation replaces the failed attempt. The gaps let a failure take the+// rank between two statuses. Never renumber a rank, because rows already written keep their versions.+const STATUS_RANK: Record<EmailStatus, bigint> = {+ queued: 0n,+ failed: 2n,+ sent: 3n,+ delivered: 5n,+ bounced: 6n,+ complained: 7n,+}++// 2^48 ms is past the year 10000, so the status time never reaches into the rank bits.+const STATUS_AT_BITS = 48n++// How long a flush waits for its rows. See `flush`.+const EMAIL_STATUS_FLUSH_MAX_WAIT_MS = 1000++const UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i++export interface EmailStatusRow {+ team_id: number+ message_id: string+ status: EmailStatus+ // A decimal string, because the value is past Number.MAX_SAFE_INTEGER. ClickHouse parses it as UInt64.+ version: string+ created_at: string+ status_at: string+ reason: string+}++export interface EmailStatusUpdate {+ teamId: number+ // The hog flow invocation id, which the transactional email API returns as `message_id`.+ messageId: string+ status: EmailStatus+ reason?: EmailStatusReason+ // When the status changed. Defaults to now.+ statusAt?: Date+}++const isoMicroseconds = (date: Date): string => {+ // ClickHouse DateTime64(6) accepts 'YYYY-MM-DD HH:MM:SS.ffffff'.+ return date.toISOString().replace('T', ' ').replace('Z', '000')+}++// A UUIDT starts with its creation time as 48 bits of Unix milliseconds.+const uuidtCreatedAt = (messageId: string): Date => {+ if (!UUID_REGEX.test(messageId)) {+ throw new Error('message id is not a UUID')+ }+ return new Date(parseInt(messageId.slice(0, 8) + messageId.slice(9, 13), 16))+}++export const buildEmailStatusRow = (update: EmailStatusUpdate): EmailStatusRow => {+ const statusAt = update.statusAt && !Number.isNaN(update.statusAt.getTime()) ? update.statusAt : new Date()+ const version = (STATUS_RANK[update.status] << STATUS_AT_BITS) + BigInt(statusAt.getTime())+ return {+ team_id: update.teamId,+ message_id: update.messageId,+ status: update.status,+ version: version.toString(),+ // Derived from the message id rather than the clock, so every row of a message lands in+ // the same `toYYYYMMDD(created_at)` partition and the rows can collapse.+ created_at: isoMicroseconds(uuidtCreatedAt(update.messageId)),+ status_at: isoMicroseconds(statusAt),+ reason: update.reason ?? '',+ }+}++// A transactional email run is triggered by the synthetic event that the transactional email API creates.+export const isTransactionalEmailSend = (invocation: CyclotronJobInvocationHogFunction): boolean =>+ invocation.state?.globals?.event?.event === TRANSACTIONAL_EMAIL_EVENT++/**+ * Records each status change of a transactional email for the email_status ClickHouse table.+ *+ * Rows are buffered by `queue` and produced by `flush`. Both are best-effort and never throw:+ * a status row that cannot be written is logged and counted, and the send it describes goes on.+ */+export class EmailStatusService {+ private queuedRows: EmailStatusRow[] = []++ constructor(+ private outputs: IngestionOutputs<EmailStatusOutput>,+ private config: { EMAIL_STATUS_ENABLED: boolean },+ private maxFlushWaitMs = EMAIL_STATUS_FLUSH_MAX_WAIT_MS+ ) {}++ queue(update: EmailStatusUpdate): void {+ if (!this.config.EMAIL_STATUS_ENABLED) {+ return+ }+ try {+ this.queuedRows.push(buildEmailStatusRow(update))+ } catch (error) {+ counterEmailStatusWriteFailed.inc({ status: update.status })+ logger.error('⚠️', 'failed to build an email status row', {+ error: String(error),+ team_id: update.teamId,+ message_id: update.messageId,+ status: update.status,+ })+ }+ }++ // Required by ResultSink. The rows go through the shared Kafka outputs, which the server+ // disconnects separately on shutdown.+ async stop(): Promise<void> {+ return Promise.resolve()+ }++ // The caller waits at most `maxFlushWaitMs`, and rows still in flight are produced in the background.+ // A stalled produce then cannot hold up the send API's reply, the SES webhook, or a worker batch past+ // its timeout. The status API already documents that a status can arrive a few seconds late.+ async flush(): Promise<void> {+ if (this.queuedRows.length === 0) {+ return+ }+ const rows = this.queuedRows+ this.queuedRows = []+ await Promise.race([this.produce(rows), delay(this.maxFlushWaitMs, undefined, { ref: false })])+ }++ private async produce(rows: EmailStatusRow[]): Promise<void> {+ await Promise.all(+ rows.map(async (row) => {+ try {+ await this.outputs.produce(EMAIL_STATUS_OUTPUT, {+ // Keyed by message so all rows of a message go to the same partition.+ key: Buffer.from(row.message_id),+ value: Buffer.from(JSON.stringify(row)),+ })+ counterEmailStatusRowsProduced.inc({ status: row.status })+ } catch (error) {+ counterEmailStatusWriteFailed.inc({ status: row.status })+ logger.error('⚠️', 'failed to produce an email status row', {+ error: String(error),+ team_id: row.team_id,+ message_id: row.message_id,+ status: row.status,+ })+ }+ })+ )+ }+}diff --git a/nodejs/src/cdp/services/messaging/email.service.test.ts b/nodejs/src/cdp/services/messaging/email.service.test.tsindex 459f664a7da7..d3655ef28311 100644--- a/nodejs/src/cdp/services/messaging/email.service.test.ts+++ b/nodejs/src/cdp/services/messaging/email.service.test.ts@@ -3,15 +3,19 @@ import { mockFetch } from '~/tests/helpers/mocks/request.mock'import { MessageRejected, SendingPausedException, TooManyRequestsException } from '@aws-sdk/client-sesv2'import { createExampleInvocation, insertIntegration } from '~/cdp/_tests/fixtures'+import { EmailStatusOutput } from '~/cdp/outputs/outputs'import {CyclotronInvocationQueueParametersEmailSchema,CyclotronInvocationQueueParametersEmailType,} from '~/cdp/schema/cyclotron'+import { TRANSACTIONAL_EMAIL_EVENT } from '~/cdp/services/hogflows/hogflow-sensitivity'import { CyclotronJobInvocationHogFunction } from '~/cdp/types'import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'import { createRedisV2PoolFromConfig } from '~/common/redis/redis-v2'import { closeHub, createHub } from '~/common/utils/db/hub'import { PostgresUse } from '~/common/utils/db/postgres'+import { parseJSON } from '~/common/utils/json-parse'+import { UUIDT } from '~/common/utils/utils'import { waitForExpect } from '~/tests/helpers/expectations'import { createTestTeamFixture } from '~/tests/helpers/sql'import { INVALID_TEAM_ID_KEYS } from '~/tests/helpers/team'@@ -21,6 +25,7 @@ import { RecipientsManagerService } from '../managers/recipients-manager.serviceimport { TeamWorkflowsConfigService } from '../managers/team-workflows-config.service'import { RateLimiterService } from '../rate-limiter/rate-limiter.service'import { selectEmailSenderIntegrationId } from './email-sender-selection'+import { EmailStatusService } from './email-status.service'import { EmailSuppressionService, emailSuppressionConfigFromEnv } from './email-suppression.service'import {EmailService,@@ -144,7 +149,8 @@ describe('EmailService', () => {hub.SITE_URL,new EmailTrackingCodeSigner(hub.ENCRYPTION_SALT_KEYS, hub.CDP_EMAIL_TRACKING_URL),new EmailSuppressionService(hub.postgres, emailSuppressionConfigFromEnv()),- new RecipientsManagerService(hub.postgres)+ new RecipientsManagerService(hub.postgres),+ null)mockFetch.mockClear()})@@ -168,7 +174,8 @@ describe('EmailService', () => {hub.SITE_URL,new EmailTrackingCodeSigner(hub.ENCRYPTION_SALT_KEYS, hub.CDP_EMAIL_TRACKING_URL),new EmailSuppressionService(hub.postgres, emailSuppressionConfigFromEnv()),- new RecipientsManagerService(hub.postgres)+ new RecipientsManagerService(hub.postgres),+ null)expect(serviceWithoutSES.sesV2Client).toBeNull()@@ -525,6 +532,7 @@ describe('EmailService', () => {new EmailTrackingCodeSigner(hub.ENCRYPTION_SALT_KEYS, hub.CDP_EMAIL_TRACKING_URL),new EmailSuppressionService(hub.postgres, emailSuppressionConfigFromEnv()),new RecipientsManagerService(hub.postgres),+ null,undefined,{ claimOrReserve } as unknown as RateLimiterService)@@ -696,6 +704,7 @@ describe('EmailService', () => {new EmailTrackingCodeSigner(hub.ENCRYPTION_SALT_KEYS, hub.CDP_EMAIL_TRACKING_URL),new EmailSuppressionService(hub.postgres, emailSuppressionConfigFromEnv()),new RecipientsManagerService(hub.postgres),+ null,undefined,new RateLimiterService(redis, { name: 'workflow-email-backlog-test' }))@@ -766,6 +775,7 @@ describe('EmailService', () => {new EmailTrackingCodeSigner(hub.ENCRYPTION_SALT_KEYS, hub.CDP_EMAIL_TRACKING_URL),new EmailSuppressionService(hub.postgres, emailSuppressionConfigFromEnv()),new RecipientsManagerService(hub.postgres),+ null,undefined,null,{ claimAllOrNothingPair } as unknown as RateLimiterService@@ -894,6 +904,7 @@ describe('EmailService', () => {new EmailTrackingCodeSigner(hub.ENCRYPTION_SALT_KEYS, hub.CDP_EMAIL_TRACKING_URL),new EmailSuppressionService(hub.postgres, emailSuppressionConfigFromEnv()),new RecipientsManagerService(hub.postgres),+ null,undefined,null,limiter@@ -1082,6 +1093,45 @@ describe('EmailService', () => {})})+ const sesAccepts = (): void => {+ sendEmailSpy.mockResolvedValue({ MessageId: 'test-message-id' })+ }+ const sesRejects = (): void => {+ sendEmailSpy.mockRejectedValue(new Error('SES error'))+ }+ it.each([+ ['a transactional email that SES accepts', TRANSACTIONAL_EMAIL_EVENT, false, sesAccepts, 'sent', ''],+ [+ 'a transactional email that SES rejects',+ TRANSACTIONAL_EMAIL_EVENT,+ false,+ sesRejects,+ 'failed',+ 'provider_error',+ ],+ ['a test send of a transactional email', TRANSACTIONAL_EMAIL_EVENT, true, sesAccepts, null, null],+ ['a workflow email that SES accepts', '$pageview', false, sesAccepts, null, null],+ ['a workflow email that SES rejects', '$pageview', false, sesRejects, null, null],+ ])('records the status of %s', async (_name, triggerEvent, isTest, setup, status, reason) => {+ const produce = jest.fn().mockResolvedValue(undefined)+ const emailStatusService = new EmailStatusService(+ { produce } as unknown as IngestionOutputs<EmailStatusOutput>,+ { EMAIL_STATUS_ENABLED: true }+ )+ service['emailStatusService'] = emailStatusService+ invocation.id = new UUIDT().toString()+ invocation.state.globals.event.event = triggerEvent+ setup()++ await service.executeSendEmail(invocation, isTest)+ await emailStatusService.flush()++ const rows = produce.mock.calls.map(([, message]) => parseJSON(message.value.toString()))+ expect(rows).toEqual(+ status ? [expect.objectContaining({ team_id: team.id, message_id: invocation.id, status, reason })] : []+ )+ })+it('records a send-time metric for normal sends but not for test sends', async () => {sendEmailSpy.mockResolvedValue({ MessageId: 'test-message-id' })diff --git a/nodejs/src/cdp/services/messaging/email.service.ts b/nodejs/src/cdp/services/messaging/email.service.tsindex c0f4581a699c..2bfa8417d268 100644--- a/nodejs/src/cdp/services/messaging/email.service.ts+++ b/nodejs/src/cdp/services/messaging/email.service.ts@@ -21,6 +21,7 @@ import { RecipientManagerRecipient, RecipientsManagerService } from '../managersimport { TeamWorkflowsConfigService } from '../managers/team-workflows-config.service'import { RateLimiterService } from '../rate-limiter/rate-limiter.service'import { selectEmailSenderIntegrationId } from './email-sender-selection'+import { EmailStatusService, isTransactionalEmailSend } from './email-status.service'import { EmailSuppressionService, SuppressionCheckOptions } from './email-suppression.service'import {addTrackingToEmail,@@ -364,6 +365,9 @@ export class EmailService {private trackingCodeSigner: EmailTrackingCodeSigner,private emailSuppressionService: EmailSuppressionService,private recipientsManager: RecipientsManagerService,+ // Required, so that a caller which builds the service without it fails to compile. Pass null+ // only where no email status is recorded.+ private emailStatusService: EmailStatusService | null,private messageAssetsService?: MessageAssetsService,private workflowEmailRateLimiter: RateLimiterService | null = null,private teamEmailRateLimiter: RateLimiterService | null = null@@ -753,6 +757,17 @@ export class EmailService {if (success && assetRow) {result.messageAssets.push(assetRow)}++ // A failure here is final: SES throttling returned above to reschedule, and nothing+ // retries a failed send.+ if (this.emailStatusService && isTransactionalEmailSend(invocation)) {+ this.emailStatusService.queue({+ teamId: invocation.teamId,+ messageId: invocation.id,+ status: success ? 'sent' : 'failed',+ reason: success ? undefined : 'provider_error',+ })+ }}const distinctId = resolveEmailEngagementDistinctId(invocation)diff --git a/nodejs/src/cdp/templates/test/test-helpers.ts b/nodejs/src/cdp/templates/test/test-helpers.tsindex 0f78d88d7e5e..6fb026c1b3b1 100644--- a/nodejs/src/cdp/templates/test/test-helpers.ts+++ b/nodejs/src/cdp/templates/test/test-helpers.ts@@ -220,7 +220,8 @@ export class TemplateTester {config.SITE_URL,new EmailTrackingCodeSigner(config.ENCRYPTION_SALT_KEYS, config.CDP_EMAIL_TRACKING_URL),undefined as any,- undefined as any+ undefined as any,+ null)return new HogExecutorAsyncService(new HogExecutorService(diff --git a/nodejs/src/common/config/kafka-topics.ts b/nodejs/src/common/config/kafka-topics.tsindex ff32d96febe9..2222d8bc9bdb 100644--- a/nodejs/src/common/config/kafka-topics.ts+++ b/nodejs/src/common/config/kafka-topics.ts@@ -84,6 +84,9 @@ export const KAFKA_LOG_ENTRIES = `${prefix}log_entries${suffix}`// per-invocation result rows for hog functions and hog flowsexport const KAFKA_HOG_INVOCATION_RESULTS = `${prefix}clickhouse_hog_invocation_results${suffix}`+// one row per status change of a transactional email, for the email_status ClickHouse table+export const KAFKA_EMAIL_STATUS = `${prefix}clickhouse_email_status${suffix}`+// metadata rows for sent message assets (rendered emails stored in object storage)export const KAFKA_MESSAGE_ASSETS = `${prefix}clickhouse_message_assets${suffix}`