Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(cdp): record the queued and sent status of each transactional email (#124736)

Harley Alexanderef2369a75734
diff --git a/nodejs/src/cdp/cdp-api.serial.test.ts b/nodejs/src/cdp/cdp-api.serial.test.ts
index 233988af94a2..419637585c47 100644
--- a/nodejs/src/cdp/cdp-api.serial.test.ts
+++ b/nodejs/src/cdp/cdp-api.serial.test.ts
@@ -2148,7 +2148,14 @@ describe('CDP API', () => {
.set(headers)
.send(payload)
+ let produceOutput: jest.SpyInstance
+ const emailStatusRows = (): Record<string, unknown>[] =>
+ produceOutput.mock.calls
+ .filter(([output]) => output === 'email_status')
+ .map(([, message]) => parseJSON(message.value.toString()))
+
beforeEach(async () => {
+ produceOutput = jest.spyOn(api['emailStatusService']['outputs'], 'produce')
mockQueueInvocations = jest.fn().mockResolvedValue(undefined)
api['hogflowQueue'] = { queueInvocations: mockQueueInvocations } as any
hogFlow = await insertHogFlow(transactionalFlow('api', 'active'))
@@ -2186,6 +2193,9 @@ describe('CDP API', () => {
expect.objectContaining({ app_source_id: hogFlow.id, metric_name: 'triggered', count: 1 }),
'hog_flow'
)
+ expect(emailStatusRows()).toEqual([
+ expect.objectContaining({ team_id: team.id, message_id: res.body.invocation_id, status: 'queued' }),
+ ])
})
describe('with an idempotency key', () => {
@@ -2211,6 +2221,9 @@ describe('CDP API', () => {
replayed: true,
})
expect(mockQueueInvocations).toHaveBeenCalledTimes(1)
+ expect(emailStatusRows()).toEqual([
+ expect.objectContaining({ message_id: first.body.invocation_id, status: 'queued' }),
+ ])
})
it('rejects the key for a different request', async () => {
@@ -2333,6 +2346,7 @@ describe('CDP API', () => {
expect(refused.body).toEqual({ error: code })
expect(await outcomeCount(code)).toEqual(refusedBefore + 1)
expect(mockQueueInvocations).not.toHaveBeenCalled()
+ expect(emailStatusRows()).toEqual([])
const retry = await send(hogFlow.id, headers, keyed)
diff --git a/nodejs/src/cdp/cdp-api.ts b/nodejs/src/cdp/cdp-api.ts
index 225ea19ba3c3..e3bd345a759e 100644
--- a/nodejs/src/cdp/cdp-api.ts
+++ b/nodejs/src/cdp/cdp-api.ts
@@ -64,6 +64,7 @@ import { GroupsManagerService } from './services/managers/groups-manager.service
import { HogFunctionManagerService } from './services/managers/hog-function-manager.service'
import { TeamWorkflowsConfigService } from './services/managers/team-workflows-config.service'
import { EMAIL_QUEUE_PRIORITY } from './services/messaging/email-priority'
+import { EmailStatusService } from './services/messaging/email-status.service'
import { EmailSuppressionService } from './services/messaging/email-suppression.service'
import { EmailTrackingService } from './services/messaging/email-tracking.service'
import { EmailValidationService } from './services/messaging/email-validation.service'
@@ -202,6 +203,7 @@ export class CdpApi {
private emailValidationService: EmailValidationService
private teamWorkflowsConfigService: TeamWorkflowsConfigService
private hogFunctionMonitoringService: HogFunctionMonitoringService
+ private emailStatusService: EmailStatusService
private resumeBatchJwt: ScopedServiceJwt
constructor(
@@ -226,6 +228,7 @@ export class CdpApi {
this.hogWatcher = services.hogWatcher
this.hogWatcherMirror = services.hogWatcherMirror
this.invocationResultsService = services.invocationResultsService
+ this.emailStatusService = services.emailStatusService
// API-only services. The hog-transformer's monitoring service reuses the same
// resolved outputs registry as the core CDP services — no separate construction.
@@ -1260,7 +1263,12 @@ export class CdpApi {
if (idempotencyKey) {
await idempotencyKeys.markQueued(team.id, idempotencyKey, fingerprint, invocation.id)
}
- await this.invocationResultsService.invocationResultsRowsService.flush()
+ this.emailStatusService.queue({ teamId: team.id, messageId: invocation.id, status: 'queued' })
+ // Flushed together, and bounded, so the best-effort status row can't hold up the 202.
+ await Promise.all([
+ this.invocationResultsService.invocationResultsRowsService.flush(),
+ this.emailStatusService.flush(),
+ ])
// The run skips the trigger pipeline that counts runs for event-triggered workflows.
this.hogFunctionMonitoringService.queueAppMetric(
{
diff --git a/nodejs/src/cdp/cdp-services.ts b/nodejs/src/cdp/cdp-services.ts
index 47643b470597..a61771e3e788 100644
--- a/nodejs/src/cdp/cdp-services.ts
+++ b/nodejs/src/cdp/cdp-services.ts
@@ -34,6 +34,7 @@ import { IntegrationManagerService } from './services/managers/integration-manag
import { PersonsManagerService } from './services/managers/persons-manager.service'
import { RecipientsManagerService } from './services/managers/recipients-manager.service'
import { TeamWorkflowsConfigService } from './services/managers/team-workflows-config.service'
+import { EmailStatusService } from './services/messaging/email-status.service'
import { EmailSuppressionService } from './services/messaging/email-suppression.service'
import { EmailValidationService } from './services/messaging/email-validation.service'
import { EmailService, parseTeamEmailCapMode, parseTierCaps } from './services/messaging/email.service'
@@ -118,6 +119,8 @@ export interface CdpCoreServices {
emailService: EmailService
/** Buffers rendered-email asset rows and bulk-flushes them at the batch boundary. */
messageAssetsService: MessageAssetsService
+ /** Buffers transactional email status rows. `invocationResultsService.flush()` produces them. */
+ emailStatusService: EmailStatusService
}
export type CdpCoreServicesConfig = Pick<
@@ -188,6 +191,9 @@ export type CdpCoreServicesConfig = Pick<
| 'HOG_INVOCATION_RESULTS_ENABLED'
| 'MESSAGE_ASSETS_TOPIC'
| 'MESSAGE_ASSETS_PRODUCER'
+ | 'EMAIL_STATUS_TOPIC'
+ | 'EMAIL_STATUS_PRODUCER'
+ | 'EMAIL_STATUS_ENABLED'
| 'CDP_WAREHOUSE_SOURCE_WEBHOOKS_TOPIC'
| 'CDP_WAREHOUSE_SOURCE_WEBHOOKS_PRODUCER'
| 'CDP_DLQ_ENABLED'
@@ -435,6 +441,7 @@ export function createCdpCoreServices(
const teamWorkflowsConfigService = new TeamWorkflowsConfigService(deps.postgres, deps.pubSub)
const outputs = createCdpOutputsRegistry().build(deps.cdpProducerRegistry, config)
const messageAssetsService = new MessageAssetsService(outputs)
+ const emailStatusService = new EmailStatusService(outputs, config)
// Constructed here (rather than below with the other messaging services) so it can be threaded
// into EmailService — the pre-send suppression check lives there so every send path shares one
// choke point regardless of whether the invocation came from a workflow action or a hog function.
@@ -472,6 +479,7 @@ export function createCdpCoreServices(
trackingCodeSigner,
emailSuppressionService,
recipientsManager,
+ emailStatusService,
messageAssetsService,
workflowEmailRateLimiter,
teamEmailRateLimiter
@@ -576,7 +584,8 @@ export function createCdpCoreServices(
warehouseWebhooksService,
capturedEventsService,
messageAssetsService,
- conversionWatchersService
+ conversionWatchersService,
+ emailStatusService
)
const nativeDestinationExecutorService = new NativeDestinationExecutorService(config)
@@ -612,5 +621,6 @@ export function createCdpCoreServices(
outputs,
emailService,
messageAssetsService,
+ emailStatusService,
}
}
diff --git a/nodejs/src/cdp/config.ts b/nodejs/src/cdp/config.ts
index dd7198cb4a09..6d798251fc7d 100644
--- a/nodejs/src/cdp/config.ts
+++ b/nodejs/src/cdp/config.ts
@@ -1,6 +1,7 @@
import {
KAFKA_APP_METRICS_2,
KAFKA_CDP_EVENTS_DLQ,
+ KAFKA_EMAIL_STATUS,
KAFKA_EVENTS_JSON,
KAFKA_HOG_INVOCATION_RESULTS,
KAFKA_LOG_ENTRIES,
@@ -133,6 +134,10 @@ export type CdpConfig = ClickhouseConfig & {
// "Assets" tab.
MESSAGE_ASSETS_TOPIC: string
MESSAGE_ASSETS_PRODUCER: CdpProducerName
+ // One row per status change of a transactional email, for the email_status ClickHouse table.
+ EMAIL_STATUS_TOPIC: string
+ EMAIL_STATUS_PRODUCER: CdpProducerName
+ EMAIL_STATUS_ENABLED: boolean
HOG_INVOCATION_RERUN_MAX_COUNT: number
// How many rerun wrapper jobs the worker dequeues per cyclotron-v2 poll.
// Kept small by default — each job runs a full ClickHouse query per page.
@@ -374,6 +379,11 @@ export function getDefaultCdpConfig(): CdpConfig {
// Same cyclotron Warpstream cluster as hog_invocation_results — ClickHouse
// consumes message_assets from the warpstream_cyclotron named collection.
MESSAGE_ASSETS_PRODUCER: WARPSTREAM_CYCLOTRON_PRODUCER,
+ EMAIL_STATUS_TOPIC: KAFKA_EMAIL_STATUS,
+ // ClickHouse consumes email_status from the warpstream_cyclotron named collection.
+ EMAIL_STATUS_PRODUCER: WARPSTREAM_CYCLOTRON_PRODUCER,
+ // Off by default in production, so that writes start only once the email_status table exists.
+ EMAIL_STATUS_ENABLED: isDevEnv() || isTestEnv(),
// Hard cap on rows a single rerun wrapper job will drain. Mirrors the
// Django serializer's HOG_INVOCATION_RERUN_MAX_COUNT (same env var).
HOG_INVOCATION_RERUN_MAX_COUNT: 10000,
diff --git a/nodejs/src/cdp/outputs/outputs.ts b/nodejs/src/cdp/outputs/outputs.ts
index 6cda4e049958..48ad7789d397 100644
--- a/nodejs/src/cdp/outputs/outputs.ts
+++ b/nodejs/src/cdp/outputs/outputs.ts
@@ -12,3 +12,6 @@ export type WarehouseSourceWebhooksOutput = typeof WAREHOUSE_SOURCE_WEBHOOKS_OUT
export const CDP_EVENTS_DLQ_OUTPUT = 'cdp_events_dlq' as const
export type CdpEventsDlqOutput = typeof CDP_EVENTS_DLQ_OUTPUT
+
+export const EMAIL_STATUS_OUTPUT = 'email_status' as const
+export type EmailStatusOutput = typeof EMAIL_STATUS_OUTPUT
diff --git a/nodejs/src/cdp/outputs/registry.ts b/nodejs/src/cdp/outputs/registry.ts
index 94b64ebbfaf9..7c62868954ae 100644
--- a/nodejs/src/cdp/outputs/registry.ts
+++ b/nodejs/src/cdp/outputs/registry.ts
@@ -6,7 +6,7 @@ import {
} from '~/common/outputs'
import { IngestionOutputsBuilder } from '~/common/outputs/ingestion-outputs-builder'
-import { CDP_EVENTS_DLQ_OUTPUT, WAREHOUSE_SOURCE_WEBHOOKS_OUTPUT } from './outputs'
+import { CDP_EVENTS_DLQ_OUTPUT, EMAIL_STATUS_OUTPUT, WAREHOUSE_SOURCE_WEBHOOKS_OUTPUT } from './outputs'
/**
* Outputs the CDP deployments write to. Each output's topic + producer are
@@ -18,6 +18,7 @@ import { CDP_EVENTS_DLQ_OUTPUT, WAREHOUSE_SOURCE_WEBHOOKS_OUTPUT } from './outpu
* `clickhouse_app_metrics2` schema).
* - `WAREHOUSE_SOURCE_WEBHOOKS_OUTPUT` — warehouse source webhook payloads.
* - `CDP_EVENTS_DLQ_OUTPUT` — events that matched a function but produced no invocation.
+ * - `EMAIL_STATUS_OUTPUT` — status changes of transactional emails.
*/
export function createCdpOutputsRegistry() {
return new IngestionOutputsBuilder()
@@ -45,4 +46,8 @@ export function createCdpOutputsRegistry() {
topicKey: 'CDP_EVENTS_DLQ_TOPIC',
producerKey: 'CDP_EVENTS_DLQ_PRODUCER',
})
+ .register(EMAIL_STATUS_OUTPUT, {
+ topicKey: 'EMAIL_STATUS_TOPIC',
+ producerKey: 'EMAIL_STATUS_PRODUCER',
+ })
}
diff --git a/nodejs/src/cdp/services/hog-executor.service.test.ts b/nodejs/src/cdp/services/hog-executor.service.test.ts
index 2ae11dcd4a1e..994f90a42ca5 100644
--- a/nodejs/src/cdp/services/hog-executor.service.test.ts
+++ b/nodejs/src/cdp/services/hog-executor.service.test.ts
@@ -88,7 +88,8 @@ describe('Hog Executor', () => {
hub.SITE_URL,
new EmailTrackingCodeSigner(hub.ENCRYPTION_SALT_KEYS, hub.CDP_EMAIL_TRACKING_URL),
new EmailSuppressionService(hub.postgres, emailSuppressionConfigFromEnv()),
- new RecipientsManagerService(hub.postgres)
+ new RecipientsManagerService(hub.postgres),
+ null
)
const recipientTokensService = new RecipientTokensService(hub.ENCRYPTION_SALT_KEYS, hub.SITE_URL)
return new HogExecutorAsyncService(
diff --git a/nodejs/src/cdp/services/hogflows/actions/hog_function.test.ts b/nodejs/src/cdp/services/hogflows/actions/hog_function.test.ts
index 60e53e0ed5f7..a268665367ac 100644
--- a/nodejs/src/cdp/services/hogflows/actions/hog_function.test.ts
+++ b/nodejs/src/cdp/services/hogflows/actions/hog_function.test.ts
@@ -87,7 +87,8 @@ describe('HogFunctionHandler', () => {
hub.SITE_URL,
new EmailTrackingCodeSigner(hub.ENCRYPTION_SALT_KEYS, hub.CDP_EMAIL_TRACKING_URL),
new EmailSuppressionService(hub.postgres, emailSuppressionConfigFromEnv()),
- new RecipientsManagerService(hub.postgres)
+ new RecipientsManagerService(hub.postgres),
+ null
)
mockHogFunctionExecutor = new HogExecutorAsyncService(
new HogExecutorService({ executionTimeoutMs: hub.CDP_WATCHER_HOG_COST_TIMING_UPPER_MS }, hogInputsService),
diff --git a/nodejs/src/cdp/services/hogflows/hogflow-executor.service.test.ts b/nodejs/src/cdp/services/hogflows/hogflow-executor.service.test.ts
index 5f6df00a7697..6b467b61d305 100644
--- a/nodejs/src/cdp/services/hogflows/hogflow-executor.service.test.ts
+++ b/nodejs/src/cdp/services/hogflows/hogflow-executor.service.test.ts
@@ -99,7 +99,8 @@ describe('Hogflow Executor', () => {
hub.SITE_URL,
new EmailTrackingCodeSigner(hub.ENCRYPTION_SALT_KEYS, hub.CDP_EMAIL_TRACKING_URL),
emailSuppressionService,
- new RecipientsManagerService(hub.postgres)
+ new RecipientsManagerService(hub.postgres),
+ null
)
const recipientTokensService = new RecipientTokensService(hub.ENCRYPTION_SALT_KEYS, hub.SITE_URL)
const hogExecutor = new HogExecutorAsyncService(
diff --git a/nodejs/src/cdp/services/invocation-results.service.test.ts b/nodejs/src/cdp/services/invocation-results.service.test.ts
index 00ab9ed588f3..c6bc739339b1 100644
--- a/nodejs/src/cdp/services/invocation-results.service.test.ts
+++ b/nodejs/src/cdp/services/invocation-results.service.test.ts
@@ -2,6 +2,7 @@ import { CyclotronJobInvocationResult } from '../types'
import { CapturedEventsService } from './captured-events/captured-events.service'
import { ConversionWatchersService } from './conversion-watchers/conversion-watchers.service'
import { InvocationResultsService } from './invocation-results.service'
+import { EmailStatusService } from './messaging/email-status.service'
import { MessageAssetsService } from './messaging/message-assets.service'
import { HogFunctionMonitoringService } from './monitoring/hog-function-monitoring.service'
import { HogInvocationResultsService } from './monitoring/hog-invocation-results.service'
@@ -14,6 +15,7 @@ describe('InvocationResultsService', () => {
let capturedEventsService: jest.Mocked<CapturedEventsService>
let messageAssetsService: jest.Mocked<MessageAssetsService>
let conversionWatchersService: jest.Mocked<ConversionWatchersService>
+ let emailStatusService: jest.Mocked<EmailStatusService>
let service: InvocationResultsService
const results = [
@@ -52,13 +54,18 @@ describe('InvocationResultsService', () => {
flush: jest.fn().mockResolvedValue(undefined),
} as unknown as jest.Mocked<ConversionWatchersService>
+ emailStatusService = {
+ flush: jest.fn().mockResolvedValue(undefined),
+ } as unknown as jest.Mocked<EmailStatusService>
+
service = new InvocationResultsService(
monitoringService,
invocationResultsRowsService,
warehouseWebhooksService,
capturedEventsService,
messageAssetsService,
- conversionWatchersService
+ conversionWatchersService,
+ emailStatusService
)
})
@@ -114,6 +121,7 @@ describe('InvocationResultsService', () => {
expect(warehouseWebhooksService.flush).toHaveBeenCalledTimes(1)
expect(capturedEventsService.flush).toHaveBeenCalledTimes(1)
expect(messageAssetsService.flush).toHaveBeenCalledTimes(1)
+ expect(emailStatusService.flush).toHaveBeenCalledTimes(1)
})
it('flushes all in parallel (all start before any finish)', async () => {
diff --git a/nodejs/src/cdp/services/invocation-results.service.ts b/nodejs/src/cdp/services/invocation-results.service.ts
index 200fdb0798ba..b11ae9cc3a24 100644
--- a/nodejs/src/cdp/services/invocation-results.service.ts
+++ b/nodejs/src/cdp/services/invocation-results.service.ts
@@ -3,6 +3,7 @@ import { instrumentFn } from '~/common/tracing/tracing-utils'
import { CyclotronJobInvocationResult } from '../types'
import { CapturedEventsService } from './captured-events/captured-events.service'
import { ConversionWatchersService } from './conversion-watchers/conversion-watchers.service'
+import { EmailStatusService } from './messaging/email-status.service'
import { MessageAssetsService } from './messaging/message-assets.service'
import { HogFunctionMonitoringService } from './monitoring/hog-function-monitoring.service'
import { HogInvocationResultsService } from './monitoring/hog-invocation-results.service'
@@ -32,6 +33,8 @@ export interface ResultSink {
* Assets tab
* - `ConversionWatchersService` — per-run conversion watchers, which outlive the
* run so a late conversion stays observable
+ * - `EmailStatusService` — status rows of transactional emails that the
+ * email service queues at send time
*
* Callers interact with this one service instead of coordinating queue/flush
* calls across the five individually. `queueInvocationResultsAndFlush` is the
@@ -48,7 +51,8 @@ export class InvocationResultsService {
public readonly warehouseWebhooksService: WarehouseWebhooksService,
public readonly capturedEventsService: CapturedEventsService,
public readonly messageAssetsService: MessageAssetsService,
- public readonly conversionWatchersService: ConversionWatchersService
+ public readonly conversionWatchersService: ConversionWatchersService,
+ public readonly emailStatusService: EmailStatusService
) {
this.sinks = [
monitoringService,
@@ -57,6 +61,7 @@ export class InvocationResultsService {
capturedEventsService,
messageAssetsService,
conversionWatchersService,
+ emailStatusService,
]
}
diff --git a/nodejs/src/cdp/services/messaging/email-status.service.test.ts b/nodejs/src/cdp/services/messaging/email-status.service.test.ts
new file mode 100644
index 000000000000..97e6f4017823
--- /dev/null
+++ b/nodejs/src/cdp/services/messaging/email-status.service.test.ts
@@ -0,0 +1,87 @@
+import { register } from 'prom-client'
+
+import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
+import { UUIDT } from '~/common/utils/utils'
+
+import { EmailStatusOutput } from '../../outputs/outputs'
+import { EmailStatus, EmailStatusService, buildEmailStatusRow } from './email-status.service'
+
+const failedWrites = async (status: EmailStatus): Promise<number> =>
+ (await register.getSingleMetric('cdp_email_status_write_failed')!.get()).values.find(
+ (v) => v.labels.status === status
+ )?.value ?? 0
+
+describe('EmailStatusService', () => {
+ const createdAt = Date.UTC(2026, 0, 2, 3, 4, 5, 678)
+ const messageId = new UUIDT(createdAt).toString()
+
+ it('builds a row whose created_at comes from the message id and whose version leads with the status rank', () => {
+ const statusAt = new Date(Date.UTC(2026, 0, 3, 10, 0, 0, 123))
+
+ expect(
+ buildEmailStatusRow({ teamId: 2, messageId, status: 'failed', reason: 'provider_error', statusAt })
+ ).toEqual({
+ team_id: 2,
+ message_id: messageId,
+ status: 'failed',
+ // 2 * 2^48 + status_at in ms, as a decimal string for ClickHouse's UInt64.
+ version: (2n * 2n ** 48n + BigInt(statusAt.getTime())).toString(),
+ created_at: '2026-01-02 03:04:05.678000',
+ status_at: '2026-01-03 10:00:00.123000',
+ reason: 'provider_error',
+ })
+ })
+
+ it('ranks every status above the previous one even when it is timestamped earlier', () => {
+ const statuses: EmailStatus[] = ['queued', 'failed', 'sent', 'delivered', 'bounced', 'complained']
+ const versions = statuses.map((status, index) =>
+ BigInt(
+ buildEmailStatusRow({
+ teamId: 2,
+ messageId,
+ status,
+ // Each later status carries an earlier time, as a late webhook or a clock skew would.
+ statusAt: new Date(createdAt + (statuses.length - index) * 60_000),
+ }).version
+ )
+ )
+
+ expect(versions).toEqual([...versions].sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)))
+ expect(new Set(versions).size).toEqual(statuses.length)
+ })
+
+ it('returns from a flush whose produce stalls once the wait runs out', async () => {
+ const produce = jest.fn(() => new Promise<void>(() => {}))
+ const service = new EmailStatusService(
+ { produce } as unknown as IngestionOutputs<EmailStatusOutput>,
+ { EMAIL_STATUS_ENABLED: true },
+ 10
+ )
+
+ service.queue({ teamId: 2, messageId, status: 'sent' })
+
+ await expect(service.flush()).resolves.toBeUndefined()
+ expect(produce).toHaveBeenCalledTimes(1)
+ })
+
+ it.each([
+ ['the produce fails', messageId, () => Promise.reject(new Error('broker down'))],
+ ['the message id is not a UUID', 'not-a-uuid', () => Promise.resolve()],
+ ])('drops and counts a row when %s, and still writes the others', async (_desc, badMessageId, produceBad) => {
+ const otherMessageId = new UUIDT().toString()
+ const produce = jest.fn((_output, message: { key: Buffer }) =>
+ message.key.toString() === badMessageId ? produceBad() : Promise.resolve()
+ )
+ const service = new EmailStatusService({ produce } as unknown as IngestionOutputs<EmailStatusOutput>, {
+ EMAIL_STATUS_ENABLED: true,
+ })
+ const failedBefore = await failedWrites('delivered')
+
+ service.queue({ teamId: 2, messageId: badMessageId, status: 'delivered' })
+ service.queue({ teamId: 2, messageId: otherMessageId, status: 'delivered' })
+ await expect(service.flush()).resolves.toBeUndefined()
+
+ expect(await failedWrites('delivered')).toEqual(failedBefore + 1)
+ expect(produce.mock.calls.map(([, message]) => message.key.toString())).toContain(otherMessageId)
+ })
+})
diff --git a/nodejs/src/cdp/services/messaging/email-status.service.ts b/nodejs/src/cdp/services/messaging/email-status.service.ts
new file mode 100644
index 000000000000..ce4f19d13c55
--- /dev/null
+++ b/nodejs/src/cdp/services/messaging/email-status.service.ts
@@ -0,0 +1,176 @@
+import { Counter } from 'prom-client'
+import { setTimeout as delay } from 'timers/promises'
+
+import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
+import { logger } from '~/common/utils/logger'
+
+import { EMAIL_STATUS_OUTPUT, EmailStatusOutput } from '../../outputs/outputs'
+import { CyclotronJobInvocationHogFunction } from '../../types'
+import { TRANSACTIONAL_EMAIL_EVENT } from '../hogflows/hogflow-sensitivity'
+
+const counterEmailStatusRowsProduced = new Counter({
+ name: 'cdp_email_status_rows_produced',
+ help: 'Transactional email status rows produced to the email_status Kafka topic.',
+ labelNames: ['status'],
+})
+
+const counterEmailStatusWriteFailed = new Counter({
+ name: 'cdp_email_status_write_failed',
+ help: 'Transactional email status rows dropped because they could not be built or produced. The send is never affected.',
+ labelNames: ['status'],
+})
+
+export type EmailStatus = 'queued' | 'failed' | 'sent' | 'delivered' | 'bounced' | 'complained'
+
+export type EmailStatusReason =
+ // A failed send. The provider rejected the email, or the send could not be made.
+ 'provider_error'
+
+// The rank is the high part of `version`, so a message's status only moves forward in the
+// ReplacingMergeTree, whatever order the rows arrive in. `failed` ranks below `sent` so that a
+// successful rerun of the same invocation replaces the failed attempt. The gaps let a failure take the
+// rank between two statuses. Never renumber a rank, because rows already written keep their versions.
+const STATUS_RANK: Record<EmailStatus, bigint> = {
+ queued: 0n,
+ failed: 2n,
+ sent: 3n,
+ delivered: 5n,
+ bounced: 6n,
+ complained: 7n,
+}
+
+// 2^48 ms is past the year 10000, so the status time never reaches into the rank bits.
+const STATUS_AT_BITS = 48n
+
+// How long a flush waits for its rows. See `flush`.
+const EMAIL_STATUS_FLUSH_MAX_WAIT_MS = 1000
+
+const UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i
+
+export interface EmailStatusRow {
+ team_id: number
+ message_id: string
+ status: EmailStatus
+ // A decimal string, because the value is past Number.MAX_SAFE_INTEGER. ClickHouse parses it as UInt64.
+ version: string
+ created_at: string
+ status_at: string
+ reason: string
+}
+
+export interface EmailStatusUpdate {
+ teamId: number
+ // The hog flow invocation id, which the transactional email API returns as `message_id`.
+ messageId: string
+ status: EmailStatus
+ reason?: EmailStatusReason
+ // When the status changed. Defaults to now.
+ statusAt?: Date
+}
+
+const isoMicroseconds = (date: Date): string => {
+ // ClickHouse DateTime64(6) accepts 'YYYY-MM-DD HH:MM:SS.ffffff'.
+ return date.toISOString().replace('T', ' ').replace('Z', '000')
+}
+
+// A UUIDT starts with its creation time as 48 bits of Unix milliseconds.
+const uuidtCreatedAt = (messageId: string): Date => {
+ if (!UUID_REGEX.test(messageId)) {
+ throw new Error('message id is not a UUID')
+ }
+ return new Date(parseInt(messageId.slice(0, 8) + messageId.slice(9, 13), 16))
+}
+
+export const buildEmailStatusRow = (update: EmailStatusUpdate): EmailStatusRow => {
+ const statusAt = update.statusAt && !Number.isNaN(update.statusAt.getTime()) ? update.statusAt : new Date()
+ const version = (STATUS_RANK[update.status] << STATUS_AT_BITS) + BigInt(statusAt.getTime())
+ return {
+ team_id: update.teamId,
+ message_id: update.messageId,
+ status: update.status,
+ version: version.toString(),
+ // Derived from the message id rather than the clock, so every row of a message lands in
+ // the same `toYYYYMMDD(created_at)` partition and the rows can collapse.
+ created_at: isoMicroseconds(uuidtCreatedAt(update.messageId)),
+ status_at: isoMicroseconds(statusAt),
+ reason: update.reason ?? '',
+ }
+}
+
+// A transactional email run is triggered by the synthetic event that the transactional email API creates.
+export const isTransactionalEmailSend = (invocation: CyclotronJobInvocationHogFunction): boolean =>
+ invocation.state?.globals?.event?.event === TRANSACTIONAL_EMAIL_EVENT
+
+/**
+ * Records each status change of a transactional email for the email_status ClickHouse table.
+ *
+ * Rows are buffered by `queue` and produced by `flush`. Both are best-effort and never throw:
+ * a status row that cannot be written is logged and counted, and the send it describes goes on.
+ */
+export class EmailStatusService {
+ private queuedRows: EmailStatusRow[] = []
+
+ constructor(
+ private outputs: IngestionOutputs<EmailStatusOutput>,
+ private config: { EMAIL_STATUS_ENABLED: boolean },
+ private maxFlushWaitMs = EMAIL_STATUS_FLUSH_MAX_WAIT_MS
+ ) {}
+
+ queue(update: EmailStatusUpdate): void {
+ if (!this.config.EMAIL_STATUS_ENABLED) {
+ return
+ }
+ try {
+ this.queuedRows.push(buildEmailStatusRow(update))
+ } catch (error) {
+ counterEmailStatusWriteFailed.inc({ status: update.status })
+ logger.error('⚠️', 'failed to build an email status row', {
+ error: String(error),
+ team_id: update.teamId,
+ message_id: update.messageId,
+ status: update.status,
+ })
+ }
+ }
+
+ // Required by ResultSink. The rows go through the shared Kafka outputs, which the server
+ // disconnects separately on shutdown.
+ async stop(): Promise<void> {
+ return Promise.resolve()
+ }
+
+ // The caller waits at most `maxFlushWaitMs`, and rows still in flight are produced in the background.
+ // A stalled produce then cannot hold up the send API's reply, the SES webhook, or a worker batch past
+ // its timeout. The status API already documents that a status can arrive a few seconds late.
+ async flush(): Promise<void> {
+ if (this.queuedRows.length === 0) {
+ return
+ }
+ const rows = this.queuedRows
+ this.queuedRows = []
+ await Promise.race([this.produce(rows), delay(this.maxFlushWaitMs, undefined, { ref: false })])
+ }
+
+ private async produce(rows: EmailStatusRow[]): Promise<void> {
+ await Promise.all(
+ rows.map(async (row) => {
+ try {
+ await this.outputs.produce(EMAIL_STATUS_OUTPUT, {
+ // Keyed by message so all rows of a message go to the same partition.
+ key: Buffer.from(row.message_id),
+ value: Buffer.from(JSON.stringify(row)),
+ })
+ counterEmailStatusRowsProduced.inc({ status: row.status })
+ } catch (error) {
+ counterEmailStatusWriteFailed.inc({ status: row.status })
+ logger.error('⚠️', 'failed to produce an email status row', {
+ error: String(error),
+ team_id: row.team_id,
+ message_id: row.message_id,
+ status: row.status,
+ })
+ }
+ })
+ )
+ }
+}
diff --git a/nodejs/src/cdp/services/messaging/email.service.test.ts b/nodejs/src/cdp/services/messaging/email.service.test.ts
index 459f664a7da7..d3655ef28311 100644
--- a/nodejs/src/cdp/services/messaging/email.service.test.ts
+++ b/nodejs/src/cdp/services/messaging/email.service.test.ts
@@ -3,15 +3,19 @@ import { mockFetch } from '~/tests/helpers/mocks/request.mock'
import { MessageRejected, SendingPausedException, TooManyRequestsException } from '@aws-sdk/client-sesv2'
import { createExampleInvocation, insertIntegration } from '~/cdp/_tests/fixtures'
+import { EmailStatusOutput } from '~/cdp/outputs/outputs'
import {
CyclotronInvocationQueueParametersEmailSchema,
CyclotronInvocationQueueParametersEmailType,
} from '~/cdp/schema/cyclotron'
+import { TRANSACTIONAL_EMAIL_EVENT } from '~/cdp/services/hogflows/hogflow-sensitivity'
import { CyclotronJobInvocationHogFunction } from '~/cdp/types'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { createRedisV2PoolFromConfig } from '~/common/redis/redis-v2'
import { closeHub, createHub } from '~/common/utils/db/hub'
import { PostgresUse } from '~/common/utils/db/postgres'
+import { parseJSON } from '~/common/utils/json-parse'
+import { UUIDT } from '~/common/utils/utils'
import { waitForExpect } from '~/tests/helpers/expectations'
import { createTestTeamFixture } from '~/tests/helpers/sql'
import { INVALID_TEAM_ID_KEYS } from '~/tests/helpers/team'
@@ -21,6 +25,7 @@ import { RecipientsManagerService } from '../managers/recipients-manager.service
import { TeamWorkflowsConfigService } from '../managers/team-workflows-config.service'
import { RateLimiterService } from '../rate-limiter/rate-limiter.service'
import { selectEmailSenderIntegrationId } from './email-sender-selection'
+import { EmailStatusService } from './email-status.service'
import { EmailSuppressionService, emailSuppressionConfigFromEnv } from './email-suppression.service'
import {
EmailService,
@@ -144,7 +149,8 @@ describe('EmailService', () => {
hub.SITE_URL,
new EmailTrackingCodeSigner(hub.ENCRYPTION_SALT_KEYS, hub.CDP_EMAIL_TRACKING_URL),
new EmailSuppressionService(hub.postgres, emailSuppressionConfigFromEnv()),
- new RecipientsManagerService(hub.postgres)
+ new RecipientsManagerService(hub.postgres),
+ null
)
mockFetch.mockClear()
})
@@ -168,7 +174,8 @@ describe('EmailService', () => {
hub.SITE_URL,
new EmailTrackingCodeSigner(hub.ENCRYPTION_SALT_KEYS, hub.CDP_EMAIL_TRACKING_URL),
new EmailSuppressionService(hub.postgres, emailSuppressionConfigFromEnv()),
- new RecipientsManagerService(hub.postgres)
+ new RecipientsManagerService(hub.postgres),
+ null
)
expect(serviceWithoutSES.sesV2Client).toBeNull()
@@ -525,6 +532,7 @@ describe('EmailService', () => {
new EmailTrackingCodeSigner(hub.ENCRYPTION_SALT_KEYS, hub.CDP_EMAIL_TRACKING_URL),
new EmailSuppressionService(hub.postgres, emailSuppressionConfigFromEnv()),
new RecipientsManagerService(hub.postgres),
+ null,
undefined,
{ claimOrReserve } as unknown as RateLimiterService
)
@@ -696,6 +704,7 @@ describe('EmailService', () => {
new EmailTrackingCodeSigner(hub.ENCRYPTION_SALT_KEYS, hub.CDP_EMAIL_TRACKING_URL),
new EmailSuppressionService(hub.postgres, emailSuppressionConfigFromEnv()),
new RecipientsManagerService(hub.postgres),
+ null,
undefined,
new RateLimiterService(redis, { name: 'workflow-email-backlog-test' })
)
@@ -766,6 +775,7 @@ describe('EmailService', () => {
new EmailTrackingCodeSigner(hub.ENCRYPTION_SALT_KEYS, hub.CDP_EMAIL_TRACKING_URL),
new EmailSuppressionService(hub.postgres, emailSuppressionConfigFromEnv()),
new RecipientsManagerService(hub.postgres),
+ null,
undefined,
null,
{ claimAllOrNothingPair } as unknown as RateLimiterService
@@ -894,6 +904,7 @@ describe('EmailService', () => {
new EmailTrackingCodeSigner(hub.ENCRYPTION_SALT_KEYS, hub.CDP_EMAIL_TRACKING_URL),
new EmailSuppressionService(hub.postgres, emailSuppressionConfigFromEnv()),
new RecipientsManagerService(hub.postgres),
+ null,
undefined,
null,
limiter
@@ -1082,6 +1093,45 @@ describe('EmailService', () => {
})
})
+ const sesAccepts = (): void => {
+ sendEmailSpy.mockResolvedValue({ MessageId: 'test-message-id' })
+ }
+ const sesRejects = (): void => {
+ sendEmailSpy.mockRejectedValue(new Error('SES error'))
+ }
+ it.each([
+ ['a transactional email that SES accepts', TRANSACTIONAL_EMAIL_EVENT, false, sesAccepts, 'sent', ''],
+ [
+ 'a transactional email that SES rejects',
+ TRANSACTIONAL_EMAIL_EVENT,
+ false,
+ sesRejects,
+ 'failed',
+ 'provider_error',
+ ],
+ ['a test send of a transactional email', TRANSACTIONAL_EMAIL_EVENT, true, sesAccepts, null, null],
+ ['a workflow email that SES accepts', '$pageview', false, sesAccepts, null, null],
+ ['a workflow email that SES rejects', '$pageview', false, sesRejects, null, null],
+ ])('records the status of %s', async (_name, triggerEvent, isTest, setup, status, reason) => {
+ const produce = jest.fn().mockResolvedValue(undefined)
+ const emailStatusService = new EmailStatusService(
+ { produce } as unknown as IngestionOutputs<EmailStatusOutput>,
+ { EMAIL_STATUS_ENABLED: true }
+ )
+ service['emailStatusService'] = emailStatusService
+ invocation.id = new UUIDT().toString()
+ invocation.state.globals.event.event = triggerEvent
+ setup()
+
+ await service.executeSendEmail(invocation, isTest)
+ await emailStatusService.flush()
+
+ const rows = produce.mock.calls.map(([, message]) => parseJSON(message.value.toString()))
+ expect(rows).toEqual(
+ status ? [expect.objectContaining({ team_id: team.id, message_id: invocation.id, status, reason })] : []
+ )
+ })
+
it('records a send-time metric for normal sends but not for test sends', async () => {
sendEmailSpy.mockResolvedValue({ MessageId: 'test-message-id' })
diff --git a/nodejs/src/cdp/services/messaging/email.service.ts b/nodejs/src/cdp/services/messaging/email.service.ts
index c0f4581a699c..2bfa8417d268 100644
--- a/nodejs/src/cdp/services/messaging/email.service.ts
+++ b/nodejs/src/cdp/services/messaging/email.service.ts
@@ -21,6 +21,7 @@ import { RecipientManagerRecipient, RecipientsManagerService } from '../managers
import { TeamWorkflowsConfigService } from '../managers/team-workflows-config.service'
import { RateLimiterService } from '../rate-limiter/rate-limiter.service'
import { selectEmailSenderIntegrationId } from './email-sender-selection'
+import { EmailStatusService, isTransactionalEmailSend } from './email-status.service'
import { EmailSuppressionService, SuppressionCheckOptions } from './email-suppression.service'
import {
addTrackingToEmail,
@@ -364,6 +365,9 @@ export class EmailService {
private trackingCodeSigner: EmailTrackingCodeSigner,
private emailSuppressionService: EmailSuppressionService,
private recipientsManager: RecipientsManagerService,
+ // Required, so that a caller which builds the service without it fails to compile. Pass null
+ // only where no email status is recorded.
+ private emailStatusService: EmailStatusService | null,
private messageAssetsService?: MessageAssetsService,
private workflowEmailRateLimiter: RateLimiterService | null = null,
private teamEmailRateLimiter: RateLimiterService | null = null
@@ -753,6 +757,17 @@ export class EmailService {
if (success && assetRow) {
result.messageAssets.push(assetRow)
}
+
+ // A failure here is final: SES throttling returned above to reschedule, and nothing
+ // retries a failed send.
+ if (this.emailStatusService && isTransactionalEmailSend(invocation)) {
+ this.emailStatusService.queue({
+ teamId: invocation.teamId,
+ messageId: invocation.id,
+ status: success ? 'sent' : 'failed',
+ reason: success ? undefined : 'provider_error',
+ })
+ }
}
const distinctId = resolveEmailEngagementDistinctId(invocation)
diff --git a/nodejs/src/cdp/templates/test/test-helpers.ts b/nodejs/src/cdp/templates/test/test-helpers.ts
index 0f78d88d7e5e..6fb026c1b3b1 100644
--- a/nodejs/src/cdp/templates/test/test-helpers.ts
+++ b/nodejs/src/cdp/templates/test/test-helpers.ts
@@ -220,7 +220,8 @@ export class TemplateTester {
config.SITE_URL,
new EmailTrackingCodeSigner(config.ENCRYPTION_SALT_KEYS, config.CDP_EMAIL_TRACKING_URL),
undefined as any,
- undefined as any
+ undefined as any,
+ null
)
return new HogExecutorAsyncService(
new HogExecutorService(
diff --git a/nodejs/src/common/config/kafka-topics.ts b/nodejs/src/common/config/kafka-topics.ts
index ff32d96febe9..2222d8bc9bdb 100644
--- a/nodejs/src/common/config/kafka-topics.ts
+++ b/nodejs/src/common/config/kafka-topics.ts
@@ -84,6 +84,9 @@ export const KAFKA_LOG_ENTRIES = `${prefix}log_entries${suffix}`
// per-invocation result rows for hog functions and hog flows
export const KAFKA_HOG_INVOCATION_RESULTS = `${prefix}clickhouse_hog_invocation_results${suffix}`
+// one row per status change of a transactional email, for the email_status ClickHouse table
+export const KAFKA_EMAIL_STATUS = `${prefix}clickhouse_email_status${suffix}`
+
// metadata rows for sent message assets (rendered emails stored in object storage)
export const KAFKA_MESSAGE_ASSETS = `${prefix}clickhouse_message_assets${suffix}`