:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import { Counter } from 'prom-client'
import { setTimeout as delay } from 'timers/promises'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { logger } from '~/common/utils/logger'
import { EMAIL_STATUS_OUTPUT, EmailStatusOutput } from '../../outputs/outputs'
import { CyclotronJobInvocation, CyclotronJobInvocationHogFunction, CyclotronJobInvocationResult } from '../../types'
import { TRANSACTIONAL_EMAIL_EVENT } from '../hogflows/hogflow-sensitivity'
import { classifyError } from '../monitoring/hog-invocation-results.service'
const counterEmailStatusRowsProduced = new Counter({
name: 'cdp_email_status_rows_produced',
help: 'Transactional email status rows produced to the email_status Kafka topic, by status and reason. The reason is empty for a status without one, a skip reason or a run error kind for failed, and the bounce type for bounced.',
labelNames: ['status', 'reason'],
})
const counterEmailStatusWriteFailed = new Counter({
name: 'cdp_email_status_write_failed',
help: 'Transactional email status rows dropped because they could not be built or produced. The send is never affected.',
labelNames: ['status'],
})
export type EmailDeliveryStatus = 'queued' | 'failed' | 'sent' | 'delivered' | 'bounced' | 'complained'
// Only email_status_events keeps these, so they never replace a delivery status.
export type EmailEngagementStatus = 'opened' | 'clicked'
export type EmailStatus = EmailDeliveryStatus | EmailEngagementStatus
export type EmailStatusReason =
// A bounce type, in the words the `$bounce_type` event property uses.
| 'hard'
| 'soft'
| 'unknown'
// A failed send. The provider rejected the email, or the send could not be made.
| 'provider_error'
// The provider refused the email after it accepted the send, for example because it found a virus.
| 'provider_rejected'
// A send that was skipped. The first four are also the codes the transactional email API
// returns when it refuses a send before it queues the run.
| EmailSkipReason
// A run that was canceled before it finished.
| 'canceled'
// A run whose workflow or team no longer exists when it is dequeued.
| 'workflow_unavailable'
// A run that failed. The reason is the `error_kind` that hog_invocation_results stores for the
// run, for example `hog_error` or `timeout`.
| (string & {})
export type EmailSkipReason =
| 'recipient_suppressed'
| 'sending_paused'
| 'sending_suspended'
| 'recipient_unreachable'
| 'missing_recipient'
| 'recipient_opted_out'
| 'frequency_capped'
// The rank is the high part of `version`, so a message's status only moves forward in the
// ReplacingMergeTree, whatever order the rows arrive in. `failed` ranks below `sent` so that a
// successful rerun of the same invocation replaces the failed attempt. The gaps let a failure take the
// rank between two statuses. Never renumber a rank, because rows already written keep their versions.
const STATUS_RANK: Record<EmailStatus, bigint> = {
queued: 0n,
failed: 2n,
sent: 3n,
delivered: 5n,
bounced: 6n,
complained: 7n,
opened: 8n,
clicked: 9n,
}
// A run that fails on its email step fails after the step recorded why, for example
// `provider_error`. The run's own failure ranks lower, so it never replaces that reason.
const RUN_FAILED_RANK = 1n
// A provider rejection comes after the provider accepted the send, so it must replace `sent`. The
// provider never delivers a rejected email, so `delivered` and later statuses still rank above it.
const PROVIDER_REJECTED_RANK = 4n
// 2^48 ms is past the year 10000, so the status time never reaches into the rank bits.
const STATUS_AT_BITS = 48n
// How long a flush waits for its rows. See `flush`.
const EMAIL_STATUS_FLUSH_MAX_WAIT_MS = 1000
const UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i
export interface EmailStatusRow {
team_id: number
message_id: string
status: EmailStatus
// A decimal string, because the value is past Number.MAX_SAFE_INTEGER. ClickHouse parses it as UInt64.
version: string
created_at: string
status_at: string
reason: string
recipient: string
function_id: string
sending_domain: string
}
export interface EmailStatusUpdate {
teamId: number
// The hog flow invocation id, which the transactional email API returns as `message_id`.
messageId: string
status: EmailStatus
reason?: EmailStatusReason
// When the status changed. Defaults to now.
statusAt?: Date
// A failure of the whole run rather than of the email step.
runOutcome?: boolean
recipient?: string
functionId?: string
sendingDomain?: string
}
const isoMicroseconds = (date: Date): string => {
// ClickHouse DateTime64(6) accepts 'YYYY-MM-DD HH:MM:SS.ffffff'.
return date.toISOString().replace('T', ' ').replace('Z', '000')
}
// A UUIDT starts with its creation time as 48 bits of Unix milliseconds.
const uuidtCreatedAt = (messageId: string): Date => {
if (!UUID_REGEX.test(messageId)) {
throw new Error('message id is not a UUID')
}
return new Date(parseInt(messageId.slice(0, 8) + messageId.slice(9, 13), 16))
}
export const buildEmailStatusRow = (update: EmailStatusUpdate): EmailStatusRow => {
const statusAt = update.statusAt && !Number.isNaN(update.statusAt.getTime()) ? update.statusAt : new Date()
const rank =
update.status !== 'failed'
? STATUS_RANK[update.status]
: update.runOutcome
? RUN_FAILED_RANK
: update.reason === 'provider_rejected'
? PROVIDER_REJECTED_RANK
: STATUS_RANK.failed
const version = (rank << STATUS_AT_BITS) + BigInt(statusAt.getTime())
return {
team_id: update.teamId,
message_id: update.messageId,
status: update.status,
version: version.toString(),
// Derived from the message id rather than the clock, so every row of a message lands in
// the same `toYYYYMMDD(created_at)` partition and the rows can collapse.
created_at: isoMicroseconds(uuidtCreatedAt(update.messageId)),
status_at: isoMicroseconds(statusAt),
reason: update.reason ?? '',
recipient: update.recipient ?? '',
function_id: update.functionId ?? '',
sending_domain: update.sendingDomain ?? '',
}
}
// A transactional email run is triggered by the synthetic event that the transactional email API creates.
export const isTransactionalEmailSend = (invocation: CyclotronJobInvocationHogFunction): boolean =>
invocation.state?.globals?.event?.event === TRANSACTIONAL_EMAIL_EVENT
const isTransactionalEmailRun = (invocation: CyclotronJobInvocation): boolean =>
'hogFlow' in invocation && invocation.state?.event?.event === TRANSACTIONAL_EMAIL_EVENT
// A dequeued job has no `hogFlow` yet, so only its state says which event triggered it.
const isTransactionalEmailJob = (job: CyclotronJobInvocation): boolean =>
(job.state as { event?: { event?: string } } | null)?.event?.event === TRANSACTIONAL_EMAIL_EVENT
/**
* Records each status change of a transactional email for the email_status ClickHouse table.
*
* Rows are buffered by `queue` and produced by `flush`. Both are best-effort and never throw:
* a status row that cannot be written is logged and counted, and the send it describes goes on.
*/
export class EmailStatusService {
private queuedRows: EmailStatusRow[] = []
constructor(
private outputs: IngestionOutputs<EmailStatusOutput>,
private config: { EMAIL_STATUS_ENABLED: boolean },
private maxFlushWaitMs = EMAIL_STATUS_FLUSH_MAX_WAIT_MS
) {}
queue(update: EmailStatusUpdate): void {
if (!this.config.EMAIL_STATUS_ENABLED) {
return
}
try {
this.queuedRows.push(buildEmailStatusRow(update))
} catch (error) {
counterEmailStatusWriteFailed.inc({ status: update.status })
logger.error('⚠️', 'failed to build an email status row', {
error: String(error),
team_id: update.teamId,
message_id: update.messageId,
status: update.status,
})
}
}
// A skipped send has no later status, so it is recorded as failed. Otherwise the message stays
// queued, and the caller never learns that the email did not go out.
queueSkippedSend(invocation: CyclotronJobInvocationHogFunction, reason: EmailSkipReason, isTest: boolean): void {
if (isTest || !isTransactionalEmailSend(invocation)) {
return
}
this.queue({ teamId: invocation.teamId, messageId: invocation.id, status: 'failed', reason })
}
// A run that fails or is canceled before its email step never reaches the email service, so the
// run's outcome is the email's. A failed row is written even when the email went out or failed
// earlier in the run, because a run failure ranks below both and so never replaces them.
queueInvocationResults(results: CyclotronJobInvocationResult[]): void {
for (const result of results) {
if (!(result.error || result.canceled) || !isTransactionalEmailRun(result.invocation)) {
continue
}
this.queue({
teamId: result.invocation.teamId,
messageId: result.invocation.id,
status: 'failed',
// Same precedence as the lifecycle row in hog_invocation_results: an error wins over a cancel.
reason: result.error ? classifyError(result.error).kind : 'canceled',
runOutcome: true,
})
}
}
// A job whose workflow or team is gone is dropped without a result, so nothing else records it.
queueUnavailableRun(job: CyclotronJobInvocation): void {
if (!isTransactionalEmailJob(job)) {
return
}
this.queue({
teamId: job.teamId,
messageId: job.id,
status: 'failed',
reason: 'workflow_unavailable',
runOutcome: true,
})
}
// Required by ResultSink. The rows go through the shared Kafka outputs, which the server
// disconnects separately on shutdown.
async stop(): Promise<void> {
return Promise.resolve()
}
// The caller waits at most `maxFlushWaitMs`, and rows still in flight are produced in the background.
// A stalled produce then cannot hold up the send API's reply, the SES webhook, or a worker batch past
// its timeout. The status API already documents that a status can arrive a few seconds late.
async flush(): Promise<void> {
if (this.queuedRows.length === 0) {
return
}
const rows = this.queuedRows
this.queuedRows = []
await Promise.race([this.produce(rows), delay(this.maxFlushWaitMs, undefined, { ref: false })])
}
private async produce(rows: EmailStatusRow[]): Promise<void> {
await Promise.all(
rows.map(async (row) => {
try {
await this.outputs.produce(EMAIL_STATUS_OUTPUT, {
// Keyed by message so all rows of a message go to the same partition.
key: Buffer.from(row.message_id),
value: Buffer.from(JSON.stringify(row)),
})
counterEmailStatusRowsProduced.inc({ status: row.status, reason: row.reason })
} catch (error) {
counterEmailStatusWriteFailed.inc({ status: row.status })
logger.error('⚠️', 'failed to produce an email status row', {
error: String(error),
team_id: row.team_id,
message_id: row.message_id,
status: row.status,
})
}
})
)
}
}