Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(cdp): add transactional email latency and SES send metrics (#125066)

Harley Alexander59d675c0aa7c
diff --git a/nodejs/src/cdp/cdp-api.serial.test.ts b/nodejs/src/cdp/cdp-api.serial.test.ts
index 419637585c47..a3156c0f3e87 100644
--- a/nodejs/src/cdp/cdp-api.serial.test.ts
+++ b/nodejs/src/cdp/cdp-api.serial.test.ts
@@ -36,7 +36,10 @@ import { CdpApi } from './cdp-api'
import { CdpConsumerBaseDeps } from './consumers/cdp-base.consumer'
import { posthogFilterOutPlugin } from './legacy-plugins/_transformations/posthog-filter-out-plugin/template'
import { IdempotencyKeys, createIdempotencyValkeyPool } from './services/hogflows/idempotency-keys'
-import { transactionalEmailRequestsTotal } from './services/hogflows/transactional-email'
+import {
+ transactionalEmailRequestsTotal,
+ transactionalEmailStageDurationMs,
+} from './services/hogflows/transactional-email'
import { BASE_REDIS_KEY, HogWatcherState } from './services/monitoring/hog-watcher.service'
import { compileHog } from './templates/compiler'
import { HogFunctionInvocationGlobals, HogFunctionType } from './types'
@@ -2141,6 +2144,18 @@ describe('CDP API', () => {
// The alerts on this endpoint key on these outcome labels.
const outcomeCount = async (outcome: string): Promise<number> =>
(await transactionalEmailRequestsTotal.get()).values.find((v) => v.labels.outcome === outcome)?.value ?? 0
+ const stageCounts = async (): Promise<Record<string, number>> =>
+ Object.fromEntries(
+ (await transactionalEmailStageDurationMs.get()).values
+ .filter((v) => v.metricName === 'cdp_transactional_email_stage_duration_ms_count')
+ .map((v) => [String(v.labels.stage), v.value])
+ )
+ const stageCountChanges = (before: Record<string, number>, after: Record<string, number>) =>
+ Object.fromEntries(
+ Object.entries(after)
+ .map(([stage, count]) => [stage, count - (before[stage] ?? 0)])
+ .filter(([, change]) => change !== 0)
+ )
const send = (flowId: string, headers: Record<string, string>, payload: Record<string, unknown> = body) =>
supertest(app)
@@ -2170,11 +2185,23 @@ describe('CDP API', () => {
it('queues one run on the email fast lane with the recipient and the data', async () => {
const queueAppMetric = jest.spyOn(api['hogFunctionMonitoringService'], 'queueAppMetric')
const queuedBefore = await outcomeCount('queued')
+ const stagesBefore = await stageCounts()
const res = await send(hogFlow.id, { Authorization: `Bearer ${mintToken(team.id, hogFlow.id)}` })
expect(res.status).toEqual(202)
expect(await outcomeCount('queued')).toEqual(queuedBefore + 1)
+ // Without an Idempotency-Key the request makes no idempotency call.
+ expect(stageCountChanges(stagesBefore, await stageCounts())).toEqual({
+ auth: 1,
+ validate: 1,
+ load: 1,
+ build: 1,
+ checks: 1,
+ queue_insert: 1,
+ metrics_flush: 1,
+ total: 1,
+ })
expect(mockQueueInvocations).toHaveBeenCalledTimes(1)
const [invocation] = mockQueueInvocations.mock.calls[0][0]
expect(invocation).toMatchObject({
@@ -2337,6 +2364,7 @@ describe('CDP API', () => {
] as const)('refuses a send the email worker would skip because %s', async (_desc, code, refuse) => {
refuse()
const refusedBefore = await outcomeCount(code)
+ const stagesBefore = await stageCounts()
const keyed = { ...body, idempotency_key: `key-${new UUIDT().toString()}` }
const headers = { Authorization: `Bearer ${mintToken(team.id, hogFlow.id)}` }
@@ -2345,6 +2373,16 @@ describe('CDP API', () => {
expect(refused.status).toEqual(422)
expect(refused.body).toEqual({ error: code })
expect(await outcomeCount(code)).toEqual(refusedBefore + 1)
+ // The reserve and the release of the key count once, as one idempotency observation.
+ expect(stageCountChanges(stagesBefore, await stageCounts())).toEqual({
+ auth: 1,
+ validate: 1,
+ load: 1,
+ build: 1,
+ idempotency: 1,
+ checks: 1,
+ total: 1,
+ })
expect(mockQueueInvocations).not.toHaveBeenCalled()
expect(emailStatusRows()).toEqual([])
diff --git a/nodejs/src/cdp/cdp-api.ts b/nodejs/src/cdp/cdp-api.ts
index c08af8df2435..ed3459b1cd62 100644
--- a/nodejs/src/cdp/cdp-api.ts
+++ b/nodejs/src/cdp/cdp-api.ts
@@ -56,6 +56,7 @@ import { WorkflowStepResumeSchema } from './services/hogflows/step-resume.servic
import {
TransactionalEmailOutcome,
TransactionalEmailRequestSchema,
+ TransactionalEmailStageTimer,
transactionalEmailRequestsTotal,
} from './services/hogflows/transactional-email'
import { InvocationResultsService } from './services/invocation-results.service'
@@ -63,7 +64,7 @@ import { JobQueue } from './services/job-queue/job-queue.interface'
import { GroupsManagerService } from './services/managers/groups-manager.service'
import { HogFunctionManagerService } from './services/managers/hog-function-manager.service'
import { TeamWorkflowsConfigService } from './services/managers/team-workflows-config.service'
-import { EMAIL_QUEUE_PRIORITY } from './services/messaging/email-priority'
+import { EMAIL_QUEUE_PRIORITY, EmailQueuePriorityClass, cdpEmailQueuedTotal } from './services/messaging/email-priority'
import { EmailStatusService } from './services/messaging/email-status.service'
import { EmailSuppressionService } from './services/messaging/email-suppression.service'
import { EmailTrackingService } from './services/messaging/email-tracking.service'
@@ -1159,11 +1160,13 @@ export class CdpApi {
// Auth: a per-call JWT minted by Django on its own audience and key, pinned to the team and the workflow,
// never the fleet-wide internal secret.
private postTransactionalEmail = async (req: ModifiedRequest, res: express.Response): Promise<any> => {
+ const stages = new TransactionalEmailStageTimer()
const reply = (outcome: TransactionalEmailOutcome, status: number, body: Record<string, unknown>) => {
transactionalEmailRequestsTotal.inc({ outcome })
return res.status(status).json(body)
}
try {
+ stages.start('auth')
if (!this.transactionalEmailJwt.enabled) {
return reply('not_configured', 503, {
error: 'Transactional email auth not configured (WORKFLOWS_TRANSACTIONAL_EMAIL_JWT_SECRET unset)',
@@ -1173,12 +1176,14 @@ export class CdpApi {
transactionalEmailRequestsTotal.inc({ outcome: 'unauthorized' })
return
}
+ stages.start('validate')
const parsed = TransactionalEmailRequestSchema.safeParse(req.body)
if (!parsed.success) {
return reply('invalid_request', 400, { error: parsed.error.message })
}
const { to, distinct_id: distinctId, variables, idempotency_key: idempotencyKey } = parsed.data
+ stages.start('load')
const team = await this.deps.teamManager.getTeam(parseInt(req.params.team_id)).catch(() => null)
if (!team) {
return reply('not_found', 404, { error: 'Team not found' })
@@ -1191,6 +1196,7 @@ export class CdpApi {
return reply('not_sendable', 400, { error: 'not_sendable' })
}
+ stages.start('build')
const event: HogFunctionInvocationGlobals['event'] = {
uuid: new UUIDT().toString(),
event: TRANSACTIONAL_EMAIL_EVENT,
@@ -1215,14 +1221,16 @@ export class CdpApi {
const invocation = createHogFlowInvocation(triggerGlobals, hogFlow, filterGlobals)
// Straight to the email queue: the flow's only step is the email, so a hop through the
// workflow queue would only add latency to a password reset or a verification code.
+ const priorityClass: EmailQueuePriorityClass = 'fast'
invocation.queue = 'email'
- invocation.queuePriority = EMAIL_QUEUE_PRIORITY.fast
+ invocation.queuePriority = EMAIL_QUEUE_PRIORITY[priorityClass]
const idempotencyKeys = this.transactionalEmailIdempotencyKeys
const fingerprint = IdempotencyKeys.fingerprint([hogFlow.id, to, distinctId, variables])
if (idempotencyKey) {
// Checked before the quota, so a retry of a send that went out gets its answer even
// when the team has hit its quota since.
+ stages.start('idempotency')
const reservation = await idempotencyKeys.reserve(team.id, idempotencyKey, fingerprint, invocation.id)
switch (reservation.status) {
case 'replayed':
@@ -1240,13 +1248,16 @@ export class CdpApi {
}
}
try {
+ stages.start('checks')
const refusal = await this.refuseTransactionalEmail(team.id, hogFlow, to)
if (refusal) {
if (idempotencyKey) {
+ stages.start('idempotency')
await idempotencyKeys.release(team.id, idempotencyKey, invocation.id)
}
return reply(refusal.outcome, refusal.status, { error: refusal.outcome })
}
+ stages.start('queue_insert')
this.invocationResultsService.invocationResultsRowsService.queueLifecycleRow(invocation, 'running')
try {
await this.hogflowQueue.queueInvocations([invocation])
@@ -1254,16 +1265,20 @@ export class CdpApi {
this.invocationResultsService.invocationResultsRowsService.dropQueuedRowsFor([invocation.id])
throw error
}
+ cdpEmailQueuedTotal.labels(priorityClass).inc()
} catch (error) {
// Nothing was queued, so the caller's retry with the same key must be able to send.
if (idempotencyKey) {
+ stages.start('idempotency')
await idempotencyKeys.release(team.id, idempotencyKey, invocation.id)
}
throw error
}
if (idempotencyKey) {
+ stages.start('idempotency')
await idempotencyKeys.markQueued(team.id, idempotencyKey, fingerprint, invocation.id)
}
+ stages.start('metrics_flush')
this.emailStatusService.queue({ teamId: team.id, messageId: invocation.id, status: 'queued' })
// Flushed together, and bounded, so the best-effort status row can't hold up the 202.
await Promise.all([
@@ -1290,6 +1305,8 @@ export class CdpApi {
error: e instanceof Error ? e.message : String(e),
})
return reply('error', 500, { error: e instanceof Error ? e.message : String(e) })
+ } finally {
+ stages.finish()
}
}
diff --git a/nodejs/src/cdp/services/cyclotron-v2/cyclotron-v2.test.ts b/nodejs/src/cdp/services/cyclotron-v2/cyclotron-v2.test.ts
index 188fb18c0c2b..faf24792b7ea 100644
--- a/nodejs/src/cdp/services/cyclotron-v2/cyclotron-v2.test.ts
+++ b/nodejs/src/cdp/services/cyclotron-v2/cyclotron-v2.test.ts
@@ -170,8 +170,8 @@ async function dequeueOneBatch(worker: CyclotronV2Worker, timeoutMs = 2000): Pro
// ── Tests ────────────────────────────────────────────────────────────
-async function gaugeValueForQueue(queue: string): Promise<number | null> {
- const metric = await register.getSingleMetricAsString('cdp_cyclotron_v2_queue_depth')
+async function gaugeValueForQueue(queue: string, metricName = 'cdp_cyclotron_v2_queue_depth'): Promise<number | null> {
+ const metric = await register.getSingleMetricAsString(metricName)
const line = metric.split('\n').find((l) => l.includes(`queue="${queue}"`))
return line ? Number(line.trim().split(' ').pop()) : null
}
@@ -2757,19 +2757,28 @@ describe('Cyclotron V2', () => {
it('measureQueueDepths reports 0 for a queue that drains', async () => {
// GROUP BY returns no row for an empty queue. Without the zero write the gauge
// keeps the last depth, and a depth alert then fires on an idle queue.
+ const ageMetric = 'cdp_cyclotron_v2_queue_oldest_due_age_ms'
const jobId = uuidv7()
- await insertRawJob({ id: jobId, queue_name: 'queue-drains', status: 'available' })
+ await insertRawJob({
+ id: jobId,
+ queue_name: 'queue-drains',
+ status: 'available',
+ scheduled: new Date(Date.now() - 30_000),
+ })
const janitor = createJanitor({ stallTimeoutMs: 60_000 })
const before = await janitor.runOnce()
expect(before.depths.get('queue-drains')).toBe(1)
expect(await gaugeValueForQueue('queue-drains')).toBe(1)
+ expect(await gaugeValueForQueue('queue-drains', ageMetric)).toBeGreaterThanOrEqual(30_000)
+ expect(await gaugeValueForQueue('queue-drains', ageMetric)).toBeLessThan(60_000)
await assertPool.query('DELETE FROM cyclotron_jobs WHERE id = $1', [jobId])
await janitor.runOnce()
await janitor.stop()
expect(await gaugeValueForQueue('queue-drains')).toBe(0)
+ expect(await gaugeValueForQueue('queue-drains', ageMetric)).toBe(0)
})
it('measureQueueDepths returns correct counts per queue', async () => {
diff --git a/nodejs/src/cdp/services/cyclotron-v2/janitor.ts b/nodejs/src/cdp/services/cyclotron-v2/janitor.ts
index cd15867705f0..aad8bf55d709 100644
--- a/nodejs/src/cdp/services/cyclotron-v2/janitor.ts
+++ b/nodejs/src/cdp/services/cyclotron-v2/janitor.ts
@@ -81,6 +81,12 @@ const queueDepthGauge = new Gauge({
labelNames: ['queue'],
})
+const queueOldestDueAgeGauge = new Gauge({
+ name: 'cdp_cyclotron_v2_queue_oldest_due_age_ms',
+ help: 'Milliseconds since the oldest available job per queue became due (its scheduled time). 0 when the queue has no due jobs.',
+ labelNames: ['queue'],
+})
+
interface PoisonRow {
id: string
team_id: number
@@ -525,8 +531,9 @@ export class CyclotronV2Janitor {
}
async measureQueueDepths(): Promise<Map<string, number>> {
- const result = await this.pool.query<{ queue_name: string; count: string }>(
- `SELECT queue_name, COUNT(*) as count
+ const result = await this.pool.query<{ queue_name: string; count: string; oldest_due_age_ms: number }>(
+ `SELECT queue_name, COUNT(*) as count,
+ EXTRACT(EPOCH FROM NOW() - MIN(scheduled))::float8 * 1000 as oldest_due_age_ms
FROM cyclotron_jobs
WHERE status = 'available' AND scheduled <= NOW()
GROUP BY queue_name`
@@ -538,6 +545,7 @@ export class CyclotronV2Janitor {
depths.set(row.queue_name, count)
this.seenQueues.add(row.queue_name)
queueDepthGauge.labels({ queue: row.queue_name }).set(count)
+ queueOldestDueAgeGauge.labels({ queue: row.queue_name }).set(row.oldest_due_age_ms)
}
// GROUP BY returns no row for a queue that is empty. Without the write below,
@@ -548,6 +556,7 @@ export class CyclotronV2Janitor {
for (const queue of this.seenQueues) {
if (!depths.has(queue)) {
queueDepthGauge.labels({ queue }).set(0)
+ queueOldestDueAgeGauge.labels({ queue }).set(0)
}
}
diff --git a/nodejs/src/cdp/services/hog-executor-async.service.ts b/nodejs/src/cdp/services/hog-executor-async.service.ts
index c914db4cba34..8a06dcfcf045 100644
--- a/nodejs/src/cdp/services/hog-executor-async.service.ts
+++ b/nodejs/src/cdp/services/hog-executor-async.service.ts
@@ -1,5 +1,4 @@
import { DateTime } from 'luxon'
-import { Counter } from 'prom-client'
import { instrumented } from '~/common/tracing/tracing-utils'
import { parseJSON } from '~/common/utils/json-parse'
@@ -31,7 +30,7 @@ import { mergeSecretHeaders, resolveSecretHeaders } from '../utils/secret-header
import { resolveStandardWebhooksKey, signStandardWebhooksRequest } from '../utils/standard-webhooks'
import { HogExecutorExecuteOptions, HogExecutorPreviousResult, HogExecutorService } from './hog-executor.service'
import { HogInputsService } from './hog-inputs.service'
-import { EMAIL_QUEUE_PRIORITY, getEmailQueuePriorityClass } from './messaging/email-priority'
+import { EMAIL_QUEUE_PRIORITY, cdpEmailQueuedTotal, getEmailQueuePriorityClass } from './messaging/email-priority'
import { EmailService } from './messaging/email.service'
import { PushNotificationService } from './messaging/push-notification.service'
import { RecipientTokensService, preferencesTokenSourceForInvocation } from './messaging/recipient-tokens.service'
@@ -44,12 +43,6 @@ import {
selfLoopGuardCounter,
} from './self-loop-guard'
-const cdpEmailQueuedTotal = new Counter({
- name: 'cdp_email_queued_total',
- help: 'Total emails routed to the dedicated email queue',
- labelNames: ['priority_class'] as const,
-})
-
export interface HogExecutorAsyncConfig {
googleAdwordsDeveloperToken: string
fetchRetries: number
diff --git a/nodejs/src/cdp/services/hogflows/transactional-email.ts b/nodejs/src/cdp/services/hogflows/transactional-email.ts
index 5e92bc1f38c2..c24b3b4bf6ba 100644
--- a/nodejs/src/cdp/services/hogflows/transactional-email.ts
+++ b/nodejs/src/cdp/services/hogflows/transactional-email.ts
@@ -1,4 +1,4 @@
-import { Counter } from 'prom-client'
+import { Counter, Histogram } from 'prom-client'
import { z } from 'zod'
// What Django sends to run a transactional email's workflow once.
@@ -35,3 +35,53 @@ export const transactionalEmailRequestsTotal = new Counter({
help: 'Requests from Django to run a transactional email, by outcome. Every outcome except `queued` means this request queued no email.',
labelNames: ['outcome'],
})
+
+export const TRANSACTIONAL_EMAIL_STAGES = [
+ 'auth',
+ 'validate',
+ 'load',
+ 'build',
+ 'idempotency',
+ 'checks',
+ 'queue_insert',
+ 'metrics_flush',
+] as const
+export type TransactionalEmailStage = (typeof TRANSACTIONAL_EMAIL_STAGES)[number]
+
+export const transactionalEmailStageDurationMs = new Histogram({
+ name: 'cdp_transactional_email_stage_duration_ms',
+ help: 'Milliseconds one transactional email request spends in each stage: auth (the JWT), validate (the body), load (the team and the workflow), build (the run and its idempotency fingerprint), idempotency (every Idempotency-Key call to Valkey), checks (suspension, pause, suppression, quota and MX), queue_insert (the email queue insert), metrics_flush (the result, status and app metric writes), and total. A request records only the stages it reached, so a refusal shows in the stage that refused it.',
+ labelNames: ['stage'],
+ buckets: [1, 2.5, 5, 10, 25, 50, 100, 250, 500, 1_000, 2_500, 5_000, 10_000],
+})
+
+// The stages of a request run back to back, so starting a stage ends the one before it. finish() records each stage
+// once, with its summed time, so a stage entered twice (idempotency) still counts as one observation per request.
+export class TransactionalEmailStageTimer {
+ private readonly startedAt = performance.now()
+ private current: { stage: TransactionalEmailStage; startedAt: number } | null = null
+ private readonly durations = new Map<TransactionalEmailStage, number>()
+
+ start(stage: TransactionalEmailStage): void {
+ const now = performance.now()
+ this.endCurrent(now)
+ this.current = { stage, startedAt: now }
+ }
+
+ finish(): void {
+ const now = performance.now()
+ this.endCurrent(now)
+ for (const [stage, durationMs] of this.durations) {
+ transactionalEmailStageDurationMs.labels({ stage }).observe(durationMs)
+ }
+ transactionalEmailStageDurationMs.labels({ stage: 'total' }).observe(now - this.startedAt)
+ }
+
+ private endCurrent(now: number): void {
+ if (this.current) {
+ const { stage, startedAt } = this.current
+ this.durations.set(stage, (this.durations.get(stage) ?? 0) + now - startedAt)
+ this.current = null
+ }
+ }
+}
diff --git a/nodejs/src/cdp/services/job-queue/job-queue-postgres-v2.ts b/nodejs/src/cdp/services/job-queue/job-queue-postgres-v2.ts
index eb215a386d5b..1d7822536967 100644
--- a/nodejs/src/cdp/services/job-queue/job-queue-postgres-v2.ts
+++ b/nodejs/src/cdp/services/job-queue/job-queue-postgres-v2.ts
@@ -356,6 +356,8 @@ export function v2JobToInvocation(job: CyclotronV2DequeuedJob): CyclotronJobInvo
queue: job.queueName as CyclotronJobQueueKind,
queuePriority: job.priority,
queueScheduledAt: job.scheduled ?? undefined,
+ queueCreatedAt: job.created,
+ queueDueAt: job.scheduled ?? undefined,
queueMetadata: parsed.queueMetadata ?? undefined,
queueParameters: parsed.queueParameters ?? undefined,
state: parsed.state,
diff --git a/nodejs/src/cdp/services/messaging/email-priority.ts b/nodejs/src/cdp/services/messaging/email-priority.ts
index 0b6c0550830f..b4f3353e94a7 100644
--- a/nodejs/src/cdp/services/messaging/email-priority.ts
+++ b/nodejs/src/cdp/services/messaging/email-priority.ts
@@ -1,3 +1,5 @@
+import { Counter } from 'prom-client'
+
import { HogFunctionType } from '../../types'
/**
@@ -36,3 +38,9 @@ export const EMAIL_QUEUE_PRIORITY: Record<EmailQueuePriorityClass, number> = {
export function getEmailQueuePriorityClass(metadata: HogFunctionType['metadata']): EmailQueuePriorityClass {
return metadata?.message_category_type === 'transactional' ? 'fast' : 'bulk'
}
+
+export const cdpEmailQueuedTotal = new Counter({
+ name: 'cdp_email_queued_total',
+ help: 'Total emails routed to the dedicated email queue',
+ labelNames: ['priority_class'] as const,
+})
diff --git a/nodejs/src/cdp/services/messaging/email.service.test.ts b/nodejs/src/cdp/services/messaging/email.service.test.ts
index fcfade0db311..390512f5959d 100644
--- a/nodejs/src/cdp/services/messaging/email.service.test.ts
+++ b/nodejs/src/cdp/services/messaging/email.service.test.ts
@@ -1,6 +1,8 @@
import { mockFetch } from '~/tests/helpers/mocks/request.mock'
import { MessageRejected, SendingPausedException, TooManyRequestsException } from '@aws-sdk/client-sesv2'
+import { DateTime } from 'luxon'
+import { register } from 'prom-client'
import { createExampleInvocation, insertIntegration } from '~/cdp/_tests/fixtures'
import { EmailStatusOutput } from '~/cdp/outputs/outputs'
@@ -46,6 +48,18 @@ class ThrottlingException extends Error {
}
}
+async function metricValue(name: string, labels: Record<string, string>, metricName = name): Promise<number> {
+ const metric = await register.getSingleMetric(name)!.get()
+ const values: { labels: Partial<Record<string, string | number>>; value: number; metricName?: string }[] =
+ metric.values
+ const sample = values.find(
+ (value) =>
+ (value.metricName ?? name) === metricName &&
+ Object.entries(labels).every(([key, labelValue]) => value.labels[key] === labelValue)
+ )
+ return sample?.value ?? 0
+}
+
describe('sanitizeEmailSubject', () => {
it.each([
['passes through normal text', 'Hello World', 'Hello World'],
@@ -493,6 +507,29 @@ describe('EmailService', () => {
)
})
+ it.each<[string, () => Error, string]>([
+ [
+ 'TooManyRequestsException',
+ () => new TooManyRequestsException({ $metadata: {}, message: 'Too many requests' }),
+ 'throttled',
+ ],
+ ['ThrottlingException', () => new ThrottlingException('Rate exceeded'), 'throttled'],
+ [
+ 'SendingPausedException',
+ () => new SendingPausedException({ $metadata: {}, message: 'Sending paused' }),
+ 'SendingPausedException',
+ ],
+ ['an unknown error', () => new Error('connect ECONNRESET for [email protected]'), 'other'],
+ ])('counts an SES send that fails with %s as %s', async (_name, makeError, expectedResult) => {
+ const labels = { result: expectedResult }
+ const before = await metricValue('cdp_ses_send_results_total', labels)
+ sendEmailSpy.mockRejectedValueOnce(makeError())
+
+ await service.executeSendEmail(invocation)
+
+ expect(await metricValue('cdp_ses_send_results_total', labels)).toBe(before + 1)
+ })
+
it('still fails the job for non-throttle SES errors', async () => {
sendEmailSpy.mockRejectedValueOnce(
new MessageRejected({ $metadata: {}, message: 'something else broke' })
@@ -509,6 +546,45 @@ describe('EmailService', () => {
})
})
+ describe('queue-to-send latency', () => {
+ // The job was created a minute ago and rescheduled after a throttle, so it became due again 5 seconds ago.
+ // Running the email step clones the invocation, which resets queueScheduledAt before the send.
+ it.each([
+ ['workflow', 'bulk', 'its rescheduled time', (): void => {}, 5_000, 60_000],
+ [
+ 'transactional_api',
+ 'fast',
+ 'its creation',
+ (): void => {
+ invocation.state.globals.event.event = '$transactional_email'
+ invocation.hogFunction.metadata = { message_category_type: 'transactional' }
+ },
+ 60_000,
+ Infinity,
+ ],
+ ])(
+ 'observes the wait of a retried %s send on the %s lane from %s',
+ async (source, priorityClass, _from, setUp, minWaitMs, maxWaitMs) => {
+ setUp()
+ invocation.queue = 'email'
+ invocation.queueCreatedAt = DateTime.utc().minus({ seconds: 60 })
+ invocation.queueDueAt = DateTime.utc().minus({ seconds: 5 })
+ invocation.queueScheduledAt = undefined
+ const name = 'cdp_email_queue_to_send_ms'
+ const labels = { source, priority_class: priorityClass }
+ const countBefore = await metricValue(name, labels, `${name}_count`)
+ const sumBefore = await metricValue(name, labels, `${name}_sum`)
+
+ await service.executeSendEmail(invocation)
+
+ expect(await metricValue(name, labels, `${name}_count`)).toBe(countBefore + 1)
+ const waitMs = (await metricValue(name, labels, `${name}_sum`)) - sumBefore
+ expect(waitMs).toBeGreaterThanOrEqual(minWaitMs)
+ expect(waitMs).toBeLessThan(maxWaitMs)
+ }
+ )
+ })
+
describe('workflow sending rate limit', () => {
let claimOrReserve: jest.Mock
let limitedService: EmailService
diff --git a/nodejs/src/cdp/services/messaging/email.service.ts b/nodejs/src/cdp/services/messaging/email.service.ts
index cb74135e7723..d99cf20fa3a8 100644
--- a/nodejs/src/cdp/services/messaging/email.service.ts
+++ b/nodejs/src/cdp/services/messaging/email.service.ts
@@ -16,10 +16,12 @@ import { createAddLogFunction, logEntry } from '~/cdp/utils'
import { createInvocationResult } from '~/cdp/utils/invocation-utils'
import { logger } from '~/common/utils/logger'
+import { TRANSACTIONAL_EMAIL_EVENT } from '../hogflows/hogflow-sensitivity'
import { IntegrationManagerService } from '../managers/integration-manager.service'
import { RecipientManagerRecipient, RecipientsManagerService } from '../managers/recipients-manager.service'
import { TeamWorkflowsConfigService } from '../managers/team-workflows-config.service'
import { RateLimiterService } from '../rate-limiter/rate-limiter.service'
+import { getEmailQueuePriorityClass } from './email-priority'
import { selectEmailSenderIntegrationId } from './email-sender-selection'
import { EmailStatusService, isTransactionalEmailSend } from './email-status.service'
import { EmailSuppressionService, SuppressionCheckOptions } from './email-suppression.service'
@@ -80,6 +82,59 @@ function isSesThrottleError(error: unknown): error is Error & { name: SesThrottl
return error instanceof Error && (SES_THROTTLE_ERROR_NAMES as readonly string[]).includes(error.name)
}
+// Every other error name reads as `other`, so an unexpected SDK error cannot add a series.
+const SES_SEND_ERROR_RESULTS = [
+ 'MessageRejected',
+ 'MailFromDomainNotVerifiedException',
+ 'AccountSuspendedException',
+ 'SendingPausedException',
+ 'LimitExceededException',
+ 'NotFoundException',
+ 'BadRequestException',
+ 'TimeoutError',
+] as const
+type SesSendResult = 'success' | 'throttled' | (typeof SES_SEND_ERROR_RESULTS)[number] | 'other'
+
+function classifySesSendError(error: unknown): SesSendResult {
+ if (isSesThrottleError(error)) {
+ return 'throttled'
+ }
+ const name = error instanceof Error ? error.name : undefined
+ return SES_SEND_ERROR_RESULTS.find((known) => known === name) ?? 'other'
+}
+
+const sesSendResultsTotal = new Counter({
+ name: 'cdp_ses_send_results_total',
+ help: 'SES SendEmail calls by result: success, throttled (rescheduled, with the error code in cdp_ses_throttle_responses_total), a known SES error name, or other.',
+ labelNames: ['result'],
+})
+
+const sesSendDurationMs = new Histogram({
+ name: 'cdp_ses_send_duration_ms',
+ help: 'Duration of the SES SendEmail call in milliseconds, by whether SES accepted the email (success or error).',
+ labelNames: ['result'],
+ buckets: [25, 50, 100, 250, 500, 1_000, 2_500, 5_000, 10_000, 30_000],
+})
+
+const emailQueueToSendMs = new Histogram({
+ name: 'cdp_email_queue_to_send_ms',
+ help: 'Milliseconds an email waits on the email queue before its handoff to SES. For source="transactional_api" the wait starts when cdp-api created the job, so it is the time from the API\'s 202 to SES and includes every throttled or rate-limited retry. For source="workflow" the wait starts at the job\'s scheduled time: when the job joined the email queue, or when a delayed or rescheduled send became due, so a retried send measures only its last wait. priority_class is the email queue lane.',
+ labelNames: ['source', 'priority_class'],
+ buckets: [100, 250, 500, 1_000, 2_500, 5_000, 10_000, 30_000, 60_000, 120_000, 300_000],
+})
+
+type EmailQueueWait = { source: 'transactional_api' | 'workflow'; startedAt: DateTime }
+
+// A transactional API job is created on the email queue, so its creation time is when the caller got its 202. A
+// workflow job can join the email queue long after it was created, so its wait starts at the time it became due.
+// Both come from fields a clone keeps: running the email step clones the invocation and resets queueScheduledAt.
+function emailQueueWait(invocation: CyclotronJobInvocationHogFunction): EmailQueueWait | undefined {
+ const source =
+ invocation.state.globals.event?.event === TRANSACTIONAL_EMAIL_EVENT ? 'transactional_api' : 'workflow'
+ const startedAt = source === 'transactional_api' ? invocation.queueCreatedAt : invocation.queueDueAt
+ return startedAt ? { source, startedAt } : undefined
+}
+
/**
* Tagged error signalling that SES rejected the send for a transient,
* rate-limit-shaped reason. The caller schedules a retry instead of failing
@@ -404,6 +459,9 @@ export class EmailService {
)
const addLog = createAddLogFunction(result.logs)
+ // Read from the incoming invocation: result.invocation is a clone, and a clone drops the scheduled time.
+ const queueWait = invocation.queue === 'email' && !isTest ? emailQueueWait(invocation) : undefined
+
const params = invocation.queueParameters
const integrationId = selectEmailSenderIntegrationId(invocation.id, params.from)
const integration = await this.integrationManager.get(integrationId)
@@ -684,7 +742,7 @@ export class EmailService {
await this.sendEmailWithMaildev(result, sendParams, from, trackingEnabled, isTest)
break
case 'ses':
- await this.sendEmailWithSES(result, sendParams, from, trackingEnabled, isTest)
+ await this.sendEmailWithSES(result, sendParams, from, trackingEnabled, isTest, queueWait)
break
case 'unsupported':
@@ -1109,7 +1167,8 @@ export class EmailService {
params: CyclotronInvocationQueueParametersEmailType,
from: { email: string; name: string },
trackingEnabled: boolean,
- isTest = false
+ isTest = false,
+ queueWait?: EmailQueueWait
): Promise<void> {
if (!this.sesV2Client) {
throw new Error('SES is not configured - set SES_REGION and AWS credentials')
@@ -1220,12 +1279,26 @@ export class EmailService {
sendEmailParams.Destination!.BccAddresses = bccAddresses
}
+ if (queueWait) {
+ emailQueueToSendMs
+ .labels({
+ source: queueWait.source,
+ priority_class: getEmailQueuePriorityClass(result.invocation.hogFunction.metadata),
+ })
+ .observe(Math.max(0, Date.now() - queueWait.startedAt.toMillis()))
+ }
+
+ const sendStartedAt = performance.now()
try {
const response = await this.sesV2Client.send(new SendEmailCommand(sendEmailParams))
if (!response.MessageId) {
throw new Error('No messageId returned from SES')
}
+ sesSendDurationMs.labels({ result: 'success' }).observe(performance.now() - sendStartedAt)
+ sesSendResultsTotal.labels({ result: 'success' }).inc()
} catch (error: unknown) {
+ sesSendDurationMs.labels({ result: 'error' }).observe(performance.now() - sendStartedAt)
+ sesSendResultsTotal.labels({ result: classifySesSendError(error) }).inc()
if (isSesThrottleError(error)) {
sesThrottleResponsesTotal.inc({ error_code: error.name })
throw new SESThrottleError(error.name, pickThrottleRetryDelayMs(), error.message)
diff --git a/nodejs/src/cdp/types.ts b/nodejs/src/cdp/types.ts
index c22bc248ae0b..b59606543a4a 100644
--- a/nodejs/src/cdp/types.ts
+++ b/nodejs/src/cdp/types.ts
@@ -342,6 +342,10 @@ export type CyclotronJobInvocation = {
queuePriority: number
// When the invocation is scheduled to run
queueScheduledAt?: DateTime
+ // When the queue first stored the job. A reschedule moves queueScheduledAt but not this, and a clone keeps it.
+ queueCreatedAt?: DateTime
+ // The scheduled time this job had when it was dequeued. A clone resets queueScheduledAt but keeps this.
+ queueDueAt?: DateTime
// Metadata for the invocation - TODO: check when this gets cleared
queueMetadata?: Record<string, any> | null
// Where the invocation came from (kafka or postgres)
diff --git a/products/messaging/backend/facade/transactional_email.py b/products/messaging/backend/facade/transactional_email.py
index 4e5fe7ec755d..dbff0513fde0 100644
--- a/products/messaging/backend/facade/transactional_email.py
+++ b/products/messaging/backend/facade/transactional_email.py
@@ -9,12 +9,15 @@ from products.messaging.backend.facade.contracts import (
)
from products.messaging.backend.services.transactional_email import (
SENDER_ADDRESS_MAX_LENGTH,
+ TransactionalEmailStage,
check_transactional_update,
lock_transactional_workflow_id,
make_transactional,
+ observe_transactional_email_stage,
send_transactional_email,
sender_address_error,
sync_transactional_workflow,
+ transactional_email_stage,
)
from products.messaging.backend.services.transactional_email_status import get_email_status
@@ -25,11 +28,14 @@ __all__ = [
"EmailStatus",
"QueuedEmail",
"TransactionalEmailError",
+ "TransactionalEmailStage",
"check_transactional_update",
"get_email_status",
"lock_transactional_workflow_id",
"make_transactional",
+ "observe_transactional_email_stage",
"send_transactional_email",
"sender_address_error",
"sync_transactional_workflow",
+ "transactional_email_stage",
]
diff --git a/products/messaging/backend/presentation/views/message_templates.py b/products/messaging/backend/presentation/views/message_templates.py
index c7000d0069e1..3dce0c0caef7 100644
--- a/products/messaging/backend/presentation/views/message_templates.py
+++ b/products/messaging/backend/presentation/views/message_templates.py
@@ -1,7 +1,9 @@
+import time
from collections.abc import Sequence
+from contextlib import AbstractContextManager, nullcontext
from dataclasses import fields as dataclass_fields
from types import SimpleNamespace
-from typing import Any
+from typing import Any, cast
from django.db import transaction
@@ -59,12 +61,15 @@ from products.messaging.backend.facade.transactional_email import (
SENDER_ADDRESS_MAX_LENGTH,
DeliveryUnavailable,
TransactionalEmailError,
+ TransactionalEmailStage,
check_transactional_update,
lock_transactional_workflow_id,
make_transactional,
+ observe_transactional_email_stage,
send_transactional_email,
sender_address_error,
sync_transactional_workflow,
+ transactional_email_stage,
)
from products.messaging.backend.presentation.views.serializers import DesignOperationSerializer
from products.notifications.backend.facade.api import publish_resource_edited
@@ -537,13 +542,55 @@ class TransactionalEmailSendMixin:
team_id: int
organization: Any
user_access_control: Any
+ # The actions of the viewset that send an email. Only these record stage durations.
+ transactional_email_send_actions: tuple[str, ...] = ()
+ _validation_started_at: float | None = None
+
+ def _is_transactional_email_send(self) -> bool:
+ return getattr(self, "action", None) in self.transactional_email_send_actions
+
+ def _send_stage(self, stage: TransactionalEmailStage) -> AbstractContextManager[None]:
+ return transactional_email_stage(stage) if self._is_transactional_email_send() else nullcontext()
+
+ def dispatch(self, request: Any, *args: Any, **kwargs: Any) -> Any:
+ started_at = time.monotonic()
+ try:
+ return cast(Any, super()).dispatch(request, *args, **kwargs)
+ finally:
+ # A body that fails validation never reaches the action, so its validation time is recorded here.
+ self._end_request_validation()
+ if self._is_transactional_email_send():
+ observe_transactional_email_stage("total", time.monotonic() - started_at)
+
+ def perform_authentication(self, request: Any) -> None:
+ with self._send_stage("auth"):
+ cast(Any, super()).perform_authentication(request)
+
+ def check_permissions(self, request: Any) -> None:
+ with self._send_stage("permissions"):
+ cast(Any, super()).check_permissions(request)
+
+ def check_throttles(self, request: Any) -> None:
+ with self._send_stage("throttle"):
+ cast(Any, super()).check_throttles(request)
+
+ def initial(self, request: Any, *args: Any, **kwargs: Any) -> None:
+ cast(Any, super()).initial(request, *args, **kwargs)
+ # @validated_request parses and validates the body after initial() returns and before the action body runs.
+ self._validation_started_at = time.monotonic()
+
+ def _end_request_validation(self) -> None:
+ if self._validation_started_at is not None and self._is_transactional_email_send():
+ observe_transactional_email_stage("validation", time.monotonic() - self._validation_started_at)
+ self._validation_started_at = None
def _require_transactional_email_flag(self) -> None:
require_transactional_email_api(self.team)
def _queue_transactional_email(self, request: Any, template: MessageTemplateRow, *, route: str) -> Response:
- if template.transactional_hog_flow_id:
- self._require_workflow_editor(template.transactional_hog_flow_id)
+ with transactional_email_stage("workflow_access"):
+ if template.transactional_hog_flow_id:
+ self._require_workflow_editor(template.transactional_hog_flow_id)
data = request.validated_data
idempotency_key = request.headers.get("Idempotency-Key")
outcome: dict[str, Any] = {
@@ -568,7 +615,8 @@ class TransactionalEmailSendMixin:
outcome["outcome"] = "unavailable"
raise TransactionalEmailUnavailable() from None
finally:
- self._report_transactional_email_send(request, template, outcome)
+ with transactional_email_stage("usage_event"):
+ self._report_transactional_email_send(request, template, outcome)
return Response(
SendTransactionalEmailResponseSerializer(queued).data,
status=status.HTTP_202_ACCEPTED,
@@ -644,6 +692,7 @@ class MessageTemplatesViewSet(
authentication_classes = [ProjectSecretAPIKeyAuthentication]
# A project secret API key can only send. Template CRUD stays session/personal API key/OAuth only.
psak_allowed_actions = ["send"]
+ transactional_email_send_actions = ("send",)
serializer_class = MessageTemplateSerializer
@@ -850,8 +899,12 @@ class MessageTemplatesViewSet(
)
@action(detail=True, methods=["POST"], pagination_class=None)
def send(self, request: Any, *args: Any, **kwargs: Any) -> Response:
- self._require_transactional_email_flag()
- return self._queue_transactional_email(request, self.get_object(), route="template")
+ self._end_request_validation()
+ with transactional_email_stage("feature_flag"):
+ self._require_transactional_email_flag()
+ with transactional_email_stage("template_lookup"):
+ template = self.get_object()
+ return self._queue_transactional_email(request, template, route="template")
@validated_request(
MakeTransactionalSerializer,
@@ -910,6 +963,7 @@ class MessagingSendViewSet(TransactionalEmailSendMixin, TeamAndOrgViewSetMixin,
permission_classes = [IsAuthenticated]
authentication_classes = [ProjectSecretAPIKeyAuthentication]
psak_allowed_actions = ["email"]
+ transactional_email_send_actions = ("email",)
def get_throttles(self):
return transactional_email_throttles()
@@ -928,10 +982,13 @@ class MessagingSendViewSet(TransactionalEmailSendMixin, TeamAndOrgViewSetMixin,
)
@action(detail=False, methods=["POST"])
def email(self, request: Any, *args: Any, **kwargs: Any) -> Response:
- self._require_transactional_email_flag()
- try:
- template = get_template(self.team_id, request.validated_data["template"]["id"])
- except MessageTemplateMissing:
- raise NotFound()
- self.check_object_permissions(request, template)
+ self._end_request_validation()
+ with transactional_email_stage("feature_flag"):
+ self._require_transactional_email_flag()
+ with transactional_email_stage("template_lookup"):
+ try:
+ template = get_template(self.team_id, request.validated_data["template"]["id"])
+ except MessageTemplateMissing:
+ raise NotFound()
+ self.check_object_permissions(request, template)
return self._queue_transactional_email(request, template, route="send_email")
diff --git a/products/messaging/backend/services/transactional_email.py b/products/messaging/backend/services/transactional_email.py
index 938afdfa4c8d..804975351759 100644
--- a/products/messaging/backend/services/transactional_email.py
+++ b/products/messaging/backend/services/transactional_email.py
@@ -2,12 +2,17 @@
per send, so sends get the workflow's metrics, run history and versions."""
import re
-from typing import TYPE_CHECKING, Any
+import time
+from collections.abc import Iterator
+from contextlib import contextmanager
+from typing import Any, Literal
from uuid import UUID
from django.db import transaction
+import requests
import structlog
+from prometheus_client import Counter, Histogram
from posthog.models.integration import Integration
from posthog.models.team import Team
@@ -47,11 +52,78 @@ from products.workflows.backend.facade.transactional_emails import (
update_transactional_email_workflow,
)
-if TYPE_CHECKING:
- import requests
-
logger = structlog.get_logger(__name__)
+CDP_SEND_DURATION = Histogram(
+ "messaging_transactional_email_cdp_request_duration_seconds",
+ "Time for the CDP service to answer a request to queue one transactional email, including failed requests.",
+ buckets=(0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 7.5, 10, 15),
+)
+CDP_SEND_RESULTS = Counter(
+ "messaging_transactional_email_cdp_requests_total",
+ "Requests to the CDP service to queue one transactional email, by result: success (202), the HTTP status class "
+ "of any other answer, timeout, connection_error, or error.",
+ labelnames=["result"],
+)
+
+
+TransactionalEmailStage = Literal[
+ "auth",
+ "permissions",
+ "throttle",
+ "validation",
+ "feature_flag",
+ "template_lookup",
+ "workflow_access",
+ "workflow_lookup",
+ "cdp_call",
+ "usage_event",
+ "total",
+]
+STAGE_DURATION = Histogram(
+ "messaging_transactional_email_stage_duration_seconds",
+ "Time one transactional email send request spends in each stage, including requests that fail in that stage: "
+ "auth, permissions and throttle (DRF's checks), validation (the request body), feature_flag, template_lookup, "
+ "workflow_access (the caller's editor access to the workflow), workflow_lookup (the template and workflow "
+ "checks before the send), cdp_call (the same time as messaging_transactional_email_cdp_request_duration_seconds), "
+ "usage_event, and total (the DRF view from start to response).",
+ labelnames=["stage"],
+ buckets=(0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 15),
+)
+
+
+def observe_transactional_email_stage(stage: TransactionalEmailStage, seconds: float) -> None:
+ STAGE_DURATION.labels(stage=stage).observe(seconds)
+
+
+@contextmanager
+def transactional_email_stage(stage: TransactionalEmailStage) -> Iterator[None]:
+ started_at = time.monotonic()
+ try:
+ yield
+ finally:
+ observe_transactional_email_stage(stage, time.monotonic() - started_at)
+
+
+def _cdp_error_result(error: Exception) -> str:
+ # ConnectTimeout is both a Timeout and a ConnectionError, so the timeout check goes first.
+ if isinstance(error, requests.Timeout):
+ return "timeout"
+ if isinstance(error, requests.ConnectionError):
+ return "connection_error"
+ return "error"
+
+
+def _cdp_response_result(status_code: int) -> str:
+ return "success" if status_code == 202 else f"{status_code // 100}xx"
+
+
+def _observe_cdp_call(seconds: float) -> None:
+ # Both histograms get the same value, so the stage breakdown and the CDP request metric can't disagree.
+ CDP_SEND_DURATION.observe(seconds)
+ observe_transactional_email_stage("cdp_call", seconds)
+
+
# The CDP service names the reason it refused a send in the body's `error`. A retry of the same request gets the
# same refusal, so each of these reaches the caller as a 4xx and not as a retryable 503.
PIPELINE_REFUSALS: dict[str, tuple[type[TransactionalEmailError], str]] = {
@@ -314,7 +386,8 @@ def send_transactional_email(
if "{" in to or "}" in to:
raise InvalidRecipient("The recipient address can't contain { or }.")
- hog_flow_id = _transactional_workflow_id(team.id, template_id)
+ with transactional_email_stage("workflow_lookup"):
+ hog_flow_id = _transactional_workflow_id(team.id, template_id)
payload = {
"to": to,
# The recipient is the person the send is tracked against. Without a distinct_id, the address is one.
@@ -324,11 +397,16 @@ def send_transactional_email(
if idempotency_key is not None:
# The CDP service holds the key, because only it knows whether the email was queued.
payload["idempotency_key"] = idempotency_key
+ started_at = time.monotonic()
try:
response = run_transactional_workflow(team.id, str(hog_flow_id), payload, timeout=timeout)
except Exception as e:
+ _observe_cdp_call(time.monotonic() - started_at)
+ CDP_SEND_RESULTS.labels(result=_cdp_error_result(e)).inc()
logger.exception("transactional_email_queue_failed", team_id=team.id, template_id=str(template_id))
raise DeliveryUnavailable() from e
+ _observe_cdp_call(time.monotonic() - started_at)
+ CDP_SEND_RESULTS.labels(result=_cdp_response_result(response.status_code)).inc()
if response.status_code != 202:
refusal = PIPELINE_REFUSALS.get(_refusal_code(response) or "")
@@ -347,7 +425,7 @@ def send_transactional_email(
return QueuedEmail(message_id=str(body["invocation_id"]), replayed=bool(body.get("replayed", False)))
-def _refusal_code(response: "requests.Response") -> str | None:
+def _refusal_code(response: requests.Response) -> str | None:
try:
body = response.json()
except ValueError:
diff --git a/products/messaging/backend/tests/api/test_transactional_email.py b/products/messaging/backend/tests/api/test_transactional_email.py
index 786ad7893242..03ae73b8f81a 100644
--- a/products/messaging/backend/tests/api/test_transactional_email.py
+++ b/products/messaging/backend/tests/api/test_transactional_email.py
@@ -1,4 +1,4 @@
-from typing import Any
+from typing import Any, get_args
from uuid import uuid4
import pytest
@@ -7,6 +7,7 @@ from unittest.mock import MagicMock, patch
import requests
from parameterized import parameterized
+from prometheus_client import REGISTRY
from rest_framework import status
from rest_framework.test import APIClient
@@ -19,10 +20,18 @@ from posthog.models.utils import generate_random_token_secret
from products.access_control.backend.facade.testing import create_access_control
from products.messaging.backend.models.message_category import MessageCategory, MessageCategoryType
from products.messaging.backend.models.message_template import MessageTemplate
-from products.messaging.backend.services.transactional_email import make_transactional
+from products.messaging.backend.services.transactional_email import TransactionalEmailStage, make_transactional
from products.workflows.backend.facade.api import set_default_email_sender_if_unset
+def _stage_counts() -> dict[str, float]:
+ return {
+ stage: REGISTRY.get_sample_value("messaging_transactional_email_stage_duration_seconds_count", {"stage": stage})
+ or 0
+ for stage in get_args(TransactionalEmailStage)
+ }
+
+
@patch("products.messaging.backend.presentation.views.message_templates.posthoganalytics.capture")
@patch(
"products.messaging.backend.presentation.views.message_templates.posthoganalytics.feature_enabled",
@@ -102,10 +111,16 @@ class TestTransactionalEmail(APIBaseTest):
if identifiers
else {k: v for k, v in self.body.items() if k != "identifiers"}
)
+ client = self._psak_client(["hog_flow:write"])
+ stages_before = _stage_counts()
- response = self._send(self._psak_client(["hog_flow:write"]), route, body)
+ response = self._send(client, route, body)
assert response.status_code == status.HTTP_202_ACCEPTED, response.json()
+ stages_after = _stage_counts()
+ assert {stage: stages_after[stage] - stages_before[stage] for stage in stages_after} == dict.fromkeys(
+ stages_after, 1
+ )
assert response.json() == {"message_id": "invocation-1", "replayed": False}
assert capture.call_args.kwargs["properties"]["route"] == route
assert mock_post.call_args.args[0].endswith(
@@ -200,20 +215,27 @@ class TestTransactionalEmail(APIBaseTest):
@parameterized.expand(
[
- ("timeout", requests.ReadTimeout()),
- ("cdp_api_error", MagicMock(status_code=500)),
+ ("timeout", requests.ReadTimeout(), "timeout"),
+ ("connect_timeout", requests.ConnectTimeout(), "timeout"),
+ ("connection_error", requests.ConnectionError(), "connection_error"),
+ ("cdp_api_error", MagicMock(status_code=500), "5xx"),
]
)
- def test_a_send_the_pipeline_does_not_confirm_is_unavailable(self, mock_post, _flag, _capture, _name, outcome):
+ def test_a_send_the_pipeline_does_not_confirm_is_unavailable(
+ self, mock_post, _flag, _capture, _name, outcome, expected_result
+ ):
if isinstance(outcome, Exception):
mock_post.side_effect = outcome
else:
mock_post.return_value = outcome
+ metric = "messaging_transactional_email_cdp_requests_total"
+ before = REGISTRY.get_sample_value(metric, {"result": expected_result}) or 0
response = self.client.post(self.url, self.body, format="json")
assert response.status_code == status.HTTP_503_SERVICE_UNAVAILABLE
assert response.json()["code"] == "delivery_unavailable"
+ assert REGISTRY.get_sample_value(metric, {"result": expected_result}) == before + 1
@parameterized.expand(
[