:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(cdp): add transactional email latency and SES send metrics (#125066)
Harley Alexander59d675c0aa7c
diff --git a/nodejs/src/cdp/cdp-api.serial.test.ts b/nodejs/src/cdp/cdp-api.serial.test.tsindex 419637585c47..a3156c0f3e87 100644--- a/nodejs/src/cdp/cdp-api.serial.test.ts+++ b/nodejs/src/cdp/cdp-api.serial.test.ts@@ -36,7 +36,10 @@ import { CdpApi } from './cdp-api'import { CdpConsumerBaseDeps } from './consumers/cdp-base.consumer'import { posthogFilterOutPlugin } from './legacy-plugins/_transformations/posthog-filter-out-plugin/template'import { IdempotencyKeys, createIdempotencyValkeyPool } from './services/hogflows/idempotency-keys'-import { transactionalEmailRequestsTotal } from './services/hogflows/transactional-email'+import {+ transactionalEmailRequestsTotal,+ transactionalEmailStageDurationMs,+} from './services/hogflows/transactional-email'import { BASE_REDIS_KEY, HogWatcherState } from './services/monitoring/hog-watcher.service'import { compileHog } from './templates/compiler'import { HogFunctionInvocationGlobals, HogFunctionType } from './types'@@ -2141,6 +2144,18 @@ describe('CDP API', () => {// The alerts on this endpoint key on these outcome labels.const outcomeCount = async (outcome: string): Promise<number> =>(await transactionalEmailRequestsTotal.get()).values.find((v) => v.labels.outcome === outcome)?.value ?? 0+ const stageCounts = async (): Promise<Record<string, number>> =>+ Object.fromEntries(+ (await transactionalEmailStageDurationMs.get()).values+ .filter((v) => v.metricName === 'cdp_transactional_email_stage_duration_ms_count')+ .map((v) => [String(v.labels.stage), v.value])+ )+ const stageCountChanges = (before: Record<string, number>, after: Record<string, number>) =>+ Object.fromEntries(+ Object.entries(after)+ .map(([stage, count]) => [stage, count - (before[stage] ?? 0)])+ .filter(([, change]) => change !== 0)+ )const send = (flowId: string, headers: Record<string, string>, payload: Record<string, unknown> = body) =>supertest(app)@@ -2170,11 +2185,23 @@ describe('CDP API', () => {it('queues one run on the email fast lane with the recipient and the data', async () => {const queueAppMetric = jest.spyOn(api['hogFunctionMonitoringService'], 'queueAppMetric')const queuedBefore = await outcomeCount('queued')+ const stagesBefore = await stageCounts()const res = await send(hogFlow.id, { Authorization: `Bearer ${mintToken(team.id, hogFlow.id)}` })expect(res.status).toEqual(202)expect(await outcomeCount('queued')).toEqual(queuedBefore + 1)+ // Without an Idempotency-Key the request makes no idempotency call.+ expect(stageCountChanges(stagesBefore, await stageCounts())).toEqual({+ auth: 1,+ validate: 1,+ load: 1,+ build: 1,+ checks: 1,+ queue_insert: 1,+ metrics_flush: 1,+ total: 1,+ })expect(mockQueueInvocations).toHaveBeenCalledTimes(1)const [invocation] = mockQueueInvocations.mock.calls[0][0]expect(invocation).toMatchObject({@@ -2337,6 +2364,7 @@ describe('CDP API', () => {] as const)('refuses a send the email worker would skip because %s', async (_desc, code, refuse) => {refuse()const refusedBefore = await outcomeCount(code)+ const stagesBefore = await stageCounts()const keyed = { ...body, idempotency_key: `key-${new UUIDT().toString()}` }const headers = { Authorization: `Bearer ${mintToken(team.id, hogFlow.id)}` }@@ -2345,6 +2373,16 @@ describe('CDP API', () => {expect(refused.status).toEqual(422)expect(refused.body).toEqual({ error: code })expect(await outcomeCount(code)).toEqual(refusedBefore + 1)+ // The reserve and the release of the key count once, as one idempotency observation.+ expect(stageCountChanges(stagesBefore, await stageCounts())).toEqual({+ auth: 1,+ validate: 1,+ load: 1,+ build: 1,+ idempotency: 1,+ checks: 1,+ total: 1,+ })expect(mockQueueInvocations).not.toHaveBeenCalled()expect(emailStatusRows()).toEqual([])diff --git a/nodejs/src/cdp/cdp-api.ts b/nodejs/src/cdp/cdp-api.tsindex c08af8df2435..ed3459b1cd62 100644--- a/nodejs/src/cdp/cdp-api.ts+++ b/nodejs/src/cdp/cdp-api.ts@@ -56,6 +56,7 @@ import { WorkflowStepResumeSchema } from './services/hogflows/step-resume.servicimport {TransactionalEmailOutcome,TransactionalEmailRequestSchema,+ TransactionalEmailStageTimer,transactionalEmailRequestsTotal,} from './services/hogflows/transactional-email'import { InvocationResultsService } from './services/invocation-results.service'@@ -63,7 +64,7 @@ import { JobQueue } from './services/job-queue/job-queue.interface'import { GroupsManagerService } from './services/managers/groups-manager.service'import { HogFunctionManagerService } from './services/managers/hog-function-manager.service'import { TeamWorkflowsConfigService } from './services/managers/team-workflows-config.service'-import { EMAIL_QUEUE_PRIORITY } from './services/messaging/email-priority'+import { EMAIL_QUEUE_PRIORITY, EmailQueuePriorityClass, cdpEmailQueuedTotal } from './services/messaging/email-priority'import { EmailStatusService } from './services/messaging/email-status.service'import { EmailSuppressionService } from './services/messaging/email-suppression.service'import { EmailTrackingService } from './services/messaging/email-tracking.service'@@ -1159,11 +1160,13 @@ export class CdpApi {// Auth: a per-call JWT minted by Django on its own audience and key, pinned to the team and the workflow,// never the fleet-wide internal secret.private postTransactionalEmail = async (req: ModifiedRequest, res: express.Response): Promise<any> => {+ const stages = new TransactionalEmailStageTimer()const reply = (outcome: TransactionalEmailOutcome, status: number, body: Record<string, unknown>) => {transactionalEmailRequestsTotal.inc({ outcome })return res.status(status).json(body)}try {+ stages.start('auth')if (!this.transactionalEmailJwt.enabled) {return reply('not_configured', 503, {error: 'Transactional email auth not configured (WORKFLOWS_TRANSACTIONAL_EMAIL_JWT_SECRET unset)',@@ -1173,12 +1176,14 @@ export class CdpApi {transactionalEmailRequestsTotal.inc({ outcome: 'unauthorized' })return}+ stages.start('validate')const parsed = TransactionalEmailRequestSchema.safeParse(req.body)if (!parsed.success) {return reply('invalid_request', 400, { error: parsed.error.message })}const { to, distinct_id: distinctId, variables, idempotency_key: idempotencyKey } = parsed.data+ stages.start('load')const team = await this.deps.teamManager.getTeam(parseInt(req.params.team_id)).catch(() => null)if (!team) {return reply('not_found', 404, { error: 'Team not found' })@@ -1191,6 +1196,7 @@ export class CdpApi {return reply('not_sendable', 400, { error: 'not_sendable' })}+ stages.start('build')const event: HogFunctionInvocationGlobals['event'] = {uuid: new UUIDT().toString(),event: TRANSACTIONAL_EMAIL_EVENT,@@ -1215,14 +1221,16 @@ export class CdpApi {const invocation = createHogFlowInvocation(triggerGlobals, hogFlow, filterGlobals)// Straight to the email queue: the flow's only step is the email, so a hop through the// workflow queue would only add latency to a password reset or a verification code.+ const priorityClass: EmailQueuePriorityClass = 'fast'invocation.queue = 'email'- invocation.queuePriority = EMAIL_QUEUE_PRIORITY.fast+ invocation.queuePriority = EMAIL_QUEUE_PRIORITY[priorityClass]const idempotencyKeys = this.transactionalEmailIdempotencyKeysconst fingerprint = IdempotencyKeys.fingerprint([hogFlow.id, to, distinctId, variables])if (idempotencyKey) {// Checked before the quota, so a retry of a send that went out gets its answer even// when the team has hit its quota since.+ stages.start('idempotency')const reservation = await idempotencyKeys.reserve(team.id, idempotencyKey, fingerprint, invocation.id)switch (reservation.status) {case 'replayed':@@ -1240,13 +1248,16 @@ export class CdpApi {}}try {+ stages.start('checks')const refusal = await this.refuseTransactionalEmail(team.id, hogFlow, to)if (refusal) {if (idempotencyKey) {+ stages.start('idempotency')await idempotencyKeys.release(team.id, idempotencyKey, invocation.id)}return reply(refusal.outcome, refusal.status, { error: refusal.outcome })}+ stages.start('queue_insert')this.invocationResultsService.invocationResultsRowsService.queueLifecycleRow(invocation, 'running')try {await this.hogflowQueue.queueInvocations([invocation])@@ -1254,16 +1265,20 @@ export class CdpApi {this.invocationResultsService.invocationResultsRowsService.dropQueuedRowsFor([invocation.id])throw error}+ cdpEmailQueuedTotal.labels(priorityClass).inc()} catch (error) {// Nothing was queued, so the caller's retry with the same key must be able to send.if (idempotencyKey) {+ stages.start('idempotency')await idempotencyKeys.release(team.id, idempotencyKey, invocation.id)}throw error}if (idempotencyKey) {+ stages.start('idempotency')await idempotencyKeys.markQueued(team.id, idempotencyKey, fingerprint, invocation.id)}+ stages.start('metrics_flush')this.emailStatusService.queue({ teamId: team.id, messageId: invocation.id, status: 'queued' })// Flushed together, and bounded, so the best-effort status row can't hold up the 202.await Promise.all([@@ -1290,6 +1305,8 @@ export class CdpApi {error: e instanceof Error ? e.message : String(e),})return reply('error', 500, { error: e instanceof Error ? e.message : String(e) })+ } finally {+ stages.finish()}}diff --git a/nodejs/src/cdp/services/cyclotron-v2/cyclotron-v2.test.ts b/nodejs/src/cdp/services/cyclotron-v2/cyclotron-v2.test.tsindex 188fb18c0c2b..faf24792b7ea 100644--- a/nodejs/src/cdp/services/cyclotron-v2/cyclotron-v2.test.ts+++ b/nodejs/src/cdp/services/cyclotron-v2/cyclotron-v2.test.ts@@ -170,8 +170,8 @@ async function dequeueOneBatch(worker: CyclotronV2Worker, timeoutMs = 2000): Pro// ── Tests ────────────────────────────────────────────────────────────-async function gaugeValueForQueue(queue: string): Promise<number | null> {- const metric = await register.getSingleMetricAsString('cdp_cyclotron_v2_queue_depth')+async function gaugeValueForQueue(queue: string, metricName = 'cdp_cyclotron_v2_queue_depth'): Promise<number | null> {+ const metric = await register.getSingleMetricAsString(metricName)const line = metric.split('\n').find((l) => l.includes(`queue="${queue}"`))return line ? Number(line.trim().split(' ').pop()) : null}@@ -2757,19 +2757,28 @@ describe('Cyclotron V2', () => {it('measureQueueDepths reports 0 for a queue that drains', async () => {// GROUP BY returns no row for an empty queue. Without the zero write the gauge// keeps the last depth, and a depth alert then fires on an idle queue.+ const ageMetric = 'cdp_cyclotron_v2_queue_oldest_due_age_ms'const jobId = uuidv7()- await insertRawJob({ id: jobId, queue_name: 'queue-drains', status: 'available' })+ await insertRawJob({+ id: jobId,+ queue_name: 'queue-drains',+ status: 'available',+ scheduled: new Date(Date.now() - 30_000),+ })const janitor = createJanitor({ stallTimeoutMs: 60_000 })const before = await janitor.runOnce()expect(before.depths.get('queue-drains')).toBe(1)expect(await gaugeValueForQueue('queue-drains')).toBe(1)+ expect(await gaugeValueForQueue('queue-drains', ageMetric)).toBeGreaterThanOrEqual(30_000)+ expect(await gaugeValueForQueue('queue-drains', ageMetric)).toBeLessThan(60_000)await assertPool.query('DELETE FROM cyclotron_jobs WHERE id = $1', [jobId])await janitor.runOnce()await janitor.stop()expect(await gaugeValueForQueue('queue-drains')).toBe(0)+ expect(await gaugeValueForQueue('queue-drains', ageMetric)).toBe(0)})it('measureQueueDepths returns correct counts per queue', async () => {diff --git a/nodejs/src/cdp/services/cyclotron-v2/janitor.ts b/nodejs/src/cdp/services/cyclotron-v2/janitor.tsindex cd15867705f0..aad8bf55d709 100644--- a/nodejs/src/cdp/services/cyclotron-v2/janitor.ts+++ b/nodejs/src/cdp/services/cyclotron-v2/janitor.ts@@ -81,6 +81,12 @@ const queueDepthGauge = new Gauge({labelNames: ['queue'],})+const queueOldestDueAgeGauge = new Gauge({+ name: 'cdp_cyclotron_v2_queue_oldest_due_age_ms',+ help: 'Milliseconds since the oldest available job per queue became due (its scheduled time). 0 when the queue has no due jobs.',+ labelNames: ['queue'],+})+interface PoisonRow {id: stringteam_id: number@@ -525,8 +531,9 @@ export class CyclotronV2Janitor {}async measureQueueDepths(): Promise<Map<string, number>> {- const result = await this.pool.query<{ queue_name: string; count: string }>(- `SELECT queue_name, COUNT(*) as count+ const result = await this.pool.query<{ queue_name: string; count: string; oldest_due_age_ms: number }>(+ `SELECT queue_name, COUNT(*) as count,+ EXTRACT(EPOCH FROM NOW() - MIN(scheduled))::float8 * 1000 as oldest_due_age_msFROM cyclotron_jobsWHERE status = 'available' AND scheduled <= NOW()GROUP BY queue_name`@@ -538,6 +545,7 @@ export class CyclotronV2Janitor {depths.set(row.queue_name, count)this.seenQueues.add(row.queue_name)queueDepthGauge.labels({ queue: row.queue_name }).set(count)+ queueOldestDueAgeGauge.labels({ queue: row.queue_name }).set(row.oldest_due_age_ms)}// GROUP BY returns no row for a queue that is empty. Without the write below,@@ -548,6 +556,7 @@ export class CyclotronV2Janitor {for (const queue of this.seenQueues) {if (!depths.has(queue)) {queueDepthGauge.labels({ queue }).set(0)+ queueOldestDueAgeGauge.labels({ queue }).set(0)}}diff --git a/nodejs/src/cdp/services/hog-executor-async.service.ts b/nodejs/src/cdp/services/hog-executor-async.service.tsindex c914db4cba34..8a06dcfcf045 100644--- a/nodejs/src/cdp/services/hog-executor-async.service.ts+++ b/nodejs/src/cdp/services/hog-executor-async.service.ts@@ -1,5 +1,4 @@import { DateTime } from 'luxon'-import { Counter } from 'prom-client'import { instrumented } from '~/common/tracing/tracing-utils'import { parseJSON } from '~/common/utils/json-parse'@@ -31,7 +30,7 @@ import { mergeSecretHeaders, resolveSecretHeaders } from '../utils/secret-headerimport { resolveStandardWebhooksKey, signStandardWebhooksRequest } from '../utils/standard-webhooks'import { HogExecutorExecuteOptions, HogExecutorPreviousResult, HogExecutorService } from './hog-executor.service'import { HogInputsService } from './hog-inputs.service'-import { EMAIL_QUEUE_PRIORITY, getEmailQueuePriorityClass } from './messaging/email-priority'+import { EMAIL_QUEUE_PRIORITY, cdpEmailQueuedTotal, getEmailQueuePriorityClass } from './messaging/email-priority'import { EmailService } from './messaging/email.service'import { PushNotificationService } from './messaging/push-notification.service'import { RecipientTokensService, preferencesTokenSourceForInvocation } from './messaging/recipient-tokens.service'@@ -44,12 +43,6 @@ import {selfLoopGuardCounter,} from './self-loop-guard'-const cdpEmailQueuedTotal = new Counter({- name: 'cdp_email_queued_total',- help: 'Total emails routed to the dedicated email queue',- labelNames: ['priority_class'] as const,-})-export interface HogExecutorAsyncConfig {googleAdwordsDeveloperToken: stringfetchRetries: numberdiff --git a/nodejs/src/cdp/services/hogflows/transactional-email.ts b/nodejs/src/cdp/services/hogflows/transactional-email.tsindex 5e92bc1f38c2..c24b3b4bf6ba 100644--- a/nodejs/src/cdp/services/hogflows/transactional-email.ts+++ b/nodejs/src/cdp/services/hogflows/transactional-email.ts@@ -1,4 +1,4 @@-import { Counter } from 'prom-client'+import { Counter, Histogram } from 'prom-client'import { z } from 'zod'// What Django sends to run a transactional email's workflow once.@@ -35,3 +35,53 @@ export const transactionalEmailRequestsTotal = new Counter({help: 'Requests from Django to run a transactional email, by outcome. Every outcome except `queued` means this request queued no email.',labelNames: ['outcome'],})++export const TRANSACTIONAL_EMAIL_STAGES = [+ 'auth',+ 'validate',+ 'load',+ 'build',+ 'idempotency',+ 'checks',+ 'queue_insert',+ 'metrics_flush',+] as const+export type TransactionalEmailStage = (typeof TRANSACTIONAL_EMAIL_STAGES)[number]++export const transactionalEmailStageDurationMs = new Histogram({+ name: 'cdp_transactional_email_stage_duration_ms',+ help: 'Milliseconds one transactional email request spends in each stage: auth (the JWT), validate (the body), load (the team and the workflow), build (the run and its idempotency fingerprint), idempotency (every Idempotency-Key call to Valkey), checks (suspension, pause, suppression, quota and MX), queue_insert (the email queue insert), metrics_flush (the result, status and app metric writes), and total. A request records only the stages it reached, so a refusal shows in the stage that refused it.',+ labelNames: ['stage'],+ buckets: [1, 2.5, 5, 10, 25, 50, 100, 250, 500, 1_000, 2_500, 5_000, 10_000],+})++// The stages of a request run back to back, so starting a stage ends the one before it. finish() records each stage+// once, with its summed time, so a stage entered twice (idempotency) still counts as one observation per request.+export class TransactionalEmailStageTimer {+ private readonly startedAt = performance.now()+ private current: { stage: TransactionalEmailStage; startedAt: number } | null = null+ private readonly durations = new Map<TransactionalEmailStage, number>()++ start(stage: TransactionalEmailStage): void {+ const now = performance.now()+ this.endCurrent(now)+ this.current = { stage, startedAt: now }+ }++ finish(): void {+ const now = performance.now()+ this.endCurrent(now)+ for (const [stage, durationMs] of this.durations) {+ transactionalEmailStageDurationMs.labels({ stage }).observe(durationMs)+ }+ transactionalEmailStageDurationMs.labels({ stage: 'total' }).observe(now - this.startedAt)+ }++ private endCurrent(now: number): void {+ if (this.current) {+ const { stage, startedAt } = this.current+ this.durations.set(stage, (this.durations.get(stage) ?? 0) + now - startedAt)+ this.current = null+ }+ }+}diff --git a/nodejs/src/cdp/services/job-queue/job-queue-postgres-v2.ts b/nodejs/src/cdp/services/job-queue/job-queue-postgres-v2.tsindex eb215a386d5b..1d7822536967 100644--- a/nodejs/src/cdp/services/job-queue/job-queue-postgres-v2.ts+++ b/nodejs/src/cdp/services/job-queue/job-queue-postgres-v2.ts@@ -356,6 +356,8 @@ export function v2JobToInvocation(job: CyclotronV2DequeuedJob): CyclotronJobInvoqueue: job.queueName as CyclotronJobQueueKind,queuePriority: job.priority,queueScheduledAt: job.scheduled ?? undefined,+ queueCreatedAt: job.created,+ queueDueAt: job.scheduled ?? undefined,queueMetadata: parsed.queueMetadata ?? undefined,queueParameters: parsed.queueParameters ?? undefined,state: parsed.state,diff --git a/nodejs/src/cdp/services/messaging/email-priority.ts b/nodejs/src/cdp/services/messaging/email-priority.tsindex 0b6c0550830f..b4f3353e94a7 100644--- a/nodejs/src/cdp/services/messaging/email-priority.ts+++ b/nodejs/src/cdp/services/messaging/email-priority.ts@@ -1,3 +1,5 @@+import { Counter } from 'prom-client'+import { HogFunctionType } from '../../types'/**@@ -36,3 +38,9 @@ export const EMAIL_QUEUE_PRIORITY: Record<EmailQueuePriorityClass, number> = {export function getEmailQueuePriorityClass(metadata: HogFunctionType['metadata']): EmailQueuePriorityClass {return metadata?.message_category_type === 'transactional' ? 'fast' : 'bulk'}++export const cdpEmailQueuedTotal = new Counter({+ name: 'cdp_email_queued_total',+ help: 'Total emails routed to the dedicated email queue',+ labelNames: ['priority_class'] as const,+})diff --git a/nodejs/src/cdp/services/messaging/email.service.test.ts b/nodejs/src/cdp/services/messaging/email.service.test.tsindex fcfade0db311..390512f5959d 100644--- a/nodejs/src/cdp/services/messaging/email.service.test.ts+++ b/nodejs/src/cdp/services/messaging/email.service.test.ts@@ -1,6 +1,8 @@import { mockFetch } from '~/tests/helpers/mocks/request.mock'import { MessageRejected, SendingPausedException, TooManyRequestsException } from '@aws-sdk/client-sesv2'+import { DateTime } from 'luxon'+import { register } from 'prom-client'import { createExampleInvocation, insertIntegration } from '~/cdp/_tests/fixtures'import { EmailStatusOutput } from '~/cdp/outputs/outputs'@@ -46,6 +48,18 @@ class ThrottlingException extends Error {}}+async function metricValue(name: string, labels: Record<string, string>, metricName = name): Promise<number> {+ const metric = await register.getSingleMetric(name)!.get()+ const values: { labels: Partial<Record<string, string | number>>; value: number; metricName?: string }[] =+ metric.values+ const sample = values.find(+ (value) =>+ (value.metricName ?? name) === metricName &&+ Object.entries(labels).every(([key, labelValue]) => value.labels[key] === labelValue)+ )+ return sample?.value ?? 0+}+describe('sanitizeEmailSubject', () => {it.each([['passes through normal text', 'Hello World', 'Hello World'],@@ -493,6 +507,29 @@ describe('EmailService', () => {)})+ it.each<[string, () => Error, string]>([+ [+ 'TooManyRequestsException',+ () => new TooManyRequestsException({ $metadata: {}, message: 'Too many requests' }),+ 'throttled',+ ],+ ['ThrottlingException', () => new ThrottlingException('Rate exceeded'), 'throttled'],+ [+ 'SendingPausedException',+ () => new SendingPausedException({ $metadata: {}, message: 'Sending paused' }),+ 'SendingPausedException',+ ],+ ['an unknown error', () => new Error('connect ECONNRESET for [email protected]'), 'other'],+ ])('counts an SES send that fails with %s as %s', async (_name, makeError, expectedResult) => {+ const labels = { result: expectedResult }+ const before = await metricValue('cdp_ses_send_results_total', labels)+ sendEmailSpy.mockRejectedValueOnce(makeError())++ await service.executeSendEmail(invocation)++ expect(await metricValue('cdp_ses_send_results_total', labels)).toBe(before + 1)+ })+it('still fails the job for non-throttle SES errors', async () => {sendEmailSpy.mockRejectedValueOnce(new MessageRejected({ $metadata: {}, message: 'something else broke' })@@ -509,6 +546,45 @@ describe('EmailService', () => {})})+ describe('queue-to-send latency', () => {+ // The job was created a minute ago and rescheduled after a throttle, so it became due again 5 seconds ago.+ // Running the email step clones the invocation, which resets queueScheduledAt before the send.+ it.each([+ ['workflow', 'bulk', 'its rescheduled time', (): void => {}, 5_000, 60_000],+ [+ 'transactional_api',+ 'fast',+ 'its creation',+ (): void => {+ invocation.state.globals.event.event = '$transactional_email'+ invocation.hogFunction.metadata = { message_category_type: 'transactional' }+ },+ 60_000,+ Infinity,+ ],+ ])(+ 'observes the wait of a retried %s send on the %s lane from %s',+ async (source, priorityClass, _from, setUp, minWaitMs, maxWaitMs) => {+ setUp()+ invocation.queue = 'email'+ invocation.queueCreatedAt = DateTime.utc().minus({ seconds: 60 })+ invocation.queueDueAt = DateTime.utc().minus({ seconds: 5 })+ invocation.queueScheduledAt = undefined+ const name = 'cdp_email_queue_to_send_ms'+ const labels = { source, priority_class: priorityClass }+ const countBefore = await metricValue(name, labels, `${name}_count`)+ const sumBefore = await metricValue(name, labels, `${name}_sum`)++ await service.executeSendEmail(invocation)++ expect(await metricValue(name, labels, `${name}_count`)).toBe(countBefore + 1)+ const waitMs = (await metricValue(name, labels, `${name}_sum`)) - sumBefore+ expect(waitMs).toBeGreaterThanOrEqual(minWaitMs)+ expect(waitMs).toBeLessThan(maxWaitMs)+ }+ )+ })+describe('workflow sending rate limit', () => {let claimOrReserve: jest.Mocklet limitedService: EmailServicediff --git a/nodejs/src/cdp/services/messaging/email.service.ts b/nodejs/src/cdp/services/messaging/email.service.tsindex cb74135e7723..d99cf20fa3a8 100644--- a/nodejs/src/cdp/services/messaging/email.service.ts+++ b/nodejs/src/cdp/services/messaging/email.service.ts@@ -16,10 +16,12 @@ import { createAddLogFunction, logEntry } from '~/cdp/utils'import { createInvocationResult } from '~/cdp/utils/invocation-utils'import { logger } from '~/common/utils/logger'+import { TRANSACTIONAL_EMAIL_EVENT } from '../hogflows/hogflow-sensitivity'import { IntegrationManagerService } from '../managers/integration-manager.service'import { RecipientManagerRecipient, RecipientsManagerService } from '../managers/recipients-manager.service'import { TeamWorkflowsConfigService } from '../managers/team-workflows-config.service'import { RateLimiterService } from '../rate-limiter/rate-limiter.service'+import { getEmailQueuePriorityClass } from './email-priority'import { selectEmailSenderIntegrationId } from './email-sender-selection'import { EmailStatusService, isTransactionalEmailSend } from './email-status.service'import { EmailSuppressionService, SuppressionCheckOptions } from './email-suppression.service'@@ -80,6 +82,59 @@ function isSesThrottleError(error: unknown): error is Error & { name: SesThrottlreturn error instanceof Error && (SES_THROTTLE_ERROR_NAMES as readonly string[]).includes(error.name)}+// Every other error name reads as `other`, so an unexpected SDK error cannot add a series.+const SES_SEND_ERROR_RESULTS = [+ 'MessageRejected',+ 'MailFromDomainNotVerifiedException',+ 'AccountSuspendedException',+ 'SendingPausedException',+ 'LimitExceededException',+ 'NotFoundException',+ 'BadRequestException',+ 'TimeoutError',+] as const+type SesSendResult = 'success' | 'throttled' | (typeof SES_SEND_ERROR_RESULTS)[number] | 'other'++function classifySesSendError(error: unknown): SesSendResult {+ if (isSesThrottleError(error)) {+ return 'throttled'+ }+ const name = error instanceof Error ? error.name : undefined+ return SES_SEND_ERROR_RESULTS.find((known) => known === name) ?? 'other'+}++const sesSendResultsTotal = new Counter({+ name: 'cdp_ses_send_results_total',+ help: 'SES SendEmail calls by result: success, throttled (rescheduled, with the error code in cdp_ses_throttle_responses_total), a known SES error name, or other.',+ labelNames: ['result'],+})++const sesSendDurationMs = new Histogram({+ name: 'cdp_ses_send_duration_ms',+ help: 'Duration of the SES SendEmail call in milliseconds, by whether SES accepted the email (success or error).',+ labelNames: ['result'],+ buckets: [25, 50, 100, 250, 500, 1_000, 2_500, 5_000, 10_000, 30_000],+})++const emailQueueToSendMs = new Histogram({+ name: 'cdp_email_queue_to_send_ms',+ help: 'Milliseconds an email waits on the email queue before its handoff to SES. For source="transactional_api" the wait starts when cdp-api created the job, so it is the time from the API\'s 202 to SES and includes every throttled or rate-limited retry. For source="workflow" the wait starts at the job\'s scheduled time: when the job joined the email queue, or when a delayed or rescheduled send became due, so a retried send measures only its last wait. priority_class is the email queue lane.',+ labelNames: ['source', 'priority_class'],+ buckets: [100, 250, 500, 1_000, 2_500, 5_000, 10_000, 30_000, 60_000, 120_000, 300_000],+})++type EmailQueueWait = { source: 'transactional_api' | 'workflow'; startedAt: DateTime }++// A transactional API job is created on the email queue, so its creation time is when the caller got its 202. A+// workflow job can join the email queue long after it was created, so its wait starts at the time it became due.+// Both come from fields a clone keeps: running the email step clones the invocation and resets queueScheduledAt.+function emailQueueWait(invocation: CyclotronJobInvocationHogFunction): EmailQueueWait | undefined {+ const source =+ invocation.state.globals.event?.event === TRANSACTIONAL_EMAIL_EVENT ? 'transactional_api' : 'workflow'+ const startedAt = source === 'transactional_api' ? invocation.queueCreatedAt : invocation.queueDueAt+ return startedAt ? { source, startedAt } : undefined+}+/*** Tagged error signalling that SES rejected the send for a transient,* rate-limit-shaped reason. The caller schedules a retry instead of failing@@ -404,6 +459,9 @@ export class EmailService {)const addLog = createAddLogFunction(result.logs)+ // Read from the incoming invocation: result.invocation is a clone, and a clone drops the scheduled time.+ const queueWait = invocation.queue === 'email' && !isTest ? emailQueueWait(invocation) : undefined+const params = invocation.queueParametersconst integrationId = selectEmailSenderIntegrationId(invocation.id, params.from)const integration = await this.integrationManager.get(integrationId)@@ -684,7 +742,7 @@ export class EmailService {await this.sendEmailWithMaildev(result, sendParams, from, trackingEnabled, isTest)breakcase 'ses':- await this.sendEmailWithSES(result, sendParams, from, trackingEnabled, isTest)+ await this.sendEmailWithSES(result, sendParams, from, trackingEnabled, isTest, queueWait)breakcase 'unsupported':@@ -1109,7 +1167,8 @@ export class EmailService {params: CyclotronInvocationQueueParametersEmailType,from: { email: string; name: string },trackingEnabled: boolean,- isTest = false+ isTest = false,+ queueWait?: EmailQueueWait): Promise<void> {if (!this.sesV2Client) {throw new Error('SES is not configured - set SES_REGION and AWS credentials')@@ -1220,12 +1279,26 @@ export class EmailService {sendEmailParams.Destination!.BccAddresses = bccAddresses}+ if (queueWait) {+ emailQueueToSendMs+ .labels({+ source: queueWait.source,+ priority_class: getEmailQueuePriorityClass(result.invocation.hogFunction.metadata),+ })+ .observe(Math.max(0, Date.now() - queueWait.startedAt.toMillis()))+ }++ const sendStartedAt = performance.now()try {const response = await this.sesV2Client.send(new SendEmailCommand(sendEmailParams))if (!response.MessageId) {throw new Error('No messageId returned from SES')}+ sesSendDurationMs.labels({ result: 'success' }).observe(performance.now() - sendStartedAt)+ sesSendResultsTotal.labels({ result: 'success' }).inc()} catch (error: unknown) {+ sesSendDurationMs.labels({ result: 'error' }).observe(performance.now() - sendStartedAt)+ sesSendResultsTotal.labels({ result: classifySesSendError(error) }).inc()if (isSesThrottleError(error)) {sesThrottleResponsesTotal.inc({ error_code: error.name })throw new SESThrottleError(error.name, pickThrottleRetryDelayMs(), error.message)diff --git a/nodejs/src/cdp/types.ts b/nodejs/src/cdp/types.tsindex c22bc248ae0b..b59606543a4a 100644--- a/nodejs/src/cdp/types.ts+++ b/nodejs/src/cdp/types.ts@@ -342,6 +342,10 @@ export type CyclotronJobInvocation = {queuePriority: number// When the invocation is scheduled to runqueueScheduledAt?: DateTime+ // When the queue first stored the job. A reschedule moves queueScheduledAt but not this, and a clone keeps it.+ queueCreatedAt?: DateTime+ // The scheduled time this job had when it was dequeued. A clone resets queueScheduledAt but keeps this.+ queueDueAt?: DateTime// Metadata for the invocation - TODO: check when this gets clearedqueueMetadata?: Record<string, any> | null// Where the invocation came from (kafka or postgres)diff --git a/products/messaging/backend/facade/transactional_email.py b/products/messaging/backend/facade/transactional_email.pyindex 4e5fe7ec755d..dbff0513fde0 100644--- a/products/messaging/backend/facade/transactional_email.py+++ b/products/messaging/backend/facade/transactional_email.py@@ -9,12 +9,15 @@ from products.messaging.backend.facade.contracts import ()from products.messaging.backend.services.transactional_email import (SENDER_ADDRESS_MAX_LENGTH,+ TransactionalEmailStage,check_transactional_update,lock_transactional_workflow_id,make_transactional,+ observe_transactional_email_stage,send_transactional_email,sender_address_error,sync_transactional_workflow,+ transactional_email_stage,)from products.messaging.backend.services.transactional_email_status import get_email_status@@ -25,11 +28,14 @@ __all__ = ["EmailStatus","QueuedEmail","TransactionalEmailError",+ "TransactionalEmailStage","check_transactional_update","get_email_status","lock_transactional_workflow_id","make_transactional",+ "observe_transactional_email_stage","send_transactional_email","sender_address_error","sync_transactional_workflow",+ "transactional_email_stage",]diff --git a/products/messaging/backend/presentation/views/message_templates.py b/products/messaging/backend/presentation/views/message_templates.pyindex c7000d0069e1..3dce0c0caef7 100644--- a/products/messaging/backend/presentation/views/message_templates.py+++ b/products/messaging/backend/presentation/views/message_templates.py@@ -1,7 +1,9 @@+import timefrom collections.abc import Sequence+from contextlib import AbstractContextManager, nullcontextfrom dataclasses import fields as dataclass_fieldsfrom types import SimpleNamespace-from typing import Any+from typing import Any, castfrom django.db import transaction@@ -59,12 +61,15 @@ from products.messaging.backend.facade.transactional_email import (SENDER_ADDRESS_MAX_LENGTH,DeliveryUnavailable,TransactionalEmailError,+ TransactionalEmailStage,check_transactional_update,lock_transactional_workflow_id,make_transactional,+ observe_transactional_email_stage,send_transactional_email,sender_address_error,sync_transactional_workflow,+ transactional_email_stage,)from products.messaging.backend.presentation.views.serializers import DesignOperationSerializerfrom products.notifications.backend.facade.api import publish_resource_edited@@ -537,13 +542,55 @@ class TransactionalEmailSendMixin:team_id: intorganization: Anyuser_access_control: Any+ # The actions of the viewset that send an email. Only these record stage durations.+ transactional_email_send_actions: tuple[str, ...] = ()+ _validation_started_at: float | None = None++ def _is_transactional_email_send(self) -> bool:+ return getattr(self, "action", None) in self.transactional_email_send_actions++ def _send_stage(self, stage: TransactionalEmailStage) -> AbstractContextManager[None]:+ return transactional_email_stage(stage) if self._is_transactional_email_send() else nullcontext()++ def dispatch(self, request: Any, *args: Any, **kwargs: Any) -> Any:+ started_at = time.monotonic()+ try:+ return cast(Any, super()).dispatch(request, *args, **kwargs)+ finally:+ # A body that fails validation never reaches the action, so its validation time is recorded here.+ self._end_request_validation()+ if self._is_transactional_email_send():+ observe_transactional_email_stage("total", time.monotonic() - started_at)++ def perform_authentication(self, request: Any) -> None:+ with self._send_stage("auth"):+ cast(Any, super()).perform_authentication(request)++ def check_permissions(self, request: Any) -> None:+ with self._send_stage("permissions"):+ cast(Any, super()).check_permissions(request)++ def check_throttles(self, request: Any) -> None:+ with self._send_stage("throttle"):+ cast(Any, super()).check_throttles(request)++ def initial(self, request: Any, *args: Any, **kwargs: Any) -> None:+ cast(Any, super()).initial(request, *args, **kwargs)+ # @validated_request parses and validates the body after initial() returns and before the action body runs.+ self._validation_started_at = time.monotonic()++ def _end_request_validation(self) -> None:+ if self._validation_started_at is not None and self._is_transactional_email_send():+ observe_transactional_email_stage("validation", time.monotonic() - self._validation_started_at)+ self._validation_started_at = Nonedef _require_transactional_email_flag(self) -> None:require_transactional_email_api(self.team)def _queue_transactional_email(self, request: Any, template: MessageTemplateRow, *, route: str) -> Response:- if template.transactional_hog_flow_id:- self._require_workflow_editor(template.transactional_hog_flow_id)+ with transactional_email_stage("workflow_access"):+ if template.transactional_hog_flow_id:+ self._require_workflow_editor(template.transactional_hog_flow_id)data = request.validated_dataidempotency_key = request.headers.get("Idempotency-Key")outcome: dict[str, Any] = {@@ -568,7 +615,8 @@ class TransactionalEmailSendMixin:outcome["outcome"] = "unavailable"raise TransactionalEmailUnavailable() from Nonefinally:- self._report_transactional_email_send(request, template, outcome)+ with transactional_email_stage("usage_event"):+ self._report_transactional_email_send(request, template, outcome)return Response(SendTransactionalEmailResponseSerializer(queued).data,status=status.HTTP_202_ACCEPTED,@@ -644,6 +692,7 @@ class MessageTemplatesViewSet(authentication_classes = [ProjectSecretAPIKeyAuthentication]# A project secret API key can only send. Template CRUD stays session/personal API key/OAuth only.psak_allowed_actions = ["send"]+ transactional_email_send_actions = ("send",)serializer_class = MessageTemplateSerializer@@ -850,8 +899,12 @@ class MessageTemplatesViewSet()@action(detail=True, methods=["POST"], pagination_class=None)def send(self, request: Any, *args: Any, **kwargs: Any) -> Response:- self._require_transactional_email_flag()- return self._queue_transactional_email(request, self.get_object(), route="template")+ self._end_request_validation()+ with transactional_email_stage("feature_flag"):+ self._require_transactional_email_flag()+ with transactional_email_stage("template_lookup"):+ template = self.get_object()+ return self._queue_transactional_email(request, template, route="template")@validated_request(MakeTransactionalSerializer,@@ -910,6 +963,7 @@ class MessagingSendViewSet(TransactionalEmailSendMixin, TeamAndOrgViewSetMixin,permission_classes = [IsAuthenticated]authentication_classes = [ProjectSecretAPIKeyAuthentication]psak_allowed_actions = ["email"]+ transactional_email_send_actions = ("email",)def get_throttles(self):return transactional_email_throttles()@@ -928,10 +982,13 @@ class MessagingSendViewSet(TransactionalEmailSendMixin, TeamAndOrgViewSetMixin,)@action(detail=False, methods=["POST"])def email(self, request: Any, *args: Any, **kwargs: Any) -> Response:- self._require_transactional_email_flag()- try:- template = get_template(self.team_id, request.validated_data["template"]["id"])- except MessageTemplateMissing:- raise NotFound()- self.check_object_permissions(request, template)+ self._end_request_validation()+ with transactional_email_stage("feature_flag"):+ self._require_transactional_email_flag()+ with transactional_email_stage("template_lookup"):+ try:+ template = get_template(self.team_id, request.validated_data["template"]["id"])+ except MessageTemplateMissing:+ raise NotFound()+ self.check_object_permissions(request, template)return self._queue_transactional_email(request, template, route="send_email")diff --git a/products/messaging/backend/services/transactional_email.py b/products/messaging/backend/services/transactional_email.pyindex 938afdfa4c8d..804975351759 100644--- a/products/messaging/backend/services/transactional_email.py+++ b/products/messaging/backend/services/transactional_email.py@@ -2,12 +2,17 @@per send, so sends get the workflow's metrics, run history and versions."""import re-from typing import TYPE_CHECKING, Any+import time+from collections.abc import Iterator+from contextlib import contextmanager+from typing import Any, Literalfrom uuid import UUIDfrom django.db import transaction+import requestsimport structlog+from prometheus_client import Counter, Histogramfrom posthog.models.integration import Integrationfrom posthog.models.team import Team@@ -47,11 +52,78 @@ from products.workflows.backend.facade.transactional_emails import (update_transactional_email_workflow,)-if TYPE_CHECKING:- import requests-logger = structlog.get_logger(__name__)+CDP_SEND_DURATION = Histogram(+ "messaging_transactional_email_cdp_request_duration_seconds",+ "Time for the CDP service to answer a request to queue one transactional email, including failed requests.",+ buckets=(0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 7.5, 10, 15),+)+CDP_SEND_RESULTS = Counter(+ "messaging_transactional_email_cdp_requests_total",+ "Requests to the CDP service to queue one transactional email, by result: success (202), the HTTP status class "+ "of any other answer, timeout, connection_error, or error.",+ labelnames=["result"],+)+++TransactionalEmailStage = Literal[+ "auth",+ "permissions",+ "throttle",+ "validation",+ "feature_flag",+ "template_lookup",+ "workflow_access",+ "workflow_lookup",+ "cdp_call",+ "usage_event",+ "total",+]+STAGE_DURATION = Histogram(+ "messaging_transactional_email_stage_duration_seconds",+ "Time one transactional email send request spends in each stage, including requests that fail in that stage: "+ "auth, permissions and throttle (DRF's checks), validation (the request body), feature_flag, template_lookup, "+ "workflow_access (the caller's editor access to the workflow), workflow_lookup (the template and workflow "+ "checks before the send), cdp_call (the same time as messaging_transactional_email_cdp_request_duration_seconds), "+ "usage_event, and total (the DRF view from start to response).",+ labelnames=["stage"],+ buckets=(0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 15),+)+++def observe_transactional_email_stage(stage: TransactionalEmailStage, seconds: float) -> None:+ STAGE_DURATION.labels(stage=stage).observe(seconds)+++@contextmanager+def transactional_email_stage(stage: TransactionalEmailStage) -> Iterator[None]:+ started_at = time.monotonic()+ try:+ yield+ finally:+ observe_transactional_email_stage(stage, time.monotonic() - started_at)+++def _cdp_error_result(error: Exception) -> str:+ # ConnectTimeout is both a Timeout and a ConnectionError, so the timeout check goes first.+ if isinstance(error, requests.Timeout):+ return "timeout"+ if isinstance(error, requests.ConnectionError):+ return "connection_error"+ return "error"+++def _cdp_response_result(status_code: int) -> str:+ return "success" if status_code == 202 else f"{status_code // 100}xx"+++def _observe_cdp_call(seconds: float) -> None:+ # Both histograms get the same value, so the stage breakdown and the CDP request metric can't disagree.+ CDP_SEND_DURATION.observe(seconds)+ observe_transactional_email_stage("cdp_call", seconds)++# The CDP service names the reason it refused a send in the body's `error`. A retry of the same request gets the# same refusal, so each of these reaches the caller as a 4xx and not as a retryable 503.PIPELINE_REFUSALS: dict[str, tuple[type[TransactionalEmailError], str]] = {@@ -314,7 +386,8 @@ def send_transactional_email(if "{" in to or "}" in to:raise InvalidRecipient("The recipient address can't contain { or }.")- hog_flow_id = _transactional_workflow_id(team.id, template_id)+ with transactional_email_stage("workflow_lookup"):+ hog_flow_id = _transactional_workflow_id(team.id, template_id)payload = {"to": to,# The recipient is the person the send is tracked against. Without a distinct_id, the address is one.@@ -324,11 +397,16 @@ def send_transactional_email(if idempotency_key is not None:# The CDP service holds the key, because only it knows whether the email was queued.payload["idempotency_key"] = idempotency_key+ started_at = time.monotonic()try:response = run_transactional_workflow(team.id, str(hog_flow_id), payload, timeout=timeout)except Exception as e:+ _observe_cdp_call(time.monotonic() - started_at)+ CDP_SEND_RESULTS.labels(result=_cdp_error_result(e)).inc()logger.exception("transactional_email_queue_failed", team_id=team.id, template_id=str(template_id))raise DeliveryUnavailable() from e+ _observe_cdp_call(time.monotonic() - started_at)+ CDP_SEND_RESULTS.labels(result=_cdp_response_result(response.status_code)).inc()if response.status_code != 202:refusal = PIPELINE_REFUSALS.get(_refusal_code(response) or "")@@ -347,7 +425,7 @@ def send_transactional_email(return QueuedEmail(message_id=str(body["invocation_id"]), replayed=bool(body.get("replayed", False)))-def _refusal_code(response: "requests.Response") -> str | None:+def _refusal_code(response: requests.Response) -> str | None:try:body = response.json()except ValueError:diff --git a/products/messaging/backend/tests/api/test_transactional_email.py b/products/messaging/backend/tests/api/test_transactional_email.pyindex 786ad7893242..03ae73b8f81a 100644--- a/products/messaging/backend/tests/api/test_transactional_email.py+++ b/products/messaging/backend/tests/api/test_transactional_email.py@@ -1,4 +1,4 @@-from typing import Any+from typing import Any, get_argsfrom uuid import uuid4import pytest@@ -7,6 +7,7 @@ from unittest.mock import MagicMock, patchimport requestsfrom parameterized import parameterized+from prometheus_client import REGISTRYfrom rest_framework import statusfrom rest_framework.test import APIClient@@ -19,10 +20,18 @@ from posthog.models.utils import generate_random_token_secretfrom products.access_control.backend.facade.testing import create_access_controlfrom products.messaging.backend.models.message_category import MessageCategory, MessageCategoryTypefrom products.messaging.backend.models.message_template import MessageTemplate-from products.messaging.backend.services.transactional_email import make_transactional+from products.messaging.backend.services.transactional_email import TransactionalEmailStage, make_transactionalfrom products.workflows.backend.facade.api import set_default_email_sender_if_unset+def _stage_counts() -> dict[str, float]:+ return {+ stage: REGISTRY.get_sample_value("messaging_transactional_email_stage_duration_seconds_count", {"stage": stage})+ or 0+ for stage in get_args(TransactionalEmailStage)+ }++@patch("products.messaging.backend.presentation.views.message_templates.posthoganalytics.capture")@patch("products.messaging.backend.presentation.views.message_templates.posthoganalytics.feature_enabled",@@ -102,10 +111,16 @@ class TestTransactionalEmail(APIBaseTest):if identifierselse {k: v for k, v in self.body.items() if k != "identifiers"})+ client = self._psak_client(["hog_flow:write"])+ stages_before = _stage_counts()- response = self._send(self._psak_client(["hog_flow:write"]), route, body)+ response = self._send(client, route, body)assert response.status_code == status.HTTP_202_ACCEPTED, response.json()+ stages_after = _stage_counts()+ assert {stage: stages_after[stage] - stages_before[stage] for stage in stages_after} == dict.fromkeys(+ stages_after, 1+ )assert response.json() == {"message_id": "invocation-1", "replayed": False}assert capture.call_args.kwargs["properties"]["route"] == routeassert mock_post.call_args.args[0].endswith(@@ -200,20 +215,27 @@ class TestTransactionalEmail(APIBaseTest):@parameterized.expand([- ("timeout", requests.ReadTimeout()),- ("cdp_api_error", MagicMock(status_code=500)),+ ("timeout", requests.ReadTimeout(), "timeout"),+ ("connect_timeout", requests.ConnectTimeout(), "timeout"),+ ("connection_error", requests.ConnectionError(), "connection_error"),+ ("cdp_api_error", MagicMock(status_code=500), "5xx"),])- def test_a_send_the_pipeline_does_not_confirm_is_unavailable(self, mock_post, _flag, _capture, _name, outcome):+ def test_a_send_the_pipeline_does_not_confirm_is_unavailable(+ self, mock_post, _flag, _capture, _name, outcome, expected_result+ ):if isinstance(outcome, Exception):mock_post.side_effect = outcomeelse:mock_post.return_value = outcome+ metric = "messaging_transactional_email_cdp_requests_total"+ before = REGISTRY.get_sample_value(metric, {"result": expected_result}) or 0response = self.client.post(self.url, self.body, format="json")assert response.status_code == status.HTTP_503_SERVICE_UNAVAILABLEassert response.json()["code"] == "delivery_unavailable"+ assert REGISTRY.get_sample_value(metric, {"result": expected_result}) == before + 1@parameterized.expand([