:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(cdp): label email queue waits that the team sending cap delayed (#126304)
Harley Alexander55638a0f8628
diff --git a/nodejs/src/cdp/services/messaging/email.service.test.ts b/nodejs/src/cdp/services/messaging/email.service.test.tsindex 825423616ecc..d42d6399c11a 100644--- a/nodejs/src/cdp/services/messaging/email.service.test.ts+++ b/nodejs/src/cdp/services/messaging/email.service.test.ts@@ -565,29 +565,37 @@ describe('EmailService', () => {describe('queue-to-send latency', () => {// The job was created a minute ago and rescheduled after a throttle, so it became due again 5 seconds ago.// Running the email step clones the invocation, which resets queueScheduledAt before the send.+ const transactionalSend = (): void => {+ invocation.state.globals.event.event = '$transactional_email'+ invocation.hogFunction.metadata = { message_category_type: 'transactional' }+ }+it.each([- ['workflow', 'bulk', 'its rescheduled time', (): void => {}, 5_000, 60_000],+ ['workflow', 'bulk', 'false', 'its rescheduled time', (): void => {}, 5_000, 60_000],+ ['transactional_api', 'fast', 'false', 'its creation', transactionalSend, 60_000, Infinity],+ // Alerts leave out a wait that the team sending cap causes on purpose.['transactional_api','fast',- 'its creation',+ 'true',+ 'its creation after a team cap delay',(): void => {- invocation.state.globals.event.event = '$transactional_email'- invocation.hogFunction.metadata = { message_category_type: 'transactional' }+ transactionalSend()+ invocation.queueMetadata = { teamCapped: true }},60_000,Infinity,],])(- 'observes the wait of a retried %s send on the %s lane from %s',- async (source, priorityClass, _from, setUp, minWaitMs, maxWaitMs) => {+ 'observes the wait of a retried %s send on the %s lane with team_capped=%s from %s',+ async (source, priorityClass, teamCapped, _from, setUp, minWaitMs, maxWaitMs) => {setUp()invocation.queue = 'email'invocation.queueCreatedAt = DateTime.utc().minus({ seconds: 60 })invocation.queueDueAt = DateTime.utc().minus({ seconds: 5 })invocation.queueScheduledAt = undefinedconst name = 'cdp_email_queue_to_send_ms'- const labels = { source, priority_class: priorityClass }+ const labels = { source, priority_class: priorityClass, team_capped: teamCapped }const countBefore = await metricValue(name, labels, `${name}_count`)const sumBefore = await metricValue(name, labels, `${name}_sum`)@@ -904,6 +912,8 @@ describe('EmailService', () => {expect(result.finished).toBe(false)// The reschedule must carry the email payload forward, same as the workflow limit.expect(result.invocation.queueParameters).toEqual(invocation.queueParameters)+ // The flag labels the eventual send's queue wait, so alerts can leave out the cap delay.+ expect(result.invocation.queueMetadata).toMatchObject({ teamCapped: true })const scheduledMs = result.invocation.queueScheduledAt!.toMillis()expect(scheduledMs).toBeGreaterThanOrEqual(before + minMs)expect(scheduledMs).toBeLessThan(before + maxMs + 5000)diff --git a/nodejs/src/cdp/services/messaging/email.service.ts b/nodejs/src/cdp/services/messaging/email.service.tsindex 27ad77432e00..c67d2f7e592b 100644--- a/nodejs/src/cdp/services/messaging/email.service.ts+++ b/nodejs/src/cdp/services/messaging/email.service.ts@@ -118,12 +118,12 @@ const sesSendDurationMs = new Histogram({const emailQueueToSendMs = new Histogram({name: 'cdp_email_queue_to_send_ms',- help: 'Milliseconds an email waits on the email queue before its handoff to SES. For source="transactional_api" the wait starts when cdp-api created the job, so it is the time from the API\'s 202 to SES and includes every throttled or rate-limited retry. For source="workflow" the wait starts at the job\'s scheduled time: when the job joined the email queue, or when a delayed or rescheduled send became due, so a retried send measures only its last wait. priority_class is the email queue lane.',- labelNames: ['source', 'priority_class'],+ help: 'Milliseconds an email waits on the email queue before its handoff to SES. For source="transactional_api" the wait starts when cdp-api created the job, so it is the time from the API\'s 202 to SES and includes every throttled or rate-limited retry. For source="workflow" the wait starts at the job\'s scheduled time: when the job joined the email queue, or when a delayed or rescheduled send became due, so a retried send measures only its last wait. priority_class is the email queue lane. team_capped is true when the team sending cap delayed the send, so that alerts can leave out a wait that the cap causes on purpose.',+ labelNames: ['source', 'priority_class', 'team_capped'],buckets: [100, 250, 500, 1_000, 2_500, 5_000, 10_000, 30_000, 60_000, 120_000, 300_000],})-type EmailQueueWait = { source: 'transactional_api' | 'workflow'; startedAt: DateTime }+type EmailQueueWait = { source: 'transactional_api' | 'workflow'; startedAt: DateTime; teamCapped: boolean }// A transactional API job is created on the email queue, so its creation time is when the caller got its 202. A// workflow job can join the email queue long after it was created, so its wait starts at the time it became due.@@ -132,7 +132,7 @@ function emailQueueWait(invocation: CyclotronJobInvocationHogFunction): EmailQueconst source =invocation.state.globals.event?.event === TRANSACTIONAL_EMAIL_EVENT ? 'transactional_api' : 'workflow'const startedAt = source === 'transactional_api' ? invocation.queueCreatedAt : invocation.queueDueAt- return startedAt ? { source, startedAt } : undefined+ return startedAt ? { source, startedAt, teamCapped: invocation.queueMetadata?.teamCapped === true } : undefined}/**@@ -694,6 +694,7 @@ export class EmailService {// per-workflow limit above: createInvocationResult cleared queueParameters, and// without them the rescheduled dequeue resumes the Hog VM and drops the send.result.invocation.queueParameters = params+ result.invocation.queueMetadata = { ...invocation.queueMetadata, teamCapped: true }result.invocation.queueScheduledAt = DateTime.utc().plus({ milliseconds: capDelay.retryDelayMs })const capRetrySeconds = Math.round(capDelay.retryDelayMs / 1000)addLog(@@ -1293,6 +1294,7 @@ export class EmailService {.labels({source: queueWait.source,priority_class: getEmailQueuePriorityClass(result.invocation.hogFunction.metadata),+ team_capped: queueWait.teamCapped ? 'true' : 'false',}).observe(Math.max(0, Date.now() - queueWait.startedAt.toMillis()))}