Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(cdp): label email queue waits that the team sending cap delayed (#126304)

Harley Alexander55638a0f8628
diff --git a/nodejs/src/cdp/services/messaging/email.service.test.ts b/nodejs/src/cdp/services/messaging/email.service.test.ts
index 825423616ecc..d42d6399c11a 100644
--- a/nodejs/src/cdp/services/messaging/email.service.test.ts
+++ b/nodejs/src/cdp/services/messaging/email.service.test.ts
@@ -565,29 +565,37 @@ describe('EmailService', () => {
describe('queue-to-send latency', () => {
// The job was created a minute ago and rescheduled after a throttle, so it became due again 5 seconds ago.
// Running the email step clones the invocation, which resets queueScheduledAt before the send.
+ const transactionalSend = (): void => {
+ invocation.state.globals.event.event = '$transactional_email'
+ invocation.hogFunction.metadata = { message_category_type: 'transactional' }
+ }
+
it.each([
- ['workflow', 'bulk', 'its rescheduled time', (): void => {}, 5_000, 60_000],
+ ['workflow', 'bulk', 'false', 'its rescheduled time', (): void => {}, 5_000, 60_000],
+ ['transactional_api', 'fast', 'false', 'its creation', transactionalSend, 60_000, Infinity],
+ // Alerts leave out a wait that the team sending cap causes on purpose.
[
'transactional_api',
'fast',
- 'its creation',
+ 'true',
+ 'its creation after a team cap delay',
(): void => {
- invocation.state.globals.event.event = '$transactional_email'
- invocation.hogFunction.metadata = { message_category_type: 'transactional' }
+ transactionalSend()
+ invocation.queueMetadata = { teamCapped: true }
},
60_000,
Infinity,
],
])(
- 'observes the wait of a retried %s send on the %s lane from %s',
- async (source, priorityClass, _from, setUp, minWaitMs, maxWaitMs) => {
+ 'observes the wait of a retried %s send on the %s lane with team_capped=%s from %s',
+ async (source, priorityClass, teamCapped, _from, setUp, minWaitMs, maxWaitMs) => {
setUp()
invocation.queue = 'email'
invocation.queueCreatedAt = DateTime.utc().minus({ seconds: 60 })
invocation.queueDueAt = DateTime.utc().minus({ seconds: 5 })
invocation.queueScheduledAt = undefined
const name = 'cdp_email_queue_to_send_ms'
- const labels = { source, priority_class: priorityClass }
+ const labels = { source, priority_class: priorityClass, team_capped: teamCapped }
const countBefore = await metricValue(name, labels, `${name}_count`)
const sumBefore = await metricValue(name, labels, `${name}_sum`)
@@ -904,6 +912,8 @@ describe('EmailService', () => {
expect(result.finished).toBe(false)
// The reschedule must carry the email payload forward, same as the workflow limit.
expect(result.invocation.queueParameters).toEqual(invocation.queueParameters)
+ // The flag labels the eventual send's queue wait, so alerts can leave out the cap delay.
+ expect(result.invocation.queueMetadata).toMatchObject({ teamCapped: true })
const scheduledMs = result.invocation.queueScheduledAt!.toMillis()
expect(scheduledMs).toBeGreaterThanOrEqual(before + minMs)
expect(scheduledMs).toBeLessThan(before + maxMs + 5000)
diff --git a/nodejs/src/cdp/services/messaging/email.service.ts b/nodejs/src/cdp/services/messaging/email.service.ts
index 27ad77432e00..c67d2f7e592b 100644
--- a/nodejs/src/cdp/services/messaging/email.service.ts
+++ b/nodejs/src/cdp/services/messaging/email.service.ts
@@ -118,12 +118,12 @@ const sesSendDurationMs = new Histogram({
const emailQueueToSendMs = new Histogram({
name: 'cdp_email_queue_to_send_ms',
- help: 'Milliseconds an email waits on the email queue before its handoff to SES. For source="transactional_api" the wait starts when cdp-api created the job, so it is the time from the API\'s 202 to SES and includes every throttled or rate-limited retry. For source="workflow" the wait starts at the job\'s scheduled time: when the job joined the email queue, or when a delayed or rescheduled send became due, so a retried send measures only its last wait. priority_class is the email queue lane.',
- labelNames: ['source', 'priority_class'],
+ help: 'Milliseconds an email waits on the email queue before its handoff to SES. For source="transactional_api" the wait starts when cdp-api created the job, so it is the time from the API\'s 202 to SES and includes every throttled or rate-limited retry. For source="workflow" the wait starts at the job\'s scheduled time: when the job joined the email queue, or when a delayed or rescheduled send became due, so a retried send measures only its last wait. priority_class is the email queue lane. team_capped is true when the team sending cap delayed the send, so that alerts can leave out a wait that the cap causes on purpose.',
+ labelNames: ['source', 'priority_class', 'team_capped'],
buckets: [100, 250, 500, 1_000, 2_500, 5_000, 10_000, 30_000, 60_000, 120_000, 300_000],
})
-type EmailQueueWait = { source: 'transactional_api' | 'workflow'; startedAt: DateTime }
+type EmailQueueWait = { source: 'transactional_api' | 'workflow'; startedAt: DateTime; teamCapped: boolean }
// A transactional API job is created on the email queue, so its creation time is when the caller got its 202. A
// workflow job can join the email queue long after it was created, so its wait starts at the time it became due.
@@ -132,7 +132,7 @@ function emailQueueWait(invocation: CyclotronJobInvocationHogFunction): EmailQue
const source =
invocation.state.globals.event?.event === TRANSACTIONAL_EMAIL_EVENT ? 'transactional_api' : 'workflow'
const startedAt = source === 'transactional_api' ? invocation.queueCreatedAt : invocation.queueDueAt
- return startedAt ? { source, startedAt } : undefined
+ return startedAt ? { source, startedAt, teamCapped: invocation.queueMetadata?.teamCapped === true } : undefined
}
/**
@@ -694,6 +694,7 @@ export class EmailService {
// per-workflow limit above: createInvocationResult cleared queueParameters, and
// without them the rescheduled dequeue resumes the Hog VM and drops the send.
result.invocation.queueParameters = params
+ result.invocation.queueMetadata = { ...invocation.queueMetadata, teamCapped: true }
result.invocation.queueScheduledAt = DateTime.utc().plus({ milliseconds: capDelay.retryDelayMs })
const capRetrySeconds = Math.round(capDelay.retryDelayMs / 1000)
addLog(
@@ -1293,6 +1294,7 @@ export class EmailService {
.labels({
source: queueWait.source,
priority_class: getEmailQueuePriorityClass(result.invocation.hogFunction.metadata),
+ team_capped: queueWait.teamCapped ? 'true' : 'false',
})
.observe(Math.max(0, Date.now() - queueWait.startedAt.toMillis()))
}