Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(cdp): answer transactional email sends without waiting on kafka (#125444)

Harley Alexanderb30cac61768a
diff --git a/nodejs/src/cdp/cdp-api.serial.test.ts b/nodejs/src/cdp/cdp-api.serial.test.ts
index 165b29fa49a7..5366677985c9 100644
--- a/nodejs/src/cdp/cdp-api.serial.test.ts
+++ b/nodejs/src/cdp/cdp-api.serial.test.ts
@@ -2210,7 +2210,7 @@ describe('CDP API', () => {
})
it('queues one run on the email fast lane with the recipient and the data', async () => {
- const queueAppMetric = jest.spyOn(api['hogFunctionMonitoringService'], 'queueAppMetric')
+ const queueAppMetric = jest.spyOn(api['invocationResultsService'].monitoringService, 'queueAppMetric')
const queuedBefore = await outcomeCount('queued')
const stagesBefore = await stageCounts()
@@ -2240,18 +2240,45 @@ describe('CDP API', () => {
state: {
event: { distinct_id: 'distinct_A_1', properties: { to: '[email protected]' } },
variables: { reset_link: 'https://example.com/reset' },
+ triggerMetricPending: true,
},
})
- // The run counts as started in the workflow's metrics, like an event-triggered run.
- expect(queueAppMetric).toHaveBeenCalledWith(
- expect.objectContaining({ app_source_id: hogFlow.id, metric_name: 'triggered', count: 1 }),
- 'hog_flow'
+ // The email worker counts the run as triggered, so a count here would count it twice.
+ expect(queueAppMetric).not.toHaveBeenCalledWith(
+ expect.objectContaining({ metric_name: 'triggered' }),
+ expect.anything()
)
expect(emailStatusRows()).toEqual([
expect.objectContaining({ team_id: team.id, message_id: res.body.invocation_id, status: 'queued' }),
])
})
+ it('replies before Kafka acks the result and status rows', async () => {
+ let ackProduce: () => void = () => {}
+ const acked = new Promise<void>((resolve) => (ackProduce = resolve))
+ produceOutput.mockImplementation(() => acked)
+ const resultsOutputs = api['invocationResultsService'].invocationResultsRowsService['outputs']
+ const produceResultRow = jest.spyOn(resultsOutputs, 'produce').mockImplementation(() => acked)
+ const resultsEnabled = hub.HOG_INVOCATION_RESULTS_ENABLED
+ hub.HOG_INVOCATION_RESULTS_ENABLED = true
+
+ try {
+ const res = await send(hogFlow.id, { Authorization: `Bearer ${mintToken(team.id, hogFlow.id)}` })
+
+ expect(res.status).toEqual(202)
+ expect(produceResultRow).toHaveBeenCalledWith(
+ 'hog_invocation_results',
+ expect.objectContaining({ key: Buffer.from(res.body.invocation_id) })
+ )
+ expect(emailStatusRows()).toEqual([
+ expect.objectContaining({ message_id: res.body.invocation_id, status: 'queued' }),
+ ])
+ } finally {
+ hub.HOG_INVOCATION_RESULTS_ENABLED = resultsEnabled
+ ackProduce()
+ }
+ })
+
describe('with an idempotency key', () => {
let keyed: Record<string, unknown>
const sendKeyed = (payload: Record<string, unknown> = keyed) =>
diff --git a/nodejs/src/cdp/cdp-api.ts b/nodejs/src/cdp/cdp-api.ts
index 5bce94f57786..bbde6e02e343 100644
--- a/nodejs/src/cdp/cdp-api.ts
+++ b/nodejs/src/cdp/cdp-api.ts
@@ -71,7 +71,6 @@ import { EmailTrackingService } from './services/messaging/email-tracking.servic
import { EmailValidationService } from './services/messaging/email-validation.service'
import { EmailTrackingCodeSigner } from './services/messaging/helpers/tracking-code'
import { RecipientTokensService } from './services/messaging/recipient-tokens.service'
-import { HogFunctionMonitoringService } from './services/monitoring/hog-function-monitoring.service'
import {
HogWatcherService,
HogWatcherState,
@@ -203,7 +202,6 @@ export class CdpApi {
private emailSuppressionService: EmailSuppressionService
private emailValidationService: EmailValidationService
private teamWorkflowsConfigService: TeamWorkflowsConfigService
- private hogFunctionMonitoringService: HogFunctionMonitoringService
private emailStatusService: EmailStatusService
private resumeBatchJwt: ScopedServiceJwt
@@ -245,7 +243,6 @@ export class CdpApi {
jobQueues,
new IdempotencyKeys(deps.idempotencyValkey ?? null, 'workflow-webhook')
)
- this.hogFunctionMonitoringService = services.hogFunctionMonitoringService
this.emailTrackingService = new EmailTrackingService(
this.hogFunctionManager,
this.hogFlowManager,
@@ -1221,6 +1218,9 @@ export class CdpApi {
const filterGlobals = convertToHogFunctionFilterGlobal({ event, person: undefined, groups: {}, variables })
const invocation = createHogFlowInvocation(triggerGlobals, hogFlow, filterGlobals)
+ // The run skips the trigger pipeline that counts runs for event-triggered workflows, so the email
+ // worker counts it instead.
+ invocation.state.triggerMetricPending = true
// Straight to the email queue: the flow's only step is the email, so a hop through the
// workflow queue would only add latency to a password reset or a verification code.
const priorityClass: EmailQueuePriorityClass = 'fast'
@@ -1282,24 +1282,15 @@ export class CdpApi {
}
stages.start('metrics_flush')
this.emailStatusService.queue({ teamId: team.id, messageId: invocation.id, status: 'queued' })
- // Flushed together, and bounded, so the best-effort status row can't hold up the 202.
- await Promise.all([
- this.invocationResultsService.invocationResultsRowsService.flush(),
- this.emailStatusService.flush(),
- ])
- // The run skips the trigger pipeline that counts runs for event-triggered workflows.
- this.hogFunctionMonitoringService.queueAppMetric(
- {
- team_id: team.id,
- app_source_id: hogFlow.id,
- metric_kind: 'other',
- metric_name: 'triggered',
- count: 1,
- app_source_version: { id: hogFlow.id, version: hogFlow.version },
- },
- 'hog_flow'
- )
- await this.hogFunctionMonitoringService.flush()
+ // Best-effort rows, so the 202 waits for the hand-off to Kafka (flushed on shutdown) but not the acks.
+ const warnTelemetryFlushFailed = (error: unknown) =>
+ logger.warn('⚠️', 'Transactional email telemetry flush failed', {
+ error: error instanceof Error ? error.message : String(error),
+ invocation_id: invocation.id,
+ })
+ void this.emailStatusService.flush().catch(warnTelemetryFlushFailed)
+ const { produced } = await this.invocationResultsService.invocationResultsRowsService.handOffQueuedRows()
+ void produced.catch(warnTelemetryFlushFailed)
return reply('queued', 202, { status: 'queued', invocation_id: invocation.id, replayed: false })
} catch (e) {
diff --git a/nodejs/src/cdp/consumers/cdp-cyclotron-worker-email.consumer.test.ts b/nodejs/src/cdp/consumers/cdp-cyclotron-worker-email.consumer.test.ts
index ac61b9e062b4..e48ed6a0f72c 100644
--- a/nodejs/src/cdp/consumers/cdp-cyclotron-worker-email.consumer.test.ts
+++ b/nodejs/src/cdp/consumers/cdp-cyclotron-worker-email.consumer.test.ts
@@ -1,9 +1,16 @@
import { createMockJobQueue } from '~/tests/helpers/mocks/job-queue.mock'
+import { HogFlow } from '~/cdp/schema/hogflow'
import { closeHub, createHub } from '~/common/utils/db/hub'
+import { parseJSON } from '~/common/utils/json-parse'
+import { UUIDT } from '~/common/utils/utils'
import { createCdpConsumerDeps } from '~/tests/helpers/cdp'
+import { createTestTeamFixture } from '~/tests/helpers/sql'
import { Hub } from '../../types'
+import { FixtureHogFlowBuilder } from '../_tests/builders/hogflow.builder'
+import { createHogFlowInvocationContext, insertHogFlow } from '../_tests/fixtures-hogflows'
+import { CyclotronJobInvocation, CyclotronJobInvocationResult } from '../types'
import { CdpCyclotronWorkerEmail } from './cdp-cyclotron-worker-email.consumer'
jest.setTimeout(5000)
@@ -28,4 +35,68 @@ describe('CdpCyclotronWorkerEmail', () => {
const worker = new CdpCyclotronWorkerEmail(hub, createCdpConsumerDeps(hub), createMockJobQueue())
expect(worker['name']).toBe('CdpCyclotronWorkerEmail')
})
+
+ it.each([
+ ['counts a job the transactional email API marked once, not again on its reschedule', true, [1, 0]],
+ ['does not count a job without the mark', false, [0, 0]],
+ ])('%s', async (_desc, marked, expectedTriggeredPerRun) => {
+ const { team } = await createTestTeamFixture(hub.postgres)
+ // The delay parks the run, so the second dequeue is a reschedule of the same job.
+ const hogFlow: HogFlow = await insertHogFlow(
+ hub.postgres,
+ new FixtureHogFlowBuilder()
+ .withTeamId(team.id)
+ .withStatus('active')
+ .withWorkflow({
+ actions: {
+ trigger: { type: 'trigger', config: { type: 'api' } },
+ delay: { type: 'delay', config: { delay_duration: '1h' } },
+ exit: { type: 'exit', config: {} },
+ },
+ edges: [
+ { from: 'trigger', to: 'delay', type: 'continue' },
+ { from: 'delay', to: 'exit', type: 'continue' },
+ ],
+ } as any)
+ .build()
+ )
+ const worker = new CdpCyclotronWorkerEmail(hub, createCdpConsumerDeps(hub), createMockJobQueue())
+ const triggered = (result: CyclotronJobInvocationResult) =>
+ result.metrics.filter((metric) => metric.metric_name === 'triggered')
+
+ let job: CyclotronJobInvocation = {
+ id: new UUIDT().toString(),
+ teamId: team.id,
+ functionId: hogFlow.id,
+ queue: 'email',
+ queuePriority: 0,
+ state: createHogFlowInvocationContext({
+ event: { distinct_id: 'distinct_A_1', properties: { to: '[email protected]' } } as any,
+ ...(marked ? { triggerMetricPending: true } : {}),
+ }),
+ }
+ const triggeredPerRun: CyclotronJobInvocationResult['metrics'][] = []
+ for (let run = 0; run < 2; run++) {
+ const [result] = await worker.processInvocations([job])
+ expect(result.finished).toBe(false)
+ triggeredPerRun.push(triggered(result))
+ // The job queue persists the result's state as JSON, and the next dequeue reads it back.
+ job = { ...job, state: parseJSON(JSON.stringify(result.invocation.state)) }
+ }
+
+ expect(triggeredPerRun.map((metrics) => metrics.length)).toEqual(expectedTriggeredPerRun)
+ expect(triggeredPerRun.flat()).toEqual(
+ marked
+ ? [
+ {
+ team_id: team.id,
+ app_source_id: hogFlow.id,
+ metric_kind: 'other',
+ metric_name: 'triggered',
+ count: 1,
+ },
+ ]
+ : []
+ )
+ })
})
diff --git a/nodejs/src/cdp/consumers/cdp-cyclotron-worker-email.consumer.ts b/nodejs/src/cdp/consumers/cdp-cyclotron-worker-email.consumer.ts
index 276a5a4f63a9..42938bedc88b 100644
--- a/nodejs/src/cdp/consumers/cdp-cyclotron-worker-email.consumer.ts
+++ b/nodejs/src/cdp/consumers/cdp-cyclotron-worker-email.consumer.ts
@@ -1,6 +1,7 @@
import { PluginsServerConfig } from '~/types'
import { JobQueue } from '../services/job-queue/job-queue.interface'
+import { CyclotronJobInvocation, CyclotronJobInvocationHogFlow, CyclotronJobInvocationResult } from '../types'
import { CdpConsumerBaseDeps } from './cdp-base.consumer'
import { CdpCyclotronWorkerHogFlow } from './cdp-cyclotron-worker-hogflow.consumer'
@@ -11,4 +12,30 @@ export class CdpCyclotronWorkerEmail extends CdpCyclotronWorkerHogFlow {
super(config, deps, jobQueue)
this.queue = 'email'
}
+
+ public override async processInvocations(
+ invocations: CyclotronJobInvocation[]
+ ): Promise<CyclotronJobInvocationResult[]> {
+ const results = await super.processInvocations(invocations)
+ results.forEach(countPendingTrigger)
+ return results
+ }
+}
+
+// The transactional email API queues its runs straight onto this queue, so this worker counts them as triggered.
+// The flag is cleared on the state that the job queue persists, so a throttled or rate-limited reschedule of the
+// same job does not count it again. A canceled run counts too, because the API accepted it.
+function countPendingTrigger(result: CyclotronJobInvocationResult): void {
+ const state = result.invocation.state as CyclotronJobInvocationHogFlow['state'] | null | undefined
+ if (!state?.triggerMetricPending) {
+ return
+ }
+ delete state.triggerMetricPending
+ result.metrics.push({
+ team_id: result.invocation.teamId,
+ app_source_id: result.invocation.functionId,
+ metric_kind: 'other',
+ metric_name: 'triggered',
+ count: 1,
+ })
}
diff --git a/nodejs/src/cdp/services/hogflows/transactional-email.ts b/nodejs/src/cdp/services/hogflows/transactional-email.ts
index c24b3b4bf6ba..e1671bf1afd1 100644
--- a/nodejs/src/cdp/services/hogflows/transactional-email.ts
+++ b/nodejs/src/cdp/services/hogflows/transactional-email.ts
@@ -50,7 +50,7 @@ export type TransactionalEmailStage = (typeof TRANSACTIONAL_EMAIL_STAGES)[number
export const transactionalEmailStageDurationMs = new Histogram({
name: 'cdp_transactional_email_stage_duration_ms',
- help: 'Milliseconds one transactional email request spends in each stage: auth (the JWT), validate (the body), load (the team and the workflow), build (the run and its idempotency fingerprint), idempotency (every Idempotency-Key call to Valkey), checks (suspension, pause, suppression, quota and MX), queue_insert (the email queue insert), metrics_flush (the result, status and app metric writes), and total. A request records only the stages it reached, so a refusal shows in the stage that refused it.',
+ help: 'Milliseconds one transactional email request spends in each stage: auth (the JWT), validate (the body), load (the team and the workflow), build (the run and its idempotency fingerprint), idempotency (every Idempotency-Key call to Valkey), checks (suspension, pause, suppression, quota and MX), queue_insert (the email queue insert), metrics_flush (the hand-off of the result and status rows to the Kafka producer, without the wait for the broker), and total. A request records only the stages it reached, so a refusal shows in the stage that refused it.',
labelNames: ['stage'],
buckets: [1, 2.5, 5, 10, 25, 50, 100, 250, 500, 1_000, 2_500, 5_000, 10_000],
})
diff --git a/nodejs/src/cdp/services/monitoring/hog-invocation-results.service.ts b/nodejs/src/cdp/services/monitoring/hog-invocation-results.service.ts
index 3fd19719bca8..056a370f91b5 100644
--- a/nodejs/src/cdp/services/monitoring/hog-invocation-results.service.ts
+++ b/nodejs/src/cdp/services/monitoring/hog-invocation-results.service.ts
@@ -490,7 +490,11 @@ export class HogInvocationResultsService {
}
private async produceRow(row: HogInvocationResultRow): Promise<void> {
- const value = Buffer.from(
+ await this.produceEncodedRow(row, await this.encodeRow(row))
+ }
+
+ private async encodeRow(row: HogInvocationResultRow): Promise<Buffer> {
+ return Buffer.from(
safeClickhouseString(
JSON.stringify({
...row,
@@ -498,6 +502,9 @@ export class HogInvocationResultsService {
})
)
)
+ }
+
+ private async produceEncodedRow(row: HogInvocationResultRow, value: Buffer): Promise<void> {
await this.outputs.produce(HOG_INVOCATION_RESULTS_OUTPUT, {
// Partition by invocation_id so all rows for a single invocation
// land on the same Kafka partition (and the same ClickHouse shard).
@@ -618,27 +625,49 @@ export class HogInvocationResultsService {
}
async flush(): Promise<void> {
+ const { produced } = await this.handOffQueuedRows()
+ await produced
+ }
+
+ // Resolves once every queued row is encoded and passed to the Kafka producer, which flushes the rows it
+ // holds on shutdown. `produced` resolves when the broker has acked them, so a caller that must not wait
+ // on the broker awaits only the hand-off. Neither promise rejects.
+ async handOffQueuedRows(): Promise<{ produced: Promise<void> }> {
if (this.queuedRows.length === 0) {
- return
+ return { produced: Promise.resolve() }
}
const rows = this.queuedRows
this.queuedRows = []
hogInvocationResultsPendingMessages.set(0)
- await Promise.all(
+ const encoded = await Promise.all(
rows.map((row) =>
- this.produceRow(row).catch((error) => {
- counterHogInvocationResultProduceFailed.inc()
- // Best-effort — never disrupt invocation processing for a
- // monitoring write.
- logger.error('⚠️', `failed to produce hog invocation result: ${error}`, {
- error: String(error),
- invocation_id: row.invocation_id,
- })
- captureException(error)
+ this.encodeRow(row).catch((error) => {
+ this.onRowProduceFailed(row, error)
+ return null
})
)
)
+ const produced = Promise.all(
+ rows.flatMap((row, i) => {
+ const value = encoded[i]
+ return value
+ ? [this.produceEncodedRow(row, value).catch((error) => this.onRowProduceFailed(row, error))]
+ : []
+ })
+ ).then(() => undefined)
+ return { produced }
+ }
+
+ private onRowProduceFailed(row: HogInvocationResultRow, error: unknown): void {
+ counterHogInvocationResultProduceFailed.inc()
+ // Best-effort — never disrupt invocation processing for a
+ // monitoring write.
+ logger.error('⚠️', `failed to produce hog invocation result: ${error}`, {
+ error: String(error),
+ invocation_id: row.invocation_id,
+ })
+ captureException(error)
}
}
diff --git a/nodejs/src/cdp/types.ts b/nodejs/src/cdp/types.ts
index b59606543a4a..c8d5575c9c3a 100644
--- a/nodejs/src/cdp/types.ts
+++ b/nodejs/src/cdp/types.ts
@@ -486,6 +486,9 @@ export type HogFlowInvocationContext = {
// Stamped at run start and persisted with the state, so a recovery path that holds only the
// workflow id still keeps a sensitive run's message data out of its snapshot.
sensitive?: boolean
+ // Set by the transactional email API, which skips the trigger pipeline. The email worker counts the run as
+ // `triggered` and clears this flag, so a rescheduled run does not count again.
+ triggerMetricPending?: boolean
actionStepCount: number
// Missing on legacy runs, which must keep run:action keys even when no function state was persisted.
customerTaskIdempotencyVersion?: 1