:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(cdp): answer transactional email sends without waiting on kafka (#125444)
Harley Alexanderb30cac61768a
diff --git a/nodejs/src/cdp/cdp-api.serial.test.ts b/nodejs/src/cdp/cdp-api.serial.test.tsindex 165b29fa49a7..5366677985c9 100644--- a/nodejs/src/cdp/cdp-api.serial.test.ts+++ b/nodejs/src/cdp/cdp-api.serial.test.ts@@ -2210,7 +2210,7 @@ describe('CDP API', () => {})it('queues one run on the email fast lane with the recipient and the data', async () => {- const queueAppMetric = jest.spyOn(api['hogFunctionMonitoringService'], 'queueAppMetric')+ const queueAppMetric = jest.spyOn(api['invocationResultsService'].monitoringService, 'queueAppMetric')const queuedBefore = await outcomeCount('queued')const stagesBefore = await stageCounts()@@ -2240,18 +2240,45 @@ describe('CDP API', () => {state: {event: { distinct_id: 'distinct_A_1', properties: { to: '[email protected]' } },variables: { reset_link: 'https://example.com/reset' },+ triggerMetricPending: true,},})- // The run counts as started in the workflow's metrics, like an event-triggered run.- expect(queueAppMetric).toHaveBeenCalledWith(- expect.objectContaining({ app_source_id: hogFlow.id, metric_name: 'triggered', count: 1 }),- 'hog_flow'+ // The email worker counts the run as triggered, so a count here would count it twice.+ expect(queueAppMetric).not.toHaveBeenCalledWith(+ expect.objectContaining({ metric_name: 'triggered' }),+ expect.anything())expect(emailStatusRows()).toEqual([expect.objectContaining({ team_id: team.id, message_id: res.body.invocation_id, status: 'queued' }),])})+ it('replies before Kafka acks the result and status rows', async () => {+ let ackProduce: () => void = () => {}+ const acked = new Promise<void>((resolve) => (ackProduce = resolve))+ produceOutput.mockImplementation(() => acked)+ const resultsOutputs = api['invocationResultsService'].invocationResultsRowsService['outputs']+ const produceResultRow = jest.spyOn(resultsOutputs, 'produce').mockImplementation(() => acked)+ const resultsEnabled = hub.HOG_INVOCATION_RESULTS_ENABLED+ hub.HOG_INVOCATION_RESULTS_ENABLED = true++ try {+ const res = await send(hogFlow.id, { Authorization: `Bearer ${mintToken(team.id, hogFlow.id)}` })++ expect(res.status).toEqual(202)+ expect(produceResultRow).toHaveBeenCalledWith(+ 'hog_invocation_results',+ expect.objectContaining({ key: Buffer.from(res.body.invocation_id) })+ )+ expect(emailStatusRows()).toEqual([+ expect.objectContaining({ message_id: res.body.invocation_id, status: 'queued' }),+ ])+ } finally {+ hub.HOG_INVOCATION_RESULTS_ENABLED = resultsEnabled+ ackProduce()+ }+ })+describe('with an idempotency key', () => {let keyed: Record<string, unknown>const sendKeyed = (payload: Record<string, unknown> = keyed) =>diff --git a/nodejs/src/cdp/cdp-api.ts b/nodejs/src/cdp/cdp-api.tsindex 5bce94f57786..bbde6e02e343 100644--- a/nodejs/src/cdp/cdp-api.ts+++ b/nodejs/src/cdp/cdp-api.ts@@ -71,7 +71,6 @@ import { EmailTrackingService } from './services/messaging/email-tracking.servicimport { EmailValidationService } from './services/messaging/email-validation.service'import { EmailTrackingCodeSigner } from './services/messaging/helpers/tracking-code'import { RecipientTokensService } from './services/messaging/recipient-tokens.service'-import { HogFunctionMonitoringService } from './services/monitoring/hog-function-monitoring.service'import {HogWatcherService,HogWatcherState,@@ -203,7 +202,6 @@ export class CdpApi {private emailSuppressionService: EmailSuppressionServiceprivate emailValidationService: EmailValidationServiceprivate teamWorkflowsConfigService: TeamWorkflowsConfigService- private hogFunctionMonitoringService: HogFunctionMonitoringServiceprivate emailStatusService: EmailStatusServiceprivate resumeBatchJwt: ScopedServiceJwt@@ -245,7 +243,6 @@ export class CdpApi {jobQueues,new IdempotencyKeys(deps.idempotencyValkey ?? null, 'workflow-webhook'))- this.hogFunctionMonitoringService = services.hogFunctionMonitoringServicethis.emailTrackingService = new EmailTrackingService(this.hogFunctionManager,this.hogFlowManager,@@ -1221,6 +1218,9 @@ export class CdpApi {const filterGlobals = convertToHogFunctionFilterGlobal({ event, person: undefined, groups: {}, variables })const invocation = createHogFlowInvocation(triggerGlobals, hogFlow, filterGlobals)+ // The run skips the trigger pipeline that counts runs for event-triggered workflows, so the email+ // worker counts it instead.+ invocation.state.triggerMetricPending = true// Straight to the email queue: the flow's only step is the email, so a hop through the// workflow queue would only add latency to a password reset or a verification code.const priorityClass: EmailQueuePriorityClass = 'fast'@@ -1282,24 +1282,15 @@ export class CdpApi {}stages.start('metrics_flush')this.emailStatusService.queue({ teamId: team.id, messageId: invocation.id, status: 'queued' })- // Flushed together, and bounded, so the best-effort status row can't hold up the 202.- await Promise.all([- this.invocationResultsService.invocationResultsRowsService.flush(),- this.emailStatusService.flush(),- ])- // The run skips the trigger pipeline that counts runs for event-triggered workflows.- this.hogFunctionMonitoringService.queueAppMetric(- {- team_id: team.id,- app_source_id: hogFlow.id,- metric_kind: 'other',- metric_name: 'triggered',- count: 1,- app_source_version: { id: hogFlow.id, version: hogFlow.version },- },- 'hog_flow'- )- await this.hogFunctionMonitoringService.flush()+ // Best-effort rows, so the 202 waits for the hand-off to Kafka (flushed on shutdown) but not the acks.+ const warnTelemetryFlushFailed = (error: unknown) =>+ logger.warn('⚠️', 'Transactional email telemetry flush failed', {+ error: error instanceof Error ? error.message : String(error),+ invocation_id: invocation.id,+ })+ void this.emailStatusService.flush().catch(warnTelemetryFlushFailed)+ const { produced } = await this.invocationResultsService.invocationResultsRowsService.handOffQueuedRows()+ void produced.catch(warnTelemetryFlushFailed)return reply('queued', 202, { status: 'queued', invocation_id: invocation.id, replayed: false })} catch (e) {diff --git a/nodejs/src/cdp/consumers/cdp-cyclotron-worker-email.consumer.test.ts b/nodejs/src/cdp/consumers/cdp-cyclotron-worker-email.consumer.test.tsindex ac61b9e062b4..e48ed6a0f72c 100644--- a/nodejs/src/cdp/consumers/cdp-cyclotron-worker-email.consumer.test.ts+++ b/nodejs/src/cdp/consumers/cdp-cyclotron-worker-email.consumer.test.ts@@ -1,9 +1,16 @@import { createMockJobQueue } from '~/tests/helpers/mocks/job-queue.mock'+import { HogFlow } from '~/cdp/schema/hogflow'import { closeHub, createHub } from '~/common/utils/db/hub'+import { parseJSON } from '~/common/utils/json-parse'+import { UUIDT } from '~/common/utils/utils'import { createCdpConsumerDeps } from '~/tests/helpers/cdp'+import { createTestTeamFixture } from '~/tests/helpers/sql'import { Hub } from '../../types'+import { FixtureHogFlowBuilder } from '../_tests/builders/hogflow.builder'+import { createHogFlowInvocationContext, insertHogFlow } from '../_tests/fixtures-hogflows'+import { CyclotronJobInvocation, CyclotronJobInvocationResult } from '../types'import { CdpCyclotronWorkerEmail } from './cdp-cyclotron-worker-email.consumer'jest.setTimeout(5000)@@ -28,4 +35,68 @@ describe('CdpCyclotronWorkerEmail', () => {const worker = new CdpCyclotronWorkerEmail(hub, createCdpConsumerDeps(hub), createMockJobQueue())expect(worker['name']).toBe('CdpCyclotronWorkerEmail')})++ it.each([+ ['counts a job the transactional email API marked once, not again on its reschedule', true, [1, 0]],+ ['does not count a job without the mark', false, [0, 0]],+ ])('%s', async (_desc, marked, expectedTriggeredPerRun) => {+ const { team } = await createTestTeamFixture(hub.postgres)+ // The delay parks the run, so the second dequeue is a reschedule of the same job.+ const hogFlow: HogFlow = await insertHogFlow(+ hub.postgres,+ new FixtureHogFlowBuilder()+ .withTeamId(team.id)+ .withStatus('active')+ .withWorkflow({+ actions: {+ trigger: { type: 'trigger', config: { type: 'api' } },+ delay: { type: 'delay', config: { delay_duration: '1h' } },+ exit: { type: 'exit', config: {} },+ },+ edges: [+ { from: 'trigger', to: 'delay', type: 'continue' },+ { from: 'delay', to: 'exit', type: 'continue' },+ ],+ } as any)+ .build()+ )+ const worker = new CdpCyclotronWorkerEmail(hub, createCdpConsumerDeps(hub), createMockJobQueue())+ const triggered = (result: CyclotronJobInvocationResult) =>+ result.metrics.filter((metric) => metric.metric_name === 'triggered')++ let job: CyclotronJobInvocation = {+ id: new UUIDT().toString(),+ teamId: team.id,+ functionId: hogFlow.id,+ queue: 'email',+ queuePriority: 0,+ state: createHogFlowInvocationContext({+ event: { distinct_id: 'distinct_A_1', properties: { to: '[email protected]' } } as any,+ ...(marked ? { triggerMetricPending: true } : {}),+ }),+ }+ const triggeredPerRun: CyclotronJobInvocationResult['metrics'][] = []+ for (let run = 0; run < 2; run++) {+ const [result] = await worker.processInvocations([job])+ expect(result.finished).toBe(false)+ triggeredPerRun.push(triggered(result))+ // The job queue persists the result's state as JSON, and the next dequeue reads it back.+ job = { ...job, state: parseJSON(JSON.stringify(result.invocation.state)) }+ }++ expect(triggeredPerRun.map((metrics) => metrics.length)).toEqual(expectedTriggeredPerRun)+ expect(triggeredPerRun.flat()).toEqual(+ marked+ ? [+ {+ team_id: team.id,+ app_source_id: hogFlow.id,+ metric_kind: 'other',+ metric_name: 'triggered',+ count: 1,+ },+ ]+ : []+ )+ })})diff --git a/nodejs/src/cdp/consumers/cdp-cyclotron-worker-email.consumer.ts b/nodejs/src/cdp/consumers/cdp-cyclotron-worker-email.consumer.tsindex 276a5a4f63a9..42938bedc88b 100644--- a/nodejs/src/cdp/consumers/cdp-cyclotron-worker-email.consumer.ts+++ b/nodejs/src/cdp/consumers/cdp-cyclotron-worker-email.consumer.ts@@ -1,6 +1,7 @@import { PluginsServerConfig } from '~/types'import { JobQueue } from '../services/job-queue/job-queue.interface'+import { CyclotronJobInvocation, CyclotronJobInvocationHogFlow, CyclotronJobInvocationResult } from '../types'import { CdpConsumerBaseDeps } from './cdp-base.consumer'import { CdpCyclotronWorkerHogFlow } from './cdp-cyclotron-worker-hogflow.consumer'@@ -11,4 +12,30 @@ export class CdpCyclotronWorkerEmail extends CdpCyclotronWorkerHogFlow {super(config, deps, jobQueue)this.queue = 'email'}++ public override async processInvocations(+ invocations: CyclotronJobInvocation[]+ ): Promise<CyclotronJobInvocationResult[]> {+ const results = await super.processInvocations(invocations)+ results.forEach(countPendingTrigger)+ return results+ }+}++// The transactional email API queues its runs straight onto this queue, so this worker counts them as triggered.+// The flag is cleared on the state that the job queue persists, so a throttled or rate-limited reschedule of the+// same job does not count it again. A canceled run counts too, because the API accepted it.+function countPendingTrigger(result: CyclotronJobInvocationResult): void {+ const state = result.invocation.state as CyclotronJobInvocationHogFlow['state'] | null | undefined+ if (!state?.triggerMetricPending) {+ return+ }+ delete state.triggerMetricPending+ result.metrics.push({+ team_id: result.invocation.teamId,+ app_source_id: result.invocation.functionId,+ metric_kind: 'other',+ metric_name: 'triggered',+ count: 1,+ })}diff --git a/nodejs/src/cdp/services/hogflows/transactional-email.ts b/nodejs/src/cdp/services/hogflows/transactional-email.tsindex c24b3b4bf6ba..e1671bf1afd1 100644--- a/nodejs/src/cdp/services/hogflows/transactional-email.ts+++ b/nodejs/src/cdp/services/hogflows/transactional-email.ts@@ -50,7 +50,7 @@ export type TransactionalEmailStage = (typeof TRANSACTIONAL_EMAIL_STAGES)[numberexport const transactionalEmailStageDurationMs = new Histogram({name: 'cdp_transactional_email_stage_duration_ms',- help: 'Milliseconds one transactional email request spends in each stage: auth (the JWT), validate (the body), load (the team and the workflow), build (the run and its idempotency fingerprint), idempotency (every Idempotency-Key call to Valkey), checks (suspension, pause, suppression, quota and MX), queue_insert (the email queue insert), metrics_flush (the result, status and app metric writes), and total. A request records only the stages it reached, so a refusal shows in the stage that refused it.',+ help: 'Milliseconds one transactional email request spends in each stage: auth (the JWT), validate (the body), load (the team and the workflow), build (the run and its idempotency fingerprint), idempotency (every Idempotency-Key call to Valkey), checks (suspension, pause, suppression, quota and MX), queue_insert (the email queue insert), metrics_flush (the hand-off of the result and status rows to the Kafka producer, without the wait for the broker), and total. A request records only the stages it reached, so a refusal shows in the stage that refused it.',labelNames: ['stage'],buckets: [1, 2.5, 5, 10, 25, 50, 100, 250, 500, 1_000, 2_500, 5_000, 10_000],})diff --git a/nodejs/src/cdp/services/monitoring/hog-invocation-results.service.ts b/nodejs/src/cdp/services/monitoring/hog-invocation-results.service.tsindex 3fd19719bca8..056a370f91b5 100644--- a/nodejs/src/cdp/services/monitoring/hog-invocation-results.service.ts+++ b/nodejs/src/cdp/services/monitoring/hog-invocation-results.service.ts@@ -490,7 +490,11 @@ export class HogInvocationResultsService {}private async produceRow(row: HogInvocationResultRow): Promise<void> {- const value = Buffer.from(+ await this.produceEncodedRow(row, await this.encodeRow(row))+ }++ private async encodeRow(row: HogInvocationResultRow): Promise<Buffer> {+ return Buffer.from(safeClickhouseString(JSON.stringify({...row,@@ -498,6 +502,9 @@ export class HogInvocationResultsService {})))+ }++ private async produceEncodedRow(row: HogInvocationResultRow, value: Buffer): Promise<void> {await this.outputs.produce(HOG_INVOCATION_RESULTS_OUTPUT, {// Partition by invocation_id so all rows for a single invocation// land on the same Kafka partition (and the same ClickHouse shard).@@ -618,27 +625,49 @@ export class HogInvocationResultsService {}async flush(): Promise<void> {+ const { produced } = await this.handOffQueuedRows()+ await produced+ }++ // Resolves once every queued row is encoded and passed to the Kafka producer, which flushes the rows it+ // holds on shutdown. `produced` resolves when the broker has acked them, so a caller that must not wait+ // on the broker awaits only the hand-off. Neither promise rejects.+ async handOffQueuedRows(): Promise<{ produced: Promise<void> }> {if (this.queuedRows.length === 0) {- return+ return { produced: Promise.resolve() }}const rows = this.queuedRowsthis.queuedRows = []hogInvocationResultsPendingMessages.set(0)- await Promise.all(+ const encoded = await Promise.all(rows.map((row) =>- this.produceRow(row).catch((error) => {- counterHogInvocationResultProduceFailed.inc()- // Best-effort — never disrupt invocation processing for a- // monitoring write.- logger.error('⚠️', `failed to produce hog invocation result: ${error}`, {- error: String(error),- invocation_id: row.invocation_id,- })- captureException(error)+ this.encodeRow(row).catch((error) => {+ this.onRowProduceFailed(row, error)+ return null})))+ const produced = Promise.all(+ rows.flatMap((row, i) => {+ const value = encoded[i]+ return value+ ? [this.produceEncodedRow(row, value).catch((error) => this.onRowProduceFailed(row, error))]+ : []+ })+ ).then(() => undefined)+ return { produced }+ }++ private onRowProduceFailed(row: HogInvocationResultRow, error: unknown): void {+ counterHogInvocationResultProduceFailed.inc()+ // Best-effort — never disrupt invocation processing for a+ // monitoring write.+ logger.error('⚠️', `failed to produce hog invocation result: ${error}`, {+ error: String(error),+ invocation_id: row.invocation_id,+ })+ captureException(error)}}diff --git a/nodejs/src/cdp/types.ts b/nodejs/src/cdp/types.tsindex b59606543a4a..c8d5575c9c3a 100644--- a/nodejs/src/cdp/types.ts+++ b/nodejs/src/cdp/types.ts@@ -486,6 +486,9 @@ export type HogFlowInvocationContext = {// Stamped at run start and persisted with the state, so a recovery path that holds only the// workflow id still keeps a sensitive run's message data out of its snapshot.sensitive?: boolean+ // Set by the transactional email API, which skips the trigger pipeline. The email worker counts the run as+ // `triggered` and clears this flag, so a rescheduled run does not count again.+ triggerMetricPending?: booleanactionStepCount: number// Missing on legacy runs, which must keep run:action keys even when no function state was persisted.customerTaskIdempotencyVersion?: 1