:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(cdp): deliver a code send's content on its own transactional email route (#127122)
Michael Reichenbacha69f8ef98b9a
diff --git a/nodejs/src/cdp/_tests/fixtures-hogflows.ts b/nodejs/src/cdp/_tests/fixtures-hogflows.tsindex fc7d6858397b..3d48cc3d3ec9 100644--- a/nodejs/src/cdp/_tests/fixtures-hogflows.ts+++ b/nodejs/src/cdp/_tests/fixtures-hogflows.ts@@ -1,5 +1,5 @@import { HogFlow } from '~/cdp/schema/hogflow'-import { PostgresRouter } from '~/common/utils/db/postgres'+import { PostgresRouter, PostgresUse } from '~/common/utils/db/postgres'import { UUIDT } from '~/common/utils/utils'import { insertRow } from '~/tests/helpers/sql'@@ -20,6 +20,20 @@ export const insertHogFlow = async (postgres: PostgresRouter, hogFlow: HogFlow):return res}+export const hideHogFlowsFromReplica = (+ postgres: PostgresRouter,+ ids: HogFlow['id'][]+): jest.SpyInstance<ReturnType<PostgresRouter['query']>, Parameters<PostgresRouter['query']>> => {+ const query = postgres.query.bind(postgres)+ return jest.spyOn(postgres, 'query').mockImplementation(async (target, queryString, values, tag, logLevel) => {+ const result = await query(target, queryString, values, tag, logLevel)+ if (target === PostgresUse.COMMON_READ && tag === 'fetchHogFlows') {+ return { ...result, rows: result.rows.filter((row) => !ids.includes(row.id)) }+ }+ return result+ })+}+export const createHogFlowInvocationContext = (data: Partial<HogFlowInvocationContext> = {}): HogFlowInvocationContext => {diff --git a/nodejs/src/cdp/async-functions/send-email.ts b/nodejs/src/cdp/async-functions/send-email.tsindex df495c5b3aa8..7b52a3be7edc 100644--- a/nodejs/src/cdp/async-functions/send-email.ts+++ b/nodejs/src/cdp/async-functions/send-email.ts@@ -1,13 +1,15 @@import { DateTime } from 'luxon'import { CyclotronInvocationQueueParametersEmailSchema } from '~/cdp/schema/cyclotron'+import { acceptedSendEmailFields } from '~/cdp/services/hogflows/transactional-email'import { registerAsyncFunction } from '../async-function-registry'registerAsyncFunction('sendEmail', {- execute: (args, _context, result) => {+ execute: (args, context, result) => {result.invocation.queueParameters = CyclotronInvocationQueueParametersEmailSchema.parse({...args[0],+ ...acceptedSendEmailFields(context.invocation.state.send),type: 'email',})},diff --git a/nodejs/src/cdp/cdp-api.serial.test.ts b/nodejs/src/cdp/cdp-api.serial.test.tsindex 288a88ffb517..394cfad1b36e 100644--- a/nodejs/src/cdp/cdp-api.serial.test.ts+++ b/nodejs/src/cdp/cdp-api.serial.test.ts@@ -32,7 +32,7 @@ import {insertHogFunctionTemplate,insertIntegration,} from './_tests/fixtures'-import { insertHogFlow as _insertHogFlow } from './_tests/fixtures-hogflows'+import { insertHogFlow as _insertHogFlow, hideHogFlowsFromReplica } from './_tests/fixtures-hogflows'import { CdpApi } from './cdp-api'import { CdpConsumerBaseDeps } from './consumers/cdp-base.consumer'import { posthogFilterOutPlugin } from './legacy-plugins/_transformations/posthog-filter-out-plugin/template'@@ -2145,8 +2145,11 @@ describe('CDP API', () => {{secret = 'local-dev-workflows-transactional-email-jwt',audience = 'posthog:workflows:send_transactional_email',- } = {}- ) => jwt.sign({ team_id: teamId, hog_flow_id: hogFlowId }, secret, { audience, expiresIn: '2m' })+ claims = {},+ }: { secret?: string; audience?: string; claims?: Record<string, string> } = {}+ ) => jwt.sign({ team_id: teamId, hog_flow_id: hogFlowId, ...claims }, secret, { audience, expiresIn: '2m' })+ const tokenFor = (route: string, teamId: number, hogFlowId: string) =>+ mintToken(teamId, hogFlowId, route.endsWith('/send') ? { claims: { kind: 'code_send' } } : {})const transactionalFlow = (trigger: 'api' | 'schedule',@@ -2197,6 +2200,26 @@ describe('CDP API', () => {.filter(([output]) => output === 'email_status').map(([, message]) => parseJSON(message.value.toString()))+ const sendAccepted = (flowId: string, payload: Record<string, unknown>) =>+ supertest(app)+ .post(`/api/projects/${team.id}/hog_flows/${flowId}/transactional_email/send`)+ .set({ Authorization: `Bearer ${tokenFor('transactional_email/send', team.id, flowId)}` })+ .send(payload)++ const sendOn = (+ route: string,+ flowId: string,+ headers: Record<string, string>,+ payload: Record<string, unknown>+ ) => supertest(app).post(`/api/projects/${team.id}/hog_flows/${flowId}/${route}`).set(headers).send(payload)++ const codeSend = {+ content: { subject: 'Reset your password', html: '<p>Hi {{ variables.name }}</p>', text: 'Hi' },+ from: { integrationId: 12, email: '[email protected]', name: 'Example' },+ replyTo: ['[email protected]'],+ tracking: { captureEngagement: true },+ }+beforeEach(async () => {produceOutput = jest.spyOn(api['emailStatusService']['outputs'], 'produce')mockQueueInvocations = jest.fn().mockResolvedValue(undefined)@@ -2284,39 +2307,69 @@ describe('CDP API', () => {let keyed: Record<string, unknown>const sendKeyed = (payload: Record<string, unknown> = keyed) =>send(hogFlow.id, { Authorization: `Bearer ${mintToken(team.id, hogFlow.id)}` }, payload)+ const sendKeyedOn = (route: 'transactional_email' | 'transactional_email/send', payload: object) =>+ supertest(app)+ .post(`/api/projects/${team.id}/hog_flows/${hogFlow.id}/${route}`)+ .set({ Authorization: `Bearer ${tokenFor(route, team.id, hogFlow.id)}` })+ .send({ ...payload, idempotency_key: keyed.idempotency_key })+ const changedContent = { ...codeSend, content: { ...codeSend.content, subject: 'Your new code' } }beforeEach(() => {- // A fresh key per test, so no test reads a key another one left in Valkey.keyed = { ...body, idempotency_key: `key-${new UUIDT().toString()}` }})- it('returns the first send for a retry and queues nothing new', async () => {- const first = await sendKeyed()- const retry = await sendKeyed()-- expect(first.status).toEqual(202)- expect(first.body).toMatchObject({ replayed: false })- expect(retry.status).toEqual(202)- expect(retry.body).toEqual({- status: 'queued',- invocation_id: first.body.invocation_id,- replayed: true,- })- expect(mockQueueInvocations).toHaveBeenCalledTimes(1)- expect(emailStatusRows()).toEqual([- expect.objectContaining({ message_id: first.body.invocation_id, status: 'queued' }),- ])- })+ it.each([+ ['a template send', 'transactional_email', body],+ ['a code send', 'transactional_email/send', { ...body, send: codeSend }],+ ] as const)(+ 'returns the first send for a retry of %s and queues nothing new',+ async (_desc, route, payload) => {+ const first = await sendKeyedOn(route, payload)+ const retry = await sendKeyedOn(route, payload)++ expect(first.status).toEqual(202)+ expect(first.body).toMatchObject({ replayed: false })+ expect(retry.status).toEqual(202)+ expect(retry.body).toEqual({+ status: 'queued',+ invocation_id: first.body.invocation_id,+ replayed: true,+ })+ expect(mockQueueInvocations).toHaveBeenCalledTimes(1)+ expect(emailStatusRows()).toEqual([+ expect.objectContaining({ message_id: first.body.invocation_id, status: 'queued' }),+ ])+ }+ )- it('rejects the key for a different request', async () => {- await sendKeyed()+ it.each([+ [+ 'another recipient',+ ['transactional_email', body],+ ['transactional_email', { ...body, to: '[email protected]' }],+ ],+ [+ 'changed content',+ ['transactional_email/send', { ...body, send: codeSend }],+ ['transactional_email/send', { ...body, send: changedContent }],+ ],+ [+ 'the other route',+ ['transactional_email', body],+ ['transactional_email/send', { ...body, send: codeSend }],+ ],+ ] as const)(+ 'rejects the key for a request with %s',+ async (_desc, [firstRoute, first], [route, payload]) => {+ await sendKeyedOn(firstRoute, first)- const res = await sendKeyed({ ...keyed, to: '[email protected]' })+ const res = await sendKeyedOn(route, payload)- expect(res.status).toEqual(422)- expect(res.body).toEqual({ error: 'idempotency_key_reused' })- expect(mockQueueInvocations).toHaveBeenCalledTimes(1)- })+ expect(res.status).toEqual(422)+ expect(res.body).toEqual({ error: 'idempotency_key_reused' })+ expect(mockQueueInvocations).toHaveBeenCalledTimes(1)+ }+ )it('tells a retry to wait while the first request is still queueing', async () => {let finishQueueing: () => void = () => {}@@ -2361,6 +2414,71 @@ describe('CDP API', () => {})})+ it.each([+ ['a code send with rendered content', codeSend],+ [+ 'a code send with html that has no text to derive',+ { ...codeSend, content: { subject: 'Hi', html: '<img src="https://example.com/a.png" alt="Hi">' } },+ ],+ ['a code send for a local template', { tracking: { captureEngagement: false } }],+ ])('stamps %s on the queued run', async (_desc, acceptedSend) => {+ const res = await sendAccepted(hogFlow.id, { ...body, send: acceptedSend })++ expect(res.status).toEqual(202)+ const [invocation] = mockQueueInvocations.mock.calls[0][0]+ expect(invocation.state.send).toEqual(acceptedSend)+ })++ it('queues a template send without an accepted send, even when the request carries one', async () => {+ const res = await send(+ hogFlow.id,+ { Authorization: `Bearer ${mintToken(team.id, hogFlow.id)}` },+ { ...body, send: codeSend }+ )++ expect(res.status).toEqual(202)+ const [invocation] = mockQueueInvocations.mock.calls[0][0]+ expect(invocation.state.send).toBeUndefined()+ })++ it('refuses a send on the accepted send route without the send', async () => {+ const invalidBefore = await outcomeCount('invalid_request')++ const res = await sendAccepted(hogFlow.id, body)++ expect(res.status).toEqual(400)+ expect(await outcomeCount('invalid_request')).toEqual(invalidBefore + 1)+ expect(mockQueueInvocations).not.toHaveBeenCalled()+ })++ it.each([+ ['a field this version does not know', { ...codeSend, priority: 'high' }],+ [+ 'a content field this version does not know',+ { ...codeSend, content: { ...codeSend.content, amp: '<p/>' } },+ ],+ [+ 'a sender field this version does not know',+ { ...codeSend, from: { ...codeSend.from, replyName: 'Ops' } },+ ],+ [+ 'a tracking field this version does not know',+ { ...codeSend, tracking: { captureEngagement: true, pixel: false } },+ ],+ ['content without a body', { ...codeSend, content: { subject: 'Hi', html: '', text: '' } }],+ ['content with an empty subject', { ...codeSend, content: { subject: '', text: 'Hi' } }],+ ['a sender without an integration', { ...codeSend, from: { email: '[email protected]' } }],+ ['no tracking choice', { content: codeSend.content }],+ ])('refuses a code send with %s', async (_desc, acceptedSend) => {+ const invalidBefore = await outcomeCount('invalid_request')++ const res = await sendAccepted(hogFlow.id, { ...body, send: acceptedSend })++ expect(res.status).toEqual(400)+ expect(await outcomeCount('invalid_request')).toEqual(invalidBefore + 1)+ expect(mockQueueInvocations).not.toHaveBeenCalled()+ })+it('queues a send when the team is over its email quota', async () => {jest.spyOn(api['deps'].quotaLimiting, 'isTeamQuotaLimited').mockResolvedValue(true)const queuedBefore = await outcomeCount('queued')@@ -2386,69 +2504,110 @@ describe('CDP API', () => {expect(mockQueueInvocations).not.toHaveBeenCalled()})- it.each([- [- 'email sending is suspended for the project',- 'sending_suspended',- () =>- jest- .spyOn(api['teamWorkflowsConfigService'], 'getEmailSendingSuspension')- .mockResolvedValueOnce('staff'),- ],- [- "the transactional email's sending is paused",- 'sending_paused',- () =>- jest- .spyOn(api['hogFlowManager'], 'getHogFlow')- .mockResolvedValueOnce({ ...hogFlow, email_sending_paused_at: Date.now() }),- ],- [- 'the recipient is suppressed',- 'recipient_suppressed',- () => jest.spyOn(api['emailSuppressionService'], 'isSuppressed').mockResolvedValueOnce(true),- ],- [- "the recipient's domain has no mail servers",- 'recipient_unreachable',- () =>- jest- .spyOn(api['emailValidationService'], 'getRecipientSkipReason')- .mockResolvedValueOnce('Skipping send: the domain has no reachable mail servers.'),- ],- ] as const)('refuses a send the email worker would skip because %s', async (_desc, code, refuse) => {- refuse()- const refusedBefore = await outcomeCount(code)- const stagesBefore = await stageCounts()- const keyed = { ...body, idempotency_key: `key-${new UUIDT().toString()}` }- const headers = { Authorization: `Bearer ${mintToken(team.id, hogFlow.id)}` }+ describe.each([+ ['a template send', 'transactional_email', {}],+ ['a code send', 'transactional_email/send', { send: codeSend }],+ ])('for %s', (_kind, route, extra) => {+ it.each([+ [+ 'email sending is suspended for the project',+ 'sending_suspended',+ () =>+ jest+ .spyOn(api['teamWorkflowsConfigService'], 'getEmailSendingSuspension')+ .mockResolvedValueOnce('staff'),+ ],+ [+ "the transactional email's sending is paused",+ 'sending_paused',+ () =>+ jest+ .spyOn(api['hogFlowManager'], 'getHogFlowCheckingPrimaryOnMiss')+ .mockResolvedValueOnce({ ...hogFlow, email_sending_paused_at: Date.now() }),+ ],+ [+ 'the recipient is suppressed',+ 'recipient_suppressed',+ () => jest.spyOn(api['emailSuppressionService'], 'isSuppressed').mockResolvedValueOnce(true),+ ],+ [+ "the recipient's domain has no mail servers",+ 'recipient_unreachable',+ () =>+ jest+ .spyOn(api['emailValidationService'], 'getRecipientSkipReason')+ .mockResolvedValueOnce('Skipping send: the domain has no reachable mail servers.'),+ ],+ ] as const)('refuses a send the email worker would skip because %s', async (_desc, code, refuse) => {+ refuse()+ const refusedBefore = await outcomeCount(code)+ const stagesBefore = await stageCounts()+ const keyed = { ...body, ...extra, idempotency_key: `key-${new UUIDT().toString()}` }+ const headers = { Authorization: `Bearer ${tokenFor(route, team.id, hogFlow.id)}` }++ const refused = await sendOn(route, hogFlow.id, headers, keyed)++ expect(refused.status).toEqual(422)+ expect(refused.body).toEqual({ error: code })+ expect(await outcomeCount(code)).toEqual(refusedBefore + 1)+ // The reserve and the release of the key count once, as one idempotency observation.+ expect(stageCountChanges(stagesBefore, await stageCounts())).toEqual({+ auth: 1,+ validate: 1,+ load: 1,+ build: 1,+ idempotency: 1,+ checks: 1,+ total: 1,+ })+ expect(mockQueueInvocations).not.toHaveBeenCalled()+ expect(emailStatusRows()).toEqual([])- const refused = await send(hogFlow.id, headers, keyed)+ const retry = await sendOn(route, hogFlow.id, headers, keyed)- expect(refused.status).toEqual(422)- expect(refused.body).toEqual({ error: code })- expect(await outcomeCount(code)).toEqual(refusedBefore + 1)- // The reserve and the release of the key count once, as one idempotency observation.- expect(stageCountChanges(stagesBefore, await stageCounts())).toEqual({- auth: 1,- validate: 1,- load: 1,- build: 1,- idempotency: 1,- checks: 1,- total: 1,+ expect(retry.status).toEqual(202)+ expect(mockQueueInvocations).toHaveBeenCalledTimes(1)})- expect(mockQueueInvocations).not.toHaveBeenCalled()- expect(emailStatusRows()).toEqual([])- const retry = await send(hogFlow.id, headers, keyed)+ it('replays a keyed send whose workflow stopped being active since it went out', async () => {+ const keyed = { ...body, ...extra, idempotency_key: `key-${new UUIDT().toString()}` }+ const headers = { Authorization: `Bearer ${tokenFor(route, team.id, hogFlow.id)}` }+ const first = await sendOn(route, hogFlow.id, headers, keyed)+ await hub.postgres.query(+ PostgresUse.COMMON_WRITE,+ `UPDATE posthog_hogflow SET status = 'draft' WHERE id = $1`,+ [hogFlow.id],+ 'deactivate-workflow'+ )+ api['hogFlowManager']['onHogFlowsReloaded'](team.id, [hogFlow.id])- expect(retry.status).toEqual(202)- expect(mockQueueInvocations).toHaveBeenCalledTimes(1)+ const retry = await sendOn(route, hogFlow.id, headers, keyed)++ expect(retry.status).toEqual(202)+ expect(retry.body).toEqual({ ...first.body, replayed: true })+ expect(mockQueueInvocations).toHaveBeenCalledTimes(1)+ })})it.each([['a valid address', { to: '[email protected]' }, 202, 'queued'],+ ...[+ '[email protected]',+ '[email protected]',+ 'jane#[email protected]',+ "o'[email protected]",+ 'a!b$c%d&e*f/g?h^i`j|[email protected]',+ '[email protected]',+ ].map((to) => [to, { to }, 202, 'queued'] as const),+ ...[+ 'jane@bücher.example',+ 'jane@[192.0.2.1]',+ '"jane doe"@example.com',+ 'jane@localhost',+ '[email protected]',+ '[email protected]',+ '[email protected]',+ ].map((to) => [to, { to }, 422, 'recipient_unreachable'] as const),['a domain with a fullwidth dot', { to: 'user@ph\uFF0Eexample.com' }, 422, 'recipient_unreachable'],['an ideographic full stop', { to: 'user@ph\u3002example.com' }, 422, 'recipient_unreachable'],['a zero-width space', { to: 'user\[email protected]' }, 422, 'recipient_unreachable'],@@ -2513,6 +2672,65 @@ describe('CDP API', () => {expect(mockQueueInvocations).not.toHaveBeenCalled()})+ it.each([+ ['transactional_email', body],+ ['transactional_email/send', { ...body, send: codeSend }],+ ])(+ 'sends on %s under a workflow the replica does not have yet, reading the primary once',+ async (route, payload) => {+ const unseen = await _insertHogFlow(hub.postgres, transactionalFlow('api', 'active'))+ const query = hideHogFlowsFromReplica(hub.postgres, [unseen.id])+ const post = () =>+ supertest(app)+ .post(`/api/projects/${team.id}/hog_flows/${unseen.id}/${route}`)+ .set({ Authorization: `Bearer ${tokenFor(route, team.id, unseen.id)}` })+ .send(payload)++ expect((await post()).status).toEqual(202)+ expect((await post()).status).toEqual(202)++ expect(mockQueueInvocations).toHaveBeenCalledTimes(2)+ expect(+ query.mock.calls.filter(+ ([use, , , tag]) => use === PostgresUse.COMMON_WRITE && tag === 'fetchHogFlows'+ )+ ).toHaveLength(1)+ }+ )++ it.each(['transactional_email', 'transactional_email/send'])(+ 'authenticates every send on %s',+ async (route) => {+ const post = (headers: Record<string, string>) =>+ supertest(app)+ .post(`/api/projects/${team.id}/hog_flows/${hogFlow.id}/${route}`)+ .set(headers)+ .send({ ...body, send: codeSend })++ expect((await post({})).status).toEqual(401)+ expect(+ (await post({ Authorization: `Bearer ${tokenFor(route, team.id + 1, hogFlow.id)}` })).status+ ).toEqual(401)+ expect(mockQueueInvocations).not.toHaveBeenCalled()+ }+ )++ it.each([+ ['a template send token on the code send route', 'transactional_email/send', 'transactional_email'],+ ['a code send token on the template send route', 'transactional_email', 'transactional_email/send'],+ ])('refuses %s', async (_desc, route, tokenRoute) => {+ const unauthorizedBefore = await outcomeCount('unauthorized')++ const res = await supertest(app)+ .post(`/api/projects/${team.id}/hog_flows/${hogFlow.id}/${route}`)+ .set({ Authorization: `Bearer ${tokenFor(tokenRoute, team.id, hogFlow.id)}` })+ .send({ ...body, send: codeSend })++ expect(res.status).toEqual(401)+ expect(await outcomeCount('unauthorized')).toEqual(unauthorizedBefore + 1)+ expect(mockQueueInvocations).not.toHaveBeenCalled()+ })+it.each([['no token', () => ({})],[diff --git a/nodejs/src/cdp/cdp-api.ts b/nodejs/src/cdp/cdp-api.tsindex ccf4d277cf6d..03ea68d80d37 100644--- a/nodejs/src/cdp/cdp-api.ts+++ b/nodejs/src/cdp/cdp-api.ts@@ -54,7 +54,11 @@ import { matchesWaitUntilCondition } from './services/hogflows/hogflow-utils'import { IdempotencyKeys } from './services/hogflows/idempotency-keys'import { WorkflowStepResumeSchema } from './services/hogflows/step-resume.service'import {+ AcceptedSendRequestSchema,+ CODE_SEND_TOKEN_CLAIMS,+ TEMPLATE_SEND_TOKEN_CLAIMS,TransactionalEmailOutcome,+ TransactionalEmailRequest,TransactionalEmailRequestSchema,TransactionalEmailStageTimer,transactionalEmailRequestsTotal,@@ -167,6 +171,18 @@ function parseRealtimeCohort(value: unknown): BatchResolverState['realtimeCohortreturn { id: id as number, teamId: team_id as number, waitUntil: waitUntil.toISOString() }}+const ACCEPTED_SEND_FINGERPRINT_MARKER = 'accepted_send'++const transactionalEmailFingerprint = (+ hogFlowId: string,+ { to, distinct_id, variables, send }: TransactionalEmailRequest+): string => {+ const templateRequest = [hogFlowId, to, distinct_id, variables]+ return IdempotencyKeys.fingerprint(+ send ? [...templateRequest, ACCEPTED_SEND_FINGERPRINT_MARKER, send] : templateRequest+ )+}+export class CdpApi {private hogExecutorAsync: HogExecutorAsyncServiceprivate hogInputsService: HogInputsService@@ -382,7 +398,11 @@ export class CdpApi {router.post('/api/projects/:team_id/workflow_steps/resume', asyncHandler(this.postWorkflowStepResume))router.post('/api/projects/:team_id/hog_flows/:id/transactional_email',- asyncHandler(this.postTransactionalEmail)+ asyncHandler(this.postTransactionalEmail(TransactionalEmailRequestSchema, TEMPLATE_SEND_TOKEN_CLAIMS))+ )+ router.post(+ '/api/projects/:team_id/hog_flows/:id/transactional_email/send',+ asyncHandler(this.postTransactionalEmail(AcceptedSendRequestSchema, CODE_SEND_TOKEN_CLAIMS)))router.get('/api/projects/:team_id/hog_functions/:id/status', asyncHandler(this.getFunctionStatus()))router.patch('/api/projects/:team_id/hog_functions/:id/status', asyncHandler(this.patchFunctionStatus()))@@ -1158,161 +1178,185 @@ export class CdpApi {//// Auth: a per-call JWT minted by Django on its own audience and key, pinned to the team and the workflow,// never the fleet-wide internal secret.- private postTransactionalEmail = async (req: ModifiedRequest, res: express.Response): Promise<any> => {- const stages = new TransactionalEmailStageTimer()- const reply = (outcome: TransactionalEmailOutcome, status: number, body: Record<string, unknown>) => {- transactionalEmailRequestsTotal.inc({ outcome })- return res.status(status).json(body)- }- try {- stages.start('auth')- if (!this.transactionalEmailJwt.enabled) {- return reply('not_configured', 503, {- error: 'Transactional email auth not configured (WORKFLOWS_TRANSACTIONAL_EMAIL_JWT_SECRET unset)',- })- }- if (!this.verifyScopedWorkflowJwt(this.transactionalEmailJwt, req, res, 'transactional email')) {- transactionalEmailRequestsTotal.inc({ outcome: 'unauthorized' })- return+ private postTransactionalEmail =+ (schema: z.ZodType<TransactionalEmailRequest, unknown>, tokenClaims: Record<string, string | undefined>) =>+ async (req: ModifiedRequest, res: express.Response): Promise<any> => {+ const stages = new TransactionalEmailStageTimer()+ const reply = (outcome: TransactionalEmailOutcome, status: number, body: Record<string, unknown>) => {+ transactionalEmailRequestsTotal.inc({ outcome })+ return res.status(status).json(body)}- stages.start('validate')- const parsed = TransactionalEmailRequestSchema.safeParse(req.body)- if (!parsed.success) {- const issues = parsed.error.issues- // Signup accepts some addresses this check refuses, and no retry can fix an address.- if (issues.every((issue) => issue.path[0] === 'to' && issue.code === 'invalid_format')) {- return reply('recipient_unreachable', 422, { error: 'recipient_unreachable' })+ try {+ stages.start('auth')+ if (!this.transactionalEmailJwt.enabled) {+ return reply('not_configured', 503, {+ error: 'Transactional email auth not configured (WORKFLOWS_TRANSACTIONAL_EMAIL_JWT_SECRET unset)',+ })}- // Paths and codes only: the values include the recipient's address.- logger.warn('⚠️', 'Transactional email request failed validation', {- team_id: req.params.team_id,- hog_flow_id: req.params.id,- issues: issues.map((issue) => ({ path: issue.path.join('.'), code: issue.code })),- })- return reply('invalid_request', 400, { error: parsed.error.message })- }- const { to, distinct_id: distinctId, variables, idempotency_key: idempotencyKey } = parsed.data+ if (+ !this.verifyScopedWorkflowJwt(+ this.transactionalEmailJwt,+ req,+ res,+ 'transactional email',+ tokenClaims+ )+ ) {+ transactionalEmailRequestsTotal.inc({ outcome: 'unauthorized' })+ return+ }+ stages.start('validate')+ const parsed = schema.safeParse(req.body)+ if (!parsed.success) {+ const issues = parsed.error.issues+ // Signup accepts some addresses this check refuses, and no retry can fix an address.+ if (issues.every((issue) => issue.path[0] === 'to' && issue.code === 'invalid_format')) {+ return reply('recipient_unreachable', 422, { error: 'recipient_unreachable' })+ }+ // Paths and codes only: the values include the recipient's address.+ logger.warn('⚠️', 'Transactional email request failed validation', {+ team_id: req.params.team_id,+ hog_flow_id: req.params.id,+ issues: issues.map((issue) => ({ path: issue.path.join('.'), code: issue.code })),+ })+ return reply('invalid_request', 400, { error: parsed.error.message })+ }+ const { to, distinct_id: distinctId, variables, idempotency_key: idempotencyKey, send } = parsed.data- stages.start('load')- const team = await this.deps.teamManager.getTeam(parseInt(req.params.team_id)).catch(() => null)- if (!team) {- return reply('not_found', 404, { error: 'Team not found' })- }- const hogFlow = await this.hogFlowManager.getHogFlow(req.params.id)- if (!hogFlow || hogFlow.team_id !== team.id) {- return reply('not_found', 404, { error: 'Workflow not found' })- }- if (hogFlow.trigger?.type !== 'api' || hogFlow.status !== 'active') {- return reply('not_sendable', 400, { error: 'not_sendable' })- }+ stages.start('load')+ const team = await this.deps.teamManager.getTeam(parseInt(req.params.team_id)).catch(() => null)+ if (!team) {+ return reply('not_found', 404, { error: 'Team not found' })+ }+ const hogFlow = await this.hogFlowManager.getHogFlowCheckingPrimaryOnMiss(req.params.id)+ if (!hogFlow || hogFlow.team_id !== team.id) {+ return reply('not_found', 404, { error: 'Workflow not found' })+ }+ if (hogFlow.trigger?.type !== 'api') {+ return reply('not_sendable', 400, { error: 'not_sendable' })+ }- stages.start('build')- const event: HogFunctionInvocationGlobals['event'] = {- uuid: new UUIDT().toString(),- event: TRANSACTIONAL_EMAIL_EVENT,- distinct_id: distinctId,- timestamp: DateTime.now().toISO(),- url: '',- // The email step's recipient reads this.- properties: { to },- elements_chain: '',- }- const triggerGlobals: HogFunctionInvocationGlobals = {- event,- project: {- id: team.id,- name: team.name,- url: `${this.config.SITE_URL ?? 'http://localhost:8000'}/project/${team.id}`,- },- variables,- }- const filterGlobals = convertToHogFunctionFilterGlobal({ event, person: undefined, groups: {}, variables })+ stages.start('build')+ const event: HogFunctionInvocationGlobals['event'] = {+ uuid: new UUIDT().toString(),+ event: TRANSACTIONAL_EMAIL_EVENT,+ distinct_id: distinctId,+ timestamp: DateTime.now().toISO(),+ url: '',+ // The email step's recipient reads this.+ properties: { to },+ elements_chain: '',+ }+ const triggerGlobals: HogFunctionInvocationGlobals = {+ event,+ project: {+ id: team.id,+ name: team.name,+ url: `${this.config.SITE_URL ?? 'http://localhost:8000'}/project/${team.id}`,+ },+ variables,+ }+ const filterGlobals = convertToHogFunctionFilterGlobal({+ event,+ person: undefined,+ groups: {},+ variables,+ })- const invocation = createHogFlowInvocation(triggerGlobals, hogFlow, filterGlobals)- // The run skips the trigger pipeline that counts runs for event-triggered workflows, so the email- // worker counts it instead.- invocation.state.triggerMetricPending = true- // Straight to the email queue: the flow's only step is the email, so a hop through the- // workflow queue would only add latency to a password reset or a verification code.- const priorityClass: EmailQueuePriorityClass = 'fast'- invocation.queue = 'email'- invocation.queuePriority = EMAIL_QUEUE_PRIORITY[priorityClass]-- const idempotencyKeys = this.transactionalEmailIdempotencyKeys- const fingerprint = IdempotencyKeys.fingerprint([hogFlow.id, to, distinctId, variables])- if (idempotencyKey) {- // Checked before the quota, so a retry of a send that went out gets its answer even- // when the team has hit its quota since.- stages.start('idempotency')- const reservation = await idempotencyKeys.reserve(team.id, idempotencyKey, fingerprint, invocation.id)- switch (reservation.status) {- case 'replayed':- return reply('replayed', 202, {- status: 'queued',- invocation_id: reservation.invocationId,- replayed: true,- })- case 'in_flight':- return reply('idempotency_key_in_flight', 409, { error: 'idempotency_key_in_flight' })- case 'conflict':- return reply('idempotency_key_reused', 422, { error: 'idempotency_key_reused' })- case 'unavailable':- return reply('idempotency_unavailable', 503, { error: 'idempotency_unavailable' })+ const invocation = createHogFlowInvocation(triggerGlobals, hogFlow, filterGlobals)+ if (send) {+ invocation.state.send = send}- }- try {- stages.start('checks')- const refusal = await this.refuseTransactionalEmail(team.id, hogFlow, to)- if (refusal) {- if (idempotencyKey) {- stages.start('idempotency')- await idempotencyKeys.release(team.id, idempotencyKey, invocation.id)+ // The run skips the trigger pipeline that counts runs for event-triggered workflows, so the email+ // worker counts it instead.+ invocation.state.triggerMetricPending = true+ // Straight to the email queue: the flow's only step is the email, so a hop through the+ // workflow queue would only add latency to a password reset or a verification code.+ const priorityClass: EmailQueuePriorityClass = 'fast'+ invocation.queue = 'email'+ invocation.queuePriority = EMAIL_QUEUE_PRIORITY[priorityClass]++ const idempotencyKeys = this.transactionalEmailIdempotencyKeys+ const fingerprint = transactionalEmailFingerprint(hogFlow.id, parsed.data)+ if (idempotencyKey) {+ // Checked before the quota, so a retry of a send that went out gets its answer even+ // when the team has hit its quota since.+ stages.start('idempotency')+ const reservation = await idempotencyKeys.reserve(+ team.id,+ idempotencyKey,+ fingerprint,+ invocation.id+ )+ switch (reservation.status) {+ case 'replayed':+ return reply('replayed', 202, {+ status: 'queued',+ invocation_id: reservation.invocationId,+ replayed: true,+ })+ case 'in_flight':+ return reply('idempotency_key_in_flight', 409, { error: 'idempotency_key_in_flight' })+ case 'conflict':+ return reply('idempotency_key_reused', 422, { error: 'idempotency_key_reused' })+ case 'unavailable':+ return reply('idempotency_unavailable', 503, { error: 'idempotency_unavailable' })}- return reply(refusal.outcome, refusal.status, { error: refusal.outcome })}- stages.start('queue_insert')- this.invocationResultsService.invocationResultsRowsService.queueLifecycleRow(invocation, 'running')try {- await this.hogflowQueue.queueInvocations([invocation])+ stages.start('checks')+ const refusal = await this.refuseTransactionalEmail(team.id, hogFlow, to)+ if (refusal) {+ if (idempotencyKey) {+ stages.start('idempotency')+ await idempotencyKeys.release(team.id, idempotencyKey, invocation.id)+ }+ return reply(refusal.outcome, refusal.status, { error: refusal.outcome })+ }+ stages.start('queue_insert')+ this.invocationResultsService.invocationResultsRowsService.queueLifecycleRow(invocation, 'running')+ try {+ await this.hogflowQueue.queueInvocations([invocation])+ } catch (error) {+ this.invocationResultsService.invocationResultsRowsService.dropQueuedRowsFor([invocation.id])+ throw error+ }+ cdpEmailQueuedTotal.labels(priorityClass).inc()} catch (error) {- this.invocationResultsService.invocationResultsRowsService.dropQueuedRowsFor([invocation.id])+ // Nothing was queued, so the caller's retry with the same key must be able to send.+ if (idempotencyKey) {+ stages.start('idempotency')+ await idempotencyKeys.release(team.id, idempotencyKey, invocation.id)+ }throw error}- cdpEmailQueuedTotal.labels(priorityClass).inc()- } catch (error) {- // Nothing was queued, so the caller's retry with the same key must be able to send.if (idempotencyKey) {stages.start('idempotency')- await idempotencyKeys.release(team.id, idempotencyKey, invocation.id)+ await idempotencyKeys.markQueued(team.id, idempotencyKey, fingerprint, invocation.id)}- throw error- }- if (idempotencyKey) {- stages.start('idempotency')- await idempotencyKeys.markQueued(team.id, idempotencyKey, fingerprint, invocation.id)- }- stages.start('metrics_flush')- this.emailStatusService.queue({ teamId: team.id, messageId: invocation.id, status: 'queued' })- // Best-effort rows, so the 202 waits for the hand-off to Kafka (flushed on shutdown) but not the acks.- const warnTelemetryFlushFailed = (error: unknown) =>- logger.warn('⚠️', 'Transactional email telemetry flush failed', {- error: error instanceof Error ? error.message : String(error),- invocation_id: invocation.id,- })- void this.emailStatusService.flush().catch(warnTelemetryFlushFailed)- const { produced } = await this.invocationResultsService.invocationResultsRowsService.handOffQueuedRows()- void produced.catch(warnTelemetryFlushFailed)+ stages.start('metrics_flush')+ this.emailStatusService.queue({ teamId: team.id, messageId: invocation.id, status: 'queued' })+ // Best-effort rows, so the 202 waits for the hand-off to Kafka (flushed on shutdown) but not the acks.+ const warnTelemetryFlushFailed = (error: unknown) =>+ logger.warn('⚠️', 'Transactional email telemetry flush failed', {+ error: error instanceof Error ? error.message : String(error),+ invocation_id: invocation.id,+ })+ void this.emailStatusService.flush().catch(warnTelemetryFlushFailed)+ const { produced } =+ await this.invocationResultsService.invocationResultsRowsService.handOffQueuedRows()+ void produced.catch(warnTelemetryFlushFailed)- return reply('queued', 202, { status: 'queued', invocation_id: invocation.id, replayed: false })- } catch (e) {- logger.error('Error queueing transactional email', {- error: e instanceof Error ? e.message : String(e),- })- return reply('error', 500, { error: e instanceof Error ? e.message : String(e) })- } finally {- stages.finish()+ return reply('queued', 202, { status: 'queued', invocation_id: invocation.id, replayed: false })+ } catch (e) {+ logger.error('Error queueing transactional email', {+ error: e instanceof Error ? e.message : String(e),+ })+ return reply('error', 500, { error: e instanceof Error ? e.message : String(e) })+ } finally {+ stages.finish()+ }}- }// The email worker skips a send for these reasons too, but only after the caller has its 202. Checking them// here tells the caller that the email will not go out. The worker keeps its checks, because the state can@@ -1322,6 +1366,9 @@ export class CdpApi {hogFlow: HogFlow,to: string): Promise<{ outcome: TransactionalEmailOutcome; status: number } | null> {+ if (hogFlow.status !== 'active') {+ return { outcome: 'not_sendable', status: 400 }+ }if (await this.teamWorkflowsConfigService.getEmailSendingSuspension(teamId)) {return { outcome: 'sending_suspended', status: 422 }}@@ -1347,7 +1394,7 @@ export class CdpApi {req: ModifiedRequest,res: express.Response,label: string,- extraClaims?: Record<string, string>+ extraClaims?: Record<string, string | undefined>): boolean {const { team_id, id } = req.paramsconst authHeader = req.headers['authorization']diff --git a/nodejs/src/cdp/consumers/cdp-cyclotron-worker-hogflow.consumer.ts b/nodejs/src/cdp/consumers/cdp-cyclotron-worker-hogflow.consumer.tsindex 6e6718bbf887..f7e2639d86c7 100644--- a/nodejs/src/cdp/consumers/cdp-cyclotron-worker-hogflow.consumer.ts+++ b/nodejs/src/cdp/consumers/cdp-cyclotron-worker-hogflow.consumer.ts@@ -89,6 +89,9 @@ export class CdpCyclotronWorkerHogFlow extends CdpCyclotronWorker {const loadedInvocations: CyclotronJobInvocationHogFlow[] = []const failedInvocations: CyclotronJobInvocation[] = []const canceledResults: CyclotronJobInvocationResult[] = []+ const hogFlows = await this.hogFlowManager.getHogFlowsCheckingPrimaryOnMiss(+ invocations.filter((item) => !item.cancelRequestedAt).map((item) => item.functionId)+ )await Promise.all(invocations.map(async (item) => {@@ -103,7 +106,7 @@ export class CdpCyclotronWorkerHogFlow extends CdpCyclotronWorker {}const team = await this.deps.teamManager.getTeam(item.teamId)- const hogFlow = await this.hogFlowManager.getHogFlow(item.functionId)+ const hogFlow = hogFlows[item.functionId]if (!hogFlow || !team) {logger.error('⚠️', 'Error finding hog flow', {id: item.functionId,diff --git a/nodejs/src/cdp/consumers/cdp-cyclotron-worker-hogflow.test.ts b/nodejs/src/cdp/consumers/cdp-cyclotron-worker-hogflow.test.tsindex f1d624cc7c69..98640b5d993b 100644--- a/nodejs/src/cdp/consumers/cdp-cyclotron-worker-hogflow.test.ts+++ b/nodejs/src/cdp/consumers/cdp-cyclotron-worker-hogflow.test.ts@@ -12,7 +12,7 @@ import { createTeam, createTestTeamFixture, getTeam } from '~/tests/helpers/sql'import { Hub, InternalPerson, Team } from '../../types'import { FixtureHogFlowBuilder } from '../_tests/builders/hogflow.builder'-import { createHogFlowInvocationContext, insertHogFlow } from '../_tests/fixtures-hogflows'+import { createHogFlowInvocationContext, hideHogFlowsFromReplica, insertHogFlow } from '../_tests/fixtures-hogflows'import {CyclotronJobInvocation,CyclotronJobInvocationHogFlow,@@ -88,6 +88,7 @@ describe('CdpCyclotronWorkerHogFlow', () => {let team: Teamlet team2: Teamlet hogFlows: HogFlow[]+ let jobQueue: ReturnType<typeof createMockJobQueue>const createSerializedHogFlowInvocation = (hogFlow: HogFlow,@@ -160,11 +161,11 @@ describe('CdpCyclotronWorkerHogFlow', () => {]const mockRepo = createMockPersonReadRepository(testPersons)- const mockJobQueue = createMockJobQueue()+ jobQueue = createMockJobQueue()processor = new CdpCyclotronWorkerHogFlow(hub,{ ...createCdpConsumerDeps(hub), personRepository: mockRepo },- mockJobQueue+ jobQueue)hogFlows = []@@ -587,6 +588,33 @@ describe('CdpCyclotronWorkerHogFlow', () => {})})+ it('runs a flow the replica does not have yet, and fails one missing from both', async () => {+ const [lateFlow, missingFlow] = ['Flow only the primary has', 'Flow neither database has'].map((name) =>+ new FixtureHogFlowBuilder()+ .withName(name)+ .withTeamId(team.id)+ .withStatus('active')+ .withSimpleWorkflow()+ .build()+ )+ await insertHogFlow(hub.postgres, lateFlow)+ hideHogFlowsFromReplica(hub.postgres, [lateFlow.id])+ const [lateInvocation, missingInvocation] = [lateFlow, missingFlow].map((flow) =>+ createSerializedHogFlowInvocation(flow, {+ event: { distinct_id: 'distinct_A_1', properties: {} } as any,+ })+ )++ const results = await processor.processInvocations([lateInvocation, missingInvocation])++ expect(results).toHaveLength(1)+ expect(results[0]).toMatchObject({+ finished: true,+ invocation: { hogFlow: { id: lateFlow.id } },+ })+ expect(jobQueue.dequeueInvocations).toHaveBeenLastCalledWith([missingInvocation])+ })+it('terminates invocations as canceled when the workflow is disabled after being queued', async () => {const hogFlow = hogFlows[0]diff --git a/nodejs/src/cdp/rerun/rerun-job.manager.test.ts b/nodejs/src/cdp/rerun/rerun-job.manager.test.tsindex e11e969ed4d4..48b48a05e46d 100644--- a/nodejs/src/cdp/rerun/rerun-job.manager.test.ts+++ b/nodejs/src/cdp/rerun/rerun-job.manager.test.ts@@ -76,9 +76,11 @@ describe('RerunJobManager', () => {const jobId = await manager.enqueue(7, 'hog_function', functionId, {filter: { ...baseFilter, invocation_ids: ids },+ replay_code_sends: true,})const state = fetchState(await queryJob(jobId))+ expect(state.request.replay_code_sends).toBe(true)expect(state.function_kind).toBe('hog_function')expect(state.function_id).toBe(functionId)expect(state.request.filter.invocation_ids).toEqual(ids)@@ -118,6 +120,7 @@ describe('RerunJobManager', () => {expect(state.function_kind).toBe('hog_flow')expect(state.request.filter).toMatchObject(filter)expect(state.request.filter.invocation_ids).toBeUndefined()+ expect(state.request.replay_code_sends).toBe(false)expect(state.progress.cursor).toBeUndefined()expect(state.progress.done).toBe(false)})diff --git a/nodejs/src/cdp/rerun/rerun-job.manager.ts b/nodejs/src/cdp/rerun/rerun-job.manager.tsindex d721e8ddc4c2..d477f3791691 100644--- a/nodejs/src/cdp/rerun/rerun-job.manager.ts+++ b/nodejs/src/cdp/rerun/rerun-job.manager.ts@@ -83,7 +83,7 @@ export class RerunJobManager {const state: RerunJobState = {function_kind: functionKind,function_id: functionId,- request: { filter },+ request: { filter, replay_code_sends: request.replay_code_sends === true },progress: {queued: 0,skipped: 0,diff --git a/nodejs/src/cdp/rerun/rerun-job.types.ts b/nodejs/src/cdp/rerun/rerun-job.types.tsindex 98e1ffc215cc..1bae853a2ff9 100644--- a/nodejs/src/cdp/rerun/rerun-job.types.ts+++ b/nodejs/src/cdp/rerun/rerun-job.types.ts@@ -68,6 +68,7 @@ export interface RerunFilter {export interface RerunRequest {filter: RerunFilter+ replay_code_sends?: boolean}/**diff --git a/nodejs/src/cdp/rerun/rerun-paginator.replay-fidelity.test.ts b/nodejs/src/cdp/rerun/rerun-paginator.replay-fidelity.test.tsindex 8f7b6850ff5f..6b2f645ac8f5 100644--- a/nodejs/src/cdp/rerun/rerun-paginator.replay-fidelity.test.ts+++ b/nodejs/src/cdp/rerun/rerun-paginator.replay-fidelity.test.ts@@ -39,7 +39,11 @@ describe('RerunPaginatorService replay fidelity (hog_flow)', () => {function buildPaginator(clickhouse: ClickHouseClient,- hogFlowQueue: jest.Mocked<CyclotronJobQueuePostgresV2>+ hogFlowQueue: jest.Mocked<CyclotronJobQueuePostgresV2>,+ monitoring = {+ queueLogs: jest.fn(),+ flush: jest.fn().mockResolvedValue(undefined),+ } as unknown as jest.Mocked<HogFunctionMonitoringService>): RerunPaginatorService {const hogFlowManager = {getHogFlow: jest.fn().mockResolvedValue({ id: functionId, team_id: teamId, variables: [] }),@@ -53,11 +57,6 @@ describe('RerunPaginatorService replay fidelity (hog_flow)', () => {flush: jest.fn().mockResolvedValue(undefined),dropQueuedRowsFor: jest.fn(),} as unknown as jest.Mocked<HogInvocationResultsService>- const monitoring = {- queueLogs: jest.fn(),- flush: jest.fn().mockResolvedValue(undefined),- } as unknown as jest.Mocked<HogFunctionMonitoringService>-return new RerunPaginatorService(clickhouse,hogFunctionManager,@@ -99,6 +98,12 @@ describe('RerunPaginatorService replay fidelity (hog_flow)', () => {},personId: 'person-1',messageOptOuts: [{ identifier: '[email protected]', categoryId: '$all' }],+ send: {+ content: { subject: 'Welcome', html: '<p>Hi</p>', text: 'Hi' },+ from: { integrationId: 12, email: '[email protected]', name: 'Example' },+ replyTo: ['[email protected]'],+ tracking: { captureEngagement: false },+ },}const rows = [{@@ -117,7 +122,11 @@ describe('RerunPaginatorService replay fidelity (hog_flow)', () => {} as unknown as jest.Mocked<CyclotronJobQueuePostgresV2>const paginator = buildPaginator(fakeClickhouse(rows), hogFlowQueue)- await paginator.processPage(teamId, state, { jobId: 'rerun-1', createdAt: DateTime.now() })+ await paginator.processPage(+ teamId,+ { ...state, request: { ...state.request, replay_code_sends: true } },+ { jobId: 'rerun-1', createdAt: DateTime.now() }+ )expect(hogFlowQueue.queueInvocations).toHaveBeenCalledTimes(1)const [enqueued, opts] = hogFlowQueue.queueInvocations.mock.calls[0]@@ -133,10 +142,73 @@ describe('RerunPaginatorService replay fidelity (hog_flow)', () => {expect(invocation.state?.actionStepCount).toBe(3)expect(invocation.state?.customerTaskIdempotencyVersion).toBe(version)expect(invocation.state?.variables).toEqual({ ticket_id: '123' })+ expect(invocation.state?.send).toEqual(persistedState.send)// Sticky rerun counter still increments so max_attempts guards apply.expect(invocation.state?.rerunAttempts).toBe(1)})+ const codeSend = {+ content: { subject: 'Welcome', html: '<p>Hi</p>', text: 'Hi' },+ from: { integrationId: 12, email: '[email protected]', name: 'Example' },+ tracking: { captureEngagement: false },+ }++ it.each([+ ['an email sent from code while replay_code_sends is undefined', { send: codeSend }, undefined, []],+ ['an email sent from code while replay_code_sends is false', { send: codeSend }, false, []],+ [+ 'a sensitive message whose content was not stored',+ { sensitive_variables_omitted: true },+ true,+ [+ expect.objectContaining({+ instance_id: 'inv-3',+ level: 'warn',+ message:+ "Not re-run: this run's message is sensitive, so its content wasn't stored and can't be sent again.",+ }),+ ],+ ],+ ] as const)('skips a run with %s', async (_name, stored, allowed, expectedLogs) => {+ const persistedState = {+ event: { uuid: 'evt-3', distinct_id: 'd-3', properties: {}, timestamp: '2026-06-01T09:00:00Z' },+ actionStepCount: 1,+ variables: {},+ ...stored,+ }+ const rows = [+ {+ invocation_id: 'inv-3',+ parent_run_id: '',+ attempts: 0,+ last_scheduled_at: '2026-06-01 09:00:00.000000',+ first_scheduled_at: '2026-06-01 09:00:00.000000',+ invocation_globals: JSON.stringify(persistedState),+ },+ ]++ const hogFlowQueue = {+ queueInvocations: jest.fn().mockResolvedValue(undefined),+ } as unknown as jest.Mocked<CyclotronJobQueuePostgresV2>++ const monitoring = {+ queueLogs: jest.fn(),+ flush: jest.fn().mockResolvedValue(undefined),+ } as unknown as jest.Mocked<HogFunctionMonitoringService>+ const paginator = buildPaginator(fakeClickhouse(rows), hogFlowQueue, monitoring)+ await paginator.processPage(+ teamId,+ { ...state, request: { ...state.request, replay_code_sends: allowed } },+ { jobId: 'rerun-3', createdAt: DateTime.now() }+ )++ expect(hogFlowQueue.queueInvocations.mock.calls.flatMap(([invocations]) => invocations)).toEqual([])+ const runLogs = monitoring.queueLogs.mock.calls+ .flatMap(([entries]) => entries)+ .filter((entry) => entry.instance_id === 'inv-3')+ expect(runLogs).toEqual(expectedLogs)+ })+it('leaves currentAction undefined for a flow that never progressed (fresh start is correct there)', async () => {const persistedState = {event: { uuid: 'evt-2', distinct_id: 'd-2', properties: {}, timestamp: '2026-06-01T09:00:00Z' },diff --git a/nodejs/src/cdp/rerun/rerun-paginator.service.ts b/nodejs/src/cdp/rerun/rerun-paginator.service.tsindex 9dcf2ac5bbc6..60c46883825b 100644--- a/nodejs/src/cdp/rerun/rerun-paginator.service.ts+++ b/nodejs/src/cdp/rerun/rerun-paginator.service.ts@@ -641,7 +641,8 @@ export class RerunPaginatorService {teamId,state.function_kind,state.function_id,- row+ row,+ state.request.replay_code_sends === true)if (!invocation) {counterRerunInvocationsSkipped.labels(state.function_kind, 'rehydrate_failed').inc()@@ -682,7 +683,8 @@ export class RerunPaginatorService {teamId: number,functionKind: RerunFunctionKind,functionId: string,- row: InvocationRow+ row: InvocationRow,+ replayCodeSends: boolean): Promise<CyclotronJobInvocation | null> {let parsedGlobals: unknowntry {@@ -774,6 +776,29 @@ export class RerunPaginatorService {teamId,invocation_id: row.invocation_id,})+ this.monitoringService.queueLogs(+ [+ {+ team_id: teamId,+ log_source: 'hog_flow',+ log_source_id: functionId,+ instance_id: row.invocation_id,+ timestamp: DateTime.now(),+ level: 'warn',+ message:+ "Not re-run: this run's message is sensitive, so its content wasn't stored and can't be sent again.",+ },+ ],+ 'hog_flow'+ )+ return null+ }+ if (persistedState.send && !replayCodeSends) {+ logger.warn('⚠️', 'Skipping rerun of an email sent from code while code sends are off', {+ functionId,+ teamId,+ invocation_id: row.invocation_id,+ })return null}const eventForFilter = persistedState.event ?? {}@@ -839,6 +864,7 @@ export class RerunPaginatorService {personId: persistedState.personId,// Later message steps check these before the per-worker preferences cache, which can predate them.messageOptOuts: persistedState.messageOptOuts,+ send: persistedState.send,// Sticky rerun counter — mirror the hog function path so the// lifecycle row producer can derive `attempts` / `is_retry`// for flows too, and the `max_attempts` guard actually trips.diff --git a/nodejs/src/cdp/services/hogflows/hogflow-executor.service.test.ts b/nodejs/src/cdp/services/hogflows/hogflow-executor.service.test.tsindex aec05b4a71df..a17da3387f1c 100644--- a/nodejs/src/cdp/services/hogflows/hogflow-executor.service.test.ts+++ b/nodejs/src/cdp/services/hogflows/hogflow-executor.service.test.ts@@ -13,12 +13,14 @@ import { template as setVariableTemplate } from '~/cdp/templates/_destinations/pimport { compileHog } from '~/cdp/templates/compiler'import { convertToHogFunctionFilterGlobal } from '~/cdp/utils/hog-function-filtering'import { createHub } from '~/common/utils/db/hub'+import { PostgresUse } from '~/common/utils/db/postgres'import { parseJSON } from '~/common/utils/json-parse'import { logger } from '~/common/utils/logger'import { FetchResponse, fetch, internalFetch } from '~/common/utils/request'import { createTestTeamFixture, uniqueTestId } from '~/tests/helpers/sql'import { Hub, Team } from '../../../types'+import { CyclotronJobInvocationHogFlow } from '../../types'import { HOG_FILTERS_EXAMPLES } from '../../_tests/examples'import { createExampleHogFlowInvocation } from '../../_tests/fixtures-hogflows'import { CohortMembershipRepository } from '../cohorts/cohort-membership-repository'@@ -37,6 +39,7 @@ import { RecipientPreferencesService } from '../messaging/recipient-preferences.import { RecipientTokensService } from '../messaging/recipient-tokens.service'import { HogFlowExecutorService, createHogFlowInvocation } from './hogflow-executor.service'import { HogFlowFunctionsService } from './hogflow-functions.service'+import { AcceptedSend } from './transactional-email'// Mock before importing fetchjest.mock('~/common/utils/request', () => {@@ -3298,6 +3301,70 @@ describe('Hogflow Executor', () => {},}) as const+ const emailFlow = (email: Record<string, any>): HogFlow =>+ new FixtureHogFlowBuilder()+ .withTeamId(team.id)+ .withExitCondition('exit_only_at_end')+ .withWorkflow({+ actions: {+ trigger: {+ type: 'trigger',+ config: {+ type: 'event',+ filters: HOG_FILTERS_EXAMPLES.no_filters.filters ?? {},+ },+ },+ email_1: {+ type: 'function_email',+ config: {+ template_id: 'template-email-routing-test',+ inputs: { email: { value: email } },+ },+ },+ exit: {+ type: 'exit',+ config: {},+ },+ },+ edges: [+ { from: 'trigger', to: 'email_1', type: 'continue' },+ { from: 'email_1', to: 'exit', type: 'continue' },+ ],+ })+ .build()++ const flowInvocation = (hogFlow: HogFlow): CyclotronJobInvocationHogFlow =>+ createExampleHogFlowInvocation(hogFlow, {+ event: {+ ...createHogExecutionGlobals().event,+ event: '$pageview',+ },+ })++ const storedStep = () => ({+ to: { email: '[email protected]', name: 'Recipient' },+ from: { integrationId, email: '[email protected]' },+ subject: 'Test Email',+ text: 'Test text',+ html: '<p>Test html</p>',+ })++ const codeKeyStep = () => ({+ to: { email: '[email protected]', name: 'Recipient' },+ from: { integrationId: null },+ cc: '[email protected]',+ subject: '',+ text: '',+ html: '',+ })++ const codeSend = (): AcceptedSend => ({+ content: { subject: 'Reset your password', html: '<p>Hi {{ variables.name }}</p>', text: 'Hi' },+ from: { integrationId, email: '[email protected]', name: 'Security' },+ replyTo: ['[email protected]', '[email protected]'],+ tracking: { captureEngagement: true },+ })+it('should route email actions to the email queue', async () => {await insertEmailSetup('maildev')@@ -3528,5 +3595,115 @@ if (not res.success) { throw Error(res.error) }`,expect(result.invocation.state.currentAction?.id).toBe('email_2')expect(result.invocation.queueMetadata).toEqual({ originQueue: 'hogflow', originPriority: 0 })})++ it.each([+ {+ name: 'a template send as the step renders it',+ step: storedStep,+ send: (): AcceptedSend | undefined => undefined,+ expected: () => ({ type: 'email', ...storedStep() }),+ },+ {+ name: 'a code send for a local template as the step renders it',+ step: storedStep,+ send: (): AcceptedSend => ({ tracking: { captureEngagement: false } }),+ expected: () => ({ type: 'email', ...storedStep() }),+ },+ {+ name: "a code send with the caller's content, sender and reply-to over the step",+ step: codeKeyStep,+ send: codeSend,+ expected: () => ({+ type: 'email',+ to: { email: '[email protected]', name: 'Recipient' },+ from: { integrationId, email: '[email protected]', name: 'Security' },+ cc: '[email protected]',+ replyTo: '[email protected], [email protected]',+ subject: 'Reset your password',+ preheader: '',+ text: 'Hi',+ html: '<p>Hi {{ variables.name }}</p>',+ }),+ },+ ])('hands the email queue $name', async ({ step, send, expected }) => {+ await insertEmailSetup('ses')+ const invocation = flowInvocation(emailFlow(step()))+ invocation.state.variables = { name: 'Ada' }+ invocation.state.send = send()++ const result = await executor.execute(invocation)++ expect(result.invocation.queueParameters).toEqual(expected())+ })++ it.each([+ {+ name: 'fails a code-key step without an accepted send',+ send: (): AcceptedSend | undefined => undefined,+ sent: 0,+ },+ { name: 'delivers a code-key step with an accepted send', send: codeSend, sent: 1 },+ ])('$name', async ({ send, sent }) => {+ await insertEmailSetup('ses')+ const sesSend = jest.spyOn(SESv2Client.prototype, 'send') as jest.SpyInstance+ sesSend.mockResolvedValue({ MessageId: 'code-key-email' })+ const invocation = flowInvocation(emailFlow(codeKeyStep()))+ invocation.state.send = send()++ let result = await executor.execute(invocation)+ const metrics = [...result.metrics]+ while (!result.finished) {+ result = await executor.execute(result.invocation)+ metrics.push(...result.metrics)+ }++ expect(metrics.filter((m) => m.metric_name === 'email_sent')).toHaveLength(sent)+ expect(sesSend).toHaveBeenCalledTimes(sent)+ })++ it.each([+ {+ captureEngagement: true,+ teamCaptures: true,+ savedWithoutSend: false,+ captured: ['$workflows_email_sent'],+ },+ { captureEngagement: false, teamCaptures: true, savedWithoutSend: false, captured: [] },+ { captureEngagement: true, teamCaptures: false, savedWithoutSend: false, captured: [] },+ { captureEngagement: false, teamCaptures: true, savedWithoutSend: true, captured: [] },+ ])(+ "captures the email's events by the code send's choice $captureEngagement and the team switch $teamCaptures, step state saved without the send: $savedWithoutSend",+ async ({ captureEngagement, teamCaptures, savedWithoutSend, captured }) => {+ await insertEmailSetup('ses')+ const sesSend = jest.spyOn(SESv2Client.prototype, 'send') as jest.SpyInstance+ sesSend.mockResolvedValue({ MessageId: 'captured-email' })+ await hub.postgres.query(+ PostgresUse.COMMON_WRITE,+ `INSERT INTO workflows_teamworkflowsconfig+ (team_id, capture_workflows_engagement_events, email_tracking_consent_mode,+ email_sending_suspension_reason, ses_tenant_sending_status, email_sending_tier)+ VALUES ($1, $2, 'off', '', '', 0)`,+ [team.id, teamCaptures],+ 'test-team-engagement-capture'+ )+ const invocation = flowInvocation(emailFlow(storedStep()))+ invocation.state.send = { tracking: { captureEngagement } }++ let result = await executor.execute(invocation)+ const events = [...result.capturedPostHogEvents]+ if (savedWithoutSend) {+ delete result.invocation.state.currentAction!.hogFunctionState!.send+ }+ while (!result.finished) {+ result = await executor.execute(result.invocation)+ events.push(...result.capturedPostHogEvents)+ }++ expect(+ events.map((event) => event.event).filter((event) => event.startsWith('$workflows_email_'))+ ).toEqual(captured)+ expect(sesSend).toHaveBeenCalledTimes(1)+ }+ )})})diff --git a/nodejs/src/cdp/services/hogflows/hogflow-functions.service.ts b/nodejs/src/cdp/services/hogflows/hogflow-functions.service.tsindex 0272e55af062..a154f0cd4885 100644--- a/nodejs/src/cdp/services/hogflows/hogflow-functions.service.ts+++ b/nodejs/src/cdp/services/hogflows/hogflow-functions.service.ts@@ -121,26 +121,28 @@ export class HogFlowFunctionsService {},}+ const functionState = invocation.state.currentAction?.hogFunctionState ?? {+ globals: await this.hogFunctionExecutor.hogExecutor.buildInputsWithGlobals(+ hogFunction,+ globalsWithSource,+ undefined,+ {+ appSourceId: invocation.parentRunId ?? invocation.hogFlow.id,+ instanceId: invocation.state.currentAction?.id,+ }+ ),+ timings: [],+ attempts: 0,+ actionId: invocation.state.currentAction?.id,+ actionStepCount: invocation.state.actionStepCount,+ customerTaskIdempotencyVersion: invocation.state.customerTaskIdempotencyVersion,+ rerunAttempts: invocation.state.rerunAttempts,+ }+const hogFunctionInvocation: CyclotronJobInvocationHogFunction = {...invocation,hogFunction,- state: invocation.state.currentAction?.hogFunctionState ?? {- globals: await this.hogFunctionExecutor.hogExecutor.buildInputsWithGlobals(- hogFunction,- globalsWithSource,- undefined,- {- appSourceId: invocation.parentRunId ?? invocation.hogFlow.id,- instanceId: invocation.state.currentAction?.id,- }- ),- timings: [],- attempts: 0,- actionId: invocation.state.currentAction?.id,- actionStepCount: invocation.state.actionStepCount,- customerTaskIdempotencyVersion: invocation.state.customerTaskIdempotencyVersion,- rerunAttempts: invocation.state.rerunAttempts,- },+ state: { ...functionState, ...(invocation.state.send && { send: invocation.state.send }) },}return hogFunctionInvocationdiff --git a/nodejs/src/cdp/services/hogflows/hogflow-manager.service.test.ts b/nodejs/src/cdp/services/hogflows/hogflow-manager.service.test.tsindex c4bdd4223116..6f3fe6bfcb5b 100644--- a/nodejs/src/cdp/services/hogflows/hogflow-manager.service.test.ts+++ b/nodejs/src/cdp/services/hogflows/hogflow-manager.service.test.ts@@ -10,7 +10,7 @@ import { createTeam, createTestTeamFixture } from '~/tests/helpers/sql'import { INVALID_TEAM_ID_KEYS } from '~/tests/helpers/team'import { Hub } from '~/types'-import { insertHogFlow } from '../../_tests/fixtures-hogflows'+import { hideHogFlowsFromReplica, insertHogFlow } from '../../_tests/fixtures-hogflows'import { HogFlowManagerService } from './hogflow-manager.service'describe('HogFlowManager', () => {@@ -148,6 +148,133 @@ describe('HogFlowManager', () => {}, 10000)})+ it('reads the primary once per batch of workflows the database no longer has, and again once the miss expires', async () => {+ const [deletedId, secondDeletedId, thirdDeletedId] = Array.from(+ { length: 3 },+ () => new FixtureHogFlowBuilder().withTeamId(teamId1).build().id+ )+ const querySpy = jest.spyOn(hub.postgres, 'query')+ const primaryReads = (): number =>+ querySpy.mock.calls.filter(([use, , , tag]) => use === PostgresUse.COMMON_WRITE && tag === 'fetchHogFlows')+ .length+ const baseNow = Date.now()++ try {+ const flows = await Promise.all(+ Array.from({ length: 5 }, () => manager.getHogFlowCheckingPrimaryOnMiss(deletedId))+ )+ expect(flows).toEqual(Array(5).fill(null))+ expect(primaryReads()).toBe(1)++ jest.spyOn(Date, 'now').mockReturnValue(baseNow + 59_000)+ expect(await manager.getHogFlowCheckingPrimaryOnMiss(deletedId)).toBeNull()+ expect(primaryReads()).toBe(1)++ jest.spyOn(Date, 'now').mockReturnValue(baseNow + 61_000)+ expect(await manager.getHogFlowCheckingPrimaryOnMiss(deletedId)).toBeNull()+ expect(primaryReads()).toBe(2)++ const batch = await manager.getHogFlowsCheckingPrimaryOnMiss([+ secondDeletedId,+ thirdDeletedId,+ secondDeletedId,+ hogFlows[0].id,+ ])+ expect(batch).toEqual({+ [secondDeletedId]: null,+ [thirdDeletedId]: null,+ [hogFlows[0].id]: expect.objectContaining({ id: hogFlows[0].id }),+ })+ expect(primaryReads()).toBe(3)+ } finally {+ jest.spyOn(Date, 'now').mockReturnValue(baseNow)+ }+ })++ it('serves a workflow only the primary has from cache until a reload names it or the entry expires', async () => {+ const lateFlow = await insertHogFlow(+ hub.postgres,+ new FixtureHogFlowBuilder().withTeamId(teamId1).withStatus('active').build()+ )+ const querySpy = hideHogFlowsFromReplica(hub.postgres, [lateFlow.id])+ const reads = (): number[] =>+ [PostgresUse.COMMON_READ, PostgresUse.COMMON_WRITE].map(+ (use) =>+ querySpy.mock.calls.filter(([target, , , tag]) => target === use && tag === 'fetchHogFlows').length+ )+ const lateFlowMatch = expect.objectContaining({ id: lateFlow.id })+ const baseNow = Date.now()++ try {+ expect(await manager.getHogFlowCheckingPrimaryOnMiss(lateFlow.id)).toEqual(lateFlowMatch)+ expect(reads()).toEqual([1, 1])++ jest.spyOn(Date, 'now').mockReturnValue(baseNow + 59_000)+ expect(await manager.getHogFlowsCheckingPrimaryOnMiss([lateFlow.id])).toEqual({+ [lateFlow.id]: lateFlowMatch,+ })+ expect(reads()).toEqual([1, 1])++ manager['onHogFlowsReloaded'](teamId1, [lateFlow.id])+ expect(await manager.getHogFlowCheckingPrimaryOnMiss(lateFlow.id)).toEqual(lateFlowMatch)+ expect(reads()).toEqual([2, 2])++ jest.spyOn(Date, 'now').mockReturnValue(baseNow + 120_000)+ expect(await manager.getHogFlowCheckingPrimaryOnMiss(lateFlow.id)).toEqual(lateFlowMatch)+ expect(reads()).toEqual([3, 3])+ } finally {+ jest.spyOn(Date, 'now').mockReturnValue(baseNow)+ }+ })++ it('keeps no primary result from a read that a reload overtook', async () => {+ const lateFlow = await insertHogFlow(+ hub.postgres,+ new FixtureHogFlowBuilder().withTeamId(teamId1).withStatus('active').build()+ )+ const querySpy = hideHogFlowsFromReplica(hub.postgres, [lateFlow.id])+ const hideFromReplica = querySpy.getMockImplementation()!+ let primaryReadStarted: () => void = () => {}+ const primaryReadInFlight = new Promise<void>((resolve) => {+ primaryReadStarted = resolve+ })+ let releasePrimaryRead: () => void = () => {}+ const primaryReadReleased = new Promise<void>((resolve) => {+ releasePrimaryRead = resolve+ })+ let holdNextPrimaryRead = true+ querySpy.mockImplementation(async (target, queryString, values, tag, logLevel) => {+ const result = await hideFromReplica(target, queryString, values, tag, logLevel)+ if (holdNextPrimaryRead && target === PostgresUse.COMMON_WRITE && tag === 'fetchHogFlows') {+ holdNextPrimaryRead = false+ primaryReadStarted()+ await primaryReadReleased+ }+ return result+ })++ try {+ const overtakenLookup = manager.getHogFlowCheckingPrimaryOnMiss(lateFlow.id)+ await primaryReadInFlight+ await hub.postgres.query(+ PostgresUse.COMMON_WRITE,+ `UPDATE posthog_hogflow SET status = 'draft' WHERE id = $1`,+ [lateFlow.id],+ 'testKey'+ )+ manager['onHogFlowsReloaded'](teamId1, [lateFlow.id])+ releasePrimaryRead()+ await overtakenLookup++ expect(await manager.getHogFlowCheckingPrimaryOnMiss(lateFlow.id)).toMatchObject({+ id: lateFlow.id,+ status: 'draft',+ })+ } finally {+ querySpy.mockRestore()+ }+ })+describe('email sending pause', () => {// The pause only reaches the send choke point through the manager's column list, so a// column left out of HOG_FLOW_FIELDS silently lets a paused workflow keep sending.diff --git a/nodejs/src/cdp/services/hogflows/hogflow-manager.service.ts b/nodejs/src/cdp/services/hogflows/hogflow-manager.service.tsindex d5c3baf2461f..9b5b033b98a5 100644--- a/nodejs/src/cdp/services/hogflows/hogflow-manager.service.ts+++ b/nodejs/src/cdp/services/hogflows/hogflow-manager.service.ts@@ -49,9 +49,14 @@ const HOG_FLOW_FIELDS = [export type HogFlowTeamInfo = Pick<HogFlow, 'id' | 'team_id' | 'version'>+const PRIMARY_RESULT_TTL_MS = 60 * 1000+const PRIMARY_RESULT_MAX_SIZE = 10_000+export class HogFlowManagerService {private lazyLoader: LazyLoader<HogFlow>private lazyLoaderByTeam: LazyLoader<HogFlowTeamInfo[]>+ private primaryReads = new Map<HogFlow['id'], Promise<HogFlow | null>>()+ private primaryResults = new Map<HogFlow['id'], { flow: HogFlow | null; until: number }>()constructor(private postgres: PostgresRouter,@@ -142,6 +147,76 @@ export class HogFlowManagerService {return (await this.lazyLoader.get(id)) ?? null}+ public async getHogFlowCheckingPrimaryOnMiss(id: HogFlow['id']): Promise<HogFlow | null> {+ return (await this.getHogFlowsCheckingPrimaryOnMiss([id]))[id]+ }++ public async getHogFlowsCheckingPrimaryOnMiss(+ ids: HogFlow['id'][]+ ): Promise<Record<HogFlow['id'], HogFlow | null>> {+ const remembered = Object.fromEntries(ids.map((id) => [id, this.rememberedPrimaryResult(id)]))+ const cached = await this.lazyLoader.getMany(ids.filter((id) => !remembered[id]))+ const misses = [...new Set(ids.filter((id) => remembered[id] === undefined && !cached[id]))]+ const fromPrimary = await this.readPrimary(misses)+ return Object.fromEntries(ids.map((id) => [id, remembered[id] ?? cached[id] ?? fromPrimary[id] ?? null]))+ }++ private async readPrimary(ids: HogFlow['id'][]): Promise<Record<HogFlow['id'], HogFlow | null>> {+ const unread = ids.filter((id) => !this.primaryReads.has(id))+ if (unread.length > 0) {+ const read = this.fetchHogFlows(unread, PostgresUse.COMMON_WRITE)+ for (const id of unread) {+ const settled: Promise<HogFlow | null> = read.then((flows) =>+ this.settlePrimaryRead(id, flows[id] ?? null, settled)+ )+ this.primaryReads.set(id, settled)+ const forget = (): void => {+ if (this.primaryReads.get(id) === settled) {+ this.primaryReads.delete(id)+ }+ }+ settled.then(forget, forget)+ }+ }+ const reads = ids.map((id) => this.primaryReads.get(id) ?? Promise.resolve(null))+ const flows = await Promise.all(reads)+ return Object.fromEntries(ids.map((id, index) => [id, flows[index]]))+ }++ private settlePrimaryRead(id: HogFlow['id'], flow: HogFlow | null, read: Promise<HogFlow | null>): HogFlow | null {+ if (this.primaryReads.get(id) !== read) {+ return flow+ }+ if (flow) {+ this.lazyLoader.markForRefresh(id)+ }+ this.rememberPrimaryResult(id, flow)+ return flow+ }++ private rememberedPrimaryResult(id: HogFlow['id']): HogFlow | null | undefined {+ const remembered = this.primaryResults.get(id)+ if (remembered === undefined) {+ return undefined+ }+ if (remembered.until > Date.now()) {+ return remembered.flow+ }+ this.primaryResults.delete(id)+ return undefined+ }++ private rememberPrimaryResult(id: HogFlow['id'], flow: HogFlow | null): void {+ this.primaryResults.delete(id)+ if (this.primaryResults.size >= PRIMARY_RESULT_MAX_SIZE) {+ const oldest = this.primaryResults.keys().next().value+ if (oldest !== undefined) {+ this.primaryResults.delete(oldest)+ }+ }+ this.primaryResults.set(id, { flow, until: Date.now() + PRIMARY_RESULT_TTL_MS })+ }+public async getHogFlows(ids: HogFlow['id'][]): Promise<Record<HogFlow['id'], HogFlow | null>> {return await this.lazyLoader.getMany(ids)}@@ -149,6 +224,10 @@ export class HogFlowManagerService {private onHogFlowsReloaded(teamId: Team['id'], hogFlowIds: HogFlow['id'][]): void {this.lazyLoaderByTeam.markForRefresh(teamId.toString())this.lazyLoader.markForRefresh(hogFlowIds)+ hogFlowIds.forEach((id) => {+ this.primaryResults.delete(id)+ this.primaryReads.delete(id)+ })}private async fetchTeamHogFlows(teamIds: string[]): Promise<Record<string, HogFlowTeamInfo[]>> {@@ -173,11 +252,14 @@ export class HogFlowManagerService {return byTeam}- private async fetchHogFlows(ids: string[]): Promise<Record<string, HogFlow | undefined>> {+ private async fetchHogFlows(+ ids: string[],+ use: PostgresUse = PostgresUse.COMMON_READ+ ): Promise<Record<string, HogFlow | undefined>> {logger.debug('[HogFlowManager]', 'Fetching hog flows', { ids })const response = await this.postgres.query<HogFlow>(- PostgresUse.COMMON_READ,+ use,`SELECT ${HOG_FLOW_FIELDS.join(', ')} FROM posthog_hogflow WHERE id = ANY($1)`,[ids],'fetchHogFlows'diff --git a/nodejs/src/cdp/services/hogflows/transactional-email.ts b/nodejs/src/cdp/services/hogflows/transactional-email.tsindex e1671bf1afd1..1577b5d207a4 100644--- a/nodejs/src/cdp/services/hogflows/transactional-email.ts+++ b/nodejs/src/cdp/services/hogflows/transactional-email.ts@@ -1,9 +1,56 @@import { Counter, Histogram } from 'prom-client'import { z } from 'zod'+import { CyclotronInvocationQueueParametersEmailType } from '~/cdp/schema/cyclotron'++export const AcceptedSendSchema = z+ .object({+ content: z+ .object({+ subject: z.string().min(1),+ html: z.string().optional(),+ text: z.string().optional(),+ })+ .strict()+ .readonly()+ .refine((content) => !!content.html || !!content.text, { message: 'content needs html or text' })+ .optional(),+ from: z+ .object({+ integrationId: z.number().int(),+ email: z.string().optional(),+ name: z.string().optional(),+ })+ .strict()+ .readonly()+ .optional(),+ replyTo: z.array(z.string()).readonly().optional(),+ tracking: z.object({ captureEngagement: z.boolean() }).strict().readonly(),+ })+ .strict()+ .readonly()++export type AcceptedSend = z.infer<typeof AcceptedSendSchema>++export const acceptedSendEmailFields = (+ send: AcceptedSend | undefined+): Partial<CyclotronInvocationQueueParametersEmailType> => ({+ ...(send?.content && {+ subject: send.content.subject,+ html: send.content.html ?? '',+ text: send.content.text ?? '',+ preheader: '',+ }),+ ...(send?.from && { from: { ...send.from } }),+ ...(send?.replyTo && { replyTo: send.replyTo.join(', ') }),+})++export const RECIPIENT_ADDRESS_PATTERN =+ /^[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)*@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)+$/+// What Django sends to run a transactional email's workflow once.export const TransactionalEmailRequestSchema = z.object({- to: z.string().email(),+ to: z.string().email({ pattern: RECIPIENT_ADDRESS_PATTERN }),// The person the send is tracked against. Django falls back to the recipient's address.distinct_id: z.string().min(1),// The caller's `message_data`, available to the template as `variables`.@@ -12,6 +59,13 @@ export const TransactionalEmailRequestSchema = z.object({idempotency_key: z.string().min(1).max(255).optional(),})+export const TEMPLATE_SEND_TOKEN_CLAIMS = { kind: undefined }+export const CODE_SEND_TOKEN_CLAIMS = { kind: 'code_send' }++export const AcceptedSendRequestSchema = TransactionalEmailRequestSchema.extend({ send: AcceptedSendSchema })++export type TransactionalEmailRequest = z.infer<typeof TransactionalEmailRequestSchema> & { send?: AcceptedSend }+export type TransactionalEmailOutcome =| 'queued'| 'replayed'diff --git a/nodejs/src/cdp/services/messaging/email-tracking.service.test.ts b/nodejs/src/cdp/services/messaging/email-tracking.service.test.tsindex bedfc201df44..1a6ea76fb6e1 100644--- a/nodejs/src/cdp/services/messaging/email-tracking.service.test.ts+++ b/nodejs/src/cdp/services/messaging/email-tracking.service.test.ts@@ -1,14 +1,15 @@import { createMockJobQueue } from '~/tests/helpers/mocks/job-queue.mock'import { mockProducerObserver } from '~/tests/helpers/mocks/producer.mock'-import { mockFetch } from '~/tests/helpers/mocks/request.mock'+import { mockFetch, mockInternalFetch } from '~/tests/helpers/mocks/request.mock'import { Server } from 'http'+import { register } from 'prom-client'import supertest from 'supertest'import express from 'ultimate-express'import { FixtureHogFlowBuilder } from '~/cdp/_tests/builders/hogflow.builder'-import { insertHogFunction } from '~/cdp/_tests/fixtures'-import { insertHogFlow } from '~/cdp/_tests/fixtures-hogflows'+import { createExampleInvocation, insertHogFunction, insertIntegration } from '~/cdp/_tests/fixtures'+import { hideHogFlowsFromReplica, insertHogFlow } from '~/cdp/_tests/fixtures-hogflows'import { CdpApi } from '~/cdp/cdp-api'import { CyclotronJobInvocationHogFunction, HogFunctionType } from '~/cdp/types'import { setupExpressApp } from '~/common/api/router'@@ -17,6 +18,7 @@ import { KAFKA_APP_METRICS_2, KAFKA_EMAIL_STATUS, KAFKA_LOG_ENTRIES } from '~/coimport { closeHub, createHub } from '~/common/utils/db/hub'import { PostgresUse } from '~/common/utils/db/postgres'import * as envUtils from '~/common/utils/env-utils'+import { parseJSON } from '~/common/utils/json-parse'import { UUIDT } from '~/common/utils/utils'import { createCdpConsumerDeps } from '~/tests/helpers/cdp'import { waitForExpect } from '~/tests/helpers/expectations'@@ -46,6 +48,7 @@ describe('EmailTrackingService', () => {team = (await createTestTeamFixture(hub.postgres)).teammockFetch.mockClear()+ mockInternalFetch.mockClear()mockProducerObserver.resetKafkaProducer()})@@ -201,14 +204,31 @@ describe('EmailTrackingService', () => {} as anyconst phIdOf = (html: string): string => html.match(/ph_id=([A-Za-z0-9._-]+)/)![1]- it('includes distinct_id in the public tracking URLs in dev/test', () => {- const out = addTrackingToEmail(- '<body><a href="https://example.com">x</a></body>',- invocationWithDistinctId,- signer- )- expect(signer.parse(phIdOf(out))?.distinctId).toBe('leaky-id')- })+ it.each([+ { captureEngagement: undefined, distinctId: 'leaky-id' },+ { captureEngagement: true, distinctId: 'leaky-id' },+ { captureEngagement: false, distinctId: undefined },+ ])(+ 'mints the pixel and redirect codes in dev/test with the distinct_id the send choice $captureEngagement allows',+ ({ captureEngagement, distinctId }) => {+ const out = addTrackingToEmail(+ '<body><a href="https://example.com">x</a></body>',+ {+ ...invocationWithDistinctId,+ state: {+ ...invocationWithDistinctId.state,+ send: captureEngagement === undefined ? undefined : { tracking: { captureEngagement } },+ },+ },+ signer+ )+ const codes = [...out.matchAll(/ph_id=([A-Za-z0-9_.-]+)/g)].map((match) => signer.parse(match[1]))+ expect(codes).toHaveLength(2)+ for (const code of codes) {+ expect(code).toMatchObject({ functionId: 'fn-1', invocationId: 'inv-1', distinctId })+ }+ }+ )it('omits distinct_id from the public tracking URLs in production (Referer-leak guard)', () => {const devSpy = jest.spyOn(envUtils, 'isDevEnv').mockReturnValue(false)@@ -457,15 +477,15 @@ describe('EmailTrackingService', () => {parentRunId,workflowVersion,transactionalMessageId,+ headerCode,}: {functionId: stringparentRunId?: stringworkflowVersion?: number// Sends the bounce as SES reports it for a transactional email with this message id.transactionalMessageId?: string+ headerCode?: string}): Promise<supertest.Response> => {- // Third arg opts into the versioned payload, which `generate` won't emit by default- // until phase two of the rollout — see EMIT_VERSIONED_PAYLOAD in tracking-code.ts.const trackingCode = signer.generate({functionId,@@ -484,7 +504,7 @@ describe('EmailTrackingService', () => {source: '[email protected]',messageId: 'ses-message-id',destination: ['[email protected]'],- headers: [{ name: TRACKING_CODE_HEADER_NAME, value: trackingCode }],+ headers: [{ name: TRACKING_CODE_HEADER_NAME, value: headerCode ?? trackingCode }],...(transactionalMessageId? {commonHeaders: { to: ['[email protected]'] },@@ -514,11 +534,24 @@ describe('EmailTrackingService', () => {.send(JSON.stringify(envelope))}- it('writes a hog_flow log entry for a bounce that resolves to a flow', async () => {+ it.each([+ ['on the replica', false, false],+ ['only on the primary, during replica lag', true, false],+ ['on the replica, while the hog function lookup fails', false, true],+ ['only on the primary, while the hog function lookup fails', true, true],+ ])('writes a hog_flow log entry for a bounce that resolves to a flow %s', async (_, lagging, fail) => {const hogFlow = await insertHogFlow(hub.postgres,new FixtureHogFlowBuilder().withTeamId(team.id).build())+ if (lagging) {+ hideHogFlowsFromReplica(hub.postgres, [hogFlow.id])+ }+ if (fail) {+ jest.spyOn(api['emailTrackingService']['hogFunctionManager'], 'getHogFunctions').mockRejectedValue(+ new Error('hog function lookup failed')+ )+ }const res = await postBounce({ functionId: hogFlow.id })expect(res.status).toBe(200)@@ -541,11 +574,36 @@ describe('EmailTrackingService', () => {expect(metrics.map((m) => m.value.metric_name)).toEqual(['email_bounced', 'email_bounced_hard'])expect(metrics[0].value).toMatchObject({team_id: team.id,+ app_source: 'hog_flow',+ app_source_id: hogFlow.id,metric_name: 'email_bounced',metric_kind: 'email',})})+ it('counts a failed primary lookup as a lookup failure for a bounce on a flow only the primary has', async () => {+ const hogFlow = await insertHogFlow(+ hub.postgres,+ new FixtureHogFlowBuilder().withTeamId(team.id).build()+ )+ hideHogFlowsFromReplica(hub.postgres, [hogFlow.id])+ jest.spyOn(+ api['emailTrackingService']['hogFlowManager'],+ 'getHogFlowsCheckingPrimaryOnMiss'+ ).mockRejectedValue(new Error('primary lookup failed'))+ const lookupFailures = async (): Promise<number> =>+ (await register.getSingleMetric('email_tracking_errors_total')!.get()).values.find(+ (v) => v.labels.error_type === 'hog_flow_lookup_failed' && v.labels.source === 'ses'+ )?.value ?? 0+ const before = await lookupFailures()++ const res = await postBounce({ functionId: hogFlow.id })+ expect(res.status).toBe(200)++ expect(await lookupFailures()).toBeGreaterThan(before)+ expect(mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_LOG_ENTRIES)).toHaveLength(0)+ })+it('records the bounced status of a transactional email', async () => {const hogFlow = await insertHogFlow(hub.postgres,@@ -570,7 +628,9 @@ describe('EmailTrackingService', () => {])})- it('records the metric but writes no log entry when the bounce resolves to a hog_function', async () => {+ it('records the metric but writes no log entry and reads no workflow from the primary when the bounce resolves to a hog_function', async () => {+ const query = jest.spyOn(hub.postgres, 'query')+const res = await postBounce({ functionId: hogFunction.id })expect(res.status).toBe(200)@@ -586,6 +646,11 @@ describe('EmailTrackingService', () => {const logs = mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_LOG_ENTRIES)expect(logs).toHaveLength(0)+ expect(+ query.mock.calls.filter(+ ([target, , , tag]) => target === PostgresUse.COMMON_WRITE && tag === 'fetchHogFlows'+ )+ ).toHaveLength(0)})it('attributes the bounce to the version that sent it, not the version live when it lands', async () => {@@ -613,6 +678,76 @@ describe('EmailTrackingService', () => {})})+ it.each([+ { captureEngagement: true, teamCaptures: true, captured: 1 },+ { captureEngagement: undefined, teamCaptures: true, captured: 1 },+ { captureEngagement: false, teamCaptures: true, captured: 0 },+ { captureEngagement: true, teamCaptures: false, captured: 0 },+ ])(+ "captures the bounce of a sent email by the send's choice $captureEngagement and the team switch $teamCaptures",+ async ({ captureEngagement, teamCaptures, captured }) => {+ await hub.postgres.query(+ PostgresUse.COMMON_WRITE,+ `INSERT INTO workflows_teamworkflowsconfig+ (team_id, capture_workflows_engagement_events, email_tracking_consent_mode,+ email_sending_suspension_reason, ses_tenant_sending_status, email_sending_tier)+ VALUES ($1, $2, 'off', '', '', 0)`,+ [team.id, teamCaptures],+ 'test-set-engagement-capture'+ )+ const hogFlow = await insertHogFlow(+ hub.postgres,+ new FixtureHogFlowBuilder().withTeamId(team.id).build()+ )+ const integration = await insertIntegration(hub.postgres, team.id, {+ kind: 'email',+ config: {+ email: '[email protected]',+ name: 'Sender',+ domain: 'posthog-test.com',+ verified: true,+ provider: 'ses',+ },+ })+ const emailService = api['hogExecutorAsync']['deps'].emailService+ const sesSend = jest+ .spyOn(emailService.sesV2Client!, 'send')+ .mockResolvedValue({ MessageId: 'ses-message-id' } as never)+ const invocation = createExampleInvocation({+ id: hogFlow.id,+ team_id: team.id,+ })+ invocation.state.globals.event.distinct_id = 'user-1'+ if (captureEngagement !== undefined) {+ invocation.state.send = { tracking: { captureEngagement } }+ }+ invocation.queueParameters = {+ type: 'email',+ to: { email: '[email protected]', name: 'User' },+ from: { integrationId: integration.id },+ subject: 'Reset your password',+ text: 'Hi',+ html: '<p>Hi</p>',+ }++ expect((await emailService.executeSendEmail(invocation)).error).toBeUndefined()+ const headers: { Name: string; Value: string }[] = (sesSend.mock.calls[0][0] as any).input.Content+ .Simple.Headers+ const headerCode = headers.find((header) => header.Name === TRACKING_CODE_HEADER_NAME)!.Value+ mockInternalFetch.mockClear()++ const res = await postBounce({ functionId: hogFlow.id, headerCode })+ expect(res.status).toBe(200)++ const capturedEvents = mockInternalFetch.mock.calls.map(+ ([, options]) => parseJSON(String(options?.body)).event+ )+ expect(capturedEvents.filter((event) => event === '$workflows_email_bounced')).toHaveLength(+ captured+ )+ }+ )+it('keys the log entry under parentRunId for batch-triggered runs', async () => {const hogFlow = await insertHogFlow(hub.postgres,diff --git a/nodejs/src/cdp/services/messaging/email-tracking.service.ts b/nodejs/src/cdp/services/messaging/email-tracking.service.tsindex 723e8040506f..6bcea0f6056f 100644--- a/nodejs/src/cdp/services/messaging/email-tracking.service.ts+++ b/nodejs/src/cdp/services/messaging/email-tracking.service.ts@@ -6,6 +6,7 @@ import { HogFlow } from '~/cdp/schema/hogflow'import {CyclotronJobInvocationHogFlow,CyclotronJobInvocationHogFunction,+ HogFunctionType,LogEntry,LogEntryLevel,MinimalAppMetric,@@ -127,6 +128,14 @@ export const resolveEmailSendingVersion = (invocation: CyclotronJobInvocationHog: undefined}+export const resolveEmailTrackingDistinctId = (+ invocation: Pick<CyclotronJobInvocationHogFunction, 'state' | 'hogFunction'>+): string | undefined => {+ return invocation.state?.send?.tracking.captureEngagement === false+ ? undefined+ : resolveEmailEngagementDistinctId(invocation)+}+// HTML attribute values arrive entity-encoded (e.g. `&`, `&`). Decode before// percent-encoding for the tracking redirect, otherwise `?a=1&b=2` round-trips// through `target=` as a literal `&` and breaks the destination page's query string.@@ -156,7 +165,7 @@ export const addTrackingToEmail = (// record metrics. In production they don't (SES webhooks own open/click attribution via the// signed header), so embedding distinct_id in the public `ph_id` would be unused — and worse,// a tracked-link click could leak the recipient identifier to the destination via the Referer.- const distinctId = isDevEnv() || isTestEnv() ? resolveEmailEngagementDistinctId(invocation) : undefined+ const distinctId = isDevEnv() || isTestEnv() ? resolveEmailTrackingDistinctId(invocation) : undefined// The pixel and redirect codes carry the sending version too. SES-attributed opens and clicks// read the version off the signed header instead, but a provider we track ourselves has these// URLs as its only carrier, and without the version its engagement cannot split by version.@@ -338,7 +347,9 @@ export class EmailTrackingService {// The function ID could be one or the other so we load bothconst [hogFunction, hogFlow] = await Promise.all([this.hogFunctionManager.getHogFunction(functionId).catch(() => null),- this.hogFlowManager.getHogFlow(functionId).catch(() => null),+ this.getFeedbackHogFlows([functionId], source)+ .then((hogFlows) => hogFlows[functionId])+ .catch(() => null),])let teamId = hogFunction?.team_id ?? hogFlow?.team_id@@ -438,7 +449,7 @@ export class EmailTrackingService {let hogFlows: Record<string, HogFlow | null> = {}try {- hogFlows = await this.hogFlowManager.getHogFlows(uniqueFunctionIds)+ hogFlows = await this.getFeedbackHogFlows(uniqueFunctionIds, 'ses')} catch (error) {logger.error('[EmailTrackingService] trackLogs: Failed to load hog flows', { error })emailTrackingErrorsCounter.inc({ error_type: 'hog_flow_lookup_failed', source: 'ses' })@@ -609,6 +620,32 @@ export class EmailTrackingService {}}+ private async getFeedbackHogFlows(+ ids: HogFlow['id'][],+ source: 'direct' | 'ses'+ ): Promise<Record<HogFlow['id'], HogFlow | null>> {+ const [onReplica, hogFunctions] = await Promise.all([+ this.hogFlowManager.getHogFlows(ids),+ this.hogFunctionManager.getHogFunctions(ids).catch((error): Record<string, HogFunctionType | null> => {+ logger.warn('[EmailTrackingService] Failed to load hog functions', { error })+ return {}+ }),+ ])+ const unknown = ids.filter((id) => !onReplica[id] && !hogFunctions[id])+ const onPrimary: Record<HogFlow['id'], HogFlow | null> =+ unknown.length > 0+ ? await this.hogFlowManager.getHogFlowsCheckingPrimaryOnMiss(unknown).catch((error) => {+ logger.error('[EmailTrackingService] Failed to load hog flows from the primary', {+ error,+ ids: unknown,+ })+ emailTrackingErrorsCounter.inc({ error_type: 'hog_flow_lookup_failed', source })+ return {}+ })+ : {}+ return Object.fromEntries(ids.map((id) => [id, onReplica[id] ?? onPrimary[id] ?? null]))+ }+private parseTrackingParams(query: Record<string, any>): {functionId?: stringinvocationId?: stringdiff --git a/nodejs/src/cdp/services/messaging/email.service.ts b/nodejs/src/cdp/services/messaging/email.service.tsindex ae1f2961ce96..768117da547b 100644--- a/nodejs/src/cdp/services/messaging/email.service.ts+++ b/nodejs/src/cdp/services/messaging/email.service.ts@@ -30,8 +30,8 @@ import { EmailStatusService, isTransactionalEmailSend } from './email-status.serimport { EmailSuppressionService, SuppressionCheckOptions } from './email-suppression.service'import {addTrackingToEmail,- resolveEmailEngagementDistinctId,resolveEmailSendingVersion,+ resolveEmailTrackingDistinctId,} from './email-tracking.service'import { mailDevTransport, mailDevWebUrl } from './helpers/maildev'import { maybeAddPreheaderToEmail } from './helpers/preheader'@@ -878,7 +878,7 @@ export class EmailService {}}- const distinctId = resolveEmailEngagementDistinctId(invocation)+ const distinctId = resolveEmailTrackingDistinctId(invocation)if (distinctId &&!isTest &&@@ -1227,7 +1227,7 @@ export class EmailService {if (!this.sesV2Client) {throw new Error('SES is not configured - set SES_REGION and AWS credentials')}- const distinctId = resolveEmailEngagementDistinctId(result.invocation)+ const distinctId = resolveEmailTrackingDistinctId(result.invocation)// Full signed code (with distinct_id + isTest) rides in the header; the short unsigned// carrier (no distinct_id/isTest) goes in the SES EmailTag, guaranteed under the 256-char// tag-value limit. The webhook reads the header first and only falls back to the tag.diff --git a/nodejs/src/cdp/services/monitoring/hog-invocation-results.service.test.ts b/nodejs/src/cdp/services/monitoring/hog-invocation-results.service.test.tsindex 02009788a033..38157d680e8f 100644--- a/nodejs/src/cdp/services/monitoring/hog-invocation-results.service.test.ts+++ b/nodejs/src/cdp/services/monitoring/hog-invocation-results.service.test.ts@@ -144,22 +144,29 @@ describe('HogInvocationResultsService', () => {expect(decoded).not.toContain('abc123')})+ const send = {+ content: { subject: 'Your login code', html: '<p>Your code is 482913</p>' },+ tracking: { captureEngagement: false },+ }+ const messageData = { variables: { code: '482913' }, send }+ const noMessageData = { variables: undefined, send: undefined }+it.each([- ['an ordinary email', false, {}, { code: '482913' }],- ['a sensitive email', true, {}, undefined],- ['an ordinary push', false, {}, { code: '482913' }, 'function_push'],- ['a sensitive push', true, {}, undefined, 'function_push'],- ['a recovery stub of a sensitive run', undefined, { sensitive: true }, undefined],+ ['an ordinary email', false, {}, messageData],+ ['a sensitive email', true, {}, noMessageData],+ ['an ordinary push', false, {}, messageData, 'function_push'],+ ['a sensitive push', true, {}, noMessageData, 'function_push'],+ ['a recovery stub of a sensitive run', undefined, { sensitive: true }, noMessageData],['a recovery stub of an unstamped transactional run',undefined,{ event: { event: '$transactional_email' } },- undefined,+ noMessageData,],- ['a recovery stub of an ordinary run', undefined, {}, { code: '482913' }],+ ['a recovery stub of an ordinary run', undefined, {}, messageData],])('keeps the message data in a workflow snapshot only for %s',- async (_name, sensitive, state, variables, stepType = 'function_email') => {+ async (_name, sensitive, state, expected, stepType = 'function_email') => {const hogFlow =sensitive === undefined? ({ id: 'flow-1' } as any)@@ -181,7 +188,7 @@ describe('HogInvocationResultsService', () => {edges: [],}).build()- const invocation = createExampleHogFlowInvocation(hogFlow, { variables: { code: '482913' } })+ const invocation = createExampleHogFlowInvocation(hogFlow, { variables: { code: '482913' }, send })// A step paused for a provider retry keeps the rendered message on its VM stack.const vmState = { stack: [{ title: 'Your code is 482913' }] }const person = { id: 'person-1', properties: { code: '482913' } }@@ -198,9 +205,9 @@ describe('HogInvocationResultsService', () => {const globals = (await decodeInvocationGlobals(parseProducedRows(outputs)[0].invocation_globals)) as Record<string, any>- expect(globals.variables).toEqual(variables)- expect(globals.currentAction.hogFunctionState.vmState).toEqual(variables && vmState)- expect(globals.currentAction.hogFunctionState.globals.person).toEqual(variables && person)+ expect({ variables: globals.variables, send: globals.send }).toEqual(expected)+ expect(globals.currentAction.hogFunctionState.vmState).toEqual(expected.variables && vmState)+ expect(globals.currentAction.hogFunctionState.globals.person).toEqual(expected.variables && person)})diff --git a/nodejs/src/cdp/services/monitoring/hog-invocation-results.service.ts b/nodejs/src/cdp/services/monitoring/hog-invocation-results.service.tsindex 9eea0595b43d..8bd839575b82 100644--- a/nodejs/src/cdp/services/monitoring/hog-invocation-results.service.ts+++ b/nodejs/src/cdp/services/monitoring/hog-invocation-results.service.ts@@ -208,7 +208,7 @@ const INPUT_KEYS: ReadonlySet<string> = new Set(['inputs'])// A sensitive message carries reset links, login codes or private text, so its snapshot keeps no// variables either, and a rerun can't rebuild the send. A step paused mid-send (e.g. for a provider// retry) holds the rendered message on its VM stack, and the person and group properties it read, so those go too.-const SENSITIVE_KEYS: ReadonlySet<string> = new Set(['inputs', 'variables', 'vmState', 'person', 'groups'])+const SENSITIVE_KEYS: ReadonlySet<string> = new Set(['inputs', 'variables', 'vmState', 'person', 'groups', 'send'])export const SENSITIVE_VARIABLES_OMITTED = 'sensitive_variables_omitted'const stripKeys = <T>(value: T, keys: ReadonlySet<string>): T => {diff --git a/nodejs/src/cdp/types.ts b/nodejs/src/cdp/types.tsindex 710bbb724e6b..62ee80fa99ca 100644--- a/nodejs/src/cdp/types.ts+++ b/nodejs/src/cdp/types.ts@@ -13,6 +13,7 @@ import {PersonPropertyFilter,Team,} from '../types'+import type { AcceptedSend } from './services/hogflows/transactional-email'import type { HogErrorClass } from './utils/hog-error-classification'export type HogBytecode = any[]@@ -431,6 +432,7 @@ export type CyclotronJobInvocationHogFunctionContext = {sensitive?: booleanactionStepCount?: numbercustomerTaskIdempotencyVersion?: 1+ send?: AcceptedSend}export type WorkflowStepResumeStatus = 'completed' | 'failed' | 'cancelled'@@ -577,6 +579,7 @@ export type HogFlowInvocationContext = {// conversions are counted by the matcher and never touch this flag.conversionCounted?: booleanvariables?: Record<string, any>+ send?: AcceptedSend// Sticky counter incremented by the rerun paginator on rehydration. Lets// the lifecycle row producer derive `attempts` / `is_retry` for hog flows// the same way it does for hog functions, so the `max_attempts` guard ondiff --git a/nodejs/src/common/api/middleware/internal-api-auth.test.ts b/nodejs/src/common/api/middleware/internal-api-auth.test.tsindex b4c6c554b132..9c9b5c3b491b 100644--- a/nodejs/src/common/api/middleware/internal-api-auth.test.ts+++ b/nodejs/src/common/api/middleware/internal-api-auth.test.ts@@ -192,6 +192,8 @@ describe('createInternalApiAuthMiddleware', () => {['/api/projects/1/hog_flows/some-uuid/reschedule_parked'],['/api/projects/1/hog_flows/some-uuid/invocations/cancel'],['/api/projects/1/hog_flows/some-uuid/batch_jobs/some-job-id/cancel'],+ ['/api/projects/1/hog_flows/some-uuid/transactional_email'],+ ['/api/projects/1/hog_flows/some-uuid/transactional_email/send'],])('should skip auth for scoped-JWT route %s, which authenticates in its handler', (path) => {const middleware = createInternalApiAuthMiddleware({ secret: 'test-secret' })const req = mockRequest(path, {})@@ -210,6 +212,7 @@ describe('createInternalApiAuthMiddleware', () => {['/api/projects/1/hog_flows/some-uuid/batch_invocations/cancel'],['/api/some/other/route/cancel'],['/api/projects/1/hog_flows/some-uuid/reschedule_parked/extra'],+ ['/api/projects/1/hog_flows/some-uuid/transactional_email/send/extra'],])('should still require the secret for %s despite resembling a scoped route', (path) => {const middleware = createInternalApiAuthMiddleware({ secret: 'test-secret' })const req = mockRequest(path, {})diff --git a/nodejs/src/common/api/middleware/internal-api-auth.ts b/nodejs/src/common/api/middleware/internal-api-auth.tsindex 9d14943f7e4a..71e280506b84 100644--- a/nodejs/src/common/api/middleware/internal-api-auth.ts+++ b/nodejs/src/common/api/middleware/internal-api-auth.ts@@ -40,6 +40,7 @@ const SCOPED_AUTH_PATH_PATTERNS = [/^\/api\/projects\/[^/]+\/workflow_steps\/resume$/,// CdpApi.postTransactionalEmail, verified against WORKFLOWS_TRANSACTIONAL_EMAIL_JWT_SECRET, audience-pinned./^\/api\/projects\/[^/]+\/hog_flows\/[^/]+\/transactional_email$/,+ /^\/api\/projects\/[^/]+\/hog_flows\/[^/]+\/transactional_email\/send$/,]export interface InternalApiAuthOptions {