Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(cdp): deliver a code send's content on its own transactional email route (#127122)

Michael Reichenbacha69f8ef98b9a
diff --git a/nodejs/src/cdp/_tests/fixtures-hogflows.ts b/nodejs/src/cdp/_tests/fixtures-hogflows.ts
index fc7d6858397b..3d48cc3d3ec9 100644
--- a/nodejs/src/cdp/_tests/fixtures-hogflows.ts
+++ b/nodejs/src/cdp/_tests/fixtures-hogflows.ts
@@ -1,5 +1,5 @@
import { HogFlow } from '~/cdp/schema/hogflow'
-import { PostgresRouter } from '~/common/utils/db/postgres'
+import { PostgresRouter, PostgresUse } from '~/common/utils/db/postgres'
import { UUIDT } from '~/common/utils/utils'
import { insertRow } from '~/tests/helpers/sql'
@@ -20,6 +20,20 @@ export const insertHogFlow = async (postgres: PostgresRouter, hogFlow: HogFlow):
return res
}
+export const hideHogFlowsFromReplica = (
+ postgres: PostgresRouter,
+ ids: HogFlow['id'][]
+): jest.SpyInstance<ReturnType<PostgresRouter['query']>, Parameters<PostgresRouter['query']>> => {
+ const query = postgres.query.bind(postgres)
+ return jest.spyOn(postgres, 'query').mockImplementation(async (target, queryString, values, tag, logLevel) => {
+ const result = await query(target, queryString, values, tag, logLevel)
+ if (target === PostgresUse.COMMON_READ && tag === 'fetchHogFlows') {
+ return { ...result, rows: result.rows.filter((row) => !ids.includes(row.id)) }
+ }
+ return result
+ })
+}
+
export const createHogFlowInvocationContext = (
data: Partial<HogFlowInvocationContext> = {}
): HogFlowInvocationContext => {
diff --git a/nodejs/src/cdp/async-functions/send-email.ts b/nodejs/src/cdp/async-functions/send-email.ts
index df495c5b3aa8..7b52a3be7edc 100644
--- a/nodejs/src/cdp/async-functions/send-email.ts
+++ b/nodejs/src/cdp/async-functions/send-email.ts
@@ -1,13 +1,15 @@
import { DateTime } from 'luxon'
import { CyclotronInvocationQueueParametersEmailSchema } from '~/cdp/schema/cyclotron'
+import { acceptedSendEmailFields } from '~/cdp/services/hogflows/transactional-email'
import { registerAsyncFunction } from '../async-function-registry'
registerAsyncFunction('sendEmail', {
- execute: (args, _context, result) => {
+ execute: (args, context, result) => {
result.invocation.queueParameters = CyclotronInvocationQueueParametersEmailSchema.parse({
...args[0],
+ ...acceptedSendEmailFields(context.invocation.state.send),
type: 'email',
})
},
diff --git a/nodejs/src/cdp/cdp-api.serial.test.ts b/nodejs/src/cdp/cdp-api.serial.test.ts
index 288a88ffb517..394cfad1b36e 100644
--- a/nodejs/src/cdp/cdp-api.serial.test.ts
+++ b/nodejs/src/cdp/cdp-api.serial.test.ts
@@ -32,7 +32,7 @@ import {
insertHogFunctionTemplate,
insertIntegration,
} from './_tests/fixtures'
-import { insertHogFlow as _insertHogFlow } from './_tests/fixtures-hogflows'
+import { insertHogFlow as _insertHogFlow, hideHogFlowsFromReplica } from './_tests/fixtures-hogflows'
import { CdpApi } from './cdp-api'
import { CdpConsumerBaseDeps } from './consumers/cdp-base.consumer'
import { posthogFilterOutPlugin } from './legacy-plugins/_transformations/posthog-filter-out-plugin/template'
@@ -2145,8 +2145,11 @@ describe('CDP API', () => {
{
secret = 'local-dev-workflows-transactional-email-jwt',
audience = 'posthog:workflows:send_transactional_email',
- } = {}
- ) => jwt.sign({ team_id: teamId, hog_flow_id: hogFlowId }, secret, { audience, expiresIn: '2m' })
+ claims = {},
+ }: { secret?: string; audience?: string; claims?: Record<string, string> } = {}
+ ) => jwt.sign({ team_id: teamId, hog_flow_id: hogFlowId, ...claims }, secret, { audience, expiresIn: '2m' })
+ const tokenFor = (route: string, teamId: number, hogFlowId: string) =>
+ mintToken(teamId, hogFlowId, route.endsWith('/send') ? { claims: { kind: 'code_send' } } : {})
const transactionalFlow = (
trigger: 'api' | 'schedule',
@@ -2197,6 +2200,26 @@ describe('CDP API', () => {
.filter(([output]) => output === 'email_status')
.map(([, message]) => parseJSON(message.value.toString()))
+ const sendAccepted = (flowId: string, payload: Record<string, unknown>) =>
+ supertest(app)
+ .post(`/api/projects/${team.id}/hog_flows/${flowId}/transactional_email/send`)
+ .set({ Authorization: `Bearer ${tokenFor('transactional_email/send', team.id, flowId)}` })
+ .send(payload)
+
+ const sendOn = (
+ route: string,
+ flowId: string,
+ headers: Record<string, string>,
+ payload: Record<string, unknown>
+ ) => supertest(app).post(`/api/projects/${team.id}/hog_flows/${flowId}/${route}`).set(headers).send(payload)
+
+ const codeSend = {
+ content: { subject: 'Reset your password', html: '<p>Hi {{ variables.name }}</p>', text: 'Hi' },
+ from: { integrationId: 12, email: '[email protected]', name: 'Example' },
+ replyTo: ['[email protected]'],
+ tracking: { captureEngagement: true },
+ }
+
beforeEach(async () => {
produceOutput = jest.spyOn(api['emailStatusService']['outputs'], 'produce')
mockQueueInvocations = jest.fn().mockResolvedValue(undefined)
@@ -2284,39 +2307,69 @@ describe('CDP API', () => {
let keyed: Record<string, unknown>
const sendKeyed = (payload: Record<string, unknown> = keyed) =>
send(hogFlow.id, { Authorization: `Bearer ${mintToken(team.id, hogFlow.id)}` }, payload)
+ const sendKeyedOn = (route: 'transactional_email' | 'transactional_email/send', payload: object) =>
+ supertest(app)
+ .post(`/api/projects/${team.id}/hog_flows/${hogFlow.id}/${route}`)
+ .set({ Authorization: `Bearer ${tokenFor(route, team.id, hogFlow.id)}` })
+ .send({ ...payload, idempotency_key: keyed.idempotency_key })
+ const changedContent = { ...codeSend, content: { ...codeSend.content, subject: 'Your new code' } }
beforeEach(() => {
- // A fresh key per test, so no test reads a key another one left in Valkey.
keyed = { ...body, idempotency_key: `key-${new UUIDT().toString()}` }
})
- it('returns the first send for a retry and queues nothing new', async () => {
- const first = await sendKeyed()
- const retry = await sendKeyed()
-
- expect(first.status).toEqual(202)
- expect(first.body).toMatchObject({ replayed: false })
- expect(retry.status).toEqual(202)
- expect(retry.body).toEqual({
- status: 'queued',
- invocation_id: first.body.invocation_id,
- replayed: true,
- })
- expect(mockQueueInvocations).toHaveBeenCalledTimes(1)
- expect(emailStatusRows()).toEqual([
- expect.objectContaining({ message_id: first.body.invocation_id, status: 'queued' }),
- ])
- })
+ it.each([
+ ['a template send', 'transactional_email', body],
+ ['a code send', 'transactional_email/send', { ...body, send: codeSend }],
+ ] as const)(
+ 'returns the first send for a retry of %s and queues nothing new',
+ async (_desc, route, payload) => {
+ const first = await sendKeyedOn(route, payload)
+ const retry = await sendKeyedOn(route, payload)
+
+ expect(first.status).toEqual(202)
+ expect(first.body).toMatchObject({ replayed: false })
+ expect(retry.status).toEqual(202)
+ expect(retry.body).toEqual({
+ status: 'queued',
+ invocation_id: first.body.invocation_id,
+ replayed: true,
+ })
+ expect(mockQueueInvocations).toHaveBeenCalledTimes(1)
+ expect(emailStatusRows()).toEqual([
+ expect.objectContaining({ message_id: first.body.invocation_id, status: 'queued' }),
+ ])
+ }
+ )
- it('rejects the key for a different request', async () => {
- await sendKeyed()
+ it.each([
+ [
+ 'another recipient',
+ ['transactional_email', body],
+ ['transactional_email', { ...body, to: '[email protected]' }],
+ ],
+ [
+ 'changed content',
+ ['transactional_email/send', { ...body, send: codeSend }],
+ ['transactional_email/send', { ...body, send: changedContent }],
+ ],
+ [
+ 'the other route',
+ ['transactional_email', body],
+ ['transactional_email/send', { ...body, send: codeSend }],
+ ],
+ ] as const)(
+ 'rejects the key for a request with %s',
+ async (_desc, [firstRoute, first], [route, payload]) => {
+ await sendKeyedOn(firstRoute, first)
- const res = await sendKeyed({ ...keyed, to: '[email protected]' })
+ const res = await sendKeyedOn(route, payload)
- expect(res.status).toEqual(422)
- expect(res.body).toEqual({ error: 'idempotency_key_reused' })
- expect(mockQueueInvocations).toHaveBeenCalledTimes(1)
- })
+ expect(res.status).toEqual(422)
+ expect(res.body).toEqual({ error: 'idempotency_key_reused' })
+ expect(mockQueueInvocations).toHaveBeenCalledTimes(1)
+ }
+ )
it('tells a retry to wait while the first request is still queueing', async () => {
let finishQueueing: () => void = () => {}
@@ -2361,6 +2414,71 @@ describe('CDP API', () => {
})
})
+ it.each([
+ ['a code send with rendered content', codeSend],
+ [
+ 'a code send with html that has no text to derive',
+ { ...codeSend, content: { subject: 'Hi', html: '<img src="https://example.com/a.png" alt="Hi">' } },
+ ],
+ ['a code send for a local template', { tracking: { captureEngagement: false } }],
+ ])('stamps %s on the queued run', async (_desc, acceptedSend) => {
+ const res = await sendAccepted(hogFlow.id, { ...body, send: acceptedSend })
+
+ expect(res.status).toEqual(202)
+ const [invocation] = mockQueueInvocations.mock.calls[0][0]
+ expect(invocation.state.send).toEqual(acceptedSend)
+ })
+
+ it('queues a template send without an accepted send, even when the request carries one', async () => {
+ const res = await send(
+ hogFlow.id,
+ { Authorization: `Bearer ${mintToken(team.id, hogFlow.id)}` },
+ { ...body, send: codeSend }
+ )
+
+ expect(res.status).toEqual(202)
+ const [invocation] = mockQueueInvocations.mock.calls[0][0]
+ expect(invocation.state.send).toBeUndefined()
+ })
+
+ it('refuses a send on the accepted send route without the send', async () => {
+ const invalidBefore = await outcomeCount('invalid_request')
+
+ const res = await sendAccepted(hogFlow.id, body)
+
+ expect(res.status).toEqual(400)
+ expect(await outcomeCount('invalid_request')).toEqual(invalidBefore + 1)
+ expect(mockQueueInvocations).not.toHaveBeenCalled()
+ })
+
+ it.each([
+ ['a field this version does not know', { ...codeSend, priority: 'high' }],
+ [
+ 'a content field this version does not know',
+ { ...codeSend, content: { ...codeSend.content, amp: '<p/>' } },
+ ],
+ [
+ 'a sender field this version does not know',
+ { ...codeSend, from: { ...codeSend.from, replyName: 'Ops' } },
+ ],
+ [
+ 'a tracking field this version does not know',
+ { ...codeSend, tracking: { captureEngagement: true, pixel: false } },
+ ],
+ ['content without a body', { ...codeSend, content: { subject: 'Hi', html: '', text: '' } }],
+ ['content with an empty subject', { ...codeSend, content: { subject: '', text: 'Hi' } }],
+ ['a sender without an integration', { ...codeSend, from: { email: '[email protected]' } }],
+ ['no tracking choice', { content: codeSend.content }],
+ ])('refuses a code send with %s', async (_desc, acceptedSend) => {
+ const invalidBefore = await outcomeCount('invalid_request')
+
+ const res = await sendAccepted(hogFlow.id, { ...body, send: acceptedSend })
+
+ expect(res.status).toEqual(400)
+ expect(await outcomeCount('invalid_request')).toEqual(invalidBefore + 1)
+ expect(mockQueueInvocations).not.toHaveBeenCalled()
+ })
+
it('queues a send when the team is over its email quota', async () => {
jest.spyOn(api['deps'].quotaLimiting, 'isTeamQuotaLimited').mockResolvedValue(true)
const queuedBefore = await outcomeCount('queued')
@@ -2386,69 +2504,110 @@ describe('CDP API', () => {
expect(mockQueueInvocations).not.toHaveBeenCalled()
})
- it.each([
- [
- 'email sending is suspended for the project',
- 'sending_suspended',
- () =>
- jest
- .spyOn(api['teamWorkflowsConfigService'], 'getEmailSendingSuspension')
- .mockResolvedValueOnce('staff'),
- ],
- [
- "the transactional email's sending is paused",
- 'sending_paused',
- () =>
- jest
- .spyOn(api['hogFlowManager'], 'getHogFlow')
- .mockResolvedValueOnce({ ...hogFlow, email_sending_paused_at: Date.now() }),
- ],
- [
- 'the recipient is suppressed',
- 'recipient_suppressed',
- () => jest.spyOn(api['emailSuppressionService'], 'isSuppressed').mockResolvedValueOnce(true),
- ],
- [
- "the recipient's domain has no mail servers",
- 'recipient_unreachable',
- () =>
- jest
- .spyOn(api['emailValidationService'], 'getRecipientSkipReason')
- .mockResolvedValueOnce('Skipping send: the domain has no reachable mail servers.'),
- ],
- ] as const)('refuses a send the email worker would skip because %s', async (_desc, code, refuse) => {
- refuse()
- const refusedBefore = await outcomeCount(code)
- const stagesBefore = await stageCounts()
- const keyed = { ...body, idempotency_key: `key-${new UUIDT().toString()}` }
- const headers = { Authorization: `Bearer ${mintToken(team.id, hogFlow.id)}` }
+ describe.each([
+ ['a template send', 'transactional_email', {}],
+ ['a code send', 'transactional_email/send', { send: codeSend }],
+ ])('for %s', (_kind, route, extra) => {
+ it.each([
+ [
+ 'email sending is suspended for the project',
+ 'sending_suspended',
+ () =>
+ jest
+ .spyOn(api['teamWorkflowsConfigService'], 'getEmailSendingSuspension')
+ .mockResolvedValueOnce('staff'),
+ ],
+ [
+ "the transactional email's sending is paused",
+ 'sending_paused',
+ () =>
+ jest
+ .spyOn(api['hogFlowManager'], 'getHogFlowCheckingPrimaryOnMiss')
+ .mockResolvedValueOnce({ ...hogFlow, email_sending_paused_at: Date.now() }),
+ ],
+ [
+ 'the recipient is suppressed',
+ 'recipient_suppressed',
+ () => jest.spyOn(api['emailSuppressionService'], 'isSuppressed').mockResolvedValueOnce(true),
+ ],
+ [
+ "the recipient's domain has no mail servers",
+ 'recipient_unreachable',
+ () =>
+ jest
+ .spyOn(api['emailValidationService'], 'getRecipientSkipReason')
+ .mockResolvedValueOnce('Skipping send: the domain has no reachable mail servers.'),
+ ],
+ ] as const)('refuses a send the email worker would skip because %s', async (_desc, code, refuse) => {
+ refuse()
+ const refusedBefore = await outcomeCount(code)
+ const stagesBefore = await stageCounts()
+ const keyed = { ...body, ...extra, idempotency_key: `key-${new UUIDT().toString()}` }
+ const headers = { Authorization: `Bearer ${tokenFor(route, team.id, hogFlow.id)}` }
+
+ const refused = await sendOn(route, hogFlow.id, headers, keyed)
+
+ expect(refused.status).toEqual(422)
+ expect(refused.body).toEqual({ error: code })
+ expect(await outcomeCount(code)).toEqual(refusedBefore + 1)
+ // The reserve and the release of the key count once, as one idempotency observation.
+ expect(stageCountChanges(stagesBefore, await stageCounts())).toEqual({
+ auth: 1,
+ validate: 1,
+ load: 1,
+ build: 1,
+ idempotency: 1,
+ checks: 1,
+ total: 1,
+ })
+ expect(mockQueueInvocations).not.toHaveBeenCalled()
+ expect(emailStatusRows()).toEqual([])
- const refused = await send(hogFlow.id, headers, keyed)
+ const retry = await sendOn(route, hogFlow.id, headers, keyed)
- expect(refused.status).toEqual(422)
- expect(refused.body).toEqual({ error: code })
- expect(await outcomeCount(code)).toEqual(refusedBefore + 1)
- // The reserve and the release of the key count once, as one idempotency observation.
- expect(stageCountChanges(stagesBefore, await stageCounts())).toEqual({
- auth: 1,
- validate: 1,
- load: 1,
- build: 1,
- idempotency: 1,
- checks: 1,
- total: 1,
+ expect(retry.status).toEqual(202)
+ expect(mockQueueInvocations).toHaveBeenCalledTimes(1)
})
- expect(mockQueueInvocations).not.toHaveBeenCalled()
- expect(emailStatusRows()).toEqual([])
- const retry = await send(hogFlow.id, headers, keyed)
+ it('replays a keyed send whose workflow stopped being active since it went out', async () => {
+ const keyed = { ...body, ...extra, idempotency_key: `key-${new UUIDT().toString()}` }
+ const headers = { Authorization: `Bearer ${tokenFor(route, team.id, hogFlow.id)}` }
+ const first = await sendOn(route, hogFlow.id, headers, keyed)
+ await hub.postgres.query(
+ PostgresUse.COMMON_WRITE,
+ `UPDATE posthog_hogflow SET status = 'draft' WHERE id = $1`,
+ [hogFlow.id],
+ 'deactivate-workflow'
+ )
+ api['hogFlowManager']['onHogFlowsReloaded'](team.id, [hogFlow.id])
- expect(retry.status).toEqual(202)
- expect(mockQueueInvocations).toHaveBeenCalledTimes(1)
+ const retry = await sendOn(route, hogFlow.id, headers, keyed)
+
+ expect(retry.status).toEqual(202)
+ expect(retry.body).toEqual({ ...first.body, replayed: true })
+ expect(mockQueueInvocations).toHaveBeenCalledTimes(1)
+ })
})
it.each([
['a valid address', { to: '[email protected]' }, 202, 'queued'],
+ ...[
+ 'a!b$c%d&e*f/g?h^i`j|[email protected]',
+ ].map((to) => [to, { to }, 202, 'queued'] as const),
+ ...[
+ 'jane@bücher.example',
+ 'jane@[192.0.2.1]',
+ '"jane doe"@example.com',
+ 'jane@localhost',
+ ].map((to) => [to, { to }, 422, 'recipient_unreachable'] as const),
['a domain with a fullwidth dot', { to: 'user@ph\uFF0Eexample.com' }, 422, 'recipient_unreachable'],
['an ideographic full stop', { to: 'user@ph\u3002example.com' }, 422, 'recipient_unreachable'],
['a zero-width space', { to: 'user\[email protected]' }, 422, 'recipient_unreachable'],
@@ -2513,6 +2672,65 @@ describe('CDP API', () => {
expect(mockQueueInvocations).not.toHaveBeenCalled()
})
+ it.each([
+ ['transactional_email', body],
+ ['transactional_email/send', { ...body, send: codeSend }],
+ ])(
+ 'sends on %s under a workflow the replica does not have yet, reading the primary once',
+ async (route, payload) => {
+ const unseen = await _insertHogFlow(hub.postgres, transactionalFlow('api', 'active'))
+ const query = hideHogFlowsFromReplica(hub.postgres, [unseen.id])
+ const post = () =>
+ supertest(app)
+ .post(`/api/projects/${team.id}/hog_flows/${unseen.id}/${route}`)
+ .set({ Authorization: `Bearer ${tokenFor(route, team.id, unseen.id)}` })
+ .send(payload)
+
+ expect((await post()).status).toEqual(202)
+ expect((await post()).status).toEqual(202)
+
+ expect(mockQueueInvocations).toHaveBeenCalledTimes(2)
+ expect(
+ query.mock.calls.filter(
+ ([use, , , tag]) => use === PostgresUse.COMMON_WRITE && tag === 'fetchHogFlows'
+ )
+ ).toHaveLength(1)
+ }
+ )
+
+ it.each(['transactional_email', 'transactional_email/send'])(
+ 'authenticates every send on %s',
+ async (route) => {
+ const post = (headers: Record<string, string>) =>
+ supertest(app)
+ .post(`/api/projects/${team.id}/hog_flows/${hogFlow.id}/${route}`)
+ .set(headers)
+ .send({ ...body, send: codeSend })
+
+ expect((await post({})).status).toEqual(401)
+ expect(
+ (await post({ Authorization: `Bearer ${tokenFor(route, team.id + 1, hogFlow.id)}` })).status
+ ).toEqual(401)
+ expect(mockQueueInvocations).not.toHaveBeenCalled()
+ }
+ )
+
+ it.each([
+ ['a template send token on the code send route', 'transactional_email/send', 'transactional_email'],
+ ['a code send token on the template send route', 'transactional_email', 'transactional_email/send'],
+ ])('refuses %s', async (_desc, route, tokenRoute) => {
+ const unauthorizedBefore = await outcomeCount('unauthorized')
+
+ const res = await supertest(app)
+ .post(`/api/projects/${team.id}/hog_flows/${hogFlow.id}/${route}`)
+ .set({ Authorization: `Bearer ${tokenFor(tokenRoute, team.id, hogFlow.id)}` })
+ .send({ ...body, send: codeSend })
+
+ expect(res.status).toEqual(401)
+ expect(await outcomeCount('unauthorized')).toEqual(unauthorizedBefore + 1)
+ expect(mockQueueInvocations).not.toHaveBeenCalled()
+ })
+
it.each([
['no token', () => ({})],
[
diff --git a/nodejs/src/cdp/cdp-api.ts b/nodejs/src/cdp/cdp-api.ts
index ccf4d277cf6d..03ea68d80d37 100644
--- a/nodejs/src/cdp/cdp-api.ts
+++ b/nodejs/src/cdp/cdp-api.ts
@@ -54,7 +54,11 @@ import { matchesWaitUntilCondition } from './services/hogflows/hogflow-utils'
import { IdempotencyKeys } from './services/hogflows/idempotency-keys'
import { WorkflowStepResumeSchema } from './services/hogflows/step-resume.service'
import {
+ AcceptedSendRequestSchema,
+ CODE_SEND_TOKEN_CLAIMS,
+ TEMPLATE_SEND_TOKEN_CLAIMS,
TransactionalEmailOutcome,
+ TransactionalEmailRequest,
TransactionalEmailRequestSchema,
TransactionalEmailStageTimer,
transactionalEmailRequestsTotal,
@@ -167,6 +171,18 @@ function parseRealtimeCohort(value: unknown): BatchResolverState['realtimeCohort
return { id: id as number, teamId: team_id as number, waitUntil: waitUntil.toISOString() }
}
+const ACCEPTED_SEND_FINGERPRINT_MARKER = 'accepted_send'
+
+const transactionalEmailFingerprint = (
+ hogFlowId: string,
+ { to, distinct_id, variables, send }: TransactionalEmailRequest
+): string => {
+ const templateRequest = [hogFlowId, to, distinct_id, variables]
+ return IdempotencyKeys.fingerprint(
+ send ? [...templateRequest, ACCEPTED_SEND_FINGERPRINT_MARKER, send] : templateRequest
+ )
+}
+
export class CdpApi {
private hogExecutorAsync: HogExecutorAsyncService
private hogInputsService: HogInputsService
@@ -382,7 +398,11 @@ export class CdpApi {
router.post('/api/projects/:team_id/workflow_steps/resume', asyncHandler(this.postWorkflowStepResume))
router.post(
'/api/projects/:team_id/hog_flows/:id/transactional_email',
- asyncHandler(this.postTransactionalEmail)
+ asyncHandler(this.postTransactionalEmail(TransactionalEmailRequestSchema, TEMPLATE_SEND_TOKEN_CLAIMS))
+ )
+ router.post(
+ '/api/projects/:team_id/hog_flows/:id/transactional_email/send',
+ asyncHandler(this.postTransactionalEmail(AcceptedSendRequestSchema, CODE_SEND_TOKEN_CLAIMS))
)
router.get('/api/projects/:team_id/hog_functions/:id/status', asyncHandler(this.getFunctionStatus()))
router.patch('/api/projects/:team_id/hog_functions/:id/status', asyncHandler(this.patchFunctionStatus()))
@@ -1158,161 +1178,185 @@ export class CdpApi {
//
// Auth: a per-call JWT minted by Django on its own audience and key, pinned to the team and the workflow,
// never the fleet-wide internal secret.
- private postTransactionalEmail = async (req: ModifiedRequest, res: express.Response): Promise<any> => {
- const stages = new TransactionalEmailStageTimer()
- const reply = (outcome: TransactionalEmailOutcome, status: number, body: Record<string, unknown>) => {
- transactionalEmailRequestsTotal.inc({ outcome })
- return res.status(status).json(body)
- }
- try {
- stages.start('auth')
- if (!this.transactionalEmailJwt.enabled) {
- return reply('not_configured', 503, {
- error: 'Transactional email auth not configured (WORKFLOWS_TRANSACTIONAL_EMAIL_JWT_SECRET unset)',
- })
- }
- if (!this.verifyScopedWorkflowJwt(this.transactionalEmailJwt, req, res, 'transactional email')) {
- transactionalEmailRequestsTotal.inc({ outcome: 'unauthorized' })
- return
+ private postTransactionalEmail =
+ (schema: z.ZodType<TransactionalEmailRequest, unknown>, tokenClaims: Record<string, string | undefined>) =>
+ async (req: ModifiedRequest, res: express.Response): Promise<any> => {
+ const stages = new TransactionalEmailStageTimer()
+ const reply = (outcome: TransactionalEmailOutcome, status: number, body: Record<string, unknown>) => {
+ transactionalEmailRequestsTotal.inc({ outcome })
+ return res.status(status).json(body)
}
- stages.start('validate')
- const parsed = TransactionalEmailRequestSchema.safeParse(req.body)
- if (!parsed.success) {
- const issues = parsed.error.issues
- // Signup accepts some addresses this check refuses, and no retry can fix an address.
- if (issues.every((issue) => issue.path[0] === 'to' && issue.code === 'invalid_format')) {
- return reply('recipient_unreachable', 422, { error: 'recipient_unreachable' })
+ try {
+ stages.start('auth')
+ if (!this.transactionalEmailJwt.enabled) {
+ return reply('not_configured', 503, {
+ error: 'Transactional email auth not configured (WORKFLOWS_TRANSACTIONAL_EMAIL_JWT_SECRET unset)',
+ })
}
- // Paths and codes only: the values include the recipient's address.
- logger.warn('⚠️', 'Transactional email request failed validation', {
- team_id: req.params.team_id,
- hog_flow_id: req.params.id,
- issues: issues.map((issue) => ({ path: issue.path.join('.'), code: issue.code })),
- })
- return reply('invalid_request', 400, { error: parsed.error.message })
- }
- const { to, distinct_id: distinctId, variables, idempotency_key: idempotencyKey } = parsed.data
+ if (
+ !this.verifyScopedWorkflowJwt(
+ this.transactionalEmailJwt,
+ req,
+ res,
+ 'transactional email',
+ tokenClaims
+ )
+ ) {
+ transactionalEmailRequestsTotal.inc({ outcome: 'unauthorized' })
+ return
+ }
+ stages.start('validate')
+ const parsed = schema.safeParse(req.body)
+ if (!parsed.success) {
+ const issues = parsed.error.issues
+ // Signup accepts some addresses this check refuses, and no retry can fix an address.
+ if (issues.every((issue) => issue.path[0] === 'to' && issue.code === 'invalid_format')) {
+ return reply('recipient_unreachable', 422, { error: 'recipient_unreachable' })
+ }
+ // Paths and codes only: the values include the recipient's address.
+ logger.warn('⚠️', 'Transactional email request failed validation', {
+ team_id: req.params.team_id,
+ hog_flow_id: req.params.id,
+ issues: issues.map((issue) => ({ path: issue.path.join('.'), code: issue.code })),
+ })
+ return reply('invalid_request', 400, { error: parsed.error.message })
+ }
+ const { to, distinct_id: distinctId, variables, idempotency_key: idempotencyKey, send } = parsed.data
- stages.start('load')
- const team = await this.deps.teamManager.getTeam(parseInt(req.params.team_id)).catch(() => null)
- if (!team) {
- return reply('not_found', 404, { error: 'Team not found' })
- }
- const hogFlow = await this.hogFlowManager.getHogFlow(req.params.id)
- if (!hogFlow || hogFlow.team_id !== team.id) {
- return reply('not_found', 404, { error: 'Workflow not found' })
- }
- if (hogFlow.trigger?.type !== 'api' || hogFlow.status !== 'active') {
- return reply('not_sendable', 400, { error: 'not_sendable' })
- }
+ stages.start('load')
+ const team = await this.deps.teamManager.getTeam(parseInt(req.params.team_id)).catch(() => null)
+ if (!team) {
+ return reply('not_found', 404, { error: 'Team not found' })
+ }
+ const hogFlow = await this.hogFlowManager.getHogFlowCheckingPrimaryOnMiss(req.params.id)
+ if (!hogFlow || hogFlow.team_id !== team.id) {
+ return reply('not_found', 404, { error: 'Workflow not found' })
+ }
+ if (hogFlow.trigger?.type !== 'api') {
+ return reply('not_sendable', 400, { error: 'not_sendable' })
+ }
- stages.start('build')
- const event: HogFunctionInvocationGlobals['event'] = {
- uuid: new UUIDT().toString(),
- event: TRANSACTIONAL_EMAIL_EVENT,
- distinct_id: distinctId,
- timestamp: DateTime.now().toISO(),
- url: '',
- // The email step's recipient reads this.
- properties: { to },
- elements_chain: '',
- }
- const triggerGlobals: HogFunctionInvocationGlobals = {
- event,
- project: {
- id: team.id,
- name: team.name,
- url: `${this.config.SITE_URL ?? 'http://localhost:8000'}/project/${team.id}`,
- },
- variables,
- }
- const filterGlobals = convertToHogFunctionFilterGlobal({ event, person: undefined, groups: {}, variables })
+ stages.start('build')
+ const event: HogFunctionInvocationGlobals['event'] = {
+ uuid: new UUIDT().toString(),
+ event: TRANSACTIONAL_EMAIL_EVENT,
+ distinct_id: distinctId,
+ timestamp: DateTime.now().toISO(),
+ url: '',
+ // The email step's recipient reads this.
+ properties: { to },
+ elements_chain: '',
+ }
+ const triggerGlobals: HogFunctionInvocationGlobals = {
+ event,
+ project: {
+ id: team.id,
+ name: team.name,
+ url: `${this.config.SITE_URL ?? 'http://localhost:8000'}/project/${team.id}`,
+ },
+ variables,
+ }
+ const filterGlobals = convertToHogFunctionFilterGlobal({
+ event,
+ person: undefined,
+ groups: {},
+ variables,
+ })
- const invocation = createHogFlowInvocation(triggerGlobals, hogFlow, filterGlobals)
- // The run skips the trigger pipeline that counts runs for event-triggered workflows, so the email
- // worker counts it instead.
- invocation.state.triggerMetricPending = true
- // Straight to the email queue: the flow's only step is the email, so a hop through the
- // workflow queue would only add latency to a password reset or a verification code.
- const priorityClass: EmailQueuePriorityClass = 'fast'
- invocation.queue = 'email'
- invocation.queuePriority = EMAIL_QUEUE_PRIORITY[priorityClass]
-
- const idempotencyKeys = this.transactionalEmailIdempotencyKeys
- const fingerprint = IdempotencyKeys.fingerprint([hogFlow.id, to, distinctId, variables])
- if (idempotencyKey) {
- // Checked before the quota, so a retry of a send that went out gets its answer even
- // when the team has hit its quota since.
- stages.start('idempotency')
- const reservation = await idempotencyKeys.reserve(team.id, idempotencyKey, fingerprint, invocation.id)
- switch (reservation.status) {
- case 'replayed':
- return reply('replayed', 202, {
- status: 'queued',
- invocation_id: reservation.invocationId,
- replayed: true,
- })
- case 'in_flight':
- return reply('idempotency_key_in_flight', 409, { error: 'idempotency_key_in_flight' })
- case 'conflict':
- return reply('idempotency_key_reused', 422, { error: 'idempotency_key_reused' })
- case 'unavailable':
- return reply('idempotency_unavailable', 503, { error: 'idempotency_unavailable' })
+ const invocation = createHogFlowInvocation(triggerGlobals, hogFlow, filterGlobals)
+ if (send) {
+ invocation.state.send = send
}
- }
- try {
- stages.start('checks')
- const refusal = await this.refuseTransactionalEmail(team.id, hogFlow, to)
- if (refusal) {
- if (idempotencyKey) {
- stages.start('idempotency')
- await idempotencyKeys.release(team.id, idempotencyKey, invocation.id)
+ // The run skips the trigger pipeline that counts runs for event-triggered workflows, so the email
+ // worker counts it instead.
+ invocation.state.triggerMetricPending = true
+ // Straight to the email queue: the flow's only step is the email, so a hop through the
+ // workflow queue would only add latency to a password reset or a verification code.
+ const priorityClass: EmailQueuePriorityClass = 'fast'
+ invocation.queue = 'email'
+ invocation.queuePriority = EMAIL_QUEUE_PRIORITY[priorityClass]
+
+ const idempotencyKeys = this.transactionalEmailIdempotencyKeys
+ const fingerprint = transactionalEmailFingerprint(hogFlow.id, parsed.data)
+ if (idempotencyKey) {
+ // Checked before the quota, so a retry of a send that went out gets its answer even
+ // when the team has hit its quota since.
+ stages.start('idempotency')
+ const reservation = await idempotencyKeys.reserve(
+ team.id,
+ idempotencyKey,
+ fingerprint,
+ invocation.id
+ )
+ switch (reservation.status) {
+ case 'replayed':
+ return reply('replayed', 202, {
+ status: 'queued',
+ invocation_id: reservation.invocationId,
+ replayed: true,
+ })
+ case 'in_flight':
+ return reply('idempotency_key_in_flight', 409, { error: 'idempotency_key_in_flight' })
+ case 'conflict':
+ return reply('idempotency_key_reused', 422, { error: 'idempotency_key_reused' })
+ case 'unavailable':
+ return reply('idempotency_unavailable', 503, { error: 'idempotency_unavailable' })
}
- return reply(refusal.outcome, refusal.status, { error: refusal.outcome })
}
- stages.start('queue_insert')
- this.invocationResultsService.invocationResultsRowsService.queueLifecycleRow(invocation, 'running')
try {
- await this.hogflowQueue.queueInvocations([invocation])
+ stages.start('checks')
+ const refusal = await this.refuseTransactionalEmail(team.id, hogFlow, to)
+ if (refusal) {
+ if (idempotencyKey) {
+ stages.start('idempotency')
+ await idempotencyKeys.release(team.id, idempotencyKey, invocation.id)
+ }
+ return reply(refusal.outcome, refusal.status, { error: refusal.outcome })
+ }
+ stages.start('queue_insert')
+ this.invocationResultsService.invocationResultsRowsService.queueLifecycleRow(invocation, 'running')
+ try {
+ await this.hogflowQueue.queueInvocations([invocation])
+ } catch (error) {
+ this.invocationResultsService.invocationResultsRowsService.dropQueuedRowsFor([invocation.id])
+ throw error
+ }
+ cdpEmailQueuedTotal.labels(priorityClass).inc()
} catch (error) {
- this.invocationResultsService.invocationResultsRowsService.dropQueuedRowsFor([invocation.id])
+ // Nothing was queued, so the caller's retry with the same key must be able to send.
+ if (idempotencyKey) {
+ stages.start('idempotency')
+ await idempotencyKeys.release(team.id, idempotencyKey, invocation.id)
+ }
throw error
}
- cdpEmailQueuedTotal.labels(priorityClass).inc()
- } catch (error) {
- // Nothing was queued, so the caller's retry with the same key must be able to send.
if (idempotencyKey) {
stages.start('idempotency')
- await idempotencyKeys.release(team.id, idempotencyKey, invocation.id)
+ await idempotencyKeys.markQueued(team.id, idempotencyKey, fingerprint, invocation.id)
}
- throw error
- }
- if (idempotencyKey) {
- stages.start('idempotency')
- await idempotencyKeys.markQueued(team.id, idempotencyKey, fingerprint, invocation.id)
- }
- stages.start('metrics_flush')
- this.emailStatusService.queue({ teamId: team.id, messageId: invocation.id, status: 'queued' })
- // Best-effort rows, so the 202 waits for the hand-off to Kafka (flushed on shutdown) but not the acks.
- const warnTelemetryFlushFailed = (error: unknown) =>
- logger.warn('⚠️', 'Transactional email telemetry flush failed', {
- error: error instanceof Error ? error.message : String(error),
- invocation_id: invocation.id,
- })
- void this.emailStatusService.flush().catch(warnTelemetryFlushFailed)
- const { produced } = await this.invocationResultsService.invocationResultsRowsService.handOffQueuedRows()
- void produced.catch(warnTelemetryFlushFailed)
+ stages.start('metrics_flush')
+ this.emailStatusService.queue({ teamId: team.id, messageId: invocation.id, status: 'queued' })
+ // Best-effort rows, so the 202 waits for the hand-off to Kafka (flushed on shutdown) but not the acks.
+ const warnTelemetryFlushFailed = (error: unknown) =>
+ logger.warn('⚠️', 'Transactional email telemetry flush failed', {
+ error: error instanceof Error ? error.message : String(error),
+ invocation_id: invocation.id,
+ })
+ void this.emailStatusService.flush().catch(warnTelemetryFlushFailed)
+ const { produced } =
+ await this.invocationResultsService.invocationResultsRowsService.handOffQueuedRows()
+ void produced.catch(warnTelemetryFlushFailed)
- return reply('queued', 202, { status: 'queued', invocation_id: invocation.id, replayed: false })
- } catch (e) {
- logger.error('Error queueing transactional email', {
- error: e instanceof Error ? e.message : String(e),
- })
- return reply('error', 500, { error: e instanceof Error ? e.message : String(e) })
- } finally {
- stages.finish()
+ return reply('queued', 202, { status: 'queued', invocation_id: invocation.id, replayed: false })
+ } catch (e) {
+ logger.error('Error queueing transactional email', {
+ error: e instanceof Error ? e.message : String(e),
+ })
+ return reply('error', 500, { error: e instanceof Error ? e.message : String(e) })
+ } finally {
+ stages.finish()
+ }
}
- }
// The email worker skips a send for these reasons too, but only after the caller has its 202. Checking them
// here tells the caller that the email will not go out. The worker keeps its checks, because the state can
@@ -1322,6 +1366,9 @@ export class CdpApi {
hogFlow: HogFlow,
to: string
): Promise<{ outcome: TransactionalEmailOutcome; status: number } | null> {
+ if (hogFlow.status !== 'active') {
+ return { outcome: 'not_sendable', status: 400 }
+ }
if (await this.teamWorkflowsConfigService.getEmailSendingSuspension(teamId)) {
return { outcome: 'sending_suspended', status: 422 }
}
@@ -1347,7 +1394,7 @@ export class CdpApi {
req: ModifiedRequest,
res: express.Response,
label: string,
- extraClaims?: Record<string, string>
+ extraClaims?: Record<string, string | undefined>
): boolean {
const { team_id, id } = req.params
const authHeader = req.headers['authorization']
diff --git a/nodejs/src/cdp/consumers/cdp-cyclotron-worker-hogflow.consumer.ts b/nodejs/src/cdp/consumers/cdp-cyclotron-worker-hogflow.consumer.ts
index 6e6718bbf887..f7e2639d86c7 100644
--- a/nodejs/src/cdp/consumers/cdp-cyclotron-worker-hogflow.consumer.ts
+++ b/nodejs/src/cdp/consumers/cdp-cyclotron-worker-hogflow.consumer.ts
@@ -89,6 +89,9 @@ export class CdpCyclotronWorkerHogFlow extends CdpCyclotronWorker {
const loadedInvocations: CyclotronJobInvocationHogFlow[] = []
const failedInvocations: CyclotronJobInvocation[] = []
const canceledResults: CyclotronJobInvocationResult[] = []
+ const hogFlows = await this.hogFlowManager.getHogFlowsCheckingPrimaryOnMiss(
+ invocations.filter((item) => !item.cancelRequestedAt).map((item) => item.functionId)
+ )
await Promise.all(
invocations.map(async (item) => {
@@ -103,7 +106,7 @@ export class CdpCyclotronWorkerHogFlow extends CdpCyclotronWorker {
}
const team = await this.deps.teamManager.getTeam(item.teamId)
- const hogFlow = await this.hogFlowManager.getHogFlow(item.functionId)
+ const hogFlow = hogFlows[item.functionId]
if (!hogFlow || !team) {
logger.error('⚠️', 'Error finding hog flow', {
id: item.functionId,
diff --git a/nodejs/src/cdp/consumers/cdp-cyclotron-worker-hogflow.test.ts b/nodejs/src/cdp/consumers/cdp-cyclotron-worker-hogflow.test.ts
index f1d624cc7c69..98640b5d993b 100644
--- a/nodejs/src/cdp/consumers/cdp-cyclotron-worker-hogflow.test.ts
+++ b/nodejs/src/cdp/consumers/cdp-cyclotron-worker-hogflow.test.ts
@@ -12,7 +12,7 @@ import { createTeam, createTestTeamFixture, getTeam } from '~/tests/helpers/sql'
import { Hub, InternalPerson, Team } from '../../types'
import { FixtureHogFlowBuilder } from '../_tests/builders/hogflow.builder'
-import { createHogFlowInvocationContext, insertHogFlow } from '../_tests/fixtures-hogflows'
+import { createHogFlowInvocationContext, hideHogFlowsFromReplica, insertHogFlow } from '../_tests/fixtures-hogflows'
import {
CyclotronJobInvocation,
CyclotronJobInvocationHogFlow,
@@ -88,6 +88,7 @@ describe('CdpCyclotronWorkerHogFlow', () => {
let team: Team
let team2: Team
let hogFlows: HogFlow[]
+ let jobQueue: ReturnType<typeof createMockJobQueue>
const createSerializedHogFlowInvocation = (
hogFlow: HogFlow,
@@ -160,11 +161,11 @@ describe('CdpCyclotronWorkerHogFlow', () => {
]
const mockRepo = createMockPersonReadRepository(testPersons)
- const mockJobQueue = createMockJobQueue()
+ jobQueue = createMockJobQueue()
processor = new CdpCyclotronWorkerHogFlow(
hub,
{ ...createCdpConsumerDeps(hub), personRepository: mockRepo },
- mockJobQueue
+ jobQueue
)
hogFlows = []
@@ -587,6 +588,33 @@ describe('CdpCyclotronWorkerHogFlow', () => {
})
})
+ it('runs a flow the replica does not have yet, and fails one missing from both', async () => {
+ const [lateFlow, missingFlow] = ['Flow only the primary has', 'Flow neither database has'].map((name) =>
+ new FixtureHogFlowBuilder()
+ .withName(name)
+ .withTeamId(team.id)
+ .withStatus('active')
+ .withSimpleWorkflow()
+ .build()
+ )
+ await insertHogFlow(hub.postgres, lateFlow)
+ hideHogFlowsFromReplica(hub.postgres, [lateFlow.id])
+ const [lateInvocation, missingInvocation] = [lateFlow, missingFlow].map((flow) =>
+ createSerializedHogFlowInvocation(flow, {
+ event: { distinct_id: 'distinct_A_1', properties: {} } as any,
+ })
+ )
+
+ const results = await processor.processInvocations([lateInvocation, missingInvocation])
+
+ expect(results).toHaveLength(1)
+ expect(results[0]).toMatchObject({
+ finished: true,
+ invocation: { hogFlow: { id: lateFlow.id } },
+ })
+ expect(jobQueue.dequeueInvocations).toHaveBeenLastCalledWith([missingInvocation])
+ })
+
it('terminates invocations as canceled when the workflow is disabled after being queued', async () => {
const hogFlow = hogFlows[0]
diff --git a/nodejs/src/cdp/rerun/rerun-job.manager.test.ts b/nodejs/src/cdp/rerun/rerun-job.manager.test.ts
index e11e969ed4d4..48b48a05e46d 100644
--- a/nodejs/src/cdp/rerun/rerun-job.manager.test.ts
+++ b/nodejs/src/cdp/rerun/rerun-job.manager.test.ts
@@ -76,9 +76,11 @@ describe('RerunJobManager', () => {
const jobId = await manager.enqueue(7, 'hog_function', functionId, {
filter: { ...baseFilter, invocation_ids: ids },
+ replay_code_sends: true,
})
const state = fetchState(await queryJob(jobId))
+ expect(state.request.replay_code_sends).toBe(true)
expect(state.function_kind).toBe('hog_function')
expect(state.function_id).toBe(functionId)
expect(state.request.filter.invocation_ids).toEqual(ids)
@@ -118,6 +120,7 @@ describe('RerunJobManager', () => {
expect(state.function_kind).toBe('hog_flow')
expect(state.request.filter).toMatchObject(filter)
expect(state.request.filter.invocation_ids).toBeUndefined()
+ expect(state.request.replay_code_sends).toBe(false)
expect(state.progress.cursor).toBeUndefined()
expect(state.progress.done).toBe(false)
})
diff --git a/nodejs/src/cdp/rerun/rerun-job.manager.ts b/nodejs/src/cdp/rerun/rerun-job.manager.ts
index d721e8ddc4c2..d477f3791691 100644
--- a/nodejs/src/cdp/rerun/rerun-job.manager.ts
+++ b/nodejs/src/cdp/rerun/rerun-job.manager.ts
@@ -83,7 +83,7 @@ export class RerunJobManager {
const state: RerunJobState = {
function_kind: functionKind,
function_id: functionId,
- request: { filter },
+ request: { filter, replay_code_sends: request.replay_code_sends === true },
progress: {
queued: 0,
skipped: 0,
diff --git a/nodejs/src/cdp/rerun/rerun-job.types.ts b/nodejs/src/cdp/rerun/rerun-job.types.ts
index 98e1ffc215cc..1bae853a2ff9 100644
--- a/nodejs/src/cdp/rerun/rerun-job.types.ts
+++ b/nodejs/src/cdp/rerun/rerun-job.types.ts
@@ -68,6 +68,7 @@ export interface RerunFilter {
export interface RerunRequest {
filter: RerunFilter
+ replay_code_sends?: boolean
}
/**
diff --git a/nodejs/src/cdp/rerun/rerun-paginator.replay-fidelity.test.ts b/nodejs/src/cdp/rerun/rerun-paginator.replay-fidelity.test.ts
index 8f7b6850ff5f..6b2f645ac8f5 100644
--- a/nodejs/src/cdp/rerun/rerun-paginator.replay-fidelity.test.ts
+++ b/nodejs/src/cdp/rerun/rerun-paginator.replay-fidelity.test.ts
@@ -39,7 +39,11 @@ describe('RerunPaginatorService replay fidelity (hog_flow)', () => {
function buildPaginator(
clickhouse: ClickHouseClient,
- hogFlowQueue: jest.Mocked<CyclotronJobQueuePostgresV2>
+ hogFlowQueue: jest.Mocked<CyclotronJobQueuePostgresV2>,
+ monitoring = {
+ queueLogs: jest.fn(),
+ flush: jest.fn().mockResolvedValue(undefined),
+ } as unknown as jest.Mocked<HogFunctionMonitoringService>
): RerunPaginatorService {
const hogFlowManager = {
getHogFlow: jest.fn().mockResolvedValue({ id: functionId, team_id: teamId, variables: [] }),
@@ -53,11 +57,6 @@ describe('RerunPaginatorService replay fidelity (hog_flow)', () => {
flush: jest.fn().mockResolvedValue(undefined),
dropQueuedRowsFor: jest.fn(),
} as unknown as jest.Mocked<HogInvocationResultsService>
- const monitoring = {
- queueLogs: jest.fn(),
- flush: jest.fn().mockResolvedValue(undefined),
- } as unknown as jest.Mocked<HogFunctionMonitoringService>
-
return new RerunPaginatorService(
clickhouse,
hogFunctionManager,
@@ -99,6 +98,12 @@ describe('RerunPaginatorService replay fidelity (hog_flow)', () => {
},
personId: 'person-1',
messageOptOuts: [{ identifier: '[email protected]', categoryId: '$all' }],
+ send: {
+ content: { subject: 'Welcome', html: '<p>Hi</p>', text: 'Hi' },
+ from: { integrationId: 12, email: '[email protected]', name: 'Example' },
+ replyTo: ['[email protected]'],
+ tracking: { captureEngagement: false },
+ },
}
const rows = [
{
@@ -117,7 +122,11 @@ describe('RerunPaginatorService replay fidelity (hog_flow)', () => {
} as unknown as jest.Mocked<CyclotronJobQueuePostgresV2>
const paginator = buildPaginator(fakeClickhouse(rows), hogFlowQueue)
- await paginator.processPage(teamId, state, { jobId: 'rerun-1', createdAt: DateTime.now() })
+ await paginator.processPage(
+ teamId,
+ { ...state, request: { ...state.request, replay_code_sends: true } },
+ { jobId: 'rerun-1', createdAt: DateTime.now() }
+ )
expect(hogFlowQueue.queueInvocations).toHaveBeenCalledTimes(1)
const [enqueued, opts] = hogFlowQueue.queueInvocations.mock.calls[0]
@@ -133,10 +142,73 @@ describe('RerunPaginatorService replay fidelity (hog_flow)', () => {
expect(invocation.state?.actionStepCount).toBe(3)
expect(invocation.state?.customerTaskIdempotencyVersion).toBe(version)
expect(invocation.state?.variables).toEqual({ ticket_id: '123' })
+ expect(invocation.state?.send).toEqual(persistedState.send)
// Sticky rerun counter still increments so max_attempts guards apply.
expect(invocation.state?.rerunAttempts).toBe(1)
})
+ const codeSend = {
+ content: { subject: 'Welcome', html: '<p>Hi</p>', text: 'Hi' },
+ from: { integrationId: 12, email: '[email protected]', name: 'Example' },
+ tracking: { captureEngagement: false },
+ }
+
+ it.each([
+ ['an email sent from code while replay_code_sends is undefined', { send: codeSend }, undefined, []],
+ ['an email sent from code while replay_code_sends is false', { send: codeSend }, false, []],
+ [
+ 'a sensitive message whose content was not stored',
+ { sensitive_variables_omitted: true },
+ true,
+ [
+ expect.objectContaining({
+ instance_id: 'inv-3',
+ level: 'warn',
+ message:
+ "Not re-run: this run's message is sensitive, so its content wasn't stored and can't be sent again.",
+ }),
+ ],
+ ],
+ ] as const)('skips a run with %s', async (_name, stored, allowed, expectedLogs) => {
+ const persistedState = {
+ event: { uuid: 'evt-3', distinct_id: 'd-3', properties: {}, timestamp: '2026-06-01T09:00:00Z' },
+ actionStepCount: 1,
+ variables: {},
+ ...stored,
+ }
+ const rows = [
+ {
+ invocation_id: 'inv-3',
+ parent_run_id: '',
+ attempts: 0,
+ last_scheduled_at: '2026-06-01 09:00:00.000000',
+ first_scheduled_at: '2026-06-01 09:00:00.000000',
+ invocation_globals: JSON.stringify(persistedState),
+ },
+ ]
+
+ const hogFlowQueue = {
+ queueInvocations: jest.fn().mockResolvedValue(undefined),
+ } as unknown as jest.Mocked<CyclotronJobQueuePostgresV2>
+
+ const monitoring = {
+ queueLogs: jest.fn(),
+ flush: jest.fn().mockResolvedValue(undefined),
+ } as unknown as jest.Mocked<HogFunctionMonitoringService>
+ const paginator = buildPaginator(fakeClickhouse(rows), hogFlowQueue, monitoring)
+ await paginator.processPage(
+ teamId,
+ { ...state, request: { ...state.request, replay_code_sends: allowed } },
+ { jobId: 'rerun-3', createdAt: DateTime.now() }
+ )
+
+ expect(hogFlowQueue.queueInvocations.mock.calls.flatMap(([invocations]) => invocations)).toEqual([])
+ const runLogs = monitoring.queueLogs.mock.calls
+ .flatMap(([entries]) => entries)
+ .filter((entry) => entry.instance_id === 'inv-3')
+ expect(runLogs).toEqual(expectedLogs)
+ })
+
it('leaves currentAction undefined for a flow that never progressed (fresh start is correct there)', async () => {
const persistedState = {
event: { uuid: 'evt-2', distinct_id: 'd-2', properties: {}, timestamp: '2026-06-01T09:00:00Z' },
diff --git a/nodejs/src/cdp/rerun/rerun-paginator.service.ts b/nodejs/src/cdp/rerun/rerun-paginator.service.ts
index 9dcf2ac5bbc6..60c46883825b 100644
--- a/nodejs/src/cdp/rerun/rerun-paginator.service.ts
+++ b/nodejs/src/cdp/rerun/rerun-paginator.service.ts
@@ -641,7 +641,8 @@ export class RerunPaginatorService {
teamId,
state.function_kind,
state.function_id,
- row
+ row,
+ state.request.replay_code_sends === true
)
if (!invocation) {
counterRerunInvocationsSkipped.labels(state.function_kind, 'rehydrate_failed').inc()
@@ -682,7 +683,8 @@ export class RerunPaginatorService {
teamId: number,
functionKind: RerunFunctionKind,
functionId: string,
- row: InvocationRow
+ row: InvocationRow,
+ replayCodeSends: boolean
): Promise<CyclotronJobInvocation | null> {
let parsedGlobals: unknown
try {
@@ -774,6 +776,29 @@ export class RerunPaginatorService {
teamId,
invocation_id: row.invocation_id,
})
+ this.monitoringService.queueLogs(
+ [
+ {
+ team_id: teamId,
+ log_source: 'hog_flow',
+ log_source_id: functionId,
+ instance_id: row.invocation_id,
+ timestamp: DateTime.now(),
+ level: 'warn',
+ message:
+ "Not re-run: this run's message is sensitive, so its content wasn't stored and can't be sent again.",
+ },
+ ],
+ 'hog_flow'
+ )
+ return null
+ }
+ if (persistedState.send && !replayCodeSends) {
+ logger.warn('⚠️', 'Skipping rerun of an email sent from code while code sends are off', {
+ functionId,
+ teamId,
+ invocation_id: row.invocation_id,
+ })
return null
}
const eventForFilter = persistedState.event ?? {}
@@ -839,6 +864,7 @@ export class RerunPaginatorService {
personId: persistedState.personId,
// Later message steps check these before the per-worker preferences cache, which can predate them.
messageOptOuts: persistedState.messageOptOuts,
+ send: persistedState.send,
// Sticky rerun counter — mirror the hog function path so the
// lifecycle row producer can derive `attempts` / `is_retry`
// for flows too, and the `max_attempts` guard actually trips.
diff --git a/nodejs/src/cdp/services/hogflows/hogflow-executor.service.test.ts b/nodejs/src/cdp/services/hogflows/hogflow-executor.service.test.ts
index aec05b4a71df..a17da3387f1c 100644
--- a/nodejs/src/cdp/services/hogflows/hogflow-executor.service.test.ts
+++ b/nodejs/src/cdp/services/hogflows/hogflow-executor.service.test.ts
@@ -13,12 +13,14 @@ import { template as setVariableTemplate } from '~/cdp/templates/_destinations/p
import { compileHog } from '~/cdp/templates/compiler'
import { convertToHogFunctionFilterGlobal } from '~/cdp/utils/hog-function-filtering'
import { createHub } from '~/common/utils/db/hub'
+import { PostgresUse } from '~/common/utils/db/postgres'
import { parseJSON } from '~/common/utils/json-parse'
import { logger } from '~/common/utils/logger'
import { FetchResponse, fetch, internalFetch } from '~/common/utils/request'
import { createTestTeamFixture, uniqueTestId } from '~/tests/helpers/sql'
import { Hub, Team } from '../../../types'
+import { CyclotronJobInvocationHogFlow } from '../../types'
import { HOG_FILTERS_EXAMPLES } from '../../_tests/examples'
import { createExampleHogFlowInvocation } from '../../_tests/fixtures-hogflows'
import { CohortMembershipRepository } from '../cohorts/cohort-membership-repository'
@@ -37,6 +39,7 @@ import { RecipientPreferencesService } from '../messaging/recipient-preferences.
import { RecipientTokensService } from '../messaging/recipient-tokens.service'
import { HogFlowExecutorService, createHogFlowInvocation } from './hogflow-executor.service'
import { HogFlowFunctionsService } from './hogflow-functions.service'
+import { AcceptedSend } from './transactional-email'
// Mock before importing fetch
jest.mock('~/common/utils/request', () => {
@@ -3298,6 +3301,70 @@ describe('Hogflow Executor', () => {
},
}) as const
+ const emailFlow = (email: Record<string, any>): HogFlow =>
+ new FixtureHogFlowBuilder()
+ .withTeamId(team.id)
+ .withExitCondition('exit_only_at_end')
+ .withWorkflow({
+ actions: {
+ trigger: {
+ type: 'trigger',
+ config: {
+ type: 'event',
+ filters: HOG_FILTERS_EXAMPLES.no_filters.filters ?? {},
+ },
+ },
+ email_1: {
+ type: 'function_email',
+ config: {
+ template_id: 'template-email-routing-test',
+ inputs: { email: { value: email } },
+ },
+ },
+ exit: {
+ type: 'exit',
+ config: {},
+ },
+ },
+ edges: [
+ { from: 'trigger', to: 'email_1', type: 'continue' },
+ { from: 'email_1', to: 'exit', type: 'continue' },
+ ],
+ })
+ .build()
+
+ const flowInvocation = (hogFlow: HogFlow): CyclotronJobInvocationHogFlow =>
+ createExampleHogFlowInvocation(hogFlow, {
+ event: {
+ ...createHogExecutionGlobals().event,
+ event: '$pageview',
+ },
+ })
+
+ const storedStep = () => ({
+ to: { email: '[email protected]', name: 'Recipient' },
+ from: { integrationId, email: '[email protected]' },
+ subject: 'Test Email',
+ text: 'Test text',
+ html: '<p>Test html</p>',
+ })
+
+ const codeKeyStep = () => ({
+ to: { email: '[email protected]', name: 'Recipient' },
+ from: { integrationId: null },
+ subject: '',
+ text: '',
+ html: '',
+ })
+
+ const codeSend = (): AcceptedSend => ({
+ content: { subject: 'Reset your password', html: '<p>Hi {{ variables.name }}</p>', text: 'Hi' },
+ from: { integrationId, email: '[email protected]', name: 'Security' },
+ tracking: { captureEngagement: true },
+ })
+
it('should route email actions to the email queue', async () => {
await insertEmailSetup('maildev')
@@ -3528,5 +3595,115 @@ if (not res.success) { throw Error(res.error) }`,
expect(result.invocation.state.currentAction?.id).toBe('email_2')
expect(result.invocation.queueMetadata).toEqual({ originQueue: 'hogflow', originPriority: 0 })
})
+
+ it.each([
+ {
+ name: 'a template send as the step renders it',
+ step: storedStep,
+ send: (): AcceptedSend | undefined => undefined,
+ expected: () => ({ type: 'email', ...storedStep() }),
+ },
+ {
+ name: 'a code send for a local template as the step renders it',
+ step: storedStep,
+ send: (): AcceptedSend => ({ tracking: { captureEngagement: false } }),
+ expected: () => ({ type: 'email', ...storedStep() }),
+ },
+ {
+ name: "a code send with the caller's content, sender and reply-to over the step",
+ step: codeKeyStep,
+ send: codeSend,
+ expected: () => ({
+ type: 'email',
+ to: { email: '[email protected]', name: 'Recipient' },
+ from: { integrationId, email: '[email protected]', name: 'Security' },
+ subject: 'Reset your password',
+ preheader: '',
+ text: 'Hi',
+ html: '<p>Hi {{ variables.name }}</p>',
+ }),
+ },
+ ])('hands the email queue $name', async ({ step, send, expected }) => {
+ await insertEmailSetup('ses')
+ const invocation = flowInvocation(emailFlow(step()))
+ invocation.state.variables = { name: 'Ada' }
+ invocation.state.send = send()
+
+ const result = await executor.execute(invocation)
+
+ expect(result.invocation.queueParameters).toEqual(expected())
+ })
+
+ it.each([
+ {
+ name: 'fails a code-key step without an accepted send',
+ send: (): AcceptedSend | undefined => undefined,
+ sent: 0,
+ },
+ { name: 'delivers a code-key step with an accepted send', send: codeSend, sent: 1 },
+ ])('$name', async ({ send, sent }) => {
+ await insertEmailSetup('ses')
+ const sesSend = jest.spyOn(SESv2Client.prototype, 'send') as jest.SpyInstance
+ sesSend.mockResolvedValue({ MessageId: 'code-key-email' })
+ const invocation = flowInvocation(emailFlow(codeKeyStep()))
+ invocation.state.send = send()
+
+ let result = await executor.execute(invocation)
+ const metrics = [...result.metrics]
+ while (!result.finished) {
+ result = await executor.execute(result.invocation)
+ metrics.push(...result.metrics)
+ }
+
+ expect(metrics.filter((m) => m.metric_name === 'email_sent')).toHaveLength(sent)
+ expect(sesSend).toHaveBeenCalledTimes(sent)
+ })
+
+ it.each([
+ {
+ captureEngagement: true,
+ teamCaptures: true,
+ savedWithoutSend: false,
+ captured: ['$workflows_email_sent'],
+ },
+ { captureEngagement: false, teamCaptures: true, savedWithoutSend: false, captured: [] },
+ { captureEngagement: true, teamCaptures: false, savedWithoutSend: false, captured: [] },
+ { captureEngagement: false, teamCaptures: true, savedWithoutSend: true, captured: [] },
+ ])(
+ "captures the email's events by the code send's choice $captureEngagement and the team switch $teamCaptures, step state saved without the send: $savedWithoutSend",
+ async ({ captureEngagement, teamCaptures, savedWithoutSend, captured }) => {
+ await insertEmailSetup('ses')
+ const sesSend = jest.spyOn(SESv2Client.prototype, 'send') as jest.SpyInstance
+ sesSend.mockResolvedValue({ MessageId: 'captured-email' })
+ await hub.postgres.query(
+ PostgresUse.COMMON_WRITE,
+ `INSERT INTO workflows_teamworkflowsconfig
+ (team_id, capture_workflows_engagement_events, email_tracking_consent_mode,
+ email_sending_suspension_reason, ses_tenant_sending_status, email_sending_tier)
+ VALUES ($1, $2, 'off', '', '', 0)`,
+ [team.id, teamCaptures],
+ 'test-team-engagement-capture'
+ )
+ const invocation = flowInvocation(emailFlow(storedStep()))
+ invocation.state.send = { tracking: { captureEngagement } }
+
+ let result = await executor.execute(invocation)
+ const events = [...result.capturedPostHogEvents]
+ if (savedWithoutSend) {
+ delete result.invocation.state.currentAction!.hogFunctionState!.send
+ }
+ while (!result.finished) {
+ result = await executor.execute(result.invocation)
+ events.push(...result.capturedPostHogEvents)
+ }
+
+ expect(
+ events.map((event) => event.event).filter((event) => event.startsWith('$workflows_email_'))
+ ).toEqual(captured)
+ expect(sesSend).toHaveBeenCalledTimes(1)
+ }
+ )
})
})
diff --git a/nodejs/src/cdp/services/hogflows/hogflow-functions.service.ts b/nodejs/src/cdp/services/hogflows/hogflow-functions.service.ts
index 0272e55af062..a154f0cd4885 100644
--- a/nodejs/src/cdp/services/hogflows/hogflow-functions.service.ts
+++ b/nodejs/src/cdp/services/hogflows/hogflow-functions.service.ts
@@ -121,26 +121,28 @@ export class HogFlowFunctionsService {
},
}
+ const functionState = invocation.state.currentAction?.hogFunctionState ?? {
+ globals: await this.hogFunctionExecutor.hogExecutor.buildInputsWithGlobals(
+ hogFunction,
+ globalsWithSource,
+ undefined,
+ {
+ appSourceId: invocation.parentRunId ?? invocation.hogFlow.id,
+ instanceId: invocation.state.currentAction?.id,
+ }
+ ),
+ timings: [],
+ attempts: 0,
+ actionId: invocation.state.currentAction?.id,
+ actionStepCount: invocation.state.actionStepCount,
+ customerTaskIdempotencyVersion: invocation.state.customerTaskIdempotencyVersion,
+ rerunAttempts: invocation.state.rerunAttempts,
+ }
+
const hogFunctionInvocation: CyclotronJobInvocationHogFunction = {
...invocation,
hogFunction,
- state: invocation.state.currentAction?.hogFunctionState ?? {
- globals: await this.hogFunctionExecutor.hogExecutor.buildInputsWithGlobals(
- hogFunction,
- globalsWithSource,
- undefined,
- {
- appSourceId: invocation.parentRunId ?? invocation.hogFlow.id,
- instanceId: invocation.state.currentAction?.id,
- }
- ),
- timings: [],
- attempts: 0,
- actionId: invocation.state.currentAction?.id,
- actionStepCount: invocation.state.actionStepCount,
- customerTaskIdempotencyVersion: invocation.state.customerTaskIdempotencyVersion,
- rerunAttempts: invocation.state.rerunAttempts,
- },
+ state: { ...functionState, ...(invocation.state.send && { send: invocation.state.send }) },
}
return hogFunctionInvocation
diff --git a/nodejs/src/cdp/services/hogflows/hogflow-manager.service.test.ts b/nodejs/src/cdp/services/hogflows/hogflow-manager.service.test.ts
index c4bdd4223116..6f3fe6bfcb5b 100644
--- a/nodejs/src/cdp/services/hogflows/hogflow-manager.service.test.ts
+++ b/nodejs/src/cdp/services/hogflows/hogflow-manager.service.test.ts
@@ -10,7 +10,7 @@ import { createTeam, createTestTeamFixture } from '~/tests/helpers/sql'
import { INVALID_TEAM_ID_KEYS } from '~/tests/helpers/team'
import { Hub } from '~/types'
-import { insertHogFlow } from '../../_tests/fixtures-hogflows'
+import { hideHogFlowsFromReplica, insertHogFlow } from '../../_tests/fixtures-hogflows'
import { HogFlowManagerService } from './hogflow-manager.service'
describe('HogFlowManager', () => {
@@ -148,6 +148,133 @@ describe('HogFlowManager', () => {
}, 10000)
})
+ it('reads the primary once per batch of workflows the database no longer has, and again once the miss expires', async () => {
+ const [deletedId, secondDeletedId, thirdDeletedId] = Array.from(
+ { length: 3 },
+ () => new FixtureHogFlowBuilder().withTeamId(teamId1).build().id
+ )
+ const querySpy = jest.spyOn(hub.postgres, 'query')
+ const primaryReads = (): number =>
+ querySpy.mock.calls.filter(([use, , , tag]) => use === PostgresUse.COMMON_WRITE && tag === 'fetchHogFlows')
+ .length
+ const baseNow = Date.now()
+
+ try {
+ const flows = await Promise.all(
+ Array.from({ length: 5 }, () => manager.getHogFlowCheckingPrimaryOnMiss(deletedId))
+ )
+ expect(flows).toEqual(Array(5).fill(null))
+ expect(primaryReads()).toBe(1)
+
+ jest.spyOn(Date, 'now').mockReturnValue(baseNow + 59_000)
+ expect(await manager.getHogFlowCheckingPrimaryOnMiss(deletedId)).toBeNull()
+ expect(primaryReads()).toBe(1)
+
+ jest.spyOn(Date, 'now').mockReturnValue(baseNow + 61_000)
+ expect(await manager.getHogFlowCheckingPrimaryOnMiss(deletedId)).toBeNull()
+ expect(primaryReads()).toBe(2)
+
+ const batch = await manager.getHogFlowsCheckingPrimaryOnMiss([
+ secondDeletedId,
+ thirdDeletedId,
+ secondDeletedId,
+ hogFlows[0].id,
+ ])
+ expect(batch).toEqual({
+ [secondDeletedId]: null,
+ [thirdDeletedId]: null,
+ [hogFlows[0].id]: expect.objectContaining({ id: hogFlows[0].id }),
+ })
+ expect(primaryReads()).toBe(3)
+ } finally {
+ jest.spyOn(Date, 'now').mockReturnValue(baseNow)
+ }
+ })
+
+ it('serves a workflow only the primary has from cache until a reload names it or the entry expires', async () => {
+ const lateFlow = await insertHogFlow(
+ hub.postgres,
+ new FixtureHogFlowBuilder().withTeamId(teamId1).withStatus('active').build()
+ )
+ const querySpy = hideHogFlowsFromReplica(hub.postgres, [lateFlow.id])
+ const reads = (): number[] =>
+ [PostgresUse.COMMON_READ, PostgresUse.COMMON_WRITE].map(
+ (use) =>
+ querySpy.mock.calls.filter(([target, , , tag]) => target === use && tag === 'fetchHogFlows').length
+ )
+ const lateFlowMatch = expect.objectContaining({ id: lateFlow.id })
+ const baseNow = Date.now()
+
+ try {
+ expect(await manager.getHogFlowCheckingPrimaryOnMiss(lateFlow.id)).toEqual(lateFlowMatch)
+ expect(reads()).toEqual([1, 1])
+
+ jest.spyOn(Date, 'now').mockReturnValue(baseNow + 59_000)
+ expect(await manager.getHogFlowsCheckingPrimaryOnMiss([lateFlow.id])).toEqual({
+ [lateFlow.id]: lateFlowMatch,
+ })
+ expect(reads()).toEqual([1, 1])
+
+ manager['onHogFlowsReloaded'](teamId1, [lateFlow.id])
+ expect(await manager.getHogFlowCheckingPrimaryOnMiss(lateFlow.id)).toEqual(lateFlowMatch)
+ expect(reads()).toEqual([2, 2])
+
+ jest.spyOn(Date, 'now').mockReturnValue(baseNow + 120_000)
+ expect(await manager.getHogFlowCheckingPrimaryOnMiss(lateFlow.id)).toEqual(lateFlowMatch)
+ expect(reads()).toEqual([3, 3])
+ } finally {
+ jest.spyOn(Date, 'now').mockReturnValue(baseNow)
+ }
+ })
+
+ it('keeps no primary result from a read that a reload overtook', async () => {
+ const lateFlow = await insertHogFlow(
+ hub.postgres,
+ new FixtureHogFlowBuilder().withTeamId(teamId1).withStatus('active').build()
+ )
+ const querySpy = hideHogFlowsFromReplica(hub.postgres, [lateFlow.id])
+ const hideFromReplica = querySpy.getMockImplementation()!
+ let primaryReadStarted: () => void = () => {}
+ const primaryReadInFlight = new Promise<void>((resolve) => {
+ primaryReadStarted = resolve
+ })
+ let releasePrimaryRead: () => void = () => {}
+ const primaryReadReleased = new Promise<void>((resolve) => {
+ releasePrimaryRead = resolve
+ })
+ let holdNextPrimaryRead = true
+ querySpy.mockImplementation(async (target, queryString, values, tag, logLevel) => {
+ const result = await hideFromReplica(target, queryString, values, tag, logLevel)
+ if (holdNextPrimaryRead && target === PostgresUse.COMMON_WRITE && tag === 'fetchHogFlows') {
+ holdNextPrimaryRead = false
+ primaryReadStarted()
+ await primaryReadReleased
+ }
+ return result
+ })
+
+ try {
+ const overtakenLookup = manager.getHogFlowCheckingPrimaryOnMiss(lateFlow.id)
+ await primaryReadInFlight
+ await hub.postgres.query(
+ PostgresUse.COMMON_WRITE,
+ `UPDATE posthog_hogflow SET status = 'draft' WHERE id = $1`,
+ [lateFlow.id],
+ 'testKey'
+ )
+ manager['onHogFlowsReloaded'](teamId1, [lateFlow.id])
+ releasePrimaryRead()
+ await overtakenLookup
+
+ expect(await manager.getHogFlowCheckingPrimaryOnMiss(lateFlow.id)).toMatchObject({
+ id: lateFlow.id,
+ status: 'draft',
+ })
+ } finally {
+ querySpy.mockRestore()
+ }
+ })
+
describe('email sending pause', () => {
// The pause only reaches the send choke point through the manager's column list, so a
// column left out of HOG_FLOW_FIELDS silently lets a paused workflow keep sending.
diff --git a/nodejs/src/cdp/services/hogflows/hogflow-manager.service.ts b/nodejs/src/cdp/services/hogflows/hogflow-manager.service.ts
index d5c3baf2461f..9b5b033b98a5 100644
--- a/nodejs/src/cdp/services/hogflows/hogflow-manager.service.ts
+++ b/nodejs/src/cdp/services/hogflows/hogflow-manager.service.ts
@@ -49,9 +49,14 @@ const HOG_FLOW_FIELDS = [
export type HogFlowTeamInfo = Pick<HogFlow, 'id' | 'team_id' | 'version'>
+const PRIMARY_RESULT_TTL_MS = 60 * 1000
+const PRIMARY_RESULT_MAX_SIZE = 10_000
+
export class HogFlowManagerService {
private lazyLoader: LazyLoader<HogFlow>
private lazyLoaderByTeam: LazyLoader<HogFlowTeamInfo[]>
+ private primaryReads = new Map<HogFlow['id'], Promise<HogFlow | null>>()
+ private primaryResults = new Map<HogFlow['id'], { flow: HogFlow | null; until: number }>()
constructor(
private postgres: PostgresRouter,
@@ -142,6 +147,76 @@ export class HogFlowManagerService {
return (await this.lazyLoader.get(id)) ?? null
}
+ public async getHogFlowCheckingPrimaryOnMiss(id: HogFlow['id']): Promise<HogFlow | null> {
+ return (await this.getHogFlowsCheckingPrimaryOnMiss([id]))[id]
+ }
+
+ public async getHogFlowsCheckingPrimaryOnMiss(
+ ids: HogFlow['id'][]
+ ): Promise<Record<HogFlow['id'], HogFlow | null>> {
+ const remembered = Object.fromEntries(ids.map((id) => [id, this.rememberedPrimaryResult(id)]))
+ const cached = await this.lazyLoader.getMany(ids.filter((id) => !remembered[id]))
+ const misses = [...new Set(ids.filter((id) => remembered[id] === undefined && !cached[id]))]
+ const fromPrimary = await this.readPrimary(misses)
+ return Object.fromEntries(ids.map((id) => [id, remembered[id] ?? cached[id] ?? fromPrimary[id] ?? null]))
+ }
+
+ private async readPrimary(ids: HogFlow['id'][]): Promise<Record<HogFlow['id'], HogFlow | null>> {
+ const unread = ids.filter((id) => !this.primaryReads.has(id))
+ if (unread.length > 0) {
+ const read = this.fetchHogFlows(unread, PostgresUse.COMMON_WRITE)
+ for (const id of unread) {
+ const settled: Promise<HogFlow | null> = read.then((flows) =>
+ this.settlePrimaryRead(id, flows[id] ?? null, settled)
+ )
+ this.primaryReads.set(id, settled)
+ const forget = (): void => {
+ if (this.primaryReads.get(id) === settled) {
+ this.primaryReads.delete(id)
+ }
+ }
+ settled.then(forget, forget)
+ }
+ }
+ const reads = ids.map((id) => this.primaryReads.get(id) ?? Promise.resolve(null))
+ const flows = await Promise.all(reads)
+ return Object.fromEntries(ids.map((id, index) => [id, flows[index]]))
+ }
+
+ private settlePrimaryRead(id: HogFlow['id'], flow: HogFlow | null, read: Promise<HogFlow | null>): HogFlow | null {
+ if (this.primaryReads.get(id) !== read) {
+ return flow
+ }
+ if (flow) {
+ this.lazyLoader.markForRefresh(id)
+ }
+ this.rememberPrimaryResult(id, flow)
+ return flow
+ }
+
+ private rememberedPrimaryResult(id: HogFlow['id']): HogFlow | null | undefined {
+ const remembered = this.primaryResults.get(id)
+ if (remembered === undefined) {
+ return undefined
+ }
+ if (remembered.until > Date.now()) {
+ return remembered.flow
+ }
+ this.primaryResults.delete(id)
+ return undefined
+ }
+
+ private rememberPrimaryResult(id: HogFlow['id'], flow: HogFlow | null): void {
+ this.primaryResults.delete(id)
+ if (this.primaryResults.size >= PRIMARY_RESULT_MAX_SIZE) {
+ const oldest = this.primaryResults.keys().next().value
+ if (oldest !== undefined) {
+ this.primaryResults.delete(oldest)
+ }
+ }
+ this.primaryResults.set(id, { flow, until: Date.now() + PRIMARY_RESULT_TTL_MS })
+ }
+
public async getHogFlows(ids: HogFlow['id'][]): Promise<Record<HogFlow['id'], HogFlow | null>> {
return await this.lazyLoader.getMany(ids)
}
@@ -149,6 +224,10 @@ export class HogFlowManagerService {
private onHogFlowsReloaded(teamId: Team['id'], hogFlowIds: HogFlow['id'][]): void {
this.lazyLoaderByTeam.markForRefresh(teamId.toString())
this.lazyLoader.markForRefresh(hogFlowIds)
+ hogFlowIds.forEach((id) => {
+ this.primaryResults.delete(id)
+ this.primaryReads.delete(id)
+ })
}
private async fetchTeamHogFlows(teamIds: string[]): Promise<Record<string, HogFlowTeamInfo[]>> {
@@ -173,11 +252,14 @@ export class HogFlowManagerService {
return byTeam
}
- private async fetchHogFlows(ids: string[]): Promise<Record<string, HogFlow | undefined>> {
+ private async fetchHogFlows(
+ ids: string[],
+ use: PostgresUse = PostgresUse.COMMON_READ
+ ): Promise<Record<string, HogFlow | undefined>> {
logger.debug('[HogFlowManager]', 'Fetching hog flows', { ids })
const response = await this.postgres.query<HogFlow>(
- PostgresUse.COMMON_READ,
+ use,
`SELECT ${HOG_FLOW_FIELDS.join(', ')} FROM posthog_hogflow WHERE id = ANY($1)`,
[ids],
'fetchHogFlows'
diff --git a/nodejs/src/cdp/services/hogflows/transactional-email.ts b/nodejs/src/cdp/services/hogflows/transactional-email.ts
index e1671bf1afd1..1577b5d207a4 100644
--- a/nodejs/src/cdp/services/hogflows/transactional-email.ts
+++ b/nodejs/src/cdp/services/hogflows/transactional-email.ts
@@ -1,9 +1,56 @@
import { Counter, Histogram } from 'prom-client'
import { z } from 'zod'
+import { CyclotronInvocationQueueParametersEmailType } from '~/cdp/schema/cyclotron'
+
+export const AcceptedSendSchema = z
+ .object({
+ content: z
+ .object({
+ subject: z.string().min(1),
+ html: z.string().optional(),
+ text: z.string().optional(),
+ })
+ .strict()
+ .readonly()
+ .refine((content) => !!content.html || !!content.text, { message: 'content needs html or text' })
+ .optional(),
+ from: z
+ .object({
+ integrationId: z.number().int(),
+ email: z.string().optional(),
+ name: z.string().optional(),
+ })
+ .strict()
+ .readonly()
+ .optional(),
+ replyTo: z.array(z.string()).readonly().optional(),
+ tracking: z.object({ captureEngagement: z.boolean() }).strict().readonly(),
+ })
+ .strict()
+ .readonly()
+
+export type AcceptedSend = z.infer<typeof AcceptedSendSchema>
+
+export const acceptedSendEmailFields = (
+ send: AcceptedSend | undefined
+): Partial<CyclotronInvocationQueueParametersEmailType> => ({
+ ...(send?.content && {
+ subject: send.content.subject,
+ html: send.content.html ?? '',
+ text: send.content.text ?? '',
+ preheader: '',
+ }),
+ ...(send?.from && { from: { ...send.from } }),
+ ...(send?.replyTo && { replyTo: send.replyTo.join(', ') }),
+})
+
+export const RECIPIENT_ADDRESS_PATTERN =
+ /^[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-zA-Z0-9!#$%&'*+/=?^_`{|}~-]+)*@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)+$/
+
// What Django sends to run a transactional email's workflow once.
export const TransactionalEmailRequestSchema = z.object({
- to: z.string().email(),
+ to: z.string().email({ pattern: RECIPIENT_ADDRESS_PATTERN }),
// The person the send is tracked against. Django falls back to the recipient's address.
distinct_id: z.string().min(1),
// The caller's `message_data`, available to the template as `variables`.
@@ -12,6 +59,13 @@ export const TransactionalEmailRequestSchema = z.object({
idempotency_key: z.string().min(1).max(255).optional(),
})
+export const TEMPLATE_SEND_TOKEN_CLAIMS = { kind: undefined }
+export const CODE_SEND_TOKEN_CLAIMS = { kind: 'code_send' }
+
+export const AcceptedSendRequestSchema = TransactionalEmailRequestSchema.extend({ send: AcceptedSendSchema })
+
+export type TransactionalEmailRequest = z.infer<typeof TransactionalEmailRequestSchema> & { send?: AcceptedSend }
+
export type TransactionalEmailOutcome =
| 'queued'
| 'replayed'
diff --git a/nodejs/src/cdp/services/messaging/email-tracking.service.test.ts b/nodejs/src/cdp/services/messaging/email-tracking.service.test.ts
index bedfc201df44..1a6ea76fb6e1 100644
--- a/nodejs/src/cdp/services/messaging/email-tracking.service.test.ts
+++ b/nodejs/src/cdp/services/messaging/email-tracking.service.test.ts
@@ -1,14 +1,15 @@
import { createMockJobQueue } from '~/tests/helpers/mocks/job-queue.mock'
import { mockProducerObserver } from '~/tests/helpers/mocks/producer.mock'
-import { mockFetch } from '~/tests/helpers/mocks/request.mock'
+import { mockFetch, mockInternalFetch } from '~/tests/helpers/mocks/request.mock'
import { Server } from 'http'
+import { register } from 'prom-client'
import supertest from 'supertest'
import express from 'ultimate-express'
import { FixtureHogFlowBuilder } from '~/cdp/_tests/builders/hogflow.builder'
-import { insertHogFunction } from '~/cdp/_tests/fixtures'
-import { insertHogFlow } from '~/cdp/_tests/fixtures-hogflows'
+import { createExampleInvocation, insertHogFunction, insertIntegration } from '~/cdp/_tests/fixtures'
+import { hideHogFlowsFromReplica, insertHogFlow } from '~/cdp/_tests/fixtures-hogflows'
import { CdpApi } from '~/cdp/cdp-api'
import { CyclotronJobInvocationHogFunction, HogFunctionType } from '~/cdp/types'
import { setupExpressApp } from '~/common/api/router'
@@ -17,6 +18,7 @@ import { KAFKA_APP_METRICS_2, KAFKA_EMAIL_STATUS, KAFKA_LOG_ENTRIES } from '~/co
import { closeHub, createHub } from '~/common/utils/db/hub'
import { PostgresUse } from '~/common/utils/db/postgres'
import * as envUtils from '~/common/utils/env-utils'
+import { parseJSON } from '~/common/utils/json-parse'
import { UUIDT } from '~/common/utils/utils'
import { createCdpConsumerDeps } from '~/tests/helpers/cdp'
import { waitForExpect } from '~/tests/helpers/expectations'
@@ -46,6 +48,7 @@ describe('EmailTrackingService', () => {
team = (await createTestTeamFixture(hub.postgres)).team
mockFetch.mockClear()
+ mockInternalFetch.mockClear()
mockProducerObserver.resetKafkaProducer()
})
@@ -201,14 +204,31 @@ describe('EmailTrackingService', () => {
} as any
const phIdOf = (html: string): string => html.match(/ph_id=([A-Za-z0-9._-]+)/)![1]
- it('includes distinct_id in the public tracking URLs in dev/test', () => {
- const out = addTrackingToEmail(
- '<body><a href="https://example.com">x</a></body>',
- invocationWithDistinctId,
- signer
- )
- expect(signer.parse(phIdOf(out))?.distinctId).toBe('leaky-id')
- })
+ it.each([
+ { captureEngagement: undefined, distinctId: 'leaky-id' },
+ { captureEngagement: true, distinctId: 'leaky-id' },
+ { captureEngagement: false, distinctId: undefined },
+ ])(
+ 'mints the pixel and redirect codes in dev/test with the distinct_id the send choice $captureEngagement allows',
+ ({ captureEngagement, distinctId }) => {
+ const out = addTrackingToEmail(
+ '<body><a href="https://example.com">x</a></body>',
+ {
+ ...invocationWithDistinctId,
+ state: {
+ ...invocationWithDistinctId.state,
+ send: captureEngagement === undefined ? undefined : { tracking: { captureEngagement } },
+ },
+ },
+ signer
+ )
+ const codes = [...out.matchAll(/ph_id=([A-Za-z0-9_.-]+)/g)].map((match) => signer.parse(match[1]))
+ expect(codes).toHaveLength(2)
+ for (const code of codes) {
+ expect(code).toMatchObject({ functionId: 'fn-1', invocationId: 'inv-1', distinctId })
+ }
+ }
+ )
it('omits distinct_id from the public tracking URLs in production (Referer-leak guard)', () => {
const devSpy = jest.spyOn(envUtils, 'isDevEnv').mockReturnValue(false)
@@ -457,15 +477,15 @@ describe('EmailTrackingService', () => {
parentRunId,
workflowVersion,
transactionalMessageId,
+ headerCode,
}: {
functionId: string
parentRunId?: string
workflowVersion?: number
// Sends the bounce as SES reports it for a transactional email with this message id.
transactionalMessageId?: string
+ headerCode?: string
}): Promise<supertest.Response> => {
- // Third arg opts into the versioned payload, which `generate` won't emit by default
- // until phase two of the rollout — see EMIT_VERSIONED_PAYLOAD in tracking-code.ts.
const trackingCode = signer.generate(
{
functionId,
@@ -484,7 +504,7 @@ describe('EmailTrackingService', () => {
source: '[email protected]',
messageId: 'ses-message-id',
destination: ['[email protected]'],
- headers: [{ name: TRACKING_CODE_HEADER_NAME, value: trackingCode }],
+ headers: [{ name: TRACKING_CODE_HEADER_NAME, value: headerCode ?? trackingCode }],
...(transactionalMessageId
? {
commonHeaders: { to: ['[email protected]'] },
@@ -514,11 +534,24 @@ describe('EmailTrackingService', () => {
.send(JSON.stringify(envelope))
}
- it('writes a hog_flow log entry for a bounce that resolves to a flow', async () => {
+ it.each([
+ ['on the replica', false, false],
+ ['only on the primary, during replica lag', true, false],
+ ['on the replica, while the hog function lookup fails', false, true],
+ ['only on the primary, while the hog function lookup fails', true, true],
+ ])('writes a hog_flow log entry for a bounce that resolves to a flow %s', async (_, lagging, fail) => {
const hogFlow = await insertHogFlow(
hub.postgres,
new FixtureHogFlowBuilder().withTeamId(team.id).build()
)
+ if (lagging) {
+ hideHogFlowsFromReplica(hub.postgres, [hogFlow.id])
+ }
+ if (fail) {
+ jest.spyOn(api['emailTrackingService']['hogFunctionManager'], 'getHogFunctions').mockRejectedValue(
+ new Error('hog function lookup failed')
+ )
+ }
const res = await postBounce({ functionId: hogFlow.id })
expect(res.status).toBe(200)
@@ -541,11 +574,36 @@ describe('EmailTrackingService', () => {
expect(metrics.map((m) => m.value.metric_name)).toEqual(['email_bounced', 'email_bounced_hard'])
expect(metrics[0].value).toMatchObject({
team_id: team.id,
+ app_source: 'hog_flow',
+ app_source_id: hogFlow.id,
metric_name: 'email_bounced',
metric_kind: 'email',
})
})
+ it('counts a failed primary lookup as a lookup failure for a bounce on a flow only the primary has', async () => {
+ const hogFlow = await insertHogFlow(
+ hub.postgres,
+ new FixtureHogFlowBuilder().withTeamId(team.id).build()
+ )
+ hideHogFlowsFromReplica(hub.postgres, [hogFlow.id])
+ jest.spyOn(
+ api['emailTrackingService']['hogFlowManager'],
+ 'getHogFlowsCheckingPrimaryOnMiss'
+ ).mockRejectedValue(new Error('primary lookup failed'))
+ const lookupFailures = async (): Promise<number> =>
+ (await register.getSingleMetric('email_tracking_errors_total')!.get()).values.find(
+ (v) => v.labels.error_type === 'hog_flow_lookup_failed' && v.labels.source === 'ses'
+ )?.value ?? 0
+ const before = await lookupFailures()
+
+ const res = await postBounce({ functionId: hogFlow.id })
+ expect(res.status).toBe(200)
+
+ expect(await lookupFailures()).toBeGreaterThan(before)
+ expect(mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_LOG_ENTRIES)).toHaveLength(0)
+ })
+
it('records the bounced status of a transactional email', async () => {
const hogFlow = await insertHogFlow(
hub.postgres,
@@ -570,7 +628,9 @@ describe('EmailTrackingService', () => {
])
})
- it('records the metric but writes no log entry when the bounce resolves to a hog_function', async () => {
+ it('records the metric but writes no log entry and reads no workflow from the primary when the bounce resolves to a hog_function', async () => {
+ const query = jest.spyOn(hub.postgres, 'query')
+
const res = await postBounce({ functionId: hogFunction.id })
expect(res.status).toBe(200)
@@ -586,6 +646,11 @@ describe('EmailTrackingService', () => {
const logs = mockProducerObserver.getProducedKafkaMessagesForTopic(KAFKA_LOG_ENTRIES)
expect(logs).toHaveLength(0)
+ expect(
+ query.mock.calls.filter(
+ ([target, , , tag]) => target === PostgresUse.COMMON_WRITE && tag === 'fetchHogFlows'
+ )
+ ).toHaveLength(0)
})
it('attributes the bounce to the version that sent it, not the version live when it lands', async () => {
@@ -613,6 +678,76 @@ describe('EmailTrackingService', () => {
})
})
+ it.each([
+ { captureEngagement: true, teamCaptures: true, captured: 1 },
+ { captureEngagement: undefined, teamCaptures: true, captured: 1 },
+ { captureEngagement: false, teamCaptures: true, captured: 0 },
+ { captureEngagement: true, teamCaptures: false, captured: 0 },
+ ])(
+ "captures the bounce of a sent email by the send's choice $captureEngagement and the team switch $teamCaptures",
+ async ({ captureEngagement, teamCaptures, captured }) => {
+ await hub.postgres.query(
+ PostgresUse.COMMON_WRITE,
+ `INSERT INTO workflows_teamworkflowsconfig
+ (team_id, capture_workflows_engagement_events, email_tracking_consent_mode,
+ email_sending_suspension_reason, ses_tenant_sending_status, email_sending_tier)
+ VALUES ($1, $2, 'off', '', '', 0)`,
+ [team.id, teamCaptures],
+ 'test-set-engagement-capture'
+ )
+ const hogFlow = await insertHogFlow(
+ hub.postgres,
+ new FixtureHogFlowBuilder().withTeamId(team.id).build()
+ )
+ const integration = await insertIntegration(hub.postgres, team.id, {
+ kind: 'email',
+ config: {
+ email: '[email protected]',
+ name: 'Sender',
+ domain: 'posthog-test.com',
+ verified: true,
+ provider: 'ses',
+ },
+ })
+ const emailService = api['hogExecutorAsync']['deps'].emailService
+ const sesSend = jest
+ .spyOn(emailService.sesV2Client!, 'send')
+ .mockResolvedValue({ MessageId: 'ses-message-id' } as never)
+ const invocation = createExampleInvocation({
+ id: hogFlow.id,
+ team_id: team.id,
+ })
+ invocation.state.globals.event.distinct_id = 'user-1'
+ if (captureEngagement !== undefined) {
+ invocation.state.send = { tracking: { captureEngagement } }
+ }
+ invocation.queueParameters = {
+ type: 'email',
+ to: { email: '[email protected]', name: 'User' },
+ from: { integrationId: integration.id },
+ subject: 'Reset your password',
+ text: 'Hi',
+ html: '<p>Hi</p>',
+ }
+
+ expect((await emailService.executeSendEmail(invocation)).error).toBeUndefined()
+ const headers: { Name: string; Value: string }[] = (sesSend.mock.calls[0][0] as any).input.Content
+ .Simple.Headers
+ const headerCode = headers.find((header) => header.Name === TRACKING_CODE_HEADER_NAME)!.Value
+ mockInternalFetch.mockClear()
+
+ const res = await postBounce({ functionId: hogFlow.id, headerCode })
+ expect(res.status).toBe(200)
+
+ const capturedEvents = mockInternalFetch.mock.calls.map(
+ ([, options]) => parseJSON(String(options?.body)).event
+ )
+ expect(capturedEvents.filter((event) => event === '$workflows_email_bounced')).toHaveLength(
+ captured
+ )
+ }
+ )
+
it('keys the log entry under parentRunId for batch-triggered runs', async () => {
const hogFlow = await insertHogFlow(
hub.postgres,
diff --git a/nodejs/src/cdp/services/messaging/email-tracking.service.ts b/nodejs/src/cdp/services/messaging/email-tracking.service.ts
index 723e8040506f..6bcea0f6056f 100644
--- a/nodejs/src/cdp/services/messaging/email-tracking.service.ts
+++ b/nodejs/src/cdp/services/messaging/email-tracking.service.ts
@@ -6,6 +6,7 @@ import { HogFlow } from '~/cdp/schema/hogflow'
import {
CyclotronJobInvocationHogFlow,
CyclotronJobInvocationHogFunction,
+ HogFunctionType,
LogEntry,
LogEntryLevel,
MinimalAppMetric,
@@ -127,6 +128,14 @@ export const resolveEmailSendingVersion = (invocation: CyclotronJobInvocationHog
: undefined
}
+export const resolveEmailTrackingDistinctId = (
+ invocation: Pick<CyclotronJobInvocationHogFunction, 'state' | 'hogFunction'>
+): string | undefined => {
+ return invocation.state?.send?.tracking.captureEngagement === false
+ ? undefined
+ : resolveEmailEngagementDistinctId(invocation)
+}
+
// HTML attribute values arrive entity-encoded (e.g. `&amp;`, `&#38;`). Decode before
// percent-encoding for the tracking redirect, otherwise `?a=1&amp;b=2` round-trips
// through `target=` as a literal `&amp;` and breaks the destination page's query string.
@@ -156,7 +165,7 @@ export const addTrackingToEmail = (
// record metrics. In production they don't (SES webhooks own open/click attribution via the
// signed header), so embedding distinct_id in the public `ph_id` would be unused — and worse,
// a tracked-link click could leak the recipient identifier to the destination via the Referer.
- const distinctId = isDevEnv() || isTestEnv() ? resolveEmailEngagementDistinctId(invocation) : undefined
+ const distinctId = isDevEnv() || isTestEnv() ? resolveEmailTrackingDistinctId(invocation) : undefined
// The pixel and redirect codes carry the sending version too. SES-attributed opens and clicks
// read the version off the signed header instead, but a provider we track ourselves has these
// URLs as its only carrier, and without the version its engagement cannot split by version.
@@ -338,7 +347,9 @@ export class EmailTrackingService {
// The function ID could be one or the other so we load both
const [hogFunction, hogFlow] = await Promise.all([
this.hogFunctionManager.getHogFunction(functionId).catch(() => null),
- this.hogFlowManager.getHogFlow(functionId).catch(() => null),
+ this.getFeedbackHogFlows([functionId], source)
+ .then((hogFlows) => hogFlows[functionId])
+ .catch(() => null),
])
let teamId = hogFunction?.team_id ?? hogFlow?.team_id
@@ -438,7 +449,7 @@ export class EmailTrackingService {
let hogFlows: Record<string, HogFlow | null> = {}
try {
- hogFlows = await this.hogFlowManager.getHogFlows(uniqueFunctionIds)
+ hogFlows = await this.getFeedbackHogFlows(uniqueFunctionIds, 'ses')
} catch (error) {
logger.error('[EmailTrackingService] trackLogs: Failed to load hog flows', { error })
emailTrackingErrorsCounter.inc({ error_type: 'hog_flow_lookup_failed', source: 'ses' })
@@ -609,6 +620,32 @@ export class EmailTrackingService {
}
}
+ private async getFeedbackHogFlows(
+ ids: HogFlow['id'][],
+ source: 'direct' | 'ses'
+ ): Promise<Record<HogFlow['id'], HogFlow | null>> {
+ const [onReplica, hogFunctions] = await Promise.all([
+ this.hogFlowManager.getHogFlows(ids),
+ this.hogFunctionManager.getHogFunctions(ids).catch((error): Record<string, HogFunctionType | null> => {
+ logger.warn('[EmailTrackingService] Failed to load hog functions', { error })
+ return {}
+ }),
+ ])
+ const unknown = ids.filter((id) => !onReplica[id] && !hogFunctions[id])
+ const onPrimary: Record<HogFlow['id'], HogFlow | null> =
+ unknown.length > 0
+ ? await this.hogFlowManager.getHogFlowsCheckingPrimaryOnMiss(unknown).catch((error) => {
+ logger.error('[EmailTrackingService] Failed to load hog flows from the primary', {
+ error,
+ ids: unknown,
+ })
+ emailTrackingErrorsCounter.inc({ error_type: 'hog_flow_lookup_failed', source })
+ return {}
+ })
+ : {}
+ return Object.fromEntries(ids.map((id) => [id, onReplica[id] ?? onPrimary[id] ?? null]))
+ }
+
private parseTrackingParams(query: Record<string, any>): {
functionId?: string
invocationId?: string
diff --git a/nodejs/src/cdp/services/messaging/email.service.ts b/nodejs/src/cdp/services/messaging/email.service.ts
index ae1f2961ce96..768117da547b 100644
--- a/nodejs/src/cdp/services/messaging/email.service.ts
+++ b/nodejs/src/cdp/services/messaging/email.service.ts
@@ -30,8 +30,8 @@ import { EmailStatusService, isTransactionalEmailSend } from './email-status.ser
import { EmailSuppressionService, SuppressionCheckOptions } from './email-suppression.service'
import {
addTrackingToEmail,
- resolveEmailEngagementDistinctId,
resolveEmailSendingVersion,
+ resolveEmailTrackingDistinctId,
} from './email-tracking.service'
import { mailDevTransport, mailDevWebUrl } from './helpers/maildev'
import { maybeAddPreheaderToEmail } from './helpers/preheader'
@@ -878,7 +878,7 @@ export class EmailService {
}
}
- const distinctId = resolveEmailEngagementDistinctId(invocation)
+ const distinctId = resolveEmailTrackingDistinctId(invocation)
if (
distinctId &&
!isTest &&
@@ -1227,7 +1227,7 @@ export class EmailService {
if (!this.sesV2Client) {
throw new Error('SES is not configured - set SES_REGION and AWS credentials')
}
- const distinctId = resolveEmailEngagementDistinctId(result.invocation)
+ const distinctId = resolveEmailTrackingDistinctId(result.invocation)
// Full signed code (with distinct_id + isTest) rides in the header; the short unsigned
// carrier (no distinct_id/isTest) goes in the SES EmailTag, guaranteed under the 256-char
// tag-value limit. The webhook reads the header first and only falls back to the tag.
diff --git a/nodejs/src/cdp/services/monitoring/hog-invocation-results.service.test.ts b/nodejs/src/cdp/services/monitoring/hog-invocation-results.service.test.ts
index 02009788a033..38157d680e8f 100644
--- a/nodejs/src/cdp/services/monitoring/hog-invocation-results.service.test.ts
+++ b/nodejs/src/cdp/services/monitoring/hog-invocation-results.service.test.ts
@@ -144,22 +144,29 @@ describe('HogInvocationResultsService', () => {
expect(decoded).not.toContain('abc123')
})
+ const send = {
+ content: { subject: 'Your login code', html: '<p>Your code is 482913</p>' },
+ tracking: { captureEngagement: false },
+ }
+ const messageData = { variables: { code: '482913' }, send }
+ const noMessageData = { variables: undefined, send: undefined }
+
it.each([
- ['an ordinary email', false, {}, { code: '482913' }],
- ['a sensitive email', true, {}, undefined],
- ['an ordinary push', false, {}, { code: '482913' }, 'function_push'],
- ['a sensitive push', true, {}, undefined, 'function_push'],
- ['a recovery stub of a sensitive run', undefined, { sensitive: true }, undefined],
+ ['an ordinary email', false, {}, messageData],
+ ['a sensitive email', true, {}, noMessageData],
+ ['an ordinary push', false, {}, messageData, 'function_push'],
+ ['a sensitive push', true, {}, noMessageData, 'function_push'],
+ ['a recovery stub of a sensitive run', undefined, { sensitive: true }, noMessageData],
[
'a recovery stub of an unstamped transactional run',
undefined,
{ event: { event: '$transactional_email' } },
- undefined,
+ noMessageData,
],
- ['a recovery stub of an ordinary run', undefined, {}, { code: '482913' }],
+ ['a recovery stub of an ordinary run', undefined, {}, messageData],
])(
'keeps the message data in a workflow snapshot only for %s',
- async (_name, sensitive, state, variables, stepType = 'function_email') => {
+ async (_name, sensitive, state, expected, stepType = 'function_email') => {
const hogFlow =
sensitive === undefined
? ({ id: 'flow-1' } as any)
@@ -181,7 +188,7 @@ describe('HogInvocationResultsService', () => {
edges: [],
})
.build()
- const invocation = createExampleHogFlowInvocation(hogFlow, { variables: { code: '482913' } })
+ const invocation = createExampleHogFlowInvocation(hogFlow, { variables: { code: '482913' }, send })
// A step paused for a provider retry keeps the rendered message on its VM stack.
const vmState = { stack: [{ title: 'Your code is 482913' }] }
const person = { id: 'person-1', properties: { code: '482913' } }
@@ -198,9 +205,9 @@ describe('HogInvocationResultsService', () => {
const globals = (await decodeInvocationGlobals(
parseProducedRows(outputs)[0].invocation_globals
)) as Record<string, any>
- expect(globals.variables).toEqual(variables)
- expect(globals.currentAction.hogFunctionState.vmState).toEqual(variables && vmState)
- expect(globals.currentAction.hogFunctionState.globals.person).toEqual(variables && person)
+ expect({ variables: globals.variables, send: globals.send }).toEqual(expected)
+ expect(globals.currentAction.hogFunctionState.vmState).toEqual(expected.variables && vmState)
+ expect(globals.currentAction.hogFunctionState.globals.person).toEqual(expected.variables && person)
}
)
diff --git a/nodejs/src/cdp/services/monitoring/hog-invocation-results.service.ts b/nodejs/src/cdp/services/monitoring/hog-invocation-results.service.ts
index 9eea0595b43d..8bd839575b82 100644
--- a/nodejs/src/cdp/services/monitoring/hog-invocation-results.service.ts
+++ b/nodejs/src/cdp/services/monitoring/hog-invocation-results.service.ts
@@ -208,7 +208,7 @@ const INPUT_KEYS: ReadonlySet<string> = new Set(['inputs'])
// A sensitive message carries reset links, login codes or private text, so its snapshot keeps no
// variables either, and a rerun can't rebuild the send. A step paused mid-send (e.g. for a provider
// retry) holds the rendered message on its VM stack, and the person and group properties it read, so those go too.
-const SENSITIVE_KEYS: ReadonlySet<string> = new Set(['inputs', 'variables', 'vmState', 'person', 'groups'])
+const SENSITIVE_KEYS: ReadonlySet<string> = new Set(['inputs', 'variables', 'vmState', 'person', 'groups', 'send'])
export const SENSITIVE_VARIABLES_OMITTED = 'sensitive_variables_omitted'
const stripKeys = <T>(value: T, keys: ReadonlySet<string>): T => {
diff --git a/nodejs/src/cdp/types.ts b/nodejs/src/cdp/types.ts
index 710bbb724e6b..62ee80fa99ca 100644
--- a/nodejs/src/cdp/types.ts
+++ b/nodejs/src/cdp/types.ts
@@ -13,6 +13,7 @@ import {
PersonPropertyFilter,
Team,
} from '../types'
+import type { AcceptedSend } from './services/hogflows/transactional-email'
import type { HogErrorClass } from './utils/hog-error-classification'
export type HogBytecode = any[]
@@ -431,6 +432,7 @@ export type CyclotronJobInvocationHogFunctionContext = {
sensitive?: boolean
actionStepCount?: number
customerTaskIdempotencyVersion?: 1
+ send?: AcceptedSend
}
export type WorkflowStepResumeStatus = 'completed' | 'failed' | 'cancelled'
@@ -577,6 +579,7 @@ export type HogFlowInvocationContext = {
// conversions are counted by the matcher and never touch this flag.
conversionCounted?: boolean
variables?: Record<string, any>
+ send?: AcceptedSend
// Sticky counter incremented by the rerun paginator on rehydration. Lets
// the lifecycle row producer derive `attempts` / `is_retry` for hog flows
// the same way it does for hog functions, so the `max_attempts` guard on
diff --git a/nodejs/src/common/api/middleware/internal-api-auth.test.ts b/nodejs/src/common/api/middleware/internal-api-auth.test.ts
index b4c6c554b132..9c9b5c3b491b 100644
--- a/nodejs/src/common/api/middleware/internal-api-auth.test.ts
+++ b/nodejs/src/common/api/middleware/internal-api-auth.test.ts
@@ -192,6 +192,8 @@ describe('createInternalApiAuthMiddleware', () => {
['/api/projects/1/hog_flows/some-uuid/reschedule_parked'],
['/api/projects/1/hog_flows/some-uuid/invocations/cancel'],
['/api/projects/1/hog_flows/some-uuid/batch_jobs/some-job-id/cancel'],
+ ['/api/projects/1/hog_flows/some-uuid/transactional_email'],
+ ['/api/projects/1/hog_flows/some-uuid/transactional_email/send'],
])('should skip auth for scoped-JWT route %s, which authenticates in its handler', (path) => {
const middleware = createInternalApiAuthMiddleware({ secret: 'test-secret' })
const req = mockRequest(path, {})
@@ -210,6 +212,7 @@ describe('createInternalApiAuthMiddleware', () => {
['/api/projects/1/hog_flows/some-uuid/batch_invocations/cancel'],
['/api/some/other/route/cancel'],
['/api/projects/1/hog_flows/some-uuid/reschedule_parked/extra'],
+ ['/api/projects/1/hog_flows/some-uuid/transactional_email/send/extra'],
])('should still require the secret for %s despite resembling a scoped route', (path) => {
const middleware = createInternalApiAuthMiddleware({ secret: 'test-secret' })
const req = mockRequest(path, {})
diff --git a/nodejs/src/common/api/middleware/internal-api-auth.ts b/nodejs/src/common/api/middleware/internal-api-auth.ts
index 9d14943f7e4a..71e280506b84 100644
--- a/nodejs/src/common/api/middleware/internal-api-auth.ts
+++ b/nodejs/src/common/api/middleware/internal-api-auth.ts
@@ -40,6 +40,7 @@ const SCOPED_AUTH_PATH_PATTERNS = [
/^\/api\/projects\/[^/]+\/workflow_steps\/resume$/,
// CdpApi.postTransactionalEmail, verified against WORKFLOWS_TRANSACTIONAL_EMAIL_JWT_SECRET, audience-pinned.
/^\/api\/projects\/[^/]+\/hog_flows\/[^/]+\/transactional_email$/,
+ /^\/api\/projects\/[^/]+\/hog_flows\/[^/]+\/transactional_email\/send$/,
]
export interface InternalApiAuthOptions {