:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
perf(cdp): run transactional email checks concurrently (#125765)
Harley Alexander8d41f9742817
diff --git a/nodejs/src/cdp/cdp-api.serial.test.ts b/nodejs/src/cdp/cdp-api.serial.test.tsindex 394cfad1b36e..b24fe27c6dfc 100644--- a/nodejs/src/cdp/cdp-api.serial.test.ts+++ b/nodejs/src/cdp/cdp-api.serial.test.ts@@ -2479,13 +2479,36 @@ describe('CDP API', () => {expect(mockQueueInvocations).not.toHaveBeenCalled()})- it('queues a send when the team is over its email quota', async () => {- jest.spyOn(api['deps'].quotaLimiting, 'isTeamQuotaLimited').mockResolvedValue(true)+ it.each<[string, () => { payload: Record<string, unknown>; verify?: () => void }]>([+ [+ 'the team is over its email quota',+ () => {+ jest.spyOn(api['deps'].quotaLimiting, 'isTeamQuotaLimited').mockResolvedValue(true)+ return { payload: body }+ },+ ],+ [+ "the lookup of the recipient's mail servers fails",+ () => {+ jest.spyOn(api['emailValidationService'], 'getRecipientSkipReason').mockRestore()+ const resolveMx = jest.mocked(api['emailValidationService']['resolver'].resolveMx)+ resolveMx.mockRejectedValueOnce(Object.assign(new Error('queryMx ETIMEOUT'), { code: 'ETIMEOUT' }))+ // A domain no other test sends to, so no cached verdict answers before the lookup.+ const domain = `${new UUIDT().toString()}.example.org`+ return {+ payload: { ...body, to: `user@${domain}` },+ verify: () => expect(resolveMx).toHaveBeenCalledWith(domain),+ }+ },+ ],+ ])('queues a send when %s', async (_desc, arrange) => {+ const { payload, verify } = arrange()const queuedBefore = await outcomeCount('queued')- const res = await send(hogFlow.id, { Authorization: `Bearer ${mintToken(team.id, hogFlow.id)}` })+ const res = await send(hogFlow.id, { Authorization: `Bearer ${mintToken(team.id, hogFlow.id)}` }, payload)expect(res.status).toEqual(202)+ verify?.()expect(await outcomeCount('queued')).toEqual(queuedBefore + 1)expect(mockQueueInvocations).toHaveBeenCalledTimes(1)})@@ -2641,6 +2664,57 @@ describe('CDP API', () => {expect(JSON.stringify(validationLogs)).not.toContain('example.com')})+ it.each([+ [+ 'sending_suspended',+ 422,+ [+ 'sending_suspended',+ 'sending_paused',+ 'recipient_suppressed',+ 'quota_limited',+ 'recipient_unreachable',+ ],+ ],+ [+ 'sending_paused',+ 422,+ ['sending_paused', 'recipient_suppressed', 'quota_limited', 'recipient_unreachable'],+ ],+ ['recipient_suppressed', 422, ['recipient_suppressed', 'quota_limited', 'recipient_unreachable']],+ ['quota_limited', 429, ['quota_limited', 'recipient_unreachable']],+ ] as const)('refuses with %s when it and every later check would refuse', async (code, status, reasons) => {+ // Marketing, because a transactional email bypasses the quota.+ const marketingFlow = await insertHogFlow(transactionalFlow('api', 'active', 'marketing'))+ const arrange: Record<(typeof reasons)[number], () => void> = {+ sending_suspended: () =>+ jest+ .spyOn(api['teamWorkflowsConfigService'], 'getEmailSendingSuspension')+ .mockResolvedValue('staff'),+ sending_paused: () =>+ jest+ .spyOn(api['hogFlowManager'], 'getHogFlowCheckingPrimaryOnMiss')+ .mockResolvedValue({ ...marketingFlow, email_sending_paused_at: Date.now() }),+ recipient_suppressed: () =>+ jest.spyOn(api['emailSuppressionService'], 'isSuppressed').mockResolvedValue(true),+ quota_limited: () =>+ jest.spyOn(api['deps'].quotaLimiting, 'isTeamQuotaLimited').mockResolvedValue(true),+ recipient_unreachable: () =>+ jest+ .spyOn(api['emailValidationService'], 'getRecipientSkipReason')+ .mockResolvedValue('Skipping send: the domain has no reachable mail servers.'),+ }+ reasons.forEach((reason) => arrange[reason]())++ const res = await send(marketingFlow.id, {+ Authorization: `Bearer ${mintToken(team.id, marketingFlow.id)}`,+ })++ expect(res.status).toEqual(status)+ expect(res.body).toEqual({ error: code })+ expect(mockQueueInvocations).not.toHaveBeenCalled()+ })+it('sends to a recipient whose only suppression is a spam complaint', async () => {await hub.postgres.query(PostgresUse.COMMON_WRITE,diff --git a/nodejs/src/cdp/cdp-api.ts b/nodejs/src/cdp/cdp-api.tsindex 03ea68d80d37..b73f70012b1a 100644--- a/nodejs/src/cdp/cdp-api.ts+++ b/nodejs/src/cdp/cdp-api.ts@@ -1224,11 +1224,15 @@ export class CdpApi {const { to, distinct_id: distinctId, variables, idempotency_key: idempotencyKey, send } = parsed.datastages.start('load')- const team = await this.deps.teamManager.getTeam(parseInt(req.params.team_id)).catch(() => null)+ const teamLookup = this.deps.teamManager.getTeam(parseInt(req.params.team_id)).catch(() => null)+ const hogFlowLookup = this.hogFlowManager.getHogFlowCheckingPrimaryOnMiss(req.params.id)+ // A missing team returns before the workflow lookup is awaited, so its failure must not go unhandled.+ hogFlowLookup.catch(() => undefined)+ const team = await teamLookupif (!team) {return reply('not_found', 404, { error: 'Team not found' })}- const hogFlow = await this.hogFlowManager.getHogFlowCheckingPrimaryOnMiss(req.params.id)+ const hogFlow = await hogFlowLookupif (!hogFlow || hogFlow.team_id !== team.id) {return reply('not_found', 404, { error: 'Workflow not found' })}@@ -1369,21 +1373,33 @@ export class CdpApi {if (hogFlow.status !== 'active') {return { outcome: 'not_sendable', status: 400 }}- if (await this.teamWorkflowsConfigService.getEmailSendingSuspension(teamId)) {+ // The lookups run at the same time, and the checks below await them in order, so the first refusal in that+ // order wins. The MX check's DNS lookup runs even when an earlier check refuses, which is acceptable because+ // refusals are rare and the verdict is cached per domain.+ const suspension = this.teamWorkflowsConfigService.getEmailSendingSuspension(teamId)+ const suppressed = this.emailSuppressionService.isSuppressed(teamId, to, { transactional: true })+ const quota = checkHogFlowQuotaLimits(hogFlow, teamId, this.deps.quotaLimiting)+ // A DNS failure lets the send through.+ const recipientSkipReason = this.emailValidationService.getRecipientSkipReason(teamId, to)+ // An earlier refusal returns before the later lookups are awaited, so their failures must not go unhandled.+ const lookups: Promise<unknown>[] = [suspension, suppressed, quota, recipientSkipReason]+ for (const lookup of lookups) {+ lookup.catch(() => undefined)+ }++ if (await suspension) {return { outcome: 'sending_suspended', status: 422 }}if (hogFlow.email_sending_paused_at) {return { outcome: 'sending_paused', status: 422 }}- if (await this.emailSuppressionService.isSuppressed(teamId, to, { transactional: true })) {+ if (await suppressed) {return { outcome: 'recipient_suppressed', status: 422 }}- const { isLimited } = await checkHogFlowQuotaLimits(hogFlow, teamId, this.deps.quotaLimiting)- if (isLimited) {+ if ((await quota).isLimited) {return { outcome: 'quota_limited', status: 429 }}- // Last, because a cache miss costs a DNS lookup. A DNS failure lets the send through.- if (await this.emailValidationService.getRecipientSkipReason(teamId, to)) {+ if (await recipientSkipReason) {return { outcome: 'recipient_unreachable', status: 422 }}return nulldiff --git a/nodejs/src/cdp/cdp-services.ts b/nodejs/src/cdp/cdp-services.tsindex 6e3eb56bb824..03a5fdb36ef5 100644--- a/nodejs/src/cdp/cdp-services.ts+++ b/nodejs/src/cdp/cdp-services.ts@@ -220,8 +220,9 @@ export interface CdpCoreServicesDeps {internalCaptureService: InternalCaptureService/*** SES Valkey pool shared with the SES rate limiter, opened only on pods whose- * capabilities actually execute email actions (hogflow/email cyclotron workers).- * `null` on every other CDP consumer and on cdp-api so idle pods don't hold+ * capabilities actually execute email actions (hogflow/email cyclotron workers)+ * and on cdp-api, which checks a transactional email's recipient against the+ * MX-verdict cache. `null` on every other CDP consumer so idle pods don't hold* open connections against the SES Valkey instance.*/emailValidationValkey: RedisV2 | null@@ -459,6 +460,7 @@ export function createCdpCoreServices(const workflowsActivationReporter = new WorkflowsActivationReporter(deps.teamManager)// Per-workflow send pacing rides the SES Valkey pool, which is only populated on pods that// execute email actions — exactly where the limit is enforced. Null elsewhere disables it.+ // cdp-api also gets the pool, but its sends are all test sends, which skip both limits.const workflowEmailRateLimiter = deps.emailValidationValkey? new RateLimiterService(deps.emailValidationValkey, { name: 'workflow-email' }): null@@ -558,8 +560,8 @@ export function createCdpCoreServices()// MX verdicts live on the dedicated SES Valkey (same instance as the SES rate// limiter, separate pool). The pool is created by the server only on pods- // whose capabilities execute email actions; everywhere else this is null- // and EmailValidationService degrades to the local cache + DNS.+ // whose capabilities execute email actions and on cdp-api; everywhere else+ // this is null and EmailValidationService degrades to the local cache + DNS.const emailValidationService = new EmailValidationService(deps.emailValidationValkey)const cohortMembershipRepository = new PostgresCohortMembershipRepository(deps.postgres)// Observer writes to both stores; the read source decides which verdict drives the metric.diff --git a/nodejs/src/cdp/services/messaging/email-validation.service.test.ts b/nodejs/src/cdp/services/messaging/email-validation.service.test.tsindex e4a0b95a3f32..480b5bf5ee3b 100644--- a/nodejs/src/cdp/services/messaging/email-validation.service.test.ts+++ b/nodejs/src/cdp/services/messaging/email-validation.service.test.ts@@ -237,6 +237,18 @@ describe('EmailValidationService', () => {expect(reason).toContain('no reachable mail servers')expect(mockResolveMx).not.toHaveBeenCalled()})++ it('falls back to DNS when the Valkey pool cannot hand out a client', async () => {+ const valkey: RedisV2 = {+ useClient: jest.fn().mockRejectedValue(new Error('ResourceRequest timed out')),+ usePipeline: jest.fn(),+ }+ const withValkey = new EmailValidationService(valkey)++ const reason = await withValkey.getSkipReason(emailInvocation('[email protected]'), emailAction)+ expect(reason).toBeNull()+ expect(mockResolveMx).toHaveBeenCalledWith('example.com')+ })})describe('out-of-band domain detection', () => {const originalCloudDeployment = process.env.CLOUD_DEPLOYMENTdiff --git a/nodejs/src/cdp/services/messaging/email-validation.service.ts b/nodejs/src/cdp/services/messaging/email-validation.service.tsindex 3d9f675ddb2f..05af0deec3c9 100644--- a/nodejs/src/cdp/services/messaging/email-validation.service.ts+++ b/nodejs/src/cdp/services/messaging/email-validation.service.ts@@ -259,10 +259,11 @@ export class EmailValidationService {if (!this.valkey) {return null}- // failOpen: a Valkey outage returns null (treated as a cache miss), never throws.- const value = await this.valkey.useClient({ name: 'email-mx-read', failOpen: true }, (client) =>- client.get(VALKEY_KEY_PREFIX + domain)- )+ // failOpen covers a failed command, but useClient still rethrows when the pool cannot hand out a client+ // in time. The cache is optional, so both failures count as a cache miss and the lookup goes to DNS.+ const value = await this.valkey+ .useClient({ name: 'email-mx-read', failOpen: true }, (client) => client.get(VALKEY_KEY_PREFIX + domain))+ .catch(() => null)if (value === '1') {return true}@@ -277,8 +278,10 @@ export class EmailValidationService {return}const ttlSeconds = Math.floor((deliverable ? POSITIVE_TTL_MS : NEGATIVE_TTL_MS) / 1000)- await this.valkey.useClient({ name: 'email-mx-write', failOpen: true }, (client) =>- client.set(VALKEY_KEY_PREFIX + domain, deliverable ? '1' : '0', 'EX', ttlSeconds)- )+ await this.valkey+ .useClient({ name: 'email-mx-write', failOpen: true }, (client) =>+ client.set(VALKEY_KEY_PREFIX + domain, deliverable ? '1' : '0', 'EX', ttlSeconds)+ )+ .catch(() => undefined)}}diff --git a/nodejs/src/cdp/services/rate-limiter/rate-limiter-valkey-pool.ts b/nodejs/src/cdp/services/rate-limiter/rate-limiter-valkey-pool.tsindex 7e9eaed1011b..a92e8ae99036 100644--- a/nodejs/src/cdp/services/rate-limiter/rate-limiter-valkey-pool.ts+++ b/nodejs/src/cdp/services/rate-limiter/rate-limiter-valkey-pool.ts@@ -4,6 +4,8 @@ import { logger } from '~/common/utils/logger'import type { CdpConfig } from '../../config'+const OPTIONAL_VALKEY_TIMEOUT_MS = 250+export type RateLimiterValkeyConfig = Pick<CdpConfig,| 'SES_RATE_LIMITER_VALKEY_HOST'@@ -26,7 +28,8 @@ export type RateLimiterValkeyConfig = Pick<*/export function createSesRateLimiterValkeyPool(config: RateLimiterValkeyConfig,- name = 'ses-rate-limiter'+ name = 'ses-rate-limiter',+ { optional = false }: { optional?: boolean } = {}): RedisV2 | null {if (!config.SES_RATE_LIMITER_VALKEY_HOST) {logger.info('🪙', `[${name}] no SES Valkey host configured — feature disabled`)@@ -40,6 +43,25 @@ export function createSesRateLimiterValkeyPool(const tls = config.SES_RATE_LIMITER_VALKEY_TLS ? {} : undefined+ if (optional) {+ // A caller that works without this Valkey must not stop when it is down, or wait on it long.+ return createRedisV2PoolFromConfig({+ connection: {+ url: config.SES_RATE_LIMITER_VALKEY_HOST,+ options: {+ port: config.SES_RATE_LIMITER_VALKEY_PORT,+ password: config.SES_RATE_LIMITER_VALKEY_PASSWORD,+ tls,+ commandTimeout: OPTIONAL_VALKEY_TIMEOUT_MS,+ },+ name,+ exitOnRepeatedErrors: false,+ },+ poolMinSize: 0,+ poolMaxSize: config.REDIS_POOL_MAX_SIZE,+ acquireTimeoutMillis: OPTIONAL_VALKEY_TIMEOUT_MS,+ })+ }return createRedisV2PoolFromConfig({connection: {url: config.SES_RATE_LIMITER_VALKEY_HOST,diff --git a/nodejs/src/server.ts b/nodejs/src/server.tsindex 81391973e04b..a4dac1d7a454 100644--- a/nodejs/src/server.ts+++ b/nodejs/src/server.ts@@ -138,7 +138,8 @@ export class PluginServer implements NodeServer {// Build typed deps objects for consumers. `emailValidationValkey` and `frequencyCapValkey`// are null in the shared deps and set only by the cyclotron workers that run the hogflow// email action (see `withEmailActionValkeys` at their loaders below) — no- // other consumer touches the SES or frequency cap Valkey.+ // other consumer touches the SES or frequency cap Valkey, except cdp-api, which reads+ // the MX-verdict cache to check a transactional email's recipient.const cdpDeps: CdpConsumerBaseDeps | undefined = needsCdp? {postgres: this.postgres!,@@ -245,7 +246,13 @@ export class PluginServer implements NodeServer {: nullconst api = new CdpApi(this.config,- { ...cdpDeps!, idempotencyValkey: createIdempotencyValkeyPool(this.config) },+ {+ ...cdpDeps!,+ idempotencyValkey: createIdempotencyValkeyPool(this.config),+ emailValidationValkey: createSesRateLimiterValkeyPool(this.config, 'email-mx-validation', {+ optional: true,+ }),+ },{ hogQueue: kafkaQueue, hogflowQueue: postgresV2Queue },batchResolverProducer)@@ -457,8 +464,9 @@ export class PluginServer implements NodeServer {/*** Grants the SES Valkey pool that backs the shared MX-verdict cache to a worker's* deps. Only the cyclotron workers that run the hogflow email action call this, so- * the pool is never opened on other CDP consumers or cdp-api — an idle Valkey sized- * for the SES rate limiter shouldn't hold connections from pods that never validate.+ * the pool is never opened on other CDP consumers — an idle Valkey sized for the SES+ * rate limiter shouldn't hold connections from pods that never validate. cdp-api+ * opens the same MX pool itself, for the transactional email recipient check.* Null when no SES Valkey host is configured (local dev), in which case* EmailValidationService degrades to its local cache + DNS.*