Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

perf(cdp): run transactional email checks concurrently (#125765)

Harley Alexander8d41f9742817
diff --git a/nodejs/src/cdp/cdp-api.serial.test.ts b/nodejs/src/cdp/cdp-api.serial.test.ts
index 394cfad1b36e..b24fe27c6dfc 100644
--- a/nodejs/src/cdp/cdp-api.serial.test.ts
+++ b/nodejs/src/cdp/cdp-api.serial.test.ts
@@ -2479,13 +2479,36 @@ describe('CDP API', () => {
expect(mockQueueInvocations).not.toHaveBeenCalled()
})
- it('queues a send when the team is over its email quota', async () => {
- jest.spyOn(api['deps'].quotaLimiting, 'isTeamQuotaLimited').mockResolvedValue(true)
+ it.each<[string, () => { payload: Record<string, unknown>; verify?: () => void }]>([
+ [
+ 'the team is over its email quota',
+ () => {
+ jest.spyOn(api['deps'].quotaLimiting, 'isTeamQuotaLimited').mockResolvedValue(true)
+ return { payload: body }
+ },
+ ],
+ [
+ "the lookup of the recipient's mail servers fails",
+ () => {
+ jest.spyOn(api['emailValidationService'], 'getRecipientSkipReason').mockRestore()
+ const resolveMx = jest.mocked(api['emailValidationService']['resolver'].resolveMx)
+ resolveMx.mockRejectedValueOnce(Object.assign(new Error('queryMx ETIMEOUT'), { code: 'ETIMEOUT' }))
+ // A domain no other test sends to, so no cached verdict answers before the lookup.
+ const domain = `${new UUIDT().toString()}.example.org`
+ return {
+ payload: { ...body, to: `user@${domain}` },
+ verify: () => expect(resolveMx).toHaveBeenCalledWith(domain),
+ }
+ },
+ ],
+ ])('queues a send when %s', async (_desc, arrange) => {
+ const { payload, verify } = arrange()
const queuedBefore = await outcomeCount('queued')
- const res = await send(hogFlow.id, { Authorization: `Bearer ${mintToken(team.id, hogFlow.id)}` })
+ const res = await send(hogFlow.id, { Authorization: `Bearer ${mintToken(team.id, hogFlow.id)}` }, payload)
expect(res.status).toEqual(202)
+ verify?.()
expect(await outcomeCount('queued')).toEqual(queuedBefore + 1)
expect(mockQueueInvocations).toHaveBeenCalledTimes(1)
})
@@ -2641,6 +2664,57 @@ describe('CDP API', () => {
expect(JSON.stringify(validationLogs)).not.toContain('example.com')
})
+ it.each([
+ [
+ 'sending_suspended',
+ 422,
+ [
+ 'sending_suspended',
+ 'sending_paused',
+ 'recipient_suppressed',
+ 'quota_limited',
+ 'recipient_unreachable',
+ ],
+ ],
+ [
+ 'sending_paused',
+ 422,
+ ['sending_paused', 'recipient_suppressed', 'quota_limited', 'recipient_unreachable'],
+ ],
+ ['recipient_suppressed', 422, ['recipient_suppressed', 'quota_limited', 'recipient_unreachable']],
+ ['quota_limited', 429, ['quota_limited', 'recipient_unreachable']],
+ ] as const)('refuses with %s when it and every later check would refuse', async (code, status, reasons) => {
+ // Marketing, because a transactional email bypasses the quota.
+ const marketingFlow = await insertHogFlow(transactionalFlow('api', 'active', 'marketing'))
+ const arrange: Record<(typeof reasons)[number], () => void> = {
+ sending_suspended: () =>
+ jest
+ .spyOn(api['teamWorkflowsConfigService'], 'getEmailSendingSuspension')
+ .mockResolvedValue('staff'),
+ sending_paused: () =>
+ jest
+ .spyOn(api['hogFlowManager'], 'getHogFlowCheckingPrimaryOnMiss')
+ .mockResolvedValue({ ...marketingFlow, email_sending_paused_at: Date.now() }),
+ recipient_suppressed: () =>
+ jest.spyOn(api['emailSuppressionService'], 'isSuppressed').mockResolvedValue(true),
+ quota_limited: () =>
+ jest.spyOn(api['deps'].quotaLimiting, 'isTeamQuotaLimited').mockResolvedValue(true),
+ recipient_unreachable: () =>
+ jest
+ .spyOn(api['emailValidationService'], 'getRecipientSkipReason')
+ .mockResolvedValue('Skipping send: the domain has no reachable mail servers.'),
+ }
+ reasons.forEach((reason) => arrange[reason]())
+
+ const res = await send(marketingFlow.id, {
+ Authorization: `Bearer ${mintToken(team.id, marketingFlow.id)}`,
+ })
+
+ expect(res.status).toEqual(status)
+ expect(res.body).toEqual({ error: code })
+ expect(mockQueueInvocations).not.toHaveBeenCalled()
+ })
+
it('sends to a recipient whose only suppression is a spam complaint', async () => {
await hub.postgres.query(
PostgresUse.COMMON_WRITE,
diff --git a/nodejs/src/cdp/cdp-api.ts b/nodejs/src/cdp/cdp-api.ts
index 03ea68d80d37..b73f70012b1a 100644
--- a/nodejs/src/cdp/cdp-api.ts
+++ b/nodejs/src/cdp/cdp-api.ts
@@ -1224,11 +1224,15 @@ export class CdpApi {
const { to, distinct_id: distinctId, variables, idempotency_key: idempotencyKey, send } = parsed.data
stages.start('load')
- const team = await this.deps.teamManager.getTeam(parseInt(req.params.team_id)).catch(() => null)
+ const teamLookup = this.deps.teamManager.getTeam(parseInt(req.params.team_id)).catch(() => null)
+ const hogFlowLookup = this.hogFlowManager.getHogFlowCheckingPrimaryOnMiss(req.params.id)
+ // A missing team returns before the workflow lookup is awaited, so its failure must not go unhandled.
+ hogFlowLookup.catch(() => undefined)
+ const team = await teamLookup
if (!team) {
return reply('not_found', 404, { error: 'Team not found' })
}
- const hogFlow = await this.hogFlowManager.getHogFlowCheckingPrimaryOnMiss(req.params.id)
+ const hogFlow = await hogFlowLookup
if (!hogFlow || hogFlow.team_id !== team.id) {
return reply('not_found', 404, { error: 'Workflow not found' })
}
@@ -1369,21 +1373,33 @@ export class CdpApi {
if (hogFlow.status !== 'active') {
return { outcome: 'not_sendable', status: 400 }
}
- if (await this.teamWorkflowsConfigService.getEmailSendingSuspension(teamId)) {
+ // The lookups run at the same time, and the checks below await them in order, so the first refusal in that
+ // order wins. The MX check's DNS lookup runs even when an earlier check refuses, which is acceptable because
+ // refusals are rare and the verdict is cached per domain.
+ const suspension = this.teamWorkflowsConfigService.getEmailSendingSuspension(teamId)
+ const suppressed = this.emailSuppressionService.isSuppressed(teamId, to, { transactional: true })
+ const quota = checkHogFlowQuotaLimits(hogFlow, teamId, this.deps.quotaLimiting)
+ // A DNS failure lets the send through.
+ const recipientSkipReason = this.emailValidationService.getRecipientSkipReason(teamId, to)
+ // An earlier refusal returns before the later lookups are awaited, so their failures must not go unhandled.
+ const lookups: Promise<unknown>[] = [suspension, suppressed, quota, recipientSkipReason]
+ for (const lookup of lookups) {
+ lookup.catch(() => undefined)
+ }
+
+ if (await suspension) {
return { outcome: 'sending_suspended', status: 422 }
}
if (hogFlow.email_sending_paused_at) {
return { outcome: 'sending_paused', status: 422 }
}
- if (await this.emailSuppressionService.isSuppressed(teamId, to, { transactional: true })) {
+ if (await suppressed) {
return { outcome: 'recipient_suppressed', status: 422 }
}
- const { isLimited } = await checkHogFlowQuotaLimits(hogFlow, teamId, this.deps.quotaLimiting)
- if (isLimited) {
+ if ((await quota).isLimited) {
return { outcome: 'quota_limited', status: 429 }
}
- // Last, because a cache miss costs a DNS lookup. A DNS failure lets the send through.
- if (await this.emailValidationService.getRecipientSkipReason(teamId, to)) {
+ if (await recipientSkipReason) {
return { outcome: 'recipient_unreachable', status: 422 }
}
return null
diff --git a/nodejs/src/cdp/cdp-services.ts b/nodejs/src/cdp/cdp-services.ts
index 6e3eb56bb824..03a5fdb36ef5 100644
--- a/nodejs/src/cdp/cdp-services.ts
+++ b/nodejs/src/cdp/cdp-services.ts
@@ -220,8 +220,9 @@ export interface CdpCoreServicesDeps {
internalCaptureService: InternalCaptureService
/**
* SES Valkey pool shared with the SES rate limiter, opened only on pods whose
- * capabilities actually execute email actions (hogflow/email cyclotron workers).
- * `null` on every other CDP consumer and on cdp-api so idle pods don't hold
+ * capabilities actually execute email actions (hogflow/email cyclotron workers)
+ * and on cdp-api, which checks a transactional email's recipient against the
+ * MX-verdict cache. `null` on every other CDP consumer so idle pods don't hold
* open connections against the SES Valkey instance.
*/
emailValidationValkey: RedisV2 | null
@@ -459,6 +460,7 @@ export function createCdpCoreServices(
const workflowsActivationReporter = new WorkflowsActivationReporter(deps.teamManager)
// Per-workflow send pacing rides the SES Valkey pool, which is only populated on pods that
// execute email actions — exactly where the limit is enforced. Null elsewhere disables it.
+ // cdp-api also gets the pool, but its sends are all test sends, which skip both limits.
const workflowEmailRateLimiter = deps.emailValidationValkey
? new RateLimiterService(deps.emailValidationValkey, { name: 'workflow-email' })
: null
@@ -558,8 +560,8 @@ export function createCdpCoreServices(
)
// MX verdicts live on the dedicated SES Valkey (same instance as the SES rate
// limiter, separate pool). The pool is created by the server only on pods
- // whose capabilities execute email actions; everywhere else this is null
- // and EmailValidationService degrades to the local cache + DNS.
+ // whose capabilities execute email actions and on cdp-api; everywhere else
+ // this is null and EmailValidationService degrades to the local cache + DNS.
const emailValidationService = new EmailValidationService(deps.emailValidationValkey)
const cohortMembershipRepository = new PostgresCohortMembershipRepository(deps.postgres)
// Observer writes to both stores; the read source decides which verdict drives the metric.
diff --git a/nodejs/src/cdp/services/messaging/email-validation.service.test.ts b/nodejs/src/cdp/services/messaging/email-validation.service.test.ts
index e4a0b95a3f32..480b5bf5ee3b 100644
--- a/nodejs/src/cdp/services/messaging/email-validation.service.test.ts
+++ b/nodejs/src/cdp/services/messaging/email-validation.service.test.ts
@@ -237,6 +237,18 @@ describe('EmailValidationService', () => {
expect(reason).toContain('no reachable mail servers')
expect(mockResolveMx).not.toHaveBeenCalled()
})
+
+ it('falls back to DNS when the Valkey pool cannot hand out a client', async () => {
+ const valkey: RedisV2 = {
+ useClient: jest.fn().mockRejectedValue(new Error('ResourceRequest timed out')),
+ usePipeline: jest.fn(),
+ }
+ const withValkey = new EmailValidationService(valkey)
+
+ const reason = await withValkey.getSkipReason(emailInvocation('[email protected]'), emailAction)
+ expect(reason).toBeNull()
+ expect(mockResolveMx).toHaveBeenCalledWith('example.com')
+ })
})
describe('out-of-band domain detection', () => {
const originalCloudDeployment = process.env.CLOUD_DEPLOYMENT
diff --git a/nodejs/src/cdp/services/messaging/email-validation.service.ts b/nodejs/src/cdp/services/messaging/email-validation.service.ts
index 3d9f675ddb2f..05af0deec3c9 100644
--- a/nodejs/src/cdp/services/messaging/email-validation.service.ts
+++ b/nodejs/src/cdp/services/messaging/email-validation.service.ts
@@ -259,10 +259,11 @@ export class EmailValidationService {
if (!this.valkey) {
return null
}
- // failOpen: a Valkey outage returns null (treated as a cache miss), never throws.
- const value = await this.valkey.useClient({ name: 'email-mx-read', failOpen: true }, (client) =>
- client.get(VALKEY_KEY_PREFIX + domain)
- )
+ // failOpen covers a failed command, but useClient still rethrows when the pool cannot hand out a client
+ // in time. The cache is optional, so both failures count as a cache miss and the lookup goes to DNS.
+ const value = await this.valkey
+ .useClient({ name: 'email-mx-read', failOpen: true }, (client) => client.get(VALKEY_KEY_PREFIX + domain))
+ .catch(() => null)
if (value === '1') {
return true
}
@@ -277,8 +278,10 @@ export class EmailValidationService {
return
}
const ttlSeconds = Math.floor((deliverable ? POSITIVE_TTL_MS : NEGATIVE_TTL_MS) / 1000)
- await this.valkey.useClient({ name: 'email-mx-write', failOpen: true }, (client) =>
- client.set(VALKEY_KEY_PREFIX + domain, deliverable ? '1' : '0', 'EX', ttlSeconds)
- )
+ await this.valkey
+ .useClient({ name: 'email-mx-write', failOpen: true }, (client) =>
+ client.set(VALKEY_KEY_PREFIX + domain, deliverable ? '1' : '0', 'EX', ttlSeconds)
+ )
+ .catch(() => undefined)
}
}
diff --git a/nodejs/src/cdp/services/rate-limiter/rate-limiter-valkey-pool.ts b/nodejs/src/cdp/services/rate-limiter/rate-limiter-valkey-pool.ts
index 7e9eaed1011b..a92e8ae99036 100644
--- a/nodejs/src/cdp/services/rate-limiter/rate-limiter-valkey-pool.ts
+++ b/nodejs/src/cdp/services/rate-limiter/rate-limiter-valkey-pool.ts
@@ -4,6 +4,8 @@ import { logger } from '~/common/utils/logger'
import type { CdpConfig } from '../../config'
+const OPTIONAL_VALKEY_TIMEOUT_MS = 250
+
export type RateLimiterValkeyConfig = Pick<
CdpConfig,
| 'SES_RATE_LIMITER_VALKEY_HOST'
@@ -26,7 +28,8 @@ export type RateLimiterValkeyConfig = Pick<
*/
export function createSesRateLimiterValkeyPool(
config: RateLimiterValkeyConfig,
- name = 'ses-rate-limiter'
+ name = 'ses-rate-limiter',
+ { optional = false }: { optional?: boolean } = {}
): RedisV2 | null {
if (!config.SES_RATE_LIMITER_VALKEY_HOST) {
logger.info('🪙', `[${name}] no SES Valkey host configured — feature disabled`)
@@ -40,6 +43,25 @@ export function createSesRateLimiterValkeyPool(
const tls = config.SES_RATE_LIMITER_VALKEY_TLS ? {} : undefined
+ if (optional) {
+ // A caller that works without this Valkey must not stop when it is down, or wait on it long.
+ return createRedisV2PoolFromConfig({
+ connection: {
+ url: config.SES_RATE_LIMITER_VALKEY_HOST,
+ options: {
+ port: config.SES_RATE_LIMITER_VALKEY_PORT,
+ password: config.SES_RATE_LIMITER_VALKEY_PASSWORD,
+ tls,
+ commandTimeout: OPTIONAL_VALKEY_TIMEOUT_MS,
+ },
+ name,
+ exitOnRepeatedErrors: false,
+ },
+ poolMinSize: 0,
+ poolMaxSize: config.REDIS_POOL_MAX_SIZE,
+ acquireTimeoutMillis: OPTIONAL_VALKEY_TIMEOUT_MS,
+ })
+ }
return createRedisV2PoolFromConfig({
connection: {
url: config.SES_RATE_LIMITER_VALKEY_HOST,
diff --git a/nodejs/src/server.ts b/nodejs/src/server.ts
index 81391973e04b..a4dac1d7a454 100644
--- a/nodejs/src/server.ts
+++ b/nodejs/src/server.ts
@@ -138,7 +138,8 @@ export class PluginServer implements NodeServer {
// Build typed deps objects for consumers. `emailValidationValkey` and `frequencyCapValkey`
// are null in the shared deps and set only by the cyclotron workers that run the hogflow
// email action (see `withEmailActionValkeys` at their loaders below) — no
- // other consumer touches the SES or frequency cap Valkey.
+ // other consumer touches the SES or frequency cap Valkey, except cdp-api, which reads
+ // the MX-verdict cache to check a transactional email's recipient.
const cdpDeps: CdpConsumerBaseDeps | undefined = needsCdp
? {
postgres: this.postgres!,
@@ -245,7 +246,13 @@ export class PluginServer implements NodeServer {
: null
const api = new CdpApi(
this.config,
- { ...cdpDeps!, idempotencyValkey: createIdempotencyValkeyPool(this.config) },
+ {
+ ...cdpDeps!,
+ idempotencyValkey: createIdempotencyValkeyPool(this.config),
+ emailValidationValkey: createSesRateLimiterValkeyPool(this.config, 'email-mx-validation', {
+ optional: true,
+ }),
+ },
{ hogQueue: kafkaQueue, hogflowQueue: postgresV2Queue },
batchResolverProducer
)
@@ -457,8 +464,9 @@ export class PluginServer implements NodeServer {
/**
* Grants the SES Valkey pool that backs the shared MX-verdict cache to a worker's
* deps. Only the cyclotron workers that run the hogflow email action call this, so
- * the pool is never opened on other CDP consumers or cdp-api — an idle Valkey sized
- * for the SES rate limiter shouldn't hold connections from pods that never validate.
+ * the pool is never opened on other CDP consumers — an idle Valkey sized for the SES
+ * rate limiter shouldn't hold connections from pods that never validate. cdp-api
+ * opens the same MX pool itself, for the transactional email recipient check.
* Null when no SES Valkey host is configured (local dev), in which case
* EmailValidationService degrades to its local cache + DNS.
*