Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(cdp): claim one email cap bucket instead of two (#129163)

Meikel Ratz887d0fb45c4e
diff --git a/nodejs/src/cdp/cdp-services.ts b/nodejs/src/cdp/cdp-services.ts
index f17f1840ab81..a58385794e57 100644
--- a/nodejs/src/cdp/cdp-services.ts
+++ b/nodejs/src/cdp/cdp-services.ts
@@ -183,7 +183,6 @@ export type CdpCoreServicesConfig = Pick<
| 'EMAIL_SUPPRESSION_TRANSIENT_BOUNCE_THRESHOLD'
| 'EMAIL_SUPPRESSION_GLOBAL_HARD_BOUNCE_ENFORCE'
| 'EMAIL_TEAM_SENDING_CAP_MODE'
- | 'EMAIL_TEAM_SENDING_CAP_HOURLY_BY_TIER'
| 'EMAIL_TEAM_SENDING_CAP_DAILY_BY_TIER'
| 'CDP_EMAIL_UTM_LINK_SCANNER_TEAMS'
| 'CDP_GOOGLE_ADWORDS_DEVELOPER_TOKEN'
@@ -489,7 +488,6 @@ export function createCdpCoreServices(
sesTrackedConfigurationSet: config.SES_TRACKED_CONFIGURATION_SET,
sesUntrackedConfigurationSet: config.SES_UNTRACKED_CONFIGURATION_SET,
teamEmailCapMode: parseTeamEmailCapMode(config.EMAIL_TEAM_SENDING_CAP_MODE),
- teamEmailTierHourlyCaps: parseTierCaps(config.EMAIL_TEAM_SENDING_CAP_HOURLY_BY_TIER),
teamEmailTierDailyCaps: parseTierCaps(config.EMAIL_TEAM_SENDING_CAP_DAILY_BY_TIER),
utmLinkScannerTeams: config.CDP_EMAIL_UTM_LINK_SCANNER_TEAMS,
},
diff --git a/nodejs/src/cdp/config.ts b/nodejs/src/cdp/config.ts
index 0a798c47ce61..d6a96fef1d3e 100644
--- a/nodejs/src/cdp/config.ts
+++ b/nodejs/src/cdp/config.ts
@@ -187,15 +187,14 @@ export type CdpConfig = ClickhouseConfig & {
EMAIL_SUPPRESSION_GLOBAL_HARD_BOUNCE_ENFORCE: boolean
// Trust-tiered per-team workflow email caps ("team warming"). A team's tier, stored by the
- // Django side, picks an hourly and a daily cap from these tables. See
- // WORKFLOWS_EMAIL_TIER_* in posthog/settings/web.py, which must stay in step with them.
+ // Django side, picks a daily cap from this table. See WORKFLOWS_EMAIL_TIER_* in
+ // posthog/settings/web.py, which must stay in step with it.
// "off" - the caps are not consulted.
// "shadow" - the caps are evaluated and every send they would delay is logged and counted,
// but nothing is delayed.
// "enforce" - a reached cap reschedules the send.
EMAIL_TEAM_SENDING_CAP_MODE: string
- // Comma-separated caps indexed by tier. Both lists must be the same length.
- EMAIL_TEAM_SENDING_CAP_HOURLY_BY_TIER: string
+ // Comma-separated caps indexed by tier.
EMAIL_TEAM_SENDING_CAP_DAILY_BY_TIER: string
// Comma-separated team ids, or "*", whose UTM link tagging runs the scanner in helpers/utm.ts.
// Every other team runs the previous scanner in helpers/utm-legacy.ts.
@@ -437,7 +436,6 @@ export function getDefaultCdpConfig(): CdpConfig {
// Ships dark: tiers are computed and stored first, then observed in shadow mode, and only
// then enforced. Match the defaults in posthog/settings/web.py.
EMAIL_TEAM_SENDING_CAP_MODE: 'off',
- EMAIL_TEAM_SENDING_CAP_HOURLY_BY_TIER: '50,200,600,2000,6000,20000,60000,200000',
EMAIL_TEAM_SENDING_CAP_DAILY_BY_TIER: '100,1000,3000,10000,30000,100000,300000,1000000',
CDP_EMAIL_UTM_LINK_SCANNER_TEAMS: '',
diff --git a/nodejs/src/cdp/services/managers/team-workflows-config.service.ts b/nodejs/src/cdp/services/managers/team-workflows-config.service.ts
index 59e5ddb039ba..c218209e28dd 100644
--- a/nodejs/src/cdp/services/managers/team-workflows-config.service.ts
+++ b/nodejs/src/cdp/services/managers/team-workflows-config.service.ts
@@ -18,7 +18,7 @@ export type TeamWorkflowsConfig = {
email_tracking_consent_mode: EmailTrackingConsentMode
email_sending_suspended: boolean
ses_tenant_provider_suspended: boolean
- /** Trust tier that picks the team's hourly and daily workflow email caps. */
+ /** Trust tier that picks the team's daily workflow email cap. */
email_sending_tier: number
/** Max marketing messages one person gets per window. Null on either field means no cap. */
marketing_frequency_cap_max_messages: number | null
@@ -103,7 +103,7 @@ export class TeamWorkflowsConfigService {
}
/**
- * Trust tier for the team's workflow email, which the send path turns into an hourly and a
+ * Trust tier for the team's workflow email, which the send path turns into a
* daily cap. Fails open with `null`: a lookup error must let the send through rather than
* throttle a legitimate customer, same stance as `isEmailSendingSuspended`.
*/
diff --git a/nodejs/src/cdp/services/messaging/email.service.test.ts b/nodejs/src/cdp/services/messaging/email.service.test.ts
index 137dfdd05408..c2f14667d74a 100644
--- a/nodejs/src/cdp/services/messaging/email.service.test.ts
+++ b/nodejs/src/cdp/services/messaging/email.service.test.ts
@@ -37,8 +37,9 @@ import {
EmailService,
formatMailbox,
parseAddressList,
+ parseTierCaps,
sanitizeEmailSubject,
- teamEmailCapBuckets,
+ teamEmailCapBucket,
} from './email.service'
import { MailDevAPI } from './helpers/maildev'
import { EmailTrackingCodeSigner } from './helpers/tracking-code'
@@ -917,14 +918,28 @@ describe('EmailService', () => {
})
})
+ describe('parseTierCaps', () => {
+ it.each([
+ ['a whole table', '50,200', [50, 200]],
+ // A decimal cap would make a claim fractional, and the limiter floors its grant
+ // while the caller compares against the unfloored request. The claim then charges
+ // the bucket and reads back as a denial, which starves the team on every retry.
+ ['a decimal cap', '50,2.5', []],
+ ['a zero cap', '50,0', []],
+ ['nonsense', 'fifty', []],
+ ])('%s', (_name, value, expected) => {
+ expect(parseTierCaps(value)).toEqual(expected)
+ })
+ })
+
describe('team sending cap (enforce mode)', () => {
- let claimAllOrNothingPair: jest.Mock
+ let claimAllOrNothing: jest.Mock
let claimUpTo: jest.Mock
let cappedService: EmailService
let cappedSendSpy: jest.SpyInstance
beforeEach(() => {
- claimAllOrNothingPair = jest.fn()
+ claimAllOrNothing = jest.fn()
claimUpTo = jest.fn().mockResolvedValue(0)
const configService = new TeamWorkflowsConfigService(hub.postgres, hub.pubSub)
jest.spyOn(configService, 'getEmailSendingTier').mockResolvedValue(0)
@@ -937,7 +952,6 @@ describe('EmailService', () => {
sesTrackedConfigurationSet: hub.SES_TRACKED_CONFIGURATION_SET,
sesUntrackedConfigurationSet: hub.SES_UNTRACKED_CONFIGURATION_SET,
teamEmailCapMode: 'enforce',
- teamEmailTierHourlyCaps: [100],
teamEmailTierDailyCaps: [200],
},
hub.integrationManager,
@@ -951,7 +965,7 @@ describe('EmailService', () => {
{ report: jest.fn() },
undefined,
null,
- { claimAllOrNothingPair, claimUpTo } as unknown as RateLimiterService
+ { claimAllOrNothing, claimUpTo } as unknown as RateLimiterService
)
cappedSendSpy = jest.spyOn(cappedService.sesV2Client!, 'send') as any
cappedSendSpy.mockResolvedValue({ MessageId: 'test-message-id' })
@@ -963,7 +977,6 @@ describe('EmailService', () => {
const buildEnforcedService = (
name: string,
- hourlyCap: number,
dailyCap: number
): { service: EmailService; sendSpy: jest.SpyInstance; limiter: RateLimiterService } => {
const redis = createRedisV2PoolFromConfig({
@@ -988,7 +1001,6 @@ describe('EmailService', () => {
sesTrackedConfigurationSet: hub.SES_TRACKED_CONFIGURATION_SET,
sesUntrackedConfigurationSet: hub.SES_UNTRACKED_CONFIGURATION_SET,
teamEmailCapMode: 'enforce',
- teamEmailTierHourlyCaps: [hourlyCap],
teamEmailTierDailyCaps: [dailyCap],
},
hub.integrationManager,
@@ -1023,7 +1035,7 @@ describe('EmailService', () => {
2 * 60 * 60 * 1000,
],
])('%s', async (_name, retryAfterMs, reserved, minMs, maxMs) => {
- claimAllOrNothingPair.mockResolvedValue({ granted: false, deniedIndex: 1, retryAfterMs, reserved })
+ claimAllOrNothing.mockResolvedValue({ granted: false, retryAfterMs, reserved })
const before = Date.now()
const result = await cappedService.executeSendEmail(invocation)
@@ -1047,12 +1059,7 @@ describe('EmailService', () => {
// A Valkey fault denies with no horizon. Parking on the daily cap's pacing
// interval would hold the email long after the limiter recovered.
jest.spyOn(Math, 'random').mockReturnValue(0)
- claimAllOrNothingPair.mockResolvedValue({
- granted: false,
- deniedIndex: null,
- retryAfterMs: null,
- reserved: false,
- })
+ claimAllOrNothing.mockResolvedValue({ granted: false, retryAfterMs: null, reserved: false })
const before = Date.now()
const result = await cappedService.executeSendEmail(invocation)
@@ -1062,19 +1069,14 @@ describe('EmailService', () => {
const scheduledMs = result.invocation.queueScheduledAt!.toMillis()
expect(scheduledMs).toBeGreaterThanOrEqual(before + 5 * 60 * 1000)
expect(scheduledMs).toBeLessThan(before + 5 * 60 * 1000 + 5000)
- // No bucket denied, so the customer must not be told they hit a cap they never hit.
+ // No cap was reached, so the customer must not be told they hit one.
const messages = result.logs.map((log) => log.message).join(' ')
expect(messages).not.toContain('reached its email sending limit')
expect(messages).toContain("Could not check this project's email sending limit")
})
- it('sends a transactional email past the cap and charges the buckets anyway', async () => {
- claimAllOrNothingPair.mockResolvedValue({
- granted: false,
- deniedIndex: 1,
- retryAfterMs: 30 * 60 * 1000,
- reserved: true,
- })
+ it('sends a transactional email past the cap and charges the bucket anyway', async () => {
+ claimAllOrNothing.mockResolvedValue({ granted: false, retryAfterMs: 30 * 60 * 1000, reserved: true })
invocation.hogFunction.metadata = { message_category_type: 'transactional' }
const result = await cappedService.executeSendEmail(invocation)
@@ -1082,29 +1084,20 @@ describe('EmailService', () => {
expect(result.finished).toBe(true)
expect(cappedSendSpy).toHaveBeenCalledTimes(1)
expect(result.invocation.queueScheduledAt).toBeUndefined()
- expect(claimAllOrNothingPair).not.toHaveBeenCalled()
- expect(claimUpTo).toHaveBeenCalledTimes(2)
+ expect(claimAllOrNothing).not.toHaveBeenCalled()
+ expect(claimUpTo).toHaveBeenCalledTimes(1)
// No cap was applied, so the customer must not be told one was.
expect(result.logs.map((log) => log.message).join(' ')).not.toContain('reached its email sending limit')
})
it('sends when the claim is granted', async () => {
- claimAllOrNothingPair.mockResolvedValue({
- granted: true,
- deniedIndex: null,
- retryAfterMs: null,
- reserved: false,
- })
+ claimAllOrNothing.mockResolvedValue({ granted: true, retryAfterMs: null, reserved: false })
const result = await cappedService.executeSendEmail(invocation)
// The reservation horizon must reach the limiter, or denials fall back to
// shared-horizon wakes and a denied backlog re-herds.
- expect(claimAllOrNothingPair).toHaveBeenCalledWith(
- expect.anything(),
- expect.any(Number),
- 60 * 60 * 1000
- )
+ expect(claimAllOrNothing).toHaveBeenCalledWith(expect.anything(), 60 * 60 * 1000)
expect(result.finished).toBe(true)
expect(cappedSendSpy).toHaveBeenCalled()
})
@@ -1113,23 +1106,17 @@ describe('EmailService', () => {
// their own future slots instead of all waking together, and another team's send
// goes straight out. Runs against the real limiter.
it('spreads a capped team over distinct slots while another team keeps sending', async () => {
- const hourlyCap = 360
const dailyCap = 8640
const {
service: enforcedService,
sendSpy: enforcedSendSpy,
limiter,
- } = buildEnforcedService('team-email-cap-test', hourlyCap, dailyCap)
+ } = buildEnforcedService('team-email-cap-test', dailyCap)
- // Drain the capped team's hourly bucket so every send below is a denial. The
+ // Drain the capped team's bucket so every send below is a denial. The
// 0.1 tokens/s refill makes the slot spacing 10s, far above test wall-clock.
- const buckets = teamEmailCapBuckets(team.id, hourlyCap, dailyCap)
- await limiter.claimUpTo({
- key: buckets[0].key,
- requested: hourlyCap,
- capacity: buckets[0].capacity,
- refillPerSecond: buckets[0].refillPerSecond,
- })
+ const bucket = teamEmailCapBucket(team.id, dailyCap)
+ await limiter.claimUpTo({ ...bucket, requested: dailyCap })
const parkedAt: number[] = []
for (let i = 0; i < 4; i++) {
@@ -1169,16 +1156,15 @@ describe('EmailService', () => {
expect(enforcedSendSpy).toHaveBeenCalledTimes(1)
})
- // Enforce mode has only ever issued claimAllOrNothingPair against real Valkey, so
- // transactional through the single-key claimUpTo has never run there. Real limiter.
- it('charges a transactional send against the real buckets without delaying it', async () => {
- const hourlyCap = 60
+ // Enforce mode has only ever issued claimAllOrNothing against real Valkey, so
+ // transactional through claimUpTo has never run there. Real limiter.
+ it('charges a transactional send against the real bucket without delaying it', async () => {
const dailyCap = 50
const {
service: enforcedService,
sendSpy: enforcedSendSpy,
limiter,
- } = buildEnforcedService('team-email-cap-transactional-test', hourlyCap, dailyCap)
+ } = buildEnforcedService('team-email-cap-transactional-test', dailyCap)
// A team of its own, so the buckets start cold whatever else touched Valkey.
const capTeam = (await createTestTeamFixture(hub.postgres)).team
@@ -1205,34 +1191,25 @@ describe('EmailService', () => {
return next
}
- // Empty the hourly bucket only, so the daily one can show the charge.
- const buckets = teamEmailCapBuckets(capTeam.id, hourlyCap, dailyCap)
- await limiter.claimUpTo({
- key: buckets[0].key,
- requested: hourlyCap,
- capacity: buckets[0].capacity,
- refillPerSecond: buckets[0].refillPerSecond,
- })
+ // Leave one token, so the next send can show the charge before the cap closes.
+ const bucket = teamEmailCapBucket(capTeam.id, dailyCap)
+ await limiter.claimUpTo({ ...bucket, requested: dailyCap - 1 })
+
+ // The charge: this send takes the last token rather than riding past the bucket.
+ const charged = await enforcedService.executeSendEmail(sendFor('charged', true))
+ expect(charged.finished).toBe(true)
+ expect(enforcedSendSpy).toHaveBeenCalledTimes(1)
- // Control: the cap really is reached, so a marketing send parks.
+ // Control: the cap really is closed now, so a marketing send parks.
const marketing = await enforcedService.executeSendEmail(sendFor('marketing', false))
expect(marketing.finished).toBe(false)
- expect(enforcedSendSpy).not.toHaveBeenCalled()
+ expect(enforcedSendSpy).toHaveBeenCalledTimes(1)
+ // The exemption: a transactional send still goes out against the closed cap.
const transactional = await enforcedService.executeSendEmail(sendFor('transactional', true))
expect(transactional.finished).toBe(true)
expect(transactional.invocation.queueScheduledAt).toBeUndefined()
- expect(enforcedSendSpy).toHaveBeenCalledTimes(1)
-
- // A range, not an exact count, because the daily bucket refills as the test runs.
- const dailyLeft = await limiter.claimUpTo({
- key: buckets[1].key,
- requested: dailyCap,
- capacity: buckets[1].capacity,
- refillPerSecond: buckets[1].refillPerSecond,
- })
- expect(dailyLeft).toBeLessThan(dailyCap)
- expect(dailyLeft).toBeGreaterThanOrEqual(dailyCap - 2)
+ expect(enforcedSendSpy).toHaveBeenCalledTimes(2)
})
})
})
diff --git a/nodejs/src/cdp/services/messaging/email.service.ts b/nodejs/src/cdp/services/messaging/email.service.ts
index 9765aead3ae0..835f66774a85 100644
--- a/nodejs/src/cdp/services/messaging/email.service.ts
+++ b/nodejs/src/cdp/services/messaging/email.service.ts
@@ -301,33 +301,31 @@ export function parseTeamEmailCapMode(value: string | undefined): TeamEmailCapMo
return mode === 'shadow' || mode === 'enforce' ? mode : 'off'
}
-/** Parses a comma-separated cap table. Any unusable entry drops the whole table, which turns the
- * cap off rather than applying a wrong number to a paying customer. */
+/** Parses a comma-separated cap table. Any unusable entry drops the whole table, and the send path
+ * then applies DEFAULT_TEAM_EMAIL_TIER_DAILY_CAPS. A non-integer cap is unusable because the
+ * all-or-nothing claim needs a whole-number request. */
export function parseTierCaps(value: string | undefined): number[] {
const parts = (value ?? '')
.split(',')
.map((part) => part.trim())
.filter((part) => part.length > 0)
const caps = parts.map((part) => Number(part))
- if (caps.length === 0 || caps.some((cap) => !Number.isFinite(cap) || cap <= 0)) {
+ if (caps.length === 0 || caps.some((cap) => !Number.isInteger(cap) || cap <= 0)) {
return []
}
return caps
}
-// Mirrors WORKFLOWS_EMAIL_TIER_HOURLY_CAPS / _DAILY_CAPS in posthog/settings/web.py. Both sides
-// read the same tier index, so the two tables must stay in step.
-const DEFAULT_TEAM_EMAIL_TIER_HOURLY_CAPS = [50, 200, 600, 2000, 6000, 20000, 60000, 200000]
+// Mirrors WORKFLOWS_EMAIL_TIER_DAILY_CAPS in posthog/settings/web.py. Both sides read the same
+// tier index, so the two tables must stay in step.
const DEFAULT_TEAM_EMAIL_TIER_DAILY_CAPS = [100, 1000, 3000, 10000, 30000, 100000, 300000, 1000000]
-// Comfortably longer than the window each bucket paces, so an idle bucket is never reset to full
+// Comfortably longer than the window the bucket paces, so an idle bucket is never reset to full
// by expiry before it would have refilled on its own. Without that, a team that pauses for a day
// would get a fresh daily allowance instead of waiting for the refill.
-const TEAM_EMAIL_HOUR_BUCKET_TTL_SECONDS = 6 * 60 * 60
const TEAM_EMAIL_DAY_BUCKET_TTL_SECONDS = 3 * 24 * 60 * 60
type TeamEmailCapBucket = {
- name: 'hour' | 'day'
key: string
capacity: number
refillPerSecond: number
@@ -336,28 +334,14 @@ type TeamEmailCapBucket = {
label: string
}
-// The `{teamId}` hash tag puts both bucket keys in the same cluster slot. Enforce mode claims
-// both keys in one Lua call (claimAllOrNothingPair), and clustered Valkey rejects a multi-key
-// call whose keys hash to different slots with a CROSSSLOT error.
-export function teamEmailCapBuckets(teamId: number, hourlyCap: number, dailyCap: number): TeamEmailCapBucket[] {
- return [
- {
- name: 'hour',
- key: `@posthog/team-email-rate-hour/{${teamId}}`,
- capacity: hourlyCap,
- refillPerSecond: hourlyCap / 3600,
- ttlSeconds: TEAM_EMAIL_HOUR_BUCKET_TTL_SECONDS,
- label: `${hourlyCap.toLocaleString('en-US')} emails per hour`,
- },
- {
- name: 'day',
- key: `@posthog/team-email-rate-day/{${teamId}}`,
- capacity: dailyCap,
- refillPerSecond: dailyCap / 86400,
- ttlSeconds: TEAM_EMAIL_DAY_BUCKET_TTL_SECONDS,
- label: `${dailyCap.toLocaleString('en-US')} emails per day`,
- },
- ]
+export function teamEmailCapBucket(teamId: number, dailyCap: number): TeamEmailCapBucket {
+ return {
+ key: `@posthog/team-email-rate-day/{${teamId}}`,
+ capacity: dailyCap,
+ refillPerSecond: dailyCap / 86400,
+ ttlSeconds: TEAM_EMAIL_DAY_BUCKET_TTL_SECONDS,
+ label: `${dailyCap.toLocaleString('en-US')} emails per day`,
+ }
}
export interface EmailServiceConfig {
@@ -373,7 +357,6 @@ export interface EmailServiceConfig {
// Trust-tiered per-team sending caps. Optional and defaulting to off, so every send path that
// builds an EmailService without them keeps its pre-cap behavior.
teamEmailCapMode?: TeamEmailCapMode
- teamEmailTierHourlyCaps?: number[]
teamEmailTierDailyCaps?: number[]
utmLinkScannerTeams?: string
}
@@ -951,25 +934,25 @@ export class EmailService {
/**
* Claims one send (as `recipients` tokens, one per delivered copy) against the team's
- * trust-tier buckets.
+ * trust-tier daily bucket.
*
- * Returns the delay to reschedule with when a cap is reached, or null when the send may go out.
- * Two buckets, not one: the daily cap bounds how much damage a team can do to the shared SES
- * account's reputation, and the hourly cap forces that volume to spread out so complaint
- * feedback (which lags by hours) arrives while the team's total volume is still small.
+ * Returns the delay to reschedule with when the cap is reached, or null when the send may go
+ * out. The cap bounds how much damage a team can do to the shared SES account's reputation
+ * in a day. It does not spread that volume out: a full bucket lets the team send its whole
+ * daily cap as one burst, and only volume past that waits for the refill.
*
* Failure stances differ on purpose. A failed tier lookup lets the send through, because a
- * config blip must never throttle a legitimate customer. A failed bucket claim returns 0 from
- * the limiter, which delays the send, because we must not send when we cannot account for it.
- * That second case returns a null `label`: the send waits, but no cap was reached, so the
- * caller must not tell the customer it hit one.
+ * config blip must never throttle a legitimate customer. A failed bucket claim delays the
+ * send, because we must not send when we cannot account for it. That second case returns a
+ * null `label`: the send waits, but no cap was reached, so the caller must not tell the
+ * customer it hit one.
*
* Transactional sends are charged but never delayed, because a password reset that arrives
* two hours late has already failed the person waiting on it. The charge does not bound the
- * exemption: claimUpTo records no debt, so once a bucket empties further transactional sends
- * cost nothing. No other control bounds that volume per team either. The SES limiter paces
- * the whole account, so what is left is suspension and the daily tier job, which demotes a
- * team on its complaint and bounce rates after the fact.
+ * exemption: claimUpTo records no debt, so once the bucket empties further transactional
+ * sends cost nothing. No other control bounds that volume per team either. The SES limiter
+ * paces the whole account, so what is left is suspension and the daily tier job, which
+ * demotes a team on its complaint and bounce rates after the fact.
*/
private async claimTeamSendingBudget(
invocation: CyclotronJobInvocationHogFunction,
@@ -986,78 +969,44 @@ export class EmailService {
return null
}
- const hourlyCaps = this.sesConfig.teamEmailTierHourlyCaps?.length
- ? this.sesConfig.teamEmailTierHourlyCaps
- : DEFAULT_TEAM_EMAIL_TIER_HOURLY_CAPS
const dailyCaps = this.sesConfig.teamEmailTierDailyCaps?.length
? this.sesConfig.teamEmailTierDailyCaps
: DEFAULT_TEAM_EMAIL_TIER_DAILY_CAPS
- const topTier = Math.min(hourlyCaps.length, dailyCaps.length) - 1
- if (topTier < 0) {
- return null
- }
- const tier = Math.min(Math.max(teamTier, 0), topTier)
+ const tier = Math.min(Math.max(teamTier, 0), dailyCaps.length - 1)
- const buckets = teamEmailCapBuckets(invocation.teamId, hourlyCaps[tier], dailyCaps[tier])
- // Clamped to the smaller capacity: a send with more copies than a whole allowance could
- // never be granted and would reschedule forever. It charges the entire bucket instead, so
- // it still waits for full capacity and pays everything there is.
- const requested = Math.min(Math.max(1, recipients), buckets[0].capacity, buckets[1].capacity)
+ const bucket = teamEmailCapBucket(invocation.teamId, dailyCaps[tier])
+ // Clamped to the capacity: a send with more copies than a whole allowance could never be
+ // granted and would reschedule forever. It charges the entire bucket instead, so it still
+ // waits for full capacity and pays everything there is.
+ const requested = Math.min(Math.max(1, recipients), bucket.capacity)
const isTransactional = invocation.hogFunction?.metadata?.message_category_type === 'transactional'
if (mode === 'enforce' && !isTransactional) {
- // Both buckets in one atomic claim, granted whole or not at all. A denial consumes
- // nothing, so a rescheduled multi-recipient send cannot burn the partial refill on
- // every retry and starve the team's other emails while never sending itself.
- const claim = await this.teamEmailRateLimiter.claimAllOrNothingPair(
- [buckets[0], buckets[1]],
- requested,
- CAP_RETRY_MAX_MS
- )
+ const claim = await this.teamEmailRateLimiter.claimAllOrNothing({ ...bucket, requested }, CAP_RETRY_MAX_MS)
if (claim.granted) {
return null
}
- // `deniedIndex` is null when the limiter itself failed, not when a bucket was full. The
- // send still waits, because we must not send what we cannot account for, but no cap was
- // reached, so a null label tells the caller it has no cap to name to the customer.
- const denied = claim.deniedIndex === null ? null : buckets[claim.deniedIndex]
- // The daily bucket paces a limiter failure because it is the slower of the two, so the
- // retry backs off at the safer rate while the limiter is unreachable.
- const pacing = denied ?? buckets[1]
+ const capReached = claim.retryAfterMs !== null
teamEmailCapDelayedTotal.inc({
tier: String(tier),
- bucket: denied?.name ?? 'limiter_unavailable',
+ bucket: capReached ? 'day' : 'limiter_unavailable',
mode,
})
- const retryDelayMs = pickReservedRetryDelayMs(claim.retryAfterMs, pacing.refillPerSecond, claim.reserved)
+ const retryDelayMs = pickReservedRetryDelayMs(claim.retryAfterMs, bucket.refillPerSecond, claim.reserved)
emailReservedParkMs.labels('team-email').observe(retryDelayMs)
return {
retryDelayMs,
- label: denied?.label ?? null,
+ label: capReached ? bucket.label : null,
}
}
// Shadow mode, and transactional sends in any mode: the send goes out regardless, so
- // drain each bucket and log the cap. Separate labels keep shadow a forecast of enforce.
+ // drain the bucket and log the cap. Separate labels keep shadow a forecast of enforce.
const observedMode = isTransactional ? 'transactional' : mode
- let firstDenial: TeamEmailCapBucket | null = null
- for (const bucket of buckets) {
- const granted = await this.teamEmailRateLimiter.claimUpTo({
- key: bucket.key,
- requested,
- capacity: bucket.capacity,
- refillPerSecond: bucket.refillPerSecond,
- ttlSeconds: bucket.ttlSeconds,
- })
- if (granted >= requested) {
- continue
- }
- teamEmailCapDelayedTotal.inc({ tier: String(tier), bucket: bucket.name, mode: observedMode })
- firstDenial = firstDenial ?? bucket
- }
-
- if (firstDenial) {
+ const granted = await this.teamEmailRateLimiter.claimUpTo({ ...bucket, requested })
+ if (granted < requested) {
+ teamEmailCapDelayedTotal.inc({ tier: String(tier), bucket: 'day', mode: observedMode })
logger.info(
'📧',
isTransactional
@@ -1066,8 +1015,8 @@ export class EmailService {
{
teamId: invocation.teamId,
tier,
- bucket: firstDenial.name,
- cap: firstDenial.label,
+ bucket: 'day',
+ cap: bucket.label,
}
)
}
diff --git a/nodejs/src/cdp/services/rate-limiter/rate-limiter.service.test.ts b/nodejs/src/cdp/services/rate-limiter/rate-limiter.service.test.ts
index 490f0fee49ef..2b85bd4dab30 100644
--- a/nodejs/src/cdp/services/rate-limiter/rate-limiter.service.test.ts
+++ b/nodejs/src/cdp/services/rate-limiter/rate-limiter.service.test.ts
@@ -197,66 +197,38 @@ describe('RateLimiterService', () => {
})
})
- describe('claimAllOrNothingPair', () => {
- const KEY_A = `${KEY}/pair-a`
- const KEY_B = `${KEY}/pair-b`
-
- it('grants from both buckets when both can cover the request', async () => {
- const claim = await limiter.claimAllOrNothingPair(
- [
- { key: KEY_A, capacity: 50, refillPerSecond: 0 },
- { key: KEY_B, capacity: 100, refillPerSecond: 0 },
- ],
- 30
+ describe('claimAllOrNothing', () => {
+ const AON_KEY = `${KEY}/all-or-nothing`
+
+ it('grants and charges the pool when the bucket covers the request', async () => {
+ const claim = await limiter.claimAllOrNothing(
+ { key: AON_KEY, requested: 30, capacity: 50, refillPerSecond: 0 },
+ 600_000
)
- expect(claim).toEqual({ granted: true, deniedIndex: null, retryAfterMs: null, reserved: false })
- // Both pools were charged: the remainder is all that is left to claim.
- expect(await limiter.claimUpTo({ key: KEY_A, requested: 50, capacity: 50, refillPerSecond: 0 })).toBe(20)
- expect(await limiter.claimUpTo({ key: KEY_B, requested: 100, capacity: 100, refillPerSecond: 0 })).toBe(70)
+ expect(claim).toEqual({ granted: true, retryAfterMs: null, reserved: false })
+ // The remainder is all that is left to claim.
+ expect(await limiter.claimUpTo({ key: AON_KEY, requested: 50, capacity: 50, refillPerSecond: 0 })).toBe(20)
})
- it('consumes nothing on denial, so retries cannot starve the buckets', async () => {
- // The second bucket cannot cover the request, so the claim must leave BOTH pools
- // untouched. Without that, a rescheduled multi-recipient send would burn the first
- // bucket's refill on every retry while never sending.
- const claim = await limiter.claimAllOrNothingPair(
- [
- { key: KEY_A, capacity: 50, refillPerSecond: 0 },
- { key: KEY_B, capacity: 10, refillPerSecond: 0 },
- ],
- 30
- )
+ it('consumes nothing on denial, so retries cannot starve the bucket', async () => {
+ // The bucket cannot cover the request, so the claim must leave the pool untouched.
+ // Without that, a rescheduled multi-recipient send would burn the refill on every
+ // retry while never sending.
+ const req = { key: `${AON_KEY}/denied`, requested: 30, capacity: 10, refillPerSecond: 0 }
+ const claim = await limiter.claimAllOrNothing(req, 600_000)
// refillPerSecond 0 means the missing tokens never accrue, so no horizon is reported.
- expect(claim).toEqual({ granted: false, deniedIndex: 1, retryAfterMs: null, reserved: false })
- expect(await limiter.claimUpTo({ key: KEY_A, requested: 50, capacity: 50, refillPerSecond: 0 })).toBe(50)
- expect(await limiter.claimUpTo({ key: KEY_B, requested: 10, capacity: 10, refillPerSecond: 0 })).toBe(10)
+ expect(claim).toEqual({ granted: false, retryAfterMs: null, reserved: false })
+ expect(await limiter.claimUpTo({ ...req, requested: 10 })).toBe(10)
})
it('reports on denial how long until the missing tokens accrue', async () => {
- // Cold start: the denying bucket holds exactly its capacity of 10, so the request
- // for 30 is missing 20 tokens. At 2 tokens/s that is 10 seconds.
- const claim = await limiter.claimAllOrNothingPair(
- [
- { key: KEY_A, capacity: 50, refillPerSecond: 0 },
- { key: KEY_B, capacity: 10, refillPerSecond: 2 },
- ],
- 30
+ // Cold start: the bucket holds exactly its capacity of 10, so the request for 30 is
+ // missing 20 tokens. At 2 tokens/s that is 10 seconds.
+ const claim = await limiter.claimAllOrNothing(
+ { key: `${AON_KEY}/horizon`, requested: 30, capacity: 10, refillPerSecond: 2 },
+ 600_000
)
- expect(claim).toEqual({ granted: false, deniedIndex: 1, retryAfterMs: 10_000, reserved: false })
- })
-
- it('reports the slower bucket when both are short', async () => {
- // Both buckets are 20 tokens short of the request. The first one covers that in 10
- // seconds, the second needs 40. Reporting the first would wake the caller while the
- // second still cannot grant, costing a whole extra dequeue and reschedule.
- const claim = await limiter.claimAllOrNothingPair(
- [
- { key: KEY_A, capacity: 10, refillPerSecond: 2 },
- { key: KEY_B, capacity: 10, refillPerSecond: 0.5 },
- ],
- 30
- )
- expect(claim).toEqual({ granted: false, deniedIndex: 0, retryAfterMs: 40_000, reserved: false })
+ expect(claim).toEqual({ granted: false, retryAfterMs: 10_000, reserved: true })
})
it('fails closed when the Lua call throws', async () => {
@@ -265,85 +237,12 @@ describe('RateLimiterService', () => {
usePipeline: jest.fn(),
} as unknown as RedisV2
const brokenLimiter = new RateLimiterService(brokenValkey, { name: 'broken-limiter' })
- const claim = await brokenLimiter.claimAllOrNothingPair(
- [
- { key: KEY_A, capacity: 50, refillPerSecond: 0 },
- { key: KEY_B, capacity: 10, refillPerSecond: 0 },
- ],
- 5
+ // A null horizon is how the caller tells an unreachable limiter from a reached cap.
+ const claim = await brokenLimiter.claimAllOrNothing(
+ { key: AON_KEY, requested: 5, capacity: 50, refillPerSecond: 1 },
+ 600_000
)
- expect(claim).toEqual({ granted: false, deniedIndex: null, retryAfterMs: null, reserved: false })
- })
-
- it('hands successive reserved denials distinct, later slots, paced by the slower bucket', async () => {
- // Both buckets are short, so the slower one (0.5/s) sets the pace. The first
- // caller waits 40s (its missing tokens). The next one gets the slot after
- // that, a full 60s later. Nobody wakes at the same time.
- const buckets: [
- { key: string; capacity: number; refillPerSecond: number },
- { key: string; capacity: number; refillPerSecond: number },
- ] = [
- { key: `${KEY_A}/resv`, capacity: 10, refillPerSecond: 2 },
- { key: `${KEY_B}/resv`, capacity: 10, refillPerSecond: 0.5 },
- ]
- const first = await limiter.claimAllOrNothingPair(buckets, 30, 600_000)
- const second = await limiter.claimAllOrNothingPair(buckets, 30, 600_000)
-
- expect(first.granted).toBe(false)
- expect(first.reserved).toBe(true)
- // First in line pays only the 40s shortfall, not a full 60s slot.
- expect(first.retryAfterMs).toBe(40_000)
- expect(second.reserved).toBe(true)
- expect(second.retryAfterMs!).toBeGreaterThan(first.retryAfterMs!)
- expect(second.retryAfterMs!).toBeLessThanOrEqual(100_000)
- })
-
- it('stops advancing the reservation cursor at the horizon', async () => {
- const buckets: [
- { key: string; capacity: number; refillPerSecond: number },
- { key: string; capacity: number; refillPerSecond: number },
- ] = [
- { key: `${KEY_A}/resv-cap`, capacity: 50, refillPerSecond: 0 },
- { key: `${KEY_B}/resv-cap`, capacity: 10, refillPerSecond: 2 },
- ]
- // The first slot (10s) fits inside the 20s horizon. The next one would not,
- // so everyone after the first just gets "come back in 20s" with no slot.
- const first = await limiter.claimAllOrNothingPair(buckets, 30, 20_000)
- const second = await limiter.claimAllOrNothingPair(buckets, 30, 20_000)
- const third = await limiter.claimAllOrNothingPair(buckets, 30, 20_000)
-
- expect(first.retryAfterMs).toBe(10_000)
- expect(first.reserved).toBe(true)
- expect(second.retryAfterMs).toBe(20_000)
- expect(second.reserved).toBe(false)
- expect(third.retryAfterMs).toBe(20_000)
- expect(third.reserved).toBe(false)
- })
-
- it('keeps one reservation line when the slower bucket changes', async () => {
- const buckets: [
- { key: string; capacity: number; refillPerSecond: number },
- { key: string; capacity: number; refillPerSecond: number },
- ] = [
- { key: `${KEY_A}/line`, capacity: 10, refillPerSecond: 2 },
- { key: `${KEY_B}/line`, capacity: 100, refillPerSecond: 0.5 },
- ]
- // The second bucket still covers the request, so the first one (10s
- // shortfall, 15s spacing) sets the pace for the first two denials.
- const first = await limiter.claimAllOrNothingPair(buckets, 30, 600_000)
- const second = await limiter.claimAllOrNothingPair(buckets, 30, 600_000)
-
- // Now drain the second bucket so it becomes the slower one. The line has
- // to continue behind the parked sends. A cursor kept only on the bucket
- // that is currently slower would start a fresh line here and hand this
- // send a slot at ~16s, ahead of the send already parked at ~25s.
- await limiter.claimUpTo({ key: buckets[1].key, requested: 78, capacity: 100, refillPerSecond: 0.5 })
- const third = await limiter.claimAllOrNothingPair(buckets, 30, 600_000)
-
- expect(first.reserved).toBe(true)
- expect(second.retryAfterMs!).toBeGreaterThan(first.retryAfterMs!)
- expect(third.reserved).toBe(true)
- expect(third.retryAfterMs!).toBeGreaterThan(second.retryAfterMs!)
+ expect(claim).toEqual({ granted: false, retryAfterMs: null, reserved: false })
})
})
diff --git a/nodejs/src/cdp/services/rate-limiter/rate-limiter.service.ts b/nodejs/src/cdp/services/rate-limiter/rate-limiter.service.ts
index 1d4b178a0574..c8925ee9a41d 100644
--- a/nodejs/src/cdp/services/rate-limiter/rate-limiter.service.ts
+++ b/nodejs/src/cdp/services/rate-limiter/rate-limiter.service.ts
@@ -41,6 +41,10 @@ const claimLatency = new Histogram({
// ARGV[4] = TTL seconds — bucket auto-expires when idle so cold-start
// gives full capacity rather than a stale negative pool.
// ARGV[5] = reserve-on-deny horizon in ms; 0 disables reservation.
+// ARGV[6] = 1 to make the claim all-or-nothing: a bucket that cannot cover
+// the whole request grants nothing and consumes nothing, so a
+// caller retrying a multi-token claim cannot drain the bucket
+// while never succeeding.
//
// `now` is sourced via `redis.call('TIME')` so all pods share a single
// monotonic clock — NTP drift between workers can't over- or under-refill
@@ -63,6 +67,7 @@ local capacity = tonumber(ARGV[2])
local refillPerSecond = tonumber(ARGV[3])
local ttlSeconds = tonumber(ARGV[4])
local reserveOnDenyMaxMs = tonumber(ARGV[5])
+local allOrNothing = tonumber(ARGV[6])
local time = redis.call('TIME')
-- TIME returns {seconds, microseconds}; flatten to epoch ms.
@@ -92,12 +97,14 @@ else
end
local rawGranted = math.min(available, requested)
-if rawGranted < 0 then
+if rawGranted < 0 or (allOrNothing == 1 and available < requested) then
rawGranted = 0
end
--- Floor to int — the grant flows into a SQL LIMIT, which must be integer.
--- The fractional residual stays in the pool so partial refills accumulate
--- across calls instead of being silently dropped.
+-- Floor to int — the grant flows into a SQL LIMIT, which must be integer, and
+-- Redis truncates a Lua float in the reply regardless. The fractional residual
+-- stays in the pool so partial refills accumulate across calls instead of being
+-- silently dropped. An all-or-nothing caller must therefore request a whole
+-- number, or the floored grant reads back as a denial after the bucket paid.
local granted = math.floor(rawGranted)
local tokensAfter = available - granted
@@ -129,126 +136,6 @@ redis.call('hset', key, 'resv', slotAt)
return {0, math.ceil(slotAt - now), 1}
`
-// Atomic all-or-nothing claim across two buckets. Same refill math as CLAIM_UP_TO_LUA per bucket,
-// but the claim succeeds only when BOTH buckets can cover the full request, and a denial writes
-// nothing at all — no partial consumption, no timestamp refresh. Without that, a caller retrying a
-// multi-token claim would burn each bucket's partial refill on every attempt and could keep a
-// shared bucket empty without ever succeeding.
-// KEYS[1..2] = the two bucket hash keys
-// ARGV[1] = requested tokens (integer)
-// ARGV[2..4] = capacity, refill/sec, TTL seconds for KEYS[1]
-// ARGV[5..7] = capacity, refill/sec, TTL seconds for KEYS[2]
-// ARGV[8] = reserve-on-deny horizon in ms; 0 disables reservation.
-// Returns {1, 0, 0, 0} when granted. On denial: {0, i, retryAfterMs, reserved},
-// where i is the first bucket (1-based) that came up short. With ARGV[8] = 0,
-// retryAfterMs is only how long until BOTH buckets have their missing tokens back
-// (the slower one decides). Every denied caller gets that same answer, so they all
-// wake together. With ARGV[8] > 0 the denial also books the caller a slot on the
-// slowest short bucket, the same ticket-at-a-counter idea as the claim-up-to script
-// above: the first denied caller waits only for the missing tokens, everyone after
-// gets the next slot, one interval later. Slots never go out past ARGV[8]; past
-// that the caller gets ARGV[8] back with reserved=0 and asks again when it wakes.
-// A slot is a place in line, not a promise: the wake still has to claim. A bucket
-// that never refills has no horizon, and then the denial reports none and books
-// nothing.
-const CLAIM_ALL_OR_NOTHING_PAIR_LUA = `
-local time = redis.call('TIME')
-local now = tonumber(time[1]) * 1000 + math.floor(tonumber(time[2]) / 1000)
-local requested = tonumber(ARGV[1])
-local reserveOnDenyMaxMs = tonumber(ARGV[8])
-
-local available = {}
-local deniedIndex = 0
-local retryAfterMs = 0
-local horizonKnown = true
-local slowRefill = 0
-for i = 1, 2 do
- local capacity = tonumber(ARGV[(i - 1) * 3 + 2])
- local refillPerSecond = tonumber(ARGV[(i - 1) * 3 + 3])
- local existing = redis.call('hmget', KEYS[i], 'ts', 'pool')
- local avail
- if existing[1] == false then
- avail = capacity
- else
- local elapsedMs = now - tonumber(existing[1])
- if elapsedMs < 0 then
- elapsedMs = 0
- end
- local currentTokens = capacity
- if existing[2] ~= false then
- currentTokens = tonumber(existing[2])
- end
- avail = math.min(capacity, currentTokens + (elapsedMs / 1000.0) * refillPerSecond)
- end
- if avail < requested then
- if deniedIndex == 0 then
- deniedIndex = i
- end
- if refillPerSecond > 0 then
- local bucketRetryMs = math.ceil(((requested - avail) / refillPerSecond) * 1000)
- if bucketRetryMs > retryAfterMs then
- retryAfterMs = bucketRetryMs
- slowRefill = refillPerSecond
- end
- else
- horizonKnown = false
- end
- end
- available[i] = avail
-end
-
-if deniedIndex > 0 then
- if not horizonKnown then
- return {0, deniedIndex, 0, 0}
- end
- if reserveOnDenyMaxMs <= 0 then
- return {0, deniedIndex, retryAfterMs, 0}
- end
- -- First in line: wait only for the pair's shortfall (that is retryAfterMs).
- -- Behind someone: take the slot after theirs, one interval of the slowest
- -- bucket later. Same idea as the claim-up-to script above.
- -- There is ONE line for the pair, kept on both keys: the base is the later of
- -- the two cursors and the new slot is written back to both. Keeping the line
- -- on only the currently slower bucket splits it when the slower bucket
- -- changes (hourly runs dry first, then daily), and a fresh line lets new
- -- arrivals cut ahead of sends that are already parked.
- local rawResvA = redis.call('hget', KEYS[1], 'resv')
- local rawResvB = redis.call('hget', KEYS[2], 'resv')
- local base = now
- if rawResvA ~= false and tonumber(rawResvA) > base then
- base = tonumber(rawResvA)
- end
- if rawResvB ~= false and tonumber(rawResvB) > base then
- base = tonumber(rawResvB)
- end
- local slotAt
- if base > now then
- slotAt = base + (requested / slowRefill) * 1000
- else
- slotAt = now + retryAfterMs
- end
- if slotAt - now < retryAfterMs then
- slotAt = now + retryAfterMs
- end
- if slotAt - now > reserveOnDenyMaxMs then
- return {0, deniedIndex, reserveOnDenyMaxMs, 0}
- end
- redis.call('hset', KEYS[1], 'resv', slotAt)
- redis.call('expire', KEYS[1], tonumber(ARGV[4]))
- redis.call('hset', KEYS[2], 'resv', slotAt)
- redis.call('expire', KEYS[2], tonumber(ARGV[7]))
- return {0, deniedIndex, math.ceil(slotAt - now), 1}
-end
-
-for i = 1, 2 do
- local ttlSeconds = tonumber(ARGV[(i - 1) * 3 + 4])
- redis.call('hset', KEYS[i], 'ts', now, 'pool', available[i] - requested)
- redis.call('expire', KEYS[i], ttlSeconds)
-end
-
-return {1, 0, 0, 0}
-`
-
export interface RateLimiterConfig {
/** Logical name for metrics/logging only (e.g. 'ses'). */
name: string
@@ -299,6 +186,24 @@ export class RateLimiterService {
return (await this.evalClaim(req, 0)).granted
}
+ /**
+ * Like claimOrReserve, but the bucket must cover the whole request or grant nothing. A denial
+ * consumes no tokens, so a caller retrying a multi-token claim cannot drain the bucket while
+ * never succeeding.
+ *
+ * `retryAfterMs` is null when the limiter itself failed, which is how a caller tells a reached
+ * cap from an unreachable Valkey. A denial reports a horizon only when `reserveOnDenyMs` and
+ * `refillPerSecond` are both above 0, so `reserveOnDenyMs` has no default: with 0 a reached
+ * cap would also read back as null.
+ */
+ public async claimAllOrNothing(
+ req: ClaimRequest,
+ reserveOnDenyMs: number
+ ): Promise<{ granted: boolean; retryAfterMs: number | null; reserved: boolean }> {
+ const { granted, retryAfterMs, reserved } = await this.evalClaim(req, reserveOnDenyMs, true)
+ return { granted: granted >= req.requested, retryAfterMs, reserved }
+ }
+
/**
* Like claimUpTo, but when the bucket says no, the caller also gets a time to come
* back at. First denial: wait only for the missing tokens. Every denial after that:
@@ -322,7 +227,8 @@ export class RateLimiterService {
private async evalClaim(
req: ClaimRequest,
- reserveOnDenyMs: number
+ reserveOnDenyMs: number,
+ allOrNothing: boolean = false
): Promise<{ granted: number; retryAfterMs: number | null; reserved: boolean }> {
const endTimer = claimLatency.startTimer({ limiter: this.config.name })
const ttlSeconds = req.ttlSeconds ?? 3600
@@ -340,7 +246,8 @@ export class RateLimiterService {
String(req.capacity),
String(req.refillPerSecond),
String(ttlSeconds),
- String(reserveOnDenyMs)
+ String(reserveOnDenyMs),
+ allOrNothing ? '1' : '0'
)
)
@@ -372,81 +279,4 @@ export class RateLimiterService {
endTimer()
}
}
-
- /**
- * Atomically claim `requested` tokens from BOTH buckets, or neither. A denial consumes no
- * tokens, so a caller that retries a multi-token claim cannot drain the buckets while never
- * succeeding. Returns which bucket denied (index into `buckets`), or null when granted.
- * A denial also says when to come back (`retryAfterMs`), paced by the slower of the two
- * buckets. With `reserveOnDenyMs` = 0 that is a shared answer and every denied caller gets
- * the same one. With `reserveOnDenyMs` > 0 the denial books the caller its own slot in
- * line instead, like claimOrReserve does, never further out than `reserveOnDenyMs`.
- * `reserved` tells the caller which of the two it got. A slot is a place in line, not a
- * promise: the wake still has to claim.
- * Runtime errors deny with `deniedIndex: null` and `retryAfterMs: null` — fail-closed, like
- * claimUpTo.
- *
- * On clustered Valkey the two keys must hash to the same slot (give them the same `{...}`
- * hash tag), because the Lua script touches both keys in one call and the cluster rejects
- * cross-slot calls with a CROSSSLOT error. Single-node Valkey accepts any pair of keys, so
- * tests and local dev do not catch a violation; production does, on every claim.
- */
- public async claimAllOrNothingPair(
- buckets: [Omit<ClaimRequest, 'requested'>, Omit<ClaimRequest, 'requested'>],
- requested: number,
- reserveOnDenyMs: number = 0
- ): Promise<{ granted: boolean; deniedIndex: 0 | 1 | null; retryAfterMs: number | null; reserved: boolean }> {
- const endTimer = claimLatency.startTimer({ limiter: this.config.name })
- try {
- const result = await this.valkey.useClient(
- { name: `rate-limiter:${this.config.name}:claimPair`, timeout: 1000 },
- (client) =>
- client.eval(
- CLAIM_ALL_OR_NOTHING_PAIR_LUA,
- 2,
- buckets[0].key,
- buckets[1].key,
- String(requested),
- String(buckets[0].capacity),
- String(buckets[0].refillPerSecond),
- String(buckets[0].ttlSeconds ?? 3600),
- String(buckets[1].capacity),
- String(buckets[1].refillPerSecond),
- String(buckets[1].ttlSeconds ?? 3600),
- String(reserveOnDenyMs)
- )
- )
-
- const [granted, deniedBucket, retryAfterMs, reserved] = Array.isArray(result)
- ? result.map(Number)
- : [NaN, NaN, NaN, 0]
- if (granted !== 0 && granted !== 1) {
- logger.warn('🪙', `RateLimiterService(${this.config.name}) pair claim returned invalid result`, {
- keys: [buckets[0].key, buckets[1].key],
- raw: result,
- })
- claimCounter.inc({ limiter: this.config.name, result: 'valkey_error' })
- return { granted: false, deniedIndex: null, retryAfterMs: null, reserved: false }
- }
-
- claimCounter.inc({ limiter: this.config.name, result: granted === 1 ? 'granted_full' : 'denied' })
- return granted === 1
- ? { granted: true, deniedIndex: null, retryAfterMs: null, reserved: false }
- : {
- granted: false,
- deniedIndex: deniedBucket === 2 ? 1 : 0,
- retryAfterMs: Number.isFinite(retryAfterMs) && retryAfterMs > 0 ? retryAfterMs : null,
- reserved: reserved === 1,
- }
- } catch (err) {
- logger.warn('🪙', `RateLimiterService(${this.config.name}) pair claim threw`, {
- keys: [buckets[0].key, buckets[1].key],
- error: String(err),
- })
- claimCounter.inc({ limiter: this.config.name, result: 'valkey_error' })
- return { granted: false, deniedIndex: null, retryAfterMs: null, reserved: false }
- } finally {
- endTimer()
- }
- }
}
diff --git a/nodejs/src/cdp/services/rate-limiter/rate-limiter.valkey.integration.test.ts b/nodejs/src/cdp/services/rate-limiter/rate-limiter.valkey.integration.test.ts
deleted file mode 100644
index 20bb408223fc..000000000000
--- a/nodejs/src/cdp/services/rate-limiter/rate-limiter.valkey.integration.test.ts
+++ /dev/null
@@ -1,46 +0,0 @@
-import Redis from 'ioredis'
-
-import { RedisClient, RedisClientPipeline, RedisV2 } from '~/common/redis/redis-v2'
-
-import { teamEmailCapBuckets } from '../messaging/email.service'
-import { RateLimiterService } from './rate-limiter.service'
-
-const host = process.env.CDP_VALKEY_HOST ?? '127.0.0.1'
-const port = Number(process.env.CDP_VALKEY_PORT ?? 6390)
-
-describe('RateLimiterService on Valkey Cluster', () => {
- let valkey: Redis.Redis
- let valkeyPool: RedisV2
-
- beforeAll(async () => {
- valkey = new Redis(port, host, { maxRetriesPerRequest: 1 })
- await valkey.ping()
- valkeyPool = {
- useClient: async (_options, callback) => callback(valkey as unknown as RedisClient),
- usePipeline: async (_options, callback) => {
- const pipeline = valkey.pipeline() as RedisClientPipeline
- callback(pipeline)
- return pipeline.exec()
- },
- }
- })
-
- afterAll(async () => {
- await valkey.quit()
- })
-
- it('pair-claims the team email cap buckets without a cross-slot rejection', async () => {
- // Prove the backing store enforces cluster key-slot rules. Without this, swapping the
- // compose service for a non-cluster Valkey would make the claim below pass vacuously.
- await expect(valkey.mget('rate-limiter-test-key-a', 'rate-limiter-test-key-b')).rejects.toThrow('CROSSSLOT')
-
- // A fresh team id per run keeps the buckets cold, so a full grant is the only correct
- // outcome. The cluster container keeps data between local runs.
- const teamId = Math.floor(Math.random() * 1_000_000_000)
- const buckets = teamEmailCapBuckets(teamId, 50, 100)
- const limiter = new RateLimiterService(valkeyPool, { name: 'team-email-cluster-test' })
-
- const claim = await limiter.claimAllOrNothingPair([buckets[0], buckets[1]], 10)
- expect(claim).toEqual({ granted: true, deniedIndex: null, retryAfterMs: null, reserved: false })
- })
-})
diff --git a/nodejs/src/cdp/workflows-e2e.serial.test.ts b/nodejs/src/cdp/workflows-e2e.serial.test.ts
index 4846d3d62f79..2247b2b8fc0d 100644
--- a/nodejs/src/cdp/workflows-e2e.serial.test.ts
+++ b/nodejs/src/cdp/workflows-e2e.serial.test.ts
@@ -3941,22 +3941,21 @@ describe('Workflows E2E (email queue)', () => {
it("a team over its tier cap does not hold up another team's emails", async () => {
// Two teams, each with one email workflow. Team A sits on a tier that allows
- // 2 emails per hour and gets 6 sends queued; team B is on a high tier with 3
+ // 2 emails per day and gets 6 sends queued; team B is on a high tier with 3
// sends. The whole pipeline is real and the tier caps are enforced: B's emails
// must all go out while A's over-cap sends park on future slots from A's own
// refill, without cycling.
//
// The default email worker was built with the caps off, so swap in one that
- // enforces them. Tier 0 allows 2/hour, tier 1 is effectively unlimited.
+ // enforces them. Tier 0 allows 2/day, tier 1 is effectively unlimited.
await emailWorker.stop()
hub.EMAIL_TEAM_SENDING_CAP_MODE = 'enforce'
- hub.EMAIL_TEAM_SENDING_CAP_HOURLY_BY_TIER = '2,10000'
- hub.EMAIL_TEAM_SENDING_CAP_DAILY_BY_TIER = '48,240000'
+ hub.EMAIL_TEAM_SENDING_CAP_DAILY_BY_TIER = '2,240000'
const enforcedQueue = new CyclotronJobQueuePostgresV2(hub.CONSUMER_BATCH_SIZE, hub)
emailWorker = new CdpCyclotronWorkerEmail(hub, deps, enforcedQueue)
await emailWorker.start()
- // The tier buckets are keyed by team id and the test Redis keeps data between
+ // The tier bucket is keyed by team id and the test Redis keeps data between
// runs, so team A's bucket must start full or a previous run drains this one.
const capValkey = createRedisV2PoolFromConfig({
connection: hub.CDP_REDIS_HOST
@@ -3968,7 +3967,6 @@ describe('Workflows E2E (email queue)', () => {
poolMinSize: hub.REDIS_POOL_MIN_SIZE,
poolMaxSize: hub.REDIS_POOL_MAX_SIZE,
})
- await deleteKeysWithPrefix(capValkey, '@posthog/team-email-rate-hour')
await deleteKeysWithPrefix(capValkey, '@posthog/team-email-rate-day')
// Team A has no config row and defaults to tier 0. Team B gets tier 1.
@@ -4076,7 +4074,7 @@ describe('Workflows E2E (email queue)', () => {
}, 15000)
// Team A's 4 over-cap sends are parked far in the future (its refill is one
- // token per 30 minutes), each dequeued once on the email queue.
+ // token per 12 hours), each dequeued once on the email queue.
const jobs = await queryCyclotronJobs()
const parkedA = jobs.filter(
(j: any) => j.function_id === flowA.id && j.queue_name === 'email' && j.status === 'available'