:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(cdp): claim one email cap bucket instead of two (#129163)
Meikel Ratz887d0fb45c4e
diff --git a/nodejs/src/cdp/cdp-services.ts b/nodejs/src/cdp/cdp-services.tsindex f17f1840ab81..a58385794e57 100644--- a/nodejs/src/cdp/cdp-services.ts+++ b/nodejs/src/cdp/cdp-services.ts@@ -183,7 +183,6 @@ export type CdpCoreServicesConfig = Pick<| 'EMAIL_SUPPRESSION_TRANSIENT_BOUNCE_THRESHOLD'| 'EMAIL_SUPPRESSION_GLOBAL_HARD_BOUNCE_ENFORCE'| 'EMAIL_TEAM_SENDING_CAP_MODE'- | 'EMAIL_TEAM_SENDING_CAP_HOURLY_BY_TIER'| 'EMAIL_TEAM_SENDING_CAP_DAILY_BY_TIER'| 'CDP_EMAIL_UTM_LINK_SCANNER_TEAMS'| 'CDP_GOOGLE_ADWORDS_DEVELOPER_TOKEN'@@ -489,7 +488,6 @@ export function createCdpCoreServices(sesTrackedConfigurationSet: config.SES_TRACKED_CONFIGURATION_SET,sesUntrackedConfigurationSet: config.SES_UNTRACKED_CONFIGURATION_SET,teamEmailCapMode: parseTeamEmailCapMode(config.EMAIL_TEAM_SENDING_CAP_MODE),- teamEmailTierHourlyCaps: parseTierCaps(config.EMAIL_TEAM_SENDING_CAP_HOURLY_BY_TIER),teamEmailTierDailyCaps: parseTierCaps(config.EMAIL_TEAM_SENDING_CAP_DAILY_BY_TIER),utmLinkScannerTeams: config.CDP_EMAIL_UTM_LINK_SCANNER_TEAMS,},diff --git a/nodejs/src/cdp/config.ts b/nodejs/src/cdp/config.tsindex 0a798c47ce61..d6a96fef1d3e 100644--- a/nodejs/src/cdp/config.ts+++ b/nodejs/src/cdp/config.ts@@ -187,15 +187,14 @@ export type CdpConfig = ClickhouseConfig & {EMAIL_SUPPRESSION_GLOBAL_HARD_BOUNCE_ENFORCE: boolean// Trust-tiered per-team workflow email caps ("team warming"). A team's tier, stored by the- // Django side, picks an hourly and a daily cap from these tables. See- // WORKFLOWS_EMAIL_TIER_* in posthog/settings/web.py, which must stay in step with them.+ // Django side, picks a daily cap from this table. See WORKFLOWS_EMAIL_TIER_* in+ // posthog/settings/web.py, which must stay in step with it.// "off" - the caps are not consulted.// "shadow" - the caps are evaluated and every send they would delay is logged and counted,// but nothing is delayed.// "enforce" - a reached cap reschedules the send.EMAIL_TEAM_SENDING_CAP_MODE: string- // Comma-separated caps indexed by tier. Both lists must be the same length.- EMAIL_TEAM_SENDING_CAP_HOURLY_BY_TIER: string+ // Comma-separated caps indexed by tier.EMAIL_TEAM_SENDING_CAP_DAILY_BY_TIER: string// Comma-separated team ids, or "*", whose UTM link tagging runs the scanner in helpers/utm.ts.// Every other team runs the previous scanner in helpers/utm-legacy.ts.@@ -437,7 +436,6 @@ export function getDefaultCdpConfig(): CdpConfig {// Ships dark: tiers are computed and stored first, then observed in shadow mode, and only// then enforced. Match the defaults in posthog/settings/web.py.EMAIL_TEAM_SENDING_CAP_MODE: 'off',- EMAIL_TEAM_SENDING_CAP_HOURLY_BY_TIER: '50,200,600,2000,6000,20000,60000,200000',EMAIL_TEAM_SENDING_CAP_DAILY_BY_TIER: '100,1000,3000,10000,30000,100000,300000,1000000',CDP_EMAIL_UTM_LINK_SCANNER_TEAMS: '',diff --git a/nodejs/src/cdp/services/managers/team-workflows-config.service.ts b/nodejs/src/cdp/services/managers/team-workflows-config.service.tsindex 59e5ddb039ba..c218209e28dd 100644--- a/nodejs/src/cdp/services/managers/team-workflows-config.service.ts+++ b/nodejs/src/cdp/services/managers/team-workflows-config.service.ts@@ -18,7 +18,7 @@ export type TeamWorkflowsConfig = {email_tracking_consent_mode: EmailTrackingConsentModeemail_sending_suspended: booleanses_tenant_provider_suspended: boolean- /** Trust tier that picks the team's hourly and daily workflow email caps. */+ /** Trust tier that picks the team's daily workflow email cap. */email_sending_tier: number/** Max marketing messages one person gets per window. Null on either field means no cap. */marketing_frequency_cap_max_messages: number | null@@ -103,7 +103,7 @@ export class TeamWorkflowsConfigService {}/**- * Trust tier for the team's workflow email, which the send path turns into an hourly and a+ * Trust tier for the team's workflow email, which the send path turns into a* daily cap. Fails open with `null`: a lookup error must let the send through rather than* throttle a legitimate customer, same stance as `isEmailSendingSuspended`.*/diff --git a/nodejs/src/cdp/services/messaging/email.service.test.ts b/nodejs/src/cdp/services/messaging/email.service.test.tsindex 137dfdd05408..c2f14667d74a 100644--- a/nodejs/src/cdp/services/messaging/email.service.test.ts+++ b/nodejs/src/cdp/services/messaging/email.service.test.ts@@ -37,8 +37,9 @@ import {EmailService,formatMailbox,parseAddressList,+ parseTierCaps,sanitizeEmailSubject,- teamEmailCapBuckets,+ teamEmailCapBucket,} from './email.service'import { MailDevAPI } from './helpers/maildev'import { EmailTrackingCodeSigner } from './helpers/tracking-code'@@ -917,14 +918,28 @@ describe('EmailService', () => {})})+ describe('parseTierCaps', () => {+ it.each([+ ['a whole table', '50,200', [50, 200]],+ // A decimal cap would make a claim fractional, and the limiter floors its grant+ // while the caller compares against the unfloored request. The claim then charges+ // the bucket and reads back as a denial, which starves the team on every retry.+ ['a decimal cap', '50,2.5', []],+ ['a zero cap', '50,0', []],+ ['nonsense', 'fifty', []],+ ])('%s', (_name, value, expected) => {+ expect(parseTierCaps(value)).toEqual(expected)+ })+ })+describe('team sending cap (enforce mode)', () => {- let claimAllOrNothingPair: jest.Mock+ let claimAllOrNothing: jest.Mocklet claimUpTo: jest.Mocklet cappedService: EmailServicelet cappedSendSpy: jest.SpyInstancebeforeEach(() => {- claimAllOrNothingPair = jest.fn()+ claimAllOrNothing = jest.fn()claimUpTo = jest.fn().mockResolvedValue(0)const configService = new TeamWorkflowsConfigService(hub.postgres, hub.pubSub)jest.spyOn(configService, 'getEmailSendingTier').mockResolvedValue(0)@@ -937,7 +952,6 @@ describe('EmailService', () => {sesTrackedConfigurationSet: hub.SES_TRACKED_CONFIGURATION_SET,sesUntrackedConfigurationSet: hub.SES_UNTRACKED_CONFIGURATION_SET,teamEmailCapMode: 'enforce',- teamEmailTierHourlyCaps: [100],teamEmailTierDailyCaps: [200],},hub.integrationManager,@@ -951,7 +965,7 @@ describe('EmailService', () => {{ report: jest.fn() },undefined,null,- { claimAllOrNothingPair, claimUpTo } as unknown as RateLimiterService+ { claimAllOrNothing, claimUpTo } as unknown as RateLimiterService)cappedSendSpy = jest.spyOn(cappedService.sesV2Client!, 'send') as anycappedSendSpy.mockResolvedValue({ MessageId: 'test-message-id' })@@ -963,7 +977,6 @@ describe('EmailService', () => {const buildEnforcedService = (name: string,- hourlyCap: number,dailyCap: number): { service: EmailService; sendSpy: jest.SpyInstance; limiter: RateLimiterService } => {const redis = createRedisV2PoolFromConfig({@@ -988,7 +1001,6 @@ describe('EmailService', () => {sesTrackedConfigurationSet: hub.SES_TRACKED_CONFIGURATION_SET,sesUntrackedConfigurationSet: hub.SES_UNTRACKED_CONFIGURATION_SET,teamEmailCapMode: 'enforce',- teamEmailTierHourlyCaps: [hourlyCap],teamEmailTierDailyCaps: [dailyCap],},hub.integrationManager,@@ -1023,7 +1035,7 @@ describe('EmailService', () => {2 * 60 * 60 * 1000,],])('%s', async (_name, retryAfterMs, reserved, minMs, maxMs) => {- claimAllOrNothingPair.mockResolvedValue({ granted: false, deniedIndex: 1, retryAfterMs, reserved })+ claimAllOrNothing.mockResolvedValue({ granted: false, retryAfterMs, reserved })const before = Date.now()const result = await cappedService.executeSendEmail(invocation)@@ -1047,12 +1059,7 @@ describe('EmailService', () => {// A Valkey fault denies with no horizon. Parking on the daily cap's pacing// interval would hold the email long after the limiter recovered.jest.spyOn(Math, 'random').mockReturnValue(0)- claimAllOrNothingPair.mockResolvedValue({- granted: false,- deniedIndex: null,- retryAfterMs: null,- reserved: false,- })+ claimAllOrNothing.mockResolvedValue({ granted: false, retryAfterMs: null, reserved: false })const before = Date.now()const result = await cappedService.executeSendEmail(invocation)@@ -1062,19 +1069,14 @@ describe('EmailService', () => {const scheduledMs = result.invocation.queueScheduledAt!.toMillis()expect(scheduledMs).toBeGreaterThanOrEqual(before + 5 * 60 * 1000)expect(scheduledMs).toBeLessThan(before + 5 * 60 * 1000 + 5000)- // No bucket denied, so the customer must not be told they hit a cap they never hit.+ // No cap was reached, so the customer must not be told they hit one.const messages = result.logs.map((log) => log.message).join(' ')expect(messages).not.toContain('reached its email sending limit')expect(messages).toContain("Could not check this project's email sending limit")})- it('sends a transactional email past the cap and charges the buckets anyway', async () => {- claimAllOrNothingPair.mockResolvedValue({- granted: false,- deniedIndex: 1,- retryAfterMs: 30 * 60 * 1000,- reserved: true,- })+ it('sends a transactional email past the cap and charges the bucket anyway', async () => {+ claimAllOrNothing.mockResolvedValue({ granted: false, retryAfterMs: 30 * 60 * 1000, reserved: true })invocation.hogFunction.metadata = { message_category_type: 'transactional' }const result = await cappedService.executeSendEmail(invocation)@@ -1082,29 +1084,20 @@ describe('EmailService', () => {expect(result.finished).toBe(true)expect(cappedSendSpy).toHaveBeenCalledTimes(1)expect(result.invocation.queueScheduledAt).toBeUndefined()- expect(claimAllOrNothingPair).not.toHaveBeenCalled()- expect(claimUpTo).toHaveBeenCalledTimes(2)+ expect(claimAllOrNothing).not.toHaveBeenCalled()+ expect(claimUpTo).toHaveBeenCalledTimes(1)// No cap was applied, so the customer must not be told one was.expect(result.logs.map((log) => log.message).join(' ')).not.toContain('reached its email sending limit')})it('sends when the claim is granted', async () => {- claimAllOrNothingPair.mockResolvedValue({- granted: true,- deniedIndex: null,- retryAfterMs: null,- reserved: false,- })+ claimAllOrNothing.mockResolvedValue({ granted: true, retryAfterMs: null, reserved: false })const result = await cappedService.executeSendEmail(invocation)// The reservation horizon must reach the limiter, or denials fall back to// shared-horizon wakes and a denied backlog re-herds.- expect(claimAllOrNothingPair).toHaveBeenCalledWith(- expect.anything(),- expect.any(Number),- 60 * 60 * 1000- )+ expect(claimAllOrNothing).toHaveBeenCalledWith(expect.anything(), 60 * 60 * 1000)expect(result.finished).toBe(true)expect(cappedSendSpy).toHaveBeenCalled()})@@ -1113,23 +1106,17 @@ describe('EmailService', () => {// their own future slots instead of all waking together, and another team's send// goes straight out. Runs against the real limiter.it('spreads a capped team over distinct slots while another team keeps sending', async () => {- const hourlyCap = 360const dailyCap = 8640const {service: enforcedService,sendSpy: enforcedSendSpy,limiter,- } = buildEnforcedService('team-email-cap-test', hourlyCap, dailyCap)+ } = buildEnforcedService('team-email-cap-test', dailyCap)- // Drain the capped team's hourly bucket so every send below is a denial. The+ // Drain the capped team's bucket so every send below is a denial. The// 0.1 tokens/s refill makes the slot spacing 10s, far above test wall-clock.- const buckets = teamEmailCapBuckets(team.id, hourlyCap, dailyCap)- await limiter.claimUpTo({- key: buckets[0].key,- requested: hourlyCap,- capacity: buckets[0].capacity,- refillPerSecond: buckets[0].refillPerSecond,- })+ const bucket = teamEmailCapBucket(team.id, dailyCap)+ await limiter.claimUpTo({ ...bucket, requested: dailyCap })const parkedAt: number[] = []for (let i = 0; i < 4; i++) {@@ -1169,16 +1156,15 @@ describe('EmailService', () => {expect(enforcedSendSpy).toHaveBeenCalledTimes(1)})- // Enforce mode has only ever issued claimAllOrNothingPair against real Valkey, so- // transactional through the single-key claimUpTo has never run there. Real limiter.- it('charges a transactional send against the real buckets without delaying it', async () => {- const hourlyCap = 60+ // Enforce mode has only ever issued claimAllOrNothing against real Valkey, so+ // transactional through claimUpTo has never run there. Real limiter.+ it('charges a transactional send against the real bucket without delaying it', async () => {const dailyCap = 50const {service: enforcedService,sendSpy: enforcedSendSpy,limiter,- } = buildEnforcedService('team-email-cap-transactional-test', hourlyCap, dailyCap)+ } = buildEnforcedService('team-email-cap-transactional-test', dailyCap)// A team of its own, so the buckets start cold whatever else touched Valkey.const capTeam = (await createTestTeamFixture(hub.postgres)).team@@ -1205,34 +1191,25 @@ describe('EmailService', () => {return next}- // Empty the hourly bucket only, so the daily one can show the charge.- const buckets = teamEmailCapBuckets(capTeam.id, hourlyCap, dailyCap)- await limiter.claimUpTo({- key: buckets[0].key,- requested: hourlyCap,- capacity: buckets[0].capacity,- refillPerSecond: buckets[0].refillPerSecond,- })+ // Leave one token, so the next send can show the charge before the cap closes.+ const bucket = teamEmailCapBucket(capTeam.id, dailyCap)+ await limiter.claimUpTo({ ...bucket, requested: dailyCap - 1 })++ // The charge: this send takes the last token rather than riding past the bucket.+ const charged = await enforcedService.executeSendEmail(sendFor('charged', true))+ expect(charged.finished).toBe(true)+ expect(enforcedSendSpy).toHaveBeenCalledTimes(1)- // Control: the cap really is reached, so a marketing send parks.+ // Control: the cap really is closed now, so a marketing send parks.const marketing = await enforcedService.executeSendEmail(sendFor('marketing', false))expect(marketing.finished).toBe(false)- expect(enforcedSendSpy).not.toHaveBeenCalled()+ expect(enforcedSendSpy).toHaveBeenCalledTimes(1)+ // The exemption: a transactional send still goes out against the closed cap.const transactional = await enforcedService.executeSendEmail(sendFor('transactional', true))expect(transactional.finished).toBe(true)expect(transactional.invocation.queueScheduledAt).toBeUndefined()- expect(enforcedSendSpy).toHaveBeenCalledTimes(1)-- // A range, not an exact count, because the daily bucket refills as the test runs.- const dailyLeft = await limiter.claimUpTo({- key: buckets[1].key,- requested: dailyCap,- capacity: buckets[1].capacity,- refillPerSecond: buckets[1].refillPerSecond,- })- expect(dailyLeft).toBeLessThan(dailyCap)- expect(dailyLeft).toBeGreaterThanOrEqual(dailyCap - 2)+ expect(enforcedSendSpy).toHaveBeenCalledTimes(2)})})})diff --git a/nodejs/src/cdp/services/messaging/email.service.ts b/nodejs/src/cdp/services/messaging/email.service.tsindex 9765aead3ae0..835f66774a85 100644--- a/nodejs/src/cdp/services/messaging/email.service.ts+++ b/nodejs/src/cdp/services/messaging/email.service.ts@@ -301,33 +301,31 @@ export function parseTeamEmailCapMode(value: string | undefined): TeamEmailCapMoreturn mode === 'shadow' || mode === 'enforce' ? mode : 'off'}-/** Parses a comma-separated cap table. Any unusable entry drops the whole table, which turns the- * cap off rather than applying a wrong number to a paying customer. */+/** Parses a comma-separated cap table. Any unusable entry drops the whole table, and the send path+ * then applies DEFAULT_TEAM_EMAIL_TIER_DAILY_CAPS. A non-integer cap is unusable because the+ * all-or-nothing claim needs a whole-number request. */export function parseTierCaps(value: string | undefined): number[] {const parts = (value ?? '').split(',').map((part) => part.trim()).filter((part) => part.length > 0)const caps = parts.map((part) => Number(part))- if (caps.length === 0 || caps.some((cap) => !Number.isFinite(cap) || cap <= 0)) {+ if (caps.length === 0 || caps.some((cap) => !Number.isInteger(cap) || cap <= 0)) {return []}return caps}-// Mirrors WORKFLOWS_EMAIL_TIER_HOURLY_CAPS / _DAILY_CAPS in posthog/settings/web.py. Both sides-// read the same tier index, so the two tables must stay in step.-const DEFAULT_TEAM_EMAIL_TIER_HOURLY_CAPS = [50, 200, 600, 2000, 6000, 20000, 60000, 200000]+// Mirrors WORKFLOWS_EMAIL_TIER_DAILY_CAPS in posthog/settings/web.py. Both sides read the same+// tier index, so the two tables must stay in step.const DEFAULT_TEAM_EMAIL_TIER_DAILY_CAPS = [100, 1000, 3000, 10000, 30000, 100000, 300000, 1000000]-// Comfortably longer than the window each bucket paces, so an idle bucket is never reset to full+// Comfortably longer than the window the bucket paces, so an idle bucket is never reset to full// by expiry before it would have refilled on its own. Without that, a team that pauses for a day// would get a fresh daily allowance instead of waiting for the refill.-const TEAM_EMAIL_HOUR_BUCKET_TTL_SECONDS = 6 * 60 * 60const TEAM_EMAIL_DAY_BUCKET_TTL_SECONDS = 3 * 24 * 60 * 60type TeamEmailCapBucket = {- name: 'hour' | 'day'key: stringcapacity: numberrefillPerSecond: number@@ -336,28 +334,14 @@ type TeamEmailCapBucket = {label: string}-// The `{teamId}` hash tag puts both bucket keys in the same cluster slot. Enforce mode claims-// both keys in one Lua call (claimAllOrNothingPair), and clustered Valkey rejects a multi-key-// call whose keys hash to different slots with a CROSSSLOT error.-export function teamEmailCapBuckets(teamId: number, hourlyCap: number, dailyCap: number): TeamEmailCapBucket[] {- return [- {- name: 'hour',- key: `@posthog/team-email-rate-hour/{${teamId}}`,- capacity: hourlyCap,- refillPerSecond: hourlyCap / 3600,- ttlSeconds: TEAM_EMAIL_HOUR_BUCKET_TTL_SECONDS,- label: `${hourlyCap.toLocaleString('en-US')} emails per hour`,- },- {- name: 'day',- key: `@posthog/team-email-rate-day/{${teamId}}`,- capacity: dailyCap,- refillPerSecond: dailyCap / 86400,- ttlSeconds: TEAM_EMAIL_DAY_BUCKET_TTL_SECONDS,- label: `${dailyCap.toLocaleString('en-US')} emails per day`,- },- ]+export function teamEmailCapBucket(teamId: number, dailyCap: number): TeamEmailCapBucket {+ return {+ key: `@posthog/team-email-rate-day/{${teamId}}`,+ capacity: dailyCap,+ refillPerSecond: dailyCap / 86400,+ ttlSeconds: TEAM_EMAIL_DAY_BUCKET_TTL_SECONDS,+ label: `${dailyCap.toLocaleString('en-US')} emails per day`,+ }}export interface EmailServiceConfig {@@ -373,7 +357,6 @@ export interface EmailServiceConfig {// Trust-tiered per-team sending caps. Optional and defaulting to off, so every send path that// builds an EmailService without them keeps its pre-cap behavior.teamEmailCapMode?: TeamEmailCapMode- teamEmailTierHourlyCaps?: number[]teamEmailTierDailyCaps?: number[]utmLinkScannerTeams?: string}@@ -951,25 +934,25 @@ export class EmailService {/*** Claims one send (as `recipients` tokens, one per delivered copy) against the team's- * trust-tier buckets.+ * trust-tier daily bucket.*- * Returns the delay to reschedule with when a cap is reached, or null when the send may go out.- * Two buckets, not one: the daily cap bounds how much damage a team can do to the shared SES- * account's reputation, and the hourly cap forces that volume to spread out so complaint- * feedback (which lags by hours) arrives while the team's total volume is still small.+ * Returns the delay to reschedule with when the cap is reached, or null when the send may go+ * out. The cap bounds how much damage a team can do to the shared SES account's reputation+ * in a day. It does not spread that volume out: a full bucket lets the team send its whole+ * daily cap as one burst, and only volume past that waits for the refill.** Failure stances differ on purpose. A failed tier lookup lets the send through, because a- * config blip must never throttle a legitimate customer. A failed bucket claim returns 0 from- * the limiter, which delays the send, because we must not send when we cannot account for it.- * That second case returns a null `label`: the send waits, but no cap was reached, so the- * caller must not tell the customer it hit one.+ * config blip must never throttle a legitimate customer. A failed bucket claim delays the+ * send, because we must not send when we cannot account for it. That second case returns a+ * null `label`: the send waits, but no cap was reached, so the caller must not tell the+ * customer it hit one.** Transactional sends are charged but never delayed, because a password reset that arrives* two hours late has already failed the person waiting on it. The charge does not bound the- * exemption: claimUpTo records no debt, so once a bucket empties further transactional sends- * cost nothing. No other control bounds that volume per team either. The SES limiter paces- * the whole account, so what is left is suspension and the daily tier job, which demotes a- * team on its complaint and bounce rates after the fact.+ * exemption: claimUpTo records no debt, so once the bucket empties further transactional+ * sends cost nothing. No other control bounds that volume per team either. The SES limiter+ * paces the whole account, so what is left is suspension and the daily tier job, which+ * demotes a team on its complaint and bounce rates after the fact.*/private async claimTeamSendingBudget(invocation: CyclotronJobInvocationHogFunction,@@ -986,78 +969,44 @@ export class EmailService {return null}- const hourlyCaps = this.sesConfig.teamEmailTierHourlyCaps?.length- ? this.sesConfig.teamEmailTierHourlyCaps- : DEFAULT_TEAM_EMAIL_TIER_HOURLY_CAPSconst dailyCaps = this.sesConfig.teamEmailTierDailyCaps?.length? this.sesConfig.teamEmailTierDailyCaps: DEFAULT_TEAM_EMAIL_TIER_DAILY_CAPS- const topTier = Math.min(hourlyCaps.length, dailyCaps.length) - 1- if (topTier < 0) {- return null- }- const tier = Math.min(Math.max(teamTier, 0), topTier)+ const tier = Math.min(Math.max(teamTier, 0), dailyCaps.length - 1)- const buckets = teamEmailCapBuckets(invocation.teamId, hourlyCaps[tier], dailyCaps[tier])- // Clamped to the smaller capacity: a send with more copies than a whole allowance could- // never be granted and would reschedule forever. It charges the entire bucket instead, so- // it still waits for full capacity and pays everything there is.- const requested = Math.min(Math.max(1, recipients), buckets[0].capacity, buckets[1].capacity)+ const bucket = teamEmailCapBucket(invocation.teamId, dailyCaps[tier])+ // Clamped to the capacity: a send with more copies than a whole allowance could never be+ // granted and would reschedule forever. It charges the entire bucket instead, so it still+ // waits for full capacity and pays everything there is.+ const requested = Math.min(Math.max(1, recipients), bucket.capacity)const isTransactional = invocation.hogFunction?.metadata?.message_category_type === 'transactional'if (mode === 'enforce' && !isTransactional) {- // Both buckets in one atomic claim, granted whole or not at all. A denial consumes- // nothing, so a rescheduled multi-recipient send cannot burn the partial refill on- // every retry and starve the team's other emails while never sending itself.- const claim = await this.teamEmailRateLimiter.claimAllOrNothingPair(- [buckets[0], buckets[1]],- requested,- CAP_RETRY_MAX_MS- )+ const claim = await this.teamEmailRateLimiter.claimAllOrNothing({ ...bucket, requested }, CAP_RETRY_MAX_MS)if (claim.granted) {return null}- // `deniedIndex` is null when the limiter itself failed, not when a bucket was full. The- // send still waits, because we must not send what we cannot account for, but no cap was- // reached, so a null label tells the caller it has no cap to name to the customer.- const denied = claim.deniedIndex === null ? null : buckets[claim.deniedIndex]- // The daily bucket paces a limiter failure because it is the slower of the two, so the- // retry backs off at the safer rate while the limiter is unreachable.- const pacing = denied ?? buckets[1]+ const capReached = claim.retryAfterMs !== nullteamEmailCapDelayedTotal.inc({tier: String(tier),- bucket: denied?.name ?? 'limiter_unavailable',+ bucket: capReached ? 'day' : 'limiter_unavailable',mode,})- const retryDelayMs = pickReservedRetryDelayMs(claim.retryAfterMs, pacing.refillPerSecond, claim.reserved)+ const retryDelayMs = pickReservedRetryDelayMs(claim.retryAfterMs, bucket.refillPerSecond, claim.reserved)emailReservedParkMs.labels('team-email').observe(retryDelayMs)return {retryDelayMs,- label: denied?.label ?? null,+ label: capReached ? bucket.label : null,}}// Shadow mode, and transactional sends in any mode: the send goes out regardless, so- // drain each bucket and log the cap. Separate labels keep shadow a forecast of enforce.+ // drain the bucket and log the cap. Separate labels keep shadow a forecast of enforce.const observedMode = isTransactional ? 'transactional' : mode- let firstDenial: TeamEmailCapBucket | null = null- for (const bucket of buckets) {- const granted = await this.teamEmailRateLimiter.claimUpTo({- key: bucket.key,- requested,- capacity: bucket.capacity,- refillPerSecond: bucket.refillPerSecond,- ttlSeconds: bucket.ttlSeconds,- })- if (granted >= requested) {- continue- }- teamEmailCapDelayedTotal.inc({ tier: String(tier), bucket: bucket.name, mode: observedMode })- firstDenial = firstDenial ?? bucket- }-- if (firstDenial) {+ const granted = await this.teamEmailRateLimiter.claimUpTo({ ...bucket, requested })+ if (granted < requested) {+ teamEmailCapDelayedTotal.inc({ tier: String(tier), bucket: 'day', mode: observedMode })logger.info('📧',isTransactional@@ -1066,8 +1015,8 @@ export class EmailService {{teamId: invocation.teamId,tier,- bucket: firstDenial.name,- cap: firstDenial.label,+ bucket: 'day',+ cap: bucket.label,})}diff --git a/nodejs/src/cdp/services/rate-limiter/rate-limiter.service.test.ts b/nodejs/src/cdp/services/rate-limiter/rate-limiter.service.test.tsindex 490f0fee49ef..2b85bd4dab30 100644--- a/nodejs/src/cdp/services/rate-limiter/rate-limiter.service.test.ts+++ b/nodejs/src/cdp/services/rate-limiter/rate-limiter.service.test.ts@@ -197,66 +197,38 @@ describe('RateLimiterService', () => {})})- describe('claimAllOrNothingPair', () => {- const KEY_A = `${KEY}/pair-a`- const KEY_B = `${KEY}/pair-b`-- it('grants from both buckets when both can cover the request', async () => {- const claim = await limiter.claimAllOrNothingPair(- [- { key: KEY_A, capacity: 50, refillPerSecond: 0 },- { key: KEY_B, capacity: 100, refillPerSecond: 0 },- ],- 30+ describe('claimAllOrNothing', () => {+ const AON_KEY = `${KEY}/all-or-nothing`++ it('grants and charges the pool when the bucket covers the request', async () => {+ const claim = await limiter.claimAllOrNothing(+ { key: AON_KEY, requested: 30, capacity: 50, refillPerSecond: 0 },+ 600_000)- expect(claim).toEqual({ granted: true, deniedIndex: null, retryAfterMs: null, reserved: false })- // Both pools were charged: the remainder is all that is left to claim.- expect(await limiter.claimUpTo({ key: KEY_A, requested: 50, capacity: 50, refillPerSecond: 0 })).toBe(20)- expect(await limiter.claimUpTo({ key: KEY_B, requested: 100, capacity: 100, refillPerSecond: 0 })).toBe(70)+ expect(claim).toEqual({ granted: true, retryAfterMs: null, reserved: false })+ // The remainder is all that is left to claim.+ expect(await limiter.claimUpTo({ key: AON_KEY, requested: 50, capacity: 50, refillPerSecond: 0 })).toBe(20)})- it('consumes nothing on denial, so retries cannot starve the buckets', async () => {- // The second bucket cannot cover the request, so the claim must leave BOTH pools- // untouched. Without that, a rescheduled multi-recipient send would burn the first- // bucket's refill on every retry while never sending.- const claim = await limiter.claimAllOrNothingPair(- [- { key: KEY_A, capacity: 50, refillPerSecond: 0 },- { key: KEY_B, capacity: 10, refillPerSecond: 0 },- ],- 30- )+ it('consumes nothing on denial, so retries cannot starve the bucket', async () => {+ // The bucket cannot cover the request, so the claim must leave the pool untouched.+ // Without that, a rescheduled multi-recipient send would burn the refill on every+ // retry while never sending.+ const req = { key: `${AON_KEY}/denied`, requested: 30, capacity: 10, refillPerSecond: 0 }+ const claim = await limiter.claimAllOrNothing(req, 600_000)// refillPerSecond 0 means the missing tokens never accrue, so no horizon is reported.- expect(claim).toEqual({ granted: false, deniedIndex: 1, retryAfterMs: null, reserved: false })- expect(await limiter.claimUpTo({ key: KEY_A, requested: 50, capacity: 50, refillPerSecond: 0 })).toBe(50)- expect(await limiter.claimUpTo({ key: KEY_B, requested: 10, capacity: 10, refillPerSecond: 0 })).toBe(10)+ expect(claim).toEqual({ granted: false, retryAfterMs: null, reserved: false })+ expect(await limiter.claimUpTo({ ...req, requested: 10 })).toBe(10)})it('reports on denial how long until the missing tokens accrue', async () => {- // Cold start: the denying bucket holds exactly its capacity of 10, so the request- // for 30 is missing 20 tokens. At 2 tokens/s that is 10 seconds.- const claim = await limiter.claimAllOrNothingPair(- [- { key: KEY_A, capacity: 50, refillPerSecond: 0 },- { key: KEY_B, capacity: 10, refillPerSecond: 2 },- ],- 30+ // Cold start: the bucket holds exactly its capacity of 10, so the request for 30 is+ // missing 20 tokens. At 2 tokens/s that is 10 seconds.+ const claim = await limiter.claimAllOrNothing(+ { key: `${AON_KEY}/horizon`, requested: 30, capacity: 10, refillPerSecond: 2 },+ 600_000)- expect(claim).toEqual({ granted: false, deniedIndex: 1, retryAfterMs: 10_000, reserved: false })- })-- it('reports the slower bucket when both are short', async () => {- // Both buckets are 20 tokens short of the request. The first one covers that in 10- // seconds, the second needs 40. Reporting the first would wake the caller while the- // second still cannot grant, costing a whole extra dequeue and reschedule.- const claim = await limiter.claimAllOrNothingPair(- [- { key: KEY_A, capacity: 10, refillPerSecond: 2 },- { key: KEY_B, capacity: 10, refillPerSecond: 0.5 },- ],- 30- )- expect(claim).toEqual({ granted: false, deniedIndex: 0, retryAfterMs: 40_000, reserved: false })+ expect(claim).toEqual({ granted: false, retryAfterMs: 10_000, reserved: true })})it('fails closed when the Lua call throws', async () => {@@ -265,85 +237,12 @@ describe('RateLimiterService', () => {usePipeline: jest.fn(),} as unknown as RedisV2const brokenLimiter = new RateLimiterService(brokenValkey, { name: 'broken-limiter' })- const claim = await brokenLimiter.claimAllOrNothingPair(- [- { key: KEY_A, capacity: 50, refillPerSecond: 0 },- { key: KEY_B, capacity: 10, refillPerSecond: 0 },- ],- 5+ // A null horizon is how the caller tells an unreachable limiter from a reached cap.+ const claim = await brokenLimiter.claimAllOrNothing(+ { key: AON_KEY, requested: 5, capacity: 50, refillPerSecond: 1 },+ 600_000)- expect(claim).toEqual({ granted: false, deniedIndex: null, retryAfterMs: null, reserved: false })- })-- it('hands successive reserved denials distinct, later slots, paced by the slower bucket', async () => {- // Both buckets are short, so the slower one (0.5/s) sets the pace. The first- // caller waits 40s (its missing tokens). The next one gets the slot after- // that, a full 60s later. Nobody wakes at the same time.- const buckets: [- { key: string; capacity: number; refillPerSecond: number },- { key: string; capacity: number; refillPerSecond: number },- ] = [- { key: `${KEY_A}/resv`, capacity: 10, refillPerSecond: 2 },- { key: `${KEY_B}/resv`, capacity: 10, refillPerSecond: 0.5 },- ]- const first = await limiter.claimAllOrNothingPair(buckets, 30, 600_000)- const second = await limiter.claimAllOrNothingPair(buckets, 30, 600_000)-- expect(first.granted).toBe(false)- expect(first.reserved).toBe(true)- // First in line pays only the 40s shortfall, not a full 60s slot.- expect(first.retryAfterMs).toBe(40_000)- expect(second.reserved).toBe(true)- expect(second.retryAfterMs!).toBeGreaterThan(first.retryAfterMs!)- expect(second.retryAfterMs!).toBeLessThanOrEqual(100_000)- })-- it('stops advancing the reservation cursor at the horizon', async () => {- const buckets: [- { key: string; capacity: number; refillPerSecond: number },- { key: string; capacity: number; refillPerSecond: number },- ] = [- { key: `${KEY_A}/resv-cap`, capacity: 50, refillPerSecond: 0 },- { key: `${KEY_B}/resv-cap`, capacity: 10, refillPerSecond: 2 },- ]- // The first slot (10s) fits inside the 20s horizon. The next one would not,- // so everyone after the first just gets "come back in 20s" with no slot.- const first = await limiter.claimAllOrNothingPair(buckets, 30, 20_000)- const second = await limiter.claimAllOrNothingPair(buckets, 30, 20_000)- const third = await limiter.claimAllOrNothingPair(buckets, 30, 20_000)-- expect(first.retryAfterMs).toBe(10_000)- expect(first.reserved).toBe(true)- expect(second.retryAfterMs).toBe(20_000)- expect(second.reserved).toBe(false)- expect(third.retryAfterMs).toBe(20_000)- expect(third.reserved).toBe(false)- })-- it('keeps one reservation line when the slower bucket changes', async () => {- const buckets: [- { key: string; capacity: number; refillPerSecond: number },- { key: string; capacity: number; refillPerSecond: number },- ] = [- { key: `${KEY_A}/line`, capacity: 10, refillPerSecond: 2 },- { key: `${KEY_B}/line`, capacity: 100, refillPerSecond: 0.5 },- ]- // The second bucket still covers the request, so the first one (10s- // shortfall, 15s spacing) sets the pace for the first two denials.- const first = await limiter.claimAllOrNothingPair(buckets, 30, 600_000)- const second = await limiter.claimAllOrNothingPair(buckets, 30, 600_000)-- // Now drain the second bucket so it becomes the slower one. The line has- // to continue behind the parked sends. A cursor kept only on the bucket- // that is currently slower would start a fresh line here and hand this- // send a slot at ~16s, ahead of the send already parked at ~25s.- await limiter.claimUpTo({ key: buckets[1].key, requested: 78, capacity: 100, refillPerSecond: 0.5 })- const third = await limiter.claimAllOrNothingPair(buckets, 30, 600_000)-- expect(first.reserved).toBe(true)- expect(second.retryAfterMs!).toBeGreaterThan(first.retryAfterMs!)- expect(third.reserved).toBe(true)- expect(third.retryAfterMs!).toBeGreaterThan(second.retryAfterMs!)+ expect(claim).toEqual({ granted: false, retryAfterMs: null, reserved: false })})})diff --git a/nodejs/src/cdp/services/rate-limiter/rate-limiter.service.ts b/nodejs/src/cdp/services/rate-limiter/rate-limiter.service.tsindex 1d4b178a0574..c8925ee9a41d 100644--- a/nodejs/src/cdp/services/rate-limiter/rate-limiter.service.ts+++ b/nodejs/src/cdp/services/rate-limiter/rate-limiter.service.ts@@ -41,6 +41,10 @@ const claimLatency = new Histogram({// ARGV[4] = TTL seconds — bucket auto-expires when idle so cold-start// gives full capacity rather than a stale negative pool.// ARGV[5] = reserve-on-deny horizon in ms; 0 disables reservation.+// ARGV[6] = 1 to make the claim all-or-nothing: a bucket that cannot cover+// the whole request grants nothing and consumes nothing, so a+// caller retrying a multi-token claim cannot drain the bucket+// while never succeeding.//// `now` is sourced via `redis.call('TIME')` so all pods share a single// monotonic clock — NTP drift between workers can't over- or under-refill@@ -63,6 +67,7 @@ local capacity = tonumber(ARGV[2])local refillPerSecond = tonumber(ARGV[3])local ttlSeconds = tonumber(ARGV[4])local reserveOnDenyMaxMs = tonumber(ARGV[5])+local allOrNothing = tonumber(ARGV[6])local time = redis.call('TIME')-- TIME returns {seconds, microseconds}; flatten to epoch ms.@@ -92,12 +97,14 @@ elseendlocal rawGranted = math.min(available, requested)-if rawGranted < 0 then+if rawGranted < 0 or (allOrNothing == 1 and available < requested) thenrawGranted = 0end--- Floor to int — the grant flows into a SQL LIMIT, which must be integer.--- The fractional residual stays in the pool so partial refills accumulate--- across calls instead of being silently dropped.+-- Floor to int — the grant flows into a SQL LIMIT, which must be integer, and+-- Redis truncates a Lua float in the reply regardless. The fractional residual+-- stays in the pool so partial refills accumulate across calls instead of being+-- silently dropped. An all-or-nothing caller must therefore request a whole+-- number, or the floored grant reads back as a denial after the bucket paid.local granted = math.floor(rawGranted)local tokensAfter = available - granted@@ -129,126 +136,6 @@ redis.call('hset', key, 'resv', slotAt)return {0, math.ceil(slotAt - now), 1}`-// Atomic all-or-nothing claim across two buckets. Same refill math as CLAIM_UP_TO_LUA per bucket,-// but the claim succeeds only when BOTH buckets can cover the full request, and a denial writes-// nothing at all — no partial consumption, no timestamp refresh. Without that, a caller retrying a-// multi-token claim would burn each bucket's partial refill on every attempt and could keep a-// shared bucket empty without ever succeeding.-// KEYS[1..2] = the two bucket hash keys-// ARGV[1] = requested tokens (integer)-// ARGV[2..4] = capacity, refill/sec, TTL seconds for KEYS[1]-// ARGV[5..7] = capacity, refill/sec, TTL seconds for KEYS[2]-// ARGV[8] = reserve-on-deny horizon in ms; 0 disables reservation.-// Returns {1, 0, 0, 0} when granted. On denial: {0, i, retryAfterMs, reserved},-// where i is the first bucket (1-based) that came up short. With ARGV[8] = 0,-// retryAfterMs is only how long until BOTH buckets have their missing tokens back-// (the slower one decides). Every denied caller gets that same answer, so they all-// wake together. With ARGV[8] > 0 the denial also books the caller a slot on the-// slowest short bucket, the same ticket-at-a-counter idea as the claim-up-to script-// above: the first denied caller waits only for the missing tokens, everyone after-// gets the next slot, one interval later. Slots never go out past ARGV[8]; past-// that the caller gets ARGV[8] back with reserved=0 and asks again when it wakes.-// A slot is a place in line, not a promise: the wake still has to claim. A bucket-// that never refills has no horizon, and then the denial reports none and books-// nothing.-const CLAIM_ALL_OR_NOTHING_PAIR_LUA = `-local time = redis.call('TIME')-local now = tonumber(time[1]) * 1000 + math.floor(tonumber(time[2]) / 1000)-local requested = tonumber(ARGV[1])-local reserveOnDenyMaxMs = tonumber(ARGV[8])--local available = {}-local deniedIndex = 0-local retryAfterMs = 0-local horizonKnown = true-local slowRefill = 0-for i = 1, 2 do- local capacity = tonumber(ARGV[(i - 1) * 3 + 2])- local refillPerSecond = tonumber(ARGV[(i - 1) * 3 + 3])- local existing = redis.call('hmget', KEYS[i], 'ts', 'pool')- local avail- if existing[1] == false then- avail = capacity- else- local elapsedMs = now - tonumber(existing[1])- if elapsedMs < 0 then- elapsedMs = 0- end- local currentTokens = capacity- if existing[2] ~= false then- currentTokens = tonumber(existing[2])- end- avail = math.min(capacity, currentTokens + (elapsedMs / 1000.0) * refillPerSecond)- end- if avail < requested then- if deniedIndex == 0 then- deniedIndex = i- end- if refillPerSecond > 0 then- local bucketRetryMs = math.ceil(((requested - avail) / refillPerSecond) * 1000)- if bucketRetryMs > retryAfterMs then- retryAfterMs = bucketRetryMs- slowRefill = refillPerSecond- end- else- horizonKnown = false- end- end- available[i] = avail-end--if deniedIndex > 0 then- if not horizonKnown then- return {0, deniedIndex, 0, 0}- end- if reserveOnDenyMaxMs <= 0 then- return {0, deniedIndex, retryAfterMs, 0}- end- -- First in line: wait only for the pair's shortfall (that is retryAfterMs).- -- Behind someone: take the slot after theirs, one interval of the slowest- -- bucket later. Same idea as the claim-up-to script above.- -- There is ONE line for the pair, kept on both keys: the base is the later of- -- the two cursors and the new slot is written back to both. Keeping the line- -- on only the currently slower bucket splits it when the slower bucket- -- changes (hourly runs dry first, then daily), and a fresh line lets new- -- arrivals cut ahead of sends that are already parked.- local rawResvA = redis.call('hget', KEYS[1], 'resv')- local rawResvB = redis.call('hget', KEYS[2], 'resv')- local base = now- if rawResvA ~= false and tonumber(rawResvA) > base then- base = tonumber(rawResvA)- end- if rawResvB ~= false and tonumber(rawResvB) > base then- base = tonumber(rawResvB)- end- local slotAt- if base > now then- slotAt = base + (requested / slowRefill) * 1000- else- slotAt = now + retryAfterMs- end- if slotAt - now < retryAfterMs then- slotAt = now + retryAfterMs- end- if slotAt - now > reserveOnDenyMaxMs then- return {0, deniedIndex, reserveOnDenyMaxMs, 0}- end- redis.call('hset', KEYS[1], 'resv', slotAt)- redis.call('expire', KEYS[1], tonumber(ARGV[4]))- redis.call('hset', KEYS[2], 'resv', slotAt)- redis.call('expire', KEYS[2], tonumber(ARGV[7]))- return {0, deniedIndex, math.ceil(slotAt - now), 1}-end--for i = 1, 2 do- local ttlSeconds = tonumber(ARGV[(i - 1) * 3 + 4])- redis.call('hset', KEYS[i], 'ts', now, 'pool', available[i] - requested)- redis.call('expire', KEYS[i], ttlSeconds)-end--return {1, 0, 0, 0}-`-export interface RateLimiterConfig {/** Logical name for metrics/logging only (e.g. 'ses'). */name: string@@ -299,6 +186,24 @@ export class RateLimiterService {return (await this.evalClaim(req, 0)).granted}+ /**+ * Like claimOrReserve, but the bucket must cover the whole request or grant nothing. A denial+ * consumes no tokens, so a caller retrying a multi-token claim cannot drain the bucket while+ * never succeeding.+ *+ * `retryAfterMs` is null when the limiter itself failed, which is how a caller tells a reached+ * cap from an unreachable Valkey. A denial reports a horizon only when `reserveOnDenyMs` and+ * `refillPerSecond` are both above 0, so `reserveOnDenyMs` has no default: with 0 a reached+ * cap would also read back as null.+ */+ public async claimAllOrNothing(+ req: ClaimRequest,+ reserveOnDenyMs: number+ ): Promise<{ granted: boolean; retryAfterMs: number | null; reserved: boolean }> {+ const { granted, retryAfterMs, reserved } = await this.evalClaim(req, reserveOnDenyMs, true)+ return { granted: granted >= req.requested, retryAfterMs, reserved }+ }+/*** Like claimUpTo, but when the bucket says no, the caller also gets a time to come* back at. First denial: wait only for the missing tokens. Every denial after that:@@ -322,7 +227,8 @@ export class RateLimiterService {private async evalClaim(req: ClaimRequest,- reserveOnDenyMs: number+ reserveOnDenyMs: number,+ allOrNothing: boolean = false): Promise<{ granted: number; retryAfterMs: number | null; reserved: boolean }> {const endTimer = claimLatency.startTimer({ limiter: this.config.name })const ttlSeconds = req.ttlSeconds ?? 3600@@ -340,7 +246,8 @@ export class RateLimiterService {String(req.capacity),String(req.refillPerSecond),String(ttlSeconds),- String(reserveOnDenyMs)+ String(reserveOnDenyMs),+ allOrNothing ? '1' : '0'))@@ -372,81 +279,4 @@ export class RateLimiterService {endTimer()}}-- /**- * Atomically claim `requested` tokens from BOTH buckets, or neither. A denial consumes no- * tokens, so a caller that retries a multi-token claim cannot drain the buckets while never- * succeeding. Returns which bucket denied (index into `buckets`), or null when granted.- * A denial also says when to come back (`retryAfterMs`), paced by the slower of the two- * buckets. With `reserveOnDenyMs` = 0 that is a shared answer and every denied caller gets- * the same one. With `reserveOnDenyMs` > 0 the denial books the caller its own slot in- * line instead, like claimOrReserve does, never further out than `reserveOnDenyMs`.- * `reserved` tells the caller which of the two it got. A slot is a place in line, not a- * promise: the wake still has to claim.- * Runtime errors deny with `deniedIndex: null` and `retryAfterMs: null` — fail-closed, like- * claimUpTo.- *- * On clustered Valkey the two keys must hash to the same slot (give them the same `{...}`- * hash tag), because the Lua script touches both keys in one call and the cluster rejects- * cross-slot calls with a CROSSSLOT error. Single-node Valkey accepts any pair of keys, so- * tests and local dev do not catch a violation; production does, on every claim.- */- public async claimAllOrNothingPair(- buckets: [Omit<ClaimRequest, 'requested'>, Omit<ClaimRequest, 'requested'>],- requested: number,- reserveOnDenyMs: number = 0- ): Promise<{ granted: boolean; deniedIndex: 0 | 1 | null; retryAfterMs: number | null; reserved: boolean }> {- const endTimer = claimLatency.startTimer({ limiter: this.config.name })- try {- const result = await this.valkey.useClient(- { name: `rate-limiter:${this.config.name}:claimPair`, timeout: 1000 },- (client) =>- client.eval(- CLAIM_ALL_OR_NOTHING_PAIR_LUA,- 2,- buckets[0].key,- buckets[1].key,- String(requested),- String(buckets[0].capacity),- String(buckets[0].refillPerSecond),- String(buckets[0].ttlSeconds ?? 3600),- String(buckets[1].capacity),- String(buckets[1].refillPerSecond),- String(buckets[1].ttlSeconds ?? 3600),- String(reserveOnDenyMs)- )- )-- const [granted, deniedBucket, retryAfterMs, reserved] = Array.isArray(result)- ? result.map(Number)- : [NaN, NaN, NaN, 0]- if (granted !== 0 && granted !== 1) {- logger.warn('🪙', `RateLimiterService(${this.config.name}) pair claim returned invalid result`, {- keys: [buckets[0].key, buckets[1].key],- raw: result,- })- claimCounter.inc({ limiter: this.config.name, result: 'valkey_error' })- return { granted: false, deniedIndex: null, retryAfterMs: null, reserved: false }- }-- claimCounter.inc({ limiter: this.config.name, result: granted === 1 ? 'granted_full' : 'denied' })- return granted === 1- ? { granted: true, deniedIndex: null, retryAfterMs: null, reserved: false }- : {- granted: false,- deniedIndex: deniedBucket === 2 ? 1 : 0,- retryAfterMs: Number.isFinite(retryAfterMs) && retryAfterMs > 0 ? retryAfterMs : null,- reserved: reserved === 1,- }- } catch (err) {- logger.warn('🪙', `RateLimiterService(${this.config.name}) pair claim threw`, {- keys: [buckets[0].key, buckets[1].key],- error: String(err),- })- claimCounter.inc({ limiter: this.config.name, result: 'valkey_error' })- return { granted: false, deniedIndex: null, retryAfterMs: null, reserved: false }- } finally {- endTimer()- }- }}diff --git a/nodejs/src/cdp/services/rate-limiter/rate-limiter.valkey.integration.test.ts b/nodejs/src/cdp/services/rate-limiter/rate-limiter.valkey.integration.test.tsdeleted file mode 100644index 20bb408223fc..000000000000--- a/nodejs/src/cdp/services/rate-limiter/rate-limiter.valkey.integration.test.ts+++ /dev/null@@ -1,46 +0,0 @@-import Redis from 'ioredis'--import { RedisClient, RedisClientPipeline, RedisV2 } from '~/common/redis/redis-v2'--import { teamEmailCapBuckets } from '../messaging/email.service'-import { RateLimiterService } from './rate-limiter.service'--const host = process.env.CDP_VALKEY_HOST ?? '127.0.0.1'-const port = Number(process.env.CDP_VALKEY_PORT ?? 6390)--describe('RateLimiterService on Valkey Cluster', () => {- let valkey: Redis.Redis- let valkeyPool: RedisV2-- beforeAll(async () => {- valkey = new Redis(port, host, { maxRetriesPerRequest: 1 })- await valkey.ping()- valkeyPool = {- useClient: async (_options, callback) => callback(valkey as unknown as RedisClient),- usePipeline: async (_options, callback) => {- const pipeline = valkey.pipeline() as RedisClientPipeline- callback(pipeline)- return pipeline.exec()- },- }- })-- afterAll(async () => {- await valkey.quit()- })-- it('pair-claims the team email cap buckets without a cross-slot rejection', async () => {- // Prove the backing store enforces cluster key-slot rules. Without this, swapping the- // compose service for a non-cluster Valkey would make the claim below pass vacuously.- await expect(valkey.mget('rate-limiter-test-key-a', 'rate-limiter-test-key-b')).rejects.toThrow('CROSSSLOT')-- // A fresh team id per run keeps the buckets cold, so a full grant is the only correct- // outcome. The cluster container keeps data between local runs.- const teamId = Math.floor(Math.random() * 1_000_000_000)- const buckets = teamEmailCapBuckets(teamId, 50, 100)- const limiter = new RateLimiterService(valkeyPool, { name: 'team-email-cluster-test' })-- const claim = await limiter.claimAllOrNothingPair([buckets[0], buckets[1]], 10)- expect(claim).toEqual({ granted: true, deniedIndex: null, retryAfterMs: null, reserved: false })- })-})diff --git a/nodejs/src/cdp/workflows-e2e.serial.test.ts b/nodejs/src/cdp/workflows-e2e.serial.test.tsindex 4846d3d62f79..2247b2b8fc0d 100644--- a/nodejs/src/cdp/workflows-e2e.serial.test.ts+++ b/nodejs/src/cdp/workflows-e2e.serial.test.ts@@ -3941,22 +3941,21 @@ describe('Workflows E2E (email queue)', () => {it("a team over its tier cap does not hold up another team's emails", async () => {// Two teams, each with one email workflow. Team A sits on a tier that allows- // 2 emails per hour and gets 6 sends queued; team B is on a high tier with 3+ // 2 emails per day and gets 6 sends queued; team B is on a high tier with 3// sends. The whole pipeline is real and the tier caps are enforced: B's emails// must all go out while A's over-cap sends park on future slots from A's own// refill, without cycling.//// The default email worker was built with the caps off, so swap in one that- // enforces them. Tier 0 allows 2/hour, tier 1 is effectively unlimited.+ // enforces them. Tier 0 allows 2/day, tier 1 is effectively unlimited.await emailWorker.stop()hub.EMAIL_TEAM_SENDING_CAP_MODE = 'enforce'- hub.EMAIL_TEAM_SENDING_CAP_HOURLY_BY_TIER = '2,10000'- hub.EMAIL_TEAM_SENDING_CAP_DAILY_BY_TIER = '48,240000'+ hub.EMAIL_TEAM_SENDING_CAP_DAILY_BY_TIER = '2,240000'const enforcedQueue = new CyclotronJobQueuePostgresV2(hub.CONSUMER_BATCH_SIZE, hub)emailWorker = new CdpCyclotronWorkerEmail(hub, deps, enforcedQueue)await emailWorker.start()- // The tier buckets are keyed by team id and the test Redis keeps data between+ // The tier bucket is keyed by team id and the test Redis keeps data between// runs, so team A's bucket must start full or a previous run drains this one.const capValkey = createRedisV2PoolFromConfig({connection: hub.CDP_REDIS_HOST@@ -3968,7 +3967,6 @@ describe('Workflows E2E (email queue)', () => {poolMinSize: hub.REDIS_POOL_MIN_SIZE,poolMaxSize: hub.REDIS_POOL_MAX_SIZE,})- await deleteKeysWithPrefix(capValkey, '@posthog/team-email-rate-hour')await deleteKeysWithPrefix(capValkey, '@posthog/team-email-rate-day')// Team A has no config row and defaults to tier 0. Team B gets tier 1.@@ -4076,7 +4074,7 @@ describe('Workflows E2E (email queue)', () => {}, 15000)// Team A's 4 over-cap sends are parked far in the future (its refill is one- // token per 30 minutes), each dequeued once on the email queue.+ // token per 12 hours), each dequeued once on the email queue.const jobs = await queryCyclotronJobs()const parkedA = jobs.filter((j: any) => j.function_id === flowA.id && j.queue_name === 'email' && j.status === 'available'