The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Use one atomic token bucket for every rate limit (#27418)
Félix Malfait0ab948d5ddca
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/services/application-job-enqueue-throttler.service.ts b/packages/twenty-server/src/engine/core-modules/message-queue/services/application-job-enqueue-throttler.service.tsindex 3773cd96b63..d3cdcd15cd8 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/services/application-job-enqueue-throttler.service.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/services/application-job-enqueue-throttler.service.ts@@ -29,29 +29,33 @@ export class ApplicationJobEnqueueThrottlerService {const timeWindow = this.twentyConfigService.get('APPLICATION_JOB_ENQUEUE_RATE_LIMITING_TTL_IN_MS',);- const applicationKey = `enqueue:throttler:application:${applicationId}`;const applicationLimit = this.twentyConfigService.get('APPLICATION_JOB_ENQUEUE_RATE_LIMITING_LIMIT',);- const registrationKey = `enqueue:throttler:application-registration:${applicationRegistrationId}`;const registrationLimit = this.twentyConfigService.get('APPLICATION_REGISTRATION_JOB_ENQUEUE_RATE_LIMITING_LIMIT',);- const [applicationTokens, registrationTokens] = await Promise.all([- this.throttlerService.getAvailableTokensCount(- applicationKey,- applicationLimit,- timeWindow,- ),- this.throttlerService.getAvailableTokensCount(- registrationKey,- registrationLimit,- timeWindow,- ),- ]);+ const [admittedCount] = await this.throttlerService.tryConsumeTokenBuckets({+ buckets: [+ {+ key: `enqueue:throttler:{${applicationRegistrationId}}:application:${applicationId}`,+ burst: applicationLimit,+ refillPerWindow: applicationLimit,+ windowMs: timeWindow,+ },+ {+ key: `enqueue:throttler:{${applicationRegistrationId}}:application-registration`,+ burst: registrationLimit,+ refillPerWindow: registrationLimit,+ windowMs: timeWindow,+ },+ ],+ tokensToConsume: jobCount,+ allowPartial: false,+ });- if (applicationTokens < jobCount || registrationTokens < jobCount) {+ if (admittedCount !== jobCount) {await this.metricsService.incrementCounterForEvent({key: MetricsKeys.JobEnqueueApplicationRateLimited,shouldStoreInCache: false,@@ -66,20 +70,5 @@ export class ApplicationJobEnqueueThrottlerService {ThrottlerExceptionCode.LIMIT_REACHED,);}-- await Promise.all([- this.throttlerService.consumeTokens(- applicationKey,- jobCount,- applicationLimit,- timeWindow,- ),- this.throttlerService.consumeTokens(- registrationKey,- jobCount,- registrationLimit,- timeWindow,- ),- ]);}}diff --git a/packages/twenty-server/src/engine/core-modules/throttler/throttler.service.ts b/packages/twenty-server/src/engine/core-modules/throttler/throttler.service.tsindex 2d04629d988..ebdafac5da1 100644--- a/packages/twenty-server/src/engine/core-modules/throttler/throttler.service.ts+++ b/packages/twenty-server/src/engine/core-modules/throttler/throttler.service.ts@@ -14,6 +14,13 @@ import {TRY_CONSUME_TOKEN_BUCKETS_SCRIPT,} from 'src/engine/core-modules/throttler/constants/try-consume-token-buckets-script.constant';+type TokenBucket = {+ key: string;+ burst: number;+ refillPerWindow: number;+ windowMs: number;+};+@Injectable()export class ThrottlerService {constructor(@@ -21,22 +28,59 @@ export class ThrottlerService {private readonly cacheStorage: CacheStorageService,) {}+ async tryConsumeTokenBuckets({+ buckets,+ tokensToConsume,+ allowPartial,+ }: {+ buckets: TokenBucket[];+ tokensToConsume: number;+ allowPartial: boolean;+ }): Promise<+ [+ admittedCount: number,+ exhaustedBucketPosition: number,+ retryAfterMs: number,+ ]+ > {+ return this.cacheStorage.runScript({+ script: TRY_CONSUME_TOKEN_BUCKETS_SCRIPT,+ keys: buckets.map(+ (bucket) => `${TOKEN_BUCKET_THROTTLE_KEY_PREFIX}:${bucket.key}`,+ ),+ args: [+ String(tokensToConsume),+ JSON.stringify(+ buckets.map((bucket) => ({+ burst: bucket.burst,+ refill: bucket.refillPerWindow,+ windowMs: bucket.windowMs,+ })),+ ),+ allowPartial+ ? TOKEN_BUCKETS_ALLOW_PARTIAL_ARG+ : TOKEN_BUCKETS_DENY_PARTIAL_ARG,+ ],+ });+ }+async tokenBucketThrottleOrThrow(key: string,tokensToConsume: number,maxTokens: number,timeWindow: number,): Promise<void> {- const [admittedCount] = await this.cacheStorage.runScript<number[]>({- script: TRY_CONSUME_TOKEN_BUCKETS_SCRIPT,- keys: [`${TOKEN_BUCKET_THROTTLE_KEY_PREFIX}:${key}`],- args: [- String(tokensToConsume),- JSON.stringify([- { burst: maxTokens, refill: maxTokens, windowMs: timeWindow },- ]),- TOKEN_BUCKETS_DENY_PARTIAL_ARG,+ const [admittedCount] = await this.tryConsumeTokenBuckets({+ buckets: [+ {+ key,+ burst: maxTokens,+ refillPerWindow: maxTokens,+ windowMs: timeWindow,+ },],+ tokensToConsume,+ allowPartial: false,});if (admittedCount !== tokensToConsume) {@@ -53,60 +97,19 @@ export class ThrottlerService {maxTokens: number,timeWindow: number,): Promise<number> {- const [admittedCount] = await this.cacheStorage.runScript<number[]>({- script: TRY_CONSUME_TOKEN_BUCKETS_SCRIPT,- keys: [`${TOKEN_BUCKET_THROTTLE_KEY_PREFIX}:${key}`],- args: [- String(tokensToConsume),- JSON.stringify([- { burst: maxTokens, refill: maxTokens, windowMs: timeWindow },- ]),- TOKEN_BUCKETS_ALLOW_PARTIAL_ARG,+ const [admittedCount] = await this.tryConsumeTokenBuckets({+ buckets: [+ {+ key,+ burst: maxTokens,+ refillPerWindow: maxTokens,+ windowMs: timeWindow,+ },],+ tokensToConsume,+ allowPartial: true,});return admittedCount;}-- async consumeTokens(- key: string,- tokensToConsume: number,- maxTokens: number,- timeWindow: number,- ) {- const now = Date.now();- const availableTokens = await this.getAvailableTokensCount(- key,- maxTokens,- timeWindow,- now,- );-- await this.cacheStorage.set(- key,- {- tokens: availableTokens - tokensToConsume,- lastRefillAt: now,- },- timeWindow * 2,- );- }-- async getAvailableTokensCount(- key: string,- maxTokens: number,- timeWindow: number,- now = Date.now(),- ): Promise<number> {- const refillRate = maxTokens / timeWindow;-- const { tokens, lastRefillAt } = (await this.cacheStorage.get<{- tokens: number;- lastRefillAt: number;- }>(key)) || { tokens: maxTokens, lastRefillAt: now };-- const refillAmount = Math.floor((now - lastRefillAt) * refillRate);-- return Math.min(tokens + refillAmount, maxTokens);- }}diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit-speed.service.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit-speed.service.tsindex 0ed007e0e2c..0380c4c6639 100644--- a/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit-speed.service.ts+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit-speed.service.ts@@ -3,16 +3,9 @@ import { Injectable, Logger } from '@nestjs/common';import { isDefined } from 'twenty-shared/utils';import { type WorkspaceAuthContext } from 'src/engine/core-modules/auth/types/workspace-auth-context.type';-import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';import { CacheStorageException } from 'src/engine/core-modules/cache-storage/exceptions/cache-storage.exception';-import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';-import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';+import { ThrottlerService } from 'src/engine/core-modules/throttler/throttler.service';import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';-import {- TOKEN_BUCKETS_ALLOW_PARTIAL_ARG,- TOKEN_BUCKETS_DENY_PARTIAL_ARG,- TRY_CONSUME_TOKEN_BUCKETS_SCRIPT,-} from 'src/engine/core-modules/throttler/constants/try-consume-token-buckets-script.constant';import {UsageLimitException,UsageLimitExceptionCode,@@ -34,8 +27,7 @@ export class UsageLimitSpeedService {private readonly logger = new Logger(UsageLimitSpeedService.name);constructor(- @InjectCacheStorage(CacheStorageNamespace.EngineUsageLimit)- private readonly cacheStorage: CacheStorageService,+ private readonly throttlerService: ThrottlerService,private readonly workspaceCacheService: WorkspaceCacheService,private readonly twentyConfigService: TwentyConfigService,private readonly usageLimitEntitlementService: UsageLimitEntitlementService,@@ -176,23 +168,11 @@ export class UsageLimitSpeedService {cost: number;allowPartial: boolean;}): Promise<SpeedBucketOutcome> {- const bucketConfigs = buckets.map((bucket) => ({- burst: bucket.burst,- refill: bucket.refillPerWindow,- windowMs: bucket.windowMs,- }));-const [admittedCount, exhaustedIndex, retryAfterMs] =- await this.cacheStorage.runScript<number[]>({- script: TRY_CONSUME_TOKEN_BUCKETS_SCRIPT,- keys: buckets.map((bucket) => bucket.key),- args: [- String(cost),- JSON.stringify(bucketConfigs),- allowPartial- ? TOKEN_BUCKETS_ALLOW_PARTIAL_ARG- : TOKEN_BUCKETS_DENY_PARTIAL_ARG,- ],+ await this.throttlerService.tryConsumeTokenBuckets({+ buckets,+ tokensToConsume: cost,+ allowPartial,});if (admittedCount === cost) {diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/usage-limit.module.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/usage-limit.module.tsindex f9bd1913119..830634f570e 100644--- a/packages/twenty-server/src/engine/core-modules/usage-limit/usage-limit.module.ts+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/usage-limit.module.ts@@ -5,6 +5,7 @@ import { TypeOrmModule } from '@nestjs/typeorm';import { ApiKeyEntity } from 'src/engine/core-modules/api-key/api-key.entity';import { CacheLockModule } from 'src/engine/core-modules/cache-lock/cache-lock.module';import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module';+import { ThrottlerModule } from 'src/engine/core-modules/throttler/throttler.module';import { UsageLimitQuotaService } from 'src/engine/core-modules/usage-limit/services/usage-limit-quota.service';import { UsagePeriodService } from 'src/engine/core-modules/usage-limit/services/usage-period.service';import { UsageAnalyticsModule } from 'src/engine/core-modules/usage/usage-analytics.module';@@ -42,6 +43,7 @@ import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cacheDiscoveryModule,CacheLockModule,MetricsModule,+ ThrottlerModule,UsageAnalyticsModule,UsageModule,],diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-run-enqueue.workspace-service.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-run-enqueue.workspace-service.tsindex 6e15c18bf1d..b6e8dc7a665 100644--- a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-run-enqueue.workspace-service.ts+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-run-enqueue.workspace-service.ts@@ -71,19 +71,10 @@ export class WorkflowRunEnqueueWorkspaceService {return;}- let remainingWorkflowRunToEnqueueCount =- await this.workflowThrottlingWorkspaceService.getRemainingRunsToEnqueueCount(- workspaceId,- );-let totalEnqueuedCount = 0;+ let isSoftThrottled = false;- while (remainingWorkflowRunToEnqueueCount > 0) {- const batchSize = Math.min(- remainingWorkflowRunToEnqueueCount,- QUERY_MAX_RECORDS,- );-+ while (!isSoftThrottled) {const batchRuns = await workflowRunRepository.find({where: NOT_STARTED_RUNS_FIND_OPTIONS,select: {@@ -92,16 +83,29 @@ export class WorkflowRunEnqueueWorkspaceService {order: {createdAt: 'ASC',},- take: batchSize,+ take: QUERY_MAX_RECORDS,});if (batchRuns.length === 0) {break;}- const batchIds = batchRuns.map(- (workflowRun: WorkflowRunWorkspaceEntity) => workflowRun.id,- );+ // Runs are fetched before consuming so the soft throttle is only charged for runs that exist+ const admittedRunCount =+ await this.workflowThrottlingWorkspaceService.consumeRemainingRunsToEnqueueCount(+ workspaceId,+ batchRuns.length,+ );++ if (admittedRunCount === 0) {+ break;+ }++ isSoftThrottled = admittedRunCount < batchRuns.length;++ const batchIds = batchRuns+ .slice(0, admittedRunCount)+ .map((workflowRun: WorkflowRunWorkspaceEntity) => workflowRun.id);await workflowRunRepository.update(batchIds, {enqueuedAt: new Date().toISOString(),@@ -119,8 +123,7 @@ export class WorkflowRunEnqueueWorkspaceService {);}- totalEnqueuedCount += batchRuns.length;- remainingWorkflowRunToEnqueueCount -= batchRuns.length;+ totalEnqueuedCount += batchIds.length;}if (totalEnqueuedCount === 0) {@@ -133,11 +136,6 @@ export class WorkflowRunEnqueueWorkspaceService {return;}- await this.workflowThrottlingWorkspaceService.consumeRemainingRunsToEnqueueCount(- workspaceId,- totalEnqueuedCount,- );-if (isCacheMode) {await this.workflowThrottlingWorkspaceService.decreaseWorkflowRunNotStartedCount(workspaceId,diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-throttling.workspace-service.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-throttling.workspace-service.tsindex dfb2e8c1d9c..4d6f0e46a4c 100644--- a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-throttling.workspace-service.ts+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-throttling.workspace-service.ts@@ -20,21 +20,13 @@ export class WorkflowThrottlingWorkspaceService {private readonly twentyConfigService: TwentyConfigService,) {}- async getRemainingRunsToEnqueueCount(workspaceId: string) {- return this.throttlerService.getAvailableTokensCount(- this.getWorkflowExecutionSoftThrottleCacheKey(workspaceId),- this.twentyConfigService.get('WORKFLOW_EXEC_SOFT_THROTTLE_LIMIT'),- this.twentyConfigService.get('WORKFLOW_EXEC_SOFT_THROTTLE_TTL'),- );- }-async consumeRemainingRunsToEnqueueCount(workspaceId: string,- runsToConsume: number,- ) {- await this.throttlerService.consumeTokens(+ requestedRunCount: number,+ ): Promise<number> {+ return this.throttlerService.tokenBucketConsumeUpTo(this.getWorkflowExecutionSoftThrottleCacheKey(workspaceId),- runsToConsume,+ requestedRunCount,this.twentyConfigService.get('WORKFLOW_EXEC_SOFT_THROTTLE_LIMIT'),this.twentyConfigService.get('WORKFLOW_EXEC_SOFT_THROTTLE_TTL'),);diff --git a/packages/twenty-server/test/integration/graphql/suites/workflow/core-workflow-execution.integration-spec.ts b/packages/twenty-server/test/integration/graphql/suites/workflow/core-workflow-execution.integration-spec.tsindex a47074380b2..998871697b6 100644--- a/packages/twenty-server/test/integration/graphql/suites/workflow/core-workflow-execution.integration-spec.ts+++ b/packages/twenty-server/test/integration/graphql/suites/workflow/core-workflow-execution.integration-spec.ts@@ -1864,7 +1864,7 @@ describe('core workflow execution and queue compatibility (e2e)', () => {const capacity = jest.spyOn(global.workflowTestServices.throttling,- 'getRemainingRunsToEnqueueCount',+ 'consumeRemainingRunsToEnqueueCount',).mockResolvedValue(0);const runId = await runFixture(fixture);