Skip to content

The open alternative to Salesforce, designed for AI.

Use one atomic token bucket for every rate limit (#27418)

Félix Malfait0ab948d5ddca
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/services/application-job-enqueue-throttler.service.ts b/packages/twenty-server/src/engine/core-modules/message-queue/services/application-job-enqueue-throttler.service.ts
index 3773cd96b63..d3cdcd15cd8 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/services/application-job-enqueue-throttler.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/services/application-job-enqueue-throttler.service.ts
@@ -29,29 +29,33 @@ export class ApplicationJobEnqueueThrottlerService {
const timeWindow = this.twentyConfigService.get(
'APPLICATION_JOB_ENQUEUE_RATE_LIMITING_TTL_IN_MS',
);
- const applicationKey = `enqueue:throttler:application:${applicationId}`;
const applicationLimit = this.twentyConfigService.get(
'APPLICATION_JOB_ENQUEUE_RATE_LIMITING_LIMIT',
);
- const registrationKey = `enqueue:throttler:application-registration:${applicationRegistrationId}`;
const registrationLimit = this.twentyConfigService.get(
'APPLICATION_REGISTRATION_JOB_ENQUEUE_RATE_LIMITING_LIMIT',
);
- const [applicationTokens, registrationTokens] = await Promise.all([
- this.throttlerService.getAvailableTokensCount(
- applicationKey,
- applicationLimit,
- timeWindow,
- ),
- this.throttlerService.getAvailableTokensCount(
- registrationKey,
- registrationLimit,
- timeWindow,
- ),
- ]);
+ const [admittedCount] = await this.throttlerService.tryConsumeTokenBuckets({
+ buckets: [
+ {
+ key: `enqueue:throttler:{${applicationRegistrationId}}:application:${applicationId}`,
+ burst: applicationLimit,
+ refillPerWindow: applicationLimit,
+ windowMs: timeWindow,
+ },
+ {
+ key: `enqueue:throttler:{${applicationRegistrationId}}:application-registration`,
+ burst: registrationLimit,
+ refillPerWindow: registrationLimit,
+ windowMs: timeWindow,
+ },
+ ],
+ tokensToConsume: jobCount,
+ allowPartial: false,
+ });
- if (applicationTokens < jobCount || registrationTokens < jobCount) {
+ if (admittedCount !== jobCount) {
await this.metricsService.incrementCounterForEvent({
key: MetricsKeys.JobEnqueueApplicationRateLimited,
shouldStoreInCache: false,
@@ -66,20 +70,5 @@ export class ApplicationJobEnqueueThrottlerService {
ThrottlerExceptionCode.LIMIT_REACHED,
);
}
-
- await Promise.all([
- this.throttlerService.consumeTokens(
- applicationKey,
- jobCount,
- applicationLimit,
- timeWindow,
- ),
- this.throttlerService.consumeTokens(
- registrationKey,
- jobCount,
- registrationLimit,
- timeWindow,
- ),
- ]);
}
}
diff --git a/packages/twenty-server/src/engine/core-modules/throttler/throttler.service.ts b/packages/twenty-server/src/engine/core-modules/throttler/throttler.service.ts
index 2d04629d988..ebdafac5da1 100644
--- a/packages/twenty-server/src/engine/core-modules/throttler/throttler.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/throttler/throttler.service.ts
@@ -14,6 +14,13 @@ import {
TRY_CONSUME_TOKEN_BUCKETS_SCRIPT,
} from 'src/engine/core-modules/throttler/constants/try-consume-token-buckets-script.constant';
+type TokenBucket = {
+ key: string;
+ burst: number;
+ refillPerWindow: number;
+ windowMs: number;
+};
+
@Injectable()
export class ThrottlerService {
constructor(
@@ -21,22 +28,59 @@ export class ThrottlerService {
private readonly cacheStorage: CacheStorageService,
) {}
+ async tryConsumeTokenBuckets({
+ buckets,
+ tokensToConsume,
+ allowPartial,
+ }: {
+ buckets: TokenBucket[];
+ tokensToConsume: number;
+ allowPartial: boolean;
+ }): Promise<
+ [
+ admittedCount: number,
+ exhaustedBucketPosition: number,
+ retryAfterMs: number,
+ ]
+ > {
+ return this.cacheStorage.runScript({
+ script: TRY_CONSUME_TOKEN_BUCKETS_SCRIPT,
+ keys: buckets.map(
+ (bucket) => `${TOKEN_BUCKET_THROTTLE_KEY_PREFIX}:${bucket.key}`,
+ ),
+ args: [
+ String(tokensToConsume),
+ JSON.stringify(
+ buckets.map((bucket) => ({
+ burst: bucket.burst,
+ refill: bucket.refillPerWindow,
+ windowMs: bucket.windowMs,
+ })),
+ ),
+ allowPartial
+ ? TOKEN_BUCKETS_ALLOW_PARTIAL_ARG
+ : TOKEN_BUCKETS_DENY_PARTIAL_ARG,
+ ],
+ });
+ }
+
async tokenBucketThrottleOrThrow(
key: string,
tokensToConsume: number,
maxTokens: number,
timeWindow: number,
): Promise<void> {
- const [admittedCount] = await this.cacheStorage.runScript<number[]>({
- script: TRY_CONSUME_TOKEN_BUCKETS_SCRIPT,
- keys: [`${TOKEN_BUCKET_THROTTLE_KEY_PREFIX}:${key}`],
- args: [
- String(tokensToConsume),
- JSON.stringify([
- { burst: maxTokens, refill: maxTokens, windowMs: timeWindow },
- ]),
- TOKEN_BUCKETS_DENY_PARTIAL_ARG,
+ const [admittedCount] = await this.tryConsumeTokenBuckets({
+ buckets: [
+ {
+ key,
+ burst: maxTokens,
+ refillPerWindow: maxTokens,
+ windowMs: timeWindow,
+ },
],
+ tokensToConsume,
+ allowPartial: false,
});
if (admittedCount !== tokensToConsume) {
@@ -53,60 +97,19 @@ export class ThrottlerService {
maxTokens: number,
timeWindow: number,
): Promise<number> {
- const [admittedCount] = await this.cacheStorage.runScript<number[]>({
- script: TRY_CONSUME_TOKEN_BUCKETS_SCRIPT,
- keys: [`${TOKEN_BUCKET_THROTTLE_KEY_PREFIX}:${key}`],
- args: [
- String(tokensToConsume),
- JSON.stringify([
- { burst: maxTokens, refill: maxTokens, windowMs: timeWindow },
- ]),
- TOKEN_BUCKETS_ALLOW_PARTIAL_ARG,
+ const [admittedCount] = await this.tryConsumeTokenBuckets({
+ buckets: [
+ {
+ key,
+ burst: maxTokens,
+ refillPerWindow: maxTokens,
+ windowMs: timeWindow,
+ },
],
+ tokensToConsume,
+ allowPartial: true,
});
return admittedCount;
}
-
- async consumeTokens(
- key: string,
- tokensToConsume: number,
- maxTokens: number,
- timeWindow: number,
- ) {
- const now = Date.now();
- const availableTokens = await this.getAvailableTokensCount(
- key,
- maxTokens,
- timeWindow,
- now,
- );
-
- await this.cacheStorage.set(
- key,
- {
- tokens: availableTokens - tokensToConsume,
- lastRefillAt: now,
- },
- timeWindow * 2,
- );
- }
-
- async getAvailableTokensCount(
- key: string,
- maxTokens: number,
- timeWindow: number,
- now = Date.now(),
- ): Promise<number> {
- const refillRate = maxTokens / timeWindow;
-
- const { tokens, lastRefillAt } = (await this.cacheStorage.get<{
- tokens: number;
- lastRefillAt: number;
- }>(key)) || { tokens: maxTokens, lastRefillAt: now };
-
- const refillAmount = Math.floor((now - lastRefillAt) * refillRate);
-
- return Math.min(tokens + refillAmount, maxTokens);
- }
}
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit-speed.service.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit-speed.service.ts
index 0ed007e0e2c..0380c4c6639 100644
--- a/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit-speed.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/services/usage-limit-speed.service.ts
@@ -3,16 +3,9 @@ import { Injectable, Logger } from '@nestjs/common';
import { isDefined } from 'twenty-shared/utils';
import { type WorkspaceAuthContext } from 'src/engine/core-modules/auth/types/workspace-auth-context.type';
-import { InjectCacheStorage } from 'src/engine/core-modules/cache-storage/decorators/cache-storage.decorator';
import { CacheStorageException } from 'src/engine/core-modules/cache-storage/exceptions/cache-storage.exception';
-import { CacheStorageService } from 'src/engine/core-modules/cache-storage/services/cache-storage.service';
-import { CacheStorageNamespace } from 'src/engine/core-modules/cache-storage/types/cache-storage-namespace.enum';
+import { ThrottlerService } from 'src/engine/core-modules/throttler/throttler.service';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
-import {
- TOKEN_BUCKETS_ALLOW_PARTIAL_ARG,
- TOKEN_BUCKETS_DENY_PARTIAL_ARG,
- TRY_CONSUME_TOKEN_BUCKETS_SCRIPT,
-} from 'src/engine/core-modules/throttler/constants/try-consume-token-buckets-script.constant';
import {
UsageLimitException,
UsageLimitExceptionCode,
@@ -34,8 +27,7 @@ export class UsageLimitSpeedService {
private readonly logger = new Logger(UsageLimitSpeedService.name);
constructor(
- @InjectCacheStorage(CacheStorageNamespace.EngineUsageLimit)
- private readonly cacheStorage: CacheStorageService,
+ private readonly throttlerService: ThrottlerService,
private readonly workspaceCacheService: WorkspaceCacheService,
private readonly twentyConfigService: TwentyConfigService,
private readonly usageLimitEntitlementService: UsageLimitEntitlementService,
@@ -176,23 +168,11 @@ export class UsageLimitSpeedService {
cost: number;
allowPartial: boolean;
}): Promise<SpeedBucketOutcome> {
- const bucketConfigs = buckets.map((bucket) => ({
- burst: bucket.burst,
- refill: bucket.refillPerWindow,
- windowMs: bucket.windowMs,
- }));
-
const [admittedCount, exhaustedIndex, retryAfterMs] =
- await this.cacheStorage.runScript<number[]>({
- script: TRY_CONSUME_TOKEN_BUCKETS_SCRIPT,
- keys: buckets.map((bucket) => bucket.key),
- args: [
- String(cost),
- JSON.stringify(bucketConfigs),
- allowPartial
- ? TOKEN_BUCKETS_ALLOW_PARTIAL_ARG
- : TOKEN_BUCKETS_DENY_PARTIAL_ARG,
- ],
+ await this.throttlerService.tryConsumeTokenBuckets({
+ buckets,
+ tokensToConsume: cost,
+ allowPartial,
});
if (admittedCount === cost) {
diff --git a/packages/twenty-server/src/engine/core-modules/usage-limit/usage-limit.module.ts b/packages/twenty-server/src/engine/core-modules/usage-limit/usage-limit.module.ts
index f9bd1913119..830634f570e 100644
--- a/packages/twenty-server/src/engine/core-modules/usage-limit/usage-limit.module.ts
+++ b/packages/twenty-server/src/engine/core-modules/usage-limit/usage-limit.module.ts
@@ -5,6 +5,7 @@ import { TypeOrmModule } from '@nestjs/typeorm';
import { ApiKeyEntity } from 'src/engine/core-modules/api-key/api-key.entity';
import { CacheLockModule } from 'src/engine/core-modules/cache-lock/cache-lock.module';
import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module';
+import { ThrottlerModule } from 'src/engine/core-modules/throttler/throttler.module';
import { UsageLimitQuotaService } from 'src/engine/core-modules/usage-limit/services/usage-limit-quota.service';
import { UsagePeriodService } from 'src/engine/core-modules/usage-limit/services/usage-period.service';
import { UsageAnalyticsModule } from 'src/engine/core-modules/usage/usage-analytics.module';
@@ -42,6 +43,7 @@ import { WorkspaceCacheModule } from 'src/engine/workspace-cache/workspace-cache
DiscoveryModule,
CacheLockModule,
MetricsModule,
+ ThrottlerModule,
UsageAnalyticsModule,
UsageModule,
],
diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-run-enqueue.workspace-service.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-run-enqueue.workspace-service.ts
index 6e15c18bf1d..b6e8dc7a665 100644
--- a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-run-enqueue.workspace-service.ts
+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-run-enqueue.workspace-service.ts
@@ -71,19 +71,10 @@ export class WorkflowRunEnqueueWorkspaceService {
return;
}
- let remainingWorkflowRunToEnqueueCount =
- await this.workflowThrottlingWorkspaceService.getRemainingRunsToEnqueueCount(
- workspaceId,
- );
-
let totalEnqueuedCount = 0;
+ let isSoftThrottled = false;
- while (remainingWorkflowRunToEnqueueCount > 0) {
- const batchSize = Math.min(
- remainingWorkflowRunToEnqueueCount,
- QUERY_MAX_RECORDS,
- );
-
+ while (!isSoftThrottled) {
const batchRuns = await workflowRunRepository.find({
where: NOT_STARTED_RUNS_FIND_OPTIONS,
select: {
@@ -92,16 +83,29 @@ export class WorkflowRunEnqueueWorkspaceService {
order: {
createdAt: 'ASC',
},
- take: batchSize,
+ take: QUERY_MAX_RECORDS,
});
if (batchRuns.length === 0) {
break;
}
- const batchIds = batchRuns.map(
- (workflowRun: WorkflowRunWorkspaceEntity) => workflowRun.id,
- );
+ // Runs are fetched before consuming so the soft throttle is only charged for runs that exist
+ const admittedRunCount =
+ await this.workflowThrottlingWorkspaceService.consumeRemainingRunsToEnqueueCount(
+ workspaceId,
+ batchRuns.length,
+ );
+
+ if (admittedRunCount === 0) {
+ break;
+ }
+
+ isSoftThrottled = admittedRunCount < batchRuns.length;
+
+ const batchIds = batchRuns
+ .slice(0, admittedRunCount)
+ .map((workflowRun: WorkflowRunWorkspaceEntity) => workflowRun.id);
await workflowRunRepository.update(batchIds, {
enqueuedAt: new Date().toISOString(),
@@ -119,8 +123,7 @@ export class WorkflowRunEnqueueWorkspaceService {
);
}
- totalEnqueuedCount += batchRuns.length;
- remainingWorkflowRunToEnqueueCount -= batchRuns.length;
+ totalEnqueuedCount += batchIds.length;
}
if (totalEnqueuedCount === 0) {
@@ -133,11 +136,6 @@ export class WorkflowRunEnqueueWorkspaceService {
return;
}
- await this.workflowThrottlingWorkspaceService.consumeRemainingRunsToEnqueueCount(
- workspaceId,
- totalEnqueuedCount,
- );
-
if (isCacheMode) {
await this.workflowThrottlingWorkspaceService.decreaseWorkflowRunNotStartedCount(
workspaceId,
diff --git a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-throttling.workspace-service.ts b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-throttling.workspace-service.ts
index dfb2e8c1d9c..4d6f0e46a4c 100644
--- a/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-throttling.workspace-service.ts
+++ b/packages/twenty-server/src/modules/workflow/workflow-runner/workflow-run-queue/workspace-services/workflow-throttling.workspace-service.ts
@@ -20,21 +20,13 @@ export class WorkflowThrottlingWorkspaceService {
private readonly twentyConfigService: TwentyConfigService,
) {}
- async getRemainingRunsToEnqueueCount(workspaceId: string) {
- return this.throttlerService.getAvailableTokensCount(
- this.getWorkflowExecutionSoftThrottleCacheKey(workspaceId),
- this.twentyConfigService.get('WORKFLOW_EXEC_SOFT_THROTTLE_LIMIT'),
- this.twentyConfigService.get('WORKFLOW_EXEC_SOFT_THROTTLE_TTL'),
- );
- }
-
async consumeRemainingRunsToEnqueueCount(
workspaceId: string,
- runsToConsume: number,
- ) {
- await this.throttlerService.consumeTokens(
+ requestedRunCount: number,
+ ): Promise<number> {
+ return this.throttlerService.tokenBucketConsumeUpTo(
this.getWorkflowExecutionSoftThrottleCacheKey(workspaceId),
- runsToConsume,
+ requestedRunCount,
this.twentyConfigService.get('WORKFLOW_EXEC_SOFT_THROTTLE_LIMIT'),
this.twentyConfigService.get('WORKFLOW_EXEC_SOFT_THROTTLE_TTL'),
);
diff --git a/packages/twenty-server/test/integration/graphql/suites/workflow/core-workflow-execution.integration-spec.ts b/packages/twenty-server/test/integration/graphql/suites/workflow/core-workflow-execution.integration-spec.ts
index a47074380b2..998871697b6 100644
--- a/packages/twenty-server/test/integration/graphql/suites/workflow/core-workflow-execution.integration-spec.ts
+++ b/packages/twenty-server/test/integration/graphql/suites/workflow/core-workflow-execution.integration-spec.ts
@@ -1864,7 +1864,7 @@ describe('core workflow execution and queue compatibility (e2e)', () => {
const capacity = jest
.spyOn(
global.workflowTestServices.throttling,
- 'getRemainingRunsToEnqueueCount',
+ 'consumeRemainingRunsToEnqueueCount',
)
.mockResolvedValue(0);
const runId = await runFixture(fixture);