Skip to content

The open alternative to Salesforce, designed for AI.

Add application job enqueue limits guard on all message queue drivers (#23570)

martmulla99e62fbca8a
diff --git a/packages/twenty-server/src/engine/core-modules/application/application-install/application-install.service.ts b/packages/twenty-server/src/engine/core-modules/application/application-install/application-install.service.ts
index 748feba076a..4dad1707bbb 100644
--- a/packages/twenty-server/src/engine/core-modules/application/application-install/application-install.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/application/application-install/application-install.service.ts
@@ -572,6 +572,7 @@ export class ApplicationInstallService {
},
{ retryLimit: 3 },
);
+
return;
}
diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job.ts
index f9a891e06be..1e03dcb7032 100644
--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job.ts
+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job.ts
@@ -5,6 +5,7 @@ import { isDefined } from 'twenty-shared/utils';
import { WorkspaceActivationStatus } from 'twenty-shared/workspace';
import { Repository } from 'typeorm';
+import { findActiveFlatApplicationById } from 'src/engine/core-modules/application/utils/find-active-flat-application-by-id.util';
import { CronTriggerDeduplicationService } from 'src/engine/core-modules/cron/services/cron-trigger-deduplication.service';
import { SentryCronMonitor } from 'src/engine/core-modules/cron/sentry-cron-monitor.decorator';
import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';
@@ -12,6 +13,7 @@ import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decora
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
+import { ApplicationJobEnqueueThrottlerService } from 'src/engine/core-modules/message-queue/services/application-job-enqueue-throttler.service';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';
import {
@@ -34,6 +36,7 @@ export class CronTriggerCronJob {
private readonly workspaceCacheService: WorkspaceCacheService,
private readonly exceptionHandlerService: ExceptionHandlerService,
private readonly cronTriggerDeduplicationService: CronTriggerDeduplicationService,
+ private readonly applicationJobEnqueueThrottlerService: ApplicationJobEnqueueThrottlerService,
) {}
@Process(CronTriggerCronJob.name)
@@ -50,9 +53,10 @@ export class CronTriggerCronJob {
for (const activeWorkspace of activeWorkspaces) {
try {
- const { flatLogicFunctionMaps } =
+ const { flatLogicFunctionMaps, flatApplicationMaps } =
await this.workspaceCacheService.getOrRecompute(activeWorkspace.id, [
'flatLogicFunctionMaps',
+ 'flatApplicationMaps',
]);
const logicFunctions = Object.values(
@@ -85,6 +89,22 @@ export class CronTriggerCronJob {
continue;
}
+ const application = findActiveFlatApplicationById(
+ flatApplicationMaps,
+ logicFunction.applicationId,
+ );
+ const applicationRegistrationId =
+ application?.applicationRegistrationId;
+
+ if (!isDefined(applicationRegistrationId)) {
+ continue;
+ }
+
+ await this.applicationJobEnqueueThrottlerService.throttleOrThrow({
+ applicationId: logicFunction.applicationId,
+ applicationRegistrationId,
+ });
+
await this.messageQueueService.add<LogicFunctionTriggerJobData>(
LogicFunctionTriggerJob.name,
{
diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.ts
index dbdd5d4439c..c89b5ae3dc5 100644
--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.ts
+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.ts
@@ -2,10 +2,12 @@ import { isDefined } from 'twenty-shared/utils';
import type { ObjectRecordEvent } from 'twenty-shared/database-events';
+import { findActiveFlatApplicationById } from 'src/engine/core-modules/application/utils/find-active-flat-application-by-id.util';
import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
+import { ApplicationJobEnqueueThrottlerService } from 'src/engine/core-modules/message-queue/services/application-job-enqueue-throttler.service';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import { transformEventBatchToEventPayloads } from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/utils/transform-event-batch-to-event-payloads';
import {
@@ -21,14 +23,15 @@ export class CallDatabaseEventTriggerJobsJob {
@InjectMessageQueue(MessageQueue.logicFunctionQueue)
private readonly messageQueueService: MessageQueueService,
private readonly workspaceCacheService: WorkspaceCacheService,
+ private readonly applicationJobEnqueueThrottlerService: ApplicationJobEnqueueThrottlerService,
) {}
@Process(CallDatabaseEventTriggerJobsJob.name)
async handle(workspaceEventBatch: WorkspaceEventBatch<ObjectRecordEvent>) {
- const { flatLogicFunctionMaps } =
+ const { flatLogicFunctionMaps, flatApplicationMaps } =
await this.workspaceCacheService.getOrRecompute(
workspaceEventBatch.workspaceId,
- ['flatLogicFunctionMaps'],
+ ['flatLogicFunctionMaps', 'flatApplicationMaps'],
);
const logicFunctionsWithDatabaseEventTrigger = Object.values(
@@ -51,16 +54,57 @@ export class CallDatabaseEventTriggerJobsJob {
}),
);
- const logicFunctionPayloads = transformEventBatchToEventPayloads({
- logicFunctions: logicFunctionsToTrigger,
- workspaceEventBatch,
- });
+ const logicFunctionsByApplicationId = new Map<
+ string,
+ typeof logicFunctionsToTrigger
+ >();
- await this.messageQueueService.bulkAdd<LogicFunctionTriggerJobData>(
- LogicFunctionTriggerJob.name,
- logicFunctionPayloads,
- { retryLimit: 3 },
- );
+ for (const logicFunction of logicFunctionsToTrigger) {
+ const applicationLogicFunctions =
+ logicFunctionsByApplicationId.get(logicFunction.applicationId) ?? [];
+
+ applicationLogicFunctions.push(logicFunction);
+ logicFunctionsByApplicationId.set(
+ logicFunction.applicationId,
+ applicationLogicFunctions,
+ );
+ }
+
+ for (const [
+ applicationId,
+ logicFunctions,
+ ] of logicFunctionsByApplicationId) {
+ const application = findActiveFlatApplicationById(
+ flatApplicationMaps,
+ applicationId,
+ );
+ const applicationRegistrationId = application?.applicationRegistrationId;
+
+ if (!isDefined(applicationRegistrationId)) {
+ continue;
+ }
+
+ const logicFunctionPayloads = transformEventBatchToEventPayloads({
+ logicFunctions,
+ workspaceEventBatch,
+ });
+
+ if (logicFunctionPayloads.length === 0) {
+ continue;
+ }
+
+ await this.applicationJobEnqueueThrottlerService.throttleOrThrow({
+ applicationId,
+ applicationRegistrationId,
+ jobCount: logicFunctionPayloads.length,
+ });
+
+ await this.messageQueueService.bulkAdd<LogicFunctionTriggerJobData>(
+ LogicFunctionTriggerJob.name,
+ logicFunctionPayloads,
+ { retryLimit: 3 },
+ );
+ }
}
private shouldTriggerJob({
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-core.module.ts b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-core.module.ts
index b6160b1e656..368d22409b0 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-core.module.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-core.module.ts
@@ -20,9 +20,11 @@ import {
ConfigurableModuleClass,
type OPTIONS_TYPE,
} from 'src/engine/core-modules/message-queue/message-queue.module-definition';
+import { ApplicationJobEnqueueThrottlerService } from 'src/engine/core-modules/message-queue/services/application-job-enqueue-throttler.service';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import { getQueueToken } from 'src/engine/core-modules/message-queue/utils/get-queue-token.util';
import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module';
+import { ThrottlerModule } from 'src/engine/core-modules/throttler/throttler.module';
@Global()
@Module({})
@@ -43,13 +45,20 @@ export class MessageQueueCoreModule extends ConfigurableModuleClass {
return {
...dynamicModule,
+ imports: [
+ ...(dynamicModule.imports ?? []),
+ MetricsModule,
+ ThrottlerModule,
+ ],
providers: [
...(dynamicModule.providers ?? []),
+ ApplicationJobEnqueueThrottlerService,
driverProvider,
...queueProviders,
],
exports: [
...(dynamicModule.exports ?? []),
+ ApplicationJobEnqueueThrottlerService,
...Object.values(MessageQueue).map((queueName) =>
getQueueToken(queueName),
),
@@ -78,14 +87,20 @@ export class MessageQueueCoreModule extends ConfigurableModuleClass {
return {
...dynamicModule,
- imports: [...(dynamicModule.imports ?? []), MetricsModule],
+ imports: [
+ ...(dynamicModule.imports ?? []),
+ MetricsModule,
+ ThrottlerModule,
+ ],
providers: [
...(dynamicModule.providers ?? []),
+ ApplicationJobEnqueueThrottlerService,
driverProvider,
...queueProviders,
],
exports: [
...(dynamicModule.exports ?? []),
+ ApplicationJobEnqueueThrottlerService,
...Object.values(MessageQueue).map((queueName) =>
getQueueToken(queueName),
),
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/services/__tests__/application-job-enqueue-throttler.service.spec.ts b/packages/twenty-server/src/engine/core-modules/message-queue/services/__tests__/application-job-enqueue-throttler.service.spec.ts
new file mode 100644
index 00000000000..7ff94bfaaff
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/services/__tests__/application-job-enqueue-throttler.service.spec.ts
@@ -0,0 +1,123 @@
+import { Test, type TestingModule } from '@nestjs/testing';
+
+import { ApplicationJobEnqueueThrottlerService } from 'src/engine/core-modules/message-queue/services/application-job-enqueue-throttler.service';
+import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
+import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
+import { ThrottlerException } from 'src/engine/core-modules/throttler/throttler.exception';
+import { ThrottlerService } from 'src/engine/core-modules/throttler/throttler.service';
+import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
+
+const CONFIG = {
+ APPLICATION_JOB_ENQUEUE_RATE_LIMITING_TTL_IN_MS: 60_000,
+ APPLICATION_JOB_ENQUEUE_RATE_LIMITING_LIMIT: 500,
+ APPLICATION_REGISTRATION_JOB_ENQUEUE_RATE_LIMITING_LIMIT: 2000,
+} as const;
+
+const context = {
+ applicationId: 'application-id',
+ applicationRegistrationId: 'registration-id',
+};
+
+const APPLICATION_KEY = 'enqueue:throttler:application:application-id';
+const REGISTRATION_KEY =
+ 'enqueue:throttler:application-registration:registration-id';
+
+describe('ApplicationJobEnqueueThrottlerService', () => {
+ let service: ApplicationJobEnqueueThrottlerService;
+ let throttlerService: {
+ getAvailableTokensCount: jest.Mock;
+ consumeTokens: jest.Mock;
+ };
+ let metricsService: { incrementCounterForEvent: jest.Mock };
+
+ const mockAvailableTokens = (byKey: Record<string, number>) => {
+ throttlerService.getAvailableTokensCount.mockImplementation(
+ async (key: string) => byKey[key] ?? 0,
+ );
+ };
+
+ beforeEach(async () => {
+ throttlerService = {
+ getAvailableTokensCount: jest.fn().mockResolvedValue(1000),
+ consumeTokens: jest.fn().mockResolvedValue(undefined),
+ };
+ metricsService = {
+ incrementCounterForEvent: jest.fn().mockResolvedValue(undefined),
+ };
+
+ const module: TestingModule = await Test.createTestingModule({
+ providers: [
+ ApplicationJobEnqueueThrottlerService,
+ { provide: ThrottlerService, useValue: throttlerService },
+ { provide: MetricsService, useValue: metricsService },
+ {
+ provide: TwentyConfigService,
+ useValue: {
+ get: (key: keyof typeof CONFIG) => CONFIG[key],
+ },
+ },
+ ],
+ }).compile();
+
+ service = module.get(ApplicationJobEnqueueThrottlerService);
+ });
+
+ it('debits both application and registration buckets with distinct limits', async () => {
+ await service.throttleOrThrow(context);
+
+ expect(throttlerService.consumeTokens).toHaveBeenCalledWith(
+ APPLICATION_KEY,
+ 1,
+ CONFIG.APPLICATION_JOB_ENQUEUE_RATE_LIMITING_LIMIT,
+ CONFIG.APPLICATION_JOB_ENQUEUE_RATE_LIMITING_TTL_IN_MS,
+ );
+ expect(throttlerService.consumeTokens).toHaveBeenCalledWith(
+ REGISTRATION_KEY,
+ 1,
+ CONFIG.APPLICATION_REGISTRATION_JOB_ENQUEUE_RATE_LIMITING_LIMIT,
+ CONFIG.APPLICATION_JOB_ENQUEUE_RATE_LIMITING_TTL_IN_MS,
+ );
+ });
+
+ it('consumes one token per job for bulk enqueue', async () => {
+ await service.throttleOrThrow({ ...context, jobCount: 5 });
+
+ expect(throttlerService.consumeTokens).toHaveBeenCalledWith(
+ APPLICATION_KEY,
+ 5,
+ expect.any(Number),
+ expect.any(Number),
+ );
+ });
+
+ it('does not debit any bucket when the registration bucket is exhausted', async () => {
+ mockAvailableTokens({
+ [APPLICATION_KEY]: 1000,
+ [REGISTRATION_KEY]: 0,
+ });
+
+ await expect(service.throttleOrThrow(context)).rejects.toThrow(
+ ThrottlerException,
+ );
+
+ expect(throttlerService.consumeTokens).not.toHaveBeenCalled();
+ expect(metricsService.incrementCounterForEvent).toHaveBeenCalledWith(
+ expect.objectContaining({
+ key: MetricsKeys.JobEnqueueApplicationRateLimited,
+ }),
+ );
+ });
+
+ it('does not debit any bucket when the application bucket is exhausted', async () => {
+ mockAvailableTokens({
+ [APPLICATION_KEY]: 0,
+ [REGISTRATION_KEY]: 1000,
+ });
+
+ await expect(service.throttleOrThrow(context)).rejects.toThrow(
+ ThrottlerException,
+ );
+
+ expect(throttlerService.consumeTokens).not.toHaveBeenCalled();
+ });
+});
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/services/application-job-enqueue-throttler.service.ts b/packages/twenty-server/src/engine/core-modules/message-queue/services/application-job-enqueue-throttler.service.ts
new file mode 100644
index 00000000000..3773cd96b63
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/services/application-job-enqueue-throttler.service.ts
@@ -0,0 +1,85 @@
+import { Injectable } from '@nestjs/common';
+
+import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';
+import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';
+import {
+ ThrottlerException,
+ ThrottlerExceptionCode,
+} from 'src/engine/core-modules/throttler/throttler.exception';
+import { ThrottlerService } from 'src/engine/core-modules/throttler/throttler.service';
+import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
+
+@Injectable()
+export class ApplicationJobEnqueueThrottlerService {
+ constructor(
+ private readonly throttlerService: ThrottlerService,
+ private readonly twentyConfigService: TwentyConfigService,
+ private readonly metricsService: MetricsService,
+ ) {}
+
+ async throttleOrThrow({
+ applicationId,
+ applicationRegistrationId,
+ jobCount = 1,
+ }: {
+ applicationId: string;
+ applicationRegistrationId: string;
+ jobCount?: number;
+ }): Promise<void> {
+ const timeWindow = this.twentyConfigService.get(
+ 'APPLICATION_JOB_ENQUEUE_RATE_LIMITING_TTL_IN_MS',
+ );
+ const applicationKey = `enqueue:throttler:application:${applicationId}`;
+ const applicationLimit = this.twentyConfigService.get(
+ 'APPLICATION_JOB_ENQUEUE_RATE_LIMITING_LIMIT',
+ );
+ const registrationKey = `enqueue:throttler:application-registration:${applicationRegistrationId}`;
+ const registrationLimit = this.twentyConfigService.get(
+ 'APPLICATION_REGISTRATION_JOB_ENQUEUE_RATE_LIMITING_LIMIT',
+ );
+
+ const [applicationTokens, registrationTokens] = await Promise.all([
+ this.throttlerService.getAvailableTokensCount(
+ applicationKey,
+ applicationLimit,
+ timeWindow,
+ ),
+ this.throttlerService.getAvailableTokensCount(
+ registrationKey,
+ registrationLimit,
+ timeWindow,
+ ),
+ ]);
+
+ if (applicationTokens < jobCount || registrationTokens < jobCount) {
+ await this.metricsService.incrementCounterForEvent({
+ key: MetricsKeys.JobEnqueueApplicationRateLimited,
+ shouldStoreInCache: false,
+ attributes: {
+ application_id: applicationId,
+ application_registration_id: applicationRegistrationId,
+ },
+ });
+
+ throw new ThrottlerException(
+ 'Application job enqueue limit reached',
+ ThrottlerExceptionCode.LIMIT_REACHED,
+ );
+ }
+
+ await Promise.all([
+ this.throttlerService.consumeTokens(
+ applicationKey,
+ jobCount,
+ applicationLimit,
+ timeWindow,
+ ),
+ this.throttlerService.consumeTokens(
+ registrationKey,
+ jobCount,
+ registrationLimit,
+ timeWindow,
+ ),
+ ]);
+ }
+}
diff --git a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts
index c9e766a5f7f..aa242b7676d 100644
--- a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts
+++ b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts
@@ -59,6 +59,7 @@ export enum MetricsKeys {
SignUpSuccess = 'sign-up/success',
CommonApiQueryRateLimited = 'common-api-query/rate-limited',
CommonApiApplicationQueryRateLimited = 'common-api-query/application-rate-limited',
+ JobEnqueueApplicationRateLimited = 'job/enqueue-application-rate-limited',
JobCompleted = 'job/completed',
JobFailed = 'job/failed',
JobStalled = 'job/stalled',
diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts
index a5caf8dcd08..35575b3c9aa 100644
--- a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts
+++ b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts
@@ -1521,6 +1521,33 @@ export class ConfigVariables {
@CastToPositiveNumber()
APPLICATION_API_RATE_LIMITING_LIMIT = 500;
+ @ConfigVariablesMetadata({
+ group: ConfigVariablesGroup.RATE_LIMITING,
+ description:
+ 'Time-to-live for application job enqueue rate limiting in milliseconds',
+ type: ConfigVariableType.NUMBER,
+ })
+ @CastToPositiveNumber()
+ APPLICATION_JOB_ENQUEUE_RATE_LIMITING_TTL_IN_MS = 60_000;
+
+ @ConfigVariablesMetadata({
+ group: ConfigVariablesGroup.RATE_LIMITING,
+ description:
+ 'Maximum number of jobs a single application installation can enqueue in the rate limiting window',
+ type: ConfigVariableType.NUMBER,
+ })
+ @CastToPositiveNumber()
+ APPLICATION_JOB_ENQUEUE_RATE_LIMITING_LIMIT = 500;
+
+ @ConfigVariablesMetadata({
+ group: ConfigVariablesGroup.RATE_LIMITING,
+ description:
+ 'Maximum number of jobs enqueued per application registration across all workspaces in the rate limiting window',
+ type: ConfigVariableType.NUMBER,
+ })
+ @CastToPositiveNumber()
+ APPLICATION_REGISTRATION_JOB_ENQUEUE_RATE_LIMITING_LIMIT = 2000;
+
@CastToPositiveNumber()
@ConfigVariablesMetadata({
group: ConfigVariablesGroup.RATE_LIMITING,