The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Add application job enqueue limits guard on all message queue drivers (#23570)
martmulla99e62fbca8a
diff --git a/packages/twenty-server/src/engine/core-modules/application/application-install/application-install.service.ts b/packages/twenty-server/src/engine/core-modules/application/application-install/application-install.service.tsindex 748feba076a..4dad1707bbb 100644--- a/packages/twenty-server/src/engine/core-modules/application/application-install/application-install.service.ts+++ b/packages/twenty-server/src/engine/core-modules/application/application-install/application-install.service.ts@@ -572,6 +572,7 @@ export class ApplicationInstallService {},{ retryLimit: 3 },);+return;}diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job.tsindex f9a891e06be..1e03dcb7032 100644--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job.ts+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/cron/cron-trigger.cron.job.ts@@ -5,6 +5,7 @@ import { isDefined } from 'twenty-shared/utils';import { WorkspaceActivationStatus } from 'twenty-shared/workspace';import { Repository } from 'typeorm';+import { findActiveFlatApplicationById } from 'src/engine/core-modules/application/utils/find-active-flat-application-by-id.util';import { CronTriggerDeduplicationService } from 'src/engine/core-modules/cron/services/cron-trigger-deduplication.service';import { SentryCronMonitor } from 'src/engine/core-modules/cron/sentry-cron-monitor.decorator';import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';@@ -12,6 +13,7 @@ import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decoraimport { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';+import { ApplicationJobEnqueueThrottlerService } from 'src/engine/core-modules/message-queue/services/application-job-enqueue-throttler.service';import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';import { WorkspaceEntity } from 'src/engine/core-modules/workspace/workspace.entity';import {@@ -34,6 +36,7 @@ export class CronTriggerCronJob {private readonly workspaceCacheService: WorkspaceCacheService,private readonly exceptionHandlerService: ExceptionHandlerService,private readonly cronTriggerDeduplicationService: CronTriggerDeduplicationService,+ private readonly applicationJobEnqueueThrottlerService: ApplicationJobEnqueueThrottlerService,) {}@Process(CronTriggerCronJob.name)@@ -50,9 +53,10 @@ export class CronTriggerCronJob {for (const activeWorkspace of activeWorkspaces) {try {- const { flatLogicFunctionMaps } =+ const { flatLogicFunctionMaps, flatApplicationMaps } =await this.workspaceCacheService.getOrRecompute(activeWorkspace.id, ['flatLogicFunctionMaps',+ 'flatApplicationMaps',]);const logicFunctions = Object.values(@@ -85,6 +89,22 @@ export class CronTriggerCronJob {continue;}+ const application = findActiveFlatApplicationById(+ flatApplicationMaps,+ logicFunction.applicationId,+ );+ const applicationRegistrationId =+ application?.applicationRegistrationId;++ if (!isDefined(applicationRegistrationId)) {+ continue;+ }++ await this.applicationJobEnqueueThrottlerService.throttleOrThrow({+ applicationId: logicFunction.applicationId,+ applicationRegistrationId,+ });+await this.messageQueueService.add<LogicFunctionTriggerJobData>(LogicFunctionTriggerJob.name,{diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.tsindex dbdd5d4439c..c89b5ae3dc5 100644--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.ts+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/call-database-event-trigger-jobs.job.ts@@ -2,10 +2,12 @@ import { isDefined } from 'twenty-shared/utils';import type { ObjectRecordEvent } from 'twenty-shared/database-events';+import { findActiveFlatApplicationById } from 'src/engine/core-modules/application/utils/find-active-flat-application-by-id.util';import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';+import { ApplicationJobEnqueueThrottlerService } from 'src/engine/core-modules/message-queue/services/application-job-enqueue-throttler.service';import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';import { transformEventBatchToEventPayloads } from 'src/engine/core-modules/logic-function/logic-function-trigger/triggers/database-event/utils/transform-event-batch-to-event-payloads';import {@@ -21,14 +23,15 @@ export class CallDatabaseEventTriggerJobsJob {@InjectMessageQueue(MessageQueue.logicFunctionQueue)private readonly messageQueueService: MessageQueueService,private readonly workspaceCacheService: WorkspaceCacheService,+ private readonly applicationJobEnqueueThrottlerService: ApplicationJobEnqueueThrottlerService,) {}@Process(CallDatabaseEventTriggerJobsJob.name)async handle(workspaceEventBatch: WorkspaceEventBatch<ObjectRecordEvent>) {- const { flatLogicFunctionMaps } =+ const { flatLogicFunctionMaps, flatApplicationMaps } =await this.workspaceCacheService.getOrRecompute(workspaceEventBatch.workspaceId,- ['flatLogicFunctionMaps'],+ ['flatLogicFunctionMaps', 'flatApplicationMaps'],);const logicFunctionsWithDatabaseEventTrigger = Object.values(@@ -51,16 +54,57 @@ export class CallDatabaseEventTriggerJobsJob {}),);- const logicFunctionPayloads = transformEventBatchToEventPayloads({- logicFunctions: logicFunctionsToTrigger,- workspaceEventBatch,- });+ const logicFunctionsByApplicationId = new Map<+ string,+ typeof logicFunctionsToTrigger+ >();- await this.messageQueueService.bulkAdd<LogicFunctionTriggerJobData>(- LogicFunctionTriggerJob.name,- logicFunctionPayloads,- { retryLimit: 3 },- );+ for (const logicFunction of logicFunctionsToTrigger) {+ const applicationLogicFunctions =+ logicFunctionsByApplicationId.get(logicFunction.applicationId) ?? [];++ applicationLogicFunctions.push(logicFunction);+ logicFunctionsByApplicationId.set(+ logicFunction.applicationId,+ applicationLogicFunctions,+ );+ }++ for (const [+ applicationId,+ logicFunctions,+ ] of logicFunctionsByApplicationId) {+ const application = findActiveFlatApplicationById(+ flatApplicationMaps,+ applicationId,+ );+ const applicationRegistrationId = application?.applicationRegistrationId;++ if (!isDefined(applicationRegistrationId)) {+ continue;+ }++ const logicFunctionPayloads = transformEventBatchToEventPayloads({+ logicFunctions,+ workspaceEventBatch,+ });++ if (logicFunctionPayloads.length === 0) {+ continue;+ }++ await this.applicationJobEnqueueThrottlerService.throttleOrThrow({+ applicationId,+ applicationRegistrationId,+ jobCount: logicFunctionPayloads.length,+ });++ await this.messageQueueService.bulkAdd<LogicFunctionTriggerJobData>(+ LogicFunctionTriggerJob.name,+ logicFunctionPayloads,+ { retryLimit: 3 },+ );+ }}private shouldTriggerJob({diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-core.module.ts b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-core.module.tsindex b6160b1e656..368d22409b0 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-core.module.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-core.module.ts@@ -20,9 +20,11 @@ import {ConfigurableModuleClass,type OPTIONS_TYPE,} from 'src/engine/core-modules/message-queue/message-queue.module-definition';+import { ApplicationJobEnqueueThrottlerService } from 'src/engine/core-modules/message-queue/services/application-job-enqueue-throttler.service';import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';import { getQueueToken } from 'src/engine/core-modules/message-queue/utils/get-queue-token.util';import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module';+import { ThrottlerModule } from 'src/engine/core-modules/throttler/throttler.module';@Global()@Module({})@@ -43,13 +45,20 @@ export class MessageQueueCoreModule extends ConfigurableModuleClass {return {...dynamicModule,+ imports: [+ ...(dynamicModule.imports ?? []),+ MetricsModule,+ ThrottlerModule,+ ],providers: [...(dynamicModule.providers ?? []),+ ApplicationJobEnqueueThrottlerService,driverProvider,...queueProviders,],exports: [...(dynamicModule.exports ?? []),+ ApplicationJobEnqueueThrottlerService,...Object.values(MessageQueue).map((queueName) =>getQueueToken(queueName),),@@ -78,14 +87,20 @@ export class MessageQueueCoreModule extends ConfigurableModuleClass {return {...dynamicModule,- imports: [...(dynamicModule.imports ?? []), MetricsModule],+ imports: [+ ...(dynamicModule.imports ?? []),+ MetricsModule,+ ThrottlerModule,+ ],providers: [...(dynamicModule.providers ?? []),+ ApplicationJobEnqueueThrottlerService,driverProvider,...queueProviders,],exports: [...(dynamicModule.exports ?? []),+ ApplicationJobEnqueueThrottlerService,...Object.values(MessageQueue).map((queueName) =>getQueueToken(queueName),),diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/services/__tests__/application-job-enqueue-throttler.service.spec.ts b/packages/twenty-server/src/engine/core-modules/message-queue/services/__tests__/application-job-enqueue-throttler.service.spec.tsnew file mode 100644index 00000000000..7ff94bfaaff--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/message-queue/services/__tests__/application-job-enqueue-throttler.service.spec.ts@@ -0,0 +1,123 @@+import { Test, type TestingModule } from '@nestjs/testing';++import { ApplicationJobEnqueueThrottlerService } from 'src/engine/core-modules/message-queue/services/application-job-enqueue-throttler.service';+import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';+import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';+import { ThrottlerException } from 'src/engine/core-modules/throttler/throttler.exception';+import { ThrottlerService } from 'src/engine/core-modules/throttler/throttler.service';+import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';++const CONFIG = {+ APPLICATION_JOB_ENQUEUE_RATE_LIMITING_TTL_IN_MS: 60_000,+ APPLICATION_JOB_ENQUEUE_RATE_LIMITING_LIMIT: 500,+ APPLICATION_REGISTRATION_JOB_ENQUEUE_RATE_LIMITING_LIMIT: 2000,+} as const;++const context = {+ applicationId: 'application-id',+ applicationRegistrationId: 'registration-id',+};++const APPLICATION_KEY = 'enqueue:throttler:application:application-id';+const REGISTRATION_KEY =+ 'enqueue:throttler:application-registration:registration-id';++describe('ApplicationJobEnqueueThrottlerService', () => {+ let service: ApplicationJobEnqueueThrottlerService;+ let throttlerService: {+ getAvailableTokensCount: jest.Mock;+ consumeTokens: jest.Mock;+ };+ let metricsService: { incrementCounterForEvent: jest.Mock };++ const mockAvailableTokens = (byKey: Record<string, number>) => {+ throttlerService.getAvailableTokensCount.mockImplementation(+ async (key: string) => byKey[key] ?? 0,+ );+ };++ beforeEach(async () => {+ throttlerService = {+ getAvailableTokensCount: jest.fn().mockResolvedValue(1000),+ consumeTokens: jest.fn().mockResolvedValue(undefined),+ };+ metricsService = {+ incrementCounterForEvent: jest.fn().mockResolvedValue(undefined),+ };++ const module: TestingModule = await Test.createTestingModule({+ providers: [+ ApplicationJobEnqueueThrottlerService,+ { provide: ThrottlerService, useValue: throttlerService },+ { provide: MetricsService, useValue: metricsService },+ {+ provide: TwentyConfigService,+ useValue: {+ get: (key: keyof typeof CONFIG) => CONFIG[key],+ },+ },+ ],+ }).compile();++ service = module.get(ApplicationJobEnqueueThrottlerService);+ });++ it('debits both application and registration buckets with distinct limits', async () => {+ await service.throttleOrThrow(context);++ expect(throttlerService.consumeTokens).toHaveBeenCalledWith(+ APPLICATION_KEY,+ 1,+ CONFIG.APPLICATION_JOB_ENQUEUE_RATE_LIMITING_LIMIT,+ CONFIG.APPLICATION_JOB_ENQUEUE_RATE_LIMITING_TTL_IN_MS,+ );+ expect(throttlerService.consumeTokens).toHaveBeenCalledWith(+ REGISTRATION_KEY,+ 1,+ CONFIG.APPLICATION_REGISTRATION_JOB_ENQUEUE_RATE_LIMITING_LIMIT,+ CONFIG.APPLICATION_JOB_ENQUEUE_RATE_LIMITING_TTL_IN_MS,+ );+ });++ it('consumes one token per job for bulk enqueue', async () => {+ await service.throttleOrThrow({ ...context, jobCount: 5 });++ expect(throttlerService.consumeTokens).toHaveBeenCalledWith(+ APPLICATION_KEY,+ 5,+ expect.any(Number),+ expect.any(Number),+ );+ });++ it('does not debit any bucket when the registration bucket is exhausted', async () => {+ mockAvailableTokens({+ [APPLICATION_KEY]: 1000,+ [REGISTRATION_KEY]: 0,+ });++ await expect(service.throttleOrThrow(context)).rejects.toThrow(+ ThrottlerException,+ );++ expect(throttlerService.consumeTokens).not.toHaveBeenCalled();+ expect(metricsService.incrementCounterForEvent).toHaveBeenCalledWith(+ expect.objectContaining({+ key: MetricsKeys.JobEnqueueApplicationRateLimited,+ }),+ );+ });++ it('does not debit any bucket when the application bucket is exhausted', async () => {+ mockAvailableTokens({+ [APPLICATION_KEY]: 0,+ [REGISTRATION_KEY]: 1000,+ });++ await expect(service.throttleOrThrow(context)).rejects.toThrow(+ ThrottlerException,+ );++ expect(throttlerService.consumeTokens).not.toHaveBeenCalled();+ });+});diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/services/application-job-enqueue-throttler.service.ts b/packages/twenty-server/src/engine/core-modules/message-queue/services/application-job-enqueue-throttler.service.tsnew file mode 100644index 00000000000..3773cd96b63--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/message-queue/services/application-job-enqueue-throttler.service.ts@@ -0,0 +1,85 @@+import { Injectable } from '@nestjs/common';++import { MetricsService } from 'src/engine/core-modules/metrics/metrics.service';+import { MetricsKeys } from 'src/engine/core-modules/metrics/types/metrics-keys.type';+import {+ ThrottlerException,+ ThrottlerExceptionCode,+} from 'src/engine/core-modules/throttler/throttler.exception';+import { ThrottlerService } from 'src/engine/core-modules/throttler/throttler.service';+import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';++@Injectable()+export class ApplicationJobEnqueueThrottlerService {+ constructor(+ private readonly throttlerService: ThrottlerService,+ private readonly twentyConfigService: TwentyConfigService,+ private readonly metricsService: MetricsService,+ ) {}++ async throttleOrThrow({+ applicationId,+ applicationRegistrationId,+ jobCount = 1,+ }: {+ applicationId: string;+ applicationRegistrationId: string;+ jobCount?: number;+ }): Promise<void> {+ const timeWindow = this.twentyConfigService.get(+ 'APPLICATION_JOB_ENQUEUE_RATE_LIMITING_TTL_IN_MS',+ );+ const applicationKey = `enqueue:throttler:application:${applicationId}`;+ const applicationLimit = this.twentyConfigService.get(+ 'APPLICATION_JOB_ENQUEUE_RATE_LIMITING_LIMIT',+ );+ const registrationKey = `enqueue:throttler:application-registration:${applicationRegistrationId}`;+ const registrationLimit = this.twentyConfigService.get(+ 'APPLICATION_REGISTRATION_JOB_ENQUEUE_RATE_LIMITING_LIMIT',+ );++ const [applicationTokens, registrationTokens] = await Promise.all([+ this.throttlerService.getAvailableTokensCount(+ applicationKey,+ applicationLimit,+ timeWindow,+ ),+ this.throttlerService.getAvailableTokensCount(+ registrationKey,+ registrationLimit,+ timeWindow,+ ),+ ]);++ if (applicationTokens < jobCount || registrationTokens < jobCount) {+ await this.metricsService.incrementCounterForEvent({+ key: MetricsKeys.JobEnqueueApplicationRateLimited,+ shouldStoreInCache: false,+ attributes: {+ application_id: applicationId,+ application_registration_id: applicationRegistrationId,+ },+ });++ throw new ThrottlerException(+ 'Application job enqueue limit reached',+ ThrottlerExceptionCode.LIMIT_REACHED,+ );+ }++ await Promise.all([+ this.throttlerService.consumeTokens(+ applicationKey,+ jobCount,+ applicationLimit,+ timeWindow,+ ),+ this.throttlerService.consumeTokens(+ registrationKey,+ jobCount,+ registrationLimit,+ timeWindow,+ ),+ ]);+ }+}diff --git a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.tsindex c9e766a5f7f..aa242b7676d 100644--- a/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts+++ b/packages/twenty-server/src/engine/core-modules/metrics/types/metrics-keys.type.ts@@ -59,6 +59,7 @@ export enum MetricsKeys {SignUpSuccess = 'sign-up/success',CommonApiQueryRateLimited = 'common-api-query/rate-limited',CommonApiApplicationQueryRateLimited = 'common-api-query/application-rate-limited',+ JobEnqueueApplicationRateLimited = 'job/enqueue-application-rate-limited',JobCompleted = 'job/completed',JobFailed = 'job/failed',JobStalled = 'job/stalled',diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.tsindex a5caf8dcd08..35575b3c9aa 100644--- a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts+++ b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts@@ -1521,6 +1521,33 @@ export class ConfigVariables {@CastToPositiveNumber()APPLICATION_API_RATE_LIMITING_LIMIT = 500;+ @ConfigVariablesMetadata({+ group: ConfigVariablesGroup.RATE_LIMITING,+ description:+ 'Time-to-live for application job enqueue rate limiting in milliseconds',+ type: ConfigVariableType.NUMBER,+ })+ @CastToPositiveNumber()+ APPLICATION_JOB_ENQUEUE_RATE_LIMITING_TTL_IN_MS = 60_000;++ @ConfigVariablesMetadata({+ group: ConfigVariablesGroup.RATE_LIMITING,+ description:+ 'Maximum number of jobs a single application installation can enqueue in the rate limiting window',+ type: ConfigVariableType.NUMBER,+ })+ @CastToPositiveNumber()+ APPLICATION_JOB_ENQUEUE_RATE_LIMITING_LIMIT = 500;++ @ConfigVariablesMetadata({+ group: ConfigVariablesGroup.RATE_LIMITING,+ description:+ 'Maximum number of jobs enqueued per application registration across all workspaces in the rate limiting window',+ type: ConfigVariableType.NUMBER,+ })+ @CastToPositiveNumber()+ APPLICATION_REGISTRATION_JOB_ENQUEUE_RATE_LIMITING_LIMIT = 2000;+@CastToPositiveNumber()@ConfigVariablesMetadata({group: ConfigVariablesGroup.RATE_LIMITING,