The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Run prebuilt warm-ups on a rate-limited lambda-management-queue (#28343)
Charles Bochet524d35db650b
diff --git a/packages/twenty-server/src/engine/core-modules/application/application-install/application-install.service.ts b/packages/twenty-server/src/engine/core-modules/application/application-install/application-install.service.tsindex fce39194d2c..83a54a41a43 100644--- a/packages/twenty-server/src/engine/core-modules/application/application-install/application-install.service.ts+++ b/packages/twenty-server/src/engine/core-modules/application/application-install/application-install.service.ts@@ -53,10 +53,10 @@ import {type ApplicationLifecycleHookJobData,} from 'src/engine/core-modules/logic-function/logic-function-trigger/jobs/application-lifecycle-hook.job';import {- WARM_UP_APPLICATION_LOGIC_FUNCTIONS_JOB_NAME,- WARM_UP_APPLICATION_LOGIC_FUNCTIONS_JOB_OPTIONS,- type WarmUpApplicationLogicFunctionsJobData,-} from 'src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-application-logic-functions.job-constants';+ WARM_UP_LOGIC_FUNCTION_JOB_NAME,+ WARM_UP_LOGIC_FUNCTION_JOB_OPTIONS,+ type WarmUpLogicFunctionJobData,+} from 'src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-logic-function.job-constants';import { findLogicFunctionUniversalIdentifiersToWarmUp } from 'src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/utils/find-logic-function-universal-identifiers-to-warm-up.util';import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';import { type MessageQueueJobProgressContext } from 'src/engine/core-modules/message-queue/interfaces/message-queue-job.interface';@@ -85,8 +85,8 @@ export class ApplicationInstallService {private readonly cacheLockService: CacheLockService,@InjectMessageQueue(MessageQueue.applicationLifecycleHookQueue)private readonly messageQueueService: MessageQueueService,- @InjectMessageQueue(MessageQueue.workspaceQueue)- private readonly workspaceQueueService: MessageQueueService,+ @InjectMessageQueue(MessageQueue.lambdaManagementQueue)+ private readonly lambdaManagementQueueService: MessageQueueService,private readonly workspaceCacheService: WorkspaceCacheService,private readonly metricsService: MetricsService,) {}@@ -615,22 +615,36 @@ export class ApplicationInstallService {}}- private async enqueueLogicFunctionWarmUp(- data: WarmUpApplicationLogicFunctionsJobData,- ): Promise<void> {- if (data.logicFunctionUniversalIdentifiers.length === 0) {+ private async enqueueLogicFunctionWarmUp({+ workspaceId,+ applicationId,+ logicFunctionUniversalIdentifiers,+ }: {+ workspaceId: string;+ applicationId: string;+ logicFunctionUniversalIdentifiers: string[];+ }): Promise<void> {+ if (logicFunctionUniversalIdentifiers.length === 0) {return;}try {- await this.workspaceQueueService.add<WarmUpApplicationLogicFunctionsJobData>(- WARM_UP_APPLICATION_LOGIC_FUNCTIONS_JOB_NAME,- data,- WARM_UP_APPLICATION_LOGIC_FUNCTIONS_JOB_OPTIONS,+ await this.lambdaManagementQueueService.bulkAdd<WarmUpLogicFunctionJobData>(+ WARM_UP_LOGIC_FUNCTION_JOB_NAME,+ logicFunctionUniversalIdentifiers.map(+ (logicFunctionUniversalIdentifier) => ({+ data: {+ workspaceId,+ applicationId,+ logicFunctionUniversalIdentifier,+ },+ }),+ ),+ WARM_UP_LOGIC_FUNCTION_JOB_OPTIONS,);} catch (error) {this.logger.error(- `Failed to enqueue prebuilt warm-up for application ${data.applicationId} in workspace ${data.workspaceId}: ` ++ `Failed to enqueue prebuilt warm-up for application ${applicationId} in workspace ${workspaceId}: ` +`${error instanceof Error ? error.message : String(error)}`,error instanceof Error ? error.stack : undefined,);diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-aws-client.service.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-aws-client.service.tsindex 7d49ae371bd..9dd4b1c1037 100644--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-aws-client.service.ts+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-aws-client.service.ts@@ -19,6 +19,7 @@ import {UPDATE_FUNCTION_DURATION_TIMEOUT_IN_SECONDS,} from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/constants/lambda-driver.constant';import { type LambdaDriverOptions } from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/types/lambda-driver.type';+import { addLambdaRateLimitErrorMiddleware } from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/utils/add-lambda-rate-limit-error-middleware.util';import { buildAwsRequestHandlerOptions } from 'src/utils/aws-request-handler.util';export class LambdaAwsClientService {@@ -38,7 +39,7 @@ export class LambdaAwsClientService {(isDefined(this.options.subhostingRole) &&this.areAssumeRoleCredentialsExpired())) {- this.lambdaClient = new Lambda({+ const lambdaClient = new Lambda({...this.options,...(isDefined(this.options.subhostingRole) && {credentials: await this.getAssumeRoleCredentials(),@@ -51,6 +52,10 @@ export class LambdaAwsClientService {maxSockets: LAMBDA_CLIENT_MAX_SOCKETS,}),});++ addLambdaRateLimitErrorMiddleware(lambdaClient);++ this.lambdaClient = lambdaClient;}return this.lambdaClient;diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-executor-manager.service.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-executor-manager.service.tsindex 0904fa63887..57d1c760a6d 100644--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-executor-manager.service.ts+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-executor-manager.service.ts@@ -320,8 +320,9 @@ export class LambdaExecutorManagerService {} catch (error) {if (error instanceof LogicFunctionException &&- error.code ===- LogicFunctionExceptionCode.LOGIC_FUNCTION_DEPENDENCIES_SIZE_EXCEEDED+ (error.code ===+ LogicFunctionExceptionCode.LOGIC_FUNCTION_DEPENDENCIES_SIZE_EXCEEDED ||+ error.code === LogicFunctionExceptionCode.LOGIC_FUNCTION_RATE_LIMITED)) {throw error;}@@ -344,6 +345,13 @@ export class LambdaExecutorManagerService {applicationUniversalIdentifier,});} catch (error) {+ if (+ error instanceof LogicFunctionException &&+ error.code === LogicFunctionExceptionCode.LOGIC_FUNCTION_RATE_LIMITED+ ) {+ throw error;+ }+this.logger.error(`Failed to get SDK layer for function ${flatLogicFunction.id}: ${error instanceof Error ? error.message : String(error)}`,error instanceof Error ? error.stack : undefined,diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/utils/__tests__/is-lambda-rate-limit-error.util.spec.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/utils/__tests__/is-lambda-rate-limit-error.util.spec.tsnew file mode 100644index 00000000000..6c4555676fa--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/utils/__tests__/is-lambda-rate-limit-error.util.spec.ts@@ -0,0 +1,22 @@+import { isLambdaRateLimitError } from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/utils/is-lambda-rate-limit-error.util';++const buildError = (name: string) =>+ Object.assign(new Error('Rate exceeded'), { name });++describe('isLambdaRateLimitError', () => {+ it('detects the throttling errors the Lambda API returns', () => {+ expect(isLambdaRateLimitError(buildError('TooManyRequestsException'))).toBe(+ true,+ );+ expect(isLambdaRateLimitError(buildError('ThrottlingException'))).toBe(+ true,+ );+ });++ it('ignores other errors', () => {+ expect(+ isLambdaRateLimitError(buildError('ResourceConflictException')),+ ).toBe(false);+ expect(isLambdaRateLimitError('Rate exceeded')).toBe(false);+ });+});diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/utils/add-lambda-rate-limit-error-middleware.util.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/utils/add-lambda-rate-limit-error-middleware.util.tsnew file mode 100644index 00000000000..9a75ee707d8--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/utils/add-lambda-rate-limit-error-middleware.util.ts@@ -0,0 +1,27 @@+import { type Lambda } from '@aws-sdk/client-lambda';++import { isLambdaRateLimitError } from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/utils/is-lambda-rate-limit-error.util';+import {+ LogicFunctionException,+ LogicFunctionExceptionCode,+} from 'src/engine/metadata-modules/logic-function/logic-function.exception';++export const addLambdaRateLimitErrorMiddleware = (lambdaClient: Lambda) => {+ lambdaClient.middlewareStack.add(+ (next, context) => async (args) => {+ try {+ return await next(args);+ } catch (error) {+ if (isLambdaRateLimitError(error)) {+ throw new LogicFunctionException(+ `Lambda rate limited ${context.commandName ?? 'a request'}`,+ LogicFunctionExceptionCode.LOGIC_FUNCTION_RATE_LIMITED,+ );+ }++ throw error;+ }+ },+ { step: 'initialize', name: 'lambdaRateLimitErrorMiddleware' },+ );+};diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/utils/is-lambda-rate-limit-error.util.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/utils/is-lambda-rate-limit-error.util.tsnew file mode 100644index 00000000000..62bb503d419--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/utils/is-lambda-rate-limit-error.util.ts@@ -0,0 +1,7 @@+const LAMBDA_RATE_LIMIT_ERROR_NAMES = new Set([+ 'TooManyRequestsException',+ 'ThrottlingException',+]);++export const isLambdaRateLimitError = (error: unknown): boolean =>+ error instanceof Error && LAMBDA_RATE_LIMIT_ERROR_NAMES.has(error.name);diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-application-logic-functions.job-constants.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-application-logic-functions.job-constants.tsdeleted file mode 100644index f700735e891..00000000000--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-application-logic-functions.job-constants.ts+++ /dev/null@@ -1,16 +0,0 @@-import { type QueueJobOptions } from 'src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface';--export const WARM_UP_APPLICATION_LOGIC_FUNCTIONS_JOB_NAME =- 'WarmUpApplicationLogicFunctionsJob';--export const WARM_UP_APPLICATION_LOGIC_FUNCTIONS_JOB_OPTIONS: QueueJobOptions =- {- retryLimit: 1,- backoff: { strategy: 'exponential', initialDelayMilliseconds: 1_000 },- };--export type WarmUpApplicationLogicFunctionsJobData = {- workspaceId: string;- applicationId: string;- logicFunctionUniversalIdentifiers: string[];-};diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-logic-function.job-constants.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-logic-function.job-constants.tsnew file mode 100644index 00000000000..9d97db1761e--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-logic-function.job-constants.ts@@ -0,0 +1,14 @@+import { type QueueJobOptions } from 'src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface';++export const WARM_UP_LOGIC_FUNCTION_JOB_NAME = 'WarmUpLogicFunctionJob';++export const WARM_UP_LOGIC_FUNCTION_JOB_OPTIONS: QueueJobOptions = {+ retryLimit: 5,+ backoff: { strategy: 'exponential', initialDelayMilliseconds: 30_000 },+};++export type WarmUpLogicFunctionJobData = {+ workspaceId: string;+ applicationId: string;+ logicFunctionUniversalIdentifier: string;+};diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-application-logic-functions.job.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-logic-function.job.tssimilarity index 56%rename from packages/twenty-server/src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-application-logic-functions.job.tsrename to packages/twenty-server/src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-logic-function.job.tsindex 02984b15a81..5888f75111f 100644--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-application-logic-functions.job.ts+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-logic-function.job.ts@@ -1,22 +1,20 @@import { LogicFunctionPrebuiltWarmUpService } from 'src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/logic-function-prebuilt-warm-up.service';import {- WARM_UP_APPLICATION_LOGIC_FUNCTIONS_JOB_NAME,- type WarmUpApplicationLogicFunctionsJobData,-} from 'src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-application-logic-functions.job-constants';+ WARM_UP_LOGIC_FUNCTION_JOB_NAME,+ type WarmUpLogicFunctionJobData,+} from 'src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-logic-function.job-constants';import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';-@Processor(MessageQueue.workspaceQueue)-export class WarmUpApplicationLogicFunctionsJob {+@Processor(MessageQueue.lambdaManagementQueue)+export class WarmUpLogicFunctionJob {constructor(private readonly logicFunctionPrebuiltWarmUpService: LogicFunctionPrebuiltWarmUpService,) {}- @Process(WARM_UP_APPLICATION_LOGIC_FUNCTIONS_JOB_NAME)- async handle(data: WarmUpApplicationLogicFunctionsJobData): Promise<void> {- await this.logicFunctionPrebuiltWarmUpService.warmUpApplicationLogicFunctions(- data,- );+ @Process(WARM_UP_LOGIC_FUNCTION_JOB_NAME)+ async handle(data: WarmUpLogicFunctionJobData): Promise<void> {+ await this.logicFunctionPrebuiltWarmUpService.warmUpLogicFunction(data);}}diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/logic-function-prebuilt-warm-up.service.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/logic-function-prebuilt-warm-up.service.tsindex a85652e6a32..568d683bcb8 100644--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/logic-function-prebuilt-warm-up.service.ts+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/logic-function-prebuilt-warm-up.service.ts@@ -1,6 +1,5 @@import { Inject, Injectable, Logger } from '@nestjs/common';-import chunk from 'lodash.chunk';import { isDefined } from 'twenty-shared/utils';import { type FlatApplication } from 'src/engine/core-modules/application/types/flat-application.type';@@ -14,8 +13,6 @@ import { type FlatLogicFunction } from 'src/engine/metadata-modules/logic-functiimport { isLogicFunctionReadyForPrebuiltInstall } from 'src/engine/metadata-modules/logic-function/utils/is-logic-function-ready-for-prebuilt-install.util';import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';-const WARM_UP_CHUNK_SIZE = 10;-@Injectable()export class LogicFunctionPrebuiltWarmUpService {private readonly logger = new Logger(LogicFunctionPrebuiltWarmUpService.name);@@ -50,6 +47,13 @@ export class LogicFunctionPrebuiltWarmUpService {applicationUniversalIdentifier: flatApplication.universalIdentifier,});} catch (error) {+ if (+ error instanceof LogicFunctionException &&+ error.code === LogicFunctionExceptionCode.LOGIC_FUNCTION_RATE_LIMITED+ ) {+ throw error;+ }+const cause = error instanceof Error ? error.message : String(error);this.logger.error(@@ -66,14 +70,14 @@ export class LogicFunctionPrebuiltWarmUpService {}}- async warmUpApplicationLogicFunctions({+ async warmUpLogicFunction({workspaceId,applicationId,- logicFunctionUniversalIdentifiers,+ logicFunctionUniversalIdentifier,}: {workspaceId: string;applicationId: string;- logicFunctionUniversalIdentifiers: string[];+ logicFunctionUniversalIdentifier: string;}): Promise<void> {const { flatLogicFunctionMaps, flatApplicationMaps } =await this.workspaceCacheService.getOrRecompute(workspaceId, [@@ -91,48 +95,22 @@ export class LogicFunctionPrebuiltWarmUpService {return;}- const flatLogicFunctions = logicFunctionUniversalIdentifiers- .map(- (universalIdentifier) =>- flatLogicFunctionMaps.byUniversalIdentifier[universalIdentifier],- )- .filter(isDefined)- .filter(- (flatLogicFunction) =>- flatLogicFunction.applicationId === applicationId &&- isLogicFunctionReadyForPrebuiltInstall(flatLogicFunction),- );-- const failures: string[] = [];-- for (const flatLogicFunctionsChunk of chunk(- flatLogicFunctions,- WARM_UP_CHUNK_SIZE,- )) {- const results = await Promise.allSettled(- flatLogicFunctionsChunk.map((flatLogicFunction) =>- this.ensurePrebuiltBundleInstalled({- flatLogicFunction,- flatApplication,- }),- ),- );+ const flatLogicFunction =+ flatLogicFunctionMaps.byUniversalIdentifier[+ logicFunctionUniversalIdentifier+ ];- results.forEach((result, index) => {- if (result.status === 'rejected') {- failures.push(- `${flatLogicFunctionsChunk[index].id}: ${result.reason instanceof Error ? result.reason.message : String(result.reason)}`,- );- }- });+ if (+ !isDefined(flatLogicFunction) ||+ flatLogicFunction.applicationId !== applicationId ||+ !isLogicFunctionReadyForPrebuiltInstall(flatLogicFunction)+ ) {+ return;}- if (failures.length > 0) {- throw new LogicFunctionException(- `Failed to warm up ${failures.length} of ${flatLogicFunctions.length} prebuilt logic functions ` +- `for application ${applicationId} in workspace ${workspaceId}: ${failures.join('; ')}`,- LogicFunctionExceptionCode.LOGIC_FUNCTION_PREBUILT_BUNDLE_NOT_INSTALLED,- );- }+ await this.ensurePrebuiltBundleInstalled({+ flatLogicFunction,+ flatApplication,+ });}}diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.tsindex cb97ceedb3c..d737c631708 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts@@ -254,6 +254,14 @@ export class BullMQDriver...(isDefined(options?.maxStalledCount)? { maxStalledCount: options.maxStalledCount }: {}),+ ...(isDefined(options?.rateLimit)+ ? {+ limiter: {+ max: options.rateLimit.max,+ duration: options.rateLimit.durationMs,+ },+ }+ : {}),metrics: {maxDataPoints: MetricsTime.ONE_WEEK,collectInterval: 60000,diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/interfaces/message-queue-worker-options.interface.ts b/packages/twenty-server/src/engine/core-modules/message-queue/interfaces/message-queue-worker-options.interface.tsindex 088a18b2220..fa6d5ed3740 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/interfaces/message-queue-worker-options.interface.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/interfaces/message-queue-worker-options.interface.ts@@ -1,6 +1,7 @@export interface MessageQueueWorkerOptions {concurrency?: number;globalConcurrency?: number;+ rateLimit?: { max: number; durationMs: number };lockDuration?: number;maxStalledCount?: number;boundedShutdownDrain?: boolean;diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/jobs.module.ts b/packages/twenty-server/src/engine/core-modules/message-queue/jobs.module.tsindex 1005f4fdfc9..9e47c2a29fb 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/jobs.module.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/jobs.module.ts@@ -52,7 +52,7 @@ import { WorkspaceModule } from 'src/engine/core-modules/workspace/workspace.modimport { AiAgentTriggerModule } from 'src/engine/metadata-modules/ai/ai-agent-trigger/ai-agent-trigger.module';import { AiChatModule } from 'src/engine/metadata-modules/ai/ai-chat/ai-chat.module';import { LogicFunctionPrebuiltWarmUpModule } from 'src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/logic-function-prebuilt-warm-up.module';-import { WarmUpApplicationLogicFunctionsJob } from 'src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-application-logic-functions.job';+import { WarmUpLogicFunctionJob } from 'src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-logic-function.job';import { LogicFunctionModule } from 'src/engine/metadata-modules/logic-function/logic-function.module';import { NavigationMenuItemModule } from 'src/engine/metadata-modules/navigation-menu-item/navigation-menu-item.module';import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module';@@ -144,7 +144,7 @@ import { WorkflowModule } from 'src/modules/workflow/workflow.module';TriggerInstallApplicationJob,TriggerUninstallApplicationJob,TriggerUpgradeApplicationJob,- WarmUpApplicationLogicFunctionsJob,+ WarmUpLogicFunctionJob,RunDeferredWorkspaceMigrationActionsJob,DeferredWorkspaceMigrationActionRecoveryCronJob,],diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-worker-config.constant.ts b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-worker-config.constant.tsindex 71c74ae0db9..712a0479f85 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-worker-config.constant.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-worker-config.constant.ts@@ -10,6 +10,7 @@ import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queu// concurrency: max jobs processed in parallel per worker process// globalConcurrency: optional, max jobs processed in parallel across all// worker processes+// rateLimit: optional, max jobs started per durationMs across all workers// lockDuration: ms a job may run before BullMQ considers it stalled// maxStalledCount: times a stalled job is re-queued before failing permanently// boundedShutdownDrain: on shutdown, abort still-active jobs after@@ -18,9 +19,9 @@ import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queuexport type MessageQueueWorkerConfig = {priority: number;workerOptions: Required<- Omit<MessageQueueWorkerOptions, 'globalConcurrency'>+ Omit<MessageQueueWorkerOptions, 'globalConcurrency' | 'rateLimit'>> &- Pick<MessageQueueWorkerOptions, 'globalConcurrency'>;+ Pick<MessageQueueWorkerOptions, 'globalConcurrency' | 'rateLimit'>;};export const MESSAGE_QUEUE_WORKER_CONFIG: Record<@@ -218,6 +219,16 @@ export const MESSAGE_QUEUE_WORKER_CONFIG: Record<boundedShutdownDrain: false,},},+ [MessageQueue.lambdaManagementQueue]: {+ priority: 6,+ workerOptions: {+ concurrency: 2,+ rateLimit: { max: 2, durationMs: 1_000 },+ lockDuration: 60_000,+ maxStalledCount: 1,+ boundedShutdownDrain: false,+ },+ },[MessageQueue.applicationLifecycleHookQueue]: {priority: 4,workerOptions: {diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.constants.ts b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.constants.tsindex bed18e85a5d..83be4810cea 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.constants.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.constants.ts@@ -24,6 +24,7 @@ export enum MessageQueue {delayedJobsQueue = 'delayed-jobs-queue',deleteCascadeQueue = 'delete-cascade-queue',logicFunctionQueue = 'logic-function-queue',+ lambdaManagementQueue = 'lambda-management-queue',applicationLifecycleHookQueue = 'application-lifecycle-hook-queue',applicationUpgradeQueue = 'application-upgrade-queue',triggerQueue = 'trigger-queue',diff --git a/packages/twenty-server/src/engine/metadata-modules/logic-function/logic-function.exception.ts b/packages/twenty-server/src/engine/metadata-modules/logic-function/logic-function.exception.tsindex 4874acbaaea..1cb5f04f32b 100644--- a/packages/twenty-server/src/engine/metadata-modules/logic-function/logic-function.exception.ts+++ b/packages/twenty-server/src/engine/metadata-modules/logic-function/logic-function.exception.ts@@ -1,4 +1,6 @@import { type MessageDescriptor } from '@lingui/core';+import { HttpStatus } from '@nestjs/common';+import { msg } from '@lingui/core/macro';import { assertUnreachable } from 'twenty-shared/utils';@@ -22,6 +24,7 @@ export enum LogicFunctionExceptionCode {LOGIC_FUNCTION_INVALID_SEED_PROJECT = 'LOGIC_FUNCTION_INVALID_SEED_PROJECT',INVALID_LOGIC_FUNCTION_INPUT = 'INVALID_LOGIC_FUNCTION_INPUT',LOGIC_FUNCTION_PREBUILT_BUNDLE_NOT_INSTALLED = 'LOGIC_FUNCTION_PREBUILT_BUNDLE_NOT_INSTALLED',+ LOGIC_FUNCTION_RATE_LIMITED = 'LOGIC_FUNCTION_RATE_LIMITED',}const getLogicFunctionExceptionUserFriendlyMessage = (@@ -62,6 +65,8 @@ const getLogicFunctionExceptionUserFriendlyMessage = (return msg`Invalid logic function input.`;case LogicFunctionExceptionCode.LOGIC_FUNCTION_PREBUILT_BUNDLE_NOT_INSTALLED:return msg`Prebuilt bundle is not installed on the function. Rebuild and try again.`;+ case LogicFunctionExceptionCode.LOGIC_FUNCTION_RATE_LIMITED:+ return msg`Function provider is busy. Try again in a few minutes.`;default:assertUnreachable(code);}@@ -77,6 +82,10 @@ export class LogicFunctionException extends CustomException<LogicFunctionExceptiuserFriendlyMessage:userFriendlyMessage ??getLogicFunctionExceptionUserFriendlyMessage(code),+ statusCode:+ code === LogicFunctionExceptionCode.LOGIC_FUNCTION_RATE_LIMITED+ ? HttpStatus.TOO_MANY_REQUESTS+ : undefined,});}}diff --git a/packages/twenty-server/src/engine/metadata-modules/logic-function/utils/logic-function-graphql-api-exception-handler.utils.ts b/packages/twenty-server/src/engine/metadata-modules/logic-function/utils/logic-function-graphql-api-exception-handler.utils.tsindex 2ed710ef4e9..680499ab39d 100644--- a/packages/twenty-server/src/engine/metadata-modules/logic-function/utils/logic-function-graphql-api-exception-handler.utils.ts+++ b/packages/twenty-server/src/engine/metadata-modules/logic-function/utils/logic-function-graphql-api-exception-handler.utils.ts@@ -32,6 +32,7 @@ export const logicFunctionGraphQLApiExceptionHandler = (error: any) => {case LogicFunctionExceptionCode.LOGIC_FUNCTION_INVALID_SEED_PROJECT:case LogicFunctionExceptionCode.LOGIC_FUNCTION_PLATFORM_EXECUTION_ERROR:case LogicFunctionExceptionCode.LOGIC_FUNCTION_LAYER_BUILD_FAILED:+ case LogicFunctionExceptionCode.LOGIC_FUNCTION_RATE_LIMITED:throw error;case LogicFunctionExceptionCode.LOGIC_FUNCTION_COMPILATION_FAILED:case LogicFunctionExceptionCode.LOGIC_FUNCTION_DEPENDENCIES_SIZE_EXCEEDED: