Skip to content

The open alternative to Salesforce, designed for AI.

Run prebuilt warm-ups on a rate-limited lambda-management-queue (#28343)

Charles Bochet524d35db650b
diff --git a/packages/twenty-server/src/engine/core-modules/application/application-install/application-install.service.ts b/packages/twenty-server/src/engine/core-modules/application/application-install/application-install.service.ts
index fce39194d2c..83a54a41a43 100644
--- a/packages/twenty-server/src/engine/core-modules/application/application-install/application-install.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/application/application-install/application-install.service.ts
@@ -53,10 +53,10 @@ import {
type ApplicationLifecycleHookJobData,
} from 'src/engine/core-modules/logic-function/logic-function-trigger/jobs/application-lifecycle-hook.job';
import {
- WARM_UP_APPLICATION_LOGIC_FUNCTIONS_JOB_NAME,
- WARM_UP_APPLICATION_LOGIC_FUNCTIONS_JOB_OPTIONS,
- type WarmUpApplicationLogicFunctionsJobData,
-} from 'src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-application-logic-functions.job-constants';
+ WARM_UP_LOGIC_FUNCTION_JOB_NAME,
+ WARM_UP_LOGIC_FUNCTION_JOB_OPTIONS,
+ type WarmUpLogicFunctionJobData,
+} from 'src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-logic-function.job-constants';
import { findLogicFunctionUniversalIdentifiersToWarmUp } from 'src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/utils/find-logic-function-universal-identifiers-to-warm-up.util';
import { InjectMessageQueue } from 'src/engine/core-modules/message-queue/decorators/message-queue.decorator';
import { type MessageQueueJobProgressContext } from 'src/engine/core-modules/message-queue/interfaces/message-queue-job.interface';
@@ -85,8 +85,8 @@ export class ApplicationInstallService {
private readonly cacheLockService: CacheLockService,
@InjectMessageQueue(MessageQueue.applicationLifecycleHookQueue)
private readonly messageQueueService: MessageQueueService,
- @InjectMessageQueue(MessageQueue.workspaceQueue)
- private readonly workspaceQueueService: MessageQueueService,
+ @InjectMessageQueue(MessageQueue.lambdaManagementQueue)
+ private readonly lambdaManagementQueueService: MessageQueueService,
private readonly workspaceCacheService: WorkspaceCacheService,
private readonly metricsService: MetricsService,
) {}
@@ -615,22 +615,36 @@ export class ApplicationInstallService {
}
}
- private async enqueueLogicFunctionWarmUp(
- data: WarmUpApplicationLogicFunctionsJobData,
- ): Promise<void> {
- if (data.logicFunctionUniversalIdentifiers.length === 0) {
+ private async enqueueLogicFunctionWarmUp({
+ workspaceId,
+ applicationId,
+ logicFunctionUniversalIdentifiers,
+ }: {
+ workspaceId: string;
+ applicationId: string;
+ logicFunctionUniversalIdentifiers: string[];
+ }): Promise<void> {
+ if (logicFunctionUniversalIdentifiers.length === 0) {
return;
}
try {
- await this.workspaceQueueService.add<WarmUpApplicationLogicFunctionsJobData>(
- WARM_UP_APPLICATION_LOGIC_FUNCTIONS_JOB_NAME,
- data,
- WARM_UP_APPLICATION_LOGIC_FUNCTIONS_JOB_OPTIONS,
+ await this.lambdaManagementQueueService.bulkAdd<WarmUpLogicFunctionJobData>(
+ WARM_UP_LOGIC_FUNCTION_JOB_NAME,
+ logicFunctionUniversalIdentifiers.map(
+ (logicFunctionUniversalIdentifier) => ({
+ data: {
+ workspaceId,
+ applicationId,
+ logicFunctionUniversalIdentifier,
+ },
+ }),
+ ),
+ WARM_UP_LOGIC_FUNCTION_JOB_OPTIONS,
);
} catch (error) {
this.logger.error(
- `Failed to enqueue prebuilt warm-up for application ${data.applicationId} in workspace ${data.workspaceId}: ` +
+ `Failed to enqueue prebuilt warm-up for application ${applicationId} in workspace ${workspaceId}: ` +
`${error instanceof Error ? error.message : String(error)}`,
error instanceof Error ? error.stack : undefined,
);
diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-aws-client.service.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-aws-client.service.ts
index 7d49ae371bd..9dd4b1c1037 100644
--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-aws-client.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-aws-client.service.ts
@@ -19,6 +19,7 @@ import {
UPDATE_FUNCTION_DURATION_TIMEOUT_IN_SECONDS,
} from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/constants/lambda-driver.constant';
import { type LambdaDriverOptions } from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/types/lambda-driver.type';
+import { addLambdaRateLimitErrorMiddleware } from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/utils/add-lambda-rate-limit-error-middleware.util';
import { buildAwsRequestHandlerOptions } from 'src/utils/aws-request-handler.util';
export class LambdaAwsClientService {
@@ -38,7 +39,7 @@ export class LambdaAwsClientService {
(isDefined(this.options.subhostingRole) &&
this.areAssumeRoleCredentialsExpired())
) {
- this.lambdaClient = new Lambda({
+ const lambdaClient = new Lambda({
...this.options,
...(isDefined(this.options.subhostingRole) && {
credentials: await this.getAssumeRoleCredentials(),
@@ -51,6 +52,10 @@ export class LambdaAwsClientService {
maxSockets: LAMBDA_CLIENT_MAX_SOCKETS,
}),
});
+
+ addLambdaRateLimitErrorMiddleware(lambdaClient);
+
+ this.lambdaClient = lambdaClient;
}
return this.lambdaClient;
diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-executor-manager.service.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-executor-manager.service.ts
index 0904fa63887..57d1c760a6d 100644
--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-executor-manager.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/services/lambda-executor-manager.service.ts
@@ -320,8 +320,9 @@ export class LambdaExecutorManagerService {
} catch (error) {
if (
error instanceof LogicFunctionException &&
- error.code ===
- LogicFunctionExceptionCode.LOGIC_FUNCTION_DEPENDENCIES_SIZE_EXCEEDED
+ (error.code ===
+ LogicFunctionExceptionCode.LOGIC_FUNCTION_DEPENDENCIES_SIZE_EXCEEDED ||
+ error.code === LogicFunctionExceptionCode.LOGIC_FUNCTION_RATE_LIMITED)
) {
throw error;
}
@@ -344,6 +345,13 @@ export class LambdaExecutorManagerService {
applicationUniversalIdentifier,
});
} catch (error) {
+ if (
+ error instanceof LogicFunctionException &&
+ error.code === LogicFunctionExceptionCode.LOGIC_FUNCTION_RATE_LIMITED
+ ) {
+ throw error;
+ }
+
this.logger.error(
`Failed to get SDK layer for function ${flatLogicFunction.id}: ${error instanceof Error ? error.message : String(error)}`,
error instanceof Error ? error.stack : undefined,
diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/utils/__tests__/is-lambda-rate-limit-error.util.spec.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/utils/__tests__/is-lambda-rate-limit-error.util.spec.ts
new file mode 100644
index 00000000000..6c4555676fa
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/utils/__tests__/is-lambda-rate-limit-error.util.spec.ts
@@ -0,0 +1,22 @@
+import { isLambdaRateLimitError } from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/utils/is-lambda-rate-limit-error.util';
+
+const buildError = (name: string) =>
+ Object.assign(new Error('Rate exceeded'), { name });
+
+describe('isLambdaRateLimitError', () => {
+ it('detects the throttling errors the Lambda API returns', () => {
+ expect(isLambdaRateLimitError(buildError('TooManyRequestsException'))).toBe(
+ true,
+ );
+ expect(isLambdaRateLimitError(buildError('ThrottlingException'))).toBe(
+ true,
+ );
+ });
+
+ it('ignores other errors', () => {
+ expect(
+ isLambdaRateLimitError(buildError('ResourceConflictException')),
+ ).toBe(false);
+ expect(isLambdaRateLimitError('Rate exceeded')).toBe(false);
+ });
+});
diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/utils/add-lambda-rate-limit-error-middleware.util.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/utils/add-lambda-rate-limit-error-middleware.util.ts
new file mode 100644
index 00000000000..9a75ee707d8
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/utils/add-lambda-rate-limit-error-middleware.util.ts
@@ -0,0 +1,27 @@
+import { type Lambda } from '@aws-sdk/client-lambda';
+
+import { isLambdaRateLimitError } from 'src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/utils/is-lambda-rate-limit-error.util';
+import {
+ LogicFunctionException,
+ LogicFunctionExceptionCode,
+} from 'src/engine/metadata-modules/logic-function/logic-function.exception';
+
+export const addLambdaRateLimitErrorMiddleware = (lambdaClient: Lambda) => {
+ lambdaClient.middlewareStack.add(
+ (next, context) => async (args) => {
+ try {
+ return await next(args);
+ } catch (error) {
+ if (isLambdaRateLimitError(error)) {
+ throw new LogicFunctionException(
+ `Lambda rate limited ${context.commandName ?? 'a request'}`,
+ LogicFunctionExceptionCode.LOGIC_FUNCTION_RATE_LIMITED,
+ );
+ }
+
+ throw error;
+ }
+ },
+ { step: 'initialize', name: 'lambdaRateLimitErrorMiddleware' },
+ );
+};
diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/utils/is-lambda-rate-limit-error.util.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/utils/is-lambda-rate-limit-error.util.ts
new file mode 100644
index 00000000000..62bb503d419
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-drivers/drivers/lambda/utils/is-lambda-rate-limit-error.util.ts
@@ -0,0 +1,7 @@
+const LAMBDA_RATE_LIMIT_ERROR_NAMES = new Set([
+ 'TooManyRequestsException',
+ 'ThrottlingException',
+]);
+
+export const isLambdaRateLimitError = (error: unknown): boolean =>
+ error instanceof Error && LAMBDA_RATE_LIMIT_ERROR_NAMES.has(error.name);
diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-application-logic-functions.job-constants.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-application-logic-functions.job-constants.ts
deleted file mode 100644
index f700735e891..00000000000
--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-application-logic-functions.job-constants.ts
+++ /dev/null
@@ -1,16 +0,0 @@
-import { type QueueJobOptions } from 'src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface';
-
-export const WARM_UP_APPLICATION_LOGIC_FUNCTIONS_JOB_NAME =
- 'WarmUpApplicationLogicFunctionsJob';
-
-export const WARM_UP_APPLICATION_LOGIC_FUNCTIONS_JOB_OPTIONS: QueueJobOptions =
- {
- retryLimit: 1,
- backoff: { strategy: 'exponential', initialDelayMilliseconds: 1_000 },
- };
-
-export type WarmUpApplicationLogicFunctionsJobData = {
- workspaceId: string;
- applicationId: string;
- logicFunctionUniversalIdentifiers: string[];
-};
diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-logic-function.job-constants.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-logic-function.job-constants.ts
new file mode 100644
index 00000000000..9d97db1761e
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-logic-function.job-constants.ts
@@ -0,0 +1,14 @@
+import { type QueueJobOptions } from 'src/engine/core-modules/message-queue/drivers/interfaces/job-options.interface';
+
+export const WARM_UP_LOGIC_FUNCTION_JOB_NAME = 'WarmUpLogicFunctionJob';
+
+export const WARM_UP_LOGIC_FUNCTION_JOB_OPTIONS: QueueJobOptions = {
+ retryLimit: 5,
+ backoff: { strategy: 'exponential', initialDelayMilliseconds: 30_000 },
+};
+
+export type WarmUpLogicFunctionJobData = {
+ workspaceId: string;
+ applicationId: string;
+ logicFunctionUniversalIdentifier: string;
+};
diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-application-logic-functions.job.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-logic-function.job.ts
similarity index 56%
rename from packages/twenty-server/src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-application-logic-functions.job.ts
rename to packages/twenty-server/src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-logic-function.job.ts
index 02984b15a81..5888f75111f 100644
--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-application-logic-functions.job.ts
+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-logic-function.job.ts
@@ -1,22 +1,20 @@
import { LogicFunctionPrebuiltWarmUpService } from 'src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/logic-function-prebuilt-warm-up.service';
import {
- WARM_UP_APPLICATION_LOGIC_FUNCTIONS_JOB_NAME,
- type WarmUpApplicationLogicFunctionsJobData,
-} from 'src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-application-logic-functions.job-constants';
+ WARM_UP_LOGIC_FUNCTION_JOB_NAME,
+ type WarmUpLogicFunctionJobData,
+} from 'src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-logic-function.job-constants';
import { Process } from 'src/engine/core-modules/message-queue/decorators/process.decorator';
import { Processor } from 'src/engine/core-modules/message-queue/decorators/processor.decorator';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
-@Processor(MessageQueue.workspaceQueue)
-export class WarmUpApplicationLogicFunctionsJob {
+@Processor(MessageQueue.lambdaManagementQueue)
+export class WarmUpLogicFunctionJob {
constructor(
private readonly logicFunctionPrebuiltWarmUpService: LogicFunctionPrebuiltWarmUpService,
) {}
- @Process(WARM_UP_APPLICATION_LOGIC_FUNCTIONS_JOB_NAME)
- async handle(data: WarmUpApplicationLogicFunctionsJobData): Promise<void> {
- await this.logicFunctionPrebuiltWarmUpService.warmUpApplicationLogicFunctions(
- data,
- );
+ @Process(WARM_UP_LOGIC_FUNCTION_JOB_NAME)
+ async handle(data: WarmUpLogicFunctionJobData): Promise<void> {
+ await this.logicFunctionPrebuiltWarmUpService.warmUpLogicFunction(data);
}
}
diff --git a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/logic-function-prebuilt-warm-up.service.ts b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/logic-function-prebuilt-warm-up.service.ts
index a85652e6a32..568d683bcb8 100644
--- a/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/logic-function-prebuilt-warm-up.service.ts
+++ b/packages/twenty-server/src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/logic-function-prebuilt-warm-up.service.ts
@@ -1,6 +1,5 @@
import { Inject, Injectable, Logger } from '@nestjs/common';
-import chunk from 'lodash.chunk';
import { isDefined } from 'twenty-shared/utils';
import { type FlatApplication } from 'src/engine/core-modules/application/types/flat-application.type';
@@ -14,8 +13,6 @@ import { type FlatLogicFunction } from 'src/engine/metadata-modules/logic-functi
import { isLogicFunctionReadyForPrebuiltInstall } from 'src/engine/metadata-modules/logic-function/utils/is-logic-function-ready-for-prebuilt-install.util';
import { WorkspaceCacheService } from 'src/engine/workspace-cache/services/workspace-cache.service';
-const WARM_UP_CHUNK_SIZE = 10;
-
@Injectable()
export class LogicFunctionPrebuiltWarmUpService {
private readonly logger = new Logger(LogicFunctionPrebuiltWarmUpService.name);
@@ -50,6 +47,13 @@ export class LogicFunctionPrebuiltWarmUpService {
applicationUniversalIdentifier: flatApplication.universalIdentifier,
});
} catch (error) {
+ if (
+ error instanceof LogicFunctionException &&
+ error.code === LogicFunctionExceptionCode.LOGIC_FUNCTION_RATE_LIMITED
+ ) {
+ throw error;
+ }
+
const cause = error instanceof Error ? error.message : String(error);
this.logger.error(
@@ -66,14 +70,14 @@ export class LogicFunctionPrebuiltWarmUpService {
}
}
- async warmUpApplicationLogicFunctions({
+ async warmUpLogicFunction({
workspaceId,
applicationId,
- logicFunctionUniversalIdentifiers,
+ logicFunctionUniversalIdentifier,
}: {
workspaceId: string;
applicationId: string;
- logicFunctionUniversalIdentifiers: string[];
+ logicFunctionUniversalIdentifier: string;
}): Promise<void> {
const { flatLogicFunctionMaps, flatApplicationMaps } =
await this.workspaceCacheService.getOrRecompute(workspaceId, [
@@ -91,48 +95,22 @@ export class LogicFunctionPrebuiltWarmUpService {
return;
}
- const flatLogicFunctions = logicFunctionUniversalIdentifiers
- .map(
- (universalIdentifier) =>
- flatLogicFunctionMaps.byUniversalIdentifier[universalIdentifier],
- )
- .filter(isDefined)
- .filter(
- (flatLogicFunction) =>
- flatLogicFunction.applicationId === applicationId &&
- isLogicFunctionReadyForPrebuiltInstall(flatLogicFunction),
- );
-
- const failures: string[] = [];
-
- for (const flatLogicFunctionsChunk of chunk(
- flatLogicFunctions,
- WARM_UP_CHUNK_SIZE,
- )) {
- const results = await Promise.allSettled(
- flatLogicFunctionsChunk.map((flatLogicFunction) =>
- this.ensurePrebuiltBundleInstalled({
- flatLogicFunction,
- flatApplication,
- }),
- ),
- );
+ const flatLogicFunction =
+ flatLogicFunctionMaps.byUniversalIdentifier[
+ logicFunctionUniversalIdentifier
+ ];
- results.forEach((result, index) => {
- if (result.status === 'rejected') {
- failures.push(
- `${flatLogicFunctionsChunk[index].id}: ${result.reason instanceof Error ? result.reason.message : String(result.reason)}`,
- );
- }
- });
+ if (
+ !isDefined(flatLogicFunction) ||
+ flatLogicFunction.applicationId !== applicationId ||
+ !isLogicFunctionReadyForPrebuiltInstall(flatLogicFunction)
+ ) {
+ return;
}
- if (failures.length > 0) {
- throw new LogicFunctionException(
- `Failed to warm up ${failures.length} of ${flatLogicFunctions.length} prebuilt logic functions ` +
- `for application ${applicationId} in workspace ${workspaceId}: ${failures.join('; ')}`,
- LogicFunctionExceptionCode.LOGIC_FUNCTION_PREBUILT_BUNDLE_NOT_INSTALLED,
- );
- }
+ await this.ensurePrebuiltBundleInstalled({
+ flatLogicFunction,
+ flatApplication,
+ });
}
}
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
index cb97ceedb3c..d737c631708 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
@@ -254,6 +254,14 @@ export class BullMQDriver
...(isDefined(options?.maxStalledCount)
? { maxStalledCount: options.maxStalledCount }
: {}),
+ ...(isDefined(options?.rateLimit)
+ ? {
+ limiter: {
+ max: options.rateLimit.max,
+ duration: options.rateLimit.durationMs,
+ },
+ }
+ : {}),
metrics: {
maxDataPoints: MetricsTime.ONE_WEEK,
collectInterval: 60000,
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/interfaces/message-queue-worker-options.interface.ts b/packages/twenty-server/src/engine/core-modules/message-queue/interfaces/message-queue-worker-options.interface.ts
index 088a18b2220..fa6d5ed3740 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/interfaces/message-queue-worker-options.interface.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/interfaces/message-queue-worker-options.interface.ts
@@ -1,6 +1,7 @@
export interface MessageQueueWorkerOptions {
concurrency?: number;
globalConcurrency?: number;
+ rateLimit?: { max: number; durationMs: number };
lockDuration?: number;
maxStalledCount?: number;
boundedShutdownDrain?: boolean;
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/jobs.module.ts b/packages/twenty-server/src/engine/core-modules/message-queue/jobs.module.ts
index 1005f4fdfc9..9e47c2a29fb 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/jobs.module.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/jobs.module.ts
@@ -52,7 +52,7 @@ import { WorkspaceModule } from 'src/engine/core-modules/workspace/workspace.mod
import { AiAgentTriggerModule } from 'src/engine/metadata-modules/ai/ai-agent-trigger/ai-agent-trigger.module';
import { AiChatModule } from 'src/engine/metadata-modules/ai/ai-chat/ai-chat.module';
import { LogicFunctionPrebuiltWarmUpModule } from 'src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/logic-function-prebuilt-warm-up.module';
-import { WarmUpApplicationLogicFunctionsJob } from 'src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-application-logic-functions.job';
+import { WarmUpLogicFunctionJob } from 'src/engine/core-modules/logic-function/logic-function-prebuilt-warm-up/jobs/warm-up-logic-function.job';
import { LogicFunctionModule } from 'src/engine/metadata-modules/logic-function/logic-function.module';
import { NavigationMenuItemModule } from 'src/engine/metadata-modules/navigation-menu-item/navigation-menu-item.module';
import { ObjectMetadataModule } from 'src/engine/metadata-modules/object-metadata/object-metadata.module';
@@ -144,7 +144,7 @@ import { WorkflowModule } from 'src/modules/workflow/workflow.module';
TriggerInstallApplicationJob,
TriggerUninstallApplicationJob,
TriggerUpgradeApplicationJob,
- WarmUpApplicationLogicFunctionsJob,
+ WarmUpLogicFunctionJob,
RunDeferredWorkspaceMigrationActionsJob,
DeferredWorkspaceMigrationActionRecoveryCronJob,
],
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-worker-config.constant.ts b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-worker-config.constant.ts
index 71c74ae0db9..712a0479f85 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-worker-config.constant.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-worker-config.constant.ts
@@ -10,6 +10,7 @@ import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queu
// concurrency: max jobs processed in parallel per worker process
// globalConcurrency: optional, max jobs processed in parallel across all
// worker processes
+// rateLimit: optional, max jobs started per durationMs across all workers
// lockDuration: ms a job may run before BullMQ considers it stalled
// maxStalledCount: times a stalled job is re-queued before failing permanently
// boundedShutdownDrain: on shutdown, abort still-active jobs after
@@ -18,9 +19,9 @@ import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queu
export type MessageQueueWorkerConfig = {
priority: number;
workerOptions: Required<
- Omit<MessageQueueWorkerOptions, 'globalConcurrency'>
+ Omit<MessageQueueWorkerOptions, 'globalConcurrency' | 'rateLimit'>
> &
- Pick<MessageQueueWorkerOptions, 'globalConcurrency'>;
+ Pick<MessageQueueWorkerOptions, 'globalConcurrency' | 'rateLimit'>;
};
export const MESSAGE_QUEUE_WORKER_CONFIG: Record<
@@ -218,6 +219,16 @@ export const MESSAGE_QUEUE_WORKER_CONFIG: Record<
boundedShutdownDrain: false,
},
},
+ [MessageQueue.lambdaManagementQueue]: {
+ priority: 6,
+ workerOptions: {
+ concurrency: 2,
+ rateLimit: { max: 2, durationMs: 1_000 },
+ lockDuration: 60_000,
+ maxStalledCount: 1,
+ boundedShutdownDrain: false,
+ },
+ },
[MessageQueue.applicationLifecycleHookQueue]: {
priority: 4,
workerOptions: {
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.constants.ts b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.constants.ts
index bed18e85a5d..83be4810cea 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.constants.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.constants.ts
@@ -24,6 +24,7 @@ export enum MessageQueue {
delayedJobsQueue = 'delayed-jobs-queue',
deleteCascadeQueue = 'delete-cascade-queue',
logicFunctionQueue = 'logic-function-queue',
+ lambdaManagementQueue = 'lambda-management-queue',
applicationLifecycleHookQueue = 'application-lifecycle-hook-queue',
applicationUpgradeQueue = 'application-upgrade-queue',
triggerQueue = 'trigger-queue',
diff --git a/packages/twenty-server/src/engine/metadata-modules/logic-function/logic-function.exception.ts b/packages/twenty-server/src/engine/metadata-modules/logic-function/logic-function.exception.ts
index 4874acbaaea..1cb5f04f32b 100644
--- a/packages/twenty-server/src/engine/metadata-modules/logic-function/logic-function.exception.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/logic-function/logic-function.exception.ts
@@ -1,4 +1,6 @@
import { type MessageDescriptor } from '@lingui/core';
+import { HttpStatus } from '@nestjs/common';
+
import { msg } from '@lingui/core/macro';
import { assertUnreachable } from 'twenty-shared/utils';
@@ -22,6 +24,7 @@ export enum LogicFunctionExceptionCode {
LOGIC_FUNCTION_INVALID_SEED_PROJECT = 'LOGIC_FUNCTION_INVALID_SEED_PROJECT',
INVALID_LOGIC_FUNCTION_INPUT = 'INVALID_LOGIC_FUNCTION_INPUT',
LOGIC_FUNCTION_PREBUILT_BUNDLE_NOT_INSTALLED = 'LOGIC_FUNCTION_PREBUILT_BUNDLE_NOT_INSTALLED',
+ LOGIC_FUNCTION_RATE_LIMITED = 'LOGIC_FUNCTION_RATE_LIMITED',
}
const getLogicFunctionExceptionUserFriendlyMessage = (
@@ -62,6 +65,8 @@ const getLogicFunctionExceptionUserFriendlyMessage = (
return msg`Invalid logic function input.`;
case LogicFunctionExceptionCode.LOGIC_FUNCTION_PREBUILT_BUNDLE_NOT_INSTALLED:
return msg`Prebuilt bundle is not installed on the function. Rebuild and try again.`;
+ case LogicFunctionExceptionCode.LOGIC_FUNCTION_RATE_LIMITED:
+ return msg`Function provider is busy. Try again in a few minutes.`;
default:
assertUnreachable(code);
}
@@ -77,6 +82,10 @@ export class LogicFunctionException extends CustomException<LogicFunctionExcepti
userFriendlyMessage:
userFriendlyMessage ??
getLogicFunctionExceptionUserFriendlyMessage(code),
+ statusCode:
+ code === LogicFunctionExceptionCode.LOGIC_FUNCTION_RATE_LIMITED
+ ? HttpStatus.TOO_MANY_REQUESTS
+ : undefined,
});
}
}
diff --git a/packages/twenty-server/src/engine/metadata-modules/logic-function/utils/logic-function-graphql-api-exception-handler.utils.ts b/packages/twenty-server/src/engine/metadata-modules/logic-function/utils/logic-function-graphql-api-exception-handler.utils.ts
index 2ed710ef4e9..680499ab39d 100644
--- a/packages/twenty-server/src/engine/metadata-modules/logic-function/utils/logic-function-graphql-api-exception-handler.utils.ts
+++ b/packages/twenty-server/src/engine/metadata-modules/logic-function/utils/logic-function-graphql-api-exception-handler.utils.ts
@@ -32,6 +32,7 @@ export const logicFunctionGraphQLApiExceptionHandler = (error: any) => {
case LogicFunctionExceptionCode.LOGIC_FUNCTION_INVALID_SEED_PROJECT:
case LogicFunctionExceptionCode.LOGIC_FUNCTION_PLATFORM_EXECUTION_ERROR:
case LogicFunctionExceptionCode.LOGIC_FUNCTION_LAYER_BUILD_FAILED:
+ case LogicFunctionExceptionCode.LOGIC_FUNCTION_RATE_LIMITED:
throw error;
case LogicFunctionExceptionCode.LOGIC_FUNCTION_COMPILATION_FAILED:
case LogicFunctionExceptionCode.LOGIC_FUNCTION_DEPENDENCIES_SIZE_EXCEEDED: