The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Consolidate per-queue worker tuning in one explicit config file (#23229)
martmullabc82d66b72c
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/constants/ai-stream-lock-duration.constant.ts b/packages/twenty-server/src/engine/core-modules/message-queue/constants/ai-stream-lock-duration.constant.tsdeleted file mode 100644index c739aa89437..00000000000--- a/packages/twenty-server/src/engine/core-modules/message-queue/constants/ai-stream-lock-duration.constant.ts+++ /dev/null@@ -1 +0,0 @@-export const AI_STREAM_LOCK_DURATION_MS = 10 * 60 * 1000;diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.tsindex 0aefd7278d4..8fb5080f4d0 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts@@ -30,7 +30,7 @@ import {import { type MessageQueueWorkerOptions } from 'src/engine/core-modules/message-queue/interfaces/message-queue-worker-options.interface';import { QUEUE_RETENTION } from 'src/engine/core-modules/message-queue/constants/queue-retention.constants';-import { MESSAGE_QUEUE_PRIORITY } from 'src/engine/core-modules/message-queue/message-queue-priority.constant';+import { MESSAGE_QUEUE_WORKER_CONFIG } from 'src/engine/core-modules/message-queue/message-queue-worker-config.constant';import { type MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';import { getJobKey } from 'src/engine/core-modules/message-queue/utils/get-job-key.util';import { type MetricsService } from 'src/engine/core-modules/metrics/metrics.service';@@ -337,7 +337,8 @@ export class BullMQDriverreturn {// We suffix the id with V4() to make sure ids are unique so we can add a waiting job when a job related with the same option.id is runningjobId: options?.id ? `${options.id}-${v4()}` : undefined,- priority: options?.priority ?? MESSAGE_QUEUE_PRIORITY[queueName],+ priority:+ options?.priority ?? MESSAGE_QUEUE_WORKER_CONFIG[queueName].priority,attempts: 1 + (options?.retryLimit || 0),removeOnComplete: {age: QUEUE_RETENTION.completedMaxAge,diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-priority.constant.ts b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-priority.constant.tsdeleted file mode 100644index 5a8a158153e..00000000000--- a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-priority.constant.ts+++ /dev/null@@ -1,21 +0,0 @@-import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';--export const MESSAGE_QUEUE_PRIORITY = {- [MessageQueue.billingQueue]: 1,- [MessageQueue.entityEventsToDbQueue]: 1,- [MessageQueue.emailQueue]: 1,- [MessageQueue.workflowQueue]: 2,- [MessageQueue.webhookQueue]: 2,- [MessageQueue.messagingQueue]: 2,- [MessageQueue.delayedJobsQueue]: 3,- [MessageQueue.calendarQueue]: 4,- [MessageQueue.contactCreationQueue]: 4,- [MessageQueue.taskAssignedQueue]: 4,- [MessageQueue.logicFunctionQueue]: 4,- [MessageQueue.workspaceQueue]: 5,- [MessageQueue.triggerQueue]: 5,- [MessageQueue.deleteCascadeQueue]: 6,- [MessageQueue.cronQueue]: 7,- [MessageQueue.aiQueue]: 5,- [MessageQueue.aiStreamQueue]: 2,-};diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-worker-config.constant.ts b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-worker-config.constant.tsnew file mode 100644index 00000000000..624b2e96d0c--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-worker-config.constant.ts@@ -0,0 +1,180 @@+import { type MessageQueueWorkerOptions } from 'src/engine/core-modules/message-queue/interfaces/message-queue-worker-options.interface';+import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';++// Single source of truth to pilot worker behavior per queue. Every value is+// explicit on purpose (Record + Required): adding a queue without declaring+// its full configuration is a compile error, and no queue silently relies on+// implicit BullMQ defaults.+//+// priority: applied when enqueuing, lower value is processed first+// concurrency: max jobs processed in parallel per worker process+// lockDuration: ms a job may run before BullMQ considers it stalled+// maxStalledCount: times a stalled job is re-queued before failing permanently+// boundedShutdownDrain: on shutdown, abort still-active jobs after+// AI_STREAM_SHUTDOWN_DRAIN_MS instead of waiting for them to finish++export type MessageQueueWorkerConfig = {+ priority: number;+ workerOptions: Required<MessageQueueWorkerOptions>;+};++export const MESSAGE_QUEUE_WORKER_CONFIG: Record<+ MessageQueue,+ MessageQueueWorkerConfig+> = {+ [MessageQueue.taskAssignedQueue]: {+ priority: 4,+ workerOptions: {+ concurrency: 1,+ lockDuration: 30_000,+ maxStalledCount: 1,+ boundedShutdownDrain: false,+ },+ },+ [MessageQueue.messagingQueue]: {+ priority: 2,+ workerOptions: {+ concurrency: 1,+ lockDuration: 30_000,+ maxStalledCount: 1,+ boundedShutdownDrain: false,+ },+ },+ [MessageQueue.webhookQueue]: {+ priority: 2,+ workerOptions: {+ concurrency: 1,+ lockDuration: 30_000,+ maxStalledCount: 1,+ boundedShutdownDrain: false,+ },+ },+ [MessageQueue.cronQueue]: {+ priority: 7,+ workerOptions: {+ concurrency: 1,+ lockDuration: 30_000,+ maxStalledCount: 1,+ boundedShutdownDrain: false,+ },+ },+ [MessageQueue.emailQueue]: {+ priority: 1,+ workerOptions: {+ concurrency: 1,+ lockDuration: 30_000,+ maxStalledCount: 1,+ boundedShutdownDrain: false,+ },+ },+ [MessageQueue.calendarQueue]: {+ priority: 4,+ workerOptions: {+ concurrency: 1,+ lockDuration: 30_000,+ maxStalledCount: 1,+ boundedShutdownDrain: false,+ },+ },+ [MessageQueue.contactCreationQueue]: {+ priority: 4,+ workerOptions: {+ concurrency: 1,+ lockDuration: 30_000,+ maxStalledCount: 1,+ boundedShutdownDrain: false,+ },+ },+ [MessageQueue.billingQueue]: {+ priority: 1,+ workerOptions: {+ concurrency: 1,+ lockDuration: 30_000,+ maxStalledCount: 1,+ boundedShutdownDrain: false,+ },+ },+ [MessageQueue.workspaceQueue]: {+ priority: 5,+ workerOptions: {+ concurrency: 1,+ lockDuration: 30_000,+ maxStalledCount: 1,+ boundedShutdownDrain: false,+ },+ },+ [MessageQueue.entityEventsToDbQueue]: {+ priority: 1,+ workerOptions: {+ concurrency: 1,+ lockDuration: 30_000,+ maxStalledCount: 1,+ boundedShutdownDrain: false,+ },+ },+ [MessageQueue.workflowQueue]: {+ priority: 2,+ workerOptions: {+ concurrency: 1,+ lockDuration: 30_000,+ maxStalledCount: 1,+ boundedShutdownDrain: false,+ },+ },+ [MessageQueue.delayedJobsQueue]: {+ priority: 3,+ workerOptions: {+ concurrency: 1,+ lockDuration: 30_000,+ maxStalledCount: 1,+ boundedShutdownDrain: false,+ },+ },+ [MessageQueue.deleteCascadeQueue]: {+ priority: 6,+ workerOptions: {+ concurrency: 1,+ lockDuration: 30_000,+ maxStalledCount: 1,+ boundedShutdownDrain: false,+ },+ },+ [MessageQueue.logicFunctionQueue]: {+ priority: 4,+ workerOptions: {+ concurrency: 10,+ lockDuration: 30_000,+ maxStalledCount: 1,+ boundedShutdownDrain: false,+ },+ },+ [MessageQueue.triggerQueue]: {+ priority: 5,+ workerOptions: {+ concurrency: 1,+ lockDuration: 30_000,+ maxStalledCount: 1,+ boundedShutdownDrain: false,+ },+ },+ [MessageQueue.aiQueue]: {+ priority: 5,+ workerOptions: {+ concurrency: 1,+ lockDuration: 30_000,+ maxStalledCount: 1,+ boundedShutdownDrain: false,+ },+ },+ [MessageQueue.aiStreamQueue]: {+ priority: 2,+ workerOptions: {+ concurrency: 20,+ // 10 minutes: a stream job holds its lock for the whole stream duration+ lockDuration: 600_000,+ // A stalled stream cannot be resumed client-side, never re-queue it+ maxStalledCount: 0,+ boundedShutdownDrain: true,+ },+ },+};diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-worker-options.constant.ts b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-worker-options.constant.tsdeleted file mode 100644index 8d5ee0defa1..00000000000--- a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-worker-options.constant.ts+++ /dev/null@@ -1,15 +0,0 @@-import { AI_STREAM_LOCK_DURATION_MS } from 'src/engine/core-modules/message-queue/constants/ai-stream-lock-duration.constant';-import { type MessageQueueWorkerOptions } from 'src/engine/core-modules/message-queue/interfaces/message-queue-worker-options.interface';-import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';--export const QUEUE_WORKER_OPTIONS: Partial<- Record<MessageQueue, MessageQueueWorkerOptions>-> = {- [MessageQueue.aiStreamQueue]: {- concurrency: 20,- lockDuration: AI_STREAM_LOCK_DURATION_MS,- maxStalledCount: 0,- boundedShutdownDrain: true,- },- [MessageQueue.logicFunctionQueue]: { concurrency: 10 },-};diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.explorer.ts b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.explorer.tsindex 6030ccf8e67..333e4a2f794 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.explorer.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.explorer.ts@@ -18,8 +18,8 @@ import { type MessageQueueWorkerOptions } from 'src/engine/core-modules/message-import { ExceptionHandlerService } from 'src/engine/core-modules/exception-handler/exception-handler.service';import { MessageQueueMetadataAccessor } from 'src/engine/core-modules/message-queue/message-queue-metadata.accessor';import { type MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';+import { MESSAGE_QUEUE_WORKER_CONFIG } from 'src/engine/core-modules/message-queue/message-queue-worker-config.constant';import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';-import { QUEUE_WORKER_OPTIONS } from 'src/engine/core-modules/message-queue/message-queue-worker-options.constant';import { getQueueToken } from 'src/engine/core-modules/message-queue/utils/get-queue-token.util';import { shouldCreateWorkerForQueue } from 'src/engine/core-modules/message-queue/utils/should-create-worker-for-queue.util';import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';@@ -103,7 +103,7 @@ export class MessageQueueExplorer implements OnModuleInit {this.handleProcessorGroupCollection(processorGroupCollection,messageQueueService,- QUEUE_WORKER_OPTIONS[queueName],+ MESSAGE_QUEUE_WORKER_CONFIG[queueName].workerOptions,);}}@@ -181,7 +181,7 @@ export class MessageQueueExplorer implements OnModuleInit {private handleProcessorGroupCollection(processorGroupCollection: ProcessorGroup[],queue: MessageQueueService,- options?: MessageQueueWorkerOptions,+ options: MessageQueueWorkerOptions,) {queue.work(async (job) => {for (const processorGroup of processorGroupCollection) {