Skip to content

The open alternative to Salesforce, designed for AI.

fix: limit record export concurrency across workers (#26665)

Weikoef4f289ebad1
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts
index 8e30ebe4117..b0cd26f9840 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/__tests__/bullmq.driver.spec.ts
@@ -7,6 +7,8 @@ const mockGetJobs = jest.fn();
const mockGetJob = jest.fn();
const mockAdd = jest.fn();
const mockAddBulk = jest.fn();
+const mockSetGlobalConcurrency = jest.fn();
+const mockRemoveGlobalConcurrency = jest.fn();
jest.mock('bullmq', () => ({
Queue: jest.fn().mockImplementation(() => ({
@@ -14,6 +16,8 @@ jest.mock('bullmq', () => ({
getJob: mockGetJob,
add: mockAdd,
addBulk: mockAddBulk,
+ setGlobalConcurrency: mockSetGlobalConcurrency,
+ removeGlobalConcurrency: mockRemoveGlobalConcurrency,
})),
Worker: jest.fn().mockImplementation(() => ({ on: jest.fn() })),
MetricsTime: { ONE_WEEK: 1 },
@@ -191,3 +195,34 @@ describe('BullMQDriver progress', () => {
},
);
});
+
+describe('BullMQDriver global concurrency', () => {
+ const driver = new BullMQDriver(
+ {} as never,
+ {} as never,
+ {} as never,
+ {} as never,
+ );
+
+ driver.register(MessageQueue.recordExportQueue);
+
+ beforeEach(() => {
+ jest.clearAllMocks();
+ });
+
+ it('sets the global concurrency when the queue declares one', () => {
+ driver.work(MessageQueue.recordExportQueue, jest.fn(), {
+ globalConcurrency: 2,
+ });
+
+ expect(mockSetGlobalConcurrency).toHaveBeenCalledWith(2);
+ expect(mockRemoveGlobalConcurrency).not.toHaveBeenCalled();
+ });
+
+ it('removes the global concurrency when the queue declares none', () => {
+ driver.work(MessageQueue.recordExportQueue, jest.fn(), {});
+
+ expect(mockRemoveGlobalConcurrency).toHaveBeenCalledTimes(1);
+ expect(mockSetGlobalConcurrency).not.toHaveBeenCalled();
+ });
+});
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
index b1a3365eb9c..a8e0713d7ee 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
@@ -181,6 +181,28 @@ export class BullMQDriver
}
}
+ private async writeGlobalConcurrency(
+ queueName: MessageQueue,
+ globalConcurrency: number | undefined,
+ ): Promise<void> {
+ const queue = this.queueMap[queueName];
+
+ try {
+ if (isDefined(globalConcurrency)) {
+ await queue.setGlobalConcurrency(globalConcurrency);
+
+ return;
+ }
+
+ await queue.removeGlobalConcurrency();
+ } catch (error) {
+ this.logger.error(
+ `Failed to write global concurrency for queue ${queueName}`,
+ error,
+ );
+ }
+ }
+
work<T>(
queueName: MessageQueue,
handler: (job: MessageQueueJob<T>) => Promise<void>,
@@ -205,6 +227,8 @@ export class BullMQDriver
this.workerOptionsMap[queueName] = options;
+ void this.writeGlobalConcurrency(queueName, options?.globalConcurrency);
+
this.workerMap[queueName] = new Worker(
queueName,
async (job, _token, abortSignal) =>
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/interfaces/message-queue-worker-options.interface.ts b/packages/twenty-server/src/engine/core-modules/message-queue/interfaces/message-queue-worker-options.interface.ts
index 4fb070c2db5..088a18b2220 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/interfaces/message-queue-worker-options.interface.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/interfaces/message-queue-worker-options.interface.ts
@@ -1,5 +1,6 @@
export interface MessageQueueWorkerOptions {
concurrency?: number;
+ globalConcurrency?: number;
lockDuration?: number;
maxStalledCount?: number;
boundedShutdownDrain?: boolean;
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-worker-config.constant.ts b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-worker-config.constant.ts
index 72601162fe2..46416ebdf8e 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-worker-config.constant.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-worker-config.constant.ts
@@ -8,6 +8,8 @@ import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queu
//
// priority: applied when enqueuing, lower value is processed first
// concurrency: max jobs processed in parallel per worker process
+// globalConcurrency: optional, max jobs processed in parallel across all
+// worker processes
// lockDuration: ms a job may run before BullMQ considers it stalled
// maxStalledCount: times a stalled job is re-queued before failing permanently
// boundedShutdownDrain: on shutdown, abort still-active jobs after
@@ -15,7 +17,10 @@ import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queu
export type MessageQueueWorkerConfig = {
priority: number;
- workerOptions: Required<MessageQueueWorkerOptions>;
+ workerOptions: Required<
+ Omit<MessageQueueWorkerOptions, 'globalConcurrency'>
+ > &
+ Pick<MessageQueueWorkerOptions, 'globalConcurrency'>;
};
export const MESSAGE_QUEUE_WORKER_CONFIG: Record<
@@ -26,6 +31,7 @@ export const MESSAGE_QUEUE_WORKER_CONFIG: Record<
priority: 7,
workerOptions: {
concurrency: 1,
+ globalConcurrency: 2,
lockDuration: 60_000,
maxStalledCount: 0,
boundedShutdownDrain: false,