Skip to content

The open alternative to Salesforce, designed for AI.

Drain queue jobs before the worker closes its clients (#28176)

Charles Bochetedf62a4bf05f
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
index aa58a869f39..f99f7783aee 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts
@@ -72,6 +72,7 @@ export class BullMQDriver
private workerOptionsMap: Partial<
Record<MessageQueue, MessageQueueWorkerOptions>
> = {};
+ private workersDrain: Promise<void> | undefined;
constructor(
private options: BullMQDriverOptions,
@@ -136,23 +137,12 @@ export class BullMQDriver
}
async onModuleDestroy() {
- const workers = Object.entries(this.workerMap) as [MessageQueue, Worker][];
const queues = Object.values(this.queueMap);
- if (workers.length > 0) {
- this.logger.log(
- `Draining active jobs on queues: ${workers.map(([queueName]) => queueName).join(', ')}`,
- );
- }
-
let workerCloseError: unknown;
try {
- await Promise.all(
- workers.map(([queueName, worker]) =>
- this.closeWorker(queueName, worker),
- ),
- );
+ await this.closeWorkers();
} catch (error) {
workerCloseError = error;
}
@@ -176,6 +166,35 @@ export class BullMQDriver
this.logger.log('Message queue shutdown complete');
}
+ closeWorkers(): Promise<void> {
+ this.workersDrain ??= this.drainWorkers();
+
+ return this.workersDrain;
+ }
+
+ private async drainWorkers(): Promise<void> {
+ const workers = Object.entries(this.workerMap) as [MessageQueue, Worker][];
+
+ if (workers.length > 0) {
+ this.logger.log(
+ `Draining active jobs on queues: ${workers.map(([queueName]) => queueName).join(', ')}`,
+ );
+ }
+
+ const closeResults = await Promise.allSettled(
+ workers.map(([queueName, worker]) => this.closeWorker(queueName, worker)),
+ );
+
+ const failedClose = closeResults.find(
+ (closeResult): closeResult is PromiseRejectedResult =>
+ closeResult.status === 'rejected',
+ );
+
+ if (isDefined(failedClose)) {
+ throw failedClose.reason;
+ }
+ }
+
private async closeWorker(
queueName: MessageQueue,
worker: Worker,
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface.ts
index 0a3d0474bb3..73b873677d2 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface.ts
@@ -54,6 +54,7 @@ export interface MessageQueueDriver {
jobId?: string;
}): Promise<void>;
register?(queueName: MessageQueue): void;
+ closeWorkers?(): Promise<void>;
getInFlightJobs?<T extends MessageQueueJobData>(
queueName: MessageQueue,
): Promise<InFlightQueueJob<T>[]>;
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-core.module.ts b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-core.module.ts
index 9f4219bb09b..79107373eb0 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-core.module.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-core.module.ts
@@ -22,6 +22,7 @@ import {
} from 'src/engine/core-modules/message-queue/message-queue.module-definition';
import { ApplicationJobEnqueueThrottlerService } from 'src/engine/core-modules/message-queue/services/application-job-enqueue-throttler.service';
import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
+import { MessageQueueWorkerDrainService } from 'src/engine/core-modules/message-queue/services/message-queue-worker-drain.service';
import { getQueueToken } from 'src/engine/core-modules/message-queue/utils/get-queue-token.util';
import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module';
import { ThrottlerModule } from 'src/engine/core-modules/throttler/throttler.module';
@@ -53,12 +54,14 @@ export class MessageQueueCoreModule extends ConfigurableModuleClass {
providers: [
...(dynamicModule.providers ?? []),
ApplicationJobEnqueueThrottlerService,
+ MessageQueueWorkerDrainService,
driverProvider,
...queueProviders,
],
exports: [
...(dynamicModule.exports ?? []),
ApplicationJobEnqueueThrottlerService,
+ MessageQueueWorkerDrainService,
...Object.values(MessageQueue).map((queueName) =>
getQueueToken(queueName),
),
@@ -95,12 +98,14 @@ export class MessageQueueCoreModule extends ConfigurableModuleClass {
providers: [
...(dynamicModule.providers ?? []),
ApplicationJobEnqueueThrottlerService,
+ MessageQueueWorkerDrainService,
driverProvider,
...queueProviders,
],
exports: [
...(dynamicModule.exports ?? []),
ApplicationJobEnqueueThrottlerService,
+ MessageQueueWorkerDrainService,
...Object.values(MessageQueue).map((queueName) =>
getQueueToken(queueName),
),
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/services/message-queue-worker-drain.service.ts b/packages/twenty-server/src/engine/core-modules/message-queue/services/message-queue-worker-drain.service.ts
new file mode 100644
index 00000000000..04e389f50e7
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/services/message-queue-worker-drain.service.ts
@@ -0,0 +1,25 @@
+import { Inject, Injectable, Logger } from '@nestjs/common';
+
+import { MessageQueueDriver } from 'src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface';
+
+import { QUEUE_DRIVER } from 'src/engine/core-modules/message-queue/message-queue.constants';
+
+@Injectable()
+export class MessageQueueWorkerDrainService {
+ private readonly logger = new Logger(MessageQueueWorkerDrainService.name);
+
+ constructor(
+ @Inject(QUEUE_DRIVER) private readonly driver: MessageQueueDriver,
+ ) {}
+
+ async drainWorkers(): Promise<void> {
+ try {
+ await this.driver.closeWorkers?.();
+ } catch (error) {
+ this.logger.error(
+ 'Failed to drain queue workers, continuing shutdown',
+ error,
+ );
+ }
+ }
+}
diff --git a/packages/twenty-server/src/queue-worker/queue-worker.module.ts b/packages/twenty-server/src/queue-worker/queue-worker.module.ts
index 3151c40f544..6cdbc4222a0 100644
--- a/packages/twenty-server/src/queue-worker/queue-worker.module.ts
+++ b/packages/twenty-server/src/queue-worker/queue-worker.module.ts
@@ -1,8 +1,9 @@
-import { Module } from '@nestjs/common';
+import { Module, type OnModuleDestroy } from '@nestjs/common';
import { CoreEngineModule } from 'src/engine/core-modules/core-engine.module';
import { JobsModule } from 'src/engine/core-modules/message-queue/jobs.module';
import { MessageQueueModule } from 'src/engine/core-modules/message-queue/message-queue.module';
+import { MessageQueueWorkerDrainService } from 'src/engine/core-modules/message-queue/services/message-queue-worker-drain.service';
import { TwentyOrmModule } from 'src/engine/twenty-orm/twenty-orm.module';
import { WorkspaceEventEmitterModule } from 'src/engine/workspace-event-emitter/workspace-event-emitter.module';
import { ModulesModule } from 'src/modules/modules.module';
@@ -17,4 +18,12 @@ import { ModulesModule } from 'src/modules/modules.module';
TwentyOrmModule,
],
})
-export class QueueWorkerModule {}
+export class QueueWorkerModule implements OnModuleDestroy {
+ constructor(
+ private readonly messageQueueWorkerDrainService: MessageQueueWorkerDrainService,
+ ) {}
+
+ async onModuleDestroy() {
+ await this.messageQueueWorkerDrainService.drainWorkers();
+ }
+}