The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
Drain queue jobs before the worker closes its clients (#28176)
Charles Bochetedf62a4bf05f
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.tsindex aa58a869f39..f99f7783aee 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/bullmq.driver.ts@@ -72,6 +72,7 @@ export class BullMQDriverprivate workerOptionsMap: Partial<Record<MessageQueue, MessageQueueWorkerOptions>> = {};+ private workersDrain: Promise<void> | undefined;constructor(private options: BullMQDriverOptions,@@ -136,23 +137,12 @@ export class BullMQDriver}async onModuleDestroy() {- const workers = Object.entries(this.workerMap) as [MessageQueue, Worker][];const queues = Object.values(this.queueMap);- if (workers.length > 0) {- this.logger.log(- `Draining active jobs on queues: ${workers.map(([queueName]) => queueName).join(', ')}`,- );- }-let workerCloseError: unknown;try {- await Promise.all(- workers.map(([queueName, worker]) =>- this.closeWorker(queueName, worker),- ),- );+ await this.closeWorkers();} catch (error) {workerCloseError = error;}@@ -176,6 +166,35 @@ export class BullMQDriverthis.logger.log('Message queue shutdown complete');}+ closeWorkers(): Promise<void> {+ this.workersDrain ??= this.drainWorkers();++ return this.workersDrain;+ }++ private async drainWorkers(): Promise<void> {+ const workers = Object.entries(this.workerMap) as [MessageQueue, Worker][];++ if (workers.length > 0) {+ this.logger.log(+ `Draining active jobs on queues: ${workers.map(([queueName]) => queueName).join(', ')}`,+ );+ }++ const closeResults = await Promise.allSettled(+ workers.map(([queueName, worker]) => this.closeWorker(queueName, worker)),+ );++ const failedClose = closeResults.find(+ (closeResult): closeResult is PromiseRejectedResult =>+ closeResult.status === 'rejected',+ );++ if (isDefined(failedClose)) {+ throw failedClose.reason;+ }+ }+private async closeWorker(queueName: MessageQueue,worker: Worker,diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface.ts b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface.tsindex 0a3d0474bb3..73b873677d2 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface.ts@@ -54,6 +54,7 @@ export interface MessageQueueDriver {jobId?: string;}): Promise<void>;register?(queueName: MessageQueue): void;+ closeWorkers?(): Promise<void>;getInFlightJobs?<T extends MessageQueueJobData>(queueName: MessageQueue,): Promise<InFlightQueueJob<T>[]>;diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-core.module.ts b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-core.module.tsindex 9f4219bb09b..79107373eb0 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-core.module.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue-core.module.ts@@ -22,6 +22,7 @@ import {} from 'src/engine/core-modules/message-queue/message-queue.module-definition';import { ApplicationJobEnqueueThrottlerService } from 'src/engine/core-modules/message-queue/services/application-job-enqueue-throttler.service';import { MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';+import { MessageQueueWorkerDrainService } from 'src/engine/core-modules/message-queue/services/message-queue-worker-drain.service';import { getQueueToken } from 'src/engine/core-modules/message-queue/utils/get-queue-token.util';import { MetricsModule } from 'src/engine/core-modules/metrics/metrics.module';import { ThrottlerModule } from 'src/engine/core-modules/throttler/throttler.module';@@ -53,12 +54,14 @@ export class MessageQueueCoreModule extends ConfigurableModuleClass {providers: [...(dynamicModule.providers ?? []),ApplicationJobEnqueueThrottlerService,+ MessageQueueWorkerDrainService,driverProvider,...queueProviders,],exports: [...(dynamicModule.exports ?? []),ApplicationJobEnqueueThrottlerService,+ MessageQueueWorkerDrainService,...Object.values(MessageQueue).map((queueName) =>getQueueToken(queueName),),@@ -95,12 +98,14 @@ export class MessageQueueCoreModule extends ConfigurableModuleClass {providers: [...(dynamicModule.providers ?? []),ApplicationJobEnqueueThrottlerService,+ MessageQueueWorkerDrainService,driverProvider,...queueProviders,],exports: [...(dynamicModule.exports ?? []),ApplicationJobEnqueueThrottlerService,+ MessageQueueWorkerDrainService,...Object.values(MessageQueue).map((queueName) =>getQueueToken(queueName),),diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/services/message-queue-worker-drain.service.ts b/packages/twenty-server/src/engine/core-modules/message-queue/services/message-queue-worker-drain.service.tsnew file mode 100644index 00000000000..04e389f50e7--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/message-queue/services/message-queue-worker-drain.service.ts@@ -0,0 +1,25 @@+import { Inject, Injectable, Logger } from '@nestjs/common';++import { MessageQueueDriver } from 'src/engine/core-modules/message-queue/drivers/interfaces/message-queue-driver.interface';++import { QUEUE_DRIVER } from 'src/engine/core-modules/message-queue/message-queue.constants';++@Injectable()+export class MessageQueueWorkerDrainService {+ private readonly logger = new Logger(MessageQueueWorkerDrainService.name);++ constructor(+ @Inject(QUEUE_DRIVER) private readonly driver: MessageQueueDriver,+ ) {}++ async drainWorkers(): Promise<void> {+ try {+ await this.driver.closeWorkers?.();+ } catch (error) {+ this.logger.error(+ 'Failed to drain queue workers, continuing shutdown',+ error,+ );+ }+ }+}diff --git a/packages/twenty-server/src/queue-worker/queue-worker.module.ts b/packages/twenty-server/src/queue-worker/queue-worker.module.tsindex 3151c40f544..6cdbc4222a0 100644--- a/packages/twenty-server/src/queue-worker/queue-worker.module.ts+++ b/packages/twenty-server/src/queue-worker/queue-worker.module.ts@@ -1,8 +1,9 @@-import { Module } from '@nestjs/common';+import { Module, type OnModuleDestroy } from '@nestjs/common';import { CoreEngineModule } from 'src/engine/core-modules/core-engine.module';import { JobsModule } from 'src/engine/core-modules/message-queue/jobs.module';import { MessageQueueModule } from 'src/engine/core-modules/message-queue/message-queue.module';+import { MessageQueueWorkerDrainService } from 'src/engine/core-modules/message-queue/services/message-queue-worker-drain.service';import { TwentyOrmModule } from 'src/engine/twenty-orm/twenty-orm.module';import { WorkspaceEventEmitterModule } from 'src/engine/workspace-event-emitter/workspace-event-emitter.module';import { ModulesModule } from 'src/modules/modules.module';@@ -17,4 +18,12 @@ import { ModulesModule } from 'src/modules/modules.module';TwentyOrmModule,],})-export class QueueWorkerModule {}+export class QueueWorkerModule implements OnModuleDestroy {+ constructor(+ private readonly messageQueueWorkerDrainService: MessageQueueWorkerDrainService,+ ) {}++ async onModuleDestroy() {+ await this.messageQueueWorkerDrainService.drainWorkers();+ }+}