The open alternative to Salesforce, designed for AI.
TypeScript crmcrm-systemcustomergood-first-issuegraphqlhacktoberfestjavascriptmarketingmonoreponestjsopen-sourcepostgresqlreactreactjssalestypescriptweb
feat(server): log in-flight jobs when the worker event loop stalls (#24767)
Charles Bochet2ae1dfbce810
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.explorer.ts b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.explorer.tsindex 7ed5c15d4f8..8cdd1f3903e 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.explorer.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.explorer.ts@@ -20,6 +20,7 @@ import { MessageQueueMetadataAccessor } from 'src/engine/core-modules/message-quimport { type MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';import { MESSAGE_QUEUE_WORKER_CONFIG } from 'src/engine/core-modules/message-queue/message-queue-worker-config.constant';import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';+import { EventLoopStallMonitorService } from 'src/engine/core-modules/message-queue/services/event-loop-stall-monitor.service';import { getQueueToken } from 'src/engine/core-modules/message-queue/utils/get-queue-token.util';import { shouldCreateWorkerForQueue } from 'src/engine/core-modules/message-queue/utils/should-create-worker-for-queue.util';import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';@@ -44,6 +45,7 @@ export class MessageQueueExplorer implements OnModuleInit {private readonly metadataScanner: MetadataScanner,private readonly exceptionHandlerService: ExceptionHandlerService,private readonly twentyConfigService: TwentyConfigService,+ private readonly eventLoopStallMonitorService: EventLoopStallMonitorService,) {}onModuleInit() {@@ -104,6 +106,7 @@ export class MessageQueueExplorer implements OnModuleInit {processorGroupCollection,messageQueueService,MESSAGE_QUEUE_WORKER_CONFIG[queueName].workerOptions,+ queueName,);}}@@ -182,10 +185,22 @@ export class MessageQueueExplorer implements OnModuleInit {processorGroupCollection: ProcessorGroup[],queue: MessageQueueService,options: MessageQueueWorkerOptions,+ queueName: MessageQueue,) {queue.work(async (job) => {- for (const processorGroup of processorGroupCollection) {- await this.handleProcessor(processorGroup, job);+ const stallMonitorToken =+ this.eventLoopStallMonitorService.registerJobStart({+ queueName,+ jobName: job.name,+ workspaceId: job.data?.workspaceId,+ });++ try {+ for (const processorGroup of processorGroupCollection) {+ await this.handleProcessor(processorGroup, job);+ }+ } finally {+ this.eventLoopStallMonitorService.registerJobEnd(stallMonitorToken);}}, options);}diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.module.ts b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.module.tsindex cfc74c7ad93..658c04a1b7a 100644--- a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.module.ts+++ b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.module.ts@@ -4,6 +4,7 @@ import { DiscoveryModule } from '@nestjs/core';import { MessageQueueCoreModule } from 'src/engine/core-modules/message-queue/message-queue-core.module';import { MessageQueueMetadataAccessor } from 'src/engine/core-modules/message-queue/message-queue-metadata.accessor';import { MessageQueueExplorer } from 'src/engine/core-modules/message-queue/message-queue.explorer';+import { EventLoopStallMonitorService } from 'src/engine/core-modules/message-queue/services/event-loop-stall-monitor.service';import {type ASYNC_OPTIONS_TYPE,type OPTIONS_TYPE,@@ -23,7 +24,11 @@ export class MessageQueueModule {return {module: MessageQueueModule,imports: [DiscoveryModule],- providers: [MessageQueueExplorer, MessageQueueMetadataAccessor],+ providers: [+ MessageQueueExplorer,+ MessageQueueMetadataAccessor,+ EventLoopStallMonitorService,+ ],};}diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/services/event-loop-stall-monitor.service.ts b/packages/twenty-server/src/engine/core-modules/message-queue/services/event-loop-stall-monitor.service.tsnew file mode 100644index 00000000000..36f84445398--- /dev/null+++ b/packages/twenty-server/src/engine/core-modules/message-queue/services/event-loop-stall-monitor.service.ts@@ -0,0 +1,138 @@+import {+ Injectable,+ Logger,+ type OnModuleDestroy,+ type OnModuleInit,+} from '@nestjs/common';++import { isNonEmptyString } from '@sniptt/guards';+import { isDefined } from 'twenty-shared/utils';++const TICK_INTERVAL_MS = 500;+const STALL_THRESHOLD_MS = 5_000;+const LOG_MIN_INTERVAL_MS = 30_000;++type MonitoredJob = {+ queueName: string;+ jobName: string;+ workspaceId?: string;+ startedAt: number;+ endedAt?: number;+};++@Injectable()+export class EventLoopStallMonitorService+ implements OnModuleInit, OnModuleDestroy+{+ private readonly logger = new Logger(EventLoopStallMonitorService.name);++ private readonly jobs = new Map<symbol, MonitoredJob>();+ private tickHandle?: NodeJS.Timeout;+ private lastTickAt = 0;+ private lastLogAt = 0;+ private suppressedStallCount = 0;++ onModuleInit() {+ this.lastTickAt = Date.now();+ this.tickHandle = setInterval(() => this.tick(), TICK_INTERVAL_MS);+ this.tickHandle.unref();+ }++ onModuleDestroy() {+ if (isDefined(this.tickHandle)) {+ clearInterval(this.tickHandle);+ }+ }++ registerJobStart({+ queueName,+ jobName,+ workspaceId,+ }: {+ queueName: string;+ jobName: string;+ workspaceId?: string;+ }): symbol {+ const token = Symbol(jobName);++ this.jobs.set(token, {+ queueName,+ jobName,+ workspaceId,+ startedAt: Date.now(),+ });++ return token;+ }++ // Jobs are kept until the next tick instead of being removed: after a long+ // synchronous block, the job's finally runs in a microtask before the+ // overdue timer fires, so removing it here would hide the stalling job from+ // the very log line meant to name it.+ registerJobEnd(token: symbol) {+ const job = this.jobs.get(token);++ if (isDefined(job)) {+ job.endedAt = Date.now();+ }+ }++ private tick() {+ const now = Date.now();+ const previousTickAt = this.lastTickAt;+ const stallMs = now - previousTickAt - TICK_INTERVAL_MS;++ this.lastTickAt = now;++ if (stallMs >= STALL_THRESHOLD_MS) {+ if (now - this.lastLogAt >= LOG_MIN_INTERVAL_MS) {+ this.logStall({ stallMs, now, previousTickAt });+ this.lastLogAt = now;+ this.suppressedStallCount = 0;+ } else {+ this.suppressedStallCount += 1;+ }+ }++ for (const [token, job] of this.jobs) {+ if (isDefined(job.endedAt) && job.endedAt < previousTickAt) {+ this.jobs.delete(token);+ }+ }+ }++ private logStall({+ stallMs,+ now,+ previousTickAt,+ }: {+ stallMs: number;+ now: number;+ previousTickAt: number;+ }) {+ const inFlight = [...this.jobs.values()].filter(+ (job) => !isDefined(job.endedAt) || job.endedAt >= previousTickAt,+ );++ const jobs = inFlight+ .map(+ (job) =>+ `${job.queueName}/${job.jobName}` ++ (isNonEmptyString(job.workspaceId)+ ? ` workspace=${job.workspaceId}`+ : '') ++ ` elapsedMs=${(job.endedAt ?? now) - job.startedAt}` ++ (isDefined(job.endedAt) ? ' (just finished)' : ''),+ )+ .join('; ');++ const suppressedSuffix =+ this.suppressedStallCount > 0+ ? ` (${this.suppressedStallCount} earlier stall(s) not logged)`+ : '';++ this.logger.warn(+ `Event loop stalled for ${stallMs}ms with ${inFlight.length} job(s) in flight${suppressedSuffix}: ${isNonEmptyString(jobs) ? jobs : 'none'}`,+ );+ }+}