Skip to content

The open alternative to Salesforce, designed for AI.

feat(server): log in-flight jobs when the worker event loop stalls (#24767)

Charles Bochet2ae1dfbce810
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.explorer.ts b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.explorer.ts
index 7ed5c15d4f8..8cdd1f3903e 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.explorer.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.explorer.ts
@@ -20,6 +20,7 @@ import { MessageQueueMetadataAccessor } from 'src/engine/core-modules/message-qu
import { type MessageQueueService } from 'src/engine/core-modules/message-queue/services/message-queue.service';
import { MESSAGE_QUEUE_WORKER_CONFIG } from 'src/engine/core-modules/message-queue/message-queue-worker-config.constant';
import { MessageQueue } from 'src/engine/core-modules/message-queue/message-queue.constants';
+import { EventLoopStallMonitorService } from 'src/engine/core-modules/message-queue/services/event-loop-stall-monitor.service';
import { getQueueToken } from 'src/engine/core-modules/message-queue/utils/get-queue-token.util';
import { shouldCreateWorkerForQueue } from 'src/engine/core-modules/message-queue/utils/should-create-worker-for-queue.util';
import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service';
@@ -44,6 +45,7 @@ export class MessageQueueExplorer implements OnModuleInit {
private readonly metadataScanner: MetadataScanner,
private readonly exceptionHandlerService: ExceptionHandlerService,
private readonly twentyConfigService: TwentyConfigService,
+ private readonly eventLoopStallMonitorService: EventLoopStallMonitorService,
) {}
onModuleInit() {
@@ -104,6 +106,7 @@ export class MessageQueueExplorer implements OnModuleInit {
processorGroupCollection,
messageQueueService,
MESSAGE_QUEUE_WORKER_CONFIG[queueName].workerOptions,
+ queueName,
);
}
}
@@ -182,10 +185,22 @@ export class MessageQueueExplorer implements OnModuleInit {
processorGroupCollection: ProcessorGroup[],
queue: MessageQueueService,
options: MessageQueueWorkerOptions,
+ queueName: MessageQueue,
) {
queue.work(async (job) => {
- for (const processorGroup of processorGroupCollection) {
- await this.handleProcessor(processorGroup, job);
+ const stallMonitorToken =
+ this.eventLoopStallMonitorService.registerJobStart({
+ queueName,
+ jobName: job.name,
+ workspaceId: job.data?.workspaceId,
+ });
+
+ try {
+ for (const processorGroup of processorGroupCollection) {
+ await this.handleProcessor(processorGroup, job);
+ }
+ } finally {
+ this.eventLoopStallMonitorService.registerJobEnd(stallMonitorToken);
}
}, options);
}
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.module.ts b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.module.ts
index cfc74c7ad93..658c04a1b7a 100644
--- a/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.module.ts
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/message-queue.module.ts
@@ -4,6 +4,7 @@ import { DiscoveryModule } from '@nestjs/core';
import { MessageQueueCoreModule } from 'src/engine/core-modules/message-queue/message-queue-core.module';
import { MessageQueueMetadataAccessor } from 'src/engine/core-modules/message-queue/message-queue-metadata.accessor';
import { MessageQueueExplorer } from 'src/engine/core-modules/message-queue/message-queue.explorer';
+import { EventLoopStallMonitorService } from 'src/engine/core-modules/message-queue/services/event-loop-stall-monitor.service';
import {
type ASYNC_OPTIONS_TYPE,
type OPTIONS_TYPE,
@@ -23,7 +24,11 @@ export class MessageQueueModule {
return {
module: MessageQueueModule,
imports: [DiscoveryModule],
- providers: [MessageQueueExplorer, MessageQueueMetadataAccessor],
+ providers: [
+ MessageQueueExplorer,
+ MessageQueueMetadataAccessor,
+ EventLoopStallMonitorService,
+ ],
};
}
diff --git a/packages/twenty-server/src/engine/core-modules/message-queue/services/event-loop-stall-monitor.service.ts b/packages/twenty-server/src/engine/core-modules/message-queue/services/event-loop-stall-monitor.service.ts
new file mode 100644
index 00000000000..36f84445398
--- /dev/null
+++ b/packages/twenty-server/src/engine/core-modules/message-queue/services/event-loop-stall-monitor.service.ts
@@ -0,0 +1,138 @@
+import {
+ Injectable,
+ Logger,
+ type OnModuleDestroy,
+ type OnModuleInit,
+} from '@nestjs/common';
+
+import { isNonEmptyString } from '@sniptt/guards';
+import { isDefined } from 'twenty-shared/utils';
+
+const TICK_INTERVAL_MS = 500;
+const STALL_THRESHOLD_MS = 5_000;
+const LOG_MIN_INTERVAL_MS = 30_000;
+
+type MonitoredJob = {
+ queueName: string;
+ jobName: string;
+ workspaceId?: string;
+ startedAt: number;
+ endedAt?: number;
+};
+
+@Injectable()
+export class EventLoopStallMonitorService
+ implements OnModuleInit, OnModuleDestroy
+{
+ private readonly logger = new Logger(EventLoopStallMonitorService.name);
+
+ private readonly jobs = new Map<symbol, MonitoredJob>();
+ private tickHandle?: NodeJS.Timeout;
+ private lastTickAt = 0;
+ private lastLogAt = 0;
+ private suppressedStallCount = 0;
+
+ onModuleInit() {
+ this.lastTickAt = Date.now();
+ this.tickHandle = setInterval(() => this.tick(), TICK_INTERVAL_MS);
+ this.tickHandle.unref();
+ }
+
+ onModuleDestroy() {
+ if (isDefined(this.tickHandle)) {
+ clearInterval(this.tickHandle);
+ }
+ }
+
+ registerJobStart({
+ queueName,
+ jobName,
+ workspaceId,
+ }: {
+ queueName: string;
+ jobName: string;
+ workspaceId?: string;
+ }): symbol {
+ const token = Symbol(jobName);
+
+ this.jobs.set(token, {
+ queueName,
+ jobName,
+ workspaceId,
+ startedAt: Date.now(),
+ });
+
+ return token;
+ }
+
+ // Jobs are kept until the next tick instead of being removed: after a long
+ // synchronous block, the job's finally runs in a microtask before the
+ // overdue timer fires, so removing it here would hide the stalling job from
+ // the very log line meant to name it.
+ registerJobEnd(token: symbol) {
+ const job = this.jobs.get(token);
+
+ if (isDefined(job)) {
+ job.endedAt = Date.now();
+ }
+ }
+
+ private tick() {
+ const now = Date.now();
+ const previousTickAt = this.lastTickAt;
+ const stallMs = now - previousTickAt - TICK_INTERVAL_MS;
+
+ this.lastTickAt = now;
+
+ if (stallMs >= STALL_THRESHOLD_MS) {
+ if (now - this.lastLogAt >= LOG_MIN_INTERVAL_MS) {
+ this.logStall({ stallMs, now, previousTickAt });
+ this.lastLogAt = now;
+ this.suppressedStallCount = 0;
+ } else {
+ this.suppressedStallCount += 1;
+ }
+ }
+
+ for (const [token, job] of this.jobs) {
+ if (isDefined(job.endedAt) && job.endedAt < previousTickAt) {
+ this.jobs.delete(token);
+ }
+ }
+ }
+
+ private logStall({
+ stallMs,
+ now,
+ previousTickAt,
+ }: {
+ stallMs: number;
+ now: number;
+ previousTickAt: number;
+ }) {
+ const inFlight = [...this.jobs.values()].filter(
+ (job) => !isDefined(job.endedAt) || job.endedAt >= previousTickAt,
+ );
+
+ const jobs = inFlight
+ .map(
+ (job) =>
+ `${job.queueName}/${job.jobName}` +
+ (isNonEmptyString(job.workspaceId)
+ ? ` workspace=${job.workspaceId}`
+ : '') +
+ ` elapsedMs=${(job.endedAt ?? now) - job.startedAt}` +
+ (isDefined(job.endedAt) ? ' (just finished)' : ''),
+ )
+ .join('; ');
+
+ const suppressedSuffix =
+ this.suppressedStallCount > 0
+ ? ` (${this.suppressedStallCount} earlier stall(s) not logged)`
+ : '';
+
+ this.logger.warn(
+ `Event loop stalled for ${stallMs}ms with ${inFlight.length} job(s) in flight${suppressedSuffix}: ${isNonEmptyString(jobs) ? jobs : 'none'}`,
+ );
+ }
+}