Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(metrics): record event-loop stalls that begin right after a render
Yudhi Armyndhariscc8e30433d63
diff --git a/src/modules/metrics/metrics.service.spec.ts b/src/modules/metrics/metrics.service.spec.tsindex 75b91f47d..b4af95f1c 100644--- a/src/modules/metrics/metrics.service.spec.ts+++ b/src/modules/metrics/metrics.service.spec.ts@@ -250,6 +250,25 @@ describe('MetricsService runtime series', () => {expect(sample(text, 'openwa_event_loop_delay_max_seconds')).toBeCloseTo(0.2, 9);});+ // Resetting Node's interval histogram also dropped its previous-tick timestamp, so the first gap+ // after each render went unrecorded, and with it a stall that began right there.+ it('records a stall that begins right after an uncached render', async () => {+ const svc = make();+ const idle = (): Promise<void> => new Promise(resolve => setTimeout(resolve, 60));+ await idle();+ await svc.render();+ const until = Date.now() + 300;+ while (Date.now() < until) {+ // block the event loop+ }+ await idle();+ (svc as unknown as { cachedRender: unknown }).cachedRender = null;++ const text = await svc.render();++ expect(sample(text, 'openwa_event_loop_delay_max_seconds')).toBeGreaterThanOrEqual(0.25);+ });+it('counts unhandled rejections by kind', async () => {const before = getUnhandledRejections();incrementUnhandledRejections('other');diff --git a/src/modules/metrics/metrics.service.ts b/src/modules/metrics/metrics.service.tsindex ce16907d3..f910330b3 100644--- a/src/modules/metrics/metrics.service.ts+++ b/src/modules/metrics/metrics.service.ts@@ -1,4 +1,4 @@-import { monitorEventLoopDelay } from 'node:perf_hooks';+import { createHistogram, performance } from 'node:perf_hooks';import {HttpException,HttpStatus,@@ -69,8 +69,19 @@ export class MetricsService implements OnModuleDestroy {*/private readonly failedScrapeLimiter = new SlidingWindowLimiter(10, 60_000);- /** Event-loop delay sampled between uncached renders (reset after each one). */- private readonly loopDelay = monitorEventLoopDelay({ resolution: LOOP_DELAY_RESOLUTION_MS });+ /**+ * Event-loop delay sampled between uncached renders (reset after each one). Sampled by a timer of+ * our own rather than monitorEventLoopDelay: resetting that histogram also drops its previous-tick+ * timestamp, so the first gap after every render, and any stall that began in it, went unrecorded.+ * Here reset() clears only the counts; `lastTick` carries across it.+ */+ private readonly loopDelay = createHistogram();+ private lastTick = performance.now();+ private readonly loopSampler = setInterval(() => {+ const now = performance.now();+ this.loopDelay.record(Math.max(1, Math.round((now - this.lastTick) * 1e6)));+ this.lastTick = now;+ }, LOOP_DELAY_RESOLUTION_MS).unref();constructor(private readonly config: ConfigService,@@ -78,12 +89,10 @@ export class MetricsService implements OnModuleDestroy {// Registered only with QUEUE_ENABLED=true (see metrics.module.ts); absent otherwise.@Optional() @InjectQueue(QUEUE_NAMES.WEBHOOK) private readonly webhookQueue?: Queue,@Optional() @InjectQueue(QUEUE_NAMES.INGRESS) private readonly ingressQueue?: Queue,- ) {- this.loopDelay.enable();- }+ ) {}onModuleDestroy(): void {- this.loopDelay.disable();+ clearInterval(this.loopSampler);}private get token(): string {