Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(metrics): record event-loop stalls that begin right after a render

Yudhi Armyndhariscc8e30433d63
diff --git a/src/modules/metrics/metrics.service.spec.ts b/src/modules/metrics/metrics.service.spec.ts
index 75b91f47d..b4af95f1c 100644
--- a/src/modules/metrics/metrics.service.spec.ts
+++ b/src/modules/metrics/metrics.service.spec.ts
@@ -250,6 +250,25 @@ describe('MetricsService runtime series', () => {
expect(sample(text, 'openwa_event_loop_delay_max_seconds')).toBeCloseTo(0.2, 9);
});
+ // Resetting Node's interval histogram also dropped its previous-tick timestamp, so the first gap
+ // after each render went unrecorded, and with it a stall that began right there.
+ it('records a stall that begins right after an uncached render', async () => {
+ const svc = make();
+ const idle = (): Promise<void> => new Promise(resolve => setTimeout(resolve, 60));
+ await idle();
+ await svc.render();
+ const until = Date.now() + 300;
+ while (Date.now() < until) {
+ // block the event loop
+ }
+ await idle();
+ (svc as unknown as { cachedRender: unknown }).cachedRender = null;
+
+ const text = await svc.render();
+
+ expect(sample(text, 'openwa_event_loop_delay_max_seconds')).toBeGreaterThanOrEqual(0.25);
+ });
+
it('counts unhandled rejections by kind', async () => {
const before = getUnhandledRejections();
incrementUnhandledRejections('other');
diff --git a/src/modules/metrics/metrics.service.ts b/src/modules/metrics/metrics.service.ts
index ce16907d3..f910330b3 100644
--- a/src/modules/metrics/metrics.service.ts
+++ b/src/modules/metrics/metrics.service.ts
@@ -1,4 +1,4 @@
-import { monitorEventLoopDelay } from 'node:perf_hooks';
+import { createHistogram, performance } from 'node:perf_hooks';
import {
HttpException,
HttpStatus,
@@ -69,8 +69,19 @@ export class MetricsService implements OnModuleDestroy {
*/
private readonly failedScrapeLimiter = new SlidingWindowLimiter(10, 60_000);
- /** Event-loop delay sampled between uncached renders (reset after each one). */
- private readonly loopDelay = monitorEventLoopDelay({ resolution: LOOP_DELAY_RESOLUTION_MS });
+ /**
+ * Event-loop delay sampled between uncached renders (reset after each one). Sampled by a timer of
+ * our own rather than monitorEventLoopDelay: resetting that histogram also drops its previous-tick
+ * timestamp, so the first gap after every render, and any stall that began in it, went unrecorded.
+ * Here reset() clears only the counts; `lastTick` carries across it.
+ */
+ private readonly loopDelay = createHistogram();
+ private lastTick = performance.now();
+ private readonly loopSampler = setInterval(() => {
+ const now = performance.now();
+ this.loopDelay.record(Math.max(1, Math.round((now - this.lastTick) * 1e6)));
+ this.lastTick = now;
+ }, LOOP_DELAY_RESOLUTION_MS).unref();
constructor(
private readonly config: ConfigService,
@@ -78,12 +89,10 @@ export class MetricsService implements OnModuleDestroy {
// Registered only with QUEUE_ENABLED=true (see metrics.module.ts); absent otherwise.
@Optional() @InjectQueue(QUEUE_NAMES.WEBHOOK) private readonly webhookQueue?: Queue,
@Optional() @InjectQueue(QUEUE_NAMES.INGRESS) private readonly ingressQueue?: Queue,
- ) {
- this.loopDelay.enable();
- }
+ ) {}
onModuleDestroy(): void {
- this.loopDelay.disable();
+ clearInterval(this.loopSampler);
}
private get token(): string {