Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
feat(metrics): export event-loop delay, queue depth and rejections
Yudhi Armyndharisf1ab97254acf
diff --git a/docs/10-devops-infrastructure.md b/docs/10-devops-infrastructure.mdindex 77309ead5..ca13938ee 100644--- a/docs/10-devops-infrastructure.md+++ b/docs/10-devops-infrastructure.md@@ -943,6 +943,10 @@ export class MetricsService {| `openwa_process_uptime_seconds` | gauge | — | Process uptime || `openwa_process_resident_memory_bytes` | gauge | — | RSS || `openwa_process_heap_used_bytes` | gauge | — | V8 heap used |+| `openwa_event_loop_delay_p99_seconds` | gauge | none | p99 event-loop delay since the previous uncached scrape |+| `openwa_event_loop_delay_max_seconds` | gauge | none | Maximum event-loop delay since the previous uncached scrape |+| `openwa_unhandled_rejections_total` | counter | `kind` | Unhandled promise rejections since process start (`other` or `page_context_lost`) |+| `openwa_queue_jobs` | gauge | `queue`, `state` | BullMQ jobs per queue in `wait`/`active`/`delayed`/`failed` (cluster-wide, from Redis) || `openwa_stats_available` | gauge | — | 1 when the last overview read of the database-derived series below succeeded, 0 if it failed || `openwa_sessions_total` | gauge | — | Configured sessions || `openwa_sessions_active` | gauge | — | READY (active) sessions |@@ -965,7 +969,15 @@ Not every row appears on every scrape, and the difference matters when you writedatabase-derived series (`openwa_sessions*`, `openwa_messages*`) are **omitted entirely** when theoverview cannot be read — `openwa_stats_available` is what tells the two cases apart, so alert on itrather than reading a missing series as zero. `openwa_send_pacing_refusals_total` appears only once-the governor has refused something. For these, `absent()` is the correct alerting primitive.+the governor has refused something. `openwa_queue_jobs` appears only with `QUEUE_ENABLED=true`, and a+queue whose counts cannot be read within 2 s is left out of that scrape. For these, `absent()` is the+correct alerting primitive.++Two rows need care when aggregating. `openwa_queue_jobs` is read from the shared Redis, so every node+reports the same cluster-wide value: aggregate it with `max`, not `sum`. The event-loop delay window+runs from the previous uncached render (renders are cached for 5 s), so with several scrapers it is+shared between them. `openwa_unhandled_rejections_total{kind="page_context_lost"}` counts the expected+whatsapp-web.js navigation rejections; alert on `kind="other"`.`src/common/docs-metrics-list.spec.ts` compares this table against the metric names declared in`metrics.service.ts` and `request-metrics.ts`, and checks that every helper `render()` splices in isdiff --git a/src/common/metrics/process-error-metrics.ts b/src/common/metrics/process-error-metrics.tsnew file mode 100644index 000000000..9c53ee3dc--- /dev/null+++ b/src/common/metrics/process-error-metrics.ts@@ -0,0 +1,16 @@+/**+ * Process-lifetime count of promise rejections that reached the process-level backstop+ * (`registerUnhandledRejectionHandler`), split by the handler's own classification so an alert can+ * target `other` without firing on the expected whatsapp-web.js navigation rejections.+ */+export type UnhandledRejectionKind = 'page_context_lost' | 'other';++const unhandledRejections: Record<UnhandledRejectionKind, number> = { page_context_lost: 0, other: 0 };++export function incrementUnhandledRejections(kind: UnhandledRejectionKind): void {+ unhandledRejections[kind] += 1;+}++export function getUnhandledRejections(): Readonly<Record<UnhandledRejectionKind, number>> {+ return { ...unhandledRejections };+}diff --git a/src/config/process-error-monitor.spec.ts b/src/config/process-error-monitor.spec.tsindex 1440372ef..a9b39098d 100644--- a/src/config/process-error-monitor.spec.ts+++ b/src/config/process-error-monitor.spec.ts@@ -1,4 +1,5 @@import { registerUncaughtExceptionMonitor, registerUnhandledRejectionHandler } from './process-error-monitor';+import { getUnhandledRejections } from '../common/metrics/process-error-metrics';const EVENT = 'uncaughtExceptionMonitor';@@ -122,6 +123,19 @@ describe('registerUnhandledRejectionHandler', () => {expect(errors).toHaveLength(1);});+ it('counts each rejection under its classification', () => {+ const { handler } = register();+ const before = getUnhandledRejections();+ handler(new Error('something genuinely broke'));+ handler(new Error('Execution context was destroyed'));+ handler('a bare string');++ expect(getUnhandledRejections()).toEqual({+ other: before.other + 2,+ page_context_lost: before.page_context_lost + 1,+ });+ });+it('stringifies a non-Error rejection without throwing', () => {const { handler, errors } = register();expect(() => handler('a bare string')).not.toThrow();diff --git a/src/config/process-error-monitor.ts b/src/config/process-error-monitor.tsindex 17634a547..d15d2a63b 100644--- a/src/config/process-error-monitor.ts+++ b/src/config/process-error-monitor.ts@@ -1,3 +1,5 @@+import { incrementUnhandledRejections } from '../common/metrics/process-error-metrics';+/** Minimal structured-logger surface the monitor needs (satisfied by createLogger()'s result). */interface FatalLogger {error: (message: string, detail?: string) => void;@@ -81,7 +83,9 @@ export function registerUnhandledRejectionHandler(logger: RejectionLogger): voidprocess.on('unhandledRejection', (reason: unknown) => {const message = reason instanceof Error ? reason.message : String(reason);const detail = reason instanceof Error ? reason.stack : String(reason);- if (PAGE_CONTEXT_LOST_REJECTION.test(message)) {+ const pageContextLost = PAGE_CONTEXT_LOST_REJECTION.test(message);+ incrementUnhandledRejections(pageContextLost ? 'page_context_lost' : 'other');+ if (pageContextLost) {// The stack goes in the context object, not the second positional slot: `warn`'s second// parameter is the log context, and a string there becomes the line's scope name.logger.warn(diff --git a/src/modules/metrics/metrics.module.ts b/src/modules/metrics/metrics.module.tsindex 8fc7f6704..f6884540a 100644--- a/src/modules/metrics/metrics.module.ts+++ b/src/modules/metrics/metrics.module.ts@@ -1,4 +1,4 @@-import { MiddlewareConsumer, Module, NestModule } from '@nestjs/common';+import { DynamicModule, MiddlewareConsumer, Module, NestModule, Type } from '@nestjs/common';import { APP_INTERCEPTOR } from '@nestjs/core';import { ConfigModule } from '@nestjs/config';import { MetricsController } from './metrics.controller';@@ -7,8 +7,17 @@ import { StatsModule } from '../stats/stats.module';import { RequestMetricsInterceptor } from '../../common/interceptors/request-metrics.interceptor';import { requestMetricsBoundaryMiddleware } from '../../common/middleware/request-metrics.middleware';+// QueueModule registers the webhook and ingress queues MetricsService reports (@Optional). Imported+// only when enabled, like infra.module.ts, so a disabled queue never dials Redis.+const queueModules: Array<Type | DynamicModule> = [];+if (process.env.QUEUE_ENABLED === 'true') {+ // eslint-disable-next-line @typescript-eslint/no-require-imports+ const queueModule = require('../queue/queue.module') as { QueueModule: Type };+ queueModules.push(queueModule.QueueModule);+}+@Module({- imports: [ConfigModule, StatsModule],+ imports: [ConfigModule, StatsModule, ...queueModules],controllers: [MetricsController],providers: [MetricsService,diff --git a/src/modules/metrics/metrics.service.spec.ts b/src/modules/metrics/metrics.service.spec.tsindex 0af6150d5..75b91f47d 100644--- a/src/modules/metrics/metrics.service.spec.ts+++ b/src/modules/metrics/metrics.service.spec.ts@@ -1,6 +1,7 @@import { HttpException, HttpStatus, NotFoundException, UnauthorizedException } from '@nestjs/common';import { ConfigService } from '@nestjs/config';-import { MetricsService, METRICS_RENDER_TTL_MS } from './metrics.service';+import { Queue } from 'bullmq';+import { MetricsService, METRICS_RENDER_TTL_MS, QUEUE_READ_TIMEOUT_MS } from './metrics.service';import { StatsService, OverviewStats } from '../stats/stats.service';import { getWebhookDeliveryFailuresTotal } from '../../common/metrics/webhook-delivery-metrics';import {@@ -8,6 +9,7 @@ import {getSessionReconnectLoopAlertsTotal,} from '../../common/metrics/session-reconnect-metrics';import { setRestrictedSessionCount } from '../../common/metrics/session-restriction-metrics';+import { getUnhandledRejections, incrementUnhandledRejections } from '../../common/metrics/process-error-metrics';describe('MetricsService', () => {const overview: OverviewStats = {@@ -190,3 +192,110 @@ describe('MetricsService.render survives a failing stats query', () => {expect(text).toContain('openwa_sessions_active 2');});});++describe('MetricsService runtime series', () => {+ const overview: OverviewStats = {+ sessions: { active: 0, total: 0, byStatus: {} },+ messages: { sent: 0, received: 0, failed: 0, today: { sent: 0, received: 0 } },+ };+ const config = { get: () => undefined } as unknown as ConfigService;+ const stats = { getOverview: jest.fn().mockResolvedValue(overview) } as unknown as StatsService;+ const queue = (getJobCounts: () => Promise<Record<string, number>>): Queue => ({ getJobCounts }) as unknown as Queue;+ const services: MetricsService[] = [];+ const make = (webhook?: Queue, ingress?: Queue): MetricsService => {+ const svc = new MetricsService(config, stats, webhook, ingress);+ services.push(svc);+ return svc;+ };++ afterEach(() => {+ services.splice(0).forEach(svc => svc.onModuleDestroy());+ jest.useRealTimers();+ });++ const sample = (text: string, series: string): number => {+ const line = text.split('\n').find(l => l.startsWith(`${series} `));+ expect(line).toBeDefined();+ return Number(line!.split(' ')[1]);+ };++ it('reports event-loop delay since the previous uncached render, then starts a new window', async () => {+ const svc = make();+ const histogram = (svc as unknown as { loopDelay: { reset: () => void } }).loopDelay;+ const reset = jest.spyOn(histogram, 'reset');++ const text = await svc.render();++ expect(text).toContain('# TYPE openwa_event_loop_delay_p99_seconds gauge');+ expect(text).toContain('# TYPE openwa_event_loop_delay_max_seconds gauge');+ for (const series of ['openwa_event_loop_delay_p99_seconds', 'openwa_event_loop_delay_max_seconds']) {+ const value = sample(text, series);+ expect(Number.isFinite(value)).toBe(true);+ expect(value).toBeGreaterThanOrEqual(0);+ }+ expect(reset).toHaveBeenCalledTimes(1);+ });++ it('reports event-loop delay net of the sampling interval, so an idle loop reads zero', async () => {+ const svc = make();+ const histogram = (svc as unknown as { loopDelay: { percentile: (p: number) => number; max: number } }).loopDelay;+ // The histogram records the whole gap between its 20 ms timer callbacks: an idle loop reads about+ // 20 ms, and a 200 ms synchronous block reads about 220 ms.+ jest.spyOn(histogram, 'percentile').mockReturnValue(19.8e6);+ jest.spyOn(histogram, 'max', 'get').mockReturnValue(220e6);++ const text = await svc.render();++ expect(sample(text, 'openwa_event_loop_delay_p99_seconds')).toBe(0);+ expect(sample(text, 'openwa_event_loop_delay_max_seconds')).toBeCloseTo(0.2, 9);+ });++ it('counts unhandled rejections by kind', async () => {+ const before = getUnhandledRejections();+ incrementUnhandledRejections('other');+ const text = await make().render();++ expect(text).toContain('# TYPE openwa_unhandled_rejections_total counter');+ expect(sample(text, 'openwa_unhandled_rejections_total{kind="other"}')).toBe(before.other + 1);+ expect(sample(text, 'openwa_unhandled_rejections_total{kind="page_context_lost"}')).toBe(before.page_context_lost);+ });++ it('reports job counts per queue and state', async () => {+ const text = await make(+ queue(() => Promise.resolve({ wait: 4, active: 1, delayed: 2, failed: 3 })),+ queue(() => Promise.resolve({ wait: 0, active: 0, delayed: 0, failed: 7 })),+ ).render();++ expect(text).toContain('# TYPE openwa_queue_jobs gauge');+ expect(text).toContain('openwa_queue_jobs{queue="webhook-queue",state="wait"} 4');+ expect(text).toContain('openwa_queue_jobs{queue="webhook-queue",state="failed"} 3');+ expect(text).toContain('openwa_queue_jobs{queue="ingress-queue",state="failed"} 7');+ });++ it('omits a queue whose read fails or hangs instead of reporting zero', async () => {+ jest.useFakeTimers();+ const rendering = make(+ queue(() => new Promise(() => undefined)),+ queue(() => Promise.reject(new Error('Connection is closed.'))),+ ).render();+ await jest.advanceTimersByTimeAsync(QUEUE_READ_TIMEOUT_MS);+ const text = await rendering;++ expect(text).toContain('openwa_up 1');+ expect(text).not.toContain('openwa_queue_jobs');+ });++ it('keeps the healthy queue when only the other one fails', async () => {+ const text = await make(+ queue(() => Promise.resolve({ wait: 1, active: 0, delayed: 0, failed: 0 })),+ queue(() => Promise.reject(new Error('Connection is closed.'))),+ ).render();++ expect(text).toContain('openwa_queue_jobs{queue="webhook-queue",state="wait"} 1');+ expect(text).not.toContain('queue="ingress-queue"');+ });++ it('emits no queue series when the queue is disabled', async () => {+ expect(await make().render()).not.toContain('openwa_queue_jobs');+ });+});diff --git a/src/modules/metrics/metrics.service.ts b/src/modules/metrics/metrics.service.tsindex 1f731fe7f..5280cdc7e 100644--- a/src/modules/metrics/metrics.service.ts+++ b/src/modules/metrics/metrics.service.ts@@ -1,5 +1,17 @@-import { HttpException, HttpStatus, Injectable, NotFoundException, UnauthorizedException } from '@nestjs/common';+import { monitorEventLoopDelay } from 'node:perf_hooks';+import {+ HttpException,+ HttpStatus,+ Injectable,+ NotFoundException,+ OnModuleDestroy,+ Optional,+ UnauthorizedException,+} from '@nestjs/common';import { ConfigService } from '@nestjs/config';+import { InjectQueue } from '@nestjs/bullmq';+import type { Queue } from 'bullmq';+import { QUEUE_NAMES } from '../queue/queue-names';import { constantTimeEqual } from '../../common/security/constantTimeEqual';import { limiterKeyForIp, resolveClientIp, type RequestLike } from '../../common/utils/ip';import { SlidingWindowLimiter } from '../events/ws-rate-limit';@@ -12,6 +24,7 @@ import {import { getRestrictedSessionCount } from '../../common/metrics/session-restriction-metrics';import { getSendPacingRefusals } from '../../common/metrics/send-pacing-metrics';import { renderHttpRequestMetrics } from '../../common/metrics/request-metrics';+import { getUnhandledRejections } from '../../common/metrics/process-error-metrics';import { createLogger } from '../../common/services/logger.service';/**@@ -30,8 +43,21 @@ import { createLogger } from '../../common/services/logger.service';*/export const METRICS_RENDER_TTL_MS = 5000;+/** Bound on each queue's job-count read, so a connected but unresponsive Redis cannot stall the scrape. */+export const QUEUE_READ_TIMEOUT_MS = 2000;++/** The job states reported per queue. `completed` is left out: removeOnComplete trims it, so it means nothing. */+const QUEUE_JOB_STATES = ['wait', 'active', 'delayed', 'failed'] as const;++/**+ * Sampling interval of the event-loop delay histogram. Each sample is the whole gap between two timer+ * callbacks, this interval included, so it is subtracted before export: an idle loop reads zero.+ */+const LOOP_DELAY_RESOLUTION_MS = 20;+const LOOP_DELAY_FLOOR_NS = LOOP_DELAY_RESOLUTION_MS * 1e6;+@Injectable()-export class MetricsService {+export class MetricsService implements OnModuleDestroy {private readonly logger = createLogger('MetricsService');private cachedRender: { at: number; text: string } | null = null;@@ -43,10 +69,22 @@ export class MetricsService {*/private readonly failedScrapeLimiter = new SlidingWindowLimiter(10, 60_000);+ /** Event-loop delay sampled between uncached renders (reset after each one). */+ private readonly loopDelay = monitorEventLoopDelay({ resolution: LOOP_DELAY_RESOLUTION_MS });+constructor(private readonly config: ConfigService,private readonly statsService: StatsService,- ) {}+ // Registered only with QUEUE_ENABLED=true (see metrics.module.ts); absent otherwise.+ @Optional() @InjectQueue(QUEUE_NAMES.WEBHOOK) private readonly webhookQueue?: Queue,+ @Optional() @InjectQueue(QUEUE_NAMES.INGRESS) private readonly ingressQueue?: Queue,+ ) {+ this.loopDelay.enable();+ }++ onModuleDestroy(): void {+ this.loopDelay.disable();+ }private get token(): string {return (this.config.get<string>('METRICS_TOKEN') ?? '').trim();@@ -104,6 +142,7 @@ export class MetricsService {`Metrics scrape could not read stats; database-derived series omitted: ${err instanceof Error ? err.message : String(err)}`,);}+ const queueCounts = await this.readQueueCounts();const mem = process.memoryUsage();const lines: string[] = [];@@ -117,6 +156,37 @@ export class MetricsService {gauge('openwa_process_uptime_seconds', 'Process uptime in seconds.', Math.round(process.uptime()));gauge('openwa_process_resident_memory_bytes', 'Resident set size in bytes.', mem.rss);gauge('openwa_process_heap_used_bytes', 'V8 heap used in bytes.', mem.heapUsed);+ gauge(+ 'openwa_event_loop_delay_p99_seconds',+ 'p99 event-loop delay since the previous uncached scrape, in seconds.',+ Math.max(0, this.loopDelay.percentile(99) - LOOP_DELAY_FLOOR_NS) / 1e9,+ );+ gauge(+ 'openwa_event_loop_delay_max_seconds',+ 'Maximum event-loop delay since the previous uncached scrape, in seconds.',+ Math.max(0, this.loopDelay.max - LOOP_DELAY_FLOOR_NS) / 1e9,+ );+ this.loopDelay.reset();++ const rejections = getUnhandledRejections();+ lines.push(+ '# HELP openwa_unhandled_rejections_total Promise rejections that reached the process-level handler since process start.',+ );+ lines.push('# TYPE openwa_unhandled_rejections_total counter');+ lines.push(`openwa_unhandled_rejections_total{kind="other"} ${rejections.other}`);+ lines.push(`openwa_unhandled_rejections_total{kind="page_context_lost"} ${rejections.page_context_lost}`);++ // Read from the shared Redis, so every node reports the same cluster-wide value. A queue that is+ // disabled or could not be read is omitted, not reported as empty.+ if (queueCounts.length > 0) {+ lines.push('# HELP openwa_queue_jobs Jobs in each BullMQ queue by state (cluster-wide).');+ lines.push('# TYPE openwa_queue_jobs gauge');+ for (const [name, counts] of queueCounts) {+ for (const state of QUEUE_JOB_STATES) {+ lines.push(`openwa_queue_jobs{queue="${name}",state="${state}"} ${counts[state] ?? 0}`);+ }+ }+ }gauge('openwa_stats_available',@@ -185,6 +255,35 @@ export class MetricsService {return text;}+ /** Job counts for each registered queue that answered within QUEUE_READ_TIMEOUT_MS. */+ private async readQueueCounts(): Promise<Array<[string, Record<string, number>]>> {+ const queues = [+ [QUEUE_NAMES.WEBHOOK, this.webhookQueue],+ [QUEUE_NAMES.INGRESS, this.ingressQueue],+ ] as const;+ const results: Array<[string, Record<string, number>]> = [];+ for (const [name, queue] of queues) {+ if (!queue) continue;+ let timer: NodeJS.Timeout | undefined;+ try {+ const counts = await Promise.race([+ queue.getJobCounts(...QUEUE_JOB_STATES),+ new Promise<never>((_, reject) => {+ timer = setTimeout(() => reject(new Error('timed out')), QUEUE_READ_TIMEOUT_MS);+ }),+ ]);+ results.push([name, counts]);+ } catch (err) {+ this.logger.warn(+ `Metrics scrape could not read ${name} job counts; series omitted: ${err instanceof Error ? err.message : String(err)}`,+ );+ } finally {+ clearTimeout(timer);+ }+ }+ return results;+ }+private escapeLabel(value: string): string {return value.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, '\\n');}