Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

feat(metrics): export event-loop delay, queue depth and rejections

Yudhi Armyndharisf1ab97254acf
diff --git a/docs/10-devops-infrastructure.md b/docs/10-devops-infrastructure.md
index 77309ead5..ca13938ee 100644
--- a/docs/10-devops-infrastructure.md
+++ b/docs/10-devops-infrastructure.md
@@ -943,6 +943,10 @@ export class MetricsService {
| `openwa_process_uptime_seconds` | gauge | — | Process uptime |
| `openwa_process_resident_memory_bytes` | gauge | — | RSS |
| `openwa_process_heap_used_bytes` | gauge | — | V8 heap used |
+| `openwa_event_loop_delay_p99_seconds` | gauge | none | p99 event-loop delay since the previous uncached scrape |
+| `openwa_event_loop_delay_max_seconds` | gauge | none | Maximum event-loop delay since the previous uncached scrape |
+| `openwa_unhandled_rejections_total` | counter | `kind` | Unhandled promise rejections since process start (`other` or `page_context_lost`) |
+| `openwa_queue_jobs` | gauge | `queue`, `state` | BullMQ jobs per queue in `wait`/`active`/`delayed`/`failed` (cluster-wide, from Redis) |
| `openwa_stats_available` | gauge | — | 1 when the last overview read of the database-derived series below succeeded, 0 if it failed |
| `openwa_sessions_total` | gauge | — | Configured sessions |
| `openwa_sessions_active` | gauge | — | READY (active) sessions |
@@ -965,7 +969,15 @@ Not every row appears on every scrape, and the difference matters when you write
database-derived series (`openwa_sessions*`, `openwa_messages*`) are **omitted entirely** when the
overview cannot be read — `openwa_stats_available` is what tells the two cases apart, so alert on it
rather than reading a missing series as zero. `openwa_send_pacing_refusals_total` appears only once
-the governor has refused something. For these, `absent()` is the correct alerting primitive.
+the governor has refused something. `openwa_queue_jobs` appears only with `QUEUE_ENABLED=true`, and a
+queue whose counts cannot be read within 2 s is left out of that scrape. For these, `absent()` is the
+correct alerting primitive.
+
+Two rows need care when aggregating. `openwa_queue_jobs` is read from the shared Redis, so every node
+reports the same cluster-wide value: aggregate it with `max`, not `sum`. The event-loop delay window
+runs from the previous uncached render (renders are cached for 5 s), so with several scrapers it is
+shared between them. `openwa_unhandled_rejections_total{kind="page_context_lost"}` counts the expected
+whatsapp-web.js navigation rejections; alert on `kind="other"`.
`src/common/docs-metrics-list.spec.ts` compares this table against the metric names declared in
`metrics.service.ts` and `request-metrics.ts`, and checks that every helper `render()` splices in is
diff --git a/src/common/metrics/process-error-metrics.ts b/src/common/metrics/process-error-metrics.ts
new file mode 100644
index 000000000..9c53ee3dc
--- /dev/null
+++ b/src/common/metrics/process-error-metrics.ts
@@ -0,0 +1,16 @@
+/**
+ * Process-lifetime count of promise rejections that reached the process-level backstop
+ * (`registerUnhandledRejectionHandler`), split by the handler's own classification so an alert can
+ * target `other` without firing on the expected whatsapp-web.js navigation rejections.
+ */
+export type UnhandledRejectionKind = 'page_context_lost' | 'other';
+
+const unhandledRejections: Record<UnhandledRejectionKind, number> = { page_context_lost: 0, other: 0 };
+
+export function incrementUnhandledRejections(kind: UnhandledRejectionKind): void {
+ unhandledRejections[kind] += 1;
+}
+
+export function getUnhandledRejections(): Readonly<Record<UnhandledRejectionKind, number>> {
+ return { ...unhandledRejections };
+}
diff --git a/src/config/process-error-monitor.spec.ts b/src/config/process-error-monitor.spec.ts
index 1440372ef..a9b39098d 100644
--- a/src/config/process-error-monitor.spec.ts
+++ b/src/config/process-error-monitor.spec.ts
@@ -1,4 +1,5 @@
import { registerUncaughtExceptionMonitor, registerUnhandledRejectionHandler } from './process-error-monitor';
+import { getUnhandledRejections } from '../common/metrics/process-error-metrics';
const EVENT = 'uncaughtExceptionMonitor';
@@ -122,6 +123,19 @@ describe('registerUnhandledRejectionHandler', () => {
expect(errors).toHaveLength(1);
});
+ it('counts each rejection under its classification', () => {
+ const { handler } = register();
+ const before = getUnhandledRejections();
+ handler(new Error('something genuinely broke'));
+ handler(new Error('Execution context was destroyed'));
+ handler('a bare string');
+
+ expect(getUnhandledRejections()).toEqual({
+ other: before.other + 2,
+ page_context_lost: before.page_context_lost + 1,
+ });
+ });
+
it('stringifies a non-Error rejection without throwing', () => {
const { handler, errors } = register();
expect(() => handler('a bare string')).not.toThrow();
diff --git a/src/config/process-error-monitor.ts b/src/config/process-error-monitor.ts
index 17634a547..d15d2a63b 100644
--- a/src/config/process-error-monitor.ts
+++ b/src/config/process-error-monitor.ts
@@ -1,3 +1,5 @@
+import { incrementUnhandledRejections } from '../common/metrics/process-error-metrics';
+
/** Minimal structured-logger surface the monitor needs (satisfied by createLogger()'s result). */
interface FatalLogger {
error: (message: string, detail?: string) => void;
@@ -81,7 +83,9 @@ export function registerUnhandledRejectionHandler(logger: RejectionLogger): void
process.on('unhandledRejection', (reason: unknown) => {
const message = reason instanceof Error ? reason.message : String(reason);
const detail = reason instanceof Error ? reason.stack : String(reason);
- if (PAGE_CONTEXT_LOST_REJECTION.test(message)) {
+ const pageContextLost = PAGE_CONTEXT_LOST_REJECTION.test(message);
+ incrementUnhandledRejections(pageContextLost ? 'page_context_lost' : 'other');
+ if (pageContextLost) {
// The stack goes in the context object, not the second positional slot: `warn`'s second
// parameter is the log context, and a string there becomes the line's scope name.
logger.warn(
diff --git a/src/modules/metrics/metrics.module.ts b/src/modules/metrics/metrics.module.ts
index 8fc7f6704..f6884540a 100644
--- a/src/modules/metrics/metrics.module.ts
+++ b/src/modules/metrics/metrics.module.ts
@@ -1,4 +1,4 @@
-import { MiddlewareConsumer, Module, NestModule } from '@nestjs/common';
+import { DynamicModule, MiddlewareConsumer, Module, NestModule, Type } from '@nestjs/common';
import { APP_INTERCEPTOR } from '@nestjs/core';
import { ConfigModule } from '@nestjs/config';
import { MetricsController } from './metrics.controller';
@@ -7,8 +7,17 @@ import { StatsModule } from '../stats/stats.module';
import { RequestMetricsInterceptor } from '../../common/interceptors/request-metrics.interceptor';
import { requestMetricsBoundaryMiddleware } from '../../common/middleware/request-metrics.middleware';
+// QueueModule registers the webhook and ingress queues MetricsService reports (@Optional). Imported
+// only when enabled, like infra.module.ts, so a disabled queue never dials Redis.
+const queueModules: Array<Type | DynamicModule> = [];
+if (process.env.QUEUE_ENABLED === 'true') {
+ // eslint-disable-next-line @typescript-eslint/no-require-imports
+ const queueModule = require('../queue/queue.module') as { QueueModule: Type };
+ queueModules.push(queueModule.QueueModule);
+}
+
@Module({
- imports: [ConfigModule, StatsModule],
+ imports: [ConfigModule, StatsModule, ...queueModules],
controllers: [MetricsController],
providers: [
MetricsService,
diff --git a/src/modules/metrics/metrics.service.spec.ts b/src/modules/metrics/metrics.service.spec.ts
index 0af6150d5..75b91f47d 100644
--- a/src/modules/metrics/metrics.service.spec.ts
+++ b/src/modules/metrics/metrics.service.spec.ts
@@ -1,6 +1,7 @@
import { HttpException, HttpStatus, NotFoundException, UnauthorizedException } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
-import { MetricsService, METRICS_RENDER_TTL_MS } from './metrics.service';
+import { Queue } from 'bullmq';
+import { MetricsService, METRICS_RENDER_TTL_MS, QUEUE_READ_TIMEOUT_MS } from './metrics.service';
import { StatsService, OverviewStats } from '../stats/stats.service';
import { getWebhookDeliveryFailuresTotal } from '../../common/metrics/webhook-delivery-metrics';
import {
@@ -8,6 +9,7 @@ import {
getSessionReconnectLoopAlertsTotal,
} from '../../common/metrics/session-reconnect-metrics';
import { setRestrictedSessionCount } from '../../common/metrics/session-restriction-metrics';
+import { getUnhandledRejections, incrementUnhandledRejections } from '../../common/metrics/process-error-metrics';
describe('MetricsService', () => {
const overview: OverviewStats = {
@@ -190,3 +192,110 @@ describe('MetricsService.render survives a failing stats query', () => {
expect(text).toContain('openwa_sessions_active 2');
});
});
+
+describe('MetricsService runtime series', () => {
+ const overview: OverviewStats = {
+ sessions: { active: 0, total: 0, byStatus: {} },
+ messages: { sent: 0, received: 0, failed: 0, today: { sent: 0, received: 0 } },
+ };
+ const config = { get: () => undefined } as unknown as ConfigService;
+ const stats = { getOverview: jest.fn().mockResolvedValue(overview) } as unknown as StatsService;
+ const queue = (getJobCounts: () => Promise<Record<string, number>>): Queue => ({ getJobCounts }) as unknown as Queue;
+ const services: MetricsService[] = [];
+ const make = (webhook?: Queue, ingress?: Queue): MetricsService => {
+ const svc = new MetricsService(config, stats, webhook, ingress);
+ services.push(svc);
+ return svc;
+ };
+
+ afterEach(() => {
+ services.splice(0).forEach(svc => svc.onModuleDestroy());
+ jest.useRealTimers();
+ });
+
+ const sample = (text: string, series: string): number => {
+ const line = text.split('\n').find(l => l.startsWith(`${series} `));
+ expect(line).toBeDefined();
+ return Number(line!.split(' ')[1]);
+ };
+
+ it('reports event-loop delay since the previous uncached render, then starts a new window', async () => {
+ const svc = make();
+ const histogram = (svc as unknown as { loopDelay: { reset: () => void } }).loopDelay;
+ const reset = jest.spyOn(histogram, 'reset');
+
+ const text = await svc.render();
+
+ expect(text).toContain('# TYPE openwa_event_loop_delay_p99_seconds gauge');
+ expect(text).toContain('# TYPE openwa_event_loop_delay_max_seconds gauge');
+ for (const series of ['openwa_event_loop_delay_p99_seconds', 'openwa_event_loop_delay_max_seconds']) {
+ const value = sample(text, series);
+ expect(Number.isFinite(value)).toBe(true);
+ expect(value).toBeGreaterThanOrEqual(0);
+ }
+ expect(reset).toHaveBeenCalledTimes(1);
+ });
+
+ it('reports event-loop delay net of the sampling interval, so an idle loop reads zero', async () => {
+ const svc = make();
+ const histogram = (svc as unknown as { loopDelay: { percentile: (p: number) => number; max: number } }).loopDelay;
+ // The histogram records the whole gap between its 20 ms timer callbacks: an idle loop reads about
+ // 20 ms, and a 200 ms synchronous block reads about 220 ms.
+ jest.spyOn(histogram, 'percentile').mockReturnValue(19.8e6);
+ jest.spyOn(histogram, 'max', 'get').mockReturnValue(220e6);
+
+ const text = await svc.render();
+
+ expect(sample(text, 'openwa_event_loop_delay_p99_seconds')).toBe(0);
+ expect(sample(text, 'openwa_event_loop_delay_max_seconds')).toBeCloseTo(0.2, 9);
+ });
+
+ it('counts unhandled rejections by kind', async () => {
+ const before = getUnhandledRejections();
+ incrementUnhandledRejections('other');
+ const text = await make().render();
+
+ expect(text).toContain('# TYPE openwa_unhandled_rejections_total counter');
+ expect(sample(text, 'openwa_unhandled_rejections_total{kind="other"}')).toBe(before.other + 1);
+ expect(sample(text, 'openwa_unhandled_rejections_total{kind="page_context_lost"}')).toBe(before.page_context_lost);
+ });
+
+ it('reports job counts per queue and state', async () => {
+ const text = await make(
+ queue(() => Promise.resolve({ wait: 4, active: 1, delayed: 2, failed: 3 })),
+ queue(() => Promise.resolve({ wait: 0, active: 0, delayed: 0, failed: 7 })),
+ ).render();
+
+ expect(text).toContain('# TYPE openwa_queue_jobs gauge');
+ expect(text).toContain('openwa_queue_jobs{queue="webhook-queue",state="wait"} 4');
+ expect(text).toContain('openwa_queue_jobs{queue="webhook-queue",state="failed"} 3');
+ expect(text).toContain('openwa_queue_jobs{queue="ingress-queue",state="failed"} 7');
+ });
+
+ it('omits a queue whose read fails or hangs instead of reporting zero', async () => {
+ jest.useFakeTimers();
+ const rendering = make(
+ queue(() => new Promise(() => undefined)),
+ queue(() => Promise.reject(new Error('Connection is closed.'))),
+ ).render();
+ await jest.advanceTimersByTimeAsync(QUEUE_READ_TIMEOUT_MS);
+ const text = await rendering;
+
+ expect(text).toContain('openwa_up 1');
+ expect(text).not.toContain('openwa_queue_jobs');
+ });
+
+ it('keeps the healthy queue when only the other one fails', async () => {
+ const text = await make(
+ queue(() => Promise.resolve({ wait: 1, active: 0, delayed: 0, failed: 0 })),
+ queue(() => Promise.reject(new Error('Connection is closed.'))),
+ ).render();
+
+ expect(text).toContain('openwa_queue_jobs{queue="webhook-queue",state="wait"} 1');
+ expect(text).not.toContain('queue="ingress-queue"');
+ });
+
+ it('emits no queue series when the queue is disabled', async () => {
+ expect(await make().render()).not.toContain('openwa_queue_jobs');
+ });
+});
diff --git a/src/modules/metrics/metrics.service.ts b/src/modules/metrics/metrics.service.ts
index 1f731fe7f..5280cdc7e 100644
--- a/src/modules/metrics/metrics.service.ts
+++ b/src/modules/metrics/metrics.service.ts
@@ -1,5 +1,17 @@
-import { HttpException, HttpStatus, Injectable, NotFoundException, UnauthorizedException } from '@nestjs/common';
+import { monitorEventLoopDelay } from 'node:perf_hooks';
+import {
+ HttpException,
+ HttpStatus,
+ Injectable,
+ NotFoundException,
+ OnModuleDestroy,
+ Optional,
+ UnauthorizedException,
+} from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
+import { InjectQueue } from '@nestjs/bullmq';
+import type { Queue } from 'bullmq';
+import { QUEUE_NAMES } from '../queue/queue-names';
import { constantTimeEqual } from '../../common/security/constantTimeEqual';
import { limiterKeyForIp, resolveClientIp, type RequestLike } from '../../common/utils/ip';
import { SlidingWindowLimiter } from '../events/ws-rate-limit';
@@ -12,6 +24,7 @@ import {
import { getRestrictedSessionCount } from '../../common/metrics/session-restriction-metrics';
import { getSendPacingRefusals } from '../../common/metrics/send-pacing-metrics';
import { renderHttpRequestMetrics } from '../../common/metrics/request-metrics';
+import { getUnhandledRejections } from '../../common/metrics/process-error-metrics';
import { createLogger } from '../../common/services/logger.service';
/**
@@ -30,8 +43,21 @@ import { createLogger } from '../../common/services/logger.service';
*/
export const METRICS_RENDER_TTL_MS = 5000;
+/** Bound on each queue's job-count read, so a connected but unresponsive Redis cannot stall the scrape. */
+export const QUEUE_READ_TIMEOUT_MS = 2000;
+
+/** The job states reported per queue. `completed` is left out: removeOnComplete trims it, so it means nothing. */
+const QUEUE_JOB_STATES = ['wait', 'active', 'delayed', 'failed'] as const;
+
+/**
+ * Sampling interval of the event-loop delay histogram. Each sample is the whole gap between two timer
+ * callbacks, this interval included, so it is subtracted before export: an idle loop reads zero.
+ */
+const LOOP_DELAY_RESOLUTION_MS = 20;
+const LOOP_DELAY_FLOOR_NS = LOOP_DELAY_RESOLUTION_MS * 1e6;
+
@Injectable()
-export class MetricsService {
+export class MetricsService implements OnModuleDestroy {
private readonly logger = createLogger('MetricsService');
private cachedRender: { at: number; text: string } | null = null;
@@ -43,10 +69,22 @@ export class MetricsService {
*/
private readonly failedScrapeLimiter = new SlidingWindowLimiter(10, 60_000);
+ /** Event-loop delay sampled between uncached renders (reset after each one). */
+ private readonly loopDelay = monitorEventLoopDelay({ resolution: LOOP_DELAY_RESOLUTION_MS });
+
constructor(
private readonly config: ConfigService,
private readonly statsService: StatsService,
- ) {}
+ // Registered only with QUEUE_ENABLED=true (see metrics.module.ts); absent otherwise.
+ @Optional() @InjectQueue(QUEUE_NAMES.WEBHOOK) private readonly webhookQueue?: Queue,
+ @Optional() @InjectQueue(QUEUE_NAMES.INGRESS) private readonly ingressQueue?: Queue,
+ ) {
+ this.loopDelay.enable();
+ }
+
+ onModuleDestroy(): void {
+ this.loopDelay.disable();
+ }
private get token(): string {
return (this.config.get<string>('METRICS_TOKEN') ?? '').trim();
@@ -104,6 +142,7 @@ export class MetricsService {
`Metrics scrape could not read stats; database-derived series omitted: ${err instanceof Error ? err.message : String(err)}`,
);
}
+ const queueCounts = await this.readQueueCounts();
const mem = process.memoryUsage();
const lines: string[] = [];
@@ -117,6 +156,37 @@ export class MetricsService {
gauge('openwa_process_uptime_seconds', 'Process uptime in seconds.', Math.round(process.uptime()));
gauge('openwa_process_resident_memory_bytes', 'Resident set size in bytes.', mem.rss);
gauge('openwa_process_heap_used_bytes', 'V8 heap used in bytes.', mem.heapUsed);
+ gauge(
+ 'openwa_event_loop_delay_p99_seconds',
+ 'p99 event-loop delay since the previous uncached scrape, in seconds.',
+ Math.max(0, this.loopDelay.percentile(99) - LOOP_DELAY_FLOOR_NS) / 1e9,
+ );
+ gauge(
+ 'openwa_event_loop_delay_max_seconds',
+ 'Maximum event-loop delay since the previous uncached scrape, in seconds.',
+ Math.max(0, this.loopDelay.max - LOOP_DELAY_FLOOR_NS) / 1e9,
+ );
+ this.loopDelay.reset();
+
+ const rejections = getUnhandledRejections();
+ lines.push(
+ '# HELP openwa_unhandled_rejections_total Promise rejections that reached the process-level handler since process start.',
+ );
+ lines.push('# TYPE openwa_unhandled_rejections_total counter');
+ lines.push(`openwa_unhandled_rejections_total{kind="other"} ${rejections.other}`);
+ lines.push(`openwa_unhandled_rejections_total{kind="page_context_lost"} ${rejections.page_context_lost}`);
+
+ // Read from the shared Redis, so every node reports the same cluster-wide value. A queue that is
+ // disabled or could not be read is omitted, not reported as empty.
+ if (queueCounts.length > 0) {
+ lines.push('# HELP openwa_queue_jobs Jobs in each BullMQ queue by state (cluster-wide).');
+ lines.push('# TYPE openwa_queue_jobs gauge');
+ for (const [name, counts] of queueCounts) {
+ for (const state of QUEUE_JOB_STATES) {
+ lines.push(`openwa_queue_jobs{queue="${name}",state="${state}"} ${counts[state] ?? 0}`);
+ }
+ }
+ }
gauge(
'openwa_stats_available',
@@ -185,6 +255,35 @@ export class MetricsService {
return text;
}
+ /** Job counts for each registered queue that answered within QUEUE_READ_TIMEOUT_MS. */
+ private async readQueueCounts(): Promise<Array<[string, Record<string, number>]>> {
+ const queues = [
+ [QUEUE_NAMES.WEBHOOK, this.webhookQueue],
+ [QUEUE_NAMES.INGRESS, this.ingressQueue],
+ ] as const;
+ const results: Array<[string, Record<string, number>]> = [];
+ for (const [name, queue] of queues) {
+ if (!queue) continue;
+ let timer: NodeJS.Timeout | undefined;
+ try {
+ const counts = await Promise.race([
+ queue.getJobCounts(...QUEUE_JOB_STATES),
+ new Promise<never>((_, reject) => {
+ timer = setTimeout(() => reject(new Error('timed out')), QUEUE_READ_TIMEOUT_MS);
+ }),
+ ]);
+ results.push([name, counts]);
+ } catch (err) {
+ this.logger.warn(
+ `Metrics scrape could not read ${name} job counts; series omitted: ${err instanceof Error ? err.message : String(err)}`,
+ );
+ } finally {
+ clearTimeout(timer);
+ }
+ }
+ return results;
+ }
+
private escapeLabel(value: string): string {
return value.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, '\\n');
}