Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(metrics): bound the statistics read of a scrape
Yudhi Armyndharis1a2b65ba11d6
diff --git a/src/modules/metrics/metrics-render-budget.spec.ts b/src/modules/metrics/metrics-render-budget.spec.tsnew file mode 100644index 000000000..bacb39da6--- /dev/null+++ b/src/modules/metrics/metrics-render-budget.spec.ts@@ -0,0 +1,203 @@+import { MetricsService, METRICS_RENDER_TTL_MS } from './metrics.service';+import type { OverviewStats } from '../stats/stats.service';++const STATS_BUDGET_MS = 2000;+const overview: OverviewStats = {+ sessions: { active: 2, total: 3, byStatus: { ready: 2, failed: 1 } },+ messages: { sent: 4, received: 5, failed: 1, today: { sent: 1, received: 1 } },+};+const deferred = <T>() => {+ let resolve!: (value: T) => void;+ let reject!: (error: Error) => void;+ const promise = new Promise<T>((done, fail) => {+ resolve = done;+ reject = fail;+ });+ return { promise, resolve, reject };+};+const services: MetricsService[] = [];+function fixture(read: () => Promise<OverviewStats>) {+ // This stand-in has no cache or inflight map, matching a disabled stats memo.+ const stats = { getOverview: jest.fn(read) };+ const config = {+ get: (key: string) => (key === 'METRICS_TOKEN' ? 'fixture-token' : key === 'stats.cacheTtlMs' ? 0 : undefined),+ };+ const service = new MetricsService(config as never, stats as never);+ services.push(service);+ const scrape = () => {+ service.assertScrapeAuthorized('Bearer fixture-token', {+ headers: {},+ socket: { remoteAddress: '192.0.2.1' },+ });+ return service.render();+ };+ return { service, stats, scrape };+}+const observe = (pending: Promise<string>) => {+ const result: { text?: string; error?: unknown } = {};+ const done = pending.then(+ text => {+ result.text = text;+ },+ error => {+ result.error = error;+ },+ );+ return { result, done };+};+function degraded(result: { text?: string; error?: unknown }) {+ expect(result.error).toBeUndefined();+ expect(result.text).toBeDefined();+ expect(result.text).toContain('openwa_up 1');+ expect(result.text).toContain('openwa_stats_available 0');+ expect(result.text).toContain('openwa_process_resident_memory_bytes');+ expect(result.text).toContain('openwa_webhook_delivery_failures_total');+ expect(result.text).not.toContain('openwa_sessions_active');+ expect(result.text).not.toContain('openwa_messages_total');+}++describe('MetricsService bounded degraded renders', () => {+ beforeEach(() => jest.useFakeTimers({ now: 1000000 }));+ afterEach(() => {+ for (const service of services.splice(0)) service.onModuleDestroy();+ jest.clearAllTimers();+ jest.restoreAllMocks();+ jest.useRealTimers();+ });++ it('returns independent process series after a pending stats read spends its budget', async () => {+ const held = deferred<OverviewStats>();+ const f = fixture(() => held.promise);+ const call = observe(f.scrape());+ try {+ await jest.advanceTimersByTimeAsync(STATS_BUDGET_MS);+ degraded(call.result);+ expect(f.stats.getOverview).toHaveBeenCalledTimes(1);+ } finally {+ held.resolve(overview);+ await call.done;+ }+ });++ it('shares one pending overview across concurrent and repeated uncached scrapes', async () => {+ const held = deferred<OverviewStats>();+ const f = fixture(() => held.promise);+ const calls = [observe(f.scrape()), observe(f.scrape())];+ try {+ await jest.advanceTimersByTimeAsync(0);+ expect(f.stats.getOverview).toHaveBeenCalledTimes(1);+ await jest.advanceTimersByTimeAsync(STATS_BUDGET_MS);+ calls.forEach(call => degraded(call.result));+ const cached = observe(f.scrape());+ calls.push(cached);+ await jest.advanceTimersByTimeAsync(0);+ degraded(cached.result);+ await jest.advanceTimersByTimeAsync(METRICS_RENDER_TTL_MS + 1);+ const repeated = observe(f.scrape());+ calls.push(repeated);+ await jest.advanceTimersByTimeAsync(STATS_BUDGET_MS);+ degraded(repeated.result);+ expect(f.stats.getOverview).toHaveBeenCalledTimes(1);+ } finally {+ held.resolve(overview);+ await Promise.all(calls.map(call => call.done));+ }+ });++ it.each(['resolve', 'reject'] as const)('ignores a late %s after a newer degraded render', async outcome => {+ const held = deferred<OverviewStats>();+ const f = fixture(() => held.promise);+ const uptime = jest.spyOn(process, 'uptime').mockReturnValue(100);+ const unhandled: unknown[] = [];+ const listener = (error: unknown) => {+ unhandled.push(error);+ };+ process.on('unhandledRejection', listener);+ const calls = [observe(f.scrape())];+ try {+ await jest.advanceTimersByTimeAsync(STATS_BUDGET_MS);+ degraded(calls[0].result);+ await jest.advanceTimersByTimeAsync(METRICS_RENDER_TTL_MS + 1);+ uptime.mockReturnValue(200);+ const newer = observe(f.scrape());+ calls.push(newer);+ await jest.advanceTimersByTimeAsync(STATS_BUDGET_MS);+ degraded(newer.result);+ expect(newer.result.text).toContain('openwa_process_uptime_seconds 200');+ expect(f.stats.getOverview).toHaveBeenCalledTimes(1);+ if (outcome === 'resolve') held.resolve(overview);+ else held.reject(new Error('late stats failure'));+ await jest.advanceTimersByTimeAsync(1);+ expect(await f.scrape()).toBe(newer.result.text);+ expect(unhandled).toEqual([]);+ } finally {+ held.resolve(overview);+ await Promise.allSettled([held.promise, ...calls.map(call => call.done)]);+ process.removeListener('unhandledRejection', listener);+ }+ });++ it('retains healthy values and serves the completed render from cache', async () => {+ const f = fixture(() => Promise.resolve(overview));+ const first = await f.scrape();+ expect(first).toContain('openwa_stats_available 1');+ expect(first).toContain('openwa_sessions_active 2');+ expect(await f.scrape()).toBe(first);+ expect(f.stats.getOverview).toHaveBeenCalledTimes(1);+ });++ it('omits only database series when the overview immediately rejects', async () => {+ const f = fixture(() => Promise.reject(new Error('stats unavailable')));+ const call = observe(f.scrape());+ await jest.advanceTimersByTimeAsync(0);+ await call.done;+ degraded(call.result);+ expect(f.stats.getOverview).toHaveBeenCalledTimes(1);+ });++ it('clears wait timers and resumes database series after a timed-out read settles', async () => {+ const held = deferred<OverviewStats>();+ const f = fixture(() => Promise.resolve(overview));+ f.stats.getOverview.mockImplementationOnce(() => held.promise);+ const samplerTimers = jest.getTimerCount();+ const call = observe(f.scrape());+ try {+ await jest.advanceTimersByTimeAsync(STATS_BUDGET_MS);+ degraded(call.result);+ expect(jest.getTimerCount()).toBe(samplerTimers);+ held.resolve(overview);+ await jest.advanceTimersByTimeAsync(0);+ expect(await f.scrape()).toBe(call.result.text);+ await jest.advanceTimersByTimeAsync(METRICS_RENDER_TTL_MS + 1);+ const recovered = await f.scrape();+ expect(recovered).toContain('openwa_stats_available 1');+ expect(recovered).toContain('openwa_sessions_active 2');+ expect(f.stats.getOverview).toHaveBeenCalledTimes(2);+ expect(jest.getTimerCount()).toBe(samplerTimers);+ } finally {+ held.resolve(overview);+ await call.done;+ }+ });++ it('reuses a timed-out wait without starting another deadline while its read is pending', async () => {+ const held = deferred<OverviewStats>();+ const f = fixture(() => held.promise);+ const calls = [observe(f.scrape())];+ try {+ await jest.advanceTimersByTimeAsync(STATS_BUDGET_MS);+ degraded(calls[0].result);+ await jest.advanceTimersByTimeAsync(METRICS_RENDER_TTL_MS + 1);+ const samplerTimers = jest.getTimerCount();+ const repeated = observe(f.scrape());+ calls.push(repeated);+ await jest.advanceTimersByTimeAsync(0);+ degraded(repeated.result);+ expect(f.stats.getOverview).toHaveBeenCalledTimes(1);+ expect(jest.getTimerCount()).toBe(samplerTimers);+ } finally {+ held.resolve(overview);+ await Promise.all(calls.map(call => call.done));+ }+ });+});diff --git a/src/modules/metrics/metrics.service.ts b/src/modules/metrics/metrics.service.tsindex 6db07712c..e61338669 100644--- a/src/modules/metrics/metrics.service.ts+++ b/src/modules/metrics/metrics.service.ts@@ -16,7 +16,7 @@ import { producerReady } from '../queue/redis-connection';import { constantTimeEqual } from '../../common/security/constantTimeEqual';import { limiterKeyForIp, resolveClientIp, type RequestLike } from '../../common/utils/ip';import { SlidingWindowLimiter } from '../events/ws-rate-limit';-import { StatsService } from '../stats/stats.service';+import { StatsService, type OverviewStats } from '../stats/stats.service';import { getWebhookDeliveryFailuresTotal } from '../../common/metrics/webhook-delivery-metrics';import {getSessionReconnectAttemptsTotal,@@ -47,6 +47,9 @@ export const METRICS_RENDER_TTL_MS = 5000;/** Bound on each queue's job-count read, so a connected but unresponsive Redis cannot stall the scrape. */export const QUEUE_READ_TIMEOUT_MS = 2000;+/** Budget for waiting on database-derived statistics during a scrape. */+export const STATS_READ_TIMEOUT_MS = 2000;+/** The job states reported per queue. `completed` is left out: removeOnComplete trims it, so it means nothing. */const QUEUE_JOB_STATES = ['wait', 'active', 'delayed', 'failed'] as const;@@ -62,6 +65,7 @@ export class MetricsService implements OnModuleDestroy {private readonly logger = createLogger('MetricsService');private cachedRender: { at: number; text: string } | null = null;+ private pendingOverview: Promise<OverviewStats | null> | null = null;/*** The route skips the shared throttler so a scrape interval costs no request budget, which left@@ -138,16 +142,8 @@ export class MetricsService implements OnModuleDestroy {// too, and `up` conflated "process dead" with "database unreachable": the exact incident this// endpoint exists to describe. The series that need no database are emitted either way, and the// ones that do are OMITTED rather than reported as zero, because a zero would fire an alert- // claiming every session had dropped.- let overview: Awaited<ReturnType<StatsService['getOverview']>> | null = null;- try {- overview = await this.statsService.getOverview();- } catch (err) {- this.logger.warn(- `Metrics scrape could not read stats; database-derived series omitted: ${err instanceof Error ? err.message : String(err)}`,- );- }- const queueCounts = await this.readQueueCounts();+ // claiming every session had dropped. A read that merely hangs is bounded the same way.+ const [overview, queueCounts] = await Promise.all([this.readOverviewOrNull(), this.readQueueCounts()]);const mem = process.memoryUsage();const lines: string[] = [];@@ -260,6 +256,41 @@ export class MetricsService implements OnModuleDestroy {return text;}+ /**+ * The overview, or null when it failed or did not settle within STATS_READ_TIMEOUT_MS. The wait is shared+ * until the read itself settles, so scrapes arriving behind a stuck read neither queue up nor each start a+ * deadline of their own.+ */+ private readOverviewOrNull(): Promise<OverviewStats | null> {+ if (this.pendingOverview) return this.pendingOverview;+ const work = Promise.resolve().then(() => this.statsService.getOverview());+ let timer: ReturnType<typeof setTimeout> | undefined;+ const waiting = Promise.race([+ work,+ new Promise<never>((_, reject) => {+ timer = setTimeout(() => reject(new Error('statistics read timed out')), STATS_READ_TIMEOUT_MS);+ }),+ ]).then(+ overview => {+ clearTimeout(timer);+ return overview;+ },+ err => {+ clearTimeout(timer);+ this.logger.warn(+ `Metrics scrape could not read stats; database-derived series omitted: ${err instanceof Error ? err.message : String(err)}`,+ );+ return null;+ },+ );+ this.pendingOverview = waiting;+ const clear = (): void => {+ if (this.pendingOverview === waiting) this.pendingOverview = null;+ };+ void work.then(clear, clear);+ return waiting;+ }+/*** Job counts for each registered queue that answered within QUEUE_READ_TIMEOUT_MS. producerReady keeps a* scrape from leaving a read pending on a Redis that has never connected.