Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(metrics): bound the statistics read of a scrape

Yudhi Armyndharis1a2b65ba11d6
diff --git a/src/modules/metrics/metrics-render-budget.spec.ts b/src/modules/metrics/metrics-render-budget.spec.ts
new file mode 100644
index 000000000..bacb39da6
--- /dev/null
+++ b/src/modules/metrics/metrics-render-budget.spec.ts
@@ -0,0 +1,203 @@
+import { MetricsService, METRICS_RENDER_TTL_MS } from './metrics.service';
+import type { OverviewStats } from '../stats/stats.service';
+
+const STATS_BUDGET_MS = 2000;
+const overview: OverviewStats = {
+ sessions: { active: 2, total: 3, byStatus: { ready: 2, failed: 1 } },
+ messages: { sent: 4, received: 5, failed: 1, today: { sent: 1, received: 1 } },
+};
+const deferred = <T>() => {
+ let resolve!: (value: T) => void;
+ let reject!: (error: Error) => void;
+ const promise = new Promise<T>((done, fail) => {
+ resolve = done;
+ reject = fail;
+ });
+ return { promise, resolve, reject };
+};
+const services: MetricsService[] = [];
+function fixture(read: () => Promise<OverviewStats>) {
+ // This stand-in has no cache or inflight map, matching a disabled stats memo.
+ const stats = { getOverview: jest.fn(read) };
+ const config = {
+ get: (key: string) => (key === 'METRICS_TOKEN' ? 'fixture-token' : key === 'stats.cacheTtlMs' ? 0 : undefined),
+ };
+ const service = new MetricsService(config as never, stats as never);
+ services.push(service);
+ const scrape = () => {
+ service.assertScrapeAuthorized('Bearer fixture-token', {
+ headers: {},
+ socket: { remoteAddress: '192.0.2.1' },
+ });
+ return service.render();
+ };
+ return { service, stats, scrape };
+}
+const observe = (pending: Promise<string>) => {
+ const result: { text?: string; error?: unknown } = {};
+ const done = pending.then(
+ text => {
+ result.text = text;
+ },
+ error => {
+ result.error = error;
+ },
+ );
+ return { result, done };
+};
+function degraded(result: { text?: string; error?: unknown }) {
+ expect(result.error).toBeUndefined();
+ expect(result.text).toBeDefined();
+ expect(result.text).toContain('openwa_up 1');
+ expect(result.text).toContain('openwa_stats_available 0');
+ expect(result.text).toContain('openwa_process_resident_memory_bytes');
+ expect(result.text).toContain('openwa_webhook_delivery_failures_total');
+ expect(result.text).not.toContain('openwa_sessions_active');
+ expect(result.text).not.toContain('openwa_messages_total');
+}
+
+describe('MetricsService bounded degraded renders', () => {
+ beforeEach(() => jest.useFakeTimers({ now: 1000000 }));
+ afterEach(() => {
+ for (const service of services.splice(0)) service.onModuleDestroy();
+ jest.clearAllTimers();
+ jest.restoreAllMocks();
+ jest.useRealTimers();
+ });
+
+ it('returns independent process series after a pending stats read spends its budget', async () => {
+ const held = deferred<OverviewStats>();
+ const f = fixture(() => held.promise);
+ const call = observe(f.scrape());
+ try {
+ await jest.advanceTimersByTimeAsync(STATS_BUDGET_MS);
+ degraded(call.result);
+ expect(f.stats.getOverview).toHaveBeenCalledTimes(1);
+ } finally {
+ held.resolve(overview);
+ await call.done;
+ }
+ });
+
+ it('shares one pending overview across concurrent and repeated uncached scrapes', async () => {
+ const held = deferred<OverviewStats>();
+ const f = fixture(() => held.promise);
+ const calls = [observe(f.scrape()), observe(f.scrape())];
+ try {
+ await jest.advanceTimersByTimeAsync(0);
+ expect(f.stats.getOverview).toHaveBeenCalledTimes(1);
+ await jest.advanceTimersByTimeAsync(STATS_BUDGET_MS);
+ calls.forEach(call => degraded(call.result));
+ const cached = observe(f.scrape());
+ calls.push(cached);
+ await jest.advanceTimersByTimeAsync(0);
+ degraded(cached.result);
+ await jest.advanceTimersByTimeAsync(METRICS_RENDER_TTL_MS + 1);
+ const repeated = observe(f.scrape());
+ calls.push(repeated);
+ await jest.advanceTimersByTimeAsync(STATS_BUDGET_MS);
+ degraded(repeated.result);
+ expect(f.stats.getOverview).toHaveBeenCalledTimes(1);
+ } finally {
+ held.resolve(overview);
+ await Promise.all(calls.map(call => call.done));
+ }
+ });
+
+ it.each(['resolve', 'reject'] as const)('ignores a late %s after a newer degraded render', async outcome => {
+ const held = deferred<OverviewStats>();
+ const f = fixture(() => held.promise);
+ const uptime = jest.spyOn(process, 'uptime').mockReturnValue(100);
+ const unhandled: unknown[] = [];
+ const listener = (error: unknown) => {
+ unhandled.push(error);
+ };
+ process.on('unhandledRejection', listener);
+ const calls = [observe(f.scrape())];
+ try {
+ await jest.advanceTimersByTimeAsync(STATS_BUDGET_MS);
+ degraded(calls[0].result);
+ await jest.advanceTimersByTimeAsync(METRICS_RENDER_TTL_MS + 1);
+ uptime.mockReturnValue(200);
+ const newer = observe(f.scrape());
+ calls.push(newer);
+ await jest.advanceTimersByTimeAsync(STATS_BUDGET_MS);
+ degraded(newer.result);
+ expect(newer.result.text).toContain('openwa_process_uptime_seconds 200');
+ expect(f.stats.getOverview).toHaveBeenCalledTimes(1);
+ if (outcome === 'resolve') held.resolve(overview);
+ else held.reject(new Error('late stats failure'));
+ await jest.advanceTimersByTimeAsync(1);
+ expect(await f.scrape()).toBe(newer.result.text);
+ expect(unhandled).toEqual([]);
+ } finally {
+ held.resolve(overview);
+ await Promise.allSettled([held.promise, ...calls.map(call => call.done)]);
+ process.removeListener('unhandledRejection', listener);
+ }
+ });
+
+ it('retains healthy values and serves the completed render from cache', async () => {
+ const f = fixture(() => Promise.resolve(overview));
+ const first = await f.scrape();
+ expect(first).toContain('openwa_stats_available 1');
+ expect(first).toContain('openwa_sessions_active 2');
+ expect(await f.scrape()).toBe(first);
+ expect(f.stats.getOverview).toHaveBeenCalledTimes(1);
+ });
+
+ it('omits only database series when the overview immediately rejects', async () => {
+ const f = fixture(() => Promise.reject(new Error('stats unavailable')));
+ const call = observe(f.scrape());
+ await jest.advanceTimersByTimeAsync(0);
+ await call.done;
+ degraded(call.result);
+ expect(f.stats.getOverview).toHaveBeenCalledTimes(1);
+ });
+
+ it('clears wait timers and resumes database series after a timed-out read settles', async () => {
+ const held = deferred<OverviewStats>();
+ const f = fixture(() => Promise.resolve(overview));
+ f.stats.getOverview.mockImplementationOnce(() => held.promise);
+ const samplerTimers = jest.getTimerCount();
+ const call = observe(f.scrape());
+ try {
+ await jest.advanceTimersByTimeAsync(STATS_BUDGET_MS);
+ degraded(call.result);
+ expect(jest.getTimerCount()).toBe(samplerTimers);
+ held.resolve(overview);
+ await jest.advanceTimersByTimeAsync(0);
+ expect(await f.scrape()).toBe(call.result.text);
+ await jest.advanceTimersByTimeAsync(METRICS_RENDER_TTL_MS + 1);
+ const recovered = await f.scrape();
+ expect(recovered).toContain('openwa_stats_available 1');
+ expect(recovered).toContain('openwa_sessions_active 2');
+ expect(f.stats.getOverview).toHaveBeenCalledTimes(2);
+ expect(jest.getTimerCount()).toBe(samplerTimers);
+ } finally {
+ held.resolve(overview);
+ await call.done;
+ }
+ });
+
+ it('reuses a timed-out wait without starting another deadline while its read is pending', async () => {
+ const held = deferred<OverviewStats>();
+ const f = fixture(() => held.promise);
+ const calls = [observe(f.scrape())];
+ try {
+ await jest.advanceTimersByTimeAsync(STATS_BUDGET_MS);
+ degraded(calls[0].result);
+ await jest.advanceTimersByTimeAsync(METRICS_RENDER_TTL_MS + 1);
+ const samplerTimers = jest.getTimerCount();
+ const repeated = observe(f.scrape());
+ calls.push(repeated);
+ await jest.advanceTimersByTimeAsync(0);
+ degraded(repeated.result);
+ expect(f.stats.getOverview).toHaveBeenCalledTimes(1);
+ expect(jest.getTimerCount()).toBe(samplerTimers);
+ } finally {
+ held.resolve(overview);
+ await Promise.all(calls.map(call => call.done));
+ }
+ });
+});
diff --git a/src/modules/metrics/metrics.service.ts b/src/modules/metrics/metrics.service.ts
index 6db07712c..e61338669 100644
--- a/src/modules/metrics/metrics.service.ts
+++ b/src/modules/metrics/metrics.service.ts
@@ -16,7 +16,7 @@ import { producerReady } from '../queue/redis-connection';
import { constantTimeEqual } from '../../common/security/constantTimeEqual';
import { limiterKeyForIp, resolveClientIp, type RequestLike } from '../../common/utils/ip';
import { SlidingWindowLimiter } from '../events/ws-rate-limit';
-import { StatsService } from '../stats/stats.service';
+import { StatsService, type OverviewStats } from '../stats/stats.service';
import { getWebhookDeliveryFailuresTotal } from '../../common/metrics/webhook-delivery-metrics';
import {
getSessionReconnectAttemptsTotal,
@@ -47,6 +47,9 @@ export const METRICS_RENDER_TTL_MS = 5000;
/** Bound on each queue's job-count read, so a connected but unresponsive Redis cannot stall the scrape. */
export const QUEUE_READ_TIMEOUT_MS = 2000;
+/** Budget for waiting on database-derived statistics during a scrape. */
+export const STATS_READ_TIMEOUT_MS = 2000;
+
/** The job states reported per queue. `completed` is left out: removeOnComplete trims it, so it means nothing. */
const QUEUE_JOB_STATES = ['wait', 'active', 'delayed', 'failed'] as const;
@@ -62,6 +65,7 @@ export class MetricsService implements OnModuleDestroy {
private readonly logger = createLogger('MetricsService');
private cachedRender: { at: number; text: string } | null = null;
+ private pendingOverview: Promise<OverviewStats | null> | null = null;
/**
* The route skips the shared throttler so a scrape interval costs no request budget, which left
@@ -138,16 +142,8 @@ export class MetricsService implements OnModuleDestroy {
// too, and `up` conflated "process dead" with "database unreachable": the exact incident this
// endpoint exists to describe. The series that need no database are emitted either way, and the
// ones that do are OMITTED rather than reported as zero, because a zero would fire an alert
- // claiming every session had dropped.
- let overview: Awaited<ReturnType<StatsService['getOverview']>> | null = null;
- try {
- overview = await this.statsService.getOverview();
- } catch (err) {
- this.logger.warn(
- `Metrics scrape could not read stats; database-derived series omitted: ${err instanceof Error ? err.message : String(err)}`,
- );
- }
- const queueCounts = await this.readQueueCounts();
+ // claiming every session had dropped. A read that merely hangs is bounded the same way.
+ const [overview, queueCounts] = await Promise.all([this.readOverviewOrNull(), this.readQueueCounts()]);
const mem = process.memoryUsage();
const lines: string[] = [];
@@ -260,6 +256,41 @@ export class MetricsService implements OnModuleDestroy {
return text;
}
+ /**
+ * The overview, or null when it failed or did not settle within STATS_READ_TIMEOUT_MS. The wait is shared
+ * until the read itself settles, so scrapes arriving behind a stuck read neither queue up nor each start a
+ * deadline of their own.
+ */
+ private readOverviewOrNull(): Promise<OverviewStats | null> {
+ if (this.pendingOverview) return this.pendingOverview;
+ const work = Promise.resolve().then(() => this.statsService.getOverview());
+ let timer: ReturnType<typeof setTimeout> | undefined;
+ const waiting = Promise.race([
+ work,
+ new Promise<never>((_, reject) => {
+ timer = setTimeout(() => reject(new Error('statistics read timed out')), STATS_READ_TIMEOUT_MS);
+ }),
+ ]).then(
+ overview => {
+ clearTimeout(timer);
+ return overview;
+ },
+ err => {
+ clearTimeout(timer);
+ this.logger.warn(
+ `Metrics scrape could not read stats; database-derived series omitted: ${err instanceof Error ? err.message : String(err)}`,
+ );
+ return null;
+ },
+ );
+ this.pendingOverview = waiting;
+ const clear = (): void => {
+ if (this.pendingOverview === waiting) this.pendingOverview = null;
+ };
+ void work.then(clear, clear);
+ return waiting;
+ }
+
/**
* Job counts for each registered queue that answered within QUEUE_READ_TIMEOUT_MS. producerReady keeps a
* scrape from leaving a read pending on a Redis that has never connected.