Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(metrics): keep the last statistics when a read runs late
Yudhi Armyndharisa94bbb750fcb
diff --git a/src/modules/metrics/metrics-render-budget.spec.ts b/src/modules/metrics/metrics-render-budget.spec.tsindex bacb39da6..7aa93b4fe 100644--- a/src/modules/metrics/metrics-render-budget.spec.ts+++ b/src/modules/metrics/metrics-render-budget.spec.ts@@ -1,5 +1,5 @@-import { MetricsService, METRICS_RENDER_TTL_MS } from './metrics.service';-import type { OverviewStats } from '../stats/stats.service';+import { MetricsService, METRICS_RENDER_TTL_MS, STATS_STALE_MAX_MS } from './metrics.service';+import { StatsService, type OverviewStats } from '../stats/stats.service';const STATS_BUDGET_MS = 2000;const overview: OverviewStats = {@@ -201,3 +201,80 @@ describe('MetricsService bounded degraded renders', () => {}});});++describe('MetricsService statistics slower than its budget, with the real stats memo', () => {+ // The memo lives 30 s from completion, so a scan of 3 s run by a scrape every 15 s or 60 s always meets it cold.+ const SCAN_MS = 3000;+ const later = <T>(value: T, delay = SCAN_MS) => new Promise<T>(resolve => setTimeout(() => resolve(value), delay));+ function slowDatabase() {+ const state = { fail: false };+ const query = (): Record<string, unknown> => {+ const chain: Record<string, unknown> = {};+ for (const step of ['select', 'addSelect', 'where', 'groupBy']) chain[step] = () => chain;+ chain.getRawMany = () => later([{ direction: 'outgoing', count: '4' }]);+ return chain;+ };+ const sessions = {+ find: () => (state.fail ? Promise.reject(new Error('pool exhausted')) : later([{ status: 'ready' }])),+ };+ const messages = { createQueryBuilder: query, count: () => later(1) };+ const stats = new StatsService(+ sessions as never,+ messages as never,+ {+ get: (key: string, fallback?: unknown) => (key === 'stats.cacheTtlMs' ? 30000 : fallback),+ } as never,+ );+ const service = new MetricsService(+ { get: (key: string) => (key === 'METRICS_TOKEN' ? 'fixture-token' : undefined) } as never,+ stats,+ );+ services.push(service);+ const scrape = () => {+ service.assertScrapeAuthorized('Bearer fixture-token', { headers: {}, socket: { remoteAddress: '192.0.2.1' } });+ return service.render();+ };+ return { state, scrape };+ }+ const scrapeAfter = async (scrape: () => Promise<string>, interval: number): Promise<string> => {+ const pending = scrape();+ await jest.advanceTimersByTimeAsync(interval);+ return pending;+ };++ beforeEach(() => jest.useFakeTimers({ now: 1000000 }));+ afterEach(() => {+ for (const service of services.splice(0)) service.onModuleDestroy();+ jest.clearAllTimers();+ jest.restoreAllMocks();+ jest.useRealTimers();+ });++ it.each([15_000, 60_000])('keeps the database series on every scrape after the first, at %i ms', async interval => {+ const db = slowDatabase();+ const texts: string[] = [];+ for (let i = 0; i < 6; i++) texts.push(await scrapeAfter(db.scrape, interval));++ expect(texts[0]).toContain('openwa_stats_available 0');+ for (const text of texts.slice(1)) {+ expect(text).toContain('openwa_stats_available 1');+ expect(text).toContain('openwa_sessions_active');+ expect(text).toContain('openwa_messages_total{direction="outgoing"} 4');+ }+ });++ it('drops the series once the last result is older than the stale bound', async () => {+ const db = slowDatabase();+ await scrapeAfter(db.scrape, SCAN_MS);+ expect(await scrapeAfter(db.scrape, STATS_STALE_MAX_MS)).toContain('openwa_stats_available 0');+ });++ it('does not stand in for a read that failed', async () => {+ const db = slowDatabase();+ await scrapeAfter(db.scrape, 60_000);+ db.state.fail = true;+ const failed = await scrapeAfter(db.scrape, 60_000);+ expect(failed).toContain('openwa_stats_available 0');+ expect(failed).not.toContain('openwa_sessions_active');+ });+});diff --git a/src/modules/metrics/metrics.service.ts b/src/modules/metrics/metrics.service.tsindex e61338669..9d898e1d5 100644--- a/src/modules/metrics/metrics.service.ts+++ b/src/modules/metrics/metrics.service.ts@@ -50,6 +50,13 @@ export const QUEUE_READ_TIMEOUT_MS = 2000;/** Budget for waiting on database-derived statistics during a scrape. */export const STATS_READ_TIMEOUT_MS = 2000;+/**+ * How long the last overview read stands in for one that is slower than the budget. A read that outlasts the+ * budget still completes, and the stats memo it fills expires before a scrape interval of a minute comes round+ * again, so without this every such scrape would meet a cold memo and drop the series.+ */+export const STATS_STALE_MAX_MS = 5 * 60_000;+/** The job states reported per queue. `completed` is left out: removeOnComplete trims it, so it means nothing. */const QUEUE_JOB_STATES = ['wait', 'active', 'delayed', 'failed'] as const;@@ -66,6 +73,7 @@ export class MetricsService implements OnModuleDestroy {private cachedRender: { at: number; text: string } | null = null;private pendingOverview: Promise<OverviewStats | null> | null = null;+ private lastOverview: { overview: OverviewStats; at: number } | null = null;/*** The route skips the shared throttler so a scrape interval costs no request budget, which left@@ -257,18 +265,20 @@ export class MetricsService implements OnModuleDestroy {}/**- * The overview, or null when it failed or did not settle within STATS_READ_TIMEOUT_MS. The wait is shared- * until the read itself settles, so scrapes arriving behind a stuck read neither queue up nor each start a- * deadline of their own.+ * The overview, or null when it failed or did not settle within STATS_READ_TIMEOUT_MS and no recent one is+ * known. The wait is shared until the read itself settles, so scrapes arriving behind a stuck read neither+ * queue up nor each start a deadline of their own. A read that only runs late is kept when it completes, and+ * stands in for the next scrape that finds it late again (STATS_STALE_MAX_MS); a failed read never does.*/private readOverviewOrNull(): Promise<OverviewStats | null> {if (this.pendingOverview) return this.pendingOverview;const work = Promise.resolve().then(() => this.statsService.getOverview());+ const late = new Error('statistics read timed out');let timer: ReturnType<typeof setTimeout> | undefined;const waiting = Promise.race([work,new Promise<never>((_, reject) => {- timer = setTimeout(() => reject(new Error('statistics read timed out')), STATS_READ_TIMEOUT_MS);+ timer = setTimeout(() => reject(late), STATS_READ_TIMEOUT_MS);}),]).then(overview => {@@ -277,20 +287,30 @@ export class MetricsService implements OnModuleDestroy {},err => {clearTimeout(timer);+ const recent = err === late ? this.recentOverview() : null;this.logger.warn(- `Metrics scrape could not read stats; database-derived series omitted: ${err instanceof Error ? err.message : String(err)}`,+ `Metrics scrape could not read stats; ${recent ? 'serving the last result' : 'database-derived series omitted'}: ` ++ (err instanceof Error ? err.message : String(err)),);- return null;+ return recent;},);this.pendingOverview = waiting;- const clear = (): void => {+ const settled = (): void => {if (this.pendingOverview === waiting) this.pendingOverview = null;};- void work.then(clear, clear);+ void work.then(overview => {+ this.lastOverview = { overview, at: Date.now() };+ settled();+ }, settled);return waiting;}+ private recentOverview(): OverviewStats | null {+ const last = this.lastOverview;+ return last && Date.now() - last.at < STATS_STALE_MAX_MS ? last.overview : null;+ }+/*** Job counts for each registered queue that answered within QUEUE_READ_TIMEOUT_MS. producerReady keeps a* scrape from leaving a read pending on a Redis that has never connected.