Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(metrics): keep the last statistics when a read runs late

Yudhi Armyndharisa94bbb750fcb
diff --git a/src/modules/metrics/metrics-render-budget.spec.ts b/src/modules/metrics/metrics-render-budget.spec.ts
index bacb39da6..7aa93b4fe 100644
--- a/src/modules/metrics/metrics-render-budget.spec.ts
+++ b/src/modules/metrics/metrics-render-budget.spec.ts
@@ -1,5 +1,5 @@
-import { MetricsService, METRICS_RENDER_TTL_MS } from './metrics.service';
-import type { OverviewStats } from '../stats/stats.service';
+import { MetricsService, METRICS_RENDER_TTL_MS, STATS_STALE_MAX_MS } from './metrics.service';
+import { StatsService, type OverviewStats } from '../stats/stats.service';
const STATS_BUDGET_MS = 2000;
const overview: OverviewStats = {
@@ -201,3 +201,80 @@ describe('MetricsService bounded degraded renders', () => {
}
});
});
+
+describe('MetricsService statistics slower than its budget, with the real stats memo', () => {
+ // The memo lives 30 s from completion, so a scan of 3 s run by a scrape every 15 s or 60 s always meets it cold.
+ const SCAN_MS = 3000;
+ const later = <T>(value: T, delay = SCAN_MS) => new Promise<T>(resolve => setTimeout(() => resolve(value), delay));
+ function slowDatabase() {
+ const state = { fail: false };
+ const query = (): Record<string, unknown> => {
+ const chain: Record<string, unknown> = {};
+ for (const step of ['select', 'addSelect', 'where', 'groupBy']) chain[step] = () => chain;
+ chain.getRawMany = () => later([{ direction: 'outgoing', count: '4' }]);
+ return chain;
+ };
+ const sessions = {
+ find: () => (state.fail ? Promise.reject(new Error('pool exhausted')) : later([{ status: 'ready' }])),
+ };
+ const messages = { createQueryBuilder: query, count: () => later(1) };
+ const stats = new StatsService(
+ sessions as never,
+ messages as never,
+ {
+ get: (key: string, fallback?: unknown) => (key === 'stats.cacheTtlMs' ? 30000 : fallback),
+ } as never,
+ );
+ const service = new MetricsService(
+ { get: (key: string) => (key === 'METRICS_TOKEN' ? 'fixture-token' : undefined) } as never,
+ stats,
+ );
+ services.push(service);
+ const scrape = () => {
+ service.assertScrapeAuthorized('Bearer fixture-token', { headers: {}, socket: { remoteAddress: '192.0.2.1' } });
+ return service.render();
+ };
+ return { state, scrape };
+ }
+ const scrapeAfter = async (scrape: () => Promise<string>, interval: number): Promise<string> => {
+ const pending = scrape();
+ await jest.advanceTimersByTimeAsync(interval);
+ return pending;
+ };
+
+ beforeEach(() => jest.useFakeTimers({ now: 1000000 }));
+ afterEach(() => {
+ for (const service of services.splice(0)) service.onModuleDestroy();
+ jest.clearAllTimers();
+ jest.restoreAllMocks();
+ jest.useRealTimers();
+ });
+
+ it.each([15_000, 60_000])('keeps the database series on every scrape after the first, at %i ms', async interval => {
+ const db = slowDatabase();
+ const texts: string[] = [];
+ for (let i = 0; i < 6; i++) texts.push(await scrapeAfter(db.scrape, interval));
+
+ expect(texts[0]).toContain('openwa_stats_available 0');
+ for (const text of texts.slice(1)) {
+ expect(text).toContain('openwa_stats_available 1');
+ expect(text).toContain('openwa_sessions_active');
+ expect(text).toContain('openwa_messages_total{direction="outgoing"} 4');
+ }
+ });
+
+ it('drops the series once the last result is older than the stale bound', async () => {
+ const db = slowDatabase();
+ await scrapeAfter(db.scrape, SCAN_MS);
+ expect(await scrapeAfter(db.scrape, STATS_STALE_MAX_MS)).toContain('openwa_stats_available 0');
+ });
+
+ it('does not stand in for a read that failed', async () => {
+ const db = slowDatabase();
+ await scrapeAfter(db.scrape, 60_000);
+ db.state.fail = true;
+ const failed = await scrapeAfter(db.scrape, 60_000);
+ expect(failed).toContain('openwa_stats_available 0');
+ expect(failed).not.toContain('openwa_sessions_active');
+ });
+});
diff --git a/src/modules/metrics/metrics.service.ts b/src/modules/metrics/metrics.service.ts
index e61338669..9d898e1d5 100644
--- a/src/modules/metrics/metrics.service.ts
+++ b/src/modules/metrics/metrics.service.ts
@@ -50,6 +50,13 @@ export const QUEUE_READ_TIMEOUT_MS = 2000;
/** Budget for waiting on database-derived statistics during a scrape. */
export const STATS_READ_TIMEOUT_MS = 2000;
+/**
+ * How long the last overview read stands in for one that is slower than the budget. A read that outlasts the
+ * budget still completes, and the stats memo it fills expires before a scrape interval of a minute comes round
+ * again, so without this every such scrape would meet a cold memo and drop the series.
+ */
+export const STATS_STALE_MAX_MS = 5 * 60_000;
+
/** The job states reported per queue. `completed` is left out: removeOnComplete trims it, so it means nothing. */
const QUEUE_JOB_STATES = ['wait', 'active', 'delayed', 'failed'] as const;
@@ -66,6 +73,7 @@ export class MetricsService implements OnModuleDestroy {
private cachedRender: { at: number; text: string } | null = null;
private pendingOverview: Promise<OverviewStats | null> | null = null;
+ private lastOverview: { overview: OverviewStats; at: number } | null = null;
/**
* The route skips the shared throttler so a scrape interval costs no request budget, which left
@@ -257,18 +265,20 @@ export class MetricsService implements OnModuleDestroy {
}
/**
- * The overview, or null when it failed or did not settle within STATS_READ_TIMEOUT_MS. The wait is shared
- * until the read itself settles, so scrapes arriving behind a stuck read neither queue up nor each start a
- * deadline of their own.
+ * The overview, or null when it failed or did not settle within STATS_READ_TIMEOUT_MS and no recent one is
+ * known. The wait is shared until the read itself settles, so scrapes arriving behind a stuck read neither
+ * queue up nor each start a deadline of their own. A read that only runs late is kept when it completes, and
+ * stands in for the next scrape that finds it late again (STATS_STALE_MAX_MS); a failed read never does.
*/
private readOverviewOrNull(): Promise<OverviewStats | null> {
if (this.pendingOverview) return this.pendingOverview;
const work = Promise.resolve().then(() => this.statsService.getOverview());
+ const late = new Error('statistics read timed out');
let timer: ReturnType<typeof setTimeout> | undefined;
const waiting = Promise.race([
work,
new Promise<never>((_, reject) => {
- timer = setTimeout(() => reject(new Error('statistics read timed out')), STATS_READ_TIMEOUT_MS);
+ timer = setTimeout(() => reject(late), STATS_READ_TIMEOUT_MS);
}),
]).then(
overview => {
@@ -277,20 +287,30 @@ export class MetricsService implements OnModuleDestroy {
},
err => {
clearTimeout(timer);
+ const recent = err === late ? this.recentOverview() : null;
this.logger.warn(
- `Metrics scrape could not read stats; database-derived series omitted: ${err instanceof Error ? err.message : String(err)}`,
+ `Metrics scrape could not read stats; ${recent ? 'serving the last result' : 'database-derived series omitted'}: ` +
+ (err instanceof Error ? err.message : String(err)),
);
- return null;
+ return recent;
},
);
this.pendingOverview = waiting;
- const clear = (): void => {
+ const settled = (): void => {
if (this.pendingOverview === waiting) this.pendingOverview = null;
};
- void work.then(clear, clear);
+ void work.then(overview => {
+ this.lastOverview = { overview, at: Date.now() };
+ settled();
+ }, settled);
return waiting;
}
+ private recentOverview(): OverviewStats | null {
+ const last = this.lastOverview;
+ return last && Date.now() - last.at < STATS_STALE_MAX_MS ? last.overview : null;
+ }
+
/**
* Job counts for each registered queue that answered within QUEUE_READ_TIMEOUT_MS. producerReady keeps a
* scrape from leaving a read pending on a Redis that has never connected.