Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
import { createHistogram, performance } from 'node:perf_hooks';
import {
HttpException,
HttpStatus,
Injectable,
NotFoundException,
OnModuleDestroy,
Optional,
UnauthorizedException,
} from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { InjectQueue } from '@nestjs/bullmq';
import type { Queue } from 'bullmq';
import { QUEUE_NAMES } from '../queue/queue-names';
import { producerReady } from '../queue/redis-connection';
import { constantTimeEqual } from '../../common/security/constantTimeEqual';
import { limiterKeyForIp, resolveClientIp, type RequestLike } from '../../common/utils/ip';
import { SlidingWindowLimiter } from '../events/ws-rate-limit';
import { StatsService, type OverviewStats } from '../stats/stats.service';
import { getWebhookDeliveryFailuresTotal } from '../../common/metrics/webhook-delivery-metrics';
import {
getSessionReconnectAttemptsTotal,
getSessionReconnectLoopAlertsTotal,
} from '../../common/metrics/session-reconnect-metrics';
import { getRestrictedSessionCount } from '../../common/metrics/session-restriction-metrics';
import { getSendPacingRefusals } from '../../common/metrics/send-pacing-metrics';
import { renderHttpRequestMetrics } from '../../common/metrics/request-metrics';
import { getUnhandledRejections } from '../../common/metrics/process-error-metrics';
import { createLogger } from '../../common/services/logger.service';
/**
* Prometheus exposition for OpenWA. Kept dependency-free (no prom-client) — the
* surface is small and the text format (v0.0.4) is trivial to emit by hand.
*
* Scraping is gated by METRICS_TOKEN: when it is unset the endpoint is disabled entirely
* (404, so a scanner cannot even confirm it exists); when set, a matching `Bearer` token
* is required. This keeps operational internals (session counts, failure totals) from
* being exposed publicly by default on a self-hosted box.
*/
/**
* How long a rendered scrape is reused before recomputing. getOverview() runs a full session scan plus
* several aggregate queries, so back-to-back scrapes (or several Prometheus replicas) would otherwise
* each pay the full DB cost. Stale-by-a-few-seconds metrics are fine for Prometheus.
*/
export const METRICS_RENDER_TTL_MS = 5000;
/** Bound on each queue's job-count read, so a connected but unresponsive Redis cannot stall the scrape. */
export const QUEUE_READ_TIMEOUT_MS = 2000;
/** Budget for waiting on database-derived statistics during a scrape. */
export const STATS_READ_TIMEOUT_MS = 2000;
/**
* How long the last overview read stands in for one that is slower than the budget, counted from when that read
* started. A read that outlasts the budget still completes, and the stats memo it fills expires before a scrape
* interval of a minute comes round again, so without this every such scrape would meet a cold memo and drop the
* series. A read still pending after this long is given up on and a new one starts.
*/
export const STATS_STALE_MAX_MS = 5 * 60_000;
/** The job states reported per queue. `completed` is left out: removeOnComplete trims it, so it means nothing. */
const QUEUE_JOB_STATES = ['wait', 'active', 'delayed', 'failed'] as const;
/**
* Sampling interval of the event-loop delay histogram. Each sample is the whole gap between two timer
* callbacks, this interval included, so it is subtracted before export: an idle loop reads zero.
*/
const LOOP_DELAY_RESOLUTION_MS = 20;
const LOOP_DELAY_FLOOR_NS = LOOP_DELAY_RESOLUTION_MS * 1e6;
@Injectable()
export class MetricsService implements OnModuleDestroy {
private readonly logger = createLogger('MetricsService');
private cachedRender: { at: number; text: string } | null = null;
private pendingOverview: Promise<OverviewStats | null> | null = null;
private pendingSince = 0;
/** What the newest read to finish kept: its result and when it started, or nothing after a failure. */
private lastOverview: { overview: OverviewStats; at: number } | null = null;
private readSeq = 0;
private keptSeq = 0;
/**
* The route skips the shared throttler so a scrape interval costs no request budget, which left
* token guesses unbounded. This window is charged per client before the compare and refunded when
* it succeeds, so only failures spend it: 10 a minute, then 429 until the window slides.
*/
private readonly failedScrapeLimiter = new SlidingWindowLimiter(10, 60_000);
/**
* Event-loop delay sampled between uncached renders (reset after each one). Sampled by a timer of
* our own rather than monitorEventLoopDelay: resetting that histogram also drops its previous-tick
* timestamp, so the first gap after every render, and any stall that began in it, went unrecorded.
* Here reset() clears only the counts; `lastTick` carries across it.
*/
private readonly loopDelay = createHistogram();
private lastTick = performance.now();
private readonly loopSampler = setInterval(() => {
const now = performance.now();
this.loopDelay.record(Math.max(1, Math.round((now - this.lastTick) * 1e6)));
this.lastTick = now;
}, LOOP_DELAY_RESOLUTION_MS).unref();
constructor(
private readonly config: ConfigService,
private readonly statsService: StatsService,
// Registered only with QUEUE_ENABLED=true (see metrics.module.ts); absent otherwise.
@Optional() @InjectQueue(QUEUE_NAMES.WEBHOOK) private readonly webhookQueue?: Queue,
@Optional() @InjectQueue(QUEUE_NAMES.INGRESS) private readonly ingressQueue?: Queue,
) {}
onModuleDestroy(): void {
clearInterval(this.loopSampler);
}
private get token(): string {
return (this.config.get<string>('METRICS_TOKEN') ?? '').trim();
}
/**
* Throws if the caller may not scrape: 404 when metrics are disabled (no token configured),
* 401 when a token is configured but the request's bearer is missing or wrong, 429 once `client`
* has spent its failed-attempt window.
*/
assertScrapeAuthorized(authorizationHeader: string | undefined, client?: RequestLike): void {
const expected = this.token;
if (!expected) {
throw new NotFoundException('Metrics endpoint is disabled (set METRICS_TOKEN to enable)');
}
const subject = client
? limiterKeyForIp(resolveClientIp(client, this.config.get<string[]>('security.trustedProxies') ?? []))
: '';
if (!this.failedScrapeLimiter.allow(subject)) {
throw new HttpException('Too many failed metrics token attempts', HttpStatus.TOO_MANY_REQUESTS);
}
const provided = (authorizationHeader ?? '').replace(/^Bearer\s+/i, '').trim();
// Length-hiding constant-time compare, so the expected token's byte-length is not leaked through a fast
// length-mismatch return: this `/api/metrics` endpoint is @Public and timed by callers.
if (!provided || !constantTimeEqual(provided, expected)) {
throw new UnauthorizedException('Invalid metrics token');
}
this.failedScrapeLimiter.refund(subject);
}
/** Render the current metrics in Prometheus text exposition format (memoized for a short TTL). */
async render(): Promise<string> {
const now = Date.now();
if (this.cachedRender && now - this.cachedRender.at < METRICS_RENDER_TTL_MS) {
return this.cachedRender.text;
}
// The database-derived series are best-effort. Awaiting them unguarded meant a single rejected
// query — a statement timeout, pool exhaustion, SQLITE_BUSY under load, or a genuine outage —
// answered the whole scrape with a 500, so Prometheus lost the process, HTTP and webhook series
// too, and `up` conflated "process dead" with "database unreachable": the exact incident this
// endpoint exists to describe. The series that need no database are emitted either way, and the
// ones that do are OMITTED rather than reported as zero, because a zero would fire an alert
// claiming every session had dropped. A read that merely hangs is bounded the same way.
const [overview, queueCounts] = await Promise.all([this.readOverviewOrNull(), this.readQueueCounts()]);
const mem = process.memoryUsage();
const lines: string[] = [];
const gauge = (name: string, help: string, value: number, labels = ''): void => {
lines.push(`# HELP ${name} ${help}`);
lines.push(`# TYPE ${name} gauge`);
lines.push(`${name}${labels} ${value}`);
};
gauge('openwa_up', 'Whether the OpenWA process is up (always 1 when scraped).', 1);
gauge('openwa_process_uptime_seconds', 'Process uptime in seconds.', Math.round(process.uptime()));
gauge('openwa_process_resident_memory_bytes', 'Resident set size in bytes.', mem.rss);
gauge('openwa_process_heap_used_bytes', 'V8 heap used in bytes.', mem.heapUsed);
gauge(
'openwa_event_loop_delay_p99_seconds',
'p99 event-loop delay since the previous uncached scrape, in seconds.',
Math.max(0, this.loopDelay.percentile(99) - LOOP_DELAY_FLOOR_NS) / 1e9,
);
gauge(
'openwa_event_loop_delay_max_seconds',
'Maximum event-loop delay since the previous uncached scrape, in seconds.',
Math.max(0, this.loopDelay.max - LOOP_DELAY_FLOOR_NS) / 1e9,
);
this.loopDelay.reset();
const rejections = getUnhandledRejections();
lines.push(
'# HELP openwa_unhandled_rejections_total Promise rejections that reached the process-level handler since process start.',
);
lines.push('# TYPE openwa_unhandled_rejections_total counter');
lines.push(`openwa_unhandled_rejections_total{kind="other"} ${rejections.other}`);
lines.push(`openwa_unhandled_rejections_total{kind="page_context_lost"} ${rejections.page_context_lost}`);
// Read from the shared Redis, so every node reports the same cluster-wide value. A queue that is
// disabled or could not be read is omitted, not reported as empty.
if (queueCounts.length > 0) {
lines.push('# HELP openwa_queue_jobs Jobs in each BullMQ queue by state (cluster-wide).');
lines.push('# TYPE openwa_queue_jobs gauge');
for (const [name, counts] of queueCounts) {
for (const state of QUEUE_JOB_STATES) {
lines.push(`openwa_queue_jobs{queue="${name}",state="${state}"} ${counts[state] ?? 0}`);
}
}
}
gauge(
'openwa_stats_available',
'Whether the database-derived series below could be read (1) or not (0); they can trail the database by STATS_CACHE_TTL_MS + 5 s, or up to 5 min while a read runs late.',
overview ? 1 : 0,
);
if (overview) {
gauge('openwa_sessions_total', 'Total number of configured sessions.', overview.sessions.total);
gauge('openwa_sessions_active', 'Number of READY (active) sessions.', overview.sessions.active);
// Per-status session counts share one metric name with a `status` label.
lines.push('# HELP openwa_sessions Number of sessions by status.');
lines.push('# TYPE openwa_sessions gauge');
for (const [status, count] of Object.entries(overview.sessions.byStatus)) {
lines.push(`openwa_sessions{status="${this.escapeLabel(status)}"} ${count}`);
}
lines.push('# HELP openwa_messages_total Current stored messages by direction.');
lines.push('# TYPE openwa_messages_total gauge');
lines.push(`openwa_messages_total{direction="outgoing"} ${overview.messages.sent}`);
lines.push(`openwa_messages_total{direction="incoming"} ${overview.messages.received}`);
lines.push('# HELP openwa_messages_failed_total Current stored messages in FAILED state.');
lines.push('# TYPE openwa_messages_failed_total gauge');
lines.push(`openwa_messages_failed_total ${overview.messages.failed}`);
}
lines.push(
'# HELP openwa_webhook_delivery_failures_total Webhook delivery failures recorded since process start: retries exhausted, never sent (shed, refused at shutdown, rejected before sending, or an opt-in event dropped past the bound of its webhook), or stopped by shutdown between direct retries.',
);
lines.push('# TYPE openwa_webhook_delivery_failures_total counter');
lines.push(`openwa_webhook_delivery_failures_total ${getWebhookDeliveryFailuresTotal()}`);
lines.push(
'# HELP openwa_session_reconnect_attempts_total Reconnect attempts scheduled across all sessions since process start.',
);
lines.push('# TYPE openwa_session_reconnect_attempts_total counter');
lines.push(`openwa_session_reconnect_attempts_total ${getSessionReconnectAttemptsTotal()}`);
lines.push('# HELP openwa_session_reconnect_loop_alerts_total Reconnect-loop alerts emitted since process start.');
lines.push('# TYPE openwa_session_reconnect_loop_alerts_total counter');
lines.push(`openwa_session_reconnect_loop_alerts_total ${getSessionReconnectLoopAlertsTotal()}`);
lines.push('# HELP openwa_sessions_restricted Sessions whose account WhatsApp is currently restricting.');
lines.push('# TYPE openwa_sessions_restricted gauge');
lines.push(`openwa_sessions_restricted ${getRestrictedSessionCount()}`);
// Emitted only once a refusal has actually happened, like the HTTP series: a family that appears
// at its first occurrence is easier to alert on than one pinned at zero for every reason.
const refusals = getSendPacingRefusals();
if (refusals.size > 0) {
lines.push('# HELP openwa_send_pacing_refusals_total Sends refused by the pacing governor since process start.');
lines.push('# TYPE openwa_send_pacing_refusals_total counter');
for (const [reason, count] of refusals) {
lines.push(`openwa_send_pacing_refusals_total{reason="${this.escapeLabel(reason)}"} ${count}`);
}
}
// HTTP RED metrics (request rate + duration per route), recorded by RequestMetricsInterceptor.
// Included in the same cached render — a few seconds of staleness is fine for Prometheus.
lines.push(...renderHttpRequestMetrics());
const text = lines.join('\n') + '\n';
this.cachedRender = { at: now, text };
return text;
}
/**
* The overview, or null when it failed or did not settle within STATS_READ_TIMEOUT_MS and no recent one is
* kept. Scrapes arriving inside that wait share it, so they neither queue up nor start a deadline of their own.
* Once it has run out, scrapes behind the same read do not wait again: they get the kept result while it is
* younger than STATS_STALE_MAX_MS, else nothing, and past that age the read is given up on and a new one starts.
* A read that only runs late is kept when it completes; a failed one discards what an earlier read kept.
*/
private readOverviewOrNull(): Promise<OverviewStats | null> {
const now = Date.now();
if (this.pendingOverview) {
const waited = now - this.pendingSince;
if (waited < STATS_READ_TIMEOUT_MS) return this.pendingOverview;
if (waited < STATS_STALE_MAX_MS) return Promise.resolve(this.recentOverview());
this.logger.warn('Metrics scrape gave up on a statistics read that never settled; starting a new one');
}
this.pendingSince = now;
const seq = ++this.readSeq;
const work = Promise.resolve().then(() => this.statsService.getOverview());
const late = new Error('statistics read timed out');
let timer: ReturnType<typeof setTimeout> | undefined;
const waiting = Promise.race([
work,
new Promise<never>((_, reject) => {
timer = setTimeout(() => reject(late), STATS_READ_TIMEOUT_MS);
}),
]).then(
overview => {
clearTimeout(timer);
return overview;
},
err => {
clearTimeout(timer);
const recent = err === late ? this.recentOverview() : null;
this.logger.warn(
`Metrics scrape could not read stats; ${recent ? 'serving the last result' : 'database-derived series omitted'}: ` +
(err instanceof Error ? err.message : String(err)),
);
return recent;
},
);
this.pendingOverview = waiting;
// Only a read newer than the one the kept result came from may replace it, so an abandoned read that
// settles after its successor changes nothing.
const settled = (kept: { overview: OverviewStats; at: number } | null): void => {
if (seq > this.keptSeq) {
this.keptSeq = seq;
this.lastOverview = kept;
}
if (this.pendingOverview === waiting) this.pendingOverview = null;
};
void work.then(
overview => settled({ overview, at: now }),
() => settled(null),
);
return waiting;
}
private recentOverview(): OverviewStats | null {
const last = this.lastOverview;
return last && Date.now() - last.at < STATS_STALE_MAX_MS ? last.overview : null;
}
/**
* Job counts for each registered queue that answered within QUEUE_READ_TIMEOUT_MS. producerReady keeps a
* scrape from leaving a read pending on a Redis that has never connected.
*/
private async readQueueCounts(): Promise<Array<[string, Record<string, number>]>> {
const queues = [
[QUEUE_NAMES.WEBHOOK, this.webhookQueue],
[QUEUE_NAMES.INGRESS, this.ingressQueue],
] as const;
const results: Array<[string, Record<string, number>]> = [];
for (const [name, queue] of queues) {
if (!queue) continue;
let timer: NodeJS.Timeout | undefined;
try {
const counts = await Promise.race([
producerReady(queue).then(() => queue.getJobCounts(...QUEUE_JOB_STATES)),
new Promise<never>((_, reject) => {
timer = setTimeout(() => reject(new Error('timed out')), QUEUE_READ_TIMEOUT_MS);
}),
]);
results.push([name, counts]);
} catch (err) {
this.logger.warn(
`Metrics scrape could not read ${name} job counts; series omitted: ${err instanceof Error ? err.message : String(err)}`,
);
} finally {
clearTimeout(timer);
}
}
return results;
}
private escapeLabel(value: string): string {
return value.replace(/\\/g, '\\\\').replace(/"/g, '\\"').replace(/\n/g, '\\n');
}
}