Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

15.5 KB
Raw Blame
import { HttpException, HttpStatus, NotFoundException, UnauthorizedException } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { Queue } from 'bullmq';
import { MetricsService, METRICS_RENDER_TTL_MS, QUEUE_READ_TIMEOUT_MS } from './metrics.service';
import { StatsService, OverviewStats } from '../stats/stats.service';
import { producerConnectWaitMs } from '../queue/redis-connection';
import { getWebhookDeliveryFailuresTotal } from '../../common/metrics/webhook-delivery-metrics';
import {
  getSessionReconnectAttemptsTotal,
  getSessionReconnectLoopAlertsTotal,
} from '../../common/metrics/session-reconnect-metrics';
import { setRestrictedSessionCount } from '../../common/metrics/session-restriction-metrics';
import { getUnhandledRejections, incrementUnhandledRejections } from '../../common/metrics/process-error-metrics';

describe('MetricsService', () => {
  const overview: OverviewStats = {
    sessions: { active: 2, total: 3, byStatus: { ready: 2, failed: 1 } },
    messages: { sent: 100, received: 50, failed: 3, today: { sent: 10, received: 5 } },
  };

  const makeService = (token?: string): MetricsService => {
    const config = { get: (k: string) => (k === 'METRICS_TOKEN' ? token : undefined) } as unknown as ConfigService;
    const stats = { getOverview: jest.fn().mockResolvedValue(overview) } as unknown as StatsService;
    return new MetricsService(config, stats);
  };

  describe('assertScrapeAuthorized', () => {
    it('returns 404 when no token is configured (endpoint disabled by default)', () => {
      const svc = makeService(undefined);
      expect(() => svc.assertScrapeAuthorized('Bearer anything')).toThrow(NotFoundException);
    });

    it('rejects a missing bearer with 401 when a token is configured', () => {
      const svc = makeService('s3cret');
      expect(() => svc.assertScrapeAuthorized(undefined)).toThrow(UnauthorizedException);
    });

    it('rejects a wrong token with 401', () => {
      const svc = makeService('s3cret');
      expect(() => svc.assertScrapeAuthorized('Bearer nope')).toThrow(UnauthorizedException);
    });

    it('accepts a correct bearer token', () => {
      const svc = makeService('s3cret');
      expect(() => svc.assertScrapeAuthorized('Bearer s3cret')).not.toThrow();
    });

    it('is tolerant of bearer casing/whitespace', () => {
      const svc = makeService('s3cret');
      expect(() => svc.assertScrapeAuthorized('bearer   s3cret')).not.toThrow();
    });

    // The route skips the shared throttler so scrapes cost no budget; failed compares get their own.
    it('answers 429 once one client has failed too often, without comparing further guesses', () => {
      const svc = makeService('s3cret');
      const guesser = { headers: {}, socket: { remoteAddress: '198.51.100.7' } };
      for (let i = 0; i < 10; i++) {
        expect(() => svc.assertScrapeAuthorized('Bearer nope', guesser)).toThrow(UnauthorizedException);
      }
      const locked = (): void => svc.assertScrapeAuthorized('Bearer s3cret', guesser);
      expect(locked).toThrow(HttpException);
      try {
        locked();
      } catch (err) {
        expect((err as HttpException).getStatus()).toBe(HttpStatus.TOO_MANY_REQUESTS);
      }
      // Another client is unaffected.
      expect(() =>
        svc.assertScrapeAuthorized('Bearer s3cret', { headers: {}, socket: { remoteAddress: '203.0.113.9' } }),
      ).not.toThrow();
    });

    it('never charges a successful scrape against that budget', () => {
      const svc = makeService('s3cret');
      const scraper = { headers: {}, socket: { remoteAddress: '198.51.100.8' } };
      for (let i = 0; i < 50; i++) {
        expect(() => svc.assertScrapeAuthorized('Bearer s3cret', scraper)).not.toThrow();
      }
    });
  });

  describe('render', () => {
    it('emits Prometheus exposition with session + message gauges', async () => {
      const svc = makeService('s3cret');
      const out = await svc.render();

      expect(out).toContain('openwa_up 1');
      expect(out).toContain('openwa_sessions_active 2');
      expect(out).toContain('openwa_sessions_total 3');
      expect(out).toContain('openwa_sessions{status="ready"} 2');
      expect(out).toContain('openwa_sessions{status="failed"} 1');
      expect(out).toContain('openwa_messages_total{direction="outgoing"} 100');
      expect(out).toContain('openwa_messages_total{direction="incoming"} 50');
      expect(out).toContain('openwa_messages_failed_total 3');
      // Every metric must declare HELP/TYPE before its sample.
      expect(out).toContain('# TYPE openwa_messages_total gauge');
      expect(out).toContain('# TYPE openwa_messages_failed_total gauge');
      // Webhook terminal-failure counter is emitted with correct counter typing + current total.
      expect(out).toContain('# TYPE openwa_webhook_delivery_failures_total counter');
      expect(out).toContain(`openwa_webhook_delivery_failures_total ${getWebhookDeliveryFailuresTotal()}`);
      // Reconnect observability counters are emitted with correct counter typing + current totals.
      expect(out).toContain('# TYPE openwa_session_reconnect_attempts_total counter');
      expect(out).toContain(`openwa_session_reconnect_attempts_total ${getSessionReconnectAttemptsTotal()}`);
      expect(out).toContain('# TYPE openwa_session_reconnect_loop_alerts_total counter');
      expect(out).toContain(`openwa_session_reconnect_loop_alerts_total ${getSessionReconnectLoopAlertsTotal()}`);
      expect(out.endsWith('\n')).toBe(true);
    });

    // A gauge, not a counter: what matters is how many accounts are restricted right now, and a
    // restriction that is applied, lifted and re-applied is one recurring fact, not a running total.
    it('emits the restricted-session gauge from the live count', async () => {
      setRestrictedSessionCount(2);
      const out = await makeService('s3cret').render();

      expect(out).toContain('# TYPE openwa_sessions_restricted gauge');
      expect(out).toContain('openwa_sessions_restricted 2');
    });

    it('reports zero restricted sessions rather than omitting the gauge', async () => {
      setRestrictedSessionCount(0);
      const out = await makeService('s3cret').render();

      expect(out).toContain('openwa_sessions_restricted 0');
    });

    it('memoizes the rendered output within the TTL (one getOverview per window)', async () => {
      jest.useFakeTimers();
      try {
        const config = {
          get: (k: string) => (k === 'METRICS_TOKEN' ? 's3cret' : undefined),
        } as unknown as ConfigService;
        const getOverview = jest.fn().mockResolvedValue(overview);
        const svc = new MetricsService(config, { getOverview } as unknown as StatsService);

        await svc.render();
        await svc.render();
        expect(getOverview).toHaveBeenCalledTimes(1); // 2nd scrape served from the memo, no DB work

        jest.advanceTimersByTime(METRICS_RENDER_TTL_MS + 1);
        await svc.render();
        expect(getOverview).toHaveBeenCalledTimes(2); // window expired → recomputed
      } finally {
        jest.useRealTimers();
      }
    });
  });
});

// The scrape had a hard runtime dependency on the data database: render() awaited
// StatsService.getOverview() unguarded, so once the stats memo lapsed during a database problem
// EVERY scrape answered 500 and Prometheus lost the whole target — including the process and HTTP
// series that need no database at all, during the exact incident they exist for.
describe('MetricsService.render survives a failing stats query', () => {
  const healthyOverview: OverviewStats = {
    sessions: { active: 2, total: 3, byStatus: { ready: 2, failed: 1 } },
    messages: { sent: 100, received: 50, failed: 3, today: { sent: 10, received: 5 } },
  };

  const failing = (): MetricsService => {
    const config = { get: () => undefined } as unknown as ConfigService;
    const stats = {
      getOverview: jest.fn().mockRejectedValue(new Error('SQLITE_BUSY: database is locked')),
    } as unknown as StatsService;
    return new MetricsService(config, stats);
  };

  it('still serves the series that need no database', async () => {
    const text = await failing().render();

    expect(text).toContain('openwa_up 1');
    expect(text).toContain('openwa_process_uptime_seconds');
    expect(text).toContain('openwa_process_resident_memory_bytes');
    expect(text).toContain('openwa_webhook_delivery_failures_total');
  });

  it('signals that the database-derived series are missing rather than reporting them as zero', async () => {
    const text = await failing().render();

    expect(text).toContain('openwa_stats_available 0');
    // A stale or invented 0 would be worse than an absent series: an alert on
    // openwa_sessions_active would fire as if every session had dropped.
    expect(text).not.toContain('openwa_sessions_active');
    expect(text).not.toContain('openwa_messages_total');
  });

  // Negative twin: a healthy scrape must still carry the database-derived series and say so.
  it('reports the stats source as available on a healthy scrape', async () => {
    const config = { get: () => undefined } as unknown as ConfigService;
    const stats = { getOverview: jest.fn().mockResolvedValue(healthyOverview) } as unknown as StatsService;
    const text = await new MetricsService(config, stats).render();

    expect(text).toContain('openwa_stats_available 1');
    expect(text).toContain('openwa_sessions_active 2');
  });
});

describe('MetricsService runtime series', () => {
  const overview: OverviewStats = {
    sessions: { active: 0, total: 0, byStatus: {} },
    messages: { sent: 0, received: 0, failed: 0, today: { sent: 0, received: 0 } },
  };
  const config = { get: () => undefined } as unknown as ConfigService;
  const stats = { getOverview: jest.fn().mockResolvedValue(overview) } as unknown as StatsService;
  const queue = (getJobCounts: () => Promise<Record<string, number>>): Queue =>
    ({ waitUntilReady: () => Promise.resolve(), getJobCounts }) as unknown as Queue;
  const services: MetricsService[] = [];
  const make = (webhook?: Queue, ingress?: Queue): MetricsService => {
    const svc = new MetricsService(config, stats, webhook, ingress);
    services.push(svc);
    return svc;
  };

  afterEach(() => {
    services.splice(0).forEach(svc => svc.onModuleDestroy());
    jest.useRealTimers();
  });

  const sample = (text: string, series: string): number => {
    const line = text.split('\n').find(l => l.startsWith(`${series} `));
    expect(line).toBeDefined();
    return Number(line!.split(' ')[1]);
  };

  it('reports event-loop delay since the previous uncached render, then starts a new window', async () => {
    const svc = make();
    const histogram = (svc as unknown as { loopDelay: { reset: () => void } }).loopDelay;
    const reset = jest.spyOn(histogram, 'reset');

    const text = await svc.render();

    expect(text).toContain('# TYPE openwa_event_loop_delay_p99_seconds gauge');
    expect(text).toContain('# TYPE openwa_event_loop_delay_max_seconds gauge');
    for (const series of ['openwa_event_loop_delay_p99_seconds', 'openwa_event_loop_delay_max_seconds']) {
      const value = sample(text, series);
      expect(Number.isFinite(value)).toBe(true);
      expect(value).toBeGreaterThanOrEqual(0);
    }
    expect(reset).toHaveBeenCalledTimes(1);
  });

  it('reports event-loop delay net of the sampling interval, so an idle loop reads zero', async () => {
    const svc = make();
    const histogram = (svc as unknown as { loopDelay: { percentile: (p: number) => number; max: number } }).loopDelay;
    // The histogram records the whole gap between its 20 ms timer callbacks: an idle loop reads about
    // 20 ms, and a 200 ms synchronous block reads about 220 ms.
    jest.spyOn(histogram, 'percentile').mockReturnValue(19.8e6);
    jest.spyOn(histogram, 'max', 'get').mockReturnValue(220e6);

    const text = await svc.render();

    expect(sample(text, 'openwa_event_loop_delay_p99_seconds')).toBe(0);
    expect(sample(text, 'openwa_event_loop_delay_max_seconds')).toBeCloseTo(0.2, 9);
  });

  // Resetting Node's interval histogram also dropped its previous-tick timestamp, so the first gap
  // after each render went unrecorded, and with it a stall that began right there.
  it('records a stall that begins right after an uncached render', async () => {
    const svc = make();
    const idle = (): Promise<void> => new Promise(resolve => setTimeout(resolve, 60));
    await idle();
    await svc.render();
    const until = Date.now() + 300;
    while (Date.now() < until) {
      // block the event loop
    }
    await idle();
    (svc as unknown as { cachedRender: unknown }).cachedRender = null;

    const text = await svc.render();

    expect(sample(text, 'openwa_event_loop_delay_max_seconds')).toBeGreaterThanOrEqual(0.25);
  });

  it('counts unhandled rejections by kind', async () => {
    const before = getUnhandledRejections();
    incrementUnhandledRejections('other');
    const text = await make().render();

    expect(text).toContain('# TYPE openwa_unhandled_rejections_total counter');
    expect(sample(text, 'openwa_unhandled_rejections_total{kind="other"}')).toBe(before.other + 1);
    expect(sample(text, 'openwa_unhandled_rejections_total{kind="page_context_lost"}')).toBe(before.page_context_lost);
  });

  it('reports job counts per queue and state', async () => {
    const text = await make(
      queue(() => Promise.resolve({ wait: 4, active: 1, delayed: 2, failed: 3 })),
      queue(() => Promise.resolve({ wait: 0, active: 0, delayed: 0, failed: 7 })),
    ).render();

    expect(text).toContain('# TYPE openwa_queue_jobs gauge');
    expect(text).toContain('openwa_queue_jobs{queue="webhook-queue",state="wait"} 4');
    expect(text).toContain('openwa_queue_jobs{queue="webhook-queue",state="failed"} 3');
    expect(text).toContain('openwa_queue_jobs{queue="ingress-queue",state="failed"} 7');
  });

  it('omits a queue whose read fails or hangs instead of reporting zero', async () => {
    jest.useFakeTimers();
    const rendering = make(
      queue(() => new Promise(() => undefined)),
      queue(() => Promise.reject(new Error('Connection is closed.'))),
    ).render();
    await jest.advanceTimersByTimeAsync(QUEUE_READ_TIMEOUT_MS);
    const text = await rendering;

    expect(text).toContain('openwa_up 1');
    expect(text).not.toContain('openwa_queue_jobs');
  });

  it('leaves no job-count read pending on a queue whose Redis has never connected', async () => {
    jest.useFakeTimers();
    const getJobCounts = jest.fn(() => Promise.resolve({ wait: 1, active: 0, delayed: 0, failed: 0 }));
    const svc = make({ waitUntilReady: () => new Promise(() => undefined), getJobCounts } as unknown as Queue);

    const rendering = svc.render();
    await jest.advanceTimersByTimeAsync(QUEUE_READ_TIMEOUT_MS);
    expect(await rendering).not.toContain('openwa_queue_jobs');

    // Once the first-connect wait has passed, a scrape skips the queue without waiting.
    await jest.advanceTimersByTimeAsync(producerConnectWaitMs());
    (svc as unknown as { cachedRender: unknown }).cachedRender = null;
    expect(await svc.render()).not.toContain('openwa_queue_jobs');
    expect(getJobCounts).not.toHaveBeenCalled();
  });

  it('keeps the healthy queue when only the other one fails', async () => {
    const text = await make(
      queue(() => Promise.resolve({ wait: 1, active: 0, delayed: 0, failed: 0 })),
      queue(() => Promise.reject(new Error('Connection is closed.'))),
    ).render();

    expect(text).toContain('openwa_queue_jobs{queue="webhook-queue",state="wait"} 1');
    expect(text).not.toContain('queue="ingress-queue"');
  });

  it('emits no queue series when the queue is disabled', async () => {
    expect(await make().render()).not.toContain('openwa_queue_jobs');
  });
});