Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

17.3 KB
Raw Blame
import { INestApplicationContext } from '@nestjs/common';
import { IoAdapter } from '@nestjs/platform-socket.io';
import { createAdapter } from '@socket.io/redis-adapter';
import Redis, { type RedisOptions } from 'ioredis';
import type { Server, ServerOptions } from 'socket.io';
import { createLogger } from '../../common/services/logger.service';
import { redisConnectionOptions } from '../../config/redis-options';

const logger = createLogger('RedisIoAdapter');

/** Max time to await a graceful `quit()` per client on shutdown before force-disconnecting. */
export const WS_REDIS_QUIT_TIMEOUT_MS = 2000;

/**
 * How long the promise of one fan-out publish waits for its reply. Only the promise: the command stays on the
 * wire, and a Redis that was merely slow still delivers it, so this bounds nothing the client holds.
 */
export const WS_REDIS_COMMAND_TIMEOUT_MS = 2000;

/**
 * How long the publish client's socket may stay silent after a write before it is destroyed, which is what
 * stops a stalled Redis from filling the client. The timer runs on this process's own event loop, so a healthy
 * Redis looks silent whenever the loop stays blocked for that long right after a write, and the connection would
 * be dropped for nothing. That is why it sits far above any block this process runs, and above the command
 * timeout. It bounds a Redis that sends nothing: after this long the socket is destroyed and nothing more is
 * held. What was already held, at most this many seconds of events, stays until the client has reconnected. A
 * Redis that answers, however slowly, resets the timer with every reply, and is bounded by
 * WS_REDIS_MAX_UNANSWERED instead.
 */
export const WS_REDIS_SOCKET_TIMEOUT_MS = 10_000;

/**
 * How often the subscriber connection is pinged. It only receives, so a connection that stops answering without
 * being closed (a NAT or a firewall that drops the flow, a proxy that keeps its own side open) would otherwise go
 * unnoticed until TCP keepalive gives up, or for good behind such a proxy, and the node would miss what the other
 * replicas publish meanwhile. A ping arms the subscriber's socket deadline, so a connection that does not answer is
 * replaced, and its subscriptions restored, within this interval plus WS_REDIS_SUBSCRIBER_SOCKET_TIMEOUT_MS.
 */
export const WS_REDIS_SUBSCRIBER_PING_MS = 15_000;

/**
 * How long the subscriber's socket may stay silent after a ping before it is destroyed. Above the publisher's
 * deadline on purpose: a replaced subscriber misses what Redis delivers before it has subscribed again, so through
 * a pause longer than WS_REDIS_SOCKET_TIMEOUT_MS this node still receives, late, what the others had written before
 * their own deadline, rather than losing it. Up to this long a pause costs nothing more than it did without pings.
 */
export const WS_REDIS_SUBSCRIBER_SOCKET_TIMEOUT_MS = 30_000;

/**
 * Most commands the publisher keeps written and unanswered once the backlog has lasted. A Redis that answers slower
 * than events are raised never trips the socket deadline, so without a cap the backlog grows with the load for as
 * long as the slowness lasts. The rule: frames are dropped for the other replicas while the queue holds this many
 * commands and has not been below half of it for longer than a command is given to be answered
 * (WS_REDIS_COMMAND_TIMEOUT_MS), and they go on being dropped until the queue is back below half of it. Depth alone
 * says nothing, since a burst raised in one tick is all on the queue before a reply can be read, so such a burst is
 * not cut as long as raising it past half the cap takes less than a command is given. One that takes longer, a
 * synchronous loop that cannot yield for over 2 s, is cut at the cap whatever the peer is doing, and the warning
 * names the unanswered count. Dropping starts with a warning and ends with another, with the count. The queue can
 * pass the cap by the traffic of one command deadline before dropping starts. 12,000 holds a full socket deadline
 * of traffic at 1,000 events a second with a fifth to spare, which measured about 25 MB of heap with 1 KiB frames.
 * It counts commands, not bytes: events with inline media are larger.
 */
export const WS_REDIS_MAX_UNANSWERED = 12_000;

/** The queue length below which a backlog counts as over, and dropped frames are published again. */
const WS_REDIS_RESUME_UNANSWERED = WS_REDIS_MAX_UNANSWERED / 2;

/**
 * Commands the client has written and not had answered. ioredis keeps them on an internal queue; if this version
 * does not expose it, the answer is 0 and the cap does nothing, so a change of ioredis loses the bound rather than
 * the fan-out (the socket deadline still applies).
 */
function unansweredCommands(client: Redis): number {
  const queue = (client as unknown as { commandQueue?: { length?: unknown } }).commandQueue;
  return typeof queue?.length === 'number' ? queue.length : 0;
}

/**
 * Whether cross-replica WebSocket fan-out is switched on. Same `REDIS_ENABLED === 'true'` gate the
 * throttler and cache read at boot, so a deployment already using Redis for those gets event
 * fan-out with no extra flag, and a single-node deployment (the default) never opens a connection.
 */
export function isWsRedisEnabled(): boolean {
  return process.env.REDIS_ENABLED === 'true';
}

/** ioredis options for the pub/sub pair, mirroring the throttler/cache connection env exactly. */
export function wsRedisOptions(): RedisOptions {
  return {
    ...redisConnectionOptions(),
    // The adapter's SUBSCRIBE connection cannot issue ordinary commands, so a bounded retry that
    // gives up (returning null) would strand fan-out permanently after one blip. Reconnect forever
    // with capped backoff, matching the cache client.
    retryStrategy: times => Math.min(times * 500, 5000),
  };
}

/**
 * Socket.IO always creates the root namespace `/` and accepts a CONNECT to it with no middleware, while
 * everything this gateway enforces (the handshake limit, the per-key socket cap, the audit rows) lives on
 * `/events`. The 45 s connect timeout only closes a peer that joined no namespace at all, so a peer that
 * joins `/` and answers pings would stay connected for ever, outside every limit. Refusing `/` leaves such a
 * peer unjoined, and the timeout reaps it. No client of this gateway uses the root namespace.
 */
export function rejectRootNamespace(server: Server): void {
  server.of('/').use((_socket, next) => next(new Error('Invalid namespace')));
}

/**
 * A Socket.IO server whose broadcasts reach clients connected to OTHER replicas, via the Redis
 * pub/sub adapter.
 *
 * The gateway already broadcasts to rooms (`server.to(room).emit(...)`); the default in-memory
 * adapter only reaches sockets on THIS process, so a client connected to replica A never sees an
 * event raised on replica B. Attaching the Redis adapter on the root `io` — before any namespace
 * is used — makes every namespace, including `/events`, fan its room broadcasts out through Redis.
 * No emit site changes.
 *
 * SCOPE (honest): this distributes event FAN-OUT only. Key eviction (`socketsByKeyId`), the WS
 * rate-limit buckets, and the engine registry remain process-local — a key revoked on replica A
 * still tears down only A's sockets, and per-key WS limits are counted per replica. See
 * docs/13-horizontal-scaling.md.
 *
 * Failure posture: if the pub/sub clients cannot be created the server falls back to the in-memory
 * adapter — the local node keeps working, only cross-node fan-out is lost — rather than refusing to
 * boot. A Redis outage, at boot or after it, is ioredis's problem to retry; the adapter recovers on reconnect.
 * The subscriber only listens, so a connection that stops answering without being closed errors nothing until TCP
 * keepalive gives up, and never when a proxy in the path keeps its side open: it is pinged every
 * WS_REDIS_SUBSCRIBER_PING_MS and replaced, and subscribed again, when an answer takes longer than
 * WS_REDIS_SUBSCRIBER_SOCKET_TIMEOUT_MS.
 *
 * Delivery while Redis is away or slow: the fan-out to the other replicas is best effort and keeps no replay
 * queue. A Redis that pauses for less than WS_REDIS_SOCKET_TIMEOUT_MS answers afterwards, and every event raised
 * meanwhile reaches them late, at up to about 1,200 events a second; above that the cap below applies first, and
 * drops the frames beyond it. Silence for that long destroys the publisher's socket, and so does a restart, a
 * failover or an outage: what was written to the dropped connection may be lost with it, or may still reach the
 * other replicas late and after newer events. A subscriber whose ping waits through a silence longer than
 * WS_REDIS_SUBSCRIBER_SOCKET_TIMEOUT_MS is replaced too, and this node then misses what the others had published
 * into it. The events this node raises while it is disconnected, plus its reconnect backoff of up to 5 s once Redis
 * is back (wsRedisOptions), are dropped for the other replicas, never delivered late and never twice. A Redis that
 * keeps answering, but slower than events are raised, never looks silent: it is bounded by WS_REDIS_MAX_UNANSWERED,
 * which drops frames for the other replicas once the backlog has outlived a command's deadline and until it is back
 * below half of it (a burst raised in one tick is not cut unless raising it past half the cap takes over 2 s).
 * Clients on this node are served by the local broadcast and webhooks do not use this path, so neither is
 * affected. What the publisher holds is bounded by one socket deadline of traffic for a silent Redis and by that
 * cap, plus the traffic of one command deadline, for a slow one.
 */
export class RedisIoAdapter extends IoAdapter {
  private pubClient?: Redis;
  private subClient?: Redis;
  private subPing?: NodeJS.Timeout;

  constructor(app: INestApplicationContext) {
    super(app);
  }

  createIOServer(port: number, options?: ServerOptions): Server {
    const server = super.createIOServer(port, options) as Server;
    rejectRootNamespace(server);
    if (!isWsRedisEnabled()) return server;

    try {
      // The publisher is the only client that waits for replies, and a Redis that keeps the connection open but
      // stops answering (a frozen process, a stalled disk) never errors it: every broadcast would wait in the
      // client, payload and all, for as long as the stall lasts. So the promise of a command the peer does not
      // answer in time is rejected (the command stays on the wire, so a slow Redis still delivers it), a socket
      // that stays silent for the socket deadline is destroyed and the client reconnects, nothing is queued for
      // a connection it does not have, and nothing the old socket may already have delivered is sent again. An
      // event raised while disconnected is therefore dropped for the other replicas rather than replayed late.
      // A silent Redis holds at most one socket deadline of traffic; a slow one that keeps answering is held to
      // WS_REDIS_MAX_UNANSWERED (see there for the rule) by the wrapper below.
      const pubClient = new Redis({
        ...wsRedisOptions(),
        commandTimeout: WS_REDIS_COMMAND_TIMEOUT_MS,
        socketTimeout: WS_REDIS_SOCKET_TIMEOUT_MS,
        enableOfflineQueue: false,
        autoResendUnfulfilledCommands: false,
      });
      // The adapter issues its SUBSCRIBE/PSUBSCRIBE once, at construction. With a bounded per-request
      // retry, an outage at boot flushes them before Redis is ever reached, and ioredis only replays
      // subscriptions the server confirmed, so the replica would never subscribe. This connection
      // carries nothing else, so keep its commands queued until the first connect. Nothing waits on
      // its replies, so the publisher's command deadline and dropped queues do not apply to it. A
      // socket deadline of its own does, armed by the pings below: a destroyed socket loses no
      // subscription, since on the next ready ioredis subscribes again to what the server had
      // confirmed and resends the commands it had not answered.
      const subClient = new Redis({
        ...wsRedisOptions(),
        maxRetriesPerRequest: null,
        socketTimeout: WS_REDIS_SUBSCRIBER_SOCKET_TIMEOUT_MS,
      });
      for (const [name, client] of [
        ['pub', pubClient],
        ['sub', subClient],
      ] as const) {
        client.on('error', err => logger.warn(`Redis ${name} client error: ${err.message}`));
      }
      // The adapter fires publish() on every broadcast and drops the promise. During an outage each of
      // those commands is rejected, so every event emitted meanwhile surfaced as an unhandled
      // rejection. The 'error' listener above already reports the outage once; a fan-out frame without a
      // confirmed delivery is only worth a debug line. A timed-out command was still written, and a slow
      // Redis delivers it late; one rejected for lack of a connection was never sent.
      const publish = pubClient.publish.bind(pubClient);
      let dropping = false;
      let dropped = 0;
      // When the queue was last seen below half the cap: the backlog is as old as the time since. Read from the
      // monotonic clock: the wall clock can step back (an NTP correction, a resumed VM), and an age taken from it
      // would be negative for as long as the step is large, leaving a slow Redis uncapped.
      let lastBelowResume = 0;
      pubClient.publish = (channel: string, message: string | Buffer) => {
        const unanswered = unansweredCommands(pubClient);
        const now = performance.now();
        if (unanswered < WS_REDIS_RESUME_UNANSWERED) {
          lastBelowResume = now;
          if (dropping) {
            logger.warn(`Redis caught up; ${dropped} fan-out frames were dropped for the other replicas`);
            dropping = false;
            dropped = 0;
          }
        } else if (
          !dropping &&
          unanswered >= WS_REDIS_MAX_UNANSWERED &&
          now - lastBelowResume > WS_REDIS_COMMAND_TIMEOUT_MS
        ) {
          // Nothing else reports a Redis that answers slowly, so the start of an episode is warned about.
          dropping = true;
          logger.warn(
            `Redis has ${unanswered} fan-out frames unanswered; fan-out frames are dropped for the other replicas until it catches up`,
          );
        }
        if (dropping) {
          dropped++;
          return Promise.resolve(0);
        }
        return publish(channel, message).catch((err: Error) => {
          logger.debug(`Redis publish not confirmed: ${err.message}`);
          return 0;
        });
      };
      this.pubClient = pubClient;
      this.subClient = subClient;
      server.adapter(createAdapter(pubClient, subClient));
      // Only a ready connection is pinged: one issued while reconnecting would wait in the offline queue
      // for as long as the outage lasts. A failed ping needs nothing more, the deadline or the next ping
      // covers the connection. The timer alone does not keep the process alive.
      this.subPing = setInterval(() => {
        if (subClient.status === 'ready') subClient.ping().catch(() => undefined);
      }, WS_REDIS_SUBSCRIBER_PING_MS).unref();
      logger.log('WebSocket events fan out across replicas via the Redis adapter');
    } catch (error) {
      logger.error(
        'Could not attach the Redis WebSocket adapter; falling back to single-node event delivery. ' +
          'Events raised on other replicas will not reach clients connected here.',
        error instanceof Error ? error.stack : String(error),
      );
    }
    return server;
  }

  /** Close the pub/sub connections so a shutdown does not leak them. Called by Nest on app close. */
  async close(server: Server): Promise<void> {
    // Socket.IO's adapters unsubscribe from their Redis channels as the server closes. Tearing the
    // clients down FIRST left those unsubscribes to be issued on dead clients, and each rejection
    // surfaced as an unhandled rejection — a handful of ERROR lines on every graceful shutdown, for
    // a shutdown that had in fact gone fine. Close the server first, then release the pair. Under
    // forceCloseConnections Nest skips the server close when the socket.io server shares the HTTP
    // server, so only the pub/sub pair is released here and the HTTP close tears the sockets down.
    clearInterval(this.subPing);
    await super.close(server);
    await Promise.allSettled([this.quitClient(this.pubClient), this.quitClient(this.subClient)]);
    this.pubClient = undefined;
    this.subClient = undefined;
  }

  /**
   * Release one client on shutdown without ever blocking. `quit()` waits for the QUIT reply, which
   * never arrives on a half-open/partitioned socket — and with the never-give-up retryStrategy and
   * ioredis's default `enableOfflineQueue:true`, quit() is queued and hangs indefinitely rather
   * than rejecting. Race it against a short deadline, then ALWAYS `disconnect()` in the finally so
   * the socket and its reconnect timer are gone regardless of connection state (mirrors
   * CacheService/RedisThrottlerStorage). disconnect() is idempotent after a clean quit.
   */
  private async quitClient(client: Redis | undefined): Promise<void> {
    if (!client) return;
    let timer: NodeJS.Timeout | undefined;
    const deadline = new Promise<void>(resolve => {
      timer = setTimeout(resolve, WS_REDIS_QUIT_TIMEOUT_MS);
      timer.unref();
    });
    try {
      await Promise.race([client.quit().catch(() => undefined), deadline]);
    } finally {
      if (timer) clearTimeout(timer);
      client.disconnect();
    }
  }
}