Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

13.6 KB
Raw Blame
import { Injectable, OnApplicationBootstrap, Optional } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { InjectQueue } from '@nestjs/bullmq';
import { JobState, Queue } from 'bullmq';
import { createHash } from 'crypto';
import { PluginLoaderService } from '../../core/plugins/plugin-loader.service';
import { IngressJobData } from '../queue/processors/ingress.processor';
import { IntegrationDeliveryFailure } from './entities/integration-delivery-failure.entity';
import { QUEUE_NAMES } from '../queue/queue-names';
import { producerReady } from '../queue/redis-connection';
import { createLogger } from '../../common/services/logger.service';
import { resolveNonNegativeIntEnv } from '../../config/configuration';

/**
 * Outcome of an enqueue attempt. 'queued' = handed to BullMQ; 'dispatched' = delivered inline; 'failed'
 * = inline dispatch threw and was swallowed (`error` carries the message). enqueue() does not throw
 * (bar an aborted `signal`, see enqueue), so callers use the outcome (not exceptions) to decide
 * durability follow-up: a LIVE ingress delivery must persist a dead-letter row on 'failed' (see
 * buildIngressDeadLetterRow, wired at the IngressService factory) so RedriveService can replay it;
 * RedriveService itself calls enqueue() directly, because a failed replay must keep its EXISTING
 * dead-letter row redrivable rather than write a second one.
 */
export type EnqueueOutcome = { outcome: 'queued' | 'dispatched' | 'failed'; error?: string };

/**
 * Retry policy for inbound ingress jobs. Previously the enqueue passed only a jobId, so BullMQ ran a
 * SINGLE attempt — a transient plugin-sandbox 5xx went straight to the DLQ with no retry (asymmetric
 * with the webhook queue). A few exponential-backoff attempts absorb transient failures; the
 * final-attempt DLQ write in ingress.processor is gated on `job.opts.attempts`, so it still fires
 * exactly once, only after these are exhausted. Env-overridable (INGRESS_MAX_ATTEMPTS /
 * INGRESS_RETRY_DELAY_MS); an invalid value falls back to the default.
 */
export function resolveIngressJobOptions(): { attempts: number; backoff: { type: 'exponential'; delay: number } } {
  return {
    attempts: resolveNonNegativeIntEnv(process.env.INGRESS_MAX_ATTEMPTS, 0) || 3,
    backoff: { type: 'exponential', delay: resolveNonNegativeIntEnv(process.env.INGRESS_RETRY_DELAY_MS, 5000) },
  };
}

/**
 * BullMQ refuses several jobId shapes outright, at two validation sites: Job.validateOptions rejects
 * an exact integer string ("Custom Id cannot be integers") and a colon id that does not split into
 * exactly 3 parts ("Custom Id cannot contain :"); Queue.addJob additionally rejects '0' and any
 * '0:'-prefixed id ("JobId cannot be '0' or start with '0:'"). Providers send numeric dedup headers
 * (`svix-Id: 12345`), and the redrive path mints `redrive:<uuid>`, so these refusals happen in
 * practice, and because enqueue()'s catch-all treats ANY add() throw as "Redis unreachable", the job
 * silently degraded to inline dispatch with no retry, no backoff, and a blocked redrive loop. Every id
 * therefore maps to a deterministic sha256 prefix, which BullMQ always accepts: its exactly-once dedup
 * only needs the id STABLE per delivery, not recognizable, and the reconciler replays through this
 * same function so its dedup against an earlier enqueue is preserved.
 *
 * The hash input is namespaced with the plugin/instance pair, for EVERY id rather than only the
 * refused shapes. BullMQ dedups jobIds across the WHOLE shared ingress queue, while the database dedup
 * is (pluginId, instanceId, providerDeliveryId), and two instances can share a provider id: numeric
 * per-account sequences, or one Standard Webhooks message fanned out to two endpoints with the same
 * `webhook-id`. Without the namespace, the second instance's delivery would collide with the first's
 * job id and BullMQ would silently discard it (resolved as the existing job, no DLQ row); with it, the
 * queue-level dedup matches the database-level scope. A non-string id (a duplicated header can surface
 * as string[]) is coerced rather than trusted to reach BullMQ's own checks.
 */
export function sanitizeIngressJobId(jobId: string, namespace = ''): string {
  const raw = typeof jobId === 'string' ? jobId : String(jobId);
  return `ing-${createHash('sha256').update(`${namespace}\u0000${raw}`).digest('hex').slice(0, 40)}`;
}

function queueJobId(data: IngressJobData, jobId: string): string {
  return sanitizeIngressJobId(jobId, `${data.pluginId}\u0000${data.instanceId}`);
}

/**
 * The two ids IngressProcessor re-queues a delivery under when its dead-letter row cannot be written.
 * A copy that fails the same way re-queues under the other one, so the ids stay fixed and a live copy
 * can be looked up by id instead of being mistaken for a dead-lettered delivery.
 */
export function requeuedJobIds(baseJobId: string): [string, string] {
  return [`${baseJobId}-requeued-1`, `${baseJobId}-requeued-2`];
}

// A job in one of these states still owns the delivery, or already delivered it.
const isLiveOrCompleted = (state: JobState | 'unknown'): state is JobState => state !== 'failed' && state !== 'unknown';

/**
 * Build the dead-letter row for an ingress delivery whose inline-dispatch fallback failed. The shape
 * mirrors the row IngressProcessor writes on a final-attempt failure (direction / pluginId / instanceId
 * / sessionId / deliveryId / attempts / lastError / payload / redriven), so RedriveService reads either
 * back identically. `attempts` is 1 — the inline path makes a single dispatch attempt (no BullMQ retries).
 */
export function buildIngressDeadLetterRow(data: IngressJobData, error?: string): Partial<IntegrationDeliveryFailure> {
  return {
    direction: 'inbound',
    pluginId: data.pluginId,
    instanceId: data.instanceId,
    sessionId: data.sessionId ?? null,
    deliveryId: data.deliveryId,
    attempts: 1,
    lastError: error ?? 'inline ingress dispatch failed',
    payload: {
      route: data.route,
      method: data.method,
      providerConversationId: data.providerConversationId,
      ingress: data.payload,
    },
    redriven: false,
  };
}

/**
 * Shared queue-or-inline enqueue for inbound ingress jobs. Extracted out of IngressService's DI
 * factory (integration.module.ts) so RedriveService can reuse the exact same behavior when replaying
 * DLQ rows: same queue.add args, same inline dispatch-after-persist fallback, same error swallow.
 * The ingress queue stays @Optional so a queue-off boot (QUEUE_ENABLED unset/false) needs no
 * QueueModule — and no Redis — at all; a missing injection then means inline dispatch, mirroring
 * WebhookService's direct fallback. Under QUEUE_ENABLED=true, IntegrationModule imports QueueModule
 * so the queue provider MUST resolve; onApplicationBootstrap below fails the boot if it did not, so
 * a broken queue wiring crashes at startup instead of silently running inline forever (the queued
 * dispatch contract — fast-ack, retries, ordered processing — would otherwise be dead with no signal).
 */
@Injectable()
export class IngressEnqueueService implements OnApplicationBootstrap {
  private readonly logger = createLogger('IngressEnqueueService');

  constructor(
    private readonly loader: PluginLoaderService,
    private readonly config: ConfigService,
    @Optional() @InjectQueue(QUEUE_NAMES.INGRESS) private readonly ingressQueue?: Queue<IngressJobData>,
  ) {}

  onApplicationBootstrap(): void {
    // Reads the same module-eval signal as the conditional QueueModule imports (integration.module.ts /
    // webhook.module.ts), not the runtime config, so the check guards exactly the wiring that should
    // have happened at import time.
    if (process.env.QUEUE_ENABLED === 'true' && !this.ingressQueue) {
      throw new Error(
        `QUEUE_ENABLED=true but the '${QUEUE_NAMES.INGRESS}' BullMQ queue did not resolve — ` +
          'IntegrationModule must import QueueModule (see the QUEUE_ENABLED conditional in integration.module.ts). ' +
          'Refusing to boot: ingress deliveries would silently dispatch inline, defeating the queued-dispatch contract.',
      );
    }
  }

  /**
   * Whether a delivery of this plugin must wait to be dispatched: it would run inline, because the queue is off,
   * in a plugin the boot restore has not reached, which has no worker to run it in. With the queue on, the job
   * is enqueued instead, and IngressProcessor holds it until the plugin is up.
   */
  defersInlineDispatch(pluginId: string): boolean {
    const queued = this.config.get<boolean>('queue.enabled', false) && !!this.ingressQueue;
    return !queued && this.loader.isRestorePending(pluginId);
  }

  /**
   * State of the job an earlier enqueue() of this delivery left in the queue, or undefined when the
   * queue is off, holds no such job, or cannot answer. add() resolves for a duplicate jobId whatever
   * state the existing job is in, so a caller replaying under the original jobId must look first: a
   * retained failed job (removeOnFail keeps it for a day) would silently swallow the replay. When that
   * job failed or is gone, a live or completed re-queued copy (see requeuedJobIds) answers for it: the
   * copy owns the delivery, so neither a replay nor a dead-letter row may stand in for it.
   */
  async existingJobState(data: IngressJobData, jobId: string): Promise<JobState | undefined> {
    if (!this.config.get<boolean>('queue.enabled', false) || !this.ingressQueue) return undefined;
    try {
      await producerReady(this.ingressQueue);
      const id = queueJobId(data, jobId);
      const state = await this.ingressQueue.getJobState(id);
      if (isLiveOrCompleted(state)) return state;
      for (const copyId of requeuedJobIds(id)) {
        const copyState = await this.ingressQueue.getJobState(copyId);
        if (isLiveOrCompleted(copyState)) return copyState;
      }
      return state === 'unknown' ? undefined : state;
    } catch (err) {
      // Redis unreachable or not yet connected: enqueue() then takes its own inline fallback, as it would
      // without the probe.
      this.logger.warn('Ingress job state lookup failed', {
        pluginId: data.pluginId,
        instanceId: data.instanceId,
        deliveryId: data.deliveryId,
        error: err instanceof Error ? err.message : String(err),
        action: 'ingress_job_state_lookup_failed',
      });
      return undefined;
    }
  }

  /**
   * `signal` lets a caller that stops at shutdown (the reconciler) refuse the inline fallback once it
   * has aborted: a queue add that fails after the stop rethrows the abort reason instead of
   * dispatching, since the plugin may already be torn down. That is the only way enqueue() throws.
   */
  async enqueue(data: IngressJobData, jobId: string, signal?: AbortSignal): Promise<EnqueueOutcome> {
    const queueEnabled = this.config.get<boolean>('queue.enabled', false);
    const useQueue = queueEnabled && !!this.ingressQueue;

    if (useQueue && this.ingressQueue) {
      try {
        // jobId = deliveryId gives BullMQ exactly-once enqueue semantics; the retry policy adds bounded
        // exponential-backoff attempts so a transient failure retries before landing in the DLQ. The id
        // is sanitized because BullMQ refuses several id shapes at add() (see sanitizeIngressJobId),
        // which would otherwise read as a Redis failure here and fall through to inline dispatch.
        await producerReady(this.ingressQueue);
        await this.ingressQueue.add('ingress', data, {
          jobId: queueJobId(data, jobId),
          ...resolveIngressJobOptions(),
        });
        return { outcome: 'queued' };
      } catch (err) {
        // Redis unreachable (enableOfflineQueue:false makes add() reject) — fall through to inline
        // dispatch (before the first connect, producerReady rejects instead). Without this, the
        // already-persisted event would be lost forever: the throw would 500 the ingress request, the
        // provider retries, dedup returns "duplicate", and no job was ever enqueued (no DLQ row either).
        // Mirrors WebhookService's queue-add fallback.
        signal?.throwIfAborted();
        this.logger.error(
          'Ingress queue add failed; dispatching inline',
          err instanceof Error ? err.message : String(err),
          {
            pluginId: data.pluginId,
            instanceId: data.instanceId,
            route: data.route,
            deliveryId: data.deliveryId,
            action: 'ingress_queue_add_failed',
          },
        );
      }
    }
    // Queue disabled OR queue.add() failed: dispatch inline AFTER the ingress_events row was persisted
    // (persist-before-dispatch still holds), mirroring the webhook direct-delivery fallback.
    try {
      await this.loader.dispatchWebhookForInstance(data);
      return { outcome: 'dispatched' };
    } catch (err) {
      // A duplicate delivery is already acked before this point, so a failure here is a real dispatch error.
      // Log and swallow so the provider still gets its 202 (at-least-once, like the webhook fallback).
      // enqueue() intentionally does NOT write a dead-letter row here — it is shared with RedriveService
      // (a failed replay must not spawn a second DLQ row). The 'failed' outcome + error is returned so the
      // caller decides durability: the live-ingress wiring persists a DLQ row (buildIngressDeadLetterRow).
      const error = err instanceof Error ? err.message : String(err);
      this.logger.error('Inline ingress dispatch failed', error, {
        pluginId: data.pluginId,
        instanceId: data.instanceId,
        route: data.route,
        deliveryId: data.deliveryId,
        action: 'ingress_inline_dispatch_failed',
      });
      return { outcome: 'failed', error };
    }
  }
}