Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

18.3 KB
Raw Blame
import { Processor, WorkerHost, OnWorkerEvent } from '@nestjs/bullmq';
import { DelayedError, Job } from 'bullmq';
import { InjectRepository } from '@nestjs/typeorm';
import { ConfigService } from '@nestjs/config';
import { Repository } from 'typeorm';
import { createLogger } from '../../../common/services/logger.service';
import { QUEUE_NAMES } from '../queue-names';
import { workerConnectionOptions, webhookWorkerConcurrency } from '../redis-connection';
import { closeWorkerIfStarted } from './close-worker';
import { WebhookJobData, WebhookPayload } from '../../webhook/webhook.service';
import { Webhook } from '../../webhook/entities/webhook.entity';
import { WebhookOutboxService } from '../../webhook/webhook-outbox.service';
import { WebhookDeliveryFailure } from '../../webhook/entities/webhook-delivery-failure.entity';
import {
  clearDeliveryFailureRows,
  recordWebhookDeliveryFailure,
  statusCodeFromError,
} from '../../webhook/utils/record-delivery-failure';
import { buildDeliveryHeaders, isDeliverableWebhook, postWebhookPayload } from '../../webhook/utils/deliver-once';
import { HookManager } from '../../../core/hooks';
import { redactSsrfError } from '../../../common/security/ssrf-guard';
import { incrementWebhookDeliveryFailures } from '../../../common/metrics/webhook-delivery-metrics';

export interface WebhookJobResult {
  statusCode: number;
  success: boolean;
  error?: string;
  responseTime: number;
}

/**
 * The exact `failedReason` BullMQ 6.x sets when a job stalls more than `maxStalledCount` (worker
 * default 1, so the SECOND genuine stall): the stalled checker (moveStalledJobsToWait Lua script)
 * stores it as the job's deferred failure, and the worker then fails the job itself — emitting
 * 'failed' WITHOUT ever calling process(). Lock renewal means a slow-but-alive processor never
 * stalls, so reaching this sentinel implies the job genuinely died twice mid-processing.
 */
const STALL_EXHAUSTION_MESSAGE = 'job stalled more than allowable limit';

/** Per-attempt delivery context threaded through the process() pipeline stages (was closure state). */
interface WebhookDeliveryContext {
  job: Job<WebhookJobData>;
  webhookId: string;
  url: string;
  event: string;
  payload: WebhookPayload;
  maxRetries: number;
  sessionId: string;
  startTime: number;
}

// Override the Worker's connection so it does NOT inherit the producer's `enableOfflineQueue: false`
// from the shared BullModule connection — the Worker must tolerate a brief Redis reconnect. Set an
// explicit concurrency: BullMQ defaults a Worker to 1, which serializes every session's webhook
// deliveries behind one slow/timing-out receiver.
@Processor(QUEUE_NAMES.WEBHOOK, { connection: workerConnectionOptions(), concurrency: webhookWorkerConcurrency() })
export class WebhookProcessor extends WorkerHost {
  private readonly logger = createLogger('WebhookProcessor');
  /**
   * Webhooks whose last attempt on this worker failed, until one succeeds. A job for one of them
   * takes a per-session slot first, at most `degradedSessionConcurrency` per session; over that it
   * goes back to the delayed set without spending an attempt. One session's failing receivers then
   * cannot fill the shared pool, while healthy receivers are never held back.
   */
  private readonly failingWebhooks = new Set<string>();
  private readonly degradedInFlight = new Map<string, number>();
  private readonly degradedSessionConcurrency: number;

  constructor(
    @InjectRepository(Webhook, 'data')
    private readonly webhookRepository: Repository<Webhook>,
    @InjectRepository(WebhookDeliveryFailure, 'data')
    private readonly failureRepository: Repository<WebhookDeliveryFailure>,
    private readonly hookManager: HookManager,
    private readonly configService: ConfigService,
    private readonly outbox: WebhookOutboxService,
  ) {
    super();
    // WEBHOOK_DEGRADED_SESSION_CONCURRENCY, else a quarter of the worker pool.
    this.degradedSessionConcurrency =
      this.configService.get<number | undefined>('webhook.degradedSessionConcurrency') ??
      Math.max(1, Math.floor(webhookWorkerConcurrency() / 4));
  }

  // Stop taking jobs before the plugins are torn down, so a final attempt's webhook:error hook still
  // reaches them; see IngressProcessor.onModuleDestroy. The wait is min(WEBHOOK_TIMEOUT + 5 s, 10 s), so it
  // covers one POST plus its bookkeeping only while WEBHOOK_TIMEOUT is 5 s or less; a job still active at the
  // cap is re-run after the next start.
  async onModuleDestroy(): Promise<void> {
    await closeWorkerIfStarted(this, this.configService.get<number>('webhook.timeout', 10000) + 5000);
  }

  async process(job: Job<WebhookJobData>, token?: string): Promise<WebhookJobResult> {
    const { webhookId, payload } = job.data;
    const sessionId = payload.sessionId;
    const gated = this.failingWebhooks.has(webhookId);
    if (gated) {
      const inFlight = this.degradedInFlight.get(sessionId) ?? 0;
      if (inFlight >= this.degradedSessionConcurrency) {
        // Before the try below on purpose: this is not an attempt, so it must never be logged as a
        // failure or file a dead-letter row. moveToDelayed does not spend one of the job's attempts.
        // At least a second: a job delayed to "now" is promoted straight back and would spin.
        const base = Math.max(1000, this.configService.get<number>('webhook.retryDelay', 5000));
        // Each bounce doubles the job's wait, up to 64x, so a dead receiver's backlog is not promoted
        // and bounced again every few seconds, a churn that would grow with the backlog. attemptsStarted
        // counts every activation and attemptsMade only real attempts, so the gap, less this
        // activation, is how many times this job was already bounced.
        const bounces = Math.max(0, job.attemptsStarted - job.attemptsMade - 1);
        const delay = base * 2 ** Math.min(bounces, 6);
        await job.moveToDelayed(Date.now() + delay + Math.floor(Math.random() * delay), token);
        throw new DelayedError();
      }
      this.degradedInFlight.set(sessionId, inFlight + 1);
    }
    try {
      return await this.deliver(job);
    } finally {
      if (gated) {
        const left = (this.degradedInFlight.get(sessionId) ?? 1) - 1;
        if (left > 0) this.degradedInFlight.set(sessionId, left);
        else this.degradedInFlight.delete(sessionId);
      }
    }
  }

  private async deliver(job: Job<WebhookJobData>): Promise<WebhookJobResult> {
    const { webhookId, event, payload, maxRetries } = job.data;
    const startTime = Date.now();
    const sessionId = payload.sessionId;

    this.logger.log(`Processing webhook job ${job.id}`, {
      webhookId,
      event,
      deliveryId: payload.deliveryId,
      idempotencyKey: payload.idempotencyKey,
      attempt: job.attemptsMade + 1,
      action: 'webhook_process_start',
    });

    const ctx: WebhookDeliveryContext = {
      job,
      webhookId,
      url: job.data.url,
      event,
      payload,
      maxRetries,
      sessionId,
      startTime,
    };

    try {
      // The job carries a snapshot taken at enqueue time. Re-read the row before every attempt: a
      // webhook that was deleted, disabled or unsubscribed from this event since then must not
      // receive it (the reconciler applies the same test; neither re-applies the webhook's filters,
      // which need the event data). Completing the job instead of throwing
      // stops the retries and files no dead-letter row. Otherwise deliver with the CURRENT url,
      // headers and secret, as the reconciler's replay does, so a receiver move or a rotated
      // secret or auth header applies to jobs already waiting in the queue.
      // A read error lands in the catch below and counts as a failed attempt, like a failed POST.
      const current = await this.loadDeliverableWebhook(webhookId, event, sessionId);
      if (!current) {
        this.logger.warn('Skipping queued webhook delivery: webhook removed, disabled or unsubscribed', {
          webhookId,
          event,
          deliveryId: payload.deliveryId,
          idempotencyKey: payload.idempotencyKey,
          action: 'webhook_skipped_stale',
        });
        this.failingWebhooks.delete(webhookId);
        await this.outbox.close(webhookId, payload.idempotencyKey, 'dispatched');
        return { statusCode: 0, success: false, error: 'webhook removed, disabled or unsubscribed', responseTime: 0 };
      }
      ctx.url = current.url;
      const body = JSON.stringify(payload);
      const requestHeaders = buildDeliveryHeaders(
        current,
        event,
        payload.idempotencyKey,
        payload.deliveryId,
        body,
        job.attemptsMade,
      );

      const { status, responseTime } = await this.postToReceiver(ctx, body, requestHeaders);
      this.failingWebhooks.delete(webhookId);
      await this.outbox.close(webhookId, payload.idempotencyKey, 'dispatched');
      await this.recordSuccessfulDelivery(ctx, status, responseTime);
      return {
        statusCode: status,
        success: true,
        responseTime,
      };
    } catch (error) {
      this.failingWebhooks.add(webhookId);
      await this.recordDeliveryFailure(ctx, error);
      // Re-throw to trigger BullMQ retry
      throw error;
    }
  }

  /** The webhook row as it is now, or null when it was deleted, disabled or no longer takes `event`. */
  private async loadDeliverableWebhook(webhookId: string, event: string, sessionId: string): Promise<Webhook | null> {
    const row = await this.webhookRepository.findOne({ where: { id: webhookId } });
    return isDeliverableWebhook(row, event, sessionId) ? row : null;
  }

  /**
   * POST the payload to the receiver through the SSRF-guarded fetch and classify the response:
   * a non-ok status throws into the failure path. Returns the status and measured response time.
   */
  private async postToReceiver(
    ctx: WebhookDeliveryContext,
    body: string,
    requestHeaders: Record<string, string>,
  ): Promise<{ status: number; responseTime: number }> {
    const { url, startTime } = ctx;
    const { status } = await postWebhookPayload(
      url,
      body,
      requestHeaders,
      // Honor WEBHOOK_TIMEOUT on the queued path too, as the direct path does.
      this.configService.get<number>('webhook.timeout', 10000),
    );

    const responseTime = Date.now() - startTime;
    return { status, responseTime };
  }

  /**
   * Post-delivery bookkeeping for a 2xx answer: guarded lastTriggeredAt update, the
   * webhook:delivered hook, and the success log.
   */
  private async recordSuccessfulDelivery(
    ctx: WebhookDeliveryContext,
    status: number,
    responseTime: number,
  ): Promise<void> {
    const { job, webhookId, event, payload, sessionId } = ctx;
    // The receiver already answered 2xx — the delivery SUCCEEDED. Everything up to the return is
    // bookkeeping and must never throw back into the failure path: a rethrow would make BullMQ
    // retry (a duplicate POST for an already-delivered event) and, on the final attempt, file a
    // false dead-letter row. Log a bookkeeping failure and keep the success outcome.
    try {
      await this.webhookRepository.update(webhookId, {
        lastTriggeredAt: new Date(),
      });
    } catch (bookkeepingError) {
      this.logger.error(
        'Webhook delivered but lastTriggeredAt update failed',
        bookkeepingError instanceof Error ? bookkeepingError.message : String(bookkeepingError),
        { webhookId, deliveryId: payload.deliveryId, action: 'webhook_bookkeeping_failed' },
      );
    }

    // A delivered event must not stay listed as lost. A failure row exists for it only when an
    // earlier dispatch of the same delivery was shed, refused or failed before this job ran. An
    // indexed delete that usually matches nothing.
    await clearDeliveryFailureRows(this.failureRepository, this.logger, webhookId, payload.idempotencyKey);

    // Execute hook after successful delivery
    await this.hookManager.execute(
      'webhook:delivered',
      {
        sessionId,
        event,
        webhookId,
        deliveryId: payload.deliveryId,
        statusCode: status,
        responseTime,
        attempt: job.attemptsMade + 1,
      },
      { sessionId, source: 'WebhookProcessor' },
    );

    this.logger.log(`Webhook delivered successfully`, {
      webhookId,
      event,
      deliveryId: payload.deliveryId,
      idempotencyKey: payload.idempotencyKey,
      statusCode: status,
      responseTime,
      attempt: job.attemptsMade + 1,
      action: 'webhook_delivered',
    });
  }

  /**
   * Failure-path bookkeeping: log the delivery failure and, on the final attempt, fire the
   * webhook:error hook, persist the durable dead-letter row, and bump the failures metric.
   */
  private async recordDeliveryFailure(ctx: WebhookDeliveryContext, error: unknown): Promise<void> {
    const { job, webhookId, url, event, payload, maxRetries, sessionId, startTime } = ctx;
    const responseTime = Date.now() - startTime;
    const errorMessage = error instanceof Error ? error.message : String(error);
    const isFinalAttempt = job.attemptsMade + 1 >= maxRetries;

    this.logger.error(`Webhook delivery failed`, errorMessage, {
      webhookId,
      event,
      deliveryId: payload.deliveryId,
      idempotencyKey: payload.idempotencyKey,
      responseTime,
      attempt: job.attemptsMade + 1,
      maxRetries,
      isFinalAttempt,
      action: 'webhook_failed',
    });

    // On final failure (all retries exhausted): fire the error hook AND persist a durable record so
    // the lost event is visible after the BullMQ failed-set / logs roll off.
    if (isFinalAttempt) {
      // The hook payload and the durable row are surfaced to operators/plugins — redact SSRF detail
      // (resolved internal IP) from the client-facing message. The full `errorMessage` is already
      // logged server-side above; statusCodeFromError never matches an SSRF block (matches ^HTTP \d{3}).
      const clientError = redactSsrfError(error);
      await this.hookManager.execute(
        'webhook:error',
        {
          sessionId,
          event,
          webhookId,
          deliveryId: payload.deliveryId,
          error: clientError,
          attempt: job.attemptsMade + 1,
        },
        { sessionId, source: 'WebhookProcessor' },
      );
      const recorded = await recordWebhookDeliveryFailure(this.failureRepository, this.logger, {
        webhookId,
        sessionId,
        event,
        url,
        idempotencyKey: payload.idempotencyKey,
        deliveryId: payload.deliveryId,
        attempts: job.attemptsMade + 1,
        lastStatusCode: statusCodeFromError(errorMessage),
        lastError: clientError,
        // The producer attaches the pre-hook event data only while payload retention is on.
        ...(this.configService.get<number>('webhook.failurePayloadRetentionHours', 0) > 0 && job.data.replayData
          ? { payload: job.data.replayData }
          : {}),
      });
      if (recorded !== false) incrementWebhookDeliveryFailures();
      if (recorded !== null) await this.outbox.close(webhookId, payload.idempotencyKey, 'dispatched');
    }
  }

  /**
   * A job failed by stall exhaustion never enters process(): the worker fails it internally after
   * the second stall and only emits 'failed'. Without this handler such a job bypasses every product
   * failure channel — no dead-letter row, no metric, no webhook:error hook. Normal delivery failures
   * are already recorded by process() on the final attempt (and non-final ones are retried), so this
   * handler MUST ignore anything but the stall-exhaustion sentinel, or every failure is recorded
   * twice. `job` can be undefined when the queue's bounded `removeOnFail` window (see QueueModule's
   * WEBHOOK_QUEUE_JOB_OPTIONS) pruned it before this event fired — that window keeps failed-job
   * retention bounded, and each retained payload was size-gated before enqueue.
   */
  @OnWorkerEvent('failed')
  async onWorkerFailed(job: Job<WebhookJobData> | undefined, error: Error): Promise<void> {
    // The sentinel text alone is not proof: a failed attempt whose error reads the same is retried and is not
    // this. BullMQ marks a job it fails for stalling by storing the reason as its deferred failure.
    if (!job || job.deferredFailure !== STALL_EXHAUSTION_MESSAGE || error.message !== STALL_EXHAUSTION_MESSAGE) {
      return;
    }

    const { webhookId, event, payload } = job.data;
    const sessionId = payload.sessionId;

    // Same rule as process(): no dead-letter row or webhook:error for a webhook that is gone, disabled
    // or unsubscribed, and the row records the URL a retry would have used. If the read itself fails,
    // record against the enqueue-time snapshot rather than lose the failure.
    let url = job.data.url;
    try {
      const current = await this.loadDeliverableWebhook(webhookId, event, sessionId);
      if (!current) {
        return;
      }
      url = current.url;
    } catch (readError) {
      this.logger.warn('Could not re-read webhook for a stalled job; recording the enqueue-time URL', {
        webhookId,
        error: readError instanceof Error ? readError.message : String(readError),
      });
    }

    this.logger.error('Webhook job failed after stalling beyond the recovery limit', error.message, {
      webhookId,
      event,
      deliveryId: payload.deliveryId,
      idempotencyKey: payload.idempotencyKey,
      attemptsMade: job.attemptsMade,
      action: 'webhook_stall_exhausted',
    });

    await this.hookManager.execute(
      'webhook:error',
      {
        sessionId,
        event,
        webhookId,
        deliveryId: payload.deliveryId,
        error: error.message,
        attempt: job.attemptsMade,
      },
      { sessionId, source: 'WebhookProcessor' },
    );

    const recorded = await recordWebhookDeliveryFailure(this.failureRepository, this.logger, {
      webhookId,
      sessionId,
      event,
      url,
      idempotencyKey: payload.idempotencyKey,
      deliveryId: payload.deliveryId,
      attempts: Math.max(1, job.attemptsMade),
      lastStatusCode: null, // no HTTP exchange completed on the stalled attempts
      lastError: error.message,
      ...(this.configService.get<number>('webhook.failurePayloadRetentionHours', 0) > 0 && job.data.replayData
        ? { payload: job.data.replayData }
        : {}),
    });
    if (recorded !== false) incrementWebhookDeliveryFailures();
    if (recorded !== null) await this.outbox.close(webhookId, payload.idempotencyKey, 'dispatched');
  }
}