Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

17.7 KB
Raw Blame
import type {
  MigrationTables,
  SessionRow,
  WebhookRow,
  MessageRow,
  MessageBatchRow,
  WebhookDeliveryFailureRow,
} from './migration-tables.types';
import { renderTemplate, templateVariableNames } from '../../common/utils/template-render';

/**
 * A `data` value that is a POINTER rather than bytes. `metadata.media.data` holds `base64 || dto.url!`
 * (message.service.ts), and the URL form is the only one the send examples offer — so a URL must never
 * be treated as a payload: dropping it destroys the reference for a few dozen bytes that were never a
 * 413 risk, and reports a `sizeBytes` that is URL text measured as base64, the size of nothing.
 */
function isMediaPointer(data: string): boolean {
  // Case-insensitive to match the two adapters that actually fetch these (`isHttpUrl` in
  // wwebjs-messaging.ts, `resolveMediaBuffer` in baileys-messaging.ts). `@IsUrl()` accepts an
  // uppercase scheme, so a row can hold one; disagreeing with the senders here would destroy a
  // reference to media they had just delivered.
  return /^https?:\/\//i.test(data);
}

/**
 * Webhook credentials (the HMAC secret and custom headers, which may carry receiver tokens) do
 * not belong in a backup payload — omit them from the export. The importer restores the row
 * with secret null and headers {} when they are absent.
 */
function redactWebhookCredentials(rows: WebhookRow[]): void {
  for (const row of rows) {
    delete row.secret;
    delete row.headers;
  }
}

/**
 * A delivery-failure row's `payload` is a short-lived replay copy of the event (a whole message body,
 * cleared after WEBHOOK_FAILURE_PAYLOAD_RETENTION_HOURS). It is not part of the record a backup keeps,
 * and the importer never restores it, so it is left out of the archive rather than carried in it.
 */
function stripWebhookFailurePayload(rows: WebhookDeliveryFailureRow[]): void {
  for (const row of rows) {
    delete row.payload;
  }
}

/**
 * `sessions.proxyUrl` may embed `user:pass` (docs/04 section 4.3), and the sibling webhook secret
 * above is already kept out of this payload. Strip the userinfo and keep the rest of the URL.
 *
 * Dropping the column instead would be worse than it looks: the importer would restore the session
 * with no proxy at all, and it would then connect DIRECT on the next start, leaking the host's real
 * egress IP to WhatsApp with nothing to notice. That is the exact failure mode the engine refuses
 * elsewhere, where an unusable proxy value fails the session rather than silently bypassing it
 * (baileys-lifecycle.ts). A credential-stripped URL keeps the operator's intent visible, restores as
 * `hasCredentials: false` on `GET /proxy`, and fails loudly on the next start when the proxy needs
 * auth, which is the signal to re-enter it.
 *
 * A value the URL parser rejects is cleared rather than passed through, since it cannot be proven
 * free of credentials.
 */
function redactSessionProxyCredentials(rows: SessionRow[]): void {
  for (const row of rows) {
    if (!row.proxyUrl) continue;
    try {
      const parsed = new URL(row.proxyUrl);
      if (!parsed.username && !parsed.password) continue;
      parsed.username = '';
      parsed.password = '';
      row.proxyUrl = parsed.toString();
    } catch {
      row.proxyUrl = null;
    }
  }
}

/**
 * Postgres carries a STORED generated tsvector column `body_ts` (FTS) that `SELECT *` picks up.
 * It is a server-maintained index artifact, not payload: strip it so backups stay dialect-neutral
 * (and small). The import's explicit column list already ignores it in older archives.
 */
function stripBodyTs(rows: MessageRow[]): void {
  for (const row of rows) {
    delete row.body_ts;
  }
}

/**
 * Replace an over-budget inline payload on a message row with the engine's own omitted marker
 * (`{ mimetype, filename?, omitted: true, sizeBytes }`, `capInboundMedia`), so a restored row is
 * indistinguishable from one whose media was skipped on the way in rather than a new shape consumers
 * must learn. `mediaPath`/`mediaMimetype` are untouched.
 *
 * NOTE: this bounds the media, not the export. A text-only history is still unbounded — every row
 * carries a few hundred bytes of scaffolding regardless of what was said.
 */
function stripInlineMediaPayload(row: MessageRow, exceedsBudget: (encodedBytes: number) => boolean): void {
  // `metadata` is TEXT on both dialects and the export reads through a raw query that never hydrates
  // an entity, so the value is always a string here.
  const raw = row.metadata;
  if (typeof raw !== 'string') return;
  let parsed: unknown;
  try {
    parsed = JSON.parse(raw);
  } catch {
    // Metadata we cannot parse is not ours to rewrite; the importer round-trips it verbatim.
    return;
  }
  // `JSON.parse('null')` yields null, and the import accepts a hand-edited archive verbatim, so this
  // is reachable — reading `.media` off it would 500 every export until the row is deleted by hand.
  if (typeof parsed !== 'object' || parsed === null) return;
  const bag = parsed as Record<string, unknown>;
  const media = bag.media as { data?: unknown; sizeBytes?: number } | null | undefined;
  if (!media || typeof media.data !== 'string' || isMediaPointer(media.data)) return;
  if (!exceedsBudget(Buffer.byteLength(media.data, 'utf8'))) return;
  const { data, ...withoutPayload } = media;
  bag.media = {
    ...withoutPayload,
    omitted: true,
    // Decoded bytes, matching what capInboundMedia reports — the caller asked how big it WAS.
    sizeBytes: media.sizeBytes ?? Buffer.byteLength(data, 'base64'),
  };
  row.metadata = JSON.stringify(bag);
}

/**
 * The same budget applied to a bulk batch's stored message list.
 *
 * `message_batches.messages` carries the whole outbound list, base64 included, for the entire
 * duration of a run — `stripBatchMediaPayloads` only fires on the four terminal transitions, and a
 * batch left PROCESSING by another node keeps its payloads indefinitely. Bounding only `messages`
 * would let the 413 arrive by this route instead. The batch shape keeps `url` in its own field, so
 * unlike a message row there is no pointer to confuse with a payload.
 */
function stripBatchInlineMedia(row: MessageBatchRow, exceedsBudget: (encodedBytes: number) => boolean): void {
  const raw = row.messages;
  if (typeof raw !== 'string') return;
  let parsed: unknown;
  try {
    parsed = JSON.parse(raw);
  } catch {
    return;
  }
  if (!Array.isArray(parsed)) return;
  let stripped = false;
  for (const entry of parsed) {
    const content = (entry as { content?: unknown } | null)?.content;
    if (typeof content !== 'object' || content === null) continue;
    // A `base64` may be a placeholder whose payload sits in the item's variables, so those bytes are the field's
    // too. A variable two fields share is stored once: it is charged to the first and dropped only if every
    // field that names it was refused.
    const rawVariables = (entry as { variables?: unknown }).variables;
    const variables =
      rawVariables && typeof rawVariables === 'object' && !Array.isArray(rawVariables)
        ? (rawVariables as Record<string, unknown>)
        : {};
    const kept = new Set<string>();
    const refused = new Set<string>();
    for (const key of ['image', 'video', 'audio', 'document']) {
      const media = (content as Record<string, unknown>)[key] as { base64?: unknown } | null | undefined;
      if (!media || typeof media !== 'object' || typeof media.base64 !== 'string') continue;
      const template = media.base64;
      // One pass over the template with the renderer's own grammar, whatever the number of variables.
      const used = new Set(templateVariableNames(template));
      const named = Object.keys(variables).filter(name => used.has(name) && variables[name] != null);
      const scaffolding = renderTemplate(template, Object.fromEntries(named.map(name => [name, ''])));
      const encodedBytes = named.reduce(
        (bytes, name) => {
          if (kept.has(name)) return bytes;
          const value = variables[name];
          return bytes + Buffer.byteLength(typeof value === 'string' ? value : JSON.stringify(value), 'utf8');
        },
        Buffer.byteLength(scaffolding, 'utf8'),
      );
      if (!exceedsBudget(encodedBytes)) {
        for (const name of named) kept.add(name);
        continue;
      }
      for (const name of named) refused.add(name);
      delete media.base64;
      stripped = true;
    }
    for (const name of refused) {
      if (!kept.has(name)) delete variables[name];
    }
  }
  if (stripped) row.messages = JSON.stringify(parsed);
}

// A per-table export step for InfraDataService.exportData: which backup key the rows land under, the
// physical table to read, whether a genuinely-absent table is tolerable, and the per-table hooks that
// stay explicit because they encode real decisions (credential redaction, dialect artifacts, the
// inline-media budget).
export interface ExportTable<K extends keyof MigrationTables = keyof MigrationTables> {
  /** Backup key in `tables`/`counts` — the published response shape (see MigrationTablesDto). */
  key: K;
  /**
   * Physical table name, exactly as the data DataSource's entity metadata reports it. Every export
   * validates the registry against that metadata BEFORE reading: a table the metadata does not know
   * (renamed or dropped entity) fails the request rather than exporting empty.
   */
  table: string;
  /**
   * The table may legitimately not exist in this DB (created by a migration an older database has
   * not run). Only a GENUINE missing-table error (isMissingTableError) is tolerated, and the table
   * is reported in `skippedTables`. Tables without the flag are required: a missing one fails the
   * export, because a backup that silently omits them is worse than no backup.
   */
  optional?: boolean;
  /** Columns read from SQL, excluding transient payloads before they can be materialized. */
  columns?: readonly string[];
  /**
   * Rows carry an FK `sessionId` to sessions. The reads share no snapshot, so a session created after
   * `sessions` was read can leave child rows here that would fail the restore's FK check and roll the
   * whole import back; the export drops rows whose session is not in the archive.
   */
  sessionFk?: boolean;
  /** In-place row mutation applied right after the read (redaction, artifact stripping). */
  afterRead?: (rows: MigrationTables[K]) => void;
  /**
   * Spends the export's shared inline-media budget on this table's payloads. Such a table is never
   * read whole: the export reads `id`, `recencyColumn` and the stored size of `payloadColumn` for
   * every row, then the full rows in chunks, newest-first, stripping each chunk before reading the next.
   */
  inlineMedia?: {
    /** Which arm of `omittedInlineMedia` reports this table's dropped payloads. */
    bucket: 'messages' | 'messageBatches';
    /** Column `newestFirst` reads. */
    recencyColumn: keyof MigrationTables[K][number] & string;
    /** Column holding the inline payloads; its stored size bounds each chunk of full rows. */
    payloadColumn: keyof MigrationTables[K][number] & string;
    /**
     * Recency key: the budget is spent newest-first, so the most recent media survives. It is called
     * on the key rows, which hold only `id`, `recencyColumn` and the payload size, so it must read
     * nothing but `recencyColumn`.
     */
    newestFirst: (row: MigrationTables[K][number]) => number;
    /** Drops the row's inline payload in place when it does not fit the budget. */
    strip: (row: MigrationTables[K][number], exceedsBudget: (encodedBytes: number) => boolean) => void;
  };
}

/**
 * A registered table with its row type erased, which is what the union-keyed EXPORT_TABLES array
 * holds (the same shape as AnyTableImporter: `never` is the only parameter type every concrete
 * ExportTable<K> can be assigned to, and soundness comes from the export loop only ever handing an
 * entry rows it read for that entry's own table).
 */
export type AnyExportTable = Omit<ExportTable, 'afterRead' | 'inlineMedia'> & {
  afterRead?: (rows: never[]) => void;
  inlineMedia?: {
    bucket: 'messages' | 'messageBatches';
    recencyColumn: string;
    payloadColumn: string;
    newestFirst: (row: never) => number;
    strip: (row: never, exceedsBudget: (encodedBytes: number) => boolean) => void;
  };
};

// Registers one concrete descriptor into the union-keyed EXPORT_TABLES array.
function defineExportTable<K extends keyof MigrationTables>(table: ExportTable<K>): AnyExportTable {
  return table;
}

/**
 * Every data-DB table a backup must carry, in the SAME FK-safe order TABLE_IMPORTERS restores them:
 * sessions first (everything else references it or cascades from it), then the standalone
 * cache/DLQ tables. The export-tables parity spec pins this order to TABLE_IMPORTERS, the table set
 * to the data connection's entity metadata, and the keys to the published counts DTO — so a new
 * entity table cannot silently miss a backup: it must be registered here (with its import descriptor
 * and DTO keys) or excluded below with a reason.
 */
export const EXPORT_TABLES: AnyExportTable[] = [
  // sessions first: webhooks/messages/templates/etc. all reference it (some via FK, all by sessionId).
  defineExportTable({ key: 'sessions', table: 'sessions', afterRead: redactSessionProxyCredentials }),
  defineExportTable({ key: 'webhooks', table: 'webhooks', sessionFk: true, afterRead: redactWebhookCredentials }),

  // Both carry a full inline base64 payload, so they share ONE budget: messages are served first
  // (newest media kept), batches spend what is left. Optional — an older DB may predate them.
  defineExportTable({
    key: 'messages',
    table: 'messages',
    optional: true,
    afterRead: stripBodyTs,
    inlineMedia: {
      bucket: 'messages',
      recencyColumn: 'timestamp',
      payloadColumn: 'metadata',
      newestFirst: (row: MessageRow) => Number(row.timestamp),
      strip: stripInlineMediaPayload,
    },
  }),
  defineExportTable({
    key: 'messageBatches',
    table: 'message_batches',
    optional: true,
    inlineMedia: {
      bucket: 'messageBatches',
      recencyColumn: 'created_at',
      payloadColumn: 'messages',
      newestFirst: (row: MessageBatchRow) => Date.parse(row.created_at),
      strip: stripBatchInlineMedia,
    },
  }),

  // templates + baileys_stored_messages both FK sessions ON DELETE CASCADE, so the import's
  // `DELETE FROM sessions` wipes them; they must be exported and re-inserted or the documented
  // backup flow loses them permanently.
  defineExportTable({ key: 'templates', table: 'templates', optional: true, sessionFk: true }),
  defineExportTable({
    key: 'baileysStoredMessages',
    table: 'baileys_stored_messages',
    optional: true,
    sessionFk: true,
  }),

  // The persisted lid->phone resolution cache. sessionId names the session whose negative may clear the
  // phone; it is not a FK to sessions, so the
  // import's `DELETE FROM sessions` never clears it — it must be exported + re-inserted explicitly
  // or a backup→restore into a fresh DB loses the whole cache (it self-heals, but lossily).
  defineExportTable({ key: 'lidMappings', table: 'lid_mappings', optional: true }),

  // Persisted per-session mute/archive/pin. Like lid_mappings it is not a FK to sessions, so the
  // import's `DELETE FROM sessions` never clears it; WhatsApp does not re-deliver it on reconnect,
  // so a backup that omits it would show a muted chat as unmuted after a restore.
  defineExportTable({ key: 'chatStates', table: 'chat_states', optional: true }),

  // Integration Fabric + both DLQs: none carry an FK constraint to sessions (sessionId is
  // provenance), so the import clears them explicitly before the sessions DELETE to keep the
  // replace-semantics complete.
  defineExportTable({ key: 'pluginInstances', table: 'plugin_instances', optional: true }),
  defineExportTable({ key: 'conversationMappings', table: 'conversation_mappings', optional: true }),
  defineExportTable({ key: 'ingressEvents', table: 'ingress_events', optional: true }),
  defineExportTable({
    key: 'webhookDeliveryFailures',
    table: 'webhook_delivery_failures',
    optional: true,
    columns: [
      'id',
      'webhookId',
      'sessionId',
      'event',
      'url',
      'idempotencyKey',
      'deliveryId',
      'attempts',
      'lastStatusCode',
      'lastError',
      'createdAt',
    ],
    afterRead: stripWebhookFailurePayload,
  }),
  defineExportTable({ key: 'webhookOutboxEvents', table: 'webhook_outbox_events', optional: true }),
  defineExportTable({
    key: 'integrationDeliveryFailures',
    table: 'integration_delivery_failures',
    optional: true,
  }),

  // status_updates has no FK to sessions (plain columns), so the sessions DELETE never clears it —
  // it must be exported + re-inserted explicitly like lid_mappings.
  defineExportTable({ key: 'statusUpdates', table: 'status_updates', optional: true }),

  // automation_rules has an ON DELETE CASCADE FK to sessions, so the sessions DELETE takes every
  // rule with it — exporting and re-inserting it is not optional, or a restore silently destroys
  // every autoreply rule.
  defineExportTable({ key: 'automationRules', table: 'automation_rules', optional: true, sessionFk: true }),
];

/**
 * Data-connection entity tables deliberately absent from the export, each with a one-line reason.
 * The parity spec (and every export, at runtime) fails when an entity table is in neither list, so
 * adding an entity forces an explicit backup decision here rather than a silent miss. Note the
 * TypeORM migration ledger (`migrations`) never appears: it is not an entity, so the connection's
 * entity metadata does not report it.
 */
export const EXPORT_TABLE_EXCLUSIONS: Readonly<Record<string, string>> = {
  // Claims expire within 24 hours; a restored key would only block or replay a send it never saw.
  send_idempotency_keys: 'short-lived Idempotency-Key claims (24 h TTL), meaningless after a restore',
};