Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

21.2 KB
Raw Blame
import { Injectable, OnModuleDestroy, OnModuleInit, Optional } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { ConfigService } from '@nestjs/config';
import { In, IsNull, LessThan, MoreThan, Not, Repository } from 'typeorm';
import { randomUUID } from 'crypto';
import { StatusUpdate } from './entities/status-update.entity';
import type { IncomingStatus } from './incoming-status';
import type { Status } from '../../engine/interfaces/whatsapp-engine.interface';
import { StorageService } from '../../common/storage/storage.service';
import { sweepOrphanedFiles } from '../../common/storage/orphan-sweep';
import { startGuardedInterval } from '../../common/utils/guarded-interval';
import { extFromMimetype } from '../../common/media/mime-ext';
import { isUniqueViolation } from '../../common/utils/db-errors';
import { LidMappingStoreService } from '../../engine/identity/lid-mapping-store.service';
import { parseWaId, userPart } from '../../engine/identity/wa-id';
import { createLogger } from '../../common/services/logger.service';
import { captureSessionWrites, SessionWriteScope } from '../../common/utils/session-write-scope';

/** A status/story lives for 24h from posting, matching WhatsApp's own expiry. Exported for the
 * session service's seed, which skips backfilling statuses that have already run out their TTL. */
export const STATUS_TTL_MS = 24 * 60 * 60 * 1000;
/** How often the TTL purge sweeps expired rows. */
const PURGE_INTERVAL_MS = 15 * 60 * 1000;
/** Rows removed per purge statement. Keeps the id list far below the drivers' bind-parameter
 *  ceilings (SQLite 32766, Postgres 65535) and bounds how much work one failure can undo. */
const PURGE_BATCH_SIZE = 500;
/** Cap on batches per run, so one tick cannot monopolise the process draining a huge backlog;
 *  whatever is left is picked up by the next scheduled run. */
const PURGE_MAX_BATCHES_PER_RUN = 200;
/** Default per-file cap on persisted status media. Exported for the session service's seed, which
 * pre-gates history downloads at the same cap so over-cap blobs are never fetched. */
export const DEFAULT_MEDIA_MAX_BYTES = 10 * 1024 * 1024;
/** Default cadence of the orphaned-media reconciliation sweep (overridable via
 * STATUS_ORPHAN_SWEEP_INTERVAL_MS). Lighter-than-it-sounds: one streamed enumeration + one indexed query. */
const DEFAULT_ORPHAN_SWEEP_INTERVAL_MS = 60 * 60 * 1000;
/** Default grace window before an unreferenced status media file is deleted (overridable via
 * STATUS_ORPHAN_GRACE_MS), so a file mid-ingest is never reaped. */
const DEFAULT_ORPHAN_GRACE_MS = 60 * 60 * 1000;

/** Storage key prefix owned by the status store; the media bucket is shared with chat media. */
const STATUS_MEDIA_PREFIX = 'statuses/';

/**
 * Persists inbound status/story broadcasts (`StatusUpdate` rows) with a 24h TTL, storing any attached
 * media through `StorageService`. Runs two recurring sweeps: the TTL purge (once at startup, then
 * every 15 minutes) so the store never accumulates stories past their WhatsApp-side expiry, and an
 * hourly reconciliation sweep that reaps media files no row references (crash leftovers).
 */
@Injectable()
export class StatusStoreService implements OnModuleInit, OnModuleDestroy {
  private readonly logger = createLogger('StatusStoreService');
  /** Stops the TTL purge and the orphan sweep; set while the service is running. */
  private stopMaintenance: Array<() => void> = [];
  /** First sweep sighting (epoch ms) of each unreferenced status media file, for the grace window. */
  private readonly orphanFirstSeenAt = new Map<string, number>();
  /** Last row id the TTL purge walked past; the next run resumes after it. */
  private purgeCursor?: string;

  constructor(
    @InjectRepository(StatusUpdate, 'data')
    private readonly repository: Repository<StatusUpdate>,
    private readonly storageService: StorageService,
    private readonly configService: ConfigService,
    // Optional, mirroring WebhookService: resolution is a best-effort display concern — without the
    // store, contacts just show under whichever JID the status arrived with.
    @Optional() private readonly lidMappingStore?: LidMappingStoreService,
  ) {}

  onModuleInit(): void {
    const failed = (what: string) => (err: unknown) =>
      this.logger.error(`${what} failed`, err instanceof Error ? err.stack : String(err));
    const sweepIntervalMs = this.configService.get<number>(
      'status.orphanSweepIntervalMs',
      DEFAULT_ORPHAN_SWEEP_INTERVAL_MS,
    );
    this.stopMaintenance = [
      // Once at startup, then on a timer; a pass that outlasts its interval is not stacked on.
      startGuardedInterval(() => this.purgeExpired(Date.now()), PURGE_INTERVAL_MS, failed('Status purge')),
      // The first pass only records first-seen; nothing is deleted before the grace window.
      startGuardedInterval(
        () => this.sweepOrphanedMedia(Date.now()),
        sweepIntervalMs,
        failed('Status media orphan sweep'),
      ),
    ];
  }

  onModuleDestroy(): void {
    for (const stop of this.stopMaintenance) stop();
    this.stopMaintenance = [];
  }

  /**
   * Insert a status row (idempotent on `(sessionId, waStatusId)`), persisting any attached media.
   * `created` tells the caller whether this call actually inserted the row: false for a duplicate
   * delivery, a lost insert race, or a row removed while its media was written, so a once-per-status
   * side effect (the status.received webhook) doesn't fire again for a status consumers already saw.
   *
   * Row-first ordering: the row is saved BEFORE its media file is written. A crash mid-ingest then
   * leaves a consistent media-omitted row behind (the worst case is a missing attachment), never a
   * permanent orphan file; the narrow window between the file write and the row update is the only
   * remaining orphan source, and the reconciliation sweep reaps those.
   */
  async ingest(
    sessionId: string,
    s: IncomingStatus,
    lifetime?: SessionWriteScope,
  ): Promise<{ row: StatusUpdate; created: boolean }> {
    const scope = lifetime ?? this.captureWrites(sessionId);
    try {
      const existing = await this.repository.findOne({ where: { sessionId, waStatusId: s.waStatusId } });
      if (existing) return { row: existing, created: false };

      const row = new StatusUpdate();
      row.sessionId = sessionId;
      row.contactJid = s.contactJid;
      row.contactName = s.contactName;
      row.contactPushName = s.contactPushName;
      row.waStatusId = s.waStatusId;
      row.type = s.type;
      row.caption = s.caption;
      row.backgroundColor = s.backgroundColor;
      row.font = s.font;
      row.postedAt = s.postedAt;
      row.expiresAt = s.postedAt + STATUS_TTL_MS;
      // Settles every media outcome that does NOT need a file write; a keepable blob stays
      // omitted-for-now until attachMedia lands the file and flips it (post-save).
      const keepMedia = this.applyMediaDecision(row, s);

      let saved: StatusUpdate;
      try {
        // Without a transaction: a single-row save() wraps itself in one, and on SQLite's single shared
        // connection the rollback of the unique violation expected below also discards the rows other callers
        // wrote while it was open, after they were reported stored.
        saved = await scope.write(() => this.repository.save(row, { transaction: false }));
      } catch (error) {
        // The unique (sessionId, waStatusId) index is the idempotency backstop for a concurrent ingest
        // of the same status racing past the findOne check above; on a unique-constraint violation the
        // loser re-reads and returns the row the winner just inserted instead of surfacing the
        // constraint error to the caller. Any OTHER save error is a genuine persistence failure and
        // must propagate — never be masked by a coincidentally matching row. No media file has been
        // written at this point (row-first ordering), so unlike a file-first ingest there is nothing
        // to reap here — the winner's own call owns its file write.
        const winner = await this.repository.findOne({ where: { sessionId, waStatusId: s.waStatusId } });
        if (winner && isUniqueViolation(error)) return { row: winner, created: false };
        throw error;
      }

      // A row removed while its media was being written (a purge, a session delete) was never stored.
      if (keepMedia && !(await this.attachMedia(saved, sessionId, s))) return { row: saved, created: false };
      return { row: saved, created: true };
    } finally {
      if (!lifetime) scope.close();
    }
  }

  captureWrites(sessionId: string): SessionWriteScope {
    return captureSessionWrites(this.repository.manager?.connection ?? this.repository, sessionId);
  }

  /**
   * Settle the row's mediaOmitted/omitReason fields for every outcome that does not require a file
   * write (no media, engine-omitted, over the store cap). Returns true when the blob should be
   * written post-save; the row is then saved as omitted-for-now (no omitReason) and attachMedia
   * upgrades it once the file exists.
   */
  private applyMediaDecision(row: StatusUpdate, s: IncomingStatus): boolean {
    const media = s.media;
    if (!media) {
      row.mediaOmitted = false;
      return false;
    }

    const maxBytes = this.configService.get<number>('status.mediaMaxBytes', DEFAULT_MEDIA_MAX_BYTES);
    const sizeBytes = media.sizeBytes ?? (media.data ? Buffer.byteLength(media.data, 'base64') : undefined);
    const withinCap = sizeBytes !== undefined && sizeBytes <= maxBytes;

    if (!media.omitted && media.data && withinCap) {
      row.mediaOmitted = true; // pending the post-save write; attachMedia flips it back off
      return true;
    }

    row.mediaOmitted = true;
    // A blob the engine skipped precisely because the caller tightened the cap below it (the status
    // seed's pre-gate) is over the STORE's cap — keep the reason truthful on both arrival paths.
    row.omitReason =
      sizeBytes !== undefined && sizeBytes > maxBytes ? 'over_cap' : media.omitted ? 'engine_omitted' : 'over_cap';
    return false;
  }

  /**
   * Write the media file for a row already saved in the omitted-for-now state, then point the row at
   * it. A failed write is recorded as omitReason 'write_failed' rather than thrown — a storage
   * hiccup must not lose the status itself. A failure of the follow-up row update leaves the file
   * unreferenced; that is the reconciliation sweep's job, so it is logged and swallowed here. Returns
   * false when the row no longer exists: the file is removed and nothing was published.
   */
  private async attachMedia(row: StatusUpdate, sessionId: string, s: IncomingStatus): Promise<boolean> {
    const media = s.media;
    if (!media?.data) return true;

    const key = `${STATUS_MEDIA_PREFIX}${sessionId}/${randomUUID()}.${extFromMimetype(media.mimetype)}`;
    try {
      await this.storageService.putFile(key, Buffer.from(media.data, 'base64'));
    } catch (error) {
      this.logger.error(
        `Failed to persist status media for session ${sessionId}, status ${s.waStatusId}`,
        error instanceof Error ? error.stack : String(error),
      );
      row.omitReason = 'write_failed';
      await this.repository.update({ id: row.id, sessionId }, { omitReason: row.omitReason }).catch(err =>
        this.logger.warn(`Failed to record the write_failed omission for status ${s.waStatusId}`, {
          error: String(err),
        }),
      );
      return true;
    }

    row.mediaPath = key;
    row.mediaMimetype = media.mimetype;
    row.mediaOmitted = false;
    try {
      const result = await this.repository.update(
        { id: row.id, sessionId },
        {
          mediaPath: row.mediaPath,
          mediaMimetype: row.mediaMimetype,
          mediaOmitted: false,
        },
      );
      if (result.affected === 0) {
        row.mediaPath = undefined;
        row.mediaMimetype = undefined;
        row.mediaOmitted = true;
        await this.storageService.deleteFile(key).catch(() => undefined); // else the orphan sweep reaps it
        return false;
      }
    } catch (error) {
      // The file exists but the row cannot reference it — an orphan the reconciliation sweep reaps
      // after its grace window. The persisted row stays consistent (still omitted from the first
      // save), so don't fail the ingest; restore the matching in-memory state too.
      this.logger.warn(`Status media ${key} written but the row update failed; leaving it for the orphan sweep`, {
        error: String(error),
      });
      row.mediaPath = undefined;
      row.mediaMimetype = undefined;
      row.mediaOmitted = true;
      // This same object is what the status.received webhook carries, and every other omission path
      // names its reason ('engine_omitted' / 'over_cap' / 'write_failed'). Leaving it unset here
      // would hand a consumer a media-less status it cannot tell apart from an unexplained
      // omission — the media really is gone (nothing retries), so say so.
      row.omitReason = 'write_failed';
    }
    return true;
  }

  // Reads exclude already-expired rows: WhatsApp hides a status the moment its 24h are up, but the
  // purge sweep only reaps rows every 15 minutes — without the filter an expired status would stay
  // visible in the gap.
  async list(sessionId: string): Promise<Status[]> {
    const rows = await this.repository.find({
      where: { sessionId, expiresAt: MoreThan(Date.now()) },
      order: { postedAt: 'DESC' },
    });
    return rows.map(row => this.toStatus(row));
  }

  async listByContact(sessionId: string, contactJid: string): Promise<Status[]> {
    // The same person may hold rows under both a @lid and their @c.us (the mapping was learned
    // mid-window) — match every candidate so a resolved-form query doesn't silently miss lid rows.
    const candidates = new Set<string>([contactJid]);
    if (this.lidMappingStore) {
      // parseWaId classifies the input, so an upper-case or hosted.lid input is a lid too.
      const parsed = parseWaId(contactJid);
      if (parsed.kind === 'lid') {
        // A lid's digits are not a phone (the map's keys are raw digits, so a phone could collide
        // with a lid): add the folded `<lid>@lid` form it is stored under and forward-resolve it, so
        // rows ingested after the mapping was learned (stored under @c.us) still match.
        candidates.add(`${parsed.userPart}@lid`);
        const resolved = await this.lidMappingStore.findPhoneForLid(parsed.userPart);
        if (resolved) candidates.add(`${resolved}@c.us`);
      } else {
        const phone = userPart(contactJid);
        candidates.add(`${phone}@c.us`);
        for (const lid of await this.lidMappingStore.findLidsForPhone(phone)) candidates.add(`${lid}@lid`);
      }
    }
    const rows = await this.repository.find({
      where: { sessionId, contactJid: In([...candidates]), expiresAt: MoreThan(Date.now()) },
      order: { postedAt: 'DESC' },
    });
    return rows.map(row => this.toStatus(row));
  }

  /**
   * Canonical display form of a contact JID: resolve a @lid to its phone via the shared mapping.
   * Read-time (not ingest-time) on purpose: a mapping learned after the status arrived still merges
   * the contact's rows into one group. Only @lid inputs go through the map — its keys are raw digit
   * strings, so resolving a phone-shaped JID could collide with an unrelated lid. Unknown or
   * known-unresolved lids stay as-is.
   */
  private canonicalContactJid(jid: string): string {
    if (!jid.endsWith('@lid')) return jid;
    const phone = this.lidMappingStore?.resolveLid(jid);
    return phone ? `${phone}@c.us` : jid;
  }

  // The nullable columns read back as null, while Status declares these fields optional: `?? undefined`
  // keeps them off the wire, as the contract (StatusDto, openapi) says.
  private toStatus(row: StatusUpdate): Status {
    return {
      id: row.waStatusId,
      contact: {
        id: this.canonicalContactJid(row.contactJid),
        name: row.contactName ?? undefined,
        pushName: row.contactPushName ?? undefined,
      },
      type: row.type,
      caption: row.caption ?? undefined,
      mediaUrl:
        row.mediaPath && !row.mediaOmitted
          ? `/api/sessions/${row.sessionId}/status/${row.waStatusId}/media`
          : undefined,
      backgroundColor: row.backgroundColor ?? undefined,
      font: row.font ?? undefined,
      timestamp: new Date(row.postedAt),
      expiresAt: new Date(row.expiresAt),
    };
  }

  async getMedia(sessionId: string, statusId: string): Promise<{ path: string; mimetype: string } | null> {
    const row = await this.repository.findOne({
      where: { sessionId, waStatusId: statusId, expiresAt: MoreThan(Date.now()) },
    });
    // Same condition toStatus() advertises a mediaUrl on. Media stored without a type (an engine that
    // reported none) is still served, as octet-stream.
    if (!row || row.mediaOmitted || !row.mediaPath) return null;
    return { path: row.mediaPath, mimetype: row.mediaMimetype || 'application/octet-stream' };
  }

  /**
   * Deletes rows (and their media files) whose `expiresAt` is before `now`. Returns the count removed.
   * A row whose media delete FAILS is kept — deleting it would orphan the file permanently (a row-less
   * file is only rediscovered by the orphan sweep after its grace window), so the row stays and the
   * next sweep retries the delete. A missing file is a successful delete (see StorageService), so an
   * already-gone file never wedges its row.
   *
   * Drained in bounded batches. The 24h TTL does not bound a backlog: a restart after downtime, or
   * media deletes that keep failing, leave any number of expired rows, and a single
   * `DELETE ... WHERE id IN (...)` past the driver's bind-parameter ceiling throws after the files are
   * gone, then fails the same way on every later run. The batches walk the expired rows by id, resuming
   * across runs, so rows whose delete keeps failing are stepped over instead of blocking newer ones. A
   * batch in which every delete fails ends the run: that means the store is down, and walking on would
   * only repeat the failure.
   */
  async purgeExpired(now: number): Promise<number> {
    let removed = 0;
    let after = this.purgeCursor;
    this.purgeCursor = undefined;
    for (let batch = 0; batch < PURGE_MAX_BATCHES_PER_RUN; batch++) {
      const expired = await this.repository.find({
        where: { expiresAt: LessThan(now), ...(after ? { id: MoreThan(after) } : {}) },
        select: { id: true, mediaPath: true },
        order: { id: 'ASC' },
        take: PURGE_BATCH_SIZE,
      });
      if (expired.length === 0) break;
      after = expired[expired.length - 1].id;

      const deletableIds: string[] = [];
      for (const row of expired) {
        try {
          if (row.mediaPath) await this.storageService.deleteFile(row.mediaPath);
          deletableIds.push(row.id);
        } catch (err) {
          this.logger.warn(
            `Failed to delete expired status media ${row.mediaPath}; keeping the row for the next sweep`,
            { error: String(err) },
          );
        }
      }

      // Rows whose delete failed are kept; they are retried once the walk wraps around.
      if (deletableIds.length === 0) {
        this.purgeCursor = after;
        break;
      }
      const result = await this.repository.delete(deletableIds);
      removed += result.affected ?? deletableIds.length;
      // A short batch means the backlog is drained and the next run starts over from the lowest id;
      // a run that hits the batch cap resumes where it stopped.
      if (expired.length < PURGE_BATCH_SIZE) break;
      if (batch === PURGE_MAX_BATCHES_PER_RUN - 1) this.purgeCursor = after;
    }
    return removed;
  }

  /**
   * Reconcile the media store with the status rows: delete `statuses/` files no row references.
   * Orphans arise from the narrow crash window between the media write and its row update (row-first
   * ingest keeps every other path consistent). Scoped to the statuses/ prefix — the media store is
   * shared with chat media, which this sweep must never touch.
   *
   * A file is deleted only after the sweep has seen it unreferenced for at least the grace window:
   * first-seen is tracked in memory (a restart simply restarts the grace clock, which fails safe),
   * so a file mid-ingest or freshly written is never reaped. Returns the count removed.
   */
  async sweepOrphanedMedia(now: number = Date.now()): Promise<number> {
    const graceMs = this.configService.get<number>('status.orphanGraceMs', DEFAULT_ORPHAN_GRACE_MS);
    // The referenced set is bounded by the 24h TTL, so a single whole-set query is the cheap shape
    // here (no chunking) — unlike the chat-media archive, whose rows can accumulate without bound.
    const removed = await sweepOrphanedFiles({
      storage: this.storageService,
      prefix: STATUS_MEDIA_PREFIX,
      graceMs,
      now,
      firstSeenAt: this.orphanFirstSeenAt,
      referencedAmong: async () => {
        const rows = await this.repository.find({
          where: { mediaPath: Not(IsNull()) },
          select: { mediaPath: true },
        });
        return new Set(rows.map(row => row.mediaPath));
      },
      onDeleteFailed: (file, err) =>
        this.logger.warn(`Failed to delete orphaned status media ${file}`, { error: String(err) }),
    });
    if (removed > 0) this.logger.log(`Status media orphan sweep removed ${removed} file(s)`);
    return removed;
  }
}