Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

16.1 KB
Raw Blame
import * as tar from 'tar-stream';
import { createGunzip, createGzip } from 'zlib';
import { Readable, PassThrough, Transform, pipeline } from 'stream';
import { LoggerService } from '../services/logger.service';
import { positiveIntFromEnv } from '../utils/env-int';

/** Per-entry buffer cap for an import (200 MiB — 4× the inbound media cap). Bounds a decompression bomb. */
export const DEFAULT_IMPORT_MAX_BYTES = 200 * 1024 * 1024;
/** Max number of entries an import archive may contain. Bounds an entry-count DoS. */
export const DEFAULT_IMPORT_MAX_ENTRIES = 100_000;
/** Cap on the bytes one import extracts in total (10 GiB). The two caps above still allow terabytes. */
export const DEFAULT_IMPORT_MAX_TOTAL_BYTES = 10 * 1024 * 1024 * 1024;

/**
 * True when a storage read failed because the object is simply not there.
 *
 * Both backends must be covered, and they report it differently: the local backend raises a POSIX
 * `ENOENT` (a `.code`), while S3 raises `NoSuchKey`/`NotFound`, which carries a `.name` and no
 * `.code` at all; `StorageService.getS3File` rethrows that original error when the local read-through
 * also misses. Checking only `.code` turns a missing S3 object into a 500 on the one backend where
 * retention and bucket lifecycle rules make a miss most likely.
 */
export function isMissingObjectError(error: unknown): boolean {
  const e = error as { code?: string; name?: string; $metadata?: { httpStatusCode?: number } };
  return (
    e?.code === 'ENOENT' || e?.name === 'NoSuchKey' || e?.name === 'NotFound' || e?.$metadata?.httpStatusCode === 404
  );
}

// ============================================================================
// Export - Create tar.gz stream from current storage
// ============================================================================

/**
 * A file opened for export. `size` is the byte length the tar header declares; without it the entry
 * has to be buffered to learn its length.
 */
export interface ExportFileSource {
  stream: Readable;
  size?: number;
}

type TarPack = ReturnType<typeof tar.pack>;

// Returns the output as soon as the file list is known and fills the archive in the background, one
// entry at a time: each file is opened only after the previous entry has been written through to the
// consumer, so memory holds one file's in-flight chunks, not the whole store. A file deleted since the
// listing is skipped with a warning. Any other open failure (a throttled or unreachable store) destroys
// the output, so the export fails instead of reporting success for an archive missing that file; so
// does a read that fails part-way, because its tar header is already written and skipping it would
// leave a corrupt archive.
export async function createExportStream(
  listFiles: () => Promise<string[]>,
  openFile: (filePath: string) => Promise<ExportFileSource>,
  logger: LoggerService,
): Promise<PassThrough> {
  const files = await listFiles();
  // The importer aborts a whole archive past STORAGE_IMPORT_MAX_ENTRIES. The export carries the whole
  // store, so a large one can produce an archive this gateway refuses to restore, and the operator
  // would only find out at restore time, after decommissioning the source. Say it at EXPORT time instead.
  // Compared against the LOWER of this deployment's configured limit and the shipped default. The
  // default alone is destination-agnostic, which is right when restoring elsewhere — but it stays
  // silent for an operator who LOWERED the limit here and restores onto this same gateway, which is
  // the one destination whose limit we actually know.
  const localImportCap = positiveIntFromEnv('STORAGE_IMPORT_MAX_ENTRIES', DEFAULT_IMPORT_MAX_ENTRIES);
  const warnAbove = Math.min(localImportCap, DEFAULT_IMPORT_MAX_ENTRIES);
  if (files.length > warnAbove) {
    logger.warn(
      `Export contains ${files.length} files, above the import limit of ${warnAbove} ` +
        `(this deployment: ${localImportCap}, shipped default: ${DEFAULT_IMPORT_MAX_ENTRIES}). ` +
        'Raise STORAGE_IMPORT_MAX_ENTRIES on the destination before restoring this archive.',
    );
  }
  const output = new PassThrough();

  const pack = tar.pack();
  const gzip = createGzip({ level: 6 });

  // Surface archive-level failures (tar/gzip) on the returned stream instead of letting them become an
  // uncaught 'error' event or a silently truncated download.
  const failArchive = (err: Error): void => {
    logger.error('Export archive failed', String(err));
    output.destroy(err);
  };
  pack.on('error', failArchive);
  gzip.on('error', failArchive);

  pack.pipe(gzip).pipe(output);
  // The output closes early when the archive fails or the consumer goes away: release the tar and gzip
  // streams with whatever they still buffer. After a clean finish both have ended, so this does nothing.
  output.once('close', () => {
    pack.destroy();
    gzip.destroy();
  });

  appendEntries(files, openFile, pack, output, logger).catch((err: Error) => output.destroy(err));
  return output;
}

function enforceSize(file: string, size: number | undefined, onRead: (bytes: number) => void): Transform {
  let seen = 0;
  const mismatch = (): Error => new Error(`Export failed: ${file} changed size while being read (expected ${size})`);
  return new Transform({
    transform(chunk: Buffer, _encoding, callback) {
      seen += chunk.length;
      callback(size !== undefined && seen > size ? mismatch() : null, chunk);
    },
    flush(callback) {
      if (size !== undefined && seen < size) return callback(mismatch());
      onRead(seen);
      callback();
    },
  });
}

/**
 * Write one tar entry from `source` and call `done` once it is complete or torn down. A known size is
 * piped straight into the entry; without one the source is buffered first, since a tar header has to
 * declare the length up front. A failed entry also destroys the pack, which fails the output through
 * its 'error' listener, so the entry's own error is not needed here; it still needs a listener of its
 * own, because an 'error' event nothing listens for would crash the process.
 */
function writeEntry(
  pack: TarPack,
  source: Readable,
  header: { name: string; mode: number; mtime: Date },
  size: number | undefined,
  output: PassThrough,
  done: () => void,
): void {
  if (size !== undefined) {
    const sink = pack.entry({ ...header, size }, () => done());
    sink.on('error', () => undefined);
    source.pipe(sink);
    return;
  }
  const chunks: Buffer[] = [];
  source.on('data', (chunk: Buffer) => chunks.push(chunk));
  source.on('end', () => {
    // The pack is gone once the output closed, and adding an entry to a destroyed pack throws.
    if (output.destroyed) return;
    pack.entry(header, Buffer.concat(chunks), () => done()).on('error', () => undefined);
  });
}

async function appendEntries(
  files: string[],
  openFile: (filePath: string) => Promise<ExportFileSource>,
  pack: TarPack,
  output: PassThrough,
  logger: LoggerService,
): Promise<void> {
  let exportedBytes = 0;
  let largest = { file: '', size: 0 };
  for (const file of files) {
    let source: ExportFileSource;
    try {
      source = await openFile(file);
    } catch (error) {
      // Only a per-object miss is skippable: ENOENT locally, NoSuchKey from S3 (openS3File returns it
      // once the local fallback also misses). isMissingObjectError also counts any 404, which includes
      // a bucket that is gone, and skipping that would report a partial archive as a finished export.
      const { code, name } = error as { code?: string; name?: string };
      if (code !== 'ENOENT' && name !== 'NoSuchKey') throw error;
      logger.warn(`Failed to export file: ${file}`, { error: String(error) });
      continue;
    }
    const { stream, size } = source;
    // The output closes early when the archive fails or the consumer goes away: stop reading files.
    // Checked synchronously before waiting, so a 'close' that already fired cannot be missed.
    if (output.destroyed) {
      stream.destroy();
      break;
    }
    // The tar sink fails with a bare "Size mismatch" when an entry's length differs from the size in
    // its header. A file rewritten between open and read does that, so the length is checked here and
    // a mismatch fails the export like a read error, naming the file, before the sink sees the extra
    // byte or the early end. pipeline() also forwards a source error, which pipe() does not.
    // The import-limit warnings below count the bytes read, since a backend may not report a length.
    const entry = pipeline(
      stream,
      enforceSize(file, size, bytes => {
        exportedBytes += bytes;
        if (bytes > largest.size) largest = { file, size: bytes };
      }),
      err => {
        if (!err || output.destroyed) return;
        logger.error(`Export failed reading file: ${file}`, String(err));
        output.destroy(err);
      },
    );
    await new Promise<void>(resolve => {
      const done = (): void => {
        output.removeListener('close', done);
        resolve();
      };
      output.on('close', done);
      writeEntry(pack, entry, { name: file, mode: 0o644, mtime: new Date() }, size, output, done);
    });
    if (output.destroyed) {
      stream.destroy();
      break;
    }
  }
  if (output.destroyed) return;
  // Same reasoning as the entry-count warning in createExportStream, for the import's total-bytes cap.
  const localByteCap = positiveIntFromEnv('STORAGE_IMPORT_MAX_TOTAL_BYTES', DEFAULT_IMPORT_MAX_TOTAL_BYTES);
  const warnAboveBytes = Math.min(localByteCap, DEFAULT_IMPORT_MAX_TOTAL_BYTES);
  if (exportedBytes > warnAboveBytes) {
    logger.warn(
      `Export contains ${exportedBytes} bytes, above the import limit of ${warnAboveBytes} ` +
        `(this deployment: ${localByteCap}, shipped default: ${DEFAULT_IMPORT_MAX_TOTAL_BYTES}). ` +
        'Raise STORAGE_IMPORT_MAX_TOTAL_BYTES on the destination before restoring this archive.',
    );
  }
  // And for the per-entry cap: one file above it aborts the whole import.
  const localEntryCap = positiveIntFromEnv('STORAGE_IMPORT_MAX_BYTES', DEFAULT_IMPORT_MAX_BYTES);
  const warnAboveEntry = Math.min(localEntryCap, DEFAULT_IMPORT_MAX_BYTES);
  if (largest.size > warnAboveEntry) {
    logger.warn(
      `Export contains ${largest.file} (${largest.size} bytes), above the per-entry import limit of ${warnAboveEntry} ` +
        `(this deployment: ${localEntryCap}, shipped default: ${DEFAULT_IMPORT_MAX_BYTES}). ` +
        'Raise STORAGE_IMPORT_MAX_BYTES on the destination before restoring this archive.',
    );
  }
  pack.finalize();
}

// ============================================================================
// Import - Extract tar.gz stream to current storage
// ============================================================================

// Best-effort, NOT atomic: a single bad/traversing entry is skipped and the rest still import, and a
// resource-cap breach aborts the rest but KEEPS the entries already written (no rollback). Callers
// re-running an import is safe (putFile overwrites). A staging-dir + atomic promote would make it
// transactional, but is out of scope here. Resolves with how many entries were written and how many
// putFile refused, so a caller can tell an import that wrote nothing from an empty archive.
export async function importFromStream(
  inputStream: Readable,
  putFile: (filePath: string, data: Buffer) => Promise<void>,
  logger: LoggerService,
): Promise<{ imported: number; failed: number }> {
  let importedCount = 0;
  let failedCount = 0;
  let entryCount = 0;
  let totalBytes = 0;
  const maxEntryBytes = positiveIntFromEnv('STORAGE_IMPORT_MAX_BYTES', DEFAULT_IMPORT_MAX_BYTES);
  const maxEntries = positiveIntFromEnv('STORAGE_IMPORT_MAX_ENTRIES', DEFAULT_IMPORT_MAX_ENTRIES);
  const maxTotalBytes = positiveIntFromEnv('STORAGE_IMPORT_MAX_TOTAL_BYTES', DEFAULT_IMPORT_MAX_TOTAL_BYTES);

  const extract = tar.extract();
  const gunzip = createGunzip();

  return new Promise<{ imported: number; failed: number }>((resolve, reject) => {
    let settled = false;
    // Abort the whole import, not just the entry: a cap breach can be a decompression bomb or a store
    // larger than the configured limit. Tear down the pipeline and reject so nothing further is buffered
    // or written.
    const fail = (err: Error): void => {
      if (settled) return;
      settled = true;
      extract.destroy();
      gunzip.destroy();
      // Destroying the input mid-pipe stops the source; without an error arg it emits no 'error'.
      inputStream.destroy();
      // The counts reached so far ride on the rejection for the caller's audit row. A lower bound: a
      // putFile already in flight when a gzip or input error lands can still complete afterwards.
      reject(Object.assign(err, { imported: importedCount, failed: failedCount }));
    };
    // Every stream in the pipeline needs an 'error' listener: an EventEmitter with none CRASHES the
    // process on error. pipe() does not forward errors, so a corrupt gzip (zlib error on gunzip) or
    // an input read failure (disk I/O, file replaced mid-read) would otherwise kill the server
    // mid-request instead of failing the import.
    gunzip.on('error', (err: Error) => {
      logger.error('Import failed (gzip)', String(err));
      fail(err);
    });
    inputStream.on('error', (err: Error) => {
      logger.error('Import failed (input)', String(err));
      fail(err);
    });

    extract.on('entry', (header, stream, next) => {
      if (settled) {
        stream.resume();
        return;
      }
      if (++entryCount > maxEntries) {
        stream.resume();
        fail(new Error(`Import aborted: archive exceeds the ${maxEntries}-entry limit (STORAGE_IMPORT_MAX_ENTRIES)`));
        return;
      }
      // Only regular files are media. A directory or link entry carries no content, and writing it
      // through putFile would store an empty file over any existing object with that key.
      if (header.type !== 'file' && header.type !== 'contiguous-file') {
        logger.debug(`Skipped ${header.type ?? 'unknown'} entry: ${header.name}`);
        stream.on('end', () => next());
        stream.resume();
        return;
      }
      // An archive packed with `tar -C dir .` names every entry `./…`; the export's keys have no such
      // segment, and on S3 it would become part of the object key.
      const key = header.name.replace(/^(?:\.\/)+/, '');

      const chunks: Buffer[] = [];
      let entryBytes = 0;
      let entryAborted = false;

      stream.on('data', (chunk: Buffer) => {
        if (entryAborted || settled) return;
        entryBytes += chunk.length;
        totalBytes += chunk.length;
        if (entryBytes > maxEntryBytes) {
          entryAborted = true;
          stream.resume(); // drain the remainder so the source can end
          fail(
            new Error(
              `Import aborted: entry "${key}" exceeds the ${maxEntryBytes}-byte per-entry cap (STORAGE_IMPORT_MAX_BYTES)`,
            ),
          );
        } else if (totalBytes > maxTotalBytes) {
          // The entry that crosses the cap is dropped, so no import writes more than maxTotalBytes.
          entryAborted = true;
          stream.resume();
          fail(
            new Error(
              `Import aborted: archive exceeds the ${maxTotalBytes}-byte total cap (STORAGE_IMPORT_MAX_TOTAL_BYTES)`,
            ),
          );
        } else {
          chunks.push(chunk);
        }
      });

      stream.on('end', () => {
        if (entryAborted || settled) return;
        const data = Buffer.concat(chunks);
        putFile(key, data)
          .then(() => {
            importedCount++;
            logger.debug(`Imported file: ${key}`);
            next();
          })
          .catch((error: unknown) => {
            failedCount++;
            logger.error(`Failed to import file: ${key}`, String(error));
            next();
          });
      });
      stream.resume();
    });

    extract.on('finish', () => {
      if (settled) return;
      settled = true;
      logger.log(`Import completed: ${importedCount} files, ${failedCount} failed`);
      resolve({ imported: importedCount, failed: failedCount });
    });

    extract.on('error', (err: Error) => {
      logger.error('Import failed', String(err));
      fail(err);
    });

    inputStream.pipe(gunzip).pipe(extract);
  });
}