Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

20.7 KB
Raw Blame
import { randomBytes } from 'crypto';
import { PassThrough, Readable } from 'stream';
import * as tar from 'tar-stream';
import { createGzip } from 'zlib';

import {
  createExportStream,
  DEFAULT_IMPORT_MAX_BYTES,
  DEFAULT_IMPORT_MAX_ENTRIES,
  DEFAULT_IMPORT_MAX_TOTAL_BYTES,
  ExportFileSource,
  importFromStream,
} from './storage-transfer';

const FILE_BYTES = 256 * 1024;
const files = Array.from({ length: 12 }, (_, i) => `media/file-${String.fromCharCode(97 + i)}.bin`);

const makeLogger = () => ({ warn: jest.fn(), error: jest.fn(), log: jest.fn(), debug: jest.fn() });

// Random bytes do not compress, so gzip cannot hide a file inside a few kilobytes of stream buffer.
const contents = new Map<string, Buffer>();
const contentFor = (name: string): Buffer => {
  if (!contents.has(name)) contents.set(name, randomBytes(FILE_BYTES));
  return contents.get(name)!;
};

/** An openFile that counts the file streams open at once and the files ever opened. */
function trackingOpener(failing: Record<string, 'gone' | 'bucket' | 'open' | 'read'> = {}, withSize = true) {
  const stats = { opened: 0, open: 0, maxOpen: 0 };
  const openFile = (name: string): Promise<ExportFileSource> => {
    if (failing[name] === 'gone')
      return Promise.reject(Object.assign(new Error(`ENOENT: ${name}`), { code: 'ENOENT' }));
    if (failing[name] === 'bucket')
      return Promise.reject(
        Object.assign(new Error('The specified bucket does not exist'), {
          name: 'NoSuchBucket',
          $metadata: { httpStatusCode: 404 },
        }),
      );
    if (failing[name] === 'open') return Promise.reject(new Error(`SlowDown: ${name}`));
    const data = contentFor(name);
    stats.opened++;
    stats.open++;
    stats.maxOpen = Math.max(stats.maxOpen, stats.open);
    let offset = 0;
    const stream = new Readable({
      highWaterMark: 16 * 1024,
      read() {
        if (failing[name] === 'read' && offset > 0) {
          this.destroy(new Error(`read failed: ${name}`));
          return;
        }
        const chunk = data.subarray(offset, offset + 16 * 1024);
        offset += chunk.length;
        this.push(chunk.length ? chunk : null);
      },
    });
    stream.once('close', () => stats.open--);
    return Promise.resolve({ stream, size: withSize ? data.length : undefined });
  };
  return { stats, openFile };
}

const settle = (ms: number): Promise<void> => new Promise(resolve => setTimeout(resolve, ms));

async function importAll(output: PassThrough, logger: object): Promise<Map<string, Buffer>> {
  const imported = new Map<string, Buffer>();
  await importFromStream(
    output,
    (name, data) => {
      imported.set(name, data);
      return Promise.resolve();
    },
    logger as never,
  );
  return imported;
}

describe('createExportStream streams one file at a time', () => {
  it('returns before reading the store and opens no further files while nothing consumes the output', async () => {
    const { stats, openFile } = trackingOpener();
    const output = await createExportStream(() => Promise.resolve(files), openFile, makeLogger() as never);

    await settle(100);
    // Stream buffers absorb a file or so; every file past that waits for a reader.
    expect(stats.opened).toBeLessThanOrEqual(2);
    output.destroy();
  });

  it('holds one file stream open at a time, skips a file deleted since the listing, and round-trips the rest', async () => {
    const { stats, openFile } = trackingOpener({ 'media/file-c.bin': 'gone' });
    const logger = makeLogger();
    const output = await createExportStream(() => Promise.resolve(files), openFile, logger as never);

    const imported = await importAll(output, logger);

    expect(stats.maxOpen).toBe(1);
    expect(stats.open).toBe(0);
    expect(logger.warn).toHaveBeenCalledWith('Failed to export file: media/file-c.bin', expect.anything());
    expect([...imported.keys()].sort()).toEqual(files.filter(f => f !== 'media/file-c.bin'));
    for (const [name, data] of imported) expect(data.equals(contentFor(name))).toBe(true);
  });

  it('fails the output when a listed file cannot be opened for any other reason', async () => {
    const { stats, openFile } = trackingOpener({ 'media/file-c.bin': 'open' });
    const output = await createExportStream(() => Promise.resolve(files), openFile, makeLogger() as never);

    const error = await new Promise<Error>(resolve => {
      output.on('error', resolve);
      output.resume();
    });
    await settle(20);

    expect(error.message).toBe('SlowDown: media/file-c.bin');
    expect(stats.opened).toBe(2);
  });

  it('fails the output when the bucket itself is gone, not only one object', async () => {
    const { stats, openFile } = trackingOpener({ 'media/file-c.bin': 'bucket' });
    const logger = makeLogger();
    const output = await createExportStream(() => Promise.resolve(files), openFile, logger as never);

    const error = await new Promise<Error>(resolve => {
      output.on('error', resolve);
      output.resume();
    });
    await settle(20);

    expect(error.name).toBe('NoSuchBucket');
    expect(stats.opened).toBe(2);
    expect(logger.warn).not.toHaveBeenCalledWith('Failed to export file: media/file-c.bin', expect.anything());
  });

  it('still exports a file whose size the backend did not report', async () => {
    const { openFile } = trackingOpener({}, false);
    const logger = makeLogger();
    const output = await createExportStream(() => Promise.resolve(files.slice(0, 2)), openFile, logger as never);

    const imported = await importAll(output, logger);

    expect(imported.get(files[1])?.equals(contentFor(files[1]))).toBe(true);
  });

  // A key is an opaque name. A backslash, a doubled slash or a leading `word:` belongs to it, and an archive that
  // rewrites any of them restores the file under a different key.
  it.each([true, false])(
    'writes every key under exactly the name it is stored by (size reported: %s)',
    async withSize => {
      const names = [
        'media/back\\slash.bin',
        'media/dir//double.bin',
        '628123:[email protected]/c.jpg',
        'status:x/y.bin',
      ];
      const { openFile } = trackingOpener({}, withSize);
      const logger = makeLogger();
      const output = await createExportStream(() => Promise.resolve(names), openFile, logger as never);

      const imported = await importAll(output, logger);

      expect([...imported.keys()]).toEqual(names);
      expect(imported.get(names[0])?.equals(contentFor(names[0]))).toBe(true);
    },
  );

  it('fails the output with a read error part-way through a file and stops opening files', async () => {
    const { stats, openFile } = trackingOpener({ 'media/file-b.bin': 'read' });
    const output = await createExportStream(() => Promise.resolve(files), openFile, makeLogger() as never);

    const error = await new Promise<Error>(resolve => {
      output.on('error', resolve);
      output.resume();
    });
    await settle(20);

    expect(error.message).toBe('read failed: media/file-b.bin');
    expect(stats.opened).toBe(2);
    expect(stats.open).toBe(0);
  });

  it('stops and releases the open file when the consumer destroys the output', async () => {
    const { stats, openFile } = trackingOpener();
    const output = await createExportStream(() => Promise.resolve(files), openFile, makeLogger() as never);
    await settle(50);
    const openedBefore = stats.opened;

    output.destroy();
    await settle(50);

    expect(stats.open).toBe(0);
    expect(stats.opened).toBe(openedBefore);
  });
});

describe('createExportStream releases the archive streams', () => {
  // The tar and gzip streams hold whatever the consumer had not read yet, and zlib a native handle. Destroying
  // the output has to release them at once, not leave them to the garbage collector. Nothing else fails when it
  // does not: the export stops opening files either way, so only the two streams themselves can show it.
  it('destroys the tar and gzip streams when the consumer destroys the output mid-entry', async () => {
    const realZlib = jest.requireActual<typeof import('zlib')>('zlib');
    const realCreateGzip = realZlib.createGzip;
    const readOnlyExport = { configurable: true, enumerable: true, writable: false };
    const gzips: Array<{ destroyed: boolean }> = [];
    // createGzip is a read-only export. jest.spyOn cannot wrap it, and the failed attempt makes the suite fail
    // when the mocks are restored, so the export is replaced by hand for the length of the test.
    Object.defineProperty(realZlib, 'createGzip', {
      ...readOnlyExport,
      value: (...args: Parameters<typeof realCreateGzip>) => {
        const gzip = realCreateGzip(...args);
        gzips.push(gzip);
        return gzip;
      },
    });
    const pack = jest.spyOn(jest.requireActual<typeof tar>('tar-stream'), 'pack');
    try {
      let reading = false;
      // An entry that never ends, so the export is parked inside it when the consumer goes away.
      const stuck = new Readable({
        read() {
          reading = true;
        },
      });
      const output = await createExportStream(
        () => Promise.resolve(['media/stuck.bin']),
        () => Promise.resolve({ stream: stuck, size: FILE_BYTES }),
        makeLogger() as never,
      );
      for (let waited = 0; !reading && waited < 1000; waited += 10) await settle(10);
      expect(reading).toBe(true);
      expect(pack).toHaveBeenCalledTimes(1);
      expect(gzips).toHaveLength(1);
      const tarPack = pack.mock.results[0].value as { destroyed: boolean };
      const gzip = gzips[0];
      expect(tarPack.destroyed).toBe(false);
      expect(gzip.destroyed).toBe(false);

      output.destroy();
      // Destroying a stream finishes on a later tick; a missing teardown shows here as a stream still open.
      for (let waited = 0; !(tarPack.destroyed && gzip.destroyed) && waited < 1000; waited += 10) await settle(10);

      expect(tarPack.destroyed).toBe(true);
      expect(gzip.destroyed).toBe(true);
    } finally {
      pack.mockRestore();
      Object.defineProperty(realZlib, 'createGzip', { ...readOnlyExport, value: realCreateGzip });
    }
  });
});

describe('importFromStream reports refused writes', () => {
  it('counts every entry putFile rejects as failed, and none as imported', async () => {
    const { openFile } = trackingOpener();
    const names = files.slice(0, 3);
    const output = await createExportStream(() => Promise.resolve(names), openFile, makeLogger() as never);

    const result = await importFromStream(
      output,
      () => Promise.reject(new Error('AccessDenied')),
      makeLogger() as never,
    );

    expect(result).toEqual({ imported: 0, failed: names.length });
  });
});

describe('importFromStream writes regular files only', () => {
  it('skips directory and link entries and drops a leading ./ from the key', async () => {
    const pack = tar.pack();
    pack.entry({ name: './', type: 'directory' });
    pack.entry({ name: './status/', type: 'directory' });
    pack.entry({ name: './status/link.jpg', type: 'symlink', linkname: 'a.jpg' });
    pack.entry({ name: './status/hard.jpg', type: 'link', linkname: './status/a.jpg' });
    pack.entry({ name: './status/a.jpg' }, Buffer.from('jpeg!!'));
    pack.finalize();
    const written = new Map<string, Buffer>();

    const result = await importFromStream(
      pack.pipe(createGzip()),
      (name, data) => {
        written.set(name, data);
        return Promise.resolve();
      },
      makeLogger() as never,
    );

    expect([...written.keys()]).toEqual(['status/a.jpg']);
    expect(written.get('status/a.jpg')?.toString()).toBe('jpeg!!');
    expect(result).toEqual({ imported: 1, failed: 0 });
  });
});

describe('importFromStream reports partial counts on an abort', () => {
  it('carries the entries already written on an entry-cap rejection', async () => {
    const prev = process.env.STORAGE_IMPORT_MAX_ENTRIES;
    process.env.STORAGE_IMPORT_MAX_ENTRIES = '2';
    const pack = tar.pack();
    for (const name of ['a.jpg', 'b.jpg', 'c.jpg']) pack.entry({ name }, Buffer.from(name));
    pack.finalize();
    const written: string[] = [];
    try {
      const rejection = await importFromStream(
        pack.pipe(createGzip()),
        name => {
          written.push(name);
          return Promise.resolve();
        },
        makeLogger() as never,
      ).catch((err: unknown) => err);

      expect(rejection).toBeInstanceOf(Error);
      expect((rejection as Error).message).toMatch(/2-entry limit \(STORAGE_IMPORT_MAX_ENTRIES\)/);
      expect(rejection).toMatchObject({ imported: 2, failed: 0 });
      expect(written).toEqual(['a.jpg', 'b.jpg']);
    } finally {
      if (prev === undefined) delete process.env.STORAGE_IMPORT_MAX_ENTRIES;
      else process.env.STORAGE_IMPORT_MAX_ENTRIES = prev;
    }
  });
});

describe('importFromStream caps the total bytes one import extracts', () => {
  const withTotalCap = async <T>(cap: string, run: () => Promise<T>): Promise<T> => {
    const prev = process.env.STORAGE_IMPORT_MAX_TOTAL_BYTES;
    process.env.STORAGE_IMPORT_MAX_TOTAL_BYTES = cap;
    try {
      return await run();
    } finally {
      if (prev === undefined) delete process.env.STORAGE_IMPORT_MAX_TOTAL_BYTES;
      else process.env.STORAGE_IMPORT_MAX_TOTAL_BYTES = prev;
    }
  };
  const importEntries = (written: Map<string, Buffer>, sizes: number[]) => {
    const pack = tar.pack();
    sizes.forEach((size, i) => pack.entry({ name: `e${i}.bin` }, Buffer.alloc(size, 0x61)));
    pack.finalize();
    return importFromStream(
      pack.pipe(createGzip()),
      (name, data) => {
        written.set(name, data);
        return Promise.resolve();
      },
      makeLogger() as never,
    );
  };
  const bytesIn = (written: Map<string, Buffer>): number => [...written.values()].reduce((n, b) => n + b.length, 0);

  it('aborts when entries that each fit the per-entry cap add up past the total cap', async () => {
    const written = new Map<string, Buffer>();
    const rejection = await withTotalCap('10000', () => importEntries(written, [4000, 4000, 4000, 4000])).catch(
      (err: unknown) => err,
    );

    expect(rejection).toBeInstanceOf(Error);
    expect((rejection as Error).message).toMatch(/10000-byte total cap \(STORAGE_IMPORT_MAX_TOTAL_BYTES\)/);
    expect(rejection).toMatchObject({ imported: 2, failed: 0 });
    expect([...written.keys()]).toEqual(['e0.bin', 'e1.bin']);
    expect(bytesIn(written)).toBeLessThanOrEqual(10000);
  });

  it('imports an archive whose total is exactly the cap', async () => {
    const written = new Map<string, Buffer>();
    const result = await withTotalCap('8000', () => importEntries(written, [4000, 4000]));

    expect(result).toEqual({ imported: 2, failed: 0 });
    expect(bytesIn(written)).toBe(8000);
  });

  it('warns at export time when the archive holds more bytes than the import accepts', async () => {
    const exportWith = async (cap: string) => {
      const logger = makeLogger();
      await withTotalCap(cap, async () => {
        const { openFile } = trackingOpener();
        const output = await createExportStream(() => Promise.resolve(files.slice(0, 3)), openFile, logger as never);
        await new Promise<void>(resolve => output.on('end', resolve).resume());
      });
      return logger.warn;
    };

    expect(await exportWith(String(3 * FILE_BYTES))).not.toHaveBeenCalled();
    expect(await exportWith(String(3 * FILE_BYTES - 1))).toHaveBeenCalledWith(
      expect.stringContaining('Raise STORAGE_IMPORT_MAX_TOTAL_BYTES'),
    );
  });
});

describe('createExportStream and the per-entry import cap', () => {
  it('warns at export time when one file is larger than the import accepts', async () => {
    const exportWith = async (cap: string) => {
      const logger = makeLogger();
      const prev = process.env.STORAGE_IMPORT_MAX_BYTES;
      process.env.STORAGE_IMPORT_MAX_BYTES = cap;
      try {
        const { openFile } = trackingOpener();
        const output = await createExportStream(() => Promise.resolve(files.slice(0, 3)), openFile, logger as never);
        await new Promise<void>(resolve => output.on('end', resolve).resume());
      } finally {
        if (prev === undefined) delete process.env.STORAGE_IMPORT_MAX_BYTES;
        else process.env.STORAGE_IMPORT_MAX_BYTES = prev;
      }
      return logger.warn;
    };

    expect(await exportWith(String(FILE_BYTES))).not.toHaveBeenCalled();
    const warn = await exportWith(String(FILE_BYTES - 1));
    expect(warn).toHaveBeenCalledTimes(1);
    expect(warn).toHaveBeenCalledWith(expect.stringContaining('Raise STORAGE_IMPORT_MAX_BYTES'));
    expect(warn).toHaveBeenCalledWith(expect.stringContaining(`(${FILE_BYTES} bytes)`));
  });
});

describe('the per-entry import cap boundary', () => {
  const importOne = async (cap: string, size: number) => {
    const prev = process.env.STORAGE_IMPORT_MAX_BYTES;
    process.env.STORAGE_IMPORT_MAX_BYTES = cap;
    const pack = tar.pack();
    pack.entry({ name: 'only.bin' }, Buffer.alloc(size, 0x61));
    pack.finalize();
    try {
      return await importFromStream(pack.pipe(createGzip()), () => Promise.resolve(), makeLogger() as never).catch(
        (err: unknown) => err,
      );
    } finally {
      if (prev === undefined) delete process.env.STORAGE_IMPORT_MAX_BYTES;
      else process.env.STORAGE_IMPORT_MAX_BYTES = prev;
    }
  };

  it('imports an entry of exactly the cap and aborts on one byte more', async () => {
    expect(await importOne('10', 10)).toEqual({ imported: 1, failed: 0 });
    const over = await importOne('10', 11);
    expect(over).toBeInstanceOf(Error);
    expect((over as Error).message).toMatch(/10-byte per-entry cap \(STORAGE_IMPORT_MAX_BYTES\)/);
  });
});

describe('createExportStream and the entry-count limit this deployment lowered', () => {
  it('warns above the lower limit, though the shipped default would not', async () => {
    const prev = process.env.STORAGE_IMPORT_MAX_ENTRIES;
    process.env.STORAGE_IMPORT_MAX_ENTRIES = '2';
    const logger = makeLogger();
    try {
      const { openFile } = trackingOpener();
      const output = await createExportStream(() => Promise.resolve(files.slice(0, 3)), openFile, logger as never);
      await new Promise<void>(resolve => output.on('end', resolve).resume());
    } finally {
      if (prev === undefined) delete process.env.STORAGE_IMPORT_MAX_ENTRIES;
      else process.env.STORAGE_IMPORT_MAX_ENTRIES = prev;
    }
    expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('above the import limit of 2'));
    expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('Raise STORAGE_IMPORT_MAX_ENTRIES'));
  });
});

// A backend that does not report a length (ExportFileSource.size is optional) still exports a file the
// destination may refuse, so the warning is worked out from the bytes actually read.
describe('createExportStream warns about import limits for files of unreported size', () => {
  const exportWithLimits = async (limits: Record<string, string>, sizes: number[]) => {
    const previous = Object.fromEntries(Object.keys(limits).map(key => [key, process.env[key]]));
    Object.assign(process.env, limits);
    const logger = makeLogger();
    try {
      const names = sizes.map((_, index) => `media/unknown-${index}.bin`);
      const output = await createExportStream(
        () => Promise.resolve(names),
        name => Promise.resolve({ stream: Readable.from([Buffer.alloc(sizes[names.indexOf(name)], 0x61)]) }),
        logger as never,
      );
      await new Promise<void>(resolve => output.on('end', resolve).resume());
    } finally {
      for (const [key, value] of Object.entries(previous)) {
        if (value === undefined) delete process.env[key];
        else process.env[key] = value;
      }
    }
    return logger;
  };

  it('warns when one file is above the per-entry limit', async () => {
    const logger = await exportWithLimits({ STORAGE_IMPORT_MAX_BYTES: '5' }, [6]);
    expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('Raise STORAGE_IMPORT_MAX_BYTES'));
  });

  it('warns when the files together are above the total limit', async () => {
    const logger = await exportWithLimits({ STORAGE_IMPORT_MAX_TOTAL_BYTES: '5' }, [3, 3]);
    expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('Raise STORAGE_IMPORT_MAX_TOTAL_BYTES'));
  });

  it('stays quiet at exactly the limits', async () => {
    const logger = await exportWithLimits(
      { STORAGE_IMPORT_MAX_BYTES: '3', STORAGE_IMPORT_MAX_TOTAL_BYTES: '6' },
      [3, 3],
    );
    expect(logger.warn).not.toHaveBeenCalled();
  });
});

// docs/14, .env.example and the upgrade notes state these figures, and an operator sizes an import
// against them: a drifted default changes what an import is allowed to extract without a word.
describe('storage import caps', () => {
  it('ship at the documented defaults', () => {
    expect(DEFAULT_IMPORT_MAX_BYTES).toBe(209_715_200);
    expect(DEFAULT_IMPORT_MAX_ENTRIES).toBe(100_000);
    expect(DEFAULT_IMPORT_MAX_TOTAL_BYTES).toBe(10_737_418_240);
  });
});