Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(storage): stream the export one file at a time
Yudhi Armyndharis2c091fa91f49
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 3ce9b6e97..480b2a5b1 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -20,6 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0- A session whose automatic reconnect fails to relaunch the engine (a network, DNS or browser launch error) keeps retrying with backoff instead of stopping in `failed` until restarted by hand; an authentication failure or a stale browser profile still ends in `failed` ([#1580](https://github.com/rmyndharis/OpenWA/issues/1580)).- A session that runs out of reconnect attempts fires the `session:error` plugin hook when it lands in `failed`.- An explicit `maxReconnectAttempts` is honoured and the reconnect delay is capped at 5 minutes; the budget used to restart once the backoff passed 5 minutes, so the limit was never reached and the documented 1-hour cap never applied.+- `GET /api/infra/storage/export` streams files into the archive one at a time instead of loading the whole media store into memory first, so exporting a large local or S3 store no longer exhausts memory.## [0.23.5] - 2026-09-14diff --git a/src/common/storage/storage-local-files.ts b/src/common/storage/storage-local-files.tsindex 31733432b..c71cee96c 100644--- a/src/common/storage/storage-local-files.ts+++ b/src/common/storage/storage-local-files.ts@@ -1,5 +1,6 @@import * as fs from 'fs';import * as path from 'path';+import { Readable } from 'stream';import { isPathWithin } from '../utils/path-safety';/** Max number of local files a single traversal enumerates. Bounds a count DoS on a huge media dir. */@@ -67,11 +68,29 @@ export function getLocalFile(localPath: string, filePath: string): Promise<Buffethrow new Error(`Refusing to read outside storage root: ${filePath}`);}const fullPath = path.join(localPath, filePath);- // Async read so the export loop (the only caller) yields the event loop per file instead of- // blocking it with a synchronous read for every media file.+ // Async read so a media read yields the event loop instead of blocking it for the whole file.return fs.promises.readFile(fullPath);}+/**+ * Open a local file for a streamed read. The handle is opened and stat'ed up front, so a missing or+ * unreadable file rejects here, before the caller commits to it, and the size is the one the open+ * handle sees. The stream closes the handle when it ends or is destroyed.+ */+export async function openLocalFile(localPath: string, filePath: string): Promise<{ stream: Readable; size: number }> {+ if (!isPathWithin(localPath, filePath)) {+ throw new Error(`Refusing to read outside storage root: ${filePath}`);+ }+ const handle = await fs.promises.open(path.join(localPath, filePath), 'r');+ try {+ const { size } = await handle.stat();+ return { stream: handle.createReadStream(), size };+ } catch (error) {+ await handle.close();+ throw error;+ }+}+export async function putLocalFile(localPath: string, filePath: string, data: Buffer): Promise<void> {if (!isPathWithin(localPath, filePath)) {throw new Error(`Refusing to write outside storage root: ${filePath}`);diff --git a/src/common/storage/storage-transfer.spec.ts b/src/common/storage/storage-transfer.spec.tsnew file mode 100644index 000000000..ab60abd3a--- /dev/null+++ b/src/common/storage/storage-transfer.spec.ts@@ -0,0 +1,164 @@+import { randomBytes } from 'crypto';+import { PassThrough, Readable } from 'stream';++// archiver v8 is ESM-only and ts-jest cannot load it here. This stand-in keeps the parts of its+// TarArchive contract the export relies on: a stream entry with a known size is piped straight into+// the tar pack (without one it is collected first), 'entry' fires once the entry is written, and an+// entry failure is emitted as 'error'. The real archiver runs in test/storage-export-streaming.e2e-spec.ts.+jest.mock('archiver', () => {+ /* eslint-disable @typescript-eslint/no-require-imports */+ const { EventEmitter } = require('events') as typeof import('events');+ const tar = require('tar-stream') as typeof import('tar-stream');+ const { createGzip } = require('zlib') as typeof import('zlib');+ /* eslint-enable @typescript-eslint/no-require-imports */+ class TarArchive extends EventEmitter {+ private readonly pack = tar.pack();+ private readonly gzip = createGzip();+ constructor() {+ super();+ this.pack.pipe(this.gzip);+ }+ pipe(destination: NodeJS.WritableStream): NodeJS.WritableStream {+ return this.gzip.pipe(destination);+ }+ append(source: Readable, data: { name: string; stats?: { size: number } }): void {+ const done = (err?: Error | null): void => void (err ? this.emit('error', err) : this.emit('entry', data));+ if (data.stats) {+ source.pipe(this.pack.entry({ name: data.name, size: data.stats.size }, done));+ return;+ }+ const chunks: Buffer[] = [];+ source.on('data', (chunk: Buffer) => chunks.push(chunk));+ source.on('end', () => this.pack.entry({ name: data.name }, Buffer.concat(chunks), done));+ }+ finalize(): Promise<void> {+ this.pack.finalize();+ return Promise.resolve();+ }+ abort(): void {+ this.pack.destroy();+ }+ }+ return { TarArchive };+});++import { createExportStream, ExportFileSource, importFromStream } from './storage-transfer';++const FILE_BYTES = 256 * 1024;+const files = Array.from({ length: 12 }, (_, i) => `media/file-${String.fromCharCode(97 + i)}.bin`);++const makeLogger = () => ({ warn: jest.fn(), error: jest.fn(), log: jest.fn(), debug: jest.fn() });++// Random bytes do not compress, so gzip cannot hide a file inside a few kilobytes of stream buffer.+const contents = new Map<string, Buffer>();+const contentFor = (name: string): Buffer => {+ if (!contents.has(name)) contents.set(name, randomBytes(FILE_BYTES));+ return contents.get(name)!;+};++/** An openFile that counts the file streams open at once and the files ever opened. */+function trackingOpener(failing: Record<string, 'open' | 'read'> = {}, withSize = true) {+ const stats = { opened: 0, open: 0, maxOpen: 0 };+ const openFile = (name: string): Promise<ExportFileSource> => {+ if (failing[name] === 'open') return Promise.reject(new Error(`ENOENT: ${name}`));+ const data = contentFor(name);+ stats.opened++;+ stats.open++;+ stats.maxOpen = Math.max(stats.maxOpen, stats.open);+ let offset = 0;+ const stream = new Readable({+ highWaterMark: 16 * 1024,+ read() {+ if (failing[name] === 'read' && offset > 0) {+ this.destroy(new Error(`read failed: ${name}`));+ return;+ }+ const chunk = data.subarray(offset, offset + 16 * 1024);+ offset += chunk.length;+ this.push(chunk.length ? chunk : null);+ },+ });+ stream.once('close', () => stats.open--);+ return Promise.resolve({ stream, size: withSize ? data.length : undefined });+ };+ return { stats, openFile };+}++const settle = (ms: number): Promise<void> => new Promise(resolve => setTimeout(resolve, ms));++async function importAll(output: PassThrough, logger: object): Promise<Map<string, Buffer>> {+ const imported = new Map<string, Buffer>();+ await importFromStream(+ output,+ (name, data) => {+ imported.set(name, data);+ return Promise.resolve();+ },+ logger as never,+ );+ return imported;+}++describe('createExportStream streams one file at a time', () => {+ it('returns before reading the store and opens no further files while nothing consumes the output', async () => {+ const { stats, openFile } = trackingOpener();+ const output = await createExportStream(() => Promise.resolve(files), openFile, makeLogger() as never);++ await settle(100);+ // Stream buffers absorb a file or so; every file past that waits for a reader.+ expect(stats.opened).toBeLessThanOrEqual(2);+ output.destroy();+ });++ it('holds one file stream open at a time, skips an unopenable file, and round-trips the rest', async () => {+ const { stats, openFile } = trackingOpener({ 'media/file-c.bin': 'open' });+ const logger = makeLogger();+ const output = await createExportStream(() => Promise.resolve(files), openFile, logger as never);++ const imported = await importAll(output, logger);++ expect(stats.maxOpen).toBe(1);+ expect(stats.open).toBe(0);+ expect(logger.warn).toHaveBeenCalledWith('Failed to export file: media/file-c.bin', expect.anything());+ expect([...imported.keys()].sort()).toEqual(files.filter(f => f !== 'media/file-c.bin'));+ for (const [name, data] of imported) expect(data.equals(contentFor(name))).toBe(true);+ });++ it('still exports a file whose size the backend did not report', async () => {+ const { openFile } = trackingOpener({}, false);+ const logger = makeLogger();+ const output = await createExportStream(() => Promise.resolve(files.slice(0, 2)), openFile, logger as never);++ const imported = await importAll(output, logger);++ expect(imported.get(files[1])?.equals(contentFor(files[1]))).toBe(true);+ });++ it('fails the output with a read error part-way through a file and stops opening files', async () => {+ const { stats, openFile } = trackingOpener({ 'media/file-b.bin': 'read' });+ const output = await createExportStream(() => Promise.resolve(files), openFile, makeLogger() as never);++ const error = await new Promise<Error>(resolve => {+ output.on('error', resolve);+ output.resume();+ });+ await settle(20);++ expect(error.message).toBe('read failed: media/file-b.bin');+ expect(stats.opened).toBe(2);+ expect(stats.open).toBe(0);+ });++ it('stops and releases the open file when the consumer destroys the output', async () => {+ const { stats, openFile } = trackingOpener();+ const output = await createExportStream(() => Promise.resolve(files), openFile, makeLogger() as never);+ await settle(50);+ const openedBefore = stats.opened;++ output.destroy();+ await settle(50);++ expect(stats.open).toBe(0);+ expect(stats.opened).toBe(openedBefore);+ });+});diff --git a/src/common/storage/storage-transfer.ts b/src/common/storage/storage-transfer.tsindex 9de62a85e..479d14c52 100644--- a/src/common/storage/storage-transfer.ts+++ b/src/common/storage/storage-transfer.ts@@ -1,7 +1,8 @@import { TarArchive } from 'archiver';import * as tar from 'tar-stream';import { createGunzip } from 'zlib';-import { Readable, PassThrough } from 'stream';+import * as fs from 'fs';+import { Readable, PassThrough, Transform, pipeline } from 'stream';import { LoggerService } from '../services/logger.service';/** Per-entry buffer cap for an import (200 MiB — 4× the inbound media cap). Bounds a decompression bomb. */@@ -18,9 +19,23 @@ function positiveIntFromEnv(name: string, fallback: number): number {// Export - Create tar.gz stream from current storage// ============================================================================+/**+ * A file opened for export. `size` is the byte length the tar header declares; without it archiver+ * has to buffer that one entry to learn its length.+ */+export interface ExportFileSource {+ stream: Readable;+ size?: number;+}++// Returns the output as soon as the file list is known and fills the archive in the background, one+// entry at a time: each file is opened only after the previous entry has been written through to the+// consumer, so memory holds one file's in-flight chunks, not the whole store. A file that cannot be+// opened is skipped with a warning; a read that fails part-way destroys the output with that error,+// because its tar header is already written and skipping it would leave a corrupt archive.export async function createExportStream(listFiles: () => Promise<string[]>,- getFile: (filePath: string) => Promise<Buffer>,+ openFile: (filePath: string) => Promise<ExportFileSource>,logger: LoggerService,): Promise<PassThrough> {const files = await listFiles();@@ -56,20 +71,81 @@ export async function createExportStream(archive.pipe(output);- // Add files to archive+ appendEntries(files, openFile, archive, output, logger).catch((err: Error) => output.destroy(err));+ return output;+}++function enforceSize(file: string, size: number | undefined): Transform {+ let seen = 0;+ const mismatch = (): Error => new Error(`Export failed: ${file} changed size while being read (expected ${size})`);+ return new Transform({+ transform(chunk: Buffer, _encoding, callback) {+ seen += chunk.length;+ callback(size !== undefined && seen > size ? mismatch() : null, chunk);+ },+ flush(callback) {+ callback(size !== undefined && seen < size ? mismatch() : null);+ },+ });+}++async function appendEntries(+ files: string[],+ openFile: (filePath: string) => Promise<ExportFileSource>,+ archive: TarArchive,+ output: PassThrough,+ logger: LoggerService,+): Promise<void> {for (const file of files) {+ let source: ExportFileSource;try {- const data = await getFile(file);- archive.append(data, { name: file });+ source = await openFile(file);} catch (error) {logger.warn(`Failed to export file: ${file}`, { error: String(error) });+ continue;+ }+ const { stream, size } = source;+ // The output closes early when the archive fails or the consumer goes away: stop reading files.+ // Checked synchronously before waiting, so a 'close' that already fired cannot be missed.+ if (output.destroyed) {+ stream.destroy();+ break;+ }+ // The tar sink fails with an error nothing can listen for when an entry's length differs from the+ // size in its header, which would crash the process. A file rewritten between open and read does+ // that, so the length is checked here and a mismatch fails the export like a read error, before+ // the sink sees the extra byte or the early end. pipeline() also forwards a source error, which+ // pipe() does not.+ const entry = pipeline(stream, enforceSize(file, size), err => {+ if (!err || output.destroyed) return;+ logger.error(`Export failed reading file: ${file}`, String(err));+ output.destroy(err);+ });+ await new Promise<void>(resolve => {+ const done = (): void => {+ archive.removeListener('entry', done);+ output.removeListener('close', done);+ resolve();+ };+ archive.on('entry', done);+ output.on('close', done);+ // Mode and date are set explicitly: archiver derives both from `stats` when present, and this+ // stand-in carries only the size.+ const stats = size === undefined ? undefined : ({ size } as fs.Stats);+ archive.append(entry, { name: file, mode: 0o644, date: new Date(), stats });+ });+ if (output.destroyed) {+ stream.destroy();+ break;}}-+ if (output.destroyed) {+ archive.abort();+ return;+ }// finalize() rejections also emit via the 'error' handler above; catch the promise so it// never surfaces as an unhandled rejection.archive.finalize().catch(() => undefined);- return output;}// ============================================================================diff --git a/src/common/storage/storage.service.s3-reprobe.spec.ts b/src/common/storage/storage.service.s3-reprobe.spec.tsindex b0ebcabf6..159b3007a 100644--- a/src/common/storage/storage.service.s3-reprobe.spec.ts+++ b/src/common/storage/storage.service.s3-reprobe.spec.ts@@ -219,6 +219,33 @@ describe('StorageService S3 re-probe and recovery', () => {expect(data.toString()).toBe('from-s3');});+ it('openFile hands the S3 body through unread, with the object length as its size', async () => {+ const body = Readable.from([Buffer.from('from-s3')]);+ mockSend.mockImplementation((cmd: unknown) => {+ if (cmd instanceof GetObjectCommand) return Promise.resolve({ Body: body, ContentLength: 7 });+ return Promise.resolve({});+ });+ const svc = new StorageService(makeConfig());+ await flush();++ await expect(svc.openFile('present.bin')).resolves.toEqual({ stream: body, size: 7 });+ });++ it('openFile reads a missing S3 object through from the local fallback dir, and rejects when neither has it', async () => {+ mockSend.mockImplementation((cmd: unknown) => {+ if (cmd instanceof GetObjectCommand) return Promise.reject(s3Error('NoSuchKey'));+ return Promise.resolve({});+ });+ const svc = new StorageService(makeConfig());+ await flush();+ fs.writeFileSync(path.join(localPath, 'gap.bin'), 'gap-media');++ const { stream, size } = await svc.openFile('gap.bin');+ stream.destroy();+ expect(size).toBe(9);+ await expect(svc.openFile('missing.bin')).rejects.toThrow('NoSuchKey');+ });+it('deleteFile removes the local fallback copy as well as the S3 object', async () => {mockSend.mockResolvedValue({}); // HeadBucket at boot, DeleteObject laterconst svc = new StorageService(makeConfig());diff --git a/src/common/storage/storage.service.spec.ts b/src/common/storage/storage.service.spec.tsindex 8f048f1aa..6b2957777 100644--- a/src/common/storage/storage.service.spec.ts+++ b/src/common/storage/storage.service.spec.ts@@ -147,6 +147,29 @@ describe('StorageService put/getFile containment is backend-agnostic', () => {});});+describe('StorageService.openFile (the export read path)', () => {+ const readAll = async (stream: Readable): Promise<Buffer> => {+ const chunks: Buffer[] = [];+ for await (const chunk of stream) chunks.push(chunk as Buffer);+ return Buffer.concat(chunks);+ };++ it('streams a local file with its size, and rejects a missing file before any stream exists', async () => {+ const { service, baseDir, localPath } = makeLocalService();+ fs.mkdirSync(path.join(localPath, 'sub'), { recursive: true });+ fs.writeFileSync(path.join(localPath, 'sub/a.bin'), 'local-bytes');++ const { stream, size } = await service.openFile('sub/a.bin');+ expect(size).toBe(11);+ expect((await readAll(stream)).toString()).toBe('local-bytes');+ expect((await service.getFile('sub/a.bin')).toString()).toBe('local-bytes');+ await expect(service.openFile('sub/missing.bin')).rejects.toThrow(/ENOENT/);+ await expect(service.openFile('../../etc/passwd')).rejects.toThrow(/unsafe storage key/);++ fs.rmSync(baseDir, { recursive: true, force: true });+ });+});+describe('StorageService getFileCount (S3 size)', () => {it('sums the real Size of each S3 object instead of estimating', async () => {const { service, baseDir } = makeLocalService();diff --git a/src/common/storage/storage.service.ts b/src/common/storage/storage.service.tsindex f5d30516a..618638619 100644--- a/src/common/storage/storage.service.ts+++ b/src/common/storage/storage.service.ts@@ -14,8 +14,15 @@ import {} from '@aws-sdk/client-s3';import { createLogger } from '../services/logger.service';import { isSafeStorageKey } from '../utils/path-safety';-import { createExportStream, importFromStream } from './storage-transfer';-import { listLocalFiles, iterateLocalFiles, getLocalFile, putLocalFile, deleteLocalFile } from './storage-local-files';+import { createExportStream, ExportFileSource, importFromStream } from './storage-transfer';+import {+ listLocalFiles,+ iterateLocalFiles,+ getLocalFile,+ openLocalFile,+ putLocalFile,+ deleteLocalFile,+} from './storage-local-files';interface S3Config {endpoint?: string;@@ -285,6 +292,20 @@ export class StorageService implements OnModuleDestroy {return this.getLocalFile(filePath);}+ /**+ * Open a file as a stream instead of reading it into memory, with the same key guard and S3+ * read-through as getFile. A missing or unreadable file rejects here rather than on the stream.+ */+ async openFile(filePath: string): Promise<ExportFileSource> {+ if (!isSafeStorageKey(filePath)) {+ throw new Error(`Refusing to read an unsafe storage key: ${filePath}`);+ }+ if (this.storageType === 's3' && this.s3Client && this.s3Available) {+ return this.openS3File(filePath);+ }+ return openLocalFile(this.localPath, filePath);+ }+async putFile(filePath: string, data: Buffer): Promise<void> {// Centralized containment so BOTH backends inherit it: putLocalFile has its own isPathWithin// guard, but putS3File builds `media/${filePath}` with none — reject a traversing key here.@@ -401,7 +422,7 @@ export class StorageService implements OnModuleDestroy {// could not reveal the gap. An export exists to be complete; that is what the uncapped walk is for.return createExportStream(() => this.listAllFiles(),- filePath => this.getFile(filePath),+ filePath => this.openFile(filePath),this.logger,);}@@ -489,6 +510,15 @@ export class StorageService implements OnModuleDestroy {}private async getS3File(filePath: string): Promise<Buffer> {+ const { stream } = await this.openS3File(filePath);+ const chunks: Buffer[] = [];+ for await (const chunk of stream) {+ chunks.push(Buffer.from(chunk as ArrayBuffer));+ }+ return Buffer.concat(chunks);+ }++ private async openS3File(filePath: string): Promise<ExportFileSource> {if (!this.s3Client) throw new Error('S3 client not initialized');let response;@@ -505,26 +535,19 @@ export class StorageService implements OnModuleDestroy {// fine during the outage). Fall through to the local copy; if there is none, surface the// original S3 error so "not found" semantics are unchanged.if ((error as { name?: string }).name !== 'NoSuchKey') throw error;+ let local: ExportFileSource;try {- const local = await this.getLocalFile(filePath);- this.logger.debug(`Served '${filePath}' from the local fallback dir (not yet in S3)`);- return local;+ local = await openLocalFile(this.localPath, filePath);} catch {throw error;}+ this.logger.debug(`Served '${filePath}' from the local fallback dir (not yet in S3)`);+ return local;}if (!response.Body) throw new Error('Empty response body');- // Convert stream to buffer- const chunks: Buffer[] = [];- const stream = response.Body as Readable;-- for await (const chunk of stream) {- chunks.push(Buffer.from(chunk as ArrayBuffer));- }-- return Buffer.concat(chunks);+ return { stream: response.Body as Readable, size: response.ContentLength };}private async putS3File(filePath: string, data: Buffer): Promise<void> {diff --git a/src/modules/infra/infra-storage.controller.spec.ts b/src/modules/infra/infra-storage.controller.spec.tsindex f59bea0a9..dbb521fec 100644--- a/src/modules/infra/infra-storage.controller.spec.ts+++ b/src/modules/infra/infra-storage.controller.spec.ts@@ -1,7 +1,7 @@import * as fs from 'fs';import * as os from 'os';import * as path from 'path';-import { Readable } from 'stream';+import { Readable, Writable } from 'stream';// StorageService (imported transitively by the infra controllers) pulls in `archiver`// v8, which is ESM-only and cannot be parsed by ts-jest. The controller logic@@ -22,6 +22,7 @@ jest.mock('fs', () => {existsSync: jest.fn().mockReturnValue(false),readFileSync: jest.fn().mockReturnValue(''),createReadStream: jest.fn(() => jest.requireActual<typeof import('stream')>('stream').Readable.from([])),+ createWriteStream: jest.fn(actual.createWriteStream),};});@@ -225,6 +226,32 @@ describe('InfraStorageController storage stream failures surface as request erro}});+ it('exportStorage destroys the export stream when the archive file cannot be written', async () => {+ const cwd = fs.mkdtempSync(path.join(os.tmpdir(), 'owa-export-sink-'));+ const cwdSpy = jest.spyOn(process, 'cwd').mockReturnValue(cwd);+ const sink = new Writable({+ write(_chunk, _encoding, callback) {+ callback(new Error('disk full'));+ },+ });+ (fs.createWriteStream as jest.Mock).mockReturnValueOnce(sink);+ try {+ // The export fills the archive while it is being written; a failed sink must stop it, or it+ // keeps a file open waiting for a reader that is gone.+ const source = new Readable({+ read() {+ this.push(Buffer.alloc(1024));+ },+ });+ const storage = { createExportStream: jest.fn().mockResolvedValue(source) };+ await expect(new InfraStorageController(storage as never).exportStorage()).rejects.toThrow(/disk full/);+ expect(source.destroyed).toBe(true);+ } finally {+ cwdSpy.mockRestore();+ fs.rmSync(cwd, { recursive: true, force: true });+ }+ });+it('exportStorage rejects (and the process lives) when the export source stream errors', async () => {const cwd = fs.mkdtempSync(path.join(os.tmpdir(), 'owa-export-err-'));const cwdSpy = jest.spyOn(process, 'cwd').mockReturnValue(cwd);diff --git a/src/modules/infra/infra-storage.controller.ts b/src/modules/infra/infra-storage.controller.tsindex 397b12082..9f8596807 100644--- a/src/modules/infra/infra-storage.controller.ts+++ b/src/modules/infra/infra-storage.controller.ts@@ -156,7 +156,12 @@ export class InfraStorageController implements OnApplicationBootstrap {await new Promise<void>((resolve, reject) => {writeStream.on('finish', resolve);- writeStream.on('error', reject);+ // The archive is filled while it is written: destroying the source stops the export from+ // opening further files once the sink has failed.+ writeStream.on('error', (err: Error) => {+ stream.destroy();+ reject(err);+ });// pipe() does NOT forward source errors: an archiver/gzip failure surfaces as an 'error' event on// the source stream, which without a listener crashes the process. Fail the request instead and// tear down the sink so its fd isn't held open waiting for a 'finish' that never comes.diff --git a/test/jest-e2e.json b/test/jest-e2e.jsonindex 75824be58..37642d36d 100644--- a/test/jest-e2e.json+++ b/test/jest-e2e.json@@ -8,7 +8,7 @@"^.+\\.(t|j)s$": "ts-jest"},"transformIgnorePatterns": [- "node_modules/(?!(archiver|compress-commons|crc32-stream|zip-stream|archiver-utils|https-proxy-agent|socks-proxy-agent|agent-base|proxy-agent-negotiate)/)"+ "node_modules/(?!(archiver|compress-commons|crc32-stream|zip-stream|archiver-utils|is-stream|readdir-glob|https-proxy-agent|socks-proxy-agent|agent-base|proxy-agent-negotiate)/)"],"moduleNameMapper": {"^puppeteer$": "<rootDir>/__mocks__/puppeteer.ts"diff --git a/test/storage-export-streaming.e2e-spec.ts b/test/storage-export-streaming.e2e-spec.tsnew file mode 100644index 000000000..eeb61f0ef--- /dev/null+++ b/test/storage-export-streaming.e2e-spec.ts@@ -0,0 +1,134 @@+// test/__mocks__/archiver.ts replaces archiver in every suite under test/; this one needs the real one.+jest.unmock('archiver');++import { randomBytes } from 'crypto';+import { Readable } from 'stream';+import { createExportStream, ExportFileSource, importFromStream } from '../src/common/storage/storage-transfer';++/**+ * The storage export runs against the real archiver here (the unit config stubs it). An export of a+ * large media store must not read every file into memory before anything consumes the archive:+ * files are opened one at a time, each only after the previous entry was written through.+ */+describe('Storage export streams one file at a time (e2e, real archiver)', () => {+ const FILE_BYTES = 1024 * 1024;++ const makeLogger = () => ({ warn: jest.fn(), error: jest.fn(), log: jest.fn(), debug: jest.fn() });++ // Random bytes do not compress, so gzip cannot hide a file inside a few kilobytes of stream buffer.+ const contents = new Map<string, Buffer>();+ const contentFor = (name: string): Buffer => {+ if (!contents.has(name)) contents.set(name, randomBytes(FILE_BYTES));+ return contents.get(name)!;+ };++ /** An openFile that tracks how many file streams are open at once and how many were ever opened. */+ function trackingOpener(failing: Record<string, 'open' | 'read'> = {}) {+ const stats = { opened: 0, open: 0, maxOpen: 0 };+ const openFile = (name: string): Promise<ExportFileSource> => {+ if (failing[name] === 'open') return Promise.reject(new Error(`ENOENT: ${name}`));+ const data = contentFor(name);+ stats.opened++;+ stats.open++;+ stats.maxOpen = Math.max(stats.maxOpen, stats.open);+ let offset = 0;+ const stream = new Readable({+ highWaterMark: 16 * 1024,+ read() {+ if (failing[name] === 'read' && offset > 0) {+ this.destroy(new Error(`read failed: ${name}`));+ return;+ }+ const chunk = data.subarray(offset, offset + 16 * 1024);+ offset += chunk.length;+ this.push(chunk.length ? chunk : null);+ },+ });+ stream.once('close', () => stats.open--);+ return Promise.resolve({ stream, size: data.length });+ };+ return { stats, openFile };+ }++ const files = Array.from({ length: 12 }, (_, i) => `media/file-${String.fromCharCode(97 + i)}.bin`);++ it('returns before reading the store and opens no further files while nothing consumes the output', async () => {+ const { stats, openFile } = trackingOpener();+ const output = await createExportStream(() => Promise.resolve(files), openFile, makeLogger() as never);++ // Let the background loop run as far as it can without a reader. The pipeline's own buffers (about+ // a megabyte in archiver) absorb a file or two; everything past that has to wait for the reader.+ await new Promise(resolve => setTimeout(resolve, 300));+ expect(stats.opened).toBeLessThanOrEqual(3);+ output.destroy();+ });++ it('never holds more than one file stream open, and the archive round-trips through the importer', async () => {+ const { stats, openFile } = trackingOpener({ 'media/file-c.bin': 'open' });+ const logger = makeLogger();+ const output = await createExportStream(() => Promise.resolve(files), openFile, logger as never);++ const imported = new Map<string, Buffer>();+ const count = await importFromStream(+ output,+ (name, data) => {+ imported.set(name, data);+ return Promise.resolve();+ },+ logger as never,+ );++ expect(stats.maxOpen).toBe(1);+ expect(stats.open).toBe(0);+ // The unopenable file is skipped with a warning, as before; every other file arrives intact.+ expect(logger.warn).toHaveBeenCalledWith('Failed to export file: media/file-c.bin', expect.anything());+ expect(count).toBe(files.length - 1);+ for (const name of files.filter(f => f !== 'media/file-c.bin')) {+ expect(imported.get(name)?.equals(contentFor(name))).toBe(true);+ }+ });++ it('fails the output with the read error when a file fails part-way, and stops opening files', async () => {+ const { stats, openFile } = trackingOpener({ 'media/file-b.bin': 'read' });+ const output = await createExportStream(() => Promise.resolve(files), openFile, makeLogger() as never);++ const error = await new Promise<Error>(resolve => {+ output.on('error', resolve);+ output.resume();+ });++ expect(error.message).toBe('read failed: media/file-b.bin');+ await new Promise(resolve => setTimeout(resolve, 50));+ expect(stats.opened).toBe(2);+ expect(stats.open).toBe(0);+ });++ // The tar entry declares the size seen at open time. A file that is rewritten before it is read+ // yields a different length, and handing that to the tar sink raises an error nothing can catch.+ it.each([+ ['longer', 2 * FILE_BYTES],+ ['shorter', FILE_BYTES / 2],+ ])('fails the output, without an uncaught exception, when a file reads %s than its size', async (_, length) => {+ const uncaught: Error[] = [];+ const onUncaught = (err: Error): void => void uncaught.push(err);+ process.on('uncaughtException', onUncaught);+ try {+ const openFile = (name: string): Promise<ExportFileSource> =>+ Promise.resolve({+ stream: Readable.from([randomBytes(length)]),+ size: name === 'media/file-b.bin' ? FILE_BYTES : length,+ });+ const output = await createExportStream(() => Promise.resolve(files), openFile, makeLogger() as never);++ const error = await new Promise<Error>(resolve => {+ output.on('error', resolve);+ output.resume();+ });+ expect(error.message).toMatch(/media\/file-b\.bin/);+ await new Promise(resolve => setTimeout(resolve, 100));+ expect(uncaught).toEqual([]);+ } finally {+ process.removeListener('uncaughtException', onUncaught);+ }+ });+});