Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(storage): stream the export one file at a time

Yudhi Armyndharis2c091fa91f49
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3ce9b6e97..480b2a5b1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -20,6 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- A session whose automatic reconnect fails to relaunch the engine (a network, DNS or browser launch error) keeps retrying with backoff instead of stopping in `failed` until restarted by hand; an authentication failure or a stale browser profile still ends in `failed` ([#1580](https://github.com/rmyndharis/OpenWA/issues/1580)).
- A session that runs out of reconnect attempts fires the `session:error` plugin hook when it lands in `failed`.
- An explicit `maxReconnectAttempts` is honoured and the reconnect delay is capped at 5 minutes; the budget used to restart once the backoff passed 5 minutes, so the limit was never reached and the documented 1-hour cap never applied.
+- `GET /api/infra/storage/export` streams files into the archive one at a time instead of loading the whole media store into memory first, so exporting a large local or S3 store no longer exhausts memory.
## [0.23.5] - 2026-09-14
diff --git a/src/common/storage/storage-local-files.ts b/src/common/storage/storage-local-files.ts
index 31733432b..c71cee96c 100644
--- a/src/common/storage/storage-local-files.ts
+++ b/src/common/storage/storage-local-files.ts
@@ -1,5 +1,6 @@
import * as fs from 'fs';
import * as path from 'path';
+import { Readable } from 'stream';
import { isPathWithin } from '../utils/path-safety';
/** Max number of local files a single traversal enumerates. Bounds a count DoS on a huge media dir. */
@@ -67,11 +68,29 @@ export function getLocalFile(localPath: string, filePath: string): Promise<Buffe
throw new Error(`Refusing to read outside storage root: ${filePath}`);
}
const fullPath = path.join(localPath, filePath);
- // Async read so the export loop (the only caller) yields the event loop per file instead of
- // blocking it with a synchronous read for every media file.
+ // Async read so a media read yields the event loop instead of blocking it for the whole file.
return fs.promises.readFile(fullPath);
}
+/**
+ * Open a local file for a streamed read. The handle is opened and stat'ed up front, so a missing or
+ * unreadable file rejects here, before the caller commits to it, and the size is the one the open
+ * handle sees. The stream closes the handle when it ends or is destroyed.
+ */
+export async function openLocalFile(localPath: string, filePath: string): Promise<{ stream: Readable; size: number }> {
+ if (!isPathWithin(localPath, filePath)) {
+ throw new Error(`Refusing to read outside storage root: ${filePath}`);
+ }
+ const handle = await fs.promises.open(path.join(localPath, filePath), 'r');
+ try {
+ const { size } = await handle.stat();
+ return { stream: handle.createReadStream(), size };
+ } catch (error) {
+ await handle.close();
+ throw error;
+ }
+}
+
export async function putLocalFile(localPath: string, filePath: string, data: Buffer): Promise<void> {
if (!isPathWithin(localPath, filePath)) {
throw new Error(`Refusing to write outside storage root: ${filePath}`);
diff --git a/src/common/storage/storage-transfer.spec.ts b/src/common/storage/storage-transfer.spec.ts
new file mode 100644
index 000000000..ab60abd3a
--- /dev/null
+++ b/src/common/storage/storage-transfer.spec.ts
@@ -0,0 +1,164 @@
+import { randomBytes } from 'crypto';
+import { PassThrough, Readable } from 'stream';
+
+// archiver v8 is ESM-only and ts-jest cannot load it here. This stand-in keeps the parts of its
+// TarArchive contract the export relies on: a stream entry with a known size is piped straight into
+// the tar pack (without one it is collected first), 'entry' fires once the entry is written, and an
+// entry failure is emitted as 'error'. The real archiver runs in test/storage-export-streaming.e2e-spec.ts.
+jest.mock('archiver', () => {
+ /* eslint-disable @typescript-eslint/no-require-imports */
+ const { EventEmitter } = require('events') as typeof import('events');
+ const tar = require('tar-stream') as typeof import('tar-stream');
+ const { createGzip } = require('zlib') as typeof import('zlib');
+ /* eslint-enable @typescript-eslint/no-require-imports */
+ class TarArchive extends EventEmitter {
+ private readonly pack = tar.pack();
+ private readonly gzip = createGzip();
+ constructor() {
+ super();
+ this.pack.pipe(this.gzip);
+ }
+ pipe(destination: NodeJS.WritableStream): NodeJS.WritableStream {
+ return this.gzip.pipe(destination);
+ }
+ append(source: Readable, data: { name: string; stats?: { size: number } }): void {
+ const done = (err?: Error | null): void => void (err ? this.emit('error', err) : this.emit('entry', data));
+ if (data.stats) {
+ source.pipe(this.pack.entry({ name: data.name, size: data.stats.size }, done));
+ return;
+ }
+ const chunks: Buffer[] = [];
+ source.on('data', (chunk: Buffer) => chunks.push(chunk));
+ source.on('end', () => this.pack.entry({ name: data.name }, Buffer.concat(chunks), done));
+ }
+ finalize(): Promise<void> {
+ this.pack.finalize();
+ return Promise.resolve();
+ }
+ abort(): void {
+ this.pack.destroy();
+ }
+ }
+ return { TarArchive };
+});
+
+import { createExportStream, ExportFileSource, importFromStream } from './storage-transfer';
+
+const FILE_BYTES = 256 * 1024;
+const files = Array.from({ length: 12 }, (_, i) => `media/file-${String.fromCharCode(97 + i)}.bin`);
+
+const makeLogger = () => ({ warn: jest.fn(), error: jest.fn(), log: jest.fn(), debug: jest.fn() });
+
+// Random bytes do not compress, so gzip cannot hide a file inside a few kilobytes of stream buffer.
+const contents = new Map<string, Buffer>();
+const contentFor = (name: string): Buffer => {
+ if (!contents.has(name)) contents.set(name, randomBytes(FILE_BYTES));
+ return contents.get(name)!;
+};
+
+/** An openFile that counts the file streams open at once and the files ever opened. */
+function trackingOpener(failing: Record<string, 'open' | 'read'> = {}, withSize = true) {
+ const stats = { opened: 0, open: 0, maxOpen: 0 };
+ const openFile = (name: string): Promise<ExportFileSource> => {
+ if (failing[name] === 'open') return Promise.reject(new Error(`ENOENT: ${name}`));
+ const data = contentFor(name);
+ stats.opened++;
+ stats.open++;
+ stats.maxOpen = Math.max(stats.maxOpen, stats.open);
+ let offset = 0;
+ const stream = new Readable({
+ highWaterMark: 16 * 1024,
+ read() {
+ if (failing[name] === 'read' && offset > 0) {
+ this.destroy(new Error(`read failed: ${name}`));
+ return;
+ }
+ const chunk = data.subarray(offset, offset + 16 * 1024);
+ offset += chunk.length;
+ this.push(chunk.length ? chunk : null);
+ },
+ });
+ stream.once('close', () => stats.open--);
+ return Promise.resolve({ stream, size: withSize ? data.length : undefined });
+ };
+ return { stats, openFile };
+}
+
+const settle = (ms: number): Promise<void> => new Promise(resolve => setTimeout(resolve, ms));
+
+async function importAll(output: PassThrough, logger: object): Promise<Map<string, Buffer>> {
+ const imported = new Map<string, Buffer>();
+ await importFromStream(
+ output,
+ (name, data) => {
+ imported.set(name, data);
+ return Promise.resolve();
+ },
+ logger as never,
+ );
+ return imported;
+}
+
+describe('createExportStream streams one file at a time', () => {
+ it('returns before reading the store and opens no further files while nothing consumes the output', async () => {
+ const { stats, openFile } = trackingOpener();
+ const output = await createExportStream(() => Promise.resolve(files), openFile, makeLogger() as never);
+
+ await settle(100);
+ // Stream buffers absorb a file or so; every file past that waits for a reader.
+ expect(stats.opened).toBeLessThanOrEqual(2);
+ output.destroy();
+ });
+
+ it('holds one file stream open at a time, skips an unopenable file, and round-trips the rest', async () => {
+ const { stats, openFile } = trackingOpener({ 'media/file-c.bin': 'open' });
+ const logger = makeLogger();
+ const output = await createExportStream(() => Promise.resolve(files), openFile, logger as never);
+
+ const imported = await importAll(output, logger);
+
+ expect(stats.maxOpen).toBe(1);
+ expect(stats.open).toBe(0);
+ expect(logger.warn).toHaveBeenCalledWith('Failed to export file: media/file-c.bin', expect.anything());
+ expect([...imported.keys()].sort()).toEqual(files.filter(f => f !== 'media/file-c.bin'));
+ for (const [name, data] of imported) expect(data.equals(contentFor(name))).toBe(true);
+ });
+
+ it('still exports a file whose size the backend did not report', async () => {
+ const { openFile } = trackingOpener({}, false);
+ const logger = makeLogger();
+ const output = await createExportStream(() => Promise.resolve(files.slice(0, 2)), openFile, logger as never);
+
+ const imported = await importAll(output, logger);
+
+ expect(imported.get(files[1])?.equals(contentFor(files[1]))).toBe(true);
+ });
+
+ it('fails the output with a read error part-way through a file and stops opening files', async () => {
+ const { stats, openFile } = trackingOpener({ 'media/file-b.bin': 'read' });
+ const output = await createExportStream(() => Promise.resolve(files), openFile, makeLogger() as never);
+
+ const error = await new Promise<Error>(resolve => {
+ output.on('error', resolve);
+ output.resume();
+ });
+ await settle(20);
+
+ expect(error.message).toBe('read failed: media/file-b.bin');
+ expect(stats.opened).toBe(2);
+ expect(stats.open).toBe(0);
+ });
+
+ it('stops and releases the open file when the consumer destroys the output', async () => {
+ const { stats, openFile } = trackingOpener();
+ const output = await createExportStream(() => Promise.resolve(files), openFile, makeLogger() as never);
+ await settle(50);
+ const openedBefore = stats.opened;
+
+ output.destroy();
+ await settle(50);
+
+ expect(stats.open).toBe(0);
+ expect(stats.opened).toBe(openedBefore);
+ });
+});
diff --git a/src/common/storage/storage-transfer.ts b/src/common/storage/storage-transfer.ts
index 9de62a85e..479d14c52 100644
--- a/src/common/storage/storage-transfer.ts
+++ b/src/common/storage/storage-transfer.ts
@@ -1,7 +1,8 @@
import { TarArchive } from 'archiver';
import * as tar from 'tar-stream';
import { createGunzip } from 'zlib';
-import { Readable, PassThrough } from 'stream';
+import * as fs from 'fs';
+import { Readable, PassThrough, Transform, pipeline } from 'stream';
import { LoggerService } from '../services/logger.service';
/** Per-entry buffer cap for an import (200 MiB — 4× the inbound media cap). Bounds a decompression bomb. */
@@ -18,9 +19,23 @@ function positiveIntFromEnv(name: string, fallback: number): number {
// Export - Create tar.gz stream from current storage
// ============================================================================
+/**
+ * A file opened for export. `size` is the byte length the tar header declares; without it archiver
+ * has to buffer that one entry to learn its length.
+ */
+export interface ExportFileSource {
+ stream: Readable;
+ size?: number;
+}
+
+// Returns the output as soon as the file list is known and fills the archive in the background, one
+// entry at a time: each file is opened only after the previous entry has been written through to the
+// consumer, so memory holds one file's in-flight chunks, not the whole store. A file that cannot be
+// opened is skipped with a warning; a read that fails part-way destroys the output with that error,
+// because its tar header is already written and skipping it would leave a corrupt archive.
export async function createExportStream(
listFiles: () => Promise<string[]>,
- getFile: (filePath: string) => Promise<Buffer>,
+ openFile: (filePath: string) => Promise<ExportFileSource>,
logger: LoggerService,
): Promise<PassThrough> {
const files = await listFiles();
@@ -56,20 +71,81 @@ export async function createExportStream(
archive.pipe(output);
- // Add files to archive
+ appendEntries(files, openFile, archive, output, logger).catch((err: Error) => output.destroy(err));
+ return output;
+}
+
+function enforceSize(file: string, size: number | undefined): Transform {
+ let seen = 0;
+ const mismatch = (): Error => new Error(`Export failed: ${file} changed size while being read (expected ${size})`);
+ return new Transform({
+ transform(chunk: Buffer, _encoding, callback) {
+ seen += chunk.length;
+ callback(size !== undefined && seen > size ? mismatch() : null, chunk);
+ },
+ flush(callback) {
+ callback(size !== undefined && seen < size ? mismatch() : null);
+ },
+ });
+}
+
+async function appendEntries(
+ files: string[],
+ openFile: (filePath: string) => Promise<ExportFileSource>,
+ archive: TarArchive,
+ output: PassThrough,
+ logger: LoggerService,
+): Promise<void> {
for (const file of files) {
+ let source: ExportFileSource;
try {
- const data = await getFile(file);
- archive.append(data, { name: file });
+ source = await openFile(file);
} catch (error) {
logger.warn(`Failed to export file: ${file}`, { error: String(error) });
+ continue;
+ }
+ const { stream, size } = source;
+ // The output closes early when the archive fails or the consumer goes away: stop reading files.
+ // Checked synchronously before waiting, so a 'close' that already fired cannot be missed.
+ if (output.destroyed) {
+ stream.destroy();
+ break;
+ }
+ // The tar sink fails with an error nothing can listen for when an entry's length differs from the
+ // size in its header, which would crash the process. A file rewritten between open and read does
+ // that, so the length is checked here and a mismatch fails the export like a read error, before
+ // the sink sees the extra byte or the early end. pipeline() also forwards a source error, which
+ // pipe() does not.
+ const entry = pipeline(stream, enforceSize(file, size), err => {
+ if (!err || output.destroyed) return;
+ logger.error(`Export failed reading file: ${file}`, String(err));
+ output.destroy(err);
+ });
+ await new Promise<void>(resolve => {
+ const done = (): void => {
+ archive.removeListener('entry', done);
+ output.removeListener('close', done);
+ resolve();
+ };
+ archive.on('entry', done);
+ output.on('close', done);
+ // Mode and date are set explicitly: archiver derives both from `stats` when present, and this
+ // stand-in carries only the size.
+ const stats = size === undefined ? undefined : ({ size } as fs.Stats);
+ archive.append(entry, { name: file, mode: 0o644, date: new Date(), stats });
+ });
+ if (output.destroyed) {
+ stream.destroy();
+ break;
}
}
-
+ if (output.destroyed) {
+ archive.abort();
+ return;
+ }
// finalize() rejections also emit via the 'error' handler above; catch the promise so it
// never surfaces as an unhandled rejection.
archive.finalize().catch(() => undefined);
- return output;
}
// ============================================================================
diff --git a/src/common/storage/storage.service.s3-reprobe.spec.ts b/src/common/storage/storage.service.s3-reprobe.spec.ts
index b0ebcabf6..159b3007a 100644
--- a/src/common/storage/storage.service.s3-reprobe.spec.ts
+++ b/src/common/storage/storage.service.s3-reprobe.spec.ts
@@ -219,6 +219,33 @@ describe('StorageService S3 re-probe and recovery', () => {
expect(data.toString()).toBe('from-s3');
});
+ it('openFile hands the S3 body through unread, with the object length as its size', async () => {
+ const body = Readable.from([Buffer.from('from-s3')]);
+ mockSend.mockImplementation((cmd: unknown) => {
+ if (cmd instanceof GetObjectCommand) return Promise.resolve({ Body: body, ContentLength: 7 });
+ return Promise.resolve({});
+ });
+ const svc = new StorageService(makeConfig());
+ await flush();
+
+ await expect(svc.openFile('present.bin')).resolves.toEqual({ stream: body, size: 7 });
+ });
+
+ it('openFile reads a missing S3 object through from the local fallback dir, and rejects when neither has it', async () => {
+ mockSend.mockImplementation((cmd: unknown) => {
+ if (cmd instanceof GetObjectCommand) return Promise.reject(s3Error('NoSuchKey'));
+ return Promise.resolve({});
+ });
+ const svc = new StorageService(makeConfig());
+ await flush();
+ fs.writeFileSync(path.join(localPath, 'gap.bin'), 'gap-media');
+
+ const { stream, size } = await svc.openFile('gap.bin');
+ stream.destroy();
+ expect(size).toBe(9);
+ await expect(svc.openFile('missing.bin')).rejects.toThrow('NoSuchKey');
+ });
+
it('deleteFile removes the local fallback copy as well as the S3 object', async () => {
mockSend.mockResolvedValue({}); // HeadBucket at boot, DeleteObject later
const svc = new StorageService(makeConfig());
diff --git a/src/common/storage/storage.service.spec.ts b/src/common/storage/storage.service.spec.ts
index 8f048f1aa..6b2957777 100644
--- a/src/common/storage/storage.service.spec.ts
+++ b/src/common/storage/storage.service.spec.ts
@@ -147,6 +147,29 @@ describe('StorageService put/getFile containment is backend-agnostic', () => {
});
});
+describe('StorageService.openFile (the export read path)', () => {
+ const readAll = async (stream: Readable): Promise<Buffer> => {
+ const chunks: Buffer[] = [];
+ for await (const chunk of stream) chunks.push(chunk as Buffer);
+ return Buffer.concat(chunks);
+ };
+
+ it('streams a local file with its size, and rejects a missing file before any stream exists', async () => {
+ const { service, baseDir, localPath } = makeLocalService();
+ fs.mkdirSync(path.join(localPath, 'sub'), { recursive: true });
+ fs.writeFileSync(path.join(localPath, 'sub/a.bin'), 'local-bytes');
+
+ const { stream, size } = await service.openFile('sub/a.bin');
+ expect(size).toBe(11);
+ expect((await readAll(stream)).toString()).toBe('local-bytes');
+ expect((await service.getFile('sub/a.bin')).toString()).toBe('local-bytes');
+ await expect(service.openFile('sub/missing.bin')).rejects.toThrow(/ENOENT/);
+ await expect(service.openFile('../../etc/passwd')).rejects.toThrow(/unsafe storage key/);
+
+ fs.rmSync(baseDir, { recursive: true, force: true });
+ });
+});
+
describe('StorageService getFileCount (S3 size)', () => {
it('sums the real Size of each S3 object instead of estimating', async () => {
const { service, baseDir } = makeLocalService();
diff --git a/src/common/storage/storage.service.ts b/src/common/storage/storage.service.ts
index f5d30516a..618638619 100644
--- a/src/common/storage/storage.service.ts
+++ b/src/common/storage/storage.service.ts
@@ -14,8 +14,15 @@ import {
} from '@aws-sdk/client-s3';
import { createLogger } from '../services/logger.service';
import { isSafeStorageKey } from '../utils/path-safety';
-import { createExportStream, importFromStream } from './storage-transfer';
-import { listLocalFiles, iterateLocalFiles, getLocalFile, putLocalFile, deleteLocalFile } from './storage-local-files';
+import { createExportStream, ExportFileSource, importFromStream } from './storage-transfer';
+import {
+ listLocalFiles,
+ iterateLocalFiles,
+ getLocalFile,
+ openLocalFile,
+ putLocalFile,
+ deleteLocalFile,
+} from './storage-local-files';
interface S3Config {
endpoint?: string;
@@ -285,6 +292,20 @@ export class StorageService implements OnModuleDestroy {
return this.getLocalFile(filePath);
}
+ /**
+ * Open a file as a stream instead of reading it into memory, with the same key guard and S3
+ * read-through as getFile. A missing or unreadable file rejects here rather than on the stream.
+ */
+ async openFile(filePath: string): Promise<ExportFileSource> {
+ if (!isSafeStorageKey(filePath)) {
+ throw new Error(`Refusing to read an unsafe storage key: ${filePath}`);
+ }
+ if (this.storageType === 's3' && this.s3Client && this.s3Available) {
+ return this.openS3File(filePath);
+ }
+ return openLocalFile(this.localPath, filePath);
+ }
+
async putFile(filePath: string, data: Buffer): Promise<void> {
// Centralized containment so BOTH backends inherit it: putLocalFile has its own isPathWithin
// guard, but putS3File builds `media/${filePath}` with none — reject a traversing key here.
@@ -401,7 +422,7 @@ export class StorageService implements OnModuleDestroy {
// could not reveal the gap. An export exists to be complete; that is what the uncapped walk is for.
return createExportStream(
() => this.listAllFiles(),
- filePath => this.getFile(filePath),
+ filePath => this.openFile(filePath),
this.logger,
);
}
@@ -489,6 +510,15 @@ export class StorageService implements OnModuleDestroy {
}
private async getS3File(filePath: string): Promise<Buffer> {
+ const { stream } = await this.openS3File(filePath);
+ const chunks: Buffer[] = [];
+ for await (const chunk of stream) {
+ chunks.push(Buffer.from(chunk as ArrayBuffer));
+ }
+ return Buffer.concat(chunks);
+ }
+
+ private async openS3File(filePath: string): Promise<ExportFileSource> {
if (!this.s3Client) throw new Error('S3 client not initialized');
let response;
@@ -505,26 +535,19 @@ export class StorageService implements OnModuleDestroy {
// fine during the outage). Fall through to the local copy; if there is none, surface the
// original S3 error so "not found" semantics are unchanged.
if ((error as { name?: string }).name !== 'NoSuchKey') throw error;
+ let local: ExportFileSource;
try {
- const local = await this.getLocalFile(filePath);
- this.logger.debug(`Served '${filePath}' from the local fallback dir (not yet in S3)`);
- return local;
+ local = await openLocalFile(this.localPath, filePath);
} catch {
throw error;
}
+ this.logger.debug(`Served '${filePath}' from the local fallback dir (not yet in S3)`);
+ return local;
}
if (!response.Body) throw new Error('Empty response body');
- // Convert stream to buffer
- const chunks: Buffer[] = [];
- const stream = response.Body as Readable;
-
- for await (const chunk of stream) {
- chunks.push(Buffer.from(chunk as ArrayBuffer));
- }
-
- return Buffer.concat(chunks);
+ return { stream: response.Body as Readable, size: response.ContentLength };
}
private async putS3File(filePath: string, data: Buffer): Promise<void> {
diff --git a/src/modules/infra/infra-storage.controller.spec.ts b/src/modules/infra/infra-storage.controller.spec.ts
index f59bea0a9..dbb521fec 100644
--- a/src/modules/infra/infra-storage.controller.spec.ts
+++ b/src/modules/infra/infra-storage.controller.spec.ts
@@ -1,7 +1,7 @@
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
-import { Readable } from 'stream';
+import { Readable, Writable } from 'stream';
// StorageService (imported transitively by the infra controllers) pulls in `archiver`
// v8, which is ESM-only and cannot be parsed by ts-jest. The controller logic
@@ -22,6 +22,7 @@ jest.mock('fs', () => {
existsSync: jest.fn().mockReturnValue(false),
readFileSync: jest.fn().mockReturnValue(''),
createReadStream: jest.fn(() => jest.requireActual<typeof import('stream')>('stream').Readable.from([])),
+ createWriteStream: jest.fn(actual.createWriteStream),
};
});
@@ -225,6 +226,32 @@ describe('InfraStorageController storage stream failures surface as request erro
}
});
+ it('exportStorage destroys the export stream when the archive file cannot be written', async () => {
+ const cwd = fs.mkdtempSync(path.join(os.tmpdir(), 'owa-export-sink-'));
+ const cwdSpy = jest.spyOn(process, 'cwd').mockReturnValue(cwd);
+ const sink = new Writable({
+ write(_chunk, _encoding, callback) {
+ callback(new Error('disk full'));
+ },
+ });
+ (fs.createWriteStream as jest.Mock).mockReturnValueOnce(sink);
+ try {
+ // The export fills the archive while it is being written; a failed sink must stop it, or it
+ // keeps a file open waiting for a reader that is gone.
+ const source = new Readable({
+ read() {
+ this.push(Buffer.alloc(1024));
+ },
+ });
+ const storage = { createExportStream: jest.fn().mockResolvedValue(source) };
+ await expect(new InfraStorageController(storage as never).exportStorage()).rejects.toThrow(/disk full/);
+ expect(source.destroyed).toBe(true);
+ } finally {
+ cwdSpy.mockRestore();
+ fs.rmSync(cwd, { recursive: true, force: true });
+ }
+ });
+
it('exportStorage rejects (and the process lives) when the export source stream errors', async () => {
const cwd = fs.mkdtempSync(path.join(os.tmpdir(), 'owa-export-err-'));
const cwdSpy = jest.spyOn(process, 'cwd').mockReturnValue(cwd);
diff --git a/src/modules/infra/infra-storage.controller.ts b/src/modules/infra/infra-storage.controller.ts
index 397b12082..9f8596807 100644
--- a/src/modules/infra/infra-storage.controller.ts
+++ b/src/modules/infra/infra-storage.controller.ts
@@ -156,7 +156,12 @@ export class InfraStorageController implements OnApplicationBootstrap {
await new Promise<void>((resolve, reject) => {
writeStream.on('finish', resolve);
- writeStream.on('error', reject);
+ // The archive is filled while it is written: destroying the source stops the export from
+ // opening further files once the sink has failed.
+ writeStream.on('error', (err: Error) => {
+ stream.destroy();
+ reject(err);
+ });
// pipe() does NOT forward source errors: an archiver/gzip failure surfaces as an 'error' event on
// the source stream, which without a listener crashes the process. Fail the request instead and
// tear down the sink so its fd isn't held open waiting for a 'finish' that never comes.
diff --git a/test/jest-e2e.json b/test/jest-e2e.json
index 75824be58..37642d36d 100644
--- a/test/jest-e2e.json
+++ b/test/jest-e2e.json
@@ -8,7 +8,7 @@
"^.+\\.(t|j)s$": "ts-jest"
},
"transformIgnorePatterns": [
- "node_modules/(?!(archiver|compress-commons|crc32-stream|zip-stream|archiver-utils|https-proxy-agent|socks-proxy-agent|agent-base|proxy-agent-negotiate)/)"
+ "node_modules/(?!(archiver|compress-commons|crc32-stream|zip-stream|archiver-utils|is-stream|readdir-glob|https-proxy-agent|socks-proxy-agent|agent-base|proxy-agent-negotiate)/)"
],
"moduleNameMapper": {
"^puppeteer$": "<rootDir>/__mocks__/puppeteer.ts"
diff --git a/test/storage-export-streaming.e2e-spec.ts b/test/storage-export-streaming.e2e-spec.ts
new file mode 100644
index 000000000..eeb61f0ef
--- /dev/null
+++ b/test/storage-export-streaming.e2e-spec.ts
@@ -0,0 +1,134 @@
+// test/__mocks__/archiver.ts replaces archiver in every suite under test/; this one needs the real one.
+jest.unmock('archiver');
+
+import { randomBytes } from 'crypto';
+import { Readable } from 'stream';
+import { createExportStream, ExportFileSource, importFromStream } from '../src/common/storage/storage-transfer';
+
+/**
+ * The storage export runs against the real archiver here (the unit config stubs it). An export of a
+ * large media store must not read every file into memory before anything consumes the archive:
+ * files are opened one at a time, each only after the previous entry was written through.
+ */
+describe('Storage export streams one file at a time (e2e, real archiver)', () => {
+ const FILE_BYTES = 1024 * 1024;
+
+ const makeLogger = () => ({ warn: jest.fn(), error: jest.fn(), log: jest.fn(), debug: jest.fn() });
+
+ // Random bytes do not compress, so gzip cannot hide a file inside a few kilobytes of stream buffer.
+ const contents = new Map<string, Buffer>();
+ const contentFor = (name: string): Buffer => {
+ if (!contents.has(name)) contents.set(name, randomBytes(FILE_BYTES));
+ return contents.get(name)!;
+ };
+
+ /** An openFile that tracks how many file streams are open at once and how many were ever opened. */
+ function trackingOpener(failing: Record<string, 'open' | 'read'> = {}) {
+ const stats = { opened: 0, open: 0, maxOpen: 0 };
+ const openFile = (name: string): Promise<ExportFileSource> => {
+ if (failing[name] === 'open') return Promise.reject(new Error(`ENOENT: ${name}`));
+ const data = contentFor(name);
+ stats.opened++;
+ stats.open++;
+ stats.maxOpen = Math.max(stats.maxOpen, stats.open);
+ let offset = 0;
+ const stream = new Readable({
+ highWaterMark: 16 * 1024,
+ read() {
+ if (failing[name] === 'read' && offset > 0) {
+ this.destroy(new Error(`read failed: ${name}`));
+ return;
+ }
+ const chunk = data.subarray(offset, offset + 16 * 1024);
+ offset += chunk.length;
+ this.push(chunk.length ? chunk : null);
+ },
+ });
+ stream.once('close', () => stats.open--);
+ return Promise.resolve({ stream, size: data.length });
+ };
+ return { stats, openFile };
+ }
+
+ const files = Array.from({ length: 12 }, (_, i) => `media/file-${String.fromCharCode(97 + i)}.bin`);
+
+ it('returns before reading the store and opens no further files while nothing consumes the output', async () => {
+ const { stats, openFile } = trackingOpener();
+ const output = await createExportStream(() => Promise.resolve(files), openFile, makeLogger() as never);
+
+ // Let the background loop run as far as it can without a reader. The pipeline's own buffers (about
+ // a megabyte in archiver) absorb a file or two; everything past that has to wait for the reader.
+ await new Promise(resolve => setTimeout(resolve, 300));
+ expect(stats.opened).toBeLessThanOrEqual(3);
+ output.destroy();
+ });
+
+ it('never holds more than one file stream open, and the archive round-trips through the importer', async () => {
+ const { stats, openFile } = trackingOpener({ 'media/file-c.bin': 'open' });
+ const logger = makeLogger();
+ const output = await createExportStream(() => Promise.resolve(files), openFile, logger as never);
+
+ const imported = new Map<string, Buffer>();
+ const count = await importFromStream(
+ output,
+ (name, data) => {
+ imported.set(name, data);
+ return Promise.resolve();
+ },
+ logger as never,
+ );
+
+ expect(stats.maxOpen).toBe(1);
+ expect(stats.open).toBe(0);
+ // The unopenable file is skipped with a warning, as before; every other file arrives intact.
+ expect(logger.warn).toHaveBeenCalledWith('Failed to export file: media/file-c.bin', expect.anything());
+ expect(count).toBe(files.length - 1);
+ for (const name of files.filter(f => f !== 'media/file-c.bin')) {
+ expect(imported.get(name)?.equals(contentFor(name))).toBe(true);
+ }
+ });
+
+ it('fails the output with the read error when a file fails part-way, and stops opening files', async () => {
+ const { stats, openFile } = trackingOpener({ 'media/file-b.bin': 'read' });
+ const output = await createExportStream(() => Promise.resolve(files), openFile, makeLogger() as never);
+
+ const error = await new Promise<Error>(resolve => {
+ output.on('error', resolve);
+ output.resume();
+ });
+
+ expect(error.message).toBe('read failed: media/file-b.bin');
+ await new Promise(resolve => setTimeout(resolve, 50));
+ expect(stats.opened).toBe(2);
+ expect(stats.open).toBe(0);
+ });
+
+ // The tar entry declares the size seen at open time. A file that is rewritten before it is read
+ // yields a different length, and handing that to the tar sink raises an error nothing can catch.
+ it.each([
+ ['longer', 2 * FILE_BYTES],
+ ['shorter', FILE_BYTES / 2],
+ ])('fails the output, without an uncaught exception, when a file reads %s than its size', async (_, length) => {
+ const uncaught: Error[] = [];
+ const onUncaught = (err: Error): void => void uncaught.push(err);
+ process.on('uncaughtException', onUncaught);
+ try {
+ const openFile = (name: string): Promise<ExportFileSource> =>
+ Promise.resolve({
+ stream: Readable.from([randomBytes(length)]),
+ size: name === 'media/file-b.bin' ? FILE_BYTES : length,
+ });
+ const output = await createExportStream(() => Promise.resolve(files), openFile, makeLogger() as never);
+
+ const error = await new Promise<Error>(resolve => {
+ output.on('error', resolve);
+ output.resume();
+ });
+ expect(error.message).toMatch(/media\/file-b\.bin/);
+ await new Promise(resolve => setTimeout(resolve, 100));
+ expect(uncaught).toEqual([]);
+ } finally {
+ process.removeListener('uncaughtException', onUncaught);
+ }
+ });
+});