Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(infra): cap total bytes restored by storage import (#1810)

Yudhi Armyndharis4dfba2cdd193
diff --git a/.env.example b/.env.example
index 928b53595..6c00d41fd 100644
--- a/.env.example
+++ b/.env.example
@@ -679,6 +679,7 @@ WEBHOOK_SSRF_PROTECT=true
# Storage import/export limits (ADMIN /infra/storage/* endpoints):
# STORAGE_IMPORT_MAX_BYTES=209715200 # per-entry cap for a tar.gz import; aborts on overflow (default 200 MiB)
# STORAGE_IMPORT_MAX_ENTRIES=100000 # max entries in an import archive; aborts beyond this (default 100000)
+# STORAGE_IMPORT_MAX_TOTAL_BYTES=10737418240 # total bytes one import may write; aborts on overflow (default 10 GiB)
# The two STORAGE_EXPORT_* values are raw milliseconds: a value that is not a positive integer
# (`1h` included) fails the boot, and so does a TTL above 2147483647 ms (Node's timer ceiling).
# STORAGE_EXPORT_TTL_MS=3600000 # auto-delete an export archive after this long (default 1h)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 78e87cc59..78b7f8853 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -28,6 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
+- Cap the total bytes one storage import writes at `STORAGE_IMPORT_MAX_TOTAL_BYTES`.
- Close the Baileys session proxy's fetch dispatcher when the session disconnects, logs out or is destroyed.
- Apply the Java SDK request timeout to the response body as well as the headers.
- Keep a new dashboard template draft when an earlier template save finishes.
@@ -94,6 +95,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Upgrade notes (behavior changes)
+- A storage import stops at `STORAGE_IMPORT_MAX_TOTAL_BYTES` (default 10 GiB); raise it before importing a larger export.
- Baileys answers `400` for an animated sticker with more than 500 frames.
- `backup.sh` and `restore.sh` exit with status 2 when `./.env` or `.env.generated` sets a key in a form they cannot parse; fix the line or pass the key in the environment.
- With `DATABASE_SSL=true`, `backup.sh` now fails against a PostgreSQL server whose certificate the app would reject; trust a private CA through `NODE_EXTRA_CA_CERTS` or `PGSSLROOTCERT`.
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 7a25810a9..a42fd6c06 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -267,6 +267,7 @@ services:
- STORAGE_LIST_MAX_FILES=${STORAGE_LIST_MAX_FILES:-}
- STORAGE_IMPORT_MAX_BYTES=${STORAGE_IMPORT_MAX_BYTES:-}
- STORAGE_IMPORT_MAX_ENTRIES=${STORAGE_IMPORT_MAX_ENTRIES:-}
+ - STORAGE_IMPORT_MAX_TOTAL_BYTES=${STORAGE_IMPORT_MAX_TOTAL_BYTES:-}
- STORAGE_EXPORT_TTL_MS=${STORAGE_EXPORT_TTL_MS:-}
- STORAGE_EXPORT_SWEEP_MAX_AGE_MS=${STORAGE_EXPORT_SWEEP_MAX_AGE_MS:-}
- S3_REPROBE_INTERVAL_MS=${S3_REPROBE_INTERVAL_MS:-}
diff --git a/docker-compose.yml b/docker-compose.yml
index c0a6e4b11..dbfbbb73e 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -369,6 +369,7 @@ services:
- STORAGE_LIST_MAX_FILES=${STORAGE_LIST_MAX_FILES:-}
- STORAGE_IMPORT_MAX_BYTES=${STORAGE_IMPORT_MAX_BYTES:-}
- STORAGE_IMPORT_MAX_ENTRIES=${STORAGE_IMPORT_MAX_ENTRIES:-}
+ - STORAGE_IMPORT_MAX_TOTAL_BYTES=${STORAGE_IMPORT_MAX_TOTAL_BYTES:-}
- STORAGE_EXPORT_TTL_MS=${STORAGE_EXPORT_TTL_MS:-}
- STORAGE_EXPORT_SWEEP_MAX_AGE_MS=${STORAGE_EXPORT_SWEEP_MAX_AGE_MS:-}
- S3_REPROBE_INTERVAL_MS=${S3_REPROBE_INTERVAL_MS:-}
diff --git a/docs/14-migration-guide.md b/docs/14-migration-guide.md
index b3b22e40d..bb3709ba7 100644
--- a/docs/14-migration-guide.md
+++ b/docs/14-migration-guide.md
@@ -269,6 +269,10 @@ curl -X POST 'http://localhost:2785/api/infra/storage/import' \
-d '{"filePath": "data/exports/storage-export-xxx.tar.gz"}'
```
+The import aborts once it has written more than `STORAGE_IMPORT_MAX_TOTAL_BYTES` (default 10 GiB). For a
+larger store, raise it on the destination before Step 5; the export logs a warning when it exceeds that
+limit. An aborted import keeps what it wrote and can be re-run.
+
| Scenario | Support | Method |
| ---------------------------- | ------- | ------------------------ |
| Local → Built-in MinIO | ✅ | Export → Config → Import |
diff --git a/src/common/storage/storage-transfer.spec.ts b/src/common/storage/storage-transfer.spec.ts
index 502f0376b..304e02378 100644
--- a/src/common/storage/storage-transfer.spec.ts
+++ b/src/common/storage/storage-transfer.spec.ts
@@ -274,3 +274,68 @@ describe('importFromStream reports partial counts on an abort', () => {
}
});
});
+
+describe('importFromStream caps the total bytes one import writes', () => {
+ const withTotalCap = async <T>(cap: string, run: () => Promise<T>): Promise<T> => {
+ const prev = process.env.STORAGE_IMPORT_MAX_TOTAL_BYTES;
+ process.env.STORAGE_IMPORT_MAX_TOTAL_BYTES = cap;
+ try {
+ return await run();
+ } finally {
+ if (prev === undefined) delete process.env.STORAGE_IMPORT_MAX_TOTAL_BYTES;
+ else process.env.STORAGE_IMPORT_MAX_TOTAL_BYTES = prev;
+ }
+ };
+ const importEntries = (written: Map<string, Buffer>, sizes: number[]) => {
+ const pack = tar.pack();
+ sizes.forEach((size, i) => pack.entry({ name: `e${i}.bin` }, Buffer.alloc(size, 0x61)));
+ pack.finalize();
+ return importFromStream(
+ pack.pipe(createGzip()),
+ (name, data) => {
+ written.set(name, data);
+ return Promise.resolve();
+ },
+ makeLogger() as never,
+ );
+ };
+ const bytesIn = (written: Map<string, Buffer>): number => [...written.values()].reduce((n, b) => n + b.length, 0);
+
+ it('aborts when entries that each fit the per-entry cap add up past the total cap', async () => {
+ const written = new Map<string, Buffer>();
+ const rejection = await withTotalCap('10000', () => importEntries(written, [4000, 4000, 4000, 4000])).catch(
+ (err: unknown) => err,
+ );
+
+ expect(rejection).toBeInstanceOf(Error);
+ expect((rejection as Error).message).toMatch(/10000-byte total cap \(STORAGE_IMPORT_MAX_TOTAL_BYTES\)/);
+ expect(rejection).toMatchObject({ imported: 2, failed: 0 });
+ expect([...written.keys()]).toEqual(['e0.bin', 'e1.bin']);
+ expect(bytesIn(written)).toBeLessThanOrEqual(10000);
+ });
+
+ it('imports an archive whose total is exactly the cap', async () => {
+ const written = new Map<string, Buffer>();
+ const result = await withTotalCap('8000', () => importEntries(written, [4000, 4000]));
+
+ expect(result).toEqual({ imported: 2, failed: 0 });
+ expect(bytesIn(written)).toBe(8000);
+ });
+
+ it('warns at export time when the archive holds more bytes than the import accepts', async () => {
+ const exportWith = async (cap: string) => {
+ const logger = makeLogger();
+ await withTotalCap(cap, async () => {
+ const { openFile } = trackingOpener();
+ const output = await createExportStream(() => Promise.resolve(files.slice(0, 3)), openFile, logger as never);
+ await new Promise<void>(resolve => output.on('end', resolve).resume());
+ });
+ return logger.warn;
+ };
+
+ expect(await exportWith(String(3 * FILE_BYTES))).not.toHaveBeenCalled();
+ expect(await exportWith(String(3 * FILE_BYTES - 1))).toHaveBeenCalledWith(
+ expect.stringContaining('Raise STORAGE_IMPORT_MAX_TOTAL_BYTES'),
+ );
+ });
+});
diff --git a/src/common/storage/storage-transfer.ts b/src/common/storage/storage-transfer.ts
index c9a3dbabf..0583a38a7 100644
--- a/src/common/storage/storage-transfer.ts
+++ b/src/common/storage/storage-transfer.ts
@@ -9,6 +9,8 @@ import { LoggerService } from '../services/logger.service';
const DEFAULT_IMPORT_MAX_BYTES = 200 * 1024 * 1024;
/** Max number of entries an import archive may contain. Bounds an entry-count DoS. */
const DEFAULT_IMPORT_MAX_ENTRIES = 100_000;
+/** Cap on the bytes one import writes in total (10 GiB). The two caps above still allow terabytes. */
+const DEFAULT_IMPORT_MAX_TOTAL_BYTES = 10 * 1024 * 1024 * 1024;
/**
* True when a storage read failed because the object is simply not there.
@@ -114,6 +116,7 @@ async function appendEntries(
output: PassThrough,
logger: LoggerService,
): Promise<void> {
+ let exportedBytes = 0;
for (const file of files) {
let source: ExportFileSource;
try {
@@ -128,6 +131,7 @@ async function appendEntries(
continue;
}
const { stream, size } = source;
+ exportedBytes += size ?? 0;
// The output closes early when the archive fails or the consumer goes away: stop reading files.
// Checked synchronously before waiting, so a 'close' that already fired cannot be missed.
if (output.destroyed) {
@@ -166,6 +170,16 @@ async function appendEntries(
archive.abort();
return;
}
+ // Same reasoning as the entry-count warning in createExportStream, for the import's total-bytes cap.
+ const localByteCap = positiveIntFromEnv('STORAGE_IMPORT_MAX_TOTAL_BYTES', DEFAULT_IMPORT_MAX_TOTAL_BYTES);
+ const warnAboveBytes = Math.min(localByteCap, DEFAULT_IMPORT_MAX_TOTAL_BYTES);
+ if (exportedBytes > warnAboveBytes) {
+ logger.warn(
+ `Export contains ${exportedBytes} bytes, above the import limit of ${warnAboveBytes} ` +
+ `(this deployment: ${localByteCap}, shipped default: ${DEFAULT_IMPORT_MAX_TOTAL_BYTES}). ` +
+ 'Raise STORAGE_IMPORT_MAX_TOTAL_BYTES on the destination before restoring this archive.',
+ );
+ }
// finalize() rejections also emit via the 'error' handler above; catch the promise so it
// never surfaces as an unhandled rejection.
archive.finalize().catch(() => undefined);
@@ -188,15 +202,17 @@ export async function importFromStream(
let importedCount = 0;
let failedCount = 0;
let entryCount = 0;
+ let totalBytes = 0;
const maxEntryBytes = positiveIntFromEnv('STORAGE_IMPORT_MAX_BYTES', DEFAULT_IMPORT_MAX_BYTES);
const maxEntries = positiveIntFromEnv('STORAGE_IMPORT_MAX_ENTRIES', DEFAULT_IMPORT_MAX_ENTRIES);
+ const maxTotalBytes = positiveIntFromEnv('STORAGE_IMPORT_MAX_TOTAL_BYTES', DEFAULT_IMPORT_MAX_TOTAL_BYTES);
const extract = tar.extract();
const gunzip = createGunzip();
return new Promise<{ imported: number; failed: number }>((resolve, reject) => {
let settled = false;
- // Abort the whole import: a per-entry overflow or too many entries is a (zip-bomb) attack, not
+ // Abort the whole import: a per-entry or total overflow or too many entries is a (zip-bomb) attack, not
// a per-file skip — tear down the pipeline and reject so nothing further is buffered or written.
const fail = (err: Error): void => {
if (settled) return;
@@ -251,10 +267,20 @@ export async function importFromStream(
stream.on('data', (chunk: Buffer) => {
if (entryAborted || settled) return;
entryBytes += chunk.length;
+ totalBytes += chunk.length;
if (entryBytes > maxEntryBytes) {
entryAborted = true;
stream.resume(); // drain the remainder so the source can end
fail(new Error(`Import aborted: entry "${key}" exceeds the ${maxEntryBytes}-byte per-entry cap`));
+ } else if (totalBytes > maxTotalBytes) {
+ // The entry that crosses the cap is dropped, so no import writes more than maxTotalBytes.
+ entryAborted = true;
+ stream.resume();
+ fail(
+ new Error(
+ `Import aborted: archive exceeds the ${maxTotalBytes}-byte total cap (STORAGE_IMPORT_MAX_TOTAL_BYTES)`,
+ ),
+ );
} else {
chunks.push(chunk);
}
diff --git a/src/config/env-precedence.ts b/src/config/env-precedence.ts
index 695fbe33f..10ebd9ad7 100644
--- a/src/config/env-precedence.ts
+++ b/src/config/env-precedence.ts
@@ -225,6 +225,7 @@ export const BLANK_SHADOWED_ENV_KEYS: string[] = [
'STORAGE_LIST_MAX_FILES',
'STORAGE_IMPORT_MAX_BYTES',
'STORAGE_IMPORT_MAX_ENTRIES',
+ 'STORAGE_IMPORT_MAX_TOTAL_BYTES',
'STORAGE_EXPORT_TTL_MS',
'STORAGE_EXPORT_SWEEP_MAX_AGE_MS',
'S3_REPROBE_INTERVAL_MS',
diff --git a/src/config/env.validation.spec.ts b/src/config/env.validation.spec.ts
index 5406c00b7..4b80e0ebb 100644
--- a/src/config/env.validation.spec.ts
+++ b/src/config/env.validation.spec.ts
@@ -664,6 +664,7 @@ describe('validateEnv', () => {
['TEMPLATE_RENDER_MAX_CHARS', '64k'],
['STORAGE_IMPORT_MAX_BYTES', '200mb'],
['STORAGE_IMPORT_MAX_ENTRIES', '1e5'],
+ ['STORAGE_IMPORT_MAX_TOTAL_BYTES', '10gb'],
['STORAGE_LIST_MAX_FILES', '100k'],
['BAILEYS_MESSAGE_STORE_LIMIT', '5k'],
])('rejects a unit-suffixed or zero %s', (key, bad) => {
diff --git a/src/config/env.validation.ts b/src/config/env.validation.ts
index 5328acd66..4a4ec980b 100644
--- a/src/config/env.validation.ts
+++ b/src/config/env.validation.ts
@@ -420,6 +420,7 @@ export function validateEnv(config: EnvConfig): EnvConfig {
'TEMPLATE_RENDER_MAX_CHARS',
'STORAGE_IMPORT_MAX_BYTES',
'STORAGE_IMPORT_MAX_ENTRIES',
+ 'STORAGE_IMPORT_MAX_TOTAL_BYTES',
'STORAGE_LIST_MAX_FILES',
'BAILEYS_MESSAGE_STORE_LIMIT',
// Each read fell back to its default on 0 or garbage, or passed a negative or fractional value on