Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(infra): cap total bytes restored by storage import (#1810)
Yudhi Armyndharis4dfba2cdd193
diff --git a/.env.example b/.env.exampleindex 928b53595..6c00d41fd 100644--- a/.env.example+++ b/.env.example@@ -679,6 +679,7 @@ WEBHOOK_SSRF_PROTECT=true# Storage import/export limits (ADMIN /infra/storage/* endpoints):# STORAGE_IMPORT_MAX_BYTES=209715200 # per-entry cap for a tar.gz import; aborts on overflow (default 200 MiB)# STORAGE_IMPORT_MAX_ENTRIES=100000 # max entries in an import archive; aborts beyond this (default 100000)+# STORAGE_IMPORT_MAX_TOTAL_BYTES=10737418240 # total bytes one import may write; aborts on overflow (default 10 GiB)# The two STORAGE_EXPORT_* values are raw milliseconds: a value that is not a positive integer# (`1h` included) fails the boot, and so does a TTL above 2147483647 ms (Node's timer ceiling).# STORAGE_EXPORT_TTL_MS=3600000 # auto-delete an export archive after this long (default 1h)diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 78e87cc59..78b7f8853 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -28,6 +28,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Fixed+- Cap the total bytes one storage import writes at `STORAGE_IMPORT_MAX_TOTAL_BYTES`.- Close the Baileys session proxy's fetch dispatcher when the session disconnects, logs out or is destroyed.- Apply the Java SDK request timeout to the response body as well as the headers.- Keep a new dashboard template draft when an earlier template save finishes.@@ -94,6 +95,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Upgrade notes (behavior changes)+- A storage import stops at `STORAGE_IMPORT_MAX_TOTAL_BYTES` (default 10 GiB); raise it before importing a larger export.- Baileys answers `400` for an animated sticker with more than 500 frames.- `backup.sh` and `restore.sh` exit with status 2 when `./.env` or `.env.generated` sets a key in a form they cannot parse; fix the line or pass the key in the environment.- With `DATABASE_SSL=true`, `backup.sh` now fails against a PostgreSQL server whose certificate the app would reject; trust a private CA through `NODE_EXTRA_CA_CERTS` or `PGSSLROOTCERT`.diff --git a/docker-compose.dev.yml b/docker-compose.dev.ymlindex 7a25810a9..a42fd6c06 100644--- a/docker-compose.dev.yml+++ b/docker-compose.dev.yml@@ -267,6 +267,7 @@ services:- STORAGE_LIST_MAX_FILES=${STORAGE_LIST_MAX_FILES:-}- STORAGE_IMPORT_MAX_BYTES=${STORAGE_IMPORT_MAX_BYTES:-}- STORAGE_IMPORT_MAX_ENTRIES=${STORAGE_IMPORT_MAX_ENTRIES:-}+ - STORAGE_IMPORT_MAX_TOTAL_BYTES=${STORAGE_IMPORT_MAX_TOTAL_BYTES:-}- STORAGE_EXPORT_TTL_MS=${STORAGE_EXPORT_TTL_MS:-}- STORAGE_EXPORT_SWEEP_MAX_AGE_MS=${STORAGE_EXPORT_SWEEP_MAX_AGE_MS:-}- S3_REPROBE_INTERVAL_MS=${S3_REPROBE_INTERVAL_MS:-}diff --git a/docker-compose.yml b/docker-compose.ymlindex c0a6e4b11..dbfbbb73e 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -369,6 +369,7 @@ services:- STORAGE_LIST_MAX_FILES=${STORAGE_LIST_MAX_FILES:-}- STORAGE_IMPORT_MAX_BYTES=${STORAGE_IMPORT_MAX_BYTES:-}- STORAGE_IMPORT_MAX_ENTRIES=${STORAGE_IMPORT_MAX_ENTRIES:-}+ - STORAGE_IMPORT_MAX_TOTAL_BYTES=${STORAGE_IMPORT_MAX_TOTAL_BYTES:-}- STORAGE_EXPORT_TTL_MS=${STORAGE_EXPORT_TTL_MS:-}- STORAGE_EXPORT_SWEEP_MAX_AGE_MS=${STORAGE_EXPORT_SWEEP_MAX_AGE_MS:-}- S3_REPROBE_INTERVAL_MS=${S3_REPROBE_INTERVAL_MS:-}diff --git a/docs/14-migration-guide.md b/docs/14-migration-guide.mdindex b3b22e40d..bb3709ba7 100644--- a/docs/14-migration-guide.md+++ b/docs/14-migration-guide.md@@ -269,6 +269,10 @@ curl -X POST 'http://localhost:2785/api/infra/storage/import' \-d '{"filePath": "data/exports/storage-export-xxx.tar.gz"}'```+The import aborts once it has written more than `STORAGE_IMPORT_MAX_TOTAL_BYTES` (default 10 GiB). For a+larger store, raise it on the destination before Step 5; the export logs a warning when it exceeds that+limit. An aborted import keeps what it wrote and can be re-run.+| Scenario | Support | Method || ---------------------------- | ------- | ------------------------ || Local → Built-in MinIO | ✅ | Export → Config → Import |diff --git a/src/common/storage/storage-transfer.spec.ts b/src/common/storage/storage-transfer.spec.tsindex 502f0376b..304e02378 100644--- a/src/common/storage/storage-transfer.spec.ts+++ b/src/common/storage/storage-transfer.spec.ts@@ -274,3 +274,68 @@ describe('importFromStream reports partial counts on an abort', () => {}});});++describe('importFromStream caps the total bytes one import writes', () => {+ const withTotalCap = async <T>(cap: string, run: () => Promise<T>): Promise<T> => {+ const prev = process.env.STORAGE_IMPORT_MAX_TOTAL_BYTES;+ process.env.STORAGE_IMPORT_MAX_TOTAL_BYTES = cap;+ try {+ return await run();+ } finally {+ if (prev === undefined) delete process.env.STORAGE_IMPORT_MAX_TOTAL_BYTES;+ else process.env.STORAGE_IMPORT_MAX_TOTAL_BYTES = prev;+ }+ };+ const importEntries = (written: Map<string, Buffer>, sizes: number[]) => {+ const pack = tar.pack();+ sizes.forEach((size, i) => pack.entry({ name: `e${i}.bin` }, Buffer.alloc(size, 0x61)));+ pack.finalize();+ return importFromStream(+ pack.pipe(createGzip()),+ (name, data) => {+ written.set(name, data);+ return Promise.resolve();+ },+ makeLogger() as never,+ );+ };+ const bytesIn = (written: Map<string, Buffer>): number => [...written.values()].reduce((n, b) => n + b.length, 0);++ it('aborts when entries that each fit the per-entry cap add up past the total cap', async () => {+ const written = new Map<string, Buffer>();+ const rejection = await withTotalCap('10000', () => importEntries(written, [4000, 4000, 4000, 4000])).catch(+ (err: unknown) => err,+ );++ expect(rejection).toBeInstanceOf(Error);+ expect((rejection as Error).message).toMatch(/10000-byte total cap \(STORAGE_IMPORT_MAX_TOTAL_BYTES\)/);+ expect(rejection).toMatchObject({ imported: 2, failed: 0 });+ expect([...written.keys()]).toEqual(['e0.bin', 'e1.bin']);+ expect(bytesIn(written)).toBeLessThanOrEqual(10000);+ });++ it('imports an archive whose total is exactly the cap', async () => {+ const written = new Map<string, Buffer>();+ const result = await withTotalCap('8000', () => importEntries(written, [4000, 4000]));++ expect(result).toEqual({ imported: 2, failed: 0 });+ expect(bytesIn(written)).toBe(8000);+ });++ it('warns at export time when the archive holds more bytes than the import accepts', async () => {+ const exportWith = async (cap: string) => {+ const logger = makeLogger();+ await withTotalCap(cap, async () => {+ const { openFile } = trackingOpener();+ const output = await createExportStream(() => Promise.resolve(files.slice(0, 3)), openFile, logger as never);+ await new Promise<void>(resolve => output.on('end', resolve).resume());+ });+ return logger.warn;+ };++ expect(await exportWith(String(3 * FILE_BYTES))).not.toHaveBeenCalled();+ expect(await exportWith(String(3 * FILE_BYTES - 1))).toHaveBeenCalledWith(+ expect.stringContaining('Raise STORAGE_IMPORT_MAX_TOTAL_BYTES'),+ );+ });+});diff --git a/src/common/storage/storage-transfer.ts b/src/common/storage/storage-transfer.tsindex c9a3dbabf..0583a38a7 100644--- a/src/common/storage/storage-transfer.ts+++ b/src/common/storage/storage-transfer.ts@@ -9,6 +9,8 @@ import { LoggerService } from '../services/logger.service';const DEFAULT_IMPORT_MAX_BYTES = 200 * 1024 * 1024;/** Max number of entries an import archive may contain. Bounds an entry-count DoS. */const DEFAULT_IMPORT_MAX_ENTRIES = 100_000;+/** Cap on the bytes one import writes in total (10 GiB). The two caps above still allow terabytes. */+const DEFAULT_IMPORT_MAX_TOTAL_BYTES = 10 * 1024 * 1024 * 1024;/*** True when a storage read failed because the object is simply not there.@@ -114,6 +116,7 @@ async function appendEntries(output: PassThrough,logger: LoggerService,): Promise<void> {+ let exportedBytes = 0;for (const file of files) {let source: ExportFileSource;try {@@ -128,6 +131,7 @@ async function appendEntries(continue;}const { stream, size } = source;+ exportedBytes += size ?? 0;// The output closes early when the archive fails or the consumer goes away: stop reading files.// Checked synchronously before waiting, so a 'close' that already fired cannot be missed.if (output.destroyed) {@@ -166,6 +170,16 @@ async function appendEntries(archive.abort();return;}+ // Same reasoning as the entry-count warning in createExportStream, for the import's total-bytes cap.+ const localByteCap = positiveIntFromEnv('STORAGE_IMPORT_MAX_TOTAL_BYTES', DEFAULT_IMPORT_MAX_TOTAL_BYTES);+ const warnAboveBytes = Math.min(localByteCap, DEFAULT_IMPORT_MAX_TOTAL_BYTES);+ if (exportedBytes > warnAboveBytes) {+ logger.warn(+ `Export contains ${exportedBytes} bytes, above the import limit of ${warnAboveBytes} ` ++ `(this deployment: ${localByteCap}, shipped default: ${DEFAULT_IMPORT_MAX_TOTAL_BYTES}). ` ++ 'Raise STORAGE_IMPORT_MAX_TOTAL_BYTES on the destination before restoring this archive.',+ );+ }// finalize() rejections also emit via the 'error' handler above; catch the promise so it// never surfaces as an unhandled rejection.archive.finalize().catch(() => undefined);@@ -188,15 +202,17 @@ export async function importFromStream(let importedCount = 0;let failedCount = 0;let entryCount = 0;+ let totalBytes = 0;const maxEntryBytes = positiveIntFromEnv('STORAGE_IMPORT_MAX_BYTES', DEFAULT_IMPORT_MAX_BYTES);const maxEntries = positiveIntFromEnv('STORAGE_IMPORT_MAX_ENTRIES', DEFAULT_IMPORT_MAX_ENTRIES);+ const maxTotalBytes = positiveIntFromEnv('STORAGE_IMPORT_MAX_TOTAL_BYTES', DEFAULT_IMPORT_MAX_TOTAL_BYTES);const extract = tar.extract();const gunzip = createGunzip();return new Promise<{ imported: number; failed: number }>((resolve, reject) => {let settled = false;- // Abort the whole import: a per-entry overflow or too many entries is a (zip-bomb) attack, not+ // Abort the whole import: a per-entry or total overflow or too many entries is a (zip-bomb) attack, not// a per-file skip — tear down the pipeline and reject so nothing further is buffered or written.const fail = (err: Error): void => {if (settled) return;@@ -251,10 +267,20 @@ export async function importFromStream(stream.on('data', (chunk: Buffer) => {if (entryAborted || settled) return;entryBytes += chunk.length;+ totalBytes += chunk.length;if (entryBytes > maxEntryBytes) {entryAborted = true;stream.resume(); // drain the remainder so the source can endfail(new Error(`Import aborted: entry "${key}" exceeds the ${maxEntryBytes}-byte per-entry cap`));+ } else if (totalBytes > maxTotalBytes) {+ // The entry that crosses the cap is dropped, so no import writes more than maxTotalBytes.+ entryAborted = true;+ stream.resume();+ fail(+ new Error(+ `Import aborted: archive exceeds the ${maxTotalBytes}-byte total cap (STORAGE_IMPORT_MAX_TOTAL_BYTES)`,+ ),+ );} else {chunks.push(chunk);}diff --git a/src/config/env-precedence.ts b/src/config/env-precedence.tsindex 695fbe33f..10ebd9ad7 100644--- a/src/config/env-precedence.ts+++ b/src/config/env-precedence.ts@@ -225,6 +225,7 @@ export const BLANK_SHADOWED_ENV_KEYS: string[] = ['STORAGE_LIST_MAX_FILES','STORAGE_IMPORT_MAX_BYTES','STORAGE_IMPORT_MAX_ENTRIES',+ 'STORAGE_IMPORT_MAX_TOTAL_BYTES','STORAGE_EXPORT_TTL_MS','STORAGE_EXPORT_SWEEP_MAX_AGE_MS','S3_REPROBE_INTERVAL_MS',diff --git a/src/config/env.validation.spec.ts b/src/config/env.validation.spec.tsindex 5406c00b7..4b80e0ebb 100644--- a/src/config/env.validation.spec.ts+++ b/src/config/env.validation.spec.ts@@ -664,6 +664,7 @@ describe('validateEnv', () => {['TEMPLATE_RENDER_MAX_CHARS', '64k'],['STORAGE_IMPORT_MAX_BYTES', '200mb'],['STORAGE_IMPORT_MAX_ENTRIES', '1e5'],+ ['STORAGE_IMPORT_MAX_TOTAL_BYTES', '10gb'],['STORAGE_LIST_MAX_FILES', '100k'],['BAILEYS_MESSAGE_STORE_LIMIT', '5k'],])('rejects a unit-suffixed or zero %s', (key, bad) => {diff --git a/src/config/env.validation.ts b/src/config/env.validation.tsindex 5328acd66..4a4ec980b 100644--- a/src/config/env.validation.ts+++ b/src/config/env.validation.ts@@ -420,6 +420,7 @@ export function validateEnv(config: EnvConfig): EnvConfig {'TEMPLATE_RENDER_MAX_CHARS','STORAGE_IMPORT_MAX_BYTES','STORAGE_IMPORT_MAX_ENTRIES',+ 'STORAGE_IMPORT_MAX_TOTAL_BYTES','STORAGE_LIST_MAX_FILES','BAILEYS_MESSAGE_STORE_LIMIT',// Each read fell back to its default on 0 or garbage, or passed a negative or fractional value on