Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(storage): warn about import limits for files of unreported length
Yudhi Armyndharis7a6d036861e3
diff --git a/src/common/storage/storage-transfer.spec.ts b/src/common/storage/storage-transfer.spec.tsindex 36766f36f..926b71b4d 100644--- a/src/common/storage/storage-transfer.spec.ts+++ b/src/common/storage/storage-transfer.spec.ts@@ -451,6 +451,49 @@ describe('createExportStream and the entry-count limit this deployment lowered',});});+// A backend that does not report a length (ExportFileSource.size is optional) still exports a file the+// destination may refuse, so the warning is worked out from the bytes actually read.+describe('createExportStream warns about import limits for files of unreported size', () => {+ const exportWithLimits = async (limits: Record<string, string>, sizes: number[]) => {+ const previous = Object.fromEntries(Object.keys(limits).map(key => [key, process.env[key]]));+ Object.assign(process.env, limits);+ const logger = makeLogger();+ try {+ const names = sizes.map((_, index) => `media/unknown-${index}.bin`);+ const output = await createExportStream(+ () => Promise.resolve(names),+ name => Promise.resolve({ stream: Readable.from([Buffer.alloc(sizes[names.indexOf(name)], 0x61)]) }),+ logger as never,+ );+ await new Promise<void>(resolve => output.on('end', resolve).resume());+ } finally {+ for (const [key, value] of Object.entries(previous)) {+ if (value === undefined) delete process.env[key];+ else process.env[key] = value;+ }+ }+ return logger;+ };++ it('warns when one file is above the per-entry limit', async () => {+ const logger = await exportWithLimits({ STORAGE_IMPORT_MAX_BYTES: '5' }, [6]);+ expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('Raise STORAGE_IMPORT_MAX_BYTES'));+ });++ it('warns when the files together are above the total limit', async () => {+ const logger = await exportWithLimits({ STORAGE_IMPORT_MAX_TOTAL_BYTES: '5' }, [3, 3]);+ expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('Raise STORAGE_IMPORT_MAX_TOTAL_BYTES'));+ });++ it('stays quiet at exactly the limits', async () => {+ const logger = await exportWithLimits(+ { STORAGE_IMPORT_MAX_BYTES: '3', STORAGE_IMPORT_MAX_TOTAL_BYTES: '6' },+ [3, 3],+ );+ expect(logger.warn).not.toHaveBeenCalled();+ });+});+// docs/14, .env.example and the upgrade notes state these figures, and an operator sizes an import// against them: a drifted default changes what an import is allowed to extract without a word.describe('storage import caps', () => {diff --git a/src/common/storage/storage-transfer.ts b/src/common/storage/storage-transfer.tsindex 9909040ef..9c6237dd0 100644--- a/src/common/storage/storage-transfer.ts+++ b/src/common/storage/storage-transfer.ts@@ -97,7 +97,7 @@ export async function createExportStream(return output;}-function enforceSize(file: string, size: number | undefined): Transform {+function enforceSize(file: string, size: number | undefined, onRead: (bytes: number) => void): Transform {let seen = 0;const mismatch = (): Error => new Error(`Export failed: ${file} changed size while being read (expected ${size})`);return new Transform({@@ -106,7 +106,9 @@ function enforceSize(file: string, size: number | undefined): Transform {callback(size !== undefined && seen > size ? mismatch() : null, chunk);},flush(callback) {- callback(size !== undefined && seen < size ? mismatch() : null);+ if (size !== undefined && seen < size) return callback(mismatch());+ onRead(seen);+ callback();},});}@@ -164,8 +166,6 @@ async function appendEntries(continue;}const { stream, size } = source;- exportedBytes += size ?? 0;- if ((size ?? 0) > largest.size) largest = { file, size: size ?? 0 };// The output closes early when the archive fails or the consumer goes away: stop reading files.// Checked synchronously before waiting, so a 'close' that already fired cannot be missed.if (output.destroyed) {@@ -176,11 +176,19 @@ async function appendEntries(// its header. A file rewritten between open and read does that, so the length is checked here and// a mismatch fails the export like a read error, naming the file, before the sink sees the extra// byte or the early end. pipeline() also forwards a source error, which pipe() does not.- const entry = pipeline(stream, enforceSize(file, size), err => {- if (!err || output.destroyed) return;- logger.error(`Export failed reading file: ${file}`, String(err));- output.destroy(err);- });+ // The import-limit warnings below count the bytes read, since a backend may not report a length.+ const entry = pipeline(+ stream,+ enforceSize(file, size, bytes => {+ exportedBytes += bytes;+ if (bytes > largest.size) largest = { file, size: bytes };+ }),+ err => {+ if (!err || output.destroyed) return;+ logger.error(`Export failed reading file: ${file}`, String(err));+ output.destroy(err);+ },+ );await new Promise<void>(resolve => {const done = (): void => {output.removeListener('close', done);