Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(storage): warn about import limits for files of unreported length

Yudhi Armyndharis7a6d036861e3
diff --git a/src/common/storage/storage-transfer.spec.ts b/src/common/storage/storage-transfer.spec.ts
index 36766f36f..926b71b4d 100644
--- a/src/common/storage/storage-transfer.spec.ts
+++ b/src/common/storage/storage-transfer.spec.ts
@@ -451,6 +451,49 @@ describe('createExportStream and the entry-count limit this deployment lowered',
});
});
+// A backend that does not report a length (ExportFileSource.size is optional) still exports a file the
+// destination may refuse, so the warning is worked out from the bytes actually read.
+describe('createExportStream warns about import limits for files of unreported size', () => {
+ const exportWithLimits = async (limits: Record<string, string>, sizes: number[]) => {
+ const previous = Object.fromEntries(Object.keys(limits).map(key => [key, process.env[key]]));
+ Object.assign(process.env, limits);
+ const logger = makeLogger();
+ try {
+ const names = sizes.map((_, index) => `media/unknown-${index}.bin`);
+ const output = await createExportStream(
+ () => Promise.resolve(names),
+ name => Promise.resolve({ stream: Readable.from([Buffer.alloc(sizes[names.indexOf(name)], 0x61)]) }),
+ logger as never,
+ );
+ await new Promise<void>(resolve => output.on('end', resolve).resume());
+ } finally {
+ for (const [key, value] of Object.entries(previous)) {
+ if (value === undefined) delete process.env[key];
+ else process.env[key] = value;
+ }
+ }
+ return logger;
+ };
+
+ it('warns when one file is above the per-entry limit', async () => {
+ const logger = await exportWithLimits({ STORAGE_IMPORT_MAX_BYTES: '5' }, [6]);
+ expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('Raise STORAGE_IMPORT_MAX_BYTES'));
+ });
+
+ it('warns when the files together are above the total limit', async () => {
+ const logger = await exportWithLimits({ STORAGE_IMPORT_MAX_TOTAL_BYTES: '5' }, [3, 3]);
+ expect(logger.warn).toHaveBeenCalledWith(expect.stringContaining('Raise STORAGE_IMPORT_MAX_TOTAL_BYTES'));
+ });
+
+ it('stays quiet at exactly the limits', async () => {
+ const logger = await exportWithLimits(
+ { STORAGE_IMPORT_MAX_BYTES: '3', STORAGE_IMPORT_MAX_TOTAL_BYTES: '6' },
+ [3, 3],
+ );
+ expect(logger.warn).not.toHaveBeenCalled();
+ });
+});
+
// docs/14, .env.example and the upgrade notes state these figures, and an operator sizes an import
// against them: a drifted default changes what an import is allowed to extract without a word.
describe('storage import caps', () => {
diff --git a/src/common/storage/storage-transfer.ts b/src/common/storage/storage-transfer.ts
index 9909040ef..9c6237dd0 100644
--- a/src/common/storage/storage-transfer.ts
+++ b/src/common/storage/storage-transfer.ts
@@ -97,7 +97,7 @@ export async function createExportStream(
return output;
}
-function enforceSize(file: string, size: number | undefined): Transform {
+function enforceSize(file: string, size: number | undefined, onRead: (bytes: number) => void): Transform {
let seen = 0;
const mismatch = (): Error => new Error(`Export failed: ${file} changed size while being read (expected ${size})`);
return new Transform({
@@ -106,7 +106,9 @@ function enforceSize(file: string, size: number | undefined): Transform {
callback(size !== undefined && seen > size ? mismatch() : null, chunk);
},
flush(callback) {
- callback(size !== undefined && seen < size ? mismatch() : null);
+ if (size !== undefined && seen < size) return callback(mismatch());
+ onRead(seen);
+ callback();
},
});
}
@@ -164,8 +166,6 @@ async function appendEntries(
continue;
}
const { stream, size } = source;
- exportedBytes += size ?? 0;
- if ((size ?? 0) > largest.size) largest = { file, size: size ?? 0 };
// The output closes early when the archive fails or the consumer goes away: stop reading files.
// Checked synchronously before waiting, so a 'close' that already fired cannot be missed.
if (output.destroyed) {
@@ -176,11 +176,19 @@ async function appendEntries(
// its header. A file rewritten between open and read does that, so the length is checked here and
// a mismatch fails the export like a read error, naming the file, before the sink sees the extra
// byte or the early end. pipeline() also forwards a source error, which pipe() does not.
- const entry = pipeline(stream, enforceSize(file, size), err => {
- if (!err || output.destroyed) return;
- logger.error(`Export failed reading file: ${file}`, String(err));
- output.destroy(err);
- });
+ // The import-limit warnings below count the bytes read, since a backend may not report a length.
+ const entry = pipeline(
+ stream,
+ enforceSize(file, size, bytes => {
+ exportedBytes += bytes;
+ if (bytes > largest.size) largest = { file, size: bytes };
+ }),
+ err => {
+ if (!err || output.destroyed) return;
+ logger.error(`Export failed reading file: ${file}`, String(err));
+ output.destroy(err);
+ },
+ );
await new Promise<void>(resolve => {
const done = (): void => {
output.removeListener('close', done);