Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(storage): count only what the export carries
Yudhi Armyndharis98fdb2b796bb
diff --git a/src/common/storage/storage.service.s3-reprobe.spec.ts b/src/common/storage/storage.service.s3-reprobe.spec.tsindex 7144a2222..b546ccb73 100644--- a/src/common/storage/storage.service.s3-reprobe.spec.ts+++ b/src/common/storage/storage.service.s3-reprobe.spec.ts@@ -1,7 +1,7 @@import * as fs from 'fs';import * as os from 'os';import * as path from 'path';-import { Readable } from 'stream';+import { PassThrough, Readable } from 'stream';import { ConfigService } from '@nestjs/config';// Mock the AWS SDK so no real network call is made. Each test drives `mockSend`: HeadBucket probes@@ -33,6 +33,7 @@ import {S3_PROBE_TIMEOUT_MS,StorageService,} from './storage.service';+import * as transfer from './storage-transfer';const ENV_KEYS = ['S3_ENDPOINT',@@ -480,6 +481,40 @@ describe('StorageService S3 re-probe and recovery', () => {expect(result.sizeBytes).toBe(1000 + 10 + 500);});+ // The count is the pre-check an operator runs before the export, so it must promise exactly the files the+ // archive will hold: not a console-made folder placeholder, and not a key the export cannot open.+ it('getFileCount counts the files the export carries, not folder placeholders or unsafe keys', async () => {+ mockSend.mockImplementation((cmd: unknown) => {+ if (cmd instanceof ListObjectsV2Command) {+ return Promise.resolve({+ Contents: [+ { Key: 'media/a.bin', Size: 1000 },+ { Key: 'media/sess1/', Size: 0 },+ { Key: 'media/sess1/b.bin', Size: 2000 },+ { Key: 'media/empty.bin', Size: 0 },+ { Key: 'media/../outside.bin', Size: 7 },+ ],+ });+ }+ return Promise.resolve({});+ });+ const svc = new StorageService(makeConfig());+ warnSpyOf(svc);+ await flush();+ fs.writeFileSync(path.join(localPath, 'local-only.bin'), Buffer.alloc(500));+ fs.writeFileSync(path.join(localPath, 'bad\u0001name.bin'), Buffer.alloc(9));+ const exported = jest.spyOn(transfer, 'createExportStream').mockResolvedValue(new PassThrough());++ const { count, sizeBytes } = await svc.getFileCount();+ await svc.createExportStream();+ const carried = await exported.mock.calls[0][0]();+ exported.mockRestore();++ expect(carried.sort()).toEqual(['empty.bin', 'local-only.bin', 'a.bin', 'sess1/b.bin'].sort());+ expect(count).toBe(carried.length);+ expect(sizeBytes).toBe(1000 + 2000 + 500);+ });+it('iterateFiles paginates S3 and unions the local fallback dir (each key once)', async () => {const pages = [{ Contents: [{ Key: 'media/p1.bin' }], NextContinuationToken: 'tok' },diff --git a/src/common/storage/storage.service.spec.ts b/src/common/storage/storage.service.spec.tsindex 973728c05..e26bf797f 100644--- a/src/common/storage/storage.service.spec.ts+++ b/src/common/storage/storage.service.spec.ts@@ -400,6 +400,20 @@ describe('StorageService.createExportStream enumerates the whole store', () => {expect(count).toBe(2); // the iterator's items are what was counted, not an unrelated walk});+ it('counts only what the export carries: a folder placeholder and an unsafe key are left out', async () => {+ const { service } = makeService();+ jest.spyOn(service, 'iterateFiles').mockImplementation(async function* () {+ yield await Promise.resolve('sess1/');+ yield 'sess1/a.bin';+ yield 'media/../x.bin';+ });+ jest.spyOn((service as unknown as { logger: { warn: () => void } }).logger, 'warn').mockImplementation(() => {});++ const { count } = await service.getFileCount();++ expect(count).toBe(1);+ });+/*** The size loop under the whole-store walk must not stat every file synchronously: on a large store* that holds the event loop for the whole walk, and health checks, webhooks and every in-flightdiff --git a/src/common/storage/storage.service.ts b/src/common/storage/storage.service.tsindex 3158a384b..0df995d16 100644--- a/src/common/storage/storage.service.ts+++ b/src/common/storage/storage.service.ts@@ -362,7 +362,7 @@ export class StorageService implements OnModuleDestroy {const s3Keys = new Set(s3.keys);let { count, sizeBytes } = s3;for await (const file of this.iterateLocalFiles()) {- if (s3Keys.has(file)) continue;+ if (s3Keys.has(file) || !this.isExportableKey(file)) continue;count += 1;try {// Same reason as the local branch below: the walk covers the whole store, so the stat must yield.@@ -376,7 +376,7 @@ export class StorageService implements OnModuleDestroy {// The whole store is walked, for the same reason createExportStream does: this is the pre-check an// operator runs BEFORE the export/import migration, so it has to see every file the export will carry.- const files = await this.listAllFiles();+ const files = await this.listExportableFiles();let sizeBytes = 0;for (const file of files) {try {@@ -413,7 +413,7 @@ export class StorageService implements OnModuleDestroy {for (const obj of response.Contents ?? []) {const key = this.stripS3KeyRoot(obj.Key);- if (key === null) continue;+ if (key === null || !this.isExportableKey(key)) continue;count += 1;sizeBytes += obj.Size ?? 0;keys.push(key);@@ -437,25 +437,31 @@ export class StorageService implements OnModuleDestroy {// root) is left out here: the export fails on any open error other than a missing object, and a// key that can never be opened must not fail every export.return createExportStream(- async () => (await this.listAllFiles()).filter(file => this.isExportableKey(file)),+ () => this.listExportableFiles(),filePath => this.openFile(filePath),this.logger,);}+ /**+ * Whether the export carries a listed key. The file count applies the same rule, so the pre-check never+ * promises a file the archive will not hold.+ */private isExportableKey(file: string): boolean {// A key ending in "/" is a folder placeholder an S3 console leaves behind: it holds no file, and written to// the archive as one it would fail the import that restores it onto a local store.if (file.endsWith('/')) return false;if (isSafeStorageKey(file)) return true;- this.logger.warn(`Skipping an unsafe storage key in the export: ${JSON.stringify(file)}`);+ this.logger.warn(`Leaving an unsafe storage key out of the export and the file count: ${JSON.stringify(file)}`);return false;}- /** Every key in the store. */- private async listAllFiles(): Promise<string[]> {+ /** Every key in the store that the export carries. */+ private async listExportableFiles(): Promise<string[]> {const files: string[] = [];- for await (const file of this.iterateFiles()) files.push(file);+ for await (const file of this.iterateFiles()) {+ if (this.isExportableKey(file)) files.push(file);+ }return files;}