Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(storage): count only what the export carries

Yudhi Armyndharis98fdb2b796bb
diff --git a/src/common/storage/storage.service.s3-reprobe.spec.ts b/src/common/storage/storage.service.s3-reprobe.spec.ts
index 7144a2222..b546ccb73 100644
--- a/src/common/storage/storage.service.s3-reprobe.spec.ts
+++ b/src/common/storage/storage.service.s3-reprobe.spec.ts
@@ -1,7 +1,7 @@
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
-import { Readable } from 'stream';
+import { PassThrough, Readable } from 'stream';
import { ConfigService } from '@nestjs/config';
// Mock the AWS SDK so no real network call is made. Each test drives `mockSend`: HeadBucket probes
@@ -33,6 +33,7 @@ import {
S3_PROBE_TIMEOUT_MS,
StorageService,
} from './storage.service';
+import * as transfer from './storage-transfer';
const ENV_KEYS = [
'S3_ENDPOINT',
@@ -480,6 +481,40 @@ describe('StorageService S3 re-probe and recovery', () => {
expect(result.sizeBytes).toBe(1000 + 10 + 500);
});
+ // The count is the pre-check an operator runs before the export, so it must promise exactly the files the
+ // archive will hold: not a console-made folder placeholder, and not a key the export cannot open.
+ it('getFileCount counts the files the export carries, not folder placeholders or unsafe keys', async () => {
+ mockSend.mockImplementation((cmd: unknown) => {
+ if (cmd instanceof ListObjectsV2Command) {
+ return Promise.resolve({
+ Contents: [
+ { Key: 'media/a.bin', Size: 1000 },
+ { Key: 'media/sess1/', Size: 0 },
+ { Key: 'media/sess1/b.bin', Size: 2000 },
+ { Key: 'media/empty.bin', Size: 0 },
+ { Key: 'media/../outside.bin', Size: 7 },
+ ],
+ });
+ }
+ return Promise.resolve({});
+ });
+ const svc = new StorageService(makeConfig());
+ warnSpyOf(svc);
+ await flush();
+ fs.writeFileSync(path.join(localPath, 'local-only.bin'), Buffer.alloc(500));
+ fs.writeFileSync(path.join(localPath, 'bad\u0001name.bin'), Buffer.alloc(9));
+ const exported = jest.spyOn(transfer, 'createExportStream').mockResolvedValue(new PassThrough());
+
+ const { count, sizeBytes } = await svc.getFileCount();
+ await svc.createExportStream();
+ const carried = await exported.mock.calls[0][0]();
+ exported.mockRestore();
+
+ expect(carried.sort()).toEqual(['empty.bin', 'local-only.bin', 'a.bin', 'sess1/b.bin'].sort());
+ expect(count).toBe(carried.length);
+ expect(sizeBytes).toBe(1000 + 2000 + 500);
+ });
+
it('iterateFiles paginates S3 and unions the local fallback dir (each key once)', async () => {
const pages = [
{ Contents: [{ Key: 'media/p1.bin' }], NextContinuationToken: 'tok' },
diff --git a/src/common/storage/storage.service.spec.ts b/src/common/storage/storage.service.spec.ts
index 973728c05..e26bf797f 100644
--- a/src/common/storage/storage.service.spec.ts
+++ b/src/common/storage/storage.service.spec.ts
@@ -400,6 +400,20 @@ describe('StorageService.createExportStream enumerates the whole store', () => {
expect(count).toBe(2); // the iterator's items are what was counted, not an unrelated walk
});
+ it('counts only what the export carries: a folder placeholder and an unsafe key are left out', async () => {
+ const { service } = makeService();
+ jest.spyOn(service, 'iterateFiles').mockImplementation(async function* () {
+ yield await Promise.resolve('sess1/');
+ yield 'sess1/a.bin';
+ yield 'media/../x.bin';
+ });
+ jest.spyOn((service as unknown as { logger: { warn: () => void } }).logger, 'warn').mockImplementation(() => {});
+
+ const { count } = await service.getFileCount();
+
+ expect(count).toBe(1);
+ });
+
/**
* The size loop under the whole-store walk must not stat every file synchronously: on a large store
* that holds the event loop for the whole walk, and health checks, webhooks and every in-flight
diff --git a/src/common/storage/storage.service.ts b/src/common/storage/storage.service.ts
index 3158a384b..0df995d16 100644
--- a/src/common/storage/storage.service.ts
+++ b/src/common/storage/storage.service.ts
@@ -362,7 +362,7 @@ export class StorageService implements OnModuleDestroy {
const s3Keys = new Set(s3.keys);
let { count, sizeBytes } = s3;
for await (const file of this.iterateLocalFiles()) {
- if (s3Keys.has(file)) continue;
+ if (s3Keys.has(file) || !this.isExportableKey(file)) continue;
count += 1;
try {
// Same reason as the local branch below: the walk covers the whole store, so the stat must yield.
@@ -376,7 +376,7 @@ export class StorageService implements OnModuleDestroy {
// The whole store is walked, for the same reason createExportStream does: this is the pre-check an
// operator runs BEFORE the export/import migration, so it has to see every file the export will carry.
- const files = await this.listAllFiles();
+ const files = await this.listExportableFiles();
let sizeBytes = 0;
for (const file of files) {
try {
@@ -413,7 +413,7 @@ export class StorageService implements OnModuleDestroy {
for (const obj of response.Contents ?? []) {
const key = this.stripS3KeyRoot(obj.Key);
- if (key === null) continue;
+ if (key === null || !this.isExportableKey(key)) continue;
count += 1;
sizeBytes += obj.Size ?? 0;
keys.push(key);
@@ -437,25 +437,31 @@ export class StorageService implements OnModuleDestroy {
// root) is left out here: the export fails on any open error other than a missing object, and a
// key that can never be opened must not fail every export.
return createExportStream(
- async () => (await this.listAllFiles()).filter(file => this.isExportableKey(file)),
+ () => this.listExportableFiles(),
filePath => this.openFile(filePath),
this.logger,
);
}
+ /**
+ * Whether the export carries a listed key. The file count applies the same rule, so the pre-check never
+ * promises a file the archive will not hold.
+ */
private isExportableKey(file: string): boolean {
// A key ending in "/" is a folder placeholder an S3 console leaves behind: it holds no file, and written to
// the archive as one it would fail the import that restores it onto a local store.
if (file.endsWith('/')) return false;
if (isSafeStorageKey(file)) return true;
- this.logger.warn(`Skipping an unsafe storage key in the export: ${JSON.stringify(file)}`);
+ this.logger.warn(`Leaving an unsafe storage key out of the export and the file count: ${JSON.stringify(file)}`);
return false;
}
- /** Every key in the store. */
- private async listAllFiles(): Promise<string[]> {
+ /** Every key in the store that the export carries. */
+ private async listExportableFiles(): Promise<string[]> {
const files: string[] = [];
- for await (const file of this.iterateFiles()) files.push(file);
+ for await (const file of this.iterateFiles()) {
+ if (this.isExportableKey(file)) files.push(file);
+ }
return files;
}