Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
refactor(storage): remove the file listing cap
Yudhi Armyndharis91648f390122
diff --git a/.env.example b/.env.exampleindex 80c8d7a62..bce621078 100644--- a/.env.example+++ b/.env.example@@ -419,11 +419,6 @@ DATABASE_SYNCHRONIZE=false # WARNING: Set false in production! (data DB# Local storage path (if STORAGE_TYPE=local)# STORAGE_LOCAL_PATH=./data/media-# Cap on how many files one local-storage listing call enumerates, so a large media directory cannot-# be walked into memory in a single request. A value that is not a positive integer fails the boot.-# Streaming reads are not affected — only the listing call is truncated.-# STORAGE_LIST_MAX_FILES=100000-# S3/MinIO settings (if STORAGE_TYPE=s3). Saving storage from the dashboard writes the credentials# below for you when you pick the built-in MinIO. Setting MINIO_BUILTIN=true by hand does not: it# starts the bundled container and nothing else, so a hand-edited file still needs the keys set.diff --git a/docker-compose.dev.yml b/docker-compose.dev.ymlindex 0c0ecf3b1..b39e7c901 100644--- a/docker-compose.dev.yml+++ b/docker-compose.dev.yml@@ -265,7 +265,6 @@ services:- STATUS_MEDIA_MAX_BYTES=${STATUS_MEDIA_MAX_BYTES:-}- STATUS_ORPHAN_SWEEP_INTERVAL_MS=${STATUS_ORPHAN_SWEEP_INTERVAL_MS:-}- STATUS_ORPHAN_GRACE_MS=${STATUS_ORPHAN_GRACE_MS:-}- - STORAGE_LIST_MAX_FILES=${STORAGE_LIST_MAX_FILES:-}- STORAGE_IMPORT_MAX_BYTES=${STORAGE_IMPORT_MAX_BYTES:-}- STORAGE_IMPORT_MAX_ENTRIES=${STORAGE_IMPORT_MAX_ENTRIES:-}- STORAGE_IMPORT_MAX_TOTAL_BYTES=${STORAGE_IMPORT_MAX_TOTAL_BYTES:-}diff --git a/docker-compose.yml b/docker-compose.ymlindex 5b4f9031c..5a913dbce 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -370,7 +370,6 @@ services:- STATUS_MEDIA_MAX_BYTES=${STATUS_MEDIA_MAX_BYTES:-}- STATUS_ORPHAN_SWEEP_INTERVAL_MS=${STATUS_ORPHAN_SWEEP_INTERVAL_MS:-}- STATUS_ORPHAN_GRACE_MS=${STATUS_ORPHAN_GRACE_MS:-}- - STORAGE_LIST_MAX_FILES=${STORAGE_LIST_MAX_FILES:-}- STORAGE_IMPORT_MAX_BYTES=${STORAGE_IMPORT_MAX_BYTES:-}- STORAGE_IMPORT_MAX_ENTRIES=${STORAGE_IMPORT_MAX_ENTRIES:-}- STORAGE_IMPORT_MAX_TOTAL_BYTES=${STORAGE_IMPORT_MAX_TOTAL_BYTES:-}diff --git a/docs/03-system-architecture.md b/docs/03-system-architecture.mdindex bed2f0baa..a68456781 100644--- a/docs/03-system-architecture.md+++ b/docs/03-system-architecture.md@@ -1333,7 +1333,7 @@ On S3 every key sits under the `S3_KEY_PREFIX` root (default `media/`).**MinIO is not a separate type** — it is the `s3` backend. The S3 client is created from credentialsalone, so plain AWS S3 works with no endpoint (the SDK derives one from the region); `S3_ENDPOINT` isfor S3-compatible stores (MinIO, R2, …), and setting it is also what enables `forcePathStyle: true`.-The public method set is `putFile` / `getFile` / `deleteFile` / `listFiles` / `iterateFiles` /+The public method set is `putFile` / `getFile` / `deleteFile` / `iterateFiles` /`createExportStream` (export) / `importFromStream` (import), plus `getFileCount`,`getCurrentStorageType` and `refreshS3Availability` (re-probes the bucket; the infra status endpointcalls it to report S3 reachability).@@ -1382,7 +1382,7 @@ export class StorageService {async getFile(filePath: string): Promise<Buffer> {/* mirrors putFile */}- async listFiles(): Promise<string[]> {+ async *iterateFiles(prefix = ''): AsyncGenerator<string> {/* local recurse, or S3 ListObjectsV2 under <S3_KEY_PREFIX, default media/> */}// createExportStream(): tar.gz of all files; importFromStream(): extract with zip-bomb capsdiff --git a/src/common/storage/orphan-sweep.ts b/src/common/storage/orphan-sweep.tsindex 60158d609..aea00ce4b 100644--- a/src/common/storage/orphan-sweep.ts+++ b/src/common/storage/orphan-sweep.ts@@ -10,9 +10,8 @@ import { StorageService } from './storage.service';* restart simply restarts the grace clock (fails safe) and a file mid-write is never reaped. The map* is pruned to the keys still orphaned at the end of the pass, so it cannot grow unbounded.*- * Files are enumerated with iterateFiles(), NOT listFiles(): listFiles() truncates at- * STORAGE_LIST_MAX_FILES (a per-call DoS guard, not a completeness contract), which would strand- * every orphan past the cap.+ * Files are streamed from iterateFiles() under the sweep's prefix, so an orphan is found however+ * large the store is.** Returns the number of files deleted. The summary log stays with the caller (each sweep reports in* its own wording); a failed delete is reported through `onDeleteFailed` and retried on a later pass.diff --git a/src/common/storage/s3-key-prefix.spec.ts b/src/common/storage/s3-key-prefix.spec.tsindex 4c1d2306c..dbad32089 100644--- a/src/common/storage/s3-key-prefix.spec.ts+++ b/src/common/storage/s3-key-prefix.spec.ts@@ -80,7 +80,7 @@ describe('StorageService S3 key prefix', () => {} as unknown as ConfigService;svc = new StorageService(config);await new Promise(resolve => setImmediate(resolve));- expect(svc.isS3Available()).toBe(true);+ expect((svc as unknown as { s3Available: boolean }).s3Available).toBe(true);return svc;}diff --git a/src/common/storage/storage-local-files.ts b/src/common/storage/storage-local-files.tsindex e811166a3..18bea94ca 100644--- a/src/common/storage/storage-local-files.ts+++ b/src/common/storage/storage-local-files.ts@@ -3,35 +3,13 @@ import * as path from 'path';import { Readable } from 'stream';import { isPathWithin } from '../utils/path-safety';-/** Max number of local files a single traversal enumerates. Bounds a count DoS on a huge media dir. */-const DEFAULT_LIST_MAX_FILES = 100_000;/** Max directory depth a local traversal descends. Prevents a pathological tree from running unbounded. */const LOCAL_TRAVERSAL_MAX_DEPTH = 20;-function positiveIntFromEnv(name: string, fallback: number): number {- const parsed = Number.parseInt(process.env[name] ?? '', 10);- return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;-}--/**- * Enumerate local files under the storage root, capped at STORAGE_LIST_MAX_FILES — a per-call- * DoS guard (a healthy media store stays well under it), NOT a completeness contract. Callers- * that must see the whole tree use iterateFiles().- */-export async function listLocalFiles(localPath: string): Promise<string[]> {- const maxFiles = positiveIntFromEnv('STORAGE_LIST_MAX_FILES', DEFAULT_LIST_MAX_FILES);- const files: string[] = [];- for await (const file of iterateLocalFiles(localPath)) {- files.push(file);- if (files.length >= maxFiles) break; // cap reached — stop early- }- return files;-}-/**- * Full local enumeration as a stream — no count cap. Async + iterative (a work queue, not- * recursion) so a deep/wide media tree can't block the event loop or stack-overflow; still- * bounded by the max directory depth. An incomplete traversal fails instead of hiding files.+ * Local enumeration as a stream. Async + iterative (a work queue, not recursion) so a deep/wide+ * media tree can't block the event loop or stack-overflow; still bounded by the max directory+ * depth. An incomplete traversal fails instead of hiding files.*/export async function* iterateLocalFiles(localPath: string, prefix = ''): AsyncGenerator<string> {// Iterative BFS: a queue of [relativeDir, depth] avoids unbounded call-stack growth. A prefixdiff --git a/src/common/storage/storage-transfer.ts b/src/common/storage/storage-transfer.tsindex 0b372789b..9909040ef 100644--- a/src/common/storage/storage-transfer.ts+++ b/src/common/storage/storage-transfer.ts@@ -55,9 +55,9 @@ export async function createExportStream(logger: LoggerService,): Promise<PassThrough> {const files = await listFiles();- // The importer aborts a whole archive past STORAGE_IMPORT_MAX_ENTRIES. Now that the export is- // uncapped, a large store can produce one this gateway refuses to restore — and the operator would- // only find out at restore time, after decommissioning the source. Say it at EXPORT time instead.+ // The importer aborts a whole archive past STORAGE_IMPORT_MAX_ENTRIES. The export carries the whole+ // store, so a large one can produce an archive this gateway refuses to restore, and the operator+ // would only find out at restore time, after decommissioning the source. Say it at EXPORT time instead.// Compared against the LOWER of this deployment's configured limit and the shipped default. The// default alone is destination-agnostic, which is right when restoring elsewhere — but it stays// silent for an operator who LOWERED the limit here and restores onto this same gateway, which isdiff --git a/src/common/storage/storage.service.s3-reprobe.spec.ts b/src/common/storage/storage.service.s3-reprobe.spec.tsindex 1bd8b9e1b..7144a2222 100644--- a/src/common/storage/storage.service.s3-reprobe.spec.ts+++ b/src/common/storage/storage.service.s3-reprobe.spec.ts@@ -87,13 +87,17 @@ describe('StorageService S3 re-probe and recovery', () => {return jest.spyOn(logger, 'warn').mockImplementation(() => undefined);}+ function s3AvailableOf(svc: StorageService): boolean {+ return (svc as unknown as { s3Available: boolean }).s3Available;+ }+it('falls back to local with a WARN when S3 is down at boot', async () => {mockSend.mockRejectedValue(s3Error('NetworkingError'));const svc = new StorageService(makeConfig());const warn = warnSpyOf(svc);await flush();- expect(svc.isS3Available()).toBe(false);+ expect(s3AvailableOf(svc)).toBe(false);expect(svc.getCurrentStorageType()).toBe('s3');expect(warn).toHaveBeenCalledWith(expect.stringContaining('degraded'));});@@ -109,7 +113,7 @@ describe('StorageService S3 re-probe and recovery', () => {expect(warn).toHaveBeenCalledTimes(2);await jest.advanceTimersByTimeAsync(DEFAULT_S3_REPROBE_INTERVAL_MS);expect(warn).toHaveBeenCalledTimes(3);- expect(svc.isS3Available()).toBe(false);+ expect(s3AvailableOf(svc)).toBe(false);});it('recovers to S3 via the periodic probe (no dashboard poll) and stops probing afterwards', async () => {@@ -117,12 +121,12 @@ describe('StorageService S3 re-probe and recovery', () => {const svc = new StorageService(makeConfig());const warn = warnSpyOf(svc);await flush();- expect(svc.isS3Available()).toBe(false);+ expect(s3AvailableOf(svc)).toBe(false);mockSend.mockResolvedValue({}); // S3 is backawait jest.advanceTimersByTimeAsync(DEFAULT_S3_REPROBE_INTERVAL_MS);- expect(svc.isS3Available()).toBe(true);+ expect(s3AvailableOf(svc)).toBe(true);expect(warn).toHaveBeenCalledWith(expect.stringContaining('recovered'));// Timer stops after recovery: no further probes, and availability never flips back on its own.@@ -130,7 +134,7 @@ describe('StorageService S3 re-probe and recovery', () => {mockSend.mockRejectedValue(s3Error('NetworkingError'));await jest.advanceTimersByTimeAsync(DEFAULT_S3_REPROBE_INTERVAL_MS * 5);expect(mockSend).not.toHaveBeenCalled();- expect(svc.isS3Available()).toBe(true);+ expect(s3AvailableOf(svc)).toBe(true);});it('creates the bucket on re-probe when the store was down at boot and came back empty', async () => {@@ -138,7 +142,7 @@ describe('StorageService S3 re-probe and recovery', () => {const svc = new StorageService(makeConfig());warnSpyOf(svc);await flush();- expect(svc.isS3Available()).toBe(false);+ expect(s3AvailableOf(svc)).toBe(false);// The store is up now, but on a fresh volume: HeadBucket answers NotFound, CreateBucket succeeds.mockSend.mockImplementation((cmd: unknown) =>@@ -147,7 +151,7 @@ describe('StorageService S3 re-probe and recovery', () => {await jest.advanceTimersByTimeAsync(DEFAULT_S3_REPROBE_INTERVAL_MS);expect(mockSend.mock.calls.some(([cmd]) => cmd instanceof CreateBucketCommand)).toBe(true);- expect(svc.isS3Available()).toBe(true);+ expect(s3AvailableOf(svc)).toBe(true);});it('treats losing a concurrent CreateBucket race (BucketAlreadyOwnedByYou) as the bucket being ready', async () => {@@ -162,7 +166,7 @@ describe('StorageService S3 re-probe and recovery', () => {);await expect(svc.refreshS3Availability()).resolves.toBe(true);- expect(svc.isS3Available()).toBe(true);+ expect(s3AvailableOf(svc)).toBe(true);});it('stays on the local fallback when the bucket name is owned by another account', async () => {@@ -176,7 +180,7 @@ describe('StorageService S3 re-probe and recovery', () => {);await expect(svc.refreshS3Availability()).resolves.toBe(false);- expect(svc.isS3Available()).toBe(false);+ expect(s3AvailableOf(svc)).toBe(false);});it('stays on the local fallback when the re-probe cannot create the missing bucket, and says why', async () => {@@ -191,7 +195,7 @@ describe('StorageService S3 re-probe and recovery', () => {);await jest.advanceTimersByTimeAsync(DEFAULT_S3_REPROBE_INTERVAL_MS);- expect(svc.isS3Available()).toBe(false);+ expect(s3AvailableOf(svc)).toBe(false);// The store answered: the periodic warning names the refused create, not the boot-time outage.expect(warn).toHaveBeenLastCalledWith(expect.stringContaining('AccessDenied'));});@@ -200,14 +204,14 @@ describe('StorageService S3 re-probe and recovery', () => {mockSend.mockResolvedValue({}); // boot probe succeedsconst svc = new StorageService(makeConfig());await flush();- expect(svc.isS3Available()).toBe(true);+ expect(s3AvailableOf(svc)).toBe(true);mockSend.mockClear();mockSend.mockRejectedValue(s3Error('NetworkingError'));await jest.advanceTimersByTimeAsync(DEFAULT_S3_REPROBE_INTERVAL_MS * 3);expect(mockSend).not.toHaveBeenCalled(); // no re-probe of a healthy backend- expect(svc.isS3Available()).toBe(true);+ expect(s3AvailableOf(svc)).toBe(true);});it('honors S3_REPROBE_INTERVAL_MS for the periodic probe', async () => {@@ -243,7 +247,7 @@ describe('StorageService S3 re-probe and recovery', () => {});const svc = new StorageService(makeConfig());await flush();- expect(svc.isS3Available()).toBe(true);+ expect(s3AvailableOf(svc)).toBe(true);// Simulate gap media: written to the local fallback dir while S3 was down, never uploaded.fs.mkdirSync(path.join(localPath, 'sub'), { recursive: true });@@ -337,7 +341,7 @@ describe('StorageService S3 re-probe and recovery', () => {mockSend.mockResolvedValue({}); // HeadBucket at boot, DeleteObject laterconst svc = new StorageService(makeConfig());await flush();- expect(svc.isS3Available()).toBe(true);+ expect(s3AvailableOf(svc)).toBe(true);// Gap media: written to the local fallback dir while S3 was down, never uploaded. Without the// delete-through, deleting the (missing) S3 key would leave these bytes orphaned forever.@@ -421,7 +425,7 @@ describe('StorageService S3 re-probe and recovery', () => {warnSpyOf(svc);await flush();await jest.advanceTimersByTimeAsync(S3_PROBE_TIMEOUT_MS); // the boot probe gives up too- expect(svc.isS3Available()).toBe(false);+ expect(s3AvailableOf(svc)).toBe(false);await expect(settleWithin(svc.refreshS3Availability())).resolves.toBe(false);@@ -435,7 +439,7 @@ describe('StorageService S3 re-probe and recovery', () => {}});- it('listFiles unions S3 objects with the local fallback dir (each key once)', async () => {+ it('iterateFiles unions S3 objects with the local fallback dir (each key once)', async () => {mockSend.mockImplementation((cmd: unknown) => {if (cmd instanceof ListObjectsV2Command) {return Promise.resolve({ Contents: [{ Key: 'media/s3-only.bin' }, { Key: 'media/shared.bin' }] });@@ -448,7 +452,8 @@ describe('StorageService S3 re-probe and recovery', () => {fs.writeFileSync(path.join(localPath, 'local-only.bin'), 'x');fs.writeFileSync(path.join(localPath, 'shared.bin'), 'stale-local-copy');- const files = await svc.listFiles();+ const files: string[] = [];+ for await (const file of svc.iterateFiles()) files.push(file);expect(files.sort()).toEqual(['local-only.bin', 's3-only.bin', 'shared.bin']);});diff --git a/src/common/storage/storage.service.s3.spec.ts b/src/common/storage/storage.service.s3.spec.tsindex 1578c14a9..713efa94e 100644--- a/src/common/storage/storage.service.s3.spec.ts+++ b/src/common/storage/storage.service.s3.spec.ts@@ -48,6 +48,9 @@ const ENV_KEYS = [// initializeS3Bucket() runs as a fire-and-forget promise; let its HeadBucket settle before asserting.const flush = (): Promise<void> => new Promise(resolve => setImmediate(resolve));+/** Whether the service currently routes to the bucket (the flag the boot probe and the re-probe set). */+const s3AvailableOf = (svc: StorageService): boolean => (svc as unknown as { s3Available: boolean }).s3Available;+/** The config passed to the most recent S3Client construction (throws if it was never constructed). */function lastConfig(): S3ClientConfig {const last = mockedS3Client.mock.calls.at(-1);@@ -97,7 +100,7 @@ describe('StorageService (s3) client init', () => {expect(cfg.forcePathStyle).toBeFalsy(); // AWS uses virtual-hosted addressingexpect(cfg.region).toBe('ap-southeast-1');expect(cfg.credentials).toEqual({ accessKeyId: 'AKIAEXAMPLE', secretAccessKey: 'shhh-secret' });- expect(svc.isS3Available()).toBe(true);+ expect(s3AvailableOf(svc)).toBe(true);});it('keeps endpoint + forcePathStyle for an S3-compatible store (MinIO)', async () => {@@ -111,7 +114,7 @@ describe('StorageService (s3) client init', () => {const cfg = lastConfig();expect(cfg.endpoint).toBe('http://minio:9000');expect(cfg.forcePathStyle).toBe(true);- expect(svc.isS3Available()).toBe(true);+ expect(s3AvailableOf(svc)).toBe(true);});// A store that accepts the connection and never answers would otherwise leave every request (the@@ -135,7 +138,7 @@ describe('StorageService (s3) client init', () => {await flush();expect(mockedS3Client).not.toHaveBeenCalled();- expect(svc.isS3Available()).toBe(false);+ expect(s3AvailableOf(svc)).toBe(false);expect(svc.getCurrentStorageType()).toBe('s3');expect(warn).toHaveBeenCalledWith(expect.stringContaining('S3_ACCESS_KEY_ID'));expect(warn).toHaveBeenCalledWith(expect.stringContaining('S3_SECRET_ACCESS_KEY'));@@ -164,6 +167,6 @@ describe('StorageService (s3) client init', () => {await flush();expect(lastConfig().credentials).toEqual({ accessKeyId: 'ENVKEY', secretAccessKey: 'ENVSECRET' });- expect(svc.isS3Available()).toBe(true);+ expect(s3AvailableOf(svc)).toBe(true);});});diff --git a/src/common/storage/storage.service.spec.ts b/src/common/storage/storage.service.spec.tsindex 488c48c7c..83a10fd95 100644--- a/src/common/storage/storage.service.spec.ts+++ b/src/common/storage/storage.service.spec.ts@@ -296,46 +296,33 @@ describe('StorageService local traversal (async + bounded)', () => {afterEach(() => {fs.rmSync(baseDir, { recursive: true, force: true });- delete process.env.STORAGE_LIST_MAX_FILES;});- it('lists files across nested subdirectories (async traversal)', async () => {+ it('enumerates files across nested subdirectories (async traversal)', async () => {await service.putFile('a.txt', Buffer.from('a'));await service.putFile('sub/b.txt', Buffer.from('b'));await service.putFile('sub/deep/c.txt', Buffer.from('c'));- const files = await service.listFiles();+ const files: string[] = [];+ for await (const file of service.iterateFiles()) files.push(file);expect(files.sort()).toEqual(['a.txt', 'sub/b.txt', 'sub/deep/c.txt']);});- it('stops at the STORAGE_LIST_MAX_FILES cap instead of enumerating a huge tree', async () => {- process.env.STORAGE_LIST_MAX_FILES = '5';- for (let i = 0; i < 20; i++) {- await service.putFile(`file${i}.txt`, Buffer.from('x'));- }-- const files = await service.listFiles();- expect(files.length).toBe(5); // capped, not 20- });-- it('iterateFiles enumerates the full tree, ignoring the STORAGE_LIST_MAX_FILES per-call cap', async () => {- process.env.STORAGE_LIST_MAX_FILES = '5';+ it('iterateFiles enumerates every file of a wide directory', async () => {for (let i = 0; i < 20; i++) {await service.putFile(`file${i}.txt`, Buffer.from('x'));}const seen: string[] = [];for await (const file of service.iterateFiles()) seen.push(file);- expect(seen.length).toBe(20); // complete where listFiles() above truncates at 5+ expect(seen.length).toBe(20);});});/**- * The export enumerated with listFiles(), which stops at STORAGE_LIST_MAX_FILES and returns without- * logging or throwing — so the documented local→S3 migration (export, repoint STORAGE_TYPE, import)- * silently left media behind, and the operator's own files/count pre-check was truncated by the same- * path. iterateFiles() exists for exactly this case: its own doc calls the cap "a per-call DoS guard,- * not a completeness contract" and tells callers needing the whole store to iterate instead.+ * The export and the files/count pre-check both walk the whole store through iterateFiles(), so the+ * documented local→S3 migration (export, repoint STORAGE_TYPE, import) cannot silently leave media+ * behind on the old backend.*/describe('StorageService.createExportStream enumerates the whole store', () => {const baseDirs: string[] = [];@@ -349,9 +336,8 @@ describe('StorageService.createExportStream enumerates the whole store', () => {for (const dir of baseDirs.splice(0)) fs.rmSync(dir, { recursive: true, force: true });});- it('walks the uncapped iterator rather than the capped listing', async () => {+ it('walks the whole store through iterateFiles', async () => {const { service } = makeService();- const listFiles = jest.spyOn(service, 'listFiles');const iterateFiles = jest.spyOn(service, 'iterateFiles').mockImplementation(async function* () {yield await Promise.resolve('media/a.bin');});@@ -361,7 +347,6 @@ describe('StorageService.createExportStream enumerates the whole store', () => {output.destroy();expect(iterateFiles).toHaveBeenCalled();- expect(listFiles).not.toHaveBeenCalled();});// The export fails on any open error other than a missing object, so a listed key openFile always@@ -399,14 +384,11 @@ describe('StorageService.createExportStream enumerates the whole store', () => {});/**- * The count is the OTHER half of the same fix and had no test of its own — reverting it to the- * capped listing left the suite green. It is the pre-check an operator runs before that migration,- * so a count truncated at the cap hides precisely the gap they are checking for, and hides it- * while agreeing with itself.+ * The count is the pre-check an operator runs before that migration, so it has to see every file+ * the export will carry.*/- it('counts with the uncapped iterator too, so the pre-check cannot hide the gap', async () => {+ it('counts through iterateFiles too, so the pre-check sees what the export carries', async () => {const { service } = makeService();- const listFiles = jest.spyOn(service, 'listFiles');const iterateFiles = jest.spyOn(service, 'iterateFiles').mockImplementation(async function* () {yield await Promise.resolve('media/a.bin');yield await Promise.resolve('media/b.bin');@@ -415,14 +397,13 @@ describe('StorageService.createExportStream enumerates the whole store', () => {const { count } = await service.getFileCount();expect(iterateFiles).toHaveBeenCalled();- expect(listFiles).not.toHaveBeenCalled();expect(count).toBe(2); // the iterator's items are what was counted, not an unrelated walk});/**- * Removing the cap from the enumeration also removed the bound on the SIZE loop underneath it,- * which stat'ed every file synchronously. On a large store that holds the event loop for the whole- * walk: health checks, webhooks and every in-flight request wait behind a count.+ * The size loop under the whole-store walk must not stat every file synchronously: on a large store+ * that holds the event loop for the whole walk, and health checks, webhooks and every in-flight+ * request wait behind a count.** Measured, not asserted structurally. A single "did something run during the call?" flag is NOT* discriminating here — the enumeration awaits before the stat loop begins, so that flag flipsdiff --git a/src/common/storage/storage.service.ts b/src/common/storage/storage.service.tsindex 52eba8956..a4e0a286d 100644--- a/src/common/storage/storage.service.ts+++ b/src/common/storage/storage.service.ts@@ -17,14 +17,7 @@ import { positiveIntFromEnv } from '../utils/env-int';import { isSafeStorageKey } from '../utils/path-safety';import { DEFAULT_S3_KEY_PREFIX, normalizeS3KeyPrefix } from './s3-key-prefix';import { createExportStream, ExportFileSource, importFromStream, isMissingObjectError } from './storage-transfer';-import {- listLocalFiles,- iterateLocalFiles,- getLocalFile,- openLocalFile,- putLocalFile,- deleteLocalFile,-} from './storage-local-files';+import { iterateLocalFiles, getLocalFile, openLocalFile, putLocalFile, deleteLocalFile } from './storage-local-files';// Re-exported so feature modules keep importing it next to StorageService.export { isMissingObjectError } from './storage-transfer';@@ -229,10 +222,6 @@ export class StorageService implements OnModuleDestroy {return this.storageType;}- isS3Available(): boolean {- return this.s3Available;- }-private lastS3Check = 0;/** Why the last probe failed. The fallback warning names it: a store refusing the create is not an outage. */private lastS3Error = '';@@ -275,29 +264,16 @@ export class StorageService implements OnModuleDestroy {return this.s3Available;}- async listFiles(): Promise<string[]> {- if (this.storageType === 's3' && this.s3Client && this.s3Available) {- // Union with the local fallback dir: media written locally while S3 was down is otherwise- // invisible to enumeration — getFile reads through to it, so a listing that omits it- // contradicts what reads serve, and a sweep reconciling against this listing could never- // reclaim it.- const files = new Set(await this.listS3Files());- for (const file of await this.listLocalFiles()) files.add(file);- return [...files];- }- return this.listLocalFiles();- }-/*** Enumerate every file in the store (all S3 pages unioned with the local fallback dir, each key- * once) as a stream — unlike listFiles(), whose STORAGE_LIST_MAX_FILES truncation is a per-call- * DoS guard, not a completeness contract. Callers that must reconcile against the full store- * (e.g. the status-media orphan sweep) should iterate this instead of a single capped listing.+ * once) as a stream. The union matters: media written locally while S3 was down is otherwise+ * invisible to enumeration, although getFile reads through to it, and a sweep reconciling against+ * the listing could never reclaim it. Callers that must reconcile against the full store (e.g.+ * the status-media orphan sweep) iterate this.*- * `prefix` narrows the walk at the source — the S3 ListObjectsV2 prefix and the local traversal- * root — so a caller reconciling one subtree neither pages the whole store nor holds every key- * of it in the dedupe Set. Removing the per-call cap must not trade one unbounded read for- * another.+ * `prefix` narrows the walk at the source (the S3 ListObjectsV2 prefix and the local traversal+ * root), so a caller reconciling one subtree neither pages the whole store nor holds every key+ * of it in the dedupe Set.*/async *iterateFiles(prefix = ''): AsyncGenerator<string> {if (this.storageType === 's3' && this.s3Client && this.s3Available) {@@ -376,15 +352,16 @@ export class StorageService implements OnModuleDestroy {// ListObjectsV2 already returns each object's Size, so report the real total instead of a// 100KB-per-file estimate — no extra API calls beyond the listing we'd do anyway.const s3 = await this.getS3CountAndSize();- // Union with the local fallback dir (same split-brain gap as listFiles): a local-only file- // counts with its real on-disk size; for a key present in both, the S3 copy is authoritative.+ // Union with the local fallback dir (media written locally while S3 was down is not in the+ // bucket listing): a local-only file counts with its real on-disk size; for a key present in+ // both, the S3 copy is authoritative.const s3Keys = new Set(s3.keys);let { count, sizeBytes } = s3;for await (const file of this.iterateLocalFiles()) {if (s3Keys.has(file)) continue;count += 1;try {- // Same reason as the local branch below: this walk is uncapped too, so the stat must yield.+ // Same reason as the local branch below: the walk covers the whole store, so the stat must yield.sizeBytes += (await fs.promises.stat(path.join(this.localPath, file))).size;} catch (error) {this.logger.debug(`Failed to stat file: ${file}`, { error: String(error) });@@ -393,17 +370,16 @@ export class StorageService implements OnModuleDestroy {return { count, sizeBytes };}- // The uncapped walk, for the same reason createExportStream uses it: this is the pre-check an- // operator runs BEFORE the export/import migration, so a count truncated at STORAGE_LIST_MAX_FILES- // would hide exactly the gap they are checking for.+ // The whole store is walked, for the same reason createExportStream does: this is the pre-check an+ // operator runs BEFORE the export/import migration, so it has to see every file the export will carry.const files = await this.listAllFiles();let sizeBytes = 0;for (const file of files) {try {const fullPath = path.join(this.localPath, file);- // Awaited, not statSync: uncapping the walk above also uncapped this loop, and a synchronous- // stat per file holds the event loop for the whole store — health checks, webhooks and every- // in-flight request wait behind a count. Measured at one event-loop tick for 2000 files.+ // Awaited, not statSync: a synchronous stat per file holds the event loop for the whole store,+ // so health checks, webhooks and every in-flight request wait behind a count. Measured at one+ // event-loop tick for 2000 files.const stats = await fs.promises.stat(fullPath);sizeBytes += stats.size;} catch (error) {@@ -450,12 +426,9 @@ export class StorageService implements OnModuleDestroy {// ============================================================================createExportStream(): Promise<PassThrough> {- // Enumerated with iterateFiles(), NOT listFiles(). listFiles() stops at STORAGE_LIST_MAX_FILES- // and returns without logging or throwing — a per-call DoS guard, as its own doc says, and not a- // completeness contract. Spending it here made the documented local→S3 migration (export,- // repoint STORAGE_TYPE, import) leave media behind on the old backend silently, and the- // operator's own files/count pre-check was truncated by the same path, so the consistency check- // could not reveal the gap. An export exists to be complete; that is what the uncapped walk is for.+ // The whole store is enumerated (iterateFiles): an export exists to be complete, and a short one+ // would leave media behind on the old backend silently after the documented local→S3 migration+ // (export, repoint STORAGE_TYPE, import).// A listed key openFile would refuse (an S3 object named `..`, or one sitting exactly at the key// root) is left out here: the export fails on any open error other than a missing object, and a// key that can never be opened must not fail every export.@@ -475,7 +448,7 @@ export class StorageService implements OnModuleDestroy {return false;}- /** Every key in the store, uncapped — the completeness counterpart to the capped listFiles(). */+ /** Every key in the store. */private async listAllFiles(): Promise<string[]> {const files: string[] = [];for await (const file of this.iterateFiles()) files.push(file);@@ -492,18 +465,9 @@ export class StorageService implements OnModuleDestroy {// ============================================================================/**- * Enumerate local files under the storage root, capped at STORAGE_LIST_MAX_FILES — a per-call- * DoS guard (a healthy media store stays well under it), NOT a completeness contract. Callers- * that must see the whole tree use iterateFiles().- */- private listLocalFiles(): Promise<string[]> {- return listLocalFiles(this.localPath);- }-- /**- * Full local enumeration as a stream — no count cap. Async + iterative (a work queue, not- * recursion) so a deep/wide media tree can't block the event loop or stack-overflow; still- * bounded by the max directory depth so a pathological tree can't descend unbounded.+ * Local enumeration as a stream. Async + iterative (a work queue, not recursion) so a deep/wide+ * media tree can't block the event loop or stack-overflow; still bounded by the max directory+ * depth so a pathological tree can't descend unbounded.*/private iterateLocalFiles(prefix = ''): AsyncGenerator<string> {return iterateLocalFiles(this.localPath, prefix);@@ -525,12 +489,6 @@ export class StorageService implements OnModuleDestroy {// S3 Storage Operations// ============================================================================- private async listS3Files(): Promise<string[]> {- const files: string[] = [];- for await (const file of this.iterateS3Files()) files.push(file);- return files;- }-/** Stream every S3 object key (prefix-stripped), following the ListObjectsV2 pagination token. */private async *iterateS3Files(prefix = ''): AsyncGenerator<string> {if (!this.s3Client) return;diff --git a/src/config/env-precedence.ts b/src/config/env-precedence.tsindex 10ebd9ad7..16090d262 100644--- a/src/config/env-precedence.ts+++ b/src/config/env-precedence.ts@@ -222,7 +222,6 @@ export const BLANK_SHADOWED_ENV_KEYS: string[] = ['STATUS_MEDIA_MAX_BYTES','STATUS_ORPHAN_SWEEP_INTERVAL_MS','STATUS_ORPHAN_GRACE_MS',- 'STORAGE_LIST_MAX_FILES','STORAGE_IMPORT_MAX_BYTES','STORAGE_IMPORT_MAX_ENTRIES','STORAGE_IMPORT_MAX_TOTAL_BYTES',diff --git a/src/config/env.validation.spec.ts b/src/config/env.validation.spec.tsindex cf7d4ab27..ecd8d61ac 100644--- a/src/config/env.validation.spec.ts+++ b/src/config/env.validation.spec.ts@@ -687,7 +687,6 @@ describe('validateEnv', () => {['STORAGE_IMPORT_MAX_BYTES', '200mb'],['STORAGE_IMPORT_MAX_ENTRIES', '1e5'],['STORAGE_IMPORT_MAX_TOTAL_BYTES', '10gb'],- ['STORAGE_LIST_MAX_FILES', '100k'],['BAILEYS_MESSAGE_STORE_LIMIT', '5k'],])('rejects a unit-suffixed or zero %s', (key, bad) => {expect(() => validateEnv({ [key]: bad })).toThrow(new RegExp(`${key} must be a positive integer`));diff --git a/src/config/env.validation.ts b/src/config/env.validation.tsindex e7ac04b28..c8ba71018 100644--- a/src/config/env.validation.ts+++ b/src/config/env.validation.ts@@ -428,7 +428,6 @@ export function validateEnv(config: EnvConfig): EnvConfig {'STORAGE_IMPORT_MAX_BYTES','STORAGE_IMPORT_MAX_ENTRIES','STORAGE_IMPORT_MAX_TOTAL_BYTES',- 'STORAGE_LIST_MAX_FILES','BAILEYS_MESSAGE_STORE_LIMIT',// Each read fell back to its default on 0 or garbage, or passed a negative or fractional value on// (SEARCH_LIMIT_MAX reached plugin search providers as-is).diff --git a/src/modules/infra/infra-status.controller.spec.ts b/src/modules/infra/infra-status.controller.spec.tsindex a025aefed..2d6bfd228 100644--- a/src/modules/infra/infra-status.controller.spec.ts+++ b/src/modules/infra/infra-status.controller.spec.ts@@ -140,7 +140,7 @@ describe('InfraStatusController.getStatus engine (reads the real engine.puppetee{} as never, // engineFactory{ isDockerAvailable: () => false } as never, // dockerService — no Docker in unit testscache as never,- { isS3Available: () => false, refreshS3Availability: () => Promise.resolve(false) } as never, // storageService+ { refreshS3Availability: () => Promise.resolve(false) } as never, // storageService);const status = await controller.getStatus();@@ -159,7 +159,7 @@ describe('InfraStatusController.getStatus engine (reads the real engine.puppetee{} as never,{ isDockerAvailable: () => false } as never,{ isAvailable: () => Promise.resolve(false) } as never,- { isS3Available: () => false, refreshS3Availability: () => Promise.resolve(false) } as never,+ { refreshS3Availability: () => Promise.resolve(false) } as never,);const status = await controller.getStatus();@@ -189,7 +189,7 @@ describe('InfraStatusController.getStatus storage (reads the real storage.localP{} as never, // engineFactory{ isDockerAvailable: () => false } as never, // dockerService — no Docker in unit testscache as never,- { isS3Available: () => false, refreshS3Availability: () => Promise.resolve(false) } as never, // storageService+ { refreshS3Availability: () => Promise.resolve(false) } as never, // storageService);};@@ -240,7 +240,7 @@ describe('InfraStatusController.getStatus envPinned', () => {{} as never,{ isDockerAvailable: () => false } as never,{ isAvailable: () => Promise.resolve(false) } as never,- { isS3Available: () => false, refreshS3Availability: () => Promise.resolve(false) } as never,+ { refreshS3Availability: () => Promise.resolve(false) } as never,);};diff --git a/src/modules/infra/infra-storage.controller.ts b/src/modules/infra/infra-storage.controller.tsindex 247a7da02..6d44b269d 100644--- a/src/modules/infra/infra-storage.controller.ts+++ b/src/modules/infra/infra-storage.controller.ts@@ -187,7 +187,7 @@ export class InfraStorageController implements OnApplicationBootstrap {stream.destroy();reject(err);});- // pipe() does NOT forward source errors: an archiver/gzip failure surfaces as an 'error' event on+ // pipe() does NOT forward source errors: a tar/gzip failure surfaces as an 'error' event on// the source stream, which without a listener crashes the process. Fail the request instead and// tear down the sink so its fd isn't held open waiting for a 'finish' that never comes.stream.on('error', (err: Error) => {diff --git a/src/modules/status-store/status-store.service.spec.ts b/src/modules/status-store/status-store.service.spec.tsindex dac588925..6288464f9 100644--- a/src/modules/status-store/status-store.service.spec.ts+++ b/src/modules/status-store/status-store.service.spec.ts@@ -874,22 +874,13 @@ describe('StatusStoreService.sweepOrphanedMedia', () => {expect(fs.existsSync(path.join(baseDir, 'media', 'statuses', 'sess', 'orphan.jpg'))).toBe(false);});- it('enumerates past the listFiles() cap so orphans beyond it are still reaped', async () => {- // listFiles() truncates at STORAGE_LIST_MAX_FILES (a per-call DoS guard, not a completeness- // contract); the sweep reconciles against the FULL store, so it streams iterateFiles() instead.- process.env.STORAGE_LIST_MAX_FILES = '5';- try {- for (let i = 0; i < 8; i++) writeFile(`statuses/sess/orphan-${i}.jpg`, 'orphan');- // The capped call the sweep used to make would see only 5 of the 8 orphans.- expect((await storageService.listFiles()).filter(f => f.startsWith('statuses/'))).toHaveLength(5);-- const t0 = Date.now();- expect(await service.sweepOrphanedMedia(t0)).toBe(0); // first sighting starts the grace clock- expect(await service.sweepOrphanedMedia(t0 + 61 * 60 * 1000)).toBe(8); // all 8, not the capped 5- expect(fs.readdirSync(path.join(baseDir, 'media', 'statuses', 'sess'))).toHaveLength(0);- } finally {- delete process.env.STORAGE_LIST_MAX_FILES;- }+ it('reaps every orphan under its prefix, not just the first few', async () => {+ for (let i = 0; i < 8; i++) writeFile(`statuses/sess/orphan-${i}.jpg`, 'orphan');++ const t0 = Date.now();+ expect(await service.sweepOrphanedMedia(t0)).toBe(0); // first sighting starts the grace clock+ expect(await service.sweepOrphanedMedia(t0 + 61 * 60 * 1000)).toBe(8);+ expect(fs.readdirSync(path.join(baseDir, 'media', 'statuses', 'sess'))).toHaveLength(0);});});