Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

refactor(storage): remove the file listing cap

Yudhi Armyndharis91648f390122
diff --git a/.env.example b/.env.example
index 80c8d7a62..bce621078 100644
--- a/.env.example
+++ b/.env.example
@@ -419,11 +419,6 @@ DATABASE_SYNCHRONIZE=false # WARNING: Set false in production! (data DB
# Local storage path (if STORAGE_TYPE=local)
# STORAGE_LOCAL_PATH=./data/media
-# Cap on how many files one local-storage listing call enumerates, so a large media directory cannot
-# be walked into memory in a single request. A value that is not a positive integer fails the boot.
-# Streaming reads are not affected — only the listing call is truncated.
-# STORAGE_LIST_MAX_FILES=100000
-
# S3/MinIO settings (if STORAGE_TYPE=s3). Saving storage from the dashboard writes the credentials
# below for you when you pick the built-in MinIO. Setting MINIO_BUILTIN=true by hand does not: it
# starts the bundled container and nothing else, so a hand-edited file still needs the keys set.
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index 0c0ecf3b1..b39e7c901 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -265,7 +265,6 @@ services:
- STATUS_MEDIA_MAX_BYTES=${STATUS_MEDIA_MAX_BYTES:-}
- STATUS_ORPHAN_SWEEP_INTERVAL_MS=${STATUS_ORPHAN_SWEEP_INTERVAL_MS:-}
- STATUS_ORPHAN_GRACE_MS=${STATUS_ORPHAN_GRACE_MS:-}
- - STORAGE_LIST_MAX_FILES=${STORAGE_LIST_MAX_FILES:-}
- STORAGE_IMPORT_MAX_BYTES=${STORAGE_IMPORT_MAX_BYTES:-}
- STORAGE_IMPORT_MAX_ENTRIES=${STORAGE_IMPORT_MAX_ENTRIES:-}
- STORAGE_IMPORT_MAX_TOTAL_BYTES=${STORAGE_IMPORT_MAX_TOTAL_BYTES:-}
diff --git a/docker-compose.yml b/docker-compose.yml
index 5b4f9031c..5a913dbce 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -370,7 +370,6 @@ services:
- STATUS_MEDIA_MAX_BYTES=${STATUS_MEDIA_MAX_BYTES:-}
- STATUS_ORPHAN_SWEEP_INTERVAL_MS=${STATUS_ORPHAN_SWEEP_INTERVAL_MS:-}
- STATUS_ORPHAN_GRACE_MS=${STATUS_ORPHAN_GRACE_MS:-}
- - STORAGE_LIST_MAX_FILES=${STORAGE_LIST_MAX_FILES:-}
- STORAGE_IMPORT_MAX_BYTES=${STORAGE_IMPORT_MAX_BYTES:-}
- STORAGE_IMPORT_MAX_ENTRIES=${STORAGE_IMPORT_MAX_ENTRIES:-}
- STORAGE_IMPORT_MAX_TOTAL_BYTES=${STORAGE_IMPORT_MAX_TOTAL_BYTES:-}
diff --git a/docs/03-system-architecture.md b/docs/03-system-architecture.md
index bed2f0baa..a68456781 100644
--- a/docs/03-system-architecture.md
+++ b/docs/03-system-architecture.md
@@ -1333,7 +1333,7 @@ On S3 every key sits under the `S3_KEY_PREFIX` root (default `media/`).
**MinIO is not a separate type** — it is the `s3` backend. The S3 client is created from credentials
alone, so plain AWS S3 works with no endpoint (the SDK derives one from the region); `S3_ENDPOINT` is
for S3-compatible stores (MinIO, R2, …), and setting it is also what enables `forcePathStyle: true`.
-The public method set is `putFile` / `getFile` / `deleteFile` / `listFiles` / `iterateFiles` /
+The public method set is `putFile` / `getFile` / `deleteFile` / `iterateFiles` /
`createExportStream` (export) / `importFromStream` (import), plus `getFileCount`,
`getCurrentStorageType` and `refreshS3Availability` (re-probes the bucket; the infra status endpoint
calls it to report S3 reachability).
@@ -1382,7 +1382,7 @@ export class StorageService {
async getFile(filePath: string): Promise<Buffer> {
/* mirrors putFile */
}
- async listFiles(): Promise<string[]> {
+ async *iterateFiles(prefix = ''): AsyncGenerator<string> {
/* local recurse, or S3 ListObjectsV2 under <S3_KEY_PREFIX, default media/> */
}
// createExportStream(): tar.gz of all files; importFromStream(): extract with zip-bomb caps
diff --git a/src/common/storage/orphan-sweep.ts b/src/common/storage/orphan-sweep.ts
index 60158d609..aea00ce4b 100644
--- a/src/common/storage/orphan-sweep.ts
+++ b/src/common/storage/orphan-sweep.ts
@@ -10,9 +10,8 @@ import { StorageService } from './storage.service';
* restart simply restarts the grace clock (fails safe) and a file mid-write is never reaped. The map
* is pruned to the keys still orphaned at the end of the pass, so it cannot grow unbounded.
*
- * Files are enumerated with iterateFiles(), NOT listFiles(): listFiles() truncates at
- * STORAGE_LIST_MAX_FILES (a per-call DoS guard, not a completeness contract), which would strand
- * every orphan past the cap.
+ * Files are streamed from iterateFiles() under the sweep's prefix, so an orphan is found however
+ * large the store is.
*
* Returns the number of files deleted. The summary log stays with the caller (each sweep reports in
* its own wording); a failed delete is reported through `onDeleteFailed` and retried on a later pass.
diff --git a/src/common/storage/s3-key-prefix.spec.ts b/src/common/storage/s3-key-prefix.spec.ts
index 4c1d2306c..dbad32089 100644
--- a/src/common/storage/s3-key-prefix.spec.ts
+++ b/src/common/storage/s3-key-prefix.spec.ts
@@ -80,7 +80,7 @@ describe('StorageService S3 key prefix', () => {
} as unknown as ConfigService;
svc = new StorageService(config);
await new Promise(resolve => setImmediate(resolve));
- expect(svc.isS3Available()).toBe(true);
+ expect((svc as unknown as { s3Available: boolean }).s3Available).toBe(true);
return svc;
}
diff --git a/src/common/storage/storage-local-files.ts b/src/common/storage/storage-local-files.ts
index e811166a3..18bea94ca 100644
--- a/src/common/storage/storage-local-files.ts
+++ b/src/common/storage/storage-local-files.ts
@@ -3,35 +3,13 @@ import * as path from 'path';
import { Readable } from 'stream';
import { isPathWithin } from '../utils/path-safety';
-/** Max number of local files a single traversal enumerates. Bounds a count DoS on a huge media dir. */
-const DEFAULT_LIST_MAX_FILES = 100_000;
/** Max directory depth a local traversal descends. Prevents a pathological tree from running unbounded. */
const LOCAL_TRAVERSAL_MAX_DEPTH = 20;
-function positiveIntFromEnv(name: string, fallback: number): number {
- const parsed = Number.parseInt(process.env[name] ?? '', 10);
- return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
-}
-
-/**
- * Enumerate local files under the storage root, capped at STORAGE_LIST_MAX_FILES — a per-call
- * DoS guard (a healthy media store stays well under it), NOT a completeness contract. Callers
- * that must see the whole tree use iterateFiles().
- */
-export async function listLocalFiles(localPath: string): Promise<string[]> {
- const maxFiles = positiveIntFromEnv('STORAGE_LIST_MAX_FILES', DEFAULT_LIST_MAX_FILES);
- const files: string[] = [];
- for await (const file of iterateLocalFiles(localPath)) {
- files.push(file);
- if (files.length >= maxFiles) break; // cap reached — stop early
- }
- return files;
-}
-
/**
- * Full local enumeration as a stream — no count cap. Async + iterative (a work queue, not
- * recursion) so a deep/wide media tree can't block the event loop or stack-overflow; still
- * bounded by the max directory depth. An incomplete traversal fails instead of hiding files.
+ * Local enumeration as a stream. Async + iterative (a work queue, not recursion) so a deep/wide
+ * media tree can't block the event loop or stack-overflow; still bounded by the max directory
+ * depth. An incomplete traversal fails instead of hiding files.
*/
export async function* iterateLocalFiles(localPath: string, prefix = ''): AsyncGenerator<string> {
// Iterative BFS: a queue of [relativeDir, depth] avoids unbounded call-stack growth. A prefix
diff --git a/src/common/storage/storage-transfer.ts b/src/common/storage/storage-transfer.ts
index 0b372789b..9909040ef 100644
--- a/src/common/storage/storage-transfer.ts
+++ b/src/common/storage/storage-transfer.ts
@@ -55,9 +55,9 @@ export async function createExportStream(
logger: LoggerService,
): Promise<PassThrough> {
const files = await listFiles();
- // The importer aborts a whole archive past STORAGE_IMPORT_MAX_ENTRIES. Now that the export is
- // uncapped, a large store can produce one this gateway refuses to restore — and the operator would
- // only find out at restore time, after decommissioning the source. Say it at EXPORT time instead.
+ // The importer aborts a whole archive past STORAGE_IMPORT_MAX_ENTRIES. The export carries the whole
+ // store, so a large one can produce an archive this gateway refuses to restore, and the operator
+ // would only find out at restore time, after decommissioning the source. Say it at EXPORT time instead.
// Compared against the LOWER of this deployment's configured limit and the shipped default. The
// default alone is destination-agnostic, which is right when restoring elsewhere — but it stays
// silent for an operator who LOWERED the limit here and restores onto this same gateway, which is
diff --git a/src/common/storage/storage.service.s3-reprobe.spec.ts b/src/common/storage/storage.service.s3-reprobe.spec.ts
index 1bd8b9e1b..7144a2222 100644
--- a/src/common/storage/storage.service.s3-reprobe.spec.ts
+++ b/src/common/storage/storage.service.s3-reprobe.spec.ts
@@ -87,13 +87,17 @@ describe('StorageService S3 re-probe and recovery', () => {
return jest.spyOn(logger, 'warn').mockImplementation(() => undefined);
}
+ function s3AvailableOf(svc: StorageService): boolean {
+ return (svc as unknown as { s3Available: boolean }).s3Available;
+ }
+
it('falls back to local with a WARN when S3 is down at boot', async () => {
mockSend.mockRejectedValue(s3Error('NetworkingError'));
const svc = new StorageService(makeConfig());
const warn = warnSpyOf(svc);
await flush();
- expect(svc.isS3Available()).toBe(false);
+ expect(s3AvailableOf(svc)).toBe(false);
expect(svc.getCurrentStorageType()).toBe('s3');
expect(warn).toHaveBeenCalledWith(expect.stringContaining('degraded'));
});
@@ -109,7 +113,7 @@ describe('StorageService S3 re-probe and recovery', () => {
expect(warn).toHaveBeenCalledTimes(2);
await jest.advanceTimersByTimeAsync(DEFAULT_S3_REPROBE_INTERVAL_MS);
expect(warn).toHaveBeenCalledTimes(3);
- expect(svc.isS3Available()).toBe(false);
+ expect(s3AvailableOf(svc)).toBe(false);
});
it('recovers to S3 via the periodic probe (no dashboard poll) and stops probing afterwards', async () => {
@@ -117,12 +121,12 @@ describe('StorageService S3 re-probe and recovery', () => {
const svc = new StorageService(makeConfig());
const warn = warnSpyOf(svc);
await flush();
- expect(svc.isS3Available()).toBe(false);
+ expect(s3AvailableOf(svc)).toBe(false);
mockSend.mockResolvedValue({}); // S3 is back
await jest.advanceTimersByTimeAsync(DEFAULT_S3_REPROBE_INTERVAL_MS);
- expect(svc.isS3Available()).toBe(true);
+ expect(s3AvailableOf(svc)).toBe(true);
expect(warn).toHaveBeenCalledWith(expect.stringContaining('recovered'));
// Timer stops after recovery: no further probes, and availability never flips back on its own.
@@ -130,7 +134,7 @@ describe('StorageService S3 re-probe and recovery', () => {
mockSend.mockRejectedValue(s3Error('NetworkingError'));
await jest.advanceTimersByTimeAsync(DEFAULT_S3_REPROBE_INTERVAL_MS * 5);
expect(mockSend).not.toHaveBeenCalled();
- expect(svc.isS3Available()).toBe(true);
+ expect(s3AvailableOf(svc)).toBe(true);
});
it('creates the bucket on re-probe when the store was down at boot and came back empty', async () => {
@@ -138,7 +142,7 @@ describe('StorageService S3 re-probe and recovery', () => {
const svc = new StorageService(makeConfig());
warnSpyOf(svc);
await flush();
- expect(svc.isS3Available()).toBe(false);
+ expect(s3AvailableOf(svc)).toBe(false);
// The store is up now, but on a fresh volume: HeadBucket answers NotFound, CreateBucket succeeds.
mockSend.mockImplementation((cmd: unknown) =>
@@ -147,7 +151,7 @@ describe('StorageService S3 re-probe and recovery', () => {
await jest.advanceTimersByTimeAsync(DEFAULT_S3_REPROBE_INTERVAL_MS);
expect(mockSend.mock.calls.some(([cmd]) => cmd instanceof CreateBucketCommand)).toBe(true);
- expect(svc.isS3Available()).toBe(true);
+ expect(s3AvailableOf(svc)).toBe(true);
});
it('treats losing a concurrent CreateBucket race (BucketAlreadyOwnedByYou) as the bucket being ready', async () => {
@@ -162,7 +166,7 @@ describe('StorageService S3 re-probe and recovery', () => {
);
await expect(svc.refreshS3Availability()).resolves.toBe(true);
- expect(svc.isS3Available()).toBe(true);
+ expect(s3AvailableOf(svc)).toBe(true);
});
it('stays on the local fallback when the bucket name is owned by another account', async () => {
@@ -176,7 +180,7 @@ describe('StorageService S3 re-probe and recovery', () => {
);
await expect(svc.refreshS3Availability()).resolves.toBe(false);
- expect(svc.isS3Available()).toBe(false);
+ expect(s3AvailableOf(svc)).toBe(false);
});
it('stays on the local fallback when the re-probe cannot create the missing bucket, and says why', async () => {
@@ -191,7 +195,7 @@ describe('StorageService S3 re-probe and recovery', () => {
);
await jest.advanceTimersByTimeAsync(DEFAULT_S3_REPROBE_INTERVAL_MS);
- expect(svc.isS3Available()).toBe(false);
+ expect(s3AvailableOf(svc)).toBe(false);
// The store answered: the periodic warning names the refused create, not the boot-time outage.
expect(warn).toHaveBeenLastCalledWith(expect.stringContaining('AccessDenied'));
});
@@ -200,14 +204,14 @@ describe('StorageService S3 re-probe and recovery', () => {
mockSend.mockResolvedValue({}); // boot probe succeeds
const svc = new StorageService(makeConfig());
await flush();
- expect(svc.isS3Available()).toBe(true);
+ expect(s3AvailableOf(svc)).toBe(true);
mockSend.mockClear();
mockSend.mockRejectedValue(s3Error('NetworkingError'));
await jest.advanceTimersByTimeAsync(DEFAULT_S3_REPROBE_INTERVAL_MS * 3);
expect(mockSend).not.toHaveBeenCalled(); // no re-probe of a healthy backend
- expect(svc.isS3Available()).toBe(true);
+ expect(s3AvailableOf(svc)).toBe(true);
});
it('honors S3_REPROBE_INTERVAL_MS for the periodic probe', async () => {
@@ -243,7 +247,7 @@ describe('StorageService S3 re-probe and recovery', () => {
});
const svc = new StorageService(makeConfig());
await flush();
- expect(svc.isS3Available()).toBe(true);
+ expect(s3AvailableOf(svc)).toBe(true);
// Simulate gap media: written to the local fallback dir while S3 was down, never uploaded.
fs.mkdirSync(path.join(localPath, 'sub'), { recursive: true });
@@ -337,7 +341,7 @@ describe('StorageService S3 re-probe and recovery', () => {
mockSend.mockResolvedValue({}); // HeadBucket at boot, DeleteObject later
const svc = new StorageService(makeConfig());
await flush();
- expect(svc.isS3Available()).toBe(true);
+ expect(s3AvailableOf(svc)).toBe(true);
// Gap media: written to the local fallback dir while S3 was down, never uploaded. Without the
// delete-through, deleting the (missing) S3 key would leave these bytes orphaned forever.
@@ -421,7 +425,7 @@ describe('StorageService S3 re-probe and recovery', () => {
warnSpyOf(svc);
await flush();
await jest.advanceTimersByTimeAsync(S3_PROBE_TIMEOUT_MS); // the boot probe gives up too
- expect(svc.isS3Available()).toBe(false);
+ expect(s3AvailableOf(svc)).toBe(false);
await expect(settleWithin(svc.refreshS3Availability())).resolves.toBe(false);
@@ -435,7 +439,7 @@ describe('StorageService S3 re-probe and recovery', () => {
}
});
- it('listFiles unions S3 objects with the local fallback dir (each key once)', async () => {
+ it('iterateFiles unions S3 objects with the local fallback dir (each key once)', async () => {
mockSend.mockImplementation((cmd: unknown) => {
if (cmd instanceof ListObjectsV2Command) {
return Promise.resolve({ Contents: [{ Key: 'media/s3-only.bin' }, { Key: 'media/shared.bin' }] });
@@ -448,7 +452,8 @@ describe('StorageService S3 re-probe and recovery', () => {
fs.writeFileSync(path.join(localPath, 'local-only.bin'), 'x');
fs.writeFileSync(path.join(localPath, 'shared.bin'), 'stale-local-copy');
- const files = await svc.listFiles();
+ const files: string[] = [];
+ for await (const file of svc.iterateFiles()) files.push(file);
expect(files.sort()).toEqual(['local-only.bin', 's3-only.bin', 'shared.bin']);
});
diff --git a/src/common/storage/storage.service.s3.spec.ts b/src/common/storage/storage.service.s3.spec.ts
index 1578c14a9..713efa94e 100644
--- a/src/common/storage/storage.service.s3.spec.ts
+++ b/src/common/storage/storage.service.s3.spec.ts
@@ -48,6 +48,9 @@ const ENV_KEYS = [
// initializeS3Bucket() runs as a fire-and-forget promise; let its HeadBucket settle before asserting.
const flush = (): Promise<void> => new Promise(resolve => setImmediate(resolve));
+/** Whether the service currently routes to the bucket (the flag the boot probe and the re-probe set). */
+const s3AvailableOf = (svc: StorageService): boolean => (svc as unknown as { s3Available: boolean }).s3Available;
+
/** The config passed to the most recent S3Client construction (throws if it was never constructed). */
function lastConfig(): S3ClientConfig {
const last = mockedS3Client.mock.calls.at(-1);
@@ -97,7 +100,7 @@ describe('StorageService (s3) client init', () => {
expect(cfg.forcePathStyle).toBeFalsy(); // AWS uses virtual-hosted addressing
expect(cfg.region).toBe('ap-southeast-1');
expect(cfg.credentials).toEqual({ accessKeyId: 'AKIAEXAMPLE', secretAccessKey: 'shhh-secret' });
- expect(svc.isS3Available()).toBe(true);
+ expect(s3AvailableOf(svc)).toBe(true);
});
it('keeps endpoint + forcePathStyle for an S3-compatible store (MinIO)', async () => {
@@ -111,7 +114,7 @@ describe('StorageService (s3) client init', () => {
const cfg = lastConfig();
expect(cfg.endpoint).toBe('http://minio:9000');
expect(cfg.forcePathStyle).toBe(true);
- expect(svc.isS3Available()).toBe(true);
+ expect(s3AvailableOf(svc)).toBe(true);
});
// A store that accepts the connection and never answers would otherwise leave every request (the
@@ -135,7 +138,7 @@ describe('StorageService (s3) client init', () => {
await flush();
expect(mockedS3Client).not.toHaveBeenCalled();
- expect(svc.isS3Available()).toBe(false);
+ expect(s3AvailableOf(svc)).toBe(false);
expect(svc.getCurrentStorageType()).toBe('s3');
expect(warn).toHaveBeenCalledWith(expect.stringContaining('S3_ACCESS_KEY_ID'));
expect(warn).toHaveBeenCalledWith(expect.stringContaining('S3_SECRET_ACCESS_KEY'));
@@ -164,6 +167,6 @@ describe('StorageService (s3) client init', () => {
await flush();
expect(lastConfig().credentials).toEqual({ accessKeyId: 'ENVKEY', secretAccessKey: 'ENVSECRET' });
- expect(svc.isS3Available()).toBe(true);
+ expect(s3AvailableOf(svc)).toBe(true);
});
});
diff --git a/src/common/storage/storage.service.spec.ts b/src/common/storage/storage.service.spec.ts
index 488c48c7c..83a10fd95 100644
--- a/src/common/storage/storage.service.spec.ts
+++ b/src/common/storage/storage.service.spec.ts
@@ -296,46 +296,33 @@ describe('StorageService local traversal (async + bounded)', () => {
afterEach(() => {
fs.rmSync(baseDir, { recursive: true, force: true });
- delete process.env.STORAGE_LIST_MAX_FILES;
});
- it('lists files across nested subdirectories (async traversal)', async () => {
+ it('enumerates files across nested subdirectories (async traversal)', async () => {
await service.putFile('a.txt', Buffer.from('a'));
await service.putFile('sub/b.txt', Buffer.from('b'));
await service.putFile('sub/deep/c.txt', Buffer.from('c'));
- const files = await service.listFiles();
+ const files: string[] = [];
+ for await (const file of service.iterateFiles()) files.push(file);
expect(files.sort()).toEqual(['a.txt', 'sub/b.txt', 'sub/deep/c.txt']);
});
- it('stops at the STORAGE_LIST_MAX_FILES cap instead of enumerating a huge tree', async () => {
- process.env.STORAGE_LIST_MAX_FILES = '5';
- for (let i = 0; i < 20; i++) {
- await service.putFile(`file${i}.txt`, Buffer.from('x'));
- }
-
- const files = await service.listFiles();
- expect(files.length).toBe(5); // capped, not 20
- });
-
- it('iterateFiles enumerates the full tree, ignoring the STORAGE_LIST_MAX_FILES per-call cap', async () => {
- process.env.STORAGE_LIST_MAX_FILES = '5';
+ it('iterateFiles enumerates every file of a wide directory', async () => {
for (let i = 0; i < 20; i++) {
await service.putFile(`file${i}.txt`, Buffer.from('x'));
}
const seen: string[] = [];
for await (const file of service.iterateFiles()) seen.push(file);
- expect(seen.length).toBe(20); // complete where listFiles() above truncates at 5
+ expect(seen.length).toBe(20);
});
});
/**
- * The export enumerated with listFiles(), which stops at STORAGE_LIST_MAX_FILES and returns without
- * logging or throwing — so the documented local→S3 migration (export, repoint STORAGE_TYPE, import)
- * silently left media behind, and the operator's own files/count pre-check was truncated by the same
- * path. iterateFiles() exists for exactly this case: its own doc calls the cap "a per-call DoS guard,
- * not a completeness contract" and tells callers needing the whole store to iterate instead.
+ * The export and the files/count pre-check both walk the whole store through iterateFiles(), so the
+ * documented local→S3 migration (export, repoint STORAGE_TYPE, import) cannot silently leave media
+ * behind on the old backend.
*/
describe('StorageService.createExportStream enumerates the whole store', () => {
const baseDirs: string[] = [];
@@ -349,9 +336,8 @@ describe('StorageService.createExportStream enumerates the whole store', () => {
for (const dir of baseDirs.splice(0)) fs.rmSync(dir, { recursive: true, force: true });
});
- it('walks the uncapped iterator rather than the capped listing', async () => {
+ it('walks the whole store through iterateFiles', async () => {
const { service } = makeService();
- const listFiles = jest.spyOn(service, 'listFiles');
const iterateFiles = jest.spyOn(service, 'iterateFiles').mockImplementation(async function* () {
yield await Promise.resolve('media/a.bin');
});
@@ -361,7 +347,6 @@ describe('StorageService.createExportStream enumerates the whole store', () => {
output.destroy();
expect(iterateFiles).toHaveBeenCalled();
- expect(listFiles).not.toHaveBeenCalled();
});
// The export fails on any open error other than a missing object, so a listed key openFile always
@@ -399,14 +384,11 @@ describe('StorageService.createExportStream enumerates the whole store', () => {
});
/**
- * The count is the OTHER half of the same fix and had no test of its own — reverting it to the
- * capped listing left the suite green. It is the pre-check an operator runs before that migration,
- * so a count truncated at the cap hides precisely the gap they are checking for, and hides it
- * while agreeing with itself.
+ * The count is the pre-check an operator runs before that migration, so it has to see every file
+ * the export will carry.
*/
- it('counts with the uncapped iterator too, so the pre-check cannot hide the gap', async () => {
+ it('counts through iterateFiles too, so the pre-check sees what the export carries', async () => {
const { service } = makeService();
- const listFiles = jest.spyOn(service, 'listFiles');
const iterateFiles = jest.spyOn(service, 'iterateFiles').mockImplementation(async function* () {
yield await Promise.resolve('media/a.bin');
yield await Promise.resolve('media/b.bin');
@@ -415,14 +397,13 @@ describe('StorageService.createExportStream enumerates the whole store', () => {
const { count } = await service.getFileCount();
expect(iterateFiles).toHaveBeenCalled();
- expect(listFiles).not.toHaveBeenCalled();
expect(count).toBe(2); // the iterator's items are what was counted, not an unrelated walk
});
/**
- * Removing the cap from the enumeration also removed the bound on the SIZE loop underneath it,
- * which stat'ed every file synchronously. On a large store that holds the event loop for the whole
- * walk: health checks, webhooks and every in-flight request wait behind a count.
+ * The size loop under the whole-store walk must not stat every file synchronously: on a large store
+ * that holds the event loop for the whole walk, and health checks, webhooks and every in-flight
+ * request wait behind a count.
*
* Measured, not asserted structurally. A single "did something run during the call?" flag is NOT
* discriminating here — the enumeration awaits before the stat loop begins, so that flag flips
diff --git a/src/common/storage/storage.service.ts b/src/common/storage/storage.service.ts
index 52eba8956..a4e0a286d 100644
--- a/src/common/storage/storage.service.ts
+++ b/src/common/storage/storage.service.ts
@@ -17,14 +17,7 @@ import { positiveIntFromEnv } from '../utils/env-int';
import { isSafeStorageKey } from '../utils/path-safety';
import { DEFAULT_S3_KEY_PREFIX, normalizeS3KeyPrefix } from './s3-key-prefix';
import { createExportStream, ExportFileSource, importFromStream, isMissingObjectError } from './storage-transfer';
-import {
- listLocalFiles,
- iterateLocalFiles,
- getLocalFile,
- openLocalFile,
- putLocalFile,
- deleteLocalFile,
-} from './storage-local-files';
+import { iterateLocalFiles, getLocalFile, openLocalFile, putLocalFile, deleteLocalFile } from './storage-local-files';
// Re-exported so feature modules keep importing it next to StorageService.
export { isMissingObjectError } from './storage-transfer';
@@ -229,10 +222,6 @@ export class StorageService implements OnModuleDestroy {
return this.storageType;
}
- isS3Available(): boolean {
- return this.s3Available;
- }
-
private lastS3Check = 0;
/** Why the last probe failed. The fallback warning names it: a store refusing the create is not an outage. */
private lastS3Error = '';
@@ -275,29 +264,16 @@ export class StorageService implements OnModuleDestroy {
return this.s3Available;
}
- async listFiles(): Promise<string[]> {
- if (this.storageType === 's3' && this.s3Client && this.s3Available) {
- // Union with the local fallback dir: media written locally while S3 was down is otherwise
- // invisible to enumeration — getFile reads through to it, so a listing that omits it
- // contradicts what reads serve, and a sweep reconciling against this listing could never
- // reclaim it.
- const files = new Set(await this.listS3Files());
- for (const file of await this.listLocalFiles()) files.add(file);
- return [...files];
- }
- return this.listLocalFiles();
- }
-
/**
* Enumerate every file in the store (all S3 pages unioned with the local fallback dir, each key
- * once) as a stream — unlike listFiles(), whose STORAGE_LIST_MAX_FILES truncation is a per-call
- * DoS guard, not a completeness contract. Callers that must reconcile against the full store
- * (e.g. the status-media orphan sweep) should iterate this instead of a single capped listing.
+ * once) as a stream. The union matters: media written locally while S3 was down is otherwise
+ * invisible to enumeration, although getFile reads through to it, and a sweep reconciling against
+ * the listing could never reclaim it. Callers that must reconcile against the full store (e.g.
+ * the status-media orphan sweep) iterate this.
*
- * `prefix` narrows the walk at the source — the S3 ListObjectsV2 prefix and the local traversal
- * root — so a caller reconciling one subtree neither pages the whole store nor holds every key
- * of it in the dedupe Set. Removing the per-call cap must not trade one unbounded read for
- * another.
+ * `prefix` narrows the walk at the source (the S3 ListObjectsV2 prefix and the local traversal
+ * root), so a caller reconciling one subtree neither pages the whole store nor holds every key
+ * of it in the dedupe Set.
*/
async *iterateFiles(prefix = ''): AsyncGenerator<string> {
if (this.storageType === 's3' && this.s3Client && this.s3Available) {
@@ -376,15 +352,16 @@ export class StorageService implements OnModuleDestroy {
// ListObjectsV2 already returns each object's Size, so report the real total instead of a
// 100KB-per-file estimate — no extra API calls beyond the listing we'd do anyway.
const s3 = await this.getS3CountAndSize();
- // Union with the local fallback dir (same split-brain gap as listFiles): a local-only file
- // counts with its real on-disk size; for a key present in both, the S3 copy is authoritative.
+ // Union with the local fallback dir (media written locally while S3 was down is not in the
+ // bucket listing): a local-only file counts with its real on-disk size; for a key present in
+ // both, the S3 copy is authoritative.
const s3Keys = new Set(s3.keys);
let { count, sizeBytes } = s3;
for await (const file of this.iterateLocalFiles()) {
if (s3Keys.has(file)) continue;
count += 1;
try {
- // Same reason as the local branch below: this walk is uncapped too, so the stat must yield.
+ // Same reason as the local branch below: the walk covers the whole store, so the stat must yield.
sizeBytes += (await fs.promises.stat(path.join(this.localPath, file))).size;
} catch (error) {
this.logger.debug(`Failed to stat file: ${file}`, { error: String(error) });
@@ -393,17 +370,16 @@ export class StorageService implements OnModuleDestroy {
return { count, sizeBytes };
}
- // The uncapped walk, for the same reason createExportStream uses it: this is the pre-check an
- // operator runs BEFORE the export/import migration, so a count truncated at STORAGE_LIST_MAX_FILES
- // would hide exactly the gap they are checking for.
+ // The whole store is walked, for the same reason createExportStream does: this is the pre-check an
+ // operator runs BEFORE the export/import migration, so it has to see every file the export will carry.
const files = await this.listAllFiles();
let sizeBytes = 0;
for (const file of files) {
try {
const fullPath = path.join(this.localPath, file);
- // Awaited, not statSync: uncapping the walk above also uncapped this loop, and a synchronous
- // stat per file holds the event loop for the whole store — health checks, webhooks and every
- // in-flight request wait behind a count. Measured at one event-loop tick for 2000 files.
+ // Awaited, not statSync: a synchronous stat per file holds the event loop for the whole store,
+ // so health checks, webhooks and every in-flight request wait behind a count. Measured at one
+ // event-loop tick for 2000 files.
const stats = await fs.promises.stat(fullPath);
sizeBytes += stats.size;
} catch (error) {
@@ -450,12 +426,9 @@ export class StorageService implements OnModuleDestroy {
// ============================================================================
createExportStream(): Promise<PassThrough> {
- // Enumerated with iterateFiles(), NOT listFiles(). listFiles() stops at STORAGE_LIST_MAX_FILES
- // and returns without logging or throwing — a per-call DoS guard, as its own doc says, and not a
- // completeness contract. Spending it here made the documented local→S3 migration (export,
- // repoint STORAGE_TYPE, import) leave media behind on the old backend silently, and the
- // operator's own files/count pre-check was truncated by the same path, so the consistency check
- // could not reveal the gap. An export exists to be complete; that is what the uncapped walk is for.
+ // The whole store is enumerated (iterateFiles): an export exists to be complete, and a short one
+ // would leave media behind on the old backend silently after the documented local→S3 migration
+ // (export, repoint STORAGE_TYPE, import).
// A listed key openFile would refuse (an S3 object named `..`, or one sitting exactly at the key
// root) is left out here: the export fails on any open error other than a missing object, and a
// key that can never be opened must not fail every export.
@@ -475,7 +448,7 @@ export class StorageService implements OnModuleDestroy {
return false;
}
- /** Every key in the store, uncapped — the completeness counterpart to the capped listFiles(). */
+ /** Every key in the store. */
private async listAllFiles(): Promise<string[]> {
const files: string[] = [];
for await (const file of this.iterateFiles()) files.push(file);
@@ -492,18 +465,9 @@ export class StorageService implements OnModuleDestroy {
// ============================================================================
/**
- * Enumerate local files under the storage root, capped at STORAGE_LIST_MAX_FILES — a per-call
- * DoS guard (a healthy media store stays well under it), NOT a completeness contract. Callers
- * that must see the whole tree use iterateFiles().
- */
- private listLocalFiles(): Promise<string[]> {
- return listLocalFiles(this.localPath);
- }
-
- /**
- * Full local enumeration as a stream — no count cap. Async + iterative (a work queue, not
- * recursion) so a deep/wide media tree can't block the event loop or stack-overflow; still
- * bounded by the max directory depth so a pathological tree can't descend unbounded.
+ * Local enumeration as a stream. Async + iterative (a work queue, not recursion) so a deep/wide
+ * media tree can't block the event loop or stack-overflow; still bounded by the max directory
+ * depth so a pathological tree can't descend unbounded.
*/
private iterateLocalFiles(prefix = ''): AsyncGenerator<string> {
return iterateLocalFiles(this.localPath, prefix);
@@ -525,12 +489,6 @@ export class StorageService implements OnModuleDestroy {
// S3 Storage Operations
// ============================================================================
- private async listS3Files(): Promise<string[]> {
- const files: string[] = [];
- for await (const file of this.iterateS3Files()) files.push(file);
- return files;
- }
-
/** Stream every S3 object key (prefix-stripped), following the ListObjectsV2 pagination token. */
private async *iterateS3Files(prefix = ''): AsyncGenerator<string> {
if (!this.s3Client) return;
diff --git a/src/config/env-precedence.ts b/src/config/env-precedence.ts
index 10ebd9ad7..16090d262 100644
--- a/src/config/env-precedence.ts
+++ b/src/config/env-precedence.ts
@@ -222,7 +222,6 @@ export const BLANK_SHADOWED_ENV_KEYS: string[] = [
'STATUS_MEDIA_MAX_BYTES',
'STATUS_ORPHAN_SWEEP_INTERVAL_MS',
'STATUS_ORPHAN_GRACE_MS',
- 'STORAGE_LIST_MAX_FILES',
'STORAGE_IMPORT_MAX_BYTES',
'STORAGE_IMPORT_MAX_ENTRIES',
'STORAGE_IMPORT_MAX_TOTAL_BYTES',
diff --git a/src/config/env.validation.spec.ts b/src/config/env.validation.spec.ts
index cf7d4ab27..ecd8d61ac 100644
--- a/src/config/env.validation.spec.ts
+++ b/src/config/env.validation.spec.ts
@@ -687,7 +687,6 @@ describe('validateEnv', () => {
['STORAGE_IMPORT_MAX_BYTES', '200mb'],
['STORAGE_IMPORT_MAX_ENTRIES', '1e5'],
['STORAGE_IMPORT_MAX_TOTAL_BYTES', '10gb'],
- ['STORAGE_LIST_MAX_FILES', '100k'],
['BAILEYS_MESSAGE_STORE_LIMIT', '5k'],
])('rejects a unit-suffixed or zero %s', (key, bad) => {
expect(() => validateEnv({ [key]: bad })).toThrow(new RegExp(`${key} must be a positive integer`));
diff --git a/src/config/env.validation.ts b/src/config/env.validation.ts
index e7ac04b28..c8ba71018 100644
--- a/src/config/env.validation.ts
+++ b/src/config/env.validation.ts
@@ -428,7 +428,6 @@ export function validateEnv(config: EnvConfig): EnvConfig {
'STORAGE_IMPORT_MAX_BYTES',
'STORAGE_IMPORT_MAX_ENTRIES',
'STORAGE_IMPORT_MAX_TOTAL_BYTES',
- 'STORAGE_LIST_MAX_FILES',
'BAILEYS_MESSAGE_STORE_LIMIT',
// Each read fell back to its default on 0 or garbage, or passed a negative or fractional value on
// (SEARCH_LIMIT_MAX reached plugin search providers as-is).
diff --git a/src/modules/infra/infra-status.controller.spec.ts b/src/modules/infra/infra-status.controller.spec.ts
index a025aefed..2d6bfd228 100644
--- a/src/modules/infra/infra-status.controller.spec.ts
+++ b/src/modules/infra/infra-status.controller.spec.ts
@@ -140,7 +140,7 @@ describe('InfraStatusController.getStatus engine (reads the real engine.puppetee
{} as never, // engineFactory
{ isDockerAvailable: () => false } as never, // dockerService — no Docker in unit tests
cache as never,
- { isS3Available: () => false, refreshS3Availability: () => Promise.resolve(false) } as never, // storageService
+ { refreshS3Availability: () => Promise.resolve(false) } as never, // storageService
);
const status = await controller.getStatus();
@@ -159,7 +159,7 @@ describe('InfraStatusController.getStatus engine (reads the real engine.puppetee
{} as never,
{ isDockerAvailable: () => false } as never,
{ isAvailable: () => Promise.resolve(false) } as never,
- { isS3Available: () => false, refreshS3Availability: () => Promise.resolve(false) } as never,
+ { refreshS3Availability: () => Promise.resolve(false) } as never,
);
const status = await controller.getStatus();
@@ -189,7 +189,7 @@ describe('InfraStatusController.getStatus storage (reads the real storage.localP
{} as never, // engineFactory
{ isDockerAvailable: () => false } as never, // dockerService — no Docker in unit tests
cache as never,
- { isS3Available: () => false, refreshS3Availability: () => Promise.resolve(false) } as never, // storageService
+ { refreshS3Availability: () => Promise.resolve(false) } as never, // storageService
);
};
@@ -240,7 +240,7 @@ describe('InfraStatusController.getStatus envPinned', () => {
{} as never,
{ isDockerAvailable: () => false } as never,
{ isAvailable: () => Promise.resolve(false) } as never,
- { isS3Available: () => false, refreshS3Availability: () => Promise.resolve(false) } as never,
+ { refreshS3Availability: () => Promise.resolve(false) } as never,
);
};
diff --git a/src/modules/infra/infra-storage.controller.ts b/src/modules/infra/infra-storage.controller.ts
index 247a7da02..6d44b269d 100644
--- a/src/modules/infra/infra-storage.controller.ts
+++ b/src/modules/infra/infra-storage.controller.ts
@@ -187,7 +187,7 @@ export class InfraStorageController implements OnApplicationBootstrap {
stream.destroy();
reject(err);
});
- // pipe() does NOT forward source errors: an archiver/gzip failure surfaces as an 'error' event on
+ // pipe() does NOT forward source errors: a tar/gzip failure surfaces as an 'error' event on
// the source stream, which without a listener crashes the process. Fail the request instead and
// tear down the sink so its fd isn't held open waiting for a 'finish' that never comes.
stream.on('error', (err: Error) => {
diff --git a/src/modules/status-store/status-store.service.spec.ts b/src/modules/status-store/status-store.service.spec.ts
index dac588925..6288464f9 100644
--- a/src/modules/status-store/status-store.service.spec.ts
+++ b/src/modules/status-store/status-store.service.spec.ts
@@ -874,22 +874,13 @@ describe('StatusStoreService.sweepOrphanedMedia', () => {
expect(fs.existsSync(path.join(baseDir, 'media', 'statuses', 'sess', 'orphan.jpg'))).toBe(false);
});
- it('enumerates past the listFiles() cap so orphans beyond it are still reaped', async () => {
- // listFiles() truncates at STORAGE_LIST_MAX_FILES (a per-call DoS guard, not a completeness
- // contract); the sweep reconciles against the FULL store, so it streams iterateFiles() instead.
- process.env.STORAGE_LIST_MAX_FILES = '5';
- try {
- for (let i = 0; i < 8; i++) writeFile(`statuses/sess/orphan-${i}.jpg`, 'orphan');
- // The capped call the sweep used to make would see only 5 of the 8 orphans.
- expect((await storageService.listFiles()).filter(f => f.startsWith('statuses/'))).toHaveLength(5);
-
- const t0 = Date.now();
- expect(await service.sweepOrphanedMedia(t0)).toBe(0); // first sighting starts the grace clock
- expect(await service.sweepOrphanedMedia(t0 + 61 * 60 * 1000)).toBe(8); // all 8, not the capped 5
- expect(fs.readdirSync(path.join(baseDir, 'media', 'statuses', 'sess'))).toHaveLength(0);
- } finally {
- delete process.env.STORAGE_LIST_MAX_FILES;
- }
+ it('reaps every orphan under its prefix, not just the first few', async () => {
+ for (let i = 0; i < 8; i++) writeFile(`statuses/sess/orphan-${i}.jpg`, 'orphan');
+
+ const t0 = Date.now();
+ expect(await service.sweepOrphanedMedia(t0)).toBe(0); // first sighting starts the grace clock
+ expect(await service.sweepOrphanedMedia(t0 + 61 * 60 * 1000)).toBe(8);
+ expect(fs.readdirSync(path.join(baseDir, 'media', 'statuses', 'sess'))).toHaveLength(0);
});
});