Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(storage): let the orphan sweeps skip unreadable directories
Yudhi Armyndharisf82852b8dcec
diff --git a/src/common/storage/orphan-sweep.spec.ts b/src/common/storage/orphan-sweep.spec.tsindex cfd7c3309..b327eee23 100644--- a/src/common/storage/orphan-sweep.spec.ts+++ b/src/common/storage/orphan-sweep.spec.ts@@ -1,4 +1,9 @@+import * as fs from 'fs';+import * as os from 'os';+import * as path from 'path';+import type { ConfigService } from '@nestjs/config';import { sweepOrphanedFiles } from './orphan-sweep';+import { StorageService } from './storage.service';describe('sweepOrphanedFiles', () => {const PREFIX = 'status/';@@ -94,3 +99,66 @@ describe('sweepOrphanedFiles', () => {expect(referencedAmong.mock.calls.map(([keys]) => keys.length)).toEqual([2, 2, 1]);});});++/**+ * The export and the file count refuse a store they cannot enumerate completely, because a short archive+ * would lose media on the next backend. A sweep has no such duty: it deletes only files it saw, so a+ * subtree it cannot read is merely left alone and must not stop it from reclaiming everything else.+ */+describe('sweepOrphanedFiles over a store with a directory it cannot read', () => {+ let baseDir: string;+ let storage: StorageService;++ beforeEach(() => {+ baseDir = fs.mkdtempSync(path.join(os.tmpdir(), 'owa-sweep-'));+ const localPath = path.join(baseDir, 'media');+ for (const file of ['statuses/s1/orphan.bin', 'statuses/locked/hidden.bin', 'chat-media/s1/other.bin']) {+ fs.mkdirSync(path.dirname(path.join(localPath, file)), { recursive: true });+ fs.writeFileSync(path.join(localPath, file), 'x');+ }+ storage = new StorageService({+ get: (key: string) => ({ 'storage.type': 'local', 'storage.localPath': localPath })[key],+ } as unknown as ConfigService);+ // A mode-000 directory is still readable by root, so the failure is injected for that one path.+ const readdir = fs.promises.readdir.bind(fs.promises) as (...args: unknown[]) => Promise<unknown>;+ jest+ .spyOn(fs.promises, 'readdir')+ .mockImplementation(((target: fs.PathLike, ...rest: unknown[]) =>+ String(target).endsWith(path.join('statuses', 'locked'))+ ? Promise.reject(Object.assign(new Error('EACCES: permission denied, scandir'), { code: 'EACCES' }))+ : readdir(target, ...rest)) as never);+ jest.spyOn(console, 'warn').mockImplementation(() => undefined);+ });++ afterEach(() => {+ jest.restoreAllMocks();+ storage.onModuleDestroy();+ fs.rmSync(baseDir, { recursive: true, force: true });+ });++ const sweep = (prefix: string) =>+ sweepOrphanedFiles({+ storage,+ prefix,+ graceMs: 0,+ now: 1000,+ firstSeenAt: new Map(),+ referencedAmong: () => Promise.resolve(new Set<string | undefined>()),+ onDeleteFailed: jest.fn(),+ });++ it('reclaims the orphans it can see and never touches the subtree it cannot', async () => {+ expect(await sweep('statuses/')).toBe(1);++ const present = (file: string): boolean => fs.existsSync(path.join(baseDir, 'media', file));+ expect(present('statuses/s1/orphan.bin')).toBe(false);+ expect(present('statuses/locked/hidden.bin')).toBe(true);+ expect(present('chat-media/s1/other.bin')).toBe(true);+ const warnings = (console.warn as jest.Mock).mock.calls.map(([line]) => String(line));+ expect(warnings.some(line => line.includes('statuses/locked'))).toBe(true);+ });++ it('keeps refusing a file count of the same store', async () => {+ await expect(storage.getFileCount()).rejects.toThrow(/EACCES/);+ });+});diff --git a/src/common/storage/orphan-sweep.ts b/src/common/storage/orphan-sweep.tsindex aea00ce4b..89c04e005 100644--- a/src/common/storage/orphan-sweep.ts+++ b/src/common/storage/orphan-sweep.ts@@ -72,7 +72,8 @@ export async function sweepOrphanedFiles(options: {chunk = [];};- for await (const file of storage.iterateFiles(prefix)) {+ // A directory the walk cannot read is left alone: this pass deletes only files it was shown.+ for await (const file of storage.iterateFiles(prefix, { skipUnreadable: true })) {if (!file.startsWith(prefix)) continue;chunk.push(file);if (options.chunkSize !== undefined && chunk.length >= options.chunkSize) await flush();diff --git a/src/common/storage/storage-local-files.spec.ts b/src/common/storage/storage-local-files.spec.tsnew file mode 100644index 000000000..931d42afb--- /dev/null+++ b/src/common/storage/storage-local-files.spec.ts@@ -0,0 +1,109 @@+import * as fs from 'fs';+import * as os from 'os';+import * as path from 'path';+import { iterateLocalFiles, type LocalWalkOptions } from './storage-local-files';++describe('iterateLocalFiles', () => {+ let root: string;++ beforeEach(() => {+ root = fs.mkdtempSync(path.join(os.tmpdir(), 'owa-walk-'));+ });++ afterEach(() => {+ jest.restoreAllMocks();+ fs.rmSync(root, { recursive: true, force: true });+ });++ const write = (file: string): void => {+ fs.mkdirSync(path.dirname(path.join(root, file)), { recursive: true });+ fs.writeFileSync(path.join(root, file), 'x');+ };++ const walk = async (prefix = '', options?: LocalWalkOptions): Promise<string[]> => {+ const files: string[] = [];+ for await (const file of iterateLocalFiles(root, prefix, options)) files.push(file);+ return files.sort();+ };++ /** A mode-000 directory is still readable by root, so the failure is injected for the one path. */+ const failReaddirAt = (suffix: string, code = 'EACCES'): Error => {+ const error = Object.assign(new Error(`${code}: cannot read ${suffix}`), { code });+ const readdir = fs.promises.readdir.bind(fs.promises) as (...args: unknown[]) => Promise<unknown>;+ jest+ .spyOn(fs.promises, 'readdir')+ .mockImplementation(((target: fs.PathLike, ...rest: unknown[]) =>+ String(target).endsWith(suffix) ? Promise.reject(error) : readdir(target, ...rest)) as never);+ return error;+ };++ const deepKey = [...Array<string>(20).fill('d'), 'deep.bin'].join('/');++ it('lists every file with "/" separated keys', async () => {+ write('a.bin');+ write('x/y/b.bin');++ expect(await walk()).toEqual(['a.bin', 'x/y/b.bin']);+ expect(await walk('x/')).toEqual(['x/y/b.bin']);+ });++ describe('by default', () => {+ it('fails on a directory it cannot read, with the error it got', async () => {+ write('ok.bin');+ write('locked/hidden.bin');+ const error = failReaddirAt(path.join(root, 'locked'));++ await expect(walk()).rejects.toBe(error);+ });++ it('fails on a directory below the maximum depth', async () => {+ write(deepKey);++ await expect(walk()).rejects.toThrow(/maximum traversal depth/);+ });++ it('treats a directory removed during the walk as empty', async () => {+ write('ok.bin');+ write('gone/file.bin');+ failReaddirAt(path.join(root, 'gone'), 'ENOENT');++ expect(await walk()).toEqual(['ok.bin']);+ });+ });++ describe('when asked to skip what it cannot read', () => {+ const lenient = (): { options: LocalWalkOptions; warn: jest.Mock } => {+ const warn = jest.fn();+ return { options: { skipUnreadable: true, logger: { warn } }, warn };+ };++ it('skips an unreadable directory, names it, and still lists the rest', async () => {+ write('ok.bin');+ write('locked/hidden.bin');+ write('other/fine.bin');+ failReaddirAt(path.join(root, 'locked'));+ const { options, warn } = lenient();++ expect(await walk('', options)).toEqual(['ok.bin', 'other/fine.bin']);+ expect(warn).toHaveBeenCalledTimes(1);+ expect(warn).toHaveBeenCalledWith(expect.stringContaining('"locked"'));+ });++ it('skips a directory below the maximum depth and still lists the rest', async () => {+ write('ok.bin');+ write(deepKey);+ const { options, warn } = lenient();++ expect(await walk('', options)).toEqual(['ok.bin']);+ expect(warn).toHaveBeenCalledWith(expect.stringContaining('maximum traversal depth'));+ });++ it('skips without a logger too', async () => {+ write('ok.bin');+ write('locked/hidden.bin');+ failReaddirAt(path.join(root, 'locked'));++ expect(await walk('', { skipUnreadable: true })).toEqual(['ok.bin']);+ });+ });+});diff --git a/src/common/storage/storage-local-files.ts b/src/common/storage/storage-local-files.tsindex 18bea94ca..d180c83c8 100644--- a/src/common/storage/storage-local-files.ts+++ b/src/common/storage/storage-local-files.ts@@ -1,17 +1,39 @@import * as fs from 'fs';import * as path from 'path';import { Readable } from 'stream';+import type { LoggerService } from '../services/logger.service';import { isPathWithin } from '../utils/path-safety';/** Max directory depth a local traversal descends. Prevents a pathological tree from running unbounded. */const LOCAL_TRAVERSAL_MAX_DEPTH = 20;+export interface LocalWalkOptions {+ /**+ * Skip a path the walk cannot enumerate, naming it in a warning, instead of failing the walk. Only for a+ * caller that acts on the files it saw and nothing else, such as an orphan sweep: it leaves the subtree it+ * could not read alone. An export or a count must not set it, because a short answer hides files.+ */+ skipUnreadable?: boolean;+ logger?: Pick<LoggerService, 'warn'>;+}+/*** Local enumeration as a stream. Async + iterative (a work queue, not recursion) so a deep/wide* media tree can't block the event loop or stack-overflow; still bounded by the max directory- * depth. An incomplete traversal fails instead of hiding files.+ * depth. An incomplete traversal fails instead of hiding files, unless the caller opts out.*/-export async function* iterateLocalFiles(localPath: string, prefix = ''): AsyncGenerator<string> {+export async function* iterateLocalFiles(+ localPath: string,+ prefix = '',+ options: LocalWalkOptions = {},+): AsyncGenerator<string> {+ /** A path the walk cannot enumerate: fail the walk, or skip it and say so when the caller allows it. */+ const unreadable = (entry: string, error: Error): void => {+ if (!options.skipUnreadable) throw error;+ options.logger?.warn(+ `Skipping a storage path that cannot be enumerated: ${JSON.stringify(entry)} (${error.message})`,+ );+ };// Iterative BFS: a queue of [relativeDir, depth] avoids unbounded call-stack growth. A prefix// ending in '/' names a subtree, so start the walk there instead of filtering afterwards.const root = prefix.endsWith('/') ? prefix.slice(0, -1) : '';@@ -21,7 +43,8 @@ export async function* iterateLocalFiles(localPath: string, prefix = ''): AsyncGwhile (queue.length > 0) {const { dir, depth } = queue.shift()!;if (depth >= LOCAL_TRAVERSAL_MAX_DEPTH) {- throw new Error(`Storage directory exceeds the maximum traversal depth: ${dir}`);+ unreadable(dir, new Error(`Storage directory exceeds the maximum traversal depth: ${dir}`));+ continue;}const fullPath = path.join(localPath, dir);@@ -29,8 +52,9 @@ export async function* iterateLocalFiles(localPath: string, prefix = ''): AsyncGtry {entries = await fs.promises.readdir(fullPath, { withFileTypes: true });} catch (error) {- if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;- continue; // A directory removed during enumeration has no files left to export.+ // A directory removed during enumeration has no files left to export.+ if ((error as NodeJS.ErrnoException).code !== 'ENOENT') unreadable(dir, error as Error);+ continue;}for (const entry of entries) {diff --git a/src/common/storage/storage.service.ts b/src/common/storage/storage.service.tsindex a4e0a286d..3158a384b 100644--- a/src/common/storage/storage.service.ts+++ b/src/common/storage/storage.service.ts@@ -274,20 +274,24 @@ export class StorageService implements OnModuleDestroy {* `prefix` narrows the walk at the source (the S3 ListObjectsV2 prefix and the local traversal* root), so a caller reconciling one subtree neither pages the whole store nor holds every key* of it in the dedupe Set.+ *+ * A local directory the walk cannot read fails it, so no caller is handed a short listing without+ * knowing. `skipUnreadable` is for a caller that only acts on the files it was shown (an orphan+ * sweep): the directory is named in a warning and left alone.*/- async *iterateFiles(prefix = ''): AsyncGenerator<string> {+ async *iterateFiles(prefix = '', options: { skipUnreadable?: boolean } = {}): AsyncGenerator<string> {if (this.storageType === 's3' && this.s3Client && this.s3Available) {const seen = new Set<string>();for await (const file of this.iterateS3Files(prefix)) {seen.add(file);yield file;}- for await (const file of this.iterateLocalFiles(prefix)) {+ for await (const file of this.iterateLocalFiles(prefix, options.skipUnreadable)) {if (!seen.has(file)) yield file;}return;}- yield* this.iterateLocalFiles(prefix);+ yield* this.iterateLocalFiles(prefix, options.skipUnreadable);}async getFile(filePath: string): Promise<Buffer> {@@ -469,8 +473,8 @@ export class StorageService implements OnModuleDestroy {* media tree can't block the event loop or stack-overflow; still bounded by the max directory* depth so a pathological tree can't descend unbounded.*/- private iterateLocalFiles(prefix = ''): AsyncGenerator<string> {- return iterateLocalFiles(this.localPath, prefix);+ private iterateLocalFiles(prefix = '', skipUnreadable = false): AsyncGenerator<string> {+ return iterateLocalFiles(this.localPath, prefix, { skipUnreadable, logger: this.logger });}private getLocalFile(filePath: string): Promise<Buffer> {