Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(storage): let the orphan sweeps skip unreadable directories

Yudhi Armyndharisf82852b8dcec
diff --git a/src/common/storage/orphan-sweep.spec.ts b/src/common/storage/orphan-sweep.spec.ts
index cfd7c3309..b327eee23 100644
--- a/src/common/storage/orphan-sweep.spec.ts
+++ b/src/common/storage/orphan-sweep.spec.ts
@@ -1,4 +1,9 @@
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import type { ConfigService } from '@nestjs/config';
import { sweepOrphanedFiles } from './orphan-sweep';
+import { StorageService } from './storage.service';
describe('sweepOrphanedFiles', () => {
const PREFIX = 'status/';
@@ -94,3 +99,66 @@ describe('sweepOrphanedFiles', () => {
expect(referencedAmong.mock.calls.map(([keys]) => keys.length)).toEqual([2, 2, 1]);
});
});
+
+/**
+ * The export and the file count refuse a store they cannot enumerate completely, because a short archive
+ * would lose media on the next backend. A sweep has no such duty: it deletes only files it saw, so a
+ * subtree it cannot read is merely left alone and must not stop it from reclaiming everything else.
+ */
+describe('sweepOrphanedFiles over a store with a directory it cannot read', () => {
+ let baseDir: string;
+ let storage: StorageService;
+
+ beforeEach(() => {
+ baseDir = fs.mkdtempSync(path.join(os.tmpdir(), 'owa-sweep-'));
+ const localPath = path.join(baseDir, 'media');
+ for (const file of ['statuses/s1/orphan.bin', 'statuses/locked/hidden.bin', 'chat-media/s1/other.bin']) {
+ fs.mkdirSync(path.dirname(path.join(localPath, file)), { recursive: true });
+ fs.writeFileSync(path.join(localPath, file), 'x');
+ }
+ storage = new StorageService({
+ get: (key: string) => ({ 'storage.type': 'local', 'storage.localPath': localPath })[key],
+ } as unknown as ConfigService);
+ // A mode-000 directory is still readable by root, so the failure is injected for that one path.
+ const readdir = fs.promises.readdir.bind(fs.promises) as (...args: unknown[]) => Promise<unknown>;
+ jest
+ .spyOn(fs.promises, 'readdir')
+ .mockImplementation(((target: fs.PathLike, ...rest: unknown[]) =>
+ String(target).endsWith(path.join('statuses', 'locked'))
+ ? Promise.reject(Object.assign(new Error('EACCES: permission denied, scandir'), { code: 'EACCES' }))
+ : readdir(target, ...rest)) as never);
+ jest.spyOn(console, 'warn').mockImplementation(() => undefined);
+ });
+
+ afterEach(() => {
+ jest.restoreAllMocks();
+ storage.onModuleDestroy();
+ fs.rmSync(baseDir, { recursive: true, force: true });
+ });
+
+ const sweep = (prefix: string) =>
+ sweepOrphanedFiles({
+ storage,
+ prefix,
+ graceMs: 0,
+ now: 1000,
+ firstSeenAt: new Map(),
+ referencedAmong: () => Promise.resolve(new Set<string | undefined>()),
+ onDeleteFailed: jest.fn(),
+ });
+
+ it('reclaims the orphans it can see and never touches the subtree it cannot', async () => {
+ expect(await sweep('statuses/')).toBe(1);
+
+ const present = (file: string): boolean => fs.existsSync(path.join(baseDir, 'media', file));
+ expect(present('statuses/s1/orphan.bin')).toBe(false);
+ expect(present('statuses/locked/hidden.bin')).toBe(true);
+ expect(present('chat-media/s1/other.bin')).toBe(true);
+ const warnings = (console.warn as jest.Mock).mock.calls.map(([line]) => String(line));
+ expect(warnings.some(line => line.includes('statuses/locked'))).toBe(true);
+ });
+
+ it('keeps refusing a file count of the same store', async () => {
+ await expect(storage.getFileCount()).rejects.toThrow(/EACCES/);
+ });
+});
diff --git a/src/common/storage/orphan-sweep.ts b/src/common/storage/orphan-sweep.ts
index aea00ce4b..89c04e005 100644
--- a/src/common/storage/orphan-sweep.ts
+++ b/src/common/storage/orphan-sweep.ts
@@ -72,7 +72,8 @@ export async function sweepOrphanedFiles(options: {
chunk = [];
};
- for await (const file of storage.iterateFiles(prefix)) {
+ // A directory the walk cannot read is left alone: this pass deletes only files it was shown.
+ for await (const file of storage.iterateFiles(prefix, { skipUnreadable: true })) {
if (!file.startsWith(prefix)) continue;
chunk.push(file);
if (options.chunkSize !== undefined && chunk.length >= options.chunkSize) await flush();
diff --git a/src/common/storage/storage-local-files.spec.ts b/src/common/storage/storage-local-files.spec.ts
new file mode 100644
index 000000000..931d42afb
--- /dev/null
+++ b/src/common/storage/storage-local-files.spec.ts
@@ -0,0 +1,109 @@
+import * as fs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+import { iterateLocalFiles, type LocalWalkOptions } from './storage-local-files';
+
+describe('iterateLocalFiles', () => {
+ let root: string;
+
+ beforeEach(() => {
+ root = fs.mkdtempSync(path.join(os.tmpdir(), 'owa-walk-'));
+ });
+
+ afterEach(() => {
+ jest.restoreAllMocks();
+ fs.rmSync(root, { recursive: true, force: true });
+ });
+
+ const write = (file: string): void => {
+ fs.mkdirSync(path.dirname(path.join(root, file)), { recursive: true });
+ fs.writeFileSync(path.join(root, file), 'x');
+ };
+
+ const walk = async (prefix = '', options?: LocalWalkOptions): Promise<string[]> => {
+ const files: string[] = [];
+ for await (const file of iterateLocalFiles(root, prefix, options)) files.push(file);
+ return files.sort();
+ };
+
+ /** A mode-000 directory is still readable by root, so the failure is injected for the one path. */
+ const failReaddirAt = (suffix: string, code = 'EACCES'): Error => {
+ const error = Object.assign(new Error(`${code}: cannot read ${suffix}`), { code });
+ const readdir = fs.promises.readdir.bind(fs.promises) as (...args: unknown[]) => Promise<unknown>;
+ jest
+ .spyOn(fs.promises, 'readdir')
+ .mockImplementation(((target: fs.PathLike, ...rest: unknown[]) =>
+ String(target).endsWith(suffix) ? Promise.reject(error) : readdir(target, ...rest)) as never);
+ return error;
+ };
+
+ const deepKey = [...Array<string>(20).fill('d'), 'deep.bin'].join('/');
+
+ it('lists every file with "/" separated keys', async () => {
+ write('a.bin');
+ write('x/y/b.bin');
+
+ expect(await walk()).toEqual(['a.bin', 'x/y/b.bin']);
+ expect(await walk('x/')).toEqual(['x/y/b.bin']);
+ });
+
+ describe('by default', () => {
+ it('fails on a directory it cannot read, with the error it got', async () => {
+ write('ok.bin');
+ write('locked/hidden.bin');
+ const error = failReaddirAt(path.join(root, 'locked'));
+
+ await expect(walk()).rejects.toBe(error);
+ });
+
+ it('fails on a directory below the maximum depth', async () => {
+ write(deepKey);
+
+ await expect(walk()).rejects.toThrow(/maximum traversal depth/);
+ });
+
+ it('treats a directory removed during the walk as empty', async () => {
+ write('ok.bin');
+ write('gone/file.bin');
+ failReaddirAt(path.join(root, 'gone'), 'ENOENT');
+
+ expect(await walk()).toEqual(['ok.bin']);
+ });
+ });
+
+ describe('when asked to skip what it cannot read', () => {
+ const lenient = (): { options: LocalWalkOptions; warn: jest.Mock } => {
+ const warn = jest.fn();
+ return { options: { skipUnreadable: true, logger: { warn } }, warn };
+ };
+
+ it('skips an unreadable directory, names it, and still lists the rest', async () => {
+ write('ok.bin');
+ write('locked/hidden.bin');
+ write('other/fine.bin');
+ failReaddirAt(path.join(root, 'locked'));
+ const { options, warn } = lenient();
+
+ expect(await walk('', options)).toEqual(['ok.bin', 'other/fine.bin']);
+ expect(warn).toHaveBeenCalledTimes(1);
+ expect(warn).toHaveBeenCalledWith(expect.stringContaining('"locked"'));
+ });
+
+ it('skips a directory below the maximum depth and still lists the rest', async () => {
+ write('ok.bin');
+ write(deepKey);
+ const { options, warn } = lenient();
+
+ expect(await walk('', options)).toEqual(['ok.bin']);
+ expect(warn).toHaveBeenCalledWith(expect.stringContaining('maximum traversal depth'));
+ });
+
+ it('skips without a logger too', async () => {
+ write('ok.bin');
+ write('locked/hidden.bin');
+ failReaddirAt(path.join(root, 'locked'));
+
+ expect(await walk('', { skipUnreadable: true })).toEqual(['ok.bin']);
+ });
+ });
+});
diff --git a/src/common/storage/storage-local-files.ts b/src/common/storage/storage-local-files.ts
index 18bea94ca..d180c83c8 100644
--- a/src/common/storage/storage-local-files.ts
+++ b/src/common/storage/storage-local-files.ts
@@ -1,17 +1,39 @@
import * as fs from 'fs';
import * as path from 'path';
import { Readable } from 'stream';
+import type { LoggerService } from '../services/logger.service';
import { isPathWithin } from '../utils/path-safety';
/** Max directory depth a local traversal descends. Prevents a pathological tree from running unbounded. */
const LOCAL_TRAVERSAL_MAX_DEPTH = 20;
+export interface LocalWalkOptions {
+ /**
+ * Skip a path the walk cannot enumerate, naming it in a warning, instead of failing the walk. Only for a
+ * caller that acts on the files it saw and nothing else, such as an orphan sweep: it leaves the subtree it
+ * could not read alone. An export or a count must not set it, because a short answer hides files.
+ */
+ skipUnreadable?: boolean;
+ logger?: Pick<LoggerService, 'warn'>;
+}
+
/**
* Local enumeration as a stream. Async + iterative (a work queue, not recursion) so a deep/wide
* media tree can't block the event loop or stack-overflow; still bounded by the max directory
- * depth. An incomplete traversal fails instead of hiding files.
+ * depth. An incomplete traversal fails instead of hiding files, unless the caller opts out.
*/
-export async function* iterateLocalFiles(localPath: string, prefix = ''): AsyncGenerator<string> {
+export async function* iterateLocalFiles(
+ localPath: string,
+ prefix = '',
+ options: LocalWalkOptions = {},
+): AsyncGenerator<string> {
+ /** A path the walk cannot enumerate: fail the walk, or skip it and say so when the caller allows it. */
+ const unreadable = (entry: string, error: Error): void => {
+ if (!options.skipUnreadable) throw error;
+ options.logger?.warn(
+ `Skipping a storage path that cannot be enumerated: ${JSON.stringify(entry)} (${error.message})`,
+ );
+ };
// Iterative BFS: a queue of [relativeDir, depth] avoids unbounded call-stack growth. A prefix
// ending in '/' names a subtree, so start the walk there instead of filtering afterwards.
const root = prefix.endsWith('/') ? prefix.slice(0, -1) : '';
@@ -21,7 +43,8 @@ export async function* iterateLocalFiles(localPath: string, prefix = ''): AsyncG
while (queue.length > 0) {
const { dir, depth } = queue.shift()!;
if (depth >= LOCAL_TRAVERSAL_MAX_DEPTH) {
- throw new Error(`Storage directory exceeds the maximum traversal depth: ${dir}`);
+ unreadable(dir, new Error(`Storage directory exceeds the maximum traversal depth: ${dir}`));
+ continue;
}
const fullPath = path.join(localPath, dir);
@@ -29,8 +52,9 @@ export async function* iterateLocalFiles(localPath: string, prefix = ''): AsyncG
try {
entries = await fs.promises.readdir(fullPath, { withFileTypes: true });
} catch (error) {
- if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error;
- continue; // A directory removed during enumeration has no files left to export.
+ // A directory removed during enumeration has no files left to export.
+ if ((error as NodeJS.ErrnoException).code !== 'ENOENT') unreadable(dir, error as Error);
+ continue;
}
for (const entry of entries) {
diff --git a/src/common/storage/storage.service.ts b/src/common/storage/storage.service.ts
index a4e0a286d..3158a384b 100644
--- a/src/common/storage/storage.service.ts
+++ b/src/common/storage/storage.service.ts
@@ -274,20 +274,24 @@ export class StorageService implements OnModuleDestroy {
* `prefix` narrows the walk at the source (the S3 ListObjectsV2 prefix and the local traversal
* root), so a caller reconciling one subtree neither pages the whole store nor holds every key
* of it in the dedupe Set.
+ *
+ * A local directory the walk cannot read fails it, so no caller is handed a short listing without
+ * knowing. `skipUnreadable` is for a caller that only acts on the files it was shown (an orphan
+ * sweep): the directory is named in a warning and left alone.
*/
- async *iterateFiles(prefix = ''): AsyncGenerator<string> {
+ async *iterateFiles(prefix = '', options: { skipUnreadable?: boolean } = {}): AsyncGenerator<string> {
if (this.storageType === 's3' && this.s3Client && this.s3Available) {
const seen = new Set<string>();
for await (const file of this.iterateS3Files(prefix)) {
seen.add(file);
yield file;
}
- for await (const file of this.iterateLocalFiles(prefix)) {
+ for await (const file of this.iterateLocalFiles(prefix, options.skipUnreadable)) {
if (!seen.has(file)) yield file;
}
return;
}
- yield* this.iterateLocalFiles(prefix);
+ yield* this.iterateLocalFiles(prefix, options.skipUnreadable);
}
async getFile(filePath: string): Promise<Buffer> {
@@ -469,8 +473,8 @@ export class StorageService implements OnModuleDestroy {
* media tree can't block the event loop or stack-overflow; still bounded by the max directory
* depth so a pathological tree can't descend unbounded.
*/
- private iterateLocalFiles(prefix = ''): AsyncGenerator<string> {
- return iterateLocalFiles(this.localPath, prefix);
+ private iterateLocalFiles(prefix = '', skipUnreadable = false): AsyncGenerator<string> {
+ return iterateLocalFiles(this.localPath, prefix, { skipUnreadable, logger: this.logger });
}
private getLocalFile(filePath: string): Promise<Buffer> {