Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(infra): answer 409 when the storage walk refuses a store
Yudhi Armyndharis995d0fa9a8f0
diff --git a/docs/06-api-specification.md b/docs/06-api-specification.mdindex 81c75ce58..6053c8ba1 100644--- a/docs/06-api-specification.md+++ b/docs/06-api-specification.md@@ -5982,7 +5982,7 @@ File count and total size in the active storage backend.{ "storageType": "local", "count": 128, "sizeBytes": 5242880, "sizeMB": "5.00" }```-**Errors:** `401` · `403` · `500` · `503` `STORAGE_TYPE=s3` but the bucket has not been reachable since boot, or its credentials are missing (this route, the export and the import refuse rather than report on the local fallback directory; an outage after the bucket was reachable answers `500`)+**Errors:** `401` · `403` · `409` the local storage directory cannot be enumerated completely, so the count would leave files out: it holds a symbolic link, a directory the app cannot read (permission denied), or one nested 20 levels deep or more; `message` names the path relative to the storage root and the cause, and the export refuses the same store the same way. Replace a link with a bind mount or fix the permissions, then retry · `500` · `503` `STORAGE_TYPE=s3` but the bucket has not been reachable since boot, or its credentials are missing (this route, the export and the import refuse rather than report on the local fallback directory; an outage after the bucket was reachable answers `500`)---@@ -6000,7 +6000,7 @@ Export all storage files into a `tar.gz` under `data/exports` and return its **s`download` is a server filesystem path — feed it back to `POST /api/infra/storage/import`. The archive is auto-deleted after `STORAGE_EXPORT_TTL_MS` (default 1h).-**Errors:** `401` · `403` · `500` · `503` S3 configured but not reachable since boot (a later outage answers `500`)+**Errors:** `401` · `403` · `409` the local storage directory cannot be enumerated completely, so the archive would leave files out (a symbolic link, a directory the app cannot read, or one nested too deep; `message` names the path relative to the storage root and the cause; nothing is written) · `500` · `503` S3 configured but not reachable since boot (a later outage answers `500`)---diff --git a/openapi.json b/openapi.jsonindex 09cd4af33..0084b0fb2 100644--- a/openapi.json+++ b/openapi.json@@ -8548,6 +8548,16 @@}}},+ "409": {+ "description": "The local storage directory cannot be enumerated completely, so the answer would silently leave files out: it holds a symbolic link, a directory the app cannot read, or one nested too deep. `message` names the path relative to the storage root and the cause. Replace a link with a bind mount or fix the permissions, then retry; nothing was written.",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/ErrorResponse"+ }+ }+ }+ },"503": {"description": "STORAGE_TYPE is s3 but the bucket is unreachable or its credentials are missing; nothing was read or written.","content": {@@ -8600,6 +8610,16 @@}}},+ "409": {+ "description": "The local storage directory cannot be enumerated completely, so the answer would silently leave files out: it holds a symbolic link, a directory the app cannot read, or one nested too deep. `message` names the path relative to the storage root and the cause. Replace a link with a bind mount or fix the permissions, then retry; nothing was written.",+ "content": {+ "application/json": {+ "schema": {+ "$ref": "#/components/schemas/ErrorResponse"+ }+ }+ }+ },"503": {"description": "STORAGE_TYPE is s3 but the bucket is unreachable or its credentials are missing; nothing was read or written.","content": {diff --git a/src/common/storage/storage-local-files.spec.ts b/src/common/storage/storage-local-files.spec.tsindex 40711421b..da16acc00 100644--- a/src/common/storage/storage-local-files.spec.ts+++ b/src/common/storage/storage-local-files.spec.ts@@ -1,7 +1,7 @@import * as fs from 'fs';import * as os from 'os';import * as path from 'path';-import { iterateLocalFiles, type LocalWalkOptions } from './storage-local-files';+import { iterateLocalFiles, StorageWalkRefusedError, type LocalWalkOptions } from './storage-local-files';describe('iterateLocalFiles', () => {let root: string;@@ -39,6 +39,16 @@ describe('iterateLocalFiles', () => {const deepKey = [...Array<string>(20).fill('d'), 'deep.bin'].join('/');+ /** Walk and return what it threw. */+ const refusal = async (prefix = '', options?: LocalWalkOptions): Promise<StorageWalkRefusedError> => {+ const error = await walk(prefix, options).then(+ () => undefined,+ (e: unknown) => e,+ );+ expect(error).toBeInstanceOf(StorageWalkRefusedError);+ return error as StorageWalkRefusedError;+ };+it('lists every file with "/" separated keys', async () => {write('a.bin');write('x/y/b.bin');@@ -48,10 +58,30 @@ describe('iterateLocalFiles', () => {});describe('by default', () => {- it('fails on a directory it cannot read, with the error it got', async () => {+ // The message reaches the API caller, so it names the path relative to the storage root and the cause, never the+ // host path. The error the host raised, with its path, stays on `cause` for the log.+ it.each(['EACCES', 'EPERM'])('refuses a directory it cannot read (%s), naming it and the cause', async code => {write('ok.bin');write('locked/hidden.bin');- const error = failReaddirAt(path.join(root, 'locked'));+ const error = failReaddirAt(path.join(root, 'locked'), code);++ const refused = await refusal();++ expect(refused.message).toBe(`Storage directory cannot be read: locked (${code})`);+ expect(refused.message).not.toContain(root);+ expect(refused.name).toBe('StorageWalkRefusedError');+ expect(refused.cause).toBe(error);+ });++ it('names the storage root as "." when the root itself cannot be read', async () => {+ failReaddirAt(root, 'EACCES');++ expect((await refusal()).message).toBe('Storage directory cannot be read: . (EACCES)');+ });++ it('lets any other failure through as it came, so it stays an internal error', async () => {+ write('broken/file.bin');+ const error = failReaddirAt(path.join(root, 'broken'), 'EIO');await expect(walk()).rejects.toBe(error);});@@ -79,9 +109,9 @@ describe('iterateLocalFiles', () => {it('still fails on one that is not at the storage root', async () => {write(`nested/${lostAndFound}/recovered.bin`);- const error = failReaddirAt(path.join(root, 'nested', lostAndFound));+ failReaddirAt(path.join(root, 'nested', lostAndFound));- await expect(walk()).rejects.toBe(error);+ expect((await refusal()).message).toBe(`Storage directory cannot be read: nested/${lostAndFound} (EACCES)`);});it('still fails on an error that is not a permission error', async () => {@@ -92,10 +122,13 @@ describe('iterateLocalFiles', () => {});});- it('fails on a directory below the maximum depth', async () => {+ it('refuses a directory below the maximum depth, naming it', async () => {write(deepKey);- await expect(walk()).rejects.toThrow(/maximum traversal depth/);+ const refused = await refusal();++ expect(refused.message).toMatch(/^Storage directory exceeds the maximum traversal depth: d(\/d){19}$/);+ expect(refused.message).not.toContain(root);});// A link is neither followed (it can loop, or leave the store) nor listed, so what it points at would be@@ -103,12 +136,15 @@ describe('iterateLocalFiles', () => {it.each([['directory', 'linked'],['file', 'alias.bin'],- ])('fails on a symbolic link to a %s, naming it', async (kind, name) => {+ ])('refuses a symbolic link to a %s, naming it', async (kind, name) => {write('real/inside.bin');write('real.bin');fs.symlinkSync(path.join(root, kind === 'directory' ? 'real' : 'real.bin'), path.join(root, name));- await expect(walk()).rejects.toThrow(new RegExp(`symbolic link.*${name.replace('.', '\\.')}`));+ const refused = await refusal();++ expect(refused.message).toBe(`Storage path is a symbolic link and is not followed: ${name}`);+ expect(refused.message).not.toContain(root);});it('walks a prefix that is itself a symbolic link, since the walk starts from a path', async () => {@@ -143,6 +179,8 @@ describe('iterateLocalFiles', () => {expect(await walk('', options)).toEqual(['ok.bin', 'other/fine.bin']);expect(warn).toHaveBeenCalledTimes(1);expect(warn).toHaveBeenCalledWith(expect.stringContaining('"locked"'));+ // The log keeps the host's own account of the failure, with its path; only the API message drops it.+ expect(warn).toHaveBeenCalledWith(expect.stringContaining(path.join(root, 'locked')));});it('skips a directory below the maximum depth and still lists the rest', async () => {diff --git a/src/common/storage/storage-local-files.ts b/src/common/storage/storage-local-files.tsindex 6ac06a12d..4d7c21614 100644--- a/src/common/storage/storage-local-files.ts+++ b/src/common/storage/storage-local-files.ts@@ -7,6 +7,19 @@ import { isPathWithin } from '../utils/path-safety';/** Max directory depth a local traversal descends. Prevents a pathological tree from running unbounded. */const LOCAL_TRAVERSAL_MAX_DEPTH = 20;+/**+ * The walk refuses to go on because it cannot enumerate part of the store completely: a directory the app cannot+ * read, a symbolic link it does not follow, a directory nested past the depth limit. A count or an export that went+ * on would leave files out without a word. The message reaches the API caller, so it names the path relative to+ * the storage root and the cause, never the host path; the error the host raised, with its path, is on `cause`.+ */+export class StorageWalkRefusedError extends Error {+ constructor(message: string, options?: { cause?: unknown }) {+ super(message, options);+ this.name = 'StorageWalkRefusedError';+ }+}+export interface LocalWalkOptions {/*** Skip a path the walk cannot enumerate, naming it in a warning, instead of failing the walk. Only for a@@ -33,8 +46,10 @@ export async function* iterateLocalFiles(/** A path the walk cannot enumerate: fail the walk, or skip it and say so when the caller allows it. */const unreadable = (entry: string, error: Error): void => {if (!options.skipUnreadable) throw error;+ // The log gets the host's own account of the failure, with its path.+ const detail = error instanceof StorageWalkRefusedError && error.cause instanceof Error ? error.cause : error;options.logger?.warn(- `Skipping a storage path that cannot be enumerated: ${JSON.stringify(entry)} (${error.message})`,+ `Skipping a storage path that cannot be enumerated: ${JSON.stringify(entry)} (${detail.message})`,);};// Iterative BFS: a queue of [relativeDir, depth] avoids unbounded call-stack growth. A prefix@@ -46,7 +61,7 @@ export async function* iterateLocalFiles(while (queue.length > 0) {const { dir, depth } = queue.shift()!;if (depth >= LOCAL_TRAVERSAL_MAX_DEPTH) {- unreadable(dir, new Error(`Storage directory exceeds the maximum traversal depth: ${dir}`));+ unreadable(dir, new StorageWalkRefusedError(`Storage directory exceeds the maximum traversal depth: ${dir}`));continue;}@@ -58,11 +73,19 @@ export async function* iterateLocalFiles(const code = (error as NodeJS.ErrnoException).code;// A directory removed during enumeration has no files left to export.if (code === 'ENOENT') continue;- if (dir === LOST_AND_FOUND && (code === 'EACCES' || code === 'EPERM')) {+ const permission = code === 'EACCES' || code === 'EPERM';+ if (dir === LOST_AND_FOUND && permission) {options.logger?.log(`Skipping the unreadable ${LOST_AND_FOUND} directory at the storage root`);continue;}- unreadable(dir, error as Error);+ // Only a permission failure is one the operator can fix, so only it is refused with a cause the caller can+ // read. Any other failure (an I/O error, say) goes on as it came, and is an internal error.+ unreadable(+ dir,+ permission+ ? new StorageWalkRefusedError(`Storage directory cannot be read: ${dir || '.'} (${code})`, { cause: error })+ : (error as Error),+ );continue;}@@ -77,7 +100,10 @@ export async function* iterateLocalFiles(} else if (entry.isSymbolicLink()) {// Not followed: a link can loop or leave the store, and what it points at would be missing from the// listing. getFile and a walk that starts below the link still reach it.- unreadable(relativePath, new Error(`Storage path is a symbolic link and is not followed: ${relativePath}`));+ unreadable(+ relativePath,+ new StorageWalkRefusedError(`Storage path is a symbolic link and is not followed: ${relativePath}`),+ );}}}diff --git a/src/modules/infra/infra-storage.controller.refusal.spec.ts b/src/modules/infra/infra-storage.controller.refusal.spec.tsnew file mode 100644index 000000000..e040179b3--- /dev/null+++ b/src/modules/infra/infra-storage.controller.refusal.spec.ts@@ -0,0 +1,124 @@+import * as fs from 'fs';+import type { Server } from 'http';+import * as os from 'os';+import * as path from 'path';+import type { ConfigService } from '@nestjs/config';+import type { INestApplication } from '@nestjs/common';+import { Test } from '@nestjs/testing';+import request from 'supertest';+import { StorageService } from '../../common/storage/storage.service';+import { InfraStorageController } from './infra-storage.controller';++/**+ * The count and the export refuse a store the walk cannot enumerate completely, because an answer that leaves+ * files out would lose them on the next backend. That refusal has to reach the ADMIN who runs the migration with+ * its cause, not as a bare 500, and only that refusal: any other failure stays an internal error.+ */+describe('GET /infra/storage/files/count and GET /infra/storage/export over a store the walk refuses', () => {+ let tmp: string;+ let media: string;+ let app: INestApplication;+ let storage: StorageService;++ beforeEach(async () => {+ tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'owa-refusal-'));+ media = path.join(tmp, 'media');+ fs.mkdirSync(media, { recursive: true });+ fs.writeFileSync(path.join(media, 'real.bin'), 'real');+ // The export writes under <cwd>/data/exports.+ jest.spyOn(process, 'cwd').mockReturnValue(tmp);+ storage = new StorageService({+ get: (key: string) => ({ 'storage.type': 'local', 'storage.localPath': media })[key],+ } as unknown as ConfigService);+ const moduleRef = await Test.createTestingModule({+ controllers: [InfraStorageController],+ providers: [{ provide: StorageService, useValue: storage }],+ }).compile();+ app = moduleRef.createNestApplication({ logger: false });+ await app.init();+ });++ afterEach(async () => {+ await app.close();+ storage.onModuleDestroy();+ jest.restoreAllMocks();+ fs.rmSync(tmp, { recursive: true, force: true });+ });++ /** A mode-000 directory is still readable by root, so the failure is injected for the one path. */+ const failReaddirAt = (suffix: string, code: string): void => {+ const error = Object.assign(new Error(`${code}: cannot read ${path.join(media, suffix)}`), { code });+ const readdir = fs.promises.readdir.bind(fs.promises) as (...args: unknown[]) => Promise<unknown>;+ jest+ .spyOn(fs.promises, 'readdir')+ .mockImplementation(((target: fs.PathLike, ...rest: unknown[]) =>+ String(target).endsWith(path.join(media, suffix)) ? Promise.reject(error) : readdir(target, ...rest)) as never);+ };++ const linkMediaAway = (): void => {+ fs.mkdirSync(path.join(tmp, 'bigdisk'));+ fs.writeFileSync(path.join(tmp, 'bigdisk', 'archived.bin'), 'archived');+ fs.symlinkSync(path.join(tmp, 'bigdisk'), path.join(media, 'chat-media'));+ };++ const exportsLeft = (): string[] => {+ const dir = path.join(tmp, 'data', 'exports');+ return fs.existsSync(dir) ? fs.readdirSync(dir) : [];+ };++ it.each(['/infra/storage/files/count', '/infra/storage/export'])(+ 'answers 409 naming the symbolic link, and not the host path, for %s',+ async route => {+ linkMediaAway();++ const response = await request(app.getHttpServer() as Server).get(route);++ expect(response.status).toBe(409);+ expect(response.body).toEqual({+ statusCode: 409,+ message: 'Storage path is a symbolic link and is not followed: chat-media',+ error: 'Conflict',+ });+ expect(JSON.stringify(response.body)).not.toContain(tmp);+ expect(exportsLeft()).toEqual([]);+ },+ );++ it.each(['/infra/storage/files/count', '/infra/storage/export'])(+ 'answers 409 naming a directory the app cannot read, and the cause, for %s',+ async route => {+ fs.mkdirSync(path.join(media, 'locked'));+ failReaddirAt('locked', 'EACCES');++ const response = await request(app.getHttpServer() as Server).get(route);++ expect(response.status).toBe(409);+ expect(response.body).toMatchObject({+ statusCode: 409,+ message: 'Storage directory cannot be read: locked (EACCES)',+ });+ expect(JSON.stringify(response.body)).not.toContain(tmp);+ },+ );++ it.each(['/infra/storage/files/count', '/infra/storage/export'])(+ 'still answers 500 for a failure that is not a refusal of the walk, for %s',+ async route => {+ fs.mkdirSync(path.join(media, 'broken'));+ failReaddirAt('broken', 'EIO');++ const response = await request(app.getHttpServer() as Server).get(route);++ expect(response.status).toBe(500);+ expect(response.body).toEqual({ statusCode: 500, message: 'Internal server error' });+ expect(exportsLeft()).toEqual([]);+ },+ );++ it('counts a store with nothing to refuse', async () => {+ const response = await request(app.getHttpServer() as Server).get('/infra/storage/files/count');++ expect(response.status).toBe(200);+ expect(response.body).toMatchObject({ storageType: 'local', count: 1, sizeBytes: 4 });+ });+});diff --git a/src/modules/infra/infra-storage.controller.ts b/src/modules/infra/infra-storage.controller.tsindex 6d44b269d..0a17b50ae 100644--- a/src/modules/infra/infra-storage.controller.ts+++ b/src/modules/infra/infra-storage.controller.ts@@ -4,6 +4,7 @@ import {Post,Body,BadRequestException,+ ConflictException,HttpCode,HttpStatus,OnApplicationBootstrap,@@ -20,6 +21,7 @@ import { RequireRole, RequireUnscopedKey } from '../auth/decorators/auth.decoratimport { ApiKeyRole } from '../auth/entities/api-key.entity';import { isPathWithin } from '../../common/utils/path-safety';import { StorageService } from '../../common/storage/storage.service';+import { StorageWalkRefusedError } from '../../common/storage/storage-local-files';import { createLogger } from '../../common/services/logger.service';import { AuditService } from '../audit/audit.service';import { AuditAction } from '../audit/entities/audit-log.entity';@@ -31,6 +33,12 @@ import { randomUUID } from 'crypto';const S3_UNAVAILABLE_DESCRIPTION ='STORAGE_TYPE is s3 but the bucket is unreachable or its credentials are missing; nothing was read or written.';+const WALK_REFUSED_DESCRIPTION =+ 'The local storage directory cannot be enumerated completely, so the answer would silently leave files out: ' ++ 'it holds a symbolic link, a directory the app cannot read, or one nested too deep. `message` names the path ' ++ 'relative to the storage root and the cause. Replace a link with a bind mount or fix the permissions, then ' ++ 'retry; nothing was written.';+@ApiTags('infrastructure')@Controller('infra')// Every route here is deployment-global (data export/import, infra config, service orchestration),@@ -111,6 +119,15 @@ export class InfraStorageController implements OnApplicationBootstrap {// STORAGE MIGRATION API// ============================================================================+ /**+ * A store the walk refuses to enumerate completely is a state the operator can fix, so the caller is told what and+ * where instead of a bare 500. Any other failure stays an internal error.+ */+ private static refusalAsConflict(error: unknown): never {+ if (error instanceof StorageWalkRefusedError) throw new ConflictException(error.message);+ throw error;+ }+/*** With STORAGE_TYPE=s3 and the bucket unusable (credentials missing, or unreachable since boot),* every StorageService operation quietly uses the local fallback directory. A count, export or@@ -130,6 +147,7 @@ export class InfraStorageController implements OnApplicationBootstrap {@RequireRole(ApiKeyRole.ADMIN)@ApiOperation({ summary: 'Get file count in current storage' })@ApiResponse({ status: 200, description: 'File count and size', type: StorageFileCountResponseDto })+ @ApiResponse({ status: 409, description: WALK_REFUSED_DESCRIPTION })@ApiResponse({ status: 503, description: S3_UNAVAILABLE_DESCRIPTION })async getStorageFileCount(): Promise<{storageType: string;@@ -138,7 +156,9 @@ export class InfraStorageController implements OnApplicationBootstrap {sizeMB: string;}> {await this.assertStorageBackendReady();- const { count, sizeBytes } = await this.storageService.getFileCount();+ const { count, sizeBytes } = await this.storageService+ .getFileCount()+ .catch((error: unknown) => InfraStorageController.refusalAsConflict(error));return {storageType: this.storageService.getCurrentStorageType(),count,@@ -158,12 +178,15 @@ export class InfraStorageController implements OnApplicationBootstrap {'POST /api/infra/storage/import.',type: StorageExportResponseDto,})+ @ApiResponse({ status: 409, description: WALK_REFUSED_DESCRIPTION })@ApiResponse({ status: 503, description: S3_UNAVAILABLE_DESCRIPTION })async exportStorage(): Promise<{ message: string; download: string }> {await this.assertStorageBackendReady();// Note: In production, this would return a StreamableFile// For simplicity, we'll save to a temp file and return the path- const stream = await this.storageService.createExportStream();+ const stream = await this.storageService+ .createExportStream()+ .catch((error: unknown) => InfraStorageController.refusalAsConflict(error));// Keep the export INSIDE data/ (under data/exports/): the import handler only accepts paths under// data/, and the documented backend-migration flow re-imports this file AFTER a container restart,// so it must live on the persistent volume — the OS temp dir is wiped on restart. The originaldiff --git a/test/storage-export-streaming.e2e-spec.ts b/test/storage-export-streaming.e2e-spec.tsindex 4948560b4..cad9a32f5 100644--- a/test/storage-export-streaming.e2e-spec.ts+++ b/test/storage-export-streaming.e2e-spec.ts@@ -4,6 +4,7 @@ import * as os from 'os';import * as path from 'path';import { Readable } from 'stream';import { ConfigService } from '@nestjs/config';+import { StorageWalkRefusedError } from '../src/common/storage/storage-local-files';import { createExportStream, ExportFileSource, importFromStream } from '../src/common/storage/storage-transfer';import { StorageService } from '../src/common/storage/storage.service';@@ -24,14 +25,31 @@ describe('Local storage export completeness', () => {fs.rmSync(localPath, { recursive: true, force: true });});- it.each(['EACCES', 'EIO'])('refuses an export and count when directory enumeration fails with %s', async code => {+ // A permission failure is one the operator can fix, so it is refused with a typed error that names the cause and+ // keeps the host's own error for the log.+ it.each(['EACCES', 'EPERM'])('refuses an export and count when a directory is unreadable (%s)', async code => {await storage.putFile('nested/media.bin', Buffer.from('preserve this file'));const error = Object.assign(new Error(`Cannot enumerate storage: ${code}`), { code });jest.spyOn(fs.promises, 'readdir').mockRejectedValue(error);+ const refusals = [+ await storage.createExportStream().catch((e: unknown) => e),+ await storage.getFileCount().catch((e: unknown) => e),+ ];+ for (const refusal of refusals) {+ expect(refusal).toBeInstanceOf(StorageWalkRefusedError);+ expect(refusal).toMatchObject({ message: `Storage directory cannot be read: . (${code})`, cause: error });+ }+ expect((await storage.getFile('nested/media.bin')).toString()).toBe('preserve this file');+ });++ it('refuses an export and count when enumeration fails with an error that is not a refusal, as it came', async () => {+ await storage.putFile('nested/media.bin', Buffer.from('preserve this file'));+ const error = Object.assign(new Error('Cannot enumerate storage: EIO'), { code: 'EIO' });+ jest.spyOn(fs.promises, 'readdir').mockRejectedValue(error);+await expect(storage.createExportStream()).rejects.toBe(error);await expect(storage.getFileCount()).rejects.toBe(error);- expect((await storage.getFile('nested/media.bin')).toString()).toBe('preserve this file');});it('refuses an export and count when an accepted key exceeds the traversal depth', async () => {@@ -39,6 +57,7 @@ describe('Local storage export completeness', () => {await storage.putFile(key, Buffer.from('deep media'));expect((await storage.getFile(key)).toString()).toBe('deep media');+ await expect(storage.createExportStream()).rejects.toBeInstanceOf(StorageWalkRefusedError);await expect(storage.createExportStream()).rejects.toThrow(/maximum traversal depth/);await expect(storage.getFileCount()).rejects.toThrow(/maximum traversal depth/);});