Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(infra): refuse storage migration against the local fallback
Yudhi Armyndharisd0eb6b669c94
diff --git a/docs/06-api-specification.md b/docs/06-api-specification.mdindex 747dbf6bf..20af2b9a9 100644--- a/docs/06-api-specification.md+++ b/docs/06-api-specification.md@@ -5783,7 +5783,7 @@ File count and total size in the active storage backend.{ "storageType": "local", "count": 128, "sizeBytes": 5242880, "sizeMB": "5.00" }```-**Errors:** `401` · `403` · `500`+**Errors:** `401` · `403` · `500` · `503` `STORAGE_TYPE=s3` but the bucket is unreachable or its credentials are missing (this route, the export and the import refuse rather than report on the local fallback directory)---@@ -5801,7 +5801,7 @@ Export all storage files into a `tar.gz` under `data/exports` and return its **s`download` is a server filesystem path — feed it back to `POST /api/infra/storage/import`. The archive is auto-deleted after `STORAGE_EXPORT_TTL_MS` (default 1h).-**Errors:** `401` · `403` · `500`+**Errors:** `401` · `403` · `500` · `503` S3 configured but unavailable---@@ -5824,10 +5824,12 @@ Import storage files from a `tar.gz` located inside the `data/` directory.**Response** `200````json-{ "imported": true, "count": 128, "storageType": "local" }+{ "imported": true, "count": 128, "failed": 0, "storageType": "local" }```-**Errors:** `400` missing/out-of-`data/`/not-found path · `401` · `403` · `500`+`failed` counts archive entries the store refused to write; a bad or traversing entry is skipped without failing the rest. `imported` is `false` when entries failed and none was written.++**Errors:** `400` missing/out-of-`data/`/not-found path · `401` · `403` · `500` · `503` S3 configured but unavailable---diff --git a/docs/14-migration-guide.md b/docs/14-migration-guide.mdindex 779dd7543..4c668a5c9 100644--- a/docs/14-migration-guide.md+++ b/docs/14-migration-guide.md@@ -205,6 +205,10 @@ curl -X POST 'http://localhost:2785/api/infra/import-data' \OpenWA v0.2+ supports migrating media files between storage backends:+On an S3/MinIO backend, confirm `GET /api/infra/status` reports `storage.s3Available: true` before the+export (S3 to Local) and before the import (Local to S3). While the bucket is unusable these routes+answer `503` instead of running against the local fallback directory.+```bash# Step 1: Check current storage file countcurl -s 'http://localhost:2785/api/infra/storage/files/count' \diff --git a/openapi.json b/openapi.jsonindex edda37249..bbd93bed9 100644--- a/openapi.json+++ b/openapi.json@@ -4477,6 +4477,9 @@}}}+ },+ "503": {+ "description": "STORAGE_TYPE is s3 but the bucket is unreachable or its credentials are missing; nothing was read or written."}},"summary": "Get file count in current storage",@@ -4499,6 +4502,9 @@}}}+ },+ "503": {+ "description": "STORAGE_TYPE is s3 but the bucket is unreachable or its credentials are missing; nothing was read or written."}},"summary": "Export all storage files as tar.gz",@@ -4532,6 +4538,9 @@}}}+ },+ "503": {+ "description": "STORAGE_TYPE is s3 but the bucket is unreachable or its credentials are missing; nothing was read or written."}},"summary": "Import storage files from tar.gz",@@ -14727,6 +14736,11 @@"description": "Objects written.","example": 128},+ "failed": {+ "type": "number",+ "description": "Archive entries the store refused to write. `imported` is false when entries failed and none was written.",+ "example": 0+ },"storageType": {"type": "string","enum": [@@ -14739,6 +14753,7 @@"required": ["imported","count",+ "failed","storageType"]},diff --git a/src/common/storage/storage-transfer.spec.ts b/src/common/storage/storage-transfer.spec.tsindex ab60abd3a..d36819f12 100644--- a/src/common/storage/storage-transfer.spec.ts+++ b/src/common/storage/storage-transfer.spec.ts@@ -162,3 +162,19 @@ describe('createExportStream streams one file at a time', () => {expect(stats.opened).toBe(openedBefore);});});++describe('importFromStream reports refused writes', () => {+ it('counts every entry putFile rejects as failed, and none as imported', async () => {+ const { openFile } = trackingOpener();+ const names = files.slice(0, 3);+ const output = await createExportStream(() => Promise.resolve(names), openFile, makeLogger() as never);++ const result = await importFromStream(+ output,+ () => Promise.reject(new Error('AccessDenied')),+ makeLogger() as never,+ );++ expect(result).toEqual({ imported: 0, failed: names.length });+ });+});diff --git a/src/common/storage/storage-transfer.ts b/src/common/storage/storage-transfer.tsindex 479d14c52..488bb2147 100644--- a/src/common/storage/storage-transfer.ts+++ b/src/common/storage/storage-transfer.ts@@ -155,13 +155,15 @@ async function appendEntries(// Best-effort, NOT atomic: a single bad/traversing entry is skipped and the rest still import, and a// resource-cap breach aborts the rest but KEEPS the entries already written (no rollback). Callers// re-running an import is safe (putFile overwrites). A staging-dir + atomic promote would make it-// transactional, but is out of scope here.+// transactional, but is out of scope here. Resolves with how many entries were written and how many+// putFile refused, so a caller can tell an import that wrote nothing from an empty archive.export async function importFromStream(inputStream: Readable,putFile: (filePath: string, data: Buffer) => Promise<void>,logger: LoggerService,-): Promise<number> {+): Promise<{ imported: number; failed: number }> {let importedCount = 0;+ let failedCount = 0;let entryCount = 0;const maxEntryBytes = positiveIntFromEnv('STORAGE_IMPORT_MAX_BYTES', DEFAULT_IMPORT_MAX_BYTES);const maxEntries = positiveIntFromEnv('STORAGE_IMPORT_MAX_ENTRIES', DEFAULT_IMPORT_MAX_ENTRIES);@@ -169,7 +171,7 @@ export async function importFromStream(const extract = tar.extract();const gunzip = createGunzip();- return new Promise<number>((resolve, reject) => {+ return new Promise<{ imported: number; failed: number }>((resolve, reject) => {let settled = false;// Abort the whole import: a per-entry overflow or too many entries is a (zip-bomb) attack, not// a per-file skip — tear down the pipeline and reject so nothing further is buffered or written.@@ -232,6 +234,7 @@ export async function importFromStream(next();}).catch((error: unknown) => {+ failedCount++;logger.error(`Failed to import file: ${header.name}`, String(error));next();});@@ -242,8 +245,8 @@ export async function importFromStream(extract.on('finish', () => {if (settled) return;settled = true;- logger.log(`Import completed: ${importedCount} files`);- resolve(importedCount);+ logger.log(`Import completed: ${importedCount} files, ${failedCount} failed`);+ resolve({ imported: importedCount, failed: failedCount });});extract.on('error', (err: Error) => {diff --git a/src/common/storage/storage.service.spec.ts b/src/common/storage/storage.service.spec.tsindex 7b460110e..aed67c011 100644--- a/src/common/storage/storage.service.spec.ts+++ b/src/common/storage/storage.service.spec.ts@@ -97,11 +97,11 @@ describe('StorageService (local) path traversal protection', () => {{ name: '../evil.txt', data: 'bad' },]);- const count = await service.importFromStream(Readable.from(gz));+ const result = await service.importFromStream(Readable.from(gz));expect(fs.readFileSync(path.join(localPath, 'safe.txt'), 'utf8')).toBe('good');expect(fs.existsSync(path.join(baseDir, 'evil.txt'))).toBe(false);- expect(count).toBe(1);+ expect(result).toEqual({ imported: 1, failed: 1 });});});@@ -244,8 +244,8 @@ describe('StorageService import resource caps (decompression-bomb defense)', ()it('imports normally within the (generous default) caps', async () => {const gz = await makeTarGz([{ name: 'ok.txt', data: 'fine' }]);- const count = await service.importFromStream(Readable.from(gz));- expect(count).toBe(1);+ const result = await service.importFromStream(Readable.from(gz));+ expect(result).toEqual({ imported: 1, failed: 0 });});});diff --git a/src/common/storage/storage.service.ts b/src/common/storage/storage.service.tsindex 618638619..10139d491 100644--- a/src/common/storage/storage.service.ts+++ b/src/common/storage/storage.service.ts@@ -435,7 +435,7 @@ export class StorageService implements OnModuleDestroy {}// Best-effort, NOT atomic: see the implementation in storage-transfer.ts for the full contract.- importFromStream(inputStream: Readable): Promise<number> {+ importFromStream(inputStream: Readable): Promise<{ imported: number; failed: number }> {return importFromStream(inputStream, (filePath, data) => this.putFile(filePath, data), this.logger);}diff --git a/src/modules/infra/dto/infra-response.dto.ts b/src/modules/infra/dto/infra-response.dto.tsindex 2448a38cb..d2b1914f3 100644--- a/src/modules/infra/dto/infra-response.dto.ts+++ b/src/modules/infra/dto/infra-response.dto.ts@@ -550,6 +550,13 @@ export class StorageImportResponseDto {@ApiProperty({ description: 'Objects written.', example: 128 })count!: number;+ @ApiProperty({+ description:+ 'Archive entries the store refused to write. `imported` is false when entries failed and none was written.',+ example: 0,+ })+ failed!: number;+@ApiProperty({ enum: ['local', 's3'], example: 'local' })storageType!: string;}diff --git a/src/modules/infra/infra-storage.controller.spec.ts b/src/modules/infra/infra-storage.controller.spec.tsindex dbb521fec..524f99a67 100644--- a/src/modules/infra/infra-storage.controller.spec.ts+++ b/src/modules/infra/infra-storage.controller.spec.ts@@ -26,7 +26,7 @@ jest.mock('fs', () => {};});-import { BadRequestException } from '@nestjs/common';+import { BadRequestException, ServiceUnavailableException } from '@nestjs/common';import { InfraStorageController } from './infra-storage.controller';import { AuditAction } from '../audit/entities/audit-log.entity';@@ -50,14 +50,14 @@ describe('InfraStorageController.importStorage filePath validation', () => {(fs.existsSync as jest.Mock).mockImplementation((p: string) => p === '/srv/openwa/data/exports/export.tar.gz');(fs.createReadStream as jest.Mock).mockClear();const storage = {- importFromStream: jest.fn().mockResolvedValue(3),+ importFromStream: jest.fn().mockResolvedValue({ imported: 3, failed: 0 }),getCurrentStorageType: jest.fn(() => 'local'),};try {const result = await buildController(storage).importStorage({ filePath: 'data/exports/export.tar.gz' });expect(fs.createReadStream).toHaveBeenCalledWith('/srv/openwa/data/exports/export.tar.gz');expect(storage.importFromStream).toHaveBeenCalledTimes(1);- expect(result).toEqual({ imported: true, count: 3, storageType: 'local' });+ expect(result).toEqual({ imported: true, count: 3, failed: 0, storageType: 'local' });} finally {cwdSpy.mockRestore();(fs.existsSync as jest.Mock).mockReturnValue(false);@@ -67,7 +67,7 @@ describe('InfraStorageController.importStorage filePath validation', () => {describe('InfraStorageController.exportStorage keeps the export import-able and sweeps it', () => {function buildController(storage: Partial<{ createExportStream: jest.Mock }>) {- return new InfraStorageController(storage as never);+ return new InfraStorageController({ getCurrentStorageType: () => 'local', ...storage } as never);}// fs.existsSync is globally mocked in this file, so probe the real filesystem via fs.promises.access.@@ -243,7 +243,7 @@ describe('InfraStorageController storage stream failures surface as request errothis.push(Buffer.alloc(1024));},});- const storage = { createExportStream: jest.fn().mockResolvedValue(source) };+ const storage = { createExportStream: jest.fn().mockResolvedValue(source), getCurrentStorageType: () => 'local' };await expect(new InfraStorageController(storage as never).exportStorage()).rejects.toThrow(/disk full/);expect(source.destroyed).toBe(true);} finally {@@ -263,7 +263,10 @@ describe('InfraStorageController storage stream failures surface as request errothis.destroy(new Error('archive boom'));},});- const storage = { createExportStream: jest.fn().mockResolvedValue(errStream) };+ const storage = {+ createExportStream: jest.fn().mockResolvedValue(errStream),+ getCurrentStorageType: () => 'local',+ };const controller = new InfraStorageController(storage as never);await expect(controller.exportStorage()).rejects.toThrow(/archive boom/);} finally {@@ -291,7 +294,10 @@ describe('InfraStorageController audit trail (light-dependency handlers)', () =>const cwdSpy = jest.spyOn(process, 'cwd').mockReturnValue('/srv/openwa');(fs.existsSync as jest.Mock).mockImplementation((p: string) => p === '/srv/openwa/data/exports/x.tar.gz');try {- const storageService = { importFromStream: jest.fn().mockResolvedValue(5), getCurrentStorageType: () => 'local' };+ const storageService = {+ importFromStream: jest.fn().mockResolvedValue({ imported: 5, failed: 0 }),+ getCurrentStorageType: () => 'local',+ };await build(audit, { storageService }).importStorage({ filePath: 'data/exports/x.tar.gz' });const calls = audit.logInfo.mock.calls as Array<[AuditAction, { metadata: { count: number; storageType: string } }]@@ -310,7 +316,10 @@ describe('InfraStorageController audit trail (light-dependency handlers)', () =>const cwdSpy = jest.spyOn(process, 'cwd').mockReturnValue(cwd);process.env.STORAGE_EXPORT_TTL_MS = '30';try {- const storageService = { createExportStream: jest.fn().mockResolvedValue(Readable.from([Buffer.from('x')])) };+ const storageService = {+ createExportStream: jest.fn().mockResolvedValue(Readable.from([Buffer.from('x')])),+ getCurrentStorageType: () => 'local',+ };await build(audit, { storageService }).exportStorage();const calls = audit.logInfo.mock.calls as Array<[AuditAction, { metadata: { download: string } }]>;expect(calls[0][0]).toBe(AuditAction.INFRA_STORAGE_EXPORTED);@@ -322,3 +331,81 @@ describe('InfraStorageController audit trail (light-dependency handlers)', () =>}});});++describe('InfraStorageController refuses to run a storage migration against the local fallback', () => {+ function unavailableS3() {+ return {+ getCurrentStorageType: jest.fn(() => 's3'),+ refreshS3Availability: jest.fn().mockResolvedValue(false),+ getFileCount: jest.fn(),+ createExportStream: jest.fn(),+ importFromStream: jest.fn(),+ };+ }++ it('answers 503 for the file count, the export and the import when the configured bucket is unusable', async () => {+ const storage = unavailableS3();+ const controller = new InfraStorageController(storage as never);++ await expect(controller.getStorageFileCount()).rejects.toBeInstanceOf(ServiceUnavailableException);+ await expect(controller.exportStorage()).rejects.toBeInstanceOf(ServiceUnavailableException);+ await expect(controller.importStorage({ filePath: 'data/exports/x.tar.gz' })).rejects.toBeInstanceOf(+ ServiceUnavailableException,+ );+ expect(storage.refreshS3Availability).toHaveBeenCalledTimes(3);+ expect(storage.getFileCount).not.toHaveBeenCalled();+ expect(storage.createExportStream).not.toHaveBeenCalled();+ expect(storage.importFromStream).not.toHaveBeenCalled();+ });++ it('proceeds once the re-probe finds the bucket', async () => {+ const storage = unavailableS3();+ storage.refreshS3Availability.mockResolvedValue(true);+ storage.getFileCount.mockResolvedValue({ count: 2, sizeBytes: 0 });++ await expect(new InfraStorageController(storage as never).getStorageFileCount()).resolves.toMatchObject({+ storageType: 's3',+ count: 2,+ });+ });++ it('never probes S3 for a local backend', async () => {+ const storage = { ...unavailableS3(), getCurrentStorageType: jest.fn(() => 'local') };+ storage.getFileCount.mockResolvedValue({ count: 0, sizeBytes: 0 });++ await new InfraStorageController(storage as never).getStorageFileCount();++ expect(storage.refreshS3Availability).not.toHaveBeenCalled();+ });+});++describe('InfraStorageController.importStorage reports an import that wrote nothing', () => {+ async function importWith(result: { imported: number; failed: number }) {+ const cwdSpy = jest.spyOn(process, 'cwd').mockReturnValue('/srv/openwa');+ (fs.existsSync as jest.Mock).mockImplementation((p: string) => p === '/srv/openwa/data/exports/x.tar.gz');+ try {+ const storage = { importFromStream: jest.fn().mockResolvedValue(result), getCurrentStorageType: () => 'local' };+ return await new InfraStorageController(storage as never).importStorage({ filePath: 'data/exports/x.tar.gz' });+ } finally {+ cwdSpy.mockRestore();+ (fs.existsSync as jest.Mock).mockReturnValue(false);+ }+ }++ it('answers imported:false when every entry was refused', async () => {+ await expect(importWith({ imported: 0, failed: 4 })).resolves.toEqual({+ imported: false,+ count: 0,+ failed: 4,+ storageType: 'local',+ });+ });++ it('keeps imported:true when only some entries were skipped', async () => {+ await expect(importWith({ imported: 3, failed: 1 })).resolves.toMatchObject({ imported: true, failed: 1 });+ });++ it('keeps imported:true for an empty archive', async () => {+ await expect(importWith({ imported: 0, failed: 0 })).resolves.toMatchObject({ imported: true, count: 0 });+ });+});diff --git a/src/modules/infra/infra-storage.controller.ts b/src/modules/infra/infra-storage.controller.tsindex 3cbc3476b..c350ce62b 100644--- a/src/modules/infra/infra-storage.controller.ts+++ b/src/modules/infra/infra-storage.controller.ts@@ -8,6 +8,7 @@ import {HttpStatus,OnApplicationBootstrap,Optional,+ ServiceUnavailableException,} from '@nestjs/common';import { ApiTags, ApiOperation, ApiResponse, ApiBody } from '@nestjs/swagger';import {@@ -27,6 +28,9 @@ import * as fs from 'fs';import * as path from 'path';import { randomUUID } from 'crypto';+const S3_UNAVAILABLE_DESCRIPTION =+ 'STORAGE_TYPE is s3 but the bucket is unreachable or its credentials are missing; nothing was read or written.';+@ApiTags('infrastructure')@Controller('infra')// Every route here is deployment-global (data export/import, infra config, service orchestration),@@ -107,16 +111,33 @@ export class InfraStorageController implements OnApplicationBootstrap {// STORAGE MIGRATION API// ============================================================================+ /**+ * With STORAGE_TYPE=s3 and the bucket unusable (credentials missing, or unreachable since boot),+ * every StorageService operation quietly uses the local fallback directory. A count, export or+ * import run in that state reports on the wrong store: an import lands on local disk while answering+ * storageType 's3', and an export archives only the fallback files. Refuse instead, so the operator+ * retries once the bucket is back rather than trusting a migration that did not reach it.+ */+ private async assertStorageBackendReady(): Promise<void> {+ if (this.storageService.getCurrentStorageType() !== 's3') return;+ if (await this.storageService.refreshS3Availability()) return;+ throw new ServiceUnavailableException(+ 'S3 storage is not available (bucket unreachable or credentials missing); retry once GET /api/infra/status reports s3Available: true',+ );+ }+@Get('storage/files/count')@RequireRole(ApiKeyRole.ADMIN)@ApiOperation({ summary: 'Get file count in current storage' })@ApiResponse({ status: 200, description: 'File count and size', type: StorageFileCountResponseDto })+ @ApiResponse({ status: 503, description: S3_UNAVAILABLE_DESCRIPTION })async getStorageFileCount(): Promise<{storageType: string;count: number;sizeBytes: number;sizeMB: string;}> {+ await this.assertStorageBackendReady();const { count, sizeBytes } = await this.storageService.getFileCount();return {storageType: this.storageService.getCurrentStorageType(),@@ -137,7 +158,9 @@ export class InfraStorageController implements OnApplicationBootstrap {'POST /api/infra/storage/import.',type: StorageExportResponseDto,})+ @ApiResponse({ status: 503, description: S3_UNAVAILABLE_DESCRIPTION })async exportStorage(): Promise<{ message: string; download: string }> {+ await this.assertStorageBackendReady();// Note: In production, this would return a StreamableFile// For simplicity, we'll save to a temp file and return the pathconst stream = await this.storageService.createExportStream();@@ -202,9 +225,11 @@ export class InfraStorageController implements OnApplicationBootstrap {// string while the handler takes an object.@ApiBody({ type: ImportStorageDto, description: 'Path to tar.gz file to import' })@ApiResponse({ status: 200, description: 'Import result', type: StorageImportResponseDto })+ @ApiResponse({ status: 503, description: S3_UNAVAILABLE_DESCRIPTION })async importStorage(@Body() body: ImportStorageDto,- ): Promise<{ imported: boolean; count: number; storageType: string }> {+ ): Promise<{ imported: boolean; count: number; failed: number; storageType: string }> {+ await this.assertStorageBackendReady();const { filePath } = body;// `filePath` is fully caller-controlled. Restrict it to the app's data@@ -228,8 +253,9 @@ export class InfraStorageController implements OnApplicationBootstrap {// not a tar, unreadable, over the resource caps), so surface them as a 400 with the real reason// rather than an opaque 500.let count: number;+ let failed: number;try {- count = await this.storageService.importFromStream(readStream);+ ({ imported: count, failed } = await this.storageService.importFromStream(readStream));} catch (error) {throw new BadRequestException(`Storage import failed: ${error instanceof Error ? error.message : String(error)}`);}@@ -240,9 +266,12 @@ export class InfraStorageController implements OnApplicationBootstrap {metadata: { count, storageType },});+ // A bad or traversing entry is skipped by design, so one refusal does not fail the import. An+ // archive whose every entry was refused (the store rejects writes) wrote nothing and says so.return {- imported: true,+ imported: !(failed > 0 && count === 0),count,+ failed,storageType,};}diff --git a/test/storage-export-streaming.e2e-spec.ts b/test/storage-export-streaming.e2e-spec.tsindex eeb61f0ef..e942090f7 100644--- a/test/storage-export-streaming.e2e-spec.ts+++ b/test/storage-export-streaming.e2e-spec.ts@@ -69,7 +69,7 @@ describe('Storage export streams one file at a time (e2e, real archiver)', () =>const output = await createExportStream(() => Promise.resolve(files), openFile, logger as never);const imported = new Map<string, Buffer>();- const count = await importFromStream(+ const { imported: count } = await importFromStream(output,(name, data) => {imported.set(name, data);