Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
feat(storage): add S3_KEY_PREFIX for the bucket key root
Yudhi Armyndharisc2e968275425
diff --git a/.env.example b/.env.exampleindex 837932993..0890e6b4a 100644--- a/.env.example+++ b/.env.example@@ -412,6 +412,13 @@ DATABASE_SYNCHRONIZE=false # WARNING: Set false in production! (data DB# S3_ENDPOINT=http://localhost:9000 # uncomment / set ONLY for an S3-compatible store (MinIO, R2)# S3_BUCKET=openwa# S3_REGION=us-east-1+# Key root inside the bucket (default media/). Two deployments with separate databases need their own+# bucket or prefixes that do not overlap: neither may start with the other (use siblings such as+# prod/ and staging/, never media/ and media/staging/), and never the same STORAGE_LOCAL_PATH. An+# overlap puts one deployment's objects in the other's storage stats and export, and its orphan+# sweeps delete the files its own database does not reference. Changing the prefix on a live+# deployment hides existing objects; move them with a storage export, the change, then an import.+# S3_KEY_PREFIX=media/# MUST set strong, unique values before using S3/MinIO — no defaults shipped.# Production refuses to start if these are empty/placeholder when STORAGE_TYPE=s3 (external).# Canonical names (what the app reads first and the dashboard writes); the legacydiff --git a/docker-compose.dev.yml b/docker-compose.dev.ymlindex 9e3272b6e..bbae0c367 100644--- a/docker-compose.dev.yml+++ b/docker-compose.dev.yml@@ -181,6 +181,7 @@ services:- DATABASE_SSL_REJECT_UNAUTHORIZED=${DATABASE_SSL_REJECT_UNAUTHORIZED:-}- S3_ENDPOINT=${S3_ENDPOINT:-}- S3_BUCKET=${S3_BUCKET:-}+ - S3_KEY_PREFIX=${S3_KEY_PREFIX:-}- S3_REGION=${S3_REGION:-}- S3_ACCESS_KEY_ID=${S3_ACCESS_KEY_ID:-}- S3_SECRET_ACCESS_KEY=${S3_SECRET_ACCESS_KEY:-}diff --git a/docker-compose.yml b/docker-compose.ymlindex 0aafcf582..a278c47a0 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -194,6 +194,7 @@ services:- S3_SECRET_KEY=${S3_SECRET_KEY:-}- S3_REGION=${S3_REGION:-}- S3_BUCKET=${S3_BUCKET:-}+ - S3_KEY_PREFIX=${S3_KEY_PREFIX:-}# Chat-media archiving (opt-in). Forwarded explicitly: this file has no env_file, so a# variable absent from this list simply never reaches the container and the feature stays off# no matter what the operator's .env says.diff --git a/docs/03-system-architecture.md b/docs/03-system-architecture.mdindex 2265b7a6d..0d893ae29 100644--- a/docs/03-system-architecture.md+++ b/docs/03-system-architecture.md@@ -1329,7 +1329,7 @@ export class StorageService {async putFile(filePath: string, data: Buffer): Promise<void> {if (!isSafeStorageKey(filePath)) throw new Error(`Refusing unsafe storage key: ${filePath}`);return this.storageType === 's3' && this.s3Client- ? this.putS3File(filePath, data) // keyed under media/<filePath>+ ? this.putS3File(filePath, data) // keyed under <S3_KEY_PREFIX, default media/><filePath>: this.putLocalFile(filePath, data);}@@ -1337,7 +1337,7 @@ export class StorageService {/* mirrors putFile */}async listFiles(): Promise<string[]> {- /* local recurse, or S3 ListObjectsV2 under media/ */+ /* local recurse, or S3 ListObjectsV2 under <S3_KEY_PREFIX, default media/> */}// createExportStream(): tar.gz of all files; importFromStream(): extract with zip-bomb caps}diff --git a/docs/10-devops-infrastructure.md b/docs/10-devops-infrastructure.mdindex ea859adb0..fa0b98fbb 100644--- a/docs/10-devops-infrastructure.md+++ b/docs/10-devops-infrastructure.md@@ -450,6 +450,14 @@ LOG_FORMAT=json# S3_ACCESS_KEY_ID=minioadmin# S3_SECRET_ACCESS_KEY=minioadmin+# One deployment per bucket and key prefix. S3_KEY_PREFIX (default media/) is the key root: give a+# second deployment with its own database its own bucket or a non-overlapping prefix (neither may+# start with the other: use siblings such as prod/ and staging/, never media/ and media/staging/).+# An overlap puts the other deployment's objects in this one's storage stats and export, and each+# one's orphan sweeps delete the other's media. The same applies to a shared STORAGE_LOCAL_PATH. Changing the prefix on a+# live deployment hides existing objects: export storage, change it, then import.+# S3_KEY_PREFIX=media/+# ===========================================# CACHE & QUEUE# ===========================================diff --git a/src/common/storage/s3-key-prefix.spec.ts b/src/common/storage/s3-key-prefix.spec.tsnew file mode 100644index 000000000..4c14df9e2--- /dev/null+++ b/src/common/storage/s3-key-prefix.spec.ts@@ -0,0 +1,159 @@+import * as fs from 'fs';+import * as os from 'os';+import * as path from 'path';+import { ConfigService } from '@nestjs/config';++// archiver is ESM-only and never used here; stub it before StorageService is imported.+jest.mock('archiver', () => ({ default: jest.fn() }));++const mockSend = jest.fn();+jest.mock('@aws-sdk/client-s3', () => ({+ S3Client: jest.fn().mockImplementation(() => ({ send: mockSend })),+ HeadBucketCommand: jest.fn(),+ CreateBucketCommand: jest.fn(),+ ListObjectsV2Command: jest.fn(),+ GetObjectCommand: jest.fn(),+ PutObjectCommand: jest.fn(),+ DeleteObjectCommand: jest.fn(),+}));++import { DeleteObjectCommand, GetObjectCommand, ListObjectsV2Command, PutObjectCommand } from '@aws-sdk/client-s3';+import { Readable } from 'stream';+import { validateEnv } from '../../config/env.validation';+import { normalizeS3KeyPrefix } from './s3-key-prefix';+import { StorageService } from './storage.service';++/** The input object of every construction of a mocked AWS command. */+function inputs(command: unknown): Record<string, unknown>[] {+ return (command as jest.Mock).mock.calls.map(([input]) => input as Record<string, unknown>);+}++describe('normalizeS3KeyPrefix', () => {+ it.each([+ [undefined, 'media/'],+ ['', 'media/'],+ [' ', 'media/'],+ ['staging', 'staging/'],+ ['staging/', 'staging/'],+ [' tenants/prod// ', 'tenants/prod/'],+ ])('%j gives %j', (raw, root) => {+ expect(normalizeS3KeyPrefix(raw)).toBe(root);+ });++ it.each(['/', '//', '/abs', '../x', 'a/../b', 'a\u0000b', '.', './', 'a//b', 'a/./b'])('rejects %j', raw => {+ expect(normalizeS3KeyPrefix(raw)).toBeNull();+ });++ it('fails boot on an unsafe S3_KEY_PREFIX and accepts a relative one', () => {+ expect(() => validateEnv({ S3_KEY_PREFIX: '../x' })).toThrow(/S3_KEY_PREFIX/);+ expect(() => validateEnv({ S3_KEY_PREFIX: '/abs' })).toThrow(/S3_KEY_PREFIX/);+ expect(() => validateEnv({ S3_KEY_PREFIX: '/' })).toThrow(/S3_KEY_PREFIX/);+ expect(() => validateEnv({ S3_KEY_PREFIX: 'tenants//prod' })).toThrow(/S3_KEY_PREFIX/);+ expect(() => validateEnv({ S3_KEY_PREFIX: 'staging/' })).not.toThrow();+ expect(() => validateEnv({ S3_KEY_PREFIX: '' })).not.toThrow();+ });+});++describe('StorageService S3 key prefix', () => {+ let tmpRoot: string;+ let svc: StorageService | undefined;++ beforeEach(() => {+ tmpRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'owa-s3-prefix-'));+ jest.clearAllMocks();+ mockSend.mockResolvedValue({});+ delete process.env.S3_KEY_PREFIX;+ });++ afterEach(() => {+ svc?.onModuleDestroy();+ svc = undefined;+ delete process.env.S3_KEY_PREFIX;+ fs.rmSync(tmpRoot, { recursive: true, force: true });+ });++ async function makeService(keyPrefix?: string): Promise<StorageService> {+ const config = {+ get: (key: string) => {+ if (key === 'storage.type') return 's3';+ if (key === 'storage.localPath') return path.join(tmpRoot, 'media');+ if (key === 'storage.s3') return { accessKeyId: 'k', secretAccessKey: 's', keyPrefix };+ return undefined;+ },+ } as unknown as ConfigService;+ svc = new StorageService(config);+ await new Promise(resolve => setImmediate(resolve));+ expect(svc.isS3Available()).toBe(true);+ return svc;+ }++ it('routes put, get and delete through a custom prefix, adding the missing trailing slash', async () => {+ mockSend.mockImplementation((cmd: unknown) =>+ Promise.resolve(cmd instanceof GetObjectCommand ? { Body: Readable.from([Buffer.from('x')]) } : {}),+ );+ const storage = await makeService('staging');++ await storage.putFile('x.bin', Buffer.from('x'));+ await storage.getFile('x.bin');+ await storage.deleteFile('x.bin');++ expect(inputs(PutObjectCommand)[0].Key).toBe('staging/x.bin');+ expect(inputs(GetObjectCommand)[0].Key).toBe('staging/x.bin');+ expect(inputs(DeleteObjectCommand)[0].Key).toBe('staging/x.bin');+ });++ it('S3_KEY_PREFIX in the environment wins over the config value', async () => {+ process.env.S3_KEY_PREFIX = 'from-env/';+ const storage = await makeService('from-config');++ await storage.putFile('x.bin', Buffer.from('x'));++ expect(inputs(PutObjectCommand)[0].Key).toBe('from-env/x.bin');+ });++ it('lists under the prefix and never yields a key outside it', async () => {+ mockSend.mockImplementation((cmd: unknown) =>+ Promise.resolve(+ cmd instanceof ListObjectsV2Command+ ? {+ Contents: [+ { Key: 'staging/chat-media/a', Size: 3 },+ { Key: 'staging-old/chat-media/b', Size: 5 },+ { Key: 'media/chat-media/c', Size: 7 },+ ],+ }+ : {},+ ),+ );+ const storage = await makeService('staging');++ const seen: string[] = [];+ for await (const key of storage.iterateFiles('chat-media/')) seen.push(key);+ expect(inputs(ListObjectsV2Command)[0].Prefix).toBe('staging/chat-media/');+ expect(seen).toEqual(['chat-media/a']);++ const { count, sizeBytes } = await storage.getFileCount();+ expect(inputs(ListObjectsV2Command)[1].Prefix).toBe('staging/');+ expect({ count, sizeBytes }).toEqual({ count: 1, sizeBytes: 3 });+ });++ it.each([undefined, '', ' '])('keeps the media/ root when the prefix is %j', async keyPrefix => {+ const storage = await makeService(keyPrefix);++ await storage.putFile('x.bin', Buffer.from('x'));++ expect(inputs(PutObjectCommand)[0].Key).toBe('media/x.bin');+ });+});++describe('S3_KEY_PREFIX sharing guidance', () => {+ // Listing uses the root as a plain key prefix, so a nested root (media/ and media/staging/) puts+ // the inner deployment's objects in the outer one's stats, export and orphan sweeps.+ it.each(['.env.example', 'docs/10-devops-infrastructure.md'])('%s rules out nested prefixes', file => {+ const text = fs+ .readFileSync(path.join(__dirname, '../../..', file), 'utf8')+ .replace(/^\s*# ?/gm, '')+ .replace(/\s+/g, ' ');+ expect(text).toContain('neither may start with the other');+ });+});diff --git a/src/common/storage/s3-key-prefix.ts b/src/common/storage/s3-key-prefix.tsnew file mode 100644index 000000000..aa1cee3ea--- /dev/null+++ b/src/common/storage/s3-key-prefix.ts@@ -0,0 +1,20 @@+import { isSafeStorageKey } from '../utils/path-safety';++/** The S3 key root when S3_KEY_PREFIX is unset or blank, and the layout every existing bucket uses. */+export const DEFAULT_S3_KEY_PREFIX = 'media/';++/**+ * Normalise S3_KEY_PREFIX into the key root every object lives under: trimmed, with exactly one+ * trailing '/', so `staging` never also matches `staging-old/...` in a prefix listing (the orphan+ * sweeps delete what they list). Unset or blank gives `media/`. Returns null for a value that is+ * absolute, contains control characters or an empty, `.` or `..` segment, or is only slashes (keys+ * at the bucket root); boot validation rejects those, as S3-compatible stores such as MinIO refuse+ * object keys with empty or `.` segments.+ */+export function normalizeS3KeyPrefix(raw: string | undefined): string | null {+ const trimmed = (raw ?? '').trim();+ if (!trimmed) return DEFAULT_S3_KEY_PREFIX;+ const body = trimmed.replace(/\/+$/, '');+ if (!body || !isSafeStorageKey(body) || body.split('/').some(s => s === '' || s === '.')) return null;+ return `${body}/`;+}diff --git a/src/common/storage/storage.service.ts b/src/common/storage/storage.service.tsindex 1baf13c69..af8dcaf5f 100644--- a/src/common/storage/storage.service.ts+++ b/src/common/storage/storage.service.ts@@ -14,6 +14,7 @@ import {} from '@aws-sdk/client-s3';import { createLogger } from '../services/logger.service';import { isSafeStorageKey } from '../utils/path-safety';+import { DEFAULT_S3_KEY_PREFIX, normalizeS3KeyPrefix } from './s3-key-prefix';import { createExportStream, ExportFileSource, importFromStream } from './storage-transfer';import {listLocalFiles,@@ -30,6 +31,7 @@ interface S3Config {secretAccessKey?: string;region?: string;bucket?: string;+ keyPrefix?: string;}/** How often an S3-configured service re-probes a bucket that was unreachable at boot. */@@ -63,6 +65,7 @@ export class StorageService implements OnModuleDestroy {private readonly localPath: string;private s3Client: S3Client | null = null;private s3Bucket = 'openwa';+ private s3KeyRoot = DEFAULT_S3_KEY_PREFIX;private s3Available = false;private s3ReprobeTimer: NodeJS.Timeout | null = null;private readonly s3ReprobeIntervalMs = positiveIntFromEnv('S3_REPROBE_INTERVAL_MS', DEFAULT_S3_REPROBE_INTERVAL_MS);@@ -97,6 +100,9 @@ export class StorageService implements OnModuleDestroy {...(endpoint ? { forcePathStyle: true } : {}), // Required for path-style stores (MinIO)});this.s3Bucket = process.env.S3_BUCKET || s3Config.bucket || 'openwa';+ const keyRoot = normalizeS3KeyPrefix(process.env.S3_KEY_PREFIX || s3Config.keyPrefix);+ if (!keyRoot) throw new Error('S3_KEY_PREFIX is not a safe relative key prefix');+ this.s3KeyRoot = keyRoot;void this.initializeS3Bucket();this.startS3Reprobe();} else {@@ -294,7 +300,7 @@ export class StorageService implements OnModuleDestroy {async getFile(filePath: string): Promise<Buffer> {// Mirror putFile: getLocalFile has its own isPathWithin guard, but getS3File builds- // `media/${filePath}` with none — contain both read backends at this boundary.+ // `${s3KeyRoot}${filePath}` with none, so contain both read backends at this boundary.if (!isSafeStorageKey(filePath)) {throw new Error(`Refusing to read an unsafe storage key: ${filePath}`);}@@ -320,7 +326,7 @@ export class StorageService implements OnModuleDestroy {async putFile(filePath: string, data: Buffer): Promise<void> {// Centralized containment so BOTH backends inherit it: putLocalFile has its own isPathWithin- // guard, but putS3File builds `media/${filePath}` with none — reject a traversing key here.+ // guard, but putS3File builds `${s3KeyRoot}${filePath}` with none, so reject a traversing key here.if (!isSafeStorageKey(filePath)) {throw new Error(`Refusing to store an unsafe storage key: ${filePath}`);}@@ -404,15 +410,17 @@ export class StorageService implements OnModuleDestroy {const response = await this.s3Client!.send(new ListObjectsV2Command({Bucket: this.s3Bucket,- Prefix: 'media/',+ Prefix: this.s3KeyRoot,ContinuationToken: continuationToken,}),);for (const obj of response.Contents ?? []) {+ const key = this.stripS3KeyRoot(obj.Key);+ if (key === null) continue;count += 1;sizeBytes += obj.Size ?? 0;- if (obj.Key) keys.push(obj.Key.replace(/^media\//, ''));+ keys.push(key);}continuationToken = response.NextContinuationToken;@@ -505,22 +513,25 @@ export class StorageService implements OnModuleDestroy {const response = await this.s3Client.send(new ListObjectsV2Command({Bucket: this.s3Bucket,- Prefix: `media/${prefix}`,+ Prefix: `${this.s3KeyRoot}${prefix}`,ContinuationToken: continuationToken,}),);for (const obj of response.Contents ?? []) {- if (obj.Key) {- // Remove 'media/' prefix- yield obj.Key.replace(/^media\//, '');- }+ const key = this.stripS3KeyRoot(obj.Key);+ if (key !== null) yield key;}continuationToken = response.NextContinuationToken;} while (continuationToken);}+ /** The store-relative key for a listed object, or null for one outside this deployment's key root. */+ private stripS3KeyRoot(key: string | undefined): string | null {+ return key?.startsWith(this.s3KeyRoot) ? key.slice(this.s3KeyRoot.length) : null;+ }+private async getS3File(filePath: string): Promise<Buffer> {const { stream } = await this.openS3File(filePath);const chunks: Buffer[] = [];@@ -538,7 +549,7 @@ export class StorageService implements OnModuleDestroy {response = await this.s3Client.send(new GetObjectCommand({Bucket: this.s3Bucket,- Key: `media/${filePath}`,+ Key: `${this.s3KeyRoot}${filePath}`,}),);} catch (error: unknown) {@@ -568,7 +579,7 @@ export class StorageService implements OnModuleDestroy {await this.s3Client.send(new PutObjectCommand({Bucket: this.s3Bucket,- Key: `media/${filePath}`,+ Key: `${this.s3KeyRoot}${filePath}`,Body: data,}),);@@ -580,7 +591,7 @@ export class StorageService implements OnModuleDestroy {await this.s3Client.send(new DeleteObjectCommand({Bucket: this.s3Bucket,- Key: `media/${filePath}`,+ Key: `${this.s3KeyRoot}${filePath}`,}),);}diff --git a/src/common/utils/path-safety.ts b/src/common/utils/path-safety.tsindex 236bc6e7a..6ef5a80af 100644--- a/src/common/utils/path-safety.ts+++ b/src/common/utils/path-safety.ts@@ -20,8 +20,8 @@ export function isPathWithin(root: string, target: string): boolean {* Returns true if `key` is a safe, contained relative storage key: a non-empty relative path with no* `..` traversal segment. Used to validate untrusted archive entry names / object keys at the* backend-agnostic `putFile`/`getFile` boundary so an S3 key (which has no host filesystem root to- * check against `isPathWithin`) still can't escape the intended `media/` prefix. Ordinary keys —- * including plugin/JID-style ones with `:`, `@`, `.`, `-` — are preserved.+ * check against `isPathWithin`) still can't escape the configured S3 key root (`S3_KEY_PREFIX`,+ * default `media/`). Ordinary keys — including plugin/JID-style ones with `:`, `@`, `.`, `-` — are preserved.*/export function isSafeStorageKey(key: string): boolean {if (typeof key !== 'string' || key.length === 0) return false;diff --git a/src/config/configuration.ts b/src/config/configuration.tsindex 35b5ac79c..6711d18a7 100644--- a/src/config/configuration.ts+++ b/src/config/configuration.ts@@ -558,6 +558,7 @@ export default () => ({accessKeyId: process.env.S3_ACCESS_KEY_ID,secretAccessKey: process.env.S3_SECRET_ACCESS_KEY,endpoint: process.env.S3_ENDPOINT,+ keyPrefix: process.env.S3_KEY_PREFIX,},},});diff --git a/src/config/env-precedence.ts b/src/config/env-precedence.tsindex d3309d4e3..00d1aca34 100644--- a/src/config/env-precedence.ts+++ b/src/config/env-precedence.ts@@ -50,6 +50,7 @@ export const BLANK_SHADOWED_ENV_KEYS: string[] = ['STORAGE_TYPE','STORAGE_LOCAL_PATH','S3_BUCKET',+ 'S3_KEY_PREFIX','S3_ENDPOINT','S3_REGION','S3_ACCESS_KEY_ID',diff --git a/src/config/env.validation.ts b/src/config/env.validation.tsindex 30f0fdf6e..26f2423b0 100644--- a/src/config/env.validation.ts+++ b/src/config/env.validation.ts@@ -1,4 +1,5 @@import { resolve } from 'path';+import { normalizeS3KeyPrefix } from '../common/storage/s3-key-prefix';type EnvConfig = Record<string, unknown>;@@ -97,6 +98,15 @@ export function validateEnv(config: EnvConfig): EnvConfig {};checkEnum('ENGINE_TYPE', ['whatsapp-web.js', 'baileys']);checkEnum('STORAGE_TYPE', ['local', 's3']);+ // The S3 key root. A prefix that is absolute, traverses, or is only slashes would put this+ // deployment's objects (and its orphan sweeps' deletes) at the bucket root or outside its own root;+ // one with an empty or '.' segment builds keys an S3-compatible store refuses on every write.+ const s3KeyPrefix = str('S3_KEY_PREFIX');+ if (s3KeyPrefix !== undefined && normalizeS3KeyPrefix(s3KeyPrefix) === null) {+ errors.push(+ `S3_KEY_PREFIX must be a relative key prefix such as "staging/" with no empty, "." or ".." segment (got ${JSON.stringify(s3KeyPrefix)})`,+ );+ }// Every production hardening in the repo gates on the exact string 'production', so an// unrecognised value silently selects the permissive branch of each one — CORS, Swagger, DTO// error detail, the default-secret guard and the ALLOW_DEV_API_KEY rejection that stops the public