Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
import { Injectable, OnModuleDestroy } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import * as fs from 'fs';
import * as path from 'path';
import { Readable, PassThrough } from 'stream';
import {
S3Client,
ListObjectsV2Command,
GetObjectCommand,
PutObjectCommand,
DeleteObjectCommand,
HeadBucketCommand,
CreateBucketCommand,
} from '@aws-sdk/client-s3';
import { createLogger } from '../services/logger.service';
import { positiveIntFromEnv } from '../utils/env-int';
import { isSafeStorageKey } from '../utils/path-safety';
import { DEFAULT_S3_KEY_PREFIX, normalizeS3KeyPrefix } from './s3-key-prefix';
import { createExportStream, ExportFileSource, importFromStream, isMissingObjectError } from './storage-transfer';
import { iterateLocalFiles, getLocalFile, openLocalFile, putLocalFile, deleteLocalFile } from './storage-local-files';
// Re-exported so feature modules keep importing it next to StorageService.
export { isMissingObjectError } from './storage-transfer';
interface S3Config {
endpoint?: string;
accessKeyId?: string;
secretAccessKey?: string;
region?: string;
bucket?: string;
keyPrefix?: string;
}
/** How often an S3-configured service re-probes a bucket that was unreachable at boot. */
export const DEFAULT_S3_REPROBE_INTERVAL_MS = 60_000;
/**
* Cap on one S3 DeleteObject. The retention purges delete row by row behind a single-flight guard,
* so a delete that never settles would hold that guard and stop every later purge until a restart.
*/
export const S3_DELETE_TIMEOUT_MS = 30_000;
/**
* Cap on one bucket probe request. The status endpoint and the storage migration routes await the
* probe, and every later re-probe waits on one already in flight, so it gets a tighter bound than
* the idle-socket timeout below, and one that also stops the SDK's retries.
*/
export const S3_PROBE_TIMEOUT_MS = 10_000;
/**
* Connect and idle-socket bounds for every S3 request. Without them a store that accepts the
* connection and never answers (a paused container, a stuck proxy) leaves the request pending
* forever, holding its media buffer and the caller. The socket bound is an idle timeout, so a long
* upload or download that keeps moving bytes is not cut off. A bare requestTimeout would not do:
* the HTTP handler only logs a warning when it expires.
*/
export const S3_CONNECT_TIMEOUT_MS = 5_000;
export const S3_SOCKET_TIMEOUT_MS = 30_000;
@Injectable()
export class StorageService implements OnModuleDestroy {
private readonly logger = createLogger('StorageService');
private readonly storageType: string;
private readonly localPath: string;
private s3Client: S3Client | null = null;
private s3Bucket = 'openwa';
private s3KeyRoot = DEFAULT_S3_KEY_PREFIX;
private s3Available = false;
private s3ReprobeTimer: NodeJS.Timeout | null = null;
private readonly s3ReprobeIntervalMs = positiveIntFromEnv('S3_REPROBE_INTERVAL_MS', DEFAULT_S3_REPROBE_INTERVAL_MS);
constructor(private readonly configService: ConfigService) {
this.storageType = this.configService.get<string>('storage.type') || 'local';
this.localPath = this.configService.get<string>('storage.localPath') || './data/media';
// Initialize S3 client if storage type is s3
if (this.storageType === 's3') {
const s3Config = this.configService.get<S3Config>('storage.s3') || {};
const endpoint = process.env.S3_ENDPOINT || s3Config.endpoint;
// Canonical names are S3_ACCESS_KEY_ID / S3_SECRET_ACCESS_KEY (what configuration.ts
// and the dashboard write). The legacy S3_ACCESS_KEY / S3_SECRET_KEY are still read as
// a fallback so existing .env files keep working.
const accessKeyId = process.env.S3_ACCESS_KEY_ID || process.env.S3_ACCESS_KEY || s3Config.accessKeyId;
const secretAccessKey = process.env.S3_SECRET_ACCESS_KEY || process.env.S3_SECRET_KEY || s3Config.secretAccessKey;
const region = process.env.S3_REGION || s3Config.region || 'us-east-1';
// Standard AWS S3 needs only credentials + region — the SDK derives the regional endpoint, and
// an `endpoint` is only required for S3-compatible stores (MinIO, R2, …). Requiring it dropped
// valid AWS configs to a silent local fallback (#735). forcePathStyle is likewise a path-style
// concern (MinIO/etc.); AWS S3 uses virtual-hosted addressing — so tie both to the endpoint.
if (accessKeyId && secretAccessKey) {
this.s3Client = new S3Client({
...(endpoint ? { endpoint } : {}),
region,
credentials: {
accessKeyId,
secretAccessKey,
},
...(endpoint ? { forcePathStyle: true } : {}), // Required for path-style stores (MinIO)
requestHandler: { connectionTimeout: S3_CONNECT_TIMEOUT_MS, socketTimeout: S3_SOCKET_TIMEOUT_MS },
});
this.s3Bucket = process.env.S3_BUCKET || s3Config.bucket || 'openwa';
const keyRoot = normalizeS3KeyPrefix(process.env.S3_KEY_PREFIX || s3Config.keyPrefix);
if (!keyRoot) throw new Error('S3_KEY_PREFIX is not a safe relative key prefix');
this.s3KeyRoot = keyRoot;
void this.initializeS3Bucket();
this.startS3Reprobe();
} else {
// Every other degradation in this service announces itself, but this one could not: the
// logging all lives past the client construction above, so an s3 deployment missing its
// credentials built no client, wrote every file to local disk, and said nothing at all.
// The operator's first symptom was an empty bucket with no failure to point at.
//
// Name the variables actually missing rather than declaring all of them absent: reaching
// here with one of the pair set is a plain typo in the other, and "no credentials found"
// would send that operator looking at the one they got right.
const missing = [
accessKeyId ? null : 'S3_ACCESS_KEY_ID',
secretAccessKey ? null : 'S3_SECRET_ACCESS_KEY',
].filter((name): name is string => name !== null);
this.logger.warn(
`STORAGE_TYPE=s3 but ${missing.join(' and ')} is not set; media is being written to the ` +
`local dir '${this.localPath}' instead of the bucket. The built-in MinIO uses ` +
`minioadmin/minioadmin.`,
);
}
}
// Ensure local directory exists
if (!fs.existsSync(this.localPath)) {
fs.mkdirSync(this.localPath, { recursive: true });
}
}
onModuleDestroy(): void {
this.clearS3Reprobe();
}
private async initializeS3Bucket(): Promise<void> {
if (!this.s3Client) return;
try {
await this.ensureS3Bucket();
this.s3Available = true;
this.logger.log(`S3 bucket '${this.s3Bucket}' is available`);
} catch (error: unknown) {
this.logger.error('S3 bucket check failed', String(error));
this.lastS3Error = String(error);
this.warnLocalFallback();
}
}
/**
* HeadBucket, and create the bucket when the store answers that it does not exist. Throws when the
* store is unreachable or the create fails. The boot probe and every re-probe share it: a store that
* was down at boot and comes back empty (a fresh MinIO volume) must still get its bucket, or the
* re-probe would see NotFound forever and S3 would never become available without a restart.
*/
private async ensureS3Bucket(): Promise<void> {
try {
await this.s3Client!.send(new HeadBucketCommand({ Bucket: this.s3Bucket }), {
abortSignal: AbortSignal.timeout(S3_PROBE_TIMEOUT_MS),
});
return;
} catch (error: unknown) {
const name = (error as { name?: string }).name;
if (name !== 'NotFound' && name !== 'NoSuchBucket') throw error;
}
try {
await this.s3Client!.send(new CreateBucketCommand({ Bucket: this.s3Bucket }), {
abortSignal: AbortSignal.timeout(S3_PROBE_TIMEOUT_MS),
});
} catch (error: unknown) {
// Another replica (or an overlapping probe) created it first, and this deployment owns it.
// BucketAlreadyExists means another account owns the name, so that one still throws.
if ((error as { name?: string }).name !== 'BucketAlreadyOwnedByYou') throw error;
return;
}
this.logger.log(`Created S3 bucket '${this.s3Bucket}'`);
}
private warnLocalFallback(): void {
this.logger.warn(
`S3 bucket '${this.s3Bucket}' is unavailable (${this.lastS3Error}); media storage degraded, using the ` +
`local fallback dir '${this.localPath}'. Re-probing every ${this.s3ReprobeIntervalMs}ms; writes return ` +
'to S3 once it recovers.',
);
}
/**
* Periodically re-probe S3 while the effective backend is the local fallback (S3 was unreachable at
* boot). Recovery is one-way: availability only ever transitions false→true here, and once S3 is
* back the timer stops — a session already on S3 is never dropped to local by a transient error
* without an explicit re-evaluation.
*/
private startS3Reprobe(): void {
this.s3ReprobeTimer = setInterval(() => {
if (this.s3Available) {
this.clearS3Reprobe();
return;
}
void this.refreshS3Availability().then(available => {
if (!available) this.warnLocalFallback();
});
}, this.s3ReprobeIntervalMs);
// Don't keep the process alive for a probe; shutdown clears it via onModuleDestroy.
this.s3ReprobeTimer.unref();
}
private clearS3Reprobe(): void {
if (this.s3ReprobeTimer) {
clearInterval(this.s3ReprobeTimer);
this.s3ReprobeTimer = null;
}
}
// ============================================================================
// Current Storage Operations
// ============================================================================
getCurrentStorageType(): string {
return this.storageType;
}
private lastS3Check = 0;
/** Why the last probe failed. The fallback warning names it: a store refusing the create is not an outage. */
private lastS3Error = '';
private s3CheckInFlight: Promise<void> | null = null;
/**
* Re-probe S3/MinIO reachability when it's currently marked unavailable — e.g. a bundled MinIO that
* came up AFTER the app booted (the init HeadBucket raced and latched false). Throttled (10s) and
* in-flight-deduped so the status endpoint and the periodic re-probe can call it cheaply. Once
* available it stays available (no need to re-probe a healthy backend here).
*/
async refreshS3Availability(): Promise<boolean> {
if (this.storageType !== 's3' || !this.s3Client || this.s3Available) return this.s3Available;
if (this.s3CheckInFlight) {
await this.s3CheckInFlight;
return this.s3Available;
}
const now = Date.now();
if (now - this.lastS3Check < 10_000) return this.s3Available;
this.lastS3Check = now;
this.s3CheckInFlight = (async () => {
try {
await this.ensureS3Bucket();
this.s3Available = true;
// WARN (not log): the degraded window matters — media written to the local fallback dir while
// S3 was down stays local-only; reads for those keys keep working via the NoSuchKey
// read-through in getS3File, but the operator should reconcile/acknowledge the gap.
this.logger.warn(
`S3 bucket '${this.s3Bucket}' recovered — media storage back on S3. Files written to the local ` +
`fallback dir '${this.localPath}' during the outage remain there (still readable via read-through).`,
);
} catch (error: unknown) {
// still unavailable: leave s3Available false; a later poll retries after the throttle window
this.lastS3Error = String(error);
} finally {
this.s3CheckInFlight = null;
}
})();
await this.s3CheckInFlight;
return this.s3Available;
}
/**
* Enumerate every file in the store (all S3 pages unioned with the local fallback dir, each key
* once) as a stream. The union matters: media written locally while S3 was down is otherwise
* invisible to enumeration, although getFile reads through to it, and a sweep reconciling against
* the listing could never reclaim it. Callers that must reconcile against the full store (e.g.
* the status-media orphan sweep) iterate this.
*
* `prefix` narrows the walk at the source (the S3 ListObjectsV2 prefix and the local traversal
* root), so a caller reconciling one subtree neither pages the whole store nor holds every key
* of it in the dedupe Set.
*
* A local directory the walk cannot read fails it, so no caller is handed a short listing without
* knowing. `skipUnreadable` is for a caller that only acts on the files it was shown (an orphan
* sweep): the directory is named in a warning and left alone.
*/
async *iterateFiles(prefix = '', options: { skipUnreadable?: boolean } = {}): AsyncGenerator<string> {
if (this.storageType === 's3' && this.s3Client && this.s3Available) {
const seen = new Set<string>();
for await (const file of this.iterateS3Files(prefix)) {
seen.add(file);
yield file;
}
for await (const file of this.iterateLocalFiles(prefix, options.skipUnreadable)) {
if (!seen.has(file)) yield file;
}
return;
}
yield* this.iterateLocalFiles(prefix, options.skipUnreadable);
}
async getFile(filePath: string): Promise<Buffer> {
// Mirror putFile: getLocalFile has its own isPathWithin guard, but getS3File builds
// `${s3KeyRoot}${filePath}` with none, so contain both read backends at this boundary.
if (!isSafeStorageKey(filePath)) {
throw new Error(`Refusing to read an unsafe storage key: ${filePath}`);
}
if (this.storageType === 's3' && this.s3Client && this.s3Available) {
return this.getS3File(filePath);
}
return this.getLocalFile(filePath);
}
/**
* Open a file as a stream instead of reading it into memory, with the same key guard and S3
* read-through as getFile. A missing or unreadable file rejects here rather than on the stream.
*/
async openFile(filePath: string): Promise<ExportFileSource> {
if (!isSafeStorageKey(filePath)) {
throw new Error(`Refusing to read an unsafe storage key: ${filePath}`);
}
if (this.storageType === 's3' && this.s3Client && this.s3Available) {
return this.openS3File(filePath);
}
return openLocalFile(this.localPath, filePath);
}
async putFile(filePath: string, data: Buffer): Promise<void> {
// Centralized containment so BOTH backends inherit it: putLocalFile has its own isPathWithin
// guard, but putS3File builds `${s3KeyRoot}${filePath}` with none, so reject a traversing key here.
if (!isSafeStorageKey(filePath)) {
throw new Error(`Refusing to store an unsafe storage key: ${filePath}`);
}
if (this.storageType === 's3' && this.s3Client && this.s3Available) {
return this.putS3File(filePath, data);
}
return this.putLocalFile(filePath, data);
}
/**
* Delete a file previously written via `putFile`. Same unsafe-key guard as `getFile`/`putFile` (both
* backends key off it, S3's has no host filesystem to check `isPathWithin` against). When S3 is
* active the key is deleted from BOTH backends — delete-through, symmetric with getFile's
* read-through: the key may exist only in the local fallback dir (written during an S3 outage),
* and deleting just the S3 object would orphan those bytes permanently once the caller drops its
* DB reference. Missing-file is treated as success on both backends (local: ENOENT is swallowed;
* S3 DeleteObject is idempotent by design) so a caller doesn't have to special-case "already gone".
*/
async deleteFile(filePath: string): Promise<void> {
if (!isSafeStorageKey(filePath)) {
throw new Error(`Refusing to delete an unsafe storage key: ${filePath}`);
}
if (this.storageType === 's3' && this.s3Client && this.s3Available) {
await this.deleteS3File(filePath);
}
return this.deleteLocalFile(filePath);
}
async getFileCount(): Promise<{ count: number; sizeBytes: number }> {
if (this.storageType === 's3' && this.s3Client && this.s3Available) {
// ListObjectsV2 already returns each object's Size, so report the real total instead of a
// 100KB-per-file estimate — no extra API calls beyond the listing we'd do anyway.
const s3 = await this.getS3CountAndSize();
// Union with the local fallback dir (media written locally while S3 was down is not in the
// bucket listing): a local-only file counts with its real on-disk size; for a key present in
// both, the S3 copy is authoritative.
const s3Keys = new Set(s3.keys);
let { count, sizeBytes } = s3;
for await (const file of this.iterateLocalFiles()) {
if (s3Keys.has(file) || !this.isExportableKey(file)) continue;
count += 1;
try {
// Same reason as the local branch below: the walk covers the whole store, so the stat must yield.
sizeBytes += (await fs.promises.stat(path.join(this.localPath, file))).size;
} catch (error) {
this.logger.debug(`Failed to stat file: ${file}`, { error: String(error) });
}
}
return { count, sizeBytes };
}
// The whole store is walked, for the same reason createExportStream does: this is the pre-check an
// operator runs BEFORE the export/import migration, so it has to see every file the export will carry.
const files = await this.listExportableFiles();
let sizeBytes = 0;
for (const file of files) {
try {
const fullPath = path.join(this.localPath, file);
// Awaited, not statSync: a synchronous stat per file holds the event loop for the whole store,
// so health checks, webhooks and every in-flight request wait behind a count. Measured at one
// event-loop tick for 2000 files.
const stats = await fs.promises.stat(fullPath);
sizeBytes += stats.size;
} catch (error) {
this.logger.debug(`Failed to stat file: ${file}`, { error: String(error) });
}
}
return { count: files.length, sizeBytes };
}
private async getS3CountAndSize(): Promise<{ count: number; sizeBytes: number; keys: string[] }> {
let count = 0;
let sizeBytes = 0;
// Keys (prefix-stripped) are kept so getFileCount can union the local fallback dir without a
// second listing.
const keys: string[] = [];
let continuationToken: string | undefined;
do {
const response = await this.s3Client!.send(
new ListObjectsV2Command({
Bucket: this.s3Bucket,
Prefix: this.s3KeyRoot,
ContinuationToken: continuationToken,
}),
);
for (const obj of response.Contents ?? []) {
const key = this.stripS3KeyRoot(obj.Key);
if (key === null || !this.isExportableKey(key)) continue;
count += 1;
sizeBytes += obj.Size ?? 0;
keys.push(key);
}
continuationToken = response.NextContinuationToken;
} while (continuationToken);
return { count, sizeBytes, keys };
}
// ============================================================================
// Export/Import - tar.gz streams (implemented in storage-transfer.ts)
// ============================================================================
createExportStream(): Promise<PassThrough> {
// The whole store is enumerated (iterateFiles): an export exists to be complete, and a short one
// would leave media behind on the old backend silently after the documented local→S3 migration
// (export, repoint STORAGE_TYPE, import).
// A listed key openFile would refuse (an S3 object named `..`, or one sitting exactly at the key
// root) is left out here: the export fails on any open error other than a missing object, and a
// key that can never be opened must not fail every export.
return createExportStream(
() => this.listExportableFiles(),
filePath => this.openFile(filePath),
this.logger,
);
}
/**
* Whether the export carries a listed key. The file count applies the same rule, so the pre-check never
* promises a file the archive will not hold.
*/
private isExportableKey(file: string): boolean {
// A key ending in "/" is a folder placeholder an S3 console leaves behind: it holds no file, and written to
// the archive as one it would fail the import that restores it onto a local store.
if (file.endsWith('/')) return false;
if (isSafeStorageKey(file)) return true;
this.logger.warn(`Leaving an unsafe storage key out of the export and the file count: ${JSON.stringify(file)}`);
return false;
}
/** Every key in the store that the export carries. */
private async listExportableFiles(): Promise<string[]> {
const files: string[] = [];
for await (const file of this.iterateFiles()) {
if (this.isExportableKey(file)) files.push(file);
}
return files;
}
// Best-effort, NOT atomic: see the implementation in storage-transfer.ts for the full contract.
importFromStream(inputStream: Readable): Promise<{ imported: number; failed: number }> {
return importFromStream(inputStream, (filePath, data) => this.putFile(filePath, data), this.logger);
}
// ============================================================================
// Local Storage Operations (implemented in storage-local-files.ts)
// ============================================================================
/**
* Local enumeration as a stream. Async + iterative (a work queue, not recursion) so a deep/wide
* media tree can't block the event loop or stack-overflow; still bounded by the max directory
* depth so a pathological tree can't descend unbounded.
*/
private iterateLocalFiles(prefix = '', skipUnreadable = false): AsyncGenerator<string> {
return iterateLocalFiles(this.localPath, prefix, { skipUnreadable, logger: this.logger });
}
private getLocalFile(filePath: string): Promise<Buffer> {
return getLocalFile(this.localPath, filePath);
}
private putLocalFile(filePath: string, data: Buffer): Promise<void> {
return putLocalFile(this.localPath, filePath, data);
}
private deleteLocalFile(filePath: string): Promise<void> {
return deleteLocalFile(this.localPath, filePath);
}
// ============================================================================
// S3 Storage Operations
// ============================================================================
/** Stream every S3 object key (prefix-stripped), following the ListObjectsV2 pagination token. */
private async *iterateS3Files(prefix = ''): AsyncGenerator<string> {
if (!this.s3Client) return;
let continuationToken: string | undefined;
do {
const response = await this.s3Client.send(
new ListObjectsV2Command({
Bucket: this.s3Bucket,
Prefix: `${this.s3KeyRoot}${prefix}`,
ContinuationToken: continuationToken,
}),
);
for (const obj of response.Contents ?? []) {
const key = this.stripS3KeyRoot(obj.Key);
if (key !== null) yield key;
}
continuationToken = response.NextContinuationToken;
} while (continuationToken);
}
/** The store-relative key for a listed object, or null for one outside this deployment's key root. */
private stripS3KeyRoot(key: string | undefined): string | null {
return key?.startsWith(this.s3KeyRoot) ? key.slice(this.s3KeyRoot.length) : null;
}
private async getS3File(filePath: string): Promise<Buffer> {
const { stream } = await this.openS3File(filePath);
const chunks: Buffer[] = [];
for await (const chunk of stream) {
chunks.push(Buffer.from(chunk as ArrayBuffer));
}
return Buffer.concat(chunks);
}
private async openS3File(filePath: string): Promise<ExportFileSource> {
if (!this.s3Client) throw new Error('S3 client not initialized');
let response;
try {
response = await this.s3Client.send(
new GetObjectCommand({
Bucket: this.s3Bucket,
Key: `${this.s3KeyRoot}${filePath}`,
}),
);
} catch (error: unknown) {
// Read-through: media written while S3 was down lives only in the local fallback dir, so after
// recovery a plain S3 read would split-brain (NoSuchKey even though the app served the file
// fine during the outage). Fall through to the local copy; if there is none, surface the
// original S3 error so "not found" semantics are unchanged. A local copy that exists but cannot
// be opened surfaces its own error, so it is never mistaken for a missing file.
if ((error as { name?: string }).name !== 'NoSuchKey') throw error;
let local: ExportFileSource;
try {
local = await openLocalFile(this.localPath, filePath);
} catch (localError: unknown) {
throw isMissingObjectError(localError) ? error : localError;
}
this.logger.debug(`Served '${filePath}' from the local fallback dir (not yet in S3)`);
return local;
}
if (!response.Body) throw new Error('Empty response body');
return { stream: response.Body as Readable, size: response.ContentLength };
}
private async putS3File(filePath: string, data: Buffer): Promise<void> {
if (!this.s3Client) throw new Error('S3 client not initialized');
await this.s3Client.send(
new PutObjectCommand({
Bucket: this.s3Bucket,
Key: `${this.s3KeyRoot}${filePath}`,
Body: data,
}),
);
}
private async deleteS3File(filePath: string): Promise<void> {
if (!this.s3Client) throw new Error('S3 client not initialized');
await this.s3Client.send(
new DeleteObjectCommand({
Bucket: this.s3Bucket,
Key: `${this.s3KeyRoot}${filePath}`,
}),
{ abortSignal: AbortSignal.timeout(S3_DELETE_TIMEOUT_MS) },
);
}
}