Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(storage): fail the export when a listed file cannot be opened

Yudhi Armyndharis0a7403ee199e
diff --git a/src/common/storage/storage-transfer.spec.ts b/src/common/storage/storage-transfer.spec.ts
index d36819f12..b32730314 100644
--- a/src/common/storage/storage-transfer.spec.ts
+++ b/src/common/storage/storage-transfer.spec.ts
@@ -57,10 +57,12 @@ const contentFor = (name: string): Buffer => {
};
/** An openFile that counts the file streams open at once and the files ever opened. */
-function trackingOpener(failing: Record<string, 'open' | 'read'> = {}, withSize = true) {
+function trackingOpener(failing: Record<string, 'gone' | 'open' | 'read'> = {}, withSize = true) {
const stats = { opened: 0, open: 0, maxOpen: 0 };
const openFile = (name: string): Promise<ExportFileSource> => {
- if (failing[name] === 'open') return Promise.reject(new Error(`ENOENT: ${name}`));
+ if (failing[name] === 'gone')
+ return Promise.reject(Object.assign(new Error(`ENOENT: ${name}`), { code: 'ENOENT' }));
+ if (failing[name] === 'open') return Promise.reject(new Error(`SlowDown: ${name}`));
const data = contentFor(name);
stats.opened++;
stats.open++;
@@ -110,8 +112,8 @@ describe('createExportStream streams one file at a time', () => {
output.destroy();
});
- it('holds one file stream open at a time, skips an unopenable file, and round-trips the rest', async () => {
- const { stats, openFile } = trackingOpener({ 'media/file-c.bin': 'open' });
+ it('holds one file stream open at a time, skips a file deleted since the listing, and round-trips the rest', async () => {
+ const { stats, openFile } = trackingOpener({ 'media/file-c.bin': 'gone' });
const logger = makeLogger();
const output = await createExportStream(() => Promise.resolve(files), openFile, logger as never);
@@ -124,6 +126,20 @@ describe('createExportStream streams one file at a time', () => {
for (const [name, data] of imported) expect(data.equals(contentFor(name))).toBe(true);
});
+ it('fails the output when a listed file cannot be opened for any other reason', async () => {
+ const { stats, openFile } = trackingOpener({ 'media/file-c.bin': 'open' });
+ const output = await createExportStream(() => Promise.resolve(files), openFile, makeLogger() as never);
+
+ const error = await new Promise<Error>(resolve => {
+ output.on('error', resolve);
+ output.resume();
+ });
+ await settle(20);
+
+ expect(error.message).toBe('SlowDown: media/file-c.bin');
+ expect(stats.opened).toBe(2);
+ });
+
it('still exports a file whose size the backend did not report', async () => {
const { openFile } = trackingOpener({}, false);
const logger = makeLogger();
diff --git a/src/common/storage/storage-transfer.ts b/src/common/storage/storage-transfer.ts
index 488bb2147..b55d9b0ac 100644
--- a/src/common/storage/storage-transfer.ts
+++ b/src/common/storage/storage-transfer.ts
@@ -10,6 +10,22 @@ const DEFAULT_IMPORT_MAX_BYTES = 200 * 1024 * 1024;
/** Max number of entries an import archive may contain. Bounds an entry-count DoS. */
const DEFAULT_IMPORT_MAX_ENTRIES = 100_000;
+/**
+ * True when a storage read failed because the object is simply not there.
+ *
+ * Both backends must be covered, and they report it differently: the local backend raises a POSIX
+ * `ENOENT` (a `.code`), while S3 raises `NoSuchKey`/`NotFound`, which carries a `.name` and no
+ * `.code` at all; `StorageService.getS3File` rethrows that original error when the local read-through
+ * also misses. Checking only `.code` turns a missing S3 object into a 500 on the one backend where
+ * retention and bucket lifecycle rules make a miss most likely.
+ */
+export function isMissingObjectError(error: unknown): boolean {
+ const e = error as { code?: string; name?: string; $metadata?: { httpStatusCode?: number } };
+ return (
+ e?.code === 'ENOENT' || e?.name === 'NoSuchKey' || e?.name === 'NotFound' || e?.$metadata?.httpStatusCode === 404
+ );
+}
+
function positiveIntFromEnv(name: string, fallback: number): number {
const parsed = Number.parseInt(process.env[name] ?? '', 10);
return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
@@ -30,9 +46,11 @@ export interface ExportFileSource {
// Returns the output as soon as the file list is known and fills the archive in the background, one
// entry at a time: each file is opened only after the previous entry has been written through to the
-// consumer, so memory holds one file's in-flight chunks, not the whole store. A file that cannot be
-// opened is skipped with a warning; a read that fails part-way destroys the output with that error,
-// because its tar header is already written and skipping it would leave a corrupt archive.
+// consumer, so memory holds one file's in-flight chunks, not the whole store. A file deleted since the
+// listing is skipped with a warning. Any other open failure (a throttled or unreachable store) destroys
+// the output, so the export fails instead of reporting success for an archive missing that file; so
+// does a read that fails part-way, because its tar header is already written and skipping it would
+// leave a corrupt archive.
export async function createExportStream(
listFiles: () => Promise<string[]>,
openFile: (filePath: string) => Promise<ExportFileSource>,
@@ -101,6 +119,7 @@ async function appendEntries(
try {
source = await openFile(file);
} catch (error) {
+ if (!isMissingObjectError(error)) throw error;
logger.warn(`Failed to export file: ${file}`, { error: String(error) });
continue;
}
diff --git a/src/common/storage/storage.service.s3-reprobe.spec.ts b/src/common/storage/storage.service.s3-reprobe.spec.ts
index 9b74da0c4..4d7f46193 100644
--- a/src/common/storage/storage.service.s3-reprobe.spec.ts
+++ b/src/common/storage/storage.service.s3-reprobe.spec.ts
@@ -312,6 +312,24 @@ describe('StorageService S3 re-probe and recovery', () => {
await expect(svc.openFile('missing.bin')).rejects.toThrow('NoSuchKey');
});
+ it('openFile surfaces the local error when the fallback copy exists but cannot be opened', async () => {
+ mockSend.mockImplementation((cmd: unknown) => {
+ if (cmd instanceof GetObjectCommand) return Promise.reject(s3Error('NoSuchKey'));
+ return Promise.resolve({});
+ });
+ const svc = new StorageService(makeConfig());
+ await flush();
+ fs.writeFileSync(path.join(localPath, 'gap.bin'), 'gap-media');
+ const denied = Object.assign(new Error('EACCES: permission denied'), { code: 'EACCES' });
+ const openSpy = jest.spyOn(fs.promises, 'open').mockRejectedValueOnce(denied);
+
+ try {
+ await expect(svc.openFile('gap.bin')).rejects.toBe(denied);
+ } finally {
+ openSpy.mockRestore();
+ }
+ });
+
it('deleteFile removes the local fallback copy as well as the S3 object', async () => {
mockSend.mockResolvedValue({}); // HeadBucket at boot, DeleteObject later
const svc = new StorageService(makeConfig());
diff --git a/src/common/storage/storage.service.spec.ts b/src/common/storage/storage.service.spec.ts
index aed67c011..18f0895fd 100644
--- a/src/common/storage/storage.service.spec.ts
+++ b/src/common/storage/storage.service.spec.ts
@@ -3,7 +3,7 @@ import * as os from 'os';
import * as path from 'path';
import * as tar from 'tar-stream';
import { createGzip } from 'zlib';
-import { Readable } from 'stream';
+import { PassThrough, Readable } from 'stream';
import { ConfigService } from '@nestjs/config';
// `archiver` v8 ships as ESM only, which ts-jest cannot parse when StorageService
@@ -12,6 +12,7 @@ import { ConfigService } from '@nestjs/config';
jest.mock('archiver', () => ({ default: jest.fn() }));
import { StorageService } from './storage.service';
+import * as transfer from './storage-transfer';
/** Build an in-memory gzipped tar archive from the given entries. */
function makeTarGz(entries: { name: string; data: string }[]): Promise<Buffer> {
@@ -345,6 +346,23 @@ describe('StorageService.createExportStream enumerates the whole store', () => {
expect(listFiles).not.toHaveBeenCalled();
});
+ // The export fails on any open error other than a missing object, so a listed key openFile always
+ // refuses (an S3 object at the bare key root, or one with a `..` segment) must not reach it.
+ it('leaves out a listed key that openFile would refuse', async () => {
+ const { service } = makeLocalService();
+ jest.spyOn(service, 'iterateFiles').mockImplementation(async function* () {
+ yield await Promise.resolve('media/a.bin');
+ yield '';
+ yield 'media/../x.bin';
+ });
+ const create = jest.spyOn(transfer, 'createExportStream').mockResolvedValue(new PassThrough());
+
+ await service.createExportStream();
+
+ expect(await create.mock.calls[0][0]()).toEqual(['media/a.bin']);
+ create.mockRestore();
+ });
+
/**
* The count is the OTHER half of the same fix and had no test of its own — reverting it to the
* capped listing left the suite green. It is the pre-check an operator runs before that migration,
diff --git a/src/common/storage/storage.service.ts b/src/common/storage/storage.service.ts
index af8dcaf5f..453be9f59 100644
--- a/src/common/storage/storage.service.ts
+++ b/src/common/storage/storage.service.ts
@@ -15,7 +15,7 @@ import {
import { createLogger } from '../services/logger.service';
import { isSafeStorageKey } from '../utils/path-safety';
import { DEFAULT_S3_KEY_PREFIX, normalizeS3KeyPrefix } from './s3-key-prefix';
-import { createExportStream, ExportFileSource, importFromStream } from './storage-transfer';
+import { createExportStream, ExportFileSource, importFromStream, isMissingObjectError } from './storage-transfer';
import {
listLocalFiles,
iterateLocalFiles,
@@ -25,6 +25,9 @@ import {
deleteLocalFile,
} from './storage-local-files';
+// Re-exported so feature modules keep importing it next to StorageService.
+export { isMissingObjectError } from './storage-transfer';
+
interface S3Config {
endpoint?: string;
accessKeyId?: string;
@@ -37,22 +40,6 @@ interface S3Config {
/** How often an S3-configured service re-probes a bucket that was unreachable at boot. */
export const DEFAULT_S3_REPROBE_INTERVAL_MS = 60_000;
-/**
- * True when a storage read failed because the object is simply not there.
- *
- * Both backends must be covered, and they report it differently: the local backend raises a POSIX
- * `ENOENT` (a `.code`), while S3 raises `NoSuchKey`/`NotFound`, which carries a `.name` and no
- * `.code` at all — `getS3File` below rethrows that original error when the local read-through also
- * misses. Checking only `.code` turns a missing S3 object into a 500 on the one backend where
- * retention and bucket lifecycle rules make a miss most likely.
- */
-export function isMissingObjectError(error: unknown): boolean {
- const e = error as { code?: string; name?: string; $metadata?: { httpStatusCode?: number } };
- return (
- e?.code === 'ENOENT' || e?.name === 'NoSuchKey' || e?.name === 'NotFound' || e?.$metadata?.httpStatusCode === 404
- );
-}
-
function positiveIntFromEnv(name: string, fallback: number): number {
const parsed = Number.parseInt(process.env[name] ?? '', 10);
return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
@@ -440,13 +427,22 @@ export class StorageService implements OnModuleDestroy {
// repoint STORAGE_TYPE, import) leave media behind on the old backend silently, and the
// operator's own files/count pre-check was truncated by the same path, so the consistency check
// could not reveal the gap. An export exists to be complete; that is what the uncapped walk is for.
+ // A listed key openFile would refuse (an S3 object named `..`, or one sitting exactly at the key
+ // root) is left out here: the export fails on any open error other than a missing object, and a
+ // key that can never be opened must not fail every export.
return createExportStream(
- () => this.listAllFiles(),
+ async () => (await this.listAllFiles()).filter(file => this.isExportableKey(file)),
filePath => this.openFile(filePath),
this.logger,
);
}
+ private isExportableKey(file: string): boolean {
+ if (isSafeStorageKey(file)) return true;
+ this.logger.warn(`Skipping an unsafe storage key in the export: ${JSON.stringify(file)}`);
+ return false;
+ }
+
/** Every key in the store, uncapped — the completeness counterpart to the capped listFiles(). */
private async listAllFiles(): Promise<string[]> {
const files: string[] = [];
@@ -556,13 +552,14 @@ export class StorageService implements OnModuleDestroy {
// Read-through: media written while S3 was down lives only in the local fallback dir, so after
// recovery a plain S3 read would split-brain (NoSuchKey even though the app served the file
// fine during the outage). Fall through to the local copy; if there is none, surface the
- // original S3 error so "not found" semantics are unchanged.
+ // original S3 error so "not found" semantics are unchanged. A local copy that exists but cannot
+ // be opened surfaces its own error, so it is never mistaken for a missing file.
if ((error as { name?: string }).name !== 'NoSuchKey') throw error;
let local: ExportFileSource;
try {
local = await openLocalFile(this.localPath, filePath);
- } catch {
- throw error;
+ } catch (localError: unknown) {
+ throw isMissingObjectError(localError) ? error : localError;
}
this.logger.debug(`Served '${filePath}' from the local fallback dir (not yet in S3)`);
return local;