Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(storage): create a missing bucket when the S3 re-probe recovers

Yudhi Armyndharis36fc6fa40c78
diff --git a/src/common/storage/storage.service.s3-reprobe.spec.ts b/src/common/storage/storage.service.s3-reprobe.spec.ts
index 159b3007a..9b74da0c4 100644
--- a/src/common/storage/storage.service.s3-reprobe.spec.ts
+++ b/src/common/storage/storage.service.s3-reprobe.spec.ts
@@ -25,7 +25,13 @@ jest.mock('@aws-sdk/client-s3', () => {
};
});
-import { DeleteObjectCommand, GetObjectCommand, ListObjectsV2Command } from '@aws-sdk/client-s3';
+import {
+ CreateBucketCommand,
+ DeleteObjectCommand,
+ GetObjectCommand,
+ HeadBucketCommand,
+ ListObjectsV2Command,
+} from '@aws-sdk/client-s3';
import { DEFAULT_S3_REPROBE_INTERVAL_MS, StorageService } from './storage.service';
const ENV_KEYS = [
@@ -127,6 +133,66 @@ describe('StorageService S3 re-probe and recovery', () => {
expect(svc.isS3Available()).toBe(true);
});
+ it('creates the bucket on re-probe when the store was down at boot and came back empty', async () => {
+ mockSend.mockRejectedValueOnce(s3Error('NetworkingError')); // boot probe: store not up yet
+ const svc = new StorageService(makeConfig());
+ warnSpyOf(svc);
+ await flush();
+ expect(svc.isS3Available()).toBe(false);
+
+ // The store is up now, but on a fresh volume: HeadBucket answers NotFound, CreateBucket succeeds.
+ mockSend.mockImplementation((cmd: unknown) =>
+ cmd instanceof HeadBucketCommand ? Promise.reject(s3Error('NotFound')) : Promise.resolve({}),
+ );
+ await jest.advanceTimersByTimeAsync(DEFAULT_S3_REPROBE_INTERVAL_MS);
+
+ expect(mockSend.mock.calls.some(([cmd]) => cmd instanceof CreateBucketCommand)).toBe(true);
+ expect(svc.isS3Available()).toBe(true);
+ });
+
+ it('treats losing a concurrent CreateBucket race (BucketAlreadyOwnedByYou) as the bucket being ready', async () => {
+ mockSend.mockRejectedValueOnce(s3Error('NetworkingError'));
+ const svc = new StorageService(makeConfig());
+ warnSpyOf(svc);
+ await flush();
+
+ // Another replica created the bucket between this node's HeadBucket and its CreateBucket.
+ mockSend.mockImplementation((cmd: unknown) =>
+ Promise.reject(s3Error(cmd instanceof HeadBucketCommand ? 'NotFound' : 'BucketAlreadyOwnedByYou')),
+ );
+
+ await expect(svc.refreshS3Availability()).resolves.toBe(true);
+ expect(svc.isS3Available()).toBe(true);
+ });
+
+ it('stays on the local fallback when the bucket name is owned by another account', async () => {
+ mockSend.mockRejectedValueOnce(s3Error('NetworkingError'));
+ const svc = new StorageService(makeConfig());
+ warnSpyOf(svc);
+ await flush();
+
+ mockSend.mockImplementation((cmd: unknown) =>
+ Promise.reject(s3Error(cmd instanceof HeadBucketCommand ? 'NotFound' : 'BucketAlreadyExists')),
+ );
+
+ await expect(svc.refreshS3Availability()).resolves.toBe(false);
+ expect(svc.isS3Available()).toBe(false);
+ });
+
+ it('stays on the local fallback when the re-probe cannot create the missing bucket', async () => {
+ mockSend.mockRejectedValueOnce(s3Error('NetworkingError'));
+ const svc = new StorageService(makeConfig());
+ warnSpyOf(svc);
+ await flush();
+
+ mockSend.mockImplementation((cmd: unknown) =>
+ Promise.reject(s3Error(cmd instanceof HeadBucketCommand ? 'NoSuchBucket' : 'AccessDenied')),
+ );
+ await jest.advanceTimersByTimeAsync(DEFAULT_S3_REPROBE_INTERVAL_MS);
+
+ expect(svc.isS3Available()).toBe(false);
+ });
+
it('never transitions true→false: an S3-healthy boot ignores later transient probe failures', async () => {
mockSend.mockResolvedValue({}); // boot probe succeeds
const svc = new StorageService(makeConfig());
diff --git a/src/common/storage/storage.service.ts b/src/common/storage/storage.service.ts
index 10139d491..1baf13c69 100644
--- a/src/common/storage/storage.service.ts
+++ b/src/common/storage/storage.service.ts
@@ -134,26 +134,38 @@ export class StorageService implements OnModuleDestroy {
if (!this.s3Client) return;
try {
- await this.s3Client.send(new HeadBucketCommand({ Bucket: this.s3Bucket }));
+ await this.ensureS3Bucket();
this.s3Available = true;
this.logger.log(`S3 bucket '${this.s3Bucket}' is available`);
} catch (error: unknown) {
- const err = error as { name?: string };
- if (err.name === 'NotFound' || err.name === 'NoSuchBucket') {
- // Create bucket
- try {
- await this.s3Client.send(new CreateBucketCommand({ Bucket: this.s3Bucket }));
- this.s3Available = true;
- this.logger.log(`Created S3 bucket '${this.s3Bucket}'`);
- } catch (createError) {
- this.logger.error('Failed to create S3 bucket', String(createError));
- this.warnLocalFallback();
- }
- } else {
- this.logger.error('S3 bucket check failed', String(error));
- this.warnLocalFallback();
- }
+ this.logger.error('S3 bucket check failed', String(error));
+ this.warnLocalFallback();
+ }
+ }
+
+ /**
+ * HeadBucket, and create the bucket when the store answers that it does not exist. Throws when the
+ * store is unreachable or the create fails. The boot probe and every re-probe share it: a store that
+ * was down at boot and comes back empty (a fresh MinIO volume) must still get its bucket, or the
+ * re-probe would see NotFound forever and S3 would never become available without a restart.
+ */
+ private async ensureS3Bucket(): Promise<void> {
+ try {
+ await this.s3Client!.send(new HeadBucketCommand({ Bucket: this.s3Bucket }));
+ return;
+ } catch (error: unknown) {
+ const name = (error as { name?: string }).name;
+ if (name !== 'NotFound' && name !== 'NoSuchBucket') throw error;
+ }
+ try {
+ await this.s3Client!.send(new CreateBucketCommand({ Bucket: this.s3Bucket }));
+ } catch (error: unknown) {
+ // Another replica (or an overlapping probe) created it first, and this deployment owns it.
+ // BucketAlreadyExists means another account owns the name, so that one still throws.
+ if ((error as { name?: string }).name !== 'BucketAlreadyOwnedByYou') throw error;
+ return;
}
+ this.logger.log(`Created S3 bucket '${this.s3Bucket}'`);
}
private warnLocalFallback(): void {
@@ -222,7 +234,7 @@ export class StorageService implements OnModuleDestroy {
this.lastS3Check = now;
this.s3CheckInFlight = (async () => {
try {
- await this.s3Client!.send(new HeadBucketCommand({ Bucket: this.s3Bucket }));
+ await this.ensureS3Bucket();
this.s3Available = true;
// WARN (not log): the degraded window matters — media written to the local fallback dir while
// S3 was down stays local-only; reads for those keys keep working via the NoSuchKey