Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(storage): create a missing bucket when the S3 re-probe recovers
Yudhi Armyndharis36fc6fa40c78
diff --git a/src/common/storage/storage.service.s3-reprobe.spec.ts b/src/common/storage/storage.service.s3-reprobe.spec.tsindex 159b3007a..9b74da0c4 100644--- a/src/common/storage/storage.service.s3-reprobe.spec.ts+++ b/src/common/storage/storage.service.s3-reprobe.spec.ts@@ -25,7 +25,13 @@ jest.mock('@aws-sdk/client-s3', () => {};});-import { DeleteObjectCommand, GetObjectCommand, ListObjectsV2Command } from '@aws-sdk/client-s3';+import {+ CreateBucketCommand,+ DeleteObjectCommand,+ GetObjectCommand,+ HeadBucketCommand,+ ListObjectsV2Command,+} from '@aws-sdk/client-s3';import { DEFAULT_S3_REPROBE_INTERVAL_MS, StorageService } from './storage.service';const ENV_KEYS = [@@ -127,6 +133,66 @@ describe('StorageService S3 re-probe and recovery', () => {expect(svc.isS3Available()).toBe(true);});+ it('creates the bucket on re-probe when the store was down at boot and came back empty', async () => {+ mockSend.mockRejectedValueOnce(s3Error('NetworkingError')); // boot probe: store not up yet+ const svc = new StorageService(makeConfig());+ warnSpyOf(svc);+ await flush();+ expect(svc.isS3Available()).toBe(false);++ // The store is up now, but on a fresh volume: HeadBucket answers NotFound, CreateBucket succeeds.+ mockSend.mockImplementation((cmd: unknown) =>+ cmd instanceof HeadBucketCommand ? Promise.reject(s3Error('NotFound')) : Promise.resolve({}),+ );+ await jest.advanceTimersByTimeAsync(DEFAULT_S3_REPROBE_INTERVAL_MS);++ expect(mockSend.mock.calls.some(([cmd]) => cmd instanceof CreateBucketCommand)).toBe(true);+ expect(svc.isS3Available()).toBe(true);+ });++ it('treats losing a concurrent CreateBucket race (BucketAlreadyOwnedByYou) as the bucket being ready', async () => {+ mockSend.mockRejectedValueOnce(s3Error('NetworkingError'));+ const svc = new StorageService(makeConfig());+ warnSpyOf(svc);+ await flush();++ // Another replica created the bucket between this node's HeadBucket and its CreateBucket.+ mockSend.mockImplementation((cmd: unknown) =>+ Promise.reject(s3Error(cmd instanceof HeadBucketCommand ? 'NotFound' : 'BucketAlreadyOwnedByYou')),+ );++ await expect(svc.refreshS3Availability()).resolves.toBe(true);+ expect(svc.isS3Available()).toBe(true);+ });++ it('stays on the local fallback when the bucket name is owned by another account', async () => {+ mockSend.mockRejectedValueOnce(s3Error('NetworkingError'));+ const svc = new StorageService(makeConfig());+ warnSpyOf(svc);+ await flush();++ mockSend.mockImplementation((cmd: unknown) =>+ Promise.reject(s3Error(cmd instanceof HeadBucketCommand ? 'NotFound' : 'BucketAlreadyExists')),+ );++ await expect(svc.refreshS3Availability()).resolves.toBe(false);+ expect(svc.isS3Available()).toBe(false);+ });++ it('stays on the local fallback when the re-probe cannot create the missing bucket', async () => {+ mockSend.mockRejectedValueOnce(s3Error('NetworkingError'));+ const svc = new StorageService(makeConfig());+ warnSpyOf(svc);+ await flush();++ mockSend.mockImplementation((cmd: unknown) =>+ Promise.reject(s3Error(cmd instanceof HeadBucketCommand ? 'NoSuchBucket' : 'AccessDenied')),+ );+ await jest.advanceTimersByTimeAsync(DEFAULT_S3_REPROBE_INTERVAL_MS);++ expect(svc.isS3Available()).toBe(false);+ });+it('never transitions true→false: an S3-healthy boot ignores later transient probe failures', async () => {mockSend.mockResolvedValue({}); // boot probe succeedsconst svc = new StorageService(makeConfig());diff --git a/src/common/storage/storage.service.ts b/src/common/storage/storage.service.tsindex 10139d491..1baf13c69 100644--- a/src/common/storage/storage.service.ts+++ b/src/common/storage/storage.service.ts@@ -134,26 +134,38 @@ export class StorageService implements OnModuleDestroy {if (!this.s3Client) return;try {- await this.s3Client.send(new HeadBucketCommand({ Bucket: this.s3Bucket }));+ await this.ensureS3Bucket();this.s3Available = true;this.logger.log(`S3 bucket '${this.s3Bucket}' is available`);} catch (error: unknown) {- const err = error as { name?: string };- if (err.name === 'NotFound' || err.name === 'NoSuchBucket') {- // Create bucket- try {- await this.s3Client.send(new CreateBucketCommand({ Bucket: this.s3Bucket }));- this.s3Available = true;- this.logger.log(`Created S3 bucket '${this.s3Bucket}'`);- } catch (createError) {- this.logger.error('Failed to create S3 bucket', String(createError));- this.warnLocalFallback();- }- } else {- this.logger.error('S3 bucket check failed', String(error));- this.warnLocalFallback();- }+ this.logger.error('S3 bucket check failed', String(error));+ this.warnLocalFallback();+ }+ }++ /**+ * HeadBucket, and create the bucket when the store answers that it does not exist. Throws when the+ * store is unreachable or the create fails. The boot probe and every re-probe share it: a store that+ * was down at boot and comes back empty (a fresh MinIO volume) must still get its bucket, or the+ * re-probe would see NotFound forever and S3 would never become available without a restart.+ */+ private async ensureS3Bucket(): Promise<void> {+ try {+ await this.s3Client!.send(new HeadBucketCommand({ Bucket: this.s3Bucket }));+ return;+ } catch (error: unknown) {+ const name = (error as { name?: string }).name;+ if (name !== 'NotFound' && name !== 'NoSuchBucket') throw error;+ }+ try {+ await this.s3Client!.send(new CreateBucketCommand({ Bucket: this.s3Bucket }));+ } catch (error: unknown) {+ // Another replica (or an overlapping probe) created it first, and this deployment owns it.+ // BucketAlreadyExists means another account owns the name, so that one still throws.+ if ((error as { name?: string }).name !== 'BucketAlreadyOwnedByYou') throw error;+ return;}+ this.logger.log(`Created S3 bucket '${this.s3Bucket}'`);}private warnLocalFallback(): void {@@ -222,7 +234,7 @@ export class StorageService implements OnModuleDestroy {this.lastS3Check = now;this.s3CheckInFlight = (async () => {try {- await this.s3Client!.send(new HeadBucketCommand({ Bucket: this.s3Bucket }));+ await this.ensureS3Bucket();this.s3Available = true;// WARN (not log): the degraded window matters — media written to the local fallback dir while// S3 was down stays local-only; reads for those keys keep working via the NoSuchKey