Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(status-store): purge expired statuses in bounded batches
Yudhi Armyndharis91035b2a1b07
diff --git a/src/modules/chat-media/chat-media-archive.service.spec.ts b/src/modules/chat-media/chat-media-archive.service.spec.tsindex 3dd9b873b..61e614650 100644--- a/src/modules/chat-media/chat-media-archive.service.spec.ts+++ b/src/modules/chat-media/chat-media-archive.service.spec.ts@@ -342,6 +342,59 @@ describe('ChatMediaArchiveService', () => {update.mockRestore();}, 30_000);+ it('does not let a batch of undeletable files block newer expired rows', async () => {+ // Ids sort the undeletable rows first: a purge that restarted from the lowest id after an+ // all-failed batch would never reach the rows behind them.+ const id = (prefix: string, i: number): string => `${prefix}-0000-4000-8000-${String(i).padStart(12, '0')}`;+ const rows = [+ ...Array.from({ length: 510 }, (_, i) => ({+ id: id('00000000', i),+ mediaPath: `${CHAT_MEDIA_PREFIX}bad/${i}.png`,+ })),+ ...Array.from({ length: 5 }, (_, i) => ({+ id: id('ffffffff', i),+ mediaPath: `${CHAT_MEDIA_PREFIX}good/${i}.png`,+ })),+ ];+ for (let i = 0; i < rows.length; i += 100) {+ await repository.insert(+ rows.slice(i, i + 100).map(r => ({+ ...r,+ sessionId: 'sess-1',+ chatId: '[email protected]',+ waMessageId: r.id,+ from: '[email protected]',+ to: '[email protected]',+ body: '',+ type: 'image',+ direction: MessageDirection.INCOMING,+ status: MessageStatus.SENT,+ mediaMimetype: 'image/png',+ })),+ );+ }+ await repository.query('UPDATE messages SET createdAt = ?', [+ new Date(Date.now() - 10 * 24 * 60 * 60 * 1000).toISOString(),+ ]);+ const del = jest+ .spyOn(storageService, 'deleteFile')+ .mockImplementation(key => (key.includes('/bad/') ? Promise.reject(new Error('EACCES')) : Promise.resolve()));++ const svc = enabled({ 'chatMedia.ttlDays': 7 });++ // A batch where every delete fails ends the run instead of walking on through the backlog.+ expect(await svc.purgeExpired(Date.now())).toBe(0);+ expect(del.mock.calls.length).toBe(500);+ // The next run resumes after that batch.+ expect(await svc.purgeExpired(Date.now())).toBe(5);+ expect(del.mock.calls.length).toBe(515);+ expect(await repository.count({ where: { mediaPath: Not(IsNull()) } })).toBe(510);+ // Once the walk is drained, the next run starts over and retries the undeletable rows.+ await svc.purgeExpired(Date.now());+ expect(del.mock.calls[515][0]).toBe(`${CHAT_MEDIA_PREFIX}bad/0.png`);+ del.mockRestore();+ }, 30_000);+it('stops instead of spinning when every delete in a batch fails', async () => {await seedExpired(3);const del = jest.spyOn(storageService, 'deleteFile').mockRejectedValue(new Error('s3 down'));@@ -466,6 +519,37 @@ describe('ChatMediaArchiveService', () => {await expect(storageService.getFile(`${CHAT_MEDIA_PREFIX}sess-1/orphan.png`)).rejects.toThrow();});+ it('skips a purge tick while the previous purge is still running', async () => {+ const row = await saveRow(undefined, {+ mediaPath: `${CHAT_MEDIA_PREFIX}sess-1/f.png`,+ mediaMimetype: 'image/png',+ });+ await backdate(row.id, 10);+ jest.useFakeTimers({ doNotFake: ['setImmediate', 'nextTick'] });+ let releaseDelete: () => void = () => undefined;+ const del = jest+ .spyOn(storageService, 'deleteFile')+ .mockImplementationOnce(() => new Promise<void>(resolve => (releaseDelete = resolve)));+ const svc = build({ 'chatMedia.ttlDays': 7 });+ const sweep = jest.spyOn(svc, 'sweepOrphanedMedia').mockResolvedValue(0);+ const find = jest.spyOn(repository, 'find');+ try {+ svc.onModuleInit();+ await jest.advanceTimersByTimeAsync(15 * 60 * 1000);+ await jest.advanceTimersByTimeAsync(15 * 60 * 1000);+ expect(find).toHaveBeenCalledTimes(1);++ releaseDelete();+ await jest.advanceTimersByTimeAsync(15 * 60 * 1000);+ expect(find.mock.calls.length).toBeGreaterThan(1);+ } finally {+ svc.onModuleDestroy();+ releaseDelete();+ [del, sweep, find].forEach(spy => spy.mockRestore());+ jest.useRealTimers();+ }+ });+it('schedules both sweeps once archiving is on, and clears them on destroy', () => {const setInterval = jest.spyOn(global, 'setInterval');const clearInterval = jest.spyOn(global, 'clearInterval');diff --git a/src/modules/chat-media/chat-media-archive.service.ts b/src/modules/chat-media/chat-media-archive.service.tsindex 0a06598d6..26894ae42 100644--- a/src/modules/chat-media/chat-media-archive.service.ts+++ b/src/modules/chat-media/chat-media-archive.service.ts@@ -1,7 +1,7 @@import { Injectable, OnModuleDestroy, OnModuleInit } from '@nestjs/common';import { InjectRepository } from '@nestjs/typeorm';import { ConfigService } from '@nestjs/config';-import { In, IsNull, LessThan, Not, Repository } from 'typeorm';+import { In, IsNull, LessThan, MoreThan, Not, Repository } from 'typeorm';import { randomUUID } from 'crypto';import { Message } from '../message/entities/message.entity';import { StorageService } from '../../common/storage/storage.service';@@ -71,6 +71,10 @@ export class ChatMediaArchiveService implements OnModuleInit, OnModuleDestroy {private orphanSweepTimer?: ReturnType<typeof setInterval>;/** First sweep sighting (epoch ms) of each unreferenced archive file, for the grace window. */private readonly orphanFirstSeenAt = new Map<string, number>();+ /** Last row id the retention purge walked past; the next run resumes after it. */+ private purgeCursor?: string;+ /** A purge is still running, so the next tick is skipped rather than stacked on top of it. */+ private purging = false;constructor(@InjectRepository(Message, 'data')@@ -85,9 +89,11 @@ export class ChatMediaArchiveService implements OnModuleInit, OnModuleDestroy {onModuleInit(): void {const runPurge = (): void => {- this.purgeExpired(Date.now()).catch(err =>- this.logger.error('Chat media purge failed', err instanceof Error ? err.stack : String(err)),- );+ if (this.purging) return;+ this.purging = true;+ this.purgeExpired(Date.now())+ .catch(err => this.logger.error('Chat media purge failed', err instanceof Error ? err.stack : String(err)))+ .finally(() => (this.purging = false));};runPurge(); // sweep once at startupthis.purgeTimer = setInterval(runPurge, PURGE_INTERVAL_MS);@@ -214,20 +220,30 @@ export class ChatMediaArchiveService implements OnModuleInit, OnModuleDestroy {let cleared = 0;// Batched, not one pass. The archive's default TTL is 0 (keep forever), so the first run after- // an operator sets a retention can face an arbitrarily large backlog — unlike the status store,- // whose 24h TTL bounds every batch by construction. Unbatched that backlog broke three ways at- // once: tens of thousands of concurrent deletes, and a single `UPDATE ... WHERE id IN (...)`- // past the driver's bind-parameter ceiling (SQLite 32766, Postgres 65535) that throws AFTER the- // files are already gone — leaving rows pointing at deleted files, and every later tick- // repeating the same failure. Draining in bounded batches keeps each statement small and makes- // partial progress durable.+ // an operator sets a retention can face an arbitrarily large backlog. Unbatched that backlog+ // broke three ways at once: tens of thousands of concurrent deletes, and a single+ // `UPDATE ... WHERE id IN (...)` past the driver's bind-parameter ceiling (SQLite 32766, Postgres+ // 65535) that throws AFTER the files are already gone, leaving rows pointing at deleted files and+ // every later tick repeating the same failure. Draining in bounded batches keeps each statement+ // small and makes partial progress durable. The batches walk the rows by id, resuming across runs,+ // so rows whose delete keeps failing are stepped over instead of being re-selected ahead of every+ // newer row. A batch in which every delete fails ends the run: a missing file already counts as+ // deleted, so that means the store is down, and walking on would only repeat the failure.+ let after = this.purgeCursor;+ this.purgeCursor = undefined;for (let batch = 0; batch < PURGE_MAX_BATCHES_PER_RUN; batch++) {const expired = await this.repository.find({- where: { mediaPath: Not(IsNull()), createdAt: LessThan(cutoff) },+ where: {+ mediaPath: Not(IsNull()),+ createdAt: LessThan(cutoff),+ ...(after ? { id: MoreThan(after) } : {}),+ },select: { id: true, mediaPath: true },+ order: { id: 'ASC' },take: PURGE_BATCH_SIZE,});if (expired.length === 0) break;+ after = expired[expired.length - 1].id;const clearableIds: string[] = [];for (const row of expired) {@@ -241,18 +257,21 @@ export class ChatMediaArchiveService implements OnModuleInit, OnModuleDestroy {}}- // Every delete in the batch failed: the same rows would be re-selected forever, so stop- // rather than spin. The next scheduled run retries them.- if (clearableIds.length === 0) break;-+ if (clearableIds.length === 0) {+ this.purgeCursor = after;+ break;+ }+ // Rows whose delete failed keep their columns; they are retried once the walk wraps around.await this.repository.update(clearableIds, {mediaPath: null as unknown as undefined,mediaMimetype: null as unknown as undefined,});cleared += clearableIds.length;- // A short batch means the backlog is drained; anything else costs an extra empty query.+ // A short batch means the backlog is drained and the next run starts over from the lowest id;+ // a run that hits the batch cap resumes where it stopped.if (expired.length < PURGE_BATCH_SIZE) break;+ if (batch === PURGE_MAX_BATCHES_PER_RUN - 1) this.purgeCursor = after;}if (cleared > 0) this.logger.log(`Chat media retention purge cleared ${cleared} file(s)`);diff --git a/src/modules/status-store/status-store.service.spec.ts b/src/modules/status-store/status-store.service.spec.tsindex 0d064ab34..1444fcd29 100644--- a/src/modules/status-store/status-store.service.spec.ts+++ b/src/modules/status-store/status-store.service.spec.ts@@ -628,9 +628,8 @@ describe('StatusStoreService.purgeExpired', () => {// Every expired row has a media file and every delete fails, so nothing is deletable — an// unguarded delete([]) would throw TypeORM's empty-criteria error instead of returning 0.- const failingStorage = {- deleteFile: jest.fn().mockRejectedValue(new Error('backend down')),- } as unknown as StorageService;+ const deleteFile = jest.fn().mockRejectedValue(new Error('backend down'));+ const failingStorage = { deleteFile } as unknown as StorageService;const failingService = new StatusStoreService(repository, failingStorage, fakeConfigService());const now = 2000 + 24 * 60 * 60 * 1000 + 1;@@ -641,7 +640,77 @@ describe('StatusStoreService.purgeExpired', () => {expect(remaining.map(r => r.waStatusId).sort()).toEqual(['expired-a', 'expired-b']);expect(fs.existsSync(path.join(baseDir, 'media', a.mediaPath!))).toBe(true);expect(fs.existsSync(path.join(baseDir, 'media', b.mediaPath!))).toBe(true);- });+ // Tried once each, not re-selected in a loop.+ expect(deleteFile).toHaveBeenCalledTimes(2);+ });++ /** Seed rows cheaply, in statements well under SQLite's bind-parameter ceiling. */+ const seed = async (rows: Array<Partial<StatusUpdate>>): Promise<void> => {+ for (let i = 0; i < rows.length; i += 100) {+ await repository.insert(+ rows.slice(i, i + 100).map(r => ({+ sessionId: 'sess',+ contactJid: '[email protected]',+ waStatusId: r.id,+ type: 'text' as const,+ mediaOmitted: false,+ postedAt: 1000,+ expiresAt: 2000,+ ...r,+ })),+ );+ }+ };+ const id = (prefix: string, i: number): string => `${prefix}-0000-4000-8000-${String(i).padStart(12, '0')}`;++ it('drains a backlog in batches, never deleting more than 500 rows in one statement', async () => {+ await seed([+ ...Array.from({ length: 1201 }, (_, i) => ({ id: id('00000000', i) })),+ { id: id('ffffffff', 0), expiresAt: Date.now() + 60_000 },+ ]);+ const del = jest.spyOn(repository, 'delete');++ expect(await service.purgeExpired(Date.now())).toBe(1201);++ const sizes = del.mock.calls.map(([ids]) => (ids as string[]).length);+ expect(sizes).toEqual([500, 500, 201]);+ expect((await repository.find()).map(r => r.id)).toEqual([id('ffffffff', 0)]);+ del.mockRestore();+ }, 30_000);++ it('does not let a batch of undeletable media block newer expired rows', async () => {+ // Ids sort the undeletable rows first: a purge that restarted from the lowest id after an+ // all-failed batch would never reach the rows behind them.+ await seed([+ ...Array.from({ length: 510 }, (_, i) => ({+ id: id('00000000', i),+ type: 'image' as const,+ mediaPath: `bad/${i}`,+ })),+ ...Array.from({ length: 5 }, (_, i) => ({+ id: id('ffffffff', i),+ type: 'image' as const,+ mediaPath: `good/${i}`,+ })),+ ]);+ const deleteFile = jest.fn((key: string) =>+ key.startsWith('bad/') ? Promise.reject(new Error('EACCES')) : Promise.resolve(),+ );+ const storage = { deleteFile } as unknown as StorageService;++ const purger = new StatusStoreService(repository, storage, fakeConfigService());++ // A batch where every delete fails ends the run instead of walking on through the backlog.+ expect(await purger.purgeExpired(Date.now())).toBe(0);+ expect(deleteFile).toHaveBeenCalledTimes(500);+ // The next run resumes after that batch.+ expect(await purger.purgeExpired(Date.now())).toBe(5);+ expect(deleteFile).toHaveBeenCalledTimes(515);+ expect(await repository.count()).toBe(510);+ // Once the walk is drained, the next run starts over and retries the undeletable rows.+ await purger.purgeExpired(Date.now());+ expect(deleteFile.mock.calls[515][0]).toBe('bad/0');+ }, 30_000);});describe('StatusStoreService.sweepOrphanedMedia', () => {@@ -768,7 +837,7 @@ describe('StatusStoreService onModuleInit/onModuleDestroy (sweep scheduling)', (return { repo, storage, find };};- it('purges once at startup and schedules a recurring sweep, cleared on destroy', () => {+ it('purges once at startup and schedules a recurring sweep, cleared on destroy', async () => {const { repo, storage } = mockDeps();const service = new StatusStoreService(repo, storage, fakeConfigService());@@ -779,18 +848,44 @@ describe('StatusStoreService onModuleInit/onModuleDestroy (sweep scheduling)', (expect(purgeSpy).toHaveBeenCalledTimes(1);purgeSpy.mockClear();- jest.advanceTimersByTime(15 * 60 * 1000);+ await jest.advanceTimersByTimeAsync(15 * 60 * 1000);expect(purgeSpy).toHaveBeenCalledTimes(1);service.onModuleDestroy();purgeSpy.mockClear();- jest.advanceTimersByTime(15 * 60 * 1000);+ await jest.advanceTimersByTimeAsync(15 * 60 * 1000);expect(purgeSpy).not.toHaveBeenCalled();} finally {jest.useRealTimers();}});+ it('skips a purge tick while the previous purge is still running', async () => {+ const { repo, storage } = mockDeps();+ const service = new StatusStoreService(repo, storage, fakeConfigService());++ jest.useFakeTimers();+ let release: () => void = () => undefined;+ const purgeSpy = jest+ .spyOn(service, 'purgeExpired')+ .mockImplementationOnce(() => new Promise<number>(resolve => (release = () => resolve(0))))+ .mockResolvedValue(0);+ try {+ service.onModuleInit();+ await jest.advanceTimersByTimeAsync(15 * 60 * 1000);+ await jest.advanceTimersByTimeAsync(15 * 60 * 1000);+ expect(purgeSpy).toHaveBeenCalledTimes(1);++ release();+ await jest.advanceTimersByTimeAsync(15 * 60 * 1000);+ expect(purgeSpy).toHaveBeenCalledTimes(2);+ } finally {+ service.onModuleDestroy();+ release();+ jest.useRealTimers();+ }+ });+it('runs the orphan sweep once at startup and on its own (slower) cadence, cleared on destroy', () => {const { repo, storage } = mockDeps();const service = new StatusStoreService(repo, storage, fakeConfigService());diff --git a/src/modules/status-store/status-store.service.ts b/src/modules/status-store/status-store.service.tsindex 33764050d..91e193ae8 100644--- a/src/modules/status-store/status-store.service.ts+++ b/src/modules/status-store/status-store.service.ts@@ -18,6 +18,12 @@ import { createLogger } from '../../common/services/logger.service';export const STATUS_TTL_MS = 24 * 60 * 60 * 1000;/** How often the TTL purge sweeps expired rows. */const PURGE_INTERVAL_MS = 15 * 60 * 1000;+/** Rows removed per purge statement. Keeps the id list far below the drivers' bind-parameter+ * ceilings (SQLite 32766, Postgres 65535) and bounds how much work one failure can undo. */+const PURGE_BATCH_SIZE = 500;+/** Cap on batches per run, so one tick cannot monopolise the process draining a huge backlog;+ * whatever is left is picked up by the next scheduled run. */+const PURGE_MAX_BATCHES_PER_RUN = 200;/** Default per-file cap on persisted status media. Exported for the session service's seed, which* pre-gates history downloads at the same cap so over-cap blobs are never fetched. */export const DEFAULT_MEDIA_MAX_BYTES = 10 * 1024 * 1024;@@ -54,6 +60,10 @@ export class StatusStoreService implements OnModuleInit, OnModuleDestroy {private orphanSweepTimer?: ReturnType<typeof setInterval>;/** First sweep sighting (epoch ms) of each unreferenced status media file, for the grace window. */private readonly orphanFirstSeenAt = new Map<string, number>();+ /** Last row id the TTL purge walked past; the next run resumes after it. */+ private purgeCursor?: string;+ /** A purge is still running, so the next tick is skipped rather than stacked on top of it. */+ private purging = false;constructor(@InjectRepository(StatusUpdate, 'data')@@ -67,9 +77,11 @@ export class StatusStoreService implements OnModuleInit, OnModuleDestroy {onModuleInit(): void {const runPurge = (): void => {- this.purgeExpired(Date.now()).catch(err =>- this.logger.error('Status purge failed', err instanceof Error ? err.stack : String(err)),- );+ if (this.purging) return;+ this.purging = true;+ this.purgeExpired(Date.now())+ .catch(err => this.logger.error('Status purge failed', err instanceof Error ? err.stack : String(err)))+ .finally(() => (this.purging = false));};runPurge(); // sweep once at startupthis.purgeTimer = setInterval(runPurge, PURGE_INTERVAL_MS);@@ -309,36 +321,55 @@ export class StatusStoreService implements OnModuleInit, OnModuleDestroy {* file is only rediscovered by the orphan sweep after its grace window), so the row stays and the* next sweep retries the delete. A missing file is a successful delete (see StorageService), so an* already-gone file never wedges its row.+ *+ * Drained in bounded batches. The 24h TTL does not bound a backlog: a restart after downtime, or+ * media deletes that keep failing, leave any number of expired rows, and a single+ * `DELETE ... WHERE id IN (...)` past the driver's bind-parameter ceiling throws after the files are+ * gone, then fails the same way on every later run. The batches walk the expired rows by id, resuming+ * across runs, so rows whose delete keeps failing are stepped over instead of blocking newer ones. A+ * batch in which every delete fails ends the run: that means the store is down, and walking on would+ * only repeat the failure.*/async purgeExpired(now: number): Promise<number> {- const expired = await this.repository.find({ where: { expiresAt: LessThan(now) } });- if (expired.length === 0) return 0;+ let removed = 0;+ let after = this.purgeCursor;+ this.purgeCursor = undefined;+ for (let batch = 0; batch < PURGE_MAX_BATCHES_PER_RUN; batch++) {+ const expired = await this.repository.find({+ where: { expiresAt: LessThan(now), ...(after ? { id: MoreThan(after) } : {}) },+ select: { id: true, mediaPath: true },+ order: { id: 'ASC' },+ take: PURGE_BATCH_SIZE,+ });+ if (expired.length === 0) break;+ after = expired[expired.length - 1].id;- const deletableIds: string[] = [];- await Promise.all(- expired.map(async row => {- if (!row.mediaPath) {- deletableIds.push(row.id);- return;- }+ const deletableIds: string[] = [];+ for (const row of expired) {try {- await this.storageService.deleteFile(row.mediaPath);+ if (row.mediaPath) await this.storageService.deleteFile(row.mediaPath);deletableIds.push(row.id);} catch (err) {this.logger.warn(`Failed to delete expired status media ${row.mediaPath}; keeping the row for the next sweep`,- {- error: String(err),- },+ { error: String(err) },);}- }),- );+ }- // Every media delete may have failed — delete([]) throws TypeORM's empty-criteria error.- if (deletableIds.length === 0) return 0;- const result = await this.repository.delete(deletableIds);- return result.affected ?? deletableIds.length;+ // Rows whose delete failed are kept; they are retried once the walk wraps around.+ if (deletableIds.length === 0) {+ this.purgeCursor = after;+ break;+ }+ const result = await this.repository.delete(deletableIds);+ removed += result.affected ?? deletableIds.length;+ // A short batch means the backlog is drained and the next run starts over from the lowest id;+ // a run that hits the batch cap resumes where it stopped.+ if (expired.length < PURGE_BATCH_SIZE) break;+ if (batch === PURGE_MAX_BATCHES_PER_RUN - 1) this.purgeCursor = after;+ }+ return removed;}/**