Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(status-store): purge expired statuses in bounded batches

Yudhi Armyndharis91035b2a1b07
diff --git a/src/modules/chat-media/chat-media-archive.service.spec.ts b/src/modules/chat-media/chat-media-archive.service.spec.ts
index 3dd9b873b..61e614650 100644
--- a/src/modules/chat-media/chat-media-archive.service.spec.ts
+++ b/src/modules/chat-media/chat-media-archive.service.spec.ts
@@ -342,6 +342,59 @@ describe('ChatMediaArchiveService', () => {
update.mockRestore();
}, 30_000);
+ it('does not let a batch of undeletable files block newer expired rows', async () => {
+ // Ids sort the undeletable rows first: a purge that restarted from the lowest id after an
+ // all-failed batch would never reach the rows behind them.
+ const id = (prefix: string, i: number): string => `${prefix}-0000-4000-8000-${String(i).padStart(12, '0')}`;
+ const rows = [
+ ...Array.from({ length: 510 }, (_, i) => ({
+ id: id('00000000', i),
+ mediaPath: `${CHAT_MEDIA_PREFIX}bad/${i}.png`,
+ })),
+ ...Array.from({ length: 5 }, (_, i) => ({
+ id: id('ffffffff', i),
+ mediaPath: `${CHAT_MEDIA_PREFIX}good/${i}.png`,
+ })),
+ ];
+ for (let i = 0; i < rows.length; i += 100) {
+ await repository.insert(
+ rows.slice(i, i + 100).map(r => ({
+ ...r,
+ sessionId: 'sess-1',
+ chatId: '[email protected]',
+ waMessageId: r.id,
+ body: '',
+ type: 'image',
+ direction: MessageDirection.INCOMING,
+ status: MessageStatus.SENT,
+ mediaMimetype: 'image/png',
+ })),
+ );
+ }
+ await repository.query('UPDATE messages SET createdAt = ?', [
+ new Date(Date.now() - 10 * 24 * 60 * 60 * 1000).toISOString(),
+ ]);
+ const del = jest
+ .spyOn(storageService, 'deleteFile')
+ .mockImplementation(key => (key.includes('/bad/') ? Promise.reject(new Error('EACCES')) : Promise.resolve()));
+
+ const svc = enabled({ 'chatMedia.ttlDays': 7 });
+
+ // A batch where every delete fails ends the run instead of walking on through the backlog.
+ expect(await svc.purgeExpired(Date.now())).toBe(0);
+ expect(del.mock.calls.length).toBe(500);
+ // The next run resumes after that batch.
+ expect(await svc.purgeExpired(Date.now())).toBe(5);
+ expect(del.mock.calls.length).toBe(515);
+ expect(await repository.count({ where: { mediaPath: Not(IsNull()) } })).toBe(510);
+ // Once the walk is drained, the next run starts over and retries the undeletable rows.
+ await svc.purgeExpired(Date.now());
+ expect(del.mock.calls[515][0]).toBe(`${CHAT_MEDIA_PREFIX}bad/0.png`);
+ del.mockRestore();
+ }, 30_000);
+
it('stops instead of spinning when every delete in a batch fails', async () => {
await seedExpired(3);
const del = jest.spyOn(storageService, 'deleteFile').mockRejectedValue(new Error('s3 down'));
@@ -466,6 +519,37 @@ describe('ChatMediaArchiveService', () => {
await expect(storageService.getFile(`${CHAT_MEDIA_PREFIX}sess-1/orphan.png`)).rejects.toThrow();
});
+ it('skips a purge tick while the previous purge is still running', async () => {
+ const row = await saveRow(undefined, {
+ mediaPath: `${CHAT_MEDIA_PREFIX}sess-1/f.png`,
+ mediaMimetype: 'image/png',
+ });
+ await backdate(row.id, 10);
+ jest.useFakeTimers({ doNotFake: ['setImmediate', 'nextTick'] });
+ let releaseDelete: () => void = () => undefined;
+ const del = jest
+ .spyOn(storageService, 'deleteFile')
+ .mockImplementationOnce(() => new Promise<void>(resolve => (releaseDelete = resolve)));
+ const svc = build({ 'chatMedia.ttlDays': 7 });
+ const sweep = jest.spyOn(svc, 'sweepOrphanedMedia').mockResolvedValue(0);
+ const find = jest.spyOn(repository, 'find');
+ try {
+ svc.onModuleInit();
+ await jest.advanceTimersByTimeAsync(15 * 60 * 1000);
+ await jest.advanceTimersByTimeAsync(15 * 60 * 1000);
+ expect(find).toHaveBeenCalledTimes(1);
+
+ releaseDelete();
+ await jest.advanceTimersByTimeAsync(15 * 60 * 1000);
+ expect(find.mock.calls.length).toBeGreaterThan(1);
+ } finally {
+ svc.onModuleDestroy();
+ releaseDelete();
+ [del, sweep, find].forEach(spy => spy.mockRestore());
+ jest.useRealTimers();
+ }
+ });
+
it('schedules both sweeps once archiving is on, and clears them on destroy', () => {
const setInterval = jest.spyOn(global, 'setInterval');
const clearInterval = jest.spyOn(global, 'clearInterval');
diff --git a/src/modules/chat-media/chat-media-archive.service.ts b/src/modules/chat-media/chat-media-archive.service.ts
index 0a06598d6..26894ae42 100644
--- a/src/modules/chat-media/chat-media-archive.service.ts
+++ b/src/modules/chat-media/chat-media-archive.service.ts
@@ -1,7 +1,7 @@
import { Injectable, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { ConfigService } from '@nestjs/config';
-import { In, IsNull, LessThan, Not, Repository } from 'typeorm';
+import { In, IsNull, LessThan, MoreThan, Not, Repository } from 'typeorm';
import { randomUUID } from 'crypto';
import { Message } from '../message/entities/message.entity';
import { StorageService } from '../../common/storage/storage.service';
@@ -71,6 +71,10 @@ export class ChatMediaArchiveService implements OnModuleInit, OnModuleDestroy {
private orphanSweepTimer?: ReturnType<typeof setInterval>;
/** First sweep sighting (epoch ms) of each unreferenced archive file, for the grace window. */
private readonly orphanFirstSeenAt = new Map<string, number>();
+ /** Last row id the retention purge walked past; the next run resumes after it. */
+ private purgeCursor?: string;
+ /** A purge is still running, so the next tick is skipped rather than stacked on top of it. */
+ private purging = false;
constructor(
@InjectRepository(Message, 'data')
@@ -85,9 +89,11 @@ export class ChatMediaArchiveService implements OnModuleInit, OnModuleDestroy {
onModuleInit(): void {
const runPurge = (): void => {
- this.purgeExpired(Date.now()).catch(err =>
- this.logger.error('Chat media purge failed', err instanceof Error ? err.stack : String(err)),
- );
+ if (this.purging) return;
+ this.purging = true;
+ this.purgeExpired(Date.now())
+ .catch(err => this.logger.error('Chat media purge failed', err instanceof Error ? err.stack : String(err)))
+ .finally(() => (this.purging = false));
};
runPurge(); // sweep once at startup
this.purgeTimer = setInterval(runPurge, PURGE_INTERVAL_MS);
@@ -214,20 +220,30 @@ export class ChatMediaArchiveService implements OnModuleInit, OnModuleDestroy {
let cleared = 0;
// Batched, not one pass. The archive's default TTL is 0 (keep forever), so the first run after
- // an operator sets a retention can face an arbitrarily large backlog — unlike the status store,
- // whose 24h TTL bounds every batch by construction. Unbatched that backlog broke three ways at
- // once: tens of thousands of concurrent deletes, and a single `UPDATE ... WHERE id IN (...)`
- // past the driver's bind-parameter ceiling (SQLite 32766, Postgres 65535) that throws AFTER the
- // files are already gone — leaving rows pointing at deleted files, and every later tick
- // repeating the same failure. Draining in bounded batches keeps each statement small and makes
- // partial progress durable.
+ // an operator sets a retention can face an arbitrarily large backlog. Unbatched that backlog
+ // broke three ways at once: tens of thousands of concurrent deletes, and a single
+ // `UPDATE ... WHERE id IN (...)` past the driver's bind-parameter ceiling (SQLite 32766, Postgres
+ // 65535) that throws AFTER the files are already gone, leaving rows pointing at deleted files and
+ // every later tick repeating the same failure. Draining in bounded batches keeps each statement
+ // small and makes partial progress durable. The batches walk the rows by id, resuming across runs,
+ // so rows whose delete keeps failing are stepped over instead of being re-selected ahead of every
+ // newer row. A batch in which every delete fails ends the run: a missing file already counts as
+ // deleted, so that means the store is down, and walking on would only repeat the failure.
+ let after = this.purgeCursor;
+ this.purgeCursor = undefined;
for (let batch = 0; batch < PURGE_MAX_BATCHES_PER_RUN; batch++) {
const expired = await this.repository.find({
- where: { mediaPath: Not(IsNull()), createdAt: LessThan(cutoff) },
+ where: {
+ mediaPath: Not(IsNull()),
+ createdAt: LessThan(cutoff),
+ ...(after ? { id: MoreThan(after) } : {}),
+ },
select: { id: true, mediaPath: true },
+ order: { id: 'ASC' },
take: PURGE_BATCH_SIZE,
});
if (expired.length === 0) break;
+ after = expired[expired.length - 1].id;
const clearableIds: string[] = [];
for (const row of expired) {
@@ -241,18 +257,21 @@ export class ChatMediaArchiveService implements OnModuleInit, OnModuleDestroy {
}
}
- // Every delete in the batch failed: the same rows would be re-selected forever, so stop
- // rather than spin. The next scheduled run retries them.
- if (clearableIds.length === 0) break;
-
+ if (clearableIds.length === 0) {
+ this.purgeCursor = after;
+ break;
+ }
+ // Rows whose delete failed keep their columns; they are retried once the walk wraps around.
await this.repository.update(clearableIds, {
mediaPath: null as unknown as undefined,
mediaMimetype: null as unknown as undefined,
});
cleared += clearableIds.length;
- // A short batch means the backlog is drained; anything else costs an extra empty query.
+ // A short batch means the backlog is drained and the next run starts over from the lowest id;
+ // a run that hits the batch cap resumes where it stopped.
if (expired.length < PURGE_BATCH_SIZE) break;
+ if (batch === PURGE_MAX_BATCHES_PER_RUN - 1) this.purgeCursor = after;
}
if (cleared > 0) this.logger.log(`Chat media retention purge cleared ${cleared} file(s)`);
diff --git a/src/modules/status-store/status-store.service.spec.ts b/src/modules/status-store/status-store.service.spec.ts
index 0d064ab34..1444fcd29 100644
--- a/src/modules/status-store/status-store.service.spec.ts
+++ b/src/modules/status-store/status-store.service.spec.ts
@@ -628,9 +628,8 @@ describe('StatusStoreService.purgeExpired', () => {
// Every expired row has a media file and every delete fails, so nothing is deletable — an
// unguarded delete([]) would throw TypeORM's empty-criteria error instead of returning 0.
- const failingStorage = {
- deleteFile: jest.fn().mockRejectedValue(new Error('backend down')),
- } as unknown as StorageService;
+ const deleteFile = jest.fn().mockRejectedValue(new Error('backend down'));
+ const failingStorage = { deleteFile } as unknown as StorageService;
const failingService = new StatusStoreService(repository, failingStorage, fakeConfigService());
const now = 2000 + 24 * 60 * 60 * 1000 + 1;
@@ -641,7 +640,77 @@ describe('StatusStoreService.purgeExpired', () => {
expect(remaining.map(r => r.waStatusId).sort()).toEqual(['expired-a', 'expired-b']);
expect(fs.existsSync(path.join(baseDir, 'media', a.mediaPath!))).toBe(true);
expect(fs.existsSync(path.join(baseDir, 'media', b.mediaPath!))).toBe(true);
- });
+ // Tried once each, not re-selected in a loop.
+ expect(deleteFile).toHaveBeenCalledTimes(2);
+ });
+
+ /** Seed rows cheaply, in statements well under SQLite's bind-parameter ceiling. */
+ const seed = async (rows: Array<Partial<StatusUpdate>>): Promise<void> => {
+ for (let i = 0; i < rows.length; i += 100) {
+ await repository.insert(
+ rows.slice(i, i + 100).map(r => ({
+ sessionId: 'sess',
+ contactJid: '[email protected]',
+ waStatusId: r.id,
+ type: 'text' as const,
+ mediaOmitted: false,
+ postedAt: 1000,
+ expiresAt: 2000,
+ ...r,
+ })),
+ );
+ }
+ };
+ const id = (prefix: string, i: number): string => `${prefix}-0000-4000-8000-${String(i).padStart(12, '0')}`;
+
+ it('drains a backlog in batches, never deleting more than 500 rows in one statement', async () => {
+ await seed([
+ ...Array.from({ length: 1201 }, (_, i) => ({ id: id('00000000', i) })),
+ { id: id('ffffffff', 0), expiresAt: Date.now() + 60_000 },
+ ]);
+ const del = jest.spyOn(repository, 'delete');
+
+ expect(await service.purgeExpired(Date.now())).toBe(1201);
+
+ const sizes = del.mock.calls.map(([ids]) => (ids as string[]).length);
+ expect(sizes).toEqual([500, 500, 201]);
+ expect((await repository.find()).map(r => r.id)).toEqual([id('ffffffff', 0)]);
+ del.mockRestore();
+ }, 30_000);
+
+ it('does not let a batch of undeletable media block newer expired rows', async () => {
+ // Ids sort the undeletable rows first: a purge that restarted from the lowest id after an
+ // all-failed batch would never reach the rows behind them.
+ await seed([
+ ...Array.from({ length: 510 }, (_, i) => ({
+ id: id('00000000', i),
+ type: 'image' as const,
+ mediaPath: `bad/${i}`,
+ })),
+ ...Array.from({ length: 5 }, (_, i) => ({
+ id: id('ffffffff', i),
+ type: 'image' as const,
+ mediaPath: `good/${i}`,
+ })),
+ ]);
+ const deleteFile = jest.fn((key: string) =>
+ key.startsWith('bad/') ? Promise.reject(new Error('EACCES')) : Promise.resolve(),
+ );
+ const storage = { deleteFile } as unknown as StorageService;
+
+ const purger = new StatusStoreService(repository, storage, fakeConfigService());
+
+ // A batch where every delete fails ends the run instead of walking on through the backlog.
+ expect(await purger.purgeExpired(Date.now())).toBe(0);
+ expect(deleteFile).toHaveBeenCalledTimes(500);
+ // The next run resumes after that batch.
+ expect(await purger.purgeExpired(Date.now())).toBe(5);
+ expect(deleteFile).toHaveBeenCalledTimes(515);
+ expect(await repository.count()).toBe(510);
+ // Once the walk is drained, the next run starts over and retries the undeletable rows.
+ await purger.purgeExpired(Date.now());
+ expect(deleteFile.mock.calls[515][0]).toBe('bad/0');
+ }, 30_000);
});
describe('StatusStoreService.sweepOrphanedMedia', () => {
@@ -768,7 +837,7 @@ describe('StatusStoreService onModuleInit/onModuleDestroy (sweep scheduling)', (
return { repo, storage, find };
};
- it('purges once at startup and schedules a recurring sweep, cleared on destroy', () => {
+ it('purges once at startup and schedules a recurring sweep, cleared on destroy', async () => {
const { repo, storage } = mockDeps();
const service = new StatusStoreService(repo, storage, fakeConfigService());
@@ -779,18 +848,44 @@ describe('StatusStoreService onModuleInit/onModuleDestroy (sweep scheduling)', (
expect(purgeSpy).toHaveBeenCalledTimes(1);
purgeSpy.mockClear();
- jest.advanceTimersByTime(15 * 60 * 1000);
+ await jest.advanceTimersByTimeAsync(15 * 60 * 1000);
expect(purgeSpy).toHaveBeenCalledTimes(1);
service.onModuleDestroy();
purgeSpy.mockClear();
- jest.advanceTimersByTime(15 * 60 * 1000);
+ await jest.advanceTimersByTimeAsync(15 * 60 * 1000);
expect(purgeSpy).not.toHaveBeenCalled();
} finally {
jest.useRealTimers();
}
});
+ it('skips a purge tick while the previous purge is still running', async () => {
+ const { repo, storage } = mockDeps();
+ const service = new StatusStoreService(repo, storage, fakeConfigService());
+
+ jest.useFakeTimers();
+ let release: () => void = () => undefined;
+ const purgeSpy = jest
+ .spyOn(service, 'purgeExpired')
+ .mockImplementationOnce(() => new Promise<number>(resolve => (release = () => resolve(0))))
+ .mockResolvedValue(0);
+ try {
+ service.onModuleInit();
+ await jest.advanceTimersByTimeAsync(15 * 60 * 1000);
+ await jest.advanceTimersByTimeAsync(15 * 60 * 1000);
+ expect(purgeSpy).toHaveBeenCalledTimes(1);
+
+ release();
+ await jest.advanceTimersByTimeAsync(15 * 60 * 1000);
+ expect(purgeSpy).toHaveBeenCalledTimes(2);
+ } finally {
+ service.onModuleDestroy();
+ release();
+ jest.useRealTimers();
+ }
+ });
+
it('runs the orphan sweep once at startup and on its own (slower) cadence, cleared on destroy', () => {
const { repo, storage } = mockDeps();
const service = new StatusStoreService(repo, storage, fakeConfigService());
diff --git a/src/modules/status-store/status-store.service.ts b/src/modules/status-store/status-store.service.ts
index 33764050d..91e193ae8 100644
--- a/src/modules/status-store/status-store.service.ts
+++ b/src/modules/status-store/status-store.service.ts
@@ -18,6 +18,12 @@ import { createLogger } from '../../common/services/logger.service';
export const STATUS_TTL_MS = 24 * 60 * 60 * 1000;
/** How often the TTL purge sweeps expired rows. */
const PURGE_INTERVAL_MS = 15 * 60 * 1000;
+/** Rows removed per purge statement. Keeps the id list far below the drivers' bind-parameter
+ * ceilings (SQLite 32766, Postgres 65535) and bounds how much work one failure can undo. */
+const PURGE_BATCH_SIZE = 500;
+/** Cap on batches per run, so one tick cannot monopolise the process draining a huge backlog;
+ * whatever is left is picked up by the next scheduled run. */
+const PURGE_MAX_BATCHES_PER_RUN = 200;
/** Default per-file cap on persisted status media. Exported for the session service's seed, which
* pre-gates history downloads at the same cap so over-cap blobs are never fetched. */
export const DEFAULT_MEDIA_MAX_BYTES = 10 * 1024 * 1024;
@@ -54,6 +60,10 @@ export class StatusStoreService implements OnModuleInit, OnModuleDestroy {
private orphanSweepTimer?: ReturnType<typeof setInterval>;
/** First sweep sighting (epoch ms) of each unreferenced status media file, for the grace window. */
private readonly orphanFirstSeenAt = new Map<string, number>();
+ /** Last row id the TTL purge walked past; the next run resumes after it. */
+ private purgeCursor?: string;
+ /** A purge is still running, so the next tick is skipped rather than stacked on top of it. */
+ private purging = false;
constructor(
@InjectRepository(StatusUpdate, 'data')
@@ -67,9 +77,11 @@ export class StatusStoreService implements OnModuleInit, OnModuleDestroy {
onModuleInit(): void {
const runPurge = (): void => {
- this.purgeExpired(Date.now()).catch(err =>
- this.logger.error('Status purge failed', err instanceof Error ? err.stack : String(err)),
- );
+ if (this.purging) return;
+ this.purging = true;
+ this.purgeExpired(Date.now())
+ .catch(err => this.logger.error('Status purge failed', err instanceof Error ? err.stack : String(err)))
+ .finally(() => (this.purging = false));
};
runPurge(); // sweep once at startup
this.purgeTimer = setInterval(runPurge, PURGE_INTERVAL_MS);
@@ -309,36 +321,55 @@ export class StatusStoreService implements OnModuleInit, OnModuleDestroy {
* file is only rediscovered by the orphan sweep after its grace window), so the row stays and the
* next sweep retries the delete. A missing file is a successful delete (see StorageService), so an
* already-gone file never wedges its row.
+ *
+ * Drained in bounded batches. The 24h TTL does not bound a backlog: a restart after downtime, or
+ * media deletes that keep failing, leave any number of expired rows, and a single
+ * `DELETE ... WHERE id IN (...)` past the driver's bind-parameter ceiling throws after the files are
+ * gone, then fails the same way on every later run. The batches walk the expired rows by id, resuming
+ * across runs, so rows whose delete keeps failing are stepped over instead of blocking newer ones. A
+ * batch in which every delete fails ends the run: that means the store is down, and walking on would
+ * only repeat the failure.
*/
async purgeExpired(now: number): Promise<number> {
- const expired = await this.repository.find({ where: { expiresAt: LessThan(now) } });
- if (expired.length === 0) return 0;
+ let removed = 0;
+ let after = this.purgeCursor;
+ this.purgeCursor = undefined;
+ for (let batch = 0; batch < PURGE_MAX_BATCHES_PER_RUN; batch++) {
+ const expired = await this.repository.find({
+ where: { expiresAt: LessThan(now), ...(after ? { id: MoreThan(after) } : {}) },
+ select: { id: true, mediaPath: true },
+ order: { id: 'ASC' },
+ take: PURGE_BATCH_SIZE,
+ });
+ if (expired.length === 0) break;
+ after = expired[expired.length - 1].id;
- const deletableIds: string[] = [];
- await Promise.all(
- expired.map(async row => {
- if (!row.mediaPath) {
- deletableIds.push(row.id);
- return;
- }
+ const deletableIds: string[] = [];
+ for (const row of expired) {
try {
- await this.storageService.deleteFile(row.mediaPath);
+ if (row.mediaPath) await this.storageService.deleteFile(row.mediaPath);
deletableIds.push(row.id);
} catch (err) {
this.logger.warn(
`Failed to delete expired status media ${row.mediaPath}; keeping the row for the next sweep`,
- {
- error: String(err),
- },
+ { error: String(err) },
);
}
- }),
- );
+ }
- // Every media delete may have failed — delete([]) throws TypeORM's empty-criteria error.
- if (deletableIds.length === 0) return 0;
- const result = await this.repository.delete(deletableIds);
- return result.affected ?? deletableIds.length;
+ // Rows whose delete failed are kept; they are retried once the walk wraps around.
+ if (deletableIds.length === 0) {
+ this.purgeCursor = after;
+ break;
+ }
+ const result = await this.repository.delete(deletableIds);
+ removed += result.affected ?? deletableIds.length;
+ // A short batch means the backlog is drained and the next run starts over from the lowest id;
+ // a run that hits the batch cap resumes where it stopped.
+ if (expired.length < PURGE_BATCH_SIZE) break;
+ if (batch === PURGE_MAX_BATCHES_PER_RUN - 1) this.purgeCursor = after;
+ }
+ return removed;
}
/**