Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(chat-media): keep the SQLite retention purge off a full table scan

Yudhi Armyndharis745096b01e85
diff --git a/src/modules/chat-media/chat-media-archive.service.spec.ts b/src/modules/chat-media/chat-media-archive.service.spec.ts
index 373b5d9f4..db6633924 100644
--- a/src/modules/chat-media/chat-media-archive.service.spec.ts
+++ b/src/modules/chat-media/chat-media-archive.service.spec.ts
@@ -424,6 +424,25 @@ describe('ChatMediaArchiveService', () => {
del.mockRestore();
}, 30_000);
+ it('does not plan the batch select as a walk of the whole table on SQLite', async () => {
+ // Ordering by the bare primary key lets SQLite satisfy ORDER BY from its autoindex and then
+ // visit every row in the table, which blocks the event loop on every tick once the backlog is
+ // drained. The plan must start from the createdAt range instead.
+ await seedExpired(1);
+ const runner = Object.getPrototypeOf(ds.createQueryRunner()) as {
+ query: (sql: string, params?: unknown[]) => Promise<unknown>;
+ };
+ const query = jest.spyOn(runner, 'query');
+
+ await enabled({ 'chatMedia.ttlDays': 7 }).purgeExpired(Date.now());
+
+ const select = query.mock.calls.find(([sql]) => /^SELECT/i.test(sql) && /LIMIT/i.test(sql));
+ query.mockRestore();
+ expect(select).toBeDefined();
+ const plan = await ds.query<{ detail: string }[]>(`EXPLAIN QUERY PLAN ${select![0]}`, select![1]);
+ expect(plan.map(p => p.detail).join('\n')).not.toMatch(/SCAN .*sqlite_autoindex_messages/);
+ });
+
it('stops instead of spinning when every delete in a batch fails', async () => {
await seedExpired(3);
const del = jest.spyOn(storageService, 'deleteFile').mockRejectedValue(new Error('s3 down'));
@@ -593,20 +612,20 @@ describe('ChatMediaArchiveService', () => {
.mockImplementationOnce(() => new Promise<void>(resolve => (releaseDelete = resolve)));
const svc = build({ 'chatMedia.ttlDays': 7 });
const sweep = jest.spyOn(svc, 'sweepOrphanedMedia').mockResolvedValue(0);
- const find = jest.spyOn(repository, 'find');
+ const select = jest.spyOn(repository, 'createQueryBuilder');
try {
svc.onModuleInit();
await jest.advanceTimersByTimeAsync(15 * 60 * 1000);
await jest.advanceTimersByTimeAsync(15 * 60 * 1000);
- expect(find).toHaveBeenCalledTimes(1);
+ expect(select).toHaveBeenCalledTimes(1);
releaseDelete();
await jest.advanceTimersByTimeAsync(15 * 60 * 1000);
- expect(find.mock.calls.length).toBeGreaterThan(1);
+ expect(select.mock.calls.length).toBeGreaterThan(1);
} finally {
svc.onModuleDestroy();
releaseDelete();
- [del, sweep, find].forEach(spy => spy.mockRestore());
+ [del, sweep, select].forEach(spy => spy.mockRestore());
jest.useRealTimers();
}
});
diff --git a/src/modules/chat-media/chat-media-archive.service.ts b/src/modules/chat-media/chat-media-archive.service.ts
index 8550d9220..7882a5d54 100644
--- a/src/modules/chat-media/chat-media-archive.service.ts
+++ b/src/modules/chat-media/chat-media-archive.service.ts
@@ -1,7 +1,7 @@
import { Injectable, OnModuleDestroy, OnModuleInit } from '@nestjs/common';
import { InjectRepository } from '@nestjs/typeorm';
import { ConfigService } from '@nestjs/config';
-import { In, IsNull, LessThan, MoreThan, Not, Repository } from 'typeorm';
+import { In, IsNull, Not, Repository } from 'typeorm';
import { randomUUID } from 'crypto';
import { Message } from '../message/entities/message.entity';
import { StorageService } from '../../common/storage/storage.service';
@@ -231,19 +231,25 @@ export class ChatMediaArchiveService implements OnModuleInit, OnModuleDestroy {
// so rows whose delete keeps failing are stepped over instead of being re-selected ahead of every
// newer row. A batch in which every delete fails ends the run: a missing file already counts as
// deleted, so that means the store is down, and walking on would only repeat the failure.
+ //
+ // On SQLite the walk orders by `+id`: a bare `id` lets the planner satisfy ORDER BY from the
+ // primary-key autoindex and visit every row of the table, even with nothing to purge, and
+ // better-sqlite3 runs that scan on the event loop. The unary plus starts the plan from the
+ // createdAt range instead. Postgres rejects unary plus on a uuid and plans this well as it is.
+ const isSqlite = ['sqlite', 'better-sqlite3'].includes(this.repository.manager.connection.options.type);
let after = this.purgeCursor;
this.purgeCursor = undefined;
for (let batch = 0; batch < PURGE_MAX_BATCHES_PER_RUN; batch++) {
- const expired = await this.repository.find({
- where: {
- mediaPath: Not(IsNull()),
- createdAt: LessThan(cutoff),
- ...(after ? { id: MoreThan(after) } : {}),
- },
- select: { id: true, mediaPath: true },
- order: { id: 'ASC' },
- take: PURGE_BATCH_SIZE,
- });
+ const qb = this.repository
+ .createQueryBuilder('m')
+ .select(['m.id', 'm.mediaPath'])
+ .where('m.mediaPath IS NOT NULL')
+ .andWhere('m.createdAt < :cutoff', { cutoff });
+ if (after) qb.andWhere('m.id > :after', { after });
+ const expired = await qb
+ .orderBy(isSqlite ? '+m.id' : 'm.id', 'ASC')
+ .limit(PURGE_BATCH_SIZE)
+ .getMany();
if (expired.length === 0) break;
after = expired[expired.length - 1].id;