Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(chat-media): keep the SQLite retention purge off a full table scan
Yudhi Armyndharis745096b01e85
diff --git a/src/modules/chat-media/chat-media-archive.service.spec.ts b/src/modules/chat-media/chat-media-archive.service.spec.tsindex 373b5d9f4..db6633924 100644--- a/src/modules/chat-media/chat-media-archive.service.spec.ts+++ b/src/modules/chat-media/chat-media-archive.service.spec.ts@@ -424,6 +424,25 @@ describe('ChatMediaArchiveService', () => {del.mockRestore();}, 30_000);+ it('does not plan the batch select as a walk of the whole table on SQLite', async () => {+ // Ordering by the bare primary key lets SQLite satisfy ORDER BY from its autoindex and then+ // visit every row in the table, which blocks the event loop on every tick once the backlog is+ // drained. The plan must start from the createdAt range instead.+ await seedExpired(1);+ const runner = Object.getPrototypeOf(ds.createQueryRunner()) as {+ query: (sql: string, params?: unknown[]) => Promise<unknown>;+ };+ const query = jest.spyOn(runner, 'query');++ await enabled({ 'chatMedia.ttlDays': 7 }).purgeExpired(Date.now());++ const select = query.mock.calls.find(([sql]) => /^SELECT/i.test(sql) && /LIMIT/i.test(sql));+ query.mockRestore();+ expect(select).toBeDefined();+ const plan = await ds.query<{ detail: string }[]>(`EXPLAIN QUERY PLAN ${select![0]}`, select![1]);+ expect(plan.map(p => p.detail).join('\n')).not.toMatch(/SCAN .*sqlite_autoindex_messages/);+ });+it('stops instead of spinning when every delete in a batch fails', async () => {await seedExpired(3);const del = jest.spyOn(storageService, 'deleteFile').mockRejectedValue(new Error('s3 down'));@@ -593,20 +612,20 @@ describe('ChatMediaArchiveService', () => {.mockImplementationOnce(() => new Promise<void>(resolve => (releaseDelete = resolve)));const svc = build({ 'chatMedia.ttlDays': 7 });const sweep = jest.spyOn(svc, 'sweepOrphanedMedia').mockResolvedValue(0);- const find = jest.spyOn(repository, 'find');+ const select = jest.spyOn(repository, 'createQueryBuilder');try {svc.onModuleInit();await jest.advanceTimersByTimeAsync(15 * 60 * 1000);await jest.advanceTimersByTimeAsync(15 * 60 * 1000);- expect(find).toHaveBeenCalledTimes(1);+ expect(select).toHaveBeenCalledTimes(1);releaseDelete();await jest.advanceTimersByTimeAsync(15 * 60 * 1000);- expect(find.mock.calls.length).toBeGreaterThan(1);+ expect(select.mock.calls.length).toBeGreaterThan(1);} finally {svc.onModuleDestroy();releaseDelete();- [del, sweep, find].forEach(spy => spy.mockRestore());+ [del, sweep, select].forEach(spy => spy.mockRestore());jest.useRealTimers();}});diff --git a/src/modules/chat-media/chat-media-archive.service.ts b/src/modules/chat-media/chat-media-archive.service.tsindex 8550d9220..7882a5d54 100644--- a/src/modules/chat-media/chat-media-archive.service.ts+++ b/src/modules/chat-media/chat-media-archive.service.ts@@ -1,7 +1,7 @@import { Injectable, OnModuleDestroy, OnModuleInit } from '@nestjs/common';import { InjectRepository } from '@nestjs/typeorm';import { ConfigService } from '@nestjs/config';-import { In, IsNull, LessThan, MoreThan, Not, Repository } from 'typeorm';+import { In, IsNull, Not, Repository } from 'typeorm';import { randomUUID } from 'crypto';import { Message } from '../message/entities/message.entity';import { StorageService } from '../../common/storage/storage.service';@@ -231,19 +231,25 @@ export class ChatMediaArchiveService implements OnModuleInit, OnModuleDestroy {// so rows whose delete keeps failing are stepped over instead of being re-selected ahead of every// newer row. A batch in which every delete fails ends the run: a missing file already counts as// deleted, so that means the store is down, and walking on would only repeat the failure.+ //+ // On SQLite the walk orders by `+id`: a bare `id` lets the planner satisfy ORDER BY from the+ // primary-key autoindex and visit every row of the table, even with nothing to purge, and+ // better-sqlite3 runs that scan on the event loop. The unary plus starts the plan from the+ // createdAt range instead. Postgres rejects unary plus on a uuid and plans this well as it is.+ const isSqlite = ['sqlite', 'better-sqlite3'].includes(this.repository.manager.connection.options.type);let after = this.purgeCursor;this.purgeCursor = undefined;for (let batch = 0; batch < PURGE_MAX_BATCHES_PER_RUN; batch++) {- const expired = await this.repository.find({- where: {- mediaPath: Not(IsNull()),- createdAt: LessThan(cutoff),- ...(after ? { id: MoreThan(after) } : {}),- },- select: { id: true, mediaPath: true },- order: { id: 'ASC' },- take: PURGE_BATCH_SIZE,- });+ const qb = this.repository+ .createQueryBuilder('m')+ .select(['m.id', 'm.mediaPath'])+ .where('m.mediaPath IS NOT NULL')+ .andWhere('m.createdAt < :cutoff', { cutoff });+ if (after) qb.andWhere('m.id > :after', { after });+ const expired = await qb+ .orderBy(isSqlite ? '+m.id' : 'm.id', 'ASC')+ .limit(PURGE_BATCH_SIZE)+ .getMany();if (expired.length === 0) break;after = expired[expired.length - 1].id;