Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(search): rebuild the SQLite index after a renumbering rebuild
Yudhi Armyndharis7a4e7cc3a223
diff --git a/src/modules/search/providers/builtin-fts.provider.backfill.spec.ts b/src/modules/search/providers/builtin-fts.provider.backfill.spec.tsindex 545d1fdca..48ccfa536 100644--- a/src/modules/search/providers/builtin-fts.provider.backfill.spec.ts+++ b/src/modules/search/providers/builtin-fts.provider.backfill.spec.ts@@ -1,5 +1,5 @@import 'reflect-metadata';-import { DataSource } from 'typeorm';+import { DataSource, TableColumn } from 'typeorm';import { Message, MessageDirection } from '../../../modules/message/entities/message.entity';import { Session } from '../../../modules/session/entities/session.entity';import { BuiltInFtsProvider } from './builtin-fts.provider';@@ -155,6 +155,33 @@ describe('BuiltInFtsProvider — backfilling an index built after the rows', ()expect(await found('legacy')).toHaveLength(2);});+ /**+ * A synchronize-driven table rebuild. TypeORM's SQLite column change copies the rows into a new+ * table without their implicit rowid, so after any delete the survivors are renumbered and every+ * index entry keyed on an old rowid now describes a different message. The gap repair alone sees+ * nothing missing for the renumbered rows and leaves the index pointing at the wrong bodies.+ */+ it('re-indexes from scratch after a table rebuild renumbered the rowids', async () => {+ expect(await nullBodies()).toBe(0);+ const repo = ds.getRepository(Message);+ await new BuiltInFtsProvider(ds).onModuleInit();+ await repo.insert([row('charlie fresh', 3)]);+ await repo.delete({ body: 'alpha legacy' });++ const qr = ds.createQueryRunner();+ await qr.addColumn('messages', new TableColumn({ name: 'rebuilt', type: 'text', isNullable: true }));+ await qr.release();+ const rowids: Array<{ rowid: number }> = await ds.query(`SELECT rowid FROM "messages" ORDER BY rowid`);+ expect(rowids.map(r => r.rowid)).toEqual([1, 2]); // renumbered: the precondition of this case++ await new BuiltInFtsProvider(ds).onModuleInit();++ expect(await found('bravo')).toEqual(['bravo legacy']);+ expect(await found('charlie')).toEqual(['charlie fresh']);+ expect(await found('alpha')).toEqual([]);+ expect(await indexMatchesContent()).toBe('consistent');+ });+/*** The write path. On an unrepaired box the `AFTER UPDATE`/`AFTER DELETE` triggers issue FTS5* 'delete' commands for documents that were never indexed and the statement is rejected, so thediff --git a/src/modules/search/providers/builtin-fts.provider.ts b/src/modules/search/providers/builtin-fts.provider.tsindex c6a81be79..12fcc5c00 100644--- a/src/modules/search/providers/builtin-fts.provider.ts+++ b/src/modules/search/providers/builtin-fts.provider.ts@@ -184,6 +184,21 @@ export class BuiltInFtsProvider implements SearchProvider, OnModuleInit {// were never added, and SQLite rejects the whole statement — so ordinary edits and deletes on// them fail, not merely searches. Reads `d."rowid"` rather than `d."id"`: same query plan,// without depending on the shadow table's column naming.+ //+ // A rowid-level gap check cannot see a renumbered table. TypeORM synchronize rebuilds `messages`+ // on SQLite by copying the rows without their implicit rowid, so after any delete the survivors+ // get new rowids and the index entries under them describe other messages. The triggers keep+ // `docsize` in step with `messages`, so an index row with no message behind it only appears after+ // such a rebuild, and any rebuild that renumbered a row leaves the old highest rowid behind. Drop+ // the whole index then and let the gap repair below re-add every row.+ const orphanRow: unknown[] = await this.dataSource.query(+ `SELECT EXISTS(SELECT 1 FROM "messages_fts_docsize" d+ WHERE NOT EXISTS (SELECT 1 FROM "messages" m WHERE m."rowid" = d."rowid")) AS orphan`,+ );+ if (Number((orphanRow as Array<{ orphan: number }>)[0]?.orphan)) {+ this.logger.warn('FTS index holds rows the messages table no longer has; re-indexing every message');+ await this.dataSource.query(`INSERT INTO "messages_fts"("messages_fts") VALUES ('delete-all')`);+ }const gapRow: unknown[] = await this.dataSource.query(`SELECT EXISTS(SELECT 1 FROM "messages" m WHERE m."body" IS NOT NULLAND NOT EXISTS (SELECT 1 FROM "messages_fts_docsize" d WHERE d."rowid" = m."rowid")) AS missing`,