Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(search): rebuild the SQLite index after a renumbering rebuild

Yudhi Armyndharis7a4e7cc3a223
diff --git a/src/modules/search/providers/builtin-fts.provider.backfill.spec.ts b/src/modules/search/providers/builtin-fts.provider.backfill.spec.ts
index 545d1fdca..48ccfa536 100644
--- a/src/modules/search/providers/builtin-fts.provider.backfill.spec.ts
+++ b/src/modules/search/providers/builtin-fts.provider.backfill.spec.ts
@@ -1,5 +1,5 @@
import 'reflect-metadata';
-import { DataSource } from 'typeorm';
+import { DataSource, TableColumn } from 'typeorm';
import { Message, MessageDirection } from '../../../modules/message/entities/message.entity';
import { Session } from '../../../modules/session/entities/session.entity';
import { BuiltInFtsProvider } from './builtin-fts.provider';
@@ -155,6 +155,33 @@ describe('BuiltInFtsProvider — backfilling an index built after the rows', ()
expect(await found('legacy')).toHaveLength(2);
});
+ /**
+ * A synchronize-driven table rebuild. TypeORM's SQLite column change copies the rows into a new
+ * table without their implicit rowid, so after any delete the survivors are renumbered and every
+ * index entry keyed on an old rowid now describes a different message. The gap repair alone sees
+ * nothing missing for the renumbered rows and leaves the index pointing at the wrong bodies.
+ */
+ it('re-indexes from scratch after a table rebuild renumbered the rowids', async () => {
+ expect(await nullBodies()).toBe(0);
+ const repo = ds.getRepository(Message);
+ await new BuiltInFtsProvider(ds).onModuleInit();
+ await repo.insert([row('charlie fresh', 3)]);
+ await repo.delete({ body: 'alpha legacy' });
+
+ const qr = ds.createQueryRunner();
+ await qr.addColumn('messages', new TableColumn({ name: 'rebuilt', type: 'text', isNullable: true }));
+ await qr.release();
+ const rowids: Array<{ rowid: number }> = await ds.query(`SELECT rowid FROM "messages" ORDER BY rowid`);
+ expect(rowids.map(r => r.rowid)).toEqual([1, 2]); // renumbered: the precondition of this case
+
+ await new BuiltInFtsProvider(ds).onModuleInit();
+
+ expect(await found('bravo')).toEqual(['bravo legacy']);
+ expect(await found('charlie')).toEqual(['charlie fresh']);
+ expect(await found('alpha')).toEqual([]);
+ expect(await indexMatchesContent()).toBe('consistent');
+ });
+
/**
* The write path. On an unrepaired box the `AFTER UPDATE`/`AFTER DELETE` triggers issue FTS5
* 'delete' commands for documents that were never indexed and the statement is rejected, so the
diff --git a/src/modules/search/providers/builtin-fts.provider.ts b/src/modules/search/providers/builtin-fts.provider.ts
index c6a81be79..12fcc5c00 100644
--- a/src/modules/search/providers/builtin-fts.provider.ts
+++ b/src/modules/search/providers/builtin-fts.provider.ts
@@ -184,6 +184,21 @@ export class BuiltInFtsProvider implements SearchProvider, OnModuleInit {
// were never added, and SQLite rejects the whole statement — so ordinary edits and deletes on
// them fail, not merely searches. Reads `d."rowid"` rather than `d."id"`: same query plan,
// without depending on the shadow table's column naming.
+ //
+ // A rowid-level gap check cannot see a renumbered table. TypeORM synchronize rebuilds `messages`
+ // on SQLite by copying the rows without their implicit rowid, so after any delete the survivors
+ // get new rowids and the index entries under them describe other messages. The triggers keep
+ // `docsize` in step with `messages`, so an index row with no message behind it only appears after
+ // such a rebuild, and any rebuild that renumbered a row leaves the old highest rowid behind. Drop
+ // the whole index then and let the gap repair below re-add every row.
+ const orphanRow: unknown[] = await this.dataSource.query(
+ `SELECT EXISTS(SELECT 1 FROM "messages_fts_docsize" d
+ WHERE NOT EXISTS (SELECT 1 FROM "messages" m WHERE m."rowid" = d."rowid")) AS orphan`,
+ );
+ if (Number((orphanRow as Array<{ orphan: number }>)[0]?.orphan)) {
+ this.logger.warn('FTS index holds rows the messages table no longer has; re-indexing every message');
+ await this.dataSource.query(`INSERT INTO "messages_fts"("messages_fts") VALUES ('delete-all')`);
+ }
const gapRow: unknown[] = await this.dataSource.query(
`SELECT EXISTS(SELECT 1 FROM "messages" m WHERE m."body" IS NOT NULL
AND NOT EXISTS (SELECT 1 FROM "messages_fts_docsize" d WHERE d."rowid" = m."rowid")) AS missing`,