Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(search): rebuild a trigram index whose triggers are gone
Yudhi Armyndharis2e989cbe8903
diff --git a/docs/26-global-search.md b/docs/26-global-search.mdindex 055508d66..3772f1aa3 100644--- a/docs/26-global-search.md+++ b/docs/26-global-search.md@@ -313,9 +313,10 @@ answering its health probe (a boot that outlasts the Helm startup probe is restameanwhile are indexed by the triggers. A trigram index holds every three-character window of the body:on a synthetic corpus of 1,000,000 messages it took 289 MB, about 3.1 times the message text, and a9.4 s backfill. Later boots only check it; an index whose document count differs from `messages` (an- interrupted backfill, or rowids renumbered by a table rebuild) is emptied and backfilled again. A build- that finds another transaction holding the connection for more than 10 seconds (a data import, for- one) tries again every 30 seconds instead of failing.+ interrupted backfill, or rowids renumbered by a table rebuild) or that is missing one of its triggers+ (a table rebuild drops them, and messages stored before the next build never reached it) is emptied and+ backfilled again. A build that finds another transaction holding the connection for more than 10+ seconds (a data import, for one) tries again every 30 seconds instead of failing.- **PostgreSQL** creates the `pg_trgm` and `unaccent` extensions when they are missing and the role may,the IMMUTABLE `openwa_unaccent(text)` wrapper, and the `idx_messages_body_trgm` GIN index on`openwa_unaccent(body)` with `CREATE INDEX CONCURRENTLY`, so writes go on during the build (it cannotdiff --git a/src/modules/search/providers/builtin-fts.provider.contains.spec.ts b/src/modules/search/providers/builtin-fts.provider.contains.spec.tsindex 010c104e3..9e3c164ec 100644--- a/src/modules/search/providers/builtin-fts.provider.contains.spec.ts+++ b/src/modules/search/providers/builtin-fts.provider.contains.spec.ts@@ -295,6 +295,26 @@ describe('BuiltInFtsProvider trigram index build (sqlite)', () => {expect((await provider.search({ q: 'receipt 2401', match: 'contains' })).total).toBe(0);});+ it('rebuilds an index whose triggers a table rebuild dropped, which missed the rows written since', async () => {+ await new BuiltInFtsProvider(ds, containsOn).buildContainsIndex();+ // A rebuild of `messages` (synchronize) drops its triggers, and sessions write before the next build.+ for (const trigger of ['ai', 'ad', 'au']) await ds.query(`DROP TRIGGER messages_trgm_${trigger}`);+ const repo = ds.getRepository(Message);+ await repo.update({ chatId: 'c2' }, { body: 'Thanks, I will pay the receipt today' });+ // The newest row goes and the next one takes its rowid, so neither count moves.+ await repo.delete({ chatId: 'c4' });+ await repo.insert(row('s1', 'c5', 'Parcel left at the warehouse', MessageDirection.INCOMING, 50));+ const provider = new BuiltInFtsProvider(ds, containsOn);++ await provider.buildContainsIndex();++ await expect(integrityCheck()).resolves.toBeDefined();+ expect((await provider.search({ q: 'receip', match: 'contains' })).hits.map(h => h.chatId)).toEqual(['c2']);+ expect((await provider.search({ q: 'invoice', match: 'contains' })).hits.map(h => h.chatId)).toEqual(['c1']);+ expect((await provider.search({ q: 'discount', match: 'contains' })).total).toBe(0);+ expect((await provider.search({ q: 'warehouse', match: 'contains' })).hits.map(h => h.chatId)).toEqual(['c5']);+ });+describe('while another transaction holds the shared connection', () => {let provider: BuiltInFtsProvider;diff --git a/src/modules/search/providers/builtin-fts.provider.ts b/src/modules/search/providers/builtin-fts.provider.tsindex a923823e0..cd1d98403 100644--- a/src/modules/search/providers/builtin-fts.provider.ts+++ b/src/modules/search/providers/builtin-fts.provider.ts@@ -416,12 +416,19 @@ export class BuiltInFtsProvider implements SearchProvider, OnModuleInit, OnApplithis.logger.warn(`${table} holds rows the messages table no longer has; re-indexing every message`);await this.dataSource.query(`INSERT INTO "${table}"("${table}") VALUES ('delete-all')`);}- // A batched index is filled after its triggers exist instead (backfillSqliteIndexInBatches).+ // A batched index is filled after its triggers exist instead (backfillSqliteIndexInBatches). It is built+ // only once the HTTP listener is up, while a table rebuild at boot (synchronize) drops its triggers with+ // the old table and sessions start writing before the listener: an edit, a delete or a rowid reused+ // after one then leaves entries for text the table no longer holds, which neither the orphan check nor+ // the count sees. Without all three triggers the index is unknown, and it is emptied like one whose+ // count is off.if (batched) {- const [counts] = await this.dataSource.query<Array<{ messages: number; indexed: number }>>(- `SELECT (SELECT count(*) FROM "messages") AS messages, (SELECT count(*) FROM "${docsize}") AS indexed`,+ const [counts] = await this.dataSource.query<Array<{ messages: number; indexed: number; triggers: number }>>(+ `SELECT (SELECT count(*) FROM "messages") AS messages, (SELECT count(*) FROM "${docsize}") AS indexed,+ (SELECT count(*) FROM sqlite_master WHERE type = 'trigger' AND tbl_name = 'messages'+ AND name IN ('${table}_ai', '${table}_ad', '${table}_au')) AS triggers`,);- if (Number(counts.messages) !== Number(counts.indexed)) {+ if (Number(counts.messages) !== Number(counts.indexed) || Number(counts.triggers) !== 3) {await this.dataSource.query(`INSERT INTO "${table}"("${table}") VALUES ('delete-all')`);}} else {