Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(search): retry a lost trigram build lock and key it per table
Yudhi Armyndharisca9fea72ff47
diff --git a/docs/26-global-search.md b/docs/26-global-search.mdindex 8e8454cf7..0de78f3f2 100644--- a/docs/26-global-search.md+++ b/docs/26-global-search.md@@ -319,8 +319,10 @@ answering its health probe (a boot that outlasts the Helm startup probe is resta`openwa_unaccent(body)` with `CREATE INDEX CONCURRENTLY`, so writes go on during the build (it cannotrun inside a migration's transaction). On 300,000 messages the index took 76 MB, next to 30 MB for theword index. The build runs on its own connection without the runtime `statement_timeout`. Replicas- that start together take a session advisory lock: one builds, the others answer `501` until the- index is valid. A build that died (a killed pod) leaves an INVALID index, which the next build drops+ that start together take a session advisory lock keyed to their `messages` table, so deployments in+ different `POSTGRES_SCHEMA`s of one database do not wait on each other: one builds, the others answer+ `501` until the index is valid, trying the lock again every 30 seconds in case the one building+ dies or fails. A build that died (a killed pod) leaves an INVALID index, which the next build dropsand rebuilds. The candidate query walks the existing `IDX_messages_createdAt` index, so contains searchadds no other index.diff --git a/src/modules/search/providers/builtin-fts.provider.contains.pg.spec.ts b/src/modules/search/providers/builtin-fts.provider.contains.pg.spec.tsindex 8df7f2cd4..0472e82e5 100644--- a/src/modules/search/providers/builtin-fts.provider.contains.pg.spec.ts+++ b/src/modules/search/providers/builtin-fts.provider.contains.pg.spec.ts@@ -5,7 +5,7 @@ import { BadRequestException, NotImplementedException } from '@nestjs/common';import { ConfigService } from '@nestjs/config';import { DataSource, DataSourceOptions } from 'typeorm';import { join } from 'path';-import { BuiltInFtsProvider, TRIGRAM_BUILD_LOCK_KEYS } from './builtin-fts.provider';+import { BuiltInFtsProvider, TRIGRAM_BUILD_LOCK, TRIGRAM_BUILD_LOCK_KEY } from './builtin-fts.provider';/*** Postgres runtime coverage for match=contains: the pg_trgm/unaccent schema buildContainsIndex creates@@ -123,22 +123,68 @@ const containsOn = new ConfigService({ search: { containsEnabled: true } });expect((await provider.search({ q: 'voic', match: 'contains' })).total).toBe(2);});+ // The lock the provider takes, keyed by the `messages` table the session's search_path resolves.+ const LOCK = TRIGRAM_BUILD_LOCK;+it('leaves the build to the replica holding the lock, then serves the index it built', async () => {const other = ds.createQueryRunner();await other.connect();+ const waiting = new BuiltInFtsProvider(ds, containsOn);try {- await other.query(`SELECT pg_advisory_lock($1, $2)`, [...TRIGRAM_BUILD_LOCK_KEYS]);- const waiting = new BuiltInFtsProvider(ds, containsOn);+ await other.query(`SELECT pg_advisory_lock(${LOCK})`, [TRIGRAM_BUILD_LOCK_KEY]);await waiting.buildContainsIndex();expect(await trigramState()).toEqual({ extensions: 0, fn: false, idx: null });await expect(waiting.search({ q: 'voic', match: 'contains' })).rejects.toThrow(/still being built/);- await other.query(`SELECT pg_advisory_unlock($1, $2)`, [...TRIGRAM_BUILD_LOCK_KEYS]);+ await other.query(`SELECT pg_advisory_unlock(${LOCK})`, [TRIGRAM_BUILD_LOCK_KEY]);await new BuiltInFtsProvider(ds, containsOn).buildContainsIndex();expect((await waiting.search({ q: 'voic', match: 'contains' })).total).toBe(2);} finally {+ waiting.onModuleDestroy();+ await other.release();+ }+ });++ it('builds the index itself once the replica holding the lock lets go without one', async () => {+ const other = ds.createQueryRunner();+ await other.connect();+ const waiting = new BuiltInFtsProvider(ds, containsOn);+ Object.assign(waiting, { containsBuildRetryMs: 50 });+ try {+ await other.query(`SELECT pg_advisory_lock(${LOCK})`, [TRIGRAM_BUILD_LOCK_KEY]);+ await waiting.buildContainsIndex();+ await expect(waiting.search({ q: 'voic', match: 'contains' })).rejects.toThrow(/still being built/);++ // The holder dies or fails: the lock goes, and no index is left behind.+ await other.query(`SELECT pg_advisory_unlock(${LOCK})`, [TRIGRAM_BUILD_LOCK_KEY]);+ for (let i = 0; i < 200 && (await trigramState()).idx !== true; i++) await new Promise(r => setTimeout(r, 50));++ expect(await trigramState()).toEqual({ extensions: 2, fn: true, idx: true });+ expect((await waiting.search({ q: 'voic', match: 'contains' })).total).toBe(2);+ } finally {+ waiting.onModuleDestroy();+ await other.release();+ }+ });++ it('is not held up by the build lock of a deployment in another schema of the database', async () => {+ const other = ds.createQueryRunner();+ await other.connect();+ try {+ await other.query(`CREATE SCHEMA "elsewhere"`);+ await other.query(`CREATE TABLE "elsewhere"."messages" (LIKE "public"."messages")`);+ await other.query(`SET search_path = "elsewhere"`);+ await other.query(`SELECT pg_advisory_lock(${LOCK})`, [TRIGRAM_BUILD_LOCK_KEY]);++ await new BuiltInFtsProvider(ds, containsOn).buildContainsIndex();++ expect((await trigramState()).idx).toBe(true);+ } finally {+ await other.query(`SELECT pg_advisory_unlock(${LOCK})`, [TRIGRAM_BUILD_LOCK_KEY]);+ await other.query(`RESET search_path`);+ await other.query(`DROP SCHEMA "elsewhere" CASCADE`);await other.release();}});@@ -176,9 +222,12 @@ const containsOn = new ConfigService({ search: { containsEnabled: true } });});it('rejects a query with no run of three letters or digits, which pg_trgm cannot look up', async () => {- for (const q of ['πππ', '...', ':-)', 'a.b.c']) {+ for (const q of ['πππ', '...', ':-)', 'a.b.c', 'μλ ', '\u02bc'.repeat(3), 'ΒΉΒ²Β³']) {await expect(provider.search({ q, match: 'contains' })).rejects.toBeInstanceOf(BadRequestException);}+ await insert('m5', 's1', 'μμΈμμμ λΌλ©μ λ¨Ήμλ€', 50);+ expect((await provider.search({ q: 'μμΈμ', match: 'contains' })).hits.map(h => h.messageId)).toEqual(['m5']);+ expect((await provider.search({ q: 'γ©γΌγ‘γ³', match: 'contains' })).total).toBe(0);expect((await provider.search({ q: ':-) invoice', match: 'contains' })).total).toBe(0);expect((await provider.search({ q: 'INV-2048', match: 'contains' })).total).toBe(1);});diff --git a/src/modules/search/providers/builtin-fts.provider.postgres.spec.ts b/src/modules/search/providers/builtin-fts.provider.postgres.spec.tsindex 1253f3379..888123b6b 100644--- a/src/modules/search/providers/builtin-fts.provider.postgres.spec.ts+++ b/src/modules/search/providers/builtin-fts.provider.postgres.spec.ts@@ -4,7 +4,7 @@ import { ConfigService } from '@nestjs/config';import { LoggerService } from '../../../common/services/logger.service';import { DataSource } from 'typeorm';import { ContainsTermsRefusedError } from '../search-terms';-import { BuiltInFtsProvider } from './builtin-fts.provider';+import { BuiltInFtsProvider, TRIGRAM_BUILD_LOCK_KEY } from './builtin-fts.provider';/*** Postgres-side probe coverage. A real PG instance isn't available to the unit suite, so the@@ -217,3 +217,78 @@ describe('BuiltInFtsProvider match=contains terms (postgres)', () => {expect(query).toHaveBeenCalledTimes(1);});});++describe('BuiltInFtsProvider contains index build lock (postgres)', () => {+ beforeEach(() => {+ jest.spyOn(LoggerService.prototype, 'log').mockImplementation(() => undefined);+ jest.spyOn(LoggerService.prototype, 'warn').mockImplementation(() => undefined);+ });+ afterEach(() => jest.restoreAllMocks());++ /**+ * A database whose build lock answers `locks` in turn and whose index exists once the lock is taken,+ * while the catalog a search reads still lacks it: only a build attempt of the provider's own can+ * make contains search ready.+ */+ const lockedOut = (locks: boolean[]) => {+ const tryLock = jest.fn<Promise<Array<{ locked: boolean }>>, [string, unknown[]]>(() =>+ Promise.resolve([{ locked: locks.shift() ?? false }]),+ );+ const runner = {+ connect: jest.fn().mockResolvedValue(undefined),+ release: jest.fn().mockResolvedValue(undefined),+ query: jest.fn().mockResolvedValue([]),+ manager: {+ query: jest.fn((sql: string, params: unknown[]) =>+ /pg_try_advisory_lock/.test(sql)+ ? tryLock(sql, params)+ : Promise.resolve([{ trgm: 'public', unaccent: 'public', fn: true, idx: true }]),+ ),+ },+ };+ const notReady = [{ trgm: 'public', unaccent: 'public', fn: false, idx: null }];+ const ds = {+ options: { type: 'postgres' },+ createQueryRunner: () => runner,+ query: jest.fn().mockResolvedValue([]),+ manager: { query: jest.fn().mockResolvedValue(notReady) },+ } as unknown as DataSource;+ const provider = new BuiltInFtsProvider(ds, new ConfigService({ search: { containsEnabled: true } }));+ Object.assign(provider, { containsBuildRetryMs: 0 });+ return { provider, tryLock };+ };+ const settle = () => new Promise(resolve => setTimeout(resolve, 20));++ it('keys the lock to the messages table and tries it again until the index is ready', async () => {+ const { provider, tryLock } = lockedOut([false, false, true]);+ const search = () => provider.search({ q: 'voic', match: 'contains' });++ await provider.buildContainsIndex();+ await expect(search()).rejects.toThrow(/still being built/);+ for (+ let i = 0;+ i < 100 &&+ (await search().then(+ () => false,+ () => true,+ ));+ i+++ )+ await settle();++ await expect(search()).resolves.toMatchObject({ total: 0 });+ expect(tryLock).toHaveBeenCalledTimes(3);+ expect(tryLock).toHaveBeenCalledWith(expect.stringContaining("to_regclass('messages')"), [TRIGRAM_BUILD_LOCK_KEY]);+ provider.onModuleDestroy();+ });++ it('stops trying once the module is destroyed', async () => {+ const { provider, tryLock } = lockedOut([]);++ await provider.buildContainsIndex();+ provider.onModuleDestroy();+ await settle();++ expect(tryLock).toHaveBeenCalledTimes(1);+ });+});diff --git a/src/modules/search/providers/builtin-fts.provider.ts b/src/modules/search/providers/builtin-fts.provider.tsindex c2ee46177..5d6b9f478 100644--- a/src/modules/search/providers/builtin-fts.provider.ts+++ b/src/modules/search/providers/builtin-fts.provider.ts@@ -3,6 +3,7 @@ import {Injectable,NotImplementedException,OnApplicationBootstrap,+ OnModuleDestroy,OnModuleInit,Optional,} from '@nestjs/common';@@ -58,9 +59,15 @@ const TRIGRAM_BACKFILL_BATCH = 1000;/*** The session-level advisory lock one replica holds while it builds the Postgres trigram index, in the- * two-int4 form of POSTGRES_BOOT_MIGRATION_LOCK_KEYS: 0x4f5741 = "OWA", 0x7472676d = "trgm".+ * two-int4 form: 0x7472676d ("trgm") first, a first key no other OpenWA lock uses (the boot migration+ * lock's is 0x4f5741, "OWA"), then the oid of the `messages` table the index is built on, so deployments+ * that share a database in different schemas (POSTGRES_SCHEMA) each build their own.*/-export const TRIGRAM_BUILD_LOCK_KEYS: readonly [number, number] = [0x4f5741, 0x7472676d];+export const TRIGRAM_BUILD_LOCK_KEY = 0x7472676d;+export const TRIGRAM_BUILD_LOCK = `$1, to_regclass('messages')::oid::int`;++/** How long a replica that found the build lock taken waits before trying it again. */+const TRIGRAM_BUILD_RETRY_MS = 30_000;/** Why match=contains answers 501, one message per cause, each naming what to change. */const CONTAINS_UNAVAILABLE = {@@ -195,7 +202,7 @@ type PlaceholderFn = () => string;* ranks a bounded window of candidates in the application (see searchContains).*/@Injectable()-export class BuiltInFtsProvider implements SearchProvider, OnModuleInit, OnApplicationBootstrap {+export class BuiltInFtsProvider implements SearchProvider, OnModuleInit, OnApplicationBootstrap, OnModuleDestroy {readonly id = 'builtin-fts';readonly label = 'Built-in database full-text search';private readonly logger = createLogger('BuiltInFtsProvider');@@ -206,6 +213,11 @@ export class BuiltInFtsProvider implements SearchProvider, OnModuleInit, OnAppli/** Why match=contains answers 501 right now (a CONTAINS_UNAVAILABLE message), or null once its index is ready. */private containsUnavailable: string | null;+ /** The next attempt at a build another replica held the lock for (see retryContainsBuild). */+ private containsBuildRetry?: NodeJS.Timeout;+ private readonly containsBuildRetryMs = TRIGRAM_BUILD_RETRY_MS;+ private destroyed = false;+// OpenWA has two TypeORM connections (main: auth/audit SQLite, data: messages). Bind explicitly to// 'data' so the provider queries the connection that owns the `messages` table + the FTS migration,// never the default/`main` one. The bare `DataSource` type alone is ambiguous with two connections.@@ -491,10 +503,30 @@ export class BuiltInFtsProvider implements SearchProvider, OnModuleInit, OnApplithis.logger.error(`Failed to build the trigram index: ${e instanceof Error ? e.message : String(e)}`);return;}- if (this.containsUnavailable) this.logger.warn(this.containsUnavailable);+ if (this.containsUnavailable === CONTAINS_UNAVAILABLE.building) this.retryContainsBuild();+ else if (this.containsUnavailable) this.logger.warn(this.containsUnavailable);else this.logger.log(`Trigram index ready in ${Date.now() - start} ms`);}+ /**+ * Another replica holds the Postgres build lock. If it dies or its build fails, the lock is released+ * with no valid index, and only a build attempt of this replica's own finds that, so it tries the lock+ * again until the index is ready or the attempt fails for a cause of its own. A search in the meantime+ * still finds an index the other replica finished (probePostgresTrigram).+ */+ private retryContainsBuild(): void {+ if (this.destroyed) return;+ this.containsBuildRetry = setTimeout(() => {+ if (this.containsUnavailable === CONTAINS_UNAVAILABLE.building) void this.buildContainsIndex();+ }, this.containsBuildRetryMs);+ this.containsBuildRetry.unref();+ }++ onModuleDestroy(): void {+ this.destroyed = true;+ clearTimeout(this.containsBuildRetry);+ }+/** Returns why this SQLite library cannot hold the index, or null once the index is complete. */private async buildSqliteTrigramIndex(): Promise<string | null> {if (!(await this.sqliteHasFts5())) return CONTAINS_UNAVAILABLE.sqliteTrigram;@@ -509,15 +541,16 @@ export class BuiltInFtsProvider implements SearchProvider, OnModuleInit, OnAppli* Runs createPostgresTrigramSchema on a connection of its own: CREATE INDEX CONCURRENTLY cannot run* inside a transaction, which rules out a migration, and outlasts the runtime pool's statement_timeout,* which is lifted here. The advisory lock lets one replica build while the others answer 501 until the- * index is valid. Returns why the index cannot be used yet, or null once it is valid.+ * index is valid (see retryContainsBuild). Returns why the index cannot be used yet, or null once it is+ * valid.*/private async buildPostgresTrigramIndex(): Promise<string | null> {const runner = this.dataSource.createQueryRunner();await runner.connect();try {const [{ locked }] = await runner.manager.query<Array<{ locked: boolean }>>(- `SELECT pg_try_advisory_lock($1, $2) AS locked`,- [...TRIGRAM_BUILD_LOCK_KEYS],+ `SELECT pg_try_advisory_lock(${TRIGRAM_BUILD_LOCK}) AS locked`,+ [TRIGRAM_BUILD_LOCK_KEY],);if (!locked) return CONTAINS_UNAVAILABLE.building;try {@@ -526,7 +559,7 @@ export class BuiltInFtsProvider implements SearchProvider, OnModuleInit, OnAppli} finally {// The connection returns to the pool: the next borrower must find its timeout and no lock.await runner.query('RESET statement_timeout');- await runner.query('SELECT pg_advisory_unlock($1, $2)', [...TRIGRAM_BUILD_LOCK_KEYS]);+ await runner.query(`SELECT pg_advisory_unlock(${TRIGRAM_BUILD_LOCK})`, [TRIGRAM_BUILD_LOCK_KEY]);}} finally {await runner.release();