Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(search): retry a lost trigram build lock and key it per table

Yudhi Armyndharisca9fea72ff47
diff --git a/docs/26-global-search.md b/docs/26-global-search.md
index 8e8454cf7..0de78f3f2 100644
--- a/docs/26-global-search.md
+++ b/docs/26-global-search.md
@@ -319,8 +319,10 @@ answering its health probe (a boot that outlasts the Helm startup probe is resta
`openwa_unaccent(body)` with `CREATE INDEX CONCURRENTLY`, so writes go on during the build (it cannot
run inside a migration's transaction). On 300,000 messages the index took 76 MB, next to 30 MB for the
word index. The build runs on its own connection without the runtime `statement_timeout`. Replicas
- that start together take a session advisory lock: one builds, the others answer `501` until the
- index is valid. A build that died (a killed pod) leaves an INVALID index, which the next build drops
+ that start together take a session advisory lock keyed to their `messages` table, so deployments in
+ different `POSTGRES_SCHEMA`s of one database do not wait on each other: one builds, the others answer
+ `501` until the index is valid, trying the lock again every 30 seconds in case the one building
+ dies or fails. A build that died (a killed pod) leaves an INVALID index, which the next build drops
and rebuilds. The candidate query walks the existing `IDX_messages_createdAt` index, so contains search
adds no other index.
diff --git a/src/modules/search/providers/builtin-fts.provider.contains.pg.spec.ts b/src/modules/search/providers/builtin-fts.provider.contains.pg.spec.ts
index 8df7f2cd4..0472e82e5 100644
--- a/src/modules/search/providers/builtin-fts.provider.contains.pg.spec.ts
+++ b/src/modules/search/providers/builtin-fts.provider.contains.pg.spec.ts
@@ -5,7 +5,7 @@ import { BadRequestException, NotImplementedException } from '@nestjs/common';
import { ConfigService } from '@nestjs/config';
import { DataSource, DataSourceOptions } from 'typeorm';
import { join } from 'path';
-import { BuiltInFtsProvider, TRIGRAM_BUILD_LOCK_KEYS } from './builtin-fts.provider';
+import { BuiltInFtsProvider, TRIGRAM_BUILD_LOCK, TRIGRAM_BUILD_LOCK_KEY } from './builtin-fts.provider';
/**
* Postgres runtime coverage for match=contains: the pg_trgm/unaccent schema buildContainsIndex creates
@@ -123,22 +123,68 @@ const containsOn = new ConfigService({ search: { containsEnabled: true } });
expect((await provider.search({ q: 'voic', match: 'contains' })).total).toBe(2);
});
+ // The lock the provider takes, keyed by the `messages` table the session's search_path resolves.
+ const LOCK = TRIGRAM_BUILD_LOCK;
+
it('leaves the build to the replica holding the lock, then serves the index it built', async () => {
const other = ds.createQueryRunner();
await other.connect();
+ const waiting = new BuiltInFtsProvider(ds, containsOn);
try {
- await other.query(`SELECT pg_advisory_lock($1, $2)`, [...TRIGRAM_BUILD_LOCK_KEYS]);
- const waiting = new BuiltInFtsProvider(ds, containsOn);
+ await other.query(`SELECT pg_advisory_lock(${LOCK})`, [TRIGRAM_BUILD_LOCK_KEY]);
await waiting.buildContainsIndex();
expect(await trigramState()).toEqual({ extensions: 0, fn: false, idx: null });
await expect(waiting.search({ q: 'voic', match: 'contains' })).rejects.toThrow(/still being built/);
- await other.query(`SELECT pg_advisory_unlock($1, $2)`, [...TRIGRAM_BUILD_LOCK_KEYS]);
+ await other.query(`SELECT pg_advisory_unlock(${LOCK})`, [TRIGRAM_BUILD_LOCK_KEY]);
await new BuiltInFtsProvider(ds, containsOn).buildContainsIndex();
expect((await waiting.search({ q: 'voic', match: 'contains' })).total).toBe(2);
} finally {
+ waiting.onModuleDestroy();
+ await other.release();
+ }
+ });
+
+ it('builds the index itself once the replica holding the lock lets go without one', async () => {
+ const other = ds.createQueryRunner();
+ await other.connect();
+ const waiting = new BuiltInFtsProvider(ds, containsOn);
+ Object.assign(waiting, { containsBuildRetryMs: 50 });
+ try {
+ await other.query(`SELECT pg_advisory_lock(${LOCK})`, [TRIGRAM_BUILD_LOCK_KEY]);
+ await waiting.buildContainsIndex();
+ await expect(waiting.search({ q: 'voic', match: 'contains' })).rejects.toThrow(/still being built/);
+
+ // The holder dies or fails: the lock goes, and no index is left behind.
+ await other.query(`SELECT pg_advisory_unlock(${LOCK})`, [TRIGRAM_BUILD_LOCK_KEY]);
+ for (let i = 0; i < 200 && (await trigramState()).idx !== true; i++) await new Promise(r => setTimeout(r, 50));
+
+ expect(await trigramState()).toEqual({ extensions: 2, fn: true, idx: true });
+ expect((await waiting.search({ q: 'voic', match: 'contains' })).total).toBe(2);
+ } finally {
+ waiting.onModuleDestroy();
+ await other.release();
+ }
+ });
+
+ it('is not held up by the build lock of a deployment in another schema of the database', async () => {
+ const other = ds.createQueryRunner();
+ await other.connect();
+ try {
+ await other.query(`CREATE SCHEMA "elsewhere"`);
+ await other.query(`CREATE TABLE "elsewhere"."messages" (LIKE "public"."messages")`);
+ await other.query(`SET search_path = "elsewhere"`);
+ await other.query(`SELECT pg_advisory_lock(${LOCK})`, [TRIGRAM_BUILD_LOCK_KEY]);
+
+ await new BuiltInFtsProvider(ds, containsOn).buildContainsIndex();
+
+ expect((await trigramState()).idx).toBe(true);
+ } finally {
+ await other.query(`SELECT pg_advisory_unlock(${LOCK})`, [TRIGRAM_BUILD_LOCK_KEY]);
+ await other.query(`RESET search_path`);
+ await other.query(`DROP SCHEMA "elsewhere" CASCADE`);
await other.release();
}
});
@@ -176,9 +222,12 @@ const containsOn = new ConfigService({ search: { containsEnabled: true } });
});
it('rejects a query with no run of three letters or digits, which pg_trgm cannot look up', async () => {
- for (const q of ['πŸ˜‚πŸ˜‚πŸ˜‚', '...', ':-)', 'a.b.c']) {
+ for (const q of ['πŸ˜‚πŸ˜‚πŸ˜‚', '...', ':-)', 'a.b.c', 'μ•ˆλ…•', '\u02bc'.repeat(3), 'ΒΉΒ²Β³']) {
await expect(provider.search({ q, match: 'contains' })).rejects.toBeInstanceOf(BadRequestException);
}
+ await insert('m5', 's1', 'μ„œμšΈμ—­μ—μ„œ λΌλ©˜μ„ λ¨Ήμ—ˆλ‹€', 50);
+ expect((await provider.search({ q: 'μ„œμšΈμ—­', match: 'contains' })).hits.map(h => h.messageId)).toEqual(['m5']);
+ expect((await provider.search({ q: 'ラーパン', match: 'contains' })).total).toBe(0);
expect((await provider.search({ q: ':-) invoice', match: 'contains' })).total).toBe(0);
expect((await provider.search({ q: 'INV-2048', match: 'contains' })).total).toBe(1);
});
diff --git a/src/modules/search/providers/builtin-fts.provider.postgres.spec.ts b/src/modules/search/providers/builtin-fts.provider.postgres.spec.ts
index 1253f3379..888123b6b 100644
--- a/src/modules/search/providers/builtin-fts.provider.postgres.spec.ts
+++ b/src/modules/search/providers/builtin-fts.provider.postgres.spec.ts
@@ -4,7 +4,7 @@ import { ConfigService } from '@nestjs/config';
import { LoggerService } from '../../../common/services/logger.service';
import { DataSource } from 'typeorm';
import { ContainsTermsRefusedError } from '../search-terms';
-import { BuiltInFtsProvider } from './builtin-fts.provider';
+import { BuiltInFtsProvider, TRIGRAM_BUILD_LOCK_KEY } from './builtin-fts.provider';
/**
* Postgres-side probe coverage. A real PG instance isn't available to the unit suite, so the
@@ -217,3 +217,78 @@ describe('BuiltInFtsProvider match=contains terms (postgres)', () => {
expect(query).toHaveBeenCalledTimes(1);
});
});
+
+describe('BuiltInFtsProvider contains index build lock (postgres)', () => {
+ beforeEach(() => {
+ jest.spyOn(LoggerService.prototype, 'log').mockImplementation(() => undefined);
+ jest.spyOn(LoggerService.prototype, 'warn').mockImplementation(() => undefined);
+ });
+ afterEach(() => jest.restoreAllMocks());
+
+ /**
+ * A database whose build lock answers `locks` in turn and whose index exists once the lock is taken,
+ * while the catalog a search reads still lacks it: only a build attempt of the provider's own can
+ * make contains search ready.
+ */
+ const lockedOut = (locks: boolean[]) => {
+ const tryLock = jest.fn<Promise<Array<{ locked: boolean }>>, [string, unknown[]]>(() =>
+ Promise.resolve([{ locked: locks.shift() ?? false }]),
+ );
+ const runner = {
+ connect: jest.fn().mockResolvedValue(undefined),
+ release: jest.fn().mockResolvedValue(undefined),
+ query: jest.fn().mockResolvedValue([]),
+ manager: {
+ query: jest.fn((sql: string, params: unknown[]) =>
+ /pg_try_advisory_lock/.test(sql)
+ ? tryLock(sql, params)
+ : Promise.resolve([{ trgm: 'public', unaccent: 'public', fn: true, idx: true }]),
+ ),
+ },
+ };
+ const notReady = [{ trgm: 'public', unaccent: 'public', fn: false, idx: null }];
+ const ds = {
+ options: { type: 'postgres' },
+ createQueryRunner: () => runner,
+ query: jest.fn().mockResolvedValue([]),
+ manager: { query: jest.fn().mockResolvedValue(notReady) },
+ } as unknown as DataSource;
+ const provider = new BuiltInFtsProvider(ds, new ConfigService({ search: { containsEnabled: true } }));
+ Object.assign(provider, { containsBuildRetryMs: 0 });
+ return { provider, tryLock };
+ };
+ const settle = () => new Promise(resolve => setTimeout(resolve, 20));
+
+ it('keys the lock to the messages table and tries it again until the index is ready', async () => {
+ const { provider, tryLock } = lockedOut([false, false, true]);
+ const search = () => provider.search({ q: 'voic', match: 'contains' });
+
+ await provider.buildContainsIndex();
+ await expect(search()).rejects.toThrow(/still being built/);
+ for (
+ let i = 0;
+ i < 100 &&
+ (await search().then(
+ () => false,
+ () => true,
+ ));
+ i++
+ )
+ await settle();
+
+ await expect(search()).resolves.toMatchObject({ total: 0 });
+ expect(tryLock).toHaveBeenCalledTimes(3);
+ expect(tryLock).toHaveBeenCalledWith(expect.stringContaining("to_regclass('messages')"), [TRIGRAM_BUILD_LOCK_KEY]);
+ provider.onModuleDestroy();
+ });
+
+ it('stops trying once the module is destroyed', async () => {
+ const { provider, tryLock } = lockedOut([]);
+
+ await provider.buildContainsIndex();
+ provider.onModuleDestroy();
+ await settle();
+
+ expect(tryLock).toHaveBeenCalledTimes(1);
+ });
+});
diff --git a/src/modules/search/providers/builtin-fts.provider.ts b/src/modules/search/providers/builtin-fts.provider.ts
index c2ee46177..5d6b9f478 100644
--- a/src/modules/search/providers/builtin-fts.provider.ts
+++ b/src/modules/search/providers/builtin-fts.provider.ts
@@ -3,6 +3,7 @@ import {
Injectable,
NotImplementedException,
OnApplicationBootstrap,
+ OnModuleDestroy,
OnModuleInit,
Optional,
} from '@nestjs/common';
@@ -58,9 +59,15 @@ const TRIGRAM_BACKFILL_BATCH = 1000;
/**
* The session-level advisory lock one replica holds while it builds the Postgres trigram index, in the
- * two-int4 form of POSTGRES_BOOT_MIGRATION_LOCK_KEYS: 0x4f5741 = "OWA", 0x7472676d = "trgm".
+ * two-int4 form: 0x7472676d ("trgm") first, a first key no other OpenWA lock uses (the boot migration
+ * lock's is 0x4f5741, "OWA"), then the oid of the `messages` table the index is built on, so deployments
+ * that share a database in different schemas (POSTGRES_SCHEMA) each build their own.
*/
-export const TRIGRAM_BUILD_LOCK_KEYS: readonly [number, number] = [0x4f5741, 0x7472676d];
+export const TRIGRAM_BUILD_LOCK_KEY = 0x7472676d;
+export const TRIGRAM_BUILD_LOCK = `$1, to_regclass('messages')::oid::int`;
+
+/** How long a replica that found the build lock taken waits before trying it again. */
+const TRIGRAM_BUILD_RETRY_MS = 30_000;
/** Why match=contains answers 501, one message per cause, each naming what to change. */
const CONTAINS_UNAVAILABLE = {
@@ -195,7 +202,7 @@ type PlaceholderFn = () => string;
* ranks a bounded window of candidates in the application (see searchContains).
*/
@Injectable()
-export class BuiltInFtsProvider implements SearchProvider, OnModuleInit, OnApplicationBootstrap {
+export class BuiltInFtsProvider implements SearchProvider, OnModuleInit, OnApplicationBootstrap, OnModuleDestroy {
readonly id = 'builtin-fts';
readonly label = 'Built-in database full-text search';
private readonly logger = createLogger('BuiltInFtsProvider');
@@ -206,6 +213,11 @@ export class BuiltInFtsProvider implements SearchProvider, OnModuleInit, OnAppli
/** Why match=contains answers 501 right now (a CONTAINS_UNAVAILABLE message), or null once its index is ready. */
private containsUnavailable: string | null;
+ /** The next attempt at a build another replica held the lock for (see retryContainsBuild). */
+ private containsBuildRetry?: NodeJS.Timeout;
+ private readonly containsBuildRetryMs = TRIGRAM_BUILD_RETRY_MS;
+ private destroyed = false;
+
// OpenWA has two TypeORM connections (main: auth/audit SQLite, data: messages). Bind explicitly to
// 'data' so the provider queries the connection that owns the `messages` table + the FTS migration,
// never the default/`main` one. The bare `DataSource` type alone is ambiguous with two connections.
@@ -491,10 +503,30 @@ export class BuiltInFtsProvider implements SearchProvider, OnModuleInit, OnAppli
this.logger.error(`Failed to build the trigram index: ${e instanceof Error ? e.message : String(e)}`);
return;
}
- if (this.containsUnavailable) this.logger.warn(this.containsUnavailable);
+ if (this.containsUnavailable === CONTAINS_UNAVAILABLE.building) this.retryContainsBuild();
+ else if (this.containsUnavailable) this.logger.warn(this.containsUnavailable);
else this.logger.log(`Trigram index ready in ${Date.now() - start} ms`);
}
+ /**
+ * Another replica holds the Postgres build lock. If it dies or its build fails, the lock is released
+ * with no valid index, and only a build attempt of this replica's own finds that, so it tries the lock
+ * again until the index is ready or the attempt fails for a cause of its own. A search in the meantime
+ * still finds an index the other replica finished (probePostgresTrigram).
+ */
+ private retryContainsBuild(): void {
+ if (this.destroyed) return;
+ this.containsBuildRetry = setTimeout(() => {
+ if (this.containsUnavailable === CONTAINS_UNAVAILABLE.building) void this.buildContainsIndex();
+ }, this.containsBuildRetryMs);
+ this.containsBuildRetry.unref();
+ }
+
+ onModuleDestroy(): void {
+ this.destroyed = true;
+ clearTimeout(this.containsBuildRetry);
+ }
+
/** Returns why this SQLite library cannot hold the index, or null once the index is complete. */
private async buildSqliteTrigramIndex(): Promise<string | null> {
if (!(await this.sqliteHasFts5())) return CONTAINS_UNAVAILABLE.sqliteTrigram;
@@ -509,15 +541,16 @@ export class BuiltInFtsProvider implements SearchProvider, OnModuleInit, OnAppli
* Runs createPostgresTrigramSchema on a connection of its own: CREATE INDEX CONCURRENTLY cannot run
* inside a transaction, which rules out a migration, and outlasts the runtime pool's statement_timeout,
* which is lifted here. The advisory lock lets one replica build while the others answer 501 until the
- * index is valid. Returns why the index cannot be used yet, or null once it is valid.
+ * index is valid (see retryContainsBuild). Returns why the index cannot be used yet, or null once it is
+ * valid.
*/
private async buildPostgresTrigramIndex(): Promise<string | null> {
const runner = this.dataSource.createQueryRunner();
await runner.connect();
try {
const [{ locked }] = await runner.manager.query<Array<{ locked: boolean }>>(
- `SELECT pg_try_advisory_lock($1, $2) AS locked`,
- [...TRIGRAM_BUILD_LOCK_KEYS],
+ `SELECT pg_try_advisory_lock(${TRIGRAM_BUILD_LOCK}) AS locked`,
+ [TRIGRAM_BUILD_LOCK_KEY],
);
if (!locked) return CONTAINS_UNAVAILABLE.building;
try {
@@ -526,7 +559,7 @@ export class BuiltInFtsProvider implements SearchProvider, OnModuleInit, OnAppli
} finally {
// The connection returns to the pool: the next borrower must find its timeout and no lock.
await runner.query('RESET statement_timeout');
- await runner.query('SELECT pg_advisory_unlock($1, $2)', [...TRIGRAM_BUILD_LOCK_KEYS]);
+ await runner.query(`SELECT pg_advisory_unlock(${TRIGRAM_BUILD_LOCK})`, [TRIGRAM_BUILD_LOCK_KEY]);
}
} finally {
await runner.release();