Free, Open Source, Self-Hosted WhatsApp API Gateway
TypeScript apibotgatewayself-hostedwhatsappwhatsapp-apiwhatsapp-automationwhatsapp-botwhatsapp-gatewaywhatsapp-gateway-apiwhatsapp-webwhatsapp-web-api
fix(search): require ASCII trigram sources on a C-ctype PostgreSQL
Yudhi Armyndharisa5ae94008c11
diff --git a/docs/26-global-search.md b/docs/26-global-search.mdindex 3772f1aa3..8bf7bc40c 100644--- a/docs/26-global-search.md+++ b/docs/26-global-search.md@@ -87,7 +87,10 @@ Japanese ones (`東京`) are: search those with `word`. What counts differs by dtrigrams from any three characters, so a term needs three (`id` and `pm` only narrow, `:-)` searches,and a Hangul syllable is one character however it is encoded); `pg_trgm` takes them from letters anddigits only, so on Postgres a term needs three of them in a row (`:-)`, `...`, `😂😂😂`, `a.b` and-superscript digits such as `¹²³` only narrow there, and alone answer `400`). The terms may also hold+superscript digits such as `¹²³` only narrow there, and alone answer `400`). On a database whose ctype+(`LC_CTYPE`, `pg_database.datctype`) is `C` or `POSIX`, `pg_trgm` knows no letter or digit outside+ASCII, so there the three must be ASCII letters or digits once accents are folded: `cafe` and `Café`+search, while `서울역` or `東京都` alone answer `400`. The terms may also holdat most **64 characters** together, spaces not counted, or the query answers `400`: every character ofa term is checked on every message holding the term's trigrams, on SQLite on the event loop everysession shares, and on 200k messages a query of 1,000 characters took 2.8 s where one of 64 took about@@ -235,8 +238,8 @@ GET /api/search?q=<term>&match=<word|contains>&sessionId=<id>&chatId=<id>&directbackend answered (e.g. `builtin-fts`).- **Errors:** `400` empty/whitespace `q` or one holding a NUL character, a non-numeric numeric param,an unknown `match`, or a `match=contains` query with no term the trigram index can look up (three- characters; on Postgres, three letters or digits in a row) or with more than 64 characters in its- terms (an FTS5 query-grammar error is also mapped to `400`, but since the+ characters; on Postgres, three letters or digits in a row, ASCII ones under a `C` ctype) or with more+ than 64 characters in its terms (an FTS5 query-grammar error is also mapped to `400`, but since theSQLite path quotes every token ordinary input no longer reaches that fallback) · `401`/`403` auth ·`501` no provider configured / FTS schema absent (e.g. a non-FTS5 SQLite build) / for`match=contains`, `SEARCH_CONTAINS_ENABLED` off, the index still building, or no way to build it (e.g.diff --git a/src/modules/search/providers/builtin-fts.provider.contains.pg.spec.ts b/src/modules/search/providers/builtin-fts.provider.contains.pg.spec.tsindex 0472e82e5..17d6536fc 100644--- a/src/modules/search/providers/builtin-fts.provider.contains.pg.spec.ts+++ b/src/modules/search/providers/builtin-fts.provider.contains.pg.spec.ts@@ -25,6 +25,8 @@ const POSTGRES_ENABLED = process.env.DATABASE_TYPE === 'postgres';/** Named for the suite so a crashed run leaves an obvious orphan rather than a mystery database. */const SCRATCH_DB = 'openwa_contains_search_probe';+/** The same, created with a C ctype. */+const SCRATCH_C_DB = 'openwa_contains_search_probe_c';const connection = (database: string): DataSourceOptions => ({type: 'postgres',@@ -295,4 +297,43 @@ const containsOn = new ConfigService({ search: { containsEnabled: true } }););});});++ // pg_trgm classifies characters with the C library there, which knows no letter outside ASCII.+ describe('on a database with a C ctype', () => {+ let cds: DataSource;++ beforeAll(async () => {+ await admin.query(`DROP DATABASE IF EXISTS "${SCRATCH_C_DB}"`);+ await admin.query(+ `CREATE DATABASE "${SCRATCH_C_DB}" TEMPLATE template0 ENCODING 'UTF8' LC_COLLATE 'C' LC_CTYPE 'C'`,+ );+ cds = new DataSource({+ ...connection(SCRATCH_C_DB),+ migrations: [join(__dirname, '../../../database/migrations/*{.ts,.js}')],+ });+ await cds.initialize();+ await cds.runMigrations();+ }, 120_000);++ afterAll(async () => {+ await cds?.destroy().catch(() => undefined);+ await admin?.query(`DROP DATABASE IF EXISTS "${SCRATCH_C_DB}"`).catch(() => undefined);+ });++ it('requires three ASCII letters or digits in a row, the only ones pg_trgm takes a trigram from', async () => {+ await cds.query(+ `INSERT INTO "messages" ("id","sessionId","chatId","from","to","body","type","direction","timestamp") ` ++ `VALUES ('c1','s1','c1-chat','c1-from','[email protected]','Lunch at 서울역 with the Café team','text','incoming',10)`,+ );+ const provider = new BuiltInFtsProvider(cds, containsOn);+ await provider.buildContainsIndex();++ expect(await cds.query(`SELECT show_trgm('서울역') AS t`)).toEqual([{ t: [] }]);+ const search = provider.search({ q: '서울역', match: 'contains' });+ await expect(search).rejects.toBeInstanceOf(BadRequestException);+ await expect(search).rejects.toThrow(/3 or more ASCII letters or digits in a row/);+ expect((await provider.search({ q: 'café', match: 'contains' })).total).toBe(1);+ expect((await provider.search({ q: '서울역 lunch', match: 'contains' })).total).toBe(1);+ });+ });});diff --git a/src/modules/search/providers/builtin-fts.provider.ts b/src/modules/search/providers/builtin-fts.provider.tsindex cd1d98403..b41424759 100644--- a/src/modules/search/providers/builtin-fts.provider.ts+++ b/src/modules/search/providers/builtin-fts.provider.ts@@ -93,6 +93,12 @@ const POSTGRES_TRIGRAM_SOURCE = new RegExp('u',);+/**+ * POSTGRES_TRIGRAM_SOURCE on a database whose ctype is C or POSIX: pg_trgm classifies characters with the+ * C library there, which knows no letter or digit outside ASCII, so it takes no trigram from any other.+ */+const POSTGRES_ASCII_TRIGRAM_SOURCE = new RegExp(`[A-Za-z0-9]{${SEARCH_CONTAINS_MIN_TERM_LENGTH}}`);+/*** The contains candidate SELECT list. It carries no snippet or score column: searchContains computes* both in the application (see containsSnippet and containsScore).@@ -164,9 +170,10 @@ function isShortTerm(term: string): boolean {/*** Whether pg_trgm can look `term` up. It takes trigrams from runs of letters and digits only, so `:-)` or* `😂😂😂` give it nothing and the search would scan the table; SQLite's trigram tokenizer indexes them.+ * `asciiOnly` is a database with a C or POSIX ctype (POSTGRES_ASCII_TRIGRAM_SOURCE).*/-function postgresCanIndex(term: string): boolean {- return POSTGRES_TRIGRAM_SOURCE.test(foldSearchText(term));+function postgresCanIndex(term: string, asciiOnly: boolean): boolean {+ return (asciiOnly ? POSTGRES_ASCII_TRIGRAM_SOURCE : POSTGRES_TRIGRAM_SOURCE).test(foldSearchText(term));}/** A build that could not get the shared SQLite connection free of another transaction: it is tried again later. */@@ -221,6 +228,9 @@ export class BuiltInFtsProvider implements SearchProvider, OnModuleInit, OnAppliprivate readonly containsBuildRetryMs = TRIGRAM_BUILD_RETRY_MS;private destroyed = false;+ /** Whether the Postgres database has a C or POSIX ctype, read by each build (see postgresCanIndex). */+ private postgresAsciiTrigrams = false;+// OpenWA has two TypeORM connections (main: auth/audit SQLite, data: messages). Bind explicitly to// 'data' so the provider queries the connection that owns the `messages` table + the FTS migration,// never the default/`main` one. The bare `DataSource` type alone is ambiguous with two connections.@@ -569,6 +579,10 @@ export class BuiltInFtsProvider implements SearchProvider, OnModuleInit, OnAppliconst runner = this.dataSource.createQueryRunner();await runner.connect();try {+ const [{ ctype }] = await runner.manager.query<Array<{ ctype: string }>>(+ `SELECT datctype AS ctype FROM pg_database WHERE datname = current_database()`,+ );+ this.postgresAsciiTrigrams = ctype === 'C' || ctype === 'POSIX';const [{ locked }] = await runner.manager.query<Array<{ locked: boolean }>>(`SELECT pg_try_advisory_lock(${TRIGRAM_BUILD_LOCK}) AS locked`,[TRIGRAM_BUILD_LOCK_KEY],@@ -711,7 +725,7 @@ export class BuiltInFtsProvider implements SearchProvider, OnModuleInit, OnAppliprivate async searchContains(query: SearchQuery): Promise<SearchResults> {await this.ensureContains();const isPostgres = this.dataSource.options.type === 'postgres';- const terms = BuiltInFtsProvider.containsTermsOf(query.q, isPostgres);+ const terms = BuiltInFtsProvider.containsTermsOf(query.q, isPostgres, this.postgresAsciiTrigrams);const start = Date.now();const limit = Math.max(1, Math.min(query.limit ?? 50, SEARCH_LIMIT_MAX));const offset = Math.max(0, query.offset ?? 0);@@ -799,14 +813,14 @@ export class BuiltInFtsProvider implements SearchProvider, OnModuleInit, OnAppli* it takes no trigram from (FTS5 returns nothing for it, pg_trgm scans every row), so such terms only* narrow the rows another term already found, and a query made of them alone is a 400 rather than a* full-table scan. SQLite takes trigrams from any three characters, pg_trgm only from three letters or- * digits in a row (see postgresCanIndex). The terms together are bounded by- * SEARCH_CONTAINS_TERMS_MAX_LENGTH, since each of their characters costs a check on every candidate.+ * digits in a row, ASCII ones on a C or POSIX ctype (see postgresCanIndex). The terms together are bounded+ * by SEARCH_CONTAINS_TERMS_MAX_LENGTH, since each of their characters costs a check on every candidate.*/- private static containsTermsOf(q: string, isPostgres: boolean): string[] {+ private static containsTermsOf(q: string, isPostgres: boolean, asciiOnly: boolean): string[] {const terms = containsTerms(q);- const canIndex = isPostgres ? postgresCanIndex : (term: string) => !isShortTerm(term);+ const canIndex = (term: string) => (isPostgres ? postgresCanIndex(term, asciiOnly) : !isShortTerm(term));if (!terms.some(canIndex)) {- const unit = isPostgres ? 'letters or digits in a row' : 'characters';+ const unit = isPostgres ? `${asciiOnly ? 'ASCII ' : ''}letters or digits in a row` : 'characters';throw new ContainsTermsRefusedError(`A contains search needs at least one term of ${SEARCH_CONTAINS_MIN_TERM_LENGTH} or more ${unit}.`,);