Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(search): require ASCII trigram sources on a C-ctype PostgreSQL

Yudhi Armyndharisa5ae94008c11
diff --git a/docs/26-global-search.md b/docs/26-global-search.md
index 3772f1aa3..8bf7bc40c 100644
--- a/docs/26-global-search.md
+++ b/docs/26-global-search.md
@@ -87,7 +87,10 @@ Japanese ones (`東京`) are: search those with `word`. What counts differs by d
trigrams from any three characters, so a term needs three (`id` and `pm` only narrow, `:-)` searches,
and a Hangul syllable is one character however it is encoded); `pg_trgm` takes them from letters and
digits only, so on Postgres a term needs three of them in a row (`:-)`, `...`, `😂😂😂`, `a.b` and
-superscript digits such as `¹²³` only narrow there, and alone answer `400`). The terms may also hold
+superscript digits such as `¹²³` only narrow there, and alone answer `400`). On a database whose ctype
+(`LC_CTYPE`, `pg_database.datctype`) is `C` or `POSIX`, `pg_trgm` knows no letter or digit outside
+ASCII, so there the three must be ASCII letters or digits once accents are folded: `cafe` and `Café`
+search, while `서울역` or `東京都` alone answer `400`. The terms may also hold
at most **64 characters** together, spaces not counted, or the query answers `400`: every character of
a term is checked on every message holding the term's trigrams, on SQLite on the event loop every
session shares, and on 200k messages a query of 1,000 characters took 2.8 s where one of 64 took about
@@ -235,8 +238,8 @@ GET /api/search?q=<term>&match=<word|contains>&sessionId=<id>&chatId=<id>&direct
backend answered (e.g. `builtin-fts`).
- **Errors:** `400` empty/whitespace `q` or one holding a NUL character, a non-numeric numeric param,
an unknown `match`, or a `match=contains` query with no term the trigram index can look up (three
- characters; on Postgres, three letters or digits in a row) or with more than 64 characters in its
- terms (an FTS5 query-grammar error is also mapped to `400`, but since the
+ characters; on Postgres, three letters or digits in a row, ASCII ones under a `C` ctype) or with more
+ than 64 characters in its terms (an FTS5 query-grammar error is also mapped to `400`, but since the
SQLite path quotes every token ordinary input no longer reaches that fallback) · `401`/`403` auth ·
`501` no provider configured / FTS schema absent (e.g. a non-FTS5 SQLite build) / for
`match=contains`, `SEARCH_CONTAINS_ENABLED` off, the index still building, or no way to build it (e.g.
diff --git a/src/modules/search/providers/builtin-fts.provider.contains.pg.spec.ts b/src/modules/search/providers/builtin-fts.provider.contains.pg.spec.ts
index 0472e82e5..17d6536fc 100644
--- a/src/modules/search/providers/builtin-fts.provider.contains.pg.spec.ts
+++ b/src/modules/search/providers/builtin-fts.provider.contains.pg.spec.ts
@@ -25,6 +25,8 @@ const POSTGRES_ENABLED = process.env.DATABASE_TYPE === 'postgres';
/** Named for the suite so a crashed run leaves an obvious orphan rather than a mystery database. */
const SCRATCH_DB = 'openwa_contains_search_probe';
+/** The same, created with a C ctype. */
+const SCRATCH_C_DB = 'openwa_contains_search_probe_c';
const connection = (database: string): DataSourceOptions => ({
type: 'postgres',
@@ -295,4 +297,43 @@ const containsOn = new ConfigService({ search: { containsEnabled: true } });
);
});
});
+
+ // pg_trgm classifies characters with the C library there, which knows no letter outside ASCII.
+ describe('on a database with a C ctype', () => {
+ let cds: DataSource;
+
+ beforeAll(async () => {
+ await admin.query(`DROP DATABASE IF EXISTS "${SCRATCH_C_DB}"`);
+ await admin.query(
+ `CREATE DATABASE "${SCRATCH_C_DB}" TEMPLATE template0 ENCODING 'UTF8' LC_COLLATE 'C' LC_CTYPE 'C'`,
+ );
+ cds = new DataSource({
+ ...connection(SCRATCH_C_DB),
+ migrations: [join(__dirname, '../../../database/migrations/*{.ts,.js}')],
+ });
+ await cds.initialize();
+ await cds.runMigrations();
+ }, 120_000);
+
+ afterAll(async () => {
+ await cds?.destroy().catch(() => undefined);
+ await admin?.query(`DROP DATABASE IF EXISTS "${SCRATCH_C_DB}"`).catch(() => undefined);
+ });
+
+ it('requires three ASCII letters or digits in a row, the only ones pg_trgm takes a trigram from', async () => {
+ await cds.query(
+ `INSERT INTO "messages" ("id","sessionId","chatId","from","to","body","type","direction","timestamp") ` +
+ `VALUES ('c1','s1','c1-chat','c1-from','[email protected]','Lunch at 서울역 with the Café team','text','incoming',10)`,
+ );
+ const provider = new BuiltInFtsProvider(cds, containsOn);
+ await provider.buildContainsIndex();
+
+ expect(await cds.query(`SELECT show_trgm('서울역') AS t`)).toEqual([{ t: [] }]);
+ const search = provider.search({ q: '서울역', match: 'contains' });
+ await expect(search).rejects.toBeInstanceOf(BadRequestException);
+ await expect(search).rejects.toThrow(/3 or more ASCII letters or digits in a row/);
+ expect((await provider.search({ q: 'café', match: 'contains' })).total).toBe(1);
+ expect((await provider.search({ q: '서울역 lunch', match: 'contains' })).total).toBe(1);
+ });
+ });
});
diff --git a/src/modules/search/providers/builtin-fts.provider.ts b/src/modules/search/providers/builtin-fts.provider.ts
index cd1d98403..b41424759 100644
--- a/src/modules/search/providers/builtin-fts.provider.ts
+++ b/src/modules/search/providers/builtin-fts.provider.ts
@@ -93,6 +93,12 @@ const POSTGRES_TRIGRAM_SOURCE = new RegExp(
'u',
);
+/**
+ * POSTGRES_TRIGRAM_SOURCE on a database whose ctype is C or POSIX: pg_trgm classifies characters with the
+ * C library there, which knows no letter or digit outside ASCII, so it takes no trigram from any other.
+ */
+const POSTGRES_ASCII_TRIGRAM_SOURCE = new RegExp(`[A-Za-z0-9]{${SEARCH_CONTAINS_MIN_TERM_LENGTH}}`);
+
/**
* The contains candidate SELECT list. It carries no snippet or score column: searchContains computes
* both in the application (see containsSnippet and containsScore).
@@ -164,9 +170,10 @@ function isShortTerm(term: string): boolean {
/**
* Whether pg_trgm can look `term` up. It takes trigrams from runs of letters and digits only, so `:-)` or
* `😂😂😂` give it nothing and the search would scan the table; SQLite's trigram tokenizer indexes them.
+ * `asciiOnly` is a database with a C or POSIX ctype (POSTGRES_ASCII_TRIGRAM_SOURCE).
*/
-function postgresCanIndex(term: string): boolean {
- return POSTGRES_TRIGRAM_SOURCE.test(foldSearchText(term));
+function postgresCanIndex(term: string, asciiOnly: boolean): boolean {
+ return (asciiOnly ? POSTGRES_ASCII_TRIGRAM_SOURCE : POSTGRES_TRIGRAM_SOURCE).test(foldSearchText(term));
}
/** A build that could not get the shared SQLite connection free of another transaction: it is tried again later. */
@@ -221,6 +228,9 @@ export class BuiltInFtsProvider implements SearchProvider, OnModuleInit, OnAppli
private readonly containsBuildRetryMs = TRIGRAM_BUILD_RETRY_MS;
private destroyed = false;
+ /** Whether the Postgres database has a C or POSIX ctype, read by each build (see postgresCanIndex). */
+ private postgresAsciiTrigrams = false;
+
// OpenWA has two TypeORM connections (main: auth/audit SQLite, data: messages). Bind explicitly to
// 'data' so the provider queries the connection that owns the `messages` table + the FTS migration,
// never the default/`main` one. The bare `DataSource` type alone is ambiguous with two connections.
@@ -569,6 +579,10 @@ export class BuiltInFtsProvider implements SearchProvider, OnModuleInit, OnAppli
const runner = this.dataSource.createQueryRunner();
await runner.connect();
try {
+ const [{ ctype }] = await runner.manager.query<Array<{ ctype: string }>>(
+ `SELECT datctype AS ctype FROM pg_database WHERE datname = current_database()`,
+ );
+ this.postgresAsciiTrigrams = ctype === 'C' || ctype === 'POSIX';
const [{ locked }] = await runner.manager.query<Array<{ locked: boolean }>>(
`SELECT pg_try_advisory_lock(${TRIGRAM_BUILD_LOCK}) AS locked`,
[TRIGRAM_BUILD_LOCK_KEY],
@@ -711,7 +725,7 @@ export class BuiltInFtsProvider implements SearchProvider, OnModuleInit, OnAppli
private async searchContains(query: SearchQuery): Promise<SearchResults> {
await this.ensureContains();
const isPostgres = this.dataSource.options.type === 'postgres';
- const terms = BuiltInFtsProvider.containsTermsOf(query.q, isPostgres);
+ const terms = BuiltInFtsProvider.containsTermsOf(query.q, isPostgres, this.postgresAsciiTrigrams);
const start = Date.now();
const limit = Math.max(1, Math.min(query.limit ?? 50, SEARCH_LIMIT_MAX));
const offset = Math.max(0, query.offset ?? 0);
@@ -799,14 +813,14 @@ export class BuiltInFtsProvider implements SearchProvider, OnModuleInit, OnAppli
* it takes no trigram from (FTS5 returns nothing for it, pg_trgm scans every row), so such terms only
* narrow the rows another term already found, and a query made of them alone is a 400 rather than a
* full-table scan. SQLite takes trigrams from any three characters, pg_trgm only from three letters or
- * digits in a row (see postgresCanIndex). The terms together are bounded by
- * SEARCH_CONTAINS_TERMS_MAX_LENGTH, since each of their characters costs a check on every candidate.
+ * digits in a row, ASCII ones on a C or POSIX ctype (see postgresCanIndex). The terms together are bounded
+ * by SEARCH_CONTAINS_TERMS_MAX_LENGTH, since each of their characters costs a check on every candidate.
*/
- private static containsTermsOf(q: string, isPostgres: boolean): string[] {
+ private static containsTermsOf(q: string, isPostgres: boolean, asciiOnly: boolean): string[] {
const terms = containsTerms(q);
- const canIndex = isPostgres ? postgresCanIndex : (term: string) => !isShortTerm(term);
+ const canIndex = (term: string) => (isPostgres ? postgresCanIndex(term, asciiOnly) : !isShortTerm(term));
if (!terms.some(canIndex)) {
- const unit = isPostgres ? 'letters or digits in a row' : 'characters';
+ const unit = isPostgres ? `${asciiOnly ? 'ASCII ' : ''}letters or digits in a row` : 'characters';
throw new ContainsTermsRefusedError(
`A contains search needs at least one term of ${SEARCH_CONTAINS_MIN_TERM_LENGTH} or more ${unit}.`,
);