Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

feat(search): build the contains index only behind SEARCH_CONTAINS_ENABLED

Luis15ca47dddc0a
diff --git a/.env.example b/.env.example
index f2c994861..931611f35 100644
--- a/.env.example
+++ b/.env.example
@@ -888,6 +888,10 @@ WEBHOOK_SSRF_PROTECT=true
# SEARCH_ENABLED=true # set to false to disable the /search route + module entirely
# SEARCH_PROVIDER=auto # auto | builtin-fts | none (plugin ids selectable once Spec 2 lands)
# SEARCH_LIMIT_MAX=100 # hard cap on the `limit` query param
+# match=contains (substring search) needs a trigram index the app builds after it starts listening:
+# about 3x the message text in size (289 MB per million messages on SQLite) and a one-time build.
+# Off by default; while off, match=contains answers 501. See docs/26.
+# SEARCH_CONTAINS_ENABLED=false
# =============================================================================
# DASHBOARD STATISTICS
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 1b0c3f950..7bd144185 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -272,7 +272,7 @@ jobs:
# spec's own two-migrator race is in-file (Promise.all) and unaffected by runInBand. The
# synchronize spec additionally CREATEs and DROPs a scratch database of its own, which is
# only safe serialized.
- run: npx jest --runInBand src/database/migrations/__tests__/1782400000000-AddMessagesFts.pg.spec.ts src/database/migrations/__tests__/1792400000000-AddMessagesTrigramSearch.pg.spec.ts src/database/migrations/__tests__/pg-boot-migrations.pg.spec.ts src/modules/message/message-list-ordering.pg.spec.ts src/database/migrations/__tests__/pg-entity-synchronize.pg.spec.ts src/database/postgres-utc.pg.spec.ts src/modules/webhook/webhook-recovery.pg.spec.ts src/modules/message/idempotency/send-idempotency.service.pg.spec.ts src/modules/infra/infra-data.import.pg.spec.ts src/modules/session/session-content-lifetime.spec.ts
+ run: npx jest --runInBand src/database/migrations/__tests__/1782400000000-AddMessagesFts.pg.spec.ts src/modules/search/providers/builtin-fts.provider.contains.pg.spec.ts src/database/migrations/__tests__/pg-boot-migrations.pg.spec.ts src/modules/message/message-list-ordering.pg.spec.ts src/database/migrations/__tests__/pg-entity-synchronize.pg.spec.ts src/database/postgres-utc.pg.spec.ts src/modules/webhook/webhook-recovery.pg.spec.ts src/modules/message/idempotency/send-idempotency.service.pg.spec.ts src/modules/infra/infra-data.import.pg.spec.ts src/modules/session/session-content-lifetime.spec.ts
env:
TZ: Asia/Jakarta
DATABASE_TYPE: postgres
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 5aca03f9b..2cf7b5a7e 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -243,7 +243,7 @@ jobs:
# tree CI had passed minutes earlier). The lock spec's own two-migrator race is in-file
# (Promise.all) and unaffected by runInBand. The synchronize spec additionally CREATEs and
# DROPs a scratch database of its own, which is only safe serialized.
- run: npx jest --runInBand src/database/migrations/__tests__/1782400000000-AddMessagesFts.pg.spec.ts src/database/migrations/__tests__/1792400000000-AddMessagesTrigramSearch.pg.spec.ts src/database/migrations/__tests__/pg-boot-migrations.pg.spec.ts src/modules/message/message-list-ordering.pg.spec.ts src/database/migrations/__tests__/pg-entity-synchronize.pg.spec.ts src/database/postgres-utc.pg.spec.ts src/modules/webhook/webhook-recovery.pg.spec.ts src/modules/message/idempotency/send-idempotency.service.pg.spec.ts src/modules/infra/infra-data.import.pg.spec.ts src/modules/session/session-content-lifetime.spec.ts
+ run: npx jest --runInBand src/database/migrations/__tests__/1782400000000-AddMessagesFts.pg.spec.ts src/modules/search/providers/builtin-fts.provider.contains.pg.spec.ts src/database/migrations/__tests__/pg-boot-migrations.pg.spec.ts src/modules/message/message-list-ordering.pg.spec.ts src/database/migrations/__tests__/pg-entity-synchronize.pg.spec.ts src/database/postgres-utc.pg.spec.ts src/modules/webhook/webhook-recovery.pg.spec.ts src/modules/message/idempotency/send-idempotency.service.pg.spec.ts src/modules/infra/infra-data.import.pg.spec.ts src/modules/session/session-content-lifetime.spec.ts
env:
TZ: Asia/Jakarta
DATABASE_TYPE: postgres
diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
index c9b9c763c..e345d0ab2 100644
--- a/docker-compose.dev.yml
+++ b/docker-compose.dev.yml
@@ -108,6 +108,7 @@ services:
- SIMULATE_TYPING=${SIMULATE_TYPING:-}
- MCP_ENABLED=${MCP_ENABLED:-}
- SEARCH_ENABLED=${SEARCH_ENABLED:-}
+ - SEARCH_CONTAINS_ENABLED=${SEARCH_CONTAINS_ENABLED:-}
- SERVE_DASHBOARD=${SERVE_DASHBOARD:-}
- CACHE_ENABLED=${CACHE_ENABLED:-}
- DATABASE_LOGGING=${DATABASE_LOGGING:-}
diff --git a/docker-compose.yml b/docker-compose.yml
index 4714b7d7d..b4d9a7aba 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -257,6 +257,7 @@ services:
- SIMULATE_TYPING=${SIMULATE_TYPING:-}
- MCP_ENABLED=${MCP_ENABLED:-}
- SEARCH_ENABLED=${SEARCH_ENABLED:-}
+ - SEARCH_CONTAINS_ENABLED=${SEARCH_CONTAINS_ENABLED:-}
- SERVE_DASHBOARD=${SERVE_DASHBOARD:-}
- CACHE_ENABLED=${CACHE_ENABLED:-}
- DATABASE_LOGGING=${DATABASE_LOGGING:-}
diff --git a/docs/05-database-design.md b/docs/05-database-design.md
index 7e1a30607..32b6ec505 100644
--- a/docs/05-database-design.md
+++ b/docs/05-database-design.md
@@ -727,7 +727,7 @@ The data connection also owns:
- **`lid_mappings`** — LID↔phone-number identity mappings (`src/engine/identity/lid-mapping.entity.ts`).
- **`chat_states`** (engine): per-session mute, archive and pin state of each Baileys chat (`src/engine/adapters/baileys-chat-state.entity.ts`), keyed `(sessionId, chatId)`. `chatId` is the person's phone JID (`<phone>@s.whatsapp.net`) whenever their lid resolves, so a state WhatsApp syncs under the lid and one set through the phone number land on the same row; it stays `<lid>@lid` while the lid is unresolved, and a row found under the lid later is folded onto the phone JID on the chat's next change (reads and the fold take each field from the newest row that observed it: `observed` lists the fields a row has seen, and null, on a row not written since the column was added, means the fields it holds a set value for; the next write that can read the row records the list). WhatsApp does not re-deliver that state, so it is kept in backups. A chat deleted on the phone or through the API loses its row, so a later message starts it clean.
-Additionally, the `AddMessagesFts` migration creates the full-text-search structures over `messages` (a FTS5 virtual table on SQLite, a generated `body_ts` `tsvector` column plus GIN index on PostgreSQL) that back the `/search` endpoint, and the `AddMessagesTrigramSearch` migration adds the trigram index behind `/search?match=contains` (a `messages_trgm` FTS5 trigram table on SQLite; the `pg_trgm` and `unaccent` extensions, an `openwa_unaccent(text)` function, a GIN trigram index and a `("createdAt", "id")` b-tree on PostgreSQL).
+Additionally, the `AddMessagesFts` migration creates the full-text-search structures over `messages` (a FTS5 virtual table on SQLite, a generated `body_ts` `tsvector` column plus GIN index on PostgreSQL) that back the `/search` endpoint. With `SEARCH_CONTAINS_ENABLED=true` (off by default) the search provider, not a migration, also builds the trigram index behind `/search?match=contains`: a `messages_trgm` FTS5 trigram table with its sync triggers on SQLite; the `pg_trgm` and `unaccent` extensions, an `openwa_unaccent(text)` function and the `idx_messages_body_trgm` GIN trigram index on PostgreSQL (see [26 - Global Search](./26-global-search.md#269-enabling-matchcontains)).
> [!NOTE]
> **Tables that do _not_ exist.** Earlier drafts referenced `contacts`, `session_logs`, `webhook_logs`, `api_key_logs`, `webhook_idempotency`, and `ip_whitelist`. None of these are implemented. Contacts are read live from the engine; auditing is the single `audit_logs` table; webhook idempotency is not a persisted table; and per-key IP restrictions are stored inline on `api_keys."allowedIps"` (a `simple-array`), not in a separate `ip_whitelist` table.
@@ -931,8 +931,7 @@ src/database/migrations/ # data connection (pluggable)
├── 1790812800000-AddMessageWindowIndexes.ts # compound timestamp indexes for message-time pagination
├── 1791100000000-AddIntegrationDeliveryFailureDeliveryIndex.ts # (pluginId, instanceId, deliveryId) index for the dead-letter guards
├── 1791200000000-AddWebhookOutboxSessionIndex.ts # (sessionId) index for the per-session outbox delete
-├── 1791300000000-BackfillLegacyMessageTypes.ts # converts raw wwebjs message types to the neutral vocabulary, once
-└── 1792400000000-AddMessagesTrigramSearch.ts # match=contains indexes: FTS5 trigram (SQLite) / pg_trgm + unaccent GIN and (createdAt, id) b-tree (Postgres)
+└── 1791300000000-BackfillLegacyMessageTypes.ts # converts raw wwebjs message types to the neutral vocabulary, once
```
> [!NOTE]
diff --git a/docs/06-api-specification.md b/docs/06-api-specification.md
index 279a65265..42fcffca6 100644
--- a/docs/06-api-specification.md
+++ b/docs/06-api-specification.md
@@ -6512,19 +6512,19 @@ Search messages across sessions (active search provider).
**Query parameters**
-| Name | Type | Required | Default | Description |
-| ----------- | ------------------------------- | -------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `q` | string | **Yes** | — | Search term. Must be non-empty after trim; `400` for whitespace-only, over 1000 characters, or over 32 words (punctuation separates words, a symbol or emoji that stands apart counts as a word, a whitespace-separated word said again counts once). Passed to the active provider's native full-text matcher. |
-| `match` | enum (`word` \| `contains`) | No | `word` | `word` matches whole words. `contains` matches any part of a word, ignoring case and accents (`voic` finds "Invoice", `cafe` finds "Café"); every term must appear, at least one term needs 3+ characters (`400` otherwise), and `%`/`_` match literally. It ranks the 1,000 most recently stored matches by match quality (whole word, then word start, then inside a word; newest first within a score), so `total` is at most 1,000. On PostgreSQL it needs the `pg_trgm` and `unaccent` extensions (`501` otherwise). See [26 - Global Search](./26-global-search.md#match-modes). |
-| `sessionId` | string | No | — | Restrict to a single session id (intersected with the key's `allowedSessions` scope). |
-| `chatId` | string | No | — | Restrict to a single chat id. A phone also matches the chat's other stored forms (the other user dialect and any lid that resolves to it). For a key restricted to selected chats this must name an allowed chat (`403` otherwise), and its hits are additionally narrowed to the allowlist. |
-| `direction` | enum (`incoming` \| `outgoing`) | No | — | Filter by message direction. |
-| `type` | string | No | — | Filter by stored message `type` (e.g. `text`, `image`, `video`). Compared against `messages.type`; not an enum validation, any string is accepted and unmatched values simply return no hits. |
-| `from` | string | No | — | Filter by sender; matches `from` or a group message's `author`. A phone also matches any lid that resolves to it. |
-| `dateFrom` | integer (epoch ms) | No | — | Inclusive lower bound on `timestamp`. A non-numeric value is rejected with `400`. |
-| `dateTo` | integer (epoch ms) | No | — | Inclusive upper bound on `timestamp`. A non-numeric value is rejected with `400`. |
-| `limit` | integer (≥ 1) | No | `50` | Max hits to return. Clamped to `SEARCH_LIMIT_MAX` (default `100`). A non-integer value is rejected with `400`. |
-| `offset` | integer (≥ 0) | No | `0` | Pagination offset, at most `100000`. A non-integer or larger value is rejected with `400`. |
+| Name | Type | Required | Default | Description |
+| ----------- | ------------------------------- | -------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `q` | string | **Yes** | — | Search term. Must be non-empty after trim; `400` for whitespace-only, over 1000 characters, or over 32 words (punctuation separates words, a symbol or emoji that stands apart counts as a word, a whitespace-separated word said again counts once). Passed to the active provider's native full-text matcher. |
+| `match` | enum (`word` \| `contains`) | No | `word` | `word` matches whole words. `contains` matches any part of a word, ignoring case and accents (`voic` finds "Invoice", `cafe` finds "Café"); every term must appear, at least one term needs 3+ characters, on PostgreSQL 3+ letters or digits in a row (`400` otherwise), and `%`/`_` match literally. It ranks the 1,000 most recently stored matches by match quality (whole word, then word start, then inside a word; newest first within a score), so `total` is at most 1,000. Opt-in on the built-in provider: `501` unless the server sets `SEARCH_CONTAINS_ENABLED=true` and has finished building the index (on PostgreSQL also the `pg_trgm` and `unaccent` extensions). PostgreSQL also folds letters such as `Ł`, `ß` and `Ø` that SQLite keeps. See [26 - Global Search](./26-global-search.md#match-modes). |
+| `sessionId` | string | No | — | Restrict to a single session id (intersected with the key's `allowedSessions` scope). |
+| `chatId` | string | No | — | Restrict to a single chat id. A phone also matches the chat's other stored forms (the other user dialect and any lid that resolves to it). For a key restricted to selected chats this must name an allowed chat (`403` otherwise), and its hits are additionally narrowed to the allowlist. |
+| `direction` | enum (`incoming` \| `outgoing`) | No | — | Filter by message direction. |
+| `type` | string | No | — | Filter by stored message `type` (e.g. `text`, `image`, `video`). Compared against `messages.type`; not an enum validation, any string is accepted and unmatched values simply return no hits. |
+| `from` | string | No | — | Filter by sender; matches `from` or a group message's `author`. A phone also matches any lid that resolves to it. |
+| `dateFrom` | integer (epoch ms) | No | — | Inclusive lower bound on `timestamp`. A non-numeric value is rejected with `400`. |
+| `dateTo` | integer (epoch ms) | No | — | Inclusive upper bound on `timestamp`. A non-numeric value is rejected with `400`. |
+| `limit` | integer (≥ 1) | No | `50` | Max hits to return. Clamped to `SEARCH_LIMIT_MAX` (default `100`). A non-integer value is rejected with `400`. |
+| `offset` | integer (≥ 0) | No | `0` | Pagination offset, at most `100000`. A non-integer or larger value is rejected with `400`. |
**Response** `200` — `SearchResults`
@@ -6565,12 +6565,13 @@ Search messages across sessions (active search provider).
word) and every key gets it, since it reads only its own message.
**Errors:** `400` empty/whitespace `q`, a non-numeric `dateFrom`/`dateTo`/`limit`/`offset`, an `offset` above `100000`, an unknown `match`, a
-`match=contains` query whose terms are all shorter than 3 characters, or a malformed
+`match=contains` query with no term of 3 or more characters (on PostgreSQL, 3 letters or digits in a row), or a malformed
SQLite FTS5 query (unbalanced quote/paren, bare operator) — Postgres's `websearch_to_tsquery` is
tolerant and has no equivalent · `401` missing/invalid `X-API-Key` · `403` key role below `OPERATOR`, or a chat-restricted key
that names a `chatId` outside its allowlist or uses a plugin provider ·
`501` no search provider configured (including a non-FTS5 SQLite build, where the provider is absent), or for
-`match=contains` the trigram index is absent (PostgreSQL without `pg_trgm` and `unaccent`, or SQLite older than 3.45) ·
+`match=contains` the trigram index is not usable: `SEARCH_CONTAINS_ENABLED` is off, the index is still being built,
+or it cannot be (PostgreSQL without `pg_trgm` and `unaccent`, or SQLite older than 3.45); the message says which ·
`502` the active plugin provider returned an invalid result shape (not retryable until the plugin is fixed) ·
`503` the active plugin provider did not answer (worker not running, timed out, or reported a failure;
retryable). The built-in provider returns neither `502` nor `503`.
diff --git a/docs/07-api-collection.md b/docs/07-api-collection.md
index e10b986fe..0586a2234 100644
--- a/docs/07-api-collection.md
+++ b/docs/07-api-collection.md
@@ -1739,7 +1739,8 @@ cannot be widened via the query. See doc 26 for the provider contract.
Search messages across sessions. `q` is required (non-empty after trim); optional filters are
`sessionId`, `chatId`, `direction` (`incoming` | `outgoing`), `type`, `from`, `dateFrom`/`dateTo`
(epoch ms), plus `limit` (default 50, clamped to `SEARCH_LIMIT_MAX` = 100) and `offset` (at most
-100000). `q` is at most 1000 characters and 32 words, and a longer one answers `400`. A key restricted to
+100000). `match` is `word` (the default, whole words) or `contains` (any part of a word, ignoring case
+and accents), which answers `501` unless the server sets `SEARCH_CONTAINS_ENABLED=true`. `q` is at most 1000 characters and 32 words, and a longer one answers `400`. A key restricted to
selected chats gets `403` for a `chatId` outside its list and for a plugin search provider. Hit
`snippet` fields carry `<mark>` highlight markers — render them as text, never as HTML.
diff --git a/docs/14-migration-guide.md b/docs/14-migration-guide.md
index 2110248b3..9057aa89b 100644
--- a/docs/14-migration-guide.md
+++ b/docs/14-migration-guide.md
@@ -883,6 +883,18 @@ with `CREATE INDEX CONCURRENTLY IF NOT EXISTS` under the same name and columns,
it. A concurrent build that fails leaves an INVALID index under that name, which the migration also
skips: check `pg_index.indisvalid` and drop the invalid index before building it again.
+> [!NOTE]
+> **Enabling `match=contains`.** `SEARCH_CONTAINS_ENABLED=true` (off by default) makes `GET /api/search`
+> and the `MessageSearch` MCP tool accept `match=contains`. No migration builds its index: the search
+> provider does, only once the HTTP listener is up, so the build never keeps a pod from its startup probe.
+> On SQLite it adds the `messages_trgm` trigram table, about three times the message text (289 MB and a
+> 9.4 s backfill on a synthetic million messages), backfilled in batches that yield to the event loop.
+> On PostgreSQL it creates the `pg_trgm` and `unaccent` extensions when the role may, and builds the
+> `idx_messages_body_trgm` GIN index with `CREATE INDEX CONCURRENTLY`, so writes go on (76 MB on 300,000
+> messages, next to 30 MB for the word index); one replica builds while the others wait. `match=contains`
+> answers `501` until the index is ready. Turning the setting off later keeps the index and its upkeep;
+> [26.9](./26-global-search.md#269-enabling-matchcontains) lists the statements that drop it.
+
### Known Upgrade Hazards
| Release | Change | Action |
diff --git a/docs/26-global-search.md b/docs/26-global-search.md
index ad5d140ce..d57e8f8a2 100644
--- a/docs/26-global-search.md
+++ b/docs/26-global-search.md
@@ -68,7 +68,9 @@ never assign the raw snippet to `innerHTML`.
### Match modes
The `match` query parameter picks how `q` matches a message body. Each mode has its own DB-maintained
-index, so both are index lookups, never table scans:
+index, so both are index lookups, never table scans. The `contains` index is opt-in
+(`SEARCH_CONTAINS_ENABLED=true`, see [26.9](#269-enabling-matchcontains)); while it is off or still
+building, `match=contains` answers `501`:
| `match` | Matches | Index | Ranking |
| ---------------- | ------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------- |
@@ -77,10 +79,13 @@ index, so both are index lookups, never table scans:
`contains` finds everything `word` finds, plus partial words. Every whitespace-separated term must
appear, as with `word`; a repeated term (also one that differs only in case or accents) runs once. A
-trigram index cannot look up a term shorter than three characters, so such a term (`id`, `pm`) only
-narrows what a longer term found, and a query made of short terms alone answers `400`. The query
-grammar of `word` mode (quoted phrases, `OR`, `-` on Postgres) does not apply: every term, `%` and `_`
-included, matches literally.
+trigram index cannot look up a term it takes no trigram from, so such a term only narrows what another
+term found, and a query made of such terms alone answers `400`. What counts differs by dialect: SQLite
+takes trigrams from any three characters, so a term needs three (`id` and `pm` only narrow, `:-)`
+searches); `pg_trgm` takes them from letters and digits only, so on Postgres a term needs three of them
+in a row (`:-)`, `...`, `😂😂😂` and `a.b` only narrow there, and alone answer `400`). The query grammar of
+`word` mode (quoted phrases, `OR`, `-` on Postgres) does not apply: every term, `%` and `_` included,
+matches literally.
**Ranking runs in two phases.** Ranking every match in SQL costs time in proportion to the number of
matches: measured on 1M messages with a term found in 200k of them, about 100 ms on SQLite (where that
@@ -88,13 +93,13 @@ time blocks the event loop every session shares) and 1.6 s on Postgres. So `cont
1. The database returns at most **1,000 candidates**: the most recently stored messages that contain
every term, newest first. Both dialects answer that order from an index and stop at 1,000, however
- many messages match. SQLite reads its trigram index in insertion order; Postgres walks the
- `idx_messages_created_at_id` b-tree backward for a common term, or reads the few matches of a rare
- one from the GIN index.
+ many messages match. SQLite reads its trigram index in insertion order; Postgres walks the existing
+ `IDX_messages_createdAt` b-tree backward for a common term, or reads the few matches of a rare one
+ from the GIN index.
2. The application ranks the candidates and pages them: each term scores **3** as a whole word, **2**
at the start of a word and **1** inside a word (compared without case or accents), the hit's `score`
is the sum over the terms, and ties go to the newest message, then to `messageId`. The same code
- ranks both dialects, so they return the same order.
+ ranks both dialects, so the same candidates come back in the same order.
Both phases together take about 2 ms on SQLite and 10 to 20 ms on Postgres at that size. The trade-off is that relevance orders the window, not the whole history: `total` is at most
1,000 in this mode, and pages past it are empty. Reach older matches by narrowing the query (more
@@ -105,19 +110,27 @@ A `contains` snippet is built by the application rather than the database (FTS5'
its window in trigrams, and Postgres has no headline function for a substring), so both dialects return
the same text, under the same `<mark>` contract and the same "not HTML-escaped" rule.
-**Requirements.** On SQLite, `contains` needs FTS5 and SQLite 3.45+ (the first with `remove_diacritics`
-for the trigram tokenizer); the bundled `better-sqlite3` ships both. On Postgres it needs the
-`pg_trgm` and `unaccent` extensions, which the migration creates when the database role may (both are
-trusted extensions from PostgreSQL 13, so a database owner can). When it may not, the migration skips
-them without failing the boot, only `match=contains` answers `501` (with the `CREATE EXTENSION`
-statements to run), and `match=word` keeps working; once an operator installs the extensions, the next
-boot builds the wrapper function and the indexes.
+**The dialects fold differently.** Both drop accents a letter decomposes into (`é`, `ã`, `ç`, `ñ`), but
+Postgres's `unaccent` also transliterates letters that have no decomposition: `Ł`, `ß`, `Ø`, `ı`, `œ`
+and `ё` become `L`, `ss`, `O`, `i`, `oe` and `е`. So on Postgres `lodz` finds "Łódź", `strasse` finds
+"Straße" and `orsted` finds "Ørsted", while SQLite's `remove_diacritics` keeps those letters and finds
+none of them. The application compares and highlights the way SQLite folds, so those extra Postgres hits
+come back with `score` 0 and a snippet without `<mark>`.
+
+**Requirements.** `SEARCH_CONTAINS_ENABLED=true`. On SQLite, `contains` needs FTS5 and SQLite 3.45+
+(the first with `remove_diacritics` for the trigram tokenizer); the bundled `better-sqlite3` ships both.
+On Postgres it needs the `pg_trgm` and `unaccent` extensions, which the provider creates when the
+database role may (both are trusted extensions from PostgreSQL 13, so a database owner can). When it
+may not, only `match=contains` answers `501` (with the `CREATE EXTENSION` statements to run), and
+`match=word` keeps working; once an operator installs the extensions, the next boot builds the wrapper
+function and the index.
## 26.4 Dual-database switching safety
Search is designed to work identically on SQLite and PostgreSQL and to survive repeated switching
between them (e.g. developing on SQLite, deploying on Postgres, exporting a SQLite DB and importing
-it into Postgres). Concretely:
+it into Postgres); `match=contains` folds a few letters differently on each, see
+[Match modes](#match-modes). Concretely:
- **Idempotent, dialect-branched migration.** The FTS migration (`1782400000000-AddMessagesFts`)
branches on the active dialect and uses `IF NOT EXISTS` / `IF NOT` guards, so `migration:run` is
@@ -141,11 +154,12 @@ it into Postgres). Concretely:
All search configuration lives in the environment (`.env` / Compose):
-| Variable | Default | Meaning |
-| ------------------ | -------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `SEARCH_ENABLED` | `true` (unset) | Set to `false` to remove the `/search` route and the entire search module — zero footprint, no DI wiring. The migration still runs (so the index is ready if you re-enable). |
-| `SEARCH_PROVIDER` | `auto` | `auto` selects the built-in provider at runtime and lets an enabled plugin provider supersede it; `builtin-fts` pins the built-in explicitly; `none` leaves the registry empty — the module and the route stay mounted, but with no active provider `/api/search` answers `501`. Those three are the only accepted values — anything else (including a plugin id) fails boot. |
-| `SEARCH_LIMIT_MAX` | `100` | Hard cap applied to the `limit` query parameter, so a caller cannot request an unbounded result set. |
+| Variable | Default | Meaning |
+| ------------------------- | -------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `SEARCH_ENABLED` | `true` (unset) | Set to `false` to remove the `/search` route and the entire search module — zero footprint, no DI wiring. The migration still runs (so the index is ready if you re-enable). |
+| `SEARCH_PROVIDER` | `auto` | `auto` selects the built-in provider at runtime and lets an enabled plugin provider supersede it; `builtin-fts` pins the built-in explicitly; `none` leaves the registry empty — the module and the route stay mounted, but with no active provider `/api/search` answers `501`. Those three are the only accepted values — anything else (including a plugin id) fails boot. |
+| `SEARCH_LIMIT_MAX` | `100` | Hard cap applied to the `limit` query parameter, so a caller cannot request an unbounded result set. |
+| `SEARCH_CONTAINS_ENABLED` | `false` | Set to `true` to accept `match=contains`: the built-in provider then builds its trigram index after the HTTP listener is up (see [26.9](#269-enabling-matchcontains)), and the `MessageSearch` MCP tool searches with `match=contains` by default. While it is `false`, `match=contains` answers `501` naming this variable, and the tool defaults to `match=word`. Like the other booleans, a value other than `true` or `false` fails boot. |
### The opt-out footprint note
@@ -206,12 +220,14 @@ GET /api/search?q=<term>&match=<word|contains>&sessionId=<id>&chatId=<id>&direct
when the page could be full). `tookMs` is the provider-side query time. `provider` names which
backend answered (e.g. `builtin-fts`).
- **Errors:** `400` empty/whitespace `q`, a non-numeric numeric param, an unknown `match`, or a
- `match=contains` query whose terms are all shorter than three characters (an FTS5 query-grammar error
- is also mapped to `400`, but since the SQLite path quotes every token ordinary input no longer reaches
- that fallback) · `401`/`403` auth · `501` no provider configured / FTS schema absent (e.g. a non-FTS5
- SQLite build) / for `match=contains`, the trigram index absent (e.g. Postgres without `pg_trgm` and
- `unaccent`) · `502` a plugin provider returned a malformed `SearchResults` payload · `503` a plugin
- provider whose worker timed out or failed (the built-in provider does not return it).
+ `match=contains` query with no term the trigram index can look up (three characters; on Postgres,
+ three letters or digits in a row) (an FTS5 query-grammar error is also mapped to `400`, but since the
+ SQLite path quotes every token ordinary input no longer reaches that fallback) · `401`/`403` auth ·
+ `501` no provider configured / FTS schema absent (e.g. a non-FTS5 SQLite build) / for
+ `match=contains`, `SEARCH_CONTAINS_ENABLED` off, the index still building, or no way to build it (e.g.
+ Postgres without `pg_trgm` and `unaccent`), with a message that says which · `502` a plugin provider
+ returned a malformed `SearchResults` payload · `503` a plugin provider whose worker timed out or failed
+ (the built-in provider does not return it).
The endpoint requires at least `OPERATOR` role.
@@ -266,19 +282,45 @@ Adding search to an existing deployment runs the one-time `1782400000000-AddMess
The migration is safe to re-run (idempotent guards) and has a `down` path that drops the FTS schema on
both dialects. After it applies, search works immediately — no separate reindex command.
-`match=contains` adds the one-time `1792400000000-AddMessagesTrigramSearch` migration:
-
-- **PostgreSQL** creates the `pg_trgm` and `unaccent` extensions inside a savepoint (a role that may
- not create them rolls back to it and the migration stops there, see [Match modes](#match-modes)), the
- IMMUTABLE `openwa_unaccent(text)` wrapper, a `GIN` trigram index on `openwa_unaccent(body)` and the
- `idx_messages_created_at_id` b-tree on `("createdAt", "id")` that serves the candidate window (the
- existing message indexes all lead with `sessionId`). Both are built **non-`CONCURRENTLY`**: plan a
- maintenance window for a very large table.
-- **SQLite** creates the `messages_trgm` FTS5 trigram table, backfills it like `messages_fts` and
- installs its own three sync triggers. A trigram index holds every three-character window of the
- body, so it takes noticeably more space than the word index.
-
-Its `down` path drops what it created except the Postgres extensions, which other objects may use.
+## 26.9 Enabling `match=contains`
+
+`SEARCH_CONTAINS_ENABLED=true` is the only switch: no migration creates the trigram index, because every
+installation would pay for it while only `match=contains` callers (the REST route and the `MessageSearch`
+MCP tool; the dashboard searches by word) use it. The built-in provider builds it instead, starting only
+once the HTTP listener is up and without holding the boot, so a long build cannot keep the process from
+answering its health probe (a boot that outlasts the Helm startup probe is restarted in a loop, see
+[14 - Migration Guide](./14-migration-guide.md)). Until the build finishes, `match=contains` answers `501`.
+
+- **SQLite** creates the `messages_trgm` FTS5 trigram table and its three sync triggers, then backfills
+ the existing rows 1,000 rowids at a time, yielding to the event loop between batches; rows stored
+ meanwhile are indexed by the triggers. A trigram index holds every three-character window of the body:
+ on a synthetic corpus of 1,000,000 messages it took 289 MB, about 3.1 times the message text, and a
+ 9.4 s backfill. Later boots only check it and re-add what is missing, so an interrupted backfill
+ resumes where it stopped.
+- **PostgreSQL** creates the `pg_trgm` and `unaccent` extensions when they are missing and the role may,
+ the IMMUTABLE `openwa_unaccent(text)` wrapper, and the `idx_messages_body_trgm` GIN index on
+ `openwa_unaccent(body)` with `CREATE INDEX CONCURRENTLY`, so writes go on during the build (it cannot
+ run inside a migration's transaction). On 300,000 messages the index took 76 MB, next to 30 MB for the
+ word index. The build runs on its own connection without the runtime `statement_timeout`. Replicas
+ that start together take a session advisory lock: one builds, the others answer `501` until the
+ index is valid. A build that died (a killed pod) leaves an INVALID index, which the next build drops
+ and rebuilds. The candidate query walks the existing `IDX_messages_createdAt` index, so contains search
+ adds no other index.
+
+**Turning it off** later stops `match=contains` (`501` again) but keeps the index, and with it the cost
+on every write, so turning it back on needs no rebuild. To reclaim the space, drop it by hand:
+
+```sql
+-- SQLite
+DROP TRIGGER IF EXISTS messages_trgm_ai;
+DROP TRIGGER IF EXISTS messages_trgm_ad;
+DROP TRIGGER IF EXISTS messages_trgm_au;
+DROP TABLE IF EXISTS messages_trgm;
+
+-- PostgreSQL (the extensions stay: other objects may use them)
+DROP INDEX CONCURRENTLY IF EXISTS idx_messages_body_trgm;
+DROP FUNCTION IF EXISTS openwa_unaccent(text);
+```
---
diff --git a/docs/27-plugin-search-providers.md b/docs/27-plugin-search-providers.md
index ac7d0cea8..48075032a 100644
--- a/docs/27-plugin-search-providers.md
+++ b/docs/27-plugin-search-providers.md
@@ -48,7 +48,7 @@ returns a `SearchResults`:
```ts
ctx.registerSearchProvider(async (query) => {
- // query: SearchQuery, { q, sessionIds?, sessionId?, chatId?, direction?, type?, from?, dateFrom?, dateTo?, limit?, offset? }
+ // query: SearchQuery, { q, match?, sessionIds?, sessionId?, chatId?, direction?, type?, from?, dateFrom?, dateTo?, limit?, offset? }
// Run your backend's query here (e.g. a Meilisearch /search call).
return {
hits: [...], // SearchHit[] — see below
@@ -59,6 +59,10 @@ ctx.registerSearchProvider(async (query) => {
});
```
+`match` is `'word'` (also when absent) or `'contains'` (any part of a word, ignoring case and accents);
+treat it as your backend allows. `SEARCH_CONTAINS_ENABLED` only gates the built-in provider's trigram
+index, so a plugin receives `contains` whatever that setting says.
+
A plugin may register **one** search handler (calling `registerSearchProvider` again replaces it; the
host is notified once). If `onEnable` throws after registering, the host cleans up (the provider is
unregistered).
diff --git a/openapi.json b/openapi.json
index 21dd66399..4a48f0a7e 100644
--- a/openapi.json
+++ b/openapi.json
@@ -17874,7 +17874,7 @@
"name": "match",
"required": false,
"in": "query",
- "description": "word (default): whole words. contains: any part of a word, ignoring case and accents; ranks the 1000 most recently stored matches by match quality",
+ "description": "word (default): whole words. contains: any part of a word, ignoring case and accents; ranks the 1000 most recently stored matches by match quality; needs SEARCH_CONTAINS_ENABLED=true",
"schema": {
"enum": [
"word",
@@ -17984,7 +17984,7 @@
}
},
"400": {
- "description": "Empty or whitespace-only \"q\", or one over 1000 characters or 32 words; an unknown \"match\"; or a \"match=contains\" query whose terms are all shorter than 3 characters",
+ "description": "Empty or whitespace-only \"q\", or one over 1000 characters or 32 words; an unknown \"match\"; or a \"match=contains\" query with no term of 3 or more characters (on PostgreSQL, 3 letters or digits in a row)",
"content": {
"application/json": {
"schema": {
@@ -18014,7 +18014,7 @@
}
},
"501": {
- "description": "No search provider configured, or the built-in provider lacks the index the requested match mode needs (for \"match=contains\" on PostgreSQL: the pg_trgm and unaccent extensions)",
+ "description": "No search provider configured, or the built-in provider lacks the index the requested match mode needs (for \"match=contains\": SEARCH_CONTAINS_ENABLED=true and a finished index build, and on PostgreSQL the pg_trgm and unaccent extensions)",
"content": {
"application/json": {
"schema": {
diff --git a/sdk/go/types_search.go b/sdk/go/types_search.go
index bf33be5de..83a70dd44 100644
--- a/sdk/go/types_search.go
+++ b/sdk/go/types_search.go
@@ -5,7 +5,7 @@ import "net/url"
// SearchQuery is the query for GET /search. Q is required; all else optional.
// DateFrom/DateTo are epoch-ms. Match is "word" (the server default: whole
// words) or "contains" (any part of a word, ignoring case and accents; needs a
-// term of 3+ characters).
+// term of 3+ characters, and a server with SEARCH_CONTAINS_ENABLED=true, else 501).
type SearchQuery struct {
Q string
Match *string
diff --git a/sdk/java/src/main/java/com/rmyndharis/openwa/model/SearchMatch.java b/sdk/java/src/main/java/com/rmyndharis/openwa/model/SearchMatch.java
index 6688533dd..dd57279f5 100644
--- a/sdk/java/src/main/java/com/rmyndharis/openwa/model/SearchMatch.java
+++ b/sdk/java/src/main/java/com/rmyndharis/openwa/model/SearchMatch.java
@@ -5,7 +5,8 @@ import com.google.gson.annotations.SerializedName;
/**
* How {@code GET /search} matches {@code q}. {@link #WORD} (the server default) matches whole words;
* {@link #CONTAINS} matches any part of a word, ignoring case and accents, and needs at least one
- * term of 3+ characters. Wire values are lowercase.
+ * term of 3+ characters and a server with {@code SEARCH_CONTAINS_ENABLED=true} (501 otherwise). Wire
+ * values are lowercase.
*/
public enum SearchMatch {
@SerializedName("word")
diff --git a/sdk/javascript/src/types.ts b/sdk/javascript/src/types.ts
index 611bbd157..edaac5930 100644
--- a/sdk/javascript/src/types.ts
+++ b/sdk/javascript/src/types.ts
@@ -1432,7 +1432,8 @@ export interface SendProductRequest {
/**
* How `q` matches: `word` (the server default) matches whole words; `contains` matches any part of a
- * word, ignoring case and accents, and needs at least one term of 3+ characters.
+ * word, ignoring case and accents, and needs at least one term of 3+ characters and a server with
+ * `SEARCH_CONTAINS_ENABLED=true` (501 otherwise).
*/
export type SearchMatch = 'word' | 'contains';
diff --git a/sdk/python/README.md b/sdk/python/README.md
index 713251ba2..0a64cedc0 100644
--- a/sdk/python/README.md
+++ b/sdk/python/README.md
@@ -55,8 +55,10 @@ client = OpenWAClient(base_url="…", api_key="…", transport=httpx.MockTranspo
## Search
`GET /search` is wrapped as `client.search.search(params)`. Only `q` is required;
-the rest (`sessionId`, `chatId`, `direction`, `type`, `from`, `dateFrom`,
+the rest (`match`, `sessionId`, `chatId`, `direction`, `type`, `from`, `dateFrom`,
`dateTo`, `limit`, `offset`) are optional. `dateFrom` / `dateTo` are epoch-ms.
+`match` is `word` (the default) or `contains`, which the server answers with 501
+unless it sets `SEARCH_CONTAINS_ENABLED=true`.
The active search provider (built-in DB full-text, or a plugin) answers; if none
is configured the server returns 501.
diff --git a/sdk/python/openwa/types.py b/sdk/python/openwa/types.py
index 961adf5bf..0dfff57a7 100644
--- a/sdk/python/openwa/types.py
+++ b/sdk/python/openwa/types.py
@@ -1454,7 +1454,8 @@ class _SearchQueryRequired(TypedDict):
# `match` is "word" (the server default: whole words) or "contains" (any part
-# of a word, ignoring case and accents; needs a term of 3+ characters).
+# of a word, ignoring case and accents; needs a term of 3+ characters, and a
+# server with SEARCH_CONTAINS_ENABLED=true, else 501).
_SearchQueryOptional = TypedDict(
"_SearchQueryOptional",
{
diff --git a/src/config/configuration.spec.ts b/src/config/configuration.spec.ts
index e58b01f4b..f54f7a8eb 100644
--- a/src/config/configuration.spec.ts
+++ b/src/config/configuration.spec.ts
@@ -261,7 +261,7 @@ describe('configuration — webhook fan-out knobs are fail-safe', () => {
describe('configuration search namespace', () => {
// Save/restore the SEARCH_* env vars so a CI .env that sets them cannot flake the default-value
// assertions below (mirrors the mutate-and-restore pattern used for PLUGIN_DOWNLOAD_MAX_BYTES etc.).
- const keys = ['SEARCH_ENABLED', 'SEARCH_PROVIDER'];
+ const keys = ['SEARCH_ENABLED', 'SEARCH_PROVIDER', 'SEARCH_CONTAINS_ENABLED'];
const orig: Record<string, string | undefined> = {};
beforeEach(() => keys.forEach(k => (orig[k] = process.env[k])));
afterEach(() =>
@@ -274,7 +274,7 @@ describe('configuration search namespace', () => {
it('exposes search defaults', () => {
keys.forEach(k => delete process.env[k]);
const cfg = configuration();
- expect(cfg.search).toEqual({ enabled: true, provider: 'auto' });
+ expect(cfg.search).toEqual({ enabled: true, provider: 'auto', containsEnabled: false });
});
});
diff --git a/src/config/configuration.ts b/src/config/configuration.ts
index f04fd6749..c9ad56612 100644
--- a/src/config/configuration.ts
+++ b/src/config/configuration.ts
@@ -129,10 +129,12 @@ export default () => ({
// Global message search. Opt-out via SEARCH_ENABLED=false. Provider defaults to 'auto' (the
// built-in DB full-text provider — Postgres tsvector/GIN, SQLite FTS5); 'none' disables the
- // /search route + module at runtime while keeping the config namespace loaded.
+ // /search route + module at runtime while keeping the config namespace loaded. match=contains is
+ // opt-in (SEARCH_CONTAINS_ENABLED=true): its trigram index costs about three times the message text.
search: {
enabled: process.env.SEARCH_ENABLED !== 'false',
provider: process.env.SEARCH_PROVIDER || 'auto',
+ containsEnabled: process.env.SEARCH_CONTAINS_ENABLED === 'true',
},
// Dashboard statistics. The /stats aggregates run GROUP BY scans over the whole messages
diff --git a/src/config/env-precedence.ts b/src/config/env-precedence.ts
index 19b9f3951..2c318c6a4 100644
--- a/src/config/env-precedence.ts
+++ b/src/config/env-precedence.ts
@@ -105,6 +105,7 @@ export const BLANK_SHADOWED_ENV_KEYS: string[] = [
'SIMULATE_TYPING',
'MCP_ENABLED',
'SEARCH_ENABLED',
+ 'SEARCH_CONTAINS_ENABLED',
'SERVE_DASHBOARD',
'CACHE_ENABLED',
'DATABASE_LOGGING',
diff --git a/src/config/env.validation.spec.ts b/src/config/env.validation.spec.ts
index ca86a6594..112638cbb 100644
--- a/src/config/env.validation.spec.ts
+++ b/src/config/env.validation.spec.ts
@@ -427,6 +427,12 @@ describe('validateEnv', () => {
expect(() => validateEnv({})).not.toThrow();
});
+ it('rejects a SEARCH_CONTAINS_ENABLED typo instead of silently leaving contains search off', () => {
+ expect(() => validateEnv({ SEARCH_CONTAINS_ENABLED: 'yes' })).toThrow(/SEARCH_CONTAINS_ENABLED/);
+ expect(() => validateEnv({ SEARCH_CONTAINS_ENABLED: 'true' })).not.toThrow();
+ expect(() => validateEnv({ SEARCH_CONTAINS_ENABLED: '' })).not.toThrow();
+ });
+
it('rejects a LOG_LEVEL misspelling instead of silently logging at info', () => {
// Plausible spellings from neighbouring vocabularies that this repo's LogLevel does not carry;
// every one of them silently meant INFO before this check existed.
diff --git a/src/config/env.validation.ts b/src/config/env.validation.ts
index 0c4b0b572..260ec5695 100644
--- a/src/config/env.validation.ts
+++ b/src/config/env.validation.ts
@@ -597,6 +597,7 @@ export function validateEnv(config: EnvConfig): EnvConfig {
'RESOLVE_LID_TO_PHONE',
'SIMULATE_TYPING',
'SEARCH_ENABLED',
+ 'SEARCH_CONTAINS_ENABLED',
// Read at boot by the throttler factory (app.module.ts) and CacheService with `=== 'true'`: a
// typo like `ture` silently downgrades rate-limit storage + cache to per-process in-memory.
'REDIS_ENABLED',
diff --git a/src/core/agent-tools/agent-tools.module.ts b/src/core/agent-tools/agent-tools.module.ts
index 61d7ece0f..6e8bd7993 100644
--- a/src/core/agent-tools/agent-tools.module.ts
+++ b/src/core/agent-tools/agent-tools.module.ts
@@ -1,4 +1,5 @@
import { Global, Module } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
import { ToolRegistryService } from './tool-registry.service';
import { SessionModule } from '../../modules/session/session.module';
import { MessageModule } from '../../modules/message/message.module';
@@ -33,6 +34,7 @@ import { allAgentTools } from './tools';
AutomationRulesService,
// Optional: SEARCH_ENABLED=false leaves SearchModule (global, so no import here) out of the app.
{ token: SearchService, optional: true },
+ ConfigService,
],
useFactory: (
session: SessionService,
@@ -42,10 +44,21 @@ import { allAgentTools } from './tools';
webhook: WebhookService,
labels: LabelService,
automation: AutomationRulesService,
- search?: SearchService,
+ search: SearchService | undefined,
+ config: ConfigService,
) =>
new ToolRegistryService(
- allAgentTools({ session, message, contact, group, webhook, labels, automation, search }),
+ allAgentTools({
+ session,
+ message,
+ contact,
+ group,
+ webhook,
+ labels,
+ automation,
+ search,
+ searchContainsEnabled: config.get<boolean>('search.containsEnabled') === true,
+ }),
),
},
],
diff --git a/src/core/agent-tools/tools/index.ts b/src/core/agent-tools/tools/index.ts
index 40236bea2..55ead1bee 100644
--- a/src/core/agent-tools/tools/index.ts
+++ b/src/core/agent-tools/tools/index.ts
@@ -27,6 +27,8 @@ export interface AgentToolDeps {
automation: AutomationRulesService;
/** Absent when SEARCH_ENABLED=false omits SearchModule; MessageSearch then answers 501. */
search?: SearchService;
+ /** SEARCH_CONTAINS_ENABLED: MessageSearch defaults to match=contains only when it is on. */
+ searchContainsEnabled?: boolean;
}
/**
@@ -46,6 +48,6 @@ export function allAgentTools(deps: AgentToolDeps): AnyToolDescriptor[] {
...webhookTools(deps.webhook),
...labelTools(deps.labels),
...automationTools(deps.automation),
- ...searchTools(deps.search),
+ ...searchTools(deps.search, deps.searchContainsEnabled),
];
}
diff --git a/src/core/agent-tools/tools/search.tools.spec.ts b/src/core/agent-tools/tools/search.tools.spec.ts
index 2e7c71cb4..696b426c8 100644
--- a/src/core/agent-tools/tools/search.tools.spec.ts
+++ b/src/core/agent-tools/tools/search.tools.spec.ts
@@ -1,6 +1,7 @@
import { BadRequestException, ForbiddenException, NotImplementedException } from '@nestjs/common';
import { ChatScopeService } from '../../../modules/auth/chat-scope.service';
import type { AuthService } from '../../../modules/auth/auth.service';
+import { ApiKeyRole } from '../../../modules/auth/entities/api-key.entity';
import type { SearchService } from '../../../modules/search/search.service';
import { invokeTool } from '../tool-invoker';
import { searchTools } from './search.tools';
@@ -18,7 +19,7 @@ function authFor(key: object): AuthService {
describe('MessageSearch tool', () => {
const search = jest.fn().mockResolvedValue(results);
- const tool = searchTools({ search } as unknown as SearchService)[0];
+ const tool = searchTools({ search } as unknown as SearchService, true)[0];
const call = (key: object, input: unknown) =>
invokeTool(tool, input, 'key', authFor(key), undefined, undefined, new ChatScopeService());
@@ -33,6 +34,17 @@ describe('MessageSearch tool', () => {
expect(search).toHaveBeenCalledWith(input, ['s1'], undefined);
});
+ it('requires OPERATOR: a below-role key is refused before the search runs', async () => {
+ expect(tool.requiredRole).toBe(ApiKeyRole.OPERATOR);
+
+ const auth = authFor({ id: 'k1', allowedSessions: null, allowedChats: null });
+ (auth.hasPermission as jest.Mock).mockReturnValue(false);
+ await expect(
+ invokeTool(tool, { q: 'invoice' }, 'key', auth, undefined, undefined, new ChatScopeService()),
+ ).rejects.toBeInstanceOf(ForbiddenException);
+ expect(search).not.toHaveBeenCalled();
+ });
+
it('searches every session with match=contains when the agent names neither', async () => {
await call({ id: 'k1', allowedSessions: null, allowedChats: null }, { q: 'invoice' });
@@ -74,6 +86,34 @@ describe('MessageSearch tool', () => {
});
});
+ describe('while SEARCH_CONTAINS_ENABLED is off', () => {
+ const wordTool = searchTools({ search } as unknown as SearchService)[0];
+ const callWord = (input: unknown) =>
+ invokeTool(
+ wordTool,
+ input,
+ 'key',
+ authFor({ id: 'k1', allowedSessions: null, allowedChats: null }),
+ undefined,
+ undefined,
+ new ChatScopeService(),
+ );
+
+ it('defaults to match=word, as REST does', async () => {
+ await callWord({ q: 'invoice' });
+
+ expect(search).toHaveBeenCalledWith({ q: 'invoice', match: 'word' }, undefined, undefined);
+ expect(wordTool.description).toContain('SEARCH_CONTAINS_ENABLED=true');
+ });
+
+ it('passes an explicit match=contains through and surfaces the 501 the provider answers', async () => {
+ search.mockRejectedValueOnce(new NotImplementedException('Contains search is disabled'));
+
+ await expect(callWord({ q: 'invoice', match: 'contains' })).rejects.toBeInstanceOf(NotImplementedException);
+ expect(search).toHaveBeenCalledWith({ q: 'invoice', match: 'contains' }, undefined, undefined);
+ });
+ });
+
it('answers 501 when the search module is disabled', async () => {
const disabled = searchTools(undefined)[0];
diff --git a/src/core/agent-tools/tools/search.tools.ts b/src/core/agent-tools/tools/search.tools.ts
index 2f3d54dd9..6f42a0781 100644
--- a/src/core/agent-tools/tools/search.tools.ts
+++ b/src/core/agent-tools/tools/search.tools.ts
@@ -19,8 +19,17 @@ import { defineTool, type AnyToolDescriptor } from '../tool-descriptor';
* `search` is undefined when SEARCH_ENABLED=false leaves SearchModule out. The tool is still
* registered, so the published tool surface (and the counts the README states) does not depend on
* configuration; calling it answers 501, as the REST route answers 404 then.
+ *
+ * `containsEnabled` is SEARCH_CONTAINS_ENABLED. With it on, `match` defaults to contains, unlike REST
+ * (word): an agent often knows only part of a word or types it without accents, and contains finds
+ * everything word finds. With it off the built-in provider answers 501 to contains, so the tool
+ * defaults to word like REST.
*/
-export function searchTools(search: SearchService | undefined): AnyToolDescriptor[] {
+export function searchTools(search: SearchService | undefined, containsEnabled = false): AnyToolDescriptor[] {
+ const defaultMatch = containsEnabled ? 'contains' : 'word';
+ const matchHelp = containsEnabled
+ ? `match=contains (the default) finds any part of a word, ignoring case and accents, and ranks the ${SEARCH_CONTAINS_WINDOW} most recently stored matches (whole words first, then newest), so total is at most ${SEARCH_CONTAINS_WINDOW}: narrow with more terms, chatId or dateTo to reach older ones. match=word finds whole words only.`
+ : 'match=word (the default) finds whole words; match=contains needs SEARCH_CONTAINS_ENABLED=true on the server.';
return [
defineTool({
name: 'MessageSearch',
@@ -30,18 +39,15 @@ export function searchTools(search: SearchService | undefined): AnyToolDescripto
chatScope: ['chatId'],
description:
'Search persisted messages across every session and chat the API key may read. Returns ranked hits with a <mark>-highlighted snippet, chatId, sender and timestamp; resolve names with SessionGetChats or ContactFindOne. ' +
- `match=contains (the default) finds any part of a word, ignoring case and accents, and ranks the ${SEARCH_CONTAINS_WINDOW} most recently stored matches (whole words first, then newest), so total is at most ${SEARCH_CONTAINS_WINDOW}: narrow with more terms, chatId or dateTo to reach older ones. ` +
- 'match=word finds whole words only. A key restricted to selected chats must pass chatId.',
+ `${matchHelp} A key restricted to selected chats must pass chatId.`,
tier: 'read',
requiredRole: ApiKeyRole.OPERATOR,
inputSchema: z.object({
q: z.string().trim().min(1).max(SEARCH_QUERY_MAX_LENGTH).describe('Search term'),
- // Defaults to contains, unlike REST (word): an agent often knows only part of a word or types
- // it without accents, and contains finds everything word finds.
match: z
.enum(SEARCH_MATCH_MODES)
- .default('contains')
- .describe('contains: any part of a word (default); word: whole words only'),
+ .default(defaultMatch)
+ .describe(`contains: any part of a word; word: whole words only. Default: ${defaultMatch}`),
chatId: chatIdSchema.optional().describe('Restrict to a single chat JID'),
from: z.string().optional().describe('Filter by sender phone or JID'),
direction: z.enum(MessageDirection).optional(),
diff --git a/src/database/migrations/1792400000000-AddMessagesTrigramSearch.ts b/src/database/migrations/1792400000000-AddMessagesTrigramSearch.ts
deleted file mode 100644
index 76d8a3234..000000000
--- a/src/database/migrations/1792400000000-AddMessagesTrigramSearch.ts
+++ /dev/null
@@ -1,123 +0,0 @@
-import { MigrationInterface, QueryRunner } from 'typeorm';
-
-/**
- * Adds the trigram index behind `GET /api/search?match=contains` (substring search, case- and
- * accent-insensitive), next to the word index of 1782400000000-AddMessagesFts:
- * - SQLite: an FTS5 external-content table with the trigram tokenizer (remove_diacritics), keyed on
- * the implicit `rowid`, kept in sync by triggers and backfilled once, exactly like `messages_fts`.
- * - Postgres: the pg_trgm and unaccent extensions, an IMMUTABLE `openwa_unaccent(text)` wrapper, a
- * GIN trigram index on `openwa_unaccent(body)`, and a b-tree on `("createdAt", "id")`. A contains
- * search ranks the most recently stored matches (SEARCH_CONTAINS_WINDOW); the existing message
- * indexes all lead with `sessionId`, so without this one a common term would sort every match to
- * find them. SQLite needs no counterpart: FTS5 walks its doclist in rowid (insertion) order.
- *
- * Never fails the boot over a missing capability. A SQLite build without FTS5 or older than 3.45 (the
- * first with remove_diacritics for trigram) gets no schema, and a Postgres role that may not create
- * the extensions gets the extension attempt rolled back to a savepoint and nothing else. In both cases
- * the provider answers 501 for match=contains only; match=word is untouched. On Postgres the provider
- * creates the function and indexes at a later boot once an operator has installed the extensions.
- *
- * Non-CONCURRENTLY on PG (one-time blocking build), as the word index. For very large `messages` tables,
- * run the upgrade during a maintenance window.
- *
- * Mirrored by BuiltInFtsProvider.ensureTrigramSchema, which brings the same schema up under
- * DATABASE_SYNCHRONIZE=true, where migrations never run. Kept self-contained rather than importing the
- * provider's helpers, so a later change to the provider cannot rewrite what this migration did.
- */
-export class AddMessagesTrigramSearch1792400000000 implements MigrationInterface {
- name = 'AddMessagesTrigramSearch1792400000000';
-
- async up(qr: QueryRunner): Promise<void> {
- if (qr.dataSource.options.type === 'postgres') {
- await this.upPostgres(qr);
- return;
- }
- const [probe] = (await qr.query(
- `SELECT sqlite_compileoption_used('ENABLE_FTS5') AS enabled, sqlite_version() AS version`,
- )) as Array<{ enabled: number; version: string }>;
- const [major, minor] = String(probe?.version ?? '')
- .split('.')
- .map(Number);
- if (!Number(probe?.enabled) || !(major > 3 || (major === 3 && minor >= 45))) return;
-
- await qr.query(
- `CREATE VIRTUAL TABLE IF NOT EXISTS "messages_trgm" USING fts5(body, content='messages', content_rowid='rowid', tokenize='trigram remove_diacritics 1')`,
- );
- await qr.query(
- `INSERT INTO "messages_trgm"("rowid", "body") SELECT "rowid", "body" FROM "messages" WHERE "body" IS NOT NULL`,
- );
- await qr.query(`DROP TRIGGER IF EXISTS messages_trgm_ai`);
- await qr.query(`CREATE TRIGGER messages_trgm_ai AFTER INSERT ON "messages" BEGIN
- INSERT INTO "messages_trgm"("rowid", "body") VALUES (new."rowid", new."body");
- END`);
- await qr.query(`DROP TRIGGER IF EXISTS messages_trgm_ad`);
- await qr.query(`CREATE TRIGGER messages_trgm_ad AFTER DELETE ON "messages" BEGIN
- INSERT INTO "messages_trgm"("messages_trgm", "rowid", "body") VALUES ('delete', old."rowid", old."body");
- END`);
- await qr.query(`DROP TRIGGER IF EXISTS messages_trgm_au`);
- // Same WHEN guard as messages_fts_au: acks, reactions and status updates do not touch the body, so
- // they skip the delete+insert on the busiest message path.
- await qr.query(`CREATE TRIGGER messages_trgm_au AFTER UPDATE ON "messages" WHEN OLD.body IS NOT NEW.body BEGIN
- INSERT INTO "messages_trgm"("messages_trgm", "rowid", "body") VALUES ('delete', old."rowid", old."body");
- INSERT INTO "messages_trgm"("rowid", "body") VALUES (new."rowid", new."body");
- END`);
- }
-
- async down(qr: QueryRunner): Promise<void> {
- if (qr.dataSource.options.type === 'postgres') {
- // The extensions stay: other objects in the database may depend on them.
- await qr.query(`DROP INDEX IF EXISTS "idx_messages_created_at_id"`);
- await qr.query(`DROP INDEX IF EXISTS "idx_messages_body_trgm"`);
- await qr.query(`DROP FUNCTION IF EXISTS openwa_unaccent(text)`);
- return;
- }
- await qr.query(`DROP TRIGGER IF EXISTS messages_trgm_au`);
- await qr.query(`DROP TRIGGER IF EXISTS messages_trgm_ad`);
- await qr.query(`DROP TRIGGER IF EXISTS messages_trgm_ai`);
- await qr.query(`DROP TABLE IF EXISTS "messages_trgm"`);
- }
-
- private async upPostgres(qr: QueryRunner): Promise<void> {
- // Guards the index build against a role- or database-level statement_timeout, as 1782400000000 does.
- await qr.query('SET LOCAL statement_timeout = 0');
- if (!(await this.createExtensions(qr))) return;
-
- const [schemas] = (await qr.query(
- `SELECT (SELECT n.nspname FROM pg_extension e JOIN pg_namespace n ON n.oid = e.extnamespace WHERE e.extname = 'pg_trgm') AS trgm,
- (SELECT n.nspname FROM pg_extension e JOIN pg_namespace n ON n.oid = e.extnamespace WHERE e.extname = 'unaccent') AS unaccent`,
- )) as Array<{ trgm: string; unaccent: string }>;
- const trgm = `"${schemas.trgm.replace(/"/g, '""')}"`;
- const unaccent = `"${schemas.unaccent.replace(/"/g, '""')}"`;
-
- // unaccent() is STABLE and Postgres indexes only IMMUTABLE expressions. Naming the dictionary and
- // schema-qualifying both references pins what the wrapper calls whatever the caller's search_path,
- // which is what makes declaring it IMMUTABLE safe.
- await qr.query(
- `CREATE OR REPLACE FUNCTION openwa_unaccent(text) RETURNS text LANGUAGE sql IMMUTABLE PARALLEL SAFE STRICT AS $$ SELECT ${unaccent}.unaccent('${unaccent.replace(/'/g, "''")}.unaccent'::regdictionary, $1) $$`,
- );
- await qr.query(
- `CREATE INDEX IF NOT EXISTS "idx_messages_body_trgm" ON "messages" USING GIN (openwa_unaccent("body") ${trgm}.gin_trgm_ops)`,
- );
- await qr.query(`CREATE INDEX IF NOT EXISTS "idx_messages_created_at_id" ON "messages" ("createdAt", "id")`);
- }
-
- /**
- * Both extensions ship with PostgreSQL's contrib package and are trusted from PostgreSQL 13, so a
- * database owner can create them; managed Postgres may still refuse. A failed statement aborts the
- * surrounding migration transaction, so the attempt runs inside a savepoint and a refusal rolls back
- * to it, leaving the rest of the chain to run. Returns whether both are installed.
- */
- private async createExtensions(qr: QueryRunner): Promise<boolean> {
- const inTransaction = qr.isTransactionActive;
- if (inTransaction) await qr.query('SAVEPOINT trigram_extensions');
- try {
- await qr.query('CREATE EXTENSION IF NOT EXISTS pg_trgm');
- await qr.query('CREATE EXTENSION IF NOT EXISTS unaccent');
- if (inTransaction) await qr.query('RELEASE SAVEPOINT trigram_extensions');
- return true;
- } catch {
- if (inTransaction) await qr.query('ROLLBACK TO SAVEPOINT trigram_extensions');
- return false;
- }
- }
-}
diff --git a/src/database/migrations/__tests__/1782400000000-AddMessagesFts.pg.spec.ts b/src/database/migrations/__tests__/1782400000000-AddMessagesFts.pg.spec.ts
index ab9ea63bd..a7022e2e6 100644
--- a/src/database/migrations/__tests__/1782400000000-AddMessagesFts.pg.spec.ts
+++ b/src/database/migrations/__tests__/1782400000000-AddMessagesFts.pg.spec.ts
@@ -4,7 +4,6 @@ import 'reflect-metadata';
import { DataSource } from 'typeorm';
import { BuiltInFtsProvider } from '../../../modules/search/providers/builtin-fts.provider';
import { AddMessagesFts1782400000000 } from '../1782400000000-AddMessagesFts';
-import { AddMessagesTrigramSearch1792400000000 } from '../1792400000000-AddMessagesTrigramSearch';
/**
* Postgres runtime coverage for BuiltInFtsProvider. Companion to the SQLite spec
@@ -52,9 +51,6 @@ const POSTGRES_ENABLED = process.env.DATABASE_TYPE === 'postgres';
// Adds the STORED generated `body_ts` tsvector + the GIN index — the exact objects the provider
// relies on; running the migration (not hand-rolling DDL) is the point of this test.
await new AddMessagesFts1782400000000().up(ds.createQueryRunner());
- // The rest of the search chain, so the boot self-heal tests below see what a migrations-based
- // deployment has: with the trigram objects missing, the provider would rightly build them at boot.
- await new AddMessagesTrigramSearch1792400000000().up(ds.createQueryRunner());
provider = new BuiltInFtsProvider(ds);
diff --git a/src/database/migrations/__tests__/1792400000000-AddMessagesTrigramSearch.pg.spec.ts b/src/database/migrations/__tests__/1792400000000-AddMessagesTrigramSearch.pg.spec.ts
deleted file mode 100644
index b401f5ce4..000000000
--- a/src/database/migrations/__tests__/1792400000000-AddMessagesTrigramSearch.pg.spec.ts
+++ /dev/null
@@ -1,156 +0,0 @@
-/* istanbul ignore file -- PG-gated: only runs under DATABASE_TYPE=postgres (test-postgres CI job);
- skipped in the default test job, so its lines would be unread and skew the global coverage gate. */
-import 'reflect-metadata';
-import { BadRequestException } from '@nestjs/common';
-import { DataSource } from 'typeorm';
-import { BuiltInFtsProvider } from '../../../modules/search/providers/builtin-fts.provider';
-import { AddMessagesFts1782400000000 } from '../1782400000000-AddMessagesFts';
-import { AddMessagesTrigramSearch1792400000000 } from '../1792400000000-AddMessagesTrigramSearch';
-
-/**
- * Postgres runtime coverage for match=contains: the pg_trgm/unaccent schema the 1792400000000
- * migration builds, queried through BuiltInFtsProvider. Companion to the SQLite spec
- * (builtin-fts.provider.contains.spec.ts); asserts the same contract (substring, case- and
- * accent-insensitive, every term required, short terms narrowing, literal wildcards, exact totals)
- * plus what only Postgres has: word_similarity ranking and the GIN index actually answering the query.
- *
- * Gated and provisioned exactly like 1782400000000-AddMessagesFts.pg.spec.ts: skipped unless
- * DATABASE_TYPE=postgres, connecting with DATABASE_HOST/PORT/USERNAME/PASSWORD/NAME. The role must be
- * allowed to create the two extensions (the CI postgres:16 service's owner role is).
- */
-const POSTGRES_ENABLED = process.env.DATABASE_TYPE === 'postgres';
-
-(POSTGRES_ENABLED ? describe : describe.skip)('AddMessagesTrigramSearch + match=contains (postgres)', () => {
- let ds: DataSource;
- let provider: BuiltInFtsProvider;
-
- const insert = (id: string, sessionId: string, body: string, ts: number) =>
- ds.query(
- `INSERT INTO "messages" ("id","sessionId","chatId","from","to","body","type","direction","timestamp") ` +
- `VALUES ($1,$2,$3,$4,'[email protected]',$5,'text','incoming',$6)`,
- [id, sessionId, `${id}-chat`, `${id}-from`, body, ts],
- );
-
- beforeEach(async () => {
- ds = new DataSource({
- type: 'postgres',
- host: process.env.DATABASE_HOST || 'localhost',
- port: Number(process.env.DATABASE_PORT || 5432),
- username: process.env.DATABASE_USERNAME || 'openwa',
- password: process.env.DATABASE_PASSWORD || 'openwa',
- database: process.env.DATABASE_NAME || 'openwa',
- });
- await ds.initialize();
- await ds.query(`DROP TABLE IF EXISTS "messages" CASCADE`);
- await ds.query(`DROP FUNCTION IF EXISTS openwa_unaccent(text)`);
- await ds.query(
- `CREATE TABLE "messages" (` +
- `"id" varchar PRIMARY KEY NOT NULL, "sessionId" varchar NOT NULL, "waMessageId" varchar, ` +
- `"chatId" varchar NOT NULL, "from" varchar NOT NULL, "to" varchar NOT NULL, "body" text, ` +
- `"type" varchar NOT NULL DEFAULT 'text', "direction" varchar NOT NULL DEFAULT 'outgoing', ` +
- `"timestamp" bigint, "status" varchar NOT NULL DEFAULT 'sent', ` +
- `"createdAt" timestamp NOT NULL DEFAULT now())`,
- );
- await new AddMessagesFts1782400000000().up(ds.createQueryRunner());
- await new AddMessagesTrigramSearch1792400000000().up(ds.createQueryRunner());
- provider = new BuiltInFtsProvider(ds);
-
- await insert('m1', 's1', 'Your invoice INV-2048 is ready for download', 10);
- await insert('m2', 's1', 'Thanks, I will pay the invoice today', 20);
- await insert('m3', 's2', 'Meeting moved to the Café Résumé at 10am', 30);
- await insert('m4', 's1', 'Order shipped with a 10% discount, tracking id 42', 40);
- });
-
- afterEach(async () => {
- if (ds && ds.isInitialized) {
- await ds.query(`DROP TABLE IF EXISTS "messages" CASCADE`).catch(() => undefined);
- await ds.query(`DROP FUNCTION IF EXISTS openwa_unaccent(text)`).catch(() => undefined);
- await ds.destroy();
- }
- });
-
- it('finds any part of a word, ignoring case and accents, and marks it', async () => {
- const res = await provider.search({ q: 'voic', match: 'contains' });
- expect(res.total).toBe(2);
- expect(res.hits.map(h => h.snippet).sort()).toEqual([
- 'Thanks, I will pay the in<mark>voic</mark>e today',
- 'Your in<mark>voic</mark>e INV-2048 is ready for download',
- ]);
-
- for (const q of ['cafe', 'CAFÉ', 'resume']) {
- expect((await provider.search({ q, match: 'contains' })).hits.map(h => h.messageId)).toEqual(['m3']);
- }
- });
-
- it('requires every term, narrows with short ones, and rejects short terms alone', async () => {
- expect((await provider.search({ q: 'invoice 2048', match: 'contains' })).hits.map(h => h.messageId)).toEqual([
- 'm1',
- ]);
- expect((await provider.search({ q: 'invoice to', match: 'contains' })).hits.map(h => h.messageId)).toEqual(['m2']);
- await expect(provider.search({ q: 'id to', match: 'contains' })).rejects.toBeInstanceOf(BadRequestException);
- });
-
- it('matches LIKE wildcards in a term literally', async () => {
- expect((await provider.search({ q: '10%', match: 'contains' })).total).toBe(1);
- expect((await provider.search({ q: 'invoice _', match: 'contains' })).total).toBe(0);
- });
-
- it('ranks whole words first, newest first within a score, and scores a scoped caller too', async () => {
- await insert('m5', 's1', 'reinvoiced items are listed below', 50);
- const res = await provider.search({ q: 'invoice', match: 'contains', sessionIds: ['s1'] });
-
- expect(res.hits.map(h => [h.messageId, h.score])).toEqual([
- ['m2', 3],
- ['m1', 3],
- ['m5', 1],
- ]);
- });
-
- it('pages with the window total', async () => {
- const page = await provider.search({ q: 'invoice', match: 'contains', limit: 1, offset: 1 });
- expect(page.total).toBe(2);
- expect(page.hits).toHaveLength(1);
- });
-
- // A four-row table is cheaper to scan than to look up, so the planner is told not to scan: what
- // these prove is that the SQL the provider emits matches the indexed expressions and CAN use them.
- // One query runner keeps SET LOCAL and EXPLAIN on the same connection and transaction.
- const plan = async (settings: string[], sql: string, params: unknown[]): Promise<string> => {
- const runner = ds.createQueryRunner();
- await runner.startTransaction();
- try {
- for (const setting of settings) await runner.query(`SET LOCAL ${setting} = off`);
- const rows = (await runner.query(`EXPLAIN ${sql}`, params)) as Array<{ 'QUERY PLAN': string }>;
- return rows.map(r => r['QUERY PLAN']).join('\n');
- } finally {
- await runner.rollbackTransaction();
- await runner.release();
- }
- };
-
- it('answers the substring predicate from the GIN trigram index', async () => {
- const sql = `SELECT 1 FROM messages m WHERE openwa_unaccent(m."body") ILIKE openwa_unaccent($1)`;
- expect(await plan(['enable_seqscan'], sql, ['%voic%'])).toContain('idx_messages_body_trgm');
- });
-
- it('can fetch the most recently stored candidates by walking the recency index', async () => {
- const sql =
- `SELECT 1 FROM messages m WHERE openwa_unaccent(m."body") ILIKE openwa_unaccent($1) ` +
- `ORDER BY m."createdAt" DESC, m."id" DESC LIMIT 1000`;
- expect(await plan(['enable_seqscan', 'enable_bitmapscan'], sql, ['%voic%'])).toContain(
- 'Index Scan Backward using idx_messages_created_at_id',
- );
- });
-
- it('builds the function and both indexes at boot when only the extensions exist', async () => {
- await new AddMessagesTrigramSearch1792400000000().down(ds.createQueryRunner());
- const fresh = new BuiltInFtsProvider(ds);
- await fresh.onModuleInit();
-
- expect((await fresh.search({ q: 'voice', match: 'contains' })).total).toBe(2);
- const indexes = await ds.query<Array<{ indexname: string }>>(
- `SELECT indexname FROM pg_indexes WHERE tablename = 'messages' AND indexname IN ('idx_messages_body_trgm', 'idx_messages_created_at_id') ORDER BY indexname`,
- );
- expect(indexes.map(i => i.indexname)).toEqual(['idx_messages_body_trgm', 'idx_messages_created_at_id']);
- });
-});
diff --git a/src/database/migrations/__tests__/1792400000000-AddMessagesTrigramSearch.spec.ts b/src/database/migrations/__tests__/1792400000000-AddMessagesTrigramSearch.spec.ts
deleted file mode 100644
index c68c4d4e1..000000000
--- a/src/database/migrations/__tests__/1792400000000-AddMessagesTrigramSearch.spec.ts
+++ /dev/null
@@ -1,120 +0,0 @@
-import 'reflect-metadata';
-import { DataSource, QueryRunner } from 'typeorm';
-import { Message, MessageDirection } from '../../../modules/message/entities/message.entity';
-import { Session } from '../../../modules/session/entities/session.entity';
-import { AddMessagesTrigramSearch1792400000000 } from '../1792400000000-AddMessagesTrigramSearch';
-
-/**
- * SQLite in-memory coverage for the AddMessagesTrigramSearch migration, plus the capability gates of
- * both dialects through stub query runners:
- * - the FTS5 trigram table + 3 sync triggers are created, backfilled, and kept in sync;
- * - re-running up() is a no-op and down() removes all four objects;
- * - a SQLite build without FTS5, or older than 3.45, gets no schema and no error;
- * - a Postgres role refused CREATE EXTENSION rolls back to the savepoint and stops there.
- *
- * The Postgres objects themselves are exercised in 1792400000000-AddMessagesTrigramSearch.pg.spec.ts.
- */
-describe('AddMessagesTrigramSearch migration (sqlite)', () => {
- let ds: DataSource;
- let runner: QueryRunner;
-
- const insert = (body: string | null) =>
- ds.getRepository(Message).insert({
- sessionId: 's1',
- chatId: 'c1',
- body: body as string,
- type: 'text',
- direction: MessageDirection.INCOMING,
- timestamp: 1,
- });
- const matches = async (q: string): Promise<number> =>
- (
- (await runner.query(`SELECT count(*) AS n FROM messages_trgm WHERE messages_trgm MATCH ?`, [q])) as Array<{
- n: number;
- }>
- )[0].n;
- const objects = async (): Promise<string[]> =>
- (
- (await runner.query(
- `SELECT name FROM sqlite_master WHERE name = 'messages_trgm' OR name LIKE 'messages_trgm\\_a_' ESCAPE '\\' ORDER BY name`,
- )) as Array<{ name: string }>
- ).map(r => r.name);
-
- beforeEach(async () => {
- ds = new DataSource({
- type: 'better-sqlite3',
- database: ':memory:',
- entities: [Session, Message],
- synchronize: true,
- migrations: [],
- });
- await ds.initialize();
- runner = ds.createQueryRunner();
- });
- afterEach(async () => {
- await ds.destroy();
- });
-
- it('creates the trigram table and triggers, backfilling the rows already stored', async () => {
- await insert('Meeting at the Café');
- await insert(null);
- await new AddMessagesTrigramSearch1792400000000().up(runner);
-
- expect(await objects()).toEqual(['messages_trgm', 'messages_trgm_ad', 'messages_trgm_ai', 'messages_trgm_au']);
- expect(await matches('"cafe"')).toBe(1);
-
- await insert('Your invoice is ready');
- expect(await matches('"voic"')).toBe(1);
- });
-
- it('is idempotent on re-run and fully reversible', async () => {
- await insert('Your invoice is ready');
- const migration = new AddMessagesTrigramSearch1792400000000();
- await migration.up(runner);
- await migration.down(runner);
- await migration.up(runner);
- expect(await matches('"invo"')).toBe(1);
-
- await migration.down(runner);
- expect(await objects()).toEqual([]);
- });
-
- it.each([
- ['a build without FTS5', { enabled: 0, version: '3.53.4' }],
- ['SQLite older than 3.45', { enabled: 1, version: '3.44.2' }],
- ])('creates nothing on %s', async (_label, probe) => {
- const query = jest.fn().mockResolvedValue([probe]);
- const stub = { dataSource: { options: { type: 'better-sqlite3' } }, query } as unknown as QueryRunner;
-
- await new AddMessagesTrigramSearch1792400000000().up(stub);
-
- expect(query).toHaveBeenCalledTimes(1);
- });
-});
-
-describe('AddMessagesTrigramSearch migration (postgres capability gate)', () => {
- it('rolls back to the savepoint and creates nothing else when CREATE EXTENSION is refused', async () => {
- const statements: string[] = [];
- const query = jest.fn((sql: string) => {
- statements.push(sql);
- if (sql.startsWith('CREATE EXTENSION')) return Promise.reject(new Error('permission denied to create extension'));
- return Promise.resolve([]);
- });
- const stub = {
- dataSource: { options: { type: 'postgres' } },
- isTransactionActive: true,
- query,
- } as unknown as QueryRunner;
-
- await new AddMessagesTrigramSearch1792400000000().up(stub);
-
- expect(statements).toEqual([
- 'SET LOCAL statement_timeout = 0',
- 'SAVEPOINT trigram_extensions',
- 'CREATE EXTENSION IF NOT EXISTS pg_trgm',
- 'ROLLBACK TO SAVEPOINT trigram_extensions',
- ]);
- });
-});
diff --git a/src/modules/search/dto/search-query.dto.ts b/src/modules/search/dto/search-query.dto.ts
index db3790eea..3df4c9755 100644
--- a/src/modules/search/dto/search-query.dto.ts
+++ b/src/modules/search/dto/search-query.dto.ts
@@ -39,8 +39,9 @@ export class SearchQueryDto {
default: 'word',
description:
'`word` (default) matches whole words. `contains` matches any part of a word, ignoring case and ' +
- `accents (\`voic\` finds "Invoice", \`cafe\` finds "Café"); it needs at least one term of ` +
- `${SEARCH_CONTAINS_MIN_TERM_LENGTH} or more characters, and on PostgreSQL the pg_trgm and unaccent extensions. ` +
+ `accents (\`voic\` finds "Invoice", \`cafe\` finds "Café"); it is opt-in on the built-in provider ` +
+ '(SEARCH_CONTAINS_ENABLED=true, else 501) and needs at least one term of ' +
+ `${SEARCH_CONTAINS_MIN_TERM_LENGTH} or more characters (on PostgreSQL, ${SEARCH_CONTAINS_MIN_TERM_LENGTH} letters or digits in a row). ` +
`It ranks the ${SEARCH_CONTAINS_WINDOW} most recently stored matches by match quality, so \`total\` is at most ` +
`${SEARCH_CONTAINS_WINDOW}.`,
})
diff --git a/src/modules/search/providers/builtin-fts.provider.contains.pg.spec.ts b/src/modules/search/providers/builtin-fts.provider.contains.pg.spec.ts
new file mode 100644
index 000000000..8df7f2cd4
--- /dev/null
+++ b/src/modules/search/providers/builtin-fts.provider.contains.pg.spec.ts
@@ -0,0 +1,249 @@
+/* istanbul ignore file -- PG-gated: only runs under DATABASE_TYPE=postgres (test-postgres CI job);
+ skipped in the default test job, so its lines would be unread and skew the global coverage gate. */
+import 'reflect-metadata';
+import { BadRequestException, NotImplementedException } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
+import { DataSource, DataSourceOptions } from 'typeorm';
+import { join } from 'path';
+import { BuiltInFtsProvider, TRIGRAM_BUILD_LOCK_KEYS } from './builtin-fts.provider';
+
+/**
+ * Postgres runtime coverage for match=contains: the pg_trgm/unaccent schema buildContainsIndex creates
+ * when SEARCH_CONTAINS_ENABLED is on, queried through BuiltInFtsProvider. Companion to the SQLite spec
+ * (builtin-fts.provider.contains.spec.ts); asserts the same contract (substring, case- and
+ * accent-insensitive, every term required, short terms narrowing, literal wildcards) plus what only
+ * Postgres has: the term rule pg_trgm imposes, the CONCURRENTLY build and its lock, and the indexes
+ * actually answering the query.
+ *
+ * `messages` comes from the real data migration chain, on a scratch database of its own, so the plans
+ * below are chosen among the indexes production has (IDX_messages_createdAt among them), not among the
+ * ones a hand-made table happens to carry. Skipped unless DATABASE_TYPE=postgres; connects with
+ * DATABASE_HOST/PORT/USERNAME/PASSWORD/NAME, and the role must be allowed to create databases and the
+ * two extensions (the CI postgres:16 service's owner role is).
+ */
+const POSTGRES_ENABLED = process.env.DATABASE_TYPE === 'postgres';
+
+/** Named for the suite so a crashed run leaves an obvious orphan rather than a mystery database. */
+const SCRATCH_DB = 'openwa_contains_search_probe';
+
+const connection = (database: string): DataSourceOptions => ({
+ type: 'postgres',
+ host: process.env.DATABASE_HOST || 'localhost',
+ port: Number(process.env.DATABASE_PORT || 5432),
+ username: process.env.DATABASE_USERNAME || 'openwa',
+ password: process.env.DATABASE_PASSWORD || 'openwa',
+ database,
+});
+
+const containsOn = new ConfigService({ search: { containsEnabled: true } });
+
+(POSTGRES_ENABLED ? describe : describe.skip)('BuiltInFtsProvider match=contains (postgres)', () => {
+ let admin: DataSource;
+ let ds: DataSource;
+
+ const insert = (id: string, sessionId: string, body: string, ts: number) =>
+ ds.query(
+ `INSERT INTO "messages" ("id","sessionId","chatId","from","to","body","type","direction","timestamp") ` +
+ `VALUES ($1,$2,$3,$4,'[email protected]',$5,'text','incoming',$6)`,
+ [id, sessionId, `${id}-chat`, `${id}-from`, body, ts],
+ );
+
+ const trigramState = async () =>
+ (
+ await ds.query<Array<{ extensions: number; fn: boolean; idx: boolean | null }>>(
+ `SELECT (SELECT count(*)::int FROM pg_extension WHERE extname IN ('pg_trgm', 'unaccent')) AS extensions,
+ to_regprocedure('openwa_unaccent(text)') IS NOT NULL AS fn,
+ (SELECT i.indisvalid FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid WHERE c.relname = 'idx_messages_body_trgm') AS idx`,
+ )
+ )[0];
+
+ beforeAll(async () => {
+ admin = new DataSource(connection(process.env.DATABASE_NAME || 'openwa'));
+ await admin.initialize();
+ await admin.query(`DROP DATABASE IF EXISTS "${SCRATCH_DB}"`);
+ await admin.query(`CREATE DATABASE "${SCRATCH_DB}"`);
+ ds = new DataSource({
+ ...connection(SCRATCH_DB),
+ migrations: [join(__dirname, '../../../database/migrations/*{.ts,.js}')],
+ });
+ await ds.initialize();
+ await ds.runMigrations();
+ }, 120_000);
+
+ afterAll(async () => {
+ await ds?.destroy().catch(() => undefined);
+ await admin?.query(`DROP DATABASE IF EXISTS "${SCRATCH_DB}"`).catch(() => undefined);
+ await admin?.destroy().catch(() => undefined);
+ });
+
+ beforeEach(async () => {
+ await ds.query(`DELETE FROM "messages"`);
+ await ds.query(`DROP INDEX IF EXISTS "idx_messages_body_trgm"`);
+ await ds.query(`DROP FUNCTION IF EXISTS openwa_unaccent(text)`);
+ await ds.query(`DROP EXTENSION IF EXISTS pg_trgm`);
+ await ds.query(`DROP EXTENSION IF EXISTS unaccent`);
+ await insert('m1', 's1', 'Your invoice INV-2048 is ready for download', 10);
+ await insert('m2', 's1', 'Thanks, I will pay the invoice today', 20);
+ await insert('m3', 's2', 'Meeting moved to the Café Résumé at 10am', 30);
+ await insert('m4', 's1', 'Order shipped with a 10% discount, tracking id 42', 40);
+ });
+
+ describe('building the schema', () => {
+ it('creates nothing and answers 501 naming SEARCH_CONTAINS_ENABLED while the setting is off', async () => {
+ const provider = new BuiltInFtsProvider(ds);
+ await provider.buildContainsIndex();
+
+ const attempt = provider.search({ q: 'voic', match: 'contains' });
+ await expect(attempt).rejects.toBeInstanceOf(NotImplementedException);
+ await expect(attempt).rejects.toThrow(/SEARCH_CONTAINS_ENABLED=true/);
+ expect(await trigramState()).toEqual({ extensions: 0, fn: false, idx: null });
+ });
+
+ it('creates the extensions, the wrapper and a valid index, and a second build changes nothing', async () => {
+ const provider = new BuiltInFtsProvider(ds, containsOn);
+ await expect(provider.search({ q: 'voic', match: 'contains' })).rejects.toThrow(/still being built/);
+
+ await provider.buildContainsIndex();
+ await provider.buildContainsIndex();
+
+ expect(await trigramState()).toEqual({ extensions: 2, fn: true, idx: true });
+ expect((await provider.search({ q: 'voic', match: 'contains' })).total).toBe(2);
+ });
+
+ it('drops and rebuilds an INVALID index a failed CONCURRENTLY build left behind', async () => {
+ await expect(
+ ds.query(`CREATE UNIQUE INDEX CONCURRENTLY "idx_messages_body_trgm" ON "messages" ("sessionId")`),
+ ).rejects.toThrow();
+ expect((await trigramState()).idx).toBe(false);
+
+ const provider = new BuiltInFtsProvider(ds, containsOn);
+ await provider.buildContainsIndex();
+
+ expect((await trigramState()).idx).toBe(true);
+ expect((await provider.search({ q: 'voic', match: 'contains' })).total).toBe(2);
+ });
+
+ it('leaves the build to the replica holding the lock, then serves the index it built', async () => {
+ const other = ds.createQueryRunner();
+ await other.connect();
+ try {
+ await other.query(`SELECT pg_advisory_lock($1, $2)`, [...TRIGRAM_BUILD_LOCK_KEYS]);
+ const waiting = new BuiltInFtsProvider(ds, containsOn);
+ await waiting.buildContainsIndex();
+
+ expect(await trigramState()).toEqual({ extensions: 0, fn: false, idx: null });
+ await expect(waiting.search({ q: 'voic', match: 'contains' })).rejects.toThrow(/still being built/);
+
+ await other.query(`SELECT pg_advisory_unlock($1, $2)`, [...TRIGRAM_BUILD_LOCK_KEYS]);
+ await new BuiltInFtsProvider(ds, containsOn).buildContainsIndex();
+
+ expect((await waiting.search({ q: 'voic', match: 'contains' })).total).toBe(2);
+ } finally {
+ await other.release();
+ }
+ });
+ });
+
+ describe('searching', () => {
+ let provider: BuiltInFtsProvider;
+
+ beforeEach(async () => {
+ provider = new BuiltInFtsProvider(ds, containsOn);
+ await provider.buildContainsIndex();
+ });
+
+ it('finds any part of a word, ignoring case and accents, and marks it', async () => {
+ const res = await provider.search({ q: 'voic', match: 'contains' });
+ expect(res.total).toBe(2);
+ expect(res.hits.map(h => h.snippet).sort()).toEqual([
+ 'Thanks, I will pay the in<mark>voic</mark>e today',
+ 'Your in<mark>voic</mark>e INV-2048 is ready for download',
+ ]);
+
+ for (const q of ['cafe', 'CAFÉ', 'resume']) {
+ expect((await provider.search({ q, match: 'contains' })).hits.map(h => h.messageId)).toEqual(['m3']);
+ }
+ });
+
+ it('requires every term, narrows with short ones, and rejects short terms alone', async () => {
+ expect((await provider.search({ q: 'invoice 2048', match: 'contains' })).hits.map(h => h.messageId)).toEqual([
+ 'm1',
+ ]);
+ expect((await provider.search({ q: 'invoice to', match: 'contains' })).hits.map(h => h.messageId)).toEqual([
+ 'm2',
+ ]);
+ await expect(provider.search({ q: 'id to', match: 'contains' })).rejects.toBeInstanceOf(BadRequestException);
+ });
+
+ it('rejects a query with no run of three letters or digits, which pg_trgm cannot look up', async () => {
+ for (const q of ['😂😂😂', '...', ':-)', 'a.b.c']) {
+ await expect(provider.search({ q, match: 'contains' })).rejects.toBeInstanceOf(BadRequestException);
+ }
+ expect((await provider.search({ q: ':-) invoice', match: 'contains' })).total).toBe(0);
+ expect((await provider.search({ q: 'INV-2048', match: 'contains' })).total).toBe(1);
+ });
+
+ it('matches LIKE wildcards literally, the fullwidth forms unaccent folds into them included', async () => {
+ await insert('m5', 's1', 'price 100 dollars', 50);
+ expect((await provider.search({ q: '10% discount', match: 'contains' })).hits.map(h => h.messageId)).toEqual([
+ 'm4',
+ ]);
+ expect((await provider.search({ q: 'invoice _', match: 'contains' })).total).toBe(0);
+ expect((await provider.search({ q: '100%', match: 'contains' })).total).toBe(0);
+ expect((await provider.search({ q: 'price_100', match: 'contains' })).total).toBe(0);
+ });
+
+ it('transliterates letters SQLite leaves alone, which docs/26 lists as a dialect difference', async () => {
+ await insert('m5', 's1', 'Shipped from Łódź to the Straße', 50);
+ expect((await provider.search({ q: 'lodz', match: 'contains' })).hits.map(h => h.messageId)).toEqual(['m5']);
+ expect((await provider.search({ q: 'strasse', match: 'contains' })).hits.map(h => h.messageId)).toEqual(['m5']);
+ });
+
+ it('ranks whole words first, newest first within a score, and scores a scoped caller too', async () => {
+ await insert('m5', 's1', 'reinvoiced items are listed below', 50);
+ const res = await provider.search({ q: 'invoice', match: 'contains', sessionIds: ['s1'] });
+
+ expect(res.hits.map(h => [h.messageId, h.score])).toEqual([
+ ['m2', 3],
+ ['m1', 3],
+ ['m5', 1],
+ ]);
+ });
+
+ it('pages with the window total', async () => {
+ const page = await provider.search({ q: 'invoice', match: 'contains', limit: 1, offset: 1 });
+ expect(page.total).toBe(2);
+ expect(page.hits).toHaveLength(1);
+ });
+
+ // A handful of rows is cheaper to scan than to look up, so the planner is told not to scan: what
+ // these prove is that the SQL the provider emits matches the indexed expressions and CAN use them.
+ // The SQL is captured from a real search; one query runner keeps SET LOCAL and EXPLAIN together.
+ const plan = async (settings: string[], q: string): Promise<string> => {
+ const query = jest.spyOn(ds, 'query');
+ await provider.search({ q, match: 'contains' });
+ const [sql, params] = query.mock.calls.find(([text]) => /ILIKE/.test(text)) as [string, unknown[]];
+ query.mockRestore();
+ const runner = ds.createQueryRunner();
+ await runner.startTransaction();
+ try {
+ for (const setting of settings) await runner.query(`SET LOCAL ${setting} = off`);
+ const rows = (await runner.query(`EXPLAIN ${sql}`, params)) as Array<{ 'QUERY PLAN': string }>;
+ return rows.map(r => r['QUERY PLAN']).join('\n');
+ } finally {
+ await runner.rollbackTransaction();
+ await runner.release();
+ }
+ };
+
+ it('answers the substring predicate from the GIN trigram index', async () => {
+ expect(await plan(['enable_seqscan', 'enable_indexscan'], 'voic')).toContain('idx_messages_body_trgm');
+ });
+
+ it('can fetch the most recently stored candidates by walking IDX_messages_createdAt', async () => {
+ expect(await plan(['enable_seqscan', 'enable_bitmapscan'], 'voic')).toContain(
+ 'Index Scan Backward using "IDX_messages_createdAt"',
+ );
+ });
+ });
+});
diff --git a/src/modules/search/providers/builtin-fts.provider.contains.spec.ts b/src/modules/search/providers/builtin-fts.provider.contains.spec.ts
index ba2ed6ba6..2eca15abb 100644
--- a/src/modules/search/providers/builtin-fts.provider.contains.spec.ts
+++ b/src/modules/search/providers/builtin-fts.provider.contains.spec.ts
@@ -1,10 +1,10 @@
import 'reflect-metadata';
import { BadRequestException, NotImplementedException } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
import { DataSource } from 'typeorm';
import { Message, MessageDirection } from '../../../modules/message/entities/message.entity';
import { Session } from '../../../modules/session/entities/session.entity';
import { AddMessagesFts1782400000000 } from '../../../database/migrations/1782400000000-AddMessagesFts';
-import { AddMessagesTrigramSearch1792400000000 } from '../../../database/migrations/1792400000000-AddMessagesTrigramSearch';
import { SEARCH_CONTAINS_WINDOW } from '../search.constants';
import {
BuiltInFtsProvider,
@@ -13,9 +13,10 @@ import {
} from './builtin-fts.provider';
/**
- * SQLite runtime coverage for match=contains: the FTS5 trigram index the 1792400000000 migration
- * builds, queried through BuiltInFtsProvider. Companion to builtin-fts.provider.spec.ts (match=word);
- * the Postgres branch runs in 1792400000000-AddMessagesTrigramSearch.pg.spec.ts.
+ * SQLite runtime coverage for match=contains: the FTS5 trigram index buildContainsIndex builds when
+ * SEARCH_CONTAINS_ENABLED is on, queried through BuiltInFtsProvider. Companion to
+ * builtin-fts.provider.spec.ts (match=word); the Postgres branch runs in
+ * builtin-fts.provider.contains.pg.spec.ts.
*/
const row = (sessionId: string, chatId: string, body: string, direction: MessageDirection, timestamp: number) => ({
sessionId,
@@ -28,7 +29,9 @@ const row = (sessionId: string, chatId: string, body: string, direction: Message
timestamp,
});
-async function boot(options: { trigramMigration: boolean }): Promise<DataSource> {
+const containsOn = new ConfigService({ search: { containsEnabled: true } });
+
+async function boot(): Promise<DataSource> {
const ds = new DataSource({
type: 'better-sqlite3',
database: ':memory:',
@@ -37,7 +40,6 @@ async function boot(options: { trigramMigration: boolean }): Promise<DataSource>
});
await ds.initialize();
await new AddMessagesFts1782400000000().up(ds.createQueryRunner());
- if (options.trigramMigration) await new AddMessagesTrigramSearch1792400000000().up(ds.createQueryRunner());
await ds
.getRepository(Message)
.insert([
@@ -54,8 +56,9 @@ describe('BuiltInFtsProvider match=contains (sqlite)', () => {
let provider: BuiltInFtsProvider;
beforeEach(async () => {
- ds = await boot({ trigramMigration: true });
- provider = new BuiltInFtsProvider(ds);
+ ds = await boot();
+ provider = new BuiltInFtsProvider(ds, containsOn);
+ await provider.buildContainsIndex();
});
afterEach(async () => {
await ds.destroy();
@@ -174,31 +177,86 @@ describe('BuiltInFtsProvider match=contains (sqlite)', () => {
});
});
-describe('BuiltInFtsProvider match=contains without the trigram index (sqlite)', () => {
+describe('BuiltInFtsProvider trigram index build (sqlite)', () => {
let ds: DataSource;
beforeEach(async () => {
- ds = await boot({ trigramMigration: false });
+ ds = await boot();
});
afterEach(async () => {
await ds.destroy();
});
- it('answers 501 for match=contains while match=word keeps working', async () => {
+ const trigramTables = async () =>
+ (await ds.query<unknown[]>(`SELECT name FROM sqlite_master WHERE name = 'messages_trgm'`)).length;
+
+ // The `rank = 1` form checks the index against the content table; valid here because no fixture body
+ // is NULL (see builtin-fts.provider.backfill.spec.ts).
+ const integrityCheck = () => ds.query(`INSERT INTO messages_trgm(messages_trgm, rank) VALUES ('integrity-check', 1)`);
+
+ const insertBulk = async (count: number, body: (i: number) => string) => {
+ const rows = Array.from({ length: count }, (_, i) => row('s1', 'c8', body(i), MessageDirection.INCOMING, 100 + i));
+ for (let i = 0; i < rows.length; i += 200) await ds.getRepository(Message).insert(rows.slice(i, i + 200));
+ };
+
+ it('answers 501 naming SEARCH_CONTAINS_ENABLED while the setting is off, builds nothing, and leaves match=word alone', async () => {
const provider = new BuiltInFtsProvider(ds);
+ await provider.buildContainsIndex();
- await expect(provider.search({ q: 'voic', match: 'contains' })).rejects.toBeInstanceOf(NotImplementedException);
+ const attempt = provider.search({ q: 'voic', match: 'contains' });
+ await expect(attempt).rejects.toBeInstanceOf(NotImplementedException);
+ await expect(attempt).rejects.toThrow(/SEARCH_CONTAINS_ENABLED=true/);
+ expect(await trigramTables()).toBe(0);
expect((await provider.search({ q: 'invoice' })).total).toBe(2);
});
- it('builds and backfills the index at boot, as under DATABASE_SYNCHRONIZE=true', async () => {
- const provider = new BuiltInFtsProvider(ds);
- await provider.onModuleInit();
+ it('answers 501 until the build has run', async () => {
+ const provider = new BuiltInFtsProvider(ds, containsOn);
+ await expect(provider.search({ q: 'voic', match: 'contains' })).rejects.toThrow(/still being built/);
+ await provider.buildContainsIndex();
expect((await provider.search({ q: 'voic', match: 'contains' })).total).toBe(2);
- await ds.getRepository(Message).insert(row('s1', 'c9', 'Invoice reminder', MessageDirection.INCOMING, 50));
+ });
+
+ it('builds over existing rows twice into a sound index the triggers then keep current', async () => {
+ await insertBulk(2500, i => `Bulk receipt ${i}`);
+ const provider = new BuiltInFtsProvider(ds, containsOn);
+
+ await provider.buildContainsIndex();
+ await provider.buildContainsIndex();
+
+ await expect(integrityCheck()).resolves.toBeDefined();
+ expect((await provider.search({ q: 'receipt', match: 'contains' })).total).toBe(SEARCH_CONTAINS_WINDOW);
+ expect((await provider.search({ q: 'receipt 2499', match: 'contains' })).total).toBe(1);
+ await ds.getRepository(Message).insert(row('s1', 'c9', 'Invoice reminder', MessageDirection.INCOMING, 5000));
expect((await provider.search({ q: 'voic', match: 'contains' })).total).toBe(3);
});
+
+ it('stays sound when a row the backfill has not reached yet is edited or deleted mid-build', async () => {
+ await insertBulk(2500, i => `Bulk receipt ${i}`);
+ const repo = ds.getRepository(Message);
+ const edited = await repo.findOneByOrFail({ body: 'Bulk receipt 2400' });
+ const deleted = await repo.findOneByOrFail({ body: 'Bulk receipt 2401' });
+ const query = ds.query.bind(ds);
+ let interleaved = false;
+ jest.spyOn(ds, 'query').mockImplementation(async (sql: string, params?: Parameters<DataSource['query']>[1]) => {
+ if (!interleaved && /m\."rowid" > \?/.test(sql)) {
+ interleaved = true;
+ await repo.update({ id: edited.id }, { body: 'Bulk invoice edited' });
+ await repo.delete({ id: deleted.id });
+ }
+ return query(sql, params);
+ });
+ const provider = new BuiltInFtsProvider(ds, containsOn);
+
+ await provider.buildContainsIndex();
+
+ expect(interleaved).toBe(true);
+ await expect(integrityCheck()).resolves.toBeDefined();
+ expect((await provider.search({ q: 'edited', match: 'contains' })).total).toBe(1);
+ expect((await provider.search({ q: 'receipt 2400', match: 'contains' })).total).toBe(0);
+ expect((await provider.search({ q: 'receipt 2401', match: 'contains' })).total).toBe(0);
+ });
});
describe('trigram schema helpers', () => {
diff --git a/src/modules/search/providers/builtin-fts.provider.ts b/src/modules/search/providers/builtin-fts.provider.ts
index 2ce76880c..31f8713af 100644
--- a/src/modules/search/providers/builtin-fts.provider.ts
+++ b/src/modules/search/providers/builtin-fts.provider.ts
@@ -1,7 +1,17 @@
-import { BadRequestException, Injectable, NotImplementedException, OnModuleInit } from '@nestjs/common';
+import {
+ BadRequestException,
+ Injectable,
+ NotImplementedException,
+ OnApplicationBootstrap,
+ OnModuleInit,
+ Optional,
+} from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
+import { HttpAdapterHost } from '@nestjs/core';
+import { setImmediate } from 'node:timers/promises';
import { createLogger } from '../../../common/services/logger.service';
import { InjectDataSource } from '@nestjs/typeorm';
-import { DataSource } from 'typeorm';
+import { DataSource, QueryRunner } from 'typeorm';
import type { MessageType } from '../../../engine/interfaces/whatsapp-engine.interface';
import { MessageDirection } from '../../message/entities/message.entity';
import type { SearchProvider, SearchQuery, SearchResults, SearchHit } from '../search.types';
@@ -20,17 +30,47 @@ const MAX_SNIPPET_WORDS = 24;
interface Fts5Index {
table: string;
tokenize?: string;
+ /**
+ * Backfill in batches that yield to the event loop, after the triggers exist, instead of in one
+ * statement before them (see backfillSqliteIndexInBatches).
+ */
+ batched?: boolean;
}
/** The match=word index (migration 1782400000000-AddMessagesFts). */
const WORD_INDEX: Fts5Index = { table: 'messages_fts' };
/**
- * The match=contains index (migration 1792400000000-AddMessagesTrigramSearch). The trigram tokenizer
- * indexes every three-character window of the body, so any substring of three or more characters is a
- * lookup instead of a scan; remove_diacritics makes it accent-insensitive like unicode61.
+ * The match=contains index, built only with SEARCH_CONTAINS_ENABLED=true (see buildContainsIndex). The
+ * trigram tokenizer indexes every three-character window of the body, so any substring of three or more
+ * characters is a lookup instead of a scan; remove_diacritics makes it accent-insensitive like unicode61.
+ */
+const TRIGRAM_INDEX: Fts5Index = { table: 'messages_trgm', tokenize: 'trigram remove_diacritics 1', batched: true };
+
+/** Rowids per backfill batch: about 10 ms of event loop each, at the 9.4 µs a row measured on 1M messages. */
+const TRIGRAM_BACKFILL_BATCH = 1000;
+
+/**
+ * The session-level advisory lock one replica holds while it builds the Postgres trigram index, in the
+ * two-int4 form of POSTGRES_BOOT_MIGRATION_LOCK_KEYS: 0x4f5741 = "OWA", 0x7472676d = "trgm".
*/
-const TRIGRAM_INDEX: Fts5Index = { table: 'messages_trgm', tokenize: 'trigram remove_diacritics 1' };
+export const TRIGRAM_BUILD_LOCK_KEYS: readonly [number, number] = [0x4f5741, 0x7472676d];
+
+/** Why match=contains answers 501, one message per cause, each naming what to change. */
+const CONTAINS_UNAVAILABLE = {
+ disabled: 'Contains search is disabled: set SEARCH_CONTAINS_ENABLED=true and restart to build its index.',
+ building:
+ 'Contains search is not ready: its index is still being built (SEARCH_CONTAINS_ENABLED=true). Retry shortly.',
+ failed:
+ 'Contains search is unavailable: building its index failed (SEARCH_CONTAINS_ENABLED=true); see the server log.',
+ postgresExtensions:
+ 'Contains search is unavailable: install the pg_trgm and unaccent extensions (CREATE EXTENSION pg_trgm; CREATE EXTENSION unaccent;) and restart.',
+ sqliteTrigram:
+ 'Contains search is unavailable: this SQLite build has no FTS5 trigram tokenizer (SQLite 3.45+ is required).',
+} as const;
+
+/** Three letters or digits in a row: pg_trgm takes trigrams from nothing else. */
+const POSTGRES_TRIGRAM_SOURCE = new RegExp(`[\\p{L}\\p{N}]{${SEARCH_CONTAINS_MIN_TERM_LENGTH}}`, 'u');
/**
* The contains candidate SELECT list. It carries no snippet or score column: searchContains computes
@@ -40,15 +80,14 @@ const CONTAINS_COLUMNS = `m."id", m."waMessageId" AS wa_message_id, m."sessionId
/**
* What the Postgres catalog holds for contains search: the schema of each extension it needs (null when
- * not installed), and whether the `openwa_unaccent` wrapper, the trigram index and the recency index
- * exist.
+ * not installed), whether the `openwa_unaccent` wrapper exists, and the trigram index: null when absent,
+ * false while a CONCURRENTLY build runs or after one died, true once usable.
*/
interface TrigramStateRow {
trgm: string | null;
unaccent: string | null;
fn: boolean;
- idx: boolean;
- recent: boolean;
+ idx: boolean | null;
}
/** Descending string order for the `id` tiebreak, as `ORDER BY "id" DESC` sorts a UUID column. */
@@ -88,11 +127,27 @@ function likePattern(term: string): string {
return `%${term.replace(/[\\%_]/g, '\\$&')}%`;
}
+/**
+ * likePattern in Postgres SQL, around the folded term: unaccent maps fullwidth `%` and `_` to `%` and
+ * `_`, so escaping before it would let them through as wildcards.
+ */
+function postgresLikePattern(placeholder: string): string {
+ return String.raw`'%' || replace(replace(replace(openwa_unaccent(${placeholder}), '\', '\\'), '%', '\%'), '_', '\_') || '%'`;
+}
+
/** Shorter than a trigram once folded, so no trigram index can look it up. */
function isShortTerm(term: string): boolean {
return [...foldSearchText(term)].length < SEARCH_CONTAINS_MIN_TERM_LENGTH;
}
+/**
+ * Whether pg_trgm can look `term` up. It takes trigrams from runs of letters and digits only, so `:-)` or
+ * `😂😂😂` give it nothing and the search would scan the table; SQLite's trigram tokenizer indexes them.
+ */
+function postgresCanIndex(term: string): boolean {
+ return POSTGRES_TRIGRAM_SOURCE.test(foldSearchText(term));
+}
+
/** Shape returned by the dialect-specific SELECT in buildSqlite / buildPostgres. */
interface FtsResultRow {
id: string;
@@ -121,20 +176,34 @@ type PlaceholderFn = () => string;
*
* Two match modes, each on its own index: `word` (the default) matches whole words through the word
* index above and ranks them in SQL; `contains` matches any substring of three or more characters,
- * case- and accent-insensitively, through the trigram index of migration
- * 1792400000000-AddMessagesTrigramSearch (an FTS5 trigram table on SQLite, a pg_trgm GIN index on
- * Postgres), and ranks a bounded window of candidates in the application (see searchContains).
+ * case- and accent-insensitively, through a trigram index (an FTS5 trigram table on SQLite, a pg_trgm
+ * GIN index on Postgres) that only SEARCH_CONTAINS_ENABLED=true builds (see buildContainsIndex), and
+ * ranks a bounded window of candidates in the application (see searchContains).
*/
@Injectable()
-export class BuiltInFtsProvider implements SearchProvider, OnModuleInit {
+export class BuiltInFtsProvider implements SearchProvider, OnModuleInit, OnApplicationBootstrap {
readonly id = 'builtin-fts';
readonly label = 'Built-in database full-text search';
private readonly logger = createLogger('BuiltInFtsProvider');
+ /** SEARCH_CONTAINS_ENABLED: the trigram index is opt-in, since it costs about three times the message text. */
+ private readonly containsEnabled: boolean;
+
+ /** Why match=contains answers 501 right now (a CONTAINS_UNAVAILABLE message), or null once its index is ready. */
+ private containsUnavailable: string | null;
+
// OpenWA has two TypeORM connections (main: auth/audit SQLite, data: messages). Bind explicitly to
// 'data' so the provider queries the connection that owns the `messages` table + the FTS migration,
// never the default/`main` one. The bare `DataSource` type alone is ambiguous with two connections.
- constructor(@InjectDataSource('data') private readonly dataSource: DataSource) {}
+ constructor(
+ @InjectDataSource('data') private readonly dataSource: DataSource,
+ // Optional so the provider still builds in a unit test without them; contains search is then off.
+ @Optional() config?: ConfigService,
+ @Optional() private readonly adapterHost?: HttpAdapterHost,
+ ) {
+ this.containsEnabled = config?.get<boolean>('search.containsEnabled') === true;
+ this.containsUnavailable = this.containsEnabled ? CONTAINS_UNAVAILABLE.building : CONTAINS_UNAVAILABLE.disabled;
+ }
/**
* Self-heals the FTS schema at bootstrap so search works under an opted-in DATABASE_SYNCHRONIZE=true
@@ -158,16 +227,17 @@ export class BuiltInFtsProvider implements SearchProvider, OnModuleInit {
// Never fail boot over index creation: the route 501s cleanly via probeFts()/ensureFts() instead.
this.logger.error(`Failed to ensure FTS schema: ${e instanceof Error ? e.message : String(e)}`);
}
- try {
- this.trigramAvailable = await this.ensureTrigramSchema();
- if (this.trigramAvailable) {
- this.logger.log('Trigram index ready');
- } else {
- this.logger.warn('Trigram index unavailable; match=contains searches will return 501');
- }
- } catch (e) {
- this.logger.error(`Failed to ensure trigram schema: ${e instanceof Error ? e.message : String(e)}`);
- }
+ }
+
+ /**
+ * Starts the contains index build once the HTTP listener is up, and does not wait for it: on a large
+ * table it runs long enough that a boot held for it would miss its health probe, and the Helm startup
+ * probe would restart the pod in a loop (docs/14).
+ */
+ onApplicationBootstrap(): void {
+ if (!this.containsEnabled || !this.adapterHost) return;
+ if (this.adapterHost.listening) void this.buildContainsIndex();
+ else this.adapterHost.listen$.subscribe(() => void this.buildContainsIndex());
}
/**
@@ -284,7 +354,7 @@ export class BuiltInFtsProvider implements SearchProvider, OnModuleInit {
* tokenizer, so both get the same repair guarantees. Idempotent: on a healthy index every step is a
* no-op or a trigger replaced by an identical one.
*/
- private async ensureSqliteIndex({ table, tokenize }: Fts5Index): Promise<void> {
+ private async ensureSqliteIndex({ table, tokenize, batched }: Fts5Index): Promise<void> {
const docsize = `${table}_docsize`;
await this.dataSource.query(
`CREATE VIRTUAL TABLE IF NOT EXISTS "${table}" USING fts5(body, content='messages', content_rowid='rowid'${tokenize ? `, tokenize='${tokenize}'` : ''})`,
@@ -317,19 +387,28 @@ export class BuiltInFtsProvider implements SearchProvider, OnModuleInit {
this.logger.warn(`${table} holds rows the messages table no longer has; re-indexing every message`);
await this.dataSource.query(`INSERT INTO "${table}"("${table}") VALUES ('delete-all')`);
}
- const gapRow: unknown[] = await this.dataSource.query(
- `SELECT EXISTS(SELECT 1 FROM "messages" m WHERE m."body" IS NOT NULL
+ // A batched index is filled after its triggers exist instead (backfillSqliteIndexInBatches).
+ if (!batched) {
+ const gapRow: unknown[] = await this.dataSource.query(
+ `SELECT EXISTS(SELECT 1 FROM "messages" m WHERE m."body" IS NOT NULL
AND NOT EXISTS (SELECT 1 FROM "${docsize}" d WHERE d."rowid" = m."rowid")) AS missing`,
- );
- if (Number((gapRow as Array<{ missing: number }>)[0]?.missing)) {
- // `body IS NOT NULL` mirrors the migration's own backfill exactly: a repair that indexed a NULL
- // body as an empty document would leave the index disagreeing with the schema it repairs.
- await this.dataSource.query(
- `INSERT INTO "${table}"("rowid", "body")
+ );
+ if (Number((gapRow as Array<{ missing: number }>)[0]?.missing)) {
+ // `body IS NOT NULL` mirrors the migration's own backfill exactly: a repair that indexed a NULL
+ // body as an empty document would leave the index disagreeing with the schema it repairs.
+ await this.dataSource.query(
+ `INSERT INTO "${table}"("rowid", "body")
SELECT m."rowid", m."body" FROM "messages" m WHERE m."body" IS NOT NULL
AND NOT EXISTS (SELECT 1 FROM "${docsize}" d WHERE d."rowid" = m."rowid")`,
- );
+ );
+ }
}
+ // While a batched backfill runs, an edited or deleted row may not be indexed yet, and an FTS5
+ // 'delete' of a document an external-content index never held corrupts it: delete only what docsize
+ // lists.
+ const deleteOld = batched
+ ? `INSERT INTO "${table}"("${table}", "rowid", "body") SELECT 'delete', old."rowid", old."body" WHERE EXISTS (SELECT 1 FROM "${docsize}" WHERE "rowid" = old."rowid");`
+ : `INSERT INTO "${table}"("${table}", "rowid", "body") VALUES ('delete', old."rowid", old."body");`;
// The three sync triggers are CREATE-OR-REPLACE by name via DROP + CREATE; idempotent on re-run.
await this.dataSource.query(`DROP TRIGGER IF EXISTS ${table}_ai`);
await this.dataSource.query(`CREATE TRIGGER ${table}_ai AFTER INSERT ON "messages" BEGIN
@@ -337,115 +416,164 @@ export class BuiltInFtsProvider implements SearchProvider, OnModuleInit {
END`);
await this.dataSource.query(`DROP TRIGGER IF EXISTS ${table}_ad`);
await this.dataSource.query(`CREATE TRIGGER ${table}_ad AFTER DELETE ON "messages" BEGIN
- INSERT INTO "${table}"("${table}", "rowid", "body") VALUES ('delete', old."rowid", old."body");
+ ${deleteOld}
END`);
await this.dataSource.query(`DROP TRIGGER IF EXISTS ${table}_au`);
// WHEN clause skips re-indexing on body-unchanged updates (acks/reactions/status) — mirrors the
// migration. NULL-safe via `IS NOT` so NULL↔non-NULL body transitions still fire.
await this.dataSource.query(
`CREATE TRIGGER ${table}_au AFTER UPDATE ON "messages" WHEN OLD.body IS NOT NEW.body BEGIN
- INSERT INTO "${table}"("${table}", "rowid", "body") VALUES ('delete', old."rowid", old."body");
+ ${deleteOld}
INSERT INTO "${table}"("rowid", "body") VALUES (new."rowid", new."body");
END`,
);
+ if (batched) await this.backfillSqliteIndexInBatches(table);
}
/**
- * Brings up the match=contains index at boot, the way ensureFtsSchema does for the word index, and
- * mirrors migration 1792400000000-AddMessagesTrigramSearch. Returns false, leaving no schema behind,
- * when the database cannot hold the index: then only match=contains answers 501, while match=word
- * keeps working.
+ * Indexes the rows the index lacks, TRIGRAM_BACKFILL_BATCH rowids at a time, yielding to the event loop
+ * between batches so requests and sessions keep being served through a build of seconds per million
+ * messages. The triggers already index every row stored meanwhile.
*/
- private async ensureTrigramSchema(): Promise<boolean> {
- if (this.dataSource.options.type === 'postgres') return this.ensurePostgresTrigramSchema();
- if (!(await this.sqliteHasFts5())) return false;
+ private async backfillSqliteIndexInBatches(table: string): Promise<void> {
+ const [{ last }] = await this.dataSource.query<Array<{ last: number | null }>>(
+ `SELECT max("rowid") AS last FROM "messages"`,
+ );
+ for (let from = 0; from < (last ?? 0); from += TRIGRAM_BACKFILL_BATCH) {
+ await this.dataSource.query(
+ `INSERT INTO "${table}"("rowid", "body")
+ SELECT m."rowid", m."body" FROM "messages" m
+ WHERE m."rowid" > ? AND m."rowid" <= ? AND m."body" IS NOT NULL
+ AND NOT EXISTS (SELECT 1 FROM "${table}_docsize" d WHERE d."rowid" = m."rowid")`,
+ [from, from + TRIGRAM_BACKFILL_BATCH],
+ );
+ await setImmediate();
+ }
+ }
+
+ /**
+ * Builds the match=contains index when SEARCH_CONTAINS_ENABLED is on, and is a no-op otherwise. Never
+ * throws: a failure leaves contains search answering 501 with the reason, and match=word untouched.
+ * Public so a spec can await the build onApplicationBootstrap leaves running.
+ */
+ async buildContainsIndex(): Promise<void> {
+ if (!this.containsEnabled) return;
+ const start = Date.now();
+ try {
+ this.containsUnavailable =
+ this.dataSource.options.type === 'postgres'
+ ? await this.buildPostgresTrigramIndex()
+ : await this.buildSqliteTrigramIndex();
+ } catch (e) {
+ this.containsUnavailable = CONTAINS_UNAVAILABLE.failed;
+ this.logger.error(`Failed to build the trigram index: ${e instanceof Error ? e.message : String(e)}`);
+ return;
+ }
+ if (this.containsUnavailable) this.logger.warn(this.containsUnavailable);
+ else this.logger.log(`Trigram index ready in ${Date.now() - start} ms`);
+ }
+
+ /** Returns why this SQLite library cannot hold the index, or null once the index is complete. */
+ private async buildSqliteTrigramIndex(): Promise<string | null> {
+ if (!(await this.sqliteHasFts5())) return CONTAINS_UNAVAILABLE.sqliteTrigram;
const rows: Array<{ version: string }> = await this.dataSource.query(`SELECT sqlite_version() AS version`);
- if (!sqliteSupportsTrigramDiacritics(rows[0]?.version ?? '')) return false;
+ if (!sqliteSupportsTrigramDiacritics(rows[0]?.version ?? '')) return CONTAINS_UNAVAILABLE.sqliteTrigram;
await this.ensureSqliteIndex(TRIGRAM_INDEX);
- return true;
+ return null;
}
/**
- * Creates the wrapper function, the trigram index and the recency index when they are missing but
- * both extensions are installed. It never runs CREATE EXTENSION: that needs a privilege managed
- * Postgres often withholds, and the migration has already tried it inside a savepoint. An operator
- * who installs the extensions later gets the indexes on the next boot. Like the word index, the
- * builds are not CONCURRENTLY, so a large table is best upgraded in a maintenance window.
+ * Runs createPostgresTrigramSchema on a connection of its own: CREATE INDEX CONCURRENTLY cannot run
+ * inside a transaction, which rules out a migration, and outlasts the runtime pool's statement_timeout,
+ * which is lifted here. The advisory lock lets one replica build while the others answer 501 until the
+ * index is valid. Returns why the index cannot be used yet, or null once it is valid.
*/
- private async ensurePostgresTrigramSchema(): Promise<boolean> {
- const [state] = await this.postgresTrigramState();
- if (!state?.trgm || !state.unaccent) return false;
- if (!state.fn) {
- await this.dataSource.query(postgresUnaccentFunctionDdl(state.unaccent));
+ private async buildPostgresTrigramIndex(): Promise<string | null> {
+ const runner = this.dataSource.createQueryRunner();
+ await runner.connect();
+ try {
+ const [{ locked }] = await runner.manager.query<Array<{ locked: boolean }>>(
+ `SELECT pg_try_advisory_lock($1, $2) AS locked`,
+ [...TRIGRAM_BUILD_LOCK_KEYS],
+ );
+ if (!locked) return CONTAINS_UNAVAILABLE.building;
+ try {
+ await runner.query('SET statement_timeout = 0');
+ return await this.createPostgresTrigramSchema(runner);
+ } finally {
+ // The connection returns to the pool: the next borrower must find its timeout and no lock.
+ await runner.query('RESET statement_timeout');
+ await runner.query('SELECT pg_advisory_unlock($1, $2)', [...TRIGRAM_BUILD_LOCK_KEYS]);
+ }
+ } finally {
+ await runner.release();
}
- if (!state.idx) {
- await this.dataSource.query(
- `CREATE INDEX IF NOT EXISTS "idx_messages_body_trgm" ON "messages" USING GIN (openwa_unaccent("body") ${quoteIdent(state.trgm)}.gin_trgm_ops)`,
+ }
+
+ /**
+ * Creates what is missing of the extensions, the wrapper function and the GIN trigram index, the index
+ * with CREATE INDEX CONCURRENTLY so writes go on during the build. Holding the build lock proves an
+ * INVALID index is what a killed build left behind rather than one in progress, so it is rebuilt.
+ */
+ private async createPostgresTrigramSchema(runner: QueryRunner): Promise<string | null> {
+ try {
+ // IF NOT EXISTS returns before any privilege check, so a role that may not create the extensions
+ // still gets past once an operator has.
+ await runner.query('CREATE EXTENSION IF NOT EXISTS pg_trgm');
+ await runner.query('CREATE EXTENSION IF NOT EXISTS unaccent');
+ } catch (e) {
+ this.logger.warn(
+ `Cannot create the pg_trgm and unaccent extensions: ${e instanceof Error ? e.message : String(e)}`,
);
+ return CONTAINS_UNAVAILABLE.postgresExtensions;
}
- if (!state.recent) {
- await this.dataSource.query(
- `CREATE INDEX IF NOT EXISTS "idx_messages_created_at_id" ON "messages" ("createdAt", "id")`,
+ const [{ trgm, unaccent, fn, idx }] = await this.postgresTrigramState(runner);
+ if (!trgm || !unaccent) return CONTAINS_UNAVAILABLE.postgresExtensions;
+ if (!fn) await runner.query(postgresUnaccentFunctionDdl(unaccent));
+ if (idx === false) await runner.query(`DROP INDEX CONCURRENTLY IF EXISTS "idx_messages_body_trgm"`);
+ if (idx !== true) {
+ await runner.query(
+ `CREATE INDEX CONCURRENTLY IF NOT EXISTS "idx_messages_body_trgm" ON "messages" USING GIN (openwa_unaccent("body") ${quoteIdent(trgm)}.gin_trgm_ops)`,
);
}
- return true;
+ return null;
}
/**
- * One catalog read of everything contains search needs on Postgres. The function and the indexes
- * are looked up through the session search_path, the same resolution the runtime queries use (see
- * the to_regclass note in probeFts).
+ * One catalog read of everything contains search needs on Postgres. The function and the index are
+ * looked up through the session search_path, the same resolution the runtime queries use (see the
+ * to_regclass note in probeFts).
*/
- private postgresTrigramState(): Promise<TrigramStateRow[]> {
- const indexExists = (name: string) =>
- `EXISTS (SELECT 1 FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid WHERE i.indrelid = to_regclass('messages') AND c.relname = '${name}')`;
- return this.dataSource.query<TrigramStateRow[]>(
+ private postgresTrigramState(runner?: QueryRunner): Promise<TrigramStateRow[]> {
+ return (runner ?? this.dataSource).manager.query<TrigramStateRow[]>(
`SELECT (SELECT n.nspname FROM pg_extension e JOIN pg_namespace n ON n.oid = e.extnamespace WHERE e.extname = 'pg_trgm') AS trgm,
(SELECT n.nspname FROM pg_extension e JOIN pg_namespace n ON n.oid = e.extnamespace WHERE e.extname = 'unaccent') AS unaccent,
to_regprocedure('openwa_unaccent(text)') IS NOT NULL AS fn,
- ${indexExists('idx_messages_body_trgm')} AS idx,
- ${indexExists('idx_messages_created_at_id')} AS recent`,
+ (SELECT i.indisvalid FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid WHERE i.indrelid = to_regclass('messages') AND c.relname = 'idx_messages_body_trgm') AS idx`,
);
}
- /** Whether the match=contains index is usable; cached like ftsAvailable after the first probe. */
- private trigramAvailable: boolean | null = null;
-
/**
- * Probes the contains index once per instance, with the same fail-closed posture as probeFts: a
- * Postgres probe error reports unavailable without caching, so a later call re-probes.
+ * Marks contains search ready once the catalog holds a valid index, which another replica may have
+ * built while this one waited on the lock. Fails closed like probeFts: an error leaves it unavailable.
*/
- private async probeTrigram(): Promise<boolean> {
- if (this.trigramAvailable !== null) return this.trigramAvailable;
- if (this.dataSource.options.type === 'postgres') {
- let state: TrigramStateRow | undefined;
- try {
- [state] = await this.postgresTrigramState();
- } catch (e) {
- this.logger.warn(
- `Trigram probe failed; treating contains search as unavailable: ${e instanceof Error ? e.message : String(e)}`,
- );
- return false;
- }
- // The recency index only speeds up the candidate query; contains search is correct without it.
- this.trigramAvailable = Boolean(state?.trgm && state.fn && state.idx);
- } else {
- const rows: unknown[] = await this.dataSource.query(
- `SELECT name FROM sqlite_master WHERE type='table' AND name='${TRIGRAM_INDEX.table}'`,
+ private async probePostgresTrigram(): Promise<void> {
+ try {
+ const [{ trgm, unaccent, fn, idx }] = await this.postgresTrigramState();
+ if (trgm && unaccent && fn && idx) this.containsUnavailable = null;
+ } catch (e) {
+ this.logger.warn(
+ `Trigram probe failed; treating contains search as unavailable: ${e instanceof Error ? e.message : String(e)}`,
);
- this.trigramAvailable = Array.isArray(rows) && rows.length === 1;
}
- return this.trigramAvailable;
}
- /** Guards a match=contains search: a missing trigram index is a 501 that names the fix. */
- private async ensureTrigram(): Promise<void> {
- if (await this.probeTrigram()) return;
- throw new NotImplementedException(
- this.dataSource.options.type === 'postgres'
- ? 'Contains search is unavailable: install the pg_trgm and unaccent extensions (CREATE EXTENSION pg_trgm; CREATE EXTENSION unaccent;) and restart.'
- : 'Contains search is unavailable: this SQLite build has no FTS5 trigram tokenizer (SQLite 3.45+ is required).',
- );
+ /** Guards a match=contains search: disabled, still building or unbuildable is a 501 that says which. */
+ private async ensureContains(): Promise<void> {
+ if (this.containsUnavailable && this.containsEnabled && this.dataSource.options.type === 'postgres') {
+ await this.probePostgresTrigram();
+ }
+ if (this.containsUnavailable) throw new NotImplementedException(this.containsUnavailable);
}
async search(query: SearchQuery): Promise<SearchResults> {
@@ -502,16 +630,16 @@ export class BuiltInFtsProvider implements SearchProvider, OnModuleInit {
* is never withheld.
*/
private async searchContains(query: SearchQuery): Promise<SearchResults> {
- const terms = BuiltInFtsProvider.containsTermsOf(query.q);
- await this.ensureTrigram();
+ await this.ensureContains();
+ const isPostgres = this.dataSource.options.type === 'postgres';
+ const terms = BuiltInFtsProvider.containsTermsOf(query.q, isPostgres);
const start = Date.now();
const limit = Math.max(1, Math.min(query.limit ?? 50, SEARCH_LIMIT_MAX));
const offset = Math.max(0, query.offset ?? 0);
- const { sql, params } =
- this.dataSource.options.type === 'postgres'
- ? this.buildPostgresCandidates(query, terms)
- : this.buildSqliteCandidates(query, terms);
+ const { sql, params } = isPostgres
+ ? this.buildPostgresCandidates(query, terms)
+ : this.buildSqliteCandidates(query, terms);
const rows = await this.dataSource.query<FtsResultRow[]>(sql, params);
const ranked = rows
@@ -589,15 +717,18 @@ export class BuiltInFtsProvider implements SearchProvider, OnModuleInit {
/**
* The whitespace-separated terms of a contains query, repeats folded away (`Invoice invoice` is one
* term). Every term must appear in the body, as with match=word. A trigram index cannot answer a term
- * shorter than three characters (FTS5 returns nothing for it, pg_trgm scans every row), so such terms
- * only narrow the rows a longer term already found, and a query made of short terms alone is a 400
- * rather than a full-table scan.
+ * it takes no trigram from (FTS5 returns nothing for it, pg_trgm scans every row), so such terms only
+ * narrow the rows another term already found, and a query made of them alone is a 400 rather than a
+ * full-table scan. SQLite takes trigrams from any three characters, pg_trgm only from three letters or
+ * digits in a row (see postgresCanIndex).
*/
- private static containsTermsOf(q: string): string[] {
+ private static containsTermsOf(q: string, isPostgres: boolean): string[] {
const terms = containsTerms(q);
- if (!terms.some(term => !isShortTerm(term))) {
+ const canIndex = isPostgres ? postgresCanIndex : (term: string) => !isShortTerm(term);
+ if (!terms.some(canIndex)) {
+ const unit = isPostgres ? 'letters or digits in a row' : 'characters';
throw new BadRequestException(
- `A contains search needs at least one term of ${SEARCH_CONTAINS_MIN_TERM_LENGTH} or more characters.`,
+ `A contains search needs at least one term of ${SEARCH_CONTAINS_MIN_TERM_LENGTH} or more ${unit}.`,
);
}
return terms;
@@ -676,17 +807,19 @@ export class BuiltInFtsProvider implements SearchProvider, OnModuleInit {
// --- Postgres pg_trgm candidates (match=contains) --------------------------
// SQL appearance order: one ILIKE per term -> filters -> LIMIT. Each ILIKE compares
// openwa_unaccent(body), the exact expression the GIN trigram index holds; ILIKE folds case and
- // openwa_unaccent folds accents, matching the SQLite trigram tokenizer, and each term has its `%`,
- // `_` and `\` escaped so it matches literally. The most recently stored first, as on SQLite:
- // idx_messages_created_at_id lets the planner walk them backward and stop at the window when a term
- // is common, while for a rare one it reads the few matches from the GIN index and sorts them.
+ // openwa_unaccent folds accents, and each term has its `%`, `_` and `\` escaped after that fold so it
+ // matches literally (postgresLikePattern). unaccent also transliterates letters remove_diacritics
+ // leaves alone (Ł, ß, Ø, œ), so Postgres finds some matches SQLite does not (docs/26). The most
+ // recently stored first, as on SQLite: IDX_messages_createdAt lets the planner walk them backward and
+ // stop at the window when a term is common, while for a rare one it reads the few matches from the
+ // GIN index and sorts them.
private buildPostgresCandidates(q: SearchQuery, terms: string[]) {
const ph = this.pgPlaceholder();
const params: unknown[] = [];
const where = terms.map(term => {
const placeholder = ph();
- params.push(likePattern(term));
- return `openwa_unaccent(m."body") ILIKE openwa_unaccent(${placeholder})`;
+ params.push(term);
+ return `openwa_unaccent(m."body") ILIKE ${postgresLikePattern(placeholder)}`;
});
this.applyFilters(where, params, q, 'm.', ph);
const sql = `SELECT ${CONTAINS_COLUMNS} FROM messages m WHERE ${where.join(' AND ')} ORDER BY m."createdAt" DESC, m."id" DESC LIMIT ${ph()}`;
diff --git a/src/modules/search/search.constants.ts b/src/modules/search/search.constants.ts
index 1dbe9e86c..4d293527d 100644
--- a/src/modules/search/search.constants.ts
+++ b/src/modules/search/search.constants.ts
@@ -37,7 +37,8 @@ export const SEARCH_MATCH_MODES = ['word', 'contains'] as const satisfies readon
/**
* Shortest term a match=contains search can look up in a trigram index: a trigram is three characters,
* so a shorter term has none (SQLite's trigram table returns nothing for it, pg_trgm scans every row).
- * A query needs at least one term this long; shorter ones only narrow what it found.
+ * A query needs at least one term this long, counted in letters and digits in a row on Postgres, whose
+ * pg_trgm takes trigrams from nothing else; shorter ones only narrow what it found.
*/
export const SEARCH_CONTAINS_MIN_TERM_LENGTH = 3;
diff --git a/src/modules/search/search.controller.ts b/src/modules/search/search.controller.ts
index afe3328f6..f9fa8cebd 100644
--- a/src/modules/search/search.controller.ts
+++ b/src/modules/search/search.controller.ts
@@ -26,14 +26,15 @@ export class SearchController {
status: 400,
description:
'Empty or whitespace-only "q", or one over 1000 characters or 32 words; an unknown "match"; or a ' +
- '"match=contains" query whose terms are all shorter than 3 characters',
+ '"match=contains" query with no term of 3 or more characters (on PostgreSQL, 3 letters or digits in a row)',
})
@ApiResponse({ status: 403, description: 'Chat scope denied or chat-restricted search uses a plugin provider' })
@ApiResponse({
status: 501,
description:
'No search provider configured, or the built-in provider lacks the index the requested match mode needs ' +
- '(for "match=contains" on PostgreSQL: the pg_trgm and unaccent extensions)',
+ '(for "match=contains": SEARCH_CONTAINS_ENABLED=true and a finished index build, and on PostgreSQL the ' +
+ 'pg_trgm and unaccent extensions)',
})
@ApiResponse({
status: 502,
@@ -58,7 +59,7 @@ export class SearchController {
enum: SEARCH_MATCH_MODES,
description:
'word (default): whole words. contains: any part of a word, ignoring case and accents; ranks the ' +
- `${SEARCH_CONTAINS_WINDOW} most recently stored matches by match quality`,
+ `${SEARCH_CONTAINS_WINDOW} most recently stored matches by match quality; needs SEARCH_CONTAINS_ENABLED=true`,
})
@ApiQuery({ name: 'sessionId', required: false, description: 'Restrict to a single session' })
@ApiQuery({
diff --git a/test/search.e2e-spec.ts b/test/search.e2e-spec.ts
index 8e6aab376..7b45b1d4d 100644
--- a/test/search.e2e-spec.ts
+++ b/test/search.e2e-spec.ts
@@ -108,6 +108,14 @@ describe('GET /api/search (e2e)', () => {
await request(app.getHttpServer()).get('/api/search?q=hello').expect(401);
});
+ it('answers 501 naming SEARCH_CONTAINS_ENABLED for match=contains while the setting is off (default)', async () => {
+ const res = await request(app.getHttpServer())
+ .get('/api/search?q=hello&match=contains')
+ .set('X-API-Key', 'dev-admin-key')
+ .expect(501);
+ expect((res.body as { message: string }).message).toContain('SEARCH_CONTAINS_ENABLED=true');
+ });
+
it('returns an empty result page (not 501) for a non-matching term', async () => {
const res = await request(app.getHttpServer())
.get('/api/search?q=zzzznomatch')
@@ -130,6 +138,62 @@ describe('GET /api/search (e2e)', () => {
});
});
+/**
+ * SEARCH_CONTAINS_ENABLED=true over the whole stack: the provider starts its trigram build only once the
+ * HTTP listener is up (HttpAdapterHost.listen$), so this instance listens on an ephemeral port, and
+ * match=contains turns from 501 into hits when the build completes.
+ */
+describe('GET /api/search?match=contains with SEARCH_CONTAINS_ENABLED=true (e2e)', () => {
+ let app: INestApplication<App>;
+ const previous = process.env.SEARCH_CONTAINS_ENABLED;
+
+ beforeAll(async () => {
+ process.env.SEARCH_CONTAINS_ENABLED = 'true';
+ const moduleFixture = await Test.createTestingModule({ imports: [AppModule] }).compile();
+ app = moduleFixture.createNestApplication();
+ applyGlobalValidation(app);
+ await app.init();
+ const dataSource = app.get<DataSource>(getDataSourceToken('data'));
+ await new AddMessagesFts1782400000000().up(dataSource.createQueryRunner());
+ await app.get<Repository<Message>>(getRepositoryToken(Message, 'data')).insert({
+ sessionId: 'sess-search-2',
+ chatId: '[email protected]',
+ body: 'Your Invoice is attached',
+ type: 'text',
+ direction: MessageDirection.INCOMING,
+ timestamp: 1700000002000,
+ });
+ await app.listen(0);
+ });
+
+ afterAll(async () => {
+ if (previous === undefined) delete process.env.SEARCH_CONTAINS_ENABLED;
+ else process.env.SEARCH_CONTAINS_ENABLED = previous;
+ try {
+ await app?.close();
+ } catch {
+ /* ignore teardown-only multi-datasource quirk */
+ }
+ });
+
+ it('finds part of a word once the build that followed listen() completes', async () => {
+ const search = () =>
+ request(app.getHttpServer()).get('/api/search?q=voic&match=contains').set('X-API-Key', 'dev-admin-key');
+ let res = await search();
+ for (let attempt = 0; res.status === 501 && attempt < 100; attempt++) {
+ await new Promise(resolve => setTimeout(resolve, 50));
+ res = await search();
+ }
+
+ expect(res.status).toBe(200);
+ const body = res.body as { hits: Array<{ sessionId: string; snippet: string }> };
+ expect(body.hits.map(h => h.sessionId)).toEqual(['sess-search-2']);
+ expect(body.hits[0].snippet).toContain('In<mark>voic</mark>e');
+ });
+});
+
/**
* The SEARCH_ENABLED=false gate (src/app.module.ts): the top-level conditional only pushes
* SearchModule into `searchModules` (spread into AppModule's @Module imports) when the env is not