TypeScript
fix(server): message search keeps results when a short non-Latin word differs in case (#30332)
MarcoDotIOb800f9919bcb
diff --git a/apps/server/src/orchestration-v2/ThreadSearch.test.ts b/apps/server/src/orchestration-v2/ThreadSearch.test.tsindex b8d75340ec6..e758280f938 100644--- a/apps/server/src/orchestration-v2/ThreadSearch.test.ts+++ b/apps/server/src/orchestration-v2/ThreadSearch.test.ts@@ -270,7 +270,7 @@ it.layer(layerTest)("ThreadSearch", (it) => {for (const match of ['"needle"', null]) {const plan = yield* sql<{readonly detail: string;- }>`EXPLAIN QUERY PLAN ${ThreadSearch.messageHitsQuery(sql, { match, substrings: "[]", projectId: null, excludedThreadIds: "[]" })}`;+ }>`EXPLAIN QUERY PLAN ${ThreadSearch.messageHitsQuery(sql, { match, substrings: "[]", shortSpellings: "[]", projectId: null, excludedThreadIds: "[]" })}`;assert.isFalse(plan.some((row) => row.detail.includes("TEMP B-TREE")),plan.map((row) => row.detail).join("\n"),@@ -732,6 +732,85 @@ it.layer(layerTest)("ThreadSearch filters before it ranks", (it) => {);});+it.effect.each([true, false])(+ "ThreadSearch narrows short words outside ASCII before the hit limit (indexed: %s)",+ (indexed) =>+ Effect.gen(function* () {+ const projections = yield* ProjectionStore.ProjectionStoreV2;+ const search = yield* ThreadSearch.ThreadSearch;+ const sql = yield* SqlClient.SqlClient;+ const project = ProjectId.make("project:short-limit");+ const older = ThreadId.make("thread:short-older");+ const newer = ThreadId.make("thread:short-newer");+ yield* createProject(project);+ yield* Effect.forEach(+ [+ thread(older, project),+ message(older, "older", "user", "В логах ошибка"),+ thread(newer, project),+ ],+ projections.apply,+ { discard: true },+ );+ if (!indexed) {+ yield* sql`DROP TRIGGER orchestration_v2_message_search_insert`;+ yield* sql`DROP TRIGGER orchestration_v2_message_search_update`;+ yield* sql`DROP TRIGGER orchestration_v2_message_search_delete`;+ yield* sql`DROP TABLE orchestration_v2_message_search`;+ }+ // More newer hits for the long word than one read holds, none with the short one.+ yield* sql`+ WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 2000)+ INSERT INTO orchestration_v2_projection_messages+ (message_id, thread_id, role, streaming, created_at, updated_at, payload_json)+ SELECT 'short-newer-' || i, ${newer}, 'assistant', 0, ${DateTime.formatIso(at(5))},+ ${DateTime.formatIso(at(5))}, json_object('text', 'ошибка здесь')+ FROM n+ `;+ const { matches } = yield* search.search({ query: "ошибка в" });+ assert.deepEqual(+ matches.map((match) => match.threadId),+ [older],+ );+ }).pipe(Effect.provide(layerTest)),+);++it.layer(layerTest)("ThreadSearch short words outside ASCII", (it) => {+ it.effect("finds them in any case or accent, with or without the index", () =>+ Effect.gen(function* () {+ const projections = yield* ProjectionStore.ProjectionStoreV2;+ const search = yield* ThreadSearch.ThreadSearch;+ const sql = yield* SqlClient.SqlClient;+ const project = ProjectId.make("project:short-non-ascii");+ const russian = ThreadId.make("thread:short-russian");+ const french = ThreadId.make("thread:short-french");+ yield* createProject(project);+ yield* Effect.forEach(+ [+ thread(russian, project),+ message(russian, "russian", "user", "В логах ошибка"),+ thread(french, project),+ message(french, "french", "user", "École fermée à Lyon"),+ ],+ projections.apply,+ { discard: true },+ );+ const threadIds = (query: string) =>+ search.search({ query }).pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId)));++ const assertMatches = Effect.gen(function* () {+ assert.deepEqual(yield* threadIds("ошибка в"), [russian]);+ assert.deepEqual(yield* threadIds("fermée éc"), [french]);+ // They still narrow: a word the message lacks rules it out.+ assert.deepEqual(yield* threadIds("ошибка я"), []);+ });+ yield* assertMatches;+ yield* sql`DROP TABLE orchestration_v2_message_search`;+ yield* assertMatches;+ }),+ );+});+it.layer(layerTest)("ThreadSearch across scripts and without the index", (it) => {it.effect("matches Korean, Japanese, and two-character Chinese words, and whole short words",diff --git a/apps/server/src/orchestration-v2/ThreadSearch.ts b/apps/server/src/orchestration-v2/ThreadSearch.tsindex 3c97122c1c6..04f68ef64c3 100644--- a/apps/server/src/orchestration-v2/ThreadSearch.ts+++ b/apps/server/src/orchestration-v2/ThreadSearch.ts@@ -67,6 +67,8 @@ const SearchRequest = Schema.Struct({match: Schema.NullOr(Schema.String),/** Tokens every hit must contain, checked on the text. */substrings: TokenList,+ /** For each short word outside ASCII, the spellings a hit must contain one of. */+ shortSpellings: Schema.fromJsonString(Schema.Array(Schema.Array(Schema.String))),projectId: Schema.NullOr(ProjectId),/** Threads the caller already lists, left out before the hit limit. */excludedThreadIds: TokenList,@@ -99,6 +101,20 @@ const MESSAGE_HIT_LIMIT = 2000;const MESSAGE_CANDIDATE_LIMIT = 10_000;// Without the index, how many of the newest messages a search scans.const MESSAGE_SCAN_LIMIT = 20_000;+const NON_ASCII = /[\u0080-\uffff]/;++/** SQLite's lower() folds ASCII only, so a short word is checked as each spelling it can match. */+function shortTokenSpellings(token: string): ReadonlyArray<string> {+ const forms = (char: string) => {+ const plain = char.normalize("NFD").replace(/\p{M}/gu, "");+ return [char, char.toUpperCase(), plain, plain.toUpperCase()];+ };+ const spellings = Array.from(token).reduce<ReadonlyArray<string>>(+ (prefixes, char) => prefixes.flatMap((prefix) => forms(char).map((form) => prefix + form)),+ [""],+ );+ return [...new Set(spellings)];+}/*** The SQL inputs for a message search, or null when the query has no token@@ -114,9 +130,11 @@ function buildMessageSearch(if (long.length === 0) return null;// Two-character CJK words are too short for trigrams.const indexed = hasIndex && long.every((token) => Array.from(token).length >= 3);+ const narrowing = short.filter((token) => !NON_ASCII.test(token));return {match: indexed ? long.map((token) => `"${token.replaceAll('"', '""')}"`).join(" ") : null,- substrings: indexed ? short : [...long, ...short],+ substrings: indexed ? narrowing : [...long, ...narrowing],+ shortSpellings: short.filter((token) => NON_ASCII.test(token)).map(shortTokenSpellings),};}@@ -160,7 +178,7 @@ export function buildSearchSnippet(text: string, matchStart: number): string {// The scan folds ASCII case only, so accent variants and non-ASCII case variants may not match.export const messageHitsQuery = (sql: SqlClient.SqlClient,- { match, substrings, projectId, excludedThreadIds }: typeof SearchRequest.Encoded,+ { match, substrings, shortSpellings, projectId, excludedThreadIds }: typeof SearchRequest.Encoded,) => sql`WITH eligible AS MATERIALIZED (SELECT threads.thread_id, threads.project_id@@ -225,6 +243,13 @@ export const messageHitsQuery = (SELECT 1 FROM json_each(${substrings}) AS tokenWHERE instr(lower(json_extract(messages.payload_json, '$.text')), token.value) = 0)+ AND NOT EXISTS (+ SELECT 1 FROM json_each(${shortSpellings}) AS token+ WHERE NOT EXISTS (+ SELECT 1 FROM json_each(token.value) AS spelling+ WHERE instr(json_extract(messages.payload_json, '$.text'), spelling.value) > 0+ )+ )-- The index's own rowid, so FTS5 returns hits newest first and stops at-- the limit; ordering by the messages table's rowid sorts every hit.ORDER BY "messageRowid" DESC