Skip to content
TypeScript

fix(server): message search keeps results when a short non-Latin word differs in case (#30332)

MarcoDotIOb800f9919bcb
diff --git a/apps/server/src/orchestration-v2/ThreadSearch.test.ts b/apps/server/src/orchestration-v2/ThreadSearch.test.ts
index b8d75340ec6..e758280f938 100644
--- a/apps/server/src/orchestration-v2/ThreadSearch.test.ts
+++ b/apps/server/src/orchestration-v2/ThreadSearch.test.ts
@@ -270,7 +270,7 @@ it.layer(layerTest)("ThreadSearch", (it) => {
for (const match of ['"needle"', null]) {
const plan = yield* sql<{
readonly detail: string;
- }>`EXPLAIN QUERY PLAN ${ThreadSearch.messageHitsQuery(sql, { match, substrings: "[]", projectId: null, excludedThreadIds: "[]" })}`;
+ }>`EXPLAIN QUERY PLAN ${ThreadSearch.messageHitsQuery(sql, { match, substrings: "[]", shortSpellings: "[]", projectId: null, excludedThreadIds: "[]" })}`;
assert.isFalse(
plan.some((row) => row.detail.includes("TEMP B-TREE")),
plan.map((row) => row.detail).join("\n"),
@@ -732,6 +732,85 @@ it.layer(layerTest)("ThreadSearch filters before it ranks", (it) => {
);
});
+it.effect.each([true, false])(
+ "ThreadSearch narrows short words outside ASCII before the hit limit (indexed: %s)",
+ (indexed) =>
+ Effect.gen(function* () {
+ const projections = yield* ProjectionStore.ProjectionStoreV2;
+ const search = yield* ThreadSearch.ThreadSearch;
+ const sql = yield* SqlClient.SqlClient;
+ const project = ProjectId.make("project:short-limit");
+ const older = ThreadId.make("thread:short-older");
+ const newer = ThreadId.make("thread:short-newer");
+ yield* createProject(project);
+ yield* Effect.forEach(
+ [
+ thread(older, project),
+ message(older, "older", "user", "В логах ошибка"),
+ thread(newer, project),
+ ],
+ projections.apply,
+ { discard: true },
+ );
+ if (!indexed) {
+ yield* sql`DROP TRIGGER orchestration_v2_message_search_insert`;
+ yield* sql`DROP TRIGGER orchestration_v2_message_search_update`;
+ yield* sql`DROP TRIGGER orchestration_v2_message_search_delete`;
+ yield* sql`DROP TABLE orchestration_v2_message_search`;
+ }
+ // More newer hits for the long word than one read holds, none with the short one.
+ yield* sql`
+ WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 2000)
+ INSERT INTO orchestration_v2_projection_messages
+ (message_id, thread_id, role, streaming, created_at, updated_at, payload_json)
+ SELECT 'short-newer-' || i, ${newer}, 'assistant', 0, ${DateTime.formatIso(at(5))},
+ ${DateTime.formatIso(at(5))}, json_object('text', 'ошибка здесь')
+ FROM n
+ `;
+ const { matches } = yield* search.search({ query: "ошибка в" });
+ assert.deepEqual(
+ matches.map((match) => match.threadId),
+ [older],
+ );
+ }).pipe(Effect.provide(layerTest)),
+);
+
+it.layer(layerTest)("ThreadSearch short words outside ASCII", (it) => {
+ it.effect("finds them in any case or accent, with or without the index", () =>
+ Effect.gen(function* () {
+ const projections = yield* ProjectionStore.ProjectionStoreV2;
+ const search = yield* ThreadSearch.ThreadSearch;
+ const sql = yield* SqlClient.SqlClient;
+ const project = ProjectId.make("project:short-non-ascii");
+ const russian = ThreadId.make("thread:short-russian");
+ const french = ThreadId.make("thread:short-french");
+ yield* createProject(project);
+ yield* Effect.forEach(
+ [
+ thread(russian, project),
+ message(russian, "russian", "user", "В логах ошибка"),
+ thread(french, project),
+ message(french, "french", "user", "École fermée à Lyon"),
+ ],
+ projections.apply,
+ { discard: true },
+ );
+ const threadIds = (query: string) =>
+ search.search({ query }).pipe(Effect.map(({ matches }) => matches.map((m) => m.threadId)));
+
+ const assertMatches = Effect.gen(function* () {
+ assert.deepEqual(yield* threadIds("ошибка в"), [russian]);
+ assert.deepEqual(yield* threadIds("fermée éc"), [french]);
+ // They still narrow: a word the message lacks rules it out.
+ assert.deepEqual(yield* threadIds("ошибка я"), []);
+ });
+ yield* assertMatches;
+ yield* sql`DROP TABLE orchestration_v2_message_search`;
+ yield* assertMatches;
+ }),
+ );
+});
+
it.layer(layerTest)("ThreadSearch across scripts and without the index", (it) => {
it.effect(
"matches Korean, Japanese, and two-character Chinese words, and whole short words",
diff --git a/apps/server/src/orchestration-v2/ThreadSearch.ts b/apps/server/src/orchestration-v2/ThreadSearch.ts
index 3c97122c1c6..04f68ef64c3 100644
--- a/apps/server/src/orchestration-v2/ThreadSearch.ts
+++ b/apps/server/src/orchestration-v2/ThreadSearch.ts
@@ -67,6 +67,8 @@ const SearchRequest = Schema.Struct({
match: Schema.NullOr(Schema.String),
/** Tokens every hit must contain, checked on the text. */
substrings: TokenList,
+ /** For each short word outside ASCII, the spellings a hit must contain one of. */
+ shortSpellings: Schema.fromJsonString(Schema.Array(Schema.Array(Schema.String))),
projectId: Schema.NullOr(ProjectId),
/** Threads the caller already lists, left out before the hit limit. */
excludedThreadIds: TokenList,
@@ -99,6 +101,20 @@ const MESSAGE_HIT_LIMIT = 2000;
const MESSAGE_CANDIDATE_LIMIT = 10_000;
// Without the index, how many of the newest messages a search scans.
const MESSAGE_SCAN_LIMIT = 20_000;
+const NON_ASCII = /[\u0080-\uffff]/;
+
+/** SQLite's lower() folds ASCII only, so a short word is checked as each spelling it can match. */
+function shortTokenSpellings(token: string): ReadonlyArray<string> {
+ const forms = (char: string) => {
+ const plain = char.normalize("NFD").replace(/\p{M}/gu, "");
+ return [char, char.toUpperCase(), plain, plain.toUpperCase()];
+ };
+ const spellings = Array.from(token).reduce<ReadonlyArray<string>>(
+ (prefixes, char) => prefixes.flatMap((prefix) => forms(char).map((form) => prefix + form)),
+ [""],
+ );
+ return [...new Set(spellings)];
+}
/**
* The SQL inputs for a message search, or null when the query has no token
@@ -114,9 +130,11 @@ function buildMessageSearch(
if (long.length === 0) return null;
// Two-character CJK words are too short for trigrams.
const indexed = hasIndex && long.every((token) => Array.from(token).length >= 3);
+ const narrowing = short.filter((token) => !NON_ASCII.test(token));
return {
match: indexed ? long.map((token) => `"${token.replaceAll('"', '""')}"`).join(" ") : null,
- substrings: indexed ? short : [...long, ...short],
+ substrings: indexed ? narrowing : [...long, ...narrowing],
+ shortSpellings: short.filter((token) => NON_ASCII.test(token)).map(shortTokenSpellings),
};
}
@@ -160,7 +178,7 @@ export function buildSearchSnippet(text: string, matchStart: number): string {
// The scan folds ASCII case only, so accent variants and non-ASCII case variants may not match.
export const messageHitsQuery = (
sql: SqlClient.SqlClient,
- { match, substrings, projectId, excludedThreadIds }: typeof SearchRequest.Encoded,
+ { match, substrings, shortSpellings, projectId, excludedThreadIds }: typeof SearchRequest.Encoded,
) => sql`
WITH eligible AS MATERIALIZED (
SELECT threads.thread_id, threads.project_id
@@ -225,6 +243,13 @@ export const messageHitsQuery = (
SELECT 1 FROM json_each(${substrings}) AS token
WHERE instr(lower(json_extract(messages.payload_json, '$.text')), token.value) = 0
)
+ AND NOT EXISTS (
+ SELECT 1 FROM json_each(${shortSpellings}) AS token
+ WHERE NOT EXISTS (
+ SELECT 1 FROM json_each(token.value) AS spelling
+ WHERE instr(json_extract(messages.payload_json, '$.text'), spelling.value) > 0
+ )
+ )
-- The index's own rowid, so FTS5 returns hits newest first and stops at
-- the limit; ordering by the messages table's rowid sorts every hit.
ORDER BY "messageRowid" DESC