Skip to content

Build your personal knowledge base with Trilium Notes

perf(search): split a note's flat text once per search, not once per token

Elian Doran5133d3210703
diff --git a/packages/trilium-core/src/services/search/expressions/note_flat_text.ts b/packages/trilium-core/src/services/search/expressions/note_flat_text.ts
index 2667d7f914..526aeb4c5b 100644
--- a/packages/trilium-core/src/services/search/expressions/note_flat_text.ts
+++ b/packages/trilium-core/src/services/search/expressions/note_flat_text.ts
@@ -3,7 +3,14 @@ import becca from "../../../becca/becca.js";
import type BNote from "../../../becca/entities/bnote.js";
import NoteSet from "../note_set.js";
import type SearchContext from "../search_context.js";
-import { fuzzyMatchWordWithResult, getAutoMaxEditDistance, normalizeSearchText } from "../utils/text_utils.js";
+import {
+ containsAnyChunk,
+ fuzzyMatchInWords,
+ fuzzyMatchWordWithResult,
+ getAutoMaxEditDistance,
+ normalizeSearchText,
+ splitIntoWords
+} from "../utils/text_utils.js";
import Expression from "./expression.js";
class NoteFlatTextExp extends Expression {
@@ -213,8 +220,32 @@ class NoteFlatTextExp extends Expression {
}
const flatText = flatTexts[i];
+ // Split on the first token that gets as far as the word scan, then reuse it for the
+ // rest. Splitting per token instead repeats work already done for this note: across
+ // a 12-token query over a 22k-note database, over half the splits are repeats.
+ let words: string[] | undefined;
+
for (const token of this.tokens) {
- if (this.smartMatch(flatText, token, searchContext)) {
+ if (flatText.includes(token)) {
+ candidateNotes.push(note);
+ break;
+ }
+
+ if (!searchContext?.enableFuzzyMatching) {
+ continue;
+ }
+
+ const maxDistance = getAutoMaxEditDistance(token.length);
+
+ if (maxDistance <= 0 || !containsAnyChunk(flatText, token, maxDistance)) {
+ continue;
+ }
+
+ words ??= splitIntoWords(flatText);
+ const matchedWord = fuzzyMatchInWords(token, words, maxDistance);
+
+ if (matchedWord) {
+ rememberFuzzyMatch(searchContext, matchedWord);
candidateNotes.push(note);
break;
}
@@ -242,10 +273,7 @@ class NoteFlatTextExp extends Expression {
if (searchContext?.enableFuzzyMatching && getAutoMaxEditDistance(token.length) > 0) {
const matchedWord = fuzzyMatchWordWithResult(token, text);
if (matchedWord) {
- // Track the fuzzy matched word for highlighting
- if (!searchContext.highlightedTokens.includes(matchedWord)) {
- searchContext.highlightedTokens.push(matchedWord);
- }
+ rememberFuzzyMatch(searchContext, matchedWord);
return true;
}
}
@@ -254,6 +282,13 @@ class NoteFlatTextExp extends Expression {
}
}
+/** Keeps a fuzzily matched word for highlighting, so a result can show why it matched. */
+function rememberFuzzyMatch(searchContext: SearchContext | undefined, word: string) {
+ if (searchContext && !searchContext.highlightedTokens.includes(word)) {
+ searchContext.highlightedTokens.push(word);
+ }
+}
+
/**
* The normalized title a note carries under one parent. A branch without a prefix shows the note's
* own title, whose normalized form the note caches, so only a prefixed branch builds a new string.
diff --git a/packages/trilium-core/src/services/search/utils/text_utils.ts b/packages/trilium-core/src/services/search/utils/text_utils.ts
index d8574f3ed1..a2435577cd 100644
--- a/packages/trilium-core/src/services/search/utils/text_utils.ts
+++ b/packages/trilium-core/src/services/search/utils/text_utils.ts
@@ -331,32 +331,16 @@ export function fuzzyMatchWordWithResult(token: string, text: string, maxDistanc
}
// For fuzzy matching, split into words and check each against the token
- const words = normalizedText.split(/\s+/).filter(word => word.length > 0);
- const originalWords = text.split(/\s+/).filter(word => word.length > 0);
+ const words = splitIntoWords(normalizedText);
+ const matched = matchingWordIndex(normalizedToken, words, maxDistance);
- for (let i = 0; i < words.length; i++) {
- const word = words[i];
- const originalWord = originalWords[i];
-
- // A word containing the token is a substring relationship, not a typo, so "sync"
- // must not fuzzy-match "async".
- if (word.length > normalizedToken.length && word.includes(normalizedToken)) {
- continue;
- }
-
- // Skip if word is too different in length for fuzzy matching
- if (Math.abs(word.length - normalizedToken.length) > maxDistance) {
- continue;
- }
-
- // Use optimized edit distance calculation
- const distance = calculateOptimizedEditDistance(normalizedToken, word, maxDistance);
- if (distance <= maxDistance) {
- return originalWord; // Return the original word with case preserved
- }
+ if (matched === -1) {
+ return null;
}
- return null;
+ // Return the original word with case preserved. Lowercasing neither adds nor removes
+ // whitespace, so the two splits agree position for position.
+ return splitIntoWords(text)[matched];
} catch (error) {
// Log error and return null for safety
console.warn('Error in fuzzy word matching:', error);
@@ -364,6 +348,47 @@ export function fuzzyMatchWordWithResult(token: string, text: string, maxDistanc
}
}
+/**
+ * Splits a normalized text into its words.
+ *
+ * Callers that test many tokens against the same text split it once and pass the result to
+ * {@link fuzzyMatchInWords}, rather than paying for the split per token.
+ */
+export function splitIntoWords(text: string): string[] {
+ return text.split(/\s+/).filter(word => word.length > 0);
+}
+
+/**
+ * The word within `maxDistance` edits of `token`, or null. Both the token and the words are
+ * expected to be normalized; the word is returned as it appears in `words`.
+ */
+export function fuzzyMatchInWords(token: string, words: string[], maxDistance: number): string | null {
+ const matched = matchingWordIndex(token, words, maxDistance);
+ return matched === -1 ? null : words[matched];
+}
+
+/** Index of the first word within `maxDistance` edits of `token`, or -1. */
+function matchingWordIndex(token: string, words: string[], maxDistance: number): number {
+ for (const [index, word] of words.entries()) {
+ // A word containing the token is a substring relationship, not a typo, so "sync"
+ // must not fuzzy-match "async".
+ if (word.length > token.length && word.includes(token)) {
+ continue;
+ }
+
+ // Skip if word is too different in length for fuzzy matching
+ if (Math.abs(word.length - token.length) > maxDistance) {
+ continue;
+ }
+
+ if (calculateOptimizedEditDistance(token, word, maxDistance) <= maxDistance) {
+ return index;
+ }
+ }
+
+ return -1;
+}
+
/**
* Whether `text` can hold a word within `maxDistance` edits of `token`.
*
@@ -372,7 +397,7 @@ export function fuzzyMatchWordWithResult(token: string, text: string, maxDistanc
* and so does any text containing that word. Each chunk costs one native substring scan, and the
* check never rejects a text the per-word scan would have matched, so it can front that scan.
*/
-function containsAnyChunk(text: string, token: string, maxDistance: number): boolean {
+export function containsAnyChunk(text: string, token: string, maxDistance: number): boolean {
const chunkCount = maxDistance + 1;
const chunkLength = Math.floor(token.length / chunkCount);