Build your personal knowledge base with Trilium Notes
TypeScript electronelectron-appknowledge-baseknowledge-graphknowledge-managementknowledge-management-graphlocal-firstnote-managmentnote-takernote-takingnotebooknotesnotes-apppersonal-knowledge-basepersonal-wikiscriptableself-hostedself-hostingwiki
perf(search): split a note's flat text once per search, not once per token
Elian Doran5133d3210703
diff --git a/packages/trilium-core/src/services/search/expressions/note_flat_text.ts b/packages/trilium-core/src/services/search/expressions/note_flat_text.tsindex 2667d7f914..526aeb4c5b 100644--- a/packages/trilium-core/src/services/search/expressions/note_flat_text.ts+++ b/packages/trilium-core/src/services/search/expressions/note_flat_text.ts@@ -3,7 +3,14 @@ import becca from "../../../becca/becca.js";import type BNote from "../../../becca/entities/bnote.js";import NoteSet from "../note_set.js";import type SearchContext from "../search_context.js";-import { fuzzyMatchWordWithResult, getAutoMaxEditDistance, normalizeSearchText } from "../utils/text_utils.js";+import {+ containsAnyChunk,+ fuzzyMatchInWords,+ fuzzyMatchWordWithResult,+ getAutoMaxEditDistance,+ normalizeSearchText,+ splitIntoWords+} from "../utils/text_utils.js";import Expression from "./expression.js";class NoteFlatTextExp extends Expression {@@ -213,8 +220,32 @@ class NoteFlatTextExp extends Expression {}const flatText = flatTexts[i];+ // Split on the first token that gets as far as the word scan, then reuse it for the+ // rest. Splitting per token instead repeats work already done for this note: across+ // a 12-token query over a 22k-note database, over half the splits are repeats.+ let words: string[] | undefined;+for (const token of this.tokens) {- if (this.smartMatch(flatText, token, searchContext)) {+ if (flatText.includes(token)) {+ candidateNotes.push(note);+ break;+ }++ if (!searchContext?.enableFuzzyMatching) {+ continue;+ }++ const maxDistance = getAutoMaxEditDistance(token.length);++ if (maxDistance <= 0 || !containsAnyChunk(flatText, token, maxDistance)) {+ continue;+ }++ words ??= splitIntoWords(flatText);+ const matchedWord = fuzzyMatchInWords(token, words, maxDistance);++ if (matchedWord) {+ rememberFuzzyMatch(searchContext, matchedWord);candidateNotes.push(note);break;}@@ -242,10 +273,7 @@ class NoteFlatTextExp extends Expression {if (searchContext?.enableFuzzyMatching && getAutoMaxEditDistance(token.length) > 0) {const matchedWord = fuzzyMatchWordWithResult(token, text);if (matchedWord) {- // Track the fuzzy matched word for highlighting- if (!searchContext.highlightedTokens.includes(matchedWord)) {- searchContext.highlightedTokens.push(matchedWord);- }+ rememberFuzzyMatch(searchContext, matchedWord);return true;}}@@ -254,6 +282,13 @@ class NoteFlatTextExp extends Expression {}}+/** Keeps a fuzzily matched word for highlighting, so a result can show why it matched. */+function rememberFuzzyMatch(searchContext: SearchContext | undefined, word: string) {+ if (searchContext && !searchContext.highlightedTokens.includes(word)) {+ searchContext.highlightedTokens.push(word);+ }+}+/*** The normalized title a note carries under one parent. A branch without a prefix shows the note's* own title, whose normalized form the note caches, so only a prefixed branch builds a new string.diff --git a/packages/trilium-core/src/services/search/utils/text_utils.ts b/packages/trilium-core/src/services/search/utils/text_utils.tsindex d8574f3ed1..a2435577cd 100644--- a/packages/trilium-core/src/services/search/utils/text_utils.ts+++ b/packages/trilium-core/src/services/search/utils/text_utils.ts@@ -331,32 +331,16 @@ export function fuzzyMatchWordWithResult(token: string, text: string, maxDistanc}// For fuzzy matching, split into words and check each against the token- const words = normalizedText.split(/\s+/).filter(word => word.length > 0);- const originalWords = text.split(/\s+/).filter(word => word.length > 0);+ const words = splitIntoWords(normalizedText);+ const matched = matchingWordIndex(normalizedToken, words, maxDistance);- for (let i = 0; i < words.length; i++) {- const word = words[i];- const originalWord = originalWords[i];-- // A word containing the token is a substring relationship, not a typo, so "sync"- // must not fuzzy-match "async".- if (word.length > normalizedToken.length && word.includes(normalizedToken)) {- continue;- }-- // Skip if word is too different in length for fuzzy matching- if (Math.abs(word.length - normalizedToken.length) > maxDistance) {- continue;- }-- // Use optimized edit distance calculation- const distance = calculateOptimizedEditDistance(normalizedToken, word, maxDistance);- if (distance <= maxDistance) {- return originalWord; // Return the original word with case preserved- }+ if (matched === -1) {+ return null;}- return null;+ // Return the original word with case preserved. Lowercasing neither adds nor removes+ // whitespace, so the two splits agree position for position.+ return splitIntoWords(text)[matched];} catch (error) {// Log error and return null for safetyconsole.warn('Error in fuzzy word matching:', error);@@ -364,6 +348,47 @@ export function fuzzyMatchWordWithResult(token: string, text: string, maxDistanc}}+/**+ * Splits a normalized text into its words.+ *+ * Callers that test many tokens against the same text split it once and pass the result to+ * {@link fuzzyMatchInWords}, rather than paying for the split per token.+ */+export function splitIntoWords(text: string): string[] {+ return text.split(/\s+/).filter(word => word.length > 0);+}++/**+ * The word within `maxDistance` edits of `token`, or null. Both the token and the words are+ * expected to be normalized; the word is returned as it appears in `words`.+ */+export function fuzzyMatchInWords(token: string, words: string[], maxDistance: number): string | null {+ const matched = matchingWordIndex(token, words, maxDistance);+ return matched === -1 ? null : words[matched];+}++/** Index of the first word within `maxDistance` edits of `token`, or -1. */+function matchingWordIndex(token: string, words: string[], maxDistance: number): number {+ for (const [index, word] of words.entries()) {+ // A word containing the token is a substring relationship, not a typo, so "sync"+ // must not fuzzy-match "async".+ if (word.length > token.length && word.includes(token)) {+ continue;+ }++ // Skip if word is too different in length for fuzzy matching+ if (Math.abs(word.length - token.length) > maxDistance) {+ continue;+ }++ if (calculateOptimizedEditDistance(token, word, maxDistance) <= maxDistance) {+ return index;+ }+ }++ return -1;+}+/*** Whether `text` can hold a word within `maxDistance` edits of `token`.*@@ -372,7 +397,7 @@ export function fuzzyMatchWordWithResult(token: string, text: string, maxDistanc* and so does any text containing that word. Each chunk costs one native substring scan, and the* check never rejects a text the per-word scan would have matched, so it can front that scan.*/-function containsAnyChunk(text: string, token: string, maxDistance: number): boolean {+export function containsAnyChunk(text: string, token: string, maxDistance: number): boolean {const chunkCount = maxDistance + 1;const chunkLength = Math.floor(token.length / chunkCount);