Build your personal knowledge base with Trilium Notes
TypeScript electronelectron-appknowledge-baseknowledge-graphknowledge-managementknowledge-management-graphlocal-firstnote-managmentnote-takernote-takingnotebooknotesnotes-apppersonal-knowledge-basepersonal-wikiscriptableself-hostedself-hostingwiki
fix(search): rank titles spelled with the query's diacritics first (#11787)
Elian Doran69c6fdff5a25
diff --git a/packages/trilium-core/src/becca/entities/bnote.ts b/packages/trilium-core/src/becca/entities/bnote.tsindex 539f299b0c..6f74f4f327 100644--- a/packages/trilium-core/src/becca/entities/bnote.ts+++ b/packages/trilium-core/src/becca/entities/bnote.ts@@ -50,9 +50,11 @@ interface ConvertOpts {}/** A note's title in the two forms scoring needs, cached together since both derive from it. */-interface SearchableTitle {+export interface SearchableTitle {normalized: string;words: string[];+ /** The title lowercased with its diacritics kept, or `null` when it has none. */+ accented: string | null;}/**@@ -827,7 +829,12 @@ class BNote extends AbstractBeccaEntity<BNote> {getSearchableTitle(): SearchableTitle {if (!this.__searchableTitleCache) {const normalized = normalizeSearchText(this.title);- this.__searchableTitleCache = { normalized, words: tokenizeNormalizedText(normalized) };+ const lowercased = this.title.toLowerCase();+ this.__searchableTitleCache = {+ normalized,+ words: tokenizeNormalizedText(normalized),+ accented: lowercased === normalized ? null : lowercased+ };}return this.__searchableTitleCache;diff --git a/packages/trilium-core/src/services/search/search_result.ts b/packages/trilium-core/src/services/search/search_result.tsindex 3ed5afa190..e637ed088c 100644--- a/packages/trilium-core/src/services/search/search_result.ts+++ b/packages/trilium-core/src/services/search/search_result.ts@@ -1,5 +1,6 @@import becca from "../../becca/becca.js";import becca_service, { type SegmentTitleCache } from "../../becca/becca_service.js";+import type { SearchableTitle } from "../../becca/entities/bnote.js";import type { ContentMatchQuality } from "./match_quality.js";import {calculateOptimizedEditDistance,@@ -17,6 +18,10 @@ const SCORE_WEIGHTS = {TITLE_EXACT_MATCH: 2000,TITLE_PREFIX_MATCH: 500,TITLE_WORD_MATCH: 300,+ // Added to a title match that also holds with the query's diacritics, so "ktorý" outranks+ // "ktory" for the query "ktorý" and the other way round. Well below the gaps between the title+ // tiers, so it only orders titles that differ in their diacritics.+ TITLE_DIACRITIC_EXACT_BONUS: 50,TOKEN_EXACT_MATCH: 4,TOKEN_PREFIX_MATCH: 2,TOKEN_CONTAINS_MATCH: 1,@@ -51,6 +56,10 @@ export interface ScoringTerms {normalizedQuery: string;queryWords: string[];normalizedTokens: string[];+ /** The query with its diacritics kept, or `null` when it has none. */+ accentedQuery: string | null;+ /** The words of {@link accentedQuery}, empty when it is `null`. */+ accentedQueryWords: string[];/*** Words per path segment title. Results under the same ancestors repeat those titles, so one* search tokenizes each distinct segment once instead of once per result.@@ -60,10 +69,15 @@ export interface ScoringTerms {export function precomputeScoringTerms(fulltextQuery: string, tokens: string[]): ScoringTerms {const normalizedQuery = normalizeSearchText(fulltextQuery);+ // `lex()` already lowercased the query, so it differs from its normalized form only in its+ // diacritics.+ const accentedQuery = fulltextQuery === normalizedQuery ? null : fulltextQuery;return {normalizedQuery,queryWords: tokenizeNormalizedText(normalizedQuery),+ accentedQuery,+ accentedQueryWords: accentedQuery ? tokenizeNormalizedText(accentedQuery) : [],normalizedTokens: tokens.map((token) => stripWordPunctuation(normalizeSearchText(token))),pathSegmentWords: new Map()};@@ -134,9 +148,11 @@ class SearchResult {this.fuzzyScore = 0; // Reset fuzzy score trackingconst note = becca.notes[this.noteId];- const { normalizedQuery, queryWords, normalizedTokens, pathSegmentWords } = terms ?? precomputeScoringTerms(fulltextQuery, tokens);+ const scoringTerms = terms ?? precomputeScoringTerms(fulltextQuery, tokens);+ const { normalizedQuery, queryWords, normalizedTokens, pathSegmentWords } = scoringTerms;// normalizeSearchText already lowercases — no need for .toLowerCase() first- const { normalized: normalizedTitle, words: titleWords } = note.getSearchableTitle();+ const searchableTitle = note.getSearchableTitle();+ const { normalized: normalizedTitle, words: titleWords } = searchableTitle;// Note ID exact match, much higher scoreif (note.noteId.toLowerCase() === fulltextQuery) {@@ -144,12 +160,16 @@ class SearchResult {}// Title matching scores with fuzzy matching support+ let titleTier: TitleTier | undefined;if (normalizedTitle === normalizedQuery) {this.score += SCORE_WEIGHTS.TITLE_EXACT_MATCH;+ titleTier = "exact";} else if (normalizedTitle.startsWith(normalizedQuery)) {this.score += SCORE_WEIGHTS.TITLE_PREFIX_MATCH;+ titleTier = "prefix";} else if (wordsContainPhrase(titleWords, queryWords)) {this.score += SCORE_WEIGHTS.TITLE_WORD_MATCH;+ titleTier = "words";} else if (enableFuzzyMatching) {// Try fuzzy matching for typos only if enabledconst fuzzyScore = this.calculateFuzzyTitleScore(normalizedTitle, normalizedQuery);@@ -157,6 +177,10 @@ class SearchResult {this.fuzzyScore += fuzzyScore; // Track fuzzy score contributions}+ if (titleTier && normalizedQuery && matchesWithDiacritics(titleTier, searchableTitle, scoringTerms)) {+ this.score += SCORE_WEIGHTS.TITLE_DIACRITIC_EXACT_BONUS;+ }+// Add scores for token matchesthis.addScoreForChunks(titleWords, tokens, SCORE_WEIGHTS.TITLE_FACTOR, enableFuzzyMatching, normalizedTokens);if (includePathScore) {@@ -323,3 +347,30 @@ class SearchResult {}export default SearchResult;++type TitleTier = "exact" | "prefix" | "words";++/**+ * Whether a title that reached `tier` after normalization still reaches it with the diacritics of+ * both sides kept. Without any on either side the two comparisons are the same one, so only an+ * accented title or query pays for the second.+ */+function matchesWithDiacritics(tier: TitleTier, title: SearchableTitle, terms: ScoringTerms) {+ if (title.accented === null && terms.accentedQuery === null) {+ return true;+ }++ const titleText = title.accented ?? title.normalized;+ const queryText = terms.accentedQuery ?? terms.normalizedQuery;+ switch (tier) {+ case "exact":+ return titleText === queryText;+ case "prefix":+ return titleText.startsWith(queryText);+ case "words":+ return wordsContainPhrase(+ title.accented === null ? title.words : tokenizeNormalizedText(title.accented),+ terms.accentedQuery === null ? terms.queryWords : terms.accentedQueryWords+ );+ }+}diff --git a/packages/trilium-core/src/services/search/services/search.spec.ts b/packages/trilium-core/src/services/search/services/search.spec.tsindex 72f1efe688..337c1200b8 100644--- a/packages/trilium-core/src/services/search/services/search.spec.ts+++ b/packages/trilium-core/src/services/search/services/search.spec.ts@@ -935,6 +935,25 @@ describe("Search", () => {expect(titledRank).toBeLessThan(bodyRank);});+ it("ranks the title spelled with the query's diacritics first (#11787)", () => {+ // Without the accent sorting first, the alphabetical tie-break settles this in favour+ // of "ktory" both ways, as "y" sorts before "ý".+ const accented = contentNote("ktorý", "");+ const plain = contentNote("ktory", "");+ const accentedPhrase = contentNote("ktorý je", "");+ const plainPhrase = contentNote("ktory je", "");++ const accentedSearch = searchService.findResultsWithQuery("ktorý", new SearchContext());+ expect(rank(accentedSearch, accented.noteId)).toBeGreaterThanOrEqual(0);+ expect(rank(accentedSearch, accented.noteId)).toBeLessThan(rank(accentedSearch, plain.noteId));+ expect(rank(accentedSearch, accentedPhrase.noteId)).toBeLessThan(rank(accentedSearch, plainPhrase.noteId));++ const plainSearch = searchService.findResultsWithQuery("ktory", new SearchContext());+ expect(rank(plainSearch, plain.noteId)).toBeGreaterThanOrEqual(0);+ expect(rank(plainSearch, plain.noteId)).toBeLessThan(rank(plainSearch, accented.noteId));+ expect(rank(plainSearch, plainPhrase.noteId)).toBeLessThan(rank(plainSearch, accentedPhrase.noteId));+ });+it("finds a body typo via phase-2 fuzzy fallback, ranked below exact matches (combinef -> combined)", () => {const exact = contentNote("ExactNote", "the combinef marker is set");const fuzzy = contentNote("FuzzyNote", "the values were combined together");