🎧 Your Personal Streaming Service
Go airsonicmadsonicmedia-servermusicmusic-servernavidromeopensubsonicraspberry-piself-hostedstreamerstreaming-apistreaming-audiosubsonicsubsonic-server
feat(search): rank exact matches above prefix matches (#5704)
Deluan Quintãoae9b8a5fe6e4
diff --git a/persistence/sql_search_fts.go b/persistence/sql_search_fts.goindex 999c904af..fce77afbb 100644--- a/persistence/sql_search_fts.go+++ b/persistence/sql_search_fts.go@@ -105,13 +105,15 @@ func isDottedAbbreviation(w string, subTokens []string) bool {}// buildFTS5Query preprocesses user input into a safe FTS5 MATCH expression.+// Plain tokens are emitted as (token OR token*) so bm25 ranks exact-token hits above prefix-only matches.// It preserves quoted phrases and * prefix wildcards, neutralizes FTS5 operators// (by lowercasing them, since FTS5 operators are case-sensitive) and strips// special characters to prevent query injection.-func buildFTS5Query(userInput string) string {+// The second return reports whether tokenization degraded the query (see ftsQueryDegraded).+func buildFTS5Query(userInput string) (string, bool) {q := strings.TrimSpace(userInput)if q == "" || q == `""` {- return ""+ return "", false}var phrases []string@@ -151,25 +153,38 @@ func buildFTS5Query(userInput string) string {result = fts5LeadingStar.ReplaceAllString(result, "$1")tokens := strings.Fields(result)- // Append * to plain tokens for prefix matching (e.g., "love" → "love*").- // Skip tokens that are already wildcarded or are quoted phrase placeholders.+ // Two forms per token: a plain prefix form (love*) used only to evaluate query+ // degradation, and the final (love OR love*) form. The OR adds no matches+ // (exact ⊂ prefix) but gives bm25 a high-IDF exact-term hit, ranking rows that+ // contain the literal word above prefix-only matches. Placeholders and+ // user-supplied wildcards pass through untouched in both forms.+ prefixTokens := make([]string, len(tokens))+ wrappedTokens := make([]string, len(tokens))for i, t := range tokens {if strings.HasPrefix(t, "\x00") || strings.HasSuffix(t, "*") {+ prefixTokens[i], wrappedTokens[i] = t, tcontinue}- tokens[i] = t + "*"+ prefixTokens[i] = t + "*"+ wrappedTokens[i] = "(" + t + " OR " + t + "*)"}// Use explicit AND between tokens — FTS5's implicit AND (space-separated)- // doesn't work correctly with parenthesized OR groups from processPunctuatedWords.- result = strings.Join(tokens, " AND ")+ // doesn't work correctly with parenthesized OR groups. The prefix form is+ // space-joined instead: it only feeds ftsQueryDegraded, which would count a+ // literal "AND" as a long token and never flag all-short-token queries.+ prefixQuery := strings.Join(prefixTokens, " ")+ result = strings.Join(wrappedTokens, " AND ")for i, phrase := range phrases {placeholder := fmt.Sprintf("\x00PHRASE%d\x00", i)+ prefixQuery = strings.ReplaceAll(prefixQuery, placeholder, phrase)result = strings.ReplaceAll(result, placeholder, phrase)}- return result+ // Degradation is evaluated on the prefix form: ftsQueryDegraded treats+ // leading-( tokens as punctuated-word groups and would never flag wrapped ones.+ return result, ftsQueryDegraded(userInput, prefixQuery)}// ftsColumn pairs an FTS5 column name with its BM25 relevance weight.@@ -209,7 +224,10 @@ var ftsColumnDefs = map[string][]ftsColumn{"artist": {{"name", 10.0},{"sort_artist_name", 1.0},- {"search_normalized", 1.0},+ // Same weight as name: for artists this column is purely the name in+ // alternate spelling (unlike media_file/album, where it mixes+ // title/album/artist variants and full weight would distort ranking).+ {"search_normalized", 10.0},},}@@ -338,8 +356,8 @@ func ftsQueryDegraded(original, ftsQuery string) bool {// tokenization stripped significant content from the query (e.g., "1+" → "1*").// Returns nil when the query produces no searchable tokens at all.func newFTSSearch(tableName, query string) searchStrategy {- q := buildFTS5Query(query)- if q == "" || ftsQueryDegraded(query, q) {+ q, degraded := buildFTS5Query(query)+ if q == "" || degraded {// Fallback: try LIKE search with the raw querycleaned := strings.TrimSpace(strings.ReplaceAll(query, `"`, ""))if cleaned != "" {diff --git a/persistence/sql_search_fts_test.go b/persistence/sql_search_fts_test.goindex 6c975c601..d0b26e8d5 100644--- a/persistence/sql_search_fts_test.go+++ b/persistence/sql_search_fts_test.go@@ -12,44 +12,45 @@ import (var _ = DescribeTable("buildFTS5Query",func(input, expected string) {- Expect(buildFTS5Query(input)).To(Equal(expected))+ q, _ := buildFTS5Query(input)+ Expect(q).To(Equal(expected))},Entry("returns empty string for empty input", "", ""),Entry("returns empty string for whitespace-only input", " ", ""),- Entry("appends * to a single word for prefix matching", "beatles", "beatles*"),- Entry("appends * to each word for prefix matching", "abbey road", "abbey* AND road*"),- Entry("preserves quoted phrases without appending *", `"the beatles"`, `"the beatles"`),- Entry("does not double-append * to existing prefix wildcard", "beat*", "beat*"),- Entry("strips FTS5 operators and appends * to lowercased words", "AND OR NOT NEAR", "and* AND or* AND not* AND near*"),- Entry("strips special FTS5 syntax characters and appends *", "test^col:val", "test* AND col* AND val*"),- Entry("handles mixed phrases and words", `"the beatles" abbey`, `"the beatles" AND abbey*`),- Entry("handles prefix with multiple words", "beat* abbey", "beat* AND abbey*"),- Entry("collapses multiple spaces", "abbey road", "abbey* AND road*"),- Entry("strips leading * from tokens and appends trailing *", "*livia", "livia*"),- Entry("strips leading * and preserves existing trailing *", "*livia oliv*", "livia* AND oliv*"),+ Entry("wraps a single word as exact OR prefix", "beatles", "(beatles OR beatles*)"),+ Entry("wraps each word as exact OR prefix", "abbey road", "(abbey OR abbey*) AND (road OR road*)"),+ Entry("preserves quoted phrases without wrapping", `"the beatles"`, `"the beatles"`),+ Entry("does not wrap user-supplied prefix wildcard", "beat*", "beat*"),+ Entry("strips FTS5 operators and wraps lowercased words", "AND OR NOT NEAR", "(and OR and*) AND (or OR or*) AND (not OR not*) AND (near OR near*)"),+ Entry("strips special FTS5 syntax characters and wraps", "test^col:val", "(test OR test*) AND (col OR col*) AND (val OR val*)"),+ Entry("handles mixed phrases and words", `"the beatles" abbey`, `"the beatles" AND (abbey OR abbey*)`),+ Entry("handles prefix with multiple words", "beat* abbey", "beat* AND (abbey OR abbey*)"),+ Entry("collapses multiple spaces", "abbey road", "(abbey OR abbey*) AND (road OR road*)"),+ Entry("strips leading * from tokens and wraps", "*livia", "(livia OR livia*)"),+ Entry("strips leading * and preserves existing trailing *", "*livia oliv*", "(livia OR livia*) AND oliv*"),Entry("strips standalone *", "*", ""),- Entry("strips apostrophe from input", "Guns N' Roses", "Guns* AND N* AND Roses*"),+ Entry("strips apostrophe from input", "Guns N' Roses", "(Guns OR Guns*) AND (N OR N*) AND (Roses OR Roses*)"),Entry("converts slashed word to phrase+concat OR", "AC/DC", `("AC DC" OR ACDC*)`),Entry("converts hyphenated word to phrase+concat OR", "a-ha", `("a ha" OR aha*)`),Entry("converts partial hyphenated word to phrase+concat OR", "a-h", `("a h" OR ah*)`),Entry("converts hyphenated name to phrase+concat OR", "Jay-Z", `("Jay Z" OR JayZ*)`),Entry("converts contraction to phrase+concat OR", "it's", `("it s" OR its*)`),- Entry("handles punctuated word mixed with plain words", "best of a-ha", `best* AND of* AND ("a ha" OR aha*)`),- Entry("handles contraction followed by plain words", "you've got", `("you ve" OR youve*) AND got*`),- Entry("strips miscellaneous punctuation", "rock & roll, vol. 2", "rock* AND roll* AND vol* AND 2*"),- Entry("transliterates NFKD-decomposable diacritics", "Björk début", "Bjork* AND debut*"),- Entry("transliterates ø to o", "Øystein", "Oystein*"),- Entry("transliterates œ ligature to oe", "œuvre", "oeuvre*"),- Entry("transliterates æ ligature to ae", "Brennæ", "Brennae*"),- Entry("transliterates mixed unicode words", "Mø Sigur Rós", "Mo* AND Sigur* AND Ros*"),- Entry("transliterates ß to ss", "Straße", "Strasse*"),+ Entry("handles punctuated word mixed with plain words", "best of a-ha", `(best OR best*) AND (of OR of*) AND ("a ha" OR aha*)`),+ Entry("handles contraction followed by plain words", "you've got", `("you ve" OR youve*) AND (got OR got*)`),+ Entry("strips miscellaneous punctuation", "rock & roll, vol. 2", "(rock OR rock*) AND (roll OR roll*) AND (vol OR vol*) AND (2 OR 2*)"),+ Entry("transliterates NFKD-decomposable diacritics", "Björk début", "(Bjork OR Bjork*) AND (debut OR debut*)"),+ Entry("transliterates ø to o", "Øystein", "(Oystein OR Oystein*)"),+ Entry("transliterates œ ligature to oe", "œuvre", "(oeuvre OR oeuvre*)"),+ Entry("transliterates æ ligature to ae", "Brennæ", "(Brennae OR Brennae*)"),+ Entry("transliterates mixed unicode words", "Mø Sigur Rós", "(Mo OR Mo*) AND (Sigur OR Sigur*) AND (Ros OR Ros*)"),+ Entry("transliterates ß to ss", "Straße", "(Strasse OR Strasse*)"),Entry("preserves quoted unicode phrase verbatim", `"Björk"`, `"Björk"`),Entry("collapses dotted abbreviation into phrase", "R.E.M.", `"R E M"`),Entry("collapses abbreviation without trailing dot", "R.E.M", `"R E M"`),- Entry("collapses abbreviation mixed with words", "best of R.E.M.", `best* AND of* AND "R E M"`),+ Entry("collapses abbreviation mixed with words", "best of R.E.M.", `(best OR best*) AND (of OR of*) AND "R E M"`),Entry("collapses two-letter abbreviation", "U.K.", `"U K"`),- Entry("does not collapse single letter surrounded by words", "I am fine", "I* AND am* AND fine*"),- Entry("does not collapse single standalone letter", "A test", "A* AND test*"),+ Entry("does not collapse single letter surrounded by words", "I am fine", "(I OR I*) AND (am OR am*) AND (fine OR fine*)"),+ Entry("does not collapse single standalone letter", "A test", "(A OR A*) AND (test OR test*)"),Entry("preserves quoted phrase with punctuation verbatim", `"ac/dc"`, `"ac/dc"`),Entry("preserves quoted abbreviation verbatim", `"R.E.M."`, `"R.E.M."`),Entry("returns empty string for punctuation-only input", "!!!!!!!", ""),@@ -57,6 +58,20 @@ var _ = DescribeTable("buildFTS5Query",Entry("returns empty string for empty quoted phrase", `""`, ""),)+var _ = DescribeTable("buildFTS5Query degraded flag",+ func(input string, expected bool) {+ _, degraded := buildFTS5Query(input)+ Expect(degraded).To(Equal(expected))+ },+ Entry("plain words are not degraded", "beatles", false),+ Entry("special chars stripped leaving short token is degraded", "1+", true),+ Entry("multiple short tokens are degraded", "1+ 2+", true),+ Entry("short tokens mixed with a long word are not degraded", "1+ beatles", false),+ Entry("quoted short-token phrase is degraded", `"1+"`, true),+ Entry("punctuated-name group is not degraded", "AC/DC", false),+ Entry("empty input is not degraded", "", false),+)+var _ = DescribeTable("ftsQueryDegraded",func(original, ftsQuery string, expected bool) {Expect(ftsQueryDegraded(original, ftsQuery)).To(Equal(expected))@@ -143,7 +158,7 @@ var _ = Describe("ftsColumnDefs helpers", func() {It("returns weight CSV for artist", func() {Expect(ftsBM25Weights).To(HaveKeyWithValue("artist",- "10.0, 1.0, 1.0",+ "10.0, 1.0, 10.0",))})@@ -237,18 +252,18 @@ var _ = Describe("newFTSSearch", func() {Expect(fts.rankExpr).To(Equal("unknown_table_fts.rank"))})- It("wraps query with column filter for known tables", func() {+ It("wraps query with column filter", func() {strategy := newFTSSearch("artist", "Beatles")fts, ok := strategy.(*ftsSearch)Expect(ok).To(BeTrue())- Expect(fts.matchExpr).To(Equal("{name sort_artist_name search_normalized} : (Beatles*)"))+ Expect(fts.matchExpr).To(Equal("{name sort_artist_name search_normalized} : ((Beatles OR Beatles*))"))})It("passes query without column filter for unknown tables", func() {strategy := newFTSSearch("unknown_table", "test")fts, ok := strategy.(*ftsSearch)Expect(ok).To(BeTrue())- Expect(fts.matchExpr).To(Equal("test*"))+ Expect(fts.matchExpr).To(Equal("(test OR test*)"))})It("preserves phrase queries inside column filter", func() {@@ -425,4 +440,38 @@ var _ = Describe("FTS5 Integration Search", func() {Expect(results).ToNot(BeEmpty(), "Max=0 should mean no limit, not LIMIT 0")})})++ Describe("Exact-match ranking", func() {+ BeforeEach(func() {+ // Registered before the inserts so a mid-loop failure cannot leak corpus rows.+ DeferCleanup(func() {+ // library_artist rows are removed by the artist_id ON DELETE CASCADE FK.+ _, err := GetDBXBuilder().NewQuery("DELETE FROM artist WHERE id LIKE 'fts-rank-%'").Execute()+ Expect(err).ToNot(HaveOccurred())+ })+ // Corpus has no competing exact-word names ("Mo X"): exact-vs-exact order depends+ // on corpus statistics; the guaranteed property is exact > prefix.+ for _, a := range []model.Artist{+ {ID: "fts-rank-1", Name: "MØ", OrderArtistName: "mø"},+ {ID: "fts-rank-2", Name: "Modest Mouse", OrderArtistName: "modest mouse"},+ {ID: "fts-rank-3", Name: "Morrissey", OrderArtistName: "morrissey"},+ } {+ Expect(createArtistWithLibrary(arr, &a, 1)).To(Succeed())+ }+ })++ It("ranks the exact transliterated match first for 'MO'", func() {+ results, err := arr.Search("MO", model.QueryOptions{Max: 10})+ Expect(err).ToNot(HaveOccurred())+ Expect(results).To(HaveLen(3))+ Expect(results[0].Name).To(Equal("MØ"), "exact match via search_normalized must outrank prefix matches")+ })++ It("ranks the exact match first for the accented query 'MØ'", func() {+ results, err := arr.Search("MØ", model.QueryOptions{Max: 10})+ Expect(err).ToNot(HaveOccurred())+ Expect(results).ToNot(BeEmpty())+ Expect(results[0].Name).To(Equal("MØ"))+ })+ })})