🎧 Your Personal Streaming Service
Go airsonicmadsonicmedia-servermusicmusic-servernavidromeopensubsonicraspberry-piself-hostedstreamerstreaming-apistreaming-audiosubsonicsubsonic-server
fix(search): artists with atomic non-ASCII names unfindable after FTS5 migration (#5703)
Deluan Quintão427d4b9bce19
diff --git a/db/migrations/20260702152457_backfill_artist_search_normalized.go b/db/migrations/20260702152457_backfill_artist_search_normalized.gonew file mode 100644index 000000000..93902e230--- /dev/null+++ b/db/migrations/20260702152457_backfill_artist_search_normalized.go@@ -0,0 +1,58 @@+package migrations++import (+ "context"+ "database/sql"+ "fmt"++ "github.com/navidrome/navidrome/utils/str"+ "github.com/pressly/goose/v3"+)++func init() {+ goose.AddMigrationContext(upBackfillArtistSearchNormalized, downBackfillArtistSearchNormalized)+}++// The FTS5 migration back-filled artist.search_normalized with a SQL approximation that+// cannot transliterate atomic letters (Ø, æ, ß, ...), and the scanner never rewrote the+// column, leaving artists like "GØGGS" unfindable by ASCII searches. Recompute it in Go;+// the artist_fts update trigger re-indexes every row that changes.+func upBackfillArtistSearchNormalized(ctx context.Context, tx *sql.Tx) error {+ notice(ctx, tx, "Rebuilding artist search index data. This may take a moment on large libraries.")++ rows, err := tx.QueryContext(ctx, "SELECT id, name, search_normalized FROM artist")+ if err != nil {+ return fmt.Errorf("querying artists: %w", err)+ }+ defer rows.Close()++ updates := map[string]string{}+ for rows.Next() {+ var id, name, current string+ if err := rows.Scan(&id, &name, ¤t); err != nil {+ return fmt.Errorf("scanning artist: %w", err)+ }+ if expected := str.NormalizeForFTS(name); expected != current {+ updates[id] = expected+ }+ }+ if err := rows.Err(); err != nil {+ return fmt.Errorf("iterating artists: %w", err)+ }++ stmt, err := tx.PrepareContext(ctx, "UPDATE artist SET search_normalized = ? WHERE id = ?")+ if err != nil {+ return fmt.Errorf("preparing update: %w", err)+ }+ defer stmt.Close()+ for id, normalized := range updates {+ if _, err := stmt.ExecContext(ctx, normalized, id); err != nil {+ return fmt.Errorf("updating artist %s: %w", id, err)+ }+ }+ return nil+}++func downBackfillArtistSearchNormalized(context.Context, *sql.Tx) error {+ return nil+}diff --git a/persistence/album_repository.go b/persistence/album_repository.goindex 31a0f5c91..34845be15 100644--- a/persistence/album_repository.go+++ b/persistence/album_repository.go@@ -17,6 +17,7 @@ import ("github.com/navidrome/navidrome/log""github.com/navidrome/navidrome/model""github.com/navidrome/navidrome/utils/slice"+ "github.com/navidrome/navidrome/utils/str""github.com/pocketbase/dbx")@@ -69,7 +70,7 @@ func (a *dbAlbum) PostMapArgs(args map[string]any) error {fullText = append(fullText, a.Album.Tags[model.TagCatalogNumber]...)args["full_text"] = formatFullText(fullText...)args["search_participants"] = strings.Join(participantNames, " ")- args["search_normalized"] = normalizeForFTS(a.Name, a.AlbumArtist)+ args["search_normalized"] = str.NormalizeForFTS(a.Name, a.AlbumArtist)args["tags"] = marshalTags(a.Album.Tags)args["participants"] = marshalParticipants(a.Album.Participants)diff --git a/persistence/artist_repository.go b/persistence/artist_repository.goindex 5152e774f..f84f410e9 100644--- a/persistence/artist_repository.go+++ b/persistence/artist_repository.go@@ -19,6 +19,7 @@ import ("github.com/navidrome/navidrome/model""github.com/navidrome/navidrome/utils""github.com/navidrome/navidrome/utils/slice"+ "github.com/navidrome/navidrome/utils/str""github.com/pocketbase/dbx")@@ -102,8 +103,10 @@ func (a *dbArtist) PostMapArgs(m map[string]any) error {}similarArtists, _ := json.Marshal(sa)m["similar_artists"] = string(similarArtists)+ // When adding a derived column here, also add it to the scanner's artist Put column list+ // in phase_1_folders.go, or rescans will never update it (how search_normalized went stale).m["full_text"] = formatFullText(a.Name, a.SortArtistName)- m["search_normalized"] = normalizeForFTS(a.Name)+ m["search_normalized"] = str.NormalizeForFTS(a.Name)// Do not override the sort_artist_name and mbz_artist_id fields if they are empty// TODO: Better way to handle this?diff --git a/persistence/mediafile_repository.go b/persistence/mediafile_repository.goindex d7d892ed1..094268783 100644--- a/persistence/mediafile_repository.go+++ b/persistence/mediafile_repository.go@@ -16,6 +16,7 @@ import ("github.com/navidrome/navidrome/log""github.com/navidrome/navidrome/model""github.com/navidrome/navidrome/utils/slice"+ "github.com/navidrome/navidrome/utils/str""github.com/pocketbase/dbx")@@ -62,7 +63,7 @@ func (m *dbMediaFile) PostMapArgs(args map[string]any) error {fullText = append(fullText, participantNames...)args["full_text"] = formatFullText(fullText...)args["search_participants"] = strings.Join(participantNames, " ")- args["search_normalized"] = normalizeForFTS(m.FullTitle(), m.Album, m.Artist, m.AlbumArtist)+ args["search_normalized"] = str.NormalizeForFTS(m.FullTitle(), m.Album, m.Artist, m.AlbumArtist)args["tags"] = marshalTags(m.MediaFile.Tags)args["participants"] = marshalParticipants(m.MediaFile.Participants)return nildiff --git a/persistence/sql_search_fts.go b/persistence/sql_search_fts.goindex bbae47fe8..999c904af 100644--- a/persistence/sql_search_fts.go+++ b/persistence/sql_search_fts.go@@ -11,6 +11,7 @@ import ("github.com/deluan/sanitize""github.com/navidrome/navidrome/log""github.com/navidrome/navidrome/model"+ "github.com/navidrome/navidrome/utils/str")// containsCJK returns true if the string contains any CJK (Chinese/Japanese/Korean) characters.@@ -35,48 +36,12 @@ func containsCJK(s string) bool {// as unbalanced string delimiters.var fts5SpecialChars = regexp.MustCompile(`[^\p{L}\p{N}\s*"\x00]`)-// fts5PunctStrip strips everything except letters and numbers (no whitespace, wildcards, or quotes).-// Used for normalizing words at index time to create concatenated forms (e.g., "R.E.M." → "REM").-var fts5PunctStrip = regexp.MustCompile(`[^\p{L}\p{N}]`)-// fts5Operators matches FTS5 boolean operators as whole words (case-insensitive).var fts5Operators = regexp.MustCompile(`(?i)\b(AND|OR|NOT|NEAR)\b`)// fts5LeadingStar matches a * at the start of a token. FTS5 only supports * at the end (prefix queries).var fts5LeadingStar = regexp.MustCompile(`(^|[\s])\*+`)-// normalizeForFTS takes multiple strings and returns a space-separated, deduplicated list of-// alternative searchable forms for each word: punctuation-stripped (R.E.M. → REM, AC/DC → ACDC)-// and ASCII-transliterated (Bjørk → Bjork, Å“uvre → oeuvre). The transliterated form is needed-// because FTS5's `unicode61 remove_diacritics 2` only handles NFKD-decomposable diacritics —-// atomic letters like ø/æ/Å“/ß survive tokenization, so the query side and index side disagree-// without an explicit transliterated entry here.-func normalizeForFTS(values ...string) string {- seen := make(map[string]struct{})- var result []string- add := func(orig, variant string) {- if variant == "" || variant == orig {- return- }- lower := strings.ToLower(variant)- if _, ok := seen[lower]; ok {- return- }- seen[lower] = struct{}{}- result = append(result, variant)- }- for _, v := range values {- for word := range strings.FieldsSeq(v) {- transliterated := sanitize.Accents(word)- // Concatenated ASCII form: R.E.M. → REM, AC/DC → ACDC, St-Étienne → StEtienne.- add(word, fts5PunctStrip.ReplaceAllString(transliterated, ""))- // Accent-only transliteration for words without name-punctuation (Bjørk → Bjork).- add(word, transliterated)- }- }- return strings.Join(result, " ")-}-// isSingleUnicodeLetter returns true if token is exactly one Unicode letter.func isSingleUnicodeLetter(token string) bool {r, size := utf8.DecodeRuneInString(token)@@ -100,7 +65,7 @@ func processPunctuatedWords(input string, phrases []string) (string, []string) {result = append(result, w)continue}- concat := fts5PunctStrip.ReplaceAllString(w, "")+ concat := str.FTSPunctStrip.ReplaceAllString(w, "")if concat == "" || concat == w {result = append(result, w)continue@@ -329,7 +294,7 @@ func ftsQueryDegraded(original, ftsQuery string) bool {// Strip quotes from original for comparison — we want the raw contentstripped := strings.ReplaceAll(original, `"`, "")// Extract the alphanumeric content from the original query- alphaNum := fts5PunctStrip.ReplaceAllString(stripped, "")+ alphaNum := str.FTSPunctStrip.ReplaceAllString(stripped, "")// If the original is entirely alphanumeric, nothing was stripped — not degradedif len(alphaNum) == len(stripped) {return false@@ -353,7 +318,7 @@ func ftsQueryDegraded(original, ftsQuery string) bool {if strings.HasPrefix(t, `"`) {// Extract content between quotesinner := strings.Trim(t, `"`)- innerAlpha := fts5PunctStrip.ReplaceAllString(inner, " ")+ innerAlpha := str.FTSPunctStrip.ReplaceAllString(inner, " ")for it := range strings.FieldsSeq(innerAlpha) {if len(it) > 2 {return falsediff --git a/persistence/sql_search_fts_test.go b/persistence/sql_search_fts_test.goindex b54e5856a..6c975c601 100644--- a/persistence/sql_search_fts_test.go+++ b/persistence/sql_search_fts_test.go@@ -74,28 +74,6 @@ var _ = DescribeTable("ftsQueryDegraded",Entry("not degraded for OR groups from processPunctuatedWords", "AC/DC", `("AC DC" OR ACDC*)`, false),)-var _ = DescribeTable("normalizeForFTS",- func(expected string, values ...string) {- Expect(normalizeForFTS(values...)).To(Equal(expected))- },- Entry("strips dots and concatenates", "REM", "R.E.M."),- Entry("strips slash", "ACDC", "AC/DC"),- Entry("strips hyphen", "Aha", "A-ha"),- Entry("skips unchanged ASCII words", "", "The Beatles"),- Entry("handles mixed input", "REM", "R.E.M.", "Automatic for the People"),- Entry("deduplicates", "REM", "R.E.M.", "R.E.M."),- Entry("strips apostrophe from word", "N", "Guns N' Roses"),- Entry("handles multiple values with punctuation", "REM ACDC", "R.E.M.", "AC/DC"),- Entry("transliterates ø to o", "Bjork", "Bjørk"),- Entry("transliterates Ø to O", "Oystein", "Øystein"),- Entry("transliterates Å“ ligature to oe", "oeuvre", "Å“uvre"),- Entry("transliterates Latin diacritics", "cafe", "café"),- Entry("transliterates only the non-ASCII words", "Mo Ros", "Mø Rós"),- Entry("combines punctuation strip and transliteration", "StEtienne St-Etienne", "St-Étienne"),- Entry("deduplicates against punctuation form", "Cafe", "Café", "Cafe"),- Entry("transliterates ß to ss", "Strasse", "Straße"),-)-var _ = DescribeTable("containsCJK",func(input string, expected bool) {Expect(containsCJK(input)).To(Equal(expected))diff --git a/scanner/phase_1_folders.go b/scanner/phase_1_folders.goindex 38967832c..5e898590b 100644--- a/scanner/phase_1_folders.go+++ b/scanner/phase_1_folders.go@@ -360,7 +360,7 @@ func (p *phaseFolders) persistChanges(entry *folderEntry) (*folderEntry, error)// Save all new/modified artists to DB. Their information will be incomplete, but they will be refreshed laterfor i := range entry.artists {err = artistRepo.Put(&entry.artists[i], "name",- "mbz_artist_id", "sort_artist_name", "order_artist_name", "full_text", "updated_at")+ "mbz_artist_id", "sort_artist_name", "order_artist_name", "full_text", "search_normalized", "updated_at")if err != nil {log.Error(p.ctx, "Scanner: Error persisting artist to DB", "folder", entry.path, "artist", entry.artists[i].Name, err)return errdiff --git a/scanner/scanner_test.go b/scanner/scanner_test.goindex cc3732bc3..7f3dca775 100644--- a/scanner/scanner_test.go+++ b/scanner/scanner_test.go@@ -189,6 +189,34 @@ var _ = Describe("Scanner", Ordered, func() {})})+ Context("Artist with atomic non-ASCII letters, 'GØGGS'", func() {+ BeforeEach(func() {+ goggs := template(_t{"albumartist": "GØGGS", "album": "Pre Strike Sweep", "year": 2018})+ createFS(fstest.MapFS{+ "GØGGS/Pre Strike Sweep/01 - Falling For You.mp3": goggs(track(1, "Falling For You")),+ })+ })++ searchNormalized := func() string {+ var sn string+ Expect(db.Db().QueryRowContext(ctx,+ "SELECT search_normalized FROM artist WHERE name = 'GØGGS'").Scan(&sn)).To(Succeed())+ return sn+ }++ It("repopulates a stale search_normalized on a full rescan", func() {+ Expect(runScanner(ctx, true)).To(Succeed())+ Expect(searchNormalized()).To(Equal("GOGGS"))++ // Simulate the stale value left by the FTS5 migration's SQL back-fill+ _, err := db.Db().ExecContext(ctx, "UPDATE artist SET search_normalized = '' WHERE name = 'GØGGS'")+ Expect(err).ToNot(HaveOccurred())++ Expect(runScanner(ctx, true)).To(Succeed())+ Expect(searchNormalized()).To(Equal("GOGGS"))+ })+ })+Context("Ignored entries", func() {BeforeEach(func() {revolver := template(_t{"albumartist": "The Beatles", "album": "Revolver", "year": 1966})diff --git a/utils/str/normalize_fts.go b/utils/str/normalize_fts.gonew file mode 100644index 000000000..994f77cc9--- /dev/null+++ b/utils/str/normalize_fts.go@@ -0,0 +1,45 @@+package str++import (+ "regexp"+ "strings"++ "github.com/deluan/sanitize"+)++// FTSPunctStrip matches any character that is not a letter or number. Index-time+// normalization (NormalizeForFTS) and query-time processing in persistence share it+// so both sides produce matching tokens.+var FTSPunctStrip = regexp.MustCompile(`[^\p{L}\p{N}]`)++// NormalizeForFTS takes multiple strings and returns a space-separated, deduplicated list of+// alternative searchable forms for each word: punctuation-stripped (R.E.M. → REM, AC/DC → ACDC)+// and ASCII-transliterated (Bjørk → Bjork, Å“uvre → oeuvre). The transliterated form is needed+// because FTS5's `unicode61 remove_diacritics 2` only handles NFKD-decomposable diacritics —+// atomic letters like ø/æ/Å“/ß survive tokenization, so the query side and index side disagree+// without an explicit transliterated entry here.+func NormalizeForFTS(values ...string) string {+ seen := make(map[string]struct{})+ var result []string+ add := func(orig, variant string) {+ if variant == "" || variant == orig {+ return+ }+ lower := strings.ToLower(variant)+ if _, ok := seen[lower]; ok {+ return+ }+ seen[lower] = struct{}{}+ result = append(result, variant)+ }+ for _, v := range values {+ for word := range strings.FieldsSeq(v) {+ transliterated := sanitize.Accents(word)+ // Concatenated ASCII form: R.E.M. → REM, AC/DC → ACDC, St-Étienne → StEtienne.+ add(word, FTSPunctStrip.ReplaceAllString(transliterated, ""))+ // Accent-only transliteration for words without name-punctuation (Bjørk → Bjork).+ add(word, transliterated)+ }+ }+ return strings.Join(result, " ")+}diff --git a/utils/str/normalize_fts_test.go b/utils/str/normalize_fts_test.gonew file mode 100644index 000000000..52387cb1d--- /dev/null+++ b/utils/str/normalize_fts_test.go@@ -0,0 +1,29 @@+package str_test++import (+ "github.com/navidrome/navidrome/utils/str"+ . "github.com/onsi/ginkgo/v2"+ . "github.com/onsi/gomega"+)++var _ = DescribeTable("NormalizeForFTS",+ func(expected string, values ...string) {+ Expect(str.NormalizeForFTS(values...)).To(Equal(expected))+ },+ Entry("strips dots and concatenates", "REM", "R.E.M."),+ Entry("strips slash", "ACDC", "AC/DC"),+ Entry("strips hyphen", "Aha", "A-ha"),+ Entry("skips unchanged ASCII words", "", "The Beatles"),+ Entry("handles mixed input", "REM", "R.E.M.", "Automatic for the People"),+ Entry("deduplicates", "REM", "R.E.M.", "R.E.M."),+ Entry("strips apostrophe from word", "N", "Guns N' Roses"),+ Entry("handles multiple values with punctuation", "REM ACDC", "R.E.M.", "AC/DC"),+ Entry("transliterates ø to o", "Bjork", "Bjørk"),+ Entry("transliterates Ø to O", "Oystein", "Øystein"),+ Entry("transliterates Å“ ligature to oe", "oeuvre", "Å“uvre"),+ Entry("transliterates Latin diacritics", "cafe", "café"),+ Entry("transliterates only the non-ASCII words", "Mo Ros", "Mø Rós"),+ Entry("combines punctuation strip and transliteration", "StEtienne St-Etienne", "St-Étienne"),+ Entry("deduplicates against punctuation form", "Cafe", "Café", "Cafe"),+ Entry("transliterates ß to ss", "Strasse", "Straße"),+)