Skip to content

🎧 Your Personal Streaming Service

fix(search): artists with atomic non-ASCII names unfindable after FTS5 migration (#5703)

Deluan Quintão427d4b9bce19
diff --git a/db/migrations/20260702152457_backfill_artist_search_normalized.go b/db/migrations/20260702152457_backfill_artist_search_normalized.go
new file mode 100644
index 000000000..93902e230
--- /dev/null
+++ b/db/migrations/20260702152457_backfill_artist_search_normalized.go
@@ -0,0 +1,58 @@
+package migrations
+
+import (
+ "context"
+ "database/sql"
+ "fmt"
+
+ "github.com/navidrome/navidrome/utils/str"
+ "github.com/pressly/goose/v3"
+)
+
+func init() {
+ goose.AddMigrationContext(upBackfillArtistSearchNormalized, downBackfillArtistSearchNormalized)
+}
+
+// The FTS5 migration back-filled artist.search_normalized with a SQL approximation that
+// cannot transliterate atomic letters (Ø, æ, ß, ...), and the scanner never rewrote the
+// column, leaving artists like "GØGGS" unfindable by ASCII searches. Recompute it in Go;
+// the artist_fts update trigger re-indexes every row that changes.
+func upBackfillArtistSearchNormalized(ctx context.Context, tx *sql.Tx) error {
+ notice(ctx, tx, "Rebuilding artist search index data. This may take a moment on large libraries.")
+
+ rows, err := tx.QueryContext(ctx, "SELECT id, name, search_normalized FROM artist")
+ if err != nil {
+ return fmt.Errorf("querying artists: %w", err)
+ }
+ defer rows.Close()
+
+ updates := map[string]string{}
+ for rows.Next() {
+ var id, name, current string
+ if err := rows.Scan(&id, &name, &current); err != nil {
+ return fmt.Errorf("scanning artist: %w", err)
+ }
+ if expected := str.NormalizeForFTS(name); expected != current {
+ updates[id] = expected
+ }
+ }
+ if err := rows.Err(); err != nil {
+ return fmt.Errorf("iterating artists: %w", err)
+ }
+
+ stmt, err := tx.PrepareContext(ctx, "UPDATE artist SET search_normalized = ? WHERE id = ?")
+ if err != nil {
+ return fmt.Errorf("preparing update: %w", err)
+ }
+ defer stmt.Close()
+ for id, normalized := range updates {
+ if _, err := stmt.ExecContext(ctx, normalized, id); err != nil {
+ return fmt.Errorf("updating artist %s: %w", id, err)
+ }
+ }
+ return nil
+}
+
+func downBackfillArtistSearchNormalized(context.Context, *sql.Tx) error {
+ return nil
+}
diff --git a/persistence/album_repository.go b/persistence/album_repository.go
index 31a0f5c91..34845be15 100644
--- a/persistence/album_repository.go
+++ b/persistence/album_repository.go
@@ -17,6 +17,7 @@ import (
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/utils/slice"
+ "github.com/navidrome/navidrome/utils/str"
"github.com/pocketbase/dbx"
)
@@ -69,7 +70,7 @@ func (a *dbAlbum) PostMapArgs(args map[string]any) error {
fullText = append(fullText, a.Album.Tags[model.TagCatalogNumber]...)
args["full_text"] = formatFullText(fullText...)
args["search_participants"] = strings.Join(participantNames, " ")
- args["search_normalized"] = normalizeForFTS(a.Name, a.AlbumArtist)
+ args["search_normalized"] = str.NormalizeForFTS(a.Name, a.AlbumArtist)
args["tags"] = marshalTags(a.Album.Tags)
args["participants"] = marshalParticipants(a.Album.Participants)
diff --git a/persistence/artist_repository.go b/persistence/artist_repository.go
index 5152e774f..f84f410e9 100644
--- a/persistence/artist_repository.go
+++ b/persistence/artist_repository.go
@@ -19,6 +19,7 @@ import (
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/utils"
"github.com/navidrome/navidrome/utils/slice"
+ "github.com/navidrome/navidrome/utils/str"
"github.com/pocketbase/dbx"
)
@@ -102,8 +103,10 @@ func (a *dbArtist) PostMapArgs(m map[string]any) error {
}
similarArtists, _ := json.Marshal(sa)
m["similar_artists"] = string(similarArtists)
+ // When adding a derived column here, also add it to the scanner's artist Put column list
+ // in phase_1_folders.go, or rescans will never update it (how search_normalized went stale).
m["full_text"] = formatFullText(a.Name, a.SortArtistName)
- m["search_normalized"] = normalizeForFTS(a.Name)
+ m["search_normalized"] = str.NormalizeForFTS(a.Name)
// Do not override the sort_artist_name and mbz_artist_id fields if they are empty
// TODO: Better way to handle this?
diff --git a/persistence/mediafile_repository.go b/persistence/mediafile_repository.go
index d7d892ed1..094268783 100644
--- a/persistence/mediafile_repository.go
+++ b/persistence/mediafile_repository.go
@@ -16,6 +16,7 @@ import (
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/utils/slice"
+ "github.com/navidrome/navidrome/utils/str"
"github.com/pocketbase/dbx"
)
@@ -62,7 +63,7 @@ func (m *dbMediaFile) PostMapArgs(args map[string]any) error {
fullText = append(fullText, participantNames...)
args["full_text"] = formatFullText(fullText...)
args["search_participants"] = strings.Join(participantNames, " ")
- args["search_normalized"] = normalizeForFTS(m.FullTitle(), m.Album, m.Artist, m.AlbumArtist)
+ args["search_normalized"] = str.NormalizeForFTS(m.FullTitle(), m.Album, m.Artist, m.AlbumArtist)
args["tags"] = marshalTags(m.MediaFile.Tags)
args["participants"] = marshalParticipants(m.MediaFile.Participants)
return nil
diff --git a/persistence/sql_search_fts.go b/persistence/sql_search_fts.go
index bbae47fe8..999c904af 100644
--- a/persistence/sql_search_fts.go
+++ b/persistence/sql_search_fts.go
@@ -11,6 +11,7 @@ import (
"github.com/deluan/sanitize"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
+ "github.com/navidrome/navidrome/utils/str"
)
// containsCJK returns true if the string contains any CJK (Chinese/Japanese/Korean) characters.
@@ -35,48 +36,12 @@ func containsCJK(s string) bool {
// as unbalanced string delimiters.
var fts5SpecialChars = regexp.MustCompile(`[^\p{L}\p{N}\s*"\x00]`)
-// fts5PunctStrip strips everything except letters and numbers (no whitespace, wildcards, or quotes).
-// Used for normalizing words at index time to create concatenated forms (e.g., "R.E.M." → "REM").
-var fts5PunctStrip = regexp.MustCompile(`[^\p{L}\p{N}]`)
-
// fts5Operators matches FTS5 boolean operators as whole words (case-insensitive).
var fts5Operators = regexp.MustCompile(`(?i)\b(AND|OR|NOT|NEAR)\b`)
// fts5LeadingStar matches a * at the start of a token. FTS5 only supports * at the end (prefix queries).
var fts5LeadingStar = regexp.MustCompile(`(^|[\s])\*+`)
-// normalizeForFTS takes multiple strings and returns a space-separated, deduplicated list of
-// alternative searchable forms for each word: punctuation-stripped (R.E.M. → REM, AC/DC → ACDC)
-// and ASCII-transliterated (Bjørk → Bjork, œuvre → oeuvre). The transliterated form is needed
-// because FTS5's `unicode61 remove_diacritics 2` only handles NFKD-decomposable diacritics —
-// atomic letters like ø/æ/œ/ß survive tokenization, so the query side and index side disagree
-// without an explicit transliterated entry here.
-func normalizeForFTS(values ...string) string {
- seen := make(map[string]struct{})
- var result []string
- add := func(orig, variant string) {
- if variant == "" || variant == orig {
- return
- }
- lower := strings.ToLower(variant)
- if _, ok := seen[lower]; ok {
- return
- }
- seen[lower] = struct{}{}
- result = append(result, variant)
- }
- for _, v := range values {
- for word := range strings.FieldsSeq(v) {
- transliterated := sanitize.Accents(word)
- // Concatenated ASCII form: R.E.M. → REM, AC/DC → ACDC, St-Étienne → StEtienne.
- add(word, fts5PunctStrip.ReplaceAllString(transliterated, ""))
- // Accent-only transliteration for words without name-punctuation (Bjørk → Bjork).
- add(word, transliterated)
- }
- }
- return strings.Join(result, " ")
-}
-
// isSingleUnicodeLetter returns true if token is exactly one Unicode letter.
func isSingleUnicodeLetter(token string) bool {
r, size := utf8.DecodeRuneInString(token)
@@ -100,7 +65,7 @@ func processPunctuatedWords(input string, phrases []string) (string, []string) {
result = append(result, w)
continue
}
- concat := fts5PunctStrip.ReplaceAllString(w, "")
+ concat := str.FTSPunctStrip.ReplaceAllString(w, "")
if concat == "" || concat == w {
result = append(result, w)
continue
@@ -329,7 +294,7 @@ func ftsQueryDegraded(original, ftsQuery string) bool {
// Strip quotes from original for comparison — we want the raw content
stripped := strings.ReplaceAll(original, `"`, "")
// Extract the alphanumeric content from the original query
- alphaNum := fts5PunctStrip.ReplaceAllString(stripped, "")
+ alphaNum := str.FTSPunctStrip.ReplaceAllString(stripped, "")
// If the original is entirely alphanumeric, nothing was stripped — not degraded
if len(alphaNum) == len(stripped) {
return false
@@ -353,7 +318,7 @@ func ftsQueryDegraded(original, ftsQuery string) bool {
if strings.HasPrefix(t, `"`) {
// Extract content between quotes
inner := strings.Trim(t, `"`)
- innerAlpha := fts5PunctStrip.ReplaceAllString(inner, " ")
+ innerAlpha := str.FTSPunctStrip.ReplaceAllString(inner, " ")
for it := range strings.FieldsSeq(innerAlpha) {
if len(it) > 2 {
return false
diff --git a/persistence/sql_search_fts_test.go b/persistence/sql_search_fts_test.go
index b54e5856a..6c975c601 100644
--- a/persistence/sql_search_fts_test.go
+++ b/persistence/sql_search_fts_test.go
@@ -74,28 +74,6 @@ var _ = DescribeTable("ftsQueryDegraded",
Entry("not degraded for OR groups from processPunctuatedWords", "AC/DC", `("AC DC" OR ACDC*)`, false),
)
-var _ = DescribeTable("normalizeForFTS",
- func(expected string, values ...string) {
- Expect(normalizeForFTS(values...)).To(Equal(expected))
- },
- Entry("strips dots and concatenates", "REM", "R.E.M."),
- Entry("strips slash", "ACDC", "AC/DC"),
- Entry("strips hyphen", "Aha", "A-ha"),
- Entry("skips unchanged ASCII words", "", "The Beatles"),
- Entry("handles mixed input", "REM", "R.E.M.", "Automatic for the People"),
- Entry("deduplicates", "REM", "R.E.M.", "R.E.M."),
- Entry("strips apostrophe from word", "N", "Guns N' Roses"),
- Entry("handles multiple values with punctuation", "REM ACDC", "R.E.M.", "AC/DC"),
- Entry("transliterates ø to o", "Bjork", "Bjørk"),
- Entry("transliterates Ø to O", "Oystein", "Øystein"),
- Entry("transliterates œ ligature to oe", "oeuvre", "œuvre"),
- Entry("transliterates Latin diacritics", "cafe", "café"),
- Entry("transliterates only the non-ASCII words", "Mo Ros", "Mø Rós"),
- Entry("combines punctuation strip and transliteration", "StEtienne St-Etienne", "St-Étienne"),
- Entry("deduplicates against punctuation form", "Cafe", "Café", "Cafe"),
- Entry("transliterates ß to ss", "Strasse", "Straße"),
-)
-
var _ = DescribeTable("containsCJK",
func(input string, expected bool) {
Expect(containsCJK(input)).To(Equal(expected))
diff --git a/scanner/phase_1_folders.go b/scanner/phase_1_folders.go
index 38967832c..5e898590b 100644
--- a/scanner/phase_1_folders.go
+++ b/scanner/phase_1_folders.go
@@ -360,7 +360,7 @@ func (p *phaseFolders) persistChanges(entry *folderEntry) (*folderEntry, error)
// Save all new/modified artists to DB. Their information will be incomplete, but they will be refreshed later
for i := range entry.artists {
err = artistRepo.Put(&entry.artists[i], "name",
- "mbz_artist_id", "sort_artist_name", "order_artist_name", "full_text", "updated_at")
+ "mbz_artist_id", "sort_artist_name", "order_artist_name", "full_text", "search_normalized", "updated_at")
if err != nil {
log.Error(p.ctx, "Scanner: Error persisting artist to DB", "folder", entry.path, "artist", entry.artists[i].Name, err)
return err
diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go
index cc3732bc3..7f3dca775 100644
--- a/scanner/scanner_test.go
+++ b/scanner/scanner_test.go
@@ -189,6 +189,34 @@ var _ = Describe("Scanner", Ordered, func() {
})
})
+ Context("Artist with atomic non-ASCII letters, 'GØGGS'", func() {
+ BeforeEach(func() {
+ goggs := template(_t{"albumartist": "GØGGS", "album": "Pre Strike Sweep", "year": 2018})
+ createFS(fstest.MapFS{
+ "GØGGS/Pre Strike Sweep/01 - Falling For You.mp3": goggs(track(1, "Falling For You")),
+ })
+ })
+
+ searchNormalized := func() string {
+ var sn string
+ Expect(db.Db().QueryRowContext(ctx,
+ "SELECT search_normalized FROM artist WHERE name = 'GØGGS'").Scan(&sn)).To(Succeed())
+ return sn
+ }
+
+ It("repopulates a stale search_normalized on a full rescan", func() {
+ Expect(runScanner(ctx, true)).To(Succeed())
+ Expect(searchNormalized()).To(Equal("GOGGS"))
+
+ // Simulate the stale value left by the FTS5 migration's SQL back-fill
+ _, err := db.Db().ExecContext(ctx, "UPDATE artist SET search_normalized = '' WHERE name = 'GØGGS'")
+ Expect(err).ToNot(HaveOccurred())
+
+ Expect(runScanner(ctx, true)).To(Succeed())
+ Expect(searchNormalized()).To(Equal("GOGGS"))
+ })
+ })
+
Context("Ignored entries", func() {
BeforeEach(func() {
revolver := template(_t{"albumartist": "The Beatles", "album": "Revolver", "year": 1966})
diff --git a/utils/str/normalize_fts.go b/utils/str/normalize_fts.go
new file mode 100644
index 000000000..994f77cc9
--- /dev/null
+++ b/utils/str/normalize_fts.go
@@ -0,0 +1,45 @@
+package str
+
+import (
+ "regexp"
+ "strings"
+
+ "github.com/deluan/sanitize"
+)
+
+// FTSPunctStrip matches any character that is not a letter or number. Index-time
+// normalization (NormalizeForFTS) and query-time processing in persistence share it
+// so both sides produce matching tokens.
+var FTSPunctStrip = regexp.MustCompile(`[^\p{L}\p{N}]`)
+
+// NormalizeForFTS takes multiple strings and returns a space-separated, deduplicated list of
+// alternative searchable forms for each word: punctuation-stripped (R.E.M. → REM, AC/DC → ACDC)
+// and ASCII-transliterated (Bjørk → Bjork, œuvre → oeuvre). The transliterated form is needed
+// because FTS5's `unicode61 remove_diacritics 2` only handles NFKD-decomposable diacritics —
+// atomic letters like ø/æ/œ/ß survive tokenization, so the query side and index side disagree
+// without an explicit transliterated entry here.
+func NormalizeForFTS(values ...string) string {
+ seen := make(map[string]struct{})
+ var result []string
+ add := func(orig, variant string) {
+ if variant == "" || variant == orig {
+ return
+ }
+ lower := strings.ToLower(variant)
+ if _, ok := seen[lower]; ok {
+ return
+ }
+ seen[lower] = struct{}{}
+ result = append(result, variant)
+ }
+ for _, v := range values {
+ for word := range strings.FieldsSeq(v) {
+ transliterated := sanitize.Accents(word)
+ // Concatenated ASCII form: R.E.M. → REM, AC/DC → ACDC, St-Étienne → StEtienne.
+ add(word, FTSPunctStrip.ReplaceAllString(transliterated, ""))
+ // Accent-only transliteration for words without name-punctuation (Bjørk → Bjork).
+ add(word, transliterated)
+ }
+ }
+ return strings.Join(result, " ")
+}
diff --git a/utils/str/normalize_fts_test.go b/utils/str/normalize_fts_test.go
new file mode 100644
index 000000000..52387cb1d
--- /dev/null
+++ b/utils/str/normalize_fts_test.go
@@ -0,0 +1,29 @@
+package str_test
+
+import (
+ "github.com/navidrome/navidrome/utils/str"
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+)
+
+var _ = DescribeTable("NormalizeForFTS",
+ func(expected string, values ...string) {
+ Expect(str.NormalizeForFTS(values...)).To(Equal(expected))
+ },
+ Entry("strips dots and concatenates", "REM", "R.E.M."),
+ Entry("strips slash", "ACDC", "AC/DC"),
+ Entry("strips hyphen", "Aha", "A-ha"),
+ Entry("skips unchanged ASCII words", "", "The Beatles"),
+ Entry("handles mixed input", "REM", "R.E.M.", "Automatic for the People"),
+ Entry("deduplicates", "REM", "R.E.M.", "R.E.M."),
+ Entry("strips apostrophe from word", "N", "Guns N' Roses"),
+ Entry("handles multiple values with punctuation", "REM ACDC", "R.E.M.", "AC/DC"),
+ Entry("transliterates ø to o", "Bjork", "Bjørk"),
+ Entry("transliterates Ø to O", "Oystein", "Øystein"),
+ Entry("transliterates œ ligature to oe", "oeuvre", "œuvre"),
+ Entry("transliterates Latin diacritics", "cafe", "café"),
+ Entry("transliterates only the non-ASCII words", "Mo Ros", "Mø Rós"),
+ Entry("combines punctuation strip and transliteration", "StEtienne St-Etienne", "St-Étienne"),
+ Entry("deduplicates against punctuation form", "Cafe", "Café", "Cafe"),
+ Entry("transliterates ß to ss", "Strasse", "Straße"),
+)