🎧 Your Personal Streaming Service
Go airsonicmadsonicmedia-servermusicmusic-servernavidromeopensubsonicraspberry-piself-hostedstreamerstreaming-apistreaming-audiosubsonicsubsonic-server
Fine tune search functionality
Deluan80c8d85cb97d
diff --git a/db/migration/20200419222708_reindex_to_change_full_text_search.go b/db/migration/20200419222708_reindex_to_change_full_text_search.gonew file mode 100644index 000000000..b92641be4--- /dev/null+++ b/db/migration/20200419222708_reindex_to_change_full_text_search.go@@ -0,0 +1,19 @@+package migration++import (+ "database/sql"+ "github.com/pressly/goose"+)++func init() {+ goose.AddMigration(Up20200419222708, Down20200419222708)+}++func Up20200419222708(tx *sql.Tx) error {+ notice(tx, "A full rescan will be performed to change the search behaviour")+ return forceFullRescan(tx)+}++func Down20200419222708(tx *sql.Tx) error {+ return nil+}diff --git a/persistence/album_repository.go b/persistence/album_repository.goindex 098885ecc..0d62843d8 100644--- a/persistence/album_repository.go+++ b/persistence/album_repository.go@@ -148,7 +148,7 @@ func (r *albumRepository) Refresh(ids ...string) error {toInsert++al.CreatedAt = time.Now()}- al.FullText = r.getFullText(al.Name, al.Artist, al.AlbumArtist, al.SongArtists)+ al.FullText = getFullText(al.Name, al.Artist, al.AlbumArtist, al.SongArtists)_, err := r.put(al.ID, al.Album)if err != nil {return errdiff --git a/persistence/album_repository_test.go b/persistence/album_repository_test.goindex 1236b57b5..e057344f2 100644--- a/persistence/album_repository_test.go+++ b/persistence/album_repository_test.go@@ -20,7 +20,7 @@ var _ = Describe("AlbumRepository", func() {Describe("Get", func() {It("returns an existent album", func() {- Expect(repo.Get("3")).To(Equal(&albumRadioactivity))+ Expect(repo.Get("103")).To(Equal(&albumRadioactivity))})It("returns ErrNotFound when the album does not exist", func() {_, err := repo.Get("666")diff --git a/persistence/artist_repository.go b/persistence/artist_repository.goindex 7496d0f53..7b161b241 100644--- a/persistence/artist_repository.go+++ b/persistence/artist_repository.go@@ -56,7 +56,7 @@ func (r *artistRepository) getIndexKey(a *model.Artist) string {}func (r *artistRepository) Put(a *model.Artist) error {- a.FullText = r.getFullText(a.Name)+ a.FullText = getFullText(a.Name)_, err := r.put(a.ID, a)return err}diff --git a/persistence/mediafile_repository.go b/persistence/mediafile_repository.goindex f6c29f4d4..a725e2f68 100644--- a/persistence/mediafile_repository.go+++ b/persistence/mediafile_repository.go@@ -41,7 +41,7 @@ func (r mediaFileRepository) Exists(id string) (bool, error) {}func (r mediaFileRepository) Put(m *model.MediaFile) error {- m.FullText = r.getFullText(m.Title, m.Album, m.Artist, m.AlbumArtist)+ m.FullText = getFullText(m.Title, m.Album, m.Artist, m.AlbumArtist)_, err := r.put(m.ID, m)return err}diff --git a/persistence/mediafile_repository_test.go b/persistence/mediafile_repository_test.goindex 20af87f46..ea09e5a64 100644--- a/persistence/mediafile_repository_test.go+++ b/persistence/mediafile_repository_test.go@@ -21,7 +21,7 @@ var _ = Describe("MediaRepository", func() {})It("gets mediafile from the DB", func() {- Expect(mr.Get("4")).To(Equal(&songAntenna))+ Expect(mr.Get("1004")).To(Equal(&songAntenna))})It("returns ErrNotFound", func() {@@ -39,7 +39,7 @@ var _ = Describe("MediaRepository", func() {})It("find mediafiles by album", func() {- Expect(mr.FindByAlbum("3")).To(Equal(model.MediaFiles{+ Expect(mr.FindByAlbum("103")).To(Equal(model.MediaFiles{songRadioactivity,songAntenna,}))diff --git a/persistence/persistence_suite_test.go b/persistence/persistence_suite_test.goindex fc2414124..4e6b8703b 100644--- a/persistence/persistence_suite_test.go+++ b/persistence/persistence_suite_test.go@@ -31,8 +31,8 @@ func TestPersistence(t *testing.T) {}var (- artistKraftwerk = model.Artist{ID: "2", Name: "Kraftwerk", AlbumCount: 1, FullText: "kraftwerk"}- artistBeatles = model.Artist{ID: "3", Name: "The Beatles", AlbumCount: 2, FullText: "beatles the"}+ artistKraftwerk = model.Artist{ID: "2", Name: "Kraftwerk", AlbumCount: 1, FullText: " kraftwerk"}+ artistBeatles = model.Artist{ID: "3", Name: "The Beatles", AlbumCount: 2, FullText: " beatles the"}testArtists = model.Artists{artistKraftwerk,artistBeatles,@@ -40,9 +40,9 @@ var ()var (- albumSgtPeppers = model.Album{ID: "1", Name: "Sgt Peppers", Artist: "The Beatles", AlbumArtistID: "3", Genre: "Rock", CoverArtId: "1", CoverArtPath: P("/beatles/1/sgt/a day.mp3"), SongCount: 1, MaxYear: 1967, FullText: "beatles peppers sgt the"}- albumAbbeyRoad = model.Album{ID: "2", Name: "Abbey Road", Artist: "The Beatles", AlbumArtistID: "3", Genre: "Rock", CoverArtId: "2", CoverArtPath: P("/beatles/1/come together.mp3"), SongCount: 1, MaxYear: 1969, FullText: "abbey beatles road the"}- albumRadioactivity = model.Album{ID: "3", Name: "Radioactivity", Artist: "Kraftwerk", AlbumArtistID: "2", Genre: "Electronic", CoverArtId: "3", CoverArtPath: P("/kraft/radio/radio.mp3"), SongCount: 2, FullText: "kraftwerk radioactivity"}+ albumSgtPeppers = model.Album{ID: "101", Name: "Sgt Peppers", Artist: "The Beatles", AlbumArtistID: "3", Genre: "Rock", CoverArtId: "1", CoverArtPath: P("/beatles/1/sgt/a day.mp3"), SongCount: 1, MaxYear: 1967, FullText: " beatles peppers sgt the"}+ albumAbbeyRoad = model.Album{ID: "102", Name: "Abbey Road", Artist: "The Beatles", AlbumArtistID: "3", Genre: "Rock", CoverArtId: "2", CoverArtPath: P("/beatles/1/come together.mp3"), SongCount: 1, MaxYear: 1969, FullText: " abbey beatles road the"}+ albumRadioactivity = model.Album{ID: "103", Name: "Radioactivity", Artist: "Kraftwerk", AlbumArtistID: "2", Genre: "Electronic", CoverArtId: "3", CoverArtPath: P("/kraft/radio/radio.mp3"), SongCount: 2, FullText: " kraftwerk radioactivity"}testAlbums = model.Albums{albumSgtPeppers,albumAbbeyRoad,@@ -51,10 +51,10 @@ var ()var (- songDayInALife = model.MediaFile{ID: "1", Title: "A Day In A Life", ArtistID: "3", Artist: "The Beatles", AlbumID: "1", Album: "Sgt Peppers", Genre: "Rock", Path: P("/beatles/1/sgt/a day.mp3"), FullText: "a beatles day in life peppers sgt the"}- songComeTogether = model.MediaFile{ID: "2", Title: "Come Together", ArtistID: "3", Artist: "The Beatles", AlbumID: "2", Album: "Abbey Road", Genre: "Rock", Path: P("/beatles/1/come together.mp3"), FullText: "abbey beatles come road the together"}- songRadioactivity = model.MediaFile{ID: "3", Title: "Radioactivity", ArtistID: "2", Artist: "Kraftwerk", AlbumID: "3", Album: "Radioactivity", Genre: "Electronic", Path: P("/kraft/radio/radio.mp3"), FullText: "kraftwerk radioactivity"}- songAntenna = model.MediaFile{ID: "4", Title: "Antenna", ArtistID: "2", Artist: "Kraftwerk", AlbumID: "3", Genre: "Electronic", Path: P("/kraft/radio/antenna.mp3"), FullText: "antenna kraftwerk"}+ songDayInALife = model.MediaFile{ID: "1001", Title: "A Day In A Life", ArtistID: "3", Artist: "The Beatles", AlbumID: "101", Album: "Sgt Peppers", Genre: "Rock", Path: P("/beatles/1/sgt/a day.mp3"), FullText: " a beatles day in life peppers sgt the"}+ songComeTogether = model.MediaFile{ID: "1002", Title: "Come Together", ArtistID: "3", Artist: "The Beatles", AlbumID: "102", Album: "Abbey Road", Genre: "Rock", Path: P("/beatles/1/come together.mp3"), FullText: " abbey beatles come road the together"}+ songRadioactivity = model.MediaFile{ID: "1003", Title: "Radioactivity", ArtistID: "2", Artist: "Kraftwerk", AlbumID: "103", Album: "Radioactivity", Genre: "Electronic", Path: P("/kraft/radio/radio.mp3"), FullText: " kraftwerk radioactivity"}+ songAntenna = model.MediaFile{ID: "1004", Title: "Antenna", ArtistID: "2", Artist: "Kraftwerk", AlbumID: "103", Genre: "Electronic", Path: P("/kraft/radio/antenna.mp3"), FullText: " antenna kraftwerk"}testSongs = model.MediaFiles{songDayInALife,songComeTogether,@@ -70,9 +70,9 @@ var (Comment: "No Comments",Owner: "userid",Public: true,- Tracks: model.MediaFiles{{ID: "1"}, {ID: "3"}},+ Tracks: model.MediaFiles{{ID: "1001"}, {ID: "1003"}},}- plsCool = model.Playlist{ID: "11", Name: "Cool", Tracks: model.MediaFiles{{ID: "4"}}}+ plsCool = model.Playlist{ID: "11", Name: "Cool", Tracks: model.MediaFiles{{ID: "1004"}}}testPlaylists = model.Playlists{plsBest, plsCool})diff --git a/persistence/playlist_repository_test.go b/persistence/playlist_repository_test.goindex 85eb96a3e..271810146 100644--- a/persistence/playlist_repository_test.go+++ b/persistence/playlist_repository_test.go@@ -63,19 +63,19 @@ var _ = Describe("PlaylistRepository", func() {Describe("Put/Exists/Delete", func() {var newPls model.PlaylistBeforeEach(func() {- newPls = model.Playlist{ID: "22", Name: "Great!", Tracks: model.MediaFiles{{ID: "4"}, {ID: "3"}}}+ newPls = model.Playlist{ID: "22", Name: "Great!", Tracks: model.MediaFiles{{ID: "1004"}, {ID: "1003"}}}})It("saves the playlist to the DB", func() {Expect(repo.Put(&newPls)).To(BeNil())})It("adds repeated songs to a playlist and keeps the order", func() {- newPls.Tracks = append(newPls.Tracks, model.MediaFile{ID: "4"})+ newPls.Tracks = append(newPls.Tracks, model.MediaFile{ID: "1004"})Expect(repo.Put(&newPls)).To(BeNil())saved, _ := repo.Get("22")Expect(saved.Tracks).To(HaveLen(3))- Expect(saved.Tracks[0].ID).To(Equal("4"))- Expect(saved.Tracks[1].ID).To(Equal("3"))- Expect(saved.Tracks[2].ID).To(Equal("4"))+ Expect(saved.Tracks[0].ID).To(Equal("1004"))+ Expect(saved.Tracks[1].ID).To(Equal("1003"))+ Expect(saved.Tracks[2].ID).To(Equal("1004"))})It("returns the newly created playlist", func() {Expect(repo.Exists("22")).To(BeTrue())diff --git a/persistence/sql_restful.go b/persistence/sql_restful.goindex eb56b5cfe..0269c29df 100644--- a/persistence/sql_restful.go+++ b/persistence/sql_restful.go@@ -7,7 +7,6 @@ import (. "github.com/Masterminds/squirrel""github.com/deluan/navidrome/model""github.com/deluan/rest"- "github.com/kennygrant/sanitize")type filterFunc = func(field string, value interface{}) Sqlizer@@ -59,15 +58,11 @@ func booleanFilter(field string, value interface{}) Sqlizer {}func fullTextFilter(field string, value interface{}) Sqlizer {- q := value.(string)- q = strings.TrimSpace(sanitize.Accents(strings.ToLower(strings.TrimSuffix(q, "*"))))+ q := sanitizeStrings(value.(string))parts := strings.Split(q, " ")filters := And{}for _, part := range parts {- filters = append(filters, Or{- Like{"full_text": part + "%"},- Like{"full_text": "%" + part + "%"},- })+ filters = append(filters, Like{"full_text": "% " + part + "%"})}return filters}diff --git a/persistence/sql_search.go b/persistence/sql_search.goindex fb7f52688..2359e5363 100644--- a/persistence/sql_search.go+++ b/persistence/sql_search.go@@ -1,6 +1,7 @@package persistenceimport (+ "regexp""sort""strings"@@ -8,7 +9,14 @@ import ("github.com/kennygrant/sanitize")-func (r sqlRepository) getFullText(text ...string) string {+var quotesRegex = regexp.MustCompile("[“”‘’'\"]")++func getFullText(text ...string) string {+ fullText := sanitizeStrings(text...)+ return " " + fullText+}++func sanitizeStrings(text ...string) string {sanitizedText := strings.Builder{}for _, txt := range text {sanitizedText.WriteString(strings.TrimSpace(sanitize.Accents(strings.ToLower(txt))) + " ")@@ -19,14 +27,18 @@ func (r sqlRepository) getFullText(text ...string) string {}var fullText []stringfor w := range words {- fullText = append(fullText, w)+ w = quotesRegex.ReplaceAllString(w, "")+ if w != "" {+ fullText = append(fullText, w)+ }}sort.Strings(fullText)return strings.Join(fullText, " ")}func (r sqlRepository) doSearch(q string, offset, size int, results interface{}, orderBys ...string) error {- q = strings.TrimSpace(sanitize.Accents(strings.ToLower(strings.TrimSuffix(q, "*"))))+ q = strings.TrimSuffix(q, "*")+ q = sanitizeStrings(q)if len(q) < 2 {return nil}@@ -37,10 +49,7 @@ func (r sqlRepository) doSearch(q string, offset, size int, results interface{},}parts := strings.Split(q, " ")for _, part := range parts {- sq = sq.Where(Or{- Like{"full_text": part + "%"},- Like{"full_text": "%" + part + "%"},- })+ sq = sq.Where(Like{"full_text": "% " + part + "%"})}err := r.queryAll(sq, results)return errdiff --git a/persistence/sql_search_test.go b/persistence/sql_search_test.goindex 64038768f..11a846663 100644--- a/persistence/sql_search_test.go+++ b/persistence/sql_search_test.go@@ -6,23 +6,25 @@ import ()var _ = Describe("sqlRepository", func() {- var sqlRepository = &sqlRepository{}-Describe("getFullText", func() {It("returns all lowercase chars", func() {- Expect(sqlRepository.getFullText("Some Text")).To(Equal("some text"))+ Expect(getFullText("Some Text")).To(Equal(" some text"))})It("removes accents", func() {- Expect(sqlRepository.getFullText("Quintão")).To(Equal("quintao"))+ Expect(getFullText("Quintão")).To(Equal(" quintao"))})It("remove extra spaces", func() {- Expect(sqlRepository.getFullText(" some text ")).To(Equal("some text"))+ Expect(getFullText(" some text ")).To(Equal(" some text"))})It("remove duplicated words", func() {- Expect(sqlRepository.getFullText("legião urbana urbana legiÃo")).To(Equal("legiao urbana"))+ Expect(getFullText("legião urbana urbana legiÃo")).To(Equal(" legiao urbana"))+ })++ It("remove symbols", func() {+ Expect(getFullText("Tom’s Diner ' “40” ‘A’")).To(Equal(" 40 a diner toms"))})})})