Skip to content

🎧 Your Personal Streaming Service

Fine tune search functionality

Deluan80c8d85cb97d
diff --git a/db/migration/20200419222708_reindex_to_change_full_text_search.go b/db/migration/20200419222708_reindex_to_change_full_text_search.go
new file mode 100644
index 000000000..b92641be4
--- /dev/null
+++ b/db/migration/20200419222708_reindex_to_change_full_text_search.go
@@ -0,0 +1,19 @@
+package migration
+
+import (
+ "database/sql"
+ "github.com/pressly/goose"
+)
+
+func init() {
+ goose.AddMigration(Up20200419222708, Down20200419222708)
+}
+
+func Up20200419222708(tx *sql.Tx) error {
+ notice(tx, "A full rescan will be performed to change the search behaviour")
+ return forceFullRescan(tx)
+}
+
+func Down20200419222708(tx *sql.Tx) error {
+ return nil
+}
diff --git a/persistence/album_repository.go b/persistence/album_repository.go
index 098885ecc..0d62843d8 100644
--- a/persistence/album_repository.go
+++ b/persistence/album_repository.go
@@ -148,7 +148,7 @@ func (r *albumRepository) Refresh(ids ...string) error {
toInsert++
al.CreatedAt = time.Now()
}
- al.FullText = r.getFullText(al.Name, al.Artist, al.AlbumArtist, al.SongArtists)
+ al.FullText = getFullText(al.Name, al.Artist, al.AlbumArtist, al.SongArtists)
_, err := r.put(al.ID, al.Album)
if err != nil {
return err
diff --git a/persistence/album_repository_test.go b/persistence/album_repository_test.go
index 1236b57b5..e057344f2 100644
--- a/persistence/album_repository_test.go
+++ b/persistence/album_repository_test.go
@@ -20,7 +20,7 @@ var _ = Describe("AlbumRepository", func() {
Describe("Get", func() {
It("returns an existent album", func() {
- Expect(repo.Get("3")).To(Equal(&albumRadioactivity))
+ Expect(repo.Get("103")).To(Equal(&albumRadioactivity))
})
It("returns ErrNotFound when the album does not exist", func() {
_, err := repo.Get("666")
diff --git a/persistence/artist_repository.go b/persistence/artist_repository.go
index 7496d0f53..7b161b241 100644
--- a/persistence/artist_repository.go
+++ b/persistence/artist_repository.go
@@ -56,7 +56,7 @@ func (r *artistRepository) getIndexKey(a *model.Artist) string {
}
func (r *artistRepository) Put(a *model.Artist) error {
- a.FullText = r.getFullText(a.Name)
+ a.FullText = getFullText(a.Name)
_, err := r.put(a.ID, a)
return err
}
diff --git a/persistence/mediafile_repository.go b/persistence/mediafile_repository.go
index f6c29f4d4..a725e2f68 100644
--- a/persistence/mediafile_repository.go
+++ b/persistence/mediafile_repository.go
@@ -41,7 +41,7 @@ func (r mediaFileRepository) Exists(id string) (bool, error) {
}
func (r mediaFileRepository) Put(m *model.MediaFile) error {
- m.FullText = r.getFullText(m.Title, m.Album, m.Artist, m.AlbumArtist)
+ m.FullText = getFullText(m.Title, m.Album, m.Artist, m.AlbumArtist)
_, err := r.put(m.ID, m)
return err
}
diff --git a/persistence/mediafile_repository_test.go b/persistence/mediafile_repository_test.go
index 20af87f46..ea09e5a64 100644
--- a/persistence/mediafile_repository_test.go
+++ b/persistence/mediafile_repository_test.go
@@ -21,7 +21,7 @@ var _ = Describe("MediaRepository", func() {
})
It("gets mediafile from the DB", func() {
- Expect(mr.Get("4")).To(Equal(&songAntenna))
+ Expect(mr.Get("1004")).To(Equal(&songAntenna))
})
It("returns ErrNotFound", func() {
@@ -39,7 +39,7 @@ var _ = Describe("MediaRepository", func() {
})
It("find mediafiles by album", func() {
- Expect(mr.FindByAlbum("3")).To(Equal(model.MediaFiles{
+ Expect(mr.FindByAlbum("103")).To(Equal(model.MediaFiles{
songRadioactivity,
songAntenna,
}))
diff --git a/persistence/persistence_suite_test.go b/persistence/persistence_suite_test.go
index fc2414124..4e6b8703b 100644
--- a/persistence/persistence_suite_test.go
+++ b/persistence/persistence_suite_test.go
@@ -31,8 +31,8 @@ func TestPersistence(t *testing.T) {
}
var (
- artistKraftwerk = model.Artist{ID: "2", Name: "Kraftwerk", AlbumCount: 1, FullText: "kraftwerk"}
- artistBeatles = model.Artist{ID: "3", Name: "The Beatles", AlbumCount: 2, FullText: "beatles the"}
+ artistKraftwerk = model.Artist{ID: "2", Name: "Kraftwerk", AlbumCount: 1, FullText: " kraftwerk"}
+ artistBeatles = model.Artist{ID: "3", Name: "The Beatles", AlbumCount: 2, FullText: " beatles the"}
testArtists = model.Artists{
artistKraftwerk,
artistBeatles,
@@ -40,9 +40,9 @@ var (
)
var (
- albumSgtPeppers = model.Album{ID: "1", Name: "Sgt Peppers", Artist: "The Beatles", AlbumArtistID: "3", Genre: "Rock", CoverArtId: "1", CoverArtPath: P("/beatles/1/sgt/a day.mp3"), SongCount: 1, MaxYear: 1967, FullText: "beatles peppers sgt the"}
- albumAbbeyRoad = model.Album{ID: "2", Name: "Abbey Road", Artist: "The Beatles", AlbumArtistID: "3", Genre: "Rock", CoverArtId: "2", CoverArtPath: P("/beatles/1/come together.mp3"), SongCount: 1, MaxYear: 1969, FullText: "abbey beatles road the"}
- albumRadioactivity = model.Album{ID: "3", Name: "Radioactivity", Artist: "Kraftwerk", AlbumArtistID: "2", Genre: "Electronic", CoverArtId: "3", CoverArtPath: P("/kraft/radio/radio.mp3"), SongCount: 2, FullText: "kraftwerk radioactivity"}
+ albumSgtPeppers = model.Album{ID: "101", Name: "Sgt Peppers", Artist: "The Beatles", AlbumArtistID: "3", Genre: "Rock", CoverArtId: "1", CoverArtPath: P("/beatles/1/sgt/a day.mp3"), SongCount: 1, MaxYear: 1967, FullText: " beatles peppers sgt the"}
+ albumAbbeyRoad = model.Album{ID: "102", Name: "Abbey Road", Artist: "The Beatles", AlbumArtistID: "3", Genre: "Rock", CoverArtId: "2", CoverArtPath: P("/beatles/1/come together.mp3"), SongCount: 1, MaxYear: 1969, FullText: " abbey beatles road the"}
+ albumRadioactivity = model.Album{ID: "103", Name: "Radioactivity", Artist: "Kraftwerk", AlbumArtistID: "2", Genre: "Electronic", CoverArtId: "3", CoverArtPath: P("/kraft/radio/radio.mp3"), SongCount: 2, FullText: " kraftwerk radioactivity"}
testAlbums = model.Albums{
albumSgtPeppers,
albumAbbeyRoad,
@@ -51,10 +51,10 @@ var (
)
var (
- songDayInALife = model.MediaFile{ID: "1", Title: "A Day In A Life", ArtistID: "3", Artist: "The Beatles", AlbumID: "1", Album: "Sgt Peppers", Genre: "Rock", Path: P("/beatles/1/sgt/a day.mp3"), FullText: "a beatles day in life peppers sgt the"}
- songComeTogether = model.MediaFile{ID: "2", Title: "Come Together", ArtistID: "3", Artist: "The Beatles", AlbumID: "2", Album: "Abbey Road", Genre: "Rock", Path: P("/beatles/1/come together.mp3"), FullText: "abbey beatles come road the together"}
- songRadioactivity = model.MediaFile{ID: "3", Title: "Radioactivity", ArtistID: "2", Artist: "Kraftwerk", AlbumID: "3", Album: "Radioactivity", Genre: "Electronic", Path: P("/kraft/radio/radio.mp3"), FullText: "kraftwerk radioactivity"}
- songAntenna = model.MediaFile{ID: "4", Title: "Antenna", ArtistID: "2", Artist: "Kraftwerk", AlbumID: "3", Genre: "Electronic", Path: P("/kraft/radio/antenna.mp3"), FullText: "antenna kraftwerk"}
+ songDayInALife = model.MediaFile{ID: "1001", Title: "A Day In A Life", ArtistID: "3", Artist: "The Beatles", AlbumID: "101", Album: "Sgt Peppers", Genre: "Rock", Path: P("/beatles/1/sgt/a day.mp3"), FullText: " a beatles day in life peppers sgt the"}
+ songComeTogether = model.MediaFile{ID: "1002", Title: "Come Together", ArtistID: "3", Artist: "The Beatles", AlbumID: "102", Album: "Abbey Road", Genre: "Rock", Path: P("/beatles/1/come together.mp3"), FullText: " abbey beatles come road the together"}
+ songRadioactivity = model.MediaFile{ID: "1003", Title: "Radioactivity", ArtistID: "2", Artist: "Kraftwerk", AlbumID: "103", Album: "Radioactivity", Genre: "Electronic", Path: P("/kraft/radio/radio.mp3"), FullText: " kraftwerk radioactivity"}
+ songAntenna = model.MediaFile{ID: "1004", Title: "Antenna", ArtistID: "2", Artist: "Kraftwerk", AlbumID: "103", Genre: "Electronic", Path: P("/kraft/radio/antenna.mp3"), FullText: " antenna kraftwerk"}
testSongs = model.MediaFiles{
songDayInALife,
songComeTogether,
@@ -70,9 +70,9 @@ var (
Comment: "No Comments",
Owner: "userid",
Public: true,
- Tracks: model.MediaFiles{{ID: "1"}, {ID: "3"}},
+ Tracks: model.MediaFiles{{ID: "1001"}, {ID: "1003"}},
}
- plsCool = model.Playlist{ID: "11", Name: "Cool", Tracks: model.MediaFiles{{ID: "4"}}}
+ plsCool = model.Playlist{ID: "11", Name: "Cool", Tracks: model.MediaFiles{{ID: "1004"}}}
testPlaylists = model.Playlists{plsBest, plsCool}
)
diff --git a/persistence/playlist_repository_test.go b/persistence/playlist_repository_test.go
index 85eb96a3e..271810146 100644
--- a/persistence/playlist_repository_test.go
+++ b/persistence/playlist_repository_test.go
@@ -63,19 +63,19 @@ var _ = Describe("PlaylistRepository", func() {
Describe("Put/Exists/Delete", func() {
var newPls model.Playlist
BeforeEach(func() {
- newPls = model.Playlist{ID: "22", Name: "Great!", Tracks: model.MediaFiles{{ID: "4"}, {ID: "3"}}}
+ newPls = model.Playlist{ID: "22", Name: "Great!", Tracks: model.MediaFiles{{ID: "1004"}, {ID: "1003"}}}
})
It("saves the playlist to the DB", func() {
Expect(repo.Put(&newPls)).To(BeNil())
})
It("adds repeated songs to a playlist and keeps the order", func() {
- newPls.Tracks = append(newPls.Tracks, model.MediaFile{ID: "4"})
+ newPls.Tracks = append(newPls.Tracks, model.MediaFile{ID: "1004"})
Expect(repo.Put(&newPls)).To(BeNil())
saved, _ := repo.Get("22")
Expect(saved.Tracks).To(HaveLen(3))
- Expect(saved.Tracks[0].ID).To(Equal("4"))
- Expect(saved.Tracks[1].ID).To(Equal("3"))
- Expect(saved.Tracks[2].ID).To(Equal("4"))
+ Expect(saved.Tracks[0].ID).To(Equal("1004"))
+ Expect(saved.Tracks[1].ID).To(Equal("1003"))
+ Expect(saved.Tracks[2].ID).To(Equal("1004"))
})
It("returns the newly created playlist", func() {
Expect(repo.Exists("22")).To(BeTrue())
diff --git a/persistence/sql_restful.go b/persistence/sql_restful.go
index eb56b5cfe..0269c29df 100644
--- a/persistence/sql_restful.go
+++ b/persistence/sql_restful.go
@@ -7,7 +7,6 @@ import (
. "github.com/Masterminds/squirrel"
"github.com/deluan/navidrome/model"
"github.com/deluan/rest"
- "github.com/kennygrant/sanitize"
)
type filterFunc = func(field string, value interface{}) Sqlizer
@@ -59,15 +58,11 @@ func booleanFilter(field string, value interface{}) Sqlizer {
}
func fullTextFilter(field string, value interface{}) Sqlizer {
- q := value.(string)
- q = strings.TrimSpace(sanitize.Accents(strings.ToLower(strings.TrimSuffix(q, "*"))))
+ q := sanitizeStrings(value.(string))
parts := strings.Split(q, " ")
filters := And{}
for _, part := range parts {
- filters = append(filters, Or{
- Like{"full_text": part + "%"},
- Like{"full_text": "%" + part + "%"},
- })
+ filters = append(filters, Like{"full_text": "% " + part + "%"})
}
return filters
}
diff --git a/persistence/sql_search.go b/persistence/sql_search.go
index fb7f52688..2359e5363 100644
--- a/persistence/sql_search.go
+++ b/persistence/sql_search.go
@@ -1,6 +1,7 @@
package persistence
import (
+ "regexp"
"sort"
"strings"
@@ -8,7 +9,14 @@ import (
"github.com/kennygrant/sanitize"
)
-func (r sqlRepository) getFullText(text ...string) string {
+var quotesRegex = regexp.MustCompile("[“”‘’'\"]")
+
+func getFullText(text ...string) string {
+ fullText := sanitizeStrings(text...)
+ return " " + fullText
+}
+
+func sanitizeStrings(text ...string) string {
sanitizedText := strings.Builder{}
for _, txt := range text {
sanitizedText.WriteString(strings.TrimSpace(sanitize.Accents(strings.ToLower(txt))) + " ")
@@ -19,14 +27,18 @@ func (r sqlRepository) getFullText(text ...string) string {
}
var fullText []string
for w := range words {
- fullText = append(fullText, w)
+ w = quotesRegex.ReplaceAllString(w, "")
+ if w != "" {
+ fullText = append(fullText, w)
+ }
}
sort.Strings(fullText)
return strings.Join(fullText, " ")
}
func (r sqlRepository) doSearch(q string, offset, size int, results interface{}, orderBys ...string) error {
- q = strings.TrimSpace(sanitize.Accents(strings.ToLower(strings.TrimSuffix(q, "*"))))
+ q = strings.TrimSuffix(q, "*")
+ q = sanitizeStrings(q)
if len(q) < 2 {
return nil
}
@@ -37,10 +49,7 @@ func (r sqlRepository) doSearch(q string, offset, size int, results interface{},
}
parts := strings.Split(q, " ")
for _, part := range parts {
- sq = sq.Where(Or{
- Like{"full_text": part + "%"},
- Like{"full_text": "%" + part + "%"},
- })
+ sq = sq.Where(Like{"full_text": "% " + part + "%"})
}
err := r.queryAll(sq, results)
return err
diff --git a/persistence/sql_search_test.go b/persistence/sql_search_test.go
index 64038768f..11a846663 100644
--- a/persistence/sql_search_test.go
+++ b/persistence/sql_search_test.go
@@ -6,23 +6,25 @@ import (
)
var _ = Describe("sqlRepository", func() {
- var sqlRepository = &sqlRepository{}
-
Describe("getFullText", func() {
It("returns all lowercase chars", func() {
- Expect(sqlRepository.getFullText("Some Text")).To(Equal("some text"))
+ Expect(getFullText("Some Text")).To(Equal(" some text"))
})
It("removes accents", func() {
- Expect(sqlRepository.getFullText("Quintão")).To(Equal("quintao"))
+ Expect(getFullText("Quintão")).To(Equal(" quintao"))
})
It("remove extra spaces", func() {
- Expect(sqlRepository.getFullText(" some text ")).To(Equal("some text"))
+ Expect(getFullText(" some text ")).To(Equal(" some text"))
})
It("remove duplicated words", func() {
- Expect(sqlRepository.getFullText("legião urbana urbana legiÃo")).To(Equal("legiao urbana"))
+ Expect(getFullText("legião urbana urbana legiÃo")).To(Equal(" legiao urbana"))
+ })
+
+ It("remove symbols", func() {
+ Expect(getFullText("Tom’s Diner ' “40” ‘A’")).To(Equal(" 40 a diner toms"))
})
})
})