🎧 Your Personal Streaming Service
Go airsonicmadsonicmedia-servermusicmusic-servernavidromeopensubsonicraspberry-piself-hostedstreamerstreaming-apistreaming-audiosubsonicsubsonic-server
perf(scanner): improve playlist importing on large libraries (#6055)
Deluan Quintãodbd26ba2e71d
diff --git a/core/playlists/parse_m3u.go b/core/playlists/parse_m3u.goindex a64c337c9..286f2e420 100644--- a/core/playlists/parse_m3u.go+++ b/core/playlists/parse_m3u.go@@ -25,8 +25,8 @@ func (s *playlists) parseM3U(ctx context.Context, pls *model.Playlist, folder *mreturn err}var mfs model.MediaFiles- // Chunk size of 100 lines, as each line can generate up to 4 lookup candidates- // (NFC/NFD × raw/lowercase), and SQLite has a max expression tree depth of 1000.+ // Chunked so a huge playlist is not held in memory at once. Each line yields up to+ // 4 lookup candidates (NFC/NFD × raw/lowercase), far below SQLite's 32766 variables.for lines := range slice.CollectChunks(slice.LinesFrom(reader), 100) {filteredLines := make([]string, 0, len(lines))for _, line := range lines {diff --git a/persistence/mediafile_repository.go b/persistence/mediafile_repository.goindex 320b95ef2..ed18333de 100644--- a/persistence/mediafile_repository.go+++ b/persistence/mediafile_repository.go@@ -4,6 +4,7 @@ import ("context""fmt""iter"+ "maps""slices""strconv""strings"@@ -355,7 +356,10 @@ func (r *mediaFileRepository) GetCursorWithArtwork(options ...model.QueryOptions// Library-qualified paths search within the specified library, while unqualified paths// search across all libraries for backward compatibility.func (r *mediaFileRepository) FindByPaths(paths []string) (model.MediaFiles, error) {- query := Or{}+ // One IN list per library instead of one OR term per path: SQLite abandons the+ // path index at just two OR-ed equality terms and scans the whole table.+ byLibrary := map[int][]string{}+ var unqualified []stringfor _, path := range paths {parts := strings.SplitN(path, ":", 2)@@ -366,17 +370,24 @@ func (r *mediaFileRepository) FindByPaths(paths []string) (model.MediaFiles, err// Invalid format, skipcontinue}- relativePath := parts[1]- query = append(query, And{- Eq{"path collate nocase": relativePath},- Eq{"library_id": libraryID},- })+ byLibrary[libraryID] = append(byLibrary[libraryID], parts[1])} else {// Unqualified path: search across all libraries- query = append(query, Eq{"path collate nocase": path})+ unqualified = append(unqualified, path)}}+ query := Or{}+ for _, libraryID := range slices.Sorted(maps.Keys(byLibrary)) {+ query = append(query, And{+ Eq{"path collate nocase": byLibrary[libraryID]},+ Eq{"library_id": libraryID},+ })+ }+ if len(unqualified) > 0 {+ query = append(query, Eq{"path collate nocase": unqualified})+ }+if len(query) == 0 {return model.MediaFiles{}, nil}diff --git a/persistence/mediafile_repository_test.go b/persistence/mediafile_repository_test.goindex c1a91c5a5..8a492a813 100644--- a/persistence/mediafile_repository_test.go+++ b/persistence/mediafile_repository_test.go@@ -1055,6 +1055,28 @@ var _ = Describe("MediaRepository", func() {Expect(results).To(HaveLen(1))Expect(results[0].ID).To(Equal("otherlib-track"))})++ It("resolves paths from multiple libraries in a single call", func() {+ adminMr := NewMediaFileRepository(request.WithUser(GinkgoT().Context(), adminUser), GetDBXBuilder())+ results, err := adminMr.FindByPaths([]string{+ "1:artist/Album/track.mp3",+ fmt.Sprintf("%d:hidden/test.mp3", otherLib.ID),+ })+ Expect(err).ToNot(HaveOccurred())+ Expect(results).To(HaveLen(2))+ Expect([]string{results[0].ID, results[1].ID}).To(ConsistOf("findpath-1", "otherlib-track"))+ })++ It("keeps each path scoped to its own library when several are queried", func() {+ adminMr := NewMediaFileRepository(request.WithUser(GinkgoT().Context(), adminUser), GetDBXBuilder())+ // Each path exists, but under the other library's ID, so neither must match.+ results, err := adminMr.FindByPaths([]string{+ fmt.Sprintf("%d:artist/Album/track.mp3", otherLib.ID),+ "1:hidden/test.mp3",+ })+ Expect(err).ToNot(HaveOccurred())+ Expect(results).To(BeEmpty())+ })})})