Skip to content

🎧 Your Personal Streaming Service

perf(scanner): improve playlist importing on large libraries (#6055)

Deluan Quintãodbd26ba2e71d
diff --git a/core/playlists/parse_m3u.go b/core/playlists/parse_m3u.go
index a64c337c9..286f2e420 100644
--- a/core/playlists/parse_m3u.go
+++ b/core/playlists/parse_m3u.go
@@ -25,8 +25,8 @@ func (s *playlists) parseM3U(ctx context.Context, pls *model.Playlist, folder *m
return err
}
var mfs model.MediaFiles
- // Chunk size of 100 lines, as each line can generate up to 4 lookup candidates
- // (NFC/NFD × raw/lowercase), and SQLite has a max expression tree depth of 1000.
+ // Chunked so a huge playlist is not held in memory at once. Each line yields up to
+ // 4 lookup candidates (NFC/NFD × raw/lowercase), far below SQLite's 32766 variables.
for lines := range slice.CollectChunks(slice.LinesFrom(reader), 100) {
filteredLines := make([]string, 0, len(lines))
for _, line := range lines {
diff --git a/persistence/mediafile_repository.go b/persistence/mediafile_repository.go
index 320b95ef2..ed18333de 100644
--- a/persistence/mediafile_repository.go
+++ b/persistence/mediafile_repository.go
@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"iter"
+ "maps"
"slices"
"strconv"
"strings"
@@ -355,7 +356,10 @@ func (r *mediaFileRepository) GetCursorWithArtwork(options ...model.QueryOptions
// Library-qualified paths search within the specified library, while unqualified paths
// search across all libraries for backward compatibility.
func (r *mediaFileRepository) FindByPaths(paths []string) (model.MediaFiles, error) {
- query := Or{}
+ // One IN list per library instead of one OR term per path: SQLite abandons the
+ // path index at just two OR-ed equality terms and scans the whole table.
+ byLibrary := map[int][]string{}
+ var unqualified []string
for _, path := range paths {
parts := strings.SplitN(path, ":", 2)
@@ -366,17 +370,24 @@ func (r *mediaFileRepository) FindByPaths(paths []string) (model.MediaFiles, err
// Invalid format, skip
continue
}
- relativePath := parts[1]
- query = append(query, And{
- Eq{"path collate nocase": relativePath},
- Eq{"library_id": libraryID},
- })
+ byLibrary[libraryID] = append(byLibrary[libraryID], parts[1])
} else {
// Unqualified path: search across all libraries
- query = append(query, Eq{"path collate nocase": path})
+ unqualified = append(unqualified, path)
}
}
+ query := Or{}
+ for _, libraryID := range slices.Sorted(maps.Keys(byLibrary)) {
+ query = append(query, And{
+ Eq{"path collate nocase": byLibrary[libraryID]},
+ Eq{"library_id": libraryID},
+ })
+ }
+ if len(unqualified) > 0 {
+ query = append(query, Eq{"path collate nocase": unqualified})
+ }
+
if len(query) == 0 {
return model.MediaFiles{}, nil
}
diff --git a/persistence/mediafile_repository_test.go b/persistence/mediafile_repository_test.go
index c1a91c5a5..8a492a813 100644
--- a/persistence/mediafile_repository_test.go
+++ b/persistence/mediafile_repository_test.go
@@ -1055,6 +1055,28 @@ var _ = Describe("MediaRepository", func() {
Expect(results).To(HaveLen(1))
Expect(results[0].ID).To(Equal("otherlib-track"))
})
+
+ It("resolves paths from multiple libraries in a single call", func() {
+ adminMr := NewMediaFileRepository(request.WithUser(GinkgoT().Context(), adminUser), GetDBXBuilder())
+ results, err := adminMr.FindByPaths([]string{
+ "1:artist/Album/track.mp3",
+ fmt.Sprintf("%d:hidden/test.mp3", otherLib.ID),
+ })
+ Expect(err).ToNot(HaveOccurred())
+ Expect(results).To(HaveLen(2))
+ Expect([]string{results[0].ID, results[1].ID}).To(ConsistOf("findpath-1", "otherlib-track"))
+ })
+
+ It("keeps each path scoped to its own library when several are queried", func() {
+ adminMr := NewMediaFileRepository(request.WithUser(GinkgoT().Context(), adminUser), GetDBXBuilder())
+ // Each path exists, but under the other library's ID, so neither must match.
+ results, err := adminMr.FindByPaths([]string{
+ fmt.Sprintf("%d:artist/Album/track.mp3", otherLib.ID),
+ "1:hidden/test.mp3",
+ })
+ Expect(err).ToNot(HaveOccurred())
+ Expect(results).To(BeEmpty())
+ })
})
})