🎧 Your Personal Streaming Service
Go airsonicmadsonicmedia-servermusicmusic-servernavidromeopensubsonicraspberry-piself-hostedstreamerstreaming-apistreaming-audiosubsonicsubsonic-server
Generalize BreakUp/RangByChunks functions
Deluan9aa7b80d0d78
diff --git a/persistence/playlist_repository.go b/persistence/playlist_repository.goindex fe1ecd5e7..f2addc150 100644--- a/persistence/playlist_repository.go+++ b/persistence/playlist_repository.go@@ -13,7 +13,7 @@ import ("github.com/navidrome/navidrome/log""github.com/navidrome/navidrome/model""github.com/navidrome/navidrome/model/criteria"- "github.com/navidrome/navidrome/utils"+ "github.com/navidrome/navidrome/utils/slice")type playlistRepository struct {@@ -297,7 +297,7 @@ func (r *playlistRepository) updatePlaylist(playlistId string, mediaFileIds []stfunc (r *playlistRepository) addTracks(playlistId string, startingPos int, mediaFileIds []string) error {// Break the track list in chunks to avoid hitting SQLITE_MAX_FUNCTION_ARG limit- chunks := utils.BreakUpStringSlice(mediaFileIds, 200)+ chunks := slice.BreakUp(mediaFileIds, 200)// Add new tracks, chunk by chunkpos := startingPosdiff --git a/persistence/playqueue_repository.go b/persistence/playqueue_repository.goindex 14bd1b9bd..beda2a832 100644--- a/persistence/playqueue_repository.go+++ b/persistence/playqueue_repository.go@@ -9,7 +9,7 @@ import ("github.com/beego/beego/v2/client/orm""github.com/navidrome/navidrome/log""github.com/navidrome/navidrome/model"- "github.com/navidrome/navidrome/utils"+ "github.com/navidrome/navidrome/utils/slice")type playQueueRepository struct {@@ -113,7 +113,7 @@ func (r *playQueueRepository) loadTracks(tracks model.MediaFiles) model.MediaFil}// Break the list in chunks, up to 50 items, to avoid hitting SQLITE_MAX_FUNCTION_ARG limit- chunks := utils.BreakUpStringSlice(ids, 50)+ chunks := slice.BreakUp(ids, 50)// Query each chunk of media_file ids and store results in a mapmfRepo := NewMediaFileRepository(r.ctx, r.ormer)diff --git a/persistence/sql_genres.go b/persistence/sql_genres.goindex 397e43368..be245721a 100644--- a/persistence/sql_genres.go+++ b/persistence/sql_genres.go@@ -3,7 +3,7 @@ package persistenceimport (. "github.com/Masterminds/squirrel""github.com/navidrome/navidrome/model"- "github.com/navidrome/navidrome/utils"+ "github.com/navidrome/navidrome/utils/slice")func (r sqlRepository) withGenres(sql SelectBuilder) SelectBuilder {@@ -25,7 +25,7 @@ func (r *sqlRepository) updateGenres(id string, tableName string, genres model.Gfor _, g := range genres {genreIds = append(genreIds, g.ID)}- err = utils.RangeByChunks(genreIds, 100, func(ids []string) error {+ err = slice.RangeByChunks(genreIds, 100, func(ids []string) error {ins := Insert(tableName+"_genres").Columns("genre_id", tableName+"_id")for _, gid := range ids {ins = ins.Values(gid, id)@@ -45,7 +45,7 @@ func (r *sqlRepository) loadMediaFileGenres(mfs *model.MediaFiles) error {m[mf.ID] = mf}- return utils.RangeByChunks(ids, 900, func(ids []string) error {+ return slice.RangeByChunks(ids, 900, func(ids []string) error {sql := Select("g.*", "mg.media_file_id").From("genre g").Join("media_file_genres mg on mg.genre_id = g.id").Where(Eq{"mg.media_file_id": ids}).OrderBy("mg.media_file_id", "mg.rowid")var genres []struct {@@ -74,7 +74,7 @@ func (r *sqlRepository) loadAlbumGenres(mfs *model.Albums) error {m[mf.ID] = mf}- return utils.RangeByChunks(ids, 900, func(ids []string) error {+ return slice.RangeByChunks(ids, 900, func(ids []string) error {sql := Select("g.*", "ag.album_id").From("genre g").Join("album_genres ag on ag.genre_id = g.id").Where(Eq{"ag.album_id": ids}).OrderBy("ag.album_id", "ag.rowid")var genres []struct {@@ -103,7 +103,7 @@ func (r *sqlRepository) loadArtistGenres(mfs *model.Artists) error {m[mf.ID] = mf}- return utils.RangeByChunks(ids, 900, func(ids []string) error {+ return slice.RangeByChunks(ids, 900, func(ids []string) error {sql := Select("g.*", "ag.artist_id").From("genre g").Join("artist_genres ag on ag.genre_id = g.id").Where(Eq{"ag.artist_id": ids}).OrderBy("ag.artist_id", "ag.rowid")var genres []struct {diff --git a/scanner/refresher.go b/scanner/refresher.goindex ec17fc1e6..3a35e8fea 100644--- a/scanner/refresher.go+++ b/scanner/refresher.go@@ -12,7 +12,6 @@ import ("github.com/navidrome/navidrome/core/artwork""github.com/navidrome/navidrome/log""github.com/navidrome/navidrome/model"- "github.com/navidrome/navidrome/utils""github.com/navidrome/navidrome/utils/slice""golang.org/x/exp/maps")@@ -71,7 +70,7 @@ func (r *refresher) flushMap(ctx context.Context, m map[string]struct{}, entity}ids := maps.Keys(m)- chunks := utils.BreakUpStringSlice(ids, 100)+ chunks := slice.BreakUp(ids, 100)for _, chunk := range chunks {err := refresh(ctx, chunk...)if err != nil {diff --git a/scanner/tag_scanner.go b/scanner/tag_scanner.goindex 1317581bc..5a719acd4 100644--- a/scanner/tag_scanner.go+++ b/scanner/tag_scanner.go@@ -19,7 +19,7 @@ import ("github.com/navidrome/navidrome/scanner/metadata"_ "github.com/navidrome/navidrome/scanner/metadata/ffmpeg"_ "github.com/navidrome/navidrome/scanner/metadata/taglib"- "github.com/navidrome/navidrome/utils"+ "github.com/navidrome/navidrome/utils/slice")type TagScanner struct {@@ -351,7 +351,7 @@ func (s *TagScanner) addOrUpdateTracksInDB(log.Trace(ctx, "Updating mediaFiles in DB", "dir", dir, "numFiles", len(filesToUpdate))// Break the file list in chunks to avoid calling ffmpeg with too many parameters- chunks := utils.BreakUpStringSlice(filesToUpdate, filesBatchSize)+ chunks := slice.BreakUp(filesToUpdate, filesBatchSize)for _, chunk := range chunks {// Load tracks Metadata from the foldernewTracks, err := s.loadTracks(chunk)diff --git a/utils/slice/slice.go b/utils/slice/slice.goindex cd012330b..78169685f 100644--- a/utils/slice/slice.go+++ b/utils/slice/slice.go@@ -54,3 +54,28 @@ func Move[T any](slice []T, srcIndex int, dstIndex int) []T {value := slice[srcIndex]return Insert(Remove(slice, srcIndex), value, dstIndex)}++func BreakUp[T any](items []T, chunkSize int) [][]T {+ numTracks := len(items)+ var chunks [][]T+ for i := 0; i < numTracks; i += chunkSize {+ end := i + chunkSize+ if end > numTracks {+ end = numTracks+ }++ chunks = append(chunks, items[i:end])+ }+ return chunks+}++func RangeByChunks[T any](items []T, chunkSize int, cb func([]T) error) error {+ chunks := BreakUp(items, chunkSize)+ for _, chunk := range chunks {+ err := cb(chunk)+ if err != nil {+ return err+ }+ }+ return nil+}diff --git a/utils/slice/slice_test.go b/utils/slice/slice_test.goindex a27416f3d..7436a34ee 100644--- a/utils/slice/slice_test.go+++ b/utils/slice/slice_test.go@@ -69,4 +69,25 @@ var _ = Describe("Slice Utils", func() {Expect(slice.Move([]string{"1", "2", "3"}, 1, 1)).To(ConsistOf("1", "2", "3"))})})++ Describe("BreakUp", func() {+ It("returns no chunks if slice is empty", func() {+ var s []string+ chunks := slice.BreakUp(s, 10)+ Expect(chunks).To(HaveLen(0))+ })+ It("returns the slice in one chunk if len < chunkSize", func() {+ s := []string{"a", "b", "c"}+ chunks := slice.BreakUp(s, 10)+ Expect(chunks).To(HaveLen(1))+ Expect(chunks[0]).To(ConsistOf("a", "b", "c"))+ })+ It("breaks up the slice if len > chunkSize", func() {+ s := []string{"a", "b", "c", "d", "e"}+ chunks := slice.BreakUp(s, 3)+ Expect(chunks).To(HaveLen(2))+ Expect(chunks[0]).To(ConsistOf("a", "b", "c"))+ Expect(chunks[1]).To(ConsistOf("d", "e"))+ })+ })})diff --git a/utils/strings.go b/utils/strings.goindex 9fbe0d448..1bd26b10a 100644--- a/utils/strings.go+++ b/utils/strings.go@@ -17,31 +17,6 @@ func NoArticle(name string) string {return name}-func BreakUpStringSlice(items []string, chunkSize int) [][]string {- numTracks := len(items)- var chunks [][]string- for i := 0; i < numTracks; i += chunkSize {- end := i + chunkSize- if end > numTracks {- end = numTracks- }-- chunks = append(chunks, items[i:end])- }- return chunks-}--func RangeByChunks(items []string, chunkSize int, cb func([]string) error) error {- chunks := BreakUpStringSlice(items, chunkSize)- for _, chunk := range chunks {- err := cb(chunk)- if err != nil {- return err- }- }- return nil-}-func LongestCommonPrefix(list []string) string {if len(list) == 0 {return ""diff --git a/utils/strings_test.go b/utils/strings_test.goindex 0983faa29..4b24dab0c 100644--- a/utils/strings_test.go+++ b/utils/strings_test.go@@ -35,27 +35,6 @@ var _ = Describe("Strings", func() {})})- Describe("BreakUpStringSlice", func() {- It("returns no chunks if slice is empty", func() {- var slice []string- chunks := BreakUpStringSlice(slice, 10)- Expect(chunks).To(HaveLen(0))- })- It("returns the slice in one chunk if len < chunkSize", func() {- slice := []string{"a", "b", "c"}- chunks := BreakUpStringSlice(slice, 10)- Expect(chunks).To(HaveLen(1))- Expect(chunks[0]).To(ConsistOf("a", "b", "c"))- })- It("breaks up the slice if len > chunkSize", func() {- slice := []string{"a", "b", "c", "d", "e"}- chunks := BreakUpStringSlice(slice, 3)- Expect(chunks).To(HaveLen(2))- Expect(chunks[0]).To(ConsistOf("a", "b", "c"))- Expect(chunks[1]).To(ConsistOf("d", "e"))- })- })-Describe("LongestCommonPrefix", func() {It("finds the longest common prefix", func() {Expect(LongestCommonPrefix(testPaths)).To(Equal("/Music/iTunes 1/iTunes Media/Music/"))