🎧 Your Personal Streaming Service
Go airsonicmadsonicmedia-servermusicmusic-servernavidromeopensubsonicraspberry-piself-hostedstreamerstreaming-apistreaming-audiosubsonicsubsonic-server
Add new Artwork Cache Warmer
Deluanb6eb60f01931
diff --git a/cmd/wire_gen.go b/cmd/wire_gen.goindex 09c5f3614..092117c38 100644--- a/cmd/wire_gen.go+++ b/cmd/wire_gen.go@@ -46,10 +46,10 @@ func CreateSubsonicAPIRouter() *subsonic.Router {sqlDB := db.Db()dataStore := persistence.New(sqlDB)fileCache := core.GetImageCache()- transcoderTranscoder := ffmpeg.New()- artwork := core.NewArtwork(dataStore, fileCache, transcoderTranscoder)+ fFmpeg := ffmpeg.New()+ artwork := core.NewArtwork(dataStore, fileCache, fFmpeg)transcodingCache := core.GetTranscodingCache()- mediaStreamer := core.NewMediaStreamer(dataStore, transcoderTranscoder, transcodingCache)+ mediaStreamer := core.NewMediaStreamer(dataStore, fFmpeg, transcodingCache)archiver := core.NewArchiver(mediaStreamer, dataStore)players := core.NewPlayers(dataStore)agentsAgents := agents.New(dataStore)@@ -80,8 +80,12 @@ func createScanner() scanner.Scanner {sqlDB := db.Db()dataStore := persistence.New(sqlDB)playlists := core.NewPlaylists(dataStore)+ fileCache := core.GetImageCache()+ fFmpeg := ffmpeg.New()+ artwork := core.NewArtwork(dataStore, fileCache, fFmpeg)+ cacheWarmer := core.NewArtworkCacheWarmer(artwork)broker := events.GetBroker()- scannerScanner := scanner.New(dataStore, playlists, broker)+ scannerScanner := scanner.New(dataStore, playlists, cacheWarmer, broker)return scannerScanner}diff --git a/core/artwork_cache_warmer.go b/core/artwork_cache_warmer.gonew file mode 100644index 000000000..ac01e3e47--- /dev/null+++ b/core/artwork_cache_warmer.go@@ -0,0 +1,63 @@+package core++import (+ "context"+ "fmt"+ "io"++ "github.com/navidrome/navidrome/conf"+ "github.com/navidrome/navidrome/log"+ "github.com/navidrome/navidrome/model"+ "github.com/navidrome/navidrome/utils/pl"+)++type ArtworkCacheWarmer interface {+ PreCache(artID model.ArtworkID)+}++func NewArtworkCacheWarmer(artwork Artwork) ArtworkCacheWarmer {+ // If image cache is disabled, return a NOOP implementation+ if conf.Server.ImageCacheSize == "0" {+ return &noopCacheWarmer{}+ }++ a := &artworkCacheWarmer{+ artwork: artwork,+ input: make(chan string),+ }+ go a.run(context.TODO())+ return a+}++type artworkCacheWarmer struct {+ artwork Artwork+ input chan string+}++func (a *artworkCacheWarmer) PreCache(artID model.ArtworkID) {+ a.input <- artID.String()+}++func (a *artworkCacheWarmer) run(ctx context.Context) {+ errs := pl.Sink(ctx, 2, a.input, a.doCacheImage)+ for err := range errs {+ log.Warn(ctx, "Error warming cache", err)+ }+}++func (a *artworkCacheWarmer) doCacheImage(ctx context.Context, id string) error {+ r, err := a.artwork.Get(ctx, id, 0)+ if err != nil {+ return fmt.Errorf("error cacheing id='%s': %w", id, err)+ }+ defer r.Close()+ _, err = io.Copy(io.Discard, r)+ if err != nil {+ return err+ }+ return nil+}++type noopCacheWarmer struct{}++func (a *noopCacheWarmer) PreCache(id model.ArtworkID) {}diff --git a/core/wire_providers.go b/core/wire_providers.goindex 50fef6760..cd0586e22 100644--- a/core/wire_providers.go+++ b/core/wire_providers.go@@ -20,4 +20,5 @@ var Set = wire.NewSet(scrobbler.GetPlayTracker,NewShare,NewPlaylists,+ NewArtworkCacheWarmer,)diff --git a/scanner/metadata/ffmpeg/ffmpeg.go b/scanner/metadata/ffmpeg/ffmpeg.goindex 4ad768b01..bbccf164f 100644--- a/scanner/metadata/ffmpeg/ffmpeg.go+++ b/scanner/metadata/ffmpeg/ffmpeg.go@@ -74,7 +74,7 @@ var (// Stream #0:0: Audio: mp3, 44100 Hz, stereo, fltp, 192 kb/s// Stream #0:0: Audio: flac, 44100 Hz, stereo, s16- audioStreamRx = regexp.MustCompile(`^\s{2,4}Stream #\d+:\d+.*: Audio: (.*), (.* Hz), ([\w\.]+),*(.*.,)*`)+ audioStreamRx = regexp.MustCompile(`^\s{2,4}Stream #\d+:\d+.*: Audio: (.*), (.* Hz), ([\w.]+),*(.*.,)*`)// Stream #0:1: Video: mjpeg, yuvj444p(pc, bt470bg/unknown/unknown), 600x600 [SAR 1:1 DAR 1:1], 90k tbr, 90k tbn, 90k tbc`coverRx = regexp.MustCompile(`^\s{2,4}Stream #\d+:.+: (Video):.*`)diff --git a/scanner/refresher.go b/scanner/refresher.goindex 15d1186af..a3f7821a5 100644--- a/scanner/refresher.go+++ b/scanner/refresher.go@@ -8,6 +8,7 @@ import ("time""github.com/Masterminds/squirrel"+ "github.com/navidrome/navidrome/core""github.com/navidrome/navidrome/log""github.com/navidrome/navidrome/model""github.com/navidrome/navidrome/utils"@@ -20,20 +21,20 @@ import (//// The actual mappings happen in MediaFiles.ToAlbum() and Albums.ToAlbumArtist()type refresher struct {- ctx context.Context- ds model.DataStore- album map[string]struct{}- artist map[string]struct{}- dirMap dirMap+ ds model.DataStore+ album map[string]struct{}+ artist map[string]struct{}+ dirMap dirMap+ cacheWarmer core.ArtworkCacheWarmer}-func newRefresher(ctx context.Context, ds model.DataStore, dirMap dirMap) *refresher {+func newRefresher(ds model.DataStore, cw core.ArtworkCacheWarmer, dirMap dirMap) *refresher {return &refresher{- ctx: ctx,- ds: ds,- album: map[string]struct{}{},- artist: map[string]struct{}{},- dirMap: dirMap,+ ds: ds,+ album: map[string]struct{}{},+ artist: map[string]struct{}{},+ dirMap: dirMap,+ cacheWarmer: cw,}}@@ -46,21 +47,23 @@ func (r *refresher) accumulate(mf model.MediaFile) {}}-func (r *refresher) flush() error {- err := r.flushMap(r.album, "album", r.refreshAlbums)+func (r *refresher) flush(ctx context.Context) error {+ err := r.flushMap(ctx, r.album, "album", r.refreshAlbums)if err != nil {return err}- err = r.flushMap(r.artist, "artist", r.refreshArtists)+ err = r.flushMap(ctx, r.artist, "artist", r.refreshArtists)if err != nil {return err}+ r.album = map[string]struct{}{}+ r.artist = map[string]struct{}{}return nil}-type refreshCallbackFunc = func(ids ...string) error+type refreshCallbackFunc = func(ctx context.Context, ids ...string) error-func (r *refresher) flushMap(m map[string]struct{}, entity string, refresh refreshCallbackFunc) error {+func (r *refresher) flushMap(ctx context.Context, m map[string]struct{}, entity string, refresh refreshCallbackFunc) error {if len(m) == 0 {return nil}@@ -71,17 +74,17 @@ func (r *refresher) flushMap(m map[string]struct{}, entity string, refresh refre}chunks := utils.BreakUpStringSlice(ids, 100)for _, chunk := range chunks {- err := refresh(chunk...)+ err := refresh(ctx, chunk...)if err != nil {- log.Error(r.ctx, fmt.Sprintf("Error writing %ss to the DB", entity), err)+ log.Error(ctx, fmt.Sprintf("Error writing %ss to the DB", entity), err)return err}}return nil}-func (r *refresher) refreshAlbums(ids ...string) error {- mfs, err := r.ds.MediaFile(r.ctx).GetAll(model.QueryOptions{Filters: squirrel.Eq{"album_id": ids}})+func (r *refresher) refreshAlbums(ctx context.Context, ids ...string) error {+ mfs, err := r.ds.MediaFile(ctx).GetAll(model.QueryOptions{Filters: squirrel.Eq{"album_id": ids}})if err != nil {return err}@@ -89,7 +92,7 @@ func (r *refresher) refreshAlbums(ids ...string) error {return nil}- repo := r.ds.Album(r.ctx)+ repo := r.ds.Album(ctx)grouped := slice.Group(mfs, func(m model.MediaFile) string { return m.AlbumID })for _, group := range grouped {songs := model.MediaFiles(group)@@ -103,6 +106,7 @@ func (r *refresher) refreshAlbums(ids ...string) error {if err != nil {return err}+ r.cacheWarmer.PreCache(a.CoverArtID())}return nil}@@ -122,8 +126,8 @@ func (r *refresher) getImageFiles(dirs []string) (string, time.Time) {return strings.Join(imageFiles, string(filepath.ListSeparator)), updatedAt}-func (r *refresher) refreshArtists(ids ...string) error {- albums, err := r.ds.Album(r.ctx).GetAll(model.QueryOptions{Filters: squirrel.Eq{"album_artist_id": ids}})+func (r *refresher) refreshArtists(ctx context.Context, ids ...string) error {+ albums, err := r.ds.Album(ctx).GetAll(model.QueryOptions{Filters: squirrel.Eq{"album_artist_id": ids}})if err != nil {return err}@@ -131,7 +135,7 @@ func (r *refresher) refreshArtists(ids ...string) error {return nil}- repo := r.ds.Artist(r.ctx)+ repo := r.ds.Artist(ctx)grouped := slice.Group(albums, func(al model.Album) string { return al.AlbumArtistID })for _, group := range grouped {a := model.Albums(group).ToAlbumArtist()diff --git a/scanner/scanner.go b/scanner/scanner.goindex 58841f90f..65dc0c55c 100644--- a/scanner/scanner.go+++ b/scanner/scanner.go@@ -40,12 +40,13 @@ type FolderScanner interface {var isScanning sync.Mutextype scanner struct {- folders map[string]FolderScanner- status map[string]*scanStatus- lock *sync.RWMutex- ds model.DataStore- pls core.Playlists- broker events.Broker+ folders map[string]FolderScanner+ status map[string]*scanStatus+ lock *sync.RWMutex+ ds model.DataStore+ pls core.Playlists+ broker events.Broker+ cacheWarmer core.ArtworkCacheWarmer}type scanStatus struct {@@ -55,14 +56,15 @@ type scanStatus struct {lastUpdate time.Time}-func New(ds model.DataStore, playlists core.Playlists, broker events.Broker) Scanner {+func New(ds model.DataStore, playlists core.Playlists, cacheWarmer core.ArtworkCacheWarmer, broker events.Broker) Scanner {s := &scanner{- ds: ds,- pls: playlists,- broker: broker,- folders: map[string]FolderScanner{},- status: map[string]*scanStatus{},- lock: &sync.RWMutex{},+ ds: ds,+ pls: playlists,+ broker: broker,+ folders: map[string]FolderScanner{},+ status: map[string]*scanStatus{},+ lock: &sync.RWMutex{},+ cacheWarmer: cacheWarmer,}s.loadFolders()return s@@ -242,5 +244,5 @@ func (s *scanner) loadFolders() {}func (s *scanner) newScanner(f model.MediaFolder) FolderScanner {- return NewTagScanner(f.Path, s.ds, s.pls)+ return NewTagScanner(f.Path, s.ds, s.pls, s.cacheWarmer)}diff --git a/scanner/tag_scanner.go b/scanner/tag_scanner.goindex 37168b2cc..134a17e50 100644--- a/scanner/tag_scanner.go+++ b/scanner/tag_scanner.go@@ -22,19 +22,23 @@ import ()type TagScanner struct {- rootFolder string- ds model.DataStore- plsSync *playlistImporter- cnt *counters- mapper *mediaFileMapper+ rootFolder string+ ds model.DataStore+ plsSync *playlistImporter+ cnt *counters+ mapper *mediaFileMapper+ cacheWarmer core.ArtworkCacheWarmer}-func NewTagScanner(rootFolder string, ds model.DataStore, playlists core.Playlists) *TagScanner {- return &TagScanner{- rootFolder: rootFolder,- plsSync: newPlaylistImporter(ds, playlists, rootFolder),- ds: ds,+func NewTagScanner(rootFolder string, ds model.DataStore, playlists core.Playlists, cacheWarmer core.ArtworkCacheWarmer) FolderScanner {+ s := &TagScanner{+ rootFolder: rootFolder,+ plsSync: newPlaylistImporter(ds, playlists, rootFolder),+ ds: ds,+ cacheWarmer: cacheWarmer,}++ return s}type dirMap map[string]dirStats@@ -96,6 +100,7 @@ func (s *TagScanner) Scan(ctx context.Context, lastModifiedSince time.Time, progs.cnt = &counters{}genres := newCachedGenreRepository(ctx, s.ds.Genre(ctx))s.mapper = newMediaFileMapper(s.rootFolder, genres)+ refresher := newRefresher(s.ds, s.cacheWarmer, allFSDirs)foldersFound, walkerError := s.getRootFolderWalker(ctx)for {@@ -109,7 +114,7 @@ func (s *TagScanner) Scan(ctx context.Context, lastModifiedSince time.Time, progif s.folderHasChanged(folderStats, allDBDirs, lastModifiedSince) {changedDirs = append(changedDirs, folderStats.Path)log.Debug("Processing changed folder", "dir", folderStats.Path)- err := s.processChangedDir(ctx, allFSDirs, folderStats.Path, fullScan)+ err := s.processChangedDir(ctx, refresher, fullScan, folderStats.Path)if err != nil {log.Error("Error updating folder in the DB", "dir", folderStats.Path, err)}@@ -128,7 +133,7 @@ func (s *TagScanner) Scan(ctx context.Context, lastModifiedSince time.Time, prog}for _, dir := range deletedDirs {- err := s.processDeletedDir(ctx, allFSDirs, dir)+ err := s.processDeletedDir(ctx, refresher, dir)if err != nil {log.Error("Error removing deleted folder from DB", "dir", dir, err)}@@ -221,9 +226,8 @@ func (s *TagScanner) getDeletedDirs(ctx context.Context, fsDirs dirMap, dbDirs mreturn deleted}-func (s *TagScanner) processDeletedDir(ctx context.Context, allFSDirs dirMap, dir string) error {+func (s *TagScanner) processDeletedDir(ctx context.Context, refresher *refresher, dir string) error {start := time.Now()- buffer := newRefresher(ctx, s.ds, allFSDirs)mfs, err := s.ds.MediaFile(ctx).FindAllByPath(dir)if err != nil {@@ -237,17 +241,16 @@ func (s *TagScanner) processDeletedDir(ctx context.Context, allFSDirs dirMap, dis.cnt.deleted += cfor _, t := range mfs {- buffer.accumulate(t)+ refresher.accumulate(t)}- err = buffer.flush()+ err = refresher.flush(ctx)log.Info(ctx, "Finished processing deleted folder", "dir", dir, "purged", len(mfs), "elapsed", time.Since(start))return err}-func (s *TagScanner) processChangedDir(ctx context.Context, allFSDirs dirMap, dir string, fullScan bool) error {+func (s *TagScanner) processChangedDir(ctx context.Context, refresher *refresher, fullScan bool, dir string) error {start := time.Now()- buffer := newRefresher(ctx, s.ds, allFSDirs)// Load folder's current tracks from DB into a mapcurrentTracks := map[string]model.MediaFile{}@@ -296,7 +299,7 @@ func (s *TagScanner) processChangedDir(ctx context.Context, allFSDirs dirMap, di}// Force a refresh of the album and artist, to cater for cover art files- buffer.accumulate(c)+ refresher.accumulate(c)// Only leaves in orphanTracks the ones not found in the folder. After this loop any remaining orphanTracks// are considered gone from the music folder and will be deleted from DB@@ -307,33 +310,38 @@ func (s *TagScanner) processChangedDir(ctx context.Context, allFSDirs dirMap, dinumPurgedTracks := 0if len(filesToUpdate) > 0 {- numUpdatedTracks, err = s.addOrUpdateTracksInDB(ctx, dir, currentTracks, filesToUpdate, buffer)+ numUpdatedTracks, err = s.addOrUpdateTracksInDB(ctx, refresher, dir, currentTracks, filesToUpdate)if err != nil {return err}}if len(orphanTracks) > 0 {- numPurgedTracks, err = s.deleteOrphanSongs(ctx, dir, orphanTracks, buffer)+ numPurgedTracks, err = s.deleteOrphanSongs(ctx, refresher, dir, orphanTracks)if err != nil {return err}}- err = buffer.flush()+ err = refresher.flush(ctx)log.Info(ctx, "Finished processing changed folder", "dir", dir, "updated", numUpdatedTracks,"deleted", numPurgedTracks, "elapsed", time.Since(start))return err}-func (s *TagScanner) deleteOrphanSongs(ctx context.Context, dir string, tracksToDelete map[string]model.MediaFile, buffer *refresher) (int, error) {+func (s *TagScanner) deleteOrphanSongs(+ ctx context.Context,+ refresher *refresher,+ dir string,+ tracksToDelete map[string]model.MediaFile,+) (int, error) {numPurgedTracks := 0log.Debug(ctx, "Deleting orphan tracks from DB", "dir", dir, "numTracks", len(tracksToDelete))// Remaining tracks from DB that are not in the folder are deletedfor _, ct := range tracksToDelete {numPurgedTracks++- buffer.accumulate(ct)+ refresher.accumulate(ct)if err := s.ds.MediaFile(ctx).Delete(ct.ID); err != nil {return 0, err}@@ -342,7 +350,13 @@ func (s *TagScanner) deleteOrphanSongs(ctx context.Context, dir string, tracksToreturn numPurgedTracks, nil}-func (s *TagScanner) addOrUpdateTracksInDB(ctx context.Context, dir string, currentTracks map[string]model.MediaFile, filesToUpdate []string, buffer *refresher) (int, error) {+func (s *TagScanner) addOrUpdateTracksInDB(+ ctx context.Context,+ refresher *refresher,+ dir string,+ currentTracks map[string]model.MediaFile,+ filesToUpdate []string,+) (int, error) {numUpdatedTracks := 0log.Trace(ctx, "Updating mediaFiles in DB", "dir", dir, "numFiles", len(filesToUpdate))@@ -367,7 +381,7 @@ func (s *TagScanner) addOrUpdateTracksInDB(ctx context.Context, dir string, currif err != nil {return 0, err}- buffer.accumulate(n)+ refresher.accumulate(n)numUpdatedTracks++}}diff --git a/utils/pl/pipelines.go b/utils/pl/pipelines.gonew file mode 100644index 000000000..ed85c6b94--- /dev/null+++ b/utils/pl/pipelines.go@@ -0,0 +1,176 @@+// Package pl implements some Data Pipeline helper functions.+// Reference: https://medium.com/amboss/applying-modern-go-concurrency-patterns-to-data-pipelines-b3b5327908d4#3a80+//+// See also:+//+// https://www.oreilly.com/library/view/concurrency-in-go/9781491941294/ch04.html#fano_fani+// https://www.youtube.com/watch?v=f6kdp27TYZs+// https://www.youtube.com/watch?v=QDDwwePbDtw+package pl++import (+ "context"+ "errors"+ "sync"++ "github.com/navidrome/navidrome/log"+ "golang.org/x/sync/semaphore"+)++func Stage[In any, Out any](+ ctx context.Context,+ maxWorkers int,+ inputChannel <-chan In,+ fn func(context.Context, In) (Out, error),+) (chan Out, chan error) {+ outputChannel := make(chan Out)+ errorChannel := make(chan error)++ limit := int64(maxWorkers)+ sem1 := semaphore.NewWeighted(limit)++ go func() {+ defer close(outputChannel)+ defer close(errorChannel)++ for s := range ReadOrDone(ctx, inputChannel) {+ if err := sem1.Acquire(ctx, 1); err != nil {+ if !errors.Is(err, context.Canceled) {+ log.Error(ctx, "Failed to acquire semaphore", err)+ }+ break+ }++ go func(s In) {+ defer sem1.Release(1)++ result, err := fn(ctx, s)+ if err != nil {+ if !errors.Is(err, context.Canceled) {+ errorChannel <- err+ }+ } else {+ outputChannel <- result+ }+ }(s)+ }++ // By using context.Background() here we are assuming the fn will stop when the context+ // is canceled. This is required so we can wait for the workers to finish and avoid closing+ // the outputChannel before they are done.+ if err := sem1.Acquire(context.Background(), limit); err != nil {+ log.Error(ctx, "Failed waiting for workers", err)+ }+ }()++ return outputChannel, errorChannel+}++func Sink[In any](+ ctx context.Context,+ maxWorkers int,+ inputChannel <-chan In,+ fn func(context.Context, In) error,+) chan error {+ results, errC := Stage(ctx, maxWorkers, inputChannel, func(ctx context.Context, in In) (bool, error) {+ err := fn(ctx, in)+ return false, err // Only err is important, results will be discarded+ })++ // Discard results+ go func() {+ for range ReadOrDone(ctx, results) {+ }+ }()++ return errC+}++func Merge[T any](ctx context.Context, cs ...<-chan T) <-chan T {+ var wg sync.WaitGroup+ out := make(chan T)++ output := func(c <-chan T) {+ defer wg.Done()+ for v := range ReadOrDone(ctx, c) {+ select {+ case out <- v:+ case <-ctx.Done():+ return+ }+ }+ }++ wg.Add(len(cs))+ for _, c := range cs {+ go output(c)+ }++ go func() {+ wg.Wait()+ close(out)+ }()++ return out+}++func SendOrDone[T any](ctx context.Context, out chan<- T, v T) {+ select {+ case out <- v:+ case <-ctx.Done():+ return+ }+}++func ReadOrDone[T any](ctx context.Context, in <-chan T) <-chan T {+ valStream := make(chan T)+ go func() {+ defer close(valStream)+ for {+ select {+ case <-ctx.Done():+ return+ case v, ok := <-in:+ if !ok {+ return+ }+ select {+ case valStream <- v:+ case <-ctx.Done():+ }+ }+ }+ }()+ return valStream+}++func Tee[T any](ctx context.Context, in <-chan T) (<-chan T, <-chan T) {+ out1 := make(chan T)+ out2 := make(chan T)+ go func() {+ defer close(out1)+ defer close(out2)+ for val := range ReadOrDone(ctx, in) {+ var out1, out2 = out1, out2+ for i := 0; i < 2; i++ {+ select {+ case <-ctx.Done():+ case out1 <- val:+ out1 = nil+ case out2 <- val:+ out2 = nil+ }+ }+ }+ }()+ return out1, out2+}++func FromSlice[T any](ctx context.Context, in []T) <-chan T {+ output := make(chan T, len(in))+ for _, c := range in {+ output <- c+ }+ close(output)+ return output+}diff --git a/utils/pl/pipelines_test.go b/utils/pl/pipelines_test.gonew file mode 100644index 000000000..f5da6e49f--- /dev/null+++ b/utils/pl/pipelines_test.go@@ -0,0 +1,168 @@+package pl_test++import (+ "context"+ "errors"+ "sync/atomic"+ "testing"+ "time"++ "github.com/navidrome/navidrome/utils/pl"+ . "github.com/onsi/ginkgo/v2"+ . "github.com/onsi/gomega"+)++func TestPipeline(t *testing.T) {+ RegisterFailHandler(Fail)+ RunSpecs(t, "Pipeline Tests Suite")+}++var _ = Describe("Pipeline", func() {+ Describe("Stage", func() {+ Context("happy path", func() {+ It("calls the 'transform' function and returns values and errors", func() {+ inC := make(chan int, 4)+ for i := 0; i < 4; i++ {+ inC <- i+ }+ close(inC)++ outC, errC := pl.Stage(context.Background(), 1, inC, func(ctx context.Context, i int) (int, error) {+ if i%2 == 0 {+ return 0, errors.New("even number")+ }+ return i * 2, nil+ })++ Expect(<-errC).To(MatchError("even number"))+ Expect(<-outC).To(Equal(2))+ Expect(<-errC).To(MatchError("even number"))+ Expect(<-outC).To(Equal(6))++ Eventually(outC).Should(BeClosed())+ Eventually(errC).Should(BeClosed())+ })+ })+ Context("Multiple workers", func() {+ const maxWorkers = 2+ const numJobs = 100+ It("starts multiple workers, respecting the limit", func() {+ inC := make(chan int, numJobs)+ for i := 0; i < numJobs; i++ {+ inC <- i+ }+ close(inC)++ current := atomic.Int32{}+ count := atomic.Int32{}+ max := atomic.Int32{}+ outC, _ := pl.Stage(context.Background(), maxWorkers, inC, func(ctx context.Context, in int) (int, error) {+ defer current.Add(-1)+ c := current.Add(1)+ count.Add(1)+ if c > max.Load() {+ max.Store(c)+ }+ time.Sleep(10 * time.Millisecond) // Slow process+ return 0, nil+ })+ // Discard output and wait for completion+ for range outC {+ }++ Expect(count.Load()).To(Equal(int32(numJobs)))+ Expect(current.Load()).To(Equal(int32(0)))+ Expect(max.Load()).To(Equal(int32(maxWorkers)))+ })+ })+ When("the context is canceled", func() {+ It("closes its output", func() {+ ctx, cancel := context.WithCancel(context.Background())+ inC := make(chan int)+ outC, errC := pl.Stage(ctx, 1, inC, func(ctx context.Context, i int) (int, error) {+ return i, nil+ })+ cancel()+ Eventually(outC).Should(BeClosed())+ Eventually(errC).Should(BeClosed())+ })+ })++ })+ Describe("Merge", func() {+ var in1, in2 chan int+ BeforeEach(func() {+ in1 = make(chan int, 4)+ in2 = make(chan int, 4)+ for i := 0; i < 4; i++ {+ in1 <- i+ in2 <- i + 4+ }+ close(in1)+ close(in2)+ })+ When("ranging through the output channel", func() {+ It("copies values from all input channels to its output channel", func() {+ var values []int+ for v := range pl.Merge(context.Background(), in1, in2) {+ values = append(values, v)+ }++ Expect(values).To(ConsistOf(0, 1, 2, 3, 4, 5, 6, 7))+ })+ })+ When("there's a blocked channel and the context is closed", func() {+ It("closes its output", func() {+ ctx, cancel := context.WithCancel(context.Background())+ in3 := make(chan int)+ out := pl.Merge(ctx, in1, in2, in3)+ cancel()+ Eventually(out).Should(BeClosed())+ })+ })+ })+ Describe("ReadOrDone", func() {+ When("values are sent", func() {+ It("copies them to its output channel", func() {+ in := make(chan int)+ out := pl.ReadOrDone(context.Background(), in)+ for i := 0; i < 4; i++ {+ in <- i+ j := <-out+ Expect(i).To(Equal(j))+ }+ close(in)+ Eventually(out).Should(BeClosed())+ })+ })+ When("the context is canceled", func() {+ It("closes its output", func() {+ ctx, cancel := context.WithCancel(context.Background())+ in := make(chan int)+ out := pl.ReadOrDone(ctx, in)+ cancel()+ Eventually(out).Should(BeClosed())+ })+ })+ })+ Describe("SendOrDone", func() {+ When("out is unblocked", func() {+ It("puts the value in the channel", func() {+ out := make(chan int)+ value := 1234+ go pl.SendOrDone(context.Background(), out, value)+ Eventually(out).Should(Receive(&value))+ })+ })+ When("out is blocked", func() {+ It("can be canceled by the context", func() {+ ctx, cancel := context.WithCancel(context.Background())+ out := make(chan int)+ go pl.SendOrDone(ctx, out, 1234)+ cancel()++ Consistently(out).ShouldNot(Receive())+ })+ })+ })+})diff --git a/utils/pool/pool.go b/utils/pool/pool.godeleted file mode 100644index 09ceaeffd..000000000--- a/utils/pool/pool.go+++ /dev/null@@ -1,90 +0,0 @@-package pool--import (- "time"-- "github.com/navidrome/navidrome/log"-)--type Executor func(workload interface{})--type Pool struct {- name string- workers []worker- exec Executor- queue chan work // receives jobs to send to workers- done chan bool // when receives bool stops workers- working bool-}--// TODO This hardcoded value will go away when the queue is persisted in disk-const bufferSize = 10000--func NewPool(name string, workerCount int, exec Executor) (*Pool, error) {- p := &Pool{- name: name,- exec: exec,- queue: make(chan work, bufferSize),- done: make(chan bool),- working: false,- }-- for i := 0; i < workerCount; i++ {- worker := worker{- p: p,- id: i,- }- worker.Start()- p.workers = append(p.workers, worker)- }-- go func() {- ticker := time.NewTicker(10 * time.Second)- defer ticker.Stop()- for {- select {- case <-ticker.C:- if len(p.queue) > 0 {- log.Debug("Queue status", "poolName", p.name, "items", len(p.queue))- } else {- if p.working {- log.Info("Queue is empty, all items processed", "poolName", p.name)- }- p.working = false- }- case <-p.done:- close(p.queue)- return- }- }- }()-- return p, nil-}--func (p *Pool) Submit(workload interface{}) {- p.working = true- p.queue <- work{workload}-}--func (p *Pool) Stop() {- p.done <- true-}--type work struct {- workload interface{}-}--type worker struct {- id int- p *Pool-}--// start worker-func (w *worker) Start() {- go func() {- for job := range w.p.queue {- w.p.exec(job.workload) // do work- }- }()-}diff --git a/utils/pool/pool_test.go b/utils/pool/pool_test.godeleted file mode 100644index c6c59e7ea..000000000--- a/utils/pool/pool_test.go+++ /dev/null@@ -1,56 +0,0 @@-package pool--import (- "sync"- "testing"-- "github.com/navidrome/navidrome/log"- "github.com/navidrome/navidrome/tests"- . "github.com/onsi/ginkgo/v2"- . "github.com/onsi/gomega"-)--func TestPool(t *testing.T) {- tests.Init(t, false)- log.SetLevel(log.LevelFatal)- RegisterFailHandler(Fail)- RunSpecs(t, "Pool Suite")-}--type testItem struct {- ID int-}--var (- processed []int- mutex sync.RWMutex-)--var _ = Describe("Pool", func() {- var pool *Pool-- BeforeEach(func() {- processed = nil- pool, _ = NewPool("test", 2, execute)- })-- It("processes items", func() {- for i := 0; i < 5; i++ {- pool.Submit(&testItem{ID: i})- }- Eventually(func() []int {- mutex.RLock()- defer mutex.RUnlock()- return processed- }, "10s").Should(HaveLen(5))-- Expect(processed).To(ContainElements(0, 1, 2, 3, 4))- })-})--func execute(workload interface{}) {- mutex.Lock()- defer mutex.Unlock()- item := workload.(*testItem)- processed = append(processed, item.ID)-}