🎧 Your Personal Streaming Service
Go airsonicmadsonicmedia-servermusicmusic-servernavidromeopensubsonicraspberry-piself-hostedstreamerstreaming-apistreaming-audiosubsonicsubsonic-server
fix(scanner): import playlists skipped when no admin existed yet (#5609)
Deluan Quintão2c90685bc29a
diff --git a/consts/consts.go b/consts/consts.goindex edd8f2b54..4baf4610d 100644--- a/consts/consts.go+++ b/consts/consts.go@@ -14,6 +14,9 @@ const (DefaultDbPath = "navidrome.db?cache=shared&_busy_timeout=15000&_journal_mode=WAL&_foreign_keys=on&synchronous=normal"InitialSetupFlagKey = "InitialSetup"FullScanAfterMigrationFlagKey = "FullScanAfterMigration"+ // PlaylistsImportPendingFlagKey marks that playlist import was deferred because+ // no admin user existed yet; the next scan with an admin imports them.+ PlaylistsImportPendingFlagKey = "PlaylistsImportPending"LastScanErrorKey = "LastScanError"LastScanTypeKey = "LastScanType"LastScanStartTimeKey = "LastScanStartTime"diff --git a/model/folder.go b/model/folder.goindex 39cb6db84..81800c072 100644--- a/model/folder.go+++ b/model/folder.go@@ -93,4 +93,7 @@ type FolderRepository interface {Put(*Folder) errorMarkMissing(missing bool, ids ...string) errorGetTouchedWithPlaylists() (FolderCursor, error)+ // GetAllWithPlaylists returns all non-missing folders with playlists, ignoring+ // the scan-timestamp gate used by GetTouchedWithPlaylists.+ GetAllWithPlaylists() (FolderCursor, error)}diff --git a/persistence/folder_repository.go b/persistence/folder_repository.goindex 624ef21b8..8fb7f0296 100644--- a/persistence/folder_repository.go+++ b/persistence/folder_repository.go@@ -250,6 +250,18 @@ func (r folderRepository) GetTouchedWithPlaylists() (model.FolderCursor, error)return wrapFolderCursor(cursor), nil}+func (r folderRepository) GetAllWithPlaylists() (model.FolderCursor, error) {+ query := r.selectFolder().Where(And{+ Eq{"missing": false},+ Gt{"num_playlists": 0},+ })+ cursor, err := queryWithStableResults[dbFolder](r.sqlRepository, query)+ if err != nil {+ return nil, err+ }+ return wrapFolderCursor(cursor), nil+}+func wrapFolderCursor(cursor iter.Seq2[dbFolder, error]) model.FolderCursor {return func(yield func(model.Folder, error) bool) {for f, err := range cursor {diff --git a/persistence/folder_repository_test.go b/persistence/folder_repository_test.goindex 413a6b38f..a8945dfee 100644--- a/persistence/folder_repository_test.go+++ b/persistence/folder_repository_test.go@@ -317,4 +317,36 @@ var _ = Describe("FolderRepository", func() {Expect(folders[0].ID).To(Equal("f1"))})})++ Describe("GetAllWithPlaylists", func() {+ It("returns all non-missing folders with playlists, ignoring the scan-timestamp gate", func() {+ withPls := model.NewFolder(testLib, "TestAllPls/WithPls")+ withPls.NumPlaylists = 2+ noPls := model.NewFolder(testLib, "TestAllPls/NoPls")+ noPls.NumPlaylists = 0+ missingWithPls := model.NewFolder(testLib, "TestAllPls/Missing")+ missingWithPls.NumPlaylists = 1+ missingWithPls.Missing = true++ Expect(repo.Put(withPls)).To(Succeed())+ Expect(repo.Put(noPls)).To(Succeed())+ Expect(repo.Put(missingWithPls)).To(Succeed())++ // Force the folder's updated_at to the past so GetTouchedWithPlaylists+ // (which gates on updated_at > last_scan_at) would NOT return it.+ _, err := conn.NewQuery("UPDATE folder SET updated_at = {:t} WHERE id = {:id}").+ Bind(dbx.Params{"t": "2000-01-01 00:00:00", "id": withPls.ID}).Execute()+ Expect(err).ToNot(HaveOccurred())++ var ids []string+ cursor, err := repo.GetAllWithPlaylists()+ Expect(err).ToNot(HaveOccurred())+ for f, err := range cursor {+ Expect(err).ToNot(HaveOccurred())+ ids = append(ids, f.ID)+ }++ Expect(ids).To(ConsistOf(withPls.ID)) // only the non-missing folder with playlists+ })+ })})diff --git a/scanner/phase_4_playlists.go b/scanner/phase_4_playlists.goindex d52743966..8ba014235 100644--- a/scanner/phase_4_playlists.go+++ b/scanner/phase_4_playlists.go@@ -2,6 +2,7 @@ package scannerimport ("context"+ "errors""fmt""os""strings"@@ -10,6 +11,7 @@ import (ppl "github.com/google/go-pipeline/pkg/pipeline""github.com/navidrome/navidrome/conf"+ "github.com/navidrome/navidrome/consts""github.com/navidrome/navidrome/core/artwork""github.com/navidrome/navidrome/core/playlists""github.com/navidrome/navidrome/log"@@ -18,12 +20,13 @@ import ()type phasePlaylists struct {- ctx context.Context- scanState *scanState- ds model.DataStore- pls playlists.Playlists- cw artwork.CacheWarmer- refreshed atomic.Uint32+ ctx context.Context+ scanState *scanState+ ds model.DataStore+ pls playlists.Playlists+ cw artwork.CacheWarmer+ refreshed atomic.Uint32+ pendingImport bool}func createPhasePlaylists(ctx context.Context, scanState *scanState, ds model.DataStore, pls playlists.Playlists, cw artwork.CacheWarmer) *phasePlaylists {@@ -49,22 +52,41 @@ func (p *phasePlaylists) produce(put func(entry *model.Folder)) error {log.Info(p.ctx, "Playlists will not be imported, AutoImportPlaylists is set to false")return nil}- u, _ := request.UserFrom(p.ctx)- if !u.IsAdmin || u.ID == "" {- log.Warn(p.ctx, "Playlists will not be imported, as there are no admin users yet, "+- "Please create an admin user first, and then update the playlists for them to be imported")- return nil++ // Resolve the admin at phase time (the producer runs late in the scan), so an+ // admin created while the scan was in progress is picked up. Assigned once,+ // before any put() below, so the channel send synchronizes it with the stages.+ admin, err := p.ds.User(p.ctx).FindFirstAdmin()+ if err != nil && !errors.Is(err, model.ErrNotFound) {+ return fmt.Errorf("finding admin user: %w", err)+ }+ noAdmin := admin == nil || admin.ID == ""+ if noAdmin {+ return p.deferImport()}+ p.ctx = request.WithUser(p.ctx, *admin)- count := 0- cursor, err := p.ds.Folder(p.ctx).GetTouchedWithPlaylists()+ // When recovering a deferred import, scan all playlist folders, not just touched ones.+ pending, err := p.importPending()if err != nil {- return fmt.Errorf("loading touched folders: %w", err)+ return fmt.Errorf("checking pending playlist import: %w", err)}- log.Debug(p.ctx, "Scanner: Checking playlists that may need refresh")+ p.pendingImport = pending+ var cursor model.FolderCursor+ if p.pendingImport {+ cursor, err = p.ds.Folder(p.ctx).GetAllWithPlaylists()+ } else {+ cursor, err = p.ds.Folder(p.ctx).GetTouchedWithPlaylists()+ }+ if err != nil {+ return fmt.Errorf("loading folders with playlists: %w", err)+ }++ count := 0+ log.Debug(p.ctx, "Scanner: Checking playlists that may need refresh", "pendingImport", p.pendingImport)for folder, err := range cursor {if err != nil {- return fmt.Errorf("loading touched folder: %w", err)+ return fmt.Errorf("loading folder with playlists: %w", err)}count++put(&folder)@@ -78,6 +100,23 @@ func (p *phasePlaylists) produce(put func(entry *model.Folder)) error {return nil}+// deferImport records the pending-import flag so a later scan with an admin can+// import the playlists, and returns an error if the flag can't be persisted (so+// the scan does not complete as successful without recording the recovery).+func (p *phasePlaylists) deferImport() error {+ if err := p.ds.Property(p.ctx).Put(consts.PlaylistsImportPendingFlagKey, "1"); err != nil {+ return fmt.Errorf("recording pending playlist import: %w", err)+ }+ log.Warn(p.ctx, "Playlists will not be imported, as there are no admin users yet. "++ "They will be imported automatically once an admin user is created.")+ return nil+}++func (p *phasePlaylists) importPending() (bool, error) {+ v, err := p.ds.Property(p.ctx).DefaultGet(consts.PlaylistsImportPendingFlagKey, "0")+ return v == "1", err+}+func (p *phasePlaylists) stages() []ppl.Stage[*model.Folder] {return []ppl.Stage[*model.Folder]{ppl.NewStage(p.processPlaylistsInFolder, ppl.Name("process playlists in folder"), ppl.Concurrency(3)),@@ -123,6 +162,11 @@ func (p *phasePlaylists) finalize(err error) error {} else {p.scanState.changesDetected.Store(true)}+ if p.pendingImport && err == nil {+ if derr := p.ds.Property(p.ctx).Delete(consts.PlaylistsImportPendingFlagKey); derr != nil {+ log.Warn(p.ctx, "Scanner: Could not clear pending playlist-import flag", derr)+ }+ }logF(p.ctx, "Scanner: Finished refreshing playlists", "refreshed", refreshed, err)return err}diff --git a/scanner/phase_4_playlists_test.go b/scanner/phase_4_playlists_test.goindex 0e01a7549..49ffc7fb7 100644--- a/scanner/phase_4_playlists_test.go+++ b/scanner/phase_4_playlists_test.go@@ -9,10 +9,10 @@ import ("github.com/navidrome/navidrome/conf""github.com/navidrome/navidrome/conf/configtest"+ "github.com/navidrome/navidrome/consts""github.com/navidrome/navidrome/core/artwork""github.com/navidrome/navidrome/core/playlists""github.com/navidrome/navidrome/model"- "github.com/navidrome/navidrome/model/request""github.com/navidrome/navidrome/tests". "github.com/onsi/ginkgo/v2". "github.com/onsi/gomega"@@ -30,14 +30,22 @@ var _ = Describe("phasePlaylists", func() {cw artwork.CacheWarmer)+ var userRepo *tests.MockedUserRepo+ var propRepo *tests.MockedPropertyRepo+BeforeEach(func() {DeferCleanup(configtest.SetupConfig())conf.Server.AutoImportPlaylists = truectx = context.Background()- ctx = request.WithUser(ctx, model.User{ID: "123", IsAdmin: true})folderRepo = &mockFolderRepository{}+ userRepo = tests.CreateMockUserRepo()+ // An admin user exists by default, so playlist import proceeds.+ Expect(userRepo.Put(&model.User{ID: "123", UserName: "admin", IsAdmin: true})).To(Succeed())+ propRepo = &tests.MockedPropertyRepo{}ds = &tests.MockDataStore{- MockedFolder: folderRepo,+ MockedFolder: folderRepo,+ MockedUser: userRepo,+ MockedProperty: propRepo,}pls = &mockPlaylists{}cw = artwork.NoopCacheWarmer()@@ -84,6 +92,81 @@ var _ = Describe("phasePlaylists", func() {Expect(called).To(BeFalse())Expect(err).To(MatchError(ContainSubstring("error loading folders")))})++ It("sets the pending flag and imports nothing when no admin user exists", func() {+ // Remove the admin user; produce resolves the admin at phase time.+ userRepo.Data = map[string]*model.User{}+ folderRepo.SetData(map[*model.Folder]error{+ {Path: "/path/to/folder1"}: nil,+ })++ called := false+ err := phase.produce(func(folder *model.Folder) { called = true })++ Expect(err).ToNot(HaveOccurred())+ Expect(called).To(BeFalse())+ v, _ := propRepo.Get(consts.PlaylistsImportPendingFlagKey)+ Expect(v).To(Equal("1"))+ })++ It("returns an error (not a silent defer) on a datastore failure resolving the admin", func() {+ userRepo.Error = errors.New("db is locked")++ err := phase.produce(func(folder *model.Folder) {})++ Expect(err).To(MatchError(ContainSubstring("finding admin user")))+ // Must NOT have set the pending flag on a real error.+ _, getErr := propRepo.Get(consts.PlaylistsImportPendingFlagKey)+ Expect(getErr).To(HaveOccurred())+ })++ It("returns an error when the pending flag cannot be persisted", func() {+ userRepo.Data = map[string]*model.User{} // no admin -> defer path+ propRepo.Error = errors.New("property table unavailable")++ err := phase.produce(func(folder *model.Folder) {})++ Expect(err).To(MatchError(ContainSubstring("recording pending playlist import")))+ })++ It("imports all playlist folders when the pending flag is set", func() {+ Expect(propRepo.Put(consts.PlaylistsImportPendingFlagKey, "1")).To(Succeed())+ folderRepo.SetAllData(map[*model.Folder]error{+ {Path: "/path/to/folder1"}: nil,+ {Path: "/path/to/folder2"}: nil,+ })+ // Touched set is empty: proves selection used GetAllWithPlaylists.+ folderRepo.SetData(map[*model.Folder]error{})++ var produced []*model.Folder+ err := phase.produce(func(folder *model.Folder) { produced = append(produced, folder) })++ Expect(err).ToNot(HaveOccurred())+ Expect(produced).To(HaveLen(2))+ Expect(phase.pendingImport).To(BeTrue())+ })+ })++ Describe("finalize", func() {+ It("clears the pending flag after a successful pending import", func() {+ Expect(propRepo.Put(consts.PlaylistsImportPendingFlagKey, "1")).To(Succeed())+ phase.pendingImport = true++ Expect(phase.finalize(nil)).To(Succeed())++ _, err := propRepo.Get(consts.PlaylistsImportPendingFlagKey)+ Expect(err).To(HaveOccurred()) // deleted+ })++ It("keeps the pending flag when the import failed", func() {+ Expect(propRepo.Put(consts.PlaylistsImportPendingFlagKey, "1")).To(Succeed())+ phase.pendingImport = true++ Expect(phase.finalize(errors.New("boom"))).To(HaveOccurred())++ v, _ := propRepo.Get(consts.PlaylistsImportPendingFlagKey)+ Expect(v).To(Equal("1"))+ })})Describe("processPlaylistsInFolder", func() {@@ -141,12 +224,13 @@ func (p *mockPlaylists) ImportFromFolder(ctx context.Context, folder *model.Foldtype mockFolderRepository struct {model.FolderRepository- data map[*model.Folder]error+ data map[*model.Folder]error+ allData map[*model.Folder]error}-func (f *mockFolderRepository) GetTouchedWithPlaylists() (model.FolderCursor, error) {+func cursorFromData(data map[*model.Folder]error) model.FolderCursor {return func(yield func(model.Folder, error) bool) {- for folder, err := range f.data {+ for folder, err := range data {if err != nil {if !yield(model.Folder{}, err) {return@@ -157,9 +241,21 @@ func (f *mockFolderRepository) GetTouchedWithPlaylists() (model.FolderCursor, erreturn}}- }, nil+ }+}++func (f *mockFolderRepository) GetTouchedWithPlaylists() (model.FolderCursor, error) {+ return cursorFromData(f.data), nil+}++func (f *mockFolderRepository) GetAllWithPlaylists() (model.FolderCursor, error) {+ return cursorFromData(f.allData), nil}func (f *mockFolderRepository) SetData(m map[*model.Folder]error) {f.data = m}++func (f *mockFolderRepository) SetAllData(m map[*model.Folder]error) {+ f.allData = m+}diff --git a/tests/mock_user_repo.go b/tests/mock_user_repo.goindex 7c7dadbc4..2d6ff3c02 100644--- a/tests/mock_user_repo.go+++ b/tests/mock_user_repo.go@@ -57,6 +57,18 @@ func (u *MockedUserRepo) FindByUsernameWithPassword(username string) (*model.Usereturn u.FindByUsername(username)}+func (u *MockedUserRepo) FindFirstAdmin() (*model.User, error) {+ if u.Error != nil {+ return nil, u.Error+ }+ for _, usr := range u.Data {+ if usr.IsAdmin {+ return usr, nil+ }+ }+ return nil, model.ErrNotFound+}+func (u *MockedUserRepo) Get(id string) (*model.User, error) {if u.Error != nil {return nil, u.Error