🎧 Your Personal Streaming Service
Go airsonicmadsonicmedia-servermusicmusic-servernavidromeopensubsonicraspberry-piself-hostedstreamerstreaming-apistreaming-audiosubsonicsubsonic-server
feat(scanner): add Scanner.PurgeMissing configuration option (#4107)
Copilot992c78376c9c
diff --git a/conf/configuration.go b/conf/configuration.goindex eebf1c004..8561f343f 100644--- a/conf/configuration.go+++ b/conf/configuration.go@@ -134,6 +134,7 @@ type scannerOptions struct {GenreSeparators string // Deprecated: Use Tags.genre.Split insteadGroupAlbumReleases bool // Deprecated: Use PID.Album insteadFollowSymlinks bool // Whether to follow symlinks when scanning directories+ PurgeMissing string // Values: "never", "always", "full"}type subsonicOptions struct {@@ -277,6 +278,7 @@ func Load(noConfigDump bool) {validateScanSchedule,validateBackupSchedule,validatePlaylistsPath,+ validatePurgeMissingOption,)if err != nil {os.Exit(1)@@ -382,6 +384,24 @@ func validatePlaylistsPath() error {return nil}+func validatePurgeMissingOption() error {+ allowedValues := []string{consts.PurgeMissingNever, consts.PurgeMissingAlways, consts.PurgeMissingFull}+ valid := false+ for _, v := range allowedValues {+ if v == Server.Scanner.PurgeMissing {+ valid = true+ break+ }+ }+ if !valid {+ err := fmt.Errorf("Invalid Scanner.PurgeMissing value: '%s'. Must be one of: %v", Server.Scanner.PurgeMissing, allowedValues)+ log.Error(err.Error())+ Server.Scanner.PurgeMissing = consts.PurgeMissingNever+ return err+ }+ return nil+}+func validateScanSchedule() error {if Server.Scanner.Schedule == "0" || Server.Scanner.Schedule == "" {Server.Scanner.Schedule = ""@@ -421,7 +441,7 @@ func AddHook(hook func()) {hooks = append(hooks, hook)}-func init() {+func setViperDefaults() {viper.SetDefault("musicfolder", filepath.Join(".", "music"))viper.SetDefault("cachefolder", "")viper.SetDefault("datafolder", ".")@@ -458,7 +478,6 @@ func init() {viper.SetDefault("indexgroups", "A B C D E F G H I J K L M N O P Q R S T U V W X-Z(XYZ) [Unknown]([)")viper.SetDefault("ffmpegpath", "")viper.SetDefault("mpvcmdtemplate", "mpv --audio-device=%d --no-audio-display --pause %f --input-ipc-server=%s")-viper.SetDefault("coverartpriority", "cover.*, folder.*, front.*, embedded, external")viper.SetDefault("coverjpegquality", 75)viper.SetDefault("artistartpriority", "artist.*, album/artist.*, external")@@ -481,19 +500,15 @@ func init() {viper.SetDefault("authrequestlimit", 5)viper.SetDefault("authwindowlength", 20*time.Second)viper.SetDefault("passwordencryptionkey", "")-viper.SetDefault("reverseproxyuserheader", "Remote-User")viper.SetDefault("reverseproxywhitelist", "")-viper.SetDefault("prometheus.enabled", false)viper.SetDefault("prometheus.metricspath", consts.PrometheusDefaultPath)viper.SetDefault("prometheus.password", "")-viper.SetDefault("jukebox.enabled", false)viper.SetDefault("jukebox.devices", []AudioDeviceDefinition{})viper.SetDefault("jukebox.default", "")viper.SetDefault("jukebox.adminonly", true)-viper.SetDefault("scanner.enabled", true)viper.SetDefault("scanner.schedule", "0")viper.SetDefault("scanner.extractor", consts.DefaultScannerExtractor)@@ -503,12 +518,11 @@ func init() {viper.SetDefault("scanner.genreseparators", "")viper.SetDefault("scanner.groupalbumreleases", false)viper.SetDefault("scanner.followsymlinks", true)-+ viper.SetDefault("scanner.purgemissing", "never")viper.SetDefault("subsonic.appendsubtitle", true)viper.SetDefault("subsonic.artistparticipations", false)viper.SetDefault("subsonic.defaultreportrealpath", false)viper.SetDefault("subsonic.legacyclients", "DSub")-viper.SetDefault("agents", "lastfm,spotify")viper.SetDefault("lastfm.enabled", true)viper.SetDefault("lastfm.language", "en")@@ -518,24 +532,17 @@ func init() {viper.SetDefault("spotify.secret", "")viper.SetDefault("listenbrainz.enabled", true)viper.SetDefault("listenbrainz.baseurl", "https://api.listenbrainz.org/1/")-viper.SetDefault("httpsecurityheaders.customframeoptionsvalue", "DENY")-viper.SetDefault("backup.path", "")viper.SetDefault("backup.schedule", "")viper.SetDefault("backup.count", 0)-viper.SetDefault("pid.track", consts.DefaultTrackPID)viper.SetDefault("pid.album", consts.DefaultAlbumPID)-viper.SetDefault("inspect.enabled", true)viper.SetDefault("inspect.maxrequests", 1)viper.SetDefault("inspect.backloglimit", consts.RequestThrottleBacklogLimit)viper.SetDefault("inspect.backlogtimeout", consts.RequestThrottleBacklogTimeout)-viper.SetDefault("lyricspriority", ".lrc,.txt,embedded")-- // DevFlags. These are used to enable/disable debugging and incomplete featuresviper.SetDefault("devlogsourceline", false)viper.SetDefault("devenableprofiler", false)viper.SetDefault("devautocreateadminpassword", "")@@ -556,6 +563,10 @@ func init() {viper.SetDefault("devenableplayerinsights", true)}+func init() {+ setViperDefaults()+}+func InitConfig(cfgFile string) {codecRegistry := viper.NewCodecRegistry()_ = codecRegistry.RegisterCodec("ini", ini.Codec{})diff --git a/conf/configuration_test.go b/conf/configuration_test.goindex f57764709..5b54e4975 100644--- a/conf/configuration_test.go+++ b/conf/configuration_test.go@@ -5,7 +5,7 @@ import ("path/filepath""testing"- . "github.com/navidrome/navidrome/conf"+ "github.com/navidrome/navidrome/conf". "github.com/onsi/ginkgo/v2". "github.com/onsi/gomega""github.com/spf13/viper"@@ -20,9 +20,10 @@ var _ = Describe("Configuration", func() {BeforeEach(func() {// Reset viper configurationviper.Reset()+ conf.SetViperDefaults()viper.SetDefault("datafolder", GinkgoT().TempDir())viper.SetDefault("loglevel", "error")- ResetConf()+ conf.ResetConf()})DescribeTable("should load configuration from",@@ -30,17 +31,17 @@ var _ = Describe("Configuration", func() {filename := filepath.Join("testdata", "cfg."+format)// Initialize config with the test file- InitConfig(filename)+ conf.InitConfig(filename)// Load the configuration (with noConfigDump=true)- Load(true)+ conf.Load(true)// Execute the format-specific assertions- Expect(Server.MusicFolder).To(Equal(fmt.Sprintf("/%s/music", format)))- Expect(Server.UIWelcomeMessage).To(Equal("Welcome " + format))- Expect(Server.Tags["custom"].Aliases).To(Equal([]string{format, "test"}))+ Expect(conf.Server.MusicFolder).To(Equal(fmt.Sprintf("/%s/music", format)))+ Expect(conf.Server.UIWelcomeMessage).To(Equal("Welcome " + format))+ Expect(conf.Server.Tags["custom"].Aliases).To(Equal([]string{format, "test"}))// The config file used should be the one we created- Expect(Server.ConfigFile).To(Equal(filename))+ Expect(conf.Server.ConfigFile).To(Equal(filename))},Entry("TOML format", "toml"),Entry("YAML format", "yaml"),diff --git a/conf/export_test.go b/conf/export_test.goindex 0bd7819eb..1b6daf036 100644--- a/conf/export_test.go+++ b/conf/export_test.go@@ -3,3 +3,5 @@ package conffunc ResetConf() {Server = &configOptions{}}++var SetViperDefaults = setViperDefaultsdiff --git a/consts/consts.go b/consts/consts.goindex 2dbb46d07..fbb2c9429 100644--- a/consts/consts.go+++ b/consts/consts.go@@ -115,6 +115,12 @@ const (InsightsInitialDelay = 30 * time.Minute)+const (+ PurgeMissingNever = "never"+ PurgeMissingAlways = "always"+ PurgeMissingFull = "full"+)+var (DefaultDownsamplingFormat = "opus"DefaultTranscodings = []struct {diff --git a/model/mediafile.go b/model/mediafile.goindex 354c5ea47..cdb001c85 100644--- a/model/mediafile.go+++ b/model/mediafile.go@@ -342,6 +342,7 @@ type MediaFileRepository interface {GetCursor(options ...QueryOptions) (MediaFileCursor, error)Delete(id string) errorDeleteMissing(ids []string) error+ DeleteAllMissing() (int64, error)FindByPaths(paths []string) (MediaFiles, error)// The following methods are used exclusively by the scanner:diff --git a/persistence/mediafile_repository.go b/persistence/mediafile_repository.goindex e236b1b29..b0ed637c1 100644--- a/persistence/mediafile_repository.go+++ b/persistence/mediafile_repository.go@@ -192,6 +192,15 @@ func (r *mediaFileRepository) Delete(id string) error {return r.delete(Eq{"id": id})}+func (r *mediaFileRepository) DeleteAllMissing() (int64, error) {+ user := loggedUser(r.ctx)+ if !user.IsAdmin {+ return 0, rest.ErrPermissionDenied+ }+ del := Delete(r.tableName).Where(Eq{"missing": true})+ return r.executeSQL(del)+}+func (r *mediaFileRepository) DeleteMissing(ids []string) error {user := loggedUser(r.ctx)if !user.IsAdmin {diff --git a/scanner/phase_2_missing_tracks.go b/scanner/phase_2_missing_tracks.goindex 352f92c34..6f56f6a52 100644--- a/scanner/phase_2_missing_tracks.go+++ b/scanner/phase_2_missing_tracks.go@@ -6,6 +6,8 @@ import ("sync/atomic"ppl "github.com/google/go-pipeline/pkg/pipeline"+ "github.com/navidrome/navidrome/conf"+ "github.com/navidrome/navidrome/consts""github.com/navidrome/navidrome/log""github.com/navidrome/navidrome/model")@@ -182,7 +184,35 @@ func (p *phaseMissingTracks) finalize(err error) error {if matched > 0 {log.Info(p.ctx, "Scanner: Found moved files", "total", matched, err)}+ if err != nil {+ return err+ }++ // Check if we should purge missing items+ if conf.Server.Scanner.PurgeMissing == consts.PurgeMissingAlways || (conf.Server.Scanner.PurgeMissing == consts.PurgeMissingFull && p.state.fullScan) {+ if err = p.purgeMissing(); err != nil {+ log.Error(p.ctx, "Scanner: Error purging missing items", err)+ }+ }+return err}+func (p *phaseMissingTracks) purgeMissing() error {+ deletedCount, err := p.ds.MediaFile(p.ctx).DeleteAllMissing()+ if err != nil {+ return fmt.Errorf("error deleting missing files: %w", err)+ }++ if deletedCount > 0 {+ log.Info(p.ctx, "Scanner: Purged missing items from the database", "mediaFiles", deletedCount)+ // Set changesDetected to true so that garbage collection will run at the end of the scan process+ p.state.changesDetected.Store(true)+ } else {+ log.Debug(p.ctx, "Scanner: No missing items to purge")+ }++ return nil+}+var _ phase[*missingTracks] = (*phaseMissingTracks)(nil)diff --git a/scanner/phase_2_missing_tracks_test.go b/scanner/phase_2_missing_tracks_test.goindex 2cd686604..5dd6cc679 100644--- a/scanner/phase_2_missing_tracks_test.go+++ b/scanner/phase_2_missing_tracks_test.go@@ -4,6 +4,8 @@ import ("context""time"+ "github.com/navidrome/navidrome/conf"+ "github.com/navidrome/navidrome/consts""github.com/navidrome/navidrome/model""github.com/navidrome/navidrome/tests". "github.com/onsi/ginkgo/v2"@@ -222,4 +224,66 @@ var _ = Describe("phaseMissingTracks", func() {Expect(state.changesDetected.Load()).To(BeFalse())})})++ Describe("finalize", func() {+ It("should return nil if no error", func() {+ err := phase.finalize(nil)+ Expect(err).To(BeNil())+ Expect(state.changesDetected.Load()).To(BeFalse())+ })++ It("should return the error if provided", func() {+ err := phase.finalize(context.DeadlineExceeded)+ Expect(err).To(Equal(context.DeadlineExceeded))+ Expect(state.changesDetected.Load()).To(BeFalse())+ })++ When("PurgeMissing is 'always'", func() {+ BeforeEach(func() {+ conf.Server.Scanner.PurgeMissing = consts.PurgeMissingAlways+ mr.CountAllValue = 3+ mr.DeleteAllMissingValue = 3+ })+ It("should purge missing files", func() {+ Expect(state.changesDetected.Load()).To(BeFalse())+ err := phase.finalize(nil)+ Expect(err).To(BeNil())+ Expect(state.changesDetected.Load()).To(BeTrue())+ })+ })++ When("PurgeMissing is 'full'", func() {+ BeforeEach(func() {+ conf.Server.Scanner.PurgeMissing = consts.PurgeMissingFull+ mr.CountAllValue = 2+ mr.DeleteAllMissingValue = 2+ })+ It("should not purge missing files if not a full scan", func() {+ state.fullScan = false+ err := phase.finalize(nil)+ Expect(err).To(BeNil())+ Expect(state.changesDetected.Load()).To(BeFalse())+ })+ It("should purge missing files if full scan", func() {+ Expect(state.changesDetected.Load()).To(BeFalse())+ state.fullScan = true+ err := phase.finalize(nil)+ Expect(err).To(BeNil())+ Expect(state.changesDetected.Load()).To(BeTrue())+ })+ })++ When("PurgeMissing is 'never'", func() {+ BeforeEach(func() {+ conf.Server.Scanner.PurgeMissing = consts.PurgeMissingNever+ mr.CountAllValue = 1+ mr.DeleteAllMissingValue = 1+ })+ It("should not purge missing files", func() {+ err := phase.finalize(nil)+ Expect(err).To(BeNil())+ Expect(state.changesDetected.Load()).To(BeFalse())+ })+ })+ })})diff --git a/scanner/scanner_test.go b/scanner/scanner_test.goindex 33c78fe7d..7c74e3f81 100644--- a/scanner/scanner_test.go+++ b/scanner/scanner_test.go@@ -10,6 +10,7 @@ import ("github.com/google/uuid""github.com/navidrome/navidrome/conf""github.com/navidrome/navidrome/conf/configtest"+ "github.com/navidrome/navidrome/consts""github.com/navidrome/navidrome/core""github.com/navidrome/navidrome/core/artwork""github.com/navidrome/navidrome/core/metrics"@@ -17,6 +18,7 @@ import ("github.com/navidrome/navidrome/db""github.com/navidrome/navidrome/log""github.com/navidrome/navidrome/model"+ "github.com/navidrome/navidrome/model/request""github.com/navidrome/navidrome/persistence""github.com/navidrome/navidrome/scanner""github.com/navidrome/navidrome/server/events"@@ -47,6 +49,7 @@ var _ = Describe("Scanner", Ordered, func() {}BeforeAll(func() {+ ctx = request.WithUser(GinkgoT().Context(), model.User{ID: "123", IsAdmin: true})tmpDir := GinkgoT().TempDir()conf.Server.DbPath = filepath.Join(tmpDir, "test-scanner.db?_journal_mode=WAL")log.Warn("Using DB at " + conf.Server.DbPath)@@ -54,7 +57,6 @@ var _ = Describe("Scanner", Ordered, func() {})BeforeEach(func() {- ctx = context.Background()db.Init(ctx)DeferCleanup(func() {Expect(tests.ClearDB()).To(Succeed())@@ -501,6 +503,113 @@ var _ = Describe("Scanner", Ordered, func() {Expect(aa[0].MbzArtistID).To(Equal(beatlesMBID))Expect(aa[0].SortArtistName).To(Equal("Beatles, The"))})++ Context("When PurgeMissing is configured", func() {+ When("PurgeMissing is set to 'never'", func() {+ BeforeEach(func() {+ DeferCleanup(configtest.SetupConfig())+ conf.Server.Scanner.PurgeMissing = consts.PurgeMissingNever+ })++ It("should mark files as missing but not delete them", func() {+ By("Running initial scan")+ Expect(runScanner(ctx, true)).To(Succeed())++ By("Removing a file")+ fsys.Remove("The Beatles/Revolver/02 - Eleanor Rigby.mp3")++ By("Running another scan")+ Expect(runScanner(ctx, true)).To(Succeed())++ By("Checking files are marked as missing but not deleted")+ count, err := ds.MediaFile(ctx).CountAll(model.QueryOptions{+ Filters: squirrel.Eq{"missing": true},+ })+ Expect(err).ToNot(HaveOccurred())+ Expect(count).To(Equal(int64(1)))++ mf, err := findByPath("The Beatles/Revolver/02 - Eleanor Rigby.mp3")+ Expect(err).ToNot(HaveOccurred())+ Expect(mf.Missing).To(BeTrue())+ })+ })++ When("PurgeMissing is set to 'always'", func() {+ BeforeEach(func() {+ conf.Server.Scanner.PurgeMissing = consts.PurgeMissingAlways+ })++ It("should purge missing files on any scan", func() {+ By("Running initial scan")+ Expect(runScanner(ctx, false)).To(Succeed())++ By("Removing a file")+ fsys.Remove("The Beatles/Revolver/02 - Eleanor Rigby.mp3")++ By("Running an incremental scan")+ Expect(runScanner(ctx, false)).To(Succeed())++ By("Checking missing files are deleted")+ count, err := ds.MediaFile(ctx).CountAll(model.QueryOptions{+ Filters: squirrel.Eq{"missing": true},+ })+ Expect(err).ToNot(HaveOccurred())+ Expect(count).To(BeZero())++ _, err = findByPath("The Beatles/Revolver/02 - Eleanor Rigby.mp3")+ Expect(err).To(MatchError(model.ErrNotFound))+ })+ })++ When("PurgeMissing is set to 'full'", func() {+ BeforeEach(func() {+ conf.Server.Scanner.PurgeMissing = consts.PurgeMissingFull+ })++ It("should not purge missing files on incremental scans", func() {+ By("Running initial scan")+ Expect(runScanner(ctx, true)).To(Succeed())++ By("Removing a file")+ fsys.Remove("The Beatles/Revolver/02 - Eleanor Rigby.mp3")++ By("Running an incremental scan")+ Expect(runScanner(ctx, false)).To(Succeed())++ By("Checking files are marked as missing but not deleted")+ count, err := ds.MediaFile(ctx).CountAll(model.QueryOptions{+ Filters: squirrel.Eq{"missing": true},+ })+ Expect(err).ToNot(HaveOccurred())+ Expect(count).To(Equal(int64(1)))++ mf, err := findByPath("The Beatles/Revolver/02 - Eleanor Rigby.mp3")+ Expect(err).ToNot(HaveOccurred())+ Expect(mf.Missing).To(BeTrue())+ })++ It("should purge missing files only on full scans", func() {+ By("Running initial scan")+ Expect(runScanner(ctx, true)).To(Succeed())++ By("Removing a file")+ fsys.Remove("The Beatles/Revolver/02 - Eleanor Rigby.mp3")++ By("Running a full scan")+ Expect(runScanner(ctx, true)).To(Succeed())++ By("Checking missing files are deleted")+ count, err := ds.MediaFile(ctx).CountAll(model.QueryOptions{+ Filters: squirrel.Eq{"missing": true},+ })+ Expect(err).ToNot(HaveOccurred())+ Expect(count).To(BeZero())++ _, err = findByPath("The Beatles/Revolver/02 - Eleanor Rigby.mp3")+ Expect(err).To(MatchError(model.ErrNotFound))+ })+ })+ })})})diff --git a/tests/mock_mediafile_repo.go b/tests/mock_mediafile_repo.goindex 01d82e03b..85adb8a25 100644--- a/tests/mock_mediafile_repo.go+++ b/tests/mock_mediafile_repo.go@@ -22,6 +22,10 @@ type MockMediaFileRepo struct {model.MediaFileRepositoryData map[string]*model.MediaFileErr bool+ // Add fields and methods for controlling CountAll and DeleteAllMissing in tests+ CountAllValue int64+ CountAllOptions model.QueryOptions+ DeleteAllMissingValue int64}func (m *MockMediaFileRepo) SetError(err bool) {@@ -161,4 +165,35 @@ func (m *MockMediaFileRepo) GetMissingAndMatching(libId int) (model.MediaFileCur}, nil}+func (m *MockMediaFileRepo) CountAll(opts ...model.QueryOptions) (int64, error) {+ if m.Err {+ return 0, errors.New("error")+ }+ if m.CountAllValue != 0 {+ if len(opts) > 0 {+ m.CountAllOptions = opts[0]+ }+ return m.CountAllValue, nil+ }+ return int64(len(m.Data)), nil+}++func (m *MockMediaFileRepo) DeleteAllMissing() (int64, error) {+ if m.Err {+ return 0, errors.New("error")+ }+ if m.DeleteAllMissingValue != 0 {+ return m.DeleteAllMissingValue, nil+ }+ // Remove all missing files from Data+ var count int64+ for id, mf := range m.Data {+ if mf.Missing {+ delete(m.Data, id)+ count+++ }+ }+ return count, nil+}+var _ model.MediaFileRepository = (*MockMediaFileRepo)(nil)