Skip to content

🎧 Your Personal Streaming Service

feat(scanner): add Scanner.PurgeMissing configuration option (#4107)

Copilot992c78376c9c
diff --git a/conf/configuration.go b/conf/configuration.go
index eebf1c004..8561f343f 100644
--- a/conf/configuration.go
+++ b/conf/configuration.go
@@ -134,6 +134,7 @@ type scannerOptions struct {
GenreSeparators string // Deprecated: Use Tags.genre.Split instead
GroupAlbumReleases bool // Deprecated: Use PID.Album instead
FollowSymlinks bool // Whether to follow symlinks when scanning directories
+ PurgeMissing string // Values: "never", "always", "full"
}
type subsonicOptions struct {
@@ -277,6 +278,7 @@ func Load(noConfigDump bool) {
validateScanSchedule,
validateBackupSchedule,
validatePlaylistsPath,
+ validatePurgeMissingOption,
)
if err != nil {
os.Exit(1)
@@ -382,6 +384,24 @@ func validatePlaylistsPath() error {
return nil
}
+func validatePurgeMissingOption() error {
+ allowedValues := []string{consts.PurgeMissingNever, consts.PurgeMissingAlways, consts.PurgeMissingFull}
+ valid := false
+ for _, v := range allowedValues {
+ if v == Server.Scanner.PurgeMissing {
+ valid = true
+ break
+ }
+ }
+ if !valid {
+ err := fmt.Errorf("Invalid Scanner.PurgeMissing value: '%s'. Must be one of: %v", Server.Scanner.PurgeMissing, allowedValues)
+ log.Error(err.Error())
+ Server.Scanner.PurgeMissing = consts.PurgeMissingNever
+ return err
+ }
+ return nil
+}
+
func validateScanSchedule() error {
if Server.Scanner.Schedule == "0" || Server.Scanner.Schedule == "" {
Server.Scanner.Schedule = ""
@@ -421,7 +441,7 @@ func AddHook(hook func()) {
hooks = append(hooks, hook)
}
-func init() {
+func setViperDefaults() {
viper.SetDefault("musicfolder", filepath.Join(".", "music"))
viper.SetDefault("cachefolder", "")
viper.SetDefault("datafolder", ".")
@@ -458,7 +478,6 @@ func init() {
viper.SetDefault("indexgroups", "A B C D E F G H I J K L M N O P Q R S T U V W X-Z(XYZ) [Unknown]([)")
viper.SetDefault("ffmpegpath", "")
viper.SetDefault("mpvcmdtemplate", "mpv --audio-device=%d --no-audio-display --pause %f --input-ipc-server=%s")
-
viper.SetDefault("coverartpriority", "cover.*, folder.*, front.*, embedded, external")
viper.SetDefault("coverjpegquality", 75)
viper.SetDefault("artistartpriority", "artist.*, album/artist.*, external")
@@ -481,19 +500,15 @@ func init() {
viper.SetDefault("authrequestlimit", 5)
viper.SetDefault("authwindowlength", 20*time.Second)
viper.SetDefault("passwordencryptionkey", "")
-
viper.SetDefault("reverseproxyuserheader", "Remote-User")
viper.SetDefault("reverseproxywhitelist", "")
-
viper.SetDefault("prometheus.enabled", false)
viper.SetDefault("prometheus.metricspath", consts.PrometheusDefaultPath)
viper.SetDefault("prometheus.password", "")
-
viper.SetDefault("jukebox.enabled", false)
viper.SetDefault("jukebox.devices", []AudioDeviceDefinition{})
viper.SetDefault("jukebox.default", "")
viper.SetDefault("jukebox.adminonly", true)
-
viper.SetDefault("scanner.enabled", true)
viper.SetDefault("scanner.schedule", "0")
viper.SetDefault("scanner.extractor", consts.DefaultScannerExtractor)
@@ -503,12 +518,11 @@ func init() {
viper.SetDefault("scanner.genreseparators", "")
viper.SetDefault("scanner.groupalbumreleases", false)
viper.SetDefault("scanner.followsymlinks", true)
-
+ viper.SetDefault("scanner.purgemissing", "never")
viper.SetDefault("subsonic.appendsubtitle", true)
viper.SetDefault("subsonic.artistparticipations", false)
viper.SetDefault("subsonic.defaultreportrealpath", false)
viper.SetDefault("subsonic.legacyclients", "DSub")
-
viper.SetDefault("agents", "lastfm,spotify")
viper.SetDefault("lastfm.enabled", true)
viper.SetDefault("lastfm.language", "en")
@@ -518,24 +532,17 @@ func init() {
viper.SetDefault("spotify.secret", "")
viper.SetDefault("listenbrainz.enabled", true)
viper.SetDefault("listenbrainz.baseurl", "https://api.listenbrainz.org/1/")
-
viper.SetDefault("httpsecurityheaders.customframeoptionsvalue", "DENY")
-
viper.SetDefault("backup.path", "")
viper.SetDefault("backup.schedule", "")
viper.SetDefault("backup.count", 0)
-
viper.SetDefault("pid.track", consts.DefaultTrackPID)
viper.SetDefault("pid.album", consts.DefaultAlbumPID)
-
viper.SetDefault("inspect.enabled", true)
viper.SetDefault("inspect.maxrequests", 1)
viper.SetDefault("inspect.backloglimit", consts.RequestThrottleBacklogLimit)
viper.SetDefault("inspect.backlogtimeout", consts.RequestThrottleBacklogTimeout)
-
viper.SetDefault("lyricspriority", ".lrc,.txt,embedded")
-
- // DevFlags. These are used to enable/disable debugging and incomplete features
viper.SetDefault("devlogsourceline", false)
viper.SetDefault("devenableprofiler", false)
viper.SetDefault("devautocreateadminpassword", "")
@@ -556,6 +563,10 @@ func init() {
viper.SetDefault("devenableplayerinsights", true)
}
+func init() {
+ setViperDefaults()
+}
+
func InitConfig(cfgFile string) {
codecRegistry := viper.NewCodecRegistry()
_ = codecRegistry.RegisterCodec("ini", ini.Codec{})
diff --git a/conf/configuration_test.go b/conf/configuration_test.go
index f57764709..5b54e4975 100644
--- a/conf/configuration_test.go
+++ b/conf/configuration_test.go
@@ -5,7 +5,7 @@ import (
"path/filepath"
"testing"
- . "github.com/navidrome/navidrome/conf"
+ "github.com/navidrome/navidrome/conf"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/spf13/viper"
@@ -20,9 +20,10 @@ var _ = Describe("Configuration", func() {
BeforeEach(func() {
// Reset viper configuration
viper.Reset()
+ conf.SetViperDefaults()
viper.SetDefault("datafolder", GinkgoT().TempDir())
viper.SetDefault("loglevel", "error")
- ResetConf()
+ conf.ResetConf()
})
DescribeTable("should load configuration from",
@@ -30,17 +31,17 @@ var _ = Describe("Configuration", func() {
filename := filepath.Join("testdata", "cfg."+format)
// Initialize config with the test file
- InitConfig(filename)
+ conf.InitConfig(filename)
// Load the configuration (with noConfigDump=true)
- Load(true)
+ conf.Load(true)
// Execute the format-specific assertions
- Expect(Server.MusicFolder).To(Equal(fmt.Sprintf("/%s/music", format)))
- Expect(Server.UIWelcomeMessage).To(Equal("Welcome " + format))
- Expect(Server.Tags["custom"].Aliases).To(Equal([]string{format, "test"}))
+ Expect(conf.Server.MusicFolder).To(Equal(fmt.Sprintf("/%s/music", format)))
+ Expect(conf.Server.UIWelcomeMessage).To(Equal("Welcome " + format))
+ Expect(conf.Server.Tags["custom"].Aliases).To(Equal([]string{format, "test"}))
// The config file used should be the one we created
- Expect(Server.ConfigFile).To(Equal(filename))
+ Expect(conf.Server.ConfigFile).To(Equal(filename))
},
Entry("TOML format", "toml"),
Entry("YAML format", "yaml"),
diff --git a/conf/export_test.go b/conf/export_test.go
index 0bd7819eb..1b6daf036 100644
--- a/conf/export_test.go
+++ b/conf/export_test.go
@@ -3,3 +3,5 @@ package conf
func ResetConf() {
Server = &configOptions{}
}
+
+var SetViperDefaults = setViperDefaults
diff --git a/consts/consts.go b/consts/consts.go
index 2dbb46d07..fbb2c9429 100644
--- a/consts/consts.go
+++ b/consts/consts.go
@@ -115,6 +115,12 @@ const (
InsightsInitialDelay = 30 * time.Minute
)
+const (
+ PurgeMissingNever = "never"
+ PurgeMissingAlways = "always"
+ PurgeMissingFull = "full"
+)
+
var (
DefaultDownsamplingFormat = "opus"
DefaultTranscodings = []struct {
diff --git a/model/mediafile.go b/model/mediafile.go
index 354c5ea47..cdb001c85 100644
--- a/model/mediafile.go
+++ b/model/mediafile.go
@@ -342,6 +342,7 @@ type MediaFileRepository interface {
GetCursor(options ...QueryOptions) (MediaFileCursor, error)
Delete(id string) error
DeleteMissing(ids []string) error
+ DeleteAllMissing() (int64, error)
FindByPaths(paths []string) (MediaFiles, error)
// The following methods are used exclusively by the scanner:
diff --git a/persistence/mediafile_repository.go b/persistence/mediafile_repository.go
index e236b1b29..b0ed637c1 100644
--- a/persistence/mediafile_repository.go
+++ b/persistence/mediafile_repository.go
@@ -192,6 +192,15 @@ func (r *mediaFileRepository) Delete(id string) error {
return r.delete(Eq{"id": id})
}
+func (r *mediaFileRepository) DeleteAllMissing() (int64, error) {
+ user := loggedUser(r.ctx)
+ if !user.IsAdmin {
+ return 0, rest.ErrPermissionDenied
+ }
+ del := Delete(r.tableName).Where(Eq{"missing": true})
+ return r.executeSQL(del)
+}
+
func (r *mediaFileRepository) DeleteMissing(ids []string) error {
user := loggedUser(r.ctx)
if !user.IsAdmin {
diff --git a/scanner/phase_2_missing_tracks.go b/scanner/phase_2_missing_tracks.go
index 352f92c34..6f56f6a52 100644
--- a/scanner/phase_2_missing_tracks.go
+++ b/scanner/phase_2_missing_tracks.go
@@ -6,6 +6,8 @@ import (
"sync/atomic"
ppl "github.com/google/go-pipeline/pkg/pipeline"
+ "github.com/navidrome/navidrome/conf"
+ "github.com/navidrome/navidrome/consts"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
)
@@ -182,7 +184,35 @@ func (p *phaseMissingTracks) finalize(err error) error {
if matched > 0 {
log.Info(p.ctx, "Scanner: Found moved files", "total", matched, err)
}
+ if err != nil {
+ return err
+ }
+
+ // Check if we should purge missing items
+ if conf.Server.Scanner.PurgeMissing == consts.PurgeMissingAlways || (conf.Server.Scanner.PurgeMissing == consts.PurgeMissingFull && p.state.fullScan) {
+ if err = p.purgeMissing(); err != nil {
+ log.Error(p.ctx, "Scanner: Error purging missing items", err)
+ }
+ }
+
return err
}
+func (p *phaseMissingTracks) purgeMissing() error {
+ deletedCount, err := p.ds.MediaFile(p.ctx).DeleteAllMissing()
+ if err != nil {
+ return fmt.Errorf("error deleting missing files: %w", err)
+ }
+
+ if deletedCount > 0 {
+ log.Info(p.ctx, "Scanner: Purged missing items from the database", "mediaFiles", deletedCount)
+ // Set changesDetected to true so that garbage collection will run at the end of the scan process
+ p.state.changesDetected.Store(true)
+ } else {
+ log.Debug(p.ctx, "Scanner: No missing items to purge")
+ }
+
+ return nil
+}
+
var _ phase[*missingTracks] = (*phaseMissingTracks)(nil)
diff --git a/scanner/phase_2_missing_tracks_test.go b/scanner/phase_2_missing_tracks_test.go
index 2cd686604..5dd6cc679 100644
--- a/scanner/phase_2_missing_tracks_test.go
+++ b/scanner/phase_2_missing_tracks_test.go
@@ -4,6 +4,8 @@ import (
"context"
"time"
+ "github.com/navidrome/navidrome/conf"
+ "github.com/navidrome/navidrome/consts"
"github.com/navidrome/navidrome/model"
"github.com/navidrome/navidrome/tests"
. "github.com/onsi/ginkgo/v2"
@@ -222,4 +224,66 @@ var _ = Describe("phaseMissingTracks", func() {
Expect(state.changesDetected.Load()).To(BeFalse())
})
})
+
+ Describe("finalize", func() {
+ It("should return nil if no error", func() {
+ err := phase.finalize(nil)
+ Expect(err).To(BeNil())
+ Expect(state.changesDetected.Load()).To(BeFalse())
+ })
+
+ It("should return the error if provided", func() {
+ err := phase.finalize(context.DeadlineExceeded)
+ Expect(err).To(Equal(context.DeadlineExceeded))
+ Expect(state.changesDetected.Load()).To(BeFalse())
+ })
+
+ When("PurgeMissing is 'always'", func() {
+ BeforeEach(func() {
+ conf.Server.Scanner.PurgeMissing = consts.PurgeMissingAlways
+ mr.CountAllValue = 3
+ mr.DeleteAllMissingValue = 3
+ })
+ It("should purge missing files", func() {
+ Expect(state.changesDetected.Load()).To(BeFalse())
+ err := phase.finalize(nil)
+ Expect(err).To(BeNil())
+ Expect(state.changesDetected.Load()).To(BeTrue())
+ })
+ })
+
+ When("PurgeMissing is 'full'", func() {
+ BeforeEach(func() {
+ conf.Server.Scanner.PurgeMissing = consts.PurgeMissingFull
+ mr.CountAllValue = 2
+ mr.DeleteAllMissingValue = 2
+ })
+ It("should not purge missing files if not a full scan", func() {
+ state.fullScan = false
+ err := phase.finalize(nil)
+ Expect(err).To(BeNil())
+ Expect(state.changesDetected.Load()).To(BeFalse())
+ })
+ It("should purge missing files if full scan", func() {
+ Expect(state.changesDetected.Load()).To(BeFalse())
+ state.fullScan = true
+ err := phase.finalize(nil)
+ Expect(err).To(BeNil())
+ Expect(state.changesDetected.Load()).To(BeTrue())
+ })
+ })
+
+ When("PurgeMissing is 'never'", func() {
+ BeforeEach(func() {
+ conf.Server.Scanner.PurgeMissing = consts.PurgeMissingNever
+ mr.CountAllValue = 1
+ mr.DeleteAllMissingValue = 1
+ })
+ It("should not purge missing files", func() {
+ err := phase.finalize(nil)
+ Expect(err).To(BeNil())
+ Expect(state.changesDetected.Load()).To(BeFalse())
+ })
+ })
+ })
})
diff --git a/scanner/scanner_test.go b/scanner/scanner_test.go
index 33c78fe7d..7c74e3f81 100644
--- a/scanner/scanner_test.go
+++ b/scanner/scanner_test.go
@@ -10,6 +10,7 @@ import (
"github.com/google/uuid"
"github.com/navidrome/navidrome/conf"
"github.com/navidrome/navidrome/conf/configtest"
+ "github.com/navidrome/navidrome/consts"
"github.com/navidrome/navidrome/core"
"github.com/navidrome/navidrome/core/artwork"
"github.com/navidrome/navidrome/core/metrics"
@@ -17,6 +18,7 @@ import (
"github.com/navidrome/navidrome/db"
"github.com/navidrome/navidrome/log"
"github.com/navidrome/navidrome/model"
+ "github.com/navidrome/navidrome/model/request"
"github.com/navidrome/navidrome/persistence"
"github.com/navidrome/navidrome/scanner"
"github.com/navidrome/navidrome/server/events"
@@ -47,6 +49,7 @@ var _ = Describe("Scanner", Ordered, func() {
}
BeforeAll(func() {
+ ctx = request.WithUser(GinkgoT().Context(), model.User{ID: "123", IsAdmin: true})
tmpDir := GinkgoT().TempDir()
conf.Server.DbPath = filepath.Join(tmpDir, "test-scanner.db?_journal_mode=WAL")
log.Warn("Using DB at " + conf.Server.DbPath)
@@ -54,7 +57,6 @@ var _ = Describe("Scanner", Ordered, func() {
})
BeforeEach(func() {
- ctx = context.Background()
db.Init(ctx)
DeferCleanup(func() {
Expect(tests.ClearDB()).To(Succeed())
@@ -501,6 +503,113 @@ var _ = Describe("Scanner", Ordered, func() {
Expect(aa[0].MbzArtistID).To(Equal(beatlesMBID))
Expect(aa[0].SortArtistName).To(Equal("Beatles, The"))
})
+
+ Context("When PurgeMissing is configured", func() {
+ When("PurgeMissing is set to 'never'", func() {
+ BeforeEach(func() {
+ DeferCleanup(configtest.SetupConfig())
+ conf.Server.Scanner.PurgeMissing = consts.PurgeMissingNever
+ })
+
+ It("should mark files as missing but not delete them", func() {
+ By("Running initial scan")
+ Expect(runScanner(ctx, true)).To(Succeed())
+
+ By("Removing a file")
+ fsys.Remove("The Beatles/Revolver/02 - Eleanor Rigby.mp3")
+
+ By("Running another scan")
+ Expect(runScanner(ctx, true)).To(Succeed())
+
+ By("Checking files are marked as missing but not deleted")
+ count, err := ds.MediaFile(ctx).CountAll(model.QueryOptions{
+ Filters: squirrel.Eq{"missing": true},
+ })
+ Expect(err).ToNot(HaveOccurred())
+ Expect(count).To(Equal(int64(1)))
+
+ mf, err := findByPath("The Beatles/Revolver/02 - Eleanor Rigby.mp3")
+ Expect(err).ToNot(HaveOccurred())
+ Expect(mf.Missing).To(BeTrue())
+ })
+ })
+
+ When("PurgeMissing is set to 'always'", func() {
+ BeforeEach(func() {
+ conf.Server.Scanner.PurgeMissing = consts.PurgeMissingAlways
+ })
+
+ It("should purge missing files on any scan", func() {
+ By("Running initial scan")
+ Expect(runScanner(ctx, false)).To(Succeed())
+
+ By("Removing a file")
+ fsys.Remove("The Beatles/Revolver/02 - Eleanor Rigby.mp3")
+
+ By("Running an incremental scan")
+ Expect(runScanner(ctx, false)).To(Succeed())
+
+ By("Checking missing files are deleted")
+ count, err := ds.MediaFile(ctx).CountAll(model.QueryOptions{
+ Filters: squirrel.Eq{"missing": true},
+ })
+ Expect(err).ToNot(HaveOccurred())
+ Expect(count).To(BeZero())
+
+ _, err = findByPath("The Beatles/Revolver/02 - Eleanor Rigby.mp3")
+ Expect(err).To(MatchError(model.ErrNotFound))
+ })
+ })
+
+ When("PurgeMissing is set to 'full'", func() {
+ BeforeEach(func() {
+ conf.Server.Scanner.PurgeMissing = consts.PurgeMissingFull
+ })
+
+ It("should not purge missing files on incremental scans", func() {
+ By("Running initial scan")
+ Expect(runScanner(ctx, true)).To(Succeed())
+
+ By("Removing a file")
+ fsys.Remove("The Beatles/Revolver/02 - Eleanor Rigby.mp3")
+
+ By("Running an incremental scan")
+ Expect(runScanner(ctx, false)).To(Succeed())
+
+ By("Checking files are marked as missing but not deleted")
+ count, err := ds.MediaFile(ctx).CountAll(model.QueryOptions{
+ Filters: squirrel.Eq{"missing": true},
+ })
+ Expect(err).ToNot(HaveOccurred())
+ Expect(count).To(Equal(int64(1)))
+
+ mf, err := findByPath("The Beatles/Revolver/02 - Eleanor Rigby.mp3")
+ Expect(err).ToNot(HaveOccurred())
+ Expect(mf.Missing).To(BeTrue())
+ })
+
+ It("should purge missing files only on full scans", func() {
+ By("Running initial scan")
+ Expect(runScanner(ctx, true)).To(Succeed())
+
+ By("Removing a file")
+ fsys.Remove("The Beatles/Revolver/02 - Eleanor Rigby.mp3")
+
+ By("Running a full scan")
+ Expect(runScanner(ctx, true)).To(Succeed())
+
+ By("Checking missing files are deleted")
+ count, err := ds.MediaFile(ctx).CountAll(model.QueryOptions{
+ Filters: squirrel.Eq{"missing": true},
+ })
+ Expect(err).ToNot(HaveOccurred())
+ Expect(count).To(BeZero())
+
+ _, err = findByPath("The Beatles/Revolver/02 - Eleanor Rigby.mp3")
+ Expect(err).To(MatchError(model.ErrNotFound))
+ })
+ })
+ })
})
})
diff --git a/tests/mock_mediafile_repo.go b/tests/mock_mediafile_repo.go
index 01d82e03b..85adb8a25 100644
--- a/tests/mock_mediafile_repo.go
+++ b/tests/mock_mediafile_repo.go
@@ -22,6 +22,10 @@ type MockMediaFileRepo struct {
model.MediaFileRepository
Data map[string]*model.MediaFile
Err bool
+ // Add fields and methods for controlling CountAll and DeleteAllMissing in tests
+ CountAllValue int64
+ CountAllOptions model.QueryOptions
+ DeleteAllMissingValue int64
}
func (m *MockMediaFileRepo) SetError(err bool) {
@@ -161,4 +165,35 @@ func (m *MockMediaFileRepo) GetMissingAndMatching(libId int) (model.MediaFileCur
}, nil
}
+func (m *MockMediaFileRepo) CountAll(opts ...model.QueryOptions) (int64, error) {
+ if m.Err {
+ return 0, errors.New("error")
+ }
+ if m.CountAllValue != 0 {
+ if len(opts) > 0 {
+ m.CountAllOptions = opts[0]
+ }
+ return m.CountAllValue, nil
+ }
+ return int64(len(m.Data)), nil
+}
+
+func (m *MockMediaFileRepo) DeleteAllMissing() (int64, error) {
+ if m.Err {
+ return 0, errors.New("error")
+ }
+ if m.DeleteAllMissingValue != 0 {
+ return m.DeleteAllMissingValue, nil
+ }
+ // Remove all missing files from Data
+ var count int64
+ for id, mf := range m.Data {
+ if mf.Missing {
+ delete(m.Data, id)
+ count++
+ }
+ }
+ return count, nil
+}
+
var _ model.MediaFileRepository = (*MockMediaFileRepo)(nil)