🎧 Your Personal Streaming Service
Go airsonicmadsonicmedia-servermusicmusic-servernavidromeopensubsonicraspberry-piself-hostedstreamerstreaming-apistreaming-audiosubsonicsubsonic-server
feat(scanner): add Scanner.IgnoreDotFolders to allow indexing dot-prefixed folders (#5568)
Deluan Quintão05105e91d90d
diff --git a/conf/configuration.go b/conf/configuration.goindex 2ae6e84ca..665a7992f 100644--- a/conf/configuration.go+++ b/conf/configuration.go@@ -162,6 +162,7 @@ type scannerOptions struct {GenreSeparators string // Deprecated: Use Tags.genre.Split insteadGroupAlbumReleases bool // Deprecated: Use PID.Album insteadFollowSymlinks bool // Whether to follow symlinks when scanning directories+ IgnoreDotFolders bool // Whether to ignore folders whose name starts with a dot when scanningPurgeMissing string // Values: "never", "always", "full"}@@ -821,6 +822,7 @@ func setViperDefaults() {viper.SetDefault("scanner.genreseparators", "")viper.SetDefault("scanner.groupalbumreleases", false)viper.SetDefault("scanner.followsymlinks", true)+ viper.SetDefault("scanner.ignoredotfolders", true)viper.SetDefault("scanner.purgemissing", consts.PurgeMissingNever)viper.SetDefault("subsonic.appendsubtitle", true)viper.SetDefault("subsonic.appendalbumversion", true)diff --git a/scanner/walk_dir_tree.go b/scanner/walk_dir_tree.goindex 78796ac5f..55bbab684 100644--- a/scanner/walk_dir_tree.go+++ b/scanner/walk_dir_tree.go@@ -123,9 +123,6 @@ func loadDir(ctx context.Context, job *scanJob, dirPath string, checker *IgnoreClog.Trace(ctx, "Scanner: Ignoring entry", "path", entryPath)continue}- if isEntryIgnored(entry.Name()) {- continue- }if ctx.Err() != nil {return folder, children, ctx.Err()}@@ -135,7 +132,10 @@ func loadDir(ctx context.Context, job *scanJob, dirPath string, checker *IgnoreClog.Warn(ctx, "Scanner: Invalid symlink", "dir", entryPath, err)continue}- if isDir && !isDirIgnored(entry.Name()) && isDirReadable(ctx, job.fs, entryPath) {+ if isIgnoredEntry(entry.Name(), isDir) {+ continue+ }+ if isDir && isDirReadable(ctx, job.fs, entryPath) {children = append(children, entryPath)folder.numSubFolders++} else {@@ -276,22 +276,35 @@ var ignoredDirs = []string{"#snapshot","@Recycle","@Recently-Snapshot",+ ".git",".streams","lost+found",}-// isDirIgnored returns true if the directory represented by dirEnt should be ignored-func isDirIgnored(name string) bool {- // allows Album folders for albums which eg start with ellipses- if strings.HasPrefix(name, ".") && !strings.HasPrefix(name, "..") {+// isIgnoredEntry returns true if a directory entry with the given name should be+// skipped during scanning. It centralizes all name- and type-based ignore policy:+// - special system directories in ignoredDirs are always ignored;+// - dot-prefixed files are always ignored;+// - dot-prefixed folders are ignored unless Scanner.IgnoreDotFolders is disabled,+// allowing albums like ".Hack Sign" to be scanned when the option is off.+func isIgnoredEntry(name string, isDir bool) bool {+ if isDir && isDirIgnored(name) {return true}- if slices.ContainsFunc(ignoredDirs, func(s string) bool { return strings.EqualFold(s, name) }) {- return true- }- return false+ return isDotEntry(name) && (!isDir || conf.Server.Scanner.IgnoreDotFolders)+}++// isDirIgnored returns true if the directory name is in the explicit ignoredDirs+// blocklist. Used both while walking the tree and by the file watcher.+func isDirIgnored(name string) bool {+ return slices.ContainsFunc(ignoredDirs, func(s string) bool { return strings.EqualFold(s, name) })}-func isEntryIgnored(name string) bool {- return strings.HasPrefix(name, ".") && !strings.HasPrefix(name, "..")+// isDotEntry returns true only for names with exactly one leading dot (the+// convention for hidden entries), e.g. ".hidden". Names with two or more leading+// dots are not considered hidden: "." and ".." are the special self/parent+// references, and anything like "..foo" or "...Album" is a regular name (album+// folders sometimes start with ellipses), so all of these return false.+func isDotEntry(name string) bool {+ return name != "." && strings.HasPrefix(name, ".") && !strings.HasPrefix(name, "..")}diff --git a/scanner/walk_dir_tree_test.go b/scanner/walk_dir_tree_test.goindex 95cbba88f..f3b13a4ef 100644--- a/scanner/walk_dir_tree_test.go+++ b/scanner/walk_dir_tree_test.go@@ -49,6 +49,10 @@ var _ = Describe("walk_dir_tree", func() {"root/f/legit.mp3": {Mode: fs.ModeSymlink, Data: []byte("realsong.mp3")},"root/f/secret": {Data: []byte("TOPSECRET")},"root/f/evil.mp3": {Mode: fs.ModeSymlink, Data: []byte("secret")},+ "root/g/.Hack Sign Original Soundtrack/track.mp3": {},+ "root/h/.hidden.mp3": {},+ "root/i/.git/config": {},+ "root/i/.streams/stream.mp3": {},},}job = &scanJob{@@ -97,6 +101,14 @@ var _ = Describe("walk_dir_tree", func() {Expect(folders["root/c"].imageFiles).To(BeEmpty())Expect(folders).ToNot(HaveKey("root/d"))+ // By default (Scanner.IgnoreDotFolders == true), dot-prefixed+ // folders are skipped, dot-prefixed files are not indexed, and+ // the special ignoredDirs (.git, .streams) are never traversed.+ Expect(folders).ToNot(HaveKey("root/g/.Hack Sign Original Soundtrack"))+ Expect(folders["root/h"].audioFiles).To(BeEmpty())+ Expect(folders).ToNot(HaveKey("root/i/.git"))+ Expect(folders).ToNot(HaveKey("root/i/.streams"))+// Symlink specific checksif followSymlinks {Expect(folders["root/e/symlink"].audioFiles).To(HaveLen(1))@@ -110,8 +122,31 @@ var _ = Describe("walk_dir_tree", func() {Expect(folders["root/f"].audioFiles).ToNot(HaveKey("evil.mp3"))}},- Entry("with symlinks enabled", true, 8),- Entry("with symlinks disabled", false, 7),+ Entry("with symlinks enabled", true, 11),+ Entry("with symlinks disabled", false, 10),+ )++ DescribeTable("dot-prefixed folders with IgnoreDotFolders disabled",+ func(followSymlinks bool) {+ conf.Server.Scanner.FollowSymlinks = followSymlinks+ conf.Server.Scanner.IgnoreDotFolders = false+ folders := getFolders()++ // Dot-prefixed album folders are now traversed and indexed+ Expect(folders["root/g/.Hack Sign Original Soundtrack"].audioFiles).To(SatisfyAll(+ HaveLen(1),+ HaveKey("track.mp3"),+ ))++ // Dot-prefixed files are still ignored, even with the flag off+ Expect(folders["root/h"].audioFiles).To(BeEmpty())++ // Special ignoredDirs remain blocked regardless of the flag+ Expect(folders).ToNot(HaveKey("root/i/.git"))+ Expect(folders).ToNot(HaveKey("root/i/.streams"))+ },+ Entry("with symlinks enabled", true),+ Entry("with symlinks disabled", false),)})@@ -450,13 +485,61 @@ var _ = Describe("walk_dir_tree", func() {Expect(isDirIgnored(dirName)).To(Equal(expected))},Entry("normal dir", "empty_folder", false),- Entry("hidden dir", ".hidden_folder", true),+ Entry("dot-prefixed album dir", ".Hack Sign Original Soundtrack", false),+ Entry("git dir", ".git", true),+ Entry("streams dir", ".streams", true),Entry("dir starting with ellipsis", "...unhidden_folder", false),Entry("recycle bin", "$Recycle.Bin", true),Entry("snapshot dir", "#snapshot", true),)})+ Describe("isIgnoredEntry", func() {+ BeforeEach(func() {+ DeferCleanup(configtest.SetupConfig())+ })++ DescribeTable("with IgnoreDotFolders enabled (default)",+ func(name string, isDir, expected bool) {+ conf.Server.Scanner.IgnoreDotFolders = true+ Expect(isIgnoredEntry(name, isDir)).To(Equal(expected))+ },+ Entry("normal dir", "Album", true, false),+ Entry("normal file", "track.mp3", false, false),+ Entry("dot folder", ".Hack Sign Original Soundtrack", true, true),+ Entry("dot file", ".hidden.mp3", false, true),+ Entry("blocklisted dir", ".git", true, true),+ Entry("ellipsis dir", "...unhidden", true, false),+ )++ DescribeTable("with IgnoreDotFolders disabled",+ func(name string, isDir, expected bool) {+ conf.Server.Scanner.IgnoreDotFolders = false+ Expect(isIgnoredEntry(name, isDir)).To(Equal(expected))+ },+ Entry("normal dir", "Album", true, false),+ Entry("normal file", "track.mp3", false, false),+ Entry("dot folder is allowed", ".Hack Sign Original Soundtrack", true, false),+ Entry("dot file is still ignored", ".hidden.mp3", false, true),+ Entry("blocklisted dir still ignored", ".git", true, true),+ )+ })++ Describe("isDotEntry", func() {+ DescribeTable("returns expected result",+ func(name string, expected bool) {+ Expect(isDotEntry(name)).To(Equal(expected))+ },+ Entry("dot folder", ".Hidden", true),+ Entry("dot file", ".hidden.mp3", true),+ Entry("current dir", ".", false),+ Entry("parent dir", "..", false),+ Entry("two leading dots", "..foo", false),+ Entry("ellipsis", "...unhidden", false),+ Entry("normal name", "Album", false),+ )+ })+Describe("fullReadDir", func() {var (fsys fakeFSdiff --git a/scanner/watcher.go b/scanner/watcher.goindex 376db910c..baf94b79b 100644--- a/scanner/watcher.go+++ b/scanner/watcher.go@@ -5,6 +5,7 @@ import ("fmt""io/fs""path/filepath"+ "strings""sync""time"@@ -320,18 +321,35 @@ func (w *watcher) shouldIgnoreFolderPath(ctx context.Context, fsys storage.Music}func isIgnoredPath(_ context.Context, _ fs.FS, path string) bool {- baseDir, name := filepath.Split(path)+ _, name := filepath.Split(path)+ // A change anywhere inside an ignored directory (a dot-folder when+ // Scanner.IgnoreDotFolders is enabled, or a special system folder) must not+ // trigger a scan, even for media files: the scan would skip it anyway.+ if isUnderIgnoredDir(path) {+ return true+ }switch {- case model.IsAudioFile(path):- return false- case model.IsValidPlaylist(path):- return false- case model.IsImageFile(path):- return false+ case model.IsAudioFile(path), model.IsValidPlaylist(path), model.IsImageFile(path):+ // A media file is normally not ignored, but a dot-prefixed one (e.g.+ // ".hidden.mp3") is always skipped by the scanner, so don't scan for it.+ return isDotEntry(name)case name == ".DS_Store":return true}- // As it can be a deletion and not a change, we cannot reliably know if the path is a file or directory.- // But at this point, we can assume it's a directory. If it's a file, it would be ignored anyway- return isDirIgnored(baseDir)+ // As it can be a deletion and not a change, we cannot reliably know if the+ // path is a file or directory. But at this point, we can assume it's a+ // directory. If it's a file, it would be ignored anyway.+ return isIgnoredEntry(name, true)+}++// isUnderIgnoredDir returns true if any parent directory component of the given+// path is an ignored directory, reusing the same policy as the scanner walk.+func isUnderIgnoredDir(path string) bool {+ dir, _ := filepath.Split(path)+ for part := range strings.SplitSeq(filepath.ToSlash(dir), "/") {+ if part != "" && isIgnoredEntry(part, true) {+ return true+ }+ }+ return false}diff --git a/scanner/watcher_test.go b/scanner/watcher_test.goindex 9795129b0..ffe9f8b15 100644--- a/scanner/watcher_test.go+++ b/scanner/watcher_test.go@@ -428,6 +428,50 @@ var _ = Describe("Watcher", func() {Expect(w.watcherNotify).To(BeEmpty(), "Expected no scan notification for file in ignored folder")})})++ })+})++var _ = Describe("isIgnoredPath", func() {+ BeforeEach(func() {+ DeferCleanup(configtest.SetupConfig())+ })++ Context("with IgnoreDotFolders enabled (default)", func() {+ BeforeEach(func() {+ conf.Server.Scanner.IgnoreDotFolders = true+ })++ DescribeTable("returns expected result",+ func(p string, expected bool) {+ Expect(isIgnoredPath(context.Background(), nil, filepath.FromSlash(p))).To(Equal(expected))+ },+ Entry("media file in normal folder", "rock/Album/track.mp3", false),+ Entry("dot-prefixed media file", "rock/Album/.hidden.mp3", true),+ Entry("media file inside a dot-folder", "rock/.Hidden Album/track.mp3", true),+ Entry("media file inside a blocklisted folder", "rock/.streams/stream.mp3", true),+ Entry("media file inside .git", "rock/.git/track.mp3", true),+ Entry("dot-folder itself", "rock/.Hidden Album", true),+ Entry("normal folder itself", "rock/Album", false),+ Entry(".DS_Store file", "rock/Album/.DS_Store", true),+ )+ })++ Context("with IgnoreDotFolders disabled", func() {+ BeforeEach(func() {+ conf.Server.Scanner.IgnoreDotFolders = false+ })++ DescribeTable("returns expected result",+ func(p string, expected bool) {+ Expect(isIgnoredPath(context.Background(), nil, filepath.FromSlash(p))).To(Equal(expected))+ },+ Entry("media file inside a dot-folder is allowed", "rock/.Hidden Album/track.mp3", false),+ Entry("dot-prefixed media file is still ignored", "rock/Album/.hidden.mp3", true),+ Entry("dot-folder itself is allowed", "rock/.Hidden Album", false),+ Entry("blocklisted folder still ignored", "rock/.streams/stream.mp3", true),+ Entry(".git still ignored", "rock/.git/config", true),+ )})})