Skip to content

🎧 Your Personal Streaming Service

feat(scanner): add Scanner.IgnoreDotFolders to allow indexing dot-prefixed folders (#5568)

Deluan Quintão05105e91d90d
diff --git a/conf/configuration.go b/conf/configuration.go
index 2ae6e84ca..665a7992f 100644
--- a/conf/configuration.go
+++ b/conf/configuration.go
@@ -162,6 +162,7 @@ type scannerOptions struct {
GenreSeparators string // Deprecated: Use Tags.genre.Split instead
GroupAlbumReleases bool // Deprecated: Use PID.Album instead
FollowSymlinks bool // Whether to follow symlinks when scanning directories
+ IgnoreDotFolders bool // Whether to ignore folders whose name starts with a dot when scanning
PurgeMissing string // Values: "never", "always", "full"
}
@@ -821,6 +822,7 @@ func setViperDefaults() {
viper.SetDefault("scanner.genreseparators", "")
viper.SetDefault("scanner.groupalbumreleases", false)
viper.SetDefault("scanner.followsymlinks", true)
+ viper.SetDefault("scanner.ignoredotfolders", true)
viper.SetDefault("scanner.purgemissing", consts.PurgeMissingNever)
viper.SetDefault("subsonic.appendsubtitle", true)
viper.SetDefault("subsonic.appendalbumversion", true)
diff --git a/scanner/walk_dir_tree.go b/scanner/walk_dir_tree.go
index 78796ac5f..55bbab684 100644
--- a/scanner/walk_dir_tree.go
+++ b/scanner/walk_dir_tree.go
@@ -123,9 +123,6 @@ func loadDir(ctx context.Context, job *scanJob, dirPath string, checker *IgnoreC
log.Trace(ctx, "Scanner: Ignoring entry", "path", entryPath)
continue
}
- if isEntryIgnored(entry.Name()) {
- continue
- }
if ctx.Err() != nil {
return folder, children, ctx.Err()
}
@@ -135,7 +132,10 @@ func loadDir(ctx context.Context, job *scanJob, dirPath string, checker *IgnoreC
log.Warn(ctx, "Scanner: Invalid symlink", "dir", entryPath, err)
continue
}
- if isDir && !isDirIgnored(entry.Name()) && isDirReadable(ctx, job.fs, entryPath) {
+ if isIgnoredEntry(entry.Name(), isDir) {
+ continue
+ }
+ if isDir && isDirReadable(ctx, job.fs, entryPath) {
children = append(children, entryPath)
folder.numSubFolders++
} else {
@@ -276,22 +276,35 @@ var ignoredDirs = []string{
"#snapshot",
"@Recycle",
"@Recently-Snapshot",
+ ".git",
".streams",
"lost+found",
}
-// isDirIgnored returns true if the directory represented by dirEnt should be ignored
-func isDirIgnored(name string) bool {
- // allows Album folders for albums which eg start with ellipses
- if strings.HasPrefix(name, ".") && !strings.HasPrefix(name, "..") {
+// isIgnoredEntry returns true if a directory entry with the given name should be
+// skipped during scanning. It centralizes all name- and type-based ignore policy:
+// - special system directories in ignoredDirs are always ignored;
+// - dot-prefixed files are always ignored;
+// - dot-prefixed folders are ignored unless Scanner.IgnoreDotFolders is disabled,
+// allowing albums like ".Hack Sign" to be scanned when the option is off.
+func isIgnoredEntry(name string, isDir bool) bool {
+ if isDir && isDirIgnored(name) {
return true
}
- if slices.ContainsFunc(ignoredDirs, func(s string) bool { return strings.EqualFold(s, name) }) {
- return true
- }
- return false
+ return isDotEntry(name) && (!isDir || conf.Server.Scanner.IgnoreDotFolders)
+}
+
+// isDirIgnored returns true if the directory name is in the explicit ignoredDirs
+// blocklist. Used both while walking the tree and by the file watcher.
+func isDirIgnored(name string) bool {
+ return slices.ContainsFunc(ignoredDirs, func(s string) bool { return strings.EqualFold(s, name) })
}
-func isEntryIgnored(name string) bool {
- return strings.HasPrefix(name, ".") && !strings.HasPrefix(name, "..")
+// isDotEntry returns true only for names with exactly one leading dot (the
+// convention for hidden entries), e.g. ".hidden". Names with two or more leading
+// dots are not considered hidden: "." and ".." are the special self/parent
+// references, and anything like "..foo" or "...Album" is a regular name (album
+// folders sometimes start with ellipses), so all of these return false.
+func isDotEntry(name string) bool {
+ return name != "." && strings.HasPrefix(name, ".") && !strings.HasPrefix(name, "..")
}
diff --git a/scanner/walk_dir_tree_test.go b/scanner/walk_dir_tree_test.go
index 95cbba88f..f3b13a4ef 100644
--- a/scanner/walk_dir_tree_test.go
+++ b/scanner/walk_dir_tree_test.go
@@ -49,6 +49,10 @@ var _ = Describe("walk_dir_tree", func() {
"root/f/legit.mp3": {Mode: fs.ModeSymlink, Data: []byte("realsong.mp3")},
"root/f/secret": {Data: []byte("TOPSECRET")},
"root/f/evil.mp3": {Mode: fs.ModeSymlink, Data: []byte("secret")},
+ "root/g/.Hack Sign Original Soundtrack/track.mp3": {},
+ "root/h/.hidden.mp3": {},
+ "root/i/.git/config": {},
+ "root/i/.streams/stream.mp3": {},
},
}
job = &scanJob{
@@ -97,6 +101,14 @@ var _ = Describe("walk_dir_tree", func() {
Expect(folders["root/c"].imageFiles).To(BeEmpty())
Expect(folders).ToNot(HaveKey("root/d"))
+ // By default (Scanner.IgnoreDotFolders == true), dot-prefixed
+ // folders are skipped, dot-prefixed files are not indexed, and
+ // the special ignoredDirs (.git, .streams) are never traversed.
+ Expect(folders).ToNot(HaveKey("root/g/.Hack Sign Original Soundtrack"))
+ Expect(folders["root/h"].audioFiles).To(BeEmpty())
+ Expect(folders).ToNot(HaveKey("root/i/.git"))
+ Expect(folders).ToNot(HaveKey("root/i/.streams"))
+
// Symlink specific checks
if followSymlinks {
Expect(folders["root/e/symlink"].audioFiles).To(HaveLen(1))
@@ -110,8 +122,31 @@ var _ = Describe("walk_dir_tree", func() {
Expect(folders["root/f"].audioFiles).ToNot(HaveKey("evil.mp3"))
}
},
- Entry("with symlinks enabled", true, 8),
- Entry("with symlinks disabled", false, 7),
+ Entry("with symlinks enabled", true, 11),
+ Entry("with symlinks disabled", false, 10),
+ )
+
+ DescribeTable("dot-prefixed folders with IgnoreDotFolders disabled",
+ func(followSymlinks bool) {
+ conf.Server.Scanner.FollowSymlinks = followSymlinks
+ conf.Server.Scanner.IgnoreDotFolders = false
+ folders := getFolders()
+
+ // Dot-prefixed album folders are now traversed and indexed
+ Expect(folders["root/g/.Hack Sign Original Soundtrack"].audioFiles).To(SatisfyAll(
+ HaveLen(1),
+ HaveKey("track.mp3"),
+ ))
+
+ // Dot-prefixed files are still ignored, even with the flag off
+ Expect(folders["root/h"].audioFiles).To(BeEmpty())
+
+ // Special ignoredDirs remain blocked regardless of the flag
+ Expect(folders).ToNot(HaveKey("root/i/.git"))
+ Expect(folders).ToNot(HaveKey("root/i/.streams"))
+ },
+ Entry("with symlinks enabled", true),
+ Entry("with symlinks disabled", false),
)
})
@@ -450,13 +485,61 @@ var _ = Describe("walk_dir_tree", func() {
Expect(isDirIgnored(dirName)).To(Equal(expected))
},
Entry("normal dir", "empty_folder", false),
- Entry("hidden dir", ".hidden_folder", true),
+ Entry("dot-prefixed album dir", ".Hack Sign Original Soundtrack", false),
+ Entry("git dir", ".git", true),
+ Entry("streams dir", ".streams", true),
Entry("dir starting with ellipsis", "...unhidden_folder", false),
Entry("recycle bin", "$Recycle.Bin", true),
Entry("snapshot dir", "#snapshot", true),
)
})
+ Describe("isIgnoredEntry", func() {
+ BeforeEach(func() {
+ DeferCleanup(configtest.SetupConfig())
+ })
+
+ DescribeTable("with IgnoreDotFolders enabled (default)",
+ func(name string, isDir, expected bool) {
+ conf.Server.Scanner.IgnoreDotFolders = true
+ Expect(isIgnoredEntry(name, isDir)).To(Equal(expected))
+ },
+ Entry("normal dir", "Album", true, false),
+ Entry("normal file", "track.mp3", false, false),
+ Entry("dot folder", ".Hack Sign Original Soundtrack", true, true),
+ Entry("dot file", ".hidden.mp3", false, true),
+ Entry("blocklisted dir", ".git", true, true),
+ Entry("ellipsis dir", "...unhidden", true, false),
+ )
+
+ DescribeTable("with IgnoreDotFolders disabled",
+ func(name string, isDir, expected bool) {
+ conf.Server.Scanner.IgnoreDotFolders = false
+ Expect(isIgnoredEntry(name, isDir)).To(Equal(expected))
+ },
+ Entry("normal dir", "Album", true, false),
+ Entry("normal file", "track.mp3", false, false),
+ Entry("dot folder is allowed", ".Hack Sign Original Soundtrack", true, false),
+ Entry("dot file is still ignored", ".hidden.mp3", false, true),
+ Entry("blocklisted dir still ignored", ".git", true, true),
+ )
+ })
+
+ Describe("isDotEntry", func() {
+ DescribeTable("returns expected result",
+ func(name string, expected bool) {
+ Expect(isDotEntry(name)).To(Equal(expected))
+ },
+ Entry("dot folder", ".Hidden", true),
+ Entry("dot file", ".hidden.mp3", true),
+ Entry("current dir", ".", false),
+ Entry("parent dir", "..", false),
+ Entry("two leading dots", "..foo", false),
+ Entry("ellipsis", "...unhidden", false),
+ Entry("normal name", "Album", false),
+ )
+ })
+
Describe("fullReadDir", func() {
var (
fsys fakeFS
diff --git a/scanner/watcher.go b/scanner/watcher.go
index 376db910c..baf94b79b 100644
--- a/scanner/watcher.go
+++ b/scanner/watcher.go
@@ -5,6 +5,7 @@ import (
"fmt"
"io/fs"
"path/filepath"
+ "strings"
"sync"
"time"
@@ -320,18 +321,35 @@ func (w *watcher) shouldIgnoreFolderPath(ctx context.Context, fsys storage.Music
}
func isIgnoredPath(_ context.Context, _ fs.FS, path string) bool {
- baseDir, name := filepath.Split(path)
+ _, name := filepath.Split(path)
+ // A change anywhere inside an ignored directory (a dot-folder when
+ // Scanner.IgnoreDotFolders is enabled, or a special system folder) must not
+ // trigger a scan, even for media files: the scan would skip it anyway.
+ if isUnderIgnoredDir(path) {
+ return true
+ }
switch {
- case model.IsAudioFile(path):
- return false
- case model.IsValidPlaylist(path):
- return false
- case model.IsImageFile(path):
- return false
+ case model.IsAudioFile(path), model.IsValidPlaylist(path), model.IsImageFile(path):
+ // A media file is normally not ignored, but a dot-prefixed one (e.g.
+ // ".hidden.mp3") is always skipped by the scanner, so don't scan for it.
+ return isDotEntry(name)
case name == ".DS_Store":
return true
}
- // As it can be a deletion and not a change, we cannot reliably know if the path is a file or directory.
- // But at this point, we can assume it's a directory. If it's a file, it would be ignored anyway
- return isDirIgnored(baseDir)
+ // As it can be a deletion and not a change, we cannot reliably know if the
+ // path is a file or directory. But at this point, we can assume it's a
+ // directory. If it's a file, it would be ignored anyway.
+ return isIgnoredEntry(name, true)
+}
+
+// isUnderIgnoredDir returns true if any parent directory component of the given
+// path is an ignored directory, reusing the same policy as the scanner walk.
+func isUnderIgnoredDir(path string) bool {
+ dir, _ := filepath.Split(path)
+ for part := range strings.SplitSeq(filepath.ToSlash(dir), "/") {
+ if part != "" && isIgnoredEntry(part, true) {
+ return true
+ }
+ }
+ return false
}
diff --git a/scanner/watcher_test.go b/scanner/watcher_test.go
index 9795129b0..ffe9f8b15 100644
--- a/scanner/watcher_test.go
+++ b/scanner/watcher_test.go
@@ -428,6 +428,50 @@ var _ = Describe("Watcher", func() {
Expect(w.watcherNotify).To(BeEmpty(), "Expected no scan notification for file in ignored folder")
})
})
+
+ })
+})
+
+var _ = Describe("isIgnoredPath", func() {
+ BeforeEach(func() {
+ DeferCleanup(configtest.SetupConfig())
+ })
+
+ Context("with IgnoreDotFolders enabled (default)", func() {
+ BeforeEach(func() {
+ conf.Server.Scanner.IgnoreDotFolders = true
+ })
+
+ DescribeTable("returns expected result",
+ func(p string, expected bool) {
+ Expect(isIgnoredPath(context.Background(), nil, filepath.FromSlash(p))).To(Equal(expected))
+ },
+ Entry("media file in normal folder", "rock/Album/track.mp3", false),
+ Entry("dot-prefixed media file", "rock/Album/.hidden.mp3", true),
+ Entry("media file inside a dot-folder", "rock/.Hidden Album/track.mp3", true),
+ Entry("media file inside a blocklisted folder", "rock/.streams/stream.mp3", true),
+ Entry("media file inside .git", "rock/.git/track.mp3", true),
+ Entry("dot-folder itself", "rock/.Hidden Album", true),
+ Entry("normal folder itself", "rock/Album", false),
+ Entry(".DS_Store file", "rock/Album/.DS_Store", true),
+ )
+ })
+
+ Context("with IgnoreDotFolders disabled", func() {
+ BeforeEach(func() {
+ conf.Server.Scanner.IgnoreDotFolders = false
+ })
+
+ DescribeTable("returns expected result",
+ func(p string, expected bool) {
+ Expect(isIgnoredPath(context.Background(), nil, filepath.FromSlash(p))).To(Equal(expected))
+ },
+ Entry("media file inside a dot-folder is allowed", "rock/.Hidden Album/track.mp3", false),
+ Entry("dot-prefixed media file is still ignored", "rock/Album/.hidden.mp3", true),
+ Entry("dot-folder itself is allowed", "rock/.Hidden Album", false),
+ Entry("blocklisted folder still ignored", "rock/.streams/stream.mp3", true),
+ Entry(".git still ignored", "rock/.git/config", true),
+ )
})
})