Skip to content

🎧 Your Personal Streaming Service

Reapply the fix from #1054, but without getting into an infinite look in case of SMB fs errors. See #1164

Deluane61cf3217d08
diff --git a/scanner/walk_dir_tree.go b/scanner/walk_dir_tree.go
index b80ede082..77db41828 100644
--- a/scanner/walk_dir_tree.go
+++ b/scanner/walk_dir_tree.go
@@ -66,11 +66,14 @@ func loadDir(ctx context.Context, dirPath string) ([]string, *dirStats, error) {
}
stats.ModTime = dirInfo.ModTime()
- dirEntries, err := fullReadDir(dirPath)
+ dir, err := os.Open(dirPath)
if err != nil {
- log.Error(ctx, "Error in ReadDir", "path", dirPath, err)
+ log.Error(ctx, "Error in Opening directory", "path", dirPath, err)
return children, stats, err
}
+ defer dir.Close()
+
+ dirEntries := fullReadDir(ctx, dir)
for _, entry := range dirEntries {
isDir, err := isDirOrSymlinkToDir(dirPath, entry)
// Skip invalid symlinks
@@ -100,20 +103,28 @@ func loadDir(ctx context.Context, dirPath string) ([]string, *dirStats, error) {
return children, stats, nil
}
-func fullReadDir(name string) ([]os.DirEntry, error) {
- f, err := os.Open(name)
- if err != nil {
- return nil, err
- }
- defer f.Close()
-
- dirs, err := f.ReadDir(-1)
- if err != nil {
- log.Warn("Skipping DirEntry", err)
- return nil, nil
+// fullReadDir reads all files in the folder, skipping the ones with errors.
+// It also detects when it is "stuck" with an error in the same directory over and over.
+// In this case, it and returns whatever it was able to read until it got stuck.
+// See discussion here: https://github.com/navidrome/navidrome/issues/1164#issuecomment-881922850
+func fullReadDir(ctx context.Context, dir fs.ReadDirFile) []os.DirEntry {
+ var allDirs []os.DirEntry
+ var prevErrStr = ""
+ for {
+ dirs, err := dir.ReadDir(-1)
+ allDirs = append(allDirs, dirs...)
+ if err == nil {
+ break
+ }
+ log.Warn(ctx, "Skipping DirEntry", err)
+ if prevErrStr == err.Error() {
+ log.Error(ctx, "Duplicate DirEntry failure, bailing", err)
+ break
+ }
+ prevErrStr = err.Error()
}
- sort.Slice(dirs, func(i, j int) bool { return dirs[i].Name() < dirs[j].Name() })
- return dirs, nil
+ sort.Slice(allDirs, func(i, j int) bool { return allDirs[i].Name() < allDirs[j].Name() })
+ return allDirs
}
// isDirOrSymlinkToDir returns true if and only if the dirEnt represents a file
diff --git a/scanner/walk_dir_tree_test.go b/scanner/walk_dir_tree_test.go
index 6c8464bfb..95a586aaa 100644
--- a/scanner/walk_dir_tree_test.go
+++ b/scanner/walk_dir_tree_test.go
@@ -2,8 +2,10 @@ package scanner
import (
"context"
+ "io/fs"
"os"
"path/filepath"
+ "testing/fstest"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
@@ -78,8 +80,85 @@ var _ = Describe("walk_dir_tree", func() {
Expect(isDirIgnored(baseDir, dirEntry)).To(BeFalse())
})
})
+
+ Describe("fullReadDir", func() {
+ var fsys fakeFS
+ var ctx context.Context
+ BeforeEach(func() {
+ ctx = context.Background()
+ fsys = fakeFS{MapFS: fstest.MapFS{
+ "root/a/f1": {},
+ "root/b/f2": {},
+ "root/c/f3": {},
+ }}
+ })
+ It("reads all entries", func() {
+ dir, _ := fsys.Open("root")
+ entries := fullReadDir(ctx, dir.(fs.ReadDirFile))
+ Expect(entries).To(HaveLen(3))
+ Expect(entries[0].Name()).To(Equal("a"))
+ Expect(entries[1].Name()).To(Equal("b"))
+ Expect(entries[2].Name()).To(Equal("c"))
+ })
+ It("skips entries with permission error", func() {
+ fsys.failOn = "b"
+ dir, _ := fsys.Open("root")
+ entries := fullReadDir(ctx, dir.(fs.ReadDirFile))
+ Expect(entries).To(HaveLen(2))
+ Expect(entries[0].Name()).To(Equal("a"))
+ Expect(entries[1].Name()).To(Equal("c"))
+ })
+ It("aborts if it keeps getting 'readdirent: no such file or directory'", func() {
+ fsys.err = fs.ErrNotExist
+ dir, _ := fsys.Open("root")
+ entries := fullReadDir(ctx, dir.(fs.ReadDirFile))
+ Expect(entries).To(BeEmpty())
+ })
+ })
})
+type fakeFS struct {
+ fstest.MapFS
+ failOn string
+ err error
+}
+
+func (f *fakeFS) Open(name string) (fs.File, error) {
+ dir, err := f.MapFS.Open(name)
+ return &fakeDirFile{File: dir, fail: f.failOn, err: f.err}, err
+}
+
+type fakeDirFile struct {
+ fs.File
+ entries []fs.DirEntry
+ pos int
+ fail string
+ err error
+}
+
+// Only works with n == -1
+func (fd *fakeDirFile) ReadDir(n int) ([]fs.DirEntry, error) {
+ if fd.err != nil {
+ return nil, fd.err
+ }
+ if fd.entries == nil {
+ fd.entries, _ = fd.File.(fs.ReadDirFile).ReadDir(-1)
+ }
+ var dirs []fs.DirEntry
+ for {
+ if fd.pos >= len(fd.entries) {
+ break
+ }
+ e := fd.entries[fd.pos]
+ fd.pos++
+ if e.Name() == fd.fail {
+ return dirs, &fs.PathError{Op: "lstat", Path: e.Name(), Err: fs.ErrPermission}
+ }
+ dirs = append(dirs, e)
+ }
+ return dirs, nil
+}
+
func getDirEntry(baseDir, name string) (os.DirEntry, error) {
dirEntries, _ := os.ReadDir(baseDir)
for _, entry := range dirEntries {