🎧 Your Personal Streaming Service
Go airsonicmadsonicmedia-servermusicmusic-servernavidromeopensubsonicraspberry-piself-hostedstreamerstreaming-apistreaming-audiosubsonicsubsonic-server
Reapply the fix from #1054, but without getting into an infinite look in case of SMB fs errors. See #1164
Deluane61cf3217d08
diff --git a/scanner/walk_dir_tree.go b/scanner/walk_dir_tree.goindex b80ede082..77db41828 100644--- a/scanner/walk_dir_tree.go+++ b/scanner/walk_dir_tree.go@@ -66,11 +66,14 @@ func loadDir(ctx context.Context, dirPath string) ([]string, *dirStats, error) {}stats.ModTime = dirInfo.ModTime()- dirEntries, err := fullReadDir(dirPath)+ dir, err := os.Open(dirPath)if err != nil {- log.Error(ctx, "Error in ReadDir", "path", dirPath, err)+ log.Error(ctx, "Error in Opening directory", "path", dirPath, err)return children, stats, err}+ defer dir.Close()++ dirEntries := fullReadDir(ctx, dir)for _, entry := range dirEntries {isDir, err := isDirOrSymlinkToDir(dirPath, entry)// Skip invalid symlinks@@ -100,20 +103,28 @@ func loadDir(ctx context.Context, dirPath string) ([]string, *dirStats, error) {return children, stats, nil}-func fullReadDir(name string) ([]os.DirEntry, error) {- f, err := os.Open(name)- if err != nil {- return nil, err- }- defer f.Close()-- dirs, err := f.ReadDir(-1)- if err != nil {- log.Warn("Skipping DirEntry", err)- return nil, nil+// fullReadDir reads all files in the folder, skipping the ones with errors.+// It also detects when it is "stuck" with an error in the same directory over and over.+// In this case, it and returns whatever it was able to read until it got stuck.+// See discussion here: https://github.com/navidrome/navidrome/issues/1164#issuecomment-881922850+func fullReadDir(ctx context.Context, dir fs.ReadDirFile) []os.DirEntry {+ var allDirs []os.DirEntry+ var prevErrStr = ""+ for {+ dirs, err := dir.ReadDir(-1)+ allDirs = append(allDirs, dirs...)+ if err == nil {+ break+ }+ log.Warn(ctx, "Skipping DirEntry", err)+ if prevErrStr == err.Error() {+ log.Error(ctx, "Duplicate DirEntry failure, bailing", err)+ break+ }+ prevErrStr = err.Error()}- sort.Slice(dirs, func(i, j int) bool { return dirs[i].Name() < dirs[j].Name() })- return dirs, nil+ sort.Slice(allDirs, func(i, j int) bool { return allDirs[i].Name() < allDirs[j].Name() })+ return allDirs}// isDirOrSymlinkToDir returns true if and only if the dirEnt represents a filediff --git a/scanner/walk_dir_tree_test.go b/scanner/walk_dir_tree_test.goindex 6c8464bfb..95a586aaa 100644--- a/scanner/walk_dir_tree_test.go+++ b/scanner/walk_dir_tree_test.go@@ -2,8 +2,10 @@ package scannerimport ("context"+ "io/fs""os""path/filepath"+ "testing/fstest". "github.com/onsi/ginkgo". "github.com/onsi/gomega"@@ -78,8 +80,85 @@ var _ = Describe("walk_dir_tree", func() {Expect(isDirIgnored(baseDir, dirEntry)).To(BeFalse())})})++ Describe("fullReadDir", func() {+ var fsys fakeFS+ var ctx context.Context+ BeforeEach(func() {+ ctx = context.Background()+ fsys = fakeFS{MapFS: fstest.MapFS{+ "root/a/f1": {},+ "root/b/f2": {},+ "root/c/f3": {},+ }}+ })+ It("reads all entries", func() {+ dir, _ := fsys.Open("root")+ entries := fullReadDir(ctx, dir.(fs.ReadDirFile))+ Expect(entries).To(HaveLen(3))+ Expect(entries[0].Name()).To(Equal("a"))+ Expect(entries[1].Name()).To(Equal("b"))+ Expect(entries[2].Name()).To(Equal("c"))+ })+ It("skips entries with permission error", func() {+ fsys.failOn = "b"+ dir, _ := fsys.Open("root")+ entries := fullReadDir(ctx, dir.(fs.ReadDirFile))+ Expect(entries).To(HaveLen(2))+ Expect(entries[0].Name()).To(Equal("a"))+ Expect(entries[1].Name()).To(Equal("c"))+ })+ It("aborts if it keeps getting 'readdirent: no such file or directory'", func() {+ fsys.err = fs.ErrNotExist+ dir, _ := fsys.Open("root")+ entries := fullReadDir(ctx, dir.(fs.ReadDirFile))+ Expect(entries).To(BeEmpty())+ })+ })})+type fakeFS struct {+ fstest.MapFS+ failOn string+ err error+}++func (f *fakeFS) Open(name string) (fs.File, error) {+ dir, err := f.MapFS.Open(name)+ return &fakeDirFile{File: dir, fail: f.failOn, err: f.err}, err+}++type fakeDirFile struct {+ fs.File+ entries []fs.DirEntry+ pos int+ fail string+ err error+}++// Only works with n == -1+func (fd *fakeDirFile) ReadDir(n int) ([]fs.DirEntry, error) {+ if fd.err != nil {+ return nil, fd.err+ }+ if fd.entries == nil {+ fd.entries, _ = fd.File.(fs.ReadDirFile).ReadDir(-1)+ }+ var dirs []fs.DirEntry+ for {+ if fd.pos >= len(fd.entries) {+ break+ }+ e := fd.entries[fd.pos]+ fd.pos+++ if e.Name() == fd.fail {+ return dirs, &fs.PathError{Op: "lstat", Path: e.Name(), Err: fs.ErrPermission}+ }+ dirs = append(dirs, e)+ }+ return dirs, nil+}+func getDirEntry(baseDir, name string) (os.DirEntry, error) {dirEntries, _ := os.ReadDir(baseDir)for _, entry := range dirEntries {