"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
local: stop --copy-links following symlink loops - fixes #4402
phatlcb6beea4b278b
diff --git a/backend/local/local.go b/backend/local/local.goindex dded823e6..e76ef3823 100644--- a/backend/local/local.go+++ b/backend/local/local.go@@ -12,6 +12,7 @@ import ("path""path/filepath""runtime"+ "slices""strings""sync""sync/atomic"@@ -438,6 +439,7 @@ var (errLinksAndCopyLinks = errors.New("can't use -l/--links with -L/--copy-links")errLinksNeedsSuffix = errors.New("need \"" + fs.LinkSuffix + "\" suffix to refer to symlink when using -l/--links")errPathEscapes = errors.New("file name is not a path within the local root - check the encoding")+ errSymlinkLoop = errors.New("loop detected: points to a parent directory"))// NewFs constructs an Fs from the path@@ -668,6 +670,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err eif err != nil {return nil, fs.ErrorDirNotFound}+ var parents []os.FileInfo // fsDirPath and its parents, read on first usefd, err := os.Open(fsDirPath)if err != nil {@@ -748,11 +751,28 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err eif f.opt.FollowSymlinks && (mode&symlinkFlag) != 0 {localPath := filepath.Join(fsDirPath, name)fi, err = os.Stat(localPath)+ // A directory symlink pointing to a parent would be followed+ // until the OS returns ELOOP, which can take a very long time+ if err == nil && fi.IsDir() {+ if parents == nil {+ parents = statParents(fsDirPath)+ }+ if slices.ContainsFunc(parents, func(parent os.FileInfo) bool { return os.SameFile(fi, parent) }) {+ // Quietly skip loops the directory filters exclude as+ // the layer above wouldn't recurse into them+ if useFilter {+ if include, dirErr := filter.IncludeDirectory(ctx, f)(newRemote); dirErr == nil && !include {+ continue+ }+ }+ err = errSymlinkLoop+ }+ }// Quietly skip errors on excluded files and directoriesif err != nil && useFilter && !filter.IncludeRemote(newRemote) {continue}- if os.IsNotExist(err) || isCircularSymlinkError(err) {+ if os.IsNotExist(err) || isCircularSymlinkError(err) || errors.Is(err, errSymlinkLoop) {// Skip bad symlinks and circular symlinkserr = fserrors.NoRetryError(fmt.Errorf("symlink: %w", err))fs.Errorf(newRemote, "Listing error: %v", err)@@ -797,6 +817,21 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err ereturn entries, nil}+// statParents returns the info for dirPath and each of its parents up+// to the root of the file system, skipping any which can't be read.+func statParents(dirPath string) (parents []os.FileInfo) {+ for {+ if fi, err := os.Stat(dirPath); err == nil {+ parents = append(parents, fi)+ }+ parent := filepath.Dir(dirPath)+ if parent == dirPath {+ return parents+ }+ dirPath = parent+ }+}+func (f *Fs) cleanRemote(dir, filename string) (remote string) {if f.opt.UTFNorm {filename = norm.NFC.String(filename)diff --git a/backend/local/local_internal_test.go b/backend/local/local_internal_test.goindex f8d1f4359..3a972c15e 100644--- a/backend/local/local_internal_test.go+++ b/backend/local/local_internal_test.go@@ -3,6 +3,7 @@ package localimport ("bytes""context"+ "errors""fmt""io""math"@@ -23,6 +24,7 @@ import ("github.com/rclone/rclone/fs/hash""github.com/rclone/rclone/fs/object""github.com/rclone/rclone/fs/operations"+ "github.com/rclone/rclone/fs/walk""github.com/rclone/rclone/fstest""github.com/rclone/rclone/lib/encoder""github.com/rclone/rclone/lib/file"@@ -1176,3 +1178,121 @@ func TestCopySymlink(t *testing.T) {want = fstest.NewItem("dst2/file.txt", "hello world", when)fstest.CompareItems(t, []fs.DirEntry{dst}, []fstest.Item{want}, nil, f.precision, "")}++// TestCopyLinksLoop checks that -L/--copy-links follows a symlink to a+// sibling directory but skips symlinks which point to the directory+// being listed or one of its parents, as these would loop forever.+func TestCopyLinksLoop(t *testing.T) {+ skipIfNoSymlinks(t)+ dir := t.TempDir()++ // folder-A/link-to-B and folder-B/link-to-A point at each other+ // and folder-A/link-to-parent points at dir+ require.NoError(t, os.Mkdir(filepath.Join(dir, "folder-A"), 0700))+ require.NoError(t, os.Mkdir(filepath.Join(dir, "folder-B"), 0700))+ require.NoError(t, os.WriteFile(filepath.Join(dir, "folder-A", "file-A"), []byte("A"), 0600))+ require.NoError(t, os.WriteFile(filepath.Join(dir, "folder-B", "file-B"), []byte("B"), 0600))+ require.NoError(t, os.Symlink(filepath.Join("..", "folder-B"), filepath.Join(dir, "folder-A", "link-to-B")))+ require.NoError(t, os.Symlink(filepath.Join("..", "folder-A"), filepath.Join(dir, "folder-B", "link-to-A")))+ require.NoError(t, os.Symlink("..", filepath.Join(dir, "folder-A", "link-to-parent")))++ wantRoot := []string{+ "folder-A",+ "folder-A/file-A",+ "folder-A/link-to-B",+ "folder-A/link-to-B/file-B",+ "folder-B",+ "folder-B/file-B",+ "folder-B/link-to-A",+ "folder-B/link-to-A/file-A",+ }+ for _, test := range []struct {+ name string+ root string+ exclude string+ want []string+ wantErrors int64+ }{{+ name: "Root",+ root: dir,+ want: wantRoot,+ // folder-A/link-to-parent, folder-A/link-to-B/link-to-A,+ // folder-B/link-to-A/link-to-B, folder-B/link-to-A/link-to-parent+ wantErrors: 4,+ }, {+ // link-to-parent points above the root of the Fs+ name: "SubDir",+ root: filepath.Join(dir, "folder-A"),+ want: []string{+ "file-A",+ "link-to-B",+ "link-to-B/file-B",+ },+ // link-to-parent, link-to-B/link-to-A+ wantErrors: 2,+ }, {+ // Excluded loops aren't reported+ name: "Excluded",+ root: dir,+ exclude: "link-to-parent/**",+ want: wantRoot,+ // folder-A/link-to-B/link-to-A, folder-B/link-to-A/link-to-B+ wantErrors: 2,+ }} {+ t.Run(test.name, func(t *testing.T) {+ ctx := context.Background()+ if test.exclude != "" {+ var fi *filter.Filter+ ctx, fi = filter.AddConfig(ctx)+ require.NoError(t, fi.AddRule("- "+test.exclude))+ }+ f, err := NewFs(ctx, "local", test.root, configmap.Simple{+ "copy_links": "true",+ })+ require.NoError(t, err)++ accounting.Stats(ctx).ResetErrors()+ defer accounting.Stats(ctx).ResetErrors()++ var got []string+ err = walk.ListR(ctx, f, "", false, -1, walk.ListAll, func(entries fs.DirEntries) error {+ for _, entry := range entries {+ got = append(got, entry.Remote())+ }+ // Stop a runaway listing rather than waiting for ELOOP+ if len(got) > 100 {+ return errors.New("too many entries - symlink loop not detected")+ }+ return nil+ })+ require.NoError(t, err)++ sort.Strings(got)+ assert.Equal(t, test.want, got)+ assert.Equal(t, test.wantErrors, accounting.Stats(ctx).GetErrors(), "global errors found")+ })+ }++ // A single List reports and skips the link to a parent, but still+ // returns the link to a sibling+ t.Run("List", func(t *testing.T) {+ ctx := context.Background()+ f, err := NewFs(ctx, "local", dir, configmap.Simple{+ "copy_links": "true",+ })+ require.NoError(t, err)++ accounting.Stats(ctx).ResetErrors()+ defer accounting.Stats(ctx).ResetErrors()++ entries, err := f.List(ctx, "folder-A")+ require.NoError(t, err)+ var got []string+ for _, entry := range entries {+ got = append(got, entry.Remote())+ }+ sort.Strings(got)+ assert.Equal(t, []string{"folder-A/file-A", "folder-A/link-to-B"}, got)+ assert.Equal(t, int64(1), accounting.Stats(ctx).GetErrors(), "global errors found")+ })+}diff --git a/docs/content/local.md b/docs/content/local.mdindex 5f2ecdeca..178fb97e4 100644--- a/docs/content/local.md+++ b/docs/content/local.md@@ -213,6 +213,9 @@ $ rclone -L ls /tmp/a6 b/one```+Symlinks which point to a parent directory would loop forever, so+rclone reports them as errors and skips them.+#### --local-links, --links, -lNormally rclone will ignore symlinks or junction points (which behave