Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

local: stop --copy-links following symlink loops - fixes #4402

phatlcb6beea4b278b
diff --git a/backend/local/local.go b/backend/local/local.go
index dded823e6..e76ef3823 100644
--- a/backend/local/local.go
+++ b/backend/local/local.go
@@ -12,6 +12,7 @@ import (
"path"
"path/filepath"
"runtime"
+ "slices"
"strings"
"sync"
"sync/atomic"
@@ -438,6 +439,7 @@ var (
errLinksAndCopyLinks = errors.New("can't use -l/--links with -L/--copy-links")
errLinksNeedsSuffix = errors.New("need \"" + fs.LinkSuffix + "\" suffix to refer to symlink when using -l/--links")
errPathEscapes = errors.New("file name is not a path within the local root - check the encoding")
+ errSymlinkLoop = errors.New("loop detected: points to a parent directory")
)
// NewFs constructs an Fs from the path
@@ -668,6 +670,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
if err != nil {
return nil, fs.ErrorDirNotFound
}
+ var parents []os.FileInfo // fsDirPath and its parents, read on first use
fd, err := os.Open(fsDirPath)
if err != nil {
@@ -748,11 +751,28 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
if f.opt.FollowSymlinks && (mode&symlinkFlag) != 0 {
localPath := filepath.Join(fsDirPath, name)
fi, err = os.Stat(localPath)
+ // A directory symlink pointing to a parent would be followed
+ // until the OS returns ELOOP, which can take a very long time
+ if err == nil && fi.IsDir() {
+ if parents == nil {
+ parents = statParents(fsDirPath)
+ }
+ if slices.ContainsFunc(parents, func(parent os.FileInfo) bool { return os.SameFile(fi, parent) }) {
+ // Quietly skip loops the directory filters exclude as
+ // the layer above wouldn't recurse into them
+ if useFilter {
+ if include, dirErr := filter.IncludeDirectory(ctx, f)(newRemote); dirErr == nil && !include {
+ continue
+ }
+ }
+ err = errSymlinkLoop
+ }
+ }
// Quietly skip errors on excluded files and directories
if err != nil && useFilter && !filter.IncludeRemote(newRemote) {
continue
}
- if os.IsNotExist(err) || isCircularSymlinkError(err) {
+ if os.IsNotExist(err) || isCircularSymlinkError(err) || errors.Is(err, errSymlinkLoop) {
// Skip bad symlinks and circular symlinks
err = fserrors.NoRetryError(fmt.Errorf("symlink: %w", err))
fs.Errorf(newRemote, "Listing error: %v", err)
@@ -797,6 +817,21 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
return entries, nil
}
+// statParents returns the info for dirPath and each of its parents up
+// to the root of the file system, skipping any which can't be read.
+func statParents(dirPath string) (parents []os.FileInfo) {
+ for {
+ if fi, err := os.Stat(dirPath); err == nil {
+ parents = append(parents, fi)
+ }
+ parent := filepath.Dir(dirPath)
+ if parent == dirPath {
+ return parents
+ }
+ dirPath = parent
+ }
+}
+
func (f *Fs) cleanRemote(dir, filename string) (remote string) {
if f.opt.UTFNorm {
filename = norm.NFC.String(filename)
diff --git a/backend/local/local_internal_test.go b/backend/local/local_internal_test.go
index f8d1f4359..3a972c15e 100644
--- a/backend/local/local_internal_test.go
+++ b/backend/local/local_internal_test.go
@@ -3,6 +3,7 @@ package local
import (
"bytes"
"context"
+ "errors"
"fmt"
"io"
"math"
@@ -23,6 +24,7 @@ import (
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/object"
"github.com/rclone/rclone/fs/operations"
+ "github.com/rclone/rclone/fs/walk"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/lib/encoder"
"github.com/rclone/rclone/lib/file"
@@ -1176,3 +1178,121 @@ func TestCopySymlink(t *testing.T) {
want = fstest.NewItem("dst2/file.txt", "hello world", when)
fstest.CompareItems(t, []fs.DirEntry{dst}, []fstest.Item{want}, nil, f.precision, "")
}
+
+// TestCopyLinksLoop checks that -L/--copy-links follows a symlink to a
+// sibling directory but skips symlinks which point to the directory
+// being listed or one of its parents, as these would loop forever.
+func TestCopyLinksLoop(t *testing.T) {
+ skipIfNoSymlinks(t)
+ dir := t.TempDir()
+
+ // folder-A/link-to-B and folder-B/link-to-A point at each other
+ // and folder-A/link-to-parent points at dir
+ require.NoError(t, os.Mkdir(filepath.Join(dir, "folder-A"), 0700))
+ require.NoError(t, os.Mkdir(filepath.Join(dir, "folder-B"), 0700))
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "folder-A", "file-A"), []byte("A"), 0600))
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "folder-B", "file-B"), []byte("B"), 0600))
+ require.NoError(t, os.Symlink(filepath.Join("..", "folder-B"), filepath.Join(dir, "folder-A", "link-to-B")))
+ require.NoError(t, os.Symlink(filepath.Join("..", "folder-A"), filepath.Join(dir, "folder-B", "link-to-A")))
+ require.NoError(t, os.Symlink("..", filepath.Join(dir, "folder-A", "link-to-parent")))
+
+ wantRoot := []string{
+ "folder-A",
+ "folder-A/file-A",
+ "folder-A/link-to-B",
+ "folder-A/link-to-B/file-B",
+ "folder-B",
+ "folder-B/file-B",
+ "folder-B/link-to-A",
+ "folder-B/link-to-A/file-A",
+ }
+ for _, test := range []struct {
+ name string
+ root string
+ exclude string
+ want []string
+ wantErrors int64
+ }{{
+ name: "Root",
+ root: dir,
+ want: wantRoot,
+ // folder-A/link-to-parent, folder-A/link-to-B/link-to-A,
+ // folder-B/link-to-A/link-to-B, folder-B/link-to-A/link-to-parent
+ wantErrors: 4,
+ }, {
+ // link-to-parent points above the root of the Fs
+ name: "SubDir",
+ root: filepath.Join(dir, "folder-A"),
+ want: []string{
+ "file-A",
+ "link-to-B",
+ "link-to-B/file-B",
+ },
+ // link-to-parent, link-to-B/link-to-A
+ wantErrors: 2,
+ }, {
+ // Excluded loops aren't reported
+ name: "Excluded",
+ root: dir,
+ exclude: "link-to-parent/**",
+ want: wantRoot,
+ // folder-A/link-to-B/link-to-A, folder-B/link-to-A/link-to-B
+ wantErrors: 2,
+ }} {
+ t.Run(test.name, func(t *testing.T) {
+ ctx := context.Background()
+ if test.exclude != "" {
+ var fi *filter.Filter
+ ctx, fi = filter.AddConfig(ctx)
+ require.NoError(t, fi.AddRule("- "+test.exclude))
+ }
+ f, err := NewFs(ctx, "local", test.root, configmap.Simple{
+ "copy_links": "true",
+ })
+ require.NoError(t, err)
+
+ accounting.Stats(ctx).ResetErrors()
+ defer accounting.Stats(ctx).ResetErrors()
+
+ var got []string
+ err = walk.ListR(ctx, f, "", false, -1, walk.ListAll, func(entries fs.DirEntries) error {
+ for _, entry := range entries {
+ got = append(got, entry.Remote())
+ }
+ // Stop a runaway listing rather than waiting for ELOOP
+ if len(got) > 100 {
+ return errors.New("too many entries - symlink loop not detected")
+ }
+ return nil
+ })
+ require.NoError(t, err)
+
+ sort.Strings(got)
+ assert.Equal(t, test.want, got)
+ assert.Equal(t, test.wantErrors, accounting.Stats(ctx).GetErrors(), "global errors found")
+ })
+ }
+
+ // A single List reports and skips the link to a parent, but still
+ // returns the link to a sibling
+ t.Run("List", func(t *testing.T) {
+ ctx := context.Background()
+ f, err := NewFs(ctx, "local", dir, configmap.Simple{
+ "copy_links": "true",
+ })
+ require.NoError(t, err)
+
+ accounting.Stats(ctx).ResetErrors()
+ defer accounting.Stats(ctx).ResetErrors()
+
+ entries, err := f.List(ctx, "folder-A")
+ require.NoError(t, err)
+ var got []string
+ for _, entry := range entries {
+ got = append(got, entry.Remote())
+ }
+ sort.Strings(got)
+ assert.Equal(t, []string{"folder-A/file-A", "folder-A/link-to-B"}, got)
+ assert.Equal(t, int64(1), accounting.Stats(ctx).GetErrors(), "global errors found")
+ })
+}
diff --git a/docs/content/local.md b/docs/content/local.md
index 5f2ecdeca..178fb97e4 100644
--- a/docs/content/local.md
+++ b/docs/content/local.md
@@ -213,6 +213,9 @@ $ rclone -L ls /tmp/a
6 b/one
```
+Symlinks which point to a parent directory would loop forever, so
+rclone reports them as errors and skips them.
+
#### --local-links, --links, -l
Normally rclone will ignore symlinks or junction points (which behave