Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

local: fix server-side copy on macOS when the destination exists - fixes #10117

Ayush Sharma2278207ab934
diff --git a/backend/local/clone_darwin.go b/backend/local/clone_darwin.go
index eec426d58..8ed3fa5e0 100644
--- a/backend/local/clone_darwin.go
+++ b/backend/local/clone_darwin.go
@@ -6,11 +6,13 @@ package local
import (
"context"
"fmt"
+ "os"
"path/filepath"
"runtime"
"github.com/go-darwin/apfs"
"github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/lib/random"
)
// Copy src to this remote using server-side copy operations.
@@ -63,7 +65,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
if err != nil {
return nil, err
}
- err = Clone(srcPath, dstPath)
+ err = cloneOver(srcPath, dstPath)
if err != nil {
return nil, err
}
@@ -79,6 +81,27 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
return f.NewObject(ctx, remote)
}
+// cloneOver clones src to dst, replacing dst if it already exists.
+//
+// COPYFILE_CLONE implies COPYFILE_EXCL so Clone fails if dst exists.
+// Cloning to a temporary name in the same directory and renaming it
+// over dst avoids this and leaves dst untouched if the clone fails.
+func cloneOver(src, dst string) error {
+ if _, err := os.Lstat(dst); os.IsNotExist(err) {
+ return Clone(src, dst)
+ }
+ tmp := filepath.Join(filepath.Dir(dst), "."+random.String(16)+".rclone-clone")
+ err := Clone(src, tmp)
+ if err == nil {
+ err = os.Rename(tmp, dst)
+ }
+ if err != nil {
+ _ = os.Remove(tmp)
+ return fmt.Errorf("failed to clone %q to %q: %w", src, dst, err)
+ }
+ return nil
+}
+
// Clone uses APFS cloning if possible, otherwise falls back to copying (with full metadata preservation)
// note that this is closely related to unix.Clonefile(src, dst, unix.CLONE_NOFOLLOW) but not 100% identical
// https://opensource.apple.com/source/copyfile/copyfile-173.40.2/copyfile.c.auto.html
diff --git a/backend/local/local.go b/backend/local/local.go
index 977a3a88e..67a3d9ea4 100644
--- a/backend/local/local.go
+++ b/backend/local/local.go
@@ -252,6 +252,13 @@ deduplicated by default (i.e. having two identical files does not consume more
storage than having just one.) However, for use cases where data redundancy is
preferable, --local-no-clone can be used to disable cloning and force "deep" copies.
+Cloning also preserves macOS-specific metadata such as extended attributes,
+which a normal copy does not, so this metadata is lost when cloning is disabled.
+
+If the destination file already exists, cloning replaces it with a new file,
+even with --inplace, so hard links to it are broken. Use --local-no-clone to
+update existing files in place.
+
Currently, cloning is only supported when using APFS on macOS (support for other
platforms may be added in the future.)`,
Default: false,
diff --git a/backend/local/local_internal_test.go b/backend/local/local_internal_test.go
index 4fd639a37..2b07ca9d2 100644
--- a/backend/local/local_internal_test.go
+++ b/backend/local/local_internal_test.go
@@ -99,6 +99,31 @@ func TestVerifyCopy(t *testing.T) {
assert.Error(t, err)
}
+// Test server-side copy over an existing file
+func TestCopyOverExisting(t *testing.T) {
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+ doCopy := r.Flocal.Features().Copy
+ if doCopy == nil {
+ t.Skip("local backend has no server-side copy on this platform")
+ }
+ t1 := fstest.Time("2001-02-03T04:05:06.499999999Z")
+ t2 := fstest.Time("2002-02-03T04:05:06.499999999Z")
+ file1 := r.WriteFile("a.txt", "hello1", t1)
+ file2 := r.WriteFile("b.txt", "HELLO2 longer", t2)
+
+ src, err := r.Flocal.NewObject(ctx, file1.Path)
+ require.NoError(t, err)
+ dst, err := doCopy(ctx, src, file2.Path)
+ require.NoError(t, err)
+ assert.Equal(t, file2.Path, dst.Remote())
+
+ file2.Size = file1.Size
+ file2.Hashes = file1.Hashes
+ file2.ModTime = file1.ModTime
+ r.CheckLocalItems(t, file1, file2)
+}
+
// skipIfNoSymlinks skips the test if this process can't create symlinks.
//
// Windows grants the privilege only to an elevated process or one running with
diff --git a/docs/content/docs.md b/docs/content/docs.md
index e0a4a2d6d..c1f85e160 100644
--- a/docs/content/docs.md
+++ b/docs/content/docs.md
@@ -1791,6 +1791,12 @@ Note that on the local file system if you don't use `--inplace` hard
links (Unix only) will be broken. And if you do use `--inplace` you
won't be able to update in use executables.
+On macOS, if rclone clones a file (see
+[--local-no-clone](/local/#local-no-clone)) and the destination
+already exists, the existing file is replaced with a new one, even
+with `--inplace`, so hard links to it will be broken. Use
+`--local-no-clone` to keep them.
+
Note also that versions of rclone prior to v1.63.0 behave as if the
`--inplace` flag is always supplied.