"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
local: fix server-side copy on macOS when the destination exists - fixes #10117
Ayush Sharma2278207ab934
diff --git a/backend/local/clone_darwin.go b/backend/local/clone_darwin.goindex eec426d58..8ed3fa5e0 100644--- a/backend/local/clone_darwin.go+++ b/backend/local/clone_darwin.go@@ -6,11 +6,13 @@ package localimport ("context""fmt"+ "os""path/filepath""runtime""github.com/go-darwin/apfs""github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/lib/random")// Copy src to this remote using server-side copy operations.@@ -63,7 +65,7 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,if err != nil {return nil, err}- err = Clone(srcPath, dstPath)+ err = cloneOver(srcPath, dstPath)if err != nil {return nil, err}@@ -79,6 +81,27 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,return f.NewObject(ctx, remote)}+// cloneOver clones src to dst, replacing dst if it already exists.+//+// COPYFILE_CLONE implies COPYFILE_EXCL so Clone fails if dst exists.+// Cloning to a temporary name in the same directory and renaming it+// over dst avoids this and leaves dst untouched if the clone fails.+func cloneOver(src, dst string) error {+ if _, err := os.Lstat(dst); os.IsNotExist(err) {+ return Clone(src, dst)+ }+ tmp := filepath.Join(filepath.Dir(dst), "."+random.String(16)+".rclone-clone")+ err := Clone(src, tmp)+ if err == nil {+ err = os.Rename(tmp, dst)+ }+ if err != nil {+ _ = os.Remove(tmp)+ return fmt.Errorf("failed to clone %q to %q: %w", src, dst, err)+ }+ return nil+}+// Clone uses APFS cloning if possible, otherwise falls back to copying (with full metadata preservation)// note that this is closely related to unix.Clonefile(src, dst, unix.CLONE_NOFOLLOW) but not 100% identical// https://opensource.apple.com/source/copyfile/copyfile-173.40.2/copyfile.c.auto.htmldiff --git a/backend/local/local.go b/backend/local/local.goindex 977a3a88e..67a3d9ea4 100644--- a/backend/local/local.go+++ b/backend/local/local.go@@ -252,6 +252,13 @@ deduplicated by default (i.e. having two identical files does not consume morestorage than having just one.) However, for use cases where data redundancy ispreferable, --local-no-clone can be used to disable cloning and force "deep" copies.+Cloning also preserves macOS-specific metadata such as extended attributes,+which a normal copy does not, so this metadata is lost when cloning is disabled.++If the destination file already exists, cloning replaces it with a new file,+even with --inplace, so hard links to it are broken. Use --local-no-clone to+update existing files in place.+Currently, cloning is only supported when using APFS on macOS (support for otherplatforms may be added in the future.)`,Default: false,diff --git a/backend/local/local_internal_test.go b/backend/local/local_internal_test.goindex 4fd639a37..2b07ca9d2 100644--- a/backend/local/local_internal_test.go+++ b/backend/local/local_internal_test.go@@ -99,6 +99,31 @@ func TestVerifyCopy(t *testing.T) {assert.Error(t, err)}+// Test server-side copy over an existing file+func TestCopyOverExisting(t *testing.T) {+ ctx := context.Background()+ r := fstest.NewRun(t)+ doCopy := r.Flocal.Features().Copy+ if doCopy == nil {+ t.Skip("local backend has no server-side copy on this platform")+ }+ t1 := fstest.Time("2001-02-03T04:05:06.499999999Z")+ t2 := fstest.Time("2002-02-03T04:05:06.499999999Z")+ file1 := r.WriteFile("a.txt", "hello1", t1)+ file2 := r.WriteFile("b.txt", "HELLO2 longer", t2)++ src, err := r.Flocal.NewObject(ctx, file1.Path)+ require.NoError(t, err)+ dst, err := doCopy(ctx, src, file2.Path)+ require.NoError(t, err)+ assert.Equal(t, file2.Path, dst.Remote())++ file2.Size = file1.Size+ file2.Hashes = file1.Hashes+ file2.ModTime = file1.ModTime+ r.CheckLocalItems(t, file1, file2)+}+// skipIfNoSymlinks skips the test if this process can't create symlinks.//// Windows grants the privilege only to an elevated process or one running withdiff --git a/docs/content/docs.md b/docs/content/docs.mdindex e0a4a2d6d..c1f85e160 100644--- a/docs/content/docs.md+++ b/docs/content/docs.md@@ -1791,6 +1791,12 @@ Note that on the local file system if you don't use `--inplace` hardlinks (Unix only) will be broken. And if you do use `--inplace` youwon't be able to update in use executables.+On macOS, if rclone clones a file (see+[--local-no-clone](/local/#local-no-clone)) and the destination+already exists, the existing file is replaced with a new one, even+with `--inplace`, so hard links to it will be broken. Use+`--local-no-clone` to keep them.+Note also that versions of rclone prior to v1.63.0 behave as if the`--inplace` flag is always supplied.