"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
local: add server-side copy with xattrs on macOS (part-fix #1710)
nielash87ec26001f39
diff --git a/backend/local/clone_darwin.go b/backend/local/clone_darwin.gonew file mode 100644index 000000000..2dbff2c05--- /dev/null+++ b/backend/local/clone_darwin.go@@ -0,0 +1,66 @@+//go:build darwin && cgo++// Package local provides a filesystem interface+package local++import (+ "context"+ "runtime"++ "github.com/go-darwin/apfs"+ "github.com/rclone/rclone/fs"+)++// Copy src to this remote using server-side copy operations.+//+// # This is stored with the remote path given+//+// # It returns the destination Object and a possible error+//+// Will only be called if src.Fs().Name() == f.Name()+//+// If it isn't possible then return fs.ErrorCantCopy+func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {+ if runtime.GOOS != "darwin" || f.opt.TranslateSymlinks {+ return nil, fs.ErrorCantCopy+ }+ srcObj, ok := src.(*Object)+ if !ok {+ fs.Debugf(src, "Can't clone - not same remote type")+ return nil, fs.ErrorCantCopy+ }++ // Create destination+ dstObj := f.newObject(remote)+ err := dstObj.mkdirAll()+ if err != nil {+ return nil, err+ }++ err = Clone(srcObj.path, f.localPath(remote))+ if err != nil {+ return nil, err+ }+ fs.Debugf(remote, "server-side cloned!")+ return f.NewObject(ctx, remote)+}++// Clone uses APFS cloning if possible, otherwise falls back to copying (with full metadata preservation)+// note that this is closely related to unix.Clonefile(src, dst, unix.CLONE_NOFOLLOW) but not 100% identical+// https://opensource.apple.com/source/copyfile/copyfile-173.40.2/copyfile.c.auto.html+func Clone(src, dst string) error {+ state := apfs.CopyFileStateAlloc()+ defer func() {+ if err := apfs.CopyFileStateFree(state); err != nil {+ fs.Errorf(dst, "free state error: %v", err)+ }+ }()+ cloned, err := apfs.CopyFile(src, dst, state, apfs.COPYFILE_CLONE)+ fs.Debugf(dst, "isCloned: %v, error: %v", cloned, err)+ return err+}++// Check the interfaces are satisfied+var (+ _ fs.Copier = &Fs{}+)diff --git a/backend/union/union_internal_test.go b/backend/union/union_internal_test.goindex b6f78ec8d..ccc130e8f 100644--- a/backend/union/union_internal_test.go+++ b/backend/union/union_internal_test.go@@ -4,6 +4,7 @@ import ("bytes""context""fmt"+ "runtime""testing""time"@@ -95,6 +96,12 @@ func TestMoveCopy(t *testing.T) {fLocal := unionFs.upstreams[0].FsfMemory := unionFs.upstreams[1].Fs+ if runtime.GOOS == "darwin" {+ // need to disable as this test specifically tests a local that can't Copy+ f.Features().Disable("Copy")+ fLocal.Features().Disable("Copy")+ }+t.Run("Features", func(t *testing.T) {assert.NotNil(t, f.Features().Move)assert.Nil(t, f.Features().Copy)diff --git a/fs/operations/copy_test.go b/fs/operations/copy_test.goindex 73aa5e05a..1e8f3ae5e 100644--- a/fs/operations/copy_test.go+++ b/fs/operations/copy_test.go@@ -7,6 +7,7 @@ import ("fmt""os""path"+ "runtime""sort""strings""testing"@@ -449,6 +450,14 @@ func TestCopyFileMaxTransfer(t *testing.T) {ci.MaxTransfer = sizeCutoffci.CutoffMode = fs.CutoffModeHard+ if runtime.GOOS == "darwin" {+ // disable server-side copies as they don't count towards transfer size stats+ r.Flocal.Features().Disable("Copy")+ if r.Fremote.Features().IsLocal {+ r.Fremote.Features().Disable("Copy")+ }+ }+// file1: Show a small file gets transferred OKaccounting.Stats(ctx).ResetCounters()err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file1.Path, file1.Path)diff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.goindex 2d221e492..f1c07921e 100644--- a/fs/sync/sync_test.go+++ b/fs/sync/sync_test.go@@ -1382,6 +1382,12 @@ func testSyncWithMaxDuration(t *testing.T, cutoffMode fs.CutoffMode) {r.CheckLocalItems(t, file1, file2)r.CheckRemoteItems(t)+ if runtime.GOOS == "darwin" {+ r.Flocal.Features().Disable("Copy") // macOS cloning is too fast for this test!+ if r.Fremote.Features().IsLocal {+ r.Fremote.Features().Disable("Copy") // macOS cloning is too fast for this test!+ }+ }accounting.GlobalStats().ResetCounters()// ctx = predictDstFromLogger(ctx) // not currently supported (but tests do pass for CutoffModeSoft)startTime := time.Now()@@ -2569,6 +2575,14 @@ func TestMaxTransfer(t *testing.T) {r.CheckLocalItems(t, file1, file2, file3)r.CheckRemoteItems(t)+ if runtime.GOOS == "darwin" {+ // disable server-side copies as they don't count towards transfer size stats+ r.Flocal.Features().Disable("Copy")+ if r.Fremote.Features().IsLocal {+ r.Fremote.Features().Disable("Copy")+ }+ }+accounting.GlobalStats().ResetCounters()// ctx = predictDstFromLogger(ctx) // not currently supporteddiff --git a/go.mod b/go.modindex a29a24326..26f74b251 100644--- a/go.mod+++ b/go.mod@@ -33,6 +33,7 @@ require (github.com/gabriel-vasile/mimetype v1.4.4github.com/gdamore/tcell/v2 v2.7.4github.com/go-chi/chi/v5 v5.1.0+ github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348github.com/go-git/go-billy/v5 v5.5.0github.com/google/uuid v1.6.0github.com/hanwen/go-fuse/v2 v2.5.1diff --git a/go.sum b/go.sumindex 1ade31d23..174178cd5 100644--- a/go.sum+++ b/go.sum@@ -225,6 +225,8 @@ github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg=github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU=github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw=github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=+github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 h1:JnrjqG5iR07/8k7NqrLNilRsl3s1EPRQEGvbPyOce68=+github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348/go.mod h1:Czxo/d1g948LtrALAZdL04TL/HnkopquAjxYUuI02bo=github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU=github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow=github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=