Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

local: add server-side copy with xattrs on macOS (part-fix #1710)

nielash87ec26001f39
diff --git a/backend/local/clone_darwin.go b/backend/local/clone_darwin.go
new file mode 100644
index 000000000..2dbff2c05
--- /dev/null
+++ b/backend/local/clone_darwin.go
@@ -0,0 +1,66 @@
+//go:build darwin && cgo
+
+// Package local provides a filesystem interface
+package local
+
+import (
+ "context"
+ "runtime"
+
+ "github.com/go-darwin/apfs"
+ "github.com/rclone/rclone/fs"
+)
+
+// Copy src to this remote using server-side copy operations.
+//
+// # This is stored with the remote path given
+//
+// # It returns the destination Object and a possible error
+//
+// Will only be called if src.Fs().Name() == f.Name()
+//
+// If it isn't possible then return fs.ErrorCantCopy
+func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object, error) {
+ if runtime.GOOS != "darwin" || f.opt.TranslateSymlinks {
+ return nil, fs.ErrorCantCopy
+ }
+ srcObj, ok := src.(*Object)
+ if !ok {
+ fs.Debugf(src, "Can't clone - not same remote type")
+ return nil, fs.ErrorCantCopy
+ }
+
+ // Create destination
+ dstObj := f.newObject(remote)
+ err := dstObj.mkdirAll()
+ if err != nil {
+ return nil, err
+ }
+
+ err = Clone(srcObj.path, f.localPath(remote))
+ if err != nil {
+ return nil, err
+ }
+ fs.Debugf(remote, "server-side cloned!")
+ return f.NewObject(ctx, remote)
+}
+
+// Clone uses APFS cloning if possible, otherwise falls back to copying (with full metadata preservation)
+// note that this is closely related to unix.Clonefile(src, dst, unix.CLONE_NOFOLLOW) but not 100% identical
+// https://opensource.apple.com/source/copyfile/copyfile-173.40.2/copyfile.c.auto.html
+func Clone(src, dst string) error {
+ state := apfs.CopyFileStateAlloc()
+ defer func() {
+ if err := apfs.CopyFileStateFree(state); err != nil {
+ fs.Errorf(dst, "free state error: %v", err)
+ }
+ }()
+ cloned, err := apfs.CopyFile(src, dst, state, apfs.COPYFILE_CLONE)
+ fs.Debugf(dst, "isCloned: %v, error: %v", cloned, err)
+ return err
+}
+
+// Check the interfaces are satisfied
+var (
+ _ fs.Copier = &Fs{}
+)
diff --git a/backend/union/union_internal_test.go b/backend/union/union_internal_test.go
index b6f78ec8d..ccc130e8f 100644
--- a/backend/union/union_internal_test.go
+++ b/backend/union/union_internal_test.go
@@ -4,6 +4,7 @@ import (
"bytes"
"context"
"fmt"
+ "runtime"
"testing"
"time"
@@ -95,6 +96,12 @@ func TestMoveCopy(t *testing.T) {
fLocal := unionFs.upstreams[0].Fs
fMemory := unionFs.upstreams[1].Fs
+ if runtime.GOOS == "darwin" {
+ // need to disable as this test specifically tests a local that can't Copy
+ f.Features().Disable("Copy")
+ fLocal.Features().Disable("Copy")
+ }
+
t.Run("Features", func(t *testing.T) {
assert.NotNil(t, f.Features().Move)
assert.Nil(t, f.Features().Copy)
diff --git a/fs/operations/copy_test.go b/fs/operations/copy_test.go
index 73aa5e05a..1e8f3ae5e 100644
--- a/fs/operations/copy_test.go
+++ b/fs/operations/copy_test.go
@@ -7,6 +7,7 @@ import (
"fmt"
"os"
"path"
+ "runtime"
"sort"
"strings"
"testing"
@@ -449,6 +450,14 @@ func TestCopyFileMaxTransfer(t *testing.T) {
ci.MaxTransfer = sizeCutoff
ci.CutoffMode = fs.CutoffModeHard
+ if runtime.GOOS == "darwin" {
+ // disable server-side copies as they don't count towards transfer size stats
+ r.Flocal.Features().Disable("Copy")
+ if r.Fremote.Features().IsLocal {
+ r.Fremote.Features().Disable("Copy")
+ }
+ }
+
// file1: Show a small file gets transferred OK
accounting.Stats(ctx).ResetCounters()
err = operations.CopyFile(ctx, r.Fremote, r.Flocal, file1.Path, file1.Path)
diff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.go
index 2d221e492..f1c07921e 100644
--- a/fs/sync/sync_test.go
+++ b/fs/sync/sync_test.go
@@ -1382,6 +1382,12 @@ func testSyncWithMaxDuration(t *testing.T, cutoffMode fs.CutoffMode) {
r.CheckLocalItems(t, file1, file2)
r.CheckRemoteItems(t)
+ if runtime.GOOS == "darwin" {
+ r.Flocal.Features().Disable("Copy") // macOS cloning is too fast for this test!
+ if r.Fremote.Features().IsLocal {
+ r.Fremote.Features().Disable("Copy") // macOS cloning is too fast for this test!
+ }
+ }
accounting.GlobalStats().ResetCounters()
// ctx = predictDstFromLogger(ctx) // not currently supported (but tests do pass for CutoffModeSoft)
startTime := time.Now()
@@ -2569,6 +2575,14 @@ func TestMaxTransfer(t *testing.T) {
r.CheckLocalItems(t, file1, file2, file3)
r.CheckRemoteItems(t)
+ if runtime.GOOS == "darwin" {
+ // disable server-side copies as they don't count towards transfer size stats
+ r.Flocal.Features().Disable("Copy")
+ if r.Fremote.Features().IsLocal {
+ r.Fremote.Features().Disable("Copy")
+ }
+ }
+
accounting.GlobalStats().ResetCounters()
// ctx = predictDstFromLogger(ctx) // not currently supported
diff --git a/go.mod b/go.mod
index a29a24326..26f74b251 100644
--- a/go.mod
+++ b/go.mod
@@ -33,6 +33,7 @@ require (
github.com/gabriel-vasile/mimetype v1.4.4
github.com/gdamore/tcell/v2 v2.7.4
github.com/go-chi/chi/v5 v5.1.0
+ github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348
github.com/go-git/go-billy/v5 v5.5.0
github.com/google/uuid v1.6.0
github.com/hanwen/go-fuse/v2 v2.5.1
diff --git a/go.sum b/go.sum
index 1ade31d23..174178cd5 100644
--- a/go.sum
+++ b/go.sum
@@ -225,6 +225,8 @@ github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg=
github.com/gin-gonic/gin v1.9.1/go.mod h1:hPrL7YrpYKXt5YId3A/Tnip5kqbEAP+KLuI3SUcPTeU=
github.com/go-chi/chi/v5 v5.1.0 h1:acVI1TYaD+hhedDJ3r54HyA6sExp3HfXq7QWEEY/xMw=
github.com/go-chi/chi/v5 v5.1.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8=
+github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348 h1:JnrjqG5iR07/8k7NqrLNilRsl3s1EPRQEGvbPyOce68=
+github.com/go-darwin/apfs v0.0.0-20211011131704-f84b94dbf348/go.mod h1:Czxo/d1g948LtrALAZdL04TL/HnkopquAjxYUuI02bo=
github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU=
github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=