"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
local: speed up default checksummed copies by writing in larger blocks
Dominik Sandere2352201d112
diff --git a/backend/local/local.go b/backend/local/local.goindex cd0cc3890..a6cf3f2eb 100644--- a/backend/local/local.go+++ b/backend/local/local.go@@ -1650,12 +1650,13 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, opout = nopWriterCloser{&symlinkData}}- // Calculate the hash of the object we are reading as we go along+ // Calculate the hash of the object as we write it+ var w io.Writer = outif hasher != nil {- in = io.TeeReader(in, hasher)+ w = io.MultiWriter(w, hasher)}- _, err = io.Copy(out, in)+ _, err = io.Copy(w, in)closeErr := out.Close()if err == nil {err = closeErrdiff --git a/backend/local/local_internal_test.go b/backend/local/local_internal_test.goindex 761aa8ea6..25adcdf80 100644--- a/backend/local/local_internal_test.go+++ b/backend/local/local_internal_test.go@@ -499,6 +499,32 @@ func TestHashOnUpdate(t *testing.T) {assert.Equal(t, "45685e95985e20822fb2538a522a5ccf", md5)}+// Test the hash cached by Update matches a HashesOption hint passed by the caller+func TestHashOnUpdateWithHashOption(t *testing.T) {+ ctx := context.Background()+ r := fstest.NewRun(t)+ const filePath = "file.txt"+ when := time.Now()+ r.WriteFile(filePath, "x", when)+ f := r.Flocal.(*Fs)++ o, err := f.NewObject(ctx, filePath)+ require.NoError(t, err)++ b := bytes.NewBufferString("content")+ src := object.NewStaticObjectInfo(filePath, when, int64(b.Len()), true, nil, f)+ options := []fs.OpenOption{&fs.HashesOption{Hashes: hash.NewHashSet(hash.MD5)}}+ require.NoError(t, o.Update(ctx, b, src, options...))++ gotContent, err := os.ReadFile(filepath.Join(f.root, filePath))+ require.NoError(t, err)+ assert.Equal(t, "content", string(gotContent))++ md5, err := o.Hash(ctx, hash.MD5)+ require.NoError(t, err)+ assert.Equal(t, "9a0364b9e99bb480dd25e1f0284c8555", md5)+}+// Test hashes on deleting an objectfunc TestHashOnDelete(t *testing.T) {ctx := context.Background()