Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

local: speed up default checksummed copies by writing in larger blocks

Dominik Sandere2352201d112
diff --git a/backend/local/local.go b/backend/local/local.go
index cd0cc3890..a6cf3f2eb 100644
--- a/backend/local/local.go
+++ b/backend/local/local.go
@@ -1650,12 +1650,13 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op
out = nopWriterCloser{&symlinkData}
}
- // Calculate the hash of the object we are reading as we go along
+ // Calculate the hash of the object as we write it
+ var w io.Writer = out
if hasher != nil {
- in = io.TeeReader(in, hasher)
+ w = io.MultiWriter(w, hasher)
}
- _, err = io.Copy(out, in)
+ _, err = io.Copy(w, in)
closeErr := out.Close()
if err == nil {
err = closeErr
diff --git a/backend/local/local_internal_test.go b/backend/local/local_internal_test.go
index 761aa8ea6..25adcdf80 100644
--- a/backend/local/local_internal_test.go
+++ b/backend/local/local_internal_test.go
@@ -499,6 +499,32 @@ func TestHashOnUpdate(t *testing.T) {
assert.Equal(t, "45685e95985e20822fb2538a522a5ccf", md5)
}
+// Test the hash cached by Update matches a HashesOption hint passed by the caller
+func TestHashOnUpdateWithHashOption(t *testing.T) {
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+ const filePath = "file.txt"
+ when := time.Now()
+ r.WriteFile(filePath, "x", when)
+ f := r.Flocal.(*Fs)
+
+ o, err := f.NewObject(ctx, filePath)
+ require.NoError(t, err)
+
+ b := bytes.NewBufferString("content")
+ src := object.NewStaticObjectInfo(filePath, when, int64(b.Len()), true, nil, f)
+ options := []fs.OpenOption{&fs.HashesOption{Hashes: hash.NewHashSet(hash.MD5)}}
+ require.NoError(t, o.Update(ctx, b, src, options...))
+
+ gotContent, err := os.ReadFile(filepath.Join(f.root, filePath))
+ require.NoError(t, err)
+ assert.Equal(t, "content", string(gotContent))
+
+ md5, err := o.Hash(ctx, hash.MD5)
+ require.NoError(t, err)
+ assert.Equal(t, "9a0364b9e99bb480dd25e1f0284c8555", md5)
+}
+
// Test hashes on deleting an object
func TestHashOnDelete(t *testing.T) {
ctx := context.Background()