"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
serve s3: don't compute the MD5 of multipart uploads unless the ETag is MD5
Nick Craig-Woodfc9e6a5ca490
diff --git a/cmd/serve/s3/multipart.go b/cmd/serve/s3/multipart.goindex 518b0afc8..da4450923 100644--- a/cmd/serve/s3/multipart.go+++ b/cmd/serve/s3/multipart.go@@ -65,7 +65,7 @@ type multipartUpload struct {vfs *vfs.VFS // the VFS fh is created on, used for all later operationstenant *tenant // the user the upload belongs to- objectMD5 hash.Hash // MD5 of the data written to the sink, so of the whole object once complete+ objectMD5 hash.Hash // MD5 of the data written to the sink, so of the whole object once complete, or nil if not needed for the ETagmu sync.Mutexchanged chan struct{} // closed when buffered shrinks, nextPart advances or the upload closes@@ -140,7 +140,6 @@ func newMultipartUpload(bucket, key, fp, streamFp string, meta map[string]stringfp: fp,streamFp: streamFp,meta: meta,- objectMD5: md5.New(),partMD5s: map[int][]byte{},partSizes: map[int]int64{},nextPart: 1,@@ -326,6 +325,9 @@ func (b *s3Backend) CreateMultipartUpload(ctx context.Context, bucketName, objecup.budget = t.budgetup.vfs = _vfsup.madeDir = madeDir+ if b.s.etagHash(_vfs) == fshash.MD5 {+ up.objectMD5 = md5.New()+ }if err := b.addUpload(t, uploadID, up); err != nil {removeMadeDirs(objectDir, madeDir, _vfs)@@ -701,6 +703,9 @@ func (up *multipartUpload) sink() (io.Writer, error) {}up.fh = fh}+ if up.objectMD5 == nil {+ return up.fh, nil+ }return hashingWriter{w: up.fh, hash: up.objectMD5}, nil}@@ -932,7 +937,7 @@ func (b *s3Backend) CompleteMultipartUpload(ctx context.Context, bucketName, obj// stored with the part sizes so HEAD returns it while the object// is unchanged.var etagHash []byte- if b.s.etagHash(up.vfs) == fshash.MD5 {+ if up.objectMD5 != nil {etagHash = up.objectMD5.Sum(nil)}_ = up.tenant.storeObjectMeta(up.vfs, up.fp, objectMeta{diff --git a/cmd/serve/s3/multipart_test.go b/cmd/serve/s3/multipart_test.goindex b1f0626ac..125b0dce4 100644--- a/cmd/serve/s3/multipart_test.go+++ b/cmd/serve/s3/multipart_test.go@@ -2103,3 +2103,32 @@ func TestETagMD5WithoutRemoteMD5(t *testing.T) {}assert.Equal(t, etags, listed)}++// TestMultipartObjectMD5OnlyForMD5ETags checks that a multipart upload only+// hashes the object as it is written when the ETag is its MD5.+func TestMultipartObjectMD5OnlyForMD5ETags(t *testing.T) {+ ctx := context.Background()+ for _, tc := range []struct {+ etagHash string+ want bool+ }{+ {"MD5", true},+ {"", false},+ {"SHA1", false},+ } {+ t.Run(fmt.Sprintf("%q", tc.etagHash), func(t *testing.T) {+ b, _ := newTestBackend(t)+ b.s.opt.EtagHash = tc.etagHash+ b.s.etagHashType = hash.None+ if tc.etagHash != "" {+ require.NoError(t, b.s.etagHashType.Set(tc.etagHash))+ }+ uploadID, err := b.CreateMultipartUpload(ctx, "bucket", "key", nil)+ require.NoError(t, err)+ up, err := b.loadUpload(ctx, uploadID)+ require.NoError(t, err)+ assert.Equal(t, tc.want, up.objectMD5 != nil)+ require.NoError(t, b.AbortMultipartUpload(ctx, "bucket", "key", uploadID))+ })+ }+}