"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
serve s3: don't buffer re-uploads of multipart parts already streamed
Nick Craig-Wood122b4e8df20a
diff --git a/cmd/serve/s3/multipart.go b/cmd/serve/s3/multipart.goindex d6ab3ed57..95c69fe56 100644--- a/cmd/serve/s3/multipart.go+++ b/cmd/serve/s3/multipart.go@@ -197,6 +197,12 @@ func (b *s3Backend) UploadPart(ctx context.Context, bucketName, objectName strinup.startActivity()defer up.endActivity()+ // A re-upload of a part already in the stream is only hashed, to check+ // it matches, so it needs no buffer.+ if md5Sum, size, ok := up.streamedPart(partNumber); ok {+ return verifyStreamedPart(partNumber, md5Sum, size, contentLength, body)+ }+// Wait until there is room to buffer this part, bounding the memory a// client which uploads faster than the backend drains can consume.if err := up.waitForTurn(partNumber, contentLength); err != nil {@@ -228,6 +234,41 @@ func (b *s3Backend) UploadPart(ctx context.Context, bucketName, objectName strinreturn etag, nil}+// streamedPart returns the MD5 sum and size of partNumber if it has already+// been streamed to the sink.+func (up *multipartUpload) streamedPart(partNumber int) (md5Sum []byte, size int64, ok bool) {+ up.mu.Lock()+ defer up.mu.Unlock()+ if partNumber >= up.nextPart {+ return nil, 0, false+ }+ return up.partMD5s[partNumber], up.partSizes[partNumber], true+}++// verifyStreamedPart reads the body of a re-upload of partNumber, which has+// already been streamed with md5Sum and size, returning its ETag if it is+// identical and an error if not, since the stream can't be rewritten.+func verifyStreamedPart(partNumber int, md5Sum []byte, size int64, contentLength int64, body io.Reader) (string, error) {+ hasher := md5.New()+ n, err := io.Copy(hasher, body)+ if err != nil {+ return "", err+ }+ if n != contentLength {+ return "", gofakes3.ErrIncompleteBody+ }+ if n != size || !bytes.Equal(hasher.Sum(nil), md5Sum) {+ return "", errPartAlreadyStreamed(partNumber)+ }+ return fmt.Sprintf("%q", hex.EncodeToString(md5Sum)), nil+}++// errPartAlreadyStreamed is returned for an attempt to replace partNumber+// with different contents after it has been streamed.+func errPartAlreadyStreamed(partNumber int) error {+ return gofakes3.ErrorMessagef(gofakes3.ErrNotImplemented, "part %d has already been streamed to the backend and cannot be replaced with different contents", partNumber)+}+// bufferCharge returns the reorder buffer memory a part of size bytes// takes: its pool.RW holds it in whole pool pages, however small it is.func bufferCharge(size int64) int64 {@@ -317,7 +358,7 @@ func (up *multipartUpload) streamPart(partNumber int, size int64, md5Sum []byte,up.mu.Unlock()_ = rw.Close()if !same {- return gofakes3.ErrorMessagef(gofakes3.ErrNotImplemented, "part %d has already been streamed to the backend and cannot be replaced with different contents", partNumber)+ return errPartAlreadyStreamed(partNumber)}return nil}diff --git a/cmd/serve/s3/multipart_test.go b/cmd/serve/s3/multipart_test.goindex 4e8a57281..03e9da423 100644--- a/cmd/serve/s3/multipart_test.go+++ b/cmd/serve/s3/multipart_test.go@@ -1114,3 +1114,55 @@ func TestMultipartBufferLimitCountsPages(t *testing.T) {require.NoError(t, err)assert.Equal(t, []byte("123456"), readObject(t, f, bucket, object))}++// poolWatchReader is a part body which records the most pool pages in use,+// above those in use when it was made, while it is being read.+type poolWatchReader struct {+ io.Reader+ baseline int+ maxPages int+}++func newPoolWatchReader(data []byte) *poolWatchReader {+ return &poolWatchReader{Reader: bytes.NewReader(data), baseline: pool.Global().InUse()}+}++func (r *poolWatchReader) Read(p []byte) (int, error) {+ r.maxPages = max(r.maxPages, pool.Global().InUse()-r.baseline)+ n, err := r.Reader.Read(p)+ r.maxPages = max(r.maxPages, pool.Global().InUse()-r.baseline)+ return n, err+}++// TestMultipartRetryAfterStreamedNotBuffered checks that a re-upload of a+// part which has already been streamed is only hashed to compare it with the+// original, not buffered in memory.+func TestMultipartRetryAfterStreamedNotBuffered(t *testing.T) {+ b, f, bucket := newPutTestBackend(t, "", nil)+ ctx := context.Background()+ const object = "retry-not-buffered.bin"+ const partSize = 8 * pool.BufferSize++ uploadID, err := b.CreateMultipartUpload(ctx, bucket, object, nil)+ require.NoError(t, err)+ part1 := []byte(random.String(partSize))+ etag, err := b.UploadPart(ctx, bucket, object, uploadID, 1, partSize, bytes.NewReader(part1))+ require.NoError(t, err)++ body := newPoolWatchReader(part1)+ retryETag, err := b.UploadPart(ctx, bucket, object, uploadID, 1, partSize, body)+ require.NoError(t, err)+ assert.Equal(t, etag, retryETag)+ assert.Less(t, body.maxPages, partSize/pool.BufferSize/2, "the retried part was buffered in memory")++ // A different re-upload is still rejected.+ other := []byte(random.String(partSize))+ _, err = b.UploadPart(ctx, bucket, object, uploadID, 1, partSize, bytes.NewReader(other))+ assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrNotImplemented), "want NotImplemented, got %v", err)++ _, _, err = b.CompleteMultipartUpload(ctx, bucket, object, uploadID, &gofakes3.CompleteMultipartUploadRequest{+ Parts: []gofakes3.CompletedPart{{PartNumber: 1, ETag: etag}},+ })+ require.NoError(t, err)+ assert.Equal(t, part1, readObject(t, f, bucket, object))+}