Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

serve s3: don't buffer re-uploads of multipart parts already streamed

Nick Craig-Wood122b4e8df20a
diff --git a/cmd/serve/s3/multipart.go b/cmd/serve/s3/multipart.go
index d6ab3ed57..95c69fe56 100644
--- a/cmd/serve/s3/multipart.go
+++ b/cmd/serve/s3/multipart.go
@@ -197,6 +197,12 @@ func (b *s3Backend) UploadPart(ctx context.Context, bucketName, objectName strin
up.startActivity()
defer up.endActivity()
+ // A re-upload of a part already in the stream is only hashed, to check
+ // it matches, so it needs no buffer.
+ if md5Sum, size, ok := up.streamedPart(partNumber); ok {
+ return verifyStreamedPart(partNumber, md5Sum, size, contentLength, body)
+ }
+
// Wait until there is room to buffer this part, bounding the memory a
// client which uploads faster than the backend drains can consume.
if err := up.waitForTurn(partNumber, contentLength); err != nil {
@@ -228,6 +234,41 @@ func (b *s3Backend) UploadPart(ctx context.Context, bucketName, objectName strin
return etag, nil
}
+// streamedPart returns the MD5 sum and size of partNumber if it has already
+// been streamed to the sink.
+func (up *multipartUpload) streamedPart(partNumber int) (md5Sum []byte, size int64, ok bool) {
+ up.mu.Lock()
+ defer up.mu.Unlock()
+ if partNumber >= up.nextPart {
+ return nil, 0, false
+ }
+ return up.partMD5s[partNumber], up.partSizes[partNumber], true
+}
+
+// verifyStreamedPart reads the body of a re-upload of partNumber, which has
+// already been streamed with md5Sum and size, returning its ETag if it is
+// identical and an error if not, since the stream can't be rewritten.
+func verifyStreamedPart(partNumber int, md5Sum []byte, size int64, contentLength int64, body io.Reader) (string, error) {
+ hasher := md5.New()
+ n, err := io.Copy(hasher, body)
+ if err != nil {
+ return "", err
+ }
+ if n != contentLength {
+ return "", gofakes3.ErrIncompleteBody
+ }
+ if n != size || !bytes.Equal(hasher.Sum(nil), md5Sum) {
+ return "", errPartAlreadyStreamed(partNumber)
+ }
+ return fmt.Sprintf("%q", hex.EncodeToString(md5Sum)), nil
+}
+
+// errPartAlreadyStreamed is returned for an attempt to replace partNumber
+// with different contents after it has been streamed.
+func errPartAlreadyStreamed(partNumber int) error {
+ return gofakes3.ErrorMessagef(gofakes3.ErrNotImplemented, "part %d has already been streamed to the backend and cannot be replaced with different contents", partNumber)
+}
+
// bufferCharge returns the reorder buffer memory a part of size bytes
// takes: its pool.RW holds it in whole pool pages, however small it is.
func bufferCharge(size int64) int64 {
@@ -317,7 +358,7 @@ func (up *multipartUpload) streamPart(partNumber int, size int64, md5Sum []byte,
up.mu.Unlock()
_ = rw.Close()
if !same {
- return gofakes3.ErrorMessagef(gofakes3.ErrNotImplemented, "part %d has already been streamed to the backend and cannot be replaced with different contents", partNumber)
+ return errPartAlreadyStreamed(partNumber)
}
return nil
}
diff --git a/cmd/serve/s3/multipart_test.go b/cmd/serve/s3/multipart_test.go
index 4e8a57281..03e9da423 100644
--- a/cmd/serve/s3/multipart_test.go
+++ b/cmd/serve/s3/multipart_test.go
@@ -1114,3 +1114,55 @@ func TestMultipartBufferLimitCountsPages(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, []byte("123456"), readObject(t, f, bucket, object))
}
+
+// poolWatchReader is a part body which records the most pool pages in use,
+// above those in use when it was made, while it is being read.
+type poolWatchReader struct {
+ io.Reader
+ baseline int
+ maxPages int
+}
+
+func newPoolWatchReader(data []byte) *poolWatchReader {
+ return &poolWatchReader{Reader: bytes.NewReader(data), baseline: pool.Global().InUse()}
+}
+
+func (r *poolWatchReader) Read(p []byte) (int, error) {
+ r.maxPages = max(r.maxPages, pool.Global().InUse()-r.baseline)
+ n, err := r.Reader.Read(p)
+ r.maxPages = max(r.maxPages, pool.Global().InUse()-r.baseline)
+ return n, err
+}
+
+// TestMultipartRetryAfterStreamedNotBuffered checks that a re-upload of a
+// part which has already been streamed is only hashed to compare it with the
+// original, not buffered in memory.
+func TestMultipartRetryAfterStreamedNotBuffered(t *testing.T) {
+ b, f, bucket := newPutTestBackend(t, "", nil)
+ ctx := context.Background()
+ const object = "retry-not-buffered.bin"
+ const partSize = 8 * pool.BufferSize
+
+ uploadID, err := b.CreateMultipartUpload(ctx, bucket, object, nil)
+ require.NoError(t, err)
+ part1 := []byte(random.String(partSize))
+ etag, err := b.UploadPart(ctx, bucket, object, uploadID, 1, partSize, bytes.NewReader(part1))
+ require.NoError(t, err)
+
+ body := newPoolWatchReader(part1)
+ retryETag, err := b.UploadPart(ctx, bucket, object, uploadID, 1, partSize, body)
+ require.NoError(t, err)
+ assert.Equal(t, etag, retryETag)
+ assert.Less(t, body.maxPages, partSize/pool.BufferSize/2, "the retried part was buffered in memory")
+
+ // A different re-upload is still rejected.
+ other := []byte(random.String(partSize))
+ _, err = b.UploadPart(ctx, bucket, object, uploadID, 1, partSize, bytes.NewReader(other))
+ assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrNotImplemented), "want NotImplemented, got %v", err)
+
+ _, _, err = b.CompleteMultipartUpload(ctx, bucket, object, uploadID, &gofakes3.CompleteMultipartUploadRequest{
+ Parts: []gofakes3.CompletedPart{{PartNumber: 1, ETag: etag}},
+ })
+ require.NoError(t, err)
+ assert.Equal(t, part1, readObject(t, f, bucket, object))
+}