"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
serve s3: count buffered multipart parts in whole memory pages
Nick Craig-Wood4aa3c41aa1d9
diff --git a/cmd/serve/s3/multipart.go b/cmd/serve/s3/multipart.goindex 11155abf9..d6ab3ed57 100644--- a/cmd/serve/s3/multipart.go+++ b/cmd/serve/s3/multipart.go@@ -23,6 +23,7 @@ import ("errors""fmt""io"+ "math""path""sort""strings"@@ -227,9 +228,19 @@ func (b *s3Backend) UploadPart(ctx context.Context, bucketName, objectName strinreturn etag, nil}-// waitForTurn blocks until size bytes can be admitted to the reorder buffer,-// then reserves them, bounding the memory an upload can consume when the-// client sends parts faster than the backend drains them.+// bufferCharge returns the reorder buffer memory a part of size bytes+// takes: its pool.RW holds it in whole pool pages, however small it is.+func bufferCharge(size int64) int64 {+ if size > math.MaxInt64-pool.BufferSize {+ return math.MaxInt64+ }+ return (size + pool.BufferSize - 1) / pool.BufferSize * pool.BufferSize+}++// waitForTurn blocks until a part of size bytes can be admitted to the+// reorder buffer, then reserves its bufferCharge, bounding the memory an+// upload can consume when the client sends parts faster than the backend+// drains them.//// The next part the stream needs (and any retry of an earlier one) is always// admitted so the sink can keep draining; so is a single part bigger than the@@ -243,6 +254,7 @@ func (up *multipartUpload) waitForTurn(partNumber int, size int64) error {if size < 0 {return gofakes3.ErrInvalidArgument}+ size = bufferCharge(size)up.mu.Lock()defer up.mu.Unlock()for {@@ -257,11 +269,11 @@ func (up *multipartUpload) waitForTurn(partNumber int, size int64) error {}}-// release returns size bytes reserved by waitForTurn to the reorder buffer-// budget and wakes any parts waiting for room.+// release returns the reservation waitForTurn made for a part of size bytes+// to the reorder buffer budget and wakes any parts waiting for room.func (up *multipartUpload) release(size int64) {up.mu.Lock()- up.buffered -= size+ up.buffered -= bufferCharge(size)up.cond.Broadcast()up.mu.Unlock()}@@ -285,7 +297,7 @@ func (up *multipartUpload) streamPart(partNumber int, size int64, md5Sum []byte,if up.closed {// The upload was aborted or completed while the part body was// being received.- up.buffered -= size+ up.buffered -= bufferCharge(size)up.cond.Broadcast()up.mu.Unlock()_ = rw.Close()@@ -294,13 +306,13 @@ func (up *multipartUpload) streamPart(partNumber int, size int64, md5Sum []byte,if oldMD5, exists := up.partMD5s[partNumber]; exists {if old, buffered := up.streamBuf[partNumber]; buffered {_ = old.Close()- up.buffered -= up.partSizes[partNumber]+ up.buffered -= bufferCharge(up.partSizes[partNumber])up.cond.Broadcast()} else {// Already streamed (or streaming right now): the stream can't be// rewritten, so accept an identical part and reject the rest.same := bytes.Equal(md5Sum, oldMD5) && size == up.partSizes[partNumber]- up.buffered -= size+ up.buffered -= bufferCharge(size)up.cond.Broadcast()up.mu.Unlock()_ = rw.Close()@@ -337,7 +349,7 @@ func (up *multipartUpload) streamPart(partNumber int, size int64, md5Sum []byte,return nil}delete(up.streamBuf, up.nextPart)- psize := prw.Size()+ psize := bufferCharge(prw.Size())up.mu.Unlock()err := pipePart(up.fh, prw)diff --git a/cmd/serve/s3/multipart_test.go b/cmd/serve/s3/multipart_test.goindex 5caea3e30..4e8a57281 100644--- a/cmd/serve/s3/multipart_test.go+++ b/cmd/serve/s3/multipart_test.go@@ -1058,3 +1058,59 @@ func TestWaitForTurnRejectsBogusSize(t *testing.T) {up.mu.Unlock()<-admitted}++// TestMultipartBufferLimitCountsPages checks that the reorder buffer limit+// is charged in the whole pool pages a buffered part occupies, not the part's+// length, so a client can't pin a page per byte by sending tiny parts out of+// order.+func TestMultipartBufferLimitCountsPages(t *testing.T) {+ b, f, bucket := newPutTestBackend(t, "", nil)+ b.s.opt.MultipartStreamingBufferLimit = 4 * pool.BufferSize+ ctx := context.Background()+ const object = "tiny-parts.bin"++ uploadID, err := b.CreateMultipartUpload(ctx, bucket, object, nil)+ require.NoError(t, err)++ // Parts 2 to 5 fill the buffer's 4 pages with one byte each.+ upload := func(partNumber int) (string, error) {+ body := []byte{byte('0' + partNumber)}+ return b.UploadPart(ctx, bucket, object, uploadID, partNumber, 1, bytes.NewReader(body))+ }+ etags := make([]string, 7)+ for partNumber := 2; partNumber <= 5; partNumber++ {+ etags[partNumber], err = upload(partNumber)+ require.NoError(t, err)+ }++ // Part 6 needs a fifth page so must wait for the buffer to drain.+ done := make(chan error, 1)+ go func() {+ var err error+ etags[6], err = upload(6)+ done <- err+ }()+ select {+ case err := <-done:+ t.Fatalf("part 6 was not blocked by the buffer limit (err=%v)", err)+ case <-time.After(200 * time.Millisecond):+ }++ // Part 1 lets the buffered parts stream, freeing the pages for part 6.+ etags[1], err = upload(1)+ require.NoError(t, err)+ select {+ case err := <-done:+ require.NoError(t, err)+ case <-time.After(10 * time.Second):+ t.Fatal("part 6 was never unblocked")+ }++ parts := make([]gofakes3.CompletedPart, 0, 6)+ for partNumber := 1; partNumber <= 6; partNumber++ {+ parts = append(parts, gofakes3.CompletedPart{PartNumber: partNumber, ETag: etags[partNumber]})+ }+ _, _, err = b.CompleteMultipartUpload(ctx, bucket, object, uploadID, &gofakes3.CompleteMultipartUploadRequest{Parts: parts})+ require.NoError(t, err)+ assert.Equal(t, []byte("123456"), readObject(t, f, bucket, object))+}diff --git a/cmd/serve/s3/serve_s3.md b/cmd/serve/s3/serve_s3.mdindex 6bfe3aaba..74d5e7aed 100644--- a/cmd/serve/s3/serve_s3.md+++ b/cmd/serve/s3/serve_s3.md@@ -192,7 +192,8 @@ remote as if it had completed.(1, 2, 3, ...). Parts the client uploads concurrently or out of orderare buffered until their turn. The memory used for this buffering iscapped, per upload, by `--multipart-streaming-buffer-limit` (default- `256M`, `0` for no limit): a part that would take the buffer over the+ `256M`, `0` for no limit), counting each buffered part in the whole+ 1 MiB memory pages it occupies: a part that would take the buffer over thelimit is stalled until the stream drains, so a client that uploadsfaster than the remote can accept sees backpressure rather thanunbounded server memory use. Since a stalled part holds its HTTP