"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
serve s3: time out multipart parts waiting for buffer space
Nick Craig-Wood603e6238eb20
diff --git a/cmd/serve/s3/multipart.go b/cmd/serve/s3/multipart.goindex 95c69fe56..1a852f138 100644--- a/cmd/serve/s3/multipart.go+++ b/cmd/serve/s3/multipart.go@@ -45,6 +45,16 @@ import (// a streamed multipart upload is written to before it is moved into place.const multipartUploadPrefix = tempObjectPrefix + "multipart_"+// multipartWaitTimeout is how long a part waits for room in the reorder+// buffer before the client is told to slow down and retry it.+var multipartWaitTimeout = time.Minute++// errSlowDown asks the client to retry the request later.+//+// FIXME gofakes3 sends this with status 500 rather than the 503 S3 uses,+// though S3 clients retry either.+const errSlowDown gofakes3.ErrorCode = "SlowDown"+// multipartUpload tracks one in-flight S3 multipart upload. The parts are// written, in part-number order, into fh - a VFS file handle which either// streams straight through to the remote (the default) or is backed by the@@ -59,7 +69,7 @@ type multipartUpload struct {vfs *vfs.VFS // the VFS fh was created on, used for all later operationsmu sync.Mutex- cond *sync.Cond // signalled when buffered shrinks, nextPart advances or the upload closes+ changed chan struct{} // closed when buffered shrinks, nextPart advances or the upload closespartMD5s map[int][]byte // raw MD5 sums per part (for the final S3 multipart ETag)partSizes map[int]int64 // observed part sizesclosed bool@@ -88,11 +98,17 @@ func newMultipartUpload(bucket, key, fp, streamFp string, meta map[string]stringstreamBuf: map[int]*pool.RW{},bufferLimit: bufferLimit,lastUsed: time.Now(),+ changed: make(chan struct{}),}- up.cond = sync.NewCond(&up.mu)return up}+// broadcast wakes everything waiting on up.changed. Call with up.mu held.+func (up *multipartUpload) broadcast() {+ close(up.changed)+ up.changed = make(chan struct{})+}+// startActivity marks the upload as having a request in flight.func (up *multipartUpload) startActivity() {up.mu.Lock()@@ -205,7 +221,7 @@ func (b *s3Backend) UploadPart(ctx context.Context, bucketName, objectName strin// Wait until there is room to buffer this part, bounding the memory a// client which uploads faster than the backend drains can consume.- if err := up.waitForTurn(partNumber, contentLength); err != nil {+ if err := up.waitForTurn(ctx, partNumber, contentLength); err != nil {return "", err}@@ -288,25 +304,40 @@ func bufferCharge(size int64) int64 {// limit when the buffer is empty, to guarantee progress. Reserved bytes are// returned with release, or by the pump as the part is streamed.//+// A part which waits for longer than multipartWaitTimeout gets errSlowDown,+// asking the client to retry it later, so a waiting request isn't held open+// indefinitely. The wait also ends if ctx is cancelled.+//// size is the client-declared part length and is not trusted: a negative value// is rejected, and the admission test is written so a huge value can't overflow// the running total and wrongly admit further parts past the limit.-func (up *multipartUpload) waitForTurn(partNumber int, size int64) error {+func (up *multipartUpload) waitForTurn(ctx context.Context, partNumber int, size int64) error {if size < 0 {return gofakes3.ErrInvalidArgument}size = bufferCharge(size)- up.mu.Lock()- defer up.mu.Unlock()+ timer := time.NewTimer(multipartWaitTimeout)+ defer timer.Stop()for {+ up.mu.Lock()if up.closed {+ up.mu.Unlock()return gofakes3.ErrNoSuchUpload}if up.bufferLimit <= 0 || partNumber <= up.nextPart || up.buffered == 0 || size <= up.bufferLimit-up.buffered {up.buffered += size+ up.mu.Unlock()return nil}- up.cond.Wait()+ changed := up.changed+ up.mu.Unlock()+ select {+ case <-changed:+ case <-ctx.Done():+ return ctx.Err()+ case <-timer.C:+ return errSlowDown+ }}}@@ -315,7 +346,7 @@ func (up *multipartUpload) waitForTurn(partNumber int, size int64) error {func (up *multipartUpload) release(size int64) {up.mu.Lock()up.buffered -= bufferCharge(size)- up.cond.Broadcast()+ up.broadcast()up.mu.Unlock()}@@ -339,7 +370,7 @@ func (up *multipartUpload) streamPart(partNumber int, size int64, md5Sum []byte,// The upload was aborted or completed while the part body was// being received.up.buffered -= bufferCharge(size)- up.cond.Broadcast()+ up.broadcast()up.mu.Unlock()_ = rw.Close()return gofakes3.ErrNoSuchUpload@@ -348,13 +379,13 @@ func (up *multipartUpload) streamPart(partNumber int, size int64, md5Sum []byte,if old, buffered := up.streamBuf[partNumber]; buffered {_ = old.Close()up.buffered -= bufferCharge(up.partSizes[partNumber])- up.cond.Broadcast()+ up.broadcast()} else {// Already streamed (or streaming right now): the stream can't be// rewritten, so accept an identical part and reject the rest.same := bytes.Equal(md5Sum, oldMD5) && size == up.partSizes[partNumber]up.buffered -= bufferCharge(size)- up.cond.Broadcast()+ up.broadcast()up.mu.Unlock()_ = rw.Close()if !same {@@ -400,7 +431,7 @@ func (up *multipartUpload) streamPart(partNumber int, size int64, md5Sum []byte,up.pumping = falseup.buffered -= psizeclosed := up.closed- up.cond.Broadcast()+ up.broadcast()up.mu.Unlock()if closed {// The write failed because the upload was aborted while@@ -414,7 +445,7 @@ func (up *multipartUpload) streamPart(partNumber int, size int64, md5Sum []byte,up.mu.Lock()up.nextPart++up.buffered -= psize- up.cond.Broadcast()+ up.broadcast()}}@@ -626,7 +657,7 @@ func (up *multipartUpload) close() error {return gofakes3.ErrInvalidPart}up.closed = true- up.cond.Broadcast()+ up.broadcast()up.mu.Unlock()return up.fh.Close()@@ -652,7 +683,7 @@ func (up *multipartUpload) abort() error {up.aborted = truestreamBuf := up.streamBufup.streamBuf = nil- up.cond.Broadcast()+ up.broadcast()up.mu.Unlock()for _, rw := range streamBuf {diff --git a/cmd/serve/s3/multipart_test.go b/cmd/serve/s3/multipart_test.goindex 03e9da423..fb483ee44 100644--- a/cmd/serve/s3/multipart_test.go+++ b/cmd/serve/s3/multipart_test.go@@ -502,7 +502,7 @@ func TestMultipartAbortDuringUploadPart(t *testing.T) {// An UploadPart in progress: the part is admitted, then the abort lands// while its body is still being received.contents := []byte("hello")- require.NoError(t, up.waitForTurn(1, int64(len(contents))))+ require.NoError(t, up.waitForTurn(context.Background(), 1, int64(len(contents))))require.NoError(t, up.abort())// The abort must abandon the write rather than committing it.@@ -555,7 +555,7 @@ func TestMultipartCloseIncomplete(t *testing.T) {_, err := rw.Write(contents)require.NoError(t, err)md5Sum := md5.Sum(contents)- require.NoError(t, up.waitForTurn(2, int64(len(contents))))+ require.NoError(t, up.waitForTurn(context.Background(), 2, int64(len(contents))))require.NoError(t, up.streamPart(2, int64(len(contents)), md5Sum[:], rw))require.ErrorIs(t, up.close(), gofakes3.ErrInvalidPart)@@ -1032,17 +1032,17 @@ func TestWaitForTurnRejectsBogusSize(t *testing.T) {up := newMultipartUpload("bucket", "key", "bucket/key", "bucket/key", nil, 1<<20)// A part length can never be negative.- require.ErrorIs(t, up.waitForTurn(1, -1), gofakes3.ErrInvalidArgument)+ require.ErrorIs(t, up.waitForTurn(context.Background(), 1, -1), gofakes3.ErrInvalidArgument)// A huge out-of-order part is admitted once because the buffer is empty,// driving buffered near the top of the int64 range.- require.NoError(t, up.waitForTurn(2, math.MaxInt64))+ require.NoError(t, up.waitForTurn(context.Background(), 2, math.MaxInt64))// A further out-of-order part must wait, not be wrongly admitted by an// overflow of buffered+size.admitted := make(chan struct{})go func() {- _ = up.waitForTurn(3, math.MaxInt64)+ _ = up.waitForTurn(context.Background(), 3, math.MaxInt64)close(admitted)}()select {@@ -1054,7 +1054,7 @@ func TestWaitForTurnRejectsBogusSize(t *testing.T) {// Wake the blocked goroutine so it doesn't leak.up.mu.Lock()up.closed = true- up.cond.Broadcast()+ up.broadcast()up.mu.Unlock()<-admitted}@@ -1166,3 +1166,34 @@ func TestMultipartRetryAfterStreamedNotBuffered(t *testing.T) {require.NoError(t, err)assert.Equal(t, part1, readObject(t, f, bucket, object))}++// TestMultipartBufferWaitGivesUp checks that a part waiting for room in the+// reorder buffer gives up when its request is cancelled, and tells the client+// to slow down if it has waited too long, so a waiting request can't be held+// open forever.+func TestMultipartBufferWaitGivesUp(t *testing.T) {+ b, _, bucket := newPutTestBackend(t, "", nil)+ b.s.opt.MultipartStreamingBufferLimit = pool.BufferSize+ oldTimeout := multipartWaitTimeout+ multipartWaitTimeout = 100 * time.Millisecond+ t.Cleanup(func() { multipartWaitTimeout = oldTimeout })+ ctx := context.Background()+ const object = "wait.bin"++ uploadID, err := b.CreateMultipartUpload(ctx, bucket, object, nil)+ require.NoError(t, err)++ // Part 2 fills the buffer, so part 3 must wait.+ _, err = b.UploadPart(ctx, bucket, object, uploadID, 2, 1, bytes.NewReader([]byte("2")))+ require.NoError(t, err)++ cancelCtx, cancel := context.WithCancel(ctx)+ time.AfterFunc(10*time.Millisecond, cancel)+ _, err = b.UploadPart(cancelCtx, bucket, object, uploadID, 3, 1, bytes.NewReader([]byte("3")))+ require.ErrorIs(t, err, context.Canceled)++ _, err = b.UploadPart(ctx, bucket, object, uploadID, 3, 1, bytes.NewReader([]byte("3")))+ assert.True(t, gofakes3.HasErrorCode(err, errSlowDown), "want SlowDown, got %v", err)++ require.NoError(t, b.AbortMultipartUpload(ctx, bucket, object, uploadID))+}diff --git a/cmd/serve/s3/serve_s3.md b/cmd/serve/s3/serve_s3.mdindex 74d5e7aed..a2d5a4b5a 100644--- a/cmd/serve/s3/serve_s3.md+++ b/cmd/serve/s3/serve_s3.md@@ -196,7 +196,8 @@ remote as if it had completed.1 MiB memory pages it occupies: a part that would take the buffer over thelimit is stalled until the stream drains, so a client that uploadsfaster than the remote can accept sees backpressure rather than- unbounded server memory use. Since a stalled part holds its HTTP+ unbounded server memory use. A part stalled for more than a minute+ is failed with a `SlowDown` error, which S3 clients retry. Since a stalled part holds its HTTPrequest open, clients whose upload concurrency times chunk sizeexceeds the limit may need a longer read timeout when the remote isslow. Non-contiguous part numbers are rejected on completion.