Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

serve s3: time out multipart parts waiting for buffer space

Nick Craig-Wood603e6238eb20
diff --git a/cmd/serve/s3/multipart.go b/cmd/serve/s3/multipart.go
index 95c69fe56..1a852f138 100644
--- a/cmd/serve/s3/multipart.go
+++ b/cmd/serve/s3/multipart.go
@@ -45,6 +45,16 @@ import (
// a streamed multipart upload is written to before it is moved into place.
const multipartUploadPrefix = tempObjectPrefix + "multipart_"
+// multipartWaitTimeout is how long a part waits for room in the reorder
+// buffer before the client is told to slow down and retry it.
+var multipartWaitTimeout = time.Minute
+
+// errSlowDown asks the client to retry the request later.
+//
+// FIXME gofakes3 sends this with status 500 rather than the 503 S3 uses,
+// though S3 clients retry either.
+const errSlowDown gofakes3.ErrorCode = "SlowDown"
+
// multipartUpload tracks one in-flight S3 multipart upload. The parts are
// written, in part-number order, into fh - a VFS file handle which either
// streams straight through to the remote (the default) or is backed by the
@@ -59,7 +69,7 @@ type multipartUpload struct {
vfs *vfs.VFS // the VFS fh was created on, used for all later operations
mu sync.Mutex
- cond *sync.Cond // signalled when buffered shrinks, nextPart advances or the upload closes
+ changed chan struct{} // closed when buffered shrinks, nextPart advances or the upload closes
partMD5s map[int][]byte // raw MD5 sums per part (for the final S3 multipart ETag)
partSizes map[int]int64 // observed part sizes
closed bool
@@ -88,11 +98,17 @@ func newMultipartUpload(bucket, key, fp, streamFp string, meta map[string]string
streamBuf: map[int]*pool.RW{},
bufferLimit: bufferLimit,
lastUsed: time.Now(),
+ changed: make(chan struct{}),
}
- up.cond = sync.NewCond(&up.mu)
return up
}
+// broadcast wakes everything waiting on up.changed. Call with up.mu held.
+func (up *multipartUpload) broadcast() {
+ close(up.changed)
+ up.changed = make(chan struct{})
+}
+
// startActivity marks the upload as having a request in flight.
func (up *multipartUpload) startActivity() {
up.mu.Lock()
@@ -205,7 +221,7 @@ func (b *s3Backend) UploadPart(ctx context.Context, bucketName, objectName strin
// Wait until there is room to buffer this part, bounding the memory a
// client which uploads faster than the backend drains can consume.
- if err := up.waitForTurn(partNumber, contentLength); err != nil {
+ if err := up.waitForTurn(ctx, partNumber, contentLength); err != nil {
return "", err
}
@@ -288,25 +304,40 @@ func bufferCharge(size int64) int64 {
// limit when the buffer is empty, to guarantee progress. Reserved bytes are
// returned with release, or by the pump as the part is streamed.
//
+// A part which waits for longer than multipartWaitTimeout gets errSlowDown,
+// asking the client to retry it later, so a waiting request isn't held open
+// indefinitely. The wait also ends if ctx is cancelled.
+//
// size is the client-declared part length and is not trusted: a negative value
// is rejected, and the admission test is written so a huge value can't overflow
// the running total and wrongly admit further parts past the limit.
-func (up *multipartUpload) waitForTurn(partNumber int, size int64) error {
+func (up *multipartUpload) waitForTurn(ctx context.Context, partNumber int, size int64) error {
if size < 0 {
return gofakes3.ErrInvalidArgument
}
size = bufferCharge(size)
- up.mu.Lock()
- defer up.mu.Unlock()
+ timer := time.NewTimer(multipartWaitTimeout)
+ defer timer.Stop()
for {
+ up.mu.Lock()
if up.closed {
+ up.mu.Unlock()
return gofakes3.ErrNoSuchUpload
}
if up.bufferLimit <= 0 || partNumber <= up.nextPart || up.buffered == 0 || size <= up.bufferLimit-up.buffered {
up.buffered += size
+ up.mu.Unlock()
return nil
}
- up.cond.Wait()
+ changed := up.changed
+ up.mu.Unlock()
+ select {
+ case <-changed:
+ case <-ctx.Done():
+ return ctx.Err()
+ case <-timer.C:
+ return errSlowDown
+ }
}
}
@@ -315,7 +346,7 @@ func (up *multipartUpload) waitForTurn(partNumber int, size int64) error {
func (up *multipartUpload) release(size int64) {
up.mu.Lock()
up.buffered -= bufferCharge(size)
- up.cond.Broadcast()
+ up.broadcast()
up.mu.Unlock()
}
@@ -339,7 +370,7 @@ func (up *multipartUpload) streamPart(partNumber int, size int64, md5Sum []byte,
// The upload was aborted or completed while the part body was
// being received.
up.buffered -= bufferCharge(size)
- up.cond.Broadcast()
+ up.broadcast()
up.mu.Unlock()
_ = rw.Close()
return gofakes3.ErrNoSuchUpload
@@ -348,13 +379,13 @@ func (up *multipartUpload) streamPart(partNumber int, size int64, md5Sum []byte,
if old, buffered := up.streamBuf[partNumber]; buffered {
_ = old.Close()
up.buffered -= bufferCharge(up.partSizes[partNumber])
- up.cond.Broadcast()
+ up.broadcast()
} else {
// Already streamed (or streaming right now): the stream can't be
// rewritten, so accept an identical part and reject the rest.
same := bytes.Equal(md5Sum, oldMD5) && size == up.partSizes[partNumber]
up.buffered -= bufferCharge(size)
- up.cond.Broadcast()
+ up.broadcast()
up.mu.Unlock()
_ = rw.Close()
if !same {
@@ -400,7 +431,7 @@ func (up *multipartUpload) streamPart(partNumber int, size int64, md5Sum []byte,
up.pumping = false
up.buffered -= psize
closed := up.closed
- up.cond.Broadcast()
+ up.broadcast()
up.mu.Unlock()
if closed {
// The write failed because the upload was aborted while
@@ -414,7 +445,7 @@ func (up *multipartUpload) streamPart(partNumber int, size int64, md5Sum []byte,
up.mu.Lock()
up.nextPart++
up.buffered -= psize
- up.cond.Broadcast()
+ up.broadcast()
}
}
@@ -626,7 +657,7 @@ func (up *multipartUpload) close() error {
return gofakes3.ErrInvalidPart
}
up.closed = true
- up.cond.Broadcast()
+ up.broadcast()
up.mu.Unlock()
return up.fh.Close()
@@ -652,7 +683,7 @@ func (up *multipartUpload) abort() error {
up.aborted = true
streamBuf := up.streamBuf
up.streamBuf = nil
- up.cond.Broadcast()
+ up.broadcast()
up.mu.Unlock()
for _, rw := range streamBuf {
diff --git a/cmd/serve/s3/multipart_test.go b/cmd/serve/s3/multipart_test.go
index 03e9da423..fb483ee44 100644
--- a/cmd/serve/s3/multipart_test.go
+++ b/cmd/serve/s3/multipart_test.go
@@ -502,7 +502,7 @@ func TestMultipartAbortDuringUploadPart(t *testing.T) {
// An UploadPart in progress: the part is admitted, then the abort lands
// while its body is still being received.
contents := []byte("hello")
- require.NoError(t, up.waitForTurn(1, int64(len(contents))))
+ require.NoError(t, up.waitForTurn(context.Background(), 1, int64(len(contents))))
require.NoError(t, up.abort())
// The abort must abandon the write rather than committing it.
@@ -555,7 +555,7 @@ func TestMultipartCloseIncomplete(t *testing.T) {
_, err := rw.Write(contents)
require.NoError(t, err)
md5Sum := md5.Sum(contents)
- require.NoError(t, up.waitForTurn(2, int64(len(contents))))
+ require.NoError(t, up.waitForTurn(context.Background(), 2, int64(len(contents))))
require.NoError(t, up.streamPart(2, int64(len(contents)), md5Sum[:], rw))
require.ErrorIs(t, up.close(), gofakes3.ErrInvalidPart)
@@ -1032,17 +1032,17 @@ func TestWaitForTurnRejectsBogusSize(t *testing.T) {
up := newMultipartUpload("bucket", "key", "bucket/key", "bucket/key", nil, 1<<20)
// A part length can never be negative.
- require.ErrorIs(t, up.waitForTurn(1, -1), gofakes3.ErrInvalidArgument)
+ require.ErrorIs(t, up.waitForTurn(context.Background(), 1, -1), gofakes3.ErrInvalidArgument)
// A huge out-of-order part is admitted once because the buffer is empty,
// driving buffered near the top of the int64 range.
- require.NoError(t, up.waitForTurn(2, math.MaxInt64))
+ require.NoError(t, up.waitForTurn(context.Background(), 2, math.MaxInt64))
// A further out-of-order part must wait, not be wrongly admitted by an
// overflow of buffered+size.
admitted := make(chan struct{})
go func() {
- _ = up.waitForTurn(3, math.MaxInt64)
+ _ = up.waitForTurn(context.Background(), 3, math.MaxInt64)
close(admitted)
}()
select {
@@ -1054,7 +1054,7 @@ func TestWaitForTurnRejectsBogusSize(t *testing.T) {
// Wake the blocked goroutine so it doesn't leak.
up.mu.Lock()
up.closed = true
- up.cond.Broadcast()
+ up.broadcast()
up.mu.Unlock()
<-admitted
}
@@ -1166,3 +1166,34 @@ func TestMultipartRetryAfterStreamedNotBuffered(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, part1, readObject(t, f, bucket, object))
}
+
+// TestMultipartBufferWaitGivesUp checks that a part waiting for room in the
+// reorder buffer gives up when its request is cancelled, and tells the client
+// to slow down if it has waited too long, so a waiting request can't be held
+// open forever.
+func TestMultipartBufferWaitGivesUp(t *testing.T) {
+ b, _, bucket := newPutTestBackend(t, "", nil)
+ b.s.opt.MultipartStreamingBufferLimit = pool.BufferSize
+ oldTimeout := multipartWaitTimeout
+ multipartWaitTimeout = 100 * time.Millisecond
+ t.Cleanup(func() { multipartWaitTimeout = oldTimeout })
+ ctx := context.Background()
+ const object = "wait.bin"
+
+ uploadID, err := b.CreateMultipartUpload(ctx, bucket, object, nil)
+ require.NoError(t, err)
+
+ // Part 2 fills the buffer, so part 3 must wait.
+ _, err = b.UploadPart(ctx, bucket, object, uploadID, 2, 1, bytes.NewReader([]byte("2")))
+ require.NoError(t, err)
+
+ cancelCtx, cancel := context.WithCancel(ctx)
+ time.AfterFunc(10*time.Millisecond, cancel)
+ _, err = b.UploadPart(cancelCtx, bucket, object, uploadID, 3, 1, bytes.NewReader([]byte("3")))
+ require.ErrorIs(t, err, context.Canceled)
+
+ _, err = b.UploadPart(ctx, bucket, object, uploadID, 3, 1, bytes.NewReader([]byte("3")))
+ assert.True(t, gofakes3.HasErrorCode(err, errSlowDown), "want SlowDown, got %v", err)
+
+ require.NoError(t, b.AbortMultipartUpload(ctx, bucket, object, uploadID))
+}
diff --git a/cmd/serve/s3/serve_s3.md b/cmd/serve/s3/serve_s3.md
index 74d5e7aed..a2d5a4b5a 100644
--- a/cmd/serve/s3/serve_s3.md
+++ b/cmd/serve/s3/serve_s3.md
@@ -196,7 +196,8 @@ remote as if it had completed.
1 MiB memory pages it occupies: a part that would take the buffer over the
limit is stalled until the stream drains, so a client that uploads
faster than the remote can accept sees backpressure rather than
- unbounded server memory use. Since a stalled part holds its HTTP
+ unbounded server memory use. A part stalled for more than a minute
+ is failed with a `SlowDown` error, which S3 clients retry. Since a stalled part holds its HTTP
request open, clients whose upload concurrency times chunk size
exceeds the limit may need a longer read timeout when the remote is
slow. Non-contiguous part numbers are rejected on completion.