Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

serve s3: stream in-order multipart parts without buffering them

Nick Craig-Wood7773bb648787
diff --git a/cmd/serve/s3/multipart.go b/cmd/serve/s3/multipart.go
index 1a852f138..9f1ae1427 100644
--- a/cmd/serve/s3/multipart.go
+++ b/cmd/serve/s3/multipart.go
@@ -19,9 +19,11 @@ import (
"bytes"
"context"
"crypto/md5"
+ "encoding"
"encoding/hex"
"errors"
"fmt"
+ "hash"
"io"
"math"
"path"
@@ -80,6 +82,9 @@ type multipartUpload struct {
nextPart int // next part number to stream (1-based)
streamBuf map[int]*pool.RW // parts received ahead of nextPart, awaiting their turn
pumping bool // a goroutine is currently writing to the sink
+ partialSize int64 // bytes of part nextPart in the sink from an attempt which failed part way
+ poisoned bool // the stream holds data the upload can never complete with
+ partialMD5 []byte // MD5 state after those partialSize bytes (encoding.BinaryMarshaler)
buffered int64 // bytes of parts admitted but not yet streamed or released
bufferLimit int64 // max buffered before parts ahead of nextPart must wait (<= 0 for no limit)
}
@@ -205,6 +210,11 @@ func (b *s3Backend) CreateMultipartUpload(ctx context.Context, bucketName, objec
}
// UploadPart writes a single part from the S3 client into the streaming upload.
+//
+// The next part the stream needs is written straight into the sink as it is
+// received. A part which arrives ahead of its turn is buffered in memory until
+// the stream reaches it, and a re-upload of a part already streamed is just
+// hashed to check it matches.
func (b *s3Backend) UploadPart(ctx context.Context, bucketName, objectName string, uploadID gofakes3.UploadID, partNumber int, contentLength int64, body io.Reader) (string, error) {
up, err := b.loadUpload(uploadID)
if err != nil {
@@ -213,17 +223,32 @@ func (b *s3Backend) UploadPart(ctx context.Context, bucketName, objectName strin
up.startActivity()
defer up.endActivity()
- // A re-upload of a part already in the stream is only hashed, to check
- // it matches, so it needs no buffer.
- if md5Sum, size, ok := up.streamedPart(partNumber); ok {
- return verifyStreamedPart(partNumber, md5Sum, size, contentLength, body)
- }
+ // The sink must never get more than the declared length, and the body
+ // must be read to its end so that an error it only reports there - as
+ // the Content-MD5 check does - is not missed.
+ body = newLimitedBody(body, contentLength)
- // Wait until there is room to buffer this part, bounding the memory a
- // client which uploads faster than the backend drains can consume.
- if err := up.waitForTurn(ctx, partNumber, contentLength); err != nil {
+ turn, err := up.waitForTurn(ctx, partNumber, contentLength)
+ if err != nil {
return "", err
}
+ switch turn {
+ case turnVerify:
+ md5Sum, size, _ := up.streamedPart(partNumber)
+ return verifyStreamedPart(partNumber, md5Sum, size, contentLength, body)
+ case turnStream:
+ md5Sum, err := up.streamDirect(partNumber, contentLength, body)
+ if err != nil {
+ if up.isPoisoned() {
+ // The stream holds data the upload can never complete
+ // with, so tear it down rather than let the client retry
+ // a part which can't be accepted.
+ b.failUpload(uploadID, up)
+ }
+ return "", err
+ }
+ return fmt.Sprintf("%q", hex.EncodeToString(md5Sum)), nil
+ }
// Buffer the part in a pool-backed RW so we can MD5 it (for the ETag) and
// stream it once it is this part's turn. The RW grows a page at a time as
@@ -244,12 +269,54 @@ func (b *s3Backend) UploadPart(ctx context.Context, bucketName, objectName strin
md5Sum := hasher.Sum(nil)
etag := fmt.Sprintf("%q", hex.EncodeToString(md5Sum))
- if err := up.streamPart(partNumber, n, md5Sum, rw); err != nil {
+ if err := up.bufferPart(ctx, partNumber, n, md5Sum, rw); err != nil {
return "", err
}
return etag, nil
}
+// limitedBody reads the declared length of a part body and no more, then
+// checks the body really ends there. The final read of the underlying body
+// is what reports an error such as a Content-MD5 mismatch, so it must
+// happen even though its data is not wanted.
+type limitedBody struct {
+ body io.Reader
+ remaining int64
+}
+
+// newLimitedBody returns body limited to size bytes.
+func newLimitedBody(body io.Reader, size int64) *limitedBody {
+ return &limitedBody{body: body, remaining: max(size, 0)}
+}
+
+// maxEmptyReads is how many reads returning nothing are tolerated before a
+// body is declared stuck, as io.Copy does.
+const maxEmptyReads = 100
+
+func (r *limitedBody) Read(p []byte) (int, error) {
+ if r.remaining <= 0 {
+ // The declared length has been read, so all that is left is to
+ // find out how the body ends.
+ var b [1]byte
+ for range maxEmptyReads {
+ n, err := r.body.Read(b[:])
+ if n > 0 {
+ return 0, gofakes3.ErrIncompleteBody
+ }
+ if err != nil {
+ return 0, err
+ }
+ }
+ return 0, io.ErrNoProgress
+ }
+ if int64(len(p)) > r.remaining {
+ p = p[:r.remaining]
+ }
+ n, err := r.body.Read(p)
+ r.remaining -= int64(n)
+ return n, err
+}
+
// streamedPart returns the MD5 sum and size of partNumber if it has already
// been streamed to the sink.
func (up *multipartUpload) streamedPart(partNumber int) (md5Sum []byte, size int64, ok bool) {
@@ -294,15 +361,31 @@ func bufferCharge(size int64) int64 {
return (size + pool.BufferSize - 1) / pool.BufferSize * pool.BufferSize
}
-// waitForTurn blocks until a part of size bytes can be admitted to the
-// reorder buffer, then reserves its bufferCharge, bounding the memory an
-// upload can consume when the client sends parts faster than the backend
-// drains them.
+// partTurn says how an admitted part is to be received.
+type partTurn int
+
+const (
+ // turnBuffer means buffer the part until the stream reaches it; its
+ // bufferCharge has been reserved.
+ turnBuffer partTurn = iota
+ // turnStream means stream the part straight into the sink with
+ // streamDirect: the caller now owns the sink.
+ turnStream
+ // turnVerify means the part has already been streamed, so the
+ // re-upload can only be checked against it.
+ turnVerify
+)
+
+// waitForTurn blocks until a part of size bytes can be received, and says
+// how.
//
-// The next part the stream needs (and any retry of an earlier one) is always
-// admitted so the sink can keep draining; so is a single part bigger than the
-// limit when the buffer is empty, to guarantee progress. Reserved bytes are
-// returned with release, or by the pump as the part is streamed.
+// The next part the stream needs is streamed straight into the sink, once
+// no other request is writing to it, so it needs no buffer. A part ahead of
+// it must wait until its bufferCharge fits within the reorder buffer limit,
+// which it then reserves, bounding the memory an upload can consume when
+// the client sends parts faster than the backend drains them. A part too big
+// for the limit waits until it is the next part. Reserved bytes are returned
+// with release, or by the pump as the part is streamed.
//
// A part which waits for longer than multipartWaitTimeout gets errSlowDown,
// asking the client to retry it later, so a waiting request isn't held open
@@ -311,32 +394,40 @@ func bufferCharge(size int64) int64 {
// size is the client-declared part length and is not trusted: a negative value
// is rejected, and the admission test is written so a huge value can't overflow
// the running total and wrongly admit further parts past the limit.
-func (up *multipartUpload) waitForTurn(ctx context.Context, partNumber int, size int64) error {
+func (up *multipartUpload) waitForTurn(ctx context.Context, partNumber int, size int64) (partTurn, error) {
if size < 0 {
- return gofakes3.ErrInvalidArgument
+ return 0, gofakes3.ErrInvalidArgument
}
- size = bufferCharge(size)
+ charge := bufferCharge(size)
timer := time.NewTimer(multipartWaitTimeout)
defer timer.Stop()
for {
up.mu.Lock()
if up.closed {
up.mu.Unlock()
- return gofakes3.ErrNoSuchUpload
+ return 0, gofakes3.ErrNoSuchUpload
}
- if up.bufferLimit <= 0 || partNumber <= up.nextPart || up.buffered == 0 || size <= up.bufferLimit-up.buffered {
- up.buffered += size
+ switch {
+ case partNumber < up.nextPart:
up.mu.Unlock()
- return nil
+ return turnVerify, nil
+ case partNumber == up.nextPart && !up.pumping:
+ up.pumping = true
+ up.mu.Unlock()
+ return turnStream, nil
+ case partNumber > up.nextPart && (up.bufferLimit <= 0 || charge <= up.bufferLimit-up.buffered):
+ up.buffered += charge
+ up.mu.Unlock()
+ return turnBuffer, nil
}
changed := up.changed
up.mu.Unlock()
select {
case <-changed:
case <-ctx.Done():
- return ctx.Err()
+ return 0, ctx.Err()
case <-timer.C:
- return errSlowDown
+ return 0, errSlowDown
}
}
}
@@ -350,13 +441,146 @@ func (up *multipartUpload) release(size int64) {
up.mu.Unlock()
}
-// streamPart records a part and streams the parts into the sink in order.
+// streamDirect streams body, the size bytes of partNumber, straight into
+// the sink, returning its MD5 sum. partNumber must be the next part the
+// stream needs and the caller must own the sink (up.pumping), which is
+// passed on to pump any buffered parts which follow, or given up.
+//
+// If the body fails part way, what reached the sink can't be taken back, so
+// the bytes streamed and the MD5 state after them are recorded instead. The
+// next attempt at the part - typically the client's retry - must start with
+// the same bytes, which are only hashed to check them, and then carries on
+// from where the failed one stopped.
+func (up *multipartUpload) streamDirect(partNumber int, size int64, body io.Reader) (md5Sum []byte, err error) {
+ up.mu.Lock()
+ resumeAt, resumeState := up.partialSize, up.partialMD5
+ up.mu.Unlock()
+
+ hasher := md5.New()
+ if resumeAt > size {
+ return nil, up.stopStreaming(errPartAlreadyStreamed(partNumber))
+ }
+ if resumeAt > 0 {
+ if _, err := io.CopyN(hasher, body, resumeAt); err != nil {
+ if err == io.EOF {
+ err = gofakes3.ErrIncompleteBody
+ }
+ return nil, up.stopStreaming(err)
+ }
+ if state, _ := hasher.(encoding.BinaryMarshaler).MarshalBinary(); !bytes.Equal(state, resumeState) {
+ return nil, up.stopStreaming(errPartAlreadyStreamed(partNumber))
+ }
+ }
+
+ n, sinkErr, bodyErr := copyToSink(up.fh, hasher, body)
+ if n > 0 {
+ state, _ := hasher.(encoding.BinaryMarshaler).MarshalBinary()
+ up.mu.Lock()
+ up.partialSize, up.partialMD5 = resumeAt+n, state
+ up.mu.Unlock()
+ }
+ if sinkErr != nil {
+ return nil, up.stopStreaming(sinkErr)
+ }
+ if bodyErr != nil {
+ if resumeAt+n == size {
+ // The whole part reached the sink but the client disowned it
+ // (for example it failed its Content-MD5 check), so the stream
+ // holds data which can neither be completed nor replaced.
+ return nil, up.poison(bodyErr)
+ }
+ return nil, up.stopStreaming(bodyErr)
+ }
+ if resumeAt+n != size {
+ return nil, up.stopStreaming(gofakes3.ErrIncompleteBody)
+ }
+
+ md5Sum = hasher.Sum(nil)
+ up.mu.Lock()
+ if up.closed {
+ up.pumping = false
+ up.mu.Unlock()
+ return nil, gofakes3.ErrNoSuchUpload
+ }
+ up.partMD5s[partNumber] = md5Sum
+ up.partSizes[partNumber] = size
+ up.partialSize, up.partialMD5 = 0, nil
+ up.nextPart++
+ up.broadcast()
+ return md5Sum, up.pump()
+}
+
+// poison marks the upload as one which can never be completed, because the
+// stream holds data which can't be taken back, then gives up the sink as
+// stopStreaming does.
+func (up *multipartUpload) poison(err error) error {
+ up.mu.Lock()
+ up.poisoned = true
+ up.mu.Unlock()
+ return up.stopStreaming(err)
+}
+
+// isPoisoned reports whether the upload can never be completed.
+func (up *multipartUpload) isPoisoned() bool {
+ up.mu.Lock()
+ defer up.mu.Unlock()
+ return up.poisoned
+}
+
+// stopStreaming gives up the sink after a part failed with err, returning
+// the error to report: ErrNoSuchUpload if the failure was because the
+// upload was aborted.
+func (up *multipartUpload) stopStreaming(err error) error {
+ up.mu.Lock()
+ defer up.mu.Unlock()
+ up.pumping = false
+ up.broadcast()
+ if up.closed {
+ // The write failed because the upload was aborted while this part
+ // was being written: report the upload gone rather than the sink's
+ // ECLOSED as an internal error.
+ return gofakes3.ErrNoSuchUpload
+ }
+ return err
+}
+
+// copyToSink copies body into sink and hasher, returning the number of
+// bytes written to the sink, which are also the bytes hashed, and the error
+// which stopped it, from either the sink or the body.
+func copyToSink(sink io.Writer, hasher hash.Hash, body io.Reader) (n int64, sinkErr, bodyErr error) {
+ buf := make([]byte, 32*1024)
+ for {
+ nr, err := body.Read(buf)
+ if nr > 0 {
+ nw, err := sink.Write(buf[:nr])
+ _, _ = hasher.Write(buf[:nw])
+ n += int64(nw)
+ if err == nil && nw != nr {
+ err = io.ErrShortWrite
+ }
+ if err != nil {
+ return n, err, nil
+ }
+ }
+ if err == io.EOF {
+ return n, nil, nil
+ }
+ if err != nil {
+ return n, nil, err
+ }
+ }
+}
+
+// bufferPart records a part received ahead of its turn, holding it in rw
+// until the stream reaches it, then streams the parts into the sink in
+// order.
//
-// Parts must be uploaded in ascending, contiguous part-number order. A part
-// that arrives ahead of the next expected one is buffered until its turn; the
-// parts are then pumped into the sink in order. Whichever goroutine finds the
-// next part available does the pumping, so concurrent (but in-order) clients
-// are tolerated, with the buffering bounded by waitForTurn.
+// Parts must be uploaded in ascending, contiguous part-number order. Whichever
+// goroutine finds the next part available does the pumping, so concurrent
+// (but in-order) clients are tolerated, with the buffering bounded by
+// waitForTurn. If the stream has reached the part while it was being
+// received, it is streamed now, once any other request writing to the sink
+// has finished.
//
// A part number may be uploaded more than once - typically a client retrying
// after its request timed out, but real S3 also allows replacing a part. If
@@ -364,47 +588,80 @@ func (up *multipartUpload) release(size int64) {
// has already been streamed it can't be replaced: an identical re-upload is
// accepted idempotently (the data is already in the stream) and a different
// one is rejected.
-func (up *multipartUpload) streamPart(partNumber int, size int64, md5Sum []byte, rw *pool.RW) error {
+func (up *multipartUpload) bufferPart(ctx context.Context, partNumber int, size int64, md5Sum []byte, rw *pool.RW) error {
+ charge := bufferCharge(size)
+ // done releases the part's buffer. Call with up.mu held.
+ done := func() {
+ up.buffered -= charge
+ up.broadcast()
+ _ = rw.Close()
+ }
+ timer := time.NewTimer(multipartWaitTimeout)
+ defer timer.Stop()
up.mu.Lock()
- if up.closed {
+ for partNumber == up.nextPart && up.pumping && !up.closed {
+ // Another request is writing this part to the sink: wait to see
+ // whether it gets there.
+ changed := up.changed
+ up.mu.Unlock()
+ var err error
+ select {
+ case <-changed:
+ case <-ctx.Done():
+ err = ctx.Err()
+ case <-timer.C:
+ err = errSlowDown
+ }
+ up.mu.Lock()
+ if err != nil {
+ done()
+ up.mu.Unlock()
+ return err
+ }
+ }
+ switch {
+ case up.closed:
// The upload was aborted or completed while the part body was
// being received.
- up.buffered -= bufferCharge(size)
- up.broadcast()
+ done()
up.mu.Unlock()
- _ = rw.Close()
return gofakes3.ErrNoSuchUpload
- }
- if oldMD5, exists := up.partMD5s[partNumber]; exists {
- if old, buffered := up.streamBuf[partNumber]; buffered {
- _ = old.Close()
- up.buffered -= bufferCharge(up.partSizes[partNumber])
- up.broadcast()
- } else {
- // Already streamed (or streaming right now): the stream can't be
- // rewritten, so accept an identical part and reject the rest.
- same := bytes.Equal(md5Sum, oldMD5) && size == up.partSizes[partNumber]
- up.buffered -= bufferCharge(size)
- up.broadcast()
- up.mu.Unlock()
- _ = rw.Close()
- if !same {
- return errPartAlreadyStreamed(partNumber)
- }
- return nil
+ case partNumber < up.nextPart:
+ // Already streamed: the stream can't be rewritten, so accept an
+ // identical part and reject the rest.
+ same := bytes.Equal(md5Sum, up.partMD5s[partNumber]) && size == up.partSizes[partNumber]
+ done()
+ up.mu.Unlock()
+ if !same {
+ return errPartAlreadyStreamed(partNumber)
}
+ return nil
+ case partNumber == up.nextPart:
+ // The stream has reached this part while it was being received.
+ up.pumping = true
+ up.mu.Unlock()
+ _, err := up.streamDirect(partNumber, size, rw)
+ up.mu.Lock()
+ done()
+ up.mu.Unlock()
+ return err
+ }
+ if old, buffered := up.streamBuf[partNumber]; buffered {
+ _ = old.Close()
+ up.buffered -= bufferCharge(up.partSizes[partNumber])
+ up.broadcast()
}
up.partMD5s[partNumber] = md5Sum
up.partSizes[partNumber] = size
up.streamBuf[partNumber] = rw
+ up.mu.Unlock()
+ return nil
+}
- if up.pumping {
- // Another goroutine owns the sink and will pump this part in turn.
- up.mu.Unlock()
- return nil
- }
- up.pumping = true
-
+// pump streams buffered parts into the sink in order until it reaches one
+// which hasn't arrived, then gives up the sink. Call with up.mu held and
+// the sink owned (up.pumping); returns with up.mu released.
+func (up *multipartUpload) pump() error {
for {
if up.closed {
// Aborted while pumping: the sink is closed and any
@@ -417,6 +674,7 @@ func (up *multipartUpload) streamPart(partNumber int, size int64, md5Sum []byte,
prw, ok := up.streamBuf[up.nextPart]
if !ok {
up.pumping = false
+ up.broadcast()
up.mu.Unlock()
return nil
}
@@ -428,18 +686,9 @@ func (up *multipartUpload) streamPart(partNumber int, size int64, md5Sum []byte,
_ = prw.Close()
if err != nil {
up.mu.Lock()
- up.pumping = false
up.buffered -= psize
- closed := up.closed
- up.broadcast()
up.mu.Unlock()
- if closed {
- // The write failed because the upload was aborted while
- // this part was being pumped: report the upload gone
- // rather than the sink's ECLOSED as an internal error.
- return gofakes3.ErrNoSuchUpload
- }
- return err
+ return up.stopStreaming(err)
}
up.mu.Lock()
@@ -531,6 +780,18 @@ func (b *s3Backend) AbortMultipartUpload(ctx context.Context, bucketName, object
return nil
}
+// failUpload tears down an upload which can never be completed, so that
+// the client starts again instead of retrying a part which can't be
+// accepted.
+func (b *s3Backend) failUpload(uploadID gofakes3.UploadID, up *multipartUpload) {
+ fs.Errorf(up.fp, "failing multipart upload %s: %v", uploadID, errMultipartPoisoned)
+ b.multipartUploads.Delete(uploadID)
+ if err := up.abort(); err != nil {
+ fs.Errorf(up.fp, "aborting multipart upload: %v", err)
+ }
+ b.discardUpload(up)
+}
+
// startReaper starts a goroutine which aborts incomplete multipart
// uploads once they have been idle for expiry. Stopped by stopReaper.
func (b *s3Backend) startReaper(expiry time.Duration) {
@@ -652,7 +913,7 @@ func (up *multipartUpload) close() error {
}
return nil
}
- if len(up.streamBuf) != 0 || up.nextPart-1 != len(up.partSizes) {
+ if len(up.streamBuf) != 0 || up.nextPart-1 != len(up.partSizes) || up.pumping || up.partialSize != 0 {
up.mu.Unlock()
return gofakes3.ErrInvalidPart
}
@@ -663,6 +924,10 @@ func (up *multipartUpload) close() error {
return up.fh.Close()
}
+// errMultipartPoisoned is why an upload whose stream holds data it can
+// never complete with is failed.
+var errMultipartPoisoned = errors.New("a part was rejected after it had been streamed to the backend, so the upload must be started again")
+
// errMultipartAborted is the reason an aborted upload's write is abandoned
// with, so the streaming upload fails instead of committing what it has.
var errMultipartAborted = errors.New("serve s3: multipart upload aborted")
diff --git a/cmd/serve/s3/multipart_test.go b/cmd/serve/s3/multipart_test.go
index fb483ee44..2d61817c9 100644
--- a/cmd/serve/s3/multipart_test.go
+++ b/cmd/serve/s3/multipart_test.go
@@ -6,6 +6,7 @@ import (
"bytes"
"context"
"crypto/md5"
+ "encoding/base64"
"fmt"
"io"
"math"
@@ -411,7 +412,7 @@ func TestMultipartReplaceBuffered(t *testing.T) {
// missing part arrives and the buffer drains.
func TestMultipartBufferLimit(t *testing.T) {
core, f, bucket := newMultipartTestServerOpt(t, "", false, func(opt *Options) {
- opt.MultipartStreamingBufferLimit = 50 * 1024
+ opt.MultipartStreamingBufferLimit = pool.BufferSize
})
ctx := context.Background()
const object = "buffer-limit.bin"
@@ -427,8 +428,8 @@ func TestMultipartBufferLimit(t *testing.T) {
uploadID, err := core.NewMultipartUpload(ctx, bucket, object, minio.PutObjectOptions{})
require.NoError(t, err)
- // Part 3 arrives first: the buffer is empty so it is admitted even though
- // it must wait for its turn.
+ // Part 3 arrives first: it fits in the buffer so it is admitted even
+ // though it must wait for its turn.
p3, err := core.PutObjectPart(ctx, bucket, object, uploadID, 3, bytes.NewReader(datas[2]), int64(sizes[2]), minio.PutObjectPartOptions{})
require.NoError(t, err)
@@ -490,10 +491,10 @@ func (s *stubSink) CloseWithError(err error) error {
return nil
}
-// TestMultipartAbortDuringUploadPart aborts the upload between a part's
-// waitForTurn and its streamPart - as happens when the abort arrives while
-// the part body is still being received from the client - and checks that
-// streamPart fails cleanly instead of panicking on the torn-down upload.
+// TestMultipartAbortDuringUploadPart aborts the upload between a buffered
+// part's waitForTurn and its bufferPart - as happens when the abort arrives
+// while the part body is still being received from the client - and checks
+// that bufferPart fails cleanly instead of panicking on the torn-down upload.
func TestMultipartAbortDuringUploadPart(t *testing.T) {
up := newMultipartUpload("bucket", "key", "bucket/key", "bucket/key", nil, 0)
sink := &stubSink{}
@@ -502,19 +503,21 @@ func TestMultipartAbortDuringUploadPart(t *testing.T) {
// An UploadPart in progress: the part is admitted, then the abort lands
// while its body is still being received.
contents := []byte("hello")
- require.NoError(t, up.waitForTurn(context.Background(), 1, int64(len(contents))))
+ turn, err := up.waitForTurn(context.Background(), 2, int64(len(contents)))
+ require.NoError(t, err)
+ require.Equal(t, turnBuffer, turn)
require.NoError(t, up.abort())
// The abort must abandon the write rather than committing it.
assert.True(t, sink.closed)
assert.Equal(t, errMultipartAborted, sink.abortErr)
- // The UploadPart resumes: it buffers the part and calls streamPart.
+ // The UploadPart resumes: it buffers the part and calls bufferPart.
rw := multipart.NewRW()
- _, err := rw.Write(contents)
+ _, err = rw.Write(contents)
require.NoError(t, err)
md5Sum := md5.Sum(contents)
- err = up.streamPart(1, int64(len(contents)), md5Sum[:], rw)
+ err = up.bufferPart(context.Background(), 2, int64(len(contents)), md5Sum[:], rw)
require.ErrorIs(t, err, gofakes3.ErrNoSuchUpload)
// The reorder buffer reservation must have been returned
@@ -555,8 +558,10 @@ func TestMultipartCloseIncomplete(t *testing.T) {
_, err := rw.Write(contents)
require.NoError(t, err)
md5Sum := md5.Sum(contents)
- require.NoError(t, up.waitForTurn(context.Background(), 2, int64(len(contents))))
- require.NoError(t, up.streamPart(2, int64(len(contents)), md5Sum[:], rw))
+ turn, err := up.waitForTurn(context.Background(), 2, int64(len(contents)))
+ require.NoError(t, err)
+ require.Equal(t, turnBuffer, turn)
+ require.NoError(t, up.bufferPart(context.Background(), 2, int64(len(contents)), md5Sum[:], rw))
require.ErrorIs(t, up.close(), gofakes3.ErrInvalidPart)
assert.False(t, sink.closed)
@@ -1025,29 +1030,35 @@ func TestUploadPartNoReserveBeforeBody(t *testing.T) {
}
// TestWaitForTurnRejectsBogusSize checks that the reorder-buffer admission
-// rejects a negative client-declared part length and that a huge declared
-// length cannot overflow the running total so as to admit a further part past
-// the buffer limit.
+// rejects a negative client-declared part length, and that a huge declared
+// length is never admitted to the buffer - it must wait to be streamed - nor
+// can it overflow the running total.
func TestWaitForTurnRejectsBogusSize(t *testing.T) {
up := newMultipartUpload("bucket", "key", "bucket/key", "bucket/key", nil, 1<<20)
+ ctx := context.Background()
// A part length can never be negative.
- require.ErrorIs(t, up.waitForTurn(context.Background(), 1, -1), gofakes3.ErrInvalidArgument)
+ _, err := up.waitForTurn(ctx, 1, -1)
+ require.ErrorIs(t, err, gofakes3.ErrInvalidArgument)
- // A huge out-of-order part is admitted once because the buffer is empty,
- // driving buffered near the top of the int64 range.
- require.NoError(t, up.waitForTurn(context.Background(), 2, math.MaxInt64))
+ // The next part is streamed, whatever its size, without touching the
+ // buffer.
+ turn, err := up.waitForTurn(ctx, 1, math.MaxInt64)
+ require.NoError(t, err)
+ assert.Equal(t, turnStream, turn)
+ up.mu.Lock()
+ assert.Equal(t, int64(0), up.buffered)
+ up.mu.Unlock()
- // A further out-of-order part must wait, not be wrongly admitted by an
- // overflow of buffered+size.
+ // A huge out-of-order part must wait for its turn, not be admitted.
admitted := make(chan struct{})
go func() {
- _ = up.waitForTurn(context.Background(), 3, math.MaxInt64)
+ _, _ = up.waitForTurn(ctx, 2, math.MaxInt64)
close(admitted)
}()
select {
case <-admitted:
- t.Fatal("out-of-order part admitted past the buffer limit via overflow")
+ t.Fatal("huge out-of-order part admitted past the buffer limit")
case <-time.After(50 * time.Millisecond):
}
@@ -1197,3 +1208,277 @@ func TestMultipartBufferWaitGivesUp(t *testing.T) {
require.NoError(t, b.AbortMultipartUpload(ctx, bucket, object, uploadID))
}
+
+// bufferSink is a multipartUpload sink which collects what is written to it.
+type bufferSink struct {
+ mu sync.Mutex
+ buf bytes.Buffer
+}
+
+func (s *bufferSink) Write(p []byte) (int, error) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ return s.buf.Write(p)
+}
+
+func (s *bufferSink) Close() error { return nil }
+
+func (s *bufferSink) Bytes() []byte {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ return bytes.Clone(s.buf.Bytes())
+}
+
+// newBufferSinkUpload starts a multipart upload on b whose parts are
+// collected in the returned sink instead of being written to the VFS, so the
+// memory used by the VFS upload doesn't hide the memory used by serve s3.
+func newBufferSinkUpload(t *testing.T, b *s3Backend, bucket, object string) (gofakes3.UploadID, *bufferSink) {
+ uploadID, err := b.CreateMultipartUpload(context.Background(), bucket, object, nil)
+ require.NoError(t, err)
+ up, err := b.loadUpload(uploadID)
+ require.NoError(t, err)
+ if up.fh != nil {
+ // This fails the VFS upload, returning errBoom
+ _ = up.fh.(interface{ CloseWithError(error) error }).CloseWithError(errBoom)
+ }
+ sink := &bufferSink{}
+ up.fh = sink
+ t.Cleanup(func() { _ = b.AbortMultipartUpload(context.Background(), bucket, object, uploadID) })
+ return uploadID, sink
+}
+
+// TestMultipartInOrderPartNotBuffered checks that the part the stream needs
+// is written straight to the sink rather than buffered in memory, however
+// big it is compared with the buffer limit.
+func TestMultipartInOrderPartNotBuffered(t *testing.T) {
+ b, _, bucket := newPutTestBackend(t, "", nil)
+ b.s.opt.MultipartStreamingBufferLimit = pool.BufferSize
+ ctx := context.Background()
+ const object = "in-order.bin"
+ const partSize = 8 * pool.BufferSize
+
+ uploadID, sink := newBufferSinkUpload(t, b, bucket, object)
+ data := []byte(random.String(partSize))
+ body := newPoolWatchReader(data)
+ _, err := b.UploadPart(ctx, bucket, object, uploadID, 1, partSize, body)
+ require.NoError(t, err)
+ assert.Equal(t, 0, body.maxPages, "the in-order part was buffered in memory")
+ assert.Equal(t, data, sink.Bytes())
+}
+
+// failAfterReader returns data and then fails with err.
+func failAfterReader(data []byte, err error) io.Reader {
+ return io.MultiReader(bytes.NewReader(data), iotestErrReader{err})
+}
+
+type iotestErrReader struct{ err error }
+
+func (r iotestErrReader) Read([]byte) (int, error) { return 0, r.err }
+
+// TestMultipartResumeFailedPart checks that when the body of a part being
+// streamed fails part way, a retry of the part with the same contents
+// carries on from where it stopped, while one with different contents is
+// rejected without disturbing the stream.
+func TestMultipartResumeFailedPart(t *testing.T) {
+ b, f, bucket := newPutTestBackend(t, "", nil)
+ ctx := context.Background()
+ const object = "resume.bin"
+
+ part1 := []byte(random.String(300 * 1024))
+ part2 := []byte(random.String(100 * 1024))
+ want := append(append([]byte(nil), part1...), part2...)
+
+ uploadID, err := b.CreateMultipartUpload(ctx, bucket, object, nil)
+ require.NoError(t, err)
+
+ // The first attempt at part 1 fails after 123457 bytes reach the sink.
+ _, err = b.UploadPart(ctx, bucket, object, uploadID, 1, int64(len(part1)), failAfterReader(part1[:123457], errBoom))
+ require.ErrorIs(t, err, errBoom)
+
+ // A retry which fails before the resume point changes nothing.
+ _, err = b.UploadPart(ctx, bucket, object, uploadID, 1, int64(len(part1)), failAfterReader(part1[:1000], errBoom))
+ require.ErrorIs(t, err, errBoom)
+
+ // A retry with different contents is rejected.
+ other := append([]byte(nil), part1...)
+ other[100] ^= 1
+ _, err = b.UploadPart(ctx, bucket, object, uploadID, 1, int64(len(other)), bytes.NewReader(other))
+ assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrNotImplemented), "want NotImplemented, got %v", err)
+
+ // As is a retry shorter than what is already in the stream.
+ _, err = b.UploadPart(ctx, bucket, object, uploadID, 1, 1000, bytes.NewReader(part1[:1000]))
+ assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrNotImplemented), "want NotImplemented, got %v", err)
+
+ // A second failure part way through extends what is in the stream.
+ _, err = b.UploadPart(ctx, bucket, object, uploadID, 1, int64(len(part1)), failAfterReader(part1[:200000], errBoom))
+ require.ErrorIs(t, err, errBoom)
+
+ // The whole retry completes the part.
+ p1, err := b.UploadPart(ctx, bucket, object, uploadID, 1, int64(len(part1)), bytes.NewReader(part1))
+ require.NoError(t, err)
+ p2, err := b.UploadPart(ctx, bucket, object, uploadID, 2, int64(len(part2)), bytes.NewReader(part2))
+ require.NoError(t, err)
+
+ _, _, err = b.CompleteMultipartUpload(ctx, bucket, object, uploadID, &gofakes3.CompleteMultipartUploadRequest{
+ Parts: []gofakes3.CompletedPart{{PartNumber: 1, ETag: p1}, {PartNumber: 2, ETag: p2}},
+ })
+ require.NoError(t, err)
+ assert.Equal(t, want, readObject(t, f, bucket, object))
+}
+
+// TestMultipartCompleteWithPartialPart checks that an upload whose stream
+// holds part of a failed part can't be completed without it.
+func TestMultipartCompleteWithPartialPart(t *testing.T) {
+ b, f, bucket := newPutTestBackend(t, "", nil)
+ ctx := context.Background()
+ const object = "partial.bin"
+
+ uploadID, err := b.CreateMultipartUpload(ctx, bucket, object, nil)
+ require.NoError(t, err)
+ part1 := []byte(random.String(200 * 1024))
+ _, err = b.UploadPart(ctx, bucket, object, uploadID, 1, int64(len(part1)), failAfterReader(part1[:150000], errBoom))
+ require.ErrorIs(t, err, errBoom)
+
+ _, _, err = b.CompleteMultipartUpload(ctx, bucket, object, uploadID, &gofakes3.CompleteMultipartUploadRequest{})
+ require.ErrorIs(t, err, gofakes3.ErrInvalidPart)
+ _, err = f.NewObject(ctx, path.Join(bucket, object))
+ require.ErrorIs(t, err, fs.ErrorObjectNotFound)
+}
+
+// TestMultipartConcurrentDuplicateParts checks that further requests for the
+// part being streamed wait, without buffering it, for the first to finish,
+// then are checked against it.
+func TestMultipartConcurrentDuplicateParts(t *testing.T) {
+ b, _, bucket := newPutTestBackend(t, "", nil)
+ ctx := context.Background()
+ const object = "duplicates.bin"
+ const partSize = 8 * pool.BufferSize
+
+ uploadID, sink := newBufferSinkUpload(t, b, bucket, object)
+ data := []byte(random.String(partSize))
+
+ // The first request stalls half way through its body.
+ pr, pw := io.Pipe()
+ first := make(chan error, 1)
+ var firstETag string
+ go func() {
+ var err error
+ firstETag, err = b.UploadPart(ctx, bucket, object, uploadID, 1, partSize, pr)
+ first <- err
+ }()
+ _, err := pw.Write(data[:partSize/2])
+ require.NoError(t, err)
+
+ type result struct {
+ etag string
+ err error
+ }
+ const duplicates = 4
+ bodies := make([]*poolWatchReader, duplicates)
+ results := make(chan result, duplicates)
+ for i := range bodies {
+ bodies[i] = newPoolWatchReader(data)
+ go func(body io.Reader) {
+ etag, err := b.UploadPart(ctx, bucket, object, uploadID, 1, partSize, body)
+ results <- result{etag, err}
+ }(bodies[i])
+ }
+ select {
+ case r := <-results:
+ t.Fatalf("a duplicate finished while the first request was streaming (err=%v)", r.err)
+ case <-time.After(200 * time.Millisecond):
+ }
+
+ _, err = pw.Write(data[partSize/2:])
+ require.NoError(t, err)
+ // The body must end, as a request body does, so that the part is
+ // read to its end.
+ require.NoError(t, pw.Close())
+ require.NoError(t, <-first)
+ for range duplicates {
+ r := <-results
+ require.NoError(t, r.err)
+ assert.Equal(t, firstETag, r.etag)
+ }
+ for _, body := range bodies {
+ assert.Equal(t, 0, body.maxPages, "a duplicate part was buffered in memory")
+ }
+ assert.Equal(t, data, sink.Bytes())
+}
+
+// TestMultipartPartFailsAfterWholeBody checks that a part whose body is
+// delivered in full but is then rejected - as gofakes3's Content-MD5 check
+// does, after every byte has reached the backend - fails the whole upload,
+// rather than leaving it in a state no retry of the part can escape.
+func TestMultipartPartFailsAfterWholeBody(t *testing.T) {
+ b, _, bucket := newPutTestBackend(t, "", nil)
+ ctx := context.Background()
+ const object = "bad-digest.bin"
+
+ uploadID, err := b.CreateMultipartUpload(ctx, bucket, object, nil)
+ require.NoError(t, err)
+ part1 := []byte(random.String(100 * 1024))
+
+ // The whole body arrives, then the integrity check rejects it.
+ _, err = b.UploadPart(ctx, bucket, object, uploadID, 1, int64(len(part1)),
+ failAfterReader(part1, gofakes3.ErrBadDigest))
+ require.Error(t, err)
+
+ // The upload is gone, so the client starts again rather than retrying
+ // a part which could never be accepted.
+ _, err = b.UploadPart(ctx, bucket, object, uploadID, 1, int64(len(part1)), bytes.NewReader(part1))
+ require.ErrorIs(t, err, gofakes3.ErrNoSuchUpload)
+ _, err = b.loadUpload(uploadID)
+ require.ErrorIs(t, err, gofakes3.ErrNoSuchUpload)
+}
+
+// TestMultipartPartBadMD5 checks that a part whose body doesn't match the
+// Content-MD5 the client declared is rejected, and that the upload it
+// corrupted is failed rather than left for the client to retry.
+func TestMultipartPartBadMD5(t *testing.T) {
+ core, f, bucket := newMultipartTestServer(t, false)
+ ctx := context.Background()
+ const object = "bad-md5.bin"
+
+ uploadID, err := core.NewMultipartUpload(ctx, bucket, object, minio.PutObjectOptions{})
+ require.NoError(t, err)
+
+ part1 := []byte(random.String(60 * 1024))
+ wrongMD5 := md5.Sum([]byte("not the part"))
+ _, err = core.PutObjectPart(ctx, bucket, object, uploadID, 1, bytes.NewReader(part1), int64(len(part1)),
+ minio.PutObjectPartOptions{Md5Base64: base64.StdEncoding.EncodeToString(wrongMD5[:])})
+ require.Error(t, err)
+
+ // Nothing was committed at the key and the upload is gone.
+ _, err = f.NewObject(ctx, path.Join(bucket, object))
+ require.ErrorIs(t, err, fs.ErrorObjectNotFound)
+ _, err = core.PutObjectPart(ctx, bucket, object, uploadID, 1, bytes.NewReader(part1), int64(len(part1)), minio.PutObjectPartOptions{})
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "NoSuchUpload")
+ requireOnly(t, f, bucket)
+}
+
+// TestLimitedBody checks that a part body is read to its end, so that an
+// error reported there is not missed, and that a body longer than declared
+// is rejected.
+func TestLimitedBody(t *testing.T) {
+ for _, test := range []struct {
+ name string
+ body io.Reader
+ want error
+ }{
+ {"EndError", failAfterReader([]byte("hello"), gofakes3.ErrBadDigest), gofakes3.ErrBadDigest},
+ {"TooLong", bytes.NewReader([]byte("hello world")), gofakes3.ErrIncompleteBody},
+ {"Exact", bytes.NewReader([]byte("hello")), nil},
+ } {
+ t.Run(test.name, func(t *testing.T) {
+ n, err := io.Copy(io.Discard, newLimitedBody(test.body, 5))
+ assert.Equal(t, int64(5), n)
+ if test.want == nil {
+ assert.NoError(t, err)
+ } else {
+ assert.ErrorIs(t, err, test.want)
+ }
+ })
+ }
+}
diff --git a/cmd/serve/s3/serve_s3.md b/cmd/serve/s3/serve_s3.md
index a2d5a4b5a..7f31ebfc1 100644
--- a/cmd/serve/s3/serve_s3.md
+++ b/cmd/serve/s3/serve_s3.md
@@ -189,12 +189,15 @@ remote as if it had completed.
**Limitations**
- Parts must arrive in ascending, contiguous part-number order
- (1, 2, 3, ...). Parts the client uploads concurrently or out of order
- are buffered until their turn. The memory used for this buffering is
+ (1, 2, 3, ...). The part the stream needs next is written straight
+ through as it arrives, so it is never buffered, whatever its size.
+ Parts the client uploads concurrently or out of order are buffered
+ until their turn. The memory used for this buffering is
capped, per upload, by `--multipart-streaming-buffer-limit` (default
`256M`, `0` for no limit), counting each buffered part in the whole
1 MiB memory pages it occupies: a part that would take the buffer over the
- limit is stalled until the stream drains, so a client that uploads
+ limit is stalled until the stream drains (or, if it is bigger than the
+ limit, until it is the next part), so a client that uploads
faster than the remote can accept sees backpressure rather than
unbounded server memory use. A part stalled for more than a minute
is failed with a `SlowDown` error, which S3 clients retry. Since a stalled part holds its HTTP
@@ -206,8 +209,11 @@ remote as if it had completed.
- A part uploaded again before completion - typically a client retrying
after a timeout - is accepted: if the earlier copy is still buffered
it is replaced, and if it has already been streamed an identical
- re-upload is a no-op. What isn't possible is replacing a part that has
- already been streamed with *different* content - that is rejected. A
+ re-upload is a no-op. If a part fails part way through being streamed
+ (for example the client's connection drops) the client's retry of it
+ carries on where it stopped, provided it has the same content. What
+ isn't possible is replacing a part that has already been streamed,
+ in whole or in part, with *different* content - that is rejected. A
failure in the stream to the remote itself still aborts the whole
upload and the client must start it again. (The remote's own upload
still retries its internal chunks.)