"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
serve s3: stream in-order multipart parts without buffering them
Nick Craig-Wood7773bb648787
diff --git a/cmd/serve/s3/multipart.go b/cmd/serve/s3/multipart.goindex 1a852f138..9f1ae1427 100644--- a/cmd/serve/s3/multipart.go+++ b/cmd/serve/s3/multipart.go@@ -19,9 +19,11 @@ import ("bytes""context""crypto/md5"+ "encoding""encoding/hex""errors""fmt"+ "hash""io""math""path"@@ -80,6 +82,9 @@ type multipartUpload struct {nextPart int // next part number to stream (1-based)streamBuf map[int]*pool.RW // parts received ahead of nextPart, awaiting their turnpumping bool // a goroutine is currently writing to the sink+ partialSize int64 // bytes of part nextPart in the sink from an attempt which failed part way+ poisoned bool // the stream holds data the upload can never complete with+ partialMD5 []byte // MD5 state after those partialSize bytes (encoding.BinaryMarshaler)buffered int64 // bytes of parts admitted but not yet streamed or releasedbufferLimit int64 // max buffered before parts ahead of nextPart must wait (<= 0 for no limit)}@@ -205,6 +210,11 @@ func (b *s3Backend) CreateMultipartUpload(ctx context.Context, bucketName, objec}// UploadPart writes a single part from the S3 client into the streaming upload.+//+// The next part the stream needs is written straight into the sink as it is+// received. A part which arrives ahead of its turn is buffered in memory until+// the stream reaches it, and a re-upload of a part already streamed is just+// hashed to check it matches.func (b *s3Backend) UploadPart(ctx context.Context, bucketName, objectName string, uploadID gofakes3.UploadID, partNumber int, contentLength int64, body io.Reader) (string, error) {up, err := b.loadUpload(uploadID)if err != nil {@@ -213,17 +223,32 @@ func (b *s3Backend) UploadPart(ctx context.Context, bucketName, objectName strinup.startActivity()defer up.endActivity()- // A re-upload of a part already in the stream is only hashed, to check- // it matches, so it needs no buffer.- if md5Sum, size, ok := up.streamedPart(partNumber); ok {- return verifyStreamedPart(partNumber, md5Sum, size, contentLength, body)- }+ // The sink must never get more than the declared length, and the body+ // must be read to its end so that an error it only reports there - as+ // the Content-MD5 check does - is not missed.+ body = newLimitedBody(body, contentLength)- // Wait until there is room to buffer this part, bounding the memory a- // client which uploads faster than the backend drains can consume.- if err := up.waitForTurn(ctx, partNumber, contentLength); err != nil {+ turn, err := up.waitForTurn(ctx, partNumber, contentLength)+ if err != nil {return "", err}+ switch turn {+ case turnVerify:+ md5Sum, size, _ := up.streamedPart(partNumber)+ return verifyStreamedPart(partNumber, md5Sum, size, contentLength, body)+ case turnStream:+ md5Sum, err := up.streamDirect(partNumber, contentLength, body)+ if err != nil {+ if up.isPoisoned() {+ // The stream holds data the upload can never complete+ // with, so tear it down rather than let the client retry+ // a part which can't be accepted.+ b.failUpload(uploadID, up)+ }+ return "", err+ }+ return fmt.Sprintf("%q", hex.EncodeToString(md5Sum)), nil+ }// Buffer the part in a pool-backed RW so we can MD5 it (for the ETag) and// stream it once it is this part's turn. The RW grows a page at a time as@@ -244,12 +269,54 @@ func (b *s3Backend) UploadPart(ctx context.Context, bucketName, objectName strinmd5Sum := hasher.Sum(nil)etag := fmt.Sprintf("%q", hex.EncodeToString(md5Sum))- if err := up.streamPart(partNumber, n, md5Sum, rw); err != nil {+ if err := up.bufferPart(ctx, partNumber, n, md5Sum, rw); err != nil {return "", err}return etag, nil}+// limitedBody reads the declared length of a part body and no more, then+// checks the body really ends there. The final read of the underlying body+// is what reports an error such as a Content-MD5 mismatch, so it must+// happen even though its data is not wanted.+type limitedBody struct {+ body io.Reader+ remaining int64+}++// newLimitedBody returns body limited to size bytes.+func newLimitedBody(body io.Reader, size int64) *limitedBody {+ return &limitedBody{body: body, remaining: max(size, 0)}+}++// maxEmptyReads is how many reads returning nothing are tolerated before a+// body is declared stuck, as io.Copy does.+const maxEmptyReads = 100++func (r *limitedBody) Read(p []byte) (int, error) {+ if r.remaining <= 0 {+ // The declared length has been read, so all that is left is to+ // find out how the body ends.+ var b [1]byte+ for range maxEmptyReads {+ n, err := r.body.Read(b[:])+ if n > 0 {+ return 0, gofakes3.ErrIncompleteBody+ }+ if err != nil {+ return 0, err+ }+ }+ return 0, io.ErrNoProgress+ }+ if int64(len(p)) > r.remaining {+ p = p[:r.remaining]+ }+ n, err := r.body.Read(p)+ r.remaining -= int64(n)+ return n, err+}+// streamedPart returns the MD5 sum and size of partNumber if it has already// been streamed to the sink.func (up *multipartUpload) streamedPart(partNumber int) (md5Sum []byte, size int64, ok bool) {@@ -294,15 +361,31 @@ func bufferCharge(size int64) int64 {return (size + pool.BufferSize - 1) / pool.BufferSize * pool.BufferSize}-// waitForTurn blocks until a part of size bytes can be admitted to the-// reorder buffer, then reserves its bufferCharge, bounding the memory an-// upload can consume when the client sends parts faster than the backend-// drains them.+// partTurn says how an admitted part is to be received.+type partTurn int++const (+ // turnBuffer means buffer the part until the stream reaches it; its+ // bufferCharge has been reserved.+ turnBuffer partTurn = iota+ // turnStream means stream the part straight into the sink with+ // streamDirect: the caller now owns the sink.+ turnStream+ // turnVerify means the part has already been streamed, so the+ // re-upload can only be checked against it.+ turnVerify+)++// waitForTurn blocks until a part of size bytes can be received, and says+// how.//-// The next part the stream needs (and any retry of an earlier one) is always-// admitted so the sink can keep draining; so is a single part bigger than the-// limit when the buffer is empty, to guarantee progress. Reserved bytes are-// returned with release, or by the pump as the part is streamed.+// The next part the stream needs is streamed straight into the sink, once+// no other request is writing to it, so it needs no buffer. A part ahead of+// it must wait until its bufferCharge fits within the reorder buffer limit,+// which it then reserves, bounding the memory an upload can consume when+// the client sends parts faster than the backend drains them. A part too big+// for the limit waits until it is the next part. Reserved bytes are returned+// with release, or by the pump as the part is streamed.//// A part which waits for longer than multipartWaitTimeout gets errSlowDown,// asking the client to retry it later, so a waiting request isn't held open@@ -311,32 +394,40 @@ func bufferCharge(size int64) int64 {// size is the client-declared part length and is not trusted: a negative value// is rejected, and the admission test is written so a huge value can't overflow// the running total and wrongly admit further parts past the limit.-func (up *multipartUpload) waitForTurn(ctx context.Context, partNumber int, size int64) error {+func (up *multipartUpload) waitForTurn(ctx context.Context, partNumber int, size int64) (partTurn, error) {if size < 0 {- return gofakes3.ErrInvalidArgument+ return 0, gofakes3.ErrInvalidArgument}- size = bufferCharge(size)+ charge := bufferCharge(size)timer := time.NewTimer(multipartWaitTimeout)defer timer.Stop()for {up.mu.Lock()if up.closed {up.mu.Unlock()- return gofakes3.ErrNoSuchUpload+ return 0, gofakes3.ErrNoSuchUpload}- if up.bufferLimit <= 0 || partNumber <= up.nextPart || up.buffered == 0 || size <= up.bufferLimit-up.buffered {- up.buffered += size+ switch {+ case partNumber < up.nextPart:up.mu.Unlock()- return nil+ return turnVerify, nil+ case partNumber == up.nextPart && !up.pumping:+ up.pumping = true+ up.mu.Unlock()+ return turnStream, nil+ case partNumber > up.nextPart && (up.bufferLimit <= 0 || charge <= up.bufferLimit-up.buffered):+ up.buffered += charge+ up.mu.Unlock()+ return turnBuffer, nil}changed := up.changedup.mu.Unlock()select {case <-changed:case <-ctx.Done():- return ctx.Err()+ return 0, ctx.Err()case <-timer.C:- return errSlowDown+ return 0, errSlowDown}}}@@ -350,13 +441,146 @@ func (up *multipartUpload) release(size int64) {up.mu.Unlock()}-// streamPart records a part and streams the parts into the sink in order.+// streamDirect streams body, the size bytes of partNumber, straight into+// the sink, returning its MD5 sum. partNumber must be the next part the+// stream needs and the caller must own the sink (up.pumping), which is+// passed on to pump any buffered parts which follow, or given up.+//+// If the body fails part way, what reached the sink can't be taken back, so+// the bytes streamed and the MD5 state after them are recorded instead. The+// next attempt at the part - typically the client's retry - must start with+// the same bytes, which are only hashed to check them, and then carries on+// from where the failed one stopped.+func (up *multipartUpload) streamDirect(partNumber int, size int64, body io.Reader) (md5Sum []byte, err error) {+ up.mu.Lock()+ resumeAt, resumeState := up.partialSize, up.partialMD5+ up.mu.Unlock()++ hasher := md5.New()+ if resumeAt > size {+ return nil, up.stopStreaming(errPartAlreadyStreamed(partNumber))+ }+ if resumeAt > 0 {+ if _, err := io.CopyN(hasher, body, resumeAt); err != nil {+ if err == io.EOF {+ err = gofakes3.ErrIncompleteBody+ }+ return nil, up.stopStreaming(err)+ }+ if state, _ := hasher.(encoding.BinaryMarshaler).MarshalBinary(); !bytes.Equal(state, resumeState) {+ return nil, up.stopStreaming(errPartAlreadyStreamed(partNumber))+ }+ }++ n, sinkErr, bodyErr := copyToSink(up.fh, hasher, body)+ if n > 0 {+ state, _ := hasher.(encoding.BinaryMarshaler).MarshalBinary()+ up.mu.Lock()+ up.partialSize, up.partialMD5 = resumeAt+n, state+ up.mu.Unlock()+ }+ if sinkErr != nil {+ return nil, up.stopStreaming(sinkErr)+ }+ if bodyErr != nil {+ if resumeAt+n == size {+ // The whole part reached the sink but the client disowned it+ // (for example it failed its Content-MD5 check), so the stream+ // holds data which can neither be completed nor replaced.+ return nil, up.poison(bodyErr)+ }+ return nil, up.stopStreaming(bodyErr)+ }+ if resumeAt+n != size {+ return nil, up.stopStreaming(gofakes3.ErrIncompleteBody)+ }++ md5Sum = hasher.Sum(nil)+ up.mu.Lock()+ if up.closed {+ up.pumping = false+ up.mu.Unlock()+ return nil, gofakes3.ErrNoSuchUpload+ }+ up.partMD5s[partNumber] = md5Sum+ up.partSizes[partNumber] = size+ up.partialSize, up.partialMD5 = 0, nil+ up.nextPart+++ up.broadcast()+ return md5Sum, up.pump()+}++// poison marks the upload as one which can never be completed, because the+// stream holds data which can't be taken back, then gives up the sink as+// stopStreaming does.+func (up *multipartUpload) poison(err error) error {+ up.mu.Lock()+ up.poisoned = true+ up.mu.Unlock()+ return up.stopStreaming(err)+}++// isPoisoned reports whether the upload can never be completed.+func (up *multipartUpload) isPoisoned() bool {+ up.mu.Lock()+ defer up.mu.Unlock()+ return up.poisoned+}++// stopStreaming gives up the sink after a part failed with err, returning+// the error to report: ErrNoSuchUpload if the failure was because the+// upload was aborted.+func (up *multipartUpload) stopStreaming(err error) error {+ up.mu.Lock()+ defer up.mu.Unlock()+ up.pumping = false+ up.broadcast()+ if up.closed {+ // The write failed because the upload was aborted while this part+ // was being written: report the upload gone rather than the sink's+ // ECLOSED as an internal error.+ return gofakes3.ErrNoSuchUpload+ }+ return err+}++// copyToSink copies body into sink and hasher, returning the number of+// bytes written to the sink, which are also the bytes hashed, and the error+// which stopped it, from either the sink or the body.+func copyToSink(sink io.Writer, hasher hash.Hash, body io.Reader) (n int64, sinkErr, bodyErr error) {+ buf := make([]byte, 32*1024)+ for {+ nr, err := body.Read(buf)+ if nr > 0 {+ nw, err := sink.Write(buf[:nr])+ _, _ = hasher.Write(buf[:nw])+ n += int64(nw)+ if err == nil && nw != nr {+ err = io.ErrShortWrite+ }+ if err != nil {+ return n, err, nil+ }+ }+ if err == io.EOF {+ return n, nil, nil+ }+ if err != nil {+ return n, nil, err+ }+ }+}++// bufferPart records a part received ahead of its turn, holding it in rw+// until the stream reaches it, then streams the parts into the sink in+// order.//-// Parts must be uploaded in ascending, contiguous part-number order. A part-// that arrives ahead of the next expected one is buffered until its turn; the-// parts are then pumped into the sink in order. Whichever goroutine finds the-// next part available does the pumping, so concurrent (but in-order) clients-// are tolerated, with the buffering bounded by waitForTurn.+// Parts must be uploaded in ascending, contiguous part-number order. Whichever+// goroutine finds the next part available does the pumping, so concurrent+// (but in-order) clients are tolerated, with the buffering bounded by+// waitForTurn. If the stream has reached the part while it was being+// received, it is streamed now, once any other request writing to the sink+// has finished.//// A part number may be uploaded more than once - typically a client retrying// after its request timed out, but real S3 also allows replacing a part. If@@ -364,47 +588,80 @@ func (up *multipartUpload) release(size int64) {// has already been streamed it can't be replaced: an identical re-upload is// accepted idempotently (the data is already in the stream) and a different// one is rejected.-func (up *multipartUpload) streamPart(partNumber int, size int64, md5Sum []byte, rw *pool.RW) error {+func (up *multipartUpload) bufferPart(ctx context.Context, partNumber int, size int64, md5Sum []byte, rw *pool.RW) error {+ charge := bufferCharge(size)+ // done releases the part's buffer. Call with up.mu held.+ done := func() {+ up.buffered -= charge+ up.broadcast()+ _ = rw.Close()+ }+ timer := time.NewTimer(multipartWaitTimeout)+ defer timer.Stop()up.mu.Lock()- if up.closed {+ for partNumber == up.nextPart && up.pumping && !up.closed {+ // Another request is writing this part to the sink: wait to see+ // whether it gets there.+ changed := up.changed+ up.mu.Unlock()+ var err error+ select {+ case <-changed:+ case <-ctx.Done():+ err = ctx.Err()+ case <-timer.C:+ err = errSlowDown+ }+ up.mu.Lock()+ if err != nil {+ done()+ up.mu.Unlock()+ return err+ }+ }+ switch {+ case up.closed:// The upload was aborted or completed while the part body was// being received.- up.buffered -= bufferCharge(size)- up.broadcast()+ done()up.mu.Unlock()- _ = rw.Close()return gofakes3.ErrNoSuchUpload- }- if oldMD5, exists := up.partMD5s[partNumber]; exists {- if old, buffered := up.streamBuf[partNumber]; buffered {- _ = old.Close()- up.buffered -= bufferCharge(up.partSizes[partNumber])- up.broadcast()- } else {- // Already streamed (or streaming right now): the stream can't be- // rewritten, so accept an identical part and reject the rest.- same := bytes.Equal(md5Sum, oldMD5) && size == up.partSizes[partNumber]- up.buffered -= bufferCharge(size)- up.broadcast()- up.mu.Unlock()- _ = rw.Close()- if !same {- return errPartAlreadyStreamed(partNumber)- }- return nil+ case partNumber < up.nextPart:+ // Already streamed: the stream can't be rewritten, so accept an+ // identical part and reject the rest.+ same := bytes.Equal(md5Sum, up.partMD5s[partNumber]) && size == up.partSizes[partNumber]+ done()+ up.mu.Unlock()+ if !same {+ return errPartAlreadyStreamed(partNumber)}+ return nil+ case partNumber == up.nextPart:+ // The stream has reached this part while it was being received.+ up.pumping = true+ up.mu.Unlock()+ _, err := up.streamDirect(partNumber, size, rw)+ up.mu.Lock()+ done()+ up.mu.Unlock()+ return err+ }+ if old, buffered := up.streamBuf[partNumber]; buffered {+ _ = old.Close()+ up.buffered -= bufferCharge(up.partSizes[partNumber])+ up.broadcast()}up.partMD5s[partNumber] = md5Sumup.partSizes[partNumber] = sizeup.streamBuf[partNumber] = rw+ up.mu.Unlock()+ return nil+}- if up.pumping {- // Another goroutine owns the sink and will pump this part in turn.- up.mu.Unlock()- return nil- }- up.pumping = true-+// pump streams buffered parts into the sink in order until it reaches one+// which hasn't arrived, then gives up the sink. Call with up.mu held and+// the sink owned (up.pumping); returns with up.mu released.+func (up *multipartUpload) pump() error {for {if up.closed {// Aborted while pumping: the sink is closed and any@@ -417,6 +674,7 @@ func (up *multipartUpload) streamPart(partNumber int, size int64, md5Sum []byte,prw, ok := up.streamBuf[up.nextPart]if !ok {up.pumping = false+ up.broadcast()up.mu.Unlock()return nil}@@ -428,18 +686,9 @@ func (up *multipartUpload) streamPart(partNumber int, size int64, md5Sum []byte,_ = prw.Close()if err != nil {up.mu.Lock()- up.pumping = falseup.buffered -= psize- closed := up.closed- up.broadcast()up.mu.Unlock()- if closed {- // The write failed because the upload was aborted while- // this part was being pumped: report the upload gone- // rather than the sink's ECLOSED as an internal error.- return gofakes3.ErrNoSuchUpload- }- return err+ return up.stopStreaming(err)}up.mu.Lock()@@ -531,6 +780,18 @@ func (b *s3Backend) AbortMultipartUpload(ctx context.Context, bucketName, objectreturn nil}+// failUpload tears down an upload which can never be completed, so that+// the client starts again instead of retrying a part which can't be+// accepted.+func (b *s3Backend) failUpload(uploadID gofakes3.UploadID, up *multipartUpload) {+ fs.Errorf(up.fp, "failing multipart upload %s: %v", uploadID, errMultipartPoisoned)+ b.multipartUploads.Delete(uploadID)+ if err := up.abort(); err != nil {+ fs.Errorf(up.fp, "aborting multipart upload: %v", err)+ }+ b.discardUpload(up)+}+// startReaper starts a goroutine which aborts incomplete multipart// uploads once they have been idle for expiry. Stopped by stopReaper.func (b *s3Backend) startReaper(expiry time.Duration) {@@ -652,7 +913,7 @@ func (up *multipartUpload) close() error {}return nil}- if len(up.streamBuf) != 0 || up.nextPart-1 != len(up.partSizes) {+ if len(up.streamBuf) != 0 || up.nextPart-1 != len(up.partSizes) || up.pumping || up.partialSize != 0 {up.mu.Unlock()return gofakes3.ErrInvalidPart}@@ -663,6 +924,10 @@ func (up *multipartUpload) close() error {return up.fh.Close()}+// errMultipartPoisoned is why an upload whose stream holds data it can+// never complete with is failed.+var errMultipartPoisoned = errors.New("a part was rejected after it had been streamed to the backend, so the upload must be started again")+// errMultipartAborted is the reason an aborted upload's write is abandoned// with, so the streaming upload fails instead of committing what it has.var errMultipartAborted = errors.New("serve s3: multipart upload aborted")diff --git a/cmd/serve/s3/multipart_test.go b/cmd/serve/s3/multipart_test.goindex fb483ee44..2d61817c9 100644--- a/cmd/serve/s3/multipart_test.go+++ b/cmd/serve/s3/multipart_test.go@@ -6,6 +6,7 @@ import ("bytes""context""crypto/md5"+ "encoding/base64""fmt""io""math"@@ -411,7 +412,7 @@ func TestMultipartReplaceBuffered(t *testing.T) {// missing part arrives and the buffer drains.func TestMultipartBufferLimit(t *testing.T) {core, f, bucket := newMultipartTestServerOpt(t, "", false, func(opt *Options) {- opt.MultipartStreamingBufferLimit = 50 * 1024+ opt.MultipartStreamingBufferLimit = pool.BufferSize})ctx := context.Background()const object = "buffer-limit.bin"@@ -427,8 +428,8 @@ func TestMultipartBufferLimit(t *testing.T) {uploadID, err := core.NewMultipartUpload(ctx, bucket, object, minio.PutObjectOptions{})require.NoError(t, err)- // Part 3 arrives first: the buffer is empty so it is admitted even though- // it must wait for its turn.+ // Part 3 arrives first: it fits in the buffer so it is admitted even+ // though it must wait for its turn.p3, err := core.PutObjectPart(ctx, bucket, object, uploadID, 3, bytes.NewReader(datas[2]), int64(sizes[2]), minio.PutObjectPartOptions{})require.NoError(t, err)@@ -490,10 +491,10 @@ func (s *stubSink) CloseWithError(err error) error {return nil}-// TestMultipartAbortDuringUploadPart aborts the upload between a part's-// waitForTurn and its streamPart - as happens when the abort arrives while-// the part body is still being received from the client - and checks that-// streamPart fails cleanly instead of panicking on the torn-down upload.+// TestMultipartAbortDuringUploadPart aborts the upload between a buffered+// part's waitForTurn and its bufferPart - as happens when the abort arrives+// while the part body is still being received from the client - and checks+// that bufferPart fails cleanly instead of panicking on the torn-down upload.func TestMultipartAbortDuringUploadPart(t *testing.T) {up := newMultipartUpload("bucket", "key", "bucket/key", "bucket/key", nil, 0)sink := &stubSink{}@@ -502,19 +503,21 @@ func TestMultipartAbortDuringUploadPart(t *testing.T) {// An UploadPart in progress: the part is admitted, then the abort lands// while its body is still being received.contents := []byte("hello")- require.NoError(t, up.waitForTurn(context.Background(), 1, int64(len(contents))))+ turn, err := up.waitForTurn(context.Background(), 2, int64(len(contents)))+ require.NoError(t, err)+ require.Equal(t, turnBuffer, turn)require.NoError(t, up.abort())// The abort must abandon the write rather than committing it.assert.True(t, sink.closed)assert.Equal(t, errMultipartAborted, sink.abortErr)- // The UploadPart resumes: it buffers the part and calls streamPart.+ // The UploadPart resumes: it buffers the part and calls bufferPart.rw := multipart.NewRW()- _, err := rw.Write(contents)+ _, err = rw.Write(contents)require.NoError(t, err)md5Sum := md5.Sum(contents)- err = up.streamPart(1, int64(len(contents)), md5Sum[:], rw)+ err = up.bufferPart(context.Background(), 2, int64(len(contents)), md5Sum[:], rw)require.ErrorIs(t, err, gofakes3.ErrNoSuchUpload)// The reorder buffer reservation must have been returned@@ -555,8 +558,10 @@ func TestMultipartCloseIncomplete(t *testing.T) {_, err := rw.Write(contents)require.NoError(t, err)md5Sum := md5.Sum(contents)- require.NoError(t, up.waitForTurn(context.Background(), 2, int64(len(contents))))- require.NoError(t, up.streamPart(2, int64(len(contents)), md5Sum[:], rw))+ turn, err := up.waitForTurn(context.Background(), 2, int64(len(contents)))+ require.NoError(t, err)+ require.Equal(t, turnBuffer, turn)+ require.NoError(t, up.bufferPart(context.Background(), 2, int64(len(contents)), md5Sum[:], rw))require.ErrorIs(t, up.close(), gofakes3.ErrInvalidPart)assert.False(t, sink.closed)@@ -1025,29 +1030,35 @@ func TestUploadPartNoReserveBeforeBody(t *testing.T) {}// TestWaitForTurnRejectsBogusSize checks that the reorder-buffer admission-// rejects a negative client-declared part length and that a huge declared-// length cannot overflow the running total so as to admit a further part past-// the buffer limit.+// rejects a negative client-declared part length, and that a huge declared+// length is never admitted to the buffer - it must wait to be streamed - nor+// can it overflow the running total.func TestWaitForTurnRejectsBogusSize(t *testing.T) {up := newMultipartUpload("bucket", "key", "bucket/key", "bucket/key", nil, 1<<20)+ ctx := context.Background()// A part length can never be negative.- require.ErrorIs(t, up.waitForTurn(context.Background(), 1, -1), gofakes3.ErrInvalidArgument)+ _, err := up.waitForTurn(ctx, 1, -1)+ require.ErrorIs(t, err, gofakes3.ErrInvalidArgument)- // A huge out-of-order part is admitted once because the buffer is empty,- // driving buffered near the top of the int64 range.- require.NoError(t, up.waitForTurn(context.Background(), 2, math.MaxInt64))+ // The next part is streamed, whatever its size, without touching the+ // buffer.+ turn, err := up.waitForTurn(ctx, 1, math.MaxInt64)+ require.NoError(t, err)+ assert.Equal(t, turnStream, turn)+ up.mu.Lock()+ assert.Equal(t, int64(0), up.buffered)+ up.mu.Unlock()- // A further out-of-order part must wait, not be wrongly admitted by an- // overflow of buffered+size.+ // A huge out-of-order part must wait for its turn, not be admitted.admitted := make(chan struct{})go func() {- _ = up.waitForTurn(context.Background(), 3, math.MaxInt64)+ _, _ = up.waitForTurn(ctx, 2, math.MaxInt64)close(admitted)}()select {case <-admitted:- t.Fatal("out-of-order part admitted past the buffer limit via overflow")+ t.Fatal("huge out-of-order part admitted past the buffer limit")case <-time.After(50 * time.Millisecond):}@@ -1197,3 +1208,277 @@ func TestMultipartBufferWaitGivesUp(t *testing.T) {require.NoError(t, b.AbortMultipartUpload(ctx, bucket, object, uploadID))}++// bufferSink is a multipartUpload sink which collects what is written to it.+type bufferSink struct {+ mu sync.Mutex+ buf bytes.Buffer+}++func (s *bufferSink) Write(p []byte) (int, error) {+ s.mu.Lock()+ defer s.mu.Unlock()+ return s.buf.Write(p)+}++func (s *bufferSink) Close() error { return nil }++func (s *bufferSink) Bytes() []byte {+ s.mu.Lock()+ defer s.mu.Unlock()+ return bytes.Clone(s.buf.Bytes())+}++// newBufferSinkUpload starts a multipart upload on b whose parts are+// collected in the returned sink instead of being written to the VFS, so the+// memory used by the VFS upload doesn't hide the memory used by serve s3.+func newBufferSinkUpload(t *testing.T, b *s3Backend, bucket, object string) (gofakes3.UploadID, *bufferSink) {+ uploadID, err := b.CreateMultipartUpload(context.Background(), bucket, object, nil)+ require.NoError(t, err)+ up, err := b.loadUpload(uploadID)+ require.NoError(t, err)+ if up.fh != nil {+ // This fails the VFS upload, returning errBoom+ _ = up.fh.(interface{ CloseWithError(error) error }).CloseWithError(errBoom)+ }+ sink := &bufferSink{}+ up.fh = sink+ t.Cleanup(func() { _ = b.AbortMultipartUpload(context.Background(), bucket, object, uploadID) })+ return uploadID, sink+}++// TestMultipartInOrderPartNotBuffered checks that the part the stream needs+// is written straight to the sink rather than buffered in memory, however+// big it is compared with the buffer limit.+func TestMultipartInOrderPartNotBuffered(t *testing.T) {+ b, _, bucket := newPutTestBackend(t, "", nil)+ b.s.opt.MultipartStreamingBufferLimit = pool.BufferSize+ ctx := context.Background()+ const object = "in-order.bin"+ const partSize = 8 * pool.BufferSize++ uploadID, sink := newBufferSinkUpload(t, b, bucket, object)+ data := []byte(random.String(partSize))+ body := newPoolWatchReader(data)+ _, err := b.UploadPart(ctx, bucket, object, uploadID, 1, partSize, body)+ require.NoError(t, err)+ assert.Equal(t, 0, body.maxPages, "the in-order part was buffered in memory")+ assert.Equal(t, data, sink.Bytes())+}++// failAfterReader returns data and then fails with err.+func failAfterReader(data []byte, err error) io.Reader {+ return io.MultiReader(bytes.NewReader(data), iotestErrReader{err})+}++type iotestErrReader struct{ err error }++func (r iotestErrReader) Read([]byte) (int, error) { return 0, r.err }++// TestMultipartResumeFailedPart checks that when the body of a part being+// streamed fails part way, a retry of the part with the same contents+// carries on from where it stopped, while one with different contents is+// rejected without disturbing the stream.+func TestMultipartResumeFailedPart(t *testing.T) {+ b, f, bucket := newPutTestBackend(t, "", nil)+ ctx := context.Background()+ const object = "resume.bin"++ part1 := []byte(random.String(300 * 1024))+ part2 := []byte(random.String(100 * 1024))+ want := append(append([]byte(nil), part1...), part2...)++ uploadID, err := b.CreateMultipartUpload(ctx, bucket, object, nil)+ require.NoError(t, err)++ // The first attempt at part 1 fails after 123457 bytes reach the sink.+ _, err = b.UploadPart(ctx, bucket, object, uploadID, 1, int64(len(part1)), failAfterReader(part1[:123457], errBoom))+ require.ErrorIs(t, err, errBoom)++ // A retry which fails before the resume point changes nothing.+ _, err = b.UploadPart(ctx, bucket, object, uploadID, 1, int64(len(part1)), failAfterReader(part1[:1000], errBoom))+ require.ErrorIs(t, err, errBoom)++ // A retry with different contents is rejected.+ other := append([]byte(nil), part1...)+ other[100] ^= 1+ _, err = b.UploadPart(ctx, bucket, object, uploadID, 1, int64(len(other)), bytes.NewReader(other))+ assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrNotImplemented), "want NotImplemented, got %v", err)++ // As is a retry shorter than what is already in the stream.+ _, err = b.UploadPart(ctx, bucket, object, uploadID, 1, 1000, bytes.NewReader(part1[:1000]))+ assert.True(t, gofakes3.HasErrorCode(err, gofakes3.ErrNotImplemented), "want NotImplemented, got %v", err)++ // A second failure part way through extends what is in the stream.+ _, err = b.UploadPart(ctx, bucket, object, uploadID, 1, int64(len(part1)), failAfterReader(part1[:200000], errBoom))+ require.ErrorIs(t, err, errBoom)++ // The whole retry completes the part.+ p1, err := b.UploadPart(ctx, bucket, object, uploadID, 1, int64(len(part1)), bytes.NewReader(part1))+ require.NoError(t, err)+ p2, err := b.UploadPart(ctx, bucket, object, uploadID, 2, int64(len(part2)), bytes.NewReader(part2))+ require.NoError(t, err)++ _, _, err = b.CompleteMultipartUpload(ctx, bucket, object, uploadID, &gofakes3.CompleteMultipartUploadRequest{+ Parts: []gofakes3.CompletedPart{{PartNumber: 1, ETag: p1}, {PartNumber: 2, ETag: p2}},+ })+ require.NoError(t, err)+ assert.Equal(t, want, readObject(t, f, bucket, object))+}++// TestMultipartCompleteWithPartialPart checks that an upload whose stream+// holds part of a failed part can't be completed without it.+func TestMultipartCompleteWithPartialPart(t *testing.T) {+ b, f, bucket := newPutTestBackend(t, "", nil)+ ctx := context.Background()+ const object = "partial.bin"++ uploadID, err := b.CreateMultipartUpload(ctx, bucket, object, nil)+ require.NoError(t, err)+ part1 := []byte(random.String(200 * 1024))+ _, err = b.UploadPart(ctx, bucket, object, uploadID, 1, int64(len(part1)), failAfterReader(part1[:150000], errBoom))+ require.ErrorIs(t, err, errBoom)++ _, _, err = b.CompleteMultipartUpload(ctx, bucket, object, uploadID, &gofakes3.CompleteMultipartUploadRequest{})+ require.ErrorIs(t, err, gofakes3.ErrInvalidPart)+ _, err = f.NewObject(ctx, path.Join(bucket, object))+ require.ErrorIs(t, err, fs.ErrorObjectNotFound)+}++// TestMultipartConcurrentDuplicateParts checks that further requests for the+// part being streamed wait, without buffering it, for the first to finish,+// then are checked against it.+func TestMultipartConcurrentDuplicateParts(t *testing.T) {+ b, _, bucket := newPutTestBackend(t, "", nil)+ ctx := context.Background()+ const object = "duplicates.bin"+ const partSize = 8 * pool.BufferSize++ uploadID, sink := newBufferSinkUpload(t, b, bucket, object)+ data := []byte(random.String(partSize))++ // The first request stalls half way through its body.+ pr, pw := io.Pipe()+ first := make(chan error, 1)+ var firstETag string+ go func() {+ var err error+ firstETag, err = b.UploadPart(ctx, bucket, object, uploadID, 1, partSize, pr)+ first <- err+ }()+ _, err := pw.Write(data[:partSize/2])+ require.NoError(t, err)++ type result struct {+ etag string+ err error+ }+ const duplicates = 4+ bodies := make([]*poolWatchReader, duplicates)+ results := make(chan result, duplicates)+ for i := range bodies {+ bodies[i] = newPoolWatchReader(data)+ go func(body io.Reader) {+ etag, err := b.UploadPart(ctx, bucket, object, uploadID, 1, partSize, body)+ results <- result{etag, err}+ }(bodies[i])+ }+ select {+ case r := <-results:+ t.Fatalf("a duplicate finished while the first request was streaming (err=%v)", r.err)+ case <-time.After(200 * time.Millisecond):+ }++ _, err = pw.Write(data[partSize/2:])+ require.NoError(t, err)+ // The body must end, as a request body does, so that the part is+ // read to its end.+ require.NoError(t, pw.Close())+ require.NoError(t, <-first)+ for range duplicates {+ r := <-results+ require.NoError(t, r.err)+ assert.Equal(t, firstETag, r.etag)+ }+ for _, body := range bodies {+ assert.Equal(t, 0, body.maxPages, "a duplicate part was buffered in memory")+ }+ assert.Equal(t, data, sink.Bytes())+}++// TestMultipartPartFailsAfterWholeBody checks that a part whose body is+// delivered in full but is then rejected - as gofakes3's Content-MD5 check+// does, after every byte has reached the backend - fails the whole upload,+// rather than leaving it in a state no retry of the part can escape.+func TestMultipartPartFailsAfterWholeBody(t *testing.T) {+ b, _, bucket := newPutTestBackend(t, "", nil)+ ctx := context.Background()+ const object = "bad-digest.bin"++ uploadID, err := b.CreateMultipartUpload(ctx, bucket, object, nil)+ require.NoError(t, err)+ part1 := []byte(random.String(100 * 1024))++ // The whole body arrives, then the integrity check rejects it.+ _, err = b.UploadPart(ctx, bucket, object, uploadID, 1, int64(len(part1)),+ failAfterReader(part1, gofakes3.ErrBadDigest))+ require.Error(t, err)++ // The upload is gone, so the client starts again rather than retrying+ // a part which could never be accepted.+ _, err = b.UploadPart(ctx, bucket, object, uploadID, 1, int64(len(part1)), bytes.NewReader(part1))+ require.ErrorIs(t, err, gofakes3.ErrNoSuchUpload)+ _, err = b.loadUpload(uploadID)+ require.ErrorIs(t, err, gofakes3.ErrNoSuchUpload)+}++// TestMultipartPartBadMD5 checks that a part whose body doesn't match the+// Content-MD5 the client declared is rejected, and that the upload it+// corrupted is failed rather than left for the client to retry.+func TestMultipartPartBadMD5(t *testing.T) {+ core, f, bucket := newMultipartTestServer(t, false)+ ctx := context.Background()+ const object = "bad-md5.bin"++ uploadID, err := core.NewMultipartUpload(ctx, bucket, object, minio.PutObjectOptions{})+ require.NoError(t, err)++ part1 := []byte(random.String(60 * 1024))+ wrongMD5 := md5.Sum([]byte("not the part"))+ _, err = core.PutObjectPart(ctx, bucket, object, uploadID, 1, bytes.NewReader(part1), int64(len(part1)),+ minio.PutObjectPartOptions{Md5Base64: base64.StdEncoding.EncodeToString(wrongMD5[:])})+ require.Error(t, err)++ // Nothing was committed at the key and the upload is gone.+ _, err = f.NewObject(ctx, path.Join(bucket, object))+ require.ErrorIs(t, err, fs.ErrorObjectNotFound)+ _, err = core.PutObjectPart(ctx, bucket, object, uploadID, 1, bytes.NewReader(part1), int64(len(part1)), minio.PutObjectPartOptions{})+ require.Error(t, err)+ assert.Contains(t, err.Error(), "NoSuchUpload")+ requireOnly(t, f, bucket)+}++// TestLimitedBody checks that a part body is read to its end, so that an+// error reported there is not missed, and that a body longer than declared+// is rejected.+func TestLimitedBody(t *testing.T) {+ for _, test := range []struct {+ name string+ body io.Reader+ want error+ }{+ {"EndError", failAfterReader([]byte("hello"), gofakes3.ErrBadDigest), gofakes3.ErrBadDigest},+ {"TooLong", bytes.NewReader([]byte("hello world")), gofakes3.ErrIncompleteBody},+ {"Exact", bytes.NewReader([]byte("hello")), nil},+ } {+ t.Run(test.name, func(t *testing.T) {+ n, err := io.Copy(io.Discard, newLimitedBody(test.body, 5))+ assert.Equal(t, int64(5), n)+ if test.want == nil {+ assert.NoError(t, err)+ } else {+ assert.ErrorIs(t, err, test.want)+ }+ })+ }+}diff --git a/cmd/serve/s3/serve_s3.md b/cmd/serve/s3/serve_s3.mdindex a2d5a4b5a..7f31ebfc1 100644--- a/cmd/serve/s3/serve_s3.md+++ b/cmd/serve/s3/serve_s3.md@@ -189,12 +189,15 @@ remote as if it had completed.**Limitations**- Parts must arrive in ascending, contiguous part-number order- (1, 2, 3, ...). Parts the client uploads concurrently or out of order- are buffered until their turn. The memory used for this buffering is+ (1, 2, 3, ...). The part the stream needs next is written straight+ through as it arrives, so it is never buffered, whatever its size.+ Parts the client uploads concurrently or out of order are buffered+ until their turn. The memory used for this buffering iscapped, per upload, by `--multipart-streaming-buffer-limit` (default`256M`, `0` for no limit), counting each buffered part in the whole1 MiB memory pages it occupies: a part that would take the buffer over the- limit is stalled until the stream drains, so a client that uploads+ limit is stalled until the stream drains (or, if it is bigger than the+ limit, until it is the next part), so a client that uploadsfaster than the remote can accept sees backpressure rather thanunbounded server memory use. A part stalled for more than a minuteis failed with a `SlowDown` error, which S3 clients retry. Since a stalled part holds its HTTP@@ -206,8 +209,11 @@ remote as if it had completed.- A part uploaded again before completion - typically a client retryingafter a timeout - is accepted: if the earlier copy is still bufferedit is replaced, and if it has already been streamed an identical- re-upload is a no-op. What isn't possible is replacing a part that has- already been streamed with *different* content - that is rejected. A+ re-upload is a no-op. If a part fails part way through being streamed+ (for example the client's connection drops) the client's retry of it+ carries on where it stopped, provided it has the same content. What+ isn't possible is replacing a part that has already been streamed,+ in whole or in part, with *different* content - that is rejected. Afailure in the stream to the remote itself still aborts the wholeupload and the client must start it again. (The remote's own uploadstill retries its internal chunks.)