"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
serve s3: add --multipart-streaming-buffer-total to limit memory across uploads
Nick Craig-Wood87d81fcb2e41
diff --git a/cmd/serve/s3/backend.go b/cmd/serve/s3/backend.goindex d780c6527..61375e29e 100644--- a/cmd/serve/s3/backend.go+++ b/cmd/serve/s3/backend.go@@ -49,6 +49,9 @@ type s3Backend struct {// upload falls back to being buffered in memory.warnInMemoryOnce sync.Once+ // budget limits the memory buffered by all multipart uploads.+ budget *bufferBudget+reaperQuit chan struct{} // closed to stop the abandoned upload reaperreaperStop sync.Once}@@ -58,6 +61,7 @@ func newBackend(s *Server) *s3Backend {return &s3Backend{s: s,meta: new(sync.Map),+ budget: newBufferBudget(int64(s.opt.MultipartStreamingBufferTotal)),reaperQuit: make(chan struct{}),}}diff --git a/cmd/serve/s3/multipart.go b/cmd/serve/s3/multipart.goindex 9f1ae1427..cd71d7462 100644--- a/cmd/serve/s3/multipart.go+++ b/cmd/serve/s3/multipart.go@@ -87,6 +87,52 @@ type multipartUpload struct {partialMD5 []byte // MD5 state after those partialSize bytes (encoding.BinaryMarshaler)buffered int64 // bytes of parts admitted but not yet streamed or releasedbufferLimit int64 // max buffered before parts ahead of nextPart must wait (<= 0 for no limit)+ budget *bufferBudget // limits buffered across all uploads (nil for no limit)+}++// bufferBudget limits the memory used by the reorder buffers of all the+// multipart uploads of a server.+type bufferBudget struct {+ mu sync.Mutex+ limit int64 // <= 0 for no limit+ used int64 // bytes reserved+ changed chan struct{} // closed when used shrinks+}++// newBufferBudget makes a bufferBudget of limit bytes, <= 0 for no limit.+func newBufferBudget(limit int64) *bufferBudget {+ return &bufferBudget{+ limit: limit,+ changed: make(chan struct{}),+ }+}++// tryReserve reserves n bytes if they fit in the budget. If not it returns+// a channel which is closed when some are released. A nil budget has no+// limit.+func (bb *bufferBudget) tryReserve(n int64) (ok bool, changed <-chan struct{}) {+ if bb == nil {+ return true, nil+ }+ bb.mu.Lock()+ defer bb.mu.Unlock()+ if bb.limit > 0 && n > bb.limit-bb.used {+ return false, bb.changed+ }+ bb.used += n+ return true, nil+}++// release returns n bytes reserved with tryReserve.+func (bb *bufferBudget) release(n int64) {+ if bb == nil || n == 0 {+ return+ }+ bb.mu.Lock()+ defer bb.mu.Unlock()+ bb.used -= n+ close(bb.changed)+ bb.changed = make(chan struct{})}// newMultipartUpload allocates an upload struct.@@ -114,6 +160,14 @@ func (up *multipartUpload) broadcast() {up.changed = make(chan struct{})}+// unbuffer returns charge bytes of buffer reserved by waitForTurn to the+// upload's and the server's budgets. Call with up.mu held.+func (up *multipartUpload) unbuffer(charge int64) {+ up.buffered -= charge+ up.budget.release(charge)+ up.broadcast()+}+// startActivity marks the upload as having a request in flight.func (up *multipartUpload) startActivity() {up.mu.Lock()@@ -198,6 +252,7 @@ func (b *s3Backend) CreateMultipartUpload(ctx context.Context, bucketName, objec}up := newMultipartUpload(bucketName, objectName, fp, streamFp, meta, int64(b.s.opt.MultipartStreamingBufferLimit))+ up.budget = b.budgetfh, err := _vfs.Create(streamFp)if err != nil {return "", err@@ -381,10 +436,11 @@ const (//// The next part the stream needs is streamed straight into the sink, once// no other request is writing to it, so it needs no buffer. A part ahead of-// it must wait until its bufferCharge fits within the reorder buffer limit,-// which it then reserves, bounding the memory an upload can consume when-// the client sends parts faster than the backend drains them. A part too big-// for the limit waits until it is the next part. Reserved bytes are returned+// it must wait until its bufferCharge fits within both the upload's reorder+// buffer limit and the server's budget for all uploads, which it then+// reserves, bounding the memory uploads can consume when clients send parts+// faster than the backend drains them. A part too big for the limits waits+// until it is the next part. Reserved bytes are returned// with release, or by the pump as the part is streamed.//// A part which waits for longer than multipartWaitTimeout gets errSlowDown,@@ -415,15 +471,21 @@ func (up *multipartUpload) waitForTurn(ctx context.Context, partNumber int, sizeup.pumping = trueup.mu.Unlock()return turnStream, nil- case partNumber > up.nextPart && (up.bufferLimit <= 0 || charge <= up.bufferLimit-up.buffered):- up.buffered += charge- up.mu.Unlock()- return turnBuffer, nil+ }+ var budgetChanged <-chan struct{}+ if partNumber > up.nextPart && (up.bufferLimit <= 0 || charge <= up.bufferLimit-up.buffered) {+ var ok bool+ if ok, budgetChanged = up.budget.tryReserve(charge); ok {+ up.buffered += charge+ up.mu.Unlock()+ return turnBuffer, nil+ }}changed := up.changedup.mu.Unlock()select {case <-changed:+ case <-budgetChanged:case <-ctx.Done():return 0, ctx.Err()case <-timer.C:@@ -436,8 +498,7 @@ func (up *multipartUpload) waitForTurn(ctx context.Context, partNumber int, size// to the reorder buffer budget and wakes any parts waiting for room.func (up *multipartUpload) release(size int64) {up.mu.Lock()- up.buffered -= bufferCharge(size)- up.broadcast()+ up.unbuffer(bufferCharge(size))up.mu.Unlock()}@@ -592,8 +653,7 @@ func (up *multipartUpload) bufferPart(ctx context.Context, partNumber int, sizecharge := bufferCharge(size)// done releases the part's buffer. Call with up.mu held.done := func() {- up.buffered -= charge- up.broadcast()+ up.unbuffer(charge)_ = rw.Close()}timer := time.NewTimer(multipartWaitTimeout)@@ -648,8 +708,7 @@ func (up *multipartUpload) bufferPart(ctx context.Context, partNumber int, size}if old, buffered := up.streamBuf[partNumber]; buffered {_ = old.Close()- up.buffered -= bufferCharge(up.partSizes[partNumber])- up.broadcast()+ up.unbuffer(bufferCharge(up.partSizes[partNumber]))}up.partMD5s[partNumber] = md5Sumup.partSizes[partNumber] = size@@ -686,15 +745,14 @@ func (up *multipartUpload) pump() error {_ = prw.Close()if err != nil {up.mu.Lock()- up.buffered -= psize+ up.unbuffer(psize)up.mu.Unlock()return up.stopStreaming(err)}up.mu.Lock()up.nextPart++- up.buffered -= psize- up.broadcast()+ up.unbuffer(psize)}}@@ -948,6 +1006,9 @@ func (up *multipartUpload) abort() error {up.aborted = truestreamBuf := up.streamBufup.streamBuf = nil+ for _, rw := range streamBuf {+ up.unbuffer(bufferCharge(rw.Size()))+ }up.broadcast()up.mu.Unlock()diff --git a/cmd/serve/s3/multipart_test.go b/cmd/serve/s3/multipart_test.goindex 2d61817c9..a2853a060 100644--- a/cmd/serve/s3/multipart_test.go+++ b/cmd/serve/s3/multipart_test.go@@ -1482,3 +1482,54 @@ func TestLimitedBody(t *testing.T) {})}}++// TestMultipartBufferTotal checks that --multipart-streaming-buffer-total+// limits the memory buffered across all uploads, and that an aborted+// upload's buffered parts are returned to it.+func TestMultipartBufferTotal(t *testing.T) {+ b, _, bucket := newPutTestBackend(t, "", nil)+ b.budget = newBufferBudget(2 * pool.BufferSize)+ ctx := context.Background()++ upload := func(object string, uploadID gofakes3.UploadID, partNumber int) (string, error) {+ body := []byte{byte('0' + partNumber)}+ return b.UploadPart(ctx, bucket, object, uploadID, partNumber, 1, bytes.NewReader(body))+ }++ // Upload A buffers two out-of-order parts, using up the total.+ idA, err := b.CreateMultipartUpload(ctx, bucket, "a", nil)+ require.NoError(t, err)+ for partNumber := 2; partNumber <= 3; partNumber++ {+ _, err = upload("a", idA, partNumber)+ require.NoError(t, err)+ }++ // Upload B's out-of-order part must wait although its own buffer is+ // empty.+ idB, err := b.CreateMultipartUpload(ctx, bucket, "b", nil)+ require.NoError(t, err)+ done := make(chan error, 1)+ go func() {+ _, err := upload("b", idB, 2)+ done <- err+ }()+ select {+ case err := <-done:+ t.Fatalf("upload b's part was not blocked by the buffer total (err=%v)", err)+ case <-time.After(200 * time.Millisecond):+ }++ // Aborting upload A frees its buffered parts.+ require.NoError(t, b.AbortMultipartUpload(ctx, bucket, "a", idA))+ select {+ case err := <-done:+ require.NoError(t, err)+ case <-time.After(10 * time.Second):+ t.Fatal("upload b's part was never unblocked")+ }+ require.NoError(t, b.AbortMultipartUpload(ctx, bucket, "b", idB))++ b.budget.mu.Lock()+ assert.Equal(t, int64(0), b.budget.used, "buffer total not all returned")+ b.budget.mu.Unlock()+}diff --git a/cmd/serve/s3/s3.go b/cmd/serve/s3/s3.goindex 5dd9df778..587c6412a 100644--- a/cmd/serve/s3/s3.go+++ b/cmd/serve/s3/s3.go@@ -45,6 +45,10 @@ var OptionsInfo = fs.Options{{Name: "multipart_streaming_buffer_limit",Default: fs.SizeSuffix(256 * 1024 * 1024),Help: "Maximum memory buffered per streamed multipart upload for parts arriving out of order, 0 for unlimited",+}, {+ Name: "multipart_streaming_buffer_total",+ Default: fs.SizeSuffix(1024 * 1024 * 1024),+ Help: "Maximum memory buffered by all streamed multipart uploads for parts arriving out of order, 0 for unlimited",}, {Name: "multipart_expiry",Default: fs.Duration(24 * time.Hour),@@ -62,6 +66,7 @@ type Options struct {NoCleanup bool `config:"no_cleanup"`DisableMultipartStreaming bool `config:"disable_multipart_streaming"`MultipartStreamingBufferLimit fs.SizeSuffix `config:"multipart_streaming_buffer_limit"`+ MultipartStreamingBufferTotal fs.SizeSuffix `config:"multipart_streaming_buffer_total"`MultipartExpiry fs.Duration `config:"multipart_expiry"`Auth httplib.AuthConfigHTTP httplib.Configdiff --git a/cmd/serve/s3/serve_s3.md b/cmd/serve/s3/serve_s3.mdindex 7f31ebfc1..11f737544 100644--- a/cmd/serve/s3/serve_s3.md+++ b/cmd/serve/s3/serve_s3.md@@ -188,22 +188,24 @@ remote as if it had completed.**Limitations**-- Parts must arrive in ascending, contiguous part-number order- (1, 2, 3, ...). The part the stream needs next is written straight- through as it arrives, so it is never buffered, whatever its size.- Parts the client uploads concurrently or out of order are buffered- until their turn. The memory used for this buffering is- capped, per upload, by `--multipart-streaming-buffer-limit` (default- `256M`, `0` for no limit), counting each buffered part in the whole- 1 MiB memory pages it occupies: a part that would take the buffer over the- limit is stalled until the stream drains (or, if it is bigger than the- limit, until it is the next part), so a client that uploads- faster than the remote can accept sees backpressure rather than- unbounded server memory use. A part stalled for more than a minute- is failed with a `SlowDown` error, which S3 clients retry. Since a stalled part holds its HTTP- request open, clients whose upload concurrency times chunk size- exceeds the limit may need a longer read timeout when the remote is- slow. Non-contiguous part numbers are rejected on completion.+- Parts must arrive in ascending, contiguous part-number order (1, 2, 3,+ ...). The part the stream needs next is written straight through as it+ arrives, so it is never buffered, whatever its size. Parts the client+ uploads concurrently or out of order are buffered until their turn.+ The memory used for this buffering is capped, per upload, by+ `--multipart-streaming-buffer-limit` (default `256M`, `0` for no+ limit), and across all uploads by `--multipart-streaming-buffer-total`+ (default `1G`, `0` for no limit), counting each buffered part in the+ whole 1 MiB memory pages it occupies: a part that would take the+ buffer over either limit is stalled until the stream drains (or, if it+ is bigger than the limit, until it is the next part), so a client that+ uploads faster than the remote can accept sees backpressure rather+ than unbounded server memory use. A part stalled for more than a+ minute is failed with a `SlowDown` error, which S3 clients retry.+ Since a stalled part holds its HTTP request open, clients whose upload+ concurrency times chunk size exceeds the limit may need a longer read+ timeout when the remote is slow. Non-contiguous part numbers are+ rejected on completion.Configure the client to upload in part order, ideally with lowconcurrency, for the lowest memory use.- A part uploaded again before completion - typically a client retrying