Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

serve s3: add --multipart-streaming-buffer-total to limit memory across uploads

Nick Craig-Wood87d81fcb2e41
diff --git a/cmd/serve/s3/backend.go b/cmd/serve/s3/backend.go
index d780c6527..61375e29e 100644
--- a/cmd/serve/s3/backend.go
+++ b/cmd/serve/s3/backend.go
@@ -49,6 +49,9 @@ type s3Backend struct {
// upload falls back to being buffered in memory.
warnInMemoryOnce sync.Once
+ // budget limits the memory buffered by all multipart uploads.
+ budget *bufferBudget
+
reaperQuit chan struct{} // closed to stop the abandoned upload reaper
reaperStop sync.Once
}
@@ -58,6 +61,7 @@ func newBackend(s *Server) *s3Backend {
return &s3Backend{
s: s,
meta: new(sync.Map),
+ budget: newBufferBudget(int64(s.opt.MultipartStreamingBufferTotal)),
reaperQuit: make(chan struct{}),
}
}
diff --git a/cmd/serve/s3/multipart.go b/cmd/serve/s3/multipart.go
index 9f1ae1427..cd71d7462 100644
--- a/cmd/serve/s3/multipart.go
+++ b/cmd/serve/s3/multipart.go
@@ -87,6 +87,52 @@ type multipartUpload struct {
partialMD5 []byte // MD5 state after those partialSize bytes (encoding.BinaryMarshaler)
buffered int64 // bytes of parts admitted but not yet streamed or released
bufferLimit int64 // max buffered before parts ahead of nextPart must wait (<= 0 for no limit)
+ budget *bufferBudget // limits buffered across all uploads (nil for no limit)
+}
+
+// bufferBudget limits the memory used by the reorder buffers of all the
+// multipart uploads of a server.
+type bufferBudget struct {
+ mu sync.Mutex
+ limit int64 // <= 0 for no limit
+ used int64 // bytes reserved
+ changed chan struct{} // closed when used shrinks
+}
+
+// newBufferBudget makes a bufferBudget of limit bytes, <= 0 for no limit.
+func newBufferBudget(limit int64) *bufferBudget {
+ return &bufferBudget{
+ limit: limit,
+ changed: make(chan struct{}),
+ }
+}
+
+// tryReserve reserves n bytes if they fit in the budget. If not it returns
+// a channel which is closed when some are released. A nil budget has no
+// limit.
+func (bb *bufferBudget) tryReserve(n int64) (ok bool, changed <-chan struct{}) {
+ if bb == nil {
+ return true, nil
+ }
+ bb.mu.Lock()
+ defer bb.mu.Unlock()
+ if bb.limit > 0 && n > bb.limit-bb.used {
+ return false, bb.changed
+ }
+ bb.used += n
+ return true, nil
+}
+
+// release returns n bytes reserved with tryReserve.
+func (bb *bufferBudget) release(n int64) {
+ if bb == nil || n == 0 {
+ return
+ }
+ bb.mu.Lock()
+ defer bb.mu.Unlock()
+ bb.used -= n
+ close(bb.changed)
+ bb.changed = make(chan struct{})
}
// newMultipartUpload allocates an upload struct.
@@ -114,6 +160,14 @@ func (up *multipartUpload) broadcast() {
up.changed = make(chan struct{})
}
+// unbuffer returns charge bytes of buffer reserved by waitForTurn to the
+// upload's and the server's budgets. Call with up.mu held.
+func (up *multipartUpload) unbuffer(charge int64) {
+ up.buffered -= charge
+ up.budget.release(charge)
+ up.broadcast()
+}
+
// startActivity marks the upload as having a request in flight.
func (up *multipartUpload) startActivity() {
up.mu.Lock()
@@ -198,6 +252,7 @@ func (b *s3Backend) CreateMultipartUpload(ctx context.Context, bucketName, objec
}
up := newMultipartUpload(bucketName, objectName, fp, streamFp, meta, int64(b.s.opt.MultipartStreamingBufferLimit))
+ up.budget = b.budget
fh, err := _vfs.Create(streamFp)
if err != nil {
return "", err
@@ -381,10 +436,11 @@ const (
//
// The next part the stream needs is streamed straight into the sink, once
// no other request is writing to it, so it needs no buffer. A part ahead of
-// it must wait until its bufferCharge fits within the reorder buffer limit,
-// which it then reserves, bounding the memory an upload can consume when
-// the client sends parts faster than the backend drains them. A part too big
-// for the limit waits until it is the next part. Reserved bytes are returned
+// it must wait until its bufferCharge fits within both the upload's reorder
+// buffer limit and the server's budget for all uploads, which it then
+// reserves, bounding the memory uploads can consume when clients send parts
+// faster than the backend drains them. A part too big for the limits waits
+// until it is the next part. Reserved bytes are returned
// with release, or by the pump as the part is streamed.
//
// A part which waits for longer than multipartWaitTimeout gets errSlowDown,
@@ -415,15 +471,21 @@ func (up *multipartUpload) waitForTurn(ctx context.Context, partNumber int, size
up.pumping = true
up.mu.Unlock()
return turnStream, nil
- case partNumber > up.nextPart && (up.bufferLimit <= 0 || charge <= up.bufferLimit-up.buffered):
- up.buffered += charge
- up.mu.Unlock()
- return turnBuffer, nil
+ }
+ var budgetChanged <-chan struct{}
+ if partNumber > up.nextPart && (up.bufferLimit <= 0 || charge <= up.bufferLimit-up.buffered) {
+ var ok bool
+ if ok, budgetChanged = up.budget.tryReserve(charge); ok {
+ up.buffered += charge
+ up.mu.Unlock()
+ return turnBuffer, nil
+ }
}
changed := up.changed
up.mu.Unlock()
select {
case <-changed:
+ case <-budgetChanged:
case <-ctx.Done():
return 0, ctx.Err()
case <-timer.C:
@@ -436,8 +498,7 @@ func (up *multipartUpload) waitForTurn(ctx context.Context, partNumber int, size
// to the reorder buffer budget and wakes any parts waiting for room.
func (up *multipartUpload) release(size int64) {
up.mu.Lock()
- up.buffered -= bufferCharge(size)
- up.broadcast()
+ up.unbuffer(bufferCharge(size))
up.mu.Unlock()
}
@@ -592,8 +653,7 @@ func (up *multipartUpload) bufferPart(ctx context.Context, partNumber int, size
charge := bufferCharge(size)
// done releases the part's buffer. Call with up.mu held.
done := func() {
- up.buffered -= charge
- up.broadcast()
+ up.unbuffer(charge)
_ = rw.Close()
}
timer := time.NewTimer(multipartWaitTimeout)
@@ -648,8 +708,7 @@ func (up *multipartUpload) bufferPart(ctx context.Context, partNumber int, size
}
if old, buffered := up.streamBuf[partNumber]; buffered {
_ = old.Close()
- up.buffered -= bufferCharge(up.partSizes[partNumber])
- up.broadcast()
+ up.unbuffer(bufferCharge(up.partSizes[partNumber]))
}
up.partMD5s[partNumber] = md5Sum
up.partSizes[partNumber] = size
@@ -686,15 +745,14 @@ func (up *multipartUpload) pump() error {
_ = prw.Close()
if err != nil {
up.mu.Lock()
- up.buffered -= psize
+ up.unbuffer(psize)
up.mu.Unlock()
return up.stopStreaming(err)
}
up.mu.Lock()
up.nextPart++
- up.buffered -= psize
- up.broadcast()
+ up.unbuffer(psize)
}
}
@@ -948,6 +1006,9 @@ func (up *multipartUpload) abort() error {
up.aborted = true
streamBuf := up.streamBuf
up.streamBuf = nil
+ for _, rw := range streamBuf {
+ up.unbuffer(bufferCharge(rw.Size()))
+ }
up.broadcast()
up.mu.Unlock()
diff --git a/cmd/serve/s3/multipart_test.go b/cmd/serve/s3/multipart_test.go
index 2d61817c9..a2853a060 100644
--- a/cmd/serve/s3/multipart_test.go
+++ b/cmd/serve/s3/multipart_test.go
@@ -1482,3 +1482,54 @@ func TestLimitedBody(t *testing.T) {
})
}
}
+
+// TestMultipartBufferTotal checks that --multipart-streaming-buffer-total
+// limits the memory buffered across all uploads, and that an aborted
+// upload's buffered parts are returned to it.
+func TestMultipartBufferTotal(t *testing.T) {
+ b, _, bucket := newPutTestBackend(t, "", nil)
+ b.budget = newBufferBudget(2 * pool.BufferSize)
+ ctx := context.Background()
+
+ upload := func(object string, uploadID gofakes3.UploadID, partNumber int) (string, error) {
+ body := []byte{byte('0' + partNumber)}
+ return b.UploadPart(ctx, bucket, object, uploadID, partNumber, 1, bytes.NewReader(body))
+ }
+
+ // Upload A buffers two out-of-order parts, using up the total.
+ idA, err := b.CreateMultipartUpload(ctx, bucket, "a", nil)
+ require.NoError(t, err)
+ for partNumber := 2; partNumber <= 3; partNumber++ {
+ _, err = upload("a", idA, partNumber)
+ require.NoError(t, err)
+ }
+
+ // Upload B's out-of-order part must wait although its own buffer is
+ // empty.
+ idB, err := b.CreateMultipartUpload(ctx, bucket, "b", nil)
+ require.NoError(t, err)
+ done := make(chan error, 1)
+ go func() {
+ _, err := upload("b", idB, 2)
+ done <- err
+ }()
+ select {
+ case err := <-done:
+ t.Fatalf("upload b's part was not blocked by the buffer total (err=%v)", err)
+ case <-time.After(200 * time.Millisecond):
+ }
+
+ // Aborting upload A frees its buffered parts.
+ require.NoError(t, b.AbortMultipartUpload(ctx, bucket, "a", idA))
+ select {
+ case err := <-done:
+ require.NoError(t, err)
+ case <-time.After(10 * time.Second):
+ t.Fatal("upload b's part was never unblocked")
+ }
+ require.NoError(t, b.AbortMultipartUpload(ctx, bucket, "b", idB))
+
+ b.budget.mu.Lock()
+ assert.Equal(t, int64(0), b.budget.used, "buffer total not all returned")
+ b.budget.mu.Unlock()
+}
diff --git a/cmd/serve/s3/s3.go b/cmd/serve/s3/s3.go
index 5dd9df778..587c6412a 100644
--- a/cmd/serve/s3/s3.go
+++ b/cmd/serve/s3/s3.go
@@ -45,6 +45,10 @@ var OptionsInfo = fs.Options{{
Name: "multipart_streaming_buffer_limit",
Default: fs.SizeSuffix(256 * 1024 * 1024),
Help: "Maximum memory buffered per streamed multipart upload for parts arriving out of order, 0 for unlimited",
+}, {
+ Name: "multipart_streaming_buffer_total",
+ Default: fs.SizeSuffix(1024 * 1024 * 1024),
+ Help: "Maximum memory buffered by all streamed multipart uploads for parts arriving out of order, 0 for unlimited",
}, {
Name: "multipart_expiry",
Default: fs.Duration(24 * time.Hour),
@@ -62,6 +66,7 @@ type Options struct {
NoCleanup bool `config:"no_cleanup"`
DisableMultipartStreaming bool `config:"disable_multipart_streaming"`
MultipartStreamingBufferLimit fs.SizeSuffix `config:"multipart_streaming_buffer_limit"`
+ MultipartStreamingBufferTotal fs.SizeSuffix `config:"multipart_streaming_buffer_total"`
MultipartExpiry fs.Duration `config:"multipart_expiry"`
Auth httplib.AuthConfig
HTTP httplib.Config
diff --git a/cmd/serve/s3/serve_s3.md b/cmd/serve/s3/serve_s3.md
index 7f31ebfc1..11f737544 100644
--- a/cmd/serve/s3/serve_s3.md
+++ b/cmd/serve/s3/serve_s3.md
@@ -188,22 +188,24 @@ remote as if it had completed.
**Limitations**
-- Parts must arrive in ascending, contiguous part-number order
- (1, 2, 3, ...). The part the stream needs next is written straight
- through as it arrives, so it is never buffered, whatever its size.
- Parts the client uploads concurrently or out of order are buffered
- until their turn. The memory used for this buffering is
- capped, per upload, by `--multipart-streaming-buffer-limit` (default
- `256M`, `0` for no limit), counting each buffered part in the whole
- 1 MiB memory pages it occupies: a part that would take the buffer over the
- limit is stalled until the stream drains (or, if it is bigger than the
- limit, until it is the next part), so a client that uploads
- faster than the remote can accept sees backpressure rather than
- unbounded server memory use. A part stalled for more than a minute
- is failed with a `SlowDown` error, which S3 clients retry. Since a stalled part holds its HTTP
- request open, clients whose upload concurrency times chunk size
- exceeds the limit may need a longer read timeout when the remote is
- slow. Non-contiguous part numbers are rejected on completion.
+- Parts must arrive in ascending, contiguous part-number order (1, 2, 3,
+ ...). The part the stream needs next is written straight through as it
+ arrives, so it is never buffered, whatever its size. Parts the client
+ uploads concurrently or out of order are buffered until their turn.
+ The memory used for this buffering is capped, per upload, by
+ `--multipart-streaming-buffer-limit` (default `256M`, `0` for no
+ limit), and across all uploads by `--multipart-streaming-buffer-total`
+ (default `1G`, `0` for no limit), counting each buffered part in the
+ whole 1 MiB memory pages it occupies: a part that would take the
+ buffer over either limit is stalled until the stream drains (or, if it
+ is bigger than the limit, until it is the next part), so a client that
+ uploads faster than the remote can accept sees backpressure rather
+ than unbounded server memory use. A part stalled for more than a
+ minute is failed with a `SlowDown` error, which S3 clients retry.
+ Since a stalled part holds its HTTP request open, clients whose upload
+ concurrency times chunk size exceeds the limit may need a longer read
+ timeout when the remote is slow. Non-contiguous part numbers are
+ rejected on completion.
Configure the client to upload in part order, ideally with low
concurrency, for the lowest memory use.
- A part uploaded again before completion - typically a client retrying