Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

serve s3: add --multipart-max-uploads to limit uploads in progress

Nick Craig-Woodd24568e3e640
diff --git a/cmd/serve/s3/backend.go b/cmd/serve/s3/backend.go
index 61375e29e..dbb30265f 100644
--- a/cmd/serve/s3/backend.go
+++ b/cmd/serve/s3/backend.go
@@ -10,6 +10,7 @@ import (
"path"
"strings"
"sync"
+ "sync/atomic"
"time"
"github.com/google/uuid"
@@ -45,6 +46,13 @@ type s3Backend struct {
// keyed by gofakes3.UploadID.
multipartUploads sync.Map
+ // uploads counts the entries in multipartUploads.
+ uploads atomic.Int64
+
+ // warnMaxUploadsOnce logs a single NOTICE the first time a multipart
+ // upload is refused by --multipart-max-uploads.
+ warnMaxUploadsOnce sync.Once
+
// warnInMemoryOnce logs a single NOTICE the first time a multipart
// upload falls back to being buffered in memory.
warnInMemoryOnce sync.Once
diff --git a/cmd/serve/s3/multipart.go b/cmd/serve/s3/multipart.go
index 82500c466..deb2df2de 100644
--- a/cmd/serve/s3/multipart.go
+++ b/cmd/serve/s3/multipart.go
@@ -192,6 +192,40 @@ func (b *s3Backend) loadUpload(uploadID gofakes3.UploadID) (*multipartUpload, er
return v.(*multipartUpload), nil
}
+// reserveUpload reserves a place for a new upload, failing with
+// errSlowDown if --multipart-max-uploads are already in progress. The
+// place is given back by releaseUpload or deleteUpload.
+func (b *s3Backend) reserveUpload() error {
+ n := b.uploads.Add(1)
+ if maxUploads := int64(b.s.opt.MultipartMaxUploads); maxUploads > 0 && n > maxUploads {
+ b.uploads.Add(-1)
+ b.warnMaxUploadsOnce.Do(func() {
+ fs.Logf(nil, "serve s3: telling clients to slow down as --multipart-max-uploads %d multipart uploads are in progress", maxUploads)
+ })
+ return errSlowDown
+ }
+ return nil
+}
+
+// releaseUpload gives back a place reserved for an upload which was never
+// recorded with addUpload.
+func (b *s3Backend) releaseUpload() {
+ b.uploads.Add(-1)
+}
+
+// addUpload records a new in-flight upload, which must hold a place
+// reserved by reserveUpload.
+func (b *s3Backend) addUpload(uploadID gofakes3.UploadID, up *multipartUpload) {
+ b.multipartUploads.Store(uploadID, up)
+}
+
+// deleteUpload removes the record of an in-flight upload, if present.
+func (b *s3Backend) deleteUpload(uploadID gofakes3.UploadID) {
+ if _, ok := b.multipartUploads.LoadAndDelete(uploadID); ok {
+ b.uploads.Add(-1)
+ }
+}
+
// CreateMultipartUpload begins a new multipart upload.
//
// The parts are written, in part-number order, through the VFS to a temporary
@@ -215,6 +249,8 @@ func (b *s3Backend) loadUpload(uploadID gofakes3.UploadID) (*multipartUpload, er
//
// The object the parts are written to is only created when the first part
// arrives, so an upload with no parts holds no VFS or backend resources.
+// No more than --multipart-max-uploads can be in progress: after that
+// errSlowDown is returned before anything is created on the remote.
func (b *s3Backend) CreateMultipartUpload(ctx context.Context, bucketName, objectName string, meta map[string]string) (gofakes3.UploadID, error) {
_vfs, err := b.s.getVFS(ctx)
if err != nil {
@@ -235,9 +271,16 @@ func (b *s3Backend) CreateMultipartUpload(ctx context.Context, bucketName, objec
if err != nil {
return "", err
}
+ // Reserve a place before doing anything with the remote, so a refused
+ // upload leaves nothing behind.
+ if err := b.reserveUpload(); err != nil {
+ return "", err
+ }
+
objectDir := path.Dir(fp)
if objectDir != "." {
if err := mkdirRecursive(objectDir, _vfs); err != nil {
+ b.releaseUpload()
return "", err
}
}
@@ -258,7 +301,7 @@ func (b *s3Backend) CreateMultipartUpload(ctx context.Context, bucketName, objec
up.budget = b.budget
up.vfs = _vfs
- b.multipartUploads.Store(uploadID, up)
+ b.addUpload(uploadID, up)
return uploadID, nil
}
@@ -801,7 +844,7 @@ func (b *s3Backend) CompleteMultipartUpload(ctx context.Context, bucketName, obj
defer up.endActivity()
if err := up.validate(input); err != nil {
- b.multipartUploads.Delete(uploadID)
+ b.deleteUpload(uploadID)
_ = up.abort()
b.discardUpload(up)
return "", "", err
@@ -811,7 +854,7 @@ func (b *s3Backend) CompleteMultipartUpload(ctx context.Context, bucketName, obj
// streamed parts don't form the complete object; abort then tears it
// down. (After a successful or failed commit the abort is a no-op.)
if err := up.close(); err != nil {
- b.multipartUploads.Delete(uploadID)
+ b.deleteUpload(uploadID)
_ = up.abort()
b.discardUpload(up)
return "", "", err
@@ -828,7 +871,7 @@ func (b *s3Backend) CompleteMultipartUpload(ctx context.Context, bucketName, obj
return "", "", err
}
}
- b.multipartUploads.Delete(uploadID)
+ b.deleteUpload(uploadID)
b.meta.Store(up.fp, up.meta)
if val, ok := up.meta["X-Amz-Meta-Mtime"]; ok {
@@ -853,7 +896,7 @@ func (b *s3Backend) AbortMultipartUpload(ctx context.Context, bucketName, object
if err != nil {
return err
}
- defer b.multipartUploads.Delete(uploadID)
+ defer b.deleteUpload(uploadID)
if err := up.abort(); err != nil {
fs.Errorf(up.fp, "aborting multipart upload: %v", err)
}
@@ -866,7 +909,7 @@ func (b *s3Backend) AbortMultipartUpload(ctx context.Context, bucketName, object
// accepted.
func (b *s3Backend) failUpload(uploadID gofakes3.UploadID, up *multipartUpload) {
fs.Errorf(up.fp, "failing multipart upload %s: %v", uploadID, errMultipartPoisoned)
- b.multipartUploads.Delete(uploadID)
+ b.deleteUpload(uploadID)
if err := up.abort(); err != nil {
fs.Errorf(up.fp, "aborting multipart upload: %v", err)
}
@@ -911,7 +954,7 @@ func (b *s3Backend) reapExpiredUploads(now time.Time, expiry time.Duration) {
return true
}
fs.Logf(up.fp, "aborting multipart upload %s idle for more than %v", uploadID, expiry)
- b.multipartUploads.Delete(uploadID)
+ b.deleteUpload(uploadID)
if err := up.abort(); err != nil {
fs.Errorf(up.fp, "aborting abandoned multipart upload: %v", err)
}
diff --git a/cmd/serve/s3/multipart_test.go b/cmd/serve/s3/multipart_test.go
index a6ee18ccd..2a6ddef53 100644
--- a/cmd/serve/s3/multipart_test.go
+++ b/cmd/serve/s3/multipart_test.go
@@ -472,6 +472,13 @@ func TestMultipartBufferLimit(t *testing.T) {
assert.Equal(t, want, readObject(t, f, bucket, object))
}
+// storeTestUpload records up as an in-flight upload of b, as
+// CreateMultipartUpload does.
+func storeTestUpload(t *testing.T, b *s3Backend, uploadID gofakes3.UploadID, up *multipartUpload) {
+ require.NoError(t, b.reserveUpload())
+ b.addUpload(uploadID, up)
+}
+
// stubSink is a multipartUpload sink which records how it was closed.
type stubSink struct {
closed bool
@@ -592,7 +599,7 @@ func TestMultipartAbortAlwaysSucceeds(t *testing.T) {
up.fh = failingSink{}
up.vfs = _vfs
const uploadID = gofakes3.UploadID("failing-close")
- b.multipartUploads.Store(uploadID, up)
+ storeTestUpload(t, b, uploadID, up)
require.NoError(t, b.AbortMultipartUpload(ctx, bucket, "key", uploadID))
@@ -616,7 +623,7 @@ func TestMultipartCompleteRenameFailureKeepsUpload(t *testing.T) {
up.fh = &stubSink{}
up.vfs = _vfs
const uploadID = gofakes3.UploadID("rename-fails")
- b.multipartUploads.Store(uploadID, up)
+ storeTestUpload(t, b, uploadID, up)
_, _, err = b.CompleteMultipartUpload(ctx, bucket, "key", uploadID, &gofakes3.CompleteMultipartUploadRequest{})
require.Error(t, err)
@@ -670,7 +677,7 @@ func TestMultipartReapExpiredUploads(t *testing.T) {
up := newMultipartUpload(bucket, id, bucket+"/"+id, bucket+"/"+multipartUploadPrefix+id, nil, 0)
up.fh = &stubSink{}
up.vfs = _vfs
- b.multipartUploads.Store(gofakes3.UploadID(id), up)
+ storeTestUpload(t, b, gofakes3.UploadID(id), up)
return up
}
@@ -1584,3 +1591,50 @@ func TestMultipartSinkOpenedOnFirstPart(t *testing.T) {
requireOnly(t, f, bucket, "parts.bin", "empty.bin")
}
+
+// TestMultipartMaxUploads checks that no more than --multipart-max-uploads
+// multipart uploads can be in progress at once, and that finished uploads
+// make room for more.
+func TestMultipartMaxUploads(t *testing.T) {
+ b, _, bucket := newPutTestBackend(t, "", nil)
+ b.s.opt.MultipartMaxUploads = 2
+ ctx := context.Background()
+
+ id1, err := b.CreateMultipartUpload(ctx, bucket, "1", nil)
+ require.NoError(t, err)
+ id2, err := b.CreateMultipartUpload(ctx, bucket, "2", nil)
+ require.NoError(t, err)
+ _, err = b.CreateMultipartUpload(ctx, bucket, "3", nil)
+ assert.True(t, gofakes3.HasErrorCode(err, errSlowDown), "want SlowDown, got %v", err)
+
+ // Aborting an upload makes room.
+ require.NoError(t, b.AbortMultipartUpload(ctx, bucket, "1", id1))
+ id3, err := b.CreateMultipartUpload(ctx, bucket, "3", nil)
+ require.NoError(t, err)
+ _, err = b.CreateMultipartUpload(ctx, bucket, "4", nil)
+ assert.True(t, gofakes3.HasErrorCode(err, errSlowDown), "want SlowDown, got %v", err)
+
+ // So does completing one.
+ _, _, err = b.CompleteMultipartUpload(ctx, bucket, "2", id2, &gofakes3.CompleteMultipartUploadRequest{})
+ require.NoError(t, err)
+ _, err = b.CreateMultipartUpload(ctx, bucket, "4", nil)
+ require.NoError(t, err)
+
+ // A refused upload leaves nothing behind on the remote.
+ _, err = b.CreateMultipartUpload(ctx, bucket, "dir/deep/4", nil)
+ assert.True(t, gofakes3.HasErrorCode(err, errSlowDown), "want SlowDown, got %v", err)
+ _vfs, err := b.s.getVFS(ctx)
+ require.NoError(t, err)
+ _, err = _vfs.Stat(bucket + "/dir")
+ assert.ErrorIs(t, err, vfs.ENOENT, "a refused upload created directories")
+
+ // And so does expiring one.
+ up, err := b.loadUpload(id3)
+ require.NoError(t, err)
+ up.mu.Lock()
+ up.lastUsed = time.Now().Add(-2 * time.Hour)
+ up.mu.Unlock()
+ b.reapExpiredUploads(time.Now(), time.Hour)
+ _, err = b.CreateMultipartUpload(ctx, bucket, "5", nil)
+ require.NoError(t, err)
+}
diff --git a/cmd/serve/s3/s3.go b/cmd/serve/s3/s3.go
index 587c6412a..b8586cd31 100644
--- a/cmd/serve/s3/s3.go
+++ b/cmd/serve/s3/s3.go
@@ -49,6 +49,10 @@ var OptionsInfo = fs.Options{{
Name: "multipart_streaming_buffer_total",
Default: fs.SizeSuffix(1024 * 1024 * 1024),
Help: "Maximum memory buffered by all streamed multipart uploads for parts arriving out of order, 0 for unlimited",
+}, {
+ Name: "multipart_max_uploads",
+ Default: 1000,
+ Help: "Maximum number of streamed multipart uploads in progress at once, 0 for unlimited",
}, {
Name: "multipart_expiry",
Default: fs.Duration(24 * time.Hour),
@@ -67,6 +71,7 @@ type Options struct {
DisableMultipartStreaming bool `config:"disable_multipart_streaming"`
MultipartStreamingBufferLimit fs.SizeSuffix `config:"multipart_streaming_buffer_limit"`
MultipartStreamingBufferTotal fs.SizeSuffix `config:"multipart_streaming_buffer_total"`
+ MultipartMaxUploads int `config:"multipart_max_uploads"`
MultipartExpiry fs.Duration `config:"multipart_expiry"`
Auth httplib.AuthConfig
HTTP httplib.Config
diff --git a/cmd/serve/s3/serve_s3.md b/cmd/serve/s3/serve_s3.md
index 11f737544..3ab888ed9 100644
--- a/cmd/serve/s3/serve_s3.md
+++ b/cmd/serve/s3/serve_s3.md
@@ -319,7 +319,17 @@ clock, so the expiry only needs to outlast the client's pauses
Late operations on an expired upload fail with `NoSuchUpload`, as they
do on real S3 when a lifecycle rule has aborted the upload. Set
-`--multipart-expiry 0` to keep incomplete uploads forever.
+`--multipart-expiry 0` to keep incomplete uploads forever - but note
+that abandoned uploads are then never cleaned up and go on counting
+towards `--multipart-max-uploads` below, so enough of them will stop
+any new multipart upload being started until the server is restarted.
+
+Each multipart upload in progress holds server resources - for example
+the buffers the remote's own upload uses once the first part has
+arrived - so no more than `--multipart-max-uploads` (default `1000`,
+`0` for no limit) can be in progress at once. Starting another fails
+with a `SlowDown` error, which S3 clients retry, and a one-off `NOTICE`
+is logged.
#### Disabling streaming
diff --git a/cmd/serve/s3/server.go b/cmd/serve/s3/server.go
index c94e414bc..84a51c9b2 100644
--- a/cmd/serve/s3/server.go
+++ b/cmd/serve/s3/server.go
@@ -88,6 +88,8 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Opt
w.backend = newBackend(w)
if w.opt.MultipartExpiry > 0 {
w.backend.startReaper(time.Duration(w.opt.MultipartExpiry))
+ } else if w.opt.MultipartMaxUploads > 0 {
+ fs.Logf("serve s3", "--multipart-expiry 0 means abandoned multipart uploads are never cleaned up and go on counting towards --multipart-max-uploads %d", w.opt.MultipartMaxUploads)
}
var newLogger logger