"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
serve s3: add --multipart-max-uploads to limit uploads in progress
Nick Craig-Woodd24568e3e640
diff --git a/cmd/serve/s3/backend.go b/cmd/serve/s3/backend.goindex 61375e29e..dbb30265f 100644--- a/cmd/serve/s3/backend.go+++ b/cmd/serve/s3/backend.go@@ -10,6 +10,7 @@ import ("path""strings""sync"+ "sync/atomic""time""github.com/google/uuid"@@ -45,6 +46,13 @@ type s3Backend struct {// keyed by gofakes3.UploadID.multipartUploads sync.Map+ // uploads counts the entries in multipartUploads.+ uploads atomic.Int64++ // warnMaxUploadsOnce logs a single NOTICE the first time a multipart+ // upload is refused by --multipart-max-uploads.+ warnMaxUploadsOnce sync.Once+// warnInMemoryOnce logs a single NOTICE the first time a multipart// upload falls back to being buffered in memory.warnInMemoryOnce sync.Oncediff --git a/cmd/serve/s3/multipart.go b/cmd/serve/s3/multipart.goindex 82500c466..deb2df2de 100644--- a/cmd/serve/s3/multipart.go+++ b/cmd/serve/s3/multipart.go@@ -192,6 +192,40 @@ func (b *s3Backend) loadUpload(uploadID gofakes3.UploadID) (*multipartUpload, erreturn v.(*multipartUpload), nil}+// reserveUpload reserves a place for a new upload, failing with+// errSlowDown if --multipart-max-uploads are already in progress. The+// place is given back by releaseUpload or deleteUpload.+func (b *s3Backend) reserveUpload() error {+ n := b.uploads.Add(1)+ if maxUploads := int64(b.s.opt.MultipartMaxUploads); maxUploads > 0 && n > maxUploads {+ b.uploads.Add(-1)+ b.warnMaxUploadsOnce.Do(func() {+ fs.Logf(nil, "serve s3: telling clients to slow down as --multipart-max-uploads %d multipart uploads are in progress", maxUploads)+ })+ return errSlowDown+ }+ return nil+}++// releaseUpload gives back a place reserved for an upload which was never+// recorded with addUpload.+func (b *s3Backend) releaseUpload() {+ b.uploads.Add(-1)+}++// addUpload records a new in-flight upload, which must hold a place+// reserved by reserveUpload.+func (b *s3Backend) addUpload(uploadID gofakes3.UploadID, up *multipartUpload) {+ b.multipartUploads.Store(uploadID, up)+}++// deleteUpload removes the record of an in-flight upload, if present.+func (b *s3Backend) deleteUpload(uploadID gofakes3.UploadID) {+ if _, ok := b.multipartUploads.LoadAndDelete(uploadID); ok {+ b.uploads.Add(-1)+ }+}+// CreateMultipartUpload begins a new multipart upload.//// The parts are written, in part-number order, through the VFS to a temporary@@ -215,6 +249,8 @@ func (b *s3Backend) loadUpload(uploadID gofakes3.UploadID) (*multipartUpload, er//// The object the parts are written to is only created when the first part// arrives, so an upload with no parts holds no VFS or backend resources.+// No more than --multipart-max-uploads can be in progress: after that+// errSlowDown is returned before anything is created on the remote.func (b *s3Backend) CreateMultipartUpload(ctx context.Context, bucketName, objectName string, meta map[string]string) (gofakes3.UploadID, error) {_vfs, err := b.s.getVFS(ctx)if err != nil {@@ -235,9 +271,16 @@ func (b *s3Backend) CreateMultipartUpload(ctx context.Context, bucketName, objecif err != nil {return "", err}+ // Reserve a place before doing anything with the remote, so a refused+ // upload leaves nothing behind.+ if err := b.reserveUpload(); err != nil {+ return "", err+ }+objectDir := path.Dir(fp)if objectDir != "." {if err := mkdirRecursive(objectDir, _vfs); err != nil {+ b.releaseUpload()return "", err}}@@ -258,7 +301,7 @@ func (b *s3Backend) CreateMultipartUpload(ctx context.Context, bucketName, objecup.budget = b.budgetup.vfs = _vfs- b.multipartUploads.Store(uploadID, up)+ b.addUpload(uploadID, up)return uploadID, nil}@@ -801,7 +844,7 @@ func (b *s3Backend) CompleteMultipartUpload(ctx context.Context, bucketName, objdefer up.endActivity()if err := up.validate(input); err != nil {- b.multipartUploads.Delete(uploadID)+ b.deleteUpload(uploadID)_ = up.abort()b.discardUpload(up)return "", "", err@@ -811,7 +854,7 @@ func (b *s3Backend) CompleteMultipartUpload(ctx context.Context, bucketName, obj// streamed parts don't form the complete object; abort then tears it// down. (After a successful or failed commit the abort is a no-op.)if err := up.close(); err != nil {- b.multipartUploads.Delete(uploadID)+ b.deleteUpload(uploadID)_ = up.abort()b.discardUpload(up)return "", "", err@@ -828,7 +871,7 @@ func (b *s3Backend) CompleteMultipartUpload(ctx context.Context, bucketName, objreturn "", "", err}}- b.multipartUploads.Delete(uploadID)+ b.deleteUpload(uploadID)b.meta.Store(up.fp, up.meta)if val, ok := up.meta["X-Amz-Meta-Mtime"]; ok {@@ -853,7 +896,7 @@ func (b *s3Backend) AbortMultipartUpload(ctx context.Context, bucketName, objectif err != nil {return err}- defer b.multipartUploads.Delete(uploadID)+ defer b.deleteUpload(uploadID)if err := up.abort(); err != nil {fs.Errorf(up.fp, "aborting multipart upload: %v", err)}@@ -866,7 +909,7 @@ func (b *s3Backend) AbortMultipartUpload(ctx context.Context, bucketName, object// accepted.func (b *s3Backend) failUpload(uploadID gofakes3.UploadID, up *multipartUpload) {fs.Errorf(up.fp, "failing multipart upload %s: %v", uploadID, errMultipartPoisoned)- b.multipartUploads.Delete(uploadID)+ b.deleteUpload(uploadID)if err := up.abort(); err != nil {fs.Errorf(up.fp, "aborting multipart upload: %v", err)}@@ -911,7 +954,7 @@ func (b *s3Backend) reapExpiredUploads(now time.Time, expiry time.Duration) {return true}fs.Logf(up.fp, "aborting multipart upload %s idle for more than %v", uploadID, expiry)- b.multipartUploads.Delete(uploadID)+ b.deleteUpload(uploadID)if err := up.abort(); err != nil {fs.Errorf(up.fp, "aborting abandoned multipart upload: %v", err)}diff --git a/cmd/serve/s3/multipart_test.go b/cmd/serve/s3/multipart_test.goindex a6ee18ccd..2a6ddef53 100644--- a/cmd/serve/s3/multipart_test.go+++ b/cmd/serve/s3/multipart_test.go@@ -472,6 +472,13 @@ func TestMultipartBufferLimit(t *testing.T) {assert.Equal(t, want, readObject(t, f, bucket, object))}+// storeTestUpload records up as an in-flight upload of b, as+// CreateMultipartUpload does.+func storeTestUpload(t *testing.T, b *s3Backend, uploadID gofakes3.UploadID, up *multipartUpload) {+ require.NoError(t, b.reserveUpload())+ b.addUpload(uploadID, up)+}+// stubSink is a multipartUpload sink which records how it was closed.type stubSink struct {closed bool@@ -592,7 +599,7 @@ func TestMultipartAbortAlwaysSucceeds(t *testing.T) {up.fh = failingSink{}up.vfs = _vfsconst uploadID = gofakes3.UploadID("failing-close")- b.multipartUploads.Store(uploadID, up)+ storeTestUpload(t, b, uploadID, up)require.NoError(t, b.AbortMultipartUpload(ctx, bucket, "key", uploadID))@@ -616,7 +623,7 @@ func TestMultipartCompleteRenameFailureKeepsUpload(t *testing.T) {up.fh = &stubSink{}up.vfs = _vfsconst uploadID = gofakes3.UploadID("rename-fails")- b.multipartUploads.Store(uploadID, up)+ storeTestUpload(t, b, uploadID, up)_, _, err = b.CompleteMultipartUpload(ctx, bucket, "key", uploadID, &gofakes3.CompleteMultipartUploadRequest{})require.Error(t, err)@@ -670,7 +677,7 @@ func TestMultipartReapExpiredUploads(t *testing.T) {up := newMultipartUpload(bucket, id, bucket+"/"+id, bucket+"/"+multipartUploadPrefix+id, nil, 0)up.fh = &stubSink{}up.vfs = _vfs- b.multipartUploads.Store(gofakes3.UploadID(id), up)+ storeTestUpload(t, b, gofakes3.UploadID(id), up)return up}@@ -1584,3 +1591,50 @@ func TestMultipartSinkOpenedOnFirstPart(t *testing.T) {requireOnly(t, f, bucket, "parts.bin", "empty.bin")}++// TestMultipartMaxUploads checks that no more than --multipart-max-uploads+// multipart uploads can be in progress at once, and that finished uploads+// make room for more.+func TestMultipartMaxUploads(t *testing.T) {+ b, _, bucket := newPutTestBackend(t, "", nil)+ b.s.opt.MultipartMaxUploads = 2+ ctx := context.Background()++ id1, err := b.CreateMultipartUpload(ctx, bucket, "1", nil)+ require.NoError(t, err)+ id2, err := b.CreateMultipartUpload(ctx, bucket, "2", nil)+ require.NoError(t, err)+ _, err = b.CreateMultipartUpload(ctx, bucket, "3", nil)+ assert.True(t, gofakes3.HasErrorCode(err, errSlowDown), "want SlowDown, got %v", err)++ // Aborting an upload makes room.+ require.NoError(t, b.AbortMultipartUpload(ctx, bucket, "1", id1))+ id3, err := b.CreateMultipartUpload(ctx, bucket, "3", nil)+ require.NoError(t, err)+ _, err = b.CreateMultipartUpload(ctx, bucket, "4", nil)+ assert.True(t, gofakes3.HasErrorCode(err, errSlowDown), "want SlowDown, got %v", err)++ // So does completing one.+ _, _, err = b.CompleteMultipartUpload(ctx, bucket, "2", id2, &gofakes3.CompleteMultipartUploadRequest{})+ require.NoError(t, err)+ _, err = b.CreateMultipartUpload(ctx, bucket, "4", nil)+ require.NoError(t, err)++ // A refused upload leaves nothing behind on the remote.+ _, err = b.CreateMultipartUpload(ctx, bucket, "dir/deep/4", nil)+ assert.True(t, gofakes3.HasErrorCode(err, errSlowDown), "want SlowDown, got %v", err)+ _vfs, err := b.s.getVFS(ctx)+ require.NoError(t, err)+ _, err = _vfs.Stat(bucket + "/dir")+ assert.ErrorIs(t, err, vfs.ENOENT, "a refused upload created directories")++ // And so does expiring one.+ up, err := b.loadUpload(id3)+ require.NoError(t, err)+ up.mu.Lock()+ up.lastUsed = time.Now().Add(-2 * time.Hour)+ up.mu.Unlock()+ b.reapExpiredUploads(time.Now(), time.Hour)+ _, err = b.CreateMultipartUpload(ctx, bucket, "5", nil)+ require.NoError(t, err)+}diff --git a/cmd/serve/s3/s3.go b/cmd/serve/s3/s3.goindex 587c6412a..b8586cd31 100644--- a/cmd/serve/s3/s3.go+++ b/cmd/serve/s3/s3.go@@ -49,6 +49,10 @@ var OptionsInfo = fs.Options{{Name: "multipart_streaming_buffer_total",Default: fs.SizeSuffix(1024 * 1024 * 1024),Help: "Maximum memory buffered by all streamed multipart uploads for parts arriving out of order, 0 for unlimited",+}, {+ Name: "multipart_max_uploads",+ Default: 1000,+ Help: "Maximum number of streamed multipart uploads in progress at once, 0 for unlimited",}, {Name: "multipart_expiry",Default: fs.Duration(24 * time.Hour),@@ -67,6 +71,7 @@ type Options struct {DisableMultipartStreaming bool `config:"disable_multipart_streaming"`MultipartStreamingBufferLimit fs.SizeSuffix `config:"multipart_streaming_buffer_limit"`MultipartStreamingBufferTotal fs.SizeSuffix `config:"multipart_streaming_buffer_total"`+ MultipartMaxUploads int `config:"multipart_max_uploads"`MultipartExpiry fs.Duration `config:"multipart_expiry"`Auth httplib.AuthConfigHTTP httplib.Configdiff --git a/cmd/serve/s3/serve_s3.md b/cmd/serve/s3/serve_s3.mdindex 11f737544..3ab888ed9 100644--- a/cmd/serve/s3/serve_s3.md+++ b/cmd/serve/s3/serve_s3.md@@ -319,7 +319,17 @@ clock, so the expiry only needs to outlast the client's pausesLate operations on an expired upload fail with `NoSuchUpload`, as theydo on real S3 when a lifecycle rule has aborted the upload. Set-`--multipart-expiry 0` to keep incomplete uploads forever.+`--multipart-expiry 0` to keep incomplete uploads forever - but note+that abandoned uploads are then never cleaned up and go on counting+towards `--multipart-max-uploads` below, so enough of them will stop+any new multipart upload being started until the server is restarted.++Each multipart upload in progress holds server resources - for example+the buffers the remote's own upload uses once the first part has+arrived - so no more than `--multipart-max-uploads` (default `1000`,+`0` for no limit) can be in progress at once. Starting another fails+with a `SlowDown` error, which S3 clients retry, and a one-off `NOTICE`+is logged.#### Disabling streamingdiff --git a/cmd/serve/s3/server.go b/cmd/serve/s3/server.goindex c94e414bc..84a51c9b2 100644--- a/cmd/serve/s3/server.go+++ b/cmd/serve/s3/server.go@@ -88,6 +88,8 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Optw.backend = newBackend(w)if w.opt.MultipartExpiry > 0 {w.backend.startReaper(time.Duration(w.opt.MultipartExpiry))+ } else if w.opt.MultipartMaxUploads > 0 {+ fs.Logf("serve s3", "--multipart-expiry 0 means abandoned multipart uploads are never cleaned up and go on counting towards --multipart-max-uploads %d", w.opt.MultipartMaxUploads)}var newLogger logger