"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
serve s3: clean up abandoned multipart uploads after --multipart-expiry
Nick Craig-Wood1947e4217c63
diff --git a/cmd/serve/s3/backend.go b/cmd/serve/s3/backend.goindex 8c135e969..337cff5ea 100644--- a/cmd/serve/s3/backend.go+++ b/cmd/serve/s3/backend.go@@ -48,13 +48,17 @@ type s3Backend struct {// warnInMemoryOnce logs a single NOTICE the first time a multipart// upload falls back to being buffered in memory.warnInMemoryOnce sync.Once++ reaperQuit chan struct{} // closed to stop the abandoned upload reaper+ reaperStop sync.Once}// newBackend creates a new SimpleBucketBackend.-func newBackend(s *Server) gofakes3.Backend {+func newBackend(s *Server) *s3Backend {return &s3Backend{- s: s,- meta: new(sync.Map),+ s: s,+ meta: new(sync.Map),+ reaperQuit: make(chan struct{}),}}diff --git a/cmd/serve/s3/backend_test.go b/cmd/serve/s3/backend_test.goindex dfbddede1..574bbc56c 100644--- a/cmd/serve/s3/backend_test.go+++ b/cmd/serve/s3/backend_test.go@@ -37,7 +37,7 @@ func newTestBackend(t *testing.T) (*s3Backend, string) {w, err := newServer(ctx, f, &opt, &vfscommon.Opt, &proxy.Opt)require.NoError(t, err)- return newBackend(w).(*s3Backend), root+ return newBackend(w), root}const rootSecret = "ROOT_LEVEL_SECRET_MARKER"diff --git a/cmd/serve/s3/multipart.go b/cmd/serve/s3/multipart.goindex 304aaf69f..869127ba0 100644--- a/cmd/serve/s3/multipart.go+++ b/cmd/serve/s3/multipart.go@@ -27,6 +27,7 @@ import ("sort""strings""sync"+ "time""github.com/google/uuid""github.com/ncw/swift/v2"@@ -61,7 +62,9 @@ type multipartUpload struct {partMD5s map[int][]byte // raw MD5 sums per part (for the final S3 multipart ETag)partSizes map[int]int64 // observed part sizesclosed bool- aborted bool // closed by abort, so nothing was committed+ aborted bool // closed by abort, so nothing was committed+ active int // requests in flight for this upload, protecting it from the reaper+ lastUsed time.Time // when the last request for this upload finishednextPart int // next part number to stream (1-based)streamBuf map[int]*pool.RW // parts received ahead of nextPart, awaiting their turn@@ -83,11 +86,27 @@ func newMultipartUpload(bucket, key, fp, streamFp string, meta map[string]stringnextPart: 1,streamBuf: map[int]*pool.RW{},bufferLimit: bufferLimit,+ lastUsed: time.Now(),}up.cond = sync.NewCond(&up.mu)return up}+// startActivity marks the upload as having a request in flight.+func (up *multipartUpload) startActivity() {+ up.mu.Lock()+ up.active+++ up.mu.Unlock()+}++// endActivity marks the request done and restarts the upload's idle time.+func (up *multipartUpload) endActivity() {+ up.mu.Lock()+ up.active--+ up.lastUsed = time.Now()+ up.mu.Unlock()+}+// loadUpload looks up an in-flight upload by ID.func (b *s3Backend) loadUpload(uploadID gofakes3.UploadID) (*multipartUpload, error) {v, ok := b.multipartUploads.Load(uploadID)@@ -174,6 +193,8 @@ func (b *s3Backend) UploadPart(ctx context.Context, bucketName, objectName strinif err != nil {return "", err}+ up.startActivity()+ defer up.endActivity()// Wait until there is room to buffer this part, bounding the memory a// client which uploads faster than the backend drains can consume.@@ -354,6 +375,8 @@ func (b *s3Backend) CompleteMultipartUpload(ctx context.Context, bucketName, objif err != nil {return "", "", err}+ up.startActivity()+ defer up.endActivity()if err := up.validate(input); err != nil {b.multipartUploads.Delete(uploadID)@@ -416,6 +439,53 @@ func (b *s3Backend) AbortMultipartUpload(ctx context.Context, bucketName, objectreturn nil}+// startReaper starts a goroutine which aborts incomplete multipart+// uploads once they have been idle for expiry. Stopped by stopReaper.+func (b *s3Backend) startReaper(expiry time.Duration) {+ interval := min(expiry/2, time.Minute)+ go func() {+ ticker := time.NewTicker(interval)+ defer ticker.Stop()+ for {+ select {+ case <-b.reaperQuit:+ return+ case <-ticker.C:+ b.reapExpiredUploads(time.Now(), expiry)+ }+ }+ }()+}++// stopReaper stops the abandoned upload reaper, if running.+func (b *s3Backend) stopReaper() {+ b.reaperStop.Do(func() {+ close(b.reaperQuit)+ })+}++// reapExpiredUploads aborts and cleans up multipart uploads which have had+// no request activity for longer than expiry.+func (b *s3Backend) reapExpiredUploads(now time.Time, expiry time.Duration) {+ b.multipartUploads.Range(func(key, value any) bool {+ uploadID := key.(gofakes3.UploadID)+ up := value.(*multipartUpload)+ up.mu.Lock()+ expired := up.active == 0 && now.Sub(up.lastUsed) >= expiry+ up.mu.Unlock()+ if !expired {+ return true+ }+ fs.Logf(up.fp, "aborting multipart upload %s idle for more than %v", uploadID, expiry)+ b.multipartUploads.Delete(uploadID)+ if err := up.abort(); err != nil {+ fs.Errorf(up.fp, "aborting abandoned multipart upload: %v", err)+ }+ b.discardUpload(up)+ return true+ })+}+// discardUpload cleans up after a failed or aborted upload, removing the// temporary object and any stale VFS state for it. It never removes the// object at the final path: when the parts were written straight to thediff --git a/cmd/serve/s3/multipart_test.go b/cmd/serve/s3/multipart_test.goindex a0b585b56..acf36317a 100644--- a/cmd/serve/s3/multipart_test.go+++ b/cmd/serve/s3/multipart_test.go@@ -617,6 +617,84 @@ func TestMultipartCompleteRenameFailureKeepsUpload(t *testing.T) {require.NoError(t, err, "the upload record must survive a retryable Complete failure")}+// TestMultipartReaper checks that an incomplete multipart upload abandoned+// by its client is aborted and cleaned up after --multipart-expiry, and that+// late operations on it fail with NoSuchUpload.+func TestMultipartReaper(t *testing.T) {+ core, f, bucket := newMultipartTestServerOpt(t, "", false, func(opt *Options) {+ opt.MultipartExpiry = fs.Duration(100 * time.Millisecond)+ })+ ctx := context.Background()+ const object = "abandoned.bin"++ uploadID, err := core.NewMultipartUpload(ctx, bucket, object, minio.PutObjectOptions{})+ require.NoError(t, err)+ data := []byte(random.String(50 * 1024))+ _, err = core.PutObjectPart(ctx, bucket, object, uploadID, 1, bytes.NewReader(data), int64(len(data)), minio.PutObjectPartOptions{})+ require.NoError(t, err)++ // Wait for well over the expiry and the reaper interval, then the+ // upload must be gone.+ time.Sleep(time.Second)+ _, err = core.PutObjectPart(ctx, bucket, object, uploadID, 2, bytes.NewReader(data), int64(len(data)), minio.PutObjectPartOptions{})+ require.Error(t, err)+ assert.Contains(t, err.Error(), "NoSuchUpload")+ err = core.AbortMultipartUpload(ctx, bucket, object, uploadID)+ require.Error(t, err)+ assert.Contains(t, err.Error(), "NoSuchUpload")++ // Nothing is left at the key or as a temporary object.+ _, err = f.NewObject(ctx, path.Join(bucket, object))+ require.ErrorIs(t, err, fs.ErrorObjectNotFound)+ requireOnly(t, f, bucket)+}++// TestMultipartReapExpiredUploads checks the reaper's rules directly: an+// idle upload past the expiry is aborted and cleaned up, one with a request+// in flight is left alone however stale its idle time, and a fresh one is+// kept.+func TestMultipartReapExpiredUploads(t *testing.T) {+ b, _, bucket := newPutTestBackend(t, "", nil)+ ctx := context.Background()+ _vfs, err := b.s.getVFS(ctx)+ require.NoError(t, err)++ newUp := func(id string) *multipartUpload {+ up := newMultipartUpload(bucket, id, bucket+"/"+id, bucket+"/"+multipartUploadPrefix+id, nil, 0)+ up.fh = &stubSink{}+ up.vfs = _vfs+ b.multipartUploads.Store(gofakes3.UploadID(id), up)+ return up+ }++ const expiry = time.Hour+ now := time.Now()++ newUp("fresh")+ idle := newUp("idle")+ busy := newUp("busy")+ busy.startActivity()+ for _, up := range []*multipartUpload{idle, busy} {+ up.mu.Lock()+ up.lastUsed = now.Add(-2 * expiry)+ up.mu.Unlock()+ }++ b.reapExpiredUploads(now, expiry)++ _, err = b.loadUpload("fresh")+ assert.NoError(t, err, "a fresh upload must not be reaped")+ _, err = b.loadUpload("busy")+ assert.NoError(t, err, "an upload with a request in flight must not be reaped")+ _, err = b.loadUpload("idle")+ assert.ErrorIs(t, err, gofakes3.ErrNoSuchUpload, "an idle upload past the expiry must be reaped")++ // The reaped upload was aborted, not committed.+ sink := idle.fh.(*stubSink)+ assert.True(t, sink.closed)+ assert.Equal(t, errMultipartAborted, sink.abortErr)+}+// cacheWritesVFSOpt returns VFS options with --vfs-cache-mode writes and the// given write-back delay.func cacheWritesVFSOpt(writeBack time.Duration) *vfscommon.Options {diff --git a/cmd/serve/s3/put_test.go b/cmd/serve/s3/put_test.goindex 88998b1b3..242d028b5 100644--- a/cmd/serve/s3/put_test.go+++ b/cmd/serve/s3/put_test.go@@ -69,7 +69,7 @@ func newPutTestBackend(t *testing.T, backing string, vfsOpt *vfscommon.Options)w, err := newServer(ctx, f, &opt, vfsOpt, &proxy.Opt)require.NoError(t, err)t.Cleanup(func() { _ = w.Shutdown() })- return newBackend(w).(*s3Backend), f, bucket+ return newBackend(w), f, bucket}var errBoom = errors.New("boom")diff --git a/cmd/serve/s3/s3.go b/cmd/serve/s3/s3.goindex 1b0c92e61..94cb75ca5 100644--- a/cmd/serve/s3/s3.go+++ b/cmd/serve/s3/s3.go@@ -4,6 +4,7 @@ import ("context"_ "embed""strings"+ "time""github.com/rclone/rclone/cmd""github.com/rclone/rclone/cmd/serve"@@ -39,11 +40,15 @@ var OptionsInfo = fs.Options{{}, {Name: "disable_multipart_streaming",Default: false,- Help: "Buffer multipart uploads in memory instead of streaming them to the backend (see the Multipart uploads docs section)",+ Help: "Buffer multipart uploads in memory instead of streaming them to the backend",}, {Name: "multipart_streaming_buffer_limit",Default: fs.SizeSuffix(256 * 1024 * 1024),- Help: "Maximum memory buffered per streamed multipart upload for parts arriving out of order, 0 for unlimited (see the Multipart uploads docs section)",+ Help: "Maximum memory buffered per streamed multipart upload for parts arriving out of order, 0 for unlimited",+}, {+ Name: "multipart_expiry",+ Default: fs.Duration(24 * time.Hour),+ Help: "Abort incomplete multipart uploads idle for longer than this, 0 to keep forever",}}.Add(httplib.ConfigInfo).Add(httplib.AuthConfigInfo)@@ -57,6 +62,7 @@ type Options struct {NoCleanup bool `config:"no_cleanup"`DisableMultipartStreaming bool `config:"disable_multipart_streaming"`MultipartStreamingBufferLimit fs.SizeSuffix `config:"multipart_streaming_buffer_limit"`+ MultipartExpiry fs.Duration `config:"multipart_expiry"`Auth httplib.AuthConfigHTTP httplib.Config}diff --git a/cmd/serve/s3/serve_s3.md b/cmd/serve/s3/serve_s3.mdindex 23e89ec5d..4ba573b88 100644--- a/cmd/serve/s3/serve_s3.md+++ b/cmd/serve/s3/serve_s3.md@@ -283,6 +283,26 @@ rclone v1.75 named its temporary multipart objects`.rclone_multipart_upload_*`; leftovers from an older server are alsohidden from listings and can be cleaned up the same way.+#### Abandoned uploads++A client which starts a multipart upload and vanishes without either+completing or aborting it would otherwise hold on to its resources+forever.++An incomplete multipart upload which has had no activity for+`--multipart-expiry` (default `24h`) is therefore aborted and cleaned+up, exactly as if the client had called `AbortMultipartUpload`, and a+`NOTICE` is logged.++An upload with a part still being received is never expired, however+slowly the part is arriving, and each completed part restarts the+clock, so the expiry only needs to outlast the client's pauses+*between* parts, not the whole upload.++Late operations on an expired upload fail with `NoSuchUpload`, as they+do on real S3 when a lifecycle rule has aborted the upload. Set+`--multipart-expiry 0` to keep incomplete uploads forever.+#### Disabling streamingIf you pass `--disable-multipart-streaming`, multipart uploads arediff --git a/cmd/serve/s3/server.go b/cmd/serve/s3/server.goindex 260508a00..9043d7716 100644--- a/cmd/serve/s3/server.go+++ b/cmd/serve/s3/server.go@@ -11,6 +11,7 @@ import ("net""net/http""strings"+ "time""github.com/go-chi/chi/v5""github.com/rclone/gofakes3"@@ -36,6 +37,7 @@ type Server struct {f fs.Fs_vfs *vfs.VFS // don't use directly, use getVFSfaker *gofakes3.GoFakeS3+ backend *s3Backendhandler http.Handlerproxy *proxy.Proxyctx context.Context // for global config@@ -75,9 +77,14 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Optreturn nil, fmt.Errorf("parsing auth list failed: %q", err)}+ w.backend = newBackend(w)+ if w.opt.MultipartExpiry > 0 {+ w.backend.startReaper(time.Duration(w.opt.MultipartExpiry))+ }+var newLogger loggerw.faker = gofakes3.New(- newBackend(w),+ w.backend,gofakes3.WithHostBucket(!opt.ForcePathStyle),gofakes3.WithLogger(newLogger),gofakes3.WithRequestID(rand.Uint64()),@@ -161,6 +168,7 @@ func (w *Server) Addr() net.Addr {// Shutdown the serverfunc (w *Server) Shutdown() error {+ w.backend.stopReaper()return w.server.Shutdown()}