"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
serve s3: make conditional writes atomic against other writes through the server
Nick Craig-Woodae473af277b4
diff --git a/cmd/serve/s3/backend.go b/cmd/serve/s3/backend.goindex 5df593707..fa1218e06 100644--- a/cmd/serve/s3/backend.go+++ b/cmd/serve/s3/backend.go@@ -58,6 +58,8 @@ type s3Backend struct {reaperQuit chan struct{} // closed to stop the abandoned upload reaperreaperStop sync.Once++ commitLocks pathLocks // serialises moving objects into place, see commitObject}// tenant is the state kept between requests for one user of the server.@@ -629,9 +631,9 @@ func (b *s3Backend) PutObject(return result, err}- // Rename the temporary object into place+ // Move the temporary object into placeif tmpFp != fp {- if err := _vfs.Rename(tmpFp, fp); err != nil {+ if err := b.commitObject(ctx, _vfs, tmpFp, fp); err != nil {cleanup()return result, err}@@ -645,6 +647,35 @@ func (b *s3Backend) PutObject(return result, b.tenant(ctx).storeMeta(_vfs, fp, meta)}+// commitObject moves the object written to tmpFp into place at fp,+// first checking the request's conditions on writing it.+//+// Moving objects into place through this server is serialised per+// object, so the check and the move are atomic against other writes+// through it, though not against changes made some other way.+func (b *s3Backend) commitObject(ctx context.Context, _vfs *vfs.VFS, tmpFp, fp string) error {+ unlock := b.commitLocks.lock(fs.ConfigString(_vfs.Fs()) + "\x00" + fp)+ defer unlock()+ if gofakes3.HasWriteConditions(ctx) {+ exists, etag := false, ""+ if node, err := _vfs.Stat(fp); err == nil && node.IsFile() {+ exists, etag = true, hex.EncodeToString(b.objectHash(ctx, _vfs, fp, node))+ }+ if err := gofakes3.CheckWriteConditions(ctx, exists, etag); err != nil {+ return err+ }+ }+ return _vfs.Rename(tmpFp, fp)+}++// objectHash returns the hash returned as the ETag of the object at fp.+func (b *s3Backend) objectHash(ctx context.Context, _vfs *vfs.VFS, fp string, node vfs.Node) []byte {+ if om, ok := b.tenant(ctx).getMeta(ctx, fp, node); ok && om.hash != nil {+ return om.hash+ }+ return getFileHashByte(node, b.s.etagHash(_vfs))+}+// DeleteMulti deletes multiple objects in a single request.func (b *s3Backend) DeleteMulti(ctx context.Context, bucketName string, objects ...string) (result gofakes3.MultiDeleteResult, rerr error) {for _, object := range objects {diff --git a/cmd/serve/s3/multipart.go b/cmd/serve/s3/multipart.goindex fb7f1caf5..3a356efcd 100644--- a/cmd/serve/s3/multipart.go+++ b/cmd/serve/s3/multipart.go@@ -920,7 +920,7 @@ func (b *s3Backend) CompleteMultipartUpload(ctx context.Context, bucketName, obj// so that the client can retry the CompleteMultipartUpload: the// committed close is idempotent, so the retry just renames again.if up.streamFp != up.fp {- if err := up.vfs.Rename(up.streamFp, up.fp); err != nil {+ if err := b.commitObject(ctx, up.vfs, up.streamFp, up.fp); err != nil {return "", "", err}}diff --git a/cmd/serve/s3/put_test.go b/cmd/serve/s3/put_test.goindex 6f6fc08c3..1b0c9653b 100644--- a/cmd/serve/s3/put_test.go+++ b/cmd/serve/s3/put_test.go@@ -12,9 +12,11 @@ import ("fmt""io""path"+ "sync""testing""time"+ "github.com/minio/minio-go/v7""github.com/ncw/swift/v2""github.com/rclone/gofakes3""github.com/rclone/rclone/cmd/serve/proxy"@@ -338,3 +340,43 @@ func TestMetadataAfterWriteback(t *testing.T) {b.forgetPath(_vfs, path.Join(bucket, "key"))assert.Equal(t, "red", colour(), "after writeback")}++// TestConditionalPutRace checks that of many concurrent PUTs of a new+// object with If-None-Match: *, which all find the object missing when+// they start, only one writes it.+func TestConditionalPutRace(t *testing.T) {+ core, f, bucket := newMultipartTestServer(t, false)+ ctx := context.Background()+ const object = "race.bin"+ const n = 20+ data := []byte(random.String(256 * 1024))++ var wg sync.WaitGroup+ var start sync.WaitGroup+ start.Add(1)+ errs := make([]error, n)+ for i := range n {+ wg.Add(1)+ go func() {+ defer wg.Done()+ opts := minio.PutObjectOptions{}+ opts.SetMatchETagExcept("*")+ start.Wait()+ _, errs[i] = core.PutObject(ctx, bucket, object, bytes.NewReader(data), int64(len(data)), "", "", opts)+ }()+ }+ start.Done()+ wg.Wait()++ succeeded := 0+ for _, err := range errs {+ if err == nil {+ succeeded+++ } else {+ assert.Equal(t, "PreconditionFailed", minio.ToErrorResponse(err).Code, "%v", err)+ }+ }+ assert.Equal(t, 1, succeeded)+ assert.Equal(t, data, readObject(t, f, bucket, object))+ requireOnly(t, f, bucket, object)+}diff --git a/cmd/serve/s3/serve_s3.md b/cmd/serve/s3/serve_s3.mdindex ac5cd1a88..6a2fcfe6c 100644--- a/cmd/serve/s3/serve_s3.md+++ b/cmd/serve/s3/serve_s3.md@@ -457,6 +457,17 @@ the prefix walking, unless the delimiter is in the directory names.Versioning is not currently supported.+Conditional writes, `PutObject` and `CompleteMultipartUpload` with+`If-None-Match` or `If-Match`, are supported. The conditions are+checked before the object is written, and again as it is moved into+place when it is written to a temporary object first. This happens for+multipart uploads and, for `PutObject`, with remotes which can upload+partial files (such as local disk) or with `--vfs-cache-mode minimal`+or above. The second check is atomic against other writes through+`serve s3`, but neither is atomic against changes made to the remote+some other way. If the remote doesn't support the `--etag-hash`,+objects have no `ETag` so `If-Match` with an `ETag` always fails.+Metadata will only be saved in memory other than the rclone `mtime`metadata which will be set as the modification time of the file. Themetadata of at most `--metadata-max-objects` objects (default `100000`,diff --git a/cmd/serve/s3/utils.go b/cmd/serve/s3/utils.goindex c00930181..37de2636f 100644--- a/cmd/serve/s3/utils.go+++ b/cmd/serve/s3/utils.go@@ -8,6 +8,7 @@ import ("os""path""strings"+ "sync""github.com/rclone/gofakes3""github.com/rclone/rclone/fs"@@ -231,6 +232,43 @@ func rmdirRecursive(p string, VFS *vfs.VFS) {rmdirRecursive(dir, VFS)}+// pathLocks are locks on paths, made as needed.+type pathLocks struct {+ mu sync.Mutex+ locks map[string]*pathLock+}++// pathLock is a lock on a path wanted by refs users.+type pathLock struct {+ mu sync.Mutex+ refs int+}++// lock locks path, returning a function to unlock it.+func (p *pathLocks) lock(path string) (unlock func()) {+ p.mu.Lock()+ if p.locks == nil {+ p.locks = map[string]*pathLock{}+ }+ l := p.locks[path]+ if l == nil {+ l = &pathLock{}+ p.locks[path] = l+ }+ l.refs+++ p.mu.Unlock()+ l.mu.Lock()+ return func() {+ l.mu.Unlock()+ p.mu.Lock()+ l.refs--+ if l.refs == 0 {+ delete(p.locks, path)+ }+ p.mu.Unlock()+ }+}+// errNotEmpty is returned by removeEmptyTree for a directory holding// files.var errNotEmpty = errors.New("directory not empty")