Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

serve s3: make conditional writes atomic against other writes through the server

Nick Craig-Woodae473af277b4
diff --git a/cmd/serve/s3/backend.go b/cmd/serve/s3/backend.go
index 5df593707..fa1218e06 100644
--- a/cmd/serve/s3/backend.go
+++ b/cmd/serve/s3/backend.go
@@ -58,6 +58,8 @@ type s3Backend struct {
reaperQuit chan struct{} // closed to stop the abandoned upload reaper
reaperStop sync.Once
+
+ commitLocks pathLocks // serialises moving objects into place, see commitObject
}
// tenant is the state kept between requests for one user of the server.
@@ -629,9 +631,9 @@ func (b *s3Backend) PutObject(
return result, err
}
- // Rename the temporary object into place
+ // Move the temporary object into place
if tmpFp != fp {
- if err := _vfs.Rename(tmpFp, fp); err != nil {
+ if err := b.commitObject(ctx, _vfs, tmpFp, fp); err != nil {
cleanup()
return result, err
}
@@ -645,6 +647,35 @@ func (b *s3Backend) PutObject(
return result, b.tenant(ctx).storeMeta(_vfs, fp, meta)
}
+// commitObject moves the object written to tmpFp into place at fp,
+// first checking the request's conditions on writing it.
+//
+// Moving objects into place through this server is serialised per
+// object, so the check and the move are atomic against other writes
+// through it, though not against changes made some other way.
+func (b *s3Backend) commitObject(ctx context.Context, _vfs *vfs.VFS, tmpFp, fp string) error {
+ unlock := b.commitLocks.lock(fs.ConfigString(_vfs.Fs()) + "\x00" + fp)
+ defer unlock()
+ if gofakes3.HasWriteConditions(ctx) {
+ exists, etag := false, ""
+ if node, err := _vfs.Stat(fp); err == nil && node.IsFile() {
+ exists, etag = true, hex.EncodeToString(b.objectHash(ctx, _vfs, fp, node))
+ }
+ if err := gofakes3.CheckWriteConditions(ctx, exists, etag); err != nil {
+ return err
+ }
+ }
+ return _vfs.Rename(tmpFp, fp)
+}
+
+// objectHash returns the hash returned as the ETag of the object at fp.
+func (b *s3Backend) objectHash(ctx context.Context, _vfs *vfs.VFS, fp string, node vfs.Node) []byte {
+ if om, ok := b.tenant(ctx).getMeta(ctx, fp, node); ok && om.hash != nil {
+ return om.hash
+ }
+ return getFileHashByte(node, b.s.etagHash(_vfs))
+}
+
// DeleteMulti deletes multiple objects in a single request.
func (b *s3Backend) DeleteMulti(ctx context.Context, bucketName string, objects ...string) (result gofakes3.MultiDeleteResult, rerr error) {
for _, object := range objects {
diff --git a/cmd/serve/s3/multipart.go b/cmd/serve/s3/multipart.go
index fb7f1caf5..3a356efcd 100644
--- a/cmd/serve/s3/multipart.go
+++ b/cmd/serve/s3/multipart.go
@@ -920,7 +920,7 @@ func (b *s3Backend) CompleteMultipartUpload(ctx context.Context, bucketName, obj
// so that the client can retry the CompleteMultipartUpload: the
// committed close is idempotent, so the retry just renames again.
if up.streamFp != up.fp {
- if err := up.vfs.Rename(up.streamFp, up.fp); err != nil {
+ if err := b.commitObject(ctx, up.vfs, up.streamFp, up.fp); err != nil {
return "", "", err
}
}
diff --git a/cmd/serve/s3/put_test.go b/cmd/serve/s3/put_test.go
index 6f6fc08c3..1b0c9653b 100644
--- a/cmd/serve/s3/put_test.go
+++ b/cmd/serve/s3/put_test.go
@@ -12,9 +12,11 @@ import (
"fmt"
"io"
"path"
+ "sync"
"testing"
"time"
+ "github.com/minio/minio-go/v7"
"github.com/ncw/swift/v2"
"github.com/rclone/gofakes3"
"github.com/rclone/rclone/cmd/serve/proxy"
@@ -338,3 +340,43 @@ func TestMetadataAfterWriteback(t *testing.T) {
b.forgetPath(_vfs, path.Join(bucket, "key"))
assert.Equal(t, "red", colour(), "after writeback")
}
+
+// TestConditionalPutRace checks that of many concurrent PUTs of a new
+// object with If-None-Match: *, which all find the object missing when
+// they start, only one writes it.
+func TestConditionalPutRace(t *testing.T) {
+ core, f, bucket := newMultipartTestServer(t, false)
+ ctx := context.Background()
+ const object = "race.bin"
+ const n = 20
+ data := []byte(random.String(256 * 1024))
+
+ var wg sync.WaitGroup
+ var start sync.WaitGroup
+ start.Add(1)
+ errs := make([]error, n)
+ for i := range n {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ opts := minio.PutObjectOptions{}
+ opts.SetMatchETagExcept("*")
+ start.Wait()
+ _, errs[i] = core.PutObject(ctx, bucket, object, bytes.NewReader(data), int64(len(data)), "", "", opts)
+ }()
+ }
+ start.Done()
+ wg.Wait()
+
+ succeeded := 0
+ for _, err := range errs {
+ if err == nil {
+ succeeded++
+ } else {
+ assert.Equal(t, "PreconditionFailed", minio.ToErrorResponse(err).Code, "%v", err)
+ }
+ }
+ assert.Equal(t, 1, succeeded)
+ assert.Equal(t, data, readObject(t, f, bucket, object))
+ requireOnly(t, f, bucket, object)
+}
diff --git a/cmd/serve/s3/serve_s3.md b/cmd/serve/s3/serve_s3.md
index ac5cd1a88..6a2fcfe6c 100644
--- a/cmd/serve/s3/serve_s3.md
+++ b/cmd/serve/s3/serve_s3.md
@@ -457,6 +457,17 @@ the prefix walking, unless the delimiter is in the directory names.
Versioning is not currently supported.
+Conditional writes, `PutObject` and `CompleteMultipartUpload` with
+`If-None-Match` or `If-Match`, are supported. The conditions are
+checked before the object is written, and again as it is moved into
+place when it is written to a temporary object first. This happens for
+multipart uploads and, for `PutObject`, with remotes which can upload
+partial files (such as local disk) or with `--vfs-cache-mode minimal`
+or above. The second check is atomic against other writes through
+`serve s3`, but neither is atomic against changes made to the remote
+some other way. If the remote doesn't support the `--etag-hash`,
+objects have no `ETag` so `If-Match` with an `ETag` always fails.
+
Metadata will only be saved in memory other than the rclone `mtime`
metadata which will be set as the modification time of the file. The
metadata of at most `--metadata-max-objects` objects (default `100000`,
diff --git a/cmd/serve/s3/utils.go b/cmd/serve/s3/utils.go
index c00930181..37de2636f 100644
--- a/cmd/serve/s3/utils.go
+++ b/cmd/serve/s3/utils.go
@@ -8,6 +8,7 @@ import (
"os"
"path"
"strings"
+ "sync"
"github.com/rclone/gofakes3"
"github.com/rclone/rclone/fs"
@@ -231,6 +232,43 @@ func rmdirRecursive(p string, VFS *vfs.VFS) {
rmdirRecursive(dir, VFS)
}
+// pathLocks are locks on paths, made as needed.
+type pathLocks struct {
+ mu sync.Mutex
+ locks map[string]*pathLock
+}
+
+// pathLock is a lock on a path wanted by refs users.
+type pathLock struct {
+ mu sync.Mutex
+ refs int
+}
+
+// lock locks path, returning a function to unlock it.
+func (p *pathLocks) lock(path string) (unlock func()) {
+ p.mu.Lock()
+ if p.locks == nil {
+ p.locks = map[string]*pathLock{}
+ }
+ l := p.locks[path]
+ if l == nil {
+ l = &pathLock{}
+ p.locks[path] = l
+ }
+ l.refs++
+ p.mu.Unlock()
+ l.mu.Lock()
+ return func() {
+ l.mu.Unlock()
+ p.mu.Lock()
+ l.refs--
+ if l.refs == 0 {
+ delete(p.locks, path)
+ }
+ p.mu.Unlock()
+ }
+}
+
// errNotEmpty is returned by removeEmptyTree for a directory holding
// files.
var errNotEmpty = errors.New("directory not empty")