"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
serve s3: fix stale metadata being returned for objects changed elsewhere
Nick Craig-Wood221398e2d051
diff --git a/cmd/serve/s3/backend.go b/cmd/serve/s3/backend.goindex 58a9136bf..14090ed44 100644--- a/cmd/serve/s3/backend.go+++ b/cmd/serve/s3/backend.go@@ -43,7 +43,7 @@ const putObjectPrefix = tempObjectPrefix + "put_"// PutStream, instead of being buffered in memory by gofakes3.type s3Backend struct {s *Server- meta *lru.Cache[metaKey, map[string]string] // object metadata of all tenants+ meta *lru.Cache[metaKey, objectMeta] // object metadata of all tenantstenantsMu sync.Mutextenants map[string]*tenant // per-user state by tenant ID@@ -86,14 +86,22 @@ type metaKey struct {fp string // object path}+// objectMeta is the metadata stored for an object, with the size and+// modification time the object had when it was stored.+type objectMeta struct {+ meta map[string]string+ size int64+ modTime time.Time+}+// newMetadataStore makes a store for the metadata of up to maxObjects// objects, forgetting that of the least recently used after that. If// maxObjects <= 0 there is no limit.-func newMetadataStore(maxObjects int) *lru.Cache[metaKey, map[string]string] {+func newMetadataStore(maxObjects int) *lru.Cache[metaKey, objectMeta] {if maxObjects <= 0 {maxObjects = math.MaxInt}- meta, err := lru.New[metaKey, map[string]string](maxObjects)+ meta, err := lru.New[metaKey, objectMeta](maxObjects)if err != nil {panic(err) // only fails if maxObjects <= 0}@@ -136,14 +144,56 @@ func (t *tenant) context() context.Context {return context.WithValue(context.Background(), ctxKeyAccessKeyID, t.id)}-// getMeta returns the stored metadata of the object at fp.-func (t *tenant) getMeta(fp string) (meta map[string]string, ok bool) {- return t.b.meta.Get(metaKey{tenant: t.id, fp: fp})+// getMeta returns the stored metadata of node, the object at fp.+//+// Nothing is returned if the object has changed since the metadata was+// stored, as it has been replaced by something other than this user.+func (t *tenant) getMeta(ctx context.Context, fp string, node vfs.Node) (meta map[string]string, ok bool) {+ om, ok := t.b.meta.Get(metaKey{tenant: t.id, fp: fp})+ if !ok || om.size != node.Size() {+ return nil, false+ }+ window := fs.GetModifyWindow(ctx, node.VFS().Fs())+ if window != fs.ModTimeNotSupported {+ if dt := node.ModTime().Sub(om.modTime); dt < -window || dt > window {+ return nil, false+ }+ }+ return om.meta, true}-// setMeta stores meta as the metadata of the object at fp.-func (t *tenant) setMeta(fp string, meta map[string]string) {- t.b.meta.Add(metaKey{tenant: t.id, fp: fp}, meta)+// storeMeta stores meta as the metadata of the object at fp in _vfs.+//+// If meta has a modification time in "X-Amz-Meta-Mtime" or "mtime" the+// object's modification time is set to it, returning any error, and+// both are set to it in meta.+func (t *tenant) storeMeta(_vfs *vfs.VFS, fp string, meta map[string]string) (err error) {+ for _, key := range []string{"X-Amz-Meta-Mtime", "mtime"} {+ val, ok := meta[key]+ if !ok {+ continue+ }+ ti, parseErr := swift.FloatStringToTime(val)+ if parseErr != nil {+ continue+ }+ meta["X-Amz-Meta-Mtime"] = val+ meta["mtime"] = val+ err = _vfs.Chtimes(fp, ti, ti)+ break+ }+ // Stat after setting the modification time as getMeta compares with it+ node, statErr := _vfs.Stat(fp)+ if statErr != nil {+ t.removeMeta(fp)+ return err+ }+ t.b.meta.Add(metaKey{tenant: t.id, fp: fp}, objectMeta{+ meta: meta,+ size: node.Size(),+ modTime: node.ModTime(),+ })+ return err}// removeMeta forgets the metadata of the object at fp.@@ -264,7 +314,7 @@ func (b *s3Backend) HeadObject(ctx context.Context, bucketName, objectName strin"Content-Type": mimeType,}- if metaMap, ok := b.tenant(ctx).getMeta(fp); ok {+ if metaMap, ok := b.tenant(ctx).getMeta(ctx, fp, node); ok {maps.Copy(meta, metaMap)}@@ -344,7 +394,7 @@ func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string"Content-Type": mimeType,}- if metaMap, ok := b.tenant(ctx).getMeta(fp); ok {+ if metaMap, ok := b.tenant(ctx).getMeta(ctx, fp, node); ok {maps.Copy(meta, metaMap)}@@ -358,14 +408,6 @@ func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string}, nil}-// storeModtime sets both "mtime" and "X-Amz-Meta-Mtime" to val in the-// stored metadata. Call this whenever modtime is updated.-func (t *tenant) storeModtime(fp string, meta map[string]string, val string) {- meta["X-Amz-Meta-Mtime"] = val- meta["mtime"] = val- t.setMeta(fp, meta)-}-// TouchObject creates or updates meta on specified object.func (b *s3Backend) TouchObject(ctx context.Context, fp string, meta map[string]string) (result gofakes3.PutObjectResult, err error) {_vfs, err := b.s.getVFS(ctx)@@ -389,27 +431,7 @@ func (b *s3Backend) TouchObject(ctx context.Context, fp string, meta map[string]return result, err}- b.tenant(ctx).setMeta(fp, meta)-- if val, ok := meta["X-Amz-Meta-Mtime"]; ok {- ti, err := swift.FloatStringToTime(val)- if err == nil {- b.tenant(ctx).storeModtime(fp, meta, val)- return result, _vfs.Chtimes(fp, ti, ti)- }- // ignore error since the file is successfully created- }-- if val, ok := meta["mtime"]; ok {- ti, err := swift.FloatStringToTime(val)- if err == nil {- b.tenant(ctx).storeModtime(fp, meta, val)- return result, _vfs.Chtimes(fp, ti, ti)- }- // ignore error since the file is successfully created- }-- return result, nil+ return result, b.tenant(ctx).storeMeta(_vfs, fp, meta)}// PutObject creates or overwrites the object with the given name.@@ -510,27 +532,7 @@ func (b *s3Backend) PutObject(return result, err}- b.tenant(ctx).setMeta(fp, meta)-- if val, ok := meta["X-Amz-Meta-Mtime"]; ok {- ti, err := swift.FloatStringToTime(val)- if err == nil {- b.tenant(ctx).storeModtime(fp, meta, val)- return result, _vfs.Chtimes(fp, ti, ti)- }- // ignore error since the file is successfully created- }-- if val, ok := meta["mtime"]; ok {- ti, err := swift.FloatStringToTime(val)- if err == nil {- b.tenant(ctx).storeModtime(fp, meta, val)- return result, _vfs.Chtimes(fp, ti, ti)- }- // ignore error since the file is successfully created- }-- return result, nil+ return result, b.tenant(ctx).storeMeta(_vfs, fp, meta)}// DeleteMulti deletes multiple objects in a single request.@@ -653,22 +655,7 @@ func (b *s3Backend) CopyObject(ctx context.Context, srcBucket, srcKey, dstBucketreturn result, gofakes3.KeyNotFound(srcKey)}if srcBucket == dstBucket && srcKey == dstKey {- b.tenant(ctx).setMeta(fp, meta)-- val, ok := meta["X-Amz-Meta-Mtime"]- if !ok {- if val, ok = meta["mtime"]; !ok {- return- }- }- // update modtime- ti, err := swift.FloatStringToTime(val)- if err != nil {- return result, nil- }- b.tenant(ctx).storeModtime(fp, meta, val)-- return result, _vfs.Chtimes(fp, ti, ti)+ return result, b.tenant(ctx).storeMeta(_vfs, fp, meta)}c, err := b.GetObject(ctx, srcBucket, srcKey, nil)diff --git a/cmd/serve/s3/backend_test.go b/cmd/serve/s3/backend_test.goindex 78d74d958..54bef0f52 100644--- a/cmd/serve/s3/backend_test.go+++ b/cmd/serve/s3/backend_test.go@@ -8,6 +8,7 @@ import ("path/filepath""strings""testing"+ "time""github.com/rclone/gofakes3"_ "github.com/rclone/rclone/backend/crypt"@@ -271,3 +272,38 @@ func TestMetadataOtherTenant(t *testing.T) {require.NoError(t, obj.Contents.Close())assert.NotContains(t, obj.Metadata, secretKey)}++// TestMetadataStale checks that the metadata stored for an object isn't+// returned once the object has been changed by someone else: another+// auth proxy user sharing the backend or a change to the backend itself.+func TestMetadataStale(t *testing.T) {+ b, root := newTestBackend(t)+ ctxA, ctxB := tenantCtx("tenantA"), tenantCtx("tenantB")+ const key = "X-Amz-Meta-Colour"+ objPath := filepath.Join(root, "bucket", "meta.txt")++ put := func(ctx context.Context, meta map[string]string, data string) {+ _, err := b.PutObject(ctx, "bucket", "meta.txt", meta, strings.NewReader(data), int64(len(data)))+ require.NoError(t, err)+ }+ colour := func(ctx context.Context) string {+ _vfs, err := b.s.getVFS(ctx)+ require.NoError(t, err)+ b.forgetPath(_vfs, "bucket/meta.txt")+ obj, err := b.HeadObject(ctx, "bucket", "meta.txt")+ require.NoError(t, err)+ return obj.Metadata[key]+ }++ put(ctxA, map[string]string{key: "red"}, "data")+ assert.Equal(t, "red", colour(ctxA))++ put(ctxB, map[string]string{}, "other data")+ assert.Equal(t, "", colour(ctxA), "metadata kept after another user replaced the object")++ put(ctxA, map[string]string{key: "blue"}, "data")+ assert.Equal(t, "blue", colour(ctxA))+ later := time.Now().Add(time.Hour)+ require.NoError(t, os.Chtimes(objPath, later, later))+ assert.Equal(t, "", colour(ctxA), "metadata kept after the object was changed on the backend")+}diff --git a/cmd/serve/s3/multipart.go b/cmd/serve/s3/multipart.goindex c7e17fdc9..dcdfd5292 100644--- a/cmd/serve/s3/multipart.go+++ b/cmd/serve/s3/multipart.go@@ -33,7 +33,6 @@ import ("time""github.com/google/uuid"- "github.com/ncw/swift/v2""github.com/rclone/gofakes3""github.com/rclone/rclone/fs""github.com/rclone/rclone/fs/operations"@@ -895,18 +894,7 @@ func (b *s3Backend) CompleteMultipartUpload(ctx context.Context, bucketName, obj}b.deleteUpload(uploadID, up)- up.tenant.setMeta(up.fp, up.meta)- if val, ok := up.meta["X-Amz-Meta-Mtime"]; ok {- if ti, err := swift.FloatStringToTime(val); err == nil {- up.tenant.storeModtime(up.fp, up.meta, val)- _ = up.vfs.Chtimes(up.fp, ti, ti)- }- } else if val, ok := up.meta["mtime"]; ok {- if ti, err := swift.FloatStringToTime(val); err == nil {- up.tenant.storeModtime(up.fp, up.meta, val)- _ = up.vfs.Chtimes(up.fp, ti, ti)- }- }+ _ = up.tenant.storeMeta(up.vfs, up.fp, up.meta)return "", up.multipartETag(input), nil}diff --git a/cmd/serve/s3/put_test.go b/cmd/serve/s3/put_test.goindex 604647782..f38334ba5 100644--- a/cmd/serve/s3/put_test.go+++ b/cmd/serve/s3/put_test.go@@ -275,3 +275,28 @@ func TestMetadataMaxObjects(t *testing.T) {assert.Equal(t, want, obj.Metadata["X-Amz-Meta-Key"], key)}}++// TestMetadataAfterWriteback checks the metadata of an object written to+// the VFS cache is still returned once it has been written back to the+// backend.+func TestMetadataAfterWriteback(t *testing.T) {+ b, _, bucket := newPutTestBackend(t, "", cacheWritesVFSOpt(100*time.Millisecond))+ ctx := context.Background()+ _vfs, err := b.s.getVFS(ctx)+ require.NoError(t, err)++ mtime := swift.TimeToFloatString(time.Now().Add(-time.Hour))+ meta := map[string]string{"X-Amz-Meta-Colour": "red", "X-Amz-Meta-Mtime": mtime}+ _, err = b.PutObject(ctx, bucket, "key", meta, bytes.NewReader([]byte("data")), 4)+ require.NoError(t, err)++ colour := func() string {+ obj, err := b.HeadObject(ctx, bucket, "key")+ require.NoError(t, err)+ return obj.Metadata["X-Amz-Meta-Colour"]+ }+ assert.Equal(t, "red", colour(), "before writeback")+ _vfs.WaitForWriters(10 * time.Second)+ b.forgetPath(_vfs, path.Join(bucket, "key"))+ assert.Equal(t, "red", colour(), "after writeback")+}