Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

serve s3: fix stale metadata being returned for objects changed elsewhere

Nick Craig-Wood221398e2d051
diff --git a/cmd/serve/s3/backend.go b/cmd/serve/s3/backend.go
index 58a9136bf..14090ed44 100644
--- a/cmd/serve/s3/backend.go
+++ b/cmd/serve/s3/backend.go
@@ -43,7 +43,7 @@ const putObjectPrefix = tempObjectPrefix + "put_"
// PutStream, instead of being buffered in memory by gofakes3.
type s3Backend struct {
s *Server
- meta *lru.Cache[metaKey, map[string]string] // object metadata of all tenants
+ meta *lru.Cache[metaKey, objectMeta] // object metadata of all tenants
tenantsMu sync.Mutex
tenants map[string]*tenant // per-user state by tenant ID
@@ -86,14 +86,22 @@ type metaKey struct {
fp string // object path
}
+// objectMeta is the metadata stored for an object, with the size and
+// modification time the object had when it was stored.
+type objectMeta struct {
+ meta map[string]string
+ size int64
+ modTime time.Time
+}
+
// newMetadataStore makes a store for the metadata of up to maxObjects
// objects, forgetting that of the least recently used after that. If
// maxObjects <= 0 there is no limit.
-func newMetadataStore(maxObjects int) *lru.Cache[metaKey, map[string]string] {
+func newMetadataStore(maxObjects int) *lru.Cache[metaKey, objectMeta] {
if maxObjects <= 0 {
maxObjects = math.MaxInt
}
- meta, err := lru.New[metaKey, map[string]string](maxObjects)
+ meta, err := lru.New[metaKey, objectMeta](maxObjects)
if err != nil {
panic(err) // only fails if maxObjects <= 0
}
@@ -136,14 +144,56 @@ func (t *tenant) context() context.Context {
return context.WithValue(context.Background(), ctxKeyAccessKeyID, t.id)
}
-// getMeta returns the stored metadata of the object at fp.
-func (t *tenant) getMeta(fp string) (meta map[string]string, ok bool) {
- return t.b.meta.Get(metaKey{tenant: t.id, fp: fp})
+// getMeta returns the stored metadata of node, the object at fp.
+//
+// Nothing is returned if the object has changed since the metadata was
+// stored, as it has been replaced by something other than this user.
+func (t *tenant) getMeta(ctx context.Context, fp string, node vfs.Node) (meta map[string]string, ok bool) {
+ om, ok := t.b.meta.Get(metaKey{tenant: t.id, fp: fp})
+ if !ok || om.size != node.Size() {
+ return nil, false
+ }
+ window := fs.GetModifyWindow(ctx, node.VFS().Fs())
+ if window != fs.ModTimeNotSupported {
+ if dt := node.ModTime().Sub(om.modTime); dt < -window || dt > window {
+ return nil, false
+ }
+ }
+ return om.meta, true
}
-// setMeta stores meta as the metadata of the object at fp.
-func (t *tenant) setMeta(fp string, meta map[string]string) {
- t.b.meta.Add(metaKey{tenant: t.id, fp: fp}, meta)
+// storeMeta stores meta as the metadata of the object at fp in _vfs.
+//
+// If meta has a modification time in "X-Amz-Meta-Mtime" or "mtime" the
+// object's modification time is set to it, returning any error, and
+// both are set to it in meta.
+func (t *tenant) storeMeta(_vfs *vfs.VFS, fp string, meta map[string]string) (err error) {
+ for _, key := range []string{"X-Amz-Meta-Mtime", "mtime"} {
+ val, ok := meta[key]
+ if !ok {
+ continue
+ }
+ ti, parseErr := swift.FloatStringToTime(val)
+ if parseErr != nil {
+ continue
+ }
+ meta["X-Amz-Meta-Mtime"] = val
+ meta["mtime"] = val
+ err = _vfs.Chtimes(fp, ti, ti)
+ break
+ }
+ // Stat after setting the modification time as getMeta compares with it
+ node, statErr := _vfs.Stat(fp)
+ if statErr != nil {
+ t.removeMeta(fp)
+ return err
+ }
+ t.b.meta.Add(metaKey{tenant: t.id, fp: fp}, objectMeta{
+ meta: meta,
+ size: node.Size(),
+ modTime: node.ModTime(),
+ })
+ return err
}
// removeMeta forgets the metadata of the object at fp.
@@ -264,7 +314,7 @@ func (b *s3Backend) HeadObject(ctx context.Context, bucketName, objectName strin
"Content-Type": mimeType,
}
- if metaMap, ok := b.tenant(ctx).getMeta(fp); ok {
+ if metaMap, ok := b.tenant(ctx).getMeta(ctx, fp, node); ok {
maps.Copy(meta, metaMap)
}
@@ -344,7 +394,7 @@ func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string
"Content-Type": mimeType,
}
- if metaMap, ok := b.tenant(ctx).getMeta(fp); ok {
+ if metaMap, ok := b.tenant(ctx).getMeta(ctx, fp, node); ok {
maps.Copy(meta, metaMap)
}
@@ -358,14 +408,6 @@ func (b *s3Backend) GetObject(ctx context.Context, bucketName, objectName string
}, nil
}
-// storeModtime sets both "mtime" and "X-Amz-Meta-Mtime" to val in the
-// stored metadata. Call this whenever modtime is updated.
-func (t *tenant) storeModtime(fp string, meta map[string]string, val string) {
- meta["X-Amz-Meta-Mtime"] = val
- meta["mtime"] = val
- t.setMeta(fp, meta)
-}
-
// TouchObject creates or updates meta on specified object.
func (b *s3Backend) TouchObject(ctx context.Context, fp string, meta map[string]string) (result gofakes3.PutObjectResult, err error) {
_vfs, err := b.s.getVFS(ctx)
@@ -389,27 +431,7 @@ func (b *s3Backend) TouchObject(ctx context.Context, fp string, meta map[string]
return result, err
}
- b.tenant(ctx).setMeta(fp, meta)
-
- if val, ok := meta["X-Amz-Meta-Mtime"]; ok {
- ti, err := swift.FloatStringToTime(val)
- if err == nil {
- b.tenant(ctx).storeModtime(fp, meta, val)
- return result, _vfs.Chtimes(fp, ti, ti)
- }
- // ignore error since the file is successfully created
- }
-
- if val, ok := meta["mtime"]; ok {
- ti, err := swift.FloatStringToTime(val)
- if err == nil {
- b.tenant(ctx).storeModtime(fp, meta, val)
- return result, _vfs.Chtimes(fp, ti, ti)
- }
- // ignore error since the file is successfully created
- }
-
- return result, nil
+ return result, b.tenant(ctx).storeMeta(_vfs, fp, meta)
}
// PutObject creates or overwrites the object with the given name.
@@ -510,27 +532,7 @@ func (b *s3Backend) PutObject(
return result, err
}
- b.tenant(ctx).setMeta(fp, meta)
-
- if val, ok := meta["X-Amz-Meta-Mtime"]; ok {
- ti, err := swift.FloatStringToTime(val)
- if err == nil {
- b.tenant(ctx).storeModtime(fp, meta, val)
- return result, _vfs.Chtimes(fp, ti, ti)
- }
- // ignore error since the file is successfully created
- }
-
- if val, ok := meta["mtime"]; ok {
- ti, err := swift.FloatStringToTime(val)
- if err == nil {
- b.tenant(ctx).storeModtime(fp, meta, val)
- return result, _vfs.Chtimes(fp, ti, ti)
- }
- // ignore error since the file is successfully created
- }
-
- return result, nil
+ return result, b.tenant(ctx).storeMeta(_vfs, fp, meta)
}
// DeleteMulti deletes multiple objects in a single request.
@@ -653,22 +655,7 @@ func (b *s3Backend) CopyObject(ctx context.Context, srcBucket, srcKey, dstBucket
return result, gofakes3.KeyNotFound(srcKey)
}
if srcBucket == dstBucket && srcKey == dstKey {
- b.tenant(ctx).setMeta(fp, meta)
-
- val, ok := meta["X-Amz-Meta-Mtime"]
- if !ok {
- if val, ok = meta["mtime"]; !ok {
- return
- }
- }
- // update modtime
- ti, err := swift.FloatStringToTime(val)
- if err != nil {
- return result, nil
- }
- b.tenant(ctx).storeModtime(fp, meta, val)
-
- return result, _vfs.Chtimes(fp, ti, ti)
+ return result, b.tenant(ctx).storeMeta(_vfs, fp, meta)
}
c, err := b.GetObject(ctx, srcBucket, srcKey, nil)
diff --git a/cmd/serve/s3/backend_test.go b/cmd/serve/s3/backend_test.go
index 78d74d958..54bef0f52 100644
--- a/cmd/serve/s3/backend_test.go
+++ b/cmd/serve/s3/backend_test.go
@@ -8,6 +8,7 @@ import (
"path/filepath"
"strings"
"testing"
+ "time"
"github.com/rclone/gofakes3"
_ "github.com/rclone/rclone/backend/crypt"
@@ -271,3 +272,38 @@ func TestMetadataOtherTenant(t *testing.T) {
require.NoError(t, obj.Contents.Close())
assert.NotContains(t, obj.Metadata, secretKey)
}
+
+// TestMetadataStale checks that the metadata stored for an object isn't
+// returned once the object has been changed by someone else: another
+// auth proxy user sharing the backend or a change to the backend itself.
+func TestMetadataStale(t *testing.T) {
+ b, root := newTestBackend(t)
+ ctxA, ctxB := tenantCtx("tenantA"), tenantCtx("tenantB")
+ const key = "X-Amz-Meta-Colour"
+ objPath := filepath.Join(root, "bucket", "meta.txt")
+
+ put := func(ctx context.Context, meta map[string]string, data string) {
+ _, err := b.PutObject(ctx, "bucket", "meta.txt", meta, strings.NewReader(data), int64(len(data)))
+ require.NoError(t, err)
+ }
+ colour := func(ctx context.Context) string {
+ _vfs, err := b.s.getVFS(ctx)
+ require.NoError(t, err)
+ b.forgetPath(_vfs, "bucket/meta.txt")
+ obj, err := b.HeadObject(ctx, "bucket", "meta.txt")
+ require.NoError(t, err)
+ return obj.Metadata[key]
+ }
+
+ put(ctxA, map[string]string{key: "red"}, "data")
+ assert.Equal(t, "red", colour(ctxA))
+
+ put(ctxB, map[string]string{}, "other data")
+ assert.Equal(t, "", colour(ctxA), "metadata kept after another user replaced the object")
+
+ put(ctxA, map[string]string{key: "blue"}, "data")
+ assert.Equal(t, "blue", colour(ctxA))
+ later := time.Now().Add(time.Hour)
+ require.NoError(t, os.Chtimes(objPath, later, later))
+ assert.Equal(t, "", colour(ctxA), "metadata kept after the object was changed on the backend")
+}
diff --git a/cmd/serve/s3/multipart.go b/cmd/serve/s3/multipart.go
index c7e17fdc9..dcdfd5292 100644
--- a/cmd/serve/s3/multipart.go
+++ b/cmd/serve/s3/multipart.go
@@ -33,7 +33,6 @@ import (
"time"
"github.com/google/uuid"
- "github.com/ncw/swift/v2"
"github.com/rclone/gofakes3"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/operations"
@@ -895,18 +894,7 @@ func (b *s3Backend) CompleteMultipartUpload(ctx context.Context, bucketName, obj
}
b.deleteUpload(uploadID, up)
- up.tenant.setMeta(up.fp, up.meta)
- if val, ok := up.meta["X-Amz-Meta-Mtime"]; ok {
- if ti, err := swift.FloatStringToTime(val); err == nil {
- up.tenant.storeModtime(up.fp, up.meta, val)
- _ = up.vfs.Chtimes(up.fp, ti, ti)
- }
- } else if val, ok := up.meta["mtime"]; ok {
- if ti, err := swift.FloatStringToTime(val); err == nil {
- up.tenant.storeModtime(up.fp, up.meta, val)
- _ = up.vfs.Chtimes(up.fp, ti, ti)
- }
- }
+ _ = up.tenant.storeMeta(up.vfs, up.fp, up.meta)
return "", up.multipartETag(input), nil
}
diff --git a/cmd/serve/s3/put_test.go b/cmd/serve/s3/put_test.go
index 604647782..f38334ba5 100644
--- a/cmd/serve/s3/put_test.go
+++ b/cmd/serve/s3/put_test.go
@@ -275,3 +275,28 @@ func TestMetadataMaxObjects(t *testing.T) {
assert.Equal(t, want, obj.Metadata["X-Amz-Meta-Key"], key)
}
}
+
+// TestMetadataAfterWriteback checks the metadata of an object written to
+// the VFS cache is still returned once it has been written back to the
+// backend.
+func TestMetadataAfterWriteback(t *testing.T) {
+ b, _, bucket := newPutTestBackend(t, "", cacheWritesVFSOpt(100*time.Millisecond))
+ ctx := context.Background()
+ _vfs, err := b.s.getVFS(ctx)
+ require.NoError(t, err)
+
+ mtime := swift.TimeToFloatString(time.Now().Add(-time.Hour))
+ meta := map[string]string{"X-Amz-Meta-Colour": "red", "X-Amz-Meta-Mtime": mtime}
+ _, err = b.PutObject(ctx, bucket, "key", meta, bytes.NewReader([]byte("data")), 4)
+ require.NoError(t, err)
+
+ colour := func() string {
+ obj, err := b.HeadObject(ctx, bucket, "key")
+ require.NoError(t, err)
+ return obj.Metadata["X-Amz-Meta-Colour"]
+ }
+ assert.Equal(t, "red", colour(), "before writeback")
+ _vfs.WaitForWriters(10 * time.Second)
+ b.forgetPath(_vfs, path.Join(bucket, "key"))
+ assert.Equal(t, "red", colour(), "after writeback")
+}