Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

serve s3: apply --metadata-max-objects to each user separately

Nick Craig-Woodc2ce9a0bfb26
diff --git a/cmd/serve/s3/backend.go b/cmd/serve/s3/backend.go
index d2476cbe2..ffa18deaa 100644
--- a/cmd/serve/s3/backend.go
+++ b/cmd/serve/s3/backend.go
@@ -42,8 +42,7 @@ const putObjectPrefix = tempObjectPrefix + "put_"
// multipart uploads stream straight through to the underlying Fs via
// PutStream, instead of being buffered in memory by gofakes3.
type s3Backend struct {
- s *Server
- meta *lru.Cache[metaKey, objectMeta] // object metadata of all tenants
+ s *Server
tenantsMu sync.Mutex
tenants map[string]*tenant // per-user state by tenant ID
@@ -80,12 +79,9 @@ type tenant struct {
// budget limits the memory buffered by all the multipart uploads,
// set by --multipart-streaming-buffer-total.
budget *bufferBudget
-}
-// metaKey is the key of an object's metadata in s3Backend.meta.
-type metaKey struct {
- tenant string // tenant ID
- fp string // object path
+ // meta is the stored metadata of objects, by object path.
+ meta *lru.Cache[string, objectMeta]
}
// objectMeta is the metadata stored for an object, with the size and
@@ -99,11 +95,11 @@ type objectMeta struct {
// newMetadataStore makes a store for the metadata of up to maxObjects
// objects, forgetting that of the least recently used after that. If
// maxObjects <= 0 there is no limit.
-func newMetadataStore(maxObjects int) *lru.Cache[metaKey, objectMeta] {
+func newMetadataStore(maxObjects int) *lru.Cache[string, objectMeta] {
if maxObjects <= 0 {
maxObjects = math.MaxInt
}
- meta, err := lru.New[metaKey, objectMeta](maxObjects)
+ meta, err := lru.New[string, objectMeta](maxObjects)
if err != nil {
panic(err) // only fails if maxObjects <= 0
}
@@ -114,7 +110,6 @@ func newMetadataStore(maxObjects int) *lru.Cache[metaKey, objectMeta] {
func newBackend(s *Server) *s3Backend {
return &s3Backend{
s: s,
- meta: newMetadataStore(s.opt.MetadataMaxObjects),
tenants: map[string]*tenant{},
reaperQuit: make(chan struct{}),
}
@@ -131,6 +126,7 @@ func (b *s3Backend) tenant(ctx context.Context) *tenant {
b: b,
id: id,
budget: newBufferBudget(int64(b.s.opt.MultipartStreamingBufferTotal)),
+ meta: newMetadataStore(b.s.opt.MetadataMaxObjects),
}
b.tenants[id] = t
}
@@ -154,7 +150,7 @@ func (t *tenant) context() context.Context {
// Nothing is returned if the object has changed since the metadata was
// stored, as it has been replaced by something other than this user.
func (t *tenant) getMeta(ctx context.Context, fp string, node vfs.Node) (meta map[string]string, ok bool) {
- om, ok := t.b.meta.Get(metaKey{tenant: t.id, fp: fp})
+ om, ok := t.meta.Get(fp)
if !ok || om.size != node.Size() {
return nil, false
}
@@ -193,7 +189,7 @@ func (t *tenant) storeMeta(_vfs *vfs.VFS, fp string, meta map[string]string) (er
t.removeMeta(fp)
return err
}
- t.b.meta.Add(metaKey{tenant: t.id, fp: fp}, objectMeta{
+ t.meta.Add(fp, objectMeta{
meta: meta,
size: node.Size(),
modTime: node.ModTime(),
@@ -203,7 +199,7 @@ func (t *tenant) storeMeta(_vfs *vfs.VFS, fp string, meta map[string]string) (er
// removeMeta forgets the metadata of the object at fp.
func (t *tenant) removeMeta(fp string) {
- t.b.meta.Remove(metaKey{tenant: t.id, fp: fp})
+ t.meta.Remove(fp)
}
// ListBuckets always returns the default bucket.
diff --git a/cmd/serve/s3/put_test.go b/cmd/serve/s3/put_test.go
index f38334ba5..64d829807 100644
--- a/cmd/serve/s3/put_test.go
+++ b/cmd/serve/s3/put_test.go
@@ -256,24 +256,34 @@ func TestDeleteObjectForgetsMetadata(t *testing.T) {
}
// TestMetadataMaxObjects checks that the metadata of no more than
-// --metadata-max-objects objects is kept, forgetting that of the least
-// recently used.
+// --metadata-max-objects objects of a user is kept, forgetting that of
+// the least recently used, and that one user's objects don't push out
+// the metadata of another's.
func TestMetadataMaxObjects(t *testing.T) {
b, _, bucket := newPutTestBackend(t, "", nil)
- b.meta = newMetadataStore(2)
- ctx := context.Background()
+ b.s.opt.MetadataMaxObjects = 2
+ ctx, ctxOther := context.Background(), tenantCtx("other")
- for _, key := range []string{"a", "b", "c"} {
+ put := func(ctx context.Context, key string) {
meta := map[string]string{"X-Amz-Meta-Key": key}
- _, err := b.PutObject(ctx, bucket, key, meta, bytes.NewReader([]byte(key)), 1)
+ _, err := b.PutObject(ctx, bucket, key, meta, bytes.NewReader([]byte(key)), int64(len(key)))
require.NoError(t, err)
}
-
- for key, want := range map[string]string{"a": "", "b": "b", "c": "c"} {
+ head := func(ctx context.Context, key string) string {
obj, err := b.HeadObject(ctx, bucket, key)
require.NoError(t, err)
- assert.Equal(t, want, obj.Metadata["X-Amz-Meta-Key"], key)
+ return obj.Metadata["X-Amz-Meta-Key"]
+ }
+
+ put(ctxOther, "other")
+ for _, key := range []string{"a", "b", "c"} {
+ put(ctx, key)
+ }
+
+ for key, want := range map[string]string{"a": "", "b": "b", "c": "c"} {
+ assert.Equal(t, want, head(ctx, key), key)
}
+ assert.Equal(t, "other", head(ctxOther, "other"), "another user's metadata was pushed out")
}
// TestMetadataAfterWriteback checks the metadata of an object written to
diff --git a/cmd/serve/s3/s3.go b/cmd/serve/s3/s3.go
index 23edf1612..47a65a04b 100644
--- a/cmd/serve/s3/s3.go
+++ b/cmd/serve/s3/s3.go
@@ -56,7 +56,7 @@ var OptionsInfo = fs.Options{{
}, {
Name: "metadata_max_objects",
Default: 100000,
- Help: "Maximum number of objects whose metadata is kept in memory, 0 for unlimited",
+ Help: "Maximum number of objects of a user whose metadata is kept in memory, 0 for unlimited",
}, {
Name: "multipart_expiry",
Default: fs.Duration(24 * time.Hour),
diff --git a/cmd/serve/s3/serve_s3.md b/cmd/serve/s3/serve_s3.md
index 27ca1d43f..a7b5f85f8 100644
--- a/cmd/serve/s3/serve_s3.md
+++ b/cmd/serve/s3/serve_s3.md
@@ -387,7 +387,7 @@ make it use unbounded memory.
`--s3-upload-concurrency`). Plain `PutObject` uploads use the same
while they are in progress.
- **Metadata.** The metadata of at most `--metadata-max-objects`
- objects is kept, each limited to 2 KB.
+ objects of each user is kept, each limited to 2 KB.
- **Request bodies.** The XML bodies of requests such as
`CompleteMultipartUpload` and `DeleteObjects` are limited to 10 MiB,
and `DeleteObjects` to 1000 keys. A browser form (`POST`) upload
@@ -451,8 +451,8 @@ Versioning is not currently supported.
Metadata will only be saved in memory other than the rclone `mtime`
metadata which will be set as the modification time of the file. The
metadata of at most `--metadata-max-objects` objects (default `100000`,
-`0` for no limit) is kept; after that the metadata of the least
-recently used object is forgotten. The metadata of an object is
+`0` for no limit) of each user (each access key) is kept; after that
+the metadata of the user's least recently used object is forgotten. The metadata of an object is
ignored once the object has been changed some other way, for example
by another `--auth-proxy` user or directly on the backend.