"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
serve s3: apply --metadata-max-objects to each user separately
Nick Craig-Woodc2ce9a0bfb26
diff --git a/cmd/serve/s3/backend.go b/cmd/serve/s3/backend.goindex d2476cbe2..ffa18deaa 100644--- a/cmd/serve/s3/backend.go+++ b/cmd/serve/s3/backend.go@@ -42,8 +42,7 @@ const putObjectPrefix = tempObjectPrefix + "put_"// multipart uploads stream straight through to the underlying Fs via// PutStream, instead of being buffered in memory by gofakes3.type s3Backend struct {- s *Server- meta *lru.Cache[metaKey, objectMeta] // object metadata of all tenants+ s *ServertenantsMu sync.Mutextenants map[string]*tenant // per-user state by tenant ID@@ -80,12 +79,9 @@ type tenant struct {// budget limits the memory buffered by all the multipart uploads,// set by --multipart-streaming-buffer-total.budget *bufferBudget-}-// metaKey is the key of an object's metadata in s3Backend.meta.-type metaKey struct {- tenant string // tenant ID- fp string // object path+ // meta is the stored metadata of objects, by object path.+ meta *lru.Cache[string, objectMeta]}// objectMeta is the metadata stored for an object, with the size and@@ -99,11 +95,11 @@ type objectMeta struct {// newMetadataStore makes a store for the metadata of up to maxObjects// objects, forgetting that of the least recently used after that. If// maxObjects <= 0 there is no limit.-func newMetadataStore(maxObjects int) *lru.Cache[metaKey, objectMeta] {+func newMetadataStore(maxObjects int) *lru.Cache[string, objectMeta] {if maxObjects <= 0 {maxObjects = math.MaxInt}- meta, err := lru.New[metaKey, objectMeta](maxObjects)+ meta, err := lru.New[string, objectMeta](maxObjects)if err != nil {panic(err) // only fails if maxObjects <= 0}@@ -114,7 +110,6 @@ func newMetadataStore(maxObjects int) *lru.Cache[metaKey, objectMeta] {func newBackend(s *Server) *s3Backend {return &s3Backend{s: s,- meta: newMetadataStore(s.opt.MetadataMaxObjects),tenants: map[string]*tenant{},reaperQuit: make(chan struct{}),}@@ -131,6 +126,7 @@ func (b *s3Backend) tenant(ctx context.Context) *tenant {b: b,id: id,budget: newBufferBudget(int64(b.s.opt.MultipartStreamingBufferTotal)),+ meta: newMetadataStore(b.s.opt.MetadataMaxObjects),}b.tenants[id] = t}@@ -154,7 +150,7 @@ func (t *tenant) context() context.Context {// Nothing is returned if the object has changed since the metadata was// stored, as it has been replaced by something other than this user.func (t *tenant) getMeta(ctx context.Context, fp string, node vfs.Node) (meta map[string]string, ok bool) {- om, ok := t.b.meta.Get(metaKey{tenant: t.id, fp: fp})+ om, ok := t.meta.Get(fp)if !ok || om.size != node.Size() {return nil, false}@@ -193,7 +189,7 @@ func (t *tenant) storeMeta(_vfs *vfs.VFS, fp string, meta map[string]string) (ert.removeMeta(fp)return err}- t.b.meta.Add(metaKey{tenant: t.id, fp: fp}, objectMeta{+ t.meta.Add(fp, objectMeta{meta: meta,size: node.Size(),modTime: node.ModTime(),@@ -203,7 +199,7 @@ func (t *tenant) storeMeta(_vfs *vfs.VFS, fp string, meta map[string]string) (er// removeMeta forgets the metadata of the object at fp.func (t *tenant) removeMeta(fp string) {- t.b.meta.Remove(metaKey{tenant: t.id, fp: fp})+ t.meta.Remove(fp)}// ListBuckets always returns the default bucket.diff --git a/cmd/serve/s3/put_test.go b/cmd/serve/s3/put_test.goindex f38334ba5..64d829807 100644--- a/cmd/serve/s3/put_test.go+++ b/cmd/serve/s3/put_test.go@@ -256,24 +256,34 @@ func TestDeleteObjectForgetsMetadata(t *testing.T) {}// TestMetadataMaxObjects checks that the metadata of no more than-// --metadata-max-objects objects is kept, forgetting that of the least-// recently used.+// --metadata-max-objects objects of a user is kept, forgetting that of+// the least recently used, and that one user's objects don't push out+// the metadata of another's.func TestMetadataMaxObjects(t *testing.T) {b, _, bucket := newPutTestBackend(t, "", nil)- b.meta = newMetadataStore(2)- ctx := context.Background()+ b.s.opt.MetadataMaxObjects = 2+ ctx, ctxOther := context.Background(), tenantCtx("other")- for _, key := range []string{"a", "b", "c"} {+ put := func(ctx context.Context, key string) {meta := map[string]string{"X-Amz-Meta-Key": key}- _, err := b.PutObject(ctx, bucket, key, meta, bytes.NewReader([]byte(key)), 1)+ _, err := b.PutObject(ctx, bucket, key, meta, bytes.NewReader([]byte(key)), int64(len(key)))require.NoError(t, err)}-- for key, want := range map[string]string{"a": "", "b": "b", "c": "c"} {+ head := func(ctx context.Context, key string) string {obj, err := b.HeadObject(ctx, bucket, key)require.NoError(t, err)- assert.Equal(t, want, obj.Metadata["X-Amz-Meta-Key"], key)+ return obj.Metadata["X-Amz-Meta-Key"]+ }++ put(ctxOther, "other")+ for _, key := range []string{"a", "b", "c"} {+ put(ctx, key)+ }++ for key, want := range map[string]string{"a": "", "b": "b", "c": "c"} {+ assert.Equal(t, want, head(ctx, key), key)}+ assert.Equal(t, "other", head(ctxOther, "other"), "another user's metadata was pushed out")}// TestMetadataAfterWriteback checks the metadata of an object written todiff --git a/cmd/serve/s3/s3.go b/cmd/serve/s3/s3.goindex 23edf1612..47a65a04b 100644--- a/cmd/serve/s3/s3.go+++ b/cmd/serve/s3/s3.go@@ -56,7 +56,7 @@ var OptionsInfo = fs.Options{{}, {Name: "metadata_max_objects",Default: 100000,- Help: "Maximum number of objects whose metadata is kept in memory, 0 for unlimited",+ Help: "Maximum number of objects of a user whose metadata is kept in memory, 0 for unlimited",}, {Name: "multipart_expiry",Default: fs.Duration(24 * time.Hour),diff --git a/cmd/serve/s3/serve_s3.md b/cmd/serve/s3/serve_s3.mdindex 27ca1d43f..a7b5f85f8 100644--- a/cmd/serve/s3/serve_s3.md+++ b/cmd/serve/s3/serve_s3.md@@ -387,7 +387,7 @@ make it use unbounded memory.`--s3-upload-concurrency`). Plain `PutObject` uploads use the samewhile they are in progress.- **Metadata.** The metadata of at most `--metadata-max-objects`- objects is kept, each limited to 2 KB.+ objects of each user is kept, each limited to 2 KB.- **Request bodies.** The XML bodies of requests such as`CompleteMultipartUpload` and `DeleteObjects` are limited to 10 MiB,and `DeleteObjects` to 1000 keys. A browser form (`POST`) upload@@ -451,8 +451,8 @@ Versioning is not currently supported.Metadata will only be saved in memory other than the rclone `mtime`metadata which will be set as the modification time of the file. Themetadata of at most `--metadata-max-objects` objects (default `100000`,-`0` for no limit) is kept; after that the metadata of the least-recently used object is forgotten. The metadata of an object is+`0` for no limit) of each user (each access key) is kept; after that+the metadata of the user's least recently used object is forgotten. The metadata of an object isignored once the object has been changed some other way, for exampleby another `--auth-proxy` user or directly on the backend.