"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
serve s3: list the directory markers put for directories holding other keys
Nick Craig-Wood85f1f9d39b03
diff --git a/cmd/serve/s3/backend.go b/cmd/serve/s3/backend.goindex 81605ce28..a1e86ad32 100644--- a/cmd/serve/s3/backend.go+++ b/cmd/serve/s3/backend.go@@ -86,7 +86,8 @@ type tenant struct {// set by --multipart-streaming-buffer-total.budget *bufferBudget- // meta is the stored metadata of objects, by object path.+ // meta is the stored metadata of objects, by object path, and+ // the directory markers put, by directory path.meta *lru.Cache[string, objectMeta]}@@ -98,6 +99,7 @@ type objectMeta struct {partSizes []int64 // the sizes of the parts of an object made by a multipart uploadsize int64modTime time.Time+ dirMarker bool // a directory marker was put for the directory at this path}// newMetadataStore makes a store for the metadata of up to maxObjects@@ -159,7 +161,7 @@ func (t *tenant) context() context.Context {// stored, as it has been replaced by something other than this user.func (t *tenant) getMeta(ctx context.Context, fp string, node vfs.Node) (om objectMeta, ok bool) {om, ok = t.meta.Get(fp)- if !ok || om.size != node.Size() {+ if !ok || om.dirMarker || om.size != node.Size() {return objectMeta{}, false}window := fs.GetModifyWindow(ctx, node.VFS().Fs())@@ -242,11 +244,38 @@ func (b *s3Backend) SetObjectTagging(ctx context.Context, bucketName, objectNamereturn t.storeObjectMeta(_vfs, fp, om)}-// removeMeta forgets the metadata of the object at fp.+// removeMeta forgets the metadata of the object at fp, or the+// directory marker of the directory at fp.func (t *tenant) removeMeta(fp string) {t.meta.Remove(fp)}+// storeDirMarker records that a directory marker was put for the+// directory at fp, so it is listed while the directory holds other+// keys.+func (t *tenant) storeDirMarker(fp string) {+ t.meta.Add(fp, objectMeta{dirMarker: true})+}++// hasDirMarker reports whether a directory marker was put for the+// directory at fp.+func (t *tenant) hasDirMarker(fp string) bool {+ om, ok := t.meta.Get(fp)+ return ok && om.dirMarker+}++// removeDirMarkers forgets the directory markers of every user below+// the directory dir.+func (b *s3Backend) removeDirMarkers(dir string) {+ for _, t := range b.allTenants() {+ for _, fp := range t.meta.Keys() {+ if strings.HasPrefix(fp, dir+"/") {+ t.meta.Remove(fp)+ }+ }+ }+}+// ListBuckets always returns the default bucket.func (b *s3Backend) ListBuckets(ctx context.Context) ([]gofakes3.BucketInfo, error) {_vfs, err := b.s.getVFS(ctx)@@ -566,7 +595,11 @@ func (b *s3Backend) PutObject(if b.s.etagHash(_vfs) != fshash.MD5 {result.ETag = quoteETag(emptyHash(b.s.etagHash(_vfs)))}- return result, putDirMarker(_vfs, fp, objectName, input)+ if err := putDirMarker(_vfs, fp, objectName, input); err != nil {+ return result, err+ }+ b.tenant(ctx).storeDirMarker(fp)+ return result, nil}fp, err := bucketObjectPath(bucketName, objectName)if err != nil {@@ -751,6 +784,7 @@ func (b *s3Backend) deleteObject(ctx context.Context, bucketName, objectName strif err != nil {return gofakes3.BucketNotFound(bucketName)}+ t := b.tenant(ctx)if fp, isMarker, err := bucketDirMarkerPath(bucketName, objectName); isMarker {if err != nil {@@ -758,11 +792,12 @@ func (b *s3Backend) deleteObject(ctx context.Context, bucketName, objectName str}// Deleting a directory marker leaves the objects in its// directory, so the directory is only removed if empty.+ t.removeMeta(fp)if isEmptyDir(fp, _vfs) {if err := _vfs.Remove(fp); err != nil {return err}- rmdirRecursive(fp, _vfs)+ rmdirRecursive(fp, _vfs, t.hasDirMarker)}return nil}@@ -775,10 +810,10 @@ func (b *s3Backend) deleteObject(ctx context.Context, bucketName, objectName strif err := _vfs.Remove(fp); err != nil && !os.IsNotExist(err) {return err}- b.tenant(ctx).removeMeta(fp)+ t.removeMeta(fp)// FIXME: unsafe operation- rmdirRecursive(fp, _vfs)+ rmdirRecursive(fp, _vfs, t.hasDirMarker)return nil}@@ -822,6 +857,7 @@ func (b *s3Backend) DeleteBucket(ctx context.Context, name string) error {}return err}+ b.removeDirMarkers(name)return nil}diff --git a/cmd/serve/s3/list.go b/cmd/serve/s3/list.goindex d5d5ec6a5..96f7b2eb4 100644--- a/cmd/serve/s3/list.go+++ b/cmd/serve/s3/list.go@@ -30,6 +30,7 @@ var errPageFull = errors.New("listing page full")// the size of the subtree below the prefix.type lister struct {vfs *vfs.VFS+ t *tenant // the user listinghashType hash.Type // hash used for ETagsetag func(fp string, entry vfs.Node) []byte // returns the ETag of the object at fpbucket string@@ -60,6 +61,7 @@ func newLister(ctx context.Context, b *s3Backend, _vfs *vfs.VFS, bucket string,}return &lister{vfs: _vfs,+ t: b.tenant(ctx),hashType: b.s.etagHash(_vfs),bucket: bucket,marker: page.Marker,@@ -136,8 +138,8 @@ func (l *lister) addObject(key string, entry vfs.Node) error {return nil}-// addDirMarker adds the directory marker key of the empty directory-// fp to the page, returning errPageFull if the page is now complete.+// addDirMarker adds the directory marker key of the directory fp to+// the page, returning errPageFull if the page is now complete.func (l *lister) addDirMarker(key, fp string) error {if l.skipKey(key) {return nil@@ -204,8 +206,8 @@ func (l *lister) hasKeys(fdPath string) bool {if err != nil {return false}- if len(dirEntries) == 0 {- // An empty directory has its directory marker key+ if len(dirEntries) == 0 || l.t.hasDirMarker(fp) {+ // The directory has its directory marker keyreturn true}for _, entry := range dirEntries {@@ -246,11 +248,14 @@ func (l *lister) list(fdPath, name string, addPrefix bool) error {}l.dirsRead++- // An empty directory is listed as its directory marker, which is- // the only key below it, as S3 would list the marker which made- // it. Other directories may have been made implicitly, so aren't.- if len(dirEntries) == 0 && name == "" && fdPath != "" {- return l.addDirMarker(fdPath+"/", fp)+ // A directory is listed as its directory marker, which sorts+ // before the keys below it, if it is empty, as S3 would list the+ // marker which made it, or if a marker was put for it. Other+ // directories may have been made implicitly, so aren't.+ if name == "" && fdPath != "" && (len(dirEntries) == 0 || l.t.hasDirMarker(fp)) {+ if err := l.addDirMarker(fdPath+"/", fp); err != nil || len(dirEntries) == 0 {+ return err+ }}// Emit the entries in the order their keys have in a flat keyspace,diff --git a/cmd/serve/s3/list_test.go b/cmd/serve/s3/list_test.goindex 47d9711d1..0f370ba72 100644--- a/cmd/serve/s3/list_test.go+++ b/cmd/serve/s3/list_test.go@@ -345,3 +345,58 @@ func TestListETagQuoted(t *testing.T) {require.NoError(t, err)assert.Equal(t, `"`+hex.EncodeToString(obj.Hash)+`"`, list.Contents[0].ETag)}++// TestListPutDirMarkers checks that the directory marker put for a+// directory is listed, and kept, while the directory holds other keys,+// as on S3, but not that of a directory made by putting a key below it.+func TestListPutDirMarkers(t *testing.T) {+ b := newListBackend(t, "bucket/implicit/file")+ ctx := context.Background()+ put := func(bucket, key string) {+ _, err := b.PutObject(ctx, bucket, key, nil, strings.NewReader(""), 0)+ require.NoError(t, err)+ }+ del := func(key string) {+ require.NoError(t, b.deleteObject(ctx, "bucket", key))+ }+ all := func() []string {+ return listPages(t, b, listPrefix("", ""), 1000)+ }++ put("bucket", "dir/")+ put("bucket", "dir/a")+ put("bucket", "dir/sub/b")+ want := []string{"dir/", "dir/a", "dir/sub/b", "implicit/file"}+ assert.Equal(t, want, all())+ assert.Equal(t, want, listPages(t, b, listPrefix("", ""), 1))+ keys, _ := listOnce(t, b, listPrefix("", "/"), gofakes3.ListBucketPage{})+ assert.Equal(t, []string{"dir/", "implicit/"}, keys)+ keys, _ = listOnce(t, b, listPrefix("dir/", "/"), gofakes3.ListBucketPage{})+ assert.Equal(t, []string{"dir/", "dir/a", "dir/sub/"}, keys)+ keys, _ = listOnce(t, b, listPrefix("", "r"), gofakes3.ListBucketPage{})+ assert.Equal(t, []string{"dir", "implicit/file"}, keys)++ // The marker keeps its directory when the keys below it are deleted+ del("dir/a")+ del("dir/sub/b")+ assert.Equal(t, []string{"dir/", "implicit/file"}, all())++ // Deleting the marker leaves the keys below it, and then the+ // directory goes with the last of them+ put("bucket", "dir/c")+ del("dir/")+ assert.Equal(t, []string{"dir/c", "implicit/file"}, all())+ del("dir/c")+ assert.Equal(t, []string{"implicit/file"}, all())++ // The markers of a deleted bucket are forgotten+ require.NoError(t, b.CreateBucket(ctx, "other"))+ put("other", "d/")+ require.NoError(t, b.DeleteBucket(ctx, "other"))+ require.NoError(t, b.CreateBucket(ctx, "other"))+ put("other", "d/f")+ list, err := b.ListBucket(ctx, "other", listPrefix("", ""), gofakes3.ListBucketPage{})+ require.NoError(t, err)+ require.Len(t, list.Contents, 1)+ assert.Equal(t, "d/f", list.Contents[0].Key)+}diff --git a/cmd/serve/s3/serve_s3.md b/cmd/serve/s3/serve_s3.mdindex 5cd22bd9d..929abcdde 100644--- a/cmd/serve/s3/serve_s3.md+++ b/cmd/serve/s3/serve_s3.md@@ -488,9 +488,14 @@ for the directory `dir`. Putting an empty directory marker creates thedirectory (one with data is rejected with `400 Bad Request`), anddeleting it removes the directory if it is empty. Directory markersare found by `HeadObject`, `GetObject` and `CopyObject` for any-directory, but only the markers of empty directories are listed as-objects, as other directories may not have been made by putting a-marker.+directory, but only the markers of empty directories, and of the+directories a marker was put for, are listed as objects, as other+directories may not have been made by putting a marker. The markers+put are remembered in memory with the metadata (see+[Limitations](#limitations)), so after a restart, or once forgotten,+the marker of a directory holding other keys is no longer listed.+While a marker is remembered its directory isn't removed when the+last key below it is deleted.As keys are file paths, a key can't be both an object and a directoryholding other objects, for example `a/b` and `a/b/c`. Writing an@@ -518,6 +523,7 @@ ignored.| Authentication | Partial | AWS Signature Version 4, in headers, presigned URLs and browser form (`POST`) uploads. Signature Version 2 isn't supported. || Anonymous access | Partial | All requests are allowed without `--auth-key` or `--auth-proxy`, and none are with them, as buckets and objects can't be made public. || Metadata | Partial | `x-amz-meta-*` and the standard headers such as `Content-Type` are kept in memory only (see [Limitations](#limitations)), except the modification time. |+| Directory markers | Partial | Keys ending in `/` are stored as directories, so can't hold data. A marker put for a directory holding other keys is only listed while it is remembered in memory (see [Object names](#object-names)). || Checksums | Partial | The `x-amz-checksum-*` of a `PutObject` (CRC32, CRC32C, CRC64NVME, SHA1 or SHA256) is checked and returned with the metadata. The checksums of multipart uploads aren't supported. || `GetObjectAttributes` | Partial | Everything but the checksums of multipart uploads. || Object tagging | Partial | `GetObjectTagging`, `PutObjectTagging` and `DeleteObjectTagging` work, but tags are kept in memory with the metadata. |diff --git a/cmd/serve/s3/utils.go b/cmd/serve/s3/utils.goindex 790755f3b..f151d7ab9 100644--- a/cmd/serve/s3/utils.go+++ b/cmd/serve/s3/utils.go@@ -228,8 +228,8 @@ func removeMadeDirs(dir, made string, VFS *vfs.VFS) {}// rmdirRecursive removes the parent directories of p which are empty,-// stopping at the bucket.-func rmdirRecursive(p string, VFS *vfs.VFS) {+// stopping at the bucket or at a directory for which keep is true.+func rmdirRecursive(p string, VFS *vfs.VFS, keep func(dir string) bool) {dir := path.Dir(p)if !strings.ContainsAny(dir, "/\\") {// might be bucket(root)@@ -237,13 +237,13 @@ func rmdirRecursive(p string, VFS *vfs.VFS) {}// Check the directory is empty first as the VFS logs an error// removing one which isn't- if !isEmptyDir(dir, VFS) {+ if keep(dir) || !isEmptyDir(dir, VFS) {return}if err := VFS.Remove(dir); err != nil {return}- rmdirRecursive(dir, VFS)+ rmdirRecursive(dir, VFS, keep)}// pathLocks are locks on paths, made as needed.