Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

serve s3: list the directory markers put for directories holding other keys

Nick Craig-Wood85f1f9d39b03
diff --git a/cmd/serve/s3/backend.go b/cmd/serve/s3/backend.go
index 81605ce28..a1e86ad32 100644
--- a/cmd/serve/s3/backend.go
+++ b/cmd/serve/s3/backend.go
@@ -86,7 +86,8 @@ type tenant struct {
// set by --multipart-streaming-buffer-total.
budget *bufferBudget
- // meta is the stored metadata of objects, by object path.
+ // meta is the stored metadata of objects, by object path, and
+ // the directory markers put, by directory path.
meta *lru.Cache[string, objectMeta]
}
@@ -98,6 +99,7 @@ type objectMeta struct {
partSizes []int64 // the sizes of the parts of an object made by a multipart upload
size int64
modTime time.Time
+ dirMarker bool // a directory marker was put for the directory at this path
}
// newMetadataStore makes a store for the metadata of up to maxObjects
@@ -159,7 +161,7 @@ func (t *tenant) context() context.Context {
// stored, as it has been replaced by something other than this user.
func (t *tenant) getMeta(ctx context.Context, fp string, node vfs.Node) (om objectMeta, ok bool) {
om, ok = t.meta.Get(fp)
- if !ok || om.size != node.Size() {
+ if !ok || om.dirMarker || om.size != node.Size() {
return objectMeta{}, false
}
window := fs.GetModifyWindow(ctx, node.VFS().Fs())
@@ -242,11 +244,38 @@ func (b *s3Backend) SetObjectTagging(ctx context.Context, bucketName, objectName
return t.storeObjectMeta(_vfs, fp, om)
}
-// removeMeta forgets the metadata of the object at fp.
+// removeMeta forgets the metadata of the object at fp, or the
+// directory marker of the directory at fp.
func (t *tenant) removeMeta(fp string) {
t.meta.Remove(fp)
}
+// storeDirMarker records that a directory marker was put for the
+// directory at fp, so it is listed while the directory holds other
+// keys.
+func (t *tenant) storeDirMarker(fp string) {
+ t.meta.Add(fp, objectMeta{dirMarker: true})
+}
+
+// hasDirMarker reports whether a directory marker was put for the
+// directory at fp.
+func (t *tenant) hasDirMarker(fp string) bool {
+ om, ok := t.meta.Get(fp)
+ return ok && om.dirMarker
+}
+
+// removeDirMarkers forgets the directory markers of every user below
+// the directory dir.
+func (b *s3Backend) removeDirMarkers(dir string) {
+ for _, t := range b.allTenants() {
+ for _, fp := range t.meta.Keys() {
+ if strings.HasPrefix(fp, dir+"/") {
+ t.meta.Remove(fp)
+ }
+ }
+ }
+}
+
// ListBuckets always returns the default bucket.
func (b *s3Backend) ListBuckets(ctx context.Context) ([]gofakes3.BucketInfo, error) {
_vfs, err := b.s.getVFS(ctx)
@@ -566,7 +595,11 @@ func (b *s3Backend) PutObject(
if b.s.etagHash(_vfs) != fshash.MD5 {
result.ETag = quoteETag(emptyHash(b.s.etagHash(_vfs)))
}
- return result, putDirMarker(_vfs, fp, objectName, input)
+ if err := putDirMarker(_vfs, fp, objectName, input); err != nil {
+ return result, err
+ }
+ b.tenant(ctx).storeDirMarker(fp)
+ return result, nil
}
fp, err := bucketObjectPath(bucketName, objectName)
if err != nil {
@@ -751,6 +784,7 @@ func (b *s3Backend) deleteObject(ctx context.Context, bucketName, objectName str
if err != nil {
return gofakes3.BucketNotFound(bucketName)
}
+ t := b.tenant(ctx)
if fp, isMarker, err := bucketDirMarkerPath(bucketName, objectName); isMarker {
if err != nil {
@@ -758,11 +792,12 @@ func (b *s3Backend) deleteObject(ctx context.Context, bucketName, objectName str
}
// Deleting a directory marker leaves the objects in its
// directory, so the directory is only removed if empty.
+ t.removeMeta(fp)
if isEmptyDir(fp, _vfs) {
if err := _vfs.Remove(fp); err != nil {
return err
}
- rmdirRecursive(fp, _vfs)
+ rmdirRecursive(fp, _vfs, t.hasDirMarker)
}
return nil
}
@@ -775,10 +810,10 @@ func (b *s3Backend) deleteObject(ctx context.Context, bucketName, objectName str
if err := _vfs.Remove(fp); err != nil && !os.IsNotExist(err) {
return err
}
- b.tenant(ctx).removeMeta(fp)
+ t.removeMeta(fp)
// FIXME: unsafe operation
- rmdirRecursive(fp, _vfs)
+ rmdirRecursive(fp, _vfs, t.hasDirMarker)
return nil
}
@@ -822,6 +857,7 @@ func (b *s3Backend) DeleteBucket(ctx context.Context, name string) error {
}
return err
}
+ b.removeDirMarkers(name)
return nil
}
diff --git a/cmd/serve/s3/list.go b/cmd/serve/s3/list.go
index d5d5ec6a5..96f7b2eb4 100644
--- a/cmd/serve/s3/list.go
+++ b/cmd/serve/s3/list.go
@@ -30,6 +30,7 @@ var errPageFull = errors.New("listing page full")
// the size of the subtree below the prefix.
type lister struct {
vfs *vfs.VFS
+ t *tenant // the user listing
hashType hash.Type // hash used for ETags
etag func(fp string, entry vfs.Node) []byte // returns the ETag of the object at fp
bucket string
@@ -60,6 +61,7 @@ func newLister(ctx context.Context, b *s3Backend, _vfs *vfs.VFS, bucket string,
}
return &lister{
vfs: _vfs,
+ t: b.tenant(ctx),
hashType: b.s.etagHash(_vfs),
bucket: bucket,
marker: page.Marker,
@@ -136,8 +138,8 @@ func (l *lister) addObject(key string, entry vfs.Node) error {
return nil
}
-// addDirMarker adds the directory marker key of the empty directory
-// fp to the page, returning errPageFull if the page is now complete.
+// addDirMarker adds the directory marker key of the directory fp to
+// the page, returning errPageFull if the page is now complete.
func (l *lister) addDirMarker(key, fp string) error {
if l.skipKey(key) {
return nil
@@ -204,8 +206,8 @@ func (l *lister) hasKeys(fdPath string) bool {
if err != nil {
return false
}
- if len(dirEntries) == 0 {
- // An empty directory has its directory marker key
+ if len(dirEntries) == 0 || l.t.hasDirMarker(fp) {
+ // The directory has its directory marker key
return true
}
for _, entry := range dirEntries {
@@ -246,11 +248,14 @@ func (l *lister) list(fdPath, name string, addPrefix bool) error {
}
l.dirsRead++
- // An empty directory is listed as its directory marker, which is
- // the only key below it, as S3 would list the marker which made
- // it. Other directories may have been made implicitly, so aren't.
- if len(dirEntries) == 0 && name == "" && fdPath != "" {
- return l.addDirMarker(fdPath+"/", fp)
+ // A directory is listed as its directory marker, which sorts
+ // before the keys below it, if it is empty, as S3 would list the
+ // marker which made it, or if a marker was put for it. Other
+ // directories may have been made implicitly, so aren't.
+ if name == "" && fdPath != "" && (len(dirEntries) == 0 || l.t.hasDirMarker(fp)) {
+ if err := l.addDirMarker(fdPath+"/", fp); err != nil || len(dirEntries) == 0 {
+ return err
+ }
}
// Emit the entries in the order their keys have in a flat keyspace,
diff --git a/cmd/serve/s3/list_test.go b/cmd/serve/s3/list_test.go
index 47d9711d1..0f370ba72 100644
--- a/cmd/serve/s3/list_test.go
+++ b/cmd/serve/s3/list_test.go
@@ -345,3 +345,58 @@ func TestListETagQuoted(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, `"`+hex.EncodeToString(obj.Hash)+`"`, list.Contents[0].ETag)
}
+
+// TestListPutDirMarkers checks that the directory marker put for a
+// directory is listed, and kept, while the directory holds other keys,
+// as on S3, but not that of a directory made by putting a key below it.
+func TestListPutDirMarkers(t *testing.T) {
+ b := newListBackend(t, "bucket/implicit/file")
+ ctx := context.Background()
+ put := func(bucket, key string) {
+ _, err := b.PutObject(ctx, bucket, key, nil, strings.NewReader(""), 0)
+ require.NoError(t, err)
+ }
+ del := func(key string) {
+ require.NoError(t, b.deleteObject(ctx, "bucket", key))
+ }
+ all := func() []string {
+ return listPages(t, b, listPrefix("", ""), 1000)
+ }
+
+ put("bucket", "dir/")
+ put("bucket", "dir/a")
+ put("bucket", "dir/sub/b")
+ want := []string{"dir/", "dir/a", "dir/sub/b", "implicit/file"}
+ assert.Equal(t, want, all())
+ assert.Equal(t, want, listPages(t, b, listPrefix("", ""), 1))
+ keys, _ := listOnce(t, b, listPrefix("", "/"), gofakes3.ListBucketPage{})
+ assert.Equal(t, []string{"dir/", "implicit/"}, keys)
+ keys, _ = listOnce(t, b, listPrefix("dir/", "/"), gofakes3.ListBucketPage{})
+ assert.Equal(t, []string{"dir/", "dir/a", "dir/sub/"}, keys)
+ keys, _ = listOnce(t, b, listPrefix("", "r"), gofakes3.ListBucketPage{})
+ assert.Equal(t, []string{"dir", "implicit/file"}, keys)
+
+ // The marker keeps its directory when the keys below it are deleted
+ del("dir/a")
+ del("dir/sub/b")
+ assert.Equal(t, []string{"dir/", "implicit/file"}, all())
+
+ // Deleting the marker leaves the keys below it, and then the
+ // directory goes with the last of them
+ put("bucket", "dir/c")
+ del("dir/")
+ assert.Equal(t, []string{"dir/c", "implicit/file"}, all())
+ del("dir/c")
+ assert.Equal(t, []string{"implicit/file"}, all())
+
+ // The markers of a deleted bucket are forgotten
+ require.NoError(t, b.CreateBucket(ctx, "other"))
+ put("other", "d/")
+ require.NoError(t, b.DeleteBucket(ctx, "other"))
+ require.NoError(t, b.CreateBucket(ctx, "other"))
+ put("other", "d/f")
+ list, err := b.ListBucket(ctx, "other", listPrefix("", ""), gofakes3.ListBucketPage{})
+ require.NoError(t, err)
+ require.Len(t, list.Contents, 1)
+ assert.Equal(t, "d/f", list.Contents[0].Key)
+}
diff --git a/cmd/serve/s3/serve_s3.md b/cmd/serve/s3/serve_s3.md
index 5cd22bd9d..929abcdde 100644
--- a/cmd/serve/s3/serve_s3.md
+++ b/cmd/serve/s3/serve_s3.md
@@ -488,9 +488,14 @@ for the directory `dir`. Putting an empty directory marker creates the
directory (one with data is rejected with `400 Bad Request`), and
deleting it removes the directory if it is empty. Directory markers
are found by `HeadObject`, `GetObject` and `CopyObject` for any
-directory, but only the markers of empty directories are listed as
-objects, as other directories may not have been made by putting a
-marker.
+directory, but only the markers of empty directories, and of the
+directories a marker was put for, are listed as objects, as other
+directories may not have been made by putting a marker. The markers
+put are remembered in memory with the metadata (see
+[Limitations](#limitations)), so after a restart, or once forgotten,
+the marker of a directory holding other keys is no longer listed.
+While a marker is remembered its directory isn't removed when the
+last key below it is deleted.
As keys are file paths, a key can't be both an object and a directory
holding other objects, for example `a/b` and `a/b/c`. Writing an
@@ -518,6 +523,7 @@ ignored.
| Authentication | Partial | AWS Signature Version 4, in headers, presigned URLs and browser form (`POST`) uploads. Signature Version 2 isn't supported. |
| Anonymous access | Partial | All requests are allowed without `--auth-key` or `--auth-proxy`, and none are with them, as buckets and objects can't be made public. |
| Metadata | Partial | `x-amz-meta-*` and the standard headers such as `Content-Type` are kept in memory only (see [Limitations](#limitations)), except the modification time. |
+| Directory markers | Partial | Keys ending in `/` are stored as directories, so can't hold data. A marker put for a directory holding other keys is only listed while it is remembered in memory (see [Object names](#object-names)). |
| Checksums | Partial | The `x-amz-checksum-*` of a `PutObject` (CRC32, CRC32C, CRC64NVME, SHA1 or SHA256) is checked and returned with the metadata. The checksums of multipart uploads aren't supported. |
| `GetObjectAttributes` | Partial | Everything but the checksums of multipart uploads. |
| Object tagging | Partial | `GetObjectTagging`, `PutObjectTagging` and `DeleteObjectTagging` work, but tags are kept in memory with the metadata. |
diff --git a/cmd/serve/s3/utils.go b/cmd/serve/s3/utils.go
index 790755f3b..f151d7ab9 100644
--- a/cmd/serve/s3/utils.go
+++ b/cmd/serve/s3/utils.go
@@ -228,8 +228,8 @@ func removeMadeDirs(dir, made string, VFS *vfs.VFS) {
}
// rmdirRecursive removes the parent directories of p which are empty,
-// stopping at the bucket.
-func rmdirRecursive(p string, VFS *vfs.VFS) {
+// stopping at the bucket or at a directory for which keep is true.
+func rmdirRecursive(p string, VFS *vfs.VFS, keep func(dir string) bool) {
dir := path.Dir(p)
if !strings.ContainsAny(dir, "/\\") {
// might be bucket(root)
@@ -237,13 +237,13 @@ func rmdirRecursive(p string, VFS *vfs.VFS) {
}
// Check the directory is empty first as the VFS logs an error
// removing one which isn't
- if !isEmptyDir(dir, VFS) {
+ if keep(dir) || !isEmptyDir(dir, VFS) {
return
}
if err := VFS.Remove(dir); err != nil {
return
}
- rmdirRecursive(dir, VFS)
+ rmdirRecursive(dir, VFS, keep)
}
// pathLocks are locks on paths, made as needed.