Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

serve s3: remove a directory with its last key after its marker is deleted

Nick Craig-Wood399fd8e102f9
diff --git a/cmd/serve/s3/backend.go b/cmd/serve/s3/backend.go
index c8ac6b74c..0847eddbc 100644
--- a/cmd/serve/s3/backend.go
+++ b/cmd/serve/s3/backend.go
@@ -94,12 +94,13 @@ type tenant struct {
// objectMeta is the metadata stored for an object, with the size and
// modification time the object had when it was stored.
type objectMeta struct {
- meta map[string]string
- hash []byte // the ETag of the object, if not its hash
- partSizes []int64 // the sizes of the parts of an object made by a multipart upload
- size int64
- modTime time.Time
- dirMarker bool // a directory marker was put for the directory at this path
+ meta map[string]string
+ hash []byte // the ETag of the object, if not its hash
+ partSizes []int64 // the sizes of the parts of an object made by a multipart upload
+ size int64
+ modTime time.Time
+ dirMarker bool // a directory marker was put for the directory at this path
+ noDirMarker bool // the directory marker of the directory at this path was deleted, with --directory-markers
}
// newMetadataStore makes a store for the metadata of up to maxObjects
@@ -161,7 +162,7 @@ func (t *tenant) context() context.Context {
// stored, as it has been replaced by something other than this user.
func (t *tenant) getMeta(ctx context.Context, fp string, node vfs.Node) (om objectMeta, ok bool) {
om, ok = t.meta.Get(fp)
- if !ok || om.dirMarker || om.size != node.Size() {
+ if !ok || om.dirMarker || om.noDirMarker || om.size != node.Size() {
return objectMeta{}, false
}
window := fs.GetModifyWindow(ctx, node.VFS().Fs())
@@ -258,12 +259,13 @@ func (t *tenant) storeDirMarker(fp string) {
}
// hasDirMarker reports whether the directory at fp has a directory
-// marker: one was put for it, or --directory-markers is set.
+// marker: one was put for it, or --directory-markers is set and it
+// hasn't been deleted.
func (t *tenant) hasDirMarker(fp string) bool {
+ om, ok := t.meta.Get(fp)
if t.b.s.opt.DirectoryMarkers {
- return true
+ return !ok || !om.noDirMarker
}
- om, ok := t.meta.Get(fp)
return ok && om.dirMarker
}
@@ -800,7 +802,10 @@ func (b *s3Backend) deleteObject(ctx context.Context, bucketName, objectName str
if err := _vfs.Remove(fp); err != nil {
return err
}
- rmdirRecursive(fp, _vfs, t.hasDirMarker)
+ rmdirRecursive(fp, _vfs, t)
+ } else if b.s.opt.DirectoryMarkers {
+ // Otherwise it is removed with the last key below it
+ t.meta.Add(fp, objectMeta{noDirMarker: true})
}
return nil
}
@@ -816,7 +821,7 @@ func (b *s3Backend) deleteObject(ctx context.Context, bucketName, objectName str
t.removeMeta(fp)
// FIXME: unsafe operation
- rmdirRecursive(fp, _vfs, t.hasDirMarker)
+ rmdirRecursive(fp, _vfs, t)
return nil
}
diff --git a/cmd/serve/s3/list_test.go b/cmd/serve/s3/list_test.go
index a5839b5ef..dc4a754ad 100644
--- a/cmd/serve/s3/list_test.go
+++ b/cmd/serve/s3/list_test.go
@@ -451,4 +451,22 @@ func TestListDirectoryMarkers(t *testing.T) {
assert.Equal(t, []string{"a/", "a/b/", "implicit/", "implicit/file"}, all())
require.NoError(t, b.deleteObject(ctx, "bucket", "a/b/"))
assert.Equal(t, []string{"a/", "implicit/", "implicit/file"}, all())
+
+ // A marker deleted while its directory holds other keys isn't
+ // listed, and the directory goes with the last key below it, in
+ // whatever order the keys are deleted
+ _, err := b.PutObject(ctx, "bucket", "x/y/z", nil, strings.NewReader("z"), 1)
+ require.NoError(t, err)
+ assert.Equal(t, []string{"a/", "implicit/", "implicit/file", "x/", "x/y/", "x/y/z"}, all())
+ require.NoError(t, b.deleteObject(ctx, "bucket", "x/"))
+ assert.Equal(t, []string{"a/", "implicit/", "implicit/file", "x/y/", "x/y/z"}, all())
+ require.NoError(t, b.deleteObject(ctx, "bucket", "x/y/z"))
+ assert.Equal(t, []string{"a/", "implicit/", "implicit/file", "x/y/"}, all())
+ require.NoError(t, b.deleteObject(ctx, "bucket", "x/y/"))
+ assert.Equal(t, []string{"a/", "implicit/", "implicit/file"}, all())
+
+ // A directory made again has its marker
+ _, err = b.PutObject(ctx, "bucket", "x/w", nil, strings.NewReader("w"), 1)
+ require.NoError(t, err)
+ assert.Equal(t, []string{"a/", "implicit/", "implicit/file", "x/", "x/w"}, all())
}
diff --git a/cmd/serve/s3/serve_s3.md b/cmd/serve/s3/serve_s3.md
index 4f3943ad1..e8660a117 100644
--- a/cmd/serve/s3/serve_s3.md
+++ b/cmd/serve/s3/serve_s3.md
@@ -506,8 +506,11 @@ With `--directory-markers` every directory is listed as a directory
marker, so the markers are listed after a restart too, as needed by
clients which rely on them, such as rclone's s3 backend with
`--s3-directory-markers`. As every directory has a marker, a directory
-isn't removed when the last key below it is deleted, and deleting the
-marker of a directory which holds other keys has no effect.
+isn't removed when the last key below it is deleted. Deleting the
+marker of a directory which holds other keys stops it being listed,
+and the directory is then removed with the last key below it, but this
+is only remembered in memory, like the markers put without
+`--directory-markers`.
As keys are file paths, a key can't be both an object and a directory
holding other objects, for example `a/b` and `a/b/c`. Writing an
diff --git a/cmd/serve/s3/utils.go b/cmd/serve/s3/utils.go
index f151d7ab9..5e6caeca6 100644
--- a/cmd/serve/s3/utils.go
+++ b/cmd/serve/s3/utils.go
@@ -228,8 +228,9 @@ func removeMadeDirs(dir, made string, VFS *vfs.VFS) {
}
// rmdirRecursive removes the parent directories of p which are empty,
-// stopping at the bucket or at a directory for which keep is true.
-func rmdirRecursive(p string, VFS *vfs.VFS, keep func(dir string) bool) {
+// stopping at the bucket or at a directory with a directory marker of
+// the user t, and forgets what t remembers of the directories removed.
+func rmdirRecursive(p string, VFS *vfs.VFS, t *tenant) {
dir := path.Dir(p)
if !strings.ContainsAny(dir, "/\\") {
// might be bucket(root)
@@ -237,13 +238,14 @@ func rmdirRecursive(p string, VFS *vfs.VFS, keep func(dir string) bool) {
}
// Check the directory is empty first as the VFS logs an error
// removing one which isn't
- if keep(dir) || !isEmptyDir(dir, VFS) {
+ if t.hasDirMarker(dir) || !isEmptyDir(dir, VFS) {
return
}
if err := VFS.Remove(dir); err != nil {
return
}
- rmdirRecursive(dir, VFS, keep)
+ t.removeMeta(dir)
+ rmdirRecursive(dir, VFS, t)
}
// pathLocks are locks on paths, made as needed.