Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

serve s3: add --directory-markers to list every directory as a marker

Nick Craig-Wood4dfbe7f32085
diff --git a/cmd/serve/s3/backend.go b/cmd/serve/s3/backend.go
index a1e86ad32..c8ac6b74c 100644
--- a/cmd/serve/s3/backend.go
+++ b/cmd/serve/s3/backend.go
@@ -257,9 +257,12 @@ func (t *tenant) storeDirMarker(fp string) {
t.meta.Add(fp, objectMeta{dirMarker: true})
}
-// hasDirMarker reports whether a directory marker was put for the
-// directory at fp.
+// hasDirMarker reports whether the directory at fp has a directory
+// marker: one was put for it, or --directory-markers is set.
func (t *tenant) hasDirMarker(fp string) bool {
+ if t.b.s.opt.DirectoryMarkers {
+ return true
+ }
om, ok := t.meta.Get(fp)
return ok && om.dirMarker
}
diff --git a/cmd/serve/s3/list_test.go b/cmd/serve/s3/list_test.go
index 3d0b25c9b..a5839b5ef 100644
--- a/cmd/serve/s3/list_test.go
+++ b/cmd/serve/s3/list_test.go
@@ -24,6 +24,11 @@ import (
// newListBackend serves a temporary directory populated with the given files,
// whose paths are relative to the serve root and so start with a bucket name.
func newListBackend(t *testing.T, files ...string) *s3Backend {
+ return newListBackendOpt(t, Opt, files...)
+}
+
+// newListBackendOpt is newListBackend with the options opt.
+func newListBackendOpt(t *testing.T, opt Options, files ...string) *s3Backend {
fstest.Initialise()
ctx := context.Background()
root := t.TempDir()
@@ -36,7 +41,6 @@ func newListBackend(t *testing.T, files ...string) *s3Backend {
f, err := fs.NewFs(ctx, root)
require.NoError(t, err)
- opt := Opt
opt.HTTP.ListenAddr = []string{endpoint}
w, err := newServer(ctx, f, &opt, &vfscommon.Opt, &proxy.Opt)
require.NoError(t, err)
@@ -422,3 +426,29 @@ func TestListStorageClass(t *testing.T) {
"plain": gofakes3.StorageStandard,
}, got)
}
+
+// TestListDirectoryMarkers checks every directory is listed as a
+// directory marker with --directory-markers.
+func TestListDirectoryMarkers(t *testing.T) {
+ opt := Opt
+ opt.DirectoryMarkers = true
+ b := newListBackendOpt(t, opt, "bucket/implicit/file", "bucket/a/b/c")
+ ctx := context.Background()
+ all := func() []string {
+ return listPages(t, b, listPrefix("", ""), 1000)
+ }
+
+ want := []string{"a/", "a/b/", "a/b/c", "implicit/", "implicit/file"}
+ assert.Equal(t, want, all())
+ assert.Equal(t, want, listPages(t, b, listPrefix("", ""), 1))
+ keys, _ := listOnce(t, b, listPrefix("a/", "/"), gofakes3.ListBucketPage{})
+ assert.Equal(t, []string{"a/", "a/b/"}, keys)
+ keys, _ = listOnce(t, b, listPrefix("", "b"), gofakes3.ListBucketPage{})
+ assert.Equal(t, []string{"a/", "a/b", "implicit/", "implicit/file"}, keys)
+
+ // The directories are kept when the keys below them are deleted
+ require.NoError(t, b.deleteObject(ctx, "bucket", "a/b/c"))
+ assert.Equal(t, []string{"a/", "a/b/", "implicit/", "implicit/file"}, all())
+ require.NoError(t, b.deleteObject(ctx, "bucket", "a/b/"))
+ assert.Equal(t, []string{"a/", "implicit/", "implicit/file"}, all())
+}
diff --git a/cmd/serve/s3/s3.go b/cmd/serve/s3/s3.go
index 47a65a04b..c69812ee5 100644
--- a/cmd/serve/s3/s3.go
+++ b/cmd/serve/s3/s3.go
@@ -57,6 +57,10 @@ var OptionsInfo = fs.Options{{
Name: "metadata_max_objects",
Default: 100000,
Help: "Maximum number of objects of a user whose metadata is kept in memory, 0 for unlimited",
+}, {
+ Name: "directory_markers",
+ Default: false,
+ Help: "List every directory as a directory marker object",
}, {
Name: "multipart_expiry",
Default: fs.Duration(24 * time.Hour),
@@ -78,6 +82,7 @@ type Options struct {
MultipartMaxUploads int `config:"multipart_max_uploads"`
MetadataMaxObjects int `config:"metadata_max_objects"`
MultipartExpiry fs.Duration `config:"multipart_expiry"`
+ DirectoryMarkers bool `config:"directory_markers"`
Auth httplib.AuthConfig
HTTP httplib.Config
}
diff --git a/cmd/serve/s3/serve_s3.md b/cmd/serve/s3/serve_s3.md
index 6711745b0..c0155aa44 100644
--- a/cmd/serve/s3/serve_s3.md
+++ b/cmd/serve/s3/serve_s3.md
@@ -502,6 +502,13 @@ the marker of a directory holding other keys is no longer listed.
While a marker is remembered its directory isn't removed when the
last key below it is deleted.
+With `--directory-markers` every directory is listed as a directory
+marker, so the markers are listed after a restart too, as needed by
+clients which rely on them, such as rclone's s3 backend with
+`--s3-directory-markers`. As every directory has a marker, a directory
+isn't removed when the last key below it is deleted, and deleting the
+marker of a directory which holds other keys has no effect.
+
As keys are file paths, a key can't be both an object and a directory
holding other objects, for example `a/b` and `a/b/c`. Writing an
object which would need this fails with `400 Bad Request`
@@ -528,7 +535,7 @@ ignored.
| Authentication | Partial | AWS Signature Version 4, in headers, presigned URLs and browser form (`POST`) uploads. Signature Version 2 isn't supported. |
| Anonymous access | Partial | All requests are allowed without `--auth-key` or `--auth-proxy`, and none are with them, as buckets and objects can't be made public. |
| Metadata | Partial | `x-amz-meta-*`, the storage class and the standard headers such as `Content-Type` are kept in memory only (see [Limitations](#limitations)), except the modification time. The storage class is only recorded, so doesn't change how the object is stored. |
-| Directory markers | Partial | Keys ending in `/` are stored as directories, so can't hold data. A marker put for a directory holding other keys is only listed while it is remembered in memory (see [Object names](#object-names)). |
+| Directory markers | Partial | Keys ending in `/` are stored as directories, so can't hold data. A marker put for a directory holding other keys is only listed while it is remembered in memory, unless `--directory-markers` is set (see [Object names](#object-names)). |
| Checksums | Partial | The `x-amz-checksum-*` of a `PutObject` (CRC32, CRC32C, CRC64NVME, SHA1 or SHA256) is checked and returned with the metadata. The checksums of multipart uploads aren't supported. |
| `GetObjectAttributes` | Partial | Everything but the checksums of multipart uploads. |
| Object tagging | Partial | `GetObjectTagging`, `PutObjectTagging` and `DeleteObjectTagging` work, but tags are kept in memory with the metadata. |