"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
serve s3: add --directory-markers to list every directory as a marker
Nick Craig-Wood4dfbe7f32085
diff --git a/cmd/serve/s3/backend.go b/cmd/serve/s3/backend.goindex a1e86ad32..c8ac6b74c 100644--- a/cmd/serve/s3/backend.go+++ b/cmd/serve/s3/backend.go@@ -257,9 +257,12 @@ func (t *tenant) storeDirMarker(fp string) {t.meta.Add(fp, objectMeta{dirMarker: true})}-// hasDirMarker reports whether a directory marker was put for the-// directory at fp.+// hasDirMarker reports whether the directory at fp has a directory+// marker: one was put for it, or --directory-markers is set.func (t *tenant) hasDirMarker(fp string) bool {+ if t.b.s.opt.DirectoryMarkers {+ return true+ }om, ok := t.meta.Get(fp)return ok && om.dirMarker}diff --git a/cmd/serve/s3/list_test.go b/cmd/serve/s3/list_test.goindex 3d0b25c9b..a5839b5ef 100644--- a/cmd/serve/s3/list_test.go+++ b/cmd/serve/s3/list_test.go@@ -24,6 +24,11 @@ import (// newListBackend serves a temporary directory populated with the given files,// whose paths are relative to the serve root and so start with a bucket name.func newListBackend(t *testing.T, files ...string) *s3Backend {+ return newListBackendOpt(t, Opt, files...)+}++// newListBackendOpt is newListBackend with the options opt.+func newListBackendOpt(t *testing.T, opt Options, files ...string) *s3Backend {fstest.Initialise()ctx := context.Background()root := t.TempDir()@@ -36,7 +41,6 @@ func newListBackend(t *testing.T, files ...string) *s3Backend {f, err := fs.NewFs(ctx, root)require.NoError(t, err)- opt := Optopt.HTTP.ListenAddr = []string{endpoint}w, err := newServer(ctx, f, &opt, &vfscommon.Opt, &proxy.Opt)require.NoError(t, err)@@ -422,3 +426,29 @@ func TestListStorageClass(t *testing.T) {"plain": gofakes3.StorageStandard,}, got)}++// TestListDirectoryMarkers checks every directory is listed as a+// directory marker with --directory-markers.+func TestListDirectoryMarkers(t *testing.T) {+ opt := Opt+ opt.DirectoryMarkers = true+ b := newListBackendOpt(t, opt, "bucket/implicit/file", "bucket/a/b/c")+ ctx := context.Background()+ all := func() []string {+ return listPages(t, b, listPrefix("", ""), 1000)+ }++ want := []string{"a/", "a/b/", "a/b/c", "implicit/", "implicit/file"}+ assert.Equal(t, want, all())+ assert.Equal(t, want, listPages(t, b, listPrefix("", ""), 1))+ keys, _ := listOnce(t, b, listPrefix("a/", "/"), gofakes3.ListBucketPage{})+ assert.Equal(t, []string{"a/", "a/b/"}, keys)+ keys, _ = listOnce(t, b, listPrefix("", "b"), gofakes3.ListBucketPage{})+ assert.Equal(t, []string{"a/", "a/b", "implicit/", "implicit/file"}, keys)++ // The directories are kept when the keys below them are deleted+ require.NoError(t, b.deleteObject(ctx, "bucket", "a/b/c"))+ assert.Equal(t, []string{"a/", "a/b/", "implicit/", "implicit/file"}, all())+ require.NoError(t, b.deleteObject(ctx, "bucket", "a/b/"))+ assert.Equal(t, []string{"a/", "implicit/", "implicit/file"}, all())+}diff --git a/cmd/serve/s3/s3.go b/cmd/serve/s3/s3.goindex 47a65a04b..c69812ee5 100644--- a/cmd/serve/s3/s3.go+++ b/cmd/serve/s3/s3.go@@ -57,6 +57,10 @@ var OptionsInfo = fs.Options{{Name: "metadata_max_objects",Default: 100000,Help: "Maximum number of objects of a user whose metadata is kept in memory, 0 for unlimited",+}, {+ Name: "directory_markers",+ Default: false,+ Help: "List every directory as a directory marker object",}, {Name: "multipart_expiry",Default: fs.Duration(24 * time.Hour),@@ -78,6 +82,7 @@ type Options struct {MultipartMaxUploads int `config:"multipart_max_uploads"`MetadataMaxObjects int `config:"metadata_max_objects"`MultipartExpiry fs.Duration `config:"multipart_expiry"`+ DirectoryMarkers bool `config:"directory_markers"`Auth httplib.AuthConfigHTTP httplib.Config}diff --git a/cmd/serve/s3/serve_s3.md b/cmd/serve/s3/serve_s3.mdindex 6711745b0..c0155aa44 100644--- a/cmd/serve/s3/serve_s3.md+++ b/cmd/serve/s3/serve_s3.md@@ -502,6 +502,13 @@ the marker of a directory holding other keys is no longer listed.While a marker is remembered its directory isn't removed when thelast key below it is deleted.+With `--directory-markers` every directory is listed as a directory+marker, so the markers are listed after a restart too, as needed by+clients which rely on them, such as rclone's s3 backend with+`--s3-directory-markers`. As every directory has a marker, a directory+isn't removed when the last key below it is deleted, and deleting the+marker of a directory which holds other keys has no effect.+As keys are file paths, a key can't be both an object and a directoryholding other objects, for example `a/b` and `a/b/c`. Writing anobject which would need this fails with `400 Bad Request`@@ -528,7 +535,7 @@ ignored.| Authentication | Partial | AWS Signature Version 4, in headers, presigned URLs and browser form (`POST`) uploads. Signature Version 2 isn't supported. || Anonymous access | Partial | All requests are allowed without `--auth-key` or `--auth-proxy`, and none are with them, as buckets and objects can't be made public. || Metadata | Partial | `x-amz-meta-*`, the storage class and the standard headers such as `Content-Type` are kept in memory only (see [Limitations](#limitations)), except the modification time. The storage class is only recorded, so doesn't change how the object is stored. |-| Directory markers | Partial | Keys ending in `/` are stored as directories, so can't hold data. A marker put for a directory holding other keys is only listed while it is remembered in memory (see [Object names](#object-names)). |+| Directory markers | Partial | Keys ending in `/` are stored as directories, so can't hold data. A marker put for a directory holding other keys is only listed while it is remembered in memory, unless `--directory-markers` is set (see [Object names](#object-names)). || Checksums | Partial | The `x-amz-checksum-*` of a `PutObject` (CRC32, CRC32C, CRC64NVME, SHA1 or SHA256) is checked and returned with the metadata. The checksums of multipart uploads aren't supported. || `GetObjectAttributes` | Partial | Everything but the checksums of multipart uploads. || Object tagging | Partial | `GetObjectTagging`, `PutObjectTagging` and `DeleteObjectTagging` work, but tags are kept in memory with the metadata. |