"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
serve s3: list empty directories as directory markers
Nick Craig-Wood617476d0e7a6
diff --git a/cmd/serve/s3/list.go b/cmd/serve/s3/list.goindex b238c2b99..0bc3ba7ba 100644--- a/cmd/serve/s3/list.go+++ b/cmd/serve/s3/list.go@@ -131,6 +131,32 @@ func (l *lister) addObject(key string, entry vfs.Node) error {return nil}+// addDirMarker adds the directory marker key of the empty directory+// fp to the page, returning errPageFull if the page is now complete.+func (l *lister) addDirMarker(key, fp string) error {+ if l.skipKey(key) {+ return nil+ }+ node, err := l.vfs.Stat(fp)+ if err != nil {+ // The directory went away while we were listing it+ return nil+ }+ if l.keys >= l.max {+ return l.truncate()+ }+ l.response.Add(&gofakes3.Content{+ Key: key,+ LastModified: gofakes3.NewContentTime(node.ModTime()),+ ETag: quoteETag(emptyHash(l.hashType)),+ Size: 0,+ StorageClass: gofakes3.StorageStandard,+ })+ l.keys+++ l.lastKey = key+ return nil+}+// commonPrefix returns the common prefix key rolls up into, if the// listing has a delimiter other than "/" and key has it after the// listing's prefix.@@ -173,6 +199,10 @@ func (l *lister) hasKeys(fdPath string) bool {if err != nil {return false}+ if len(dirEntries) == 0 {+ // An empty directory has its directory marker key+ return true+ }for _, entry := range dirEntries {if isTempObject(entry.Name()) {continue@@ -211,6 +241,13 @@ func (l *lister) list(fdPath, name string, addPrefix bool) error {}l.dirsRead+++ // An empty directory is listed as its directory marker, which is+ // the only key below it, as S3 would list the marker which made+ // it. Other directories may have been made implicitly, so aren't.+ if len(dirEntries) == 0 && name == "" && fdPath != "" {+ return l.addDirMarker(fdPath+"/", fp)+ }+// Emit the entries in the order their keys have in a flat keyspace,// which is not the plain name order getDirEntries returns.slices.SortFunc(dirEntries, func(a, b vfs.Node) int {diff --git a/cmd/serve/s3/list_test.go b/cmd/serve/s3/list_test.goindex 0851162a1..7a08e20f1 100644--- a/cmd/serve/s3/list_test.go+++ b/cmd/serve/s3/list_test.go@@ -277,19 +277,22 @@ func TestListAnyDelimiter(t *testing.T) {for _, file := range files {paths = append(paths, "bucket/"+file)}+ // A directory holding only the temporary object of an upload has+ // no keys+ paths = append(paths, "bucket/pending/"+putObjectPrefix+"x")b := newListBackend(t, paths...)- // An empty directory has no keys, so mustn't be a common prefix- // for delimiters other than "/"+ // An empty directory has its directory marker as its keyrequire.NoError(t, os.MkdirAll(filepath.Join(b.s.f.Root(), "bucket", "empty", "abc"), 0777))+ files = append(files, "empty/abc/")// want returns the keys and common prefixes S3 listswant := func(prefix, delimiter string) (keys []string) {seen := map[string]bool{}keySpace := files- if delimiter == "/" {- // Listing directories with "/" shows empty ones too, as- // it always has- keySpace = append(slices.Clone(files), "empty/abc/")+ if delimiter == "/" && prefix != "pending/" {+ // Listing directories with "/" shows every directory+ // as a common prefix, as it always has+ keySpace = append(slices.Clone(files), "pending/")}for _, key := range keySpace {if !strings.HasPrefix(key, prefix) {@@ -309,7 +312,7 @@ func TestListAnyDelimiter(t *testing.T) {return keys}- for _, prefix := range []string{"", "b", "ba", "baz/", "c", "c/a", "e", "x"} {+ for _, prefix := range []string{"", "b", "ba", "baz/", "c", "c/a", "e", "empty/", "empty/abc", "empty/abc/", "p", "pending/", "x"} {for _, delimiter := range []string{"", "/", "a", "ba", " ", ".", "%", "/a", "b/", "z"} {t.Run(fmt.Sprintf("prefix=%q,delimiter=%q", prefix, delimiter), func(t *testing.T) {expected := want(prefix, delimiter)diff --git a/cmd/serve/s3/multipart_test.go b/cmd/serve/s3/multipart_test.goindex 94246f160..8fc710b1c 100644--- a/cmd/serve/s3/multipart_test.go+++ b/cmd/serve/s3/multipart_test.go@@ -2034,7 +2034,6 @@ func TestETagHashConsistent(t *testing.T) {for _, obj := range result.Contents {listed[obj.Key] = strings.Trim(obj.ETag, `"`)}- delete(etags, "dir/")assert.Equal(t, etags, listed)})}diff --git a/cmd/serve/s3/serve_s3.md b/cmd/serve/s3/serve_s3.mdindex 037a3b244..3a6fdc0e8 100644--- a/cmd/serve/s3/serve_s3.md+++ b/cmd/serve/s3/serve_s3.md@@ -451,7 +451,8 @@ When using `ListObjects` with the delimiter `/`, rclone lists thedirectory the prefix is in, showing its subdirectories as commonprefixes, including empty ones. Without a delimiter the listing isrecursive: rclone walks the directory tree below the prefix, reading-only as many directories as it needs to fill each page. Any other+only as many directories as it needs to fill each page, and lists+empty directories as their directory markers (see below). Any otherdelimiter is supported too, but needs the whole directory tree belowthe prefix walking, unless the delimiter is in the directory names.@@ -492,8 +493,9 @@ A key ending in `/`, such as `dir/`, is a directory marker, standingfor the directory `dir`. Putting an empty directory marker creates thedirectory (one with data is rejected with `400 Bad Request`), anddeleting it removes the directory if it is empty. Directory markers-are found by `HeadObject` and `GetObject` for any directory, but-aren't listed as objects.+are found by `HeadObject` and `GetObject` for any directory, but only+the markers of empty directories are listed as objects, as other+directories may not have been made by putting a marker.As keys are file paths, a key can't be both an object and a directoryholding other objects, for example `a/b` and `a/b/c`. Writing an