Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

serve s3: list empty directories as directory markers

Nick Craig-Wood617476d0e7a6
diff --git a/cmd/serve/s3/list.go b/cmd/serve/s3/list.go
index b238c2b99..0bc3ba7ba 100644
--- a/cmd/serve/s3/list.go
+++ b/cmd/serve/s3/list.go
@@ -131,6 +131,32 @@ func (l *lister) addObject(key string, entry vfs.Node) error {
return nil
}
+// addDirMarker adds the directory marker key of the empty directory
+// fp to the page, returning errPageFull if the page is now complete.
+func (l *lister) addDirMarker(key, fp string) error {
+ if l.skipKey(key) {
+ return nil
+ }
+ node, err := l.vfs.Stat(fp)
+ if err != nil {
+ // The directory went away while we were listing it
+ return nil
+ }
+ if l.keys >= l.max {
+ return l.truncate()
+ }
+ l.response.Add(&gofakes3.Content{
+ Key: key,
+ LastModified: gofakes3.NewContentTime(node.ModTime()),
+ ETag: quoteETag(emptyHash(l.hashType)),
+ Size: 0,
+ StorageClass: gofakes3.StorageStandard,
+ })
+ l.keys++
+ l.lastKey = key
+ return nil
+}
+
// commonPrefix returns the common prefix key rolls up into, if the
// listing has a delimiter other than "/" and key has it after the
// listing's prefix.
@@ -173,6 +199,10 @@ func (l *lister) hasKeys(fdPath string) bool {
if err != nil {
return false
}
+ if len(dirEntries) == 0 {
+ // An empty directory has its directory marker key
+ return true
+ }
for _, entry := range dirEntries {
if isTempObject(entry.Name()) {
continue
@@ -211,6 +241,13 @@ func (l *lister) list(fdPath, name string, addPrefix bool) error {
}
l.dirsRead++
+ // An empty directory is listed as its directory marker, which is
+ // the only key below it, as S3 would list the marker which made
+ // it. Other directories may have been made implicitly, so aren't.
+ if len(dirEntries) == 0 && name == "" && fdPath != "" {
+ return l.addDirMarker(fdPath+"/", fp)
+ }
+
// Emit the entries in the order their keys have in a flat keyspace,
// which is not the plain name order getDirEntries returns.
slices.SortFunc(dirEntries, func(a, b vfs.Node) int {
diff --git a/cmd/serve/s3/list_test.go b/cmd/serve/s3/list_test.go
index 0851162a1..7a08e20f1 100644
--- a/cmd/serve/s3/list_test.go
+++ b/cmd/serve/s3/list_test.go
@@ -277,19 +277,22 @@ func TestListAnyDelimiter(t *testing.T) {
for _, file := range files {
paths = append(paths, "bucket/"+file)
}
+ // A directory holding only the temporary object of an upload has
+ // no keys
+ paths = append(paths, "bucket/pending/"+putObjectPrefix+"x")
b := newListBackend(t, paths...)
- // An empty directory has no keys, so mustn't be a common prefix
- // for delimiters other than "/"
+ // An empty directory has its directory marker as its key
require.NoError(t, os.MkdirAll(filepath.Join(b.s.f.Root(), "bucket", "empty", "abc"), 0777))
+ files = append(files, "empty/abc/")
// want returns the keys and common prefixes S3 lists
want := func(prefix, delimiter string) (keys []string) {
seen := map[string]bool{}
keySpace := files
- if delimiter == "/" {
- // Listing directories with "/" shows empty ones too, as
- // it always has
- keySpace = append(slices.Clone(files), "empty/abc/")
+ if delimiter == "/" && prefix != "pending/" {
+ // Listing directories with "/" shows every directory
+ // as a common prefix, as it always has
+ keySpace = append(slices.Clone(files), "pending/")
}
for _, key := range keySpace {
if !strings.HasPrefix(key, prefix) {
@@ -309,7 +312,7 @@ func TestListAnyDelimiter(t *testing.T) {
return keys
}
- for _, prefix := range []string{"", "b", "ba", "baz/", "c", "c/a", "e", "x"} {
+ for _, prefix := range []string{"", "b", "ba", "baz/", "c", "c/a", "e", "empty/", "empty/abc", "empty/abc/", "p", "pending/", "x"} {
for _, delimiter := range []string{"", "/", "a", "ba", " ", ".", "%", "/a", "b/", "z"} {
t.Run(fmt.Sprintf("prefix=%q,delimiter=%q", prefix, delimiter), func(t *testing.T) {
expected := want(prefix, delimiter)
diff --git a/cmd/serve/s3/multipart_test.go b/cmd/serve/s3/multipart_test.go
index 94246f160..8fc710b1c 100644
--- a/cmd/serve/s3/multipart_test.go
+++ b/cmd/serve/s3/multipart_test.go
@@ -2034,7 +2034,6 @@ func TestETagHashConsistent(t *testing.T) {
for _, obj := range result.Contents {
listed[obj.Key] = strings.Trim(obj.ETag, `"`)
}
- delete(etags, "dir/")
assert.Equal(t, etags, listed)
})
}
diff --git a/cmd/serve/s3/serve_s3.md b/cmd/serve/s3/serve_s3.md
index 037a3b244..3a6fdc0e8 100644
--- a/cmd/serve/s3/serve_s3.md
+++ b/cmd/serve/s3/serve_s3.md
@@ -451,7 +451,8 @@ When using `ListObjects` with the delimiter `/`, rclone lists the
directory the prefix is in, showing its subdirectories as common
prefixes, including empty ones. Without a delimiter the listing is
recursive: rclone walks the directory tree below the prefix, reading
-only as many directories as it needs to fill each page. Any other
+only as many directories as it needs to fill each page, and lists
+empty directories as their directory markers (see below). Any other
delimiter is supported too, but needs the whole directory tree below
the prefix walking, unless the delimiter is in the directory names.
@@ -492,8 +493,9 @@ A key ending in `/`, such as `dir/`, is a directory marker, standing
for the directory `dir`. Putting an empty directory marker creates the
directory (one with data is rejected with `400 Bad Request`), and
deleting it removes the directory if it is empty. Directory markers
-are found by `HeadObject` and `GetObject` for any directory, but
-aren't listed as objects.
+are found by `HeadObject` and `GetObject` for any directory, but only
+the markers of empty directories are listed as objects, as other
+directories may not have been made by putting a marker.
As keys are file paths, a key can't be both an object and a directory
holding other objects, for example `a/b` and `a/b/c`. Writing an