"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
serve s3: support listing with delimiters other than "/"
Nick Craig-Woodb80e7e8a7e58
diff --git a/cmd/serve/s3/backend.go b/cmd/serve/s3/backend.goindex f29f4f575..8da2e5830 100644--- a/cmd/serve/s3/backend.go+++ b/cmd/serve/s3/backend.go@@ -272,18 +272,13 @@ func (b *s3Backend) ListBucket(ctx context.Context, bucket string, prefix *gofakprefix = emptyPrefix}- // workaround- if strings.TrimSpace(prefix.Prefix) == "" {- prefix.HasPrefix = false- }- if strings.TrimSpace(prefix.Delimiter) == "" {- prefix.HasDelimiter = false- }-response := gofakes3.NewObjectList()- path, remaining := prefixParser(prefix)+ // Keys can't start with "/" so such a prefix matches nothing+ if strings.HasPrefix(prefix.Prefix, "/") {+ return response, nil+ }- err = b.listPage(_vfs, bucket, path, remaining, prefix.HasDelimiter, page, response)+ err = b.listPage(_vfs, bucket, prefix.Prefix, prefix.Delimiter, page, response)if err == gofakes3.ErrNoSuchKey {// AWS just returns an empty listresponse = gofakes3.NewObjectList()diff --git a/cmd/serve/s3/list.go b/cmd/serve/s3/list.goindex eb71d5021..f174d0037 100644--- a/cmd/serve/s3/list.go+++ b/cmd/serve/s3/list.go@@ -38,6 +38,13 @@ type lister struct {keys int // keys added to the page so farlastKey string // the last key added to the pagedirsRead int // directories read, for debug logging++ // With a delimiter other than "/", which can't be found by+ // listing directories, the whole tree is walked and the keys are+ // rolled up into common prefixes.+ prefix string // the prefix of the listing+ delimiter string // the delimiter, or "" if none or "/"+ lastPrefix string // the last common prefix added to the page}// newLister makes a lister which fills response with at most one page of the@@ -128,6 +135,65 @@ func (l *lister) addObject(key string, entry vfs.Node) error {return nil}+// commonPrefix returns the common prefix key rolls up into, if the+// listing has a delimiter other than "/" and key has it after the+// listing's prefix.+func (l *lister) commonPrefix(key string) (prefix string, ok bool) {+ if l.delimiter == "" || !strings.HasPrefix(key, l.prefix) {+ return "", false+ }+ i := strings.Index(key[len(l.prefix):], l.delimiter)+ if i < 0 {+ return "", false+ }+ return key[:len(l.prefix)+i+len(l.delimiter)], true+}++// addRolledUp adds the common prefix a key rolls up into to the page+// unless it has been added already, returning errPageFull if the page+// is now complete.+//+// The keys rolling up into a common prefix are contiguous in the+// listing, so it only needs comparing with the last one.+func (l *lister) addRolledUp(prefix string) error {+ if prefix == l.lastPrefix || l.skipKey(prefix) {+ return nil+ }+ if err := l.addCommonPrefix(prefix); err != nil {+ return err+ }+ l.lastPrefix = prefix+ return nil+}++// hasKeys reports whether the directory fp in the bucket has any keys+// below it.+func (l *lister) hasKeys(fdPath string) bool {+ fp, err := bucketDirPath(l.bucket, fdPath)+ if err != nil {+ return false+ }+ dirEntries, err := getDirEntries(fp, l.vfs)+ if err != nil {+ return false+ }+ for _, entry := range dirEntries {+ if isTempObject(entry.Name()) {+ continue+ }+ if !entry.IsDir() || l.hasKeys(path.Join(fdPath, entry.Name())) {+ return true+ }+ }+ return false+}++// isTempObject reports whether name is a temporary object of an+// in-progress upload, which is hidden from listings.+func isTempObject(name string) bool {+ return strings.HasPrefix(name, tempObjectPrefix) || strings.HasPrefix(name, legacyMultipartUploadPrefix)+}+// list walks the directory fdPath, adding the entries whose leaf name starts// with name to the page.//@@ -159,7 +225,7 @@ func (l *lister) list(fdPath, name string, addPrefix bool) error {object := entry.Name()// Hide the temporary objects of in-progress uploads- if strings.HasPrefix(object, tempObjectPrefix) || strings.HasPrefix(object, legacyMultipartUploadPrefix) {+ if isTempObject(object) {continue}@@ -184,6 +250,17 @@ func (l *lister) list(fdPath, name string, addPrefix bool) error {if l.skipDir(prefixWithTrailingSlash) {continue}+ // If the delimiter is in the directory's path every+ // key below it rolls up into the same common prefix,+ // so the directory needn't be read.+ if prefix, ok := l.commonPrefix(prefixWithTrailingSlash); ok {+ if l.hasKeys(objectPath) {+ if err := l.addRolledUp(prefix); err != nil {+ return err+ }+ }+ continue+ }err := l.list(objectPath, "", false)if errors.Is(err, gofakes3.ErrNoSuchKey) {// The directory went away while we were listing it, so@@ -193,6 +270,12 @@ func (l *lister) list(fdPath, name string, addPrefix bool) error {return err}} else {+ if prefix, ok := l.commonPrefix(objectPath); ok {+ if err := l.addRolledUp(prefix); err != nil {+ return err+ }+ continue+ }if l.skipKey(objectPath) {continue}@@ -204,16 +287,25 @@ func (l *lister) list(fdPath, name string, addPrefix bool) error {return nil}-// listPage fills response with one page of the listing of the directory-// fdPath in bucket, as described by page.-//-// Only entries whose leaf name starts with name are listed. If addPrefix is-// set, subdirectories are reported as common prefixes, otherwise the whole-// subtree below fdPath is listed.+// listPage fills response with one page of the listing of bucket with+// prefix and delimiter, as described by page.//-// It returns gofakes3.ErrNoSuchKey if fdPath is not a directory.-func (b *s3Backend) listPage(_vfs *vfs.VFS, bucket, fdPath, name string, addPrefix bool, page gofakes3.ListBucketPage, response *gofakes3.ObjectList) error {+// It returns gofakes3.ErrNoSuchKey if the directory the prefix is in+// doesn't exist.+func (b *s3Backend) listPage(_vfs *vfs.VFS, bucket, prefix, delimiter string, page gofakes3.ListBucketPage, response *gofakes3.ObjectList) error {l := newLister(b, _vfs, bucket, page, response)+ // The prefix is split into the directory to list and the start of+ // the names in it to list.+ fdPath, name := "", prefix+ if i := strings.LastIndexByte(prefix, '/'); i >= 0 {+ fdPath, name = prefix[:i], prefix[i+1:]+ }+ // A "/" delimiter is found by listing directories, any other by+ // walking the whole tree.+ addPrefix := delimiter == "/"+ if delimiter != "" && delimiter != "/" {+ l.prefix, l.delimiter = prefix, delimiter+ }err := l.list(fdPath, name, addPrefix)if err != nil && !errors.Is(err, errPageFull) {return errdiff --git a/cmd/serve/s3/list_test.go b/cmd/serve/s3/list_test.goindex 575f3313c..0851162a1 100644--- a/cmd/serve/s3/list_test.go+++ b/cmd/serve/s3/list_test.go@@ -8,6 +8,7 @@ import ("os""path/filepath""slices"+ "strings""testing""github.com/rclone/gofakes3"@@ -263,6 +264,74 @@ func TestListSkipDir(t *testing.T) {}}+// TestListAnyDelimiter checks listings with any delimiter, and with a+// prefix, against the keys an S3 listing of the same flat keyspace has,+// whatever the page size.+func TestListAnyDelimiter(t *testing.T) {+ files := []string{+ "a.txt", "a/b/c", "a/bc", "a0",+ "b%ar", "b ar", "b.ar", "bar", "bazar", "baz/qux", "baz/q ux/x",+ "cab", "c/a/b/z", "c/ab", "dir/sub/file", "foo",+ }+ var paths []string+ for _, file := range files {+ paths = append(paths, "bucket/"+file)+ }+ b := newListBackend(t, paths...)+ // An empty directory has no keys, so mustn't be a common prefix+ // for delimiters other than "/"+ require.NoError(t, os.MkdirAll(filepath.Join(b.s.f.Root(), "bucket", "empty", "abc"), 0777))++ // want returns the keys and common prefixes S3 lists+ want := func(prefix, delimiter string) (keys []string) {+ seen := map[string]bool{}+ keySpace := files+ if delimiter == "/" {+ // Listing directories with "/" shows empty ones too, as+ // it always has+ keySpace = append(slices.Clone(files), "empty/abc/")+ }+ for _, key := range keySpace {+ if !strings.HasPrefix(key, prefix) {+ continue+ }+ if delimiter != "" {+ if i := strings.Index(key[len(prefix):], delimiter); i >= 0 {+ key = key[:len(prefix)+i+len(delimiter)]+ }+ }+ if !seen[key] {+ seen[key] = true+ keys = append(keys, key)+ }+ }+ slices.Sort(keys)+ return keys+ }++ for _, prefix := range []string{"", "b", "ba", "baz/", "c", "c/a", "e", "x"} {+ for _, delimiter := range []string{"", "/", "a", "ba", " ", ".", "%", "/a", "b/", "z"} {+ t.Run(fmt.Sprintf("prefix=%q,delimiter=%q", prefix, delimiter), func(t *testing.T) {+ expected := want(prefix, delimiter)+ for _, maxKeys := range []int{1, 2, 3, 1000} {+ got := listPages(t, b, listPrefix(prefix, delimiter), maxKeys)+ assert.Equal(t, expected, got, "max-keys %d", maxKeys)+ }+ })+ }+ }+}++// TestListPrefixLeadingSlash checks a prefix starting with "/" matches+// nothing, as no key starts with "/".+func TestListPrefixLeadingSlash(t *testing.T) {+ b := newListBackend(t, "bucket/a/b", "bucket/c")+ keys, _ := listOnce(t, b, listPrefix("/", ""), gofakes3.ListBucketPage{MaxKeys: 1000})+ assert.Empty(t, keys)+ keys, _ = listOnce(t, b, listPrefix("/a", "/"), gofakes3.ListBucketPage{MaxKeys: 1000})+ assert.Empty(t, keys)+}+// TestListETagQuoted checks the ETags in listings are quoted as on S3// and in HEAD responses.func TestListETagQuoted(t *testing.T) {diff --git a/cmd/serve/s3/serve_s3.md b/cmd/serve/s3/serve_s3.mdindex 529b7d37f..fcc91a1cc 100644--- a/cmd/serve/s3/serve_s3.md+++ b/cmd/serve/s3/serve_s3.md@@ -441,10 +441,13 @@ When using `PutObject` or `DeleteObject`, rclone will automaticallycreate or clean up empty folders. If you don't want to clean up emptyfolders automatically, use `--no-cleanup`.-When using `ListObjects`, rclone treats any delimiter as `/`. Without a-delimiter the listing is recursive: rclone walks the directory tree-below the prefix, reading only as many directories as it needs to fill-each page.+When using `ListObjects` with the delimiter `/`, rclone lists the+directory the prefix is in, showing its subdirectories as common+prefixes, including empty ones. Without a delimiter the listing is+recursive: rclone walks the directory tree below the prefix, reading+only as many directories as it needs to fill each page. Any other+delimiter is supported too, but needs the whole directory tree below+the prefix walking, unless the delimiter is in the directory names.Versioning is not currently supported.diff --git a/cmd/serve/s3/utils.go b/cmd/serve/s3/utils.goindex e5f21a14c..e6498874d 100644--- a/cmd/serve/s3/utils.go+++ b/cmd/serve/s3/utils.go@@ -130,14 +130,6 @@ func bucketDirPath(bucketName, dirName string) (string, error) {return bucketObjectPath(bucketName, dirName)}-func prefixParser(p *gofakes3.Prefix) (path, remaining string) {- idx := strings.LastIndexByte(p.Prefix, '/')- if idx < 0 {- return "", p.Prefix- }- return p.Prefix[:idx], p.Prefix[idx+1:]-}-// mkdirRecursive makes the directory path and any of its parents which// don't exist, returning the top-most directory it made, or "" if path// existed already.