"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
combine: implement ListP interface
Nick Craig-Woodb1d774c2e3ca
diff --git a/backend/combine/combine.go b/backend/combine/combine.goindex d1036ae3a..914eaca6e 100644--- a/backend/combine/combine.go+++ b/backend/combine/combine.go@@ -20,6 +20,7 @@ import ("github.com/rclone/rclone/fs/config/configmap""github.com/rclone/rclone/fs/config/configstruct""github.com/rclone/rclone/fs/hash"+ "github.com/rclone/rclone/fs/list""github.com/rclone/rclone/fs/operations""github.com/rclone/rclone/fs/walk""golang.org/x/sync/errgroup"@@ -265,6 +266,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs}}+ // Enable ListP always+ features.ListP = f.ListP+// Enable Purge when any upstreams support itif features.Purge == nil {for _, u := range f.upstreams {@@ -809,24 +813,52 @@ func (u *upstream) wrapEntries(ctx context.Context, entries fs.DirEntries) (fs.D// This should return ErrDirNotFound if the directory isn't// found.func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {+ return list.WithListP(ctx, dir, f)+}++// ListP lists the objects and directories of the Fs starting+// from dir non recursively into out.+//+// dir should be "" to start from the root, and should not+// have trailing slashes.+//+// This should return ErrDirNotFound if the directory isn't+// found.+//+// It should call callback for each tranche of entries read.+// These need not be returned in any particular order. If+// callback returns an error then the listing will stop+// immediately.+func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) error {// defer log.Trace(f, "dir=%q", dir)("entries = %v, err=%v", &entries, &err)if f.root == "" && dir == "" {- entries = make(fs.DirEntries, 0, len(f.upstreams))+ entries := make(fs.DirEntries, 0, len(f.upstreams))for combineDir := range f.upstreams {d := fs.NewLimitedDirWrapper(combineDir, fs.NewDir(combineDir, f.when))entries = append(entries, d)}- return entries, nil+ return callback(entries)}u, uRemote, err := f.findUpstream(dir)if err != nil {- return nil, err+ return err}- entries, err = u.f.List(ctx, uRemote)- if err != nil {- return nil, err+ wrappedCallback := func(entries fs.DirEntries) error {+ entries, err := u.wrapEntries(ctx, entries)+ if err != nil {+ return err+ }+ return callback(entries)+ }+ listP := u.f.Features().ListP+ if listP == nil {+ entries, err := u.f.List(ctx, uRemote)+ if err != nil {+ return err+ }+ return wrappedCallback(entries)}- return u.wrapEntries(ctx, entries)+ return listP(ctx, dir, wrappedCallback)}// ListR lists the objects and directories of the Fs starting