Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

combine: implement ListP interface

Nick Craig-Woodb1d774c2e3ca
diff --git a/backend/combine/combine.go b/backend/combine/combine.go
index d1036ae3a..914eaca6e 100644
--- a/backend/combine/combine.go
+++ b/backend/combine/combine.go
@@ -20,6 +20,7 @@ import (
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/hash"
+ "github.com/rclone/rclone/fs/list"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/fs/walk"
"golang.org/x/sync/errgroup"
@@ -265,6 +266,9 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (outFs fs
}
}
+ // Enable ListP always
+ features.ListP = f.ListP
+
// Enable Purge when any upstreams support it
if features.Purge == nil {
for _, u := range f.upstreams {
@@ -809,24 +813,52 @@ func (u *upstream) wrapEntries(ctx context.Context, entries fs.DirEntries) (fs.D
// This should return ErrDirNotFound if the directory isn't
// found.
func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) {
+ return list.WithListP(ctx, dir, f)
+}
+
+// ListP lists the objects and directories of the Fs starting
+// from dir non recursively into out.
+//
+// dir should be "" to start from the root, and should not
+// have trailing slashes.
+//
+// This should return ErrDirNotFound if the directory isn't
+// found.
+//
+// It should call callback for each tranche of entries read.
+// These need not be returned in any particular order. If
+// callback returns an error then the listing will stop
+// immediately.
+func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) error {
// defer log.Trace(f, "dir=%q", dir)("entries = %v, err=%v", &entries, &err)
if f.root == "" && dir == "" {
- entries = make(fs.DirEntries, 0, len(f.upstreams))
+ entries := make(fs.DirEntries, 0, len(f.upstreams))
for combineDir := range f.upstreams {
d := fs.NewLimitedDirWrapper(combineDir, fs.NewDir(combineDir, f.when))
entries = append(entries, d)
}
- return entries, nil
+ return callback(entries)
}
u, uRemote, err := f.findUpstream(dir)
if err != nil {
- return nil, err
+ return err
}
- entries, err = u.f.List(ctx, uRemote)
- if err != nil {
- return nil, err
+ wrappedCallback := func(entries fs.DirEntries) error {
+ entries, err := u.wrapEntries(ctx, entries)
+ if err != nil {
+ return err
+ }
+ return callback(entries)
+ }
+ listP := u.f.Features().ListP
+ if listP == nil {
+ entries, err := u.f.List(ctx, uRemote)
+ if err != nil {
+ return err
+ }
+ return wrappedCallback(entries)
}
- return u.wrapEntries(ctx, entries)
+ return listP(ctx, dir, wrappedCallback)
}
// ListR lists the objects and directories of the Fs starting