"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
walk: move NewListRHelper into list.Helper to avoid circular dependency
Nick Craig-Woodcba653d502ec
diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.goindex 4ce27db2c..6a966bb75 100644--- a/backend/azureblob/azureblob.go+++ b/backend/azureblob/azureblob.go@@ -44,7 +44,7 @@ import ("github.com/rclone/rclone/fs/fserrors""github.com/rclone/rclone/fs/fshttp""github.com/rclone/rclone/fs/hash"- "github.com/rclone/rclone/fs/walk"+ "github.com/rclone/rclone/fs/list""github.com/rclone/rclone/lib/atexit""github.com/rclone/rclone/lib/bucket""github.com/rclone/rclone/lib/encoder"@@ -1378,7 +1378,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e// of listing recursively that doing a directory traversal.func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {containerName, directory := f.split(dir)- list := walk.NewListRHelper(callback)+ list := list.NewHelper(callback)listR := func(containerName, directory, prefix string, addContainer bool) error {return f.list(ctx, containerName, directory, prefix, addContainer, true, int32(f.opt.ListChunkSize), func(remote string, object *container.BlobItem, isDirectory bool) error {entry, err := f.itemToDirEntry(ctx, remote, object, isDirectory)diff --git a/backend/b2/b2.go b/backend/b2/b2.goindex 92e7675f0..e8ed3bd72 100644--- a/backend/b2/b2.go+++ b/backend/b2/b2.go@@ -31,8 +31,8 @@ import ("github.com/rclone/rclone/fs/fserrors""github.com/rclone/rclone/fs/fshttp""github.com/rclone/rclone/fs/hash"+ "github.com/rclone/rclone/fs/list""github.com/rclone/rclone/fs/operations"- "github.com/rclone/rclone/fs/walk""github.com/rclone/rclone/lib/bucket""github.com/rclone/rclone/lib/encoder""github.com/rclone/rclone/lib/multipart"@@ -918,7 +918,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e// of listing recursively that doing a directory traversal.func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {bucket, directory := f.split(dir)- list := walk.NewListRHelper(callback)+ list := list.NewHelper(callback)listR := func(bucket, directory, prefix string, addBucket bool) error {last := ""return f.list(ctx, bucket, directory, prefix, addBucket, true, 0, f.opt.Versions, false, func(remote string, object *api.File, isDirectory bool) error {diff --git a/backend/cache/cache.go b/backend/cache/cache.goindex e7a61e922..6977ff2d7 100644--- a/backend/cache/cache.go+++ b/backend/cache/cache.go@@ -29,6 +29,7 @@ import ("github.com/rclone/rclone/fs/config/obscure""github.com/rclone/rclone/fs/fspath""github.com/rclone/rclone/fs/hash"+ "github.com/rclone/rclone/fs/list""github.com/rclone/rclone/fs/rc""github.com/rclone/rclone/fs/walk""github.com/rclone/rclone/lib/atexit"@@ -1086,7 +1087,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err ereturn cachedEntries, nil}-func (f *Fs) recurse(ctx context.Context, dir string, list *walk.ListRHelper) error {+func (f *Fs) recurse(ctx context.Context, dir string, list *list.Helper) error {entries, err := f.List(ctx, dir)if err != nil {return err@@ -1138,7 +1139,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (}// if we're here, we're gonna do a standard recursive traversal and cache everything- list := walk.NewListRHelper(callback)+ list := list.NewHelper(callback)err = f.recurse(ctx, dir, list)if err != nil {return errdiff --git a/backend/drive/drive.go b/backend/drive/drive.goindex 6cbb1d8d9..de83a902d 100644--- a/backend/drive/drive.go+++ b/backend/drive/drive.go@@ -38,8 +38,8 @@ import ("github.com/rclone/rclone/fs/fshttp""github.com/rclone/rclone/fs/fspath""github.com/rclone/rclone/fs/hash"+ "github.com/rclone/rclone/fs/list""github.com/rclone/rclone/fs/operations"- "github.com/rclone/rclone/fs/walk""github.com/rclone/rclone/lib/dircache""github.com/rclone/rclone/lib/encoder""github.com/rclone/rclone/lib/env"@@ -2189,7 +2189,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (wg := sync.WaitGroup{}in := make(chan listREntry, listRInputBuffer)out := make(chan error, f.ci.Checkers)- list := walk.NewListRHelper(callback)+ list := list.NewHelper(callback)overflow := []listREntry{}listed := 0diff --git a/backend/gofile/gofile.go b/backend/gofile/gofile.goindex ca4091046..d8981b9c6 100644--- a/backend/gofile/gofile.go+++ b/backend/gofile/gofile.go@@ -25,7 +25,7 @@ import ("github.com/rclone/rclone/fs/fserrors""github.com/rclone/rclone/fs/fshttp""github.com/rclone/rclone/fs/hash"- "github.com/rclone/rclone/fs/walk"+ "github.com/rclone/rclone/fs/list""github.com/rclone/rclone/lib/dircache""github.com/rclone/rclone/lib/encoder""github.com/rclone/rclone/lib/pacer"@@ -734,7 +734,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e}// implementation of ListR-func (f *Fs) listR(ctx context.Context, dir string, list *walk.ListRHelper) (err error) {+func (f *Fs) listR(ctx context.Context, dir string, list *list.Helper) (err error) {directoryID, err := f.dirCache.FindDir(ctx, dir, false)if err != nil {return err@@ -820,7 +820,7 @@ func (f *Fs) listR(ctx context.Context, dir string, list *walk.ListRHelper) (err// Don't implement this unless you have a more efficient way// of listing recursively than doing a directory traversal.func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {- list := walk.NewListRHelper(callback)+ list := list.NewHelper(callback)err = f.listR(ctx, dir, list)if err != nil {return errdiff --git a/backend/googlecloudstorage/googlecloudstorage.go b/backend/googlecloudstorage/googlecloudstorage.goindex cd57edd53..7c7bec373 100644--- a/backend/googlecloudstorage/googlecloudstorage.go+++ b/backend/googlecloudstorage/googlecloudstorage.go@@ -35,7 +35,7 @@ import ("github.com/rclone/rclone/fs/fserrors""github.com/rclone/rclone/fs/fshttp""github.com/rclone/rclone/fs/hash"- "github.com/rclone/rclone/fs/walk"+ "github.com/rclone/rclone/fs/list""github.com/rclone/rclone/lib/bucket""github.com/rclone/rclone/lib/encoder""github.com/rclone/rclone/lib/env"@@ -845,7 +845,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e// of listing recursively that doing a directory traversal.func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {bucket, directory := f.split(dir)- list := walk.NewListRHelper(callback)+ list := list.NewHelper(callback)listR := func(bucket, directory, prefix string, addBucket bool) error {return f.list(ctx, bucket, directory, prefix, addBucket, true, func(remote string, object *storage.Object, isDirectory bool) error {entry, err := f.itemToDirEntry(ctx, remote, object, isDirectory)diff --git a/backend/jottacloud/jottacloud.go b/backend/jottacloud/jottacloud.goindex 6482070ab..f074ceea2 100644--- a/backend/jottacloud/jottacloud.go+++ b/backend/jottacloud/jottacloud.go@@ -31,7 +31,7 @@ import ("github.com/rclone/rclone/fs/fserrors""github.com/rclone/rclone/fs/fshttp""github.com/rclone/rclone/fs/hash"- "github.com/rclone/rclone/fs/walk"+ "github.com/rclone/rclone/fs/list""github.com/rclone/rclone/lib/encoder""github.com/rclone/rclone/lib/oauthutil""github.com/rclone/rclone/lib/pacer"@@ -1264,7 +1264,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (Parameters: url.Values{},}opts.Parameters.Set("mode", "liststream")- list := walk.NewListRHelper(callback)+ list := list.NewHelper(callback)var resp *http.Responseerr = f.pacer.Call(func() (bool, error) {diff --git a/backend/memory/memory.go b/backend/memory/memory.goindex 48cc9678b..9842c6eb2 100644--- a/backend/memory/memory.go+++ b/backend/memory/memory.go@@ -17,7 +17,7 @@ import ("github.com/rclone/rclone/fs/config/configmap""github.com/rclone/rclone/fs/config/configstruct""github.com/rclone/rclone/fs/hash"- "github.com/rclone/rclone/fs/walk"+ "github.com/rclone/rclone/fs/list""github.com/rclone/rclone/lib/bucket")@@ -383,7 +383,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e// of listing recursively that doing a directory traversal.func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {bucket, directory := f.split(dir)- list := walk.NewListRHelper(callback)+ list := list.NewHelper(callback)entries := fs.DirEntries{}listR := func(bucket, directory, prefix string, addBucket bool) error {err = f.list(ctx, bucket, directory, prefix, addBucket, true, func(remote string, entry fs.DirEntry, isDirectory bool) error {diff --git a/backend/netstorage/netstorage.go b/backend/netstorage/netstorage.goindex 4bada7f66..74964bd07 100755--- a/backend/netstorage/netstorage.go+++ b/backend/netstorage/netstorage.go@@ -28,7 +28,7 @@ import ("github.com/rclone/rclone/fs/fserrors""github.com/rclone/rclone/fs/fshttp""github.com/rclone/rclone/fs/hash"- "github.com/rclone/rclone/fs/walk"+ "github.com/rclone/rclone/fs/list""github.com/rclone/rclone/lib/pacer""github.com/rclone/rclone/lib/rest")@@ -516,7 +516,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (return fs.ErrorDirNotFound}- list := walk.NewListRHelper(callback)+ list := list.NewHelper(callback)for resumeStart := u.Path; resumeStart != ""; {var files []Filefiles, resumeStart, err = f.netStorageListRequest(ctx, URL, u.Path)diff --git a/backend/onedrive/onedrive.go b/backend/onedrive/onedrive.goindex 0a9aad7f4..bfa6ce61c 100644--- a/backend/onedrive/onedrive.go+++ b/backend/onedrive/onedrive.go@@ -30,6 +30,7 @@ import ("github.com/rclone/rclone/fs/fserrors""github.com/rclone/rclone/fs/fshttp""github.com/rclone/rclone/fs/hash"+ "github.com/rclone/rclone/fs/list""github.com/rclone/rclone/fs/log""github.com/rclone/rclone/fs/operations""github.com/rclone/rclone/fs/walk"@@ -1396,7 +1397,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (// So we have to filter things outside of the root which is// inefficient.- list := walk.NewListRHelper(callback)+ list := list.NewHelper(callback)// list a folder conventionally - used for shared foldersvar listFolder func(dir string) errordiff --git a/backend/oracleobjectstorage/oracleobjectstorage.go b/backend/oracleobjectstorage/oracleobjectstorage.goindex a4d0f00fa..2e8c06581 100644--- a/backend/oracleobjectstorage/oracleobjectstorage.go+++ b/backend/oracleobjectstorage/oracleobjectstorage.go@@ -18,8 +18,8 @@ import ("github.com/rclone/rclone/fs/config/configmap""github.com/rclone/rclone/fs/config/configstruct""github.com/rclone/rclone/fs/hash"+ "github.com/rclone/rclone/fs/list""github.com/rclone/rclone/fs/operations"- "github.com/rclone/rclone/fs/walk""github.com/rclone/rclone/lib/bucket""github.com/rclone/rclone/lib/pacer")@@ -649,7 +649,7 @@ of listing recursively that doing a directory traversal.*/func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {bucketName, directory := f.split(dir)- list := walk.NewListRHelper(callback)+ list := list.NewHelper(callback)listR := func(bucket, directory, prefix string, addBucket bool) error {return f.list(ctx, bucket, directory, prefix, addBucket, true, 0, func(remote string, object *objectstorage.ObjectSummary, isDirectory bool) error {entry, err := f.itemToDirEntry(ctx, remote, object, isDirectory)diff --git a/backend/pcloud/pcloud.go b/backend/pcloud/pcloud.goindex 0fe34d56e..8c43df9bd 100644--- a/backend/pcloud/pcloud.go+++ b/backend/pcloud/pcloud.go@@ -27,7 +27,7 @@ import ("github.com/rclone/rclone/fs/fserrors""github.com/rclone/rclone/fs/fshttp""github.com/rclone/rclone/fs/hash"- "github.com/rclone/rclone/fs/walk"+ "github.com/rclone/rclone/fs/list""github.com/rclone/rclone/lib/dircache""github.com/rclone/rclone/lib/encoder""github.com/rclone/rclone/lib/oauthutil"@@ -631,7 +631,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e// ListR lists the objects and directories of the Fs starting// from dir recursively into out.func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {- list := walk.NewListRHelper(callback)+ list := list.NewHelper(callback)err = f.listHelper(ctx, dir, true, func(o fs.DirEntry) error {return list.Add(o)})diff --git a/backend/qingstor/qingstor.go b/backend/qingstor/qingstor.goindex 3dfa36b15..39113d149 100644--- a/backend/qingstor/qingstor.go+++ b/backend/qingstor/qingstor.go@@ -22,7 +22,7 @@ import ("github.com/rclone/rclone/fs/config/configstruct""github.com/rclone/rclone/fs/fshttp""github.com/rclone/rclone/fs/hash"- "github.com/rclone/rclone/fs/walk"+ "github.com/rclone/rclone/fs/list""github.com/rclone/rclone/lib/bucket""github.com/rclone/rclone/lib/encoder"qsConfig "github.com/yunify/qingstor-sdk-go/v3/config"@@ -704,7 +704,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e// of listing recursively that doing a directory traversal.func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {bucket, directory := f.split(dir)- list := walk.NewListRHelper(callback)+ list := list.NewHelper(callback)listR := func(bucket, directory, prefix string, addBucket bool) error {return f.list(ctx, bucket, directory, prefix, addBucket, true, func(remote string, object *qs.KeyType, isDirectory bool) error {entry, err := f.itemToDirEntry(remote, object, isDirectory)diff --git a/backend/s3/s3.go b/backend/s3/s3.goindex 89faffcc2..b0f54b690 100644--- a/backend/s3/s3.go+++ b/backend/s3/s3.go@@ -48,8 +48,8 @@ import ("github.com/rclone/rclone/fs/fserrors""github.com/rclone/rclone/fs/fshttp""github.com/rclone/rclone/fs/hash"+ "github.com/rclone/rclone/fs/list""github.com/rclone/rclone/fs/operations"- "github.com/rclone/rclone/fs/walk""github.com/rclone/rclone/lib/atexit""github.com/rclone/rclone/lib/bucket""github.com/rclone/rclone/lib/encoder"@@ -4567,7 +4567,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e// of listing recursively than doing a directory traversal.func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {bucket, directory := f.split(dir)- list := walk.NewListRHelper(callback)+ list := list.NewHelper(callback)listR := func(bucket, directory, prefix string, addBucket bool) error {return f.list(ctx, listOpt{bucket: bucket,diff --git a/backend/swift/swift.go b/backend/swift/swift.goindex b7b227bc7..e50e17782 100644--- a/backend/swift/swift.go+++ b/backend/swift/swift.go@@ -25,8 +25,8 @@ import ("github.com/rclone/rclone/fs/fserrors""github.com/rclone/rclone/fs/fshttp""github.com/rclone/rclone/fs/hash"+ "github.com/rclone/rclone/fs/list""github.com/rclone/rclone/fs/operations"- "github.com/rclone/rclone/fs/walk""github.com/rclone/rclone/lib/atexit""github.com/rclone/rclone/lib/bucket""github.com/rclone/rclone/lib/encoder"@@ -846,7 +846,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e// of listing recursively than doing a directory traversal.func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {container, directory := f.split(dir)- list := walk.NewListRHelper(callback)+ list := list.NewHelper(callback)listR := func(container, directory, prefix string, addContainer bool) error {return f.list(ctx, container, directory, prefix, addContainer, true, false, func(entry fs.DirEntry) error {return list.Add(entry)diff --git a/fs/walk/helpers.go b/fs/list/helpers.gosimilarity index 59%rename from fs/walk/helpers.gorename to fs/list/helpers.goindex 6b14b8586..e9ec5d92a 100644--- a/fs/walk/helpers.go+++ b/fs/list/helpers.go@@ -1,25 +1,25 @@-package walk+package listimport "github.com/rclone/rclone/fs"// Listing helpers used by backends-// ListRHelper is used in the implementation of ListR to accumulate DirEntries-type ListRHelper struct {+// Helper is used in the implementation of ListR to accumulate DirEntries+type Helper struct {callback fs.ListRCallbackentries fs.DirEntries}-// NewListRHelper should be called from ListR with the callback passed in-func NewListRHelper(callback fs.ListRCallback) *ListRHelper {- return &ListRHelper{+// NewHelper should be called from ListR with the callback passed in+func NewHelper(callback fs.ListRCallback) *Helper {+ return &Helper{callback: callback,}}// send sends the stored entries to the callback if there are >= max// entries.-func (lh *ListRHelper) send(max int) (err error) {+func (lh *Helper) send(max int) (err error) {if len(lh.entries) >= max {err = lh.callback(lh.entries)lh.entries = lh.entries[:0]@@ -29,7 +29,7 @@ func (lh *ListRHelper) send(max int) (err error) {// Add an entry to the stored entries and send them if there are more// than a certain amount-func (lh *ListRHelper) Add(entry fs.DirEntry) error {+func (lh *Helper) Add(entry fs.DirEntry) error {if entry == nil {return nil}@@ -38,6 +38,6 @@ func (lh *ListRHelper) Add(entry fs.DirEntry) error {}// Flush the stored entries (if any) sending them to the callback-func (lh *ListRHelper) Flush() error {+func (lh *Helper) Flush() error {return lh.send(1)}diff --git a/fs/walk/helpers_test.go b/fs/list/helpers_test.gosimilarity index 93%rename from fs/walk/helpers_test.gorename to fs/list/helpers_test.goindex a028fcde3..5066c5449 100644--- a/fs/walk/helpers_test.go+++ b/fs/list/helpers_test.go@@ -1,4 +1,4 @@-package walk+package listimport ("fmt"@@ -18,7 +18,7 @@ func mockCallback(entries fs.DirEntries) error {func TestNewListRHelper(t *testing.T) {callback := mockCallback- helper := NewListRHelper(callback)+ helper := NewHelper(callback)assert.NotNil(t, helper)assert.Equal(t, fmt.Sprintf("%p", callback), fmt.Sprintf("%p", helper.callback))@@ -32,7 +32,7 @@ func TestListRHelperAdd(t *testing.T) {return nil}- helper := NewListRHelper(callback)+ helper := NewHelper(callback)entry := mockobject.Object("A")require.NoError(t, helper.Add(entry))@@ -57,7 +57,7 @@ func TestListRHelperSend(t *testing.T) {return nil}- helper := NewListRHelper(callback)+ helper := NewHelper(callback)// Add 100 entries to force the callback to be invokedfor i := 0; i < 100; i++ {@@ -80,7 +80,7 @@ func TestListRHelperFlush(t *testing.T) {return nil}- helper := NewListRHelper(callback)+ helper := NewHelper(callback)require.NoError(t, helper.Add(entry))assert.False(t, callbackInvoked, "Callback should not have been invoked yet")require.NoError(t, helper.Flush())diff --git a/fs/walk/walk.go b/fs/walk/walk.goindex 5d64f97d3..d9857a2f1 100644--- a/fs/walk/walk.go+++ b/fs/walk/walk.go@@ -274,7 +274,7 @@ func (dm *dirMap) sendEntries(fn fs.ListRCallback) (err error) {sort.Strings(dirs)// Now convert to bulkier Dir in batches and sendnow := time.Now()- list := NewListRHelper(fn)+ list := list.NewHelper(fn)for _, dir := range dirs {err = list.Add(fs.NewDir(dir, now))if err != nil {