Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

walk: move NewListRHelper into list.Helper to avoid circular dependency

Nick Craig-Woodcba653d502ec
diff --git a/backend/azureblob/azureblob.go b/backend/azureblob/azureblob.go
index 4ce27db2c..6a966bb75 100644
--- a/backend/azureblob/azureblob.go
+++ b/backend/azureblob/azureblob.go
@@ -44,7 +44,7 @@ import (
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
- "github.com/rclone/rclone/fs/walk"
+ "github.com/rclone/rclone/fs/list"
"github.com/rclone/rclone/lib/atexit"
"github.com/rclone/rclone/lib/bucket"
"github.com/rclone/rclone/lib/encoder"
@@ -1378,7 +1378,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
// of listing recursively that doing a directory traversal.
func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
containerName, directory := f.split(dir)
- list := walk.NewListRHelper(callback)
+ list := list.NewHelper(callback)
listR := func(containerName, directory, prefix string, addContainer bool) error {
return f.list(ctx, containerName, directory, prefix, addContainer, true, int32(f.opt.ListChunkSize), func(remote string, object *container.BlobItem, isDirectory bool) error {
entry, err := f.itemToDirEntry(ctx, remote, object, isDirectory)
diff --git a/backend/b2/b2.go b/backend/b2/b2.go
index 92e7675f0..e8ed3bd72 100644
--- a/backend/b2/b2.go
+++ b/backend/b2/b2.go
@@ -31,8 +31,8 @@ import (
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
+ "github.com/rclone/rclone/fs/list"
"github.com/rclone/rclone/fs/operations"
- "github.com/rclone/rclone/fs/walk"
"github.com/rclone/rclone/lib/bucket"
"github.com/rclone/rclone/lib/encoder"
"github.com/rclone/rclone/lib/multipart"
@@ -918,7 +918,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
// of listing recursively that doing a directory traversal.
func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
bucket, directory := f.split(dir)
- list := walk.NewListRHelper(callback)
+ list := list.NewHelper(callback)
listR := func(bucket, directory, prefix string, addBucket bool) error {
last := ""
return f.list(ctx, bucket, directory, prefix, addBucket, true, 0, f.opt.Versions, false, func(remote string, object *api.File, isDirectory bool) error {
diff --git a/backend/cache/cache.go b/backend/cache/cache.go
index e7a61e922..6977ff2d7 100644
--- a/backend/cache/cache.go
+++ b/backend/cache/cache.go
@@ -29,6 +29,7 @@ import (
"github.com/rclone/rclone/fs/config/obscure"
"github.com/rclone/rclone/fs/fspath"
"github.com/rclone/rclone/fs/hash"
+ "github.com/rclone/rclone/fs/list"
"github.com/rclone/rclone/fs/rc"
"github.com/rclone/rclone/fs/walk"
"github.com/rclone/rclone/lib/atexit"
@@ -1086,7 +1087,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
return cachedEntries, nil
}
-func (f *Fs) recurse(ctx context.Context, dir string, list *walk.ListRHelper) error {
+func (f *Fs) recurse(ctx context.Context, dir string, list *list.Helper) error {
entries, err := f.List(ctx, dir)
if err != nil {
return err
@@ -1138,7 +1139,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
}
// if we're here, we're gonna do a standard recursive traversal and cache everything
- list := walk.NewListRHelper(callback)
+ list := list.NewHelper(callback)
err = f.recurse(ctx, dir, list)
if err != nil {
return err
diff --git a/backend/drive/drive.go b/backend/drive/drive.go
index 6cbb1d8d9..de83a902d 100644
--- a/backend/drive/drive.go
+++ b/backend/drive/drive.go
@@ -38,8 +38,8 @@ import (
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/fspath"
"github.com/rclone/rclone/fs/hash"
+ "github.com/rclone/rclone/fs/list"
"github.com/rclone/rclone/fs/operations"
- "github.com/rclone/rclone/fs/walk"
"github.com/rclone/rclone/lib/dircache"
"github.com/rclone/rclone/lib/encoder"
"github.com/rclone/rclone/lib/env"
@@ -2189,7 +2189,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
wg := sync.WaitGroup{}
in := make(chan listREntry, listRInputBuffer)
out := make(chan error, f.ci.Checkers)
- list := walk.NewListRHelper(callback)
+ list := list.NewHelper(callback)
overflow := []listREntry{}
listed := 0
diff --git a/backend/gofile/gofile.go b/backend/gofile/gofile.go
index ca4091046..d8981b9c6 100644
--- a/backend/gofile/gofile.go
+++ b/backend/gofile/gofile.go
@@ -25,7 +25,7 @@ import (
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
- "github.com/rclone/rclone/fs/walk"
+ "github.com/rclone/rclone/fs/list"
"github.com/rclone/rclone/lib/dircache"
"github.com/rclone/rclone/lib/encoder"
"github.com/rclone/rclone/lib/pacer"
@@ -734,7 +734,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
}
// implementation of ListR
-func (f *Fs) listR(ctx context.Context, dir string, list *walk.ListRHelper) (err error) {
+func (f *Fs) listR(ctx context.Context, dir string, list *list.Helper) (err error) {
directoryID, err := f.dirCache.FindDir(ctx, dir, false)
if err != nil {
return err
@@ -820,7 +820,7 @@ func (f *Fs) listR(ctx context.Context, dir string, list *walk.ListRHelper) (err
// Don't implement this unless you have a more efficient way
// of listing recursively than doing a directory traversal.
func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
- list := walk.NewListRHelper(callback)
+ list := list.NewHelper(callback)
err = f.listR(ctx, dir, list)
if err != nil {
return err
diff --git a/backend/googlecloudstorage/googlecloudstorage.go b/backend/googlecloudstorage/googlecloudstorage.go
index cd57edd53..7c7bec373 100644
--- a/backend/googlecloudstorage/googlecloudstorage.go
+++ b/backend/googlecloudstorage/googlecloudstorage.go
@@ -35,7 +35,7 @@ import (
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
- "github.com/rclone/rclone/fs/walk"
+ "github.com/rclone/rclone/fs/list"
"github.com/rclone/rclone/lib/bucket"
"github.com/rclone/rclone/lib/encoder"
"github.com/rclone/rclone/lib/env"
@@ -845,7 +845,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
// of listing recursively that doing a directory traversal.
func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
bucket, directory := f.split(dir)
- list := walk.NewListRHelper(callback)
+ list := list.NewHelper(callback)
listR := func(bucket, directory, prefix string, addBucket bool) error {
return f.list(ctx, bucket, directory, prefix, addBucket, true, func(remote string, object *storage.Object, isDirectory bool) error {
entry, err := f.itemToDirEntry(ctx, remote, object, isDirectory)
diff --git a/backend/jottacloud/jottacloud.go b/backend/jottacloud/jottacloud.go
index 6482070ab..f074ceea2 100644
--- a/backend/jottacloud/jottacloud.go
+++ b/backend/jottacloud/jottacloud.go
@@ -31,7 +31,7 @@ import (
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
- "github.com/rclone/rclone/fs/walk"
+ "github.com/rclone/rclone/fs/list"
"github.com/rclone/rclone/lib/encoder"
"github.com/rclone/rclone/lib/oauthutil"
"github.com/rclone/rclone/lib/pacer"
@@ -1264,7 +1264,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
Parameters: url.Values{},
}
opts.Parameters.Set("mode", "liststream")
- list := walk.NewListRHelper(callback)
+ list := list.NewHelper(callback)
var resp *http.Response
err = f.pacer.Call(func() (bool, error) {
diff --git a/backend/memory/memory.go b/backend/memory/memory.go
index 48cc9678b..9842c6eb2 100644
--- a/backend/memory/memory.go
+++ b/backend/memory/memory.go
@@ -17,7 +17,7 @@ import (
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/hash"
- "github.com/rclone/rclone/fs/walk"
+ "github.com/rclone/rclone/fs/list"
"github.com/rclone/rclone/lib/bucket"
)
@@ -383,7 +383,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
// of listing recursively that doing a directory traversal.
func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
bucket, directory := f.split(dir)
- list := walk.NewListRHelper(callback)
+ list := list.NewHelper(callback)
entries := fs.DirEntries{}
listR := func(bucket, directory, prefix string, addBucket bool) error {
err = f.list(ctx, bucket, directory, prefix, addBucket, true, func(remote string, entry fs.DirEntry, isDirectory bool) error {
diff --git a/backend/netstorage/netstorage.go b/backend/netstorage/netstorage.go
index 4bada7f66..74964bd07 100755
--- a/backend/netstorage/netstorage.go
+++ b/backend/netstorage/netstorage.go
@@ -28,7 +28,7 @@ import (
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
- "github.com/rclone/rclone/fs/walk"
+ "github.com/rclone/rclone/fs/list"
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/rest"
)
@@ -516,7 +516,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
return fs.ErrorDirNotFound
}
- list := walk.NewListRHelper(callback)
+ list := list.NewHelper(callback)
for resumeStart := u.Path; resumeStart != ""; {
var files []File
files, resumeStart, err = f.netStorageListRequest(ctx, URL, u.Path)
diff --git a/backend/onedrive/onedrive.go b/backend/onedrive/onedrive.go
index 0a9aad7f4..bfa6ce61c 100644
--- a/backend/onedrive/onedrive.go
+++ b/backend/onedrive/onedrive.go
@@ -30,6 +30,7 @@ import (
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
+ "github.com/rclone/rclone/fs/list"
"github.com/rclone/rclone/fs/log"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/fs/walk"
@@ -1396,7 +1397,7 @@ func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (
// So we have to filter things outside of the root which is
// inefficient.
- list := walk.NewListRHelper(callback)
+ list := list.NewHelper(callback)
// list a folder conventionally - used for shared folders
var listFolder func(dir string) error
diff --git a/backend/oracleobjectstorage/oracleobjectstorage.go b/backend/oracleobjectstorage/oracleobjectstorage.go
index a4d0f00fa..2e8c06581 100644
--- a/backend/oracleobjectstorage/oracleobjectstorage.go
+++ b/backend/oracleobjectstorage/oracleobjectstorage.go
@@ -18,8 +18,8 @@ import (
"github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/hash"
+ "github.com/rclone/rclone/fs/list"
"github.com/rclone/rclone/fs/operations"
- "github.com/rclone/rclone/fs/walk"
"github.com/rclone/rclone/lib/bucket"
"github.com/rclone/rclone/lib/pacer"
)
@@ -649,7 +649,7 @@ of listing recursively that doing a directory traversal.
*/
func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
bucketName, directory := f.split(dir)
- list := walk.NewListRHelper(callback)
+ list := list.NewHelper(callback)
listR := func(bucket, directory, prefix string, addBucket bool) error {
return f.list(ctx, bucket, directory, prefix, addBucket, true, 0, func(remote string, object *objectstorage.ObjectSummary, isDirectory bool) error {
entry, err := f.itemToDirEntry(ctx, remote, object, isDirectory)
diff --git a/backend/pcloud/pcloud.go b/backend/pcloud/pcloud.go
index 0fe34d56e..8c43df9bd 100644
--- a/backend/pcloud/pcloud.go
+++ b/backend/pcloud/pcloud.go
@@ -27,7 +27,7 @@ import (
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
- "github.com/rclone/rclone/fs/walk"
+ "github.com/rclone/rclone/fs/list"
"github.com/rclone/rclone/lib/dircache"
"github.com/rclone/rclone/lib/encoder"
"github.com/rclone/rclone/lib/oauthutil"
@@ -631,7 +631,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
// ListR lists the objects and directories of the Fs starting
// from dir recursively into out.
func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
- list := walk.NewListRHelper(callback)
+ list := list.NewHelper(callback)
err = f.listHelper(ctx, dir, true, func(o fs.DirEntry) error {
return list.Add(o)
})
diff --git a/backend/qingstor/qingstor.go b/backend/qingstor/qingstor.go
index 3dfa36b15..39113d149 100644
--- a/backend/qingstor/qingstor.go
+++ b/backend/qingstor/qingstor.go
@@ -22,7 +22,7 @@ import (
"github.com/rclone/rclone/fs/config/configstruct"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
- "github.com/rclone/rclone/fs/walk"
+ "github.com/rclone/rclone/fs/list"
"github.com/rclone/rclone/lib/bucket"
"github.com/rclone/rclone/lib/encoder"
qsConfig "github.com/yunify/qingstor-sdk-go/v3/config"
@@ -704,7 +704,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
// of listing recursively that doing a directory traversal.
func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
bucket, directory := f.split(dir)
- list := walk.NewListRHelper(callback)
+ list := list.NewHelper(callback)
listR := func(bucket, directory, prefix string, addBucket bool) error {
return f.list(ctx, bucket, directory, prefix, addBucket, true, func(remote string, object *qs.KeyType, isDirectory bool) error {
entry, err := f.itemToDirEntry(remote, object, isDirectory)
diff --git a/backend/s3/s3.go b/backend/s3/s3.go
index 89faffcc2..b0f54b690 100644
--- a/backend/s3/s3.go
+++ b/backend/s3/s3.go
@@ -48,8 +48,8 @@ import (
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
+ "github.com/rclone/rclone/fs/list"
"github.com/rclone/rclone/fs/operations"
- "github.com/rclone/rclone/fs/walk"
"github.com/rclone/rclone/lib/atexit"
"github.com/rclone/rclone/lib/bucket"
"github.com/rclone/rclone/lib/encoder"
@@ -4567,7 +4567,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
// of listing recursively than doing a directory traversal.
func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
bucket, directory := f.split(dir)
- list := walk.NewListRHelper(callback)
+ list := list.NewHelper(callback)
listR := func(bucket, directory, prefix string, addBucket bool) error {
return f.list(ctx, listOpt{
bucket: bucket,
diff --git a/backend/swift/swift.go b/backend/swift/swift.go
index b7b227bc7..e50e17782 100644
--- a/backend/swift/swift.go
+++ b/backend/swift/swift.go
@@ -25,8 +25,8 @@ import (
"github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/hash"
+ "github.com/rclone/rclone/fs/list"
"github.com/rclone/rclone/fs/operations"
- "github.com/rclone/rclone/fs/walk"
"github.com/rclone/rclone/lib/atexit"
"github.com/rclone/rclone/lib/bucket"
"github.com/rclone/rclone/lib/encoder"
@@ -846,7 +846,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e
// of listing recursively than doing a directory traversal.
func (f *Fs) ListR(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
container, directory := f.split(dir)
- list := walk.NewListRHelper(callback)
+ list := list.NewHelper(callback)
listR := func(container, directory, prefix string, addContainer bool) error {
return f.list(ctx, container, directory, prefix, addContainer, true, false, func(entry fs.DirEntry) error {
return list.Add(entry)
diff --git a/fs/walk/helpers.go b/fs/list/helpers.go
similarity index 59%
rename from fs/walk/helpers.go
rename to fs/list/helpers.go
index 6b14b8586..e9ec5d92a 100644
--- a/fs/walk/helpers.go
+++ b/fs/list/helpers.go
@@ -1,25 +1,25 @@
-package walk
+package list
import "github.com/rclone/rclone/fs"
// Listing helpers used by backends
-// ListRHelper is used in the implementation of ListR to accumulate DirEntries
-type ListRHelper struct {
+// Helper is used in the implementation of ListR to accumulate DirEntries
+type Helper struct {
callback fs.ListRCallback
entries fs.DirEntries
}
-// NewListRHelper should be called from ListR with the callback passed in
-func NewListRHelper(callback fs.ListRCallback) *ListRHelper {
- return &ListRHelper{
+// NewHelper should be called from ListR with the callback passed in
+func NewHelper(callback fs.ListRCallback) *Helper {
+ return &Helper{
callback: callback,
}
}
// send sends the stored entries to the callback if there are >= max
// entries.
-func (lh *ListRHelper) send(max int) (err error) {
+func (lh *Helper) send(max int) (err error) {
if len(lh.entries) >= max {
err = lh.callback(lh.entries)
lh.entries = lh.entries[:0]
@@ -29,7 +29,7 @@ func (lh *ListRHelper) send(max int) (err error) {
// Add an entry to the stored entries and send them if there are more
// than a certain amount
-func (lh *ListRHelper) Add(entry fs.DirEntry) error {
+func (lh *Helper) Add(entry fs.DirEntry) error {
if entry == nil {
return nil
}
@@ -38,6 +38,6 @@ func (lh *ListRHelper) Add(entry fs.DirEntry) error {
}
// Flush the stored entries (if any) sending them to the callback
-func (lh *ListRHelper) Flush() error {
+func (lh *Helper) Flush() error {
return lh.send(1)
}
diff --git a/fs/walk/helpers_test.go b/fs/list/helpers_test.go
similarity index 93%
rename from fs/walk/helpers_test.go
rename to fs/list/helpers_test.go
index a028fcde3..5066c5449 100644
--- a/fs/walk/helpers_test.go
+++ b/fs/list/helpers_test.go
@@ -1,4 +1,4 @@
-package walk
+package list
import (
"fmt"
@@ -18,7 +18,7 @@ func mockCallback(entries fs.DirEntries) error {
func TestNewListRHelper(t *testing.T) {
callback := mockCallback
- helper := NewListRHelper(callback)
+ helper := NewHelper(callback)
assert.NotNil(t, helper)
assert.Equal(t, fmt.Sprintf("%p", callback), fmt.Sprintf("%p", helper.callback))
@@ -32,7 +32,7 @@ func TestListRHelperAdd(t *testing.T) {
return nil
}
- helper := NewListRHelper(callback)
+ helper := NewHelper(callback)
entry := mockobject.Object("A")
require.NoError(t, helper.Add(entry))
@@ -57,7 +57,7 @@ func TestListRHelperSend(t *testing.T) {
return nil
}
- helper := NewListRHelper(callback)
+ helper := NewHelper(callback)
// Add 100 entries to force the callback to be invoked
for i := 0; i < 100; i++ {
@@ -80,7 +80,7 @@ func TestListRHelperFlush(t *testing.T) {
return nil
}
- helper := NewListRHelper(callback)
+ helper := NewHelper(callback)
require.NoError(t, helper.Add(entry))
assert.False(t, callbackInvoked, "Callback should not have been invoked yet")
require.NoError(t, helper.Flush())
diff --git a/fs/walk/walk.go b/fs/walk/walk.go
index 5d64f97d3..d9857a2f1 100644
--- a/fs/walk/walk.go
+++ b/fs/walk/walk.go
@@ -274,7 +274,7 @@ func (dm *dirMap) sendEntries(fn fs.ListRCallback) (err error) {
sort.Strings(dirs)
// Now convert to bulkier Dir in batches and send
now := time.Now()
- list := NewListRHelper(fn)
+ list := list.NewHelper(fn)
for _, dir := range dirs {
err = list.Add(fs.NewDir(dir, now))
if err != nil {