Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

list: add WithListP helper to implement List for ListP backends

Nick Craig-Wood37120ef7bdbe
diff --git a/fs/list/helpers.go b/fs/list/helpers.go
index e9ec5d92a..54303ef61 100644
--- a/fs/list/helpers.go
+++ b/fs/list/helpers.go
@@ -1,6 +1,11 @@
package list
-import "github.com/rclone/rclone/fs"
+import (
+ "context"
+
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/accounting"
+)
// Listing helpers used by backends
@@ -41,3 +46,16 @@ func (lh *Helper) Add(entry fs.DirEntry) error {
func (lh *Helper) Flush() error {
return lh.send(1)
}
+
+// WithListP implements the List interface with ListP
+//
+// It should be used in backends which support ListP to implement
+// List.
+func WithListP(ctx context.Context, dir string, list fs.ListPer) (entries fs.DirEntries, err error) {
+ err = list.ListP(ctx, dir, func(newEntries fs.DirEntries) error {
+ accounting.Stats(ctx).Listed(int64(len(newEntries)))
+ entries = append(entries, newEntries...)
+ return nil
+ })
+ return entries, err
+}
diff --git a/fs/list/helpers_test.go b/fs/list/helpers_test.go
index 5066c5449..8973b9ddd 100644
--- a/fs/list/helpers_test.go
+++ b/fs/list/helpers_test.go
@@ -1,6 +1,8 @@
package list
import (
+ "context"
+ "errors"
"fmt"
"testing"
@@ -88,3 +90,56 @@ func TestListRHelperFlush(t *testing.T) {
assert.True(t, callbackInvoked, "Callback should be invoked on flush")
assert.Len(t, helper.entries, 0, "Entries should be cleared after flush")
}
+
+type mockListPfs struct {
+ t *testing.T
+ entries fs.DirEntries
+ err error
+ errorAfter int
+}
+
+func (f *mockListPfs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) (err error) {
+ assert.Equal(f.t, "dir", dir)
+ count := 0
+ for entries := f.entries; len(entries) > 0; entries = entries[2:] {
+ err = callback(entries[:2])
+ if err != nil {
+ return err
+ }
+ count += 2
+ if f.err != nil && count >= f.errorAfter {
+ return f.err
+ }
+ }
+ return nil
+}
+
+// check interface
+var _ fs.ListPer = (*mockListPfs)(nil)
+
+func TestListWithListP(t *testing.T) {
+ ctx := context.Background()
+ var entries fs.DirEntries
+ for i := 0; i < 26; i++ {
+ entries = append(entries, mockobject.New(fmt.Sprintf("%c", 'A'+i)))
+ }
+ t.Run("NoError", func(t *testing.T) {
+ f := &mockListPfs{
+ t: t,
+ entries: entries,
+ }
+ gotEntries, err := WithListP(ctx, "dir", f)
+ require.NoError(t, err)
+ assert.Equal(t, entries, gotEntries)
+ })
+ t.Run("Error", func(t *testing.T) {
+ f := &mockListPfs{t: t,
+ entries: entries,
+ err: errors.New("BOOM"),
+ errorAfter: 10,
+ }
+ gotEntries, err := WithListP(ctx, "dir", f)
+ assert.Equal(t, f.err, err)
+ assert.Equal(t, entries[:10], gotEntries)
+ })
+}