Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

walk: factor Listing helpers into their own file and add tests

Nick Craig-Wood117f583ebe75
diff --git a/fs/walk/helpers.go b/fs/walk/helpers.go
new file mode 100644
index 000000000..6b14b8586
--- /dev/null
+++ b/fs/walk/helpers.go
@@ -0,0 +1,43 @@
+package walk
+
+import "github.com/rclone/rclone/fs"
+
+// Listing helpers used by backends
+
+// ListRHelper is used in the implementation of ListR to accumulate DirEntries
+type ListRHelper struct {
+ callback fs.ListRCallback
+ entries fs.DirEntries
+}
+
+// NewListRHelper should be called from ListR with the callback passed in
+func NewListRHelper(callback fs.ListRCallback) *ListRHelper {
+ return &ListRHelper{
+ callback: callback,
+ }
+}
+
+// send sends the stored entries to the callback if there are >= max
+// entries.
+func (lh *ListRHelper) send(max int) (err error) {
+ if len(lh.entries) >= max {
+ err = lh.callback(lh.entries)
+ lh.entries = lh.entries[:0]
+ }
+ return err
+}
+
+// Add an entry to the stored entries and send them if there are more
+// than a certain amount
+func (lh *ListRHelper) Add(entry fs.DirEntry) error {
+ if entry == nil {
+ return nil
+ }
+ lh.entries = append(lh.entries, entry)
+ return lh.send(100)
+}
+
+// Flush the stored entries (if any) sending them to the callback
+func (lh *ListRHelper) Flush() error {
+ return lh.send(1)
+}
diff --git a/fs/walk/helpers_test.go b/fs/walk/helpers_test.go
new file mode 100644
index 000000000..a028fcde3
--- /dev/null
+++ b/fs/walk/helpers_test.go
@@ -0,0 +1,90 @@
+package walk
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fstest/mockobject"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// Mock callback to collect the entries
+func mockCallback(entries fs.DirEntries) error {
+ // Do nothing or log for testing purposes
+ return nil
+}
+
+func TestNewListRHelper(t *testing.T) {
+ callback := mockCallback
+ helper := NewListRHelper(callback)
+
+ assert.NotNil(t, helper)
+ assert.Equal(t, fmt.Sprintf("%p", callback), fmt.Sprintf("%p", helper.callback))
+ assert.Empty(t, helper.entries)
+}
+
+func TestListRHelperAdd(t *testing.T) {
+ callbackInvoked := false
+ callback := func(entries fs.DirEntries) error {
+ callbackInvoked = true
+ return nil
+ }
+
+ helper := NewListRHelper(callback)
+ entry := mockobject.Object("A")
+ require.NoError(t, helper.Add(entry))
+
+ assert.Len(t, helper.entries, 1)
+ assert.False(t, callbackInvoked, "Callback should not be invoked before reaching 100 entries")
+
+ // Check adding a nil entry doesn't change anything
+ require.NoError(t, helper.Add(nil))
+ assert.Len(t, helper.entries, 1)
+ assert.False(t, callbackInvoked, "Callback should not be invoked before reaching 100 entries")
+}
+
+func TestListRHelperSend(t *testing.T) {
+ entry := mockobject.Object("A")
+ callbackInvoked := false
+ callback := func(entries fs.DirEntries) error {
+ callbackInvoked = true
+ assert.Equal(t, 100, len(entries))
+ for _, obj := range entries {
+ assert.Equal(t, entry, obj)
+ }
+ return nil
+ }
+
+ helper := NewListRHelper(callback)
+
+ // Add 100 entries to force the callback to be invoked
+ for i := 0; i < 100; i++ {
+ require.NoError(t, helper.Add(entry))
+ }
+
+ assert.Len(t, helper.entries, 0)
+ assert.True(t, callbackInvoked, "Callback should be invoked after 100 entries")
+}
+
+func TestListRHelperFlush(t *testing.T) {
+ entry := mockobject.Object("A")
+ callbackInvoked := false
+ callback := func(entries fs.DirEntries) error {
+ callbackInvoked = true
+ assert.Equal(t, 1, len(entries))
+ for _, obj := range entries {
+ assert.Equal(t, entry, obj)
+ }
+ return nil
+ }
+
+ helper := NewListRHelper(callback)
+ require.NoError(t, helper.Add(entry))
+ assert.False(t, callbackInvoked, "Callback should not have been invoked yet")
+ require.NoError(t, helper.Flush())
+
+ assert.True(t, callbackInvoked, "Callback should be invoked on flush")
+ assert.Len(t, helper.entries, 0, "Entries should be cleared after flush")
+}
diff --git a/fs/walk/walk.go b/fs/walk/walk.go
index 9e54d4c7f..ab157e3a1 100644
--- a/fs/walk/walk.go
+++ b/fs/walk/walk.go
@@ -641,41 +641,3 @@ func GetAll(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel
})
return
}
-
-// ListRHelper is used in the implementation of ListR to accumulate DirEntries
-type ListRHelper struct {
- callback fs.ListRCallback
- entries fs.DirEntries
-}
-
-// NewListRHelper should be called from ListR with the callback passed in
-func NewListRHelper(callback fs.ListRCallback) *ListRHelper {
- return &ListRHelper{
- callback: callback,
- }
-}
-
-// send sends the stored entries to the callback if there are >= max
-// entries.
-func (lh *ListRHelper) send(max int) (err error) {
- if len(lh.entries) >= max {
- err = lh.callback(lh.entries)
- lh.entries = lh.entries[:0]
- }
- return err
-}
-
-// Add an entry to the stored entries and send them if there are more
-// than a certain amount
-func (lh *ListRHelper) Add(entry fs.DirEntry) error {
- if entry == nil {
- return nil
- }
- lh.entries = append(lh.entries, entry)
- return lh.send(100)
-}
-
-// Flush the stored entries (if any) sending them to the callback
-func (lh *ListRHelper) Flush() error {
- return lh.send(1)
-}