"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
walk: factor Listing helpers into their own file and add tests
Nick Craig-Wood117f583ebe75
diff --git a/fs/walk/helpers.go b/fs/walk/helpers.gonew file mode 100644index 000000000..6b14b8586--- /dev/null+++ b/fs/walk/helpers.go@@ -0,0 +1,43 @@+package walk++import "github.com/rclone/rclone/fs"++// Listing helpers used by backends++// ListRHelper is used in the implementation of ListR to accumulate DirEntries+type ListRHelper struct {+ callback fs.ListRCallback+ entries fs.DirEntries+}++// NewListRHelper should be called from ListR with the callback passed in+func NewListRHelper(callback fs.ListRCallback) *ListRHelper {+ return &ListRHelper{+ callback: callback,+ }+}++// send sends the stored entries to the callback if there are >= max+// entries.+func (lh *ListRHelper) send(max int) (err error) {+ if len(lh.entries) >= max {+ err = lh.callback(lh.entries)+ lh.entries = lh.entries[:0]+ }+ return err+}++// Add an entry to the stored entries and send them if there are more+// than a certain amount+func (lh *ListRHelper) Add(entry fs.DirEntry) error {+ if entry == nil {+ return nil+ }+ lh.entries = append(lh.entries, entry)+ return lh.send(100)+}++// Flush the stored entries (if any) sending them to the callback+func (lh *ListRHelper) Flush() error {+ return lh.send(1)+}diff --git a/fs/walk/helpers_test.go b/fs/walk/helpers_test.gonew file mode 100644index 000000000..a028fcde3--- /dev/null+++ b/fs/walk/helpers_test.go@@ -0,0 +1,90 @@+package walk++import (+ "fmt"+ "testing"++ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fstest/mockobject"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++// Mock callback to collect the entries+func mockCallback(entries fs.DirEntries) error {+ // Do nothing or log for testing purposes+ return nil+}++func TestNewListRHelper(t *testing.T) {+ callback := mockCallback+ helper := NewListRHelper(callback)++ assert.NotNil(t, helper)+ assert.Equal(t, fmt.Sprintf("%p", callback), fmt.Sprintf("%p", helper.callback))+ assert.Empty(t, helper.entries)+}++func TestListRHelperAdd(t *testing.T) {+ callbackInvoked := false+ callback := func(entries fs.DirEntries) error {+ callbackInvoked = true+ return nil+ }++ helper := NewListRHelper(callback)+ entry := mockobject.Object("A")+ require.NoError(t, helper.Add(entry))++ assert.Len(t, helper.entries, 1)+ assert.False(t, callbackInvoked, "Callback should not be invoked before reaching 100 entries")++ // Check adding a nil entry doesn't change anything+ require.NoError(t, helper.Add(nil))+ assert.Len(t, helper.entries, 1)+ assert.False(t, callbackInvoked, "Callback should not be invoked before reaching 100 entries")+}++func TestListRHelperSend(t *testing.T) {+ entry := mockobject.Object("A")+ callbackInvoked := false+ callback := func(entries fs.DirEntries) error {+ callbackInvoked = true+ assert.Equal(t, 100, len(entries))+ for _, obj := range entries {+ assert.Equal(t, entry, obj)+ }+ return nil+ }++ helper := NewListRHelper(callback)++ // Add 100 entries to force the callback to be invoked+ for i := 0; i < 100; i++ {+ require.NoError(t, helper.Add(entry))+ }++ assert.Len(t, helper.entries, 0)+ assert.True(t, callbackInvoked, "Callback should be invoked after 100 entries")+}++func TestListRHelperFlush(t *testing.T) {+ entry := mockobject.Object("A")+ callbackInvoked := false+ callback := func(entries fs.DirEntries) error {+ callbackInvoked = true+ assert.Equal(t, 1, len(entries))+ for _, obj := range entries {+ assert.Equal(t, entry, obj)+ }+ return nil+ }++ helper := NewListRHelper(callback)+ require.NoError(t, helper.Add(entry))+ assert.False(t, callbackInvoked, "Callback should not have been invoked yet")+ require.NoError(t, helper.Flush())++ assert.True(t, callbackInvoked, "Callback should be invoked on flush")+ assert.Len(t, helper.entries, 0, "Entries should be cleared after flush")+}diff --git a/fs/walk/walk.go b/fs/walk/walk.goindex 9e54d4c7f..ab157e3a1 100644--- a/fs/walk/walk.go+++ b/fs/walk/walk.go@@ -641,41 +641,3 @@ func GetAll(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel})return}--// ListRHelper is used in the implementation of ListR to accumulate DirEntries-type ListRHelper struct {- callback fs.ListRCallback- entries fs.DirEntries-}--// NewListRHelper should be called from ListR with the callback passed in-func NewListRHelper(callback fs.ListRCallback) *ListRHelper {- return &ListRHelper{- callback: callback,- }-}--// send sends the stored entries to the callback if there are >= max-// entries.-func (lh *ListRHelper) send(max int) (err error) {- if len(lh.entries) >= max {- err = lh.callback(lh.entries)- lh.entries = lh.entries[:0]- }- return err-}--// Add an entry to the stored entries and send them if there are more-// than a certain amount-func (lh *ListRHelper) Add(entry fs.DirEntry) error {- if entry == nil {- return nil- }- lh.entries = append(lh.entries, entry)- return lh.send(100)-}--// Flush the stored entries (if any) sending them to the callback-func (lh *ListRHelper) Flush() error {- return lh.send(1)-}