Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

list: Implement Sorter to sort directory entries

Nick Craig-Wood08e81f8420d7
diff --git a/fs/list/sorter.go b/fs/list/sorter.go
new file mode 100644
index 000000000..b38d66fa3
--- /dev/null
+++ b/fs/list/sorter.go
@@ -0,0 +1,105 @@
+package list
+
+import (
+ "cmp"
+ "context"
+ "slices"
+ "sync"
+
+ "github.com/rclone/rclone/fs"
+)
+
+// Sorter implements an efficient mechanism for sorting list entries.
+//
+// If there are a large number of entries, this may be done on disk
+// instead of in memory.
+//
+// Supply entries with the Add method, call Send at the end to deliver
+// the sorted entries and finalise with CleanUp regardless of whether
+// you called Add or Send.
+//
+// Sorted entries are delivered to the callback supplied to NewSorter
+// when the Send method is called.
+type Sorter struct {
+ ctx context.Context
+ mu sync.Mutex
+ callback fs.ListRCallback
+ entries fs.DirEntries
+ keyFn KeyFn
+}
+
+// KeyFn turns an entry into a sort key
+type KeyFn func(entry fs.DirEntry) string
+
+// identityKeyFn maps an entry to its Remote
+func identityKeyFn(entry fs.DirEntry) string {
+ return entry.Remote()
+}
+
+// NewSorter creates a new Sorter with callback for sorted entries to
+// be delivered to. keyFn is used to process each entry to get a key
+// function, if nil then it will just use entry.Remote()
+func NewSorter(ctx context.Context, callback fs.ListRCallback, keyFn KeyFn) (*Sorter, error) {
+ if keyFn == nil {
+ keyFn = identityKeyFn
+ }
+ return &Sorter{
+ ctx: ctx,
+ callback: callback,
+ keyFn: keyFn,
+ }, nil
+}
+
+// Add entries to the list sorter.
+//
+// Does not call the callback.
+//
+// Safe to call from concurrent go routines
+func (ls *Sorter) Add(entries fs.DirEntries) error {
+ ls.mu.Lock()
+ defer ls.mu.Unlock()
+ ls.entries = append(ls.entries, entries...)
+ return nil
+}
+
+// Send the sorted entries to the callback.
+func (ls *Sorter) Send() error {
+ ls.mu.Lock()
+ defer ls.mu.Unlock()
+
+ // Sort the directory entries by Remote
+ //
+ // We use a stable sort here just in case there are
+ // duplicates. Assuming the remote delivers the entries in a
+ // consistent order, this will give the best user experience
+ // in syncing as it will use the first entry for the sync
+ // comparison.
+ slices.SortStableFunc(ls.entries, func(a, b fs.DirEntry) int {
+ return cmp.Compare(ls.keyFn(a), ls.keyFn(b))
+ })
+ return ls.callback(ls.entries)
+}
+
+// CleanUp the Sorter, cleaning up any memory / files.
+//
+// It is safe and encouraged to call this regardless of whether you
+// called Send or not.
+//
+// This does not call the callback
+func (ls *Sorter) CleanUp() {
+ ls.mu.Lock()
+ defer ls.mu.Unlock()
+
+ ls.entries = nil
+}
+
+// SortToChan makes a callback for the Sorter which sends the output
+// to the channel provided.
+func SortToChan(out chan<- fs.DirEntry) fs.ListRCallback {
+ return func(entries fs.DirEntries) error {
+ for _, entry := range entries {
+ out <- entry
+ }
+ return nil
+ }
+}
diff --git a/fs/list/sorter_test.go b/fs/list/sorter_test.go
new file mode 100644
index 000000000..992389887
--- /dev/null
+++ b/fs/list/sorter_test.go
@@ -0,0 +1,104 @@
+package list
+
+import (
+ "cmp"
+ "context"
+ "fmt"
+ "slices"
+ "testing"
+
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fstest/mockdir"
+ "github.com/rclone/rclone/fstest/mockobject"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestSorter(t *testing.T) {
+ ctx := context.Background()
+ da := mockdir.New("a")
+ oA := mockobject.Object("A")
+ callback := func(entries fs.DirEntries) error {
+ require.Equal(t, fs.DirEntries{oA, da}, entries)
+ return nil
+ }
+ ls, err := NewSorter(ctx, callback, nil)
+ require.NoError(t, err)
+ assert.Equal(t, fmt.Sprintf("%p", callback), fmt.Sprintf("%p", ls.callback))
+ assert.Equal(t, fmt.Sprintf("%p", identityKeyFn), fmt.Sprintf("%p", ls.keyFn))
+ assert.Equal(t, fs.DirEntries(nil), ls.entries)
+
+ // Test Add
+ err = ls.Add(fs.DirEntries{da})
+ require.NoError(t, err)
+ assert.Equal(t, fs.DirEntries{da}, ls.entries)
+ err = ls.Add(fs.DirEntries{oA})
+ require.NoError(t, err)
+ assert.Equal(t, fs.DirEntries{da, oA}, ls.entries)
+
+ // Test Send
+ err = ls.Send()
+ require.NoError(t, err)
+
+ // Test Cleanup
+ ls.CleanUp()
+ assert.Equal(t, fs.DirEntries(nil), ls.entries)
+}
+
+func TestSorterIdentity(t *testing.T) {
+ ctx := context.Background()
+ cmpFn := func(a, b fs.DirEntry) int {
+ return cmp.Compare(a.Remote(), b.Remote())
+ }
+ callback := func(entries fs.DirEntries) error {
+ assert.True(t, slices.IsSortedFunc(entries, cmpFn))
+ assert.Equal(t, "a", entries[0].Remote())
+ return nil
+ }
+ ls, err := NewSorter(ctx, callback, nil)
+ require.NoError(t, err)
+ defer ls.CleanUp()
+
+ // Add things in reverse alphabetical order
+ for i := 'z'; i >= 'a'; i-- {
+ err = ls.Add(fs.DirEntries{mockobject.Object(string(i))})
+ require.NoError(t, err)
+ }
+ assert.Equal(t, "z", ls.entries[0].Remote())
+ assert.False(t, slices.IsSortedFunc(ls.entries, cmpFn))
+
+ // Check they get sorted
+ err = ls.Send()
+ require.NoError(t, err)
+}
+
+func TestSorterKeyFn(t *testing.T) {
+ ctx := context.Background()
+ keyFn := func(entry fs.DirEntry) string {
+ s := entry.Remote()
+ return string('z' - s[0])
+ }
+ cmpFn := func(a, b fs.DirEntry) int {
+ return cmp.Compare(keyFn(a), keyFn(b))
+ }
+ callback := func(entries fs.DirEntries) error {
+ assert.True(t, slices.IsSortedFunc(entries, cmpFn))
+ assert.Equal(t, "z", entries[0].Remote())
+ return nil
+ }
+ ls, err := NewSorter(ctx, callback, keyFn)
+ require.NoError(t, err)
+ defer ls.CleanUp()
+
+ // Add things in reverse sorted order
+ for i := 'a'; i <= 'z'; i++ {
+ err = ls.Add(fs.DirEntries{mockobject.Object(string(i))})
+ require.NoError(t, err)
+ }
+ assert.Equal(t, "a", ls.entries[0].Remote())
+ assert.False(t, slices.IsSortedFunc(ls.entries, cmpFn))
+
+ // Check they get sorted
+ err = ls.Send()
+ require.NoError(t, err)
+}