"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
list: Implement Sorter to sort directory entries
Nick Craig-Wood08e81f8420d7
diff --git a/fs/list/sorter.go b/fs/list/sorter.gonew file mode 100644index 000000000..b38d66fa3--- /dev/null+++ b/fs/list/sorter.go@@ -0,0 +1,105 @@+package list++import (+ "cmp"+ "context"+ "slices"+ "sync"++ "github.com/rclone/rclone/fs"+)++// Sorter implements an efficient mechanism for sorting list entries.+//+// If there are a large number of entries, this may be done on disk+// instead of in memory.+//+// Supply entries with the Add method, call Send at the end to deliver+// the sorted entries and finalise with CleanUp regardless of whether+// you called Add or Send.+//+// Sorted entries are delivered to the callback supplied to NewSorter+// when the Send method is called.+type Sorter struct {+ ctx context.Context+ mu sync.Mutex+ callback fs.ListRCallback+ entries fs.DirEntries+ keyFn KeyFn+}++// KeyFn turns an entry into a sort key+type KeyFn func(entry fs.DirEntry) string++// identityKeyFn maps an entry to its Remote+func identityKeyFn(entry fs.DirEntry) string {+ return entry.Remote()+}++// NewSorter creates a new Sorter with callback for sorted entries to+// be delivered to. keyFn is used to process each entry to get a key+// function, if nil then it will just use entry.Remote()+func NewSorter(ctx context.Context, callback fs.ListRCallback, keyFn KeyFn) (*Sorter, error) {+ if keyFn == nil {+ keyFn = identityKeyFn+ }+ return &Sorter{+ ctx: ctx,+ callback: callback,+ keyFn: keyFn,+ }, nil+}++// Add entries to the list sorter.+//+// Does not call the callback.+//+// Safe to call from concurrent go routines+func (ls *Sorter) Add(entries fs.DirEntries) error {+ ls.mu.Lock()+ defer ls.mu.Unlock()+ ls.entries = append(ls.entries, entries...)+ return nil+}++// Send the sorted entries to the callback.+func (ls *Sorter) Send() error {+ ls.mu.Lock()+ defer ls.mu.Unlock()++ // Sort the directory entries by Remote+ //+ // We use a stable sort here just in case there are+ // duplicates. Assuming the remote delivers the entries in a+ // consistent order, this will give the best user experience+ // in syncing as it will use the first entry for the sync+ // comparison.+ slices.SortStableFunc(ls.entries, func(a, b fs.DirEntry) int {+ return cmp.Compare(ls.keyFn(a), ls.keyFn(b))+ })+ return ls.callback(ls.entries)+}++// CleanUp the Sorter, cleaning up any memory / files.+//+// It is safe and encouraged to call this regardless of whether you+// called Send or not.+//+// This does not call the callback+func (ls *Sorter) CleanUp() {+ ls.mu.Lock()+ defer ls.mu.Unlock()++ ls.entries = nil+}++// SortToChan makes a callback for the Sorter which sends the output+// to the channel provided.+func SortToChan(out chan<- fs.DirEntry) fs.ListRCallback {+ return func(entries fs.DirEntries) error {+ for _, entry := range entries {+ out <- entry+ }+ return nil+ }+}diff --git a/fs/list/sorter_test.go b/fs/list/sorter_test.gonew file mode 100644index 000000000..992389887--- /dev/null+++ b/fs/list/sorter_test.go@@ -0,0 +1,104 @@+package list++import (+ "cmp"+ "context"+ "fmt"+ "slices"+ "testing"++ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fstest/mockdir"+ "github.com/rclone/rclone/fstest/mockobject"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++func TestSorter(t *testing.T) {+ ctx := context.Background()+ da := mockdir.New("a")+ oA := mockobject.Object("A")+ callback := func(entries fs.DirEntries) error {+ require.Equal(t, fs.DirEntries{oA, da}, entries)+ return nil+ }+ ls, err := NewSorter(ctx, callback, nil)+ require.NoError(t, err)+ assert.Equal(t, fmt.Sprintf("%p", callback), fmt.Sprintf("%p", ls.callback))+ assert.Equal(t, fmt.Sprintf("%p", identityKeyFn), fmt.Sprintf("%p", ls.keyFn))+ assert.Equal(t, fs.DirEntries(nil), ls.entries)++ // Test Add+ err = ls.Add(fs.DirEntries{da})+ require.NoError(t, err)+ assert.Equal(t, fs.DirEntries{da}, ls.entries)+ err = ls.Add(fs.DirEntries{oA})+ require.NoError(t, err)+ assert.Equal(t, fs.DirEntries{da, oA}, ls.entries)++ // Test Send+ err = ls.Send()+ require.NoError(t, err)++ // Test Cleanup+ ls.CleanUp()+ assert.Equal(t, fs.DirEntries(nil), ls.entries)+}++func TestSorterIdentity(t *testing.T) {+ ctx := context.Background()+ cmpFn := func(a, b fs.DirEntry) int {+ return cmp.Compare(a.Remote(), b.Remote())+ }+ callback := func(entries fs.DirEntries) error {+ assert.True(t, slices.IsSortedFunc(entries, cmpFn))+ assert.Equal(t, "a", entries[0].Remote())+ return nil+ }+ ls, err := NewSorter(ctx, callback, nil)+ require.NoError(t, err)+ defer ls.CleanUp()++ // Add things in reverse alphabetical order+ for i := 'z'; i >= 'a'; i-- {+ err = ls.Add(fs.DirEntries{mockobject.Object(string(i))})+ require.NoError(t, err)+ }+ assert.Equal(t, "z", ls.entries[0].Remote())+ assert.False(t, slices.IsSortedFunc(ls.entries, cmpFn))++ // Check they get sorted+ err = ls.Send()+ require.NoError(t, err)+}++func TestSorterKeyFn(t *testing.T) {+ ctx := context.Background()+ keyFn := func(entry fs.DirEntry) string {+ s := entry.Remote()+ return string('z' - s[0])+ }+ cmpFn := func(a, b fs.DirEntry) int {+ return cmp.Compare(keyFn(a), keyFn(b))+ }+ callback := func(entries fs.DirEntries) error {+ assert.True(t, slices.IsSortedFunc(entries, cmpFn))+ assert.Equal(t, "z", entries[0].Remote())+ return nil+ }+ ls, err := NewSorter(ctx, callback, keyFn)+ require.NoError(t, err)+ defer ls.CleanUp()++ // Add things in reverse sorted order+ for i := 'a'; i <= 'z'; i++ {+ err = ls.Add(fs.DirEntries{mockobject.Object(string(i))})+ require.NoError(t, err)+ }+ assert.Equal(t, "a", ls.entries[0].Remote())+ assert.False(t, slices.IsSortedFunc(ls.entries, cmpFn))++ // Check they get sorted+ err = ls.Send()+ require.NoError(t, err)+}