"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
list: add ListDirSortedFn for callback oriented directory listing
Nick Craig-Wood0f7ecf6f06a6
diff --git a/fs/list/list.go b/fs/list/list.goindex edd56f889..d2caf0323 100644--- a/fs/list/list.go+++ b/fs/list/list.go@@ -39,8 +39,64 @@ func DirSorted(ctx context.Context, f fs.Fs, includeAll bool, dir string) (entrireturn filterAndSortDir(ctx, entries, includeAll, dir, fi.IncludeObject, fi.IncludeDirectory(ctx, f))}-// filter (if required) and check the entries, then sort them-func filterAndSortDir(ctx context.Context, entries fs.DirEntries, includeAll bool, dir string,+// listP for every backend+func listP(ctx context.Context, f fs.Fs, dir string, callback fs.ListRCallback) error {+ if doListP := f.Features().ListP; doListP != nil {+ return doListP(ctx, dir, callback)+ }+ // Fallback to List+ entries, err := f.List(ctx, dir)+ if err != nil {+ return err+ }+ return callback(entries)+}++// DirSortedFn reads Object and *Dir into entries for the given Fs.+//+// dir is the start directory, "" for root+//+// If includeAll is specified all files will be added, otherwise only+// files and directories passing the filter will be added.+//+// Files will be returned through callback in sorted order+func DirSortedFn(ctx context.Context, f fs.Fs, includeAll bool, dir string, callback fs.ListRCallback, keyFn KeyFn) (err error) {+ stats := accounting.Stats(ctx)+ fi := filter.GetConfig(ctx)++ // Sort the entries, in or out of memory+ sorter, err := NewSorter(ctx, callback, keyFn)+ if err != nil {+ return fmt.Errorf("failed to create directory sorter: %w", err)+ }+ defer sorter.CleanUp()++ // Get unfiltered entries from the fs+ err = listP(ctx, f, dir, func(entries fs.DirEntries) error {+ stats.Listed(int64(len(entries)))++ // This should happen only if exclude files lives in the+ // starting directory, otherwise ListDirSorted should not be+ // called.+ if !includeAll && fi.ListContainsExcludeFile(entries) {+ fs.Debugf(dir, "Excluded")+ return nil+ }++ entries, err := filterDir(ctx, entries, includeAll, dir, fi.IncludeObject, fi.IncludeDirectory(ctx, f))+ if err != nil {+ return err+ }+ return sorter.Add(entries)+ })+ if err != nil {+ return err+ }+ return sorter.Send()+}++// Filter the entries passed in+func filterDir(ctx context.Context, entries fs.DirEntries, includeAll bool, dir string,IncludeObject func(ctx context.Context, o fs.Object) bool,IncludeDirectory func(remote string) (bool, error)) (newEntries fs.DirEntries, err error) {newEntries = entries[:0] // in place filter@@ -96,7 +152,18 @@ func filterAndSortDir(ctx context.Context, entries fs.DirEntries, includeAll boonewEntries = append(newEntries, entry)}}- entries = newEntries+ return newEntries, nil+}++// filter and sort the entries+func filterAndSortDir(ctx context.Context, entries fs.DirEntries, includeAll bool, dir string,+ IncludeObject func(ctx context.Context, o fs.Object) bool,+ IncludeDirectory func(remote string) (bool, error)) (newEntries fs.DirEntries, err error) {+ // Filter the directory entries (in place)+ entries, err = filterDir(ctx, entries, includeAll, dir, IncludeObject, IncludeDirectory)+ if err != nil {+ return nil, err+ }// Sort the directory entries by Remote//diff --git a/fs/operations/listdirsorted_test.go b/fs/operations/listdirsorted_test.goindex 1bcd8860e..4719d6c55 100644--- a/fs/operations/listdirsorted_test.go+++ b/fs/operations/listdirsorted_test.go@@ -12,9 +12,9 @@ import ("github.com/stretchr/testify/require")-// TestListDirSorted is integration testing code in fs/list/list.go+// testListDirSorted is integration testing code in fs/list/list.go// which can't be tested there due to import loops.-func TestListDirSorted(t *testing.T) {+func testListDirSorted(t *testing.T, listFn func(ctx context.Context, f fs.Fs, includeAll bool, dir string) (entries fs.DirEntries, err error)) {r := fstest.NewRun(t)ctx := context.Background()@@ -52,20 +52,20 @@ func TestListDirSorted(t *testing.T) {return name}- items, err = list.DirSorted(context.Background(), r.Fremote, true, "")+ items, err = listFn(context.Background(), r.Fremote, true, "")require.NoError(t, err)require.Len(t, items, 3)assert.Equal(t, "a.txt", str(0))assert.Equal(t, "sub dir/", str(1))assert.Equal(t, "zend.txt", str(2))- items, err = list.DirSorted(context.Background(), r.Fremote, false, "")+ items, err = listFn(context.Background(), r.Fremote, false, "")require.NoError(t, err)require.Len(t, items, 2)assert.Equal(t, "sub dir/", str(0))assert.Equal(t, "zend.txt", str(1))- items, err = list.DirSorted(context.Background(), r.Fremote, true, "sub dir")+ items, err = listFn(context.Background(), r.Fremote, true, "sub dir")require.NoError(t, err)require.Len(t, items, 4)assert.Equal(t, "sub dir/hello world", str(0))@@ -73,7 +73,7 @@ func TestListDirSorted(t *testing.T) {assert.Equal(t, "sub dir/ignore dir/", str(2))assert.Equal(t, "sub dir/sub sub dir/", str(3))- items, err = list.DirSorted(context.Background(), r.Fremote, false, "sub dir")+ items, err = listFn(context.Background(), r.Fremote, false, "sub dir")require.NoError(t, err)require.Len(t, items, 2)assert.Equal(t, "sub dir/ignore dir/", str(0))@@ -82,25 +82,45 @@ func TestListDirSorted(t *testing.T) {// testing ignore filefi.Opt.ExcludeFile = []string{".ignore"}- items, err = list.DirSorted(context.Background(), r.Fremote, false, "sub dir")+ items, err = listFn(context.Background(), r.Fremote, false, "sub dir")require.NoError(t, err)require.Len(t, items, 1)assert.Equal(t, "sub dir/sub sub dir/", str(0))- items, err = list.DirSorted(context.Background(), r.Fremote, false, "sub dir/ignore dir")+ items, err = listFn(context.Background(), r.Fremote, false, "sub dir/ignore dir")require.NoError(t, err)require.Len(t, items, 0)- items, err = list.DirSorted(context.Background(), r.Fremote, true, "sub dir/ignore dir")+ items, err = listFn(context.Background(), r.Fremote, true, "sub dir/ignore dir")require.NoError(t, err)require.Len(t, items, 2)assert.Equal(t, "sub dir/ignore dir/.ignore", str(0))assert.Equal(t, "sub dir/ignore dir/should be ignored", str(1))fi.Opt.ExcludeFile = nil- items, err = list.DirSorted(context.Background(), r.Fremote, false, "sub dir/ignore dir")+ items, err = listFn(context.Background(), r.Fremote, false, "sub dir/ignore dir")require.NoError(t, err)require.Len(t, items, 2)assert.Equal(t, "sub dir/ignore dir/.ignore", str(0))assert.Equal(t, "sub dir/ignore dir/should be ignored", str(1))}++// TestListDirSorted is integration testing code in fs/list/list.go+// which can't be tested there due to import loops.+func TestListDirSorted(t *testing.T) {+ testListDirSorted(t, list.DirSorted)+}++// TestListDirSortedFn is integration testing code in fs/list/list.go+// which can't be tested there due to import loops.+func TestListDirSortedFn(t *testing.T) {+ listFn := func(ctx context.Context, f fs.Fs, includeAll bool, dir string) (entries fs.DirEntries, err error) {+ callback := func(newEntries fs.DirEntries) error {+ entries = append(entries, newEntries...)+ return nil+ }+ err = list.DirSortedFn(ctx, f, includeAll, dir, callback, nil)+ return entries, err+ }+ testListDirSorted(t, listFn)+}