Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

list: add ListDirSortedFn for callback oriented directory listing

Nick Craig-Wood0f7ecf6f06a6
diff --git a/fs/list/list.go b/fs/list/list.go
index edd56f889..d2caf0323 100644
--- a/fs/list/list.go
+++ b/fs/list/list.go
@@ -39,8 +39,64 @@ func DirSorted(ctx context.Context, f fs.Fs, includeAll bool, dir string) (entri
return filterAndSortDir(ctx, entries, includeAll, dir, fi.IncludeObject, fi.IncludeDirectory(ctx, f))
}
-// filter (if required) and check the entries, then sort them
-func filterAndSortDir(ctx context.Context, entries fs.DirEntries, includeAll bool, dir string,
+// listP for every backend
+func listP(ctx context.Context, f fs.Fs, dir string, callback fs.ListRCallback) error {
+ if doListP := f.Features().ListP; doListP != nil {
+ return doListP(ctx, dir, callback)
+ }
+ // Fallback to List
+ entries, err := f.List(ctx, dir)
+ if err != nil {
+ return err
+ }
+ return callback(entries)
+}
+
+// DirSortedFn reads Object and *Dir into entries for the given Fs.
+//
+// dir is the start directory, "" for root
+//
+// If includeAll is specified all files will be added, otherwise only
+// files and directories passing the filter will be added.
+//
+// Files will be returned through callback in sorted order
+func DirSortedFn(ctx context.Context, f fs.Fs, includeAll bool, dir string, callback fs.ListRCallback, keyFn KeyFn) (err error) {
+ stats := accounting.Stats(ctx)
+ fi := filter.GetConfig(ctx)
+
+ // Sort the entries, in or out of memory
+ sorter, err := NewSorter(ctx, callback, keyFn)
+ if err != nil {
+ return fmt.Errorf("failed to create directory sorter: %w", err)
+ }
+ defer sorter.CleanUp()
+
+ // Get unfiltered entries from the fs
+ err = listP(ctx, f, dir, func(entries fs.DirEntries) error {
+ stats.Listed(int64(len(entries)))
+
+ // This should happen only if exclude files lives in the
+ // starting directory, otherwise ListDirSorted should not be
+ // called.
+ if !includeAll && fi.ListContainsExcludeFile(entries) {
+ fs.Debugf(dir, "Excluded")
+ return nil
+ }
+
+ entries, err := filterDir(ctx, entries, includeAll, dir, fi.IncludeObject, fi.IncludeDirectory(ctx, f))
+ if err != nil {
+ return err
+ }
+ return sorter.Add(entries)
+ })
+ if err != nil {
+ return err
+ }
+ return sorter.Send()
+}
+
+// Filter the entries passed in
+func filterDir(ctx context.Context, entries fs.DirEntries, includeAll bool, dir string,
IncludeObject func(ctx context.Context, o fs.Object) bool,
IncludeDirectory func(remote string) (bool, error)) (newEntries fs.DirEntries, err error) {
newEntries = entries[:0] // in place filter
@@ -96,7 +152,18 @@ func filterAndSortDir(ctx context.Context, entries fs.DirEntries, includeAll boo
newEntries = append(newEntries, entry)
}
}
- entries = newEntries
+ return newEntries, nil
+}
+
+// filter and sort the entries
+func filterAndSortDir(ctx context.Context, entries fs.DirEntries, includeAll bool, dir string,
+ IncludeObject func(ctx context.Context, o fs.Object) bool,
+ IncludeDirectory func(remote string) (bool, error)) (newEntries fs.DirEntries, err error) {
+ // Filter the directory entries (in place)
+ entries, err = filterDir(ctx, entries, includeAll, dir, IncludeObject, IncludeDirectory)
+ if err != nil {
+ return nil, err
+ }
// Sort the directory entries by Remote
//
diff --git a/fs/operations/listdirsorted_test.go b/fs/operations/listdirsorted_test.go
index 1bcd8860e..4719d6c55 100644
--- a/fs/operations/listdirsorted_test.go
+++ b/fs/operations/listdirsorted_test.go
@@ -12,9 +12,9 @@ import (
"github.com/stretchr/testify/require"
)
-// TestListDirSorted is integration testing code in fs/list/list.go
+// testListDirSorted is integration testing code in fs/list/list.go
// which can't be tested there due to import loops.
-func TestListDirSorted(t *testing.T) {
+func testListDirSorted(t *testing.T, listFn func(ctx context.Context, f fs.Fs, includeAll bool, dir string) (entries fs.DirEntries, err error)) {
r := fstest.NewRun(t)
ctx := context.Background()
@@ -52,20 +52,20 @@ func TestListDirSorted(t *testing.T) {
return name
}
- items, err = list.DirSorted(context.Background(), r.Fremote, true, "")
+ items, err = listFn(context.Background(), r.Fremote, true, "")
require.NoError(t, err)
require.Len(t, items, 3)
assert.Equal(t, "a.txt", str(0))
assert.Equal(t, "sub dir/", str(1))
assert.Equal(t, "zend.txt", str(2))
- items, err = list.DirSorted(context.Background(), r.Fremote, false, "")
+ items, err = listFn(context.Background(), r.Fremote, false, "")
require.NoError(t, err)
require.Len(t, items, 2)
assert.Equal(t, "sub dir/", str(0))
assert.Equal(t, "zend.txt", str(1))
- items, err = list.DirSorted(context.Background(), r.Fremote, true, "sub dir")
+ items, err = listFn(context.Background(), r.Fremote, true, "sub dir")
require.NoError(t, err)
require.Len(t, items, 4)
assert.Equal(t, "sub dir/hello world", str(0))
@@ -73,7 +73,7 @@ func TestListDirSorted(t *testing.T) {
assert.Equal(t, "sub dir/ignore dir/", str(2))
assert.Equal(t, "sub dir/sub sub dir/", str(3))
- items, err = list.DirSorted(context.Background(), r.Fremote, false, "sub dir")
+ items, err = listFn(context.Background(), r.Fremote, false, "sub dir")
require.NoError(t, err)
require.Len(t, items, 2)
assert.Equal(t, "sub dir/ignore dir/", str(0))
@@ -82,25 +82,45 @@ func TestListDirSorted(t *testing.T) {
// testing ignore file
fi.Opt.ExcludeFile = []string{".ignore"}
- items, err = list.DirSorted(context.Background(), r.Fremote, false, "sub dir")
+ items, err = listFn(context.Background(), r.Fremote, false, "sub dir")
require.NoError(t, err)
require.Len(t, items, 1)
assert.Equal(t, "sub dir/sub sub dir/", str(0))
- items, err = list.DirSorted(context.Background(), r.Fremote, false, "sub dir/ignore dir")
+ items, err = listFn(context.Background(), r.Fremote, false, "sub dir/ignore dir")
require.NoError(t, err)
require.Len(t, items, 0)
- items, err = list.DirSorted(context.Background(), r.Fremote, true, "sub dir/ignore dir")
+ items, err = listFn(context.Background(), r.Fremote, true, "sub dir/ignore dir")
require.NoError(t, err)
require.Len(t, items, 2)
assert.Equal(t, "sub dir/ignore dir/.ignore", str(0))
assert.Equal(t, "sub dir/ignore dir/should be ignored", str(1))
fi.Opt.ExcludeFile = nil
- items, err = list.DirSorted(context.Background(), r.Fremote, false, "sub dir/ignore dir")
+ items, err = listFn(context.Background(), r.Fremote, false, "sub dir/ignore dir")
require.NoError(t, err)
require.Len(t, items, 2)
assert.Equal(t, "sub dir/ignore dir/.ignore", str(0))
assert.Equal(t, "sub dir/ignore dir/should be ignored", str(1))
}
+
+// TestListDirSorted is integration testing code in fs/list/list.go
+// which can't be tested there due to import loops.
+func TestListDirSorted(t *testing.T) {
+ testListDirSorted(t, list.DirSorted)
+}
+
+// TestListDirSortedFn is integration testing code in fs/list/list.go
+// which can't be tested there due to import loops.
+func TestListDirSortedFn(t *testing.T) {
+ listFn := func(ctx context.Context, f fs.Fs, includeAll bool, dir string) (entries fs.DirEntries, err error) {
+ callback := func(newEntries fs.DirEntries) error {
+ entries = append(entries, newEntries...)
+ return nil
+ }
+ err = list.DirSortedFn(ctx, f, includeAll, dir, callback, nil)
+ return entries, err
+ }
+ testListDirSorted(t, listFn)
+}