Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

filter: fix --files-from copy stopping at the first unreadable file

Nick Craig-Wood9ab8e4724a03
diff --git a/fs/filter/filter.go b/fs/filter/filter.go
index 18590c2fe..ab0310dc1 100644
--- a/fs/filter/filter.go
+++ b/fs/filter/filter.go
@@ -631,7 +631,11 @@ func (f *Filter) MakeListR(ctx context.Context, NewObject func(ctx context.Conte
if err == fs.ErrorObjectNotFound {
// Skip files that are not found
} else if err != nil {
- return err
+ // Count and log the error but carry on so that one
+ // unreadable file (e.g. permission denied) doesn't stop
+ // the other --files-from files being processed.
+ fs.Errorf(remote, "--files-from failed to read file: %v", err)
+ _ = fs.CountError(gCtx, err)
} else {
err = callback(entries)
if err != nil {
diff --git a/fs/filter/filter_test.go b/fs/filter/filter_test.go
index a5e82d3ab..67c8b9bc1 100644
--- a/fs/filter/filter_test.go
+++ b/fs/filter/filter_test.go
@@ -391,19 +391,22 @@ func TestNewFilterMakeListR(t *testing.T) {
assert.Equal(t, want, newObjects)
assert.Equal(t, want, listRObjects)
- // Now check an error is returned from NewObject
+ // Check that a file which errors on NewObject doesn't stop the
+ // others being listed - the error is counted and logged rather than
+ // being returned.
require.NoError(t, f.AddFile("error"))
err = listR(context.Background(), "", listRcallback)
- require.EqualError(t, err, assert.AnError.Error())
+ require.NoError(t, err)
+ assert.Equal(t, want, listRObjects)
- // The checker will exit by the error above
+ // The same with a single checker
ci := fs.GetConfig(context.Background())
ci.Checkers = 1
- // Now check an error is returned from NewObject
require.NoError(t, f.AddFile("error"))
err = listR(context.Background(), "", listRcallback)
- require.EqualError(t, err, assert.AnError.Error())
+ require.NoError(t, err)
+ assert.Equal(t, want, listRObjects)
}
func TestNewFilterMinSize(t *testing.T) {