Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

filter: add --files-from0 to support NUL-delimited input - fixes #9537

Gaurav761af75a252f
diff --git a/docs/content/docs.md b/docs/content/docs.md
index 6971d3513..517b1942a 100644
--- a/docs/content/docs.md
+++ b/docs/content/docs.md
@@ -3320,6 +3320,7 @@ For the filtering options
- `--include-from`
- `--files-from`
- `--files-from-raw`
+- `--files-from0`
- `--min-size`
- `--max-size`
- `--min-age`
diff --git a/docs/content/filtering.md b/docs/content/filtering.md
index 8c587c968..399b8da05 100644
--- a/docs/content/filtering.md
+++ b/docs/content/filtering.md
@@ -272,8 +272,8 @@ is included.
Any path/file included at that stage is processed by the rclone
command.
-`--files-from` and `--files-from-raw` flags over-ride and cannot be
-combined with other filter options.
+`--files-from`, `--files-from-raw` and `--files-from0` flags
+over-ride and cannot be combined with other filter options.
To see the internal combined rule list, in regular expression form,
for a command add the `--dump filters` flag. Running an rclone command
@@ -396,8 +396,8 @@ processed in.
`--exclude` should not be used with `--include`, `--include-from`,
`--filter` or `--filter-from` flags.
-`--exclude` has no effect when combined with `--files-from` or
-`--files-from-raw` flags.
+`--exclude` has no effect when combined with `--files-from`,
+`--files-from-raw` or `--files-from0` flags.
E.g. `rclone ls remote: --exclude *.bak` excludes all .bak files
from listing.
@@ -440,8 +440,8 @@ are applied to an rclone command.
`--exclude-from` should not be used with `--include`, `--include-from`,
`--filter` or `--filter-from` flags.
-`--exclude-from` has no effect when combined with `--files-from` or
-`--files-from-raw` flags.
+`--exclude-from` has no effect when combined with `--files-from`,
+`--files-from-raw` or `--files-from0` flags.
`--exclude-from` followed by `-` reads filter rules from standard input.
@@ -453,8 +453,8 @@ command.
This flag can be repeated. See above for the order filter flags are
processed in.
-`--include` has no effect when combined with `--files-from` or
-`--files-from-raw` flags.
+`--include` has no effect when combined with `--files-from`,
+`--files-from-raw` or `--files-from0` flags.
`--include` implies `--exclude **` at the end of an rclone internal
filter list. Therefore if you mix `--include` and `--include-from`
@@ -513,8 +513,8 @@ flags with `--exclude`, `--exclude-from`, `--filter` or `--filter-from`,
you must use include rules for all the files you want in the include
statement. For more flexibility use the `--filter-from` flag.
-`--include-from` has no effect when combined with `--files-from` or
-`--files-from-raw` flags.
+`--include-from` has no effect when combined with `--files-from`,
+`--files-from-raw` or `--files-from0` flags.
`--include-from` followed by `-` reads filter rules from standard input.
@@ -531,8 +531,8 @@ implies an `--exclude *` rule which it adds to the bottom of the internal rule
list. `--filter...+` does not imply
that rule.
-`--filter` has no effect when combined with `--files-from` or
-`--files-from-raw` flags.
+`--filter` has no effect when combined with `--files-from`,
+`--files-from-raw` or `--files-from0` flags.
`--filter` should not be used with `--include`, `--include-from`,
`--exclude` or `--exclude-from` flags.
@@ -612,7 +612,7 @@ no others.
Other filter flags (`--include`, `--include-from`, `--exclude`,
`--exclude-from`, `--filter` and `--filter-from`) are ignored when
-`--files-from` is used.
+`--files-from`, `--files-from-raw` or `--files-from0` is used.
`--files-from` expects a list of files as its input. Leading or
trailing whitespace is stripped from the input lines. Lines starting
@@ -719,6 +719,21 @@ with `;` or `#` are read without any processing. [rclone lsf](/commands/rclone_l
has a compatible format that can be used to export file lists from remotes for
input to `--files-from-raw`.
+### `--files-from0` - Read NUL separated list of source-file names
+
+This flag is the same as `--files-from-raw` except that input is
+split on NUL (`\0`) characters instead of newlines. This allows
+handling filenames that contain embedded newline characters.
+
+It is similar to the `-print0` option of `find` and the `-0`
+option of `xargs`.
+
+E.g. to copy files listed by `find`:
+
+```console
+find /path -print0 | rclone copy --files-from0 - / remote:path
+```
+
### `--ignore-case` - make searches case insensitive
By default, rclone filter patterns are case sensitive. The `--ignore-case`
@@ -846,7 +861,7 @@ This will stay constant across retries.
- Safe to use with `rclone sync`; source and destination selections will match.
- **Do not** use with `--delete-excluded`, as this could delete unselected files.
-- Ignored if `--files-from` is used.
+- Ignored if `--files-from`, `--files-from-raw` or `--files-from0` is used.
#### Examples
diff --git a/fs/filter/filter.go b/fs/filter/filter.go
index ab0310dc1..e00d0e8b3 100644
--- a/fs/filter/filter.go
+++ b/fs/filter/filter.go
@@ -45,6 +45,11 @@ var OptionsInfo = fs.Options{{
Default: []string{},
Help: "Read list of source-file names from file without any processing of lines (use - to read from stdin)",
Groups: "Filter",
+}, {
+ Name: "files_from0",
+ Default: []string{},
+ Help: "Read list of source-file names from file using NUL as separator (use - to read from stdin)",
+ Groups: "Filter",
}, {
Name: "min_age",
Default: fs.DurationOff,
@@ -145,6 +150,7 @@ type Options struct {
ExcludeFile []string `config:"exclude_if_present"`
FilesFrom []string `config:"files_from"`
FilesFromRaw []string `config:"files_from_raw"`
+ FilesFrom0 []string `config:"files_from0"`
MetaRules RulesOpt `config:"metadata"`
MinAge fs.Duration `config:"min_age"`
MaxAge fs.Duration `config:"max_age"`
@@ -226,10 +232,10 @@ func NewFilter(opt *Options) (f *Filter, err error) {
for _, rule := range f.Opt.FilesFrom {
if !inActive {
- return nil, fmt.Errorf("the usage of --files-from overrides all other filters, it should be used alone or with --files-from-raw")
+ return nil, fmt.Errorf("the usage of --files-from overrides all other filters, it should be used alone or with --files-from-raw or --files-from0")
}
f.initAddFile() // init to show --files-from set even if no files within
- err := forEachLine(rule, false, func(line string) error {
+ err := forEachLine(rule, false, false, func(line string) error {
return f.AddFile(line)
})
if err != nil {
@@ -241,10 +247,23 @@ func NewFilter(opt *Options) (f *Filter, err error) {
// --files-from-raw can be used with --files-from, hence we do
// not need to get the value of f.InActive again
if !inActive {
- return nil, fmt.Errorf("the usage of --files-from-raw overrides all other filters, it should be used alone or with --files-from")
+ return nil, fmt.Errorf("the usage of --files-from-raw overrides all other filters, it should be used alone or with --files-from or --files-from0")
+ }
+ f.initAddFile() // init to show --files-from set even if no files within
+ err := forEachLine(rule, true, false, func(line string) error {
+ return f.AddFile(line)
+ })
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ for _, rule := range f.Opt.FilesFrom0 {
+ if !inActive {
+ return nil, fmt.Errorf("the usage of --files-from0 overrides all other filters, it should be used alone or with --files-from or --files-from-raw")
}
f.initAddFile() // init to show --files-from set even if no files within
- err := forEachLine(rule, true, func(line string) error {
+ err := forEachLine(rule, true, true, func(line string) error {
return f.AddFile(line)
})
if err != nil {
diff --git a/fs/filter/filter_test.go b/fs/filter/filter_test.go
index 67c8b9bc1..75f8c77e1 100644
--- a/fs/filter/filter_test.go
+++ b/fs/filter/filter_test.go
@@ -176,6 +176,57 @@ func TestNewFilterWithFilesFromRaw(t *testing.T) {
}
}
+func TestNewFilterForbiddenMixOfFilesFrom0AndFilterRule(t *testing.T) {
+ Opt := Opt
+
+ // Set up the input
+ Opt.FilterRule = []string{"- filter1", "- filter1b"}
+ Opt.FilesFrom0 = []string{testFile(t, "#comment\x00files1\x00files2\x00")}
+
+ rm := func(p string) {
+ err := os.Remove(p)
+ if err != nil {
+ t.Logf("error removing %q: %v", p, err)
+ }
+ }
+ // Reset the input
+ defer func() {
+ rm(Opt.FilesFrom0[0])
+ }()
+
+ _, err := NewFilter(&Opt)
+ require.Error(t, err)
+ require.Contains(t, err.Error(), "the usage of --files-from0 overrides all other filters")
+}
+
+func TestNewFilterWithFilesFrom0(t *testing.T) {
+ Opt := Opt
+
+ // Set up the input: NUL-separated, with an embedded newline in one entry
+ Opt.FilesFrom0 = []string{testFile(t, "#comment\x00files1\nmore\x00files2\x00")}
+
+ rm := func(p string) {
+ err := os.Remove(p)
+ if err != nil {
+ t.Logf("error removing %q: %v", p, err)
+ }
+ }
+ // Reset the input
+ defer func() {
+ rm(Opt.FilesFrom0[0])
+ }()
+
+ f, err := NewFilter(&Opt)
+ require.NoError(t, err)
+ assert.Len(t, f.files, 3)
+ for _, name := range []string{"#comment", "files1\nmore", "files2"} {
+ _, ok := f.files[name]
+ if !ok {
+ t.Errorf("Didn't find file %q in f.files", name)
+ }
+ }
+}
+
func TestNewFilterFullExceptFilesFromOpt(t *testing.T) {
Opt := Opt
@@ -770,7 +821,7 @@ five
}()
fileName = "-"
}
- err := forEachLine(fileName, raw, func(s string) error {
+ err := forEachLine(fileName, raw, false, func(s string) error {
lines = append(lines, s)
return nil
})
@@ -799,6 +850,71 @@ func TestFilterForEachLineStdinWithRaw(t *testing.T) {
testFilterForEachLine(t, true, true)
}
+func testFilterForEachLineNul(t *testing.T, useStdin bool) {
+ file := testFile(t, "one\x00two\nthree\x00four\x00five\x00")
+ defer func() {
+ err := os.Remove(file)
+ require.NoError(t, err)
+ }()
+ lines := []string{}
+ fileName := file
+ if useStdin {
+ in, err := os.Open(file)
+ require.NoError(t, err)
+ oldStdin := os.Stdin
+ os.Stdin = in
+ defer func() {
+ os.Stdin = oldStdin
+ _ = in.Close()
+ }()
+ fileName = "-"
+ }
+ err := forEachLine(fileName, true, true, func(s string) error {
+ lines = append(lines, s)
+ return nil
+ })
+ require.NoError(t, err)
+ assert.Equal(t, []string{"one", "two\nthree", "four", "five"}, lines)
+}
+
+func TestFilterForEachLineNul(t *testing.T) {
+ testFilterForEachLineNul(t, false)
+}
+
+func TestFilterForEachLineNulStdin(t *testing.T) {
+ testFilterForEachLineNul(t, true)
+}
+
+func TestFilterForEachLineNulNoTrailing(t *testing.T) {
+ file := testFile(t, "one\x00two\x00three")
+ defer func() {
+ err := os.Remove(file)
+ require.NoError(t, err)
+ }()
+ var lines []string
+ err := forEachLine(file, true, true, func(s string) error {
+ lines = append(lines, s)
+ return nil
+ })
+ require.NoError(t, err)
+ assert.Equal(t, []string{"one", "two", "three"}, lines)
+}
+
+func TestFilterForEachLineNulConsecutive(t *testing.T) {
+ file := testFile(t, "one\x00\x00two\x00")
+ defer func() {
+ err := os.Remove(file)
+ require.NoError(t, err)
+ }()
+ var lines []string
+ err := forEachLine(file, true, true, func(s string) error {
+ lines = append(lines, s)
+ return nil
+ })
+ require.NoError(t, err)
+ assert.Equal(t, []string{"one", "", "two"}, lines)
+}
+
func TestFilterMatchesFromDocs(t *testing.T) {
for _, test := range []struct {
glob string
diff --git a/fs/filter/rules.go b/fs/filter/rules.go
index 085a02aa5..e5ee2626e 100644
--- a/fs/filter/rules.go
+++ b/fs/filter/rules.go
@@ -2,6 +2,7 @@ package filter
import (
"bufio"
+ "bytes"
"fmt"
"os"
"regexp"
@@ -114,10 +115,26 @@ func (rs *rules) includeMany(remotes []string) bool {
return true
}
+// scanNul is a split function for a Scanner that returns each NUL-terminated
+// sequence of bytes. It correctly handles the final segment even if it
+// lacks a trailing NUL.
+func scanNul(data []byte, atEOF bool) (advance int, token []byte, err error) {
+ if atEOF && len(data) == 0 {
+ return 0, nil, nil
+ }
+ if i := bytes.IndexByte(data, '\x00'); i >= 0 {
+ return i + 1, data[:i], nil
+ }
+ if atEOF {
+ return len(data), data, nil
+ }
+ return 0, nil, nil
+}
+
// forEachLine calls fn on every line in the file pointed to by path
//
// It ignores empty lines and lines starting with '#' or ';' if raw is false
-func forEachLine(path string, raw bool, fn func(string) error) (err error) {
+func forEachLine(path string, raw bool, useNulDelimiter bool, fn func(string) error) (err error) {
var scanner *bufio.Scanner
if path == "-" {
scanner = bufio.NewScanner(os.Stdin)
@@ -129,6 +146,11 @@ func forEachLine(path string, raw bool, fn func(string) error) (err error) {
scanner = bufio.NewScanner(in)
defer fs.CheckClose(in, &err)
}
+
+ if useNulDelimiter {
+ scanner.Split(scanNul)
+ }
+
for scanner.Scan() {
line := scanner.Text()
if !raw {
@@ -199,7 +221,7 @@ func parseRules(opt *RulesOpt, add addFn, clear clearFn) (err error) {
addImplicitExclude = true
}
for _, rule := range opt.IncludeFrom {
- err := forEachLine(rule, false, func(line string) error {
+ err := forEachLine(rule, false, false, func(line string) error {
return add(true, line)
})
if err != nil {
@@ -215,7 +237,7 @@ func parseRules(opt *RulesOpt, add addFn, clear clearFn) (err error) {
foundExcludeRule = true
}
for _, rule := range opt.ExcludeFrom {
- err := forEachLine(rule, false, func(line string) error {
+ err := forEachLine(rule, false, false, func(line string) error {
return add(false, line)
})
if err != nil {
@@ -235,7 +257,7 @@ func parseRules(opt *RulesOpt, add addFn, clear clearFn) (err error) {
}
}
for _, rule := range opt.FilterFrom {
- err := forEachLine(rule, false, func(rule string) error {
+ err := forEachLine(rule, false, false, func(rule string) error {
return addRule(rule, add, clear)
})
if err != nil {