"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
filter: add --files-from0 to support NUL-delimited input - fixes #9537
Gaurav761af75a252f
diff --git a/docs/content/docs.md b/docs/content/docs.mdindex 6971d3513..517b1942a 100644--- a/docs/content/docs.md+++ b/docs/content/docs.md@@ -3320,6 +3320,7 @@ For the filtering options- `--include-from`- `--files-from`- `--files-from-raw`+- `--files-from0`- `--min-size`- `--max-size`- `--min-age`diff --git a/docs/content/filtering.md b/docs/content/filtering.mdindex 8c587c968..399b8da05 100644--- a/docs/content/filtering.md+++ b/docs/content/filtering.md@@ -272,8 +272,8 @@ is included.Any path/file included at that stage is processed by the rclonecommand.-`--files-from` and `--files-from-raw` flags over-ride and cannot be-combined with other filter options.+`--files-from`, `--files-from-raw` and `--files-from0` flags+over-ride and cannot be combined with other filter options.To see the internal combined rule list, in regular expression form,for a command add the `--dump filters` flag. Running an rclone command@@ -396,8 +396,8 @@ processed in.`--exclude` should not be used with `--include`, `--include-from`,`--filter` or `--filter-from` flags.-`--exclude` has no effect when combined with `--files-from` or-`--files-from-raw` flags.+`--exclude` has no effect when combined with `--files-from`,+`--files-from-raw` or `--files-from0` flags.E.g. `rclone ls remote: --exclude *.bak` excludes all .bak filesfrom listing.@@ -440,8 +440,8 @@ are applied to an rclone command.`--exclude-from` should not be used with `--include`, `--include-from`,`--filter` or `--filter-from` flags.-`--exclude-from` has no effect when combined with `--files-from` or-`--files-from-raw` flags.+`--exclude-from` has no effect when combined with `--files-from`,+`--files-from-raw` or `--files-from0` flags.`--exclude-from` followed by `-` reads filter rules from standard input.@@ -453,8 +453,8 @@ command.This flag can be repeated. See above for the order filter flags areprocessed in.-`--include` has no effect when combined with `--files-from` or-`--files-from-raw` flags.+`--include` has no effect when combined with `--files-from`,+`--files-from-raw` or `--files-from0` flags.`--include` implies `--exclude **` at the end of an rclone internalfilter list. Therefore if you mix `--include` and `--include-from`@@ -513,8 +513,8 @@ flags with `--exclude`, `--exclude-from`, `--filter` or `--filter-from`,you must use include rules for all the files you want in the includestatement. For more flexibility use the `--filter-from` flag.-`--include-from` has no effect when combined with `--files-from` or-`--files-from-raw` flags.+`--include-from` has no effect when combined with `--files-from`,+`--files-from-raw` or `--files-from0` flags.`--include-from` followed by `-` reads filter rules from standard input.@@ -531,8 +531,8 @@ implies an `--exclude *` rule which it adds to the bottom of the internal rulelist. `--filter...+` does not implythat rule.-`--filter` has no effect when combined with `--files-from` or-`--files-from-raw` flags.+`--filter` has no effect when combined with `--files-from`,+`--files-from-raw` or `--files-from0` flags.`--filter` should not be used with `--include`, `--include-from`,`--exclude` or `--exclude-from` flags.@@ -612,7 +612,7 @@ no others.Other filter flags (`--include`, `--include-from`, `--exclude`,`--exclude-from`, `--filter` and `--filter-from`) are ignored when-`--files-from` is used.+`--files-from`, `--files-from-raw` or `--files-from0` is used.`--files-from` expects a list of files as its input. Leading ortrailing whitespace is stripped from the input lines. Lines starting@@ -719,6 +719,21 @@ with `;` or `#` are read without any processing. [rclone lsf](/commands/rclone_lhas a compatible format that can be used to export file lists from remotes forinput to `--files-from-raw`.+### `--files-from0` - Read NUL separated list of source-file names++This flag is the same as `--files-from-raw` except that input is+split on NUL (`\0`) characters instead of newlines. This allows+handling filenames that contain embedded newline characters.++It is similar to the `-print0` option of `find` and the `-0`+option of `xargs`.++E.g. to copy files listed by `find`:++```console+find /path -print0 | rclone copy --files-from0 - / remote:path+```+### `--ignore-case` - make searches case insensitiveBy default, rclone filter patterns are case sensitive. The `--ignore-case`@@ -846,7 +861,7 @@ This will stay constant across retries.- Safe to use with `rclone sync`; source and destination selections will match.- **Do not** use with `--delete-excluded`, as this could delete unselected files.-- Ignored if `--files-from` is used.+- Ignored if `--files-from`, `--files-from-raw` or `--files-from0` is used.#### Examplesdiff --git a/fs/filter/filter.go b/fs/filter/filter.goindex ab0310dc1..e00d0e8b3 100644--- a/fs/filter/filter.go+++ b/fs/filter/filter.go@@ -45,6 +45,11 @@ var OptionsInfo = fs.Options{{Default: []string{},Help: "Read list of source-file names from file without any processing of lines (use - to read from stdin)",Groups: "Filter",+}, {+ Name: "files_from0",+ Default: []string{},+ Help: "Read list of source-file names from file using NUL as separator (use - to read from stdin)",+ Groups: "Filter",}, {Name: "min_age",Default: fs.DurationOff,@@ -145,6 +150,7 @@ type Options struct {ExcludeFile []string `config:"exclude_if_present"`FilesFrom []string `config:"files_from"`FilesFromRaw []string `config:"files_from_raw"`+ FilesFrom0 []string `config:"files_from0"`MetaRules RulesOpt `config:"metadata"`MinAge fs.Duration `config:"min_age"`MaxAge fs.Duration `config:"max_age"`@@ -226,10 +232,10 @@ func NewFilter(opt *Options) (f *Filter, err error) {for _, rule := range f.Opt.FilesFrom {if !inActive {- return nil, fmt.Errorf("the usage of --files-from overrides all other filters, it should be used alone or with --files-from-raw")+ return nil, fmt.Errorf("the usage of --files-from overrides all other filters, it should be used alone or with --files-from-raw or --files-from0")}f.initAddFile() // init to show --files-from set even if no files within- err := forEachLine(rule, false, func(line string) error {+ err := forEachLine(rule, false, false, func(line string) error {return f.AddFile(line)})if err != nil {@@ -241,10 +247,23 @@ func NewFilter(opt *Options) (f *Filter, err error) {// --files-from-raw can be used with --files-from, hence we do// not need to get the value of f.InActive againif !inActive {- return nil, fmt.Errorf("the usage of --files-from-raw overrides all other filters, it should be used alone or with --files-from")+ return nil, fmt.Errorf("the usage of --files-from-raw overrides all other filters, it should be used alone or with --files-from or --files-from0")+ }+ f.initAddFile() // init to show --files-from set even if no files within+ err := forEachLine(rule, true, false, func(line string) error {+ return f.AddFile(line)+ })+ if err != nil {+ return nil, err+ }+ }++ for _, rule := range f.Opt.FilesFrom0 {+ if !inActive {+ return nil, fmt.Errorf("the usage of --files-from0 overrides all other filters, it should be used alone or with --files-from or --files-from-raw")}f.initAddFile() // init to show --files-from set even if no files within- err := forEachLine(rule, true, func(line string) error {+ err := forEachLine(rule, true, true, func(line string) error {return f.AddFile(line)})if err != nil {diff --git a/fs/filter/filter_test.go b/fs/filter/filter_test.goindex 67c8b9bc1..75f8c77e1 100644--- a/fs/filter/filter_test.go+++ b/fs/filter/filter_test.go@@ -176,6 +176,57 @@ func TestNewFilterWithFilesFromRaw(t *testing.T) {}}+func TestNewFilterForbiddenMixOfFilesFrom0AndFilterRule(t *testing.T) {+ Opt := Opt++ // Set up the input+ Opt.FilterRule = []string{"- filter1", "- filter1b"}+ Opt.FilesFrom0 = []string{testFile(t, "#comment\x00files1\x00files2\x00")}++ rm := func(p string) {+ err := os.Remove(p)+ if err != nil {+ t.Logf("error removing %q: %v", p, err)+ }+ }+ // Reset the input+ defer func() {+ rm(Opt.FilesFrom0[0])+ }()++ _, err := NewFilter(&Opt)+ require.Error(t, err)+ require.Contains(t, err.Error(), "the usage of --files-from0 overrides all other filters")+}++func TestNewFilterWithFilesFrom0(t *testing.T) {+ Opt := Opt++ // Set up the input: NUL-separated, with an embedded newline in one entry+ Opt.FilesFrom0 = []string{testFile(t, "#comment\x00files1\nmore\x00files2\x00")}++ rm := func(p string) {+ err := os.Remove(p)+ if err != nil {+ t.Logf("error removing %q: %v", p, err)+ }+ }+ // Reset the input+ defer func() {+ rm(Opt.FilesFrom0[0])+ }()++ f, err := NewFilter(&Opt)+ require.NoError(t, err)+ assert.Len(t, f.files, 3)+ for _, name := range []string{"#comment", "files1\nmore", "files2"} {+ _, ok := f.files[name]+ if !ok {+ t.Errorf("Didn't find file %q in f.files", name)+ }+ }+}+func TestNewFilterFullExceptFilesFromOpt(t *testing.T) {Opt := Opt@@ -770,7 +821,7 @@ five}()fileName = "-"}- err := forEachLine(fileName, raw, func(s string) error {+ err := forEachLine(fileName, raw, false, func(s string) error {lines = append(lines, s)return nil})@@ -799,6 +850,71 @@ func TestFilterForEachLineStdinWithRaw(t *testing.T) {testFilterForEachLine(t, true, true)}+func testFilterForEachLineNul(t *testing.T, useStdin bool) {+ file := testFile(t, "one\x00two\nthree\x00four\x00five\x00")+ defer func() {+ err := os.Remove(file)+ require.NoError(t, err)+ }()+ lines := []string{}+ fileName := file+ if useStdin {+ in, err := os.Open(file)+ require.NoError(t, err)+ oldStdin := os.Stdin+ os.Stdin = in+ defer func() {+ os.Stdin = oldStdin+ _ = in.Close()+ }()+ fileName = "-"+ }+ err := forEachLine(fileName, true, true, func(s string) error {+ lines = append(lines, s)+ return nil+ })+ require.NoError(t, err)+ assert.Equal(t, []string{"one", "two\nthree", "four", "five"}, lines)+}++func TestFilterForEachLineNul(t *testing.T) {+ testFilterForEachLineNul(t, false)+}++func TestFilterForEachLineNulStdin(t *testing.T) {+ testFilterForEachLineNul(t, true)+}++func TestFilterForEachLineNulNoTrailing(t *testing.T) {+ file := testFile(t, "one\x00two\x00three")+ defer func() {+ err := os.Remove(file)+ require.NoError(t, err)+ }()+ var lines []string+ err := forEachLine(file, true, true, func(s string) error {+ lines = append(lines, s)+ return nil+ })+ require.NoError(t, err)+ assert.Equal(t, []string{"one", "two", "three"}, lines)+}++func TestFilterForEachLineNulConsecutive(t *testing.T) {+ file := testFile(t, "one\x00\x00two\x00")+ defer func() {+ err := os.Remove(file)+ require.NoError(t, err)+ }()+ var lines []string+ err := forEachLine(file, true, true, func(s string) error {+ lines = append(lines, s)+ return nil+ })+ require.NoError(t, err)+ assert.Equal(t, []string{"one", "", "two"}, lines)+}+func TestFilterMatchesFromDocs(t *testing.T) {for _, test := range []struct {glob stringdiff --git a/fs/filter/rules.go b/fs/filter/rules.goindex 085a02aa5..e5ee2626e 100644--- a/fs/filter/rules.go+++ b/fs/filter/rules.go@@ -2,6 +2,7 @@ package filterimport ("bufio"+ "bytes""fmt""os""regexp"@@ -114,10 +115,26 @@ func (rs *rules) includeMany(remotes []string) bool {return true}+// scanNul is a split function for a Scanner that returns each NUL-terminated+// sequence of bytes. It correctly handles the final segment even if it+// lacks a trailing NUL.+func scanNul(data []byte, atEOF bool) (advance int, token []byte, err error) {+ if atEOF && len(data) == 0 {+ return 0, nil, nil+ }+ if i := bytes.IndexByte(data, '\x00'); i >= 0 {+ return i + 1, data[:i], nil+ }+ if atEOF {+ return len(data), data, nil+ }+ return 0, nil, nil+}+// forEachLine calls fn on every line in the file pointed to by path//// It ignores empty lines and lines starting with '#' or ';' if raw is false-func forEachLine(path string, raw bool, fn func(string) error) (err error) {+func forEachLine(path string, raw bool, useNulDelimiter bool, fn func(string) error) (err error) {var scanner *bufio.Scannerif path == "-" {scanner = bufio.NewScanner(os.Stdin)@@ -129,6 +146,11 @@ func forEachLine(path string, raw bool, fn func(string) error) (err error) {scanner = bufio.NewScanner(in)defer fs.CheckClose(in, &err)}++ if useNulDelimiter {+ scanner.Split(scanNul)+ }+for scanner.Scan() {line := scanner.Text()if !raw {@@ -199,7 +221,7 @@ func parseRules(opt *RulesOpt, add addFn, clear clearFn) (err error) {addImplicitExclude = true}for _, rule := range opt.IncludeFrom {- err := forEachLine(rule, false, func(line string) error {+ err := forEachLine(rule, false, false, func(line string) error {return add(true, line)})if err != nil {@@ -215,7 +237,7 @@ func parseRules(opt *RulesOpt, add addFn, clear clearFn) (err error) {foundExcludeRule = true}for _, rule := range opt.ExcludeFrom {- err := forEachLine(rule, false, func(line string) error {+ err := forEachLine(rule, false, false, func(line string) error {return add(false, line)})if err != nil {@@ -235,7 +257,7 @@ func parseRules(opt *RulesOpt, add addFn, clear clearFn) (err error) {}}for _, rule := range opt.FilterFrom {- err := forEachLine(rule, false, func(rule string) error {+ err := forEachLine(rule, false, false, func(rule string) error {return addRule(rule, add, clear)})if err != nil {