Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

filter: convert options to new style

Nick Craig-Woodc954c397d9c7
diff --git a/fs/filter/filter.go b/fs/filter/filter.go
index 277623600..408165080 100644
--- a/fs/filter/filter.go
+++ b/fs/filter/filter.go
@@ -19,24 +19,137 @@ import (
// This is accessed through GetConfig and AddConfig
var globalConfig = mustNewFilter(nil)
+// OptionsInfo describes the Options in use
+var OptionsInfo = fs.Options{{
+ Name: "delete_excluded",
+ Default: false,
+ Help: "Delete files on dest excluded from sync",
+ Groups: "Filter",
+}, {
+ Name: "exclude_if_present",
+ Default: []string{},
+ Help: "Exclude directories if filename is present",
+ Groups: "Filter",
+}, {
+ Name: "files_from",
+ Default: []string{},
+ Help: "Read list of source-file names from file (use - to read from stdin)",
+ Groups: "Filter",
+}, {
+ Name: "files_from_raw",
+ Default: []string{},
+ Help: "Read list of source-file names from file without any processing of lines (use - to read from stdin)",
+ Groups: "Filter",
+}, {
+ Name: "min_age",
+ Default: fs.DurationOff,
+ Help: "Only transfer files older than this in s or suffix ms|s|m|h|d|w|M|y",
+ Groups: "Filter",
+}, {
+ Name: "max_age",
+ Default: fs.DurationOff,
+ Help: "Only transfer files younger than this in s or suffix ms|s|m|h|d|w|M|y",
+ Groups: "Filter",
+}, {
+ Name: "min_size",
+ Default: fs.SizeSuffix(-1),
+ Help: "Only transfer files bigger than this in KiB or suffix B|K|M|G|T|P",
+ Groups: "Filter",
+}, {
+ Name: "max_size",
+ Default: fs.SizeSuffix(-1),
+ Help: "Only transfer files smaller than this in KiB or suffix B|K|M|G|T|P",
+ Groups: "Filter",
+}, {
+ Name: "ignore_case",
+ Default: false,
+ Help: "Ignore case in filters (case insensitive)",
+ Groups: "Filter",
+}, {
+ Name: "filter",
+ Default: []string{},
+ ShortOpt: "f",
+ Help: "Add a file filtering rule",
+ Groups: "Filter",
+}, {
+ Name: "filter_from",
+ Default: []string{},
+ Help: "Read file filtering patterns from a file (use - to read from stdin)",
+ Groups: "Filter",
+}, {
+ Name: "exclude",
+ Default: []string{},
+ Help: "Exclude files matching pattern",
+ Groups: "Filter",
+}, {
+ Name: "exclude_from",
+ Default: []string{},
+ Help: "Read file exclude patterns from file (use - to read from stdin)",
+ Groups: "Filter",
+}, {
+ Name: "include",
+ Default: []string{},
+ Help: "Include files matching pattern",
+ Groups: "Filter",
+}, {
+ Name: "include_from",
+ Default: []string{},
+ Help: "Read file include patterns from file (use - to read from stdin)",
+ Groups: "Filter",
+}, {
+ Name: "metadata_filter",
+ Default: []string{},
+ Help: "Add a metadata filtering rule",
+ Groups: "Filter,Metadata",
+}, {
+ Name: "metadata_filter_from",
+ Default: []string{},
+ Help: "Read metadata filtering patterns from a file (use - to read from stdin)",
+ Groups: "Filter,Metadata",
+}, {
+ Name: "metadata_exclude",
+ Default: []string{},
+ Help: "Exclude metadatas matching pattern",
+ Groups: "Filter,Metadata",
+}, {
+ Name: "metadata_exclude_from",
+ Default: []string{},
+ Help: "Read metadata exclude patterns from file (use - to read from stdin)",
+ Groups: "Filter,Metadata",
+}, {
+ Name: "metadata_include",
+ Default: []string{},
+ Help: "Include metadatas matching pattern",
+ Groups: "Filter,Metadata",
+}, {
+ Name: "metadata_include_from",
+ Default: []string{},
+ Help: "Read metadata include patterns from file (use - to read from stdin)",
+ Groups: "Filter,Metadata",
+}}
+
// Options configures the filter
type Options struct {
- DeleteExcluded bool
- RulesOpt // embedded so we don't change the JSON API
- ExcludeFile []string
- FilesFrom []string
- FilesFromRaw []string
- MetaRules RulesOpt
- MinAge fs.Duration
- MaxAge fs.Duration
- MinSize fs.SizeSuffix
- MaxSize fs.SizeSuffix
- IgnoreCase bool
+ DeleteExcluded bool `config:"delete_excluded"`
+ RulesOpt // embedded so we don't change the JSON API
+ ExcludeFile []string `config:"exclude_if_present"`
+ FilesFrom []string `config:"files_from"`
+ FilesFromRaw []string `config:"files_from_raw"`
+ MetaRules RulesOpt `config:"metadata"`
+ MinAge fs.Duration `config:"min_age"`
+ MaxAge fs.Duration `config:"max_age"`
+ MinSize fs.SizeSuffix `config:"min_size"`
+ MaxSize fs.SizeSuffix `config:"max_size"`
+ IgnoreCase bool `config:"ignore_case"`
+}
+
+func init() {
+ fs.RegisterGlobalOptions(fs.OptionsInfo{Name: "filter", Opt: &Opt, Options: OptionsInfo, Reload: Reload})
}
-// DefaultOpt is the default config for the filter
-var DefaultOpt = Options{
- MinAge: fs.DurationOff,
+// Opt is the default config for the filter
+var Opt = Options{
+ MinAge: fs.DurationOff, // These have to be set here as the options are parsed once before the defaults are set
MaxAge: fs.DurationOff,
MinSize: fs.SizeSuffix(-1),
MaxSize: fs.SizeSuffix(-1),
@@ -66,7 +179,7 @@ func NewFilter(opt *Options) (f *Filter, err error) {
if opt != nil {
f.Opt = *opt
} else {
- f.Opt = DefaultOpt
+ f.Opt = Opt
}
// Filter flags
@@ -77,7 +190,7 @@ func NewFilter(opt *Options) (f *Filter, err error) {
if f.Opt.MaxAge.IsSet() {
f.ModTimeFrom = time.Now().Add(-time.Duration(f.Opt.MaxAge))
if !f.ModTimeTo.IsZero() && f.ModTimeTo.Before(f.ModTimeFrom) {
- log.Fatal("filter: --min-age can't be larger than --max-age")
+ log.Fatalf("filter: --min-age %q can't be larger than --max-age %q", opt.MinAge, opt.MaxAge)
}
fs.Debugf(nil, "--max-age %v to %v", f.Opt.MaxAge, f.ModTimeFrom)
}
@@ -568,3 +681,14 @@ func SetUseFilter(ctx context.Context, useFilter bool) context.Context {
*pVal = useFilter
return context.WithValue(ctx, useFlagContextKey, pVal)
}
+
+// Reload the filters from the flags
+func Reload(ctx context.Context) (err error) {
+ fi := GetConfig(ctx)
+ newFilter, err := NewFilter(&Opt)
+ if err != nil {
+ return err
+ }
+ *fi = *newFilter
+ return nil
+}
diff --git a/fs/filter/filter_test.go b/fs/filter/filter_test.go
index 7821cec74..0b5f934da 100644
--- a/fs/filter/filter_test.go
+++ b/fs/filter/filter_test.go
@@ -43,7 +43,7 @@ func testFile(t *testing.T, contents string) string {
}
func TestNewFilterForbiddenMixOfFilesFromAndFilterRule(t *testing.T) {
- Opt := DefaultOpt
+ Opt := Opt
// Set up the input
Opt.FilterRule = []string{"- filter1", "- filter1b"}
@@ -66,7 +66,7 @@ func TestNewFilterForbiddenMixOfFilesFromAndFilterRule(t *testing.T) {
}
func TestNewFilterForbiddenMixOfFilesFromRawAndFilterRule(t *testing.T) {
- Opt := DefaultOpt
+ Opt := Opt
// Set up the input
Opt.FilterRule = []string{"- filter1", "- filter1b"}
@@ -89,7 +89,7 @@ func TestNewFilterForbiddenMixOfFilesFromRawAndFilterRule(t *testing.T) {
}
func TestNewFilterWithFilesFromAlone(t *testing.T) {
- Opt := DefaultOpt
+ Opt := Opt
// Set up the input
Opt.FilesFrom = []string{testFile(t, "#comment\nfiles1\nfiles2\n")}
@@ -117,7 +117,7 @@ func TestNewFilterWithFilesFromAlone(t *testing.T) {
}
func TestNewFilterWithFilesFromRaw(t *testing.T) {
- Opt := DefaultOpt
+ Opt := Opt
// Set up the input
Opt.FilesFromRaw = []string{testFile(t, "#comment\nfiles1\nfiles2\n")}
@@ -145,7 +145,7 @@ func TestNewFilterWithFilesFromRaw(t *testing.T) {
}
func TestNewFilterFullExceptFilesFromOpt(t *testing.T) {
- Opt := DefaultOpt
+ Opt := Opt
mins := fs.SizeSuffix(100 * 1024)
maxs := fs.SizeSuffix(1000 * 1024)
diff --git a/fs/filter/filterflags/filterflags.go b/fs/filter/filterflags/filterflags.go
index e5202bfc1..eaead5e79 100644
--- a/fs/filter/filterflags/filterflags.go
+++ b/fs/filter/filterflags/filterflags.go
@@ -2,62 +2,12 @@
package filterflags
import (
- "context"
- "fmt"
-
"github.com/rclone/rclone/fs/config/flags"
"github.com/rclone/rclone/fs/filter"
- "github.com/rclone/rclone/fs/rc"
"github.com/spf13/pflag"
)
-// Options set by command line flags
-var (
- Opt = filter.DefaultOpt
-)
-
-// Reload the filters from the flags
-func Reload(ctx context.Context) (err error) {
- fi := filter.GetConfig(ctx)
- newFilter, err := filter.NewFilter(&Opt)
- if err != nil {
- return err
- }
- *fi = *newFilter
- return nil
-}
-
-// AddRuleFlags add a set of rules flags with prefix
-func AddRuleFlags(flagSet *pflag.FlagSet, Opt *filter.RulesOpt, what, prefix string) {
- shortFilter := ""
- if prefix == "" {
- shortFilter = "f"
- }
- group := "Filter"
- if prefix == "metadata-" {
- group += ",Metadata"
- }
- flags.StringArrayVarP(flagSet, &Opt.FilterRule, prefix+"filter", shortFilter, nil, fmt.Sprintf("Add a %s filtering rule", what), group)
- flags.StringArrayVarP(flagSet, &Opt.FilterFrom, prefix+"filter-from", "", nil, fmt.Sprintf("Read %s filtering patterns from a file (use - to read from stdin)", what), group)
- flags.StringArrayVarP(flagSet, &Opt.ExcludeRule, prefix+"exclude", "", nil, fmt.Sprintf("Exclude %ss matching pattern", what), group)
- flags.StringArrayVarP(flagSet, &Opt.ExcludeFrom, prefix+"exclude-from", "", nil, fmt.Sprintf("Read %s exclude patterns from file (use - to read from stdin)", what), group)
- flags.StringArrayVarP(flagSet, &Opt.IncludeRule, prefix+"include", "", nil, fmt.Sprintf("Include %ss matching pattern", what), group)
- flags.StringArrayVarP(flagSet, &Opt.IncludeFrom, prefix+"include-from", "", nil, fmt.Sprintf("Read %s include patterns from file (use - to read from stdin)", what), group)
-}
-
// AddFlags adds the non filing system specific flags to the command
func AddFlags(flagSet *pflag.FlagSet) {
- rc.AddOptionReload("filter", &Opt, Reload)
- flags.BoolVarP(flagSet, &Opt.DeleteExcluded, "delete-excluded", "", false, "Delete files on dest excluded from sync", "Filter")
- AddRuleFlags(flagSet, &Opt.RulesOpt, "file", "")
- AddRuleFlags(flagSet, &Opt.MetaRules, "metadata", "metadata-")
- flags.StringArrayVarP(flagSet, &Opt.ExcludeFile, "exclude-if-present", "", nil, "Exclude directories if filename is present", "Filter")
- flags.StringArrayVarP(flagSet, &Opt.FilesFrom, "files-from", "", nil, "Read list of source-file names from file (use - to read from stdin)", "Filter")
- flags.StringArrayVarP(flagSet, &Opt.FilesFromRaw, "files-from-raw", "", nil, "Read list of source-file names from file without any processing of lines (use - to read from stdin)", "Filter")
- flags.FVarP(flagSet, &Opt.MinAge, "min-age", "", "Only transfer files older than this in s or suffix ms|s|m|h|d|w|M|y", "Filter")
- flags.FVarP(flagSet, &Opt.MaxAge, "max-age", "", "Only transfer files younger than this in s or suffix ms|s|m|h|d|w|M|y", "Filter")
- flags.FVarP(flagSet, &Opt.MinSize, "min-size", "", "Only transfer files bigger than this in KiB or suffix B|K|M|G|T|P", "Filter")
- flags.FVarP(flagSet, &Opt.MaxSize, "max-size", "", "Only transfer files smaller than this in KiB or suffix B|K|M|G|T|P", "Filter")
- flags.BoolVarP(flagSet, &Opt.IgnoreCase, "ignore-case", "", false, "Ignore case in filters (case insensitive)", "Filter")
- //cvsExclude = BoolP("cvs-exclude", "C", false, "Exclude files in the same way CVS does")
+ flags.AddFlagsFromOptions(flagSet, "", filter.OptionsInfo)
}
diff --git a/fs/filter/rules.go b/fs/filter/rules.go
index 683cc7037..dce5851a6 100644
--- a/fs/filter/rules.go
+++ b/fs/filter/rules.go
@@ -12,12 +12,12 @@ import (
// RulesOpt is configuration for a rule set
type RulesOpt struct {
- FilterRule []string
- FilterFrom []string
- ExcludeRule []string
- ExcludeFrom []string
- IncludeRule []string
- IncludeFrom []string
+ FilterRule []string `config:"filter"`
+ FilterFrom []string `config:"filter_from"`
+ ExcludeRule []string `config:"exclude"`
+ ExcludeFrom []string `config:"exclude_from"`
+ IncludeRule []string `config:"include"`
+ IncludeFrom []string `config:"include_from"`
}
// rule is one filter rule