"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
filter: convert options to new style
Nick Craig-Woodc954c397d9c7
diff --git a/fs/filter/filter.go b/fs/filter/filter.goindex 277623600..408165080 100644--- a/fs/filter/filter.go+++ b/fs/filter/filter.go@@ -19,24 +19,137 @@ import (// This is accessed through GetConfig and AddConfigvar globalConfig = mustNewFilter(nil)+// OptionsInfo describes the Options in use+var OptionsInfo = fs.Options{{+ Name: "delete_excluded",+ Default: false,+ Help: "Delete files on dest excluded from sync",+ Groups: "Filter",+}, {+ Name: "exclude_if_present",+ Default: []string{},+ Help: "Exclude directories if filename is present",+ Groups: "Filter",+}, {+ Name: "files_from",+ Default: []string{},+ Help: "Read list of source-file names from file (use - to read from stdin)",+ Groups: "Filter",+}, {+ Name: "files_from_raw",+ Default: []string{},+ Help: "Read list of source-file names from file without any processing of lines (use - to read from stdin)",+ Groups: "Filter",+}, {+ Name: "min_age",+ Default: fs.DurationOff,+ Help: "Only transfer files older than this in s or suffix ms|s|m|h|d|w|M|y",+ Groups: "Filter",+}, {+ Name: "max_age",+ Default: fs.DurationOff,+ Help: "Only transfer files younger than this in s or suffix ms|s|m|h|d|w|M|y",+ Groups: "Filter",+}, {+ Name: "min_size",+ Default: fs.SizeSuffix(-1),+ Help: "Only transfer files bigger than this in KiB or suffix B|K|M|G|T|P",+ Groups: "Filter",+}, {+ Name: "max_size",+ Default: fs.SizeSuffix(-1),+ Help: "Only transfer files smaller than this in KiB or suffix B|K|M|G|T|P",+ Groups: "Filter",+}, {+ Name: "ignore_case",+ Default: false,+ Help: "Ignore case in filters (case insensitive)",+ Groups: "Filter",+}, {+ Name: "filter",+ Default: []string{},+ ShortOpt: "f",+ Help: "Add a file filtering rule",+ Groups: "Filter",+}, {+ Name: "filter_from",+ Default: []string{},+ Help: "Read file filtering patterns from a file (use - to read from stdin)",+ Groups: "Filter",+}, {+ Name: "exclude",+ Default: []string{},+ Help: "Exclude files matching pattern",+ Groups: "Filter",+}, {+ Name: "exclude_from",+ Default: []string{},+ Help: "Read file exclude patterns from file (use - to read from stdin)",+ Groups: "Filter",+}, {+ Name: "include",+ Default: []string{},+ Help: "Include files matching pattern",+ Groups: "Filter",+}, {+ Name: "include_from",+ Default: []string{},+ Help: "Read file include patterns from file (use - to read from stdin)",+ Groups: "Filter",+}, {+ Name: "metadata_filter",+ Default: []string{},+ Help: "Add a metadata filtering rule",+ Groups: "Filter,Metadata",+}, {+ Name: "metadata_filter_from",+ Default: []string{},+ Help: "Read metadata filtering patterns from a file (use - to read from stdin)",+ Groups: "Filter,Metadata",+}, {+ Name: "metadata_exclude",+ Default: []string{},+ Help: "Exclude metadatas matching pattern",+ Groups: "Filter,Metadata",+}, {+ Name: "metadata_exclude_from",+ Default: []string{},+ Help: "Read metadata exclude patterns from file (use - to read from stdin)",+ Groups: "Filter,Metadata",+}, {+ Name: "metadata_include",+ Default: []string{},+ Help: "Include metadatas matching pattern",+ Groups: "Filter,Metadata",+}, {+ Name: "metadata_include_from",+ Default: []string{},+ Help: "Read metadata include patterns from file (use - to read from stdin)",+ Groups: "Filter,Metadata",+}}+// Options configures the filtertype Options struct {- DeleteExcluded bool- RulesOpt // embedded so we don't change the JSON API- ExcludeFile []string- FilesFrom []string- FilesFromRaw []string- MetaRules RulesOpt- MinAge fs.Duration- MaxAge fs.Duration- MinSize fs.SizeSuffix- MaxSize fs.SizeSuffix- IgnoreCase bool+ DeleteExcluded bool `config:"delete_excluded"`+ RulesOpt // embedded so we don't change the JSON API+ ExcludeFile []string `config:"exclude_if_present"`+ FilesFrom []string `config:"files_from"`+ FilesFromRaw []string `config:"files_from_raw"`+ MetaRules RulesOpt `config:"metadata"`+ MinAge fs.Duration `config:"min_age"`+ MaxAge fs.Duration `config:"max_age"`+ MinSize fs.SizeSuffix `config:"min_size"`+ MaxSize fs.SizeSuffix `config:"max_size"`+ IgnoreCase bool `config:"ignore_case"`+}++func init() {+ fs.RegisterGlobalOptions(fs.OptionsInfo{Name: "filter", Opt: &Opt, Options: OptionsInfo, Reload: Reload})}-// DefaultOpt is the default config for the filter-var DefaultOpt = Options{- MinAge: fs.DurationOff,+// Opt is the default config for the filter+var Opt = Options{+ MinAge: fs.DurationOff, // These have to be set here as the options are parsed once before the defaults are setMaxAge: fs.DurationOff,MinSize: fs.SizeSuffix(-1),MaxSize: fs.SizeSuffix(-1),@@ -66,7 +179,7 @@ func NewFilter(opt *Options) (f *Filter, err error) {if opt != nil {f.Opt = *opt} else {- f.Opt = DefaultOpt+ f.Opt = Opt}// Filter flags@@ -77,7 +190,7 @@ func NewFilter(opt *Options) (f *Filter, err error) {if f.Opt.MaxAge.IsSet() {f.ModTimeFrom = time.Now().Add(-time.Duration(f.Opt.MaxAge))if !f.ModTimeTo.IsZero() && f.ModTimeTo.Before(f.ModTimeFrom) {- log.Fatal("filter: --min-age can't be larger than --max-age")+ log.Fatalf("filter: --min-age %q can't be larger than --max-age %q", opt.MinAge, opt.MaxAge)}fs.Debugf(nil, "--max-age %v to %v", f.Opt.MaxAge, f.ModTimeFrom)}@@ -568,3 +681,14 @@ func SetUseFilter(ctx context.Context, useFilter bool) context.Context {*pVal = useFilterreturn context.WithValue(ctx, useFlagContextKey, pVal)}++// Reload the filters from the flags+func Reload(ctx context.Context) (err error) {+ fi := GetConfig(ctx)+ newFilter, err := NewFilter(&Opt)+ if err != nil {+ return err+ }+ *fi = *newFilter+ return nil+}diff --git a/fs/filter/filter_test.go b/fs/filter/filter_test.goindex 7821cec74..0b5f934da 100644--- a/fs/filter/filter_test.go+++ b/fs/filter/filter_test.go@@ -43,7 +43,7 @@ func testFile(t *testing.T, contents string) string {}func TestNewFilterForbiddenMixOfFilesFromAndFilterRule(t *testing.T) {- Opt := DefaultOpt+ Opt := Opt// Set up the inputOpt.FilterRule = []string{"- filter1", "- filter1b"}@@ -66,7 +66,7 @@ func TestNewFilterForbiddenMixOfFilesFromAndFilterRule(t *testing.T) {}func TestNewFilterForbiddenMixOfFilesFromRawAndFilterRule(t *testing.T) {- Opt := DefaultOpt+ Opt := Opt// Set up the inputOpt.FilterRule = []string{"- filter1", "- filter1b"}@@ -89,7 +89,7 @@ func TestNewFilterForbiddenMixOfFilesFromRawAndFilterRule(t *testing.T) {}func TestNewFilterWithFilesFromAlone(t *testing.T) {- Opt := DefaultOpt+ Opt := Opt// Set up the inputOpt.FilesFrom = []string{testFile(t, "#comment\nfiles1\nfiles2\n")}@@ -117,7 +117,7 @@ func TestNewFilterWithFilesFromAlone(t *testing.T) {}func TestNewFilterWithFilesFromRaw(t *testing.T) {- Opt := DefaultOpt+ Opt := Opt// Set up the inputOpt.FilesFromRaw = []string{testFile(t, "#comment\nfiles1\nfiles2\n")}@@ -145,7 +145,7 @@ func TestNewFilterWithFilesFromRaw(t *testing.T) {}func TestNewFilterFullExceptFilesFromOpt(t *testing.T) {- Opt := DefaultOpt+ Opt := Optmins := fs.SizeSuffix(100 * 1024)maxs := fs.SizeSuffix(1000 * 1024)diff --git a/fs/filter/filterflags/filterflags.go b/fs/filter/filterflags/filterflags.goindex e5202bfc1..eaead5e79 100644--- a/fs/filter/filterflags/filterflags.go+++ b/fs/filter/filterflags/filterflags.go@@ -2,62 +2,12 @@package filterflagsimport (- "context"- "fmt"-"github.com/rclone/rclone/fs/config/flags""github.com/rclone/rclone/fs/filter"- "github.com/rclone/rclone/fs/rc""github.com/spf13/pflag")-// Options set by command line flags-var (- Opt = filter.DefaultOpt-)--// Reload the filters from the flags-func Reload(ctx context.Context) (err error) {- fi := filter.GetConfig(ctx)- newFilter, err := filter.NewFilter(&Opt)- if err != nil {- return err- }- *fi = *newFilter- return nil-}--// AddRuleFlags add a set of rules flags with prefix-func AddRuleFlags(flagSet *pflag.FlagSet, Opt *filter.RulesOpt, what, prefix string) {- shortFilter := ""- if prefix == "" {- shortFilter = "f"- }- group := "Filter"- if prefix == "metadata-" {- group += ",Metadata"- }- flags.StringArrayVarP(flagSet, &Opt.FilterRule, prefix+"filter", shortFilter, nil, fmt.Sprintf("Add a %s filtering rule", what), group)- flags.StringArrayVarP(flagSet, &Opt.FilterFrom, prefix+"filter-from", "", nil, fmt.Sprintf("Read %s filtering patterns from a file (use - to read from stdin)", what), group)- flags.StringArrayVarP(flagSet, &Opt.ExcludeRule, prefix+"exclude", "", nil, fmt.Sprintf("Exclude %ss matching pattern", what), group)- flags.StringArrayVarP(flagSet, &Opt.ExcludeFrom, prefix+"exclude-from", "", nil, fmt.Sprintf("Read %s exclude patterns from file (use - to read from stdin)", what), group)- flags.StringArrayVarP(flagSet, &Opt.IncludeRule, prefix+"include", "", nil, fmt.Sprintf("Include %ss matching pattern", what), group)- flags.StringArrayVarP(flagSet, &Opt.IncludeFrom, prefix+"include-from", "", nil, fmt.Sprintf("Read %s include patterns from file (use - to read from stdin)", what), group)-}-// AddFlags adds the non filing system specific flags to the commandfunc AddFlags(flagSet *pflag.FlagSet) {- rc.AddOptionReload("filter", &Opt, Reload)- flags.BoolVarP(flagSet, &Opt.DeleteExcluded, "delete-excluded", "", false, "Delete files on dest excluded from sync", "Filter")- AddRuleFlags(flagSet, &Opt.RulesOpt, "file", "")- AddRuleFlags(flagSet, &Opt.MetaRules, "metadata", "metadata-")- flags.StringArrayVarP(flagSet, &Opt.ExcludeFile, "exclude-if-present", "", nil, "Exclude directories if filename is present", "Filter")- flags.StringArrayVarP(flagSet, &Opt.FilesFrom, "files-from", "", nil, "Read list of source-file names from file (use - to read from stdin)", "Filter")- flags.StringArrayVarP(flagSet, &Opt.FilesFromRaw, "files-from-raw", "", nil, "Read list of source-file names from file without any processing of lines (use - to read from stdin)", "Filter")- flags.FVarP(flagSet, &Opt.MinAge, "min-age", "", "Only transfer files older than this in s or suffix ms|s|m|h|d|w|M|y", "Filter")- flags.FVarP(flagSet, &Opt.MaxAge, "max-age", "", "Only transfer files younger than this in s or suffix ms|s|m|h|d|w|M|y", "Filter")- flags.FVarP(flagSet, &Opt.MinSize, "min-size", "", "Only transfer files bigger than this in KiB or suffix B|K|M|G|T|P", "Filter")- flags.FVarP(flagSet, &Opt.MaxSize, "max-size", "", "Only transfer files smaller than this in KiB or suffix B|K|M|G|T|P", "Filter")- flags.BoolVarP(flagSet, &Opt.IgnoreCase, "ignore-case", "", false, "Ignore case in filters (case insensitive)", "Filter")- //cvsExclude = BoolP("cvs-exclude", "C", false, "Exclude files in the same way CVS does")+ flags.AddFlagsFromOptions(flagSet, "", filter.OptionsInfo)}diff --git a/fs/filter/rules.go b/fs/filter/rules.goindex 683cc7037..dce5851a6 100644--- a/fs/filter/rules.go+++ b/fs/filter/rules.go@@ -12,12 +12,12 @@ import (// RulesOpt is configuration for a rule settype RulesOpt struct {- FilterRule []string- FilterFrom []string- ExcludeRule []string- ExcludeFrom []string- IncludeRule []string- IncludeFrom []string+ FilterRule []string `config:"filter"`+ FilterFrom []string `config:"filter_from"`+ ExcludeRule []string `config:"exclude"`+ ExcludeFrom []string `config:"exclude_from"`+ IncludeRule []string `config:"include"`+ IncludeFrom []string `config:"include_from"`}// rule is one filter rule