"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
filter: speed up large --include-from lists of directories - fixes #9014
kaii9af364f5468a2
diff --git a/fs/filter/filter.go b/fs/filter/filter.goindex 4c5e1c5b4..504be6251 100644--- a/fs/filter/filter.go+++ b/fs/filter/filter.go@@ -724,13 +724,15 @@ func CopyConfig(dstCtx, srcCtx context.Context) context.Context {return context.WithValue(dstCtx, configContextKey, c)}-// AddConfig returns a mutable config structure based on a shallow-// copy of that found in ctx and returns a new context with that added-// to it.+// AddConfig returns a mutable copy of the config in ctx with independent+// rule sets and a new context containing it. Other fields are shallow-copied.func AddConfig(ctx context.Context) (context.Context, *Filter) {c := GetConfig(ctx)cCopy := new(Filter)*cCopy = *c+ cCopy.fileRules = c.fileRules.clone()+ cCopy.dirRules = c.dirRules.clone()+ cCopy.metaRules = c.metaRules.clone()newCtx := context.WithValue(ctx, configContextKey, cCopy)return newCtx, cCopy}diff --git a/fs/filter/filter_test.go b/fs/filter/filter_test.goindex 75f8c77e1..64d20bfbe 100644--- a/fs/filter/filter_test.go+++ b/fs/filter/filter_test.go@@ -1043,6 +1043,39 @@ func TestNewFilterUsesDirectoryFilters(t *testing.T) {}}+func TestAddConfigRulesIndependent(t *testing.T) {+ for _, ignoreCase := range []bool{false, true} {+ t.Run(fmt.Sprintf("ignoreCase=%t", ignoreCase), func(t *testing.T) {+ opt := Opt+ opt.IgnoreCase = ignoreCase+ for i := range minIndexedRules {+ opt.FilterRule = append(opt.FilterRule, fmt.Sprintf("- /old_%d/**", i))+ opt.MetaRules.FilterRule = append(opt.MetaRules.FilterRule, fmt.Sprintf("- /old_%d/**", i))+ }+ original, err := NewFilter(&opt)+ require.NoError(t, err)+ ctx := ReplaceConfig(context.Background(), original)+ _, cloned := AddConfig(ctx)+ require.NoError(t, cloned.Add(false, "/new/**"))+ require.NoError(t, cloned.metaRules.Add(false, "/new/**"))+ assert.False(t, cloned.IncludeRemote("new/file"))+ assert.True(t, original.IncludeRemote("new/file"))+ assert.False(t, cloned.metaRules.include("new/file"))+ assert.True(t, original.metaRules.include("new/file"))+ include := original.IncludeDirectory(ctx, nil)+ got, err := include("new")+ require.NoError(t, err)+ assert.True(t, got)+ // The original must not inherit the clone's deduplication state.+ require.NoError(t, original.Add(false, "/new/**"))+ assert.False(t, original.IncludeRemote("new/file"))+ cloned.Clear()+ assert.True(t, cloned.IncludeRemote("old_0/file"))+ assert.False(t, original.IncludeRemote("old_0/file"))+ })+ }+}+func TestGetConfig(t *testing.T) {ctx := context.Background()diff --git a/fs/filter/rules.go b/fs/filter/rules.goindex e5ee2626e..46776248a 100644--- a/fs/filter/rules.go+++ b/fs/filter/rules.go@@ -4,10 +4,14 @@ import ("bufio""bytes""fmt"+ "maps""os""regexp"+ "regexp/syntax""slices""strings"+ "unicode"+ "unicode/utf8""github.com/rclone/rclone/fs")@@ -44,10 +48,16 @@ func (r *rule) String() string {// rules is a slice of rulestype rules struct {- rules []rule- existing map[string]struct{}+ rules []rule+ existing map[string]struct{}+ byDirectory map[string][]int+ foldedDirectory map[string][]int+ otherRules []int}+// Small rule sets are faster to scan without maintaining an index.+const minIndexedRules = 64+type addFn func(Include bool, glob string) error// add adds a rule if it doesn't exist already@@ -65,6 +75,52 @@ func (rs *rules) add(Include bool, re *regexp.Regexp) {}rs.rules = append(rs.rules, newRule)rs.existing[newRuleString] = struct{}{}+ if rs.byDirectory != nil {+ rs.index(len(rs.rules) - 1)+ } else if len(rs.rules) == minIndexedRules {+ rs.byDirectory = make(map[string][]int)+ for i := range rs.rules {+ rs.index(i)+ }+ }+}++// foldDirectory uses the same Unicode simple-fold equivalence as regexp.+func foldDirectory(directory string) string {+ return strings.Map(func(r rune) rune {+ if r < utf8.RuneSelf {+ if r >= 'a' && r <= 'z' {+ return r - 'a' + 'A'+ }+ return r+ }+ minimum := r+ for folded := unicode.SimpleFold(r); folded != r; folded = unicode.SimpleFold(folded) {+ minimum = min(minimum, folded)+ }+ return minimum+ }, directory)+}++// index groups anchored rules by their first literal directory.+func (rs *rules) index(i int) {+ parsed, err := syntax.Parse(rs.rules[i].Regexp.String(), syntax.Perl)+ if err == nil && parsed.Op == syntax.OpConcat && len(parsed.Sub) >= 2 &&+ parsed.Sub[0].Op == syntax.OpBeginText && parsed.Sub[1].Op == syntax.OpLiteral {+ if directory, _, ok := strings.Cut(string(parsed.Sub[1].Rune), "/"); ok {+ if parsed.Sub[1].Flags&syntax.FoldCase != 0 {+ if rs.foldedDirectory == nil {+ rs.foldedDirectory = make(map[string][]int)+ }+ directory = foldDirectory(directory)+ rs.foldedDirectory[directory] = append(rs.foldedDirectory[directory], i)+ } else {+ rs.byDirectory[directory] = append(rs.byDirectory[directory], i)+ }+ return+ }+ }+ rs.otherRules = append(rs.otherRules, i)}// Add adds a filter rule with include or exclude status indicated@@ -83,6 +139,28 @@ type clearFn func()func (rs *rules) clear() {rs.rules = nilrs.existing = nil+ rs.byDirectory = nil+ rs.foldedDirectory = nil+ rs.otherRules = nil+}++// clone copies the mutable rule storage, including directory buckets.+func (rs *rules) clone() rules {+ copy := *rs+ copy.rules = slices.Clone(rs.rules)+ copy.existing = maps.Clone(rs.existing)+ copy.otherRules = slices.Clone(rs.otherRules)+ copy.byDirectory = cloneDirectoryIndex(rs.byDirectory)+ copy.foldedDirectory = cloneDirectoryIndex(rs.foldedDirectory)+ return copy+}++func cloneDirectoryIndex(index map[string][]int) map[string][]int {+ copy := maps.Clone(index)+ for directory, bucket := range copy {+ copy[directory] = slices.Clone(bucket)+ }+ return copy}// len returns the number of rules@@ -92,6 +170,40 @@ func (rs *rules) len() int {// include returns whether this remote passes the filter rules.func (rs *rules) include(remote string) bool {+ if len(rs.byDirectory) != 0 || len(rs.foldedDirectory) != 0 {+ directory, _, _ := strings.Cut(remote, "/")+ // Regexp decodes invalid UTF-8 bytes as U+FFFD, which can match a+ // literal U+FFFD. Use the linear scan when byte lookup could miss it.+ if !utf8.ValidString(directory) {+ for _, rule := range rs.rules {+ if rule.Match(remote) {+ return rule.Include+ }+ }+ return true+ }+ candidates := [3][]int{rs.byDirectory[directory], nil, rs.otherRules}+ if len(rs.foldedDirectory) != 0 {+ candidates[1] = rs.foldedDirectory[foldDirectory(directory)]+ }+ for {+ // Merge the sorted buckets to preserve first-match precedence.+ first := -1+ for j, bucket := range candidates {+ if len(bucket) != 0 && (first == -1 || bucket[0] < candidates[first][0]) {+ first = j+ }+ }+ if first == -1 {+ return true+ }+ i := candidates[first][0]+ candidates[first] = candidates[first][1:]+ if rs.rules[i].Match(remote) {+ return rs.rules[i].Include+ }+ }+ }for _, rule := range rs.rules {if rule.Match(remote) {return rule.Includediff --git a/fs/filter/rules_test.go b/fs/filter/rules_test.gonew file mode 100644index 000000000..dd14b630a--- /dev/null+++ b/fs/filter/rules_test.go@@ -0,0 +1,205 @@+package filter++import (+ "context"+ "fmt"+ "regexp"+ "strings"+ "testing"++ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++func TestRulesDirectoryIndexMatchesLinear(t *testing.T) {+ patterns := []string{+ `^alpha/.*$`, `^beta/private/.*$`, `^beta/.*$`, `.*secret.*`,+ `^gamma/(?:one|two)/.*$`, `^a\.b/.*$`, `^日本語/.*$`,+ `(?i)^ALPHA/.*$`, `^alpha/.*$|beta/.*$`, `(?m)^alpha/.*$`,+ `^(alpha/.*)$`, `^replacement�/.*$`, `^alpha/file$`, `.*`,+ `(?i)^kelvin/.*$`, `(?i)^ſea/.*$`, `(?i)^Σ/.*$`,+ `(?i)^İ/.*$`, `(?i)^ı/.*$`, `(?i)^ß/.*$`,+ `(?i)^replacement�/.*$`,+ }+ remotes := []string{+ "", "alpha", "alpha/", "alpha/file", "alpha/secret", "ALPHA/file",+ "beta/private/file", "beta/public/file", "gamma/one/file", "gamma/three/file",+ "a.b/file", "axb/file", "日本語/file", "unknown/file", "prefix\nalpha/file",+ "replacement�/file", "replacement\xff/file", "replacement\xff\xff/file",+ "kelvin/file", "Kelvin/file", "Kelvin/file", "SEA/file", "sea/file", "ſea/file",+ "Σ/file", "σ/file", "ς/file", "İ/file", "ı/file", "I/file", "i/file",+ "ß/file", "ẞ/file", "ss/file",+ "REPLACEMENT\xff/file",+ }+ // Move generic rules through the list to check first-match precedence.+ for _, count := range []int{len(patterns), 63, 64, 65, 128} {+ for offset := range patterns {+ t.Run(fmt.Sprintf("rules=%d/offset=%d", count, offset), func(t *testing.T) {+ var rs rules+ for i := range count - len(patterns) {+ rs.add(i%2 == 0, regexp.MustCompile(fmt.Sprintf(`^padding_%d/.*$`, i)))+ }+ for i := range patterns {+ rs.add(i%2 == 0, regexp.MustCompile(patterns[(i+offset)%len(patterns)]))+ }+ require.Len(t, rs.rules, count)+ for _, remote := range remotes {+ want := true+ for _, r := range rs.rules {+ if r.Match(remote) {+ want = r.Include+ break+ }+ }+ assert.Equal(t, want, rs.include(remote), "remote %q", remote)+ }+ })+ }+ }+}++func TestRulesDirectoryIndexIgnoreCase(t *testing.T) {+ for _, test := range []struct {+ pattern string+ matches []string+ misses []string+ }{+ {"kelvin", []string{"kelvin", "KELVIN", "Kelvin"}, []string{"other"}},+ {"ſea", []string{"ſea", "sea", "SEA"}, []string{"other"}},+ {"Σ", []string{"Σ", "σ", "ς"}, []string{"other"}},+ {"İ", []string{"İ"}, []string{"I", "i", "ı"}},+ {"ı", []string{"ı"}, []string{"I", "i", "İ"}},+ {"ß", []string{"ß", "ẞ"}, []string{"ss", "SS"}},+ {"日本語", []string{"日本語"}, []string{"日本"}},+ } {+ t.Run(test.pattern, func(t *testing.T) {+ var rs rules+ for i := range minIndexedRules - 1 {+ rs.add(false, regexp.MustCompile(fmt.Sprintf(`^padding_%d/.*$`, i)))+ }+ rs.add(false, regexp.MustCompile(`(?i)^`+regexp.QuoteMeta(test.pattern)+`/.*$`))+ assert.Empty(t, rs.otherRules, "anchored ignore-case rules should use the index")+ for _, directory := range test.matches {+ assert.False(t, rs.include(directory+"/file"), "directory %q", directory)+ }+ for _, directory := range test.misses {+ assert.True(t, rs.include(directory+"/file"), "directory %q", directory)+ }+ })+ }+}++func TestFilterDirectoryIndexIgnoreCase(t *testing.T) {+ opt := Opt+ opt.IgnoreCase = true+ for i := range minIndexedRules {+ opt.IncludeRule = append(opt.IncludeRule, fmt.Sprintf("/dataset_%06d/**", i))+ }+ f, err := NewFilter(&opt)+ require.NoError(t, err)+ // Only the implicit exclude-all rule needs the general scan.+ assert.Len(t, f.fileRules.otherRules, 1)+ assert.Len(t, f.dirRules.otherRules, 1)+ assert.True(t, f.IncludeRemote("DATASET_000063/file"))+ include := f.IncludeDirectory(context.Background(), nil)+ got, err := include("DATASET_000063")+ require.NoError(t, err)+ assert.True(t, got)+ got, err = include("UNSELECTED")+ require.NoError(t, err)+ assert.False(t, got)+}++func TestRulesDirectoryIndexAddAndClear(t *testing.T) {+ var rs rules+ for i := range 128 {+ require.NoError(t, rs.Add(false, fmt.Sprintf("/padding_%d/**", i)))+ }+ require.NoError(t, rs.Add(false, "/target/private/**"))+ require.NoError(t, rs.Add(true, "/target/**"))+ count := rs.len()+ require.NoError(t, rs.Add(true, "/target/**"))+ assert.Equal(t, count, rs.len())+ require.NoError(t, rs.Add(false, "**"))+ assert.False(t, rs.include("target/private/file"))+ assert.True(t, rs.include("target/public/file"))+ assert.False(t, rs.include("other/file"))+ rs.clear()+ assert.Zero(t, rs.len())+ assert.True(t, rs.include("other/file"))+ require.NoError(t, rs.Add(false, "/target/**"))+ assert.False(t, rs.include("target/public/file"))+ assert.True(t, rs.include("other/file"))+ // Rebuild an index after a reset, with the opposite rule for the same path.+ for i := range 128 {+ require.NoError(t, rs.Add(false, fmt.Sprintf("/padding_%d/**", i)))+ }+ assert.False(t, rs.include("target/public/file"))+}++func BenchmarkFilterLiteralDirectories(b *testing.B) {+ benchmarkFilterLiteralDirectories(b, false)+}++func BenchmarkFilterLiteralDirectoriesIgnoreCase(b *testing.B) {+ benchmarkFilterLiteralDirectories(b, true)+}++func benchmarkFilterLiteralDirectories(b *testing.B, ignoreCase bool) {+ for _, count := range []int{4, 32, 1000, 10000, 100000} {+ b.Run(fmt.Sprintf("rules=%d", count), func(b *testing.B) {+ opt := Opt+ opt.IgnoreCase = ignoreCase+ opt.IncludeRule = make([]string, count)+ for i := range opt.IncludeRule {+ opt.IncludeRule[i] = fmt.Sprintf("/dataset_%06d/**", i)+ }+ f, err := NewFilter(&opt)+ if err != nil {+ b.Fatal(err)+ }+ include := f.IncludeDirectory(context.Background(), nil)+ for _, remote := range []string{"dataset_000000", fmt.Sprintf("dataset_%06d", count-1), "unselected_directory"} {+ if ignoreCase {+ remote = strings.ToUpper(remote)+ }+ b.Run(remote, func(b *testing.B) {+ b.ReportAllocs()+ for b.Loop() {+ _, err := include(remote)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+ })+ }+}++func BenchmarkFilterLiteralDirectoriesSetup(b *testing.B) {+ benchmarkFilterLiteralDirectoriesSetup(b, false)+}++func BenchmarkFilterLiteralDirectoriesSetupIgnoreCase(b *testing.B) {+ benchmarkFilterLiteralDirectoriesSetup(b, true)+}++func benchmarkFilterLiteralDirectoriesSetup(b *testing.B, ignoreCase bool) {+ for _, count := range []int{32, 1000, 10000} {+ b.Run(fmt.Sprintf("rules=%d", count), func(b *testing.B) {+ opt := Opt+ opt.IgnoreCase = ignoreCase+ opt.IncludeRule = make([]string, count)+ for i := range opt.IncludeRule {+ opt.IncludeRule[i] = fmt.Sprintf("/dataset_%06d/**", i)+ }+ b.ReportAllocs()+ for b.Loop() {+ if _, err := NewFilter(&opt); err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}