Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

filter: speed up large --include-from lists of directories - fixes #9014

kaii9af364f5468a2
diff --git a/fs/filter/filter.go b/fs/filter/filter.go
index 4c5e1c5b4..504be6251 100644
--- a/fs/filter/filter.go
+++ b/fs/filter/filter.go
@@ -724,13 +724,15 @@ func CopyConfig(dstCtx, srcCtx context.Context) context.Context {
return context.WithValue(dstCtx, configContextKey, c)
}
-// AddConfig returns a mutable config structure based on a shallow
-// copy of that found in ctx and returns a new context with that added
-// to it.
+// AddConfig returns a mutable copy of the config in ctx with independent
+// rule sets and a new context containing it. Other fields are shallow-copied.
func AddConfig(ctx context.Context) (context.Context, *Filter) {
c := GetConfig(ctx)
cCopy := new(Filter)
*cCopy = *c
+ cCopy.fileRules = c.fileRules.clone()
+ cCopy.dirRules = c.dirRules.clone()
+ cCopy.metaRules = c.metaRules.clone()
newCtx := context.WithValue(ctx, configContextKey, cCopy)
return newCtx, cCopy
}
diff --git a/fs/filter/filter_test.go b/fs/filter/filter_test.go
index 75f8c77e1..64d20bfbe 100644
--- a/fs/filter/filter_test.go
+++ b/fs/filter/filter_test.go
@@ -1043,6 +1043,39 @@ func TestNewFilterUsesDirectoryFilters(t *testing.T) {
}
}
+func TestAddConfigRulesIndependent(t *testing.T) {
+ for _, ignoreCase := range []bool{false, true} {
+ t.Run(fmt.Sprintf("ignoreCase=%t", ignoreCase), func(t *testing.T) {
+ opt := Opt
+ opt.IgnoreCase = ignoreCase
+ for i := range minIndexedRules {
+ opt.FilterRule = append(opt.FilterRule, fmt.Sprintf("- /old_%d/**", i))
+ opt.MetaRules.FilterRule = append(opt.MetaRules.FilterRule, fmt.Sprintf("- /old_%d/**", i))
+ }
+ original, err := NewFilter(&opt)
+ require.NoError(t, err)
+ ctx := ReplaceConfig(context.Background(), original)
+ _, cloned := AddConfig(ctx)
+ require.NoError(t, cloned.Add(false, "/new/**"))
+ require.NoError(t, cloned.metaRules.Add(false, "/new/**"))
+ assert.False(t, cloned.IncludeRemote("new/file"))
+ assert.True(t, original.IncludeRemote("new/file"))
+ assert.False(t, cloned.metaRules.include("new/file"))
+ assert.True(t, original.metaRules.include("new/file"))
+ include := original.IncludeDirectory(ctx, nil)
+ got, err := include("new")
+ require.NoError(t, err)
+ assert.True(t, got)
+ // The original must not inherit the clone's deduplication state.
+ require.NoError(t, original.Add(false, "/new/**"))
+ assert.False(t, original.IncludeRemote("new/file"))
+ cloned.Clear()
+ assert.True(t, cloned.IncludeRemote("old_0/file"))
+ assert.False(t, original.IncludeRemote("old_0/file"))
+ })
+ }
+}
+
func TestGetConfig(t *testing.T) {
ctx := context.Background()
diff --git a/fs/filter/rules.go b/fs/filter/rules.go
index e5ee2626e..46776248a 100644
--- a/fs/filter/rules.go
+++ b/fs/filter/rules.go
@@ -4,10 +4,14 @@ import (
"bufio"
"bytes"
"fmt"
+ "maps"
"os"
"regexp"
+ "regexp/syntax"
"slices"
"strings"
+ "unicode"
+ "unicode/utf8"
"github.com/rclone/rclone/fs"
)
@@ -44,10 +48,16 @@ func (r *rule) String() string {
// rules is a slice of rules
type rules struct {
- rules []rule
- existing map[string]struct{}
+ rules []rule
+ existing map[string]struct{}
+ byDirectory map[string][]int
+ foldedDirectory map[string][]int
+ otherRules []int
}
+// Small rule sets are faster to scan without maintaining an index.
+const minIndexedRules = 64
+
type addFn func(Include bool, glob string) error
// add adds a rule if it doesn't exist already
@@ -65,6 +75,52 @@ func (rs *rules) add(Include bool, re *regexp.Regexp) {
}
rs.rules = append(rs.rules, newRule)
rs.existing[newRuleString] = struct{}{}
+ if rs.byDirectory != nil {
+ rs.index(len(rs.rules) - 1)
+ } else if len(rs.rules) == minIndexedRules {
+ rs.byDirectory = make(map[string][]int)
+ for i := range rs.rules {
+ rs.index(i)
+ }
+ }
+}
+
+// foldDirectory uses the same Unicode simple-fold equivalence as regexp.
+func foldDirectory(directory string) string {
+ return strings.Map(func(r rune) rune {
+ if r < utf8.RuneSelf {
+ if r >= 'a' && r <= 'z' {
+ return r - 'a' + 'A'
+ }
+ return r
+ }
+ minimum := r
+ for folded := unicode.SimpleFold(r); folded != r; folded = unicode.SimpleFold(folded) {
+ minimum = min(minimum, folded)
+ }
+ return minimum
+ }, directory)
+}
+
+// index groups anchored rules by their first literal directory.
+func (rs *rules) index(i int) {
+ parsed, err := syntax.Parse(rs.rules[i].Regexp.String(), syntax.Perl)
+ if err == nil && parsed.Op == syntax.OpConcat && len(parsed.Sub) >= 2 &&
+ parsed.Sub[0].Op == syntax.OpBeginText && parsed.Sub[1].Op == syntax.OpLiteral {
+ if directory, _, ok := strings.Cut(string(parsed.Sub[1].Rune), "/"); ok {
+ if parsed.Sub[1].Flags&syntax.FoldCase != 0 {
+ if rs.foldedDirectory == nil {
+ rs.foldedDirectory = make(map[string][]int)
+ }
+ directory = foldDirectory(directory)
+ rs.foldedDirectory[directory] = append(rs.foldedDirectory[directory], i)
+ } else {
+ rs.byDirectory[directory] = append(rs.byDirectory[directory], i)
+ }
+ return
+ }
+ }
+ rs.otherRules = append(rs.otherRules, i)
}
// Add adds a filter rule with include or exclude status indicated
@@ -83,6 +139,28 @@ type clearFn func()
func (rs *rules) clear() {
rs.rules = nil
rs.existing = nil
+ rs.byDirectory = nil
+ rs.foldedDirectory = nil
+ rs.otherRules = nil
+}
+
+// clone copies the mutable rule storage, including directory buckets.
+func (rs *rules) clone() rules {
+ copy := *rs
+ copy.rules = slices.Clone(rs.rules)
+ copy.existing = maps.Clone(rs.existing)
+ copy.otherRules = slices.Clone(rs.otherRules)
+ copy.byDirectory = cloneDirectoryIndex(rs.byDirectory)
+ copy.foldedDirectory = cloneDirectoryIndex(rs.foldedDirectory)
+ return copy
+}
+
+func cloneDirectoryIndex(index map[string][]int) map[string][]int {
+ copy := maps.Clone(index)
+ for directory, bucket := range copy {
+ copy[directory] = slices.Clone(bucket)
+ }
+ return copy
}
// len returns the number of rules
@@ -92,6 +170,40 @@ func (rs *rules) len() int {
// include returns whether this remote passes the filter rules.
func (rs *rules) include(remote string) bool {
+ if len(rs.byDirectory) != 0 || len(rs.foldedDirectory) != 0 {
+ directory, _, _ := strings.Cut(remote, "/")
+ // Regexp decodes invalid UTF-8 bytes as U+FFFD, which can match a
+ // literal U+FFFD. Use the linear scan when byte lookup could miss it.
+ if !utf8.ValidString(directory) {
+ for _, rule := range rs.rules {
+ if rule.Match(remote) {
+ return rule.Include
+ }
+ }
+ return true
+ }
+ candidates := [3][]int{rs.byDirectory[directory], nil, rs.otherRules}
+ if len(rs.foldedDirectory) != 0 {
+ candidates[1] = rs.foldedDirectory[foldDirectory(directory)]
+ }
+ for {
+ // Merge the sorted buckets to preserve first-match precedence.
+ first := -1
+ for j, bucket := range candidates {
+ if len(bucket) != 0 && (first == -1 || bucket[0] < candidates[first][0]) {
+ first = j
+ }
+ }
+ if first == -1 {
+ return true
+ }
+ i := candidates[first][0]
+ candidates[first] = candidates[first][1:]
+ if rs.rules[i].Match(remote) {
+ return rs.rules[i].Include
+ }
+ }
+ }
for _, rule := range rs.rules {
if rule.Match(remote) {
return rule.Include
diff --git a/fs/filter/rules_test.go b/fs/filter/rules_test.go
new file mode 100644
index 000000000..dd14b630a
--- /dev/null
+++ b/fs/filter/rules_test.go
@@ -0,0 +1,205 @@
+package filter
+
+import (
+ "context"
+ "fmt"
+ "regexp"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestRulesDirectoryIndexMatchesLinear(t *testing.T) {
+ patterns := []string{
+ `^alpha/.*$`, `^beta/private/.*$`, `^beta/.*$`, `.*secret.*`,
+ `^gamma/(?:one|two)/.*$`, `^a\.b/.*$`, `^日本語/.*$`,
+ `(?i)^ALPHA/.*$`, `^alpha/.*$|beta/.*$`, `(?m)^alpha/.*$`,
+ `^(alpha/.*)$`, `^replacement�/.*$`, `^alpha/file$`, `.*`,
+ `(?i)^kelvin/.*$`, `(?i)^ſea/.*$`, `(?i)^Σ/.*$`,
+ `(?i)^İ/.*$`, `(?i)^ı/.*$`, `(?i)^ß/.*$`,
+ `(?i)^replacement�/.*$`,
+ }
+ remotes := []string{
+ "", "alpha", "alpha/", "alpha/file", "alpha/secret", "ALPHA/file",
+ "beta/private/file", "beta/public/file", "gamma/one/file", "gamma/three/file",
+ "a.b/file", "axb/file", "日本語/file", "unknown/file", "prefix\nalpha/file",
+ "replacement�/file", "replacement\xff/file", "replacement\xff\xff/file",
+ "kelvin/file", "Kelvin/file", "Kelvin/file", "SEA/file", "sea/file", "ſea/file",
+ "Σ/file", "σ/file", "ς/file", "İ/file", "ı/file", "I/file", "i/file",
+ "ß/file", "ẞ/file", "ss/file",
+ "REPLACEMENT\xff/file",
+ }
+ // Move generic rules through the list to check first-match precedence.
+ for _, count := range []int{len(patterns), 63, 64, 65, 128} {
+ for offset := range patterns {
+ t.Run(fmt.Sprintf("rules=%d/offset=%d", count, offset), func(t *testing.T) {
+ var rs rules
+ for i := range count - len(patterns) {
+ rs.add(i%2 == 0, regexp.MustCompile(fmt.Sprintf(`^padding_%d/.*$`, i)))
+ }
+ for i := range patterns {
+ rs.add(i%2 == 0, regexp.MustCompile(patterns[(i+offset)%len(patterns)]))
+ }
+ require.Len(t, rs.rules, count)
+ for _, remote := range remotes {
+ want := true
+ for _, r := range rs.rules {
+ if r.Match(remote) {
+ want = r.Include
+ break
+ }
+ }
+ assert.Equal(t, want, rs.include(remote), "remote %q", remote)
+ }
+ })
+ }
+ }
+}
+
+func TestRulesDirectoryIndexIgnoreCase(t *testing.T) {
+ for _, test := range []struct {
+ pattern string
+ matches []string
+ misses []string
+ }{
+ {"kelvin", []string{"kelvin", "KELVIN", "Kelvin"}, []string{"other"}},
+ {"ſea", []string{"ſea", "sea", "SEA"}, []string{"other"}},
+ {"Σ", []string{"Σ", "σ", "ς"}, []string{"other"}},
+ {"İ", []string{"İ"}, []string{"I", "i", "ı"}},
+ {"ı", []string{"ı"}, []string{"I", "i", "İ"}},
+ {"ß", []string{"ß", "ẞ"}, []string{"ss", "SS"}},
+ {"日本語", []string{"日本語"}, []string{"日本"}},
+ } {
+ t.Run(test.pattern, func(t *testing.T) {
+ var rs rules
+ for i := range minIndexedRules - 1 {
+ rs.add(false, regexp.MustCompile(fmt.Sprintf(`^padding_%d/.*$`, i)))
+ }
+ rs.add(false, regexp.MustCompile(`(?i)^`+regexp.QuoteMeta(test.pattern)+`/.*$`))
+ assert.Empty(t, rs.otherRules, "anchored ignore-case rules should use the index")
+ for _, directory := range test.matches {
+ assert.False(t, rs.include(directory+"/file"), "directory %q", directory)
+ }
+ for _, directory := range test.misses {
+ assert.True(t, rs.include(directory+"/file"), "directory %q", directory)
+ }
+ })
+ }
+}
+
+func TestFilterDirectoryIndexIgnoreCase(t *testing.T) {
+ opt := Opt
+ opt.IgnoreCase = true
+ for i := range minIndexedRules {
+ opt.IncludeRule = append(opt.IncludeRule, fmt.Sprintf("/dataset_%06d/**", i))
+ }
+ f, err := NewFilter(&opt)
+ require.NoError(t, err)
+ // Only the implicit exclude-all rule needs the general scan.
+ assert.Len(t, f.fileRules.otherRules, 1)
+ assert.Len(t, f.dirRules.otherRules, 1)
+ assert.True(t, f.IncludeRemote("DATASET_000063/file"))
+ include := f.IncludeDirectory(context.Background(), nil)
+ got, err := include("DATASET_000063")
+ require.NoError(t, err)
+ assert.True(t, got)
+ got, err = include("UNSELECTED")
+ require.NoError(t, err)
+ assert.False(t, got)
+}
+
+func TestRulesDirectoryIndexAddAndClear(t *testing.T) {
+ var rs rules
+ for i := range 128 {
+ require.NoError(t, rs.Add(false, fmt.Sprintf("/padding_%d/**", i)))
+ }
+ require.NoError(t, rs.Add(false, "/target/private/**"))
+ require.NoError(t, rs.Add(true, "/target/**"))
+ count := rs.len()
+ require.NoError(t, rs.Add(true, "/target/**"))
+ assert.Equal(t, count, rs.len())
+ require.NoError(t, rs.Add(false, "**"))
+ assert.False(t, rs.include("target/private/file"))
+ assert.True(t, rs.include("target/public/file"))
+ assert.False(t, rs.include("other/file"))
+ rs.clear()
+ assert.Zero(t, rs.len())
+ assert.True(t, rs.include("other/file"))
+ require.NoError(t, rs.Add(false, "/target/**"))
+ assert.False(t, rs.include("target/public/file"))
+ assert.True(t, rs.include("other/file"))
+ // Rebuild an index after a reset, with the opposite rule for the same path.
+ for i := range 128 {
+ require.NoError(t, rs.Add(false, fmt.Sprintf("/padding_%d/**", i)))
+ }
+ assert.False(t, rs.include("target/public/file"))
+}
+
+func BenchmarkFilterLiteralDirectories(b *testing.B) {
+ benchmarkFilterLiteralDirectories(b, false)
+}
+
+func BenchmarkFilterLiteralDirectoriesIgnoreCase(b *testing.B) {
+ benchmarkFilterLiteralDirectories(b, true)
+}
+
+func benchmarkFilterLiteralDirectories(b *testing.B, ignoreCase bool) {
+ for _, count := range []int{4, 32, 1000, 10000, 100000} {
+ b.Run(fmt.Sprintf("rules=%d", count), func(b *testing.B) {
+ opt := Opt
+ opt.IgnoreCase = ignoreCase
+ opt.IncludeRule = make([]string, count)
+ for i := range opt.IncludeRule {
+ opt.IncludeRule[i] = fmt.Sprintf("/dataset_%06d/**", i)
+ }
+ f, err := NewFilter(&opt)
+ if err != nil {
+ b.Fatal(err)
+ }
+ include := f.IncludeDirectory(context.Background(), nil)
+ for _, remote := range []string{"dataset_000000", fmt.Sprintf("dataset_%06d", count-1), "unselected_directory"} {
+ if ignoreCase {
+ remote = strings.ToUpper(remote)
+ }
+ b.Run(remote, func(b *testing.B) {
+ b.ReportAllocs()
+ for b.Loop() {
+ _, err := include(remote)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+ })
+ }
+}
+
+func BenchmarkFilterLiteralDirectoriesSetup(b *testing.B) {
+ benchmarkFilterLiteralDirectoriesSetup(b, false)
+}
+
+func BenchmarkFilterLiteralDirectoriesSetupIgnoreCase(b *testing.B) {
+ benchmarkFilterLiteralDirectoriesSetup(b, true)
+}
+
+func benchmarkFilterLiteralDirectoriesSetup(b *testing.B, ignoreCase bool) {
+ for _, count := range []int{32, 1000, 10000} {
+ b.Run(fmt.Sprintf("rules=%d", count), func(b *testing.B) {
+ opt := Opt
+ opt.IgnoreCase = ignoreCase
+ opt.IncludeRule = make([]string, count)
+ for i := range opt.IncludeRule {
+ opt.IncludeRule[i] = fmt.Sprintf("/dataset_%06d/**", i)
+ }
+ b.ReportAllocs()
+ for b.Loop() {
+ if _, err := NewFilter(&opt); err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}