Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

sync: add pattern-based transfer ordering

Shurong Caobb7c081ec520
diff --git a/docs/content/docs.md b/docs/content/docs.md
index 4ee271b4c..14d563c32 100644
--- a/docs/content/docs.md
+++ b/docs/content/docs.md
@@ -2378,8 +2378,10 @@ describes what aspect is being measured:
- `size` - order by the size of the files
- `name` - order by the full path of the files
- `modtime` - order by the modification date of the files
+- `pattern` - order by a list of path glob patterns
-This can have a modifier appended with a comma:
+The `size`, `name` and `modtime` comparisons can have a modifier appended with
+a comma:
- `ascending` or `asc` - order so that the smallest (or oldest) is processed first
- `descending` or `desc` - order so that the largest (or newest) is processed first
@@ -2398,11 +2400,18 @@ processed continuously.
If no modifier is supplied then the order is `ascending`.
+For `pattern`, append one or more comma-separated patterns. Patterns use the
+[filter pattern syntax](/filtering/#patterns). Files matching the first pattern
+are processed first, followed by files matching the second pattern, and so on.
+Files which do not match a pattern are processed last. Within each group, files
+are ordered by full path. Patterns cannot contain a comma.
+
For example
- `--order-by size,desc` - send the largest files first
- `--order-by modtime,ascending` - send the oldest files first
- `--order-by name` - send the files with alphabetically by path first
+- `--order-by 'pattern,*.tar.gz,*.md5'` - send archives before checksum files
If the `--order-by` flag is not supplied or it is supplied with an
empty string then the default ordering will be used which is as
diff --git a/fs/sync/pipe.go b/fs/sync/pipe.go
index 124a07780..42d3e9234 100644
--- a/fs/sync/pipe.go
+++ b/fs/sync/pipe.go
@@ -10,6 +10,7 @@ import (
"github.com/aalpar/deheap"
"github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/filter"
"github.com/rclone/rclone/fs/fserrors"
)
@@ -187,8 +188,8 @@ func newLess(orderBy string) (less lessFn, fraction int, err error) {
if orderBy == "" {
return nil, fraction, nil
}
- parts := strings.Split(strings.ToLower(orderBy), ",")
- switch parts[0] {
+ parts := strings.Split(orderBy, ",")
+ switch strings.ToLower(parts[0]) {
case "name":
less = func(a, b fs.ObjectPair) bool {
return a.Src.Remote() < b.Src.Remote()
@@ -202,12 +203,44 @@ func newLess(orderBy string) (less lessFn, fraction int, err error) {
ctx := context.Background()
return a.Src.ModTime(ctx).Before(b.Src.ModTime(ctx))
}
+ case "pattern":
+ if len(parts) < 2 {
+ return nil, fraction, fmt.Errorf("pattern requires at least one glob")
+ }
+ matches := make([]func(string) bool, 0, len(parts)-1)
+ for _, glob := range parts[1:] {
+ if glob == "" {
+ return nil, fraction, fmt.Errorf("empty pattern glob")
+ }
+ re, err := filter.GlobPathToRegexp(glob, false)
+ if err != nil {
+ return nil, fraction, fmt.Errorf("bad pattern glob %q: %w", glob, err)
+ }
+ matches = append(matches, re.MatchString)
+ }
+ rank := func(remote string) int {
+ for i, match := range matches {
+ if match(remote) {
+ return i
+ }
+ }
+ return len(matches)
+ }
+ less = func(a, b fs.ObjectPair) bool {
+ aRemote, bRemote := a.Src.Remote(), b.Src.Remote()
+ aRank, bRank := rank(aRemote), rank(bRemote)
+ if aRank != bRank {
+ return aRank < bRank
+ }
+ return aRemote < bRemote
+ }
+ return less, fraction, nil
default:
return nil, fraction, fmt.Errorf("unknown --order-by comparison %q", parts[0])
}
descending := false
if len(parts) > 1 {
- switch parts[1] {
+ switch strings.ToLower(parts[1]) {
case "ascending", "asc":
case "descending", "desc":
descending = true
diff --git a/fs/sync/pipe_test.go b/fs/sync/pipe_test.go
index 94916aa82..c08ca926c 100644
--- a/fs/sync/pipe_test.go
+++ b/fs/sync/pipe_test.go
@@ -250,6 +250,41 @@ func TestNewLess(t *testing.T) {
assert.Contains(t, err.Error(), "unknown --order-by sort direction")
})
+ t.Run("patternNeedsGlob", func(t *testing.T) {
+ _, _, err := newLess("pattern")
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "pattern requires at least one glob")
+ })
+
+ t.Run("patternRejectsEmptyGlob", func(t *testing.T) {
+ _, _, err := newLess("pattern,")
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "empty pattern glob")
+ })
+
+ t.Run("patternRejectsBadGlob", func(t *testing.T) {
+ _, _, err := newLess("pattern,[")
+ require.Error(t, err)
+ assert.Contains(t, err.Error(), "bad pattern glob")
+ })
+
+ t.Run("patternOrder", func(t *testing.T) {
+ less, fraction, err := newLess("pattern,*.tar.gz,*.md5")
+ require.NoError(t, err)
+ assert.Equal(t, -1, fraction)
+
+ archiveA := fs.ObjectPair{Src: mockobject.New("a/archive.tar.gz")}
+ archiveB := fs.ObjectPair{Src: mockobject.New("b/archive.tar.gz")}
+ checksum := fs.ObjectPair{Src: mockobject.New("a/archive.md5")}
+ unmatched := fs.ObjectPair{Src: mockobject.New("a/README")}
+
+ assert.True(t, less(archiveA, checksum))
+ assert.True(t, less(checksum, unmatched))
+ assert.False(t, less(unmatched, archiveA))
+ assert.True(t, less(archiveA, archiveB))
+ assert.False(t, less(archiveB, archiveA))
+ })
+
var (
obj1 = mockobject.New("b").WithContent([]byte("1"), mockobject.SeekModeNone)
obj2 = mockobject.New("a").WithContent([]byte("22"), mockobject.SeekModeNone)