"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
sync: add pattern-based transfer ordering
Shurong Caobb7c081ec520
diff --git a/docs/content/docs.md b/docs/content/docs.mdindex 4ee271b4c..14d563c32 100644--- a/docs/content/docs.md+++ b/docs/content/docs.md@@ -2378,8 +2378,10 @@ describes what aspect is being measured:- `size` - order by the size of the files- `name` - order by the full path of the files- `modtime` - order by the modification date of the files+- `pattern` - order by a list of path glob patterns-This can have a modifier appended with a comma:+The `size`, `name` and `modtime` comparisons can have a modifier appended with+a comma:- `ascending` or `asc` - order so that the smallest (or oldest) is processed first- `descending` or `desc` - order so that the largest (or newest) is processed first@@ -2398,11 +2400,18 @@ processed continuously.If no modifier is supplied then the order is `ascending`.+For `pattern`, append one or more comma-separated patterns. Patterns use the+[filter pattern syntax](/filtering/#patterns). Files matching the first pattern+are processed first, followed by files matching the second pattern, and so on.+Files which do not match a pattern are processed last. Within each group, files+are ordered by full path. Patterns cannot contain a comma.+For example- `--order-by size,desc` - send the largest files first- `--order-by modtime,ascending` - send the oldest files first- `--order-by name` - send the files with alphabetically by path first+- `--order-by 'pattern,*.tar.gz,*.md5'` - send archives before checksum filesIf the `--order-by` flag is not supplied or it is supplied with anempty string then the default ordering will be used which is asdiff --git a/fs/sync/pipe.go b/fs/sync/pipe.goindex 124a07780..42d3e9234 100644--- a/fs/sync/pipe.go+++ b/fs/sync/pipe.go@@ -10,6 +10,7 @@ import ("github.com/aalpar/deheap""github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/filter""github.com/rclone/rclone/fs/fserrors")@@ -187,8 +188,8 @@ func newLess(orderBy string) (less lessFn, fraction int, err error) {if orderBy == "" {return nil, fraction, nil}- parts := strings.Split(strings.ToLower(orderBy), ",")- switch parts[0] {+ parts := strings.Split(orderBy, ",")+ switch strings.ToLower(parts[0]) {case "name":less = func(a, b fs.ObjectPair) bool {return a.Src.Remote() < b.Src.Remote()@@ -202,12 +203,44 @@ func newLess(orderBy string) (less lessFn, fraction int, err error) {ctx := context.Background()return a.Src.ModTime(ctx).Before(b.Src.ModTime(ctx))}+ case "pattern":+ if len(parts) < 2 {+ return nil, fraction, fmt.Errorf("pattern requires at least one glob")+ }+ matches := make([]func(string) bool, 0, len(parts)-1)+ for _, glob := range parts[1:] {+ if glob == "" {+ return nil, fraction, fmt.Errorf("empty pattern glob")+ }+ re, err := filter.GlobPathToRegexp(glob, false)+ if err != nil {+ return nil, fraction, fmt.Errorf("bad pattern glob %q: %w", glob, err)+ }+ matches = append(matches, re.MatchString)+ }+ rank := func(remote string) int {+ for i, match := range matches {+ if match(remote) {+ return i+ }+ }+ return len(matches)+ }+ less = func(a, b fs.ObjectPair) bool {+ aRemote, bRemote := a.Src.Remote(), b.Src.Remote()+ aRank, bRank := rank(aRemote), rank(bRemote)+ if aRank != bRank {+ return aRank < bRank+ }+ return aRemote < bRemote+ }+ return less, fraction, nildefault:return nil, fraction, fmt.Errorf("unknown --order-by comparison %q", parts[0])}descending := falseif len(parts) > 1 {- switch parts[1] {+ switch strings.ToLower(parts[1]) {case "ascending", "asc":case "descending", "desc":descending = truediff --git a/fs/sync/pipe_test.go b/fs/sync/pipe_test.goindex 94916aa82..c08ca926c 100644--- a/fs/sync/pipe_test.go+++ b/fs/sync/pipe_test.go@@ -250,6 +250,41 @@ func TestNewLess(t *testing.T) {assert.Contains(t, err.Error(), "unknown --order-by sort direction")})+ t.Run("patternNeedsGlob", func(t *testing.T) {+ _, _, err := newLess("pattern")+ require.Error(t, err)+ assert.Contains(t, err.Error(), "pattern requires at least one glob")+ })++ t.Run("patternRejectsEmptyGlob", func(t *testing.T) {+ _, _, err := newLess("pattern,")+ require.Error(t, err)+ assert.Contains(t, err.Error(), "empty pattern glob")+ })++ t.Run("patternRejectsBadGlob", func(t *testing.T) {+ _, _, err := newLess("pattern,[")+ require.Error(t, err)+ assert.Contains(t, err.Error(), "bad pattern glob")+ })++ t.Run("patternOrder", func(t *testing.T) {+ less, fraction, err := newLess("pattern,*.tar.gz,*.md5")+ require.NoError(t, err)+ assert.Equal(t, -1, fraction)++ archiveA := fs.ObjectPair{Src: mockobject.New("a/archive.tar.gz")}+ archiveB := fs.ObjectPair{Src: mockobject.New("b/archive.tar.gz")}+ checksum := fs.ObjectPair{Src: mockobject.New("a/archive.md5")}+ unmatched := fs.ObjectPair{Src: mockobject.New("a/README")}++ assert.True(t, less(archiveA, checksum))+ assert.True(t, less(checksum, unmatched))+ assert.False(t, less(unmatched, archiveA))+ assert.True(t, less(archiveA, archiveB))+ assert.False(t, less(archiveB, archiveA))+ })+var (obj1 = mockobject.New("b").WithContent([]byte("1"), mockobject.SeekModeNone)obj2 = mockobject.New("a").WithContent([]byte("22"), mockobject.SeekModeNone)