"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
march: split src and dst
nielash41a407dcc9ec
diff --git a/fs/march/march.go b/fs/march/march.goindex 578aea512..def481ac9 100644--- a/fs/march/march.go+++ b/fs/march/march.go@@ -60,9 +60,9 @@ type Marcher interface {// Note: this will flag filter-aware backends on the source sidefunc (m *March) init(ctx context.Context) {ci := fs.GetConfig(ctx)- m.srcListDir = m.makeListDir(ctx, m.Fsrc, m.SrcIncludeAll)+ m.srcListDir = m.makeListDir(ctx, m.Fsrc, m.SrcIncludeAll, m.srcKey)if !m.NoTraverse {- m.dstListDir = m.makeListDir(ctx, m.Fdst, m.DstIncludeAll)+ m.dstListDir = m.makeListDir(ctx, m.Fdst, m.DstIncludeAll, m.dstKey)}// Now create the matching transform// ..normalise the UTF8 first@@ -80,8 +80,8 @@ func (m *March) init(ctx context.Context) {}}-// key turns a directory entry into a sort key using the defined transforms.-func (m *March) key(entry fs.DirEntry) string {+// srcKey turns a directory entry into a sort key using the defined transforms.+func (m *March) srcKey(entry fs.DirEntry) string {if entry == nil {return ""}@@ -99,17 +99,22 @@ func (m *March) key(entry fs.DirEntry) string {return name}+// dstKey turns a directory entry into a sort key using the defined transforms.+func (m *March) dstKey(entry fs.DirEntry) string {+ return m.srcKey(entry) // FIXME actually do something different+}+// makeListDir makes constructs a listing function for the given fs// and includeAll flags for marching through the file system.// Note: this will optionally flag filter-aware backends!-func (m *March) makeListDir(ctx context.Context, f fs.Fs, includeAll bool) listDirFn {+func (m *March) makeListDir(ctx context.Context, f fs.Fs, includeAll bool, keyFn list.KeyFn) listDirFn {ci := fs.GetConfig(ctx)fi := filter.GetConfig(ctx)if !(ci.UseListR && f.Features().ListR != nil) && // !--fast-list active and!(ci.NoTraverse && fi.HaveFilesFrom()) { // !(--files-from and --no-traverse)return func(dir string, callback fs.ListRCallback) (err error) {dirCtx := filter.SetUseFilter(m.Ctx, f.Features().FilterAware && !includeAll) // make filter-aware backends constrain List- return list.DirSortedFn(dirCtx, f, includeAll, dir, callback, m.key)+ return list.DirSortedFn(dirCtx, f, includeAll, dir, callback, keyFn)}}@@ -144,7 +149,7 @@ func (m *March) makeListDir(ctx context.Context, f fs.Fs, includeAll bool) listD// in syncing as it will use the first entry for the sync// comparison.slices.SortStableFunc(entries, func(a, b fs.DirEntry) int {- return cmp.Compare(m.key(a), m.key(b))+ return cmp.Compare(keyFn(a), keyFn(b))})return callback(entries)}@@ -297,11 +302,11 @@ func (m *March) matchListings(srcChan, dstChan <-chan fs.DirEntry, srcOnly, dstO// Reload src and dst if needed - we set them to nil if usedif src == nil {src = <-srcChan- srcName = m.key(src)+ srcName = m.srcKey(src)}if dst == nil {dst = <-dstChan- dstName = m.key(dst)+ dstName = m.dstKey(dst)}if src == nil && dst == nil {break@@ -406,7 +411,7 @@ func (m *March) processJob(job listDirJob) ([]listDirJob, error) {if m.NoTraverse && !m.NoCheckDest {originalSrcChan := srcChansrcChan = make(chan fs.DirEntry, 100)- ls, err := list.NewSorter(m.Ctx, m.Fdst, list.SortToChan(dstChan), m.key)+ ls, err := list.NewSorter(m.Ctx, m.Fdst, list.SortToChan(dstChan), m.dstKey)if err != nil {return nil, err}diff --git a/fs/march/march_test.go b/fs/march/march_test.goindex ad71e6dd3..76747287e 100644--- a/fs/march/march_test.go+++ b/fs/march/march_test.go@@ -502,7 +502,11 @@ func TestMatchListings(t *testing.T) {// Make a channel to send the source (0) or dest (1) using a list.SortermakeChan := func(offset int) <-chan fs.DirEntry {out := make(chan fs.DirEntry)- ls, err := list.NewSorter(ctx, nil, list.SortToChan(out), m.key)+ key := m.dstKey+ if offset == 0 {+ key = m.srcKey+ }+ ls, err := list.NewSorter(ctx, nil, list.SortToChan(out), key)require.NoError(t, err)wg.Add(1)go func() {