Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

march: split src and dst

nielash41a407dcc9ec
diff --git a/fs/march/march.go b/fs/march/march.go
index 578aea512..def481ac9 100644
--- a/fs/march/march.go
+++ b/fs/march/march.go
@@ -60,9 +60,9 @@ type Marcher interface {
// Note: this will flag filter-aware backends on the source side
func (m *March) init(ctx context.Context) {
ci := fs.GetConfig(ctx)
- m.srcListDir = m.makeListDir(ctx, m.Fsrc, m.SrcIncludeAll)
+ m.srcListDir = m.makeListDir(ctx, m.Fsrc, m.SrcIncludeAll, m.srcKey)
if !m.NoTraverse {
- m.dstListDir = m.makeListDir(ctx, m.Fdst, m.DstIncludeAll)
+ m.dstListDir = m.makeListDir(ctx, m.Fdst, m.DstIncludeAll, m.dstKey)
}
// Now create the matching transform
// ..normalise the UTF8 first
@@ -80,8 +80,8 @@ func (m *March) init(ctx context.Context) {
}
}
-// key turns a directory entry into a sort key using the defined transforms.
-func (m *March) key(entry fs.DirEntry) string {
+// srcKey turns a directory entry into a sort key using the defined transforms.
+func (m *March) srcKey(entry fs.DirEntry) string {
if entry == nil {
return ""
}
@@ -99,17 +99,22 @@ func (m *March) key(entry fs.DirEntry) string {
return name
}
+// dstKey turns a directory entry into a sort key using the defined transforms.
+func (m *March) dstKey(entry fs.DirEntry) string {
+ return m.srcKey(entry) // FIXME actually do something different
+}
+
// makeListDir makes constructs a listing function for the given fs
// and includeAll flags for marching through the file system.
// Note: this will optionally flag filter-aware backends!
-func (m *March) makeListDir(ctx context.Context, f fs.Fs, includeAll bool) listDirFn {
+func (m *March) makeListDir(ctx context.Context, f fs.Fs, includeAll bool, keyFn list.KeyFn) listDirFn {
ci := fs.GetConfig(ctx)
fi := filter.GetConfig(ctx)
if !(ci.UseListR && f.Features().ListR != nil) && // !--fast-list active and
!(ci.NoTraverse && fi.HaveFilesFrom()) { // !(--files-from and --no-traverse)
return func(dir string, callback fs.ListRCallback) (err error) {
dirCtx := filter.SetUseFilter(m.Ctx, f.Features().FilterAware && !includeAll) // make filter-aware backends constrain List
- return list.DirSortedFn(dirCtx, f, includeAll, dir, callback, m.key)
+ return list.DirSortedFn(dirCtx, f, includeAll, dir, callback, keyFn)
}
}
@@ -144,7 +149,7 @@ func (m *March) makeListDir(ctx context.Context, f fs.Fs, includeAll bool) listD
// in syncing as it will use the first entry for the sync
// comparison.
slices.SortStableFunc(entries, func(a, b fs.DirEntry) int {
- return cmp.Compare(m.key(a), m.key(b))
+ return cmp.Compare(keyFn(a), keyFn(b))
})
return callback(entries)
}
@@ -297,11 +302,11 @@ func (m *March) matchListings(srcChan, dstChan <-chan fs.DirEntry, srcOnly, dstO
// Reload src and dst if needed - we set them to nil if used
if src == nil {
src = <-srcChan
- srcName = m.key(src)
+ srcName = m.srcKey(src)
}
if dst == nil {
dst = <-dstChan
- dstName = m.key(dst)
+ dstName = m.dstKey(dst)
}
if src == nil && dst == nil {
break
@@ -406,7 +411,7 @@ func (m *March) processJob(job listDirJob) ([]listDirJob, error) {
if m.NoTraverse && !m.NoCheckDest {
originalSrcChan := srcChan
srcChan = make(chan fs.DirEntry, 100)
- ls, err := list.NewSorter(m.Ctx, m.Fdst, list.SortToChan(dstChan), m.key)
+ ls, err := list.NewSorter(m.Ctx, m.Fdst, list.SortToChan(dstChan), m.dstKey)
if err != nil {
return nil, err
}
diff --git a/fs/march/march_test.go b/fs/march/march_test.go
index ad71e6dd3..76747287e 100644
--- a/fs/march/march_test.go
+++ b/fs/march/march_test.go
@@ -502,7 +502,11 @@ func TestMatchListings(t *testing.T) {
// Make a channel to send the source (0) or dest (1) using a list.Sorter
makeChan := func(offset int) <-chan fs.DirEntry {
out := make(chan fs.DirEntry)
- ls, err := list.NewSorter(ctx, nil, list.SortToChan(out), m.key)
+ key := m.dstKey
+ if offset == 0 {
+ key = m.srcKey
+ }
+ ls, err := list.NewSorter(ctx, nil, list.SortToChan(out), key)
require.NoError(t, err)
wg.Add(1)
go func() {