"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
march: fix unnecessarily listing dst directory when src listing finished
Nick Craig-Wood9e0a5b66a4a9
diff --git a/fs/march/march.go b/fs/march/march.goindex 4ba97a2f9..effd240bd 100644--- a/fs/march/march.go+++ b/fs/march/march.go@@ -44,6 +44,7 @@ type March struct {Callback Marcher // object to call with resultsNoCheckDest bool // transfer all objects regardless without checking dstNoUnicodeNormalization bool // don't normalize unicode characters in filenames+ NoProcessDstOnly bool // if set, when source listing finishes, cancel the dst listing// internal statesrcListDir listDirFn // function to call to list a directory in the srcdstListDir listDirFn // function to call to list a directory in the dst@@ -297,7 +298,7 @@ func (m *March) aborting() bool {// Into match go matchPair's of src and dst which have the same name//// This checks for duplicates and checks the list is sorted.-func (m *March) matchListings(srcChan, dstChan <-chan fs.DirEntry, srcOnly, dstOnly func(fs.DirEntry), match func(dst, src fs.DirEntry)) error {+func (m *March) matchListings(srcChan, dstChan <-chan fs.DirEntry, dstCancel func(), srcOnly, dstOnly func(fs.DirEntry), match func(dst, src fs.DirEntry)) error {var (srcPrev, dstPrev fs.DirEntrysrcPrevName, dstPrevName string@@ -326,6 +327,12 @@ func (m *March) matchListings(srcChan, dstChan <-chan fs.DirEntry, srcOnly, dstOsrc, srcHasMore = <-srcChansrcName = m.srcKey(src)}+ // If the source listing is finished and we don't need+ // dst-only entries, cancel the dst listing early.+ if !srcHasMore && m.NoProcessDstOnly {+ dstCancel()+ break+ }if dst == nil {dst, dstHasMore = <-dstChandstName = m.dstKey(dst)@@ -392,9 +399,12 @@ func (m *March) processJob(job listDirJob) ([]listDirJob, error) {srcChan = make(chan fs.DirEntry, 100)dstChan = make(chan fs.DirEntry, 100)srcListErr, dstListErr error+ dstListCancelled boolwg sync.WaitGroupci = fs.GetConfig(m.Ctx)+ dstCtx, dstCancel = context.WithCancel(m.Ctx))+ defer dstCancel()// List the src and dst directoriesif !job.noSrc {@@ -415,12 +425,19 @@ func (m *March) processJob(job listDirJob) ([]listDirJob, error) {if !m.NoTraverse && !job.noDst {startedDst = truewg.Go(func() {- dstListErr = m.dstListDir(m.Ctx, job.dstRemote, func(entries fs.DirEntries) error {+ dstListErr = m.dstListDir(dstCtx, job.dstRemote, func(entries fs.DirEntries) error {for _, entry := range entries {- dstChan <- entry+ select {+ case <-dstCtx.Done():+ return dstCtx.Err()+ case dstChan <- entry:+ }}return nil})+ if dstCtx.Err() != nil {+ dstListCancelled = true+ }close(dstChan)})}@@ -502,7 +519,7 @@ func (m *March) processJob(job listDirJob) ([]listDirJob, error) {}// Work out what to do and do it- err := m.matchListings(srcChan, dstChan, func(src fs.DirEntry) {+ err := m.matchListings(srcChan, dstChan, dstCancel, func(src fs.DirEntry) {recurse := m.Callback.SrcOnly(src)if recurse && job.srcDepth > 0 {jobs = append(jobs, listDirJob{@@ -548,7 +565,9 @@ func (m *March) processJob(job listDirJob) ([]listDirJob, error) {srcListErr = fs.CountError(m.Ctx, srcListErr)return nil, srcListErr}- if dstListErr == fs.ErrorDirNotFound {+ if dstListCancelled {+ // Ignore dst listing errors if we cancelled it+ } else if dstListErr == fs.ErrorDirNotFound {// Copy the stuff anyway} else if dstListErr != nil {if job.dstRemote != "" {diff --git a/fs/march/march_test.go b/fs/march/march_test.goindex d1a866538..2a916c351 100644--- a/fs/march/march_test.go+++ b/fs/march/march_test.go@@ -274,6 +274,110 @@ func TestMarch(t *testing.T) {}}+func TestMarchNoProcessDstOnly(t *testing.T) {+ for _, test := range []struct {+ what string+ fileSrcOnly []string+ dirSrcOnly []string+ fileDstOnly []string+ dirDstOnly []string+ fileMatch []string+ dirMatch []string+ noProcessDstOnly bool+ fastList bool+ }{+ {+ // dst-only files sort after all src files so they are+ // skipped when NoProcessDstOnly cancels the dst listing.+ what: "no process dst only - typical sync",+ fileSrcOnly: []string{"a_srcOnly"},+ fileMatch: []string{"b_match"},+ fileDstOnly: []string{"z_dstOnly1", "z_dstOnly2"},+ noProcessDstOnly: true,+ },+ {+ what: "no process dst only - fast list",+ fileSrcOnly: []string{"a_srcOnly"},+ fileMatch: []string{"b_match"},+ fileDstOnly: []string{"z_dstOnly1", "z_dstOnly2"},+ noProcessDstOnly: true,+ fastList: true,+ },+ {+ what: "no process dst only - source only",+ fileSrcOnly: []string{"test", "test2"},+ noProcessDstOnly: true,+ },+ {+ what: "no process dst only - dest only skipped",+ fileSrcOnly: []string{"a_src"},+ fileDstOnly: []string{"z_dstOnly1", "z_dstOnly2"},+ noProcessDstOnly: true,+ },+ } {+ t.Run(fmt.Sprintf("TestMarch-%s", test.what), func(t *testing.T) {+ r := fstest.NewRun(t)++ var srcOnly []fstest.Item+ var match []fstest.Item++ ctx, cancel := context.WithCancel(context.Background())++ for _, f := range test.fileSrcOnly {+ srcOnly = append(srcOnly, r.WriteFile(f, "hello world", t1))+ }+ for _, f := range test.fileDstOnly {+ r.WriteObject(ctx, f, "hello world", t1)+ }+ for _, f := range test.fileMatch {+ match = append(match, r.WriteBoth(ctx, f, "hello world", t1))+ }++ ctx, ci := fs.AddConfig(ctx)+ ci.UseListR = test.fastList++ fi := filter.GetConfig(ctx)++ // Local backend doesn't implement ListR, so monkey patch it for this test+ if test.fastList && r.Flocal.Features().ListR == nil {+ r.Flocal.Features().ListR = func(ctx context.Context, dir string, callback fs.ListRCallback) error {+ r.Flocal.Features().ListR = nil+ return walk.ListR(ctx, r.Flocal, dir, true, -1, walk.ListAll, callback)+ }+ defer func() {+ r.Flocal.Features().ListR = nil+ }()+ }++ mt := &marchTester{+ ctx: ctx,+ cancel: cancel,+ }+ m := &March{+ Ctx: ctx,+ Fdst: r.Fremote,+ Fsrc: r.Flocal,+ Dir: "",+ Callback: mt,+ DstIncludeAll: fi.Opt.DeleteExcluded,+ NoProcessDstOnly: test.noProcessDstOnly,+ }++ mt.processError(m.Run(ctx))+ mt.cancel()+ err := mt.currentError()+ require.NoError(t, err)++ precision := fs.GetModifyWindow(ctx, r.Fremote, r.Flocal)++ // With NoProcessDstOnly, dst-only entries should not be reported+ fstest.CompareItems(t, mt.srcOnly, srcOnly, test.dirSrcOnly, precision, "srcOnly")+ assert.Empty(t, mt.dstOnly, "dstOnly should be empty with NoProcessDstOnly")+ fstest.CompareItems(t, mt.match, match, test.dirMatch, precision, "match")+ })+ }+}+// matchPair is a matched pair of direntries returned by matchListingstype matchPair struct {src, dst fs.DirEntry@@ -535,7 +639,7 @@ func TestMatchListings(t *testing.T) {matches = append(matches, matchPair{dst: dst, src: src})}- err := m.matchListings(makeChan(0), makeChan(1), srcOnlyFn, dstOnlyFn, matchFn)+ err := m.matchListings(makeChan(0), makeChan(1), func() {}, srcOnlyFn, dstOnlyFn, matchFn)require.NoError(t, err)wg.Wait()assert.Equal(t, test.srcOnly, srcOnly, test.what, "srcOnly differ")@@ -544,7 +648,7 @@ func TestMatchListings(t *testing.T) {// now swap src and dstsrcOnly, dstOnly, matches = nil, nil, nil- err = m.matchListings(makeChan(0), makeChan(1), srcOnlyFn, dstOnlyFn, matchFn)+ err = m.matchListings(makeChan(0), makeChan(1), func() {}, srcOnlyFn, dstOnlyFn, matchFn)require.NoError(t, err)wg.Wait()assert.Equal(t, test.srcOnly, srcOnly, test.what, "srcOnly differ")@@ -553,3 +657,138 @@ func TestMatchListings(t *testing.T) {})}}++func TestMatchListingsNoProcessDstOnly(t *testing.T) {+ var (+ a = mockobject.Object("a")+ b = mockobject.Object("b")+ c = mockobject.Object("c")+ d = mockobject.Object("d")+ )++ for _, test := range []struct {+ what string+ input fs.DirEntries // pairs of input src, dst+ srcOnly fs.DirEntries+ matches []matchPair+ }{+ {+ what: "src and dst with dst-only skipped",+ input: fs.DirEntries{+ a, nil,+ b, b,+ nil, c,+ nil, d,+ },+ srcOnly: fs.DirEntries{+ a,+ },+ matches: []matchPair{+ {b, b},+ },+ },+ {+ what: "all dst-only skipped",+ input: fs.DirEntries{+ nil, a,+ nil, b,+ nil, c,+ },+ },+ {+ what: "all src-only",+ input: fs.DirEntries{+ a, nil,+ b, nil,+ c, nil,+ },+ srcOnly: fs.DirEntries{+ a, b, c,+ },+ },+ {+ what: "matches then dst-only skipped",+ input: fs.DirEntries{+ a, a,+ b, b,+ nil, c,+ nil, d,+ },+ matches: []matchPair{+ {a, a},+ {b, b},+ },+ },+ } {+ t.Run(test.what, func(t *testing.T) {+ ctx := context.Background()+ var wg sync.WaitGroup++ m := March{+ Ctx: context.Background(),+ NoProcessDstOnly: true,+ }++ // makeChan creates a channel sending sorted entries.+ // It returns the channel and a drain function to unblock+ // senders if the channel isn't fully consumed.+ makeChan := func(offset int) (<-chan fs.DirEntry, func()) {+ out := make(chan fs.DirEntry)+ key := m.dstKey+ if offset == 0 {+ key = m.srcKey+ }+ ls, err := list.NewSorter(ctx, nil, list.SortToChan(out), key)+ require.NoError(t, err)+ wg.Go(func() {+ for i := 0; i < len(test.input); i += 2 {+ entry := test.input[i+offset]+ if entry != nil {+ require.NoError(t, ls.Add(fs.DirEntries{entry}))+ }+ }+ require.NoError(t, ls.Send())+ ls.CleanUp()+ close(out)+ })+ drain := func() {+ for range out {+ }+ }+ return out, drain+ }++ var srcOnly fs.DirEntries+ srcOnlyFn := func(entry fs.DirEntry) {+ srcOnly = append(srcOnly, entry)+ }+ var dstOnly fs.DirEntries+ dstOnlyFn := func(entry fs.DirEntry) {+ dstOnly = append(dstOnly, entry)+ }+ var matches []matchPair+ matchFn := func(dst, src fs.DirEntry) {+ matches = append(matches, matchPair{dst: dst, src: src})+ }++ srcChan, _ := makeChan(0)+ dstChan, drainDst := makeChan(1)++ cancelled := false+ dstCancel := func() {+ cancelled = true+ // Drain remaining dst entries to unblock the sender,+ // mimicking what the real code does via context cancellation.+ go drainDst()+ }++ err := m.matchListings(srcChan, dstChan, dstCancel, srcOnlyFn, dstOnlyFn, matchFn)+ require.NoError(t, err)+ wg.Wait()+ assert.Equal(t, test.srcOnly, srcOnly, test.what, "srcOnly differ")+ assert.Empty(t, dstOnly, test.what, "dstOnly should be empty")+ assert.Equal(t, test.matches, matches, test.what, "matches differ")+ assert.True(t, cancelled, "dstCancel should have been called")+ })+ }+}diff --git a/fs/sync/sync.go b/fs/sync/sync.goindex 075e18188..4892204ea 100644--- a/fs/sync/sync.go+++ b/fs/sync/sync.go@@ -961,6 +961,7 @@ func (s *syncCopyMove) run() error {DstIncludeAll: s.fi.Opt.DeleteExcluded,NoCheckDest: s.noCheckDest,NoUnicodeNormalization: s.noUnicodeNormalization,+ NoProcessDstOnly: s.deleteMode == fs.DeleteModeOff && !s.usingLogger,}s.processError(m.Run(s.ctx))