"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
bisync: merge copies and deletes, support --track-renames and --backup-dir -- fixes #5690 fixes #5685
nielash0e5f12126f8a
diff --git a/cmd/bisync/deltas.go b/cmd/bisync/deltas.goindex ccd43edbe..6e0542dfb 100644--- a/cmd/bisync/deltas.go+++ b/cmd/bisync/deltas.go@@ -327,6 +327,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (changeif !in2 {b.indent("Path2", p2, "Queue delete")delete2.Add(file)+ copy1to2.Add(file)} else if d2.is(deltaOther) {b.indent("Path2", p1, "Queue copy to Path1")copy2to1.Add(file)@@ -351,6 +352,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change// Deletedb.indent("Path1", p1, "Queue delete")delete1.Add(file)+ copy2to1.Add(file)}}@@ -380,25 +382,17 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change}if delete1.NotEmpty() {- changes1 = true- b.indent("", "Path1", "Do queued deletes on")- err = b.fastDelete(ctx, b.fs1, delete1, "delete1")- if err != nil {+ if err = b.saveQueue(delete1, "delete1"); err != nil {return}-//propagate deletions of empty dirs from path2 to path1 (if --create-empty-src-dirs)b.syncEmptyDirs(ctx, b.fs1, delete1, dirs1, "remove")}if delete2.NotEmpty() {- changes2 = true- b.indent("", "Path2", "Do queued deletes on")- err = b.fastDelete(ctx, b.fs2, delete2, "delete2")- if err != nil {+ if err = b.saveQueue(delete2, "delete2"); err != nil {return}-//propagate deletions of empty dirs from path1 to path2 (if --create-empty-src-dirs)b.syncEmptyDirs(ctx, b.fs2, delete2, dirs2, "remove")}diff --git a/cmd/bisync/queue.go b/cmd/bisync/queue.goindex 701c8a922..d213ff74a 100644--- a/cmd/bisync/queue.go+++ b/cmd/bisync/queue.go@@ -24,39 +24,11 @@ func (b *bisyncRun) fastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib.}}- return sync.CopyDir(ctxCopy, fdst, fsrc, b.opt.CreateEmptySrcDirs)-}--func (b *bisyncRun) fastDelete(ctx context.Context, f fs.Fs, files bilib.Names, queueName string) error {- if err := b.saveQueue(files, queueName); err != nil {- return err- }-- transfers := fs.GetConfig(ctx).Transfers-- ctxRun, filterDelete := filter.AddConfig(b.opt.setDryRun(ctx))-- for _, file := range files.ToList() {- if err := filterDelete.AddFile(file); err != nil {- return err- }- }-- objChan := make(fs.ObjectsChan, transfers)- errChan := make(chan error, 1)- go func() {- errChan <- operations.DeleteFiles(ctxRun, objChan)- }()- err := operations.ListFn(ctxRun, f, func(obj fs.Object) {- remote := obj.Remote()- if files.Has(remote) {- objChan <- obj- }- })- close(objChan)- opErr := <-errChan- if err == nil {- err = opErr+ var err error+ if b.opt.Resync {+ err = sync.CopyDir(ctxCopy, fdst, fsrc, b.opt.CreateEmptySrcDirs)+ } else {+ err = sync.Sync(ctxCopy, fdst, fsrc, b.opt.CreateEmptySrcDirs)}return err}@@ -75,8 +47,9 @@ func (b *bisyncRun) syncEmptyDirs(ctx context.Context, dst fs.Fs, candidates bilvar direrr errorif dirsList.has(s) { //make sure it's a dir, not a fileif operation == "remove" {- //note: we need to use Rmdirs instead of Rmdir because directories will fail to delete if they have other empty dirs inside of them.- direrr = operations.Rmdirs(ctx, dst, s, false)+ // directories made empty by the sync will have already been deleted during the sync+ // this just catches the already-empty ones (excluded from sync by --files-from filter)+ direrr = operations.TryRmdir(ctx, dst, s)} else if operation == "make" {direrr = operations.Mkdir(ctx, dst, s)} else {diff --git a/cmd/bisync/testdata/test_changes/golden/_testdir_path1.._testdir_path2.copy1to2.que b/cmd/bisync/testdata/test_changes/golden/_testdir_path1.._testdir_path2.copy1to2.queindex 38a6601db..5725b8184 100644--- a/cmd/bisync/testdata/test_changes/golden/_testdir_path1.._testdir_path2.copy1to2.que+++ b/cmd/bisync/testdata/test_changes/golden/_testdir_path1.._testdir_path2.copy1to2.que@@ -1,4 +1,5 @@"file11.txt""file2.txt"+"file4.txt""file5.txt..path1""file7.txt"diff --git a/cmd/bisync/testdata/test_changes/golden/_testdir_path1.._testdir_path2.copy2to1.que b/cmd/bisync/testdata/test_changes/golden/_testdir_path1.._testdir_path2.copy2to1.queindex 4146ce3e0..3b79e78bd 100644--- a/cmd/bisync/testdata/test_changes/golden/_testdir_path1.._testdir_path2.copy2to1.que+++ b/cmd/bisync/testdata/test_changes/golden/_testdir_path1.._testdir_path2.copy2to1.que@@ -1,4 +1,5 @@"file1.txt""file10.txt"+"file3.txt""file5.txt..path2""file6.txt"diff --git a/cmd/bisync/testdata/test_changes/golden/test.log b/cmd/bisync/testdata/test_changes/golden/test.logindex 222b29758..f77d40340 100644--- a/cmd/bisync/testdata/test_changes/golden/test.log+++ b/cmd/bisync/testdata/test_changes/golden/test.log@@ -88,8 +88,6 @@ INFO : - Path2 Queue copy to Path1 - {path1/}file10.txtINFO : - Path1 Queue delete - {path1/}file3.txtINFO : - Path2 Do queued copies to - Path1INFO : - Path1 Do queued copies to - Path2-INFO : - Do queued deletes on - Path1-INFO : - Do queued deletes on - Path2INFO : Updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : Bisync successfuldiff --git a/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log b/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.logindex b8799524a..c63c8850c 100644--- a/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log+++ b/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log@@ -77,7 +77,7 @@ INFO : - Path2 File was deleted - subdirINFO : Path2: 1 changes: 0 new, 0 newer, 0 older, 1 deletedINFO : Applying changesINFO : - Path2 Queue delete - {path2/}RCLONE_TEST-INFO : - Do queued deletes on - Path2+INFO : - Path1 Do queued copies to - Path2INFO : Updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : Bisync successful@@ -121,7 +121,7 @@ INFO : Path1: 1 changes: 0 new, 0 newer, 0 older, 1 deletedINFO : Path2 checking for diffsINFO : Applying changesINFO : - Path2 Queue delete - {path2/}subdir-INFO : - Do queued deletes on - Path2+INFO : - Path1 Do queued copies to - Path2INFO : subdir: Removing directoryINFO : Updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"diff --git a/cmd/bisync/testdata/test_dry_run/golden/_testdir_path1.._testdir_path2.copy1to2.que b/cmd/bisync/testdata/test_dry_run/golden/_testdir_path1.._testdir_path2.copy1to2.queindex 38a6601db..5725b8184 100644--- a/cmd/bisync/testdata/test_dry_run/golden/_testdir_path1.._testdir_path2.copy1to2.que+++ b/cmd/bisync/testdata/test_dry_run/golden/_testdir_path1.._testdir_path2.copy1to2.que@@ -1,4 +1,5 @@"file11.txt""file2.txt"+"file4.txt""file5.txt..path1""file7.txt"diff --git a/cmd/bisync/testdata/test_dry_run/golden/_testdir_path1.._testdir_path2.copy2to1.que b/cmd/bisync/testdata/test_dry_run/golden/_testdir_path1.._testdir_path2.copy2to1.queindex 4146ce3e0..3b79e78bd 100644--- a/cmd/bisync/testdata/test_dry_run/golden/_testdir_path1.._testdir_path2.copy2to1.que+++ b/cmd/bisync/testdata/test_dry_run/golden/_testdir_path1.._testdir_path2.copy2to1.que@@ -1,4 +1,5 @@"file1.txt""file10.txt"+"file3.txt""file5.txt..path2""file6.txt"diff --git a/cmd/bisync/testdata/test_dry_run/golden/dryrun._testdir_path1.._testdir_path2.copy1to2.que b/cmd/bisync/testdata/test_dry_run/golden/dryrun._testdir_path1.._testdir_path2.copy1to2.queindex 38a6601db..5725b8184 100644--- a/cmd/bisync/testdata/test_dry_run/golden/dryrun._testdir_path1.._testdir_path2.copy1to2.que+++ b/cmd/bisync/testdata/test_dry_run/golden/dryrun._testdir_path1.._testdir_path2.copy1to2.que@@ -1,4 +1,5 @@"file11.txt""file2.txt"+"file4.txt""file5.txt..path1""file7.txt"diff --git a/cmd/bisync/testdata/test_dry_run/golden/dryrun._testdir_path1.._testdir_path2.copy2to1.que b/cmd/bisync/testdata/test_dry_run/golden/dryrun._testdir_path1.._testdir_path2.copy2to1.queindex 4146ce3e0..3b79e78bd 100644--- a/cmd/bisync/testdata/test_dry_run/golden/dryrun._testdir_path1.._testdir_path2.copy2to1.que+++ b/cmd/bisync/testdata/test_dry_run/golden/dryrun._testdir_path1.._testdir_path2.copy2to1.que@@ -1,4 +1,5 @@"file1.txt""file10.txt"+"file3.txt""file5.txt..path2""file6.txt"diff --git a/cmd/bisync/testdata/test_dry_run/golden/test.log b/cmd/bisync/testdata/test_dry_run/golden/test.logindex 13bd9654a..3d749965b 100644--- a/cmd/bisync/testdata/test_dry_run/golden/test.log+++ b/cmd/bisync/testdata/test_dry_run/golden/test.log@@ -106,15 +106,13 @@ INFO : - Path1 Queue delete - {path1/}file3.txtINFO : - Path2 Do queued copies to - Path1NOTICE: file1.txt: Skipped copy as --dry-run is set (size 19)NOTICE: file10.txt: Skipped copy as --dry-run is set (size 19)+NOTICE: file3.txt: Skipped delete as --dry-run is set (size 0)NOTICE: file6.txt: Skipped copy as --dry-run is set (size 19)INFO : - Path1 Do queued copies to - Path2NOTICE: file11.txt: Skipped copy as --dry-run is set (size 19)NOTICE: file2.txt: Skipped copy as --dry-run is set (size 13)-NOTICE: file7.txt: Skipped copy as --dry-run is set (size 19)-INFO : - Do queued deletes on - Path1-NOTICE: file3.txt: Skipped delete as --dry-run is set (size 0)-INFO : - Do queued deletes on - Path2NOTICE: file4.txt: Skipped delete as --dry-run is set (size 0)+NOTICE: file7.txt: Skipped copy as --dry-run is set (size 19)INFO : Updating listingsINFO : Bisync successful(32) : copy-listings dryrun@@ -159,8 +157,6 @@ INFO : - Path2 Queue copy to Path1 - {path1/}file10.txtINFO : - Path1 Queue delete - {path1/}file3.txtINFO : - Path2 Do queued copies to - Path1INFO : - Path1 Do queued copies to - Path2-INFO : - Do queued deletes on - Path1-INFO : - Do queued deletes on - Path2INFO : Updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : Bisync successfuldiff --git a/cmd/bisync/testdata/test_extended_filenames/golden/_testdir_path1.._testdir_path2.copy2to1.que b/cmd/bisync/testdata/test_extended_filenames/golden/_testdir_path1.._testdir_path2.copy2to1.queindex eece110c1..5c39d3955 100644--- a/cmd/bisync/testdata/test_extended_filenames/golden/_testdir_path1.._testdir_path2.copy2to1.que+++ b/cmd/bisync/testdata/test_extended_filenames/golden/_testdir_path1.._testdir_path2.copy2to1.que@@ -1,7 +1,9 @@"New_top_level_mañana_funcionará.txt""file_enconde_mañana_funcionará.txt"+"filename_contains_ࢺ_.txt""subdir with␊white space.txt/file2 with␊white space.txt""subdir_rawchars_␙_\x81_\xfe/file3_␙_\x81_\xfe""subdir_with_ࢺ_/filechangedbothpaths_ࢺ_.txt..path2"+"subdir_with_ࢺ_/filename_contains_ě_.txt""subdir_with_ࢺ_/filename_contains_ࢺ_p2s.txt""Русский.txt"diff --git a/cmd/bisync/testdata/test_extended_filenames/golden/test.log b/cmd/bisync/testdata/test_extended_filenames/golden/test.logindex 5a7f6412a..d1db495d9 100644--- a/cmd/bisync/testdata/test_extended_filenames/golden/test.log+++ b/cmd/bisync/testdata/test_extended_filenames/golden/test.log@@ -84,7 +84,6 @@ INFO : - Path1 Queue delete - {path1/}subdir_with_ࢺINFO : - Path2 Queue copy to Path1 - {path1/}subdir_with_ࢺ_/filename_contains_ࢺ_p2s.txtINFO : - Path2 Do queued copies to - Path1INFO : - Path1 Do queued copies to - Path2-INFO : - Do queued deletes on - Path1INFO : Updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : Bisync successfuldiff --git a/cmd/bisync/testdata/test_max_delete_path1/golden/_testdir_path1.._testdir_path2.copy1to2.que b/cmd/bisync/testdata/test_max_delete_path1/golden/_testdir_path1.._testdir_path2.copy1to2.quenew file mode 100644index 000000000..9c041c2ba--- /dev/null+++ b/cmd/bisync/testdata/test_max_delete_path1/golden/_testdir_path1.._testdir_path2.copy1to2.que@@ -0,0 +1,5 @@+"file1.txt"+"file2.txt"+"file3.txt"+"file4.txt"+"file5.txt"diff --git a/cmd/bisync/testdata/test_max_delete_path1/golden/test.log b/cmd/bisync/testdata/test_max_delete_path1/golden/test.logindex c7c208211..7d83e9150 100644--- a/cmd/bisync/testdata/test_max_delete_path1/golden/test.log+++ b/cmd/bisync/testdata/test_max_delete_path1/golden/test.log@@ -49,7 +49,7 @@ INFO : - Path2 Queue delete - {path2/}file2.txtINFO : - Path2 Queue delete - {path2/}file3.txtINFO : - Path2 Queue delete - {path2/}file4.txtINFO : - Path2 Queue delete - {path2/}file5.txt-INFO : - Do queued deletes on - Path2+INFO : - Path1 Do queued copies to - Path2INFO : Updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : Bisync successfuldiff --git a/cmd/bisync/testdata/test_max_delete_path2_force/golden/_testdir_path1.._testdir_path2.copy2to1.que b/cmd/bisync/testdata/test_max_delete_path2_force/golden/_testdir_path1.._testdir_path2.copy2to1.quenew file mode 100644index 000000000..9c041c2ba--- /dev/null+++ b/cmd/bisync/testdata/test_max_delete_path2_force/golden/_testdir_path1.._testdir_path2.copy2to1.que@@ -0,0 +1,5 @@+"file1.txt"+"file2.txt"+"file3.txt"+"file4.txt"+"file5.txt"diff --git a/cmd/bisync/testdata/test_max_delete_path2_force/golden/test.log b/cmd/bisync/testdata/test_max_delete_path2_force/golden/test.logindex 58da0c3a3..23cc350c4 100644--- a/cmd/bisync/testdata/test_max_delete_path2_force/golden/test.log+++ b/cmd/bisync/testdata/test_max_delete_path2_force/golden/test.log@@ -49,7 +49,7 @@ INFO : - Path1 Queue delete - {path1/}file2.txtINFO : - Path1 Queue delete - {path1/}file3.txtINFO : - Path1 Queue delete - {path1/}file4.txtINFO : - Path1 Queue delete - {path1/}file5.txt-INFO : - Do queued deletes on - Path1+INFO : - Path2 Do queued copies to - Path1INFO : Updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : Bisync successfuldiff --git a/cmd/bisync/testdata/test_rmdirs/golden/_testdir_path1.._testdir_path2.copy1to2.que b/cmd/bisync/testdata/test_rmdirs/golden/_testdir_path1.._testdir_path2.copy1to2.quenew file mode 100644index 000000000..a49deb16c--- /dev/null+++ b/cmd/bisync/testdata/test_rmdirs/golden/_testdir_path1.._testdir_path2.copy1to2.que@@ -0,0 +1 @@+"subdir/file20.txt"diff --git a/cmd/bisync/testdata/test_rmdirs/golden/test.log b/cmd/bisync/testdata/test_rmdirs/golden/test.logindex 936af897b..b770c2482 100644--- a/cmd/bisync/testdata/test_rmdirs/golden/test.log+++ b/cmd/bisync/testdata/test_rmdirs/golden/test.log@@ -21,7 +21,7 @@ INFO : Path1: 1 changes: 0 new, 0 newer, 0 older, 1 deletedINFO : Path2 checking for diffsINFO : Applying changesINFO : - Path2 Queue delete - {path2/}subdir/file20.txt-INFO : - Do queued deletes on - Path2+INFO : - Path1 Do queued copies to - Path2INFO : Updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : Bisync successfuldiff --git a/docs/content/bisync.md b/docs/content/bisync.mdindex be03f8cb8..94dc815e3 100644--- a/docs/content/bisync.md+++ b/docs/content/bisync.md@@ -552,11 +552,17 @@ and use `--resync` when you need to switch.### Renamed directories-Renaming a folder on the Path1 side results in deleting all files on+By default, renaming a folder on the Path1 side results in deleting all files onthe Path2 side and then copying all files again from Path1 to Path2.Bisync sees this as all files in the old directory name as deleted and allfiles in the new directory name as new.-Currently, the most effective and efficient method of renaming a directory++A recommended solution is to use [`--track-renames`](/docs/#track-renames),+which is now supported in bisync as of `rclone v1.65`.+Note that `--track-renames` is not available during `--resync`,+as `--resync` does not delete anything (`--track-renames` only supports `sync`, not `copy`.)++Otherwise, the most effective and efficient method of renaming a directoryis to rename it to the same name on both sides. (As of `rclone v1.64`,a `--resync` is no longer required after doing so, as bisync will automaticallydetect that Path1 and Path2 are in agreement.)@@ -1263,6 +1269,10 @@ about _Unison_ and synchronization in general.## Changelog+### `v1.65`+* Copies and deletes are now handled in one operation instead of two+* `--track-renames` and `--backup-dir` are now supported+### `v1.64`* Fixed an [issue](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=1.%20Dry%20runs%20are%20not%20completely%20dry)causing dry runs to inadvertently commit filter changes