Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

bisync: merge copies and deletes, support --track-renames and --backup-dir -- fixes #5690 fixes #5685

nielash0e5f12126f8a
diff --git a/cmd/bisync/deltas.go b/cmd/bisync/deltas.go
index ccd43edbe..6e0542dfb 100644
--- a/cmd/bisync/deltas.go
+++ b/cmd/bisync/deltas.go
@@ -327,6 +327,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change
if !in2 {
b.indent("Path2", p2, "Queue delete")
delete2.Add(file)
+ copy1to2.Add(file)
} else if d2.is(deltaOther) {
b.indent("Path2", p1, "Queue copy to Path1")
copy2to1.Add(file)
@@ -351,6 +352,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change
// Deleted
b.indent("Path1", p1, "Queue delete")
delete1.Add(file)
+ copy2to1.Add(file)
}
}
@@ -380,25 +382,17 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change
}
if delete1.NotEmpty() {
- changes1 = true
- b.indent("", "Path1", "Do queued deletes on")
- err = b.fastDelete(ctx, b.fs1, delete1, "delete1")
- if err != nil {
+ if err = b.saveQueue(delete1, "delete1"); err != nil {
return
}
-
//propagate deletions of empty dirs from path2 to path1 (if --create-empty-src-dirs)
b.syncEmptyDirs(ctx, b.fs1, delete1, dirs1, "remove")
}
if delete2.NotEmpty() {
- changes2 = true
- b.indent("", "Path2", "Do queued deletes on")
- err = b.fastDelete(ctx, b.fs2, delete2, "delete2")
- if err != nil {
+ if err = b.saveQueue(delete2, "delete2"); err != nil {
return
}
-
//propagate deletions of empty dirs from path1 to path2 (if --create-empty-src-dirs)
b.syncEmptyDirs(ctx, b.fs2, delete2, dirs2, "remove")
}
diff --git a/cmd/bisync/queue.go b/cmd/bisync/queue.go
index 701c8a922..d213ff74a 100644
--- a/cmd/bisync/queue.go
+++ b/cmd/bisync/queue.go
@@ -24,39 +24,11 @@ func (b *bisyncRun) fastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib.
}
}
- return sync.CopyDir(ctxCopy, fdst, fsrc, b.opt.CreateEmptySrcDirs)
-}
-
-func (b *bisyncRun) fastDelete(ctx context.Context, f fs.Fs, files bilib.Names, queueName string) error {
- if err := b.saveQueue(files, queueName); err != nil {
- return err
- }
-
- transfers := fs.GetConfig(ctx).Transfers
-
- ctxRun, filterDelete := filter.AddConfig(b.opt.setDryRun(ctx))
-
- for _, file := range files.ToList() {
- if err := filterDelete.AddFile(file); err != nil {
- return err
- }
- }
-
- objChan := make(fs.ObjectsChan, transfers)
- errChan := make(chan error, 1)
- go func() {
- errChan <- operations.DeleteFiles(ctxRun, objChan)
- }()
- err := operations.ListFn(ctxRun, f, func(obj fs.Object) {
- remote := obj.Remote()
- if files.Has(remote) {
- objChan <- obj
- }
- })
- close(objChan)
- opErr := <-errChan
- if err == nil {
- err = opErr
+ var err error
+ if b.opt.Resync {
+ err = sync.CopyDir(ctxCopy, fdst, fsrc, b.opt.CreateEmptySrcDirs)
+ } else {
+ err = sync.Sync(ctxCopy, fdst, fsrc, b.opt.CreateEmptySrcDirs)
}
return err
}
@@ -75,8 +47,9 @@ func (b *bisyncRun) syncEmptyDirs(ctx context.Context, dst fs.Fs, candidates bil
var direrr error
if dirsList.has(s) { //make sure it's a dir, not a file
if operation == "remove" {
- //note: we need to use Rmdirs instead of Rmdir because directories will fail to delete if they have other empty dirs inside of them.
- direrr = operations.Rmdirs(ctx, dst, s, false)
+ // directories made empty by the sync will have already been deleted during the sync
+ // this just catches the already-empty ones (excluded from sync by --files-from filter)
+ direrr = operations.TryRmdir(ctx, dst, s)
} else if operation == "make" {
direrr = operations.Mkdir(ctx, dst, s)
} else {
diff --git a/cmd/bisync/testdata/test_changes/golden/_testdir_path1.._testdir_path2.copy1to2.que b/cmd/bisync/testdata/test_changes/golden/_testdir_path1.._testdir_path2.copy1to2.que
index 38a6601db..5725b8184 100644
--- a/cmd/bisync/testdata/test_changes/golden/_testdir_path1.._testdir_path2.copy1to2.que
+++ b/cmd/bisync/testdata/test_changes/golden/_testdir_path1.._testdir_path2.copy1to2.que
@@ -1,4 +1,5 @@
"file11.txt"
"file2.txt"
+"file4.txt"
"file5.txt..path1"
"file7.txt"
diff --git a/cmd/bisync/testdata/test_changes/golden/_testdir_path1.._testdir_path2.copy2to1.que b/cmd/bisync/testdata/test_changes/golden/_testdir_path1.._testdir_path2.copy2to1.que
index 4146ce3e0..3b79e78bd 100644
--- a/cmd/bisync/testdata/test_changes/golden/_testdir_path1.._testdir_path2.copy2to1.que
+++ b/cmd/bisync/testdata/test_changes/golden/_testdir_path1.._testdir_path2.copy2to1.que
@@ -1,4 +1,5 @@
"file1.txt"
"file10.txt"
+"file3.txt"
"file5.txt..path2"
"file6.txt"
diff --git a/cmd/bisync/testdata/test_changes/golden/test.log b/cmd/bisync/testdata/test_changes/golden/test.log
index 222b29758..f77d40340 100644
--- a/cmd/bisync/testdata/test_changes/golden/test.log
+++ b/cmd/bisync/testdata/test_changes/golden/test.log
@@ -88,8 +88,6 @@ INFO : - Path2 Queue copy to Path1 - {path1/}file10.txt
INFO : - Path1 Queue delete - {path1/}file3.txt
INFO : - Path2 Do queued copies to - Path1
INFO : - Path1 Do queued copies to - Path2
-INFO : - Do queued deletes on - Path1
-INFO : - Do queued deletes on - Path2
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
diff --git a/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log b/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log
index b8799524a..c63c8850c 100644
--- a/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log
+++ b/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log
@@ -77,7 +77,7 @@ INFO : - Path2 File was deleted - subdir
INFO : Path2: 1 changes: 0 new, 0 newer, 0 older, 1 deleted
INFO : Applying changes
INFO : - Path2 Queue delete - {path2/}RCLONE_TEST
-INFO : - Do queued deletes on - Path2
+INFO : - Path1 Do queued copies to - Path2
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -121,7 +121,7 @@ INFO : Path1: 1 changes: 0 new, 0 newer, 0 older, 1 deleted
INFO : Path2 checking for diffs
INFO : Applying changes
INFO : - Path2 Queue delete - {path2/}subdir
-INFO : - Do queued deletes on - Path2
+INFO : - Path1 Do queued copies to - Path2
INFO : subdir: Removing directory
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
diff --git a/cmd/bisync/testdata/test_dry_run/golden/_testdir_path1.._testdir_path2.copy1to2.que b/cmd/bisync/testdata/test_dry_run/golden/_testdir_path1.._testdir_path2.copy1to2.que
index 38a6601db..5725b8184 100644
--- a/cmd/bisync/testdata/test_dry_run/golden/_testdir_path1.._testdir_path2.copy1to2.que
+++ b/cmd/bisync/testdata/test_dry_run/golden/_testdir_path1.._testdir_path2.copy1to2.que
@@ -1,4 +1,5 @@
"file11.txt"
"file2.txt"
+"file4.txt"
"file5.txt..path1"
"file7.txt"
diff --git a/cmd/bisync/testdata/test_dry_run/golden/_testdir_path1.._testdir_path2.copy2to1.que b/cmd/bisync/testdata/test_dry_run/golden/_testdir_path1.._testdir_path2.copy2to1.que
index 4146ce3e0..3b79e78bd 100644
--- a/cmd/bisync/testdata/test_dry_run/golden/_testdir_path1.._testdir_path2.copy2to1.que
+++ b/cmd/bisync/testdata/test_dry_run/golden/_testdir_path1.._testdir_path2.copy2to1.que
@@ -1,4 +1,5 @@
"file1.txt"
"file10.txt"
+"file3.txt"
"file5.txt..path2"
"file6.txt"
diff --git a/cmd/bisync/testdata/test_dry_run/golden/dryrun._testdir_path1.._testdir_path2.copy1to2.que b/cmd/bisync/testdata/test_dry_run/golden/dryrun._testdir_path1.._testdir_path2.copy1to2.que
index 38a6601db..5725b8184 100644
--- a/cmd/bisync/testdata/test_dry_run/golden/dryrun._testdir_path1.._testdir_path2.copy1to2.que
+++ b/cmd/bisync/testdata/test_dry_run/golden/dryrun._testdir_path1.._testdir_path2.copy1to2.que
@@ -1,4 +1,5 @@
"file11.txt"
"file2.txt"
+"file4.txt"
"file5.txt..path1"
"file7.txt"
diff --git a/cmd/bisync/testdata/test_dry_run/golden/dryrun._testdir_path1.._testdir_path2.copy2to1.que b/cmd/bisync/testdata/test_dry_run/golden/dryrun._testdir_path1.._testdir_path2.copy2to1.que
index 4146ce3e0..3b79e78bd 100644
--- a/cmd/bisync/testdata/test_dry_run/golden/dryrun._testdir_path1.._testdir_path2.copy2to1.que
+++ b/cmd/bisync/testdata/test_dry_run/golden/dryrun._testdir_path1.._testdir_path2.copy2to1.que
@@ -1,4 +1,5 @@
"file1.txt"
"file10.txt"
+"file3.txt"
"file5.txt..path2"
"file6.txt"
diff --git a/cmd/bisync/testdata/test_dry_run/golden/test.log b/cmd/bisync/testdata/test_dry_run/golden/test.log
index 13bd9654a..3d749965b 100644
--- a/cmd/bisync/testdata/test_dry_run/golden/test.log
+++ b/cmd/bisync/testdata/test_dry_run/golden/test.log
@@ -106,15 +106,13 @@ INFO : - Path1 Queue delete - {path1/}file3.txt
INFO : - Path2 Do queued copies to - Path1
NOTICE: file1.txt: Skipped copy as --dry-run is set (size 19)
NOTICE: file10.txt: Skipped copy as --dry-run is set (size 19)
+NOTICE: file3.txt: Skipped delete as --dry-run is set (size 0)
NOTICE: file6.txt: Skipped copy as --dry-run is set (size 19)
INFO : - Path1 Do queued copies to - Path2
NOTICE: file11.txt: Skipped copy as --dry-run is set (size 19)
NOTICE: file2.txt: Skipped copy as --dry-run is set (size 13)
-NOTICE: file7.txt: Skipped copy as --dry-run is set (size 19)
-INFO : - Do queued deletes on - Path1
-NOTICE: file3.txt: Skipped delete as --dry-run is set (size 0)
-INFO : - Do queued deletes on - Path2
NOTICE: file4.txt: Skipped delete as --dry-run is set (size 0)
+NOTICE: file7.txt: Skipped copy as --dry-run is set (size 19)
INFO : Updating listings
INFO : Bisync successful
(32) : copy-listings dryrun
@@ -159,8 +157,6 @@ INFO : - Path2 Queue copy to Path1 - {path1/}file10.txt
INFO : - Path1 Queue delete - {path1/}file3.txt
INFO : - Path2 Do queued copies to - Path1
INFO : - Path1 Do queued copies to - Path2
-INFO : - Do queued deletes on - Path1
-INFO : - Do queued deletes on - Path2
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
diff --git a/cmd/bisync/testdata/test_extended_filenames/golden/_testdir_path1.._testdir_path2.copy2to1.que b/cmd/bisync/testdata/test_extended_filenames/golden/_testdir_path1.._testdir_path2.copy2to1.que
index eece110c1..5c39d3955 100644
--- a/cmd/bisync/testdata/test_extended_filenames/golden/_testdir_path1.._testdir_path2.copy2to1.que
+++ b/cmd/bisync/testdata/test_extended_filenames/golden/_testdir_path1.._testdir_path2.copy2to1.que
@@ -1,7 +1,9 @@
"New_top_level_mañana_funcionará.txt"
"file_enconde_mañana_funcionará.txt"
+"filename_contains_ࢺ_.txt"
"subdir with␊white space.txt/file2 with␊white space.txt"
"subdir_rawchars_␙_\x81_\xfe/file3_␙_\x81_\xfe"
"subdir_with_ࢺ_/filechangedbothpaths_ࢺ_.txt..path2"
+"subdir_with_ࢺ_/filename_contains_ě_.txt"
"subdir_with_ࢺ_/filename_contains_ࢺ_p2s.txt"
"Русский.txt"
diff --git a/cmd/bisync/testdata/test_extended_filenames/golden/test.log b/cmd/bisync/testdata/test_extended_filenames/golden/test.log
index 5a7f6412a..d1db495d9 100644
--- a/cmd/bisync/testdata/test_extended_filenames/golden/test.log
+++ b/cmd/bisync/testdata/test_extended_filenames/golden/test.log
@@ -84,7 +84,6 @@ INFO : - Path1 Queue delete - {path1/}subdir_with_ࢺ
INFO : - Path2 Queue copy to Path1 - {path1/}subdir_with_ࢺ_/filename_contains_ࢺ_p2s.txt
INFO : - Path2 Do queued copies to - Path1
INFO : - Path1 Do queued copies to - Path2
-INFO : - Do queued deletes on - Path1
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
diff --git a/cmd/bisync/testdata/test_max_delete_path1/golden/_testdir_path1.._testdir_path2.copy1to2.que b/cmd/bisync/testdata/test_max_delete_path1/golden/_testdir_path1.._testdir_path2.copy1to2.que
new file mode 100644
index 000000000..9c041c2ba
--- /dev/null
+++ b/cmd/bisync/testdata/test_max_delete_path1/golden/_testdir_path1.._testdir_path2.copy1to2.que
@@ -0,0 +1,5 @@
+"file1.txt"
+"file2.txt"
+"file3.txt"
+"file4.txt"
+"file5.txt"
diff --git a/cmd/bisync/testdata/test_max_delete_path1/golden/test.log b/cmd/bisync/testdata/test_max_delete_path1/golden/test.log
index c7c208211..7d83e9150 100644
--- a/cmd/bisync/testdata/test_max_delete_path1/golden/test.log
+++ b/cmd/bisync/testdata/test_max_delete_path1/golden/test.log
@@ -49,7 +49,7 @@ INFO : - Path2 Queue delete - {path2/}file2.txt
INFO : - Path2 Queue delete - {path2/}file3.txt
INFO : - Path2 Queue delete - {path2/}file4.txt
INFO : - Path2 Queue delete - {path2/}file5.txt
-INFO : - Do queued deletes on - Path2
+INFO : - Path1 Do queued copies to - Path2
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
diff --git a/cmd/bisync/testdata/test_max_delete_path2_force/golden/_testdir_path1.._testdir_path2.copy2to1.que b/cmd/bisync/testdata/test_max_delete_path2_force/golden/_testdir_path1.._testdir_path2.copy2to1.que
new file mode 100644
index 000000000..9c041c2ba
--- /dev/null
+++ b/cmd/bisync/testdata/test_max_delete_path2_force/golden/_testdir_path1.._testdir_path2.copy2to1.que
@@ -0,0 +1,5 @@
+"file1.txt"
+"file2.txt"
+"file3.txt"
+"file4.txt"
+"file5.txt"
diff --git a/cmd/bisync/testdata/test_max_delete_path2_force/golden/test.log b/cmd/bisync/testdata/test_max_delete_path2_force/golden/test.log
index 58da0c3a3..23cc350c4 100644
--- a/cmd/bisync/testdata/test_max_delete_path2_force/golden/test.log
+++ b/cmd/bisync/testdata/test_max_delete_path2_force/golden/test.log
@@ -49,7 +49,7 @@ INFO : - Path1 Queue delete - {path1/}file2.txt
INFO : - Path1 Queue delete - {path1/}file3.txt
INFO : - Path1 Queue delete - {path1/}file4.txt
INFO : - Path1 Queue delete - {path1/}file5.txt
-INFO : - Do queued deletes on - Path1
+INFO : - Path2 Do queued copies to - Path1
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
diff --git a/cmd/bisync/testdata/test_rmdirs/golden/_testdir_path1.._testdir_path2.copy1to2.que b/cmd/bisync/testdata/test_rmdirs/golden/_testdir_path1.._testdir_path2.copy1to2.que
new file mode 100644
index 000000000..a49deb16c
--- /dev/null
+++ b/cmd/bisync/testdata/test_rmdirs/golden/_testdir_path1.._testdir_path2.copy1to2.que
@@ -0,0 +1 @@
+"subdir/file20.txt"
diff --git a/cmd/bisync/testdata/test_rmdirs/golden/test.log b/cmd/bisync/testdata/test_rmdirs/golden/test.log
index 936af897b..b770c2482 100644
--- a/cmd/bisync/testdata/test_rmdirs/golden/test.log
+++ b/cmd/bisync/testdata/test_rmdirs/golden/test.log
@@ -21,7 +21,7 @@ INFO : Path1: 1 changes: 0 new, 0 newer, 0 older, 1 deleted
INFO : Path2 checking for diffs
INFO : Applying changes
INFO : - Path2 Queue delete - {path2/}subdir/file20.txt
-INFO : - Do queued deletes on - Path2
+INFO : - Path1 Do queued copies to - Path2
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
diff --git a/docs/content/bisync.md b/docs/content/bisync.md
index be03f8cb8..94dc815e3 100644
--- a/docs/content/bisync.md
+++ b/docs/content/bisync.md
@@ -552,11 +552,17 @@ and use `--resync` when you need to switch.
### Renamed directories
-Renaming a folder on the Path1 side results in deleting all files on
+By default, renaming a folder on the Path1 side results in deleting all files on
the Path2 side and then copying all files again from Path1 to Path2.
Bisync sees this as all files in the old directory name as deleted and all
files in the new directory name as new.
-Currently, the most effective and efficient method of renaming a directory
+
+A recommended solution is to use [`--track-renames`](/docs/#track-renames),
+which is now supported in bisync as of `rclone v1.65`.
+Note that `--track-renames` is not available during `--resync`,
+as `--resync` does not delete anything (`--track-renames` only supports `sync`, not `copy`.)
+
+Otherwise, the most effective and efficient method of renaming a directory
is to rename it to the same name on both sides. (As of `rclone v1.64`,
a `--resync` is no longer required after doing so, as bisync will automatically
detect that Path1 and Path2 are in agreement.)
@@ -1263,6 +1269,10 @@ about _Unison_ and synchronization in general.
## Changelog
+### `v1.65`
+* Copies and deletes are now handled in one operation instead of two
+* `--track-renames` and `--backup-dir` are now supported
+
### `v1.64`
* Fixed an [issue](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=1.%20Dry%20runs%20are%20not%20completely%20dry)
causing dry runs to inadvertently commit filter changes