"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
bisync: support two --backup-dir paths on different remotes
nielashbbf9b1b3d212
diff --git a/cmd/bisync/cmd.go b/cmd/bisync/cmd.goindex 54a417cb4..fafbfaff5 100644--- a/cmd/bisync/cmd.go+++ b/cmd/bisync/cmd.go@@ -40,6 +40,9 @@ type Options struct {Force boolFiltersFile stringWorkdir string+ OrigBackupDir string+ BackupDir1 string+ BackupDir2 stringDryRun boolNoCleanup boolSaveQueues bool // save extra debugging files (test only flag)@@ -107,6 +110,8 @@ func init() {Opt.Retries = 3cmd.Root.AddCommand(commandDefinition)cmdFlags := commandDefinition.Flags()+ // when adding new flags, remember to also update the rc params:+ // cmd/bisync/rc.go cmd/bisync/help.go (not docs/content/rc.md)flags.BoolVarP(cmdFlags, &Opt.Resync, "resync", "1", Opt.Resync, "Performs the resync run. Path1 files may overwrite Path2 versions. Consider using --verbose or --dry-run first.", "")flags.BoolVarP(cmdFlags, &Opt.CheckAccess, "check-access", "", Opt.CheckAccess, makeHelp("Ensure expected {CHECKFILE} files are found on both Path1 and Path2 filesystems, else abort."), "")flags.StringVarP(cmdFlags, &Opt.CheckFilename, "check-filename", "", Opt.CheckFilename, makeHelp("Filename for --check-access (default: {CHECKFILE})"), "")@@ -116,6 +121,8 @@ func init() {flags.BoolVarP(cmdFlags, &Opt.RemoveEmptyDirs, "remove-empty-dirs", "", Opt.RemoveEmptyDirs, "Remove ALL empty directories at the final cleanup step.", "")flags.StringVarP(cmdFlags, &Opt.FiltersFile, "filters-file", "", Opt.FiltersFile, "Read filtering patterns from a file", "")flags.StringVarP(cmdFlags, &Opt.Workdir, "workdir", "", Opt.Workdir, makeHelp("Use custom working dir - useful for testing. (default: {WORKDIR})"), "")+ flags.StringVarP(cmdFlags, &Opt.BackupDir1, "backup-dir1", "", Opt.BackupDir1, "--backup-dir for Path1. Must be a non-overlapping path on the same remote.", "")+ flags.StringVarP(cmdFlags, &Opt.BackupDir2, "backup-dir2", "", Opt.BackupDir2, "--backup-dir for Path2. Must be a non-overlapping path on the same remote.", "")flags.BoolVarP(cmdFlags, &tzLocal, "localtime", "", tzLocal, "Use local time in listings (default: UTC)", "")flags.BoolVarP(cmdFlags, &Opt.NoCleanup, "no-cleanup", "", Opt.NoCleanup, "Retain working files (useful for troubleshooting and testing).", "")flags.BoolVarP(cmdFlags, &Opt.IgnoreListingChecksum, "ignore-listing-checksum", "", Opt.IgnoreListingChecksum, "Do not use checksums for listings (add --ignore-checksum to additionally skip post-copy checksum checks)", "")diff --git a/cmd/bisync/deltas.go b/cmd/bisync/deltas.goindex 770dac750..23a10540f 100644--- a/cmd/bisync/deltas.go+++ b/cmd/bisync/deltas.go@@ -346,6 +346,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change} else {fs.Debugf(nil, "Files are NOT equal: %s", file)b.indent("!Path1", p1+"..path1", "Renaming Path1 copy")+ ctxMove = b.setBackupDir(ctxMove, 1) // in case already a file with new nameif err = operations.MoveFile(ctxMove, b.fs1, b.fs1, file+"..path1", file); err != nil {err = fmt.Errorf("path1 rename failed for %s: %w", p1, err)b.critical = true@@ -360,6 +361,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (changecopy1to2.Add(file + "..path1")b.indent("!Path2", p2+"..path2", "Renaming Path2 copy")+ ctxMove = b.setBackupDir(ctxMove, 2) // in case already a file with new nameif err = operations.MoveFile(ctxMove, b.fs2, b.fs2, alias+"..path2", alias); err != nil {err = fmt.Errorf("path2 rename failed for %s: %w", alias, err)return@@ -426,6 +428,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (changeif copy2to1.NotEmpty() {changes1 = trueb.indent("Path2", "Path1", "Do queued copies to")+ ctx = b.setBackupDir(ctx, 1)results2to1, err = b.fastCopy(ctx, b.fs2, b.fs1, copy2to1, "copy2to1")// retries, if any@@ -442,6 +445,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (changeif copy1to2.NotEmpty() {changes2 = trueb.indent("Path1", "Path2", "Do queued copies to")+ ctx = b.setBackupDir(ctx, 2)results1to2, err = b.fastCopy(ctx, b.fs1, b.fs2, copy1to2, "copy1to2")// retries, if anydiff --git a/cmd/bisync/help.go b/cmd/bisync/help.goindex 84e78ab9d..61c944a59 100644--- a/cmd/bisync/help.go+++ b/cmd/bisync/help.go@@ -10,7 +10,7 @@ func makeHelp(help string) string {"|", "`","{MAXDELETE}", strconv.Itoa(DefaultMaxDelete),"{CHECKFILE}", DefaultCheckFilename,- "{WORKDIR}", DefaultWorkdir,+ // "{WORKDIR}", DefaultWorkdir,)return replacer.Replace(help)}@@ -37,7 +37,9 @@ var rcHelp = makeHelp(`This takes the following parameters- ignoreListingChecksum - Do not use checksums for listings- resilient - Allow future runs to retry after certain less-serious errors, instead of requiring resync.Use at your own risk!-- workdir - server directory for history files (default: {WORKDIR})+- workdir - server directory for history files (default: |~/.cache/rclone/bisync|)+- backupdir1 - --backup-dir for Path1. Must be a non-overlapping path on the same remote.+- backupdir2 - --backup-dir for Path2. Must be a non-overlapping path on the same remote.- noCleanup - retain working filesSee [bisync command help](https://rclone.org/commands/rclone_bisync/)diff --git a/cmd/bisync/operations.go b/cmd/bisync/operations.goindex 5d4926c7d..d845b4826 100644--- a/cmd/bisync/operations.go+++ b/cmd/bisync/operations.go@@ -68,6 +68,8 @@ func Bisync(ctx context.Context, fs1, fs2 fs.Fs, optArg *Options) (err error) {if opt.Workdir == "" {opt.Workdir = DefaultWorkdir}+ ci := fs.GetConfig(ctx)+ opt.OrigBackupDir = ci.BackupDirif !opt.DryRun && !opt.Force {if fs1.Precision() == fs.ModTimeNotSupported {@@ -358,7 +360,9 @@ func (b *bisyncRun) runLocked(octx context.Context) (err error) {// Optional rmdirs for empty directoriesif opt.RemoveEmptyDirs {fs.Infof(nil, "Removing empty directories")+ fctx = b.setBackupDir(fctx, 1)err1 := operations.Rmdirs(fctx, b.fs1, "", true)+ fctx = b.setBackupDir(fctx, 2)err2 := operations.Rmdirs(fctx, b.fs2, "", true)err := err1if err == nil {@@ -445,6 +449,8 @@ func (b *bisyncRun) resync(octx, fctx context.Context) error {}ci := fs.GetConfig(ctxSync)ci.IgnoreExisting = true+ ctxSync = b.setBackupDir(ctxSync, 1)+ // 2 to 1if results2to1, err = b.resyncDir(ctxSync, b.fs2, b.fs1); err != nil {b.critical = truereturn err@@ -452,6 +458,8 @@ func (b *bisyncRun) resync(octx, fctx context.Context) error {b.indent("Path1", "Path2", "Resync is copying UNIQUE OR DIFFERING files to")ci.IgnoreExisting = false+ ctxSync = b.setBackupDir(ctxSync, 2)+ // 1 to 2if results1to2, err = b.resyncDir(ctxSync, b.fs1, b.fs2); err != nil {b.critical = truereturn err@@ -581,3 +589,17 @@ func (b *bisyncRun) handleErr(o interface{}, msg string, err error, critical, re}}}++// setBackupDir overrides --backup-dir with path-specific version, if set, in each direction+func (b *bisyncRun) setBackupDir(ctx context.Context, destPath int) context.Context {+ ci := fs.GetConfig(ctx)+ ci.BackupDir = b.opt.OrigBackupDir+ if destPath == 1 && b.opt.BackupDir1 != "" {+ ci.BackupDir = b.opt.BackupDir1+ }+ if destPath == 2 && b.opt.BackupDir2 != "" {+ ci.BackupDir = b.opt.BackupDir1+ }+ fs.Debugf(ci.BackupDir, "updated backup-dir for Path%d", destPath)+ return ctx+}diff --git a/cmd/bisync/rc.go b/cmd/bisync/rc.goindex 550be5e38..c0d36a372 100644--- a/cmd/bisync/rc.go+++ b/cmd/bisync/rc.go@@ -74,6 +74,12 @@ func rcBisync(ctx context.Context, in rc.Params) (out rc.Params, err error) {if opt.Workdir, err = in.GetString("workdir"); rc.NotErrParamNotFound(err) {return}+ if opt.BackupDir1, err = in.GetString("backupdir1"); rc.NotErrParamNotFound(err) {+ return+ }+ if opt.BackupDir2, err = in.GetString("backupdir2"); rc.NotErrParamNotFound(err) {+ return+ }checkSync, err := in.GetString("checkSync")if rc.NotErrParamNotFound(err) {diff --git a/docs/content/bisync.md b/docs/content/bisync.mdindex 67d338287..f3354dc1e 100644--- a/docs/content/bisync.md+++ b/docs/content/bisync.md@@ -105,6 +105,8 @@ Optional Flags:--no-cleanup Retain working files (useful for troubleshooting and testing).--workdir PATH Use custom working directory (useful for testing).(default: `~/.cache/rclone/bisync`)+ --backup-dir1 PATH --backup-dir for Path1. Must be a non-overlapping path on the same remote.+ --backup-dir2 PATH --backup-dir for Path2. Must be a non-overlapping path on the same remote.-n, --dry-run Go through the motions - No files are copied/deleted.-v, --verbose Increases logging verbosity.May be specified more than once for more details.@@ -357,6 +359,42 @@ Certain more serious errors will still enforce a `--resync` lockout, even in `--Behavior of `--resilient` may change in a future version.+#### --backup-dir1 and --backup-dir2++As of `v1.65`, [`--backup-dir`](/docs/#backup-dir-dir) is supported in bisync.+Because `--backup-dir` must be a non-overlapping path on the same remote,+Bisync has introduced new `--backup-dir1` and `--backup-dir2` flags to support+separate backup-dirs for `Path1` and `Path2` (bisyncing between different+remotes with `--backup-dir` would not otherwise be possible.) `--backup-dir1`+and `--backup-dir2` can use different remotes from each other, but+`--backup-dir1` must use the same remote as `Path1`, and `--backup-dir2` must+use the same remote as `Path2`. Each backup directory must not overlap its+respective bisync Path without being excluded by a filter rule.++The standard `--backup-dir` will also work, if both paths use the same remote+(but note that deleted files from both paths would be mixed together in the+same dir). If either `--backup-dir1` and `--backup-dir2` are set, they will+override `--backup-dir`.++Example:+```+rclone bisync /Users/someuser/some/local/path/Bisync gdrive:Bisync --backup-dir1 /Users/someuser/some/local/path/BackupDir --backup-dir2 gdrive:BackupDir --suffix -2023-08-26 --suffix-keep-extension --check-access --max-delete 10 --filters-file /Users/someuser/some/local/path/bisync_filters.txt --no-cleanup --ignore-listing-checksum --checkers=16 --drive-pacer-min-sleep=10ms --create-empty-src-dirs --resilient -MvP --drive-skip-gdocs --fix-case+```++In this example, if the user deletes a file in+`/Users/someuser/some/local/path/Bisync`, bisync will propagate the delete to+the other side by moving the corresponding file from `gdrive:Bisync` to+`gdrive:BackupDir`. If the user deletes a file from `gdrive:Bisync`, bisync+moves it from `/Users/someuser/some/local/path/Bisync` to+`/Users/someuser/some/local/path/BackupDir`.++In the event of a `..path1` / `..path2` rename due to a sync conflict, the+rename is not considered a delete, unless a previous conflict with the same+name already exists and would get overwritten.++See also: [`--suffix`](/docs/#suffix-suffix),+[`--suffix-keep-extension`](/docs/#suffix-keep-extension)+## Operation### Runtime flow details