"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
bisync: add support for --retries-sleep - fixes #7555
nielashb14269fd23a3
diff --git a/cmd/bisync/cmd.go b/cmd/bisync/cmd.goindex 805e9897d..e50ae9e16 100644--- a/cmd/bisync/cmd.go+++ b/cmd/bisync/cmd.go@@ -52,6 +52,7 @@ type Options struct {Recover boolTestFn TestFunc // test-only option, for mocking errorsRetries int+ RetriesInterval time.DurationCompare CompareOptCompareFlag stringDebugName string@@ -143,7 +144,8 @@ func init() {flags.BoolVarP(cmdFlags, &Opt.IgnoreListingChecksum, "ignore-listing-checksum", "", Opt.IgnoreListingChecksum, "Do not use checksums for listings (add --ignore-checksum to additionally skip post-copy checksum checks)", "")flags.BoolVarP(cmdFlags, &Opt.Resilient, "resilient", "", Opt.Resilient, "Allow future runs to retry after certain less-serious errors, instead of requiring --resync. Use at your own risk!", "")flags.BoolVarP(cmdFlags, &Opt.Recover, "recover", "", Opt.Recover, "Automatically recover from interruptions without requiring --resync.", "")- flags.IntVarP(cmdFlags, &Opt.Retries, "retries", "", Opt.Retries, "Retry operations this many times if they fail", "")+ flags.IntVarP(cmdFlags, &Opt.Retries, "retries", "", Opt.Retries, "Retry operations this many times if they fail (requires --resilient).", "")+ flags.DurationVarP(cmdFlags, &Opt.RetriesInterval, "retries-sleep", "", 0, "Interval between retrying operations if they fail, e.g. 500ms, 60s, 5m (0 to disable)", "")flags.StringVarP(cmdFlags, &Opt.CompareFlag, "compare", "", Opt.CompareFlag, "Comma-separated list of bisync-specific compare options ex. 'size,modtime,checksum' (default: 'size,modtime')", "")flags.BoolVarP(cmdFlags, &Opt.Compare.NoSlowHash, "no-slow-hash", "", Opt.Compare.NoSlowHash, "Ignore listing checksums only on backends where they are slow", "")flags.BoolVarP(cmdFlags, &Opt.Compare.SlowHashSyncOnly, "slow-hash-sync-only", "", Opt.Compare.SlowHashSyncOnly, "Ignore slow checksums for listings and deltas, but still consider them during sync calls.", "")diff --git a/cmd/bisync/operations.go b/cmd/bisync/operations.goindex 432ceaee4..4b5a288b1 100644--- a/cmd/bisync/operations.go+++ b/cmd/bisync/operations.go@@ -127,19 +127,6 @@ func Bisync(ctx context.Context, fs1, fs2 fs.Fs, optArg *Options) (err error) {// Handle SIGINTvar finaliseOnce gosync.Once- // waitFor runs fn() until it returns true or the timeout expires- waitFor := func(msg string, totalWait time.Duration, fn func() bool) (ok bool) {- const individualWait = 1 * time.Second- for i := 0; i < int(totalWait/individualWait); i++ {- ok = fn()- if ok {- return ok- }- fs.Infof(nil, Color(terminal.YellowFg, "%s: %v"), msg, int(totalWait/individualWait)-i)- time.Sleep(individualWait)- }- return false- }finalise := func() {finaliseOnce.Do(func() {if atexit.Signalled() {@@ -648,6 +635,20 @@ func (b *bisyncRun) debugFn(nametocheck string, fn func()) {}}+// waitFor runs fn() until it returns true or the timeout expires+func waitFor(msg string, totalWait time.Duration, fn func() bool) (ok bool) {+ const individualWait = 1 * time.Second+ for i := 0; i < int(totalWait/individualWait); i++ {+ ok = fn()+ if ok {+ return ok+ }+ fs.Infof(nil, Color(terminal.YellowFg, "%s: %vs"), msg, int(totalWait/individualWait)-i)+ time.Sleep(individualWait)+ }+ return false+}+// mainly to make sure tests don't interfere with each other when running more than onefunc resetGlobals() {downloadHash = falsediff --git a/cmd/bisync/queue.go b/cmd/bisync/queue.goindex 73d9f18e6..bdd85b665 100644--- a/cmd/bisync/queue.go+++ b/cmd/bisync/queue.go@@ -266,6 +266,16 @@ func (b *bisyncRun) retryFastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bfor tries := 1; tries <= b.opt.Retries; tries++ {fs.Logf(queueName, Color(terminal.YellowFg, "Received error: %v - retrying as --resilient is set. Retry %d/%d"), err, tries, b.opt.Retries)accounting.GlobalStats().ResetErrors()+ if retryAfter := accounting.GlobalStats().RetryAfter(); !retryAfter.IsZero() {+ d := time.Until(retryAfter)+ if d > 0 {+ fs.Logf(nil, "Received retry after error - sleeping until %s (%v)", retryAfter.Format(time.RFC3339Nano), d)+ time.Sleep(d)+ }+ }+ if b.opt.RetriesInterval > 0 {+ naptime(b.opt.RetriesInterval)+ }results, err = b.fastCopy(ctx, fsrc, fdst, files, queueName)if err == nil || b.InGracefulShutdown {return results, err@@ -362,3 +372,16 @@ func (b *bisyncRun) saveQueue(files bilib.Names, jobName string) error {queueFile := fmt.Sprintf("%s.%s.que", b.basePath, jobName)return files.Save(queueFile)}++func naptime(totalWait time.Duration) {+ expireTime := time.Now().Add(totalWait)+ fs.Logf(nil, "will retry in %v at %v", totalWait, expireTime.Format("2006-01-02 15:04:05 MST"))+ for i := 0; time.Until(expireTime) > 0; i++ {+ if i > 0 && i%10 == 0 {+ fs.Infof(nil, Color(terminal.Dim, "retrying in %v..."), time.Until(expireTime).Round(1*time.Second))+ } else {+ fs.Debugf(nil, Color(terminal.Dim, "retrying in %v..."), time.Until(expireTime).Round(1*time.Second))+ }+ time.Sleep(1 * time.Second)+ }+}diff --git a/docs/content/bisync.md b/docs/content/bisync.mdindex 93b819297..85250985c 100644--- a/docs/content/bisync.md+++ b/docs/content/bisync.md@@ -114,7 +114,8 @@ Optional Flags:--resilient Allow future runs to retry after certain less-serious errors, instead of requiring --resync. Use at your own risk!-1, --resync Performs the resync run. Equivalent to --resync-mode path1. Consider using --verbose or --dry-run first.--resync-mode string During resync, prefer the version that is: path1, path2, newer, older, larger, smaller (default: path1 if --resync, otherwise none for no resync.) (default "none")- --retries int Retry operations this many times if they fail (default 3)+ --retries int Retry operations this many times if they fail (requires --resilient). (default 3)+ --retries-sleep Duration Interval between retrying operations if they fail, e.g. 500ms, 60s, 5m (0 to disable) (default 0s)--slow-hash-sync-only Ignore slow checksums for listings and deltas, but still consider them during sync calls.--workdir string Use custom working dir - useful for testing. (default: {WORKDIR})--max-delete PERCENT Safety check on maximum percentage of deleted files allowed. If exceeded, the bisync run will abort. (default: 50%)@@ -1833,6 +1834,7 @@ instead of of `--size-only`, when `check` is not available.* A new `--max-lock` setting allows lock files to automatically renew and expire, for better automatic recovery when a run is interrupted.* Bisync now supports auto-resolving sync conflicts and customizing rename behavior with new [`--conflict-resolve`](#conflict-resolve), [`--conflict-loser`](#conflict-loser), and [`--conflict-suffix`](#conflict-suffix) flags.* A new [`--resync-mode`](#resync-mode) flag allows more control over which version of a file gets kept during a `--resync`.+* Bisync now supports [`--retries`](/docs/#retries-int) and [`--retries-sleep`](/docs/#retries-sleep-time) (when [`--resilient`](#resilient) is set.)### `v1.64`* Fixed an [issue](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=1.%20Dry%20runs%20are%20not%20completely%20dry)