"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
bisync: high-level retries if --resilient
nielash44637dcd7fe1
diff --git a/cmd/bisync/cmd.go b/cmd/bisync/cmd.goindex c3a2e1e57..54a417cb4 100644--- a/cmd/bisync/cmd.go+++ b/cmd/bisync/cmd.go@@ -46,6 +46,7 @@ type Options struct {IgnoreListingChecksum boolResilient boolTestFn TestFunc // test-only option, for mocking errors+ Retries int}// Default values@@ -103,6 +104,7 @@ func (x *CheckSyncMode) Type() string {var Opt Optionsfunc init() {+ Opt.Retries = 3cmd.Root.AddCommand(commandDefinition)cmdFlags := commandDefinition.Flags()flags.BoolVarP(cmdFlags, &Opt.Resync, "resync", "1", Opt.Resync, "Performs the resync run. Path1 files may overwrite Path2 versions. Consider using --verbose or --dry-run first.", "")@@ -118,6 +120,7 @@ func init() {flags.BoolVarP(cmdFlags, &Opt.NoCleanup, "no-cleanup", "", Opt.NoCleanup, "Retain working files (useful for troubleshooting and testing).", "")flags.BoolVarP(cmdFlags, &Opt.IgnoreListingChecksum, "ignore-listing-checksum", "", Opt.IgnoreListingChecksum, "Do not use checksums for listings (add --ignore-checksum to additionally skip post-copy checksum checks)", "")flags.BoolVarP(cmdFlags, &Opt.Resilient, "resilient", "", Opt.Resilient, "Allow future runs to retry after certain less-serious errors, instead of requiring --resync. Use at your own risk!", "")+ flags.IntVarP(cmdFlags, &Opt.Retries, "retries", "", Opt.Retries, "Retry operations this many times if they fail", "")}// bisync command definitiondiff --git a/cmd/bisync/deltas.go b/cmd/bisync/deltas.goindex 68fb538c3..770dac750 100644--- a/cmd/bisync/deltas.go+++ b/cmd/bisync/deltas.go@@ -427,6 +427,10 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (changechanges1 = trueb.indent("Path2", "Path1", "Do queued copies to")results2to1, err = b.fastCopy(ctx, b.fs2, b.fs1, copy2to1, "copy2to1")++ // retries, if any+ results2to1, err = b.retryFastCopy(ctx, b.fs2, b.fs1, copy2to1, "copy2to1", results2to1, err)+if err != nil {return}@@ -439,6 +443,10 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (changechanges2 = trueb.indent("Path1", "Path2", "Do queued copies to")results1to2, err = b.fastCopy(ctx, b.fs1, b.fs2, copy1to2, "copy1to2")++ // retries, if any+ results1to2, err = b.retryFastCopy(ctx, b.fs1, b.fs2, copy1to2, "copy1to2", results1to2, err)+if err != nil {return}diff --git a/cmd/bisync/queue.go b/cmd/bisync/queue.goindex f08a82f89..1c4a56b8e 100644--- a/cmd/bisync/queue.go+++ b/cmd/bisync/queue.go@@ -14,6 +14,7 @@ import ("github.com/rclone/rclone/fs/filter""github.com/rclone/rclone/fs/operations""github.com/rclone/rclone/fs/sync"+ "github.com/rclone/rclone/lib/terminal")// Results represents a pair of synced files, as reported by the LoggerFn@@ -182,6 +183,16 @@ func (b *bisyncRun) fastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib.return getResults, err}+func (b *bisyncRun) retryFastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib.Names, queueName string, results []Results, err error) ([]Results, error) {+ if err != nil && b.opt.Resilient && b.opt.Retries > 1 {+ for tries := 1; tries <= b.opt.Retries; tries++ {+ fs.Logf(queueName, Color(terminal.YellowFg, "Received error: %v - retrying as --resilient is set. Retry %d/%d"), err, tries, b.opt.Retries)+ results, err = b.fastCopy(ctx, fsrc, fdst, files, queueName)+ }+ }+ return results, err+}+func (b *bisyncRun) resyncDir(ctx context.Context, fsrc, fdst fs.Fs) ([]Results, error) {ignoreListingChecksum = b.opt.IgnoreListingChecksumlogger.LoggerFn = WriteResults