"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
bisync: auto-generate rc help docs
nielash72c561d20958
diff --git a/Makefile b/Makefileindex 07f66379e..1e4127994 100644--- a/Makefile+++ b/Makefile@@ -146,6 +146,7 @@ MANUAL.txt: MANUAL.mdcommanddocs: rclonego generate ./lib/transform+ go generate ./cmd/bisync-@rmdir -p '$$HOME/.config/rclone'XDG_CACHE_HOME="" XDG_CONFIG_HOME="" HOME="\$$HOME" USER="\$$USER" rclone gendocs --config=/notfound docs/content/@[ ! -e '$$HOME' ] || (echo 'Error: created unwanted directory named $$HOME' && exit 1)diff --git a/cmd/bisync/cmd.go b/cmd/bisync/cmd.goindex dd22e48c5..f1b3367bb 100644--- a/cmd/bisync/cmd.go+++ b/cmd/bisync/cmd.go@@ -127,14 +127,14 @@ func init() {// and the Command line syntax section of docs/content/bisync.md (it doesn't update automatically)flags.BoolVarP(cmdFlags, &Opt.Resync, "resync", "1", Opt.Resync, "Performs the resync run. Equivalent to --resync-mode path1. Consider using --verbose or --dry-run first.", "")flags.FVarP(cmdFlags, &Opt.ResyncMode, "resync-mode", "", "During resync, prefer the version that is: path1, path2, newer, older, larger, smaller (default: path1 if --resync, otherwise none for no resync.)", "")- flags.BoolVarP(cmdFlags, &Opt.CheckAccess, "check-access", "", Opt.CheckAccess, makeHelp("Ensure expected {CHECKFILE} files are found on both Path1 and Path2 filesystems, else abort."), "")- flags.StringVarP(cmdFlags, &Opt.CheckFilename, "check-filename", "", Opt.CheckFilename, makeHelp("Filename for --check-access (default: {CHECKFILE})"), "")+ flags.BoolVarP(cmdFlags, &Opt.CheckAccess, "check-access", "", Opt.CheckAccess, MakeHelp("Ensure expected {CHECKFILE} files are found on both Path1 and Path2 filesystems, else abort."), "")+ flags.StringVarP(cmdFlags, &Opt.CheckFilename, "check-filename", "", Opt.CheckFilename, MakeHelp("Filename for --check-access (default: {CHECKFILE})"), "")flags.BoolVarP(cmdFlags, &Opt.Force, "force", "", Opt.Force, "Bypass --max-delete safety check and run the sync. Consider using with --verbose", "")flags.FVarP(cmdFlags, &Opt.CheckSync, "check-sync", "", "Controls comparison of final listings: true|false|only (default: true)", "")flags.BoolVarP(cmdFlags, &Opt.CreateEmptySrcDirs, "create-empty-src-dirs", "", Opt.CreateEmptySrcDirs, "Sync creation and deletion of empty directories. (Not compatible with --remove-empty-dirs)", "")flags.BoolVarP(cmdFlags, &Opt.RemoveEmptyDirs, "remove-empty-dirs", "", Opt.RemoveEmptyDirs, "Remove ALL empty directories at the final cleanup step.", "")flags.StringVarP(cmdFlags, &Opt.FiltersFile, "filters-file", "", Opt.FiltersFile, "Read filtering patterns from a file", "")- flags.StringVarP(cmdFlags, &Opt.Workdir, "workdir", "", Opt.Workdir, makeHelp("Use custom working dir - useful for testing. (default: {WORKDIR})"), "")+ flags.StringVarP(cmdFlags, &Opt.Workdir, "workdir", "", Opt.Workdir, MakeHelp("Use custom working dir - useful for testing. (default: {WORKDIR})"), "")flags.StringVarP(cmdFlags, &Opt.BackupDir1, "backup-dir1", "", Opt.BackupDir1, "--backup-dir for Path1. Must be a non-overlapping path on the same remote.", "")flags.StringVarP(cmdFlags, &Opt.BackupDir2, "backup-dir2", "", Opt.BackupDir2, "--backup-dir for Path2. Must be a non-overlapping path on the same remote.", "")flags.StringVarP(cmdFlags, &Opt.DebugName, "debugname", "", Opt.DebugName, "Debug by tracking one file at various points throughout a bisync run (when -v or -vv)", "")@@ -153,6 +153,7 @@ func init() {flags.StringVarP(cmdFlags, &Opt.ConflictSuffixFlag, "conflict-suffix", "", Opt.ConflictSuffixFlag, "Suffix to use when renaming a --conflict-loser. Can be either one string or two comma-separated strings to assign different suffixes to Path1/Path2. (default: 'conflict')", "")_ = cmdFlags.MarkHidden("debugname")_ = cmdFlags.MarkHidden("localtime")+ addRC()}// bisync command definitiondiff --git a/cmd/bisync/help.go b/cmd/bisync/help.goindex a9699a848..84cd55967 100644--- a/cmd/bisync/help.go+++ b/cmd/bisync/help.go@@ -1,65 +1,77 @@-package bisync+//go:build none++// Create the help text for the rc+//+// Run with go generate ./cmd/bisync (defined in rc.go)+package mainimport (- "strconv"+ "fmt"+ "os""strings"++ "github.com/muesli/reflow/wordwrap"+ "github.com/rclone/rclone/cmd"+ "github.com/rclone/rclone/cmd/bisync"+ "github.com/rclone/rclone/fs"+ "github.com/spf13/pflag"+ "golang.org/x/text/cases"+ "golang.org/x/text/language")-func makeHelp(help string) string {- replacer := strings.NewReplacer(- "|", "`",- "{MAXDELETE}", strconv.Itoa(DefaultMaxDelete),- "{CHECKFILE}", DefaultCheckFilename,- // "{WORKDIR}", DefaultWorkdir,- )- return replacer.Replace(help)+// Output the help to stdout+func main() {+ out := os.Stdout+ if len(os.Args) > 1 {+ var err error+ out, err = os.Create(os.Args[1])+ if err != nil {+ fs.Fatalf(nil, "Open output failed: %v", err)+ }+ defer out.Close()+ }+ fmt.Fprintf(out, "<!--- Docs generated by help.go - use go generate to rebuild - DO NOT EDIT --->\n\n")+ fmt.Fprint(out, RcHelp())}-var shortHelp = `Perform bidirectional synchronization between two paths.`--var rcHelp = makeHelp(`This takes the following parameters--- path1 - a remote directory string e.g. |drive:path1|-- path2 - a remote directory string e.g. |drive:path2|-- dryRun - dry-run mode-- resync - performs the resync run-- checkAccess - abort if {CHECKFILE} files are not found on both filesystems-- checkFilename - file name for checkAccess (default: {CHECKFILE})-- maxDelete - abort sync if percentage of deleted files is above- this threshold (default: {MAXDELETE})-- force - Bypass maxDelete safety check and run the sync-- checkSync - |true| by default, |false| disables comparison of final listings,- |only| will skip sync, only compare listings from the last run-- createEmptySrcDirs - Sync creation and deletion of empty directories.- (Not compatible with --remove-empty-dirs)-- removeEmptyDirs - remove empty directories at the final cleanup step-- filtersFile - read filtering patterns from a file-- ignoreListingChecksum - Do not use checksums for listings-- resilient - Allow future runs to retry after certain less-serious errors, instead of requiring resync.-- workdir - server directory for history files (default: |~/.cache/rclone/bisync|)-- backupdir1 - --backup-dir for Path1. Must be a non-overlapping path on the same remote.-- backupdir2 - --backup-dir for Path2. Must be a non-overlapping path on the same remote.-- noCleanup - retain working files+// RcHelp returns the rc help+func RcHelp() string {+ return wordwrap.String(bisync.MakeHelp(`This takes the following parameters:+- path1 (required) - (string) a remote directory string e.g. ||drive:path1||+- path2 (required) - (string) a remote directory string e.g. ||drive:path2||+- dryRun - (bool) dry-run mode+`+GenerateParams()+`See [bisync command help](https://rclone.org/commands/rclone_bisync/)and [full bisync description](https://rclone.org/bisync/)-for more information.`)--var longHelp = shortHelp + makeHelp(`--[Bisync](https://rclone.org/bisync/) provides a-bidirectional cloud sync solution in rclone.-It retains the Path1 and Path2 filesystem listings from the prior run.-On each successive run it will:--- list files on Path1 and Path2, and check for changes on each side.- Changes include |New|, |Newer|, |Older|, and |Deleted| files.-- Propagate changes on Path1 to Path2, and vice-versa.+for more information.+`), 76)+}-Bisync is considered an **advanced command**, so use with care.-Make sure you have read and understood the entire [manual](https://rclone.org/bisync)-(especially the [Limitations](https://rclone.org/bisync/#limitations) section)-before using, or data loss can result. Questions can be asked in the-[Rclone Forum](https://forum.rclone.org/).+// example: "create-empty-src-dirs" -> "createEmptySrcDirs"+func toCamel(s string) string {+ split := strings.Split(s, "-")+ builder := strings.Builder{}+ for i, word := range split {+ if i == 0 { // first word always all lowercase+ builder.WriteString(strings.ToLower(word))+ continue+ }+ builder.WriteString(cases.Title(language.AmericanEnglish).String(word))+ }+ return builder.String()+}-See [full bisync description](https://rclone.org/bisync/) for details.`)+// GenerateParams automatically generates the param list from commandDefinition.Flags+func GenerateParams() string {+ builder := strings.Builder{}+ fn := func(flag *pflag.Flag) {+ if flag.Hidden {+ return+ }+ builder.WriteString(fmt.Sprintf("- %s - (%s) %s \n", toCamel(flag.Name), flag.Value.Type(), flag.Usage))+ }+ commandDefinition, _, _ := cmd.Root.Find([]string{"bisync"})+ commandDefinition.Flags().VisitAll(fn)+ return builder.String()+}diff --git a/cmd/bisync/rc.go b/cmd/bisync/rc.goindex 4e50614ae..0cb677910 100644--- a/cmd/bisync/rc.go+++ b/cmd/bisync/rc.go@@ -1,10 +1,14 @@+//go:generate go run help.go rc.mdpackage bisyncimport ("context"+ _ "embed""errors""log""path/filepath"+ "strconv"+ "strings""github.com/rclone/rclone/cmd/bisync/bilib""github.com/rclone/rclone/fs"@@ -12,7 +16,7 @@ import ("github.com/rclone/rclone/fs/rc")-func init() {+func addRC() {rc.Add(rc.Call{Path: "sync/bisync",AuthRequired: true,@@ -22,6 +26,42 @@ func init() {})}+//go:embed rc.md+var rcHelp string++var shortHelp = `Perform bidirectional synchronization between two paths.`++var longHelp = shortHelp + MakeHelp(`++[Bisync](https://rclone.org/bisync/) provides a+bidirectional cloud sync solution in rclone.+It retains the Path1 and Path2 filesystem listings from the prior run.+On each successive run it will:++- list files on Path1 and Path2, and check for changes on each side.+ Changes include ||New||, ||Newer||, ||Older||, and ||Deleted|| files.+- Propagate changes on Path1 to Path2, and vice-versa.++Bisync is considered an **advanced command**, so use with care.+Make sure you have read and understood the entire [manual](https://rclone.org/bisync)+(especially the [Limitations](https://rclone.org/bisync/#limitations) section)+before using, or data loss can result. Questions can be asked in the+[Rclone Forum](https://forum.rclone.org/).++See [full bisync description](https://rclone.org/bisync/) for details.+`)++// MakeHelp replaces some dynamic variables for the help docs+func MakeHelp(help string) string {+ replacer := strings.NewReplacer(+ "||", "`",+ "{MAXDELETE}", strconv.Itoa(DefaultMaxDelete),+ "{CHECKFILE}", DefaultCheckFilename,+ "{WORKDIR}", DefaultWorkdir,+ )+ return replacer.Replace(help)+}+func rcBisync(ctx context.Context, in rc.Params) (out rc.Params, err error) {opt := &Options{}octx, ci := fs.AddConfig(ctx)diff --git a/cmd/bisync/rc.md b/cmd/bisync/rc.mdnew file mode 100644index 000000000..271d816ea--- /dev/null+++ b/cmd/bisync/rc.md@@ -0,0 +1,61 @@+<!--- Docs generated by help.go - use go generate to rebuild - DO NOT EDIT --->++This takes the following parameters:++- path1 (required) - (string) a remote directory string e.g. `drive:path1`+- path2 (required) - (string) a remote directory string e.g. `drive:path2`+- dryRun - (bool) dry-run mode+- backupDir1 - (string) --backup-dir for Path1. Must be a non-overlapping path on+the same remote.+- backupDir2 - (string) --backup-dir for Path2. Must be a non-overlapping path on+the same remote.+- checkAccess - (bool) Ensure expected RCLONE_TEST files are found on both+Path1 and Path2 filesystems, else abort.+- checkFilename - (string) Filename for --check-access (default: RCLONE_TEST)+- checkSync - (string) Controls comparison of final listings: true|false|only+(default: true)+- compare - (string) Comma-separated list of bisync-specific compare options ex.+'size,modtime,checksum' (default: 'size,modtime')+- conflictLoser - (ConflictLoserAction) Action to take on the loser of a sync+conflict (when there is a winner) or on both files (when there is no+winner): , num, pathname, delete (default: num)+- conflictResolve - (string) Automatically resolve conflicts by preferring the+version that is: none, path1, path2, newer, older, larger, smaller (default:+none)+- conflictSuffix - (string) Suffix to use when renaming a --conflict-loser. Can+be either one string or two comma-separated strings to assign different+suffixes to Path1/Path2. (default: 'conflict')+- createEmptySrcDirs - (bool) Sync creation and deletion of empty directories.+(Not compatible with --remove-empty-dirs)+- downloadHash - (bool) Compute hash by downloading when otherwise+unavailable. (warning: may be slow and use lots of data!)+- filtersFile - (string) Read filtering patterns from a file+- force - (bool) Bypass --max-delete safety check and run the sync. Consider+using with --verbose+- ignoreListingChecksum - (bool) Do not use checksums for listings (add --ignore-+checksum to additionally skip post-copy checksum checks)+- maxLock - (Duration) Consider lock files older than this to be expired+(default: 0 (never expire)) (minimum: 2m)+- noCleanup - (bool) Retain working files (useful for troubleshooting and+testing).+- noSlowHash - (bool) Ignore listing checksums only on backends where they are+slow+- recover - (bool) Automatically recover from interruptions without requiring --+resync.+- removeEmptyDirs - (bool) Remove ALL empty directories at the final cleanup+step.+- resilient - (bool) Allow future runs to retry after certain less-serious+errors, instead of requiring --resync.+- resync - (bool) Performs the resync run. Equivalent to --resync-mode path1.+Consider using --verbose or --dry-run first.+- resyncMode - (string) During resync, prefer the version that is: path1,+path2, newer, older, larger, smaller (default: path1 if --resync, otherwise+none for no resync.)+- slowHashSyncOnly - (bool) Ignore slow checksums for listings and deltas, but+still consider them during sync calls.+- workdir - (string) Use custom working dir - useful for testing. (default:+~/.cache/rclone/bisync)++See [bisync command help](https://rclone.org/commands/rclone_bisync/)+and [full bisync description](https://rclone.org/bisync/)+for more information.diff --git a/go.mod b/go.modindex 5cbd60bf2..61b5edd6b 100644--- a/go.mod+++ b/go.mod@@ -60,6 +60,7 @@ require (github.com/minio/minio-go/v7 v7.0.98github.com/mitchellh/go-homedir v1.1.0github.com/moby/sys/mountinfo v0.7.2+ github.com/muesli/reflow v0.3.0github.com/ncw/swift/v2 v2.0.5github.com/oracle/oci-go-sdk/v65 v65.108.2github.com/patrickmn/go-cache v2.1.0+incompatiblediff --git a/go.sum b/go.sumindex e969577c3..0515fa7e4 100644--- a/go.sum+++ b/go.sum@@ -499,6 +499,7 @@ github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stggithub.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=+github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=github.com/mattn/go-runewidth v0.0.20 h1:WcT52H91ZUAwy8+HUkdM3THM6gXqXuLJi9O3rjcQQaQ=github.com/mattn/go-runewidth v0.0.20/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=github.com/mholt/archives v0.1.5 h1:Fh2hl1j7VEhc6DZs2DLMgiBNChUux154a1G+2esNvzQ=@@ -523,6 +524,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/wgithub.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=+github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s=+github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8=github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=github.com/ncw/swift/v2 v2.0.5 h1:9o5Gsd7bInAFEqsGPcaUdsboMbqf8lnNtxqWKFT9iz8=@@ -598,6 +601,8 @@ github.com/relvacode/iso8601 v1.7.0 h1:BXy+V60stMP6cpswc+a93Mq3e65PfXCgDFfhvNNGrgithub.com/relvacode/iso8601 v1.7.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I=github.com/rfjakob/eme v1.2.0 h1:8dAHL+WVAw06+7DkRKnRiFp1JL3QjcJEZFqDnndUaSI=github.com/rfjakob/eme v1.2.0/go.mod h1:cVvpasglm/G3ngEfcfT/Wt0GwhkuO32pf/poW6Nyk1k=+github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=