Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

bisync: normalize session name to non-canonical - fixes #7423

nielash199d82969b45
diff --git a/cmd/bisync/bilib/canonical.go b/cmd/bisync/bilib/canonical.go
index be99d7f2d..dac147670 100644
--- a/cmd/bisync/bilib/canonical.go
+++ b/cmd/bisync/bilib/canonical.go
@@ -2,12 +2,15 @@
package bilib
import (
+ "context"
"os"
+ "path/filepath"
"regexp"
"runtime"
"strings"
"github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/operations"
)
// FsPath converts Fs to a suitable rclone argument
@@ -38,5 +41,57 @@ var nonCanonicalChars = regexp.MustCompile(`[\s\\/:?*]`)
// SessionName makes a unique base name for the sync operation
func SessionName(fs1, fs2 fs.Fs) string {
- return CanonicalPath(FsPath(fs1)) + ".." + CanonicalPath(FsPath(fs2))
+ return StripHexString(CanonicalPath(FsPath(fs1))) + ".." + StripHexString(CanonicalPath(FsPath(fs2)))
+}
+
+// StripHexString strips the (first) canonical {hexstring} suffix
+func StripHexString(path string) string {
+ open := strings.IndexRune(path, '{')
+ close := strings.IndexRune(path, '}')
+ if open >= 0 && close > open {
+ return path[:open] + path[close+1:] // (trailing underscore)
+ }
+ return path
+}
+
+// HasHexString returns true if path contains at least one canonical {hexstring} suffix
+func HasHexString(path string) bool {
+ open := strings.IndexRune(path, '{')
+ if open >= 0 && strings.IndexRune(path, '}') > open {
+ return true
+ }
+ return false
+}
+
+// BasePath joins the workDir with the SessionName, stripping {hexstring} suffix if necessary
+func BasePath(ctx context.Context, workDir string, fs1, fs2 fs.Fs) string {
+ suffixedSession := CanonicalPath(FsPath(fs1)) + ".." + CanonicalPath(FsPath(fs2))
+ suffixedBasePath := filepath.Join(workDir, suffixedSession)
+ listing1 := suffixedBasePath + ".path1.lst"
+ listing2 := suffixedBasePath + ".path2.lst"
+
+ sessionName := SessionName(fs1, fs2)
+ basePath := filepath.Join(workDir, sessionName)
+
+ // Normalize to non-canonical version for overridden configs
+ // to ensure that backend-specific flags don't change the listing filename.
+ // For backward-compatibility, we first check if we found a listing file with the suffixed version.
+ // If so, we rename it (and overwrite non-suffixed version, if any.)
+ // If not, we carry on with the non-suffixed version.
+ // We should only find a suffixed version if bisync v1.66 or older created it.
+ if HasHexString(suffixedSession) && FileExists(listing1) {
+ fs.Infof(listing1, "renaming to: %s", basePath+".path1.lst")
+ if !operations.SkipDestructive(ctx, listing1, "rename to "+basePath+".path1.lst") {
+ _ = os.Rename(listing1, basePath+".path1.lst")
+ }
+ }
+ if HasHexString(suffixedSession) && FileExists(listing2) {
+ fs.Infof(listing2, "renaming to: %s", basePath+".path2.lst")
+ if !operations.SkipDestructive(ctx, listing1, "rename to "+basePath+".path2.lst") {
+ _ = os.Rename(listing2, basePath+".path2.lst")
+ } else {
+ return suffixedBasePath
+ }
+ }
+ return basePath
}
diff --git a/cmd/bisync/operations.go b/cmd/bisync/operations.go
index d845b4826..e7c56c2f3 100644
--- a/cmd/bisync/operations.go
+++ b/cmd/bisync/operations.go
@@ -88,7 +88,7 @@ func Bisync(ctx context.Context, fs1, fs2 fs.Fs, optArg *Options) (err error) {
}
// Produce a unique name for the sync operation
- b.basePath = filepath.Join(b.workDir, bilib.SessionName(b.fs1, b.fs2))
+ b.basePath = bilib.BasePath(ctx, b.workDir, b.fs1, b.fs2)
b.listing1 = b.basePath + ".path1.lst"
b.listing2 = b.basePath + ".path2.lst"
b.newListing1 = b.listing1 + "-new"
diff --git a/docs/content/bisync.md b/docs/content/bisync.md
index c4758a4e5..30d45b884 100644
--- a/docs/content/bisync.md
+++ b/docs/content/bisync.md
@@ -614,19 +614,6 @@ and there is also a [known issue concerning Google Drive users with many empty d
For now, the recommended way to avoid using `--fast-list` is to add `--disable ListR`
to all bisync commands. The default behavior may change in a future version.
-### Overridden Configs
-
-When rclone detects an overridden config, it adds a suffix like `{ABCDE}` on the fly
-to the internal name of the remote. Bisync follows suit by including this suffix in its listing filenames.
-However, this suffix does not necessarily persist from run to run, especially if different flags are provided.
-So if next time the suffix assigned is `{FGHIJ}`, bisync will get confused,
-because it's looking for a listing file with `{FGHIJ}`, when the file it wants has `{ABCDE}`.
-As a result, it throws
-`Bisync critical error: cannot find prior Path1 or Path2 listings, likely due to critical error on prior run`
-and refuses to run again until the user runs a `--resync` (unless using `--resilient`).
-The best workaround at the moment is to set any backend-specific flags in the [config file](/commands/rclone_config/)
-instead of specifying them with command flags. (You can still override them as needed for other rclone commands.)
-
### Case (and unicode) sensitivity {#case-sensitivity}
As of `v1.66`, case and unicode form differences no longer cause critical errors,
@@ -974,10 +961,6 @@ skip them.) To work around this, use the default (modtime and size) instead of
To ignore Google Docs entirely, use
[`--drive-skip-gdocs`](/drive/#drive-skip-gdocs).
-(Note that all flags starting with `--drive` are backend-specific, and
-therefore will cause the behavior explained in [Overridden
-Configs](/#overridden-configs).)
-
## Usage examples
### Cron {#cron}
@@ -1369,6 +1352,7 @@ for performance improvements and less [risk of error](https://forum.rclone.org/t
* Google Docs (and other files of unknown size) are now supported (with the same options as in `sync`)
* Equality checks before a sync conflict rename now fall back to `cryptcheck` (when possible) or `--download`,
instead of of `--size-only`, when `check` is not available.
+* Bisync no longer fails to find the correct listing file when configs are overridden with backend-specific flags.
### `v1.64`
* Fixed an [issue](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=1.%20Dry%20runs%20are%20not%20completely%20dry)