"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
sync: --fix-case flag to rename case insensitive dest - fixes #4854
nielash11afc3dde072
diff --git a/docs/content/docs.md b/docs/content/docs.mdindex ee3f7e4d7..821d6f4b0 100644--- a/docs/content/docs.md+++ b/docs/content/docs.md@@ -1107,6 +1107,26 @@ triggering follow-on actions if data was copied, or skipping if not.NB: Enabling this option turns a usually non-fatal error into a potentiallyfatal one - please check and adjust your scripts accordingly!+### --fix-case ###++Normally, a sync to a case insensitive dest (such as macOS / Windows) will+not result in a matching filename if the source and dest filenames have+casing differences but are otherwise identical. For example, syncing `hello.txt`+to `HELLO.txt` will normally result in the dest filename remaining `HELLO.txt`.+If `--fix-case` is set, then `HELLO.txt` will be renamed to `hello.txt`+to match the source.++NB:+- directory names with incorrect casing will also be fixed+- `--fix-case` will be ignored if `--immutable` is set+- using `--local-case-sensitive` instead is not advisable;+it will cause `HELLO.txt` to get deleted!+- the old dest filename must not be excluded by filters.+Be especially careful with [`--files-from`](/filtering/#files-from-read-list-of-source-file-names),+which does not respect [`--ignore-case`](/filtering/#ignore-case-make-searches-case-insensitive)!+- on remotes that do not support server-side move, `--fix-case` will require+downloading the file and re-uploading it. To avoid this, do not use `--fix-case`.+### --fs-cache-expire-duration=TIMEWhen using rclone via the API rclone caches created remotes for 5diff --git a/fs/config.go b/fs/config.goindex d405f55bb..5b1e483dc 100644--- a/fs/config.go+++ b/fs/config.go@@ -81,6 +81,7 @@ type ConfigInfo struct {IgnoreSize boolIgnoreChecksum boolIgnoreCaseSync bool+ FixCase boolNoTraverse boolCheckFirst boolNoCheckDest booldiff --git a/fs/config/configflags/configflags.go b/fs/config/configflags/configflags.goindex 7d856cf12..232af61d5 100644--- a/fs/config/configflags/configflags.go+++ b/fs/config/configflags/configflags.go@@ -83,6 +83,7 @@ func AddFlags(ci *fs.ConfigInfo, flagSet *pflag.FlagSet) {flags.BoolVarP(flagSet, &ci.IgnoreSize, "ignore-size", "", false, "Ignore size when skipping use modtime or checksum", "Copy")flags.BoolVarP(flagSet, &ci.IgnoreChecksum, "ignore-checksum", "", ci.IgnoreChecksum, "Skip post copy check of checksums", "Copy")flags.BoolVarP(flagSet, &ci.IgnoreCaseSync, "ignore-case-sync", "", ci.IgnoreCaseSync, "Ignore case when synchronizing", "Copy")+ flags.BoolVarP(flagSet, &ci.FixCase, "fix-case", "", ci.FixCase, "Force rename of case insensitive dest to match source", "Sync")flags.BoolVarP(flagSet, &ci.NoTraverse, "no-traverse", "", ci.NoTraverse, "Don't traverse destination file system on copy", "Copy")flags.BoolVarP(flagSet, &ci.CheckFirst, "check-first", "", ci.CheckFirst, "Do all the checks before starting transfers", "Copy")flags.BoolVarP(flagSet, &ci.NoCheckDest, "no-check-dest", "", ci.NoCheckDest, "Don't check the destination, copy regardless", "Copy")diff --git a/fs/sync/sync.go b/fs/sync/sync.goindex 9154171c5..39dfb3818 100644--- a/fs/sync/sync.go+++ b/fs/sync/sync.go@@ -6,6 +6,7 @@ import ("errors""fmt""path"+ "path/filepath""sort""strings""sync"@@ -18,6 +19,7 @@ import ("github.com/rclone/rclone/fs/hash""github.com/rclone/rclone/fs/march""github.com/rclone/rclone/fs/operations"+ "github.com/rclone/rclone/lib/random")// ErrorMaxDurationReached defines error when transfer duration is reached@@ -363,6 +365,16 @@ func (s *syncCopyMove) pairChecker(in *pipe, out *pipe, fraction int, wg *sync.WneedTransfer = false}}+ // Fix case for case insensitive filesystems+ if s.ci.FixCase && !s.ci.Immutable && src.Remote() != pair.Dst.Remote() {+ if newDst, err := operations.Move(s.ctx, s.fdst, nil, src.Remote(), pair.Dst); err != nil {+ fs.Errorf(pair.Dst, "Error while attempting to rename to %s: %v", src.Remote(), err)+ s.processError(err)+ } else {+ fs.Infof(pair.Dst, "Fixed case by renaming to: %s", src.Remote())+ pair.Dst = newDst+ }+ }if needTransfer {// If files are treated as immutable, fail if destination exists and does not matchif s.ci.Immutable && pair.Dst != nil {@@ -1136,6 +1148,30 @@ func (s *syncCopyMove) Match(ctx context.Context, dst, src fs.DirEntry) (recurses.srcEmptyDirs[src.Remote()] = srcs.srcEmptyDirsMu.Unlock()}+ if s.ci.FixCase && !s.ci.Immutable && src.Remote() != dst.Remote() {+ // Fix case for case insensitive filesystems+ // Fix each dir before recursing into subdirs and files+ oldDirFs, err := fs.NewFs(s.ctx, filepath.Join(s.fdst.Root(), dst.Remote()))+ s.processError(err)+ newDirPath := filepath.Join(s.fdst.Root(), filepath.Dir(dst.Remote()), filepath.Base(src.Remote()))+ newDirFs, err := fs.NewFs(s.ctx, newDirPath)+ s.processError(err)+ // Create random name to temporarily move dir to+ tmpDirName := newDirPath + "-rclone-move-" + random.String(8)+ tmpDirFs, err := fs.NewFs(s.ctx, tmpDirName)+ s.processError(err)+ if err = MoveDir(s.ctx, tmpDirFs, oldDirFs, s.deleteEmptySrcDirs, s.copyEmptySrcDirs); err != nil {+ fs.Errorf(dst, "Error while attempting to move dir to temporary location %s: %v", tmpDirName, err)+ s.processError(err)+ } else {+ if err = MoveDir(s.ctx, newDirFs, tmpDirFs, s.deleteEmptySrcDirs, s.copyEmptySrcDirs); err != nil {+ fs.Errorf(dst, "Error while attempting to rename to %s: %v", src.Remote(), err)+ s.processError(err)+ } else {+ fs.Infof(dst, "Fixed case by renaming to: %s", src.Remote())+ }+ }+ }return true}diff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.goindex 4438477f4..34085ee6e 100644--- a/fs/sync/sync_test.go+++ b/fs/sync/sync_test.go@@ -2197,6 +2197,39 @@ func TestSyncIgnoreCase(t *testing.T) {r.CheckRemoteItems(t, file2)}+// Test --fix-case+func TestFixCase(t *testing.T) {+ ctx := context.Background()+ ctx, ci := fs.AddConfig(ctx)+ r := fstest.NewRun(t)++ // Only test if remote is case insensitive+ if !r.Fremote.Features().CaseInsensitive {+ t.Skip("Skipping test as local or remote are case-sensitive")+ }++ ci.FixCase = true++ // Create files with different filename casing+ file1a := r.WriteFile("existing", "potato", t1)+ file1b := r.WriteFile("existingbutdifferent", "donut", t1)+ file1c := r.WriteFile("subdira/subdirb/subdirc/hello", "donut", t1)+ file1d := r.WriteFile("subdira/subdirb/subdirc/subdird/filewithoutcasedifferences", "donut", t1)+ r.CheckLocalItems(t, file1a, file1b, file1c, file1d)+ file2a := r.WriteObject(ctx, "EXISTING", "potato", t1)+ file2b := r.WriteObject(ctx, "EXISTINGBUTDIFFERENT", "lemonade", t1)+ file2c := r.WriteObject(ctx, "SUBDIRA/subdirb/SUBDIRC/HELLO", "lemonade", t1)+ file2d := r.WriteObject(ctx, "SUBDIRA/subdirb/SUBDIRC/subdird/filewithoutcasedifferences", "lemonade", t1)+ r.CheckRemoteItems(t, file2a, file2b, file2c, file2d)++ // Should force rename of dest file that is differently-cased+ accounting.GlobalStats().ResetCounters()+ err := Sync(ctx, r.Fremote, r.Flocal, false)+ require.NoError(t, err)+ r.CheckLocalItems(t, file1a, file1b, file1c, file1d)+ r.CheckRemoteItems(t, file1a, file1b, file1c, file1d)+}+// Test that aborting on --max-transfer worksfunc TestMaxTransfer(t *testing.T) {ctx := context.Background()