"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
bisync: handle unicode and case normalization consistently - mostly-fixes #7270
nielashf7f465182832
diff --git a/cmd/bisync/bisync_test.go b/cmd/bisync/bisync_test.goindex 11a0862c3..185981a2c 100644--- a/cmd/bisync/bisync_test.go+++ b/cmd/bisync/bisync_test.go@@ -98,10 +98,13 @@ var logHoppers = []string{// subdirectories. The order inconsistency initially showed up in the// listings and triggered reordering of log messages, but the actual// files will in fact match.- `ERROR : - +Access test failed: Path[12] file not found in Path[12] - .*`,+ `.* +.....Access test failed: Path[12] file not found in Path[12].*`,// Test case `resync` suffered from the order of queued copies.`(?:INFO |NOTICE): - Path2 Resync will copy to Path1 +- .*`,++ // Test case `normalization` can have random order of fix-case files.+ `(?:INFO |NOTICE): .*: Fixed case by renaming to: .*`,}// Some log lines can contain Windows path separator that must be@@ -546,6 +549,16 @@ func (b *bisyncTest) runTestStep(ctx context.Context, line string) (err error) {case "copy-as":b.checkArgs(args, 3, 3)return b.copyFile(ctx, args[1], args[2], args[3])+ case "copy-as-NFC":+ b.checkArgs(args, 3, 3)+ ci.NoUnicodeNormalization = true+ ci.FixCase = true+ return b.copyFile(ctx, args[1], norm.NFC.String(args[2]), norm.NFC.String(args[3]))+ case "copy-as-NFD":+ b.checkArgs(args, 3, 3)+ ci.NoUnicodeNormalization = true+ ci.FixCase = true+ return b.copyFile(ctx, args[1], norm.NFD.String(args[2]), norm.NFD.String(args[3]))case "copy-dir", "sync-dir":b.checkArgs(args, 2, 2)if fsrc, err = cache.Get(ctx, args[1]); err != nil {@@ -565,6 +578,9 @@ func (b *bisyncTest) runTestStep(ctx context.Context, line string) (err error) {b.checkArgs(args, 1, 1)return b.listSubdirs(ctx, args[1])case "bisync":+ ci.NoUnicodeNormalization = false+ ci.IgnoreCaseSync = false+ // ci.FixCase = truereturn b.runBisync(ctx, args[1:])case "test-func":b.TestFn = testFunc@@ -668,6 +684,16 @@ func (b *bisyncTest) runBisync(ctx context.Context, args []string) (err error) {fs2 = addSubdir(b.path2, val)case "ignore-listing-checksum":opt.IgnoreListingChecksum = true+ case "no-norm":+ ci.NoUnicodeNormalization = true+ ci.IgnoreCaseSync = false+ case "norm":+ ci.NoUnicodeNormalization = false+ ci.IgnoreCaseSync = true+ case "fix-case":+ ci.NoUnicodeNormalization = false+ ci.IgnoreCaseSync = true+ ci.FixCase = truedefault:return fmt.Errorf("invalid bisync option %q", arg)}diff --git a/cmd/bisync/deltas.go b/cmd/bisync/deltas.goindex 3167dfb00..d0c0cd8aa 100644--- a/cmd/bisync/deltas.go+++ b/cmd/bisync/deltas.go@@ -381,11 +381,17 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change}}+ // find alternate names according to normalization settings+ altNames1to2 := bilib.Names{}+ altNames2to1 := bilib.Names{}+ b.findAltNames(ctx, b.fs1, copy2to1, b.newListing1, altNames2to1)+ b.findAltNames(ctx, b.fs2, copy1to2, b.newListing2, altNames1to2)+// Do the batch operationif copy2to1.NotEmpty() {changes1 = trueb.indent("Path2", "Path1", "Do queued copies to")- results2to1, err = b.fastCopy(ctx, b.fs2, b.fs1, copy2to1, "copy2to1")+ results2to1, err = b.fastCopy(ctx, b.fs2, b.fs1, copy2to1, "copy2to1", altNames2to1)if err != nil {return}@@ -397,7 +403,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (changeif copy1to2.NotEmpty() {changes2 = trueb.indent("Path1", "Path2", "Do queued copies to")- results1to2, err = b.fastCopy(ctx, b.fs1, b.fs2, copy1to2, "copy1to2")+ results1to2, err = b.fastCopy(ctx, b.fs1, b.fs2, copy1to2, "copy1to2", altNames1to2)if err != nil {return}diff --git a/cmd/bisync/listing.go b/cmd/bisync/listing.goindex 7875036fa..f94883d97 100644--- a/cmd/bisync/listing.go+++ b/cmd/bisync/listing.go@@ -20,6 +20,7 @@ import ("github.com/rclone/rclone/fs/hash""github.com/rclone/rclone/fs/operations""golang.org/x/exp/slices"+ "golang.org/x/text/unicode/norm")// ListingHeader defines first line of a listing@@ -400,6 +401,18 @@ func ConvertPrecision(Modtime time.Time, dst fs.Fs) time.Time {return Modtime}+// ApplyTransforms handles unicode and case normalization+func ApplyTransforms(ctx context.Context, dst fs.Fs, s string) string {+ ci := fs.GetConfig(ctx)+ if !ci.NoUnicodeNormalization {+ s = norm.NFC.String(s)+ }+ if ci.IgnoreCaseSync || dst.Features().CaseInsensitive {+ s = strings.ToLower(s)+ }+ return s+}+// modifyListing will modify the listing based on the results of the syncfunc (b *bisyncRun) modifyListing(ctx context.Context, src fs.Fs, dst fs.Fs, results []Results, queues queues, is1to2 bool) (err error) {queue := queues.copy2to1@@ -429,6 +442,10 @@ func (b *bisyncRun) modifyListing(ctx context.Context, src fs.Fs, dst fs.Fs, ressrcList.hash = src.Hashes().GetOne()dstList.hash = dst.Hashes().GetOne()}+ dstListNew, err := b.loadListing(dstListing + "-new")+ if err != nil {+ return fmt.Errorf("cannot read new listing: %w", err)+ }srcWinners := newFileList()dstWinners := newFileList()@@ -466,20 +483,47 @@ func (b *bisyncRun) modifyListing(ctx context.Context, src fs.Fs, dst fs.Fs, res}updateLists := func(side string, winners, list *fileList) {- // removals from side- for _, oldFile := range queue.ToList() {- if !winners.has(oldFile) && list.has(oldFile) && !errors.has(oldFile) {- list.remove(oldFile)- fs.Debugf(nil, "decision: removed from %s: %v", side, oldFile)- } else if winners.has(oldFile) {+ for _, queueFile := range queue.ToList() {+ if !winners.has(queueFile) && list.has(queueFile) && !errors.has(queueFile) {+ // removals from side+ list.remove(queueFile)+ fs.Debugf(nil, "decision: removed from %s: %v", side, queueFile)+ } else if winners.has(queueFile) {// copies to side- new := winners.get(oldFile)- list.put(oldFile, new.size, new.time, new.hash, new.id, new.flags)- fs.Debugf(nil, "decision: copied to %s: %v", side, oldFile)+ new := winners.get(queueFile)++ // handle normalization according to settings+ ci := fs.GetConfig(ctx)+ if side == "dst" && (!ci.NoUnicodeNormalization || ci.IgnoreCaseSync || dst.Features().CaseInsensitive) {+ // search list for existing file that matches queueFile when normalized+ normalizedName := ApplyTransforms(ctx, dst, queueFile)+ matchFound := false+ matchedName := ""+ for _, filename := range dstListNew.list {+ if ApplyTransforms(ctx, dst, filename) == normalizedName {+ matchFound = true+ matchedName = filename // original, not normalized+ break+ }+ }+ if matchFound && matchedName != queueFile {+ // use the (non-identical) existing name, unless --fix-case+ if ci.FixCase {+ fs.Debugf(direction, "removing %s and adding %s as --fix-case was specified", matchedName, queueFile)+ list.remove(matchedName)+ } else {+ fs.Debugf(direction, "casing/unicode difference detected. using %s instead of %s", matchedName, queueFile)+ queueFile = matchedName+ }+ }+ }++ list.put(queueFile, new.size, new.time, new.hash, new.id, new.flags)+ fs.Debugf(nil, "decision: copied to %s: %v", side, queueFile)} else {- fs.Debugf(oldFile, "file in queue but missing from %s transfers", side)- if err := filterRecheck.AddFile(oldFile); err != nil {- fs.Debugf(oldFile, "error adding file to recheck filter: %v", err)+ fs.Debugf(queueFile, "file in queue but missing from %s transfers", side)+ if err := filterRecheck.AddFile(queueFile); err != nil {+ fs.Debugf(queueFile, "error adding file to recheck filter: %v", err)}}}diff --git a/cmd/bisync/march.go b/cmd/bisync/march.goindex 229f3ed63..63536063b 100644--- a/cmd/bisync/march.go+++ b/cmd/bisync/march.go@@ -183,7 +183,3 @@ func whichPath(isPath1 bool) string {}return s}--// TODO:-// equality check?-// unicode stuffdiff --git a/cmd/bisync/operations.go b/cmd/bisync/operations.goindex 67cb5b440..fce29f4d0 100644--- a/cmd/bisync/operations.go+++ b/cmd/bisync/operations.go@@ -428,8 +428,11 @@ func (b *bisyncRun) resync(octx, fctx context.Context) error {if len(copy2to1) > 0 {b.indent("Path2", "Path1", "Resync is doing queued copies to")+ resync2to1 := bilib.ToNames(copy2to1)+ altNames2to1 := bilib.Names{}+ b.findAltNames(octx, b.fs1, resync2to1, b.newListing1, altNames2to1)// octx does not have extra filters!- results2to1, err = b.fastCopy(octx, b.fs2, b.fs1, bilib.ToNames(copy2to1), "resync-copy2to1")+ results2to1, err = b.fastCopy(octx, b.fs2, b.fs1, resync2to1, "resync-copy2to1", altNames2to1)if err != nil {b.critical = truereturn err@@ -516,15 +519,27 @@ func (b *bisyncRun) checkSync(listing1, listing2 string) error {return fmt.Errorf("cannot read prior listing of Path2: %w", err)}+ transformList := func(files *fileList, fs fs.Fs) *fileList {+ transformed := newFileList()+ for _, file := range files.list {+ f := files.get(file)+ transformed.put(ApplyTransforms(context.Background(), fs, file), f.size, f.time, f.hash, f.id, f.flags)+ }+ return transformed+ }++ files1Transformed := transformList(files1, b.fs1)+ files2Transformed := transformList(files2, b.fs2)+ok := truefor _, file := range files1.list {- if !files2.has(file) {+ if !files2.has(file) && !files2Transformed.has(ApplyTransforms(context.Background(), b.fs1, file)) {b.indent("ERROR", file, "Path1 file not found in Path2")ok = false}}for _, file := range files2.list {- if !files1.has(file) {+ if !files1.has(file) && !files1Transformed.has(ApplyTransforms(context.Background(), b.fs2, file)) {b.indent("ERROR", file, "Path2 file not found in Path1")ok = false}diff --git a/cmd/bisync/queue.go b/cmd/bisync/queue.goindex d670c033d..d16f06fcc 100644--- a/cmd/bisync/queue.go+++ b/cmd/bisync/queue.go@@ -14,7 +14,6 @@ import ("github.com/rclone/rclone/fs/filter""github.com/rclone/rclone/fs/operations""github.com/rclone/rclone/fs/sync"- "golang.org/x/text/unicode/norm")// Results represents a pair of synced files, as reported by the LoggerFn@@ -130,7 +129,7 @@ func ReadResults(results io.Reader) []Results {return slice}-func (b *bisyncRun) fastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib.Names, queueName string) ([]Results, error) {+func (b *bisyncRun) fastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib.Names, queueName string, altNames bilib.Names) ([]Results, error) {if err := b.saveQueue(files, queueName); err != nil {return nil, err}@@ -140,9 +139,12 @@ func (b *bisyncRun) fastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib.if err := filterCopy.AddFile(file); err != nil {return nil, err}- // macOS- if err := filterCopy.AddFile(norm.NFD.String(file)); err != nil {- return nil, err+ }+ if altNames.NotEmpty() {+ for _, file := range altNames.ToList() {+ if err := filterCopy.AddFile(file); err != nil {+ return nil, err+ }}}@@ -252,3 +254,22 @@ func (b *bisyncRun) saveQueue(files bilib.Names, jobName string) error {queueFile := fmt.Sprintf("%s.%s.que", b.basePath, jobName)return files.Save(queueFile)}++func (b *bisyncRun) findAltNames(ctx context.Context, dst fs.Fs, queue bilib.Names, newListing string, altNames bilib.Names) {+ ci := fs.GetConfig(ctx)+ if queue.NotEmpty() && (!ci.NoUnicodeNormalization || ci.IgnoreCaseSync || b.fs1.Features().CaseInsensitive || b.fs2.Features().CaseInsensitive) {+ // search list for existing file that matches queueFile when normalized+ for _, queueFile := range queue.ToList() {+ normalizedName := ApplyTransforms(ctx, dst, queueFile)+ candidates, err := b.loadListing(newListing)+ if err != nil {+ fs.Errorf(candidates, "cannot read new listing: %v", err)+ }+ for _, filename := range candidates.list {+ if ApplyTransforms(ctx, dst, filename) == normalizedName && filename != queueFile {+ altNames.Add(filename) // original, not normalized+ }+ }+ }+ }+}diff --git a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lstindex 7471a27cf..f0208cb9a 100644--- a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst+++ b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst@@ -1,10 +1,9 @@# bisync listing v1 from test-- 6148 md5:23b446fda9938c607142c5133cf90689 - 2000-01-01T00:00:00.000000000+0000 ".DS_Store"-- 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt"-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt"-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt"-- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "file1.txt"-- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "subdir/file20.txt"+- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt"+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt"+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt"+- 19 - - 2001-01-02T00:00:00.000000000+0000 "file1.txt"+- 19 - - 2001-01-02T00:00:00.000000000+0000 "subdir/file20.txt"diff --git a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst-new b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst-newindex f243ade04..465d2455d 100644--- a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst-new+++ b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst-new@@ -1,5 +1,4 @@# bisync listing v1 from test-- 6148 md5:23b446fda9938c607142c5133cf90689 - 2000-01-01T00:00:00.000000000+0000 ".DS_Store"- 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"diff --git a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst-old b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst-oldindex 991f89b82..7819f9efc 100644--- a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst-old+++ b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst-old@@ -1,10 +1,9 @@# bisync listing v1 from test-- 6148 md5:23b446fda9938c607142c5133cf90689 - 2000-01-01T00:00:00.000000000+0000 ".DS_Store"-- 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt"-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt"-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt"-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.txt"-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "subdir/file20.txt"+- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt"+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt"+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt"+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.txt"+- 0 - - 2000-01-01T00:00:00.000000000+0000 "subdir/file20.txt"diff --git a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lstindex 7471a27cf..f0208cb9a 100644--- a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst+++ b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst@@ -1,10 +1,9 @@# bisync listing v1 from test-- 6148 md5:23b446fda9938c607142c5133cf90689 - 2000-01-01T00:00:00.000000000+0000 ".DS_Store"-- 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt"-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt"-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt"-- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "file1.txt"-- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "subdir/file20.txt"+- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt"+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt"+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt"+- 19 - - 2001-01-02T00:00:00.000000000+0000 "file1.txt"+- 19 - - 2001-01-02T00:00:00.000000000+0000 "subdir/file20.txt"diff --git a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst-new b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst-newindex dac88bd2c..9e8bc8a1a 100644--- a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst-new+++ b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst-new@@ -1,5 +1,4 @@# bisync listing v1 from test-- 6148 md5:23b446fda9938c607142c5133cf90689 - 2000-01-01T00:00:00.000000000+0000 ".DS_Store"- 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"diff --git a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst-old b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst-oldindex 991f89b82..7819f9efc 100644--- a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst-old+++ b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst-old@@ -1,10 +1,9 @@# bisync listing v1 from test-- 6148 md5:23b446fda9938c607142c5133cf90689 - 2000-01-01T00:00:00.000000000+0000 ".DS_Store"-- 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt"-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt"-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt"-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.txt"-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "subdir/file20.txt"+- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt"+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt"+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt"+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.txt"+- 0 - - 2000-01-01T00:00:00.000000000+0000 "subdir/file20.txt"diff --git a/cmd/bisync/testdata/test_check_access_filters/golden/test.log b/cmd/bisync/testdata/test_check_access_filters/golden/test.logindex c12126374..9ade7ee1b 100644--- a/cmd/bisync/testdata/test_check_access_filters/golden/test.log+++ b/cmd/bisync/testdata/test_check_access_filters/golden/test.log@@ -142,8 +142,8 @@ INFO : Path2: 1 changes: 0 new, 0 newer, 0 older, 1 deletedINFO : Checking access healthERROR : Access test failed: Path1 count 3, Path2 count 4 - RCLONE_TESTERROR : - [34m[0m [35mAccess test failed: Path1 file not found in Path2[0m - [36mRCLONE_TEST[0m-ERROR : - [34m[0m [35mAccess test failed: Path2 file not found in Path1[0m - [36msubdir/RCLONE_TEST[0mERROR : - [34m[0m [35mAccess test failed: Path2 file not found in Path1[0m - [36msubdirX/subdirX1/RCLONE_TEST[0m+ERROR : - [34m[0m [35mAccess test failed: Path2 file not found in Path1[0m - [36msubdir/RCLONE_TEST[0mERROR : [31mBisync critical error: check file check failed[0mERROR : [31mBisync aborted. Must run --resync to recover.[0mBisync error: bisync aborteddiff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.copy1to2.que b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.copy1to2.quenew file mode 100644index 000000000..bd7f8e7dd--- /dev/null+++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.copy1to2.que@@ -0,0 +1,3 @@+"folder/HeLlO,wOrLd!.txt"+"folder/éééö.txt"+"測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.copy2to1.que b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.copy2to1.quenew file mode 100644index 000000000..85deec9f9--- /dev/null+++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.copy2to1.que@@ -0,0 +1,3 @@+"file1.txt"+"folder/hello,WORLD!.txt"+"folder/éééö.txt"diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lstnew file mode 100644index 000000000..0cdecd335--- /dev/null+++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst@@ -0,0 +1,5 @@+# bisync listing v1 from test+- 19 - - 2001-01-05T00:00:00.000000000+0000 "file1.txt"+- 19 - - 2001-01-05T00:00:00.000000000+0000 "folder/HeLlO,wOrLd!.txt"+- 19 - - 2001-01-05T00:00:00.000000000+0000 "folder/éééö.txt"+- 19 - - 2001-01-05T00:00:00.000000000+0000 "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst-new b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst-newnew file mode 100644index 000000000..bd44aea5b--- /dev/null+++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst-new@@ -0,0 +1,5 @@+# bisync listing v1 from test+- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-03T00:00:00.000000000+0000 "file1.txt"+- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-05T00:00:00.000000000+0000 "folder/HeLlO,wOrLd!.txt"+- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-05T00:00:00.000000000+0000 "folder/éééö.txt"+- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-05T00:00:00.000000000+0000 "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst-old b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst-oldnew file mode 100644index 000000000..441434161--- /dev/null+++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst-old@@ -0,0 +1,5 @@+# bisync listing v1 from test+- 19 - - 2001-01-03T00:00:00.000000000+0000 "file1.txt"+- 19 - - 2001-01-03T00:00:00.000000000+0000 "folder/HeLlO,wOrLd!.txt"+- 19 - - 2001-01-03T00:00:00.000000000+0000 "folder/éééö.txt"+- 19 - - 2001-01-02T00:00:00.000000000+0000 "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lstnew file mode 100644index 000000000..f7c2c0ea6--- /dev/null+++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst@@ -0,0 +1,5 @@+# bisync listing v1 from test+- 19 - - 2001-01-05T00:00:00.000000000+0000 "file1.txt"+- 19 - - 2001-01-05T00:00:00.000000000+0000 "folder/hello,WORLD!.txt"+- 19 - - 2001-01-05T00:00:00.000000000+0000 "folder/éééö.txt"+- 19 - - 2001-01-05T00:00:00.000000000+0000 "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-new b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-newnew file mode 100644index 000000000..225fe66c1--- /dev/null+++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-new@@ -0,0 +1,5 @@+# bisync listing v1 from test+- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-05T00:00:00.000000000+0000 "file1.txt"+- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-05T00:00:00.000000000+0000 "folder/hello,WORLD!.txt"+- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-05T00:00:00.000000000+0000 "folder/éééö.txt"+- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-old b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-oldnew file mode 100644index 000000000..ab1695601--- /dev/null+++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-old@@ -0,0 +1,5 @@+# bisync listing v1 from test+- 19 - - 2001-01-03T00:00:00.000000000+0000 "file1.txt"+- 19 - - 2001-01-03T00:00:00.000000000+0000 "folder/hello,WORLD!.txt"+- 19 - - 2001-01-03T00:00:00.000000000+0000 "folder/éééö.txt"+- 19 - - 2001-01-02T00:00:00.000000000+0000 "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.resync-copy2to1.que b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.resync-copy2to1.quenew file mode 100644index 000000000..007f0aada--- /dev/null+++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.resync-copy2to1.que@@ -0,0 +1,2 @@+"folder/hello,WORLD!.txt"+"folder/éééö.txt"diff --git a/cmd/bisync/testdata/test_normalization/golden/test.log b/cmd/bisync/testdata/test_normalization/golden/test.lognew file mode 100644index 000000000..accd69d14--- /dev/null+++ b/cmd/bisync/testdata/test_normalization/golden/test.log@@ -0,0 +1,144 @@+[36m(01) :[0m [34mtest normalization[0m+++[36m(02) :[0m [34mtouch-copy 2001-01-02 {datadir/}file1.txt {path2/}[0m+[36m(03) :[0m [34mtest initial bisync[0m+[36m(04) :[0m [34mbisync resync[0m+INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"+INFO : Copying unique Path2 files to Path1+INFO : Resynching Path1 to Path2+INFO : Resync updating listings+INFO : [32mBisync successful[0m+++[36m(05) :[0m [34mcopy-as-NFC {datadir/}file1.txt {path1/}測試_Русский___ě_áñ👸🏼🧝🏾♀️💆🏿♂️🐨🤙🏼🤮🧑🏻🔧🧑🔬éééö 測試_Русский___ě_áñ👸🏼🧝🏾♀️💆🏿♂️🐨🤙🏼🤮🧑🏻🔧🧑🔬éééö.txt[0m+[36m(06) :[0m [34mcopy-as-NFC {datadir/}file1.txt {path1/}folder éééö.txt[0m+[36m(07) :[0m [34mcopy-as-NFC {datadir/}file1.txt {path1/}folder HeLlO,wOrLd!.txt[0m+++[36m(08) :[0m [34mtouch-copy 2001-01-03 {datadir/}file1.txt {path2/}[0m+[36m(09) :[0m [34mcopy-as-NFD {datadir/}file1.txt {path2/}folder éééö.txt[0m+[36m(10) :[0m [34mcopy-as-NFD {datadir/}file1.txt {path2/}folder hello,WORLD!.txt[0m++[36m(11) :[0m [34mtest bisync run with fix-case[0m+[36m(12) :[0m [34mbisync fix-case[0m+INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"+INFO : Building Path1 and Path2 listings+INFO : Path1 checking for diffs+INFO : - [34mPath1[0m [35mFile is new[0m - [36mfolder/HeLlO,wOrLd!.txt[0m+INFO : - [34mPath1[0m [35mFile is new[0m - [36mfolder/éééö.txt[0m+INFO : - [34mPath1[0m [35mFile is new[0m - [36m"測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"[0m+INFO : Path1: 3 changes: 3 new, 0 newer, 0 older, 0 deleted+INFO : Path2 checking for diffs+INFO : - [34mPath2[0m [35mFile is newer[0m - [36mfile1.txt[0m+INFO : - [34mPath2[0m [35mFile is new[0m - [36mfolder/éééö.txt[0m+INFO : - [34mPath2[0m [35mFile is new[0m - [36mfolder/hello,WORLD!.txt[0m+INFO : Path2: 3 changes: 2 new, 1 newer, 0 older, 0 deleted+INFO : Applying changes+INFO : - [34mPath1[0m [35mQueue copy to Path2[0m - [36m{path2/}folder/HeLlO,wOrLd!.txt[0m+INFO : - [34mPath1[0m [35mQueue copy to Path2[0m - [36m{path2/}folder/éééö.txt[0m+INFO : - [34mPath1[0m [35mQueue copy to Path2[0m - [36m"{path2/}測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"[0m+INFO : - [34mPath2[0m [35mQueue copy to Path1[0m - [36m{path1/}file1.txt[0m+INFO : - [34mPath2[0m [35mQueue copy to Path1[0m - [36m{path1/}folder/éééö.txt[0m+INFO : - [34mPath2[0m [35mQueue copy to Path1[0m - [36m{path1/}folder/hello,WORLD!.txt[0m+INFO : - [34mPath2[0m [35mDo queued copies to[0m - [36mPath1[0m+INFO : folder/HeLlO,wOrLd!.txt: Fixed case by renaming to: folder/hello,WORLD!.txt+INFO : folder/éééö.txt: Fixed case by renaming to: folder/éééö.txt+INFO : - [34mPath1[0m [35mDo queued copies to[0m - [36mPath2[0m+INFO : Updating listings+INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"+INFO : [32mBisync successful[0m+++[36m(13) :[0m [34mpurge-children {path1/}[0m+[36m(14) :[0m [34mpurge-children {path2/}[0m+[36m(15) :[0m [34mtouch-copy 2001-01-02 {datadir/}file1.txt {path2/}[0m+[36m(16) :[0m [34mbisync resync[0m+INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"+INFO : Copying unique Path2 files to Path1+INFO : - [34mPath2[0m [35mResync will copy to Path1[0m - [36mfile1.txt[0m+INFO : - [34mPath2[0m [35mResync is doing queued copies to[0m - [36mPath1[0m+INFO : Resynching Path1 to Path2+INFO : Resync updating listings+INFO : [32mBisync successful[0m+++[36m(17) :[0m [34mcopy-as-NFC {datadir/}file1.txt {path1/}測試_Русский___ě_áñ👸🏼🧝🏾♀️💆🏿♂️🐨🤙🏼🤮🧑🏻🔧🧑🔬éééö 測試_Русский___ě_áñ👸🏼🧝🏾♀️💆🏿♂️🐨🤙🏼🤮🧑🏻🔧🧑🔬éééö.txt[0m+[36m(18) :[0m [34mcopy-as-NFC {datadir/}file1.txt {path1/}folder éééö.txt[0m+[36m(19) :[0m [34mcopy-as-NFC {datadir/}file1.txt {path1/}folder HeLlO,wOrLd!.txt[0m+++[36m(20) :[0m [34mtouch-copy 2001-01-03 {datadir/}file1.txt {path2/}[0m+[36m(21) :[0m [34mcopy-as-NFD {datadir/}file1.txt {path2/}folder éééö.txt[0m+[36m(22) :[0m [34mcopy-as-NFD {datadir/}file1.txt {path2/}folder hello,WORLD!.txt[0m++[36m(23) :[0m [34mtest bisync run with normalization[0m+[36m(24) :[0m [34mbisync norm force[0m+INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"+INFO : Building Path1 and Path2 listings+INFO : Path1 checking for diffs+INFO : - [34mPath1[0m [35mFile is new[0m - [36mfolder/HeLlO,wOrLd!.txt[0m+INFO : - [34mPath1[0m [35mFile is new[0m - [36mfolder/éééö.txt[0m+INFO : - [34mPath1[0m [35mFile is new[0m - [36m"測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"[0m+INFO : Path1: 3 changes: 3 new, 0 newer, 0 older, 0 deleted+INFO : Path2 checking for diffs+INFO : - [34mPath2[0m [35mFile is newer[0m - [36mfile1.txt[0m+INFO : - [34mPath2[0m [35mFile is new[0m - [36mfolder/éééö.txt[0m+INFO : - [34mPath2[0m [35mFile is new[0m - [36mfolder/hello,WORLD!.txt[0m+INFO : Path2: 3 changes: 2 new, 1 newer, 0 older, 0 deleted+INFO : Applying changes+INFO : - [34mPath1[0m [35mQueue copy to Path2[0m - [36m{path2/}folder/HeLlO,wOrLd!.txt[0m+INFO : - [34mPath1[0m [35mQueue copy to Path2[0m - [36m{path2/}folder/éééö.txt[0m+INFO : - [34mPath1[0m [35mQueue copy to Path2[0m - [36m"{path2/}測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"[0m+INFO : - [34mPath2[0m [35mQueue copy to Path1[0m - [36m{path1/}file1.txt[0m+INFO : - [34mPath2[0m [35mQueue copy to Path1[0m - [36m{path1/}folder/éééö.txt[0m+INFO : - [34mPath2[0m [35mQueue copy to Path1[0m - [36m{path1/}folder/hello,WORLD!.txt[0m+INFO : - [34mPath2[0m [35mDo queued copies to[0m - [36mPath1[0m+INFO : - [34mPath1[0m [35mDo queued copies to[0m - [36mPath2[0m+INFO : Updating listings+INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"+INFO : [32mBisync successful[0m++[36m(25) :[0m [34mtest resync[0m+[36m(26) :[0m [34mbisync resync norm[0m+INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"+INFO : Copying unique Path2 files to Path1+INFO : - [34mPath2[0m [35mResync will copy to Path1[0m - [36mfolder/éééö.txt[0m+INFO : - [34mPath2[0m [35mResync will copy to Path1[0m - [36mfolder/hello,WORLD!.txt[0m+INFO : - [34mPath2[0m [35mResync is doing queued copies to[0m - [36mPath1[0m+INFO : Resynching Path1 to Path2+INFO : Resync updating listings+INFO : [32mBisync successful[0m++[36m(27) :[0m [34mtest changed on both paths[0m+[36m(28) :[0m [34mtouch-copy 2001-01-05 {datadir/}file1.txt {path2/}[0m+[36m(29) :[0m [34mcopy-as-NFC {datadir/}file1.txt {path1/}測試_Русский___ě_áñ👸🏼🧝🏾♀️💆🏿♂️🐨🤙🏼🤮🧑🏻🔧🧑🔬éééö 測試_Русский___ě_áñ👸🏼🧝🏾♀️💆🏿♂️🐨🤙🏼🤮🧑🏻🔧🧑🔬éééö.txt[0m+[36m(30) :[0m [34mcopy-as-NFC {datadir/}file1.txt {path1/}folder éééö.txt[0m+[36m(31) :[0m [34mcopy-as-NFC {datadir/}file1.txt {path1/}folder HeLlO,wOrLd!.txt[0m+[36m(32) :[0m [34mcopy-as-NFD {datadir/}file1.txt {path2/}folder éééö.txt[0m+[36m(33) :[0m [34mcopy-as-NFD {datadir/}file1.txt {path2/}folder hello,WORLD!.txt[0m+[36m(34) :[0m [34mbisync norm[0m+INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"+INFO : Building Path1 and Path2 listings+INFO : Path1 checking for diffs+INFO : - [34mPath1[0m [35mFile is newer[0m - [36mfolder/HeLlO,wOrLd!.txt[0m+INFO : - [34mPath1[0m [35mFile is newer[0m - [36mfolder/éééö.txt[0m+INFO : - [34mPath1[0m [35mFile is newer[0m - [36m"測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"[0m+INFO : Path1: 3 changes: 0 new, 3 newer, 0 older, 0 deleted+INFO : Path2 checking for diffs+INFO : - [34mPath2[0m [35mFile is newer[0m - [36mfile1.txt[0m+INFO : - [34mPath2[0m [35mFile is newer[0m - [36mfolder/éééö.txt[0m+INFO : - [34mPath2[0m [35mFile is newer[0m - [36mfolder/hello,WORLD!.txt[0m+INFO : Path2: 3 changes: 0 new, 3 newer, 0 older, 0 deleted+INFO : Applying changes+INFO : - [34mPath1[0m [35mQueue copy to Path2[0m - [36m{path2/}folder/HeLlO,wOrLd!.txt[0m+INFO : - [34mPath1[0m [35mQueue copy to Path2[0m - [36m{path2/}folder/éééö.txt[0m+INFO : - [34mPath1[0m [35mQueue copy to Path2[0m - [36m"{path2/}測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"[0m+INFO : - [34mPath2[0m [35mQueue copy to Path1[0m - [36m{path1/}file1.txt[0m+INFO : - [34mPath2[0m [35mQueue copy to Path1[0m - [36m{path1/}folder/éééö.txt[0m+INFO : - [34mPath2[0m [35mQueue copy to Path1[0m - [36m{path1/}folder/hello,WORLD!.txt[0m+INFO : - [34mPath2[0m [35mDo queued copies to[0m - [36mPath1[0m+INFO : - [34mPath1[0m [35mDo queued copies to[0m - [36mPath2[0m+INFO : Updating listings+INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"+INFO : [32mBisync successful[0mdiff --git a/cmd/bisync/testdata/test_normalization/initial/RCLONE_TEST b/cmd/bisync/testdata/test_normalization/initial/RCLONE_TESTnew file mode 100644index 000000000..d8ca97c2a--- /dev/null+++ b/cmd/bisync/testdata/test_normalization/initial/RCLONE_TEST@@ -0,0 +1 @@+This file is used for testing the health of rclone accesses to the local/remote file system. Do not delete.diff --git a/cmd/bisync/testdata/test_normalization/initial/file1.txt b/cmd/bisync/testdata/test_normalization/initial/file1.txtnew file mode 100644index 000000000..e69de29bbdiff --git "a/cmd/bisync/testdata/test_normalization/initial/\346\270\254\350\251\246_\320\240\321\203\321\201\321\201\320\272\320\270\320\271_ _ _\304\233_\303\241\303\261/filename_contains_\304\233_.txt" "b/cmd/bisync/testdata/test_normalization/initial/\346\270\254\350\251\246_\320\240\321\203\321\201\321\201\320\272\320\270\320\271_ _ _\304\233_\303\241\303\261/filename_contains_\304\233_.txt"new file mode 100644index 000000000..a378eaef5--- /dev/null+++ "b/cmd/bisync/testdata/test_normalization/initial/\346\270\254\350\251\246_\320\240\321\203\321\201\321\201\320\272\320\270\320\271_ _ _\304\233_\303\241\303\261/filename_contains_\304\233_.txt"@@ -0,0 +1,7 @@+ma�ana << Coded as xF1, which is the unicode ID name, but not the correct byte stream coding.+filename_contains_\u011b_+filename_contains_e_++_mañana_funcionará.txt++file_enconde_mañana_funcionará << Valid byte stream unicode is read correctlydiff --git "a/cmd/bisync/testdata/test_normalization/initial/\346\270\254\350\251\246_\320\240\321\203\321\201\321\201\320\272\320\270\320\271_ _ _\304\233_\303\241\303\261/filename_contains_\340\242\272_.txt" "b/cmd/bisync/testdata/test_normalization/initial/\346\270\254\350\251\246_\320\240\321\203\321\201\321\201\320\272\320\270\320\271_ _ _\304\233_\303\241\303\261/filename_contains_\340\242\272_.txt"new file mode 100644index 000000000..a378eaef5--- /dev/null+++ "b/cmd/bisync/testdata/test_normalization/initial/\346\270\254\350\251\246_\320\240\321\203\321\201\321\201\320\272\320\270\320\271_ _ _\304\233_\303\241\303\261/filename_contains_\340\242\272_.txt"@@ -0,0 +1,7 @@+ma�ana << Coded as xF1, which is the unicode ID name, but not the correct byte stream coding.+filename_contains_\u011b_+filename_contains_e_++_mañana_funcionará.txt++file_enconde_mañana_funcionará << Valid byte stream unicode is read correctlydiff --git "a/cmd/bisync/testdata/test_normalization/initial/\346\270\254\350\251\246_\320\240\321\203\321\201\321\201\320\272\320\270\320\271_ _ _\304\233_\303\241\303\261/\346\270\254\350\251\246_check file" "b/cmd/bisync/testdata/test_normalization/initial/\346\270\254\350\251\246_\320\240\321\203\321\201\321\201\320\272\320\270\320\271_ _ _\304\233_\303\241\303\261/\346\270\254\350\251\246_check file"new file mode 100644index 000000000..e69de29bbdiff --git a/cmd/bisync/testdata/test_normalization/modfiles/file1.txt b/cmd/bisync/testdata/test_normalization/modfiles/file1.txtnew file mode 100644index 000000000..464147f09--- /dev/null+++ b/cmd/bisync/testdata/test_normalization/modfiles/file1.txt@@ -0,0 +1 @@+This file is newerdiff --git "a/cmd/bisync/testdata/test_normalization/modfiles/\346\270\254\350\251\246_filtersfile.txt" "b/cmd/bisync/testdata/test_normalization/modfiles/\346\270\254\350\251\246_filtersfile.txt"new file mode 100644index 000000000..c85f097fc--- /dev/null+++ "b/cmd/bisync/testdata/test_normalization/modfiles/\346\270\254\350\251\246_filtersfile.txt"@@ -0,0 +1,5 @@+# Test filters file+# Note that this test checks for bisync's access to and usage of a filters file, not an extensive test of rclone's filter capability++# Exclude fileZ.txt in root only. The copy in the subdir should be found and synched.+- /fileZ.txtdiff --git a/cmd/bisync/testdata/test_normalization/scenario.txt b/cmd/bisync/testdata/test_normalization/scenario.txtnew file mode 100644index 000000000..e44265fd2--- /dev/null+++ b/cmd/bisync/testdata/test_normalization/scenario.txt@@ -0,0 +1,53 @@+test normalization+# Tests support for --no-unicode-normalization and --ignore-case-sync+# note: this test is written carefully to be runnable regardless of case/unicode sensitivity+# i.e. the results should be the same on linux and macOS++# force specific modification time since file time is lost through git+touch-copy 2001-01-02 {datadir/}file1.txt {path2/}+test initial bisync+bisync resync++# copy NFC version to Path1+copy-as-NFC {datadir/}file1.txt {path1/}測試_Русский___ě_áñ👸🏼🧝🏾♀️💆🏿♂️🐨🤙🏼🤮🧑🏻🔧🧑🔬éééö 測試_Русский___ě_áñ👸🏼🧝🏾♀️💆🏿♂️🐨🤙🏼🤮🧑🏻🔧🧑🔬éééö.txt+copy-as-NFC {datadir/}file1.txt {path1/}folder éééö.txt+copy-as-NFC {datadir/}file1.txt {path1/}folder HeLlO,wOrLd!.txt++# place newer NFD version on Path2+touch-copy 2001-01-03 {datadir/}file1.txt {path2/}+copy-as-NFD {datadir/}file1.txt {path2/}folder éééö.txt+copy-as-NFD {datadir/}file1.txt {path2/}folder hello,WORLD!.txt++test bisync run with fix-case+bisync fix-case++# purge and reset+purge-children {path1/}+purge-children {path2/}+touch-copy 2001-01-02 {datadir/}file1.txt {path2/}+bisync resync++# copy NFC version to Path1+copy-as-NFC {datadir/}file1.txt {path1/}測試_Русский___ě_áñ👸🏼🧝🏾♀️💆🏿♂️🐨🤙🏼🤮🧑🏻🔧🧑🔬éééö 測試_Русский___ě_áñ👸🏼🧝🏾♀️💆🏿♂️🐨🤙🏼🤮🧑🏻🔧🧑🔬éééö.txt+copy-as-NFC {datadir/}file1.txt {path1/}folder éééö.txt+copy-as-NFC {datadir/}file1.txt {path1/}folder HeLlO,wOrLd!.txt++# place newer NFD version on Path2+touch-copy 2001-01-03 {datadir/}file1.txt {path2/}+copy-as-NFD {datadir/}file1.txt {path2/}folder éééö.txt+copy-as-NFD {datadir/}file1.txt {path2/}folder hello,WORLD!.txt++test bisync run with normalization+bisync norm force++test resync+bisync resync norm++test changed on both paths+touch-copy 2001-01-05 {datadir/}file1.txt {path2/}+copy-as-NFC {datadir/}file1.txt {path1/}測試_Русский___ě_áñ👸🏼🧝🏾♀️💆🏿♂️🐨🤙🏼🤮🧑🏻🔧🧑🔬éééö 測試_Русский___ě_áñ👸🏼🧝🏾♀️💆🏿♂️🐨🤙🏼🤮🧑🏻🔧🧑🔬éééö.txt+copy-as-NFC {datadir/}file1.txt {path1/}folder éééö.txt+copy-as-NFC {datadir/}file1.txt {path1/}folder HeLlO,wOrLd!.txt+copy-as-NFD {datadir/}file1.txt {path2/}folder éééö.txt+copy-as-NFD {datadir/}file1.txt {path2/}folder hello,WORLD!.txt+bisync norm\ No newline at end of filediff --git a/docs/content/bisync.md b/docs/content/bisync.mdindex b01e82cc0..109010827 100644--- a/docs/content/bisync.md+++ b/docs/content/bisync.md@@ -588,13 +588,26 @@ and refuses to run again until the user runs a `--resync` (unless using `--resilThe best workaround at the moment is to set any backend-specific flags in the [config file](/commands/rclone_config/)instead of specifying them with command flags. (You can still override them as needed for other rclone commands.)-### Case sensitivity+### Case (and unicode) sensitivity {#case-sensitivity}Synching with **case-insensitive** filesystems, such as Windows or `Box`,-can result in file name conflicts. This will be fixed in a future release.-The near-term workaround is to make sure that files on both sides+can result in unusual behavior. As of `v1.65`, case and unicode form differences no longer cause critical errors,+however they may cause unexpected delta outcomes, due to the delta engine still being case-sensitive.+This will be fixed in a future release. The near-term workaround is to make sure that files on both sidesdon't have spelling case differences (`Smile.jpg` vs. `smile.jpg`).+The same limitation applies to Unicode normalization forms.+This [particularly applies to macOS](https://github.com/rclone/rclone/issues/7270),+which prefers NFD and sometimes auto-converts filenames from the NFC form used by most other platforms.+This should no longer cause bisync to fail entirely, but may cause surprising delta results, as explained above.++See the following options (all of which are supported by bisync) to control this behavior more granularly:+- [`--fix-case`](/docs/#fix-case)+- [`--ignore-case-sync`](/docs/#ignore-case-sync)+- [`--no-unicode-normalization`](/docs/#no-unicode-normalization)+- [`--local-unicode-normalization`](/local/#local-unicode-normalization) and+[`--local-case-sensitive`](/local/#local-case-sensitive) (caution: these are normally not what you want.)+## Windows support {#windows}Bisync has been tested on Windows 8.1, Windows 10 Pro 64-bit and on Windows@@ -1278,6 +1291,7 @@ about _Unison_ and synchronization in general.* A few basic terminal colors are now supported, controllable with [`--color`](/docs/#color-when) (`AUTO`|`NEVER`|`ALWAYS`)* Initial listing snapshots of Path1 and Path2 are now generated concurrently, using the same "march" infrastructure as `check` and `sync`,for performance improvements and less [risk of error](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=4.%20Listings%20should%20alternate%20between%20paths%20to%20minimize%20errors).+* Better handling of unicode normalization and case insensitivity, support for [`--fix-case`](/docs/#fix-case), [`--ignore-case-sync`](/docs/#ignore-case-sync), [`--no-unicode-normalization`](/docs/#no-unicode-normalization)### `v1.64`* Fixed an [issue](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=1.%20Dry%20runs%20are%20not%20completely%20dry)