Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

bisync: handle unicode and case normalization consistently - mostly-fixes #7270

nielashf7f465182832
diff --git a/cmd/bisync/bisync_test.go b/cmd/bisync/bisync_test.go
index 11a0862c3..185981a2c 100644
--- a/cmd/bisync/bisync_test.go
+++ b/cmd/bisync/bisync_test.go
@@ -98,10 +98,13 @@ var logHoppers = []string{
// subdirectories. The order inconsistency initially showed up in the
// listings and triggered reordering of log messages, but the actual
// files will in fact match.
- `ERROR : - +Access test failed: Path[12] file not found in Path[12] - .*`,
+ `.* +.....Access test failed: Path[12] file not found in Path[12].*`,
// Test case `resync` suffered from the order of queued copies.
`(?:INFO |NOTICE): - Path2 Resync will copy to Path1 +- .*`,
+
+ // Test case `normalization` can have random order of fix-case files.
+ `(?:INFO |NOTICE): .*: Fixed case by renaming to: .*`,
}
// Some log lines can contain Windows path separator that must be
@@ -546,6 +549,16 @@ func (b *bisyncTest) runTestStep(ctx context.Context, line string) (err error) {
case "copy-as":
b.checkArgs(args, 3, 3)
return b.copyFile(ctx, args[1], args[2], args[3])
+ case "copy-as-NFC":
+ b.checkArgs(args, 3, 3)
+ ci.NoUnicodeNormalization = true
+ ci.FixCase = true
+ return b.copyFile(ctx, args[1], norm.NFC.String(args[2]), norm.NFC.String(args[3]))
+ case "copy-as-NFD":
+ b.checkArgs(args, 3, 3)
+ ci.NoUnicodeNormalization = true
+ ci.FixCase = true
+ return b.copyFile(ctx, args[1], norm.NFD.String(args[2]), norm.NFD.String(args[3]))
case "copy-dir", "sync-dir":
b.checkArgs(args, 2, 2)
if fsrc, err = cache.Get(ctx, args[1]); err != nil {
@@ -565,6 +578,9 @@ func (b *bisyncTest) runTestStep(ctx context.Context, line string) (err error) {
b.checkArgs(args, 1, 1)
return b.listSubdirs(ctx, args[1])
case "bisync":
+ ci.NoUnicodeNormalization = false
+ ci.IgnoreCaseSync = false
+ // ci.FixCase = true
return b.runBisync(ctx, args[1:])
case "test-func":
b.TestFn = testFunc
@@ -668,6 +684,16 @@ func (b *bisyncTest) runBisync(ctx context.Context, args []string) (err error) {
fs2 = addSubdir(b.path2, val)
case "ignore-listing-checksum":
opt.IgnoreListingChecksum = true
+ case "no-norm":
+ ci.NoUnicodeNormalization = true
+ ci.IgnoreCaseSync = false
+ case "norm":
+ ci.NoUnicodeNormalization = false
+ ci.IgnoreCaseSync = true
+ case "fix-case":
+ ci.NoUnicodeNormalization = false
+ ci.IgnoreCaseSync = true
+ ci.FixCase = true
default:
return fmt.Errorf("invalid bisync option %q", arg)
}
diff --git a/cmd/bisync/deltas.go b/cmd/bisync/deltas.go
index 3167dfb00..d0c0cd8aa 100644
--- a/cmd/bisync/deltas.go
+++ b/cmd/bisync/deltas.go
@@ -381,11 +381,17 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change
}
}
+ // find alternate names according to normalization settings
+ altNames1to2 := bilib.Names{}
+ altNames2to1 := bilib.Names{}
+ b.findAltNames(ctx, b.fs1, copy2to1, b.newListing1, altNames2to1)
+ b.findAltNames(ctx, b.fs2, copy1to2, b.newListing2, altNames1to2)
+
// Do the batch operation
if copy2to1.NotEmpty() {
changes1 = true
b.indent("Path2", "Path1", "Do queued copies to")
- results2to1, err = b.fastCopy(ctx, b.fs2, b.fs1, copy2to1, "copy2to1")
+ results2to1, err = b.fastCopy(ctx, b.fs2, b.fs1, copy2to1, "copy2to1", altNames2to1)
if err != nil {
return
}
@@ -397,7 +403,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change
if copy1to2.NotEmpty() {
changes2 = true
b.indent("Path1", "Path2", "Do queued copies to")
- results1to2, err = b.fastCopy(ctx, b.fs1, b.fs2, copy1to2, "copy1to2")
+ results1to2, err = b.fastCopy(ctx, b.fs1, b.fs2, copy1to2, "copy1to2", altNames1to2)
if err != nil {
return
}
diff --git a/cmd/bisync/listing.go b/cmd/bisync/listing.go
index 7875036fa..f94883d97 100644
--- a/cmd/bisync/listing.go
+++ b/cmd/bisync/listing.go
@@ -20,6 +20,7 @@ import (
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/operations"
"golang.org/x/exp/slices"
+ "golang.org/x/text/unicode/norm"
)
// ListingHeader defines first line of a listing
@@ -400,6 +401,18 @@ func ConvertPrecision(Modtime time.Time, dst fs.Fs) time.Time {
return Modtime
}
+// ApplyTransforms handles unicode and case normalization
+func ApplyTransforms(ctx context.Context, dst fs.Fs, s string) string {
+ ci := fs.GetConfig(ctx)
+ if !ci.NoUnicodeNormalization {
+ s = norm.NFC.String(s)
+ }
+ if ci.IgnoreCaseSync || dst.Features().CaseInsensitive {
+ s = strings.ToLower(s)
+ }
+ return s
+}
+
// modifyListing will modify the listing based on the results of the sync
func (b *bisyncRun) modifyListing(ctx context.Context, src fs.Fs, dst fs.Fs, results []Results, queues queues, is1to2 bool) (err error) {
queue := queues.copy2to1
@@ -429,6 +442,10 @@ func (b *bisyncRun) modifyListing(ctx context.Context, src fs.Fs, dst fs.Fs, res
srcList.hash = src.Hashes().GetOne()
dstList.hash = dst.Hashes().GetOne()
}
+ dstListNew, err := b.loadListing(dstListing + "-new")
+ if err != nil {
+ return fmt.Errorf("cannot read new listing: %w", err)
+ }
srcWinners := newFileList()
dstWinners := newFileList()
@@ -466,20 +483,47 @@ func (b *bisyncRun) modifyListing(ctx context.Context, src fs.Fs, dst fs.Fs, res
}
updateLists := func(side string, winners, list *fileList) {
- // removals from side
- for _, oldFile := range queue.ToList() {
- if !winners.has(oldFile) && list.has(oldFile) && !errors.has(oldFile) {
- list.remove(oldFile)
- fs.Debugf(nil, "decision: removed from %s: %v", side, oldFile)
- } else if winners.has(oldFile) {
+ for _, queueFile := range queue.ToList() {
+ if !winners.has(queueFile) && list.has(queueFile) && !errors.has(queueFile) {
+ // removals from side
+ list.remove(queueFile)
+ fs.Debugf(nil, "decision: removed from %s: %v", side, queueFile)
+ } else if winners.has(queueFile) {
// copies to side
- new := winners.get(oldFile)
- list.put(oldFile, new.size, new.time, new.hash, new.id, new.flags)
- fs.Debugf(nil, "decision: copied to %s: %v", side, oldFile)
+ new := winners.get(queueFile)
+
+ // handle normalization according to settings
+ ci := fs.GetConfig(ctx)
+ if side == "dst" && (!ci.NoUnicodeNormalization || ci.IgnoreCaseSync || dst.Features().CaseInsensitive) {
+ // search list for existing file that matches queueFile when normalized
+ normalizedName := ApplyTransforms(ctx, dst, queueFile)
+ matchFound := false
+ matchedName := ""
+ for _, filename := range dstListNew.list {
+ if ApplyTransforms(ctx, dst, filename) == normalizedName {
+ matchFound = true
+ matchedName = filename // original, not normalized
+ break
+ }
+ }
+ if matchFound && matchedName != queueFile {
+ // use the (non-identical) existing name, unless --fix-case
+ if ci.FixCase {
+ fs.Debugf(direction, "removing %s and adding %s as --fix-case was specified", matchedName, queueFile)
+ list.remove(matchedName)
+ } else {
+ fs.Debugf(direction, "casing/unicode difference detected. using %s instead of %s", matchedName, queueFile)
+ queueFile = matchedName
+ }
+ }
+ }
+
+ list.put(queueFile, new.size, new.time, new.hash, new.id, new.flags)
+ fs.Debugf(nil, "decision: copied to %s: %v", side, queueFile)
} else {
- fs.Debugf(oldFile, "file in queue but missing from %s transfers", side)
- if err := filterRecheck.AddFile(oldFile); err != nil {
- fs.Debugf(oldFile, "error adding file to recheck filter: %v", err)
+ fs.Debugf(queueFile, "file in queue but missing from %s transfers", side)
+ if err := filterRecheck.AddFile(queueFile); err != nil {
+ fs.Debugf(queueFile, "error adding file to recheck filter: %v", err)
}
}
}
diff --git a/cmd/bisync/march.go b/cmd/bisync/march.go
index 229f3ed63..63536063b 100644
--- a/cmd/bisync/march.go
+++ b/cmd/bisync/march.go
@@ -183,7 +183,3 @@ func whichPath(isPath1 bool) string {
}
return s
}
-
-// TODO:
-// equality check?
-// unicode stuff
diff --git a/cmd/bisync/operations.go b/cmd/bisync/operations.go
index 67cb5b440..fce29f4d0 100644
--- a/cmd/bisync/operations.go
+++ b/cmd/bisync/operations.go
@@ -428,8 +428,11 @@ func (b *bisyncRun) resync(octx, fctx context.Context) error {
if len(copy2to1) > 0 {
b.indent("Path2", "Path1", "Resync is doing queued copies to")
+ resync2to1 := bilib.ToNames(copy2to1)
+ altNames2to1 := bilib.Names{}
+ b.findAltNames(octx, b.fs1, resync2to1, b.newListing1, altNames2to1)
// octx does not have extra filters!
- results2to1, err = b.fastCopy(octx, b.fs2, b.fs1, bilib.ToNames(copy2to1), "resync-copy2to1")
+ results2to1, err = b.fastCopy(octx, b.fs2, b.fs1, resync2to1, "resync-copy2to1", altNames2to1)
if err != nil {
b.critical = true
return err
@@ -516,15 +519,27 @@ func (b *bisyncRun) checkSync(listing1, listing2 string) error {
return fmt.Errorf("cannot read prior listing of Path2: %w", err)
}
+ transformList := func(files *fileList, fs fs.Fs) *fileList {
+ transformed := newFileList()
+ for _, file := range files.list {
+ f := files.get(file)
+ transformed.put(ApplyTransforms(context.Background(), fs, file), f.size, f.time, f.hash, f.id, f.flags)
+ }
+ return transformed
+ }
+
+ files1Transformed := transformList(files1, b.fs1)
+ files2Transformed := transformList(files2, b.fs2)
+
ok := true
for _, file := range files1.list {
- if !files2.has(file) {
+ if !files2.has(file) && !files2Transformed.has(ApplyTransforms(context.Background(), b.fs1, file)) {
b.indent("ERROR", file, "Path1 file not found in Path2")
ok = false
}
}
for _, file := range files2.list {
- if !files1.has(file) {
+ if !files1.has(file) && !files1Transformed.has(ApplyTransforms(context.Background(), b.fs2, file)) {
b.indent("ERROR", file, "Path2 file not found in Path1")
ok = false
}
diff --git a/cmd/bisync/queue.go b/cmd/bisync/queue.go
index d670c033d..d16f06fcc 100644
--- a/cmd/bisync/queue.go
+++ b/cmd/bisync/queue.go
@@ -14,7 +14,6 @@ import (
"github.com/rclone/rclone/fs/filter"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/fs/sync"
- "golang.org/x/text/unicode/norm"
)
// Results represents a pair of synced files, as reported by the LoggerFn
@@ -130,7 +129,7 @@ func ReadResults(results io.Reader) []Results {
return slice
}
-func (b *bisyncRun) fastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib.Names, queueName string) ([]Results, error) {
+func (b *bisyncRun) fastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib.Names, queueName string, altNames bilib.Names) ([]Results, error) {
if err := b.saveQueue(files, queueName); err != nil {
return nil, err
}
@@ -140,9 +139,12 @@ func (b *bisyncRun) fastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib.
if err := filterCopy.AddFile(file); err != nil {
return nil, err
}
- // macOS
- if err := filterCopy.AddFile(norm.NFD.String(file)); err != nil {
- return nil, err
+ }
+ if altNames.NotEmpty() {
+ for _, file := range altNames.ToList() {
+ if err := filterCopy.AddFile(file); err != nil {
+ return nil, err
+ }
}
}
@@ -252,3 +254,22 @@ func (b *bisyncRun) saveQueue(files bilib.Names, jobName string) error {
queueFile := fmt.Sprintf("%s.%s.que", b.basePath, jobName)
return files.Save(queueFile)
}
+
+func (b *bisyncRun) findAltNames(ctx context.Context, dst fs.Fs, queue bilib.Names, newListing string, altNames bilib.Names) {
+ ci := fs.GetConfig(ctx)
+ if queue.NotEmpty() && (!ci.NoUnicodeNormalization || ci.IgnoreCaseSync || b.fs1.Features().CaseInsensitive || b.fs2.Features().CaseInsensitive) {
+ // search list for existing file that matches queueFile when normalized
+ for _, queueFile := range queue.ToList() {
+ normalizedName := ApplyTransforms(ctx, dst, queueFile)
+ candidates, err := b.loadListing(newListing)
+ if err != nil {
+ fs.Errorf(candidates, "cannot read new listing: %v", err)
+ }
+ for _, filename := range candidates.list {
+ if ApplyTransforms(ctx, dst, filename) == normalizedName && filename != queueFile {
+ altNames.Add(filename) // original, not normalized
+ }
+ }
+ }
+ }
+}
diff --git a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst
index 7471a27cf..f0208cb9a 100644
--- a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst
+++ b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst
@@ -1,10 +1,9 @@
# bisync listing v1 from test
-- 6148 md5:23b446fda9938c607142c5133cf90689 - 2000-01-01T00:00:00.000000000+0000 ".DS_Store"
-- 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"
-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"
-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt"
-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt"
-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt"
-- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "file1.txt"
-- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "subdir/file20.txt"
+- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"
+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"
+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt"
+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt"
+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt"
+- 19 - - 2001-01-02T00:00:00.000000000+0000 "file1.txt"
+- 19 - - 2001-01-02T00:00:00.000000000+0000 "subdir/file20.txt"
diff --git a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst-new b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst-new
index f243ade04..465d2455d 100644
--- a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst-new
+++ b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst-new
@@ -1,5 +1,4 @@
# bisync listing v1 from test
-- 6148 md5:23b446fda9938c607142c5133cf90689 - 2000-01-01T00:00:00.000000000+0000 ".DS_Store"
- 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"
diff --git a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst-old b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst-old
index 991f89b82..7819f9efc 100644
--- a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst-old
+++ b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path1.lst-old
@@ -1,10 +1,9 @@
# bisync listing v1 from test
-- 6148 md5:23b446fda9938c607142c5133cf90689 - 2000-01-01T00:00:00.000000000+0000 ".DS_Store"
-- 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"
-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"
-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt"
-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt"
-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt"
-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.txt"
-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "subdir/file20.txt"
+- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"
+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"
+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt"
+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt"
+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt"
+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.txt"
+- 0 - - 2000-01-01T00:00:00.000000000+0000 "subdir/file20.txt"
diff --git a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst
index 7471a27cf..f0208cb9a 100644
--- a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst
+++ b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst
@@ -1,10 +1,9 @@
# bisync listing v1 from test
-- 6148 md5:23b446fda9938c607142c5133cf90689 - 2000-01-01T00:00:00.000000000+0000 ".DS_Store"
-- 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"
-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"
-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt"
-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt"
-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt"
-- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "file1.txt"
-- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "subdir/file20.txt"
+- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"
+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"
+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt"
+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt"
+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt"
+- 19 - - 2001-01-02T00:00:00.000000000+0000 "file1.txt"
+- 19 - - 2001-01-02T00:00:00.000000000+0000 "subdir/file20.txt"
diff --git a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst-new b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst-new
index dac88bd2c..9e8bc8a1a 100644
--- a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst-new
+++ b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst-new
@@ -1,5 +1,4 @@
# bisync listing v1 from test
-- 6148 md5:23b446fda9938c607142c5133cf90689 - 2000-01-01T00:00:00.000000000+0000 ".DS_Store"
- 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"
- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"
diff --git a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst-old b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst-old
index 991f89b82..7819f9efc 100644
--- a/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst-old
+++ b/cmd/bisync/testdata/test_basic/golden/_testdir_path1.._testdir_path2.path2.lst-old
@@ -1,10 +1,9 @@
# bisync listing v1 from test
-- 6148 md5:23b446fda9938c607142c5133cf90689 - 2000-01-01T00:00:00.000000000+0000 ".DS_Store"
-- 109 md5:294d25b294ff26a5243dba914ac3fbf7 - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"
-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"
-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt"
-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt"
-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt"
-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "file1.txt"
-- 0 md5:d41d8cd98f00b204e9800998ecf8427e - 2000-01-01T00:00:00.000000000+0000 "subdir/file20.txt"
+- 109 - - 2000-01-01T00:00:00.000000000+0000 "RCLONE_TEST"
+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy1.txt"
+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy2.txt"
+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy3.txt"
+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy4.txt"
+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.copy5.txt"
+- 0 - - 2000-01-01T00:00:00.000000000+0000 "file1.txt"
+- 0 - - 2000-01-01T00:00:00.000000000+0000 "subdir/file20.txt"
diff --git a/cmd/bisync/testdata/test_check_access_filters/golden/test.log b/cmd/bisync/testdata/test_check_access_filters/golden/test.log
index c12126374..9ade7ee1b 100644
--- a/cmd/bisync/testdata/test_check_access_filters/golden/test.log
+++ b/cmd/bisync/testdata/test_check_access_filters/golden/test.log
@@ -142,8 +142,8 @@ INFO : Path2: 1 changes: 0 new, 0 newer, 0 older, 1 deleted
INFO : Checking access health
ERROR : Access test failed: Path1 count 3, Path2 count 4 - RCLONE_TEST
ERROR : -  Access test failed: Path1 file not found in Path2 - RCLONE_TEST
-ERROR : -  Access test failed: Path2 file not found in Path1 - subdir/RCLONE_TEST
ERROR : -  Access test failed: Path2 file not found in Path1 - subdirX/subdirX1/RCLONE_TEST
+ERROR : -  Access test failed: Path2 file not found in Path1 - subdir/RCLONE_TEST
ERROR : Bisync critical error: check file check failed
ERROR : Bisync aborted. Must run --resync to recover.
Bisync error: bisync aborted
diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.copy1to2.que b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.copy1to2.que
new file mode 100644
index 000000000..bd7f8e7dd
--- /dev/null
+++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.copy1to2.que
@@ -0,0 +1,3 @@
+"folder/HeLlO,wOrLd!.txt"
+"folder/éééö.txt"
+"測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"
diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.copy2to1.que b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.copy2to1.que
new file mode 100644
index 000000000..85deec9f9
--- /dev/null
+++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.copy2to1.que
@@ -0,0 +1,3 @@
+"file1.txt"
+"folder/hello,WORLD!.txt"
+"folder/éééö.txt"
diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst
new file mode 100644
index 000000000..0cdecd335
--- /dev/null
+++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst
@@ -0,0 +1,5 @@
+# bisync listing v1 from test
+- 19 - - 2001-01-05T00:00:00.000000000+0000 "file1.txt"
+- 19 - - 2001-01-05T00:00:00.000000000+0000 "folder/HeLlO,wOrLd!.txt"
+- 19 - - 2001-01-05T00:00:00.000000000+0000 "folder/éééö.txt"
+- 19 - - 2001-01-05T00:00:00.000000000+0000 "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"
diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst-new b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst-new
new file mode 100644
index 000000000..bd44aea5b
--- /dev/null
+++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst-new
@@ -0,0 +1,5 @@
+# bisync listing v1 from test
+- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-03T00:00:00.000000000+0000 "file1.txt"
+- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-05T00:00:00.000000000+0000 "folder/HeLlO,wOrLd!.txt"
+- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-05T00:00:00.000000000+0000 "folder/éééö.txt"
+- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-05T00:00:00.000000000+0000 "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"
diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst-old b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst-old
new file mode 100644
index 000000000..441434161
--- /dev/null
+++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path1.lst-old
@@ -0,0 +1,5 @@
+# bisync listing v1 from test
+- 19 - - 2001-01-03T00:00:00.000000000+0000 "file1.txt"
+- 19 - - 2001-01-03T00:00:00.000000000+0000 "folder/HeLlO,wOrLd!.txt"
+- 19 - - 2001-01-03T00:00:00.000000000+0000 "folder/éééö.txt"
+- 19 - - 2001-01-02T00:00:00.000000000+0000 "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"
diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst
new file mode 100644
index 000000000..f7c2c0ea6
--- /dev/null
+++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst
@@ -0,0 +1,5 @@
+# bisync listing v1 from test
+- 19 - - 2001-01-05T00:00:00.000000000+0000 "file1.txt"
+- 19 - - 2001-01-05T00:00:00.000000000+0000 "folder/hello,WORLD!.txt"
+- 19 - - 2001-01-05T00:00:00.000000000+0000 "folder/éééö.txt"
+- 19 - - 2001-01-05T00:00:00.000000000+0000 "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"
diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-new b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-new
new file mode 100644
index 000000000..225fe66c1
--- /dev/null
+++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-new
@@ -0,0 +1,5 @@
+# bisync listing v1 from test
+- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-05T00:00:00.000000000+0000 "file1.txt"
+- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-05T00:00:00.000000000+0000 "folder/hello,WORLD!.txt"
+- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-05T00:00:00.000000000+0000 "folder/éééö.txt"
+- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"
diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-old b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-old
new file mode 100644
index 000000000..ab1695601
--- /dev/null
+++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-old
@@ -0,0 +1,5 @@
+# bisync listing v1 from test
+- 19 - - 2001-01-03T00:00:00.000000000+0000 "file1.txt"
+- 19 - - 2001-01-03T00:00:00.000000000+0000 "folder/hello,WORLD!.txt"
+- 19 - - 2001-01-03T00:00:00.000000000+0000 "folder/éééö.txt"
+- 19 - - 2001-01-02T00:00:00.000000000+0000 "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"
diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.resync-copy2to1.que b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.resync-copy2to1.que
new file mode 100644
index 000000000..007f0aada
--- /dev/null
+++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.resync-copy2to1.que
@@ -0,0 +1,2 @@
+"folder/hello,WORLD!.txt"
+"folder/éééö.txt"
diff --git a/cmd/bisync/testdata/test_normalization/golden/test.log b/cmd/bisync/testdata/test_normalization/golden/test.log
new file mode 100644
index 000000000..accd69d14
--- /dev/null
+++ b/cmd/bisync/testdata/test_normalization/golden/test.log
@@ -0,0 +1,144 @@
+(01) : test normalization
+
+
+(02) : touch-copy 2001-01-02 {datadir/}file1.txt {path2/}
+(03) : test initial bisync
+(04) : bisync resync
+INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
+INFO : Copying unique Path2 files to Path1
+INFO : Resynching Path1 to Path2
+INFO : Resync updating listings
+INFO : Bisync successful
+
+
+(05) : copy-as-NFC {datadir/}file1.txt {path1/}測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö 測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö.txt
+(06) : copy-as-NFC {datadir/}file1.txt {path1/}folder éééö.txt
+(07) : copy-as-NFC {datadir/}file1.txt {path1/}folder HeLlO,wOrLd!.txt
+
+
+(08) : touch-copy 2001-01-03 {datadir/}file1.txt {path2/}
+(09) : copy-as-NFD {datadir/}file1.txt {path2/}folder éééö.txt
+(10) : copy-as-NFD {datadir/}file1.txt {path2/}folder hello,WORLD!.txt
+
+(11) : test bisync run with fix-case
+(12) : bisync fix-case
+INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
+INFO : Building Path1 and Path2 listings
+INFO : Path1 checking for diffs
+INFO : - Path1 File is new - folder/HeLlO,wOrLd!.txt
+INFO : - Path1 File is new - folder/éééö.txt
+INFO : - Path1 File is new - "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"
+INFO : Path1: 3 changes: 3 new, 0 newer, 0 older, 0 deleted
+INFO : Path2 checking for diffs
+INFO : - Path2 File is newer - file1.txt
+INFO : - Path2 File is new - folder/éééö.txt
+INFO : - Path2 File is new - folder/hello,WORLD!.txt
+INFO : Path2: 3 changes: 2 new, 1 newer, 0 older, 0 deleted
+INFO : Applying changes
+INFO : - Path1 Queue copy to Path2 - {path2/}folder/HeLlO,wOrLd!.txt
+INFO : - Path1 Queue copy to Path2 - {path2/}folder/éééö.txt
+INFO : - Path1 Queue copy to Path2 - "{path2/}測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"
+INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt
+INFO : - Path2 Queue copy to Path1 - {path1/}folder/éééö.txt
+INFO : - Path2 Queue copy to Path1 - {path1/}folder/hello,WORLD!.txt
+INFO : - Path2 Do queued copies to - Path1
+INFO : folder/HeLlO,wOrLd!.txt: Fixed case by renaming to: folder/hello,WORLD!.txt
+INFO : folder/éééö.txt: Fixed case by renaming to: folder/éééö.txt
+INFO : - Path1 Do queued copies to - Path2
+INFO : Updating listings
+INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
+INFO : Bisync successful
+
+
+(13) : purge-children {path1/}
+(14) : purge-children {path2/}
+(15) : touch-copy 2001-01-02 {datadir/}file1.txt {path2/}
+(16) : bisync resync
+INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
+INFO : Copying unique Path2 files to Path1
+INFO : - Path2 Resync will copy to Path1 - file1.txt
+INFO : - Path2 Resync is doing queued copies to - Path1
+INFO : Resynching Path1 to Path2
+INFO : Resync updating listings
+INFO : Bisync successful
+
+
+(17) : copy-as-NFC {datadir/}file1.txt {path1/}測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö 測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö.txt
+(18) : copy-as-NFC {datadir/}file1.txt {path1/}folder éééö.txt
+(19) : copy-as-NFC {datadir/}file1.txt {path1/}folder HeLlO,wOrLd!.txt
+
+
+(20) : touch-copy 2001-01-03 {datadir/}file1.txt {path2/}
+(21) : copy-as-NFD {datadir/}file1.txt {path2/}folder éééö.txt
+(22) : copy-as-NFD {datadir/}file1.txt {path2/}folder hello,WORLD!.txt
+
+(23) : test bisync run with normalization
+(24) : bisync norm force
+INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
+INFO : Building Path1 and Path2 listings
+INFO : Path1 checking for diffs
+INFO : - Path1 File is new - folder/HeLlO,wOrLd!.txt
+INFO : - Path1 File is new - folder/éééö.txt
+INFO : - Path1 File is new - "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"
+INFO : Path1: 3 changes: 3 new, 0 newer, 0 older, 0 deleted
+INFO : Path2 checking for diffs
+INFO : - Path2 File is newer - file1.txt
+INFO : - Path2 File is new - folder/éééö.txt
+INFO : - Path2 File is new - folder/hello,WORLD!.txt
+INFO : Path2: 3 changes: 2 new, 1 newer, 0 older, 0 deleted
+INFO : Applying changes
+INFO : - Path1 Queue copy to Path2 - {path2/}folder/HeLlO,wOrLd!.txt
+INFO : - Path1 Queue copy to Path2 - {path2/}folder/éééö.txt
+INFO : - Path1 Queue copy to Path2 - "{path2/}測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"
+INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt
+INFO : - Path2 Queue copy to Path1 - {path1/}folder/éééö.txt
+INFO : - Path2 Queue copy to Path1 - {path1/}folder/hello,WORLD!.txt
+INFO : - Path2 Do queued copies to - Path1
+INFO : - Path1 Do queued copies to - Path2
+INFO : Updating listings
+INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
+INFO : Bisync successful
+
+(25) : test resync
+(26) : bisync resync norm
+INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
+INFO : Copying unique Path2 files to Path1
+INFO : - Path2 Resync will copy to Path1 - folder/éééö.txt
+INFO : - Path2 Resync will copy to Path1 - folder/hello,WORLD!.txt
+INFO : - Path2 Resync is doing queued copies to - Path1
+INFO : Resynching Path1 to Path2
+INFO : Resync updating listings
+INFO : Bisync successful
+
+(27) : test changed on both paths
+(28) : touch-copy 2001-01-05 {datadir/}file1.txt {path2/}
+(29) : copy-as-NFC {datadir/}file1.txt {path1/}測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö 測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö.txt
+(30) : copy-as-NFC {datadir/}file1.txt {path1/}folder éééö.txt
+(31) : copy-as-NFC {datadir/}file1.txt {path1/}folder HeLlO,wOrLd!.txt
+(32) : copy-as-NFD {datadir/}file1.txt {path2/}folder éééö.txt
+(33) : copy-as-NFD {datadir/}file1.txt {path2/}folder hello,WORLD!.txt
+(34) : bisync norm
+INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
+INFO : Building Path1 and Path2 listings
+INFO : Path1 checking for diffs
+INFO : - Path1 File is newer - folder/HeLlO,wOrLd!.txt
+INFO : - Path1 File is newer - folder/éééö.txt
+INFO : - Path1 File is newer - "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"
+INFO : Path1: 3 changes: 0 new, 3 newer, 0 older, 0 deleted
+INFO : Path2 checking for diffs
+INFO : - Path2 File is newer - file1.txt
+INFO : - Path2 File is newer - folder/éééö.txt
+INFO : - Path2 File is newer - folder/hello,WORLD!.txt
+INFO : Path2: 3 changes: 0 new, 3 newer, 0 older, 0 deleted
+INFO : Applying changes
+INFO : - Path1 Queue copy to Path2 - {path2/}folder/HeLlO,wOrLd!.txt
+INFO : - Path1 Queue copy to Path2 - {path2/}folder/éééö.txt
+INFO : - Path1 Queue copy to Path2 - "{path2/}測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"
+INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt
+INFO : - Path2 Queue copy to Path1 - {path1/}folder/éééö.txt
+INFO : - Path2 Queue copy to Path1 - {path1/}folder/hello,WORLD!.txt
+INFO : - Path2 Do queued copies to - Path1
+INFO : - Path1 Do queued copies to - Path2
+INFO : Updating listings
+INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
+INFO : Bisync successful
diff --git a/cmd/bisync/testdata/test_normalization/initial/RCLONE_TEST b/cmd/bisync/testdata/test_normalization/initial/RCLONE_TEST
new file mode 100644
index 000000000..d8ca97c2a
--- /dev/null
+++ b/cmd/bisync/testdata/test_normalization/initial/RCLONE_TEST
@@ -0,0 +1 @@
+This file is used for testing the health of rclone accesses to the local/remote file system. Do not delete.
diff --git a/cmd/bisync/testdata/test_normalization/initial/file1.txt b/cmd/bisync/testdata/test_normalization/initial/file1.txt
new file mode 100644
index 000000000..e69de29bb
diff --git "a/cmd/bisync/testdata/test_normalization/initial/\346\270\254\350\251\246_\320\240\321\203\321\201\321\201\320\272\320\270\320\271_ _ _\304\233_\303\241\303\261/filename_contains_\304\233_.txt" "b/cmd/bisync/testdata/test_normalization/initial/\346\270\254\350\251\246_\320\240\321\203\321\201\321\201\320\272\320\270\320\271_ _ _\304\233_\303\241\303\261/filename_contains_\304\233_.txt"
new file mode 100644
index 000000000..a378eaef5
--- /dev/null
+++ "b/cmd/bisync/testdata/test_normalization/initial/\346\270\254\350\251\246_\320\240\321\203\321\201\321\201\320\272\320\270\320\271_ _ _\304\233_\303\241\303\261/filename_contains_\304\233_.txt"
@@ -0,0 +1,7 @@
+ma�ana << Coded as xF1, which is the unicode ID name, but not the correct byte stream coding.
+filename_contains_\u011b_
+filename_contains_e_
+
+_mañana_funcionará.txt
+
+file_enconde_mañana_funcionará << Valid byte stream unicode is read correctly
diff --git "a/cmd/bisync/testdata/test_normalization/initial/\346\270\254\350\251\246_\320\240\321\203\321\201\321\201\320\272\320\270\320\271_ _ _\304\233_\303\241\303\261/filename_contains_\340\242\272_.txt" "b/cmd/bisync/testdata/test_normalization/initial/\346\270\254\350\251\246_\320\240\321\203\321\201\321\201\320\272\320\270\320\271_ _ _\304\233_\303\241\303\261/filename_contains_\340\242\272_.txt"
new file mode 100644
index 000000000..a378eaef5
--- /dev/null
+++ "b/cmd/bisync/testdata/test_normalization/initial/\346\270\254\350\251\246_\320\240\321\203\321\201\321\201\320\272\320\270\320\271_ _ _\304\233_\303\241\303\261/filename_contains_\340\242\272_.txt"
@@ -0,0 +1,7 @@
+ma�ana << Coded as xF1, which is the unicode ID name, but not the correct byte stream coding.
+filename_contains_\u011b_
+filename_contains_e_
+
+_mañana_funcionará.txt
+
+file_enconde_mañana_funcionará << Valid byte stream unicode is read correctly
diff --git "a/cmd/bisync/testdata/test_normalization/initial/\346\270\254\350\251\246_\320\240\321\203\321\201\321\201\320\272\320\270\320\271_ _ _\304\233_\303\241\303\261/\346\270\254\350\251\246_check file" "b/cmd/bisync/testdata/test_normalization/initial/\346\270\254\350\251\246_\320\240\321\203\321\201\321\201\320\272\320\270\320\271_ _ _\304\233_\303\241\303\261/\346\270\254\350\251\246_check file"
new file mode 100644
index 000000000..e69de29bb
diff --git a/cmd/bisync/testdata/test_normalization/modfiles/file1.txt b/cmd/bisync/testdata/test_normalization/modfiles/file1.txt
new file mode 100644
index 000000000..464147f09
--- /dev/null
+++ b/cmd/bisync/testdata/test_normalization/modfiles/file1.txt
@@ -0,0 +1 @@
+This file is newer
diff --git "a/cmd/bisync/testdata/test_normalization/modfiles/\346\270\254\350\251\246_filtersfile.txt" "b/cmd/bisync/testdata/test_normalization/modfiles/\346\270\254\350\251\246_filtersfile.txt"
new file mode 100644
index 000000000..c85f097fc
--- /dev/null
+++ "b/cmd/bisync/testdata/test_normalization/modfiles/\346\270\254\350\251\246_filtersfile.txt"
@@ -0,0 +1,5 @@
+# Test filters file
+# Note that this test checks for bisync's access to and usage of a filters file, not an extensive test of rclone's filter capability
+
+# Exclude fileZ.txt in root only. The copy in the subdir should be found and synched.
+- /fileZ.txt
diff --git a/cmd/bisync/testdata/test_normalization/scenario.txt b/cmd/bisync/testdata/test_normalization/scenario.txt
new file mode 100644
index 000000000..e44265fd2
--- /dev/null
+++ b/cmd/bisync/testdata/test_normalization/scenario.txt
@@ -0,0 +1,53 @@
+test normalization
+# Tests support for --no-unicode-normalization and --ignore-case-sync
+# note: this test is written carefully to be runnable regardless of case/unicode sensitivity
+# i.e. the results should be the same on linux and macOS
+
+# force specific modification time since file time is lost through git
+touch-copy 2001-01-02 {datadir/}file1.txt {path2/}
+test initial bisync
+bisync resync
+
+# copy NFC version to Path1
+copy-as-NFC {datadir/}file1.txt {path1/}測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö 測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö.txt
+copy-as-NFC {datadir/}file1.txt {path1/}folder éééö.txt
+copy-as-NFC {datadir/}file1.txt {path1/}folder HeLlO,wOrLd!.txt
+
+# place newer NFD version on Path2
+touch-copy 2001-01-03 {datadir/}file1.txt {path2/}
+copy-as-NFD {datadir/}file1.txt {path2/}folder éééö.txt
+copy-as-NFD {datadir/}file1.txt {path2/}folder hello,WORLD!.txt
+
+test bisync run with fix-case
+bisync fix-case
+
+# purge and reset
+purge-children {path1/}
+purge-children {path2/}
+touch-copy 2001-01-02 {datadir/}file1.txt {path2/}
+bisync resync
+
+# copy NFC version to Path1
+copy-as-NFC {datadir/}file1.txt {path1/}測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö 測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö.txt
+copy-as-NFC {datadir/}file1.txt {path1/}folder éééö.txt
+copy-as-NFC {datadir/}file1.txt {path1/}folder HeLlO,wOrLd!.txt
+
+# place newer NFD version on Path2
+touch-copy 2001-01-03 {datadir/}file1.txt {path2/}
+copy-as-NFD {datadir/}file1.txt {path2/}folder éééö.txt
+copy-as-NFD {datadir/}file1.txt {path2/}folder hello,WORLD!.txt
+
+test bisync run with normalization
+bisync norm force
+
+test resync
+bisync resync norm
+
+test changed on both paths
+touch-copy 2001-01-05 {datadir/}file1.txt {path2/}
+copy-as-NFC {datadir/}file1.txt {path1/}測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö 測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éééö.txt
+copy-as-NFC {datadir/}file1.txt {path1/}folder éééö.txt
+copy-as-NFC {datadir/}file1.txt {path1/}folder HeLlO,wOrLd!.txt
+copy-as-NFD {datadir/}file1.txt {path2/}folder éééö.txt
+copy-as-NFD {datadir/}file1.txt {path2/}folder hello,WORLD!.txt
+bisync norm
\ No newline at end of file
diff --git a/docs/content/bisync.md b/docs/content/bisync.md
index b01e82cc0..109010827 100644
--- a/docs/content/bisync.md
+++ b/docs/content/bisync.md
@@ -588,13 +588,26 @@ and refuses to run again until the user runs a `--resync` (unless using `--resil
The best workaround at the moment is to set any backend-specific flags in the [config file](/commands/rclone_config/)
instead of specifying them with command flags. (You can still override them as needed for other rclone commands.)
-### Case sensitivity
+### Case (and unicode) sensitivity {#case-sensitivity}
Synching with **case-insensitive** filesystems, such as Windows or `Box`,
-can result in file name conflicts. This will be fixed in a future release.
-The near-term workaround is to make sure that files on both sides
+can result in unusual behavior. As of `v1.65`, case and unicode form differences no longer cause critical errors,
+however they may cause unexpected delta outcomes, due to the delta engine still being case-sensitive.
+This will be fixed in a future release. The near-term workaround is to make sure that files on both sides
don't have spelling case differences (`Smile.jpg` vs. `smile.jpg`).
+The same limitation applies to Unicode normalization forms.
+This [particularly applies to macOS](https://github.com/rclone/rclone/issues/7270),
+which prefers NFD and sometimes auto-converts filenames from the NFC form used by most other platforms.
+This should no longer cause bisync to fail entirely, but may cause surprising delta results, as explained above.
+
+See the following options (all of which are supported by bisync) to control this behavior more granularly:
+- [`--fix-case`](/docs/#fix-case)
+- [`--ignore-case-sync`](/docs/#ignore-case-sync)
+- [`--no-unicode-normalization`](/docs/#no-unicode-normalization)
+- [`--local-unicode-normalization`](/local/#local-unicode-normalization) and
+[`--local-case-sensitive`](/local/#local-case-sensitive) (caution: these are normally not what you want.)
+
## Windows support {#windows}
Bisync has been tested on Windows 8.1, Windows 10 Pro 64-bit and on Windows
@@ -1278,6 +1291,7 @@ about _Unison_ and synchronization in general.
* A few basic terminal colors are now supported, controllable with [`--color`](/docs/#color-when) (`AUTO`|`NEVER`|`ALWAYS`)
* Initial listing snapshots of Path1 and Path2 are now generated concurrently, using the same "march" infrastructure as `check` and `sync`,
for performance improvements and less [risk of error](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=4.%20Listings%20should%20alternate%20between%20paths%20to%20minimize%20errors).
+* Better handling of unicode normalization and case insensitivity, support for [`--fix-case`](/docs/#fix-case), [`--ignore-case-sync`](/docs/#ignore-case-sync), [`--no-unicode-normalization`](/docs/#no-unicode-normalization)
### `v1.64`
* Fixed an [issue](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=1.%20Dry%20runs%20are%20not%20completely%20dry)