"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
bisync: refactor normalization code, fix deltas - fixes #7270
nielash98f539de8f91
diff --git a/cmd/bisync/bilib/names.go b/cmd/bisync/bilib/names.goindex 41c4692b5..3093dff55 100644--- a/cmd/bisync/bilib/names.go+++ b/cmd/bisync/bilib/names.go@@ -59,3 +59,27 @@ func SaveList(list []string, path string) error {}return os.WriteFile(path, buf.Bytes(), PermSecure)}++// AliasMap comprises a pair of names that are not equal but treated as equal for comparison purposes+// For example, when normalizing unicode and casing+// This helps reduce repeated normalization functions, which really slow things down+type AliasMap map[string]string++// Add adds new pair to the set, in both directions+func (am AliasMap) Add(name1, name2 string) {+ if name1 != name2 {+ am[name1] = name2+ am[name2] = name1+ }+}++// Alias returns the alternate version, if any, else the original.+func (am AliasMap) Alias(name1 string) string {+ // note: we don't need to check normalization settings, because we already did it in March.+ // the AliasMap will only exist if March paired up two unequal filenames.+ name2, ok := am[name1]+ if ok {+ return name2+ }+ return name1+}diff --git a/cmd/bisync/deltas.go b/cmd/bisync/deltas.goindex d0c0cd8aa..68fb538c3 100644--- a/cmd/bisync/deltas.go+++ b/cmd/bisync/deltas.go@@ -16,6 +16,7 @@ import ("github.com/rclone/rclone/fs/accounting""github.com/rclone/rclone/fs/filter""github.com/rclone/rclone/fs/operations"+ "golang.org/x/text/unicode/norm")// delta@@ -240,6 +241,9 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (changectxMove := b.opt.setDryRun(ctx)+ // update AliasMap for deleted files, as march does not know about them+ b.updateAliases(ctx, ds1, ds2)+// efficient isDir check// we load the listing just once and store only the dirsdirs1, dirs1Err := b.listDirsOnly(1)@@ -270,14 +274,24 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (changectxCheck, filterCheck := filter.AddConfig(ctxNew)for _, file := range ds1.sort() {+ alias := b.aliases.Alias(file)d1 := ds1.deltas[file]if d1.is(deltaOther) {- d2 := ds2.deltas[file]+ d2, in2 := ds2.deltas[file]+ if !in2 && file != alias {+ d2 = ds2.deltas[alias]+ }if d2.is(deltaOther) {- if err := filterCheck.AddFile(file); err != nil {- fs.Debugf(nil, "Non-critical error adding file to list of potential conflicts to check: %s", err)- } else {- fs.Debugf(nil, "Added file to list of potential conflicts to check: %s", file)+ checkit := func(filename string) {+ if err := filterCheck.AddFile(filename); err != nil {+ fs.Debugf(nil, "Non-critical error adding file to list of potential conflicts to check: %s", err)+ } else {+ fs.Debugf(nil, "Added file to list of potential conflicts to check: %s", filename)+ }+ }+ checkit(file)+ if file != alias {+ checkit(alias)}}}@@ -287,12 +301,17 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (changematches, err := b.checkconflicts(ctxCheck, filterCheck, b.fs1, b.fs2)for _, file := range ds1.sort() {+ alias := b.aliases.Alias(file)p1 := path1 + file- p2 := path2 + file+ p2 := path2 + aliasd1 := ds1.deltas[file]if d1.is(deltaOther) {d2, in2 := ds2.deltas[file]+ // try looking under alternate name+ if !in2 && file != alias {+ d2, in2 = ds2.deltas[alias]+ }if !in2 {b.indent("Path1", p2, "Queue copy to Path2")copy1to2.Add(file)@@ -304,15 +323,26 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (changeb.indent("!WARNING", file, "New or changed in both paths")//if files are identical, leave them alone instead of renaming- if dirs1.has(file) && dirs2.has(file) {+ if (dirs1.has(file) || dirs1.has(alias)) && (dirs2.has(file) || dirs2.has(alias)) {fs.Debugf(nil, "This is a directory, not a file. Skipping equality check and will not rename: %s", file)ls1.getPut(file, skippedDirs1)ls2.getPut(file, skippedDirs2)} else {equal := matches.Has(file)+ if !equal {+ equal = matches.Has(alias)+ }if equal {- fs.Infof(nil, "Files are equal! Skipping: %s", file)- renameSkipped.Add(file)+ if ciCheck.FixCase && file != alias {+ // the content is equal but filename still needs to be FixCase'd, so copy1to2+ // the Path1 version is deemed "correct" in this scenario+ fs.Infof(alias, "Files are equal but will copy anyway to fix case to %s", file)+ copy1to2.Add(file)+ } else {+ fs.Infof(nil, "Files are equal! Skipping: %s", file)+ renameSkipped.Add(file)+ renameSkipped.Add(alias)+ }} else {fs.Debugf(nil, "Files are NOT equal: %s", file)b.indent("!Path1", p1+"..path1", "Renaming Path1 copy")@@ -330,17 +360,17 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (changecopy1to2.Add(file + "..path1")b.indent("!Path2", p2+"..path2", "Renaming Path2 copy")- if err = operations.MoveFile(ctxMove, b.fs2, b.fs2, file+"..path2", file); err != nil {- err = fmt.Errorf("path2 rename failed for %s: %w", file, err)+ if err = operations.MoveFile(ctxMove, b.fs2, b.fs2, alias+"..path2", alias); err != nil {+ err = fmt.Errorf("path2 rename failed for %s: %w", alias, err)return}if b.opt.DryRun {- renameSkipped.Add(file)+ renameSkipped.Add(alias)} else {- renamed2.Add(file)+ renamed2.Add(alias)}b.indent("!Path2", p1+"..path2", "Queue copy to Path1")- copy2to1.Add(file + "..path2")+ copy2to1.Add(alias + "..path2")}}handled.Add(file)@@ -348,6 +378,15 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change} else {// Path1 deletedd2, in2 := ds2.deltas[file]+ // try looking under alternate name+ fs.Debugf(file, "alias: %s, in2: %v", alias, in2)+ if !in2 && file != alias {+ fs.Debugf(file, "looking for alias: %s", alias)+ d2, in2 = ds2.deltas[alias]+ if in2 {+ fs.Debugf(file, "detected alias: %s", alias)+ }+ }if !in2 {b.indent("Path2", p2, "Queue delete")delete2.Add(file)@@ -359,15 +398,17 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change} else if d2.is(deltaDeleted) {handled.Add(file)deletedonboth.Add(file)+ deletedonboth.Add(alias)}}}for _, file := range ds2.sort() {- p1 := path1 + file+ alias := b.aliases.Alias(file)+ p1 := path1 + aliasd2 := ds2.deltas[file]- if handled.Has(file) {+ if handled.Has(file) || handled.Has(alias) {continue}if d2.is(deltaOther) {@@ -381,17 +422,11 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (change}}- // find alternate names according to normalization settings- altNames1to2 := bilib.Names{}- altNames2to1 := bilib.Names{}- b.findAltNames(ctx, b.fs1, copy2to1, b.newListing1, altNames2to1)- b.findAltNames(ctx, b.fs2, copy1to2, b.newListing2, altNames1to2)-// Do the batch operationif copy2to1.NotEmpty() {changes1 = trueb.indent("Path2", "Path1", "Do queued copies to")- results2to1, err = b.fastCopy(ctx, b.fs2, b.fs1, copy2to1, "copy2to1", altNames2to1)+ results2to1, err = b.fastCopy(ctx, b.fs2, b.fs1, copy2to1, "copy2to1")if err != nil {return}@@ -403,7 +438,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (changeif copy1to2.NotEmpty() {changes2 = trueb.indent("Path1", "Path2", "Do queued copies to")- results1to2, err = b.fastCopy(ctx, b.fs1, b.fs2, copy1to2, "copy1to2", altNames1to2)+ results1to2, err = b.fastCopy(ctx, b.fs1, b.fs2, copy1to2, "copy1to2")if err != nil {return}@@ -458,3 +493,65 @@ func (ds *deltaSet) excessDeletes() bool {maxDelete, ds.deleted, ds.oldCount, ds.msg, quotePath(bilib.FsPath(ds.fs)))return true}++// normally we build the AliasMap from march results,+// however, march does not know about deleted files, so need to manually check them for aliases+func (b *bisyncRun) updateAliases(ctx context.Context, ds1, ds2 *deltaSet) {+ ci := fs.GetConfig(ctx)+ // skip if not needed+ if ci.NoUnicodeNormalization && !ci.IgnoreCaseSync && !b.fs1.Features().CaseInsensitive && !b.fs2.Features().CaseInsensitive {+ return+ }+ if ds1.deleted < 1 && ds2.deleted < 1 {+ return+ }++ fs.Debugf(nil, "Updating AliasMap")++ transform := func(s string) string {+ if !ci.NoUnicodeNormalization {+ s = norm.NFC.String(s)+ }+ // note: march only checks the dest, but we check both here+ if ci.IgnoreCaseSync || b.fs1.Features().CaseInsensitive || b.fs2.Features().CaseInsensitive {+ s = strings.ToLower(s)+ }+ return s+ }++ delMap1 := map[string]string{} // [transformedname]originalname+ delMap2 := map[string]string{} // [transformedname]originalname+ fullMap1 := map[string]string{} // [transformedname]originalname+ fullMap2 := map[string]string{} // [transformedname]originalname++ for _, name := range ls1.list {+ fullMap1[transform(name)] = name+ }+ for _, name := range ls2.list {+ fullMap2[transform(name)] = name+ }++ addDeletes := func(ds *deltaSet, delMap, fullMap map[string]string) {+ for _, file := range ds.sort() {+ d := ds.deltas[file]+ if d.is(deltaDeleted) {+ delMap[transform(file)] = file+ fullMap[transform(file)] = file+ }+ }+ }+ addDeletes(ds1, delMap1, fullMap1)+ addDeletes(ds2, delMap2, fullMap2)++ addAliases := func(delMap, fullMap map[string]string) {+ for transformedname, name := range delMap {+ matchedName, found := fullMap[transformedname]+ if found && name != matchedName {+ fs.Debugf(name, "adding alias %s", matchedName)+ b.aliases.Add(name, matchedName)+ }+ }+ }+ addAliases(delMap1, fullMap2)+ addAliases(delMap2, fullMap1)+}diff --git a/cmd/bisync/listing.go b/cmd/bisync/listing.goindex 70a2aaf5f..5c44101f0 100644--- a/cmd/bisync/listing.go+++ b/cmd/bisync/listing.go@@ -5,6 +5,7 @@ package bisyncimport ("bufio""context"+ "errors""fmt""io""os"@@ -20,7 +21,6 @@ import ("github.com/rclone/rclone/fs/hash""github.com/rclone/rclone/fs/operations""golang.org/x/exp/slices"- "golang.org/x/text/unicode/norm")// ListingHeader defines first line of a listing@@ -407,18 +407,6 @@ func ConvertPrecision(Modtime time.Time, dst fs.Fs) time.Time {return Modtime}-// ApplyTransforms handles unicode and case normalization-func ApplyTransforms(ctx context.Context, dst fs.Fs, s string) string {- ci := fs.GetConfig(ctx)- if !ci.NoUnicodeNormalization {- s = norm.NFC.String(s)- }- if ci.IgnoreCaseSync || dst.Features().CaseInsensitive {- s = strings.ToLower(s)- }- return s-}-// modifyListing will modify the listing based on the results of the syncfunc (b *bisyncRun) modifyListing(ctx context.Context, src fs.Fs, dst fs.Fs, results []Results, queues queues, is1to2 bool) (err error) {queue := queues.copy2to1@@ -448,18 +436,6 @@ func (b *bisyncRun) modifyListing(ctx context.Context, src fs.Fs, dst fs.Fs, ressrcList.hash = src.Hashes().GetOne()dstList.hash = dst.Hashes().GetOne()}- dstListNew, err := b.loadListing(dstListing + "-new")- if err != nil {- return fmt.Errorf("cannot read new listing: %w", err)- }- // for resync only, dstListNew will be empty, so need to use results instead- if b.opt.Resync {- for _, result := range results {- if result.Name != "" && result.IsDst {- dstListNew.put(result.Name, result.Size, result.Modtime, result.Hash, "-", result.Flags)- }- }- }srcWinners := newFileList()dstWinners := newFileList()@@ -471,6 +447,10 @@ func (b *bisyncRun) modifyListing(ctx context.Context, src fs.Fs, dst fs.Fs, rescontinue}+ if result.AltName != "" {+ b.aliases.Add(result.Name, result.AltName)+ }+if result.Flags == "d" && !b.opt.CreateEmptySrcDirs {continue}@@ -496,6 +476,7 @@ func (b *bisyncRun) modifyListing(ctx context.Context, src fs.Fs, dst fs.Fs, res}}+ ci := fs.GetConfig(ctx)updateLists := func(side string, winners, list *fileList) {for _, queueFile := range queue.ToList() {if !winners.has(queueFile) && list.has(queueFile) && !errors.has(queueFile) {@@ -506,28 +487,17 @@ func (b *bisyncRun) modifyListing(ctx context.Context, src fs.Fs, dst fs.Fs, res// copies to sidenew := winners.get(queueFile)- // handle normalization according to settings- ci := fs.GetConfig(ctx)- if side == "dst" && (!ci.NoUnicodeNormalization || ci.IgnoreCaseSync || dst.Features().CaseInsensitive) {- // search list for existing file that matches queueFile when normalized- normalizedName := ApplyTransforms(ctx, dst, queueFile)- matchFound := false- matchedName := ""- for _, filename := range dstListNew.list {- if ApplyTransforms(ctx, dst, filename) == normalizedName {- matchFound = true- matchedName = filename // original, not normalized- break- }- }- if matchFound && matchedName != queueFile {+ // handle normalization+ if side == "dst" {+ alias := b.aliases.Alias(queueFile)+ if alias != queueFile {// use the (non-identical) existing name, unless --fix-caseif ci.FixCase {- fs.Debugf(direction, "removing %s and adding %s as --fix-case was specified", matchedName, queueFile)- list.remove(matchedName)+ fs.Debugf(direction, "removing %s and adding %s as --fix-case was specified", alias, queueFile)+ list.remove(alias)} else {- fs.Debugf(direction, "casing/unicode difference detected. using %s instead of %s", matchedName, queueFile)- queueFile = matchedName+ fs.Debugf(direction, "casing/unicode difference detected. using %s instead of %s", alias, queueFile)+ queueFile = alias}}}@@ -613,6 +583,16 @@ func (b *bisyncRun) modifyListing(ctx context.Context, src fs.Fs, dst fs.Fs, res}if filterRecheck.HaveFilesFrom() {+ // also include any aliases+ recheckFiles := filterRecheck.Files()+ for recheckFile := range recheckFiles {+ alias := b.aliases.Alias(recheckFile)+ if recheckFile != alias {+ if err := filterRecheck.AddFile(alias); err != nil {+ fs.Debugf(alias, "error adding file to recheck filter: %v", err)+ }+ }+ }b.recheck(ctxRecheck, src, dst, srcList, dstList, is1to2)}@@ -661,7 +641,7 @@ func (b *bisyncRun) recheck(ctxRecheck context.Context, src, dst fs.Fs, srcList,for _, srcObj := range srcObjs {fs.Debugf(srcObj, "rechecking")for _, dstObj := range dstObjs {- if srcObj.Remote() == dstObj.Remote() {+ if srcObj.Remote() == dstObj.Remote() || srcObj.Remote() == b.aliases.Alias(dstObj.Remote()) {if operations.Equal(ctxRecheck, srcObj, dstObj) || b.opt.DryRun {putObj(srcObj, src, srcList)putObj(dstObj, dst, dstList)@@ -673,8 +653,13 @@ func (b *bisyncRun) recheck(ctxRecheck context.Context, src, dst fs.Fs, srcList,}// if srcObj not resolved by now (either because no dstObj match or files not equal),// roll it back to old version, so it gets retried next time.+ // skip and error during --resync, as rollback is not possibleif !slices.Contains(resolved, srcObj.Remote()) && !b.opt.DryRun {- toRollback = append(toRollback, srcObj.Remote())+ if b.opt.Resync {+ b.handleErr(srcObj, "Unable to rollback during --resync", errors.New("no dstObj match or files not equal"), true, false)+ } else {+ toRollback = append(toRollback, srcObj.Remote())+ }}}if len(toRollback) > 0 {@@ -683,6 +668,9 @@ func (b *bisyncRun) recheck(ctxRecheck context.Context, src, dst fs.Fs, srcList,b.handleErr(oldSrc, "error loading old src listing", err, true, true)oldDst, err := b.loadListing(dstListing + "-old")b.handleErr(oldDst, "error loading old dst listing", err, true, true)+ if b.critical {+ return+ }for _, item := range toRollback {rollback(item, oldSrc, srcList)diff --git a/cmd/bisync/march.go b/cmd/bisync/march.goindex 6e8c5bf81..66d8fdb05 100644--- a/cmd/bisync/march.go+++ b/cmd/bisync/march.go@@ -15,6 +15,7 @@ var ls1 = newFileList()var ls2 = newFileList()var err errorvar firstErr error+var marchAliasLock sync.Mutexvar marchLsLock sync.Mutexvar marchErrLock sync.Mutexvar marchCtx context.Context@@ -77,6 +78,9 @@ func (b *bisyncRun) DstOnly(o fs.DirEntry) (recurse bool) {// Match is called when object exists on both path1 and path2 (whether equal or not)func (b *bisyncRun) Match(ctx context.Context, o2, o1 fs.DirEntry) (recurse bool) {fs.Debugf(o1, "both path1 and path2")+ marchAliasLock.Lock()+ b.aliases.Add(o1.Remote(), o2.Remote())+ marchAliasLock.Unlock()b.parse(o1, true)b.parse(o2, false)return isDir(o1)diff --git a/cmd/bisync/operations.go b/cmd/bisync/operations.goindex 70f759343..00d879dbc 100644--- a/cmd/bisync/operations.go+++ b/cmd/bisync/operations.go@@ -36,6 +36,7 @@ type bisyncRun struct {listing2 stringnewListing1 stringnewListing2 string+ aliases bilib.AliasMapopt *Optionsoctx context.Contextfctx context.Context@@ -90,6 +91,7 @@ func Bisync(ctx context.Context, fs1, fs2 fs.Fs, optArg *Options) (err error) {b.listing2 = b.basePath + ".path2.lst"b.newListing1 = b.listing1 + "-new"b.newListing2 = b.listing2 + "-new"+ b.aliases = bilib.AliasMap{}// Handle lock filelockFile := ""@@ -448,7 +450,7 @@ func (b *bisyncRun) resync(octx, fctx context.Context) error {}fs.Infof(nil, "Resync updating listings")- b.saveOldListings()+ b.saveOldListings() // may not exist, as this is --resyncb.replaceCurrentListings()resultsToQueue := func(results []Results) bilib.Names {@@ -496,31 +498,15 @@ func (b *bisyncRun) checkSync(listing1, listing2 string) error {return fmt.Errorf("cannot read prior listing of Path2: %w", err)}- transformList := func(files *fileList, fs fs.Fs) *fileList {- transformed := newFileList()- for _, file := range files.list {- f := files.get(file)- transformed.put(ApplyTransforms(b.fctx, fs, file), f.size, f.time, f.hash, f.id, f.flags)- }- return transformed- }-- files1Transformed := transformList(files1, b.fs1)- files2Transformed := transformList(files2, b.fs2)-- // DEBUG- fs.Debugf(nil, "files1Transformed: %v", files1Transformed)- fs.Debugf(nil, "files2Transformed: %v", files2Transformed)-ok := truefor _, file := range files1.list {- if !files2.has(file) && !files2Transformed.has(ApplyTransforms(b.fctx, b.fs1, file)) {+ if !files2.has(file) && !files2.has(b.aliases.Alias(file)) {b.indent("ERROR", file, "Path1 file not found in Path2")ok = false}}for _, file := range files2.list {- if !files1.has(file) && !files1Transformed.has(ApplyTransforms(b.fctx, b.fs2, file)) {+ if !files1.has(file) && !files1.has(b.aliases.Alias(file)) {b.indent("ERROR", file, "Path2 file not found in Path1")ok = false}@@ -580,7 +566,7 @@ func (b *bisyncRun) handleErr(o interface{}, msg string, err error, critical, reb.critical = truefs.Errorf(o, "%s: %v", msg, err)} else {- fs.Debugf(o, "%s: %v", msg, err)+ fs.Infof(o, "%s: %v", msg, err)}}}diff --git a/cmd/bisync/queue.go b/cmd/bisync/queue.goindex 12bcb9b69..f08a82f89 100644--- a/cmd/bisync/queue.go+++ b/cmd/bisync/queue.go@@ -22,6 +22,7 @@ type Results struct {Src stringDst stringName string+ AltName stringSize int64Modtime time.TimeHash string@@ -58,6 +59,21 @@ func resultName(result Results, side, src, dst fs.DirEntry) string {return ""}+// returns the opposite side's name, only if different+func altName(name string, src, dst fs.DirEntry) string {+ if src != nil && dst != nil {+ if src.Remote() != dst.Remote() {+ switch name {+ case src.Remote():+ return dst.Remote()+ case dst.Remote():+ return src.Remote()+ }+ }+ }+ return ""+}+// WriteResults is Bisync's LoggerFnfunc WriteResults(ctx context.Context, sigil operations.Sigil, src, dst fs.DirEntry, err error) {lock.Lock()@@ -77,6 +93,7 @@ func WriteResults(ctx context.Context, sigil operations.Sigil, src, dst fs.DirEnfor i, side := range fss {result.Name = resultName(result, side, src, dst)+ result.AltName = altName(result.Name, src, dst)result.IsSrc = i == 0result.IsDst = i == 1result.Flags = "-"@@ -129,7 +146,7 @@ func ReadResults(results io.Reader) []Results {return slice}-func (b *bisyncRun) fastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib.Names, queueName string, altNames bilib.Names) ([]Results, error) {+func (b *bisyncRun) fastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib.Names, queueName string) ([]Results, error) {if err := b.saveQueue(files, queueName); err != nil {return nil, err}@@ -139,10 +156,9 @@ func (b *bisyncRun) fastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib.if err := filterCopy.AddFile(file); err != nil {return nil, err}- }- if altNames.NotEmpty() {- for _, file := range altNames.ToList() {- if err := filterCopy.AddFile(file); err != nil {+ alias := b.aliases.Alias(file)+ if alias != file {+ if err := filterCopy.AddFile(alias); err != nil {return nil, err}}@@ -249,22 +265,3 @@ func (b *bisyncRun) saveQueue(files bilib.Names, jobName string) error {queueFile := fmt.Sprintf("%s.%s.que", b.basePath, jobName)return files.Save(queueFile)}--func (b *bisyncRun) findAltNames(ctx context.Context, dst fs.Fs, queue bilib.Names, newListing string, altNames bilib.Names) {- ci := fs.GetConfig(ctx)- if queue.NotEmpty() && (!ci.NoUnicodeNormalization || ci.IgnoreCaseSync || b.fs1.Features().CaseInsensitive || b.fs2.Features().CaseInsensitive) {- // search list for existing file that matches queueFile when normalized- for _, queueFile := range queue.ToList() {- normalizedName := ApplyTransforms(ctx, dst, queueFile)- candidates, err := b.loadListing(newListing)- if err != nil {- fs.Errorf(candidates, "cannot read new listing: %v", err)- }- for _, filename := range candidates.list {- if ApplyTransforms(ctx, dst, filename) == normalizedName && filename != queueFile {- altNames.Add(filename) // original, not normalized- }- }- }- }-}diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.copy2to1.que b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.copy2to1.queindex 85deec9f9..029f3e190 100644--- a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.copy2to1.que+++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.copy2to1.que@@ -1,3 +1 @@"file1.txt"-"folder/hello,WORLD!.txt"-"folder/éééö.txt"diff --git a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-new b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-newindex 225fe66c1..9afa4b347 100644--- a/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-new+++ b/cmd/bisync/testdata/test_normalization/golden/_testdir_path1.._testdir_path2.path2.lst-new@@ -1,5 +1,5 @@# bisync listing v1 from test- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-05T00:00:00.000000000+0000 "file1.txt"-- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-05T00:00:00.000000000+0000 "folder/hello,WORLD!.txt"-- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-05T00:00:00.000000000+0000 "folder/éééö.txt"+- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-03T00:00:00.000000000+0000 "folder/hello,WORLD!.txt"+- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-03T00:00:00.000000000+0000 "folder/éééö.txt"- 19 md5:7fe98ed88552b828777d8630900346b8 - 2001-01-02T00:00:00.000000000+0000 "測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"diff --git a/cmd/bisync/testdata/test_normalization/golden/test.log b/cmd/bisync/testdata/test_normalization/golden/test.logindex 54045e130..db66b5e75 100644--- a/cmd/bisync/testdata/test_normalization/golden/test.log+++ b/cmd/bisync/testdata/test_normalization/golden/test.log@@ -36,16 +36,20 @@ INFO : - [34mPath2[0m [35mFile is new[0m - [36mfINFO : - [34mPath2[0m [35mFile is new[0m - [36mfolder/hello,WORLD!.txt[0mINFO : Path2: 3 changes: 2 new, 1 newer, 0 older, 0 deletedINFO : Applying changes-INFO : - [34mPath1[0m [35mQueue copy to Path2[0m - [36m{path2/}folder/HeLlO,wOrLd!.txt[0m-INFO : - [34mPath1[0m [35mQueue copy to Path2[0m - [36m{path2/}folder/éééö.txt[0m+INFO : Checking potential conflicts...+NOTICE: Local file system at {path2}: 0 differences found+NOTICE: Local file system at {path2}: 2 matching files+INFO : Finished checking the potential conflicts. %!s(<nil>)+NOTICE: - [34mWARNING[0m [35mNew or changed in both paths[0m - [36mfolder/HeLlO,wOrLd!.txt[0m+INFO : folder/hello,WORLD!.txt: Files are equal but will copy anyway to fix case to folder/HeLlO,wOrLd!.txt+NOTICE: - [34mWARNING[0m [35mNew or changed in both paths[0m - [36mfolder/éééö.txt[0m+INFO : folder/éééö.txt: Files are equal but will copy anyway to fix case to folder/éééö.txtINFO : - [34mPath1[0m [35mQueue copy to Path2[0m - [36m"{path2/}測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"[0mINFO : - [34mPath2[0m [35mQueue copy to Path1[0m - [36m{path1/}file1.txt[0m-INFO : - [34mPath2[0m [35mQueue copy to Path1[0m - [36m{path1/}folder/éééö.txt[0m-INFO : - [34mPath2[0m [35mQueue copy to Path1[0m - [36m{path1/}folder/hello,WORLD!.txt[0mINFO : - [34mPath2[0m [35mDo queued copies to[0m - [36mPath1[0m-INFO : folder/HeLlO,wOrLd!.txt: Fixed case by renaming to: folder/hello,WORLD!.txt-INFO : folder/éééö.txt: Fixed case by renaming to: folder/éééö.txtINFO : - [34mPath1[0m [35mDo queued copies to[0m - [36mPath2[0m+INFO : folder/hello,WORLD!.txt: Fixed case by renaming to: folder/HeLlO,wOrLd!.txt+INFO : folder/éééö.txt: Fixed case by renaming to: folder/éééö.txtINFO : Updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : [32mBisync successful[0m@@ -87,12 +91,16 @@ INFO : - [34mPath2[0m [35mFile is new[0m - [36mfINFO : - [34mPath2[0m [35mFile is new[0m - [36mfolder/hello,WORLD!.txt[0mINFO : Path2: 3 changes: 2 new, 1 newer, 0 older, 0 deletedINFO : Applying changes-INFO : - [34mPath1[0m [35mQueue copy to Path2[0m - [36m{path2/}folder/HeLlO,wOrLd!.txt[0m-INFO : - [34mPath1[0m [35mQueue copy to Path2[0m - [36m{path2/}folder/éééö.txt[0m+INFO : Checking potential conflicts...+NOTICE: Local file system at {path2}: 0 differences found+NOTICE: Local file system at {path2}: 2 matching files+INFO : Finished checking the potential conflicts. %!s(<nil>)+NOTICE: - [34mWARNING[0m [35mNew or changed in both paths[0m - [36mfolder/HeLlO,wOrLd!.txt[0m+INFO : Files are equal! Skipping: folder/HeLlO,wOrLd!.txt+NOTICE: - [34mWARNING[0m [35mNew or changed in both paths[0m - [36mfolder/éééö.txt[0m+INFO : Files are equal! Skipping: folder/éééö.txtINFO : - [34mPath1[0m [35mQueue copy to Path2[0m - [36m"{path2/}測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"[0mINFO : - [34mPath2[0m [35mQueue copy to Path1[0m - [36m{path1/}file1.txt[0m-INFO : - [34mPath2[0m [35mQueue copy to Path1[0m - [36m{path1/}folder/éééö.txt[0m-INFO : - [34mPath2[0m [35mQueue copy to Path1[0m - [36m{path1/}folder/hello,WORLD!.txt[0mINFO : - [34mPath2[0m [35mDo queued copies to[0m - [36mPath1[0mINFO : - [34mPath1[0m [35mDo queued copies to[0m - [36mPath2[0mINFO : Updating listings@@ -108,14 +116,12 @@ INFO : - [34mPath1[0m [35mResync is copying UNIQUE OR DIFFERING files toINFO : Resync updating listingsINFO : [32mBisync successful[0m-[36m(27) :[0m [34mtest changed on both paths[0m+[36m(27) :[0m [34mtest changed on one path[0m[36m(28) :[0m [34mtouch-copy 2001-01-05 {datadir/}file1.txt {path2/}[0m[36m(29) :[0m [34mcopy-as-NFC {datadir/}file1.txt {path1/}測試_Русский___ě_áñ👸🏼🧝🏾♀️💆🏿♂️🐨🤙🏼🤮🧑🏻🔧🧑🔬éééö 測試_Русский___ě_áñ👸🏼🧝🏾♀️💆🏿♂️🐨🤙🏼🤮🧑🏻🔧🧑🔬éééö.txt[0m[36m(30) :[0m [34mcopy-as-NFC {datadir/}file1.txt {path1/}folder éééö.txt[0m[36m(31) :[0m [34mcopy-as-NFC {datadir/}file1.txt {path1/}folder HeLlO,wOrLd!.txt[0m-[36m(32) :[0m [34mcopy-as-NFD {datadir/}file1.txt {path2/}folder éééö.txt[0m-[36m(33) :[0m [34mcopy-as-NFD {datadir/}file1.txt {path2/}folder hello,WORLD!.txt[0m-[36m(34) :[0m [34mbisync norm[0m+[36m(32) :[0m [34mbisync norm[0mINFO : Synching Path1 "{path1/}" with Path2 "{path2/}"INFO : Building Path1 and Path2 listingsINFO : Path1 checking for diffs@@ -125,16 +131,12 @@ INFO : - [34mPath1[0m [35mFile is newer[0m - [36m"INFO : Path1: 3 changes: 0 new, 3 newer, 0 older, 0 deletedINFO : Path2 checking for diffsINFO : - [34mPath2[0m [35mFile is newer[0m - [36mfile1.txt[0m-INFO : - [34mPath2[0m [35mFile is newer[0m - [36mfolder/éééö.txt[0m-INFO : - [34mPath2[0m [35mFile is newer[0m - [36mfolder/hello,WORLD!.txt[0m-INFO : Path2: 3 changes: 0 new, 3 newer, 0 older, 0 deleted+INFO : Path2: 1 changes: 0 new, 1 newer, 0 older, 0 deletedINFO : Applying changes-INFO : - [34mPath1[0m [35mQueue copy to Path2[0m - [36m{path2/}folder/HeLlO,wOrLd!.txt[0m+INFO : - [34mPath1[0m [35mQueue copy to Path2[0m - [36m{path2/}folder/hello,WORLD!.txt[0mINFO : - [34mPath1[0m [35mQueue copy to Path2[0m - [36m{path2/}folder/éééö.txt[0mINFO : - [34mPath1[0m [35mQueue copy to Path2[0m - [36m"{path2/}測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö/測試_Русский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éééö.txt"[0mINFO : - [34mPath2[0m [35mQueue copy to Path1[0m - [36m{path1/}file1.txt[0m-INFO : - [34mPath2[0m [35mQueue copy to Path1[0m - [36m{path1/}folder/éééö.txt[0m-INFO : - [34mPath2[0m [35mQueue copy to Path1[0m - [36m{path1/}folder/hello,WORLD!.txt[0mINFO : - [34mPath2[0m [35mDo queued copies to[0m - [36mPath1[0mINFO : - [34mPath1[0m [35mDo queued copies to[0m - [36mPath2[0mINFO : Updating listingsdiff --git a/cmd/bisync/testdata/test_normalization/scenario.txt b/cmd/bisync/testdata/test_normalization/scenario.txtindex e44265fd2..bc42bbdb5 100644--- a/cmd/bisync/testdata/test_normalization/scenario.txt+++ b/cmd/bisync/testdata/test_normalization/scenario.txt@@ -43,11 +43,9 @@ bisync norm forcetest resyncbisync resync norm-test changed on both paths+test changed on one pathtouch-copy 2001-01-05 {datadir/}file1.txt {path2/}copy-as-NFC {datadir/}file1.txt {path1/}測試_Русский___ě_áñ👸🏼🧝🏾♀️💆🏿♂️🐨🤙🏼🤮🧑🏻🔧🧑🔬éééö 測試_Русский___ě_áñ👸🏼🧝🏾♀️💆🏿♂️🐨🤙🏼🤮🧑🏻🔧🧑🔬éééö.txtcopy-as-NFC {datadir/}file1.txt {path1/}folder éééö.txtcopy-as-NFC {datadir/}file1.txt {path1/}folder HeLlO,wOrLd!.txt-copy-as-NFD {datadir/}file1.txt {path2/}folder éééö.txt-copy-as-NFD {datadir/}file1.txt {path2/}folder hello,WORLD!.txtbisync norm\ No newline at end of filediff --git a/docs/content/bisync.md b/docs/content/bisync.mdindex 165e73db2..bfde514cd 100644--- a/docs/content/bisync.md+++ b/docs/content/bisync.md@@ -591,17 +591,8 @@ instead of specifying them with command flags. (You can still override them as n### Case (and unicode) sensitivity {#case-sensitivity}-Synching with **case-insensitive** filesystems, such as Windows or `Box`,-can result in unusual behavior. As of `v1.65`, case and unicode form differences no longer cause critical errors,-however they may cause unexpected delta outcomes, due to the delta engine still being case-sensitive.-This will be fixed in a future release. The near-term workaround is to make sure that files on both sides-don't have spelling case differences (`Smile.jpg` vs. `smile.jpg`).--The same limitation applies to Unicode normalization forms.-This [particularly applies to macOS](https://github.com/rclone/rclone/issues/7270),-which prefers NFD and sometimes auto-converts filenames from the NFC form used by most other platforms.-This should no longer cause bisync to fail entirely, but may cause surprising delta results, as explained above.-+As of `v1.65`, case and unicode form differences no longer cause critical errors,+and normalization (when comparing between filesystems) is handled according to the same flags and defaults as `rclone sync`.See the following options (all of which are supported by bisync) to control this behavior more granularly:- [`--fix-case`](/docs/#fix-case)- [`--ignore-case-sync`](/docs/#ignore-case-sync)@@ -609,6 +600,10 @@ See the following options (all of which are supported by bisync) to control this- [`--local-unicode-normalization`](/local/#local-unicode-normalization) and[`--local-case-sensitive`](/local/#local-case-sensitive) (caution: these are normally not what you want.)+Note that in the (probably rare) event that `--fix-case` is used AND a file is new/changed on both sides+AND the checksums match AND the filename case does not match, the Path1 filename is considered the winner,+for the purposes of `--fix-case` (Path2 will be renamed to match it).+## Windows support {#windows}Bisync has been tested on Windows 8.1, Windows 10 Pro 64-bit and on Windows@@ -1292,7 +1287,7 @@ about _Unison_ and synchronization in general.* A few basic terminal colors are now supported, controllable with [`--color`](/docs/#color-when) (`AUTO`|`NEVER`|`ALWAYS`)* Initial listing snapshots of Path1 and Path2 are now generated concurrently, using the same "march" infrastructure as `check` and `sync`,for performance improvements and less [risk of error](https://forum.rclone.org/t/bisync-bugs-and-feature-requests/37636#:~:text=4.%20Listings%20should%20alternate%20between%20paths%20to%20minimize%20errors).-* Better handling of unicode normalization and case insensitivity, support for [`--fix-case`](/docs/#fix-case), [`--ignore-case-sync`](/docs/#ignore-case-sync), [`--no-unicode-normalization`](/docs/#no-unicode-normalization)+* Fixed handling of unicode normalization and case insensitivity, support for [`--fix-case`](/docs/#fix-case), [`--ignore-case-sync`](/docs/#ignore-case-sync), [`--no-unicode-normalization`](/docs/#no-unicode-normalization)* `--resync` is now much more efficient (especially for users of `--create-empty-src-dirs`)### `v1.64`