Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

sync: don't set dir modtimes if already set

nielash8c69455c3796
diff --git a/cmd/bisync/bisync_test.go b/cmd/bisync/bisync_test.go
index 0b333b51d..c4896f65c 100644
--- a/cmd/bisync/bisync_test.go
+++ b/cmd/bisync/bisync_test.go
@@ -34,6 +34,7 @@ import (
"github.com/rclone/rclone/fs/object"
"github.com/rclone/rclone/fs/operations"
"github.com/rclone/rclone/fs/sync"
+ "github.com/rclone/rclone/fs/walk"
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/lib/atexit"
"github.com/rclone/rclone/lib/encoder"
@@ -58,6 +59,8 @@ const (
fixSlash = (runtime.GOOS == "windows")
)
+var initDate = time.Date(2000, time.January, 1, 0, 0, 0, 0, bisync.TZ)
+
// logReplacements make modern test logs comparable with golden dir.
// It is a string slice of even length with this structure:
//
@@ -87,9 +90,9 @@ var logReplacements = []string{
`^.*? for same-side diffs on .*?$`, dropMe,
`^.*?Downloading hashes.*?$`, dropMe,
// ignore timestamps in directory time updates
- `^(INFO : .*?: Made directory with (metadata|modification time)).*$`, `$1`,
+ `^(INFO : .*?: (Made directory with|Set directory) (metadata|modification time)).*$`, dropMe,
// ignore sizes in directory time updates
- `^(NOTICE: .*?: Skipped set directory modification time as --dry-run is set).*$`, `$1`,
+ `^(NOTICE: .*?: Skipped set directory modification time as --dry-run is set).*$`, dropMe,
}
// Some dry-run messages differ depending on the particular remote.
@@ -320,7 +323,6 @@ func (b *bisyncTest) runTestCase(ctx context.Context, t *testing.T, testCase str
// For test stability, jam initial dates to a fixed past date.
// Test cases that change files will touch specific files to fixed new dates.
- initDate := time.Date(2000, time.January, 1, 0, 0, 0, 0, bisync.TZ)
err = filepath.Walk(b.initDir, func(path string, info os.FileInfo, err error) error {
if err == nil && !info.IsDir() {
return os.Chtimes(path, initDate, initDate)
@@ -868,6 +870,20 @@ func (b *bisyncTest) runBisync(ctx context.Context, args []string) (err error) {
}
}
+ // set all dirs to a fixed date for test stability, as they are considered as of v1.66.
+ jamDirTimes := func(f fs.Fs) {
+ err := walk.ListR(ctx, f, "", true, -1, walk.ListDirs, func(entries fs.DirEntries) error {
+ var err error
+ entries.ForDir(func(dir fs.Directory) {
+ _, err = operations.SetDirModTime(ctx, f, dir, "", initDate)
+ })
+ return err
+ })
+ assert.NoError(b.t, err, "error jamming dirtimes")
+ }
+ jamDirTimes(fs1)
+ jamDirTimes(fs2)
+
output := bilib.CaptureOutput(func() {
err = bisync.Bisync(octx, fs1, fs2, opt)
})
@@ -1385,10 +1401,10 @@ func (b *bisyncTest) mangleListing(text string, golden bool, file string) string
lines[i] = match[1] + "-" + match[3] + match[4] // replace it with "-" for comparison purposes (see #5679)
}
// account for modtime precision
- var lineRegex = regexp.MustCompile(`^(\S) +(-?\d+) (\S+) (\S+) (\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{9}[+-]\d{4}) (".+")$`)
+ lineRegex := regexp.MustCompile(`^(\S) +(-?\d+) (\S+) (\S+) (\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{9}[+-]\d{4}) (".+")$`)
const timeFormat = "2006-01-02T15:04:05.000000000-0700"
const lineFormat = "%s %8d %s %s %s %q\n"
- var TZ = time.UTC
+ TZ := time.UTC
fields := lineRegex.FindStringSubmatch(strings.TrimSuffix(lines[i], "\n"))
if fields != nil {
sizeVal, sizeErr := strconv.ParseInt(fields[2], 10, 64)
@@ -1509,7 +1525,8 @@ func (b *bisyncTest) listDir(dir string) (names []string) {
ignoreIt := func(file string) bool {
ignoreList := []string{
// ".lst-control", ".lst-dry-control", ".lst-old", ".lst-dry-old",
- ".DS_Store"}
+ ".DS_Store",
+ }
for _, s := range ignoreList {
if strings.Contains(file, s) {
return true
diff --git a/cmd/bisync/testdata/test_all_changed/golden/test.log b/cmd/bisync/testdata/test_all_changed/golden/test.log
index 39c281bcb..cb74904c9 100644
--- a/cmd/bisync/testdata/test_all_changed/golden/test.log
+++ b/cmd/bisync/testdata/test_all_changed/golden/test.log
@@ -16,11 +16,7 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -63,8 +59,6 @@ INFO : - Path1 Queue copy to Path2 - {
INFO : - Path1 Queue copy to Path2 - {path2/}file1.txt
INFO : - Path1 Queue copy to Path2 - {path2/}subdir/file20.txt
INFO : - Path1 Do queued copies to - Path2
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -139,8 +133,6 @@ INFO : - Path1 Queue copy to Path2 - {
INFO : - Path1 Queue copy to Path2 - {path2/}file1.txt
INFO : - Path1 Queue copy to Path2 - {path2/}subdir/file20.txt
INFO : - Path1 Do queued copies to - Path2
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
diff --git a/cmd/bisync/testdata/test_basic/golden/test.log b/cmd/bisync/testdata/test_basic/golden/test.log
index 912423c26..d3086e790 100644
--- a/cmd/bisync/testdata/test_basic/golden/test.log
+++ b/cmd/bisync/testdata/test_basic/golden/test.log
@@ -16,11 +16,7 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -57,8 +53,6 @@ INFO : - Path1 Queue copy to Path2 - {
INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt
INFO : - Path2 Do queued copies to - Path1
INFO : - Path1 Do queued copies to - Path2
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
diff --git a/cmd/bisync/testdata/test_check_access/golden/test.log b/cmd/bisync/testdata/test_check_access/golden/test.log
index abc480241..4184a0f30 100644
--- a/cmd/bisync/testdata/test_check_access/golden/test.log
+++ b/cmd/bisync/testdata/test_check_access/golden/test.log
@@ -16,11 +16,7 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -91,11 +87,7 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -188,11 +180,7 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
diff --git a/cmd/bisync/testdata/test_check_access_filters/golden/test.log b/cmd/bisync/testdata/test_check_access_filters/golden/test.log
index 82cc20179..eeb347438 100644
--- a/cmd/bisync/testdata/test_check_access_filters/golden/test.log
+++ b/cmd/bisync/testdata/test_check_access_filters/golden/test.log
@@ -21,15 +21,7 @@ INFO : Using filters file {workdir/}exclude-other-filtersfile.txt
INFO : Storing filters file hash to {workdir/}exclude-other-filtersfile.txt.md5
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir/subdirB: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir/subdirB: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir/subdirB: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir/subdirB: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -144,23 +136,7 @@ INFO : Using filters file {workdir/}include-other-filtersfile.txt
INFO : Storing filters file hash to {workdir/}include-other-filtersfile.txt.md5
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdirX: Set directory modification time (using SetModTime)
-INFO : subdirX/subdirX1: Set directory modification time (using SetModTime)
-INFO : subdir/subdirB: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdirX: Set directory modification time (using SetModTime)
-INFO : subdirX/subdirX1: Set directory modification time (using SetModTime)
-INFO : subdir/subdirB: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdirX: Set directory modification time (using SetModTime)
-INFO : subdir/subdirB: Set directory modification time (using SetModTime)
-INFO : subdirX/subdirX1: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdirX: Set directory modification time (using SetModTime)
-INFO : subdir/subdirB: Set directory modification time (using SetModTime)
-INFO : subdirX/subdirX1: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
diff --git a/cmd/bisync/testdata/test_check_filename/golden/test.log b/cmd/bisync/testdata/test_check_filename/golden/test.log
index 8315bfc32..573976ede 100644
--- a/cmd/bisync/testdata/test_check_filename/golden/test.log
+++ b/cmd/bisync/testdata/test_check_filename/golden/test.log
@@ -16,11 +16,7 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -94,11 +90,7 @@ INFO : Copying Path2 files to Path1
INFO : Checking access health
INFO : Found 2 matching ".chk_file" files on both paths
INFO : - Path2 Resync is copying files to - Path1
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
diff --git a/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log b/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log
index 3cc3d7e0e..096c5f0c5 100644
--- a/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log
+++ b/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log
@@ -147,11 +147,7 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
-INFO : subdir: Made directory with metadata (mtime=2024-02-27T04:53:52.809861575-05:00)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
diff --git a/cmd/bisync/testdata/test_extended_char_paths/golden/test.log b/cmd/bisync/testdata/test_extended_char_paths/golden/test.log
index db033110e..463e8e967 100644
--- a/cmd/bisync/testdata/test_extended_char_paths/golden/test.log
+++ b/cmd/bisync/testdata/test_extended_char_paths/golden/test.log
@@ -84,11 +84,7 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
-INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
-INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
-INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
-INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -133,11 +129,7 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
-INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
-INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
-INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
-INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -182,11 +174,7 @@ INFO : Using filters file {workdir/}測試_filtersfile.txt
INFO : Storing filters file hash to {workdir/}測試_filtersfile.txt.md5
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
-INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
-INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
-INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
-INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
diff --git a/cmd/bisync/testdata/test_extended_filenames/golden/test.log b/cmd/bisync/testdata/test_extended_filenames/golden/test.log
index 37cae89d2..eda13848c 100644
--- a/cmd/bisync/testdata/test_extended_filenames/golden/test.log
+++ b/cmd/bisync/testdata/test_extended_filenames/golden/test.log
@@ -25,11 +25,7 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
-INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)
-INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
-INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)
-INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -116,15 +112,7 @@ INFO : - Path2 Queue copy to Path1 - "
INFO : - Path1 Queue delete - {path1/}subdir_with_ࢺ_/filename_contains_ě_.txt
INFO : - Path2 Queue copy to Path1 - {path1/}subdir_with_ࢺ_/filename_contains_ࢺ_p2s.txt
INFO : - Path2 Do queued copies to - Path1
-INFO : subdir with␊white space.txt: Made directory with metadata (mtime=2024-02-27T04:53:52.913860529-05:00)
-INFO : subdir_rawchars_␙_�_�: Made directory with metadata (mtime=2024-02-27T04:53:52.913860529-05:00)
-INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)
-INFO : subdir with␊white space.txt: Set directory modification time (using SetModTime)
-INFO : subdir_rawchars_␙_�_�: Set directory modification time (using SetModTime)
-INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)
INFO : - Path1 Do queued copies to - Path2
-INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)
-INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
diff --git a/cmd/bisync/testdata/test_filters/golden/test.log b/cmd/bisync/testdata/test_filters/golden/test.log
index d74044ef1..9da9b2130 100644
--- a/cmd/bisync/testdata/test_filters/golden/test.log
+++ b/cmd/bisync/testdata/test_filters/golden/test.log
@@ -20,11 +20,7 @@ INFO : Using filters file {workdir/}filtersfile.flt
INFO : Storing filters file hash to {workdir/}filtersfile.flt.md5
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -58,8 +54,6 @@ INFO : Path2 checking for diffs
INFO : Applying changes
INFO : - Path1 Queue copy to Path2 - {path2/}subdir/fileZ.txt
INFO : - Path1 Do queued copies to - Path2
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
diff --git a/cmd/bisync/testdata/test_filtersfile_checks/golden/test.log b/cmd/bisync/testdata/test_filtersfile_checks/golden/test.log
index 6a2583402..05c2abf23 100644
--- a/cmd/bisync/testdata/test_filtersfile_checks/golden/test.log
+++ b/cmd/bisync/testdata/test_filtersfile_checks/golden/test.log
@@ -16,11 +16,7 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -85,11 +81,7 @@ INFO : Using filters file {workdir/}filtersfile.txt
INFO : Storing filters file hash to {workdir/}filtersfile.txt.md5
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -154,11 +146,7 @@ INFO : Using filters file {workdir/}filtersfile.txt
INFO : Skipped storing filters file hash to {workdir/}filtersfile.txt.md5 as --dry-run is set
INFO : Copying Path2 files to Path1
NOTICE: - Path2 Resync is copying files to - Path1
-NOTICE: subdir: Skipped set directory modification time as --dry-run is set
-NOTICE: {path1String}: Skipped set directory modification time as --dry-run is set
NOTICE: - Path1 Resync is copying files to - Path2
-NOTICE: subdir: Skipped set directory modification time as --dry-run is set
-NOTICE: {path2String}: Skipped set directory modification time as --dry-run is set
INFO : Resync updating listings
INFO : Bisync successful
diff --git a/cmd/bisync/testdata/test_ignorelistingchecksum/golden/test.log b/cmd/bisync/testdata/test_ignorelistingchecksum/golden/test.log
index c7f3c1bd7..f54cd7baf 100644
--- a/cmd/bisync/testdata/test_ignorelistingchecksum/golden/test.log
+++ b/cmd/bisync/testdata/test_ignorelistingchecksum/golden/test.log
@@ -16,11 +16,7 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -37,11 +33,7 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -77,8 +69,6 @@ INFO : - Path1 Queue copy to Path2 - {
INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt
INFO : - Path2 Do queued copies to - Path1
INFO : - Path1 Do queued copies to - Path2
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
diff --git a/cmd/bisync/testdata/test_normalization/golden/test.log b/cmd/bisync/testdata/test_normalization/golden/test.log
index 98144aee1..5e4cbbccf 100644
--- a/cmd/bisync/testdata/test_normalization/golden/test.log
+++ b/cmd/bisync/testdata/test_normalization/golden/test.log
@@ -17,11 +17,7 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
-INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
-INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
-INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
-INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -74,12 +70,8 @@ INFO : - Path1 Queue copy to Path2 - "
INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt
INFO : - Path2 Do queued copies to - Path1
INFO : - Path1 Do queued copies to - Path2
-INFO : 測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Made directory with metadata (mtime=2024-02-27T04:53:52.993859723-05:00)
-INFO : folder: Set directory modification time (using SetModTime)
INFO : folder/hello,WORLD!.txt: Fixed case by renaming to: folder/HeLlO,wOrLd!.txt
INFO : folder/éééö.txt: Fixed case by renaming to: folder/éééö.txt
-INFO : 測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime)
-INFO : folder: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -158,11 +150,7 @@ INFO : - Path2 Queue copy to Path1 - {
INFO : - Path1 Queue copy to Path2 - "{path2/}測試_Руский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éö/測試_Руский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éö.txt"
INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt
INFO : - Path2 Do queued copies to - Path1
-INFO : folder: Set directory modification time (using SetModTime)
-INFO : folder: Set directory modification time (using SetModTime)
INFO : - Path1 Do queued copies to - Path2
-INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Made directory with metadata (mtime=2024-02-27T04:53:53.001859642-05:00)
-INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -182,15 +170,7 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
-INFO : folder: Set directory modification time (using SetModTime)
-INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime)
-INFO : folder: Set directory modification time (using SetModTime)
-INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
-INFO : folder: Set directory modification time (using SetModTime)
-INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime)
-INFO : folder: Set directory modification time (using SetModTime)
-INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -230,10 +210,6 @@ INFO : - Path1 Queue copy to Path2 - "
INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt
INFO : - Path2 Do queued copies to - Path1
INFO : - Path1 Do queued copies to - Path2
-INFO : folder: Set directory modification time (using SetModTime)
-INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime)
-INFO : folder: Set directory modification time (using SetModTime)
-INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
diff --git a/cmd/bisync/testdata/test_rclone_args/golden/test.log b/cmd/bisync/testdata/test_rclone_args/golden/test.log
index 4fbe6c385..73ede1e69 100644
--- a/cmd/bisync/testdata/test_rclone_args/golden/test.log
+++ b/cmd/bisync/testdata/test_rclone_args/golden/test.log
@@ -15,11 +15,7 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -111,11 +107,7 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -149,8 +141,6 @@ INFO : Applying changes
INFO : - Path2 Queue copy to Path1 - {path1/}file2.txt
INFO : - Path2 Queue copy to Path1 - {path1/}subdir/file21.txt
INFO : - Path2 Do queued copies to - Path1
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -168,11 +158,7 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -214,11 +200,7 @@ INFO : - Path1 Queue copy to Path2 - {
INFO : - Path2 Queue copy to Path1 - {path1/}file2.txt
INFO : - Path2 Queue copy to Path1 - {path1/}subdir/file21.txt
INFO : - Path2 Do queued copies to - Path1
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Do queued copies to - Path2
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
diff --git a/cmd/bisync/testdata/test_rmdirs/golden/test.log b/cmd/bisync/testdata/test_rmdirs/golden/test.log
index 899e9a5f3..4e6763801 100644
--- a/cmd/bisync/testdata/test_rmdirs/golden/test.log
+++ b/cmd/bisync/testdata/test_rmdirs/golden/test.log
@@ -16,11 +16,7 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -49,8 +45,6 @@ INFO : Path2 checking for diffs
INFO : Applying changes
INFO : - Path2 Queue delete - {path2/}subdir/file20.txt
INFO : - Path1 Do queued copies to - Path2
-INFO : subdir: Set directory modification time (using SetModTime)
-INFO : subdir: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
diff --git a/fs/config/configflags/configflags.go b/fs/config/configflags/configflags.go
index ce891f2b4..206888fc3 100644
--- a/fs/config/configflags/configflags.go
+++ b/fs/config/configflags/configflags.go
@@ -55,7 +55,7 @@ func AddFlags(ci *fs.ConfigInfo, flagSet *pflag.FlagSet) {
flags.StringVarP(flagSet, &tempDir, "temp-dir", "", os.TempDir(), "Directory rclone will use for temporary files", "Config")
flags.BoolVarP(flagSet, &ci.CheckSum, "checksum", "c", ci.CheckSum, "Check for changes with size & checksum (if available, or fallback to size only).", "Copy")
flags.BoolVarP(flagSet, &ci.SizeOnly, "size-only", "", ci.SizeOnly, "Skip based on size only, not modtime or checksum", "Copy")
- flags.BoolVarP(flagSet, &ci.IgnoreTimes, "ignore-times", "I", ci.IgnoreTimes, "Don't skip files that match size and time - transfer all files", "Copy")
+ flags.BoolVarP(flagSet, &ci.IgnoreTimes, "ignore-times", "I", ci.IgnoreTimes, "Don't skip items that match size and time - transfer all unconditionally", "Copy")
flags.BoolVarP(flagSet, &ci.IgnoreExisting, "ignore-existing", "", ci.IgnoreExisting, "Skip all files that exist on destination", "Copy")
flags.BoolVarP(flagSet, &ci.IgnoreErrors, "ignore-errors", "", ci.IgnoreErrors, "Delete even if there are I/O errors", "Sync")
flags.BoolVarP(flagSet, &ci.DryRun, "dry-run", "n", ci.DryRun, "Do a trial run with no permanent changes", "Config,Important")
diff --git a/fs/operations/operations.go b/fs/operations/operations.go
index 125608fcf..b9eed93a1 100644
--- a/fs/operations/operations.go
+++ b/fs/operations/operations.go
@@ -141,6 +141,40 @@ func Equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object) bool {
return equal(ctx, src, dst, defaultEqualOpt(ctx))
}
+// DirsEqual is like Equal but for dirs instead of objects.
+// It returns true if two dirs should be considered "equal" for the purposes of syncCopyMove
+// (in other words, true == "skip updating modtime/metadata for this dir".)
+// Unlike Equal, it does not consider size or checksum, as these do not apply to directories.
+func DirsEqual(ctx context.Context, src, dst fs.Directory, opt DirsEqualOpt) (equal bool) {
+ if dst == nil {
+ return false
+ }
+ ci := fs.GetConfig(ctx)
+ if ci.SizeOnly || ci.Immutable || ci.IgnoreExisting || opt.ModifyWindow == fs.ModTimeNotSupported {
+ return true
+ }
+ if ci.IgnoreTimes {
+ return false
+ }
+ if !(opt.SetDirModtime || opt.SetDirMetadata) {
+ return true
+ }
+ srcModTime, dstModTime := src.ModTime(ctx), dst.ModTime(ctx)
+ if srcModTime.IsZero() || dstModTime.IsZero() {
+ return false
+ }
+ dt := dstModTime.Sub(srcModTime)
+ if dt < opt.ModifyWindow && dt > -opt.ModifyWindow {
+ fs.Debugf(dst, "Directory modification time the same (differ by %s, within tolerance %s)", dt, opt.ModifyWindow)
+ return true
+ }
+ if ci.UpdateOlder && dt >= opt.ModifyWindow {
+ fs.Debugf(dst, "Destination directory is newer than source, skipping")
+ return true
+ }
+ return false
+}
+
// sizeDiffers compare the size of src and dst taking into account the
// various ways of ignoring sizes
func sizeDiffers(ctx context.Context, src, dst fs.ObjectInfo) bool {
@@ -172,6 +206,13 @@ func defaultEqualOpt(ctx context.Context) equalOpt {
}
}
+// DirsEqualOpt represents options for DirsEqual function()
+type DirsEqualOpt struct {
+ ModifyWindow time.Duration // Max time diff to be considered the same
+ SetDirModtime bool // whether to consider dir modtime
+ SetDirMetadata bool // whether to consider dir metadata
+}
+
var modTimeUploadOnce sync.Once
// emit a log if we are about to upload a file to set its modification time
diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go
index 15143ee92..4f4cf4cfd 100644
--- a/fs/operations/operations_test.go
+++ b/fs/operations/operations_test.go
@@ -1754,3 +1754,74 @@ func TestSetDirModTime(t *testing.T) {
}
fstest.CheckDirModTime(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, name), t2)
}
+
+func TestDirsEqual(t *testing.T) {
+ ctx := context.Background()
+ ctx, ci := fs.AddConfig(ctx)
+ ci.Metadata = true
+ r := fstest.NewRun(t)
+ if !r.Fremote.Features().WriteDirMetadata && r.Fremote.Features().MkdirMetadata == nil {
+ t.Skip("Skipping test as remote does not support WriteDirMetadata or MkdirMetadata")
+ }
+
+ opt := operations.DirsEqualOpt{
+ ModifyWindow: fs.GetModifyWindow(ctx, r.Flocal, r.Fremote),
+ SetDirModtime: true,
+ SetDirMetadata: true,
+ }
+
+ // Create a source local directory with metadata
+ src, err := operations.MkdirMetadata(ctx, r.Flocal, "dir with metadata to be copied", testMetadata)
+ require.NoError(t, err)
+ require.NotNil(t, src)
+
+ // try with nil dst -- should be false
+ equal := operations.DirsEqual(ctx, src, nil, opt)
+ assert.False(t, equal)
+
+ // make a dest with an equal modtime
+ dst, err := operations.MkdirModTime(ctx, r.Fremote, "dst", src.ModTime(ctx))
+ require.NoError(t, err)
+
+ // try with equal modtimes -- should be true
+ equal = operations.DirsEqual(ctx, src, dst, opt)
+ assert.True(t, equal)
+
+ // try with unequal modtimes -- should be false
+ dst, err = operations.SetDirModTime(ctx, r.Fremote, dst, "", t2)
+ require.NoError(t, err)
+ equal = operations.DirsEqual(ctx, src, dst, opt)
+ assert.False(t, equal)
+
+ // try with unequal modtimes that are within modify window -- should be true
+ halfWindow := opt.ModifyWindow / 2
+ dst, err = operations.SetDirModTime(ctx, r.Fremote, dst, "", src.ModTime(ctx).Add(halfWindow))
+ require.NoError(t, err)
+ equal = operations.DirsEqual(ctx, src, dst, opt)
+ assert.True(t, equal)
+
+ // test ignoretimes -- should be false
+ ci.IgnoreTimes = true
+ equal = operations.DirsEqual(ctx, src, dst, opt)
+ assert.False(t, equal)
+
+ // test immutable -- should be true
+ ci.IgnoreTimes = false
+ ci.Immutable = true
+ dst, err = operations.SetDirModTime(ctx, r.Fremote, dst, "", t3)
+ require.NoError(t, err)
+ equal = operations.DirsEqual(ctx, src, dst, opt)
+ assert.True(t, equal)
+
+ // test dst newer than src with --update -- should be true
+ ci.Immutable = false
+ ci.UpdateOlder = true
+ equal = operations.DirsEqual(ctx, src, dst, opt)
+ assert.True(t, equal)
+
+ // test no SetDirModtime or SetDirMetadata -- should be true
+ ci.UpdateOlder = false
+ opt.SetDirMetadata, opt.SetDirModtime = false, false
+ equal = operations.DirsEqual(ctx, src, dst, opt)
+ assert.True(t, equal)
+}
diff --git a/fs/sync/sync.go b/fs/sync/sync.go
index 0ca8e1d27..9dac7c694 100644
--- a/fs/sync/sync.go
+++ b/fs/sync/sync.go
@@ -89,9 +89,10 @@ type syncCopyMove struct {
setDirMetadata bool // if set we set the directory metadata
setDirModTime bool // if set we set the directory modtimes
setDirModTimeAfter bool // if set we set the directory modtimes at the end of the sync
- setDirModTimeMu sync.Mutex // protect setDirModTimeMu
+ setDirModTimeMu sync.Mutex // protect setDirModTimes and modifiedDirs
setDirModTimes []setDirModTime // directories that need their modtime set
setDirModTimesMaxLevel int // max level of the directories to set
+ modifiedDirs map[string]struct{} // dirs with changed contents (if s.setDirModTimeAfter)
}
// For keeping track of delayed modtime sets
@@ -155,6 +156,7 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
setDirMetadata: ci.Metadata && fsrc.Features().ReadDirMetadata && fdst.Features().WriteDirMetadata,
setDirModTime: fdst.Features().WriteDirSetModTime || fdst.Features().MkdirMetadata != nil || fdst.Features().DirSetModTime != nil,
setDirModTimeAfter: fdst.Features().DirModTimeUpdatesOnWrite,
+ modifiedDirs: make(map[string]struct{}),
}
s.logger, s.usingLogger = operations.GetLogger(ctx)
@@ -399,6 +401,11 @@ func (s *syncCopyMove) pairChecker(in *pipe, out *pipe, fraction int, wg *sync.W
fs.Errorf(pair.Dst, "Source and destination exist but do not match: %v", err)
s.processError(err)
} else {
+ if pair.Dst != nil {
+ s.markDirModifiedObject(pair.Dst)
+ } else {
+ s.markDirModifiedObject(src)
+ }
// If destination already exists, then we must move it into --backup-dir if required
if pair.Dst != nil && s.backupDir != nil {
err := operations.MoveBackupDir(s.ctx, s.backupDir, pair.Dst)
@@ -764,7 +771,6 @@ func (s *syncCopyMove) renameID(obj fs.Object, renamesStrategy trackRenamesStrat
if renamesStrategy.hash() {
var err error
hash, err := obj.Hash(s.ctx, s.commonHash)
-
if err != nil {
fs.Debugf(obj, "Hash failed: %v", err)
return ""
@@ -1079,6 +1085,26 @@ func (s *syncCopyMove) DstOnly(dst fs.DirEntry) (recurse bool) {
return false
}
+// keeps track of dirs with changed contents, to avoid setting modtimes on dirs that haven't changed
+func (s *syncCopyMove) markDirModified(dir string) {
+ if !s.setDirModTimeAfter {
+ return
+ }
+ s.setDirModTimeMu.Lock()
+ defer s.setDirModTimeMu.Unlock()
+ s.modifiedDirs[dir] = struct{}{}
+}
+
+// like markDirModified, but accepts an Object instead of a string.
+// the marked dir will be this object's parent.
+func (s *syncCopyMove) markDirModifiedObject(o fs.Object) {
+ dir := path.Dir(o.Remote())
+ if dir == "." {
+ dir = ""
+ }
+ s.markDirModified(dir)
+}
+
// copyDirMetadata copies the src directory modTime or Metadata to dst
// or f if nil. If dst is nil then it uses dir as the name of the new
// directory.
@@ -1087,17 +1113,25 @@ func (s *syncCopyMove) DstOnly(dst fs.DirEntry) (recurse bool) {
// be nil.
func (s *syncCopyMove) copyDirMetadata(ctx context.Context, f fs.Fs, dst fs.Directory, dir string, src fs.Directory) (newDst fs.Directory) {
var err error
- if s.setDirMetadata {
- newDst, err = operations.CopyDirMetadata(ctx, f, dst, dir, src)
- } else if s.setDirModTime {
- if dst == nil {
- newDst, err = operations.MkdirModTime(ctx, f, dir, src.ModTime(ctx))
- } else {
- newDst, err = operations.SetDirModTime(ctx, f, dst, dir, src.ModTime(ctx))
+ equal := operations.DirsEqual(ctx, src, dst, operations.DirsEqualOpt{ModifyWindow: s.modifyWindow, SetDirModtime: s.setDirModTime, SetDirMetadata: s.setDirMetadata})
+ if !s.setDirModTimeAfter && equal {
+ return nil
+ }
+ if s.setDirModTimeAfter && equal {
+ newDst = dst
+ } else {
+ if s.setDirMetadata {
+ newDst, err = operations.CopyDirMetadata(ctx, f, dst, dir, src)
+ } else if s.setDirModTime {
+ if dst == nil {
+ newDst, err = operations.MkdirModTime(ctx, f, dir, src.ModTime(ctx))
+ } else {
+ newDst, err = operations.SetDirModTime(ctx, f, dst, dir, src.ModTime(ctx))
+ }
+ } else if dst == nil {
+ // Create the directory if it doesn't exist
+ err = operations.Mkdir(ctx, f, dir)
}
- } else if dst == nil {
- // Create the directory if it doesn't exist
- err = operations.Mkdir(ctx, f, dir)
}
// If we need to set modtime after and we created a dir, then save it for later
if s.setDirModTime && s.setDirModTimeAfter && err == nil {
@@ -1138,7 +1172,7 @@ func (s *syncCopyMove) setDelayedDirModTimes(ctx context.Context) error {
// Timestamp all directories at the same level in parallel, deepest first
// We do this by iterating the slice multiple times to save memory
// There could be a lot of directories in this slice.
- var errCount = errcount.New()
+ errCount := errcount.New()
for level := s.setDirModTimesMaxLevel; level >= 0; level-- {
g, gCtx := errgroup.WithContext(ctx)
g.SetLimit(s.ci.Checkers)
@@ -1151,6 +1185,9 @@ func (s *syncCopyMove) setDelayedDirModTimes(ctx context.Context) error {
break
}
item := item
+ if _, ok := s.modifiedDirs[item.dir]; !ok {
+ continue
+ }
g.Go(func() error {
_, err := operations.SetDirModTime(gCtx, s.fdst, item.dst, item.dir, item.modTime)
if err != nil {
@@ -1201,6 +1238,7 @@ func (s *syncCopyMove) SrcOnly(src fs.DirEntry) (recurse bool) {
if !NoNeedTransfer {
// No need to check since doesn't exist
fs.Debugf(src, "Need to transfer - File not found at Destination")
+ s.markDirModifiedObject(x)
ok := s.toBeUploaded.Put(s.inCtx, fs.ObjectPair{Src: x, Dst: nil})
if !ok {
return
@@ -1217,6 +1255,7 @@ func (s *syncCopyMove) SrcOnly(src fs.DirEntry) (recurse bool) {
s.srcEmptyDirsMu.Unlock()
// Create the directory and make sure the Metadata/ModTime is correct
+ s.markDirModified(x.Remote())
s.copyDirMetadata(s.ctx, s.fdst, nil, x.Remote(), x)
return true
default:
diff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.go
index 1f21806fc..150051062 100644
--- a/fs/sync/sync_test.go
+++ b/fs/sync/sync_test.go
@@ -19,6 +19,7 @@ import (
mutex "sync" // renamed as "sync" already in use
_ "github.com/rclone/rclone/backend/all" // import all backends
+ "github.com/rclone/rclone/cmd/bisync/bilib"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/fs/filter"
@@ -2499,6 +2500,74 @@ func TestSyncConcurrentTruncate(t *testing.T) {
testSyncConcurrent(t, "truncate")
}
+// Tests that nothing is transferred when src and dst already match
+// Run the same sync twice, ensure no action is taken the second time
+func TestNothingToTransfer(t *testing.T) {
+ accounting.GlobalStats().ResetCounters()
+ ctx, _ := fs.AddConfig(context.Background())
+ r := fstest.NewRun(t)
+ file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
+ file2 := r.WriteFile("sub dir2/very/very/very/very/very/nested/subdir/hello world", "hello world", t1)
+ r.CheckLocalItems(t, file1, file2)
+ _, err := operations.SetDirModTime(ctx, r.Flocal, nil, "sub dir", t2)
+ if err != nil && !errors.Is(err, fs.ErrorNotImplemented) {
+ require.NoError(t, err)
+ }
+ r.Mkdir(ctx, r.Fremote)
+ _, err = operations.MkdirModTime(ctx, r.Fremote, "sub dir", t3)
+ require.NoError(t, err)
+
+ // set logging
+ // (this checks log output as DirModtime operations do not yet have stats, and r.CheckDirectoryModTimes also does not tell us what actions were taken)
+ oldLogLevel := fs.GetConfig(context.Background()).LogLevel
+ defer func() { fs.GetConfig(context.Background()).LogLevel = oldLogLevel }() // reset to old val after test
+ // need to do this as fs.Infof only respects the globalConfig
+ fs.GetConfig(context.Background()).LogLevel = fs.LogLevelInfo
+
+ accounting.GlobalStats().ResetCounters()
+ ctx = predictDstFromLogger(ctx)
+ output := bilib.CaptureOutput(func() {
+ err = CopyDir(ctx, r.Fremote, r.Flocal, true)
+ require.NoError(t, err)
+ })
+ require.NotNil(t, output)
+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
+ r.CheckLocalItems(t, file1, file2)
+ r.CheckRemoteItems(t, file1, file2)
+ // Check that the modtimes of the directories are as expected
+ r.CheckDirectoryModTimes(t, "sub dir")
+
+ // check that actions were taken
+ assert.True(t, strings.Contains(string(output), "Copied"), `expected to find at least one "Copied" log: `+string(output))
+ if r.Fremote.Features().DirSetModTime != nil || r.Fremote.Features().MkdirMetadata != nil {
+ assert.True(t, strings.Contains(string(output), "Set directory modification time"), `expected to find at least one "Set directory modification time" log: `+string(output))
+ }
+ assert.False(t, strings.Contains(string(output), "There was nothing to transfer"), `expected to find no "There was nothing to transfer" logs, but found one: `+string(output))
+ assert.Equal(t, int64(2), accounting.GlobalStats().GetTransfers())
+
+ // run it again and make sure no actions were taken
+ accounting.GlobalStats().ResetCounters()
+ ctx = predictDstFromLogger(ctx)
+ output = bilib.CaptureOutput(func() {
+ err = CopyDir(ctx, r.Fremote, r.Flocal, true)
+ require.NoError(t, err)
+ })
+ require.NotNil(t, output)
+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
+ r.CheckLocalItems(t, file1, file2)
+ r.CheckRemoteItems(t, file1, file2)
+ // Check that the modtimes of the directories are as expected
+ r.CheckDirectoryModTimes(t, "sub dir")
+
+ // check that actions were NOT taken
+ assert.False(t, strings.Contains(string(output), "Copied"), `expected to find no "Copied" logs, but found one: `+string(output))
+ if r.Fremote.Features().DirSetModTime != nil || r.Fremote.Features().MkdirMetadata != nil {
+ assert.False(t, strings.Contains(string(output), "Set directory modification time"), `expected to find no "Set directory modification time" logs, but found one: `+string(output))
+ }
+ assert.True(t, strings.Contains(string(output), "There was nothing to transfer"), `expected to find a "There was nothing to transfer" log: `+string(output))
+ assert.Equal(t, int64(0), accounting.GlobalStats().GetTransfers())
+}
+
// for testing logger:
func predictDstFromLogger(ctx context.Context) context.Context {
opt := operations.NewLoggerOpt()