Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

sync: implement directory sync for mod times and metadata

Nick Craig-Woodf5f86786b272
diff --git a/cmd/bisync/bisync_test.go b/cmd/bisync/bisync_test.go
index e9307d9ce..0b333b51d 100644
--- a/cmd/bisync/bisync_test.go
+++ b/cmd/bisync/bisync_test.go
@@ -86,6 +86,10 @@ var logReplacements = []string{
`^.*?"SlowHashDetected":.*?$`, dropMe,
`^.*? for same-side diffs on .*?$`, dropMe,
`^.*?Downloading hashes.*?$`, dropMe,
+ // ignore timestamps in directory time updates
+ `^(INFO : .*?: Made directory with (metadata|modification time)).*$`, `$1`,
+ // ignore sizes in directory time updates
+ `^(NOTICE: .*?: Skipped set directory modification time as --dry-run is set).*$`, `$1`,
}
// Some dry-run messages differ depending on the particular remote.
@@ -121,6 +125,9 @@ var logHoppers = []string{
// order of files re-checked prior to a conflict rename
`ERROR : .*: md5 differ.*`,
+
+ // Directory modification time setting can happen in any order
+ `INFO : .*: (Set directory modification time|Made directory with metadata).*`,
}
// Some log lines can contain Windows path separator that must be
diff --git a/cmd/bisync/testdata/test_all_changed/golden/test.log b/cmd/bisync/testdata/test_all_changed/golden/test.log
index cb74904c9..39c281bcb 100644
--- a/cmd/bisync/testdata/test_all_changed/golden/test.log
+++ b/cmd/bisync/testdata/test_all_changed/golden/test.log
@@ -16,7 +16,11 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -59,6 +63,8 @@ INFO : - Path1 Queue copy to Path2 - {
INFO : - Path1 Queue copy to Path2 - {path2/}file1.txt
INFO : - Path1 Queue copy to Path2 - {path2/}subdir/file20.txt
INFO : - Path1 Do queued copies to - Path2
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -133,6 +139,8 @@ INFO : - Path1 Queue copy to Path2 - {
INFO : - Path1 Queue copy to Path2 - {path2/}file1.txt
INFO : - Path1 Queue copy to Path2 - {path2/}subdir/file20.txt
INFO : - Path1 Do queued copies to - Path2
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
diff --git a/cmd/bisync/testdata/test_basic/golden/test.log b/cmd/bisync/testdata/test_basic/golden/test.log
index d3086e790..912423c26 100644
--- a/cmd/bisync/testdata/test_basic/golden/test.log
+++ b/cmd/bisync/testdata/test_basic/golden/test.log
@@ -16,7 +16,11 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -53,6 +57,8 @@ INFO : - Path1 Queue copy to Path2 - {
INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt
INFO : - Path2 Do queued copies to - Path1
INFO : - Path1 Do queued copies to - Path2
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
diff --git a/cmd/bisync/testdata/test_check_access/golden/test.log b/cmd/bisync/testdata/test_check_access/golden/test.log
index 4184a0f30..abc480241 100644
--- a/cmd/bisync/testdata/test_check_access/golden/test.log
+++ b/cmd/bisync/testdata/test_check_access/golden/test.log
@@ -16,7 +16,11 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -87,7 +91,11 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -180,7 +188,11 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
diff --git a/cmd/bisync/testdata/test_check_access_filters/golden/test.log b/cmd/bisync/testdata/test_check_access_filters/golden/test.log
index eeb347438..82cc20179 100644
--- a/cmd/bisync/testdata/test_check_access_filters/golden/test.log
+++ b/cmd/bisync/testdata/test_check_access_filters/golden/test.log
@@ -21,7 +21,15 @@ INFO : Using filters file {workdir/}exclude-other-filtersfile.txt
INFO : Storing filters file hash to {workdir/}exclude-other-filtersfile.txt.md5
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir/subdirB: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir/subdirB: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir/subdirB: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir/subdirB: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -136,7 +144,23 @@ INFO : Using filters file {workdir/}include-other-filtersfile.txt
INFO : Storing filters file hash to {workdir/}include-other-filtersfile.txt.md5
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdirX: Set directory modification time (using SetModTime)
+INFO : subdirX/subdirX1: Set directory modification time (using SetModTime)
+INFO : subdir/subdirB: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdirX: Set directory modification time (using SetModTime)
+INFO : subdirX/subdirX1: Set directory modification time (using SetModTime)
+INFO : subdir/subdirB: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdirX: Set directory modification time (using SetModTime)
+INFO : subdir/subdirB: Set directory modification time (using SetModTime)
+INFO : subdirX/subdirX1: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdirX: Set directory modification time (using SetModTime)
+INFO : subdir/subdirB: Set directory modification time (using SetModTime)
+INFO : subdirX/subdirX1: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
diff --git a/cmd/bisync/testdata/test_check_filename/golden/test.log b/cmd/bisync/testdata/test_check_filename/golden/test.log
index 573976ede..8315bfc32 100644
--- a/cmd/bisync/testdata/test_check_filename/golden/test.log
+++ b/cmd/bisync/testdata/test_check_filename/golden/test.log
@@ -16,7 +16,11 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -90,7 +94,11 @@ INFO : Copying Path2 files to Path1
INFO : Checking access health
INFO : Found 2 matching ".chk_file" files on both paths
INFO : - Path2 Resync is copying files to - Path1
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
diff --git a/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log b/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log
index 096c5f0c5..3cc3d7e0e 100644
--- a/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log
+++ b/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log
@@ -147,7 +147,11 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
+INFO : subdir: Made directory with metadata (mtime=2024-02-27T04:53:52.809861575-05:00)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
diff --git a/cmd/bisync/testdata/test_extended_char_paths/golden/test.log b/cmd/bisync/testdata/test_extended_char_paths/golden/test.log
index 463e8e967..db033110e 100644
--- a/cmd/bisync/testdata/test_extended_char_paths/golden/test.log
+++ b/cmd/bisync/testdata/test_extended_char_paths/golden/test.log
@@ -84,7 +84,11 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
+INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
+INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
+INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
+INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -129,7 +133,11 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
+INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
+INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
+INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
+INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -174,7 +182,11 @@ INFO : Using filters file {workdir/}測試_filtersfile.txt
INFO : Storing filters file hash to {workdir/}測試_filtersfile.txt.md5
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
+INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
+INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
+INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
+INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
diff --git a/cmd/bisync/testdata/test_extended_filenames/golden/test.log b/cmd/bisync/testdata/test_extended_filenames/golden/test.log
index eda13848c..37cae89d2 100644
--- a/cmd/bisync/testdata/test_extended_filenames/golden/test.log
+++ b/cmd/bisync/testdata/test_extended_filenames/golden/test.log
@@ -25,7 +25,11 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
+INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)
+INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
+INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)
+INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -112,7 +116,15 @@ INFO : - Path2 Queue copy to Path1 - "
INFO : - Path1 Queue delete - {path1/}subdir_with_ࢺ_/filename_contains_ě_.txt
INFO : - Path2 Queue copy to Path1 - {path1/}subdir_with_ࢺ_/filename_contains_ࢺ_p2s.txt
INFO : - Path2 Do queued copies to - Path1
+INFO : subdir with␊white space.txt: Made directory with metadata (mtime=2024-02-27T04:53:52.913860529-05:00)
+INFO : subdir_rawchars_␙_�_�: Made directory with metadata (mtime=2024-02-27T04:53:52.913860529-05:00)
+INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)
+INFO : subdir with␊white space.txt: Set directory modification time (using SetModTime)
+INFO : subdir_rawchars_␙_�_�: Set directory modification time (using SetModTime)
+INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)
INFO : - Path1 Do queued copies to - Path2
+INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)
+INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
diff --git a/cmd/bisync/testdata/test_filters/golden/test.log b/cmd/bisync/testdata/test_filters/golden/test.log
index 9da9b2130..d74044ef1 100644
--- a/cmd/bisync/testdata/test_filters/golden/test.log
+++ b/cmd/bisync/testdata/test_filters/golden/test.log
@@ -20,7 +20,11 @@ INFO : Using filters file {workdir/}filtersfile.flt
INFO : Storing filters file hash to {workdir/}filtersfile.flt.md5
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -54,6 +58,8 @@ INFO : Path2 checking for diffs
INFO : Applying changes
INFO : - Path1 Queue copy to Path2 - {path2/}subdir/fileZ.txt
INFO : - Path1 Do queued copies to - Path2
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
diff --git a/cmd/bisync/testdata/test_filtersfile_checks/golden/test.log b/cmd/bisync/testdata/test_filtersfile_checks/golden/test.log
index a60aa5d49..28f4cfa89 100644
--- a/cmd/bisync/testdata/test_filtersfile_checks/golden/test.log
+++ b/cmd/bisync/testdata/test_filtersfile_checks/golden/test.log
@@ -16,7 +16,11 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -81,7 +85,11 @@ INFO : Using filters file {workdir/}filtersfile.txt
INFO : Storing filters file hash to {workdir/}filtersfile.txt.md5
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -146,7 +154,11 @@ INFO : Using filters file {workdir/}filtersfile.txt
INFO : Skipped storing filters file hash to {workdir/}filtersfile.txt.md5 as --dry-run is set
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
+NOTICE: subdir: Skipped set directory modification time as --dry-run is set (size 4Ki)
+NOTICE: {path1String}: Skipped set directory modification time as --dry-run is set
INFO : - Path1 Resync is copying files to - Path2
+NOTICE: subdir: Skipped set directory modification time as --dry-run is set (size 4Ki)
+NOTICE: {path2String}: Skipped set directory modification time as --dry-run is set
INFO : Resync updating listings
INFO : Bisync successful
diff --git a/cmd/bisync/testdata/test_ignorelistingchecksum/golden/test.log b/cmd/bisync/testdata/test_ignorelistingchecksum/golden/test.log
index f54cd7baf..c7f3c1bd7 100644
--- a/cmd/bisync/testdata/test_ignorelistingchecksum/golden/test.log
+++ b/cmd/bisync/testdata/test_ignorelistingchecksum/golden/test.log
@@ -16,7 +16,11 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -33,7 +37,11 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -69,6 +77,8 @@ INFO : - Path1 Queue copy to Path2 - {
INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt
INFO : - Path2 Do queued copies to - Path1
INFO : - Path1 Do queued copies to - Path2
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
diff --git a/cmd/bisync/testdata/test_normalization/golden/test.log b/cmd/bisync/testdata/test_normalization/golden/test.log
index 5e4cbbccf..98144aee1 100644
--- a/cmd/bisync/testdata/test_normalization/golden/test.log
+++ b/cmd/bisync/testdata/test_normalization/golden/test.log
@@ -17,7 +17,11 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
+INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
+INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
+INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
+INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -70,8 +74,12 @@ INFO : - Path1 Queue copy to Path2 - "
INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt
INFO : - Path2 Do queued copies to - Path1
INFO : - Path1 Do queued copies to - Path2
+INFO : 測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Made directory with metadata (mtime=2024-02-27T04:53:52.993859723-05:00)
+INFO : folder: Set directory modification time (using SetModTime)
INFO : folder/hello,WORLD!.txt: Fixed case by renaming to: folder/HeLlO,wOrLd!.txt
INFO : folder/éééö.txt: Fixed case by renaming to: folder/éééö.txt
+INFO : 測試_Русский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime)
+INFO : folder: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -150,7 +158,11 @@ INFO : - Path2 Queue copy to Path1 - {
INFO : - Path1 Queue copy to Path2 - "{path2/}測試_Руский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éö/測試_Руский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éö.txt"
INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt
INFO : - Path2 Do queued copies to - Path1
+INFO : folder: Set directory modification time (using SetModTime)
+INFO : folder: Set directory modification time (using SetModTime)
INFO : - Path1 Do queued copies to - Path2
+INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Made directory with metadata (mtime=2024-02-27T04:53:53.001859642-05:00)
+INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -170,7 +182,15 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
+INFO : folder: Set directory modification time (using SetModTime)
+INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime)
+INFO : folder: Set directory modification time (using SetModTime)
+INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
+INFO : folder: Set directory modification time (using SetModTime)
+INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime)
+INFO : folder: Set directory modification time (using SetModTime)
+INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -210,6 +230,10 @@ INFO : - Path1 Queue copy to Path2 - "
INFO : - Path2 Queue copy to Path1 - {path1/}file1.txt
INFO : - Path2 Do queued copies to - Path1
INFO : - Path1 Do queued copies to - Path2
+INFO : folder: Set directory modification time (using SetModTime)
+INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime)
+INFO : folder: Set directory modification time (using SetModTime)
+INFO : 測試_Руский___ě_áñ👸🏼🧝🏾‍♀️💆🏿‍♂️🐨🤙🏼🤮🧑🏻‍🔧🧑‍🔬éö: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
diff --git a/cmd/bisync/testdata/test_rclone_args/golden/test.log b/cmd/bisync/testdata/test_rclone_args/golden/test.log
index 73ede1e69..4fbe6c385 100644
--- a/cmd/bisync/testdata/test_rclone_args/golden/test.log
+++ b/cmd/bisync/testdata/test_rclone_args/golden/test.log
@@ -15,7 +15,11 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -107,7 +111,11 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -141,6 +149,8 @@ INFO : Applying changes
INFO : - Path2 Queue copy to Path1 - {path1/}file2.txt
INFO : - Path2 Queue copy to Path1 - {path1/}subdir/file21.txt
INFO : - Path2 Do queued copies to - Path1
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -158,7 +168,11 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -200,7 +214,11 @@ INFO : - Path1 Queue copy to Path2 - {
INFO : - Path2 Queue copy to Path1 - {path1/}file2.txt
INFO : - Path2 Queue copy to Path1 - {path1/}subdir/file21.txt
INFO : - Path2 Do queued copies to - Path1
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Do queued copies to - Path2
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
diff --git a/cmd/bisync/testdata/test_resolve/golden/test.log b/cmd/bisync/testdata/test_resolve/golden/test.log
index 9223c3f6a..dfc303f07 100644
--- a/cmd/bisync/testdata/test_resolve/golden/test.log
+++ b/cmd/bisync/testdata/test_resolve/golden/test.log
@@ -283,8 +283,8 @@ INFO : Path2: 2 changes:  1 new,  1 modified, 
INFO : (Modified:  1 newer,  0 older,  1 larger,  0 smaller)
INFO : Applying changes
INFO : Checking potential conflicts...
-ERROR : file2.txt: md5 differ
ERROR : file1.txt: md5 differ
+ERROR : file2.txt: md5 differ
NOTICE: {path2String}: 2 differences found
NOTICE: {path2String}: 2 errors while checking
INFO : Finished checking the potential conflicts. 2 differences found
diff --git a/cmd/bisync/testdata/test_rmdirs/golden/test.log b/cmd/bisync/testdata/test_rmdirs/golden/test.log
index 4e6763801..899e9a5f3 100644
--- a/cmd/bisync/testdata/test_rmdirs/golden/test.log
+++ b/cmd/bisync/testdata/test_rmdirs/golden/test.log
@@ -16,7 +16,11 @@ INFO : Bisyncing with Comparison Settings:
INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"
INFO : Copying Path2 files to Path1
INFO : - Path2 Resync is copying files to - Path1
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : - Path1 Resync is copying files to - Path2
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : Resync updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
@@ -45,6 +49,8 @@ INFO : Path2 checking for diffs
INFO : Applying changes
INFO : - Path2 Queue delete - {path2/}subdir/file20.txt
INFO : - Path1 Do queued copies to - Path2
+INFO : subdir: Set directory modification time (using SetModTime)
+INFO : subdir: Set directory modification time (using SetModTime)
INFO : Updating listings
INFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"
INFO : Bisync successful
diff --git a/cmd/copy/copy.go b/cmd/copy/copy.go
index 745811de4..6a68a330c 100644
--- a/cmd/copy/copy.go
+++ b/cmd/copy/copy.go
@@ -79,6 +79,15 @@ recently very efficiently like this:
rclone copy --max-age 24h --no-traverse /path/to/src remote:
+
+Rclone will sync the modification times of files and directories if
+the backend supports it. If metadata syncing is required then use the
+|--metadata| flag.
+
+Note that the modification time and metadata for the root directory
+will **not** be synced. See https://github.com/rclone/rclone/issues/7652
+for more info.
+
**Note**: Use the |-P|/|--progress| flag to view real-time transfer statistics.
**Note**: Use the |--dry-run| or the |--interactive|/|-i| flag to test without copying anything.
diff --git a/cmd/move/move.go b/cmd/move/move.go
index ae7260430..ca3f448b7 100644
--- a/cmd/move/move.go
+++ b/cmd/move/move.go
@@ -55,6 +55,14 @@ whether rclone lists the destination directory or not. Supplying this
option when moving a small number of files into a large destination
can speed transfers up greatly.
+Rclone will sync the modification times of files and directories if
+the backend supports it. If metadata syncing is required then use the
+|--metadata| flag.
+
+Note that the modification time and metadata for the root directory
+will **not** be synced. See https://github.com/rclone/rclone/issues/7652
+for more info.
+
**Important**: Since this can cause data loss, test first with the
|--dry-run| or the |--interactive|/|-i| flag.
diff --git a/cmd/sync/sync.go b/cmd/sync/sync.go
index 38076541e..ba35fa3e6 100644
--- a/cmd/sync/sync.go
+++ b/cmd/sync/sync.go
@@ -170,6 +170,14 @@ the destination from the sync with a filter rule or by putting an
exclude-if-present file inside the destination directory and sync to a
destination that is inside the source directory.
+Rclone will sync the modification times of files and directories if
+the backend supports it. If metadata syncing is required then use the
+` + "`--metadata`" + ` flag.
+
+Note that the modification time and metadata for the root directory
+will **not** be synced. See https://github.com/rclone/rclone/issues/7652
+for more info.
+
**Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics
**Note**: Use the ` + "`rclone dedupe`" + ` command to deal with "Duplicate object/directory found in source/destination - ignoring" errors.
diff --git a/docs/content/docs.md b/docs/content/docs.md
index 821d6f4b0..3b3a50a63 100644
--- a/docs/content/docs.md
+++ b/docs/content/docs.md
@@ -446,18 +446,21 @@ This can be used when scripting to make aged backups efficiently, e.g.
## Metadata support {#metadata}
-Metadata is data about a file which isn't the contents of the file.
-Normally rclone only preserves the modification time and the content
-(MIME) type where possible.
+Metadata is data about a file (or directory) which isn't the contents
+of the file (or directory). Normally rclone only preserves the
+modification time and the content (MIME) type where possible.
-Rclone supports preserving all the available metadata on files (not
-directories) when using the `--metadata` or `-M` flag.
+Rclone supports preserving all the available metadata on files and
+directories when using the `--metadata` or `-M` flag.
Exactly what metadata is supported and what that support means depends
on the backend. Backends that support metadata have a metadata section
in their docs and are listed in the [features table](/overview/#features)
(Eg [local](/local/#metadata), [s3](/s3/#metadata))
+Some backends don't support metadata, some only support metadata on
+files and some support metadata on both files and directories.
+
Rclone only supports a one-time sync of metadata. This means that
metadata will be synced from the source object to the destination
object only when the source object has changed and needs to be
@@ -1560,10 +1563,10 @@ some context for the `Metadata` which may be important.
- `SrcFsType` is the name of the source backend.
- `DstFs` is the config string for the remote that the object is being copied to
- `DstFsType` is the name of the destination backend.
-- `Remote` is the path of the file relative to the root.
-- `Size`, `MimeType`, `ModTime` are attributes of the file.
+- `Remote` is the path of the object relative to the root.
+- `Size`, `MimeType`, `ModTime` are attributes of the object.
- `IsDir` is `true` if this is a directory (not yet implemented).
-- `ID` is the source `ID` of the file if known.
+- `ID` is the source `ID` of the object if known.
- `Metadata` is the backend specific metadata as described in the backend docs.
```json
diff --git a/fs/sync/sync.go b/fs/sync/sync.go
index c464235c0..0ca8e1d27 100644
--- a/fs/sync/sync.go
+++ b/fs/sync/sync.go
@@ -18,6 +18,8 @@ import (
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/march"
"github.com/rclone/rclone/fs/operations"
+ "github.com/rclone/rclone/lib/errcount"
+ "golang.org/x/sync/errgroup"
)
// ErrorMaxDurationReached defines error when transfer duration is reached
@@ -84,6 +86,20 @@ type syncCopyMove struct {
maxDurationEndTime time.Time // end time if --max-duration is set
logger operations.LoggerFn // LoggerFn used to report the results of a sync (or bisync) to an io.Writer
usingLogger bool // whether we are using logger
+ setDirMetadata bool // if set we set the directory metadata
+ setDirModTime bool // if set we set the directory modtimes
+ setDirModTimeAfter bool // if set we set the directory modtimes at the end of the sync
+ setDirModTimeMu sync.Mutex // protect setDirModTimeMu
+ setDirModTimes []setDirModTime // directories that need their modtime set
+ setDirModTimesMaxLevel int // max level of the directories to set
+}
+
+// For keeping track of delayed modtime sets
+type setDirModTime struct {
+ dst fs.Directory
+ dir string
+ modTime time.Time
+ level int // the level of the directory, 0 is root
}
type trackRenamesStrategy byte
@@ -136,6 +152,9 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.Delete
modifyWindow: fs.GetModifyWindow(ctx, fsrc, fdst),
trackRenamesCh: make(chan fs.Object, ci.Checkers),
checkFirst: ci.CheckFirst,
+ setDirMetadata: ci.Metadata && fsrc.Features().ReadDirMetadata && fdst.Features().WriteDirMetadata,
+ setDirModTime: fdst.Features().WriteDirSetModTime || fdst.Features().MkdirMetadata != nil || fdst.Features().DirSetModTime != nil,
+ setDirModTimeAfter: fdst.Features().DirModTimeUpdatesOnWrite,
}
s.logger, s.usingLogger = operations.GetLogger(ctx)
@@ -966,6 +985,11 @@ func (s *syncCopyMove) run() error {
}
}
+ // Update modtimes for directories if necessary
+ if s.setDirModTime && s.setDirModTimeAfter {
+ s.processError(s.setDelayedDirModTimes(s.ctx))
+ }
+
// Prune empty directories
if s.deleteMode != fs.DeleteModeOff {
if s.currentError() != nil && !s.ci.IgnoreErrors {
@@ -1055,6 +1079,96 @@ func (s *syncCopyMove) DstOnly(dst fs.DirEntry) (recurse bool) {
return false
}
+// copyDirMetadata copies the src directory modTime or Metadata to dst
+// or f if nil. If dst is nil then it uses dir as the name of the new
+// directory.
+//
+// It returns the destination directory if possible. Note that this may
+// be nil.
+func (s *syncCopyMove) copyDirMetadata(ctx context.Context, f fs.Fs, dst fs.Directory, dir string, src fs.Directory) (newDst fs.Directory) {
+ var err error
+ if s.setDirMetadata {
+ newDst, err = operations.CopyDirMetadata(ctx, f, dst, dir, src)
+ } else if s.setDirModTime {
+ if dst == nil {
+ newDst, err = operations.MkdirModTime(ctx, f, dir, src.ModTime(ctx))
+ } else {
+ newDst, err = operations.SetDirModTime(ctx, f, dst, dir, src.ModTime(ctx))
+ }
+ } else if dst == nil {
+ // Create the directory if it doesn't exist
+ err = operations.Mkdir(ctx, f, dir)
+ }
+ // If we need to set modtime after and we created a dir, then save it for later
+ if s.setDirModTime && s.setDirModTimeAfter && err == nil {
+ if newDst != nil {
+ dir = newDst.Remote()
+ }
+ level := strings.Count(dir, "/") + 1
+ // The root directory "" is at the top level
+ if dir == "" {
+ level = 0
+ }
+ s.setDirModTimeMu.Lock()
+ // Keep track of the maximum level inserted
+ if level > s.setDirModTimesMaxLevel {
+ s.setDirModTimesMaxLevel = level
+ }
+ s.setDirModTimes = append(s.setDirModTimes, setDirModTime{
+ dst: newDst,
+ dir: dir,
+ modTime: src.ModTime(ctx),
+ level: level,
+ })
+ s.setDirModTimeMu.Unlock()
+ fs.Debugf(nil, "Added delayed dir = %q, newDst=%v", dir, newDst)
+ }
+ s.processError(err)
+ if err != nil {
+ return nil
+ }
+ return newDst
+}
+
+// Set the modtimes for directories
+func (s *syncCopyMove) setDelayedDirModTimes(ctx context.Context) error {
+ s.setDirModTimeMu.Lock()
+ defer s.setDirModTimeMu.Unlock()
+
+ // Timestamp all directories at the same level in parallel, deepest first
+ // We do this by iterating the slice multiple times to save memory
+ // There could be a lot of directories in this slice.
+ var errCount = errcount.New()
+ for level := s.setDirModTimesMaxLevel; level >= 0; level-- {
+ g, gCtx := errgroup.WithContext(ctx)
+ g.SetLimit(s.ci.Checkers)
+ for _, item := range s.setDirModTimes {
+ if item.level != level {
+ continue
+ }
+ // End early if error
+ if gCtx.Err() != nil {
+ break
+ }
+ item := item
+ g.Go(func() error {
+ _, err := operations.SetDirModTime(gCtx, s.fdst, item.dst, item.dir, item.modTime)
+ if err != nil {
+ err = fs.CountError(err)
+ fs.Errorf(item.dir, "Failed to timestamp directory: %v", err)
+ errCount.Add(err)
+ }
+ return nil // don't return errors, just count them
+ })
+ }
+ err := g.Wait()
+ if err != nil {
+ return err
+ }
+ }
+ return errCount.Err("failed to set directory modtime")
+}
+
// SrcOnly have an object which is in the source only
func (s *syncCopyMove) SrcOnly(src fs.DirEntry) (recurse bool) {
if s.deleteMode == fs.DeleteModeOnly {
@@ -1101,6 +1215,9 @@ func (s *syncCopyMove) SrcOnly(src fs.DirEntry) (recurse bool) {
s.srcEmptyDirs[src.Remote()] = src
s.logger(s.ctx, operations.MissingOnDst, src, nil, fs.ErrorIsDir)
s.srcEmptyDirsMu.Unlock()
+
+ // Create the directory and make sure the Metadata/ModTime is correct
+ s.copyDirMetadata(s.ctx, s.fdst, nil, x.Remote(), x)
return true
default:
panic("Bad object in DirEntries")
@@ -1135,9 +1252,12 @@ func (s *syncCopyMove) Match(ctx context.Context, dst, src fs.DirEntry) (recurse
}
case fs.Directory:
// Do the same thing to the entire contents of the directory
- _, ok := dst.(fs.Directory)
+ dstX, ok := dst.(fs.Directory)
if ok {
s.logger(s.ctx, operations.Match, src, dst, fs.ErrorIsDir)
+ // Create the directory and make sure the Metadata/ModTime is correct
+ s.copyDirMetadata(s.ctx, s.fdst, dstX, "", srcX)
+
// Only record matched (src & dst) empty dirs when performing move
if s.DoMove {
// Record the src directory for deletion
diff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.go
index 7911be1d0..c0728cf06 100644
--- a/fs/sync/sync_test.go
+++ b/fs/sync/sync_test.go
@@ -7,6 +7,7 @@ import (
"context"
"errors"
"fmt"
+ "io"
"os"
"os/exec"
"runtime"
@@ -65,15 +66,88 @@ func TestCopy(t *testing.T) {
ctx := context.Background()
r := fstest.NewRun(t)
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
+ _, err := operations.SetDirModTime(ctx, r.Flocal, nil, "sub dir", t2)
+ if err != nil && !errors.Is(err, fs.ErrorNotImplemented) {
+ require.NoError(t, err)
+ }
r.Mkdir(ctx, r.Fremote)
ctx = predictDstFromLogger(ctx)
- err := CopyDir(ctx, r.Fremote, r.Flocal, false)
+ err = CopyDir(ctx, r.Fremote, r.Flocal, false)
require.NoError(t, err)
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
r.CheckLocalItems(t, file1)
r.CheckRemoteItems(t, file1)
+
+ // Check that the modtimes of the directories are as expected
+ r.CheckDirectoryModTimes(t, "sub dir")
+}
+
+func TestCopyMetadata(t *testing.T) {
+ ctx := context.Background()
+ ctx, ci := fs.AddConfig(ctx)
+ ci.Metadata = true
+ r := fstest.NewRun(t)
+ features := r.Fremote.Features()
+
+ if !features.ReadMetadata && !features.WriteMetadata && !features.UserMetadata &&
+ !features.ReadDirMetadata && !features.WriteDirMetadata && !features.UserDirMetadata {
+ t.Skip("Skipping as metadata not supported")
+ }
+
+ const content = "hello metadata world!"
+ const dirPath = "metadata sub dir"
+ const filePath = dirPath + "/hello metadata world"
+
+ fileMetadata := fs.Metadata{
+ // System metadata supported by all backends
+ "mtime": t1.Format(time.RFC3339Nano),
+ // User metadata
+ "potato": "jersey",
+ }
+
+ dirMetadata := fs.Metadata{
+ // System metadata supported by all backends
+ "mtime": t2.Format(time.RFC3339Nano),
+ // User metadata
+ "potato": "king edward",
+ }
+
+ // Make the directory with metadata - may fall back to Mkdir
+ _, err := operations.MkdirMetadata(ctx, r.Flocal, dirPath, dirMetadata)
+ require.NoError(t, err)
+
+ // Upload the file with metadata
+ in := io.NopCloser(bytes.NewBufferString(content))
+ _, err = operations.Rcat(ctx, r.Flocal, filePath, in, t1, fileMetadata)
+ require.NoError(t, err)
+ file1 := fstest.NewItem(filePath, content, t1)
+
+ // Reset the time of the directory
+ _, err = operations.SetDirModTime(ctx, r.Flocal, nil, dirPath, t2)
+ if err != nil && !errors.Is(err, fs.ErrorNotImplemented) {
+ require.NoError(t, err)
+ }
+
+ ctx = predictDstFromLogger(ctx)
+ err = CopyDir(ctx, r.Fremote, r.Flocal, false)
+ require.NoError(t, err)
+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
+
+ r.CheckLocalItems(t, file1)
+ r.CheckRemoteItems(t, file1)
+
+ // Check that the modtimes of the directories are as expected
+ r.CheckDirectoryModTimes(t, dirPath)
+
+ // Check that the metadata on the directory and file is correct
+ if features.ReadMetadata {
+ fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewObject(ctx, t, r.Fremote, filePath), fileMetadata)
+ }
+ if features.ReadDirMetadata {
+ fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, dirPath), dirMetadata)
+ }
}
func TestCopyMissingDirectory(t *testing.T) {
@@ -205,10 +279,15 @@ func TestCopyEmptyDirectories(t *testing.T) {
ctx := context.Background()
r := fstest.NewRun(t)
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
- err := operations.Mkdir(ctx, r.Flocal, "sub dir2")
+ _, err := operations.MkdirModTime(ctx, r.Flocal, "sub dir2", t2)
require.NoError(t, err)
r.Mkdir(ctx, r.Fremote)
+ // Set the modtime on "sub dir" to something specific
+ // Without this it fails on the CI and in VirtualBox with variances of up to 10mS
+ _, err = operations.SetDirModTime(ctx, r.Flocal, nil, "sub dir", t1)
+ require.NoError(t, err)
+
ctx = predictDstFromLogger(ctx)
err = CopyDir(ctx, r.Fremote, r.Flocal, true)
require.NoError(t, err)
@@ -224,6 +303,9 @@ func TestCopyEmptyDirectories(t *testing.T) {
"sub dir2",
},
)
+
+ // Check that the modtimes of the directories are as expected
+ r.CheckDirectoryModTimes(t, "sub dir", "sub dir2")
}
// Test move empty directories
@@ -231,8 +313,10 @@ func TestMoveEmptyDirectories(t *testing.T) {
ctx := context.Background()
r := fstest.NewRun(t)
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
- err := operations.Mkdir(ctx, r.Flocal, "sub dir2")
+ _, err := operations.MkdirModTime(ctx, r.Flocal, "sub dir2", t2)
require.NoError(t, err)
+ subDir := fstest.NewDirectory(ctx, t, r.Flocal, "sub dir")
+ subDirT := subDir.ModTime(ctx)
r.Mkdir(ctx, r.Fremote)
ctx = predictDstFromLogger(ctx)
@@ -250,6 +334,14 @@ func TestMoveEmptyDirectories(t *testing.T) {
"sub dir2",
},
)
+
+ // Check that the modtimes of the directories are as expected
+ r.CheckDirectoryModTimes(t, "sub dir2")
+ // Note that "sub dir" mod time is updated when file1 is deleted from it
+ // So check it more manually
+ got := fstest.NewDirectory(ctx, t, r.Fremote, "sub dir")
+ gotT := got.ModTime(ctx)
+ fstest.AssertTimeEqualWithPrecision(t, subDir.Remote(), subDirT, gotT, fs.GetModifyWindow(ctx, r.Fremote, r.Flocal))
}
// Test sync empty directories
@@ -257,8 +349,14 @@ func TestSyncEmptyDirectories(t *testing.T) {
ctx := context.Background()
r := fstest.NewRun(t)
file1 := r.WriteFile("sub dir/hello world", "hello world", t1)
- err := operations.Mkdir(ctx, r.Flocal, "sub dir2")
+ _, err := operations.MkdirModTime(ctx, r.Flocal, "sub dir2", t2)
require.NoError(t, err)
+
+ // Set the modtime on "sub dir" to something specific
+ // Without this it fails on the CI and in VirtualBox with variances of up to 10mS
+ _, err = operations.SetDirModTime(ctx, r.Flocal, nil, "sub dir", t1)
+ require.NoError(t, err)
+
r.Mkdir(ctx, r.Fremote)
ctx = predictDstFromLogger(ctx)
@@ -276,6 +374,65 @@ func TestSyncEmptyDirectories(t *testing.T) {
"sub dir2",
},
)
+
+ // Check that the modtimes of the directories are as expected
+ r.CheckDirectoryModTimes(t, "sub dir", "sub dir2")
+}
+
+// Test delayed mod time setting
+func TestSyncSetDelayedModTimes(t *testing.T) {
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+
+ if !r.Fremote.Features().DirModTimeUpdatesOnWrite {
+ t.Skip("Backend doesn't have DirModTimeUpdatesOnWrite set")
+ }
+
+ // Create directories without timestamps
+ require.NoError(t, r.Flocal.Mkdir(ctx, "a1/b1/c1/d1/e1/f1"))
+ require.NoError(t, r.Flocal.Mkdir(ctx, "a1/b2/c1/d1/e1/f1"))
+ require.NoError(t, r.Flocal.Mkdir(ctx, "a1/b1/c1/d2/e1/f1"))
+ require.NoError(t, r.Flocal.Mkdir(ctx, "a1/b1/c1/d2/e1/f2"))
+
+ dirs := []string{
+ "a1",
+ "a1/b1",
+ "a1/b1/c1",
+ "a1/b1/c1/d1",
+ "a1/b1/c1/d1/e1",
+ "a1/b1/c1/d1/e1/f1",
+ "a1/b1/c1/d2",
+ "a1/b1/c1/d2/e1",
+ "a1/b1/c1/d2/e1/f1",
+ "a1/b1/c1/d2/e1/f2",
+ "a1/b2",
+ "a1/b2/c1",
+ "a1/b2/c1/d1",
+ "a1/b2/c1/d1/e1",
+ "a1/b2/c1/d1/e1/f1",
+ }
+ r.CheckLocalListing(t, []fstest.Item{}, dirs)
+
+ // Timestamp the directories in reverse order
+ ts := t1
+ for i := len(dirs) - 1; i >= 0; i-- {
+ dir := dirs[i]
+ _, err := operations.SetDirModTime(ctx, r.Flocal, nil, dir, ts)
+ require.NoError(t, err)
+ ts = ts.Add(time.Minute)
+ }
+
+ r.Mkdir(ctx, r.Fremote)
+
+ ctx = predictDstFromLogger(ctx)
+ err := Sync(ctx, r.Fremote, r.Flocal, true)
+ require.NoError(t, err)
+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
+
+ r.CheckRemoteListing(t, []fstest.Item{}, dirs)
+
+ // Check that the modtimes of the directories are as expected
+ r.CheckDirectoryModTimes(t, dirs...)
}
// Test a server-side copy if possible, or the backup path if not
diff --git a/fstest/run.go b/fstest/run.go
index d03d0ef83..dbb794337 100644
--- a/fstest/run.go
+++ b/fstest/run.go
@@ -360,6 +360,22 @@ func (r *Run) CheckRemoteListing(t *testing.T, items []Item, expectedDirs []stri
CheckListingWithPrecision(t, r.Fremote, items, expectedDirs, r.Precision)
}
+// CheckDirectoryModTimes checks that the directory names in r.Flocal has the correct modtime compared to r.Fremote
+func (r *Run) CheckDirectoryModTimes(t *testing.T, names ...string) {
+ if r.Fremote.Features().DirSetModTime == nil && r.Fremote.Features().MkdirMetadata == nil {
+ fs.Debugf(r.Fremote, "Skipping modtime test as remote does not support DirSetModTime or MkdirMetadata")
+ return
+ }
+ ctx := context.Background()
+ for _, name := range names {
+ wantT := NewDirectory(ctx, t, r.Flocal, name).ModTime(ctx)
+ got := NewDirectory(ctx, t, r.Fremote, name)
+ gotT := got.ModTime(ctx)
+ fs.Debugf(r.Fremote, "Testing directory mod time of %q: wantT=%v, gotT=%v", name, wantT, gotT)
+ AssertTimeEqualWithPrecision(t, got.Remote(), wantT, gotT, fs.GetModifyWindow(ctx, r.Fremote, r.Flocal))
+ }
+}
+
// Clean the temporary directory
func (r *Run) cleanTempDir() {
err := os.RemoveAll(r.LocalName)