"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
sync: implement directory sync for mod times and metadata
Nick Craig-Woodf5f86786b272
diff --git a/cmd/bisync/bisync_test.go b/cmd/bisync/bisync_test.goindex e9307d9ce..0b333b51d 100644--- a/cmd/bisync/bisync_test.go+++ b/cmd/bisync/bisync_test.go@@ -86,6 +86,10 @@ var logReplacements = []string{`^.*?"SlowHashDetected":.*?$`, dropMe,`^.*? for same-side diffs on .*?$`, dropMe,`^.*?Downloading hashes.*?$`, dropMe,+ // ignore timestamps in directory time updates+ `^(INFO : .*?: Made directory with (metadata|modification time)).*$`, `$1`,+ // ignore sizes in directory time updates+ `^(NOTICE: .*?: Skipped set directory modification time as --dry-run is set).*$`, `$1`,}// Some dry-run messages differ depending on the particular remote.@@ -121,6 +125,9 @@ var logHoppers = []string{// order of files re-checked prior to a conflict rename`ERROR : .*: md5 differ.*`,++ // Directory modification time setting can happen in any order+ `INFO : .*: (Set directory modification time|Made directory with metadata).*`,}// Some log lines can contain Windows path separator that must bediff --git a/cmd/bisync/testdata/test_all_changed/golden/test.log b/cmd/bisync/testdata/test_all_changed/golden/test.logindex cb74904c9..39c281bcb 100644--- a/cmd/bisync/testdata/test_all_changed/golden/test.log+++ b/cmd/bisync/testdata/test_all_changed/golden/test.log@@ -16,7 +16,11 @@ INFO : Bisyncing with Comparison Settings:INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"INFO : Copying Path2 files to Path1INFO : - [34mPath2[0m [35mResync is copying files to[0m - [36mPath1[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : - [36mPath1[0m [35mResync is copying files to[0m - [36mPath2[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : Resync updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : [32mBisync successful[0m@@ -59,6 +63,8 @@ INFO : - [36mPath1[0m [35m[32mQueue copy to[0m Path2[0m - [36m{INFO : - [36mPath1[0m [35m[32mQueue copy to[0m Path2[0m - [36m{path2/}file1.txt[0mINFO : - [36mPath1[0m [35m[32mQueue copy to[0m Path2[0m - [36m{path2/}subdir/file20.txt[0mINFO : - [36mPath1[0m [35mDo queued copies to[0m - [36mPath2[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : Updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : [32mBisync successful[0m@@ -133,6 +139,8 @@ INFO : - [36mPath1[0m [35m[32mQueue copy to[0m Path2[0m - [36m{INFO : - [36mPath1[0m [35m[32mQueue copy to[0m Path2[0m - [36m{path2/}file1.txt[0mINFO : - [36mPath1[0m [35m[32mQueue copy to[0m Path2[0m - [36m{path2/}subdir/file20.txt[0mINFO : - [36mPath1[0m [35mDo queued copies to[0m - [36mPath2[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : Updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : [32mBisync successful[0mdiff --git a/cmd/bisync/testdata/test_basic/golden/test.log b/cmd/bisync/testdata/test_basic/golden/test.logindex d3086e790..912423c26 100644--- a/cmd/bisync/testdata/test_basic/golden/test.log+++ b/cmd/bisync/testdata/test_basic/golden/test.log@@ -16,7 +16,11 @@ INFO : Bisyncing with Comparison Settings:INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"INFO : Copying Path2 files to Path1INFO : - [34mPath2[0m [35mResync is copying files to[0m - [36mPath1[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : - [36mPath1[0m [35mResync is copying files to[0m - [36mPath2[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : Resync updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : [32mBisync successful[0m@@ -53,6 +57,8 @@ INFO : - [36mPath1[0m [35m[32mQueue copy to[0m Path2[0m - [36m{INFO : - [34mPath2[0m [35m[32mQueue copy to[0m Path1[0m - [36m{path1/}file1.txt[0mINFO : - [34mPath2[0m [35mDo queued copies to[0m - [36mPath1[0mINFO : - [36mPath1[0m [35mDo queued copies to[0m - [36mPath2[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : Updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : [32mBisync successful[0mdiff --git a/cmd/bisync/testdata/test_check_access/golden/test.log b/cmd/bisync/testdata/test_check_access/golden/test.logindex 4184a0f30..abc480241 100644--- a/cmd/bisync/testdata/test_check_access/golden/test.log+++ b/cmd/bisync/testdata/test_check_access/golden/test.log@@ -16,7 +16,11 @@ INFO : Bisyncing with Comparison Settings:INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"INFO : Copying Path2 files to Path1INFO : - [34mPath2[0m [35mResync is copying files to[0m - [36mPath1[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : - [36mPath1[0m [35mResync is copying files to[0m - [36mPath2[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : Resync updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : [32mBisync successful[0m@@ -87,7 +91,11 @@ INFO : Bisyncing with Comparison Settings:INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"INFO : Copying Path2 files to Path1INFO : - [34mPath2[0m [35mResync is copying files to[0m - [36mPath1[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : - [36mPath1[0m [35mResync is copying files to[0m - [36mPath2[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : Resync updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : [32mBisync successful[0m@@ -180,7 +188,11 @@ INFO : Bisyncing with Comparison Settings:INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"INFO : Copying Path2 files to Path1INFO : - [34mPath2[0m [35mResync is copying files to[0m - [36mPath1[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : - [36mPath1[0m [35mResync is copying files to[0m - [36mPath2[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : Resync updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : [32mBisync successful[0mdiff --git a/cmd/bisync/testdata/test_check_access_filters/golden/test.log b/cmd/bisync/testdata/test_check_access_filters/golden/test.logindex eeb347438..82cc20179 100644--- a/cmd/bisync/testdata/test_check_access_filters/golden/test.log+++ b/cmd/bisync/testdata/test_check_access_filters/golden/test.log@@ -21,7 +21,15 @@ INFO : Using filters file {workdir/}exclude-other-filtersfile.txtINFO : Storing filters file hash to {workdir/}exclude-other-filtersfile.txt.md5INFO : Copying Path2 files to Path1INFO : - [34mPath2[0m [35mResync is copying files to[0m - [36mPath1[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir/subdirB: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir/subdirB: Set directory modification time (using SetModTime)INFO : - [36mPath1[0m [35mResync is copying files to[0m - [36mPath2[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir/subdirB: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir/subdirB: Set directory modification time (using SetModTime)INFO : Resync updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : [32mBisync successful[0m@@ -136,7 +144,23 @@ INFO : Using filters file {workdir/}include-other-filtersfile.txtINFO : Storing filters file hash to {workdir/}include-other-filtersfile.txt.md5INFO : Copying Path2 files to Path1INFO : - [34mPath2[0m [35mResync is copying files to[0m - [36mPath1[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdirX: Set directory modification time (using SetModTime)+INFO : subdirX/subdirX1: Set directory modification time (using SetModTime)+INFO : subdir/subdirB: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdirX: Set directory modification time (using SetModTime)+INFO : subdirX/subdirX1: Set directory modification time (using SetModTime)+INFO : subdir/subdirB: Set directory modification time (using SetModTime)INFO : - [36mPath1[0m [35mResync is copying files to[0m - [36mPath2[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdirX: Set directory modification time (using SetModTime)+INFO : subdir/subdirB: Set directory modification time (using SetModTime)+INFO : subdirX/subdirX1: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdirX: Set directory modification time (using SetModTime)+INFO : subdir/subdirB: Set directory modification time (using SetModTime)+INFO : subdirX/subdirX1: Set directory modification time (using SetModTime)INFO : Resync updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : [32mBisync successful[0mdiff --git a/cmd/bisync/testdata/test_check_filename/golden/test.log b/cmd/bisync/testdata/test_check_filename/golden/test.logindex 573976ede..8315bfc32 100644--- a/cmd/bisync/testdata/test_check_filename/golden/test.log+++ b/cmd/bisync/testdata/test_check_filename/golden/test.log@@ -16,7 +16,11 @@ INFO : Bisyncing with Comparison Settings:INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"INFO : Copying Path2 files to Path1INFO : - [34mPath2[0m [35mResync is copying files to[0m - [36mPath1[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : - [36mPath1[0m [35mResync is copying files to[0m - [36mPath2[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : Resync updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : [32mBisync successful[0m@@ -90,7 +94,11 @@ INFO : Copying Path2 files to Path1INFO : Checking access healthINFO : Found 2 matching ".chk_file" files on both pathsINFO : - [34mPath2[0m [35mResync is copying files to[0m - [36mPath1[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : - [36mPath1[0m [35mResync is copying files to[0m - [36mPath2[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : Resync updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : [32mBisync successful[0mdiff --git a/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log b/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.logindex 096c5f0c5..3cc3d7e0e 100644--- a/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log+++ b/cmd/bisync/testdata/test_createemptysrcdirs/golden/test.log@@ -147,7 +147,11 @@ INFO : Bisyncing with Comparison Settings:INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"INFO : Copying Path2 files to Path1INFO : - [34mPath2[0m [35mResync is copying files to[0m - [36mPath1[0m+INFO : subdir: Made directory with metadata (mtime=2024-02-27T04:53:52.809861575-05:00)+INFO : subdir: Set directory modification time (using SetModTime)INFO : - [36mPath1[0m [35mResync is copying files to[0m - [36mPath2[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : Resync updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : [32mBisync successful[0mdiff --git a/cmd/bisync/testdata/test_extended_char_paths/golden/test.log b/cmd/bisync/testdata/test_extended_char_paths/golden/test.logindex 463e8e967..db033110e 100644--- a/cmd/bisync/testdata/test_extended_char_paths/golden/test.log+++ b/cmd/bisync/testdata/test_extended_char_paths/golden/test.log@@ -84,7 +84,11 @@ INFO : Bisyncing with Comparison Settings:INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"INFO : Copying Path2 files to Path1INFO : - [34mPath2[0m [35mResync is copying files to[0m - [36mPath1[0m+INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)+INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)INFO : - [36mPath1[0m [35mResync is copying files to[0m - [36mPath2[0m+INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)+INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)INFO : Resync updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : [32mBisync successful[0m@@ -129,7 +133,11 @@ INFO : Bisyncing with Comparison Settings:INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"INFO : Copying Path2 files to Path1INFO : - [34mPath2[0m [35mResync is copying files to[0m - [36mPath1[0m+INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)+INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)INFO : - [36mPath1[0m [35mResync is copying files to[0m - [36mPath2[0m+INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)+INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)INFO : Resync updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : [32mBisync successful[0m@@ -174,7 +182,11 @@ INFO : Using filters file {workdir/}測試_filtersfile.txtINFO : Storing filters file hash to {workdir/}測試_filtersfile.txt.md5INFO : Copying Path2 files to Path1INFO : - [34mPath2[0m [35mResync is copying files to[0m - [36mPath1[0m+INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)+INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)INFO : - [36mPath1[0m [35mResync is copying files to[0m - [36mPath2[0m+INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)+INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)INFO : Resync updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : [32mBisync successful[0mdiff --git a/cmd/bisync/testdata/test_extended_filenames/golden/test.log b/cmd/bisync/testdata/test_extended_filenames/golden/test.logindex eda13848c..37cae89d2 100644--- a/cmd/bisync/testdata/test_extended_filenames/golden/test.log+++ b/cmd/bisync/testdata/test_extended_filenames/golden/test.log@@ -25,7 +25,11 @@ INFO : Bisyncing with Comparison Settings:INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"INFO : Copying Path2 files to Path1INFO : - [34mPath2[0m [35mResync is copying files to[0m - [36mPath1[0m+INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)+INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)INFO : - [36mPath1[0m [35mResync is copying files to[0m - [36mPath2[0m+INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)+INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)INFO : Resync updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : [32mBisync successful[0m@@ -112,7 +116,15 @@ INFO : - [34mPath2[0m [35m[32mQueue copy to[0m Path1[0m - [36m"INFO : - [36mPath1[0m [35m[31mQueue delete[0m[0m - [36m{path1/}subdir_with_ࢺ_/filename_contains_ě_.txt[0mINFO : - [34mPath2[0m [35m[32mQueue copy to[0m Path1[0m - [36m{path1/}subdir_with_ࢺ_/filename_contains_ࢺ_p2s.txt[0mINFO : - [34mPath2[0m [35mDo queued copies to[0m - [36mPath1[0m+INFO : subdir with␊white space.txt: Made directory with metadata (mtime=2024-02-27T04:53:52.913860529-05:00)+INFO : subdir_rawchars_␙_�_�: Made directory with metadata (mtime=2024-02-27T04:53:52.913860529-05:00)+INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)+INFO : subdir with␊white space.txt: Set directory modification time (using SetModTime)+INFO : subdir_rawchars_␙_�_�: Set directory modification time (using SetModTime)+INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)INFO : - [36mPath1[0m [35mDo queued copies to[0m - [36mPath2[0m+INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)+INFO : subdir_with_ࢺ_: Set directory modification time (using SetModTime)INFO : Updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : [32mBisync successful[0mdiff --git a/cmd/bisync/testdata/test_filters/golden/test.log b/cmd/bisync/testdata/test_filters/golden/test.logindex 9da9b2130..d74044ef1 100644--- a/cmd/bisync/testdata/test_filters/golden/test.log+++ b/cmd/bisync/testdata/test_filters/golden/test.log@@ -20,7 +20,11 @@ INFO : Using filters file {workdir/}filtersfile.fltINFO : Storing filters file hash to {workdir/}filtersfile.flt.md5INFO : Copying Path2 files to Path1INFO : - [34mPath2[0m [35mResync is copying files to[0m - [36mPath1[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : - [36mPath1[0m [35mResync is copying files to[0m - [36mPath2[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : Resync updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : [32mBisync successful[0m@@ -54,6 +58,8 @@ INFO : Path2 checking for diffsINFO : Applying changesINFO : - [36mPath1[0m [35m[32mQueue copy to[0m Path2[0m - [36m{path2/}subdir/fileZ.txt[0mINFO : - [36mPath1[0m [35mDo queued copies to[0m - [36mPath2[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : Updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : [32mBisync successful[0mdiff --git a/cmd/bisync/testdata/test_filtersfile_checks/golden/test.log b/cmd/bisync/testdata/test_filtersfile_checks/golden/test.logindex a60aa5d49..28f4cfa89 100644--- a/cmd/bisync/testdata/test_filtersfile_checks/golden/test.log+++ b/cmd/bisync/testdata/test_filtersfile_checks/golden/test.log@@ -16,7 +16,11 @@ INFO : Bisyncing with Comparison Settings:INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"INFO : Copying Path2 files to Path1INFO : - [34mPath2[0m [35mResync is copying files to[0m - [36mPath1[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : - [36mPath1[0m [35mResync is copying files to[0m - [36mPath2[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : Resync updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : [32mBisync successful[0m@@ -81,7 +85,11 @@ INFO : Using filters file {workdir/}filtersfile.txtINFO : Storing filters file hash to {workdir/}filtersfile.txt.md5INFO : Copying Path2 files to Path1INFO : - [34mPath2[0m [35mResync is copying files to[0m - [36mPath1[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : - [36mPath1[0m [35mResync is copying files to[0m - [36mPath2[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : Resync updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : [32mBisync successful[0m@@ -146,7 +154,11 @@ INFO : Using filters file {workdir/}filtersfile.txtINFO : Skipped storing filters file hash to {workdir/}filtersfile.txt.md5 as --dry-run is setINFO : Copying Path2 files to Path1INFO : - [34mPath2[0m [35mResync is copying files to[0m - [36mPath1[0m+NOTICE: subdir: Skipped set directory modification time as --dry-run is set (size 4Ki)+NOTICE: {path1String}: Skipped set directory modification time as --dry-run is setINFO : - [36mPath1[0m [35mResync is copying files to[0m - [36mPath2[0m+NOTICE: subdir: Skipped set directory modification time as --dry-run is set (size 4Ki)+NOTICE: {path2String}: Skipped set directory modification time as --dry-run is setINFO : Resync updating listingsINFO : [32mBisync successful[0mdiff --git a/cmd/bisync/testdata/test_ignorelistingchecksum/golden/test.log b/cmd/bisync/testdata/test_ignorelistingchecksum/golden/test.logindex f54cd7baf..c7f3c1bd7 100644--- a/cmd/bisync/testdata/test_ignorelistingchecksum/golden/test.log+++ b/cmd/bisync/testdata/test_ignorelistingchecksum/golden/test.log@@ -16,7 +16,11 @@ INFO : Bisyncing with Comparison Settings:INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"INFO : Copying Path2 files to Path1INFO : - [34mPath2[0m [35mResync is copying files to[0m - [36mPath1[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : - [36mPath1[0m [35mResync is copying files to[0m - [36mPath2[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : Resync updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : [32mBisync successful[0m@@ -33,7 +37,11 @@ INFO : Bisyncing with Comparison Settings:INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"INFO : Copying Path2 files to Path1INFO : - [34mPath2[0m [35mResync is copying files to[0m - [36mPath1[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : - [36mPath1[0m [35mResync is copying files to[0m - [36mPath2[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : Resync updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : [32mBisync successful[0m@@ -69,6 +77,8 @@ INFO : - [36mPath1[0m [35m[32mQueue copy to[0m Path2[0m - [36m{INFO : - [34mPath2[0m [35m[32mQueue copy to[0m Path1[0m - [36m{path1/}file1.txt[0mINFO : - [34mPath2[0m [35mDo queued copies to[0m - [36mPath1[0mINFO : - [36mPath1[0m [35mDo queued copies to[0m - [36mPath2[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : Updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : [32mBisync successful[0mdiff --git a/cmd/bisync/testdata/test_normalization/golden/test.log b/cmd/bisync/testdata/test_normalization/golden/test.logindex 5e4cbbccf..98144aee1 100644--- a/cmd/bisync/testdata/test_normalization/golden/test.log+++ b/cmd/bisync/testdata/test_normalization/golden/test.log@@ -17,7 +17,11 @@ INFO : Bisyncing with Comparison Settings:INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"INFO : Copying Path2 files to Path1INFO : - [34mPath2[0m [35mResync is copying files to[0m - [36mPath1[0m+INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)+INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)INFO : - [36mPath1[0m [35mResync is copying files to[0m - [36mPath2[0m+INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)+INFO : 測試_Русский_ _ _ě_áñ: Set directory modification time (using SetModTime)INFO : Resync updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : [32mBisync successful[0m@@ -70,8 +74,12 @@ INFO : - [36mPath1[0m [35m[32mQueue copy to[0m Path2[0m - [36m"INFO : - [34mPath2[0m [35m[32mQueue copy to[0m Path1[0m - [36m{path1/}file1.txt[0mINFO : - [34mPath2[0m [35mDo queued copies to[0m - [36mPath1[0mINFO : - [36mPath1[0m [35mDo queued copies to[0m - [36mPath2[0m+INFO : 測試_Русский___ě_áñ👸🏼🧝🏾♀️💆🏿♂️🐨🤙🏼🤮🧑🏻🔧🧑🔬éö: Made directory with metadata (mtime=2024-02-27T04:53:52.993859723-05:00)+INFO : folder: Set directory modification time (using SetModTime)INFO : folder/hello,WORLD!.txt: Fixed case by renaming to: folder/HeLlO,wOrLd!.txtINFO : folder/éééö.txt: Fixed case by renaming to: folder/éééö.txt+INFO : 測試_Русский___ě_áñ👸🏼🧝🏾♀️💆🏿♂️🐨🤙🏼🤮🧑🏻🔧🧑🔬éö: Set directory modification time (using SetModTime)+INFO : folder: Set directory modification time (using SetModTime)INFO : Updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : [32mBisync successful[0m@@ -150,7 +158,11 @@ INFO : - [34mPath2[0m [35m[32mQueue copy to[0m Path1[0m - [36m{INFO : - [36mPath1[0m [35m[32mQueue copy to[0m Path2[0m - [36m"{path2/}測試_Руский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éö/測試_Руский___ě_áñ👸🏼🧝🏾\u200d♀️💆🏿\u200d♂️🐨🤙🏼🤮🧑🏻\u200d🔧🧑\u200d🔬éö.txt"[0mINFO : - [34mPath2[0m [35m[32mQueue copy to[0m Path1[0m - [36m{path1/}file1.txt[0mINFO : - [34mPath2[0m [35mDo queued copies to[0m - [36mPath1[0m+INFO : folder: Set directory modification time (using SetModTime)+INFO : folder: Set directory modification time (using SetModTime)INFO : - [36mPath1[0m [35mDo queued copies to[0m - [36mPath2[0m+INFO : 測試_Руский___ě_áñ👸🏼🧝🏾♀️💆🏿♂️🐨🤙🏼🤮🧑🏻🔧🧑🔬éö: Made directory with metadata (mtime=2024-02-27T04:53:53.001859642-05:00)+INFO : 測試_Руский___ě_áñ👸🏼🧝🏾♀️💆🏿♂️🐨🤙🏼🤮🧑🏻🔧🧑🔬éö: Set directory modification time (using SetModTime)INFO : Updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : [32mBisync successful[0m@@ -170,7 +182,15 @@ INFO : Bisyncing with Comparison Settings:INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"INFO : Copying Path2 files to Path1INFO : - [34mPath2[0m [35mResync is copying files to[0m - [36mPath1[0m+INFO : folder: Set directory modification time (using SetModTime)+INFO : 測試_Руский___ě_áñ👸🏼🧝🏾♀️💆🏿♂️🐨🤙🏼🤮🧑🏻🔧🧑🔬éö: Set directory modification time (using SetModTime)+INFO : folder: Set directory modification time (using SetModTime)+INFO : 測試_Руский___ě_áñ👸🏼🧝🏾♀️💆🏿♂️🐨🤙🏼🤮🧑🏻🔧🧑🔬éö: Set directory modification time (using SetModTime)INFO : - [36mPath1[0m [35mResync is copying files to[0m - [36mPath2[0m+INFO : folder: Set directory modification time (using SetModTime)+INFO : 測試_Руский___ě_áñ👸🏼🧝🏾♀️💆🏿♂️🐨🤙🏼🤮🧑🏻🔧🧑🔬éö: Set directory modification time (using SetModTime)+INFO : folder: Set directory modification time (using SetModTime)+INFO : 測試_Руский___ě_áñ👸🏼🧝🏾♀️💆🏿♂️🐨🤙🏼🤮🧑🏻🔧🧑🔬éö: Set directory modification time (using SetModTime)INFO : Resync updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : [32mBisync successful[0m@@ -210,6 +230,10 @@ INFO : - [36mPath1[0m [35m[32mQueue copy to[0m Path2[0m - [36m"INFO : - [34mPath2[0m [35m[32mQueue copy to[0m Path1[0m - [36m{path1/}file1.txt[0mINFO : - [34mPath2[0m [35mDo queued copies to[0m - [36mPath1[0mINFO : - [36mPath1[0m [35mDo queued copies to[0m - [36mPath2[0m+INFO : folder: Set directory modification time (using SetModTime)+INFO : 測試_Руский___ě_áñ👸🏼🧝🏾♀️💆🏿♂️🐨🤙🏼🤮🧑🏻🔧🧑🔬éö: Set directory modification time (using SetModTime)+INFO : folder: Set directory modification time (using SetModTime)+INFO : 測試_Руский___ě_áñ👸🏼🧝🏾♀️💆🏿♂️🐨🤙🏼🤮🧑🏻🔧🧑🔬éö: Set directory modification time (using SetModTime)INFO : Updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : [32mBisync successful[0mdiff --git a/cmd/bisync/testdata/test_rclone_args/golden/test.log b/cmd/bisync/testdata/test_rclone_args/golden/test.logindex 73ede1e69..4fbe6c385 100644--- a/cmd/bisync/testdata/test_rclone_args/golden/test.log+++ b/cmd/bisync/testdata/test_rclone_args/golden/test.log@@ -15,7 +15,11 @@ INFO : Bisyncing with Comparison Settings:INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"INFO : Copying Path2 files to Path1INFO : - [34mPath2[0m [35mResync is copying files to[0m - [36mPath1[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : - [36mPath1[0m [35mResync is copying files to[0m - [36mPath2[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : Resync updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : [32mBisync successful[0m@@ -107,7 +111,11 @@ INFO : Bisyncing with Comparison Settings:INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"INFO : Copying Path2 files to Path1INFO : - [34mPath2[0m [35mResync is copying files to[0m - [36mPath1[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : - [36mPath1[0m [35mResync is copying files to[0m - [36mPath2[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : Resync updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : [32mBisync successful[0m@@ -141,6 +149,8 @@ INFO : Applying changesINFO : - [34mPath2[0m [35m[32mQueue copy to[0m Path1[0m - [36m{path1/}file2.txt[0mINFO : - [34mPath2[0m [35m[32mQueue copy to[0m Path1[0m - [36m{path1/}subdir/file21.txt[0mINFO : - [34mPath2[0m [35mDo queued copies to[0m - [36mPath1[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : Updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : [32mBisync successful[0m@@ -158,7 +168,11 @@ INFO : Bisyncing with Comparison Settings:INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"INFO : Copying Path2 files to Path1INFO : - [34mPath2[0m [35mResync is copying files to[0m - [36mPath1[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : - [36mPath1[0m [35mResync is copying files to[0m - [36mPath2[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : Resync updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : [32mBisync successful[0m@@ -200,7 +214,11 @@ INFO : - [36mPath1[0m [35m[32mQueue copy to[0m Path2[0m - [36m{INFO : - [34mPath2[0m [35m[32mQueue copy to[0m Path1[0m - [36m{path1/}file2.txt[0mINFO : - [34mPath2[0m [35m[32mQueue copy to[0m Path1[0m - [36m{path1/}subdir/file21.txt[0mINFO : - [34mPath2[0m [35mDo queued copies to[0m - [36mPath1[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : - [36mPath1[0m [35mDo queued copies to[0m - [36mPath2[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : Updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : [32mBisync successful[0mdiff --git a/cmd/bisync/testdata/test_resolve/golden/test.log b/cmd/bisync/testdata/test_resolve/golden/test.logindex 9223c3f6a..dfc303f07 100644--- a/cmd/bisync/testdata/test_resolve/golden/test.log+++ b/cmd/bisync/testdata/test_resolve/golden/test.log@@ -283,8 +283,8 @@ INFO : Path2: 2 changes: [32m 1 new[0m, [33m 1 modified[0m, [31mINFO : ([33mModified[0m: [36m 1 newer[0m, [34m 0 older[0m, [36m 1 larger[0m, [34m 0 smaller[0m)INFO : Applying changesINFO : Checking potential conflicts...-ERROR : file2.txt: md5 differERROR : file1.txt: md5 differ+ERROR : file2.txt: md5 differNOTICE: {path2String}: 2 differences foundNOTICE: {path2String}: 2 errors while checkingINFO : Finished checking the potential conflicts. 2 differences founddiff --git a/cmd/bisync/testdata/test_rmdirs/golden/test.log b/cmd/bisync/testdata/test_rmdirs/golden/test.logindex 4e6763801..899e9a5f3 100644--- a/cmd/bisync/testdata/test_rmdirs/golden/test.log+++ b/cmd/bisync/testdata/test_rmdirs/golden/test.log@@ -16,7 +16,11 @@ INFO : Bisyncing with Comparison Settings:INFO : Synching Path1 "{path1/}" with Path2 "{path2/}"INFO : Copying Path2 files to Path1INFO : - [34mPath2[0m [35mResync is copying files to[0m - [36mPath1[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : - [36mPath1[0m [35mResync is copying files to[0m - [36mPath2[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : Resync updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : [32mBisync successful[0m@@ -45,6 +49,8 @@ INFO : Path2 checking for diffsINFO : Applying changesINFO : - [34mPath2[0m [35m[31mQueue delete[0m[0m - [36m{path2/}subdir/file20.txt[0mINFO : - [36mPath1[0m [35mDo queued copies to[0m - [36mPath2[0m+INFO : subdir: Set directory modification time (using SetModTime)+INFO : subdir: Set directory modification time (using SetModTime)INFO : Updating listingsINFO : Validating listings for Path1 "{path1/}" vs Path2 "{path2/}"INFO : [32mBisync successful[0mdiff --git a/cmd/copy/copy.go b/cmd/copy/copy.goindex 745811de4..6a68a330c 100644--- a/cmd/copy/copy.go+++ b/cmd/copy/copy.go@@ -79,6 +79,15 @@ recently very efficiently like this:rclone copy --max-age 24h --no-traverse /path/to/src remote:++Rclone will sync the modification times of files and directories if+the backend supports it. If metadata syncing is required then use the+|--metadata| flag.++Note that the modification time and metadata for the root directory+will **not** be synced. See https://github.com/rclone/rclone/issues/7652+for more info.+**Note**: Use the |-P|/|--progress| flag to view real-time transfer statistics.**Note**: Use the |--dry-run| or the |--interactive|/|-i| flag to test without copying anything.diff --git a/cmd/move/move.go b/cmd/move/move.goindex ae7260430..ca3f448b7 100644--- a/cmd/move/move.go+++ b/cmd/move/move.go@@ -55,6 +55,14 @@ whether rclone lists the destination directory or not. Supplying thisoption when moving a small number of files into a large destinationcan speed transfers up greatly.+Rclone will sync the modification times of files and directories if+the backend supports it. If metadata syncing is required then use the+|--metadata| flag.++Note that the modification time and metadata for the root directory+will **not** be synced. See https://github.com/rclone/rclone/issues/7652+for more info.+**Important**: Since this can cause data loss, test first with the|--dry-run| or the |--interactive|/|-i| flag.diff --git a/cmd/sync/sync.go b/cmd/sync/sync.goindex 38076541e..ba35fa3e6 100644--- a/cmd/sync/sync.go+++ b/cmd/sync/sync.go@@ -170,6 +170,14 @@ the destination from the sync with a filter rule or by putting anexclude-if-present file inside the destination directory and sync to adestination that is inside the source directory.+Rclone will sync the modification times of files and directories if+the backend supports it. If metadata syncing is required then use the+` + "`--metadata`" + ` flag.++Note that the modification time and metadata for the root directory+will **not** be synced. See https://github.com/rclone/rclone/issues/7652+for more info.+**Note**: Use the ` + "`-P`" + `/` + "`--progress`" + ` flag to view real-time transfer statistics**Note**: Use the ` + "`rclone dedupe`" + ` command to deal with "Duplicate object/directory found in source/destination - ignoring" errors.diff --git a/docs/content/docs.md b/docs/content/docs.mdindex 821d6f4b0..3b3a50a63 100644--- a/docs/content/docs.md+++ b/docs/content/docs.md@@ -446,18 +446,21 @@ This can be used when scripting to make aged backups efficiently, e.g.## Metadata support {#metadata}-Metadata is data about a file which isn't the contents of the file.-Normally rclone only preserves the modification time and the content-(MIME) type where possible.+Metadata is data about a file (or directory) which isn't the contents+of the file (or directory). Normally rclone only preserves the+modification time and the content (MIME) type where possible.-Rclone supports preserving all the available metadata on files (not-directories) when using the `--metadata` or `-M` flag.+Rclone supports preserving all the available metadata on files and+directories when using the `--metadata` or `-M` flag.Exactly what metadata is supported and what that support means dependson the backend. Backends that support metadata have a metadata sectionin their docs and are listed in the [features table](/overview/#features)(Eg [local](/local/#metadata), [s3](/s3/#metadata))+Some backends don't support metadata, some only support metadata on+files and some support metadata on both files and directories.+Rclone only supports a one-time sync of metadata. This means thatmetadata will be synced from the source object to the destinationobject only when the source object has changed and needs to be@@ -1560,10 +1563,10 @@ some context for the `Metadata` which may be important.- `SrcFsType` is the name of the source backend.- `DstFs` is the config string for the remote that the object is being copied to- `DstFsType` is the name of the destination backend.-- `Remote` is the path of the file relative to the root.-- `Size`, `MimeType`, `ModTime` are attributes of the file.+- `Remote` is the path of the object relative to the root.+- `Size`, `MimeType`, `ModTime` are attributes of the object.- `IsDir` is `true` if this is a directory (not yet implemented).-- `ID` is the source `ID` of the file if known.+- `ID` is the source `ID` of the object if known.- `Metadata` is the backend specific metadata as described in the backend docs.```jsondiff --git a/fs/sync/sync.go b/fs/sync/sync.goindex c464235c0..0ca8e1d27 100644--- a/fs/sync/sync.go+++ b/fs/sync/sync.go@@ -18,6 +18,8 @@ import ("github.com/rclone/rclone/fs/hash""github.com/rclone/rclone/fs/march""github.com/rclone/rclone/fs/operations"+ "github.com/rclone/rclone/lib/errcount"+ "golang.org/x/sync/errgroup")// ErrorMaxDurationReached defines error when transfer duration is reached@@ -84,6 +86,20 @@ type syncCopyMove struct {maxDurationEndTime time.Time // end time if --max-duration is setlogger operations.LoggerFn // LoggerFn used to report the results of a sync (or bisync) to an io.WriterusingLogger bool // whether we are using logger+ setDirMetadata bool // if set we set the directory metadata+ setDirModTime bool // if set we set the directory modtimes+ setDirModTimeAfter bool // if set we set the directory modtimes at the end of the sync+ setDirModTimeMu sync.Mutex // protect setDirModTimeMu+ setDirModTimes []setDirModTime // directories that need their modtime set+ setDirModTimesMaxLevel int // max level of the directories to set+}++// For keeping track of delayed modtime sets+type setDirModTime struct {+ dst fs.Directory+ dir string+ modTime time.Time+ level int // the level of the directory, 0 is root}type trackRenamesStrategy byte@@ -136,6 +152,9 @@ func newSyncCopyMove(ctx context.Context, fdst, fsrc fs.Fs, deleteMode fs.DeletemodifyWindow: fs.GetModifyWindow(ctx, fsrc, fdst),trackRenamesCh: make(chan fs.Object, ci.Checkers),checkFirst: ci.CheckFirst,+ setDirMetadata: ci.Metadata && fsrc.Features().ReadDirMetadata && fdst.Features().WriteDirMetadata,+ setDirModTime: fdst.Features().WriteDirSetModTime || fdst.Features().MkdirMetadata != nil || fdst.Features().DirSetModTime != nil,+ setDirModTimeAfter: fdst.Features().DirModTimeUpdatesOnWrite,}s.logger, s.usingLogger = operations.GetLogger(ctx)@@ -966,6 +985,11 @@ func (s *syncCopyMove) run() error {}}+ // Update modtimes for directories if necessary+ if s.setDirModTime && s.setDirModTimeAfter {+ s.processError(s.setDelayedDirModTimes(s.ctx))+ }+// Prune empty directoriesif s.deleteMode != fs.DeleteModeOff {if s.currentError() != nil && !s.ci.IgnoreErrors {@@ -1055,6 +1079,96 @@ func (s *syncCopyMove) DstOnly(dst fs.DirEntry) (recurse bool) {return false}+// copyDirMetadata copies the src directory modTime or Metadata to dst+// or f if nil. If dst is nil then it uses dir as the name of the new+// directory.+//+// It returns the destination directory if possible. Note that this may+// be nil.+func (s *syncCopyMove) copyDirMetadata(ctx context.Context, f fs.Fs, dst fs.Directory, dir string, src fs.Directory) (newDst fs.Directory) {+ var err error+ if s.setDirMetadata {+ newDst, err = operations.CopyDirMetadata(ctx, f, dst, dir, src)+ } else if s.setDirModTime {+ if dst == nil {+ newDst, err = operations.MkdirModTime(ctx, f, dir, src.ModTime(ctx))+ } else {+ newDst, err = operations.SetDirModTime(ctx, f, dst, dir, src.ModTime(ctx))+ }+ } else if dst == nil {+ // Create the directory if it doesn't exist+ err = operations.Mkdir(ctx, f, dir)+ }+ // If we need to set modtime after and we created a dir, then save it for later+ if s.setDirModTime && s.setDirModTimeAfter && err == nil {+ if newDst != nil {+ dir = newDst.Remote()+ }+ level := strings.Count(dir, "/") + 1+ // The root directory "" is at the top level+ if dir == "" {+ level = 0+ }+ s.setDirModTimeMu.Lock()+ // Keep track of the maximum level inserted+ if level > s.setDirModTimesMaxLevel {+ s.setDirModTimesMaxLevel = level+ }+ s.setDirModTimes = append(s.setDirModTimes, setDirModTime{+ dst: newDst,+ dir: dir,+ modTime: src.ModTime(ctx),+ level: level,+ })+ s.setDirModTimeMu.Unlock()+ fs.Debugf(nil, "Added delayed dir = %q, newDst=%v", dir, newDst)+ }+ s.processError(err)+ if err != nil {+ return nil+ }+ return newDst+}++// Set the modtimes for directories+func (s *syncCopyMove) setDelayedDirModTimes(ctx context.Context) error {+ s.setDirModTimeMu.Lock()+ defer s.setDirModTimeMu.Unlock()++ // Timestamp all directories at the same level in parallel, deepest first+ // We do this by iterating the slice multiple times to save memory+ // There could be a lot of directories in this slice.+ var errCount = errcount.New()+ for level := s.setDirModTimesMaxLevel; level >= 0; level-- {+ g, gCtx := errgroup.WithContext(ctx)+ g.SetLimit(s.ci.Checkers)+ for _, item := range s.setDirModTimes {+ if item.level != level {+ continue+ }+ // End early if error+ if gCtx.Err() != nil {+ break+ }+ item := item+ g.Go(func() error {+ _, err := operations.SetDirModTime(gCtx, s.fdst, item.dst, item.dir, item.modTime)+ if err != nil {+ err = fs.CountError(err)+ fs.Errorf(item.dir, "Failed to timestamp directory: %v", err)+ errCount.Add(err)+ }+ return nil // don't return errors, just count them+ })+ }+ err := g.Wait()+ if err != nil {+ return err+ }+ }+ return errCount.Err("failed to set directory modtime")+}+// SrcOnly have an object which is in the source onlyfunc (s *syncCopyMove) SrcOnly(src fs.DirEntry) (recurse bool) {if s.deleteMode == fs.DeleteModeOnly {@@ -1101,6 +1215,9 @@ func (s *syncCopyMove) SrcOnly(src fs.DirEntry) (recurse bool) {s.srcEmptyDirs[src.Remote()] = srcs.logger(s.ctx, operations.MissingOnDst, src, nil, fs.ErrorIsDir)s.srcEmptyDirsMu.Unlock()++ // Create the directory and make sure the Metadata/ModTime is correct+ s.copyDirMetadata(s.ctx, s.fdst, nil, x.Remote(), x)return truedefault:panic("Bad object in DirEntries")@@ -1135,9 +1252,12 @@ func (s *syncCopyMove) Match(ctx context.Context, dst, src fs.DirEntry) (recurse}case fs.Directory:// Do the same thing to the entire contents of the directory- _, ok := dst.(fs.Directory)+ dstX, ok := dst.(fs.Directory)if ok {s.logger(s.ctx, operations.Match, src, dst, fs.ErrorIsDir)+ // Create the directory and make sure the Metadata/ModTime is correct+ s.copyDirMetadata(s.ctx, s.fdst, dstX, "", srcX)+// Only record matched (src & dst) empty dirs when performing moveif s.DoMove {// Record the src directory for deletiondiff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.goindex 7911be1d0..c0728cf06 100644--- a/fs/sync/sync_test.go+++ b/fs/sync/sync_test.go@@ -7,6 +7,7 @@ import ("context""errors""fmt"+ "io""os""os/exec""runtime"@@ -65,15 +66,88 @@ func TestCopy(t *testing.T) {ctx := context.Background()r := fstest.NewRun(t)file1 := r.WriteFile("sub dir/hello world", "hello world", t1)+ _, err := operations.SetDirModTime(ctx, r.Flocal, nil, "sub dir", t2)+ if err != nil && !errors.Is(err, fs.ErrorNotImplemented) {+ require.NoError(t, err)+ }r.Mkdir(ctx, r.Fremote)ctx = predictDstFromLogger(ctx)- err := CopyDir(ctx, r.Fremote, r.Flocal, false)+ err = CopyDir(ctx, r.Fremote, r.Flocal, false)require.NoError(t, err)testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)r.CheckLocalItems(t, file1)r.CheckRemoteItems(t, file1)++ // Check that the modtimes of the directories are as expected+ r.CheckDirectoryModTimes(t, "sub dir")+}++func TestCopyMetadata(t *testing.T) {+ ctx := context.Background()+ ctx, ci := fs.AddConfig(ctx)+ ci.Metadata = true+ r := fstest.NewRun(t)+ features := r.Fremote.Features()++ if !features.ReadMetadata && !features.WriteMetadata && !features.UserMetadata &&+ !features.ReadDirMetadata && !features.WriteDirMetadata && !features.UserDirMetadata {+ t.Skip("Skipping as metadata not supported")+ }++ const content = "hello metadata world!"+ const dirPath = "metadata sub dir"+ const filePath = dirPath + "/hello metadata world"++ fileMetadata := fs.Metadata{+ // System metadata supported by all backends+ "mtime": t1.Format(time.RFC3339Nano),+ // User metadata+ "potato": "jersey",+ }++ dirMetadata := fs.Metadata{+ // System metadata supported by all backends+ "mtime": t2.Format(time.RFC3339Nano),+ // User metadata+ "potato": "king edward",+ }++ // Make the directory with metadata - may fall back to Mkdir+ _, err := operations.MkdirMetadata(ctx, r.Flocal, dirPath, dirMetadata)+ require.NoError(t, err)++ // Upload the file with metadata+ in := io.NopCloser(bytes.NewBufferString(content))+ _, err = operations.Rcat(ctx, r.Flocal, filePath, in, t1, fileMetadata)+ require.NoError(t, err)+ file1 := fstest.NewItem(filePath, content, t1)++ // Reset the time of the directory+ _, err = operations.SetDirModTime(ctx, r.Flocal, nil, dirPath, t2)+ if err != nil && !errors.Is(err, fs.ErrorNotImplemented) {+ require.NoError(t, err)+ }++ ctx = predictDstFromLogger(ctx)+ err = CopyDir(ctx, r.Fremote, r.Flocal, false)+ require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)++ r.CheckLocalItems(t, file1)+ r.CheckRemoteItems(t, file1)++ // Check that the modtimes of the directories are as expected+ r.CheckDirectoryModTimes(t, dirPath)++ // Check that the metadata on the directory and file is correct+ if features.ReadMetadata {+ fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewObject(ctx, t, r.Fremote, filePath), fileMetadata)+ }+ if features.ReadDirMetadata {+ fstest.CheckEntryMetadata(ctx, t, r.Fremote, fstest.NewDirectory(ctx, t, r.Fremote, dirPath), dirMetadata)+ }}func TestCopyMissingDirectory(t *testing.T) {@@ -205,10 +279,15 @@ func TestCopyEmptyDirectories(t *testing.T) {ctx := context.Background()r := fstest.NewRun(t)file1 := r.WriteFile("sub dir/hello world", "hello world", t1)- err := operations.Mkdir(ctx, r.Flocal, "sub dir2")+ _, err := operations.MkdirModTime(ctx, r.Flocal, "sub dir2", t2)require.NoError(t, err)r.Mkdir(ctx, r.Fremote)+ // Set the modtime on "sub dir" to something specific+ // Without this it fails on the CI and in VirtualBox with variances of up to 10mS+ _, err = operations.SetDirModTime(ctx, r.Flocal, nil, "sub dir", t1)+ require.NoError(t, err)+ctx = predictDstFromLogger(ctx)err = CopyDir(ctx, r.Fremote, r.Flocal, true)require.NoError(t, err)@@ -224,6 +303,9 @@ func TestCopyEmptyDirectories(t *testing.T) {"sub dir2",},)++ // Check that the modtimes of the directories are as expected+ r.CheckDirectoryModTimes(t, "sub dir", "sub dir2")}// Test move empty directories@@ -231,8 +313,10 @@ func TestMoveEmptyDirectories(t *testing.T) {ctx := context.Background()r := fstest.NewRun(t)file1 := r.WriteFile("sub dir/hello world", "hello world", t1)- err := operations.Mkdir(ctx, r.Flocal, "sub dir2")+ _, err := operations.MkdirModTime(ctx, r.Flocal, "sub dir2", t2)require.NoError(t, err)+ subDir := fstest.NewDirectory(ctx, t, r.Flocal, "sub dir")+ subDirT := subDir.ModTime(ctx)r.Mkdir(ctx, r.Fremote)ctx = predictDstFromLogger(ctx)@@ -250,6 +334,14 @@ func TestMoveEmptyDirectories(t *testing.T) {"sub dir2",},)++ // Check that the modtimes of the directories are as expected+ r.CheckDirectoryModTimes(t, "sub dir2")+ // Note that "sub dir" mod time is updated when file1 is deleted from it+ // So check it more manually+ got := fstest.NewDirectory(ctx, t, r.Fremote, "sub dir")+ gotT := got.ModTime(ctx)+ fstest.AssertTimeEqualWithPrecision(t, subDir.Remote(), subDirT, gotT, fs.GetModifyWindow(ctx, r.Fremote, r.Flocal))}// Test sync empty directories@@ -257,8 +349,14 @@ func TestSyncEmptyDirectories(t *testing.T) {ctx := context.Background()r := fstest.NewRun(t)file1 := r.WriteFile("sub dir/hello world", "hello world", t1)- err := operations.Mkdir(ctx, r.Flocal, "sub dir2")+ _, err := operations.MkdirModTime(ctx, r.Flocal, "sub dir2", t2)require.NoError(t, err)++ // Set the modtime on "sub dir" to something specific+ // Without this it fails on the CI and in VirtualBox with variances of up to 10mS+ _, err = operations.SetDirModTime(ctx, r.Flocal, nil, "sub dir", t1)+ require.NoError(t, err)+r.Mkdir(ctx, r.Fremote)ctx = predictDstFromLogger(ctx)@@ -276,6 +374,65 @@ func TestSyncEmptyDirectories(t *testing.T) {"sub dir2",},)++ // Check that the modtimes of the directories are as expected+ r.CheckDirectoryModTimes(t, "sub dir", "sub dir2")+}++// Test delayed mod time setting+func TestSyncSetDelayedModTimes(t *testing.T) {+ ctx := context.Background()+ r := fstest.NewRun(t)++ if !r.Fremote.Features().DirModTimeUpdatesOnWrite {+ t.Skip("Backend doesn't have DirModTimeUpdatesOnWrite set")+ }++ // Create directories without timestamps+ require.NoError(t, r.Flocal.Mkdir(ctx, "a1/b1/c1/d1/e1/f1"))+ require.NoError(t, r.Flocal.Mkdir(ctx, "a1/b2/c1/d1/e1/f1"))+ require.NoError(t, r.Flocal.Mkdir(ctx, "a1/b1/c1/d2/e1/f1"))+ require.NoError(t, r.Flocal.Mkdir(ctx, "a1/b1/c1/d2/e1/f2"))++ dirs := []string{+ "a1",+ "a1/b1",+ "a1/b1/c1",+ "a1/b1/c1/d1",+ "a1/b1/c1/d1/e1",+ "a1/b1/c1/d1/e1/f1",+ "a1/b1/c1/d2",+ "a1/b1/c1/d2/e1",+ "a1/b1/c1/d2/e1/f1",+ "a1/b1/c1/d2/e1/f2",+ "a1/b2",+ "a1/b2/c1",+ "a1/b2/c1/d1",+ "a1/b2/c1/d1/e1",+ "a1/b2/c1/d1/e1/f1",+ }+ r.CheckLocalListing(t, []fstest.Item{}, dirs)++ // Timestamp the directories in reverse order+ ts := t1+ for i := len(dirs) - 1; i >= 0; i-- {+ dir := dirs[i]+ _, err := operations.SetDirModTime(ctx, r.Flocal, nil, dir, ts)+ require.NoError(t, err)+ ts = ts.Add(time.Minute)+ }++ r.Mkdir(ctx, r.Fremote)++ ctx = predictDstFromLogger(ctx)+ err := Sync(ctx, r.Fremote, r.Flocal, true)+ require.NoError(t, err)+ testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)++ r.CheckRemoteListing(t, []fstest.Item{}, dirs)++ // Check that the modtimes of the directories are as expected+ r.CheckDirectoryModTimes(t, dirs...)}// Test a server-side copy if possible, or the backup path if notdiff --git a/fstest/run.go b/fstest/run.goindex d03d0ef83..dbb794337 100644--- a/fstest/run.go+++ b/fstest/run.go@@ -360,6 +360,22 @@ func (r *Run) CheckRemoteListing(t *testing.T, items []Item, expectedDirs []striCheckListingWithPrecision(t, r.Fremote, items, expectedDirs, r.Precision)}+// CheckDirectoryModTimes checks that the directory names in r.Flocal has the correct modtime compared to r.Fremote+func (r *Run) CheckDirectoryModTimes(t *testing.T, names ...string) {+ if r.Fremote.Features().DirSetModTime == nil && r.Fremote.Features().MkdirMetadata == nil {+ fs.Debugf(r.Fremote, "Skipping modtime test as remote does not support DirSetModTime or MkdirMetadata")+ return+ }+ ctx := context.Background()+ for _, name := range names {+ wantT := NewDirectory(ctx, t, r.Flocal, name).ModTime(ctx)+ got := NewDirectory(ctx, t, r.Fremote, name)+ gotT := got.ModTime(ctx)+ fs.Debugf(r.Fremote, "Testing directory mod time of %q: wantT=%v, gotT=%v", name, wantT, gotT)+ AssertTimeEqualWithPrecision(t, got.Remote(), wantT, gotT, fs.GetModifyWindow(ctx, r.Fremote, r.Flocal))+ }+}+// Clean the temporary directoryfunc (r *Run) cleanTempDir() {err := os.RemoveAll(r.LocalName)