"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
fs/logger: fix flaky tests by generating test data locally
Nick Craig-Wood9a49790797f6
diff --git a/fs/logger/logger_test.go b/fs/logger/logger_test.goindex c3a57840a..75639a7ed 100644--- a/fs/logger/logger_test.go+++ b/fs/logger/logger_test.go@@ -3,6 +3,8 @@package logger_testimport (+ "fmt"+ "os""path/filepath""testing"@@ -26,9 +28,70 @@ func TestLogger(t *testing.T) {testscript.Run(t, testscript.Params{Dir: "testdata/script",Setup: func(env *testscript.Env) error {- env.Setenv("SRC", filepath.Join("$WORK", "src"))- env.Setenv("DST", filepath.Join("$WORK", "dst"))+ src := filepath.Join(env.WorkDir, "src")+ dst := filepath.Join(env.WorkDir, "dst")+ // Fill src and dst with two overlapping trees of files so the+ // scripts have a realistic mix of matching, differing and+ // missing files to compare. This used to download two old+ // rclone source archives from GitHub which made the tests fail+ // whenever the network or GitHub was flaky.+ if err := makeTestTrees(src, dst); err != nil {+ return err+ }+ env.Setenv("SRC", src)+ env.Setenv("DST", dst)return nil},})}++// makeTestTrees populates src and dst with a deterministic set of files+// covering every comparison category the scripts exercise:+//+// - identical files present in both (match)+// - same-named files with different content (differ)+// - files only in src (missing on dst)+// - files only in dst (missing on src)+//+// The files are spread across the root and a shared subdirectory so the+// listings need sorting and the directory survives a sync (which deletes the+// dst-only files but keeps the matching ones).+func makeTestTrees(src, dst string) error {+ const perCategory = 25+ dirs := []string{".", "sub"}++ write := func(root, dir, name, content string) error {+ path := filepath.Join(root, dir, name)+ if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil {+ return err+ }+ return os.WriteFile(path, []byte(content), 0666)+ }++ for _, dir := range dirs {+ for i := range perCategory {+ match := fmt.Sprintf("the same content for match%02d\n", i)+ if err := write(src, dir, fmt.Sprintf("match%02d.txt", i), match); err != nil {+ return err+ }+ if err := write(dst, dir, fmt.Sprintf("match%02d.txt", i), match); err != nil {+ return err+ }++ if err := write(src, dir, fmt.Sprintf("differ%02d.txt", i), fmt.Sprintf("src content %02d\n", i)); err != nil {+ return err+ }+ if err := write(dst, dir, fmt.Sprintf("differ%02d.txt", i), fmt.Sprintf("dst content %02d differs\n", i)); err != nil {+ return err+ }++ if err := write(src, dir, fmt.Sprintf("srconly%02d.txt", i), fmt.Sprintf("only in src %02d\n", i)); err != nil {+ return err+ }+ if err := write(dst, dir, fmt.Sprintf("dstonly%02d.txt", i), fmt.Sprintf("only in dst %02d\n", i)); err != nil {+ return err+ }+ }+ }+ return nil+}diff --git a/fs/logger/testdata/script/TestBeforeVsAfter.txtar b/fs/logger/testdata/script/TestBeforeVsAfter.txtarindex 0ccadc973..3e3b09681 100644--- a/fs/logger/testdata/script/TestBeforeVsAfter.txtar+++ b/fs/logger/testdata/script/TestBeforeVsAfter.txtar@@ -1,11 +1,5 @@# tests whether an md5sum file generated post-sync matches our pre-sync prediction--# Filling src and dst with two different versions of rclone source!:-exec rclone copyurl https://github.com/rclone/rclone/archive/refs/tags/v1.49.1.zip $SRC/src.zip-exec rclone copyurl https://github.com/rclone/rclone/archive/refs/tags/v1.54.1.zip $DST/dst.zip-exec unzip $SRC/src.zip -d $SRC-exec unzip $DST/dst.zip -d $DST-+# src and dst are filled by the test Setup with two overlapping trees of files.# generating sumfiles:exec rclone md5sum $SRC --output-file $WORK/src-before.txtdiff --git a/fs/logger/testdata/script/TestRepoCompare.txtar b/fs/logger/testdata/script/TestRepoCompare.txtarindex 4bbe9220a..80dc9272c 100644--- a/fs/logger/testdata/script/TestRepoCompare.txtar+++ b/fs/logger/testdata/script/TestRepoCompare.txtar@@ -1,14 +1,7 @@# tests whether rclone check and rclone sync output exactly the same file lists.-# uses two different old versions of rclone source code for the src and dst-# to produce a more realistic test than makefiles-# (has lots of files with same name but different content)--# Filling src and dst with two different versions of rclone source!:-exec rclone copyurl https://github.com/rclone/rclone/archive/refs/tags/v1.49.1.zip $SRC/src.zip-exec rclone copyurl https://github.com/rclone/rclone/archive/refs/tags/v1.54.1.zip $DST/dst.zip-exec unzip $SRC/src.zip -d $SRC-exec unzip $DST/dst.zip -d $DST-+# src and dst are filled by the test Setup with two overlapping trees of files+# (lots of files with the same name but different content) to produce a more+# realistic test than makefiles.# running rclone check for baseline test:# error is expected here: