Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

fs/logger: fix flaky tests by generating test data locally

Nick Craig-Wood9a49790797f6
diff --git a/fs/logger/logger_test.go b/fs/logger/logger_test.go
index c3a57840a..75639a7ed 100644
--- a/fs/logger/logger_test.go
+++ b/fs/logger/logger_test.go
@@ -3,6 +3,8 @@
package logger_test
import (
+ "fmt"
+ "os"
"path/filepath"
"testing"
@@ -26,9 +28,70 @@ func TestLogger(t *testing.T) {
testscript.Run(t, testscript.Params{
Dir: "testdata/script",
Setup: func(env *testscript.Env) error {
- env.Setenv("SRC", filepath.Join("$WORK", "src"))
- env.Setenv("DST", filepath.Join("$WORK", "dst"))
+ src := filepath.Join(env.WorkDir, "src")
+ dst := filepath.Join(env.WorkDir, "dst")
+ // Fill src and dst with two overlapping trees of files so the
+ // scripts have a realistic mix of matching, differing and
+ // missing files to compare. This used to download two old
+ // rclone source archives from GitHub which made the tests fail
+ // whenever the network or GitHub was flaky.
+ if err := makeTestTrees(src, dst); err != nil {
+ return err
+ }
+ env.Setenv("SRC", src)
+ env.Setenv("DST", dst)
return nil
},
})
}
+
+// makeTestTrees populates src and dst with a deterministic set of files
+// covering every comparison category the scripts exercise:
+//
+// - identical files present in both (match)
+// - same-named files with different content (differ)
+// - files only in src (missing on dst)
+// - files only in dst (missing on src)
+//
+// The files are spread across the root and a shared subdirectory so the
+// listings need sorting and the directory survives a sync (which deletes the
+// dst-only files but keeps the matching ones).
+func makeTestTrees(src, dst string) error {
+ const perCategory = 25
+ dirs := []string{".", "sub"}
+
+ write := func(root, dir, name, content string) error {
+ path := filepath.Join(root, dir, name)
+ if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil {
+ return err
+ }
+ return os.WriteFile(path, []byte(content), 0666)
+ }
+
+ for _, dir := range dirs {
+ for i := range perCategory {
+ match := fmt.Sprintf("the same content for match%02d\n", i)
+ if err := write(src, dir, fmt.Sprintf("match%02d.txt", i), match); err != nil {
+ return err
+ }
+ if err := write(dst, dir, fmt.Sprintf("match%02d.txt", i), match); err != nil {
+ return err
+ }
+
+ if err := write(src, dir, fmt.Sprintf("differ%02d.txt", i), fmt.Sprintf("src content %02d\n", i)); err != nil {
+ return err
+ }
+ if err := write(dst, dir, fmt.Sprintf("differ%02d.txt", i), fmt.Sprintf("dst content %02d differs\n", i)); err != nil {
+ return err
+ }
+
+ if err := write(src, dir, fmt.Sprintf("srconly%02d.txt", i), fmt.Sprintf("only in src %02d\n", i)); err != nil {
+ return err
+ }
+ if err := write(dst, dir, fmt.Sprintf("dstonly%02d.txt", i), fmt.Sprintf("only in dst %02d\n", i)); err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
diff --git a/fs/logger/testdata/script/TestBeforeVsAfter.txtar b/fs/logger/testdata/script/TestBeforeVsAfter.txtar
index 0ccadc973..3e3b09681 100644
--- a/fs/logger/testdata/script/TestBeforeVsAfter.txtar
+++ b/fs/logger/testdata/script/TestBeforeVsAfter.txtar
@@ -1,11 +1,5 @@
# tests whether an md5sum file generated post-sync matches our pre-sync prediction
-
-# Filling src and dst with two different versions of rclone source!:
-exec rclone copyurl https://github.com/rclone/rclone/archive/refs/tags/v1.49.1.zip $SRC/src.zip
-exec rclone copyurl https://github.com/rclone/rclone/archive/refs/tags/v1.54.1.zip $DST/dst.zip
-exec unzip $SRC/src.zip -d $SRC
-exec unzip $DST/dst.zip -d $DST
-
+# src and dst are filled by the test Setup with two overlapping trees of files.
# generating sumfiles:
exec rclone md5sum $SRC --output-file $WORK/src-before.txt
diff --git a/fs/logger/testdata/script/TestRepoCompare.txtar b/fs/logger/testdata/script/TestRepoCompare.txtar
index 4bbe9220a..80dc9272c 100644
--- a/fs/logger/testdata/script/TestRepoCompare.txtar
+++ b/fs/logger/testdata/script/TestRepoCompare.txtar
@@ -1,14 +1,7 @@
# tests whether rclone check and rclone sync output exactly the same file lists.
-# uses two different old versions of rclone source code for the src and dst
-# to produce a more realistic test than makefiles
-# (has lots of files with same name but different content)
-
-# Filling src and dst with two different versions of rclone source!:
-exec rclone copyurl https://github.com/rclone/rclone/archive/refs/tags/v1.49.1.zip $SRC/src.zip
-exec rclone copyurl https://github.com/rclone/rclone/archive/refs/tags/v1.54.1.zip $DST/dst.zip
-exec unzip $SRC/src.zip -d $SRC
-exec unzip $DST/dst.zip -d $DST
-
+# src and dst are filled by the test Setup with two overlapping trees of files
+# (lots of files with the same name but different content) to produce a more
+# realistic test than makefiles.
# running rclone check for baseline test:
# error is expected here: