Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

cmd/gitannex: Refactor e2e tests, add layout compat tests

Dan McArdle9a2b85d71c21
diff --git a/cmd/gitannex/e2e_test.go b/cmd/gitannex/e2e_test.go
index 87704df87..c989168a2 100644
--- a/cmd/gitannex/e2e_test.go
+++ b/cmd/gitannex/e2e_test.go
@@ -1,6 +1,7 @@
package gitannex
import (
+ "bytes"
"encoding/json"
"errors"
"fmt"
@@ -50,25 +51,137 @@ func checkRcloneBinaryVersion() error {
return nil
}
-// This end-to-end test runs `git annex testremote` in a temporary git repo.
-// This test will be skipped unless the `rclone` binary on PATH reports the
-// expected version.
-//
-// When run on CI, an rclone binary built from HEAD will be on the PATH. When
-// running locally, you will likely need to ensure the current binary is on the
-// PATH like so:
-//
-// go build && PATH="$(realpath .):$PATH" go test -v ./cmd/gitannex/...
+// countFilesRecursively returns the number of files nested underneath `dir`. It
+// counts files only and excludes directories.
+func countFilesRecursively(t *testing.T, dir string) int {
+ remoteFiles, err := os.ReadDir(dir)
+ require.NoError(t, err)
+
+ var count int
+ for _, f := range remoteFiles {
+ if f.IsDir() {
+ subdir := filepath.Join(dir, f.Name())
+ count += countFilesRecursively(t, subdir)
+ } else {
+ count++
+ }
+ }
+ return count
+}
+
+func findFileWithContents(t *testing.T, dir string, wantContents []byte) bool {
+ remoteFiles, err := os.ReadDir(dir)
+ require.NoError(t, err)
+
+ for _, f := range remoteFiles {
+ fPath := filepath.Join(dir, f.Name())
+ if f.IsDir() {
+ if findFileWithContents(t, fPath, wantContents) {
+ return true
+ }
+ } else {
+ contents, err := os.ReadFile(fPath)
+ require.NoError(t, err)
+ if bytes.Equal(contents, wantContents) {
+ return true
+ }
+ }
+ }
+ return false
+}
+
+type e2eTestingContext struct {
+ tempDir string
+ binDir string
+ homeDir string
+ configDir string
+ rcloneConfigDir string
+ ephemeralRepoDir string
+}
+
+// makeE2eTestingContext sets up a new e2eTestingContext rooted under
+// `t.TempDir()`. It creates the skeleton directory structure shown below in the
+// temp directory without creating any files.
//
-// In the future, this test will probably be extended to test a number of
-// parameters like repo layouts, and runtime may suffer from a combinatorial
-// explosion.
-func TestEndToEnd(t *testing.T) {
+// .
+// |-- bin
+// | `-- git-annex-remote-rclone-builtin -> ${PATH_TO_RCLONE_BINARY}
+// |-- ephemeralRepo
+// `-- user
+// `-- .config
+// `-- rclone
+// `-- rclone.conf
+func makeE2eTestingContext(t *testing.T) e2eTestingContext {
+ tempDir := t.TempDir()
+
+ binDir := filepath.Join(tempDir, "bin")
+ homeDir := filepath.Join(tempDir, "user")
+ configDir := filepath.Join(homeDir, ".config")
+ rcloneConfigDir := filepath.Join(configDir, "rclone")
+ ephemeralRepoDir := filepath.Join(tempDir, "ephemeralRepo")
+
+ for _, dir := range []string{binDir, homeDir, configDir, rcloneConfigDir, ephemeralRepoDir} {
+ require.NoError(t, os.Mkdir(dir, 0700))
+ }
+
+ return e2eTestingContext{tempDir, binDir, homeDir, configDir, rcloneConfigDir, ephemeralRepoDir}
+}
+
+// Install the symlink that enables git-annex to invoke "rclone gitannex"
+// without explicitly specifying the subcommand.
+func (e *e2eTestingContext) installRcloneGitannexSymlink(t *testing.T) {
+ rcloneBinaryPath, err := exec.LookPath("rclone")
+ require.NoError(t, err)
+ require.NoError(t, os.Symlink(
+ rcloneBinaryPath,
+ filepath.Join(e.binDir, "git-annex-remote-rclone-builtin")))
+}
+
+// Install a rclone.conf file in an appropriate location in the fake home
+// directory. The config defines an rclone remote named "MyRcloneRemote" using
+// the local backend.
+func (e *e2eTestingContext) installRcloneConfig(t *testing.T) {
+ // Install the rclone.conf file that defines the remote.
+ rcloneConfigPath := filepath.Join(e.rcloneConfigDir, "rclone.conf")
+ rcloneConfigContents := "[MyRcloneRemote]\ntype = local"
+ require.NoError(t, os.WriteFile(rcloneConfigPath, []byte(rcloneConfigContents), 0600))
+}
+
+// runInRepo runs the given command from within the ephemeral repo directory. To
+// prevent accidental changes in the real home directory, it sets the HOME
+// variable to a subdirectory of the temp directory. It also ensures that the
+// git-annex-remote-rclone-builtin symlink will be found by extending the PATH.
+func (e *e2eTestingContext) runInRepo(t *testing.T, command string, args ...string) {
+ fmt.Printf("+ %s %v\n", command, args)
+ cmd := exec.Command(command, args...)
+ cmd.Dir = e.ephemeralRepoDir
+ cmd.Env = []string{
+ "HOME=" + e.homeDir,
+ "PATH=" + os.Getenv("PATH") + ":" + e.binDir,
+ }
+ cmd.Stdout = os.Stdout
+ cmd.Stderr = os.Stderr
+ require.NoError(t, cmd.Run())
+}
+
+// createGitRepo creates an empty git repository in the ephemeral repo
+// directory. It makes "global" config changes that are ultimately scoped to the
+// calling test thanks to runInRepo() overriding the HOME environment variable.
+func (e *e2eTestingContext) createGitRepo(t *testing.T) {
+ e.runInRepo(t, "git", "annex", "version")
+ e.runInRepo(t, "git", "config", "--global", "user.name", "User Name")
+ e.runInRepo(t, "git", "config", "--global", "user.email", "[email protected]")
+ e.runInRepo(t, "git", "config", "--global", "init.defaultBranch", "main")
+ e.runInRepo(t, "git", "init")
+ e.runInRepo(t, "git", "annex", "init")
+}
+
+func skipE2eTestIfNecessary(t *testing.T) {
if testing.Short() {
t.Skip("Skipping due to short mode.")
}
- // TODO: Support this test on Windows. Need to evaluate the semantics of the
+ // TODO: Support e2e tests on Windows. Need to evaluate the semantics of the
// HOME and PATH environment variables.
switch runtime.GOOS {
case "darwin",
@@ -89,6 +202,23 @@ func TestEndToEnd(t *testing.T) {
if _, err := exec.LookPath("git-annex"); err != nil {
t.Skipf("Skipping because git-annex was not found: %s", err)
}
+}
+
+// This end-to-end test runs `git annex testremote` in a temporary git repo.
+// This test will be skipped unless the `rclone` binary on PATH reports the
+// expected version.
+//
+// When run on CI, an rclone binary built from HEAD will be on the PATH. When
+// running locally, you will likely need to ensure the current binary is on the
+// PATH like so:
+//
+// go build && PATH="$(realpath .):$PATH" go test -v ./cmd/gitannex/...
+//
+// In the future, this test will probably be extended to test a number of
+// parameters like repo layouts, and runtime may suffer from a combinatorial
+// explosion.
+func TestEndToEnd(t *testing.T) {
+ skipE2eTestIfNecessary(t)
// Create a temp directory and chdir there, just in case.
originalWd, err := os.Getwd()
@@ -97,63 +227,114 @@ func TestEndToEnd(t *testing.T) {
require.NoError(t, os.Chdir(tempDir))
defer func() { require.NoError(t, os.Chdir(originalWd)) }()
- // Flesh out subdirectories of the temp directory:
- //
- // .
- // |-- bin
- // | `-- git-annex-remote-rclone-builtin -> ${PATH_TO_RCLONE_BINARY}
- // |-- ephemeralRepo
- // `-- user
- // `-- .config
- // `-- rclone
- // `-- rclone.conf
+ testingContext := makeE2eTestingContext(t)
+ testingContext.installRcloneGitannexSymlink(t)
+ testingContext.installRcloneConfig(t)
+ testingContext.createGitRepo(t)
- binDir := filepath.Join(tempDir, "bin")
- homeDir := filepath.Join(tempDir, "user")
- configDir := filepath.Join(homeDir, ".config")
- rcloneConfigDir := filepath.Join(configDir, "rclone")
- ephemeralRepoDir := filepath.Join(tempDir, "ephemeralRepo")
- for _, dir := range []string{binDir, homeDir, configDir, rcloneConfigDir, ephemeralRepoDir} {
- require.NoError(t, os.Mkdir(dir, 0700))
+ testingContext.runInRepo(t, "git", "annex", "initremote", "MyTestRemote",
+ "type=external", "externaltype=rclone-builtin", "encryption=none",
+ "rcloneremotename=MyRcloneRemote", "rcloneprefix="+testingContext.ephemeralRepoDir)
+
+ testingContext.runInRepo(t, "git", "annex", "testremote", "MyTestRemote")
+}
+
+// For each layout mode, ensure that we're compatible with data written by
+// git-annex-remote-rclone.
+func TestEndToEndRepoLayoutCompat(t *testing.T) {
+ skipE2eTestIfNecessary(t)
+
+ if _, err := exec.LookPath("git-annex-remote-rclone"); err != nil {
+ t.Skipf("Skipping because git-annex-remote-rclone was not found: %s", err)
}
- // Install the symlink that enables git-annex to invoke "rclone gitannex"
- // without explicitly specifying the subcommand.
- rcloneBinaryPath, err := exec.LookPath("rclone")
- require.NoError(t, err)
- require.NoError(t, os.Symlink(
- rcloneBinaryPath,
- filepath.Join(binDir, "git-annex-remote-rclone-builtin")))
+ for _, mode := range allLayoutModes() {
+ mode := mode
+ t.Run(string(mode), func(t *testing.T) {
+ t.Parallel()
- // Install the rclone.conf file that defines the remote.
- rcloneConfigPath := filepath.Join(rcloneConfigDir, "rclone.conf")
- rcloneConfigContents := "[MyRcloneRemote]\ntype = local"
- require.NoError(t, os.WriteFile(rcloneConfigPath, []byte(rcloneConfigContents), 0600))
+ // Create a temp directory and chdir there, just in case.
+ originalWd, err := os.Getwd()
+ require.NoError(t, err)
+ tempDir := t.TempDir()
+ require.NoError(t, os.Chdir(tempDir))
+ defer func() { require.NoError(t, os.Chdir(originalWd)) }()
- // NOTE: These commands must be run with HOME pointing at an ephemeral
- // directory, rather than the real home directory.
- cmds := [][]string{
- {"git", "annex", "version"},
- {"git", "config", "--global", "user.name", "User Name"},
- {"git", "config", "--global", "user.email", "[email protected]"},
- {"git", "init"},
- {"git", "annex", "init"},
- {"git", "annex", "initremote", "MyTestRemote",
- "type=external", "externaltype=rclone-builtin", "encryption=none",
- "rcloneremotename=MyRcloneRemote", "rcloneprefix=" + ephemeralRepoDir},
- {"git", "annex", "testremote", "MyTestRemote"},
- }
-
- for _, args := range cmds {
- fmt.Printf("+ %v\n", args)
- cmd := exec.Command(args[0], args[1:]...)
- cmd.Dir = ephemeralRepoDir
- cmd.Env = []string{
- "HOME=" + homeDir,
- "PATH=" + os.Getenv("PATH") + ":" + binDir,
- }
- cmd.Stdout = os.Stdout
- cmd.Stderr = os.Stderr
- require.NoError(t, cmd.Run())
+ tc := makeE2eTestingContext(t)
+ tc.installRcloneGitannexSymlink(t)
+ tc.installRcloneConfig(t)
+ tc.createGitRepo(t)
+
+ remoteStorage := filepath.Join(tc.tempDir, "remotePrefix")
+ require.NoError(t, os.Mkdir(remoteStorage, 0777))
+
+ tc.runInRepo(t,
+ "git", "annex", "initremote", "Control",
+ "type=external", "externaltype=rclone", "encryption=none",
+ "target=MyRcloneRemote",
+ "rclone_layout="+string(mode),
+ "prefix="+remoteStorage)
+
+ tc.runInRepo(t,
+ "git", "annex", "initremote", "Experiment",
+ "type=external", "externaltype=rclone-builtin", "encryption=none",
+ "rcloneremotename=MyRcloneRemote",
+ "rclonelayout="+string(mode),
+ "rcloneprefix="+remoteStorage)
+
+ fooFileContents := []byte{1, 2, 3, 4}
+ fooFilePath := filepath.Join(tc.ephemeralRepoDir, "foo")
+ require.NoError(t, os.WriteFile(fooFilePath, fooFileContents, 0700))
+ tc.runInRepo(t, "git", "annex", "add", "foo")
+ tc.runInRepo(t, "git", "commit", "-m", "Add foo file")
+ // Git-annex objects are not writable, which prevents `testing` from
+ // cleaning up the temp directory. We can work around this by
+ // explicitly dropping any files we add to the annex.
+ t.Cleanup(func() { tc.runInRepo(t, "git", "annex", "drop", "--force", "foo") })
+
+ require.Equal(t, 0, countFilesRecursively(t, remoteStorage))
+ require.False(t, findFileWithContents(t, remoteStorage, fooFileContents))
+
+ // Copy the file to Control and verify it's present on Experiment.
+
+ tc.runInRepo(t, "git", "annex", "copy", "--to=Control", "foo")
+ require.Equal(t, 1, countFilesRecursively(t, remoteStorage))
+ require.True(t, findFileWithContents(t, remoteStorage, fooFileContents))
+
+ tc.runInRepo(t, "git", "annex", "fsck", "--from=Experiment", "foo")
+ require.Equal(t, 1, countFilesRecursively(t, remoteStorage))
+ require.True(t, findFileWithContents(t, remoteStorage, fooFileContents))
+
+ // Drop the file locally and verify we can copy it back from Experiment.
+
+ tc.runInRepo(t, "git", "annex", "drop", "--force", "foo")
+ require.Equal(t, 1, countFilesRecursively(t, remoteStorage))
+ require.True(t, findFileWithContents(t, remoteStorage, fooFileContents))
+
+ tc.runInRepo(t, "git", "annex", "copy", "--from=Experiment", "foo")
+ require.Equal(t, 1, countFilesRecursively(t, remoteStorage))
+ require.True(t, findFileWithContents(t, remoteStorage, fooFileContents))
+
+ // Drop the file from Experiment, copy it back to Experiment, and
+ // verify it's still present on Control.
+
+ tc.runInRepo(t, "git", "annex", "drop", "--from=Experiment", "--force", "foo")
+ require.Equal(t, 0, countFilesRecursively(t, remoteStorage))
+ require.False(t, findFileWithContents(t, remoteStorage, fooFileContents))
+
+ tc.runInRepo(t, "git", "annex", "copy", "--to=Experiment", "foo")
+ require.Equal(t, 1, countFilesRecursively(t, remoteStorage))
+ require.True(t, findFileWithContents(t, remoteStorage, fooFileContents))
+
+ tc.runInRepo(t, "git", "annex", "fsck", "--from=Control", "foo")
+ require.Equal(t, 1, countFilesRecursively(t, remoteStorage))
+ require.True(t, findFileWithContents(t, remoteStorage, fooFileContents))
+
+ // Drop the file from Control.
+
+ tc.runInRepo(t, "git", "annex", "drop", "--from=Control", "--force", "foo")
+ require.Equal(t, 0, countFilesRecursively(t, remoteStorage))
+ require.False(t, findFileWithContents(t, remoteStorage, fooFileContents))
+ })
}
}