"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
cmd/gitannex: Refactor e2e tests, add layout compat tests
Dan McArdle9a2b85d71c21
diff --git a/cmd/gitannex/e2e_test.go b/cmd/gitannex/e2e_test.goindex 87704df87..c989168a2 100644--- a/cmd/gitannex/e2e_test.go+++ b/cmd/gitannex/e2e_test.go@@ -1,6 +1,7 @@package gitanneximport (+ "bytes""encoding/json""errors""fmt"@@ -50,25 +51,137 @@ func checkRcloneBinaryVersion() error {return nil}-// This end-to-end test runs `git annex testremote` in a temporary git repo.-// This test will be skipped unless the `rclone` binary on PATH reports the-// expected version.-//-// When run on CI, an rclone binary built from HEAD will be on the PATH. When-// running locally, you will likely need to ensure the current binary is on the-// PATH like so:-//-// go build && PATH="$(realpath .):$PATH" go test -v ./cmd/gitannex/...+// countFilesRecursively returns the number of files nested underneath `dir`. It+// counts files only and excludes directories.+func countFilesRecursively(t *testing.T, dir string) int {+ remoteFiles, err := os.ReadDir(dir)+ require.NoError(t, err)++ var count int+ for _, f := range remoteFiles {+ if f.IsDir() {+ subdir := filepath.Join(dir, f.Name())+ count += countFilesRecursively(t, subdir)+ } else {+ count+++ }+ }+ return count+}++func findFileWithContents(t *testing.T, dir string, wantContents []byte) bool {+ remoteFiles, err := os.ReadDir(dir)+ require.NoError(t, err)++ for _, f := range remoteFiles {+ fPath := filepath.Join(dir, f.Name())+ if f.IsDir() {+ if findFileWithContents(t, fPath, wantContents) {+ return true+ }+ } else {+ contents, err := os.ReadFile(fPath)+ require.NoError(t, err)+ if bytes.Equal(contents, wantContents) {+ return true+ }+ }+ }+ return false+}++type e2eTestingContext struct {+ tempDir string+ binDir string+ homeDir string+ configDir string+ rcloneConfigDir string+ ephemeralRepoDir string+}++// makeE2eTestingContext sets up a new e2eTestingContext rooted under+// `t.TempDir()`. It creates the skeleton directory structure shown below in the+// temp directory without creating any files.//-// In the future, this test will probably be extended to test a number of-// parameters like repo layouts, and runtime may suffer from a combinatorial-// explosion.-func TestEndToEnd(t *testing.T) {+// .+// |-- bin+// | `-- git-annex-remote-rclone-builtin -> ${PATH_TO_RCLONE_BINARY}+// |-- ephemeralRepo+// `-- user+// `-- .config+// `-- rclone+// `-- rclone.conf+func makeE2eTestingContext(t *testing.T) e2eTestingContext {+ tempDir := t.TempDir()++ binDir := filepath.Join(tempDir, "bin")+ homeDir := filepath.Join(tempDir, "user")+ configDir := filepath.Join(homeDir, ".config")+ rcloneConfigDir := filepath.Join(configDir, "rclone")+ ephemeralRepoDir := filepath.Join(tempDir, "ephemeralRepo")++ for _, dir := range []string{binDir, homeDir, configDir, rcloneConfigDir, ephemeralRepoDir} {+ require.NoError(t, os.Mkdir(dir, 0700))+ }++ return e2eTestingContext{tempDir, binDir, homeDir, configDir, rcloneConfigDir, ephemeralRepoDir}+}++// Install the symlink that enables git-annex to invoke "rclone gitannex"+// without explicitly specifying the subcommand.+func (e *e2eTestingContext) installRcloneGitannexSymlink(t *testing.T) {+ rcloneBinaryPath, err := exec.LookPath("rclone")+ require.NoError(t, err)+ require.NoError(t, os.Symlink(+ rcloneBinaryPath,+ filepath.Join(e.binDir, "git-annex-remote-rclone-builtin")))+}++// Install a rclone.conf file in an appropriate location in the fake home+// directory. The config defines an rclone remote named "MyRcloneRemote" using+// the local backend.+func (e *e2eTestingContext) installRcloneConfig(t *testing.T) {+ // Install the rclone.conf file that defines the remote.+ rcloneConfigPath := filepath.Join(e.rcloneConfigDir, "rclone.conf")+ rcloneConfigContents := "[MyRcloneRemote]\ntype = local"+ require.NoError(t, os.WriteFile(rcloneConfigPath, []byte(rcloneConfigContents), 0600))+}++// runInRepo runs the given command from within the ephemeral repo directory. To+// prevent accidental changes in the real home directory, it sets the HOME+// variable to a subdirectory of the temp directory. It also ensures that the+// git-annex-remote-rclone-builtin symlink will be found by extending the PATH.+func (e *e2eTestingContext) runInRepo(t *testing.T, command string, args ...string) {+ fmt.Printf("+ %s %v\n", command, args)+ cmd := exec.Command(command, args...)+ cmd.Dir = e.ephemeralRepoDir+ cmd.Env = []string{+ "HOME=" + e.homeDir,+ "PATH=" + os.Getenv("PATH") + ":" + e.binDir,+ }+ cmd.Stdout = os.Stdout+ cmd.Stderr = os.Stderr+ require.NoError(t, cmd.Run())+}++// createGitRepo creates an empty git repository in the ephemeral repo+// directory. It makes "global" config changes that are ultimately scoped to the+// calling test thanks to runInRepo() overriding the HOME environment variable.+func (e *e2eTestingContext) createGitRepo(t *testing.T) {+ e.runInRepo(t, "git", "annex", "version")+ e.runInRepo(t, "git", "config", "--global", "user.name", "User Name")+ e.runInRepo(t, "git", "config", "--global", "user.email", "[email protected]")+ e.runInRepo(t, "git", "config", "--global", "init.defaultBranch", "main")+ e.runInRepo(t, "git", "init")+ e.runInRepo(t, "git", "annex", "init")+}++func skipE2eTestIfNecessary(t *testing.T) {if testing.Short() {t.Skip("Skipping due to short mode.")}- // TODO: Support this test on Windows. Need to evaluate the semantics of the+ // TODO: Support e2e tests on Windows. Need to evaluate the semantics of the// HOME and PATH environment variables.switch runtime.GOOS {case "darwin",@@ -89,6 +202,23 @@ func TestEndToEnd(t *testing.T) {if _, err := exec.LookPath("git-annex"); err != nil {t.Skipf("Skipping because git-annex was not found: %s", err)}+}++// This end-to-end test runs `git annex testremote` in a temporary git repo.+// This test will be skipped unless the `rclone` binary on PATH reports the+// expected version.+//+// When run on CI, an rclone binary built from HEAD will be on the PATH. When+// running locally, you will likely need to ensure the current binary is on the+// PATH like so:+//+// go build && PATH="$(realpath .):$PATH" go test -v ./cmd/gitannex/...+//+// In the future, this test will probably be extended to test a number of+// parameters like repo layouts, and runtime may suffer from a combinatorial+// explosion.+func TestEndToEnd(t *testing.T) {+ skipE2eTestIfNecessary(t)// Create a temp directory and chdir there, just in case.originalWd, err := os.Getwd()@@ -97,63 +227,114 @@ func TestEndToEnd(t *testing.T) {require.NoError(t, os.Chdir(tempDir))defer func() { require.NoError(t, os.Chdir(originalWd)) }()- // Flesh out subdirectories of the temp directory:- //- // .- // |-- bin- // | `-- git-annex-remote-rclone-builtin -> ${PATH_TO_RCLONE_BINARY}- // |-- ephemeralRepo- // `-- user- // `-- .config- // `-- rclone- // `-- rclone.conf+ testingContext := makeE2eTestingContext(t)+ testingContext.installRcloneGitannexSymlink(t)+ testingContext.installRcloneConfig(t)+ testingContext.createGitRepo(t)- binDir := filepath.Join(tempDir, "bin")- homeDir := filepath.Join(tempDir, "user")- configDir := filepath.Join(homeDir, ".config")- rcloneConfigDir := filepath.Join(configDir, "rclone")- ephemeralRepoDir := filepath.Join(tempDir, "ephemeralRepo")- for _, dir := range []string{binDir, homeDir, configDir, rcloneConfigDir, ephemeralRepoDir} {- require.NoError(t, os.Mkdir(dir, 0700))+ testingContext.runInRepo(t, "git", "annex", "initremote", "MyTestRemote",+ "type=external", "externaltype=rclone-builtin", "encryption=none",+ "rcloneremotename=MyRcloneRemote", "rcloneprefix="+testingContext.ephemeralRepoDir)++ testingContext.runInRepo(t, "git", "annex", "testremote", "MyTestRemote")+}++// For each layout mode, ensure that we're compatible with data written by+// git-annex-remote-rclone.+func TestEndToEndRepoLayoutCompat(t *testing.T) {+ skipE2eTestIfNecessary(t)++ if _, err := exec.LookPath("git-annex-remote-rclone"); err != nil {+ t.Skipf("Skipping because git-annex-remote-rclone was not found: %s", err)}- // Install the symlink that enables git-annex to invoke "rclone gitannex"- // without explicitly specifying the subcommand.- rcloneBinaryPath, err := exec.LookPath("rclone")- require.NoError(t, err)- require.NoError(t, os.Symlink(- rcloneBinaryPath,- filepath.Join(binDir, "git-annex-remote-rclone-builtin")))+ for _, mode := range allLayoutModes() {+ mode := mode+ t.Run(string(mode), func(t *testing.T) {+ t.Parallel()- // Install the rclone.conf file that defines the remote.- rcloneConfigPath := filepath.Join(rcloneConfigDir, "rclone.conf")- rcloneConfigContents := "[MyRcloneRemote]\ntype = local"- require.NoError(t, os.WriteFile(rcloneConfigPath, []byte(rcloneConfigContents), 0600))+ // Create a temp directory and chdir there, just in case.+ originalWd, err := os.Getwd()+ require.NoError(t, err)+ tempDir := t.TempDir()+ require.NoError(t, os.Chdir(tempDir))+ defer func() { require.NoError(t, os.Chdir(originalWd)) }()- // NOTE: These commands must be run with HOME pointing at an ephemeral- // directory, rather than the real home directory.- cmds := [][]string{- {"git", "annex", "version"},- {"git", "config", "--global", "user.name", "User Name"},- {"git", "config", "--global", "user.email", "[email protected]"},- {"git", "init"},- {"git", "annex", "init"},- {"git", "annex", "initremote", "MyTestRemote",- "type=external", "externaltype=rclone-builtin", "encryption=none",- "rcloneremotename=MyRcloneRemote", "rcloneprefix=" + ephemeralRepoDir},- {"git", "annex", "testremote", "MyTestRemote"},- }-- for _, args := range cmds {- fmt.Printf("+ %v\n", args)- cmd := exec.Command(args[0], args[1:]...)- cmd.Dir = ephemeralRepoDir- cmd.Env = []string{- "HOME=" + homeDir,- "PATH=" + os.Getenv("PATH") + ":" + binDir,- }- cmd.Stdout = os.Stdout- cmd.Stderr = os.Stderr- require.NoError(t, cmd.Run())+ tc := makeE2eTestingContext(t)+ tc.installRcloneGitannexSymlink(t)+ tc.installRcloneConfig(t)+ tc.createGitRepo(t)++ remoteStorage := filepath.Join(tc.tempDir, "remotePrefix")+ require.NoError(t, os.Mkdir(remoteStorage, 0777))++ tc.runInRepo(t,+ "git", "annex", "initremote", "Control",+ "type=external", "externaltype=rclone", "encryption=none",+ "target=MyRcloneRemote",+ "rclone_layout="+string(mode),+ "prefix="+remoteStorage)++ tc.runInRepo(t,+ "git", "annex", "initremote", "Experiment",+ "type=external", "externaltype=rclone-builtin", "encryption=none",+ "rcloneremotename=MyRcloneRemote",+ "rclonelayout="+string(mode),+ "rcloneprefix="+remoteStorage)++ fooFileContents := []byte{1, 2, 3, 4}+ fooFilePath := filepath.Join(tc.ephemeralRepoDir, "foo")+ require.NoError(t, os.WriteFile(fooFilePath, fooFileContents, 0700))+ tc.runInRepo(t, "git", "annex", "add", "foo")+ tc.runInRepo(t, "git", "commit", "-m", "Add foo file")+ // Git-annex objects are not writable, which prevents `testing` from+ // cleaning up the temp directory. We can work around this by+ // explicitly dropping any files we add to the annex.+ t.Cleanup(func() { tc.runInRepo(t, "git", "annex", "drop", "--force", "foo") })++ require.Equal(t, 0, countFilesRecursively(t, remoteStorage))+ require.False(t, findFileWithContents(t, remoteStorage, fooFileContents))++ // Copy the file to Control and verify it's present on Experiment.++ tc.runInRepo(t, "git", "annex", "copy", "--to=Control", "foo")+ require.Equal(t, 1, countFilesRecursively(t, remoteStorage))+ require.True(t, findFileWithContents(t, remoteStorage, fooFileContents))++ tc.runInRepo(t, "git", "annex", "fsck", "--from=Experiment", "foo")+ require.Equal(t, 1, countFilesRecursively(t, remoteStorage))+ require.True(t, findFileWithContents(t, remoteStorage, fooFileContents))++ // Drop the file locally and verify we can copy it back from Experiment.++ tc.runInRepo(t, "git", "annex", "drop", "--force", "foo")+ require.Equal(t, 1, countFilesRecursively(t, remoteStorage))+ require.True(t, findFileWithContents(t, remoteStorage, fooFileContents))++ tc.runInRepo(t, "git", "annex", "copy", "--from=Experiment", "foo")+ require.Equal(t, 1, countFilesRecursively(t, remoteStorage))+ require.True(t, findFileWithContents(t, remoteStorage, fooFileContents))++ // Drop the file from Experiment, copy it back to Experiment, and+ // verify it's still present on Control.++ tc.runInRepo(t, "git", "annex", "drop", "--from=Experiment", "--force", "foo")+ require.Equal(t, 0, countFilesRecursively(t, remoteStorage))+ require.False(t, findFileWithContents(t, remoteStorage, fooFileContents))++ tc.runInRepo(t, "git", "annex", "copy", "--to=Experiment", "foo")+ require.Equal(t, 1, countFilesRecursively(t, remoteStorage))+ require.True(t, findFileWithContents(t, remoteStorage, fooFileContents))++ tc.runInRepo(t, "git", "annex", "fsck", "--from=Control", "foo")+ require.Equal(t, 1, countFilesRecursively(t, remoteStorage))+ require.True(t, findFileWithContents(t, remoteStorage, fooFileContents))++ // Drop the file from Control.++ tc.runInRepo(t, "git", "annex", "drop", "--from=Control", "--force", "foo")+ require.Equal(t, 0, countFilesRecursively(t, remoteStorage))+ require.False(t, findFileWithContents(t, remoteStorage, fooFileContents))+ })}}