"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
cmd/gitannex: Port unit tests to fstest
Dan McArdlebe33e281b3e8
diff --git a/cmd/gitannex/gitannex.go b/cmd/gitannex/gitannex.goindex 287adc7ae..d71393b05 100644--- a/cmd/gitannex/gitannex.go+++ b/cmd/gitannex/gitannex.go@@ -7,7 +7,7 @@// (Tracked in [issue #7625].)//// 1. ✅ Minimal support for the [external special remote protocol]. Tested on-// "local" and "drive" backends.+// "local", "drive", and "dropbox" backends.// 2. Add support for the ASYNC protocol extension. This may improve performance.// 3. Support the [simple export interface]. This will enable `git-annex// export` functionality.@@ -33,6 +33,7 @@ import ("github.com/rclone/rclone/cmd""github.com/rclone/rclone/fs""github.com/rclone/rclone/fs/cache"+ "github.com/rclone/rclone/fs/config""github.com/rclone/rclone/fs/operations""github.com/spf13/cobra")@@ -267,22 +268,34 @@ func (s *server) handleInitRemote() error {return fmt.Errorf("failed to get configs: %w", err)}- remoteRootFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:", s.configRcloneRemoteName))- if err != nil {- s.sendMsg("INITREMOTE-FAILURE failed to open root directory of rclone remote")- return fmt.Errorf("failed to open root directory of rclone remote: %w", err)- }+ // Explicitly check that a remote with the given name exists. If we just+ // relied on `cache.Get()` to return `fs.ErrorNotFoundInConfigFile`, this+ // function would incorrectly succeed when the given remote name is actually+ // a file path.+ //+ // The :local: backend does not correspond to a remote named by the rclone+ // config, but is permitted to enable testing. Technically, a user might hit+ // this code path, but it would be a strange choice because git-annex+ // natively supports a "directory" special remote.- if !remoteRootFs.Features().CanHaveEmptyDirectories {- s.sendMsg("INITREMOTE-FAILURE this rclone remote does not support empty directories")- return fmt.Errorf("rclone remote does not support empty directories")- }+ trimmedName := strings.TrimSuffix(s.configRcloneRemoteName, ":")- if err := operations.Mkdir(context.TODO(), remoteRootFs, s.configPrefix); err != nil {- s.sendMsg("INITREMOTE-FAILURE failed to mkdir")- return fmt.Errorf("failed to mkdir: %w", err)+ if s.configRcloneRemoteName != ":local" {+ var remoteExists bool+ for _, remoteName := range config.FileSections() {+ if remoteName == trimmedName {+ remoteExists = true+ break+ }+ }+ if !remoteExists {+ s.sendMsg("INITREMOTE-FAILURE remote does not exist: " + s.configRcloneRemoteName)+ return fmt.Errorf("remote does not exist: %s", s.configRcloneRemoteName)+ }}+ s.configRcloneRemoteName = trimmedName + ":"+s.sendMsg("INITREMOTE-SUCCESS")return nil}diff --git a/cmd/gitannex/gitannex_test.go b/cmd/gitannex/gitannex_test.goindex d53a7f921..6d60e1db1 100644--- a/cmd/gitannex/gitannex_test.go+++ b/cmd/gitannex/gitannex_test.go@@ -2,25 +2,24 @@ package gitanneximport ("bufio"- "crypto/sha256"+ "context""fmt""io""os""path/filepath""regexp"+ "runtime""strings""sync""testing"+ "time"- // Without this import, the local filesystem backend would be unavailable.- // It looks unused, but the act of importing it runs its `init()` function.- _ "github.com/rclone/rclone/backend/local"+ // Without this import, the various backends would be unavailable. It looks+ // unused, but the act of importing runs the package's `init()` function.+ _ "github.com/rclone/rclone/backend/all"- "github.com/rclone/rclone/fs"- "github.com/rclone/rclone/fs/cache"- "github.com/rclone/rclone/fs/config"- "github.com/rclone/rclone/fs/config/configfile"- "github.com/rclone/rclone/fstest/mockfs"+ "github.com/rclone/rclone/fs/fspath"+ "github.com/rclone/rclone/fstest""github.com/stretchr/testify/assert""github.com/stretchr/testify/require"@@ -256,9 +255,9 @@ type testState struct {mockStdinW *io.PipeWritermockStdoutReader *bufio.Reader- localFsDir string- configPath string- remoteName string+ fstestRun *fstest.Run+ remoteName string+ remotePrefix string}func makeTestState(t *testing.T) testState {@@ -276,6 +275,10 @@ func makeTestState(t *testing.T) testState {}}+func (h *testState) requireRemoteIsEmpty() {+ h.fstestRun.CheckRemoteItems(h.t)+}+func (h *testState) requireReadLineExact(line string) {receivedLine, err := h.mockStdoutReader.ReadString('\n')require.NoError(h.t, err)@@ -296,21 +299,106 @@ func (h *testState) requireWriteLine(line string) {// Preconfigure the handle. This enables the calling test to skip the PREPARE// handshake.func (h *testState) preconfigureServer() {- h.server.configPrefix = h.localFsDirh.server.configRcloneRemoteName = h.remoteName+ h.server.configPrefix = h.remotePrefixh.server.configRcloneLayout = string(layoutModeNodir)h.server.configsDone = true}-// getUniqueRemoteName returns a valid remote name derived from the given test's-// name. This is necessary because when a test registers a second remote with-// the same name, the original remote appears to take precedence. This function-// is injective, so each test gets a unique remote name. Returned strings-// contain no spaces.-func getUniqueRemoteName(t *testing.T) string {- // Using sha256 as a hack to ensure injectivity without adding a global- // variable.- return fmt.Sprintf("remote-%x", sha256.Sum256([]byte(t.Name())))+// Drop-in replacement for `filepath.Rel()` that works around a Windows-specific+// quirk when one of the paths begins with `\\?\` or `//?/`. It seems that+// fstest gives us paths with this prefix on Windows, which throws a wrench in+// the gitannex tests that need to construct relative paths from absolute paths.+// For a demonstration, see `TestWindowsFilepathRelQuirk` below.+//+// The `\\?\` prefix tells Windows APIs to pass strings unmodified to the+// filesystem without additional parsing [1]. Our workaround is roughly to add+// the prefix to whichever parameter doesn't have it (when the OS is Windows).+// I'm not sure this generalizes, but it works for the the kinds of inputs we're+// throwing at it.+//+// [1]: https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file?redirectedfrom=MSDN#win32-file-namespaces+func relativeFilepathWorkaround(basepath, targpath string) (string, error) {+ if runtime.GOOS != "windows" {+ return filepath.Rel(basepath, targpath)+ }+ // Canonicalize paths to use backslashes.+ basepath = filepath.Clean(basepath)+ targpath = filepath.Clean(targpath)++ const winFilePrefixDisableStringParsing = `\\?\`+ baseHasPrefix := strings.HasPrefix(basepath, winFilePrefixDisableStringParsing)+ targHasPrefix := strings.HasPrefix(targpath, winFilePrefixDisableStringParsing)++ if baseHasPrefix && !targHasPrefix {+ targpath = winFilePrefixDisableStringParsing + targpath+ }+ if !baseHasPrefix && targHasPrefix {+ basepath = winFilePrefixDisableStringParsing + basepath+ }+ return filepath.Rel(basepath, targpath)+}++func TestWindowsFilepathRelQuirk(t *testing.T) {+ if runtime.GOOS != "windows" {+ t.Skip()+ }++ t.Run("filepathRelQuirk", func(t *testing.T) {+ var err error++ _, err = filepath.Rel(`C:\foo`, `\\?\C:\foo\bar`)+ require.Error(t, err)++ _, err = filepath.Rel(`C:/foo`, `//?/C:/foo/bar`)+ require.Error(t, err)++ _, err = filepath.Rel(`\\?\C:\foo`, `C:\foo\bar`)+ require.Error(t, err)++ _, err = filepath.Rel(`//?/C:/foo`, `C:/foo/bar`)+ require.Error(t, err)++ path, err := filepath.Rel(`\\?\C:\foo`, `\\?\C:\foo\bar`)+ require.NoError(t, err)+ require.Equal(t, path, `bar`)++ path, err = filepath.Rel(`//?/C:/foo`, `//?/C:/foo/bar`)+ require.NoError(t, err)+ require.Equal(t, path, `bar`)+ })++ t.Run("fstestAndTempDirHaveDifferentPrefixes", func(t *testing.T) {+ r := fstest.NewRun(t)+ p := r.Flocal.Root()+ require.True(t, strings.HasPrefix(p, `//?/`))++ tempDir := t.TempDir()+ require.False(t, strings.HasPrefix(tempDir, `//?/`))+ require.False(t, strings.HasPrefix(tempDir, `\\?\`))+ })++ t.Run("workaroundWorks", func(t *testing.T) {+ path, err := relativeFilepathWorkaround(`C:\foo`, `\\?\C:\foo\bar`)+ require.NoError(t, err)+ require.Equal(t, path, "bar")++ path, err = relativeFilepathWorkaround(`C:/foo`, `//?/C:/foo/bar`)+ require.NoError(t, err)+ require.Equal(t, path, "bar")++ path, err = relativeFilepathWorkaround(`\\?\C:\foo`, `C:\foo\bar`)+ require.NoError(t, err)+ require.Equal(t, path, `bar`)++ path, err = relativeFilepathWorkaround(`//?/C:/foo`, `C:/foo/bar`)+ require.NoError(t, err)+ require.Equal(t, path, `bar`)++ path, err = relativeFilepathWorkaround(`\\?\C:\foo`, `\\?\C:\foo\bar`)+ require.NoError(t, err)+ require.Equal(t, path, `bar`)+ })}type testCase struct {@@ -319,8 +407,8 @@ type testCase struct {expectedError string}-// These test cases run against the "local" backend.-var localBackendTestCases = []testCase{+// These test cases run against a backend selected by the `-remote` flag.+var fstestTestCases = []testCase{{label: "HandlesInit",testProtocolFunc: func(t *testing.T, h *testState) {@@ -377,17 +465,85 @@ var localBackendTestCases = []testCase{h.requireReadLineExact("GETCONFIG rcloneremotename")h.requireWriteLine("VALUE " + h.remoteName)h.requireReadLineExact("GETCONFIG rcloneprefix")- h.requireWriteLine("VALUE " + h.localFsDir)+ h.requireWriteLine("VALUE " + h.remotePrefix)h.requireReadLineExact("GETCONFIG rclonelayout")h.requireWriteLine("VALUE foo")h.requireReadLineExact("PREPARE-SUCCESS")require.Equal(t, h.server.configRcloneRemoteName, h.remoteName)- require.Equal(t, h.server.configPrefix, h.localFsDir)+ require.Equal(t, h.server.configPrefix, h.remotePrefix)+ require.True(t, h.server.configsDone)++ require.NoError(t, h.mockStdinW.Close())+ },+ },+ {+ label: "HandlesPrepareWithNonexistentRemote",+ testProtocolFunc: func(t *testing.T, h *testState) {+ h.requireReadLineExact("VERSION 1")+ h.requireWriteLine("EXTENSIONS INFO") // Advertise that we support the INFO extension+ h.requireReadLineExact("EXTENSIONS")++ if !h.server.extensionInfo {+ t.Errorf("expected INFO extension to be enabled")+ return+ }++ h.requireWriteLine("PREPARE")+ h.requireReadLineExact("GETCONFIG rcloneremotename")+ h.requireWriteLine("VALUE thisRemoteDoesNotExist")+ h.requireReadLineExact("GETCONFIG rcloneprefix")+ h.requireWriteLine("VALUE " + h.remotePrefix)+ h.requireReadLineExact("GETCONFIG rclonelayout")+ h.requireWriteLine("VALUE foo")+ h.requireReadLineExact("PREPARE-SUCCESS")++ require.Equal(t, h.server.configRcloneRemoteName, "thisRemoteDoesNotExist")+ require.Equal(t, h.server.configPrefix, h.remotePrefix)+ require.True(t, h.server.configsDone)++ h.requireWriteLine("INITREMOTE")+ h.requireReadLineExact("INITREMOTE-FAILURE remote does not exist: thisRemoteDoesNotExist")++ require.NoError(t, h.mockStdinW.Close())+ },+ expectedError: "remote does not exist: thisRemoteDoesNotExist",+ },+ {+ label: "HandlesPrepareWithPathAsRemote",+ testProtocolFunc: func(t *testing.T, h *testState) {+ h.requireReadLineExact("VERSION 1")+ h.requireWriteLine("EXTENSIONS INFO") // Advertise that we support the INFO extension+ h.requireReadLineExact("EXTENSIONS")++ if !h.server.extensionInfo {+ t.Errorf("expected INFO extension to be enabled")+ return+ }++ h.requireWriteLine("PREPARE")+ h.requireReadLineExact("GETCONFIG rcloneremotename")+ h.requireWriteLine("VALUE " + h.remotePrefix)+ h.requireReadLineExact("GETCONFIG rcloneprefix")+ h.requireWriteLine("VALUE /foo")+ h.requireReadLineExact("GETCONFIG rclonelayout")+ h.requireWriteLine("VALUE foo")+ h.requireReadLineExact("PREPARE-SUCCESS")++ require.Equal(t, h.server.configRcloneRemoteName, h.remotePrefix)+ require.Equal(t, h.server.configPrefix, "/foo")require.True(t, h.server.configsDone)+ h.requireWriteLine("INITREMOTE")++ require.Regexp(t,+ regexp.MustCompile("^INITREMOTE-FAILURE remote does not exist: "),+ h.requireReadLine(),+ )+require.NoError(t, h.mockStdinW.Close())},+ expectedError: "remote does not exist:",},{label: "HandlesPrepareWithSynonyms",@@ -409,13 +565,13 @@ var localBackendTestCases = []testCase{h.requireWriteLine("VALUE " + h.remoteName)h.requireReadLineExact("GETCONFIG rcloneprefix")- h.requireWriteLine("VALUE " + h.localFsDir)+ h.requireWriteLine("VALUE " + h.remotePrefix)h.requireReadLineExact("GETCONFIG rclonelayout")h.requireWriteLine("VALUE foo")h.requireReadLineExact("PREPARE-SUCCESS")require.Equal(t, h.server.configRcloneRemoteName, h.remoteName)- require.Equal(t, h.server.configPrefix, h.localFsDir)+ require.Equal(t, h.server.configPrefix, h.remotePrefix)require.True(t, h.server.configsDone)require.NoError(t, h.mockStdinW.Close())@@ -437,12 +593,12 @@ var localBackendTestCases = []testCase{h.requireReadLineExact("GETCONFIG rcloneremotename")remoteNameWithSpaces := fmt.Sprintf(" %s ", h.remoteName)- localFsDirWithSpaces := fmt.Sprintf(" %s\t", h.localFsDir)+ prefixWithWhitespace := fmt.Sprintf(" %s\t", h.remotePrefix)h.requireWriteLine(fmt.Sprintf("VALUE %s", remoteNameWithSpaces))h.requireReadLineExact("GETCONFIG rcloneprefix")- h.requireWriteLine(fmt.Sprintf("VALUE %s", localFsDirWithSpaces))+ h.requireWriteLine(fmt.Sprintf("VALUE %s", prefixWithWhitespace))h.requireReadLineExact("GETCONFIG rclonelayout")h.requireWriteLine("VALUE")@@ -452,7 +608,7 @@ var localBackendTestCases = []testCase{h.requireReadLineExact("PREPARE-SUCCESS")require.Equal(t, h.server.configRcloneRemoteName, remoteNameWithSpaces)- require.Equal(t, h.server.configPrefix, localFsDirWithSpaces)+ require.Equal(t, h.server.configPrefix, prefixWithWhitespace)require.True(t, h.server.configsDone)require.NoError(t, h.mockStdinW.Close())@@ -639,20 +795,25 @@ var localBackendTestCases = []testCase{h.requireReadLineExact("INITREMOTE-SUCCESS")// Create temp file for transfer with an absolute path.- fileToTransfer := filepath.Join(t.TempDir(), "file.txt")- require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))- require.FileExists(t, fileToTransfer)- require.True(t, filepath.IsAbs(fileToTransfer))+ item := h.fstestRun.WriteFile("file.txt", "HELLO", time.Now())+ absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path)+ require.True(t, filepath.IsAbs(absPath))// Specify an absolute path to transfer.- h.requireWriteLine("TRANSFER STORE KeyAbsolute " + fileToTransfer)+ h.requireWriteLine("TRANSFER STORE KeyAbsolute " + absPath)h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyAbsolute")- require.FileExists(t, filepath.Join(h.localFsDir, "KeyAbsolute"))++ // Check that the file was transferred.+ remoteItem := fstest.NewItem("KeyAbsolute", "HELLO", item.ModTime)+ h.fstestRun.CheckRemoteItems(t, remoteItem)// Transfer the same absolute path a second time, but with a different key.- h.requireWriteLine("TRANSFER STORE KeyAbsolute2 " + fileToTransfer)+ h.requireWriteLine("TRANSFER STORE KeyAbsolute2 " + absPath)h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyAbsolute2")- require.FileExists(t, filepath.Join(h.localFsDir, "KeyAbsolute2"))++ // Check that the same file was transferred to a new name.+ remoteItem2 := fstest.NewItem("KeyAbsolute2", "HELLO", item.ModTime)+ h.fstestRun.CheckRemoteItems(t, remoteItem, remoteItem2)h.requireWriteLine("CHECKPRESENT KeyAbsolute2")h.requireReadLineExact("CHECKPRESENT-SUCCESS KeyAbsolute2")@@ -668,30 +829,36 @@ var localBackendTestCases = []testCase{{label: "TransferStoreRelative",testProtocolFunc: func(t *testing.T, h *testState) {- h.preconfigureServer()-// Save the current working directory so we can restore it when this// test ends.cwd, err := os.Getwd()require.NoError(t, err)- require.NoError(t, os.Chdir(t.TempDir()))+ tempDir := t.TempDir()++ require.NoError(t, os.Chdir(tempDir))t.Cleanup(func() { require.NoError(t, os.Chdir(cwd)) })+ h.preconfigureServer()+h.requireReadLineExact("VERSION 1")h.requireWriteLine("INITREMOTE")h.requireReadLineExact("INITREMOTE-SUCCESS")- // Create temp file for transfer with a relative path.- fileToTransfer := "file.txt"- require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))- require.FileExists(t, fileToTransfer)- require.False(t, filepath.IsAbs(fileToTransfer))+ item := h.fstestRun.WriteFile("file.txt", "HELLO", time.Now())+ absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path)++ relativePath, err := relativeFilepathWorkaround(tempDir, absPath)+ require.NoError(t, err)+ require.False(t, filepath.IsAbs(relativePath))+ require.FileExists(t, relativePath)// Specify a relative path to transfer.- h.requireWriteLine("TRANSFER STORE KeyRelative " + fileToTransfer)+ h.requireWriteLine("TRANSFER STORE KeyRelative " + relativePath)h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyRelative")- require.FileExists(t, filepath.Join(h.localFsDir, "KeyRelative"))++ remoteItem := fstest.NewItem("KeyRelative", "HELLO", item.ModTime)+ h.fstestRun.CheckRemoteItems(t, remoteItem)h.requireWriteLine("CHECKPRESENT KeyRelative")h.requireReadLineExact("CHECKPRESENT-SUCCESS KeyRelative")@@ -710,7 +877,8 @@ var localBackendTestCases = []testCase{cwd, err := os.Getwd()require.NoError(t, err)- require.NoError(t, os.Chdir(t.TempDir()))+ tempDir := t.TempDir()+ require.NoError(t, os.Chdir(tempDir))t.Cleanup(func() { require.NoError(t, os.Chdir(cwd)) })h.preconfigureServer()@@ -720,15 +888,19 @@ var localBackendTestCases = []testCase{h.requireReadLineExact("INITREMOTE-SUCCESS")// Create temp file for transfer.- fileToTransfer := "filename with spaces.txt"- require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))- require.FileExists(t, fileToTransfer)- require.False(t, filepath.IsAbs(fileToTransfer))+ item := h.fstestRun.WriteFile("filename with spaces.txt", "HELLO", time.Now())+ absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path)+ relativePath, err := relativeFilepathWorkaround(tempDir, absPath)+ require.NoError(t, err)+ require.False(t, filepath.IsAbs(relativePath))+ require.FileExists(t, relativePath)// Specify a relative path to transfer.- h.requireWriteLine("TRANSFER STORE KeyRelative " + fileToTransfer)+ h.requireWriteLine("TRANSFER STORE KeyRelative " + relativePath)h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyRelative")- require.FileExists(t, filepath.Join(h.localFsDir, "KeyRelative"))++ remoteItem := fstest.NewItem("KeyRelative", "HELLO", item.ModTime)+ h.fstestRun.CheckRemoteItems(t, remoteItem)h.requireWriteLine("CHECKPRESENT KeyRelative")h.requireReadLineExact("CHECKPRESENT-SUCCESS KeyRelative")@@ -745,8 +917,9 @@ var localBackendTestCases = []testCase{h.preconfigureServer()// Create temp file for transfer.- fileToTransfer := filepath.Join(t.TempDir(), "file.txt")- require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))+ item := h.fstestRun.WriteFile("file.txt", "HELLO", time.Now())+ absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path)+ require.True(t, filepath.IsAbs(absPath))h.requireReadLineExact("VERSION 1")h.requireWriteLine("INITREMOTE")@@ -756,10 +929,11 @@ var localBackendTestCases = []testCase{h.requireReadLineExact("CHECKPRESENT-FAILURE KeyThatDoesNotExist")// Specify an absolute path to transfer.- require.True(t, filepath.IsAbs(fileToTransfer))- h.requireWriteLine("TRANSFER STORE KeyAbsolute " + fileToTransfer)+ h.requireWriteLine("TRANSFER STORE KeyAbsolute " + absPath)h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyAbsolute")- require.FileExists(t, filepath.Join(h.localFsDir, "KeyAbsolute"))++ remoteItem := fstest.NewItem("KeyAbsolute", "HELLO", item.ModTime)+ h.fstestRun.CheckRemoteItems(t, remoteItem)require.NoError(t, h.mockStdinW.Close())},@@ -781,8 +955,9 @@ var localBackendTestCases = []testCase{h.preconfigureServer()// Create temp file for transfer.- fileToTransfer := filepath.Join(t.TempDir(), "file.txt")- require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))+ item := h.fstestRun.WriteFile("file.txt", "HELLO", time.Now())+ absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path)+ require.True(t, filepath.IsAbs(absPath))h.requireReadLineExact("VERSION 1")h.requireWriteLine("INITREMOTE")@@ -791,9 +966,11 @@ var localBackendTestCases = []testCase{h.requireWriteLine("CHECKPRESENT foo")h.requireReadLineExact("CHECKPRESENT-FAILURE foo")- h.requireWriteLine("TRANSFER STORE foo " + fileToTransfer)+ h.requireWriteLine("TRANSFER STORE foo " + absPath)h.requireReadLineExact("TRANSFER-SUCCESS STORE foo")- require.FileExists(t, filepath.Join(h.localFsDir, "foo"))++ remoteItem := fstest.NewItem("foo", "HELLO", item.ModTime)+ h.fstestRun.CheckRemoteItems(t, remoteItem)h.requireWriteLine("CHECKPRESENT foo")h.requireReadLineExact("CHECKPRESENT-SUCCESS foo")@@ -807,8 +984,9 @@ var localBackendTestCases = []testCase{h.preconfigureServer()// Create temp file for transfer.- fileToTransfer := filepath.Join(t.TempDir(), "file.txt")- require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))+ item := h.fstestRun.WriteFile("file.txt", "HELLO", time.Now())+ absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path)+ require.True(t, filepath.IsAbs(absPath))h.requireReadLineExact("VERSION 1")h.requireWriteLine("INITREMOTE")@@ -817,10 +995,11 @@ var localBackendTestCases = []testCase{realisticKey := "SHA256E-s1048576--7ba87e06b9b7903cfbaf4a38736766c161e3e7b42f06fe57f040aa410a8f0701.this-is-a-test-key"// Specify an absolute path to transfer.- require.True(t, filepath.IsAbs(fileToTransfer))- h.requireWriteLine(fmt.Sprintf("TRANSFER STORE %s %s", realisticKey, fileToTransfer))+ h.requireWriteLine(fmt.Sprintf("TRANSFER STORE %s %s", realisticKey, absPath))h.requireReadLineExact("TRANSFER-SUCCESS STORE " + realisticKey)- require.FileExists(t, filepath.Join(h.localFsDir, realisticKey))++ remoteItem := fstest.NewItem(realisticKey, "HELLO", item.ModTime)+ h.fstestRun.CheckRemoteItems(t, remoteItem)h.requireWriteLine("CHECKPRESENT " + realisticKey)h.requireReadLineExact("CHECKPRESENT-SUCCESS " + realisticKey)@@ -849,27 +1028,36 @@ var localBackendTestCases = []testCase{h.preconfigureServer()// Create temp file for transfer.- fileToTransfer := filepath.Join(t.TempDir(), "file.txt")- require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))+ item := h.fstestRun.WriteFile("file.txt", "HELLO", time.Now())+ absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path)+ require.True(t, filepath.IsAbs(absPath))h.requireReadLineExact("VERSION 1")h.requireWriteLine("INITREMOTE")h.requireReadLineExact("INITREMOTE-SUCCESS")// Specify an absolute path to transfer.- require.True(t, filepath.IsAbs(fileToTransfer))- h.requireWriteLine("TRANSFER STORE SomeKey " + fileToTransfer)+ h.requireWriteLine("TRANSFER STORE SomeKey " + absPath)h.requireReadLineExact("TRANSFER-SUCCESS STORE SomeKey")- require.FileExists(t, filepath.Join(h.localFsDir, "SomeKey"))++ remoteItem := fstest.NewItem("SomeKey", "HELLO", item.ModTime)+ h.fstestRun.CheckRemoteItems(t, remoteItem)h.requireWriteLine("CHECKPRESENT SomeKey")h.requireReadLineExact("CHECKPRESENT-SUCCESS SomeKey")- retrievedFilePath := fileToTransfer + ".retrieved"- require.NoFileExists(t, retrievedFilePath)+ h.fstestRun.CheckLocalItems(t,+ fstest.NewItem("file.txt", "HELLO", item.ModTime),+ )++ retrievedFilePath := absPath + ".retrieved"h.requireWriteLine("TRANSFER RETRIEVE SomeKey " + retrievedFilePath)h.requireReadLineExact("TRANSFER-SUCCESS RETRIEVE SomeKey")- require.FileExists(t, retrievedFilePath)++ h.fstestRun.CheckLocalItems(t,+ fstest.NewItem("file.txt", "HELLO", item.ModTime),+ fstest.NewItem("file.txt.retrieved", "HELLO", item.ModTime),+ )require.NoError(t, h.mockStdinW.Close())},@@ -879,11 +1067,13 @@ var localBackendTestCases = []testCase{testProtocolFunc: func(t *testing.T, h *testState) {h.preconfigureServer()+ ctx := context.WithoutCancel(context.Background())+// Write a file into the remote without using the git-annex// protocol.- remoteFilePath := filepath.Join(h.localFsDir, "SomeKey")- require.NoError(t, os.WriteFile(remoteFilePath, []byte("HELLO"), 0600))- require.FileExists(t, remoteFilePath)+ remoteItem := h.fstestRun.WriteObject(ctx, "SomeKey", "HELLO", time.Now())++ h.fstestRun.CheckRemoteItems(t, remoteItem)h.requireReadLineExact("VERSION 1")h.requireWriteLine("INITREMOTE")@@ -891,15 +1081,18 @@ var localBackendTestCases = []testCase{h.requireWriteLine("CHECKPRESENT SomeKey")h.requireReadLineExact("CHECKPRESENT-SUCCESS SomeKey")- require.FileExists(t, remoteFilePath)++ h.fstestRun.CheckRemoteItems(t, remoteItem)h.requireWriteLine("REMOVE SomeKey")h.requireReadLineExact("REMOVE-SUCCESS SomeKey")- require.NoFileExists(t, remoteFilePath)++ h.requireRemoteIsEmpty()h.requireWriteLine("CHECKPRESENT SomeKey")h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")- require.NoFileExists(t, remoteFilePath)++ h.requireRemoteIsEmpty()require.NoError(t, h.mockStdinW.Close())},@@ -910,8 +1103,9 @@ var localBackendTestCases = []testCase{h.preconfigureServer()// Create temp file for transfer.- fileToTransfer := filepath.Join(t.TempDir(), "file.txt")- require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))+ item := h.fstestRun.WriteFile("file.txt", "HELLO", time.Now())+ absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path)+ require.True(t, filepath.IsAbs(absPath))h.requireReadLineExact("VERSION 1")h.requireWriteLine("INITREMOTE")@@ -921,17 +1115,19 @@ var localBackendTestCases = []testCase{h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")// Specify an absolute path to transfer.- require.True(t, filepath.IsAbs(fileToTransfer))- h.requireWriteLine("TRANSFER STORE SomeKey " + fileToTransfer)+ h.requireWriteLine("TRANSFER STORE SomeKey " + absPath)h.requireReadLineExact("TRANSFER-SUCCESS STORE SomeKey")- require.FileExists(t, filepath.Join(h.localFsDir, "SomeKey"))++ remoteItem := fstest.NewItem("SomeKey", "HELLO", item.ModTime)+ h.fstestRun.CheckRemoteItems(t, remoteItem)h.requireWriteLine("CHECKPRESENT SomeKey")h.requireReadLineExact("CHECKPRESENT-SUCCESS SomeKey")h.requireWriteLine("REMOVE SomeKey")h.requireReadLineExact("REMOVE-SUCCESS SomeKey")- require.NoFileExists(t, filepath.Join(h.localFsDir, "SomeKey"))++ h.requireRemoteIsEmpty()h.requireWriteLine("CHECKPRESENT SomeKey")h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")@@ -944,10 +1140,6 @@ var localBackendTestCases = []testCase{testProtocolFunc: func(t *testing.T, h *testState) {h.preconfigureServer()- // Create temp file for transfer.- fileToTransfer := filepath.Join(t.TempDir(), "file.txt")- require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))-h.requireReadLineExact("VERSION 1")h.requireWriteLine("INITREMOTE")h.requireReadLineExact("INITREMOTE-SUCCESS")@@ -955,10 +1147,12 @@ var localBackendTestCases = []testCase{h.requireWriteLine("CHECKPRESENT SomeKey")h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")- require.NoFileExists(t, filepath.Join(h.localFsDir, "SomeKey"))+ h.requireRemoteIsEmpty()+h.requireWriteLine("REMOVE SomeKey")h.requireReadLineExact("REMOVE-SUCCESS SomeKey")- require.NoFileExists(t, filepath.Join(h.localFsDir, "SomeKey"))++ h.requireRemoteIsEmpty()h.requireWriteLine("CHECKPRESENT SomeKey")h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")@@ -983,44 +1177,40 @@ var localBackendTestCases = []testCase{},}-func TestGitAnnexLocalBackendCases(t *testing.T) {- for _, testCase := range localBackendTestCases {- // Clear global state left behind by tests that chdir to a temp directory.- cache.Clear()+// TestMain drives the tests+func TestMain(m *testing.M) {+ fstest.TestMain(m)+}++// Run fstest-compatible test cases with backend selected by `-remote`.+func TestGitAnnexFstestBackendCases(t *testing.T) {+ for _, testCase := range fstestTestCases {// TODO: Remove this when rclone requires a Go version >= 1.22. Future// versions of Go fix the semantics of capturing a range variable.// https://go.dev/blog/loopvar-previewtestCase := testCaset.Run(testCase.label, func(t *testing.T) {- tempDir := t.TempDir()+ r := fstest.NewRun(t)+ t.Cleanup(func() { r.Finalise() })- // Create temp dir for an rclone remote pointing at local filesystem.- localFsDir := filepath.Join(tempDir, "remoteTarget")- require.NoError(t, os.Mkdir(localFsDir, 0700))+ // Parse the fstest-provided remote string. It might have a path!+ remoteName, remotePath, err := fspath.SplitFs(r.FremoteName)+ require.NoError(t, err)- // Create temp config- remoteName := getUniqueRemoteName(t)- configLines := []string{- fmt.Sprintf("[%s]", remoteName),- "type = local",- fmt.Sprintf("remote = %s", localFsDir),+ // The gitannex command requires the `rcloneremotename` is the name+ // of a remote or exactly ":local", so the empty string will not+ // suffice.+ if remoteName == "" {+ require.True(t, r.Fremote.Features().IsLocal)+ remoteName = ":local"}- configContents := strings.Join(configLines, "\n")-- configPath := filepath.Join(tempDir, "rclone.conf")- require.NoError(t, os.WriteFile(configPath, []byte(configContents), 0600))- require.NoError(t, config.SetConfigPath(configPath))-- // The custom config file will be ignored unless we install the- // global config file handler.- configfile.Install()handle := makeTestState(t)- handle.localFsDir = localFsDir- handle.configPath = configPath+ handle.fstestRun = rhandle.remoteName = remoteName+ handle.remotePrefix = remotePathvar wg sync.WaitGroupwg.Add(1)@@ -1042,54 +1232,3 @@ func TestGitAnnexLocalBackendCases(t *testing.T) {})}}--// Configure the git-annex client with a mockfs backend and send it the-// "INITREMOTE" command over mocked stdin. This should fail because mockfs does-// not support empty directories.-func TestGitAnnexHandleInitRemoteBackendDoesNotSupportEmptyDirectories(t *testing.T) {- tempDir := t.TempDir()-- // Temporarily override the filesystem registry.- oldRegistry := fs.Registry- mockfs.Register()- defer func() { fs.Registry = oldRegistry }()-- // Create temp dir for an rclone remote pointing at local filesystem.- localFsDir := filepath.Join(tempDir, "remoteTarget")- require.NoError(t, os.Mkdir(localFsDir, 0700))-- // Create temp config- remoteName := getUniqueRemoteName(t)- configLines := []string{- fmt.Sprintf("[%s]", remoteName),- "type = mockfs",- fmt.Sprintf("remote = %s", localFsDir),- }- configContents := strings.Join(configLines, "\n")-- configPath := filepath.Join(tempDir, "rclone.conf")- require.NoError(t, os.WriteFile(configPath, []byte(configContents), 0600))-- // The custom config file will be ignored unless we install the global- // config file handler.- configfile.Install()- require.NoError(t, config.SetConfigPath(configPath))-- handle := makeTestState(t)- handle.server.configPrefix = localFsDir- handle.server.configRcloneRemoteName = remoteName- handle.server.configsDone = true-- var wg sync.WaitGroup- wg.Add(1)-- go func() {- require.NotNil(t, handle.server.run())- wg.Done()- }()- defer wg.Wait()-- handle.requireReadLineExact("VERSION 1")- handle.requireWriteLine("INITREMOTE")- handle.requireReadLineExact("INITREMOTE-FAILURE this rclone remote does not support empty directories")-}diff --git a/cmd/gitannex/layout.go b/cmd/gitannex/layout.goindex 81b18bfb6..897cdb5e0 100644--- a/cmd/gitannex/layout.go+++ b/cmd/gitannex/layout.go@@ -3,6 +3,8 @@ package gitanneximport ("fmt""strings"++ "github.com/rclone/rclone/fs/fspath")type layoutMode string@@ -39,8 +41,11 @@ func parseLayoutMode(mode string) layoutMode {type queryDirhashFunc func(msg string) (string, error)func buildFsString(queryDirhash queryDirhashFunc, mode layoutMode, key, remoteName, prefix string) (string, error) {+ remoteName = strings.TrimSuffix(remoteName, ":") + ":"+ remoteString := fspath.JoinRootPath(remoteName, prefix)+if mode == layoutModeNodir {- return fmt.Sprintf("%s:%s", remoteName, prefix), nil+ return remoteString, nil}var dirhash string@@ -59,13 +64,13 @@ func buildFsString(queryDirhash queryDirhashFunc, mode layoutMode, key, remoteNaswitch mode {case layoutModeLower:- return fmt.Sprintf("%s:%s/%s", remoteName, prefix, dirhash), nil+ return fmt.Sprintf("%s/%s", remoteString, dirhash), nilcase layoutModeDirectory:- return fmt.Sprintf("%s:%s/%s%s", remoteName, prefix, dirhash, key), nil+ return fmt.Sprintf("%s/%s%s", remoteString, dirhash, key), nilcase layoutModeMixed:- return fmt.Sprintf("%s:%s/%s", remoteName, prefix, dirhash), nil+ return fmt.Sprintf("%s/%s", remoteString, dirhash), nilcase layoutModeFrankencase:- return fmt.Sprintf("%s:%s/%s", remoteName, prefix, strings.ToLower(dirhash)), nil+ return fmt.Sprintf("%s/%s", remoteString, strings.ToLower(dirhash)), nildefault:panic("unreachable")}