Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

cmd/gitannex: Port unit tests to fstest

Dan McArdlebe33e281b3e8
diff --git a/cmd/gitannex/gitannex.go b/cmd/gitannex/gitannex.go
index 287adc7ae..d71393b05 100644
--- a/cmd/gitannex/gitannex.go
+++ b/cmd/gitannex/gitannex.go
@@ -7,7 +7,7 @@
// (Tracked in [issue #7625].)
//
// 1. ✅ Minimal support for the [external special remote protocol]. Tested on
-// "local" and "drive" backends.
+// "local", "drive", and "dropbox" backends.
// 2. Add support for the ASYNC protocol extension. This may improve performance.
// 3. Support the [simple export interface]. This will enable `git-annex
// export` functionality.
@@ -33,6 +33,7 @@ import (
"github.com/rclone/rclone/cmd"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/cache"
+ "github.com/rclone/rclone/fs/config"
"github.com/rclone/rclone/fs/operations"
"github.com/spf13/cobra"
)
@@ -267,22 +268,34 @@ func (s *server) handleInitRemote() error {
return fmt.Errorf("failed to get configs: %w", err)
}
- remoteRootFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:", s.configRcloneRemoteName))
- if err != nil {
- s.sendMsg("INITREMOTE-FAILURE failed to open root directory of rclone remote")
- return fmt.Errorf("failed to open root directory of rclone remote: %w", err)
- }
+ // Explicitly check that a remote with the given name exists. If we just
+ // relied on `cache.Get()` to return `fs.ErrorNotFoundInConfigFile`, this
+ // function would incorrectly succeed when the given remote name is actually
+ // a file path.
+ //
+ // The :local: backend does not correspond to a remote named by the rclone
+ // config, but is permitted to enable testing. Technically, a user might hit
+ // this code path, but it would be a strange choice because git-annex
+ // natively supports a "directory" special remote.
- if !remoteRootFs.Features().CanHaveEmptyDirectories {
- s.sendMsg("INITREMOTE-FAILURE this rclone remote does not support empty directories")
- return fmt.Errorf("rclone remote does not support empty directories")
- }
+ trimmedName := strings.TrimSuffix(s.configRcloneRemoteName, ":")
- if err := operations.Mkdir(context.TODO(), remoteRootFs, s.configPrefix); err != nil {
- s.sendMsg("INITREMOTE-FAILURE failed to mkdir")
- return fmt.Errorf("failed to mkdir: %w", err)
+ if s.configRcloneRemoteName != ":local" {
+ var remoteExists bool
+ for _, remoteName := range config.FileSections() {
+ if remoteName == trimmedName {
+ remoteExists = true
+ break
+ }
+ }
+ if !remoteExists {
+ s.sendMsg("INITREMOTE-FAILURE remote does not exist: " + s.configRcloneRemoteName)
+ return fmt.Errorf("remote does not exist: %s", s.configRcloneRemoteName)
+ }
}
+ s.configRcloneRemoteName = trimmedName + ":"
+
s.sendMsg("INITREMOTE-SUCCESS")
return nil
}
diff --git a/cmd/gitannex/gitannex_test.go b/cmd/gitannex/gitannex_test.go
index d53a7f921..6d60e1db1 100644
--- a/cmd/gitannex/gitannex_test.go
+++ b/cmd/gitannex/gitannex_test.go
@@ -2,25 +2,24 @@ package gitannex
import (
"bufio"
- "crypto/sha256"
+ "context"
"fmt"
"io"
"os"
"path/filepath"
"regexp"
+ "runtime"
"strings"
"sync"
"testing"
+ "time"
- // Without this import, the local filesystem backend would be unavailable.
- // It looks unused, but the act of importing it runs its `init()` function.
- _ "github.com/rclone/rclone/backend/local"
+ // Without this import, the various backends would be unavailable. It looks
+ // unused, but the act of importing runs the package's `init()` function.
+ _ "github.com/rclone/rclone/backend/all"
- "github.com/rclone/rclone/fs"
- "github.com/rclone/rclone/fs/cache"
- "github.com/rclone/rclone/fs/config"
- "github.com/rclone/rclone/fs/config/configfile"
- "github.com/rclone/rclone/fstest/mockfs"
+ "github.com/rclone/rclone/fs/fspath"
+ "github.com/rclone/rclone/fstest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -256,9 +255,9 @@ type testState struct {
mockStdinW *io.PipeWriter
mockStdoutReader *bufio.Reader
- localFsDir string
- configPath string
- remoteName string
+ fstestRun *fstest.Run
+ remoteName string
+ remotePrefix string
}
func makeTestState(t *testing.T) testState {
@@ -276,6 +275,10 @@ func makeTestState(t *testing.T) testState {
}
}
+func (h *testState) requireRemoteIsEmpty() {
+ h.fstestRun.CheckRemoteItems(h.t)
+}
+
func (h *testState) requireReadLineExact(line string) {
receivedLine, err := h.mockStdoutReader.ReadString('\n')
require.NoError(h.t, err)
@@ -296,21 +299,106 @@ func (h *testState) requireWriteLine(line string) {
// Preconfigure the handle. This enables the calling test to skip the PREPARE
// handshake.
func (h *testState) preconfigureServer() {
- h.server.configPrefix = h.localFsDir
h.server.configRcloneRemoteName = h.remoteName
+ h.server.configPrefix = h.remotePrefix
h.server.configRcloneLayout = string(layoutModeNodir)
h.server.configsDone = true
}
-// getUniqueRemoteName returns a valid remote name derived from the given test's
-// name. This is necessary because when a test registers a second remote with
-// the same name, the original remote appears to take precedence. This function
-// is injective, so each test gets a unique remote name. Returned strings
-// contain no spaces.
-func getUniqueRemoteName(t *testing.T) string {
- // Using sha256 as a hack to ensure injectivity without adding a global
- // variable.
- return fmt.Sprintf("remote-%x", sha256.Sum256([]byte(t.Name())))
+// Drop-in replacement for `filepath.Rel()` that works around a Windows-specific
+// quirk when one of the paths begins with `\\?\` or `//?/`. It seems that
+// fstest gives us paths with this prefix on Windows, which throws a wrench in
+// the gitannex tests that need to construct relative paths from absolute paths.
+// For a demonstration, see `TestWindowsFilepathRelQuirk` below.
+//
+// The `\\?\` prefix tells Windows APIs to pass strings unmodified to the
+// filesystem without additional parsing [1]. Our workaround is roughly to add
+// the prefix to whichever parameter doesn't have it (when the OS is Windows).
+// I'm not sure this generalizes, but it works for the the kinds of inputs we're
+// throwing at it.
+//
+// [1]: https://learn.microsoft.com/en-us/windows/win32/fileio/naming-a-file?redirectedfrom=MSDN#win32-file-namespaces
+func relativeFilepathWorkaround(basepath, targpath string) (string, error) {
+ if runtime.GOOS != "windows" {
+ return filepath.Rel(basepath, targpath)
+ }
+ // Canonicalize paths to use backslashes.
+ basepath = filepath.Clean(basepath)
+ targpath = filepath.Clean(targpath)
+
+ const winFilePrefixDisableStringParsing = `\\?\`
+ baseHasPrefix := strings.HasPrefix(basepath, winFilePrefixDisableStringParsing)
+ targHasPrefix := strings.HasPrefix(targpath, winFilePrefixDisableStringParsing)
+
+ if baseHasPrefix && !targHasPrefix {
+ targpath = winFilePrefixDisableStringParsing + targpath
+ }
+ if !baseHasPrefix && targHasPrefix {
+ basepath = winFilePrefixDisableStringParsing + basepath
+ }
+ return filepath.Rel(basepath, targpath)
+}
+
+func TestWindowsFilepathRelQuirk(t *testing.T) {
+ if runtime.GOOS != "windows" {
+ t.Skip()
+ }
+
+ t.Run("filepathRelQuirk", func(t *testing.T) {
+ var err error
+
+ _, err = filepath.Rel(`C:\foo`, `\\?\C:\foo\bar`)
+ require.Error(t, err)
+
+ _, err = filepath.Rel(`C:/foo`, `//?/C:/foo/bar`)
+ require.Error(t, err)
+
+ _, err = filepath.Rel(`\\?\C:\foo`, `C:\foo\bar`)
+ require.Error(t, err)
+
+ _, err = filepath.Rel(`//?/C:/foo`, `C:/foo/bar`)
+ require.Error(t, err)
+
+ path, err := filepath.Rel(`\\?\C:\foo`, `\\?\C:\foo\bar`)
+ require.NoError(t, err)
+ require.Equal(t, path, `bar`)
+
+ path, err = filepath.Rel(`//?/C:/foo`, `//?/C:/foo/bar`)
+ require.NoError(t, err)
+ require.Equal(t, path, `bar`)
+ })
+
+ t.Run("fstestAndTempDirHaveDifferentPrefixes", func(t *testing.T) {
+ r := fstest.NewRun(t)
+ p := r.Flocal.Root()
+ require.True(t, strings.HasPrefix(p, `//?/`))
+
+ tempDir := t.TempDir()
+ require.False(t, strings.HasPrefix(tempDir, `//?/`))
+ require.False(t, strings.HasPrefix(tempDir, `\\?\`))
+ })
+
+ t.Run("workaroundWorks", func(t *testing.T) {
+ path, err := relativeFilepathWorkaround(`C:\foo`, `\\?\C:\foo\bar`)
+ require.NoError(t, err)
+ require.Equal(t, path, "bar")
+
+ path, err = relativeFilepathWorkaround(`C:/foo`, `//?/C:/foo/bar`)
+ require.NoError(t, err)
+ require.Equal(t, path, "bar")
+
+ path, err = relativeFilepathWorkaround(`\\?\C:\foo`, `C:\foo\bar`)
+ require.NoError(t, err)
+ require.Equal(t, path, `bar`)
+
+ path, err = relativeFilepathWorkaround(`//?/C:/foo`, `C:/foo/bar`)
+ require.NoError(t, err)
+ require.Equal(t, path, `bar`)
+
+ path, err = relativeFilepathWorkaround(`\\?\C:\foo`, `\\?\C:\foo\bar`)
+ require.NoError(t, err)
+ require.Equal(t, path, `bar`)
+ })
}
type testCase struct {
@@ -319,8 +407,8 @@ type testCase struct {
expectedError string
}
-// These test cases run against the "local" backend.
-var localBackendTestCases = []testCase{
+// These test cases run against a backend selected by the `-remote` flag.
+var fstestTestCases = []testCase{
{
label: "HandlesInit",
testProtocolFunc: func(t *testing.T, h *testState) {
@@ -377,17 +465,85 @@ var localBackendTestCases = []testCase{
h.requireReadLineExact("GETCONFIG rcloneremotename")
h.requireWriteLine("VALUE " + h.remoteName)
h.requireReadLineExact("GETCONFIG rcloneprefix")
- h.requireWriteLine("VALUE " + h.localFsDir)
+ h.requireWriteLine("VALUE " + h.remotePrefix)
h.requireReadLineExact("GETCONFIG rclonelayout")
h.requireWriteLine("VALUE foo")
h.requireReadLineExact("PREPARE-SUCCESS")
require.Equal(t, h.server.configRcloneRemoteName, h.remoteName)
- require.Equal(t, h.server.configPrefix, h.localFsDir)
+ require.Equal(t, h.server.configPrefix, h.remotePrefix)
+ require.True(t, h.server.configsDone)
+
+ require.NoError(t, h.mockStdinW.Close())
+ },
+ },
+ {
+ label: "HandlesPrepareWithNonexistentRemote",
+ testProtocolFunc: func(t *testing.T, h *testState) {
+ h.requireReadLineExact("VERSION 1")
+ h.requireWriteLine("EXTENSIONS INFO") // Advertise that we support the INFO extension
+ h.requireReadLineExact("EXTENSIONS")
+
+ if !h.server.extensionInfo {
+ t.Errorf("expected INFO extension to be enabled")
+ return
+ }
+
+ h.requireWriteLine("PREPARE")
+ h.requireReadLineExact("GETCONFIG rcloneremotename")
+ h.requireWriteLine("VALUE thisRemoteDoesNotExist")
+ h.requireReadLineExact("GETCONFIG rcloneprefix")
+ h.requireWriteLine("VALUE " + h.remotePrefix)
+ h.requireReadLineExact("GETCONFIG rclonelayout")
+ h.requireWriteLine("VALUE foo")
+ h.requireReadLineExact("PREPARE-SUCCESS")
+
+ require.Equal(t, h.server.configRcloneRemoteName, "thisRemoteDoesNotExist")
+ require.Equal(t, h.server.configPrefix, h.remotePrefix)
+ require.True(t, h.server.configsDone)
+
+ h.requireWriteLine("INITREMOTE")
+ h.requireReadLineExact("INITREMOTE-FAILURE remote does not exist: thisRemoteDoesNotExist")
+
+ require.NoError(t, h.mockStdinW.Close())
+ },
+ expectedError: "remote does not exist: thisRemoteDoesNotExist",
+ },
+ {
+ label: "HandlesPrepareWithPathAsRemote",
+ testProtocolFunc: func(t *testing.T, h *testState) {
+ h.requireReadLineExact("VERSION 1")
+ h.requireWriteLine("EXTENSIONS INFO") // Advertise that we support the INFO extension
+ h.requireReadLineExact("EXTENSIONS")
+
+ if !h.server.extensionInfo {
+ t.Errorf("expected INFO extension to be enabled")
+ return
+ }
+
+ h.requireWriteLine("PREPARE")
+ h.requireReadLineExact("GETCONFIG rcloneremotename")
+ h.requireWriteLine("VALUE " + h.remotePrefix)
+ h.requireReadLineExact("GETCONFIG rcloneprefix")
+ h.requireWriteLine("VALUE /foo")
+ h.requireReadLineExact("GETCONFIG rclonelayout")
+ h.requireWriteLine("VALUE foo")
+ h.requireReadLineExact("PREPARE-SUCCESS")
+
+ require.Equal(t, h.server.configRcloneRemoteName, h.remotePrefix)
+ require.Equal(t, h.server.configPrefix, "/foo")
require.True(t, h.server.configsDone)
+ h.requireWriteLine("INITREMOTE")
+
+ require.Regexp(t,
+ regexp.MustCompile("^INITREMOTE-FAILURE remote does not exist: "),
+ h.requireReadLine(),
+ )
+
require.NoError(t, h.mockStdinW.Close())
},
+ expectedError: "remote does not exist:",
},
{
label: "HandlesPrepareWithSynonyms",
@@ -409,13 +565,13 @@ var localBackendTestCases = []testCase{
h.requireWriteLine("VALUE " + h.remoteName)
h.requireReadLineExact("GETCONFIG rcloneprefix")
- h.requireWriteLine("VALUE " + h.localFsDir)
+ h.requireWriteLine("VALUE " + h.remotePrefix)
h.requireReadLineExact("GETCONFIG rclonelayout")
h.requireWriteLine("VALUE foo")
h.requireReadLineExact("PREPARE-SUCCESS")
require.Equal(t, h.server.configRcloneRemoteName, h.remoteName)
- require.Equal(t, h.server.configPrefix, h.localFsDir)
+ require.Equal(t, h.server.configPrefix, h.remotePrefix)
require.True(t, h.server.configsDone)
require.NoError(t, h.mockStdinW.Close())
@@ -437,12 +593,12 @@ var localBackendTestCases = []testCase{
h.requireReadLineExact("GETCONFIG rcloneremotename")
remoteNameWithSpaces := fmt.Sprintf(" %s ", h.remoteName)
- localFsDirWithSpaces := fmt.Sprintf(" %s\t", h.localFsDir)
+ prefixWithWhitespace := fmt.Sprintf(" %s\t", h.remotePrefix)
h.requireWriteLine(fmt.Sprintf("VALUE %s", remoteNameWithSpaces))
h.requireReadLineExact("GETCONFIG rcloneprefix")
- h.requireWriteLine(fmt.Sprintf("VALUE %s", localFsDirWithSpaces))
+ h.requireWriteLine(fmt.Sprintf("VALUE %s", prefixWithWhitespace))
h.requireReadLineExact("GETCONFIG rclonelayout")
h.requireWriteLine("VALUE")
@@ -452,7 +608,7 @@ var localBackendTestCases = []testCase{
h.requireReadLineExact("PREPARE-SUCCESS")
require.Equal(t, h.server.configRcloneRemoteName, remoteNameWithSpaces)
- require.Equal(t, h.server.configPrefix, localFsDirWithSpaces)
+ require.Equal(t, h.server.configPrefix, prefixWithWhitespace)
require.True(t, h.server.configsDone)
require.NoError(t, h.mockStdinW.Close())
@@ -639,20 +795,25 @@ var localBackendTestCases = []testCase{
h.requireReadLineExact("INITREMOTE-SUCCESS")
// Create temp file for transfer with an absolute path.
- fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
- require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
- require.FileExists(t, fileToTransfer)
- require.True(t, filepath.IsAbs(fileToTransfer))
+ item := h.fstestRun.WriteFile("file.txt", "HELLO", time.Now())
+ absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path)
+ require.True(t, filepath.IsAbs(absPath))
// Specify an absolute path to transfer.
- h.requireWriteLine("TRANSFER STORE KeyAbsolute " + fileToTransfer)
+ h.requireWriteLine("TRANSFER STORE KeyAbsolute " + absPath)
h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyAbsolute")
- require.FileExists(t, filepath.Join(h.localFsDir, "KeyAbsolute"))
+
+ // Check that the file was transferred.
+ remoteItem := fstest.NewItem("KeyAbsolute", "HELLO", item.ModTime)
+ h.fstestRun.CheckRemoteItems(t, remoteItem)
// Transfer the same absolute path a second time, but with a different key.
- h.requireWriteLine("TRANSFER STORE KeyAbsolute2 " + fileToTransfer)
+ h.requireWriteLine("TRANSFER STORE KeyAbsolute2 " + absPath)
h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyAbsolute2")
- require.FileExists(t, filepath.Join(h.localFsDir, "KeyAbsolute2"))
+
+ // Check that the same file was transferred to a new name.
+ remoteItem2 := fstest.NewItem("KeyAbsolute2", "HELLO", item.ModTime)
+ h.fstestRun.CheckRemoteItems(t, remoteItem, remoteItem2)
h.requireWriteLine("CHECKPRESENT KeyAbsolute2")
h.requireReadLineExact("CHECKPRESENT-SUCCESS KeyAbsolute2")
@@ -668,30 +829,36 @@ var localBackendTestCases = []testCase{
{
label: "TransferStoreRelative",
testProtocolFunc: func(t *testing.T, h *testState) {
- h.preconfigureServer()
-
// Save the current working directory so we can restore it when this
// test ends.
cwd, err := os.Getwd()
require.NoError(t, err)
- require.NoError(t, os.Chdir(t.TempDir()))
+ tempDir := t.TempDir()
+
+ require.NoError(t, os.Chdir(tempDir))
t.Cleanup(func() { require.NoError(t, os.Chdir(cwd)) })
+ h.preconfigureServer()
+
h.requireReadLineExact("VERSION 1")
h.requireWriteLine("INITREMOTE")
h.requireReadLineExact("INITREMOTE-SUCCESS")
- // Create temp file for transfer with a relative path.
- fileToTransfer := "file.txt"
- require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
- require.FileExists(t, fileToTransfer)
- require.False(t, filepath.IsAbs(fileToTransfer))
+ item := h.fstestRun.WriteFile("file.txt", "HELLO", time.Now())
+ absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path)
+
+ relativePath, err := relativeFilepathWorkaround(tempDir, absPath)
+ require.NoError(t, err)
+ require.False(t, filepath.IsAbs(relativePath))
+ require.FileExists(t, relativePath)
// Specify a relative path to transfer.
- h.requireWriteLine("TRANSFER STORE KeyRelative " + fileToTransfer)
+ h.requireWriteLine("TRANSFER STORE KeyRelative " + relativePath)
h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyRelative")
- require.FileExists(t, filepath.Join(h.localFsDir, "KeyRelative"))
+
+ remoteItem := fstest.NewItem("KeyRelative", "HELLO", item.ModTime)
+ h.fstestRun.CheckRemoteItems(t, remoteItem)
h.requireWriteLine("CHECKPRESENT KeyRelative")
h.requireReadLineExact("CHECKPRESENT-SUCCESS KeyRelative")
@@ -710,7 +877,8 @@ var localBackendTestCases = []testCase{
cwd, err := os.Getwd()
require.NoError(t, err)
- require.NoError(t, os.Chdir(t.TempDir()))
+ tempDir := t.TempDir()
+ require.NoError(t, os.Chdir(tempDir))
t.Cleanup(func() { require.NoError(t, os.Chdir(cwd)) })
h.preconfigureServer()
@@ -720,15 +888,19 @@ var localBackendTestCases = []testCase{
h.requireReadLineExact("INITREMOTE-SUCCESS")
// Create temp file for transfer.
- fileToTransfer := "filename with spaces.txt"
- require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
- require.FileExists(t, fileToTransfer)
- require.False(t, filepath.IsAbs(fileToTransfer))
+ item := h.fstestRun.WriteFile("filename with spaces.txt", "HELLO", time.Now())
+ absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path)
+ relativePath, err := relativeFilepathWorkaround(tempDir, absPath)
+ require.NoError(t, err)
+ require.False(t, filepath.IsAbs(relativePath))
+ require.FileExists(t, relativePath)
// Specify a relative path to transfer.
- h.requireWriteLine("TRANSFER STORE KeyRelative " + fileToTransfer)
+ h.requireWriteLine("TRANSFER STORE KeyRelative " + relativePath)
h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyRelative")
- require.FileExists(t, filepath.Join(h.localFsDir, "KeyRelative"))
+
+ remoteItem := fstest.NewItem("KeyRelative", "HELLO", item.ModTime)
+ h.fstestRun.CheckRemoteItems(t, remoteItem)
h.requireWriteLine("CHECKPRESENT KeyRelative")
h.requireReadLineExact("CHECKPRESENT-SUCCESS KeyRelative")
@@ -745,8 +917,9 @@ var localBackendTestCases = []testCase{
h.preconfigureServer()
// Create temp file for transfer.
- fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
- require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
+ item := h.fstestRun.WriteFile("file.txt", "HELLO", time.Now())
+ absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path)
+ require.True(t, filepath.IsAbs(absPath))
h.requireReadLineExact("VERSION 1")
h.requireWriteLine("INITREMOTE")
@@ -756,10 +929,11 @@ var localBackendTestCases = []testCase{
h.requireReadLineExact("CHECKPRESENT-FAILURE KeyThatDoesNotExist")
// Specify an absolute path to transfer.
- require.True(t, filepath.IsAbs(fileToTransfer))
- h.requireWriteLine("TRANSFER STORE KeyAbsolute " + fileToTransfer)
+ h.requireWriteLine("TRANSFER STORE KeyAbsolute " + absPath)
h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyAbsolute")
- require.FileExists(t, filepath.Join(h.localFsDir, "KeyAbsolute"))
+
+ remoteItem := fstest.NewItem("KeyAbsolute", "HELLO", item.ModTime)
+ h.fstestRun.CheckRemoteItems(t, remoteItem)
require.NoError(t, h.mockStdinW.Close())
},
@@ -781,8 +955,9 @@ var localBackendTestCases = []testCase{
h.preconfigureServer()
// Create temp file for transfer.
- fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
- require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
+ item := h.fstestRun.WriteFile("file.txt", "HELLO", time.Now())
+ absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path)
+ require.True(t, filepath.IsAbs(absPath))
h.requireReadLineExact("VERSION 1")
h.requireWriteLine("INITREMOTE")
@@ -791,9 +966,11 @@ var localBackendTestCases = []testCase{
h.requireWriteLine("CHECKPRESENT foo")
h.requireReadLineExact("CHECKPRESENT-FAILURE foo")
- h.requireWriteLine("TRANSFER STORE foo " + fileToTransfer)
+ h.requireWriteLine("TRANSFER STORE foo " + absPath)
h.requireReadLineExact("TRANSFER-SUCCESS STORE foo")
- require.FileExists(t, filepath.Join(h.localFsDir, "foo"))
+
+ remoteItem := fstest.NewItem("foo", "HELLO", item.ModTime)
+ h.fstestRun.CheckRemoteItems(t, remoteItem)
h.requireWriteLine("CHECKPRESENT foo")
h.requireReadLineExact("CHECKPRESENT-SUCCESS foo")
@@ -807,8 +984,9 @@ var localBackendTestCases = []testCase{
h.preconfigureServer()
// Create temp file for transfer.
- fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
- require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
+ item := h.fstestRun.WriteFile("file.txt", "HELLO", time.Now())
+ absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path)
+ require.True(t, filepath.IsAbs(absPath))
h.requireReadLineExact("VERSION 1")
h.requireWriteLine("INITREMOTE")
@@ -817,10 +995,11 @@ var localBackendTestCases = []testCase{
realisticKey := "SHA256E-s1048576--7ba87e06b9b7903cfbaf4a38736766c161e3e7b42f06fe57f040aa410a8f0701.this-is-a-test-key"
// Specify an absolute path to transfer.
- require.True(t, filepath.IsAbs(fileToTransfer))
- h.requireWriteLine(fmt.Sprintf("TRANSFER STORE %s %s", realisticKey, fileToTransfer))
+ h.requireWriteLine(fmt.Sprintf("TRANSFER STORE %s %s", realisticKey, absPath))
h.requireReadLineExact("TRANSFER-SUCCESS STORE " + realisticKey)
- require.FileExists(t, filepath.Join(h.localFsDir, realisticKey))
+
+ remoteItem := fstest.NewItem(realisticKey, "HELLO", item.ModTime)
+ h.fstestRun.CheckRemoteItems(t, remoteItem)
h.requireWriteLine("CHECKPRESENT " + realisticKey)
h.requireReadLineExact("CHECKPRESENT-SUCCESS " + realisticKey)
@@ -849,27 +1028,36 @@ var localBackendTestCases = []testCase{
h.preconfigureServer()
// Create temp file for transfer.
- fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
- require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
+ item := h.fstestRun.WriteFile("file.txt", "HELLO", time.Now())
+ absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path)
+ require.True(t, filepath.IsAbs(absPath))
h.requireReadLineExact("VERSION 1")
h.requireWriteLine("INITREMOTE")
h.requireReadLineExact("INITREMOTE-SUCCESS")
// Specify an absolute path to transfer.
- require.True(t, filepath.IsAbs(fileToTransfer))
- h.requireWriteLine("TRANSFER STORE SomeKey " + fileToTransfer)
+ h.requireWriteLine("TRANSFER STORE SomeKey " + absPath)
h.requireReadLineExact("TRANSFER-SUCCESS STORE SomeKey")
- require.FileExists(t, filepath.Join(h.localFsDir, "SomeKey"))
+
+ remoteItem := fstest.NewItem("SomeKey", "HELLO", item.ModTime)
+ h.fstestRun.CheckRemoteItems(t, remoteItem)
h.requireWriteLine("CHECKPRESENT SomeKey")
h.requireReadLineExact("CHECKPRESENT-SUCCESS SomeKey")
- retrievedFilePath := fileToTransfer + ".retrieved"
- require.NoFileExists(t, retrievedFilePath)
+ h.fstestRun.CheckLocalItems(t,
+ fstest.NewItem("file.txt", "HELLO", item.ModTime),
+ )
+
+ retrievedFilePath := absPath + ".retrieved"
h.requireWriteLine("TRANSFER RETRIEVE SomeKey " + retrievedFilePath)
h.requireReadLineExact("TRANSFER-SUCCESS RETRIEVE SomeKey")
- require.FileExists(t, retrievedFilePath)
+
+ h.fstestRun.CheckLocalItems(t,
+ fstest.NewItem("file.txt", "HELLO", item.ModTime),
+ fstest.NewItem("file.txt.retrieved", "HELLO", item.ModTime),
+ )
require.NoError(t, h.mockStdinW.Close())
},
@@ -879,11 +1067,13 @@ var localBackendTestCases = []testCase{
testProtocolFunc: func(t *testing.T, h *testState) {
h.preconfigureServer()
+ ctx := context.WithoutCancel(context.Background())
+
// Write a file into the remote without using the git-annex
// protocol.
- remoteFilePath := filepath.Join(h.localFsDir, "SomeKey")
- require.NoError(t, os.WriteFile(remoteFilePath, []byte("HELLO"), 0600))
- require.FileExists(t, remoteFilePath)
+ remoteItem := h.fstestRun.WriteObject(ctx, "SomeKey", "HELLO", time.Now())
+
+ h.fstestRun.CheckRemoteItems(t, remoteItem)
h.requireReadLineExact("VERSION 1")
h.requireWriteLine("INITREMOTE")
@@ -891,15 +1081,18 @@ var localBackendTestCases = []testCase{
h.requireWriteLine("CHECKPRESENT SomeKey")
h.requireReadLineExact("CHECKPRESENT-SUCCESS SomeKey")
- require.FileExists(t, remoteFilePath)
+
+ h.fstestRun.CheckRemoteItems(t, remoteItem)
h.requireWriteLine("REMOVE SomeKey")
h.requireReadLineExact("REMOVE-SUCCESS SomeKey")
- require.NoFileExists(t, remoteFilePath)
+
+ h.requireRemoteIsEmpty()
h.requireWriteLine("CHECKPRESENT SomeKey")
h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")
- require.NoFileExists(t, remoteFilePath)
+
+ h.requireRemoteIsEmpty()
require.NoError(t, h.mockStdinW.Close())
},
@@ -910,8 +1103,9 @@ var localBackendTestCases = []testCase{
h.preconfigureServer()
// Create temp file for transfer.
- fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
- require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
+ item := h.fstestRun.WriteFile("file.txt", "HELLO", time.Now())
+ absPath := filepath.Join(h.fstestRun.Flocal.Root(), item.Path)
+ require.True(t, filepath.IsAbs(absPath))
h.requireReadLineExact("VERSION 1")
h.requireWriteLine("INITREMOTE")
@@ -921,17 +1115,19 @@ var localBackendTestCases = []testCase{
h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")
// Specify an absolute path to transfer.
- require.True(t, filepath.IsAbs(fileToTransfer))
- h.requireWriteLine("TRANSFER STORE SomeKey " + fileToTransfer)
+ h.requireWriteLine("TRANSFER STORE SomeKey " + absPath)
h.requireReadLineExact("TRANSFER-SUCCESS STORE SomeKey")
- require.FileExists(t, filepath.Join(h.localFsDir, "SomeKey"))
+
+ remoteItem := fstest.NewItem("SomeKey", "HELLO", item.ModTime)
+ h.fstestRun.CheckRemoteItems(t, remoteItem)
h.requireWriteLine("CHECKPRESENT SomeKey")
h.requireReadLineExact("CHECKPRESENT-SUCCESS SomeKey")
h.requireWriteLine("REMOVE SomeKey")
h.requireReadLineExact("REMOVE-SUCCESS SomeKey")
- require.NoFileExists(t, filepath.Join(h.localFsDir, "SomeKey"))
+
+ h.requireRemoteIsEmpty()
h.requireWriteLine("CHECKPRESENT SomeKey")
h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")
@@ -944,10 +1140,6 @@ var localBackendTestCases = []testCase{
testProtocolFunc: func(t *testing.T, h *testState) {
h.preconfigureServer()
- // Create temp file for transfer.
- fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
- require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
-
h.requireReadLineExact("VERSION 1")
h.requireWriteLine("INITREMOTE")
h.requireReadLineExact("INITREMOTE-SUCCESS")
@@ -955,10 +1147,12 @@ var localBackendTestCases = []testCase{
h.requireWriteLine("CHECKPRESENT SomeKey")
h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")
- require.NoFileExists(t, filepath.Join(h.localFsDir, "SomeKey"))
+ h.requireRemoteIsEmpty()
+
h.requireWriteLine("REMOVE SomeKey")
h.requireReadLineExact("REMOVE-SUCCESS SomeKey")
- require.NoFileExists(t, filepath.Join(h.localFsDir, "SomeKey"))
+
+ h.requireRemoteIsEmpty()
h.requireWriteLine("CHECKPRESENT SomeKey")
h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")
@@ -983,44 +1177,40 @@ var localBackendTestCases = []testCase{
},
}
-func TestGitAnnexLocalBackendCases(t *testing.T) {
- for _, testCase := range localBackendTestCases {
- // Clear global state left behind by tests that chdir to a temp directory.
- cache.Clear()
+// TestMain drives the tests
+func TestMain(m *testing.M) {
+ fstest.TestMain(m)
+}
+
+// Run fstest-compatible test cases with backend selected by `-remote`.
+func TestGitAnnexFstestBackendCases(t *testing.T) {
+ for _, testCase := range fstestTestCases {
// TODO: Remove this when rclone requires a Go version >= 1.22. Future
// versions of Go fix the semantics of capturing a range variable.
// https://go.dev/blog/loopvar-preview
testCase := testCase
t.Run(testCase.label, func(t *testing.T) {
- tempDir := t.TempDir()
+ r := fstest.NewRun(t)
+ t.Cleanup(func() { r.Finalise() })
- // Create temp dir for an rclone remote pointing at local filesystem.
- localFsDir := filepath.Join(tempDir, "remoteTarget")
- require.NoError(t, os.Mkdir(localFsDir, 0700))
+ // Parse the fstest-provided remote string. It might have a path!
+ remoteName, remotePath, err := fspath.SplitFs(r.FremoteName)
+ require.NoError(t, err)
- // Create temp config
- remoteName := getUniqueRemoteName(t)
- configLines := []string{
- fmt.Sprintf("[%s]", remoteName),
- "type = local",
- fmt.Sprintf("remote = %s", localFsDir),
+ // The gitannex command requires the `rcloneremotename` is the name
+ // of a remote or exactly ":local", so the empty string will not
+ // suffice.
+ if remoteName == "" {
+ require.True(t, r.Fremote.Features().IsLocal)
+ remoteName = ":local"
}
- configContents := strings.Join(configLines, "\n")
-
- configPath := filepath.Join(tempDir, "rclone.conf")
- require.NoError(t, os.WriteFile(configPath, []byte(configContents), 0600))
- require.NoError(t, config.SetConfigPath(configPath))
-
- // The custom config file will be ignored unless we install the
- // global config file handler.
- configfile.Install()
handle := makeTestState(t)
- handle.localFsDir = localFsDir
- handle.configPath = configPath
+ handle.fstestRun = r
handle.remoteName = remoteName
+ handle.remotePrefix = remotePath
var wg sync.WaitGroup
wg.Add(1)
@@ -1042,54 +1232,3 @@ func TestGitAnnexLocalBackendCases(t *testing.T) {
})
}
}
-
-// Configure the git-annex client with a mockfs backend and send it the
-// "INITREMOTE" command over mocked stdin. This should fail because mockfs does
-// not support empty directories.
-func TestGitAnnexHandleInitRemoteBackendDoesNotSupportEmptyDirectories(t *testing.T) {
- tempDir := t.TempDir()
-
- // Temporarily override the filesystem registry.
- oldRegistry := fs.Registry
- mockfs.Register()
- defer func() { fs.Registry = oldRegistry }()
-
- // Create temp dir for an rclone remote pointing at local filesystem.
- localFsDir := filepath.Join(tempDir, "remoteTarget")
- require.NoError(t, os.Mkdir(localFsDir, 0700))
-
- // Create temp config
- remoteName := getUniqueRemoteName(t)
- configLines := []string{
- fmt.Sprintf("[%s]", remoteName),
- "type = mockfs",
- fmt.Sprintf("remote = %s", localFsDir),
- }
- configContents := strings.Join(configLines, "\n")
-
- configPath := filepath.Join(tempDir, "rclone.conf")
- require.NoError(t, os.WriteFile(configPath, []byte(configContents), 0600))
-
- // The custom config file will be ignored unless we install the global
- // config file handler.
- configfile.Install()
- require.NoError(t, config.SetConfigPath(configPath))
-
- handle := makeTestState(t)
- handle.server.configPrefix = localFsDir
- handle.server.configRcloneRemoteName = remoteName
- handle.server.configsDone = true
-
- var wg sync.WaitGroup
- wg.Add(1)
-
- go func() {
- require.NotNil(t, handle.server.run())
- wg.Done()
- }()
- defer wg.Wait()
-
- handle.requireReadLineExact("VERSION 1")
- handle.requireWriteLine("INITREMOTE")
- handle.requireReadLineExact("INITREMOTE-FAILURE this rclone remote does not support empty directories")
-}
diff --git a/cmd/gitannex/layout.go b/cmd/gitannex/layout.go
index 81b18bfb6..897cdb5e0 100644
--- a/cmd/gitannex/layout.go
+++ b/cmd/gitannex/layout.go
@@ -3,6 +3,8 @@ package gitannex
import (
"fmt"
"strings"
+
+ "github.com/rclone/rclone/fs/fspath"
)
type layoutMode string
@@ -39,8 +41,11 @@ func parseLayoutMode(mode string) layoutMode {
type queryDirhashFunc func(msg string) (string, error)
func buildFsString(queryDirhash queryDirhashFunc, mode layoutMode, key, remoteName, prefix string) (string, error) {
+ remoteName = strings.TrimSuffix(remoteName, ":") + ":"
+ remoteString := fspath.JoinRootPath(remoteName, prefix)
+
if mode == layoutModeNodir {
- return fmt.Sprintf("%s:%s", remoteName, prefix), nil
+ return remoteString, nil
}
var dirhash string
@@ -59,13 +64,13 @@ func buildFsString(queryDirhash queryDirhashFunc, mode layoutMode, key, remoteNa
switch mode {
case layoutModeLower:
- return fmt.Sprintf("%s:%s/%s", remoteName, prefix, dirhash), nil
+ return fmt.Sprintf("%s/%s", remoteString, dirhash), nil
case layoutModeDirectory:
- return fmt.Sprintf("%s:%s/%s%s", remoteName, prefix, dirhash, key), nil
+ return fmt.Sprintf("%s/%s%s", remoteString, dirhash, key), nil
case layoutModeMixed:
- return fmt.Sprintf("%s:%s/%s", remoteName, prefix, dirhash), nil
+ return fmt.Sprintf("%s/%s", remoteString, dirhash), nil
case layoutModeFrankencase:
- return fmt.Sprintf("%s:%s/%s", remoteName, prefix, strings.ToLower(dirhash)), nil
+ return fmt.Sprintf("%s/%s", remoteString, strings.ToLower(dirhash)), nil
default:
panic("unreachable")
}