Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

cmd/gitannex: Add the gitannex subcommand

Dan McArdledfc329c03655
diff --git a/cmd/all/all.go b/cmd/all/all.go
index 5fb87ed16..37b15c5c3 100644
--- a/cmd/all/all.go
+++ b/cmd/all/all.go
@@ -25,6 +25,7 @@ import (
_ "github.com/rclone/rclone/cmd/deletefile"
_ "github.com/rclone/rclone/cmd/genautocomplete"
_ "github.com/rclone/rclone/cmd/gendocs"
+ _ "github.com/rclone/rclone/cmd/gitannex"
_ "github.com/rclone/rclone/cmd/hashsum"
_ "github.com/rclone/rclone/cmd/link"
_ "github.com/rclone/rclone/cmd/listremotes"
diff --git a/cmd/gitannex/gitannex.go b/cmd/gitannex/gitannex.go
new file mode 100644
index 000000000..0c085c133
--- /dev/null
+++ b/cmd/gitannex/gitannex.go
@@ -0,0 +1,515 @@
+// Package gitannex provides the "gitannex" command, which enables [git-annex]
+// to communicate with rclone by implementing the [external special remote
+// protocol]. The protocol is line delimited and spoken over stdin and stdout.
+//
+// # Milestones
+//
+// (Tracked in [issue #7625].)
+//
+// 1. ✅ Minimal support for the [external special remote protocol]. Tested on
+// "local" and "drive" backends.
+// 2. Add support for the ASYNC protocol extension. This may improve performance.
+// 3. Support the [simple export interface]. This will enable `git-annex
+// export` functionality.
+// 4. Once the draft is finalized, support import/export interface.
+//
+// [git-annex]: https://git-annex.branchable.com/
+// [external special remote protocol]: https://git-annex.branchable.com/design/external_special_remote_protocol/
+// [simple export interface]: https://git-annex.branchable.com/design/external_special_remote_protocol/export_and_import_appendix/
+// [issue #7625]: https://github.com/rclone/rclone/issues/7625
+package gitannex
+
+import (
+ "bufio"
+ "context"
+ _ "embed"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "github.com/rclone/rclone/cmd"
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/cache"
+ "github.com/rclone/rclone/fs/operations"
+ "github.com/spf13/cobra"
+)
+
+const subcommandName string = "gitannex"
+const uniqueCommandName string = "git-annex-remote-rclone-builtin"
+
+//go:embed gitannex.md
+var gitannexHelp string
+
+func init() {
+ os.Args = maybeTransformArgs(os.Args)
+ cmd.Root.AddCommand(command)
+}
+
+// maybeTransformArgs returns a modified version of `args` with the "gitannex"
+// subcommand inserted when `args` indicates that the program was executed as
+// "git-annex-remote-rclone-builtin". One way this can happen is when rclone is
+// invoked via symlink. Otherwise, returns `args`.
+func maybeTransformArgs(args []string) []string {
+ if len(args) == 0 || filepath.Base(args[0]) != uniqueCommandName {
+ return args
+ }
+ newArgs := make([]string, 0, len(args)+1)
+ newArgs = append(newArgs, args[0])
+ newArgs = append(newArgs, subcommandName)
+ newArgs = append(newArgs, args[1:]...)
+ return newArgs
+}
+
+// messageParser helps parse messages we receive from git-annex into a sequence
+// of parameters. Messages are not quite trivial to parse because they are
+// separated by spaces, but the final parameter may itself contain spaces.
+//
+// This abstraction is necessary because simply splitting on space doesn't cut
+// it. Also, we cannot know how many parameters to parse until we've parsed the
+// first parameter.
+type messageParser struct {
+ line string
+}
+
+// nextSpaceDelimitedParameter consumes the next space-delimited parameter.
+func (m *messageParser) nextSpaceDelimitedParameter() (string, error) {
+ m.line = strings.TrimRight(m.line, "\r\n")
+ if len(m.line) == 0 {
+ return "", errors.New("nothing remains to parse")
+ }
+
+ before, after, found := strings.Cut(m.line, " ")
+ if found {
+ if len(before) == 0 {
+ return "", fmt.Errorf("found an empty space-delimited parameter in line: %q", m.line)
+ }
+ m.line = after
+ return before, nil
+ }
+
+ remaining := m.line
+ m.line = ""
+ return remaining, nil
+}
+
+// finalParameter consumes the final parameter, which may contain spaces.
+func (m *messageParser) finalParameter() (string, error) {
+ m.line = strings.TrimRight(m.line, "\r\n")
+ if len(m.line) == 0 {
+ return "", errors.New("nothing remains to parse")
+ }
+
+ param := m.line
+ m.line = ""
+ return param, nil
+}
+
+// configDefinition describes a configuration value required by this command. We
+// use "GETCONFIG" messages to query git-annex for these values at runtime.
+type configDefinition struct {
+ name string
+ description string
+ destination *string
+}
+
+// server contains this command's current state.
+type server struct {
+ reader *bufio.Reader
+ writer io.Writer
+
+ // When true, the server prints a transcript of messages sent and received
+ // to stderr.
+ verbose bool
+
+ extensionInfo bool
+ extensionAsync bool
+ extensionGetGitRemoteName bool
+ extensionUnavailableResponse bool
+
+ configsDone bool
+ configPrefix string
+ configRcloneRemoteName string
+}
+
+func (s *server) sendMsg(msg string) {
+ msg = msg + "\n"
+ if _, err := io.WriteString(s.writer, msg); err != nil {
+ panic(err)
+ }
+ if s.verbose {
+ _, err := os.Stderr.WriteString(fmt.Sprintf("server sent %q\n", msg))
+ if err != nil {
+ panic(fmt.Errorf("failed to write verbose message to stderr: %w", err))
+ }
+ }
+}
+
+func (s *server) getMsg() (*messageParser, error) {
+ msg, err := s.reader.ReadString('\n')
+ if err != nil {
+ if len(msg) == 0 {
+ // Git-annex closes stdin when it is done with us, so failing to
+ // read a new line is not an error.
+ return nil, nil
+ }
+ return nil, fmt.Errorf("expected message to end with newline: %q", msg)
+ }
+ if s.verbose {
+ _, err := os.Stderr.WriteString(fmt.Sprintf("server received %q\n", msg))
+ if err != nil {
+ return nil, fmt.Errorf("failed to write verbose message to stderr: %w", err)
+ }
+ }
+ return &messageParser{msg}, nil
+}
+
+func (s *server) run() error {
+ // The remote sends the first message.
+ s.sendMsg("VERSION 2")
+
+ for {
+ message, err := s.getMsg()
+ if err != nil {
+ return fmt.Errorf("error receiving message: %w", err)
+ }
+
+ if message == nil {
+ break
+ }
+
+ command, err := message.nextSpaceDelimitedParameter()
+ if err != nil {
+ return fmt.Errorf("failed to parse command")
+ }
+
+ switch command {
+ //
+ // Git-annex requires that these requests are supported.
+ //
+ case "INITREMOTE":
+ err = s.handleInitRemote()
+ case "PREPARE":
+ err = s.handlePrepare()
+ case "EXPORTSUPPORTED":
+ // Indicate that we do not support exports.
+ s.sendMsg("EXPORTSUPPORTED-FAILURE")
+ case "TRANSFER":
+ err = s.handleTransfer(message)
+ case "CHECKPRESENT":
+ err = s.handleCheckPresent(message)
+ case "REMOVE":
+ err = s.handleRemove(message)
+ case "ERROR":
+ errorMessage, parseErr := message.finalParameter()
+ if parseErr != nil {
+ err = fmt.Errorf("error while parsing ERROR message from git-annex: %w", parseErr)
+ break
+ }
+ err = fmt.Errorf("received error message from git-annex: %s", errorMessage)
+
+ //
+ // These requests are optional.
+ //
+ case "EXTENSIONS":
+ // Git-annex just told us which protocol extensions it supports.
+ // Respond with the list of extensions that we want to use (none).
+ err = s.handleExtensions(message)
+ case "LISTCONFIGS":
+ s.handleListConfigs()
+ case "GETCOST":
+ // Git-annex wants to know the "cost" of using this remote. It
+ // probably depends on the backend we will be using, but let's just
+ // consider this an "expensive remote" per git-annex's
+ // Config/Cost.hs.
+ s.sendMsg("COST 200")
+ case "GETAVAILABILITY":
+ // Indicate that this is a cloud service.
+ s.sendMsg("AVAILABILITY GLOBAL")
+ case "CLAIMURL", "CHECKURL", "WHEREIS", "GETINFO":
+ s.sendMsg("UNSUPPORTED-REQUEST")
+ default:
+ err = fmt.Errorf("received unexpected message from git-annex: %s", message.line)
+ }
+ if err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+// Idempotently handle an incoming INITREMOTE message. This should perform
+// one-time setup operations, but we may receive the command again, e.g. when
+// this git-annex remote is initialized in a different repository.
+func (s *server) handleInitRemote() error {
+ if err := s.queryConfigs(); err != nil {
+ return fmt.Errorf("failed to get configs: %w", err)
+ }
+
+ remoteRootFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:", s.configRcloneRemoteName))
+ if err != nil {
+ s.sendMsg("INITREMOTE-FAILURE failed to open root directory of rclone remote")
+ return fmt.Errorf("failed to open root directory of rclone remote: %w", err)
+ }
+
+ if !remoteRootFs.Features().CanHaveEmptyDirectories {
+ s.sendMsg("INITREMOTE-FAILURE this rclone remote does not support empty directories")
+ return fmt.Errorf("rclone remote does not support empty directories")
+ }
+
+ if err := operations.Mkdir(context.TODO(), remoteRootFs, s.configPrefix); err != nil {
+ s.sendMsg("INITREMOTE-FAILURE failed to mkdir")
+ return fmt.Errorf("failed to mkdir: %w", err)
+ }
+
+ s.sendMsg("INITREMOTE-SUCCESS")
+ return nil
+}
+
+// Get a list of configs with pointers to fields of `s`.
+func (s *server) getRequiredConfigs() []configDefinition {
+ return []configDefinition{
+ {
+ "rcloneremotename",
+ "Name of the rclone remote to use. " +
+ "Must match a remote known to rclone. " +
+ "(Note that rclone remotes are a distinct concept from git-annex remotes.)",
+ &s.configRcloneRemoteName,
+ },
+ {
+ "rcloneprefix",
+ "Directory where rclone will write git-annex content. " +
+ "If not specified, defaults to \"git-annex-rclone\". " +
+ "This directory be created on init if it does not exist.",
+ &s.configPrefix,
+ },
+ }
+}
+
+// Query git-annex for config values.
+func (s *server) queryConfigs() error {
+ if s.configsDone {
+ return nil
+ }
+
+ // Send a "GETCONFIG" message for each required config and parse git-annex's
+ // "VALUE" response.
+ for _, config := range s.getRequiredConfigs() {
+ s.sendMsg(fmt.Sprintf("GETCONFIG %s", config.name))
+
+ message, err := s.getMsg()
+ if err != nil {
+ return err
+ }
+
+ valueKeyword, err := message.nextSpaceDelimitedParameter()
+ if err != nil || valueKeyword != "VALUE" {
+ return fmt.Errorf("failed to parse config value: %s %s", valueKeyword, message.line)
+ }
+
+ value, err := message.finalParameter()
+ if err != nil || value == "" {
+ return fmt.Errorf("config value of %q must not be empty", config.name)
+ }
+
+ *config.destination = value
+ }
+
+ s.configsDone = true
+ return nil
+}
+
+func (s *server) handlePrepare() error {
+ if err := s.queryConfigs(); err != nil {
+ s.sendMsg("PREPARE-FAILURE Error getting configs")
+ return fmt.Errorf("error getting configs: %w", err)
+ }
+ s.sendMsg("PREPARE-SUCCESS")
+ return nil
+}
+
+// Git-annex is asking us to return the list of settings that we use. Keep this
+// in sync with `handlePrepare()`.
+func (s *server) handleListConfigs() {
+ for _, config := range s.getRequiredConfigs() {
+ s.sendMsg(fmt.Sprintf("CONFIG %s %s", config.name, config.description))
+ }
+ s.sendMsg("CONFIGEND")
+}
+
+func (s *server) handleTransfer(message *messageParser) error {
+ argMode, err := message.nextSpaceDelimitedParameter()
+ if err != nil {
+ s.sendMsg("TRANSFER-FAILURE failed to parse direction")
+ return fmt.Errorf("malformed arguments for TRANSFER: %w", err)
+ }
+ argKey, err := message.nextSpaceDelimitedParameter()
+ if err != nil {
+ s.sendMsg("TRANSFER-FAILURE failed to parse key")
+ return fmt.Errorf("malformed arguments for TRANSFER: %w", err)
+ }
+ argFile, err := message.finalParameter()
+ if err != nil {
+ s.sendMsg("TRANSFER-FAILURE failed to parse file")
+ return fmt.Errorf("malformed arguments for TRANSFER: %w", err)
+ }
+
+ if err := s.queryConfigs(); err != nil {
+ s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to get configs", argMode, argKey))
+ return fmt.Errorf("error getting configs: %w", err)
+ }
+
+ remoteFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:%s", s.configRcloneRemoteName, s.configPrefix))
+ if err != nil {
+ s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to get remote fs", argMode, argKey))
+ return err
+ }
+
+ localDir := filepath.Dir(argFile)
+ localFs, err := cache.Get(context.TODO(), localDir)
+ if err != nil {
+ s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to get local fs", argMode, argKey))
+ return fmt.Errorf("failed to get local fs: %w", err)
+ }
+
+ remoteFileName := argKey
+ localFileName := filepath.Base(argFile)
+
+ switch argMode {
+ case "STORE":
+ err = operations.CopyFile(context.TODO(), remoteFs, localFs, remoteFileName, localFileName)
+ if err != nil {
+ s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to copy file: %s", argMode, argKey, err))
+ return err
+ }
+
+ case "RETRIEVE":
+ err = operations.CopyFile(context.TODO(), localFs, remoteFs, localFileName, remoteFileName)
+ // It is non-fatal when retrieval fails because the file is missing on
+ // the remote.
+ if err == fs.ErrorObjectNotFound {
+ s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s not found", argMode, argKey))
+ return nil
+ }
+ if err != nil {
+ s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to copy file: %s", argMode, argKey, err))
+ return err
+ }
+
+ default:
+ s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s unrecognized mode", argMode, argKey))
+ return fmt.Errorf("received malformed TRANSFER mode: %v", argMode)
+ }
+
+ s.sendMsg(fmt.Sprintf("TRANSFER-SUCCESS %s %s", argMode, argKey))
+ return nil
+}
+
+func (s *server) handleCheckPresent(message *messageParser) error {
+ argKey, err := message.finalParameter()
+ if err != nil {
+ return err
+ }
+
+ if err := s.queryConfigs(); err != nil {
+ s.sendMsg(fmt.Sprintf("CHECKPRESENT-FAILURE %s failed to get configs", argKey))
+ return fmt.Errorf("error getting configs: %s", err)
+ }
+
+ remoteFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:%s", s.configRcloneRemoteName, s.configPrefix))
+ if err != nil {
+ s.sendMsg(fmt.Sprintf("CHECKPRESENT-UNKNOWN %s failed to get remote fs", argKey))
+ return err
+ }
+
+ _, err = remoteFs.NewObject(context.TODO(), argKey)
+ if err == fs.ErrorObjectNotFound {
+ s.sendMsg(fmt.Sprintf("CHECKPRESENT-FAILURE %s", argKey))
+ return nil
+ }
+ if err != nil {
+ s.sendMsg(fmt.Sprintf("CHECKPRESENT-UNKNOWN %s error finding file", argKey))
+ return err
+ }
+
+ s.sendMsg(fmt.Sprintf("CHECKPRESENT-SUCCESS %s", argKey))
+ return nil
+}
+
+func (s *server) handleRemove(message *messageParser) error {
+ argKey, err := message.finalParameter()
+ if err != nil {
+ return err
+ }
+
+ remoteFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:%s", s.configRcloneRemoteName, s.configPrefix))
+ if err != nil {
+ s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s", argKey))
+ return fmt.Errorf("error getting remote fs: %w", err)
+ }
+
+ fileObj, err := remoteFs.NewObject(context.TODO(), argKey)
+ // It is non-fatal when removal fails because the file is missing on the
+ // remote.
+ if errors.Is(err, fs.ErrorObjectNotFound) {
+ s.sendMsg(fmt.Sprintf("REMOVE-SUCCESS %s", argKey))
+ return nil
+ }
+ if err != nil {
+ s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s error getting new fs object: %s", argKey, err))
+ return fmt.Errorf("error getting new fs object: %w", err)
+ }
+ if err := operations.DeleteFile(context.TODO(), fileObj); err != nil {
+ s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s error deleting file", argKey))
+ return fmt.Errorf("error deleting file: %q", argKey)
+ }
+ s.sendMsg(fmt.Sprintf("REMOVE-SUCCESS %s", argKey))
+ return nil
+}
+
+func (s *server) handleExtensions(message *messageParser) error {
+ for {
+ extension, err := message.nextSpaceDelimitedParameter()
+ if err != nil {
+ break
+ }
+ switch extension {
+ case "INFO":
+ s.extensionInfo = true
+ case "ASYNC":
+ s.extensionAsync = true
+ case "GETGITREMOTENAME":
+ s.extensionGetGitRemoteName = true
+ case "UNAVAILABLERESPONSE":
+ s.extensionUnavailableResponse = true
+ }
+ }
+ s.sendMsg("EXTENSIONS")
+ return nil
+}
+
+var command = &cobra.Command{
+ Aliases: []string{uniqueCommandName},
+ Use: subcommandName,
+ Short: "Speaks with git-annex over stdin/stdout.",
+ Long: gitannexHelp,
+ Annotations: map[string]string{
+ "versionIntroduced": "v1.67.0",
+ },
+ Run: func(command *cobra.Command, args []string) {
+ cmd.CheckArgs(0, 0, command, args)
+
+ s := server{
+ reader: bufio.NewReader(os.Stdin),
+ writer: os.Stdout,
+ }
+ err := s.run()
+ if err != nil {
+ s.sendMsg(fmt.Sprintf("ERROR %s", err.Error()))
+ panic(err)
+ }
+ },
+}
diff --git a/cmd/gitannex/gitannex.md b/cmd/gitannex/gitannex.md
new file mode 100644
index 000000000..c36affffe
--- /dev/null
+++ b/cmd/gitannex/gitannex.md
@@ -0,0 +1,38 @@
+Rclone's gitannex subcommand enables git-annex to store and retrieve content
+from an rclone remote. It expects to be run by git-annex, not directly by users.
+It is an "external special remote program" as defined by git-annex.
+
+Installation on Linux
+---------------------
+
+1. Create a symlink and ensure it's on your PATH. For example:
+
+ ln -s "$(realpath rclone)" "$HOME/bin/git-annex-remote-rclone-builtin"
+
+2. Add a new external remote to your git-annex repo.
+
+ The new remote's type should be "rclone-builtin". When git-annex interacts
+ with remotes of this type, it will try to run a command named
+ "git-annex-remote-rclone-builtin", so the symlink from the previous step
+ should be on your PATH.
+
+ The following example creates a new git-annex remote named "MyRemote" that
+ will use the rclone remote named "SomeRcloneRemote". This rclone remote must
+ be configured in your rclone.conf file, wherever that is located on your
+ system. The rcloneprefix value ensures that content is only written into the
+ rclone remote underneath the "git-annex-content" directory.
+
+ git annex initremote MyRemote \
+ type=external \
+ externaltype=rclone-builtin \
+ encryption=none \
+ rcloneremotename=SomeRcloneRemote \
+ rcloneprefix=git-annex-content
+
+3. Before you trust this command with your precious data, be sure to **test the
+ remote**. This command is very new and has not been tested on many rclone
+ backends. Caveat emptor!
+
+ git annex testremote my-rclone-remote
+
+Happy annexing!
diff --git a/cmd/gitannex/gitannex_test.go b/cmd/gitannex/gitannex_test.go
new file mode 100644
index 000000000..5581d400b
--- /dev/null
+++ b/cmd/gitannex/gitannex_test.go
@@ -0,0 +1,969 @@
+package gitannex
+
+import (
+ "bufio"
+ "crypto/sha256"
+ "fmt"
+ "io"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync"
+ "testing"
+
+ // Without this import, the local filesystem backend would be unavailable.
+ // It looks unused, but the act of importing it runs its `init()` function.
+ _ "github.com/rclone/rclone/backend/local"
+
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/cache"
+ "github.com/rclone/rclone/fs/config"
+ "github.com/rclone/rclone/fs/config/configfile"
+ "github.com/rclone/rclone/fstest/mockfs"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestFixArgsForSymlinkIdentity(t *testing.T) {
+ for _, argList := range [][]string{
+ []string{},
+ []string{"foo"},
+ []string{"foo", "bar"},
+ []string{"foo", "bar", "baz"},
+ } {
+ assert.Equal(t, maybeTransformArgs(argList), argList)
+ }
+}
+
+func TestFixArgsForSymlinkCorrectName(t *testing.T) {
+ assert.Equal(t,
+ maybeTransformArgs([]string{"git-annex-remote-rclone-builtin"}),
+ []string{"git-annex-remote-rclone-builtin", "gitannex"})
+ assert.Equal(t,
+ maybeTransformArgs([]string{"/path/to/git-annex-remote-rclone-builtin"}),
+ []string{"/path/to/git-annex-remote-rclone-builtin", "gitannex"})
+}
+
+type messageParserTestCase struct {
+ label string
+ testFunc func(*testing.T)
+}
+
+var messageParserTestCases = []messageParserTestCase{
+ {
+ "OneParam",
+ func(t *testing.T) {
+ m := messageParser{"foo\n"}
+
+ param, err := m.nextSpaceDelimitedParameter()
+ assert.NoError(t, err)
+ assert.Equal(t, param, "foo")
+
+ param, err = m.nextSpaceDelimitedParameter()
+ assert.Error(t, err)
+ assert.Equal(t, param, "")
+
+ param, err = m.finalParameter()
+ assert.Error(t, err)
+ assert.Equal(t, param, "")
+
+ param, err = m.finalParameter()
+ assert.Error(t, err)
+ assert.Equal(t, param, "")
+
+ param, err = m.nextSpaceDelimitedParameter()
+ assert.Error(t, err)
+ assert.Equal(t, param, "")
+
+ },
+ },
+ {
+ "TwoParams",
+ func(t *testing.T) {
+ m := messageParser{"foo bar\n"}
+
+ param, err := m.nextSpaceDelimitedParameter()
+ assert.NoError(t, err)
+ assert.Equal(t, param, "foo")
+
+ param, err = m.nextSpaceDelimitedParameter()
+ assert.NoError(t, err)
+ assert.Equal(t, param, "bar")
+
+ param, err = m.nextSpaceDelimitedParameter()
+ assert.Error(t, err)
+ assert.Equal(t, param, "")
+
+ param, err = m.finalParameter()
+ assert.Error(t, err)
+ assert.Equal(t, param, "")
+ },
+ },
+ {
+ "TwoParamsNoTrailingNewline",
+
+ func(t *testing.T) {
+ m := messageParser{"foo bar"}
+
+ param, err := m.nextSpaceDelimitedParameter()
+ assert.NoError(t, err)
+ assert.Equal(t, param, "foo")
+
+ param, err = m.nextSpaceDelimitedParameter()
+ assert.NoError(t, err)
+ assert.Equal(t, param, "bar")
+
+ param, err = m.nextSpaceDelimitedParameter()
+ assert.Error(t, err)
+ assert.Equal(t, param, "")
+
+ param, err = m.finalParameter()
+ assert.Error(t, err)
+ assert.Equal(t, param, "")
+ },
+ },
+ {
+ "ThreeParamsWhereFinalParamContainsSpaces",
+ func(t *testing.T) {
+ m := messageParser{"firstparam secondparam final param with spaces"}
+
+ param, err := m.nextSpaceDelimitedParameter()
+ assert.NoError(t, err)
+ assert.Equal(t, param, "firstparam")
+
+ param, err = m.nextSpaceDelimitedParameter()
+ assert.NoError(t, err)
+ assert.Equal(t, param, "secondparam")
+
+ param, err = m.finalParameter()
+ assert.NoError(t, err)
+ assert.Equal(t, param, "final param with spaces")
+ },
+ },
+ {
+ "OneLongFinalParameter",
+ func(t *testing.T) {
+ for _, lineEnding := range []string{"", "\n", "\r", "\r\n", "\n\r"} {
+ lineEnding := lineEnding
+ testName := fmt.Sprintf("lineEnding%x", lineEnding)
+
+ t.Run(testName, func(t *testing.T) {
+ m := messageParser{"one long final parameter" + lineEnding}
+
+ param, err := m.finalParameter()
+ assert.NoError(t, err)
+ assert.Equal(t, param, "one long final parameter")
+
+ param, err = m.finalParameter()
+ assert.Error(t, err)
+ assert.Equal(t, param, "")
+ })
+
+ }
+ },
+ },
+ {
+ "MultipleSpaces",
+ func(t *testing.T) {
+ m := messageParser{"foo bar\n\r"}
+
+ param, err := m.nextSpaceDelimitedParameter()
+ assert.NoError(t, err)
+ assert.Equal(t, param, "foo")
+
+ param, err = m.nextSpaceDelimitedParameter()
+ assert.Error(t, err, "blah")
+ assert.Equal(t, param, "")
+ },
+ },
+ {
+ "StartsWithSpace",
+ func(t *testing.T) {
+ m := messageParser{" foo"}
+
+ param, err := m.nextSpaceDelimitedParameter()
+ assert.Error(t, err, "blah")
+ assert.Equal(t, param, "")
+ },
+ },
+}
+
+func TestMessageParser(t *testing.T) {
+ for _, testCase := range messageParserTestCases {
+ testCase := testCase
+ t.Run(testCase.label, func(t *testing.T) {
+ t.Parallel()
+ testCase.testFunc(t)
+ })
+ }
+}
+
+type testState struct {
+ t *testing.T
+ server *server
+ mockStdinW *io.PipeWriter
+ mockStdoutReader *bufio.Reader
+
+ localFsDir string
+ configPath string
+ remoteName string
+}
+
+func makeTestState(t *testing.T) testState {
+ stdinR, stdinW := io.Pipe()
+ stdoutR, stdoutW := io.Pipe()
+
+ return testState{
+ t: t,
+ server: &server{
+ reader: bufio.NewReader(stdinR),
+ writer: stdoutW,
+ },
+ mockStdinW: stdinW,
+ mockStdoutReader: bufio.NewReader(stdoutR),
+ }
+}
+
+func (h *testState) requireReadLineExact(line string) {
+ receivedLine, err := h.mockStdoutReader.ReadString('\n')
+ require.NoError(h.t, err)
+ require.Equal(h.t, line+"\n", receivedLine)
+}
+
+func (h *testState) requireWriteLine(line string) {
+ _, err := h.mockStdinW.Write([]byte(line + "\n"))
+ require.NoError(h.t, err)
+}
+
+// Preconfigure the handle. This enables the calling test to skip the PREPARE
+// handshake.
+func (h *testState) preconfigureServer() {
+ h.server.configPrefix = h.localFsDir
+ h.server.configRcloneRemoteName = h.remoteName
+ h.server.configsDone = true
+}
+
+// getUniqueRemoteName returns a valid remote name derived from the given test's
+// name. This is necessary because when a test registers a second remote with
+// the same name, the original remote appears to take precedence. This function
+// is injective, so each test gets a unique remote name. Returned strings
+// contain no spaces.
+func getUniqueRemoteName(t *testing.T) string {
+ // Using sha256 as a hack to ensure injectivity without adding a global
+ // variable.
+ return fmt.Sprintf("remote-%x", sha256.Sum256([]byte(t.Name())))
+}
+
+type testCase struct {
+ label string
+ testProtocolFunc func(*testing.T, *testState)
+ expectedError string
+}
+
+// These test cases run against the "local" backend.
+var localBackendTestCases = []testCase{
+ {
+ label: "HandlesInit",
+ testProtocolFunc: func(t *testing.T, h *testState) {
+ h.preconfigureServer()
+
+ h.requireReadLineExact("VERSION 2")
+ h.requireWriteLine("INITREMOTE")
+ h.requireReadLineExact("INITREMOTE-SUCCESS")
+
+ require.NoError(t, h.mockStdinW.Close())
+ },
+ },
+ {
+ label: "HandlesPrepare",
+ testProtocolFunc: func(t *testing.T, h *testState) {
+ h.requireReadLineExact("VERSION 2")
+ h.requireWriteLine("EXTENSIONS INFO") // Advertise that we support the INFO extension
+ h.requireReadLineExact("EXTENSIONS")
+
+ if !h.server.extensionInfo {
+ t.Errorf("expected INFO extension to be enabled")
+ return
+ }
+
+ h.requireWriteLine("PREPARE")
+ h.requireReadLineExact("GETCONFIG rcloneremotename")
+ h.requireWriteLine("VALUE " + h.remoteName)
+ h.requireReadLineExact("GETCONFIG rcloneprefix")
+ h.requireWriteLine("VALUE " + h.localFsDir)
+ h.requireReadLineExact("PREPARE-SUCCESS")
+
+ require.Equal(t, h.server.configRcloneRemoteName, h.remoteName)
+ require.Equal(t, h.server.configPrefix, h.localFsDir)
+ require.True(t, h.server.configsDone)
+
+ require.NoError(t, h.mockStdinW.Close())
+ },
+ },
+ {
+ label: "HandlesPrepareAndDoesNotTrimWhitespaceFromValue",
+ testProtocolFunc: func(t *testing.T, h *testState) {
+ h.requireReadLineExact("VERSION 2")
+ h.requireWriteLine("EXTENSIONS INFO") // Advertise that we support the INFO extension
+ h.requireReadLineExact("EXTENSIONS")
+
+ if !h.server.extensionInfo {
+ t.Errorf("expected INFO extension to be enabled")
+ return
+ }
+
+ h.requireWriteLine("PREPARE")
+ h.requireReadLineExact("GETCONFIG rcloneremotename")
+
+ remoteNameWithSpaces := fmt.Sprintf(" %s ", h.remoteName)
+ localFsDirWithSpaces := fmt.Sprintf(" %s\t", h.localFsDir)
+
+ h.requireWriteLine(fmt.Sprintf("VALUE %s", remoteNameWithSpaces))
+ h.requireReadLineExact("GETCONFIG rcloneprefix")
+
+ h.requireWriteLine(fmt.Sprintf("VALUE %s", localFsDirWithSpaces))
+ h.requireReadLineExact("PREPARE-SUCCESS")
+
+ require.Equal(t, h.server.configRcloneRemoteName, remoteNameWithSpaces)
+ require.Equal(t, h.server.configPrefix, localFsDirWithSpaces)
+ require.True(t, h.server.configsDone)
+
+ require.NoError(t, h.mockStdinW.Close())
+ },
+ },
+ {
+ label: "HandlesEarlyError",
+ testProtocolFunc: func(t *testing.T, h *testState) {
+ h.preconfigureServer()
+
+ h.requireReadLineExact("VERSION 2")
+ h.requireWriteLine("ERROR foo")
+
+ require.NoError(t, h.mockStdinW.Close())
+ },
+ expectedError: "received error message from git-annex: foo",
+ },
+ // Test what happens when the git-annex client sends "GETCONFIG", but
+ // doesn't understand git-annex's response.
+ {
+ label: "ConfigFail",
+ testProtocolFunc: func(t *testing.T, h *testState) {
+ h.requireReadLineExact("VERSION 2")
+ h.requireWriteLine("EXTENSIONS INFO") // Advertise that we support the INFO extension
+ h.requireReadLineExact("EXTENSIONS")
+ require.True(t, h.server.extensionInfo)
+
+ h.requireWriteLine("PREPARE")
+ h.requireReadLineExact("GETCONFIG rcloneremotename")
+ h.requireWriteLine("ERROR ineffable error")
+ h.requireReadLineExact("PREPARE-FAILURE Error getting configs")
+
+ require.NoError(t, h.mockStdinW.Close())
+ },
+ expectedError: "failed to parse config value: ERROR ineffable error",
+ },
+ {
+ label: "TransferStoreEmptyPath",
+ testProtocolFunc: func(t *testing.T, h *testState) {
+ h.preconfigureServer()
+
+ h.requireReadLineExact("VERSION 2")
+ h.requireWriteLine("INITREMOTE")
+ h.requireReadLineExact("INITREMOTE-SUCCESS")
+
+ // Note the whitespace following the key.
+ h.requireWriteLine("TRANSFER STORE Key ")
+ h.requireReadLineExact("TRANSFER-FAILURE failed to parse file")
+
+ require.NoError(t, h.mockStdinW.Close())
+ },
+ expectedError: "malformed arguments for TRANSFER: nothing remains to parse",
+ },
+ // Repeated EXTENSIONS messages add to each other rather than overriding
+ // prior advertised extensions. This behavior is not mandated by the
+ // protocol design.
+ {
+ label: "ExtensionsCompound",
+ testProtocolFunc: func(t *testing.T, h *testState) {
+ h.preconfigureServer()
+
+ h.requireReadLineExact("VERSION 2")
+ h.requireWriteLine("INITREMOTE")
+ h.requireReadLineExact("INITREMOTE-SUCCESS")
+
+ h.requireWriteLine("EXTENSIONS")
+ h.requireReadLineExact("EXTENSIONS")
+ require.False(t, h.server.extensionInfo)
+ require.False(t, h.server.extensionAsync)
+ require.False(t, h.server.extensionGetGitRemoteName)
+ require.False(t, h.server.extensionUnavailableResponse)
+
+ h.requireWriteLine("EXTENSIONS INFO")
+ h.requireReadLineExact("EXTENSIONS")
+ require.True(t, h.server.extensionInfo)
+ require.False(t, h.server.extensionAsync)
+ require.False(t, h.server.extensionGetGitRemoteName)
+ require.False(t, h.server.extensionUnavailableResponse)
+
+ h.requireWriteLine("EXTENSIONS ASYNC")
+ h.requireReadLineExact("EXTENSIONS")
+ require.True(t, h.server.extensionInfo)
+ require.True(t, h.server.extensionAsync)
+ require.False(t, h.server.extensionGetGitRemoteName)
+ require.False(t, h.server.extensionUnavailableResponse)
+
+ h.requireWriteLine("EXTENSIONS GETGITREMOTENAME")
+ h.requireReadLineExact("EXTENSIONS")
+ require.True(t, h.server.extensionInfo)
+ require.True(t, h.server.extensionAsync)
+ require.True(t, h.server.extensionGetGitRemoteName)
+ require.False(t, h.server.extensionUnavailableResponse)
+
+ h.requireWriteLine("EXTENSIONS UNAVAILABLERESPONSE")
+ h.requireReadLineExact("EXTENSIONS")
+ require.True(t, h.server.extensionInfo)
+ require.True(t, h.server.extensionAsync)
+ require.True(t, h.server.extensionGetGitRemoteName)
+ require.True(t, h.server.extensionUnavailableResponse)
+
+ require.NoError(t, h.mockStdinW.Close())
+ },
+ },
+ {
+ label: "ExtensionsIdempotent",
+ testProtocolFunc: func(t *testing.T, h *testState) {
+ h.preconfigureServer()
+
+ h.requireReadLineExact("VERSION 2")
+ h.requireWriteLine("INITREMOTE")
+ h.requireReadLineExact("INITREMOTE-SUCCESS")
+
+ h.requireWriteLine("EXTENSIONS")
+ h.requireReadLineExact("EXTENSIONS")
+ require.False(t, h.server.extensionInfo)
+ require.False(t, h.server.extensionAsync)
+ require.False(t, h.server.extensionGetGitRemoteName)
+ require.False(t, h.server.extensionUnavailableResponse)
+
+ h.requireWriteLine("EXTENSIONS")
+ h.requireReadLineExact("EXTENSIONS")
+ require.False(t, h.server.extensionInfo)
+ require.False(t, h.server.extensionAsync)
+ require.False(t, h.server.extensionGetGitRemoteName)
+ require.False(t, h.server.extensionUnavailableResponse)
+
+ h.requireWriteLine("EXTENSIONS INFO")
+ h.requireReadLineExact("EXTENSIONS")
+ require.True(t, h.server.extensionInfo)
+ require.False(t, h.server.extensionAsync)
+ require.False(t, h.server.extensionGetGitRemoteName)
+ require.False(t, h.server.extensionUnavailableResponse)
+
+ h.requireWriteLine("EXTENSIONS INFO")
+ h.requireReadLineExact("EXTENSIONS")
+ require.True(t, h.server.extensionInfo)
+ require.False(t, h.server.extensionAsync)
+ require.False(t, h.server.extensionGetGitRemoteName)
+ require.False(t, h.server.extensionUnavailableResponse)
+
+ h.requireWriteLine("EXTENSIONS ASYNC ASYNC")
+ h.requireReadLineExact("EXTENSIONS")
+ require.True(t, h.server.extensionInfo)
+ require.True(t, h.server.extensionAsync)
+ require.False(t, h.server.extensionGetGitRemoteName)
+ require.False(t, h.server.extensionUnavailableResponse)
+
+ require.NoError(t, h.mockStdinW.Close())
+ },
+ },
+ {
+ label: "ExtensionsSupportsMultiple",
+ testProtocolFunc: func(t *testing.T, h *testState) {
+ h.preconfigureServer()
+
+ h.requireReadLineExact("VERSION 2")
+ h.requireWriteLine("INITREMOTE")
+ h.requireReadLineExact("INITREMOTE-SUCCESS")
+
+ h.requireWriteLine("EXTENSIONS")
+ h.requireReadLineExact("EXTENSIONS")
+ require.False(t, h.server.extensionInfo)
+ require.False(t, h.server.extensionAsync)
+ require.False(t, h.server.extensionGetGitRemoteName)
+ require.False(t, h.server.extensionUnavailableResponse)
+
+ h.requireWriteLine("EXTENSIONS INFO ASYNC")
+ h.requireReadLineExact("EXTENSIONS")
+ require.True(t, h.server.extensionInfo)
+ require.True(t, h.server.extensionAsync)
+ require.False(t, h.server.extensionGetGitRemoteName)
+ require.False(t, h.server.extensionUnavailableResponse)
+
+ require.NoError(t, h.mockStdinW.Close())
+ },
+ },
+ {
+ label: "TransferStoreAbsolute",
+ testProtocolFunc: func(t *testing.T, h *testState) {
+ h.preconfigureServer()
+
+ h.requireReadLineExact("VERSION 2")
+ h.requireWriteLine("INITREMOTE")
+ h.requireReadLineExact("INITREMOTE-SUCCESS")
+
+ // Create temp file for transfer with an absolute path.
+ fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
+ require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
+ require.FileExists(t, fileToTransfer)
+ require.True(t, filepath.IsAbs(fileToTransfer))
+
+ // Specify an absolute path to transfer.
+ h.requireWriteLine("TRANSFER STORE KeyAbsolute " + fileToTransfer)
+ h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyAbsolute")
+ require.FileExists(t, filepath.Join(h.localFsDir, "KeyAbsolute"))
+
+ // Transfer the same absolute path a second time, but with a different key.
+ h.requireWriteLine("TRANSFER STORE KeyAbsolute2 " + fileToTransfer)
+ h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyAbsolute2")
+ require.FileExists(t, filepath.Join(h.localFsDir, "KeyAbsolute2"))
+
+ h.requireWriteLine("CHECKPRESENT KeyAbsolute2")
+ h.requireReadLineExact("CHECKPRESENT-SUCCESS KeyAbsolute2")
+
+ h.requireWriteLine("CHECKPRESENT KeyThatDoesNotExist")
+ h.requireReadLineExact("CHECKPRESENT-FAILURE KeyThatDoesNotExist")
+
+ require.NoError(t, h.mockStdinW.Close())
+ },
+ },
+ // Test that the TRANSFER command understands simple relative paths
+ // consisting only of a file name.
+ {
+ label: "TransferStoreRelative",
+ testProtocolFunc: func(t *testing.T, h *testState) {
+ h.preconfigureServer()
+
+ // Save the current working directory so we can restore it when this
+ // test ends.
+ cwd, err := os.Getwd()
+ require.NoError(t, err)
+
+ require.NoError(t, os.Chdir(t.TempDir()))
+ t.Cleanup(func() { require.NoError(t, os.Chdir(cwd)) })
+
+ h.requireReadLineExact("VERSION 2")
+ h.requireWriteLine("INITREMOTE")
+ h.requireReadLineExact("INITREMOTE-SUCCESS")
+
+ // Create temp file for transfer with a relative path.
+ fileToTransfer := "file.txt"
+ require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
+ require.FileExists(t, fileToTransfer)
+ require.False(t, filepath.IsAbs(fileToTransfer))
+
+ // Specify a relative path to transfer.
+ h.requireWriteLine("TRANSFER STORE KeyRelative " + fileToTransfer)
+ h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyRelative")
+ require.FileExists(t, filepath.Join(h.localFsDir, "KeyRelative"))
+
+ h.requireWriteLine("CHECKPRESENT KeyRelative")
+ h.requireReadLineExact("CHECKPRESENT-SUCCESS KeyRelative")
+
+ h.requireWriteLine("CHECKPRESENT KeyThatDoesNotExist")
+ h.requireReadLineExact("CHECKPRESENT-FAILURE KeyThatDoesNotExist")
+
+ require.NoError(t, h.mockStdinW.Close())
+ },
+ },
+ {
+ label: "TransferStorePathWithInteriorWhitespace",
+ testProtocolFunc: func(t *testing.T, h *testState) {
+ // Save the current working directory so we can restore it when this
+ // test ends.
+ cwd, err := os.Getwd()
+ require.NoError(t, err)
+
+ require.NoError(t, os.Chdir(t.TempDir()))
+ t.Cleanup(func() { require.NoError(t, os.Chdir(cwd)) })
+
+ h.preconfigureServer()
+
+ h.requireReadLineExact("VERSION 2")
+ h.requireWriteLine("INITREMOTE")
+ h.requireReadLineExact("INITREMOTE-SUCCESS")
+
+ // Create temp file for transfer.
+ fileToTransfer := "filename with spaces.txt"
+ require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
+ require.FileExists(t, fileToTransfer)
+ require.False(t, filepath.IsAbs(fileToTransfer))
+
+ // Specify a relative path to transfer.
+ h.requireWriteLine("TRANSFER STORE KeyRelative " + fileToTransfer)
+ h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyRelative")
+ require.FileExists(t, filepath.Join(h.localFsDir, "KeyRelative"))
+
+ h.requireWriteLine("CHECKPRESENT KeyRelative")
+ h.requireReadLineExact("CHECKPRESENT-SUCCESS KeyRelative")
+
+ h.requireWriteLine("CHECKPRESENT KeyThatDoesNotExist")
+ h.requireReadLineExact("CHECKPRESENT-FAILURE KeyThatDoesNotExist")
+
+ require.NoError(t, h.mockStdinW.Close())
+ },
+ },
+ {
+ label: "CheckPresentAndTransfer",
+ testProtocolFunc: func(t *testing.T, h *testState) {
+ h.preconfigureServer()
+
+ // Create temp file for transfer.
+ fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
+ require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
+
+ h.requireReadLineExact("VERSION 2")
+ h.requireWriteLine("INITREMOTE")
+ h.requireReadLineExact("INITREMOTE-SUCCESS")
+
+ h.requireWriteLine("CHECKPRESENT KeyThatDoesNotExist")
+ h.requireReadLineExact("CHECKPRESENT-FAILURE KeyThatDoesNotExist")
+
+ // Specify an absolute path to transfer.
+ require.True(t, filepath.IsAbs(fileToTransfer))
+ h.requireWriteLine("TRANSFER STORE KeyAbsolute " + fileToTransfer)
+ h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyAbsolute")
+ require.FileExists(t, filepath.Join(h.localFsDir, "KeyAbsolute"))
+
+ require.NoError(t, h.mockStdinW.Close())
+ },
+ },
+ // Check whether a key is present, transfer a file with that key, then check
+ // again whether it is present.
+ //
+ // This is a regression test for a bug where the second CHECKPRESENT would
+ // generate the following response:
+ //
+ // CHECKPRESENT-UNKNOWN ${key} failed to read directory entry: readdirent ${filepath}: not a directory
+ //
+ // This message was generated by the local backend's `List()` function. When
+ // checking whether a file exists, we were erroneously listing its contents as
+ // if it were a directory.
+ {
+ label: "CheckpresentTransferCheckpresent",
+ testProtocolFunc: func(t *testing.T, h *testState) {
+ h.preconfigureServer()
+
+ // Create temp file for transfer.
+ fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
+ require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
+
+ h.requireReadLineExact("VERSION 2")
+ h.requireWriteLine("INITREMOTE")
+ h.requireReadLineExact("INITREMOTE-SUCCESS")
+
+ h.requireWriteLine("CHECKPRESENT foo")
+ h.requireReadLineExact("CHECKPRESENT-FAILURE foo")
+
+ h.requireWriteLine("TRANSFER STORE foo " + fileToTransfer)
+ h.requireReadLineExact("TRANSFER-SUCCESS STORE foo")
+ require.FileExists(t, filepath.Join(h.localFsDir, "foo"))
+
+ h.requireWriteLine("CHECKPRESENT foo")
+ h.requireReadLineExact("CHECKPRESENT-SUCCESS foo")
+
+ require.NoError(t, h.mockStdinW.Close())
+ },
+ },
+ {
+ label: "TransferAndCheckpresentWithRealisticKey",
+ testProtocolFunc: func(t *testing.T, h *testState) {
+ h.preconfigureServer()
+
+ // Create temp file for transfer.
+ fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
+ require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
+
+ h.requireReadLineExact("VERSION 2")
+ h.requireWriteLine("INITREMOTE")
+ h.requireReadLineExact("INITREMOTE-SUCCESS")
+
+ realisticKey := "SHA256E-s1048576--7ba87e06b9b7903cfbaf4a38736766c161e3e7b42f06fe57f040aa410a8f0701.this-is-a-test-key"
+
+ // Specify an absolute path to transfer.
+ require.True(t, filepath.IsAbs(fileToTransfer))
+ h.requireWriteLine(fmt.Sprintf("TRANSFER STORE %s %s", realisticKey, fileToTransfer))
+ h.requireReadLineExact("TRANSFER-SUCCESS STORE " + realisticKey)
+ require.FileExists(t, filepath.Join(h.localFsDir, realisticKey))
+
+ h.requireWriteLine("CHECKPRESENT " + realisticKey)
+ h.requireReadLineExact("CHECKPRESENT-SUCCESS " + realisticKey)
+
+ require.NoError(t, h.mockStdinW.Close())
+ },
+ },
+ {
+ label: "RetrieveNonexistentFile",
+ testProtocolFunc: func(t *testing.T, h *testState) {
+ h.preconfigureServer()
+
+ h.requireReadLineExact("VERSION 2")
+ h.requireWriteLine("INITREMOTE")
+ h.requireReadLineExact("INITREMOTE-SUCCESS")
+
+ h.requireWriteLine("TRANSFER RETRIEVE SomeKey path")
+ h.requireReadLineExact("TRANSFER-FAILURE RETRIEVE SomeKey not found")
+
+ require.NoError(t, h.mockStdinW.Close())
+ },
+ },
+ {
+ label: "StoreCheckpresentRetrieve",
+ testProtocolFunc: func(t *testing.T, h *testState) {
+ h.preconfigureServer()
+
+ // Create temp file for transfer.
+ fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
+ require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
+
+ h.requireReadLineExact("VERSION 2")
+ h.requireWriteLine("INITREMOTE")
+ h.requireReadLineExact("INITREMOTE-SUCCESS")
+
+ // Specify an absolute path to transfer.
+ require.True(t, filepath.IsAbs(fileToTransfer))
+ h.requireWriteLine("TRANSFER STORE SomeKey " + fileToTransfer)
+ h.requireReadLineExact("TRANSFER-SUCCESS STORE SomeKey")
+ require.FileExists(t, filepath.Join(h.localFsDir, "SomeKey"))
+
+ h.requireWriteLine("CHECKPRESENT SomeKey")
+ h.requireReadLineExact("CHECKPRESENT-SUCCESS SomeKey")
+
+ retrievedFilePath := fileToTransfer + ".retrieved"
+ require.NoFileExists(t, retrievedFilePath)
+ h.requireWriteLine("TRANSFER RETRIEVE SomeKey " + retrievedFilePath)
+ h.requireReadLineExact("TRANSFER-SUCCESS RETRIEVE SomeKey")
+ require.FileExists(t, retrievedFilePath)
+
+ require.NoError(t, h.mockStdinW.Close())
+ },
+ },
+ {
+ label: "RemovePreexistingFile",
+ testProtocolFunc: func(t *testing.T, h *testState) {
+ h.preconfigureServer()
+
+ // Write a file into the remote without using the git-annex
+ // protocol.
+ remoteFilePath := filepath.Join(h.localFsDir, "SomeKey")
+ require.NoError(t, os.WriteFile(remoteFilePath, []byte("HELLO"), 0600))
+ require.FileExists(t, remoteFilePath)
+
+ h.requireReadLineExact("VERSION 2")
+ h.requireWriteLine("INITREMOTE")
+ h.requireReadLineExact("INITREMOTE-SUCCESS")
+
+ h.requireWriteLine("CHECKPRESENT SomeKey")
+ h.requireReadLineExact("CHECKPRESENT-SUCCESS SomeKey")
+ require.FileExists(t, remoteFilePath)
+
+ h.requireWriteLine("REMOVE SomeKey")
+ h.requireReadLineExact("REMOVE-SUCCESS SomeKey")
+ require.NoFileExists(t, remoteFilePath)
+
+ h.requireWriteLine("CHECKPRESENT SomeKey")
+ h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")
+ require.NoFileExists(t, remoteFilePath)
+
+ require.NoError(t, h.mockStdinW.Close())
+ },
+ },
+ {
+ label: "Remove",
+ testProtocolFunc: func(t *testing.T, h *testState) {
+ h.preconfigureServer()
+
+ // Create temp file for transfer.
+ fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
+ require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
+
+ h.requireReadLineExact("VERSION 2")
+ h.requireWriteLine("INITREMOTE")
+ h.requireReadLineExact("INITREMOTE-SUCCESS")
+
+ h.requireWriteLine("CHECKPRESENT SomeKey")
+ h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")
+
+ // Specify an absolute path to transfer.
+ require.True(t, filepath.IsAbs(fileToTransfer))
+ h.requireWriteLine("TRANSFER STORE SomeKey " + fileToTransfer)
+ h.requireReadLineExact("TRANSFER-SUCCESS STORE SomeKey")
+ require.FileExists(t, filepath.Join(h.localFsDir, "SomeKey"))
+
+ h.requireWriteLine("CHECKPRESENT SomeKey")
+ h.requireReadLineExact("CHECKPRESENT-SUCCESS SomeKey")
+
+ h.requireWriteLine("REMOVE SomeKey")
+ h.requireReadLineExact("REMOVE-SUCCESS SomeKey")
+ require.NoFileExists(t, filepath.Join(h.localFsDir, "SomeKey"))
+
+ h.requireWriteLine("CHECKPRESENT SomeKey")
+ h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")
+
+ require.NoError(t, h.mockStdinW.Close())
+ },
+ },
+ {
+ label: "RemoveNonexistentFile",
+ testProtocolFunc: func(t *testing.T, h *testState) {
+ h.preconfigureServer()
+
+ // Create temp file for transfer.
+ fileToTransfer := filepath.Join(t.TempDir(), "file.txt")
+ require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))
+
+ h.requireReadLineExact("VERSION 2")
+ h.requireWriteLine("INITREMOTE")
+ h.requireReadLineExact("INITREMOTE-SUCCESS")
+
+ h.requireWriteLine("CHECKPRESENT SomeKey")
+ h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")
+
+ require.NoFileExists(t, filepath.Join(h.localFsDir, "SomeKey"))
+ h.requireWriteLine("REMOVE SomeKey")
+ h.requireReadLineExact("REMOVE-SUCCESS SomeKey")
+ require.NoFileExists(t, filepath.Join(h.localFsDir, "SomeKey"))
+
+ h.requireWriteLine("CHECKPRESENT SomeKey")
+ h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")
+
+ require.NoError(t, h.mockStdinW.Close())
+ },
+ },
+ {
+ label: "ExportNotSupported",
+ testProtocolFunc: func(t *testing.T, h *testState) {
+ h.preconfigureServer()
+
+ h.requireReadLineExact("VERSION 2")
+ h.requireWriteLine("INITREMOTE")
+ h.requireReadLineExact("INITREMOTE-SUCCESS")
+
+ h.requireWriteLine("EXPORTSUPPORTED")
+ h.requireReadLineExact("EXPORTSUPPORTED-FAILURE")
+
+ require.NoError(t, h.mockStdinW.Close())
+ },
+ },
+}
+
+func TestGitAnnexLocalBackendCases(t *testing.T) {
+ for _, testCase := range localBackendTestCases {
+ // Clear global state left behind by tests that chdir to a temp directory.
+ cache.Clear()
+
+ // TODO: Remove this when rclone requires a Go version >= 1.22. Future
+ // versions of Go fix the semantics of capturing a range variable.
+ // https://go.dev/blog/loopvar-preview
+ testCase := testCase
+
+ t.Run(testCase.label, func(t *testing.T) {
+ tempDir := t.TempDir()
+
+ // Create temp dir for an rclone remote pointing at local filesystem.
+ localFsDir := filepath.Join(tempDir, "remoteTarget")
+ require.NoError(t, os.Mkdir(localFsDir, 0700))
+
+ // Create temp config
+ remoteName := getUniqueRemoteName(t)
+ configLines := []string{
+ fmt.Sprintf("[%s]", remoteName),
+ "type = local",
+ fmt.Sprintf("remote = %s", localFsDir),
+ }
+ configContents := strings.Join(configLines, "\n")
+
+ configPath := filepath.Join(tempDir, "rclone.conf")
+ require.NoError(t, os.WriteFile(configPath, []byte(configContents), 0600))
+ require.NoError(t, config.SetConfigPath(configPath))
+
+ // The custom config file will be ignored unless we install the
+ // global config file handler.
+ configfile.Install()
+
+ handle := makeTestState(t)
+ handle.localFsDir = localFsDir
+ handle.configPath = configPath
+ handle.remoteName = remoteName
+
+ var wg sync.WaitGroup
+ wg.Add(1)
+
+ go func() {
+ err := handle.server.run()
+
+ if testCase.expectedError == "" {
+ require.NoError(t, err)
+ } else {
+ require.ErrorContains(t, err, testCase.expectedError)
+ }
+
+ wg.Done()
+ }()
+ defer wg.Wait()
+
+ testCase.testProtocolFunc(t, &handle)
+ })
+ }
+}
+
+// Configure the git-annex client with a mockfs backend and send it the
+// "INITREMOTE" command over mocked stdin. This should fail because mockfs does
+// not support empty directories.
+func TestGitAnnexHandleInitRemoteBackendDoesNotSupportEmptyDirectories(t *testing.T) {
+ tempDir := t.TempDir()
+
+ // Temporarily override the filesystem registry.
+ oldRegistry := fs.Registry
+ mockfs.Register()
+ defer func() { fs.Registry = oldRegistry }()
+
+ // Create temp dir for an rclone remote pointing at local filesystem.
+ localFsDir := filepath.Join(tempDir, "remoteTarget")
+ require.NoError(t, os.Mkdir(localFsDir, 0700))
+
+ // Create temp config
+ remoteName := getUniqueRemoteName(t)
+ configLines := []string{
+ fmt.Sprintf("[%s]", remoteName),
+ "type = mockfs",
+ fmt.Sprintf("remote = %s", localFsDir),
+ }
+ configContents := strings.Join(configLines, "\n")
+
+ configPath := filepath.Join(tempDir, "rclone.conf")
+ require.NoError(t, os.WriteFile(configPath, []byte(configContents), 0600))
+
+ // The custom config file will be ignored unless we install the global
+ // config file handler.
+ configfile.Install()
+ require.NoError(t, config.SetConfigPath(configPath))
+
+ handle := makeTestState(t)
+ handle.server.configPrefix = localFsDir
+ handle.server.configRcloneRemoteName = remoteName
+ handle.server.configsDone = true
+
+ var wg sync.WaitGroup
+ wg.Add(1)
+
+ go func() {
+ require.NotNil(t, handle.server.run())
+ wg.Done()
+ }()
+ defer wg.Wait()
+
+ handle.requireReadLineExact("VERSION 2")
+ handle.requireWriteLine("INITREMOTE")
+ handle.requireReadLineExact("INITREMOTE-FAILURE this rclone remote does not support empty directories")
+}
diff --git a/cmd/gitannex/run-git-annex-testremote.sh b/cmd/gitannex/run-git-annex-testremote.sh
new file mode 100755
index 000000000..98a848df7
--- /dev/null
+++ b/cmd/gitannex/run-git-annex-testremote.sh
@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+#
+# End-to-end tests for "rclone gitannex". This script runs the `git-annex
+# testremote` suite against "rclone gitannex" in an ephemeral git-annex repo.
+#
+# Assumptions:
+#
+# * This system has an rclone remote configured named "git-annex-builtin-test-remote".
+#
+# * If it uses rclone's "local" backend, /tmp/git-annex-builtin-test-remote exists.
+
+set -e
+
+TEST_DIR="$(realpath "$(mktemp -d)")"
+mkdir "$TEST_DIR/bin"
+
+function cleanup()
+{
+ rm -rf "$TEST_DIR"
+}
+
+trap cleanup EXIT
+
+RCLONE_DIR="$(git rev-parse --show-toplevel)"
+
+rm -rf /tmp/git-annex-builtin-test-remote/*
+
+set -x
+
+pushd "$RCLONE_DIR"
+go build -o "$TEST_DIR/bin" ./
+
+ln -s "$(realpath "$TEST_DIR/bin/rclone")" "$TEST_DIR/bin/git-annex-remote-rclone-builtin"
+popd
+
+pushd "$TEST_DIR"
+
+git init
+git annex init
+
+REMOTE_NAME=git-annex-builtin-test-remote
+PREFIX=/tmp/git-annex-builtin-test-remote
+
+PATH="$PATH:$TEST_DIR/bin" git annex initremote $REMOTE_NAME \
+ type=external externaltype=rclone-builtin encryption=none \
+ rcloneremotename=$REMOTE_NAME \
+ rcloneprefix="$PREFIX"
+
+PATH="$PATH:$(realpath bin)" git annex testremote $REMOTE_NAME
+
+popd
+rm -rf "$TEST_DIR"