"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
cmd/gitannex: Add the gitannex subcommand
Dan McArdledfc329c03655
diff --git a/cmd/all/all.go b/cmd/all/all.goindex 5fb87ed16..37b15c5c3 100644--- a/cmd/all/all.go+++ b/cmd/all/all.go@@ -25,6 +25,7 @@ import (_ "github.com/rclone/rclone/cmd/deletefile"_ "github.com/rclone/rclone/cmd/genautocomplete"_ "github.com/rclone/rclone/cmd/gendocs"+ _ "github.com/rclone/rclone/cmd/gitannex"_ "github.com/rclone/rclone/cmd/hashsum"_ "github.com/rclone/rclone/cmd/link"_ "github.com/rclone/rclone/cmd/listremotes"diff --git a/cmd/gitannex/gitannex.go b/cmd/gitannex/gitannex.gonew file mode 100644index 000000000..0c085c133--- /dev/null+++ b/cmd/gitannex/gitannex.go@@ -0,0 +1,515 @@+// Package gitannex provides the "gitannex" command, which enables [git-annex]+// to communicate with rclone by implementing the [external special remote+// protocol]. The protocol is line delimited and spoken over stdin and stdout.+//+// # Milestones+//+// (Tracked in [issue #7625].)+//+// 1. ✅ Minimal support for the [external special remote protocol]. Tested on+// "local" and "drive" backends.+// 2. Add support for the ASYNC protocol extension. This may improve performance.+// 3. Support the [simple export interface]. This will enable `git-annex+// export` functionality.+// 4. Once the draft is finalized, support import/export interface.+//+// [git-annex]: https://git-annex.branchable.com/+// [external special remote protocol]: https://git-annex.branchable.com/design/external_special_remote_protocol/+// [simple export interface]: https://git-annex.branchable.com/design/external_special_remote_protocol/export_and_import_appendix/+// [issue #7625]: https://github.com/rclone/rclone/issues/7625+package gitannex++import (+ "bufio"+ "context"+ _ "embed"+ "errors"+ "fmt"+ "io"+ "os"+ "path/filepath"+ "strings"++ "github.com/rclone/rclone/cmd"+ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/cache"+ "github.com/rclone/rclone/fs/operations"+ "github.com/spf13/cobra"+)++const subcommandName string = "gitannex"+const uniqueCommandName string = "git-annex-remote-rclone-builtin"++//go:embed gitannex.md+var gitannexHelp string++func init() {+ os.Args = maybeTransformArgs(os.Args)+ cmd.Root.AddCommand(command)+}++// maybeTransformArgs returns a modified version of `args` with the "gitannex"+// subcommand inserted when `args` indicates that the program was executed as+// "git-annex-remote-rclone-builtin". One way this can happen is when rclone is+// invoked via symlink. Otherwise, returns `args`.+func maybeTransformArgs(args []string) []string {+ if len(args) == 0 || filepath.Base(args[0]) != uniqueCommandName {+ return args+ }+ newArgs := make([]string, 0, len(args)+1)+ newArgs = append(newArgs, args[0])+ newArgs = append(newArgs, subcommandName)+ newArgs = append(newArgs, args[1:]...)+ return newArgs+}++// messageParser helps parse messages we receive from git-annex into a sequence+// of parameters. Messages are not quite trivial to parse because they are+// separated by spaces, but the final parameter may itself contain spaces.+//+// This abstraction is necessary because simply splitting on space doesn't cut+// it. Also, we cannot know how many parameters to parse until we've parsed the+// first parameter.+type messageParser struct {+ line string+}++// nextSpaceDelimitedParameter consumes the next space-delimited parameter.+func (m *messageParser) nextSpaceDelimitedParameter() (string, error) {+ m.line = strings.TrimRight(m.line, "\r\n")+ if len(m.line) == 0 {+ return "", errors.New("nothing remains to parse")+ }++ before, after, found := strings.Cut(m.line, " ")+ if found {+ if len(before) == 0 {+ return "", fmt.Errorf("found an empty space-delimited parameter in line: %q", m.line)+ }+ m.line = after+ return before, nil+ }++ remaining := m.line+ m.line = ""+ return remaining, nil+}++// finalParameter consumes the final parameter, which may contain spaces.+func (m *messageParser) finalParameter() (string, error) {+ m.line = strings.TrimRight(m.line, "\r\n")+ if len(m.line) == 0 {+ return "", errors.New("nothing remains to parse")+ }++ param := m.line+ m.line = ""+ return param, nil+}++// configDefinition describes a configuration value required by this command. We+// use "GETCONFIG" messages to query git-annex for these values at runtime.+type configDefinition struct {+ name string+ description string+ destination *string+}++// server contains this command's current state.+type server struct {+ reader *bufio.Reader+ writer io.Writer++ // When true, the server prints a transcript of messages sent and received+ // to stderr.+ verbose bool++ extensionInfo bool+ extensionAsync bool+ extensionGetGitRemoteName bool+ extensionUnavailableResponse bool++ configsDone bool+ configPrefix string+ configRcloneRemoteName string+}++func (s *server) sendMsg(msg string) {+ msg = msg + "\n"+ if _, err := io.WriteString(s.writer, msg); err != nil {+ panic(err)+ }+ if s.verbose {+ _, err := os.Stderr.WriteString(fmt.Sprintf("server sent %q\n", msg))+ if err != nil {+ panic(fmt.Errorf("failed to write verbose message to stderr: %w", err))+ }+ }+}++func (s *server) getMsg() (*messageParser, error) {+ msg, err := s.reader.ReadString('\n')+ if err != nil {+ if len(msg) == 0 {+ // Git-annex closes stdin when it is done with us, so failing to+ // read a new line is not an error.+ return nil, nil+ }+ return nil, fmt.Errorf("expected message to end with newline: %q", msg)+ }+ if s.verbose {+ _, err := os.Stderr.WriteString(fmt.Sprintf("server received %q\n", msg))+ if err != nil {+ return nil, fmt.Errorf("failed to write verbose message to stderr: %w", err)+ }+ }+ return &messageParser{msg}, nil+}++func (s *server) run() error {+ // The remote sends the first message.+ s.sendMsg("VERSION 2")++ for {+ message, err := s.getMsg()+ if err != nil {+ return fmt.Errorf("error receiving message: %w", err)+ }++ if message == nil {+ break+ }++ command, err := message.nextSpaceDelimitedParameter()+ if err != nil {+ return fmt.Errorf("failed to parse command")+ }++ switch command {+ //+ // Git-annex requires that these requests are supported.+ //+ case "INITREMOTE":+ err = s.handleInitRemote()+ case "PREPARE":+ err = s.handlePrepare()+ case "EXPORTSUPPORTED":+ // Indicate that we do not support exports.+ s.sendMsg("EXPORTSUPPORTED-FAILURE")+ case "TRANSFER":+ err = s.handleTransfer(message)+ case "CHECKPRESENT":+ err = s.handleCheckPresent(message)+ case "REMOVE":+ err = s.handleRemove(message)+ case "ERROR":+ errorMessage, parseErr := message.finalParameter()+ if parseErr != nil {+ err = fmt.Errorf("error while parsing ERROR message from git-annex: %w", parseErr)+ break+ }+ err = fmt.Errorf("received error message from git-annex: %s", errorMessage)++ //+ // These requests are optional.+ //+ case "EXTENSIONS":+ // Git-annex just told us which protocol extensions it supports.+ // Respond with the list of extensions that we want to use (none).+ err = s.handleExtensions(message)+ case "LISTCONFIGS":+ s.handleListConfigs()+ case "GETCOST":+ // Git-annex wants to know the "cost" of using this remote. It+ // probably depends on the backend we will be using, but let's just+ // consider this an "expensive remote" per git-annex's+ // Config/Cost.hs.+ s.sendMsg("COST 200")+ case "GETAVAILABILITY":+ // Indicate that this is a cloud service.+ s.sendMsg("AVAILABILITY GLOBAL")+ case "CLAIMURL", "CHECKURL", "WHEREIS", "GETINFO":+ s.sendMsg("UNSUPPORTED-REQUEST")+ default:+ err = fmt.Errorf("received unexpected message from git-annex: %s", message.line)+ }+ if err != nil {+ return err+ }+ }++ return nil+}++// Idempotently handle an incoming INITREMOTE message. This should perform+// one-time setup operations, but we may receive the command again, e.g. when+// this git-annex remote is initialized in a different repository.+func (s *server) handleInitRemote() error {+ if err := s.queryConfigs(); err != nil {+ return fmt.Errorf("failed to get configs: %w", err)+ }++ remoteRootFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:", s.configRcloneRemoteName))+ if err != nil {+ s.sendMsg("INITREMOTE-FAILURE failed to open root directory of rclone remote")+ return fmt.Errorf("failed to open root directory of rclone remote: %w", err)+ }++ if !remoteRootFs.Features().CanHaveEmptyDirectories {+ s.sendMsg("INITREMOTE-FAILURE this rclone remote does not support empty directories")+ return fmt.Errorf("rclone remote does not support empty directories")+ }++ if err := operations.Mkdir(context.TODO(), remoteRootFs, s.configPrefix); err != nil {+ s.sendMsg("INITREMOTE-FAILURE failed to mkdir")+ return fmt.Errorf("failed to mkdir: %w", err)+ }++ s.sendMsg("INITREMOTE-SUCCESS")+ return nil+}++// Get a list of configs with pointers to fields of `s`.+func (s *server) getRequiredConfigs() []configDefinition {+ return []configDefinition{+ {+ "rcloneremotename",+ "Name of the rclone remote to use. " ++ "Must match a remote known to rclone. " ++ "(Note that rclone remotes are a distinct concept from git-annex remotes.)",+ &s.configRcloneRemoteName,+ },+ {+ "rcloneprefix",+ "Directory where rclone will write git-annex content. " ++ "If not specified, defaults to \"git-annex-rclone\". " ++ "This directory be created on init if it does not exist.",+ &s.configPrefix,+ },+ }+}++// Query git-annex for config values.+func (s *server) queryConfigs() error {+ if s.configsDone {+ return nil+ }++ // Send a "GETCONFIG" message for each required config and parse git-annex's+ // "VALUE" response.+ for _, config := range s.getRequiredConfigs() {+ s.sendMsg(fmt.Sprintf("GETCONFIG %s", config.name))++ message, err := s.getMsg()+ if err != nil {+ return err+ }++ valueKeyword, err := message.nextSpaceDelimitedParameter()+ if err != nil || valueKeyword != "VALUE" {+ return fmt.Errorf("failed to parse config value: %s %s", valueKeyword, message.line)+ }++ value, err := message.finalParameter()+ if err != nil || value == "" {+ return fmt.Errorf("config value of %q must not be empty", config.name)+ }++ *config.destination = value+ }++ s.configsDone = true+ return nil+}++func (s *server) handlePrepare() error {+ if err := s.queryConfigs(); err != nil {+ s.sendMsg("PREPARE-FAILURE Error getting configs")+ return fmt.Errorf("error getting configs: %w", err)+ }+ s.sendMsg("PREPARE-SUCCESS")+ return nil+}++// Git-annex is asking us to return the list of settings that we use. Keep this+// in sync with `handlePrepare()`.+func (s *server) handleListConfigs() {+ for _, config := range s.getRequiredConfigs() {+ s.sendMsg(fmt.Sprintf("CONFIG %s %s", config.name, config.description))+ }+ s.sendMsg("CONFIGEND")+}++func (s *server) handleTransfer(message *messageParser) error {+ argMode, err := message.nextSpaceDelimitedParameter()+ if err != nil {+ s.sendMsg("TRANSFER-FAILURE failed to parse direction")+ return fmt.Errorf("malformed arguments for TRANSFER: %w", err)+ }+ argKey, err := message.nextSpaceDelimitedParameter()+ if err != nil {+ s.sendMsg("TRANSFER-FAILURE failed to parse key")+ return fmt.Errorf("malformed arguments for TRANSFER: %w", err)+ }+ argFile, err := message.finalParameter()+ if err != nil {+ s.sendMsg("TRANSFER-FAILURE failed to parse file")+ return fmt.Errorf("malformed arguments for TRANSFER: %w", err)+ }++ if err := s.queryConfigs(); err != nil {+ s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to get configs", argMode, argKey))+ return fmt.Errorf("error getting configs: %w", err)+ }++ remoteFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:%s", s.configRcloneRemoteName, s.configPrefix))+ if err != nil {+ s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to get remote fs", argMode, argKey))+ return err+ }++ localDir := filepath.Dir(argFile)+ localFs, err := cache.Get(context.TODO(), localDir)+ if err != nil {+ s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to get local fs", argMode, argKey))+ return fmt.Errorf("failed to get local fs: %w", err)+ }++ remoteFileName := argKey+ localFileName := filepath.Base(argFile)++ switch argMode {+ case "STORE":+ err = operations.CopyFile(context.TODO(), remoteFs, localFs, remoteFileName, localFileName)+ if err != nil {+ s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to copy file: %s", argMode, argKey, err))+ return err+ }++ case "RETRIEVE":+ err = operations.CopyFile(context.TODO(), localFs, remoteFs, localFileName, remoteFileName)+ // It is non-fatal when retrieval fails because the file is missing on+ // the remote.+ if err == fs.ErrorObjectNotFound {+ s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s not found", argMode, argKey))+ return nil+ }+ if err != nil {+ s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to copy file: %s", argMode, argKey, err))+ return err+ }++ default:+ s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s unrecognized mode", argMode, argKey))+ return fmt.Errorf("received malformed TRANSFER mode: %v", argMode)+ }++ s.sendMsg(fmt.Sprintf("TRANSFER-SUCCESS %s %s", argMode, argKey))+ return nil+}++func (s *server) handleCheckPresent(message *messageParser) error {+ argKey, err := message.finalParameter()+ if err != nil {+ return err+ }++ if err := s.queryConfigs(); err != nil {+ s.sendMsg(fmt.Sprintf("CHECKPRESENT-FAILURE %s failed to get configs", argKey))+ return fmt.Errorf("error getting configs: %s", err)+ }++ remoteFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:%s", s.configRcloneRemoteName, s.configPrefix))+ if err != nil {+ s.sendMsg(fmt.Sprintf("CHECKPRESENT-UNKNOWN %s failed to get remote fs", argKey))+ return err+ }++ _, err = remoteFs.NewObject(context.TODO(), argKey)+ if err == fs.ErrorObjectNotFound {+ s.sendMsg(fmt.Sprintf("CHECKPRESENT-FAILURE %s", argKey))+ return nil+ }+ if err != nil {+ s.sendMsg(fmt.Sprintf("CHECKPRESENT-UNKNOWN %s error finding file", argKey))+ return err+ }++ s.sendMsg(fmt.Sprintf("CHECKPRESENT-SUCCESS %s", argKey))+ return nil+}++func (s *server) handleRemove(message *messageParser) error {+ argKey, err := message.finalParameter()+ if err != nil {+ return err+ }++ remoteFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:%s", s.configRcloneRemoteName, s.configPrefix))+ if err != nil {+ s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s", argKey))+ return fmt.Errorf("error getting remote fs: %w", err)+ }++ fileObj, err := remoteFs.NewObject(context.TODO(), argKey)+ // It is non-fatal when removal fails because the file is missing on the+ // remote.+ if errors.Is(err, fs.ErrorObjectNotFound) {+ s.sendMsg(fmt.Sprintf("REMOVE-SUCCESS %s", argKey))+ return nil+ }+ if err != nil {+ s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s error getting new fs object: %s", argKey, err))+ return fmt.Errorf("error getting new fs object: %w", err)+ }+ if err := operations.DeleteFile(context.TODO(), fileObj); err != nil {+ s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s error deleting file", argKey))+ return fmt.Errorf("error deleting file: %q", argKey)+ }+ s.sendMsg(fmt.Sprintf("REMOVE-SUCCESS %s", argKey))+ return nil+}++func (s *server) handleExtensions(message *messageParser) error {+ for {+ extension, err := message.nextSpaceDelimitedParameter()+ if err != nil {+ break+ }+ switch extension {+ case "INFO":+ s.extensionInfo = true+ case "ASYNC":+ s.extensionAsync = true+ case "GETGITREMOTENAME":+ s.extensionGetGitRemoteName = true+ case "UNAVAILABLERESPONSE":+ s.extensionUnavailableResponse = true+ }+ }+ s.sendMsg("EXTENSIONS")+ return nil+}++var command = &cobra.Command{+ Aliases: []string{uniqueCommandName},+ Use: subcommandName,+ Short: "Speaks with git-annex over stdin/stdout.",+ Long: gitannexHelp,+ Annotations: map[string]string{+ "versionIntroduced": "v1.67.0",+ },+ Run: func(command *cobra.Command, args []string) {+ cmd.CheckArgs(0, 0, command, args)++ s := server{+ reader: bufio.NewReader(os.Stdin),+ writer: os.Stdout,+ }+ err := s.run()+ if err != nil {+ s.sendMsg(fmt.Sprintf("ERROR %s", err.Error()))+ panic(err)+ }+ },+}diff --git a/cmd/gitannex/gitannex.md b/cmd/gitannex/gitannex.mdnew file mode 100644index 000000000..c36affffe--- /dev/null+++ b/cmd/gitannex/gitannex.md@@ -0,0 +1,38 @@+Rclone's gitannex subcommand enables git-annex to store and retrieve content+from an rclone remote. It expects to be run by git-annex, not directly by users.+It is an "external special remote program" as defined by git-annex.++Installation on Linux+---------------------++1. Create a symlink and ensure it's on your PATH. For example:++ ln -s "$(realpath rclone)" "$HOME/bin/git-annex-remote-rclone-builtin"++2. Add a new external remote to your git-annex repo.++ The new remote's type should be "rclone-builtin". When git-annex interacts+ with remotes of this type, it will try to run a command named+ "git-annex-remote-rclone-builtin", so the symlink from the previous step+ should be on your PATH.++ The following example creates a new git-annex remote named "MyRemote" that+ will use the rclone remote named "SomeRcloneRemote". This rclone remote must+ be configured in your rclone.conf file, wherever that is located on your+ system. The rcloneprefix value ensures that content is only written into the+ rclone remote underneath the "git-annex-content" directory.++ git annex initremote MyRemote \+ type=external \+ externaltype=rclone-builtin \+ encryption=none \+ rcloneremotename=SomeRcloneRemote \+ rcloneprefix=git-annex-content++3. Before you trust this command with your precious data, be sure to **test the+ remote**. This command is very new and has not been tested on many rclone+ backends. Caveat emptor!++ git annex testremote my-rclone-remote++Happy annexing!diff --git a/cmd/gitannex/gitannex_test.go b/cmd/gitannex/gitannex_test.gonew file mode 100644index 000000000..5581d400b--- /dev/null+++ b/cmd/gitannex/gitannex_test.go@@ -0,0 +1,969 @@+package gitannex++import (+ "bufio"+ "crypto/sha256"+ "fmt"+ "io"+ "os"+ "path/filepath"+ "strings"+ "sync"+ "testing"++ // Without this import, the local filesystem backend would be unavailable.+ // It looks unused, but the act of importing it runs its `init()` function.+ _ "github.com/rclone/rclone/backend/local"++ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/cache"+ "github.com/rclone/rclone/fs/config"+ "github.com/rclone/rclone/fs/config/configfile"+ "github.com/rclone/rclone/fstest/mockfs"++ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++func TestFixArgsForSymlinkIdentity(t *testing.T) {+ for _, argList := range [][]string{+ []string{},+ []string{"foo"},+ []string{"foo", "bar"},+ []string{"foo", "bar", "baz"},+ } {+ assert.Equal(t, maybeTransformArgs(argList), argList)+ }+}++func TestFixArgsForSymlinkCorrectName(t *testing.T) {+ assert.Equal(t,+ maybeTransformArgs([]string{"git-annex-remote-rclone-builtin"}),+ []string{"git-annex-remote-rclone-builtin", "gitannex"})+ assert.Equal(t,+ maybeTransformArgs([]string{"/path/to/git-annex-remote-rclone-builtin"}),+ []string{"/path/to/git-annex-remote-rclone-builtin", "gitannex"})+}++type messageParserTestCase struct {+ label string+ testFunc func(*testing.T)+}++var messageParserTestCases = []messageParserTestCase{+ {+ "OneParam",+ func(t *testing.T) {+ m := messageParser{"foo\n"}++ param, err := m.nextSpaceDelimitedParameter()+ assert.NoError(t, err)+ assert.Equal(t, param, "foo")++ param, err = m.nextSpaceDelimitedParameter()+ assert.Error(t, err)+ assert.Equal(t, param, "")++ param, err = m.finalParameter()+ assert.Error(t, err)+ assert.Equal(t, param, "")++ param, err = m.finalParameter()+ assert.Error(t, err)+ assert.Equal(t, param, "")++ param, err = m.nextSpaceDelimitedParameter()+ assert.Error(t, err)+ assert.Equal(t, param, "")++ },+ },+ {+ "TwoParams",+ func(t *testing.T) {+ m := messageParser{"foo bar\n"}++ param, err := m.nextSpaceDelimitedParameter()+ assert.NoError(t, err)+ assert.Equal(t, param, "foo")++ param, err = m.nextSpaceDelimitedParameter()+ assert.NoError(t, err)+ assert.Equal(t, param, "bar")++ param, err = m.nextSpaceDelimitedParameter()+ assert.Error(t, err)+ assert.Equal(t, param, "")++ param, err = m.finalParameter()+ assert.Error(t, err)+ assert.Equal(t, param, "")+ },+ },+ {+ "TwoParamsNoTrailingNewline",++ func(t *testing.T) {+ m := messageParser{"foo bar"}++ param, err := m.nextSpaceDelimitedParameter()+ assert.NoError(t, err)+ assert.Equal(t, param, "foo")++ param, err = m.nextSpaceDelimitedParameter()+ assert.NoError(t, err)+ assert.Equal(t, param, "bar")++ param, err = m.nextSpaceDelimitedParameter()+ assert.Error(t, err)+ assert.Equal(t, param, "")++ param, err = m.finalParameter()+ assert.Error(t, err)+ assert.Equal(t, param, "")+ },+ },+ {+ "ThreeParamsWhereFinalParamContainsSpaces",+ func(t *testing.T) {+ m := messageParser{"firstparam secondparam final param with spaces"}++ param, err := m.nextSpaceDelimitedParameter()+ assert.NoError(t, err)+ assert.Equal(t, param, "firstparam")++ param, err = m.nextSpaceDelimitedParameter()+ assert.NoError(t, err)+ assert.Equal(t, param, "secondparam")++ param, err = m.finalParameter()+ assert.NoError(t, err)+ assert.Equal(t, param, "final param with spaces")+ },+ },+ {+ "OneLongFinalParameter",+ func(t *testing.T) {+ for _, lineEnding := range []string{"", "\n", "\r", "\r\n", "\n\r"} {+ lineEnding := lineEnding+ testName := fmt.Sprintf("lineEnding%x", lineEnding)++ t.Run(testName, func(t *testing.T) {+ m := messageParser{"one long final parameter" + lineEnding}++ param, err := m.finalParameter()+ assert.NoError(t, err)+ assert.Equal(t, param, "one long final parameter")++ param, err = m.finalParameter()+ assert.Error(t, err)+ assert.Equal(t, param, "")+ })++ }+ },+ },+ {+ "MultipleSpaces",+ func(t *testing.T) {+ m := messageParser{"foo bar\n\r"}++ param, err := m.nextSpaceDelimitedParameter()+ assert.NoError(t, err)+ assert.Equal(t, param, "foo")++ param, err = m.nextSpaceDelimitedParameter()+ assert.Error(t, err, "blah")+ assert.Equal(t, param, "")+ },+ },+ {+ "StartsWithSpace",+ func(t *testing.T) {+ m := messageParser{" foo"}++ param, err := m.nextSpaceDelimitedParameter()+ assert.Error(t, err, "blah")+ assert.Equal(t, param, "")+ },+ },+}++func TestMessageParser(t *testing.T) {+ for _, testCase := range messageParserTestCases {+ testCase := testCase+ t.Run(testCase.label, func(t *testing.T) {+ t.Parallel()+ testCase.testFunc(t)+ })+ }+}++type testState struct {+ t *testing.T+ server *server+ mockStdinW *io.PipeWriter+ mockStdoutReader *bufio.Reader++ localFsDir string+ configPath string+ remoteName string+}++func makeTestState(t *testing.T) testState {+ stdinR, stdinW := io.Pipe()+ stdoutR, stdoutW := io.Pipe()++ return testState{+ t: t,+ server: &server{+ reader: bufio.NewReader(stdinR),+ writer: stdoutW,+ },+ mockStdinW: stdinW,+ mockStdoutReader: bufio.NewReader(stdoutR),+ }+}++func (h *testState) requireReadLineExact(line string) {+ receivedLine, err := h.mockStdoutReader.ReadString('\n')+ require.NoError(h.t, err)+ require.Equal(h.t, line+"\n", receivedLine)+}++func (h *testState) requireWriteLine(line string) {+ _, err := h.mockStdinW.Write([]byte(line + "\n"))+ require.NoError(h.t, err)+}++// Preconfigure the handle. This enables the calling test to skip the PREPARE+// handshake.+func (h *testState) preconfigureServer() {+ h.server.configPrefix = h.localFsDir+ h.server.configRcloneRemoteName = h.remoteName+ h.server.configsDone = true+}++// getUniqueRemoteName returns a valid remote name derived from the given test's+// name. This is necessary because when a test registers a second remote with+// the same name, the original remote appears to take precedence. This function+// is injective, so each test gets a unique remote name. Returned strings+// contain no spaces.+func getUniqueRemoteName(t *testing.T) string {+ // Using sha256 as a hack to ensure injectivity without adding a global+ // variable.+ return fmt.Sprintf("remote-%x", sha256.Sum256([]byte(t.Name())))+}++type testCase struct {+ label string+ testProtocolFunc func(*testing.T, *testState)+ expectedError string+}++// These test cases run against the "local" backend.+var localBackendTestCases = []testCase{+ {+ label: "HandlesInit",+ testProtocolFunc: func(t *testing.T, h *testState) {+ h.preconfigureServer()++ h.requireReadLineExact("VERSION 2")+ h.requireWriteLine("INITREMOTE")+ h.requireReadLineExact("INITREMOTE-SUCCESS")++ require.NoError(t, h.mockStdinW.Close())+ },+ },+ {+ label: "HandlesPrepare",+ testProtocolFunc: func(t *testing.T, h *testState) {+ h.requireReadLineExact("VERSION 2")+ h.requireWriteLine("EXTENSIONS INFO") // Advertise that we support the INFO extension+ h.requireReadLineExact("EXTENSIONS")++ if !h.server.extensionInfo {+ t.Errorf("expected INFO extension to be enabled")+ return+ }++ h.requireWriteLine("PREPARE")+ h.requireReadLineExact("GETCONFIG rcloneremotename")+ h.requireWriteLine("VALUE " + h.remoteName)+ h.requireReadLineExact("GETCONFIG rcloneprefix")+ h.requireWriteLine("VALUE " + h.localFsDir)+ h.requireReadLineExact("PREPARE-SUCCESS")++ require.Equal(t, h.server.configRcloneRemoteName, h.remoteName)+ require.Equal(t, h.server.configPrefix, h.localFsDir)+ require.True(t, h.server.configsDone)++ require.NoError(t, h.mockStdinW.Close())+ },+ },+ {+ label: "HandlesPrepareAndDoesNotTrimWhitespaceFromValue",+ testProtocolFunc: func(t *testing.T, h *testState) {+ h.requireReadLineExact("VERSION 2")+ h.requireWriteLine("EXTENSIONS INFO") // Advertise that we support the INFO extension+ h.requireReadLineExact("EXTENSIONS")++ if !h.server.extensionInfo {+ t.Errorf("expected INFO extension to be enabled")+ return+ }++ h.requireWriteLine("PREPARE")+ h.requireReadLineExact("GETCONFIG rcloneremotename")++ remoteNameWithSpaces := fmt.Sprintf(" %s ", h.remoteName)+ localFsDirWithSpaces := fmt.Sprintf(" %s\t", h.localFsDir)++ h.requireWriteLine(fmt.Sprintf("VALUE %s", remoteNameWithSpaces))+ h.requireReadLineExact("GETCONFIG rcloneprefix")++ h.requireWriteLine(fmt.Sprintf("VALUE %s", localFsDirWithSpaces))+ h.requireReadLineExact("PREPARE-SUCCESS")++ require.Equal(t, h.server.configRcloneRemoteName, remoteNameWithSpaces)+ require.Equal(t, h.server.configPrefix, localFsDirWithSpaces)+ require.True(t, h.server.configsDone)++ require.NoError(t, h.mockStdinW.Close())+ },+ },+ {+ label: "HandlesEarlyError",+ testProtocolFunc: func(t *testing.T, h *testState) {+ h.preconfigureServer()++ h.requireReadLineExact("VERSION 2")+ h.requireWriteLine("ERROR foo")++ require.NoError(t, h.mockStdinW.Close())+ },+ expectedError: "received error message from git-annex: foo",+ },+ // Test what happens when the git-annex client sends "GETCONFIG", but+ // doesn't understand git-annex's response.+ {+ label: "ConfigFail",+ testProtocolFunc: func(t *testing.T, h *testState) {+ h.requireReadLineExact("VERSION 2")+ h.requireWriteLine("EXTENSIONS INFO") // Advertise that we support the INFO extension+ h.requireReadLineExact("EXTENSIONS")+ require.True(t, h.server.extensionInfo)++ h.requireWriteLine("PREPARE")+ h.requireReadLineExact("GETCONFIG rcloneremotename")+ h.requireWriteLine("ERROR ineffable error")+ h.requireReadLineExact("PREPARE-FAILURE Error getting configs")++ require.NoError(t, h.mockStdinW.Close())+ },+ expectedError: "failed to parse config value: ERROR ineffable error",+ },+ {+ label: "TransferStoreEmptyPath",+ testProtocolFunc: func(t *testing.T, h *testState) {+ h.preconfigureServer()++ h.requireReadLineExact("VERSION 2")+ h.requireWriteLine("INITREMOTE")+ h.requireReadLineExact("INITREMOTE-SUCCESS")++ // Note the whitespace following the key.+ h.requireWriteLine("TRANSFER STORE Key ")+ h.requireReadLineExact("TRANSFER-FAILURE failed to parse file")++ require.NoError(t, h.mockStdinW.Close())+ },+ expectedError: "malformed arguments for TRANSFER: nothing remains to parse",+ },+ // Repeated EXTENSIONS messages add to each other rather than overriding+ // prior advertised extensions. This behavior is not mandated by the+ // protocol design.+ {+ label: "ExtensionsCompound",+ testProtocolFunc: func(t *testing.T, h *testState) {+ h.preconfigureServer()++ h.requireReadLineExact("VERSION 2")+ h.requireWriteLine("INITREMOTE")+ h.requireReadLineExact("INITREMOTE-SUCCESS")++ h.requireWriteLine("EXTENSIONS")+ h.requireReadLineExact("EXTENSIONS")+ require.False(t, h.server.extensionInfo)+ require.False(t, h.server.extensionAsync)+ require.False(t, h.server.extensionGetGitRemoteName)+ require.False(t, h.server.extensionUnavailableResponse)++ h.requireWriteLine("EXTENSIONS INFO")+ h.requireReadLineExact("EXTENSIONS")+ require.True(t, h.server.extensionInfo)+ require.False(t, h.server.extensionAsync)+ require.False(t, h.server.extensionGetGitRemoteName)+ require.False(t, h.server.extensionUnavailableResponse)++ h.requireWriteLine("EXTENSIONS ASYNC")+ h.requireReadLineExact("EXTENSIONS")+ require.True(t, h.server.extensionInfo)+ require.True(t, h.server.extensionAsync)+ require.False(t, h.server.extensionGetGitRemoteName)+ require.False(t, h.server.extensionUnavailableResponse)++ h.requireWriteLine("EXTENSIONS GETGITREMOTENAME")+ h.requireReadLineExact("EXTENSIONS")+ require.True(t, h.server.extensionInfo)+ require.True(t, h.server.extensionAsync)+ require.True(t, h.server.extensionGetGitRemoteName)+ require.False(t, h.server.extensionUnavailableResponse)++ h.requireWriteLine("EXTENSIONS UNAVAILABLERESPONSE")+ h.requireReadLineExact("EXTENSIONS")+ require.True(t, h.server.extensionInfo)+ require.True(t, h.server.extensionAsync)+ require.True(t, h.server.extensionGetGitRemoteName)+ require.True(t, h.server.extensionUnavailableResponse)++ require.NoError(t, h.mockStdinW.Close())+ },+ },+ {+ label: "ExtensionsIdempotent",+ testProtocolFunc: func(t *testing.T, h *testState) {+ h.preconfigureServer()++ h.requireReadLineExact("VERSION 2")+ h.requireWriteLine("INITREMOTE")+ h.requireReadLineExact("INITREMOTE-SUCCESS")++ h.requireWriteLine("EXTENSIONS")+ h.requireReadLineExact("EXTENSIONS")+ require.False(t, h.server.extensionInfo)+ require.False(t, h.server.extensionAsync)+ require.False(t, h.server.extensionGetGitRemoteName)+ require.False(t, h.server.extensionUnavailableResponse)++ h.requireWriteLine("EXTENSIONS")+ h.requireReadLineExact("EXTENSIONS")+ require.False(t, h.server.extensionInfo)+ require.False(t, h.server.extensionAsync)+ require.False(t, h.server.extensionGetGitRemoteName)+ require.False(t, h.server.extensionUnavailableResponse)++ h.requireWriteLine("EXTENSIONS INFO")+ h.requireReadLineExact("EXTENSIONS")+ require.True(t, h.server.extensionInfo)+ require.False(t, h.server.extensionAsync)+ require.False(t, h.server.extensionGetGitRemoteName)+ require.False(t, h.server.extensionUnavailableResponse)++ h.requireWriteLine("EXTENSIONS INFO")+ h.requireReadLineExact("EXTENSIONS")+ require.True(t, h.server.extensionInfo)+ require.False(t, h.server.extensionAsync)+ require.False(t, h.server.extensionGetGitRemoteName)+ require.False(t, h.server.extensionUnavailableResponse)++ h.requireWriteLine("EXTENSIONS ASYNC ASYNC")+ h.requireReadLineExact("EXTENSIONS")+ require.True(t, h.server.extensionInfo)+ require.True(t, h.server.extensionAsync)+ require.False(t, h.server.extensionGetGitRemoteName)+ require.False(t, h.server.extensionUnavailableResponse)++ require.NoError(t, h.mockStdinW.Close())+ },+ },+ {+ label: "ExtensionsSupportsMultiple",+ testProtocolFunc: func(t *testing.T, h *testState) {+ h.preconfigureServer()++ h.requireReadLineExact("VERSION 2")+ h.requireWriteLine("INITREMOTE")+ h.requireReadLineExact("INITREMOTE-SUCCESS")++ h.requireWriteLine("EXTENSIONS")+ h.requireReadLineExact("EXTENSIONS")+ require.False(t, h.server.extensionInfo)+ require.False(t, h.server.extensionAsync)+ require.False(t, h.server.extensionGetGitRemoteName)+ require.False(t, h.server.extensionUnavailableResponse)++ h.requireWriteLine("EXTENSIONS INFO ASYNC")+ h.requireReadLineExact("EXTENSIONS")+ require.True(t, h.server.extensionInfo)+ require.True(t, h.server.extensionAsync)+ require.False(t, h.server.extensionGetGitRemoteName)+ require.False(t, h.server.extensionUnavailableResponse)++ require.NoError(t, h.mockStdinW.Close())+ },+ },+ {+ label: "TransferStoreAbsolute",+ testProtocolFunc: func(t *testing.T, h *testState) {+ h.preconfigureServer()++ h.requireReadLineExact("VERSION 2")+ h.requireWriteLine("INITREMOTE")+ h.requireReadLineExact("INITREMOTE-SUCCESS")++ // Create temp file for transfer with an absolute path.+ fileToTransfer := filepath.Join(t.TempDir(), "file.txt")+ require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))+ require.FileExists(t, fileToTransfer)+ require.True(t, filepath.IsAbs(fileToTransfer))++ // Specify an absolute path to transfer.+ h.requireWriteLine("TRANSFER STORE KeyAbsolute " + fileToTransfer)+ h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyAbsolute")+ require.FileExists(t, filepath.Join(h.localFsDir, "KeyAbsolute"))++ // Transfer the same absolute path a second time, but with a different key.+ h.requireWriteLine("TRANSFER STORE KeyAbsolute2 " + fileToTransfer)+ h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyAbsolute2")+ require.FileExists(t, filepath.Join(h.localFsDir, "KeyAbsolute2"))++ h.requireWriteLine("CHECKPRESENT KeyAbsolute2")+ h.requireReadLineExact("CHECKPRESENT-SUCCESS KeyAbsolute2")++ h.requireWriteLine("CHECKPRESENT KeyThatDoesNotExist")+ h.requireReadLineExact("CHECKPRESENT-FAILURE KeyThatDoesNotExist")++ require.NoError(t, h.mockStdinW.Close())+ },+ },+ // Test that the TRANSFER command understands simple relative paths+ // consisting only of a file name.+ {+ label: "TransferStoreRelative",+ testProtocolFunc: func(t *testing.T, h *testState) {+ h.preconfigureServer()++ // Save the current working directory so we can restore it when this+ // test ends.+ cwd, err := os.Getwd()+ require.NoError(t, err)++ require.NoError(t, os.Chdir(t.TempDir()))+ t.Cleanup(func() { require.NoError(t, os.Chdir(cwd)) })++ h.requireReadLineExact("VERSION 2")+ h.requireWriteLine("INITREMOTE")+ h.requireReadLineExact("INITREMOTE-SUCCESS")++ // Create temp file for transfer with a relative path.+ fileToTransfer := "file.txt"+ require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))+ require.FileExists(t, fileToTransfer)+ require.False(t, filepath.IsAbs(fileToTransfer))++ // Specify a relative path to transfer.+ h.requireWriteLine("TRANSFER STORE KeyRelative " + fileToTransfer)+ h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyRelative")+ require.FileExists(t, filepath.Join(h.localFsDir, "KeyRelative"))++ h.requireWriteLine("CHECKPRESENT KeyRelative")+ h.requireReadLineExact("CHECKPRESENT-SUCCESS KeyRelative")++ h.requireWriteLine("CHECKPRESENT KeyThatDoesNotExist")+ h.requireReadLineExact("CHECKPRESENT-FAILURE KeyThatDoesNotExist")++ require.NoError(t, h.mockStdinW.Close())+ },+ },+ {+ label: "TransferStorePathWithInteriorWhitespace",+ testProtocolFunc: func(t *testing.T, h *testState) {+ // Save the current working directory so we can restore it when this+ // test ends.+ cwd, err := os.Getwd()+ require.NoError(t, err)++ require.NoError(t, os.Chdir(t.TempDir()))+ t.Cleanup(func() { require.NoError(t, os.Chdir(cwd)) })++ h.preconfigureServer()++ h.requireReadLineExact("VERSION 2")+ h.requireWriteLine("INITREMOTE")+ h.requireReadLineExact("INITREMOTE-SUCCESS")++ // Create temp file for transfer.+ fileToTransfer := "filename with spaces.txt"+ require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))+ require.FileExists(t, fileToTransfer)+ require.False(t, filepath.IsAbs(fileToTransfer))++ // Specify a relative path to transfer.+ h.requireWriteLine("TRANSFER STORE KeyRelative " + fileToTransfer)+ h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyRelative")+ require.FileExists(t, filepath.Join(h.localFsDir, "KeyRelative"))++ h.requireWriteLine("CHECKPRESENT KeyRelative")+ h.requireReadLineExact("CHECKPRESENT-SUCCESS KeyRelative")++ h.requireWriteLine("CHECKPRESENT KeyThatDoesNotExist")+ h.requireReadLineExact("CHECKPRESENT-FAILURE KeyThatDoesNotExist")++ require.NoError(t, h.mockStdinW.Close())+ },+ },+ {+ label: "CheckPresentAndTransfer",+ testProtocolFunc: func(t *testing.T, h *testState) {+ h.preconfigureServer()++ // Create temp file for transfer.+ fileToTransfer := filepath.Join(t.TempDir(), "file.txt")+ require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))++ h.requireReadLineExact("VERSION 2")+ h.requireWriteLine("INITREMOTE")+ h.requireReadLineExact("INITREMOTE-SUCCESS")++ h.requireWriteLine("CHECKPRESENT KeyThatDoesNotExist")+ h.requireReadLineExact("CHECKPRESENT-FAILURE KeyThatDoesNotExist")++ // Specify an absolute path to transfer.+ require.True(t, filepath.IsAbs(fileToTransfer))+ h.requireWriteLine("TRANSFER STORE KeyAbsolute " + fileToTransfer)+ h.requireReadLineExact("TRANSFER-SUCCESS STORE KeyAbsolute")+ require.FileExists(t, filepath.Join(h.localFsDir, "KeyAbsolute"))++ require.NoError(t, h.mockStdinW.Close())+ },+ },+ // Check whether a key is present, transfer a file with that key, then check+ // again whether it is present.+ //+ // This is a regression test for a bug where the second CHECKPRESENT would+ // generate the following response:+ //+ // CHECKPRESENT-UNKNOWN ${key} failed to read directory entry: readdirent ${filepath}: not a directory+ //+ // This message was generated by the local backend's `List()` function. When+ // checking whether a file exists, we were erroneously listing its contents as+ // if it were a directory.+ {+ label: "CheckpresentTransferCheckpresent",+ testProtocolFunc: func(t *testing.T, h *testState) {+ h.preconfigureServer()++ // Create temp file for transfer.+ fileToTransfer := filepath.Join(t.TempDir(), "file.txt")+ require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))++ h.requireReadLineExact("VERSION 2")+ h.requireWriteLine("INITREMOTE")+ h.requireReadLineExact("INITREMOTE-SUCCESS")++ h.requireWriteLine("CHECKPRESENT foo")+ h.requireReadLineExact("CHECKPRESENT-FAILURE foo")++ h.requireWriteLine("TRANSFER STORE foo " + fileToTransfer)+ h.requireReadLineExact("TRANSFER-SUCCESS STORE foo")+ require.FileExists(t, filepath.Join(h.localFsDir, "foo"))++ h.requireWriteLine("CHECKPRESENT foo")+ h.requireReadLineExact("CHECKPRESENT-SUCCESS foo")++ require.NoError(t, h.mockStdinW.Close())+ },+ },+ {+ label: "TransferAndCheckpresentWithRealisticKey",+ testProtocolFunc: func(t *testing.T, h *testState) {+ h.preconfigureServer()++ // Create temp file for transfer.+ fileToTransfer := filepath.Join(t.TempDir(), "file.txt")+ require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))++ h.requireReadLineExact("VERSION 2")+ h.requireWriteLine("INITREMOTE")+ h.requireReadLineExact("INITREMOTE-SUCCESS")++ realisticKey := "SHA256E-s1048576--7ba87e06b9b7903cfbaf4a38736766c161e3e7b42f06fe57f040aa410a8f0701.this-is-a-test-key"++ // Specify an absolute path to transfer.+ require.True(t, filepath.IsAbs(fileToTransfer))+ h.requireWriteLine(fmt.Sprintf("TRANSFER STORE %s %s", realisticKey, fileToTransfer))+ h.requireReadLineExact("TRANSFER-SUCCESS STORE " + realisticKey)+ require.FileExists(t, filepath.Join(h.localFsDir, realisticKey))++ h.requireWriteLine("CHECKPRESENT " + realisticKey)+ h.requireReadLineExact("CHECKPRESENT-SUCCESS " + realisticKey)++ require.NoError(t, h.mockStdinW.Close())+ },+ },+ {+ label: "RetrieveNonexistentFile",+ testProtocolFunc: func(t *testing.T, h *testState) {+ h.preconfigureServer()++ h.requireReadLineExact("VERSION 2")+ h.requireWriteLine("INITREMOTE")+ h.requireReadLineExact("INITREMOTE-SUCCESS")++ h.requireWriteLine("TRANSFER RETRIEVE SomeKey path")+ h.requireReadLineExact("TRANSFER-FAILURE RETRIEVE SomeKey not found")++ require.NoError(t, h.mockStdinW.Close())+ },+ },+ {+ label: "StoreCheckpresentRetrieve",+ testProtocolFunc: func(t *testing.T, h *testState) {+ h.preconfigureServer()++ // Create temp file for transfer.+ fileToTransfer := filepath.Join(t.TempDir(), "file.txt")+ require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))++ h.requireReadLineExact("VERSION 2")+ h.requireWriteLine("INITREMOTE")+ h.requireReadLineExact("INITREMOTE-SUCCESS")++ // Specify an absolute path to transfer.+ require.True(t, filepath.IsAbs(fileToTransfer))+ h.requireWriteLine("TRANSFER STORE SomeKey " + fileToTransfer)+ h.requireReadLineExact("TRANSFER-SUCCESS STORE SomeKey")+ require.FileExists(t, filepath.Join(h.localFsDir, "SomeKey"))++ h.requireWriteLine("CHECKPRESENT SomeKey")+ h.requireReadLineExact("CHECKPRESENT-SUCCESS SomeKey")++ retrievedFilePath := fileToTransfer + ".retrieved"+ require.NoFileExists(t, retrievedFilePath)+ h.requireWriteLine("TRANSFER RETRIEVE SomeKey " + retrievedFilePath)+ h.requireReadLineExact("TRANSFER-SUCCESS RETRIEVE SomeKey")+ require.FileExists(t, retrievedFilePath)++ require.NoError(t, h.mockStdinW.Close())+ },+ },+ {+ label: "RemovePreexistingFile",+ testProtocolFunc: func(t *testing.T, h *testState) {+ h.preconfigureServer()++ // Write a file into the remote without using the git-annex+ // protocol.+ remoteFilePath := filepath.Join(h.localFsDir, "SomeKey")+ require.NoError(t, os.WriteFile(remoteFilePath, []byte("HELLO"), 0600))+ require.FileExists(t, remoteFilePath)++ h.requireReadLineExact("VERSION 2")+ h.requireWriteLine("INITREMOTE")+ h.requireReadLineExact("INITREMOTE-SUCCESS")++ h.requireWriteLine("CHECKPRESENT SomeKey")+ h.requireReadLineExact("CHECKPRESENT-SUCCESS SomeKey")+ require.FileExists(t, remoteFilePath)++ h.requireWriteLine("REMOVE SomeKey")+ h.requireReadLineExact("REMOVE-SUCCESS SomeKey")+ require.NoFileExists(t, remoteFilePath)++ h.requireWriteLine("CHECKPRESENT SomeKey")+ h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")+ require.NoFileExists(t, remoteFilePath)++ require.NoError(t, h.mockStdinW.Close())+ },+ },+ {+ label: "Remove",+ testProtocolFunc: func(t *testing.T, h *testState) {+ h.preconfigureServer()++ // Create temp file for transfer.+ fileToTransfer := filepath.Join(t.TempDir(), "file.txt")+ require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))++ h.requireReadLineExact("VERSION 2")+ h.requireWriteLine("INITREMOTE")+ h.requireReadLineExact("INITREMOTE-SUCCESS")++ h.requireWriteLine("CHECKPRESENT SomeKey")+ h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")++ // Specify an absolute path to transfer.+ require.True(t, filepath.IsAbs(fileToTransfer))+ h.requireWriteLine("TRANSFER STORE SomeKey " + fileToTransfer)+ h.requireReadLineExact("TRANSFER-SUCCESS STORE SomeKey")+ require.FileExists(t, filepath.Join(h.localFsDir, "SomeKey"))++ h.requireWriteLine("CHECKPRESENT SomeKey")+ h.requireReadLineExact("CHECKPRESENT-SUCCESS SomeKey")++ h.requireWriteLine("REMOVE SomeKey")+ h.requireReadLineExact("REMOVE-SUCCESS SomeKey")+ require.NoFileExists(t, filepath.Join(h.localFsDir, "SomeKey"))++ h.requireWriteLine("CHECKPRESENT SomeKey")+ h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")++ require.NoError(t, h.mockStdinW.Close())+ },+ },+ {+ label: "RemoveNonexistentFile",+ testProtocolFunc: func(t *testing.T, h *testState) {+ h.preconfigureServer()++ // Create temp file for transfer.+ fileToTransfer := filepath.Join(t.TempDir(), "file.txt")+ require.NoError(t, os.WriteFile(fileToTransfer, []byte("HELLO"), 0600))++ h.requireReadLineExact("VERSION 2")+ h.requireWriteLine("INITREMOTE")+ h.requireReadLineExact("INITREMOTE-SUCCESS")++ h.requireWriteLine("CHECKPRESENT SomeKey")+ h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")++ require.NoFileExists(t, filepath.Join(h.localFsDir, "SomeKey"))+ h.requireWriteLine("REMOVE SomeKey")+ h.requireReadLineExact("REMOVE-SUCCESS SomeKey")+ require.NoFileExists(t, filepath.Join(h.localFsDir, "SomeKey"))++ h.requireWriteLine("CHECKPRESENT SomeKey")+ h.requireReadLineExact("CHECKPRESENT-FAILURE SomeKey")++ require.NoError(t, h.mockStdinW.Close())+ },+ },+ {+ label: "ExportNotSupported",+ testProtocolFunc: func(t *testing.T, h *testState) {+ h.preconfigureServer()++ h.requireReadLineExact("VERSION 2")+ h.requireWriteLine("INITREMOTE")+ h.requireReadLineExact("INITREMOTE-SUCCESS")++ h.requireWriteLine("EXPORTSUPPORTED")+ h.requireReadLineExact("EXPORTSUPPORTED-FAILURE")++ require.NoError(t, h.mockStdinW.Close())+ },+ },+}++func TestGitAnnexLocalBackendCases(t *testing.T) {+ for _, testCase := range localBackendTestCases {+ // Clear global state left behind by tests that chdir to a temp directory.+ cache.Clear()++ // TODO: Remove this when rclone requires a Go version >= 1.22. Future+ // versions of Go fix the semantics of capturing a range variable.+ // https://go.dev/blog/loopvar-preview+ testCase := testCase++ t.Run(testCase.label, func(t *testing.T) {+ tempDir := t.TempDir()++ // Create temp dir for an rclone remote pointing at local filesystem.+ localFsDir := filepath.Join(tempDir, "remoteTarget")+ require.NoError(t, os.Mkdir(localFsDir, 0700))++ // Create temp config+ remoteName := getUniqueRemoteName(t)+ configLines := []string{+ fmt.Sprintf("[%s]", remoteName),+ "type = local",+ fmt.Sprintf("remote = %s", localFsDir),+ }+ configContents := strings.Join(configLines, "\n")++ configPath := filepath.Join(tempDir, "rclone.conf")+ require.NoError(t, os.WriteFile(configPath, []byte(configContents), 0600))+ require.NoError(t, config.SetConfigPath(configPath))++ // The custom config file will be ignored unless we install the+ // global config file handler.+ configfile.Install()++ handle := makeTestState(t)+ handle.localFsDir = localFsDir+ handle.configPath = configPath+ handle.remoteName = remoteName++ var wg sync.WaitGroup+ wg.Add(1)++ go func() {+ err := handle.server.run()++ if testCase.expectedError == "" {+ require.NoError(t, err)+ } else {+ require.ErrorContains(t, err, testCase.expectedError)+ }++ wg.Done()+ }()+ defer wg.Wait()++ testCase.testProtocolFunc(t, &handle)+ })+ }+}++// Configure the git-annex client with a mockfs backend and send it the+// "INITREMOTE" command over mocked stdin. This should fail because mockfs does+// not support empty directories.+func TestGitAnnexHandleInitRemoteBackendDoesNotSupportEmptyDirectories(t *testing.T) {+ tempDir := t.TempDir()++ // Temporarily override the filesystem registry.+ oldRegistry := fs.Registry+ mockfs.Register()+ defer func() { fs.Registry = oldRegistry }()++ // Create temp dir for an rclone remote pointing at local filesystem.+ localFsDir := filepath.Join(tempDir, "remoteTarget")+ require.NoError(t, os.Mkdir(localFsDir, 0700))++ // Create temp config+ remoteName := getUniqueRemoteName(t)+ configLines := []string{+ fmt.Sprintf("[%s]", remoteName),+ "type = mockfs",+ fmt.Sprintf("remote = %s", localFsDir),+ }+ configContents := strings.Join(configLines, "\n")++ configPath := filepath.Join(tempDir, "rclone.conf")+ require.NoError(t, os.WriteFile(configPath, []byte(configContents), 0600))++ // The custom config file will be ignored unless we install the global+ // config file handler.+ configfile.Install()+ require.NoError(t, config.SetConfigPath(configPath))++ handle := makeTestState(t)+ handle.server.configPrefix = localFsDir+ handle.server.configRcloneRemoteName = remoteName+ handle.server.configsDone = true++ var wg sync.WaitGroup+ wg.Add(1)++ go func() {+ require.NotNil(t, handle.server.run())+ wg.Done()+ }()+ defer wg.Wait()++ handle.requireReadLineExact("VERSION 2")+ handle.requireWriteLine("INITREMOTE")+ handle.requireReadLineExact("INITREMOTE-FAILURE this rclone remote does not support empty directories")+}diff --git a/cmd/gitannex/run-git-annex-testremote.sh b/cmd/gitannex/run-git-annex-testremote.shnew file mode 100755index 000000000..98a848df7--- /dev/null+++ b/cmd/gitannex/run-git-annex-testremote.sh@@ -0,0 +1,52 @@+#!/usr/bin/env bash+#+# End-to-end tests for "rclone gitannex". This script runs the `git-annex+# testremote` suite against "rclone gitannex" in an ephemeral git-annex repo.+#+# Assumptions:+#+# * This system has an rclone remote configured named "git-annex-builtin-test-remote".+#+# * If it uses rclone's "local" backend, /tmp/git-annex-builtin-test-remote exists.++set -e++TEST_DIR="$(realpath "$(mktemp -d)")"+mkdir "$TEST_DIR/bin"++function cleanup()+{+ rm -rf "$TEST_DIR"+}++trap cleanup EXIT++RCLONE_DIR="$(git rev-parse --show-toplevel)"++rm -rf /tmp/git-annex-builtin-test-remote/*++set -x++pushd "$RCLONE_DIR"+go build -o "$TEST_DIR/bin" ./++ln -s "$(realpath "$TEST_DIR/bin/rclone")" "$TEST_DIR/bin/git-annex-remote-rclone-builtin"+popd++pushd "$TEST_DIR"++git init+git annex init++REMOTE_NAME=git-annex-builtin-test-remote+PREFIX=/tmp/git-annex-builtin-test-remote++PATH="$PATH:$TEST_DIR/bin" git annex initremote $REMOTE_NAME \+ type=external externaltype=rclone-builtin encryption=none \+ rcloneremotename=$REMOTE_NAME \+ rcloneprefix="$PREFIX"++PATH="$PATH:$(realpath bin)" git annex testremote $REMOTE_NAME++popd+rm -rf "$TEST_DIR"