"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
cmd/gitannex: Add support for different layouts
Dan McArdle29b58dd4c5db
diff --git a/cmd/gitannex/gitannex.go b/cmd/gitannex/gitannex.goindex a71b04871..0eb09da12 100644--- a/cmd/gitannex/gitannex.go+++ b/cmd/gitannex/gitannex.go@@ -110,9 +110,10 @@ func (m *messageParser) finalParameter() string {// configDefinition describes a configuration value required by this command. We// use "GETCONFIG" messages to query git-annex for these values at runtime.type configDefinition struct {- name string- description string- destination *string+ name string+ description string+ destination *string+ defaultValue *string}// server contains this command's current state.@@ -132,6 +133,7 @@ type server struct {configsDone boolconfigPrefix stringconfigRcloneRemoteName string+ configRcloneLayout string}func (s *server) sendMsg(msg string) {@@ -267,6 +269,9 @@ func (s *server) handleInitRemote() error {// Get a list of configs with pointers to fields of `s`.func (s *server) getRequiredConfigs() []configDefinition {+ defaultRclonePrefix := "git-annex-rclone"+ defaultRcloneLayout := "nodir"+return []configDefinition{{"rcloneremotename",@@ -274,13 +279,23 @@ func (s *server) getRequiredConfigs() []configDefinition {"Must match a remote known to rclone. " +"(Note that rclone remotes are a distinct concept from git-annex remotes.)",&s.configRcloneRemoteName,+ nil,},{"rcloneprefix","Directory where rclone will write git-annex content. " +- "If not specified, defaults to \"git-annex-rclone\". " +- "This directory be created on init if it does not exist.",+ fmt.Sprintf("If not specified, defaults to %q. ", defaultRclonePrefix) ++ "This directory will be created on init if it does not exist.",&s.configPrefix,+ &defaultRclonePrefix,+ },+ {+ "rclonelayout",+ "Defines where, within the rcloneprefix directory, rclone will write git-annex content. " ++ fmt.Sprintf("Must be one of %v. ", allLayoutModes()) ++ fmt.Sprintf("If empty, defaults to %q.", defaultRcloneLayout),+ &s.configRcloneLayout,+ &defaultRcloneLayout,},}}@@ -307,10 +322,13 @@ func (s *server) queryConfigs() error {}value := message.finalParameter()- if value == "" {+ if value == "" && config.defaultValue == nil {return fmt.Errorf("config value of %q must not be empty", config.name)}-+ if value == "" {+ *config.destination = *config.defaultValue+ continue+ }*config.destination = value}@@ -358,7 +376,19 @@ func (s *server) handleTransfer(message *messageParser) error {return fmt.Errorf("error getting configs: %w", err)}- remoteFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:%s", s.configRcloneRemoteName, s.configPrefix))+ layout := parseLayoutMode(s.configRcloneLayout)+ if layout == layoutModeUnknown {+ s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s", argKey))+ return fmt.Errorf("error parsing layout mode: %q", s.configRcloneLayout)+ }++ remoteFsString, err := buildFsString(s.queryDirhash, layout, argKey, s.configRcloneRemoteName, s.configPrefix)+ if err != nil {+ s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s", argKey))+ return fmt.Errorf("error building fs string: %w", err)+ }++ remoteFs, err := cache.Get(context.TODO(), remoteFsString)if err != nil {s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to get remote fs", argMode, argKey))return err@@ -415,7 +445,19 @@ func (s *server) handleCheckPresent(message *messageParser) error {return fmt.Errorf("error getting configs: %s", err)}- remoteFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:%s", s.configRcloneRemoteName, s.configPrefix))+ layout := parseLayoutMode(s.configRcloneLayout)+ if layout == layoutModeUnknown {+ s.sendMsg(fmt.Sprintf("CHECKPRESENT-FAILURE %s", argKey))+ return fmt.Errorf("error parsing layout mode: %q", s.configRcloneLayout)+ }++ remoteFsString, err := buildFsString(s.queryDirhash, layout, argKey, s.configRcloneRemoteName, s.configPrefix)+ if err != nil {+ s.sendMsg(fmt.Sprintf("CHECKPRESENT-FAILURE %s", argKey))+ return fmt.Errorf("error building fs string: %w", err)+ }++ remoteFs, err := cache.Get(context.TODO(), remoteFsString)if err != nil {s.sendMsg(fmt.Sprintf("CHECKPRESENT-UNKNOWN %s failed to get remote fs", argKey))return err@@ -435,17 +477,45 @@ func (s *server) handleCheckPresent(message *messageParser) error {return nil}+func (s *server) queryDirhash(msg string) (string, error) {+ s.sendMsg(msg)+ parser, err := s.getMsg()+ if err != nil {+ return "", err+ }+ keyword, err := parser.nextSpaceDelimitedParameter()+ if err != nil {+ return "", err+ }+ if keyword != "VALUE" {+ return "", fmt.Errorf("expected VALUE keyword, but got %q", keyword)+ }+ dirhash, err := parser.nextSpaceDelimitedParameter()+ if err != nil {+ return "", fmt.Errorf("failed to parse dirhash: %w", err)+ }+ return dirhash, nil+}+func (s *server) handleRemove(message *messageParser) error {argKey := message.finalParameter()if argKey == "" {return errors.New("failed to parse key for REMOVE")}+ layout := parseLayoutMode(s.configRcloneLayout)+ if layout == layoutModeUnknown {+ s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s", argKey))+ return fmt.Errorf("error parsing layout mode: %q", s.configRcloneLayout)+ }++ remoteFsString, err := buildFsString(s.queryDirhash, layout, argKey, s.configRcloneRemoteName, s.configPrefix)if err != nil {- return err+ s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s", argKey))+ return fmt.Errorf("error building fs string: %w", err)}- remoteFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:%s", s.configRcloneRemoteName, s.configPrefix))+ remoteFs, err := cache.Get(context.TODO(), remoteFsString)if err != nil {s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s", argKey))return fmt.Errorf("error getting remote fs: %w", err)diff --git a/cmd/gitannex/gitannex_test.go b/cmd/gitannex/gitannex_test.goindex b3846221d..2b023b8c0 100644--- a/cmd/gitannex/gitannex_test.go+++ b/cmd/gitannex/gitannex_test.go@@ -234,6 +234,7 @@ func (h *testState) requireWriteLine(line string) {func (h *testState) preconfigureServer() {h.server.configPrefix = h.localFsDirh.server.configRcloneRemoteName = h.remoteName+ h.server.configRcloneLayout = string(layoutModeNodir)h.server.configsDone = true}@@ -285,6 +286,8 @@ var localBackendTestCases = []testCase{h.requireWriteLine("VALUE " + h.remoteName)h.requireReadLineExact("GETCONFIG rcloneprefix")h.requireWriteLine("VALUE " + h.localFsDir)+ h.requireReadLineExact("GETCONFIG rclonelayout")+ h.requireWriteLine("VALUE foo")h.requireReadLineExact("PREPARE-SUCCESS")require.Equal(t, h.server.configRcloneRemoteName, h.remoteName)@@ -313,9 +316,13 @@ var localBackendTestCases = []testCase{localFsDirWithSpaces := fmt.Sprintf(" %s\t", h.localFsDir)h.requireWriteLine(fmt.Sprintf("VALUE %s", remoteNameWithSpaces))- h.requireReadLineExact("GETCONFIG rcloneprefix")+ h.requireReadLineExact("GETCONFIG rcloneprefix")h.requireWriteLine(fmt.Sprintf("VALUE %s", localFsDirWithSpaces))++ h.requireReadLineExact("GETCONFIG rclonelayout")+ h.requireWriteLine("VALUE")+h.requireReadLineExact("PREPARE-SUCCESS")require.Equal(t, h.server.configRcloneRemoteName, remoteNameWithSpaces)@@ -367,11 +374,11 @@ var localBackendTestCases = []testCase{// Note the whitespace following the key.h.requireWriteLine("TRANSFER STORE Key ")- h.requireReadLineExact("TRANSFER-FAILURE failed to parse file")+ h.requireReadLineExact("TRANSFER-FAILURE failed to parse file path")require.NoError(t, h.mockStdinW.Close())},- expectedError: "malformed arguments for TRANSFER: nothing remains to parse",+ expectedError: "failed to parse file",},// Repeated EXTENSIONS messages add to each other rather than overriding// prior advertised extensions. This behavior is not mandated by thediff --git a/cmd/gitannex/layout.go b/cmd/gitannex/layout.gonew file mode 100644index 000000000..81b18bfb6--- /dev/null+++ b/cmd/gitannex/layout.go@@ -0,0 +1,72 @@+package gitannex++import (+ "fmt"+ "strings"+)++type layoutMode string++// All layout modes from git-annex-remote-rclone are supported.+const (+ layoutModeLower layoutMode = "lower"+ layoutModeDirectory layoutMode = "directory"+ layoutModeNodir layoutMode = "nodir"+ layoutModeMixed layoutMode = "mixed"+ layoutModeFrankencase layoutMode = "frankencase"+ layoutModeUnknown layoutMode = ""+)++func allLayoutModes() []layoutMode {+ return []layoutMode{+ layoutModeLower,+ layoutModeDirectory,+ layoutModeNodir,+ layoutModeMixed,+ layoutModeFrankencase,+ }+}++func parseLayoutMode(mode string) layoutMode {+ for _, knownMode := range allLayoutModes() {+ if mode == string(knownMode) {+ return knownMode+ }+ }+ return layoutModeUnknown+}++type queryDirhashFunc func(msg string) (string, error)++func buildFsString(queryDirhash queryDirhashFunc, mode layoutMode, key, remoteName, prefix string) (string, error) {+ if mode == layoutModeNodir {+ return fmt.Sprintf("%s:%s", remoteName, prefix), nil+ }++ var dirhash string+ var err error+ switch mode {+ case layoutModeLower, layoutModeDirectory:+ dirhash, err = queryDirhash("DIRHASH-LOWER " + key)+ case layoutModeMixed, layoutModeFrankencase:+ dirhash, err = queryDirhash("DIRHASH " + key)+ default:+ panic("unreachable")+ }+ if err != nil {+ return "", fmt.Errorf("buildFsString failed to query dirhash: %w", err)+ }++ switch mode {+ case layoutModeLower:+ return fmt.Sprintf("%s:%s/%s", remoteName, prefix, dirhash), nil+ case layoutModeDirectory:+ return fmt.Sprintf("%s:%s/%s%s", remoteName, prefix, dirhash, key), nil+ case layoutModeMixed:+ return fmt.Sprintf("%s:%s/%s", remoteName, prefix, dirhash), nil+ case layoutModeFrankencase:+ return fmt.Sprintf("%s:%s/%s", remoteName, prefix, strings.ToLower(dirhash)), nil+ default:+ panic("unreachable")+ }+}