Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

cmd/gitannex: Add support for different layouts

Dan McArdle29b58dd4c5db
diff --git a/cmd/gitannex/gitannex.go b/cmd/gitannex/gitannex.go
index a71b04871..0eb09da12 100644
--- a/cmd/gitannex/gitannex.go
+++ b/cmd/gitannex/gitannex.go
@@ -110,9 +110,10 @@ func (m *messageParser) finalParameter() string {
// configDefinition describes a configuration value required by this command. We
// use "GETCONFIG" messages to query git-annex for these values at runtime.
type configDefinition struct {
- name string
- description string
- destination *string
+ name string
+ description string
+ destination *string
+ defaultValue *string
}
// server contains this command's current state.
@@ -132,6 +133,7 @@ type server struct {
configsDone bool
configPrefix string
configRcloneRemoteName string
+ configRcloneLayout string
}
func (s *server) sendMsg(msg string) {
@@ -267,6 +269,9 @@ func (s *server) handleInitRemote() error {
// Get a list of configs with pointers to fields of `s`.
func (s *server) getRequiredConfigs() []configDefinition {
+ defaultRclonePrefix := "git-annex-rclone"
+ defaultRcloneLayout := "nodir"
+
return []configDefinition{
{
"rcloneremotename",
@@ -274,13 +279,23 @@ func (s *server) getRequiredConfigs() []configDefinition {
"Must match a remote known to rclone. " +
"(Note that rclone remotes are a distinct concept from git-annex remotes.)",
&s.configRcloneRemoteName,
+ nil,
},
{
"rcloneprefix",
"Directory where rclone will write git-annex content. " +
- "If not specified, defaults to \"git-annex-rclone\". " +
- "This directory be created on init if it does not exist.",
+ fmt.Sprintf("If not specified, defaults to %q. ", defaultRclonePrefix) +
+ "This directory will be created on init if it does not exist.",
&s.configPrefix,
+ &defaultRclonePrefix,
+ },
+ {
+ "rclonelayout",
+ "Defines where, within the rcloneprefix directory, rclone will write git-annex content. " +
+ fmt.Sprintf("Must be one of %v. ", allLayoutModes()) +
+ fmt.Sprintf("If empty, defaults to %q.", defaultRcloneLayout),
+ &s.configRcloneLayout,
+ &defaultRcloneLayout,
},
}
}
@@ -307,10 +322,13 @@ func (s *server) queryConfigs() error {
}
value := message.finalParameter()
- if value == "" {
+ if value == "" && config.defaultValue == nil {
return fmt.Errorf("config value of %q must not be empty", config.name)
}
-
+ if value == "" {
+ *config.destination = *config.defaultValue
+ continue
+ }
*config.destination = value
}
@@ -358,7 +376,19 @@ func (s *server) handleTransfer(message *messageParser) error {
return fmt.Errorf("error getting configs: %w", err)
}
- remoteFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:%s", s.configRcloneRemoteName, s.configPrefix))
+ layout := parseLayoutMode(s.configRcloneLayout)
+ if layout == layoutModeUnknown {
+ s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s", argKey))
+ return fmt.Errorf("error parsing layout mode: %q", s.configRcloneLayout)
+ }
+
+ remoteFsString, err := buildFsString(s.queryDirhash, layout, argKey, s.configRcloneRemoteName, s.configPrefix)
+ if err != nil {
+ s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s", argKey))
+ return fmt.Errorf("error building fs string: %w", err)
+ }
+
+ remoteFs, err := cache.Get(context.TODO(), remoteFsString)
if err != nil {
s.sendMsg(fmt.Sprintf("TRANSFER-FAILURE %s %s failed to get remote fs", argMode, argKey))
return err
@@ -415,7 +445,19 @@ func (s *server) handleCheckPresent(message *messageParser) error {
return fmt.Errorf("error getting configs: %s", err)
}
- remoteFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:%s", s.configRcloneRemoteName, s.configPrefix))
+ layout := parseLayoutMode(s.configRcloneLayout)
+ if layout == layoutModeUnknown {
+ s.sendMsg(fmt.Sprintf("CHECKPRESENT-FAILURE %s", argKey))
+ return fmt.Errorf("error parsing layout mode: %q", s.configRcloneLayout)
+ }
+
+ remoteFsString, err := buildFsString(s.queryDirhash, layout, argKey, s.configRcloneRemoteName, s.configPrefix)
+ if err != nil {
+ s.sendMsg(fmt.Sprintf("CHECKPRESENT-FAILURE %s", argKey))
+ return fmt.Errorf("error building fs string: %w", err)
+ }
+
+ remoteFs, err := cache.Get(context.TODO(), remoteFsString)
if err != nil {
s.sendMsg(fmt.Sprintf("CHECKPRESENT-UNKNOWN %s failed to get remote fs", argKey))
return err
@@ -435,17 +477,45 @@ func (s *server) handleCheckPresent(message *messageParser) error {
return nil
}
+func (s *server) queryDirhash(msg string) (string, error) {
+ s.sendMsg(msg)
+ parser, err := s.getMsg()
+ if err != nil {
+ return "", err
+ }
+ keyword, err := parser.nextSpaceDelimitedParameter()
+ if err != nil {
+ return "", err
+ }
+ if keyword != "VALUE" {
+ return "", fmt.Errorf("expected VALUE keyword, but got %q", keyword)
+ }
+ dirhash, err := parser.nextSpaceDelimitedParameter()
+ if err != nil {
+ return "", fmt.Errorf("failed to parse dirhash: %w", err)
+ }
+ return dirhash, nil
+}
+
func (s *server) handleRemove(message *messageParser) error {
argKey := message.finalParameter()
if argKey == "" {
return errors.New("failed to parse key for REMOVE")
}
+ layout := parseLayoutMode(s.configRcloneLayout)
+ if layout == layoutModeUnknown {
+ s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s", argKey))
+ return fmt.Errorf("error parsing layout mode: %q", s.configRcloneLayout)
+ }
+
+ remoteFsString, err := buildFsString(s.queryDirhash, layout, argKey, s.configRcloneRemoteName, s.configPrefix)
if err != nil {
- return err
+ s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s", argKey))
+ return fmt.Errorf("error building fs string: %w", err)
}
- remoteFs, err := cache.Get(context.TODO(), fmt.Sprintf("%s:%s", s.configRcloneRemoteName, s.configPrefix))
+ remoteFs, err := cache.Get(context.TODO(), remoteFsString)
if err != nil {
s.sendMsg(fmt.Sprintf("REMOVE-FAILURE %s", argKey))
return fmt.Errorf("error getting remote fs: %w", err)
diff --git a/cmd/gitannex/gitannex_test.go b/cmd/gitannex/gitannex_test.go
index b3846221d..2b023b8c0 100644
--- a/cmd/gitannex/gitannex_test.go
+++ b/cmd/gitannex/gitannex_test.go
@@ -234,6 +234,7 @@ func (h *testState) requireWriteLine(line string) {
func (h *testState) preconfigureServer() {
h.server.configPrefix = h.localFsDir
h.server.configRcloneRemoteName = h.remoteName
+ h.server.configRcloneLayout = string(layoutModeNodir)
h.server.configsDone = true
}
@@ -285,6 +286,8 @@ var localBackendTestCases = []testCase{
h.requireWriteLine("VALUE " + h.remoteName)
h.requireReadLineExact("GETCONFIG rcloneprefix")
h.requireWriteLine("VALUE " + h.localFsDir)
+ h.requireReadLineExact("GETCONFIG rclonelayout")
+ h.requireWriteLine("VALUE foo")
h.requireReadLineExact("PREPARE-SUCCESS")
require.Equal(t, h.server.configRcloneRemoteName, h.remoteName)
@@ -313,9 +316,13 @@ var localBackendTestCases = []testCase{
localFsDirWithSpaces := fmt.Sprintf(" %s\t", h.localFsDir)
h.requireWriteLine(fmt.Sprintf("VALUE %s", remoteNameWithSpaces))
- h.requireReadLineExact("GETCONFIG rcloneprefix")
+ h.requireReadLineExact("GETCONFIG rcloneprefix")
h.requireWriteLine(fmt.Sprintf("VALUE %s", localFsDirWithSpaces))
+
+ h.requireReadLineExact("GETCONFIG rclonelayout")
+ h.requireWriteLine("VALUE")
+
h.requireReadLineExact("PREPARE-SUCCESS")
require.Equal(t, h.server.configRcloneRemoteName, remoteNameWithSpaces)
@@ -367,11 +374,11 @@ var localBackendTestCases = []testCase{
// Note the whitespace following the key.
h.requireWriteLine("TRANSFER STORE Key ")
- h.requireReadLineExact("TRANSFER-FAILURE failed to parse file")
+ h.requireReadLineExact("TRANSFER-FAILURE failed to parse file path")
require.NoError(t, h.mockStdinW.Close())
},
- expectedError: "malformed arguments for TRANSFER: nothing remains to parse",
+ expectedError: "failed to parse file",
},
// Repeated EXTENSIONS messages add to each other rather than overriding
// prior advertised extensions. This behavior is not mandated by the
diff --git a/cmd/gitannex/layout.go b/cmd/gitannex/layout.go
new file mode 100644
index 000000000..81b18bfb6
--- /dev/null
+++ b/cmd/gitannex/layout.go
@@ -0,0 +1,72 @@
+package gitannex
+
+import (
+ "fmt"
+ "strings"
+)
+
+type layoutMode string
+
+// All layout modes from git-annex-remote-rclone are supported.
+const (
+ layoutModeLower layoutMode = "lower"
+ layoutModeDirectory layoutMode = "directory"
+ layoutModeNodir layoutMode = "nodir"
+ layoutModeMixed layoutMode = "mixed"
+ layoutModeFrankencase layoutMode = "frankencase"
+ layoutModeUnknown layoutMode = ""
+)
+
+func allLayoutModes() []layoutMode {
+ return []layoutMode{
+ layoutModeLower,
+ layoutModeDirectory,
+ layoutModeNodir,
+ layoutModeMixed,
+ layoutModeFrankencase,
+ }
+}
+
+func parseLayoutMode(mode string) layoutMode {
+ for _, knownMode := range allLayoutModes() {
+ if mode == string(knownMode) {
+ return knownMode
+ }
+ }
+ return layoutModeUnknown
+}
+
+type queryDirhashFunc func(msg string) (string, error)
+
+func buildFsString(queryDirhash queryDirhashFunc, mode layoutMode, key, remoteName, prefix string) (string, error) {
+ if mode == layoutModeNodir {
+ return fmt.Sprintf("%s:%s", remoteName, prefix), nil
+ }
+
+ var dirhash string
+ var err error
+ switch mode {
+ case layoutModeLower, layoutModeDirectory:
+ dirhash, err = queryDirhash("DIRHASH-LOWER " + key)
+ case layoutModeMixed, layoutModeFrankencase:
+ dirhash, err = queryDirhash("DIRHASH " + key)
+ default:
+ panic("unreachable")
+ }
+ if err != nil {
+ return "", fmt.Errorf("buildFsString failed to query dirhash: %w", err)
+ }
+
+ switch mode {
+ case layoutModeLower:
+ return fmt.Sprintf("%s:%s/%s", remoteName, prefix, dirhash), nil
+ case layoutModeDirectory:
+ return fmt.Sprintf("%s:%s/%s%s", remoteName, prefix, dirhash, key), nil
+ case layoutModeMixed:
+ return fmt.Sprintf("%s:%s/%s", remoteName, prefix, dirhash), nil
+ case layoutModeFrankencase:
+ return fmt.Sprintf("%s:%s/%s", remoteName, prefix, strings.ToLower(dirhash)), nil
+ default:
+ panic("unreachable")
+ }
+}