Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

configstruct: add SetAny to parse config from the rc

Nick Craig-Wood21e5fa192aaa
diff --git a/fs/config/configstruct/configstruct.go b/fs/config/configstruct/configstruct.go
index 7ad3f68b8..f90806a2d 100644
--- a/fs/config/configstruct/configstruct.go
+++ b/fs/config/configstruct/configstruct.go
@@ -86,6 +86,62 @@ func StringToInterface(def any, in string) (newValue any, err error) {
return newValue, nil
}
+// InterfaceToString turns in into a string
+//
+// This supports a subset of builtin types, string, integer types,
+// bool, time.Duration and []string.
+//
+// Builtin types are expected to be encoding as their natural
+// stringificatons as produced by fmt.Sprint except for []string which
+// is expected to be encoded a a CSV with empty array encoded as "".
+//
+// Any other types are expected to be encoded by their String()
+// methods and decoded by their `Set(s string) error` methods.
+func InterfaceToString(in any) (strValue string, err error) {
+ switch x := in.(type) {
+ case string:
+ // return strings unmodified
+ strValue = x
+ case int, int8, int16, int32, int64,
+ uint, uint8, uint16, uint32, uint64, uintptr,
+ float32, float64:
+ strValue = fmt.Sprint(in)
+ case bool:
+ strValue = fmt.Sprint(in)
+ case time.Duration:
+ strValue = fmt.Sprint(in)
+ case []string:
+ // CSV encode arrays of strings - ideally we would use
+ // fs.CommaSepList here but we can't as it would cause
+ // a circular import.
+ if len(x) == 0 {
+ strValue = ""
+ } else if len(x) == 1 && len(x[0]) == 0 {
+ strValue = `""`
+ } else {
+ var buf strings.Builder
+ w := csv.NewWriter(&buf)
+ err := w.Write(x)
+ if err != nil {
+ return "", err
+ }
+ w.Flush()
+ strValue = strings.TrimSpace(buf.String())
+ }
+ default:
+ // Try using a String method
+ if do, ok := in.(fmt.Stringer); ok {
+ strValue = do.String()
+ } else {
+ err = errors.New("don't know how to convert this")
+ }
+ }
+ if err != nil {
+ return "", fmt.Errorf("interpreting %T as string failed: %w", in, err)
+ }
+ return strValue, nil
+}
+
// Item describes a single entry in the options structure
type Item struct {
Name string // snake_case
@@ -157,6 +213,22 @@ func Items(opt any) (items []Item, err error) {
return items, nil
}
+// setValue sets newValue to configValue returning an updated newValue
+func setValue(newValue any, configValue string) (any, error) {
+ newNewValue, err := StringToInterface(newValue, configValue)
+ if err != nil {
+ // Mask errors if setting an empty string as
+ // it isn't valid for all types. This makes
+ // empty string be the equivalent of unset.
+ if configValue != "" {
+ return nil, err
+ }
+ } else {
+ newValue = newNewValue
+ }
+ return newValue, nil
+}
+
// Set interprets the field names in defaults and looks up config
// values in the config passed in. Any values found in config will be
// set in the opt structure.
@@ -178,17 +250,60 @@ func Set(config configmap.Getter, opt any) (err error) {
for _, defaultItem := range defaultItems {
newValue := defaultItem.Value
if configValue, ok := config.Get(defaultItem.Name); ok {
- var newNewValue any
- newNewValue, err = StringToInterface(newValue, configValue)
+ newValue, err = setValue(newValue, configValue)
if err != nil {
- // Mask errors if setting an empty string as
- // it isn't valid for all types. This makes
- // empty string be the equivalent of unset.
- if configValue != "" {
- return fmt.Errorf("couldn't parse config item %q = %q as %T: %w", defaultItem.Name, configValue, defaultItem.Value, err)
+ return fmt.Errorf("couldn't parse config item %q = %q as %T: %w", defaultItem.Name, configValue, defaultItem.Value, err)
+ }
+ }
+ defaultItem.Set(newValue)
+ }
+ return nil
+}
+
+// setIfSameType set aPtr with b if they are the same type or returns false.
+func setIfSameType(aPtr interface{}, b interface{}) bool {
+ aVal := reflect.ValueOf(aPtr).Elem()
+ bVal := reflect.ValueOf(b)
+
+ if aVal.Type() != bVal.Type() {
+ return false
+ }
+ aVal.Set(bVal)
+ return true
+}
+
+// SetAny interprets the field names in defaults and looks up config
+// values in the config passed in. Any values found in config will be
+// set in the opt structure.
+//
+// opt must be a pointer to a struct. The struct should have entirely
+// public fields. The field names are converted from CamelCase to
+// snake_case and looked up in the config supplied or a
+// `config:"field_name"` is looked up.
+//
+// If items are found then they are set directly if the correct type,
+// otherwise they are converted to string and then converted from
+// string to native types and set in opt.
+//
+// All the field types in the struct must implement fmt.Scanner.
+func SetAny(config map[string]any, opt any) (err error) {
+ defaultItems, err := Items(opt)
+ if err != nil {
+ return err
+ }
+ for _, defaultItem := range defaultItems {
+ newValue := defaultItem.Value
+ if configValue, ok := config[defaultItem.Name]; ok {
+ if !setIfSameType(&newValue, configValue) {
+ // Convert the config value to be a string
+ stringConfigValue, err := InterfaceToString(configValue)
+ if err != nil {
+ return err
+ }
+ newValue, err = setValue(newValue, stringConfigValue)
+ if err != nil {
+ return fmt.Errorf("couldn't parse config item %q = %q as %T: %w", defaultItem.Name, stringConfigValue, defaultItem.Value, err)
}
- } else {
- newValue = newNewValue
}
}
defaultItem.Set(newValue)
diff --git a/fs/config/configstruct/configstruct_test.go b/fs/config/configstruct/configstruct_test.go
index 4e42a661b..fb0337220 100644
--- a/fs/config/configstruct/configstruct_test.go
+++ b/fs/config/configstruct/configstruct_test.go
@@ -176,6 +176,39 @@ func TestSetFull(t *testing.T) {
assert.Equal(t, want, in)
}
+func TestSetAnyFull(t *testing.T) {
+ in := &Conf2{
+ PotatoPie: "yum",
+ BeanStew: true,
+ RaisinRoll: 42,
+ SausageOnStick: 101,
+ ForbiddenFruit: 6,
+ CookingTime: fs.Duration(42 * time.Second),
+ TotalWeight: fs.SizeSuffix(17 << 20),
+ }
+ m := map[string]any{
+ "spud_pie": "YUM",
+ "bean_stew": false,
+ "raisin_roll": "43 ",
+ "sausage_on_stick": " 102 ",
+ "forbidden_fruit": "0x7",
+ "cooking_time": 43 * time.Second,
+ "total_weight": "18M",
+ }
+ want := &Conf2{
+ PotatoPie: "YUM",
+ BeanStew: false,
+ RaisinRoll: 43,
+ SausageOnStick: 102,
+ ForbiddenFruit: 7,
+ CookingTime: fs.Duration(43 * time.Second),
+ TotalWeight: fs.SizeSuffix(18 << 20),
+ }
+ err := configstruct.SetAny(m, in)
+ require.NoError(t, err)
+ assert.Equal(t, want, in)
+}
+
func TestStringToInterface(t *testing.T) {
item := struct{ A int }{2}
for _, test := range []struct {
@@ -227,3 +260,47 @@ func TestStringToInterface(t *testing.T) {
}
}
}
+
+func TestInterfaceToString(t *testing.T) {
+ item := struct{ A int }{2}
+ for _, test := range []struct {
+ in any
+ want string
+ err string
+ }{
+ {nil, "", "interpreting <nil> as string failed: don't know how to convert this"},
+ {"", "", ""},
+ {" string ", " string ", ""},
+ {int(123), "123", ""},
+ {int(0x123), "291", ""},
+ {int(-123), "-123", ""},
+ {false, "false", ""},
+ {true, "true", ""},
+ {uint(123), "123", ""},
+ {int64(123), "123", ""},
+ {item, "", "interpreting struct { A int } as string failed: don't know how to convert this"},
+ {fs.Duration(time.Second), "1s", ""},
+ {fs.Duration(61 * time.Second), "1m1s", ""},
+ {[]string{}, ``, ""},
+ {[]string{""}, `""`, ""},
+ {[]string{"", ""}, `,`, ""},
+ {[]string{"hello"}, `hello`, ""},
+ {[]string{"hello", "world"}, `hello,world`, ""},
+ {[]string{"hello", "", "world"}, `hello,,world`, ""},
+ {[]string{`hello, world`, `goodbye, world!`}, `"hello, world","goodbye, world!"`, ""},
+ {time.Second, "1s", ""},
+ {61 * time.Second, "1m1s", ""},
+ {fs.Mebi, "1Mi", ""},
+ {fs.Gibi, "1Gi", ""},
+ } {
+ what := fmt.Sprintf("interpret %#v as string", test.in)
+ got, err := configstruct.InterfaceToString(test.in)
+ if test.err == "" {
+ require.NoError(t, err, what)
+ assert.Equal(t, test.want, got, what)
+ } else {
+ assert.Equal(t, "", got)
+ assert.EqualError(t, err, test.err, what)
+ }
+ }
+}