"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
fs: fix passwords and tokens appearing in the debug log during rclone config
Nick Craig-Wood8b812fff28cf
diff --git a/docs/content/docs.md b/docs/content/docs.mdindex 517b1942a..a402bbb13 100644--- a/docs/content/docs.md+++ b/docs/content/docs.md@@ -3267,7 +3267,9 @@ The available flags are:- `auth` dumps HTTP headers like `headers`, but also includes any `Authorization:`headers. This means the output will probably contain sensitive information.Use `headers` to dump without `Authorization:` headers. Can be very verbose.- Useful for debugging only.+ Useful for debugging only. This flag also makes the debug log of the config+ process (e.g. `rclone config -vv`) show answers to questions, passwords and+ tokens which are otherwise redacted.- `bodies` dumps HTTP headers and bodies. May contain sensitive info.Can be very verbose. Useful for debugging only. Note that the bodiesare buffered in memory so don't use this for enormous files.diff --git a/fs/backend_config.go b/fs/backend_config.goindex c8d0e0ffe..aeb450074 100644--- a/fs/backend_config.go+++ b/fs/backend_config.go@@ -11,6 +11,7 @@ import ("slices""strconv""strings"+ "sync""github.com/rclone/rclone/fs/config/configmap")@@ -462,11 +463,61 @@ func configAll(ctx context.Context, name string, m configmap.Mapper, ri *RegInforeturn nil, fmt.Errorf("internal error: bad state %q", state)}+// redactDumpAuthWarnOnce makes sure the --dump auth warning is only shown once+var redactDumpAuthWarnOnce sync.Once++// RedactValue makes value safe for inclusion in the debug log.+//+// Non-empty values are replaced with "XXX" as they may contain+// secrets such as passwords or tokens, unless --dump auth is in use+// in which case the value is returned quoted.+func RedactValue(ci *ConfigInfo, value string) string {+ if value == "" || ci.Dump&DumpAuth != 0 {+ return fmt.Sprintf("%q", value)+ }+ return "XXX"+}++// RedactOptionValue is like RedactValue except that value is only+// redacted if opt is unknown (nil), a password or sensitive.+func RedactOptionValue(ci *ConfigInfo, opt *Option, value string) string {+ if opt == nil || opt.IsPassword || opt.Sensitive {+ return RedactValue(ci, value)+ }+ return fmt.Sprintf("%q", value)+}++// redactConfigOut renders out for the debug log, redacting any+// values which may contain secrets.+func redactConfigOut(ci *ConfigInfo, out *ConfigOut) string {+ if out == nil {+ return "<nil>"+ }+ var b strings.Builder+ fmt.Fprintf(&b, "{State:%q", out.State)+ if out.Option != nil {+ fmt.Fprintf(&b, " Option:%s=%s", out.Option.Name, RedactOptionValue(ci, out.Option, out.Option.String()))+ }+ if out.OAuth != nil {+ b.WriteString(" OAuth:set")+ }+ if out.Error != "" {+ fmt.Fprintf(&b, " Error:%q", out.Error)+ }+ fmt.Fprintf(&b, " Result:%s}", RedactValue(ci, out.Result))+ return b.String()+}+func backendConfigStep(ctx context.Context, name string, m configmap.Mapper, ri *RegInfo, choices configmap.Getter, in ConfigIn) (out *ConfigOut, err error) {ci := GetConfig(ctx)- Debugf(name, "config in: state=%q, result=%q", in.State, in.Result)+ if ci.Dump&DumpAuth != 0 {+ redactDumpAuthWarnOnce.Do(func() {+ Logf(nil, "--dump auth is in use - debug output may contain secrets such as passwords and tokens")+ })+ }+ Debugf(name, "config in: state=%q, result=%s", in.State, RedactValue(ci, in.Result))defer func() {- Debugf(name, "config out: out=%+v, err=%v", out, err)+ Debugf(name, "config out: out=%s, err=%v", redactConfigOut(ci, out), err)}()switch {@@ -510,13 +561,13 @@ func backendConfigStep(ctx context.Context, name string, m configmap.Mapper, ri}// If override value is set in the choices then use thatif result, ok := choices.Get(out.Option.Name); ok {- Debugf(nil, "Override value found, choosing value %q for state %q", result, out.State)+ Debugf(nil, "Override value found, choosing value %s for state %q", RedactOptionValue(ci, out.Option, result), out.State)return ConfigResult(out.State, result)}// If AutoConfirm is set, choose the default valueif ci.AutoConfirm {result := fmt.Sprint(out.Option.Default)- Debugf(nil, "Auto confirm is set, choosing default %q for state %q, override by setting config parameter %q", result, out.State, out.Option.Name)+ Debugf(nil, "Auto confirm is set, choosing default %s for state %q, override by setting config parameter %q", RedactOptionValue(ci, out.Option, result), out.State, out.Option.Name)return ConfigResult(out.State, result)}// If fs.ConfigEdit is set then make the default value@@ -527,7 +578,7 @@ func backendConfigStep(ctx context.Context, name string, m configmap.Mapper, rioldValue := newOption.Valueerr = newOption.Set(value)if err != nil {- Errorf(nil, "Failed to set %q from %q - using default: %v", out.Option.Name, value, err)+ Errorf(nil, "Failed to set %q from %s - using default: %v", out.Option.Name, RedactOptionValue(ci, newOption, value), err)} else {newOption.Default = newOption.ValuenewOption.Value = oldValuediff --git a/fs/backend_config_test.go b/fs/backend_config_test.goindex 9c373693f..e8d2c5e49 100644--- a/fs/backend_config_test.go+++ b/fs/backend_config_test.go@@ -37,6 +37,46 @@ func TestStatePop(t *testing.T) {assert.Equal(t, "a", state)}+func TestRedactValue(t *testing.T) {+ ci := &ConfigInfo{}+ assert.Equal(t, `""`, RedactValue(ci, ""))+ assert.Equal(t, "XXX", RedactValue(ci, "potato"))+ ci.Dump = DumpAuth+ assert.Equal(t, `""`, RedactValue(ci, ""))+ assert.Equal(t, `"potato"`, RedactValue(ci, "potato"))+}++func TestRedactOptionValue(t *testing.T) {+ ci := &ConfigInfo{}+ plain := &Option{Name: "chunk_size"}+ password := &Option{Name: "pass", IsPassword: true}+ sensitive := &Option{Name: "token", Sensitive: true}+ assert.Equal(t, `"potato"`, RedactOptionValue(ci, plain, "potato"))+ assert.Equal(t, "XXX", RedactOptionValue(ci, password, "potato"))+ assert.Equal(t, "XXX", RedactOptionValue(ci, sensitive, "potato"))+ assert.Equal(t, "XXX", RedactOptionValue(ci, nil, "potato"))+ assert.Equal(t, `""`, RedactOptionValue(ci, password, ""))+ ci.Dump = DumpAuth+ assert.Equal(t, `"potato"`, RedactOptionValue(ci, password, "potato"))+ assert.Equal(t, `"potato"`, RedactOptionValue(ci, nil, "potato"))+}++func TestRedactConfigOut(t *testing.T) {+ ci := &ConfigInfo{}+ assert.Equal(t, "<nil>", redactConfigOut(ci, nil))+ assert.Equal(t, `{State:"state" Result:""}`, redactConfigOut(ci, &ConfigOut{State: "state"}))+ out := &ConfigOut{+ State: "state",+ Option: &Option{Name: "pass", IsPassword: true, Value: "obscured"},+ OAuth: struct{}{},+ Error: "boom",+ Result: "secret",+ }+ assert.Equal(t, `{State:"state" Option:pass=XXX OAuth:set Error:"boom" Result:XXX}`, redactConfigOut(ci, out))+ ci.Dump = DumpAuth+ assert.Equal(t, `{State:"state" Option:pass="obscured" OAuth:set Error:"boom" Result:"secret"}`, redactConfigOut(ci, out))+}+func TestMatchProvider(t *testing.T) {for _, test := range []struct {config stringdiff --git a/lib/oauthutil/oauthutil.go b/lib/oauthutil/oauthutil.goindex 99ac0228f..37d2f11cc 100644--- a/lib/oauthutil/oauthutil.go+++ b/lib/oauthutil/oauthutil.go@@ -716,8 +716,9 @@ version recommended):// Find the overridden optionsinM := ri.Options.NonDefault(m)delete(inM, fs.ConfigToken) // delete token as we are refreshing it+ ci := fs.GetConfig(ctx)for k, v := range inM {- fs.Debugf(nil, "sending %s = %q", k, v)+ fs.Debugf(nil, "sending %s = %s", k, fs.RedactOptionValue(ci, ri.Options.Get(k), v))}// Encode them into a stringmCopyString, err := inM.Encode()@@ -748,9 +749,10 @@ version recommended):}// Save the config updatesif newFormat {+ ci := fs.GetConfig(ctx)for k, v := range outM {m.Set(k, v)- fs.Debugf(nil, "received %s = %q", k, v)+ fs.Debugf(nil, "received %s = %s", k, fs.RedactOptionValue(ci, ri.Options.Get(k), v))}} else {m.Set(fs.ConfigToken, code)