"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
fshttp: refuse HTTPS to HTTP redirects unless --allow-https-downgrade GHSA-m97j-7x65-wh93 CVE-PENDING
Nick Craig-Woodb4659519dfe2
diff --git a/backend/http/http.go b/backend/http/http.goindex a88eefcff..ce7c85f79 100644--- a/backend/http/http.go+++ b/backend/http/http.go@@ -63,7 +63,8 @@ The headers are only sent to the host in the configured URL. If theserver redirects to another host (including a subdomain or a differentport) the headers are not sent to it, or to any further hop in thatredirect chain. When headers are set, a redirect from https to http is-refused as it would send them in cleartext.`,+refused as it would send them in cleartext, even with+--allow-https-downgrade.`,Default: fs.CommaSepList{},Advanced: true,}, {@@ -315,8 +316,9 @@ func (f *Fs) httpConnection(ctx context.Context, opt *Options) (isFile bool, err}client := fshttp.NewClient(ctx)- // Without configured headers keep the default policy which- // public mirrors that redirect from https to http rely on.+ // Without configured headers keep the default policy so+ // --allow-https-downgrade can follow public mirrors that+ // redirect from https to http.if len(opt.Headers) > 0 {client.CheckRedirect = checkRedirect(opt)}diff --git a/backend/http/http_internal_test.go b/backend/http/http_internal_test.goindex 7faabc60e..2be76a4dc 100644--- a/backend/http/http_internal_test.go+++ b/backend/http/http_internal_test.go@@ -688,7 +688,7 @@ func TestRedirectRefusesHTTPSDowngrade(t *testing.T) {// open makes a remote pointing at ts with or without headers// and opens file.txt on it- open := func(t *testing.T, withHeaders bool) (io.ReadCloser, error) {+ open := func(t *testing.T, withHeaders, allowDowngrade bool) (io.ReadCloser, error) {configfile.Install()m := configmap.Simple{"type": "http",@@ -700,6 +700,7 @@ func TestRedirectRefusesHTTPSDowngrade(t *testing.T) {}ctx, ci := fs.AddConfig(context.Background())ci.InsecureSkipVerify = true+ ci.AllowHTTPSDowngrade = allowDowngradef, err := NewFs(ctx, remoteName, "", m)require.NoError(t, err)@@ -711,13 +712,25 @@ func TestRedirectRefusesHTTPSDowngrade(t *testing.T) {t.Run("WithHeaders", func(t *testing.T) {// Headers could leak so the downgrade must be refused- _, err := open(t, true)+ _, err := open(t, true, false)+ require.ErrorIs(t, err, rest.ErrHTTPSDowngrade)+ })++ t.Run("WithHeadersAllowDowngrade", func(t *testing.T) {+ // Headers could leak so the downgrade is refused even when allowed+ _, err := open(t, true, true)require.ErrorIs(t, err, rest.ErrHTTPSDowngrade)})t.Run("WithoutHeaders", func(t *testing.T) {- // Nothing to leak so the downgrade is followed- fd, err := open(t, false)+ // The transport refuses downgrades by default+ _, err := open(t, false, false)+ require.ErrorIs(t, err, rest.ErrHTTPSDowngrade)+ })++ t.Run("WithoutHeadersAllowDowngrade", func(t *testing.T) {+ // Nothing to leak so the downgrade is followed when allowed+ fd, err := open(t, false, true)require.NoError(t, err)data, err := io.ReadAll(fd)require.NoError(t, err)diff --git a/docs/content/docs.md b/docs/content/docs.mdindex a2b681be9..bcbcf00ff 100644--- a/docs/content/docs.md+++ b/docs/content/docs.md@@ -494,6 +494,7 @@ Not all global variables make sense to be overridden like this as the config isonly applied during the remote creation. Here is a non exhaustive list of oneswhich might be useful:+- `allow_https_downgrade`- `bind_addr`- `ca_cert`- `client_cert`@@ -995,6 +996,21 @@ See also [--human-readable](#human-readable).## Main options+### --allow-https-downgrade++By default rclone refuses to follow a redirect from an `https://` URL+to a plain `http://` URL, as any credentials or other headers in the+request would then be sent unencrypted, and the response could be+altered by anyone on the network path. The request fails with an error+which mentions this flag.++Set `--allow-https-downgrade` to follow such redirects, for example+when downloading from a mirror which redirects to plain HTTP. Some+backends still refuse to follow them, as doing so would send their+credentials in cleartext.++This option defaults to `false`.+### --backup-dir stringWhen using [sync](/commands/rclone_sync/), [copy](/commands/rclone_copy/) ordiff --git a/fs/config.go b/fs/config.goindex cafca05ad..9955c374e 100644--- a/fs/config.go+++ b/fs/config.go@@ -147,6 +147,11 @@ var ConfigOptionsInfo = Options{{Default: false,Help: "Do not verify the server SSL certificate (insecure)",Groups: "Networking",+}, {+ Name: "allow_https_downgrade",+ Default: false,+ Help: "Follow redirects from HTTPS to HTTP (insecure)",+ Groups: "Networking",}, {Name: "ask_password",Default: true,@@ -605,6 +610,7 @@ type ConfigInfo struct {ExpectContinueTimeout Duration `config:"expect_continue_timeout"`Dump DumpFlags `config:"dump"`InsecureSkipVerify bool `config:"no_check_certificate"` // Skip server certificate verification+ AllowHTTPSDowngrade bool `config:"allow_https_downgrade"`DeleteMode DeleteMode `config:"delete_mode"`MaxDelete int64 `config:"max_delete"`MaxDeleteSize SizeSuffix `config:"max_delete_size"`diff --git a/fs/fshttp/http.go b/fs/fshttp/http.goindex 1b2572f2e..45f09e670 100644--- a/fs/fshttp/http.go+++ b/fs/fshttp/http.go@@ -616,8 +616,32 @@ func redirectLeavesHost(req *http.Request) bool {return false}+// isHTTPSDowngrade reports whether req is a plaintext HTTP redirect+// hop in a chain whose original request was HTTPS.+func isHTTPSDowngrade(req *http.Request) bool {+ if req.URL.Scheme != "http" {+ return false+ }+ origin := req+ for origin.Response != nil && origin.Response.Request != nil {+ origin = origin.Response.Request+ }+ return origin.URL.Scheme == "https"+}+// RoundTrip implements the RoundTripper interface.func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error) {+ // Refuse HTTPS to HTTP redirects here rather than in+ // CheckRedirect since backends and libraries replace the client's+ // CheckRedirect but still use this transport, and any credentials+ // they set would be sent in cleartext.+ if !t.ci.AllowHTTPSDowngrade && isHTTPSDowngrade(req) {+ if req.Body != nil {+ _ = req.Body.Close()+ }+ return nil, fmt.Errorf("%w: redirect to %s (use --allow-https-downgrade to follow it)", rest.ErrHTTPSDowngrade, req.URL.Redacted())+ }+// Check if certificates are being used and the certificates are expiredif isCertificateExpired(t.TLSClientConfig) {t.reloadCertificates()diff --git a/fs/fshttp/http_test.go b/fs/fshttp/http_test.goindex 3bb509c67..1fd2d7cb4 100644--- a/fs/fshttp/http_test.go+++ b/fs/fshttp/http_test.go@@ -19,6 +19,7 @@ import ("time""github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/lib/rest""github.com/stretchr/testify/assert""github.com/stretchr/testify/require""moul.io/http2curl/v2"@@ -301,7 +302,7 @@ func TestRedirectLeavesHost(t *testing.T) {// TestRedirectStripsGlobalHeadersOnDowngrade checks the headers set// with --header are not sent over plaintext HTTP when an HTTPS// request is redirected to an http:// URL with the same host and-// port.+// port and --allow-https-downgrade permits the redirect.//// The https:// request is made to the plain server's host and port// with a TLS dialer which ignores the address and connects to the@@ -322,6 +323,7 @@ func TestRedirectStripsGlobalHeadersOnDowngrade(t *testing.T) {ctx, ci := fs.AddConfig(context.Background())ci.Headers = []*fs.HTTPOption{{Key: "X-Potato", Value: "sausage"}}+ ci.AllowHTTPSDowngrade = trueclient := NewClientCustom(ctx, func(tr *http.Transport) {tr.DialTLSContext = func(ctx context.Context, network, _ string) (net.Conn, error) {return (&tls.Dialer{Config: tlsConfig}).DialContext(ctx, network, secure.Listener.Addr().String())@@ -336,3 +338,88 @@ func TestRedirectStripsGlobalHeadersOnDowngrade(t *testing.T) {assert.Equal(t, "sausage", gotTLS.Get("X-Potato"))assert.Equal(t, "", gotPlain.Get("X-Potato"))}++// TestRefuseHTTPSDowngrade checks the transport refuses to follow a+// redirect from HTTPS to HTTP unless --allow-https-downgrade is set,+// whatever CheckRedirect policy the client uses.+func TestRefuseHTTPSDowngrade(t *testing.T) {+ var plainHits int+ plain := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ plainHits+++ }))+ defer plain.Close()+ secure := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch r.URL.Path {+ case "/downgrade":+ http.Redirect(w, r, plain.URL+"/final", http.StatusFound)+ case "/secure":+ http.Redirect(w, r, "/final", http.StatusFound)+ }+ }))+ defer secure.Close()+ tlsConfig := secure.Client().Transport.(*http.Transport).TLSClientConfig++ for _, test := range []struct {+ name string+ allow bool+ checkRedirect func(req *http.Request, via []*http.Request) error+ path string+ wantErr bool+ wantPlainHits int+ }{+ {name: "Default", path: "/downgrade", wantErr: true},+ {name: "CustomCheckRedirect", checkRedirect: func(*http.Request, []*http.Request) error { return nil }, path: "/downgrade", wantErr: true},+ {name: "Allowed", allow: true, path: "/downgrade", wantPlainHits: 1},+ {name: "HTTPSToHTTPS", path: "/secure"},+ } {+ t.Run(test.name, func(t *testing.T) {+ plainHits = 0+ ctx, ci := fs.AddConfig(context.Background())+ ci.AllowHTTPSDowngrade = test.allow+ client := NewClientCustom(ctx, func(tr *http.Transport) {+ tr.TLSClientConfig = tlsConfig+ })+ if test.checkRedirect != nil {+ client.CheckRedirect = test.checkRedirect+ }+ resp, err := client.Get(secure.URL + test.path)+ if test.wantErr {+ require.Error(t, err)+ assert.ErrorIs(t, err, rest.ErrHTTPSDowngrade)+ assert.Contains(t, err.Error(), "--allow-https-downgrade")+ } else {+ require.NoError(t, err)+ require.NoError(t, resp.Body.Close())+ }+ assert.Equal(t, test.wantPlainHits, plainHits)+ })+ }+}++// TestRefuseHTTPSDowngradeClosesBody checks the transport closes the+// request body when it refuses a redirect, as RoundTrip must.+func TestRefuseHTTPSDowngradeClosesBody(t *testing.T) {+ body := &closeRecorder{Reader: strings.NewReader("potato")}+ req, err := http.NewRequest(http.MethodPost, "http://a.test/x", body)+ require.NoError(t, err)+ origin, err := http.NewRequest(http.MethodPost, "https://a.test/", nil)+ require.NoError(t, err)+ req.Response = &http.Response{Request: origin}++ ctx, _ := fs.AddConfig(context.Background())+ _, err = NewTransport(ctx).RoundTrip(req)+ assert.ErrorIs(t, err, rest.ErrHTTPSDowngrade)+ assert.True(t, body.closed)+}++// closeRecorder is an io.ReadCloser which records whether it was closed+type closeRecorder struct {+ *strings.Reader+ closed bool+}++// Close records the close+func (c *closeRecorder) Close() error {+ c.closed = true+ return nil+}