Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

fshttp: refuse HTTPS to HTTP redirects unless --allow-https-downgrade GHSA-m97j-7x65-wh93 CVE-PENDING

Nick Craig-Woodb4659519dfe2
diff --git a/backend/http/http.go b/backend/http/http.go
index a88eefcff..ce7c85f79 100644
--- a/backend/http/http.go
+++ b/backend/http/http.go
@@ -63,7 +63,8 @@ The headers are only sent to the host in the configured URL. If the
server redirects to another host (including a subdomain or a different
port) the headers are not sent to it, or to any further hop in that
redirect chain. When headers are set, a redirect from https to http is
-refused as it would send them in cleartext.`,
+refused as it would send them in cleartext, even with
+--allow-https-downgrade.`,
Default: fs.CommaSepList{},
Advanced: true,
}, {
@@ -315,8 +316,9 @@ func (f *Fs) httpConnection(ctx context.Context, opt *Options) (isFile bool, err
}
client := fshttp.NewClient(ctx)
- // Without configured headers keep the default policy which
- // public mirrors that redirect from https to http rely on.
+ // Without configured headers keep the default policy so
+ // --allow-https-downgrade can follow public mirrors that
+ // redirect from https to http.
if len(opt.Headers) > 0 {
client.CheckRedirect = checkRedirect(opt)
}
diff --git a/backend/http/http_internal_test.go b/backend/http/http_internal_test.go
index 7faabc60e..2be76a4dc 100644
--- a/backend/http/http_internal_test.go
+++ b/backend/http/http_internal_test.go
@@ -688,7 +688,7 @@ func TestRedirectRefusesHTTPSDowngrade(t *testing.T) {
// open makes a remote pointing at ts with or without headers
// and opens file.txt on it
- open := func(t *testing.T, withHeaders bool) (io.ReadCloser, error) {
+ open := func(t *testing.T, withHeaders, allowDowngrade bool) (io.ReadCloser, error) {
configfile.Install()
m := configmap.Simple{
"type": "http",
@@ -700,6 +700,7 @@ func TestRedirectRefusesHTTPSDowngrade(t *testing.T) {
}
ctx, ci := fs.AddConfig(context.Background())
ci.InsecureSkipVerify = true
+ ci.AllowHTTPSDowngrade = allowDowngrade
f, err := NewFs(ctx, remoteName, "", m)
require.NoError(t, err)
@@ -711,13 +712,25 @@ func TestRedirectRefusesHTTPSDowngrade(t *testing.T) {
t.Run("WithHeaders", func(t *testing.T) {
// Headers could leak so the downgrade must be refused
- _, err := open(t, true)
+ _, err := open(t, true, false)
+ require.ErrorIs(t, err, rest.ErrHTTPSDowngrade)
+ })
+
+ t.Run("WithHeadersAllowDowngrade", func(t *testing.T) {
+ // Headers could leak so the downgrade is refused even when allowed
+ _, err := open(t, true, true)
require.ErrorIs(t, err, rest.ErrHTTPSDowngrade)
})
t.Run("WithoutHeaders", func(t *testing.T) {
- // Nothing to leak so the downgrade is followed
- fd, err := open(t, false)
+ // The transport refuses downgrades by default
+ _, err := open(t, false, false)
+ require.ErrorIs(t, err, rest.ErrHTTPSDowngrade)
+ })
+
+ t.Run("WithoutHeadersAllowDowngrade", func(t *testing.T) {
+ // Nothing to leak so the downgrade is followed when allowed
+ fd, err := open(t, false, true)
require.NoError(t, err)
data, err := io.ReadAll(fd)
require.NoError(t, err)
diff --git a/docs/content/docs.md b/docs/content/docs.md
index a2b681be9..bcbcf00ff 100644
--- a/docs/content/docs.md
+++ b/docs/content/docs.md
@@ -494,6 +494,7 @@ Not all global variables make sense to be overridden like this as the config is
only applied during the remote creation. Here is a non exhaustive list of ones
which might be useful:
+- `allow_https_downgrade`
- `bind_addr`
- `ca_cert`
- `client_cert`
@@ -995,6 +996,21 @@ See also [--human-readable](#human-readable).
## Main options
+### --allow-https-downgrade
+
+By default rclone refuses to follow a redirect from an `https://` URL
+to a plain `http://` URL, as any credentials or other headers in the
+request would then be sent unencrypted, and the response could be
+altered by anyone on the network path. The request fails with an error
+which mentions this flag.
+
+Set `--allow-https-downgrade` to follow such redirects, for example
+when downloading from a mirror which redirects to plain HTTP. Some
+backends still refuse to follow them, as doing so would send their
+credentials in cleartext.
+
+This option defaults to `false`.
+
### --backup-dir string
When using [sync](/commands/rclone_sync/), [copy](/commands/rclone_copy/) or
diff --git a/fs/config.go b/fs/config.go
index cafca05ad..9955c374e 100644
--- a/fs/config.go
+++ b/fs/config.go
@@ -147,6 +147,11 @@ var ConfigOptionsInfo = Options{{
Default: false,
Help: "Do not verify the server SSL certificate (insecure)",
Groups: "Networking",
+}, {
+ Name: "allow_https_downgrade",
+ Default: false,
+ Help: "Follow redirects from HTTPS to HTTP (insecure)",
+ Groups: "Networking",
}, {
Name: "ask_password",
Default: true,
@@ -605,6 +610,7 @@ type ConfigInfo struct {
ExpectContinueTimeout Duration `config:"expect_continue_timeout"`
Dump DumpFlags `config:"dump"`
InsecureSkipVerify bool `config:"no_check_certificate"` // Skip server certificate verification
+ AllowHTTPSDowngrade bool `config:"allow_https_downgrade"`
DeleteMode DeleteMode `config:"delete_mode"`
MaxDelete int64 `config:"max_delete"`
MaxDeleteSize SizeSuffix `config:"max_delete_size"`
diff --git a/fs/fshttp/http.go b/fs/fshttp/http.go
index 1b2572f2e..45f09e670 100644
--- a/fs/fshttp/http.go
+++ b/fs/fshttp/http.go
@@ -616,8 +616,32 @@ func redirectLeavesHost(req *http.Request) bool {
return false
}
+// isHTTPSDowngrade reports whether req is a plaintext HTTP redirect
+// hop in a chain whose original request was HTTPS.
+func isHTTPSDowngrade(req *http.Request) bool {
+ if req.URL.Scheme != "http" {
+ return false
+ }
+ origin := req
+ for origin.Response != nil && origin.Response.Request != nil {
+ origin = origin.Response.Request
+ }
+ return origin.URL.Scheme == "https"
+}
+
// RoundTrip implements the RoundTripper interface.
func (t *Transport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
+ // Refuse HTTPS to HTTP redirects here rather than in
+ // CheckRedirect since backends and libraries replace the client's
+ // CheckRedirect but still use this transport, and any credentials
+ // they set would be sent in cleartext.
+ if !t.ci.AllowHTTPSDowngrade && isHTTPSDowngrade(req) {
+ if req.Body != nil {
+ _ = req.Body.Close()
+ }
+ return nil, fmt.Errorf("%w: redirect to %s (use --allow-https-downgrade to follow it)", rest.ErrHTTPSDowngrade, req.URL.Redacted())
+ }
+
// Check if certificates are being used and the certificates are expired
if isCertificateExpired(t.TLSClientConfig) {
t.reloadCertificates()
diff --git a/fs/fshttp/http_test.go b/fs/fshttp/http_test.go
index 3bb509c67..1fd2d7cb4 100644
--- a/fs/fshttp/http_test.go
+++ b/fs/fshttp/http_test.go
@@ -19,6 +19,7 @@ import (
"time"
"github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/lib/rest"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"moul.io/http2curl/v2"
@@ -301,7 +302,7 @@ func TestRedirectLeavesHost(t *testing.T) {
// TestRedirectStripsGlobalHeadersOnDowngrade checks the headers set
// with --header are not sent over plaintext HTTP when an HTTPS
// request is redirected to an http:// URL with the same host and
-// port.
+// port and --allow-https-downgrade permits the redirect.
//
// The https:// request is made to the plain server's host and port
// with a TLS dialer which ignores the address and connects to the
@@ -322,6 +323,7 @@ func TestRedirectStripsGlobalHeadersOnDowngrade(t *testing.T) {
ctx, ci := fs.AddConfig(context.Background())
ci.Headers = []*fs.HTTPOption{{Key: "X-Potato", Value: "sausage"}}
+ ci.AllowHTTPSDowngrade = true
client := NewClientCustom(ctx, func(tr *http.Transport) {
tr.DialTLSContext = func(ctx context.Context, network, _ string) (net.Conn, error) {
return (&tls.Dialer{Config: tlsConfig}).DialContext(ctx, network, secure.Listener.Addr().String())
@@ -336,3 +338,88 @@ func TestRedirectStripsGlobalHeadersOnDowngrade(t *testing.T) {
assert.Equal(t, "sausage", gotTLS.Get("X-Potato"))
assert.Equal(t, "", gotPlain.Get("X-Potato"))
}
+
+// TestRefuseHTTPSDowngrade checks the transport refuses to follow a
+// redirect from HTTPS to HTTP unless --allow-https-downgrade is set,
+// whatever CheckRedirect policy the client uses.
+func TestRefuseHTTPSDowngrade(t *testing.T) {
+ var plainHits int
+ plain := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ plainHits++
+ }))
+ defer plain.Close()
+ secure := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/downgrade":
+ http.Redirect(w, r, plain.URL+"/final", http.StatusFound)
+ case "/secure":
+ http.Redirect(w, r, "/final", http.StatusFound)
+ }
+ }))
+ defer secure.Close()
+ tlsConfig := secure.Client().Transport.(*http.Transport).TLSClientConfig
+
+ for _, test := range []struct {
+ name string
+ allow bool
+ checkRedirect func(req *http.Request, via []*http.Request) error
+ path string
+ wantErr bool
+ wantPlainHits int
+ }{
+ {name: "Default", path: "/downgrade", wantErr: true},
+ {name: "CustomCheckRedirect", checkRedirect: func(*http.Request, []*http.Request) error { return nil }, path: "/downgrade", wantErr: true},
+ {name: "Allowed", allow: true, path: "/downgrade", wantPlainHits: 1},
+ {name: "HTTPSToHTTPS", path: "/secure"},
+ } {
+ t.Run(test.name, func(t *testing.T) {
+ plainHits = 0
+ ctx, ci := fs.AddConfig(context.Background())
+ ci.AllowHTTPSDowngrade = test.allow
+ client := NewClientCustom(ctx, func(tr *http.Transport) {
+ tr.TLSClientConfig = tlsConfig
+ })
+ if test.checkRedirect != nil {
+ client.CheckRedirect = test.checkRedirect
+ }
+ resp, err := client.Get(secure.URL + test.path)
+ if test.wantErr {
+ require.Error(t, err)
+ assert.ErrorIs(t, err, rest.ErrHTTPSDowngrade)
+ assert.Contains(t, err.Error(), "--allow-https-downgrade")
+ } else {
+ require.NoError(t, err)
+ require.NoError(t, resp.Body.Close())
+ }
+ assert.Equal(t, test.wantPlainHits, plainHits)
+ })
+ }
+}
+
+// TestRefuseHTTPSDowngradeClosesBody checks the transport closes the
+// request body when it refuses a redirect, as RoundTrip must.
+func TestRefuseHTTPSDowngradeClosesBody(t *testing.T) {
+ body := &closeRecorder{Reader: strings.NewReader("potato")}
+ req, err := http.NewRequest(http.MethodPost, "http://a.test/x", body)
+ require.NoError(t, err)
+ origin, err := http.NewRequest(http.MethodPost, "https://a.test/", nil)
+ require.NoError(t, err)
+ req.Response = &http.Response{Request: origin}
+
+ ctx, _ := fs.AddConfig(context.Background())
+ _, err = NewTransport(ctx).RoundTrip(req)
+ assert.ErrorIs(t, err, rest.ErrHTTPSDowngrade)
+ assert.True(t, body.closed)
+}
+
+// closeRecorder is an io.ReadCloser which records whether it was closed
+type closeRecorder struct {
+ *strings.Reader
+ closed bool
+}
+
+// Close records the close
+func (c *closeRecorder) Close() error {
+ c.closed = true
+ return nil
+}