"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
lib/http: require authentication for OPTIONS requests GHSA-gj73-fh6v-92fj CVE-PENDING
Nick Craig-Wood25707f2dc632
diff --git a/cmd/serve/webdav/webdav_test.go b/cmd/serve/webdav/webdav_test.goindex 10ce4241c..b0a3310f1 100644--- a/cmd/serve/webdav/webdav_test.go+++ b/cmd/serve/webdav/webdav_test.go@@ -38,6 +38,7 @@ const (testUser = "user"testPass = "pass"testTemplate = "../http/testdata/golden/testindex.html"+ testAllowOrigin = "http://test.rclone.org")// check interfaces@@ -255,6 +256,12 @@ func HelpTestGET(t *testing.T, testURL string) {// the test files directory, starts it, waits for it to be ready, and returns// the base URL. It registers cleanup to shut the server down.func startAuthenticatedServer(t *testing.T) string {+ return startAuthenticatedServerAllowOrigin(t, "")+}++// startAuthenticatedServerAllowOrigin is startAuthenticatedServer+// with CORS enabled for allowOrigin if it is non-empty.+func startAuthenticatedServerAllowOrigin(t *testing.T, allowOrigin string) string {t.Helper()f, err := fs.NewFs(context.Background(), "../http/testdata/files")@@ -262,6 +269,7 @@ func startAuthenticatedServer(t *testing.T) string {opt := Optopt.HTTP.ListenAddr = []string{testBindAddress}+ opt.HTTP.AllowOrigin = allowOriginopt.Template.Path = testTemplateopt.Auth.BasicUser = testUseropt.Auth.BasicPass = testPass@@ -279,6 +287,64 @@ func startAuthenticatedServer(t *testing.T) string {return testURL}+// TestOPTIONSRequiresAuth checks OPTIONS can't be used to discover+// whether a path exists, and whether it is a file or a directory,+// without authenticating.+func TestOPTIONSRequiresAuth(t *testing.T) {+ doOPTIONS := func(url string, setup func(req *http.Request)) *http.Response {+ req, err := http.NewRequest("OPTIONS", url, nil)+ require.NoError(t, err)+ if setup != nil {+ setup(req)+ }+ resp, err := http.DefaultClient.Do(req)+ require.NoError(t, err)+ _ = resp.Body.Close()+ return resp+ }++ testURL := startAuthenticatedServer(t)++ // The Allow header the WebDAV handler returns differs for an+ // existing file, an existing directory and a missing path so+ // it must not be visible without credentials.+ allows := map[string]string{}+ for _, path := range []string{"two.txt", "three/", "doesnotexist"} {+ t.Run(path, func(t *testing.T) {+ resp := doOPTIONS(testURL+path, nil)+ assert.Equal(t, http.StatusUnauthorized, resp.StatusCode)+ assert.Empty(t, resp.Header.Get("Allow"))+ assert.Empty(t, resp.Header.Get("DAV"))+ assert.NotEmpty(t, resp.Header.Get("WWW-Authenticate"))++ resp = doOPTIONS(testURL+path, func(req *http.Request) {+ req.SetBasicAuth(testUser, testPass)+ })+ assert.Equal(t, http.StatusOK, resp.StatusCode)+ assert.NotEmpty(t, resp.Header.Get("Allow"))+ assert.Equal(t, "1, 2", resp.Header.Get("DAV"))+ allows[path] = resp.Header.Get("Allow")+ })+ }+ assert.NotEqual(t, allows["two.txt"], allows["three/"])+ assert.NotEqual(t, allows["three/"], allows["doesnotexist"])+ assert.NotEqual(t, allows["two.txt"], allows["doesnotexist"])++ // A browser CORS preflight can't carry credentials so it must+ // succeed, but it mustn't reach the WebDAV handler+ t.Run("Preflight", func(t *testing.T) {+ testURL := startAuthenticatedServerAllowOrigin(t, testAllowOrigin)+ resp := doOPTIONS(testURL+"two.txt", func(req *http.Request) {+ req.Header.Set("Origin", testAllowOrigin)+ req.Header.Set("Access-Control-Request-Method", "PROPFIND")+ })+ assert.Equal(t, http.StatusOK, resp.StatusCode)+ assert.Empty(t, resp.Header.Get("Allow"))+ assert.Empty(t, resp.Header.Get("DAV"))+ assert.Equal(t, testAllowOrigin, resp.Header.Get("Access-Control-Allow-Origin"))+ })+}+func TestCompressedTextFile(t *testing.T) {testURL := startAuthenticatedServer(t)diff --git a/docs/content/rc.md b/docs/content/rc.mdindex 2e58fc844..6f4cd594a 100644--- a/docs/content/rc.md+++ b/docs/content/rc.md@@ -174,9 +174,20 @@ Default Off.Set the allowed Access-Control-Allow-Origin for rc requests.-Can be used with --rc-web-gui if the rclone is running on different IP than the web-gui.+This enables cross-origin (CORS) requests from browsers on the origin+given, for example `--rc-allow-origin https://app.example.com`. Browser+CORS preflight OPTIONS requests are answered by rclone without+authentication as browsers never send credentials with them. All other+requests, including other OPTIONS requests, must be authenticated when+authentication is enabled.-Default is IP address on which rc is running.+The `rclone gui` command sets this automatically to the origin the GUI+is served from if it is not set explicitly.++Setting this to `*` allows any origin which is a security risk on an+authenticated server.++Default is off (no CORS).### --rc-web-fetch-url@@ -2666,9 +2677,11 @@ The keys in the error response are:### CORS-The sever implements basic CORS support and allows all origins for that.-The response to a preflight OPTIONS request will echo the requested-"Access-Control-Request-Headers" back.+The server implements basic CORS support when `--rc-allow-origin` is+set. Browser CORS preflight OPTIONS requests are then answered by+rclone without authentication with the `Access-Control-*` headers.+Without `--rc-allow-origin` no CORS headers are sent and a preflight+request is authenticated like any other request.### Using POST with URL parameters onlydiff --git a/lib/http/auth.go b/lib/http/auth.goindex a5358932d..f2c324447 100644--- a/lib/http/auth.go+++ b/lib/http/auth.go@@ -43,6 +43,14 @@ htpasswd -B htpasswd anotherUserThe password file can be updated while rclone is running.+When authentication is enabled every request must be authenticated,+including HTTP OPTIONS requests, as the answer to an OPTIONS request can+reveal whether a path exists. Health checks or monitoring which use+unauthenticated OPTIONS requests will need to send credentials. The+exception is browser CORS preflight requests when ` + "`--{{ .Prefix }}allow-origin`" + `+is set, which are answered without authentication as browsers never send+credentials with them.+Use ` + "`--{{ .Prefix }}realm`" + ` to set the authentication realm.Use ` + "`--{{ .Prefix }}salt`" + ` to change the password hashing salt from the default.diff --git a/lib/http/middleware.go b/lib/http/middleware.goindex 1c4721a25..eaa6b0a10 100644--- a/lib/http/middleware.go+++ b/lib/http/middleware.go@@ -58,12 +58,6 @@ func NewLoggedBasicAuthenticator(realm string, secrets goauth.SecretProvider) *Lfunc basicAuth(authenticator *LoggedBasicAuth) func(next http.Handler) http.Handler {return func(next http.Handler) http.Handler {return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {- // skip auth for CORS preflight- if r.Method == "OPTIONS" {- next.ServeHTTP(w, r)- return- }-username := authenticator.CheckAuth(r)if username == "" {authenticator.RequireAuth(w, r)@@ -123,12 +117,6 @@ func MiddlewareAuthBasic(user, pass, realm, salt string) Middleware {func MiddlewareAuthCustom(fn CustomAuthFn, realm string, userFromContext bool) Middleware {return func(next http.Handler) http.Handler {return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {- // skip auth for CORS preflight- if r.Method == "OPTIONS" {- next.ServeHTTP(w, r)- return- }-user, pass, ok := parseAuthorization(r)if !ok && userFromContext {user, ok = CtxGetUser(r.Context())@@ -180,7 +168,23 @@ func MiddlewareAuthGetUserFromHeader(header string) Middleware {var onlyOnceWarningAllowOrigin sync.Once-// MiddlewareCORS instantiates middleware that handles basic CORS protections for rcd+// isCORSPreflight returns true if r is a browser CORS preflight request+func isCORSPreflight(r *http.Request) bool {+ return r.Method == "OPTIONS" && r.Header.Get("Origin") != "" && r.Header.Get("Access-Control-Request-Method") != ""+}++// MiddlewareCORS instantiates middleware that adds the CORS headers to+// every response if allowOrigin is set.+//+// It also answers CORS preflight requests itself. Browsers never+// attach credentials to a preflight so it must succeed before any+// authentication middleware sees it, and the browser only looks at+// the Access-Control-* headers so it is not passed to the handler+// whose answer could reveal information about the path to an+// unauthenticated client.+//+// If allowOrigin is empty a preflight is treated like any other+// request.func MiddlewareCORS(allowOrigin string) Middleware {onlyOnceWarningAllowOrigin.Do(func() {if allowOrigin == "*" {@@ -196,6 +200,10 @@ func MiddlewareCORS(allowOrigin string) Middleware {w.Header().Add("Access-Control-Allow-Headers", "Authorization, Content-Type, Depth, Destination, If, Lock-Token, Overwrite, Prefer, TimeOut, Translate")w.Header().Add("Access-Control-Allow-Methods", "COPY, DELETE, GET, HEAD, LOCK, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, TRACE, UNLOCK")w.Header().Add("Access-Control-Max-Age", "86400")+ if isCORSPreflight(r) {+ w.WriteHeader(http.StatusOK)+ return+ }}next.ServeHTTP(w, r)diff --git a/lib/http/middleware_test.go b/lib/http/middleware_test.goindex ed0607742..0b9ada121 100644--- a/lib/http/middleware_test.go+++ b/lib/http/middleware_test.go@@ -205,6 +205,40 @@ func TestMiddlewareAuth(t *testing.T) {testExpectRespBody(t, resp, expected)}})++ // OPTIONS is not exempt from authentication: a handler+ // may reveal information about the path in its answer+ t.Run("NoCredsOPTIONS", func(t *testing.T) {+ client := &http.Client{}+ req, err := http.NewRequest("OPTIONS", url, nil)+ require.NoError(t, err)++ resp, err := client.Do(req)+ require.NoError(t, err)+ defer func() {+ _ = resp.Body.Close()+ }()++ require.Equal(t, http.StatusUnauthorized, resp.StatusCode, "OPTIONS with no creds should return unauthorized")+ })++ // A CORS preflight is only exempt when CORS is enabled+ // which it isn't for these servers+ t.Run("NoCredsPreflight", func(t *testing.T) {+ client := &http.Client{}+ req, err := http.NewRequest("OPTIONS", url, nil)+ require.NoError(t, err)+ req.Header.Set("Origin", "http://test.rclone.org")+ req.Header.Set("Access-Control-Request-Method", "GET")++ resp, err := client.Do(req)+ require.NoError(t, err)+ defer func() {+ _ = resp.Body.Close()+ }()++ require.Equal(t, http.StatusUnauthorized, resp.StatusCode, "preflight with no creds and no CORS should return unauthorized")+ })})}}@@ -541,7 +575,7 @@ func TestMiddlewareCORSWithAuth(t *testing.T) {auth AuthConfig}{{- name: "ServerWithAuth",+ name: "Basic",http: Config{ListenAddr: []string{"127.0.0.1:0"},AllowOrigin: "http://test.rclone.org",@@ -552,44 +586,121 @@ func TestMiddlewareCORSWithAuth(t *testing.T) {BasicPass: "test_pass",},},+ {+ name: "Custom",+ http: Config{+ ListenAddr: []string{"127.0.0.1:0"},+ AllowOrigin: "http://test.rclone.org",+ },+ auth: AuthConfig{+ Realm: "test",+ CustomAuthFn: func(r *http.Request, user, pass string) (value any, err error) {+ if user == "test_user" && pass == "test_pass" {+ return true, nil+ }+ return nil, errors.New("invalid credentials")+ },+ },+ },+ {+ name: "UserFromHeader",+ http: Config{+ ListenAddr: []string{"127.0.0.1:0"},+ AllowOrigin: "http://test.rclone.org",+ },+ auth: AuthConfig{+ Realm: "test",+ UserFromHeader: "X-Remote-User",+ },+ },}for _, ss := range authServers {t.Run(ss.name, func(t *testing.T) {- s, err := NewServer(context.Background(), WithConfig(ss.http))+ s, err := NewServer(context.Background(), WithConfig(ss.http), WithAuth(ss.auth))require.NoError(t, err)defer func() {require.NoError(t, s.Shutdown())}()- s.Router().Mount("/", testEmptyHandler())+ s.Router().Mount("/", testEchoHandler([]byte("secret-page")))s.Serve()url := testGetServerURL(t, s)- client := &http.Client{}- req, err := http.NewRequest("OPTIONS", url, nil)- require.NoError(t, err)+ doOPTIONS := func(t *testing.T, setup func(req *http.Request)) *http.Response {+ req, err := http.NewRequest("OPTIONS", url, nil)+ require.NoError(t, err)+ if setup != nil {+ setup(req)+ }+ resp, err := http.DefaultClient.Do(req)+ require.NoError(t, err)+ t.Cleanup(func() {+ _ = resp.Body.Close()+ })+ return resp+ }- resp, err := client.Do(req)- require.NoError(t, err)- defer func() {- _ = resp.Body.Close()- }()+ preflight := func(req *http.Request) {+ req.Header.Set("Origin", ss.http.AllowOrigin)+ req.Header.Set("Access-Control-Request-Method", "GET")+ }- require.Equal(t, http.StatusOK, resp.StatusCode, "OPTIONS should return ok even if not authenticated")+ // checkPreflight checks resp was answered by the CORS+ // middleware and never reached the handler behind it+ checkPreflight := func(t *testing.T, resp *http.Response) {+ require.Equal(t, http.StatusOK, resp.StatusCode, "preflight OPTIONS should return ok")+ testExpectRespBody(t, resp, []byte{})+ for _, key := range _testCORSHeaderKeys {+ require.Contains(t, resp.Header, key, "CORS headers should be sent")+ }+ require.Equal(t, ss.http.AllowOrigin, resp.Header.Get("Access-Control-Allow-Origin"), "allow origin should match")+ }- testExpectRespBody(t, resp, []byte{})+ // A browser preflight can't carry credentials so it+ // must succeed without them+ t.Run("PreflightNoCreds", func(t *testing.T) {+ checkPreflight(t, doOPTIONS(t, preflight))+ })- for _, key := range _testCORSHeaderKeys {- require.Contains(t, resp.Header, key, "CORS headers should be sent even if not authenticated")- }+ // A proxy in front of the browser may stamp an+ // Authorization header the server can't check onto the+ // preflight, which mustn't stop it succeeding+ t.Run("PreflightIgnoresAuthorization", func(t *testing.T) {+ checkPreflight(t, doOPTIONS(t, func(req *http.Request) {+ preflight(req)+ req.Header.Set("Authorization", "Bearer not-checked-here")+ }))+ checkPreflight(t, doOPTIONS(t, func(req *http.Request) {+ preflight(req)+ req.SetBasicAuth("test_user", "wrong")+ }))+ checkPreflight(t, doOPTIONS(t, func(req *http.Request) {+ preflight(req)+ req.SetBasicAuth("test_user", "test_pass")+ }))+ })- expectedOrigin := url- if ss.http.AllowOrigin != "" {- expectedOrigin = ss.http.AllowOrigin- }- require.Equal(t, expectedOrigin, resp.Header.Get("Access-Control-Allow-Origin"), "allow origin should match")+ // Any other OPTIONS is authenticated like any other+ // request as the handler's answer may reveal+ // information about the path+ t.Run("OPTIONSNoCreds", func(t *testing.T) {+ resp := doOPTIONS(t, nil)+ require.Equal(t, http.StatusUnauthorized, resp.StatusCode, "OPTIONS with no creds should return unauthorized")+ })++ t.Run("OPTIONSGoodCreds", func(t *testing.T) {+ resp := doOPTIONS(t, func(req *http.Request) {+ if ss.auth.UserFromHeader != "" {+ req.Header.Set(ss.auth.UserFromHeader, "test_user")+ } else {+ req.SetBasicAuth("test_user", "test_pass")+ }+ })+ require.Equal(t, http.StatusOK, resp.StatusCode, "OPTIONS with good creds should return ok")+ testExpectRespBody(t, resp, []byte("secret-page"))+ })})}}diff --git a/lib/http/server.go b/lib/http/server.goindex 2df99ebbc..512ff1e0d 100644--- a/lib/http/server.go+++ b/lib/http/server.go@@ -55,6 +55,13 @@ accept in the HTTP header.will overriding existing values. The flag may be repeated to add multipleheaders. Use the format ` + "`Header-Name: value`" + `.+` + "`--{{ .Prefix }}allow-origin`" + ` enables cross-origin (CORS) requests from browsers+by setting the ` + "`Access-Control-Allow-Origin`" + ` header on every response to the+origin given, e.g. ` + "`--{{ .Prefix }}allow-origin https://app.example.com`" + `. Browser+CORS preflight OPTIONS requests are answered by rclone without needing+authentication as browsers never send credentials with them. Setting this+to ` + "`*`" + ` allows any origin which is a security risk on an authenticated server.+` + "`--{{ .Prefix }}baseurl`" + ` controls the URL prefix that rclone serves from. By defaultrclone will serve from the root. If you used ` + "`--{{ .Prefix }}baseurl \"/rclone\"`" + ` thenrclone would serve from a URL starting with "/rclone/". This isdiff --git a/lib/http/server_test.go b/lib/http/server_test.goindex 1bf67a452..41123bb29 100644--- a/lib/http/server_test.go+++ b/lib/http/server_test.go@@ -16,10 +16,6 @@ import ("golang.org/x/net/http2")-func testEmptyHandler() http.Handler {- return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})-}-func testEchoHandler(data []byte) http.Handler {return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {_, _ = w.Write(data)