"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
lib/http: stop --htpasswd and --user revealing which usernames exist through timing GHSA-hwj7-wp9g-wrv9 CVE-PENDING
Nick Craig-Woode3c400f583e4
diff --git a/lib/http/auth.go b/lib/http/auth.goindex f2c324447..ded952024 100644--- a/lib/http/auth.go+++ b/lib/http/auth.go@@ -41,6 +41,10 @@ htpasswd -B htpasswd userhtpasswd -B htpasswd anotherUser` + "```" + `+Use the same hash format and cost for every entry in the file. If the+entries differ, the time taken to reject a wrong password can reveal+which usernames exist.+The password file can be updated while rclone is running.When authentication is enabled every request must be authenticated,diff --git a/lib/http/htpasswd.go b/lib/http/htpasswd.goindex 9b08b562d..9afc281aa 100644--- a/lib/http/htpasswd.go+++ b/lib/http/htpasswd.go@@ -13,6 +13,7 @@ import ("time""github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/lib/random""golang.org/x/crypto/bcrypt")@@ -97,13 +98,23 @@ func isBcrypt(secret string) bool {return false}-// supportedHash returns true if secret is in a format checkHash-// can check.-func supportedHash(secret string) bool {+// hashCost ranks secret by how expensive it is to check a password+// against it, or returns -1 if it isn't in a supported format.+func hashCost(secret string) int {+ switch {+ case strings.HasPrefix(secret, "{SHA}"):+ return 0+ case isBcrypt(secret):+ cost, err := bcrypt.Cost([]byte(secret))+ if err != nil {+ return -1+ }+ return 2 + cost+ }if _, _, ok := parseMD5Crypt(secret); ok {- return true+ return 1}- return strings.HasPrefix(secret, "{SHA}") || isBcrypt(secret)+ return -1}// checkHash returns true if password matches the hashed secret.@@ -123,6 +134,37 @@ func checkHash(secret, password string) bool {return false}+// decoyHash returns a hash of a random password in the same format+// and with the same cost as secret, so checking a password against it+// takes as long as checking against secret but never succeeds.+//+// It returns "" if secret isn't in a supported format.+func decoyHash(secret string) string {+ cost := hashCost(secret)+ if cost < 0 {+ return ""+ }+ password, err := random.Password(128)+ if err != nil {+ fs.Errorf(nil, "Failed to make htpasswd decoy hash: %v", err)+ return ""+ }+ switch cost {+ case 0:+ sum := sha1.Sum([]byte(password))+ return "{SHA}" + base64.StdEncoding.EncodeToString(sum[:])+ case 1:+ magic, salt, _ := parseMD5Crypt(secret)+ return md5Crypt(password, salt, magic)+ }+ hash, err := bcrypt.GenerateFromPassword([]byte(password), cost-2)+ if err != nil {+ fs.Errorf(nil, "Failed to make htpasswd decoy hash: %v", err)+ return ""+ }+ return string(hash)+}+// parseHtpasswd reads user:hash lines from in, returning a map of// user to hash. Blank lines and lines starting with # are ignored and// malformed lines are logged and skipped.@@ -143,7 +185,7 @@ func parseHtpasswd(name string, in io.Reader) (map[string]string, error) {fs.Errorf(name, "Ignoring malformed htpasswd line %d", lineNumber)continue}- if !supportedHash(secret) {+ if hashCost(secret) < 0 {fs.Errorf(name, "Password for htpasswd user %q is in an unsupported format - use bcrypt, MD5 or SHA1", user)}users[user] = secret@@ -160,15 +202,17 @@ type htpasswdFile struct {modTime time.Timesize int64users map[string]string // user => hashed password+ decoy string // hash to check unknown users against}-// load reads the users from the file. On error it leaves no users and-// the load is retried on the next lookup, as fixing errors such as-// permissions needn't change the file's size or modification time.+// load reads the users from the file and makes a decoy hash from+// the most expensive entry. On error it leaves no users and the load+// is retried on the next lookup, as fixing errors such as permissions+// needn't change the file's size or modification time.//// Call with mu held.func (h *htpasswdFile) load(info os.FileInfo) {- h.loaded, h.users = false, nil+ h.loaded, h.users, h.decoy = false, nil, ""f, err := os.Open(h.path)if err != nil {fs.Errorf(nil, "Failed to read htpasswd file: %v", err)@@ -180,28 +224,42 @@ func (h *htpasswdFile) load(info os.FileInfo) {fs.Errorf(nil, "Failed to read htpasswd file %q: %v", h.path, err)return}+ best, bestCost := "", -1+ for _, secret := range users {+ if cost := hashCost(secret); cost > bestCost {+ best, bestCost = secret, cost+ }+ }h.loaded, h.modTime, h.size = true, info.ModTime(), info.Size()- h.users = users+ h.users, h.decoy = users, decoyHash(best)}// lookup returns the hashed password for user, or "" if there isn't-// one.-func (h *htpasswdFile) lookup(user string) string {+// one, and the decoy hash.+func (h *htpasswdFile) lookup(user string) (secret, decoy string) {info, err := os.Stat(h.path)h.mu.Lock()defer h.mu.Unlock()if err != nil {fs.Errorf(nil, "Failed to read htpasswd file: %v", err)- h.loaded, h.users = false, nil- return ""+ h.loaded, h.users, h.decoy = false, nil, ""+ return "", ""}if !h.loaded || !info.ModTime().Equal(h.modTime) || info.Size() != h.size {h.load(info)}- return h.users[user]+ return h.users[user], h.decoy}// check returns true if user and password match an entry in the file.func (h *htpasswdFile) check(user, password string) bool {- return checkHash(h.lookup(user), password)+ secret, decoy := h.lookup(user)+ if hashCost(secret) < 0 {+ // Check unknown users against the decoy so they take as+ // long to reject as a known user with the wrong password,+ // otherwise the timing reveals which users exist.+ _ = checkHash(decoy, password)+ return false+ }+ return checkHash(secret, password)}diff --git a/lib/http/htpasswd_test.go b/lib/http/htpasswd_test.goindex 3966012aa..46d3de917 100644--- a/lib/http/htpasswd_test.go+++ b/lib/http/htpasswd_test.go@@ -71,6 +71,28 @@ func TestCheckHash(t *testing.T) {}}+func TestDecoyHash(t *testing.T) {+ for _, test := range []struct {+ secret string+ prefix string+ }{+ {"{SHA}2PRZAyDhNDqRW2OUFwZQqPNdaSY=", "{SHA}"},+ {"$apr1$s7fogein$IK9ItbnGM14ct0bY4Uyik1", "$apr1$s7fogein$"},+ {"$2y$05$K/b3mVXUA6X857TOTYIL9.Lbaeg9oBjMQwUX5NefpVUCcYP0Z5KY2", "$2a$05$"},+ {"plaintext", ""},+ {"abJnggxhB/yWI", ""},+ {"$2$05$K/b3mVXUA6X857TOTYIL9.Lbaeg9oBjMQwUX5NefpVUCcYP0Z5KY2", ""},+ } {+ decoy := decoyHash(test.secret)+ if test.prefix == "" {+ assert.Equal(t, "", decoy, test.secret)+ continue+ }+ assert.True(t, strings.HasPrefix(decoy, test.prefix), "%q: got %q", test.secret, decoy)+ assert.NotEqual(t, test.secret, decoy)+ }+}+func TestParseHtpasswd(t *testing.T) {in := strings.NewReader(`# commentuser1:{SHA}2PRZAyDhNDqRW2OUFwZQqPNdaSY=@@ -91,28 +113,38 @@ user1:{SHA}overridden=}, users)}-func TestHtpasswdFileUnsupported(t *testing.T) {+func TestHtpasswdFileDecoy(t *testing.T) {+ h := &htpasswdFile{path: "./testdata/.htpasswd"}+ _, decoy := h.lookup("nosuchuser")+ assert.True(t, strings.HasPrefix(decoy, "$2a$10$"), decoy)++ // Entries which can't be checked must not hide ones which canpath := filepath.Join(t.TempDir(), ".htpasswd")require.NoError(t, os.WriteFile(path, []byte("des:abJnggxhB/yWI\nplain:plain\nmd5:$apr1$s7fogein$IK9ItbnGM14ct0bY4Uyik1\n"), 0600))- h := &htpasswdFile{path: path}+ h = &htpasswdFile{path: path}+ _, decoy = h.lookup("nosuchuser")+ assert.True(t, strings.HasPrefix(decoy, "$apr1$s7fogein$"), decoy)assert.False(t, h.check("des", "des"))assert.False(t, h.check("plain", "plain"))assert.True(t, h.check("md5", "md5"))- assert.False(t, h.check("nosuchuser", "md5"))}-// The users must follow changes to the htpasswd file+// The users and decoy must follow changes to the htpasswd filefunc TestHtpasswdFileReload(t *testing.T) {path := filepath.Join(t.TempDir(), ".htpasswd")require.NoError(t, os.WriteFile(path, []byte("sha:{SHA}2PRZAyDhNDqRW2OUFwZQqPNdaSY=\n"), 0600))h := &htpasswdFile{path: path}assert.True(t, h.check("sha", "sha"))+ _, decoy := h.lookup("nosuchuser")+ assert.True(t, strings.HasPrefix(decoy, "{SHA}"))require.NoError(t, os.WriteFile(path, []byte("bcrypt:$2y$10$K/b3mVXUA6X857TOTYIL9.Lbaeg9oBjMQwUX5NefpVUCcYP0Z5KY2\n"), 0600))future := time.Now().Add(time.Hour)require.NoError(t, os.Chtimes(path, future, future))assert.False(t, h.check("sha", "sha"))assert.True(t, h.check("bcrypt", "bcrypt"))+ _, decoy = h.lookup("nosuchuser")+ assert.True(t, strings.HasPrefix(decoy, "$2a$10$"))// A missing file lets nobody inrequire.NoError(t, os.Remove(path))diff --git a/lib/http/middleware.go b/lib/http/middleware.goindex 31c6016ba..831339052 100644--- a/lib/http/middleware.go+++ b/lib/http/middleware.go@@ -2,6 +2,7 @@ package httpimport ("context"+ "crypto/sha256""crypto/subtle""encoding/base64""fmt"@@ -93,8 +94,16 @@ func MiddlewareAuthHtpasswd(path, realm string) Middleware {func MiddlewareAuthBasic(user, pass, realm, salt string) Middleware {fs.Infof(nil, "Using --user %s --pass XXXX as authenticated user", user)pass = md5Crypt(pass, salt, "$1$")+ // Compare hashes of the users as ConstantTimeCompare returns+ // early if the lengths differ which would reveal the user's length+ userSum := sha256.Sum256([]byte(user))return basicAuth(realm, func(u, p string) bool {- return u == user && subtle.ConstantTimeCompare([]byte(md5Crypt(p, salt, "$1$")), []byte(pass)) == 1+ // Always hash the password so a wrong user takes as long to+ // reject as a wrong password+ uSum := sha256.Sum256([]byte(u))+ userOK := subtle.ConstantTimeCompare(uSum[:], userSum[:]) == 1+ passOK := subtle.ConstantTimeCompare([]byte(md5Crypt(p, salt, "$1$")), []byte(pass)) == 1+ return userOK && passOK})}diff --git a/lib/http/middleware_test.go b/lib/http/middleware_test.goindex 510dfff55..f05ea1030 100644--- a/lib/http/middleware_test.go+++ b/lib/http/middleware_test.go@@ -9,7 +9,9 @@ import ("net/http/httptest""strings""testing"+ "time"+ "github.com/stretchr/testify/assert""github.com/stretchr/testify/require")@@ -243,6 +245,61 @@ func TestMiddlewareAuth(t *testing.T) {}}+// minAuthTime returns the shortest time handler took to reject user+// with a wrong password over a few attempts.+func minAuthTime(t *testing.T, handler http.Handler, user string) time.Duration {+ var best time.Duration+ for i := range 10 {+ req := httptest.NewRequest("GET", "/", nil)+ req.SetBasicAuth(user, "definitely-wrong-password")+ rec := httptest.NewRecorder()+ start := time.Now()+ handler.ServeHTTP(rec, req)+ elapsed := time.Since(start)+ require.Equal(t, http.StatusUnauthorized, rec.Code)+ if i == 0 || elapsed < best {+ best = elapsed+ }+ }+ return best+}++// Rejecting an unknown user must take about as long as rejecting a+// known user with a wrong password otherwise the timing reveals which+// users exist.+func TestMiddlewareAuthHtpasswdUnknownUserTiming(t *testing.T) {+ handler := MiddlewareAuthHtpasswd("./testdata/.htpasswd", "test")(testEchoHandler([]byte("ok")))+ known := minAuthTime(t, handler, "bcrypt")+ unknown := minAuthTime(t, handler, "nosuchuser")+ assert.Greater(t, unknown, known/4, "unknown user rejected in %v but known user in %v", unknown, known)+}++// As TestMiddlewareAuthHtpasswdUnknownUserTiming but for --user/--pass+func TestMiddlewareAuthBasicUnknownUserTiming(t *testing.T) {+ handler := MiddlewareAuthBasic("user", "pass", "test", "dlPL2MqE")(testEchoHandler([]byte("ok")))+ known := minAuthTime(t, handler, "user")+ unknown := minAuthTime(t, handler, "nosuchuser")+ assert.Greater(t, unknown, known/4, "unknown user rejected in %v but known user in %v", unknown, known)++ req := httptest.NewRequest("GET", "/", nil)+ req.SetBasicAuth("user", "pass")+ rec := httptest.NewRecorder()+ handler.ServeHTTP(rec, req)+ assert.Equal(t, http.StatusOK, rec.Code)+}++// An unknown user must never be authenticated by the decoy secret+func TestMiddlewareAuthHtpasswdUnknownUser(t *testing.T) {+ handler := MiddlewareAuthHtpasswd("./testdata/.htpasswd", "test")(testEchoHandler([]byte("ok")))+ for _, pass := range []string{"", "bcrypt", "md5", "sha"} {+ req := httptest.NewRequest("GET", "/", nil)+ req.SetBasicAuth("nosuchuser", pass)+ rec := httptest.NewRecorder()+ handler.ServeHTTP(rec, req)+ assert.Equal(t, http.StatusUnauthorized, rec.Code, pass)+ }+}+// A request arriving over a plain HTTP listener has no TLS state so it can't// carry a client certificate and must be rejected.func TestMiddlewareAuthCertificateUserNoTLS(t *testing.T) {