"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
serve: fix auth proxy using stale config parameters when making a backend
Nick Craig-Wooddf9935d71ef5
diff --git a/cmd/serve/proxy/proxy.go b/cmd/serve/proxy/proxy.goindex b1f5dd8dc..cb3283b14 100644--- a/cmd/serve/proxy/proxy.go+++ b/cmd/serve/proxy/proxy.go@@ -4,8 +4,11 @@ package proxyimport ("bytes""context"+ "crypto/hmac"+ "crypto/rand""crypto/sha256""crypto/subtle"+ "encoding/hex""encoding/json""errors""fmt"@@ -98,10 +101,11 @@ to make proxy to many different sftp backends, you could make thein the output and the user to |user|. For security you'd probably wantto restrict the |host| to a limited list.-Note that an internal cache is keyed on |user| so only use that for-configuration, don't use |pass| or |public_key|. This also means that if a user's-password or public-key is changed the cache will need to expire (which takes 5 mins)-before it takes effect.+An internal cache of backends is keyed on the |user| and a hash of the+|pass| or |public_key|. This means that if a user's password or+public-key changes, or the proxy returns different config parameters+(eg a rotated |api_key|), a fresh backend will be created on the next+request rather than the cached one being reused.This can be used to build general purpose proxies to any kind ofbackend that rclone supports.@@ -196,6 +200,25 @@ func (p *Proxy) run(in map[string]string) (config configmap.Simple, err error) {return config, nil}+// cacheKeyHMACKey is a per-process random key used to derive cache keys+// from auth credentials. Using a keyed hash (HMAC) rather than a bare+// SHA256 means the hash fragment that appears in logs and backend names+// cannot be used to brute-force the underlying password offline.+var cacheKeyHMACKey = func() []byte {+ key := make([]byte, 32)+ if _, err := rand.Read(key); err != nil {+ panic(fmt.Sprintf("proxy: failed to generate cache key: %v", err))+ }+ return key+}()++// generateCacheKey creates a composite cache key from user and auth credentials+func generateCacheKey(user, auth string) string {+ mac := hmac.New(sha256.New, cacheKeyHMACKey)+ mac.Write([]byte(auth))+ return user + "-" + hex.EncodeToString(mac.Sum(nil)[:8])+}+// call runs the auth proxy and returns a cacheEntry and an errorfunc (p *Proxy) call(user, auth string, isPublicKey bool) (value any, err error) {var config configmap.Simple@@ -232,12 +255,17 @@ func (p *Proxy) call(user, auth string, isPublicKey bool) (value any, err error)return nil, fmt.Errorf("proxy: couldn't find backend for %q: %w", fsName, err)}- // base name of config on user name. This may appear in logs- name := "proxy-" + user+ // Make the cache key include the auth so that changes to the+ // auth (eg the proxy returning new config) create a fresh+ // backend rather than reusing the cached one.+ cacheKey := generateCacheKey(user, auth)++ // base name of config on user name and auth hash. This may appear in logs+ name := "proxy-" + cacheKeyfsString := name + ":" + root// Look for fs in the VFS cache- value, err = p.vfsCache.Get(user, func(key string) (value any, ok bool, err error) {+ value, err = p.vfsCache.Get(cacheKey, func(key string) (value any, ok bool, err error) {// Create the Fs from the cachef, err := cache.GetFn(p.ctx, fsString, func(ctx context.Context, fsString string) (fs.Fs, error) {// Update the config with the default values@@ -271,8 +299,11 @@ func (p *Proxy) call(user, auth string, isPublicKey bool) (value any, err error)// Call runs the auth proxy with the username and password/public key provided// returning a *vfs.VFS and the key used in the VFS cache.func (p *Proxy) Call(user, auth string, isPublicKey bool) (VFS *vfs.VFS, vfsKey string, err error) {- // Look in the cache first- value, ok := p.vfsCache.GetMaybe(user)+ // Cache key includes the auth so credential changes don't hit a stale entry+ cacheKey := generateCacheKey(user, auth)++ // Look in the cache first with the credential-aware key+ value, ok := p.vfsCache.GetMaybe(cacheKey)// If not found then call the proxy for a fresh answerif !ok {@@ -288,11 +319,11 @@ func (p *Proxy) Call(user, auth string, isPublicKey bool) (VFS *vfs.VFS, vfsKeyreturn nil, "", fmt.Errorf("proxy: value is not cache entry: %#v", value)}- // Check the password / public key is correct in the cached entry. This- // prevents an attack where subsequent requests for the same- // user don't have their auth checked. It does mean that if- // the password is changed, the user will have to wait for- // cache expiry (5m) before trying again.+ // Check the password / public key matches the cached entry. The+ // cache key already includes a hash of the auth, so a changed+ // credential lands on a fresh key rather than this entry; this+ // check is a final guard against a hash collision on the key+ // returning a backend created with different auth.authHash := sha256.Sum256([]byte(auth))if subtle.ConstantTimeCompare(authHash[:], entry.pwHash[:]) != 1 {if isPublicKey {@@ -301,7 +332,7 @@ func (p *Proxy) Call(user, auth string, isPublicKey bool) (VFS *vfs.VFS, vfsKeyreturn nil, "", errors.New("proxy: incorrect password")}- return entry.vfs, user, nil+ return entry.vfs, cacheKey, nil}// Get VFS from the cache using key - returns nil if not founddiff --git a/cmd/serve/proxy/proxy_test.go b/cmd/serve/proxy/proxy_test.goindex 8f802ab12..a7c2d7dc2 100644--- a/cmd/serve/proxy/proxy_test.go+++ b/cmd/serve/proxy/proxy_test.go@@ -91,12 +91,13 @@ func TestRun(t *testing.T) {require.NotNil(t, entry.vfs)f := entry.vfs.Fs()require.NotNil(t, f)- assert.Equal(t, "proxy-"+testUser, f.Name())+ cacheKey := generateCacheKey(testUser, testPass)+ assert.Equal(t, "proxy-"+cacheKey, f.Name())assert.True(t, strings.HasPrefix(f.String(), "Local file system"))// check it is in the cacheassert.Equal(t, 1, p.vfsCache.Entries())- cacheValue, ok := p.vfsCache.GetMaybe(testUser)+ cacheValue, ok := p.vfsCache.GetMaybe(cacheKey)assert.True(t, ok)assert.Equal(t, value, cacheValue)})@@ -106,23 +107,24 @@ func TestRun(t *testing.T) {assert.Equal(t, 0, p.vfsCache.Entries())defer p.vfsCache.Clear()+ cacheKey := generateCacheKey(testUser, testPass)vfs, vfsKey, err := p.Call(testUser, testPass, false)require.NoError(t, err)require.NotNil(t, vfs)- assert.Equal(t, "proxy-"+testUser, vfs.Fs().Name())- assert.Equal(t, testUser, vfsKey)+ assert.Equal(t, "proxy-"+cacheKey, vfs.Fs().Name())+ assert.Equal(t, cacheKey, vfsKey)// check it is in the cacheassert.Equal(t, 1, p.vfsCache.Entries())- cacheValue, ok := p.vfsCache.GetMaybe(testUser)+ cacheValue, ok := p.vfsCache.GetMaybe(cacheKey)assert.True(t, ok)- cacheEntry, ok := cacheValue.(cacheEntry)+ cached, ok := cacheValue.(cacheEntry)assert.True(t, ok)- assert.Equal(t, vfs, cacheEntry.vfs)+ assert.Equal(t, vfs, cached.vfs)// Test Get works while we have something in the cachet.Run("Get", func(t *testing.T) {- assert.Equal(t, vfs, p.Get(testUser))+ assert.Equal(t, vfs, p.Get(cacheKey))assert.Nil(t, p.Get("unknown"))})@@ -130,22 +132,36 @@ func TestRun(t *testing.T) {vfs, vfsKey, err = p.Call(testUser, testPass, false)require.NoError(t, err)require.NotNil(t, vfs)- assert.Equal(t, "proxy-"+testUser, vfs.Fs().Name())- assert.Equal(t, testUser, vfsKey)+ assert.Equal(t, "proxy-"+cacheKey, vfs.Fs().Name())+ assert.Equal(t, cacheKey, vfsKey)// check cache is at the same levelassert.Equal(t, 1, p.vfsCache.Entries())- // now try again from the cache but with wrong password- vfs, vfsKey, err = p.Call(testUser, testPass+"wrong", false)+ // A different password produces a different cache key, so it+ // creates a fresh cache entry rather than hitting the existing+ // one. Authentication itself is the proxy script's job.+ vfs2, vfsKey2, err := p.Call(testUser, testPass+"different", false)+ require.NoError(t, err)+ require.NotNil(t, vfs2)+ assert.NotEqual(t, cacheKey, vfsKey2)+ assert.Equal(t, 2, p.vfsCache.Entries())++ // The underlying fs.Fs must also be a fresh instance from fs/cache+ if vfs.Fs() == vfs2.Fs() {+ t.Error("fs/cache returned the stale backend after auth change")+ }++ // If a cached entry's pwHash somehow doesn't match the supplied+ // auth (eg a hash collision on the cache key), Call must reject+ // it. Simulate by corrupting the cached pwHash.+ entry := cacheEntry{vfs: vfs, pwHash: sha256.Sum256([]byte("tampered"))}+ p.vfsCache.Put(cacheKey, entry)+ vfs, vfsKey, err = p.Call(testUser, testPass, false)require.Error(t, err)require.Contains(t, err.Error(), "incorrect password")require.Nil(t, vfs)require.Equal(t, "", vfsKey)-- // check cache is at the same level- assert.Equal(t, 1, p.vfsCache.Entries())-})privateKey, privateKeyErr := rsa.GenerateKey(rand.Reader, 2048)@@ -174,12 +190,13 @@ func TestRun(t *testing.T) {require.NotNil(t, entry.vfs)f := entry.vfs.Fs()require.NotNil(t, f)- assert.Equal(t, "proxy-"+testUser, f.Name())+ cacheKey := generateCacheKey(testUser, publicKeyString)+ assert.Equal(t, "proxy-"+cacheKey, f.Name())assert.True(t, strings.HasPrefix(f.String(), "Local file system"))// check it is in the cacheassert.Equal(t, 1, p.vfsCache.Entries())- cacheValue, ok := p.vfsCache.GetMaybe(testUser)+ cacheValue, ok := p.vfsCache.GetMaybe(cacheKey)assert.True(t, ok)assert.Equal(t, value, cacheValue)})@@ -189,6 +206,7 @@ func TestRun(t *testing.T) {assert.Equal(t, 0, p.vfsCache.Entries())defer p.vfsCache.Clear()+ cacheKey := generateCacheKey(testUser, publicKeyString)vfs, vfsKey, err := p.Call(testUser,publicKeyString,@@ -196,20 +214,20 @@ func TestRun(t *testing.T) {)require.NoError(t, err)require.NotNil(t, vfs)- assert.Equal(t, "proxy-"+testUser, vfs.Fs().Name())- assert.Equal(t, testUser, vfsKey)+ assert.Equal(t, "proxy-"+cacheKey, vfs.Fs().Name())+ assert.Equal(t, cacheKey, vfsKey)// check it is in the cacheassert.Equal(t, 1, p.vfsCache.Entries())- cacheValue, ok := p.vfsCache.GetMaybe(testUser)+ cacheValue, ok := p.vfsCache.GetMaybe(cacheKey)assert.True(t, ok)- cacheEntry, ok := cacheValue.(cacheEntry)+ cached, ok := cacheValue.(cacheEntry)assert.True(t, ok)- assert.Equal(t, vfs, cacheEntry.vfs)+ assert.Equal(t, vfs, cached.vfs)// Test Get works while we have something in the cachet.Run("Get", func(t *testing.T) {- assert.Equal(t, vfs, p.Get(testUser))+ assert.Equal(t, vfs, p.Get(cacheKey))assert.Nil(t, p.Get("unknown"))})@@ -217,20 +235,35 @@ func TestRun(t *testing.T) {vfs, vfsKey, err = p.Call(testUser, publicKeyString, true)require.NoError(t, err)require.NotNil(t, vfs)- assert.Equal(t, "proxy-"+testUser, vfs.Fs().Name())- assert.Equal(t, testUser, vfsKey)+ assert.Equal(t, "proxy-"+cacheKey, vfs.Fs().Name())+ assert.Equal(t, cacheKey, vfsKey)// check cache is at the same levelassert.Equal(t, 1, p.vfsCache.Entries())- // now try again from the cache but with wrong public key- vfs, vfsKey, err = p.Call(testUser, publicKeyString+"wrong", true)+ // A different public key produces a different cache key, so it+ // creates a fresh cache entry rather than hitting the existing+ // one. Authentication itself is the proxy script's job.+ vfs2, vfsKey2, err := p.Call(testUser, publicKeyString+"different", true)+ require.NoError(t, err)+ require.NotNil(t, vfs2)+ assert.NotEqual(t, cacheKey, vfsKey2)+ assert.Equal(t, 2, p.vfsCache.Entries())++ // The underlying fs.Fs must be a fresh instance from fs/cache+ if vfs.Fs() == vfs2.Fs() {+ t.Error("fs/cache returned the stale backend after public key change")+ }++ // If a cached entry's pwHash somehow doesn't match the supplied+ // auth (eg a hash collision on the cache key), Call must reject+ // it. Simulate by corrupting the cached pwHash.+ entry := cacheEntry{vfs: vfs, pwHash: sha256.Sum256([]byte("tampered"))}+ p.vfsCache.Put(cacheKey, entry)+ vfs, vfsKey, err = p.Call(testUser, publicKeyString, true)require.Error(t, err)require.Contains(t, err.Error(), "incorrect public key")require.Nil(t, vfs)require.Equal(t, "", vfsKey)-- // check cache is at the same level- assert.Equal(t, 1, p.vfsCache.Entries())})}