"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
swift: fix auth token leaking to other hosts or over plaintext on redirect GHSA-m97j-7x65-wh93 CVE-PENDING
Nick Craig-Wood7b29032d4d83
diff --git a/backend/swift/swift.go b/backend/swift/swift.goindex d76aaa7a6..7525a7094 100644--- a/backend/swift/swift.go+++ b/backend/swift/swift.go@@ -9,6 +9,7 @@ import ("fmt""io""maps"+ "net/http""path""regexp""slices"@@ -33,6 +34,7 @@ import ("github.com/rclone/rclone/lib/pacer""github.com/rclone/rclone/lib/random""github.com/rclone/rclone/lib/readers"+ "github.com/rclone/rclone/lib/rest")// Constants@@ -482,6 +484,76 @@ func (o *Object) split() (container, containerPath string) {return o.fs.split(o.remote)}+// authHeaders are the request headers which carry the Swift+// credentials: the token sent on every storage request and the user+// name and API key sent on a v1 auth request. They must not be+// forwarded when a redirect leaves the host of the original request.+var authHeaders = []string{"X-Auth-Token", "X-Auth-Key", "X-Auth-User"}++// transport is the http.RoundTripper for the swift connection.+//+// The swift library makes its own http.Client with the default+// redirect policy and does not let the caller replace it, so the+// redirect policy is applied here at the transport level instead. A+// redirect which downgrades HTTPS to plaintext HTTP is refused with+// rest.ErrHTTPSDowngrade and the auth headers are removed from any+// hop once the redirect chain has left the host and port of the+// original request.+type transport struct {+ *fshttp.Transport+}++// newTransport returns the transport for the swift connection+func newTransport(ctx context.Context) *transport {+ return &transport{Transport: fshttp.NewTransport(ctx)}+}++// RoundTrip implements http.RoundTripper+func (t *transport) RoundTrip(req *http.Request) (*http.Response, error) {+ // net/http fills in req.Response on the requests it makes to+ // follow a redirect, so a request without one is not a redirect.+ if req.Response != nil {+ origin, leftHost := redirectOrigin(req)+ if origin.URL.Scheme == "https" && req.URL.Scheme == "http" {+ if req.Body != nil {+ _ = req.Body.Close()+ }+ return nil, fmt.Errorf("redirect to %s: %w", req.URL.Redacted(), rest.ErrHTTPSDowngrade)+ }+ if leftHost {+ for _, header := range authHeaders {+ if req.Header.Get(header) == "" {+ continue+ }+ fs.Debugf(nil, "redirect to %s leaves %s: not sending %s", req.URL.Redacted(), origin.URL.Redacted(), header)+ req.Header.Del(header)+ }+ }+ }+ return t.Transport.RoundTrip(req)+}++// redirectOrigin walks back along the chain of redirects which led to+// req and returns the original request and whether any hop, including+// req, was to a different host or port from it.+//+// net/http copies the headers afresh from the original request for+// every hop, so once the chain has visited another host the headers+// must be stripped from every subsequent hop, even one back to the+// original host, as the other host chose the URL.+func redirectOrigin(req *http.Request) (origin *http.Request, leftHost bool) {+ origin = req+ for origin.Response != nil && origin.Response.Request != nil {+ origin = origin.Response.Request+ }+ for hop := req; hop != origin; hop = hop.Response.Request {+ if !rest.SameHost(hop.URL, origin.URL) {+ leftHost = true+ }+ }+ return origin, leftHost+}+// swiftConnection makes a connection to swiftfunc swiftConnection(ctx context.Context, opt *Options, name string) (*swift.Connection, error) {ci := fs.GetConfig(ctx)@@ -505,7 +577,7 @@ func swiftConnection(ctx context.Context, opt *Options, name string) (*swift.ConEndpointType: swift.EndpointType(opt.EndpointType),ConnectTimeout: time.Duration(10 * ci.ConnectTimeout), // Use the timeouts in the transportTimeout: time.Duration(10 * ci.Timeout), // Use the timeouts in the transport- Transport: fshttp.NewTransport(ctx),+ Transport: newTransport(ctx),FetchUntilEmptyPage: opt.FetchUntilEmptyPage,PartialPageFetchThreshold: opt.PartialPageFetchThreshold,}diff --git a/backend/swift/swift_internal_test.go b/backend/swift/swift_internal_test.goindex 49963fc61..2e936819b 100644--- a/backend/swift/swift_internal_test.go+++ b/backend/swift/swift_internal_test.go@@ -2,12 +2,21 @@ package swiftimport ("context"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync/atomic""testing""time""github.com/ncw/swift/v2"+ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/config/configmap""github.com/rclone/rclone/fs/fserrors"+ "github.com/rclone/rclone/fs/fshttp"+ "github.com/rclone/rclone/lib/rest""github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require")func TestInternalUrlEncode(t *testing.T) {@@ -66,3 +75,164 @@ func TestInternalShouldRetryHeaders(t *testing.T) {assert.True(t, dt >= time.Hour-time.Second && dt <= time.Hour+time.Second)}++// testToken is the auth token the redirect tests look for. It is+// distinctive so that it can't match anything else in a request.+const testToken = "AUTH_tk_secret_4242"++// authTestHeaders is a deliberately literal copy of authHeaders so+// that a header dropped from the production list shows up as a leak+// in the cross host tests rather than silently losing coverage.+var authTestHeaders = []string{"X-Auth-Token", "X-Auth-Key", "X-Auth-User"}++// listHandler answers any request with an empty JSON listing.+func listHandler(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte("[]"))+}++// newTestFs makes an Fs via NewFs for a container on storageURL,+// using a pre-issued token so that no auth request is made.+func newTestFs(ctx context.Context, t *testing.T, storageURL string) *Fs {+ m := configmap.Simple{+ "storage_url": storageURL + "/v1/AUTH_test",+ "auth_token": testToken,+ "chunk_size": "5Gi",+ }+ f, err := NewFs(ctx, "swift-test", "container", m)+ require.NoError(t, err)+ return f.(*Fs)+}++// assertAuthHeaders checks that every auth header is absent from r+// (!want) or that the ones named in present are there (want).+func assertAuthHeaders(t *testing.T, r *http.Request, want bool, present ...string) {+ if want {+ for _, header := range present {+ assert.NotEmpty(t, r.Header.Get(header), "%s should be sent", header)+ }+ return+ }+ for _, header := range authTestHeaders {+ assert.Empty(t, r.Header.Get(header), "%s should have been stripped", header)+ }+}++// TestRedirectStripsAuthTokenOnCrossHost checks the token is sent to+// the storage host but not to another host it redirects to, nor to+// the storage host again once the chain has been elsewhere.+func TestRedirectStripsAuthTokenOnCrossHost(t *testing.T) {+ var originHits, otherHits, backHits atomic.Int32+ var origin *httptest.Server+ other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ otherHits.Add(1)+ assertAuthHeaders(t, r, false)+ http.Redirect(w, r, origin.URL+"/back", http.StatusFound)+ }))+ defer other.Close()++ origin = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path == "/back" {+ backHits.Add(1)+ assertAuthHeaders(t, r, false)+ listHandler(w, r)+ return+ }+ originHits.Add(1)+ assertAuthHeaders(t, r, true, "X-Auth-Token")+ http.Redirect(w, r, other.URL+r.URL.Path, http.StatusFound)+ }))+ defer origin.Close()++ ctx := context.Background()+ f := newTestFs(ctx, t, origin.URL)+ _, err := f.List(ctx, "")+ require.NoError(t, err)+ assert.Equal(t, int32(1), originHits.Load())+ assert.Equal(t, int32(1), otherHits.Load())+ assert.Equal(t, int32(1), backHits.Load())+}++// TestRedirectKeepsAuthTokenOnSameHost checks the token is still sent+// after a redirect which stays on the storage host.+func TestRedirectKeepsAuthTokenOnSameHost(t *testing.T) {+ var hits atomic.Int32+ origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ hits.Add(1)+ assertAuthHeaders(t, r, true, "X-Auth-Token")+ if !strings.HasSuffix(r.URL.Path, "/moved") {+ http.Redirect(w, r, r.URL.Path+"/moved", http.StatusTemporaryRedirect)+ return+ }+ listHandler(w, r)+ }))+ defer origin.Close()++ ctx := context.Background()+ f := newTestFs(ctx, t, origin.URL)+ _, err := f.List(ctx, "")+ require.NoError(t, err)+ assert.Equal(t, int32(2), hits.Load())+}++// TestRedirectRefusesHTTPSDowngrade checks a redirect from HTTPS to+// plaintext HTTP is not followed, so the token never goes over the+// network in the clear.+func TestRedirectRefusesHTTPSDowngrade(t *testing.T) {+ var targetHits atomic.Int32+ target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ targetHits.Add(1)+ listHandler(w, r)+ }))+ defer target.Close()++ origin := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ http.Redirect(w, r, target.URL+r.URL.Path, http.StatusTemporaryRedirect)+ }))+ defer origin.Close()++ // The transport is a process wide singleton so it has to be+ // rebuilt to trust the httptest certificate, and rebuilt again+ // afterwards so later tests don't inherit that.+ fshttp.ResetTransport()+ t.Cleanup(fshttp.ResetTransport)+ ctx, ci := fs.AddConfig(context.Background())+ ci.InsecureSkipVerify = true+ f := newTestFs(ctx, t, origin.URL)+ _, err := f.List(ctx, "")+ require.ErrorIs(t, err, rest.ErrHTTPSDowngrade)+ assert.Equal(t, int32(0), targetHits.Load())+}++// TestAuthRedirectStripsCredentialsOnCrossHost checks the user name+// and API key sent on a v1 auth request go to the auth host but not+// to another host it redirects to.+func TestAuthRedirectStripsCredentialsOnCrossHost(t *testing.T) {+ var originHits, otherHits atomic.Int32+ other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ otherHits.Add(1)+ assertAuthHeaders(t, r, false)+ w.Header().Set("X-Storage-Url", "http://"+r.Host+"/v1/AUTH_test")+ w.Header().Set("X-Auth-Token", testToken)+ }))+ defer other.Close()++ origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ originHits.Add(1)+ assertAuthHeaders(t, r, true, "X-Auth-Key", "X-Auth-User")+ http.Redirect(w, r, other.URL+r.URL.Path, http.StatusFound)+ }))+ defer origin.Close()++ m := configmap.Simple{+ "auth": origin.URL + "/v1.0",+ "auth_version": "1",+ "user": "test-user",+ "key": "test-key",+ "chunk_size": "5Gi",+ }+ _, err := NewFs(context.Background(), "swift-test", "container", m)+ require.NoError(t, err)+ assert.Equal(t, int32(1), originHits.Load())+ assert.Equal(t, int32(1), otherHits.Load())+}