Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

swift: fix auth token leaking to other hosts or over plaintext on redirect GHSA-m97j-7x65-wh93 CVE-PENDING

Nick Craig-Wood7b29032d4d83
diff --git a/backend/swift/swift.go b/backend/swift/swift.go
index d76aaa7a6..7525a7094 100644
--- a/backend/swift/swift.go
+++ b/backend/swift/swift.go
@@ -9,6 +9,7 @@ import (
"fmt"
"io"
"maps"
+ "net/http"
"path"
"regexp"
"slices"
@@ -33,6 +34,7 @@ import (
"github.com/rclone/rclone/lib/pacer"
"github.com/rclone/rclone/lib/random"
"github.com/rclone/rclone/lib/readers"
+ "github.com/rclone/rclone/lib/rest"
)
// Constants
@@ -482,6 +484,76 @@ func (o *Object) split() (container, containerPath string) {
return o.fs.split(o.remote)
}
+// authHeaders are the request headers which carry the Swift
+// credentials: the token sent on every storage request and the user
+// name and API key sent on a v1 auth request. They must not be
+// forwarded when a redirect leaves the host of the original request.
+var authHeaders = []string{"X-Auth-Token", "X-Auth-Key", "X-Auth-User"}
+
+// transport is the http.RoundTripper for the swift connection.
+//
+// The swift library makes its own http.Client with the default
+// redirect policy and does not let the caller replace it, so the
+// redirect policy is applied here at the transport level instead. A
+// redirect which downgrades HTTPS to plaintext HTTP is refused with
+// rest.ErrHTTPSDowngrade and the auth headers are removed from any
+// hop once the redirect chain has left the host and port of the
+// original request.
+type transport struct {
+ *fshttp.Transport
+}
+
+// newTransport returns the transport for the swift connection
+func newTransport(ctx context.Context) *transport {
+ return &transport{Transport: fshttp.NewTransport(ctx)}
+}
+
+// RoundTrip implements http.RoundTripper
+func (t *transport) RoundTrip(req *http.Request) (*http.Response, error) {
+ // net/http fills in req.Response on the requests it makes to
+ // follow a redirect, so a request without one is not a redirect.
+ if req.Response != nil {
+ origin, leftHost := redirectOrigin(req)
+ if origin.URL.Scheme == "https" && req.URL.Scheme == "http" {
+ if req.Body != nil {
+ _ = req.Body.Close()
+ }
+ return nil, fmt.Errorf("redirect to %s: %w", req.URL.Redacted(), rest.ErrHTTPSDowngrade)
+ }
+ if leftHost {
+ for _, header := range authHeaders {
+ if req.Header.Get(header) == "" {
+ continue
+ }
+ fs.Debugf(nil, "redirect to %s leaves %s: not sending %s", req.URL.Redacted(), origin.URL.Redacted(), header)
+ req.Header.Del(header)
+ }
+ }
+ }
+ return t.Transport.RoundTrip(req)
+}
+
+// redirectOrigin walks back along the chain of redirects which led to
+// req and returns the original request and whether any hop, including
+// req, was to a different host or port from it.
+//
+// net/http copies the headers afresh from the original request for
+// every hop, so once the chain has visited another host the headers
+// must be stripped from every subsequent hop, even one back to the
+// original host, as the other host chose the URL.
+func redirectOrigin(req *http.Request) (origin *http.Request, leftHost bool) {
+ origin = req
+ for origin.Response != nil && origin.Response.Request != nil {
+ origin = origin.Response.Request
+ }
+ for hop := req; hop != origin; hop = hop.Response.Request {
+ if !rest.SameHost(hop.URL, origin.URL) {
+ leftHost = true
+ }
+ }
+ return origin, leftHost
+}
+
// swiftConnection makes a connection to swift
func swiftConnection(ctx context.Context, opt *Options, name string) (*swift.Connection, error) {
ci := fs.GetConfig(ctx)
@@ -505,7 +577,7 @@ func swiftConnection(ctx context.Context, opt *Options, name string) (*swift.Con
EndpointType: swift.EndpointType(opt.EndpointType),
ConnectTimeout: time.Duration(10 * ci.ConnectTimeout), // Use the timeouts in the transport
Timeout: time.Duration(10 * ci.Timeout), // Use the timeouts in the transport
- Transport: fshttp.NewTransport(ctx),
+ Transport: newTransport(ctx),
FetchUntilEmptyPage: opt.FetchUntilEmptyPage,
PartialPageFetchThreshold: opt.PartialPageFetchThreshold,
}
diff --git a/backend/swift/swift_internal_test.go b/backend/swift/swift_internal_test.go
index 49963fc61..2e936819b 100644
--- a/backend/swift/swift_internal_test.go
+++ b/backend/swift/swift_internal_test.go
@@ -2,12 +2,21 @@ package swift
import (
"context"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync/atomic"
"testing"
"time"
"github.com/ncw/swift/v2"
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/config/configmap"
"github.com/rclone/rclone/fs/fserrors"
+ "github.com/rclone/rclone/fs/fshttp"
+ "github.com/rclone/rclone/lib/rest"
"github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
)
func TestInternalUrlEncode(t *testing.T) {
@@ -66,3 +75,164 @@ func TestInternalShouldRetryHeaders(t *testing.T) {
assert.True(t, dt >= time.Hour-time.Second && dt <= time.Hour+time.Second)
}
+
+// testToken is the auth token the redirect tests look for. It is
+// distinctive so that it can't match anything else in a request.
+const testToken = "AUTH_tk_secret_4242"
+
+// authTestHeaders is a deliberately literal copy of authHeaders so
+// that a header dropped from the production list shows up as a leak
+// in the cross host tests rather than silently losing coverage.
+var authTestHeaders = []string{"X-Auth-Token", "X-Auth-Key", "X-Auth-User"}
+
+// listHandler answers any request with an empty JSON listing.
+func listHandler(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte("[]"))
+}
+
+// newTestFs makes an Fs via NewFs for a container on storageURL,
+// using a pre-issued token so that no auth request is made.
+func newTestFs(ctx context.Context, t *testing.T, storageURL string) *Fs {
+ m := configmap.Simple{
+ "storage_url": storageURL + "/v1/AUTH_test",
+ "auth_token": testToken,
+ "chunk_size": "5Gi",
+ }
+ f, err := NewFs(ctx, "swift-test", "container", m)
+ require.NoError(t, err)
+ return f.(*Fs)
+}
+
+// assertAuthHeaders checks that every auth header is absent from r
+// (!want) or that the ones named in present are there (want).
+func assertAuthHeaders(t *testing.T, r *http.Request, want bool, present ...string) {
+ if want {
+ for _, header := range present {
+ assert.NotEmpty(t, r.Header.Get(header), "%s should be sent", header)
+ }
+ return
+ }
+ for _, header := range authTestHeaders {
+ assert.Empty(t, r.Header.Get(header), "%s should have been stripped", header)
+ }
+}
+
+// TestRedirectStripsAuthTokenOnCrossHost checks the token is sent to
+// the storage host but not to another host it redirects to, nor to
+// the storage host again once the chain has been elsewhere.
+func TestRedirectStripsAuthTokenOnCrossHost(t *testing.T) {
+ var originHits, otherHits, backHits atomic.Int32
+ var origin *httptest.Server
+ other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ otherHits.Add(1)
+ assertAuthHeaders(t, r, false)
+ http.Redirect(w, r, origin.URL+"/back", http.StatusFound)
+ }))
+ defer other.Close()
+
+ origin = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path == "/back" {
+ backHits.Add(1)
+ assertAuthHeaders(t, r, false)
+ listHandler(w, r)
+ return
+ }
+ originHits.Add(1)
+ assertAuthHeaders(t, r, true, "X-Auth-Token")
+ http.Redirect(w, r, other.URL+r.URL.Path, http.StatusFound)
+ }))
+ defer origin.Close()
+
+ ctx := context.Background()
+ f := newTestFs(ctx, t, origin.URL)
+ _, err := f.List(ctx, "")
+ require.NoError(t, err)
+ assert.Equal(t, int32(1), originHits.Load())
+ assert.Equal(t, int32(1), otherHits.Load())
+ assert.Equal(t, int32(1), backHits.Load())
+}
+
+// TestRedirectKeepsAuthTokenOnSameHost checks the token is still sent
+// after a redirect which stays on the storage host.
+func TestRedirectKeepsAuthTokenOnSameHost(t *testing.T) {
+ var hits atomic.Int32
+ origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ hits.Add(1)
+ assertAuthHeaders(t, r, true, "X-Auth-Token")
+ if !strings.HasSuffix(r.URL.Path, "/moved") {
+ http.Redirect(w, r, r.URL.Path+"/moved", http.StatusTemporaryRedirect)
+ return
+ }
+ listHandler(w, r)
+ }))
+ defer origin.Close()
+
+ ctx := context.Background()
+ f := newTestFs(ctx, t, origin.URL)
+ _, err := f.List(ctx, "")
+ require.NoError(t, err)
+ assert.Equal(t, int32(2), hits.Load())
+}
+
+// TestRedirectRefusesHTTPSDowngrade checks a redirect from HTTPS to
+// plaintext HTTP is not followed, so the token never goes over the
+// network in the clear.
+func TestRedirectRefusesHTTPSDowngrade(t *testing.T) {
+ var targetHits atomic.Int32
+ target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ targetHits.Add(1)
+ listHandler(w, r)
+ }))
+ defer target.Close()
+
+ origin := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ http.Redirect(w, r, target.URL+r.URL.Path, http.StatusTemporaryRedirect)
+ }))
+ defer origin.Close()
+
+ // The transport is a process wide singleton so it has to be
+ // rebuilt to trust the httptest certificate, and rebuilt again
+ // afterwards so later tests don't inherit that.
+ fshttp.ResetTransport()
+ t.Cleanup(fshttp.ResetTransport)
+ ctx, ci := fs.AddConfig(context.Background())
+ ci.InsecureSkipVerify = true
+ f := newTestFs(ctx, t, origin.URL)
+ _, err := f.List(ctx, "")
+ require.ErrorIs(t, err, rest.ErrHTTPSDowngrade)
+ assert.Equal(t, int32(0), targetHits.Load())
+}
+
+// TestAuthRedirectStripsCredentialsOnCrossHost checks the user name
+// and API key sent on a v1 auth request go to the auth host but not
+// to another host it redirects to.
+func TestAuthRedirectStripsCredentialsOnCrossHost(t *testing.T) {
+ var originHits, otherHits atomic.Int32
+ other := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ otherHits.Add(1)
+ assertAuthHeaders(t, r, false)
+ w.Header().Set("X-Storage-Url", "http://"+r.Host+"/v1/AUTH_test")
+ w.Header().Set("X-Auth-Token", testToken)
+ }))
+ defer other.Close()
+
+ origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ originHits.Add(1)
+ assertAuthHeaders(t, r, true, "X-Auth-Key", "X-Auth-User")
+ http.Redirect(w, r, other.URL+r.URL.Path, http.StatusFound)
+ }))
+ defer origin.Close()
+
+ m := configmap.Simple{
+ "auth": origin.URL + "/v1.0",
+ "auth_version": "1",
+ "user": "test-user",
+ "key": "test-key",
+ "chunk_size": "5Gi",
+ }
+ _, err := NewFs(context.Background(), "swift-test", "container", m)
+ require.NoError(t, err)
+ assert.Equal(t, int32(1), originHits.Load())
+ assert.Equal(t, int32(1), otherHits.Load())
+}