Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

serve s3: fix uploads failing when the auth proxy expires the user's backend

Nick Craig-Wood0edd92134ea7
diff --git a/cmd/serve/s3/multipart.go b/cmd/serve/s3/multipart.go
index ce37b10ed..c7e17fdc9 100644
--- a/cmd/serve/s3/multipart.go
+++ b/cmd/serve/s3/multipart.go
@@ -163,11 +163,17 @@ func (up *multipartUpload) unbuffer(charge int64) {
up.broadcast()
}
-// startActivity marks the upload as having a request in flight.
-func (up *multipartUpload) startActivity() {
+// startActivity marks the upload as having a request in flight, holding
+// its VFS until the matching endActivity. It returns ErrNoSuchUpload if
+// the upload has already gone.
+func (up *multipartUpload) startActivity() error {
+ if !up.vfs.Hold() {
+ return gofakes3.ErrNoSuchUpload
+ }
up.mu.Lock()
up.active++
up.mu.Unlock()
+ return nil
}
// endActivity marks the request done and restarts the upload's idle time.
@@ -176,6 +182,7 @@ func (up *multipartUpload) endActivity() {
up.active--
up.lastUsed = time.Now()
up.mu.Unlock()
+ up.vfs.Shutdown()
}
// loadUpload looks up an in-flight upload of the user making the request
@@ -211,14 +218,23 @@ func (b *s3Backend) releaseUpload() {
// addUpload records a new in-flight upload of t, which must hold a place
// reserved by reserveUpload.
-func (b *s3Backend) addUpload(t *tenant, uploadID gofakes3.UploadID, up *multipartUpload) {
+//
+// The upload holds its VFS until deleteUpload, as an auth proxy would
+// otherwise shut it down once unused for a while, even between the
+// requests of an upload.
+func (b *s3Backend) addUpload(t *tenant, uploadID gofakes3.UploadID, up *multipartUpload) error {
+ if !up.vfs.Hold() {
+ return errors.New("serve s3: VFS shut down while creating multipart upload")
+ }
up.tenant = t
t.uploads.Store(uploadID, up)
+ return nil
}
// deleteUpload removes the record of an in-flight upload, if present.
func (b *s3Backend) deleteUpload(uploadID gofakes3.UploadID, up *multipartUpload) {
if _, ok := up.tenant.uploads.LoadAndDelete(uploadID); ok {
+ up.vfs.Shutdown()
b.uploads.Add(-1)
}
}
@@ -245,7 +261,8 @@ func (b *s3Backend) deleteUpload(uploadID gofakes3.UploadID, up *multipartUpload
// no streaming support, so it ignores the flag.
//
// The object the parts are written to is only created when the first part
-// arrives, so an upload with no parts holds no VFS or backend resources.
+// arrives, so an upload with no parts holds no file handle or backend
+// upload, only a reference to its VFS.
// No more than --multipart-max-uploads can be in progress: after that
// gofakes3.ErrSlowDown is returned before anything is created on the
// remote.
@@ -299,7 +316,10 @@ func (b *s3Backend) CreateMultipartUpload(ctx context.Context, bucketName, objec
up.budget = b.budget
up.vfs = _vfs
- b.addUpload(b.tenant(ctx), uploadID, up)
+ if err := b.addUpload(b.tenant(ctx), uploadID, up); err != nil {
+ b.releaseUpload()
+ return "", err
+ }
return uploadID, nil
}
@@ -314,7 +334,9 @@ func (b *s3Backend) UploadPart(ctx context.Context, bucketName, objectName strin
if err != nil {
return "", err
}
- up.startActivity()
+ if err := up.startActivity(); err != nil {
+ return "", err
+ }
defer up.endActivity()
// The sink must never get more than the declared length, and the body
@@ -839,7 +861,9 @@ func (b *s3Backend) CompleteMultipartUpload(ctx context.Context, bucketName, obj
if err != nil {
return "", "", err
}
- up.startActivity()
+ if err := up.startActivity(); err != nil {
+ return "", "", err
+ }
defer up.endActivity()
// gofakes3 keeps its record of an upload whose completion fails, so
@@ -894,6 +918,10 @@ func (b *s3Backend) AbortMultipartUpload(ctx context.Context, bucketName, object
if err != nil {
return err
}
+ if err := up.startActivity(); err != nil {
+ return err
+ }
+ defer up.endActivity()
defer b.deleteUpload(uploadID, up)
if err := up.abort(); err != nil {
fs.Errorf(up.fp, "aborting multipart upload: %v", err)
@@ -913,6 +941,10 @@ func (b *s3Backend) failUpload(uploadID gofakes3.UploadID, up *multipartUpload)
// forgetUpload aborts up, discards what it wrote and removes every record
// of it, ours and gofakes3's.
func (b *s3Backend) forgetUpload(uploadID gofakes3.UploadID, up *multipartUpload) {
+ if up.startActivity() != nil {
+ return // already gone
+ }
+ defer up.endActivity()
b.deleteUpload(uploadID, up)
if err := up.abort(); err != nil {
fs.Errorf(up.fp, "aborting multipart upload: %v", err)
@@ -973,6 +1005,16 @@ func (b *s3Backend) reapExpiredTenantUploads(t *tenant, now time.Time, expiry ti
})
}
+// forgetAllUploads aborts every in-flight upload, as forgetUpload does.
+func (b *s3Backend) forgetAllUploads() {
+ for _, t := range b.allTenants() {
+ t.uploads.Range(func(key, value any) bool {
+ b.forgetUpload(key.(gofakes3.UploadID), value.(*multipartUpload))
+ return true
+ })
+ }
+}
+
// discardUpload cleans up after a failed or aborted upload, removing the
// temporary object and any stale VFS state for it. It never removes the
// object at the final path: when the parts were written straight to the
diff --git a/cmd/serve/s3/multipart_test.go b/cmd/serve/s3/multipart_test.go
index edd344fc3..b82a46439 100644
--- a/cmd/serve/s3/multipart_test.go
+++ b/cmd/serve/s3/multipart_test.go
@@ -476,7 +476,7 @@ func TestMultipartBufferLimit(t *testing.T) {
// CreateMultipartUpload does.
func storeTestUpload(t *testing.T, b *s3Backend, uploadID gofakes3.UploadID, up *multipartUpload) {
require.NoError(t, b.reserveUpload())
- b.addUpload(b.tenant(context.Background()), uploadID, up)
+ require.NoError(t, b.addUpload(b.tenant(context.Background()), uploadID, up))
}
// stubSink is a multipartUpload sink which records how it was closed.
@@ -659,6 +659,48 @@ func TestMultipartOtherTenant(t *testing.T) {
assert.Equal(t, data, readObject(t, f, bucket, object))
}
+// TestMultipartVFSShutDownBetweenParts checks that a multipart upload
+// survives the auth proxy shutting down the VFS it was started with, as
+// the proxy does once the VFS has been unused in its cache for a while.
+func TestMultipartVFSShutDownBetweenParts(t *testing.T) {
+ fstest.Initialise()
+ ctx := context.Background()
+ f, err := fs.NewFs(ctx, t.TempDir())
+ require.NoError(t, err)
+ const bucket, object = "bucket", "object"
+ require.NoError(t, f.Mkdir(ctx, bucket))
+ opt := Opt
+ opt.HTTP.ListenAddr = []string{endpoint}
+ proxyOpt := proxy.Opt
+ proxyOpt.AuthProxy = "/path/to/auth/proxy"
+ w, err := newServer(ctx, f, &opt, &vfscommon.Opt, &proxyOpt)
+ require.NoError(t, err)
+ t.Cleanup(func() { _ = w.Shutdown() })
+ b := w.backend
+
+ // requestCtx returns the context of a request with a VFS from the
+ // proxy's cache, which the proxy shuts down when it expires.
+ requestCtx := func() (context.Context, *vfs.VFS) {
+ VFS := vfs.New(ctx, f, &vfscommon.Opt)
+ return context.WithValue(tenantCtx("alice"), ctxKeyID, VFS), VFS
+ }
+
+ ctx1, vfs1 := requestCtx()
+ uploadID, err := b.CreateMultipartUpload(ctx1, bucket, object, nil)
+ require.NoError(t, err)
+ vfs1.Shutdown()
+
+ ctx2, vfs2 := requestCtx()
+ defer vfs2.Shutdown()
+ data := []byte(random.String(1024))
+ etag, err := b.UploadPart(ctx2, bucket, object, uploadID, 1, int64(len(data)), bytes.NewReader(data))
+ require.NoError(t, err)
+ input := &gofakes3.CompleteMultipartUploadRequest{Parts: []gofakes3.CompletedPart{{PartNumber: 1, ETag: etag}}}
+ _, _, err = b.CompleteMultipartUpload(ctx2, bucket, object, uploadID, input)
+ require.NoError(t, err)
+ assert.Equal(t, data, readObject(t, f, bucket, object))
+}
+
// TestMultipartReaper checks that an incomplete multipart upload abandoned
// by its client is aborted and cleaned up after --multipart-expiry, and that
// late operations on it fail with NoSuchUpload.
@@ -747,7 +789,8 @@ func TestMultipartReapExpiredUploads(t *testing.T) {
newUp("fresh")
idle := newUp("idle")
busy := newUp("busy")
- busy.startActivity()
+ require.NoError(t, busy.startActivity())
+ t.Cleanup(busy.endActivity)
for _, up := range []*multipartUpload{idle, busy} {
up.mu.Lock()
up.lastUsed = now.Add(-2 * expiry)
diff --git a/cmd/serve/s3/s3_test.go b/cmd/serve/s3/s3_test.go
index b392c6b52..a1fb6f63d 100644
--- a/cmd/serve/s3/s3_test.go
+++ b/cmd/serve/s3/s3_test.go
@@ -9,6 +9,7 @@ import (
"fmt"
"io"
"net/http"
+ "net/http/httptest"
"net/url"
"os"
"path"
@@ -35,6 +36,7 @@ import (
"github.com/rclone/rclone/fstest"
"github.com/rclone/rclone/fstest/testy"
"github.com/rclone/rclone/lib/random"
+ "github.com/rclone/rclone/vfs"
"github.com/rclone/rclone/vfs/vfscommon"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -527,6 +529,42 @@ func testAuthProxyUsersKeptApart(t *testing.T, opt Options) {
assert.NotContains(t, info.UserMetadata, "Secret", "bob seeing alice's metadata")
}
+// TestAuthProxyHoldsVFS checks the VFS the auth proxy gives a request is
+// held until the request finishes, so it isn't shut down under a long
+// upload or download if it expires from the proxy's cache.
+func TestAuthProxyHoldsVFS(t *testing.T) {
+ fstest.Initialise()
+ prog, err := filepath.Abs("../servetest/proxy_code.go")
+ require.NoError(t, err)
+ t.Setenv("RCLONE_TEST_PROXY_AUTH_KEY", "alice,secret")
+ opt := Opt
+ opt.HTTP.ListenAddr = []string{endpoint}
+ proxyOpt := proxy.Opt
+ proxyOpt.AuthProxy = "go run " + prog + " " + t.TempDir()
+ w, err := newServer(context.Background(), nil, &opt, &vfscommon.Opt, &proxyOpt)
+ require.NoError(t, err)
+ t.Cleanup(func() { _ = w.Shutdown() })
+
+ var VFS *vfs.VFS
+ var inUse any
+ handler := proxyAuthMiddleware(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+ VFS, err = w.getVFS(r.Context())
+ require.NoError(t, err)
+ inUse = VFS.Stats()["inUse"]
+ }), w)
+
+ req, err := http.NewRequest("GET", "http://localhost/", nil)
+ require.NoError(t, err)
+ req.Header.Set("X-Amz-Content-Sha256", "UNSIGNED-PAYLOAD")
+ err = v4.NewSigner().SignHTTP(context.Background(), aws.Credentials{AccessKeyID: "alice", SecretAccessKey: "secret"}, req, "UNSIGNED-PAYLOAD", "s3", "us-east-1", time.Now())
+ require.NoError(t, err)
+ handler.ServeHTTP(httptest.NewRecorder(), req)
+
+ require.NotNil(t, VFS, "request not authenticated")
+ assert.Equal(t, int32(2), inUse, "VFS not held by the request")
+ assert.Equal(t, int32(1), VFS.Stats()["inUse"], "VFS still held after the request")
+}
+
// TestAuthKeyPerServer checks that two servers in the same process
// with different --auth-key pairs only accept their own credentials.
func TestAuthKeyPerServer(t *testing.T) {
diff --git a/cmd/serve/s3/server.go b/cmd/serve/s3/server.go
index fc9f0898f..49cfb78aa 100644
--- a/cmd/serve/s3/server.go
+++ b/cmd/serve/s3/server.go
@@ -228,6 +228,8 @@ func (w *Server) Addr() net.Addr {
func (w *Server) Shutdown() error {
w.backend.stopReaper()
err := w.server.Shutdown()
+ // Uploads in progress hold their VFS so must go first
+ w.backend.forgetAllUploads()
w.provider.Shutdown()
return err
}
@@ -244,11 +246,22 @@ func proxyAuthMiddleware(next http.Handler, ws *Server) http.Handler {
return
}
VFS, err := ws.auth(r, accessKey)
+ // The proxy shuts down a VFS once unused in its cache for a while,
+ // so hold it for the whole request, which may stream a long
+ // upload or download. It may have been shut down before it could
+ // be held, in which case the proxy makes a new one.
+ if err == nil && !VFS.Hold() {
+ VFS, err = ws.auth(r, accessKey)
+ if err == nil && !VFS.Hold() {
+ err = errors.New("VFS shut down")
+ }
+ }
if err != nil {
fs.Infof(r.URL.Path, "%s: Auth failed: %v", r.RemoteAddr, err)
accessDenied(w)
return
}
+ defer VFS.Shutdown()
ctx := context.WithValue(r.Context(), ctxKeyID, VFS)
ctx = context.WithValue(ctx, ctxKeyAccessKeyID, accessKey)
next.ServeHTTP(w, r.WithContext(ctx))