"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
serve s3: fix uploads failing when the auth proxy expires the user's backend
Nick Craig-Wood0edd92134ea7
diff --git a/cmd/serve/s3/multipart.go b/cmd/serve/s3/multipart.goindex ce37b10ed..c7e17fdc9 100644--- a/cmd/serve/s3/multipart.go+++ b/cmd/serve/s3/multipart.go@@ -163,11 +163,17 @@ func (up *multipartUpload) unbuffer(charge int64) {up.broadcast()}-// startActivity marks the upload as having a request in flight.-func (up *multipartUpload) startActivity() {+// startActivity marks the upload as having a request in flight, holding+// its VFS until the matching endActivity. It returns ErrNoSuchUpload if+// the upload has already gone.+func (up *multipartUpload) startActivity() error {+ if !up.vfs.Hold() {+ return gofakes3.ErrNoSuchUpload+ }up.mu.Lock()up.active++up.mu.Unlock()+ return nil}// endActivity marks the request done and restarts the upload's idle time.@@ -176,6 +182,7 @@ func (up *multipartUpload) endActivity() {up.active--up.lastUsed = time.Now()up.mu.Unlock()+ up.vfs.Shutdown()}// loadUpload looks up an in-flight upload of the user making the request@@ -211,14 +218,23 @@ func (b *s3Backend) releaseUpload() {// addUpload records a new in-flight upload of t, which must hold a place// reserved by reserveUpload.-func (b *s3Backend) addUpload(t *tenant, uploadID gofakes3.UploadID, up *multipartUpload) {+//+// The upload holds its VFS until deleteUpload, as an auth proxy would+// otherwise shut it down once unused for a while, even between the+// requests of an upload.+func (b *s3Backend) addUpload(t *tenant, uploadID gofakes3.UploadID, up *multipartUpload) error {+ if !up.vfs.Hold() {+ return errors.New("serve s3: VFS shut down while creating multipart upload")+ }up.tenant = tt.uploads.Store(uploadID, up)+ return nil}// deleteUpload removes the record of an in-flight upload, if present.func (b *s3Backend) deleteUpload(uploadID gofakes3.UploadID, up *multipartUpload) {if _, ok := up.tenant.uploads.LoadAndDelete(uploadID); ok {+ up.vfs.Shutdown()b.uploads.Add(-1)}}@@ -245,7 +261,8 @@ func (b *s3Backend) deleteUpload(uploadID gofakes3.UploadID, up *multipartUpload// no streaming support, so it ignores the flag.//// The object the parts are written to is only created when the first part-// arrives, so an upload with no parts holds no VFS or backend resources.+// arrives, so an upload with no parts holds no file handle or backend+// upload, only a reference to its VFS.// No more than --multipart-max-uploads can be in progress: after that// gofakes3.ErrSlowDown is returned before anything is created on the// remote.@@ -299,7 +316,10 @@ func (b *s3Backend) CreateMultipartUpload(ctx context.Context, bucketName, objecup.budget = b.budgetup.vfs = _vfs- b.addUpload(b.tenant(ctx), uploadID, up)+ if err := b.addUpload(b.tenant(ctx), uploadID, up); err != nil {+ b.releaseUpload()+ return "", err+ }return uploadID, nil}@@ -314,7 +334,9 @@ func (b *s3Backend) UploadPart(ctx context.Context, bucketName, objectName strinif err != nil {return "", err}- up.startActivity()+ if err := up.startActivity(); err != nil {+ return "", err+ }defer up.endActivity()// The sink must never get more than the declared length, and the body@@ -839,7 +861,9 @@ func (b *s3Backend) CompleteMultipartUpload(ctx context.Context, bucketName, objif err != nil {return "", "", err}- up.startActivity()+ if err := up.startActivity(); err != nil {+ return "", "", err+ }defer up.endActivity()// gofakes3 keeps its record of an upload whose completion fails, so@@ -894,6 +918,10 @@ func (b *s3Backend) AbortMultipartUpload(ctx context.Context, bucketName, objectif err != nil {return err}+ if err := up.startActivity(); err != nil {+ return err+ }+ defer up.endActivity()defer b.deleteUpload(uploadID, up)if err := up.abort(); err != nil {fs.Errorf(up.fp, "aborting multipart upload: %v", err)@@ -913,6 +941,10 @@ func (b *s3Backend) failUpload(uploadID gofakes3.UploadID, up *multipartUpload)// forgetUpload aborts up, discards what it wrote and removes every record// of it, ours and gofakes3's.func (b *s3Backend) forgetUpload(uploadID gofakes3.UploadID, up *multipartUpload) {+ if up.startActivity() != nil {+ return // already gone+ }+ defer up.endActivity()b.deleteUpload(uploadID, up)if err := up.abort(); err != nil {fs.Errorf(up.fp, "aborting multipart upload: %v", err)@@ -973,6 +1005,16 @@ func (b *s3Backend) reapExpiredTenantUploads(t *tenant, now time.Time, expiry ti})}+// forgetAllUploads aborts every in-flight upload, as forgetUpload does.+func (b *s3Backend) forgetAllUploads() {+ for _, t := range b.allTenants() {+ t.uploads.Range(func(key, value any) bool {+ b.forgetUpload(key.(gofakes3.UploadID), value.(*multipartUpload))+ return true+ })+ }+}+// discardUpload cleans up after a failed or aborted upload, removing the// temporary object and any stale VFS state for it. It never removes the// object at the final path: when the parts were written straight to thediff --git a/cmd/serve/s3/multipart_test.go b/cmd/serve/s3/multipart_test.goindex edd344fc3..b82a46439 100644--- a/cmd/serve/s3/multipart_test.go+++ b/cmd/serve/s3/multipart_test.go@@ -476,7 +476,7 @@ func TestMultipartBufferLimit(t *testing.T) {// CreateMultipartUpload does.func storeTestUpload(t *testing.T, b *s3Backend, uploadID gofakes3.UploadID, up *multipartUpload) {require.NoError(t, b.reserveUpload())- b.addUpload(b.tenant(context.Background()), uploadID, up)+ require.NoError(t, b.addUpload(b.tenant(context.Background()), uploadID, up))}// stubSink is a multipartUpload sink which records how it was closed.@@ -659,6 +659,48 @@ func TestMultipartOtherTenant(t *testing.T) {assert.Equal(t, data, readObject(t, f, bucket, object))}+// TestMultipartVFSShutDownBetweenParts checks that a multipart upload+// survives the auth proxy shutting down the VFS it was started with, as+// the proxy does once the VFS has been unused in its cache for a while.+func TestMultipartVFSShutDownBetweenParts(t *testing.T) {+ fstest.Initialise()+ ctx := context.Background()+ f, err := fs.NewFs(ctx, t.TempDir())+ require.NoError(t, err)+ const bucket, object = "bucket", "object"+ require.NoError(t, f.Mkdir(ctx, bucket))+ opt := Opt+ opt.HTTP.ListenAddr = []string{endpoint}+ proxyOpt := proxy.Opt+ proxyOpt.AuthProxy = "/path/to/auth/proxy"+ w, err := newServer(ctx, f, &opt, &vfscommon.Opt, &proxyOpt)+ require.NoError(t, err)+ t.Cleanup(func() { _ = w.Shutdown() })+ b := w.backend++ // requestCtx returns the context of a request with a VFS from the+ // proxy's cache, which the proxy shuts down when it expires.+ requestCtx := func() (context.Context, *vfs.VFS) {+ VFS := vfs.New(ctx, f, &vfscommon.Opt)+ return context.WithValue(tenantCtx("alice"), ctxKeyID, VFS), VFS+ }++ ctx1, vfs1 := requestCtx()+ uploadID, err := b.CreateMultipartUpload(ctx1, bucket, object, nil)+ require.NoError(t, err)+ vfs1.Shutdown()++ ctx2, vfs2 := requestCtx()+ defer vfs2.Shutdown()+ data := []byte(random.String(1024))+ etag, err := b.UploadPart(ctx2, bucket, object, uploadID, 1, int64(len(data)), bytes.NewReader(data))+ require.NoError(t, err)+ input := &gofakes3.CompleteMultipartUploadRequest{Parts: []gofakes3.CompletedPart{{PartNumber: 1, ETag: etag}}}+ _, _, err = b.CompleteMultipartUpload(ctx2, bucket, object, uploadID, input)+ require.NoError(t, err)+ assert.Equal(t, data, readObject(t, f, bucket, object))+}+// TestMultipartReaper checks that an incomplete multipart upload abandoned// by its client is aborted and cleaned up after --multipart-expiry, and that// late operations on it fail with NoSuchUpload.@@ -747,7 +789,8 @@ func TestMultipartReapExpiredUploads(t *testing.T) {newUp("fresh")idle := newUp("idle")busy := newUp("busy")- busy.startActivity()+ require.NoError(t, busy.startActivity())+ t.Cleanup(busy.endActivity)for _, up := range []*multipartUpload{idle, busy} {up.mu.Lock()up.lastUsed = now.Add(-2 * expiry)diff --git a/cmd/serve/s3/s3_test.go b/cmd/serve/s3/s3_test.goindex b392c6b52..a1fb6f63d 100644--- a/cmd/serve/s3/s3_test.go+++ b/cmd/serve/s3/s3_test.go@@ -9,6 +9,7 @@ import ("fmt""io""net/http"+ "net/http/httptest""net/url""os""path"@@ -35,6 +36,7 @@ import ("github.com/rclone/rclone/fstest""github.com/rclone/rclone/fstest/testy""github.com/rclone/rclone/lib/random"+ "github.com/rclone/rclone/vfs""github.com/rclone/rclone/vfs/vfscommon""github.com/stretchr/testify/assert""github.com/stretchr/testify/require"@@ -527,6 +529,42 @@ func testAuthProxyUsersKeptApart(t *testing.T, opt Options) {assert.NotContains(t, info.UserMetadata, "Secret", "bob seeing alice's metadata")}+// TestAuthProxyHoldsVFS checks the VFS the auth proxy gives a request is+// held until the request finishes, so it isn't shut down under a long+// upload or download if it expires from the proxy's cache.+func TestAuthProxyHoldsVFS(t *testing.T) {+ fstest.Initialise()+ prog, err := filepath.Abs("../servetest/proxy_code.go")+ require.NoError(t, err)+ t.Setenv("RCLONE_TEST_PROXY_AUTH_KEY", "alice,secret")+ opt := Opt+ opt.HTTP.ListenAddr = []string{endpoint}+ proxyOpt := proxy.Opt+ proxyOpt.AuthProxy = "go run " + prog + " " + t.TempDir()+ w, err := newServer(context.Background(), nil, &opt, &vfscommon.Opt, &proxyOpt)+ require.NoError(t, err)+ t.Cleanup(func() { _ = w.Shutdown() })++ var VFS *vfs.VFS+ var inUse any+ handler := proxyAuthMiddleware(http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {+ VFS, err = w.getVFS(r.Context())+ require.NoError(t, err)+ inUse = VFS.Stats()["inUse"]+ }), w)++ req, err := http.NewRequest("GET", "http://localhost/", nil)+ require.NoError(t, err)+ req.Header.Set("X-Amz-Content-Sha256", "UNSIGNED-PAYLOAD")+ err = v4.NewSigner().SignHTTP(context.Background(), aws.Credentials{AccessKeyID: "alice", SecretAccessKey: "secret"}, req, "UNSIGNED-PAYLOAD", "s3", "us-east-1", time.Now())+ require.NoError(t, err)+ handler.ServeHTTP(httptest.NewRecorder(), req)++ require.NotNil(t, VFS, "request not authenticated")+ assert.Equal(t, int32(2), inUse, "VFS not held by the request")+ assert.Equal(t, int32(1), VFS.Stats()["inUse"], "VFS still held after the request")+}+// TestAuthKeyPerServer checks that two servers in the same process// with different --auth-key pairs only accept their own credentials.func TestAuthKeyPerServer(t *testing.T) {diff --git a/cmd/serve/s3/server.go b/cmd/serve/s3/server.goindex fc9f0898f..49cfb78aa 100644--- a/cmd/serve/s3/server.go+++ b/cmd/serve/s3/server.go@@ -228,6 +228,8 @@ func (w *Server) Addr() net.Addr {func (w *Server) Shutdown() error {w.backend.stopReaper()err := w.server.Shutdown()+ // Uploads in progress hold their VFS so must go first+ w.backend.forgetAllUploads()w.provider.Shutdown()return err}@@ -244,11 +246,22 @@ func proxyAuthMiddleware(next http.Handler, ws *Server) http.Handler {return}VFS, err := ws.auth(r, accessKey)+ // The proxy shuts down a VFS once unused in its cache for a while,+ // so hold it for the whole request, which may stream a long+ // upload or download. It may have been shut down before it could+ // be held, in which case the proxy makes a new one.+ if err == nil && !VFS.Hold() {+ VFS, err = ws.auth(r, accessKey)+ if err == nil && !VFS.Hold() {+ err = errors.New("VFS shut down")+ }+ }if err != nil {fs.Infof(r.URL.Path, "%s: Auth failed: %v", r.RemoteAddr, err)accessDenied(w)return}+ defer VFS.Shutdown()ctx := context.WithValue(r.Context(), ctxKeyID, VFS)ctx = context.WithValue(ctx, ctxKeyAccessKeyID, accessKey)next.ServeHTTP(w, r.WithContext(ctx))