"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
serve: fix --auth-proxy backends never being shut down after simultaneous logins
Nick Craig-Wood2e9c06c84d9e
diff --git a/cmd/serve/proxy/proxy.go b/cmd/serve/proxy/proxy.goindex d3d7fb7af..bf107dcb2 100644--- a/cmd/serve/proxy/proxy.go+++ b/cmd/serve/proxy/proxy.go@@ -28,6 +28,7 @@ import (libhttp "github.com/rclone/rclone/lib/http""github.com/rclone/rclone/vfs""github.com/rclone/rclone/vfs/vfscommon"+ "golang.org/x/sync/singleflight")// Help contains text describing how to use the proxy@@ -180,7 +181,8 @@ type Proxy struct {ctx context.Context // for global configOpt OptionsvfsOpt vfscommon.Options- accessKeyMu sync.Mutex // serialises replacing a cached access key entry+ accessKeyMu sync.Mutex // serialises replacing a cached access key entry+ calls singleflight.Group // shares the calls made by logins which arrive together}// cacheEntry is what is stored in the vfsCache@@ -457,7 +459,13 @@ func (p *Proxy) Call(user, auth string, isPublicKey bool, remoteAddr string) (VFif isPublicKey {kind = authPublicKey}- value, err = p.call(user, auth, kind, clientIP)+ // Logins with the same credentials which arrive together+ // share one call, as each call which made its own cache+ // entry would take a reference to the VFS but only the+ // last entry made would be kept to give its reference back.+ value, err, _ = p.calls.Do(cacheKey, func() (any, error) {+ return p.call(user, auth, kind, clientIP)+ })if err != nil {return nil, "", err}diff --git a/cmd/serve/proxy/proxy_code.go b/cmd/serve/proxy/proxy_code.goindex 4fe45508b..f4e9948d3 100644--- a/cmd/serve/proxy/proxy_code.go+++ b/cmd/serve/proxy/proxy_code.go@@ -17,6 +17,16 @@ func main() {log.Fatal(err)}+ // Note each run in a file if asked to+ if runLog := os.Getenv("RCLONE_TEST_PROXY_RUN_LOG"); runLog != "" {+ f, err := os.OpenFile(runLog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)+ if err != nil {+ log.Fatal(err)+ }+ _, _ = f.WriteString(in["user"] + "\n")+ _ = f.Close()+ }+// Write the outputvar out = map[string]string{}for k, v := range in {diff --git a/cmd/serve/proxy/proxy_test.go b/cmd/serve/proxy/proxy_test.goindex 00c99d320..9b6dc4d72 100644--- a/cmd/serve/proxy/proxy_test.go+++ b/cmd/serve/proxy/proxy_test.go@@ -6,6 +6,8 @@ import ("crypto/rsa""crypto/sha256""encoding/base64"+ "os"+ "path/filepath""strings""sync""testing"@@ -501,3 +503,35 @@ func TestBusyVFSNotExpired(t *testing.T) {}, 10*time.Second, 10*time.Millisecond, "VFS not expired once it had nothing to write")assert.Equal(t, int32(0), VFS.Stats()["inUse"], "VFS not shut down once it had nothing to write")}++// TestSimultaneousLogins checks logins with the same credentials which+// arrive together share one run of the proxy and one reference to the+// VFS, so the VFS is shut down when it expires from the cache.+func TestSimultaneousLogins(t *testing.T) {+ runLog := filepath.Join(t.TempDir(), "runs")+ t.Setenv("RCLONE_TEST_PROXY_RUN_LOG", runLog)+ opt := Opt+ opt.AuthProxy = "go run proxy_code.go"+ p := New(context.Background(), &opt, &vfscommon.Opt)+ defer p.Shutdown()++ var wg sync.WaitGroup+ VFSes := make([]*vfs.VFS, 4)+ for i := range VFSes {+ wg.Go(func() {+ var err error+ VFSes[i], _, err = p.Call("user", "pass", false, "192.0.2.1:1024")+ assert.NoError(t, err)+ })+ }+ wg.Wait()++ runs, err := os.ReadFile(runLog)+ require.NoError(t, err)+ assert.Equal(t, "user\n", string(runs), "proxy run more than once")+ for _, VFS := range VFSes {+ require.Same(t, VFSes[0], VFS)+ }+ assert.Equal(t, 1, p.vfsCache.Entries())+ assert.Equal(t, int32(1), VFSes[0].Stats()["inUse"], "VFS referenced more than once")+}