Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

serve: fix --auth-proxy backends never being shut down after simultaneous logins

Nick Craig-Wood2e9c06c84d9e
diff --git a/cmd/serve/proxy/proxy.go b/cmd/serve/proxy/proxy.go
index d3d7fb7af..bf107dcb2 100644
--- a/cmd/serve/proxy/proxy.go
+++ b/cmd/serve/proxy/proxy.go
@@ -28,6 +28,7 @@ import (
libhttp "github.com/rclone/rclone/lib/http"
"github.com/rclone/rclone/vfs"
"github.com/rclone/rclone/vfs/vfscommon"
+ "golang.org/x/sync/singleflight"
)
// Help contains text describing how to use the proxy
@@ -180,7 +181,8 @@ type Proxy struct {
ctx context.Context // for global config
Opt Options
vfsOpt vfscommon.Options
- accessKeyMu sync.Mutex // serialises replacing a cached access key entry
+ accessKeyMu sync.Mutex // serialises replacing a cached access key entry
+ calls singleflight.Group // shares the calls made by logins which arrive together
}
// cacheEntry is what is stored in the vfsCache
@@ -457,7 +459,13 @@ func (p *Proxy) Call(user, auth string, isPublicKey bool, remoteAddr string) (VF
if isPublicKey {
kind = authPublicKey
}
- value, err = p.call(user, auth, kind, clientIP)
+ // Logins with the same credentials which arrive together
+ // share one call, as each call which made its own cache
+ // entry would take a reference to the VFS but only the
+ // last entry made would be kept to give its reference back.
+ value, err, _ = p.calls.Do(cacheKey, func() (any, error) {
+ return p.call(user, auth, kind, clientIP)
+ })
if err != nil {
return nil, "", err
}
diff --git a/cmd/serve/proxy/proxy_code.go b/cmd/serve/proxy/proxy_code.go
index 4fe45508b..f4e9948d3 100644
--- a/cmd/serve/proxy/proxy_code.go
+++ b/cmd/serve/proxy/proxy_code.go
@@ -17,6 +17,16 @@ func main() {
log.Fatal(err)
}
+ // Note each run in a file if asked to
+ if runLog := os.Getenv("RCLONE_TEST_PROXY_RUN_LOG"); runLog != "" {
+ f, err := os.OpenFile(runLog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
+ if err != nil {
+ log.Fatal(err)
+ }
+ _, _ = f.WriteString(in["user"] + "\n")
+ _ = f.Close()
+ }
+
// Write the output
var out = map[string]string{}
for k, v := range in {
diff --git a/cmd/serve/proxy/proxy_test.go b/cmd/serve/proxy/proxy_test.go
index 00c99d320..9b6dc4d72 100644
--- a/cmd/serve/proxy/proxy_test.go
+++ b/cmd/serve/proxy/proxy_test.go
@@ -6,6 +6,8 @@ import (
"crypto/rsa"
"crypto/sha256"
"encoding/base64"
+ "os"
+ "path/filepath"
"strings"
"sync"
"testing"
@@ -501,3 +503,35 @@ func TestBusyVFSNotExpired(t *testing.T) {
}, 10*time.Second, 10*time.Millisecond, "VFS not expired once it had nothing to write")
assert.Equal(t, int32(0), VFS.Stats()["inUse"], "VFS not shut down once it had nothing to write")
}
+
+// TestSimultaneousLogins checks logins with the same credentials which
+// arrive together share one run of the proxy and one reference to the
+// VFS, so the VFS is shut down when it expires from the cache.
+func TestSimultaneousLogins(t *testing.T) {
+ runLog := filepath.Join(t.TempDir(), "runs")
+ t.Setenv("RCLONE_TEST_PROXY_RUN_LOG", runLog)
+ opt := Opt
+ opt.AuthProxy = "go run proxy_code.go"
+ p := New(context.Background(), &opt, &vfscommon.Opt)
+ defer p.Shutdown()
+
+ var wg sync.WaitGroup
+ VFSes := make([]*vfs.VFS, 4)
+ for i := range VFSes {
+ wg.Go(func() {
+ var err error
+ VFSes[i], _, err = p.Call("user", "pass", false, "192.0.2.1:1024")
+ assert.NoError(t, err)
+ })
+ }
+ wg.Wait()
+
+ runs, err := os.ReadFile(runLog)
+ require.NoError(t, err)
+ assert.Equal(t, "user\n", string(runs), "proxy run more than once")
+ for _, VFS := range VFSes {
+ require.Same(t, VFSes[0], VFS)
+ }
+ assert.Equal(t, 1, p.vfsCache.Entries())
+ assert.Equal(t, int32(1), VFSes[0].Stats()["inUse"], "VFS referenced more than once")
+}