Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

serve http: fix downloads failing after 5 minutes with --auth-proxy

Nick Craig-Wood8457da3c8abb
diff --git a/cmd/serve/http/http.go b/cmd/serve/http/http.go
index c22b40d33..51d733115 100644
--- a/cmd/serve/http/http.go
+++ b/cmd/serve/http/http.go
@@ -199,6 +199,7 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Opt
router := s.server.Router()
router.Use(
+ s.provider.HoldVFS,
middleware.Compress(5),
middleware.SetHeader("Accept-Ranges", "bytes"),
middleware.SetHeader("Server", "rclone/"+fs.Version),
diff --git a/cmd/serve/http/http_test.go b/cmd/serve/http/http_test.go
index 4567fa434..8d6f2a50f 100644
--- a/cmd/serve/http/http_test.go
+++ b/cmd/serve/http/http_test.go
@@ -22,6 +22,7 @@ import (
"github.com/rclone/rclone/fs/filter"
"github.com/rclone/rclone/fs/rc"
libhttp "github.com/rclone/rclone/lib/http"
+ "github.com/rclone/rclone/lib/random"
"github.com/rclone/rclone/vfs/vfscommon"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@@ -559,3 +560,40 @@ func TestNewServerError(t *testing.T) {
require.Error(t, err)
assert.Nil(t, s)
}
+
+// TestAuthProxyDownloadOutlivesCache checks a download in progress
+// carries on working when the auth proxy drops its VFS from its cache,
+// as it does when a download takes longer than the cache expiry time.
+func TestAuthProxyDownloadOutlivesCache(t *testing.T) {
+ root := t.TempDir()
+ contents := random.String(32 * 1024 * 1024)
+ require.NoError(t, os.WriteFile(filepath.Join(root, "download.bin"), []byte(contents), 0666))
+
+ prog, err := filepath.Abs("../servetest/proxy_code.go")
+ require.NoError(t, err)
+ // FIXME this is untidy setting a global variable!
+ proxy.Opt.AuthProxy = "go run " + prog + " " + root
+ defer func() {
+ proxy.Opt.AuthProxy = ""
+ }()
+ s, testURL := start(context.Background(), t, nil)
+ defer func() { assert.NoError(t, s.Shutdown()) }()
+
+ req, err := http.NewRequest("GET", testURL+"download.bin", nil)
+ require.NoError(t, err)
+ req.SetBasicAuth(testUser, testPass)
+ resp, err := http.DefaultClient.Do(req)
+ require.NoError(t, err)
+ defer func() { _ = resp.Body.Close() }()
+ require.Equal(t, http.StatusOK, resp.StatusCode)
+ start := make([]byte, 1024)
+ _, err = io.ReadFull(resp.Body, start)
+ require.NoError(t, err)
+
+ // Drop everything from the proxy's cache as if it had expired
+ s.provider.Proxy().Shutdown()
+
+ rest, err := io.ReadAll(resp.Body)
+ require.NoError(t, err)
+ assert.True(t, contents == string(start)+string(rest), "download corrupted")
+}
diff --git a/cmd/serve/proxy/proxy.go b/cmd/serve/proxy/proxy.go
index 5718588b3..ed0bb3395 100644
--- a/cmd/serve/proxy/proxy.go
+++ b/cmd/serve/proxy/proxy.go
@@ -12,6 +12,7 @@ import (
"encoding/json"
"errors"
"fmt"
+ "net/http"
"net/netip"
"os/exec"
"strings"
@@ -577,6 +578,26 @@ func (p *Provider) Get(ctx context.Context) (*vfs.VFS, error) {
return VFS, nil
}
+// HoldVFS is HTTP middleware which holds the VFS for the request until
+// the request has been served.
+//
+// An auth proxy shuts down a VFS when it expires from its cache, which
+// a long upload or download can outlast as only the start of a request
+// counts as a use of the VFS.
+func (p *Provider) HoldVFS(next http.Handler) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ // Requests which don't need auth, eg CORS preflight, have no VFS
+ if VFS, err := p.Get(r.Context()); err == nil {
+ if !VFS.Hold() {
+ http.Error(w, "VFS has been shut down", http.StatusServiceUnavailable)
+ return
+ }
+ defer VFS.Shutdown()
+ }
+ next.ServeHTTP(w, r)
+ })
+}
+
// VFS returns the fixed VFS, or nil if using an auth proxy.
func (p *Provider) VFS() *vfs.VFS {
if p == nil {