"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
serve http: fix downloads failing after 5 minutes with --auth-proxy
Nick Craig-Wood8457da3c8abb
diff --git a/cmd/serve/http/http.go b/cmd/serve/http/http.goindex c22b40d33..51d733115 100644--- a/cmd/serve/http/http.go+++ b/cmd/serve/http/http.go@@ -199,6 +199,7 @@ func newServer(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Optrouter := s.server.Router()router.Use(+ s.provider.HoldVFS,middleware.Compress(5),middleware.SetHeader("Accept-Ranges", "bytes"),middleware.SetHeader("Server", "rclone/"+fs.Version),diff --git a/cmd/serve/http/http_test.go b/cmd/serve/http/http_test.goindex 4567fa434..8d6f2a50f 100644--- a/cmd/serve/http/http_test.go+++ b/cmd/serve/http/http_test.go@@ -22,6 +22,7 @@ import ("github.com/rclone/rclone/fs/filter""github.com/rclone/rclone/fs/rc"libhttp "github.com/rclone/rclone/lib/http"+ "github.com/rclone/rclone/lib/random""github.com/rclone/rclone/vfs/vfscommon""github.com/stretchr/testify/assert""github.com/stretchr/testify/require"@@ -559,3 +560,40 @@ func TestNewServerError(t *testing.T) {require.Error(t, err)assert.Nil(t, s)}++// TestAuthProxyDownloadOutlivesCache checks a download in progress+// carries on working when the auth proxy drops its VFS from its cache,+// as it does when a download takes longer than the cache expiry time.+func TestAuthProxyDownloadOutlivesCache(t *testing.T) {+ root := t.TempDir()+ contents := random.String(32 * 1024 * 1024)+ require.NoError(t, os.WriteFile(filepath.Join(root, "download.bin"), []byte(contents), 0666))++ prog, err := filepath.Abs("../servetest/proxy_code.go")+ require.NoError(t, err)+ // FIXME this is untidy setting a global variable!+ proxy.Opt.AuthProxy = "go run " + prog + " " + root+ defer func() {+ proxy.Opt.AuthProxy = ""+ }()+ s, testURL := start(context.Background(), t, nil)+ defer func() { assert.NoError(t, s.Shutdown()) }()++ req, err := http.NewRequest("GET", testURL+"download.bin", nil)+ require.NoError(t, err)+ req.SetBasicAuth(testUser, testPass)+ resp, err := http.DefaultClient.Do(req)+ require.NoError(t, err)+ defer func() { _ = resp.Body.Close() }()+ require.Equal(t, http.StatusOK, resp.StatusCode)+ start := make([]byte, 1024)+ _, err = io.ReadFull(resp.Body, start)+ require.NoError(t, err)++ // Drop everything from the proxy's cache as if it had expired+ s.provider.Proxy().Shutdown()++ rest, err := io.ReadAll(resp.Body)+ require.NoError(t, err)+ assert.True(t, contents == string(start)+string(rest), "download corrupted")+}diff --git a/cmd/serve/proxy/proxy.go b/cmd/serve/proxy/proxy.goindex 5718588b3..ed0bb3395 100644--- a/cmd/serve/proxy/proxy.go+++ b/cmd/serve/proxy/proxy.go@@ -12,6 +12,7 @@ import ("encoding/json""errors""fmt"+ "net/http""net/netip""os/exec""strings"@@ -577,6 +578,26 @@ func (p *Provider) Get(ctx context.Context) (*vfs.VFS, error) {return VFS, nil}+// HoldVFS is HTTP middleware which holds the VFS for the request until+// the request has been served.+//+// An auth proxy shuts down a VFS when it expires from its cache, which+// a long upload or download can outlast as only the start of a request+// counts as a use of the VFS.+func (p *Provider) HoldVFS(next http.Handler) http.Handler {+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ // Requests which don't need auth, eg CORS preflight, have no VFS+ if VFS, err := p.Get(r.Context()); err == nil {+ if !VFS.Hold() {+ http.Error(w, "VFS has been shut down", http.StatusServiceUnavailable)+ return+ }+ defer VFS.Shutdown()+ }+ next.ServeHTTP(w, r)+ })+}+// VFS returns the fixed VFS, or nil if using an auth proxy.func (p *Provider) VFS() *vfs.VFS {if p == nil {