"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
serve webdav: add gzip compression for compressible responses
Leon Brocard6f1678419fc8
diff --git a/cmd/serve/webdav/webdav.go b/cmd/serve/webdav/webdav.goindex a674a6955..12a7bcc00 100644--- a/cmd/serve/webdav/webdav.go+++ b/cmd/serve/webdav/webdav.go@@ -122,6 +122,13 @@ supported hash on the backend or you can use a named hash such as"MD5" or "SHA-1". Use the [hashsum](/commands/rclone_hashsum/) commandto see the full list.+### Gzip compression++The server will compress certain response bodies (text and XML, including+WebDAV PROPFIND responses) using gzip when the client advertises gzip+support via the ` + "`Accept-Encoding: gzip`" + ` request header. This reduces+bandwidth usage.+### Access WebDAV on WindowsWebDAV shared folder can be mapped as a drive on Windows, however the default@@ -236,6 +243,20 @@ type WebDAV struct {etagHashType hash.Type}+func webDAVCompressMiddleware() func(http.Handler) http.Handler {+ compress := middleware.Compress(5, "text/*", "application/xml")+ return func(next http.Handler) http.Handler {+ compressedNext := compress(next)+ return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {+ if r.Header.Get("Range") != "" {+ next.ServeHTTP(rw, r)+ return+ }+ compressedNext.ServeHTTP(rw, r)+ })+ }+}+// check interfacevar _ webdav.FileSystem = (*WebDAV)(nil)@@ -288,6 +309,7 @@ func newWebDAV(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Optrouter := w.server.Router()router.Use(+ webDAVCompressMiddleware(),middleware.SetHeader("Accept-Ranges", "bytes"),middleware.SetHeader("Server", "rclone/"+fs.Version),)diff --git a/cmd/serve/webdav/webdav_test.go b/cmd/serve/webdav/webdav_test.goindex 4066f6b76..77f16f18d 100644--- a/cmd/serve/webdav/webdav_test.go+++ b/cmd/serve/webdav/webdav_test.go@@ -8,6 +8,7 @@package webdavimport (+ "compress/gzip""context""flag""io"@@ -15,7 +16,6 @@ import ("os""strings""testing"- "time"_ "github.com/rclone/rclone/backend/local""github.com/rclone/rclone/cmd/serve/proxy"@@ -115,22 +115,6 @@ func TestHTTPFunction(t *testing.T) {assert.NoError(t, w.Shutdown())}()testURL := w.server.URLs()[0]- pause := time.Millisecond- i := 0- for ; i < 10; i++ {- resp, err := http.Head(testURL)- if err == nil {- _ = resp.Body.Close()- break- }- // t.Logf("couldn't connect, sleeping for %v: %v", pause, err)- time.Sleep(pause)- pause *= 2- }- if i >= 10 {- t.Fatal("couldn't connect to server")- }-HelpTestGET(t, testURL)}@@ -265,6 +249,103 @@ func HelpTestGET(t *testing.T, testURL string) {}}+// startAuthenticatedServer creates a webdav server with basic auth against+// the test files directory, starts it, waits for it to be ready, and returns+// the base URL. It registers cleanup to shut the server down.+func startAuthenticatedServer(t *testing.T) string {+ t.Helper()++ f, err := fs.NewFs(context.Background(), "../http/testdata/files")+ require.NoError(t, err)++ opt := Opt+ opt.HTTP.ListenAddr = []string{testBindAddress}+ opt.Template.Path = testTemplate+ opt.Auth.BasicUser = testUser+ opt.Auth.BasicPass = testPass++ w, err := newWebDAV(context.Background(), f, &opt, &vfscommon.Opt, &proxy.Opt)+ require.NoError(t, err)+ go func() {+ require.NoError(t, w.Serve())+ }()+ t.Cleanup(func() {+ assert.NoError(t, w.Shutdown())+ })++ testURL := w.server.URLs()[0]+ return testURL+}++func TestCompressedTextFile(t *testing.T) {+ testURL := startAuthenticatedServer(t)++ req, err := http.NewRequest("GET", testURL+"two.txt", nil)+ require.NoError(t, err)+ req.SetBasicAuth(testUser, testPass)+ req.Header.Set("Accept-Encoding", "gzip")++ resp, err := http.DefaultClient.Do(req)+ require.NoError(t, err)+ defer func() { _ = resp.Body.Close() }()++ assert.Equal(t, http.StatusOK, resp.StatusCode)+ assert.Equal(t, "gzip", resp.Header.Get("Content-Encoding"))++ gr, err := gzip.NewReader(resp.Body)+ require.NoError(t, err)+ defer func() { _ = gr.Close() }()++ body, err := io.ReadAll(gr)+ require.NoError(t, err)+ assert.Equal(t, "0123456789\n", string(body))+}++func TestCompressedPROPFIND(t *testing.T) {+ testURL := startAuthenticatedServer(t)++ req, err := http.NewRequest("PROPFIND", testURL, nil)+ require.NoError(t, err)+ req.SetBasicAuth(testUser, testPass)+ req.Header.Set("Depth", "1")+ req.Header.Set("Accept-Encoding", "gzip")++ resp, err := http.DefaultClient.Do(req)+ require.NoError(t, err)+ defer func() { _ = resp.Body.Close() }()++ assert.Equal(t, http.StatusMultiStatus, resp.StatusCode)+ assert.Equal(t, "gzip", resp.Header.Get("Content-Encoding"))++ gr, err := gzip.NewReader(resp.Body)+ require.NoError(t, err)+ defer func() { _ = gr.Close() }()++ body, err := io.ReadAll(gr)+ require.NoError(t, err)+ assert.Contains(t, string(body), "multistatus")+}++func TestRangeRequestNotCompressed(t *testing.T) {+ testURL := startAuthenticatedServer(t)++ req, err := http.NewRequest("GET", testURL+"two.txt", nil)+ require.NoError(t, err)+ req.SetBasicAuth(testUser, testPass)+ req.Header.Set("Accept-Encoding", "gzip")+ req.Header.Set("Range", "bytes=2-5")++ resp, err := http.DefaultClient.Do(req)+ require.NoError(t, err)+ defer func() { _ = resp.Body.Close() }()++ assert.Equal(t, http.StatusPartialContent, resp.StatusCode)+ assert.Empty(t, resp.Header.Get("Content-Encoding"))+ body, err := io.ReadAll(resp.Body)+ require.NoError(t, err)+ assert.Equal(t, "2345", string(body))+}+func TestRc(t *testing.T) {servetest.TestRc(t, rc.Params{"type": "webdav",