Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

serve webdav: add gzip compression for compressible responses

Leon Brocard6f1678419fc8
diff --git a/cmd/serve/webdav/webdav.go b/cmd/serve/webdav/webdav.go
index a674a6955..12a7bcc00 100644
--- a/cmd/serve/webdav/webdav.go
+++ b/cmd/serve/webdav/webdav.go
@@ -122,6 +122,13 @@ supported hash on the backend or you can use a named hash such as
"MD5" or "SHA-1". Use the [hashsum](/commands/rclone_hashsum/) command
to see the full list.
+### Gzip compression
+
+The server will compress certain response bodies (text and XML, including
+WebDAV PROPFIND responses) using gzip when the client advertises gzip
+support via the ` + "`Accept-Encoding: gzip`" + ` request header. This reduces
+bandwidth usage.
+
### Access WebDAV on Windows
WebDAV shared folder can be mapped as a drive on Windows, however the default
@@ -236,6 +243,20 @@ type WebDAV struct {
etagHashType hash.Type
}
+func webDAVCompressMiddleware() func(http.Handler) http.Handler {
+ compress := middleware.Compress(5, "text/*", "application/xml")
+ return func(next http.Handler) http.Handler {
+ compressedNext := compress(next)
+ return http.HandlerFunc(func(rw http.ResponseWriter, r *http.Request) {
+ if r.Header.Get("Range") != "" {
+ next.ServeHTTP(rw, r)
+ return
+ }
+ compressedNext.ServeHTTP(rw, r)
+ })
+ }
+}
+
// check interface
var _ webdav.FileSystem = (*WebDAV)(nil)
@@ -288,6 +309,7 @@ func newWebDAV(ctx context.Context, f fs.Fs, opt *Options, vfsOpt *vfscommon.Opt
router := w.server.Router()
router.Use(
+ webDAVCompressMiddleware(),
middleware.SetHeader("Accept-Ranges", "bytes"),
middleware.SetHeader("Server", "rclone/"+fs.Version),
)
diff --git a/cmd/serve/webdav/webdav_test.go b/cmd/serve/webdav/webdav_test.go
index 4066f6b76..77f16f18d 100644
--- a/cmd/serve/webdav/webdav_test.go
+++ b/cmd/serve/webdav/webdav_test.go
@@ -8,6 +8,7 @@
package webdav
import (
+ "compress/gzip"
"context"
"flag"
"io"
@@ -15,7 +16,6 @@ import (
"os"
"strings"
"testing"
- "time"
_ "github.com/rclone/rclone/backend/local"
"github.com/rclone/rclone/cmd/serve/proxy"
@@ -115,22 +115,6 @@ func TestHTTPFunction(t *testing.T) {
assert.NoError(t, w.Shutdown())
}()
testURL := w.server.URLs()[0]
- pause := time.Millisecond
- i := 0
- for ; i < 10; i++ {
- resp, err := http.Head(testURL)
- if err == nil {
- _ = resp.Body.Close()
- break
- }
- // t.Logf("couldn't connect, sleeping for %v: %v", pause, err)
- time.Sleep(pause)
- pause *= 2
- }
- if i >= 10 {
- t.Fatal("couldn't connect to server")
- }
-
HelpTestGET(t, testURL)
}
@@ -265,6 +249,103 @@ func HelpTestGET(t *testing.T, testURL string) {
}
}
+// startAuthenticatedServer creates a webdav server with basic auth against
+// the test files directory, starts it, waits for it to be ready, and returns
+// the base URL. It registers cleanup to shut the server down.
+func startAuthenticatedServer(t *testing.T) string {
+ t.Helper()
+
+ f, err := fs.NewFs(context.Background(), "../http/testdata/files")
+ require.NoError(t, err)
+
+ opt := Opt
+ opt.HTTP.ListenAddr = []string{testBindAddress}
+ opt.Template.Path = testTemplate
+ opt.Auth.BasicUser = testUser
+ opt.Auth.BasicPass = testPass
+
+ w, err := newWebDAV(context.Background(), f, &opt, &vfscommon.Opt, &proxy.Opt)
+ require.NoError(t, err)
+ go func() {
+ require.NoError(t, w.Serve())
+ }()
+ t.Cleanup(func() {
+ assert.NoError(t, w.Shutdown())
+ })
+
+ testURL := w.server.URLs()[0]
+ return testURL
+}
+
+func TestCompressedTextFile(t *testing.T) {
+ testURL := startAuthenticatedServer(t)
+
+ req, err := http.NewRequest("GET", testURL+"two.txt", nil)
+ require.NoError(t, err)
+ req.SetBasicAuth(testUser, testPass)
+ req.Header.Set("Accept-Encoding", "gzip")
+
+ resp, err := http.DefaultClient.Do(req)
+ require.NoError(t, err)
+ defer func() { _ = resp.Body.Close() }()
+
+ assert.Equal(t, http.StatusOK, resp.StatusCode)
+ assert.Equal(t, "gzip", resp.Header.Get("Content-Encoding"))
+
+ gr, err := gzip.NewReader(resp.Body)
+ require.NoError(t, err)
+ defer func() { _ = gr.Close() }()
+
+ body, err := io.ReadAll(gr)
+ require.NoError(t, err)
+ assert.Equal(t, "0123456789\n", string(body))
+}
+
+func TestCompressedPROPFIND(t *testing.T) {
+ testURL := startAuthenticatedServer(t)
+
+ req, err := http.NewRequest("PROPFIND", testURL, nil)
+ require.NoError(t, err)
+ req.SetBasicAuth(testUser, testPass)
+ req.Header.Set("Depth", "1")
+ req.Header.Set("Accept-Encoding", "gzip")
+
+ resp, err := http.DefaultClient.Do(req)
+ require.NoError(t, err)
+ defer func() { _ = resp.Body.Close() }()
+
+ assert.Equal(t, http.StatusMultiStatus, resp.StatusCode)
+ assert.Equal(t, "gzip", resp.Header.Get("Content-Encoding"))
+
+ gr, err := gzip.NewReader(resp.Body)
+ require.NoError(t, err)
+ defer func() { _ = gr.Close() }()
+
+ body, err := io.ReadAll(gr)
+ require.NoError(t, err)
+ assert.Contains(t, string(body), "multistatus")
+}
+
+func TestRangeRequestNotCompressed(t *testing.T) {
+ testURL := startAuthenticatedServer(t)
+
+ req, err := http.NewRequest("GET", testURL+"two.txt", nil)
+ require.NoError(t, err)
+ req.SetBasicAuth(testUser, testPass)
+ req.Header.Set("Accept-Encoding", "gzip")
+ req.Header.Set("Range", "bytes=2-5")
+
+ resp, err := http.DefaultClient.Do(req)
+ require.NoError(t, err)
+ defer func() { _ = resp.Body.Close() }()
+
+ assert.Equal(t, http.StatusPartialContent, resp.StatusCode)
+ assert.Empty(t, resp.Header.Get("Content-Encoding"))
+ body, err := io.ReadAll(resp.Body)
+ require.NoError(t, err)
+ assert.Equal(t, "2345", string(body))
+}
+
func TestRc(t *testing.T) {
servetest.TestRc(t, rc.Params{
"type": "webdav",