Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

lib/rest: validate ranged responses

Morax69e5aff2a93b
diff --git a/fs/fs.go b/fs/fs.go
index 6ca2c679b..ae630b15f 100644
--- a/fs/fs.go
+++ b/fs/fs.go
@@ -53,6 +53,7 @@ var (
ErrorFileNameTooLong = errors.New("file name too long")
ErrorCantListRoot = errors.New("can't list root")
ErrorFileTooSmall = errors.New("file too small for multipart upload")
+ ErrorRangeIgnored = errors.New("server ignored requested range")
)
// FileTooSmallError is returned by OpenChunkWriter when a file is below the
diff --git a/lib/rest/headers.go b/lib/rest/headers.go
index 3e52fbdaa..f3bd6bf30 100644
--- a/lib/rest/headers.go
+++ b/lib/rest/headers.go
@@ -1,11 +1,161 @@
package rest
import (
+ "errors"
+ "fmt"
"net/http"
"strconv"
"strings"
+
+ "github.com/rclone/rclone/fs"
)
+type contentRange struct {
+ start int64
+ end int64
+ size int64
+}
+
+func parseContentRange(value string) (contentRange, error) {
+ const prefix = "bytes "
+ if !strings.HasPrefix(value, prefix) {
+ return contentRange{}, fmt.Errorf("doesn't start with %q", prefix)
+ }
+
+ rangeAndSize := strings.Split(value[len(prefix):], "/")
+ if len(rangeAndSize) != 2 {
+ return contentRange{}, errors.New("must contain one '/'")
+ }
+ bounds := strings.Split(rangeAndSize[0], "-")
+ if len(bounds) != 2 {
+ return contentRange{}, errors.New("must contain one '-'")
+ }
+
+ start, err := strconv.ParseInt(bounds[0], 10, 64)
+ if err != nil || start < 0 {
+ return contentRange{}, errors.New("invalid start")
+ }
+ end, err := strconv.ParseInt(bounds[1], 10, 64)
+ if err != nil || end < start {
+ return contentRange{}, errors.New("invalid end")
+ }
+
+ size := int64(-1)
+ if rangeAndSize[1] != "*" {
+ size, err = strconv.ParseInt(rangeAndSize[1], 10, 64)
+ if err != nil || size < 0 || end >= size {
+ return contentRange{}, errors.New("invalid complete length")
+ }
+ }
+
+ return contentRange{start: start, end: end, size: size}, nil
+}
+
+// CheckContentRange checks that a response satisfies a Range open option.
+// The size is the expected size of the complete representation, or -1 if it is
+// unknown. Calls without a Range option are ignored.
+func CheckContentRange(resp *http.Response, options []fs.OpenOption, size int64) error {
+ var requestRange string
+ for _, option := range options {
+ key, value := option.Header()
+ if strings.EqualFold(key, "Range") {
+ requestRange = value
+ }
+ }
+ if requestRange == "" {
+ return nil
+ }
+
+ requested, err := fs.ParseRangeOption(requestRange)
+ if err != nil {
+ return fmt.Errorf("invalid requested range %q: %w", requestRange, err)
+ }
+ if requested.Start < 0 && requested.End < 0 {
+ return fmt.Errorf("invalid requested range %q", requestRange)
+ }
+ if requested.Start >= 0 && requested.End >= 0 && requested.End < requested.Start {
+ return fmt.Errorf("invalid requested range %q", requestRange)
+ }
+ if resp == nil {
+ return errors.New("nil response to range request")
+ }
+
+ if resp.StatusCode == http.StatusOK {
+ responseSize := size
+ if resp.ContentLength >= 0 {
+ if size >= 0 && resp.ContentLength != size {
+ return fmt.Errorf("Content-Length %d does not match expected size %d", resp.ContentLength, size)
+ }
+ responseSize = resp.ContentLength
+ }
+ if responseSize >= 0 {
+ offset, limit := requested.Decode(responseSize)
+ if limit < 0 {
+ limit = responseSize - offset
+ }
+ if offset == 0 && limit >= responseSize {
+ return nil
+ }
+ } else if requested.Start == 0 && requested.End < 0 {
+ return nil
+ }
+ return fmt.Errorf("%w %q", fs.ErrorRangeIgnored, requestRange)
+ }
+ if resp.StatusCode != http.StatusPartialContent {
+ return fmt.Errorf("response status %d does not satisfy requested range %q", resp.StatusCode, requestRange)
+ }
+
+ responseRange := resp.Header.Get("Content-Range")
+ got, err := parseContentRange(responseRange)
+ if err != nil {
+ return fmt.Errorf("invalid Content-Range %q: %w", responseRange, err)
+ }
+
+ if size >= 0 && got.size >= 0 && got.size != size {
+ return fmt.Errorf("Content-Range %q does not match expected size %d", responseRange, size)
+ }
+
+ rangeSize := size
+ if rangeSize < 0 {
+ rangeSize = got.size
+ }
+ expectedStart := requested.Start
+ expectedEnd := requested.End
+ if requested.Start >= 0 {
+ if rangeSize >= 0 && (expectedEnd < 0 || expectedEnd >= rangeSize) {
+ expectedEnd = rangeSize - 1
+ }
+ } else if rangeSize >= 0 {
+ expectedStart = rangeSize - requested.End
+ if expectedStart < 0 {
+ expectedStart = 0
+ }
+ expectedEnd = rangeSize - 1
+ }
+
+ var matches bool
+ if requested.Start < 0 && rangeSize < 0 {
+ matches = got.end-got.start+1 <= requested.End
+ } else {
+ matches = got.start == expectedStart
+ if expectedEnd >= 0 {
+ matches = matches && got.end == expectedEnd
+ }
+ }
+ if !matches {
+ return fmt.Errorf("Content-Range %q does not match requested range %q", responseRange, requestRange)
+ }
+
+ contentLength := got.end - got.start + 1
+ if contentLength <= 0 {
+ return fmt.Errorf("invalid Content-Range %q: length overflows", responseRange)
+ }
+ if resp.ContentLength >= 0 && resp.ContentLength != contentLength {
+ return fmt.Errorf("Content-Length %d does not match Content-Range %q", resp.ContentLength, responseRange)
+ }
+ return nil
+}
+
// ParseSizeFromHeaders parses HTTP response headers to get the full file size.
// Returns -1 if the headers did not exist or were invalid.
func ParseSizeFromHeaders(headers http.Header) (size int64) {
diff --git a/lib/rest/headers_test.go b/lib/rest/headers_test.go
index 23400062d..9901cc56f 100644
--- a/lib/rest/headers_test.go
+++ b/lib/rest/headers_test.go
@@ -4,6 +4,7 @@ import (
"net/http"
"testing"
+ "github.com/rclone/rclone/fs"
"github.com/stretchr/testify/assert"
)
@@ -39,3 +40,196 @@ func TestParseSizeFromHeaders(t *testing.T) {
assert.Equalf(t, testCase.Size, ParseSizeFromHeaders(headers), "%+v", testCase)
}
}
+
+func TestCheckContentRange(t *testing.T) {
+ testCases := []struct {
+ name string
+ status int
+ contentRange string
+ contentLength int64
+ unknownSize bool
+ options []fs.OpenOption
+ wantErr bool
+ wantErrorIs error
+ }{
+ {
+ name: "exact range",
+ status: http.StatusPartialContent,
+ contentRange: "bytes 2-4/10",
+ contentLength: 3,
+ options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},
+ },
+ {
+ name: "unknown content length",
+ status: http.StatusPartialContent,
+ contentRange: "bytes 2-4/10",
+ contentLength: -1,
+ options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},
+ },
+ {
+ name: "unknown complete length",
+ status: http.StatusPartialContent,
+ contentRange: "bytes 2-4/*",
+ contentLength: 3,
+ options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},
+ },
+ {
+ name: "open ended range",
+ status: http.StatusPartialContent,
+ contentRange: "bytes 2-9/10",
+ contentLength: 8,
+ options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: -1}},
+ },
+ {
+ name: "suffix range",
+ status: http.StatusPartialContent,
+ contentRange: "bytes 6-9/10",
+ contentLength: 4,
+ options: []fs.OpenOption{&fs.RangeOption{Start: -1, End: 4}},
+ },
+ {
+ name: "suffix range with unknown complete length",
+ status: http.StatusPartialContent,
+ contentRange: "bytes 6-9/*",
+ contentLength: 4,
+ options: []fs.OpenOption{&fs.RangeOption{Start: -1, End: 4}},
+ },
+ {
+ name: "suffix range larger than representation",
+ status: http.StatusPartialContent,
+ contentRange: "bytes 0-9/10",
+ contentLength: 10,
+ options: []fs.OpenOption{&fs.RangeOption{Start: -1, End: 20}},
+ },
+ {
+ name: "seek option",
+ status: http.StatusPartialContent,
+ contentRange: "bytes 2-9/10",
+ contentLength: 8,
+ options: []fs.OpenOption{&fs.SeekOption{Offset: 2}},
+ },
+ {
+ name: "whole representation range",
+ status: http.StatusOK,
+ contentLength: 10,
+ options: []fs.OpenOption{&fs.RangeOption{Start: 0, End: 9}},
+ },
+ {
+ name: "open ended whole representation range with unknown size",
+ status: http.StatusOK,
+ contentLength: -1,
+ unknownSize: true,
+ options: []fs.OpenOption{&fs.RangeOption{Start: 0, End: -1}},
+ },
+ {
+ name: "partial range ignored",
+ status: http.StatusOK,
+ contentLength: 10,
+ options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},
+ wantErr: true,
+ wantErrorIs: fs.ErrorRangeIgnored,
+ },
+ {
+ name: "whole range response has wrong length",
+ status: http.StatusOK,
+ contentLength: 9,
+ options: []fs.OpenOption{&fs.RangeOption{Start: 0, End: 9}},
+ wantErr: true,
+ },
+ {
+ name: "no range requested",
+ status: http.StatusPartialContent,
+ contentRange: "bytes 2-4/10",
+ contentLength: 3,
+ },
+ {
+ name: "missing content range",
+ status: http.StatusPartialContent,
+ contentLength: 3,
+ options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},
+ wantErr: true,
+ },
+ {
+ name: "wrong unit",
+ status: http.StatusPartialContent,
+ contentRange: "items 2-4/10",
+ contentLength: 3,
+ options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},
+ wantErr: true,
+ },
+ {
+ name: "wrong start",
+ status: http.StatusPartialContent,
+ contentRange: "bytes 0-2/10",
+ contentLength: 3,
+ options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},
+ wantErr: true,
+ },
+ {
+ name: "wrong end",
+ status: http.StatusPartialContent,
+ contentRange: "bytes 2-5/10",
+ contentLength: 4,
+ options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},
+ wantErr: true,
+ },
+ {
+ name: "wrong content length",
+ status: http.StatusPartialContent,
+ contentRange: "bytes 2-4/10",
+ contentLength: 4,
+ options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},
+ wantErr: true,
+ },
+ {
+ name: "invalid complete length",
+ status: http.StatusPartialContent,
+ contentRange: "bytes 2-10/10",
+ contentLength: 9,
+ options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 10}},
+ wantErr: true,
+ },
+ {
+ name: "wrong complete length",
+ status: http.StatusPartialContent,
+ contentRange: "bytes 2-4/11",
+ contentLength: 3,
+ options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},
+ wantErr: true,
+ },
+ {
+ name: "unexpected successful status",
+ status: http.StatusNoContent,
+ contentLength: 0,
+ options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},
+ wantErr: true,
+ },
+ }
+
+ for _, testCase := range testCases {
+ t.Run(testCase.name, func(t *testing.T) {
+ expectedSize := int64(10)
+ if testCase.unknownSize {
+ expectedSize = -1
+ }
+ resp := &http.Response{
+ StatusCode: testCase.status,
+ Header: make(http.Header),
+ ContentLength: testCase.contentLength,
+ }
+ if testCase.contentRange != "" {
+ resp.Header.Set("Content-Range", testCase.contentRange)
+ }
+
+ err := CheckContentRange(resp, testCase.options, expectedSize)
+ if testCase.wantErr {
+ assert.Error(t, err)
+ } else {
+ assert.NoError(t, err)
+ }
+ if testCase.wantErrorIs != nil {
+ assert.ErrorIs(t, err, testCase.wantErrorIs)
+ }
+ })
+ }
+}