"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
lib/rest: validate ranged responses
Morax69e5aff2a93b
diff --git a/fs/fs.go b/fs/fs.goindex 6ca2c679b..ae630b15f 100644--- a/fs/fs.go+++ b/fs/fs.go@@ -53,6 +53,7 @@ var (ErrorFileNameTooLong = errors.New("file name too long")ErrorCantListRoot = errors.New("can't list root")ErrorFileTooSmall = errors.New("file too small for multipart upload")+ ErrorRangeIgnored = errors.New("server ignored requested range"))// FileTooSmallError is returned by OpenChunkWriter when a file is below thediff --git a/lib/rest/headers.go b/lib/rest/headers.goindex 3e52fbdaa..f3bd6bf30 100644--- a/lib/rest/headers.go+++ b/lib/rest/headers.go@@ -1,11 +1,161 @@package restimport (+ "errors"+ "fmt""net/http""strconv""strings"++ "github.com/rclone/rclone/fs")+type contentRange struct {+ start int64+ end int64+ size int64+}++func parseContentRange(value string) (contentRange, error) {+ const prefix = "bytes "+ if !strings.HasPrefix(value, prefix) {+ return contentRange{}, fmt.Errorf("doesn't start with %q", prefix)+ }++ rangeAndSize := strings.Split(value[len(prefix):], "/")+ if len(rangeAndSize) != 2 {+ return contentRange{}, errors.New("must contain one '/'")+ }+ bounds := strings.Split(rangeAndSize[0], "-")+ if len(bounds) != 2 {+ return contentRange{}, errors.New("must contain one '-'")+ }++ start, err := strconv.ParseInt(bounds[0], 10, 64)+ if err != nil || start < 0 {+ return contentRange{}, errors.New("invalid start")+ }+ end, err := strconv.ParseInt(bounds[1], 10, 64)+ if err != nil || end < start {+ return contentRange{}, errors.New("invalid end")+ }++ size := int64(-1)+ if rangeAndSize[1] != "*" {+ size, err = strconv.ParseInt(rangeAndSize[1], 10, 64)+ if err != nil || size < 0 || end >= size {+ return contentRange{}, errors.New("invalid complete length")+ }+ }++ return contentRange{start: start, end: end, size: size}, nil+}++// CheckContentRange checks that a response satisfies a Range open option.+// The size is the expected size of the complete representation, or -1 if it is+// unknown. Calls without a Range option are ignored.+func CheckContentRange(resp *http.Response, options []fs.OpenOption, size int64) error {+ var requestRange string+ for _, option := range options {+ key, value := option.Header()+ if strings.EqualFold(key, "Range") {+ requestRange = value+ }+ }+ if requestRange == "" {+ return nil+ }++ requested, err := fs.ParseRangeOption(requestRange)+ if err != nil {+ return fmt.Errorf("invalid requested range %q: %w", requestRange, err)+ }+ if requested.Start < 0 && requested.End < 0 {+ return fmt.Errorf("invalid requested range %q", requestRange)+ }+ if requested.Start >= 0 && requested.End >= 0 && requested.End < requested.Start {+ return fmt.Errorf("invalid requested range %q", requestRange)+ }+ if resp == nil {+ return errors.New("nil response to range request")+ }++ if resp.StatusCode == http.StatusOK {+ responseSize := size+ if resp.ContentLength >= 0 {+ if size >= 0 && resp.ContentLength != size {+ return fmt.Errorf("Content-Length %d does not match expected size %d", resp.ContentLength, size)+ }+ responseSize = resp.ContentLength+ }+ if responseSize >= 0 {+ offset, limit := requested.Decode(responseSize)+ if limit < 0 {+ limit = responseSize - offset+ }+ if offset == 0 && limit >= responseSize {+ return nil+ }+ } else if requested.Start == 0 && requested.End < 0 {+ return nil+ }+ return fmt.Errorf("%w %q", fs.ErrorRangeIgnored, requestRange)+ }+ if resp.StatusCode != http.StatusPartialContent {+ return fmt.Errorf("response status %d does not satisfy requested range %q", resp.StatusCode, requestRange)+ }++ responseRange := resp.Header.Get("Content-Range")+ got, err := parseContentRange(responseRange)+ if err != nil {+ return fmt.Errorf("invalid Content-Range %q: %w", responseRange, err)+ }++ if size >= 0 && got.size >= 0 && got.size != size {+ return fmt.Errorf("Content-Range %q does not match expected size %d", responseRange, size)+ }++ rangeSize := size+ if rangeSize < 0 {+ rangeSize = got.size+ }+ expectedStart := requested.Start+ expectedEnd := requested.End+ if requested.Start >= 0 {+ if rangeSize >= 0 && (expectedEnd < 0 || expectedEnd >= rangeSize) {+ expectedEnd = rangeSize - 1+ }+ } else if rangeSize >= 0 {+ expectedStart = rangeSize - requested.End+ if expectedStart < 0 {+ expectedStart = 0+ }+ expectedEnd = rangeSize - 1+ }++ var matches bool+ if requested.Start < 0 && rangeSize < 0 {+ matches = got.end-got.start+1 <= requested.End+ } else {+ matches = got.start == expectedStart+ if expectedEnd >= 0 {+ matches = matches && got.end == expectedEnd+ }+ }+ if !matches {+ return fmt.Errorf("Content-Range %q does not match requested range %q", responseRange, requestRange)+ }++ contentLength := got.end - got.start + 1+ if contentLength <= 0 {+ return fmt.Errorf("invalid Content-Range %q: length overflows", responseRange)+ }+ if resp.ContentLength >= 0 && resp.ContentLength != contentLength {+ return fmt.Errorf("Content-Length %d does not match Content-Range %q", resp.ContentLength, responseRange)+ }+ return nil+}+// ParseSizeFromHeaders parses HTTP response headers to get the full file size.// Returns -1 if the headers did not exist or were invalid.func ParseSizeFromHeaders(headers http.Header) (size int64) {diff --git a/lib/rest/headers_test.go b/lib/rest/headers_test.goindex 23400062d..9901cc56f 100644--- a/lib/rest/headers_test.go+++ b/lib/rest/headers_test.go@@ -4,6 +4,7 @@ import ("net/http""testing"+ "github.com/rclone/rclone/fs""github.com/stretchr/testify/assert")@@ -39,3 +40,196 @@ func TestParseSizeFromHeaders(t *testing.T) {assert.Equalf(t, testCase.Size, ParseSizeFromHeaders(headers), "%+v", testCase)}}++func TestCheckContentRange(t *testing.T) {+ testCases := []struct {+ name string+ status int+ contentRange string+ contentLength int64+ unknownSize bool+ options []fs.OpenOption+ wantErr bool+ wantErrorIs error+ }{+ {+ name: "exact range",+ status: http.StatusPartialContent,+ contentRange: "bytes 2-4/10",+ contentLength: 3,+ options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},+ },+ {+ name: "unknown content length",+ status: http.StatusPartialContent,+ contentRange: "bytes 2-4/10",+ contentLength: -1,+ options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},+ },+ {+ name: "unknown complete length",+ status: http.StatusPartialContent,+ contentRange: "bytes 2-4/*",+ contentLength: 3,+ options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},+ },+ {+ name: "open ended range",+ status: http.StatusPartialContent,+ contentRange: "bytes 2-9/10",+ contentLength: 8,+ options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: -1}},+ },+ {+ name: "suffix range",+ status: http.StatusPartialContent,+ contentRange: "bytes 6-9/10",+ contentLength: 4,+ options: []fs.OpenOption{&fs.RangeOption{Start: -1, End: 4}},+ },+ {+ name: "suffix range with unknown complete length",+ status: http.StatusPartialContent,+ contentRange: "bytes 6-9/*",+ contentLength: 4,+ options: []fs.OpenOption{&fs.RangeOption{Start: -1, End: 4}},+ },+ {+ name: "suffix range larger than representation",+ status: http.StatusPartialContent,+ contentRange: "bytes 0-9/10",+ contentLength: 10,+ options: []fs.OpenOption{&fs.RangeOption{Start: -1, End: 20}},+ },+ {+ name: "seek option",+ status: http.StatusPartialContent,+ contentRange: "bytes 2-9/10",+ contentLength: 8,+ options: []fs.OpenOption{&fs.SeekOption{Offset: 2}},+ },+ {+ name: "whole representation range",+ status: http.StatusOK,+ contentLength: 10,+ options: []fs.OpenOption{&fs.RangeOption{Start: 0, End: 9}},+ },+ {+ name: "open ended whole representation range with unknown size",+ status: http.StatusOK,+ contentLength: -1,+ unknownSize: true,+ options: []fs.OpenOption{&fs.RangeOption{Start: 0, End: -1}},+ },+ {+ name: "partial range ignored",+ status: http.StatusOK,+ contentLength: 10,+ options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},+ wantErr: true,+ wantErrorIs: fs.ErrorRangeIgnored,+ },+ {+ name: "whole range response has wrong length",+ status: http.StatusOK,+ contentLength: 9,+ options: []fs.OpenOption{&fs.RangeOption{Start: 0, End: 9}},+ wantErr: true,+ },+ {+ name: "no range requested",+ status: http.StatusPartialContent,+ contentRange: "bytes 2-4/10",+ contentLength: 3,+ },+ {+ name: "missing content range",+ status: http.StatusPartialContent,+ contentLength: 3,+ options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},+ wantErr: true,+ },+ {+ name: "wrong unit",+ status: http.StatusPartialContent,+ contentRange: "items 2-4/10",+ contentLength: 3,+ options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},+ wantErr: true,+ },+ {+ name: "wrong start",+ status: http.StatusPartialContent,+ contentRange: "bytes 0-2/10",+ contentLength: 3,+ options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},+ wantErr: true,+ },+ {+ name: "wrong end",+ status: http.StatusPartialContent,+ contentRange: "bytes 2-5/10",+ contentLength: 4,+ options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},+ wantErr: true,+ },+ {+ name: "wrong content length",+ status: http.StatusPartialContent,+ contentRange: "bytes 2-4/10",+ contentLength: 4,+ options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},+ wantErr: true,+ },+ {+ name: "invalid complete length",+ status: http.StatusPartialContent,+ contentRange: "bytes 2-10/10",+ contentLength: 9,+ options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 10}},+ wantErr: true,+ },+ {+ name: "wrong complete length",+ status: http.StatusPartialContent,+ contentRange: "bytes 2-4/11",+ contentLength: 3,+ options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},+ wantErr: true,+ },+ {+ name: "unexpected successful status",+ status: http.StatusNoContent,+ contentLength: 0,+ options: []fs.OpenOption{&fs.RangeOption{Start: 2, End: 4}},+ wantErr: true,+ },+ }++ for _, testCase := range testCases {+ t.Run(testCase.name, func(t *testing.T) {+ expectedSize := int64(10)+ if testCase.unknownSize {+ expectedSize = -1+ }+ resp := &http.Response{+ StatusCode: testCase.status,+ Header: make(http.Header),+ ContentLength: testCase.contentLength,+ }+ if testCase.contentRange != "" {+ resp.Header.Set("Content-Range", testCase.contentRange)+ }++ err := CheckContentRange(resp, testCase.options, expectedSize)+ if testCase.wantErr {+ assert.Error(t, err)+ } else {+ assert.NoError(t, err)+ }+ if testCase.wantErrorIs != nil {+ assert.ErrorIs(t, err, testCase.wantErrorIs)+ }+ })+ }+}