"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
onedrive: reuse upload chunk buffers via the global memory pool
Nick Craig-Wood689081b410fd
diff --git a/backend/onedrive/onedrive.go b/backend/onedrive/onedrive.goindex daef4d817..6b6356c4c 100644--- a/backend/onedrive/onedrive.go+++ b/backend/onedrive/onedrive.go@@ -37,9 +37,9 @@ import ("github.com/rclone/rclone/lib/atexit""github.com/rclone/rclone/lib/dircache""github.com/rclone/rclone/lib/encoder"+ "github.com/rclone/rclone/lib/multipart""github.com/rclone/rclone/lib/oauthutil""github.com/rclone/rclone/lib/pacer"- "github.com/rclone/rclone/lib/readers""github.com/rclone/rclone/lib/rest")@@ -2748,12 +2748,26 @@ func (o *Object) uploadMultipart(ctx context.Context, in io.Reader, src fs.Objecposition := int64(0)for remaining > 0 {n := min(remaining, int64(o.fs.opt.ChunkSize))- seg := readers.NewRepeatableReader(io.LimitReader(in, n))+ // Buffer the chunk in memory from the global pool so it can be+ // re-sent (or partly re-sent after a 416) on retry+ rw := multipart.NewRW()+ _, err = io.CopyN(rw, in, n)+ if err != nil {+ _ = rw.Close()+ if err == io.EOF {+ err = fmt.Errorf("expected %d bytes in input, but got %d: %w", size, position, io.ErrUnexpectedEOF)+ }+ return nil, err+ }fs.Debugf(o, "Uploading segment %d/%d size %d", position, size, n)- info, err = o.uploadFragment(ctx, uploadURL, position, size, seg, n, options...)+ info, err = o.uploadFragment(ctx, uploadURL, position, size, rw, n, options...)+ closeErr := rw.Close()if err != nil {return nil, err}+ if closeErr != nil {+ return nil, closeErr+ }remaining -= nposition += n}diff --git a/backend/onedrive/upload_internal_test.go b/backend/onedrive/upload_internal_test.gonew file mode 100644index 000000000..aae575466--- /dev/null+++ b/backend/onedrive/upload_internal_test.go@@ -0,0 +1,120 @@+package onedrive++import (+ "bytes"+ "context"+ "encoding/json"+ "fmt"+ "io"+ "net/http"+ "net/http/httptest"+ "sync"+ "testing"+ "time"++ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/object"+ "github.com/rclone/rclone/lib/pacer"+ "github.com/rclone/rclone/lib/rest"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++// fragmentServer is a fake OneDrive upload session which records the+// fragments it receives. fail is called for each PUT before the body is+// accepted and may write an error response, returning true to reject it.+type fragmentServer struct {+ t *testing.T+ size int64+ mu sync.Mutex+ received []byte // bytes accepted so far+ fragments int // number of accepted fragments+ fail func(w http.ResponseWriter, attempt int) bool+ attempts int+}++func (s *fragmentServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {+ s.mu.Lock()+ defer s.mu.Unlock()+ switch {+ case r.Method == "POST" && r.URL.Path == "/root/createUploadSession":+ w.Header().Set("Content-Type", "application/json")+ _, _ = fmt.Fprintf(w, `{"uploadUrl":"http://%s/upload","nextExpectedRanges":["0-"]}`, r.Host)+ case r.Method == "GET" && r.URL.Path == "/upload":+ w.Header().Set("Content-Type", "application/json")+ _, _ = fmt.Fprintf(w, `{"nextExpectedRanges":["%d-%d"]}`, len(s.received), s.size-1)+ case r.Method == "PUT" && r.URL.Path == "/upload":+ body, err := io.ReadAll(r.Body)+ require.NoError(s.t, err)+ s.attempts+++ if s.fail != nil && s.fail(w, s.attempts) {+ return+ }+ var start, end, total int64+ _, err = fmt.Sscanf(r.Header.Get("Content-Range"), "bytes %d-%d/%d", &start, &end, &total)+ require.NoError(s.t, err)+ assert.Equal(s.t, int64(len(s.received)), start, "fragment sent out of order")+ assert.Equal(s.t, int64(len(body)), end-start+1, "Content-Range disagrees with body length")+ assert.Equal(s.t, s.size, total)+ s.received = append(s.received, body...)+ s.fragments+++ w.Header().Set("Content-Type", "application/json")+ if int64(len(s.received)) == s.size {+ w.WriteHeader(http.StatusCreated)+ _, _ = fmt.Fprintf(w, `{"id":"fake-id","name":"remote","size":%d,"file":{}}`, s.size)+ return+ }+ w.WriteHeader(http.StatusAccepted)+ _, _ = fmt.Fprintf(w, `{"nextExpectedRanges":["%d-%d"]}`, len(s.received), s.size-1)+ default:+ s.t.Errorf("unexpected request %s %s", r.Method, r.URL.Path)+ w.WriteHeader(http.StatusNotFound)+ }+}++// newUploadTest returns an Object wired to a fragmentServer which+// uploads content in chunks of chunkSize.+func newUploadTest(t *testing.T, content []byte, chunkSize int) (*Object, *fragmentServer, *httptest.Server) {+ s := &fragmentServer{t: t, size: int64(len(content))}+ server := httptest.NewServer(s)+ t.Cleanup(server.Close)+ client := rest.NewClient(server.Client()).SetRoot(server.URL)+ f := &Fs{+ ci: fs.GetConfig(context.Background()),+ srv: client,+ unAuth: client,+ pacer: fs.NewPacer(context.Background(), pacer.NewDefault(pacer.MinSleep(time.Millisecond), pacer.MaxSleep(10*time.Millisecond))),+ }+ f.opt.ChunkSize = fs.SizeSuffix(chunkSize)+ f.opt.UploadCutoff = fs.SizeSuffix(chunkSize)+ return &Object{fs: f}, s, server+}++// TestUploadMultipartSkip checks that a 416 response makes the upload+// read the server's position and re-send only the unreceived tail of+// the fragment.+func TestUploadMultipartSkip(t *testing.T) {+ content := bytes.Repeat([]byte("onedrive"), 1024)+ chunkSize := len(content) / 2+ skip := chunkSize / 4+ o, s, _ := newUploadTest(t, content, chunkSize)+ s.fail = func(w http.ResponseWriter, attempt int) bool {+ if attempt == 1 {+ // Pretend the first part of the fragment was received+ // before the connection broke+ s.received = append(s.received, content[:skip]...)+ w.Header().Set("Content-Type", "application/json")+ w.WriteHeader(http.StatusRequestedRangeNotSatisfiable)+ _ = json.NewEncoder(w).Encode(map[string]any{"error": map[string]any{"code": "invalidRange", "message": "bad range"}})+ return true+ }+ return false+ }+ src := object.NewStaticObjectInfo("remote", time.Now(), int64(len(content)), true, nil, nil)+ info, err := o.uploadMultipart(context.Background(), bytes.NewReader(content), src)+ require.NoError(t, err)+ require.NotNil(t, info)+ assert.Equal(t, content, s.received)+ assert.Equal(t, 2, s.fragments)+ assert.Equal(t, 3, s.attempts, "expected one 416, a partial re-send and the final fragment")+}