"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
rest: fix retries reading the request body after the request has finished
Nick Craig-Wooda18d9a6886a9
diff --git a/lib/rest/rest.go b/lib/rest/rest.goindex 91d324ee8..771d33263 100644--- a/lib/rest/rest.go+++ b/lib/rest/rest.go@@ -322,7 +322,16 @@ func (api *Client) Call(ctx context.Context, opts *Opts) (resp *http.Response, eif len(opts.Parameters) > 0 {url += "?" + opts.Parameters.Encode()}- body := readers.NoCloser(opts.Body)+ var bodyClosed chan struct{}+ body := opts.Body+ if _, hasClose := body.(io.Closer); hasClose {+ // Hide the body's Close method from the transport so it can't+ // close the caller's body, arranging for the transport's Close+ // to signal that it has finished with the body instead.+ bodyClosed = make(chan struct{})+ closed := bodyClosed+ body = readers.NoCloserNotify(body, func() { close(closed) })+ }// If length is set and zero then nil out the body to stop use// use of chunked encoding and insert a "Content-Length: 0"// header.@@ -331,6 +340,7 @@ func (api *Client) Call(ctx context.Context, opts *Opts) (resp *http.Response, e// files except 0 length files.if opts.ContentLength != nil && *opts.ContentLength == 0 {body = nil+ bodyClosed = nil}req, err := http.NewRequestWithContext(ctx, opts.Method, url, body)if err != nil {@@ -402,6 +412,14 @@ func (api *Client) Call(ctx context.Context, opts *Opts) (resp *http.Response, e}api.mu.RUnlock()resp, err = c.Do(req)+ if bodyClosed != nil {+ // Wait for the transport to close the request body before+ // returning. It always closes it, but is documented to+ // possibly do so in a different goroutine even after Do has+ // returned, and the caller may seek or close the body for a+ // retry as soon as we return.+ <-bodyClosed+ }api.mu.RLock()if err != nil {return nil, err@@ -433,6 +451,25 @@ func CreateFormFile(w *multipart.Writer, fieldname, filename, contentType stringreturn w.CreatePart(h)}+// multipartBody is the request body of a multipart upload as returned+// by MultipartUpload.+//+// Close stops the goroutine writing the form and waits for it to+// finish, so afterwards the file reader passed to MultipartUpload is+// no longer being read from. It is safe to call multiple times.+type multipartBody struct {+ *io.PipeReader+ done <-chan struct{}+}++// Close aborts reading the form and waits for the goroutine writing+// it to stop.+func (b *multipartBody) Close() error {+ err := b.PipeReader.Close()+ <-b.done+ return err+}+// MultipartUpload creates an io.Reader which produces an encoded a// multipart form upload from the params passed in and the passed in//@@ -443,6 +480,11 @@ func CreateFormFile(w *multipart.Writer, fieldname, filename, contentType string//// the int64 returned is the overhead in addition to the file contents, in case Content-Length is required//+// Closing the returned reader stops the goroutine writing the form+// and waits for it to finish, after which in is no longer being read+// from. Callers which retry with a seekable in should close it after+// each attempt.+//// NB This doesn't allow setting the content type of the attachmentfunc MultipartUpload(ctx context.Context, in io.Reader, params url.Values, contentName, fileName string, contentType string) (io.ReadCloser, string, int64, error) {bodyReader, bodyWriter := io.Pipe()@@ -529,7 +571,7 @@ func MultipartUpload(ctx context.Context, in io.Reader, params url.Values, conte_ = bodyWriter.Close()}()- return bodyReader, formContentType, multipartLength, nil+ return &multipartBody{PipeReader: bodyReader, done: quit}, formContentType, multipartLength, nil}// CallJSON runs Call and decodes the body as a JSON object into response (if not nil)@@ -600,14 +642,20 @@ func (api *Client) callCodec(ctx context.Context, opts *Opts, request any, respo}opts = opts.Copy()+ var mpBody io.ReadCloservar overhead int64- opts.Body, opts.ContentType, overhead, err = MultipartUpload(ctx, opts.Body, params, opts.MultipartContentName, opts.MultipartFileName, opts.MultipartContentType)+ mpBody, opts.ContentType, overhead, err = MultipartUpload(ctx, opts.Body, params, opts.MultipartContentName, opts.MultipartFileName, opts.MultipartContentType)if err != nil {return nil, err}+ opts.Body = mpBodyif opts.ContentLength != nil {*opts.ContentLength += overhead}+ // Stop the multipart writer reading the body when the call has+ // finished so the caller can seek or close the body (e.g. to+ // retry) as soon as we return.+ defer func() { _ = mpBody.Close() }()}resp, err = api.Call(ctx, opts)if err != nil {