"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
huaweidrive: reuse resumable-upload chunk buffers via the global pool
Nick Craig-Wood204f8171000c
diff --git a/backend/huaweidrive/huaweidrive.go b/backend/huaweidrive/huaweidrive.goindex e4fd732e7..81d69c9da 100644--- a/backend/huaweidrive/huaweidrive.go+++ b/backend/huaweidrive/huaweidrive.go@@ -1916,17 +1916,22 @@ func (o *Object) uploadResume(ctx context.Context, in io.Reader, leaf, directory// single upload size according to the Huawei Drive APIchunkSize := min(max(int64(o.fs.opt.ChunkSize), 256*1024), 64*1024*1024)- buf := make([]byte, chunkSize)var offset int64for offset < size {- n, err := io.ReadFull(in, buf)- if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF {+ // Buffer the chunk in pooled memory so it can be re-sent on retry+ rw := pool.NewRW(pool.Global())+ n, err := io.CopyN(rw, in, min(size-offset, chunkSize))+ if err != nil {+ // io.EOF here means the source ran out before the declared size+ if err == io.EOF {+ err = io.ErrUnexpectedEOF+ }+ _ = rw.Close()return fmt.Errorf("failed to read chunk at offset %d: %w", offset, err)}- chunk := buf[:n]- end := offset + int64(n) - 1+ end := offset + n - 1// Upload this chunk using the original client with OAuth authenticationchunkOpts := rest.Opts{@@ -1934,17 +1939,20 @@ func (o *Object) uploadResume(ctx context.Context, in io.Reader, leaf, directoryRootURL: location,Path: "", // Empty path since RootURL contains the full URLExtraHeaders: map[string]string{- "Content-Range": fmt.Sprintf("bytes %d-%d/%d", offset, end, size),- "Content-Length": strconv.Itoa(n),- "Content-Type": mimeType,+ "Content-Range": fmt.Sprintf("bytes %d-%d/%d", offset, end, size),+ "Content-Type": mimeType,},+ ContentLength: &n,+ Body: rw,}var info *api.Filevar chunkResp *http.Responseerr = o.fs.pacer.Call(func() (bool, error) {- chunkOpts.Body = bytes.NewReader(chunk)- var err error+ _, err := rw.Seek(0, io.SeekStart)+ if err != nil {+ return false, err+ }chunkResp, err = srv.CallJSON(ctx, &chunkOpts, nil, &info)// 308 means continue uploading (incomplete)if chunkResp != nil && chunkResp.StatusCode == 308 {@@ -1958,11 +1966,15 @@ func (o *Object) uploadResume(ctx context.Context, in io.Reader, leaf, directory}return shouldRetry(ctx, chunkResp, err)})+ closeErr := rw.Close()if err != nil {return fmt.Errorf("failed to upload chunk at offset %d: %w", offset, err)}+ if closeErr != nil {+ return closeErr+ }- offset += int64(n)+ offset += n// If we got file info back, we're doneif info != nil && info.ID != "" {