"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
filelu: avoid buffering entire file in memory
kingston125327ca25a4d22
diff --git a/backend/filelu/filelu_object.go b/backend/filelu/filelu_object.goindex 4c7f3145a..3867392b0 100644--- a/backend/filelu/filelu_object.go+++ b/backend/filelu/filelu_object.go@@ -1,7 +1,6 @@package fileluimport (- "bytes""context""encoding/json""fmt"@@ -109,22 +108,25 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadCloreturn false, fmt.Errorf("failed to download file: HTTP %d", resp.StatusCode)}- // Wrap the response body to handle offset and count- currentContents, err := io.ReadAll(resp.Body)- if err != nil {- return false, fmt.Errorf("failed to read response body: %w", err)- }-if offset > 0 {- if offset > int64(len(currentContents)) {- return false, fmt.Errorf("offset %d exceeds file size %d", offset, len(currentContents))+ _, err = io.CopyN(io.Discard, resp.Body, offset)+ if err != nil {+ _ = resp.Body.Close()+ return false, fmt.Errorf("failed to skip offset: %w", err)}- currentContents = currentContents[offset:]}- if count > 0 && count < int64(len(currentContents)) {- currentContents = currentContents[:count]++ if count > 0 {+ reader = struct {+ io.Reader+ io.Closer+ }{+ Reader: io.LimitReader(resp.Body, count),+ Closer: resp.Body,+ }+ } else {+ reader = resp.Body}- reader = io.NopCloser(bytes.NewReader(currentContents))return false, nil})