Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

filelu: avoid buffering entire file in memory

kingston125327ca25a4d22
diff --git a/backend/filelu/filelu_object.go b/backend/filelu/filelu_object.go
index 4c7f3145a..3867392b0 100644
--- a/backend/filelu/filelu_object.go
+++ b/backend/filelu/filelu_object.go
@@ -1,7 +1,6 @@
package filelu
import (
- "bytes"
"context"
"encoding/json"
"fmt"
@@ -109,22 +108,25 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (io.ReadClo
return false, fmt.Errorf("failed to download file: HTTP %d", resp.StatusCode)
}
- // Wrap the response body to handle offset and count
- currentContents, err := io.ReadAll(resp.Body)
- if err != nil {
- return false, fmt.Errorf("failed to read response body: %w", err)
- }
-
if offset > 0 {
- if offset > int64(len(currentContents)) {
- return false, fmt.Errorf("offset %d exceeds file size %d", offset, len(currentContents))
+ _, err = io.CopyN(io.Discard, resp.Body, offset)
+ if err != nil {
+ _ = resp.Body.Close()
+ return false, fmt.Errorf("failed to skip offset: %w", err)
}
- currentContents = currentContents[offset:]
}
- if count > 0 && count < int64(len(currentContents)) {
- currentContents = currentContents[:count]
+
+ if count > 0 {
+ reader = struct {
+ io.Reader
+ io.Closer
+ }{
+ Reader: io.LimitReader(resp.Body, count),
+ Closer: resp.Body,
+ }
+ } else {
+ reader = resp.Body
}
- reader = io.NopCloser(bytes.NewReader(currentContents))
return false, nil
})