Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

operations: add operations.ReadFile to read the contents of a file into memory

Nick Craig-Wood282e34f2d56a
diff --git a/fs/operations/operations.go b/fs/operations/operations.go
index 37dc4246d..5ec8c4c1a 100644
--- a/fs/operations/operations.go
+++ b/fs/operations/operations.go
@@ -628,6 +628,25 @@ func DeleteFiles(ctx context.Context, toBeDeleted fs.ObjectsChan) error {
return DeleteFilesWithBackupDir(ctx, toBeDeleted, nil)
}
+// ReadFile reads the object into memory and accounts it
+func ReadFile(ctx context.Context, o fs.Object) (b []byte, err error) {
+ tr := accounting.Stats(ctx).NewTransfer(o, nil)
+ defer func() {
+ tr.Done(ctx, err)
+ }()
+ in0, err := Open(ctx, o)
+ if err != nil {
+ return nil, fmt.Errorf("failed to open %v: %w", o, err)
+ }
+ in := tr.Account(ctx, in0).WithBuffer() // account and buffer the transfer
+ defer fs.CheckClose(in, &err) // closes in0 also
+ b, err = io.ReadAll(in)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read %v: %w", o, err)
+ }
+ return b, nil
+}
+
// SameRemoteType returns true if fdst and fsrc are the same type
func SameRemoteType(fdst, fsrc fs.Info) bool {
return fmt.Sprintf("%T", fdst) == fmt.Sprintf("%T", fsrc)
diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go
index 3e66bb33a..96ce5704a 100644
--- a/fs/operations/operations_test.go
+++ b/fs/operations/operations_test.go
@@ -492,6 +492,23 @@ func TestMaxDeleteSize(t *testing.T) {
assert.Equal(t, int64(1), objects) // 10 or 100 bytes
}
+func TestReadFile(t *testing.T) {
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+ defer r.Finalise()
+
+ contents := "A file to read the contents."
+ file := r.WriteObject(ctx, "ReadFile", contents, t1)
+ r.CheckRemoteItems(t, file)
+
+ o, err := r.Fremote.NewObject(ctx, file.Path)
+ require.NoError(t, err)
+
+ buf, err := operations.ReadFile(ctx, o)
+ require.NoError(t, err)
+ assert.Equal(t, contents, string(buf))
+}
+
func TestRetry(t *testing.T) {
ctx := context.Background()