Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

operations: fix silent truncation of streaming uploads whose source ends early

Nick Craig-Wood9b13247ba8ea
diff --git a/fs/operations/operations.go b/fs/operations/operations.go
index eb45a3136..fb2049e31 100644
--- a/fs/operations/operations.go
+++ b/fs/operations/operations.go
@@ -1838,6 +1838,15 @@ func RcatSize(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadClo
return nil, err
}
+
+ // Check transfer - the source may have ended before size
+ // bytes in which case the object will have been truncated
+ if sizeDiffers(ctx, info, obj) {
+ err = fmt.Errorf("corrupted on transfer: sizes differ src %d vs dst(%s) %d", info.Size(), fdst, obj.Size())
+ err = fs.CountError(ctx, err)
+ fs.Errorf(obj, "%v", err)
+ return obj, err
+ }
} else {
// Size unknown use Rcat
obj, err = Rcat(ctx, fdst, dstFileName, in, modTime, meta)
diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go
index 2ddb56a38..8c2bc868d 100644
--- a/fs/operations/operations_test.go
+++ b/fs/operations/operations_test.go
@@ -1637,6 +1637,18 @@ func TestRcatSize(t *testing.T) {
r.CheckRemoteItems(t, file1, file2)
}
+func TestRcatSizeShortEOF(t *testing.T) {
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+
+ const body = "------------------------------------------------------------"
+
+ // Upload declaring twice as many bytes as the source supplies
+ bodyReader := io.NopCloser(strings.NewReader(body))
+ _, err := operations.RcatSize(ctx, r.Fremote, "potato1", bodyReader, 2*int64(len(body)), t1, nil)
+ require.Error(t, err, "uploading a source which ends before its declared size must not succeed")
+}
+
func TestRcatSizeMetadata(t *testing.T) {
r := fstest.NewRun(t)