"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
operations: fix silent truncation of streaming uploads whose source ends early
Nick Craig-Wood9b13247ba8ea
diff --git a/fs/operations/operations.go b/fs/operations/operations.goindex eb45a3136..fb2049e31 100644--- a/fs/operations/operations.go+++ b/fs/operations/operations.go@@ -1838,6 +1838,15 @@ func RcatSize(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloreturn nil, err}++ // Check transfer - the source may have ended before size+ // bytes in which case the object will have been truncated+ if sizeDiffers(ctx, info, obj) {+ err = fmt.Errorf("corrupted on transfer: sizes differ src %d vs dst(%s) %d", info.Size(), fdst, obj.Size())+ err = fs.CountError(ctx, err)+ fs.Errorf(obj, "%v", err)+ return obj, err+ }} else {// Size unknown use Rcatobj, err = Rcat(ctx, fdst, dstFileName, in, modTime, meta)diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.goindex 2ddb56a38..8c2bc868d 100644--- a/fs/operations/operations_test.go+++ b/fs/operations/operations_test.go@@ -1637,6 +1637,18 @@ func TestRcatSize(t *testing.T) {r.CheckRemoteItems(t, file1, file2)}+func TestRcatSizeShortEOF(t *testing.T) {+ ctx := context.Background()+ r := fstest.NewRun(t)++ const body = "------------------------------------------------------------"++ // Upload declaring twice as many bytes as the source supplies+ bodyReader := io.NopCloser(strings.NewReader(body))+ _, err := operations.RcatSize(ctx, r.Fremote, "potato1", bodyReader, 2*int64(len(body)), t1, nil)+ require.Error(t, err, "uploading a source which ends before its declared size must not succeed")+}+func TestRcatSizeMetadata(t *testing.T) {r := fstest.NewRun(t)