"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
internetarchive: fix corrupted files being created when the source ends early
Nick Craig-Wood7357fb82a9ef
diff --git a/backend/internetarchive/internetarchive.go b/backend/internetarchive/internetarchive.goindex 65f045d22..cfd4e57ac 100644--- a/backend/internetarchive/internetarchive.go+++ b/backend/internetarchive/internetarchive.go@@ -30,6 +30,7 @@ import ("github.com/rclone/rclone/lib/encoder""github.com/rclone/rclone/lib/pacer""github.com/rclone/rclone/lib/random"+ "github.com/rclone/rclone/lib/readers""github.com/rclone/rclone/lib/rest")@@ -854,11 +855,12 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op}// make a PUT request at (IAS3)/encoded(:item/:path)+ counter := readers.NewCountingReader(in)var resp *http.Responseopts := rest.Opts{Method: "PUT",Path: "/" + url.PathEscape(path.Join(bucket, bucketPath)),- Body: in,+ Body: counter,ContentLength: &size,ExtraHeaders: headers,}@@ -868,6 +870,12 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, opreturn o.fs.shouldRetry(resp, err)})+ // Check the source supplied the number of bytes it declared+ // otherwise a truncated file would be stored as a good upload+ if err == nil && size >= 0 && int64(counter.BytesRead()) != size {+ err = fmt.Errorf("expected %d bytes in input, but got %d: %w", size, counter.BytesRead(), io.ErrUnexpectedEOF)+ }+// we can't update/find metadata here as IA will "ingest" uploaded file(s)// upon uploads. (you can find its progress at https://archive.org/history/ItemNameHere )// or we have to wait for finish? (needs polling (frontend)/metadata/:item or scraping (frontend)/history/:item)