"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
operations: rework rcat so that it doesn't call the --metadata-mapper twice
Nick Craig-Wooda5700a4a53ab
diff --git a/fs/operations/operations.go b/fs/operations/operations.goindex 59877bcc8..37dc4246d 100644--- a/fs/operations/operations.go+++ b/fs/operations/operations.go@@ -36,6 +36,7 @@ import ("github.com/rclone/rclone/fs/walk""github.com/rclone/rclone/lib/atexit""github.com/rclone/rclone/lib/errcount"+ "github.com/rclone/rclone/lib/pacer""github.com/rclone/rclone/lib/random""github.com/rclone/rclone/lib/readers""golang.org/x/sync/errgroup"@@ -719,13 +720,18 @@ func Retry(ctx context.Context, o interface{}, maxTries int, fn func() error) (eif err == nil {break}- // Retry if err returned a retry error+ // End if ctx is in errorif fserrors.ContextError(ctx, &err) {break}+ // Retry if err returned a retry errorif fserrors.IsRetryError(err) || fserrors.ShouldRetry(err) {fs.Debugf(o, "Received error: %v - low level retry %d/%d", err, tries, maxTries)continue+ } else if t, ok := pacer.IsRetryAfter(err); ok {+ fs.Debugf(o, "Sleeping for %v (as indicated by the server) to obey Retry-After error: %v", t, err)+ time.Sleep(t)+ continue}break}@@ -1269,22 +1275,32 @@ func Cat(ctx context.Context, f fs.Fs, w io.Writer, offset, count int64, sep []b}// Rcat reads data from the Reader until EOF and uploads it to a file on remote+//+// in is closed at the end of the transferfunc Rcat(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser, modTime time.Time, meta fs.Metadata) (dst fs.Object, err error) {return rcatSrc(ctx, fdst, dstFileName, in, modTime, meta, nil)}// rcatSrc reads data from the Reader until EOF and uploads it to a file on remote//+// in is closed at the end of the transfer+//// Pass in fsrc if known or nil if notfunc rcatSrc(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadCloser, modTime time.Time, meta fs.Metadata, fsrc fs.Fs) (dst fs.Object, err error) {+ if SkipDestructive(ctx, dstFileName, "upload from pipe") {+ // prevents "broken pipe" errors+ _, err = io.Copy(io.Discard, in)+ return nil, err+ }+ci := fs.GetConfig(ctx)tr := accounting.Stats(ctx).NewTransferRemoteSize(dstFileName, -1, nil, fdst)defer func() {tr.Done(ctx, err)}()- in = tr.Account(ctx, in).WithBuffer()+ var streamIn io.Reader = tr.Account(ctx, in).WithBuffer()- readCounter := readers.NewCountingReader(in)+ readCounter := readers.NewCountingReader(streamIn)var trackingIn io.Readervar hasher *hash.MultiHashervar options []fs.OpenOption@@ -1307,87 +1323,91 @@ func rcatSrc(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadClosoptions = append(options, fs.MetadataOption(ci.MetadataSet))}- compare := func(dst fs.Object) error {- var sums map[hash.Type]string- opt := defaultEqualOpt(ctx)+ // get the sums from the hasher if in use, or nil+ getSums := func() (sums map[hash.Type]string) {if hasher != nil {- // force --checksum on if we have hashes- opt.checkSum = truesums = hasher.Sums()}- src := object.NewStaticObjectInfo(dstFileName, modTime, int64(readCounter.BytesRead()), false, sums, fdst).WithMetadata(meta)- if !equal(ctx, src, dst, opt) {- err = fmt.Errorf("corrupted on transfer")- err = fs.CountError(err)- fs.Errorf(dst, "%v", err)- return err- }- return nil+ return sums}- // check if file small enough for direct upload+ // Read the start of the input and check if it is small enough for direct uploadbuf := make([]byte, ci.StreamingUploadCutoff)+ fileIsSmall := falseif n, err := io.ReadFull(trackingIn, buf); err == io.EOF || err == io.ErrUnexpectedEOF {- fs.Debugf(fdst, "File to upload is small (%d bytes), uploading instead of streaming", n)- src := object.NewMemoryObject(dstFileName, modTime, buf[:n]).WithMetadata(meta).SetFs(fsrc)- return Copy(ctx, fdst, nil, dstFileName, src)+ fileIsSmall = true+ buf = buf[:n]}- // Make a new ReadCloser with the bits we've already read- in = &readCloser{- Reader: io.MultiReader(bytes.NewReader(buf), trackingIn),- Closer: in,- }+ // Read the data we have already read in buf and any further unread+ streamIn = io.MultiReader(bytes.NewReader(buf), trackingIn)- fStreamTo := fdst- canStream := fdst.Features().PutStream != nil- if !canStream {- fs.Debugf(fdst, "Target remote doesn't support streaming uploads, creating temporary local FS to spool file")- tmpLocalFs, err := fs.TemporaryLocalFs(ctx)- if err != nil {- return nil, fmt.Errorf("failed to create temporary local FS to spool file: %w", err)- }- defer func() {- err := Purge(ctx, tmpLocalFs, "")+ doPutStream := fdst.Features().PutStream++ // Upload the input+ if fileIsSmall || doPutStream == nil {+ var rs io.ReadSeeker+ if fileIsSmall {+ fs.Debugf(fdst, "File to upload is small (%d bytes), uploading instead of streaming", len(buf))+ rs = bytes.NewReader(buf)+ } else {+ fs.Debugf(fdst, "Target remote doesn't support streaming uploads, creating temporary local FS to spool file")+ spool, err := os.CreateTemp("", "rclone-spool")if err != nil {- fs.Infof(tmpLocalFs, "Failed to cleanup temporary FS: %v", err)+ return nil, fmt.Errorf("failed to create temporary spool file: %v", err)}- }()- fStreamTo = tmpLocalFs+ fileName := spool.Name()+ defer func() {+ err := spool.Close()+ if err != nil {+ fs.Errorf(fileName, "Failed to close temporary spool file: %v", err)+ }+ err = os.Remove(fileName)+ if err != nil {+ fs.Errorf(fileName, "Failed to delete temporary spool file: %v", err)+ }+ }()+ _, err = io.Copy(spool, streamIn)+ if err != nil {+ return nil, fmt.Errorf("failed to copy to temporary spool file: %v", err)+ }+ rs = spool+ }+ // Upload with Put with retries - since we have downloaded the file we know the size, and the hashes+ sums := getSums()+ size := int64(readCounter.BytesRead())+ objInfo := object.NewStaticObjectInfo(dstFileName, modTime, size, false, sums, fsrc).WithMetadata(meta)+ err = Retry(ctx, objInfo, ci.LowLevelRetries, func() error {+ _, err = rs.Seek(0, io.SeekStart)+ if err != nil {+ return fmt.Errorf("failed to rewind temporary spool file: %v", err)+ }+ dst, err = fdst.Put(ctx, rs, objInfo, options...)+ return err+ })+ } else {+ // Upload with PutStream with no retries+ objInfo := object.NewStaticObjectInfo(dstFileName, modTime, -1, false, nil, fsrc).WithMetadata(meta)+ dst, err = doPutStream(ctx, streamIn, objInfo, options...)}-- if SkipDestructive(ctx, dstFileName, "upload from pipe") {- // prevents "broken pipe" errors- _, err = io.Copy(io.Discard, in)- return nil, err+ if err != nil {+ return dst, err}- objInfo := object.NewStaticObjectInfo(dstFileName, modTime, -1, false, nil, fsrc).WithMetadata(meta)- if dst, err = fStreamTo.Features().PutStream(ctx, in, objInfo, options...); err != nil {- return dst, err+ // Check transfer+ sums := getSums()+ opt := defaultEqualOpt(ctx)+ if sums != nil {+ // force --checksum on if we have hashes+ opt.checkSum = true}- if err = compare(dst); err != nil {+ src := object.NewStaticObjectInfo(dstFileName, modTime, int64(readCounter.BytesRead()), false, sums, fdst).WithMetadata(meta)+ if !equal(ctx, src, dst, opt) {+ err = fmt.Errorf("corrupted on transfer")+ err = fs.CountError(err)+ fs.Errorf(dst, "%v", err)return dst, err}- if !canStream {- // copy dst (which is the local object we have just streamed to) to the remote- newCtx := ctx- if ci.Metadata && len(meta) != 0 {- // If we have metadata and we are setting it then use- // the --metadataset mechanism to supply it to Copy- var newCi *fs.ConfigInfo- newCtx, newCi = fs.AddConfig(ctx)- if len(newCi.MetadataSet) == 0 {- newCi.MetadataSet = meta- } else {- var newMeta fs.Metadata- newMeta.Merge(meta)- newMeta.Merge(newCi.MetadataSet) // --metadata-set takes priority- newCi.MetadataSet = newMeta- }- }- return Copy(newCtx, fdst, nil, dstFileName, dst)- }return dst, nil}diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.goindex d14678f13..3e66bb33a 100644--- a/fs/operations/operations_test.go+++ b/fs/operations/operations_test.go@@ -42,6 +42,7 @@ import ("github.com/rclone/rclone/fs/operations""github.com/rclone/rclone/fstest""github.com/rclone/rclone/fstest/fstests"+ "github.com/rclone/rclone/lib/pacer""github.com/stretchr/testify/assert""github.com/stretchr/testify/require""golang.org/x/text/cases"@@ -504,12 +505,12 @@ func TestRetry(t *testing.T) {return err}- i, err = 3, io.EOF+ i, err = 3, fmt.Errorf("Wrapped EOF is retriable: %w", io.EOF)assert.Equal(t, nil, operations.Retry(ctx, nil, 5, fn))assert.Equal(t, 0, i)- i, err = 10, io.EOF- assert.Equal(t, io.EOF, operations.Retry(ctx, nil, 5, fn))+ i, err = 10, pacer.RetryAfterError(errors.New("BANG"), 10*time.Millisecond)+ assert.Equal(t, err, operations.Retry(ctx, nil, 5, fn))assert.Equal(t, 5, i)i, err = 10, fs.ErrorObjectNotFound