"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
operations: add RemoveExisting to safely remove an existing file
Nick Craig-Wood2bafbf3c0410
diff --git a/fs/operations/operations.go b/fs/operations/operations.goindex 53d846525..eeba711fb 100644--- a/fs/operations/operations.go+++ b/fs/operations/operations.go@@ -1188,6 +1188,57 @@ func Delete(ctx context.Context, f fs.Fs) error {return err}+// RemoveExisting removes an existing file in a safe way so that it+// can be restored if the operation fails.+//+// This first detects if there is an existing file and renames it to a+// temporary name if there is.+//+// The returned cleanup function should be called on a defer statement+// with a pointer to the error returned. It will revert the changes if+// there is an error or delete the existing file if not.+func RemoveExisting(ctx context.Context, f fs.Fs, remote string, operation string) (cleanup func(*error), err error) {+ existingObj, err := f.NewObject(ctx, remote)+ if err != nil {+ return func(*error) {}, nil+ }+ doMove := f.Features().Move+ if doMove == nil {+ return nil, fmt.Errorf("%s: destination file exists already and can't rename", operation)+ }++ // Avoid making the leaf name longer if it's already lengthy to avoid+ // trouble with file name length limits.+ suffix := "." + random.String(8)+ var remoteSaved string+ if len(path.Base(remote)) > 100 {+ remoteSaved = TruncateString(remote, len(remote)-len(suffix)) + suffix+ } else {+ remoteSaved = remote + suffix+ }++ fs.Debugf(existingObj, "%s: renaming existing object to %q before starting", operation, remoteSaved)+ existingObj, err = doMove(ctx, existingObj, remoteSaved)+ if err != nil {+ return nil, fmt.Errorf("%s: failed to rename existing file: %w", operation, err)+ }+ return func(perr *error) {+ if *perr == nil {+ fs.Debugf(existingObj, "%s: removing renamed existing file after operation", operation)+ err := existingObj.Remove(ctx)+ if err != nil {+ *perr = fmt.Errorf("%s: failed to remove renamed existing file: %w", operation, err)+ }+ } else {+ fs.Debugf(existingObj, "%s: renaming existing back after failed operation", operation)+ _, renameErr := doMove(ctx, existingObj, remote)+ if renameErr != nil {+ fs.Errorf(existingObj, "%s: failed to restore existing file after failed operation: %v", operation, renameErr)+ }+ }+ }, nil+}+// listToChan will transfer all objects in the listing to the output//// If an error occurs, the error will be logged, and it will close thediff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.goindex 96ce5704a..2a5c3cb6c 100644--- a/fs/operations/operations_test.go+++ b/fs/operations/operations_test.go@@ -1884,3 +1884,76 @@ func TestDirsEqual(t *testing.T) {equal = operations.DirsEqual(ctx, src, dst, opt)assert.True(t, equal)}++func TestRemoveExisting(t *testing.T) {+ ctx := context.Background()+ r := fstest.NewRun(t)+ if r.Fremote.Features().Move == nil {+ t.Skip("Skipping as remote can't Move")+ }++ file1 := r.WriteObject(ctx, "sub dir/test remove existing", "hello world", t1)+ file2 := r.WriteObject(ctx, "sub dir/test remove existing with long name 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890", "hello long name world", t1)++ r.CheckRemoteItems(t, file1, file2)++ var returnedError error++ // Check not found first+ cleanup, err := operations.RemoveExisting(ctx, r.Fremote, "not found", "TEST")+ assert.Equal(t, err, nil)+ r.CheckRemoteItems(t, file1, file2)+ cleanup(&returnedError)+ r.CheckRemoteItems(t, file1, file2)++ // Remove file1+ cleanup, err = operations.RemoveExisting(ctx, r.Fremote, file1.Path, "TEST")+ assert.Equal(t, err, nil)+ //r.CheckRemoteItems(t, file1, file2)++ // Check file1 with temporary name exists+ var buf bytes.Buffer+ err = operations.List(ctx, r.Fremote, &buf)+ require.NoError(t, err)+ res := buf.String()+ assert.NotContains(t, res, " 11 "+file1.Path+"\n")+ assert.Contains(t, res, " 11 "+file1.Path+".")+ assert.Contains(t, res, " 21 "+file2.Path+"\n")++ cleanup(&returnedError)+ r.CheckRemoteItems(t, file2)++ // Remove file2 with an error+ cleanup, err = operations.RemoveExisting(ctx, r.Fremote, file2.Path, "TEST")+ assert.Equal(t, err, nil)++ // Check file2 with truncated temporary name exists+ buf.Reset()+ err = operations.List(ctx, r.Fremote, &buf)+ require.NoError(t, err)+ res = buf.String()+ assert.NotContains(t, res, " 21 "+file2.Path+"\n")+ assert.NotContains(t, res, " 21 "+file2.Path+".")+ assert.Contains(t, res, " 21 "+file2.Path[:100])++ returnedError = errors.New("BOOM")+ cleanup(&returnedError)+ r.CheckRemoteItems(t, file2)++ // Remove file2+ cleanup, err = operations.RemoveExisting(ctx, r.Fremote, file2.Path, "TEST")+ assert.Equal(t, err, nil)++ // Check file2 with truncated temporary name exists+ buf.Reset()+ err = operations.List(ctx, r.Fremote, &buf)+ require.NoError(t, err)+ res = buf.String()+ assert.NotContains(t, res, " 21 "+file2.Path+"\n")+ assert.NotContains(t, res, " 21 "+file2.Path+".")+ assert.Contains(t, res, " 21 "+file2.Path[:100])++ returnedError = nil+ cleanup(&returnedError)+ r.CheckRemoteItems(t)+}