Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

operations: add RemoveExisting to safely remove an existing file

Nick Craig-Wood2bafbf3c0410
diff --git a/fs/operations/operations.go b/fs/operations/operations.go
index 53d846525..eeba711fb 100644
--- a/fs/operations/operations.go
+++ b/fs/operations/operations.go
@@ -1188,6 +1188,57 @@ func Delete(ctx context.Context, f fs.Fs) error {
return err
}
+// RemoveExisting removes an existing file in a safe way so that it
+// can be restored if the operation fails.
+//
+// This first detects if there is an existing file and renames it to a
+// temporary name if there is.
+//
+// The returned cleanup function should be called on a defer statement
+// with a pointer to the error returned. It will revert the changes if
+// there is an error or delete the existing file if not.
+func RemoveExisting(ctx context.Context, f fs.Fs, remote string, operation string) (cleanup func(*error), err error) {
+ existingObj, err := f.NewObject(ctx, remote)
+ if err != nil {
+ return func(*error) {}, nil
+ }
+ doMove := f.Features().Move
+ if doMove == nil {
+ return nil, fmt.Errorf("%s: destination file exists already and can't rename", operation)
+ }
+
+ // Avoid making the leaf name longer if it's already lengthy to avoid
+ // trouble with file name length limits.
+ suffix := "." + random.String(8)
+ var remoteSaved string
+ if len(path.Base(remote)) > 100 {
+ remoteSaved = TruncateString(remote, len(remote)-len(suffix)) + suffix
+ } else {
+ remoteSaved = remote + suffix
+ }
+
+ fs.Debugf(existingObj, "%s: renaming existing object to %q before starting", operation, remoteSaved)
+ existingObj, err = doMove(ctx, existingObj, remoteSaved)
+ if err != nil {
+ return nil, fmt.Errorf("%s: failed to rename existing file: %w", operation, err)
+ }
+ return func(perr *error) {
+ if *perr == nil {
+ fs.Debugf(existingObj, "%s: removing renamed existing file after operation", operation)
+ err := existingObj.Remove(ctx)
+ if err != nil {
+ *perr = fmt.Errorf("%s: failed to remove renamed existing file: %w", operation, err)
+ }
+ } else {
+ fs.Debugf(existingObj, "%s: renaming existing back after failed operation", operation)
+ _, renameErr := doMove(ctx, existingObj, remote)
+ if renameErr != nil {
+ fs.Errorf(existingObj, "%s: failed to restore existing file after failed operation: %v", operation, renameErr)
+ }
+ }
+ }, nil
+}
+
// listToChan will transfer all objects in the listing to the output
//
// If an error occurs, the error will be logged, and it will close the
diff --git a/fs/operations/operations_test.go b/fs/operations/operations_test.go
index 96ce5704a..2a5c3cb6c 100644
--- a/fs/operations/operations_test.go
+++ b/fs/operations/operations_test.go
@@ -1884,3 +1884,76 @@ func TestDirsEqual(t *testing.T) {
equal = operations.DirsEqual(ctx, src, dst, opt)
assert.True(t, equal)
}
+
+func TestRemoveExisting(t *testing.T) {
+ ctx := context.Background()
+ r := fstest.NewRun(t)
+ if r.Fremote.Features().Move == nil {
+ t.Skip("Skipping as remote can't Move")
+ }
+
+ file1 := r.WriteObject(ctx, "sub dir/test remove existing", "hello world", t1)
+ file2 := r.WriteObject(ctx, "sub dir/test remove existing with long name 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890", "hello long name world", t1)
+
+ r.CheckRemoteItems(t, file1, file2)
+
+ var returnedError error
+
+ // Check not found first
+ cleanup, err := operations.RemoveExisting(ctx, r.Fremote, "not found", "TEST")
+ assert.Equal(t, err, nil)
+ r.CheckRemoteItems(t, file1, file2)
+ cleanup(&returnedError)
+ r.CheckRemoteItems(t, file1, file2)
+
+ // Remove file1
+ cleanup, err = operations.RemoveExisting(ctx, r.Fremote, file1.Path, "TEST")
+ assert.Equal(t, err, nil)
+ //r.CheckRemoteItems(t, file1, file2)
+
+ // Check file1 with temporary name exists
+ var buf bytes.Buffer
+ err = operations.List(ctx, r.Fremote, &buf)
+ require.NoError(t, err)
+ res := buf.String()
+ assert.NotContains(t, res, " 11 "+file1.Path+"\n")
+ assert.Contains(t, res, " 11 "+file1.Path+".")
+ assert.Contains(t, res, " 21 "+file2.Path+"\n")
+
+ cleanup(&returnedError)
+ r.CheckRemoteItems(t, file2)
+
+ // Remove file2 with an error
+ cleanup, err = operations.RemoveExisting(ctx, r.Fremote, file2.Path, "TEST")
+ assert.Equal(t, err, nil)
+
+ // Check file2 with truncated temporary name exists
+ buf.Reset()
+ err = operations.List(ctx, r.Fremote, &buf)
+ require.NoError(t, err)
+ res = buf.String()
+ assert.NotContains(t, res, " 21 "+file2.Path+"\n")
+ assert.NotContains(t, res, " 21 "+file2.Path+".")
+ assert.Contains(t, res, " 21 "+file2.Path[:100])
+
+ returnedError = errors.New("BOOM")
+ cleanup(&returnedError)
+ r.CheckRemoteItems(t, file2)
+
+ // Remove file2
+ cleanup, err = operations.RemoveExisting(ctx, r.Fremote, file2.Path, "TEST")
+ assert.Equal(t, err, nil)
+
+ // Check file2 with truncated temporary name exists
+ buf.Reset()
+ err = operations.List(ctx, r.Fremote, &buf)
+ require.NoError(t, err)
+ res = buf.String()
+ assert.NotContains(t, res, " 21 "+file2.Path+"\n")
+ assert.NotContains(t, res, " 21 "+file2.Path+".")
+ assert.Contains(t, res, " 21 "+file2.Path[:100])
+
+ returnedError = nil
+ cleanup(&returnedError)
+ r.CheckRemoteItems(t)
+}