"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
b2: add SkipDestructive handling to backend commands - fixes #8194
Pat Pattersonbf5a4774c69f
diff --git a/backend/b2/b2.go b/backend/b2/b2.goindex 4d4e75424..ad58383f9 100644--- a/backend/b2/b2.go+++ b/backend/b2/b2.go@@ -30,6 +30,7 @@ import ("github.com/rclone/rclone/fs/fserrors""github.com/rclone/rclone/fs/fshttp""github.com/rclone/rclone/fs/hash"+ "github.com/rclone/rclone/fs/operations""github.com/rclone/rclone/fs/walk""github.com/rclone/rclone/lib/bucket""github.com/rclone/rclone/lib/encoder"@@ -1318,16 +1319,22 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool, deleteHidden b// Check current version of the fileif deleteHidden && object.Action == "hide" {fs.Debugf(remote, "Deleting current version (id %q) as it is a hide marker", object.ID)- toBeDeleted <- object+ if !operations.SkipDestructive(ctx, object.Name, "remove hide marker") {+ toBeDeleted <- object+ }} else if deleteUnfinished && object.Action == "start" && isUnfinishedUploadStale(object.UploadTimestamp) {fs.Debugf(remote, "Deleting current version (id %q) as it is a start marker (upload started at %s)", object.ID, time.Time(object.UploadTimestamp).Local())- toBeDeleted <- object+ if !operations.SkipDestructive(ctx, object.Name, "remove pending upload") {+ toBeDeleted <- object+ }} else {fs.Debugf(remote, "Not deleting current version (id %q) %q dated %v (%v ago)", object.ID, object.Action, time.Time(object.UploadTimestamp).Local(), time.Since(time.Time(object.UploadTimestamp)))}} else {fs.Debugf(remote, "Deleting (id %q)", object.ID)- toBeDeleted <- object+ if !operations.SkipDestructive(ctx, object.Name, "delete") {+ toBeDeleted <- object+ }}last = remotetr.Done(ctx, nil)@@ -2293,8 +2300,10 @@ func (f *Fs) lifecycleCommand(ctx context.Context, name string, arg []string, op}+ skip := operations.SkipDestructive(ctx, name, "update lifecycle rules")+var bucket *api.Bucket- if newRule.DaysFromHidingToDeleting != nil || newRule.DaysFromUploadingToHiding != nil || newRule.DaysFromStartingToCancelingUnfinishedLargeFiles != nil {+ if !skip && (newRule.DaysFromHidingToDeleting != nil || newRule.DaysFromUploadingToHiding != nil || newRule.DaysFromStartingToCancelingUnfinishedLargeFiles != nil) {bucketID, err := f.getBucketID(ctx, bucketName)if err != nil {return nil, errdiff --git a/backend/b2/b2_internal_test.go b/backend/b2/b2_internal_test.goindex e6bd8b404..c470dcc4d 100644--- a/backend/b2/b2_internal_test.go+++ b/backend/b2/b2_internal_test.go@@ -5,6 +5,7 @@ import ("crypto/sha1""fmt""path"+ "sort""strings""testing""time"@@ -13,6 +14,7 @@ import ("github.com/rclone/rclone/fs""github.com/rclone/rclone/fs/cache""github.com/rclone/rclone/fs/hash"+ "github.com/rclone/rclone/fs/object""github.com/rclone/rclone/fstest""github.com/rclone/rclone/fstest/fstests""github.com/rclone/rclone/lib/bucket"@@ -463,24 +465,161 @@ func (f *Fs) InternalTestVersions(t *testing.T) {})t.Run("Cleanup", func(t *testing.T) {- require.NoError(t, f.cleanUp(ctx, true, false, 0))- items := append([]fstest.Item{newItem}, fstests.InternalTestFiles...)- fstest.CheckListing(t, f, items)- // Set --b2-versions for this test- f.opt.Versions = true- defer func() {- f.opt.Versions = false- }()- fstest.CheckListing(t, f, items)+ t.Run("DryRun", func(t *testing.T) {+ f.opt.Versions = true+ defer func() {+ f.opt.Versions = false+ }()+ // Listing should be unchanged after dry run+ before := listAllFiles(ctx, t, f, dirName)+ ctx, ci := fs.AddConfig(ctx)+ ci.DryRun = true+ require.NoError(t, f.cleanUp(ctx, true, false, 0))+ after := listAllFiles(ctx, t, f, dirName)+ assert.Equal(t, before, after)+ })++ t.Run("RealThing", func(t *testing.T) {+ f.opt.Versions = true+ defer func() {+ f.opt.Versions = false+ }()+ // Listing should reflect current state after cleanup+ require.NoError(t, f.cleanUp(ctx, true, false, 0))+ items := append([]fstest.Item{newItem}, fstests.InternalTestFiles...)+ fstest.CheckListing(t, f, items)+ })})// Purge gets tested later}+func (f *Fs) InternalTestCleanupUnfinished(t *testing.T) {+ ctx := context.Background()++ // B2CleanupHidden tests cleaning up hidden files+ t.Run("CleanupUnfinished", func(t *testing.T) {+ dirName := "unfinished"+ fileCount := 5+ expectedFiles := []string{}+ for i := 1; i < fileCount; i++ {+ fileName := fmt.Sprintf("%s/unfinished-%d", dirName, i)+ expectedFiles = append(expectedFiles, fileName)+ obj := &Object{+ fs: f,+ remote: fileName,+ }+ objInfo := object.NewStaticObjectInfo(fileName, fstest.Time("2002-02-03T04:05:06.499999999Z"), -1, true, nil, nil)+ _, err := f.newLargeUpload(ctx, obj, nil, objInfo, f.opt.ChunkSize, false, nil)+ require.NoError(t, err)+ }+ checkListing(ctx, t, f, dirName, expectedFiles)++ t.Run("DryRun", func(t *testing.T) {+ // Listing should not change after dry run+ ctx, ci := fs.AddConfig(ctx)+ ci.DryRun = true+ require.NoError(t, f.cleanUp(ctx, false, true, 0))+ checkListing(ctx, t, f, dirName, expectedFiles)+ })++ t.Run("RealThing", func(t *testing.T) {+ // Listing should be empty after real cleanup+ require.NoError(t, f.cleanUp(ctx, false, true, 0))+ checkListing(ctx, t, f, dirName, []string{})+ })+ })+}++func listAllFiles(ctx context.Context, t *testing.T, f *Fs, dirName string) []string {+ bucket, directory := f.split(dirName)+ foundFiles := []string{}+ require.NoError(t, f.list(ctx, bucket, directory, "", false, true, 0, true, false, func(remote string, object *api.File, isDirectory bool) error {+ if !isDirectory {+ foundFiles = append(foundFiles, object.Name)+ }+ return nil+ }))+ sort.Strings(foundFiles)+ return foundFiles+}++func checkListing(ctx context.Context, t *testing.T, f *Fs, dirName string, expectedFiles []string) {+ foundFiles := listAllFiles(ctx, t, f, dirName)+ sort.Strings(expectedFiles)+ assert.Equal(t, expectedFiles, foundFiles)+}++func (f *Fs) InternalTestLifecycleRules(t *testing.T) {+ ctx := context.Background()++ opt := map[string]string{}++ t.Run("InitState", func(t *testing.T) {+ // There should be no lifecycle rules at the outset+ lifecycleRulesIf, err := f.lifecycleCommand(ctx, "lifecycle", nil, opt)+ lifecycleRules := lifecycleRulesIf.([]api.LifecycleRule)+ require.NoError(t, err)+ assert.Equal(t, 0, len(lifecycleRules))+ })++ t.Run("DryRun", func(t *testing.T) {+ // There should still be no lifecycle rules after each dry run operation+ ctx, ci := fs.AddConfig(ctx)+ ci.DryRun = true++ opt["daysFromHidingToDeleting"] = "30"+ lifecycleRulesIf, err := f.lifecycleCommand(ctx, "lifecycle", nil, opt)+ lifecycleRules := lifecycleRulesIf.([]api.LifecycleRule)+ require.NoError(t, err)+ assert.Equal(t, 0, len(lifecycleRules))++ delete(opt, "daysFromHidingToDeleting")+ opt["daysFromUploadingToHiding"] = "40"+ lifecycleRulesIf, err = f.lifecycleCommand(ctx, "lifecycle", nil, opt)+ lifecycleRules = lifecycleRulesIf.([]api.LifecycleRule)+ require.NoError(t, err)+ assert.Equal(t, 0, len(lifecycleRules))++ opt["daysFromHidingToDeleting"] = "30"+ lifecycleRulesIf, err = f.lifecycleCommand(ctx, "lifecycle", nil, opt)+ lifecycleRules = lifecycleRulesIf.([]api.LifecycleRule)+ require.NoError(t, err)+ assert.Equal(t, 0, len(lifecycleRules))+ })++ t.Run("RealThing", func(t *testing.T) {+ opt["daysFromHidingToDeleting"] = "30"+ lifecycleRulesIf, err := f.lifecycleCommand(ctx, "lifecycle", nil, opt)+ lifecycleRules := lifecycleRulesIf.([]api.LifecycleRule)+ require.NoError(t, err)+ assert.Equal(t, 1, len(lifecycleRules))+ assert.Equal(t, 30, *lifecycleRules[0].DaysFromHidingToDeleting)++ delete(opt, "daysFromHidingToDeleting")+ opt["daysFromUploadingToHiding"] = "40"+ lifecycleRulesIf, err = f.lifecycleCommand(ctx, "lifecycle", nil, opt)+ lifecycleRules = lifecycleRulesIf.([]api.LifecycleRule)+ require.NoError(t, err)+ assert.Equal(t, 1, len(lifecycleRules))+ assert.Equal(t, 40, *lifecycleRules[0].DaysFromUploadingToHiding)++ opt["daysFromHidingToDeleting"] = "30"+ lifecycleRulesIf, err = f.lifecycleCommand(ctx, "lifecycle", nil, opt)+ lifecycleRules = lifecycleRulesIf.([]api.LifecycleRule)+ require.NoError(t, err)+ assert.Equal(t, 1, len(lifecycleRules))+ assert.Equal(t, 30, *lifecycleRules[0].DaysFromHidingToDeleting)+ assert.Equal(t, 40, *lifecycleRules[0].DaysFromUploadingToHiding)+ })+}+// -run TestIntegration/FsMkdir/FsPutFiles/Internalfunc (f *Fs) InternalTest(t *testing.T) {t.Run("Metadata", f.InternalTestMetadata)t.Run("Versions", f.InternalTestVersions)+ t.Run("CleanupUnfinished", f.InternalTestCleanupUnfinished)+ t.Run("LifecycleRules", f.InternalTestLifecycleRules)}var _ fstests.InternalTester = (*Fs)(nil)