"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
bisync: fix rc ignoring global --dry-run and --max-delete settings
nielash914fa87933ce
diff --git a/cmd/bisync/bisync_test.go b/cmd/bisync/bisync_test.goindex 632b28df8..4a0a3e657 100644--- a/cmd/bisync/bisync_test.go+++ b/cmd/bisync/bisync_test.go@@ -1141,7 +1141,6 @@ func (b *bisyncTest) runBisync(ctx context.Context, args []string) (err error) {Workdir: b.workDir,NoCleanup: true,SaveQueues: true,- MaxDelete: bisync.DefaultMaxDelete,CheckFilename: bisync.DefaultCheckFilename,CheckSync: bisync.CheckSyncTrue,}@@ -1187,7 +1186,7 @@ func (b *bisyncTest) runBisync(ctx context.Context, args []string) (err error) {case "filters-file":opt.FiltersFile = valcase "max-delete":- opt.MaxDelete, err = strconv.Atoi(val)+ ci.MaxDelete, err = strconv.ParseInt(val, 10, 64)require.NoError(b.t, err, "parsing max-delete=%q", val)case "max-delete-renames-aware":opt.MaxDeleteRenamesAware = truediff --git a/cmd/bisync/cmd.go b/cmd/bisync/cmd.goindex 1e2c3b5b6..b39c2f454 100644--- a/cmd/bisync/cmd.go+++ b/cmd/bisync/cmd.go@@ -38,7 +38,7 @@ type Options struct {CheckSync CheckSyncModeCreateEmptySrcDirs boolRemoveEmptyDirs bool- MaxDelete int // percentage from 0 to 100+ MaxDelete int // percentage from 0 to 100, set by Bisync from --max-deleteMaxDeleteRenamesAware boolForce boolFiltersFile string@@ -46,7 +46,7 @@ type Options struct {OrigBackupDir stringBackupDir1 stringBackupDir2 string- DryRun bool+ DryRun bool // set by Bisync from --dry-runNoCleanup boolSaveQueues bool // save extra debugging files (test only flag)IgnoreListingChecksum bool@@ -179,7 +179,6 @@ var commandDefinition = &cobra.Command{ctx := context.Background()opt := Opt- opt.applyContext(ctx)if tzLocal {TZ = time.Local}diff --git a/cmd/bisync/help.go b/cmd/bisync/help.goindex 84cd55967..4e542775d 100644--- a/cmd/bisync/help.go+++ b/cmd/bisync/help.go@@ -41,6 +41,7 @@ func RcHelp() string {- path1 (required) - (string) a remote directory string e.g. ||drive:path1||- path2 (required) - (string) a remote directory string e.g. ||drive:path2||- dryRun - (bool) dry-run mode+- maxDelete - (int) Safety check on maximum percentage of deleted files allowed. If exceeded, the bisync run will abort. (default: {MAXDELETE}%)`+GenerateParams()+`See [bisync command help](https://rclone.org/commands/rclone_bisync/)and [full bisync description](https://rclone.org/bisync/)diff --git a/cmd/bisync/operations.go b/cmd/bisync/operations.goindex 05df06e1b..5d6379d3e 100644--- a/cmd/bisync/operations.go+++ b/cmd/bisync/operations.go@@ -72,6 +72,9 @@ type queues struct {// Bisync handles lock file, performs bisync run and checks exit statusfunc Bisync(ctx context.Context, fs1, fs2 fs.Fs, optArg *Options) (err error) {opt := *optArg // ensure that input is never changed+ // applyContext changes the config, so give it a copy+ ctx, _ = fs.AddConfig(ctx)+ opt.applyContext(ctx)b := &bisyncRun{fs1: fs1,fs2: fs2,diff --git a/cmd/bisync/rc.go b/cmd/bisync/rc.goindex fd1a67e1c..8de454618 100644--- a/cmd/bisync/rc.go+++ b/cmd/bisync/rc.go@@ -65,9 +65,9 @@ func rcBisync(ctx context.Context, in rc.Params) (out rc.Params, err error) {opt := &Options{}octx, ci := fs.AddConfig(ctx)- if dryRun, err := in.GetBool("dryRun"); err == nil {- ci.DryRun = dryRun- opt.DryRun = dryRun+ if dryRun, err := in.GetBool("dryRun"); err == nil && dryRun {+ // dryRun can ask for a dry run, but not cancel one from the global config+ ci.DryRun = true} else if rc.NotErrParamNotFound(err) {return nil, err}@@ -76,7 +76,7 @@ func rcBisync(ctx context.Context, in rc.Params) (out rc.Params, err error) {if maxDelete < 0 || maxDelete > 100 {return nil, rc.NewErrParamInvalid(errors.New("maxDelete must be a percentage between 0 and 100"))}- opt.MaxDelete = int(maxDelete)+ ci.MaxDelete = maxDelete} else if rc.NotErrParamNotFound(err) {return nil, err}diff --git a/cmd/bisync/rc_test.go b/cmd/bisync/rc_test.gonew file mode 100644index 000000000..495b6605b--- /dev/null+++ b/cmd/bisync/rc_test.go@@ -0,0 +1,114 @@+package bisync_test++import (+ "context"+ "fmt"+ "os"+ "path/filepath"+ "testing"++ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/rc"+ "github.com/rclone/rclone/fs/rc/jobs"+ "github.com/rclone/rclone/fstest"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++// newRcBisync is newRcBisyncCtx with the rc jobs run in a plain context+func newRcBisync(t *testing.T, n int) (path1, path2 string, run func(in rc.Params) error) {+ return newRcBisyncCtx(context.Background(), t, n)+}++// newRcBisyncCtx creates Path1 with n files, resyncs it to Path2, and returns+// the paths and a function which runs sync/bisync on them as an rc job. The+// jobs run in ctx, whose config stands in for the flags the rcd was started+// with.+func newRcBisyncCtx(ctx context.Context, t *testing.T, n int) (path1, path2 string, run func(in rc.Params) error) {+ if *fstest.RemoteName != "" {+ t.Skip("Skipping test on non local remote")+ }+ dir := t.TempDir()+ path1 = filepath.Join(dir, "path1")+ path2 = filepath.Join(dir, "path2")+ workdir := filepath.Join(dir, "workdir")+ require.NoError(t, os.Mkdir(path1, 0o755))+ require.NoError(t, os.Mkdir(path2, 0o755))+ for i := range n {+ require.NoError(t, os.WriteFile(filepath.Join(path1, fmt.Sprintf("file%d.txt", i)), []byte("content"), 0o644))+ }+ call := rc.Calls.Get("sync/bisync")+ require.NotNil(t, call)+ run = func(in rc.Params) error {+ in["path1"] = path1+ in["path2"] = path2+ in["workdir"] = workdir+ _, _, err := jobs.NewJob(ctx, call.Fn, in)+ return err+ }+ require.NoError(t, run(rc.Params{"resync": true}))+ return path1, path2, run+}++func TestRcDryRun(t *testing.T) {+ for _, test := range []struct {+ name string+ in rc.Params+ dryRun bool+ }{+ {"dryRun", rc.Params{"dryRun": true}, true},+ {"dry_run", rc.Params{"dry_run": true}, true},+ {"_config", rc.Params{"_config": rc.Params{"DryRun": true}}, true},+ {"dry_run and dryRun false", rc.Params{"dry_run": true, "dryRun": false}, true},+ {"_config and dryRun false", rc.Params{"_config": rc.Params{"DryRun": true}, "dryRun": false}, true},+ {"dryRun false", rc.Params{"dryRun": false}, false},+ } {+ t.Run(test.name, func(t *testing.T) {+ path1, path2, run := newRcBisync(t, 1)+ require.NoError(t, os.WriteFile(filepath.Join(path1, "new.txt"), []byte("new"), 0o644))+ require.NoError(t, run(test.in))+ if test.dryRun {+ assert.NoFileExists(t, filepath.Join(path2, "new.txt"))+ } else {+ assert.FileExists(t, filepath.Join(path2, "new.txt"))+ }+ })+ }+}++func TestRcMaxDelete(t *testing.T) {+ for _, test := range []struct {+ name string+ in rc.Params+ wantErr bool+ }{+ {"default", rc.Params{}, false},+ {"maxDelete", rc.Params{"maxDelete": 5}, true},+ {"max_delete allows", rc.Params{"max_delete": 20}, false},+ {"max_delete aborts", rc.Params{"max_delete": 5}, true},+ {"_config MaxDelete aborts", rc.Params{"_config": rc.Params{"MaxDelete": 5}}, true},+ } {+ t.Run(test.name, func(t *testing.T) {+ path1, path2, run := newRcBisync(t, 10)+ require.NoError(t, os.Remove(filepath.Join(path1, "file0.txt")))+ err := run(test.in)+ if test.wantErr {+ assert.ErrorContains(t, err, "too many deletes")+ assert.FileExists(t, filepath.Join(path2, "file0.txt"))+ } else {+ require.NoError(t, err)+ assert.NoFileExists(t, filepath.Join(path2, "file0.txt"))+ }+ })+ }+}++// Test a --max-delete the rcd was started with applies, as on the command line+func TestRcMaxDeleteFromRcd(t *testing.T) {+ ctx, ci := fs.AddConfig(context.Background())+ ci.MaxDelete = 5+ path1, path2, run := newRcBisyncCtx(ctx, t, 10)+ require.NoError(t, os.Remove(filepath.Join(path1, "file0.txt")))+ assert.ErrorContains(t, run(rc.Params{}), "too many deletes")+ assert.FileExists(t, filepath.Join(path2, "file0.txt"))+}diff --git a/docs/content/bisync.md b/docs/content/bisync.mdindex 330ff0c20..09aca5c40 100644--- a/docs/content/bisync.md+++ b/docs/content/bisync.md@@ -1947,6 +1947,10 @@ it, as with `--download`, instead of by size alone.- Fixed an issue causing a `--resync` to fail with "path1 and path2 are out ofsync", or later runs to report changes that didn't happen, when a file's newmodtime was less than a second earlier than the one in the listing.+- Fixed an issue causing the rc `sync/bisync` call to ignore a dry run set with+`_config` or the flat `dry_run` option.+- Fixed an issue causing the rc `sync/bisync` call to ignore the global+`--max-delete` and use a limit of 0% when `maxDelete` was not set.### `v1.74.2`