Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

bisync: fix rc ignoring global --dry-run and --max-delete settings

nielash914fa87933ce
diff --git a/cmd/bisync/bisync_test.go b/cmd/bisync/bisync_test.go
index 632b28df8..4a0a3e657 100644
--- a/cmd/bisync/bisync_test.go
+++ b/cmd/bisync/bisync_test.go
@@ -1141,7 +1141,6 @@ func (b *bisyncTest) runBisync(ctx context.Context, args []string) (err error) {
Workdir: b.workDir,
NoCleanup: true,
SaveQueues: true,
- MaxDelete: bisync.DefaultMaxDelete,
CheckFilename: bisync.DefaultCheckFilename,
CheckSync: bisync.CheckSyncTrue,
}
@@ -1187,7 +1186,7 @@ func (b *bisyncTest) runBisync(ctx context.Context, args []string) (err error) {
case "filters-file":
opt.FiltersFile = val
case "max-delete":
- opt.MaxDelete, err = strconv.Atoi(val)
+ ci.MaxDelete, err = strconv.ParseInt(val, 10, 64)
require.NoError(b.t, err, "parsing max-delete=%q", val)
case "max-delete-renames-aware":
opt.MaxDeleteRenamesAware = true
diff --git a/cmd/bisync/cmd.go b/cmd/bisync/cmd.go
index 1e2c3b5b6..b39c2f454 100644
--- a/cmd/bisync/cmd.go
+++ b/cmd/bisync/cmd.go
@@ -38,7 +38,7 @@ type Options struct {
CheckSync CheckSyncMode
CreateEmptySrcDirs bool
RemoveEmptyDirs bool
- MaxDelete int // percentage from 0 to 100
+ MaxDelete int // percentage from 0 to 100, set by Bisync from --max-delete
MaxDeleteRenamesAware bool
Force bool
FiltersFile string
@@ -46,7 +46,7 @@ type Options struct {
OrigBackupDir string
BackupDir1 string
BackupDir2 string
- DryRun bool
+ DryRun bool // set by Bisync from --dry-run
NoCleanup bool
SaveQueues bool // save extra debugging files (test only flag)
IgnoreListingChecksum bool
@@ -179,7 +179,6 @@ var commandDefinition = &cobra.Command{
ctx := context.Background()
opt := Opt
- opt.applyContext(ctx)
if tzLocal {
TZ = time.Local
}
diff --git a/cmd/bisync/help.go b/cmd/bisync/help.go
index 84cd55967..4e542775d 100644
--- a/cmd/bisync/help.go
+++ b/cmd/bisync/help.go
@@ -41,6 +41,7 @@ func RcHelp() string {
- path1 (required) - (string) a remote directory string e.g. ||drive:path1||
- path2 (required) - (string) a remote directory string e.g. ||drive:path2||
- dryRun - (bool) dry-run mode
+- maxDelete - (int) Safety check on maximum percentage of deleted files allowed. If exceeded, the bisync run will abort. (default: {MAXDELETE}%)
`+GenerateParams()+`
See [bisync command help](https://rclone.org/commands/rclone_bisync/)
and [full bisync description](https://rclone.org/bisync/)
diff --git a/cmd/bisync/operations.go b/cmd/bisync/operations.go
index 05df06e1b..5d6379d3e 100644
--- a/cmd/bisync/operations.go
+++ b/cmd/bisync/operations.go
@@ -72,6 +72,9 @@ type queues struct {
// Bisync handles lock file, performs bisync run and checks exit status
func Bisync(ctx context.Context, fs1, fs2 fs.Fs, optArg *Options) (err error) {
opt := *optArg // ensure that input is never changed
+ // applyContext changes the config, so give it a copy
+ ctx, _ = fs.AddConfig(ctx)
+ opt.applyContext(ctx)
b := &bisyncRun{
fs1: fs1,
fs2: fs2,
diff --git a/cmd/bisync/rc.go b/cmd/bisync/rc.go
index fd1a67e1c..8de454618 100644
--- a/cmd/bisync/rc.go
+++ b/cmd/bisync/rc.go
@@ -65,9 +65,9 @@ func rcBisync(ctx context.Context, in rc.Params) (out rc.Params, err error) {
opt := &Options{}
octx, ci := fs.AddConfig(ctx)
- if dryRun, err := in.GetBool("dryRun"); err == nil {
- ci.DryRun = dryRun
- opt.DryRun = dryRun
+ if dryRun, err := in.GetBool("dryRun"); err == nil && dryRun {
+ // dryRun can ask for a dry run, but not cancel one from the global config
+ ci.DryRun = true
} else if rc.NotErrParamNotFound(err) {
return nil, err
}
@@ -76,7 +76,7 @@ func rcBisync(ctx context.Context, in rc.Params) (out rc.Params, err error) {
if maxDelete < 0 || maxDelete > 100 {
return nil, rc.NewErrParamInvalid(errors.New("maxDelete must be a percentage between 0 and 100"))
}
- opt.MaxDelete = int(maxDelete)
+ ci.MaxDelete = maxDelete
} else if rc.NotErrParamNotFound(err) {
return nil, err
}
diff --git a/cmd/bisync/rc_test.go b/cmd/bisync/rc_test.go
new file mode 100644
index 000000000..495b6605b
--- /dev/null
+++ b/cmd/bisync/rc_test.go
@@ -0,0 +1,114 @@
+package bisync_test
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/rc"
+ "github.com/rclone/rclone/fs/rc/jobs"
+ "github.com/rclone/rclone/fstest"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// newRcBisync is newRcBisyncCtx with the rc jobs run in a plain context
+func newRcBisync(t *testing.T, n int) (path1, path2 string, run func(in rc.Params) error) {
+ return newRcBisyncCtx(context.Background(), t, n)
+}
+
+// newRcBisyncCtx creates Path1 with n files, resyncs it to Path2, and returns
+// the paths and a function which runs sync/bisync on them as an rc job. The
+// jobs run in ctx, whose config stands in for the flags the rcd was started
+// with.
+func newRcBisyncCtx(ctx context.Context, t *testing.T, n int) (path1, path2 string, run func(in rc.Params) error) {
+ if *fstest.RemoteName != "" {
+ t.Skip("Skipping test on non local remote")
+ }
+ dir := t.TempDir()
+ path1 = filepath.Join(dir, "path1")
+ path2 = filepath.Join(dir, "path2")
+ workdir := filepath.Join(dir, "workdir")
+ require.NoError(t, os.Mkdir(path1, 0o755))
+ require.NoError(t, os.Mkdir(path2, 0o755))
+ for i := range n {
+ require.NoError(t, os.WriteFile(filepath.Join(path1, fmt.Sprintf("file%d.txt", i)), []byte("content"), 0o644))
+ }
+ call := rc.Calls.Get("sync/bisync")
+ require.NotNil(t, call)
+ run = func(in rc.Params) error {
+ in["path1"] = path1
+ in["path2"] = path2
+ in["workdir"] = workdir
+ _, _, err := jobs.NewJob(ctx, call.Fn, in)
+ return err
+ }
+ require.NoError(t, run(rc.Params{"resync": true}))
+ return path1, path2, run
+}
+
+func TestRcDryRun(t *testing.T) {
+ for _, test := range []struct {
+ name string
+ in rc.Params
+ dryRun bool
+ }{
+ {"dryRun", rc.Params{"dryRun": true}, true},
+ {"dry_run", rc.Params{"dry_run": true}, true},
+ {"_config", rc.Params{"_config": rc.Params{"DryRun": true}}, true},
+ {"dry_run and dryRun false", rc.Params{"dry_run": true, "dryRun": false}, true},
+ {"_config and dryRun false", rc.Params{"_config": rc.Params{"DryRun": true}, "dryRun": false}, true},
+ {"dryRun false", rc.Params{"dryRun": false}, false},
+ } {
+ t.Run(test.name, func(t *testing.T) {
+ path1, path2, run := newRcBisync(t, 1)
+ require.NoError(t, os.WriteFile(filepath.Join(path1, "new.txt"), []byte("new"), 0o644))
+ require.NoError(t, run(test.in))
+ if test.dryRun {
+ assert.NoFileExists(t, filepath.Join(path2, "new.txt"))
+ } else {
+ assert.FileExists(t, filepath.Join(path2, "new.txt"))
+ }
+ })
+ }
+}
+
+func TestRcMaxDelete(t *testing.T) {
+ for _, test := range []struct {
+ name string
+ in rc.Params
+ wantErr bool
+ }{
+ {"default", rc.Params{}, false},
+ {"maxDelete", rc.Params{"maxDelete": 5}, true},
+ {"max_delete allows", rc.Params{"max_delete": 20}, false},
+ {"max_delete aborts", rc.Params{"max_delete": 5}, true},
+ {"_config MaxDelete aborts", rc.Params{"_config": rc.Params{"MaxDelete": 5}}, true},
+ } {
+ t.Run(test.name, func(t *testing.T) {
+ path1, path2, run := newRcBisync(t, 10)
+ require.NoError(t, os.Remove(filepath.Join(path1, "file0.txt")))
+ err := run(test.in)
+ if test.wantErr {
+ assert.ErrorContains(t, err, "too many deletes")
+ assert.FileExists(t, filepath.Join(path2, "file0.txt"))
+ } else {
+ require.NoError(t, err)
+ assert.NoFileExists(t, filepath.Join(path2, "file0.txt"))
+ }
+ })
+ }
+}
+
+// Test a --max-delete the rcd was started with applies, as on the command line
+func TestRcMaxDeleteFromRcd(t *testing.T) {
+ ctx, ci := fs.AddConfig(context.Background())
+ ci.MaxDelete = 5
+ path1, path2, run := newRcBisyncCtx(ctx, t, 10)
+ require.NoError(t, os.Remove(filepath.Join(path1, "file0.txt")))
+ assert.ErrorContains(t, run(rc.Params{}), "too many deletes")
+ assert.FileExists(t, filepath.Join(path2, "file0.txt"))
+}
diff --git a/docs/content/bisync.md b/docs/content/bisync.md
index 330ff0c20..09aca5c40 100644
--- a/docs/content/bisync.md
+++ b/docs/content/bisync.md
@@ -1947,6 +1947,10 @@ it, as with `--download`, instead of by size alone.
- Fixed an issue causing a `--resync` to fail with "path1 and path2 are out of
sync", or later runs to report changes that didn't happen, when a file's new
modtime was less than a second earlier than the one in the listing.
+- Fixed an issue causing the rc `sync/bisync` call to ignore a dry run set with
+`_config` or the flat `dry_run` option.
+- Fixed an issue causing the rc `sync/bisync` call to ignore the global
+`--max-delete` and use a limit of 0% when `maxDelete` was not set.
### `v1.74.2`