"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
bisync: fix data races on tests
nielash269abb1aeebc
diff --git a/cmd/bisync/bisync_test.go b/cmd/bisync/bisync_test.goindex 57adbd988..e0ae08185 100644--- a/cmd/bisync/bisync_test.go+++ b/cmd/bisync/bisync_test.go@@ -228,6 +228,17 @@ var color = bisync.Color// TestMain drives the testsfunc TestMain(m *testing.M) {bisync.LogTZ = time.UTC+ ci := fs.GetConfig(context.TODO())+ ciSave := *ci+ defer func() {+ *ci = ciSave+ }()+ // need to set context.TODO() here as we cannot pass a ctx to fs.LogLevelPrintf+ ci.LogLevel = fs.LogLevelInfo+ if *argDebug {+ ci.LogLevel = fs.LogLevelDebug+ }+ fstest.Initialise()fstest.TestMain(m)}@@ -240,7 +251,8 @@ func TestBisyncRemoteLocal(t *testing.T) {fs.Logf(nil, "remote: %v", remote)require.NoError(t, err)defer cleanup()- testBisync(t, remote, *argRemote2)+ ctx, _ := fs.AddConfig(context.TODO())+ testBisync(ctx, t, remote, *argRemote2)}// Path1 is local, Path2 is remote@@ -252,7 +264,8 @@ func TestBisyncLocalRemote(t *testing.T) {fs.Logf(nil, "remote: %v", remote)require.NoError(t, err)defer cleanup()- testBisync(t, *argRemote2, remote)+ ctx, _ := fs.AddConfig(context.TODO())+ testBisync(ctx, t, *argRemote2, remote)}// Path1 and Path2 are both different directories on remote@@ -262,7 +275,8 @@ func TestBisyncRemoteRemote(t *testing.T) {fs.Logf(nil, "remote: %v", remote)require.NoError(t, err)defer cleanup()- testBisync(t, remote, remote)+ ctx, _ := fs.AddConfig(context.TODO())+ testBisync(ctx, t, remote, remote)}// make sure rc can cope with running concurrent jobs@@ -285,10 +299,7 @@ func testParallel(t *testing.T) {}// TestBisync is a test engine for bisync test cases.-func testBisync(t *testing.T, path1, path2 string) {- ctx := context.Background()- fstest.Initialise()-+func testBisync(ctx context.Context, t *testing.T, path1, path2 string) {ci := fs.GetConfig(ctx)ciSave := *cidefer func() {@@ -297,7 +308,9 @@ func testBisync(t *testing.T, path1, path2 string) {if *argRefreshTimes {ci.RefreshTimes = true}+ bisync.ColorsLock.Lock()bisync.Colors = true+ bisync.ColorsLock.Unlock()ci.FsCacheExpireDuration = fs.Duration(5 * time.Hour)baseDir, err := os.Getwd()@@ -618,13 +631,8 @@ func (b *bisyncTest) makeTempRemote(ctx context.Context, remote, subdir string)}func (b *bisyncTest) cleanupCase(ctx context.Context) {- // Silence "directory not found" errors from the ftp backend- _ = bilib.CaptureOutput(func() {- _ = operations.Purge(ctx, b.fs1, "")- })- _ = bilib.CaptureOutput(func() {- _ = operations.Purge(ctx, b.fs2, "")- })+ _ = operations.Purge(ctx, b.fs1, "")+ _ = operations.Purge(ctx, b.fs2, "")_ = os.RemoveAll(b.workDir)accounting.Stats(ctx).ResetCounters()}@@ -639,11 +647,6 @@ func (b *bisyncTest) runTestStep(ctx context.Context, line string) (err error) {defer func() {*ci = ciSave}()- ci.LogLevel = fs.LogLevelInfo- if b.debug {- ci.LogLevel = fs.LogLevelDebug- }-testFunc := func() {src := filepath.Join(b.dataDir, "file7.txt")diff --git a/cmd/bisync/deltas.go b/cmd/bisync/deltas.goindex b882fb564..fa5ea785e 100644--- a/cmd/bisync/deltas.go+++ b/cmd/bisync/deltas.go@@ -396,7 +396,7 @@ func (b *bisyncRun) applyDeltas(ctx context.Context, ds1, ds2 *deltaSet) (resultb.march.ls1.getPut(file, skippedDirs1)b.march.ls2.getPut(file, skippedDirs2)b.debugFn(file, func() {- b.debug(file, fmt.Sprintf("deltas dir: %s, ls1 has name?: %v, b.march.ls2 has name?: %v", file, b.march.ls1.has(b.DebugName), b.march.ls2.has(b.DebugName)))+ b.debug(file, fmt.Sprintf("deltas dir: %s, ls1 has name?: %v, ls2 has name?: %v", file, b.march.ls1.has(b.DebugName), b.march.ls2.has(b.DebugName)))})} else {equal := matches.Has(file)diff --git a/cmd/bisync/log.go b/cmd/bisync/log.goindex 36d3ce6b8..64fda49ef 100644--- a/cmd/bisync/log.go+++ b/cmd/bisync/log.go@@ -6,6 +6,7 @@ import ("runtime""strconv""strings"+ "sync""github.com/rclone/rclone/fs""github.com/rclone/rclone/lib/encoder"@@ -67,10 +68,15 @@ func quotePath(path string) string {}// Colors controls whether terminal colors are enabled-var Colors bool+var (+ Colors bool+ ColorsLock sync.Mutex+)// Color handles terminal colors for bisyncfunc Color(style string, s string) string {+ ColorsLock.Lock()+ defer ColorsLock.Unlock()if !Colors {return s}@@ -80,6 +86,8 @@ func Color(style string, s string) string {// ColorX handles terminal colors for bisyncfunc ColorX(style string, s string) string {+ ColorsLock.Lock()+ defer ColorsLock.Unlock()if !Colors {return s}diff --git a/cmd/bisync/operations.go b/cmd/bisync/operations.goindex 5041e7109..e99ead6f8 100644--- a/cmd/bisync/operations.go+++ b/cmd/bisync/operations.go@@ -87,7 +87,9 @@ func Bisync(ctx context.Context, fs1, fs2 fs.Fs, optArg *Options) (err error) {opt.OrigBackupDir = ci.BackupDirif ci.TerminalColorMode == fs.TerminalColorModeAlways || (ci.TerminalColorMode == fs.TerminalColorModeAuto && !log.Redirected()) {+ ColorsLock.Lock()Colors = true+ ColorsLock.Unlock()}err = b.setCompareDefaults(ctx)diff --git a/cmd/bisync/queue.go b/cmd/bisync/queue.goindex f618619ff..ed0257b5d 100644--- a/cmd/bisync/queue.go+++ b/cmd/bisync/queue.go@@ -245,8 +245,10 @@ func (b *bisyncRun) fastCopy(ctx context.Context, fsrc, fdst fs.Fs, files bilib.}}- b.SyncCI = fs.GetConfig(ctxCopy) // allows us to request graceful shutdown- accounting.MaxCompletedTransfers = -1 // we need a complete list in the event of graceful shutdown+ b.SyncCI = fs.GetConfig(ctxCopy) // allows us to request graceful shutdown+ if accounting.MaxCompletedTransfers != -1 {+ accounting.MaxCompletedTransfers = -1 // we need a complete list in the event of graceful shutdown+ }ctxCopy, b.CancelSync = context.WithCancel(ctxCopy)b.testFn()err := sync.Sync(ctxCopy, fdst, fsrc, b.opt.CreateEmptySrcDirs)diff --git a/cmd/bisync/resync.go b/cmd/bisync/resync.goindex 33565db1d..4ed50133e 100644--- a/cmd/bisync/resync.go+++ b/cmd/bisync/resync.go@@ -20,7 +20,6 @@ func (b *bisyncRun) setResyncDefaults() {}if b.opt.ResyncMode != PreferNone {b.opt.Resync = true- Opt.Resync = true // shouldn't be using this one, but set to be safe}// checks and warningsdiff --git a/fs/log/slog.go b/fs/log/slog.goindex 10b64a610..0efbcbee6 100644--- a/fs/log/slog.go+++ b/fs/log/slog.go@@ -29,7 +29,7 @@ var Handler = defaultHandler()// InitLogging has been called yet or not.func defaultHandler() *OutputHandler {// Default options for default handler- var opts = &slog.HandlerOptions{+ opts := &slog.HandlerOptions{Level: fs.LogLevelToSlog(fs.InitialLogLevel()),}@@ -87,7 +87,7 @@ func getCaller(skip int) string {return ""}frames := runtime.CallersFrames(pc[:n])- var more = true+ more := truevar frame runtime.Framefor more {frame, more = frames.Next()@@ -175,11 +175,15 @@ func NewOutputHandler(out io.Writer, opts *slog.HandlerOptions, format logFormat//// This is for temporarily overriding the output.func (h *OutputHandler) SetOutput(fn outputFn) {+ h.mu.Lock()+ defer h.mu.Unlock()h.output = append(h.output, fn)}// ResetOutput resets the log output to what is was.func (h *OutputHandler) ResetOutput() {+ h.mu.Lock()+ defer h.mu.Unlock()if len(h.output) > 0 {h.output = h.output[:len(h.output)-1]}@@ -187,6 +191,8 @@ func (h *OutputHandler) ResetOutput() {// AddOutput adds an additional logging destination of the type specified.func (h *OutputHandler) AddOutput(json bool, fn outputFn) {+ h.mu.Lock()+ defer h.mu.Unlock()h.outputExtra = append(h.outputExtra, outputExtra{json: json,output: fn,@@ -195,6 +201,8 @@ func (h *OutputHandler) AddOutput(json bool, fn outputFn) {// SetLevel sets a new log level, returning the old one.func (h *OutputHandler) SetLevel(level slog.Level) slog.Level {+ h.mu.Lock()+ defer h.mu.Unlock()oldLevel := h.levelVar.Level()h.levelVar.Set(level)return oldLevel