"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
accounting: fix global error acounting
Benjamin Legrand8a6fc8535daa
diff --git a/cmd/cmd.go b/cmd/cmd.goindex 37a7d6f72..8f5f5f093 100644--- a/cmd/cmd.go+++ b/cmd/cmd.go@@ -83,12 +83,13 @@ func ShowVersion() {// It returns a string with the file name if points to a file// otherwise "".func NewFsFile(remote string) (fs.Fs, string) {+ ctx := context.Background()_, fsPath, err := fspath.SplitFs(remote)if err != nil {- err = fs.CountError(err)+ err = fs.CountError(ctx, err)fs.Fatalf(nil, "Failed to create file system for %q: %v", remote, err)}- f, err := cache.Get(context.Background(), remote)+ f, err := cache.Get(ctx, remote)switch err {case fs.ErrorIsFile:cache.Pin(f) // pin indefinitely since it was on the CLI@@ -97,7 +98,7 @@ func NewFsFile(remote string) (fs.Fs, string) {cache.Pin(f) // pin indefinitely since it was on the CLIreturn f, ""default:- err = fs.CountError(err)+ err = fs.CountError(ctx, err)fs.Fatalf(nil, "Failed to create file system for %q: %v", remote, err)}return nil, ""@@ -108,18 +109,19 @@ func NewFsFile(remote string) (fs.Fs, string) {// This works the same as NewFsFile however it adds filters to the Fs// to limit it to a single file if the remote pointed to a file.func newFsFileAddFilter(remote string) (fs.Fs, string) {- fi := filter.GetConfig(context.Background())+ ctx := context.Background()+ fi := filter.GetConfig(ctx)f, fileName := NewFsFile(remote)if fileName != "" {if !fi.InActive() {err := fmt.Errorf("can't limit to single files when using filters: %v", remote)- err = fs.CountError(err)+ err = fs.CountError(ctx, err)fs.Fatal(nil, err.Error())}// Limit transfers to this fileerr := fi.AddFile(fileName)if err != nil {- err = fs.CountError(err)+ err = fs.CountError(ctx, err)fs.Fatalf(nil, "Failed to limit to single file %q: %v", remote, err)}}@@ -139,9 +141,10 @@ func NewFsSrc(args []string) fs.Fs {//// This must point to a directoryfunc newFsDir(remote string) fs.Fs {- f, err := cache.Get(context.Background(), remote)+ ctx := context.Background()+ f, err := cache.Get(ctx, remote)if err != nil {- err = fs.CountError(err)+ err = fs.CountError(ctx, err)fs.Fatalf(nil, "Failed to create file system for %q: %v", remote, err)}cache.Pin(f) // pin indefinitely since it was on the CLI@@ -175,6 +178,7 @@ func NewFsSrcFileDst(args []string) (fsrc fs.Fs, srcFileName string, fdst fs.Fs)// NewFsSrcDstFiles creates a new src and dst fs from the arguments// If src is a file then srcFileName and dstFileName will be non-emptyfunc NewFsSrcDstFiles(args []string) (fsrc fs.Fs, srcFileName string, fdst fs.Fs, dstFileName string) {+ ctx := context.Background()fsrc, srcFileName = newFsFileAddFilter(args[0])// If copying a file...dstRemote := args[1]@@ -193,14 +197,14 @@ func NewFsSrcDstFiles(args []string) (fsrc fs.Fs, srcFileName string, fdst fs.Fsfs.Fatalf(nil, "%q is a directory", args[1])}}- fdst, err := cache.Get(context.Background(), dstRemote)+ fdst, err := cache.Get(ctx, dstRemote)switch err {case fs.ErrorIsFile:- _ = fs.CountError(err)+ _ = fs.CountError(ctx, err)fs.Fatalf(nil, "Source doesn't exist or is a directory and destination is a file")case nil:default:- _ = fs.CountError(err)+ _ = fs.CountError(ctx, err)fs.Fatalf(nil, "Failed to create file system for destination %q: %v", dstRemote, err)}cache.Pin(fdst) // pin indefinitely since it was on the CLI@@ -234,7 +238,8 @@ func ShowStats() bool {// Run the function with stats and retries if requiredfunc Run(Retry bool, showStats bool, cmd *cobra.Command, f func() error) {- ci := fs.GetConfig(context.Background())+ ctx := context.Background()+ ci := fs.GetConfig(ctx)var cmdErr errorstopStats := func() {}if !showStats && ShowStats() {@@ -248,7 +253,7 @@ func Run(Retry bool, showStats bool, cmd *cobra.Command, f func() error) {SigInfoHandler()for try := 1; try <= ci.Retries; try++ {cmdErr = f()- cmdErr = fs.CountError(cmdErr)+ cmdErr = fs.CountError(ctx, cmdErr)lastErr := accounting.GlobalStats().GetLastError()if cmdErr == nil {cmdErr = lastErr@@ -436,19 +441,19 @@ func initConfig() {fs.Infof(nil, "Creating CPU profile %q\n", *cpuProfile)f, err := os.Create(*cpuProfile)if err != nil {- err = fs.CountError(err)+ err = fs.CountError(ctx, err)fs.Fatal(nil, fmt.Sprint(err))}err = pprof.StartCPUProfile(f)if err != nil {- err = fs.CountError(err)+ err = fs.CountError(ctx, err)fs.Fatal(nil, fmt.Sprint(err))}atexit.Register(func() {pprof.StopCPUProfile()err := f.Close()if err != nil {- err = fs.CountError(err)+ err = fs.CountError(ctx, err)fs.Fatal(nil, fmt.Sprint(err))}})@@ -460,17 +465,17 @@ func initConfig() {fs.Infof(nil, "Saving Memory profile %q\n", *memProfile)f, err := os.Create(*memProfile)if err != nil {- err = fs.CountError(err)+ err = fs.CountError(ctx, err)fs.Fatal(nil, fmt.Sprint(err))}err = pprof.WriteHeapProfile(f)if err != nil {- err = fs.CountError(err)+ err = fs.CountError(ctx, err)fs.Fatal(nil, fmt.Sprint(err))}err = f.Close()if err != nil {- err = fs.CountError(err)+ err = fs.CountError(ctx, err)fs.Fatal(nil, fmt.Sprint(err))}})@@ -478,7 +483,8 @@ func initConfig() {}func resolveExitCode(err error) {- ci := fs.GetConfig(context.Background())+ ctx := context.Background()+ ci := fs.GetConfig(ctx)atexit.Run()if err == nil {if ci.ErrorOnNoTransfer {diff --git a/cmd/serve/dlna/dlna.go b/cmd/serve/dlna/dlna.goindex 29af87b4a..199350877 100644--- a/cmd/serve/dlna/dlna.go+++ b/cmd/serve/dlna/dlna.go@@ -190,16 +190,17 @@ func (s *server) ModelNumber() string {// Renders the root device descriptor.func (s *server) rootDescHandler(w http.ResponseWriter, r *http.Request) {+ ctx := r.Context()tmpl, err := data.GetTemplate()if err != nil {- serveError(s, w, "Failed to load root descriptor template", err)+ serveError(ctx, s, w, "Failed to load root descriptor template", err)return}buffer := new(bytes.Buffer)err = tmpl.Execute(buffer, s)if err != nil {- serveError(s, w, "Failed to render root descriptor XML", err)+ serveError(ctx, s, w, "Failed to render root descriptor XML", err)return}@@ -215,15 +216,16 @@ func (s *server) rootDescHandler(w http.ResponseWriter, r *http.Request) {// Handle a service control HTTP request.func (s *server) serviceControlHandler(w http.ResponseWriter, r *http.Request) {+ ctx := r.Context()soapActionString := r.Header.Get("SOAPACTION")soapAction, err := upnp.ParseActionHTTPHeader(soapActionString)if err != nil {- serveError(s, w, "Could not parse SOAPACTION header", err)+ serveError(ctx, s, w, "Could not parse SOAPACTION header", err)return}var env soap.Envelopeif err := xml.NewDecoder(r.Body).Decode(&env); err != nil {- serveError(s, w, "Could not parse SOAP request body", err)+ serveError(ctx, s, w, "Could not parse SOAP request body", err)return}@@ -257,6 +259,7 @@ func (s *server) soapActionResponse(sa upnp.SoapAction, actionRequestXML []byte,// Serves actual resources (media files).func (s *server) resourceHandler(w http.ResponseWriter, r *http.Request) {+ ctx := r.Context()remotePath := r.URL.Pathnode, err := s.vfs.Stat(r.URL.Path)if err != nil {@@ -277,7 +280,7 @@ func (s *server) resourceHandler(w http.ResponseWriter, r *http.Request) {file := node.(*vfs.File)in, err := file.Open(os.O_RDONLY)if err != nil {- serveError(node, w, "Could not open resource", err)+ serveError(ctx, node, w, "Could not open resource", err)return}defer fs.CheckClose(in, &err)diff --git a/cmd/serve/dlna/dlna_util.go b/cmd/serve/dlna/dlna_util.goindex a63f31f5b..ca61488af 100644--- a/cmd/serve/dlna/dlna_util.go+++ b/cmd/serve/dlna/dlna_util.go@@ -1,6 +1,7 @@package dlnaimport (+ "context""crypto/md5""encoding/xml""fmt"@@ -142,9 +143,10 @@ func logging(next http.Handler) http.Handler {// Error recovery and general request logging are left to logging().func traceLogging(next http.Handler) http.Handler {return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ ctx := r.Context()dump, err := httputil.DumpRequest(r, true)if err != nil {- serveError(nil, w, "error dumping request", err)+ serveError(ctx, nil, w, "error dumping request", err)return}fs.Debugf(nil, "%s", dump)@@ -182,8 +184,8 @@ func withHeader(name string, value string, next http.Handler) http.Handler {}// serveError returns an http.StatusInternalServerError and logs the error-func serveError(what interface{}, w http.ResponseWriter, text string, err error) {- err = fs.CountError(err)+func serveError(ctx context.Context, what interface{}, w http.ResponseWriter, text string, err error) {+ err = fs.CountError(ctx, err)fs.Errorf(what, "%s: %v", text, err)http.Error(w, text+".", http.StatusInternalServerError)}diff --git a/cmd/serve/http/http.go b/cmd/serve/http/http.goindex 9ace8e663..7aba1e598 100644--- a/cmd/serve/http/http.go+++ b/cmd/serve/http/http.go@@ -186,6 +186,7 @@ func (s *HTTP) handler(w http.ResponseWriter, r *http.Request) {// serveDir serves a directory index at dirRemotefunc (s *HTTP) serveDir(w http.ResponseWriter, r *http.Request, dirRemote string) {+ ctx := r.Context()VFS, err := s.getVFS(r.Context())if err != nil {http.Error(w, "Root directory not found", http.StatusNotFound)@@ -198,7 +199,7 @@ func (s *HTTP) serveDir(w http.ResponseWriter, r *http.Request, dirRemote stringhttp.Error(w, "Directory not found", http.StatusNotFound)return} else if err != nil {- serve.Error(dirRemote, w, "Failed to list directory", err)+ serve.Error(ctx, dirRemote, w, "Failed to list directory", err)return}if !node.IsDir() {@@ -208,7 +209,7 @@ func (s *HTTP) serveDir(w http.ResponseWriter, r *http.Request, dirRemote stringdir := node.(*vfs.Dir)dirEntries, err := dir.ReadDirAll()if err != nil {- serve.Error(dirRemote, w, "Failed to list directory", err)+ serve.Error(ctx, dirRemote, w, "Failed to list directory", err)return}@@ -234,6 +235,7 @@ func (s *HTTP) serveDir(w http.ResponseWriter, r *http.Request, dirRemote string// serveFile serves a file object at remotefunc (s *HTTP) serveFile(w http.ResponseWriter, r *http.Request, remote string) {+ ctx := r.Context()VFS, err := s.getVFS(r.Context())if err != nil {http.Error(w, "File not found", http.StatusNotFound)@@ -247,7 +249,7 @@ func (s *HTTP) serveFile(w http.ResponseWriter, r *http.Request, remote string)http.Error(w, "File not found", http.StatusNotFound)return} else if err != nil {- serve.Error(remote, w, "Failed to find file", err)+ serve.Error(ctx, remote, w, "Failed to find file", err)return}if !node.IsFile() {@@ -287,7 +289,7 @@ func (s *HTTP) serveFile(w http.ResponseWriter, r *http.Request, remote string)// open the objectin, err := file.Open(os.O_RDONLY)if err != nil {- serve.Error(remote, w, "Failed to open file", err)+ serve.Error(ctx, remote, w, "Failed to open file", err)return}defer func() {diff --git a/cmd/serve/webdav/webdav.go b/cmd/serve/webdav/webdav.goindex 622575e8d..0c5c16416 100644--- a/cmd/serve/webdav/webdav.go+++ b/cmd/serve/webdav/webdav.go@@ -349,6 +349,7 @@ func (w *WebDAV) ServeHTTP(rw http.ResponseWriter, r *http.Request) {// serveDir serves a directory index at dirRemote// This is similar to serveDir in serve http.func (w *WebDAV) serveDir(rw http.ResponseWriter, r *http.Request, dirRemote string) {+ ctx := r.Context()VFS, err := w.getVFS(r.Context())if err != nil {http.Error(rw, "Root directory not found", http.StatusNotFound)@@ -361,7 +362,7 @@ func (w *WebDAV) serveDir(rw http.ResponseWriter, r *http.Request, dirRemote strhttp.Error(rw, "Directory not found", http.StatusNotFound)return} else if err != nil {- serve.Error(dirRemote, rw, "Failed to list directory", err)+ serve.Error(ctx, dirRemote, rw, "Failed to list directory", err)return}if !node.IsDir() {@@ -372,7 +373,7 @@ func (w *WebDAV) serveDir(rw http.ResponseWriter, r *http.Request, dirRemote strdirEntries, err := dir.ReadDirAll()if err != nil {- serve.Error(dirRemote, rw, "Failed to list directory", err)+ serve.Error(ctx, dirRemote, rw, "Failed to list directory", err)return}diff --git a/fs/accounting/accounting.go b/fs/accounting/accounting.goindex d7d7e02dd..5a6be7df4 100644--- a/fs/accounting/accounting.go+++ b/fs/accounting/accounting.go@@ -44,7 +44,9 @@ func Start(ctx context.Context) {//// We can't do this in an init() method as it uses fs.Config// and that isn't set up then.- fs.CountError = GlobalStats().Error+ fs.CountError = func(ctx context.Context, err error) error {+ return Stats(ctx).Error(err)+ }}// Account limits and accounts for one transferdiff --git a/fs/accounting/stats_groups_test.go b/fs/accounting/stats_groups_test.goindex dda9dccef..8c02229e7 100644--- a/fs/accounting/stats_groups_test.go+++ b/fs/accounting/stats_groups_test.go@@ -7,6 +7,8 @@ import ("testing""time"+ "github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/fserrors""github.com/rclone/rclone/fs/rc""github.com/rclone/rclone/fstest/testy""github.com/stretchr/testify/assert"@@ -206,6 +208,34 @@ func TestStatsGroupOperations(t *testing.T) {})}+func TestCountError(t *testing.T) {+ ctx := context.Background()+ Start(ctx)+ defer func() {+ groups = newStatsGroups()+ }()+ t.Run("global stats", func(t *testing.T) {+ GlobalStats().ResetCounters()+ err := fs.CountError(ctx, fmt.Errorf("global err"))+ assert.Equal(t, int64(1), GlobalStats().errors)++ assert.True(t, fserrors.IsCounted(err))+ })+ t.Run("group stats", func(t *testing.T) {+ statGroupName := fmt.Sprintf("%s-error_group", t.Name())+ GlobalStats().ResetCounters()+ stCtx := WithStatsGroup(ctx, statGroupName)+ st := StatsGroup(stCtx, statGroupName)++ err := fs.CountError(stCtx, fmt.Errorf("group err"))++ assert.Equal(t, int64(0), GlobalStats().errors)+ assert.Equal(t, int64(1), st.errors)+ assert.True(t, fserrors.IsCounted(err))+ })++}+func percentDiff(start, end uint64) uint64 {return (start - end) * 100 / start}diff --git a/fs/cache/cache.go b/fs/cache/cache.goindex e2c104395..63afa94e8 100644--- a/fs/cache/cache.go+++ b/fs/cache/cache.go@@ -29,7 +29,7 @@ func createOnFirstUse() {c.SetExpireInterval(ci.FsCacheExpireInterval)c.SetFinalizer(func(value interface{}) {if s, ok := value.(fs.Shutdowner); ok {- _ = fs.CountError(s.Shutdown(context.Background()))+ _ = fs.CountError(context.Background(), s.Shutdown(context.Background()))}})})diff --git a/fs/config.go b/fs/config.goindex caf242fdf..1b11da821 100644--- a/fs/config.go+++ b/fs/config.go@@ -41,7 +41,7 @@ var (//// This is a function pointer to decouple the config// implementation from the fs- CountError = func(err error) error { return err }+ CountError = func(ctx context.Context, err error) error { return err }// ConfigProvider is the config key used for provider optionsConfigProvider = "provider"diff --git a/fs/march/march.go b/fs/march/march.goindex 856be0e7c..1116220db 100644--- a/fs/march/march.go+++ b/fs/march/march.go@@ -415,7 +415,7 @@ func (m *March) processJob(job listDirJob) ([]listDirJob, error) {} else {fs.Errorf(m.Fsrc, "error reading source root directory: %v", srcListErr)}- srcListErr = fs.CountError(srcListErr)+ srcListErr = fs.CountError(m.Ctx, srcListErr)return nil, srcListErr}if dstListErr == fs.ErrorDirNotFound {@@ -426,7 +426,7 @@ func (m *March) processJob(job listDirJob) ([]listDirJob, error) {} else {fs.Errorf(m.Fdst, "error reading destination root directory: %v", dstListErr)}- dstListErr = fs.CountError(dstListErr)+ dstListErr = fs.CountError(m.Ctx, dstListErr)return nil, dstListErr}diff --git a/fs/operations/check.go b/fs/operations/check.goindex b9fe36aab..c3c747071 100644--- a/fs/operations/check.go+++ b/fs/operations/check.go@@ -49,6 +49,7 @@ type CheckOpt struct {// checkMarch is used to march over two Fses in the same way as// sync/copytype checkMarch struct {+ ctx context.ContextioMu sync.Mutexwg sync.WaitGrouptokens chan struct{}@@ -83,7 +84,7 @@ func (c *checkMarch) DstOnly(dst fs.DirEntry) (recurse bool) {}err := fmt.Errorf("file not in %v", c.opt.Fsrc)fs.Errorf(dst, "%v", err)- _ = fs.CountError(err)+ _ = fs.CountError(c.ctx, err)c.differences.Add(1)c.srcFilesMissing.Add(1)c.report(dst, c.opt.MissingOnSrc, '-')@@ -105,7 +106,7 @@ func (c *checkMarch) SrcOnly(src fs.DirEntry) (recurse bool) {case fs.Object:err := fmt.Errorf("file not in %v", c.opt.Fdst)fs.Errorf(src, "%v", err)- _ = fs.CountError(err)+ _ = fs.CountError(c.ctx, err)c.differences.Add(1)c.dstFilesMissing.Add(1)c.report(src, c.opt.MissingOnDst, '+')@@ -155,13 +156,13 @@ func (c *checkMarch) Match(ctx context.Context, dst, src fs.DirEntry) (recurse bdiffer, noHash, err := c.checkIdentical(ctx, dstX, srcX)if err != nil {fs.Errorf(src, "%v", err)- _ = fs.CountError(err)+ _ = fs.CountError(ctx, err)c.report(src, c.opt.Error, '!')} else if differ {c.differences.Add(1)err := errors.New("files differ")// the checkFn has already logged the reason- _ = fs.CountError(err)+ _ = fs.CountError(ctx, err)c.report(src, c.opt.Differ, '*')} else {c.matches.Add(1)@@ -177,7 +178,7 @@ func (c *checkMarch) Match(ctx context.Context, dst, src fs.DirEntry) (recurse b} else {err := fmt.Errorf("is file on %v but directory on %v", c.opt.Fsrc, c.opt.Fdst)fs.Errorf(src, "%v", err)- _ = fs.CountError(err)+ _ = fs.CountError(ctx, err)c.differences.Add(1)c.dstFilesMissing.Add(1)c.report(src, c.opt.MissingOnDst, '+')@@ -190,7 +191,7 @@ func (c *checkMarch) Match(ctx context.Context, dst, src fs.DirEntry) (recurse b}err := fmt.Errorf("is file on %v but directory on %v", c.opt.Fdst, c.opt.Fsrc)fs.Errorf(dst, "%v", err)- _ = fs.CountError(err)+ _ = fs.CountError(ctx, err)c.differences.Add(1)c.srcFilesMissing.Add(1)c.report(dst, c.opt.MissingOnSrc, '-')@@ -214,6 +215,7 @@ func CheckFn(ctx context.Context, opt *CheckOpt) error {return errors.New("internal error: nil check function")}c := &checkMarch{+ ctx: ctx,tokens: make(chan struct{}, ci.Checkers),opt: *opt,}@@ -430,6 +432,7 @@ func CheckSum(ctx context.Context, fsrc, fsum fs.Fs, sumFile string, hashType haci := fs.GetConfig(ctx)c := &checkMarch{+ ctx: ctx,tokens: make(chan struct{}, ci.Checkers),opt: *opt,}@@ -450,7 +453,7 @@ func CheckSum(ctx context.Context, fsrc, fsum fs.Fs, sumFile string, hashType ha// filesystem missed the file, sum wasn't consumederr := fmt.Errorf("file not in %v", opt.Fdst)fs.Errorf(filename, "%v", err)- _ = fs.CountError(err)+ _ = fs.CountError(ctx, err)if lastErr == nil {lastErr = err}@@ -479,7 +482,7 @@ func (c *checkMarch) checkSum(ctx context.Context, obj fs.Object, download bool,if !sumFound {err = errors.New("sum not found")- _ = fs.CountError(err)+ _ = fs.CountError(ctx, err)fs.Errorf(obj, "%v", err)c.differences.Add(1)c.srcFilesMissing.Add(1)@@ -528,12 +531,12 @@ func (c *checkMarch) checkSum(ctx context.Context, obj fs.Object, download bool,func (c *checkMarch) matchSum(ctx context.Context, sumHash, objHash string, obj fs.Object, err error, hashType hash.Type) {switch {case err != nil:- _ = fs.CountError(err)+ _ = fs.CountError(ctx, err)fs.Errorf(obj, "Failed to calculate hash: %v", err)c.report(obj, c.opt.Error, '!')case sumHash == "":err = errors.New("duplicate file")- _ = fs.CountError(err)+ _ = fs.CountError(ctx, err)fs.Errorf(obj, "%v", err)c.report(obj, c.opt.Error, '!')case objHash == "":@@ -548,7 +551,7 @@ func (c *checkMarch) matchSum(ctx context.Context, sumHash, objHash string, objc.report(obj, c.opt.Match, '=')default:err = errors.New("files differ")- _ = fs.CountError(err)+ _ = fs.CountError(ctx, err)fs.Debugf(nil, "%v = %s (sum)", hashType, sumHash)fs.Debugf(obj, "%v = %s (%v)", hashType, objHash, c.opt.Fdst)fs.Errorf(obj, "%v", err)diff --git a/fs/operations/copy.go b/fs/operations/copy.goindex 4e2f09762..8228a14c7 100644--- a/fs/operations/copy.go+++ b/fs/operations/copy.go@@ -331,7 +331,7 @@ func (c *copy) copy(ctx context.Context) (newDst fs.Object, err error) {}}if err != nil {- err = fs.CountError(err)+ err = fs.CountError(ctx, err)fs.Errorf(c.src, "Failed to copy: %v", err)if !c.inplace {c.removeFailedPartialCopy(ctx, c.f, c.remoteForCopy)@@ -343,7 +343,7 @@ func (c *copy) copy(ctx context.Context) (newDst fs.Object, err error) {err = c.verify(ctx, newDst)if err != nil {fs.Errorf(newDst, "%v", err)- err = fs.CountError(err)+ err = fs.CountError(ctx, err)c.removeFailedCopy(ctx, newDst)return nil, err}@@ -353,7 +353,7 @@ func (c *copy) copy(ctx context.Context) (newDst fs.Object, err error) {movedNewDst, err := c.dstFeatures.Move(ctx, newDst, c.remote)if err != nil {fs.Errorf(newDst, "partial file rename failed: %v", err)- err = fs.CountError(err)+ err = fs.CountError(ctx, err)c.removeFailedCopy(ctx, newDst)return nil, err}diff --git a/fs/operations/dedupe.go b/fs/operations/dedupe.goindex 3e0f72cd5..60c66c4a3 100644--- a/fs/operations/dedupe.go+++ b/fs/operations/dedupe.go@@ -32,7 +32,7 @@ outer:_, err := f.NewObject(ctx, newName)for ; err != fs.ErrorObjectNotFound; suffix++ {if err != nil {- err = fs.CountError(err)+ err = fs.CountError(ctx, err)fs.Errorf(o, "Failed to check for existing object: %v", err)continue outer}@@ -46,7 +46,7 @@ outer:if !SkipDestructive(ctx, o, "rename") {newObj, err := doMove(ctx, o, newName)if err != nil {- err = fs.CountError(err)+ err = fs.CountError(ctx, err)fs.Errorf(o, "Failed to rename: %v", err)continue}@@ -374,7 +374,7 @@ func dedupeMergeDuplicateDirs(ctx context.Context, f fs.Fs, duplicateDirs [][]*dfs.Infof(fsDirs[0], "Merging contents of duplicate directories")err := mergeDirs(ctx, fsDirs)if err != nil {- err = fs.CountError(err)+ err = fs.CountError(ctx, err)fs.Errorf(nil, "merge duplicate dirs: %v", err)}}diff --git a/fs/operations/operations.go b/fs/operations/operations.goindex f14ecb61e..53d846525 100644--- a/fs/operations/operations.go+++ b/fs/operations/operations.go@@ -101,11 +101,11 @@ func checkHashes(ctx context.Context, src fs.ObjectInfo, dst fs.Object, ht hash.return true, hash.None, srcHash, dstHash, nil}if srcErr != nil {- err = fs.CountError(srcErr)+ err = fs.CountError(ctx, srcErr)fs.Errorf(src, "Failed to calculate src hash: %v", err)}if dstErr != nil {- err = fs.CountError(dstErr)+ err = fs.CountError(ctx, dstErr)fs.Errorf(dst, "Failed to calculate dst hash: %v", err)}if err != nil {@@ -340,7 +340,7 @@ func equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object, opt equalOpt)logger(ctx, Differ, src, dst, nil)return false} else if err != nil {- err = fs.CountError(err)+ err = fs.CountError(ctx, err)fs.Errorf(dst, "Failed to set modification time: %v", err)} else {fs.Infof(src, "Updated modification time in destination")@@ -481,7 +481,7 @@ func move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.fs.Debugf(src, "Can't move, switching to copy")_ = in.Close()default:- err = fs.CountError(err)+ err = fs.CountError(ctx, err)fs.Errorf(src, "Couldn't move: %v", err)_ = in.Close()return newDst, err@@ -566,7 +566,7 @@ func DeleteFileWithBackupDir(ctx context.Context, dst fs.Object, backupDir fs.Fs}if err != nil {fs.Errorf(dst, "Couldn't %s: %v", action, err)- err = fs.CountError(err)+ err = fs.CountError(ctx, err)} else if !skip {fs.Infof(dst, "%s", actioned)}@@ -974,7 +974,7 @@ func HashLister(ctx context.Context, ht hash.Type, outputBase64 bool, downloadFl}()sum, err := HashSum(ctx, ht, outputBase64, downloadFlag, o)if err != nil {- fs.Errorf(o, "%v", fs.CountError(err))+ fs.Errorf(o, "%v", fs.CountError(ctx, err))return}SyncFprintf(w, "%*s %s\n", width, sum, o.Remote())@@ -1053,7 +1053,7 @@ func Mkdir(ctx context.Context, f fs.Fs, dir string) error {fs.Debugf(fs.LogDirName(f, dir), "Making directory")err := f.Mkdir(ctx, dir)if err != nil {- err = fs.CountError(err)+ err = fs.CountError(ctx, err)return err}return nil@@ -1075,7 +1075,7 @@ func MkdirMetadata(ctx context.Context, f fs.Fs, dir string, metadata fs.Metadatfs.Debugf(fs.LogDirName(f, dir), "Making directory with metadata")newDst, err = do(ctx, dir, metadata)if err != nil {- err = fs.CountError(err)+ err = fs.CountError(ctx, err)return nil, err}if mtime, ok := metadata["mtime"]; ok {@@ -1133,7 +1133,7 @@ func TryRmdir(ctx context.Context, f fs.Fs, dir string) error {func Rmdir(ctx context.Context, f fs.Fs, dir string) error {err := TryRmdir(ctx, f, dir)if err != nil {- err = fs.CountError(err)+ err = fs.CountError(ctx, err)return err}return err@@ -1162,7 +1162,7 @@ func Purge(ctx context.Context, f fs.Fs, dir string) (err error) {err = Rmdirs(ctx, f, dir, false)}if err != nil {- err = fs.CountError(err)+ err = fs.CountError(ctx, err)return err}return nil@@ -1207,7 +1207,7 @@ func listToChan(ctx context.Context, f fs.Fs, dir string) fs.ObjectsChan {})if err != nil && err != fs.ErrorDirNotFound {err = fmt.Errorf("failed to list: %w", err)- err = fs.CountError(err)+ err = fs.CountError(ctx, err)fs.Errorf(nil, "%v", err)}}()@@ -1267,7 +1267,7 @@ func Cat(ctx context.Context, f fs.Fs, w io.Writer, offset, count int64, sep []bvar in io.ReadCloserin, err = Open(ctx, o, options...)if err != nil {- err = fs.CountError(err)+ err = fs.CountError(ctx, err)fs.Errorf(o, "Failed to open: %v", err)return}@@ -1280,13 +1280,13 @@ func Cat(ctx context.Context, f fs.Fs, w io.Writer, offset, count int64, sep []bdefer mu.Unlock()_, err = io.Copy(w, in)if err != nil {- err = fs.CountError(err)+ err = fs.CountError(ctx, err)fs.Errorf(o, "Failed to send to output: %v", err)}if len(sep) > 0 {_, err = w.Write(sep)if err != nil {- err = fs.CountError(err)+ err = fs.CountError(ctx, err)fs.Errorf(o, "Failed to send separator to output: %v", err)}}@@ -1423,7 +1423,7 @@ func rcatSrc(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadClossrc := object.NewStaticObjectInfo(dstFileName, modTime, int64(readCounter.BytesRead()), false, sums, fdst).WithMetadata(meta)if !equal(ctx, src, dst, opt) {err = fmt.Errorf("corrupted on transfer")- err = fs.CountError(err)+ err = fs.CountError(ctx, err)fs.Errorf(dst, "%v", err)return dst, err}@@ -1450,7 +1450,7 @@ func Rmdirs(ctx context.Context, f fs.Fs, dir string, leaveRoot bool) error {dirEmpty[dir] = !leaveRooterr := walk.Walk(ctx, f, dir, false, ci.MaxDepth, func(dirPath string, entries fs.DirEntries, err error) error {if err != nil {- err = fs.CountError(err)+ err = fs.CountError(ctx, err)fs.Errorf(f, "Failed to list %q: %v", dirPath, err)return nil}@@ -1526,7 +1526,7 @@ func Rmdirs(ctx context.Context, f fs.Fs, dir string, leaveRoot bool) error {g.Go(func() error {err := TryRmdir(gCtx, f, dir)if err != nil {- err = fs.CountError(err)+ err = fs.CountError(ctx, err)fs.Errorf(dir, "Failed to rmdir: %v", err)errCount.Add(err)}@@ -2096,7 +2096,7 @@ func TouchDir(ctx context.Context, f fs.Fs, remote string, t time.Time, recursiverr := o.SetModTime(ctx, t)if err != nil {err = fmt.Errorf("failed to touch: %w", err)- err = fs.CountError(err)+ err = fs.CountError(ctx, err)fs.Errorf(o, "%v", err)}}diff --git a/fs/sync/sync.go b/fs/sync/sync.goindex 67998268f..e0a3e23ac 100644--- a/fs/sync/sync.go+++ b/fs/sync/sync.go@@ -400,7 +400,7 @@ func (s *syncCopyMove) pairChecker(in *pipe, out *pipe, fraction int, wg *sync.Wif needTransfer {// If files are treated as immutable, fail if destination exists and does not matchif s.ci.Immutable && pair.Dst != nil {- err := fs.CountError(fserrors.NoRetryError(fs.ErrorImmutableModified))+ err := fs.CountError(s.ctx, fserrors.NoRetryError(fs.ErrorImmutableModified))fs.Errorf(pair.Dst, "Source and destination exist but do not match: %v", err)s.processError(err)} else {@@ -1204,7 +1204,7 @@ func (s *syncCopyMove) setDelayedDirModTimes(ctx context.Context) error {_, err = operations.SetDirModTime(gCtx, s.fdst, item.dst, item.dir, item.modTime)}if err != nil {- err = fs.CountError(err)+ err = fs.CountError(ctx, err)fs.Errorf(item.dir, "Failed to update directory timestamp or metadata: %v", err)errCount.Add(err)}@@ -1399,7 +1399,7 @@ func MoveDir(ctx context.Context, fdst, fsrc fs.Fs, deleteEmptySrcDirs bool, copfs.Infof(fdst, "Server side directory move succeeded")return nildefault:- err = fs.CountError(err)+ err = fs.CountError(ctx, err)fs.Errorf(fdst, "Server side directory move failed: %v", err)return err}diff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.goindex 288616cd5..f4f1c556a 100644--- a/fs/sync/sync_test.go+++ b/fs/sync/sync_test.go@@ -946,7 +946,7 @@ func TestSyncIgnoreErrors(t *testing.T) {accounting.GlobalStats().ResetCounters()ctx = predictDstFromLogger(ctx)- _ = fs.CountError(errors.New("boom"))+ _ = fs.CountError(ctx, errors.New("boom"))assert.NoError(t, Sync(ctx, r.Fremote, r.Flocal, false))testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)@@ -1267,7 +1267,7 @@ func TestSyncAfterRemovingAFileAndAddingAFileSubDirWithErrors(t *testing.T) {ctx = predictDstFromLogger(ctx)accounting.GlobalStats().ResetCounters()- _ = fs.CountError(errors.New("boom"))+ _ = fs.CountError(ctx, errors.New("boom"))err := Sync(ctx, r.Fremote, r.Flocal, false)assert.Equal(t, fs.ErrorNotDeleting, err)testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)diff --git a/fs/walk/walk.go b/fs/walk/walk.goindex ada7a284d..51e567a4f 100644--- a/fs/walk/walk.go+++ b/fs/walk/walk.go@@ -170,7 +170,7 @@ func listRwalk(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLe// Carry on listing but return the error at the endif err != nil {listErr = err- err = fs.CountError(err)+ err = fs.CountError(ctx, err)fs.Errorf(path, "error listing: %v", err)return nil}@@ -415,7 +415,7 @@ func walk(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel i// NB once we have passed entries to fn we mustn't touch it againif err != nil && err != ErrorSkipDir {traversing.Done()- err = fs.CountError(err)+ err = fs.CountError(ctx, err)fs.Errorf(job.remote, "error listing: %v", err)closeQuit()// Send error to error channel if spacediff --git a/lib/http/serve/dir.go b/lib/http/serve/dir.goindex 664f81cf2..5a92ddeb8 100644--- a/lib/http/serve/dir.go+++ b/lib/http/serve/dir.go@@ -2,6 +2,7 @@ package serveimport ("bytes"+ "context""fmt""html/template""net/http"@@ -124,8 +125,8 @@ func (d *Directory) AddEntry(remote string, isDir bool) {}// Error logs the error and if a ResponseWriter is given it writes an http.StatusInternalServerError-func Error(what interface{}, w http.ResponseWriter, text string, err error) {- err = fs.CountError(err)+func Error(ctx context.Context, what interface{}, w http.ResponseWriter, text string, err error) {+ err = fs.CountError(ctx, err)fs.Errorf(what, "%s: %v", text, err)if w != nil {http.Error(w, text+".", http.StatusInternalServerError)@@ -223,6 +224,7 @@ const (// Serve serves a directoryfunc (d *Directory) Serve(w http.ResponseWriter, r *http.Request) {+ ctx := r.Context()// Account the transfertr := accounting.Stats(r.Context()).NewTransferRemoteSize(d.DirRemote, -1, nil, nil)defer tr.Done(r.Context(), nil)@@ -232,12 +234,12 @@ func (d *Directory) Serve(w http.ResponseWriter, r *http.Request) {buf := &bytes.Buffer{}err := d.HTMLTemplate.Execute(buf, d)if err != nil {- Error(d.DirRemote, w, "Failed to render template", err)+ Error(ctx, d.DirRemote, w, "Failed to render template", err)return}w.Header().Set("Content-Length", fmt.Sprintf("%d", buf.Len()))_, err = buf.WriteTo(w)if err != nil {- Error(d.DirRemote, nil, "Failed to drain template buffer", err)+ Error(ctx, d.DirRemote, nil, "Failed to drain template buffer", err)}}diff --git a/lib/http/serve/dir_test.go b/lib/http/serve/dir_test.goindex 7bedc0471..19ab22f57 100644--- a/lib/http/serve/dir_test.go+++ b/lib/http/serve/dir_test.go@@ -1,6 +1,7 @@package serveimport (+ "context""errors""html/template""io"@@ -88,9 +89,10 @@ func TestAddEntry(t *testing.T) {}func TestError(t *testing.T) {+ ctx := context.Background()w := httptest.NewRecorder()err := errors.New("help")- Error("potato", w, "sausage", err)+ Error(ctx, "potato", w, "sausage", err)resp := w.Result()assert.Equal(t, http.StatusInternalServerError, resp.StatusCode)body, _ := io.ReadAll(resp.Body)