Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

accounting: fix global error acounting

Benjamin Legrand8a6fc8535daa
diff --git a/cmd/cmd.go b/cmd/cmd.go
index 37a7d6f72..8f5f5f093 100644
--- a/cmd/cmd.go
+++ b/cmd/cmd.go
@@ -83,12 +83,13 @@ func ShowVersion() {
// It returns a string with the file name if points to a file
// otherwise "".
func NewFsFile(remote string) (fs.Fs, string) {
+ ctx := context.Background()
_, fsPath, err := fspath.SplitFs(remote)
if err != nil {
- err = fs.CountError(err)
+ err = fs.CountError(ctx, err)
fs.Fatalf(nil, "Failed to create file system for %q: %v", remote, err)
}
- f, err := cache.Get(context.Background(), remote)
+ f, err := cache.Get(ctx, remote)
switch err {
case fs.ErrorIsFile:
cache.Pin(f) // pin indefinitely since it was on the CLI
@@ -97,7 +98,7 @@ func NewFsFile(remote string) (fs.Fs, string) {
cache.Pin(f) // pin indefinitely since it was on the CLI
return f, ""
default:
- err = fs.CountError(err)
+ err = fs.CountError(ctx, err)
fs.Fatalf(nil, "Failed to create file system for %q: %v", remote, err)
}
return nil, ""
@@ -108,18 +109,19 @@ func NewFsFile(remote string) (fs.Fs, string) {
// This works the same as NewFsFile however it adds filters to the Fs
// to limit it to a single file if the remote pointed to a file.
func newFsFileAddFilter(remote string) (fs.Fs, string) {
- fi := filter.GetConfig(context.Background())
+ ctx := context.Background()
+ fi := filter.GetConfig(ctx)
f, fileName := NewFsFile(remote)
if fileName != "" {
if !fi.InActive() {
err := fmt.Errorf("can't limit to single files when using filters: %v", remote)
- err = fs.CountError(err)
+ err = fs.CountError(ctx, err)
fs.Fatal(nil, err.Error())
}
// Limit transfers to this file
err := fi.AddFile(fileName)
if err != nil {
- err = fs.CountError(err)
+ err = fs.CountError(ctx, err)
fs.Fatalf(nil, "Failed to limit to single file %q: %v", remote, err)
}
}
@@ -139,9 +141,10 @@ func NewFsSrc(args []string) fs.Fs {
//
// This must point to a directory
func newFsDir(remote string) fs.Fs {
- f, err := cache.Get(context.Background(), remote)
+ ctx := context.Background()
+ f, err := cache.Get(ctx, remote)
if err != nil {
- err = fs.CountError(err)
+ err = fs.CountError(ctx, err)
fs.Fatalf(nil, "Failed to create file system for %q: %v", remote, err)
}
cache.Pin(f) // pin indefinitely since it was on the CLI
@@ -175,6 +178,7 @@ func NewFsSrcFileDst(args []string) (fsrc fs.Fs, srcFileName string, fdst fs.Fs)
// NewFsSrcDstFiles creates a new src and dst fs from the arguments
// If src is a file then srcFileName and dstFileName will be non-empty
func NewFsSrcDstFiles(args []string) (fsrc fs.Fs, srcFileName string, fdst fs.Fs, dstFileName string) {
+ ctx := context.Background()
fsrc, srcFileName = newFsFileAddFilter(args[0])
// If copying a file...
dstRemote := args[1]
@@ -193,14 +197,14 @@ func NewFsSrcDstFiles(args []string) (fsrc fs.Fs, srcFileName string, fdst fs.Fs
fs.Fatalf(nil, "%q is a directory", args[1])
}
}
- fdst, err := cache.Get(context.Background(), dstRemote)
+ fdst, err := cache.Get(ctx, dstRemote)
switch err {
case fs.ErrorIsFile:
- _ = fs.CountError(err)
+ _ = fs.CountError(ctx, err)
fs.Fatalf(nil, "Source doesn't exist or is a directory and destination is a file")
case nil:
default:
- _ = fs.CountError(err)
+ _ = fs.CountError(ctx, err)
fs.Fatalf(nil, "Failed to create file system for destination %q: %v", dstRemote, err)
}
cache.Pin(fdst) // pin indefinitely since it was on the CLI
@@ -234,7 +238,8 @@ func ShowStats() bool {
// Run the function with stats and retries if required
func Run(Retry bool, showStats bool, cmd *cobra.Command, f func() error) {
- ci := fs.GetConfig(context.Background())
+ ctx := context.Background()
+ ci := fs.GetConfig(ctx)
var cmdErr error
stopStats := func() {}
if !showStats && ShowStats() {
@@ -248,7 +253,7 @@ func Run(Retry bool, showStats bool, cmd *cobra.Command, f func() error) {
SigInfoHandler()
for try := 1; try <= ci.Retries; try++ {
cmdErr = f()
- cmdErr = fs.CountError(cmdErr)
+ cmdErr = fs.CountError(ctx, cmdErr)
lastErr := accounting.GlobalStats().GetLastError()
if cmdErr == nil {
cmdErr = lastErr
@@ -436,19 +441,19 @@ func initConfig() {
fs.Infof(nil, "Creating CPU profile %q\n", *cpuProfile)
f, err := os.Create(*cpuProfile)
if err != nil {
- err = fs.CountError(err)
+ err = fs.CountError(ctx, err)
fs.Fatal(nil, fmt.Sprint(err))
}
err = pprof.StartCPUProfile(f)
if err != nil {
- err = fs.CountError(err)
+ err = fs.CountError(ctx, err)
fs.Fatal(nil, fmt.Sprint(err))
}
atexit.Register(func() {
pprof.StopCPUProfile()
err := f.Close()
if err != nil {
- err = fs.CountError(err)
+ err = fs.CountError(ctx, err)
fs.Fatal(nil, fmt.Sprint(err))
}
})
@@ -460,17 +465,17 @@ func initConfig() {
fs.Infof(nil, "Saving Memory profile %q\n", *memProfile)
f, err := os.Create(*memProfile)
if err != nil {
- err = fs.CountError(err)
+ err = fs.CountError(ctx, err)
fs.Fatal(nil, fmt.Sprint(err))
}
err = pprof.WriteHeapProfile(f)
if err != nil {
- err = fs.CountError(err)
+ err = fs.CountError(ctx, err)
fs.Fatal(nil, fmt.Sprint(err))
}
err = f.Close()
if err != nil {
- err = fs.CountError(err)
+ err = fs.CountError(ctx, err)
fs.Fatal(nil, fmt.Sprint(err))
}
})
@@ -478,7 +483,8 @@ func initConfig() {
}
func resolveExitCode(err error) {
- ci := fs.GetConfig(context.Background())
+ ctx := context.Background()
+ ci := fs.GetConfig(ctx)
atexit.Run()
if err == nil {
if ci.ErrorOnNoTransfer {
diff --git a/cmd/serve/dlna/dlna.go b/cmd/serve/dlna/dlna.go
index 29af87b4a..199350877 100644
--- a/cmd/serve/dlna/dlna.go
+++ b/cmd/serve/dlna/dlna.go
@@ -190,16 +190,17 @@ func (s *server) ModelNumber() string {
// Renders the root device descriptor.
func (s *server) rootDescHandler(w http.ResponseWriter, r *http.Request) {
+ ctx := r.Context()
tmpl, err := data.GetTemplate()
if err != nil {
- serveError(s, w, "Failed to load root descriptor template", err)
+ serveError(ctx, s, w, "Failed to load root descriptor template", err)
return
}
buffer := new(bytes.Buffer)
err = tmpl.Execute(buffer, s)
if err != nil {
- serveError(s, w, "Failed to render root descriptor XML", err)
+ serveError(ctx, s, w, "Failed to render root descriptor XML", err)
return
}
@@ -215,15 +216,16 @@ func (s *server) rootDescHandler(w http.ResponseWriter, r *http.Request) {
// Handle a service control HTTP request.
func (s *server) serviceControlHandler(w http.ResponseWriter, r *http.Request) {
+ ctx := r.Context()
soapActionString := r.Header.Get("SOAPACTION")
soapAction, err := upnp.ParseActionHTTPHeader(soapActionString)
if err != nil {
- serveError(s, w, "Could not parse SOAPACTION header", err)
+ serveError(ctx, s, w, "Could not parse SOAPACTION header", err)
return
}
var env soap.Envelope
if err := xml.NewDecoder(r.Body).Decode(&env); err != nil {
- serveError(s, w, "Could not parse SOAP request body", err)
+ serveError(ctx, s, w, "Could not parse SOAP request body", err)
return
}
@@ -257,6 +259,7 @@ func (s *server) soapActionResponse(sa upnp.SoapAction, actionRequestXML []byte,
// Serves actual resources (media files).
func (s *server) resourceHandler(w http.ResponseWriter, r *http.Request) {
+ ctx := r.Context()
remotePath := r.URL.Path
node, err := s.vfs.Stat(r.URL.Path)
if err != nil {
@@ -277,7 +280,7 @@ func (s *server) resourceHandler(w http.ResponseWriter, r *http.Request) {
file := node.(*vfs.File)
in, err := file.Open(os.O_RDONLY)
if err != nil {
- serveError(node, w, "Could not open resource", err)
+ serveError(ctx, node, w, "Could not open resource", err)
return
}
defer fs.CheckClose(in, &err)
diff --git a/cmd/serve/dlna/dlna_util.go b/cmd/serve/dlna/dlna_util.go
index a63f31f5b..ca61488af 100644
--- a/cmd/serve/dlna/dlna_util.go
+++ b/cmd/serve/dlna/dlna_util.go
@@ -1,6 +1,7 @@
package dlna
import (
+ "context"
"crypto/md5"
"encoding/xml"
"fmt"
@@ -142,9 +143,10 @@ func logging(next http.Handler) http.Handler {
// Error recovery and general request logging are left to logging().
func traceLogging(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ ctx := r.Context()
dump, err := httputil.DumpRequest(r, true)
if err != nil {
- serveError(nil, w, "error dumping request", err)
+ serveError(ctx, nil, w, "error dumping request", err)
return
}
fs.Debugf(nil, "%s", dump)
@@ -182,8 +184,8 @@ func withHeader(name string, value string, next http.Handler) http.Handler {
}
// serveError returns an http.StatusInternalServerError and logs the error
-func serveError(what interface{}, w http.ResponseWriter, text string, err error) {
- err = fs.CountError(err)
+func serveError(ctx context.Context, what interface{}, w http.ResponseWriter, text string, err error) {
+ err = fs.CountError(ctx, err)
fs.Errorf(what, "%s: %v", text, err)
http.Error(w, text+".", http.StatusInternalServerError)
}
diff --git a/cmd/serve/http/http.go b/cmd/serve/http/http.go
index 9ace8e663..7aba1e598 100644
--- a/cmd/serve/http/http.go
+++ b/cmd/serve/http/http.go
@@ -186,6 +186,7 @@ func (s *HTTP) handler(w http.ResponseWriter, r *http.Request) {
// serveDir serves a directory index at dirRemote
func (s *HTTP) serveDir(w http.ResponseWriter, r *http.Request, dirRemote string) {
+ ctx := r.Context()
VFS, err := s.getVFS(r.Context())
if err != nil {
http.Error(w, "Root directory not found", http.StatusNotFound)
@@ -198,7 +199,7 @@ func (s *HTTP) serveDir(w http.ResponseWriter, r *http.Request, dirRemote string
http.Error(w, "Directory not found", http.StatusNotFound)
return
} else if err != nil {
- serve.Error(dirRemote, w, "Failed to list directory", err)
+ serve.Error(ctx, dirRemote, w, "Failed to list directory", err)
return
}
if !node.IsDir() {
@@ -208,7 +209,7 @@ func (s *HTTP) serveDir(w http.ResponseWriter, r *http.Request, dirRemote string
dir := node.(*vfs.Dir)
dirEntries, err := dir.ReadDirAll()
if err != nil {
- serve.Error(dirRemote, w, "Failed to list directory", err)
+ serve.Error(ctx, dirRemote, w, "Failed to list directory", err)
return
}
@@ -234,6 +235,7 @@ func (s *HTTP) serveDir(w http.ResponseWriter, r *http.Request, dirRemote string
// serveFile serves a file object at remote
func (s *HTTP) serveFile(w http.ResponseWriter, r *http.Request, remote string) {
+ ctx := r.Context()
VFS, err := s.getVFS(r.Context())
if err != nil {
http.Error(w, "File not found", http.StatusNotFound)
@@ -247,7 +249,7 @@ func (s *HTTP) serveFile(w http.ResponseWriter, r *http.Request, remote string)
http.Error(w, "File not found", http.StatusNotFound)
return
} else if err != nil {
- serve.Error(remote, w, "Failed to find file", err)
+ serve.Error(ctx, remote, w, "Failed to find file", err)
return
}
if !node.IsFile() {
@@ -287,7 +289,7 @@ func (s *HTTP) serveFile(w http.ResponseWriter, r *http.Request, remote string)
// open the object
in, err := file.Open(os.O_RDONLY)
if err != nil {
- serve.Error(remote, w, "Failed to open file", err)
+ serve.Error(ctx, remote, w, "Failed to open file", err)
return
}
defer func() {
diff --git a/cmd/serve/webdav/webdav.go b/cmd/serve/webdav/webdav.go
index 622575e8d..0c5c16416 100644
--- a/cmd/serve/webdav/webdav.go
+++ b/cmd/serve/webdav/webdav.go
@@ -349,6 +349,7 @@ func (w *WebDAV) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
// serveDir serves a directory index at dirRemote
// This is similar to serveDir in serve http.
func (w *WebDAV) serveDir(rw http.ResponseWriter, r *http.Request, dirRemote string) {
+ ctx := r.Context()
VFS, err := w.getVFS(r.Context())
if err != nil {
http.Error(rw, "Root directory not found", http.StatusNotFound)
@@ -361,7 +362,7 @@ func (w *WebDAV) serveDir(rw http.ResponseWriter, r *http.Request, dirRemote str
http.Error(rw, "Directory not found", http.StatusNotFound)
return
} else if err != nil {
- serve.Error(dirRemote, rw, "Failed to list directory", err)
+ serve.Error(ctx, dirRemote, rw, "Failed to list directory", err)
return
}
if !node.IsDir() {
@@ -372,7 +373,7 @@ func (w *WebDAV) serveDir(rw http.ResponseWriter, r *http.Request, dirRemote str
dirEntries, err := dir.ReadDirAll()
if err != nil {
- serve.Error(dirRemote, rw, "Failed to list directory", err)
+ serve.Error(ctx, dirRemote, rw, "Failed to list directory", err)
return
}
diff --git a/fs/accounting/accounting.go b/fs/accounting/accounting.go
index d7d7e02dd..5a6be7df4 100644
--- a/fs/accounting/accounting.go
+++ b/fs/accounting/accounting.go
@@ -44,7 +44,9 @@ func Start(ctx context.Context) {
//
// We can't do this in an init() method as it uses fs.Config
// and that isn't set up then.
- fs.CountError = GlobalStats().Error
+ fs.CountError = func(ctx context.Context, err error) error {
+ return Stats(ctx).Error(err)
+ }
}
// Account limits and accounts for one transfer
diff --git a/fs/accounting/stats_groups_test.go b/fs/accounting/stats_groups_test.go
index dda9dccef..8c02229e7 100644
--- a/fs/accounting/stats_groups_test.go
+++ b/fs/accounting/stats_groups_test.go
@@ -7,6 +7,8 @@ import (
"testing"
"time"
+ "github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/fserrors"
"github.com/rclone/rclone/fs/rc"
"github.com/rclone/rclone/fstest/testy"
"github.com/stretchr/testify/assert"
@@ -206,6 +208,34 @@ func TestStatsGroupOperations(t *testing.T) {
})
}
+func TestCountError(t *testing.T) {
+ ctx := context.Background()
+ Start(ctx)
+ defer func() {
+ groups = newStatsGroups()
+ }()
+ t.Run("global stats", func(t *testing.T) {
+ GlobalStats().ResetCounters()
+ err := fs.CountError(ctx, fmt.Errorf("global err"))
+ assert.Equal(t, int64(1), GlobalStats().errors)
+
+ assert.True(t, fserrors.IsCounted(err))
+ })
+ t.Run("group stats", func(t *testing.T) {
+ statGroupName := fmt.Sprintf("%s-error_group", t.Name())
+ GlobalStats().ResetCounters()
+ stCtx := WithStatsGroup(ctx, statGroupName)
+ st := StatsGroup(stCtx, statGroupName)
+
+ err := fs.CountError(stCtx, fmt.Errorf("group err"))
+
+ assert.Equal(t, int64(0), GlobalStats().errors)
+ assert.Equal(t, int64(1), st.errors)
+ assert.True(t, fserrors.IsCounted(err))
+ })
+
+}
+
func percentDiff(start, end uint64) uint64 {
return (start - end) * 100 / start
}
diff --git a/fs/cache/cache.go b/fs/cache/cache.go
index e2c104395..63afa94e8 100644
--- a/fs/cache/cache.go
+++ b/fs/cache/cache.go
@@ -29,7 +29,7 @@ func createOnFirstUse() {
c.SetExpireInterval(ci.FsCacheExpireInterval)
c.SetFinalizer(func(value interface{}) {
if s, ok := value.(fs.Shutdowner); ok {
- _ = fs.CountError(s.Shutdown(context.Background()))
+ _ = fs.CountError(context.Background(), s.Shutdown(context.Background()))
}
})
})
diff --git a/fs/config.go b/fs/config.go
index caf242fdf..1b11da821 100644
--- a/fs/config.go
+++ b/fs/config.go
@@ -41,7 +41,7 @@ var (
//
// This is a function pointer to decouple the config
// implementation from the fs
- CountError = func(err error) error { return err }
+ CountError = func(ctx context.Context, err error) error { return err }
// ConfigProvider is the config key used for provider options
ConfigProvider = "provider"
diff --git a/fs/march/march.go b/fs/march/march.go
index 856be0e7c..1116220db 100644
--- a/fs/march/march.go
+++ b/fs/march/march.go
@@ -415,7 +415,7 @@ func (m *March) processJob(job listDirJob) ([]listDirJob, error) {
} else {
fs.Errorf(m.Fsrc, "error reading source root directory: %v", srcListErr)
}
- srcListErr = fs.CountError(srcListErr)
+ srcListErr = fs.CountError(m.Ctx, srcListErr)
return nil, srcListErr
}
if dstListErr == fs.ErrorDirNotFound {
@@ -426,7 +426,7 @@ func (m *March) processJob(job listDirJob) ([]listDirJob, error) {
} else {
fs.Errorf(m.Fdst, "error reading destination root directory: %v", dstListErr)
}
- dstListErr = fs.CountError(dstListErr)
+ dstListErr = fs.CountError(m.Ctx, dstListErr)
return nil, dstListErr
}
diff --git a/fs/operations/check.go b/fs/operations/check.go
index b9fe36aab..c3c747071 100644
--- a/fs/operations/check.go
+++ b/fs/operations/check.go
@@ -49,6 +49,7 @@ type CheckOpt struct {
// checkMarch is used to march over two Fses in the same way as
// sync/copy
type checkMarch struct {
+ ctx context.Context
ioMu sync.Mutex
wg sync.WaitGroup
tokens chan struct{}
@@ -83,7 +84,7 @@ func (c *checkMarch) DstOnly(dst fs.DirEntry) (recurse bool) {
}
err := fmt.Errorf("file not in %v", c.opt.Fsrc)
fs.Errorf(dst, "%v", err)
- _ = fs.CountError(err)
+ _ = fs.CountError(c.ctx, err)
c.differences.Add(1)
c.srcFilesMissing.Add(1)
c.report(dst, c.opt.MissingOnSrc, '-')
@@ -105,7 +106,7 @@ func (c *checkMarch) SrcOnly(src fs.DirEntry) (recurse bool) {
case fs.Object:
err := fmt.Errorf("file not in %v", c.opt.Fdst)
fs.Errorf(src, "%v", err)
- _ = fs.CountError(err)
+ _ = fs.CountError(c.ctx, err)
c.differences.Add(1)
c.dstFilesMissing.Add(1)
c.report(src, c.opt.MissingOnDst, '+')
@@ -155,13 +156,13 @@ func (c *checkMarch) Match(ctx context.Context, dst, src fs.DirEntry) (recurse b
differ, noHash, err := c.checkIdentical(ctx, dstX, srcX)
if err != nil {
fs.Errorf(src, "%v", err)
- _ = fs.CountError(err)
+ _ = fs.CountError(ctx, err)
c.report(src, c.opt.Error, '!')
} else if differ {
c.differences.Add(1)
err := errors.New("files differ")
// the checkFn has already logged the reason
- _ = fs.CountError(err)
+ _ = fs.CountError(ctx, err)
c.report(src, c.opt.Differ, '*')
} else {
c.matches.Add(1)
@@ -177,7 +178,7 @@ func (c *checkMarch) Match(ctx context.Context, dst, src fs.DirEntry) (recurse b
} else {
err := fmt.Errorf("is file on %v but directory on %v", c.opt.Fsrc, c.opt.Fdst)
fs.Errorf(src, "%v", err)
- _ = fs.CountError(err)
+ _ = fs.CountError(ctx, err)
c.differences.Add(1)
c.dstFilesMissing.Add(1)
c.report(src, c.opt.MissingOnDst, '+')
@@ -190,7 +191,7 @@ func (c *checkMarch) Match(ctx context.Context, dst, src fs.DirEntry) (recurse b
}
err := fmt.Errorf("is file on %v but directory on %v", c.opt.Fdst, c.opt.Fsrc)
fs.Errorf(dst, "%v", err)
- _ = fs.CountError(err)
+ _ = fs.CountError(ctx, err)
c.differences.Add(1)
c.srcFilesMissing.Add(1)
c.report(dst, c.opt.MissingOnSrc, '-')
@@ -214,6 +215,7 @@ func CheckFn(ctx context.Context, opt *CheckOpt) error {
return errors.New("internal error: nil check function")
}
c := &checkMarch{
+ ctx: ctx,
tokens: make(chan struct{}, ci.Checkers),
opt: *opt,
}
@@ -430,6 +432,7 @@ func CheckSum(ctx context.Context, fsrc, fsum fs.Fs, sumFile string, hashType ha
ci := fs.GetConfig(ctx)
c := &checkMarch{
+ ctx: ctx,
tokens: make(chan struct{}, ci.Checkers),
opt: *opt,
}
@@ -450,7 +453,7 @@ func CheckSum(ctx context.Context, fsrc, fsum fs.Fs, sumFile string, hashType ha
// filesystem missed the file, sum wasn't consumed
err := fmt.Errorf("file not in %v", opt.Fdst)
fs.Errorf(filename, "%v", err)
- _ = fs.CountError(err)
+ _ = fs.CountError(ctx, err)
if lastErr == nil {
lastErr = err
}
@@ -479,7 +482,7 @@ func (c *checkMarch) checkSum(ctx context.Context, obj fs.Object, download bool,
if !sumFound {
err = errors.New("sum not found")
- _ = fs.CountError(err)
+ _ = fs.CountError(ctx, err)
fs.Errorf(obj, "%v", err)
c.differences.Add(1)
c.srcFilesMissing.Add(1)
@@ -528,12 +531,12 @@ func (c *checkMarch) checkSum(ctx context.Context, obj fs.Object, download bool,
func (c *checkMarch) matchSum(ctx context.Context, sumHash, objHash string, obj fs.Object, err error, hashType hash.Type) {
switch {
case err != nil:
- _ = fs.CountError(err)
+ _ = fs.CountError(ctx, err)
fs.Errorf(obj, "Failed to calculate hash: %v", err)
c.report(obj, c.opt.Error, '!')
case sumHash == "":
err = errors.New("duplicate file")
- _ = fs.CountError(err)
+ _ = fs.CountError(ctx, err)
fs.Errorf(obj, "%v", err)
c.report(obj, c.opt.Error, '!')
case objHash == "":
@@ -548,7 +551,7 @@ func (c *checkMarch) matchSum(ctx context.Context, sumHash, objHash string, obj
c.report(obj, c.opt.Match, '=')
default:
err = errors.New("files differ")
- _ = fs.CountError(err)
+ _ = fs.CountError(ctx, err)
fs.Debugf(nil, "%v = %s (sum)", hashType, sumHash)
fs.Debugf(obj, "%v = %s (%v)", hashType, objHash, c.opt.Fdst)
fs.Errorf(obj, "%v", err)
diff --git a/fs/operations/copy.go b/fs/operations/copy.go
index 4e2f09762..8228a14c7 100644
--- a/fs/operations/copy.go
+++ b/fs/operations/copy.go
@@ -331,7 +331,7 @@ func (c *copy) copy(ctx context.Context) (newDst fs.Object, err error) {
}
}
if err != nil {
- err = fs.CountError(err)
+ err = fs.CountError(ctx, err)
fs.Errorf(c.src, "Failed to copy: %v", err)
if !c.inplace {
c.removeFailedPartialCopy(ctx, c.f, c.remoteForCopy)
@@ -343,7 +343,7 @@ func (c *copy) copy(ctx context.Context) (newDst fs.Object, err error) {
err = c.verify(ctx, newDst)
if err != nil {
fs.Errorf(newDst, "%v", err)
- err = fs.CountError(err)
+ err = fs.CountError(ctx, err)
c.removeFailedCopy(ctx, newDst)
return nil, err
}
@@ -353,7 +353,7 @@ func (c *copy) copy(ctx context.Context) (newDst fs.Object, err error) {
movedNewDst, err := c.dstFeatures.Move(ctx, newDst, c.remote)
if err != nil {
fs.Errorf(newDst, "partial file rename failed: %v", err)
- err = fs.CountError(err)
+ err = fs.CountError(ctx, err)
c.removeFailedCopy(ctx, newDst)
return nil, err
}
diff --git a/fs/operations/dedupe.go b/fs/operations/dedupe.go
index 3e0f72cd5..60c66c4a3 100644
--- a/fs/operations/dedupe.go
+++ b/fs/operations/dedupe.go
@@ -32,7 +32,7 @@ outer:
_, err := f.NewObject(ctx, newName)
for ; err != fs.ErrorObjectNotFound; suffix++ {
if err != nil {
- err = fs.CountError(err)
+ err = fs.CountError(ctx, err)
fs.Errorf(o, "Failed to check for existing object: %v", err)
continue outer
}
@@ -46,7 +46,7 @@ outer:
if !SkipDestructive(ctx, o, "rename") {
newObj, err := doMove(ctx, o, newName)
if err != nil {
- err = fs.CountError(err)
+ err = fs.CountError(ctx, err)
fs.Errorf(o, "Failed to rename: %v", err)
continue
}
@@ -374,7 +374,7 @@ func dedupeMergeDuplicateDirs(ctx context.Context, f fs.Fs, duplicateDirs [][]*d
fs.Infof(fsDirs[0], "Merging contents of duplicate directories")
err := mergeDirs(ctx, fsDirs)
if err != nil {
- err = fs.CountError(err)
+ err = fs.CountError(ctx, err)
fs.Errorf(nil, "merge duplicate dirs: %v", err)
}
}
diff --git a/fs/operations/operations.go b/fs/operations/operations.go
index f14ecb61e..53d846525 100644
--- a/fs/operations/operations.go
+++ b/fs/operations/operations.go
@@ -101,11 +101,11 @@ func checkHashes(ctx context.Context, src fs.ObjectInfo, dst fs.Object, ht hash.
return true, hash.None, srcHash, dstHash, nil
}
if srcErr != nil {
- err = fs.CountError(srcErr)
+ err = fs.CountError(ctx, srcErr)
fs.Errorf(src, "Failed to calculate src hash: %v", err)
}
if dstErr != nil {
- err = fs.CountError(dstErr)
+ err = fs.CountError(ctx, dstErr)
fs.Errorf(dst, "Failed to calculate dst hash: %v", err)
}
if err != nil {
@@ -340,7 +340,7 @@ func equal(ctx context.Context, src fs.ObjectInfo, dst fs.Object, opt equalOpt)
logger(ctx, Differ, src, dst, nil)
return false
} else if err != nil {
- err = fs.CountError(err)
+ err = fs.CountError(ctx, err)
fs.Errorf(dst, "Failed to set modification time: %v", err)
} else {
fs.Infof(src, "Updated modification time in destination")
@@ -481,7 +481,7 @@ func move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.
fs.Debugf(src, "Can't move, switching to copy")
_ = in.Close()
default:
- err = fs.CountError(err)
+ err = fs.CountError(ctx, err)
fs.Errorf(src, "Couldn't move: %v", err)
_ = in.Close()
return newDst, err
@@ -566,7 +566,7 @@ func DeleteFileWithBackupDir(ctx context.Context, dst fs.Object, backupDir fs.Fs
}
if err != nil {
fs.Errorf(dst, "Couldn't %s: %v", action, err)
- err = fs.CountError(err)
+ err = fs.CountError(ctx, err)
} else if !skip {
fs.Infof(dst, "%s", actioned)
}
@@ -974,7 +974,7 @@ func HashLister(ctx context.Context, ht hash.Type, outputBase64 bool, downloadFl
}()
sum, err := HashSum(ctx, ht, outputBase64, downloadFlag, o)
if err != nil {
- fs.Errorf(o, "%v", fs.CountError(err))
+ fs.Errorf(o, "%v", fs.CountError(ctx, err))
return
}
SyncFprintf(w, "%*s %s\n", width, sum, o.Remote())
@@ -1053,7 +1053,7 @@ func Mkdir(ctx context.Context, f fs.Fs, dir string) error {
fs.Debugf(fs.LogDirName(f, dir), "Making directory")
err := f.Mkdir(ctx, dir)
if err != nil {
- err = fs.CountError(err)
+ err = fs.CountError(ctx, err)
return err
}
return nil
@@ -1075,7 +1075,7 @@ func MkdirMetadata(ctx context.Context, f fs.Fs, dir string, metadata fs.Metadat
fs.Debugf(fs.LogDirName(f, dir), "Making directory with metadata")
newDst, err = do(ctx, dir, metadata)
if err != nil {
- err = fs.CountError(err)
+ err = fs.CountError(ctx, err)
return nil, err
}
if mtime, ok := metadata["mtime"]; ok {
@@ -1133,7 +1133,7 @@ func TryRmdir(ctx context.Context, f fs.Fs, dir string) error {
func Rmdir(ctx context.Context, f fs.Fs, dir string) error {
err := TryRmdir(ctx, f, dir)
if err != nil {
- err = fs.CountError(err)
+ err = fs.CountError(ctx, err)
return err
}
return err
@@ -1162,7 +1162,7 @@ func Purge(ctx context.Context, f fs.Fs, dir string) (err error) {
err = Rmdirs(ctx, f, dir, false)
}
if err != nil {
- err = fs.CountError(err)
+ err = fs.CountError(ctx, err)
return err
}
return nil
@@ -1207,7 +1207,7 @@ func listToChan(ctx context.Context, f fs.Fs, dir string) fs.ObjectsChan {
})
if err != nil && err != fs.ErrorDirNotFound {
err = fmt.Errorf("failed to list: %w", err)
- err = fs.CountError(err)
+ err = fs.CountError(ctx, err)
fs.Errorf(nil, "%v", err)
}
}()
@@ -1267,7 +1267,7 @@ func Cat(ctx context.Context, f fs.Fs, w io.Writer, offset, count int64, sep []b
var in io.ReadCloser
in, err = Open(ctx, o, options...)
if err != nil {
- err = fs.CountError(err)
+ err = fs.CountError(ctx, err)
fs.Errorf(o, "Failed to open: %v", err)
return
}
@@ -1280,13 +1280,13 @@ func Cat(ctx context.Context, f fs.Fs, w io.Writer, offset, count int64, sep []b
defer mu.Unlock()
_, err = io.Copy(w, in)
if err != nil {
- err = fs.CountError(err)
+ err = fs.CountError(ctx, err)
fs.Errorf(o, "Failed to send to output: %v", err)
}
if len(sep) > 0 {
_, err = w.Write(sep)
if err != nil {
- err = fs.CountError(err)
+ err = fs.CountError(ctx, err)
fs.Errorf(o, "Failed to send separator to output: %v", err)
}
}
@@ -1423,7 +1423,7 @@ func rcatSrc(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadClos
src := object.NewStaticObjectInfo(dstFileName, modTime, int64(readCounter.BytesRead()), false, sums, fdst).WithMetadata(meta)
if !equal(ctx, src, dst, opt) {
err = fmt.Errorf("corrupted on transfer")
- err = fs.CountError(err)
+ err = fs.CountError(ctx, err)
fs.Errorf(dst, "%v", err)
return dst, err
}
@@ -1450,7 +1450,7 @@ func Rmdirs(ctx context.Context, f fs.Fs, dir string, leaveRoot bool) error {
dirEmpty[dir] = !leaveRoot
err := walk.Walk(ctx, f, dir, false, ci.MaxDepth, func(dirPath string, entries fs.DirEntries, err error) error {
if err != nil {
- err = fs.CountError(err)
+ err = fs.CountError(ctx, err)
fs.Errorf(f, "Failed to list %q: %v", dirPath, err)
return nil
}
@@ -1526,7 +1526,7 @@ func Rmdirs(ctx context.Context, f fs.Fs, dir string, leaveRoot bool) error {
g.Go(func() error {
err := TryRmdir(gCtx, f, dir)
if err != nil {
- err = fs.CountError(err)
+ err = fs.CountError(ctx, err)
fs.Errorf(dir, "Failed to rmdir: %v", err)
errCount.Add(err)
}
@@ -2096,7 +2096,7 @@ func TouchDir(ctx context.Context, f fs.Fs, remote string, t time.Time, recursiv
err := o.SetModTime(ctx, t)
if err != nil {
err = fmt.Errorf("failed to touch: %w", err)
- err = fs.CountError(err)
+ err = fs.CountError(ctx, err)
fs.Errorf(o, "%v", err)
}
}
diff --git a/fs/sync/sync.go b/fs/sync/sync.go
index 67998268f..e0a3e23ac 100644
--- a/fs/sync/sync.go
+++ b/fs/sync/sync.go
@@ -400,7 +400,7 @@ func (s *syncCopyMove) pairChecker(in *pipe, out *pipe, fraction int, wg *sync.W
if needTransfer {
// If files are treated as immutable, fail if destination exists and does not match
if s.ci.Immutable && pair.Dst != nil {
- err := fs.CountError(fserrors.NoRetryError(fs.ErrorImmutableModified))
+ err := fs.CountError(s.ctx, fserrors.NoRetryError(fs.ErrorImmutableModified))
fs.Errorf(pair.Dst, "Source and destination exist but do not match: %v", err)
s.processError(err)
} else {
@@ -1204,7 +1204,7 @@ func (s *syncCopyMove) setDelayedDirModTimes(ctx context.Context) error {
_, err = operations.SetDirModTime(gCtx, s.fdst, item.dst, item.dir, item.modTime)
}
if err != nil {
- err = fs.CountError(err)
+ err = fs.CountError(ctx, err)
fs.Errorf(item.dir, "Failed to update directory timestamp or metadata: %v", err)
errCount.Add(err)
}
@@ -1399,7 +1399,7 @@ func MoveDir(ctx context.Context, fdst, fsrc fs.Fs, deleteEmptySrcDirs bool, cop
fs.Infof(fdst, "Server side directory move succeeded")
return nil
default:
- err = fs.CountError(err)
+ err = fs.CountError(ctx, err)
fs.Errorf(fdst, "Server side directory move failed: %v", err)
return err
}
diff --git a/fs/sync/sync_test.go b/fs/sync/sync_test.go
index 288616cd5..f4f1c556a 100644
--- a/fs/sync/sync_test.go
+++ b/fs/sync/sync_test.go
@@ -946,7 +946,7 @@ func TestSyncIgnoreErrors(t *testing.T) {
accounting.GlobalStats().ResetCounters()
ctx = predictDstFromLogger(ctx)
- _ = fs.CountError(errors.New("boom"))
+ _ = fs.CountError(ctx, errors.New("boom"))
assert.NoError(t, Sync(ctx, r.Fremote, r.Flocal, false))
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
@@ -1267,7 +1267,7 @@ func TestSyncAfterRemovingAFileAndAddingAFileSubDirWithErrors(t *testing.T) {
ctx = predictDstFromLogger(ctx)
accounting.GlobalStats().ResetCounters()
- _ = fs.CountError(errors.New("boom"))
+ _ = fs.CountError(ctx, errors.New("boom"))
err := Sync(ctx, r.Fremote, r.Flocal, false)
assert.Equal(t, fs.ErrorNotDeleting, err)
testLoggerVsLsf(ctx, r.Fremote, operations.GetLoggerOpt(ctx).JSON, t)
diff --git a/fs/walk/walk.go b/fs/walk/walk.go
index ada7a284d..51e567a4f 100644
--- a/fs/walk/walk.go
+++ b/fs/walk/walk.go
@@ -170,7 +170,7 @@ func listRwalk(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLe
// Carry on listing but return the error at the end
if err != nil {
listErr = err
- err = fs.CountError(err)
+ err = fs.CountError(ctx, err)
fs.Errorf(path, "error listing: %v", err)
return nil
}
@@ -415,7 +415,7 @@ func walk(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel i
// NB once we have passed entries to fn we mustn't touch it again
if err != nil && err != ErrorSkipDir {
traversing.Done()
- err = fs.CountError(err)
+ err = fs.CountError(ctx, err)
fs.Errorf(job.remote, "error listing: %v", err)
closeQuit()
// Send error to error channel if space
diff --git a/lib/http/serve/dir.go b/lib/http/serve/dir.go
index 664f81cf2..5a92ddeb8 100644
--- a/lib/http/serve/dir.go
+++ b/lib/http/serve/dir.go
@@ -2,6 +2,7 @@ package serve
import (
"bytes"
+ "context"
"fmt"
"html/template"
"net/http"
@@ -124,8 +125,8 @@ func (d *Directory) AddEntry(remote string, isDir bool) {
}
// Error logs the error and if a ResponseWriter is given it writes an http.StatusInternalServerError
-func Error(what interface{}, w http.ResponseWriter, text string, err error) {
- err = fs.CountError(err)
+func Error(ctx context.Context, what interface{}, w http.ResponseWriter, text string, err error) {
+ err = fs.CountError(ctx, err)
fs.Errorf(what, "%s: %v", text, err)
if w != nil {
http.Error(w, text+".", http.StatusInternalServerError)
@@ -223,6 +224,7 @@ const (
// Serve serves a directory
func (d *Directory) Serve(w http.ResponseWriter, r *http.Request) {
+ ctx := r.Context()
// Account the transfer
tr := accounting.Stats(r.Context()).NewTransferRemoteSize(d.DirRemote, -1, nil, nil)
defer tr.Done(r.Context(), nil)
@@ -232,12 +234,12 @@ func (d *Directory) Serve(w http.ResponseWriter, r *http.Request) {
buf := &bytes.Buffer{}
err := d.HTMLTemplate.Execute(buf, d)
if err != nil {
- Error(d.DirRemote, w, "Failed to render template", err)
+ Error(ctx, d.DirRemote, w, "Failed to render template", err)
return
}
w.Header().Set("Content-Length", fmt.Sprintf("%d", buf.Len()))
_, err = buf.WriteTo(w)
if err != nil {
- Error(d.DirRemote, nil, "Failed to drain template buffer", err)
+ Error(ctx, d.DirRemote, nil, "Failed to drain template buffer", err)
}
}
diff --git a/lib/http/serve/dir_test.go b/lib/http/serve/dir_test.go
index 7bedc0471..19ab22f57 100644
--- a/lib/http/serve/dir_test.go
+++ b/lib/http/serve/dir_test.go
@@ -1,6 +1,7 @@
package serve
import (
+ "context"
"errors"
"html/template"
"io"
@@ -88,9 +89,10 @@ func TestAddEntry(t *testing.T) {
}
func TestError(t *testing.T) {
+ ctx := context.Background()
w := httptest.NewRecorder()
err := errors.New("help")
- Error("potato", w, "sausage", err)
+ Error(ctx, "potato", w, "sausage", err)
resp := w.Result()
assert.Equal(t, http.StatusInternalServerError, resp.StatusCode)
body, _ := io.ReadAll(resp.Body)