"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
accounting: Add listed stat for number of directory entries listed
Nick Craig-Woodbff229713a34
diff --git a/fs/accounting/prometheus.go b/fs/accounting/prometheus.goindex cad792804..5ae07b749 100644--- a/fs/accounting/prometheus.go+++ b/fs/accounting/prometheus.go@@ -19,6 +19,7 @@ type RcloneCollector struct {deletes *prometheus.DescdeletedDirs *prometheus.Descrenames *prometheus.Desc+ listed *prometheus.DescfatalError *prometheus.DescretryError *prometheus.Desc}@@ -59,6 +60,10 @@ func NewRcloneCollector(ctx context.Context) *RcloneCollector {"Total number of files renamed",nil, nil,),+ listed: prometheus.NewDesc(namespace+"entries_listed_total",+ "Total number of entries listed",+ nil, nil,+ ),fatalError: prometheus.NewDesc(namespace+"fatal_error","Whether a fatal error has occurred",nil, nil,@@ -80,6 +85,7 @@ func (c *RcloneCollector) Describe(ch chan<- *prometheus.Desc) {ch <- c.deletesch <- c.deletedDirsch <- c.renames+ ch <- c.listedch <- c.fatalErrorch <- c.retryError}@@ -97,6 +103,7 @@ func (c *RcloneCollector) Collect(ch chan<- prometheus.Metric) {ch <- prometheus.MustNewConstMetric(c.deletes, prometheus.CounterValue, float64(s.deletes))ch <- prometheus.MustNewConstMetric(c.deletedDirs, prometheus.CounterValue, float64(s.deletedDirs))ch <- prometheus.MustNewConstMetric(c.renames, prometheus.CounterValue, float64(s.renames))+ ch <- prometheus.MustNewConstMetric(c.listed, prometheus.CounterValue, float64(s.listed))ch <- prometheus.MustNewConstMetric(c.fatalError, prometheus.GaugeValue, bool2Float(s.fatalError))ch <- prometheus.MustNewConstMetric(c.retryError, prometheus.GaugeValue, bool2Float(s.retryError))diff --git a/fs/accounting/stats.go b/fs/accounting/stats.goindex c50ef77b0..18249e300 100644--- a/fs/accounting/stats.go+++ b/fs/accounting/stats.go@@ -46,6 +46,7 @@ type StatsInfo struct {transferring *transferMaptransferQueue inttransferQueueSize int64+ listed int64renames int64renameQueue intrenameQueueSize int64@@ -117,6 +118,7 @@ func (s *StatsInfo) RemoteStats(short bool) (out rc.Params, err error) {out["deletes"] = s.deletesout["deletedDirs"] = s.deletedDirsout["renames"] = s.renames+ out["listed"] = s.listedout["elapsedTime"] = time.Since(s.startTime).Seconds()out["serverSideCopies"] = s.serverSideCopiesout["serverSideCopyBytes"] = s.serverSideCopyBytes@@ -500,9 +502,9 @@ func (s *StatsInfo) String() string {_, _ = fmt.Fprintf(buf, "Errors: %10d%s\n",s.errors, errorDetails)}- if s.checks != 0 || ts.totalChecks != 0 {- _, _ = fmt.Fprintf(buf, "Checks: %10d / %d, %s\n",- s.checks, ts.totalChecks, percent(s.checks, ts.totalChecks))+ if s.checks != 0 || ts.totalChecks != 0 || s.listed != 0 {+ _, _ = fmt.Fprintf(buf, "Checks: %10d / %d, %s, Listed %d\n",+ s.checks, ts.totalChecks, percent(s.checks, ts.totalChecks), s.listed)}if s.deletes != 0 || s.deletedDirs != 0 {_, _ = fmt.Fprintf(buf, "Deleted: %10d (files), %d (dirs), %s (freed)\n", s.deletes, s.deletedDirs, fs.SizeSuffix(s.deletesSize).ByteUnit())@@ -718,7 +720,15 @@ func (s *StatsInfo) Renames(renames int64) int64 {return s.renames}-// ResetCounters sets the counters (bytes, checks, errors, transfers, deletes, renames) to 0 and resets lastError, fatalError and retryError+// Listed updates the stats for listed objects+func (s *StatsInfo) Listed(listed int64) int64 {+ s.mu.Lock()+ defer s.mu.Unlock()+ s.listed += listed+ return s.listed+}++// ResetCounters sets the counters (bytes, checks, errors, transfers, deletes, renames, listed) to 0 and resets lastError, fatalError and retryErrorfunc (s *StatsInfo) ResetCounters() {s.mu.Lock()defer s.mu.Unlock()@@ -734,6 +744,7 @@ func (s *StatsInfo) ResetCounters() {s.deletesSize = 0s.deletedDirs = 0s.renames = 0+ s.listed = 0s.startedTransfers = nils.oldDuration = 0diff --git a/fs/accounting/stats_groups.go b/fs/accounting/stats_groups.goindex 13236aa52..3c971d831 100644--- a/fs/accounting/stats_groups.go+++ b/fs/accounting/stats_groups.go@@ -96,6 +96,7 @@ Returns the following values:"fatalError": boolean whether there has been at least one fatal error,"lastError": last error string,"renames" : number of files renamed,+ "listed" : number of directory entries listed,"retryError": boolean showing whether there has been at least one non-NoRetryError,"serverSideCopies": number of server side copies done,"serverSideCopyBytes": number bytes server side copied,@@ -383,6 +384,7 @@ func (sg *statsGroups) sum(ctx context.Context) *StatsInfo {sum.transfers += stats.transferssum.transferring.merge(stats.transferring)sum.transferQueueSize += stats.transferQueueSize+ sum.listed += stats.listedsum.renames += stats.renamessum.renameQueue += stats.renameQueuesum.renameQueueSize += stats.renameQueueSizediff --git a/fs/list/list.go b/fs/list/list.goindex 50613779d..edd56f889 100644--- a/fs/list/list.go+++ b/fs/list/list.go@@ -8,6 +8,7 @@ import ("strings""github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/accounting""github.com/rclone/rclone/fs/filter""github.com/rclone/rclone/lib/bucket")@@ -23,6 +24,7 @@ import (func DirSorted(ctx context.Context, f fs.Fs, includeAll bool, dir string) (entries fs.DirEntries, err error) {// Get unfiltered entries from the fsentries, err = f.List(ctx, dir)+ accounting.Stats(ctx).Listed(int64(len(entries)))if err != nil {return nil, err}diff --git a/fs/operations/lsjson.go b/fs/operations/lsjson.goindex 94d405e2f..8e714efbf 100644--- a/fs/operations/lsjson.go+++ b/fs/operations/lsjson.go@@ -10,6 +10,7 @@ import ("github.com/rclone/rclone/backend/crypt""github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/accounting""github.com/rclone/rclone/fs/hash""github.com/rclone/rclone/fs/walk")@@ -283,7 +284,8 @@ func StatJSON(ctx context.Context, fsrc fs.Fs, remote string, opt *ListJSONOpt)return nil, nil}// Check the root directory exists- _, err := fsrc.List(ctx, "")+ entries, err := fsrc.List(ctx, "")+ accounting.Stats(ctx).Listed(int64(len(entries)))if err != nil {return nil, err}@@ -322,6 +324,7 @@ func StatJSON(ctx context.Context, fsrc fs.Fs, remote string, opt *ListJSONOpt)parent = ""}entries, err := fsrc.List(ctx, parent)+ accounting.Stats(ctx).Listed(int64(len(entries)))if err == fs.ErrorDirNotFound {return nil, nil} else if err != nil {diff --git a/fs/walk/walk.go b/fs/walk/walk.goindex ab157e3a1..5d64f97d3 100644--- a/fs/walk/walk.go+++ b/fs/walk/walk.go@@ -12,6 +12,7 @@ import ("time""github.com/rclone/rclone/fs"+ "github.com/rclone/rclone/fs/accounting""github.com/rclone/rclone/fs/dirtree""github.com/rclone/rclone/fs/filter""github.com/rclone/rclone/fs/list"@@ -296,6 +297,7 @@ func listR(ctx context.Context, f fs.Fs, path string, includeAll bool, listType}var mu sync.Mutexerr := doListR(ctx, path, func(entries fs.DirEntries) (err error) {+ accounting.Stats(ctx).Listed(int64(len(entries)))if synthesizeDirs {err = dm.addEntries(entries)if err != nil {@@ -465,6 +467,7 @@ func walkRDirTree(ctx context.Context, f fs.Fs, startPath string, includeAll booincludeDirectory := fi.IncludeDirectory(ctx, f)var mu sync.Mutexerr := listR(ctx, startPath, func(entries fs.DirEntries) error {+ accounting.Stats(ctx).Listed(int64(len(entries)))mu.Lock()defer mu.Unlock()for _, entry := range entries {