Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

accounting: Add listed stat for number of directory entries listed

Nick Craig-Woodbff229713a34
diff --git a/fs/accounting/prometheus.go b/fs/accounting/prometheus.go
index cad792804..5ae07b749 100644
--- a/fs/accounting/prometheus.go
+++ b/fs/accounting/prometheus.go
@@ -19,6 +19,7 @@ type RcloneCollector struct {
deletes *prometheus.Desc
deletedDirs *prometheus.Desc
renames *prometheus.Desc
+ listed *prometheus.Desc
fatalError *prometheus.Desc
retryError *prometheus.Desc
}
@@ -59,6 +60,10 @@ func NewRcloneCollector(ctx context.Context) *RcloneCollector {
"Total number of files renamed",
nil, nil,
),
+ listed: prometheus.NewDesc(namespace+"entries_listed_total",
+ "Total number of entries listed",
+ nil, nil,
+ ),
fatalError: prometheus.NewDesc(namespace+"fatal_error",
"Whether a fatal error has occurred",
nil, nil,
@@ -80,6 +85,7 @@ func (c *RcloneCollector) Describe(ch chan<- *prometheus.Desc) {
ch <- c.deletes
ch <- c.deletedDirs
ch <- c.renames
+ ch <- c.listed
ch <- c.fatalError
ch <- c.retryError
}
@@ -97,6 +103,7 @@ func (c *RcloneCollector) Collect(ch chan<- prometheus.Metric) {
ch <- prometheus.MustNewConstMetric(c.deletes, prometheus.CounterValue, float64(s.deletes))
ch <- prometheus.MustNewConstMetric(c.deletedDirs, prometheus.CounterValue, float64(s.deletedDirs))
ch <- prometheus.MustNewConstMetric(c.renames, prometheus.CounterValue, float64(s.renames))
+ ch <- prometheus.MustNewConstMetric(c.listed, prometheus.CounterValue, float64(s.listed))
ch <- prometheus.MustNewConstMetric(c.fatalError, prometheus.GaugeValue, bool2Float(s.fatalError))
ch <- prometheus.MustNewConstMetric(c.retryError, prometheus.GaugeValue, bool2Float(s.retryError))
diff --git a/fs/accounting/stats.go b/fs/accounting/stats.go
index c50ef77b0..18249e300 100644
--- a/fs/accounting/stats.go
+++ b/fs/accounting/stats.go
@@ -46,6 +46,7 @@ type StatsInfo struct {
transferring *transferMap
transferQueue int
transferQueueSize int64
+ listed int64
renames int64
renameQueue int
renameQueueSize int64
@@ -117,6 +118,7 @@ func (s *StatsInfo) RemoteStats(short bool) (out rc.Params, err error) {
out["deletes"] = s.deletes
out["deletedDirs"] = s.deletedDirs
out["renames"] = s.renames
+ out["listed"] = s.listed
out["elapsedTime"] = time.Since(s.startTime).Seconds()
out["serverSideCopies"] = s.serverSideCopies
out["serverSideCopyBytes"] = s.serverSideCopyBytes
@@ -500,9 +502,9 @@ func (s *StatsInfo) String() string {
_, _ = fmt.Fprintf(buf, "Errors: %10d%s\n",
s.errors, errorDetails)
}
- if s.checks != 0 || ts.totalChecks != 0 {
- _, _ = fmt.Fprintf(buf, "Checks: %10d / %d, %s\n",
- s.checks, ts.totalChecks, percent(s.checks, ts.totalChecks))
+ if s.checks != 0 || ts.totalChecks != 0 || s.listed != 0 {
+ _, _ = fmt.Fprintf(buf, "Checks: %10d / %d, %s, Listed %d\n",
+ s.checks, ts.totalChecks, percent(s.checks, ts.totalChecks), s.listed)
}
if s.deletes != 0 || s.deletedDirs != 0 {
_, _ = fmt.Fprintf(buf, "Deleted: %10d (files), %d (dirs), %s (freed)\n", s.deletes, s.deletedDirs, fs.SizeSuffix(s.deletesSize).ByteUnit())
@@ -718,7 +720,15 @@ func (s *StatsInfo) Renames(renames int64) int64 {
return s.renames
}
-// ResetCounters sets the counters (bytes, checks, errors, transfers, deletes, renames) to 0 and resets lastError, fatalError and retryError
+// Listed updates the stats for listed objects
+func (s *StatsInfo) Listed(listed int64) int64 {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ s.listed += listed
+ return s.listed
+}
+
+// ResetCounters sets the counters (bytes, checks, errors, transfers, deletes, renames, listed) to 0 and resets lastError, fatalError and retryError
func (s *StatsInfo) ResetCounters() {
s.mu.Lock()
defer s.mu.Unlock()
@@ -734,6 +744,7 @@ func (s *StatsInfo) ResetCounters() {
s.deletesSize = 0
s.deletedDirs = 0
s.renames = 0
+ s.listed = 0
s.startedTransfers = nil
s.oldDuration = 0
diff --git a/fs/accounting/stats_groups.go b/fs/accounting/stats_groups.go
index 13236aa52..3c971d831 100644
--- a/fs/accounting/stats_groups.go
+++ b/fs/accounting/stats_groups.go
@@ -96,6 +96,7 @@ Returns the following values:
"fatalError": boolean whether there has been at least one fatal error,
"lastError": last error string,
"renames" : number of files renamed,
+ "listed" : number of directory entries listed,
"retryError": boolean showing whether there has been at least one non-NoRetryError,
"serverSideCopies": number of server side copies done,
"serverSideCopyBytes": number bytes server side copied,
@@ -383,6 +384,7 @@ func (sg *statsGroups) sum(ctx context.Context) *StatsInfo {
sum.transfers += stats.transfers
sum.transferring.merge(stats.transferring)
sum.transferQueueSize += stats.transferQueueSize
+ sum.listed += stats.listed
sum.renames += stats.renames
sum.renameQueue += stats.renameQueue
sum.renameQueueSize += stats.renameQueueSize
diff --git a/fs/list/list.go b/fs/list/list.go
index 50613779d..edd56f889 100644
--- a/fs/list/list.go
+++ b/fs/list/list.go
@@ -8,6 +8,7 @@ import (
"strings"
"github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/fs/filter"
"github.com/rclone/rclone/lib/bucket"
)
@@ -23,6 +24,7 @@ import (
func DirSorted(ctx context.Context, f fs.Fs, includeAll bool, dir string) (entries fs.DirEntries, err error) {
// Get unfiltered entries from the fs
entries, err = f.List(ctx, dir)
+ accounting.Stats(ctx).Listed(int64(len(entries)))
if err != nil {
return nil, err
}
diff --git a/fs/operations/lsjson.go b/fs/operations/lsjson.go
index 94d405e2f..8e714efbf 100644
--- a/fs/operations/lsjson.go
+++ b/fs/operations/lsjson.go
@@ -10,6 +10,7 @@ import (
"github.com/rclone/rclone/backend/crypt"
"github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/fs/hash"
"github.com/rclone/rclone/fs/walk"
)
@@ -283,7 +284,8 @@ func StatJSON(ctx context.Context, fsrc fs.Fs, remote string, opt *ListJSONOpt)
return nil, nil
}
// Check the root directory exists
- _, err := fsrc.List(ctx, "")
+ entries, err := fsrc.List(ctx, "")
+ accounting.Stats(ctx).Listed(int64(len(entries)))
if err != nil {
return nil, err
}
@@ -322,6 +324,7 @@ func StatJSON(ctx context.Context, fsrc fs.Fs, remote string, opt *ListJSONOpt)
parent = ""
}
entries, err := fsrc.List(ctx, parent)
+ accounting.Stats(ctx).Listed(int64(len(entries)))
if err == fs.ErrorDirNotFound {
return nil, nil
} else if err != nil {
diff --git a/fs/walk/walk.go b/fs/walk/walk.go
index ab157e3a1..5d64f97d3 100644
--- a/fs/walk/walk.go
+++ b/fs/walk/walk.go
@@ -12,6 +12,7 @@ import (
"time"
"github.com/rclone/rclone/fs"
+ "github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/fs/dirtree"
"github.com/rclone/rclone/fs/filter"
"github.com/rclone/rclone/fs/list"
@@ -296,6 +297,7 @@ func listR(ctx context.Context, f fs.Fs, path string, includeAll bool, listType
}
var mu sync.Mutex
err := doListR(ctx, path, func(entries fs.DirEntries) (err error) {
+ accounting.Stats(ctx).Listed(int64(len(entries)))
if synthesizeDirs {
err = dm.addEntries(entries)
if err != nil {
@@ -465,6 +467,7 @@ func walkRDirTree(ctx context.Context, f fs.Fs, startPath string, includeAll boo
includeDirectory := fi.IncludeDirectory(ctx, f)
var mu sync.Mutex
err := listR(ctx, startPath, func(entries fs.DirEntries) error {
+ accounting.Stats(ctx).Listed(int64(len(entries)))
mu.Lock()
defer mu.Unlock()
for _, entry := range entries {