Skip to content

"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files

rcserver: implement prometheus metrics on a dedicated port - fixes #7940

Oleg Kunitsynd15704ef9f22
diff --git a/cmd/cmd.go b/cmd/cmd.go
index 2aca5341d..24e8e5160 100644
--- a/cmd/cmd.go
+++ b/cmd/cmd.go
@@ -421,11 +421,18 @@ func initConfig() {
}
// Start the remote control server if configured
- _, err = rcserver.Start(context.Background(), &rc.Opt)
+ _, err = rcserver.Start(ctx, &rc.Opt)
if err != nil {
log.Fatalf("Failed to start remote control: %v", err)
}
+ // Start the metrics server if configured
+ _, err = rcserver.MetricsStart(ctx, &rc.Opt)
+ if err != nil {
+ log.Fatalf("Failed to start metrics server: %v", err)
+
+ }
+
// Setup CPU profiling if desired
if *cpuProfile != "" {
fs.Infof(nil, "Creating CPU profile %q\n", *cpuProfile)
diff --git a/docs/content/docs.md b/docs/content/docs.md
index f8f5153d9..7f6e208e5 100644
--- a/docs/content/docs.md
+++ b/docs/content/docs.md
@@ -2836,6 +2836,17 @@ Rclone prefixes all log messages with their level in capitals, e.g. INFO
which makes it easy to grep the log file for different kinds of
information.
+Metrics
+-------
+
+Rclone can publish metrics in the OpenMetrics/Prometheus format.
+
+To enable the metrics endpoint, use the `--metrics-addr` flag. Metrics can also be published on the `--rc-addr` port if the `--rc` flag and `--rc-enable-metrics` flags are supplied or if using rclone rcd `--rc-enable-metrics`
+
+Rclone provides extensive configuration options for the metrics HTTP endpoint. These settings are grouped under the Metrics section and have a prefix `--metrics-*`.
+
+When metrics are enabled with `--rc-enable-metrics`, they will be published on the same port as the rc API. In this case, the `--metrics-*` flags will be ignored, and the HTTP endpoint configuration will be managed by the `--rc-*` parameters.
+
Exit Code
---------
diff --git a/docs/content/rc.md b/docs/content/rc.md
index 32e93867b..80c02a591 100644
--- a/docs/content/rc.md
+++ b/docs/content/rc.md
@@ -100,6 +100,7 @@ Default Off.
### --rc-enable-metrics
Enable OpenMetrics/Prometheus compatible endpoint at `/metrics`.
+If more control over the metrics is desired (for example running it on a different port or with different auth) then endpoint can be enabled with the `--metrics-*` flags instead.
Default Off.
diff --git a/fs/config/flags/flags.go b/fs/config/flags/flags.go
index db76f5d71..6389026c8 100644
--- a/fs/config/flags/flags.go
+++ b/fs/config/flags/flags.go
@@ -124,6 +124,7 @@ func init() {
All.NewGroup("Logging", "Flags for logging and statistics")
All.NewGroup("Metadata", "Flags to control metadata")
All.NewGroup("RC", "Flags to control the Remote Control API")
+ All.NewGroup("Metrics", "Flags to control the Metrics HTTP endpoint.")
}
// installFlag constructs a name from the flag passed in and
diff --git a/fs/rc/rc.go b/fs/rc/rc.go
index 55287ed6b..9c083f3b7 100644
--- a/fs/rc/rc.go
+++ b/fs/rc/rc.go
@@ -71,8 +71,8 @@ var OptionsInfo = fs.Options{{
}, {
Name: "rc_enable_metrics",
Default: false,
- Help: "Enable prometheus metrics on /metrics",
- Groups: "RC",
+ Help: "Enable the Prometheus metrics path at the remote control server",
+ Groups: "RC,Metrics",
}, {
Name: "rc_job_expire_duration",
Default: 60 * time.Second,
@@ -83,10 +83,18 @@ var OptionsInfo = fs.Options{{
Default: 10 * time.Second,
Help: "Interval to check for expired async jobs",
Groups: "RC",
+}, {
+ Name: "metrics_addr",
+ Default: []string{""},
+ Help: "IPaddress:Port or :Port to bind metrics server to",
+ Groups: "Metrics",
}}.
AddPrefix(libhttp.ConfigInfo, "rc", "RC").
AddPrefix(libhttp.AuthConfigInfo, "rc", "RC").
AddPrefix(libhttp.TemplateConfigInfo, "rc", "RC").
+ AddPrefix(libhttp.ConfigInfo, "metrics", "Metrics").
+ AddPrefix(libhttp.AuthConfigInfo, "metrics", "Metrics").
+ AddPrefix(libhttp.TemplateConfigInfo, "metrics", "Metrics").
SetDefault("rc_addr", []string{"localhost:5572"})
func init() {
@@ -109,6 +117,9 @@ type Options struct {
WebGUINoOpenBrowser bool `config:"rc_web_gui_no_open_browser"` // set to disable auto opening browser
WebGUIFetchURL string `config:"rc_web_fetch_url"` // set the default url for fetching webgui
EnableMetrics bool `config:"rc_enable_metrics"` // set to disable prometheus metrics on /metrics
+ MetricsHTTP libhttp.Config `config:"metrics"`
+ MetricsAuth libhttp.AuthConfig `config:"metrics"`
+ MetricsTemplate libhttp.TemplateConfig `config:"metrics"`
JobExpireDuration time.Duration `config:"rc_job_expire_duration"`
JobExpireInterval time.Duration `config:"rc_job_expire_interval"`
}
diff --git a/fs/rc/rcserver/metrics.go b/fs/rc/rcserver/metrics.go
new file mode 100644
index 000000000..502566f47
--- /dev/null
+++ b/fs/rc/rcserver/metrics.go
@@ -0,0 +1,97 @@
+// Package rcserver implements the HTTP endpoint to serve the remote control
+package rcserver
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/promhttp"
+ "github.com/rclone/rclone/fs/accounting"
+ "github.com/rclone/rclone/fs/fshttp"
+ "github.com/rclone/rclone/fs/rc"
+ "github.com/rclone/rclone/fs/rc/jobs"
+ libhttp "github.com/rclone/rclone/lib/http"
+)
+
+const path = "/metrics"
+
+var promHandlerFunc http.HandlerFunc
+
+func init() {
+ rcloneCollector := accounting.NewRcloneCollector(context.Background())
+ prometheus.MustRegister(rcloneCollector)
+
+ m := fshttp.NewMetrics("rclone")
+ for _, c := range m.Collectors() {
+ prometheus.MustRegister(c)
+ }
+ fshttp.DefaultMetrics = m
+
+ promHandlerFunc = promhttp.Handler().ServeHTTP
+}
+
+// MetricsStart the remote control server if configured
+//
+// If the server wasn't configured the *Server returned may be nil
+func MetricsStart(ctx context.Context, opt *rc.Options) (*MetricsServer, error) {
+ jobs.SetOpt(opt) // set the defaults for jobs
+ if opt.MetricsHTTP.ListenAddr[0] != "" {
+ // Serve on the DefaultServeMux so can have global registrations appear
+ s, err := newMetricsServer(ctx, opt)
+ if err != nil {
+ return nil, err
+ }
+ return s, s.Serve()
+ }
+ return nil, nil
+}
+
+// MetricsServer contains everything to run the rc server
+type MetricsServer struct {
+ ctx context.Context // for global config
+ server *libhttp.Server
+ promHandlerFunc http.Handler
+ opt *rc.Options
+}
+
+func newMetricsServer(ctx context.Context, opt *rc.Options) (*MetricsServer, error) {
+ s := &MetricsServer{
+ ctx: ctx,
+ opt: opt,
+ promHandlerFunc: promHandlerFunc,
+ }
+
+ var err error
+ s.server, err = libhttp.NewServer(ctx,
+ libhttp.WithConfig(opt.MetricsHTTP),
+ libhttp.WithAuth(opt.MetricsAuth),
+ libhttp.WithTemplate(opt.MetricsTemplate),
+ )
+ if err != nil {
+ return nil, fmt.Errorf("failed to init server: %w", err)
+ }
+
+ router := s.server.Router()
+ router.Get(path, promHandlerFunc)
+ return s, nil
+}
+
+// Serve runs the http server in the background.
+//
+// Use s.Close() and s.Wait() to shutdown server
+func (s *MetricsServer) Serve() error {
+ s.server.Serve()
+ return nil
+}
+
+// Wait blocks while the server is serving requests
+func (s *MetricsServer) Wait() {
+ s.server.Wait()
+}
+
+// Shutdown gracefully shuts down the server
+func (s *MetricsServer) Shutdown() error {
+ return s.server.Shutdown()
+}
diff --git a/fs/rc/rcserver/metrics_test.go b/fs/rc/rcserver/metrics_test.go
new file mode 100644
index 000000000..6141cc2a7
--- /dev/null
+++ b/fs/rc/rcserver/metrics_test.go
@@ -0,0 +1,88 @@
+package rcserver
+
+import (
+ "context"
+ "fmt"
+ "net/http"
+ "regexp"
+ "testing"
+
+ _ "github.com/rclone/rclone/backend/local"
+ "github.com/rclone/rclone/fs/accounting"
+ "github.com/rclone/rclone/fs/config/configfile"
+ "github.com/rclone/rclone/fs/rc"
+ "github.com/stretchr/testify/require"
+)
+
+// Run a suite of tests
+func testMetricsServer(t *testing.T, tests []testRun, opt *rc.Options) {
+ t.Helper()
+ ctx := context.Background()
+ configfile.Install()
+ rcServer, err := newMetricsServer(ctx, opt)
+ require.NoError(t, err)
+ testURL := rcServer.server.URLs()[0]
+ mux := rcServer.server.Router()
+ emulateCalls(t, tests, mux, testURL)
+}
+
+// return an enabled rc
+func newMetricsTestOpt() rc.Options {
+ opt := rc.Opt
+ opt.MetricsHTTP.ListenAddr = []string{testBindAddress}
+ return opt
+}
+
+func TestMetrics(t *testing.T) {
+ stats := accounting.GlobalStats()
+ tests := makeMetricsTestCases(stats)
+ opt := newMetricsTestOpt()
+ testMetricsServer(t, tests, &opt)
+
+ // Test changing a couple options
+ stats.Bytes(500)
+ for i := 0; i < 30; i++ {
+ require.NoError(t, stats.DeleteFile(context.Background(), 0))
+ }
+ stats.Errors(2)
+ stats.Bytes(324)
+
+ tests = makeMetricsTestCases(stats)
+ testMetricsServer(t, tests, &opt)
+}
+
+func makeMetricsTestCases(stats *accounting.StatsInfo) (tests []testRun) {
+ tests = []testRun{{
+ Name: "Bytes Transferred Metric",
+ URL: "metrics",
+ Method: "GET",
+ Status: http.StatusOK,
+ Contains: regexp.MustCompile(fmt.Sprintf("rclone_bytes_transferred_total %d", stats.GetBytes())),
+ }, {
+ Name: "Checked Files Metric",
+ URL: "metrics",
+ Method: "GET",
+ Status: http.StatusOK,
+ Contains: regexp.MustCompile(fmt.Sprintf("rclone_checked_files_total %d", stats.GetChecks())),
+ }, {
+ Name: "Errors Metric",
+ URL: "metrics",
+ Method: "GET",
+ Status: http.StatusOK,
+ Contains: regexp.MustCompile(fmt.Sprintf("rclone_errors_total %d", stats.GetErrors())),
+ }, {
+ Name: "Deleted Files Metric",
+ URL: "metrics",
+ Method: "GET",
+ Status: http.StatusOK,
+ Contains: regexp.MustCompile(fmt.Sprintf("rclone_files_deleted_total %d", stats.GetDeletes())),
+ }, {
+ Name: "Files Transferred Metric",
+ URL: "metrics",
+ Method: "GET",
+ Status: http.StatusOK,
+ Contains: regexp.MustCompile(fmt.Sprintf("rclone_files_transferred_total %d", stats.GetTransfers())),
+ },
+ }
+ return
+}
diff --git a/fs/rc/rcserver/rcserver.go b/fs/rc/rcserver/rcserver.go
index 88f82a484..6c91931c7 100644
--- a/fs/rc/rcserver/rcserver.go
+++ b/fs/rc/rcserver/rcserver.go
@@ -18,13 +18,9 @@ import (
"time"
"github.com/go-chi/chi/v5/middleware"
- "github.com/prometheus/client_golang/prometheus"
- "github.com/prometheus/client_golang/prometheus/promhttp"
"github.com/rclone/rclone/fs"
- "github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/fs/cache"
"github.com/rclone/rclone/fs/config"
- "github.com/rclone/rclone/fs/fshttp"
"github.com/rclone/rclone/fs/list"
"github.com/rclone/rclone/fs/rc"
"github.com/rclone/rclone/fs/rc/jobs"
@@ -35,21 +31,6 @@ import (
"github.com/skratchdot/open-golang/open"
)
-var promHandler http.Handler
-
-func init() {
- rcloneCollector := accounting.NewRcloneCollector(context.Background())
- prometheus.MustRegister(rcloneCollector)
-
- m := fshttp.NewMetrics("rclone")
- for _, c := range m.Collectors() {
- prometheus.MustRegister(c)
- }
- fshttp.DefaultMetrics = m
-
- promHandler = promhttp.Handler()
-}
-
// Start the remote control server if configured
//
// If the server wasn't configured the *Server returned may be nil
@@ -376,7 +357,7 @@ func (s *Server) handleGet(w http.ResponseWriter, r *http.Request, path string)
s.serveRemote(w, r, fsMatchResult[2], fsMatchResult[1])
return
case path == "metrics" && s.opt.EnableMetrics:
- promHandler.ServeHTTP(w, r)
+ promHandlerFunc(w, r)
return
case path == "*" && s.opt.Serve:
// Serve /* as the remote listing
diff --git a/fs/rc/rcserver/rcserver_test.go b/fs/rc/rcserver/rcserver_test.go
index 7b55f2e03..29738b614 100644
--- a/fs/rc/rcserver/rcserver_test.go
+++ b/fs/rc/rcserver/rcserver_test.go
@@ -15,9 +15,10 @@ import (
"testing"
"time"
+ "github.com/go-chi/chi/v5"
+
_ "github.com/rclone/rclone/backend/local"
"github.com/rclone/rclone/fs"
- "github.com/rclone/rclone/fs/accounting"
"github.com/rclone/rclone/fs/config/configfile"
"github.com/rclone/rclone/fs/rc"
"github.com/stretchr/testify/assert"
@@ -115,6 +116,10 @@ func testServer(t *testing.T, tests []testRun, opt *rc.Options) {
require.NoError(t, err)
testURL := rcServer.server.URLs()[0]
mux := rcServer.server.Router()
+ emulateCalls(t, tests, mux, testURL)
+}
+
+func emulateCalls(t *testing.T, tests []testRun, mux chi.Router, testURL string) {
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
t.Helper()
@@ -568,61 +573,6 @@ Unknown command
testServer(t, tests, &opt)
}
-func TestMetrics(t *testing.T) {
- stats := accounting.GlobalStats()
- tests := makeMetricsTestCases(stats)
- opt := newTestOpt()
- opt.EnableMetrics = true
- testServer(t, tests, &opt)
-
- // Test changing a couple options
- stats.Bytes(500)
- for i := 0; i < 30; i++ {
- require.NoError(t, stats.DeleteFile(context.Background(), 0))
- }
- stats.Errors(2)
- stats.Bytes(324)
-
- tests = makeMetricsTestCases(stats)
- testServer(t, tests, &opt)
-}
-
-func makeMetricsTestCases(stats *accounting.StatsInfo) (tests []testRun) {
- tests = []testRun{{
- Name: "Bytes Transferred Metric",
- URL: "/metrics",
- Method: "GET",
- Status: http.StatusOK,
- Contains: regexp.MustCompile(fmt.Sprintf("rclone_bytes_transferred_total %d", stats.GetBytes())),
- }, {
- Name: "Checked Files Metric",
- URL: "/metrics",
- Method: "GET",
- Status: http.StatusOK,
- Contains: regexp.MustCompile(fmt.Sprintf("rclone_checked_files_total %d", stats.GetChecks())),
- }, {
- Name: "Errors Metric",
- URL: "/metrics",
- Method: "GET",
- Status: http.StatusOK,
- Contains: regexp.MustCompile(fmt.Sprintf("rclone_errors_total %d", stats.GetErrors())),
- }, {
- Name: "Deleted Files Metric",
- URL: "/metrics",
- Method: "GET",
- Status: http.StatusOK,
- Contains: regexp.MustCompile(fmt.Sprintf("rclone_files_deleted_total %d", stats.GetDeletes())),
- }, {
- Name: "Files Transferred Metric",
- URL: "/metrics",
- Method: "GET",
- Status: http.StatusOK,
- Contains: regexp.MustCompile(fmt.Sprintf("rclone_files_transferred_total %d", stats.GetTransfers())),
- },
- }
- return
-}
-
var matchRemoteDirListing = regexp.MustCompile(`<title>Directory listing of /</title>`)
func TestServingRoot(t *testing.T) {