"rsync for cloud storage" - Google Drive, S3, Dropbox, Backblaze B2, One Drive, Swift, Hubic, Wasabi, Google Cloud Storage, Azure Blob, Azure Files, Yandex Files
Go azure-blobazure-blob-storageazure-filesbackblaze-b2cloud-storagedropboxencryptionftpfuse-filesystemgogolanggoogle-cloud-storagegoogle-driveonedriveopenstack-swiftrclones3sftpsyncwebdav
rcserver: implement prometheus metrics on a dedicated port - fixes #7940
Oleg Kunitsynd15704ef9f22
diff --git a/cmd/cmd.go b/cmd/cmd.goindex 2aca5341d..24e8e5160 100644--- a/cmd/cmd.go+++ b/cmd/cmd.go@@ -421,11 +421,18 @@ func initConfig() {}// Start the remote control server if configured- _, err = rcserver.Start(context.Background(), &rc.Opt)+ _, err = rcserver.Start(ctx, &rc.Opt)if err != nil {log.Fatalf("Failed to start remote control: %v", err)}+ // Start the metrics server if configured+ _, err = rcserver.MetricsStart(ctx, &rc.Opt)+ if err != nil {+ log.Fatalf("Failed to start metrics server: %v", err)++ }+// Setup CPU profiling if desiredif *cpuProfile != "" {fs.Infof(nil, "Creating CPU profile %q\n", *cpuProfile)diff --git a/docs/content/docs.md b/docs/content/docs.mdindex f8f5153d9..7f6e208e5 100644--- a/docs/content/docs.md+++ b/docs/content/docs.md@@ -2836,6 +2836,17 @@ Rclone prefixes all log messages with their level in capitals, e.g. INFOwhich makes it easy to grep the log file for different kinds ofinformation.+Metrics+-------++Rclone can publish metrics in the OpenMetrics/Prometheus format.++To enable the metrics endpoint, use the `--metrics-addr` flag. Metrics can also be published on the `--rc-addr` port if the `--rc` flag and `--rc-enable-metrics` flags are supplied or if using rclone rcd `--rc-enable-metrics`++Rclone provides extensive configuration options for the metrics HTTP endpoint. These settings are grouped under the Metrics section and have a prefix `--metrics-*`.++When metrics are enabled with `--rc-enable-metrics`, they will be published on the same port as the rc API. In this case, the `--metrics-*` flags will be ignored, and the HTTP endpoint configuration will be managed by the `--rc-*` parameters.+Exit Code---------diff --git a/docs/content/rc.md b/docs/content/rc.mdindex 32e93867b..80c02a591 100644--- a/docs/content/rc.md+++ b/docs/content/rc.md@@ -100,6 +100,7 @@ Default Off.### --rc-enable-metricsEnable OpenMetrics/Prometheus compatible endpoint at `/metrics`.+If more control over the metrics is desired (for example running it on a different port or with different auth) then endpoint can be enabled with the `--metrics-*` flags instead.Default Off.diff --git a/fs/config/flags/flags.go b/fs/config/flags/flags.goindex db76f5d71..6389026c8 100644--- a/fs/config/flags/flags.go+++ b/fs/config/flags/flags.go@@ -124,6 +124,7 @@ func init() {All.NewGroup("Logging", "Flags for logging and statistics")All.NewGroup("Metadata", "Flags to control metadata")All.NewGroup("RC", "Flags to control the Remote Control API")+ All.NewGroup("Metrics", "Flags to control the Metrics HTTP endpoint.")}// installFlag constructs a name from the flag passed in anddiff --git a/fs/rc/rc.go b/fs/rc/rc.goindex 55287ed6b..9c083f3b7 100644--- a/fs/rc/rc.go+++ b/fs/rc/rc.go@@ -71,8 +71,8 @@ var OptionsInfo = fs.Options{{}, {Name: "rc_enable_metrics",Default: false,- Help: "Enable prometheus metrics on /metrics",- Groups: "RC",+ Help: "Enable the Prometheus metrics path at the remote control server",+ Groups: "RC,Metrics",}, {Name: "rc_job_expire_duration",Default: 60 * time.Second,@@ -83,10 +83,18 @@ var OptionsInfo = fs.Options{{Default: 10 * time.Second,Help: "Interval to check for expired async jobs",Groups: "RC",+}, {+ Name: "metrics_addr",+ Default: []string{""},+ Help: "IPaddress:Port or :Port to bind metrics server to",+ Groups: "Metrics",}}.AddPrefix(libhttp.ConfigInfo, "rc", "RC").AddPrefix(libhttp.AuthConfigInfo, "rc", "RC").AddPrefix(libhttp.TemplateConfigInfo, "rc", "RC").+ AddPrefix(libhttp.ConfigInfo, "metrics", "Metrics").+ AddPrefix(libhttp.AuthConfigInfo, "metrics", "Metrics").+ AddPrefix(libhttp.TemplateConfigInfo, "metrics", "Metrics").SetDefault("rc_addr", []string{"localhost:5572"})func init() {@@ -109,6 +117,9 @@ type Options struct {WebGUINoOpenBrowser bool `config:"rc_web_gui_no_open_browser"` // set to disable auto opening browserWebGUIFetchURL string `config:"rc_web_fetch_url"` // set the default url for fetching webguiEnableMetrics bool `config:"rc_enable_metrics"` // set to disable prometheus metrics on /metrics+ MetricsHTTP libhttp.Config `config:"metrics"`+ MetricsAuth libhttp.AuthConfig `config:"metrics"`+ MetricsTemplate libhttp.TemplateConfig `config:"metrics"`JobExpireDuration time.Duration `config:"rc_job_expire_duration"`JobExpireInterval time.Duration `config:"rc_job_expire_interval"`}diff --git a/fs/rc/rcserver/metrics.go b/fs/rc/rcserver/metrics.gonew file mode 100644index 000000000..502566f47--- /dev/null+++ b/fs/rc/rcserver/metrics.go@@ -0,0 +1,97 @@+// Package rcserver implements the HTTP endpoint to serve the remote control+package rcserver++import (+ "context"+ "fmt"+ "net/http"++ "github.com/prometheus/client_golang/prometheus"+ "github.com/prometheus/client_golang/prometheus/promhttp"+ "github.com/rclone/rclone/fs/accounting"+ "github.com/rclone/rclone/fs/fshttp"+ "github.com/rclone/rclone/fs/rc"+ "github.com/rclone/rclone/fs/rc/jobs"+ libhttp "github.com/rclone/rclone/lib/http"+)++const path = "/metrics"++var promHandlerFunc http.HandlerFunc++func init() {+ rcloneCollector := accounting.NewRcloneCollector(context.Background())+ prometheus.MustRegister(rcloneCollector)++ m := fshttp.NewMetrics("rclone")+ for _, c := range m.Collectors() {+ prometheus.MustRegister(c)+ }+ fshttp.DefaultMetrics = m++ promHandlerFunc = promhttp.Handler().ServeHTTP+}++// MetricsStart the remote control server if configured+//+// If the server wasn't configured the *Server returned may be nil+func MetricsStart(ctx context.Context, opt *rc.Options) (*MetricsServer, error) {+ jobs.SetOpt(opt) // set the defaults for jobs+ if opt.MetricsHTTP.ListenAddr[0] != "" {+ // Serve on the DefaultServeMux so can have global registrations appear+ s, err := newMetricsServer(ctx, opt)+ if err != nil {+ return nil, err+ }+ return s, s.Serve()+ }+ return nil, nil+}++// MetricsServer contains everything to run the rc server+type MetricsServer struct {+ ctx context.Context // for global config+ server *libhttp.Server+ promHandlerFunc http.Handler+ opt *rc.Options+}++func newMetricsServer(ctx context.Context, opt *rc.Options) (*MetricsServer, error) {+ s := &MetricsServer{+ ctx: ctx,+ opt: opt,+ promHandlerFunc: promHandlerFunc,+ }++ var err error+ s.server, err = libhttp.NewServer(ctx,+ libhttp.WithConfig(opt.MetricsHTTP),+ libhttp.WithAuth(opt.MetricsAuth),+ libhttp.WithTemplate(opt.MetricsTemplate),+ )+ if err != nil {+ return nil, fmt.Errorf("failed to init server: %w", err)+ }++ router := s.server.Router()+ router.Get(path, promHandlerFunc)+ return s, nil+}++// Serve runs the http server in the background.+//+// Use s.Close() and s.Wait() to shutdown server+func (s *MetricsServer) Serve() error {+ s.server.Serve()+ return nil+}++// Wait blocks while the server is serving requests+func (s *MetricsServer) Wait() {+ s.server.Wait()+}++// Shutdown gracefully shuts down the server+func (s *MetricsServer) Shutdown() error {+ return s.server.Shutdown()+}diff --git a/fs/rc/rcserver/metrics_test.go b/fs/rc/rcserver/metrics_test.gonew file mode 100644index 000000000..6141cc2a7--- /dev/null+++ b/fs/rc/rcserver/metrics_test.go@@ -0,0 +1,88 @@+package rcserver++import (+ "context"+ "fmt"+ "net/http"+ "regexp"+ "testing"++ _ "github.com/rclone/rclone/backend/local"+ "github.com/rclone/rclone/fs/accounting"+ "github.com/rclone/rclone/fs/config/configfile"+ "github.com/rclone/rclone/fs/rc"+ "github.com/stretchr/testify/require"+)++// Run a suite of tests+func testMetricsServer(t *testing.T, tests []testRun, opt *rc.Options) {+ t.Helper()+ ctx := context.Background()+ configfile.Install()+ rcServer, err := newMetricsServer(ctx, opt)+ require.NoError(t, err)+ testURL := rcServer.server.URLs()[0]+ mux := rcServer.server.Router()+ emulateCalls(t, tests, mux, testURL)+}++// return an enabled rc+func newMetricsTestOpt() rc.Options {+ opt := rc.Opt+ opt.MetricsHTTP.ListenAddr = []string{testBindAddress}+ return opt+}++func TestMetrics(t *testing.T) {+ stats := accounting.GlobalStats()+ tests := makeMetricsTestCases(stats)+ opt := newMetricsTestOpt()+ testMetricsServer(t, tests, &opt)++ // Test changing a couple options+ stats.Bytes(500)+ for i := 0; i < 30; i++ {+ require.NoError(t, stats.DeleteFile(context.Background(), 0))+ }+ stats.Errors(2)+ stats.Bytes(324)++ tests = makeMetricsTestCases(stats)+ testMetricsServer(t, tests, &opt)+}++func makeMetricsTestCases(stats *accounting.StatsInfo) (tests []testRun) {+ tests = []testRun{{+ Name: "Bytes Transferred Metric",+ URL: "metrics",+ Method: "GET",+ Status: http.StatusOK,+ Contains: regexp.MustCompile(fmt.Sprintf("rclone_bytes_transferred_total %d", stats.GetBytes())),+ }, {+ Name: "Checked Files Metric",+ URL: "metrics",+ Method: "GET",+ Status: http.StatusOK,+ Contains: regexp.MustCompile(fmt.Sprintf("rclone_checked_files_total %d", stats.GetChecks())),+ }, {+ Name: "Errors Metric",+ URL: "metrics",+ Method: "GET",+ Status: http.StatusOK,+ Contains: regexp.MustCompile(fmt.Sprintf("rclone_errors_total %d", stats.GetErrors())),+ }, {+ Name: "Deleted Files Metric",+ URL: "metrics",+ Method: "GET",+ Status: http.StatusOK,+ Contains: regexp.MustCompile(fmt.Sprintf("rclone_files_deleted_total %d", stats.GetDeletes())),+ }, {+ Name: "Files Transferred Metric",+ URL: "metrics",+ Method: "GET",+ Status: http.StatusOK,+ Contains: regexp.MustCompile(fmt.Sprintf("rclone_files_transferred_total %d", stats.GetTransfers())),+ },+ }+ return+}diff --git a/fs/rc/rcserver/rcserver.go b/fs/rc/rcserver/rcserver.goindex 88f82a484..6c91931c7 100644--- a/fs/rc/rcserver/rcserver.go+++ b/fs/rc/rcserver/rcserver.go@@ -18,13 +18,9 @@ import ("time""github.com/go-chi/chi/v5/middleware"- "github.com/prometheus/client_golang/prometheus"- "github.com/prometheus/client_golang/prometheus/promhttp""github.com/rclone/rclone/fs"- "github.com/rclone/rclone/fs/accounting""github.com/rclone/rclone/fs/cache""github.com/rclone/rclone/fs/config"- "github.com/rclone/rclone/fs/fshttp""github.com/rclone/rclone/fs/list""github.com/rclone/rclone/fs/rc""github.com/rclone/rclone/fs/rc/jobs"@@ -35,21 +31,6 @@ import ("github.com/skratchdot/open-golang/open")-var promHandler http.Handler--func init() {- rcloneCollector := accounting.NewRcloneCollector(context.Background())- prometheus.MustRegister(rcloneCollector)-- m := fshttp.NewMetrics("rclone")- for _, c := range m.Collectors() {- prometheus.MustRegister(c)- }- fshttp.DefaultMetrics = m-- promHandler = promhttp.Handler()-}-// Start the remote control server if configured//// If the server wasn't configured the *Server returned may be nil@@ -376,7 +357,7 @@ func (s *Server) handleGet(w http.ResponseWriter, r *http.Request, path string)s.serveRemote(w, r, fsMatchResult[2], fsMatchResult[1])returncase path == "metrics" && s.opt.EnableMetrics:- promHandler.ServeHTTP(w, r)+ promHandlerFunc(w, r)returncase path == "*" && s.opt.Serve:// Serve /* as the remote listingdiff --git a/fs/rc/rcserver/rcserver_test.go b/fs/rc/rcserver/rcserver_test.goindex 7b55f2e03..29738b614 100644--- a/fs/rc/rcserver/rcserver_test.go+++ b/fs/rc/rcserver/rcserver_test.go@@ -15,9 +15,10 @@ import ("testing""time"+ "github.com/go-chi/chi/v5"+_ "github.com/rclone/rclone/backend/local""github.com/rclone/rclone/fs"- "github.com/rclone/rclone/fs/accounting""github.com/rclone/rclone/fs/config/configfile""github.com/rclone/rclone/fs/rc""github.com/stretchr/testify/assert"@@ -115,6 +116,10 @@ func testServer(t *testing.T, tests []testRun, opt *rc.Options) {require.NoError(t, err)testURL := rcServer.server.URLs()[0]mux := rcServer.server.Router()+ emulateCalls(t, tests, mux, testURL)+}++func emulateCalls(t *testing.T, tests []testRun, mux chi.Router, testURL string) {for _, test := range tests {t.Run(test.Name, func(t *testing.T) {t.Helper()@@ -568,61 +573,6 @@ Unknown commandtestServer(t, tests, &opt)}-func TestMetrics(t *testing.T) {- stats := accounting.GlobalStats()- tests := makeMetricsTestCases(stats)- opt := newTestOpt()- opt.EnableMetrics = true- testServer(t, tests, &opt)-- // Test changing a couple options- stats.Bytes(500)- for i := 0; i < 30; i++ {- require.NoError(t, stats.DeleteFile(context.Background(), 0))- }- stats.Errors(2)- stats.Bytes(324)-- tests = makeMetricsTestCases(stats)- testServer(t, tests, &opt)-}--func makeMetricsTestCases(stats *accounting.StatsInfo) (tests []testRun) {- tests = []testRun{{- Name: "Bytes Transferred Metric",- URL: "/metrics",- Method: "GET",- Status: http.StatusOK,- Contains: regexp.MustCompile(fmt.Sprintf("rclone_bytes_transferred_total %d", stats.GetBytes())),- }, {- Name: "Checked Files Metric",- URL: "/metrics",- Method: "GET",- Status: http.StatusOK,- Contains: regexp.MustCompile(fmt.Sprintf("rclone_checked_files_total %d", stats.GetChecks())),- }, {- Name: "Errors Metric",- URL: "/metrics",- Method: "GET",- Status: http.StatusOK,- Contains: regexp.MustCompile(fmt.Sprintf("rclone_errors_total %d", stats.GetErrors())),- }, {- Name: "Deleted Files Metric",- URL: "/metrics",- Method: "GET",- Status: http.StatusOK,- Contains: regexp.MustCompile(fmt.Sprintf("rclone_files_deleted_total %d", stats.GetDeletes())),- }, {- Name: "Files Transferred Metric",- URL: "/metrics",- Method: "GET",- Status: http.StatusOK,- Contains: regexp.MustCompile(fmt.Sprintf("rclone_files_transferred_total %d", stats.GetTransfers())),- },- }- return-}-var matchRemoteDirListing = regexp.MustCompile(`<title>Directory listing of /</title>`)func TestServingRoot(t *testing.T) {