Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Lighter prometheus analytics (#73456)

dpsutton9da28b0f337d
diff --git a/src/metabase/analytics/core.clj b/src/metabase/analytics/core.clj
index 5407f22773d..e9c3cf55afb 100644
--- a/src/metabase/analytics/core.clj
+++ b/src/metabase/analytics/core.clj
@@ -37,6 +37,7 @@
known-labels
initial-value
connection-pool-info
+ observe-initial-values
setup!
shutdown!]
diff --git a/src/metabase/analytics/impl.clj b/src/metabase/analytics/impl.clj
deleted file mode 100644
index eee9b0733a5..00000000000
--- a/src/metabase/analytics/impl.clj
+++ /dev/null
@@ -1,19 +0,0 @@
-(ns metabase.analytics.impl
- "CLJ implementation of [[metabase.analytics-interface.core/Reporter]].
- Delegates to Prometheus."
- (:require
- [metabase.analytics-interface.core :as analytics.interface]
- [metabase.analytics.prometheus :as prometheus]))
-
-(analytics.interface/set-reporter!
- (reify analytics.interface/Reporter
- (-inc! [_ metric labels amount]
- (prometheus/inc! metric labels amount))
- (-dec-gauge! [_ metric labels amount]
- (prometheus/dec! metric labels amount))
- (-set-gauge! [_ metric labels amount]
- (prometheus/set! metric labels amount))
- (-observe! [_ metric labels amount]
- (prometheus/observe! metric labels amount))
- (-clear! [_ metric]
- (prometheus/clear! metric))))
diff --git a/src/metabase/analytics/init.clj b/src/metabase/analytics/init.clj
index 8911ee49909..4b9aa733bd4 100644
--- a/src/metabase/analytics/init.clj
+++ b/src/metabase/analytics/init.clj
@@ -1,5 +1,4 @@
(ns metabase.analytics.init
(:require
- [metabase.analytics.impl] ; this registers the reporter
[metabase.analytics.settings]
[metabase.analytics.task.send-anonymous-stats]))
diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.clj
index b7c4d98bc10..475179b4e29 100644
--- a/src/metabase/analytics/prometheus.clj
+++ b/src/metabase/analytics/prometheus.clj
@@ -8,13 +8,14 @@
(:refer-clojure :exclude [set!])
(:require
[clojure.java.jmx :as jmx]
+ [clojure.string :as str]
[iapetos.collector :as collector]
[iapetos.collector.ring :as collector.ring]
[iapetos.core :as prometheus]
[iapetos.registry.collectors :as collectors]
[jvm-alloc-rate-meter.core :as alloc-rate-meter]
[jvm-hiccup-meter.core :as hiccup-meter]
- [metabase.analytics.settings :refer [prometheus-server-port]]
+ [metabase.analytics-interface.core :as analytics.interface]
[metabase.util :as u]
[metabase.util.i18n :refer [trs]]
[metabase.util.log :as log]
@@ -700,8 +701,6 @@
[@c3p0-collector]
(product-collectors)
(quartz-collectors)))]
- (doseq [{:keys [metric labels value]} (initial-labelled-metric-values)]
- (prometheus/inc registry metric (qualified-vals labels) value))
(when @jvm-hiccup-thread (@jvm-hiccup-thread))
(reset! jvm-hiccup-thread
(hiccup-meter/start-hiccup-meter
@@ -728,25 +727,41 @@
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;; Public API: call [[setup!]] once, call [[shutdown!]] on shutdown
-(def ^:private ^:dynamic *setting-up*
- "True while [[setup!]] is initializing the registry. Guards against reentrant metric
- emission during init — e.g. token-check error logging that fires while drivers are
- registering can call [[inc!]], which would otherwise loop back into [[setup!]] before
- the first call has finished. While `*setting-up*` is true the metric fns silently drop."
- false)
+(defn- parse
+ [x]
+ (when-not (str/blank? x)
+ (let [value (parse-long x)]
+ (when-not value
+ (log/warn "MB_PROMETHEUS_SERVER_PORT had a value but did not parse to an integer"))
+ value)))
+
+(declare install-real-reporter!)
(defn setup!
- "Start the prometheus metric collector and web-server."
+ "Start the prometheus metric collector and web-server. Port is optional, read from `MB_PROMETHEUS_SERVER_PORT`"
+ ([] (setup! (parse (System/getenv "MB_PROMETHEUS_SERVER_PORT"))))
+ ([port]
+ (when (not system)
+ (when-not port
+ (log/info "Running prometheus metrics without a webserver"))
+ (locking #'system
+ (when-not system
+ (let [sys (make-prometheus-system port "metabase-registry")]
+ (alter-var-root #'system (constantly sys))
+ (install-real-reporter!)))))))
+
+(defn observe-initial-values
+ "Observe initial values. Some values need a baseline otherwise their first observed value won't register as a
+ deviation from that baseline. See #52834. It's possible that before setting these baselines, some of these values
+ will be observed. Most initial values are 0 and `(inc r m ls 0)` is a no-op as to the value, but ensures the metric
+ will be scraped. Search engine uses a bit field 1 or 0 for active and default. But this is the only place these are
+ observed so it's not a double inc."
[]
- (when (and (not system) (not *setting-up*))
- (binding [*setting-up* true]
- (let [port (prometheus-server-port)]
- (when-not port
- (log/info "Running prometheus metrics without a webserver"))
- (locking #'system
- (when-not system
- (let [sys (make-prometheus-system port "metabase-registry")]
- (alter-var-root #'system (constantly sys)))))))))
+ (let [registry (:registry system)]
+ (if-not registry
+ (log/warn "Observe initial values called without initializing registry")
+ (doseq [{:keys [metric labels value]} (initial-labelled-metric-values)]
+ (prometheus/inc registry metric (qualified-vals labels) value)))))
(defn shutdown!
"Stop the prometheus metrics web-server if it is running."
@@ -774,10 +789,7 @@
(observe! metric nil labels-or-amount)
(observe! metric labels-or-amount 1)))
([metric labels amount]
- (when-not system
- (setup!))
- (when system
- (prometheus/observe (:registry system) metric (qualified-vals labels) amount))))
+ (prometheus/observe (:registry system) metric (qualified-vals labels) amount)))
(defn inc!
"Call iapetos.core/inc on the metric in the global registry.
@@ -788,10 +800,7 @@
(inc! metric nil labels-or-amount)
(inc! metric labels-or-amount 1)))
([metric labels amount]
- (when-not system
- (setup!))
- (when system
- (prometheus/inc (:registry system) metric (qualified-vals labels) amount))))
+ (prometheus/inc (:registry system) metric (qualified-vals labels) amount)))
(defn dec!
"Call iapetos.core/dec on the metric in the global registry.
@@ -804,10 +813,7 @@
(dec! metric nil labels-or-amount)
(dec! metric labels-or-amount 1)))
([metric labels amount]
- (when-not system
- (setup!))
- (when system
- (prometheus/dec (:registry system) metric (qualified-vals labels) amount))))
+ (prometheus/dec (:registry system) metric (qualified-vals labels) amount)))
(defn set!
"Call iapetos.core/set on the metric in the global registry.
@@ -817,10 +823,7 @@
;; Escape var to avoid confusing it with the special form of the same name.
(#'set! metric nil amount))
([metric labels amount]
- (when-not system
- (setup!))
- (when system
- (prometheus/set (:registry system) metric (qualified-vals labels) amount))))
+ (prometheus/set (:registry system) metric (qualified-vals labels) amount)))
(defn clear!
"Call Collector.clear() on given metric."
@@ -830,6 +833,27 @@
(when system
(.clear ^SimpleCollector (:raw (collectors/lookup (.-collectors ^iapetos.registry.IapetosRegistry (:registry system)) metric nil)))))
+(def ^:private real-reporter
+ "A real analytics.interface/Reporter that reports here"
+ (reify analytics.interface/Reporter
+ (-inc! [_ metric labels amount]
+ (inc! metric labels amount))
+ (-dec-gauge! [_ metric labels amount]
+ (dec! metric labels amount))
+ (-set-gauge! [_ metric labels amount]
+ ;; set! in first position hits the special form
+ (metabase.analytics.prometheus/set! metric labels amount))
+ (-observe! [_ metric labels amount]
+ (observe! metric labels amount))
+ (-clear! [_ metric]
+ (clear! metric))))
+
+(defn- install-real-reporter!
+ "Called after setup to wire up the real reporter"
+ []
+ (log/info "Installing real prometheus reporter")
+ (analytics.interface/set-reporter! real-reporter))
+
(comment
;; want to see what's in the registry?
(require 'iapetos.export)
diff --git a/src/metabase/analytics/settings.clj b/src/metabase/analytics/settings.clj
index 57cec0cef09..f25bacf9e07 100644
--- a/src/metabase/analytics/settings.clj
+++ b/src/metabase/analytics/settings.clj
@@ -5,23 +5,8 @@
[metabase.settings.core :as setting :refer [defsetting]]
[metabase.util.date-2 :as u.date]
[metabase.util.i18n :refer [deferred-tru]]
- [metabase.util.log :as log]
[toucan2.core :as t2]))
-(defsetting prometheus-server-port
- (deferred-tru (str "Port to serve prometheus metrics from. If set, prometheus collectors are registered"
- " and served from `localhost:<port>/metrics`."))
- :type :integer
- :visibility :internal
- ;; settable only through environmental variable
- :setter :none
- :getter (fn reading-prometheus-port-setting []
- (let [parse (fn [raw-value]
- (if-let [parsed (parse-long raw-value)]
- parsed
- (log/warnf "MB_PROMETHEUS_SERVER_PORT value of '%s' is not parseable as an integer." raw-value)))]
- (setting/get-raw-value :prometheus-server-port integer? parse))))
-
(defsetting analytics-uuid
(deferred-tru
(str "Unique identifier to be used in Snowplow analytics, to identify this instance of Metabase. "
diff --git a/src/metabase/analytics_interface/core.cljc b/src/metabase/analytics_interface/core.cljc
index 8a207ebd2d3..73120644e48 100644
--- a/src/metabase/analytics_interface/core.cljc
+++ b/src/metabase/analytics_interface/core.cljc
@@ -35,7 +35,17 @@
(-clear! [this metric]
"Clear all values for a metric (reset all label combinations)."))
-(defonce ^:private reporter (atom nil))
+(def ^:private no-op-reporter
+ "A no-op reporter. This ideally should never be used in production, as the real one is installed after setting up the
+ prometheus server. That has been moved to very early in the startup chain"
+ (reify Reporter
+ (-inc! [_this _metric _labels _amount])
+ (-dec-gauge! [_this _metric _labels _amount])
+ (-set-gauge! [_this _metric _labels _amount])
+ (-observe! [_this _metric _labels _amount])
+ (-clear! [_this _metric])))
+
+(defonce ^:private reporter (atom no-op-reporter))
(defn get-reporter
"Return the currently registered [[Reporter]], or nil."
diff --git a/src/metabase/core/core.clj b/src/metabase/core/core.clj
index f00409e464c..12b0af16812 100644
--- a/src/metabase/core/core.clj
+++ b/src/metabase/core/core.clj
@@ -170,6 +170,9 @@
(init-status/set-progress! 0.2)
;; Ensure the classloader is installed as soon as possible.
(classloader/the-classloader)
+ ;; Set up Prometheus
+ (log/info "Setting up prometheus metrics")
+ (analytics.core/setup!)
;; Initialize OpenTelemetry tracing early (before plugins, no DB dependency)
(tracing/init!)
;; load any plugins as needed
@@ -190,9 +193,7 @@
(when (cloud-migration/read-only-mode)
(cloud-migration/read-only-mode! false))
(init-status/set-progress! 0.4)
- ;; Set up Prometheus
- (log/info "Setting up prometheus metrics")
- (analytics.core/setup!)
+ (analytics.core/observe-initial-values)
(init-status/set-progress! 0.5)
(task/init-scheduler!)
(analytics.core/add-listeners-to-scheduler!)
diff --git a/test/metabase/analytics/prometheus_test.clj b/test/metabase/analytics/prometheus_test.clj
index 9a688434239..6465a5d97a1 100644
--- a/test/metabase/analytics/prometheus_test.clj
+++ b/test/metabase/analytics/prometheus_test.clj
@@ -164,12 +164,6 @@
(< (abs (- actual expected)) epsilon)))
(deftest inc!-test
- (testing "inc starts a system if it wasn't started"
- (with-redefs [prometheus/system nil]
- (mt/with-temporary-setting-values [prometheus-server-port 0]
- (prometheus/inc! :metabase-email/messages) ; << Does not throw.
- (is (approx= 1 (mt/metric-value @#'prometheus/system :metabase-email/messages))))))
-
(testing "inc throws when called with an unknown metric"
(mt/with-prometheus-system! [_ _system]
(is (thrown-with-msg? RuntimeException
@@ -186,12 +180,6 @@
(is (approx= 1 (mt/metric-value system :metabase-notification/send-ok {:payload-type :notification/card}))))))
(deftest dec!-test
- (testing "dec starts a system if it wasn't started"
- (mt/with-temporary-setting-values [prometheus-server-port 0]
- (with-redefs [prometheus/system nil]
- (prometheus/dec! :metabase-search/queue-size) ; << Does not throw.
- (is (approx= -1 (mt/metric-value @#'prometheus/system :metabase-search/queue-size))))))
-
(testing "dec throws when called with an unknown metric"
(mt/with-prometheus-system! [_ _system]
(is (thrown-with-msg? RuntimeException
@@ -208,24 +196,6 @@
(prometheus/dec! :metabase-search/engine-active {:engine :default} 1)
(is (approx= -1 (mt/metric-value system :metabase-search/engine-active {:engine :default}))))))
-(deftest observe!-test
- (testing "observe! starts a system if it wasn't started"
- (with-redefs [prometheus/system nil]
- (mt/with-temporary-setting-values [prometheus-server-port 0]
- (prometheus/observe! :metabase-notification/send-duration-ms 2) ; << Does not throw.
- (is (approx= 2 (:sum (mt/metric-value @#'prometheus/system :metabase-notification/send-duration-ms)))))))
-
- (testing "observe! with labels is correctly recorded"
- (mt/with-prometheus-system! [_ system]
- (prometheus/observe! :metabase-notification/send-duration-ms {:payload-type :notification/card} 2)
- (is (approx= 2 (:sum (mt/metric-value system :metabase-notification/send-duration-ms {:payload-type :notification/card}))))))
-
- (testing "observe! throws when called with an unknown metric"
- (mt/with-prometheus-system! [_ _system]
- (is (thrown-with-msg? RuntimeException
- #"error when updating metric"
- (prometheus/observe! :metabase-email/unknown-metric 1))))))
-
(deftest search-engine-metrics-test
(let [metrics (#'prometheus/initial-labelled-metric-values)
engine->value (fn [metric] (u/index-by (comp :engine :labels) :value (filter (comp #{metric} :metric) metrics)))
@@ -287,24 +257,3 @@
(is (approx= 1 (mt/metric-value system :metabase-embedding-iframe-static/response {:status "200"})))
(is (approx= 1 (mt/metric-value system :metabase-embedding-public/response {:status "200"})))
(is (approx= 1 (mt/metric-value system :metabase-embedding-simple/response {:status "200"})))))))
-
-(deftest ^:synchronized reentrant-setup-no-ops-test
- (testing "metric fns no-op when called reentrantly during setup! (prevents init loop)
- Background: a known-labels impl that emits a metric (e.g. via the token-check error
- handler that fires during driver init) used to loop back into setup! before alter-var-root
- completed, because the metric fns auto-init when system is nil. Guarded now by *setting-up*."
- (let [original-system @#'prometheus/system]
- (try
- (alter-var-root #'prometheus/system (constantly nil))
- (with-bindings {#'prometheus/*setting-up* true}
- (testing "setup! no-ops while *setting-up* is bound true"
- (is (nil? (prometheus/setup!)))
- (is (nil? @#'prometheus/system)))
- (testing "metric fns no-op rather than re-entering setup!"
- (is (nil? (prometheus/inc! :metabase-token-check/attempt {:status "failure"})))
- (is (nil? (prometheus/observe! :metabase-search/index-update-duration-ms 1)))
- (is (nil? (prometheus/dec! :metabase-search/queue-size)))
- (is (nil? (prometheus/set! :metabase-search/queue-size 0)))
- (is (nil? (prometheus/clear! :metabase-token-check/attempt)))))
- (finally
- (alter-var-root #'prometheus/system (constantly original-system)))))))
diff --git a/test/metabase/test_runner.clj b/test/metabase/test_runner.clj
index c7b6cc5a813..fc3b7d4771f 100644
--- a/test/metabase/test_runner.clj
+++ b/test/metabase/test_runner.clj
@@ -8,6 +8,7 @@
[clojure.string :as str]
[humane-are.core :as humane-are]
[mb.hawk.core :as hawk]
+ [metabase.analytics.core :as analytics.core]
[metabase.config.core :as config]
[metabase.core.bootstrap]
[metabase.test-runner.assert-exprs]
@@ -122,8 +123,11 @@
([options]
(hawk/find-tests-with-options (parse-options options))))
+#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}
(defn- initialize-all-fixtures []
(let [steps (initialize/all-components)]
+ (analytics.core/setup!)
+ (log/info "Initialized prometheus collector")
(u/with-timer-ms [duration-ms]
(doseq [init-step steps]
(fixtures/initialize init-step))