The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Lighter prometheus analytics (#73456)
dpsutton9da28b0f337d
diff --git a/src/metabase/analytics/core.clj b/src/metabase/analytics/core.cljindex 5407f22773d..e9c3cf55afb 100644--- a/src/metabase/analytics/core.clj+++ b/src/metabase/analytics/core.clj@@ -37,6 +37,7 @@known-labelsinitial-valueconnection-pool-info+ observe-initial-valuessetup!shutdown!]diff --git a/src/metabase/analytics/impl.clj b/src/metabase/analytics/impl.cljdeleted file mode 100644index eee9b0733a5..00000000000--- a/src/metabase/analytics/impl.clj+++ /dev/null@@ -1,19 +0,0 @@-(ns metabase.analytics.impl- "CLJ implementation of [[metabase.analytics-interface.core/Reporter]].- Delegates to Prometheus."- (:require- [metabase.analytics-interface.core :as analytics.interface]- [metabase.analytics.prometheus :as prometheus]))--(analytics.interface/set-reporter!- (reify analytics.interface/Reporter- (-inc! [_ metric labels amount]- (prometheus/inc! metric labels amount))- (-dec-gauge! [_ metric labels amount]- (prometheus/dec! metric labels amount))- (-set-gauge! [_ metric labels amount]- (prometheus/set! metric labels amount))- (-observe! [_ metric labels amount]- (prometheus/observe! metric labels amount))- (-clear! [_ metric]- (prometheus/clear! metric))))diff --git a/src/metabase/analytics/init.clj b/src/metabase/analytics/init.cljindex 8911ee49909..4b9aa733bd4 100644--- a/src/metabase/analytics/init.clj+++ b/src/metabase/analytics/init.clj@@ -1,5 +1,4 @@(ns metabase.analytics.init(:require- [metabase.analytics.impl] ; this registers the reporter[metabase.analytics.settings][metabase.analytics.task.send-anonymous-stats]))diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.cljindex b7c4d98bc10..475179b4e29 100644--- a/src/metabase/analytics/prometheus.clj+++ b/src/metabase/analytics/prometheus.clj@@ -8,13 +8,14 @@(:refer-clojure :exclude [set!])(:require[clojure.java.jmx :as jmx]+ [clojure.string :as str][iapetos.collector :as collector][iapetos.collector.ring :as collector.ring][iapetos.core :as prometheus][iapetos.registry.collectors :as collectors][jvm-alloc-rate-meter.core :as alloc-rate-meter][jvm-hiccup-meter.core :as hiccup-meter]- [metabase.analytics.settings :refer [prometheus-server-port]]+ [metabase.analytics-interface.core :as analytics.interface][metabase.util :as u][metabase.util.i18n :refer [trs]][metabase.util.log :as log]@@ -700,8 +701,6 @@[@c3p0-collector](product-collectors)(quartz-collectors)))]- (doseq [{:keys [metric labels value]} (initial-labelled-metric-values)]- (prometheus/inc registry metric (qualified-vals labels) value))(when @jvm-hiccup-thread (@jvm-hiccup-thread))(reset! jvm-hiccup-thread(hiccup-meter/start-hiccup-meter@@ -728,25 +727,41 @@;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; Public API: call [[setup!]] once, call [[shutdown!]] on shutdown-(def ^:private ^:dynamic *setting-up*- "True while [[setup!]] is initializing the registry. Guards against reentrant metric- emission during init — e.g. token-check error logging that fires while drivers are- registering can call [[inc!]], which would otherwise loop back into [[setup!]] before- the first call has finished. While `*setting-up*` is true the metric fns silently drop."- false)+(defn- parse+ [x]+ (when-not (str/blank? x)+ (let [value (parse-long x)]+ (when-not value+ (log/warn "MB_PROMETHEUS_SERVER_PORT had a value but did not parse to an integer"))+ value)))++(declare install-real-reporter!)(defn setup!- "Start the prometheus metric collector and web-server."+ "Start the prometheus metric collector and web-server. Port is optional, read from `MB_PROMETHEUS_SERVER_PORT`"+ ([] (setup! (parse (System/getenv "MB_PROMETHEUS_SERVER_PORT"))))+ ([port]+ (when (not system)+ (when-not port+ (log/info "Running prometheus metrics without a webserver"))+ (locking #'system+ (when-not system+ (let [sys (make-prometheus-system port "metabase-registry")]+ (alter-var-root #'system (constantly sys))+ (install-real-reporter!)))))))++(defn observe-initial-values+ "Observe initial values. Some values need a baseline otherwise their first observed value won't register as a+ deviation from that baseline. See #52834. It's possible that before setting these baselines, some of these values+ will be observed. Most initial values are 0 and `(inc r m ls 0)` is a no-op as to the value, but ensures the metric+ will be scraped. Search engine uses a bit field 1 or 0 for active and default. But this is the only place these are+ observed so it's not a double inc."[]- (when (and (not system) (not *setting-up*))- (binding [*setting-up* true]- (let [port (prometheus-server-port)]- (when-not port- (log/info "Running prometheus metrics without a webserver"))- (locking #'system- (when-not system- (let [sys (make-prometheus-system port "metabase-registry")]- (alter-var-root #'system (constantly sys)))))))))+ (let [registry (:registry system)]+ (if-not registry+ (log/warn "Observe initial values called without initializing registry")+ (doseq [{:keys [metric labels value]} (initial-labelled-metric-values)]+ (prometheus/inc registry metric (qualified-vals labels) value)))))(defn shutdown!"Stop the prometheus metrics web-server if it is running."@@ -774,10 +789,7 @@(observe! metric nil labels-or-amount)(observe! metric labels-or-amount 1)))([metric labels amount]- (when-not system- (setup!))- (when system- (prometheus/observe (:registry system) metric (qualified-vals labels) amount))))+ (prometheus/observe (:registry system) metric (qualified-vals labels) amount)))(defn inc!"Call iapetos.core/inc on the metric in the global registry.@@ -788,10 +800,7 @@(inc! metric nil labels-or-amount)(inc! metric labels-or-amount 1)))([metric labels amount]- (when-not system- (setup!))- (when system- (prometheus/inc (:registry system) metric (qualified-vals labels) amount))))+ (prometheus/inc (:registry system) metric (qualified-vals labels) amount)))(defn dec!"Call iapetos.core/dec on the metric in the global registry.@@ -804,10 +813,7 @@(dec! metric nil labels-or-amount)(dec! metric labels-or-amount 1)))([metric labels amount]- (when-not system- (setup!))- (when system- (prometheus/dec (:registry system) metric (qualified-vals labels) amount))))+ (prometheus/dec (:registry system) metric (qualified-vals labels) amount)))(defn set!"Call iapetos.core/set on the metric in the global registry.@@ -817,10 +823,7 @@;; Escape var to avoid confusing it with the special form of the same name.(#'set! metric nil amount))([metric labels amount]- (when-not system- (setup!))- (when system- (prometheus/set (:registry system) metric (qualified-vals labels) amount))))+ (prometheus/set (:registry system) metric (qualified-vals labels) amount)))(defn clear!"Call Collector.clear() on given metric."@@ -830,6 +833,27 @@(when system(.clear ^SimpleCollector (:raw (collectors/lookup (.-collectors ^iapetos.registry.IapetosRegistry (:registry system)) metric nil)))))+(def ^:private real-reporter+ "A real analytics.interface/Reporter that reports here"+ (reify analytics.interface/Reporter+ (-inc! [_ metric labels amount]+ (inc! metric labels amount))+ (-dec-gauge! [_ metric labels amount]+ (dec! metric labels amount))+ (-set-gauge! [_ metric labels amount]+ ;; set! in first position hits the special form+ (metabase.analytics.prometheus/set! metric labels amount))+ (-observe! [_ metric labels amount]+ (observe! metric labels amount))+ (-clear! [_ metric]+ (clear! metric))))++(defn- install-real-reporter!+ "Called after setup to wire up the real reporter"+ []+ (log/info "Installing real prometheus reporter")+ (analytics.interface/set-reporter! real-reporter))+(comment;; want to see what's in the registry?(require 'iapetos.export)diff --git a/src/metabase/analytics/settings.clj b/src/metabase/analytics/settings.cljindex 57cec0cef09..f25bacf9e07 100644--- a/src/metabase/analytics/settings.clj+++ b/src/metabase/analytics/settings.clj@@ -5,23 +5,8 @@[metabase.settings.core :as setting :refer [defsetting]][metabase.util.date-2 :as u.date][metabase.util.i18n :refer [deferred-tru]]- [metabase.util.log :as log][toucan2.core :as t2]))-(defsetting prometheus-server-port- (deferred-tru (str "Port to serve prometheus metrics from. If set, prometheus collectors are registered"- " and served from `localhost:<port>/metrics`."))- :type :integer- :visibility :internal- ;; settable only through environmental variable- :setter :none- :getter (fn reading-prometheus-port-setting []- (let [parse (fn [raw-value]- (if-let [parsed (parse-long raw-value)]- parsed- (log/warnf "MB_PROMETHEUS_SERVER_PORT value of '%s' is not parseable as an integer." raw-value)))]- (setting/get-raw-value :prometheus-server-port integer? parse))))-(defsetting analytics-uuid(deferred-tru(str "Unique identifier to be used in Snowplow analytics, to identify this instance of Metabase. "diff --git a/src/metabase/analytics_interface/core.cljc b/src/metabase/analytics_interface/core.cljcindex 8a207ebd2d3..73120644e48 100644--- a/src/metabase/analytics_interface/core.cljc+++ b/src/metabase/analytics_interface/core.cljc@@ -35,7 +35,17 @@(-clear! [this metric]"Clear all values for a metric (reset all label combinations)."))-(defonce ^:private reporter (atom nil))+(def ^:private no-op-reporter+ "A no-op reporter. This ideally should never be used in production, as the real one is installed after setting up the+ prometheus server. That has been moved to very early in the startup chain"+ (reify Reporter+ (-inc! [_this _metric _labels _amount])+ (-dec-gauge! [_this _metric _labels _amount])+ (-set-gauge! [_this _metric _labels _amount])+ (-observe! [_this _metric _labels _amount])+ (-clear! [_this _metric])))++(defonce ^:private reporter (atom no-op-reporter))(defn get-reporter"Return the currently registered [[Reporter]], or nil."diff --git a/src/metabase/core/core.clj b/src/metabase/core/core.cljindex f00409e464c..12b0af16812 100644--- a/src/metabase/core/core.clj+++ b/src/metabase/core/core.clj@@ -170,6 +170,9 @@(init-status/set-progress! 0.2);; Ensure the classloader is installed as soon as possible.(classloader/the-classloader)+ ;; Set up Prometheus+ (log/info "Setting up prometheus metrics")+ (analytics.core/setup!);; Initialize OpenTelemetry tracing early (before plugins, no DB dependency)(tracing/init!);; load any plugins as needed@@ -190,9 +193,7 @@(when (cloud-migration/read-only-mode)(cloud-migration/read-only-mode! false))(init-status/set-progress! 0.4)- ;; Set up Prometheus- (log/info "Setting up prometheus metrics")- (analytics.core/setup!)+ (analytics.core/observe-initial-values)(init-status/set-progress! 0.5)(task/init-scheduler!)(analytics.core/add-listeners-to-scheduler!)diff --git a/test/metabase/analytics/prometheus_test.clj b/test/metabase/analytics/prometheus_test.cljindex 9a688434239..6465a5d97a1 100644--- a/test/metabase/analytics/prometheus_test.clj+++ b/test/metabase/analytics/prometheus_test.clj@@ -164,12 +164,6 @@(< (abs (- actual expected)) epsilon)))(deftest inc!-test- (testing "inc starts a system if it wasn't started"- (with-redefs [prometheus/system nil]- (mt/with-temporary-setting-values [prometheus-server-port 0]- (prometheus/inc! :metabase-email/messages) ; << Does not throw.- (is (approx= 1 (mt/metric-value @#'prometheus/system :metabase-email/messages))))))-(testing "inc throws when called with an unknown metric"(mt/with-prometheus-system! [_ _system](is (thrown-with-msg? RuntimeException@@ -186,12 +180,6 @@(is (approx= 1 (mt/metric-value system :metabase-notification/send-ok {:payload-type :notification/card}))))))(deftest dec!-test- (testing "dec starts a system if it wasn't started"- (mt/with-temporary-setting-values [prometheus-server-port 0]- (with-redefs [prometheus/system nil]- (prometheus/dec! :metabase-search/queue-size) ; << Does not throw.- (is (approx= -1 (mt/metric-value @#'prometheus/system :metabase-search/queue-size))))))-(testing "dec throws when called with an unknown metric"(mt/with-prometheus-system! [_ _system](is (thrown-with-msg? RuntimeException@@ -208,24 +196,6 @@(prometheus/dec! :metabase-search/engine-active {:engine :default} 1)(is (approx= -1 (mt/metric-value system :metabase-search/engine-active {:engine :default}))))))-(deftest observe!-test- (testing "observe! starts a system if it wasn't started"- (with-redefs [prometheus/system nil]- (mt/with-temporary-setting-values [prometheus-server-port 0]- (prometheus/observe! :metabase-notification/send-duration-ms 2) ; << Does not throw.- (is (approx= 2 (:sum (mt/metric-value @#'prometheus/system :metabase-notification/send-duration-ms)))))))-- (testing "observe! with labels is correctly recorded"- (mt/with-prometheus-system! [_ system]- (prometheus/observe! :metabase-notification/send-duration-ms {:payload-type :notification/card} 2)- (is (approx= 2 (:sum (mt/metric-value system :metabase-notification/send-duration-ms {:payload-type :notification/card}))))))-- (testing "observe! throws when called with an unknown metric"- (mt/with-prometheus-system! [_ _system]- (is (thrown-with-msg? RuntimeException- #"error when updating metric"- (prometheus/observe! :metabase-email/unknown-metric 1))))))-(deftest search-engine-metrics-test(let [metrics (#'prometheus/initial-labelled-metric-values)engine->value (fn [metric] (u/index-by (comp :engine :labels) :value (filter (comp #{metric} :metric) metrics)))@@ -287,24 +257,3 @@(is (approx= 1 (mt/metric-value system :metabase-embedding-iframe-static/response {:status "200"})))(is (approx= 1 (mt/metric-value system :metabase-embedding-public/response {:status "200"})))(is (approx= 1 (mt/metric-value system :metabase-embedding-simple/response {:status "200"})))))))--(deftest ^:synchronized reentrant-setup-no-ops-test- (testing "metric fns no-op when called reentrantly during setup! (prevents init loop)- Background: a known-labels impl that emits a metric (e.g. via the token-check error- handler that fires during driver init) used to loop back into setup! before alter-var-root- completed, because the metric fns auto-init when system is nil. Guarded now by *setting-up*."- (let [original-system @#'prometheus/system]- (try- (alter-var-root #'prometheus/system (constantly nil))- (with-bindings {#'prometheus/*setting-up* true}- (testing "setup! no-ops while *setting-up* is bound true"- (is (nil? (prometheus/setup!)))- (is (nil? @#'prometheus/system)))- (testing "metric fns no-op rather than re-entering setup!"- (is (nil? (prometheus/inc! :metabase-token-check/attempt {:status "failure"})))- (is (nil? (prometheus/observe! :metabase-search/index-update-duration-ms 1)))- (is (nil? (prometheus/dec! :metabase-search/queue-size)))- (is (nil? (prometheus/set! :metabase-search/queue-size 0)))- (is (nil? (prometheus/clear! :metabase-token-check/attempt)))))- (finally- (alter-var-root #'prometheus/system (constantly original-system)))))))diff --git a/test/metabase/test_runner.clj b/test/metabase/test_runner.cljindex c7b6cc5a813..fc3b7d4771f 100644--- a/test/metabase/test_runner.clj+++ b/test/metabase/test_runner.clj@@ -8,6 +8,7 @@[clojure.string :as str][humane-are.core :as humane-are][mb.hawk.core :as hawk]+ [metabase.analytics.core :as analytics.core][metabase.config.core :as config][metabase.core.bootstrap][metabase.test-runner.assert-exprs]@@ -122,8 +123,11 @@([options](hawk/find-tests-with-options (parse-options options))))+#_{:clj-kondo/ignore [:metabase/test-helpers-use-non-thread-safe-functions]}(defn- initialize-all-fixtures [](let [steps (initialize/all-components)]+ (analytics.core/setup!)+ (log/info "Initialized prometheus collector")(u/with-timer-ms [duration-ms](doseq [init-step steps](fixtures/initialize init-step))