Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Prevent infinite loop in Prometheus setup (#73230)

metambenc8d4914484b4
diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.clj
index 33a8040336a..580d2b14530 100644
--- a/src/metabase/analytics/prometheus.clj
+++ b/src/metabase/analytics/prometheus.clj
@@ -717,17 +717,25 @@
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;; Public API: call [[setup!]] once, call [[shutdown!]] on shutdown
+(def ^:private ^:dynamic *setting-up*
+ "True while [[setup!]] is initializing the registry. Guards against reentrant metric
+ emission during init — e.g. token-check error logging that fires while drivers are
+ registering can call [[inc!]], which would otherwise loop back into [[setup!]] before
+ the first call has finished. While `*setting-up*` is true the metric fns silently drop."
+ false)
+
(defn setup!
"Start the prometheus metric collector and web-server."
[]
- (when-not system
- (let [port (prometheus-server-port)]
- (when-not port
- (log/info "Running prometheus metrics without a webserver"))
- (locking #'system
- (when-not system
- (let [sys (make-prometheus-system port "metabase-registry")]
- (alter-var-root #'system (constantly sys))))))))
+ (when (and (not system) (not *setting-up*))
+ (binding [*setting-up* true]
+ (let [port (prometheus-server-port)]
+ (when-not port
+ (log/info "Running prometheus metrics without a webserver"))
+ (locking #'system
+ (when-not system
+ (let [sys (make-prometheus-system port "metabase-registry")]
+ (alter-var-root #'system (constantly sys)))))))))
(defn shutdown!
"Stop the prometheus metrics web-server if it is running."
@@ -757,7 +765,8 @@
([metric labels amount]
(when-not system
(setup!))
- (prometheus/observe (:registry system) metric (qualified-vals labels) amount)))
+ (when system
+ (prometheus/observe (:registry system) metric (qualified-vals labels) amount))))
(defn inc!
"Call iapetos.core/inc on the metric in the global registry.
@@ -770,7 +779,8 @@
([metric labels amount]
(when-not system
(setup!))
- (prometheus/inc (:registry system) metric (qualified-vals labels) amount)))
+ (when system
+ (prometheus/inc (:registry system) metric (qualified-vals labels) amount))))
(defn dec!
"Call iapetos.core/dec on the metric in the global registry.
@@ -785,7 +795,8 @@
([metric labels amount]
(when-not system
(setup!))
- (prometheus/dec (:registry system) metric (qualified-vals labels) amount)))
+ (when system
+ (prometheus/dec (:registry system) metric (qualified-vals labels) amount))))
(defn set!
"Call iapetos.core/set on the metric in the global registry.
@@ -797,14 +808,16 @@
([metric labels amount]
(when-not system
(setup!))
- (prometheus/set (:registry system) metric (qualified-vals labels) amount)))
+ (when system
+ (prometheus/set (:registry system) metric (qualified-vals labels) amount))))
(defn clear!
"Call Collector.clear() on given metric."
[metric]
(when-not system
(setup!))
- (.clear ^SimpleCollector (:raw (collectors/lookup (.-collectors ^iapetos.registry.IapetosRegistry (:registry system)) metric nil))))
+ (when system
+ (.clear ^SimpleCollector (:raw (collectors/lookup (.-collectors ^iapetos.registry.IapetosRegistry (:registry system)) metric nil)))))
(comment
;; want to see what's in the registry?
diff --git a/test/metabase/analytics/prometheus_test.clj b/test/metabase/analytics/prometheus_test.clj
index 2d80e141ecc..9a688434239 100644
--- a/test/metabase/analytics/prometheus_test.clj
+++ b/test/metabase/analytics/prometheus_test.clj
@@ -287,3 +287,24 @@
(is (approx= 1 (mt/metric-value system :metabase-embedding-iframe-static/response {:status "200"})))
(is (approx= 1 (mt/metric-value system :metabase-embedding-public/response {:status "200"})))
(is (approx= 1 (mt/metric-value system :metabase-embedding-simple/response {:status "200"})))))))
+
+(deftest ^:synchronized reentrant-setup-no-ops-test
+ (testing "metric fns no-op when called reentrantly during setup! (prevents init loop)
+ Background: a known-labels impl that emits a metric (e.g. via the token-check error
+ handler that fires during driver init) used to loop back into setup! before alter-var-root
+ completed, because the metric fns auto-init when system is nil. Guarded now by *setting-up*."
+ (let [original-system @#'prometheus/system]
+ (try
+ (alter-var-root #'prometheus/system (constantly nil))
+ (with-bindings {#'prometheus/*setting-up* true}
+ (testing "setup! no-ops while *setting-up* is bound true"
+ (is (nil? (prometheus/setup!)))
+ (is (nil? @#'prometheus/system)))
+ (testing "metric fns no-op rather than re-entering setup!"
+ (is (nil? (prometheus/inc! :metabase-token-check/attempt {:status "failure"})))
+ (is (nil? (prometheus/observe! :metabase-search/index-update-duration-ms 1)))
+ (is (nil? (prometheus/dec! :metabase-search/queue-size)))
+ (is (nil? (prometheus/set! :metabase-search/queue-size 0)))
+ (is (nil? (prometheus/clear! :metabase-token-check/attempt)))))
+ (finally
+ (alter-var-root #'prometheus/system (constantly original-system)))))))