Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add shared search index health metrics (#78297)

Chris Truter8112036b95d5
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index abefbfc5e6c..e456148a55d 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -1048,6 +1048,7 @@
health-inspector
{:team "Querying Platform"
:api #{metabase.health-inspector.api
+ metabase.health-inspector.core
metabase.health-inspector.init}
:uses #{api
lib
@@ -2016,6 +2017,7 @@
metabase.search.core
metabase.search.engine
metabase.search.in-place.scoring
+ metabase.search.index-health
metabase.search.ingestion
metabase.search.init
metabase.search.scoring
@@ -2030,6 +2032,7 @@
collections
config
events
+ health-inspector
lib
lib-be
models
diff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/task/metric_collector.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/task/metric_collector.clj
index b2e45e054c4..8eb519ddc92 100644
--- a/enterprise/backend/src/metabase_enterprise/semantic_search/task/metric_collector.clj
+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/task/metric_collector.clj
@@ -4,10 +4,12 @@
[clojurewerkz.quartzite.schedule.simple :as simple]
[clojurewerkz.quartzite.triggers :as triggers]
[honey.sql :as sql]
+ [metabase-enterprise.semantic-search.db.datasource :as semantic.datasource]
[metabase-enterprise.semantic-search.dlq :as semantic.dlq]
[metabase-enterprise.semantic-search.env :as semantic.env]
[metabase-enterprise.semantic-search.util :as semantic.u]
[metabase.analytics-interface.core :as analytics]
+ [metabase.search.index-health :as search.index-health]
[metabase.task.core :as task]
[metabase.util.log :as log]
[next.jdbc :as jdbc]
@@ -65,13 +67,21 @@
(log/warn "DLQ table does not exist. Index may not have been initialized.")))
(defn- collect-metrics! []
- ;; Active, not merely available: on an available-but-inactive instance the index tables never exist,
- ;; and the collectors would warn about them on every run.
- (when (semantic.u/semantic-search-active?)
- (let [pgvector (semantic.env/get-pgvector-datasource!)
- index-metadata (semantic.env/get-index-metadata)]
- (collect-gate-size! pgvector)
- (collect-dlq-size! pgvector index-metadata))))
+ (try
+ ;; Active, not merely available: on an available-but-inactive instance the index tables never exist,
+ ;; and the collectors would warn about them on every run.
+ (when (semantic.u/semantic-search-active?)
+ (let [pgvector (semantic.env/get-pgvector-datasource!)
+ index-metadata (semantic.env/get-index-metadata)]
+ (collect-gate-size! pgvector)
+ (collect-dlq-size! pgvector index-metadata)))
+ (catch InterruptedException e
+ (throw e))
+ (catch Exception e
+ (log/error e "Semantic search metric collector errored")))
+ ;; Registered collectors self-gate, so refreshing them is a cheap no-op when their feature is off.
+ ;; Keep this outside the try: ordinary failures continue here, while interruption and fatal errors exit.
+ (search.index-health/refresh-search-index-metrics!))
(task/defjob ^{DisallowConcurrentExecution true
:doc "Collect expensive semantic search metrics"}
@@ -82,7 +92,13 @@
(defmethod task/init! ::SemanticMetricCollector
[_]
- (when (semantic.u/semantic-search-configured?)
+ ;; Boot-safe gate: plain env/feature checks, never a DB probe (pgvector-configured? would resolve
+ ;; pgvector-mode, probing the app db and logging a pgvector-store line on instances that can't use the
+ ;; answer). The dedicated-URL arm schedules without the feature, so a token entered post-boot starts the
+ ;; search-index gauges without a restart; app-db-pgvector instances licensed post-boot need one, matching the
+ ;; other semantic tasks (see [[semantic.u/semantic-search-configured?]]).
+ (if (or (semantic.datasource/dedicated-url-configured?)
+ (semantic.u/semantic-search-configured?))
(let [job (jobs/build
(jobs/of-type SemanticMetricCollector)
(jobs/with-identity collector-job-key))
@@ -94,4 +110,7 @@
(simple/with-interval-in-milliseconds job-interval-ms)
(simple/repeat-forever)))
(triggers/start-now))]
- (task/schedule-task! job trigger))))
+ (task/schedule-task! job trigger))
+ ;; Quartz's job store is persistent, so a collector scheduled by an earlier deploy would otherwise
+ ;; keep firing (as a no-op) on an instance whose configuration went away.
+ (task/delete-task! collector-job-key collector-trigger-key)))
diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/task/metric_collector_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/task/metric_collector_test.clj
index f416db3c3bf..a5cf3401da5 100644
--- a/enterprise/backend/test/metabase_enterprise/semantic_search/task/metric_collector_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/task/metric_collector_test.clj
@@ -12,6 +12,7 @@
[metabase-enterprise.semantic-search.task.metric-collector :as semantic.task.collector]
[metabase-enterprise.semantic-search.test-util :as semantic.tu]
[metabase-enterprise.semantic-search.util :as semantic.u]
+ [metabase.search.index-health :as search.index-health]
[metabase.test :as mt]
[next.jdbc :as jdbc]))
@@ -66,6 +67,24 @@
pgvector
(sql/format {:delete-from [[:raw (:gate-table-name index-metadata)]]})))
+(deftest shared-index-metrics-survive-semantic-collector-failure-test
+ (let [refreshes (atom 0)]
+ (mt/with-dynamic-fn-redefs
+ [semantic.u/semantic-search-active? (constantly true)
+ semantic.env/get-pgvector-datasource! #(throw (ex-info "pgvector unavailable" {}))
+ search.index-health/refresh-search-index-metrics! #(swap! refreshes inc)]
+ (@#'semantic.task.collector/collect-metrics!)
+ (is (= 1 @refreshes)))))
+
+(deftest interrupted-semantic-collector-skips-shared-refresh-test
+ (let [refreshes (atom 0)]
+ (mt/with-dynamic-fn-redefs
+ [semantic.u/semantic-search-active? (constantly true)
+ semantic.env/get-pgvector-datasource! #(throw (InterruptedException.))
+ search.index-health/refresh-search-index-metrics! #(swap! refreshes inc)]
+ (is (thrown? InterruptedException (@#'semantic.task.collector/collect-metrics!)))
+ (is (zero? @refreshes)))))
+
(deftest metric-collector-test
(mt/with-premium-features #{:semantic-search}
(mt/with-prometheus-system! [_ system]
diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.clj
index 75d2fcd102a..94f881f1c71 100644
--- a/src/metabase/analytics/prometheus.clj
+++ b/src/metabase/analytics/prometheus.clj
@@ -479,6 +479,20 @@
{:description "Number of documents in the library entity index, as of the last full reconcile."})
(prometheus/gauge :metabase-entity-retrieval/index-entities
{:description "Number of distinct entities in the library entity index, as of the last full reconcile."})
+ ;; Search-index health, one series per logical index. Implementations publish these through
+ ;; metabase.search.index-health; labels identify stable logical indexes, never physical table names.
+ (prometheus/gauge :metabase-search/index-coverage-ratio
+ {:description (str "Fraction (0-1) of the items that should be indexed that actually are, "
+ "per search index.")
+ :labels [:index]})
+ (prometheus/gauge :metabase-search/index-garbage-count
+ {:description (str "Absolute number of indexed items that should not be indexed "
+ "(orphaned or no longer a candidate), per search index.")
+ :labels [:index]})
+ (prometheus/gauge :metabase-search/index-staleness-seconds
+ {:description (str "Age in seconds of the oldest known-pending change not yet reflected "
+ "in the index (indexer/reconcile backlog), per search index.")
+ :labels [:index]})
;; data-complexity-score timing
;; 1ms → 1min buckets; widen later if real-world runs push past a minute.
(prometheus/histogram :metabase-data-complexity/scoring-duration-ms
diff --git a/src/metabase/search/index_health.clj b/src/metabase/search/index_health.clj
new file mode 100644
index 00000000000..55934b33150
--- /dev/null
+++ b/src/metabase/search/index_health.clj
@@ -0,0 +1,193 @@
+(ns metabase.search.index-health
+ "Shared health-result and metric plumbing for search indexes. Index implementations register raw coverage,
+ garbage, and staleness collectors here; this namespace publishes their Prometheus gauges and health-inspector rows."
+ (:require
+ [metabase.analytics-interface.core :as analytics]
+ [metabase.analytics.core :as analytics.core]
+ [metabase.health-inspector.core :as health-inspector]
+ [metabase.util.log :as log]))
+
+(set! *warn-on-reflection* true)
+
+(defn healthy
+ "A healthy (100) check result with `message`."
+ [message]
+ {:health 100, :message message})
+
+(defn warning
+ "A partially healthy check result. `health` defaults to 50 and must be strictly between 0 and 100; use
+ [[healthy]] or [[degraded]] for the endpoints."
+ ([message]
+ (warning 50 message))
+ ([health message]
+ {:pre [(< 0 health 100)]}
+ {:health health, :message message}))
+
+(defn degraded
+ "A degraded (0) check result with `message`."
+ [message]
+ {:health 0, :message message})
+
+(def ^:private measure->gauge
+ {:coverage :metabase-search/index-coverage-ratio
+ :garbage :metabase-search/index-garbage-count
+ :staleness :metabase-search/index-staleness-seconds})
+
+(defn- percentage
+ [ratio]
+ (cond
+ (<= ratio 0) 0
+ (>= ratio 1) 100
+ :else (-> (* 100.0 (double ratio)) Math/round (max 1) (min 99))))
+
+(defn- threshold-health
+ "Health from an absolute `value`: 100 at or below `warn`, 0 at or above `critical`, and linear between."
+ [value warn critical]
+ (cond
+ (<= value warn) 100
+ (>= value critical) 0
+ :else (-> (* 100.0 (/ (double (- critical value)) (- critical warn)))
+ Math/round
+ (max 1)
+ (min 99))))
+
+(defn coverage-result
+ "Result for `indexed` out of `expected` items. The ratio feeds Prometheus; its percentage is the health,
+ with 0 and 100 reserved for exact endpoints."
+ [indexed expected]
+ (let [ratio (if (pos? expected) (min 1.0 (/ (double indexed) expected)) 1.0)
+ health (percentage ratio)]
+ {:value ratio
+ :health health
+ :message (format "%d of %d expected items indexed (%d%%)." indexed expected health)}))
+
+(defn garbage-result
+ "Result for an absolute `orphans` count, scored against `warn` and `critical` thresholds."
+ [orphans warn critical]
+ {:value orphans
+ :health (threshold-health orphans warn critical)
+ :message (if (zero? orphans)
+ "No orphaned items in the index."
+ (format "%d orphaned item(s) in the index." orphans))})
+
+(defn- describe-age [seconds]
+ (let [seconds (long seconds)]
+ (cond
+ (>= seconds 3600) (format "%.1fh" (/ seconds 3600.0))
+ (>= seconds 60) (format "%dm" (quot seconds 60))
+ :else (format "%ds" seconds))))
+
+(defn staleness-result
+ "Result for the oldest pending change. `warn` and `critical` are seconds; `detail` is an optional clause."
+ [age-seconds warn critical detail]
+ (let [age (long (or age-seconds 0))
+ base (if (zero? age)
+ "Index current."
+ (format "Oldest pending change is %s old." (describe-age age)))]
+ {:value age
+ :health (threshold-health age warn critical)
+ :message (if detail (str base " " detail) base)}))
+
+(defonce ^:private index-measures
+ ;; Keying by check name makes namespace reloads replace descriptors instead of duplicating collectors.
+ (atom {}))
+
+(defonce ^:private live-gauge-series
+ ;; Only clear series that previously held a real value; inactive indexes should not create NaN-only series.
+ (atom #{}))
+
+(defn- set-index-gauge!
+ [gauge-key index value]
+ (if (some? value)
+ (do
+ (analytics/set-gauge! gauge-key {:index (name index)} value)
+ (swap! live-gauge-series conj [gauge-key index]))
+ (when (contains? @live-gauge-series [gauge-key index])
+ (analytics/set-gauge! gauge-key {:index (name index)} ##NaN))))
+
+(defn- run-measure!
+ "Run one collector and update its gauge. Returns nil for N/A or a health-inspector result."
+ [{:keys [gauge-key index collect check-name]}]
+ (let [{:keys [value health message]}
+ (try
+ (collect)
+ (catch InterruptedException e
+ (throw e))
+ (catch Exception e
+ (log/error e "Search index metric collector errored" {:check check-name})
+ {:health 0, :message (str "Metric collector errored: " (ex-message e))}))]
+ (set-index-gauge! gauge-key index value)
+ (when health
+ {:health health, :message message})))
+
+(defn register-index-check!
+ "Register a collector for logical `index` and `measure`. The collector returns nil for N/A or
+ `{:value :health :message}`. Returns a descriptor accepted by [[refresh-index-check!]]."
+ [index measure collect]
+ (let [descriptor {:check-name (keyword (str (name index) "-" (name measure)))
+ :gauge-key (measure->gauge measure)
+ :index index
+ :measure measure
+ :collect collect}]
+ (health-inspector/register-check! (:check-name descriptor) #(run-measure! descriptor))
+ ;; A live upgrade may leave the defonce'd registry in its former vector representation.
+ (swap! index-measures (fn [measures]
+ (let [keyed (if (map? measures)
+ measures
+ (into {} (map (juxt :check-name identity)) measures))]
+ (assoc keyed (:check-name descriptor) descriptor))))
+ descriptor))
+
+(defn refresh-index-check!
+ "Refresh one registered descriptor. Updates its gauge and, when enabled, its deduplicated health row."
+ [{:keys [check-name] :as descriptor}]
+ (try
+ (when-let [result (run-measure! descriptor)]
+ (when (health-inspector/enabled?)
+ (health-inspector/save-check-result! check-name result)))
+ (catch InterruptedException e
+ (throw e))
+ (catch Exception e
+ (log/error e "Search index health-row persist errored" {:check check-name}))))
+
+(defonce ^:private gauge-refresh-running? (atom false))
+
+(defn- submit-gauge-refresh! [f]
+ (future-call f))
+
+(defn- refresh-index-gauge! [{:keys [check-name] :as descriptor}]
+ (try
+ (run-measure! descriptor)
+ (catch InterruptedException e
+ (throw e))
+ (catch Exception e
+ (log/error e "Search index gauge refresh errored" {:check check-name}))))
+
+(defn- refresh-search-index-gauges!
+ []
+ (try
+ (run! refresh-index-gauge! (vals @index-measures))
+ (finally
+ (reset! gauge-refresh-running? false))))
+
+(defn- request-search-index-gauge-refresh!
+ []
+ (when (compare-and-set! gauge-refresh-running? false true)
+ (try
+ (submit-gauge-refresh! refresh-search-index-gauges!)
+ (catch Exception e
+ (reset! gauge-refresh-running? false)
+ (log/error e "Could not schedule search index gauge refresh"))))
+ nil)
+
+;; Prometheus scrapes every Metabase process, whereas the scheduled metric job may run on only one member of
+;; a Quartz cluster. A scrape starts a local, single-flight background refresh so every process updates its
+;; series without putting index scans on the synchronous scrape path.
+(defmethod analytics.core/pull-collector ::index-health-gauges [_]
+ {:min-interval-s 600
+ :f request-search-index-gauge-refresh!})
+
+(defn refresh-search-index-metrics!
+ "Refresh every registered measure's gauge and, when enabled, its deduplicated health row."
+ []
+ (run! refresh-index-check! (vals @index-measures)))
diff --git a/test/metabase/search/index_health_test.clj b/test/metabase/search/index_health_test.clj
new file mode 100644
index 00000000000..c21ff111fe4
--- /dev/null
+++ b/test/metabase/search/index_health_test.clj
@@ -0,0 +1,203 @@
+(ns metabase.search.index-health-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase.analytics-interface.core :as analytics]
+ [metabase.analytics.core :as analytics.core]
+ [metabase.health-inspector.core :as health-inspector]
+ [metabase.search.index-health :as index-health]
+ [metabase.test :as mt]))
+
+(set! *warn-on-reflection* true)
+
+(deftest ^:sequential register-index-check!-migrates-a-legacy-registry-test
+ (testing "a live upgrade migrates the defonce'd registry's former vector value"
+ (let [measures (atom [{:check-name :legacy-measure, :collect (constantly nil)}])
+ checks (atom {})]
+ (with-redefs [index-health/index-measures measures
+ health-inspector/checks checks]
+ (let [{:keys [check-name]}
+ (index-health/register-index-check! :test-migration :coverage (constantly nil))]
+ (is (= #{:legacy-measure check-name} (set (keys @measures))))
+ (is (contains? @checks check-name)))))))
+
+(deftest ^:parallel coverage-result-test
+ (testing "coverage health is the percentage; the ratio feeds the gauge"
+ (is (=? {:value 0.5 :health 50 :message #"5 of 10 expected items indexed \(50%\)\."}
+ (index-health/coverage-result 5 10))))
+ (testing "an empty candidate set is fully covered"
+ (is (=? {:value 1.0 :health 100 :message #"0 of 0 .*100%.*"}
+ (index-health/coverage-result 0 0))))
+ (testing "coverage clamps over-counts to 100 percent"
+ (is (=? {:value 1.0 :health 100} (index-health/coverage-result 12 10))))
+ (testing "only exact endpoints score 0 or 100"
+ (is (=? {:health 99} (index-health/coverage-result 999 1000)))
+ (is (=? {:health 1} (index-health/coverage-result 1 1000)))
+ (is (=? {:health 0} (index-health/coverage-result 0 1000)))))
+
+(deftest ^:parallel garbage-result-test
+ (testing "zero orphans is healthy"
+ (is (=? {:value 0 :health 100 :message #"No orphaned items.*"}
+ (index-health/garbage-result 0 5 100))))
+ (testing "the value is an absolute count"
+ (is (=? {:value 42 :message #"42 orphaned item\(s\) in the index\."}
+ (index-health/garbage-result 42 5 100))))
+ (testing "health is linear between the warning and critical thresholds"
+ (is (=? {:health 100} (index-health/garbage-result 5 5 100)))
+ (is (=? {:health 0} (index-health/garbage-result 100 5 100)))
+ (is (=? {:health 50} (index-health/garbage-result 52 4 100))))
+ (testing "intermediate values do not round to an endpoint"
+ (is (=? {:health 99} (index-health/garbage-result 1 0 1000)))
+ (is (=? {:health 1} (index-health/garbage-result 999 0 1000)))))
+
+(deftest ^:parallel staleness-result-test
+ (testing "values at or below the warning threshold are healthy"
+ (is (=? {:value 0 :health 100 :message #"Index current\. more"}
+ (index-health/staleness-result 0 60 600 "more")))
+ (is (=? {:health 100} (index-health/staleness-result 60 60 600 nil))))
+ (testing "values at or above the critical threshold are degraded"
+ (is (=? {:health 0, :message #"Oldest pending change is 2\.8h old\."}
+ (index-health/staleness-result 9999 60 600 nil))))
+ (testing "health is linear between the warning and critical thresholds"
+ (is (=? {:health 50} (index-health/staleness-result 330 60 600 nil)))))
+
+(deftest ^:sequential run-measure!-test
+ (let [calls (atom [])
+ live (atom #{})]
+ (with-redefs [index-health/live-gauge-series live]
+ (mt/with-dynamic-fn-redefs [analytics/set-gauge! (fn [& args] (swap! calls conj (vec args)))]
+ (testing "a collector result updates the gauge and returns the health row"
+ (is (= {:health 75 :message "ok"}
+ (#'index-health/run-measure! {:gauge-key :metabase-search/index-coverage-ratio
+ :index :semantic
+ :collect (constantly {:value 0.75
+ :health 75
+ :message "ok"})})))
+ (is (= [[:metabase-search/index-coverage-ratio {:index "semantic"} 0.75]] @calls)))
+ (testing "an inapplicable collector clears a live gauge and returns nil"
+ (reset! calls [])
+ (is (nil? (#'index-health/run-measure! {:gauge-key :metabase-search/index-coverage-ratio
+ :index :semantic
+ :collect (constantly nil)})))
+ (is (= 1 (count @calls)))
+ (is (= [:metabase-search/index-coverage-ratio {:index "semantic"}] (butlast (first @calls))))
+ (is (Double/isNaN ^double (last (first @calls)))))
+ (testing "a throwing collector clears the gauge and returns a degraded row"
+ (#'index-health/run-measure! {:gauge-key :metabase-search/index-coverage-ratio
+ :index :throwing-collector-test
+ :collect (constantly {:value 0.9 :health 90 :message "was fine"})})
+ (reset! calls [])
+ (is (=? {:health 0, :message #"Metric collector errored: collector boom"}
+ (#'index-health/run-measure! {:gauge-key :metabase-search/index-coverage-ratio
+ :index :throwing-collector-test
+ :collect (fn []
+ (throw (ex-info "collector boom" {})))})))
+ (is (= [:metabase-search/index-coverage-ratio {:index "throwing-collector-test"}]
+ (butlast (first @calls))))
+ (is (Double/isNaN ^double (last (first @calls)))))))))
+
+(deftest run-measure!-propagates-interruption-test
+ (is (thrown? InterruptedException
+ (#'index-health/run-measure! {:gauge-key :metabase-search/index-coverage-ratio
+ :index :interrupted-test
+ :collect #(throw (InterruptedException.))}))))
+
+(deftest ^:sequential inapplicable-measure-does-not-create-series-test
+ (testing "an inapplicable measure does not create a NaN-only series"
+ (let [calls (atom [])]
+ (mt/with-dynamic-fn-redefs [analytics/set-gauge! (fn [& args] (swap! calls conj (vec args)))]
+ (#'index-health/run-measure! {:gauge-key :metabase-search/index-coverage-ratio
+ :index :never-emitted-test-engine
+ :collect (constantly nil)}))
+ (is (empty? @calls)))))
+
+(deftest ^:sequential failed-first-write-does-not-mark-series-live-test
+ (testing "a failed first gauge write does not make later clears create a NaN-only series"
+ (let [set-index-gauge! @#'index-health/set-index-gauge!
+ live @#'index-health/live-gauge-series
+ series [:metabase-search/index-coverage-ratio :failed-write-test-engine]
+ calls (atom [])]
+ (try
+ (mt/with-dynamic-fn-redefs [analytics/set-gauge! (fn [& _]
+ (throw (ex-info "prometheus down" {})))]
+ (is (thrown? Exception (apply set-index-gauge! (conj series 1.0))))
+ (is (not (contains? @live series))))
+ (mt/with-dynamic-fn-redefs [analytics/set-gauge! (fn [& args] (swap! calls conj (vec args)))]
+ (apply set-index-gauge! (conj series nil))
+ (is (empty? @calls))
+ (apply set-index-gauge! (conj series 0.5))
+ (is (contains? @live series)))
+ (finally
+ (swap! live disj series))))))
+
+(deftest ^:sequential refresh-isolates-measure-failures-test
+ (testing "one collector failure does not stop later measures from refreshing"
+ (let [calls (atom [])
+ live (atom #{})
+ boom {:check-name :test-boom
+ :gauge-key :metabase-search/index-staleness-seconds
+ :index :refresh-isolation-test
+ :collect (fn [] (throw (ex-info "collector boom" {})))}
+ ok {:check-name :test-ok
+ :gauge-key :metabase-search/index-coverage-ratio
+ :index :refresh-isolation-test
+ :collect (constantly {:value 1.0 :health 100 :message "ok"})}]
+ (with-redefs [index-health/live-gauge-series live]
+ (mt/with-dynamic-fn-redefs [analytics/set-gauge! (fn [& args] (swap! calls conj (vec args)))
+ health-inspector/enabled? (constantly false)]
+ (#'index-health/run-measure! (assoc boom :collect
+ (constantly {:value 5 :health 100 :message "was fine"})))
+ (reset! calls [])
+ (run! #'index-health/refresh-index-check! [boom ok])))
+ (is (=? [[:metabase-search/index-staleness-seconds {:index "refresh-isolation-test"}
+ (fn [v] (Double/isNaN ^double v))]
+ [:metabase-search/index-coverage-ratio {:index "refresh-isolation-test"} 1.0]]
+ @calls)))))
+
+(deftest ^:sequential pull-collector-refreshes-gauges-on-each-instance-test
+ (let [measures @#'index-health/index-measures
+ running? @#'index-health/gauge-refresh-running?
+ before @measures
+ seen (atom [])
+ submitted (atom [])
+ one {:check-name :one}
+ two {:check-name :two}
+ collector (analytics.core/pull-collector :metabase.search.index-health/index-health-gauges)]
+ (try
+ (reset! measures {:one one, :two two})
+ (reset! running? false)
+ (mt/with-dynamic-fn-redefs
+ [index-health/run-measure! #(swap! seen conj %)
+ index-health/submit-gauge-refresh! #(swap! submitted conj %)
+ health-inspector/save-check-result! #(throw (ex-info "must not persist from a scrape" {}))]
+ ((:f collector))
+ ((:f collector))
+ (is (empty? @seen) "the scrape path does not run index scans")
+ (is (= 1 (count @submitted)) "overlapping scrapes share one refresh")
+ ((first @submitted)))
+ (is (= 600 (:min-interval-s collector)))
+ (is (= #{one two} (set @seen)))
+ (is (false? @running?))
+ (finally
+ (reset! running? false)
+ (reset! measures before)))))
+
+(deftest ^:sequential background-gauge-refresh-isolates-write-failures-test
+ (let [measures (atom {:one {:check-name :one}, :two {:check-name :two}})
+ running? (atom true)
+ seen (atom [])]
+ (with-redefs [index-health/index-measures measures
+ index-health/gauge-refresh-running? running?]
+ (mt/with-dynamic-fn-redefs
+ [index-health/run-measure! (fn [{:keys [check-name]}]
+ (swap! seen conj check-name)
+ (when (= :one check-name)
+ (throw (ex-info "prometheus down" {}))))]
+ (#'index-health/refresh-search-index-gauges!)))
+ (is (= #{:one :two} (set @seen)))
+ (is (false? @running?)))
+ (testing "interruption still escapes the per-descriptor boundary"
+ (is (thrown? InterruptedException
+ (#'index-health/refresh-index-gauge! {:check-name :interrupted
+ :gauge-key :metabase-search/index-coverage-ratio
+ :index :interrupted
+ :collect #(throw (InterruptedException.))})))))