The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Add shared search index health metrics (#78297)
Chris Truter8112036b95d5
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.ednindex abefbfc5e6c..e456148a55d 100644--- a/.clj-kondo/config/modules/config.edn+++ b/.clj-kondo/config/modules/config.edn@@ -1048,6 +1048,7 @@health-inspector{:team "Querying Platform":api #{metabase.health-inspector.api+ metabase.health-inspector.coremetabase.health-inspector.init}:uses #{apilib@@ -2016,6 +2017,7 @@metabase.search.coremetabase.search.enginemetabase.search.in-place.scoring+ metabase.search.index-healthmetabase.search.ingestionmetabase.search.initmetabase.search.scoring@@ -2030,6 +2032,7 @@collectionsconfigevents+ health-inspectorliblib-bemodelsdiff --git a/enterprise/backend/src/metabase_enterprise/semantic_search/task/metric_collector.clj b/enterprise/backend/src/metabase_enterprise/semantic_search/task/metric_collector.cljindex b2e45e054c4..8eb519ddc92 100644--- a/enterprise/backend/src/metabase_enterprise/semantic_search/task/metric_collector.clj+++ b/enterprise/backend/src/metabase_enterprise/semantic_search/task/metric_collector.clj@@ -4,10 +4,12 @@[clojurewerkz.quartzite.schedule.simple :as simple][clojurewerkz.quartzite.triggers :as triggers][honey.sql :as sql]+ [metabase-enterprise.semantic-search.db.datasource :as semantic.datasource][metabase-enterprise.semantic-search.dlq :as semantic.dlq][metabase-enterprise.semantic-search.env :as semantic.env][metabase-enterprise.semantic-search.util :as semantic.u][metabase.analytics-interface.core :as analytics]+ [metabase.search.index-health :as search.index-health][metabase.task.core :as task][metabase.util.log :as log][next.jdbc :as jdbc]@@ -65,13 +67,21 @@(log/warn "DLQ table does not exist. Index may not have been initialized.")))(defn- collect-metrics! []- ;; Active, not merely available: on an available-but-inactive instance the index tables never exist,- ;; and the collectors would warn about them on every run.- (when (semantic.u/semantic-search-active?)- (let [pgvector (semantic.env/get-pgvector-datasource!)- index-metadata (semantic.env/get-index-metadata)]- (collect-gate-size! pgvector)- (collect-dlq-size! pgvector index-metadata))))+ (try+ ;; Active, not merely available: on an available-but-inactive instance the index tables never exist,+ ;; and the collectors would warn about them on every run.+ (when (semantic.u/semantic-search-active?)+ (let [pgvector (semantic.env/get-pgvector-datasource!)+ index-metadata (semantic.env/get-index-metadata)]+ (collect-gate-size! pgvector)+ (collect-dlq-size! pgvector index-metadata)))+ (catch InterruptedException e+ (throw e))+ (catch Exception e+ (log/error e "Semantic search metric collector errored")))+ ;; Registered collectors self-gate, so refreshing them is a cheap no-op when their feature is off.+ ;; Keep this outside the try: ordinary failures continue here, while interruption and fatal errors exit.+ (search.index-health/refresh-search-index-metrics!))(task/defjob ^{DisallowConcurrentExecution true:doc "Collect expensive semantic search metrics"}@@ -82,7 +92,13 @@(defmethod task/init! ::SemanticMetricCollector[_]- (when (semantic.u/semantic-search-configured?)+ ;; Boot-safe gate: plain env/feature checks, never a DB probe (pgvector-configured? would resolve+ ;; pgvector-mode, probing the app db and logging a pgvector-store line on instances that can't use the+ ;; answer). The dedicated-URL arm schedules without the feature, so a token entered post-boot starts the+ ;; search-index gauges without a restart; app-db-pgvector instances licensed post-boot need one, matching the+ ;; other semantic tasks (see [[semantic.u/semantic-search-configured?]]).+ (if (or (semantic.datasource/dedicated-url-configured?)+ (semantic.u/semantic-search-configured?))(let [job (jobs/build(jobs/of-type SemanticMetricCollector)(jobs/with-identity collector-job-key))@@ -94,4 +110,7 @@(simple/with-interval-in-milliseconds job-interval-ms)(simple/repeat-forever)))(triggers/start-now))]- (task/schedule-task! job trigger))))+ (task/schedule-task! job trigger))+ ;; Quartz's job store is persistent, so a collector scheduled by an earlier deploy would otherwise+ ;; keep firing (as a no-op) on an instance whose configuration went away.+ (task/delete-task! collector-job-key collector-trigger-key)))diff --git a/enterprise/backend/test/metabase_enterprise/semantic_search/task/metric_collector_test.clj b/enterprise/backend/test/metabase_enterprise/semantic_search/task/metric_collector_test.cljindex f416db3c3bf..a5cf3401da5 100644--- a/enterprise/backend/test/metabase_enterprise/semantic_search/task/metric_collector_test.clj+++ b/enterprise/backend/test/metabase_enterprise/semantic_search/task/metric_collector_test.clj@@ -12,6 +12,7 @@[metabase-enterprise.semantic-search.task.metric-collector :as semantic.task.collector][metabase-enterprise.semantic-search.test-util :as semantic.tu][metabase-enterprise.semantic-search.util :as semantic.u]+ [metabase.search.index-health :as search.index-health][metabase.test :as mt][next.jdbc :as jdbc]))@@ -66,6 +67,24 @@pgvector(sql/format {:delete-from [[:raw (:gate-table-name index-metadata)]]})))+(deftest shared-index-metrics-survive-semantic-collector-failure-test+ (let [refreshes (atom 0)]+ (mt/with-dynamic-fn-redefs+ [semantic.u/semantic-search-active? (constantly true)+ semantic.env/get-pgvector-datasource! #(throw (ex-info "pgvector unavailable" {}))+ search.index-health/refresh-search-index-metrics! #(swap! refreshes inc)]+ (@#'semantic.task.collector/collect-metrics!)+ (is (= 1 @refreshes)))))++(deftest interrupted-semantic-collector-skips-shared-refresh-test+ (let [refreshes (atom 0)]+ (mt/with-dynamic-fn-redefs+ [semantic.u/semantic-search-active? (constantly true)+ semantic.env/get-pgvector-datasource! #(throw (InterruptedException.))+ search.index-health/refresh-search-index-metrics! #(swap! refreshes inc)]+ (is (thrown? InterruptedException (@#'semantic.task.collector/collect-metrics!)))+ (is (zero? @refreshes)))))+(deftest metric-collector-test(mt/with-premium-features #{:semantic-search}(mt/with-prometheus-system! [_ system]diff --git a/src/metabase/analytics/prometheus.clj b/src/metabase/analytics/prometheus.cljindex 75d2fcd102a..94f881f1c71 100644--- a/src/metabase/analytics/prometheus.clj+++ b/src/metabase/analytics/prometheus.clj@@ -479,6 +479,20 @@{:description "Number of documents in the library entity index, as of the last full reconcile."})(prometheus/gauge :metabase-entity-retrieval/index-entities{:description "Number of distinct entities in the library entity index, as of the last full reconcile."})+ ;; Search-index health, one series per logical index. Implementations publish these through+ ;; metabase.search.index-health; labels identify stable logical indexes, never physical table names.+ (prometheus/gauge :metabase-search/index-coverage-ratio+ {:description (str "Fraction (0-1) of the items that should be indexed that actually are, "+ "per search index.")+ :labels [:index]})+ (prometheus/gauge :metabase-search/index-garbage-count+ {:description (str "Absolute number of indexed items that should not be indexed "+ "(orphaned or no longer a candidate), per search index.")+ :labels [:index]})+ (prometheus/gauge :metabase-search/index-staleness-seconds+ {:description (str "Age in seconds of the oldest known-pending change not yet reflected "+ "in the index (indexer/reconcile backlog), per search index.")+ :labels [:index]});; data-complexity-score timing;; 1ms → 1min buckets; widen later if real-world runs push past a minute.(prometheus/histogram :metabase-data-complexity/scoring-duration-msdiff --git a/src/metabase/search/index_health.clj b/src/metabase/search/index_health.cljnew file mode 100644index 00000000000..55934b33150--- /dev/null+++ b/src/metabase/search/index_health.clj@@ -0,0 +1,193 @@+(ns metabase.search.index-health+ "Shared health-result and metric plumbing for search indexes. Index implementations register raw coverage,+ garbage, and staleness collectors here; this namespace publishes their Prometheus gauges and health-inspector rows."+ (:require+ [metabase.analytics-interface.core :as analytics]+ [metabase.analytics.core :as analytics.core]+ [metabase.health-inspector.core :as health-inspector]+ [metabase.util.log :as log]))++(set! *warn-on-reflection* true)++(defn healthy+ "A healthy (100) check result with `message`."+ [message]+ {:health 100, :message message})++(defn warning+ "A partially healthy check result. `health` defaults to 50 and must be strictly between 0 and 100; use+ [[healthy]] or [[degraded]] for the endpoints."+ ([message]+ (warning 50 message))+ ([health message]+ {:pre [(< 0 health 100)]}+ {:health health, :message message}))++(defn degraded+ "A degraded (0) check result with `message`."+ [message]+ {:health 0, :message message})++(def ^:private measure->gauge+ {:coverage :metabase-search/index-coverage-ratio+ :garbage :metabase-search/index-garbage-count+ :staleness :metabase-search/index-staleness-seconds})++(defn- percentage+ [ratio]+ (cond+ (<= ratio 0) 0+ (>= ratio 1) 100+ :else (-> (* 100.0 (double ratio)) Math/round (max 1) (min 99))))++(defn- threshold-health+ "Health from an absolute `value`: 100 at or below `warn`, 0 at or above `critical`, and linear between."+ [value warn critical]+ (cond+ (<= value warn) 100+ (>= value critical) 0+ :else (-> (* 100.0 (/ (double (- critical value)) (- critical warn)))+ Math/round+ (max 1)+ (min 99))))++(defn coverage-result+ "Result for `indexed` out of `expected` items. The ratio feeds Prometheus; its percentage is the health,+ with 0 and 100 reserved for exact endpoints."+ [indexed expected]+ (let [ratio (if (pos? expected) (min 1.0 (/ (double indexed) expected)) 1.0)+ health (percentage ratio)]+ {:value ratio+ :health health+ :message (format "%d of %d expected items indexed (%d%%)." indexed expected health)}))++(defn garbage-result+ "Result for an absolute `orphans` count, scored against `warn` and `critical` thresholds."+ [orphans warn critical]+ {:value orphans+ :health (threshold-health orphans warn critical)+ :message (if (zero? orphans)+ "No orphaned items in the index."+ (format "%d orphaned item(s) in the index." orphans))})++(defn- describe-age [seconds]+ (let [seconds (long seconds)]+ (cond+ (>= seconds 3600) (format "%.1fh" (/ seconds 3600.0))+ (>= seconds 60) (format "%dm" (quot seconds 60))+ :else (format "%ds" seconds))))++(defn staleness-result+ "Result for the oldest pending change. `warn` and `critical` are seconds; `detail` is an optional clause."+ [age-seconds warn critical detail]+ (let [age (long (or age-seconds 0))+ base (if (zero? age)+ "Index current."+ (format "Oldest pending change is %s old." (describe-age age)))]+ {:value age+ :health (threshold-health age warn critical)+ :message (if detail (str base " " detail) base)}))++(defonce ^:private index-measures+ ;; Keying by check name makes namespace reloads replace descriptors instead of duplicating collectors.+ (atom {}))++(defonce ^:private live-gauge-series+ ;; Only clear series that previously held a real value; inactive indexes should not create NaN-only series.+ (atom #{}))++(defn- set-index-gauge!+ [gauge-key index value]+ (if (some? value)+ (do+ (analytics/set-gauge! gauge-key {:index (name index)} value)+ (swap! live-gauge-series conj [gauge-key index]))+ (when (contains? @live-gauge-series [gauge-key index])+ (analytics/set-gauge! gauge-key {:index (name index)} ##NaN))))++(defn- run-measure!+ "Run one collector and update its gauge. Returns nil for N/A or a health-inspector result."+ [{:keys [gauge-key index collect check-name]}]+ (let [{:keys [value health message]}+ (try+ (collect)+ (catch InterruptedException e+ (throw e))+ (catch Exception e+ (log/error e "Search index metric collector errored" {:check check-name})+ {:health 0, :message (str "Metric collector errored: " (ex-message e))}))]+ (set-index-gauge! gauge-key index value)+ (when health+ {:health health, :message message})))++(defn register-index-check!+ "Register a collector for logical `index` and `measure`. The collector returns nil for N/A or+ `{:value :health :message}`. Returns a descriptor accepted by [[refresh-index-check!]]."+ [index measure collect]+ (let [descriptor {:check-name (keyword (str (name index) "-" (name measure)))+ :gauge-key (measure->gauge measure)+ :index index+ :measure measure+ :collect collect}]+ (health-inspector/register-check! (:check-name descriptor) #(run-measure! descriptor))+ ;; A live upgrade may leave the defonce'd registry in its former vector representation.+ (swap! index-measures (fn [measures]+ (let [keyed (if (map? measures)+ measures+ (into {} (map (juxt :check-name identity)) measures))]+ (assoc keyed (:check-name descriptor) descriptor))))+ descriptor))++(defn refresh-index-check!+ "Refresh one registered descriptor. Updates its gauge and, when enabled, its deduplicated health row."+ [{:keys [check-name] :as descriptor}]+ (try+ (when-let [result (run-measure! descriptor)]+ (when (health-inspector/enabled?)+ (health-inspector/save-check-result! check-name result)))+ (catch InterruptedException e+ (throw e))+ (catch Exception e+ (log/error e "Search index health-row persist errored" {:check check-name}))))++(defonce ^:private gauge-refresh-running? (atom false))++(defn- submit-gauge-refresh! [f]+ (future-call f))++(defn- refresh-index-gauge! [{:keys [check-name] :as descriptor}]+ (try+ (run-measure! descriptor)+ (catch InterruptedException e+ (throw e))+ (catch Exception e+ (log/error e "Search index gauge refresh errored" {:check check-name}))))++(defn- refresh-search-index-gauges!+ []+ (try+ (run! refresh-index-gauge! (vals @index-measures))+ (finally+ (reset! gauge-refresh-running? false))))++(defn- request-search-index-gauge-refresh!+ []+ (when (compare-and-set! gauge-refresh-running? false true)+ (try+ (submit-gauge-refresh! refresh-search-index-gauges!)+ (catch Exception e+ (reset! gauge-refresh-running? false)+ (log/error e "Could not schedule search index gauge refresh"))))+ nil)++;; Prometheus scrapes every Metabase process, whereas the scheduled metric job may run on only one member of+;; a Quartz cluster. A scrape starts a local, single-flight background refresh so every process updates its+;; series without putting index scans on the synchronous scrape path.+(defmethod analytics.core/pull-collector ::index-health-gauges [_]+ {:min-interval-s 600+ :f request-search-index-gauge-refresh!})++(defn refresh-search-index-metrics!+ "Refresh every registered measure's gauge and, when enabled, its deduplicated health row."+ []+ (run! refresh-index-check! (vals @index-measures)))diff --git a/test/metabase/search/index_health_test.clj b/test/metabase/search/index_health_test.cljnew file mode 100644index 00000000000..c21ff111fe4--- /dev/null+++ b/test/metabase/search/index_health_test.clj@@ -0,0 +1,203 @@+(ns metabase.search.index-health-test+ (:require+ [clojure.test :refer :all]+ [metabase.analytics-interface.core :as analytics]+ [metabase.analytics.core :as analytics.core]+ [metabase.health-inspector.core :as health-inspector]+ [metabase.search.index-health :as index-health]+ [metabase.test :as mt]))++(set! *warn-on-reflection* true)++(deftest ^:sequential register-index-check!-migrates-a-legacy-registry-test+ (testing "a live upgrade migrates the defonce'd registry's former vector value"+ (let [measures (atom [{:check-name :legacy-measure, :collect (constantly nil)}])+ checks (atom {})]+ (with-redefs [index-health/index-measures measures+ health-inspector/checks checks]+ (let [{:keys [check-name]}+ (index-health/register-index-check! :test-migration :coverage (constantly nil))]+ (is (= #{:legacy-measure check-name} (set (keys @measures))))+ (is (contains? @checks check-name)))))))++(deftest ^:parallel coverage-result-test+ (testing "coverage health is the percentage; the ratio feeds the gauge"+ (is (=? {:value 0.5 :health 50 :message #"5 of 10 expected items indexed \(50%\)\."}+ (index-health/coverage-result 5 10))))+ (testing "an empty candidate set is fully covered"+ (is (=? {:value 1.0 :health 100 :message #"0 of 0 .*100%.*"}+ (index-health/coverage-result 0 0))))+ (testing "coverage clamps over-counts to 100 percent"+ (is (=? {:value 1.0 :health 100} (index-health/coverage-result 12 10))))+ (testing "only exact endpoints score 0 or 100"+ (is (=? {:health 99} (index-health/coverage-result 999 1000)))+ (is (=? {:health 1} (index-health/coverage-result 1 1000)))+ (is (=? {:health 0} (index-health/coverage-result 0 1000)))))++(deftest ^:parallel garbage-result-test+ (testing "zero orphans is healthy"+ (is (=? {:value 0 :health 100 :message #"No orphaned items.*"}+ (index-health/garbage-result 0 5 100))))+ (testing "the value is an absolute count"+ (is (=? {:value 42 :message #"42 orphaned item\(s\) in the index\."}+ (index-health/garbage-result 42 5 100))))+ (testing "health is linear between the warning and critical thresholds"+ (is (=? {:health 100} (index-health/garbage-result 5 5 100)))+ (is (=? {:health 0} (index-health/garbage-result 100 5 100)))+ (is (=? {:health 50} (index-health/garbage-result 52 4 100))))+ (testing "intermediate values do not round to an endpoint"+ (is (=? {:health 99} (index-health/garbage-result 1 0 1000)))+ (is (=? {:health 1} (index-health/garbage-result 999 0 1000)))))++(deftest ^:parallel staleness-result-test+ (testing "values at or below the warning threshold are healthy"+ (is (=? {:value 0 :health 100 :message #"Index current\. more"}+ (index-health/staleness-result 0 60 600 "more")))+ (is (=? {:health 100} (index-health/staleness-result 60 60 600 nil))))+ (testing "values at or above the critical threshold are degraded"+ (is (=? {:health 0, :message #"Oldest pending change is 2\.8h old\."}+ (index-health/staleness-result 9999 60 600 nil))))+ (testing "health is linear between the warning and critical thresholds"+ (is (=? {:health 50} (index-health/staleness-result 330 60 600 nil)))))++(deftest ^:sequential run-measure!-test+ (let [calls (atom [])+ live (atom #{})]+ (with-redefs [index-health/live-gauge-series live]+ (mt/with-dynamic-fn-redefs [analytics/set-gauge! (fn [& args] (swap! calls conj (vec args)))]+ (testing "a collector result updates the gauge and returns the health row"+ (is (= {:health 75 :message "ok"}+ (#'index-health/run-measure! {:gauge-key :metabase-search/index-coverage-ratio+ :index :semantic+ :collect (constantly {:value 0.75+ :health 75+ :message "ok"})})))+ (is (= [[:metabase-search/index-coverage-ratio {:index "semantic"} 0.75]] @calls)))+ (testing "an inapplicable collector clears a live gauge and returns nil"+ (reset! calls [])+ (is (nil? (#'index-health/run-measure! {:gauge-key :metabase-search/index-coverage-ratio+ :index :semantic+ :collect (constantly nil)})))+ (is (= 1 (count @calls)))+ (is (= [:metabase-search/index-coverage-ratio {:index "semantic"}] (butlast (first @calls))))+ (is (Double/isNaN ^double (last (first @calls)))))+ (testing "a throwing collector clears the gauge and returns a degraded row"+ (#'index-health/run-measure! {:gauge-key :metabase-search/index-coverage-ratio+ :index :throwing-collector-test+ :collect (constantly {:value 0.9 :health 90 :message "was fine"})})+ (reset! calls [])+ (is (=? {:health 0, :message #"Metric collector errored: collector boom"}+ (#'index-health/run-measure! {:gauge-key :metabase-search/index-coverage-ratio+ :index :throwing-collector-test+ :collect (fn []+ (throw (ex-info "collector boom" {})))})))+ (is (= [:metabase-search/index-coverage-ratio {:index "throwing-collector-test"}]+ (butlast (first @calls))))+ (is (Double/isNaN ^double (last (first @calls)))))))))++(deftest run-measure!-propagates-interruption-test+ (is (thrown? InterruptedException+ (#'index-health/run-measure! {:gauge-key :metabase-search/index-coverage-ratio+ :index :interrupted-test+ :collect #(throw (InterruptedException.))}))))++(deftest ^:sequential inapplicable-measure-does-not-create-series-test+ (testing "an inapplicable measure does not create a NaN-only series"+ (let [calls (atom [])]+ (mt/with-dynamic-fn-redefs [analytics/set-gauge! (fn [& args] (swap! calls conj (vec args)))]+ (#'index-health/run-measure! {:gauge-key :metabase-search/index-coverage-ratio+ :index :never-emitted-test-engine+ :collect (constantly nil)}))+ (is (empty? @calls)))))++(deftest ^:sequential failed-first-write-does-not-mark-series-live-test+ (testing "a failed first gauge write does not make later clears create a NaN-only series"+ (let [set-index-gauge! @#'index-health/set-index-gauge!+ live @#'index-health/live-gauge-series+ series [:metabase-search/index-coverage-ratio :failed-write-test-engine]+ calls (atom [])]+ (try+ (mt/with-dynamic-fn-redefs [analytics/set-gauge! (fn [& _]+ (throw (ex-info "prometheus down" {})))]+ (is (thrown? Exception (apply set-index-gauge! (conj series 1.0))))+ (is (not (contains? @live series))))+ (mt/with-dynamic-fn-redefs [analytics/set-gauge! (fn [& args] (swap! calls conj (vec args)))]+ (apply set-index-gauge! (conj series nil))+ (is (empty? @calls))+ (apply set-index-gauge! (conj series 0.5))+ (is (contains? @live series)))+ (finally+ (swap! live disj series))))))++(deftest ^:sequential refresh-isolates-measure-failures-test+ (testing "one collector failure does not stop later measures from refreshing"+ (let [calls (atom [])+ live (atom #{})+ boom {:check-name :test-boom+ :gauge-key :metabase-search/index-staleness-seconds+ :index :refresh-isolation-test+ :collect (fn [] (throw (ex-info "collector boom" {})))}+ ok {:check-name :test-ok+ :gauge-key :metabase-search/index-coverage-ratio+ :index :refresh-isolation-test+ :collect (constantly {:value 1.0 :health 100 :message "ok"})}]+ (with-redefs [index-health/live-gauge-series live]+ (mt/with-dynamic-fn-redefs [analytics/set-gauge! (fn [& args] (swap! calls conj (vec args)))+ health-inspector/enabled? (constantly false)]+ (#'index-health/run-measure! (assoc boom :collect+ (constantly {:value 5 :health 100 :message "was fine"})))+ (reset! calls [])+ (run! #'index-health/refresh-index-check! [boom ok])))+ (is (=? [[:metabase-search/index-staleness-seconds {:index "refresh-isolation-test"}+ (fn [v] (Double/isNaN ^double v))]+ [:metabase-search/index-coverage-ratio {:index "refresh-isolation-test"} 1.0]]+ @calls)))))++(deftest ^:sequential pull-collector-refreshes-gauges-on-each-instance-test+ (let [measures @#'index-health/index-measures+ running? @#'index-health/gauge-refresh-running?+ before @measures+ seen (atom [])+ submitted (atom [])+ one {:check-name :one}+ two {:check-name :two}+ collector (analytics.core/pull-collector :metabase.search.index-health/index-health-gauges)]+ (try+ (reset! measures {:one one, :two two})+ (reset! running? false)+ (mt/with-dynamic-fn-redefs+ [index-health/run-measure! #(swap! seen conj %)+ index-health/submit-gauge-refresh! #(swap! submitted conj %)+ health-inspector/save-check-result! #(throw (ex-info "must not persist from a scrape" {}))]+ ((:f collector))+ ((:f collector))+ (is (empty? @seen) "the scrape path does not run index scans")+ (is (= 1 (count @submitted)) "overlapping scrapes share one refresh")+ ((first @submitted)))+ (is (= 600 (:min-interval-s collector)))+ (is (= #{one two} (set @seen)))+ (is (false? @running?))+ (finally+ (reset! running? false)+ (reset! measures before)))))++(deftest ^:sequential background-gauge-refresh-isolates-write-failures-test+ (let [measures (atom {:one {:check-name :one}, :two {:check-name :two}})+ running? (atom true)+ seen (atom [])]+ (with-redefs [index-health/index-measures measures+ index-health/gauge-refresh-running? running?]+ (mt/with-dynamic-fn-redefs+ [index-health/run-measure! (fn [{:keys [check-name]}]+ (swap! seen conj check-name)+ (when (= :one check-name)+ (throw (ex-info "prometheus down" {}))))]+ (#'index-health/refresh-search-index-gauges!)))+ (is (= #{:one :two} (set @seen)))+ (is (false? @running?)))+ (testing "interruption still escapes the per-descriptor boundary"+ (is (thrown? InterruptedException+ (#'index-health/refresh-index-gauge! {:check-name :interrupted+ :gauge-key :metabase-search/index-coverage-ratio+ :index :interrupted+ :collect #(throw (InterruptedException.))})))))