Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Harden health inspector check persistence (#78295)

Chris Truter26c003f858a4
diff --git a/resources/migrations/064/20260721_health_inspector_run_indexes.yaml b/resources/migrations/064/20260721_health_inspector_run_indexes.yaml
new file mode 100644
index 00000000000..85acfe690d4
--- /dev/null
+++ b/resources/migrations/064/20260721_health_inspector_run_indexes.yaml
@@ -0,0 +1,28 @@
+databaseChangeLog:
+ - changeSet:
+ id: v64.2026-07-21T12:00:00
+ author: christruter
+ comment: Index latest health-inspector check lookups
+ changes:
+ - createIndex:
+ tableName: health_inspector_runs
+ indexName: idx_health_inspector_runs_check_time
+ columns:
+ - column:
+ name: check_name
+ - column:
+ name: run_at
+ - column:
+ name: id
+
+ - changeSet:
+ id: v64.2026-07-21T12:01:00
+ author: christruter
+ comment: Index health-inspector retention cleanup
+ changes:
+ - createIndex:
+ tableName: health_inspector_runs
+ indexName: idx_health_inspector_runs_run_at
+ columns:
+ - column:
+ name: run_at
diff --git a/src/metabase/health_inspector/core.clj b/src/metabase/health_inspector/core.clj
index dc7e8dd66fd..98bce819804 100644
--- a/src/metabase/health_inspector/core.clj
+++ b/src/metabase/health_inspector/core.clj
@@ -3,12 +3,14 @@
[clojurewerkz.quartzite.jobs :as jobs]
[clojurewerkz.quartzite.schedule.cron :as cron]
[clojurewerkz.quartzite.triggers :as triggers]
+ [java-time.api :as t]
[metabase.health-inspector.settings :as setting]
[metabase.lib-be.core :as lib-be]
[metabase.lib.schema :as schema]
[metabase.task.core :as task]
[metabase.util :as u]
[metabase.util.json :as json]
+ [metabase.util.log :as log]
[metabase.util.malli.registry :as mr]
[toucan2.core :as t2]))
@@ -42,25 +44,95 @@
(defonce ^:private checks (atom {:validate-queries validate-queries}))
(defn register-check!
- "Register a new check function with a given name.
- Check functions take no args and return a map with a :health ratio and :description string."
+ "Register check `check-fn` under `name`.
+ A check takes no args and returns `{:health <int 0-100>, :message <string>}`, or nil when it doesn't
+ apply on this instance."
[name check-fn]
(swap! checks assoc name check-fn))
+(defn enabled?
+ "Whether the health inspector is enabled -- the master switch for persisting runs. Metric emitters that also
+ feed Prometheus (independent of this setting) use it to gate only their appdb persistence."
+ []
+ (setting/health-inspector-enabled))
+
+(defn- run-check
+ "Run one check in isolation. Ordinary failures read as degraded; interruption and fatal errors propagate."
+ [check-name f]
+ (try
+ (f)
+ (catch InterruptedException e
+ (throw e))
+ (catch Exception e
+ (log/error e "Health check errored" {:check check-name})
+ {:health 0, :message (str "Health check errored: " (ex-message e))})))
+
(defn report
- "Run all registered checks and produce a report describing potential problems."
+ "Run all registered checks and produce a report describing potential problems.
+ Each check is isolated, so one that throws doesn't lose the others' results."
[]
(into {} (for [[name f] @checks]
- [name (f)])))
+ [name (run-check name f)])))
-(defn save-report
- "Run a health inspector report and save it to the DB."
- []
- (doseq [[check-name result] (report)]
+(defn- persist-check-result!
+ "Persist one check result, or nothing when `result` is nil.
+ A nil result marks a check as not applicable on this instance, so it is omitted rather than recorded as
+ a misleading healthy score."
+ [check-name result]
+ (when (some? result)
(t2/insert! :health_inspector_runs (-> result
(select-keys [:health :message])
(assoc :check_name (name check-name))))))
+(def ^:private run-retention-days
+ "Days of health-inspector runs to keep. Metric checks embed changing values (coverage %, ages) in their
+ `:message`, so dedup can't collapse them and the table would grow unbounded without pruning."
+ 30)
+
+(defn- delete-old-runs!
+ "Delete health-inspector runs older than [[run-retention-days]]."
+ []
+ (t2/delete! :health_inspector_runs
+ :run_at [:< (t/minus (t/offset-date-time) (t/days run-retention-days))]))
+
+(defn save-report
+ "Run every registered check and persist the results (not-applicable (nil) checks are omitted), then prune
+ runs past the retention window."
+ []
+ (doseq [[check-name result] (report)]
+ (persist-check-result! check-name result))
+ (delete-old-runs!))
+
+(defn- latest-run
+ [check-name]
+ ;; Tie-break on id: back-to-back inserts can share a run_at, and run_at alone would then pick a
+ ;; non-deterministic row, breaking the dedup below (id is a monotonic auto-increment PK).
+ (t2/select-one [:health_inspector_runs :health :message] :check_name (name check-name)
+ {:order-by [[:run_at :desc] [:id :desc]]}))
+
+(defn save-check-result!
+ "Persist a precomputed check `result` (a `{:health :message}` map, or nil to skip). Skip the write when the
+ check's most recent result is unchanged. This is a best-effort optimization, not cross-node coordination.
+ Does NOT gate on `health-inspector-enabled` -- callers that emit results outside a check run (e.g. a
+ metric refresh that also feeds Prometheus) gate themselves."
+ [check-name result]
+ (when (some? result)
+ (let [prev (latest-run check-name)]
+ (when-not (and prev
+ (= (:health prev) (:health result))
+ (= (:message prev) (:message result)))
+ (persist-check-result! check-name result)))))
+
+(defn run-and-save-check!
+ "Run one registered check by name and persist its result, so a change can be surfaced immediately rather
+ than at the next daily report.
+ A no-op unless the health inspector is enabled and the named check is registered and applicable (non-nil).
+ Uses [[save-check-result!]] to avoid writing unchanged results in the usual single-caller case."
+ [check-name]
+ (when (enabled?)
+ (when-let [f (get @checks check-name)]
+ (save-check-result! check-name (run-check check-name f)))))
+
(defn list-runs
"Return the most recent health inspector runs from the DB."
[limit]
diff --git a/test/metabase/health_inspector/core_test.clj b/test/metabase/health_inspector/core_test.clj
index bac1fce2bc9..3584ec145ae 100644
--- a/test/metabase/health_inspector/core_test.clj
+++ b/test/metabase/health_inspector/core_test.clj
@@ -2,6 +2,7 @@
(:require
[clojure.test :refer :all]
[metabase.health-inspector.core :as hi]
+ [metabase.test :as mt]
[metabase.util.json :as json]
[toucan2.core :as t2]))
@@ -42,3 +43,53 @@
(is (= [100 100] (map :health (runs "test-check"))))
(is (= ["test check" "test check"] (map :message (runs "test-check"))))
(is (= [100 100] (map :health (runs "validate-queries"))))))
+
+;; The test-only checks below are registered into a fresh, test-scoped `checks` registry via with-redefs
+;; rather than the global one, so a nil/throwing/probe check can't leak into other health-inspector tests in
+;; the same JVM and make them order-dependent.
+
+(deftest nil-check-is-omitted-test
+ (testing "a check returning nil (not applicable on this instance) is omitted, not persisted as a green score"
+ (with-redefs [hi/checks (atom {:test-nil (constantly nil)
+ :ok (constantly {:health 100 :message "ok"})})]
+ (is (nil? (:test-nil (hi/report))))
+ (t2/delete! :health_inspector_runs)
+ (hi/save-report)
+ (is (= #{"ok"} (set (map :check_name (hi/list-runs 512))))
+ "only the applicable check is persisted"))))
+
+(deftest report-isolates-throwing-check-test
+ (testing "a throwing check reads as degraded and doesn't abort the rest of the report"
+ (with-redefs [hi/checks (atom {:boom (fn [] (throw (ex-info "kaboom" {})))
+ :test-check test-check})]
+ (let [report (hi/report)]
+ (is (=? {:health 0 :message #".*kaboom.*"} (:boom report)))
+ (is (= 100 (-> report :test-check :health)) "other checks still run")))))
+
+(deftest report-propagates-interruption-test
+ (with-redefs [hi/checks (atom {:interrupted #(throw (InterruptedException.))})]
+ (is (thrown? InterruptedException (hi/report)))))
+
+(deftest run-and-save-check!-test
+ (let [probe-result (atom {:health 0 :message "down"})]
+ (with-redefs [hi/checks (atom {:test-probe (fn [] @probe-result)})]
+ (testing "no-op when the health inspector is disabled"
+ (mt/with-temporary-setting-values [health-inspector-enabled false]
+ (t2/delete! :health_inspector_runs)
+ (hi/run-and-save-check! :test-probe)
+ (is (empty? (hi/list-runs 512)))))
+ (mt/with-temporary-setting-values [health-inspector-enabled true]
+ (testing "no-op for an unregistered check"
+ (t2/delete! :health_inspector_runs)
+ (hi/run-and-save-check! :no-such-check)
+ (is (empty? (hi/list-runs 512))))
+ (testing "persists once, then dedups an unchanged result"
+ (reset! probe-result {:health 0 :message "down"})
+ (t2/delete! :health_inspector_runs)
+ (hi/run-and-save-check! :test-probe)
+ (hi/run-and-save-check! :test-probe)
+ (is (= 1 (count (hi/list-runs 512))) "the unchanged repeat is deduped"))
+ (testing "a changed result is persisted"
+ (reset! probe-result {:health 100 :message "up"})
+ (hi/run-and-save-check! :test-probe)
+ (is (= 2 (count (hi/list-runs 512))) "a changed result is persisted"))))))