The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:
TypeScript analyticsbibusiness-intelligencebusinessintelligenceclojuredashboarddatadata-analysisdata-visualizationdatabasemetabasemysqlpostgrespostgresqlreportingslacksql-editorvisualization
Harden health inspector check persistence (#78295)
Chris Truter26c003f858a4
diff --git a/resources/migrations/064/20260721_health_inspector_run_indexes.yaml b/resources/migrations/064/20260721_health_inspector_run_indexes.yamlnew file mode 100644index 00000000000..85acfe690d4--- /dev/null+++ b/resources/migrations/064/20260721_health_inspector_run_indexes.yaml@@ -0,0 +1,28 @@+databaseChangeLog:+ - changeSet:+ id: v64.2026-07-21T12:00:00+ author: christruter+ comment: Index latest health-inspector check lookups+ changes:+ - createIndex:+ tableName: health_inspector_runs+ indexName: idx_health_inspector_runs_check_time+ columns:+ - column:+ name: check_name+ - column:+ name: run_at+ - column:+ name: id++ - changeSet:+ id: v64.2026-07-21T12:01:00+ author: christruter+ comment: Index health-inspector retention cleanup+ changes:+ - createIndex:+ tableName: health_inspector_runs+ indexName: idx_health_inspector_runs_run_at+ columns:+ - column:+ name: run_atdiff --git a/src/metabase/health_inspector/core.clj b/src/metabase/health_inspector/core.cljindex dc7e8dd66fd..98bce819804 100644--- a/src/metabase/health_inspector/core.clj+++ b/src/metabase/health_inspector/core.clj@@ -3,12 +3,14 @@[clojurewerkz.quartzite.jobs :as jobs][clojurewerkz.quartzite.schedule.cron :as cron][clojurewerkz.quartzite.triggers :as triggers]+ [java-time.api :as t][metabase.health-inspector.settings :as setting][metabase.lib-be.core :as lib-be][metabase.lib.schema :as schema][metabase.task.core :as task][metabase.util :as u][metabase.util.json :as json]+ [metabase.util.log :as log][metabase.util.malli.registry :as mr][toucan2.core :as t2]))@@ -42,25 +44,95 @@(defonce ^:private checks (atom {:validate-queries validate-queries}))(defn register-check!- "Register a new check function with a given name.- Check functions take no args and return a map with a :health ratio and :description string."+ "Register check `check-fn` under `name`.+ A check takes no args and returns `{:health <int 0-100>, :message <string>}`, or nil when it doesn't+ apply on this instance."[name check-fn](swap! checks assoc name check-fn))+(defn enabled?+ "Whether the health inspector is enabled -- the master switch for persisting runs. Metric emitters that also+ feed Prometheus (independent of this setting) use it to gate only their appdb persistence."+ []+ (setting/health-inspector-enabled))++(defn- run-check+ "Run one check in isolation. Ordinary failures read as degraded; interruption and fatal errors propagate."+ [check-name f]+ (try+ (f)+ (catch InterruptedException e+ (throw e))+ (catch Exception e+ (log/error e "Health check errored" {:check check-name})+ {:health 0, :message (str "Health check errored: " (ex-message e))})))+(defn report- "Run all registered checks and produce a report describing potential problems."+ "Run all registered checks and produce a report describing potential problems.+ Each check is isolated, so one that throws doesn't lose the others' results."[](into {} (for [[name f] @checks]- [name (f)])))+ [name (run-check name f)])))-(defn save-report- "Run a health inspector report and save it to the DB."- []- (doseq [[check-name result] (report)]+(defn- persist-check-result!+ "Persist one check result, or nothing when `result` is nil.+ A nil result marks a check as not applicable on this instance, so it is omitted rather than recorded as+ a misleading healthy score."+ [check-name result]+ (when (some? result)(t2/insert! :health_inspector_runs (-> result(select-keys [:health :message])(assoc :check_name (name check-name))))))+(def ^:private run-retention-days+ "Days of health-inspector runs to keep. Metric checks embed changing values (coverage %, ages) in their+ `:message`, so dedup can't collapse them and the table would grow unbounded without pruning."+ 30)++(defn- delete-old-runs!+ "Delete health-inspector runs older than [[run-retention-days]]."+ []+ (t2/delete! :health_inspector_runs+ :run_at [:< (t/minus (t/offset-date-time) (t/days run-retention-days))]))++(defn save-report+ "Run every registered check and persist the results (not-applicable (nil) checks are omitted), then prune+ runs past the retention window."+ []+ (doseq [[check-name result] (report)]+ (persist-check-result! check-name result))+ (delete-old-runs!))++(defn- latest-run+ [check-name]+ ;; Tie-break on id: back-to-back inserts can share a run_at, and run_at alone would then pick a+ ;; non-deterministic row, breaking the dedup below (id is a monotonic auto-increment PK).+ (t2/select-one [:health_inspector_runs :health :message] :check_name (name check-name)+ {:order-by [[:run_at :desc] [:id :desc]]}))++(defn save-check-result!+ "Persist a precomputed check `result` (a `{:health :message}` map, or nil to skip). Skip the write when the+ check's most recent result is unchanged. This is a best-effort optimization, not cross-node coordination.+ Does NOT gate on `health-inspector-enabled` -- callers that emit results outside a check run (e.g. a+ metric refresh that also feeds Prometheus) gate themselves."+ [check-name result]+ (when (some? result)+ (let [prev (latest-run check-name)]+ (when-not (and prev+ (= (:health prev) (:health result))+ (= (:message prev) (:message result)))+ (persist-check-result! check-name result)))))++(defn run-and-save-check!+ "Run one registered check by name and persist its result, so a change can be surfaced immediately rather+ than at the next daily report.+ A no-op unless the health inspector is enabled and the named check is registered and applicable (non-nil).+ Uses [[save-check-result!]] to avoid writing unchanged results in the usual single-caller case."+ [check-name]+ (when (enabled?)+ (when-let [f (get @checks check-name)]+ (save-check-result! check-name (run-check check-name f)))))+(defn list-runs"Return the most recent health inspector runs from the DB."[limit]diff --git a/test/metabase/health_inspector/core_test.clj b/test/metabase/health_inspector/core_test.cljindex bac1fce2bc9..3584ec145ae 100644--- a/test/metabase/health_inspector/core_test.clj+++ b/test/metabase/health_inspector/core_test.clj@@ -2,6 +2,7 @@(:require[clojure.test :refer :all][metabase.health-inspector.core :as hi]+ [metabase.test :as mt][metabase.util.json :as json][toucan2.core :as t2]))@@ -42,3 +43,53 @@(is (= [100 100] (map :health (runs "test-check"))))(is (= ["test check" "test check"] (map :message (runs "test-check"))))(is (= [100 100] (map :health (runs "validate-queries"))))))++;; The test-only checks below are registered into a fresh, test-scoped `checks` registry via with-redefs+;; rather than the global one, so a nil/throwing/probe check can't leak into other health-inspector tests in+;; the same JVM and make them order-dependent.++(deftest nil-check-is-omitted-test+ (testing "a check returning nil (not applicable on this instance) is omitted, not persisted as a green score"+ (with-redefs [hi/checks (atom {:test-nil (constantly nil)+ :ok (constantly {:health 100 :message "ok"})})]+ (is (nil? (:test-nil (hi/report))))+ (t2/delete! :health_inspector_runs)+ (hi/save-report)+ (is (= #{"ok"} (set (map :check_name (hi/list-runs 512))))+ "only the applicable check is persisted"))))++(deftest report-isolates-throwing-check-test+ (testing "a throwing check reads as degraded and doesn't abort the rest of the report"+ (with-redefs [hi/checks (atom {:boom (fn [] (throw (ex-info "kaboom" {})))+ :test-check test-check})]+ (let [report (hi/report)]+ (is (=? {:health 0 :message #".*kaboom.*"} (:boom report)))+ (is (= 100 (-> report :test-check :health)) "other checks still run")))))++(deftest report-propagates-interruption-test+ (with-redefs [hi/checks (atom {:interrupted #(throw (InterruptedException.))})]+ (is (thrown? InterruptedException (hi/report)))))++(deftest run-and-save-check!-test+ (let [probe-result (atom {:health 0 :message "down"})]+ (with-redefs [hi/checks (atom {:test-probe (fn [] @probe-result)})]+ (testing "no-op when the health inspector is disabled"+ (mt/with-temporary-setting-values [health-inspector-enabled false]+ (t2/delete! :health_inspector_runs)+ (hi/run-and-save-check! :test-probe)+ (is (empty? (hi/list-runs 512)))))+ (mt/with-temporary-setting-values [health-inspector-enabled true]+ (testing "no-op for an unregistered check"+ (t2/delete! :health_inspector_runs)+ (hi/run-and-save-check! :no-such-check)+ (is (empty? (hi/list-runs 512))))+ (testing "persists once, then dedups an unchanged result"+ (reset! probe-result {:health 0 :message "down"})+ (t2/delete! :health_inspector_runs)+ (hi/run-and-save-check! :test-probe)+ (hi/run-and-save-check! :test-probe)+ (is (= 1 (count (hi/list-runs 512))) "the unchanged repeat is deduped"))+ (testing "a changed result is persisted"+ (reset! probe-result {:health 100 :message "up"})+ (hi/run-and-save-check! :test-probe)+ (is (= 2 (count (hi/list-runs 512))) "a changed result is persisted"))))))