Skip to content

The easy-to-use open source Business Intelligence and Embedded Analytics tool that lets everyone work with data :bar_chart:

Add NLQ retrieval health checks (#78298)

Chris Trutera323e1c88326
diff --git a/.clj-kondo/config/modules/config.edn b/.clj-kondo/config/modules/config.edn
index e456148a55d..e6d1d7c1a8c 100644
--- a/.clj-kondo/config/modules/config.edn
+++ b/.clj-kondo/config/modules/config.edn
@@ -3262,7 +3262,9 @@
:uses #{analytics-interface
collections
entity-retrieval
+ health-inspector
premium-features
+ search
task
util
enterprise/semantic-search}
@@ -3584,6 +3586,7 @@
metabase-enterprise.semantic-search.core
metabase-enterprise.semantic-search.db.datasource
metabase-enterprise.semantic-search.embedding
+ metabase-enterprise.semantic-search.embedding-health
metabase-enterprise.semantic-search.init}
:uses #{activity-feed
analytics
diff --git a/enterprise/backend/src/metabase_enterprise/entity_retrieval/core.clj b/enterprise/backend/src/metabase_enterprise/entity_retrieval/core.clj
index f18db30057a..b50ab206dd5 100644
--- a/enterprise/backend/src/metabase_enterprise/entity_retrieval/core.clj
+++ b/enterprise/backend/src/metabase_enterprise/entity_retrieval/core.clj
@@ -13,6 +13,7 @@
[metabase-enterprise.entity-retrieval.reconcile :as reconcile]
[metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]
[metabase-enterprise.semantic-search.embedding :as embedding]
+ [metabase-enterprise.semantic-search.embedding-health :as embedding-health]
[metabase.analytics-interface.core :as analytics]
[metabase.premium-features.core :as premium-features :refer [defenterprise]]
[metabase.util :as u]
@@ -41,53 +42,106 @@
[]
(semantic.db.datasource/dedicated-url-configured?))
-(defn available?
- "Whether the entity-retrieval mirror can run right now. All four must hold:
- - a pgvector store is configured (somewhere to hold the index),
- - the license includes `:library-retrieval` (the tool is entitled),
- - the license includes `:library`: retrieval operates over library entities, so the tool is meaningless
- without it (the index would have nothing to hold) — enforced here so a token granting
- `:library-retrieval` alone degrades to \"unavailable\" explicitly, not silently via an empty index,
- - an embedding backend is configured (see [[embedding/embedding-supported?]]): without one, both
- reconcile and query embedding would throw, so we gate rather than fail mid-run.
- The feature and config checks can flip at runtime (token/settings entered post-boot), so callers
- re-evaluate per use."
+(defn- licensed?
+ "Without a library, there is nothing one is entitled to retrieve."
[]
- ;; entitlement first: short-circuit on the license flags for instances that can't use the answer, before
- ;; the config check (here a cheap dedicated-URL string check, not the app-db pgvector probe)
(and (premium-features/has-feature? :library)
- (premium-features/has-feature? :library-retrieval)
- (pgvector-configured?)
- (embedding/embedding-supported? (embedding/get-configured-model))))
-
-(defn- index-ready?
- "Whether the library entity index can serve a query right now: its meta row matches the configured
- embedding model and schema version, and it holds at least one document.
- A mismatched meta row (a model/dimension/format change whose rebuild hasn't run yet), a missing or empty
- vectors table, or any query error all read as not-ready.
- The compatibility check is what keeps a stale index from being offered: querying a vectors table built for
- a different model returns nothing (or errors) rather than answering, so the agent must get the
- general-search fallback instead."
+ (premium-features/has-feature? :library-retrieval)))
+
+(defn- embedder-configured?
+ "Whether an embedding backend is configured: without one we can neither index nor retrieve."
+ []
+ (embedding/embedding-supported? (embedding/get-configured-model)))
+
+(defn available?
+ "Whether the entity-retrieval mirror can run right now:
+ - We have this feature
+ - We have somewhere to store embedding vectors
+ - We have some way to compute embedding vectors
+
+ Re-evaluate this per use, as this configuration is dynamic."
+ []
+ (and (pgvector-configured?)
+ (licensed?)
+ (embedder-configured?)))
+
+(defn- missing-table-error?
+ "Whether we threw due to a Postgres undefined_table error (42P01), i.e. the indexes don't exist yet.
+ Expected before the first build or after a manual drop awaiting heal.
+ The SQLState comes from the database itself, so this is not a connectivity fault."
+ [e]
+ (boolean (some #(and (instance? SQLException %)
+ (= "42P01" (.getSQLState ^SQLException %)))
+ (u/full-exception-chain e))))
+
+(defn- dependencies
+ "Entity retrieval's necessary conditions, as a `{:store, :embedder, :licenses}` map of booleans."
[]
- (boolean
- (try
- (let [ds (semantic.db.datasource/ensure-initialized-data-source!)]
- (and (index-table/index-compatible? ds (embedding/get-configured-model))
- (seq (jdbc/execute! ds [(format "SELECT 1 FROM \"%s\" LIMIT 1" index-table/*vectors-table*)]))))
- (catch Throwable _ false))))
-
-;; OSS-callable surface used to decide whether to OFFER the retrieve_library_entities tool: it must
-;; be able to actually answer, so beyond config + license the index has to be built for the current model and
-;; populated. (The write/reconcile path gates on the looser [[available?]] instead, so an empty or stale
-;; index can still be rebuilt.) Runs unconditionally rather than gating on :feature — the OSS fallback
-;; `false` already covers the unlicensed case.
+ {:store (pgvector-configured?)
+ :embedder (embedder-configured?)
+ :licenses (licensed?)})
+
+(defn- index-populated? [ds]
+ (boolean (seq (jdbc/execute! ds [(format "SELECT 1 FROM \"%s\" LIMIT 1" index-table/*vectors-table*)]))))
+
+(defn- probe-index
+ "The `:index` map — `{:status <enum>}`, plus `:error` on `:unreachable`. A compatible metadata row is
+ reported as missing when its vectors table is gone; population is otherwise probed only when requested."
+ [probe-populated?]
+ (try
+ (let [ds (semantic.db.datasource/ensure-initialized-data-source!)
+ metadata-status (index-table/index-status ds (embedding/get-configured-model))
+ status (if (and (= :compatible metadata-status)
+ (not (index-table/vectors-table-exists? ds)))
+ :missing
+ metadata-status)]
+ {:status (if (and (= :compatible status) probe-populated?)
+ (if (index-populated? ds) :populated :empty)
+ status)})
+ (catch InterruptedException e
+ (throw e))
+ (catch Exception e
+ ;; A missing table is the ordinary absent-index state, not a store fault.
+ (if (missing-table-error? e)
+ {:status :missing}
+ ;; DEBUG: entity-retrieval-available? probes per tool-offering request, so a pgvector outage would
+ ;; WARN-flood the logs; the :unreachable status and the health check already carry the signal.
+ (do (log/debug e "entity-retrieval index probe failed")
+ {:status :unreachable, :error (ex-message e)})))))
+
+(defn retrieval-status
+ "Entity-retrieval health, decomposed into dependencies and index state:
+
+ {:dependencies {:store <bool>, :embedder <bool>, :licenses <bool>}
+ :index {:status <enum>}} ; present only when every dependency holds
+
+ `:status` is one of:
+ - `:missing` nothing built yet
+ - `:incompatible` built for a different model/schema — rebuild pending
+ - `:compatible` matches the current model; population not probed
+ - `:empty` compatible, holds no vectors
+ - `:populated` compatible, holds ≥1 vector
+ - `:unreachable` the store threw while probing; carries `:error`
+
+ Pass `probe-populated?` false to stop at `:compatible` and skip the population query."
+ ([] (retrieval-status true))
+ ([probe-populated?]
+ (let [deps (dependencies)]
+ (cond-> {:dependencies deps}
+ (every? val deps) (assoc :index (probe-index probe-populated?))))))
+
(defenterprise entity-retrieval-available?
- "EE impl: pgvector configured + library-retrieval licensed AND the index is ready (built for the current
- model and populated), so the tool is offered only when it can serve a query (otherwise the agent
- gets the general-search fallback)."
- :feature :none
+ "Is there a non-empty compatible index, and are we able to use it?"
+ :feature :library-retrieval
[]
- (and (available?) (index-ready?)))
+ (if-not (= :populated (get-in (retrieval-status) [:index :status]))
+ false
+ (if (embedding/embedder-circuit-untrusted?)
+ (do
+ ;; Keep serving the fallback until a throttled background trial has actually closed the breaker.
+ (embedding-health/request-circuit-recovery!)
+ false)
+ true)))
;;; ----------------------------------------- Reconcile scheduling -----------------------------------------
;;;
diff --git a/enterprise/backend/src/metabase_enterprise/entity_retrieval/health.clj b/enterprise/backend/src/metabase_enterprise/entity_retrieval/health.clj
new file mode 100644
index 00000000000..949e7e489d3
--- /dev/null
+++ b/enterprise/backend/src/metabase_enterprise/entity_retrieval/health.clj
@@ -0,0 +1,162 @@
+(ns metabase-enterprise.entity-retrieval.health
+ "Health-inspector check for NLQ (natural-language-query) curated retrieval.
+
+ The `:nlq` Metabot profile uses this tool; when the tool is unavailable we swap the profile out for
+ `:nlq-fallback`, which uses the regular search tool.
+
+ The swap is silent, so a broken index would otherwise go unnoticed -- this check surfaces it.
+
+ :health can take the following values:
+ - nil = not enabled
+ - 0 = enabled, but not available (`:message` tells why)
+ - 0<h<100 = partially available (e.g. a built but empty index)
+ - 100 = fully operational
+
+ Shares the embedding-service probe and circuit-breaker with semantic-search."
+ (:require
+ [clojure.core.memoize :as memoize]
+ [clojure.set :as set]
+ [metabase-enterprise.entity-retrieval.core :as entity-retrieval]
+ [metabase-enterprise.entity-retrieval.index-table :as index-table]
+ [metabase-enterprise.entity-retrieval.reconcile :as reconcile]
+ [metabase-enterprise.semantic-search.db.datasource :as semantic.datasource]
+ [metabase-enterprise.semantic-search.embedding :as semantic.embedding]
+ [metabase-enterprise.semantic-search.embedding-health :as embedding-health]
+ [metabase.entity-retrieval.core :as er]
+ [metabase.health-inspector.core :as health-inspector]
+ [metabase.search.index-health :as search.index-health]
+ [metabase.util :as u]
+ [next.jdbc :as jdbc]
+ [next.jdbc.result-set :as jdbc.rs]))
+
+(set! *warn-on-reflection* true)
+
+(defn nlq-retrieval-health-check
+ "Health-inspector check for NLQ curated retrieval, registered as `:nlq-retrieval`.
+ Returns nil when a dependency is unmet (the feature is off, so the check is omitted), otherwise a
+ `{:health :message}` map describing the index's own state and the embedding service it depends on."
+ []
+ ;; :index is present only when every dependency holds -- absence is the not-applicable signal.
+ (when-let [index (:index (entity-retrieval/retrieval-status))]
+ (case (:status index)
+ :unreachable (search.index-health/degraded "Index unreachable")
+ :missing (search.index-health/degraded "Index not found")
+ :incompatible (search.index-health/degraded "Index not compatible")
+ :empty (if-let [problem (embedding-health/embedding-problem)]
+ (search.index-health/degraded (u/capitalize-first-char problem))
+ (search.index-health/warning "Index empty"))
+ :populated (if-let [problem (embedding-health/embedding-problem)]
+ (search.index-health/degraded (u/capitalize-first-char problem))
+ (search.index-health/healthy "Healthy")))))
+
+(health-inspector/register-check! :nlq-retrieval nlq-retrieval-health-check)
+
+(defn- persist-nlq-check-on-breaker-change!
+ "Re-run and persist the NLQ retrieval check."
+ [_state]
+ (health-inspector/run-and-save-check! :nlq-retrieval))
+
+(swap! semantic.embedding/embedder-circuit-state-change-hooks conj #'persist-nlq-check-on-breaker-change!)
+
+;;; ----------------------------------------- Search index metrics ------------------------------------------
+;;;
+;;; Coverage / garbage / staleness for the index, at the distinct-entity grain (rows are per (entity, doc)).
+;;; Both sides are normalised through entity-class so a metric<->model relabel doesn't read as both missing
+;;; and garbage. The shared search-index health framework owns threshold, message, and gauge shaping.
+
+(def ^:private staleness-warn-seconds (* 30 60)) ; 30m -- one missed ~15m full-reconcile cycle
+(def ^:private staleness-critical-seconds (* 60 60)) ; 60m -- reconcile clearly stalled
+;; Absolute orphan counts. The curated library is a bounded, small tier, so tolerances are much lower than
+;; semantic search; reconcile GCs orphans every ~15m. Tunable (promotable to settings).
+(def ^:private garbage-warn-count 5)
+(def ^:private garbage-critical-count 100)
+
+(defn- library-datasource*
+ "pgvector datasource when NLQ library retrieval is licensed, configured (pgvector + embedder), and the
+ index is built for the current model; else nil (the metric skips -- availability is the `:nlq-retrieval`
+ check's job). An empty-but-compatible index is still measured (coverage reads 0%), so this gates on
+ compatibility, not population -- and passes `probe-populated? false` so it doesn't run the population
+ query it would discard."
+ []
+ ;; probe-populated? false stops the probe at :compatible, so :empty/:populated never appear here.
+ (when (= :compatible (get-in (entity-retrieval/retrieval-status false) [:index :status]))
+ (try (semantic.datasource/ensure-initialized-data-source!)
+ (catch InterruptedException e
+ (throw e))
+ (catch Exception _ nil))))
+
+(def ^:private library-datasource
+ "TTL-memoized so coverage, garbage, and staleness share one retrieval-status probe + datasource resolve per
+ refresh cycle rather than each re-running it."
+ (memoize/ttl library-datasource* :ttl/threshold (* 30 1000)))
+
+(defn- entity-class-set
+ "Set of entity classes for `pairs` of `[entity_type entity_local_id]`, normalised so a relabelled entity
+ collapses to one class on both the library and index sides."
+ [pairs]
+ (into #{} (map (fn [[t id]] (er/entity-class t id))) pairs))
+
+(defn- indexed-entity-classes [ds]
+ (entity-class-set
+ (map (juxt :entity_type :entity_local_id)
+ (jdbc/execute! ds
+ [(format "SELECT DISTINCT entity_type, entity_local_id FROM \"%s\"" index-table/*vectors-table*)]
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps}))))
+
+(defn- library-and-indexed-classes*
+ "The `{:library <classes> :indexed <classes>}` sets coverage and garbage both need, or nil when N/A."
+ []
+ (when-let [ds (library-datasource)]
+ {:library (entity-class-set (reconcile/library-entity-keys))
+ :indexed (indexed-entity-classes ds)}))
+
+(def ^:private library-and-indexed-classes
+ "TTL-memoized so coverage and garbage share one library scan + DISTINCT index scan per refresh cycle rather
+ than each recomputing the identical two sets."
+ (memoize/ttl library-and-indexed-classes* :ttl/threshold (* 30 1000)))
+
+(defn- nlq-coverage []
+ (when-let [{:keys [library indexed]} (library-and-indexed-classes)]
+ (search.index-health/coverage-result (count (set/intersection library indexed)) (count library))))
+
+(defn- nlq-garbage []
+ (when-let [{:keys [library indexed]} (library-and-indexed-classes)]
+ (search.index-health/garbage-result (count (set/difference indexed library))
+ garbage-warn-count garbage-critical-count)))
+
+(defn- nlq-staleness []
+ (when-let [ds (library-datasource)]
+ ;; Reconcile-lag: seconds since the last successful full reconcile began reading the appdb. Capturing the
+ ;; watermark before that read conservatively includes changes made during a long run. There's no per-change
+ ;; gate here (unlike semantic search), so this is the honest bound on undetected membership/name drift that
+ ;; the osi_ai_context write hooks don't catch. A null means no full reconcile has completed.
+ ;;
+ ;; ensure-tables! adds reconciled_at to legacy metadata when the database role owns the table. A grant-only
+ ;; role can still reconcile but cannot record freshness, which is degraded rather than silently omitted.
+ (try
+ (let [sql (format (str "WITH observed AS (SELECT clock_timestamp() AS at) "
+ "SELECT EXTRACT(EPOCH FROM ((SELECT at FROM observed) - reconciled_at)) AS age "
+ "FROM \"%s\" WHERE id = 1")
+ index-table/*meta-table*)
+ {:keys [age]} (jdbc/execute-one! ds [sql] {:builder-fn jdbc.rs/as-unqualified-lower-maps})]
+ (cond
+ (nil? age)
+ (search.index-health/degraded "No full index reconcile has completed.")
+
+ (neg? (double age))
+ (search.index-health/degraded
+ "Index reconcile freshness unavailable: database clock precedes the last reconcile.")
+
+ :else
+ (search.index-health/staleness-result
+ age staleness-warn-seconds staleness-critical-seconds
+ "Changes missed by the write hooks are picked up by the ~15m full reconcile.")))
+ (catch java.sql.SQLException e
+ ;; 42703 = undefined_column
+ (if (= "42703" (.getSQLState e))
+ (search.index-health/degraded "Index reconcile freshness unavailable: metadata column missing.")
+ (throw e))))))
+
+(search.index-health/register-index-check! :nlq-retrieval :coverage nlq-coverage)
+(search.index-health/register-index-check! :nlq-retrieval :garbage nlq-garbage)
+(search.index-health/register-index-check! :nlq-retrieval :staleness nlq-staleness)
diff --git a/enterprise/backend/src/metabase_enterprise/entity_retrieval/index_table.clj b/enterprise/backend/src/metabase_enterprise/entity_retrieval/index_table.clj
index 8c5a6fe0601..d639dfc32e3 100644
--- a/enterprise/backend/src/metabase_enterprise/entity_retrieval/index_table.clj
+++ b/enterprise/backend/src/metabase_enterprise/entity_retrieval/index_table.clj
@@ -23,6 +23,7 @@
[clojure.string :as str]
[honey.sql :as sql]
[honey.sql.helpers :as sql.helpers]
+ [metabase.util.log :as log]
[next.jdbc :as jdbc]
[next.jdbc.result-set :as jdbc.rs])
(:import
@@ -30,6 +31,8 @@
(set! *warn-on-reflection* true)
+;; TODO (Chris 2026-07-14) -- these names carry no version, so an on-disk upgrade has nothing to key off.
+;; Revisit when the semantic-search indexing mechanism is reworked, sharing its scheme.
(def ^:dynamic *vectors-table*
"Vectors table name. Dynamic so tests can rebind it to an isolated table."
"library_entity_index")
@@ -76,7 +79,11 @@
[:model_name :text :not-null]
[:vector_dimensions :int :not-null]
[:schema_version :int :not-null]
- [:updated_at :timestamp-with-time-zone :not-null]])
+ [:updated_at :timestamp-with-time-zone :not-null]
+ ;; Captured immediately before a successful full reconcile reads the appdb. Distinct from updated_at
+ ;; (which write-meta! bumps on an empty create/rebuild), so NLQ staleness isn't reset by a rebuild
+ ;; that hasn't been reconciled yet. Nullable: null = never reconciled since the (re)build.
+ [:reconciled_at :timestamp-with-time-zone]])
sql-format-quoted))
(defn- create-vectors-table-sql [dims]
@@ -122,20 +129,71 @@
(defn- create-tables! [tx dims]
(jdbc/execute! tx (create-vectors-table-sql dims)))
-(defn- vectors-table-exists? [tx]
+(defn vectors-table-exists?
+ "Whether the configured entity-retrieval vectors table exists."
+ [tx]
(some? (:exists (jdbc/execute-one! tx [(format "SELECT to_regclass('%s') AS exists" *vectors-table*)]
{:builder-fn jdbc.rs/as-unqualified-lower-maps}))))
-(defn index-compatible?
- "Whether the meta row matches `embedding-model` and the current [[schema-version]] — i.e. the vectors
- table holds embeddings the configured model can be queried against.
- False when the meta row is missing or describes a different provider/model/dimensions/format, meaning the
- index is stale and a rebuild is still pending; the same comparison [[ensure-tables!]] uses to decide a
- rebuild.
- Reads the meta table directly, so a caller must guard against the table not existing yet (it throws a SQL
- error before the first [[ensure-tables!]] of the process)."
+(defn- meta-column-exists? [tx column]
+ (some? (jdbc/execute-one! tx [(str "SELECT 1 FROM pg_attribute"
+ " WHERE attrelid = to_regclass(?) AND attname = ? AND NOT attisdropped")
+ *meta-table* column])))
+
+(defn- can-alter-meta-table? [tx]
+ (:owns_table
+ (jdbc/execute-one! tx
+ [(str "SELECT pg_has_role(c.relowner, 'USAGE') AS owns_table"
+ " FROM pg_class c WHERE c.oid = to_regclass(?)")
+ *meta-table*]
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})))
+
+(defonce ^:private warned-about-missing-reconciled-at (atom #{}))
+
+(defn- warn-about-missing-reconciled-at-once! []
+ (let [[warned-before _] (swap-vals! warned-about-missing-reconciled-at conj *meta-table*)]
+ (when-not (contains? warned-before *meta-table*)
+ (log/warnf "Cannot add reconciled_at to %s because the current role does not own it" *meta-table*))))
+
+(defn- ensure-reconciled-at-column! [tx]
+ (or (meta-column-exists? tx "reconciled_at")
+ (if (can-alter-meta-table? tx)
+ (do
+ (jdbc/execute! tx [(format "ALTER TABLE \"%s\" ADD COLUMN IF NOT EXISTS reconciled_at timestamp with time zone"
+ *meta-table*)])
+ true)
+ (do
+ ;; An ALTER permission error would abort ensure-tables!'s transaction. Grant-only installations can
+ ;; still reconcile; their staleness metric reports the unavailable timestamp as degraded instead.
+ (warn-about-missing-reconciled-at-once!)
+ false))))
+
+(defn touch-reconciled-at!
+ "Record when a successful full reconcile began reading the appdb.
+
+ Uses the pgvector clock and a separate column from `updated_at`, which changes on an empty rebuild. A
+ no-op for legacy meta tables the current database role cannot upgrade."
+ [tx reconciled-at]
+ (when (meta-column-exists? tx "reconciled_at")
+ (jdbc/execute! tx [(format "UPDATE \"%s\" SET reconciled_at = ? WHERE id = 1" *meta-table*)
+ reconciled-at])))
+
+(defn- clear-reconciled-at! [tx]
+ (when (meta-column-exists? tx "reconciled_at")
+ (jdbc/execute! tx [(format "UPDATE \"%s\" SET reconciled_at = NULL WHERE id = 1" *meta-table*)])))
+
+(defn index-status
+ "Compatibility of the built index against `embedding-model` and [[schema-version]]:
+ - `:missing` no meta row — nothing built
+ - `:incompatible` meta row for a different model/schema — rebuild pending
+ - `:compatible` meta row matches; use [[vectors-table-exists?]] when query readiness matters
+ Reads the meta table directly, so it throws an undefined-table error before the first [[ensure-tables!]]."
[pgvector embedding-model]
- (= (read-meta pgvector) (model-identity embedding-model)))
+ (let [meta (read-meta pgvector)]
+ (cond
+ (nil? meta) :missing
+ (= meta (model-identity embedding-model)) :compatible
+ :else :incompatible)))
(defn ensure-tables!
"Idempotently create the vectors + meta tables for `embedding-model`, rebuilding on model change.
@@ -152,25 +210,36 @@
(jdbc/execute! tx [(format "SELECT pg_advisory_xact_lock(%d)" ensure-lock-id)])
(jdbc/execute! tx (sql/format (sql.helpers/create-extension :vector :if-not-exists)))
(jdbc/execute! tx (create-meta-table-sql))
+ ;; CREATE TABLE IF NOT EXISTS does not add columns to an existing table. Upgrade when this role owns it;
+ ;; grant-only roles keep reconciling without the optional freshness timestamp.
+ (ensure-reconciled-at-column! tx)
(let [stored (read-meta tx)
current (model-identity embedding-model)
- dims (:vector_dimensions current)]
- (cond
- (nil? stored)
- (do (create-tables! tx dims)
- (write-meta! tx embedding-model)
- :created)
-
- (not= stored current)
- (do (jdbc/execute! tx [(format "DROP TABLE IF EXISTS \"%s\"" *vectors-table*)])
- (create-tables! tx dims)
- (write-meta! tx embedding-model)
- :rebuilt)
-
- :else
- ;; Meta matches. Re-issue the IF NOT EXISTS DDL so a manually dropped vectors table heals itself,
- ;; and report :created when it had actually gone missing — the recreated table is empty, so a
- ;; targeted reconcile must repopulate the whole library rather than fill it with one entity.
- (let [existed? (vectors-table-exists? tx)]
- (create-tables! tx dims)
- (if existed? :ok :created))))))
+ dims (:vector_dimensions current)
+ result (cond
+ (nil? stored)
+ (do (create-tables! tx dims)
+ (write-meta! tx embedding-model)
+ :created)
+
+ (not= stored current)
+ (do (jdbc/execute! tx [(format "DROP TABLE IF EXISTS \"%s\"" *vectors-table*)])
+ (create-tables! tx dims)
+ (write-meta! tx embedding-model)
+ :rebuilt)
+
+ :else
+ ;; Meta matches. Re-issue the IF NOT EXISTS DDL so a manually dropped vectors table
+ ;; heals itself, and report :created when it had actually gone missing — the recreated
+ ;; table is empty, so a targeted reconcile must repopulate the whole library rather than
+ ;; fill it with one entity.
+ (let [existed? (vectors-table-exists? tx)]
+ (create-tables! tx dims)
+ (if existed? :ok :created)))]
+ ;; Any empty (re)build invalidates reconciled_at: the table is empty until a full reconcile repopulates
+ ;; it, and neither write-meta! (:rebuilt) nor the heal branch touches reconciled_at, so a prior build's
+ ;; timestamp would linger and make NLQ staleness read fresh over an empty/incomplete index. Clear it;
+ ;; touch-reconciled-at! (converged reconciles only) sets it again once the index is actually verified.
+ (when (not= result :ok)
+ (clear-reconciled-at! tx))
+ result)))
diff --git a/enterprise/backend/src/metabase_enterprise/entity_retrieval/init.clj b/enterprise/backend/src/metabase_enterprise/entity_retrieval/init.clj
index 33ba36207c8..d86f66e7734 100644
--- a/enterprise/backend/src/metabase_enterprise/entity_retrieval/init.clj
+++ b/enterprise/backend/src/metabase_enterprise/entity_retrieval/init.clj
@@ -2,4 +2,5 @@
"Loads the entity-retrieval module's side-effecting namespaces (the background sync task) at
system startup. See [[metabase-enterprise.core.init]]."
(:require
+ [metabase-enterprise.entity-retrieval.health]
[metabase-enterprise.entity-retrieval.task.sync]))
diff --git a/enterprise/backend/src/metabase_enterprise/entity_retrieval/reconcile.clj b/enterprise/backend/src/metabase_enterprise/entity_retrieval/reconcile.clj
index 009c0aba038..da4fd974fa9 100644
--- a/enterprise/backend/src/metabase_enterprise/entity_retrieval/reconcile.clj
+++ b/enterprise/backend/src/metabase_enterprise/entity_retrieval/reconcile.clj
@@ -321,6 +321,10 @@
{:builder-fn jdbc.rs/as-unqualified-lower-maps})]
{:documents documents :entities entities}))
+(defn- capture-reconcile-watermark [conn]
+ (:now (jdbc/execute-one! conn ["SELECT now() AS now"]
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})))
+
;; Scalability — targeted writes, full-diff backstop.
;; The full diff is O(total index size) per run, not O(changes). Embeddings — the only expensive resource —
;; are already change-scoped (only a new doc_text is embedded; an idle run embeds nothing), and the O(total)
@@ -334,11 +338,12 @@
(defn- reconcile-against-appdb!
"The full diff body — assumes the reconcile advisory lock is held on `conn`. See the namespace docstring."
[conn embedding-model]
- (let [desired (desired-docs)
- desired-ids (set (map :doc_id desired))
- stored (stored-docs conn)
- to-insert (remove #(contains? stored (:doc_id %)) desired)
- orphans (remove desired-ids (keys stored))
+ (let [reconciled-at (capture-reconcile-watermark conn)
+ desired (desired-docs)
+ desired-ids (set (map :doc_id desired))
+ stored (stored-docs conn)
+ to-insert (remove #(contains? stored (:doc_id %)) desired)
+ orphans (remove desired-ids (keys stored))
;; entity classes whose insert failed this run — their orphans are spared (see below).
failed (volatile! #{})
inserted (transduce
@@ -360,6 +365,15 @@
to-delete (cond->> orphans
(seq @failed) (remove #(contains? @failed (entity-class (get stored %)))))]
(delete-rows! conn to-delete)
+ ;; Record freshness when the run converged as far as it can -- no *transient* batch failure this run
+ ;; (@failed is populated only by a caught insert exception). A doc silently dropped for exceeding the
+ ;; per-item token limit is a permanent, expected shortfall (`inserted` < `(count to-insert)` forever), so
+ ;; gating on the insert count instead would freeze reconciled_at the moment one un-embeddable doc enters
+ ;; the library -- NLQ staleness would then climb to critical (or stay NaN) even though every reconcile is
+ ;; doing all it can. Gate on the absence of retryable failures, not on full insertion. Persist the
+ ;; pre-read watermark so changes made during a long reconcile never look newer than the source snapshot.
+ (when (empty? @failed)
+ (index-table/touch-reconciled-at! conn reconciled-at))
;; index-size after the writes feeds the document/entity gauges (full reconcile only).
(merge (diff-result desired to-insert inserted (count to-delete))
(index-size conn))))
diff --git a/enterprise/backend/test/metabase_enterprise/entity_retrieval/core_test.clj b/enterprise/backend/test/metabase_enterprise/entity_retrieval/core_test.clj
index 472d0f20064..0ed1ade8b40 100644
--- a/enterprise/backend/test/metabase_enterprise/entity_retrieval/core_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/entity_retrieval/core_test.clj
@@ -6,6 +6,7 @@
[metabase-enterprise.entity-retrieval.reconcile :as reconcile]
[metabase-enterprise.semantic-search.db.datasource :as semantic.db.datasource]
[metabase-enterprise.semantic-search.embedding :as semantic.embedding]
+ [metabase-enterprise.semantic-search.embedding-health :as embedding-health]
[metabase-enterprise.semantic-search.test-util :as semantic.tu]
[metabase.entity-retrieval.mirror :as mirror]
[metabase.metabot.tools.entity-retrieval :as tools.entity-retrieval]
@@ -153,6 +154,62 @@
(mt/with-dynamic-fn-redefs [semantic.embedding/get-configured-model (constantly {:provider "no-embedder"})]
(is (false? (entity-retrieval.core/available?)))))))))
+(deftest entity-retrieval-availability-requires-a-closed-breaker-test
+ (mt/with-premium-features #{:library-retrieval}
+ (let [recovery-requested? (atom false)]
+ (mt/with-dynamic-fn-redefs
+ [entity-retrieval.core/retrieval-status (constantly {:index {:status :populated}})
+ semantic.embedding/embedder-circuit-untrusted? (constantly true)
+ embedding-health/request-circuit-recovery! #(reset! recovery-requested? true)]
+ (is (false? (entity-retrieval.core/entity-retrieval-available?)))
+ (is (true? @recovery-requested?) "an untrusted circuit starts recovery without offering the tool"))))
+ (mt/with-premium-features #{:library-retrieval}
+ (mt/with-dynamic-fn-redefs
+ [entity-retrieval.core/retrieval-status (constantly {:index {:status :populated}})
+ semantic.embedding/embedder-circuit-untrusted? (constantly false)]
+ (is (true? (entity-retrieval.core/entity-retrieval-available?))))))
+
+(deftest entity-retrieval-availability-checks-readiness-before-breaker-test
+ (mt/with-premium-features #{:library-retrieval}
+ (mt/with-dynamic-fn-redefs
+ [entity-retrieval.core/retrieval-status (constantly {:dependencies {:embedder false}})
+ semantic.embedding/embedder-circuit-untrusted? #(throw (ex-info "must not resolve an endpoint" {}))]
+ (is (false? (entity-retrieval.core/entity-retrieval-available?))))))
+
+(deftest retrieval-status-missing-table-reads-as-absence-test
+ (mt/with-premium-features #{:library :library-retrieval}
+ ;; db-url is read directly as a var, so with-redefs (not with-dynamic-fn-redefs) is required here.
+ (with-redefs [semantic.db.datasource/db-url "jdbc:postgresql://stub"]
+ (mt/with-dynamic-fn-redefs
+ [semantic.db.datasource/ensure-initialized-data-source! (constantly ::ds)
+ semantic.embedding/get-configured-model (constantly semantic.tu/mock-embedding-model)]
+ (let [deps {:store true, :embedder true, :licenses true}
+ status-when-probe-throws
+ (fn [e]
+ (mt/with-dynamic-fn-redefs [index-table/index-status (fn [& _] (throw e))]
+ (entity-retrieval.core/retrieval-status)))]
+ (testing "42P01 (first build pending / manual drop) reads as a missing index, not a store fault"
+ (is (=? {:dependencies deps, :index {:status :missing}}
+ (status-when-probe-throws
+ (java.sql.SQLException. "relation \"library_entity_index_meta\" does not exist" "42P01")))))
+ (testing "a wrapped 42P01 is found through the cause chain"
+ (is (=? {:dependencies deps, :index {:status :missing}}
+ (status-when-probe-throws
+ (ex-info "probe failed" {}
+ (java.sql.SQLException. "relation does not exist" "42P01"))))))
+ (testing "any other probe failure (a real connectivity fault) reads as :unreachable, carrying :error"
+ (is (=? {:dependencies deps, :index {:status :unreachable, :error "connection refused"}}
+ (status-when-probe-throws
+ (java.sql.SQLException. "connection refused" "08001")))))
+ (testing "interruption propagates instead of being reported as an unreachable index"
+ (is (thrown? InterruptedException
+ (status-when-probe-throws (InterruptedException.)))))
+ (testing "compatible metadata without its vectors table reads as missing, including the metric probe"
+ (mt/with-dynamic-fn-redefs [index-table/index-status (constantly :compatible)
+ index-table/vectors-table-exists? (constantly false)]
+ (is (=? {:dependencies deps, :index {:status :missing}}
+ (entity-retrieval.core/retrieval-status false))))))))))
+
(deftest ^:sequential entity-retrieval-available?-requires-a-populated-index-test
(testing "the curated tool is offered only once the index has documents (else the nlq profile falls back)"
(when semantic.db.datasource/db-url
diff --git a/enterprise/backend/test/metabase_enterprise/entity_retrieval/health_test.clj b/enterprise/backend/test/metabase_enterprise/entity_retrieval/health_test.clj
new file mode 100644
index 00000000000..80c59e1d50f
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/entity_retrieval/health_test.clj
@@ -0,0 +1,128 @@
+(ns metabase-enterprise.entity-retrieval.health-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase-enterprise.entity-retrieval.core :as entity-retrieval.core]
+ [metabase-enterprise.entity-retrieval.health :as entity-retrieval.health]
+ [metabase-enterprise.semantic-search.db.datasource :as semantic.datasource]
+ [metabase-enterprise.semantic-search.embedding :as semantic.embedding]
+ [metabase-enterprise.semantic-search.embedding-health :as embedding-health]
+ [metabase.test :as mt]
+ [next.jdbc :as jdbc]))
+
+(set! *warn-on-reflection* true)
+
+(def ^:private ready-status
+ {:dependencies {:store true, :embedder true, :licenses true}
+ :index {:status :populated}})
+
+(defn- check! [status & {:keys [circuit-untrusted? reachable]
+ :or {circuit-untrusted? false, reachable {:reachable? true, :error nil}}}]
+ (mt/with-dynamic-fn-redefs
+ [entity-retrieval.core/retrieval-status (constantly status)
+ semantic.embedding/embedder-circuit-untrusted? (constantly circuit-untrusted?)
+ embedding-health/embedding-service-reachable? (constantly reachable)
+ embedding-health/request-circuit-recovery! (constantly nil)]
+ (entity-retrieval.health/nlq-retrieval-health-check)))
+
+(defn- with-index-status [status]
+ (assoc ready-status :index {:status status}))
+
+(deftest not-enabled-is-omitted-test
+ (testing "any unmet dependency (store, embedder, or licenses) -> nil (omitted, not a misleading 100)"
+ (is (nil? (check! {:dependencies {:store false, :embedder true, :licenses true}})))
+ (is (nil? (check! {:dependencies {:store true, :embedder false, :licenses true}})))
+ (is (nil? (check! {:dependencies {:store true, :embedder true, :licenses false}})))))
+
+(deftest missing-index-is-degraded-test
+ (testing "no index built yet -> degraded"
+ (is (=? {:health 0, :message #"Index not found"}
+ (check! (with-index-status :missing))))))
+
+(deftest incompatible-index-is-degraded-test
+ (testing "the index isn't built for the current model -> degraded"
+ (is (=? {:health 0, :message #"Index not compatible"}
+ (check! (with-index-status :incompatible))))))
+
+(deftest unreachable-index-is-degraded-test
+ (testing "a thrown probe (store down) -> degraded, reported as its own state (not a misleading rebuild)"
+ (is (=? {:health 0, :message #"Index unreachable"}
+ (check! (assoc ready-status :index {:status :unreachable, :error "connection refused"}))))))
+
+(deftest empty-index-is-partially-healthy-test
+ (testing "compatible but empty with a ready embedder -> partial health (50): built, but serving nothing"
+ (is (=? {:health 50, :message #"Index empty"}
+ (check! (with-index-status :empty)))))
+ (testing "an empty index with an unavailable embedder is degraded"
+ (is (=? {:health 0, :message #"Embedding service unreachable: boom"}
+ (check! (with-index-status :empty) :reachable {:reachable? false, :error "boom"})))))
+
+(deftest untrusted-circuit-is-degraded-test
+ (testing "a non-closed breaker (open or half-open) degrades but still probes (so a recovered-but-idle
+ embedder is detectable), and reads the same in both states so open<->half-open can't flap it"
+ (let [probed? (atom false)]
+ (mt/with-dynamic-fn-redefs
+ [entity-retrieval.core/retrieval-status (constantly ready-status)
+ semantic.embedding/embedder-circuit-untrusted? (constantly true)
+ embedding-health/embedding-service-reachable? (fn [] (reset! probed? true) {:reachable? true})
+ embedding-health/request-circuit-recovery! (constantly nil)]
+ (is (=? {:health 0, :message #".*circuit open \(probe reachable.*"}
+ (entity-retrieval.health/nlq-retrieval-health-check)))
+ (is (true? @probed?) "a non-closed breaker still probes so recovery is detectable"))))
+ (testing "an untrusted breaker with an unreachable probe reads the same as unreachable with a closed one --
+ state-independent wording, so a flapping breaker's re-persisted rows dedup instead of flooding"
+ (is (=? {:health 0, :message #"Embedding service unreachable: boom.*"}
+ (check! ready-status :circuit-untrusted? true :reachable {:reachable? false, :error "boom"})))))
+
+(deftest unreachable-embedder-is-degraded-test
+ (testing "ready index but unreachable embedder -> degraded, names the error"
+ (is (=? {:health 0, :message #".*Embedding service unreachable: boom.*"}
+ (check! ready-status :reachable {:reachable? false, :error "boom"})))))
+
+(deftest healthy-test
+ (testing "ready index + reachable embedder -> healthy 100"
+ (is (=? {:health 100 :message #"Healthy"}
+ (check! ready-status)))))
+
+(deftest nlq-metrics-omitted-when-unavailable-test
+ (testing "coverage/garbage/staleness skip (nil, so omitted) when a dependency is unmet"
+ (mt/with-dynamic-fn-redefs
+ [entity-retrieval.core/retrieval-status
+ (constantly {:dependencies {:store true, :embedder true, :licenses false}})]
+ (is (nil? (#'entity-retrieval.health/nlq-coverage)))
+ (is (nil? (#'entity-retrieval.health/nlq-garbage)))
+ (is (nil? (#'entity-retrieval.health/nlq-staleness))))))
+
+(deftest nlq-staleness-reports-missing-reconciled-at-test
+ (testing "a grant-only role cannot upgrade a legacy metadata table, so the missing freshness signal is
+ reported as degraded instead of silently omitted"
+ (mt/with-dynamic-fn-redefs
+ [entity-retrieval.health/library-datasource (constantly ::ds)
+ jdbc/execute-one! (fn [& _] (throw (java.sql.SQLException. "column \"reconciled_at\" does not exist" "42703")))]
+ (is (=? {:health 0, :message "Index reconcile freshness unavailable: metadata column missing."}
+ (#'entity-retrieval.health/nlq-staleness)))))
+ (testing "any other SQL error (connectivity, permissions) propagates rather than hiding as N/A"
+ (mt/with-dynamic-fn-redefs
+ [entity-retrieval.health/library-datasource (constantly ::ds)
+ jdbc/execute-one! (fn [& _] (throw (java.sql.SQLException. "connection refused" "08006")))]
+ (is (thrown? java.sql.SQLException (#'entity-retrieval.health/nlq-staleness))))))
+
+(deftest nlq-staleness-is-degraded-before-first-reconcile-test
+ (mt/with-dynamic-fn-redefs
+ [entity-retrieval.health/library-datasource (constantly ::ds)
+ jdbc/execute-one! (constantly {:age nil})]
+ (is (=? {:health 0, :message "No full index reconcile has completed."}
+ (#'entity-retrieval.health/nlq-staleness)))))
+
+(deftest nlq-staleness-rejects-negative-database-age-test
+ (mt/with-dynamic-fn-redefs
+ [entity-retrieval.health/library-datasource (constantly ::ds)
+ jdbc/execute-one! (constantly {:age -1})]
+ (is (=? {:health 0
+ :message "Index reconcile freshness unavailable: database clock precedes the last reconcile."}
+ (#'entity-retrieval.health/nlq-staleness)))))
+
+(deftest metric-datasource-propagates-interruption-test
+ (mt/with-dynamic-fn-redefs
+ [entity-retrieval.core/retrieval-status (constantly {:index {:status :compatible}})
+ semantic.datasource/ensure-initialized-data-source! #(throw (InterruptedException.))]
+ (is (thrown? InterruptedException (#'entity-retrieval.health/library-datasource*)))))
diff --git a/enterprise/backend/test/metabase_enterprise/entity_retrieval/index_table_test.clj b/enterprise/backend/test/metabase_enterprise/entity_retrieval/index_table_test.clj
new file mode 100644
index 00000000000..e16bc4eaa4d
--- /dev/null
+++ b/enterprise/backend/test/metabase_enterprise/entity_retrieval/index_table_test.clj
@@ -0,0 +1,56 @@
+(ns metabase-enterprise.entity-retrieval.index-table-test
+ (:require
+ [clojure.test :refer :all]
+ [metabase-enterprise.entity-retrieval.index-table :as index-table]
+ [metabase-enterprise.entity-retrieval.reconcile :as reconcile]
+ [metabase.test :as mt]
+ [metabase.util.log.capture :as log.capture]
+ [next.jdbc :as jdbc]))
+
+(deftest legacy-meta-table-without-ownership-test
+ (testing "a grant-only role neither attempts ALTER nor repeatedly warns"
+ (log.capture/with-log-messages-for-level [messages [metabase-enterprise.entity-retrieval.index-table :warn]]
+ (let [executed (atom [])]
+ (with-redefs-fn {#'index-table/meta-column-exists? (constantly false)
+ #'index-table/can-alter-meta-table? (constantly false)
+ #'index-table/warned-about-missing-reconciled-at (atom #{})
+ #'jdbc/execute! (fn [& args] (swap! executed conj args))}
+ #(dotimes [_ 2]
+ (is (false? (#'index-table/ensure-reconciled-at-column! ::tx)))))
+ (is (empty? @executed))
+ (is (= 1 (count (messages)))))))
+ (testing "freshness stamping is a no-op while the optional column is unavailable"
+ (with-redefs-fn {#'index-table/meta-column-exists? (constantly false)
+ #'jdbc/execute! (fn [& _] (throw (AssertionError. "unexpected write")))}
+ #(is (nil? (index-table/touch-reconciled-at! ::tx ::reconciled-at)))))
+ (testing "freshness stamping persists the caller's pre-read watermark"
+ (let [statement (atom nil)]
+ (with-redefs-fn {#'index-table/meta-column-exists? (constantly true)
+ #'jdbc/execute! (fn [_ sql] (reset! statement sql))}
+ #(index-table/touch-reconciled-at! ::tx ::reconciled-at))
+ (is (= ::reconciled-at (second @statement))))))
+
+(deftest alter-meta-table-requires-usable-owner-role-test
+ (let [query (atom nil)]
+ (with-redefs-fn {#'jdbc/execute-one! (fn [_ [sql & _] _]
+ (reset! query sql)
+ {:owns_table false})}
+ #(is (false? (#'index-table/can-alter-meta-table? ::tx))))
+ (is (re-find #"pg_has_role\(c\.relowner, 'USAGE'\)" @query)
+ "a NOINHERIT member cannot use the owner role's ALTER privilege without SET ROLE")))
+
+(deftest reconcile-watermark-precedes-appdb-read-test
+ (let [events (atom [])]
+ (mt/with-dynamic-fn-redefs [reconcile/capture-reconcile-watermark (fn [_]
+ (swap! events conj :clock)
+ ::reconciled-at)
+ reconcile/desired-docs (fn []
+ (swap! events conj :appdb-read)
+ [])
+ reconcile/stored-docs (constantly {})
+ reconcile/delete-rows! (fn [& _])
+ reconcile/index-size (constantly {:documents 0 :entities 0})
+ index-table/touch-reconciled-at! (fn [_ reconciled-at]
+ (swap! events conj [:watermark reconciled-at]))]
+ (#'reconcile/reconcile-against-appdb! ::conn ::embedding-model))
+ (is (= [:clock :appdb-read [:watermark ::reconciled-at]] @events))))
diff --git a/enterprise/backend/test/metabase_enterprise/entity_retrieval/reconcile_test.clj b/enterprise/backend/test/metabase_enterprise/entity_retrieval/reconcile_test.clj
index fbc949501b0..b61dbebc098 100644
--- a/enterprise/backend/test/metabase_enterprise/entity_retrieval/reconcile_test.clj
+++ b/enterprise/backend/test/metabase_enterprise/entity_retrieval/reconcile_test.clj
@@ -105,6 +105,11 @@
(filter #(and (= entity-type (:entity_type %)) (= entity-local-id (:entity_local_id %)))
(index-rows ds)))
+(defn- reconciled-at! [ds]
+ (:reconciled_at (jdbc/execute-one! ds
+ [(format "SELECT reconciled_at FROM \"%s\" WHERE id = 1" index-table/*meta-table*)]
+ {:builder-fn jdbc.rs/as-unqualified-lower-maps})))
+
(deftest ^:sequential reconcile-lifecycle-test
;; :library lets us publish a Table into a library-data collection; the reconcile enumerates the
;; library tree via collections/library-collection + descendant-ids, so we build a real library root.
@@ -193,11 +198,14 @@
:name "orders" :display_name "Orders"}]
(testing "populate the index under the original model"
(reconcile/reconcile! ds (constantly model))
- (is (seq (docs-for ds "table" table-id))))
+ (is (seq (docs-for ds "table" table-id)))
+ (is (some? (reconciled-at! ds)) "a converged reconcile stamps reconciled_at"))
(testing "a model-identity change drops the vectors table and the next run re-embeds everything"
(let [new-model (assoc model :model-name "model-v2")]
(is (= :rebuilt (index-table/ensure-tables! ds new-model)))
(is (= [] (index-rows ds)))
+ (is (nil? (reconciled-at! ds))
+ "the rebuild clears reconciled_at, so NLQ staleness can't read the empty index as fresh")
(reconcile/reconcile! ds (constantly new-model))
(is (seq (docs-for ds "table" table-id)))
(testing "a schema-version mismatch alone also triggers the rebuild"
diff --git a/test/metabase/health_inspector/core_test.clj b/test/metabase/health_inspector/core_test.clj
index fe2fd0208da..08ccbe06bb2 100644
--- a/test/metabase/health_inspector/core_test.clj
+++ b/test/metabase/health_inspector/core_test.clj
@@ -1,6 +1,7 @@
(ns metabase.health-inspector.core-test
(:require
[clojure.test :refer :all]
+ [java-time.api :as t]
[metabase.health-inspector.core :as hi]
[metabase.lib.core :as lib]
[metabase.test :as mt]
@@ -49,6 +50,21 @@
(is (= ["test check" "test check"] (map :message (runs "test-check"))))
(is (= [100 100] (map :health (runs "validate-queries"))))))
+(deftest save-report-prunes-expired-runs-test
+ (t2/delete! :health_inspector_runs)
+ (let [now (t/offset-date-time)]
+ (doseq [[check-name run-at] [["expired" (t/minus now (t/days 31))]
+ ["recent" (t/minus now (t/days 29))]]]
+ (t2/insert! :health_inspector_runs
+ {:check_name check-name
+ :health 100
+ :message check-name
+ :run_at run-at})))
+ (with-redefs [hi/checks (atom {})]
+ (hi/save-report))
+ (is (= #{"recent"} (set (map :check_name (hi/list-runs 32))))
+ "runs older than 30 days are deleted while recent runs are preserved"))
+
;; The test-only checks below are registered into a fresh, test-scoped `checks` registry via with-redefs
;; rather than the global one, so a nil/throwing/probe check can't leak into other health-inspector tests in
;; the same JVM and make them order-dependent.